Joe/rust core binding poc

.agents/server-map.md

@@ -52,6 +52,50 @@ Cite file and class or function names in your answers. Do not cite line numbers,
 
 `src/DataTypes/DataTypeDecimalBase.{cpp,h}`, `src/DataTypes/DataTypesDecimal.h`, `src/DataTypes/Serializations/SerializationDecimal.{cpp,h}`. Stored as fixed-width two's-complement integer (32/64/128/256-bit) with precision and scale carried in the type string.
 
+### QBit
+
+- **Confirmed at v26.3.9.8-lts:** The type implementation is
+  `src/DataTypes/DataTypeQBit.{cpp,h}` in `DataTypeQBit`.
+  `DataTypeQBit::createColumn` creates a `ColumnQBit` backed by a tuple containing
+  16, 32, or 64 `FixedString(ceil(dimension / 8))` columns.
+  `DataTypeQBit::doGetSerialization` returns `SerializationQBit`. The column
+  wrapper and bit-plane storage accessors are in `src/Columns/ColumnQBit.{cpp,h}`.
+- **Confirmed at v26.3.9.8-lts:** Binary and Native serialization are implemented
+  by `SerializationQBit` in
+  `src/DataTypes/Serializations/SerializationQBit.{cpp,h}`. Row-wise binary uses
+  `serializeBinary`, `deserializeBinary`, `serializeFloatsFromQBit`,
+  `deserializeFloatsToQBit`, `transposeBits`, and `untransposeBitPlane`. Its wire
+  value is a VarUInt dimension followed by that many BFloat16, Float32, or Float64
+  values.
+- **Confirmed at v26.3.9.8-lts:** Native uses
+  `SerializationQBit::serializeBinaryBulkWithMultipleStreams` and
+  `deserializeBinaryBulkWithMultipleStreams`, reached through
+  `NativeWriter::writeData` and `NativeReader::readData` in
+  `src/Formats/NativeWriter.cpp` and `src/Formats/NativeReader.cpp`. These methods
+  delegate to the nested `SerializationTuple` implementation in
+  `src/DataTypes/Serializations/SerializationTuple.cpp`; each bit plane is emitted
+  as raw fixed-width bytes by `SerializationFixedString::serializeBinaryBulk` in
+  `src/DataTypes/Serializations/SerializationFixedString.cpp`. Native therefore
+  carries bit-transposed planes, not row-wise float arrays.
+- **Confirmed at v26.3.9.8-lts:** The binary type-schema encoding used by Dynamic
+  and related self-describing serializations is in
+  `src/DataTypes/DataTypesBinaryEncoding.{cpp,h}`. QBit uses type tag `0x36`,
+  followed by the encoded element type and a VarUInt dimension.
+- **Confirmed at v26.3.9.8-lts:** Focused tests are
+  `src/DataTypes/Serializations/tests/gtest_qbit_serialization.cpp`
+  (`QBitSerialization.FieldBinarySerializationFloat32` and
+  `QBitSerialization.RejectInvalidElementType`) and
+  `tests/queries/0_stateless/03371_qbit_read_write.{sh,reference}`, which exercises
+  text, RowBinary, and Native round trips.
+- **Confirmed at v26.3.9.8-lts:** Additional coverage is in
+  `tests/queries/0_stateless/03363_qbit_create_insert_select.{sql,reference}` for all
+  three element widths and byte-aligned/non-byte-aligned dimensions,
+  `03368_qbit_subcolumns.{sql,reference}` for bit-plane subcolumns,
+  `03372_qbit_mergetree_1.{sql,reference}` and
+  `03372_qbit_mergetree_2.{sql,reference}` for stored data,
+  `03373_qbit_dynamic.{sql,reference}` for Dynamic, and
+  `03374_qbit_nullable.{sql,reference}` for Nullable.
+
 ### Enum8 / Enum16
 
 `src/DataTypes/DataTypeEnum.{cpp,h}`, `src/DataTypes/EnumValues.{cpp,h}`, `src/DataTypes/Serializations/SerializationEnum.{cpp,h}`. Name-to-value map lives in the type string. Wire format is the underlying Int8/Int16.

.github/workflows/binding_ci.yml

@@ -0,0 +1,91 @@
+name: 'Rust Binding CI'
+
+on:
+  pull_request:
+    branches:
+      - main
+    paths:
+      - 'rust/**'
+      - 'clickhouse_connect/driver/rustcodec.py'
+      - 'clickhouse_connect/driver/rustnumpy.py'
+      - 'tests/integration_tests/test_rust_codec.py'
+      - '.github/workflows/binding_ci.yml'
+  push:
+    branches:
+      - main
+    paths:
+      - 'rust/**'
+      - 'clickhouse_connect/driver/rustcodec.py'
+      - 'clickhouse_connect/driver/rustnumpy.py'
+      - 'tests/integration_tests/test_rust_codec.py'
+      - '.github/workflows/binding_ci.yml'
+  workflow_dispatch:
+
+jobs:
+  rust-checks:
+    runs-on: ubuntu-latest
+    name: Cargo fmt + clippy
+    steps:
+      - uses: actions/checkout@v6
+      # pinned so new stable clippy lints don't break unrelated PRs, bump deliberately
+      - uses: dtolnay/rust-toolchain@1.97.0
+        with:
+          components: rustfmt, clippy
+      - uses: Swatinem/rust-cache@v2
+        with:
+          workspaces: rust
+      - name: Format check
+        working-directory: rust
+        run: cargo fmt --all -- --check
+      - name: Clippy (deny warnings)
+        working-directory: rust
+        run: cargo clippy --all-targets -- -D warnings
+
+  binding-tests:
+    runs-on: ubuntu-latest
+    name: Build binding and run codec tests
+    needs: rust-checks
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v6
+      - name: Start ClickHouse (latest) in Docker
+        env:
+          CLICKHOUSE_CONNECT_TEST_CH_VERSION: latest
+          COMPOSE_PROJECT_NAME: clickhouse-connect-binding-ci
+        run: docker compose -f docker-compose.yml up -d --wait clickhouse
+      - name: Set up Python 3.12
+        uses: actions/setup-python@v6
+        with:
+          python-version: '3.12'
+      - uses: dtolnay/rust-toolchain@stable
+      - uses: Swatinem/rust-cache@v2
+        with:
+          workspaces: rust
+      - name: Install Test Dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install -r tests/test_requirements.txt
+          pip install maturin
+      - name: Build cython extensions
+        run: python setup.py build_ext --inplace
+      - name: "Add distribution info"  #  This lets SQLAlchemy find entry points
+        run: python setup.py develop
+      - name: Build and install the clickhouse-connect-core wheel
+        run: |
+          maturin build --release -m rust/ch-core-py/Cargo.toml -o core-wheelhouse
+          pip install core-wheelhouse/*.whl
+      - name: Extension smoke check
+        run: python -c "import _ch_core; print(_ch_core.__version__, _ch_core.BINDING_API_VERSION)"
+      - name: Run binding tests
+        run: pytest rust/ch-core-py/tests -n 4
+      - name: Run unit tests
+        run: pytest tests/unit_tests -n 4
+      - name: Run rust codec integration tests
+        env:
+          CLICKHOUSE_CONNECT_TEST_DOCKER: 'False'
+        run: pytest tests/integration_tests/test_rust_codec.py -n 4
+      - name: Stop ClickHouse
+        if: ${{ always() && hashFiles('docker-compose.yml') != '' }}
+        env:
+          COMPOSE_PROJECT_NAME: clickhouse-connect-binding-ci
+        run: docker compose -f docker-compose.yml down --volumes --remove-orphans

.github/workflows/on_push.yml

@@ -87,8 +87,8 @@ jobs:
           "$RUNNER_TEMP/consumer/bin/mypy" --strict --follow-imports=silent consumer_smoke.py
       - name: Public type completeness ratchet
         # SQLAlchemy must be installed while verifytypes scans the public cc_sqlalchemy annotations.
-        # The pre-feature baseline is 1120 with that optional surface resolved.
-        run: python scripts/check_public_types.py clickhouse_connect --max-untyped 1120 --python "$RUNNER_TEMP/consumer/bin/python"
+        # The merged baseline is 1118 with that optional surface resolved.
+        run: python scripts/check_public_types.py clickhouse_connect --max-untyped 1118 --python "$RUNNER_TEMP/consumer/bin/python"
       - name: SQLAlchemy Select typing smoke test
         run: |
           cp tests/type_check/sqlalchemy_select_smoke.py "$RUNNER_TEMP"

.github/workflows/publish_core.yml

@@ -28,17 +28,17 @@ jobs:
       fail-fast: false
       matrix:
         include:
-          - { runner: ubuntu-latest, arch: x86_64, libc: manylinux, manylinux: auto }
+          - { runner: ubuntu-latest, arch: x86_64, libc: manylinux, manylinux: 2_28 }
           - { runner: ubuntu-latest, arch: x86_64, libc: musllinux, manylinux: musllinux_1_2 }
-          - { runner: ubuntu-24.04-arm, arch: aarch64, libc: manylinux, manylinux: auto }
+          - { runner: ubuntu-24.04-arm, arch: aarch64, libc: manylinux, manylinux: 2_28 }
           - { runner: ubuntu-24.04-arm, arch: aarch64, libc: musllinux, manylinux: musllinux_1_2 }
     steps:
       - uses: actions/checkout@v6
       - name: Build wheels
         uses: PyO3/maturin-action@v1
         with:
           manylinux: ${{ matrix.manylinux }}
-          args: --release -m ${{ env.MATURIN_MANIFEST }} -o wheelhouse --find-interpreter
+          args: --release -m ${{ env.MATURIN_MANIFEST }} -o wheelhouse -i 3.10 3.11 3.12 3.13 3.14
       - uses: actions/upload-artifact@v7
         with:
           name: core-${{ matrix.libc }}-${{ matrix.arch }}
@@ -127,7 +127,7 @@ jobs:
         uses: PyO3/maturin-action@v1
         with:
           target: aarch64-pc-windows-msvc
-          args: --release -m ${{ env.MATURIN_MANIFEST }} -o wheelhouse
+          args: --release -m ${{ env.MATURIN_MANIFEST }} -o wheelhouse -i python3.10
       - uses: actions/upload-artifact@v7
         with:
           name: core-windows-arm64-3.10

.gitignore

@@ -32,3 +32,10 @@ test.env
 # Local ClickHouse server source checkout for agent reference (see AGENTS.md)
 /.server-src/
 .server-ref
+
+# Rust build artifacts
+rust/target/
+*.so
+*.dylib
+/.cargo/config.toml
+.claude

CHANGELOG.md

@@ -2,6 +2,16 @@
 
 ## UNRELEASED
 
+## 1.8.0rc2, 2026-08-20
+
+Follow-up release candidate to 1.8.0rc1, rebased on 1.7.2 so all bug fixes from that stable release are included. The optional Rust codec itself is unchanged.
+
+## 1.8.0rc1, 2026-08-12
+
+### Improvements
+
+- Added an experimental `native_codec` client option that selects the codec for FORMAT Native query decode and insert encode. `python` is the default and uses the existing codec. `rust` prefers the compiled Rust codec and falls back to the Python codec for unsupported options and types, while `rust_strict` raises instead of falling back. Results and dtypes match the Python codec, and the Arrow methods are unaffected. The compiled codec ships as the separate clickhouse-connect-core wheel, installed with `pip install clickhouse-connect[rust]`. See the rust-codec documentation page for details. This is early access for benchmarking and is not yet a supported path.
+
 ## 1.7.2, 2026-08-19
 
 ### Bug Fixes
@@ -63,6 +73,8 @@
 - `AsyncClient` initialization no longer overwrites user-supplied session settings with generated defaults. A client created with `settings={'date_time_input_format': 'basic'}` previously had that value replaced by the generated `best_effort` default. User settings now always win, matching the sync client.
 - An `AsyncClient` created with both client certificates and an access token now sends the mutual TLS authentication headers and the `Authorization: Bearer` header together, matching the sync client. The certificates previously suppressed the token at construction, while the `token_provider` option re-added its token right after initialization, so the two async token paths disagreed with each other. The server resolves the credential precedence.
 - Dict-valued settings such as `additional_table_filters` no longer crash with `DB::Exception: Cannot parse quoted string` when passed through `query()`'s `settings` parameter. The value was rendered with Python's own `str()`/`repr()` of the dict, which mixes single and double quotes and is not valid ClickHouse map-literal syntax; it is now rendered as a properly single-quoted, escaped ClickHouse map literal. Closes [#501](https://github.com/ClickHouse/clickhouse-connect/issues/501).
+- Explicit NaN and infinity values in nullable `BFloat16` row inserts are now stored instead of being written as 0.
+- Inserting an `Array(Dynamic)` column no longer raises `ZeroDivisionError` when a sampled row holds an empty array. The insert block size estimate now treats an empty sample as minimal instead of dividing by its length.
 
 ### Improvements
 - Async clients now emit URL query parameters in the same order as the sync client on every request. The parameter names and values are unchanged, so this is only visible to systems that match or sign the exact request URL.

DISTRIBUTION_PLAN.md

@@ -0,0 +1,71 @@
+# Rust Codec Distribution Plan
+
+How the Rust native codec is packaged, versioned, released, and supported across ch-core-rs, the Python binding, and clickhouse-connect.
+
+## Decisions
+
+- Users opt in with a single switch on the client. The Rust path uses the new, correct semantics. Known behavior differences from the Python codec are enumerated and documented. Features the Rust path does not implement fall back to the Python codec.
+- ch-core-rs is open source on GitHub. It is not published to crates.io. Consumers depend on it as a git dependency pinned to a release tag. The repo is currently internal. Making it public is in progress and gates the first PyPI release.
+- The core repo stays pure Rust with no language-specific dependencies. Each language client owns its binding in its own repo.
+- The Python binding ships as a separate PyPI wheel, installed via the `rust` extra. It is not compiled into the clickhouse-connect wheel.
+- Long term: deprecation warnings on the Python codec late in 1.x, Rust becomes the only codec in 2.0.
+
+## Artifacts and repos
+
+| Piece | Lives in | Ships as | Releases when |
+|---|---|---|---|
+| ch-core-rs | own public repo, pure Rust | git tags only | core logic changes |
+| ch-core-py binding and the rustcodec.py seam | clickhouse-connect repo | `clickhouse-connect-core` wheel on PyPI, module name `_ch_core` | core repin or binding change |
+| clickhouse-connect | clickhouse-connect repo | pure/cython package with a `rust` extra | driver features, seam or floor changes |
+
+One repo can publish two PyPI artifacts. Living in the connect repo does not mean shipping in the connect wheel.
+
+## The two seams
+
+**Rust seam (ch-core-rs into the binding).** Compile time only. The core is statically linked into the extension when the wheel builds. Users never resolve it and no runtime skew is possible. Pin by git tag in ch-core-py's Cargo.toml and commit Cargo.lock for reproducible builds.
+
+**Python seam (`clickhouse-connect-core` wheel and clickhouse-connect).** The only seam users see. clickhouse-connect pins a compatible range, for example `clickhouse-connect-core>=1.2,<1.3`. The pin floor encodes the oldest core wheel this driver knows how to drive. Bump the floor only when connect starts using a new binding capability.
+
+## Packaging
+
+- PyPI name `clickhouse-connect-core`, module name `_ch_core`. The name should be recognizable in a user's pip list.
+- Install: `pip install clickhouse-connect[rust]`.
+- Build with maturin, one wheel per platform per Python version (cp310 through cp314), matching the main package's matrix.
+- Not abi3: the binding's hot paths use non-limited C API (`PyTuple_SET_ITEM`, `PyList_SET_ITEM`, presized dict construction) on purpose. Moving to the limited API would tax exactly the paths the codec exists to accelerate. Revisit only if the wheel matrix becomes a real maintenance cost.
+- If the switch is enabled without the wheel installed, raise a clear error naming the install command.
+- While ch-core-rs remains non-public, wheel builds need repo access from CI and the sdist cannot build for outside users. Making the repo public precedes the first PyPI release.
+
+## Runtime handshake
+
+- `_ch_core` exports a binding API version constant.
+- rustcodec.py checks it at import. Too old raises a legible message naming the required `clickhouse-connect-core` version. Never a crash or silent misbehavior.
+- The client's diagnostic output includes the `_ch_core` version alongside the driver version so bug reports arrive with both.
+
+## Release workflows
+
+**Core bugfix.** Fix in ch-core-rs, tag a patch release. In the connect repo bump the git tag in ch-core-py, publish a `clickhouse-connect-core` patch wheel. No clickhouse-connect release. Users run `pip install -U clickhouse-connect[rust]`.
+
+**Transparent core improvement** (faster decode, internal wins). Same as a bugfix but a minor bump. Users get it for free with a wheel upgrade.
+
+**User-facing core feature** (new setting, new type, new capability). Core minor bump, binding exposes it, `clickhouse-connect-core` minor release. Then a clickhouse-connect release that uses it and raises the pin floor. The connect release is the feature's public API.
+
+**Driver-only change.** Normal clickhouse-connect release. The wheel is untouched.
+
+## Issue handling
+
+- Users file everything on clickhouse-connect. The core repo tracker is for maintainers and binding authors.
+- If a root cause lands in ch-core-rs, fix it there but keep and close the loop in the original clickhouse-connect issue, noting the `clickhouse-connect-core` version that carries the fix.
+
+## Core repo contract
+
+- Semver on tags. No breaking changes on patch or minor.
+- A changelog maintained per release. Downstream bindings in multiple languages will depend on reading it.
+- No PyO3, napi, or other language-specific dependencies in the core crate. Language artifacts are built by the binding repos.
+- CI tests the crate on supported platforms. It builds no wheels or language artifacts.
+
+## Rollout
+
+1. Next minor: ship the opt-in switch, the `rust` extra, the known-differences documentation, and the fallback behavior.
+2. During 1.x: promote the Rust path as it proves out. Wheel-only releases carry core fixes and wins to opted-in users.
+3. Late 1.x: DeprecationWarning on the Python codec path.
+4. 2.0: Rust codec becomes the only codec. The differences list becomes the documented behavior.

clickhouse_connect/_version.py

@@ -1 +1 @@
-version = "1.7.2"
+version = "1.8.0rc2"

clickhouse_connect/cc_sqlalchemy/datatypes/sqltypes.py

@@ -259,6 +259,10 @@ class MultiLineString(ChSqlaType, UserDefinedType):  # type: ignore[misc]
     python_type = list
 
 
+class Geometry(ChSqlaType, UserDefinedType):  # type: ignore[misc]
+    python_type = object
+
+
 class Date(ChSqlaType, SqlaDate):  # type: ignore[misc]
     pass
 

clickhouse_connect/common.py

@@ -1,4 +1,6 @@
 import getpass
+import logging
+import os
 import sys
 from collections.abc import Sequence
 from dataclasses import dataclass
@@ -7,6 +9,10 @@
 from clickhouse_connect._version import version as _version_string
 from clickhouse_connect.driver.exceptions import ProgrammingError
 
+logger: logging.Logger = logging.getLogger(__name__)
+
+_NATIVE_CODEC_OPTIONS = ("python", "rust", "rust_strict")
+
 
 def version() -> str:
     return _version_string
@@ -68,6 +74,17 @@ def _init_common(name: str, options: Sequence[Any], default: Any) -> None:
     _common_settings[name] = CommonSetting(name, options, default)
 
 
+def _native_codec_env_default() -> str:
+    raw = os.environ.get("CLICKHOUSE_CONNECT_NATIVE_CODEC")
+    if raw is None:
+        return "python"
+    value = raw.strip().lower()
+    if value in _NATIVE_CODEC_OPTIONS:
+        return value
+    logger.warning("Ignoring invalid CLICKHOUSE_CONNECT_NATIVE_CODEC=%r; using 'python'", raw)
+    return "python"
+
+
 _init_common("autogenerate_session_id", (True, False), True)
 _init_common("autogenerate_query_id", (True, False), True)
 _init_common("dict_parameter_format", ("json", "map"), "json")
@@ -79,6 +96,10 @@ def _init_common(name: str, options: Sequence[Any], default: Any) -> None:
 _init_common("readonly", (0, 1), 0)  # Deprecated no-op retained for 1.x compatibility
 _init_common("send_os_user", (True, False), True)
 
+# Selects the codec for client-managed FORMAT Native query and insert paths. Seeded by
+# CLICKHOUSE_CONNECT_NATIVE_CODEC; overridable per client via the native_codec kwarg.
+_init_common("native_codec", _NATIVE_CODEC_OPTIONS, _native_codec_env_default())
+
 # Include integration tags (library name/version) in the User-Agent, e.g.:
 # pandas/2.2.5; polars/0.20.x; sqlalchemy/2.0.x. These tags are only included
 # when using relevant API methods.

clickhouse_connect/datatypes/base.py

@@ -123,6 +123,8 @@ def data_size(self, sample: Collection) -> int:
     def _data_size(self, sample: Collection) -> int:
         if self.byte_size:
             return self.byte_size
+        if len(sample) == 0:
+            return 1
         total = 0
         for x in sample:
             total += len(str(x))