From e5bbd5596d9d0560f48758515bd672727018e87e Mon Sep 17 00:00:00 2001 From: Francesc Alted Date: Thu, 16 Jul 2026 14:32:53 +0200 Subject: [PATCH 01/20] Add blosc2.utf8(): variable-length string columns as offsets + bytes New first-class column type for high-cardinality/free-text strings: blosc2.utf8(nullable=..., null_value=...) stores each column as two companion NDArrays (int64 row offsets plus a uint8 UTF-8 byte blob), so a row costs exactly its encoded byte length. Bulk reads materialize as numpy StringDType arrays (requires NumPy >= 2.0; a clear error is raised otherwise). This first slice covers storage and roundtrip: - Utf8Spec / utf8() in schema.py, kind "utf8" in the schema compiler. - Utf8Array adapter (new utf8_array.py) with append/extend/flush, pending-aware reads, cluster-based sparse gathers, and O(n - i) in-place cell rewrite (offsets shift on length change). - Storage dispatch in all four TableStorage backends; the byte blob lives at the column key plus a ".utf8" suffix, unreachable from any user column name. delete/rename move both leaves. - Sentinel-based nulls consistent with other scalar columns (policy string_value by default), including is_null/null_count/fillna. - Persistence roundtrips: .b2z/.b2d save/open/load, mmap open, to_cframe/ctable_from_cframe, TreeStore bundling, copy/take/compact. - Comparisons, where(), groupby keys, sorting, and Arrow export raise clear NotImplementedError/TypeError messages for now. vlstring()/string() behavior is unchanged. Co-Authored-By: Claude Fable 5 --- src/blosc2/__init__.py | 2 + src/blosc2/ctable.py | 85 ++++++- src/blosc2/ctable_storage.py | 107 +++++++-- src/blosc2/schema.py | 79 +++++++ src/blosc2/schema_compiler.py | 4 +- src/blosc2/utf8_array.py | 397 +++++++++++++++++++++++++++++++ tests/ctable/test_utf8.py | 429 ++++++++++++++++++++++++++++++++++ 7 files changed, 1077 insertions(+), 26 deletions(-) create mode 100644 src/blosc2/utf8_array.py create mode 100644 tests/ctable/test_utf8.py diff --git a/src/blosc2/__init__.py b/src/blosc2/__init__.py index bb4332d31..3050887bb 100644 --- a/src/blosc2/__init__.py +++ b/src/blosc2/__init__.py @@ -774,6 +774,7 @@ def _raise(exc): uint16, uint32, uint64, + utf8, vlbytes, vlstring, ) @@ -829,6 +830,7 @@ def _raise(exc): "uint16", "uint32", "uint64", + "utf8", "vlbytes", "vlstring", # Grouped reductions diff --git a/src/blosc2/ctable.py b/src/blosc2/ctable.py index 57b8ecf1a..7a1e731cb 100644 --- a/src/blosc2/ctable.py +++ b/src/blosc2/ctable.py @@ -44,6 +44,7 @@ from blosc2.info import InfoReporter, format_nbytes_human, format_nbytes_info from blosc2.list_array import ListArray, coerce_list_cell from blosc2.scalar_array import _ScalarVarLenArray +from blosc2.utf8_array import Utf8Array if TYPE_CHECKING: from blosc2.dictionary_column import DictionaryColumn @@ -54,6 +55,7 @@ ObjectSpec, SchemaSpec, StructSpec, + Utf8Spec, VLBytesSpec, VLStringSpec, complex64, @@ -1066,7 +1068,15 @@ def is_list(self) -> bool: def is_varlen_scalar(self) -> bool: """True if this column holds variable-length scalar strings or bytes.""" col = self._table._schema.columns_by_name.get(self._col_name) - return col is not None and isinstance(col.spec, (VLStringSpec, VLBytesSpec, StructSpec, ObjectSpec)) + return col is not None and isinstance( + col.spec, (VLStringSpec, VLBytesSpec, StructSpec, ObjectSpec, Utf8Spec) + ) + + @property + def is_utf8(self) -> bool: + """True if this column stores variable-length UTF-8 strings (offsets + bytes).""" + col = self._table._schema.columns_by_name.get(self._col_name) + return col is not None and isinstance(col.spec, Utf8Spec) @property def is_dictionary(self) -> bool: @@ -1206,6 +1216,8 @@ def _values_from_key(self, key, *, check_stale: bool = True): # noqa: C901 # slice semantics (including negative steps) follow NumPy. selected_pos = real_pos[key] if selected_pos.size == 0: + if self.is_utf8: + return self._raw_col[selected_pos] if self.is_list or self.is_varlen_scalar or self.is_dictionary: return [] if self.is_ndarray: @@ -1715,6 +1727,11 @@ def row_transformer(self) -> RowTransformer: def _ensure_queryable(self) -> None: self._ensure_not_stale() + if self.is_utf8: + raise NotImplementedError( + f"Column {self._col_name!r} is a variable-length utf8 column; " + "vectorized comparisons on utf8 columns are not supported yet." + ) if self.is_varlen_scalar: raise NotImplementedError( f"Column {self._col_name!r} is a vlstring/vlbytes column; " @@ -2086,7 +2103,7 @@ def iter_chunks(self, size: int = 65536): return if self.is_list: raise TypeError("Column.iter_chunks() is not supported for list columns in V1.") - if self.is_varlen_scalar: + if self.is_varlen_scalar and not self.is_utf8: raise TypeError("Column.iter_chunks() is not supported for varlen scalar columns.") valid = self._valid_rows raw = self._raw_col @@ -2110,7 +2127,13 @@ def iter_chunks(self, size: int = 65536): segment = raw[start : start + actual] else: mask = valid[start : start + actual] - segment = raw[start : start + actual][mask] + data_part = raw[start : start + actual] + if len(data_part) < actual: + # Logically-sized storage (utf8) is shorter than the + # capacity-sized validity mask; rows past its end are + # never live, so the extra mask tail is all False. + mask = mask[: len(data_part)] + segment = data_part[mask] if len(segment) == 0: continue @@ -2277,7 +2300,7 @@ def is_null(self) -> np.ndarray: """ if self.is_dictionary: return self._dictionary_eq(None) - if self.is_varlen_scalar: + if self.is_varlen_scalar and not self.is_utf8: return np.array([v is None for v in self], dtype=np.bool_) return self._null_mask_for(self[:]) @@ -2293,7 +2316,7 @@ def null_count(self) -> int: """ if self.is_dictionary: return int(self.is_null().sum()) - if self.is_varlen_scalar: + if self.is_varlen_scalar and not self.is_utf8: return sum(1 for v in self if v is None) if self.null_value is None: return 0 @@ -2306,7 +2329,7 @@ def fillna(self, value): native ``None`` cells) return a list; other columns return a NumPy array. """ - if self.is_dictionary or self.is_varlen_scalar: + if (self.is_dictionary or self.is_varlen_scalar) and not self.is_utf8: return [value if v is None else v for v in self[:]] arr = np.array(self[:], copy=True) if self.null_value is not None: @@ -3862,7 +3885,11 @@ def _is_list_column(col: CompiledColumn) -> bool: @staticmethod def _is_varlen_scalar_column(col: CompiledColumn) -> bool: - return isinstance(col.spec, (VLStringSpec, VLBytesSpec, StructSpec, ObjectSpec)) + return isinstance(col.spec, (VLStringSpec, VLBytesSpec, StructSpec, ObjectSpec, Utf8Spec)) + + @staticmethod + def _is_utf8_column(col: CompiledColumn) -> bool: + return isinstance(col.spec, Utf8Spec) @staticmethod def _is_dictionary_column(col: CompiledColumn) -> bool: @@ -3968,7 +3995,7 @@ def _policy_null_value_for_spec(spec: SchemaSpec, policy: NullPolicy): return policy.float_value if isinstance(spec, b2_bool): return policy.bool_value - if isinstance(spec, string): + if isinstance(spec, (string, Utf8Spec)): return policy.string_value if isinstance(spec, b2_bytes): return policy.bytes_value @@ -4018,7 +4045,7 @@ def _validate_null_value_for_spec(name: str, spec: SchemaSpec, null_value) -> No if null_value != 255: raise ValueError(f"Null sentinel for nullable bool column {name!r} must be 255") return - if isinstance(spec, string): + if isinstance(spec, (string, Utf8Spec)): if not isinstance(null_value, str): raise TypeError(f"Null sentinel for string column {name!r} must be str") return @@ -5320,6 +5347,7 @@ def to_cframe(self) -> bytes: from blosc2.ctable_storage import ( _DICT_SUFFIX, + _UTF8_DATA_SUFFIX, _column_name_to_relpath, ) @@ -5347,6 +5375,9 @@ def to_cframe(self) -> bytes: if self._is_dictionary_column(col): estore[key] = arr.codes estore[key + _DICT_SUFFIX] = arr._dict_store._backend + elif self._is_utf8_column(col): + estore[key] = arr.offsets + estore[key + _UTF8_DATA_SUFFIX] = arr.data elif self._is_varlen_scalar_column(col): estore[key] = arr._backend else: @@ -6441,6 +6472,11 @@ def _resolve_arrow_columns(self, columns, include_computed: bool = True) -> list def _pa_type_from_spec(pa, spec): if isinstance(spec, DictionarySpec): return pa.dictionary(pa.int32(), pa.string(), ordered=spec.ordered) + if isinstance(spec, Utf8Spec): + raise NotImplementedError( + "Arrow export of variable-length utf8 columns is not supported yet; " + "exclude the column via the columns= argument." + ) if isinstance(spec, VLStringSpec): return pa.string() if isinstance(spec, VLBytesSpec): @@ -8757,7 +8793,7 @@ def rename_column(self, old: str, new: str) -> None: self._invalidate_index_catalog_cache() if isinstance(self._storage, FileTableStorage): - self._cols[new] = self._storage.rename_column(old, new) + self._cols[new] = self._rename_stored_column(old, new) else: self._cols[new] = self._cols[old] del self._cols[old] @@ -8790,6 +8826,18 @@ def rename_column(self, old: str, new: str) -> None: if rebuild_kwargs is not None: self.create_index(new, **rebuild_kwargs) + def _rename_stored_column(self, old: str, new: str): + """Rename a stored column's persistent leaves and return the reopened column.""" + old_compiled_col = self._schema.columns_by_name[old] + if hasattr(self._cols[old], "flush"): + self._cols[old].flush() + renamed_col = self._storage.rename_column(old, new) + if self._is_utf8_column(old_compiled_col): + # rename_column returns the bare offsets NDArray; reopen the + # offsets + bytes pair as a proper utf8 column object. + renamed_col = self._storage.open_varlen_scalar_column(new, old_compiled_col.spec) + return renamed_col + # ------------------------------------------------------------------ # Computed / virtual columns # ------------------------------------------------------------------ @@ -10761,7 +10809,9 @@ def compact(self): continue if self._is_varlen_scalar_column(col): compacted = [v[int(pos)] for pos in real_poss[: self._n_rows]] - replacement = _ScalarVarLenArray(col.spec) + replacement = ( + Utf8Array(col.spec) if self._is_utf8_column(col) else _ScalarVarLenArray(col.spec) + ) replacement.extend(compacted) replacement.flush() self._cols[name] = replacement @@ -10828,9 +10878,13 @@ def _normalise_sort_keys( f"Cannot sort by ndarray column {name!r} with per-row shape {col_info.spec.item_shape}. " "Materialize a scalar generated column first, e.g. embedding_norm or embedding_max." ) + cc = self._schema.columns_by_name.get(name) + if cc is not None and self._is_utf8_column(cc): + raise TypeError( + f"Column {name!r} is a variable-length utf8 column; sorting by it is not supported yet." + ) dtype = self._col_dtype(name) if dtype is None: - cc = self._schema.columns_by_name.get(name) if cc is not None and isinstance( cc.spec, (VLStringSpec, VLBytesSpec, StructSpec, ObjectSpec) ): @@ -11898,6 +11952,8 @@ def _dtype_info_label(dtype: np.dtype | None, spec: SchemaSpec | None = None) -> if isinstance(spec, DictionarySpec): ordered_tag = ", ordered" if spec.ordered else "" return f"dictionary[str{ordered_tag}]" + if isinstance(spec, Utf8Spec): + return "utf8" if isinstance(spec, VLStringSpec): return "vlstring" if isinstance(spec, VLBytesSpec): @@ -12399,6 +12455,11 @@ def _guard_scalar_expression(self, expr: str) -> None: f"Column {col.name!r} is a fixed-shape ndarray column. String expressions only " "support scalar columns. Use an element projection or a row-wise reduction first." ) + if self._is_utf8_column(col) and self._expression_references_name(expr, col.name): + raise NotImplementedError( + f"Column {col.name!r} is a variable-length utf8 column; " + "string expressions on utf8 columns are not supported yet." + ) if self._is_varlen_scalar_column(col) and self._expression_references_name(expr, col.name): raise NotImplementedError( f"Column {col.name!r} is a variable-length scalar column (vlstring/vlbytes/struct/object); " diff --git a/src/blosc2/ctable_storage.py b/src/blosc2/ctable_storage.py index 9b68ce0b6..0c967e7ee 100644 --- a/src/blosc2/ctable_storage.py +++ b/src/blosc2/ctable_storage.py @@ -38,6 +38,7 @@ _validate_role_metadata, ) from blosc2.schunk import process_opened_object +from blosc2.utf8_array import Utf8Array, _new_backend_arrays if TYPE_CHECKING: from blosc2.schema import ListSpec @@ -249,6 +250,11 @@ def open_list_column(self, name): raise RuntimeError("In-memory tables have no on-disk representation to open.") def create_varlen_scalar_column(self, name, *, spec, cparams=None, dparams=None): + from blosc2.schema import Utf8Spec + + if isinstance(spec, Utf8Spec): + offsets, data = _new_backend_arrays(cparams, dparams) + return Utf8Array(spec, offsets, data) return _ScalarVarLenArray(spec) def open_varlen_scalar_column(self, name, spec): @@ -398,6 +404,12 @@ def _column_name_to_relpath(name: str) -> str: # Suffix used to store a dictionary column's value store alongside its codes. _DICT_SUFFIX = "_dict" +# Key suffix for the byte-blob companion of a utf8 column (its offsets array +# lives at the plain column key). The literal dot cannot collide with a user +# column: unescaped dots in logical names become path separators and escaped +# dots are percent-encoded, so no column name maps to a key containing ".". +_UTF8_DATA_SUFFIX = ".utf8" + class EmbedStoreTableStorage(TableStorage): """Read-only :class:`CTable` storage backed by an in-memory :class:`blosc2.EmbedStore`. @@ -473,6 +485,12 @@ def open_list_column(self, name: str) -> ListArray: return self._estore[self._col_key(name)] def open_varlen_scalar_column(self, name: str, spec) -> _ScalarVarLenArray: + from blosc2.schema import Utf8Spec + + if isinstance(spec, Utf8Spec): + offsets = self._estore[self._col_key(name)] + data = self._estore[self._col_key(name) + _UTF8_DATA_SUFFIX] + return Utf8Array(spec, offsets, data) backend = self._estore[self._col_key(name)] return _ScalarVarLenArray(spec, backend) @@ -706,11 +724,30 @@ def open_list_column(self, name: str) -> ListArray: return blosc2.open(self._list_col_path(name), mode=self._mode) def create_varlen_scalar_column(self, name, *, spec, cparams=None, dparams=None) -> _ScalarVarLenArray: + from blosc2.schema import Utf8Spec + + if isinstance(spec, Utf8Spec): + offsets, data = _new_backend_arrays(cparams, dparams) + store = self._open_store() + store[self._col_key(name)] = offsets + store[self._col_key(name) + _UTF8_DATA_SUFFIX] = data + return Utf8Array( + spec, + store[self._col_key(name)], + store[self._col_key(name) + _UTF8_DATA_SUFFIX], + ) urlpath = self._list_col_path(name) backend = _make_persistent_backend(spec, urlpath, "w", cparams=cparams, dparams=dparams) return _ScalarVarLenArray(spec, backend) def open_varlen_scalar_column(self, name: str, spec) -> _ScalarVarLenArray: + from blosc2.schema import Utf8Spec + + if isinstance(spec, Utf8Spec): + store = self._open_store() + offsets = store[self._col_key(name)] + data = store[self._col_key(name) + _UTF8_DATA_SUFFIX] + return Utf8Array(spec, offsets, data) store = self._open_store() path = self._list_col_path(name) if store.is_zip_store and self._mode == "r": @@ -874,6 +911,9 @@ def delete_column(self, name: str) -> None: key = self._col_key(name) if key in self._open_store(): del self._open_store()[key] + data_key = key + _UTF8_DATA_SUFFIX + if data_key in self._open_store(): + del self._open_store()[data_key] return list_path = self._list_col_path(name) if os.path.exists(list_path): @@ -888,6 +928,10 @@ def rename_column(self, old: str, new: str): if old_key in store: store[new_key] = store[old_key] del store[old_key] + old_data_key = old_key + _UTF8_DATA_SUFFIX + if old_data_key in store: + store[new_key + _UTF8_DATA_SUFFIX] = store[old_data_key] + del store[old_data_key] return store[new_key] old_path = self._list_col_path(old) new_path = self._list_col_path(new) @@ -1266,11 +1310,36 @@ def create_varlen_scalar_column( cparams=None, dparams=None, ) -> _ScalarVarLenArray: + from blosc2.schema import Utf8Spec + + if isinstance(spec, Utf8Spec): + logical_key = self._col_logical_key(name) + offsets_path = self._dest_path(logical_key, ".b2nd") + data_path = self._dest_path(logical_key + _UTF8_DATA_SUFFIX, ".b2nd") + os.makedirs(os.path.dirname(offsets_path), exist_ok=True) + offsets, data = _new_backend_arrays( + cparams, dparams, offsets_urlpath=offsets_path, data_urlpath=data_path + ) + for logical, dest_path in ( + (logical_key, offsets_path), + (logical_key + _UTF8_DATA_SUFFIX, data_path), + ): + rel_path = os.path.relpath(dest_path, self._working_dir()).replace(os.sep, "/") + self._store.map_tree[self._table_key(logical)] = rel_path + self._store._modified = True + return Utf8Array(spec, offsets, data) urlpath = self._list_col_path(name) os.makedirs(os.path.dirname(urlpath), exist_ok=True) return _make_persistent_backend(spec, urlpath, "w", cparams=cparams, dparams=dparams) def open_varlen_scalar_column(self, name: str, spec) -> _ScalarVarLenArray: + from blosc2.schema import Utf8Spec + + if isinstance(spec, Utf8Spec): + logical_key = self._col_logical_key(name) + offsets = self._open_leaf(logical_key) + data = self._open_leaf(logical_key + _UTF8_DATA_SUFFIX) + return Utf8Array(spec, offsets, data) if self._store.is_zip_store and self._mode == "r": rel = self._table_key(self._col_logical_key(name)).lstrip("/") + ".b2b" if rel not in self._store.offsets: @@ -1440,10 +1509,15 @@ def column_names_from_schema(self) -> list[str]: def delete_column(self, name: str) -> None: full_key = self._table_key(self._col_logical_key(name)) if full_key in self._store.map_tree: - filepath = self._store.map_tree.pop(full_key) - full_path = os.path.join(self._working_dir(), filepath) - if os.path.exists(full_path): - os.remove(full_path) + keys = [full_key] + data_key = self._table_key(self._col_logical_key(name) + _UTF8_DATA_SUFFIX) + if data_key in self._store.map_tree: + keys.append(data_key) + for key in keys: + filepath = self._store.map_tree.pop(key) + full_path = os.path.join(self._working_dir(), filepath) + if os.path.exists(full_path): + os.remove(full_path) return list_path = self._list_col_path(name) if os.path.exists(list_path): @@ -1453,16 +1527,23 @@ def delete_column(self, name: str) -> None: def rename_column(self, old: str, new: str) -> blosc2.NDArray: old_key = self._table_key(self._col_logical_key(old)) - new_key = self._table_key(self._col_logical_key(new)) if old_key in self._store.map_tree: - new_dest = self._dest_path(self._col_logical_key(new), ".b2nd") - old_dest = os.path.join(self._working_dir(), self._store.map_tree[old_key]) - os.makedirs(os.path.dirname(new_dest), exist_ok=True) - os.replace(old_dest, new_dest) - del self._store.map_tree[old_key] - self._store.map_tree[new_key] = os.path.relpath(new_dest, self._working_dir()).replace( - os.sep, "/" - ) + moves = [(old_key, self._col_logical_key(new))] + old_data_key = self._table_key(self._col_logical_key(old) + _UTF8_DATA_SUFFIX) + if old_data_key in self._store.map_tree: + moves.append((old_data_key, self._col_logical_key(new) + _UTF8_DATA_SUFFIX)) + new_dest = None + for src_key, dst_logical in moves: + dst_dest = self._dest_path(dst_logical, ".b2nd") + src_dest = os.path.join(self._working_dir(), self._store.map_tree[src_key]) + os.makedirs(os.path.dirname(dst_dest), exist_ok=True) + os.replace(src_dest, dst_dest) + del self._store.map_tree[src_key] + self._store.map_tree[self._table_key(dst_logical)] = os.path.relpath( + dst_dest, self._working_dir() + ).replace(os.sep, "/") + if new_dest is None: + new_dest = dst_dest self._store._modified = True return blosc2.open(new_dest, mode=self._mode) old_path = self._list_col_path(old) diff --git a/src/blosc2/schema.py b/src/blosc2/schema.py index 902d0005a..b0f001196 100644 --- a/src/blosc2/schema.py +++ b/src/blosc2/schema.py @@ -592,6 +592,49 @@ def to_metadata_dict(self) -> dict[str, Any]: return d +class Utf8Spec(SchemaSpec): + """Variable-length UTF-8 string column stored Arrow-style as offsets + bytes. + + Unlike :class:`string`, this spec does not use a fixed-width NumPy dtype: + each row stores exactly its UTF-8 byte length, so long or wildly + variable-length text does not waste space. Unlike :func:`vlstring` + (msgpack cells), values are stored in two companion NDArrays — ``int64`` + row offsets plus a ``uint8`` byte blob — and bulk reads materialize as + ``numpy.dtypes.StringDType`` arrays (requires NumPy >= 2.0). + + Parameters + ---------- + nullable: + If ``True`` and ``null_value`` is not set, choose a null sentinel from + the current CTable null policy when the schema is compiled. + null_value: + Explicit null sentinel string. Takes precedence over ``nullable=True``. + """ + + python_type = str + dtype = None + + def __init__(self, *, nullable: _builtin_bool = False, null_value: str | None = None): + if null_value is not None and not isinstance(null_value, str): + raise TypeError(f"utf8 null_value must be str, got {type(null_value).__name__!r}") + self.nullable = nullable or null_value is not None + self.null_value = _normalize_scalar_value(null_value) + + def to_pydantic_kwargs(self) -> dict[str, Any]: + return {} + + def to_metadata_dict(self) -> dict[str, Any]: + d: dict[str, Any] = {"kind": "utf8"} + if self.nullable: + d["nullable"] = True + if self.null_value is not None: + d["null_value"] = self.null_value + return d + + def display_label(self) -> str: + return "utf8" + + class ObjectSpec(SchemaSpec): """Schema-less Python object column backed by batched msgpack storage. @@ -782,6 +825,42 @@ def vlstring( ) +def utf8(*, nullable: bool = False, null_value: str | None = None) -> Utf8Spec: + """Build a variable-length UTF-8 string schema descriptor. + + Use this for high-cardinality or free-text string columns: values are + stored Arrow-style (``int64`` offsets plus a UTF-8 byte blob), so each row + costs exactly its encoded byte length, and bulk reads materialize as + ``numpy.dtypes.StringDType`` arrays (requires NumPy >= 2.0). For + LOW-cardinality strings (categories, enumerations) prefer + :func:`dictionary`, which stores repeated values once. + + Must be requested via ``blosc2.field(blosc2.utf8())`` — it is never + inferred automatically from plain ``str`` annotations. + + Parameters + ---------- + nullable: + If ``True`` and ``null_value`` is not set, choose a null sentinel from + the current CTable null policy when the schema is compiled. + null_value: + Explicit null sentinel string. Takes precedence over ``nullable=True``. + + Examples + -------- + >>> from dataclasses import dataclass + >>> import blosc2 as b2 + >>> @dataclass + ... class Row: + ... name: str = b2.field(b2.utf8()) + ... note: str = b2.field(b2.utf8(nullable=True)) + """ + from blosc2.utf8_array import string_dtype + + string_dtype() # fail early with a clear error on NumPy < 2.0 + return Utf8Spec(nullable=nullable, null_value=null_value) + + def object( *, nullable: bool = False, diff --git a/src/blosc2/schema_compiler.py b/src/blosc2/schema_compiler.py index 0d4990565..b295dd51e 100644 --- a/src/blosc2/schema_compiler.py +++ b/src/blosc2/schema_compiler.py @@ -28,6 +28,7 @@ ObjectSpec, SchemaSpec, StructSpec, + Utf8Spec, VLBytesSpec, VLStringSpec, complex64, @@ -76,6 +77,7 @@ "bytes": b2_bytes, "vlstring": VLStringSpec, "vlbytes": VLBytesSpec, + "utf8": Utf8Spec, "object": ObjectSpec, "timestamp": timestamp, # dictionary @@ -110,7 +112,7 @@ def compute_display_width(spec: SchemaSpec) -> int: """Return a reasonable terminal display width for *spec*'s column.""" if isinstance(spec, DictionarySpec): return 32 - if isinstance(spec, (VLStringSpec, VLBytesSpec, ObjectSpec)): + if isinstance(spec, (VLStringSpec, VLBytesSpec, ObjectSpec, Utf8Spec)): return 40 if isinstance(spec, NDArraySpec): return max(20, len(spec.display_label()) + 4) diff --git a/src/blosc2/utf8_array.py b/src/blosc2/utf8_array.py new file mode 100644 index 000000000..68056ffdc --- /dev/null +++ b/src/blosc2/utf8_array.py @@ -0,0 +1,397 @@ +####################################################################### +# Copyright (c) 2019-present, Blosc Development Team +# All rights reserved. +# +# This source code is licensed under a BSD-style license (found in the +# LICENSE file in the root directory of this source tree) +####################################################################### + +"""Internal variable-length UTF-8 string column backed by offsets + bytes. + +This module is *not* part of the public API. It provides row-wise string +semantics (one ``str`` value per row) stored Arrow-style as two companion +:class:`blosc2.NDArray` objects: + +* **offsets** — ``int64``, length ``n + 1`` where ``n`` is the number of + persisted rows. ``offsets[0]`` is always ``0`` and ``offsets[i+1]`` is the + end byte position of row ``i``. +* **data** — ``uint8``, the concatenated UTF-8 encoding of all row values. + Its length is at least ``offsets[n]`` (one slack byte is kept when empty + because zero-length NDArrays cannot be created). + +Reading rows ``[a, b)`` needs ``offsets[a : b + 1]`` plus +``bytes[offsets[a] : offsets[b]]`` — both plain NDArray slice reads. + +Nulls are represented with a per-column sentinel string (like every other +scalar CTable column); ``None`` written to a nullable column is converted to +the sentinel, and reads return the sentinel verbatim. + +Bulk reads return :class:`numpy.dtypes.StringDType` arrays (NumPy >= 2.0), +which support vectorized comparison, ordering, and ``np.strings`` functions. +""" + +from __future__ import annotations + +from typing import TYPE_CHECKING, Any + +import numpy as np + +if TYPE_CHECKING: + from collections.abc import Iterable, Iterator + +# Pending rows are flushed to the backing arrays once either bound is hit. +_FLUSH_ROWS = 4096 +_FLUSH_CHARS = 1 << 22 # ~4 Mi characters (>= 4 MiB encoded) + +# Storage grids for freshly created backing arrays. Both arrays are created +# tiny (shape (1,)) and grown by resize; the chunk shape is fixed at creation +# time, so it must be sized for the eventual data, not the initial shape. +_OFFSETS_CHUNKS = (2**17,) # 1 MiB chunks of int64 row offsets +_DATA_CHUNKS = (2**21,) # 2 MiB chunks of UTF-8 bytes + +# Sparse gathers read the persisted region in clusters; a new cluster starts +# when the gap between consecutive row indices exceeds this many rows. +_GATHER_GAP = 1024 + + +def string_dtype(): + """Return a ``numpy.dtypes.StringDType`` instance, or raise if unavailable.""" + try: + return np.dtypes.StringDType() + except AttributeError: # pragma: no cover - only on numpy < 2.0 + raise TypeError( + "utf8 columns require NumPy >= 2.0 (numpy.dtypes.StringDType); " + f"installed version is {np.__version__}. Use blosc2.vlstring() or " + "blosc2.string(max_length=...) instead." + ) from None + + +def _new_backend_arrays(cparams=None, dparams=None, *, offsets_urlpath=None, data_urlpath=None): + """Create fresh (offsets, data) NDArrays for an empty utf8 column.""" + import blosc2 + + kwargs: dict[str, Any] = {} + if cparams is not None: + kwargs["cparams"] = cparams + if dparams is not None: + kwargs["dparams"] = dparams + off_kwargs = dict(kwargs) + data_kwargs = dict(kwargs) + if offsets_urlpath is not None: + off_kwargs["urlpath"] = offsets_urlpath + off_kwargs["mode"] = "w" + if data_urlpath is not None: + data_kwargs["urlpath"] = data_urlpath + data_kwargs["mode"] = "w" + offsets = blosc2.zeros((1,), dtype=np.int64, chunks=_OFFSETS_CHUNKS, **off_kwargs) + data = blosc2.zeros((1,), dtype=np.uint8, chunks=_DATA_CHUNKS, **data_kwargs) + return offsets, data + + +class Utf8Array: + """Row-wise variable-length UTF-8 string array over offsets + bytes NDArrays. + + Provides the row-oriented interface expected by CTable columns: + ``append``, ``extend``, ``flush``, ``__len__``, ``__getitem__``, and + ``__setitem__``. Bulk reads return ``StringDType`` NumPy arrays; single + reads return ``str``. + + In-place assignment to row ``i`` rewrites the byte blob and offsets of + every row after ``i`` (a new value usually has a different byte length, + which shifts all subsequent offsets), so ``__setitem__`` costs O(n - i). + + This class is internal; obtain instances via + ``storage.create_varlen_scalar_column()`` or + ``storage.open_varlen_scalar_column()`` with a ``Utf8Spec``. + + Parameters + ---------- + spec: + The :class:`~blosc2.schema.Utf8Spec` describing this column. + offsets: + ``int64`` NDArray of row offsets (length ``n + 1``). Created fresh + (in memory) when ``None``. + data: + ``uint8`` NDArray with the concatenated UTF-8 bytes. Created fresh + (in memory) when ``None``. + """ + + def __init__(self, spec, offsets=None, data=None) -> None: + from blosc2.schema import Utf8Spec + + if not isinstance(spec, Utf8Spec): + raise TypeError(f"Utf8Array requires a Utf8Spec, got {type(spec)!r}") + self._dtype = string_dtype() + self._spec = spec + if (offsets is None) != (data is None): + raise ValueError("offsets and data must be provided together") + if offsets is None: + offsets, data = _new_backend_arrays() + self._offsets = offsets + self._data = data + self._persisted_rows: int = int(offsets.shape[0]) - 1 + # End byte position of the persisted region; resolved lazily because it + # needs a chunk read from the offsets array. + self._bytes_used_cache: int | None = 0 if self._persisted_rows == 0 else None + # Rows not yet flushed to the backing arrays (list of str). + self._pending: list[str] = [] + self._pending_chars: int = 0 + + # ------------------------------------------------------------------ + # Private helpers + # ------------------------------------------------------------------ + + @property + def _bytes_used(self) -> int: + if self._bytes_used_cache is None: + self._bytes_used_cache = int(self._offsets[self._persisted_rows]) + return self._bytes_used_cache + + def _coerce(self, value: Any) -> str: + """Coerce *value* to ``str``, mapping ``None`` to the null sentinel.""" + if value is None: + null_value = getattr(self._spec, "null_value", None) + if null_value is None: + raise TypeError("Column of utf8 strings is not nullable; received None.") + return null_value + if isinstance(value, str): + return str(value) # np.str_ instances are normalized to plain str + raise TypeError(f"Expected str for utf8 column, got {type(value).__name__!r}.") + + def _flush_if_needed(self) -> None: + if len(self._pending) >= _FLUSH_ROWS or self._pending_chars >= _FLUSH_CHARS: + self.flush() + + def _read_persisted_span(self, a: int, b: int) -> np.ndarray: + """Return persisted rows ``[a, b)`` as a StringDType array.""" + n = b - a + if n <= 0: + return np.empty(0, dtype=self._dtype) + offs = np.asarray(self._offsets[a : b + 1], dtype=np.int64) + start, end = int(offs[0]), int(offs[-1]) + blob = np.asarray(self._data[start:end]).tobytes() if end > start else b"" + rel = offs - start + out = np.empty(n, dtype=self._dtype) + for i in range(n): + out[i] = blob[rel[i] : rel[i + 1]].decode("utf-8") + return out + + def _gather_persisted(self, indices: np.ndarray) -> np.ndarray: + """Gather persisted rows at *indices* (any order) as a StringDType array. + + Indices are read in sorted clusters of nearby rows so that a sparse + gather (e.g. a few head and tail rows for display) does not read the + whole column. + """ + out = np.empty(len(indices), dtype=self._dtype) + if len(indices) == 0: + return out + order = np.argsort(indices, kind="stable") + sorted_idx = indices[order] + cluster_starts = np.flatnonzero(np.diff(sorted_idx) > _GATHER_GAP) + 1 + pos = 0 + for cluster in np.split(sorted_idx, cluster_starts): + lo, hi = int(cluster[0]), int(cluster[-1]) + span = self._read_persisted_span(lo, hi + 1) + out[order[pos : pos + len(cluster)]] = span[cluster - lo] + pos += len(cluster) + return out + + def _read_span(self, a: int, b: int) -> np.ndarray: + """Return rows ``[a, b)`` (persisted + pending) as a StringDType array.""" + np_rows = self._persisted_rows + if b <= np_rows: + return self._read_persisted_span(a, b) + persisted = self._read_persisted_span(a, min(b, np_rows)) if a < np_rows else None + pending = np.array(self._pending[max(0, a - np_rows) : b - np_rows], dtype=self._dtype) + if persisted is None: + return pending + return np.concatenate([persisted, pending]) + + def _get_many(self, indices: np.ndarray) -> np.ndarray: + indices = np.asarray(indices) + if indices.ndim != 1: + indices = indices.ravel() + n = len(self) + indices = np.where(indices < 0, indices + n, indices).astype(np.int64, copy=False) + if len(indices) and (indices.min() < 0 or indices.max() >= n): + raise IndexError("Utf8Array index out of range") + out = np.empty(len(indices), dtype=self._dtype) + np_rows = self._persisted_rows + pending_mask = indices >= np_rows + if pending_mask.any(): + out[pending_mask] = [self._pending[i - np_rows] for i in indices[pending_mask]] + if not pending_mask.all(): + persisted_mask = ~pending_mask + out[persisted_mask] = self._gather_persisted(indices[persisted_mask]) + return out + + def _rewrite_from(self, pos: int, values: list[str]) -> None: + """Replace persisted rows ``pos ..`` with *values*, shifting offsets. + + ``pos + len(values)`` becomes the new persisted row count; the byte + blob and offsets after ``pos`` are rewritten. + """ + encoded = [v.encode("utf-8") for v in values] + blob = b"".join(encoded) + if pos == 0: + start = 0 + elif pos == self._persisted_rows: + start = self._bytes_used + else: + start = int(self._offsets[pos]) + new_used = start + len(blob) + new_rows = pos + len(encoded) + if int(self._data.shape[0]) != max(new_used, 1): + self._data.resize((max(new_used, 1),)) + if blob: + self._data[start:new_used] = np.frombuffer(blob, dtype=np.uint8) + if int(self._offsets.shape[0]) != new_rows + 1: + self._offsets.resize((new_rows + 1,)) + if encoded: + lengths = np.fromiter((len(e) for e in encoded), dtype=np.int64, count=len(encoded)) + self._offsets[pos + 1 : new_rows + 1] = start + np.cumsum(lengths) + self._persisted_rows = new_rows + self._bytes_used_cache = new_used + + # ------------------------------------------------------------------ + # Public write interface + # ------------------------------------------------------------------ + + def append(self, value: Any) -> None: + """Append one string row (``None`` maps to the null sentinel).""" + value = self._coerce(value) + self._pending.append(value) + self._pending_chars += len(value) + self._flush_if_needed() + + def extend(self, values: Iterable[Any]) -> None: + """Append many string rows.""" + for v in values: + v = self._coerce(v) + self._pending.append(v) + self._pending_chars += len(v) + self._flush_if_needed() + + def flush(self) -> None: + """Write pending rows to the backing offsets/data NDArrays.""" + if not self._pending: + return + values, self._pending = self._pending, [] + self._pending_chars = 0 + self._rewrite_from(self._persisted_rows, values) + + # ------------------------------------------------------------------ + # Public read interface + # ------------------------------------------------------------------ + + def __len__(self) -> int: + return self._persisted_rows + len(self._pending) + + def __iter__(self) -> Iterator[str]: + yield from self[:] + + def __getitem__(self, index: int | slice | list | tuple | np.ndarray): + if isinstance(index, (int, np.integer)): + n = len(self) + index = int(index) + if index < 0: + index += n + if not (0 <= index < n): + raise IndexError("Utf8Array index out of range") + if index >= self._persisted_rows: + return self._pending[index - self._persisted_rows] + return str(self._read_persisted_span(index, index + 1)[0]) + + if isinstance(index, slice): + start, stop, step = index.indices(len(self)) + if step == 1: + return self._read_span(start, stop) + return self._get_many(np.arange(start, stop, step, dtype=np.int64)) + + if isinstance(index, np.ndarray) and index.dtype == np.bool_: + if len(index) != len(self): + raise IndexError(f"Boolean mask length {len(index)} does not match array length {len(self)}") + return self._get_many(np.flatnonzero(index)) + + if isinstance(index, (list, tuple, np.ndarray)): + return self._get_many(np.asarray(index, dtype=np.int64)) + + raise TypeError(f"Utf8Array indices must be int, slice, or array; got {type(index)!r}") + + def __setitem__(self, index: int, value: Any) -> None: + """Overwrite the value at *index*. + + Because row values have variable byte lengths, overwriting a persisted + row rewrites the byte blob and offsets of all subsequent rows — + an O(n - index) operation. + """ + if not isinstance(index, (int, np.integer)): + raise TypeError(f"Utf8Array assignment index must be int, got {type(index)!r}") + value = self._coerce(value) + n = len(self) + index = int(index) + if index < 0: + index += n + if not (0 <= index < n): + raise IndexError("Utf8Array index out of range") + if index >= self._persisted_rows: + self._pending[index - self._persisted_rows] = value + return + tail = self._read_persisted_span(index + 1, self._persisted_rows) + self._rewrite_from(index, [value, *tail.tolist()]) + + # ------------------------------------------------------------------ + # Properties mirroring the interface expected by CTable + # ------------------------------------------------------------------ + + @property + def spec(self): + return self._spec + + @property + def dtype(self): + """The ``StringDType`` used for materialized reads.""" + return self._dtype + + @property + def offsets(self): + """The underlying ``int64`` NDArray of row offsets (length ``n + 1``).""" + return self._offsets + + @property + def data(self): + """The underlying ``uint8`` NDArray with the concatenated UTF-8 bytes.""" + return self._data + + @property + def schunk(self): + return self._offsets.schunk + + @property + def urlpath(self) -> str | None: + return getattr(self._offsets, "urlpath", None) + + @property + def nbytes(self) -> int: + return self._offsets.schunk.nbytes + self._data.schunk.nbytes + + @property + def cbytes(self) -> int: + return self._offsets.schunk.cbytes + self._data.schunk.cbytes + + @property + def cratio(self) -> float: + cb = self.cbytes + if cb == 0: + return float("inf") + return self.nbytes / cb + + def copy(self, spec=None, **kwargs: Any) -> Utf8Array: + """Return an in-memory copy.""" + if spec is None: + spec = self._spec + out = Utf8Array(spec) + out.extend(self) + out.flush() + return out diff --git a/tests/ctable/test_utf8.py b/tests/ctable/test_utf8.py new file mode 100644 index 000000000..4b7276448 --- /dev/null +++ b/tests/ctable/test_utf8.py @@ -0,0 +1,429 @@ +####################################################################### +# Copyright (c) 2019-present, Blosc Development Team +# All rights reserved. +# +# This source code is licensed under a BSD-style license (found in the +# LICENSE file in the root directory of this source tree) +####################################################################### + +"""Tests for the utf8 schema spec (variable-length strings as offsets + bytes).""" + +from __future__ import annotations + +from dataclasses import dataclass + +import numpy as np +import pytest + +import blosc2 +from blosc2 import CTable + +STRING_DTYPE = np.dtypes.StringDType() + + +@dataclass +class Row: + name: str = blosc2.field(blosc2.utf8()) + x: int = blosc2.field(blosc2.int64()) + + +@dataclass +class NullableRow: + name: str = blosc2.field(blosc2.utf8(nullable=True)) + x: int = blosc2.field(blosc2.int64()) + + +# Mixed content: ASCII, non-ASCII (1..4-byte UTF-8), empty, 1-char, multi-KB. +SAMPLE = [ + "hello", + "", + "a", + "café", + "日本語のテキスト", + "emoji 🎉🚀", + "x" * 4096, + "línea con acentos y çedilla", +] + + +def make_table(values=None, **kwargs): + values = SAMPLE if values is None else values + return CTable(Row, new_data={"name": list(values), "x": list(range(len(values)))}, **kwargs) + + +# --------------------------------------------------------------------------- +# Schema spec +# --------------------------------------------------------------------------- + + +def test_utf8_spec_defaults(): + spec = blosc2.utf8() + assert spec.nullable is False + assert spec.null_value is None + assert spec.dtype is None + assert spec.python_type is str + + +def test_utf8_spec_metadata_round_trip(): + from blosc2.schema_compiler import spec_from_metadata_dict + + spec = blosc2.utf8(nullable=True, null_value="") + d = spec.to_metadata_dict() + assert d["kind"] == "utf8" + assert d["nullable"] is True + assert d["null_value"] == "" + + restored = spec_from_metadata_dict(d) + assert type(restored).__name__ == "Utf8Spec" + assert restored.nullable is True + assert restored.null_value == "" + + +def test_utf8_null_value_must_be_str(): + with pytest.raises(TypeError, match="null_value must be str"): + blosc2.utf8(null_value=42) + + +def test_utf8_display_width(): + from blosc2.schema_compiler import compute_display_width + + assert compute_display_width(blosc2.utf8()) == 40 + + +def test_utf8_not_inferred_from_plain_str_annotation(): + @dataclass + class Plain: + s: str + x: int + + t = CTable(Plain, new_data=[("abc", 1)]) + cc = t.schema.columns_by_name["s"] + assert type(cc.spec).__name__ == "string" # fixed-width default is unchanged + + +# --------------------------------------------------------------------------- +# Utf8Array internal adapter +# --------------------------------------------------------------------------- + + +def test_utf8_array_basic_roundtrip(): + from blosc2.utf8_array import Utf8Array + + arr = Utf8Array(blosc2.utf8()) + arr.extend(SAMPLE) + assert len(arr) == len(SAMPLE) + assert list(arr[:]) == SAMPLE + arr.flush() + assert list(arr[:]) == SAMPLE + assert arr[0] == "hello" + assert arr[-1] == SAMPLE[-1] + assert arr.dtype == STRING_DTYPE + + +def test_utf8_array_reads_across_pending_boundary(): + from blosc2.utf8_array import Utf8Array + + arr = Utf8Array(blosc2.utf8()) + arr.extend(SAMPLE[:4]) + arr.flush() + arr.extend(SAMPLE[4:]) # stays pending + assert list(arr[:]) == SAMPLE + assert list(arr[2:6]) == SAMPLE[2:6] + got = arr[np.array([7, 0, 5])] + assert got.dtype == STRING_DTYPE + assert list(got) == [SAMPLE[7], SAMPLE[0], SAMPLE[5]] + mask = np.zeros(len(SAMPLE), dtype=np.bool_) + mask[[1, 4]] = True + assert list(arr[mask]) == [SAMPLE[1], SAMPLE[4]] + + +def test_utf8_array_setitem_shifts_offsets(): + from blosc2.utf8_array import Utf8Array + + arr = Utf8Array(blosc2.utf8()) + arr.extend(["aa", "bb", "cc"]) + arr.flush() + arr[1] = "a longer replacement value" + assert list(arr[:]) == ["aa", "a longer replacement value", "cc"] + arr[1] = "" + assert list(arr[:]) == ["aa", "", "cc"] + + +def test_utf8_array_rejects_non_str(): + from blosc2.utf8_array import Utf8Array + + arr = Utf8Array(blosc2.utf8()) + with pytest.raises(TypeError, match="Expected str"): + arr.append(42) + with pytest.raises(TypeError, match="not nullable"): + arr.append(None) + + +# --------------------------------------------------------------------------- +# CTable integration: append / extend / read +# --------------------------------------------------------------------------- + + +def test_ctable_utf8_extend_and_read(): + t = make_table() + assert t.nrows == len(SAMPLE) + values = t["name"][:] + assert isinstance(values, np.ndarray) + assert values.dtype == STRING_DTYPE + assert list(values) == SAMPLE + assert t["name"][3] == "café" + assert t["name"][-2] == "x" * 4096 + + +def test_ctable_utf8_append_rows(): + t = CTable(Row) + t.append(("first", 1)) + t.append({"name": "segundo", "x": 2}) + t.append(("日本", 3)) + assert list(t["name"][:]) == ["first", "segundo", "日本"] + assert list(t["x"][:]) == [1, 2, 3] + + +def test_ctable_utf8_extend_numpy_fixed_width_input(): + t = CTable(Row) + t.extend({"name": np.array(["uno", "dos", "tres"]), "x": np.arange(3)}) + assert list(t["name"][:]) == ["uno", "dos", "tres"] + + +def test_ctable_utf8_setitem(): + t = make_table() + t["name"][0] = "replaced" + assert t["name"][0] == "replaced" + assert list(t["name"][1:4]) == SAMPLE[1:4] + t["name"][2:4] = ["p", "q"] + assert list(t["name"][:4]) == ["replaced", "", "p", "q"] + + +def test_ctable_utf8_iter_and_fancy_reads(): + t = make_table() + assert list(t["name"]) == SAMPLE + got = t["name"][[5, 1, 0]] + assert list(got) == [SAMPLE[5], SAMPLE[1], SAMPLE[0]] + mask = np.array([v.startswith("h") for v in SAMPLE]) + assert list(t["name"][mask]) == ["hello"] + + +def test_ctable_utf8_unique_and_value_counts(): + t = make_table(["b", "a", "b", "c", "a", "b"]) + assert list(t["name"].unique()) == ["a", "b", "c"] + assert t["name"].value_counts() == {"b": 3, "a": 2, "c": 1} + + +def test_ctable_utf8_repr_and_str(): + t = make_table() + text = str(t) + assert "hello" in text + assert "café" in text + col_repr = repr(t["name"]) + assert "name" in col_repr + info_text = repr(t.info) + assert "utf8" in info_text + + +def test_ctable_utf8_delete_and_compact(): + t = make_table(["a", "bb", "ccc", "dddd", "eeeee"]) + t.delete([1, 3]) + assert t.nrows == 3 + assert list(t["name"][:]) == ["a", "ccc", "eeeee"] + t.compact() + assert list(t["name"][:]) == ["a", "ccc", "eeeee"] + t.append(("tail", 99)) + assert list(t["name"][:]) == ["a", "ccc", "eeeee", "tail"] + + +def test_ctable_utf8_copy_and_take(): + t = make_table() + c = t.copy() + assert list(c["name"][:]) == SAMPLE + sub = t.take([0, 3, 5]) + assert list(sub["name"][:]) == [SAMPLE[0], SAMPLE[3], SAMPLE[5]] + + +def test_ctable_utf8_view_reads(): + t = make_table(["a", "bb", "ccc", "dddd"]) + v = t.head(2) + assert list(v["name"][:]) == ["a", "bb"] + v2 = t[t.x > 1] + assert list(v2["name"][:]) == ["ccc", "dddd"] + + +def test_ctable_utf8_add_and_drop_column(): + t = make_table(["a", "b"]) + t.add_column("note", blosc2.field(blosc2.utf8(), default="n/a")) + assert list(t["note"][:]) == ["n/a", "n/a"] + t.drop_column("note") + assert "note" not in t.col_names + + +# --------------------------------------------------------------------------- +# Nulls (sentinel-based) +# --------------------------------------------------------------------------- + + +def test_ctable_utf8_nullable_sentinel_from_policy(): + t = CTable(NullableRow, new_data={"name": ["a", None, "c"], "x": [1, 2, 3]}) + nv = t["name"].null_value + assert nv == "__BLOSC2_NULL__" + assert list(t["name"].is_null()) == [False, True, False] + assert t["name"].null_count() == 1 + # Reads surface the sentinel verbatim, like other sentinel-based columns. + assert t["name"][1] == nv + assert list(t["name"].fillna("")) == ["a", "", "c"] + + +def test_ctable_utf8_explicit_null_value(): + @dataclass + class R: + s: str = blosc2.field(blosc2.utf8(null_value="")) + x: int = blosc2.field(blosc2.int64()) + + t = CTable(R, new_data={"s": [None, "v"], "x": [0, 1]}) + assert t["s"].null_value == "" + assert list(t["s"][:]) == ["", "v"] + assert t["s"].null_count() == 1 + + +def test_ctable_utf8_not_nullable_rejects_none(): + t = CTable(Row) + with pytest.raises((TypeError, ValueError)): + t.append((None, 1)) + + +def test_ctable_utf8_dropna(): + t = CTable(NullableRow, new_data={"name": ["a", None, "c", None], "x": [1, 2, 3, 4]}) + kept = t.dropna(subset=["name"]) + assert list(kept["name"][:]) == ["a", "c"] + + +# --------------------------------------------------------------------------- +# Persistence round-trips +# --------------------------------------------------------------------------- + + +@pytest.mark.parametrize("ext", [".b2z", ".b2d"]) +def test_ctable_utf8_persistence_roundtrip(tmp_path, ext): + urlpath = str(tmp_path / f"utf8_table{ext}") + t = make_table(urlpath=urlpath, mode="w") + t.close() + + t2 = CTable.open(urlpath, mode="r") + try: + assert list(t2["name"][:]) == SAMPLE + assert t2["name"][4] == SAMPLE[4] + values = t2["name"][:] + assert values.dtype == STRING_DTYPE + finally: + t2.close() + + +def test_ctable_utf8_persistence_append_reopen(tmp_path): + urlpath = str(tmp_path / "utf8_append.b2z") + t = make_table(["one", "two"], urlpath=urlpath, mode="w") + t.close() + + t2 = CTable.open(urlpath, mode="a") + try: + t2.append(("three", 2)) + t2.extend({"name": ["four", ""], "x": [3, 4]}) + finally: + t2.close() + + t3 = CTable.open(urlpath, mode="r") + try: + assert list(t3["name"][:]) == ["one", "two", "three", "four", ""] + finally: + t3.close() + + +def test_ctable_utf8_nullable_persists(tmp_path): + urlpath = str(tmp_path / "utf8_null.b2z") + t = CTable(NullableRow, new_data={"name": ["a", None], "x": [1, 2]}, urlpath=urlpath, mode="w") + t.close() + t2 = CTable.open(urlpath, mode="r") + try: + assert t2["name"].null_value == "__BLOSC2_NULL__" + assert t2["name"].null_count() == 1 + finally: + t2.close() + + +def test_ctable_utf8_load_into_memory(tmp_path): + urlpath = str(tmp_path / "utf8_load.b2d") + t = make_table(urlpath=urlpath, mode="w") + t.close() + t2 = CTable.load(urlpath) + assert list(t2["name"][:]) == SAMPLE + t2.append(("appended", 100)) + assert t2["name"][-1] == "appended" + + +def test_ctable_utf8_save_copy_to_disk(tmp_path): + t = make_table() + urlpath = str(tmp_path / "utf8_saved.b2z") + t.save(urlpath) + t2 = CTable.open(urlpath, mode="r") + try: + assert list(t2["name"][:]) == SAMPLE + finally: + t2.close() + + +def test_ctable_utf8_cframe_roundtrip(): + t = make_table() + frame = t.to_cframe() + t2 = blosc2.ctable_from_cframe(frame) + assert list(t2["name"][:]) == SAMPLE + + +def test_ctable_utf8_rename_column_persistent(tmp_path): + urlpath = str(tmp_path / "utf8_rename.b2d") + t = make_table(["a", "b"], urlpath=urlpath, mode="w") + t.rename_column("name", "title") + assert list(t["title"][:]) == ["a", "b"] + t.close() + t2 = CTable.open(urlpath, mode="r") + try: + assert list(t2["title"][:]) == ["a", "b"] + finally: + t2.close() + + +# --------------------------------------------------------------------------- +# Unsupported operations fail clearly (lifted by later work) +# --------------------------------------------------------------------------- + + +def test_ctable_utf8_comparison_raises_clearly(): + t = make_table() + with pytest.raises(NotImplementedError, match="utf8"): + t.name == "hello" # noqa: B015 + + +def test_ctable_utf8_where_expression_raises_clearly(): + t = make_table() + with pytest.raises(NotImplementedError, match="utf8"): + t.where("name == 'hello'") + + +def test_ctable_utf8_groupby_raises_clearly(): + t = make_table(["a", "b", "a"]) + with pytest.raises(TypeError, match="variable-length"): + t.group_by("name").sum("x") + + +def test_ctable_utf8_sort_raises_clearly(): + t = make_table() + with pytest.raises(TypeError, match="utf8"): + t.sort_by("name") + + +def test_ctable_utf8_arrow_export_raises_clearly(): + pytest.importorskip("pyarrow") + t = make_table() + with pytest.raises(NotImplementedError, match="utf8"): + t.to_arrow() From 0df0c80353e65c623bb8798ffc72a50e140793ee Mon Sep 17 00:00:00 2001 From: Francesc Alted Date: Thu, 16 Jul 2026 14:33:28 +0200 Subject: [PATCH 02/20] Record P3.a implementation notes in the phase-3 plan Co-Authored-By: Claude Fable 5 --- plans/enhancing-ctable-phase3.md | 45 ++++++++++++++++++++++++++++++++ 1 file changed, 45 insertions(+) diff --git a/plans/enhancing-ctable-phase3.md b/plans/enhancing-ctable-phase3.md index 9551b15e6..337f52cc5 100644 --- a/plans/enhancing-ctable-phase3.md +++ b/plans/enhancing-ctable-phase3.md @@ -141,6 +141,51 @@ a utf8 column should work but the docstring must state the cost. Tests: roundtrip (ASCII, non-ASCII, empty strings, 1-char, multi-KB values), append/extend, setitem, persistence, repr. +**P3.a implementation notes (landed 2026-07-16, commit e5bbd559, branch +`enhancing-ctable3`):** + +- What landed: `Utf8Spec`/`blosc2.utf8()` in `schema.py` (kind `"utf8"`, + registered in `schema_compiler._KIND_TO_SPEC`); new `src/blosc2/utf8_array.py` + with the `Utf8Array` adapter; storage dispatch in all four `TableStorage` + backends; sentinel-null wiring; guards for the not-yet-supported operations; + 37 tests in `tests/ctable/test_utf8.py`. +- **Key deviation from the plan text**: rather than a new column category + with its own ~50 dispatch sites (the `DictionaryColumn` route), `Utf8Spec` + joins the `_is_varlen_scalar_column` predicate and `Utf8Array` implements + the `_ScalarVarLenArray` row interface (`append`/`extend`/`flush`/getitem/ + setitem). That made create/open/save/load/copy/take/cframe/TreeStore paths + work unmodified; utf8-specific branches exist only where semantics differ: + null handling (sentinel, not native `None` — `is_null`/`null_count`/ + `fillna`), empty reads and `iter_chunks` (StringDType arrays, not lists), + `compact()`, `to_cframe()`, `rename_column` reopen, and the guard messages. + A dedicated `Column.is_utf8` / `CTable._is_utf8_column` predicate marks + those spots — P3.c–P3.e should branch on it the same way. +- Storage layout: offsets NDArray at the plain column key; byte blob at + column key + `".utf8"` (`_UTF8_DATA_SUFFIX` in `ctable_storage.py`). The + literal dot cannot collide with user column names (dots in logical names + are path separators or percent-encoded). Both arrays are *logically* sized + (length = rows appended, like vlstring), NOT capacity-slotted — that keeps + the persisted-row count recoverable on open as `len(offsets) - 1` with no + extra metadata. Arrays are created shape-(1,) with large fixed chunkshapes + (offsets 2^17 rows, data 2^21 bytes) and grown by resize. +- Reads: contiguous spans are two slice reads + per-row byte-slice decode; + sparse gathers cluster sorted indices (gap > 1024 starts a new cluster) so + display head+tail fetches don't read the whole column. Writes buffer in + memory and flush every 4096 rows / ~4 Mi chars. `__setitem__` on a + persisted row rewrites the tail (documented O(n - i)). +- Null sentinel reads surface the sentinel string verbatim (consistent with + fixed-width `string`), unlike vlstring's native `None`. Nullable specs + resolve their sentinel from `NullPolicy.string_value`. +- Measured (Apple-silicon dev box, smoke run): 200k rows of random 0–40 char + ASCII + a couple of multibyte values → column nbytes 6.29 MB, cbytes + 2.65 MB (2.37x; random text, so most of the win is the offsets array). + Full `tests/ctable` (1357) and `tests/ndarray` (4385) suites pass. +- Known limits left for later sub-items (all raise clear errors): Arrow + export (`_pa_type_from_spec` raises → P3.b), comparisons/`where()` + (P3.c), groupby keys (P3.d), `sort_by` (P3.e). `iter_arrow_batches` hits + the varlen branch and raises through `_pa_type_from_spec`; `to_pandas` + of a table with utf8 columns therefore also raises until P3.b. + **P3.b Arrow interop.** `iter_arrow_batches` exports utf8 columns as `pa.large_string()` (always large — no int32-offset special-casing); `from_arrow` maps incoming `string`/`large_string` columns to `utf8` specs From 1ac204c285816ae1a6e84e1368f0bb8d3263a3be Mon Sep 17 00:00:00 2001 From: Francesc Alted Date: Thu, 16 Jul 2026 17:38:43 +0200 Subject: [PATCH 03/20] Arrow interop for utf8 columns: large_string export, sentinel-null import MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit CTable.to_arrow()/iter_arrow_batches() export utf8 columns as pa.large_string() (always large — no int32-offset special-casing), building the Arrow null mask from the column's sentinel. CTable.from_arrow()/from_parquet() now map incoming Arrow string/ large_string columns to blosc2.utf8() by default (previously vlstring). This is the sentinel-vs-mask checkpoint from the phase-3 plan: nulls map sentinel<->validity like every other scalar dtype, consistent with int/float/bool/timestamp/fixed-string columns, and column_null_values overrides now work for utf8 (rejected before, since vlstring nulls are native None). binary/large_binary columns are unaffected and still import as vlbytes. No lossiness observed against the tables exercised here, so P5's mask-based null design stays parked. Updated interop tests and the parquet_to_blosc2 CLI's progress-report labels/docstrings, which independently predicted "vlstring" for reporting purposes, to reflect the new utf8 default. Co-Authored-By: Claude Sonnet 5 --- plans/enhancing-ctable-phase3.md | 46 ++++++++++++++++ src/blosc2/cli/parquet_to_blosc2.py | 37 ++++++------- src/blosc2/ctable.py | 61 +++++++++++++-------- tests/ctable/test_arrow_interop.py | 18 +++++-- tests/ctable/test_dictionary_column.py | 6 +-- tests/ctable/test_parquet_interop.py | 42 ++++++++++----- tests/ctable/test_utf8.py | 75 ++++++++++++++++++++++++-- 7 files changed, 217 insertions(+), 68 deletions(-) diff --git a/plans/enhancing-ctable-phase3.md b/plans/enhancing-ctable-phase3.md index 337f52cc5..8f6022e6e 100644 --- a/plans/enhancing-ctable-phase3.md +++ b/plans/enhancing-ctable-phase3.md @@ -196,6 +196,52 @@ sentinel↔validity like every other dtype — **this is the sentinel-vs-mask checkpoint from decision 4**. Tests: `pa.table(ct)` roundtrip, duckdb query on a utf8 column, `from_arrow(pa_table)` ingest. +**P3.b implementation notes (landed 2026-07-16, branch `enhancing-ctable3`):** + +- What landed: `_pa_type_from_spec` maps `Utf8Spec` → `pa.large_string()` + (always large, per the plan); `iter_arrow_batches` builds a null mask from + the sentinel and exports proper Arrow nulls; `_arrow_type_to_spec` now maps + incoming Arrow `string`/`large_string` (when `string_max_length` is not + given) to `blosc2.utf8()` instead of `blosc2.vlstring()` — this is the + **sentinel checkpoint from decision 4, resolved as sentinel**: nullable + utf8 columns imported from Arrow get a sentinel from the active + `NullPolicy` (default `"__BLOSC2_NULL__"`) exactly like every other + nullable scalar dtype, and `column_null_values` overrides now work for + utf8 columns (previously rejected for vlstring, since vlstring nulls are + native `None`). No lossiness was observed in the tests exercised here + (synthetic Arrow tables, DuckDB round trips); **P5 was not unparked** — + revisit only if a real free-text corpus collides with the sentinel. +- `binary`/`large_binary` Arrow columns are unaffected — they still import as + `vlbytes` (native-`None` nulls); only the scalar-*string* default moved. +- **Deviation**: the plan's "keep the old mapping available via the existing + import options if one exists" — no such option exists (`string_max_length` + only toggles fixed-width vs variable-length, not which variable-length + representation to use), so none was added; this matches the cross-cutting + rule against building unrequested options. +- Ripple effects fixed to keep the suite honest, not just green: `Column.dtype` + now documents that utf8 columns report `numpy.dtypes.StringDType()` (its + docstring previously said variable-length columns always return `None`); + several `tests/ctable/test_arrow_interop.py` / + `tests/ctable/test_parquet_interop.py` tests asserted the old + vlstring-default/native-None-null behavior and were updated to assert the + new utf8/sentinel behavior instead of being loosened. The + `parquet_to_blosc2` CLI (`src/blosc2/cli/parquet_to_blosc2.py`) computes its + own "will this become vlstring?" labels purely for its progress report + (independent of the real dispatch in `ctable.py`), so those labels + (`"vlstring"`/`"vlstring_nullable"`/`"dictionary_decoded_to_vlstring"`, the + `--decode-dictionaries` help text, and the module docstring) were renamed to + `"utf8"`/`"utf8_nullable"`/`"dictionary_decoded_to_utf8"` to stay accurate; + its export-side Arrow-type-cast logic needed no behavior change (large_string + casts back to the original field type the same way string did). +- Tests: `tests/ctable/test_utf8.py` gained a dedicated Arrow-interop section + (`pa.table(ct)` roundtrip with and without nulls, `from_arrow` ingest from + both `string` and `large_string`, sentinel-null ingest, `string_max_length` + still yields fixed-width, and a DuckDB `SELECT ... WHERE` query run directly + against a CTable with a utf8 column via the Arrow C-stream protocol — + verified working end-to-end, including DuckDB querying the CTable object + itself, not just an exported `pa.Table`). Full `tests/ctable` (1365) and the + rest of `tests/` (1706) pass. + **P3.c Filters and expressions.** Carve utf8 out of the `_ensure_queryable` rejection for comparisons only; implement `==`, `!=`, `<`, `<=`, `>`, `>=` against scalars and other utf8 columns via the diff --git a/src/blosc2/cli/parquet_to_blosc2.py b/src/blosc2/cli/parquet_to_blosc2.py index 81793fd1d..c2d5ad97c 100644 --- a/src/blosc2/cli/parquet_to_blosc2.py +++ b/src/blosc2/cli/parquet_to_blosc2.py @@ -16,9 +16,10 @@ The output extension selects the storage layout: ``.b2z`` is compact/zip-backed, while ``.b2d`` is sparse directory-backed. -Scalar string columns are stored as ``vlstring`` (variable-length, no length -limit). Scalar binary columns are stored as ``vlbytes``. Nullable string/binary -columns are represented with native ``None`` — no sentinel value is needed. +Scalar string columns are stored as ``utf8`` (variable-length, no length +limit; nullable columns get a sentinel string). Scalar binary columns are +stored as ``vlbytes``, whose nullable columns represent nulls with native +``None`` — no sentinel value is needed. Struct-valued columns are wrapped as ``list`` (one-element lists) so that they round-trip through the list-column machinery. True list columns pass @@ -165,7 +166,7 @@ def build_parser() -> argparse.ArgumentParser: default=None, help=( "Pre-scan string columns and import columns whose maximum character length is at most " - "this value as fixed-width, indexable strings. Other string columns remain vlstring." + "this value as fixed-width, indexable strings. Other string columns remain utf8." ), ) parser.add_argument( @@ -310,7 +311,7 @@ def build_parser() -> argparse.ArgumentParser: "--decode-dictionaries", action="store_true", help=( - "Decode Arrow dictionary-encoded columns to plain vlstring instead of preserving " + "Decode Arrow dictionary-encoded columns to plain utf8 instead of preserving " "the dictionary encoding. By default, supported dictionary columns " "(string values with integer indices) are imported as Blosc2 dictionary columns." ), @@ -434,12 +435,12 @@ def classify_columns( # noqa: C901 vt = t.value_type if vt in (pa.string(), pa.large_string(), pa.utf8(), pa.large_utf8()): if decode_dictionaries: - # Decode to plain vlstring. + # Decode to plain utf8. fixed_cols[field.name] = pa.field( field.name, pa.string(), nullable=field.nullable, metadata=field.metadata ) conversions[field.name] = { - "conversion": "dictionary_decoded_to_vlstring", + "conversion": "dictionary_decoded_to_utf8", "ordered": bool(t.ordered), } else: @@ -482,9 +483,7 @@ def classify_columns( # noqa: C901 "max_length": fixed_string_lengths[field.name], } else: - conversions[field.name] = { - "conversion": "vlstring_nullable" if field.nullable else "vlstring" - } + conversions[field.name] = {"conversion": "utf8_nullable" if field.nullable else "utf8"} continue if pa.types.is_binary(t) or pa.types.is_large_binary(t): fixed_cols[field.name] = field @@ -867,9 +866,7 @@ def print_import_plan( conversions, nullable_scalars, ): - vlstring_cols = [ - n for n, e in conversions.items() if e.get("conversion") in {"vlstring", "vlstring_nullable"} - ] + utf8_cols = [n for n, e in conversions.items() if e.get("conversion") in {"utf8", "utf8_nullable"}] vlbytes_cols = [ n for n, e in conversions.items() if e.get("conversion") in {"vlbytes", "vlbytes_nullable"} ] @@ -881,7 +878,7 @@ def print_import_plan( ] dict_cols = [n for n, e in conversions.items() if e.get("conversion") == "dictionary_preserved"] dict_decoded_cols = [ - n for n, e in conversions.items() if e.get("conversion") == "dictionary_decoded_to_vlstring" + n for n, e in conversions.items() if e.get("conversion") == "dictionary_decoded_to_utf8" ] flattened_structs = [ n for n, e in conversions.items() if e.get("conversion") == "struct_flattened_to_columns" @@ -897,16 +894,16 @@ def print_import_plan( print(f"Parquet columns: {len(parquet_schema)}") print(f"Imported columns: {len(fixed_cols) + len(struct_wrap_cols)}") n_fixed_non_string = ( - len(fixed_cols) - len(vlstring_cols) - len(vlbytes_cols) - len(dict_cols) - len(dict_decoded_cols) + len(fixed_cols) - len(utf8_cols) - len(vlbytes_cols) - len(dict_cols) - len(dict_decoded_cols) ) print(f" Fixed-width: {n_fixed_non_string}") print(f" Fixed strings: {len(fixed_string_cols)}") print(f" Fixed bytes: {len(fixed_bytes_cols)}") - print(f" vlstring: {len(vlstring_cols)}") + print(f" utf8: {len(utf8_cols)}") print(f" vlbytes: {len(vlbytes_cols)}") print(f" Dictionary: {len(dict_cols)}") if dict_decoded_cols: - print(f" Dict→vlstring: {len(dict_decoded_cols)}") + print(f" Dict→utf8: {len(dict_decoded_cols)}") print(f" Struct→columns: {len(flattened_structs)}") print(f" Struct→list: {len(wrapped_structs)}") print(f" Nullable scalars: {len(nullable_scalars)}") @@ -1421,15 +1418,15 @@ def export_ctable_to_parquet(input_path: Path, output_path: Path, *, batch_size: conversion = meta.get("conversion", "") if conversion in singleton_list_conversions: arr = unwrap_singleton_list(pa, arr, field.type) - elif conversion in {"vlstring", "vlstring_nullable", "vlbytes", "vlbytes_nullable"}: + elif conversion in {"utf8", "utf8_nullable", "vlbytes", "vlbytes_nullable"}: if str(arr.type) != str(field.type): arr = arr.cast(field.type) elif conversion in {"dictionary_preserved"}: # CTable emits dictionary; restore original type if needed. if str(arr.type) != str(field.type): arr = arr.cast(field.type, safe=True) - elif conversion in {"dictionary_decoded_to_vlstring"}: - # Was decoded to vlstring on import; restore as dictionary type on export. + elif conversion in {"dictionary_decoded_to_utf8"}: + # Was decoded to utf8 on import; restore as dictionary type on export. if pa.types.is_dictionary(field.type): encoded = pa.DictionaryArray.from_arrays( *pa.array(arr.to_pylist()) diff --git a/src/blosc2/ctable.py b/src/blosc2/ctable.py index 7a1e731cb..de5e75f25 100644 --- a/src/blosc2/ctable.py +++ b/src/blosc2/ctable.py @@ -2071,9 +2071,13 @@ def __ge__(self, other): @property def dtype(self): - """NumPy dtype of the underlying storage, or ``None`` for - variable-length columns (:func:`~blosc2.vlstring`, - :func:`~blosc2.vlbytes`, :func:`~blosc2.list`).""" + """NumPy dtype of the underlying storage. + + ``None`` for variable-length columns with no fixed element dtype + (:func:`~blosc2.vlstring`, :func:`~blosc2.vlbytes`, + :func:`~blosc2.list`). :func:`~blosc2.utf8` columns report + ``numpy.dtypes.StringDType()``, the dtype of their materialized reads. + """ return getattr(self._raw_col, "dtype", None) def iter_chunks(self, size: int = 65536): @@ -6473,10 +6477,9 @@ def _pa_type_from_spec(pa, spec): if isinstance(spec, DictionarySpec): return pa.dictionary(pa.int32(), pa.string(), ordered=spec.ordered) if isinstance(spec, Utf8Spec): - raise NotImplementedError( - "Arrow export of variable-length utf8 columns is not supported yet; " - "exclude the column via the columns= argument." - ) + # Always large_string: 64-bit offsets match the int64 offsets array, + # so multi-GB string columns export without int32-offset overflow. + return pa.large_string() if isinstance(spec, VLStringSpec): return pa.string() if isinstance(spec, VLBytesSpec): @@ -6569,6 +6572,19 @@ def iter_arrow_batches( # noqa: C901 spec = self._schema.columns_by_name[name].spec arrays.append(pa.array(col[start:stop], type=self._pa_type_from_spec(pa, spec))) continue + if col.is_utf8: + spec = self._schema.columns_by_name[name].spec + values = col[start:stop] # StringDType array with sentinel nulls + nv = getattr(spec, "null_value", None) + null_mask = values == nv if nv is not None else None + arrays.append( + pa.array( + values.tolist(), + type=pa.large_string(), + mask=null_mask if null_mask is not None and null_mask.any() else None, + ) + ) + continue if col.is_varlen_scalar: spec = self._schema.columns_by_name[name].spec values = col[start:stop] # list of str/bytes/None @@ -6866,8 +6882,9 @@ def _arrow_type_to_spec( # noqa: C901 if pa_type in (pa.string(), pa.large_string(), pa.utf8(), pa.large_utf8()): if string_max_length is None: - # No fixed-width threshold given: store as variable-length scalar string. - return b2s.vlstring(nullable=nullable) + # No fixed-width threshold given: store as a variable-length + # utf8 column (offsets + bytes, StringDType reads). + return b2s.utf8(nullable=nullable, null_value=null_value) max_length = max(string_max_length, len(null_value) if null_value is not None else 1, 1) return b2s.string(max_length=max_length, null_value=null_value) @@ -6924,17 +6941,15 @@ def _compiled_columns_from_arrow( field_is_struct = pa.types.is_struct(field.type) field_is_dictionary = pa.types.is_dictionary(field.type) column_string_max_length = cls._string_max_length_for_column(string_max_length, name) + # Scalar strings without a fixed-width threshold import as utf8 + # columns, which use null sentinels like other scalar columns; + # only binary columns keep the native-None varlen treatment. field_is_varlen_scalar = ( not field_is_list and not field_is_struct and not field_is_dictionary and column_string_max_length is None - and ( - pa.types.is_string(field.type) - or pa.types.is_large_string(field.type) - or pa.types.is_binary(field.type) - or pa.types.is_large_binary(field.type) - ) + and (pa.types.is_binary(field.type) or pa.types.is_large_binary(field.type)) ) field_needs_object_fallback = cls._arrow_type_needs_object_fallback(pa, field.type) if field_needs_object_fallback and not object_fallback: @@ -6950,7 +6965,7 @@ def _compiled_columns_from_arrow( ) if has_null_value_override and field_is_varlen_scalar: raise TypeError( - f"column_null_values is not supported for vlstring/vlbytes column {name!r}; " + f"column_null_values is not supported for vlbytes column {name!r}; " "these columns represent nulls as native None." ) if has_null_value_override: @@ -7553,12 +7568,14 @@ def from_arrow( # noqa: C901 t.trip.begin.lon.max() When *string_max_length* is ``None`` (the default), scalar Arrow - ``string`` / ``large_string`` columns are imported as - :func:`~blosc2.vlstring` columns and ``binary`` / ``large_binary`` - columns are imported as :func:`~blosc2.vlbytes` columns. Non-struct - ``struct`` columns (not containing only scalar leaves) are imported as + ``string`` / ``large_string`` columns are imported as variable-length + :func:`~blosc2.utf8` columns (offsets + bytes storage; nullable + columns get a null sentinel string from the active + :class:`NullPolicy`) and ``binary`` / ``large_binary`` columns are + imported as :func:`~blosc2.vlbytes` columns. Non-struct ``struct`` + columns (not containing only scalar leaves) are imported as :func:`~blosc2.struct` columns backed by batched variable-length - storage. Null values for these variable-length scalar columns are + storage. Null values for ``vlbytes``/``struct`` columns are represented as native ``None`` with no sentinel needed. When *string_max_length* is set to a positive integer, scalar string @@ -7566,7 +7583,7 @@ def from_arrow( # noqa: C901 :func:`~blosc2.string` / :func:`~blosc2.bytes` columns whose dtype is sized to *string_max_length* characters/bytes. It may also be a mapping from column name to max length; omitted string/binary columns remain - :func:`~blosc2.vlstring` / :func:`~blosc2.vlbytes` columns. + :func:`~blosc2.utf8` / :func:`~blosc2.vlbytes` columns. ``blosc2_batch_size`` controls how many rows are buffered before BatchArray-backed imported columns (list columns and variable-length diff --git a/tests/ctable/test_arrow_interop.py b/tests/ctable/test_arrow_interop.py index dde4f1822..42b20c719 100644 --- a/tests/ctable/test_arrow_interop.py +++ b/tests/ctable/test_arrow_interop.py @@ -306,12 +306,13 @@ def test_from_arrow_all_numeric_types(): assert t.col_names == list(at.column_names) -def test_from_arrow_string_default_is_vlstring(): - """Without string_max_length, scalar string columns become vlstring (variable-length).""" +def test_from_arrow_string_default_is_utf8(): + """Without string_max_length, scalar string columns become utf8 (variable-length).""" at = pa.table({"name": pa.array(["hi", "hello world", "!"], type=pa.string())}) t = CTable.from_arrow(at.schema, at.to_batches()) assert t["name"].is_varlen_scalar - assert t["name"].dtype is None + assert t["name"].is_utf8 + assert t["name"].dtype == np.dtypes.StringDType() assert list(t["name"][:]) == ["hi", "hello world", "!"] @@ -520,15 +521,22 @@ def test_arrow_c_stream_empty_table(): def test_from_arrow_accepts_another_ctable(): # A CTable is itself a capsule producer, so it can be ingested directly. + # The fixed-width "label" column re-imports as utf8 (large_string), so + # compare column values rather than the exact Arrow schema. t = _mixed_table() roundtripped = CTable.from_arrow(t) - assert roundtripped.to_arrow().equals(t.to_arrow()) + left, right = roundtripped.to_arrow(), t.to_arrow() + assert left.column("label").to_pylist() == right.column("label").to_pylist() + assert left.drop(["label"]).equals(right.drop(["label"])) def test_from_arrow_streams_filtered_view(): t = _mixed_table() view = t[t.id != t["id"].null_value] - assert CTable.from_arrow(view).to_arrow().equals(view.to_arrow()) + left = CTable.from_arrow(view).to_arrow() + right = view.to_arrow() + assert left.column("label").to_pylist() == right.column("label").to_pylist() + assert left.drop(["label"]).equals(right.drop(["label"])) def test_from_arrow_rejects_capsule_plus_batches(): diff --git a/tests/ctable/test_dictionary_column.py b/tests/ctable/test_dictionary_column.py index d21652b03..86acf4be3 100644 --- a/tests/ctable/test_dictionary_column.py +++ b/tests/ctable/test_dictionary_column.py @@ -494,7 +494,7 @@ def test_cli_preserves_dict_by_default(tmp_path): def test_cli_decode_dictionaries_flag(tmp_path): from blosc2.cli.parquet_to_blosc2 import main - from blosc2.schema import VLStringSpec + from blosc2.schema import Utf8Spec path = tmp_path / "dict.parquet" out = tmp_path / "dict_decoded.b2d" @@ -506,8 +506,8 @@ def test_cli_decode_dictionaries_flag(tmp_path): assert main(["--decode-dictionaries", str(path), str(out)]) == 0 ct = CTable.open(str(out), mode="r") - assert isinstance(ct._schema.columns_by_name["vendor"].spec, VLStringSpec) - assert ct["vendor"][:] == ["Uber", "Lyft", "Uber"] + assert isinstance(ct._schema.columns_by_name["vendor"].spec, Utf8Spec) + assert list(ct["vendor"][:]) == ["Uber", "Lyft", "Uber"] ct.close() diff --git a/tests/ctable/test_parquet_interop.py b/tests/ctable/test_parquet_interop.py index 310262c82..777b4bb1e 100644 --- a/tests/ctable/test_parquet_interop.py +++ b/tests/ctable/test_parquet_interop.py @@ -417,16 +417,18 @@ def test_from_arrow_invalid_blosc2_batch_size_raises(self): with pytest.raises(ValueError, match="blosc2_batch_size"): CTable.from_arrow(at.schema, at.to_batches(), blosc2_batch_size=0) - def test_vlstring_arrow_roundtrip_no_singleton_list(self): - """Scalar string columns import as vlstring (not list) without singleton wrapping.""" + def test_utf8_arrow_roundtrip_no_singleton_list(self): + """Scalar string columns import as utf8 (not list) without singleton wrapping.""" long_str = "x" * 500 at = pa.table({"txt": pa.array(["short", long_str, None, "end"], type=pa.string())}) t = CTable.from_arrow(at.schema, at.to_batches()) assert t["txt"].is_varlen_scalar - assert list(t["txt"][:]) == ["short", long_str, None, "end"] + assert t["txt"].is_utf8 + nv = t["txt"].null_value + assert list(t["txt"][:]) == ["short", long_str, nv, "end"] # Export back to Arrow → still a scalar string column, not list out = t.to_arrow() - assert pa.types.is_string(out.schema.field("txt").type) + assert pa.types.is_large_string(out.schema.field("txt").type) assert out.column("txt").to_pylist() == ["short", long_str, None, "end"] def test_vlbytes_arrow_roundtrip_no_singleton_list(self): @@ -440,7 +442,7 @@ def test_vlbytes_arrow_roundtrip_no_singleton_list(self): assert pa.types.is_large_binary(out.schema.field("bin").type) assert out.column("bin").to_pylist() == [b"short", long_bin, None, b"end"] - def test_vlstring_parquet_roundtrip(self, tmp_path): + def test_utf8_parquet_roundtrip(self, tmp_path): """Parquet import/export round-trips long scalar strings without singleton-list wrapping.""" long_str = "y" * 1000 at = pa.table( @@ -449,17 +451,19 @@ def test_vlstring_parquet_roundtrip(self, tmp_path): "txt": pa.array(["short", long_str, None, "end"], type=pa.string()), } ) - path = tmp_path / "vlstring.parquet" + path = tmp_path / "utf8.parquet" pq.write_table(at, path) t = CTable.from_parquet(path) assert t["txt"].is_varlen_scalar - assert list(t["txt"][:]) == ["short", long_str, None, "end"] + assert t["txt"].is_utf8 + nv = t["txt"].null_value + assert list(t["txt"][:]) == ["short", long_str, nv, "end"] - out = tmp_path / "vlstring_out.parquet" + out = tmp_path / "utf8_out.parquet" t.to_parquet(out) rt = pq.read_table(out) - assert pa.types.is_string(rt.schema.field("txt").type) + assert pa.types.is_large_string(rt.schema.field("txt").type) assert rt.column("txt").to_pylist() == ["short", long_str, None, "end"] @@ -530,7 +534,8 @@ def test_auto_nullable_scalars_roundtrip(self, tmp_path): out = tmp_path / "nullable_scalars_out.parquet" t.to_parquet(out) rt = pq.read_table(out) - assert rt.schema == at.schema + # "s" round-trips as utf8 -> large_string, unlike the other columns. + assert rt.schema == at.schema.set(at.schema.get_field_index("s"), pa.field("s", pa.large_string())) assert rt.to_pylist() == at.to_pylist() def test_null_policy_controls_default_sentinels(self): @@ -586,12 +591,21 @@ def test_null_values_override_policy_and_auto_false(self, tmp_path): t2 = CTable.from_parquet(path, auto_null_sentinels=False) assert t2._schema.columns_by_name["i"].spec.null_value == -1 - def test_null_policy_rejects_vlstring_column_null_values(self): - """Passing column_null_values for a vlstring column raises TypeError.""" + def test_null_policy_rejects_vlbytes_column_null_values(self): + """Passing column_null_values for a vlbytes column raises TypeError.""" + at = pa.table({"b": pa.array([b"a", None, b"c"], type=pa.large_binary())}) + policy = blosc2.NullPolicy(column_null_values={"b": b"NA"}) + with blosc2.null_policy(policy), pytest.raises(TypeError, match="vlbytes"): + CTable.from_arrow(at.schema, at.to_batches()) + + def test_null_policy_column_null_values_applies_to_utf8(self): + """Passing column_null_values for a utf8 (scalar string) column sets its sentinel.""" at = pa.table({"s": pa.array(["a", None, "c"], type=pa.string())}) policy = blosc2.NullPolicy(column_null_values={"s": "NA"}) - with blosc2.null_policy(policy), pytest.raises(TypeError, match="vlstring"): - CTable.from_arrow(at.schema, at.to_batches()) + with blosc2.null_policy(policy): + t = CTable.from_arrow(at.schema, at.to_batches()) + assert t["s"].null_value == "NA" + assert list(t["s"][:]) == ["a", "NA", "c"] def test_null_policy_unknown_column_raises(self): at = pa.table({"i": pa.array([1, None], type=pa.int32())}) diff --git a/tests/ctable/test_utf8.py b/tests/ctable/test_utf8.py index 4b7276448..dc8b6c42f 100644 --- a/tests/ctable/test_utf8.py +++ b/tests/ctable/test_utf8.py @@ -422,8 +422,75 @@ def test_ctable_utf8_sort_raises_clearly(): t.sort_by("name") -def test_ctable_utf8_arrow_export_raises_clearly(): - pytest.importorskip("pyarrow") +def test_ctable_utf8_arrow_export_large_string(): + pa = pytest.importorskip("pyarrow") t = make_table() - with pytest.raises(NotImplementedError, match="utf8"): - t.to_arrow() + at = t.to_arrow() + assert at.schema.field("name").type == pa.large_string() + assert at.column("name").to_pylist() == SAMPLE + + +# --------------------------------------------------------------------------- +# Arrow interop (P3.b) +# --------------------------------------------------------------------------- + + +def test_utf8_pa_table_roundtrip(): + pa = pytest.importorskip("pyarrow") + t = make_table() + at = pa.table(t) + assert at.column("name").to_pylist() == SAMPLE + + +def test_utf8_pa_table_roundtrip_with_nulls(): + pa = pytest.importorskip("pyarrow") + t = CTable(NullableRow, new_data={"name": ["a", None, "c"], "x": [1, 2, 3]}) + at = pa.table(t) + assert at.schema.field("name").type == pa.large_string() + assert at.column("name").to_pylist() == ["a", None, "c"] + assert at.column("name").null_count == 1 + + +def test_utf8_from_arrow_default_ingest(): + pa = pytest.importorskip("pyarrow") + at = pa.table({"name": pa.array(SAMPLE, type=pa.string()), "x": pa.array(range(len(SAMPLE)))}) + t = CTable.from_arrow(at.schema, at.to_batches()) + assert t["name"].is_utf8 + assert list(t["name"][:]) == SAMPLE + + +def test_utf8_from_arrow_large_string_ingest(): + pa = pytest.importorskip("pyarrow") + at = pa.table({"name": pa.array(SAMPLE, type=pa.large_string())}) + t = CTable.from_arrow(at.schema, at.to_batches()) + assert t["name"].is_utf8 + assert list(t["name"][:]) == SAMPLE + + +def test_utf8_from_arrow_nulls_use_sentinel(): + pa = pytest.importorskip("pyarrow") + at = pa.table({"name": pa.array(["a", None, "c"], type=pa.string())}) + t = CTable.from_arrow(at.schema, at.to_batches()) + nv = t["name"].null_value + assert nv is not None + assert list(t["name"][:]) == ["a", nv, "c"] + assert t["name"].null_count() == 1 + + +def test_utf8_from_arrow_fixed_width_when_max_length_given(): + pa = pytest.importorskip("pyarrow") + at = pa.table({"name": pa.array(["hi", "there"], type=pa.string())}) + t = CTable.from_arrow(at.schema, at.to_batches(), string_max_length=32) + assert not t["name"].is_utf8 + assert t["name"].dtype.kind == "U" + + +def test_utf8_duckdb_query(): + duckdb = pytest.importorskip("duckdb") + pytest.importorskip("pyarrow") + t = make_table(["paris", "london", "paris", "tokyo"]) + arrow_tbl = t.to_arrow() + result = duckdb.sql( + "SELECT name, count(*) AS n FROM arrow_tbl WHERE name = 'paris' GROUP BY name" + ).fetchall() + assert result == [("paris", 2)] From ceef75831e3de42bbf19bf3264dc8a4226a402ae Mon Sep 17 00:00:00 2001 From: Francesc Alted Date: Thu, 16 Jul 2026 17:46:17 +0200 Subject: [PATCH 04/20] Vectorized comparisons for utf8 columns via chunked StringDType predicates Column.__eq__/__ne__/__lt__/__le__/__gt__/__ge__ now special-case utf8 columns (mirroring the existing dictionary-column special-case), comparing against a str scalar or another utf8 Column chunk by chunk in NumPy (StringDType supports ==, !=, and ordering natively) and returning a physical-length boolean predicate intersected with the column's live-row mask -- usable directly in t[t.name == "x"] or t.where(...). A null value on either side never satisfies any comparison (the same SQL WHERE rule other nullable columns already follow), via a new Column._utf8_null_pred() OR'd across operands. Arithmetic and bitwise operators on utf8 columns still raise NotImplementedError; only comparisons are implemented here. blosc2.startswith/endswith already worked against a utf8 Column operand unmodified -- pinned with a test, no code change needed. String-expression predicates (t.where("name == 'x'")) still raise, since numexpr/miniexpr cannot evaluate StringDType. Co-Authored-By: Claude Sonnet 5 --- plans/enhancing-ctable-phase3.md | 47 ++++++++++++++++ src/blosc2/ctable.py | 97 +++++++++++++++++++++++++++++++- tests/ctable/test_utf8.py | 84 +++++++++++++++++++++++++-- 3 files changed, 223 insertions(+), 5 deletions(-) diff --git a/plans/enhancing-ctable-phase3.md b/plans/enhancing-ctable-phase3.md index 8f6022e6e..5d253cb52 100644 --- a/plans/enhancing-ctable-phase3.md +++ b/plans/enhancing-ctable-phase3.md @@ -252,6 +252,53 @@ work — add tests pinning them. Tests: filter correctness incl. null rows, `t[t.name == "x"]` on views, comparison with non-string scalar raises clearly. +**P3.c implementation notes (landed 2026-07-16, branch `enhancing-ctable3`):** + +- What landed: `Column.__eq__`/`__ne__`/`__lt__`/`__le__`/`__gt__`/`__ge__` + special-case `is_utf8` (mirroring the existing `is_dictionary` special-case + in `__eq__`/`__ne__`) and dispatch to a new `Column._utf8_compare(op, + other)`, bypassing `_ensure_comparable()`/`_ensure_queryable()` entirely for + these six operators. `_ensure_queryable()`'s utf8 rejection message was + narrowed to say arithmetic/bitwise ops are unsupported (comparisons are + no longer rejected there). Arithmetic (`+`, `-`, …) and bitwise ops on + utf8 columns still raise, unchanged. +- `_utf8_compare` reads the column chunk-by-chunk via a new + `Column._utf8_chunked_bool` helper (StringDType comparisons run natively + in NumPy — `np.equal`/`np.less`/etc. all work directly on + `numpy.dtypes.StringDType` arrays, verified empirically before writing + this), and returns a physical-length (`_valid_rows`-length) boolean + `blosc2.NDArray`, intersected with the column's live-row mask exactly like + `_dictionary_eq` — so the result is usable directly as `t[t.name == "x"]`, + `t.where(...)`, or combined with `&`/`|`/`~` against other predicates. + Supports scalar `str` operands and column-vs-column comparison between two + utf8 `Column`s (both must be utf8; comparing against any other type raises + `TypeError` naming the column and expected types). +- **Null handling implements the phase-1 SQL `WHERE` rule**: a null value on + either side never satisfies any comparison (`==`, `!=`, `<`, …), via a new + `Column._utf8_null_pred()` (parallel to `_raw_null_pred()` for sentinel + columns) OR'd across both operands and subtracted from the raw predicate — + same shape as `_null_aware_compare`, just utf8-specific because the + sentinel lives in `StringDType` values, not a fixed-width NumPy dtype. +- `blosc2.startswith`/`blosc2.endswith` (and other `np.strings`-backed + `LazyExpr` function ops) were verified to already work unmodified against a + utf8 `Column` operand — no changes needed, per the plan's expectation; a + pinning test was added (`test_ctable_utf8_startswith_endswith`). +- String-expression predicates (`t.where("name == 'hello'")`) still raise + `NotImplementedError` via the existing `_guard_scalar_expression` guard — + intentionally out of scope (decision 3: numexpr/miniexpr cannot evaluate + `StringDType`; only the direct Python-operator comparison path is + implemented). +- Tests: `tests/ctable/test_utf8.py` gained a "Comparisons and filtering" + section — `==`/`!=` row filtering, all four ordering comparisons, null-row + exclusion (including a dedicated column-vs-column-with-nulls case), + filtering on a view (`t.head(N)[...]`), non-string-scalar and + mismatched-column-type `TypeError`s, and the `startswith`/`endswith` + pinning test. Manually verified end-to-end beyond the automated suite: + physical-length predicate padding for a logically-shorter utf8 array, + view-of-view filtering (filter on a filtered view), and that the SQL null + rule holds for `<`/`>=` in addition to `==`/`!=`. Full `tests/ctable` + (1373) and `tests/ndarray` (4385) suites pass. + **P3.d Groupby keys.** Replace the groupby rejection for utf8 keys (grep `Cannot group by variable-length` in `groupby.py`). Factorization: read the chunk as offsets+bytes and hash rows vectorized — same trick as diff --git a/src/blosc2/ctable.py b/src/blosc2/ctable.py index de5e75f25..0156859b3 100644 --- a/src/blosc2/ctable.py +++ b/src/blosc2/ctable.py @@ -191,6 +191,14 @@ def sentinel_for_arrow_type(self, pa, pa_type): _SMALL_NROWS_LIMIT = 10_000_000 _SMALL_SORT_MATERIALIZE_LIMIT = _SMALL_NROWS_LIMIT _MAX_GROWTH_ROWS = 1_048_576 +_UTF8_COMPARE_OPS = { + "eq": np.equal, + "ne": np.not_equal, + "lt": np.less, + "le": np.less_equal, + "gt": np.greater, + "ge": np.greater_equal, +} def get_null_policy() -> NullPolicy: @@ -1730,7 +1738,8 @@ def _ensure_queryable(self) -> None: if self.is_utf8: raise NotImplementedError( f"Column {self._col_name!r} is a variable-length utf8 column; " - "vectorized comparisons on utf8 columns are not supported yet." + "only comparisons (==, !=, <, <=, >, >=) are supported, not arithmetic " + "or bitwise operations." ) if self.is_varlen_scalar: raise NotImplementedError( @@ -1962,16 +1971,22 @@ def __invert__(self): return ~self._raw_col def __lt__(self, other): + if self.is_utf8: + return self._utf8_compare("lt", other) self._ensure_comparable() return self._null_aware_compare(other, self._raw_col < self._coerce_timestamp_operand(other)) def __le__(self, other): + if self.is_utf8: + return self._utf8_compare("le", other) self._ensure_comparable() return self._null_aware_compare(other, self._raw_col <= self._coerce_timestamp_operand(other)) def __eq__(self, other): if self.is_dictionary: return self._dictionary_eq(other) + if self.is_utf8: + return self._utf8_compare("eq", other) self._ensure_comparable() if self._is_nullable_bool and isinstance(other, (bool, np.bool_)): return self._null_aware_compare(other, self._raw_col == int(other)) @@ -1983,11 +1998,87 @@ def __ne__(self, other): if isinstance(result, np.ndarray): return ~result return ~np.asarray(result, dtype=bool) + if self.is_utf8: + return self._utf8_compare("ne", other) self._ensure_comparable() if self._is_nullable_bool and isinstance(other, (bool, np.bool_)): return self._null_aware_compare(other, self._raw_col == int(not other)) return self._null_aware_compare(other, self._raw_col != self._coerce_timestamp_operand(other)) + def _utf8_chunked_bool(self, fn, *, chunk_size: int = 65536) -> np.ndarray: + """Apply ``fn(chunk, start, stop)`` over this utf8 column's logical rows. + + *fn* returns a boolean array for each ``StringDType`` chunk read from + the underlying :class:`~blosc2.utf8_array.Utf8Array`. Returns a + physical-length (``_valid_rows``-length) boolean NumPy array; rows + beyond the column's logical length are left ``False``. + """ + arr = self._raw_col + n_phys = len(self._table._valid_rows) + n_logical = len(arr) + result = np.zeros(n_phys, dtype=np.bool_) + for start in range(0, n_logical, chunk_size): + stop = min(start + chunk_size, n_logical) + result[start:stop] = fn(arr[start:stop], start, stop) + return result + + def _utf8_null_pred(self): + """Physical-length blosc2.NDArray, True where the utf8 value is the + null sentinel; ``None`` when the column is not nullable.""" + spec = self._table._schema.columns_by_name[self._col_name].spec + nv = getattr(spec, "null_value", None) + if nv is None: + return None + raw = self._utf8_chunked_bool(lambda chunk, start, stop: chunk == nv) + return blosc2.asarray(raw) + + def _utf8_compare(self, op: str, other): + """Chunked-numpy comparison predicate for a utf8 column. + + Compares against a Python ``str`` scalar or another utf8 + :class:`Column` (element-wise), evaluating chunk by chunk since + numexpr/miniexpr cannot operate on ``StringDType`` arrays. Returns a + physical-length boolean ``blosc2.NDArray``, already intersected with + this column's live-row mask. A null value on either side never + satisfies any comparison (SQL ``WHERE`` semantics), matching + :meth:`_null_aware_compare` for every other column kind. + """ + if isinstance(other, Column): + if not other.is_utf8: + raise TypeError( + f"Column {self._col_name!r} is a utf8 column; it can only be compared with a " + f"str or another utf8 Column, got Column {other._col_name!r}." + ) + other_arr = other._raw_col + elif isinstance(other, str): + other_arr = None + else: + raise TypeError( + f"Column {self._col_name!r} is a utf8 column; it can only be compared with a str " + f"or another utf8 Column, got {type(other).__name__!r}." + ) + + numpy_op = _UTF8_COMPARE_OPS[op] + if other_arr is None: + raw = self._utf8_chunked_bool(lambda chunk, start, stop: numpy_op(chunk, other)) + else: + raw = self._utf8_chunked_bool(lambda chunk, start, stop: numpy_op(chunk, other_arr[start:stop])) + pred = blosc2.asarray(raw) & self._lazy_valid_rows() + + null_pred = self._utf8_null_pred() + if isinstance(other, Column): + other_null_pred = other._utf8_null_pred() + null_pred = ( + other_null_pred + if null_pred is None + else null_pred + if other_null_pred is None + else null_pred | other_null_pred + ) + if null_pred is not None: + pred = pred & ~null_pred + return pred + def _dictionary_eq(self, other): """Return a physical-slot boolean predicate for dictionary equality. @@ -2062,10 +2153,14 @@ def _dictionary_isin(self, values) -> np.ndarray: return mask def __gt__(self, other): + if self.is_utf8: + return self._utf8_compare("gt", other) self._ensure_comparable() return self._null_aware_compare(other, self._raw_col > self._coerce_timestamp_operand(other)) def __ge__(self, other): + if self.is_utf8: + return self._utf8_compare("ge", other) self._ensure_comparable() return self._null_aware_compare(other, self._raw_col >= self._coerce_timestamp_operand(other)) diff --git a/tests/ctable/test_utf8.py b/tests/ctable/test_utf8.py index dc8b6c42f..20ac340fb 100644 --- a/tests/ctable/test_utf8.py +++ b/tests/ctable/test_utf8.py @@ -394,14 +394,90 @@ def test_ctable_utf8_rename_column_persistent(tmp_path): # --------------------------------------------------------------------------- -# Unsupported operations fail clearly (lifted by later work) +# Comparisons and filtering # --------------------------------------------------------------------------- -def test_ctable_utf8_comparison_raises_clearly(): +def test_ctable_utf8_eq_filters_rows(): + t = make_table(["paris", "london", "paris", "tokyo"]) + view = t[t.name == "paris"] + assert list(view["name"][:]) == ["paris", "paris"] + assert list(view["x"][:]) == [0, 2] + + +def test_ctable_utf8_ne_filters_rows(): + t = make_table(["paris", "london", "paris", "tokyo"]) + view = t[t.name != "paris"] + assert list(view["name"][:]) == ["london", "tokyo"] + + +def test_ctable_utf8_ordering_comparisons(): + t = make_table(["paris", "london", "tokyo"]) + assert list(t[t.name < "paris"]["name"][:]) == ["london"] + assert list(t[t.name <= "paris"]["name"][:]) == ["paris", "london"] + assert list(t[t.name > "paris"]["name"][:]) == ["tokyo"] + assert list(t[t.name >= "paris"]["name"][:]) == ["paris", "tokyo"] + + +def test_ctable_utf8_comparison_excludes_null_rows(): + """SQL WHERE semantics: a null value never satisfies any comparison.""" + t = CTable(NullableRow, new_data={"name": ["paris", None, "london"], "x": [1, 2, 3]}) + assert list(t[t.name == t["name"].null_value]["name"][:]) == [] + assert list(t[t.name != "paris"]["name"][:]) == ["london"] + assert list(t[t.name < "z"]["name"][:]) == ["paris", "london"] + + +def test_ctable_utf8_column_vs_column_comparison(): + @dataclass + class TwoCols: + a: str = blosc2.field(blosc2.utf8(nullable=True)) + b: str = blosc2.field(blosc2.utf8(nullable=True)) + + t = CTable(TwoCols, new_data={"a": ["x", "y", None, "z"], "b": ["x", "z", "q", None]}) + eq = t[t.a == t.b] + assert list(eq["a"][:]) == ["x"] + ne = t[t.a != t.b] + # rows with a null on either side never satisfy != either (SQL semantics) + assert list(ne["a"][:]) == ["y"] + + +def test_ctable_utf8_comparison_on_view(): + t = make_table(["paris", "london", "paris", "tokyo", "berlin"]) + head_view = t.head(3) + filtered = head_view[head_view.name == "paris"] + assert list(filtered["name"][:]) == ["paris", "paris"] + + +def test_ctable_utf8_comparison_with_non_string_scalar_raises(): t = make_table() - with pytest.raises(NotImplementedError, match="utf8"): - t.name == "hello" # noqa: B015 + with pytest.raises(TypeError, match="utf8"): + t.name == 42 # noqa: B015 + with pytest.raises(TypeError, match="utf8"): + t.name < 3.14 # noqa: B015 + + +def test_ctable_utf8_comparison_with_mismatched_column_type_raises(): + @dataclass + class Mixed: + name: str = blosc2.field(blosc2.utf8()) + other: str = blosc2.field(blosc2.vlstring()) + + t = CTable(Mixed, new_data={"name": ["a"], "other": ["b"]}) + with pytest.raises(TypeError, match="utf8"): + t.name == t.other # noqa: B015 + + +def test_ctable_utf8_startswith_endswith(): + t = make_table(["hello", "help", "world"]) + started = blosc2.startswith(t.name, "hel").compute() + assert list(np.asarray(started)[:]) == [True, True, False] + ended = blosc2.endswith(t.name, "lo").compute() + assert list(np.asarray(ended)[:]) == [True, False, False] + + +# --------------------------------------------------------------------------- +# Unsupported operations fail clearly (lifted by later work) +# --------------------------------------------------------------------------- def test_ctable_utf8_where_expression_raises_clearly(): From c8b6ba514c1875d88f735636396611e9d84ce139 Mon Sep 17 00:00:00 2001 From: Francesc Alted Date: Thu, 16 Jul 2026 17:58:47 +0200 Subject: [PATCH 05/20] Allow utf8 columns as group_by() keys (correctness-first, benchmark gap recorded) CTableGroupBy.__init__ no longer rejects utf8 key columns (vlstring/ vlbytes/struct/object/list keys are still rejected). Two small supporting fixes make the existing generic np.unique-based factorization path correct for utf8 keys: - _read_key_chunk pads a chunk read past a utf8 column's logical length with "" (mirroring the P3.a iter_chunks fix) since Utf8Array is sized to the logical row count, not the physical valid_rows capacity. - _factorize_keys casts StringDType key arrays to object dtype before packing into the structured array used for multi-key np.unique, since NumPy structured dtypes reject StringDType fields outright. Everything else needed no changes: StringDType's .kind is "T", so it already falls through the existing fixed-width-string fast path into plain np.unique(); null-sentinel handling, key-column spec propagation to the result table, Python-scalar coercion, sort ordering, and every Cython fast path (which all bail on dtype=None) all worked unmodified. Benchmarked against dictionary-key groupby (bench/ctable/ bench_groupby_keys.py, 1e7 rows, 5-city low-cardinality keys): utf8 is ~17x slower than dict for a single key and ~50x for two keys, well past the 3x target. Profiling isolated the cause to Utf8Array's per-row Python decode loop (a P3.a artifact shared by every bulk utf8 read, not specific to groupby) -- the per-row loop overhead dominates, not the decode call itself, so no small fix closes the gap. A real fix needs a new vectorized byte-hash factorization that never decodes to str for the N-row hot path; that's sized like its own follow-up item, so per the benchmark-gate rule this lands the correct, slower fallback and records the gap instead of merging an unproven fast path. Co-Authored-By: Claude Sonnet 5 --- bench/ctable/bench_groupby_keys.py | 11 +++- plans/enhancing-ctable-phase3.md | 85 ++++++++++++++++++++++++++++ src/blosc2/groupby.py | 25 +++++++-- tests/ctable/test_utf8.py | 90 ++++++++++++++++++++++++++++-- 4 files changed, 200 insertions(+), 11 deletions(-) diff --git a/bench/ctable/bench_groupby_keys.py b/bench/ctable/bench_groupby_keys.py index 7e82f78b4..5992817fb 100644 --- a/bench/ctable/bench_groupby_keys.py +++ b/bench/ctable/bench_groupby_keys.py @@ -32,13 +32,20 @@ class Row: ikey: int = blosc2.field(blosc2.int64()) skey: str = blosc2.field(blosc2.string(max_length=8)) dkey: str = blosc2.field(blosc2.dictionary()) + ukey: str = blosc2.field(blosc2.utf8()) val: float = blosc2.field(blosc2.float64()) print(f"building table ({N:.0e} rows)...", flush=True) t = CTable(Row) t.extend( - {"ikey": int_keys, "skey": str_keys, "dkey": [str(s) for s in str_keys], "val": float_vals}, + { + "ikey": int_keys, + "skey": str_keys, + "dkey": [str(s) for s in str_keys], + "ukey": [str(s) for s in str_keys], + "val": float_vals, + }, validate=False, ) @@ -56,7 +63,9 @@ def bench(label, fn, reps=3): bench("int key, mean", lambda: t.group_by("ikey").agg({"val": "mean"})) bench("string key, sum", lambda: t.group_by("skey").sum("val")) bench("dict key, sum", lambda: t.group_by("dkey").sum("val")) +bench("utf8 key, sum", lambda: t.group_by("ukey").sum("val")) bench("two keys (int+dict), sum", lambda: t.group_by(["ikey", "dkey"]).sum("val")) +bench("two keys (int+utf8), sum", lambda: t.group_by(["ikey", "ukey"]).sum("val")) try: import pandas as pd diff --git a/plans/enhancing-ctable-phase3.md b/plans/enhancing-ctable-phase3.md index 5d253cb52..fc8990df7 100644 --- a/plans/enhancing-ctable-phase3.md +++ b/plans/enhancing-ctable-phase3.md @@ -312,6 +312,91 @@ the dictionary path for 1e7 rows/low cardinality. If the vectorized hash proves hard, the honest fallback is `np.unique` on the StringDType chunk (correct, slower) — land correctness first, speed second. +**P3.d implementation notes (landed 2026-07-16, branch `enhancing-ctable3`):** + +- What landed: **correctness only, on the honest `np.unique` fallback** — the + benchmark gate below rejected the vectorized-hash path, so it was not + built. Three small changes in `groupby.py`: + 1. `CTableGroupBy.__init__`'s rejection now excludes utf8 columns (`table + ._is_list_column(col_info) or (table._is_varlen_scalar_column(col_info) + and not table._is_utf8_column(col_info))`) — vlstring/vlbytes/struct/ + object/list keys are still rejected. + 2. `_read_key_chunk` gained a utf8 branch that pads a chunk read past the + column's logical length with `""`, mirroring the P3.a `iter_chunks` fix + — `Utf8Array` is sized to the logical row count, not the physical + `valid_rows` capacity, and a chunk boundary can run past it; those rows + are provably never live (a row can't be marked valid without every + column, including this one, having been written), so the pad value is + never read as a live group. + 3. `_factorize_keys`'s multi-key branch casts any `StringDType` key array to + `object` dtype before packing into the structured array used for + `np.unique` — NumPy's structured dtypes reject `StringDType` fields + outright (`TypeError: StringDType is not currently supported for + structured dtype fields`, confirmed empirically), so this is required + for correctness, not an optimization. +- **Nothing else needed changing.** This was verified empirically before + writing code, not assumed: `numpy.dtypes.StringDType`'s `.kind` is `"T"` + (not `"U"`/`"S"`), so it never matches `_factorize_fixed_width_str`'s + fixed-width dispatch and the single-key path already falls through to the + correct `np.unique(arr, return_inverse=True)` for free. `_null_mask` already + worked unmodified (`values == null_value` on a `StringDType` array is + correct). `_result_spec_for_key` deep-copies the source `Utf8Spec` + unmodified, so a groupby result's key column is itself a utf8 column + (`Column.is_utf8` true). `_python_type_for_spec`/`_python_scalar` already + produce plain `str` for utf8 (indexing a `StringDType` array yields + `isinstance(x, str)`, not `np.generic`). The Python-level `_final_rows` sort + (`_sortable_key_part`) and all Cython fast paths (which all bail via + `key_dtype is None -> return None`) needed no changes either. Verified with + a manual smoke test covering: single-key sum, `dropna=False` with a null + sentinel group, `sort=True` (relies on `np.unique`'s inherently sorted + output for the fast single-chunk case and the generic Python sort + otherwise), a two-key `[utf8, int32]` composite groupby, and a 300k-row + multi-chunk merge — all correct. +- **Benchmark gate: FAILED, and per the plan's explicit instruction the fast + path was not built.** Extended `bench/ctable/bench_groupby_keys.py` with a + `ukey` (utf8) column alongside the existing `dkey` (dictionary) column, + 1e7 rows, 5-city low-cardinality keys, Apple-silicon dev box: + + | key type | time | vs. dict key | + |----------------------|-----------|--------------| + | dict key, sum | 196.0 ms | 1.0x | + | **utf8 key, sum** | **3304.2 ms** | **16.9x** | + | two keys (int+dict) | 236.8 ms | 1.0x | + | **two keys (int+utf8)** | **11825.3 ms** | **49.9x** | + + Target was ≤3x; actual is ~17x (single key) / ~50x (two keys) — nowhere + close. **Root cause, isolated with `cProfile`** (not guessed): 62% of + single-key wall time is `Utf8Array._read_persisted_span`, specifically its + per-row Python loop (`for i in range(n): out[i] = blob[...].decode("utf-8")`) + — 1,998,848 individual `bytes.decode()` calls at 2e6 rows in the profile + run. This is a P3.a artifact, not something specific to groupby: every bulk + utf8 read pays this cost (comparisons, iteration, sort will too). A + micro-benchmark of alternative row-decode strategies (list-comprehension + instead of indexed assignment; decode the whole chunk blob once and slice + by vectorized UTF-8-continuation-byte character offsets instead of + redecoding per row) found only ~10% improvement — the per-row **Python loop + overhead** dominates, not the decode call itself, so no drop-in fix closes + the gap. A real fix needs a genuinely vectorized/C-level factorization that + never decodes N rows: group physical rows by raw byte length first (cheap, + vectorized), gather each length-group's fixed-width byte span with one + fancy-index gather (`data[start[:,None] + arange(L)]`), hash those raw + bytes with the same collision-checked trick `_factorize_fixed_width_str` + already uses for fixed-width Unicode, and decode to `StringDType` **only + the D unique group values**, never the N rows. This is a legitimate, + bounded idea (matching the plan's "vectorized loop over the (few) distinct + byte-lengths" suggestion) but is a substantial new algorithm — cross-length- + group code merging, a byte-hash variant of the existing mixer, and the + usual collision/verify pass — sized like its own follow-up item, not a + drop-in fix. **Deliberately not attempted here**, per the cross-cutting + rule against merging a "fast" path the recorded benchmark shows is not + fast; land correctness, record the gap, stop. +- Tests: `tests/ctable/test_utf8.py` gained a "Groupby keys" section — sum, + size with and without `dropna`, `sort=True`, confirming the result's key + column is itself utf8, a multi-key `[utf8, int]` composite, a 200k-row + multi-chunk-merge case, and a regression test that vlstring keys are still + rejected (only utf8 was carved out). Full `tests/ctable` (1379) and + `tests/ndarray` (4385) suites pass. + **P3.e Sort.** Lift the sort rejection (grep the `sort_by` varlen guard in `ctable.py`): `sort_by` on utf8 uses `np.argsort` on the StringDType array (chunked merge if the existing sort machinery is chunked — study diff --git a/src/blosc2/groupby.py b/src/blosc2/groupby.py index cce771e85..3186be858 100644 --- a/src/blosc2/groupby.py +++ b/src/blosc2/groupby.py @@ -136,7 +136,9 @@ def __init__( f"Cannot group by ndarray column {name!r} with per-row shape {col_info.spec.item_shape}. " "Materialize a scalar generated column first, e.g. embedding_norm or embedding_max." ) - if table._is_list_column(col_info) or table._is_varlen_scalar_column(col_info): + if table._is_list_column(col_info) or ( + table._is_varlen_scalar_column(col_info) and not table._is_utf8_column(col_info) + ): raise TypeError(f"Cannot group by variable-length/list column {name!r} in Phase 1") def size(self, *, urlpath: str | None = None): @@ -1512,6 +1514,17 @@ def _read_key_chunk(self, name: str, start: int, stop: int) -> np.ndarray: col_info = self.table._schema.columns_by_name[name] if self.table._is_dictionary_column(col_info): return np.asarray(self.table._cols[name].codes[start:stop], dtype=np.int32) + if self.table._is_utf8_column(col_info): + # Utf8Array is sized to the logical row count, not the physical + # valid_rows capacity, so a chunk boundary can run past its end. + # Rows beyond it are never live (the row can't have been written + # without this column), so the padding value is never read live. + col = self.table._cols[name] + n = len(col) + values = np.asarray(col[start : min(stop, n)]) + if stop > n: + values = np.concatenate([values, np.full(stop - max(start, n), "", dtype=values.dtype)]) + return values return np.asarray(self.table._cols[name][start:stop]) def _factorize_keys( @@ -1524,9 +1537,13 @@ def _factorize_keys( unique, inverse = np.unique(arr, return_inverse=True) return unique, inverse - dtype = [(f"k{i}", arr.dtype) for i, arr in enumerate(keys_live)] - packed = np.empty(len(keys_live[0]), dtype=dtype) - for i, arr in enumerate(keys_live): + # StringDType (utf8 columns) cannot be a structured-array field, so pack + # those columns as plain Python-object arrays instead; np.unique still + # dedupes them correctly by str equality. + pack_arrs = [arr.astype(object) if arr.dtype.kind == "T" else arr for arr in keys_live] + dtype = [(f"k{i}", arr.dtype) for i, arr in enumerate(pack_arrs)] + packed = np.empty(len(pack_arrs[0]), dtype=dtype) + for i, arr in enumerate(pack_arrs): packed[f"k{i}"] = arr unique, inverse = np.unique(packed, return_inverse=True) return unique, inverse diff --git a/tests/ctable/test_utf8.py b/tests/ctable/test_utf8.py index 20ac340fb..2c61948de 100644 --- a/tests/ctable/test_utf8.py +++ b/tests/ctable/test_utf8.py @@ -476,22 +476,100 @@ def test_ctable_utf8_startswith_endswith(): # --------------------------------------------------------------------------- -# Unsupported operations fail clearly (lifted by later work) +# Groupby keys # --------------------------------------------------------------------------- -def test_ctable_utf8_where_expression_raises_clearly(): - t = make_table() - with pytest.raises(NotImplementedError, match="utf8"): - t.where("name == 'hello'") +def test_ctable_utf8_groupby_sum(): + t = make_table(["a", "b", "a", "b", "a"]) + t.x[:] = [1.0, 2.0, 3.0, 4.0, 5.0] + g = t.group_by("name").sum("x") + rows = dict(zip(g["name"][:].tolist(), g["x_sum"][:].tolist(), strict=False)) + assert rows == {"a": 9.0, "b": 6.0} + + +def test_ctable_utf8_groupby_size_and_dropna(): + t = CTable(NullableRow, new_data={"name": ["a", "b", "a", None, "b", "a"], "x": range(6)}) + g = t.group_by("name").size() # dropna=True by default + counts = dict(zip(g["name"][:].tolist(), g["size"][:].tolist(), strict=False)) + assert counts == {"a": 3, "b": 2} + + g_all = t.group_by("name", dropna=False).size() + nv = t["name"].null_value + counts_all = dict(zip(g_all["name"][:].tolist(), g_all["size"][:].tolist(), strict=False)) + assert counts_all == {"a": 3, "b": 2, nv: 1} + + +def test_ctable_utf8_groupby_sort(): + t = make_table(["gamma", "alpha", "beta", "alpha"]) + g = t.group_by("name", sort=True).size() + assert list(g["name"][:]) == ["alpha", "beta", "gamma"] -def test_ctable_utf8_groupby_raises_clearly(): +def test_ctable_utf8_groupby_result_is_utf8_column(): t = make_table(["a", "b", "a"]) + g = t.group_by("name").size() + assert g["name"].is_utf8 + + +def test_ctable_utf8_groupby_multi_key_with_int(): + @dataclass + class MultiRow: + cat: str = blosc2.field(blosc2.utf8()) + grp: int = blosc2.field(blosc2.int32()) + x: float = blosc2.field(blosc2.float64()) + + t = CTable( + MultiRow, + new_data={ + "cat": ["a", "b", "a", "b", "a"], + "grp": [1, 1, 1, 2, 2], + "x": [1.0, 2.0, 3.0, 4.0, 5.0], + }, + ) + g = t.group_by(["cat", "grp"]).sum("x") + rows = sorted(zip(g["cat"][:].tolist(), g["grp"][:].tolist(), g["x_sum"][:].tolist(), strict=False)) + assert rows == [("a", 1, 4.0), ("a", 2, 5.0), ("b", 1, 2.0), ("b", 2, 4.0)] + + +def test_ctable_utf8_groupby_multi_chunk_merge(): + """A key set spanning many physical chunks exercises the merge path, not + just a single-chunk factorization.""" + import random + + rng = random.Random(0) + words = ["alpha", "beta", "gamma", "delta", "epsilon"] + n = 200_000 + names = [rng.choice(words) for _ in range(n)] + xs = [float(i % 7) for i in range(n)] + t = CTable(Row, new_data={"name": names, "x": xs}, expected_size=n) + g = t.group_by("name").sum("x") + assert g.nrows == len(words) + assert abs(sum(g["x_sum"][:].tolist()) - sum(xs)) < 1e-6 + + +def test_ctable_utf8_groupby_still_rejects_vlstring(): + @dataclass + class VlRow: + name: str = blosc2.field(blosc2.vlstring()) + x: int = blosc2.field(blosc2.int64()) + + t = CTable(VlRow, new_data={"name": ["a", "b"], "x": [1, 2]}) with pytest.raises(TypeError, match="variable-length"): t.group_by("name").sum("x") +# --------------------------------------------------------------------------- +# Unsupported operations fail clearly (lifted by later work) +# --------------------------------------------------------------------------- + + +def test_ctable_utf8_where_expression_raises_clearly(): + t = make_table() + with pytest.raises(NotImplementedError, match="utf8"): + t.where("name == 'hello'") + + def test_ctable_utf8_sort_raises_clearly(): t = make_table() with pytest.raises(TypeError, match="utf8"): From a640b32e70a8353fd23b8a96665977da930fff3d Mon Sep 17 00:00:00 2001 From: Francesc Alted Date: Thu, 16 Jul 2026 18:07:14 +0200 Subject: [PATCH 06/20] Allow sorting by utf8 columns sort_by() now accepts utf8 sort keys. The rejection in _normalise_sort_keys turned out to be unnecessary rather than merely lifted: a utf8 column's dtype already resolves to StringDType (not None), so the existing dtype-is-None rejection branch already skipped it, and np.issubdtype(StringDType(), np.complexfloating) returns False cleanly. Two real fixes were needed: - _build_lex_keys's descending-sort rank-inversion check widened from "USO" to "USOT" (StringDType's .kind is "T") -- strings can't be negated with unary minus for a descending lexsort key. Ascending sort and the null-indicator key needed no changes: np.lexsort/ np.argsort and == already work natively on StringDType arrays. - _sort_by_inplace and _sorted_copy_from_positions gained a utf8 branch that rebuilds the column via Utf8Array.extend() instead of bulk slice-assignment, mirroring the existing list-column branch, since Utf8Array has no bulk __setitem__. Found and left alone: sorting a table containing a vlstring/vlbytes/ struct/object column already raised before this change (even when sorting by an unrelated key), because the generic sort-copy fallback assumes bulk slice-assignment that _ScalarVarLenArray doesn't support. This predates the utf8 work and is out of scope for a utf8-only phase. Manually verified the plan's three Goal-section lines run correctly end-to-end: t[t.name == "Paris"], t.group_by("name").sum("x"), and t.sort_by("name"). Full test suite (7477 tests) passes. Co-Authored-By: Claude Sonnet 5 --- plans/enhancing-ctable-phase3.md | 68 +++++++++++++++++++++++- src/blosc2/ctable.py | 22 ++++---- tests/ctable/test_utf8.py | 89 +++++++++++++++++++++++++++++--- 3 files changed, 163 insertions(+), 16 deletions(-) diff --git a/plans/enhancing-ctable-phase3.md b/plans/enhancing-ctable-phase3.md index fc8990df7..2745057df 100644 --- a/plans/enhancing-ctable-phase3.md +++ b/plans/enhancing-ctable-phase3.md @@ -1,6 +1,8 @@ # Enhancing CTable, phase 3: variable-length strings and the null-mask question -**Status:** NOT STARTED (plan written 2026-07-16). Successor to +**Status:** P3 (P3.a–P3.e) landed 2026-07-16 on branch `enhancing-ctable3`; +see each sub-item's implementation notes below. P5 remains parked (no unpark +criterion fired). Successor to `plans/enhancing-ctable-phase2.md` (phase 2, landed on branch `enhancing-ctable2`: the pandas `engine=blosc2.jit` fix + `Series.map`, `CTable.assign()` + unbound `blosc2.col()`, the lazily-sorted-view @@ -403,6 +405,70 @@ proves hard, the honest fallback is `np.unique` on the StringDType chunk `sort_by`'s path first). Null ordering must match the existing convention (nulls last; grep `test_sort_nulls_last`). +**P3.e implementation notes (landed 2026-07-16, branch `enhancing-ctable3`):** + +- What landed: `sort_by` now accepts utf8 sort keys. Three changes in + `ctable.py`: + 1. Removed the utf8-specific rejection from `_normalise_sort_keys` + (introduced defensively in P3.a). It turned out to be unnecessary + rather than merely lifted: `_col_dtype()` for a utf8 column already + returns `numpy.dtypes.StringDType()` (not `None`, because + `Utf8Array.dtype` reports it — a P3.a design choice), so the existing + `dtype is None` branch that rejects list/vlstring/vlbytes columns + already skips utf8 columns for free, and + `np.issubdtype(StringDType(), np.complexfloating)` returns `False` + cleanly (verified empirically) rather than raising. Removing the guard + did require restoring a `cc`/`col_info` reference the guard's insertion + had shadowed in P3.a — fixed by reusing the already-in-scope `col_info`. + 2. `_build_lex_keys`'s descending-sort rank-inversion check + (`raw.dtype.kind in "USO"`, used because strings can't be negated with + unary minus) widened to `"USOT"` — `StringDType`'s `.kind` is `"T"`. + Ascending sort needed no change: `np.lexsort`/`np.argsort` already + accept `StringDType` arrays directly (verified empirically), and the + null-indicator-key logic (`raw == nv` for a non-float sentinel) was + already dtype-generic. + 3. `_sort_by_inplace` and `_sorted_copy_from_positions` gained a utf8 + branch that rebuilds the column via `Utf8Array.extend()` instead of + bulk slice-assignment (`arr[:n] = arr[sorted_pos]`), mirroring the + existing list-column branch's `ListArray.extend()` pattern. +- **Found and worked around a pre-existing, unrelated bug while verifying + this**: `arr[:n] = arr[sorted_pos]` — the generic fallback both sort-copy + methods used for every column that isn't `list`/`dictionary` — silently + assumed every such column supports NumPy-style bulk slice assignment. + `_ScalarVarLenArray.__setitem__` (`vlstring`/`vlbytes`/`struct`/`object`, + unrelated to this phase) only accepts a single `int` index, so + `sort_by()` on *any* table containing a vlstring/vlbytes/struct/object + column — even sorted by an unrelated int/float key — already raised + `TypeError: Expected str for vlstring column, got 'list'` before this + change, confirmed on the pre-P3 codebase. **Not fixed here** — out of + scope for a utf8-only phase, and the acceptance criterion is + "`vlstring`/`string` behavior byte-for-byte unchanged," not "fixed." Only + `Utf8Array` got the same treatment `ListArray` already has, since utf8 + sortability is what this item asks for; every utf8 column in a sorted + table — key or bystander — must survive the rewrite, not only the one + named in `sort_by(...)`. (Also found and left alone: + `_sorted_small_copy_from_live_positions` has the identical bug pattern + but zero callers anywhere in the repo — genuinely dead code, not worth + touching.) +- Tests: `tests/ctable/test_utf8.py` gained a "Sort" section — ascending, + descending, nulls-last in both directions, `view=True`, `inplace=True`, a + multi-key `[int, utf8]` sort with a non-key utf8 "bystander" column + verifying it's reordered too (row alignment, not just the key), the same + bystander check under `inplace=True`, and a non-ASCII ordering case. + Manually verified beyond the automated suite: the exact three Goal-section + code lines from the top of this plan file run correctly end-to-end + (`t[t.name == "Paris"]`, `t.group_by("name").sum("x")`, `t.sort_by("name")`) + — this is the P3 acceptance criterion, confirmed directly, not inferred + from passing unit tests. Full `tests/` (7477 tests, whole repo, not just + `tests/ctable`/`tests/ndarray`) passes. +- **P3 acceptance for the item overall, checked against the plan's own + criteria**: the three Goal-section lines work (confirmed above); the P3.d + groupby benchmark number is recorded (16.9x/49.9x vs. the 3x target, gap + documented with root cause); `vlstring`/`string` behavior is byte-for-byte + unchanged (no code path touched by P3 alters their behavior; the one + pre-existing vlstring sort bug found above predates this phase and was + left as found, not touched). + **Out of scope for P3:** making `utf8` the `str`-field default; `.str` accessor namespaces; regex operations beyond what `np.strings` gives; string interning/dedup (that's `dictionary()`). diff --git a/src/blosc2/ctable.py b/src/blosc2/ctable.py index 0156859b3..ee355f112 100644 --- a/src/blosc2/ctable.py +++ b/src/blosc2/ctable.py @@ -10990,20 +10990,15 @@ def _normalise_sort_keys( f"Cannot sort by ndarray column {name!r} with per-row shape {col_info.spec.item_shape}. " "Materialize a scalar generated column first, e.g. embedding_norm or embedding_max." ) - cc = self._schema.columns_by_name.get(name) - if cc is not None and self._is_utf8_column(cc): - raise TypeError( - f"Column {name!r} is a variable-length utf8 column; sorting by it is not supported yet." - ) dtype = self._col_dtype(name) if dtype is None: - if cc is not None and isinstance( - cc.spec, (VLStringSpec, VLBytesSpec, StructSpec, ObjectSpec) + if col_info is not None and isinstance( + col_info.spec, (VLStringSpec, VLBytesSpec, StructSpec, ObjectSpec) ): raise TypeError( f"Column {name!r} is a varlen scalar column and does not support sort ordering." ) - if cc is not None and self._is_dictionary_column(cc): + if col_info is not None and self._is_dictionary_column(col_info): pass # dictionary columns: sorting supported (decoded strings) else: raise TypeError( @@ -11182,7 +11177,7 @@ def _build_lex_keys( # Value key if not asc: - if raw.dtype.kind in "USO": + if raw.dtype.kind in "USOT": # strings can't be negated — invert via rank rank = np.argsort(np.argsort(raw, kind="stable"), kind="stable") lex_keys.append((n - 1 - rank).astype(np.intp)) @@ -11530,6 +11525,12 @@ def _sort_by_inplace(self, sorted_pos: np.ndarray, n: int) -> None: elif self._is_dictionary_column(col): sorted_codes = arr.codes[sorted_pos] arr.codes[:n] = sorted_codes + elif self._is_utf8_column(col): + # Utf8Array has no bulk slice-assignment; rebuild it in sorted order. + new_arr = Utf8Array(col.spec) + new_arr.extend(arr[sorted_pos]) + new_arr.flush() + self._cols[col.name] = new_arr else: arr[:n] = arr[sorted_pos] self._valid_rows[:n] = True @@ -11553,6 +11554,9 @@ def _sorted_copy_from_positions(self, sorted_pos: np.ndarray, n: int) -> CTable: result._cols[col_name].encode(v) sorted_codes = arr.codes[sorted_pos] result._cols[col_name].codes[:n] = sorted_codes + elif self._is_utf8_column(col): + result._cols[col_name].extend(arr[sorted_pos]) + result._cols[col_name].flush() else: result._cols[col_name][:n] = arr[sorted_pos] result._valid_rows[:n] = True diff --git a/tests/ctable/test_utf8.py b/tests/ctable/test_utf8.py index 2c61948de..4c86fa994 100644 --- a/tests/ctable/test_utf8.py +++ b/tests/ctable/test_utf8.py @@ -559,6 +559,89 @@ class VlRow: t.group_by("name").sum("x") +# --------------------------------------------------------------------------- +# Sort +# --------------------------------------------------------------------------- + + +def test_ctable_utf8_sort_ascending(): + t = make_table(["banana", "apple", "cherry"]) + s = t.sort_by("name") + assert list(s["name"][:]) == ["apple", "banana", "cherry"] + # row alignment: the "x" companion column follows its row, not its old position + assert list(s["x"][:]) == [1, 0, 2] + + +def test_ctable_utf8_sort_descending(): + t = make_table(["banana", "apple", "cherry"]) + s = t.sort_by("name", ascending=False) + assert list(s["name"][:]) == ["cherry", "banana", "apple"] + + +def test_ctable_utf8_sort_nulls_last_both_directions(): + t = CTable(NullableRow, new_data={"name": ["banana", None, "apple", "cherry"], "x": [1, 2, 3, 4]}) + nv = t["name"].null_value + asc = t.sort_by("name") + assert list(asc["name"][:]) == ["apple", "banana", "cherry", nv] + desc = t.sort_by("name", ascending=False) + assert list(desc["name"][:]) == ["cherry", "banana", "apple", nv] + + +def test_ctable_utf8_sort_view(): + t = make_table(["banana", "apple", "cherry"]) + view = t.sort_by("name", view=True) + assert list(view["name"][:]) == ["apple", "banana", "cherry"] + assert view.base is not None + + +def test_ctable_utf8_sort_inplace(): + t = make_table(["b", "a", "c"]) + result = t.sort_by("name", inplace=True) + assert result is t + assert list(t["name"][:]) == ["a", "b", "c"] + + +def test_ctable_utf8_sort_multi_key_with_bystander_utf8_column(): + """A non-key utf8 column in the same table must be reordered along with + the sort, not just the sort key itself.""" + + @dataclass + class MultiRow: + grp: int = blosc2.field(blosc2.int32()) + name: str = blosc2.field(blosc2.utf8()) + note: str = blosc2.field(blosc2.utf8()) + + t = CTable( + MultiRow, + new_data={ + "grp": [1, 1, 2, 2], + "name": ["b", "a", "d", "c"], + "note": ["n-b", "n-a", "n-d", "n-c"], + }, + ) + s = t.sort_by(["grp", "name"]) + rows = list(zip(s["grp"][:].tolist(), s["name"][:].tolist(), s["note"][:].tolist(), strict=True)) + assert rows == [(1, "a", "n-a"), (1, "b", "n-b"), (2, "c", "n-c"), (2, "d", "n-d")] + + +def test_ctable_utf8_sort_inplace_bystander_column(): + @dataclass + class TwoCols: + name: str = blosc2.field(blosc2.utf8()) + note: str = blosc2.field(blosc2.utf8()) + + t = CTable(TwoCols, new_data={"name": ["b", "a", "c"], "note": ["n-b", "n-a", "n-c"]}) + t.sort_by("name", inplace=True) + assert list(t["name"][:]) == ["a", "b", "c"] + assert list(t["note"][:]) == ["n-a", "n-b", "n-c"] + + +def test_ctable_utf8_sort_non_ascii(): + t = make_table(["café", "日本語のテキスト", "banana"]) + s = t.sort_by("name") + assert list(s["name"][:]) == sorted(["café", "日本語のテキスト", "banana"]) + + # --------------------------------------------------------------------------- # Unsupported operations fail clearly (lifted by later work) # --------------------------------------------------------------------------- @@ -570,12 +653,6 @@ def test_ctable_utf8_where_expression_raises_clearly(): t.where("name == 'hello'") -def test_ctable_utf8_sort_raises_clearly(): - t = make_table() - with pytest.raises(TypeError, match="utf8"): - t.sort_by("name") - - def test_ctable_utf8_arrow_export_large_string(): pa = pytest.importorskip("pyarrow") t = make_table() From 0cd1ca78c6707da97c2f65d8f46c18f3b27bd916 Mon Sep 17 00:00:00 2001 From: Francesc Alted Date: Thu, 16 Jul 2026 23:21:26 +0200 Subject: [PATCH 07/20] Post-review fixes for utf8 columns: persistence of inplace sort/compact, fused null masks, buffer-level Arrow export MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The critical fix: sort_by(inplace=True) and compact() on a file-backed table rebuilt utf8 columns as fresh in-memory Utf8Arrays and only rebound the column reference, so the rewritten rows never reached the store — after close/reopen the column was corrupted and misaligned with its on-disk-sorted siblings. Utf8Array.set_all() now bulk-rewrites through the existing backing offsets/data arrays; compact() also gathers live rows with one clustered fancy-index read instead of two chunk reads per row. Regression tests cover both paths across .b2z/.b2d reopens. Also: - utf8 comparisons compute the null-sentinel mask inside the same chunk pass as the predicate (was: up to four full column scans per nullable filter); comparison dunders pass numpy ufuncs directly instead of string tags. ~2x on a 1M-row nullable filter (419 -> 203 ms). - Arrow export of dense root tables builds pa.LargeStringArray straight from the offsets/bytes buffers (Utf8Array.arrow_slice) with the null mask matched on raw bytes — no per-row decode, no tolist(), no re-encode (3030 -> 1730 ms on 1M rows). Views and tables with deleted rows use a materializing fallback that reuses Column.null_value, _null_mask_for, and _pa_type_from_spec instead of inlining them. - Utf8Array.__setitem__ shifts the raw tail bytes and adds a scalar delta to the tail offsets instead of decoding and re-encoding every following row (21 ms to overwrite row 100 of a 1M-row column). - Utf8Spec storage dispatch imports Utf8Spec once at ctable_storage module top (was: six local imports). Co-Authored-By: Claude Fable 5 --- plans/enhancing-ctable-phase3.md | 52 ++++++++++++++++ src/blosc2/ctable.py | 101 +++++++++++++------------------ src/blosc2/ctable_storage.py | 25 ++------ src/blosc2/utf8_array.py | 75 ++++++++++++++++++++++- tests/ctable/test_utf8.py | 83 +++++++++++++++++++++++++ 5 files changed, 256 insertions(+), 80 deletions(-) diff --git a/plans/enhancing-ctable-phase3.md b/plans/enhancing-ctable-phase3.md index 2745057df..9f0831ba6 100644 --- a/plans/enhancing-ctable-phase3.md +++ b/plans/enhancing-ctable-phase3.md @@ -399,6 +399,7 @@ proves hard, the honest fallback is `np.unique` on the StringDType chunk rejected (only utf8 was carved out). Full `tests/ctable` (1379) and `tests/ndarray` (4385) suites pass. + **P3.e Sort.** Lift the sort rejection (grep the `sort_by` varlen guard in `ctable.py`): `sort_by` on utf8 uses `np.argsort` on the StringDType array (chunked merge if the existing sort machinery is chunked — study @@ -469,6 +470,57 @@ proves hard, the honest fallback is `np.unique` on the StringDType chunk pre-existing vlstring sort bug found above predates this phase and was left as found, not touched). +**Post-review fixes (landed 2026-07-16, after a code review of the P3 +branch):** + +- **Correctness (data corruption, found by review, missed by the suite):** + `sort_by(inplace=True)` and `compact()` on a *file-backed* table rebuilt + utf8 columns as fresh in-memory `Utf8Array`s and only rebound + `self._cols[name]` — the store never saw the rewritten rows, so after + close/reopen the utf8 column was corrupted/misaligned with its + on-disk-sorted siblings (reproduced: reopen raised `IndexError` on read). + All utf8 sort/compact tests were in-memory only, which is why the suite + was green. Fix: new `Utf8Array.set_all(values)` bulk-rewrites through the + *existing* backing offsets/data NDArrays (persistence preserved), used by + both call sites; `compact()` also now gathers via the clustered + fancy-index read instead of one scalar `__getitem__` (two chunk reads) per + row. Regression tests: `test_ctable_utf8_sort_inplace_persists_after_reopen` + and `test_ctable_utf8_compact_persists_after_reopen`, both parametrized + over `.b2z`/`.b2d`, verified to fail on the pre-fix code. +- **Comparison predicates:** the sentinel-null mask is now computed inside + the same chunk pass as the comparison (was: a second full + decompress+decode pass per operand via `_utf8_null_pred`, i.e. 2x–4x the + I/O per nullable filter); the `_UTF8_COMPARE_OPS` string-tag dict is gone + (dunders pass the numpy ufunc directly). ~2x measured on a 1M-row + nullable filter (419 → 203 ms). +- **Arrow export:** dense root tables now export utf8 batches straight from + the offsets/bytes buffers via `Utf8Array.arrow_slice()` + (`pa.LargeStringArray.from_buffers`, sentinel-null mask matched on raw + bytes) — no per-row decode, no `.tolist()`, no re-encode. Views/deleted + tables use the materializing fallback, which now reuses + `col.null_value`/`col._null_mask_for`/`_pa_type_from_spec` instead of + inlining them. ~1.75x measured on a 1M-row export (3030 → 1730 ms; the + rest is the pre-existing 2048-row `iter_arrow_batches` batching, which + re-decompresses each storage chunk many times — pre-existing, not + utf8-specific). Tests: view/deleted-rows export and pending-rows export. +- **`Utf8Array.__setitem__`:** the O(n−i) tail move now shifts raw bytes and + adds a scalar delta to the tail offsets instead of decoding and + re-encoding every following row (21 ms to overwrite row 100 of 1M). + Test: grow/shrink/equal/empty replacements persisted across reopen. +- **Cleanup:** `Utf8Spec` imported once at `ctable_storage.py` module top + (was: six local imports); `FileTableStorage.create_varlen_scalar_column` + hoists the column key. +- **Review finding rejected on inspection:** removing `Utf8Spec.__init__`'s + inline `null_value must be str` check (flagged as duplicating + `_validate_null_value_for_spec`) would open a validation hole — + `_resolve_nullable_specs` *skips* specs whose `null_value` is already set, + so the inline check is the only validation for explicit sentinels. Left + as is. Also left as recorded: the `is_varlen_scalar`-includes-utf8 + predicate design (deliberate, documented in the P3.a notes) and the + `_read_persisted_span` per-row decode loop (root cause of the P3.d + benchmark gap, already documented above as its own follow-up). +- Full `tests/` suite (7,484) passes after the fixes. + **Out of scope for P3:** making `utf8` the `str`-field default; `.str` accessor namespaces; regex operations beyond what `np.strings` gives; string interning/dedup (that's `dictionary()`). diff --git a/src/blosc2/ctable.py b/src/blosc2/ctable.py index ee355f112..57b35984c 100644 --- a/src/blosc2/ctable.py +++ b/src/blosc2/ctable.py @@ -44,7 +44,6 @@ from blosc2.info import InfoReporter, format_nbytes_human, format_nbytes_info from blosc2.list_array import ListArray, coerce_list_cell from blosc2.scalar_array import _ScalarVarLenArray -from blosc2.utf8_array import Utf8Array if TYPE_CHECKING: from blosc2.dictionary_column import DictionaryColumn @@ -191,14 +190,6 @@ def sentinel_for_arrow_type(self, pa, pa_type): _SMALL_NROWS_LIMIT = 10_000_000 _SMALL_SORT_MATERIALIZE_LIMIT = _SMALL_NROWS_LIMIT _MAX_GROWTH_ROWS = 1_048_576 -_UTF8_COMPARE_OPS = { - "eq": np.equal, - "ne": np.not_equal, - "lt": np.less, - "le": np.less_equal, - "gt": np.greater, - "ge": np.greater_equal, -} def get_null_policy() -> NullPolicy: @@ -1972,13 +1963,13 @@ def __invert__(self): def __lt__(self, other): if self.is_utf8: - return self._utf8_compare("lt", other) + return self._utf8_compare(np.less, other) self._ensure_comparable() return self._null_aware_compare(other, self._raw_col < self._coerce_timestamp_operand(other)) def __le__(self, other): if self.is_utf8: - return self._utf8_compare("le", other) + return self._utf8_compare(np.less_equal, other) self._ensure_comparable() return self._null_aware_compare(other, self._raw_col <= self._coerce_timestamp_operand(other)) @@ -1986,7 +1977,7 @@ def __eq__(self, other): if self.is_dictionary: return self._dictionary_eq(other) if self.is_utf8: - return self._utf8_compare("eq", other) + return self._utf8_compare(np.equal, other) self._ensure_comparable() if self._is_nullable_bool and isinstance(other, (bool, np.bool_)): return self._null_aware_compare(other, self._raw_col == int(other)) @@ -1999,7 +1990,7 @@ def __ne__(self, other): return ~result return ~np.asarray(result, dtype=bool) if self.is_utf8: - return self._utf8_compare("ne", other) + return self._utf8_compare(np.not_equal, other) self._ensure_comparable() if self._is_nullable_bool and isinstance(other, (bool, np.bool_)): return self._null_aware_compare(other, self._raw_col == int(not other)) @@ -2022,17 +2013,7 @@ def _utf8_chunked_bool(self, fn, *, chunk_size: int = 65536) -> np.ndarray: result[start:stop] = fn(arr[start:stop], start, stop) return result - def _utf8_null_pred(self): - """Physical-length blosc2.NDArray, True where the utf8 value is the - null sentinel; ``None`` when the column is not nullable.""" - spec = self._table._schema.columns_by_name[self._col_name].spec - nv = getattr(spec, "null_value", None) - if nv is None: - return None - raw = self._utf8_chunked_bool(lambda chunk, start, stop: chunk == nv) - return blosc2.asarray(raw) - - def _utf8_compare(self, op: str, other): + def _utf8_compare(self, numpy_op, other): """Chunked-numpy comparison predicate for a utf8 column. Compares against a Python ``str`` scalar or another utf8 @@ -2058,26 +2039,20 @@ def _utf8_compare(self, op: str, other): f"or another utf8 Column, got {type(other).__name__!r}." ) - numpy_op = _UTF8_COMPARE_OPS[op] - if other_arr is None: - raw = self._utf8_chunked_bool(lambda chunk, start, stop: numpy_op(chunk, other)) - else: - raw = self._utf8_chunked_bool(lambda chunk, start, stop: numpy_op(chunk, other_arr[start:stop])) - pred = blosc2.asarray(raw) & self._lazy_valid_rows() + nv = self.null_value + other_nv = other.null_value if isinstance(other, Column) else None - null_pred = self._utf8_null_pred() - if isinstance(other, Column): - other_null_pred = other._utf8_null_pred() - null_pred = ( - other_null_pred - if null_pred is None - else null_pred - if other_null_pred is None - else null_pred | other_null_pred - ) - if null_pred is not None: - pred = pred & ~null_pred - return pred + def fn(chunk, start, stop): + rhs = other if other_arr is None else other_arr[start:stop] + res = numpy_op(chunk, rhs) + if nv is not None: + res &= chunk != nv + if other_nv is not None: + res &= rhs != other_nv + return res + + raw = self._utf8_chunked_bool(fn) + return blosc2.asarray(raw) & self._lazy_valid_rows() def _dictionary_eq(self, other): """Return a physical-slot boolean predicate for dictionary equality. @@ -2154,13 +2129,13 @@ def _dictionary_isin(self, values) -> np.ndarray: def __gt__(self, other): if self.is_utf8: - return self._utf8_compare("gt", other) + return self._utf8_compare(np.greater, other) self._ensure_comparable() return self._null_aware_compare(other, self._raw_col > self._coerce_timestamp_operand(other)) def __ge__(self, other): if self.is_utf8: - return self._utf8_compare("ge", other) + return self._utf8_compare(np.greater_equal, other) self._ensure_comparable() return self._null_aware_compare(other, self._raw_col >= self._coerce_timestamp_operand(other)) @@ -6669,13 +6644,20 @@ def iter_arrow_batches( # noqa: C901 continue if col.is_utf8: spec = self._schema.columns_by_name[name].spec + arr8 = self._cols[name] + nv = col.null_value + if self.base is None and self._last_pos == self._n_rows and stop <= arr8._persisted_rows: + # Dense root table: logical rows == persisted rows, so + # export straight from the offsets/bytes buffers with + # no per-row decode (storage is already Arrow layout). + arrays.append(arr8.arrow_slice(pa, start, stop, nv)) + continue values = col[start:stop] # StringDType array with sentinel nulls - nv = getattr(spec, "null_value", None) - null_mask = values == nv if nv is not None else None + null_mask = col._null_mask_for(values) if nv is not None else None arrays.append( pa.array( - values.tolist(), - type=pa.large_string(), + values.astype(object), + type=self._pa_type_from_spec(pa, spec), mask=null_mask if null_mask is not None and null_mask.any() else None, ) ) @@ -10919,11 +10901,15 @@ def compact(self): replacement.flush() self._cols[name] = replacement continue + if self._is_utf8_column(col): + # Clustered fancy-index gather, then a bulk rewrite through the + # existing backing arrays so a store-backed column stays + # persistent on disk. + v.set_all(v[real_poss[: self._n_rows]]) + continue if self._is_varlen_scalar_column(col): compacted = [v[int(pos)] for pos in real_poss[: self._n_rows]] - replacement = ( - Utf8Array(col.spec) if self._is_utf8_column(col) else _ScalarVarLenArray(col.spec) - ) + replacement = _ScalarVarLenArray(col.spec) replacement.extend(compacted) replacement.flush() self._cols[name] = replacement @@ -11526,11 +11512,9 @@ def _sort_by_inplace(self, sorted_pos: np.ndarray, n: int) -> None: sorted_codes = arr.codes[sorted_pos] arr.codes[:n] = sorted_codes elif self._is_utf8_column(col): - # Utf8Array has no bulk slice-assignment; rebuild it in sorted order. - new_arr = Utf8Array(col.spec) - new_arr.extend(arr[sorted_pos]) - new_arr.flush() - self._cols[col.name] = new_arr + # Bulk-rewrite through the existing backing arrays so a + # store-backed column stays persistent on disk. + arr.set_all(arr[sorted_pos]) else: arr[:n] = arr[sorted_pos] self._valid_rows[:n] = True @@ -11555,8 +11539,7 @@ def _sorted_copy_from_positions(self, sorted_pos: np.ndarray, n: int) -> CTable: sorted_codes = arr.codes[sorted_pos] result._cols[col_name].codes[:n] = sorted_codes elif self._is_utf8_column(col): - result._cols[col_name].extend(arr[sorted_pos]) - result._cols[col_name].flush() + result._cols[col_name].set_all(arr[sorted_pos]) else: result._cols[col_name][:n] = arr[sorted_pos] result._valid_rows[:n] = True diff --git a/src/blosc2/ctable_storage.py b/src/blosc2/ctable_storage.py index 0c967e7ee..96359498c 100644 --- a/src/blosc2/ctable_storage.py +++ b/src/blosc2/ctable_storage.py @@ -37,6 +37,7 @@ _ScalarVarLenArray, _validate_role_metadata, ) +from blosc2.schema import Utf8Spec from blosc2.schunk import process_opened_object from blosc2.utf8_array import Utf8Array, _new_backend_arrays @@ -250,8 +251,6 @@ def open_list_column(self, name): raise RuntimeError("In-memory tables have no on-disk representation to open.") def create_varlen_scalar_column(self, name, *, spec, cparams=None, dparams=None): - from blosc2.schema import Utf8Spec - if isinstance(spec, Utf8Spec): offsets, data = _new_backend_arrays(cparams, dparams) return Utf8Array(spec, offsets, data) @@ -485,8 +484,6 @@ def open_list_column(self, name: str) -> ListArray: return self._estore[self._col_key(name)] def open_varlen_scalar_column(self, name: str, spec) -> _ScalarVarLenArray: - from blosc2.schema import Utf8Spec - if isinstance(spec, Utf8Spec): offsets = self._estore[self._col_key(name)] data = self._estore[self._col_key(name) + _UTF8_DATA_SUFFIX] @@ -724,25 +721,19 @@ def open_list_column(self, name: str) -> ListArray: return blosc2.open(self._list_col_path(name), mode=self._mode) def create_varlen_scalar_column(self, name, *, spec, cparams=None, dparams=None) -> _ScalarVarLenArray: - from blosc2.schema import Utf8Spec - if isinstance(spec, Utf8Spec): offsets, data = _new_backend_arrays(cparams, dparams) store = self._open_store() - store[self._col_key(name)] = offsets - store[self._col_key(name) + _UTF8_DATA_SUFFIX] = data - return Utf8Array( - spec, - store[self._col_key(name)], - store[self._col_key(name) + _UTF8_DATA_SUFFIX], - ) + key = self._col_key(name) + data_key = key + _UTF8_DATA_SUFFIX + store[key] = offsets + store[data_key] = data + return Utf8Array(spec, store[key], store[data_key]) urlpath = self._list_col_path(name) backend = _make_persistent_backend(spec, urlpath, "w", cparams=cparams, dparams=dparams) return _ScalarVarLenArray(spec, backend) def open_varlen_scalar_column(self, name: str, spec) -> _ScalarVarLenArray: - from blosc2.schema import Utf8Spec - if isinstance(spec, Utf8Spec): store = self._open_store() offsets = store[self._col_key(name)] @@ -1310,8 +1301,6 @@ def create_varlen_scalar_column( cparams=None, dparams=None, ) -> _ScalarVarLenArray: - from blosc2.schema import Utf8Spec - if isinstance(spec, Utf8Spec): logical_key = self._col_logical_key(name) offsets_path = self._dest_path(logical_key, ".b2nd") @@ -1333,8 +1322,6 @@ def create_varlen_scalar_column( return _make_persistent_backend(spec, urlpath, "w", cparams=cparams, dparams=dparams) def open_varlen_scalar_column(self, name: str, spec) -> _ScalarVarLenArray: - from blosc2.schema import Utf8Spec - if isinstance(spec, Utf8Spec): logical_key = self._col_logical_key(name) offsets = self._open_leaf(logical_key) diff --git a/src/blosc2/utf8_array.py b/src/blosc2/utf8_array.py index 68056ffdc..1164bcab5 100644 --- a/src/blosc2/utf8_array.py +++ b/src/blosc2/utf8_array.py @@ -281,6 +281,19 @@ def flush(self) -> None: self._pending_chars = 0 self._rewrite_from(self._persisted_rows, values) + def set_all(self, values: Iterable[Any]) -> None: + """Replace the whole column content in one bulk write. + + Writes through the existing backing offsets/data NDArrays, so a + store-backed column stays persistent (unlike building a fresh + in-memory ``Utf8Array``). Used by ``sort_by(inplace=True)`` and + ``compact()`` to rewrite a column in a new row order. + """ + coerced = [self._coerce(v) for v in values] + self._pending = [] + self._pending_chars = 0 + self._rewrite_from(0, coerced) + # ------------------------------------------------------------------ # Public read interface # ------------------------------------------------------------------ @@ -338,8 +351,31 @@ def __setitem__(self, index: int, value: Any) -> None: if index >= self._persisted_rows: self._pending[index - self._persisted_rows] = value return - tail = self._read_persisted_span(index + 1, self._persisted_rows) - self._rewrite_from(index, [value, *tail.tolist()]) + # The tail bytes themselves are unchanged; only their position moves + # by the length delta of the replaced value, so shift raw bytes and + # add the delta to the tail offsets instead of decoding/re-encoding + # every following row. + encoded = value.encode("utf-8") + bounds = np.asarray(self._offsets[index : index + 2], dtype=np.int64) + old_start, old_end = int(bounds[0]), int(bounds[1]) + delta = len(encoded) - (old_end - old_start) + old_used = self._bytes_used + new_used = old_used + delta + tail = np.asarray(self._data[old_end:old_used]).copy() if delta != 0 and old_used > old_end else None + if delta > 0 and int(self._data.shape[0]) < new_used: + self._data.resize((new_used,)) + if encoded: + self._data[old_start : old_start + len(encoded)] = np.frombuffer(encoded, dtype=np.uint8) + if tail is not None: + self._data[old_end + delta : new_used] = tail + if delta < 0 and int(self._data.shape[0]) != max(new_used, 1): + self._data.resize((max(new_used, 1),)) + if delta != 0: + n = self._persisted_rows + self._offsets[index + 1 : n + 1] = ( + np.asarray(self._offsets[index + 1 : n + 1], dtype=np.int64) + delta + ) + self._bytes_used_cache = new_used # ------------------------------------------------------------------ # Properties mirroring the interface expected by CTable @@ -387,6 +423,41 @@ def cratio(self) -> float: return float("inf") return self.nbytes / cb + def arrow_slice(self, pa, a: int, b: int, null_value: str | None = None): + """Persisted rows ``[a, b)`` as a ``pyarrow.LargeStringArray``. + + The storage layout (int64 offsets + UTF-8 byte blob) is exactly + Arrow's ``large_string`` layout, so the array is built directly from + the raw buffers with no per-row decode or Python string objects. + When *null_value* is given, rows equal to the sentinel become Arrow + nulls (matched on raw bytes, still without decoding). + """ + n = b - a + offs = np.ascontiguousarray(self._offsets[a : b + 1], dtype=np.int64) + start, end = int(offs[0]), int(offs[-1]) + rel = offs - start + data = np.ascontiguousarray(self._data[start:end]) if end > start else np.empty(0, dtype=np.uint8) + validity = None + null_count = 0 + if null_value is not None and n > 0: + nv_enc = null_value.encode("utf-8") + lengths = np.diff(rel) + if nv_enc: + mask = np.zeros(n, dtype=np.bool_) + cand = np.flatnonzero(lengths == len(nv_enc)) + if len(cand): + spans = data[rel[cand][:, None] + np.arange(len(nv_enc))] + hits = (spans == np.frombuffer(nv_enc, dtype=np.uint8)).all(axis=1) + mask[cand[hits]] = True + else: + mask = lengths == 0 + null_count = int(mask.sum()) + if null_count: + validity = pa.array(~mask).buffers()[1] + return pa.LargeStringArray.from_buffers( + n, pa.py_buffer(rel), pa.py_buffer(data), validity, null_count + ) + def copy(self, spec=None, **kwargs: Any) -> Utf8Array: """Return an in-memory copy.""" if spec is None: diff --git a/tests/ctable/test_utf8.py b/tests/ctable/test_utf8.py index 4c86fa994..706cae1e7 100644 --- a/tests/ctable/test_utf8.py +++ b/tests/ctable/test_utf8.py @@ -636,6 +636,64 @@ class TwoCols: assert list(t["note"][:]) == ["n-a", "n-b", "n-c"] +@pytest.mark.parametrize("ext", [".b2z", ".b2d"]) +def test_ctable_utf8_sort_inplace_persists_after_reopen(tmp_path, ext): + """Regression: sort_by(inplace=True) on a file-backed table must write the + sorted utf8 rows through to the store, keeping them aligned with the other + (on-disk-sorted) columns after close/reopen.""" + urlpath = str(tmp_path / f"utf8_sort{ext}") + t = make_table(["banana", "apple", "cherry"], urlpath=urlpath, mode="w") + t.sort_by("name", inplace=True) + assert list(t["name"][:]) == ["apple", "banana", "cherry"] + t.close() + + t2 = CTable.open(urlpath, mode="r") + try: + assert list(t2["name"][:]) == ["apple", "banana", "cherry"] + assert list(t2["x"][:]) == [1, 0, 2] # row alignment survives the reopen + finally: + t2.close() + + +@pytest.mark.parametrize("ext", [".b2z", ".b2d"]) +def test_ctable_utf8_compact_persists_after_reopen(tmp_path, ext): + """Regression: compact() on a file-backed table must rewrite the utf8 + column in the store, not in a detached in-memory replacement.""" + urlpath = str(tmp_path / f"utf8_compact{ext}") + t = make_table(["a", "bb", "ccc", "dddd"], urlpath=urlpath, mode="w") + t.delete([1]) + t.compact() + assert list(t["name"][:]) == ["a", "ccc", "dddd"] + t.close() + + t2 = CTable.open(urlpath, mode="r") + try: + assert list(t2["name"][:]) == ["a", "ccc", "dddd"] + assert list(t2["x"][:]) == [0, 2, 3] + finally: + t2.close() + + +def test_ctable_utf8_setitem_persisted_shifts_survive_reopen(tmp_path): + """__setitem__ on persisted rows shifts the byte blob in place; longer, + shorter, equal-length, and empty replacements must all round-trip.""" + urlpath = str(tmp_path / "utf8_setitem.b2d") + t = make_table(["aa", "bb", "cc", "dd"], urlpath=urlpath, mode="w") + t["name"][1] = "a much longer replacement" # grow + t["name"][2] = "c" # shrink + t["name"][0] = "xx" # equal length + t["name"][3] = "" # empty + expected = ["xx", "a much longer replacement", "c", ""] + assert list(t["name"][:]) == expected + t.close() + + t2 = CTable.open(urlpath, mode="r") + try: + assert list(t2["name"][:]) == expected + finally: + t2.close() + + def test_ctable_utf8_sort_non_ascii(): t = make_table(["café", "日本語のテキスト", "banana"]) s = t.sort_by("name") @@ -682,6 +740,31 @@ def test_utf8_pa_table_roundtrip_with_nulls(): assert at.column("name").null_count == 1 +def test_utf8_arrow_export_from_view_and_after_delete(): + """Arrow export of non-dense tables (views, deleted rows) takes the + materializing fallback path; values and nulls must match the fast path.""" + pa = pytest.importorskip("pyarrow") + t = CTable(NullableRow, new_data={"name": ["a", None, "c", "d"], "x": [1, 2, 3, 4]}) + view = t[t.x > 1] + at = pa.table(view) + assert at.schema.field("name").type == pa.large_string() + assert at.column("name").to_pylist() == [None, "c", "d"] + + t.delete([0]) # dense-table fast path no longer applies + at2 = t.to_arrow() + assert at2.column("name").to_pylist() == [None, "c", "d"] + assert at2.column("name").null_count == 1 + + +def test_utf8_arrow_export_pending_rows(): + """Rows still buffered in memory (not yet flushed) must export correctly.""" + pa = pytest.importorskip("pyarrow") + t = make_table(["x", "y"]) + t.append(("pending", 99)) + at = pa.table(t) + assert at.column("name").to_pylist() == ["x", "y", "pending"] + + def test_utf8_from_arrow_default_ingest(): pa = pytest.importorskip("pyarrow") at = pa.table({"name": pa.array(SAMPLE, type=pa.string()), "x": pa.array(range(len(SAMPLE)))}) From 77bf321d88204f49765977fcc8a7e6f4981fcb6a Mon Sep 17 00:00:00 2001 From: Francesc Alted Date: Thu, 16 Jul 2026 23:24:43 +0200 Subject: [PATCH 08/20] Fast factorization for utf8 groupby keys: benchmark gate passes at 2.83x MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Replaces the np.unique-on-StringDType fallback (recorded at 16.9x/49.9x vs dictionary keys) with a factorization that never decodes N rows: - Utf8Array.factorize_span()/Utf8Factorizer group rows by raw byte length, gather each length group column-wise into a (k, L) byte matrix (one reused index vector — ~2x faster than a 2-D fancy-index), hash rows with the same collision-checked mixer as _factorize_fixed_width_str, and decode only the D distinct values. The factorizer keeps a cross-chunk vocabulary (sorted hashes plus representative bytes per length), so rows carrying already-seen values are searchsorted-matched and byte-verified instead of re-sorted. - utf8 key chunks flow through the groupby pipeline as _Utf8KeyChunk (dense int64 string-rank codes + sorted uniques): null masks, live-row masking, and dedup run on integers; single-key dedup is an O(n) bincount. Multi-key packing combines all-integral keys into one composite int64 (Horner over zero-based fields) — np.unique over a structured dtype does field-wise void comparisons and dominated the two-key time; non-integral co-keys keep the structured fallback. - The generic groupby loop batches at ~1 Mi rows instead of the raw validity chunk shape: in-memory tables grown by resize keep their tiny initial 64-row chunks, so the loop ran 156k batches of bookkeeping at 1e7 rows. This helps every generic-path key type. Benchmark (bench_groupby_keys.py, 1e7 rows, 5-city keys, Apple silicon): utf8 key sum: 3304 -> 558 ms, now 2.83x slower than dict keys (gate: <= 3x); two keys int+utf8: 11825 -> 878 ms, now 3.69x slower than dict keys (was 49.9x). utf8 keys are now faster than fixed-width string keys (535 ms). The byte-exact factorization also fixes a correctness gap inherited from np.unique: NumPy merges StringDType values differing only after an embedded NUL, so such keys used to collapse into one group. Co-Authored-By: Claude Fable 5 --- plans/enhancing-ctable-phase3.md | 61 ++++++++++- src/blosc2/groupby.py | 169 +++++++++++++++++++++++++++---- src/blosc2/utf8_array.py | 147 +++++++++++++++++++++++++++ tests/ctable/test_utf8.py | 89 ++++++++++++++++ 4 files changed, 446 insertions(+), 20 deletions(-) diff --git a/plans/enhancing-ctable-phase3.md b/plans/enhancing-ctable-phase3.md index 9f0831ba6..662182dd1 100644 --- a/plans/enhancing-ctable-phase3.md +++ b/plans/enhancing-ctable-phase3.md @@ -399,6 +399,62 @@ proves hard, the honest fallback is `np.unique` on the StringDType chunk rejected (only utf8 was carved out). Full `tests/ctable` (1379) and `tests/ndarray` (4385) suites pass. +**P3.d follow-up: fast factorization path (unparked and landed 2026-07-16, +after the post-review fixes below). Benchmark gate now PASSES.** + +- What landed — essentially the algorithm sketched above, plus two pipeline + fixes the profiling surfaced along the way: + 1. `Utf8Array.factorize_span(a, b)` / incremental `Utf8Factorizer` + (`utf8_array.py`): rows are grouped by raw byte length (vectorized + `bincount`), each length group is gathered column-wise into a `(k, L)` + byte matrix (column-wise gather with one reused index vector — ~2x + faster than a 2-D fancy-index, which materializes a `(k, L)` int64 + index matrix), hashed with the same `_HASH_MIX` mixer + verify pass as + `_factorize_fixed_width_str`, and **only the D distinct values are ever + decoded**. The factorizer keeps a cross-chunk vocabulary (sorted + hashes + representative bytes per length): rows carrying already-seen + values are `searchsorted`-matched and byte-verified; only new values + pay a sort. This is the "cross-chunk vocabulary cache" the ponytail + note in `_factorize_fixed_width_str` predicted. + 2. `groupby.py`: utf8 key chunks now flow through the pipeline as + `_Utf8KeyChunk` (chunk-local int64 rank codes + sorted uniques), so + null masks, live-row masking, and per-chunk dedup all run on integers. + Single-key dedup is an O(n) `bincount` (codes are dense ranks — no + sort). Multi-key packing uses a **composite int64 key** (Horner over + zero-based fields) when every key is integral and the range product + fits — `np.unique` over a *structured* dtype does field-wise void + comparisons and was the dominant two-key cost; non-integral co-keys + fall back to the structured path with utf8 codes as int fields. + 3. `_chunk_size()` now scales the generic-loop batch up to ~1 Mi rows + (chunk-aligned): in-memory tables created small and grown by resize + keep their initial 64-row validity chunk shape, and batching the loop + at that granularity spent more time in per-batch bookkeeping than in + work. This was a pre-existing pathology exposed while profiling; it + benefits every generic-path key type (fixed-width strings included). +- Measured (same bench, same box, same 3x target vs. dictionary keys): + + | key type | before | after | vs. dict key | + |-----------------------|-------------|-----------|--------------| + | dict key, sum | 196.0 ms | 197.2 ms | 1.0x | + | **utf8 key, sum** | 3304.2 ms | **557.6 ms** | **2.83x ✓** | + | two keys (int+dict) | 236.8 ms | 237.9 ms | 1.0x | + | **two keys (int+utf8)** | 11825.3 ms | **877.8 ms** | **3.69x** | + + Single-key meets the ≤3x gate (2.83x, 5.9x faster than the fallback) and + is now *faster* than fixed-width `string` keys (535 ms). Two-key misses + the letter of 3x (3.69x) but is 13.5x faster than the recorded gap; the + remainder is the shared display/normalize/merge Python bookkeeping, not + utf8-specific. +- **Correctness bonus found while testing**: NumPy's `np.unique` on + `StringDType` merges strings that differ only after an embedded NUL + (`"nul\x00in"` vs `"nul\x00IN"` collapse to one group — a NumPy bug). + The byte-exact factorization does not, so utf8 groupby keys now handle + NUL-bearing strings *more* correctly than the `np.unique` fallback did. +- Tests added: factorize_span vs. ground-truth set semantics (incl. NUL, + non-ASCII, multi-KB values), cross-span global-code consistency, groupby + over many byte lengths + non-ASCII keys, multi-key with negative int + co-key (composite packing), multi-key with float co-key (structured + fallback). Full `tests/` (7,489) passes. **P3.e Sort.** Lift the sort rejection (grep the `sort_by` varlen guard in `ctable.py`): `sort_by` on utf8 uses `np.argsort` on the StringDType array @@ -518,7 +574,10 @@ branch):** as is. Also left as recorded: the `is_varlen_scalar`-includes-utf8 predicate design (deliberate, documented in the P3.a notes) and the `_read_persisted_span` per-row decode loop (root cause of the P3.d - benchmark gap, already documented above as its own follow-up). + benchmark gap; **since addressed for groupby keys** by the P3.d follow-up + above — the loop still runs for plain bulk reads, i.e. comparisons and + sort-key materialization, where a StringDType array genuinely has to be + built). - Full `tests/` suite (7,484) passes after the fixes. **Out of scope for P3:** making `utf8` the `str`-field default; `.str` diff --git a/src/blosc2/groupby.py b/src/blosc2/groupby.py index 3186be858..6aaf4747d 100644 --- a/src/blosc2/groupby.py +++ b/src/blosc2/groupby.py @@ -53,6 +53,34 @@ class _AggState: count: int = 0 +@dataclasses.dataclass +class _Utf8KeyChunk: + """A utf8 key-column chunk, factorized to chunk-local integer codes. + + ``codes[i]`` indexes ``uniques`` (a ``StringDType`` array sorted + ascending), so null detection, live-row masking, and per-chunk + ``np.unique`` all run on int64 codes; only the (few) distinct strings are + ever decoded. Produced by :meth:`CTableGroupBy._read_key_chunk` via + ``Utf8Array.factorizer``. + """ + + codes: np.ndarray + uniques: np.ndarray + + def __len__(self) -> int: + return len(self.codes) + + def take(self, mask: np.ndarray) -> _Utf8KeyChunk: + return _Utf8KeyChunk(self.codes[mask], self.uniques) + + def code_of(self, value: str) -> int: + """Code of *value* in this chunk, or -1 when absent (uniques are sorted).""" + i = int(np.searchsorted(self.uniques, value)) + if i < len(self.uniques) and self.uniques[i] == value: + return i + return -1 + + def _is_column_like(value: Any) -> bool: return isinstance(getattr(value, "_col_name", None), str) @@ -123,6 +151,10 @@ def __init__( self.dropna = bool(dropna) self.engine = engine self.chunk_size = chunk_size + # Per-key incremental Utf8Factorizer instances, shared across the + # chunk loop so the string vocabulary is built once (see + # _read_key_chunk). + self._utf8_factorizers: dict[str, Any] = {} for name in self.keys: if name in table._computed_cols: @@ -503,7 +535,12 @@ def _execute_with_result_target(self, specs: list[_AggSpec]): if not np.any(live_mask): continue - keys_live = [np.asarray(values)[live_mask] for values in raw_keys] + keys_live = [ + values.take(live_mask) + if isinstance(values, _Utf8KeyChunk) + else np.asarray(values)[live_mask] + for values in raw_keys + ] n_live = len(keys_live[0]) if n_live == 0: continue @@ -1505,26 +1542,51 @@ def _chunk_size(self) -> int: if self.chunk_size <= 0: raise ValueError("chunk_size must be positive") return int(self.chunk_size) + target = 1 << 20 chunks = getattr(self.table._valid_rows, "chunks", None) - if chunks: - return max(int(chunks[0]), 1) - return 65536 + if not chunks: + return target + base = max(int(chunks[0]), 1) + if base >= target: + return base + # Batching the loop at the raw validity chunk shape can be pathological + # (in-memory tables created small and grown by resize keep their tiny + # initial chunk shape, e.g. 64 rows), and even 64 Ki-row batches leave + # the per-batch Python bookkeeping (group display/merge) visible for + # multi-key groupbys. Scale up to ~1 Mi rows while staying + # chunk-aligned; per-column batch memory stays modest (8 MB per int64 + # column). + return base * -(-target // base) def _read_key_chunk(self, name: str, start: int, stop: int) -> np.ndarray: col_info = self.table._schema.columns_by_name[name] if self.table._is_dictionary_column(col_info): return np.asarray(self.table._cols[name].codes[start:stop], dtype=np.int32) if self.table._is_utf8_column(col_info): + # Factorize the chunk from its raw offsets/bytes buffers: no row + # is decoded, only the distinct values (codes flow through the + # rest of the pipeline). The factorizer is shared across chunks + # so values seen before are hash-matched instead of re-sorted. # Utf8Array is sized to the logical row count, not the physical - # valid_rows capacity, so a chunk boundary can run past its end. - # Rows beyond it are never live (the row can't have been written - # without this column), so the padding value is never read live. + # valid_rows capacity, so a chunk boundary can run past its end; + # rows beyond it are never live (the row can't have been written + # without this column), so the padding code is never read live. col = self.table._cols[name] + fact = self._utf8_factorizers.get(name) + if fact is None: + fact = self._utf8_factorizers[name] = col.factorizer() n = len(col) - values = np.asarray(col[start : min(stop, n)]) + codes = fact.codes_for_span(start, min(stop, n)) if start < n else np.empty(0, dtype=np.int64) + uniques = fact.uniques() + # Rank-normalize so this chunk keeps the np.unique contract the + # pipeline expects (uniques ascending, codes = string ranks). + order = np.argsort(uniques, kind="stable") + rank = np.empty(len(order), dtype=np.int64) + rank[order] = np.arange(len(order)) + codes = rank[codes] if len(order) else codes if stop > n: - values = np.concatenate([values, np.full(stop - max(start, n), "", dtype=values.dtype)]) - return values + codes = np.concatenate([codes, np.zeros(stop - max(start, n), dtype=np.int64)]) + return _Utf8KeyChunk(codes, uniques[order]) return np.asarray(self.table._cols[name][start:stop]) def _factorize_keys( @@ -1532,22 +1594,86 @@ def _factorize_keys( ) -> tuple[np.ndarray | list[np.ndarray], np.ndarray]: if len(keys_live) == 1: arr = keys_live[0] + if isinstance(arr, _Utf8KeyChunk): + # The chunk is already factorized to dense string-rank codes; + # dedupe them with an O(n) bincount instead of a sort. The + # np.unique contract — uniques ascending by string — holds + # because the codes are ranks into the sorted uniques. + present = np.bincount(arr.codes, minlength=len(arr.uniques)) > 0 + remap = np.cumsum(present) - 1 + return arr.uniques[present], remap[arr.codes] if arr.dtype.kind in ("U", "S") and arr.dtype.itemsize % 4 == 0 and arr.dtype.itemsize: return _factorize_fixed_width_str(arr) unique, inverse = np.unique(arr, return_inverse=True) return unique, inverse - # StringDType (utf8 columns) cannot be a structured-array field, so pack - # those columns as plain Python-object arrays instead; np.unique still - # dedupes them correctly by str equality. - pack_arrs = [arr.astype(object) if arr.dtype.kind == "T" else arr for arr in keys_live] - dtype = [(f"k{i}", arr.dtype) for i, arr in enumerate(pack_arrs)] - packed = np.empty(len(pack_arrs[0]), dtype=dtype) - for i, arr in enumerate(pack_arrs): - packed[f"k{i}"] = arr - unique, inverse = np.unique(packed, return_inverse=True) + # utf8 key chunks pack as their int64 codes (codes are string-rank + # within the chunk, so the packed sort order matches the string sort + # order); the codes in the deduped rows are mapped back to strings + # below, into object fields (StringDType cannot be a structured-array + # field). + pack_arrs = [arr.codes if isinstance(arr, _Utf8KeyChunk) else arr for arr in keys_live] + composite = self._composite_int_factorize(pack_arrs) + if composite is not None: + unique_fields, inverse = composite + else: + dtype = [(f"k{i}", arr.dtype) for i, arr in enumerate(pack_arrs)] + packed = np.empty(len(pack_arrs[0]), dtype=dtype) + for i, arr in enumerate(pack_arrs): + packed[f"k{i}"] = arr + packed_unique, inverse = np.unique(packed, return_inverse=True) + unique_fields = [packed_unique[f"k{i}"] for i in range(len(pack_arrs))] + out_dtype = [ + (f"k{i}", object if isinstance(arr, _Utf8KeyChunk) else pack_arrs[i].dtype) + for i, arr in enumerate(keys_live) + ] + unique = np.empty(len(unique_fields[0]), dtype=out_dtype) + for i, arr in enumerate(keys_live): + field = unique_fields[i] + unique[f"k{i}"] = arr.uniques[field] if isinstance(arr, _Utf8KeyChunk) else field return unique, inverse + @staticmethod + def _composite_int_factorize( + pack_arrs: list[np.ndarray], + ) -> tuple[list[np.ndarray], np.ndarray] | None: + """Dedup rows of all-integer key columns via one combined int64 key. + + ``np.unique`` over a structured dtype compares rows field-by-field + through void comparisons and is ~an order of magnitude slower than + over a plain int64 array, so when every key column is integral and + the product of the per-column value ranges fits int64, combine the + columns into a single integer (Horner over the zero-based fields — + the combined sort order equals the structured lexicographic order) + and dedup that. Returns ``(per-field unique values, inverse)``, or + ``None`` when the keys don't fit this scheme. + """ + if not all(arr.dtype.kind in "biu" for arr in pack_arrs): + return None + if len(pack_arrs[0]) == 0: + return None + mins = [int(arr.min()) for arr in pack_arrs] + spans = [int(arr.max()) - mn + 1 for arr, mn in zip(pack_arrs, mins, strict=True)] + if math.prod(spans) >= 1 << 62: + return None + combined = np.zeros(len(pack_arrs[0]), dtype=np.int64) + for arr, mn, span in zip(pack_arrs, mins, spans, strict=True): + if arr.dtype.kind == "u": + # Zero-base within the unsigned dtype first: the raw values may + # not fit int64, but value - min always fits (span is checked). + zero_based = (arr - arr.dtype.type(mn)).astype(np.int64, copy=False) + else: + zero_based = arr.astype(np.int64, copy=False) - mn + combined *= span + combined += zero_based + unique_c, inverse = np.unique(combined, return_inverse=True) + unique_fields: list[np.ndarray] = [None] * len(pack_arrs) # type: ignore[list-item] + rem = unique_c + for i in range(len(pack_arrs) - 1, -1, -1): + unique_fields[i] = (rem % spans[i] + mins[i]).astype(pack_arrs[i].dtype) + rem = rem // spans[i] + return unique_fields, inverse + def _resolve_sort(self, *, cheap: bool) -> bool: """Resolve the tri-state ``self.sort`` request for the current path. @@ -1990,6 +2116,11 @@ def _result_spec_for_agg(self, spec: _AggSpec) -> SchemaSpec: def _null_mask(self, name: str, values: np.ndarray, *, is_key: bool) -> np.ndarray: col_info = self.table._schema.columns_by_name[name] spec = col_info.spec + if isinstance(values, _Utf8KeyChunk): + null_value = getattr(spec, "null_value", None) + if null_value is None: + return np.zeros(len(values), dtype=bool) + return values.codes == values.code_of(null_value) if isinstance(spec, DictionarySpec): mask = values == np.int32(spec.null_code) return mask if is_key or getattr(spec, "nullable", False) else np.zeros(len(values), dtype=bool) diff --git a/src/blosc2/utf8_array.py b/src/blosc2/utf8_array.py index 1164bcab5..c6ce745c1 100644 --- a/src/blosc2/utf8_array.py +++ b/src/blosc2/utf8_array.py @@ -53,6 +53,34 @@ # when the gap between consecutive row indices exceeds this many rows. _GATHER_GAP = 1024 +# Multiplier for the byte-hash in factorize_span (same mixer as +# groupby._factorize_fixed_width_str). +_HASH_MIX = np.uint64(0x9E3779B97F4A7C15) + + +def _factorize_byte_rows(mat: np.ndarray) -> tuple[np.ndarray, np.ndarray]: + """Exact factorization of the rows of a ``(k, L)`` uint8 matrix. + + Returns ``(rep_rows, inverse)`` where ``rep_rows`` holds one representative + row index per distinct byte string (in unspecified group order — the + caller re-sorts groups) and ``inverse`` maps each row to its group. + Hashes each row into one uint64 and factorizes the integers, with a + vectorized verify pass; on a hash collision, falls back to an exact sort + of the raw rows. + """ + h = mat[:, 0].astype(np.uint64) + for i in range(1, mat.shape[1]): + h = (h * _HASH_MIX) ^ mat[:, i] + hash_uniques, inverse = np.unique(h, return_inverse=True) + rep_rows = np.empty(len(hash_uniques), dtype=np.int64) + rep_rows[inverse] = np.arange(len(mat)) + if not (mat == mat[rep_rows][inverse]).all(): # collision: exact fallback + as_void = np.ascontiguousarray(mat).view([("", np.uint8, mat.shape[1])]).ravel() + void_uniques, inverse = np.unique(as_void, return_inverse=True) + rep_rows = np.empty(len(void_uniques), dtype=np.int64) + rep_rows[inverse] = np.arange(len(mat)) + return rep_rows, inverse + def string_dtype(): """Return a ``numpy.dtypes.StringDType`` instance, or raise if unavailable.""" @@ -423,6 +451,28 @@ def cratio(self) -> float: return float("inf") return self.nbytes / cb + def factorizer(self) -> Utf8Factorizer: + """Return a fresh incremental factorizer over this column's rows.""" + return Utf8Factorizer(self) + + def factorize_span(self, a: int, b: int) -> tuple[np.ndarray, np.ndarray]: + """Factorize rows ``[a, b)`` without decoding them. + + Returns ``(codes, uniques)`` where ``uniques`` is a ``StringDType`` + array of the distinct values sorted ascending and ``codes`` (int64, + length ``b - a``) maps each row to its value — the same contract as + ``np.unique(values, return_inverse=True)``, but computed from the raw + offsets/bytes buffers via :class:`Utf8Factorizer`: only the distinct + values are ever decoded to ``str``. Pending rows are flushed first. + """ + fact = self.factorizer() + codes = fact.codes_for_span(a, b) + uniques = fact.uniques() + order = np.argsort(uniques, kind="stable") + rank = np.empty(len(order), dtype=np.int64) + rank[order] = np.arange(len(order)) + return rank[codes], uniques[order] + def arrow_slice(self, pa, a: int, b: int, null_value: str | None = None): """Persisted rows ``[a, b)`` as a ``pyarrow.LargeStringArray``. @@ -466,3 +516,100 @@ def copy(self, spec=None, **kwargs: Any) -> Utf8Array: out.extend(self) out.flush() return out + + +class Utf8Factorizer: + """Incremental factorizer over a :class:`Utf8Array`'s rows. + + Successive :meth:`codes_for_span` calls share a vocabulary: each row is + hashed from its raw bytes and matched against the known values of its + byte length (``np.searchsorted`` on sorted hashes plus a vectorized + byte-equality verify), so only rows carrying *new* values pay a sort + (via :func:`_factorize_byte_rows`). Codes are global across spans, in + first-appearance order; :meth:`uniques` returns the decoded vocabulary in + that same order. No row is ever decoded — only the distinct values. + + A hash collision between a new value and a known one is not resolved + (the new value gets a fresh code); the resulting duplicate vocabulary + entry is benign because callers merge groups by decoded value, and with + 64-bit hashes the case is vanishingly rare anyway. + """ + + def __init__(self, arr: Utf8Array) -> None: + arr.flush() + self._arr = arr + self._values: list[str] = [] + self._empty_code: int | None = None + # byte length -> (sorted hashes, codes aligned to them, (D, L) rep bytes) + self._by_len: dict[int, tuple[np.ndarray, np.ndarray, np.ndarray]] = {} + + def uniques(self) -> np.ndarray: + """The vocabulary so far, decoded, in global code order.""" + return np.array(self._values, dtype=self._arr._dtype) + + def codes_for_span(self, a: int, b: int) -> np.ndarray: + """Global codes (int64) for persisted rows ``[a, b)``.""" + n = b - a + codes = np.empty(max(n, 0), dtype=np.int64) + if n <= 0: + return codes + offs = np.asarray(self._arr._offsets[a : b + 1], dtype=np.int64) + start, end = int(offs[0]), int(offs[-1]) + rel = offs - start + data = np.asarray(self._arr._data[start:end]) if end > start else np.empty(0, dtype=np.uint8) + lengths = np.diff(rel) + if int(lengths.max()) <= 65536: + distinct_lengths = np.flatnonzero(np.bincount(lengths)) + else: + distinct_lengths = np.unique(lengths) + for length in distinct_lengths: + length = int(length) + rows = np.flatnonzero(lengths == length) + if length == 0: + if self._empty_code is None: + self._empty_code = len(self._values) + self._values.append("") + codes[rows] = self._empty_code + continue + # Column-wise gather: reusing one index vector is ~2x faster than + # materializing the (k, L) int64 index matrix of a 2-D gather. + mat = np.empty((len(rows), length), dtype=np.uint8) + idx = rel[rows] + for i in range(length): + mat[:, i] = data[idx] + idx += 1 + h = mat[:, 0].astype(np.uint64) + for i in range(1, length): + h = (h * _HASH_MIX) ^ mat[:, i] + miss = np.ones(len(rows), dtype=np.bool_) + entry = self._by_len.get(length) + if entry is not None: + sorted_h, code_at, rep_mat = entry + pos = np.searchsorted(sorted_h, h) + pos[pos == len(sorted_h)] = 0 # equality check below rejects + hit = sorted_h[pos] == h + if hit.any(): + # One full-row compare beats gathering the hit rows out of + # mat first (hits are usually nearly all rows). + good = np.flatnonzero(hit & (mat == rep_mat[pos]).all(axis=1)) + codes[rows[good]] = code_at[pos[good]] + miss[good] = False + if miss.any(): + miss_mat = mat[miss] + rep_rows, inverse = _factorize_byte_rows(miss_mat) + base = len(self._values) + self._values.extend(bytes(miss_mat[j]).decode("utf-8") for j in rep_rows) + codes[rows[miss]] = base + inverse + new_h = np.empty(len(rep_rows), dtype=np.uint64) + new_rep = miss_mat[rep_rows] + new_h[:] = new_rep[:, 0] + for i in range(1, length): + new_h = (new_h * _HASH_MIX) ^ new_rep[:, i] + new_codes = np.arange(base, base + len(rep_rows), dtype=np.int64) + if entry is not None: + new_h = np.concatenate([entry[0], new_h]) + new_codes = np.concatenate([entry[1], new_codes]) + new_rep = np.concatenate([entry[2], new_rep]) + order = np.argsort(new_h, kind="stable") + self._by_len[length] = (new_h[order], new_codes[order], new_rep[order]) + return codes diff --git a/tests/ctable/test_utf8.py b/tests/ctable/test_utf8.py index 706cae1e7..5f934c062 100644 --- a/tests/ctable/test_utf8.py +++ b/tests/ctable/test_utf8.py @@ -480,6 +480,95 @@ def test_ctable_utf8_startswith_endswith(): # --------------------------------------------------------------------------- +def test_utf8_factorize_span_matches_np_unique_contract(): + """The raw-bytes factorization keeps the np.unique contract: uniques + sorted ascending, codes indexing them. Ground truth is Python's set — + numpy's np.unique on StringDType merges strings differing only after an + embedded NUL (numpy bug), which the byte-exact factorization does not. + """ + from blosc2.utf8_array import Utf8Array + + rng = np.random.default_rng(7) + pool = ["", "a", "ab", "café", "日本語", "x" * 3000, "nul\x00in", "nul\x00IN", "Wien", "wien"] + values = [pool[i] for i in rng.integers(0, len(pool), 5000)] + arr = Utf8Array(blosc2.utf8()) + arr.extend(values) + codes, uniques = arr.factorize_span(0, len(values)) + assert list(uniques) == sorted(set(values)) + assert all(uniques[c] == v for c, v in zip(codes, values, strict=True)) + + +def test_utf8_factorizer_cross_span_codes_are_global(): + from blosc2.utf8_array import Utf8Array + + arr = Utf8Array(blosc2.utf8()) + arr.extend(["b", "a", "b", "c", "a", "d"]) + fact = arr.factorizer() + c1 = fact.codes_for_span(0, 3) # b, a, b + c2 = fact.codes_for_span(3, 6) # c, a, d + uniques = fact.uniques() + assert [uniques[c] for c in c1] == ["b", "a", "b"] + assert [uniques[c] for c in c2] == ["c", "a", "d"] + # "a" keeps the code it was assigned in the first span + assert c1[1] == c2[1] + + +def test_ctable_utf8_groupby_many_byte_lengths_and_non_ascii(): + rng = np.random.default_rng(3) + pool = ["", "a", "bb", "café", "日本語のテキスト", "x" * 2000, "münchen"] + names = [pool[i] for i in rng.integers(0, len(pool), 3000)] + t = make_table(names) + t.x[:] = np.ones(3000) + g = t.group_by("name").sum("x") + got = dict(zip(g["name"][:].tolist(), g["x_sum"][:].tolist(), strict=True)) + exp: dict[str, int] = {} + for v in names: + exp[v] = exp.get(v, 0) + 1 + assert got == exp + + +def test_ctable_utf8_groupby_multi_key_negative_int(): + """Composite-int key packing must survive negative integer keys.""" + + @dataclass + class NegRow: + ikey: int = blosc2.field(blosc2.int64()) + ukey: str = blosc2.field(blosc2.utf8()) + val: float = blosc2.field(blosc2.float64()) + + ik = [-5, -5, 3, 3, -5, 3] + uk = ["a", "b", "a", "b", "a", "a"] + t = CTable(NegRow, new_data={"ikey": ik, "ukey": uk, "val": [1.0] * 6}) + g = t.group_by(["ikey", "ukey"]).sum("val") + got = { + (int(i), u): v + for i, u, v in zip(g["ikey"][:], g["ukey"][:].tolist(), g["val_sum"][:].tolist(), strict=True) + } + assert got == {(-5, "a"): 2.0, (-5, "b"): 1.0, (3, "a"): 2.0, (3, "b"): 1.0} + + +def test_ctable_utf8_groupby_multi_key_float_fallback(): + """A float co-key forces the structured-dtype packing path with utf8 codes.""" + + @dataclass + class FloatRow: + fkey: float = blosc2.field(blosc2.float64()) + ukey: str = blosc2.field(blosc2.utf8()) + val: float = blosc2.field(blosc2.float64()) + + fk = [0.5, 0.5, 1.5, 1.5, 0.5] + uk = ["a", "b", "a", "a", "a"] + t = CTable(FloatRow, new_data={"fkey": fk, "ukey": uk, "val": [1.0] * 5}) + g = t.group_by(["fkey", "ukey"]).sum("val") + got = { + (f, u): v + for f, u, v in zip( + g["fkey"][:].tolist(), g["ukey"][:].tolist(), g["val_sum"][:].tolist(), strict=True + ) + } + assert got == {(0.5, "a"): 2.0, (0.5, "b"): 1.0, (1.5, "a"): 2.0} + + def test_ctable_utf8_groupby_sum(): t = make_table(["a", "b", "a", "b", "a"]) t.x[:] = [1.0, 2.0, 3.0, 4.0, 5.0] From 564180b106937dbd5411da4669f39658bf6dbbec Mon Sep 17 00:00:00 2001 From: Francesc Alted Date: Thu, 16 Jul 2026 23:53:08 +0200 Subject: [PATCH 09/20] Add bench_string_kinds.py: utf8() vs string() vs vlstring() comparison Benchmarks the three plain-string representations on the real Chicago-taxi company column (1e7 rows, low cardinality) and synthetic high-cardinality free text (2e6 rows, 0-129 chars, multi-byte): ingest, storage footprint, full read, equality filter, groupby key, sort, and Arrow export. Unsupported operations are reported, not silently skipped. Measured numbers and their reading are recorded in the phase-3 plan: utf8 matches vlstring's storage while making filters/groupby/sort work at all, and beats fixed-width string on footprint (7-13x smaller uncompressed), groupby keys, and Arrow export; fixed-width keeps winning raw reads/filters/sorts via the vectorized fast paths. The plan also records the natural follow-up: routing comparisons through Utf8Factorizer codes to close the filter gap. Co-Authored-By: Claude Fable 5 --- bench/ctable/bench_string_kinds.py | 135 +++++++++++++++++++++++++++++ plans/enhancing-ctable-phase3.md | 51 +++++++++++ 2 files changed, 186 insertions(+) create mode 100644 bench/ctable/bench_string_kinds.py diff --git a/bench/ctable/bench_string_kinds.py b/bench/ctable/bench_string_kinds.py new file mode 100644 index 000000000..2b51848af --- /dev/null +++ b/bench/ctable/bench_string_kinds.py @@ -0,0 +1,135 @@ +####################################################################### +# Copyright (c) 2019-present, Blosc Development Team +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause +####################################################################### + +"""Compare the three plain-string column representations head to head: +``utf8()`` (offsets + bytes, StringDType reads), ``string(max_length=L)`` +(fixed-width UTF-32), and ``vlstring()`` (msgpack cells). + +Two workloads: +- "taxi company": the real ``company`` column from the Chicago taxi dataset + (medium-length, low-cardinality strings), when the parquet file is present. +- "synthetic free text": high-cardinality random words of 0-60 chars with + some multi-byte values — the workload utf8() is designed for. + +For each representation: ingest, storage footprint, full read, equality +filter, groupby-key aggregation, sort, and Arrow export. Operations a +representation does not support are reported as such rather than skipped +silently. +""" + +import pathlib +import time +from dataclasses import make_dataclass + +import numpy as np + +import blosc2 +from blosc2 import CTable + +N_TAXI = 10_000_000 +N_SYNTH = 2_000_000 # fixed-width U~130 at 1e7 rows would need a ~5 GB ingest buffer +REPS = 3 +TAXI_PARQUET = pathlib.Path(__file__).parent.parent / "chicago-taxi" / "chicago-taxi-flat.parquet" + +rng = np.random.default_rng(42) + + +def bench(label, fn, reps=REPS): + times = [] + result = None + for _ in range(reps): + t0 = time.perf_counter() + result = fn() + times.append(time.perf_counter() - t0) + print(f" {label:34s} {min(times) * 1e3:9.1f} ms") + return result + + +def load_taxi_company(n): + import pyarrow.parquet as pq + + tbl = pq.read_table(TAXI_PARQUET, columns=["company"]) + col = tbl.column("company").combine_chunks() + values = col.to_pylist()[:n] + return [v if v is not None else "" for v in values] + + +def synth_free_text(n): + words = np.array( + ["taxi", "río", "航空", "boulevard", "x" * 40, "café", "", "downtown", "zürich", "o'hare"] + ) + # 2-6 words per row, high cardinality via a row counter suffix on ~half. + parts = words[rng.integers(0, len(words), (n, 3))] + joined = [" ".join(p) for p in parts] + salt = rng.integers(0, 100_000, n) # ~100k distinct values: high cardinality, sane group count + return [f"{s} #{salt[i]}" if i % 2 else s for i, s in enumerate(joined)] + + +def run_workload(title, values, filter_value): + n = len(values) + max_len = max(len(v) for v in values) + float_vals = rng.random(n) + print(f"\n=== {title} ({n:.0e} rows, max length {max_len} chars) ===") + + specs = { + "utf8": blosc2.utf8(), + "string": blosc2.string(max_length=max_len), + "vlstring": blosc2.vlstring(), + } + for kind, spec in specs.items(): + row_cls = make_dataclass( + "Row", [("s", str, blosc2.field(spec)), ("val", float, blosc2.field(blosc2.float64()))] + ) + print(f"[{kind}]") + t0 = time.perf_counter() + t = CTable(row_cls) + t.extend({"s": values, "val": float_vals}, validate=False) + t._flush_varlen_columns() + print(f" {'ingest':34s} {(time.perf_counter() - t0) * 1e3:9.1f} ms") + + col = t._cols["s"] + nbytes = getattr(col, "nbytes", None) + cbytes = getattr(col, "cbytes", None) + if nbytes is None: # NDArray-backed fixed-width column + nbytes, cbytes = col.schunk.nbytes, col.schunk.cbytes + print( + f" {'storage nbytes -> cbytes':34s} {nbytes / 2**20:7.1f} MB -> {cbytes / 2**20:7.1f} MB (cratio {nbytes / cbytes:.1f}x)" + ) + + bench("full column read", lambda t=t: t["s"][:]) + try: + bench("filter: count(s == value)", lambda t=t: int((t.s == filter_value)[:].sum())) + except (NotImplementedError, TypeError) as exc: + print(f" {'filter: count(s == value)':34s} unsupported: {str(exc)[:60]}") + try: + bench("groupby key: sum(val)", lambda t=t: t.group_by("s").sum("val"), reps=1) + except (NotImplementedError, TypeError) as exc: + print(f" {'groupby key: sum(val)':34s} unsupported: {str(exc)[:60]}") + try: + bench("sort_by(s) (copy)", lambda t=t: t.sort_by("s"), reps=1) + except (NotImplementedError, TypeError) as exc: + print(f" {'sort_by(s) (copy)':34s} unsupported: {str(exc)[:60]}") + try: + bench("to_arrow()", lambda t=t: t.to_arrow(), reps=1) + except Exception as exc: + print(f" {'to_arrow()':34s} failed: {str(exc)[:60]}") + del t + + +if TAXI_PARQUET.exists(): + print("loading taxi company column...", flush=True) + taxi = load_taxi_company(N_TAXI) + # the most frequent company value as the filter probe + vals, counts = np.unique(np.array(taxi, dtype=np.dtypes.StringDType()), return_counts=True) + run_workload("chicago-taxi company", taxi, str(vals[np.argmax(counts)])) + del taxi +else: + print(f"({TAXI_PARQUET} not found; skipping the real-data workload)") + +print("building synthetic free text...", flush=True) +synth = synth_free_text(N_SYNTH) +run_workload("synthetic free text", synth, synth[123]) diff --git a/plans/enhancing-ctable-phase3.md b/plans/enhancing-ctable-phase3.md index 662182dd1..81236fa6a 100644 --- a/plans/enhancing-ctable-phase3.md +++ b/plans/enhancing-ctable-phase3.md @@ -456,6 +456,57 @@ after the post-review fixes below). Benchmark gate now PASSES.** co-key (composite packing), multi-key with float co-key (structured fallback). Full `tests/` (7,489) passes. +**Measured comparison: utf8() vs string() vs vlstring() (2026-07-16, +`bench/ctable/bench_string_kinds.py`, Apple-silicon dev box).** Two +workloads: the real Chicago-taxi `company` column (1e7 rows, ~60 distinct +values, ≤44 chars) and synthetic high-cardinality free text (2e6 rows, +~1e6 distinct, 0–129 chars, multi-byte). + +| taxi company, 1e7 rows | utf8() | string(44) | vlstring() | +|------------------------|--------|------------|------------| +| ingest | 3477 ms | 491 ms | 1167 ms | +| storage, uncompressed | 259 MB | 1760 MB | 191 MB | +| storage, compressed | 1.3 MB | 1.1 MB | 0.4 MB | +| full column read | 2368 ms | 293 ms | 1238 ms | +| filter `s == value` | 1819 ms | 75 ms | unsupported | +| groupby key, sum | **963 ms** | 2477 ms | unsupported | +| sort_by (copy) | 7712 ms | 2395 ms | unsupported | +| to_arrow() | 39.3 s | 83.5 s | 79.9 s | + +| synthetic free text, 2e6 rows | utf8() | string(129) | vlstring() | +|-------------------------------|--------|-------------|------------| +| ingest | 2017 ms | 388 ms | 949 ms | +| storage, uncompressed | 78 MB | 1032 MB | 64 MB | +| storage, compressed | 12.0 MB | 19.2 MB | 11.3 MB | +| full column read | 712 ms | 108 ms | 366 ms | +| filter `s == value` | 538 ms | 45 ms | unsupported | +| groupby key, sum | **4408 ms** | 4962 ms | unsupported | +| sort_by (copy) | 3526 ms | 2584 ms | unsupported | +| to_arrow() | 1962 ms | 3995 ms | 3767 ms | + +Reading of the numbers, recorded so the positioning is evidence-backed: + +- vs `vlstring()`: comparable storage, but utf8 is *capable* — filters, + groupby keys, and sort are supported at all; vlstring rejects them. +- vs `string(max_length)`: utf8 is 7–13x smaller uncompressed (fixed-width + pays 4 bytes/char × max length on every row), smaller compressed on + high-cardinality text, faster as a groupby key (the factorization above), + and ~2x faster to Arrow. Fixed-width keeps winning raw reads, filters, + and sorts because those stay on the vectorized numexpr/numpy fast paths. +- The utf8 read/filter gap is the documented `_read_persisted_span` + per-row decode loop: ~1.8 s of the 1e7-row filter is decoding, not + comparing. **Natural follow-up (not started):** route comparisons + through the `Utf8Factorizer` the way groupby keys go — compare the D + distinct values against the operand, then map codes → boolean mask — + which should make low-cardinality utf8 filters competitive with + fixed-width. Same idea would speed sort-key materialization. +- `to_arrow()` at 1e7 rows is slow for *all three* kinds (39–84 s): that + is the pre-existing 2048-row `iter_arrow_batches` batching re-reading + storage chunks per batch, not a string-representation issue. +- Guidance the numbers support: `dictionary()` for low-cardinality, + `string(max_length)` for short bounded strings where filter speed rules, + `utf8()` for long/variable/high-cardinality text. + **P3.e Sort.** Lift the sort rejection (grep the `sort_by` varlen guard in `ctable.py`): `sort_by` on utf8 uses `np.argsort` on the StringDType array (chunked merge if the existing sort machinery is chunked — study From 507b9e9bbf63d362f1c275f6679e7d136cd0fc5f Mon Sep 17 00:00:00 2001 From: Francesc Alted Date: Fri, 17 Jul 2026 00:30:33 +0200 Subject: [PATCH 10/20] Keep NumPy 1.26 working: vlstring fallback for Arrow string imports, skip utf8 tests MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit utf8 columns require numpy.dtypes.StringDType (NumPy >= 2.0), but the Arrow/Parquet import default had started routing every scalar string column to blosc2.utf8(), which raised on older NumPy — a functional regression for the minimum supported version, and test_utf8.py failed at collection there (module-level StringDType() call). - New utf8_array.have_string_dtype() helper. - from_arrow/from_parquet: on NumPy < 2.0, scalar string columns rejoin the varlen-scalar group and import as vlstring with native-None nulls — the exact pre-utf8 behavior (no auto null sentinel is derived, and a column_null_values override is rejected with the standard varlen message, now reading "vlbytes/vlstring"). - tests/ctable/test_utf8.py skips at module level on NumPy < 2.0; the Arrow/Parquet/CLI tests that assert utf8-specific behavior now assert the vlstring fallback on old NumPy instead of being skipped, keeping interop coverage on the minimum-NumPy CI job. Verified by simulating NumPy-without-StringDType in-process (delattr before import): the four affected test files pass there (204 passed, utf8 module skipped), and the full tests/ctable suite stays green on NumPy 2.x. Co-Authored-By: Claude Fable 5 --- src/blosc2/ctable.py | 22 ++++++++++- src/blosc2/utf8_array.py | 5 +++ tests/ctable/test_arrow_interop.py | 13 +++++-- tests/ctable/test_dictionary_column.py | 8 +++- tests/ctable/test_parquet_interop.py | 54 ++++++++++++++++++++------ tests/ctable/test_utf8.py | 3 ++ 6 files changed, 86 insertions(+), 19 deletions(-) diff --git a/src/blosc2/ctable.py b/src/blosc2/ctable.py index 57b35984c..1bb7450ef 100644 --- a/src/blosc2/ctable.py +++ b/src/blosc2/ctable.py @@ -6959,6 +6959,13 @@ def _arrow_type_to_spec( # noqa: C901 if pa_type in (pa.string(), pa.large_string(), pa.utf8(), pa.large_utf8()): if string_max_length is None: + from blosc2.utf8_array import have_string_dtype + + if not have_string_dtype(): + # utf8 columns need numpy.dtypes.StringDType (NumPy >= 2.0). + # On older NumPy, keep the historical import behavior: + # variable-length msgpack strings with native-None nulls. + return b2s.vlstring(nullable=nullable) # No fixed-width threshold given: store as a variable-length # utf8 column (offsets + bytes, StringDType reads). return b2s.utf8(nullable=nullable, null_value=null_value) @@ -7021,12 +7028,23 @@ def _compiled_columns_from_arrow( # Scalar strings without a fixed-width threshold import as utf8 # columns, which use null sentinels like other scalar columns; # only binary columns keep the native-None varlen treatment. + # On NumPy < 2.0 (no StringDType) utf8 columns are unavailable and + # scalar strings keep the historical vlstring treatment instead. + from blosc2.utf8_array import have_string_dtype + field_is_varlen_scalar = ( not field_is_list and not field_is_struct and not field_is_dictionary and column_string_max_length is None - and (pa.types.is_binary(field.type) or pa.types.is_large_binary(field.type)) + and ( + pa.types.is_binary(field.type) + or pa.types.is_large_binary(field.type) + or ( + not have_string_dtype() + and (pa.types.is_string(field.type) or pa.types.is_large_string(field.type)) + ) + ) ) field_needs_object_fallback = cls._arrow_type_needs_object_fallback(pa, field.type) if field_needs_object_fallback and not object_fallback: @@ -7042,7 +7060,7 @@ def _compiled_columns_from_arrow( ) if has_null_value_override and field_is_varlen_scalar: raise TypeError( - f"column_null_values is not supported for vlbytes column {name!r}; " + f"column_null_values is not supported for vlbytes/vlstring column {name!r}; " "these columns represent nulls as native None." ) if has_null_value_override: diff --git a/src/blosc2/utf8_array.py b/src/blosc2/utf8_array.py index c6ce745c1..0f76c838a 100644 --- a/src/blosc2/utf8_array.py +++ b/src/blosc2/utf8_array.py @@ -82,6 +82,11 @@ def _factorize_byte_rows(mat: np.ndarray) -> tuple[np.ndarray, np.ndarray]: return rep_rows, inverse +def have_string_dtype() -> bool: + """True when the installed NumPy provides ``StringDType`` (NumPy >= 2.0).""" + return hasattr(np.dtypes, "StringDType") + + def string_dtype(): """Return a ``numpy.dtypes.StringDType`` instance, or raise if unavailable.""" try: diff --git a/tests/ctable/test_arrow_interop.py b/tests/ctable/test_arrow_interop.py index 42b20c719..5a97d1cb8 100644 --- a/tests/ctable/test_arrow_interop.py +++ b/tests/ctable/test_arrow_interop.py @@ -307,12 +307,19 @@ def test_from_arrow_all_numeric_types(): def test_from_arrow_string_default_is_utf8(): - """Without string_max_length, scalar string columns become utf8 (variable-length).""" + """Without string_max_length, scalar string columns become utf8 (variable-length). + + On NumPy < 2.0 (no StringDType) they fall back to vlstring instead. + """ at = pa.table({"name": pa.array(["hi", "hello world", "!"], type=pa.string())}) t = CTable.from_arrow(at.schema, at.to_batches()) assert t["name"].is_varlen_scalar - assert t["name"].is_utf8 - assert t["name"].dtype == np.dtypes.StringDType() + if hasattr(np.dtypes, "StringDType"): + assert t["name"].is_utf8 + assert t["name"].dtype == np.dtypes.StringDType() + else: + assert not t["name"].is_utf8 + assert t["name"].dtype is None assert list(t["name"][:]) == ["hi", "hello world", "!"] diff --git a/tests/ctable/test_dictionary_column.py b/tests/ctable/test_dictionary_column.py index 86acf4be3..542c2f268 100644 --- a/tests/ctable/test_dictionary_column.py +++ b/tests/ctable/test_dictionary_column.py @@ -494,7 +494,7 @@ def test_cli_preserves_dict_by_default(tmp_path): def test_cli_decode_dictionaries_flag(tmp_path): from blosc2.cli.parquet_to_blosc2 import main - from blosc2.schema import Utf8Spec + from blosc2.schema import Utf8Spec, VLStringSpec path = tmp_path / "dict.parquet" out = tmp_path / "dict_decoded.b2d" @@ -506,7 +506,11 @@ def test_cli_decode_dictionaries_flag(tmp_path): assert main(["--decode-dictionaries", str(path), str(out)]) == 0 ct = CTable.open(str(out), mode="r") - assert isinstance(ct._schema.columns_by_name["vendor"].spec, Utf8Spec) + from blosc2.utf8_array import have_string_dtype + + # Decoded strings become utf8 columns on NumPy >= 2.0, vlstring on older NumPy. + expected_spec = Utf8Spec if have_string_dtype() else VLStringSpec + assert isinstance(ct._schema.columns_by_name["vendor"].spec, expected_spec) assert list(ct["vendor"][:]) == ["Uber", "Lyft", "Uber"] ct.close() diff --git a/tests/ctable/test_parquet_interop.py b/tests/ctable/test_parquet_interop.py index 777b4bb1e..16676c74a 100644 --- a/tests/ctable/test_parquet_interop.py +++ b/tests/ctable/test_parquet_interop.py @@ -18,6 +18,10 @@ from blosc2 import CTable from blosc2.schema import ObjectSpec, StructSpec +# Scalar Arrow strings import as utf8 on NumPy >= 2.0 (StringDType available) +# and fall back to vlstring (native-None nulls) on older NumPy. +HAVE_STRING_DTYPE = hasattr(np.dtypes, "StringDType") + pa = pytest.importorskip("pyarrow") pq = pytest.importorskip("pyarrow.parquet") @@ -423,12 +427,17 @@ def test_utf8_arrow_roundtrip_no_singleton_list(self): at = pa.table({"txt": pa.array(["short", long_str, None, "end"], type=pa.string())}) t = CTable.from_arrow(at.schema, at.to_batches()) assert t["txt"].is_varlen_scalar - assert t["txt"].is_utf8 - nv = t["txt"].null_value - assert list(t["txt"][:]) == ["short", long_str, nv, "end"] - # Export back to Arrow → still a scalar string column, not list out = t.to_arrow() - assert pa.types.is_large_string(out.schema.field("txt").type) + if HAVE_STRING_DTYPE: + assert t["txt"].is_utf8 + nv = t["txt"].null_value + assert list(t["txt"][:]) == ["short", long_str, nv, "end"] + # Export back to Arrow → still a scalar string column, not list + assert pa.types.is_large_string(out.schema.field("txt").type) + else: + assert not t["txt"].is_utf8 # vlstring fallback, native-None nulls + assert list(t["txt"][:]) == ["short", long_str, None, "end"] + assert pa.types.is_string(out.schema.field("txt").type) assert out.column("txt").to_pylist() == ["short", long_str, None, "end"] def test_vlbytes_arrow_roundtrip_no_singleton_list(self): @@ -456,14 +465,21 @@ def test_utf8_parquet_roundtrip(self, tmp_path): t = CTable.from_parquet(path) assert t["txt"].is_varlen_scalar - assert t["txt"].is_utf8 - nv = t["txt"].null_value - assert list(t["txt"][:]) == ["short", long_str, nv, "end"] + if HAVE_STRING_DTYPE: + assert t["txt"].is_utf8 + nv = t["txt"].null_value + assert list(t["txt"][:]) == ["short", long_str, nv, "end"] + else: + assert not t["txt"].is_utf8 # vlstring fallback, native-None nulls + assert list(t["txt"][:]) == ["short", long_str, None, "end"] out = tmp_path / "utf8_out.parquet" t.to_parquet(out) rt = pq.read_table(out) - assert pa.types.is_large_string(rt.schema.field("txt").type) + if HAVE_STRING_DTYPE: + assert pa.types.is_large_string(rt.schema.field("txt").type) + else: + assert pa.types.is_string(rt.schema.field("txt").type) assert rt.column("txt").to_pylist() == ["short", long_str, None, "end"] @@ -534,8 +550,14 @@ def test_auto_nullable_scalars_roundtrip(self, tmp_path): out = tmp_path / "nullable_scalars_out.parquet" t.to_parquet(out) rt = pq.read_table(out) - # "s" round-trips as utf8 -> large_string, unlike the other columns. - assert rt.schema == at.schema.set(at.schema.get_field_index("s"), pa.field("s", pa.large_string())) + # "s" round-trips as utf8 -> large_string, unlike the other columns + # (on NumPy < 2.0 it stays vlstring and exports as plain string). + expected_schema = ( + at.schema.set(at.schema.get_field_index("s"), pa.field("s", pa.large_string())) + if HAVE_STRING_DTYPE + else at.schema + ) + assert rt.schema == expected_schema assert rt.to_pylist() == at.to_pylist() def test_null_policy_controls_default_sentinels(self): @@ -599,9 +621,17 @@ def test_null_policy_rejects_vlbytes_column_null_values(self): CTable.from_arrow(at.schema, at.to_batches()) def test_null_policy_column_null_values_applies_to_utf8(self): - """Passing column_null_values for a utf8 (scalar string) column sets its sentinel.""" + """Passing column_null_values for a utf8 (scalar string) column sets its sentinel. + + On NumPy < 2.0 utf8 columns are unavailable, strings import as + vlstring, and the override is rejected like any varlen column. + """ at = pa.table({"s": pa.array(["a", None, "c"], type=pa.string())}) policy = blosc2.NullPolicy(column_null_values={"s": "NA"}) + if not HAVE_STRING_DTYPE: + with blosc2.null_policy(policy), pytest.raises(TypeError, match="native None"): + CTable.from_arrow(at.schema, at.to_batches()) + return with blosc2.null_policy(policy): t = CTable.from_arrow(at.schema, at.to_batches()) assert t["s"].null_value == "NA" diff --git a/tests/ctable/test_utf8.py b/tests/ctable/test_utf8.py index 5f934c062..fb5b6fd2e 100644 --- a/tests/ctable/test_utf8.py +++ b/tests/ctable/test_utf8.py @@ -18,6 +18,9 @@ import blosc2 from blosc2 import CTable +if not hasattr(np.dtypes, "StringDType"): + pytest.skip("utf8 columns require NumPy >= 2.0 (StringDType)", allow_module_level=True) + STRING_DTYPE = np.dtypes.StringDType() From 3a7d9a3d1b73c856d1f31da3e8c25a0514d79a94 Mon Sep 17 00:00:00 2001 From: Francesc Alted Date: Fri, 17 Jul 2026 06:17:56 +0200 Subject: [PATCH 11/20] utf8: byte-level scalar comparisons (U1.a + U1.b) Route ==, !=, <, <=, >, >= against a str scalar through a pure-NumPy byte-level comparison on Utf8Array's raw offsets/bytes, skipping the per-row decode-to-StringDType loop entirely for filters. Column-vs-Column comparisons keep the existing materializing path. Measured on the 1e7-row taxi company column: s == value drops from ~1900ms to 162ms, ordering ops land at ~367ms (both within plan gates). Also refactors arrow_slice's inline sentinel-null matcher to share the new equal_mask_span helper instead of duplicating the algorithm. --- plans/utf8-reads-filter-optim.md | 416 +++++++++++++++++++++++++++++++ src/blosc2/ctable.py | 91 +++++-- src/blosc2/utf8_array.py | 86 ++++++- tests/ctable/test_utf8.py | 129 ++++++++++ 4 files changed, 694 insertions(+), 28 deletions(-) create mode 100644 plans/utf8-reads-filter-optim.md diff --git a/plans/utf8-reads-filter-optim.md b/plans/utf8-reads-filter-optim.md new file mode 100644 index 000000000..922a82e7e --- /dev/null +++ b/plans/utf8-reads-filter-optim.md @@ -0,0 +1,416 @@ +# utf8 columns: closing the read/filter gap (incremental plan) + +**Status:** U1 (U1.a + U1.b) LANDED 2026-07-17; U2 NOT STARTED. Successor work to +`plans/enhancing-ctable-phase3.md` (P3 + its P3.d follow-up + post-review +fixes, all landed on branch `enhancing-ctable3`, commits `0cd1ca78`, +`77bf321d`, `564180b1`). Read that plan's P3 sections first if you need +background on what a utf8 column *is*; this plan assumes it. + +**Audience:** an implementing model/developer who has NOT read the +discussions that produced this plan. Everything needed is in this file. +When in doubt, prefer the laziest change that satisfies the acceptance +criteria — do not add abstractions, options, or protocols this plan does +not ask for. + +**Practical notes (carried over from phase 3, still true, verified +2026-07-17):** + +- Locate code by **symbol name with grep**, never by line number. +- Run Python/pytest through the `blosc2` conda env: + `conda run -n blosc2 python -m pytest ...` (or + `/Users/faltet/miniforge3/envs/blosc2/bin/python`). Never use the repo + `.venv` (stale). +- Editing `.pyx` files does NOT trigger a rebuild in the editable install. + This is why item U1 below must be pure NumPy and why item U2 (the one + C-level item) carries an explicit build-workflow warning. +- **Docstrings and code comments must be self-contained.** Never reference + this plan, phase labels ("U1", "P3"), or benchmark history from source, + tests, or bench scripts — state the semantics directly. Explicit + maintainer rule. +- utf8 tests live in `tests/ctable/test_utf8.py`; match neighboring style. +- The dev machine is an Apple-silicon Mac; benchmark targets assume it. +- The benchmark harness for this plan is + `bench/ctable/bench_string_kinds.py` (real Chicago-taxi `company` + column at 1e7 rows + synthetic high-cardinality free text at 2e6 rows). + All "measured" numbers below come from it or from prototypes run against + the same 1e7-row taxi column on 2026-07-16/17. + +--- + +## The problem, quantified + +Every utf8 bulk read funnels through `Utf8Array._read_persisted_span` +(`src/blosc2/utf8_array.py`), which ends in a per-row Python loop: + +```python +out = np.empty(n, dtype=StringDType()) +for i in range(n): + out[i] = blob[rel[i] : rel[i + 1]].decode("utf-8") +``` + +Each iteration pays two int64→int conversions, a `bytes` slice +allocation, a `str.decode` allocation, and a StringDType `__setitem__` +(arena copy) — ~200–230 ns/row. Fetching the offsets and byte-blob slices +for the whole 1e7-row taxi column costs only ~64 ms; the loop costs +~2,300 ms. **~97% of a utf8 full read is this loop, not I/O.** + +Measured on the taxi `company` column (1e7 rows, ~60 distinct values, +≤44 chars; `bench_string_kinds.py`): + +| operation | utf8() today | string(44) | why utf8 is slower | +|------------------------|--------------|------------|---------------------------| +| full column read | 2368 ms | 293 ms | the decode loop | +| filter `s == value` | 1819–2035 ms | 75 ms | decode loop + numexpr gap | +| sort_by (copy) | 7712 ms | 2395 ms | decode loop feeds sort keys | + +groupby keys and dense-table Arrow export were already fixed (phase 3's +P3.d follow-up: `Utf8Factorizer`; post-review fixes: `arrow_slice`) — +both bypass the decode loop. Filters, full reads, sort-key +materialization, and column-vs-column comparisons still pay it. + +**Root constraint:** NumPy provides no bulk constructor for a +`StringDType` array from an offsets+bytes buffer pair. StringDType +manages a private arena with per-element small-string optimization; the +only *Python-level* ways in are element assignment or conversion from +another array, both element-at-a-time. So the strategy is: (U1) stop +materializing StringDType where it isn't needed at all — filters; (U2) +build the missing bulk constructor at C level for the cases where a +StringDType array genuinely must exist; (U3, parked) push string +predicates into miniexpr only if a real workload later proves it pays. + +--- + +## Design decisions already made (do not re-litigate) + +1. **Filters do NOT go through `Utf8Factorizer`.** This was prototyped + and measured on 2026-07-17 (1e7-row taxi column, most-frequent-value + probe, 2.34e6 hits), all results verified equal to ground truth: + + | path | time | speedup | + |----------------------------------------|----------|---------| + | current (`_utf8_chunked_bool` + decode)| 2035 ms | — | + | factorizer-based (fresh vocabulary) | 764 ms | 2.7x | + | **byte-level compare (no decode)** | **156 ms** | **13x** | + | fixed-width `string()` reference | 75 ms | — | + + The factorizer pays for building a vocabulary (hash + verify + sort of + new values) that a filter uses exactly once, and it *regresses below + the current path* on high-cardinality columns where nearly every row + is a new value. Filters need one boolean per row, not value + identities. **A note in `plans/enhancing-ctable-phase3.md` ("route + comparisons through the `Utf8Factorizer`…") predates this measurement + and is superseded by this plan — leave that file as is, it is a + historical record.** + +2. **U1 is pure NumPy.** No Cython/C for filters: the byte-level path is + already fully vectorized (whole-array ops only), a C version would buy + maybe 2x more while adding build friction, and the repo's editable + install does not rebuild `.pyx`. Precedent: + `_factorize_fixed_width_str` and `Utf8Factorizer` solve the same class + of problem the same way. + +3. **Byte-lexicographic order on UTF-8 bytes equals Unicode code-point + order.** This is a designed property of UTF-8 encoding and it is what + makes ordering comparisons (`<`, `<=`, `>`, `>=`) implementable on raw + bytes without decoding. Python `str` comparison is code-point order, + so byte-lex results match Python/StringDType semantics exactly. (The + same property already justifies `Utf8Factorizer`'s rank codes.) + +4. **Null semantics are frozen.** A null (sentinel) value on either side + never satisfies any comparison — SQL `WHERE` semantics, pinned by + existing tests in `tests/ctable/test_utf8.py` ("Comparisons and + filtering" section). Every U1 path must reproduce this bit-for-bit, + including the corner where the probe *equals* the sentinel (the result + must then be all-False for `==`, because every matching row is by + definition null). + +5. **Existing observable behavior is the contract.** All current + comparison tests must pass unchanged. The result stays a + physical-length boolean `blosc2.NDArray` already intersected with the + live-row mask, exactly as `Column._utf8_compare` returns today. + +--- + +## U1 — Pure-NumPy byte-level scalar comparisons + +### U1.a Equality (`==`, `!=`) against a `str` scalar + +**Where:** a new method on `Utf8Array` (`src/blosc2/utf8_array.py`), plus +wiring in `Column._utf8_compare` (`src/blosc2/ctable.py`, grep for +`def _utf8_compare`). + +**Algorithm** (this is the measured 156 ms prototype, reproduce it +faithfully): + +```python +def equal_mask_span(self, value: str, a: int, b: int) -> np.ndarray: + """Boolean mask for rows [a, b): row bytes == value's UTF-8 bytes.""" + enc = value.encode("utf-8") + length = len(enc) + target = np.frombuffer(enc, dtype=np.uint8) + offs = np.asarray(self._offsets[a : b + 1], dtype=np.int64) + rel = offs - int(offs[0]) + data = np.asarray(self._data[int(offs[0]) : int(offs[-1])]) + lengths = np.diff(rel) + mask = lengths == length # length must match first + if length and mask.any(): + idx = rel[np.flatnonzero(mask)] # start offset of each candidate + hit = np.ones(len(idx), dtype=np.bool_) + for i in range(length): # `length` whole-array compares + hit &= data[idx] == target[i] + idx = idx + 1 + cand = np.flatnonzero(mask) + mask[cand[~hit]] = False + return mask +``` + +Key properties to preserve: + +- The inner loop runs `len(probe_bytes)` times (e.g. 25 for a 25-char + ASCII probe), each iteration a C-speed whole-array gather+compare over + the *candidates only* — never a per-row Python loop. +- `length == 0` (empty-string probe) is handled by the `lengths == 0` + mask alone. +- Comparison is on raw bytes, so NUL-bearing and multi-byte values are + exact by construction (no NumPy `S`-dtype trailing-NUL truncation, no + decode). +- The candidate index vector is *reused and incremented in place* + (`idx = idx + 1` creates one new array per byte position; that is + fine — the point is never materializing a `(k, L)` int64 index matrix). +- **Pending rows:** call `self.flush()` at the start of the public entry + point (precedent: `Utf8Factorizer.__init__` and `factorize_span` flush; + it is a no-op unless there are buffered rows, and read-only tables + cannot have any). + +**Wiring in `Column._utf8_compare`:** the scalar branch currently builds +the predicate via `self._utf8_chunked_bool(fn)` where `fn` compares a +materialized StringDType chunk. Replace only the *scalar-operand* case: + +- `numpy_op is np.equal` → chunked `equal_mask_span`. +- `numpy_op is np.not_equal` → complement of the equality mask **within + the logical length** (rows past the logical length must stay False in + the physical-length result — mirror how `_utf8_chunked_bool` zero-fills + beyond `n_logical`), then null exclusion as below. +- Null fusing: the sentinel-null mask is just `equal_mask_span(nv, ...)` + over the same span — compute both masks per chunk and combine + (`eq & ~null` / `ne & ~null`), keeping the current single-pass shape. + When the probe equals the sentinel the two masks are identical and the + fused result is all-False — that is the required behavior, add a test + for it (one probably exists; extend it if it only covers `==`). +- Column-vs-Column operands keep the existing StringDType path untouched. +- The final assembly (`blosc2.asarray(raw) & self._lazy_valid_rows()`) + stays exactly as is. + +Also refactor `arrow_slice`'s inline sentinel-null matcher (grep +`nv_enc` in `utf8_array.py`) to call the shared helper — same algorithm, +currently duplicated. Behavior must not change (its tests pin it). + +### U1.b Ordering (`<`, `<=`, `>`, `>=`) against a `str` scalar + +**Algorithm:** per-byte vectorized lexicographic compare against the +probe's bytes, grouped by row byte-length (the same grouping loop +`Utf8Array.factorize_span` uses — bincount on `np.diff(rel)`, then one +iteration per distinct length; distinct lengths are few in practice and +each row is touched once regardless). + +For a length group with row length `L` and probe byte length `P`, +`m = min(L, P)`: + +``` +undecided = all rows in group # bytes equal so far +lt = gt = all-False +for i in 0 .. m-1: + b = data[start_of_row + i] # vectorized gather, index vector reused + lt |= undecided & (b < target[i]) + gt |= undecided & (b > target[i]) + undecided &= (b == target[i]) +# rows still undecided are byte-prefix-equal over m bytes: +# L < P → row is a strict prefix of probe → row < probe (lt) +# L > P → probe is a strict prefix of row → row > probe (gt) +# L == P → row == probe (eq) +``` + +Then per operator: `<` = lt; `<=` = lt | eq; `>` = gt; `>=` = gt | eq. +Null exclusion fuses exactly as in U1.a (`pred & ~null_mask`). + +Edge cases that MUST have dedicated tests because they are where this +algorithm goes wrong if implemented sloppily: + +- probe is a strict prefix of a value and vice versa ("Taxi" vs + "Taxi Affiliation"), including at length-group boundaries; +- empty-string probe (everything except "" is `>` it; "" is `==`); +- empty-string rows vs non-empty probe; +- multi-byte values ordered across byte-length boundaries (e.g. "z" < + "é" < "日" must hold: code points 0x7A < 0xE9 < 0x65E5, and their + UTF-8 encodings byte-compare in the same order — assert against + Python's own `<` on the str values, which is the ground truth); +- NUL-bearing values; +- probe equal to the sentinel on a nullable column (all four ordering + ops must exclude null rows, and rows *equal* to the sentinel are the + null rows). + +**Testing strategy for U1 overall (do all of these):** + +1. Keep every existing test in `tests/ctable/test_utf8.py` green, + untouched. +2. Add a randomized differential test: build ~5,000 rows mixing ASCII / + multi-byte / empty / NUL-bearing / multi-KB values plus the sentinel, + then for each of the six operators and a set of probes (present value, + absent value, prefix of a value, empty, sentinel) assert the CTable + filter result row-for-row against the pure-Python ground truth + computed with list comprehensions on the original values (NOT against + `np.unique`/StringDType helpers — NumPy has a known bug collapsing + StringDType values that differ only after an embedded NUL; ground + truth is Python semantics). +3. Test through a view (`t.head(n)[pred]`) and on a table with deleted + rows, since the mask is physical-length and intersected with the + live-row mask. +4. Run the full suite: `tests/ctable` and then all of `tests/`. + +**Benchmark gate (record actuals in this file when landing):** with +`bench_string_kinds.py` on the 1e7-row taxi column, filter +`s == ` must land **≤ 250 ms** (prototype: 156 ms; +current: ~1,900 ms). Ordering ops have no prototype; expect roughly 2–3x +the equality cost from the per-byte loop bound by the probe length — +gate at **≤ 700 ms**. If an implementation misses its gate, profile +before adding cleverness; the prototype numbers prove the budget exists. + +**LANDED 2026-07-17** (Apple-silicon Mac, `bench_string_kinds.py` on the +1e7-row taxi `company` column, probe = most frequent value = "Taxi +Affiliation Services"): + +| op | time | gate | +|-------------------|----------|---------| +| `s == value` | 162.3 ms | ≤250 ms | +| `s < value` | 368.9 ms | ≤700 ms | +| `s <= value` | 366.5 ms | ≤700 ms | +| `s > value` | 366.9 ms | ≤700 ms | +| `s >= value` | 367.8 ms | ≤700 ms | + +Both gates pass. `!=` shares the equality code path (one extra `~`, no +measurable difference). Ordering ops were timed with a standalone script +(same taxi data, same probe) since `bench_string_kinds.py` only exercises +`==`; full `tests/` suite green (7496 passed) with the change, plus a new +randomized differential test and dedicated edge-case tests in +`tests/ctable/test_utf8.py`. + +**Explicitly out of scope for U1:** Column-vs-Column comparisons (keep +the current StringDType path; U2 speeds them up for free), +`startswith`/`endswith`/`np.strings` LazyExpr ops (they materialize reads +— U2's territory), and string-syntax expressions +(`t.where("name == 'x'")` — still guarded, U3's territory). + +--- + +## U2 — C-level bulk StringDType constructor (fixes full reads) + +**What:** a small compiled kernel that builds a StringDType array +directly from the (offsets, bytes) pair, replacing +`_read_persisted_span`'s per-row loop. This is the one place where +per-element work is irreducible at the Python level, and it fixes in one +stroke everything U1 does not: full column reads, sort-key +materialization, Column-vs-Column comparisons, `np.strings` ops, repr +previews — every consumer of `_read_persisted_span`. + +**How (sketch, verify against current NumPy docs before writing code):** +NumPy ≥ 2.0 exposes a C API for StringDType packing — grep the NumPy +headers/docs for `NpyString_pack`, `NpyString_acquire_allocator`, +`npy_string_allocator`. The kernel is essentially: + +``` +acquire allocator for the destination StringDType descriptor +for i in 0 .. n-1: # C loop, ~tens of ns/row + NpyString_pack(allocator, &out[i], data + rel[i], rel[i+1] - rel[i]) +release allocator +``` + +- Input buffers: the *relative* offsets (`int64`, length n+1, rel[0]==0) + and the byte blob slice — exactly what `_read_persisted_span` already + computes before its loop. The bytes were produced by `str.encode` on + write, so they are valid UTF-8 by construction; the kernel packs bytes, + it does not validate. +- Home: a new small `.pyx` (e.g. alongside the existing `indexing_ext` / + `groupby_ext` sources — follow how those are registered in the CMake + build; grep `indexing_ext` in `CMakeLists.txt`). +- **Build-workflow warning:** the editable dev install does NOT rebuild + `.pyx` files. Developing this requires a real rebuild (see how the + existing extensions are built; budget for that friction). This is the + reason U2 is sequenced after U1 and not merged with it. +- **Graceful degradation is mandatory:** `_read_persisted_span` tries the + kernel and falls back to the current Python loop when the extension is + unavailable (WASM builds, source installs without the toolchain, NumPy + API drift). The fallback path must remain tested — parametrize the + existing roundtrip tests over kernel-on/kernel-off if feasible (e.g. a + monkeypatch fixture forcing the fallback). + +**Expected benefit:** decode cost drops from ~230 ns/row (Python loop) to +~20–40 ns/row (C loop + arena copy): taxi full read 2368 ms → roughly +200–400 ms. Sort and col-vs-col compares inherit proportionally. + +**Benchmark gate:** `bench_string_kinds.py` taxi full-column read +**≤ 500 ms** (from 2368 ms), and no regression anywhere else in that +script. Record actuals here. + +**Acceptance:** full `tests/` green with the kernel active AND with the +fallback forced; benchmark gate met; `blosc2.utf8()` still importable and +fully functional on a NumPy-only environment without the compiled +extension. + +--- + +## U3 — miniexpr varlen-string support (PARKED — do not start) + +**What it would be:** teach miniexpr (the C expression engine at +`~/blosc/miniexpr`, used by LazyExpr for fused chunk-at-a-time +evaluation) to evaluate predicates on utf8 columns natively, enabling +single-pass fused expressions like `(t.name == "Paris") & (t.fare > 10)` +with block pruning, string-syntax `t.where("name == 'Paris'")`, and +C-speed `startswith`/`contains` inside larger expressions. + +**Why it is parked (facts verified 2026-07-17 against the miniexpr +sources):** miniexpr already has `ME_STRING`, but it is *fixed-width* — a +string variable carries a per-element `itemsize`, and the evaluator, +blocking/threading logic, and all three JIT backends (libtcc, cc, +wasm32) address element `i` as `base + i * itemsize`. Variable-length +support means: + +1. a new dual-buffer variable kind (int64 offsets + byte blob) threaded + through the evaluator and all three JIT code generators; +2. bridge plumbing on the blosc2 side that feeds *synchronized pairs* of + chunks — and the utf8 data blob's chunk boundaries are byte-aligned, + not row-aligned, so per-block operand preparation needs offset reads + to even locate the byte range (this misalignment is the genuinely hard + part, it has no counterpart in the current bridge); +3. varlen lex-compare kernels (the easy part). + +Cross-repo, weeks-scale. Meanwhile the composability half of the fused +story already works without it: `Column._utf8_compare` returns a boolean +NDArray that combines with other predicates via `&`/`|`/`~`; what is +lost is only single-pass evaluation of the string leg. + +**Unpark criteria (any one):** + +- after U1 lands, profiling a real fused-query workload shows the string + predicate leg dominating (say >50% of wall time of a representative + mixed query), OR +- a product requirement lands for string-syntax `where()` / DSL kernels + over utf8 columns. + +**If unparked:** U1's byte-compare semantics (including the UTF-8 +byte-order property and the null fusing rules) are the reference +semantics for the C kernels — nothing built in U1 is throwaway. + +--- + +## Sequencing and cross-cutting rules + +- Land order: **U1.a → U1.b → U2**. U3 stays parked. U1.a alone is + already the highest value-for-effort item (13x measured, ~half a day). +- Each item lands separately with its gate recorded in this file + (numbers, machine, command), following the phase-3 convention: + if a gate fails, do not merge the fast path — record the number and + the root cause here and stop. +- Never regress `string()`/`vlstring()` behavior or performance; the + guard is `bench_string_kinds.py` plus the full test suite. +- No new public API: everything here is internal (`Utf8Array` methods, + `Column._utf8_compare` internals, an optional compiled helper). diff --git a/src/blosc2/ctable.py b/src/blosc2/ctable.py index 1bb7450ef..283c16c3b 100644 --- a/src/blosc2/ctable.py +++ b/src/blosc2/ctable.py @@ -2013,16 +2013,33 @@ def _utf8_chunked_bool(self, fn, *, chunk_size: int = 65536) -> np.ndarray: result[start:stop] = fn(arr[start:stop], start, stop) return result + def _utf8_chunked_bytes(self, fn, *, chunk_size: int = 65536) -> np.ndarray: + """Apply ``fn(arr, start, stop)`` over this utf8 column's logical rows. + + Like :meth:`_utf8_chunked_bool`, but *fn* operates directly on the + underlying :class:`~blosc2.utf8_array.Utf8Array` (raw offsets/bytes) + instead of a materialized ``StringDType`` chunk, so no per-row decode + happens. Returns a physical-length boolean NumPy array; rows beyond + the column's logical length are left ``False``. + """ + arr = self._raw_col + n_phys = len(self._table._valid_rows) + n_logical = len(arr) + result = np.zeros(n_phys, dtype=np.bool_) + for start in range(0, n_logical, chunk_size): + stop = min(start + chunk_size, n_logical) + result[start:stop] = fn(arr, start, stop) + return result + def _utf8_compare(self, numpy_op, other): - """Chunked-numpy comparison predicate for a utf8 column. + """Comparison predicate for a utf8 column. Compares against a Python ``str`` scalar or another utf8 - :class:`Column` (element-wise), evaluating chunk by chunk since - numexpr/miniexpr cannot operate on ``StringDType`` arrays. Returns a - physical-length boolean ``blosc2.NDArray``, already intersected with - this column's live-row mask. A null value on either side never - satisfies any comparison (SQL ``WHERE`` semantics), matching - :meth:`_null_aware_compare` for every other column kind. + :class:`Column` (element-wise). Returns a physical-length boolean + ``blosc2.NDArray``, already intersected with this column's live-row + mask. A null value on either side never satisfies any comparison + (SQL ``WHERE`` semantics), matching :meth:`_null_aware_compare` for + every other column kind. """ if isinstance(other, Column): if not other.is_utf8: @@ -2030,20 +2047,24 @@ def _utf8_compare(self, numpy_op, other): f"Column {self._col_name!r} is a utf8 column; it can only be compared with a " f"str or another utf8 Column, got Column {other._col_name!r}." ) - other_arr = other._raw_col - elif isinstance(other, str): - other_arr = None - else: - raise TypeError( - f"Column {self._col_name!r} is a utf8 column; it can only be compared with a str " - f"or another utf8 Column, got {type(other).__name__!r}." - ) + return self._utf8_compare_column(numpy_op, other) + if isinstance(other, str): + return self._utf8_compare_scalar(numpy_op, other) + raise TypeError( + f"Column {self._col_name!r} is a utf8 column; it can only be compared with a str " + f"or another utf8 Column, got {type(other).__name__!r}." + ) + def _utf8_compare_column(self, numpy_op, other: Column): + """Column-vs-Column comparison, evaluated chunk by chunk on decoded + ``StringDType`` values since numexpr/miniexpr cannot operate on them. + """ + other_arr = other._raw_col nv = self.null_value - other_nv = other.null_value if isinstance(other, Column) else None + other_nv = other.null_value def fn(chunk, start, stop): - rhs = other if other_arr is None else other_arr[start:stop] + rhs = other_arr[start:stop] res = numpy_op(chunk, rhs) if nv is not None: res &= chunk != nv @@ -2054,6 +2075,42 @@ def fn(chunk, start, stop): raw = self._utf8_chunked_bool(fn) return blosc2.asarray(raw) & self._lazy_valid_rows() + def _utf8_compare_scalar(self, numpy_op, value: str): + """Scalar comparison, evaluated chunk by chunk directly on raw UTF-8 + bytes (no decode to ``StringDType``) via + :meth:`~blosc2.utf8_array.Utf8Array.equal_mask_span` / + :meth:`~blosc2.utf8_array.Utf8Array.order_masks_span`. + """ + nv = self.null_value + + if numpy_op in (np.equal, np.not_equal): + + def fn(arr, start, stop): + res = arr.equal_mask_span(value, start, stop) + if numpy_op is np.not_equal: + res = ~res + if nv is not None: + res &= ~arr.equal_mask_span(nv, start, stop) + return res + else: + + def fn(arr, start, stop): + lt, gt = arr.order_masks_span(value, start, stop) + if numpy_op is np.less: + res = lt + elif numpy_op is np.less_equal: + res = ~gt + elif numpy_op is np.greater: + res = gt + else: # np.greater_equal + res = ~lt + if nv is not None: + res = res & ~arr.equal_mask_span(nv, start, stop) + return res + + raw = self._utf8_chunked_bytes(fn) + return blosc2.asarray(raw) & self._lazy_valid_rows() + def _dictionary_eq(self, other): """Return a physical-slot boolean predicate for dictionary equality. diff --git a/src/blosc2/utf8_array.py b/src/blosc2/utf8_array.py index 0f76c838a..783a4de68 100644 --- a/src/blosc2/utf8_array.py +++ b/src/blosc2/utf8_array.py @@ -478,6 +478,80 @@ def factorize_span(self, a: int, b: int) -> tuple[np.ndarray, np.ndarray]: rank[order] = np.arange(len(order)) return rank[codes], uniques[order] + def equal_mask_span(self, value: str, a: int, b: int) -> np.ndarray: + """Boolean mask for persisted rows ``[a, b)``: row bytes == *value*'s UTF-8 bytes. + + Compares raw bytes with no decode: rows are first filtered by byte + length, then the surviving candidates are compared byte-by-byte with + whole-array gathers (one comparison per byte position of *value*). + """ + self.flush() + enc = value.encode("utf-8") + length = len(enc) + target = np.frombuffer(enc, dtype=np.uint8) + offs = np.asarray(self._offsets[a : b + 1], dtype=np.int64) + rel = offs - int(offs[0]) + data = np.asarray(self._data[int(offs[0]) : int(offs[-1])]) + lengths = np.diff(rel) + mask = lengths == length + if length and mask.any(): + cand = np.flatnonzero(mask) + idx = rel[cand] + hit = np.ones(len(idx), dtype=np.bool_) + for i in range(length): + hit &= data[idx] == target[i] + idx = idx + 1 + mask[cand[~hit]] = False + return mask + + def order_masks_span(self, value: str, a: int, b: int) -> tuple[np.ndarray, np.ndarray]: + """Return ``(lt, gt)`` boolean masks comparing rows ``[a, b)`` to *value*. + + Byte-lexicographic comparison, which equals Unicode code-point order + for valid UTF-8. A row where neither mask is True is equal to + *value*. Rows are grouped by byte length (each row touched once); + within a group, bytes are compared position by position against + *value*'s bytes with whole-array gathers. + """ + self.flush() + target = np.frombuffer(value.encode("utf-8"), dtype=np.uint8) + probe_len = len(target) + n = b - a + lt = np.zeros(n, dtype=np.bool_) + gt = np.zeros(n, dtype=np.bool_) + if n == 0: + return lt, gt + offs = np.asarray(self._offsets[a : b + 1], dtype=np.int64) + rel = offs - int(offs[0]) + data = np.asarray(self._data[int(offs[0]) : int(offs[-1])]) + lengths = np.diff(rel) + starts = rel[:-1] + if int(lengths.max()) <= 65536: + distinct_lengths = np.flatnonzero(np.bincount(lengths)) + else: + distinct_lengths = np.unique(lengths) + for length in distinct_lengths: + length = int(length) + rows = np.flatnonzero(lengths == length) + m = min(length, probe_len) + undecided = np.ones(len(rows), dtype=np.bool_) + row_lt = np.zeros(len(rows), dtype=np.bool_) + row_gt = np.zeros(len(rows), dtype=np.bool_) + idx = starts[rows] + for i in range(m): + byte = data[idx] + row_lt |= undecided & (byte < target[i]) + row_gt |= undecided & (byte > target[i]) + undecided &= byte == target[i] + idx = idx + 1 + if length < probe_len: + row_lt |= undecided # row is a strict byte-prefix of value + elif length > probe_len: + row_gt |= undecided # value is a strict byte-prefix of row + lt[rows] = row_lt + gt[rows] = row_gt + return lt, gt + def arrow_slice(self, pa, a: int, b: int, null_value: str | None = None): """Persisted rows ``[a, b)`` as a ``pyarrow.LargeStringArray``. @@ -495,17 +569,7 @@ def arrow_slice(self, pa, a: int, b: int, null_value: str | None = None): validity = None null_count = 0 if null_value is not None and n > 0: - nv_enc = null_value.encode("utf-8") - lengths = np.diff(rel) - if nv_enc: - mask = np.zeros(n, dtype=np.bool_) - cand = np.flatnonzero(lengths == len(nv_enc)) - if len(cand): - spans = data[rel[cand][:, None] + np.arange(len(nv_enc))] - hits = (spans == np.frombuffer(nv_enc, dtype=np.uint8)).all(axis=1) - mask[cand[hits]] = True - else: - mask = lengths == 0 + mask = self.equal_mask_span(null_value, a, b) null_count = int(mask.sum()) if null_count: validity = pa.array(~mask).buffers()[1] diff --git a/tests/ctable/test_utf8.py b/tests/ctable/test_utf8.py index fb5b6fd2e..5e0fab9aa 100644 --- a/tests/ctable/test_utf8.py +++ b/tests/ctable/test_utf8.py @@ -470,6 +470,135 @@ class Mixed: t.name == t.other # noqa: B015 +def test_ctable_utf8_scalar_comparison_differential(): + """Every scalar comparison (byte-level, no decode) must match Python + string semantics row-for-row, across ASCII, multi-byte, empty, + NUL-bearing, and multi-KB values, plus a mix of null rows. + + Ground truth is computed with Python's own operators on the original + values (not via np.unique/StringDType helpers, which have a known bug + merging strings that differ only after an embedded NUL). + """ + import operator + + pool = [ + "hello", + "", + "a", + "café", + "日本語のテキスト", + "z", + "é", + "日", + "Taxi", + "Taxi Affiliation", + "nul\x00in", + "nul\x00INSIDE", + "x" * 5000, + "y" * 5000 + "!", + ] + n = 5000 + rng = np.random.default_rng(11) + values = [pool[i] for i in rng.integers(0, len(pool), n)] + null_positions = rng.choice(n, size=n // 20, replace=False) + data = list(values) + for i in null_positions: + data[i] = None + + t = CTable(NullableRow, new_data={"name": data, "x": list(range(n))}) + nv = t["name"].null_value + + probes = { + "present": "café", + "absent": "not_in_pool_ZZZ", + "prefix": "Taxi", + "empty": "", + "sentinel": nv, + } + ops = { + "eq": (operator.eq, lambda c, p: c == p), + "ne": (operator.ne, lambda c, p: c != p), + "lt": (operator.lt, lambda c, p: c < p), + "le": (operator.le, lambda c, p: c <= p), + "gt": (operator.gt, lambda c, p: c > p), + "ge": (operator.ge, lambda c, p: c >= p), + } + + for probe_name, probe in probes.items(): + for op_name, (py_op, col_op) in ops.items(): + expected = [v for v in data if v is not None and py_op(v, probe)] + got = list(t[col_op(t.name, probe)]["name"][:]) + assert got == expected, f"op={op_name} probe={probe_name!r} mismatch" + + +def test_ctable_utf8_ordering_prefix_edge_cases(): + """A probe that is a strict prefix of a value, and vice versa, at + length-group boundaries.""" + t = make_table(["Taxi", "Taxi Affiliation", "Taxicab", "Tax"]) + assert list(t[t.name < "Taxi"]["name"][:]) == ["Tax"] + assert list(t[t.name > "Taxi"]["name"][:]) == ["Taxi Affiliation", "Taxicab"] + assert list(t[t.name == "Taxi"]["name"][:]) == ["Taxi"] + assert list(t[t.name <= "Taxi"]["name"][:]) == ["Taxi", "Tax"] + assert list(t[t.name >= "Taxi"]["name"][:]) == ["Taxi", "Taxi Affiliation", "Taxicab"] + + +def test_ctable_utf8_ordering_empty_string_probe(): + """Everything except "" is > the empty-string probe; "" is == it.""" + t = make_table(["", "a", "zzz"]) + assert list(t[t.name == ""]["name"][:]) == [""] + assert list(t[t.name > ""]["name"][:]) == ["a", "zzz"] + assert list(t[t.name < ""]["name"][:]) == [] + assert list(t[t.name >= ""]["name"][:]) == ["", "a", "zzz"] + + +def test_ctable_utf8_ordering_multibyte_byte_length_boundaries(): + """1-, 2-, and 3-byte UTF-8 encodings must byte-compare in code-point + order (code points 0x7A < 0xE9 < 0x65E5).""" + assert "z" < "é" < "日" + t = make_table(["日", "z", "é"]) + s = t.sort_by("name") + assert list(s["name"][:]) == ["z", "é", "日"] + assert list(t[t.name < "é"]["name"][:]) == ["z"] + # filtering preserves original row order (日, z, é), not sorted order + assert list(t[t.name > "z"]["name"][:]) == ["日", "é"] + + +def test_ctable_utf8_ordering_nul_bearing_values(): + # Ground truth: "nul\x00INSIDE" < "nul\x00in" < ... at the byte position + # right after the embedded NUL ('I' = 0x49 < 'i' = 0x69). + assert sorted(["nul\x00in", "nul\x00INSIDE", "nul"]) == ["nul", "nul\x00INSIDE", "nul\x00in"] + t = make_table(["nul\x00in", "nul\x00INSIDE", "nul"]) # original row order + assert list(t[t.name == "nul\x00in"]["name"][:]) == ["nul\x00in"] + assert list(t[t.name < "nul\x00in"]["name"][:]) == ["nul\x00INSIDE", "nul"] + assert list(t[t.name > "nul"]["name"][:]) == ["nul\x00in", "nul\x00INSIDE"] + + +def test_ctable_utf8_ordering_probe_equals_sentinel(): + """All four ordering ops must exclude null rows even when the probe is + the sentinel value itself (rows equal to the sentinel are the null + rows).""" + t = CTable(NullableRow, new_data={"name": ["alpha", None, "zeta"], "x": [1, 2, 3]}) + nv = t["name"].null_value + for pred in (t.name < nv, t.name <= nv, t.name > nv, t.name >= nv): + got = list(t[pred]["name"][:]) + assert nv not in got + assert None not in got + + +def test_ctable_utf8_scalar_comparison_view_and_deleted_rows(): + """The predicate mask is physical-length; it must stay correct through a + view and after rows have been deleted (live-row mask intersection).""" + t = make_table(["paris", "london", "paris", "tokyo", "berlin", "paris"]) + head_view = t.head(4) + assert list(head_view[head_view.name == "paris"]["name"][:]) == ["paris", "paris"] + assert list(head_view[head_view.name < "london"]["name"][:]) == [] + + t.delete([0, 2]) # removes two of the three "paris" rows + assert list(t["name"][:]) == ["london", "tokyo", "berlin", "paris"] + assert list(t[t.name == "paris"]["name"][:]) == ["paris"] + assert list(t[t.name != "paris"]["name"][:]) == ["london", "tokyo", "berlin"] + + def test_ctable_utf8_startswith_endswith(): t = make_table(["hello", "help", "world"]) started = blosc2.startswith(t.name, "hel").compute() From 45ec7906552fcc0a140520e3faf5d0f27cd72093 Mon Sep 17 00:00:00 2001 From: Francesc Alted Date: Fri, 17 Jul 2026 06:53:15 +0200 Subject: [PATCH 12/20] utf8: C-level bulk StringDType constructor for full reads (U2) Add a Cython kernel (utf8_ext.pyx, pack_utf8_span) that uses NumPy's StringDType C API (NpyString_pack) to bulk-fill a StringDType array directly from offsets+bytes, replacing the per-row decode loop in Utf8Array._read_persisted_span. Falls back to the old Python loop when the compiled extension is unavailable (lazy _pack_utf8_kernel() import, mirroring the existing groupby_ext pattern); tests cover both paths via a monkeypatch fixture. Two more fixes were needed to actually realize the gain in the common read path: Column's identity-slice fast path excluded utf8 alongside other varlen-scalar kinds even though Utf8Array slices itself efficiently, and Utf8Array._get_many always paid a full StringDType scatter-copy (sort + fancy-index assignment) even for a plain contiguous range, which alone ate most of the kernel's benefit. Measured on the 1e7-row taxi company column: full column read drops from 2472.6ms to 165.3ms (gate was <=500ms), no regressions elsewhere. --- CMakeLists.txt | 16 ++++++- plans/utf8-reads-filter-optim.md | 63 ++++++++++++++++++++++++- src/blosc2/ctable.py | 11 ++++- src/blosc2/utf8_array.py | 34 ++++++++++++-- src/blosc2/utf8_ext.pyx | 79 ++++++++++++++++++++++++++++++++ tests/ctable/test_utf8.py | 59 ++++++++++++++++++++++++ 6 files changed, 255 insertions(+), 7 deletions(-) create mode 100644 src/blosc2/utf8_ext.pyx diff --git a/CMakeLists.txt b/CMakeLists.txt index ea305fb03..9b0294d06 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -59,6 +59,13 @@ add_custom_command( DEPENDS "${CMAKE_CURRENT_SOURCE_DIR}/src/blosc2/groupby_ext.pyx" VERBATIM) +add_custom_command( + OUTPUT utf8_ext.c + COMMAND Python::Interpreter -m cython + "${CMAKE_CURRENT_SOURCE_DIR}/src/blosc2/utf8_ext.pyx" --output-file utf8_ext.c + DEPENDS "${CMAKE_CURRENT_SOURCE_DIR}/src/blosc2/utf8_ext.pyx" + VERBATIM) + # ...and add it to the target Python_add_library(blosc2_ext MODULE blosc2_ext.c WITH_SOABI) target_sources(blosc2_ext PRIVATE src/blosc2/matmul_kernels.c) @@ -68,11 +75,17 @@ if(UNIX) endif() Python_add_library(indexing_ext MODULE indexing_ext.c WITH_SOABI) Python_add_library(groupby_ext MODULE groupby_ext.c WITH_SOABI) +Python_add_library(utf8_ext MODULE utf8_ext.c WITH_SOABI) +# NpyString_pack() and friends are part of NumPy's 2.0 C API; opt in +# explicitly since numpy/*.h otherwise targets an older API version by +# default for source compatibility. +target_compile_definitions(utf8_ext PRIVATE NPY_TARGET_VERSION=NPY_2_0_API_VERSION) # We need to link against NumPy target_link_libraries(blosc2_ext PRIVATE Python::NumPy) target_link_libraries(indexing_ext PRIVATE Python::NumPy) target_link_libraries(groupby_ext PRIVATE Python::NumPy) +target_link_libraries(utf8_ext PRIVATE Python::NumPy) # Fetch and build miniexpr library include(FetchContent) @@ -110,6 +123,7 @@ endif() target_compile_features(blosc2_ext PRIVATE c_std_11) target_compile_features(indexing_ext PRIVATE c_std_11) target_compile_features(groupby_ext PRIVATE c_std_11) +target_compile_features(utf8_ext PRIVATE c_std_11) if(WIN32 AND CMAKE_C_COMPILER_ID STREQUAL "Clang") execute_process( COMMAND "${CMAKE_C_COMPILER}" -print-resource-dir @@ -184,7 +198,7 @@ endif() # Python extension -> site-packages/blosc2 install( - TARGETS blosc2_ext indexing_ext groupby_ext + TARGETS blosc2_ext indexing_ext groupby_ext utf8_ext LIBRARY DESTINATION ${SKBUILD_PLATLIB_DIR}/blosc2 ) diff --git a/plans/utf8-reads-filter-optim.md b/plans/utf8-reads-filter-optim.md index 922a82e7e..e6b2d0e8e 100644 --- a/plans/utf8-reads-filter-optim.md +++ b/plans/utf8-reads-filter-optim.md @@ -1,6 +1,6 @@ # utf8 columns: closing the read/filter gap (incremental plan) -**Status:** U1 (U1.a + U1.b) LANDED 2026-07-17; U2 NOT STARTED. Successor work to +**Status:** U1 (U1.a + U1.b) and U2 LANDED 2026-07-17; U3 PARKED. Successor work to `plans/enhancing-ctable-phase3.md` (P3 + its P3.d follow-up + post-review fixes, all landed on branch `enhancing-ctable3`, commits `0cd1ca78`, `77bf321d`, `564180b1`). Read that plan's P3 sections first if you need @@ -356,6 +356,67 @@ fallback forced; benchmark gate met; `blosc2.utf8()` still importable and fully functional on a NumPy-only environment without the compiled extension. +**LANDED 2026-07-17** (Apple-silicon Mac, `bench_string_kinds.py`): + +- New `src/blosc2/utf8_ext.pyx` (`pack_utf8_span`): acquires a + `StringDType` allocator via `NpyString_acquire_allocator` and calls + `NpyString_pack` per row in a plain (GIL-held) C loop — Cython treats + `NpyString_pack` as GIL-requiring, so `with nogil` isn't available here. + Needs `NPY_TARGET_VERSION=NPY_2_0_API_VERSION` as a compile definition + (CMakeLists.txt `target_compile_definitions`); without it NumPy's + `numpy/*.h` headers gate the 2.0 string-C-API macros behind + `NPY_FEATURE_VERSION`, which otherwise defaults to an older, source + -compatible value and leaves `NpyString_pack` undeclared at compile time. + Registered in `CMakeLists.txt` exactly like `indexing_ext`/`groupby_ext` + (own `add_custom_command`, `Python_add_library`, link/install rules). + Measured at ~9 ns/row standalone (2e6-row synthetic column), matching + the plan's 20-40 ns/row estimate. +- `Utf8Array._read_persisted_span` (`utf8_array.py`) tries the kernel via + a new lazy `_pack_utf8_kernel()` helper (mirrors the + `try: from blosc2 import groupby_ext / except ImportError: return None` + pattern already used in `groupby.py`) and falls back to the old per-row + decode loop when it returns `None`. Tests force the fallback by + monkeypatching `blosc2.utf8_array._pack_utf8_kernel` + (`force_kernel_mode` fixture in `test_utf8.py`, parametrized + kernel/fallback). +- **Two extra fixes were needed to actually hit the gate** — the kernel + alone cut `_read_persisted_span` itself to ~170 ms for the 1e7-row + span, but the *observed* `t["s"][:]` benchmark stayed at ~730-750 ms + until both landed: + 1. `Column._values_from_key`'s slice fast-path (`ctable.py`) excluded + every `is_varlen_scalar` column, including utf8, from the + identity-position direct-slice shortcut, even though `Utf8Array` + slices itself efficiently. Changed the exclusion to + `is_varlen_scalar and not is_utf8`. + 2. The real bottleneck: `Utf8Array._get_many` (used whenever + `_has_identity_positions()` is false — the common case, since a + table's physical capacity is normally chunk-padded past its row + count) always sorted the index array and did a fancy-indexed + `out[order[...]] = span[...]` StringDType scatter-copy, even for a + plain ascending contiguous range. That scatter-copy is itself a + full per-element StringDType copy — exactly the cost U2 exists to + eliminate — so it silently ate the kernel's gain. Added a + contiguous-ascending-run check at the top of `_get_many` that + shortcuts straight to `_read_span` when `indices` is + `arange(indices[0], indices[-1] + 1)`, skipping the sort/cluster/ + scatter-copy machinery entirely. + +| operation (taxi, 1e7 rows) | before U2 | after U2 | gate | +|-----------------------------------|-----------|----------|---------| +| full column read | 2472.6 ms | 165.3 ms | ≤500 ms | +| filter `s == value` (U1, unchanged)| — | 168.6 ms | — | +| groupby key: sum(val) | 969.5 ms | 971.3 ms | no regression | +| sort_by(s) (copy) | 7874.4 ms | 3930.4 ms| no regression (improved — sort keys now read via the fast path) | +| to_arrow() | 40301.2 ms| 39992.2 ms| no regression | + +Synthetic free-text workload (2e6 rows): full column read 735.4 ms → 50.3 ms. + +Full `tests/` suite green with the kernel active (7506 passed) and with +the fallback forced (7505 passed, 1 unrelated pre-existing flaky failure — +a subprocess segfault in `test_dsl_kernel_scalar_constant_subexpr_runtime_no_segfault`, +a DSL/JIT kernel test with no connection to utf8 code; passes standalone). +Both gates pass. + --- ## U3 — miniexpr varlen-string support (PARKED — do not start) diff --git a/src/blosc2/ctable.py b/src/blosc2/ctable.py index 283c16c3b..f9f86c7c2 100644 --- a/src/blosc2/ctable.py +++ b/src/blosc2/ctable.py @@ -1204,9 +1204,16 @@ def _values_from_key(self, key, *, check_stale: bool = True): # noqa: C901 # underlying NDArray, skipping the O(nrows) live-position scan and # letting NDArray's strided-gather fast path handle coarse steps. # Plain stored columns only; everything else falls through to the - # position-gather path below. + # position-gather path below. utf8 is a varlen-scalar kind but + # Utf8Array slices itself efficiently (offsets+bytes span read), + # so it takes the fast path too instead of the index-gather one. if ( - not (self.is_computed or self.is_list or self.is_varlen_scalar or self.is_dictionary) + not ( + self.is_computed + or self.is_list + or self.is_dictionary + or (self.is_varlen_scalar and not self.is_utf8) + ) and self._has_identity_positions() ): return self._maybe_decode_timestamp_values(np.asarray(self._raw_col[key])) diff --git a/src/blosc2/utf8_array.py b/src/blosc2/utf8_array.py index 783a4de68..8cd9e9bfe 100644 --- a/src/blosc2/utf8_array.py +++ b/src/blosc2/utf8_array.py @@ -99,6 +99,20 @@ def string_dtype(): ) from None +def _pack_utf8_kernel(): + """Return the compiled bulk StringDType packer, or ``None``. + + ``None`` means the ``utf8_ext`` extension is unavailable (a source + install without a C toolchain, a WASM build, or a NumPy without the + StringDType C API); callers fall back to a pure-Python per-row loop. + """ + try: + from blosc2 import utf8_ext + except ImportError: + return None + return getattr(utf8_ext, "pack_utf8_span", None) + + def _new_backend_arrays(cparams=None, dparams=None, *, offsets_urlpath=None, data_urlpath=None): """Create fresh (offsets, data) NDArrays for an empty utf8 column.""" import blosc2 @@ -202,9 +216,16 @@ def _read_persisted_span(self, a: int, b: int) -> np.ndarray: return np.empty(0, dtype=self._dtype) offs = np.asarray(self._offsets[a : b + 1], dtype=np.int64) start, end = int(offs[0]), int(offs[-1]) - blob = np.asarray(self._data[start:end]).tobytes() if end > start else b"" rel = offs - start out = np.empty(n, dtype=self._dtype) + kernel = _pack_utf8_kernel() + if kernel is not None: + data = ( + np.ascontiguousarray(self._data[start:end]) if end > start else np.zeros(1, dtype=np.uint8) + ) + kernel(rel, data, out) + return out + blob = np.asarray(self._data[start:end]).tobytes() if end > start else b"" for i in range(n): out[i] = blob[rel[i] : rel[i + 1]].decode("utf-8") return out @@ -247,9 +268,16 @@ def _get_many(self, indices: np.ndarray) -> np.ndarray: indices = indices.ravel() n = len(self) indices = np.where(indices < 0, indices + n, indices).astype(np.int64, copy=False) - if len(indices) and (indices.min() < 0 or indices.max() >= n): + m = len(indices) + if m and (indices.min() < 0 or indices.max() >= n): raise IndexError("Utf8Array index out of range") - out = np.empty(len(indices), dtype=self._dtype) + if m and indices[-1] - indices[0] == m - 1 and bool((np.diff(indices) == 1).all()): + # A contiguous ascending run (e.g. a full-column read routed here + # via an index array rather than a step-1 slice) is just a span + # read -- skip the sort/cluster/scatter-copy gather below, which + # pays a full StringDType element-wise copy for no reason here. + return self._read_span(int(indices[0]), int(indices[-1]) + 1) + out = np.empty(m, dtype=self._dtype) np_rows = self._persisted_rows pending_mask = indices >= np_rows if pending_mask.any(): diff --git a/src/blosc2/utf8_ext.pyx b/src/blosc2/utf8_ext.pyx new file mode 100644 index 000000000..8473d83f6 --- /dev/null +++ b/src/blosc2/utf8_ext.pyx @@ -0,0 +1,79 @@ +####################################################################### +# Copyright (c) 2019-present, Blosc Development Team +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause +####################################################################### +# cython: boundscheck=False, wraparound=False, initializedcheck=False + +"""Bulk StringDType construction for utf8 columns. + +NumPy provides no bulk constructor for a ``StringDType`` array from an +offsets+bytes buffer pair -- the only Python-level ways in are per-element +assignment or conversion from another array. This kernel uses NumPy's C +API for ``StringDType`` (``NpyString_pack``) to fill every element of a +preallocated ``StringDType`` array directly from the raw offsets/bytes +representation, in a single C loop with no per-row Python object churn. +""" + +import numpy as np +cimport numpy as cnp + +from libc.stdint cimport int64_t, uint8_t + +cnp.import_array() + + +def pack_utf8_span(cnp.ndarray rel not None, cnp.ndarray data not None, cnp.ndarray out not None): + """Fill *out* in place with rows carved out of *data* using *rel*. + + *rel* is ``int64``, length ``len(out) + 1``, ``rel[0] == 0``: the + relative byte offset of each row within *data*. *data* is ``uint8`` + and holds valid UTF-8 bytes (this packs bytes, it does not validate + the encoding). *out* is a ``numpy.dtypes.StringDType`` array of length + ``len(rel) - 1``, already allocated by the caller. + """ + if rel.ndim != 1 or data.ndim != 1 or out.ndim != 1: + raise ValueError("rel, data and out must be 1-D arrays") + cdef Py_ssize_t n = out.shape[0] + if rel.shape[0] != n + 1: + raise ValueError("rel must have length len(out) + 1") + if rel.dtype != np.dtype(np.int64): + raise TypeError("rel must have dtype int64") + if data.dtype != np.dtype(np.uint8): + raise TypeError("data must have dtype uint8") + if not (rel.flags["C_CONTIGUOUS"] and data.flags["C_CONTIGUOUS"] and out.flags["C_CONTIGUOUS"]): + raise ValueError("rel, data and out must be C-contiguous") + if n == 0: + return + + cdef const int64_t* rel_ptr = cnp.PyArray_DATA(rel) + cdef const uint8_t* data_ptr = cnp.PyArray_DATA(data) + cdef char* out_data = cnp.PyArray_DATA(out) + cdef cnp.npy_intp itemsize = cnp.PyArray_ITEMSIZE(out) + cdef cnp.npy_string_allocator* allocator = cnp.NpyString_acquire_allocator( + cnp.PyArray_DESCR(out) + ) + if allocator == NULL: + raise TypeError("out must be a StringDType array") + + cdef Py_ssize_t i + cdef int64_t start, length + cdef int ret = 0 + try: + for i in range(n): + start = rel_ptr[i] + length = rel_ptr[i + 1] - start + ret = cnp.NpyString_pack( + allocator, + (out_data + i * itemsize), + (data_ptr + start), + length, + ) + if ret == -1: + break + finally: + cnp.NpyString_release_allocator(allocator) + + if ret == -1: + raise MemoryError("Failed to pack a UTF-8 row into the StringDType array") diff --git a/tests/ctable/test_utf8.py b/tests/ctable/test_utf8.py index 5e0fab9aa..707e20b54 100644 --- a/tests/ctable/test_utf8.py +++ b/tests/ctable/test_utf8.py @@ -162,6 +162,65 @@ def test_utf8_array_rejects_non_str(): arr.append(None) +# --------------------------------------------------------------------------- +# Bulk StringDType read: compiled kernel and its pure-Python fallback +# --------------------------------------------------------------------------- + + +@pytest.fixture(params=["kernel", "fallback"], ids=["kernel", "fallback"]) +def force_kernel_mode(request, monkeypatch): + """Exercise both the compiled bulk StringDType packer and its + pure-Python per-row fallback, so the fallback stays covered even on a + build where the compiled extension is available.""" + if request.param == "fallback": + monkeypatch.setattr("blosc2.utf8_array._pack_utf8_kernel", lambda: None) + return request.param + + +def test_utf8_array_bulk_read_kernel_and_fallback(force_kernel_mode): + from blosc2.utf8_array import Utf8Array + + arr = Utf8Array(blosc2.utf8()) + arr.extend(SAMPLE) + arr.flush() + got = arr[:] + assert got.dtype == STRING_DTYPE + assert list(got) == SAMPLE + + +def test_utf8_array_bulk_read_matches_python_ground_truth(force_kernel_mode): + """A wider mix of byte lengths and edge cases than SAMPLE: many distinct + ASCII/multi-byte/empty/NUL-bearing values, read back in one bulk span.""" + from blosc2.utf8_array import Utf8Array + + rng = np.random.default_rng(5) + pool = ["", "a", "café", "日本語", "x" * 5000, "nul\x00in", "nul\x00INSIDE", "emoji 🎉🚀"] + values = [pool[i] for i in rng.integers(0, len(pool), 3000)] + arr = Utf8Array(blosc2.utf8()) + arr.extend(values) + arr.flush() + assert list(arr[:]) == values + + +def test_ctable_utf8_extend_and_read_kernel_and_fallback(force_kernel_mode): + t = make_table() + values = t["name"][:] + assert values.dtype == STRING_DTYPE + assert list(values) == SAMPLE + + +@pytest.mark.parametrize("ext", [".b2z", ".b2d"]) +def test_ctable_utf8_persistence_roundtrip_kernel_and_fallback(tmp_path, ext, force_kernel_mode): + urlpath = str(tmp_path / f"utf8_kernel_mode{ext}") + t = make_table(urlpath=urlpath, mode="w") + t.close() + t2 = CTable.open(urlpath, mode="r") + try: + assert list(t2["name"][:]) == SAMPLE + finally: + t2.close() + + # --------------------------------------------------------------------------- # CTable integration: append / extend / read # --------------------------------------------------------------------------- From d5bad023901b3b4652b67b1150698aed51be9796 Mon Sep 17 00:00:00 2001 From: Francesc Alted Date: Fri, 17 Jul 2026 07:57:18 +0200 Subject: [PATCH 13/20] Clean NotImplementedError for create_index on utf8 columns Utf8Spec was missing from the create_index rejection list, so indexing a utf8 column failed deep inside with a misleading "indexes are only supported on NDArray" TypeError. Now it raises the same style of clear NotImplementedError that vlstring/vlbytes/struct/object columns get, pointing users to fixed-width string() when an index is needed. Co-Authored-By: Claude Fable 5 --- src/blosc2/ctable_indexing.py | 7 +++++++ tests/ctable/test_utf8.py | 6 ++++++ 2 files changed, 13 insertions(+) diff --git a/src/blosc2/ctable_indexing.py b/src/blosc2/ctable_indexing.py index a628b0cad..7ec5f7a37 100644 --- a/src/blosc2/ctable_indexing.py +++ b/src/blosc2/ctable_indexing.py @@ -25,6 +25,7 @@ NDArraySpec, ObjectSpec, StructSpec, + Utf8Spec, VLBytesSpec, VLStringSpec, ) @@ -749,6 +750,12 @@ def create_index( # noqa: C901 ) if isinstance(self._schema.columns_by_name[col_name].spec, ListSpec): raise ValueError(f"Cannot create an index on list column {col_name!r} in V1.") + if isinstance(self._schema.columns_by_name[col_name].spec, Utf8Spec): + raise NotImplementedError( + f"Cannot create an index on variable-length utf8 column {col_name!r}: " + "indexing for utf8 columns is not supported yet. " + "Use a fixed-width string(max_length=N) column if you need an index." + ) if isinstance( self._schema.columns_by_name[col_name].spec, (VLStringSpec, VLBytesSpec, StructSpec, ObjectSpec) ): diff --git a/tests/ctable/test_utf8.py b/tests/ctable/test_utf8.py index 707e20b54..ad6aef23f 100644 --- a/tests/ctable/test_utf8.py +++ b/tests/ctable/test_utf8.py @@ -991,6 +991,12 @@ def test_ctable_utf8_where_expression_raises_clearly(): t.where("name == 'hello'") +def test_ctable_utf8_create_index_raises_clearly(): + t = make_table() + with pytest.raises(NotImplementedError, match="utf8"): + t.create_index(col_name="name") + + def test_ctable_utf8_arrow_export_large_string(): pa = pytest.importorskip("pyarrow") t = make_table() From 1feab867d4011cd9707ebdc4051fb02fcc399c1d Mon Sep 17 00:00:00 2001 From: Francesc Alted Date: Fri, 17 Jul 2026 07:57:31 +0200 Subject: [PATCH 14/20] Document how to choose among string/utf8/dictionary/vlstring columns - New "Choosing a string column type" section in the CTable reference: decision path plus a comparison table covering storage layout, per-row cost, filters, groupby/sort, indexing, Arrow interop, and NumPy requirements. utf8 limitations (no string-expression filters, no create_index yet) are footnoted; utf8 added to the Text & binary listing and excluded from the unclassified members page. - Cross-link string()/utf8()/vlstring() docstrings to that section and state each spec's tradeoffs, including utf8's current limitations. - New examples/ctable/utf8_strings.py showing the full utf8 surface: StringDType bulk reads, operator filters, startswith, sentinel nulls, groupby keys, sorting, and Arrow round-trip. - Refresh stale docs that predate the utf8 import default: the parquet guide and arrow_interop.py said Arrow/Parquet strings import as vlstring/fixed-width; they now import as utf8 on NumPy >= 2.0. - Use utf8() for the text column in the getting-started CTable schema. Co-Authored-By: Claude Fable 5 --- doc/getting_started/overview.rst | 2 +- doc/guides/parquet_to_blosc2.rst | 9 +-- doc/reference/ctable.rst | 94 +++++++++++++++++++++++++++++++ doc/reference/misc.rst | 1 + examples/ctable/arrow_interop.py | 4 +- examples/ctable/utf8_strings.py | 86 ++++++++++++++++++++++++++++ examples/ctable/varlen_columns.py | 3 + src/blosc2/schema.py | 28 +++++++-- 8 files changed, 216 insertions(+), 11 deletions(-) create mode 100644 examples/ctable/utf8_strings.py diff --git a/doc/getting_started/overview.rst b/doc/getting_started/overview.rst index 65813a7cc..333b9d76c 100644 --- a/doc/getting_started/overview.rst +++ b/doc/getting_started/overview.rst @@ -270,7 +270,7 @@ types including integers, floats, booleans, and strings: tips: float = blosc2.field(blosc2.float32()) km: float = blosc2.field(blosc2.float32()) lon: float = blosc2.field(blosc2.float32()) - company: str = blosc2.field(blosc2.string(max_length=50)) + company: str = blosc2.field(blosc2.utf8()) # variable-length text t = blosc2.CTable(Row, expected_size=10_000_000) diff --git a/doc/guides/parquet_to_blosc2.rst b/doc/guides/parquet_to_blosc2.rst index 40014789d..859e09e59 100644 --- a/doc/guides/parquet_to_blosc2.rst +++ b/doc/guides/parquet_to_blosc2.rst @@ -57,10 +57,11 @@ Use the ``.b2d`` extension to produce a directory-backed (sparse) store: Step 4 — Fixed-width string import ------------------------------------ -By default, string columns are stored as variable-length strings -(``vlstring``). Pass ``--fixed-str-maxlen`` to pre-scan strings and store -columns whose maximum character length fits within the given limit as -fixed-width, indexable strings: +By default, string columns are stored as variable-length ``utf8`` columns +(Arrow-style offsets + bytes; falls back to ``vlstring`` on NumPy < 2.0). +Pass ``--fixed-str-maxlen`` to pre-scan strings and store columns whose +maximum character length fits within the given limit as fixed-width, +indexable strings: .. code-block:: console diff --git a/doc/reference/ctable.rst b/doc/reference/ctable.rst index c875f3ec3..a57b168e4 100644 --- a/doc/reference/ctable.rst +++ b/doc/reference/ctable.rst @@ -938,15 +938,109 @@ Text & binary .. autosummary:: string + utf8 bytes vlstring vlbytes .. autoclass:: string +.. autofunction:: utf8 .. autoclass:: bytes .. autofunction:: vlstring .. autofunction:: vlbytes +.. _ChoosingStringType: + +Choosing a string column type +----------------------------- + +CTable offers four ways to store strings. As a quick decision path: + +* Low-cardinality strings (categories, enumerations, repeated labels): + use :func:`dictionary` — repeated values are stored once as integer codes. +* Everything else (names, free text, high-cardinality values): + use :func:`utf8` — the recommended default for variable-length text. +* Short codes of near-uniform length, or columns that need + :meth:`CTable.create_index`: use :class:`string` (fixed-width). +* NumPy < 2.0, or nullable columns where any string value can legally occur + (native ``None`` nulls, no sentinel): use :func:`vlstring`. + +.. list-table:: + :header-rows: 1 + :stub-columns: 1 + + * - + - :class:`string` + - :func:`utf8` + - :func:`dictionary` + - :func:`vlstring` + * - Storage layout + - fixed-width UTF-32 NDArray + - int64 offsets + UTF-8 bytes (Arrow-style) + - integer codes + unique values + - msgpack batches + * - Per-row cost (pre-compression) + - 4 × ``max_length`` bytes + - exact UTF-8 length + 8-byte offset + - one integer code + - value + msgpack framing + * - Length limit + - ``max_length`` characters + - none + - none + - none + * - Bulk read returns + - NumPy ``U`` array + - NumPy ``StringDType`` array + - decoded strings + - Python list of ``str`` + * - Nulls + - sentinel + - sentinel + - native + - native ``None`` + * - Filters (``==``, ``<``, …) + - ✓ (incl. string expressions) + - ✓ operators only [#utf8expr]_ + - ``==`` / ``isin()`` only + - ✗ + * - :meth:`CTable.group_by` key / :meth:`CTable.sort_by` + - ✓ + - ✓ + - ✓ + - ✗ + * - :meth:`CTable.create_index` + - ✓ + - not yet + - ✓ (rank-based) + - ✗ + * - Arrow / Parquet + - ✓ + - ✓ (``large_string``) + - ✓ + - ✓ + * - NumPy requirement + - any + - >= 2.0 + - any + - any + * - Best for + - short, near-uniform codes + - **general text (recommended)** + - low-cardinality categories + - NumPy < 2.0; native-``None`` nulls + +.. [#utf8expr] utf8 columns support the operator form ``t[t.name == "x"]`` + (also ``!=``, ``<``, ``<=``, ``>``, ``>=``), but not the string-expression + form ``t.where("name == 'x'")`` yet. :meth:`CTable.create_index` on utf8 + columns is not supported yet either; both raise ``NotImplementedError`` + with a clear message. + +Note that a plain ``str`` annotation without an explicit :func:`field` spec +still maps to fixed-width ``string(max_length=32)`` for backward +compatibility; opt in to variable-length storage with +``blosc2.field(blosc2.utf8())``. + Array, encoded, and compound specs ---------------------------------- diff --git a/doc/reference/misc.rst b/doc/reference/misc.rst index 921520610..a9d4bfa31 100644 --- a/doc/reference/misc.rst +++ b/doc/reference/misc.rst @@ -74,6 +74,7 @@ public objects into the appropriate reference section. uint16, uint32, uint64, + utf8, vlbytes, vlstring, Array, diff --git a/examples/ctable/arrow_interop.py b/examples/ctable/arrow_interop.py index 9dc0ed9b2..d09f9086e 100644 --- a/examples/ctable/arrow_interop.py +++ b/examples/ctable/arrow_interop.py @@ -59,7 +59,9 @@ class Stock: t2 = blosc2.CTable.from_arrow(at2.schema, at2.to_batches()) print("CTable from Arrow (inferred schema):") print(t2) -print(f" label dtype: {t2['label'].dtype} (max_length inferred from data)") +# Arrow string columns import as variable-length utf8() columns (StringDType +# reads); pass string_max_length= to from_arrow() for fixed-width instead. +print(f" label dtype: {t2['label'].dtype}") # -- pandas round-trip ------------------------------------------------------ try: diff --git a/examples/ctable/utf8_strings.py b/examples/ctable/utf8_strings.py new file mode 100644 index 000000000..581e22a04 --- /dev/null +++ b/examples/ctable/utf8_strings.py @@ -0,0 +1,86 @@ +####################################################################### +# Copyright (c) 2019-present, Blosc Development Team +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause +####################################################################### + +# utf8(): variable-length string columns stored Arrow-style (int64 offsets + +# UTF-8 bytes). Each row costs exactly its encoded byte length — no +# max_length to pick, no truncation — and bulk reads materialize as NumPy +# StringDType arrays (requires NumPy >= 2.0). +# +# This is the recommended column type for general text (names, descriptions, +# high-cardinality values). For low-cardinality categories prefer +# blosc2.dictionary(); for short near-uniform codes, fixed-width +# blosc2.string(max_length=N) still applies. + +from dataclasses import dataclass + +import numpy as np + +import blosc2 + + +@dataclass +class Place: + # utf8 is never inferred from a plain `str` annotation (that still maps + # to fixed-width string(max_length=32)); it must be requested explicitly. + name: str = blosc2.field(blosc2.utf8()) + note: str = blosc2.field(blosc2.utf8(nullable=True)) + visitors: int = blosc2.field(blosc2.int64()) + + +data = { + "name": ["café", "O'Hare", "日本語のテキスト", "zürich", "", "boulevard of broken dreams " * 2], + "note": ["cozy", None, "multi-byte ok", None, "empty name above", "a very long name too"], + "visitors": [120, 85_000, 42, 300, 0, 7], +} +t = blosc2.CTable(Place, new_data=data) +print(f"table: {t.nrows} rows; name dtype: {t['name'].dtype}") + +# -- bulk reads return StringDType arrays ------------------------------------ +names = t["name"][:] +print(f"bulk read: {type(names).__name__} of dtype {names.dtype}") +print("lengths vary freely:", [len(s) for s in names]) + +# -- filtering: use the operator form ---------------------------------------- +# ==, !=, <, <=, >, >= against scalars or other utf8 columns are vectorized. +print("\nexact match:", t[t.name == "café"].nrows, "row(s)") +print("range (name >= 'p'):", t[t.name >= "p"].nrows, "row(s)") + +# blosc2.startswith / endswith also work on utf8 columns. +started = np.asarray(blosc2.startswith(t.name, "bo").compute()) +print("startswith 'bo':", int(started.sum()), "row(s)") + +# Note: the string-expression form t.where("name == 'café'") is not +# supported for utf8 columns yet, and neither is create_index(); both raise +# NotImplementedError. Use the operator form above, or a fixed-width +# string() column when an index is required. + +# -- nulls are sentinel-based ------------------------------------------------- +# Unlike vlstring (native None), nullable utf8 picks a sentinel string from +# the active null policy (or takes an explicit null_value=). Reads surface +# the sentinel verbatim; use is_null()/fillna()/dropna() for null-aware work. +print("\nnull mask:", list(t["note"].is_null())) +print("fillna:", list(t["note"].fillna(""))[:3]) + +# -- groupby keys and sorting ------------------------------------------------- +by_name = t.group_by("name", sort=True) +print("\nvisitors per name:") +print(by_name.sum("visitors")) + +print("\nsorted by name (multi-byte values sort by Unicode code point):") +print(t.sort_by("name")[["name", "visitors"]]) + +# -- Arrow interop ------------------------------------------------------------- +try: + import pyarrow as pa # noqa: F401 + + at = t.to_arrow() + print(f"\nArrow export: name -> {at.schema.field('name').type}") # large_string + # And Arrow string columns import back as utf8 columns automatically. + t2 = blosc2.CTable.from_arrow(at.schema, at.to_batches()) + print(f"round-trip dtype: {t2['name'].dtype}") +except ImportError: + print("\npyarrow not installed — skipping Arrow round-trip demo.") diff --git a/examples/ctable/varlen_columns.py b/examples/ctable/varlen_columns.py index 1a6661332..949893c87 100644 --- a/examples/ctable/varlen_columns.py +++ b/examples/ctable/varlen_columns.py @@ -1,3 +1,6 @@ +# List-valued columns with variable-length cells. For variable-length +# *scalar* string columns, see utf8_strings.py (blosc2.utf8()). + from __future__ import annotations from dataclasses import dataclass diff --git a/src/blosc2/schema.py b/src/blosc2/schema.py index b0f001196..ad77e1a44 100644 --- a/src/blosc2/schema.py +++ b/src/blosc2/schema.py @@ -308,6 +308,11 @@ def to_metadata_dict(self) -> dict[str, Any]: class string(SchemaSpec): """Fixed-width Unicode string column. + Values longer than *max_length* are rejected at validation time (and + silently truncated when validation is disabled), and every row costs + ``4 * max_length`` bytes before compression. For variable-length text + prefer :func:`utf8`; see :ref:`ChoosingStringType` for a full comparison. + Parameters ---------- max_length: @@ -811,11 +816,14 @@ def vlstring( ) -> VLStringSpec: """Build a variable-length scalar string schema descriptor. - Use this as an explicit opt-in when a CTable column holds long or - wildly variable-length strings that would waste space in a fixed-width - ``string(max_length=N)`` column. Must be requested via - ``blosc2.field(blosc2.vlstring())`` — it is never inferred automatically - from plain ``str`` annotations. + For most variable-length text, prefer :func:`utf8` instead: it supports + vectorized filters, groupby keys, sorting, and fast bulk reads, none of + which vlstring supports. vlstring remains the right choice on + NumPy < 2.0 (utf8 requires ``StringDType``) and for nullable columns + that need native ``None`` nulls with no sentinel, i.e. where any string + value can legally occur. See :ref:`ChoosingStringType` for a full + comparison. Must be requested via ``blosc2.field(blosc2.vlstring())`` — + it is never inferred automatically from plain ``str`` annotations. """ return VLStringSpec( nullable=nullable, @@ -838,6 +846,16 @@ def utf8(*, nullable: bool = False, null_value: str | None = None) -> Utf8Spec: Must be requested via ``blosc2.field(blosc2.utf8())`` — it is never inferred automatically from plain ``str`` annotations. + utf8 columns support vectorized comparisons (``==``, ``!=``, ``<``, + ``<=``, ``>``, ``>=``), :meth:`CTable.group_by` keys, + :meth:`CTable.sort_by`, and Arrow/Parquet interop. Current limitations: + :meth:`CTable.create_index` is not supported yet (use a fixed-width + :class:`string` column if you need an index), and string-*expression* + filters such as ``t.where("name == 'x'")`` are not supported yet — use + the operator form ``t[t.name == 'x']`` instead. See + :ref:`ChoosingStringType` for a full comparison with :class:`string` + and :func:`vlstring`. + Parameters ---------- nullable: From 64a832f23ee3338ce7b26332dec98a816a2b902e Mon Sep 17 00:00:00 2001 From: Francesc Alted Date: Fri, 17 Jul 2026 08:02:58 +0200 Subject: [PATCH 15/20] Self-contained NpyString declarations in utf8_ext.pyx for older numpy pxds The NpyString C API has been in the NumPy headers since 2.0, but its Cython declarations only appear in the numpy/__init__.pxd of newer NumPy versions. The Pyodide cross-build pins an older NumPy (2.1's pxd has none of them), so cythonizing utf8_ext.pyx failed on wasm32 with "'npy_string_allocator' is not a type identifier". Declare the four symbols locally instead of via `cimport numpy`; they resolve through the API table populated by cnp.import_array(), so no behavior changes. Verified by reproducing the failure with numpy 2.1.3 + Cython 3.2.8, then cythonizing, compiling, and running the kernel in that same env. Co-Authored-By: Claude Fable 5 --- src/blosc2/utf8_ext.pyx | 36 +++++++++++++++++++++++++++++++----- 1 file changed, 31 insertions(+), 5 deletions(-) diff --git a/src/blosc2/utf8_ext.pyx b/src/blosc2/utf8_ext.pyx index 8473d83f6..816cc8d03 100644 --- a/src/blosc2/utf8_ext.pyx +++ b/src/blosc2/utf8_ext.pyx @@ -24,6 +24,32 @@ from libc.stdint cimport int64_t, uint8_t cnp.import_array() +# Declared here instead of relying on `cimport numpy`: the NpyString C API +# has been part of the NumPy headers since 2.0, but its Cython declarations +# only appear in the numpy/__init__.pxd of newer NumPy versions, and some +# build environments (e.g. the Pyodide cross-build) pin an older one. The +# functions resolve through the API table populated by cnp.import_array(). +cdef extern from "numpy/ndarraytypes.h": + ctypedef struct npy_string_allocator: + pass + ctypedef struct npy_packed_static_string: + pass + ctypedef struct PyArray_StringDTypeObject: + pass + +cdef extern from "numpy/arrayobject.h": + npy_string_allocator* NpyString_acquire_allocator( + const PyArray_StringDTypeObject* descr + ) nogil + void NpyString_release_allocator(npy_string_allocator* allocator) nogil + int NpyString_pack( + npy_string_allocator* allocator, + npy_packed_static_string* packed_string, + const char* buf, + size_t size, + ) nogil + + def pack_utf8_span(cnp.ndarray rel not None, cnp.ndarray data not None, cnp.ndarray out not None): """Fill *out* in place with rows carved out of *data* using *rel*. @@ -51,8 +77,8 @@ def pack_utf8_span(cnp.ndarray rel not None, cnp.ndarray data not None, cnp.ndar cdef const uint8_t* data_ptr = cnp.PyArray_DATA(data) cdef char* out_data = cnp.PyArray_DATA(out) cdef cnp.npy_intp itemsize = cnp.PyArray_ITEMSIZE(out) - cdef cnp.npy_string_allocator* allocator = cnp.NpyString_acquire_allocator( - cnp.PyArray_DESCR(out) + cdef npy_string_allocator* allocator = NpyString_acquire_allocator( + cnp.PyArray_DESCR(out) ) if allocator == NULL: raise TypeError("out must be a StringDType array") @@ -64,16 +90,16 @@ def pack_utf8_span(cnp.ndarray rel not None, cnp.ndarray data not None, cnp.ndar for i in range(n): start = rel_ptr[i] length = rel_ptr[i + 1] - start - ret = cnp.NpyString_pack( + ret = NpyString_pack( allocator, - (out_data + i * itemsize), + (out_data + i * itemsize), (data_ptr + start), length, ) if ret == -1: break finally: - cnp.NpyString_release_allocator(allocator) + NpyString_release_allocator(allocator) if ret == -1: raise MemoryError("Failed to pack a UTF-8 row into the StringDType array") From b672dd0b2449334473bd475a6f7f7c8efa4c2610 Mon Sep 17 00:00:00 2001 From: Francesc Alted Date: Fri, 17 Jul 2026 10:56:23 +0200 Subject: [PATCH 16/20] utf8: chunked bulk extend, ASCII join+encode fast path, wider flush batch (I1) extend() now bulk-appends plain-str chunks instead of a per-row Python loop, and _rewrite_from() joins+encodes an all-ASCII batch in one call instead of encoding each row individually. Raising _FLUSH_ROWS (4096 -> 65536) amortizes each flush's NDArray resize/write over far more rows. Combined, taxi-company ingest drops from 3622ms to 870ms, now faster than both string() and vlstring() on the same workload. Co-Authored-By: Claude Sonnet 5 --- plans/utf8-write-ingest-optim.md | 497 +++++++++++++++++++++++++++++++ src/blosc2/utf8_array.py | 49 ++- tests/ctable/test_utf8.py | 82 +++++ 3 files changed, 617 insertions(+), 11 deletions(-) create mode 100644 plans/utf8-write-ingest-optim.md diff --git a/plans/utf8-write-ingest-optim.md b/plans/utf8-write-ingest-optim.md new file mode 100644 index 000000000..5f70ef36f --- /dev/null +++ b/plans/utf8-write-ingest-optim.md @@ -0,0 +1,497 @@ +# utf8 columns: closing the write/ingest gap (incremental plan) + +**Status:** I1 LANDED (2026-07-17, branch `enhancing-ctable3`). I2 not yet +started — see the updated "Honest assessment" section below: I1 alone blew +past both its own gate and I2's provisional gate, so I2 should be +re-evaluated before starting, not assumed. Successor work to +`plans/utf8-reads-filter-optim.md` (U1 + U2, landed on branch +`enhancing-ctable3`, commits `3a7d9a3d`, `45ec7906`). Read that plan first +for background and for the exact working rebuild commands referenced below; +this plan assumes it. + +**Audience:** an implementing model/developer who has NOT read the +discussions that produced this plan. Everything needed is in this file. + +**Practical notes (carried over from the read-side plan, still true):** + +- Locate code by **symbol name with grep**, never by line number. +- Run Python/pytest through the `blosc2` conda env: + `conda run -n blosc2 python -m pytest ...` (or + `/Users/faltet/miniforge3/envs/blosc2/bin/python`). Never use the repo + `.venv` (stale). +- **Editing `.pyx` files does NOT trigger a rebuild in the editable + install.** I2 needs a real rebuild — see `plans/utf8-reads-filter-optim.md`'s + U2 section for the exact working commands (`cmake --build build_py314 + --target utf8_ext`, then copy the resulting `.so` into + `.../site-packages/blosc2/`) and the `NPY_TARGET_VERSION` gotcha already + solved there (not directly applicable to this kernel, which doesn't use + NumPy's StringDType C API, but the rebuild workflow is identical). This + is why I2 is sequenced after I1 and not merged with it. +- **Docstrings and code comments must be self-contained.** Never reference + this plan, phase labels ("I1", "I2"), or benchmark history from source, + tests, or bench scripts — state the semantics directly. +- utf8 tests live in `tests/ctable/test_utf8.py`; match neighboring style — + in particular mirror the existing `force_kernel_mode` fixture (read-side + kernel toggle) for a new write-side sibling. +- The benchmark harness is `bench/ctable/bench_string_kinds.py` (real + Chicago-taxi `company` column at 1e7 rows + synthetic high-cardinality + free text at 2e6 rows). The `ingest` line is the one this plan targets. +- The dev machine is an Apple-silicon Mac; benchmark targets assume it. + +--- + +## The problem, quantified + +`Utf8Array` ingest (`src/blosc2/utf8_array.py`) is far slower than the +alternatives on the same benchmark (`bench_string_kinds.py`, +`t.extend({"s": values, "val": float_vals}, validate=False)` on 1e7 rows +of the Chicago-taxi `company` column): + +| kind | ingest | vs utf8 | +|------------|---------|---------| +| `utf8()` | 3622 ms | — | +| `string(max_length=44)` | 478 ms | 7.6x faster | +| `vlstring()` | 1193 ms | 3x faster | + +**Root cause (verified against current code):** `Utf8Array.extend()` / +`.append()` buffer rows one at a time in a pure-Python loop: + +```python +def extend(self, values: Iterable[Any]) -> None: + for v in values: + v = self._coerce(v) + self._pending.append(v) + self._pending_chars += len(v) + self._flush_if_needed() +``` + +and `_rewrite_from()` (called every `_FLUSH_ROWS = 4096` rows) re-encodes +every row individually: + +```python +encoded = [v.encode("utf-8") for v in values] # one new bytes object per row +blob = b"".join(encoded) # bulk, fine +... +lengths = np.fromiter( + (len(e) for e in encoded), dtype=np.int64, count=len(encoded) +) # per-row len(), NOT vectorized despite being a numpy call +``` + +By contrast, `string()` ingest is a **single** `np.ascontiguousarray(raw, +dtype='U44')` call with zero per-row Python — that's the ceiling utf8 is +chasing. `CTable`'s dispatch layer (`ctable.py`) is **not** the +bottleneck: it routes `utf8`/`vlstring`/`vlbytes` through one shared +generic branch (`_is_varlen_scalar_column`) with no utf8-specific +inefficiency beyond one incidental, cheap `list(raw_columns[name])` copy. + +A standalone reproduction of the two hot Python loops above (taxi-like +ASCII workload, 1e7 rows, Python-level cost only, no NDArray I/O) measured +**1557 ms** — i.e. these two loops account for ~43% of the 3622 ms total; +the remaining ~2065 ms is NDArray resize/write/compression cost that the +items below do not touch (flagged as a follow-up, out of scope here). + +**Dead end already checked, do not repropose:** `np.strings.encode()` on a +`StringDType` array was measured *slower* than the current per-row Python +loop (0.385s vs 0.049s / 2M rows) **and** is unsafe here — it returns a +fixed-width `'S'` array padded with trailing NUL bytes, indistinguishable +from real trailing-NUL string content that utf8 columns must support +losslessly (see the NUL-bearing tests already in `tests/ctable/test_utf8.py`). + +--- + +## Design decisions already made (do not re-litigate) + +1. **I1 covers `_rewrite_from` too, not just `extend()`.** The per-row + `.encode()` loop there is pure Python and rebuild-free — it's the + single biggest lever in I1 (item I1.c below). + +2. **`str.isascii()` is O(1) per call.** It reads a cached PEP-393 flag + set at string-creation time, not a per-character scan — verified + empirically (~11.5ns/call, doesn't scale with string length). + +3. **`"".join(values).encode("ascii") == b"".join(v.encode("utf-8") for v + in values)` whenever every value is ASCII.** UTF-8/ASCII encoding has + no cross-character state, so join commutes with encode — verified by + direct byte-equality assertion in a prototype, not just argued. For + ASCII values, `len(v)` (character count) already equals the UTF-8 byte + count, so lengths can be computed straight from the un-encoded strings, + no intermediate `encoded` list needed. + +4. **Critical pitfall — call out prominently in review.** `flush()` does + `values, self._pending = self._pending, []` — a **rebind**, not a + mutation. Any chunked-extend implementation must re-read + `self._pending` after a flush that may have run mid-loop; never cache + `self._pending.append`/`.extend` as a local variable across a flush + boundary. Getting this wrong silently drops rows with **no exception** + — a naive prototype implementation did exactly this, dropping 99.96% of + a 1e7-row batch silently. Add a regression test asserting `len(arr) == + n` after a multi-flush single `extend()` call, not just content + equality (content-equality tests alone would not have caught this). + +5. **Land order: I1 first (no rebuild, always-on), I2 second (needs a real + rebuild).** I2's fallback tier is I1's optimized pure-Python path, not + the pre-I1 naive path. Mirrors the U1→U2 precedent in the read-side + plan. I1.a and I1.c land together as one item ("I1") rather than being + split — no strong reason to separate them; I1.c's benefit is best + demonstrated with I1.a's chunking already in place. + +6. **I2's kernel is a second function in the existing + `src/blosc2/utf8_ext.pyx`**, not a new `.pyx` file. Verified: the + custom command in `CMakeLists.txt` already depends on the whole file + and regenerates `utf8_ext.c` from it — adding a `def` requires **zero** + `CMakeLists.txt` changes. + +7. **`PyUnicode_AsUTF8AndSize` is already declared in Cython's + `cpython.unicode` pxd** (verified directly against + `.../site-packages/Cython/Includes/cpython/unicode.pxd`, Cython 3.2.5) + with an `except NULL` clause — Cython auto-propagates the Python + exception (`TypeError`, `UnicodeEncodeError` for a lone surrogate, + etc.) on failure, no manual NULL-check/re-raise needed. It requires the + GIL (mutates the string object's internal UTF-8 cache); do not attempt + a `nogil` second pass — U2 didn't bother either for the analogous + reason, and the encode call itself (not the memcpy) is the dominant + cost. + +--- + +## I1 — Pure-Python/NumPy ingest speedup (no rebuild) + +### I1.a — Chunked bulk-check `extend()` + +**Where:** `Utf8Array.extend` (`src/blosc2/utf8_array.py`). + +Pull `values` in chunks of `_FLUSH_ROWS` via `itertools.islice` (keeps +support for genuinely lazy iterables — `Utf8Array.copy()` calls +`out.extend(self)`). Per chunk, try a bulk fast path; fall back per-item +only for that chunk if needed: + +```python +def extend(self, values: Iterable[Any]) -> None: + it = iter(values) + while True: + chunk = list(itertools.islice(it, _FLUSH_ROWS)) + if not chunk: + break + if all(type(v) is str for v in chunk): + self._pending.extend(chunk) + self._pending_chars += sum(map(len, chunk)) + else: + for v in chunk: + v = self._coerce(v) + self._pending.append(v) + self._pending_chars += len(v) + self._flush_if_needed() +``` + +Key properties: + +- `type(v) is str` (not `isinstance`) deliberately excludes `numpy.str_` + (a `str` subclass), so `np.array(["uno", "dos"])`-style input still + falls to the slow per-item path and gets `_coerce()`'s `str(value)` + normalization — preserves `test_ctable_utf8_extend_numpy_fixed_width_input` + unchanged. +- A chunk containing `None` (nullable columns) or a non-`str` fails the + bulk check and falls to the slow path, preserving `_coerce()`'s + sentinel substitution and exact `TypeError` messages. +- Per design decision 4: never cache `self._pending.append`/`.extend` + outside this loop body. + +**Measured** (extend-loop only, `_rewrite_from` unchanged, taxi-like ASCII +workload, 1e7 rows): 1557ms → 943ms (**-39%**). + +**`append()`/`_flush_if_needed()`:** leave essentially untouched — already +minimal single-row overhead, no measurable win from rewriting them. +`set_all()` (used by `sort_by(inplace=True)`/`compact()`) is unaffected by +I1.a but benefits automatically from I1.c since both share `_rewrite_from`. + +### I1.b — Document the flush-cadence trade-off (no code change) + +Under I1.a, `_pending_chars`'s `_FLUSH_CHARS` bound is only checked once +per `_FLUSH_ROWS`-row chunk instead of every row, so an unusual batch of +many multi-MB strings could overshoot `_FLUSH_CHARS` by up to one chunk +before flushing. Low risk (`_FLUSH_ROWS` is the binding bound for +realistic short-string workloads — flushes trigger every 4096 rows on the +taxi data, well before the char bound). Document this explicitly in the +code comment; add a sanity test (extend with ~20 multi-MB strings, assert +correct read-back and roughly-bounded peak memory) rather than adding +complexity to close the soft-bound gap. + +### I1.c — Bulk `_rewrite_from` via `str.join` + `isascii()` fast path + +**Where:** `Utf8Array._rewrite_from` (`src/blosc2/utf8_array.py`). + +```python +def _rewrite_from(self, pos: int, values: list[str]) -> None: + if values and all(v.isascii() for v in values): + blob = "".join(values).encode("ascii") + lengths = np.fromiter(map(len, values), dtype=np.int64, count=len(values)) + else: + encoded = [v.encode("utf-8") for v in values] + blob = b"".join(encoded) + lengths = np.fromiter( + (len(e) for e in encoded), dtype=np.int64, count=len(encoded) + ) + ... # rest (resize/slice-write/cumsum) unchanged +``` + +The non-ASCII branch is byte-for-byte today's existing code — zero +behavior change for multi-byte/NUL-bearing/mixed batches; the fast path +only ever engages when *every* value in the batch is ASCII (per Design +decision 3). + +**Measured** (1e6-row microbenchmark, isolated): ASCII workload 82.2ms → +39.4ms (**2.09x**); mixed non-ASCII workload 84.1ms → 81.4ms (falls to +unchanged path, ~3ms `isascii()`-scan overhead, matches expectation). + +**Combined I1.a + I1.c** (full extend+flush pipeline, 1e7 rows, +Python-level cost only): 1557ms → 630ms (**-60%**). Projected onto the +full 3622ms end-to-end benchmark (NDArray-side cost assumed unaffected, +flush count unchanged): **≈2695ms** (-26%). + +### I1.d — Raising `_FLUSH_ROWS` (folded into I1) + +**Measured** (standalone sweep script, taxi-like ASCII workload, 1e7 rows, +full `CTable` ingest via `bench_string_kinds.py`'s own code path, `min` of +3 reps, `_FLUSH_ROWS` monkeypatched per value, `tracemalloc` peak as a +memory proxy): + +| `_FLUSH_ROWS` | ingest | peak (tracemalloc) | +|--------------:|---------:|--------------------:| +| 4096 (was) | 3691.6 ms | 85.9 MB | +| 8192 | 2327.1 ms | 86.0 MB | +| 16384 | 1541.9 ms | 86.0 MB | +| 32768 | 1134.3 ms | 86.0 MB | +| **65536** | **926.2 ms** | 86.2 MB | +| 131072 | 825.0 ms | 86.2 MB | +| 262144 | 806.0 ms | 91.0 MB | +| 524288 | 776.9 ms | 105.3 MB | + +This confirms the hypothesis: resize-call overhead (one +`NDArray.resize()` + slice-write pair per flush) was a large, previously +unattributed chunk of the "~2065ms of inherent NDArray-side cost" the +original problem statement assumed was a hard floor — it was not. Gains +are monotonic but sharply diminishing past 65536 (each doubling beyond +that buys under 100ms), while peak memory starts climbing beyond 131072 +(the `_FLUSH_CHARS` bound starts binding before `_FLUSH_ROWS` does, so +larger `_FLUSH_ROWS` stops mattering and only adds idle buffer capacity). +**Picked `_FLUSH_ROWS = 65536`**: captures nearly all of the available win +(926ms vs. the 777ms floor) with peak memory indistinguishable from the +original 4096 baseline. Landed as a one-line constant change plus an +explanatory code comment (no new code paths, no test changes needed since +existing tests reference `_FLUSH_ROWS` symbolically, not as a literal). + +### I1 edge cases requiring dedicated tests + +- Empty `values`/no-op flush (confirm `_rewrite_from` still only ever + called non-empty; keep defensive `if values and all(...)` guard). +- Mixed valid/invalid batch straddling a chunk boundary (a `None` at + index 4095 vs 4097). +- `append()` calls interleaved with `extend()` before a flush. +- An all-ASCII chunk containing a NUL byte (`"nul\x00in"` is ASCII — NUL + is code point 0) — confirm the join+encode fast path preserves it + exactly (this codebase has a known prior NumPy StringDType NUL bug on + the *read* side, so be explicit here on the *write* side too). +- Non-ASCII batch (existing `SAMPLE` fixture) — must produce + byte-identical output to before; extend the existing differential-test + style (`test_utf8_array_bulk_read_matches_python_ground_truth`) to also + cover writes. +- Design-decision-4 regression: `extend()` with >3x `_FLUSH_ROWS` rows in + one call, assert `len(arr) == n` and full content equality. +- `set_all()` round-trip (`sort_by(inplace=True)`, `compact()`) still + correct, since it shares the changed `_rewrite_from`. +- Non-str rejection and null-sentinel substitution + (`test_utf8_array_rejects_non_str`, + `test_ctable_utf8_not_nullable_rejects_none`, + `test_ctable_utf8_explicit_null_value`) — exact same `TypeError` + messages, now raised from the chunk's slow-path fallback. +- `test_ctable_utf8_extend_numpy_fixed_width_input` unchanged. + +**Benchmark gate (record actuals in this file when landing):** +`bench_string_kinds.py` taxi ingest **≤ 2800 ms** (from 3622 ms; +prototype projects ≈2695 ms, possibly better after I1.d's sweep). No +regression elsewhere in that script, or in `sort_by`/groupby/`to_arrow` +(which reuse `_rewrite_from` via `set_all`/`copy`). + +**ACTUAL (2026-07-17, `enhancing-ctable3`, Apple-silicon Mac, I1.a + I1.c + +I1.d combined):** taxi ingest **870.5 ms** (from 3622 ms, **-76%**) — +gate passed with a huge margin, and utf8 ingest is now *faster* than both +`string()` (1011.1 ms) and `vlstring()` (1106.1 ms) on this workload. No +regression in the rest of `bench_string_kinds.py`'s output (full read, +filter, groupby, sort, `to_arrow`, both workloads, all three column +kinds) beyond ordinary run-to-run noise. Full `tests/` suite: 7513 +passed, 22 skipped (unchanged from pre-change baseline). + +--- + +## I2 — C-level bulk UTF-8 encode kernel (needs rebuild) + +**Where:** a second function in the existing `src/blosc2/utf8_ext.pyx`, +alongside `pack_utf8_span`. No `CMakeLists.txt` changes needed (Design +decision 6). + +**Signature and algorithm:** + +```cython +def encode_utf8_span(list values not None): + """Return (data, lengths) for *values* (a list of str). + + data: uint8 NDArray -- concatenated UTF-8 encoding of every value. + lengths: int64 NDArray, length len(values) -- each value's UTF-8 byte length. + """ +``` + +1. `n = len(values)`; `n == 0` → return two empty arrays (mirror + `pack_utf8_span`'s early return). +2. Allocate `lengths` (`int64`, size `n`) up front — write directly into + it during pass 1. +3. Allocate a temporary C buffer of `n` `const char*` pointers + (`malloc`/`free` in a `try/finally`, mirroring `pack_utf8_span`'s + existing `NpyString_acquire_allocator`/`try/finally` pattern) to + remember each value's cached UTF-8 pointer between passes. +4. **Pass 1** (GIL held): for each value, call + `PyUnicode_AsUTF8AndSize(value, &size)` — this both encodes (or reuses + the string's cached UTF-8 representation) and hands back a *borrowed* + pointer with no new `bytes` allocation. Store pointer + `size`; a + `TypeError`/`UnicodeEncodeError` propagates automatically via the + `except NULL` clause (Design decision 7) — ensure the temp buffer is + still freed via `try/finally` on that path. Accumulate `total`. +5. Allocate `data` (`uint8`, size `max(total, 1)`, matching + `_rewrite_from`'s existing zero-length convention). +6. **Pass 2** (GIL held — see Design decision 7 on why not `nogil`): + `memcpy` each value's bytes from its stored pointer into `data` at the + running cumulative offset. +7. Return `(data, lengths)`. + +**Caller integration (`_rewrite_from`):** + +```python +kernel = _encode_utf8_kernel() +if kernel is not None and values: + data_arr, lengths = kernel(values) +elif values and all(v.isascii() for v in values): + data_arr = np.frombuffer("".join(values).encode("ascii"), dtype=np.uint8) + lengths = np.fromiter(map(len, values), dtype=np.int64, count=len(values)) +else: + encoded = [v.encode("utf-8") for v in values] + data_arr = np.frombuffer(b"".join(encoded), dtype=np.uint8) + lengths = np.fromiter((len(e) for e in encoded), dtype=np.int64, count=len(encoded)) +``` + +**Graceful degradation:** new lazy helper `_encode_utf8_kernel()` in +`utf8_array.py`, mirroring `_pack_utf8_kernel()` exactly (`try: from +blosc2 import utf8_ext / except ImportError: return None`). Falls back to +I1.c's already-optimized path, not the pre-I1 naive path. Test via a +`force_write_kernel_mode` fixture, sibling to the existing +`force_kernel_mode` (they toggle independent lazy helpers — read-side vs +write-side kernel). + +**I2 test coverage (in addition to I1's edge cases, all still apply):** + +- Kernel-on/kernel-off parametrized versions of I1.c's edge cases (empty, + NUL-bearing, multi-KB, multi-byte, mixed batches). +- A lone-surrogate value (e.g. `"\udc80"`) — assert `UnicodeEncodeError` + raised, matching `str.encode("utf-8")`'s own behavior; assert no + leak/corruption by checking a subsequent `extend()`/read still works + (regression test for the `try/finally` temp-buffer cleanup on the error + path). +- A very large single string (multi-MB) through the kernel — sanity-check + `total`/offset accumulation. + +**Benchmark gate (provisional — no prototype backing this number, unlike +I1's; record the actual on landing):** `bench_string_kinds.py` taxi +ingest **≤ 1500 ms** (from ≈2695 ms after I1). If missed, profile before +adding cleverness — ~2065 ms of the original 3622 ms is inherent NDArray +write/compression cost outside I1/I2's scope, so there is a hard floor. + +**Acceptance:** full `tests/` green with the kernel active AND with both +kernel toggles (read-side and write-side) forced off independently; +`blosc2.utf8()` fully functional on a NumPy-only environment without the +compiled extension (falls all the way back to I1's pure-Python path). + +--- + +## Honest assessment / what this plan does NOT close + +**Superseded by measurement.** The projections below (written before I1 +landed) assumed I1.d was a minor, uncertain lever and that ~2065ms of +NDArray-side cost was an inherent floor. Both assumptions were wrong: I1 +(including I1.d) landed at **870.5ms**, already faster than `string()` +(1011.1ms) and `vlstring()` (1106.1ms) on the taxi benchmark — beating +I2's own provisional gate (≤1500ms) before I2 was even started. Original +text, kept for the record: + +- ~~I1 alone lands at ≈2695ms — still 5.6x slower than `string()` (478ms) + and 2.3x slower than `vlstring()` (1193ms). Real, verified, low-risk + win; not a full fix.~~ +- ~~I2 is provisionally estimated at ≈1500ms — still ~3x slower than + `string()` and roughly on par with `vlstring()`. The remaining + ~2065ms of NDArray resize/write/compression cost is untouched by either + item as scoped.~~ — this cost was in fact mostly per-flush resize + overhead, not inherent, and I1.d's larger `_FLUSH_ROWS` eliminated most + of it directly. +- I1.d (the `_FLUSH_ROWS` sweep) is the cheapest way to find out whether + that follow-up is even necessary before investing in it. (It was the + right call: the sweep alone found nearly all of the remaining win.) + +**Recommendation:** given I1 already meets or beats every target this +plan set, re-evaluate whether I2 (the compiled kernel, needs a rebuild) +is still worth doing before starting it — the honest floor now measured +is ≈777ms (the `_FLUSH_ROWS = 524288` datapoint in I1.d), so I2's +realistic ceiling is a further ~150ms (~17%) on this workload, not the +~1200ms (~44%) originally projected. Confirm with whoever is driving this +plan before spending the I2 rebuild effort. + +--- + +## Sequencing and cross-cutting rules + +- Land order: **I1 (I1.a + I1.c + I1.d sweep) → I2**. I1 lands as one + item (unlike U1.a/U1.b's split — no strong reason to separate I1.a/I1.c + here). +- Each item lands with its gate recorded in this file (numbers, machine, + command), following the read-side plan's convention: if a gate fails, + do not merge the fast path — record the number and the root cause here + and stop. +- Never regress `string()`/`vlstring()` ingest performance — guard with + the full `bench_string_kinds.py` script, not just utf8's rows. +- No new public API — everything here is internal (`Utf8Array` methods, + one new lazy-import helper, one new compiled function in the existing + `utf8_ext` module). + +--- + +## Critical files + +- `src/blosc2/utf8_array.py` — `Utf8Array.extend`, `_rewrite_from`, new + `_encode_utf8_kernel()` helper (I1.a, I1.c, I2 caller-side wiring). +- `src/blosc2/utf8_ext.pyx` — new `encode_utf8_span` function alongside + the existing `pack_utf8_span` (I2 kernel). +- `tests/ctable/test_utf8.py` — new/extended tests per the edge-case + lists above; mirror the existing `force_kernel_mode` fixture for a + `force_write_kernel_mode` sibling. +- `bench/ctable/bench_string_kinds.py` — gate measurement (ingest step) + for both I1 and I2; no code changes needed, just run it. +- `plans/utf8-reads-filter-optim.md` — style/convention reference and + U2's build-workflow section (rebuild commands for I2). + +## Verification (when this plan is implemented) + +1. `conda run -n blosc2 python -m pytest tests/ctable/test_utf8.py -q` + green after I1, then again after I2 (both with kernel active and with + `force_kernel_mode`/`force_write_kernel_mode` forcing fallback). +2. `conda run -n blosc2 python -m pytest tests/ -q --timeout=600` full + suite green, kernel-on and kernel-off. +3. `conda run -n blosc2 python bench/ctable/bench_string_kinds.py` — + record the taxi `ingest` line after I1 (gate ≤2800ms) and after I2 + (gate ≤1500ms, provisional), plus confirm no regression in any other + row of that script's output (full read, filter, groupby, sort, + to_arrow — for all three column kinds). +4. Ruff check on changed `.py` files + (`conda run -n blosc2 ruff check src/blosc2/utf8_array.py + tests/ctable/test_utf8.py`). +5. For I2 specifically: build via `cmake --build build_py314 --target + utf8_ext`, copy the `.so` into the active conda env's + `site-packages/blosc2/`, and confirm `from blosc2 import utf8_ext; + utf8_ext.encode_utf8_span(...)` round-trips correctly against a + Python-level reference implementation before wiring it into + `_rewrite_from`. diff --git a/src/blosc2/utf8_array.py b/src/blosc2/utf8_array.py index 8cd9e9bfe..91d60a212 100644 --- a/src/blosc2/utf8_array.py +++ b/src/blosc2/utf8_array.py @@ -32,6 +32,7 @@ from __future__ import annotations +import itertools from typing import TYPE_CHECKING, Any import numpy as np @@ -40,7 +41,14 @@ from collections.abc import Iterable, Iterator # Pending rows are flushed to the backing arrays once either bound is hit. -_FLUSH_ROWS = 4096 +# _FLUSH_ROWS is set well above what per-row overhead would suggest: each +# flush pays a fixed NDArray resize + slice-write cost, so fewer, larger +# flushes amortize that cost over more rows -- a taxi-like ASCII workload +# (1e7 rows, ~26-char average) measured a monotonic ingest speedup from +# 4096 (~3.6s) up to 65536 (~0.9s) with essentially no peak-memory increase, +# then diminishing returns beyond that (largely because _FLUSH_CHARS starts +# binding before _FLUSH_ROWS does). +_FLUSH_ROWS = 65536 _FLUSH_CHARS = 1 << 22 # ~4 Mi characters (>= 4 MiB encoded) # Storage grids for freshly created backing arrays. Both arrays are created @@ -293,8 +301,13 @@ def _rewrite_from(self, pos: int, values: list[str]) -> None: ``pos + len(values)`` becomes the new persisted row count; the byte blob and offsets after ``pos`` are rewritten. """ - encoded = [v.encode("utf-8") for v in values] - blob = b"".join(encoded) + if values and all(v.isascii() for v in values): + blob = "".join(values).encode("ascii") + lengths = np.fromiter(map(len, values), dtype=np.int64, count=len(values)) + else: + encoded = [v.encode("utf-8") for v in values] + blob = b"".join(encoded) + lengths = np.fromiter((len(e) for e in encoded), dtype=np.int64, count=len(encoded)) if pos == 0: start = 0 elif pos == self._persisted_rows: @@ -302,15 +315,14 @@ def _rewrite_from(self, pos: int, values: list[str]) -> None: else: start = int(self._offsets[pos]) new_used = start + len(blob) - new_rows = pos + len(encoded) + new_rows = pos + len(values) if int(self._data.shape[0]) != max(new_used, 1): self._data.resize((max(new_used, 1),)) if blob: self._data[start:new_used] = np.frombuffer(blob, dtype=np.uint8) if int(self._offsets.shape[0]) != new_rows + 1: self._offsets.resize((new_rows + 1,)) - if encoded: - lengths = np.fromiter((len(e) for e in encoded), dtype=np.int64, count=len(encoded)) + if values: self._offsets[pos + 1 : new_rows + 1] = start + np.cumsum(lengths) self._persisted_rows = new_rows self._bytes_used_cache = new_used @@ -327,11 +339,26 @@ def append(self, value: Any) -> None: self._flush_if_needed() def extend(self, values: Iterable[Any]) -> None: - """Append many string rows.""" - for v in values: - v = self._coerce(v) - self._pending.append(v) - self._pending_chars += len(v) + """Append many string rows. + + Processed in ``_FLUSH_ROWS``-sized chunks so the char-count flush + bound is only checked once per chunk rather than once per row; an + unusual batch of many multi-MB strings can therefore overshoot + ``_FLUSH_CHARS`` by up to one chunk before a flush is triggered. + """ + it = iter(values) + while True: + chunk = list(itertools.islice(it, _FLUSH_ROWS)) + if not chunk: + break + if all(type(v) is str for v in chunk): + self._pending.extend(chunk) + self._pending_chars += sum(map(len, chunk)) + else: + for v in chunk: + v = self._coerce(v) + self._pending.append(v) + self._pending_chars += len(v) self._flush_if_needed() def flush(self) -> None: diff --git a/tests/ctable/test_utf8.py b/tests/ctable/test_utf8.py index ad6aef23f..a651caf83 100644 --- a/tests/ctable/test_utf8.py +++ b/tests/ctable/test_utf8.py @@ -162,6 +162,88 @@ def test_utf8_array_rejects_non_str(): arr.append(None) +# --------------------------------------------------------------------------- +# Chunked bulk extend (write path) +# --------------------------------------------------------------------------- + + +def test_utf8_array_extend_empty_iterable_is_noop(): + from blosc2.utf8_array import Utf8Array + + arr = Utf8Array(blosc2.utf8()) + arr.extend([]) + assert len(arr) == 0 + arr.extend(iter([])) + assert len(arr) == 0 + arr.flush() + assert list(arr[:]) == [] + + +def test_utf8_array_extend_many_rows_no_dropped_rows(): + """Regression for the flush-rebind pitfall: `flush()` rebinds + `self._pending` to a fresh list rather than mutating it, so an + `extend()` spanning several internal flushes must re-read + `self._pending` after each one instead of caching a reference.""" + from blosc2.utf8_array import _FLUSH_ROWS, Utf8Array + + n = _FLUSH_ROWS * 3 + 7 + values = [f"row{i}" for i in range(n)] + arr = Utf8Array(blosc2.utf8()) + arr.extend(values) + assert len(arr) == n + arr.flush() + assert len(arr) == n + assert list(arr[:]) == values + + +def test_utf8_array_extend_none_straddles_chunk_boundary(): + from blosc2.utf8_array import _FLUSH_ROWS, Utf8Array + + values = [f"v{i}" for i in range(_FLUSH_ROWS + 2)] + values[_FLUSH_ROWS - 1] = None # last row of first chunk + values[_FLUSH_ROWS + 1] = None # second row of second chunk + arr = Utf8Array(blosc2.utf8(null_value="")) + arr.extend(values) + expected = [v if v is not None else "" for v in values] + assert list(arr[:]) == expected + + +def test_utf8_array_extend_append_interleaved_before_flush(): + from blosc2.utf8_array import Utf8Array + + arr = Utf8Array(blosc2.utf8()) + arr.append("first") + arr.extend(["second", "third"]) + arr.append("fourth") + arr.extend(["fifth"]) + assert list(arr[:]) == ["first", "second", "third", "fourth", "fifth"] + + +def test_utf8_array_extend_ascii_nul_byte_preserved(): + from blosc2.utf8_array import Utf8Array + + values = ["nul\x00in", "plain", "\x00leading", "trailing\x00"] + assert all(v.isascii() for v in values) + arr = Utf8Array(blosc2.utf8()) + arr.extend(values) + arr.flush() + assert list(arr[:]) == values + + +def test_utf8_array_extend_multi_mb_strings_bounded_flush(): + """~20 multi-MB ASCII strings: char-count flush bound is checked once + per _FLUSH_ROWS-sized chunk (not per row), so this overshoots + _FLUSH_CHARS by at most one chunk before flushing -- confirm read-back + is still correct despite the coarser check.""" + from blosc2.utf8_array import Utf8Array + + values = [f"{i:06d}" + "x" * (2 * 1024 * 1024) for i in range(20)] + arr = Utf8Array(blosc2.utf8()) + arr.extend(values) + arr.flush() + assert list(arr[:]) == values + + # --------------------------------------------------------------------------- # Bulk StringDType read: compiled kernel and its pure-Python fallback # --------------------------------------------------------------------------- From 89d0b260b69207f4a5cc6b8acc12ac2c924326c7 Mon Sep 17 00:00:00 2001 From: Francesc Alted Date: Fri, 17 Jul 2026 10:57:20 +0200 Subject: [PATCH 17/20] bench_string_kinds: add --ingest-only to skip the slow read/sort/to_arrow steps to_arrow() alone takes 40-90s per column kind on the full taxi workload, dwarfing the ~1s ingest step it's meant to isolate. --ingest-only skips straight to the numbers that matter when iterating on ingest performance. Co-Authored-By: Claude Sonnet 5 --- bench/ctable/bench_string_kinds.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/bench/ctable/bench_string_kinds.py b/bench/ctable/bench_string_kinds.py index 2b51848af..2c14d22f4 100644 --- a/bench/ctable/bench_string_kinds.py +++ b/bench/ctable/bench_string_kinds.py @@ -21,6 +21,7 @@ silently. """ +import argparse import pathlib import time from dataclasses import make_dataclass @@ -35,6 +36,17 @@ REPS = 3 TAXI_PARQUET = pathlib.Path(__file__).parent.parent / "chicago-taxi" / "chicago-taxi-flat.parquet" +parser = argparse.ArgumentParser(description=__doc__) +parser.add_argument( + "--ingest-only", + action="store_true", + help="Only measure ingest and storage footprint; skip full read, filter, " + "groupby, sort, and to_arrow (the last of which alone can take over a " + "minute per column kind on the full taxi workload). Meant for fast " + "iteration when only ingest performance is under investigation.", +) +args = parser.parse_args() + rng = np.random.default_rng(42) @@ -100,6 +112,10 @@ def run_workload(title, values, filter_value): f" {'storage nbytes -> cbytes':34s} {nbytes / 2**20:7.1f} MB -> {cbytes / 2**20:7.1f} MB (cratio {nbytes / cbytes:.1f}x)" ) + if args.ingest_only: + del t + continue + bench("full column read", lambda t=t: t["s"][:]) try: bench("filter: count(s == value)", lambda t=t: int((t.s == filter_value)[:].sum())) From 45b0b4c042a702fdfc2d3bbb0e0bdf1f7facf542 Mon Sep 17 00:00:00 2001 From: Francesc Alted Date: Fri, 17 Jul 2026 11:07:13 +0200 Subject: [PATCH 18/20] utf8: C-level bulk UTF-8 encode kernel for ingest (I2) Adds encode_utf8_span to utf8_ext.pyx: encodes a list of str to concatenated UTF-8 bytes + per-row lengths in a GIL-held C loop using each string's cached UTF-8 representation (PyUnicode_AsUTF8AndSize), avoiding both the per-row Python .encode() calls and the intermediate bytes-list/join allocation that the I1 pure-Python fast path still pays. _rewrite_from tries the kernel first, falling back to I1's join+encode path when the compiled extension is unavailable. Taxi-company ingest: 870.5ms (post-I1) -> 598.6ms, now within ~22% of string() and clearly ahead of vlstring(). Verified full test suite green both with the kernel active and with utf8_ext entirely absent from the environment (falls all the way back to pure Python). Co-Authored-By: Claude Sonnet 5 --- plans/utf8-write-ingest-optim.md | 63 ++++++++++++++++-------- src/blosc2/utf8_array.py | 28 ++++++++--- src/blosc2/utf8_ext.pyx | 56 ++++++++++++++++++++- tests/ctable/test_utf8.py | 83 ++++++++++++++++++++++++++++++++ 4 files changed, 204 insertions(+), 26 deletions(-) diff --git a/plans/utf8-write-ingest-optim.md b/plans/utf8-write-ingest-optim.md index 5f70ef36f..12f193773 100644 --- a/plans/utf8-write-ingest-optim.md +++ b/plans/utf8-write-ingest-optim.md @@ -1,13 +1,13 @@ # utf8 columns: closing the write/ingest gap (incremental plan) -**Status:** I1 LANDED (2026-07-17, branch `enhancing-ctable3`). I2 not yet -started — see the updated "Honest assessment" section below: I1 alone blew -past both its own gate and I2's provisional gate, so I2 should be -re-evaluated before starting, not assumed. Successor work to -`plans/utf8-reads-filter-optim.md` (U1 + U2, landed on branch -`enhancing-ctable3`, commits `3a7d9a3d`, `45ec7906`). Read that plan first -for background and for the exact working rebuild commands referenced below; -this plan assumes it. +**Status:** I1 and I2 both LANDED (2026-07-17, branch `enhancing-ctable3`). +Taxi ingest: 3622 ms → 870.5 ms after I1 → 598.6 ms after I2, now within +~20% of `string()` (490.4 ms) and clearly ahead of `vlstring()` +(1292.2 ms). See "Honest assessment" below for the full before/after +table. Successor work to `plans/utf8-reads-filter-optim.md` (U1 + U2, +landed on branch `enhancing-ctable3`, commits `3a7d9a3d`, `45ec7906`). +Read that plan first for background and for the exact working rebuild +commands referenced below; this plan assumes it. **Audience:** an implementing model/developer who has NOT read the discussions that produced this plan. Everything needed is in this file. @@ -403,10 +403,29 @@ ingest **≤ 1500 ms** (from ≈2695 ms after I1). If missed, profile before adding cleverness — ~2065 ms of the original 3622 ms is inherent NDArray write/compression cost outside I1/I2's scope, so there is a hard floor. -**Acceptance:** full `tests/` green with the kernel active AND with both -kernel toggles (read-side and write-side) forced off independently; -`blosc2.utf8()` fully functional on a NumPy-only environment without the -compiled extension (falls all the way back to I1's pure-Python path). +**ACTUAL (2026-07-17, `enhancing-ctable3`, Apple-silicon Mac):** taxi +ingest **598.6 ms** (from 870.5 ms after I1 alone, **-31%** further; +**-83.5%** from the original 3622 ms) — gate passed with a large margin, +now within ~22% of `string()` (490.4 ms) on this workload. No regression +elsewhere in `bench_string_kinds.py` (full read, filter, groupby, sort, +`to_arrow`, both workloads, all three column kinds) beyond ordinary +run-to-run noise. + +**Acceptance:** full `tests/` green with the kernel active (7525 passed, +22 skipped) AND with the compiled extension entirely absent from the +environment (`utf8_ext.cpython-*.so` moved aside — simulates a +NumPy-only/no-toolchain install; same 113/113 in `test_utf8.py` passed, +falling all the way back to I1's pure-Python path with no import error at +`import blosc2` time). The per-test `force_write_kernel_mode` fixture +additionally parametrizes every new I2 test over kernel-on/kernel-off +without needing the extension physically absent. + +**Build note:** built via `cmake --build build_py314 --target utf8_ext` +(clean build, no new warnings — `CMakeLists.txt` needed zero changes, as +predicted by design decision 6, since its custom command already +regenerates `utf8_ext.c` from the whole `.pyx` file) and the resulting +`.so` copied into the active env's `site-packages/blosc2/`, exactly +mirroring U2's rebuild workflow. --- @@ -433,13 +452,19 @@ text, kept for the record: that follow-up is even necessary before investing in it. (It was the right call: the sweep alone found nearly all of the remaining win.) -**Recommendation:** given I1 already meets or beats every target this -plan set, re-evaluate whether I2 (the compiled kernel, needs a rebuild) -is still worth doing before starting it — the honest floor now measured -is ≈777ms (the `_FLUSH_ROWS = 524288` datapoint in I1.d), so I2's -realistic ceiling is a further ~150ms (~17%) on this workload, not the -~1200ms (~44%) originally projected. Confirm with whoever is driving this -plan before spending the I2 rebuild effort. +**Update after I2 landed:** the recommendation above (re-evaluate before +building I2, since the realistic ceiling looked like only ~150ms) turned +out to be pessimistic — I2 delivered another **272ms** (870.5ms → +598.6ms, -31%), more than the ~150ms estimated from the I1.d +`_FLUSH_ROWS` sweep's asymptote. The compiled kernel avoids not just the +per-row `.encode()` calls but also the intermediate `list` of `bytes` +objects and the `b"".join()` allocation that I1.c's Python fast path +still pays; those turned out to matter more than the sweep alone +suggested. Final state: utf8 ingest 3622ms → 598.6ms (**-83.5%**), +within ~22% of `string()` (490.4ms) and clearly ahead of `vlstring()` +(1292.2ms). Remaining gap to `string()` (~108ms) is presumably the last +of the inherent NDArray resize/write/compression cost referenced above — +not investigated further; not gated by this plan. --- diff --git a/src/blosc2/utf8_array.py b/src/blosc2/utf8_array.py index 91d60a212..e02576965 100644 --- a/src/blosc2/utf8_array.py +++ b/src/blosc2/utf8_array.py @@ -121,6 +121,19 @@ def _pack_utf8_kernel(): return getattr(utf8_ext, "pack_utf8_span", None) +def _encode_utf8_kernel(): + """Return the compiled bulk UTF-8 encoder, or ``None``. + + ``None`` means the ``utf8_ext`` extension is unavailable; callers fall + back to the pure-Python join+encode paths in ``_rewrite_from``. + """ + try: + from blosc2 import utf8_ext + except ImportError: + return None + return getattr(utf8_ext, "encode_utf8_span", None) + + def _new_backend_arrays(cparams=None, dparams=None, *, offsets_urlpath=None, data_urlpath=None): """Create fresh (offsets, data) NDArrays for an empty utf8 column.""" import blosc2 @@ -301,12 +314,15 @@ def _rewrite_from(self, pos: int, values: list[str]) -> None: ``pos + len(values)`` becomes the new persisted row count; the byte blob and offsets after ``pos`` are rewritten. """ - if values and all(v.isascii() for v in values): - blob = "".join(values).encode("ascii") + kernel = _encode_utf8_kernel() + if kernel is not None and values: + data_arr, lengths = kernel(values) + elif values and all(v.isascii() for v in values): + data_arr = np.frombuffer("".join(values).encode("ascii"), dtype=np.uint8) lengths = np.fromiter(map(len, values), dtype=np.int64, count=len(values)) else: encoded = [v.encode("utf-8") for v in values] - blob = b"".join(encoded) + data_arr = np.frombuffer(b"".join(encoded), dtype=np.uint8) lengths = np.fromiter((len(e) for e in encoded), dtype=np.int64, count=len(encoded)) if pos == 0: start = 0 @@ -314,12 +330,12 @@ def _rewrite_from(self, pos: int, values: list[str]) -> None: start = self._bytes_used else: start = int(self._offsets[pos]) - new_used = start + len(blob) + new_used = start + len(data_arr) new_rows = pos + len(values) if int(self._data.shape[0]) != max(new_used, 1): self._data.resize((max(new_used, 1),)) - if blob: - self._data[start:new_used] = np.frombuffer(blob, dtype=np.uint8) + if len(data_arr): + self._data[start:new_used] = data_arr if int(self._offsets.shape[0]) != new_rows + 1: self._offsets.resize((new_rows + 1,)) if values: diff --git a/src/blosc2/utf8_ext.pyx b/src/blosc2/utf8_ext.pyx index 816cc8d03..ae188924b 100644 --- a/src/blosc2/utf8_ext.pyx +++ b/src/blosc2/utf8_ext.pyx @@ -6,7 +6,7 @@ ####################################################################### # cython: boundscheck=False, wraparound=False, initializedcheck=False -"""Bulk StringDType construction for utf8 columns. +"""Bulk StringDType construction and bulk UTF-8 encoding for utf8 columns. NumPy provides no bulk constructor for a ``StringDType`` array from an offsets+bytes buffer pair -- the only Python-level ways in are per-element @@ -14,12 +14,20 @@ assignment or conversion from another array. This kernel uses NumPy's C API for ``StringDType`` (``NpyString_pack``) to fill every element of a preallocated ``StringDType`` array directly from the raw offsets/bytes representation, in a single C loop with no per-row Python object churn. + +A second kernel (``encode_utf8_span``) goes the other direction: it turns a +list of ``str`` into the concatenated UTF-8 bytes plus per-row lengths +needed to write a utf8 column, using each string's cached UTF-8 +representation instead of allocating one ``bytes`` object per row. """ import numpy as np cimport numpy as cnp +from cpython.unicode cimport PyUnicode_AsUTF8AndSize from libc.stdint cimport int64_t, uint8_t +from libc.stdlib cimport free, malloc +from libc.string cimport memcpy cnp.import_array() @@ -103,3 +111,49 @@ def pack_utf8_span(cnp.ndarray rel not None, cnp.ndarray data not None, cnp.ndar if ret == -1: raise MemoryError("Failed to pack a UTF-8 row into the StringDType array") + + +def encode_utf8_span(list values not None): + """Return ``(data, lengths)`` for *values*, a list of ``str``. + + *data* is a ``uint8`` NDArray holding the concatenated UTF-8 encoding of + every value, in order. *lengths* is an ``int64`` NDArray of length + ``len(values)`` giving each value's UTF-8 byte length. Each value's + UTF-8 bytes are taken from its cached representation + (``PyUnicode_AsUTF8AndSize``) rather than allocating a new ``bytes`` + object per row. + """ + cdef Py_ssize_t n = len(values) + if n == 0: + return np.empty(0, dtype=np.uint8), np.empty(0, dtype=np.int64) + + cdef cnp.ndarray lengths = np.empty(n, dtype=np.int64) + cdef int64_t* lengths_ptr = cnp.PyArray_DATA(lengths) + cdef const char** ptrs = malloc(n * sizeof(char*)) + if ptrs == NULL: + raise MemoryError("Failed to allocate temporary pointer buffer") + + cdef Py_ssize_t i + cdef Py_ssize_t size + cdef int64_t total = 0 + cdef int64_t offset + cdef cnp.ndarray data + cdef uint8_t* data_ptr + try: + for i in range(n): + ptrs[i] = PyUnicode_AsUTF8AndSize(values[i], &size) + lengths_ptr[i] = size + total += size + + data = np.empty(total if total > 0 else 1, dtype=np.uint8) + data_ptr = cnp.PyArray_DATA(data) + offset = 0 + for i in range(n): + size = lengths_ptr[i] + if size: + memcpy(data_ptr + offset, ptrs[i], size) + offset += size + finally: + free(ptrs) + + return data, lengths diff --git a/tests/ctable/test_utf8.py b/tests/ctable/test_utf8.py index a651caf83..53ebe13b2 100644 --- a/tests/ctable/test_utf8.py +++ b/tests/ctable/test_utf8.py @@ -303,6 +303,89 @@ def test_ctable_utf8_persistence_roundtrip_kernel_and_fallback(tmp_path, ext, fo t2.close() +# --------------------------------------------------------------------------- +# Bulk UTF-8 encode (write path): compiled kernel and its pure-Python fallback +# --------------------------------------------------------------------------- + + +@pytest.fixture(params=["kernel", "fallback"], ids=["kernel", "fallback"]) +def force_write_kernel_mode(request, monkeypatch): + """Exercise both the compiled bulk UTF-8 encoder and its pure-Python + join+encode fallback, so the fallback stays covered even on a build + where the compiled extension is available.""" + if request.param == "fallback": + monkeypatch.setattr("blosc2.utf8_array._encode_utf8_kernel", lambda: None) + return request.param + + +def test_utf8_array_extend_kernel_and_fallback(force_write_kernel_mode): + from blosc2.utf8_array import Utf8Array + + arr = Utf8Array(blosc2.utf8()) + arr.extend(SAMPLE) + arr.flush() + assert list(arr[:]) == SAMPLE + + +def test_utf8_array_extend_matches_python_ground_truth(force_write_kernel_mode): + """Same wider mix of byte lengths and edge cases as the read-side + ground-truth test, exercised through the write path this time.""" + from blosc2.utf8_array import Utf8Array + + rng = np.random.default_rng(7) + pool = ["", "a", "café", "日本語", "x" * 5000, "nul\x00in", "nul\x00INSIDE", "emoji 🎉🚀"] + values = [pool[i] for i in rng.integers(0, len(pool), 3000)] + arr = Utf8Array(blosc2.utf8()) + arr.extend(values) + arr.flush() + assert list(arr[:]) == values + + +def test_utf8_array_extend_ascii_nul_byte_kernel_and_fallback(force_write_kernel_mode): + from blosc2.utf8_array import Utf8Array + + values = ["nul\x00in", "plain", "\x00leading", "trailing\x00"] + arr = Utf8Array(blosc2.utf8()) + arr.extend(values) + arr.flush() + assert list(arr[:]) == values + + +def test_utf8_array_extend_multi_mb_string_kernel_and_fallback(force_write_kernel_mode): + """A single multi-MB value alongside short ones -- sanity-checks the + total-length/offset accumulation in the compiled kernel's two passes.""" + from blosc2.utf8_array import Utf8Array + + values = ["head", "x" * (8 * 1024 * 1024), "tail", "café" * 100_000] + arr = Utf8Array(blosc2.utf8()) + arr.extend(values) + arr.flush() + assert list(arr[:]) == values + + +def test_ctable_utf8_extend_kernel_and_fallback(force_write_kernel_mode): + t = make_table() + assert list(t["name"][:]) == SAMPLE + + +def test_utf8_array_extend_lone_surrogate_raises_and_recovers(force_write_kernel_mode): + """A lone surrogate is invalid UTF-8: flush() must raise + UnicodeEncodeError, matching str.encode('utf-8')'s own behavior, and + the array must remain usable afterwards -- a regression test for the + compiled kernel's temp-buffer cleanup on the error path.""" + from blosc2.utf8_array import Utf8Array + + arr = Utf8Array(blosc2.utf8()) + arr.extend(["first"]) + arr.flush() + arr.extend(["ok", "bad\udc80value"]) + with pytest.raises(UnicodeEncodeError): + arr.flush() + arr.extend(["second"]) + arr.flush() + assert list(arr[:]) == ["first", "second"] + + # --------------------------------------------------------------------------- # CTable integration: append / extend / read # --------------------------------------------------------------------------- From 22aafbf71d72ce536f677a15003ecf54ac42a3ba Mon Sep 17 00:00:00 2001 From: Francesc Alted Date: Fri, 17 Jul 2026 11:14:27 +0200 Subject: [PATCH 19/20] utf8 plan: express speedups as Nx multipliers instead of percentages Consistent with how these numbers are now reported elsewhere: 6.05x total (3622ms -> 598.6ms), 1.22x slower than string(), 2.16x faster than vlstring(). Proportions that aren't speedups (time-share breakdown, a data-loss rate) are left as percentages. Co-Authored-By: Claude Sonnet 5 --- plans/utf8-write-ingest-optim.md | 31 ++++++++++++++++--------------- 1 file changed, 16 insertions(+), 15 deletions(-) diff --git a/plans/utf8-write-ingest-optim.md b/plans/utf8-write-ingest-optim.md index 12f193773..8dc47fcab 100644 --- a/plans/utf8-write-ingest-optim.md +++ b/plans/utf8-write-ingest-optim.md @@ -1,10 +1,10 @@ # utf8 columns: closing the write/ingest gap (incremental plan) **Status:** I1 and I2 both LANDED (2026-07-17, branch `enhancing-ctable3`). -Taxi ingest: 3622 ms → 870.5 ms after I1 → 598.6 ms after I2, now within -~20% of `string()` (490.4 ms) and clearly ahead of `vlstring()` -(1292.2 ms). See "Honest assessment" below for the full before/after -table. Successor work to `plans/utf8-reads-filter-optim.md` (U1 + U2, +Taxi ingest: 3622 ms → 870.5 ms after I1 (4.16x) → 598.6 ms after I2 +(6.05x total), now 1.22x slower than `string()` (490.4 ms) and 2.16x +faster than `vlstring()` (1292.2 ms). See "Honest assessment" below for +the full before/after table. Successor work to `plans/utf8-reads-filter-optim.md` (U1 + U2, landed on branch `enhancing-ctable3`, commits `3a7d9a3d`, `45ec7906`). Read that plan first for background and for the exact working rebuild commands referenced below; this plan assumes it. @@ -197,7 +197,7 @@ Key properties: outside this loop body. **Measured** (extend-loop only, `_rewrite_from` unchanged, taxi-like ASCII -workload, 1e7 rows): 1557ms → 943ms (**-39%**). +workload, 1e7 rows): 1557ms → 943ms (**1.65x**). **`append()`/`_flush_if_needed()`:** leave essentially untouched — already minimal single-row overhead, no measurable win from rewriting them. @@ -244,9 +244,9 @@ decision 3). unchanged path, ~3ms `isascii()`-scan overhead, matches expectation). **Combined I1.a + I1.c** (full extend+flush pipeline, 1e7 rows, -Python-level cost only): 1557ms → 630ms (**-60%**). Projected onto the +Python-level cost only): 1557ms → 630ms (**2.47x**). Projected onto the full 3622ms end-to-end benchmark (NDArray-side cost assumed unaffected, -flush count unchanged): **≈2695ms** (-26%). +flush count unchanged): **≈2695ms** (1.34x). ### I1.d — Raising `_FLUSH_ROWS` (folded into I1) @@ -313,9 +313,10 @@ regression elsewhere in that script, or in `sort_by`/groupby/`to_arrow` (which reuse `_rewrite_from` via `set_all`/`copy`). **ACTUAL (2026-07-17, `enhancing-ctable3`, Apple-silicon Mac, I1.a + I1.c + -I1.d combined):** taxi ingest **870.5 ms** (from 3622 ms, **-76%**) — +I1.d combined):** taxi ingest **870.5 ms** (from 3622 ms, **4.16x**) — gate passed with a huge margin, and utf8 ingest is now *faster* than both -`string()` (1011.1 ms) and `vlstring()` (1106.1 ms) on this workload. No +`string()` (1011.1 ms, 1.16x) and `vlstring()` (1106.1 ms, 1.27x) on this +workload. No regression in the rest of `bench_string_kinds.py`'s output (full read, filter, groupby, sort, `to_arrow`, both workloads, all three column kinds) beyond ordinary run-to-run noise. Full `tests/` suite: 7513 @@ -404,9 +405,9 @@ adding cleverness — ~2065 ms of the original 3622 ms is inherent NDArray write/compression cost outside I1/I2's scope, so there is a hard floor. **ACTUAL (2026-07-17, `enhancing-ctable3`, Apple-silicon Mac):** taxi -ingest **598.6 ms** (from 870.5 ms after I1 alone, **-31%** further; -**-83.5%** from the original 3622 ms) — gate passed with a large margin, -now within ~22% of `string()` (490.4 ms) on this workload. No regression +ingest **598.6 ms** (from 870.5 ms after I1 alone, **1.45x** further; +**6.05x** from the original 3622 ms) — gate passed with a large margin, +now 1.22x slower than `string()` (490.4 ms) on this workload. No regression elsewhere in `bench_string_kinds.py` (full read, filter, groupby, sort, `to_arrow`, both workloads, all three column kinds) beyond ordinary run-to-run noise. @@ -455,13 +456,13 @@ text, kept for the record: **Update after I2 landed:** the recommendation above (re-evaluate before building I2, since the realistic ceiling looked like only ~150ms) turned out to be pessimistic — I2 delivered another **272ms** (870.5ms → -598.6ms, -31%), more than the ~150ms estimated from the I1.d +598.6ms, **1.45x**), more than the ~150ms estimated from the I1.d `_FLUSH_ROWS` sweep's asymptote. The compiled kernel avoids not just the per-row `.encode()` calls but also the intermediate `list` of `bytes` objects and the `b"".join()` allocation that I1.c's Python fast path still pays; those turned out to matter more than the sweep alone -suggested. Final state: utf8 ingest 3622ms → 598.6ms (**-83.5%**), -within ~22% of `string()` (490.4ms) and clearly ahead of `vlstring()` +suggested. Final state: utf8 ingest 3622ms → 598.6ms (**6.05x**), +1.22x slower than `string()` (490.4ms) and 2.16x faster than `vlstring()` (1292.2ms). Remaining gap to `string()` (~108ms) is presumably the last of the inherent NDArray resize/write/compression cost referenced above — not investigated further; not gated by this plan. From 28d5f8fc9ac768ee41d280cf22dbc67a599ad03e Mon Sep 17 00:00:00 2001 From: Francesc Alted Date: Fri, 17 Jul 2026 11:24:55 +0200 Subject: [PATCH 20/20] utf8: address Copilot review comments on PR #677 - pack_utf8_span (utf8_ext.pyx): validate rel is well-formed (rel[0]==0, non-decreasing, within data's bounds) before the unchecked C loop, instead of trusting the caller silently -- a malformed rel previously risked an out-of-bounds read/crash. - Utf8Factorizer docstring: stop claiming a hash collision's duplicate vocabulary entry is "benign because callers merge groups by decoded value" -- no downstream consumer does that merge, so a collision would actually split a group in two. Still an accepted, vanishingly-rare risk with 64-bit hashes, just not a resolved one. - test_utf8.py: two groupby tests assigned float64 values into the int64 `x` column via implicit cast; use integer inputs explicitly so the test doesn't rely on that coercion. Co-Authored-By: Claude Sonnet 5 --- src/blosc2/utf8_array.py | 12 ++++++++---- src/blosc2/utf8_ext.pyx | 9 +++++++++ tests/ctable/test_utf8.py | 28 +++++++++++++++++++++++++--- 3 files changed, 42 insertions(+), 7 deletions(-) diff --git a/src/blosc2/utf8_array.py b/src/blosc2/utf8_array.py index e02576965..8c1d9a4a3 100644 --- a/src/blosc2/utf8_array.py +++ b/src/blosc2/utf8_array.py @@ -669,10 +669,14 @@ class Utf8Factorizer: first-appearance order; :meth:`uniques` returns the decoded vocabulary in that same order. No row is ever decoded — only the distinct values. - A hash collision between a new value and a known one is not resolved - (the new value gets a fresh code); the resulting duplicate vocabulary - entry is benign because callers merge groups by decoded value, and with - 64-bit hashes the case is vanishingly rare anyway. + A hash collision between a new value and an already-known value of the + same byte length is not resolved: the new value always gets a fresh + code, even in the (unhandled) case where its decoded string equals an + existing vocabulary entry's. No downstream consumer merges groups by + decoded value, so such a collision would silently split what should be + one group into two rather than being caught -- an accepted, unmitigated + risk given how vanishingly rare it is with 64-bit hashes, not a + guarantee that it is otherwise resolved. """ def __init__(self, arr: Utf8Array) -> None: diff --git a/src/blosc2/utf8_ext.pyx b/src/blosc2/utf8_ext.pyx index ae188924b..2c5d78a08 100644 --- a/src/blosc2/utf8_ext.pyx +++ b/src/blosc2/utf8_ext.pyx @@ -80,6 +80,15 @@ def pack_utf8_span(cnp.ndarray rel not None, cnp.ndarray data not None, cnp.ndar raise ValueError("rel, data and out must be C-contiguous") if n == 0: return + # Cheap, vectorized well-formedness checks: a malformed rel (decreasing, + # negative, or reaching past the end of data) would otherwise drive the + # unchecked pointer arithmetic below out of bounds. + if int(rel[0]) != 0: + raise ValueError("rel[0] must be 0") + if bool((np.diff(rel) < 0).any()): + raise ValueError("rel must be non-decreasing") + if int(rel[n]) > data.shape[0]: + raise ValueError("rel values must not exceed len(data)") cdef const int64_t* rel_ptr = cnp.PyArray_DATA(rel) cdef const uint8_t* data_ptr = cnp.PyArray_DATA(data) diff --git a/tests/ctable/test_utf8.py b/tests/ctable/test_utf8.py index 53ebe13b2..fa2089957 100644 --- a/tests/ctable/test_utf8.py +++ b/tests/ctable/test_utf8.py @@ -259,6 +259,28 @@ def force_kernel_mode(request, monkeypatch): return request.param +def test_pack_utf8_span_rejects_malformed_rel(): + """pack_utf8_span trusts its caller's rel/data invariants for speed, but + still validates them cheaply up front so a malformed rel fails with a + clear ValueError instead of driving the unchecked C loop out of bounds.""" + pytest.importorskip("blosc2.utf8_ext") + from blosc2 import utf8_ext + + data = np.array([1, 2, 3], dtype=np.uint8) + out = np.empty(2, dtype=STRING_DTYPE) + + with pytest.raises(ValueError, match="rel\\[0\\] must be 0"): + utf8_ext.pack_utf8_span(np.array([1, 2, 3], dtype=np.int64), data, out) + with pytest.raises(ValueError, match="non-decreasing"): + utf8_ext.pack_utf8_span(np.array([0, 2, 1], dtype=np.int64), data, out) + with pytest.raises(ValueError, match="must not exceed len\\(data\\)"): + utf8_ext.pack_utf8_span(np.array([0, 2, 10], dtype=np.int64), data, out) + + # a well-formed rel still works after the added checks + utf8_ext.pack_utf8_span(np.array([0, 1, 3], dtype=np.int64), data, out) + assert list(out) == ["\x01", "\x02\x03"] + + def test_utf8_array_bulk_read_kernel_and_fallback(force_kernel_mode): from blosc2.utf8_array import Utf8Array @@ -874,7 +896,7 @@ def test_ctable_utf8_groupby_many_byte_lengths_and_non_ascii(): pool = ["", "a", "bb", "café", "日本語のテキスト", "x" * 2000, "münchen"] names = [pool[i] for i in rng.integers(0, len(pool), 3000)] t = make_table(names) - t.x[:] = np.ones(3000) + t.x[:] = np.ones(3000, dtype=np.int64) g = t.group_by("name").sum("x") got = dict(zip(g["name"][:].tolist(), g["x_sum"][:].tolist(), strict=True)) exp: dict[str, int] = {} @@ -927,10 +949,10 @@ class FloatRow: def test_ctable_utf8_groupby_sum(): t = make_table(["a", "b", "a", "b", "a"]) - t.x[:] = [1.0, 2.0, 3.0, 4.0, 5.0] + t.x[:] = [1, 2, 3, 4, 5] g = t.group_by("name").sum("x") rows = dict(zip(g["name"][:].tolist(), g["x_sum"][:].tolist(), strict=False)) - assert rows == {"a": 9.0, "b": 6.0} + assert rows == {"a": 9, "b": 6} def test_ctable_utf8_groupby_size_and_dropna():