Sort dictionary columns by rank instead of decoded strings

A row's alphabetical rank orders exactly as its decoded value does --
the basis of the FULL rank index -- so _build_lex_keys can hand lexsort
an int32 key and skip both the decode and lexsort's string comparisons.
The code -> rank map is extracted as _dict_code_to_rank so the sort and
the index builder cannot drift apart.

200k rows, cardinality 5000: key construction 106.2 -> 21.4 ms,
sort_by(view=True) 246.7 -> 156.9 ms, descending 273.5 -> 156.2 ms
(an object key could not be negated, so descending went through a
double argsort that an int32 key does not need).

_sorted_small_copy_from_live_positions held a second copy of the key
builder with the same decode, a narrower dtype list (no StringDType
keys) and a KeyError for computed sort columns, and it re-read the
column after already gathering the codes.  It now calls _build_lex_keys
with the arrays it gathered, which removes the duplicate.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>

Author: FrancescAlted

RELEASE_NOTES.md

@@ -142,6 +142,13 @@ XXX version-specific blurb XXX
   lexsort-based `sort_by` cost O(N) decompressions. At 1M rows an unindexed
   `sort_by` drops from 236 s to 713 ms, and a full column read from 44 s (at
   200k rows) to 193 ms.
+- **`sort_by` on a dictionary column sorts int32 ranks, not decoded
+  strings.** A row's alphabetical rank orders exactly as its value does — the
+  trick the FULL index already used — so the sort key needs neither the decode
+  nor lexsort's string comparisons. Key construction drops from 106 ms to 21 ms
+  per 200k rows (cardinality 5000) and `sort_by(view=True)` from 247 ms to
+  157 ms. The filtered small-copy path, which had its own copy of the key
+  builder, now shares this one and picks up the same speedup.
 - **`kind=BUCKET` indexes no longer cost more than the scan they replace.**
   Scattered matches were read one bucket run at a time, re-decompressing the
   same blocks many times, and the planner measured selectivity in buckets while

src/blosc2/ctable.py

@@ -11645,13 +11645,22 @@ def _build_lex_keys(
         ascending: list[bool],
         live_pos: np.ndarray,
         n: int,
+        gathered: dict[str, np.ndarray] | None = None,
     ) -> list[np.ndarray]:
         """Build the key list for np.lexsort (innermost = last = primary key).
 
         For nullable columns a null-indicator key (0=non-null, 1=null) is
         inserted immediately after the value key, making it more significant.
         This ensures nulls sort last regardless of ascending/descending order.
+
+        *gathered* lets a caller that has already read the columns at
+        *live_pos* hand them over instead of paying for a second gather; a
+        dictionary column is expected there as its **codes**, which is what it
+        is cheap to gather.
         """
+        from blosc2.ctable_indexing import _dict_code_to_rank
+
+        gathered = gathered or {}
         lex_keys = []
         for name, asc in zip(reversed(cols), reversed(ascending), strict=True):
             cc = self._computed_cols.get(name)
@@ -11663,14 +11672,24 @@ def _build_lex_keys(
             else:
                 is_dict_col = col_info is not None and self._is_dictionary_column(col_info)
                 if is_dict_col:
-                    # Sort dictionary columns by decoded string values.
-                    decoded = self._cols[name][live_pos]
-                    raw = np.array(decoded, dtype=object)
-                    # Replace None with placeholder so lexsort never compares None.
-                    # Null indicator key (below) already places nulls last.
-                    raw[raw == None] = ""  # noqa: E711
+                    # Sort a dictionary column by the alphabetical rank of each
+                    # row's code -- the same trick the FULL index plays.  Ranks
+                    # order exactly as the decoded values do, and sorting int32
+                    # skips both the decode and lexsort's string comparisons.
+                    dict_col = self._cols[name]
+                    dictionary = list(dict_col.dictionary)
+                    code_to_rank = _dict_code_to_rank(dictionary)
+                    raw_codes = gathered[name] if name in gathered else dict_col.codes[live_pos]
+                    codes = np.asarray(raw_codes, dtype=np.int32)
+                    # The null code is reserved (-1), not a dictionary entry, so
+                    # it cannot be looked up; nulls take the largest rank.  The
+                    # null indicator key below is what actually places them.
+                    is_null = codes == col_info.spec.null_code
+                    raw = np.empty(len(codes), dtype=np.int32)
+                    raw[~is_null] = code_to_rank[codes[~is_null]]
+                    raw[is_null] = np.int32(len(dictionary))
                 else:
-                    raw = self._cols[name][live_pos]
+                    raw = gathered[name] if name in gathered else self._cols[name][live_pos]
             nv = getattr(col_info.spec, "null_value", None) if col_info else None
 
             # Value key
@@ -11689,10 +11708,7 @@ def _build_lex_keys(
             # Null indicator key — more significant than the value key above,
             # so nulls always sort last (0 before 1 → non-null before null).
             if is_dict_col and col_info.spec.nullable:
-                null_code = col_info.spec.null_code
-                codes_at_pos = np.asarray(self._cols[name].codes[live_pos], dtype=np.int32)
-                null_ind = (codes_at_pos == null_code).astype(np.intp)
-                lex_keys.append(null_ind)
+                lex_keys.append(is_null.astype(np.intp))
             elif nv is not None:
                 if isinstance(nv, float) and np.isnan(nv):
                     null_ind = np.isnan(raw).astype(np.intp)
@@ -11966,43 +11982,9 @@ def _sorted_small_copy_from_live_positions(
             else:
                 gathered[col.name] = arr[live_pos]
 
-        lex_keys = []
-        for name, asc in zip(reversed(cols), reversed(ascending), strict=True):
-            col_info = self._schema.columns_by_name.get(name)
-            is_dict_col = col_info is not None and self._is_dictionary_column(col_info)
-            if is_dict_col:
-                raw = np.array(self._cols[name][live_pos], dtype=object)
-                # Replace None with placeholder so lexsort never compares None.
-                raw[raw == None] = ""  # noqa: E711
-            else:
-                raw = gathered[name]
-
-            if not asc:
-                if raw.dtype.kind in "USO":
-                    rank = np.argsort(np.argsort(raw, kind="stable"), kind="stable")
-                    lex_keys.append((n - 1 - rank).astype(np.intp))
-                elif np.issubdtype(raw.dtype, np.unsignedinteger):
-                    lex_keys.append(-raw.astype(np.int64))
-                else:
-                    lex_keys.append(-raw)
-            else:
-                lex_keys.append(raw)
-
-            if is_dict_col and col_info.spec.nullable:
-                null_code = col_info.spec.null_code
-                codes_at_pos = np.asarray(self._cols[name].codes[live_pos], dtype=np.int32)
-                null_ind = (codes_at_pos == null_code).astype(np.intp)
-                lex_keys.append(null_ind)
-            else:
-                nv = getattr(col_info.spec, "null_value", None) if col_info else None
-                if nv is not None:
-                    if isinstance(nv, float) and np.isnan(nv):
-                        null_ind = np.isnan(raw).astype(np.intp)
-                    else:
-                        null_ind = (raw == nv).astype(np.intp)
-                    lex_keys.append(null_ind)
-
-        order = np.lexsort(lex_keys)
+        # The gather above already read every column at live_pos, dictionary
+        # columns as codes -- exactly what the key builder wants.
+        order = np.lexsort(self._build_lex_keys(cols, ascending, live_pos, n, gathered))
         result = self._empty_copy(capacity=n)
         for col in self._schema.columns:
             col_name = col.name

src/blosc2/ctable_indexing.py

@@ -58,6 +58,22 @@ def __init__(self):
         self.vlmeta = _FakeVlMeta()
 
 
+def _dict_code_to_rank(dictionary) -> np.ndarray:
+    """``code -> alphabetical rank`` lookup for a dictionary column.
+
+    Sorting by rank is sorting by decoded value, which is what lets an int32
+    array stand in for the strings — in the FULL index (:class:`_DictRankWrapper`)
+    and in ``CTable._build_lex_keys``.  The reserved null code is *not* a
+    dictionary entry, so callers assign nulls a rank of their own (``len``,
+    the largest, so nulls sort last).
+    """
+    n_entries = len(dictionary)
+    order = np.argsort(dictionary, kind="stable")
+    code_to_rank = np.empty(n_entries, dtype=np.int32)
+    code_to_rank[order] = np.arange(n_entries, dtype=np.int32)
+    return code_to_rank
+
+
 def _dict_rank_hash(dictionary) -> str:
     """Stable hash of a dictionary's entries (code position + value).
 
@@ -887,9 +903,7 @@ def create_index(  # noqa: C901
             n_live = self._n_rows if self._n_rows is not None else len(self._valid_rows)
             dictionary = list(dict_col.dictionary)
             n_entries = len(dictionary)
-            order = np.argsort(dictionary, kind="stable")
-            code_to_rank = np.empty(n_entries, dtype=np.int32)
-            code_to_rank[order] = np.arange(n_entries, dtype=np.int32)
+            code_to_rank = _dict_code_to_rank(dictionary)
             null_code = dict_col.spec.null_code
             null_rank = np.int32(n_entries)
             # Hash for staleness detection.

tests/ctable/test_dictionary_column.py

@@ -10,6 +10,7 @@
 
 from dataclasses import dataclass
 
+import numpy as np
 import pytest
 
 import blosc2
@@ -686,3 +687,61 @@ def counting_hash(dictionary):
     finally:
         ci._dict_rank_hash = _dict_rank_hash
     assert calls == 0, "value epoch was unchanged, so no hash should have been needed"
+
+
+def test_sort_by_keys_on_ranks_not_decoded_strings():
+    """sort_by must not decode a dictionary column to sort it.
+
+    Sorting by alphabetical rank is sorting by decoded value, so the key can
+    stay int32 -- which skips both the decode and lexsort's string
+    comparisons.  Correctness alone would not notice the difference, so
+    assert on the key dtype as well as the order.
+    """
+
+    @dataclass
+    class Row:
+        c: str = blosc2.field(blosc2.dictionary())
+        x: int = blosc2.field(blosc2.int64())
+
+    values = ["delta", "alpha", "Zeta", "beta", "alpha"]
+    t = CTable(Row, new_data={"c": values, "x": list(range(len(values)))})
+
+    live = np.arange(len(values))
+    keys = t._build_lex_keys(["c"], [True], live, len(values))
+    assert keys[0].dtype == np.int32
+
+    # Ranks must order exactly as Python orders the decoded strings.
+    assert list(t.sort_by("c")["c"][:]) == sorted(values)
+    assert list(t.sort_by("c", ascending=False)["c"][:]) == sorted(values, reverse=True)
+
+
+def test_sort_by_dictionary_nulls_and_multiple_keys():
+    """Nulls sort last in both directions, and rank keys compose with others."""
+
+    @dataclass
+    class Row:
+        c: str = blosc2.field(blosc2.dictionary(nullable=True))
+        x: int = blosc2.field(blosc2.int64())
+
+    values = ["b", None, "a", None, "b"]
+    t = CTable(Row, new_data={"c": values, "x": [0, 1, 2, 3, 4]})
+
+    assert list(t.sort_by("c")["c"][:]) == ["a", "b", "b", None, None]
+    assert list(t.sort_by("c", ascending=False)["c"][:]) == ["b", "b", "a", None, None]
+    # Secondary key breaks the "b" tie; nulls still trail.
+    assert list(t.sort_by(["c", "x"], [True, False])["x"][:]) == [2, 4, 0, 3, 1]
+
+
+def test_sort_by_dictionary_view_and_small_copy_agree():
+    """The filtered small-copy path builds its keys the same way sort_by does."""
+
+    @dataclass
+    class Row:
+        c: str = blosc2.field(blosc2.dictionary(nullable=True))
+        x: int = blosc2.field(blosc2.int64())
+
+    values = ["b", None, "a", "c", "b", None, "a"]
+    t = CTable(Row, new_data={"c": values, "x": list(range(len(values)))})
+    filtered = t[t.x > 1]  # small enough to take _sorted_small_copy_from_live_positions
+    assert list(filtered.sort_by("c")["c"][:]) == ["a", "a", "b", "c", None]
+    assert list(filtered.sort_by("c", ascending=False)["c"][:]) == ["c", "b", "a", "a", None]