Skip to content

Commit c5f5823

Browse files
FrancescAltedclaude
andcommitted
Make a string result refuse clashing utf8 null sentinels
utf8_span_eval took the first non-None sentinel it found and stamped it on every null row of the result, so an expression over two utf8 operands with different null_values relabelled one operand's nulls as the other's -- and which one won depended on the operand mapping's order. A result column has one sentinel, so there is no answer here to pick; refuse instead, and only where it shows. A boolean result forces nulls False whatever they were spelled as, and keeps working. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
1 parent e7b336e commit c5f5823

2 files changed

Lines changed: 36 additions & 2 deletions

File tree

‎src/blosc2/_utf8_array.py‎

Lines changed: 16 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -151,7 +151,9 @@ def utf8_span_eval(
151151
152152
Nulls are materialized to ``""`` so no string kernel ever sees a sentinel,
153153
and nullity is re-applied afterwards: a boolean result is forced ``False``
154-
(SQL ``WHERE`` semantics), a string result gets the sentinel back.
154+
(SQL ``WHERE`` semantics), a string result gets the sentinel back. A string
155+
result therefore needs the operands to agree on one sentinel, and raises
156+
``ValueError`` when they do not.
155157
156158
Span operands are handed over as blosc2 arrays rather than NumPy ones: the
157159
NumPy route evaluates through ``slices_eval``, which never reaches
@@ -163,7 +165,12 @@ def utf8_span_eval(
163165

164166
n_logical = min(len(a) for a in arrays.values())
165167
compute_kwargs = {"strict_miniexpr": True} if strict else {}
166-
null_value = next((v for v in sentinels.values() if v is not None), None)
168+
# One result column, so one sentinel. Reading it off whichever operand came
169+
# first would make the answer depend on the operand mapping's order, so a
170+
# disagreement is refused instead -- but only where it shows, on a string
171+
# result; a boolean one forces nulls False whatever they were spelled as.
172+
distinct_sentinels = sorted({v for v in sentinels.values() if v is not None})
173+
null_value = distinct_sentinels[0] if distinct_sentinels else None
167174

168175
out = None
169176
utf8_out = None
@@ -191,6 +198,13 @@ def utf8_span_eval(
191198
res.astype(f"<U{max(res.dtype.itemsize // 4, len(null_value))}"),
192199
)
193200
if res.dtype.kind == "U":
201+
if len(distinct_sentinels) > 1:
202+
raise ValueError(
203+
f"utf8 operands carry different null sentinels ({distinct_sentinels}); "
204+
"a string result can only have one, and picking one of them would "
205+
"silently relabel the other's nulls. Give the operands a common "
206+
"null_value, or compute on fixed-width arrays (blosc2.from_utf8)."
207+
)
194208
if utf8_out is None:
195209
utf8_out = UTF8Array(blosc2.utf8(null_value=null_value))
196210
# tolist() gives plain str, which is UTF8Array.extend's fast path.

‎tests/ctable/test_utf8.py‎

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1341,6 +1341,26 @@ def test_utf8_array_constructor_with_spec_and_nulls():
13411341
assert list(arr[:]) == ["a", "<NA>", "c"]
13421342

13431343

1344+
def test_utf8_string_expr_rejects_clashing_sentinels():
1345+
"""One result column means one sentinel, so operands must agree on it; the
1346+
old first-wins pick relabelled the other operand's nulls silently."""
1347+
a = blosc2.utf8_array(["x", None, "z"], blosc2.utf8(null_value="<NA_A>"))
1348+
b = blosc2.utf8_array(["1", "2", None], blosc2.utf8(null_value="<NA_B>"))
1349+
with pytest.raises(ValueError, match="different null sentinels"):
1350+
blosc2.lazyexpr("a + b", {"a": a, "b": b}).compute()
1351+
# A boolean result never carries a sentinel, so it is unaffected.
1352+
assert list(blosc2.lazyexpr("a > b", {"a": a, "b": b}).compute()) == [True, False, False]
1353+
1354+
1355+
def test_utf8_string_expr_shared_sentinel_survives():
1356+
spec = blosc2.utf8(null_value="<NA>")
1357+
a = blosc2.utf8_array(["x", None, "z"], spec)
1358+
b = blosc2.utf8_array(["1", "2", None], spec)
1359+
res = blosc2.lazyexpr("a + b", {"a": a, "b": b}).compute()
1360+
assert list(res[:]) == ["x1", "<NA>", "<NA>"]
1361+
assert res.spec.null_value == "<NA>"
1362+
1363+
13441364
def test_utf8_array_ctor_rejects_none_if_not_null():
13451365
with pytest.raises(TypeError, match="not nullable"):
13461366
blosc2.utf8_array(["a", None])

0 commit comments

Comments
 (0)