From 0c3d7fe120755f4a5675051dfa7ac3d9b4a4046c Mon Sep 17 00:00:00 2001 From: bvolpato Date: Thu, 3 Sep 2026 13:15:35 -0400 Subject: [PATCH 1/2] Merge ANTI JOIN should preserve unmatched NULL keys --- .../calcite/runtime/EnumerablesTest.java | 27 ++++++++++++++++++- .../calcite/linq4j/EnumerableDefaults.java | 13 ++++----- 2 files changed, 33 insertions(+), 7 deletions(-) diff --git a/core/src/test/java/org/apache/calcite/runtime/EnumerablesTest.java b/core/src/test/java/org/apache/calcite/runtime/EnumerablesTest.java index 575a8bfec651..e21397826c9b 100644 --- a/core/src/test/java/org/apache/calcite/runtime/EnumerablesTest.java +++ b/core/src/test/java/org/apache/calcite/runtime/EnumerablesTest.java @@ -192,30 +192,35 @@ class EnumerablesTest { newArrayList(1, 3, 4), newArrayList(1, 4), equalTo("[3]"), + equalTo("[3, null]"), JoinType.ANTI); // Matching key at start and end of right, not of left testIntersect( newArrayList(0, 1, 3, 4, 5), newArrayList(1, 4), equalTo("[0, 3, 5]"), + equalTo("[0, 3, 5, null]"), JoinType.ANTI); // Matching key at start and end of left, not right testIntersect( newArrayList(1, 3, 4), newArrayList(0, 1, 4, 5), equalTo("[3]"), + equalTo("[3, null]"), JoinType.ANTI); // Matching key not at start or end of left or right testIntersect( newArrayList(0, 2, 3, 4, 5), newArrayList(1, 3, 4, 6), equalTo("[0, 2, 5]"), + equalTo("[0, 2, 5, null]"), JoinType.ANTI); // Matching duplicated keys testIntersect( newArrayList(1, 3, 4), newArrayList(1, 1, 4, 4), equalTo("[3]"), + equalTo("[3, null]"), JoinType.ANTI); // LEFT join tests: @@ -291,24 +296,28 @@ class EnumerablesTest { newArrayList(0, 2, 4), newArrayList(1, 3, 5), equalTo("[0, 2, 4]"), + equalTo("[0, 2, 4, null]"), JoinType.ANTI); // Left empty testIntersect( new ArrayList<>(), newArrayList(1, 3, 4, 6), equalTo("[]"), + equalTo("[null]"), JoinType.ANTI); // Right empty testIntersect( newArrayList(3, 7), new ArrayList<>(), equalTo("[3, 7]"), + equalTo("[3, 7, null]"), JoinType.ANTI); // Both empty testIntersect( new ArrayList(), new ArrayList<>(), equalTo("[]"), + equalTo("[null]"), JoinType.ANTI); // LEFT join tests: @@ -342,6 +351,21 @@ class EnumerablesTest { JoinType.LEFT); } + @Test void testMergeAntiJoinWithRepeatedNullKeys() { + testIntersect( + newArrayList(1, 2, null, null), + newArrayList(1), + equalTo("[2, null, null]"), + equalTo("[2, null, null, null]"), + JoinType.ANTI); + testIntersect( + newArrayList(1, null, null), + newArrayList(1, 2), + equalTo("[null, null]"), + equalTo("[null, null, null]"), + JoinType.ANTI); + } + private static > void testIntersect( List list0, List list1, org.hamcrest.Matcher matcher, JoinType joinType) { testIntersect(list0, list1, matcher, matcher, joinType); @@ -636,7 +660,8 @@ private static > Enumerable intersect( (v0, v1) -> v0, JoinType.ANTI, null, null).toList(), - hasToString("[Emp(30, Fred), Emp(20, Sebastian), Emp(20, Zoey)]")); + hasToString("[Emp(30, Fred), Emp(20, Sebastian), Emp(20, Zoey), " + + "Emp(40, null), Emp(30, null)]")); } @Test void testMergeLeftJoin() { diff --git a/linq4j/src/main/java/org/apache/calcite/linq4j/EnumerableDefaults.java b/linq4j/src/main/java/org/apache/calcite/linq4j/EnumerableDefaults.java index 0eca4b6d5283..395d8c26bdcf 100644 --- a/linq4j/src/main/java/org/apache/calcite/linq4j/EnumerableDefaults.java +++ b/linq4j/src/main/java/org/apache/calcite/linq4j/EnumerableDefaults.java @@ -5047,11 +5047,11 @@ private Enumerator getRightEnumerator() { } /** Returns whether the left enumerator was successfully advanced to the next - * element, and it does not have a null key (except for LEFT join, that needs to process - * all elements from left. */ + * element, and it does not have a null key (except for LEFT and ANTI joins, which need to + * process all elements from left). */ private boolean leftMoveNext() { return getLeftEnumerator().moveNext() - && (joinType == JoinType.LEFT + && (isLeftOrAntiJoin() || outerKeySelector.apply(getLeftEnumerator().current()) != null); } @@ -5125,7 +5125,7 @@ private boolean advance() { // mergeJoin assumes inputs sorted in ascending order with nulls last, // if we reach a null key, we are done. if (leftKey == null || rightKey == null) { - if (joinType == JoinType.LEFT || (joinType == JoinType.ANTI && leftKey != null)) { + if (isLeftOrAntiJoin()) { // all remaining items in left are results for left/anti join remainingLeft = true; return true; @@ -5234,9 +5234,10 @@ private boolean advanceLeft(TSource left, TKey leftKey) { while (getLeftEnumerator().moveNext()) { left = getLeftEnumerator().current(); TKey leftKey2 = outerKeySelector.apply(left); - if (leftKey2 == null && joinType != JoinType.LEFT) { + if (leftKey2 == null && !isLeftOrAntiJoin()) { // mergeJoin assumes inputs sorted in ascending order with nulls last, - // if we reach a null key, we are done (except LEFT join, that needs to process LHS fully) + // if we reach a null key, we are done (except LEFT and ANTI joins, which + // process LHS fully) break; } if (!compareEquals(leftKey, leftKey2)) { From 03486d75187bb050c03de0f2fb5177bb479e52a9 Mon Sep 17 00:00:00 2001 From: Bruno Volpato Date: Thu, 8 Oct 2026 16:32:44 -0400 Subject: [PATCH 2/2] [CALCITE-7817] Add end-to-end test and correct NULL key comments Add an EnumerableMergeJoin ANTI join test with a single nullable key. The existing end-to-end test uses a composite key, which already kept NULL keys. Move the repeated NULL key cases into the ANTI section of EnumerablesTest.testMergeJoin3. Add a comment to each case that names the code path it exercises. Correct two comments in EnumerableDefaults that said a NULL key always ends the merge join. LEFT and ANTI joins continue to process left rows. The previous ANTI expectations in EnumerablesTest encoded the bug. For example, "[3]" is now "[3, null]", and "Both empty" with NULL keys is now "[null]" instead of "[]". --- .../calcite/runtime/EnumerablesTest.java | 29 +++++++++---------- .../test/enumerable/EnumerableJoinTest.java | 24 +++++++++++++++ .../calcite/linq4j/EnumerableDefaults.java | 5 ++-- 3 files changed, 41 insertions(+), 17 deletions(-) diff --git a/core/src/test/java/org/apache/calcite/runtime/EnumerablesTest.java b/core/src/test/java/org/apache/calcite/runtime/EnumerablesTest.java index e21397826c9b..52382542351c 100644 --- a/core/src/test/java/org/apache/calcite/runtime/EnumerablesTest.java +++ b/core/src/test/java/org/apache/calcite/runtime/EnumerablesTest.java @@ -319,6 +319,20 @@ class EnumerablesTest { equalTo("[]"), equalTo("[null]"), JoinType.ANTI); + // Left reaches NULL keys after right runs out + testIntersect( + newArrayList(1, 2, null, null), + newArrayList(1), + equalTo("[2, null, null]"), + equalTo("[2, null, null, null]"), + JoinType.ANTI); + // Left reaches NULL keys in advanceLeft, immediately after a match + testIntersect( + newArrayList(1, null, null), + newArrayList(1, 2), + equalTo("[null, null]"), + equalTo("[null, null, null]"), + JoinType.ANTI); // LEFT join tests: // No overlap @@ -351,21 +365,6 @@ class EnumerablesTest { JoinType.LEFT); } - @Test void testMergeAntiJoinWithRepeatedNullKeys() { - testIntersect( - newArrayList(1, 2, null, null), - newArrayList(1), - equalTo("[2, null, null]"), - equalTo("[2, null, null, null]"), - JoinType.ANTI); - testIntersect( - newArrayList(1, null, null), - newArrayList(1, 2), - equalTo("[null, null]"), - equalTo("[null, null, null]"), - JoinType.ANTI); - } - private static > void testIntersect( List list0, List list1, org.hamcrest.Matcher matcher, JoinType joinType) { testIntersect(list0, list1, matcher, matcher, joinType); diff --git a/core/src/test/java/org/apache/calcite/test/enumerable/EnumerableJoinTest.java b/core/src/test/java/org/apache/calcite/test/enumerable/EnumerableJoinTest.java index 56393e184a12..063d8046f517 100644 --- a/core/src/test/java/org/apache/calcite/test/enumerable/EnumerableJoinTest.java +++ b/core/src/test/java/org/apache/calcite/test/enumerable/EnumerableJoinTest.java @@ -336,6 +336,30 @@ class EnumerableJoinTest { "empid=7"); } + /** Test case for + * [CALCITE-7817] + * Enumerable merge ANTI join drops left rows with NULL keys. */ + @Test void testMergeJoinAntiWithSingleKeyAndNullValues() { + tester(false, new HrSchema()) + .withHook(Hook.PLANNER, (Consumer) planner -> { + planner.addRule(EnumerableRules.ENUMERABLE_MERGE_JOIN_RULE); + planner.removeRule(EnumerableRules.ENUMERABLE_JOIN_RULE); + }) + .withRel(builder -> builder + .scan("s", "emps").as("e1") + .sort(builder.field("commission")) + .scan("s", "emps").as("e2") + .filter(builder.equals(builder.field("deptno"), builder.literal(20))) + .sort(builder.field("commission")) + .antiJoin( + builder.equals(builder.field(2, 0, "commission"), + builder.field(2, 1, "commission"))) + .project(builder.field("empid")) + .build()) + .explainHookContains("EnumerableMergeJoin") + .returnsUnordered("empid=100", "empid=110", "empid=150"); + } + private void checkMergeJoinWithCompositeKeyAndNullValues(boolean bigSchema, JoinRelType joinType, String... expected) { CalciteAssert.AssertQuery checker = diff --git a/linq4j/src/main/java/org/apache/calcite/linq4j/EnumerableDefaults.java b/linq4j/src/main/java/org/apache/calcite/linq4j/EnumerableDefaults.java index 395d8c26bdcf..d83e78eb3bc4 100644 --- a/linq4j/src/main/java/org/apache/calcite/linq4j/EnumerableDefaults.java +++ b/linq4j/src/main/java/org/apache/calcite/linq4j/EnumerableDefaults.java @@ -5123,7 +5123,7 @@ private boolean advance() { // iterate until finding matching keys (or ANTI join results) for (;;) { // mergeJoin assumes inputs sorted in ascending order with nulls last, - // if we reach a null key, we are done. + // if we reach a null key, there are no more matches. if (leftKey == null || rightKey == null) { if (isLeftOrAntiJoin()) { // all remaining items in left are results for left/anti join @@ -5226,7 +5226,8 @@ private boolean advance() { * is over or a different key is found. * * @return {@code true} if there are still elements to be processed on the left enumerator, - * {@code false} otherwise (left enumerator is over or null key is found). + * {@code false} otherwise (left enumerator is over, or null key is found and the join + * is not LEFT or ANTI). */ private boolean advanceLeft(TSource left, TKey leftKey) { lefts.clear();