From 0aaca5975496073e33070fa0c3ea566adcf6afd8 Mon Sep 17 00:00:00 2001 From: Erez Shinan Date: Tue, 2 Jun 2026 06:21:34 +0200 Subject: [PATCH] Fix empty SPPF node from xearley ignore carry-over (fixes #1598) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When an item in to_scan was carried over past an ignored sequence with item.node=None (e.g. a prediction at ptr=0 that hasn't matched anything yet), the carry-over still allocated an empty SymbolNode and stored it in node_cache. That empty node later became the left child of a PackedNode when the terminal was finally scanned, and crashed ForestSumVisitor with "max() iterable argument is empty". Skip SymbolNode creation when item.node is None — there's nothing to merge, and new_item.node staying None makes the subsequent PackedNode have left=None, which is well-handled everywhere else. --- lark/parsers/xearley.py | 6 +++--- tests/test_parser.py | 15 +++++++++++++++ 2 files changed, 18 insertions(+), 3 deletions(-) diff --git a/lark/parsers/xearley.py b/lark/parsers/xearley.py index f334ed199..a015bb492 100644 --- a/lark/parsers/xearley.py +++ b/lark/parsers/xearley.py @@ -112,10 +112,10 @@ def scan(i, to_scan): else: # Handle items carried over due to ignores new_item = Item(item.rule, item.ptr, item.start) - label = (new_item.s, new_item.start, i + 1) - new_item.node = node_cache[label] if label in node_cache else node_cache.setdefault(label, self.SymbolNode(*label)) - if item.node: + if item.node is not None: # new_item.node and item.node both represent the same symbol, so merge their children + label = (new_item.s, new_item.start, i + 1) + new_item.node = node_cache[label] if label in node_cache else node_cache.setdefault(label, self.SymbolNode(*label)) for child in item.node.children: new_item.node.add_family(new_item.s, child.rule, new_item.start, child.left, child.right) diff --git a/tests/test_parser.py b/tests/test_parser.py index 533ae7890..721e8693a 100644 --- a/tests/test_parser.py +++ b/tests/test_parser.py @@ -905,6 +905,21 @@ def test_multiple_start_solutions(self): tree = l.parse('x') assert tree == Tree('start', [Tree('a', ['x'])]) + @unittest.skipIf(LEXER=='basic', 'Ignore carry-over is dynamic-lexer-specific') + def test_ignore_carryover_with_priority(self): + """Items in to_scan with item.node=None must not produce an empty SPPF + node when carried over past an ignored sequence (issue #1598).""" + grammar = r""" + start: a + a.1: NAME + NAME: /[a-z]+/ + %import common.WS + %ignore WS + """ + l = Lark(grammar, parser='earley', lexer=LEXER) + tree = l.parse(' hello') + self.assertEqual(tree, Tree('start', [Tree('a', ['hello'])])) + @unittest.skipIf(LEXER=='basic', 'This scenario only occurs with the dynamic lexers') def test_multiple_start_solutions2(self): grammar = r"""