Re-enable Joop Leo right-recursion optimization (fixes #397)

lark/parsers/earley.py

@@ -18,7 +18,7 @@
 from ..utils import logger, OrderedSet, dedup_list
 from .grammar_analysis import GrammarAnalyzer
 from ..grammar import NonTerminal
-from .earley_common import Item
+from .earley_common import Item, TransitiveItem
 from .earley_forest import ForestSumVisitor, SymbolNode, StableSymbolNode, TokenNode, ForestToParseTree
 
 if TYPE_CHECKING:
@@ -75,7 +75,7 @@ def __init__(self, lexer_conf: 'LexerConf', parser_conf: 'ParserConf', term_matc
         self.term_matcher = term_matcher
 
 
-    def predict_and_complete(self, i, to_scan, columns, transitives, node_cache):
+    def predict_and_complete(self, i, to_scan, columns, transitives, node_cache, start_symbol=None):
         """The core Earley Predictor and Completer.
 
         At each stage of the input, we handling any completed items (things
@@ -86,6 +86,70 @@ def predict_and_complete(self, i, to_scan, columns, transitives, node_cache):
         # Held Completions (H in E.Scotts paper).
         held_completions = {}
 
+        def is_quasi_complete(item):
+            quasi = item.advance()
+            while not quasi.is_complete:
+                if quasi.expect not in self.NULLABLE:
+                    return False
+                if quasi.rule.origin == start_symbol and quasi.expect == start_symbol:
+                    return False
+                quasi = quasi.advance()
+            return True
+
+        def create_leo_transitives(origin, start):
+            visited = set()
+            to_create = []
+            trule = None
+            previous = None
+
+            ### Recursively walk backwards through the Earley sets until we find the
+            #   first transitive candidate. If this is done continuously, we shouldn't
+            #   have to walk more than 1 hop.
+            while True:
+                if origin in transitives[start]:
+                    previous = trule = transitives[start][origin]
+                    break
+
+                if not self.FIRST[origin]:
+                    break
+
+                if origin in self.NULLABLE:
+                    break
+
+                candidates = [c for c in columns[start] if c.expect is not None and c.expect == origin]
+                if len(candidates) != 1:
+                    break
+                originator = candidates[0]
+
+                if originator in visited:
+                    break
+
+                visited.add(originator)
+                if not is_quasi_complete(originator):
+                    break
+
+                trule = originator.advance()
+                if originator.start != start:
+                    visited.clear()
+
+                to_create.append((origin, start, originator))
+                origin = originator.rule.origin
+                start = originator.start
+
+            # If a suitable Transitive candidate is not found, bail.
+            if trule is None:
+                return
+
+            ### Now walk forwards and create Transitive Items in each set we walked through;
+            #   and link each transitive item to the next set forwards.
+            while to_create:
+                origin, start, originator = to_create.pop()
+                if previous is not None:
+                    titem = previous.next_titem = TransitiveItem(origin, trule, originator, previous.column)
+                else:
+                    titem = TransitiveItem(origin, trule, originator, start)
+                previous = transitives[start][origin] = titem
+
         column = columns[i]
         # R (items) = Ei (column.items)
         items = deque(column)
@@ -99,24 +163,23 @@ def predict_and_complete(self, i, to_scan, columns, transitives, node_cache):
                     item.node = node_cache[label] if label in node_cache else node_cache.setdefault(label, self.SymbolNode(*label))
                     item.node.add_family(item.s, item.rule, item.start, None, None)
 
-                # create_leo_transitives(item.rule.origin, item.start)
+                create_leo_transitives(item.rule.origin, item.start)
 
                 ###R Joop Leo right recursion Completer
                 if item.rule.origin in transitives[item.start]:
-                    transitive = transitives[item.start][item.s]
-                    if transitive.previous in transitives[transitive.column]:
-                        root_transitive = transitives[transitive.column][transitive.previous]
-                    else:
-                        root_transitive = transitive
+                    transitive = transitives[item.start][item.rule.origin]
+                    # Every titem is inserted into transitives[col][previous],
+                    # so this lookup always finds the chain's root (or transitive
+                    # itself when it is the root).
+                    root_transitive = transitives[transitive.column][transitive.previous]
 
                     new_item = Item(transitive.rule, transitive.ptr, transitive.start)
                     label = (root_transitive.s, root_transitive.start, i)
                     new_item.node = node_cache[label] if label in node_cache else node_cache.setdefault(label, self.SymbolNode(*label))
                     new_item.node.add_path(root_transitive, item.node)
-                    if new_item.expect in self.TERMINALS:
-                        # Add (B :: aC.B, h, y) to Q
-                        to_scan.add(new_item)
-                    elif new_item not in column:
+                    # new_item.expect cannot be a terminal: is_quasi_complete
+                    # required the post-recursion suffix to be all-nullable.
+                    if new_item not in column:
                         # Add (B :: aC.B, h, y) to Ei and R
                         column.add(new_item)
                         items.append(new_item)
@@ -127,8 +190,7 @@ def predict_and_complete(self, i, to_scan, columns, transitives, node_cache):
                     # any predictions that result, that themselves require empty. Avoids
                     # infinite recursion on empty symbols.
                     # held_completions is 'H' in E.Scott's paper.
-                    is_empty_item = item.start == i
-                    if is_empty_item:
+                    if item.start == i:
                         held_completions[item.rule.origin] = item.node
 
                     originators = [originator for originator in columns[item.start] if originator.expect is not None and originator.expect == item.s]
@@ -169,22 +231,6 @@ def predict_and_complete(self, i, to_scan, columns, transitives, node_cache):
 
     def _parse(self, lexer, columns, to_scan, start_symbol=None):
 
-        def is_quasi_complete(item):
-            if item.is_complete:
-                return True
-
-            quasi = item.advance()
-            while not quasi.is_complete:
-                if quasi.expect not in self.NULLABLE:
-                    return False
-                if quasi.rule.origin == start_symbol and quasi.expect == start_symbol:
-                    return False
-                quasi = quasi.advance()
-            return True
-
-        # def create_leo_transitives(origin, start):
-        #   ...   # removed at commit 4c1cfb2faf24e8f8bff7112627a00b94d261b420
-
         def scan(i, token, to_scan):
             """The core Earley Scanner.
 
@@ -246,15 +292,15 @@ def scan(i, token, to_scan):
         i = 0
         node_cache = {}
         for token in lexer.lex(expects):
-            self.predict_and_complete(i, to_scan, columns, transitives, node_cache)
+            self.predict_and_complete(i, to_scan, columns, transitives, node_cache, start_symbol)
 
             to_scan, node_cache = scan(i, token, to_scan)
             i += 1
 
             expects.clear()
             expects |= {i.expect for i in to_scan}
 
-        self.predict_and_complete(i, to_scan, columns, transitives, node_cache)
+        self.predict_and_complete(i, to_scan, columns, transitives, node_cache, start_symbol)
 
         ## Column is now the final column in the parse.
         assert i == len(columns)-1

lark/parsers/earley_common.py

@@ -38,5 +38,22 @@ def __repr__(self):
         return '%s (%d)' % (symbol, self.start)
 
 
-# class TransitiveItem(Item):
-#   ...   # removed at commit 4c1cfb2faf24e8f8bff7112627a00b94d261b420
+class TransitiveItem(Item):
+    """A Joop Leo transitive item, for shortcutting right-recursive completions."""
+    __slots__ = ('recognized', 'reduction', 'column', 'next_titem')
+
+    def __init__(self, recognized, trule, originator, start):
+        super(TransitiveItem, self).__init__(trule.rule, trule.ptr, trule.start)
+        self.recognized = recognized
+        self.reduction = originator
+        self.column = start
+        self.next_titem = None
+        self._hash = hash((self.s, self.start, self.recognized))
+
+    def __eq__(self, other):
+        if not isinstance(other, TransitiveItem):
+            return False
+        return self is other or (type(self.s) == type(other.s) and self.s == other.s and self.start == other.start and self.recognized == other.recognized)
+
+    def __hash__(self):
+        return self._hash

lark/parsers/earley_forest.py

@@ -67,13 +67,27 @@ def add_path(self, transitive, node):
         self.paths.add((transitive, node))
 
     def load_paths(self):
+        # Share canonical intermediate symbol nodes across multi-hop Leo chains:
+        # SPPF identity is (s, start, end), so any path whose node matches the
+        # expected intermediate's label is reused as the canonical node, and other
+        # paths targeting the same label merge into it via add_path.
+        node_cache = {(node.s, node.start, node.end): node for (_, node) in self.paths}
         for transitive, node in self.paths:
+            next_node = node
             if transitive.next_titem is not None:
-                vn = type(self)(transitive.next_titem.s, transitive.next_titem.start, self.end)
-                vn.add_path(transitive.next_titem, node)
-                self.add_family(transitive.reduction.rule.origin, transitive.reduction.rule, transitive.reduction.start, transitive.reduction.node, vn)
-            else:
-                self.add_family(transitive.reduction.rule.origin, transitive.reduction.rule, transitive.reduction.start, transitive.reduction.node, node)
+                label = (transitive.next_titem.reduction.rule.origin,
+                         transitive.next_titem.reduction.start,
+                         self.end)
+                if label != (node.s, node.start, node.end):
+                    if label in node_cache:
+                        # Canonical already added as right-side by an earlier path; just merge.
+                        node_cache[label].add_path(transitive.next_titem, node)
+                        continue
+                    next_node = node_cache[label] = type(self)(*label)
+                    next_node.add_path(transitive.next_titem, node)
+
+            self.add_family(transitive.reduction.rule.origin, transitive.reduction.rule,
+                            transitive.reduction.start, transitive.reduction.node, next_node)
         self.paths_loaded = True
 
     @property

lark/parsers/xearley.py

@@ -156,7 +156,7 @@ def scan(i, to_scan):
         i = 0
         node_cache = {}
         for token in stream:
-            self.predict_and_complete(i, to_scan, columns, transitives, node_cache)
+            self.predict_and_complete(i, to_scan, columns, transitives, node_cache, start_symbol)
 
             to_scan, node_cache = scan(i, to_scan)
 
@@ -167,7 +167,7 @@ def scan(i, to_scan):
                 text_column += 1
             i += 1
 
-        self.predict_and_complete(i, to_scan, columns, transitives, node_cache)
+        self.predict_and_complete(i, to_scan, columns, transitives, node_cache, start_symbol)
 
         ## Column is now the final column in the parse.
         assert i == len(columns)-1

tests/test_parser.py

@@ -1070,6 +1070,75 @@ def test_consistent_derivation_order1(self):
             n = Tree('a', [])
             assert tree == Tree('start', [n, n])
 
+        def test_earley_jeo_right_recursion(self):
+            """Joop Leo optimization should correctly handle right-recursive grammars"""
+            grammar = r"""
+            start: "x" start | "x"
+            """
+            l = Lark(grammar, parser='earley', lexer=LEXER)
+            res = l.parse('xxx')
+            self.assertEqual(res, Tree('start', [Tree('start', [Tree('start', [])])]))
+
+        def test_earley_jeo_multi_start_right_recursion(self):
+            """Joop Leo optimization should work correctly with multiple start symbols (issue #397)"""
+            grammar = r"""
+            a: "x" a | "x"
+            b: "y" b | "y"
+            """
+            l = Lark(grammar, parser='earley', lexer=LEXER, start=['a', 'b'])
+
+            res_a = l.parse('xxx', start='a')
+            self.assertEqual(res_a, Tree('a', [Tree('a', [Tree('a', [])])]))
+
+            res_b = l.parse('yyy', start='b')
+            self.assertEqual(res_b, Tree('b', [Tree('b', [Tree('b', [])])]))
+
+        def test_earley_jeo_chain_stops_at_nonnullable_suffix(self):
+            """Leo chain walks outward until the outer rule has a non-nullable
+            suffix after the recursive position, at which point is_quasi_complete
+            returns False (covers the non-NULLABLE check + break in
+            create_leo_transitives)."""
+            grammar = r"""
+            start: a "z"
+            a: "x" a | "y"
+            """
+            l = Lark(grammar, parser='earley', lexer=LEXER)
+            tree = l.parse('xxyz')
+            self.assertEqual(tree, Tree('start', [Tree('a', [Tree('a', [Tree('a', [])])])]))
+
+        def test_earley_jeo_chain_extends_through_nullable_suffix(self):
+            """Leo chain extends into an outer rule when the trailing symbols
+            are all nullable: is_quasi_complete traverses them via the
+            NULLABLE check + advance loop."""
+            grammar = r"""
+            start: a opt
+            a: "x" a | "y"
+            opt: "q" |
+            """
+            l = Lark(grammar, parser='earley', lexer=LEXER)
+            tree_no_opt = l.parse('xxy')
+            self.assertEqual(tree_no_opt, Tree('start', [Tree('a', [Tree('a', [Tree('a', [])])]), Tree('opt', [])]))
+            tree_with_opt = l.parse('xxyq')
+            self.assertEqual(tree_with_opt, Tree('start', [Tree('a', [Tree('a', [Tree('a', [])])]), Tree('opt', [])]))
+
+        def test_earley_jeo_nullable_start_self_recursion(self):
+            """is_quasi_complete's start-symbol self-reference guard: when the
+            Leo chain walks into a rule of the form `start: ... <nullable> start ...`
+            with `start` itself nullable, the guard returns False so the chain
+            doesn't pretend the start rule is quasi-complete."""
+            grammar = r"""
+            start: a x start |
+            x:
+            a: "x" a | "y"
+            """
+            l = Lark(grammar, parser='earley', lexer=LEXER)
+            self.assertEqual(l.parse(''), Tree('start', []))
+            tree = l.parse('xxy')
+            self.assertEqual(
+                tree,
+                Tree('start', [Tree('a', [Tree('a', [Tree('a', [])])]), Tree('x', []), Tree('start', [])])
+            )
+
     _NAME = "TestFullEarley" + LEXER.capitalize()
     _TestFullEarley.__name__ = _NAME
     globals()[_NAME] = _TestFullEarley