Stream Parser Fix

packages/lang-core/src/parser/__tests__/parser.test.ts

@@ -287,3 +287,86 @@ root = Title("hello")
     expect(result.root?.props.text).toBe("hello");
   });
 });
+
+// ── redefined statement IDs (parse vs stream) ───────────────────────────────
+
+describe("redefined statement IDs", () => {
+  const withNewline = `root = Stack([a])
+a = Title("x")
+a = Title("y")
+`;
+  const withoutNewline = `root = Stack([a])
+a = Title("x")
+a = Title("y")`;
+
+  const titleText = (result: ReturnType<typeof parse>) => {
+    const children = result.root?.props?.children as any[] | undefined;
+    return children?.[0]?.props?.text as string | undefined;
+  };
+
+  const pushAll = (
+    text: string,
+    write: (sp: ReturnType<typeof createStreamParser>, text: string) => void,
+  ) => {
+    const sp = createStreamParser(schema);
+    write(sp, text);
+    return titleText(sp.getResult());
+  };
+
+  it("parse() last-wins with and without a trailing newline", () => {
+    expect(titleText(parse(withNewline, schema))).toBe("y");
+    expect(titleText(parse(withoutNewline, schema))).toBe("y");
+  });
+
+  it("stream parser last-wins for a single push, matching parse()", () => {
+    expect(pushAll(withNewline, (sp, text) => sp.push(text))).toBe("y");
+    expect(pushAll(withoutNewline, (sp, text) => sp.push(text))).toBe("y");
+  });
+
+  it("stream parser last-wins line-by-line and character-by-character", () => {
+    expect(
+      pushAll(withNewline, (sp, text) => {
+        for (const line of text.split(/(?<=\n)/)) {
+          if (line) sp.push(line);
+        }
+      }),
+    ).toBe("y");
+    expect(
+      pushAll(withNewline, (sp, text) => {
+        for (const ch of text) sp.push(ch);
+      }),
+    ).toBe("y");
+    expect(
+      pushAll(withoutNewline, (sp, text) => {
+        for (const ch of text) sp.push(ch);
+      }),
+    ).toBe("y");
+  });
+
+  it("stream parser last-wins across a two-chunk split", () => {
+    expect(
+      pushAll("", (sp) => {
+        sp.push(`root = Stack([a])\na = Title("x")\n`);
+        sp.push(`a = Title("y")\n`);
+      }),
+    ).toBe("y");
+    expect(
+      pushAll("", (sp) => {
+        sp.push(`root = Stack([a])\na = Title("x")\n`);
+        sp.push(`a = Title("y")`);
+      }),
+    ).toBe("y");
+  });
+
+  it("stream parser last-wins via set(), matching the renderer path", () => {
+    const sp = createStreamParser(schema);
+    expect(titleText(sp.set(withoutNewline))).toBe("y");
+  });
+
+  it("does not let an incomplete pending redefinition clobber a completed statement", () => {
+    const sp = createStreamParser(schema);
+    sp.push(`root = Stack([a])\na = Title("x")\n`);
+    expect(titleText(sp.push(`a = Title("y`))).toBe("x");
+    expect(titleText(sp.push(`")\n`))).toBe("y");
+  });
+});

packages/lang-core/src/parser/parser.ts

@@ -398,9 +398,18 @@ function stripComments(input: string): string {
     .join("\n");
 }
 
-/** Clean LLM response: strip fences, comments, whitespace. */
+/** Clean LLM response: strip fences, comments, and surrounding whitespace.
+ *
+ * Trailing newlines are preserved. The stream parser commits a statement only
+ * on newline; trimming them makes the last statement look pending forever, so a
+ * later redefinition of an earlier ID is skipped by the pending-merge guard.
+ */
 function preprocess(input: string): string {
-  return stripComments(stripFences(input.trim())).trim();
+  const stripped = stripComments(stripFences(input.trimStart()));
+  const content = stripped.trim();
+  if (!content) return "";
+  const trailingNewlines = stripped.match(/\n*$/)?.[0] ?? "";
+  return content + trailingNewlines;
 }
 
 /**
@@ -582,12 +591,14 @@ export function createStreamParser(cat: ParamMap, rootName?: string): StreamPars
     }
 
     // Merge: completed cache + re-parsed pending statement.
-    // Pending statements can only add NEW IDs — they cannot overwrite completed ones.
-    // This prevents mid-stream partial text (e.g. `root = Card`) from corrupting
-    // existing completed statements during edit streaming.
+    // Incomplete pending text cannot overwrite completed IDs — autoClose would
+    // otherwise invent closers (e.g. `root = Card(` → `root = Card()`) and
+    // clobber a finished definition mid-stream.
+    // Complete pending statements last-wins, matching parse(). Needed when the
+    // last statement has no trailing newline and therefore never hits addStmt.
     const allStmtMap = new Map(completedStmtMap);
     for (const s of stmts) {
-      if (completedStmtMap.has(s.id)) continue;
+      if (wasIncomplete && completedStmtMap.has(s.id)) continue;
       const expr = parseExpression(s.tokens);
       const stmt = classifyStatement(s, expr);
       allStmtMap.set(s.id, stmt);