From ff74f6f8125b00b8f59d611392b73866b0aecdf9 Mon Sep 17 00:00:00 2001 From: Abhin Rustagi Date: Wed, 9 Sep 2026 12:18:46 +0530 Subject: [PATCH 1/2] fix: last line win in streaming parser --- .../src/parser/__tests__/parser.test.ts | 83 +++++++++++++++++++ packages/lang-core/src/parser/parser.ts | 23 +++-- 2 files changed, 100 insertions(+), 6 deletions(-) diff --git a/packages/lang-core/src/parser/__tests__/parser.test.ts b/packages/lang-core/src/parser/__tests__/parser.test.ts index b95ed32b2..71a88ec90 100644 --- a/packages/lang-core/src/parser/__tests__/parser.test.ts +++ b/packages/lang-core/src/parser/__tests__/parser.test.ts @@ -287,3 +287,86 @@ root = Title("hello") expect(result.root?.props.text).toBe("hello"); }); }); + +// ── redefined statement IDs (parse vs stream) ─────────────────────────────── + +describe("redefined statement IDs", () => { + const withNewline = `root = Stack([a]) +a = Title("x") +a = Title("y") +`; + const withoutNewline = `root = Stack([a]) +a = Title("x") +a = Title("y")`; + + const titleText = (result: ReturnType) => { + const children = result.root?.props?.children as any[] | undefined; + return children?.[0]?.props?.text as string | undefined; + }; + + const pushAll = ( + text: string, + write: (sp: ReturnType, text: string) => void, + ) => { + const sp = createStreamParser(schema); + write(sp, text); + return titleText(sp.getResult()); + }; + + it("parse() last-wins with and without a trailing newline", () => { + expect(titleText(parse(withNewline, schema))).toBe("y"); + expect(titleText(parse(withoutNewline, schema))).toBe("y"); + }); + + it("stream parser last-wins for a single push, matching parse()", () => { + expect(pushAll(withNewline, (sp, text) => sp.push(text))).toBe("y"); + expect(pushAll(withoutNewline, (sp, text) => sp.push(text))).toBe("y"); + }); + + it("stream parser last-wins line-by-line and character-by-character", () => { + expect( + pushAll(withNewline, (sp, text) => { + for (const line of text.split(/(?<=\n)/)) { + if (line) sp.push(line); + } + }), + ).toBe("y"); + expect( + pushAll(withNewline, (sp, text) => { + for (const ch of text) sp.push(ch); + }), + ).toBe("y"); + expect( + pushAll(withoutNewline, (sp, text) => { + for (const ch of text) sp.push(ch); + }), + ).toBe("y"); + }); + + it("stream parser last-wins across a two-chunk split", () => { + expect( + pushAll("", (sp) => { + sp.push(`root = Stack([a])\na = Title("x")\n`); + sp.push(`a = Title("y")\n`); + }), + ).toBe("y"); + expect( + pushAll("", (sp) => { + sp.push(`root = Stack([a])\na = Title("x")\n`); + sp.push(`a = Title("y")`); + }), + ).toBe("y"); + }); + + it("stream parser last-wins via set(), matching the renderer path", () => { + const sp = createStreamParser(schema); + expect(titleText(sp.set(withoutNewline))).toBe("y"); + }); + + it("does not let an incomplete pending redefinition clobber a completed statement", () => { + const sp = createStreamParser(schema); + sp.push(`root = Stack([a])\na = Title("x")\n`); + expect(titleText(sp.push(`a = Title("y`))).toBe("x"); + expect(titleText(sp.push(`")\n`))).toBe("y"); + }); +}); diff --git a/packages/lang-core/src/parser/parser.ts b/packages/lang-core/src/parser/parser.ts index 2d201d559..d649c2746 100644 --- a/packages/lang-core/src/parser/parser.ts +++ b/packages/lang-core/src/parser/parser.ts @@ -398,9 +398,18 @@ function stripComments(input: string): string { .join("\n"); } -/** Clean LLM response: strip fences, comments, whitespace. */ +/** Clean LLM response: strip fences, comments, and surrounding whitespace. + * + * Trailing newlines are preserved. The stream parser commits a statement only + * on newline; trimming them makes the last statement look pending forever, so a + * later redefinition of an earlier ID is skipped by the pending-merge guard. + */ function preprocess(input: string): string { - return stripComments(stripFences(input.trim())).trim(); + const stripped = stripComments(stripFences(input.trimStart())); + const content = stripped.trim(); + if (!content) return ""; + const trailingNewlines = stripped.match(/\n*$/)?.[0] ?? ""; + return content + trailingNewlines; } /** @@ -582,12 +591,14 @@ export function createStreamParser(cat: ParamMap, rootName?: string): StreamPars } // Merge: completed cache + re-parsed pending statement. - // Pending statements can only add NEW IDs — they cannot overwrite completed ones. - // This prevents mid-stream partial text (e.g. `root = Card`) from corrupting - // existing completed statements during edit streaming. + // Incomplete pending text cannot overwrite completed IDs — autoClose would + // otherwise invent closers (e.g. `root = Card(` → `root = Card()`) and + // clobber a finished definition mid-stream. + // Complete pending statements last-wins, matching parse(). Needed when the + // last statement has no trailing newline and therefore never hits addStmt. const allStmtMap = new Map(completedStmtMap); for (const s of stmts) { - if (completedStmtMap.has(s.id)) continue; + if (wasIncomplete && completedStmtMap.has(s.id)) continue; const expr = parseExpression(s.tokens); const stmt = classifyStatement(s, expr); allStmtMap.set(s.id, stmt); From 0e62903d6dc953e7ccdaa7b3a2da23e300af6016 Mon Sep 17 00:00:00 2001 From: Abhin Rustagi Date: Wed, 16 Sep 2026 23:42:25 +0530 Subject: [PATCH 2/2] fix: retain old newline behaviour --- .changeset/fix-stream-redefined-ids.md | 5 +++++ packages/lang-core/src/parser/parser.ts | 14 +++++++------- 2 files changed, 12 insertions(+), 7 deletions(-) create mode 100644 .changeset/fix-stream-redefined-ids.md diff --git a/.changeset/fix-stream-redefined-ids.md b/.changeset/fix-stream-redefined-ids.md new file mode 100644 index 000000000..e91017950 --- /dev/null +++ b/.changeset/fix-stream-redefined-ids.md @@ -0,0 +1,5 @@ +--- +"@openuidev/lang-core": patch +--- + +Fix streaming parsing so the last complete definition of a repeated statement ID wins, matching the non-streaming parser. Keep the previous completed definition while a replacement is incomplete. diff --git a/packages/lang-core/src/parser/parser.ts b/packages/lang-core/src/parser/parser.ts index d649c2746..e5df74377 100644 --- a/packages/lang-core/src/parser/parser.ts +++ b/packages/lang-core/src/parser/parser.ts @@ -400,14 +400,14 @@ function stripComments(input: string): string { /** Clean LLM response: strip fences, comments, and surrounding whitespace. * - * Trailing newlines are preserved. The stream parser commits a statement only - * on newline; trimming them makes the last statement look pending forever, so a - * later redefinition of an earlier ID is skipped by the pending-merge guard. + * Streaming preserves trailing newlines so the final completed statement can + * enter the cache. Non-streaming keeps the original full-trimming behavior. */ -function preprocess(input: string): string { - const stripped = stripComments(stripFences(input.trimStart())); +function preprocess(input: string, preserveTrailingNewlines = false): string { + const trimmed = preserveTrailingNewlines ? input.trimStart() : input.trim(); + const stripped = stripComments(stripFences(trimmed)); const content = stripped.trim(); - if (!content) return ""; + if (!content || !preserveTrailingNewlines) return content; const trailingNewlines = stripped.match(/\n*$/)?.[0] ?? ""; return content + trailingNewlines; } @@ -483,7 +483,7 @@ export function createStreamParser(cat: ParamMap, rootName?: string): StreamPars // and re-scan. When the prefix is stable (the common streaming case) the cache // is kept, so a partial trailing statement never blanks already-completed ones. function refreshCleaned() { - const next = preprocess(buf); + const next = preprocess(buf, true); if (!next.startsWith(cleaned.slice(0, completedEnd))) { completedEnd = 0; completedStmtMap.clear();