From 7aa7cc651c2e0af82370d5b7bdc2aa9d53c184cb Mon Sep 17 00:00:00 2001 From: Joel Sahleen Date: Sat, 15 Aug 2026 20:07:02 -0600 Subject: [PATCH 1/3] scaffold: add tests for backslash sequences on XLIFF import MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Cover plain units, XML parse, PGS MF1/MF2, and export→import round-trip so JSON.parse keeps a single backslash. Refs #34 Co-authored-by: Cursor --- src/tests/format-xliff.integration.test.ts | 36 ++++++ src/tests/import-helpers.test.ts | 121 +++++++++++++++++++++ src/tests/pgs-mf1.test.ts | 14 +++ src/tests/pgs-mf2.test.ts | 15 +++ 4 files changed, 186 insertions(+) diff --git a/src/tests/format-xliff.integration.test.ts b/src/tests/format-xliff.integration.test.ts index 69aa5e2..c255da7 100644 --- a/src/tests/format-xliff.integration.test.ts +++ b/src/tests/format-xliff.integration.test.ts @@ -177,4 +177,40 @@ one {{One item}} expect(msg.value).toMatch(/\.match/); expect(msg.value).toContain("One item"); }); + + test("backslash sequences survive export→import JSON round-trip", () => { + const expected = "Hello \\{name\\} tab:\\t nl:\\n slash:\\\\"; + const project = createProject("escApp", { format: "NONE" }); + const resource = MsgResource.create( + { + title: "R", + attributes: { lang: "en", dir: "ltr", dnt: false }, + messages: [{ key: "esc", value: expected, attributes: { format: "NONE" } }], + }, + project + ); + + const exported = serializeResourceGroupsToXliff([ + { project: "escApp", resources: [resource] }, + ])[0]!.xliff; + expect(exported).toContain("\\{name\\}"); + + const bilingual = toBilingualXliff(exported, "zh"); + const parsed = parseXliff20(bilingual); + const xliffRoot = (parsed as Record).xliff as Record< + string, + unknown + >; + const fileEl = ( + Array.isArray(xliffRoot.file) ? xliffRoot.file[0] : xliffRoot.file + ) as Record; + + const imported = extractResourceFromXliffFile(fileEl, "zh", project, ["zh"]); + expect(imported).not.toBeNull(); + expect(imported!.get("esc")?.value).toBe(expected); + const json = JSON.parse(imported!.toJSON(true)) as { + messages: { value: string }[]; + }; + expect(json.messages[0]!.value).toBe(expected); + }); }); diff --git a/src/tests/import-helpers.test.ts b/src/tests/import-helpers.test.ts index 96c2cb5..5dedd9a 100644 --- a/src/tests/import-helpers.test.ts +++ b/src/tests/import-helpers.test.ts @@ -564,6 +564,127 @@ one {{一}} expect(result!.getData(true).messages![0].value).toBe("你好"); }); + test("preserves backslash sequences from XLIFF target through JSON.parse", () => { + const expected = "Hello \\{name\\} tab:\\t nl:\\n slash:\\\\"; + const fileEl = { + "@_original": "R.json", + "@_trgLang": "zh", + unit: { + "@_id": "u1", + "@_name": "esc", + segment: { source: "x", target: expected }, + }, + }; + const result = extractResourceFromXliffFile( + fileEl as unknown as Record, + "zh", + project, + ["zh"] + ); + expect(result!.get("esc")?.value).toBe(expected); + const parsed = JSON.parse(result!.toJSON(true)) as { + messages: { value: string }[]; + }; + expect(parsed.messages[0]!.value).toBe(expected); + }); + + test("preserves backslash sequences when parsing real XLIFF XML", () => { + const expected = "Hello \\{name\\} tab:\\t nl:\\n slash:\\\\"; + const xml = ` + + + + + x + ${expected} + + + +`; + const parsedXml = parseXliff20(xml) as Record; + const xliffRoot = parsedXml.xliff as Record; + const fileEl = ( + Array.isArray(xliffRoot.file) ? xliffRoot.file[0] : xliffRoot.file + ) as Record; + const result = extractResourceFromXliffFile(fileEl, "zh", project, ["zh"]); + expect(result!.get("esc")?.value).toBe(expected); + const parsed = JSON.parse(result!.toJSON(true)) as { + messages: { value: string }[]; + }; + expect(parsed.messages[0]!.value).toBe(expected); + }); + + test("preserves backslash sequences in PGS MF2 segment bodies", () => { + const body = "Hello \\{name\\}"; + const fileEl = { + "@_original": "R.json", + "@_trgLang": "zh", + unit: { + "@_id": "u1", + "@_name": "items", + "@_type": "msg:MF2", + "@_pgs:switch": "plural:n", + segment: [ + { + "@_pgs:case": "one", + source: "one", + target: body, + }, + { + "@_pgs:case": "other", + source: "other", + target: body, + }, + ], + }, + }; + const result = extractResourceFromXliffFile( + fileEl as unknown as Record, + "zh", + project, + ["zh"] + ); + const value = result!.get("items")?.value ?? ""; + expect(value).toContain("\\{name\\}"); + const parsed = JSON.parse(result!.toJSON(true)) as { + messages: { value: string }[]; + }; + expect(parsed.messages[0]!.value).toContain("\\{name\\}"); + expect(parsed.messages[0]!.value).not.toContain("\\\\{name\\\\}"); + }); + + test("preserves backslash sequences in PGS MF1 segment bodies", () => { + const body = "Hello \\{name\\}"; + const fileEl = { + "@_original": "R.json", + "@_trgLang": "zh", + unit: { + "@_id": "u1", + "@_name": "items", + "@_type": "msg:MF1", + "@_pgs:switch": "plural:count", + segment: [ + { "@_pgs:case": "one", source: "one", target: body }, + { "@_pgs:case": "other", source: "other", target: body }, + ], + }, + }; + const result = extractResourceFromXliffFile( + fileEl as unknown as Record, + "zh", + project, + ["zh"] + ); + const value = result!.get("items")?.value ?? ""; + // MF1 rebuild parses MF2-style `\{` as a literal brace and ICU-quotes it. + expect(value).toContain("Hello '{'name'}'"); + const parsed = JSON.parse(result!.toJSON(true)) as { + messages: { value: string }[]; + }; + expect(parsed.messages[0]!.value).toContain("Hello '{'name'}'"); + expect(parsed.messages[0]!.value).not.toContain("\\\\{"); + }); + test("extracts segment with target as object (inline elements)", () => { const fileEl = { "@_original": "I.json", diff --git a/src/tests/pgs-mf1.test.ts b/src/tests/pgs-mf1.test.ts index 51fdc18..998b70e 100644 --- a/src/tests/pgs-mf1.test.ts +++ b/src/tests/pgs-mf1.test.ts @@ -105,4 +105,18 @@ describe("pgs-mf1", () => { expect(back).toMatch(/\{g,\s*select,/); expect(back).toMatch(/\{n,\s*plural,/); }); + + test("import preserves backslash sequences in segment bodies through JSON.parse", () => { + const body = "tab:\\t nl:\\n slash:\\\\"; + const back = pgsImportToMf1Message("plural:n", [ + { caseAttr: "one", body }, + { caseAttr: "other", body }, + ]); + expect(back).not.toBeNull(); + expect(back).toContain("\\t"); + expect(back).toContain("\\n"); + expect(back).toContain("\\\\"); + const parsed = JSON.parse(JSON.stringify({ value: back })) as { value: string }; + expect(parsed.value).toBe(back); + }); }); diff --git a/src/tests/pgs-mf2.test.ts b/src/tests/pgs-mf2.test.ts index 4e80585..c263fbc 100644 --- a/src/tests/pgs-mf2.test.ts +++ b/src/tests/pgs-mf2.test.ts @@ -79,4 +79,19 @@ masculine {{His party}} "other", ]); }); + + test("import preserves backslash sequences in segment bodies", () => { + const body = "Hello \\{name\\} tab:\\t nl:\\n slash:\\\\"; + const back = pgsImportToSelectMessage("plural:n", [ + { caseAttr: "one", body }, + { caseAttr: "other", body }, + ]); + expect(back).not.toBeNull(); + expect(back).toContain("\\{name\\}"); + expect(back).toContain("\\t"); + expect(back).toContain("\\n"); + const parsed = JSON.parse(JSON.stringify({ value: back })) as { value: string }; + expect(parsed.value).toBe(back); + expect(parsed.value).not.toContain("\\\\{name\\\\}"); + }); }); From 53c21c975d88b3a3d0ebd005000298258030df60 Mon Sep 17 00:00:00 2001 From: Joel Sahleen Date: Sat, 15 Aug 2026 20:07:13 -0600 Subject: [PATCH 2/3] implement: splice PGS bodies so backslashes are not re-escaped parseMessage rejects \t/\n as bad MF2 escapes and dropped those segment bodies. Embed XLIFF text as-is and let JSON.stringify be the only JSON encoding step. Refs #34 Co-authored-by: Cursor --- src/lib/import-helpers.ts | 4 +++- src/lib/pgs-mf2.ts | 28 ++++++++++++++-------------- 2 files changed, 17 insertions(+), 15 deletions(-) diff --git a/src/lib/import-helpers.ts b/src/lib/import-helpers.ts index 0c8ef27..e3f2c17 100644 --- a/src/lib/import-helpers.ts +++ b/src/lib/import-helpers.ts @@ -197,7 +197,9 @@ function categoryToNoteType(category: string): string { return map[lower] ?? category.toUpperCase(); } -/** Extracts text from segment source/target, handling inline elements per XLIFF 2.0. */ +/** Extracts text from segment source/target, handling inline elements per XLIFF 2.0. + * Returns the already-decoded in-memory string; JSON encoding is left to `toJSON()`. + */ function extractSegmentText(segment: unknown): string { if (segment == null) return ""; const seg = segment as Record; diff --git a/src/lib/pgs-mf2.ts b/src/lib/pgs-mf2.ts index f05e181..db95713 100644 --- a/src/lib/pgs-mf2.ts +++ b/src/lib/pgs-mf2.ts @@ -278,18 +278,6 @@ export interface PgsSegmentImport { body: string; } -function parsePatternFromSegmentBody(body: string): unknown[] { - const trimmed = body.trim(); - if (!trimmed) return []; - try { - const m = parseMessage(trimmed); - if ((m as { type?: string }).type !== "message") return []; - return (m as { pattern: unknown[] }).pattern; - } catch { - return []; - } -} - function ensureFallbackVariant( keysCount: number, variants: SelectMessage["variants"] @@ -313,6 +301,9 @@ function ensureFallbackVariant( /** * Builds MF2 source string from PGS `pgs:switch` and segment bodies. + * + * Segment bodies are spliced into quoted patterns as-is so XLIFF `\` sequences + * are not dropped (`\t`/`\n` are invalid MF2 escapes) or double-escaped. */ export function pgsImportToSelectMessage( switchAttr: string, @@ -374,7 +365,9 @@ export function pgsImportToSelectMessage( ); variants.push({ keys, - value: parsePatternFromSegmentBody(seg.body), + // Empty pattern: stringify emits `{{}}`, then we splice the raw XLIFF + // body so `\t` / `\n` / `\{` are not dropped or double-escaped. + value: [], }); } @@ -397,9 +390,16 @@ export function pgsImportToSelectMessage( return null; } - return stringifyMessage( + const bodies = segments.map((seg) => seg.body); + while (bodies.length < msg.variants.length) { + bodies.push(segments[segments.length - 1]?.body ?? ""); + } + + const skeleton = stringifyMessage( msg as unknown as Parameters[0] ); + let i = 0; + return skeleton.replace(/\{\{\}\}/g, () => `{{${bodies[i++] ?? ""}}}`); } /** Compare two MF2 strings by parsing and re-stringifying both (for tests). */ From 2871114964cafbac617e9490f53cd84c838eef69 Mon Sep 17 00:00:00 2001 From: Joel Sahleen Date: Sat, 15 Aug 2026 20:07:18 -0600 Subject: [PATCH 3/3] document: note that import must not pre-escape backslashes Keep the import spec aligned with treating XLIFF text as already decoded before toJSON. Refs #34 Co-authored-by: Cursor --- src/specs/import-command.spec.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/specs/import-command.spec.md b/src/specs/import-command.spec.md index 7dc164f..50aae6a 100644 --- a/src/specs/import-command.spec.md +++ b/src/specs/import-command.spec.md @@ -147,7 +147,7 @@ Unit `type` values `msg:NONE` / `msg:MF1` / `msg:MF2` (or bare format tokens) ar - If there is a `translate` attribute and it is set to `no`, set the `dnt` property of the object to `true`. - If there are any `notes` associated with the `unit`, extract them to `MsgNote` objects using the uppercased category as the note `type` - Iterate through each `segment` in the `unit` and get the text for the segment translation for the `target` object. - - Reconstruct the complete translated `value` from the collected segments, according to the xliff 2.0 specification rules. + - Reconstruct the complete translated `value` from the collected segments, according to the xliff 2.0 specification rules. Treat segment text as already-decoded; do not pre-escape `\` before JSON serialization. For PGS units, splice segment bodies into the reconstructed message without re-parsing them as MF2 (so `\n`, `\t`, `\{`, and `\\` survive). - Use the `unit` element's `name` and the complete translated `value` for the unit, together with the `MsgAttribute` object and `MsgNote` array, to programmatically add a new message to the `MsgResource` created earlier. - Use MsgResource.toJSON(true) to get a serialized JSON string without notes. - Create a directory named after the project name inside `l10n/translations`, if it does not already exist