diff --git a/.github/PUBLISHING.md b/.github/PUBLISHING.md index a812aaa..17cf7c6 100644 --- a/.github/PUBLISHING.md +++ b/.github/PUBLISHING.md @@ -51,7 +51,13 @@ version number can never be reused once published. `pytest --update-golden tests/test_golden.py`, and check that `git diff tests/golden` shows only that line. Also update the version named in [`CONFORMANCE.md`](../CONFORMANCE.md) and the milestone in [`docs/roadmap.md`](../docs/roadmap.md). -3. Commit, then publish a GitHub Release with the tag `vX.Y.Z`. Its notes are where this +3. For a release that changes the DOCX writer, run the checklist below ("Checking the DOCX in Word") first, and record the + Word version and platform (for example Word for Microsoft 365, Windows 11, build number; or Word for Mac) with + its findings in the release notes. Then update what says the DOCX is not yet checked in Word: the row of + [`CONFORMANCE.md`](../CONFORMANCE.md) ("DOCX output", and its section), the heading of the milestone in + [`docs/roadmap.md`](../docs/roadmap.md), and the README's DOCX section; and decide whether the renderer now claims + DOCX for §13.6 or keeps HTML as the format it claims it with. +4. Commit, then publish a GitHub Release with the tag `vX.Y.Z`. Its notes are where this project records what changed in a release. The workflow first runs the CI workflow on the tagged commit. Only if that passes does it build the @@ -78,6 +84,176 @@ legaldown-render --version The extra index is needed because the dependencies (`legaldown-validator`, Babel, PyYAML, markdown-it-py) come from PyPI. +## Checking the DOCX in Word (before a release) + +CI proves the DOCX against the ISO/IEC 29500 schemas and in LibreOffice; it cannot run Word. Before a release, a maintainer +opens the golden documents in Word (the desktop application, a current version) and checks what only Word can show. +Make them with the Python API, which is the same on Linux, macOS and Windows (run it from the repository root, in an +environment with the package installed): + +```python +from pathlib import Path +from legaldown_render import render + +out = Path("word-check") # any folder; created here +out.mkdir(exist_ok=True) +contents = {"contents.enabled": True, "contents.depth": 3} +for doc in ("features", "template"): + source = Path(f"tests/documents/{doc}.lgd").read_text(encoding="utf-8") + for style in ("default", "continental", "outline"): + (out / f"{doc}.{style}.docx").write_bytes(render(source, format="docx", style=style, overrides=contents).binary) + # the settings whose effect only Word shows (see items 6, 7 and 9); `True` marks the ones about attachments, which + # template.lgd has none of + for name, (about_attachments, extra) in { + "attachments-not-listed": (True, {"contents.attachments": False}), + "depth-1": (False, {"contents.depth": 1}), + "no-attachments-label": (True, {"labels.attachments": ""}), + "separator-none": (True, {"attachments.separator": "none"}), + "separator-rule": (True, {"attachments.separator": "rule"}), + "scheme-none": (False, {"numbering.scheme": "none"}), + "paragraphs-numbered": (False, {"paragraphs.numbered": True}), # `1.1` paragraphs on the heading list (item 9) + }.items(): + if about_attachments and doc == "template": + continue + data = render(source, format="docx", overrides={**contents, **extra}).binary + (out / f"{doc}.{name}.docx").write_bytes(data) + +# the document of item 8's "a level nothing numbered" line: items e and f stand on a level of their list that nothing +# in the list numbered before them +uncounted = "---\ntitle: T\n---\n# A\n\n- a\n - b\n 1. c\n - d\n - e\n - f\n" +(out / "uncounted.default.docx").write_bytes(render(uncounted, format="docx").binary) +``` + +(On the command line, `legaldown-render tests/documents/features.lgd -s continental --set contents.enabled=true +--set contents.depth=3 -o word-check/features.continental.docx` makes one of them; create the folder first with +`mkdir -p word-check` on Linux and macOS or `mkdir word-check` on Windows.) + +For each file: + +1. **Opens clean.** No "unreadable content" or repair prompt, and Word does not open it in Compatibility Mode. + Answer *Yes* to the prompt to update the fields (`docx.update_fields_on_open`), or press Ctrl+A, F9. (With + `--set docx.update_fields_on_open=false` there is no prompt; the numbers and references already read right, since + the file caches what Word computes, and only the page numbers of the contents are missing until the fields are + updated.) The document's title is in File, Info, and its language in Review, Language. +2. **Table of contents.** It lists exactly the entries of the HTML contents (numbers and titles; the condition + labels of a template after the title), indented by level, with a dotted leader and page numbers that are right. + The attachments heading and each attachment's title are listed (at the levels the HTML shows) and nothing else is: + not the "Contents" label, no quote or note. A click on an entry (Ctrl+click) goes to the heading. Right-click the + TOC, *Update field*, *Update entire table*: the same entries come back. Before the update (answer *No* to the + prompt) the entries are already there and read the same, without page numbers. The headings, and nothing else, are in + the navigation pane. +3. **References.** Every `{{ref:}}` reads as in the HTML before and after the update (Ctrl+A, F9). Insert a clause + (a heading, a list item, a numbered paragraph) above a referenced one, update the fields: the numbers after it move + and every reference and the contents follow. A reference to an attachment (`{{attach:}}`) shows its title and links + to it, and a `{{term:}}` is a link (Ctrl+click) to the definition of the term; an external link opens its address. +4. **Numbering after inserting a clause.** Press Enter at the end of a numbered item and at the start of one: the new + paragraph is numbered, the rest renumbers, nested lists restart under it. In the continental style a new + sub-item continues `a)`, `b)`. A bullet list counts nothing: an ordered list after a bullet list starts at its first + number, and two bullet lists show the same bullet. +5. **Zero-length bookmarks.** A `REF \n` to the zero-length (point) bookmark at the start of a numbered paragraph + returns the number. Press Enter at the start of that paragraph and update the fields: the references still + point at the paragraph they pointed at and read right. +6. **`none` scheme** (the `scheme-none` files): a heading referenced with `{{ref:}}` shows its title text + (`REF \h`), and the references inside a heading that refers to itself are plain text: no "Error! Reference source + not found" and no self-reference. +7. **Front and back matter.** The title block (header rows, parties) is two aligned columns without borders, the + labels in italics; each attachment starts on a new page with `attachments.separator: page-break` and has a rule above + it with `rule`; with `none` (`separator-none`) there is no break and no rule; the signature blocks are borderless, + with a signing line above each name, and stay on one page when one page can hold them (try a longer document). The + settings variants: + - `attachments-not-listed`: the attachments heading and titles are **absent** from the table of contents and from + the navigation pane (outline level 9, body text). + - `depth-1`: the attachments are still in the contents, as first-level entries (the HTML indents the titles one + level more, the documented difference), and no second-level heading is. + - `no-attachments-label`: there is no attachments heading, and each title is a first-level entry. + - `\u` honours the outline levels of the attachment styles and paragraphs: the attachment titles are entries, at + the level the table of contents gives them, in every variant that lists them. If Word does not list them, the + fix is a `\t "Attachment Title,2"` list in the field (locale list separator permitting); report it. +8. **The numbering corner cases.** Make the check document with `python tests/word_check.py + word-check/numbering-assumptions.docx` (from the repository root, in the project's environment): one row for each label + form the planner relies on, each with the planner's assumption beside it, taken from the planner's own code, so that + a change to the model shows in the file. Ctrl+A, F9, then compare each row's `REF \n` and `REF \n \t` with what the + row says the planner assumes (the text after "The planner assumes", and the `[planner: ...]` of parts B and C): + - `\t` with words before the number (rows 1 to 4: `Article %1`, `§ %1`, Microsoft's own `Section %1.%2` with a + decimal and with a roman `%1`): Word shows the bare number (`I`, `1`, `1.1`, `I.1`) where LibreOffice ignores `\t` + and shows the words. This is what the default `docx.reference_text_switch: documented` relies on. + - a period after letters before the number (rows 5 and 6: `Čl. %1`, `No. %1`): the planner does not use `\t` for + them under `documented`, and takes it to give the bare number under `guess`. If Word gives `1`, a style with such a + label may set `docx.reference_text_switch: guess`; if it gives anything else, report it. + - delimiters around the counter (rows 7 to 10: `(%1)`, `%1)`, `%1:`, `%1.%2(%3)`): the planner never uses `\t` for + them (what Word does is not documented). Note what Word returns. If it leaves the delimiter out (`a` for `(a)`), + `\t` would make a reference to such an item live and the planner's refusal is only cautious (report it); if it + keeps the delimiter (`(a)`), the refusal is right and the reference stays fixed text, reported. + - words between the counters (row 11, `Part %1 of %2`): only `docx.reference_text_switch: guess` expects `\t` to drop + them (the result would be `11` for `Part 1 of 1`). If Word returns something else, report it. + - the reference to such an item reads as the HTML reads it, or the render reported `render-docx-reference` and the + reference is fixed text (with the class it falls in and the setting that could change it). + - skipped heading levels (a `###` straight after a `#`, an Error for the validator but rendered; part C of the check + document): the numbers after it, and the references to them, read as the plain-text output reads them, but the + planner caches the skipped level as 1, as LibreOffice and the renderer count it, and writes that level's part of a + reference as text (`render-docx-reference` says so). Word is expected to draw 0 for the level a heading skips: record + what it draws. The rest of such a reference is fields and follows edits. + - a level that nothing numbered before the item that stands on it (`uncounted.default.docx`: items `e` and `f` stand + on the third level of their list, `(A)`, after a list of another kind and a bullet have come between): Word is + expected to read the unused level as 0 and number `e` as `(A)` and `f` as `(B)`, as the cached results say; LibreOffice + counts the unused level as 1 and shows `(B)` and `(C)` (the render reports `render-docx-libreoffice-count`, which + names `e` and what each reads). Record what Word shows. If it shows `(B)` and `(C)`, the plan's model is wrong for + every such item: Word numbers it one higher than the renderer does, and the cached results of references to it are + one too low until the fields are updated; the model in `resolve/docx_numbering.py` needs the other reading. + - counters shared across nums: a nested list of the same kind restarts under each new parent; a list of another + kind restarts by a fixed setting (the render reported `render-docx-static-restart`): add a parent item above a nested + list of another kind (an ordered list under an enumerated item, say) and the list under the new item starts at + 1 only if you restart it by hand, which is the documented limit. + - Roman numerals of 4000 and over, and letters past 26: Word's text (`render-docx-roman-overflow`, + `render-docx-letter-overflow`) is the one the render reported, not a different one. +9. **Numbering that shares a list or a counter** (`template.continental.docx`, the continental `features` file, and + the `paragraphs-numbered` files; the planner models these from Word's documented behaviour and LibreOffice, and they are + what only Word can confirm). After Ctrl+A, F9: + - Two `w:num` of one abstract list share the counter, and a `startOverride` restarts only its own level: where a section with clauses `1.1`, `1.2` is followed by a sub-heading + (add one to the file if the golden documents have none), the sub-heading reads `1.1` for the + sub-heading (restarted at a fixed value; the render reported `render-docx-shared-counter`), and the next heading + after it reads `1.2`, not `1.3`. Deleting that sub-heading makes the headings after it read higher (the documented + limit). + - Clauses in the continental style (`5.1`, `5.2`) are not in the table of contents and not in the navigation pane + (they have no heading style). Under each clause the nested list reads `a)`, `b)`, and a new clause (Enter at the end + of a clause, typing) restarts it at `a)` under that clause; so does a sub-sub-list (`i)`). A list nested three + levels under a clause (`1.1.1`) is a separate list that restarts by a fixed setting (reported `render-docx-static-restart`). + - A level of the heading list whose text names levels around ones nothing numbers (part B of the check document: levels + 3 to 5 unused, the nested level `%1.%2(%6)`): the nested items read `1.1(a)`, `1.1(b)`, and `1.2(a)` under the next + item, never `1.1.0.0.0(a)` and never continuing after a new item. This is the layout of a `{path}` label under a + clause. + - A list in an ordered item that shows the path of its item (the style `enumeration.levels` with `{path}` labels) is on + detached levels: Word draws the item's own counter alone (`(A)`, not `1(a)(1)(i)(A)`; the render reported + `render-docx-section-label` with what Word draws), and the references read as the HTML reads them. A second + such list under a new parent restarts at `(A)` by a fixed setting; a list made new in Word by copying an item + of another goes on counting (the documented limit). + - Items nested past nine levels continue in a further list: the tenth level restarts at `(a)` by a fixed setting and its + own items restart by structure; the references to them read as the HTML reads them. + - In `template.continental.docx` a later alternative (a paragraph, clause or heading that shares a number: the + second "Disputes", say) shows the first one's number: a `numId 0` paragraph that starts with a `REF \n \h` field, + with the same hanging indent as a numbered one and its text after a tab. Update the field: it still reads the + holder's number. Its entry in the table of contents reads the shared number, a tab and the title (`2.Title`). + The numbers after the alternatives continue as the renderer's. Press Enter at the start and at the end of a + later alternative: the new paragraph has no number (it copies `numId 0`; expected, apply a list style by hand). **Delete + the first alternative:** every later alternative, their children, and every reference to the shared identifier read + "Error! Reference source not found" (expected: the render reported `render-docx-alternative`, and the final document + is made by rendering with the condition values). + - In the `paragraphs-numbered` files the paragraphs read `1.1`, `1.2` on the heading list; a sub-heading after a + list of clauses in the same section is restarted (as above). A heading added in Word with a paragraph under it + continues the numbering of the section before it (`2.3`, not `3.1`): the documented limit, which asks the lawyer to + restart the numbering by hand. +10. **Page layout and look.** The running footer shows `n / N`; a header template with `{title}`, `{version}` and + `{date}` shows them; a landscape page is landscape when printed (print preview). The paper is the style's (A4 by + default), the margins are its margins, the text is in the style's font and size, numbers hang to the left of their + text and nested lists step in, a table keeps its header row at the top of each page when it runs over a page (add + rows to the table of `features` until it does), and a drafting note is shaded and kept on one page. +11. **A viewer that never updates fields** (optional: Word on the web, or a preview such as Quick Look or Outlook's). The + numbers, the references and the contents entries read as in item 2 and 3 before any update, and the contents have no + page numbers (the documented difference). + +Record what differs in the release notes. A difference from LibreOffice is expected where the documentation says Word +and LibreOffice differ (`\t`, `lvlRestart`, `SEQ \s`, `w:updateFields`). + ## Notes - **Version numbers are permanent.** PyPI does not allow re-uploading a version, even after @@ -92,7 +268,7 @@ markdown-it-py) come from PyPI. validator's next minor version removes. - **The upload is gated on the tests.** `publish.yml` first calls [`ci.yml`](workflows/ci.yml) (`on: workflow_call`) as the job `ci`, and `build` needs it, so the lint, test - (Linux and Windows on Python 3.13) and conformance jobs must pass on the commit being published + (Linux and Windows on Python 3.13), conformance and docx (schemas and LibreOffice) jobs must pass on the commit being published before anything is built, and `testpypi` and `pypi` need `build`. The `validator-main` job is informational (`continue-on-error`) and never blocks a release. The caller grants the called workflow `contents: read`; a called workflow cannot hold more permissions than its caller gives. diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 55c2ff4..b1a860d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -75,6 +75,54 @@ jobs: env: LEGALDOWN_SPEC_DIR: .legaldown-spec + docx: + # The DOCX writer is checked against the ISO/IEC 29500 schemas (fetched, not stored: tests/schemas/README.md) + # and against LibreOffice, which opens each document, updates its fields and says what it shows + # (tests/lo/uno_export.py runs under the system Python, which python3-uno provides, not the virtual + # environment's). A missing tool fails the job instead of skipping the tests. + name: DOCX (schemas and LibreOffice) + runs-on: ubuntu-latest + env: + LEGALDOWN_REQUIRE_DOCX_TOOLS: "1" + steps: + - uses: actions/checkout@v7 + - name: Check out the LegalDown specification (its documents are validated too) + uses: actions/checkout@v7 + with: + repository: ForLegalAI/LegalDown + ref: ${{ env.LEGALDOWN_SPEC_REF }} + path: .legaldown-spec + - uses: actions/setup-python@v7 + with: + python-version: "3.13" + cache: pip + - run: | + sudo apt-get update + sudo apt-get install -y libreoffice-writer python3-uno + - name: Show the versions of the system and of LibreOffice + run: | + lsb_release -ds || cat /etc/os-release + soffice --version + - run: pip install -e ".[dev]" + - name: Cache the OOXML schemas + id: schemas + uses: actions/cache@v6 + with: + path: tests/schemas/ooxml + key: ooxml-schemas-${{ hashFiles('tests/schemas/fetch.py') }} + - if: steps.schemas.outputs.cache-hit != 'true' + run: python tests/schemas/fetch.py + - run: pytest -q -m ooxml_schema + env: + LEGALDOWN_SPEC_DIR: .legaldown-spec + # The acceptance corpus (tests/test_docx_acceptance.py) is part of both runs; it needs the specification's documents + - run: pytest -q -m libreoffice + env: + LEGALDOWN_SPEC_DIR: .legaldown-spec + - run: pytest -q -m "acceptance and not libreoffice and not ooxml_schema" + env: + LEGALDOWN_SPEC_DIR: .legaldown-spec + validator-main: # Surfaces changes in legaldown-validator's model before they are released # (docs/decisions/0001). Informational: a failure here is a heads-up. diff --git a/.gitignore b/.gitignore index c2225b2..53e0c19 100644 --- a/.gitignore +++ b/.gitignore @@ -6,3 +6,9 @@ build/ dist/ .pytest_cache/ .ruff_cache/ + +# OOXML schemas fetched for the DOCX tests (tests/schemas/README.md) +tests/schemas/ooxml/ + +# The documents made for the manual Word checks (.github/PUBLISHING.md) +word-check/ diff --git a/CONFORMANCE.md b/CONFORMANCE.md index a053c0b..c894f42 100644 --- a/CONFORMANCE.md +++ b/CONFORMANCE.md @@ -7,7 +7,7 @@ lists the Core rules it covers. It is checked against the specification's own examples and fixtures corpus (`tests/conformance/`): -- every document renders to HTML and to text +- every document renders to HTML, to text and to DOCX - every internal link resolves to an anchor that exists - each invalid fixture renders its rule's failure marker @@ -20,6 +20,7 @@ It is checked against the specification's own examples and fixtures corpus (`tes | `{{include:}}` below Full (§17.5) | ✅ Renders `[NOT PROCESSED: include path]` with a Warning (`render-not-processed`); the path is checked as a file reference (§2.3) | | Party and side display rules (§3.6) | ✅ | | At least one output format (§13.6) | ✅ HTML (recommended) and plain text (optional) | +| DOCX output (§13.6, recommended) | ⚠️ Written by `-o x.docx` / `format="docx"` (0.5): numbers, cross-references, the table of contents and page numbers are Word features, not text. Checked against the ISO/IEC 29500 schemas and in LibreOffice in CI; **not yet checked in Word itself**, which is a manual step before the release. See "DOCX output", below. HTML is the format this renderer claims §13.6 with | | A heading in a list item or a quote | ✅ Shown as a bold line: it is not a section (§4.1), so it has no number or anchor | | Comment stripping (§8.6) | ✅ Inline comments, and comment blocks across any number of lines. As in CommonMark, a comment block left unclosed runs to the end of the document | | Templates with answers (§15.8) | ✅ Assembled first with `legaldown-validator`'s Assembly capability (§15.7), then rendered: `answers=` / `--answers answers.yaml`. A template that needs other files is refused (see below) | @@ -47,6 +48,73 @@ It is checked against the specification's own examples and fixtures corpus (`tes | Question prompts next to blanks (§15.8, MAY) | ✅ `placeholders.show_prompt` | | Final check (§15.9, SHOULD) | ✅ `--final` / `final=True`: the validator's `placeholder-unfilled` and `template-construct-present` Errors; with `--strict`, the document is refused | +## DOCX output (§13.6) + +[ADR 0008](docs/decisions/0008-docx-word-native.md) decides the design: in the `.docx` every number, cross-reference, +table of contents entry and page number is a Word numbering definition or field, never literal text, and the values the +renderer computed are only the cached results of the fields. The document therefore reads right before any field is +updated, and stays right when a lawyer inserts a clause and updates fields. + +**What is written**, in the order the HTML shows it: + +- The title block (header rows and parties, borderless tables), a live table of contents (`TOC \o "1-N" \h \z \u` with cached + entries equal to the HTML contents, each a link with a `PAGEREF` page number that Word fills in when it updates the + field), the sections, the attachments (a heading and a titled placeholder, separated by a page break or a rule per + `attachments.separator`), and the signature blocks (§3.6, borderless tables with signing lines, kept together on one page as + far as one page allows). +- Headings, list items and numbered paragraphs take their numbers from Word numbering definitions: the heading styles are + linked to a multilevel list; lists restart by structure where Word can; the continental `5.1` items and the numbered + paragraphs ride the heading list's next level, with the lists nested in them on its reserved levels; a list nested past + Word's nine levels goes on in further lists, up to 36 levels. +- Every `{{ref:}}` is the delimiters of its designation and `REF` fields on bookmarks, with the values the renderer computed as the cached results (`1.1.1`, + `I.A.1.a(i)`, `1(b)(ii)`). `{{term:}}` and `{{attach:}}` are internal links; under the `none` scheme a reference is a + field for the text of the heading. +- The template view (§15.8): condition labels, drafting notes and choices are shown as in the other formats. Alternatives that + share a number are paragraphs with no number of their own whose number is a `REF` field of the first. +- Images are their alt text (no file is read, §2.3, §17.6). A link is a hyperlink for the same schemes as in HTML. +- A style setting that Word cannot express (a `%` length, a colour name Word does not know) is a style error when the + format is `docx`, before anything is rendered. + +**Differences from the other formats**, each reported with a `render-docx-*` diagnostic (the renderer's own rule ids, not the +specification's) at the line of the unit or the reference, once per finding and once for references that fail alike: + +| Rule | Level | Meaning | +|---|---|---| +| `render-docx-reference` | Warning | A `{{ref:}}` that Word's fields cannot compose is written as its text: right when the file is opened, but it does not follow edits. The message names the class: a number with text or punctuation around its counter that the reference does not have, where Word's `\t` switch may not be relied on to leave it out (`docx.reference_text_switch`: `off`, `documented`, `guess`); or a number holding the counters of other units that the reference writes itself (`{section}.{n}` under a ref `{n}`). Also this rule: the counter of a heading level the document skips (the Core Error `heading-skip`) is text between the fields of the units that do hold theirs, and a reference that reads differently in Word for another reason | +| `render-docx-reference-cycle` | Warning | Under the `none` scheme, a heading's title refers back to the heading: the reference is written as text | +| `render-docx-section-label` | Warning | A label shows a number that Word's list cannot show there (the section's number on a list that is not the heading list, or the counters of the enclosing items on a list that cannot carry them): Word draws a shorter label, named in the message. References stay fields | +| `render-docx-letter-overflow` | Warning | Past 26 a letter counter reads `aa`, `bb`, `cc` in Word and `aa`, `ab`, `ac` in the renderer | +| `render-docx-roman-overflow` | Warning | Word writes Roman numerals of 4000 and over differently | +| `render-docx-list-depth` | Warning | A list is nested past 36 levels: the items past it are numbered in the format of the last level and restart by fixed values | +| `render-docx-shared-counter` | Warning | A unit shares a counter with others in Word (a sub-heading and the items of its section, a list that goes on from another) and is restarted at its number: right when opened, but a fixed value that does not follow edits made before it | +| `render-docx-number-mismatch` | Warning | Any other difference between Word's number and the renderer's: a catch-all that does not occur outside documents that skip a heading level | +| `render-docx-alternative` | Info | Alternatives that share a number show it as a field of the first: deleting the first in Word breaks the others and every reference to the shared identifier. Render with the condition values or answers for a final document. Once per document | +| `render-docx-static-restart` | Info | A list restarts by a fixed setting, not by structure: nested in a list of another kind, in an item on the heading list that it cannot ride, in an alternative, or the one that begins the next list past nine levels. A parent item added in Word does not restart it | +| `render-docx-libreoffice-count` | Info | Items are numbered as the renderer does in Word but one or more higher in LibreOffice, which counts a level that nothing in the list numbered before a deeper item as 1. Nothing in the file is wrong for Word. Once per document | +| `render-docx-reference-word-only` | Info | A reference relies on Word's `\t` switch, which LibreOffice ignores: there it shows the whole number. Once per document | + +Two more differences are not reported, since they follow from the format: at `contents.depth: 1` the attachment titles are +first-level entries of the contents, where the HTML indents them one level more; and the contents show a later +alternative's shared number. + +**How it is checked.** In CI, on every part of every document of the corpus (the specification's examples and valid +fixtures, the test documents and documents written to stress the numbering) under the `default`, `outline` and +`continental` styles: + +- every part validates against the ISO/IEC 29500-4 transitional schemas, and rendering twice gives the same bytes; +- LibreOffice opens the document, updates its fields and indexes, and what it shows is what the plain-text writer prints + (numbers, references, contents entries, attachments, signatures), except where a diagnostic above says Word draws a + number differently (`tests/test_docx_acceptance.py`); +- after a heading, a list item, a nested item and a numbered paragraph are inserted and a clause is deleted in + LibreOffice, as a lawyer does, and the fields are updated, the document reads as the plain-text writer prints the source + with the same edits (the same module; the finer cases, with the cached results made wrong on purpose, are in + `tests/test_docx_roundtrip.py`). + +LibreOffice is the oracle available to CI, and it differs from Word in a few places: it ignores `\t`, `lvlRestart`, +`SEQ \s` and `w:updateFields`, and counts a level that nothing numbered before a deeper item as 1. What only Word shows +(the contents' page numbers, what `\t` does, the count of an unused level, numbering after an insertion, and every +other Word-only assumption of the plan) is a manual checklist before each release (`.github/PUBLISHING.md`). + ## Known limitations - **Assembly reads no other files.** A template with include fragments, LegalDown attachment @@ -55,7 +123,7 @@ It is checked against the specification's own examples and fixtures corpus (`tes - **Line numbers (§16.9).** Every diagnostic names its line, the validator's and the renderer's own (`render-ref-ambiguous`, `ref-not-enumerated`, `render-not-processed`, `render-link-refused`, `path-not-relative`, `path-outside-root`, `render-term-limit`, `render-term-too-long`, - `render-designation-cycle`, `render-designation-limit`, `render-locale-fallback`), and the CLI prints it as `file:line:`. The renderer's line is the + `render-designation-cycle`, `render-designation-limit`, `render-locale-fallback`, and the `render-docx-*` rules of the DOCX planner), and the CLI prints it as `file:line:`. The renderer's line is the **first line of the block it is in**, as `Document.line_of` gives it, whatever line the directive itself is on: a paragraph's, a table's header row, a quote's first `>`, a code block's opening fence, a heading's (a section title); for a list item, the line of **its marker** diff --git a/README.md b/README.md index a1f138c..87f2cf6 100644 --- a/README.md +++ b/README.md @@ -4,10 +4,10 @@ ### The reference renderer for [LegalDown](https://github.com/ForLegalAI/LegalDown) -**Turn a LegalDown document into HTML or plain text, with section numbers, cross-references, +**Turn a LegalDown document into HTML, plain text or a Word document, with section numbers, cross-references, defined terms, and formatted values all generated at render time.** -Targets specification **v0.2** · Conformance level **Rendering** · DOCX and PDF planned +Targets specification **v0.2** · Conformance level **Rendering** · HTML, text and Word (DOCX) output, PDF planned @@ -57,6 +57,7 @@ Python 3.11 or newer. The dependencies are all pure Python: `legaldown-validator ```bash legaldown-render contract.lgd -o contract.html # HTML page legaldown-render contract.lgd -o contract.txt # plain text (format from extension) +legaldown-render contract.lgd -o contract.docx # a Word document with live numbers and references (see "DOCX") legaldown-render contract.lgd --style continental --locale cs-CZ -o smlouva.html # Czech dates, money, durations legaldown-render contract.lgd --set numbering.scheme=legal-outline --set enumeration.enabled=false legaldown-render contract.lgd --set contents.enabled=true --set contents.depth=3 # table of contents @@ -92,11 +93,11 @@ Documents are read as UTF-8, and output is UTF-8 with LF line endings, on standa the standard streams only when it runs as the program; calling `legaldown_render.cli.main([...])` from Python leaves `sys.stdout` and `sys.stderr` as they are. Without `--format`, `-o` takes the format from the file extension: `.html` and `.htm` write HTML, -`.txt` and `.text` plain text. The extension of a document format this renderer does not write -(`.docx`, `.doc`, `.odt`, `.rtf`, `.pdf`, `.md`, `.markdown`, `.epub`, `.tex`, in any case) is a usage +`.txt` and `.text` plain text, `.docx` a Word document. `-o -` writes to standard output. The extension of a document format this renderer does not write +(`.doc`, `.odt`, `.rtf`, `.pdf`, `.md`, `.markdown`, `.epub`, `.tex`, in any case) is a usage error (exit 2) unless `--format` says what to write. The extension is the text after the last dot, also in a name that starts with one (`.pdf`), and trailing dots and spaces are ignored (`x.pdf.`, -`"x.docx "`). Any other name, with any other extension or none (`contract`, `contract.v2`, +`"x.odt "`). Any other name, with any other extension or none (`contract`, `contract.v2`, `report.2025-10-27`, `.out`), is just a name and writes HTML. ## Python @@ -112,6 +113,8 @@ with open("contract.html", "w", encoding="utf-8", newline="\n") as output: output.write(result.output) ``` +A Word document is bytes, not text: `render(text, format="docx").binary` holds the file (and `.output` is empty). + `render()` also accepts a `RenderOptions`. Pass `answers={...}` to render a template filled in. Each diagnostic has its `line` and its `file`: `render_file()` and the command name the file (its name, without the directory); for text given to `render()`, pass `filename=` (a directory in it is dropped, `/` or `\`). @@ -127,6 +130,69 @@ cannot be assembled with its answers (`RenderRefused`) do. All of them are `Rend boundaries (`multiprocessing`, `concurrent.futures`) intact. `overrides=` takes dotted settings and may not contain `version` or `extends`, so `overrides=legaldown_render.style.style_to_dict(style)` must drop those two keys. +## DOCX + +`-o contract.docx` (or `--format docx`, or `render(text, format="docx")`) writes a Word document that a lawyer can +edit. It is written with the standard library alone, so it needs nothing beyond the base install. + +**The point of it:** in the Word file the numbers, the cross-references, the table of contents and the page +numbers are Word's own features, not typed text. (This is how the file is built and how it behaves in LibreOffice, which CI +runs; it has not yet been checked in Word itself, which the maintainers do by hand before a release.) Insert a clause in Word, update the fields, and the clauses after it, +every "see 2.1(b)" and the contents follow. The renderer still works every value out; it writes the value only as the +cached result of a field, so the document reads right before any field is updated +([ADR 0008](https://github.com/ForLegalAI/legaldown-render/blob/main/docs/decisions/0008-docx-word-native.md)). + +**What is written.** The same document the HTML shows, in the same order: the title block (header rows and parties), +the table of contents, the sections, the attachments (a heading and a titled placeholder each) and the signature blocks. +Paragraphs, lists, quotes, drafting notes and the template view's condition labels, code, tables with a repeated header +row, rules, links, defined terms, blanks and choices have Word styles generated from the style template, so a change to +a style in Word applies everywhere. The page size, margins, fonts and colours are the style's, and the file carries the +document's title and language. + +**What is live** (by design; confirmed in LibreOffice, to be confirmed in Word): + +- Heading numbers (the heading styles are linked to a multilevel list), list items and numbered paragraphs are Word + numbering definitions. A nested list restarts under a new parent item, and a clause inserted before another renumbers it. +- A `{{ref: clause}}` is `REF` fields on bookmarks, joined by the delimiters of the designation, so "1(b)(ii)" or + "I.A.1" is recomputed. `{{term:}}` and `{{attach:}}` are links (an attachment's title is a field). +- The table of contents is a `TOC` field (`contents.enabled`, `contents.depth`), each entry a link with a `PAGEREF` + page number. The running header and footer are `page.header` and `page.footer`: the page number and the page count + are fields (the default footer reads "3 / 12"). + +**Opening it.** Word asks once whether to update the fields (`docx.update_fields_on_open`, on by default), and then the +cross-references and the contents with their page numbers are up to date. After an edit, press Ctrl+A, then F9, or +right-click the contents and choose *Update field*. LibreOffice ignores the request (Tools, Update, All Indexes and +Tables). A viewer that never updates fields (Word on the web, Quick Look, a Google Docs import, Outlook's preview) shows +the numbers and references right, as cached, and the contents without page numbers. + +**Settings** (the style template, [Styles and preferences](https://github.com/ForLegalAI/legaldown-render/blob/main/docs/style-templates.md#docx)): + +| Setting | Effect in the DOCX | +|---|---| +| `page.size`, `page.margin` | paper and margins; a length Word cannot express (`%`, `ch`, `ex`) is a style error, reported before anything is rendered | +| `page.header`, `page.footer` | left, centre and right text with `{page}`, `{pages}`, `{title}`, `{version}`, `{date}` | +| `docx.update_fields_on_open` | `true` (the default): Word updates every field when it opens the file; `false` writes no such request | +| `docx.reference_text_switch` | `off`, `documented` (the default) or `guess`: how far a cross-reference may rely on Word's `\t` switch to leave the words of a number out (`Article I` shown as `I`) | +| `contents.*`, `attachments.separator`, `signatures.enabled`, `numbering.*`, `enumeration.*`, `paragraphs.*`, `typography.*`, `headings.*` | as in the other formats; the DOCX meaning of each is in the style documentation | + +**Limits.** Word cannot do everything the renderer does, and the render says where, one `render-docx-*` diagnostic +with the line (the rules are listed in [the conformance notes](https://github.com/ForLegalAI/legaldown-render/blob/main/CONFORMANCE.md#docx-output-136)): + +- A cross-reference that Word's fields cannot compose (a number with punctuation around its counter that the reference + leaves out, say, or the number of a heading level the document skips) is written as its text: right when the file is + opened, but it does not follow edits. +- Where Word draws a number differently (letters past 26, Roman numerals from 4000, a label that shows the section's + number on a list that cannot carry it), or restarts a list by a fixed setting instead of by structure, the file is + right when it is opened and does not follow every edit made in Word. +- Alternatives that share a number (§15.8, in the template view) show it as a field of the first: deleting the first in + Word breaks the others. Render with the answers or condition values to get a final document. +- An image is shown as its alt text: no file is read (§2.3, §17.6). +- CI checks every part against the ISO/IEC 29500 schemas and opens the documents in LibreOffice, which updates the + fields and is compared with the plain-text output. It cannot run Word, so the maintainers check the golden documents + in Word before a release ([the checklist](https://github.com/ForLegalAI/legaldown-render/blob/main/.github/PUBLISHING.md)). +- `--fragment` (and `standalone=False` in the API, where it is ignored) does not apply: a Word document is always whole, and `--fragment` with it is a usage error. On the command line the file goes to `-o FILE.docx`, or + to standard output with `-o -` (or without `-o` when standard output is not a terminal); it is never written to a terminal. + ## Rendering documents you did not write Document source never becomes HTML or CSS: raw HTML is not emitted, every text and attribute is @@ -196,7 +262,10 @@ articles, 2.1-style items, numbered paragraphs), and `outline` (I. / A. / 1. / a ```bash pip install -e ".[dev]" -pytest # unit, golden, and CLI tests +pytest # unit, golden, and CLI tests (pytest -n auto: on all cores, about a third of the time) +python tests/schemas/fetch.py && pytest -m ooxml_schema # DOCX parts against the OOXML schemas (needs lxml) +pytest -m libreoffice # DOCX opened in LibreOffice (needs soffice and python3-uno; skipped without) +LEGALDOWN_SPEC_DIR=../LegalDown pytest -m "libreoffice or ooxml_schema" # also the DOCX acceptance corpus (tests/test_docx_acceptance.py) pytest --update-golden # regenerate tests/golden/ after an intended output change LEGALDOWN_SPEC_DIR=../LegalDown pytest -m conformance # the specification's examples and fixtures ruff check . diff --git a/docs/architecture.md b/docs/architecture.md index a3ff4a6..9d65d1c 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -29,10 +29,10 @@ This page describes how the renderer is built. The decisions behind it are in └───────────┬─────────────┘ failure markers + Rendering diagnostics ▼ ┌─────────────────────────┐ - │ 5. Write writers/ │ html │ text (docx, pdf later) + │ 5. Write writers/ │ html │ text │ docx (pdf later) └───────────┬─────────────┘ ▼ - RenderResult(output, diagnostics, format, tree, style) + RenderResult(output, diagnostics, format, tree, style, binary) ``` Stage 2, **assembly** with an answers set (§15.7), runs only when the job has answers, before @@ -132,7 +132,8 @@ Guessing at lost structure was tried, and it traded each gap for new bugs. a skipped heading level counts as 1. The style only formats them, the n-th part of a number with the n-th level format. For paragraphs and list items, **alternatives** follow the validator's rule, applied the same way: an alternative directly follows its - sibling, has the same identifier and a valid condition, and its full presence (its own condition + sibling among the units numbered together (a list or table between two top-level paragraphs does + not separate them), has the same identifier and a valid condition, and its full presence (its own condition and every enclosing one) excludes that of every unit already holding the number (§15.4, the validator's `exclusive()`). It then labels list items by depth and top-level paragraphs, and allocates output anchors. Every anchor @@ -142,7 +143,7 @@ Guessing at lost structure was tried, and it traded each gap for new bugs. | Directive | Resolves to | Failure markers | |---|---|---| -| `{{ref:}}` | `CrossRef(designation, anchor)` (§13.3). A ref to an unnumbered item or paragraph falls back to its section's designation, with `ref-not-enumerated` | `[BROKEN REF: id]` | +| `{{ref:}}` | `CrossRef(designation, anchor, target_unit, section_unit)` (§13.3). A ref to an unnumbered item or paragraph falls back to its section's designation, with `ref-not-enumerated` | `[BROKEN REF: id]` | | `{{term:}}` | `TermRef(label or the definition's rendered term text, def anchor)` (§7.3, §13.4); the text is the first definition's term resolved as at the definition, worked out on first use by `Resolver._term_text` (`_survey` only records which definition is first) | `[UNDEFINED: id]` | | `"Term" {{def:}}` | `DefinedTerm` without quotation marks, anchored at `def:` (§7.2) | n/a | | `{{date:}}` `{{money:}}` `{{duration:}}` | `Value`, formatted for the locale by Babel (ADR 0005): amounts and durations keep every written digit (a fraction beyond Python's 4300-digit limit takes its plural form from its first 30 digits), and a duration without a pattern in the locale falls back to another length, the language alone, then English with `render-locale-fallback` | `[INVALID DATE: …]`, `[INVALID AMOUNT: …]`, `[UNKNOWN CURRENCY: …]`, `[INVALID DURATION: …]`, `[INVALID DURATION UNIT: …]` | @@ -172,6 +173,33 @@ link of any `{{term:}}` whose definition got no anchor, so term links always hav Headings are numbered with their level clamped to 1–5, as the validator numbers them. +**The numbering structure in the tree.** The resolver is the only place the rules of §13 live, and +a writer that builds numbering of its own (DOCX does) must not work them out again. So the tree +carries the structure behind every number, in terms that name no output format: + +| Where | Field | Holds | +|---|---|---| +| `RenderTree.version`, `RenderTree.effective_date` | `str` | The frontmatter's version and effective date as plain text, exactly as the title block shows them (directives resolved, a valid date formatted by the style for the locale), **whether or not** `title_block` shows them: a running header or footer uses them (`{version}`, `{date}`). Empty when absent | +| `RenderTree.formats` | `NumberingFormats` | The formats the labels were made with, after the style is resolved: `scheme`; `headings`, one `LevelFormat` (counter, `label` and `ref` templates) per heading level (empty under `none`); `enumerated`, the formats by list depth (cycling); `renumber`, the one format of ordered lists (None under `enumeration.ordered: enumerate`); `paragraphs` (None unless numbered); `reference`, `references.format` | +| `Section`, `ListItem`, `Paragraph` | `unit` | An id, unique in the document and equal on every render: `s` for the k-th section, `u` for the k-th item or top-level paragraph of a section in document order, the preamble first. Not the anchor. Every section, every item (in a quote too) and every top-level paragraph of a section has one, numbered or not, since an unnumbered one can still be the target of a reference; a paragraph inside an item, or in the preamble, is not a unit (None) | +| same | `number` | The unit's own counter value (the last part of "2.1" is 1; the item's position in its list; the paragraph's in its section). None when the unit is not numbered: any section under `none`, an item of a list that is not enumerated, a paragraph when the style does not number them | +| same | `parent_unit` | The unit its designation is built on. For a section, the nearest section above it with fewer number parts (a skipped heading level counts as 1 and is no unit: "1.2.1.1" under "1.2" has a gap of one level, which `level` shows); for an item of a list not nested in an item, and for a paragraph, its section (None in the preamble); for a nested item, the item. A list in a quote or drafting note, even inside an item, is built on the section, as its designation is; to see where a list nests, follow `parent_unit` (an item or a section), never `depth` | +| same | `shares_number_with` | For a later alternative that shares its number (§15.8, §15.4), the unit of the **first**; None otherwise, and always None on an unnumbered unit. Every alternative has a unit of its own. Sections share by number, as the validator numbers them: the subsections of a later alternative section repeat the numbers of the earlier one's ("1.1" again), and each repeat shares the number of the first section that had it | +| `List` | `family` | `renumber` (ordered, counted 1., 2., … whatever the source says), `enumerated` (counted in the format of the nesting depth) or `bullet` (no numbers; `enumerated` is False) | +| `List` | `depth`, `format`, `start` | The nesting depth the format was chosen at (a list in a quote in an item is one deeper), which selects the format and nothing else; the `LevelFormat` itself, so the entry of `formats` is never selected again (None for `bullet`); and `start`, the number of the first item (`items[0].number`). A list is not always counted from 1: lists nested directly in one item share a counter per way of writing a designation (counter style and `ref`), so a second list after a paragraph, or a renumbered list beside an enumerated one with the same designations, continues the first's numbers (a, b, then c) and no designation repeats; top-level lists and lists in quotes start at 1. A list has no id of its own: name it by `items[0].unit` | +| `CrossRef` | `target_unit`, `section_unit` | The unit the identifier is on, and the section that qualifies the designation (for a section target, itself). With alternatives sharing the identifier, `target_unit` is the first, as the anchor is. For an unnumbered item or paragraph the unit is still the target and `text` is the section's designation. Under `none` both are set as usual and the section's `number` is None. A reference that does not resolve is a `FailureMarker`, and so is a `{{ref:}}` to an attachment id (§5.6); an attachment is reached with `{{attach:}}`, a `Value` whose `href` is `#` and the `AttachmentPart.anchor`. Both are None for a copy that no longer links (`unlinked`, as in the table of contents) | + +A designation is the parent's designation followed by the unit's `ref` template filled in +(`{n}` the unit's number in its counter style, `{section}` the designation of the section, which a +preamble item has none of; under `none` the heading text stands for the section's designation and a +space goes before the piece), and a label is the unit's `label` template filled in with the same and +`{path}`, the designation up to the unit. `tests/test_numbering_structure.py` writes every label, +designation and reference text again from these fields alone, for each built-in style and the +specification's corpus, so the structure is known to be enough. + +Unit ids come out the same in the second numbering pass of the `none` scheme (below): they follow +document order, and the counter starts again with each pass. + **Designations under the `none` scheme (§13.3).** A section's designation is its heading as rendered, as plain text with white space collapsed: the `{{ref:}}`s and `{{term:}}`s in it are resolved (a reference shows `references.format` applied to its target's designation, so @@ -214,6 +242,207 @@ piece after it is kept), and `render-designation-limit` says so for that heading cycle or past a cap, or one whose terms reach into one, may read differently from the designation references show; the Warning says so. Every other heading reads exactly as its designation. +### The DOCX planner (`resolve/docx_plan.py`) + +The DOCX writer contains no numbering rules either (ADR 0003, [ADR 0008](decisions/0008-docx-word-native.md)). +Between the resolved tree and that writer sits a pure function, `plan_docx(tree, *, file=None, text_switch="documented", text_width=...) -> (DocxPlan, +diagnostics)`, which maps the numbering structure above to what Word needs: the abstractNums and nums, for every +section, list item and numbered paragraph the `(numId, ilvl)` its paragraph takes, the bookmark of every anchor, the +fields of every cross-reference, and the entries of the table of contents. It reads only `tree.formats` and the +units' `number`, `parent_unit` and `List.family`/`depth`/`format` (`numbering.formats_of(style)` builds `formats` for the +resolver and for the style check), and checks every placement against the number the resolver gave the unit by +simulating Word's counters (`resolve/docx_numbering.py`), so that what Word draws is what the resolver numbered, or a +diagnostic says why not. It is a planning step, not a stage of the pipeline: `render(format="docx")` calls it after +resolution, passes the plan to the writer, and adds its diagnostics (named with the document's file, each finding once, +ordered by line with the renderer's own) to the result's. The style check, `docx_style_errors(style)`, runs earlier still, in +`style/docx.py:docx_settings`, before rendering (ADR 0006). + +**Word's list semantics**, stated once (the planner and its diagnostics rely on them): + +- Counters belong to the abstractNum and are shared by every `w:num` of it. A num with a `startOverride` restarts + the named level at the first use of that num, and the levels above keep their values: this is Word's own + "Restart at 1". A level that nothing numbered before a deeper one is read by Word as 0 (the renderer counts a + skipped heading level as 1, and so does LibreOffice). +- A paragraph restarts every deeper level. A paragraph with `numId 0` counts nothing and restarts nothing. +- A list nested in an item of the same kind goes on a deeper level of its parent's num, so it restarts by + structure when a parent item is added. A list of another kind, a list in a quote, and a list Word would continue + where the resolver restarts have a num of their own: it restarts at 1 by a fixed setting (an item added above it + in Word does not restart it). A list that the resolver continues (its first item is 3) but Word cannot continue + starts at that number by a fixed setting, which does not follow items added or removed before it. +- A bullet counts nothing: one num serves every bullet list. +- Paragraph numbering has one num per section, and a section added in Word does not restart it by itself. +- Word has no way to give two paragraphs the same number on one level, except `numId 0` and a field: a paragraph with + a counter always counts. A unit that must read the number of the one before it (an alternative, §15.8) is a + `numId 0` paragraph that starts with a `REF \n` field of that unit. +- `REF \n` returns the paragraph's level text with its counters, minus trailing periods. + +**What is planned.** Heading numbering (one multilevel abstractNum linked to the heading styles, the label templates +turned into `lvlText`, the counters into `numFmt`), list families (enumerated, renumbered, bullets) with their nesting +and restarts, depth beyond Word's nine levels (further abstractNums of the family, up to 36 levels), and numbered paragraphs. +A level of a list shows only counters of its own list, which settles where the rest of the planner puts things: + +- **Labels that need the section** (`{section}`, or the `{path}` of an item or paragraph in a section: the continental + style's `5.1`, the default style's numbered paragraphs `1.1`). Only the heading list can show the counters of a + section, so a unit whose label needs them goes on the heading abstractNum at the level after its section's + (`%1.%2` after a level-1 section), where the level reads exactly as the label does (`candidate` equals the heading + level; the first top-level list of the section gets one num with a `startOverride`, as every top-level list does). + Such an item is no heading: it has no heading style, no outline level and no entry in the table of contents, and + carries an indent of its own (`UnitPlan.ind`; a heading level's `w:ind` serves the heading). A list nested directly + in an item that is on the heading list goes on the heading abstractNum's **reserved levels**, level `4 + d` for + nesting depth `d` of 1 to 4, defined on first use with the label as the resolver writes it, every counter in it + being a level of the heading list (`candidate`): a label that is only its own counter reads `%6)`, `%7)`; a `{path}` + label reads `%1.%2(%6)` (the section's counter at heading level 0, the clause at level 1 where it is, the list's own + at level 5), and the items between at their own reserved levels (`%1.%2(%6)(%7)`). The list is on the same num as its + parent with no override: it restarts by structure under every new item and sibling lists go on counting, as in any + other list, and the label and every reference to it are live. A level may be shared by units whose definitions are + equal and is kept for the first that needs it otherwise (the level after the fifth heading level is kept for a level-5 + section's units when the document has one); a label that shows counters which are not the end of its own designation + (`{section}.{n}` at depth 3 of the continental style, under a ref `.{n}`: the designation ends in `.1`, the number + begins with the section's counter) does not take a reserved level, since no field could stand for it. What does not + fit (an outline or mixed scheme, where the level after the section's reads otherwise; the `none` scheme, where the + section's number is its text; a label whose reserved level another text has taken, depth past 4, items in a quote, + ordered and bullet lists nested in such an item) keeps the family's own abstractNum, or the **detached** one when the + enumerated level would have to show a counter of a unit that is not on it (below). A label that shows a section's + number and has no place for it is written without it and reported (`render-docx-section-label`); the references + still cover it. +- **Detached lists.** Where the enumerated level a nested list would take names a level that no unit of its chain holds + (an enumerated item below an ordered item: its `{path}` label names the ordered item's counter, which is on another + abstractNum; a quote's list in a preamble item; a path label whose reserved level is taken or past depth 4), Word + would draw `0`, or the counter some other list counted last. The list goes on the **detached** abstractNum instead: + the enumerated levels with each level's own counter alone (`{path}` shows its own `ref`), used for exactly that + case, and for every list nested in one of its items (so those restart by structure). The label Word draws is shorter + than the renderer's (`(A)` for `1(a)(1)(i)(A)`), `render-docx-section-label` names the units whose counters it + leaves out and what Word draws, and a reference is still made of fields, each unit's own counter. A detached list + restarts under a new parent at a fixed value, like any list nested in a list of another kind (`render-docx-static-restart`): + a list made new in Word by copying an item of another goes on counting where the renderer would start again. +- **Lists nested past Word's nine levels.** Word has nine levels in a list. A list nested deeper goes on in the next + abstractNum of its family (`enumerated+1` to `enumerated+3`, `renumber+1` to `renumber+3`, `bullet+1` to `bullet+3`: 36 levels in all), whose + level `j` is the family's level of depth `9k + j` (format and label with its own counter alone, indent stepping a + centimetre per level up to 60% of the page's text width, as for every nested list, `plan_docx(text_width=)`). Inside such an abstractNum the lists restart by structure; the list that + begins one restarts at a fixed value (`restart-depth`, reported as information). Only a nesting past 36 levels + uses the last level again (`clamped`, `render-docx-list-depth`). +- **Alternatives that share a number** (§15.8). A unit that shares its number with the one before it and that Word + would number differently (the live number would continue) is a `numId 0` paragraph with its own indent that starts + with a `REF \n \h` field of the unit that holds the number now, then the label's text beyond what the field returns + (the period) and a tab or space (`UnitPlan.alt`, `ind`); that unit gets a bookmark, a `{{ref:}}` to the shared + identifier is a reference to the first alternative, and the table of contents shows the shared number for the later + one. The units under a later alternative restart in a num of their own with a `startOverride` (the sections after a + later alternative section until one of its level or above; a list nested in a later alternative item), because the + alternative counts nothing and Word would go on counting from the holder's children. A later alternative whose live + number is already right (the children of a later alternative section repeat the numbers of the first's) is numbered + live. Designations below an alternative are made of the holder's counters. This is live for edits elsewhere but not + for the first alternative: deleting it in Word breaks every later alternative and every reference to the shared + identifier, and Enter at the start or end of a later alternative gives an unnumbered paragraph, so the render reports + one information per document (`render-docx-alternative`) and a final document is made by rendering with the + condition values. +- **Resynchronising.** A numbered unit that Word would number differently from the resolver for want of a place where it + can count by structure (it goes on a counter that other units of the same abstractNum have used) is put on a num with + a `startOverride` at its number, and the units after it go on counting on the original. This is a fixed value: a + restarted sub-heading keeps it when clauses or headings are inserted before it, and the headings after it read + higher if it is deleted; a restarted clause or paragraph does not follow what is inserted or deleted before it. It repairs a sub-heading after the + items and paragraphs of its section on the heading list (`1.1` for the sub-heading and for the first item share a + counter in Word and a designation in every output format: `render-docx-shared-counter` says so), a list that + resumes a sibling's count after another list in the same item. + +A list nested in an item that has an indent of its own (`UnitPlan.ind`) is indented one step in from it, on whatever +list it is. A label that shows a counter of a unit outside the unit's own chain (a list nested in another kind of +list; a quote's list) is written with the counters Word has there (the detached levels, above) and reported where it +differs from the renderer's. Nothing is left to be shown by Word differently without a diagnostic. + +**Cross-references: the designation cover** (`resolve/docx_cover.py`, §6.3, §13.3). The renderer's text of a +`{{ref:}}` is the designation of the target (`1.1.1`, `I.A.1.a(i)`, `1(b)(ii)`) inside `references.format`; the planner +writes the designation of every unit as a *token string* (a counter with the unit that holds it, literal text, or a +heading's text under the `none` scheme) in the same pass that numbers it, ported one to one from the resolver's `fill` +and `extend`. In Word every part of it that Word can compute must be a field, and nothing Word can compute may be +literal text, so the plan covers the designation by `REF \n \h` fields and the delimiters between them. +A field returns the full level text of the bookmarked paragraph with its counters, minus trailing periods (what the +simulator gives as `Step.word_ref`), and fits where its counters belong to the same units as the designation's: a +`{path}` label (`1.1.1`) is covered by one field on the target, a `{n}` label (`I.A.1`) by one field per level with +literal delimiters between (`I` `.` `A` `.` `1`), a nested item (`(b)(ii)`) by one field per level, an ordered +list in a section by `REF section` + `(` + `REF item` + `)`. Literal text may stand only for delimiters, never for a +counter. What is covered is the **whole text of the reference**, the designation inside the text of `references.format` +(with two `{designation}` the designation is there twice): a field may stand for the words of the format when the number +it returns holds the same words, so `Article {designation}` over a label `Article {n}` is one plain `REF` (`Article I`), +`§ {designation}` over `§ {n}` another, and no `\t` is needed. The cover with the fewest fields wins, then the fewest +using `\t`, then the fewest literal characters. `\t` (Word's switch that suppresses the words around a counter; +LibreOffice ignores it) is used where the number holds words the reference does not (`Article I` for a format `Art. ` +gives `I`, `§ 3` gives `3`; Microsoft's own example is `Section 1.01`, which gives `1.01`), as far as the style's +`docx.reference_text_switch` says Word may be expected to do it: `documented` (the default) drops the words before the +first counter and after the last, keeps a delimiter between counters, and does not use the switch for a number with a +delimiter around its counters (`(a)`, `a)`, `1:`, `No. 1`) or words between counters, because what Word does with them is +not documented; `off` never uses it; `guess` also drops a period after the letters that lead a number (`No. 1`, +`Čl. 1`) and words between counters. What Word does is +on the manual Word checklist (`.github/PUBLISHING.md`). Each field's cached result is **Word's** value (the simulator's), +and `cached_lo` what LibreOffice shows. A unit that a field points at has a +bookmark (its anchor's, or a hidden one); a reference to an item or paragraph the style does not number is the +designation of its section (`ref-not-enumerated`), covered as one. Under the `none` scheme a heading is a field for the +bookmarked text (`REF bookmark \h`) and the item's pieces follow after a space; what the field returns is what the +heading's runs show (`_shown`), which can differ from the renderer's designation by white space, and then the +reference is reported. A `RefPlan` is made once per target, keyed by `CrossRef.target_unit`; `exact` says that the +cached results read as the renderer's text. **What stays text.** A reference is fixed text (`RefPlan.fixed`: it reads right when the file is opened and does not +follow edits, and `render-docx-reference` says so, one finding for the references that fail alike, naming the first and +how many more) only where Word cannot express it, and the finding says which of the classes it is, so that the style +can be changed if the lawyer wants the reference live: **delimiters** (the number Word draws has text or punctuation +around its counter that the reference does not have, such as a label `({n})` under a ref `{n}`, `{n}:` or `No. {n}`, +and `\t` is off or not documented to drop it; `docx.reference_text_switch: guess` takes a period after letters), and +**counters** (the number holds the counters of other units that the reference writes itself before it, such as a label +`{section}.{n}` under a ref `{n}`: a field returns the whole number or none of it; one finding for them all, since the +cure is the style's label). Besides, **the counter of a heading level the document skips** (`heading-skip`, §4.1: an +Error the renderer still numbers) is held by no unit and no field returns it, so that part of the reference, and only +that part, is literal text (`RefPlan.ghost`, `render-docx-reference` names it); the rest is fields. The cover treats a +`Ghost` counter, never a stale counter of another level, as literal, and the Word value assumed for the cached results +there is LibreOffice's reading of a skipped level (its start value); Word itself is expected to read it as one less +(the checklist). A heading's title that refers back to itself under the `none` scheme is the third case +(`render-docx-reference-cycle`). Nothing else that Word could compute is written as literal text; the built-in styles +have no fixed reference in a document without a skipped heading level, with lists nested up to 36 levels. + +**Bookmarks** (`resolve/docx_bookmarks.py`). The planner names bookmarks and the writer gives them ids. Every anchor of +the tree has one, referenced or not: `ld_` for a section, list item or paragraph, `lddef_` for a +defined term, `ldatt_` for an attachment, `_LdAttachments` for the attachments heading, with `-` and `:` written +as `_`. A name is used only if it is valid, at most 40 characters, nothing but dashes was changed, and it is unique +ignoring case (Word compares names so); otherwise, and for a unit a field needs that has no anchor, it is hidden: +`_Ld`, counted in document order. The table of contents uses `_Toc` from 100000001. The writer wraps +the paragraph's title or first paragraph in `UnitPlan.bookmarks`, then `toc_bookmark`; a term reference and an +internal link point at `plan.anchors`. **A bookmark holds text only when a field returns that text** (`REF \h`, a +heading under the `none` scheme): every other bookmark is a point at the start of the paragraph +(`UnitPlan.collapsed`), because `REF \n` needs only the paragraph, and a field inside the bookmark it points at is +a self-reference that Word cannot show and LibreOffice expands without end. Under the `none` scheme a heading whose +title holds a reference that leads back to the heading, directly or through the titles of other headings, would have +exactly that; the plan lists those references (`UnitPlan.literal_refs`, by target) and the writer writes them as +their text, and `render-docx-reference-cycle` says so, naming the heading's line (the renderer already warns with +`render-designation-cycle`). The headings are the nodes of a graph, an edge for each heading whose text a field in +another's title returns, and a reference in a title is a cycle exactly when such a field's heading is in the title's +own strongly connected component (found once, iteratively: linear in the headings and references). The text a +heading field returns is worked out from the runs, in the order the components complete: a reference in the title is +the text of the fields it is written as (the cached results of other headings' fields), or its own text where it is +written as text. + +**The table of contents.** `ContentsEntry.unit` (the section's unit; None for the attachments heading and an +attachment) keys each entry of `tree.contents` to its section. `plan.toc` lists the entries in order with +the number Word draws for the heading (`UnitPlan.label`, so a style's `Article I` is the number Word shows, not +a rendering of the renderer's), the separator the level has (`suff`: tab or space; none for an unnumbered entry), the +title, the condition label, the `_Toc` bookmark (also `UnitPlan.toc_bookmark`) and an empty cached page number. The +complex `TOC` field that uses them is written with the title block. + +**Diagnostics** (all `render-docx-*`, with the line of the unit or of the reference, once per finding): `section-label` +(a label that shows the number of a unit Word cannot show in that list: the section's on a list that is not the heading +list's, or, for a list on the detached levels or past nine levels, the counters of the enclosing units, named, with what +Word draws), `list-depth` (a nesting past 36 levels), `letter-overflow` (Word repeats letters after 26, the renderer counts on: `aa`, `ab`), `roman-overflow` +(4000 and beyond), `shared-counter` (a unit that goes on a counter of its abstractNum that others have used, so that it is restarted at +its number, a fixed value that will not follow edits before it: a sub-heading that shares a designation with the +items of its section, a list that goes on from another, a list whose level another used in between), `number-mismatch` (any +other difference: the first unit of a run that Word counts off by the same amount, with how many follow; it is the +catch-all of the planner and does not occur outside documents that skip a heading level), `reference` (a +cross-reference that is fixed text, because Word draws delimiters around a counter that the reference does not have +and `\t` is not allowed to drop them, or the number holds counters the reference writes itself; one that has the number +of a skipped heading level as text; or one that reads differently in Word for a reason that no diagnostic of a unit +already gives; references that fail alike are one finding), `reference-cycle` (a heading's title holds a reference that +leads back to it: written as text), and, as information, `alternative` (alternatives share a number as fields of the first, which deleting in Word breaks), `static-restart` (a list that restarts at 1 by a +fixed setting: nested in a list of another kind, in an item whose number is on the heading list but which the list +cannot ride, in an alternative that shares a number, or the list that begins the next abstractNum past Word's nine +levels) `libreoffice-count` (items that Word numbers as the renderer does but LibreOffice reads higher, because a level of their list that nothing numbered before a deeper item is counted as 1 there and goes on from it; once per document) and `reference-word-only` (a cover relies on `\t`, which LibreOffice ignores, once per document). A +heading level that is skipped is the Core Error `heading-skip`; the plan only notes it on the unit. + ### 5. Write (`writers/`) > **A writer contains no LegalDown rules.** `tree.assert_resolved()` rejects a tree that still @@ -223,7 +452,8 @@ references show; the Warning says so. Every other heading reads exactly as its d |---|---| | `TextWriter` | Plain text (§13.6). A soft line break inside a paragraph is a space, a hard break a new line, and lists are indented with their labels, one quote stays one quote, and tables keep escaped cells and alignment. It takes the style's `attachments.separator` (default `page-break`; `rule` and `page-break` both print `---` before the attachments heading and between attachments). It is the **test oracle** for resolution | | `HtmlWriter` | A self-contained HTML5 page, or with `standalone=False` just the `
`. Semantic markup, with labels as real text rather than CSS counters, and a stylesheet generated from the style's presentation settings, including print rules | -| DOCX, PDF | Planned ([ADR 0004](decisions/0004-html-first.md)) | +| `DocxWriter` | A `.docx` file, as bytes (`RenderResult.binary`; `output` is empty). OOXML written directly with `zipfile` and `xml.etree` ([ADR 0008](decisions/0008-docx-word-native.md)): the parts, the styles generated from the style template, settings, page setup, running headers and footers, properties, a deterministic ZIP, and one sanitiser for every text and attribute. **The body is laid out as a `DocxPlan` says** (`resolve/docx_plan.py`, the planner's data model): `numbering.xml` from its abstract numberings and nums; each section, list item and top-level paragraph gets `w:numPr`, `w:ind`, bookmarks and, for a later alternative that shares a number (§15.8), `numId` 0 and its number as `REF` fields, as its `UnitPlan` says; a `{{ref:}}` is the literal text and `REF` fields (cached results as planned) of its `RefPlan`; a `{{term:}}` or an `{{attach:}}` is an internal hyperlink (or a `REF` for an attachment's title), a defined term a bookmark; every block (lists with their continuation blocks, quotes, drafting notes, code, tables with a repeated header row, rules) and inline (emphasis, strong, code, external links through relationships, breaks, images as alt text, blanks, choices, failure markers) has its Word form. The writer decides none of it (ADR 0003) and a field that points at a bookmark the document does not hold is a bug in the plan (`InternalError`). **Front and back matter** (`writers/docx/matter.py`, in the order the HTML shows them): the header rows and the parties as borderless tables; the contents label (style `ContentsHeading`: no outline level, so never an entry) and one complex `TOC \o "1-N" \h \z \u` field (N is `contents.depth`) whose cached result is `plan.toc` as `TOC1` to `TOC5` paragraphs (a hyperlink to the `_Toc` bookmark holding number, separator, title, condition label, a right tab with a dot leader at the text width and a `PAGEREF` field with an empty cached page), opened in the first entry and closed in a paragraph of its own; the attachments heading and each attachment's title are paragraphs of styles with an outline level (not a `\t` style list, whose separator depends on the locale), bookmarked by the plan's `ldatt_` name (so `{{attach:}}` resolves) and their `_Toc` name, with `attachments.separator` as a page break (the paragraph property `pageBreakBefore`, so no empty paragraph or blank page is left) or a rule above each attachment; the signature blocks as a borderless table, two parties to a row, rows that do not split and paragraphs that keep with the next. Every bookmark is written through the same `Bookmarks` as the body, whose first, dry run tells the second which names exist. The numbering, bookmarks and cross-reference fields come from the plan `render` computes (the writer plans a tree itself when it is used directly and given none). The DOCX-only settings are converted and checked before rendering (`style/docx.py`) | +| PDF | Planned ([ADR 0004](decisions/0004-html-first.md)) | ## Package layout @@ -238,14 +468,24 @@ src/legaldown_render/ ├── build.py stage 3 ├── resolve/ │ ├── resolver.py stage 4 -│ ├── numbering.py counters, the §13.1 scheme presets, label templates +│ ├── numbering.py counters, the §13.1 scheme presets, label templates, the formats a style gives +│ ├── docx_plan.py the DOCX plan: Word numbering, bookmarks, cross-reference fields and contents for the resolved tree, and the style check DOCX runs before rendering +│ ├── docx_numbering.py Word's counters, label templates as tokens, lvlText, the numbering simulator +│ ├── docx_cover.py the cover of a designation by REF fields and delimiters +│ ├── docx_bookmarks.py bookmark names │ └── values.py Babel formatting of dates, money, durations ├── style/ │ ├── model.py the style dataclasses (every field has a default) │ ├── loader.py layering, extends, overrides, validation +│ ├── docx.py the presentation settings in Word's units; the style errors only DOCX has │ ├── labels.py built-in labels per language (en, cs, de, fr, pl, sk) │ └── builtin/ default.yaml, continental.yaml, outline.yaml -└── writers/ html.py, text.py +└── writers/ + ├── html.py, text.py + └── docx/ the DOCX writer: ooxml.py (sanitiser, builders), package.py (parts, relationships, ZIP), + styles.py, settings.py, headers.py, properties.py, numbering.py (numbering.xml + from the plan), body.py (blocks, numbered paragraphs, tables), runs.py (inlines, fields, + hyperlinks), matter.py (title block, contents field, attachments, signature blocks) ``` ## Dependencies @@ -257,7 +497,7 @@ src/legaldown_render/ | `babel` | CLDR locale data (ADR 0005) | | `pyyaml` | Style templates | -All of them are pure Python. The DOCX and PDF writers will bring their dependencies in as extras. +All of them are pure Python. The DOCX writer needs nothing more (it writes OOXML itself, ADR 0008); the PDF writer will bring its dependency in as an extra. `lxml` is a test dependency only (`dev` extra), for validating DOCX parts against the OOXML schemas. The renderer uses only the validator's public API: `legaldown`, and the tooling modules `legaldown.syntax` and `legaldown.grammar` (roadmap item U3, done in validator 0.4.0). Before 1.0 @@ -282,7 +522,7 @@ Diagnostics reuse `legaldown.Diagnostic`. The renderer adds rules only it can ev `ref-not-enumerated`, `path-not-relative` and `path-outside-root` (Errors for a file reference that is not relative, or that leaves the document root, §2.3; reported here because validator 0.4 checks neither), specification rule ids, plus renderer-specific ids prefixed `render-`: -`render-not-processed`, `render-link-refused` (a Warning for a link whose URL scheme links may not use; its text is shown), `render-locale-fallback` (an Info when no formatting locale is known, a Warning when a duration unit has no CLDR pattern in the locale and English is used), `render-ref-ambiguous`, `render-term-limit` (a Warning when defined terms refer to each other too deeply or too often: past 32 nested terms, 200,000 inline nodes resolved inside repeat computations of a term text (only cycles cause them), or 60 inline levels nested inside an open term, a `{{term:}}` shows its id; one block's own nesting, up to the builder's 100 levels, never counts), `render-designation-cycle` (a Warning for a heading that refers back to itself under the `none` scheme) and `render-designation-limit` (a Warning under the `none` scheme for headings that refer to each other past 32 headings, 60 nested levels, or 1,000 characters, and for a reference to a heading longer than 1,000 characters, which shows its first 1,000), and `render-term-too-long` (a Warning for a defined term whose text, with its references resolved, is longer than 1,000 characters: `{{term:}}` then shows the definition id). The validator's +`render-not-processed`, `render-link-refused` (a Warning for a link whose URL scheme links may not use; its text is shown), `render-locale-fallback` (an Info when no formatting locale is known, a Warning when a duration unit has no CLDR pattern in the locale and English is used), `render-ref-ambiguous`, `render-term-limit` (a Warning when defined terms refer to each other too deeply or too often: past 32 nested terms, 200,000 inline nodes resolved inside repeat computations of a term text (only cycles cause them), or 60 inline levels nested inside an open term, a `{{term:}}` shows its id; one block's own nesting, up to the builder's 100 levels, never counts), `render-designation-cycle` (a Warning for a heading that refers back to itself under the `none` scheme) and `render-designation-limit` (a Warning under the `none` scheme for headings that refer to each other past 32 headings, 60 nested levels, or 1,000 characters, and for a reference to a heading longer than 1,000 characters, which shows its first 1,000), and `render-term-too-long` (a Warning for a defined term whose text, with its references resolved, is longer than 1,000 characters: `{{term:}}` then shows the definition id), and the rules the DOCX planner reports on what Word draws differently from the other formats (`render-docx-section-label`, `-list-depth`, `-letter-overflow`, `-roman-overflow`, `-shared-counter`, `-number-mismatch`, `-reference`, `-reference-cycle`, as Warnings, `-alternative`, `-static-restart`, `-libreoffice-count` and `-reference-word-only` as Infos; see "The DOCX planner"). The validator's diagnostics carry their line (§16.9), and so do the renderer's own, and the CLI prints it as `file:line:`. @@ -315,6 +555,14 @@ out when the validator reported the same path at the same line, or at none (`dia - Raw HTML in the source is never emitted (§8.7). - Every text node and attribute is escaped. +- DOCX: every text, attribute, field instruction, bookmark name, relationship and property value goes + through one sanitiser (`writers/docx/ooxml._t`, applied inside the element builders, so a caller + cannot forget it). It drops what XML cannot carry (C0 controls but TAB, LF and CR, DEL, C1, + U+FFFE/U+FFFF, lone surrogates): `xml.etree` alone writes `\x01` raw and a lone surrogate as a + character reference, and Word then reports the file as unreadable. TAB and line breaks in a text + become `w:tab` and `w:br`, never kept in `w:t`. A hyperlink relationship's Target is percent-encoded + to a valid URI, or the link stays text. The part contents are fuzzed for well-formedness in + `tests/test_docx_escaping.py`. The untrusted mode and the style caps apply to DOCX unchanged. - A link is a hyperlink (§8.7) only if its URL is relative (no scheme, a fragment) or its scheme is one of `urls.LINK_SCHEMES`: `http`, `https`, `mailto`, `tel`, `ftp`, `ftps`, `sftp`, `sip`, `sips`, `sms`, `urn`, `geo`, `news`. It is an allow-list, because a renderer reads documents it @@ -423,7 +671,14 @@ out when the validator reported the same path at the same line, or at none (`dia ## Testing Each module is named for its topic. Shared helpers (the minimal document header, the `text` and -`html` renderers, and running the command as a program) are in `tests/conftest.py`. +`html` renderers, and running the command as a program) are in `tests/conftest.py`; those for the DOCX tests +(reading a package, validating it, running LibreOffice) are in `tests/docx_support.py` and `tests/lo/`; the text of a LibreOffice document is read from its model (`tests/lo/uno_text.py`: a line a paragraph, the list label before a numbered one, a table's cells in order), not from its text export, whose table layout differs between versions. + +Markers: `conformance` (the specification's documents), `ooxml_schema` (DOCX parts against the schemas), `libreoffice` +(DOCX opened in LibreOffice; skipped without `soffice` and `python3-uno`, a failure when `LEGALDOWN_REQUIRE_DOCX_TOOLS` +is set, as in the CI job that installs them) and `acceptance`: the DOCX acceptance corpus, whose run takes about a +minute of LibreOffice time and is left out of a plain `pytest` (it runs with `-m`, with `LEGALDOWN_ACCEPTANCE=1`, or when +its file is named). `pytest -n auto` (pytest-xdist, in the `dev` extra) runs the whole suite in a third of the serial time. | Suite | What it checks | |---|---| @@ -431,6 +686,9 @@ Each module is named for its topic. Shared helpers (the minimal document header, | `tests/test_render.py` | Numbering schemes, designations, fallbacks, every failure marker, template view, labels by language, HTML safety, unique anchors, determinism, strict mode, locales; the renderer's reading of the validator's structure (lists, quotes, comments, markers) | | `tests/test_values.py` | Money, dates and durations in a locale: exact digits, plural of the value shown, currency display, CLDR fallback to English | | `tests/test_numbering.py` | Counter formats, label templates and the preamble's `{section}` | +| `tests/test_numbering_structure.py` | The numbering structure in the tree (`formats`, unit ids, numbers, parents, alternatives, list families, cross-reference units): headings in each scheme and with skipped levels, nested lists, section-qualified items, numbered paragraphs, alternatives, the `none` scheme and its second pass; and that every label, designation and reference can be written again from the structure alone, over the tests' and the specification's documents | +| `tests/test_docx_plan.py` | The DOCX plan: abstractNums per style and scheme, heading levels, top-level, nested, sibling, cross-family, quoted lists, lists past Word's nine levels (the next abstractNums, the cap at 36 levels), numbered paragraphs, labels that need the section and where they go (the heading list's levels, the reserved levels of lists nested in them with the path in their text, the detached levels and their reports, the fallback), alternatives that share a number (of sections, items and paragraphs, in twos and threes, with children), resynchronised units, letter and Roman overflow, diagnostics (lines, file, order, repeats, the catch-all that groups runs of units counted off), style errors (also through `docx_settings`), determinism; and, unit by unit, that Word's simulated count and label are the resolver's unless the unit's own counter reads differently in Word or the plan notes why, over the tests' and the specification's documents under each style and list setting, and over random documents and alternatives (each variant plans its own sample, and every unit is referred to by some document). Also the cross-references: the worked table of covers for each style (default, outline, mixed, continental, `none`, `Article {n}`, `§ {n}`, `Čl. {n}`, `Section {path}`: a field standing for the words of `references.format`), the cover algorithm on its own (the literal counter of a skipped heading level, what `\t` is taken to do under each `docx.reference_text_switch`), references to other sections and to unnumbered units, `references.format`, every reason a reference stays fixed text (delimiters, counters of other units, a skipped heading level) and what the message tells the style, bookmark names (prefixes, length, collisions, hidden names, determinism), the table of contents, the components of the headings' graph of title references (and that a chain of thousands of headings is planned in linear time), the render's diagnostics, and the property that every cached reference reads as the renderer's text or a diagnostic says why not, over the same documents and random documents with an anchor on every unit; and that over every variant a reference is fixed text only in a document that skips a heading level, and, over label forms the built-in styles do not have, only for the two reasons the style can change | +| `tests/test_docx_simulator.py` | Word's counters, label templates as tokens (equal to filling the template as text), `lvlText`, and the simulator: restarts, start overrides, shared counters, skipped levels, `peek`/`current` | | `tests/test_build.py` | Building the tree: comment-only items and quotes, emphasis next to a directive, spacing where an inline comment was dropped | | `tests/test_terms.py` | What `{{term:}}` shows, references inside terms, terms that name themselves or each other, the cap on a term text | | `tests/test_none_designations.py` | Designations under the `none` scheme: references and terms in headings, items and paragraphs under them, ambiguity on the final designations, cycles, the depth, length and nesting limits, the lines of the Warnings, work bounds, invariants on random documents | @@ -447,4 +705,16 @@ Each module is named for its topic. Shared helpers (the minimal document header, | `tests/test_cli_io.py` | Encodings of the streams, input that is not UTF-8, unwritable output, LF line endings, the extension rule of `-o` | | `tests/test_errors.py` | The exception hierarchy and pickling of errors | | `tests/test_escaping.py` | Control characters from a document or the command line never reach a diagnostic or standard error unescaped | +| `tests/test_docx_style.py` | `page.header`, `page.footer` and `docx.*` (`update_fields_on_open`, `reference_text_switch`): validation, and how CSS lengths, colours and fonts become Word's units, with the StyleErrors only DOCX has | +| `tests/test_docx_package.py` | The OOXML package: parts, relationships, content types, settings order, styles, page setup, headers and footers, properties, the body of a planned document, determinism, the sanitiser and builders, the API and command line | +| `tests/test_docx_body.py` | The DOCX body from hand-made plans: numbering definitions, `w:numPr`/`w:ind`/alternatives, bookmarks and `REF` fields with cached results, hyperlinks, every inline and block, escaping, schema validity (marker `ooxml_schema`) and a LibreOffice round trip of headings, nested lists and references whose cached values are wrong on purpose (marker `libreoffice`) | +| `tests/test_docx_escaping.py` | Well-formedness of every part for any text, with a seeded fuzz and the characters XML cannot carry | +| `tests/test_docx_special.py` | What the planner's hard cases become in the file: the heading list's levels that hold items, the reserved levels of lists nested in them, the detached and the next-nine-levels abstractNums, and later alternatives as `numId 0`, `w:ind` and `REF` fields; and that documents with them validate against the schemas (marker `ooxml_schema`) | +| `tests/test_docx_validation.py` | Every part of every generated document against the ISO/IEC 29500-4 schemas (marker `ooxml_schema`; needs lxml and `python tests/schemas/fetch.py`) | +| `tests/test_docx_roundtrip.py` | LibreOffice as the oracle of the numbering and the cross-references, in one LibreOffice session: every document of the corpus (the tests', the specification's, stress documents with cycles and unanchored ancestors) under the default, outline, continental, mixed and `none` schemes, with every cached field result made wrong (`XXX`), every numbered paragraph marked and a line per reference added; the number LibreOffice draws, every reference it computes and every body paragraph's text equal what the file cached and the renderer wrote, modulo the gaps the plan reports (`\t` ignored, letters past 26, labels the heading list cannot show); alternatives are checked against the renderer (their `numId 0` number is the holder's, as fields), and so are the continental items on the heading list and the lists on its reserved levels; the table of contents is built by LibreOffice from a `TOC` field the test adds and must list the headings with the numbers the plan gives and none of the items; and edit-then-update (`tests/lo/uno_edit.py`, which inserts and deletes paragraphs, and can put an inserted paragraph on another item's list as pasting a copy does): path labels on the reserved levels, in a detached list (and the documented continuation of a list copied under a new parent) and past nine levels, a heading inserted before a document that skips a level, items, headings, a clause and a nested item inserted before the targets as a lawyer does, fields updated, every reference and the numbers equal the renderer's for the edited source (the new nested item under the continental style reads `a)`), and, for a sub-heading restarted by a fixed value, the plan's warning and what an edit before it keeps right (marker `libreoffice`) | +| `tests/test_docx_acceptance.py` | The measurable claims of ADR 0008 over the whole corpus (the tests' documents, the specification's examples and valid fixtures, the stress documents) under the default, outline and continental styles, with the contents on. **Equivalence:** LibreOffice's text, with fields and indexes updated, equals the plain-text writer's, line by line, modulo the differences the formats have by nature (one documented function: borders for rules and signing lines, table cells, quote and link markup, where a condition label stands, `label: value` rows, the contents' page numbers, which are checked to be positive, never decreasing and within the page count) and modulo the gaps the plan reports; the documents that differ are exactly the stress documents that ask for what Word draws differently. **Live after edits:** in one LibreOffice session, a heading, an item, a nested item, a numbered paragraph (continental) are inserted and a clause deleted in up to ten documents under each style, and the result equals the plain-text writer's for the source with the same edits (a difference is allowed only where the render reports a fixed number). **Validity and determinism:** every part validates and two renders are the same bytes. Negative controls show the comparison fails on a wrong number, a missing line, pages out of order | +| `tests/test_docx_matter.py` | The front and back matter: the XML of the title block, the `TOC` field and its cached entries, the attachments and the signature blocks; documents with every part (hostile text too) against the schemas; LibreOffice listing exactly the contents' entries after an update, resolving `{{attach:}}`, paging the attachments as configured, taking a heading inserted in the editor into the contents, and reading in the text writer's order | +| `tests/test_docx_lo_text.py` | The reader of a LibreOffice document's text (`tests/lo/uno_text.py`) over stand-ins: one line a paragraph, the list label before a numbered paragraph, a table's cells by row then column, soft breaks as lines | +| `tests/test_docx_libreoffice.py` | The documents opened in LibreOffice through UNO (`tests/lo/uno_export.py`), fields updated: the words in order, header, footer, and the page count and the page numbers compared with each other and with page breaks, not with the page count of a font (marker `libreoffice`; skipped without `soffice` and `python3-uno`) | +| `tests/test_docx_wordcheck.py`, `tests/word_check.py` | The generator of the document for the manual Word checks of what the planner assumes (`python tests/word_check.py OUT.docx`; `.github/PUBLISHING.md` items 8 and 9): one row for each label form the plan relies on (`\t` on `Article %1`, `§ %1`, `Section %1.%2`, `Čl. %1`, `No. %1`, `(%1)`, `%1)`, `%1:`), the heading list's reserved level `%1.%2(%6)` with levels 3 to 5 unused, and a skipped heading level, each with the planner's assumption beside it taken from the planner's own code; that the package is valid (marker `ooxml_schema`) and LibreOffice computes the plain fields as the planner caches them (marker `libreoffice`); and that the Python snippet in the checklist still runs and makes every file the checklist names, including the document of the level nothing numbered | | `tests/conformance/` | Every example and fixture in the specification repository renders, every internal link has a target, and each invalid fixture renders its rule's failure marker. Needs `$LEGALDOWN_SPEC_DIR` or a sibling `../LegalDown` checkout | diff --git a/docs/concepts.md b/docs/concepts.md index 2d66b40..c946c23 100644 --- a/docs/concepts.md +++ b/docs/concepts.md @@ -21,7 +21,7 @@ these. Its inputs are: - **an answers set** (optional, templates only): the values used to assemble a template before rendering (§15.7) -Its outputs are the **rendered file** (HTML or plain text, later DOCX and PDF) and a list of +Its outputs are the **rendered file** (HTML, plain text or DOCX, later PDF) and a list of **diagnostics**. The source never changes during rendering. Given the same source, style template, and options, @@ -99,7 +99,7 @@ style template, and the answers file the CLI is given with `--answers`. | **Render tree** | This package's format-neutral tree of blocks and inlines, built from the source and document model. It is structurally complete: nested lists, inline formatting, and directives are all nodes | | **Resolution** | The stage that computes everything the specification leaves to render time: numbers, designations, link targets, display text, formatted values, markers | | **Resolved document** | The render tree after resolution. It is immutable and holds no unresolved LegalDown semantics. It is the only input to writers | -| **Writer** | A backend that turns the resolved document into a file format (`html`, `text`, later `docx` and `pdf`). A writer contains no LegalDown rules, only layout | +| **Writer** | A backend that turns the resolved document into a file format (`html`, `text`, `docx`, later `pdf`). A writer contains no LegalDown rules, only layout | | **Style template** | The configuration file that controls presentation (§13.7). See [style templates](style-templates.md) | | **Numbering scheme** | How headings are numbered: `decimal`, `legal-outline`, `mixed`, or `none` (§13.1) | | **Enumeration** | How list items are labelled at each nesting level, for example (a) / (i) / (A) (§13.2) | diff --git a/docs/decisions/0004-html-first.md b/docs/decisions/0004-html-first.md index d75eb3a..5bad51e 100644 --- a/docs/decisions/0004-html-first.md +++ b/docs/decisions/0004-html-first.md @@ -1,6 +1,7 @@ # 0004. Output formats: text and HTML, then DOCX, then PDF -- Status: Accepted (DOCX numbering and the PDF engine are still open) +- Status: Accepted. The DOCX decisions (python-docx, the numbering question) are superseded by + [0008](0008-docx-word-native.md); the PDF engine is still open - Date: 2026-09-23 ## Context @@ -13,7 +14,8 @@ the dependencies they bring. - **v0.1: `text` and `html`.** HTML has no native dependencies, supports links, and is easy to inspect. Plain text costs almost nothing and is the test oracle for resolution. -- **v0.3: `docx`** with python-docx, behind the `[docx]` extra. +- **v0.3: `docx`** with python-docx, behind the `[docx]` extra (superseded by + [0008](0008-docx-word-native.md): OOXML written directly, in the base package). - **v0.4: `pdf`**, behind the `[pdf]` extra. ## Open questions @@ -44,3 +46,7 @@ The leading option is WeasyPrint. It will be decided with a spike when v0.4 star - 2026-10-28: v0.4 became a hardening release (see the [roadmap](../roadmap.md)), so `docx` moves to v0.5 and `pdf` to v0.6. The open questions now read v0.5 (DOCX numbering) and v0.6 (PDF spike). +- 2026-10-28: [0008](0008-docx-word-native.md) decides the DOCX questions. Numbering is Word-native + (numbering definitions and `REF`/`TOC`/`PAGE` fields, never static text), and the writer produces + OOXML itself with the standard library, so DOCX needs no extra and no python-docx. "DOCX numbering" + above is closed; the PDF engine is not. diff --git a/docs/decisions/0008-docx-word-native.md b/docs/decisions/0008-docx-word-native.md new file mode 100644 index 0000000..2e511cd --- /dev/null +++ b/docs/decisions/0008-docx-word-native.md @@ -0,0 +1,195 @@ +# 0008. DOCX: Word-native numbering and fields, written as OOXML directly + +- Status: Accepted. Decides the DOCX questions [0004](0004-html-first.md) left open (the numbering + question, and the python-docx extra); the PDF engine stays open there +- Date: 2026-10-28 + +## Context + +Lawyers who receive a DOCX edit it. A clause inserted in Word must renumber the clauses after it, +and a cross-reference to "Section 2.3(b)" must follow the clause it points at. Numbers written as +text are right on the day of export and wrong after the first edit. The same holds for the table of +contents and its page numbers, and for the page numbers in a footer. + +[0003](0003-resolved-document.md) keeps one set of numbers for every format: the resolver +decides "2.3(b)", and a writer only lays it out. The question for DOCX is how a writer can lay out a +number that Word will keep recomputing without giving up that guarantee. + +## Decision + +1. **Word-native numbering and fields. Static numbers are not an option.** Heading numbers, list + enumeration and paragraph numbers are numbering definitions; cross-references, the table of + contents and page numbers are fields. The renderer still computes every value. It writes it only + as the cached result of a field, or as the input from which a numbering definition is built. + Nothing Word can compute is written as literal text. The document reads correctly before any + field is updated, and stays correct when a lawyer inserts a clause and updates fields. +2. **OOXML is written directly**, with the standard library only (`zipfile`, `xml.etree`), not with + python-docx. + - python-docx has no API for numbering definitions, fields, bookmarks, internal hyperlinks, the + table of contents or the order of the settings, so all of that would be raw XML in any case. + - Its starting template brings Letter paper, about 160 styles and stale metadata that would + have to be removed. + - A writer of our own gives schema order, byte-for-byte determinism and no new dependency, so + DOCX is part of the base package and 0004's `[docx]` extra is not needed. + - Validity becomes ours to guarantee, which CI checks against the ISO/IEC 29500 schemas. +3. **The render tree carries format-neutral numbering structure** (0003): for each numbered unit a + stable id, its counter value, its level format and its parent unit. The resolver stays the only + place where numbering rules live. +4. **A DOCX planner maps the tree to Word.** It decides lists, levels, numbering instances, + bookmarks, how each cross-reference is composed of fields, and the cached values, and it reports + the diagnostics where Word cannot do what the renderer does. The writer takes the tree and + the plan, contains no rules and emits nothing. +5. **Cached results are what Word will compute.** They come from a model of Word's numbering in the + planner, not from the renderer's designations. The cached table of contents comes from the same + model, not from the tree's contents list. Where Word's value would differ from the renderer's + (letter counters past 26, roman numerals from 4000), the planner warns. A field is never replaced + by its text. +6. **Configuration problems raise before rendering** ([0006](0006-styles-and-options.md)). A style + setting Word cannot express is a `StyleError` when the format is `docx`, not a diagnostic: a + length in `%`, `ch` or `ex`, a colour the writer cannot read, a numbering label with a literal + `%` before a digit (which a level text cannot escape). HTML keeps accepting all of them. + Lengths in `em` and `rem` are converted through `typography.font_size`. +7. **Word-equivalence is guaranteed for documents without Errors.** For example `heading-skip` + is a Core Error, so a document that renders has headings whose level is the count the resolver + already clamped, and the writer uses `Heading1` to `Heading5` as they stand. + +Also decided here, because every later part rests on them: + +- **Page furniture.** `page.header` and `page.footer` are style settings, each with a left, a + centre and a right text. They substitute `{page}`, `{pages}`, `{title}`, `{version}` and `{date}`. + The default footer is `{page} / {pages}` centred, and the default header is empty. `{page}` and + `{pages}` are `PAGE` and `NUMPAGES` fields. The PDF writer will read the same settings. +- **`docx.update_fields_on_open`** (default true) writes `w:updateFields`. Word then asks once and + updates the cross-references and the table of contents with its page numbers. LibreOffice and Word + Online ignore it. +- **Compatibility mode 15**, so Word does not open the file in Compatibility Mode. The settings are in + the order the schema requires (`updateFields` follows `characterSpacingControl` and precedes + `compat`), and contain no revision ids. +- **Front and back matter.** The table of contents is a `TOC \o "1-N" \h \z \u` field whose cached result is the + planner's entries (each a link to a `_Toc` bookmark with a `PAGEREF` field whose cached page is empty, since only + Word knows the pages). The attachments heading and each attachment's title are paragraphs of styles with an outline + level, so `\u` lists them and the field needs no `\t` list of styles (whose separator depends on the locale). The + contents label has no outline level, so it is not an entry, and a condition label is a run inside its heading's + paragraph so that the contents keep it. A page break between attachments is the paragraph property "page break + before". The title block, the parties and the signature blocks are borderless tables. +- **Output.** A DOCX is bytes: `RenderResult.binary` holds the file and `RenderResult.output` stays `str` (empty), so no + typed caller breaks. The command writes `-o FILE.docx`, or standard output with `-o -`, never to a terminal. Images + are shown as their alt text: below the Full level no file is read (§2.3, §17.6). +- **Safety.** One sanitiser is applied to every text, attribute, field instruction, bookmark + name, relationship and property value. It drops the characters XML cannot carry. Without it + `xml.etree` writes a control character raw and a lone surrogate as a character reference, and Word + reports the file as unreadable. The untrusted mode and its style caps apply unchanged. A hyperlink + relationship is made only for the schemes links may use (no relative target, no `//host`, no + backslash), because Word follows what a relationship says. +- **Deterministic output.** Fixed part order, relationship ids by order of use, fixed ZIP times, + attributes and compression, no timestamps in the properties, and prefixes from the writer's own table, + not from `xml.etree`'s global registry (which another library may change). The ZIP is + deterministic **for a given zlib**: a different zlib may compress the same parts to different, + equivalent bytes. The parts themselves, uncompressed, are the same everywhere, and the tests pin + them by SHA-256. + +### Cross-references + +A cross-reference is a `REF` field to a bookmark on the target, with the switch `\n` (the paragraph +number, no trailing period) and `\h` (a link). The renderer's designation ("2.3(b)", "I.A.1.a(i)") is +covered by segments: literals, and one such field for each numbered unit on the target's path whose +label contributes. A `{path}` label needs one field on the target. An `{n}` label needs one field per +ancestor, joined by the literals the reference templates give. What is covered is the whole text the +reader sees (the designation inside `references.format`), so a field that returns the words of the format +(`Article I` under `Article {designation}`) stands for them, and Word's `\t` switch, which leaves the words of +a number out, is needed only where the number holds words the reference does not. What Word does with `\t` +is documented for one example only (`Section 1.01` gives `1.01`), so the style says how far to trust it +(`docx.reference_text_switch`: `off`, `documented`, `guess`). + +**A reference is fixed text only where Word cannot express it.** Each such reference is reported +(`render-docx-reference`) with the class it falls in, so that the style can be changed if it matters: +a number with **delimiters around its counter** that the reference does not have (a label `({n})` under a +ref `{n}`, `{n}:`, `No. {n}`) where the switch is off or not documented to drop them; a number with **the +counters of other units** that the reference writes itself (a label `{section}.{n}` under a ref `{n}`), +which a field returns whole or not at all; and, in a document that skips a heading level (`heading-skip`, an +Error), **the counter of the skipped level**, held by no unit, which is text between the fields of the units that +do hold theirs. A heading's title that refers back to itself under the `none` scheme is the other case +(`render-docx-reference-cycle`). The built-in styles have none of these in a document without errors, and a +reference to an item in a list nested up to 36 levels is made of fields. +Under the `none` scheme a reference is a `REF` to the heading text. + +### Numbering the hard cases + +- **Items and paragraphs whose label needs the section** ("5.1" under `{section}.{n}`) go on the heading + list's level after their section's, where the level reads as the label does (`%1.%2`). Such an item has + no heading style, no outline level and so no entry in the table of contents. A list nested in one goes on + the heading list's levels 6 to 9 (headings have five), which are reserved for that: the level text is the + label's without the section (`%6)`), on the parent's num with no override, so the list restarts under every + new item and sibling lists go on counting, as in any list. When a nested list's label shows more than its + own counter (a `{path}`), the level text carries each counter at the level of the heading list that holds it + (`%1.%2(%6)`: the section's, the item's, its own), so the label and every reference to it stay live. Where the level after the + section's reads otherwise (the outline and mixed schemes, `none`), the label is written without the section and + reported. +- **Lists nested where Word cannot show the counters above them** (a list in an ordered item, a path label whose + reserved level another text has taken or that is nested past four levels, a quote's list) go on a **detached** + abstractNum: each level shows its own counter alone. The label Word draws is shorter than the renderer's + (`(A)` for `1(a)(1)(i)(A)`) and is reported with the units whose counters it leaves out; references are still + fields of each unit's own counter. The list restarts at a fixed value under a new parent (as any list nested + in a list of another kind does), and lists nested in its items restart by structure. +- **Lists nested past Word's nine levels** go on in the next abstractNum of their family (36 levels in all), whose levels are + the family's levels of those depths with their own counters, instead of repeating level 9. The list that begins + one restarts at a fixed value; inside it the lists restart by structure. Beyond 36 levels the last is used again + and reported (`render-docx-list-depth`). +- **Alternatives that share a number** (§15.8). Word cannot give two paragraphs one number on one level, except + by `numId 0` and a field: the later alternative is such a paragraph, with a `REF \n` field of the unit that + holds the number as its number. The units under it restart in a num of their own with a `startOverride`. This is + live for edits elsewhere (inserting a clause before the alternatives moves their numbers and every reference), but + **not for the first alternative itself**: deleting it in Word makes every later alternative, their children and + every reference to the shared identifier read "Error! Reference source not found", and pressing Enter at the start + or end of a later alternative gives a new paragraph with no number (it copies `numId 0`). There is no live option + that avoids this, so the render reports one information per document (`render-docx-alternative`), and the way to a + final document is to render it with the condition values, which resolves the alternatives. +- **A unit that Word would number off the resolver's** because it shares a counter with others of its + abstractNum (a sub-heading after the clauses of its section, a paragraph after clauses on the same level) is + restarted at a fixed value and reported (`render-docx-shared-counter`), so the file is right when it is opened. The + value is fixed: a restarted sub-heading keeps it when clauses or headings are inserted before it, but deleting it + makes the headings after it read higher; a restarted clause or paragraph does not follow what is inserted or + deleted before it. + +### Rejected alternatives + +- **python-docx** (0004's choice), for the reasons in point 2. +- **Static numbers.** Rejected by point 1. It would also make the DOCX the one format whose numbers + could disagree with the document after an edit, which is the reason to give lawyers a DOCX at all. +- **`\w` and `\r` for cross-references.** Word joins the numbers of the levels itself, and its rules + produce "2.(a)" for an item of section 2 and "I.A.1.a.(i)" for an outline, not the designation the + style asks for. The renderer composes the designation from `\n` fields instead. +- **List items that need the section number ("5.1", "5.2" under `{section}`) on reserved deeper + levels of the heading list.** The items would count on a level that is not the one after their + section's, and could not show the section's counters. Only the lists nested in such items use the reserved + levels 6 to 9, which no heading uses. +- **`STYLEREF "Heading 1" \n` with a `SEQ` field** for the section prefix of those items. It works + only in Word, cannot be tested in LibreOffice, and gives no number when the lawyer presses Enter in + the middle of a list. The items instead go on the heading list's next level, which numbers them + "%1.%2" and restarts them for each top-level list with a numbering instance that overrides the start. +- **Dropping the section prefix** from such labels. It changes the label the style asks for. + +## Consequences + +- A DOCX lawyers can edit: insert a clause, update fields (Word does it on open by default), and the + numbers, references, table of contents and page numbers follow. +- Some restarts cannot be kept live in Word and are documented and warned about, not prevented: a + nested list of another family than its parent (and any detached list), a list nested in an alternative, the list + that begins a further abstractNum past nine levels, a unit restarted at a fixed + value because it shares a counter, and paragraph numbering in a section the user adds; and the numbers of the + alternatives that share one live only as long as the first alternative stays. A lawyer who adds a + parent item restarts such a list by hand. +- Viewers that never update fields (Word Online, Quick Look, the Google Docs import, Outlook's + preview) show the cached results, which are right, but the table of contents without page numbers. +- Verification is part of the design: every part is validated against the schemas, the documents are + opened in LibreOffice (the oracle available in CI, which updates fields and reports the text), and + the same documents are checked by hand in Word before a release. LibreOffice ignores `\t`, + `lvlRestart`, `SEQ \s` and `w:updateFields`, fills a skipped heading level with its start value, and counts + a level that nothing in a list numbered before a deeper item as 1 where Word is expected to read it as 0 + (`render-docx-libreoffice-count`), so those differences are normalised in the comparison and the Word-only ones + are on the checklist. The acceptance corpus compares whole documents with the plain-text writer's output under + three styles, before and after edits made in the editor. +- Cost: an OOXML writer of our own, with its schema checks, instead of a library. The numbering + planner is the bulk of the work. +- Style template additions: `page.header`, `page.footer`, `docx.update_fields_on_open` and + `docx.reference_text_switch` ([style-templates.md](../style-templates.md)). The HTML writer does not read the first two yet. diff --git a/docs/decisions/README.md b/docs/decisions/README.md index f8b9470..a701cd9 100644 --- a/docs/decisions/README.md +++ b/docs/decisions/README.md @@ -9,10 +9,11 @@ later change that does not reverse a decision is appended to its Consequences as | [0001](0001-separate-package.md) | A separate package and repository, depending on `legaldown-validator` | Accepted | | [0002](0002-one-parser.md) | One LegalDown parser: the core package; CommonMark structure from markdown-it-py for now | Accepted; points 2 and 3 superseded by 0007 | | [0003](0003-resolved-document.md) | Resolve all legal semantics once, into a format-neutral resolved document | Accepted | -| [0004](0004-html-first.md) | Output formats in order: text + HTML, then DOCX, then PDF | Accepted (DOCX numbering and the PDF engine still open) | +| [0004](0004-html-first.md) | Output formats in order: text + HTML, then DOCX, then PDF | Accepted (DOCX decisions superseded by 0008; the PDF engine still open) | | [0005](0005-locale-formatting.md) | Locale formatting with Babel (CLDR) | Accepted (implemented in 0.1.0) | | [0006](0006-styles-and-options.md) | Style templates are data; render options are separate | Accepted | | [0007](0007-one-parser-validator-model.md) | One parser: build from the validator's document model; markdown-it-py for inline text only | Accepted (supersedes part of 0002) | +| [0008](0008-docx-word-native.md) | DOCX with Word-native numbering and fields, written as OOXML directly | Accepted (supersedes the DOCX part of 0004) | ## Template diff --git a/docs/roadmap.md b/docs/roadmap.md index a21ef96..e6647e7 100644 --- a/docs/roadmap.md +++ b/docs/roadmap.md @@ -90,17 +90,54 @@ Rendering what other people send, and reporting problems where they are. No new self-contained - **Project.** CI also runs on Windows, and publishing to PyPI waits for CI -### v0.5 — DOCX - -- `docx` writer behind the `[docx]` extra -- To decide first: static numbers, or Word-native numbering with `REF` fields - ([ADR 0004](decisions/0004-html-first.md)) +### v0.5 — DOCX (implemented; released after the Word checks) + +A document lawyers can edit: in the .docx every number, cross-reference, the table of contents and +the page numbers are live Word features (numbering definitions and fields), and the renderer's +values appear only as the cached results of fields. Decided in +[ADR 0008](decisions/0008-docx-word-native.md), which also settles the question 0004 left open. + +- **The writer.** A `docx` writer in the base package, written with the standard library only (no `[docx]` + extra): the OOXML parts, styles generated from the style template, settings (compatibility mode 15, + `w:updateFields`), page setup, running headers and footers with `PAGE`/`NUMPAGES` fields, document + properties, and deterministic bytes. The body is every block and inline: headings, paragraphs, lists, quotes, + drafting notes, code, tables with a repeated header row, rules, links, defined terms, blanks and choices + (images as their alt text), then the front and back matter: the title block, a live `TOC` field with cached + entries and `PAGEREF` page numbers, the attachments and the signature blocks +- **The planner** (`resolve/docx_plan.py`): the render tree carries the numbering structure of every unit, and a pure + planning step maps it to Word numbering (the heading list, list families, paragraphs, qualified `5.1` labels on the + heading list's levels, alternatives that share a number as `numId 0` paragraphs with a `REF` field, restarts by + structure or by a fixed value), to bookmarks, and to the fields of every `{{ref:}}` (the designation cover), with the + cached results taken from a model of Word's counters. Where Word draws differently from the renderer, the render + reports a `render-docx-*` diagnostic (listed in [`CONFORMANCE.md`](../CONFORMANCE.md)) +- **Style settings.** `page.header` and `page.footer` (the HTML writer ignores them for now), + `docx.update_fields_on_open` and `docx.reference_text_switch`; settings Word cannot express are a `StyleError` + for DOCX only +- **Verification.** Every part validated against the ISO/IEC 29500 schemas (fetched in CI, not stored), well-formedness + fuzzing, determinism, and LibreOffice as the oracle through UNO (a `docx` CI job): the numbers and references + unit by unit with the cached results made wrong on purpose, whole documents against the plain-text writer's + output under three styles, and edit-then-update (a heading, an item, a numbered paragraph inserted, a clause + deleted, as a lawyer does). The manual Word checklist in `.github/PUBLISHING.md` covers what only Word shows +- **Behaviour changes this brings.** + - `-o -` writes to standard output for every format (it used to create a file named `-`). + - `.docx` is a writable extension; a DOCX with no `-o` goes to standard output when that is not a terminal, and is + never written to a terminal. `--fragment` with DOCX is a usage error. + - `render(..., format="docx")` returns the file as `RenderResult.binary` (bytes); `RenderResult.output` stays + `str` and is empty for it. `FORMATS` gains `docx`. + - `RenderTree` gains fields for the numbering structure (`formats`, and on sections, list items and + paragraphs `unit`, `number`, `parent_unit`, `shares_number_with`; `List.family`, `List.depth`, `List.format` and `List.start`; `CrossRef.target_unit`, + `section_unit`) and `version` and `effective_date` (plain text, filled whether or not the title block shows + them). HTML and plain-text output are byte-identical to 0.4 (the goldens are unchanged). + - New `render-docx-*` diagnostics for DOCX renders only. +- **Next.** Running the manual Word checklist, which finds what only Word shows; its findings are recorded in the + release notes. Whether the writer then claims §13.6 for DOCX is decided in `CONFORMANCE.md` ### v0.6 — PDF - Choose the engine ([ADR 0004](decisions/0004-html-first.md)). The leading option is WeasyPrint on the HTML writer; its print stylesheet already exists -- Running headers and footers, and page numbers, from the style's `page` settings +- Running headers and footers, and page numbers, from the style's `page` settings (`page.header` and + `page.footer` exist since the DOCX writer; the HTML writer's print stylesheet does not read them yet) ### Later — Full level diff --git a/docs/style-templates.md b/docs/style-templates.md index 47af8ff..6dd95f3 100644 --- a/docs/style-templates.md +++ b/docs/style-templates.md @@ -134,10 +134,19 @@ typography: headings: # per level 1–5 1: { size: 1.15em, weight: "700", italic: false, transform: uppercase, align: left } -page: { size: A4, margin: 25mm } # print stylesheet now; the PDF writer later +page: + size: A4 # HTML print stylesheet and DOCX page setup; the PDF writer later + margin: 25mm + header: { left: "", center: "", right: "" } # running header (DOCX; not yet HTML or PDF), see "Headers and footers" + footer: { left: "", center: "{page} / {pages}", right: "" } + html: max_width: 46rem extra_css: "" # raw CSS, appended to the stylesheet as written (see "Trusted input") + +docx: # the DOCX writer only (see "DOCX") + update_fields_on_open: true + reference_text_switch: documented # off | documented | guess: what a cross-reference may expect of Word's \t switch ``` ### Level formats @@ -181,6 +190,7 @@ just text. | `references.format` | `{designation}` | `{designation}`, or every reference reads the same | | `labels.condition` | `{condition}` | `{condition}`, as §15.8 requires the condition to be shown; an empty value is allowed and shows the bare condition | | `labels.attachment_file` | `{file}` | none | +| `page.header.*`, `page.footer.*` (left, center, right) | `{page}`, `{pages}`, `{title}`, `{version}`, `{date}` | none: an empty text shows nothing | | every other label | not checked: braces are text | none | ``` @@ -192,6 +202,99 @@ legaldown-render: Invalid style 'default': A `ref` is one piece of a designation, so it cannot show the whole `{path}`. A label left unset takes the built-in one for the document language, which always passes. +### Headers and footers + +`page.header` and `page.footer` are the running text at the top and bottom of every page, each as a +`left`, a `center` and a `right` text set at the left margin, in the middle and at the right margin. The +default footer is `{page} / {pages}` in the centre ("3 / 12"), and the default header is empty. A text +may use these fields, and any other `{name}` in braces is refused when the style loads: + +| Field | Shows | +|---|---| +| `{page}` | the page number | +| `{pages}` | the number of pages | +| `{title}` | the document's title | +| `{version}` | the frontmatter's `version`; empty when the document has none | +| `{date}` | the frontmatter's `effective_date`, formatted as `values.date` says for the formatting locale; empty when the document has none | + +`{version}` and `{date}` are the same text the title block shows for them (directives in them resolved +the same way, a valid date formatted as `values.date` says), whether or not `title_block` shows them +there. A text that is empty everywhere (the header by default) writes no header at all. + +**Where it applies.** The **DOCX** writer writes them: `{page}` and `{pages}` are `PAGE` and `NUMPAGES` +fields that Word and LibreOffice count while laying out the pages (a viewer that lays out no pages shows +the cached "1", so the default footer reads "1 / 1" there), and the other three fields are text. +A single text is aligned; two or three are separated by tab stops at the middle and the right margin. +The **HTML** and plain-text writers do not read them yet; the PDF writer will (v0.6). + +```yaml +page: + header: { left: "{title}", right: "Version {version}" } + footer: { center: "Page {page} of {pages}" } +``` + +### DOCX + +The DOCX writer reads the same presentation settings as the HTML writer and converts them to what +Word has (docs/decisions/0008-docx-word-native.md). A setting Word cannot express is a **style error +when the format is `docx`**, reported before anything is rendered and naming every such setting; HTML +keeps accepting the same style. + +| Setting | In the DOCX | +|---|---| +| `page.size` | the paper size of the page: `A3` to `A5`, `B4`, `B5`, `letter`, `legal`, `ledger`, with `landscape` or `portrait`; or one or two lengths; `auto` and an orientation alone are A4 | +| `page.margin` | one to four lengths (top, right, bottom, left, as in CSS); the header and footer sit half a margin from the edge, at most 1.27 cm | +| `typography.font_family` | the **first** family: `Georgia` for `'Georgia, "Times New Roman", serif'`; `serif`, `sans-serif` and `monospace` alone are Times New Roman, Arial and Courier New. Word falls back to its own substitute when the font is not installed | +| `typography.font_size`, `headings.N.size` | half-points (Word's unit for a font size, so a size is rounded to the nearest half-point: `11.3pt` is 11.5 pt); `headings.N.size` between 1 pt and 1638 pt, and `typography.font_size` between 1 pt and 1023 pt, because the title is 1.6 times the body | +| `typography.line_height` | a number is that many lines (`1.5`), `normal` is one line, a length is the least height of a line | +| `typography.color`, `typography.link_color` | `#rgb`, `#rrggbb` (an alpha part is dropped), `rgb()`, `hsl()`, or one of `black silver gray white maroon red purple fuchsia green lime olive yellow navy blue teal aqua orange`; any other colour name is an error, so write it as `#rrggbb` | +| `headings.N.weight` | bold from 600 up (`bold`, `bolder`), otherwise regular | + +Lengths: `mm`, `cm`, `in`, `pt`, `pc` and `px` (96 to the inch) are absolute. **`em` and `rem`** are +relative to `typography.font_size`, which is itself relative to 12 pt when it is written in them, as in a +browser. **`%`, `ch` and `ex`** depend on a layout or a font that Word does not give the renderer, so +each is an error for DOCX: `page.margin: 5%` raises "page.margin: uses '%', which a Word document cannot +express". Units are not case-sensitive (`25MM`). Word's limits are enforced too, as style errors that never +quote the value: a page edge of 7.2 pt to 1584 pt (22 inches), a margin of 0 to 1584 pt, margins that leave +at least 36 pt (0.5 inch) of text in each direction, a line height of at most 1584 pt (or 100 lines), and a +number of more than 32 digits is no length at all. `html.max_width` and `html.extra_css` are never read. + +The writer generates Word styles from the template (`Normal`, `heading 1` to `heading 5`, `Title`, +`Subtitle`, `Quote`, `toc 1` to `toc 5`, `Hyperlink`, `Table Grid`, and styles of its own for drafting +notes, code, defined terms, blanks, choices, failure markers and condition labels), so a change made to a +style in Word applies to every paragraph that uses it. + +| Setting | Meaning | +|---|---| +| `docx.update_fields_on_open` | `true` (the default) asks Word to update every field when it opens the file: the cross-references, the table of contents and its page numbers. Word asks the reader once ("This document contains fields that may refer to other files. Update the fields?"). The fields hold the values the renderer computed either way, so a viewer that never updates fields (Word Online, Quick Look, Outlook's preview) shows the document right, but the table of contents without page numbers. LibreOffice and Word Online ignore the setting. `false` writes no such request | +| `docx.reference_text_switch` | What a cross-reference may expect of Word's `\t` switch (`REF bookmark \n \t`), which leaves the words of a number out. A reference is built from `REF` fields so that it stays right when a clause is inserted; where the reference shows less than the number Word draws (`Section {designation}` for a number `Article I`, `čl. {designation}` for `Čl. 1`), the switch leaves the words out and the `references.format` text goes back in as literal text. LibreOffice ignores the switch (and shows the whole number there, which the render reports once as `render-docx-reference-word-only`). Microsoft documents it with one example (`Section 1.01` gives `1.01`: the words around the number go, a delimiter between its counters stays); what Word does with a delimiter in the text around the counters is not documented. **`documented`** (the default) uses the switch only where that example settles the result: words before the first counter or after the last go, a delimiter between counters stays (words between counters are not documented: only `guess` drops them), and a label with a delimiter around its number (`(a)`, `a)`, `1:`, `No. 1`) is not asked for it. **`off`** never uses it: a reference that needs it is fixed text, reported (`render-docx-reference`). **`guess`** also expects Word to leave out a period after the letters that lead a number (`No. 1`, `Čl. 1`) and words between counters (`Part 1 of 2` gives `12`): the checklist in `.github/PUBLISHING.md` says how to check what your Word does, and a reference that is wrong in Word is fixed by setting `documented` | + +Settings that were already part of the template gain a DOCX meaning: + +| Setting | In the DOCX | +|---|---| +| `contents.enabled`, `contents.depth` | A live `TOC \o "1-" \h \z \u` field after the title block, with the contents' entries as its cached result (`toc 1` to `toc 5` paragraphs, a dotted right tab at the text width, a `PAGEREF` page number Word fills in when it updates the field). The contents label is a `Contents Heading` paragraph: no outline level, no number, so never an entry | +| `contents.attachments` | The attachments heading and each attachment title are paragraphs of the styles `Attachments Heading` and `Attachment Title`, which carry an outline level, so Word's TOC lists them (`\u`); without it they are body text for the contents. At `contents.depth: 1` the titles are listed with the headings (the first level), not under the attachments heading | +| `attachments.separator` | `page-break`: a page break before the attachments part and between attachments (the paragraph property "page break before", so no empty paragraph or blank page is left); `rule`: a line above each attachment; `none`: nothing | +| `signatures.enabled` | Borderless tables with a signing line, kept together on one page as far as one page allows. They follow the last attachment, as in the HTML. A name, title, date or place row whose label is blanked (`labels.signature_*`) and whose value is empty is left out, as the plain-text writer leaves it out | +| `title_block.parties`, `title_block.effective_date`, `title_block.version` | The header rows and the parties, as borderless tables | + +The numbering and wording settings are the same as in the other formats; what they become in the file: + +| Setting | In the DOCX | +|---|---| +| `numbering.*` | The multilevel list the heading styles are linked to: `counter` is the level's number format, `label` its level text (`{n}` the level's own counter, `{path}` the counters of the levels above it joined as the label says). A label with a literal `%` before a digit or a counter cannot be written as a level text and is a style error for DOCX. With `scheme: none` the headings are not numbered | +| `enumeration.*`, `paragraphs.*` | The lists of items and the numbered paragraphs, one numbering definition for each kind. A label that shows `{section}` can be live only on the heading list's next level, and is otherwise written without it and reported (`render-docx-section-label`) | +| `references.format`, `numbering.levels[].ref`, `enumeration.levels[].ref` | The literal text around the `REF` fields of a cross-reference and between them; the cached result of the field is what Word draws. See `docx.reference_text_switch` | +| `definitions.style`, `definitions.term_style` | The character styles of a defining occurrence and of a `{{term:}}` reference | +| `typography.justify` | The justification of `Normal` | +| `locale`, `values.*`, `placeholders.*`, `template_view.*`, `labels.*` | As in the other formats: dates, amounts, blanks, choices and labels are written as text (they are not numbers Word can compute) | +| `html.max_width`, `html.extra_css` | Not read | + +The styles for this matter (`Title Block Text`, `Party Side`, `Party Name`, `Attachments Heading`, `Attachment Title`, +`Attachment File`, `Signature Side`, `Signature Text`, `Signature Line`) are Word styles like the others: a lawyer who +changes one in Word changes every paragraph that uses it. + ### Trusted input A style template, a `--set` value, and an `extends` path are the operator's, not the document @@ -231,7 +334,8 @@ with `RenderOptions(untrusted=True, document_root=...)` (`legaldown-render --unt - `html.extra_css` must be empty. - `values.date` is `short`, `medium`, `long` or `full`, not a pattern. - Every free-text setting (`name`, `labels.*`, `references.format`, `placeholders.blank`, - `template_view.*`, and the `label` and `ref` of `numbering`, `enumeration` and `paragraphs` levels) is + `template_view.*`, `page.header.*`, `page.footer.*`, and the `label` and `ref` of `numbering`, + `enumeration` and `paragraphs` levels) is at most 200 characters, since it is repeated wherever the output shows it. NUL and lone surrogates (which no output encoding can write) are refused in any text. - YAML text is at most 64 KiB, may not use aliases, and a number in it is at most 32 characters. @@ -253,7 +357,8 @@ how a firm adds a language. - Should styles be distributable as Python packages, discovered through entry points, so a firm can `pip install` its house style? -- The DOCX writer will need a `docx` section that maps onto Word styles in a reference document. - Its shape is decided with that writer. +- Should a DOCX start from a reference document (a `.dotx` or `.docx` with the firm's styles), as Pandoc + does? The `docx` section is deliberately small for now: the writer generates its styles from the + template, so a house style stays one YAML file. - Should `references.format` gain a generated prefix ("Section 4.2")? The word would have to come from `labels`, and it could double up with source text that already says "Section". diff --git a/pyproject.toml b/pyproject.toml index eed7a93..db62ad2 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -6,14 +6,14 @@ build-backend = "hatchling.build" name = "legaldown-render" # Single-sourced from src/legaldown_render/__init__.py. dynamic = ["version"] -description = "Reference renderer for the LegalDown legal document format: HTML and plain text, DOCX and PDF planned" +description = "Reference renderer for the LegalDown legal document format: HTML, plain text and DOCX (live Word numbering and fields), PDF planned" readme = "README.md" requires-python = ">=3.11" license = "MIT" license-files = ["LICENSE"] authors = [{ name = "ForLegalAI s.r.o." }] maintainers = [{ name = "ForLegalAI s.r.o." }] -keywords = ["legaldown", "legal", "contracts", "legaltech", "markdown", "renderer", "html"] +keywords = ["legaldown", "legal", "contracts", "legaltech", "markdown", "renderer", "html", "docx", "word"] classifiers = [ "Development Status :: 3 - Alpha", "Environment :: Console", @@ -40,7 +40,9 @@ dependencies = [ ] [project.optional-dependencies] -dev = ["pytest>=9.2", "ruff>=0.17", "build>=1.6"] +# lxml validates the DOCX parts against the OOXML schemas in the tests (tests/schemas/README.md). +# pytest-xdist is optional: `pytest -n auto` runs the suite on all cores (a third of the time). +dev = ["pytest>=9.2", "ruff>=0.17", "build>=1.6", "lxml>=6.1", "pytest-xdist>=3.8"] [project.scripts] legaldown-render = "legaldown_render.cli:main" @@ -74,6 +76,9 @@ addopts = "-ra --strict-markers" pythonpath = ["src"] markers = [ "conformance: renders the LegalDown specification's examples and fixtures (needs a checkout of the specification)", + "ooxml_schema: validates DOCX parts against the ISO/IEC 29500 schemas (needs lxml and the schemas: python tests/schemas/fetch.py)", + "libreoffice: opens DOCX files in LibreOffice and compares what it shows (needs soffice and python3-uno)", + "acceptance: the DOCX acceptance corpus (tests/test_docx_acceptance.py); left out of a run that names no marker (-m), unless LEGALDOWN_ACCEPTANCE is set or the file is named", ] # A deprecated legaldown-validator API fails the tests: the next minor # release removes it, and the dependency range admits only one minor version. diff --git a/src/legaldown_render/api.py b/src/legaldown_render/api.py index e26eb34..f585b73 100644 --- a/src/legaldown_render/api.py +++ b/src/legaldown_render/api.py @@ -25,16 +25,18 @@ from .diagnostics import by_line, key_line, repeats from .errors import DocumentError, OptionsError, RenderRefused, StyleError from .resolve import Formatter, parse_locale, resolve_tree +from .resolve.docx_plan import plan_docx from .style import Style, load_style +from .style.docx import DocxSettings, docx_settings from .tree import RenderTree, assert_resolved -from .writers import FORMATS, HtmlWriter, TextWriter +from .writers import FORMATS, DocxWriter, HtmlWriter, TextWriter @dataclass(frozen=True, slots=True) class RenderOptions: """Settings for one render job.""" - #: ``html`` or ``text``. + #: ``html``, ``text`` or ``docx``. format: str = "html" #: A built-in style name, a path to a YAML style template, a Style, or (see #: :func:`~.style.load_style`) a mapping or YAML text. @@ -100,6 +102,8 @@ class RenderOptions: class RenderResult: """What a render produced.""" + #: The rendered text: HTML or plain text. Empty for ``docx``, which is bytes + #: (:attr:`binary`), not text. output: str #: Core (validator) and Rendering diagnostics, each with its stable rule id. diagnostics: list[Diagnostic] @@ -107,6 +111,9 @@ class RenderResult: #: The resolved tree, for tools that want more than the output text. tree: RenderTree style: Style + #: The rendered file for a binary format (``docx``): the bytes of the whole + #: file. None for ``html`` and ``text``. + binary: bytes | None = None @property def ok(self) -> bool: @@ -148,6 +155,8 @@ def render(source: str, options: RenderOptions | None = None, /, **settings: Any if options.locale: overrides["locale"] = options.locale style = load_style(options.style, overrides=overrides, trusted=not options.untrusted) + # A setting Word cannot express is a StyleError before any rendering starts (docs/decisions/0006). + docx = docx_settings(style, trusted=not options.untrusted) if options.format == "docx" else None source = normalize_source(source) assembled: list[Diagnostic] = [] @@ -160,6 +169,15 @@ def render(source: str, options: RenderOptions | None = None, /, **settings: Any raise RenderRefused(diagnostics) tree, extra = _resolve(document, result, style, base, options.untrusted) + assert_resolved(tree) + writer = _writer(options, style, docx) + plan = None + if isinstance(writer, DocxWriter): + # What Word will draw differently from the other formats (docs/decisions/0008) is the planner's to report; its + # findings are the renderer's, and are ordered with them. + plan, planned = plan_docx(tree, file=document.filename, text_switch=writer.settings.reference_text_switch, + text_width=writer.settings.page.text_width) + extra = [*extra, *planned] # The renderer's findings that the validator's or the assembly's already # name (the same rule, level and message), wherever they are, are not repeated; # the rest follow in the order of their lines. @@ -167,9 +185,13 @@ def render(source: str, options: RenderOptions | None = None, /, **settings: Any if options.strict and any(d.level == "error" for d in diagnostics): raise RenderRefused(diagnostics) - assert_resolved(tree) - output = _writer(options, style).write(tree) - return RenderResult(output, diagnostics, options.format, tree, style) + if plan is not None: + written: str | bytes = writer.write(tree, plan) + else: + written = writer.write(tree) + if isinstance(written, bytes): + return RenderResult("", diagnostics, options.format, tree, style, binary=written) + return RenderResult(written, diagnostics, options.format, tree, style) def render_file(path: str | Path, options: RenderOptions | None = None, /, **settings: Any) -> RenderResult: @@ -289,9 +311,13 @@ def _resolve(document: Document, result: ValidationResult, style: Style, return resolved, diagnostics + found -def _writer(options: RenderOptions, style: Style) -> HtmlWriter | TextWriter: +def _writer(options: RenderOptions, style: Style, docx: DocxSettings | None) -> HtmlWriter | TextWriter | DocxWriter: if options.format == "text": return TextWriter(style.attachments.separator) from . import __version__ - return HtmlWriter(style, standalone=options.standalone, generator=f"legaldown-render {__version__}") + generator = f"legaldown-render {__version__}" + if options.format == "docx": + assert docx is not None + return DocxWriter(style, generator=generator, settings=docx) + return HtmlWriter(style, standalone=options.standalone, generator=generator) diff --git a/src/legaldown_render/cli.py b/src/legaldown_render/cli.py index d03be48..806f690 100644 --- a/src/legaldown_render/cli.py +++ b/src/legaldown_render/cli.py @@ -8,7 +8,8 @@ Documents and output are always UTF-8 (the HTML says ````), never the console's or the locale's encoding, and output has LF line endings on -every platform. +every platform. A DOCX file is bytes: it is written to a file, or to standard output's +binary interface (``-o -``, or no ``-o`` when standard output is not a terminal), never to a terminal. """ from __future__ import annotations @@ -28,11 +29,14 @@ from .writers import FORMATS, file_extension, format_for_path #: Extensions (:func:`file_extension`, so no dot) of document formats this -#: renderer does not write. An output file named so (``x.docx``, ``.pdf``, +#: renderer does not write. An output file named so (``x.pdf``, ``.odt``, #: ``x.pdf.``) and no ``--format`` is a mistake to refuse: HTML under that name #: would mislead. Any other extension (``contract.v2``, ``report.2025-10-27``), #: or none, is just a name and keeps the HTML default. -_UNWRITTEN_EXTENSIONS = frozenset({"docx", "doc", "odt", "rtf", "pdf", "md", "markdown", "epub", "tex"}) +_UNWRITTEN_EXTENSIONS = frozenset({"doc", "odt", "rtf", "pdf", "md", "markdown", "epub", "tex"}) + +#: ``--output`` that names standard output instead of a file. +_STDOUT = "-" def _safe(text: str) -> str: @@ -70,17 +74,19 @@ def _setting(text: str) -> tuple[str, Any]: def _parser() -> argparse.ArgumentParser: parser = _Parser( prog="legaldown-render", - description="Render a LegalDown document to HTML or plain text.", + description="Render a LegalDown document to HTML, plain text or DOCX.", epilog="Style settings: --style picks a template (built-in: " + ", ".join(builtin_styles()) + ", or a .yaml file); --set overrides one setting, e.g. --set numbering.scheme=mixed. " "See every setting with --print-style.", ) parser.add_argument("input", nargs="?", help="the .lgd file to render, or - for standard input") - parser.add_argument("-o", "--output", help="output file (default: standard output). Without --format, " - ".html/.htm write HTML and .txt/.text plain text; any other name, with or without an " - "extension, writes HTML, except the extension of a format not written (.docx, .doc, " - ".odt, .rtf, .pdf, .md, .markdown, .epub, .tex), which needs --format") + parser.add_argument("-o", "--output", help="output file (default: standard output, except for DOCX, which " + "is binary and is refused on a terminal). Without --format, .html/.htm write HTML, " + ".txt/.text plain text and .docx a Word document; any other name, with or without an " + "extension, writes HTML, except the extension of a format not written (.doc, " + ".odt, .rtf, .pdf, .md, .markdown, .epub, .tex), which needs --format. -o - writes " + "to standard output") parser.add_argument("-f", "--format", choices=sorted(FORMATS), help="output format (default: html, or text for --output *.txt/*.text)") parser.add_argument("-s", "--style", help="style template: a built-in name or a path to a .yaml file") @@ -176,8 +182,16 @@ def main(argv: list[str] | None = None) -> int: file_name = Path(args.output).name output_format = format_for_path(file_name) if not output_format and file_extension(file_name) in _UNWRITTEN_EXTENSIONS: - parser.error(f"cannot tell the output format from '{printable(args.output)}'; use --format html or text") + parser.error(f"cannot tell the output format from '{printable(args.output)}'; " + "use --format html, text or docx") output_format = output_format or "html" + to_stdout = not args.output or args.output == _STDOUT + if output_format == "docx": + if args.fragment: + parser.error("--fragment applies to HTML only: a DOCX file is always the whole document") + if to_stdout and getattr(sys.stdout, "isatty", lambda: False)(): + parser.error("refusing to write a DOCX file to a terminal; redirect standard output or name a file " + "with -o FILE.docx") name = "" if args.input == "-" else args.input try: source = _read_source(args.input) @@ -224,14 +238,36 @@ def main(argv: list[str] | None = None) -> int: return 70 _report(name, result.diagnostics, quiet=args.quiet) - if args.output: - try: - Path(args.output).write_text(result.output, encoding="utf-8", newline="\n") - except OSError as error: - _complain(f"legaldown-render: cannot write {args.output}: {error.strerror or error}") - return 1 - else: + if result.binary is not None: + return _write_binary(result.binary, args.output) + if to_stdout: sys.stdout.write(result.output) + return 0 + try: + Path(args.output).write_text(result.output, encoding="utf-8", newline="\n") + except OSError as error: + _complain(f"legaldown-render: cannot write {args.output}: {error.strerror or error}") + return 1 + return 0 + + +def _write_binary(data: bytes, output: str) -> int: + """Write *data*, a rendered binary file, to the file *output*, or (with ``-``, or no + output at all) to standard output's bytes, never through its text encoding.""" + if not output or output == _STDOUT: + buffer = getattr(sys.stdout, "buffer", None) + if buffer is None: + _complain("legaldown-render: cannot write to standard output: it has no binary interface") + return 1 + sys.stdout.flush() + buffer.write(data) + buffer.flush() + return 0 + try: + Path(output).write_bytes(data) + except OSError as error: + _complain(f"legaldown-render: cannot write {output}: {error.strerror or error}") + return 1 return 0 diff --git a/src/legaldown_render/resolve/docx_bookmarks.py b/src/legaldown_render/resolve/docx_bookmarks.py new file mode 100644 index 0000000..5e23e4b --- /dev/null +++ b/src/legaldown_render/resolve/docx_bookmarks.py @@ -0,0 +1,68 @@ +"""Bookmark names for the DOCX plan (docs/decisions/0008, §6.3). + +Word reaches a paragraph by a bookmark: a ``REF`` field, an internal link and a table of contents entry all name one. +The planner gives every anchor of the tree a bookmark name; the writer allocates the ids (``writers/docx/ooxml.py`` +checks that a name is valid, at most 40 characters and unique ignoring case, which is how Word compares them). + +* A name is a prefix for what it marks and the anchor: ``ld_`` for a section, list item or paragraph, ``lddef_`` for a + defined term, ``ldatt_`` for an attachment, with ``-`` and ``:`` written as ``_`` (an identifier holds only + letters, digits and ``-``, so nothing else is lost). It is used only if it is a valid name of at most 40 + characters, nothing but the dashes was changed, and no name taken before is the same ignoring case. +* Otherwise, and for a unit a field needs that has no anchor, the name is hidden: ``_Ld``, ``k`` counting from 1 in + the order the names are given out. Word does not list names that start with ``_`` in its Bookmark dialog. +* The table of contents uses ``_Toc`` and nine digits, from 100000001, as Word does. +""" +from __future__ import annotations + +import re + +#: The longest bookmark name Word accepts, in characters. +MAX_LENGTH = 40 + +_LOSSLESS = re.compile(r"[A-Za-z0-9_:\-]+") +_VALID = re.compile(r"[A-Za-z_][A-Za-z0-9_]*") + +#: The name of the attachments heading (the anchor ``ld:attachments``). +ATTACHMENTS_HEADING = "_LdAttachments" + + +class BookmarkNames: + """Hands out bookmark names, each unique ignoring case. Give names in a fixed order and the result is the same + on every run.""" + + def __init__(self) -> None: + self._taken: set[str] = set() + self._hidden = 0 + self._toc = 100000000 + + def _take(self, name: str) -> str | None: + if len(name) > MAX_LENGTH or not _VALID.fullmatch(name) or name.lower() in self._taken: + return None + self._taken.add(name.lower()) + return name + + def anchor(self, prefix: str, anchor: str) -> str: + """The name for *anchor* with *prefix* (``ld_``, ``lddef_``, ``ldatt_``), or a hidden one when the anchor + cannot be written as one.""" + name = self._take(prefix + re.sub(r"[-:]", "_", anchor)) if _LOSSLESS.fullmatch(anchor) else None + return name if name is not None else self.hidden() + + def fixed(self, name: str) -> str: + """*name*, a name of the renderer's own (taken so that no other can be the same).""" + taken = self._take(name) + assert taken is not None, name + return taken + + def hidden(self) -> str: + """The next hidden name, ``_Ld1``, ``_Ld2``, ...""" + self._hidden += 1 + name = self._take(f"_Ld{self._hidden}") + assert name is not None + return name + + def toc(self) -> str: + """The next name for a table of contents entry, ``_Toc100000001``, ...""" + self._toc += 1 + name = self._take(f"_Toc{self._toc}") + assert name is not None + return name diff --git a/src/legaldown_render/resolve/docx_cover.py b/src/legaldown_render/resolve/docx_cover.py new file mode 100644 index 0000000..50a17b6 --- /dev/null +++ b/src/legaldown_render/resolve/docx_cover.py @@ -0,0 +1,190 @@ +"""The designation cover: how a cross-reference is built from Word's ``REF`` fields (docs/decisions/0008, §6.3, §13.3). + +A reference shows the designation of its target: the number of the section, the item or the paragraph written +with the ``ref`` templates of every unit on the way (``1.1.1``, ``I.A.1.a(i)``, ``1(b)(ii)``). In Word every part of +it that Word can compute must be a field, so that it stays right when a clause is inserted: ``REF \\n`` +returns the number Word draws for the bookmarked paragraph (its whole ``lvlText``, trailing periods dropped), and +nothing else Word could know may be written as literal text, but for the part of a number that belongs to a heading +level the document skips (§4.1: a Core error the renderer still numbers), which no unit holds. This module finds, for +one designation, the combination of such fields and literal delimiters that reads exactly like it. + +Everything here works on token strings (:mod:`.docx_numbering`): the text of a reference as the renderer wrote it (the +designation inside the text of ``references.format``, so that a field may stand for the words of the format when the +number it returns holds the same words: ``Article {designation}`` and the label ``Article {n}`` are one plain ``REF``), +and what each available field returns, are lists of :class:`~.docx_numbering.Lit` (text that is not a counter) and +:class:`~.docx_numbering.Count` (the counter of one owner, as formatted); a :class:`Title` is the text of a heading +under the ``none`` scheme (§13.3), the one thing besides counters a designation can hold that Word keeps up to date +(``REF \\h`` returns the bookmarked text). The designation is cut into **atoms** (one per character of +literal text, one per counter or title), a field is a sequence of atoms too, and a field **fits** at a position when +every atom of it equals the designation's there: characters equal, counters and titles of the same owner (their +text may differ: the field is then structurally right and its cached result is what Word will compute, which the +planner reports). A literal may stand only for characters of literal text, never for a counter or a title. Among the +covers the one with the fewest fields wins, then the fewest fields using ``\\t``, then the fewest literal +characters; of equal ones the first in the order the fields were given. + +``\\t`` (Word only; LibreOffice ignores it) suppresses the text of a number that is not a counter or a delimiter. +Microsoft documents it with one example (``Section 1.01`` gives ``1.01``): the words around the number go, the delimiter +between its counters stays. What it returns where the text around the counters holds a delimiter (``(a)``, ``a)``, +``No. 1``) is not documented, so it is modelled by the style's ``docx.reference_text_switch`` (:func:`suppressed`): +``off`` never, ``documented`` only for what the documentation settles, ``guess`` also for a period after the +letters that lead a number. +""" +from __future__ import annotations + +from collections.abc import Sequence +from typing import Literal, NamedTuple + +from .docx_numbering import Count, Ghost, Lit, Token + +#: The characters Word keeps when ``\t`` drops the text of a number. Between two counters they stay; in the text before +#: the first counter or after the last, what Word does with them is not documented (:func:`suppressed`). +DELIMITERS = frozenset(".,;:()[]/-–") + +#: What ``docx.reference_text_switch`` can say of ``\t``. +TextSwitch = Literal["off", "documented", "guess"] + + +class Title(NamedTuple): + """The text of the heading of *owner* (a unit id), as the ``none`` scheme designates a section with it.""" + + owner: object + text: str + + +Shown = Token | Title + + +class Field(NamedTuple): + """A ``REF`` field that can stand in a designation: the bookmark of unit *owner*, the switches that decide what it + returns (*number*: ``\\n``, else the bookmarked text; *suppress_text*: ``\\t``), and what it returns as tokens + (Word's value, which is its cached result).""" + + owner: str + number: bool + suppress_text: bool + tokens: tuple[Shown, ...] + + +def text_of(tokens: Sequence[Shown]) -> str: + return "".join(token.text for token in tokens) + + +def atoms(tokens: Sequence[Shown]) -> list[tuple[str, object]]: + """*tokens* as atoms: ``("d", character)`` for each character of literal text, ``("c", owner)`` for a counter and + ``("t", owner)`` for a title. The counter of a heading level the document skips (a :class:`~.docx_numbering.Ghost`: + no unit holds it, so no field can return it) is literal text, which is all that can stand for it.""" + out: list[tuple[str, object]] = [] + for token in tokens: + if isinstance(token, Lit) or (isinstance(token, Count) and isinstance(token.owner, Ghost)): + out.extend(("d", character) for character in token.text) + elif isinstance(token, Count): + out.append(("c", token.owner)) + else: + out.append(("t", token.owner)) + return out + + +def suppressed(tokens: Sequence[Shown], mode: TextSwitch = "documented") -> tuple[Shown, ...] | None: + """What ``REF \\n \\t`` returns for a field that returns *tokens* with ``\\n``, or None where the switch is not used + (the *mode* is ``off``, there is no counter, or what Word returns is not known). + + * Text before the first counter and after the last that holds no delimiter is dropped (``Article I`` gives ``I``, + ``§ 3`` gives ``3``). With a delimiter in it (``(a)``, ``a)``, ``1:``) Word's result is not documented: None. + Mode ``guess`` also drops leading text that has a period after letters in it (``No. 1``, ``Čl. 1`` give ``1``). + * Text between two counters that is nothing but delimiters stays (Microsoft's example: ``Section 1.01`` gives + ``1.01``). Words between them, with no delimiter, are dropped only in mode ``guess``: the documentation does not + say what Word does with them. A mixture is None. + """ + if mode == "off": + return None + counters = [index for index, token in enumerate(tokens) if not isinstance(token, Lit)] + if not counters: + return None + kept: list[Shown] = [] + for index, token in enumerate(tokens): + if not isinstance(token, Lit): + kept.append(token) + continue + found = {character for character in token.text if character in DELIMITERS} + if index < counters[0] or index > counters[-1]: + period_after_letters = (mode == "guess" and index < counters[0] and found == {"."} + and any(character.isalpha() for character in token.text)) + if found and not period_after_letters: + return None + elif found: + if not all(character in DELIMITERS for character in token.text): + return None + kept.append(token) + elif mode != "guess": + return None + return tuple(kept) + + +def _fits(wanted: Sequence[tuple[str, object]], at: int, field: Sequence[tuple[str, object]]) -> bool: + """True if the atoms of a field equal those of the designation from *at* on. Atoms are equal when they are the + same kind and owner or character: the *text* of a counter is not compared (that is the cached result's business).""" + return at + len(field) <= len(wanted) and all(wanted[at + k] == atom for k, atom in enumerate(field)) + + +class _Cost(NamedTuple): + fields: int + suppressing: int + literal: int + + +def cover(designation: Sequence[Shown], fields: Sequence[Field]) -> list[str | Field] | None: + """The cheapest cover of *designation* by *fields* and literal text, as literal strings and fields in order, + or None when there is none (a counter or a title no field stands for, or text around a counter that no field + drops).""" + wanted = atoms(designation) + shapes = [atoms(field.tokens) for field in fields] + size = len(wanted) + best: list[tuple[_Cost, list[str | Field]] | None] = [None] * (size + 1) + best[size] = (_Cost(0, 0, 0), []) + for at in range(size - 1, -1, -1): + options: list[tuple[_Cost, list[str | Field]]] = [] + later = best[at + 1] + if wanted[at][0] == "d" and later is not None: + cost, rest = later + options.append((_Cost(cost.fields, cost.suppressing, cost.literal + 1), [str(wanted[at][1]), *rest])) + for field, shape in zip(fields, shapes, strict=True): + if shape and shape[0] == wanted[at] and _fits(wanted, at, shape): + onward = best[at + len(shape)] + if onward is not None: + cost, rest = onward + options.append((_Cost(cost.fields + 1, cost.suppressing + field.suppress_text, cost.literal), + [field, *rest])) + if options: + best[at] = min(options, key=lambda option: option[0]) + if best[0] is None: + return None + merged: list[str | Field] = [] + for piece in best[0][1]: + if merged and isinstance(piece, str) and isinstance(merged[-1], str): + merged[-1] += piece + else: + merged.append(piece) + return merged + + +def uncovered(designation: Sequence[Shown], fields: Sequence[Field]) -> int | None: + """Why :func:`cover` finds none: the index (among the atoms of *designation*) of the last atom that a cover of what + comes before it reaches and that neither a literal nor a field can stand for; None when there is a cover.""" + wanted = atoms(designation) + shapes = [atoms(field.tokens) for field in fields] + size = len(wanted) + onward = [False] * (size + 1) + onward[size] = True + steps: list[list[int]] = [[] for _ in range(size)] + for at in range(size - 1, -1, -1): + if wanted[at][0] == "d": + steps[at].append(at + 1) + steps[at] += [at + len(shape) for shape in shapes if shape and shape[0] == wanted[at] and _fits(wanted, at, shape)] + onward[at] = any(onward[after] for after in steps[at]) + if onward[0]: + return None + reached = {0} + for at in range(size): + if at in reached: + reached.update(steps[at]) + return max(at for at in reached if at < size and not onward[at]) diff --git a/src/legaldown_render/resolve/docx_numbering.py b/src/legaldown_render/resolve/docx_numbering.py new file mode 100644 index 0000000..d99d937 --- /dev/null +++ b/src/legaldown_render/resolve/docx_numbering.py @@ -0,0 +1,340 @@ +"""Word's list numbering, as the DOCX planner needs it (docs/decisions/0008, §13.1, §13.2). + +Two things live here, both pure: + +* **Token strings.** The renderer's labels and designations, and Word's expansion of an ``lvlText``, are written as + lists of :class:`Lit` (text that is not a counter) and :class:`Count` (the counter of one owner, as formatted). The + functions :func:`fill_tokens` and :func:`extend_tokens` are :func:`.numbering.fill` and :func:`.numbering.extend` on + tokens, so that a template filled here reads exactly as the resolver filled it, with the owner of every counter kept. + :func:`level_text` derives the ``lvlText`` of a level from a label template. +* **The simulator** (:class:`WordNumbering`): what Word's numbering gives each paragraph of a list, from the + abstractNum and num definitions alone. The planner uses it to place every unit and to check each one against the + number the resolver gave it, and its results are the cached results of the fields Word will recompute. + +The simulator models Word's list semantics (docs/architecture.md, "The DOCX planner"): counters belong to the +abstractNum and are shared by all its ``w:num``; a ``startOverride`` restarts the named level at the first use of that +num; a paragraph restarts every deeper level; ``numId 0`` paragraphs count nothing (the planner never steps them). +""" +from __future__ import annotations + +import re +from collections.abc import Sequence +from typing import TYPE_CHECKING, Literal, NamedTuple + +from ..errors import InternalError +from .numbering import _FIELD_RE + +if TYPE_CHECKING: + from .docx_plan import LevelDef, Num + +# --------------------------------------------------------------------------- +# Counters +# --------------------------------------------------------------------------- + +#: The renderer's counters (``style.model.Counter``) and the ``numFmt`` that writes each in Word. +NUM_FMT: dict[str, str] = { + "decimal": "decimal", + "lower-alpha": "lowerLetter", + "upper-alpha": "upperLetter", + "lower-roman": "lowerRoman", + "upper-roman": "upperRoman", +} + +_ROMAN = ( + (900, "cm"), (500, "d"), (400, "cd"), (100, "c"), (90, "xc"), (50, "l"), (40, "xl"), + (10, "x"), (9, "ix"), (5, "v"), (4, "iv"), (1, "i"), +) + + +def word_counter(value: int, num_fmt: str) -> str: + """*value* as Word writes it in *num_fmt*. + + Letters repeat after 26 (``aa``, ``bb``, ... ``zz``, ``aaa``), as in ECMA-376 and Word's documentation; the + renderer counts ``aa``, ``ab`` (§13.2), so the two differ from 28 on. Roman numerals are the usual form below + 1000 and ``m`` repeated above, so that 4000 reads ``mmmm`` where the renderer writes digits. A value below 1 is + ``0`` for ``decimal`` and empty otherwise.""" + if value < 1: + return "0" if num_fmt == "decimal" else "" + match num_fmt: + case "lowerLetter" | "upperLetter": + letters = chr(ord("a") + (value - 1) % 26) * ((value - 1) // 26 + 1) + return letters.upper() if num_fmt == "upperLetter" else letters + case "lowerRoman" | "upperRoman": + numeral, rest = "m" * (value // 1000), value % 1000 + for amount, symbol in _ROMAN: + while rest >= amount: + numeral += symbol + rest -= amount + return numeral.upper() if num_fmt == "upperRoman" else numeral + case _: + return str(value) + + +#: How wide a counter of each ``numFmt`` is taken to be, to size a hanging indent. +SAMPLE_COUNTER: dict[str, str] = { + "decimal": "88", "lowerLetter": "mm", "upperLetter": "MM", "lowerRoman": "viii", "upperRoman": "VIII", +} + +# --------------------------------------------------------------------------- +# Token strings +# --------------------------------------------------------------------------- + + +class Lit(NamedTuple): + """Text that is not a counter: delimiters, or words of a label.""" + + text: str + + +class Count(NamedTuple): + """The counter of *owner* (a unit id, a :class:`Ghost`, or a level number in a template), as formatted.""" + + owner: object + text: str + + +class Ghost(NamedTuple): + """The counter of a skipped heading level: *level* (from 0) was never used before *unit* needed it.""" + + unit: str + level: int + + +Token = Lit | Count + + +def fill_tokens(template: str, *, n: Sequence[Token], path: Sequence[Token] = (), + section: Sequence[Token] | None = None) -> list[Token]: + """:func:`.numbering.fill` on tokens: *template* with ``{n}``, ``{path}`` and ``{section}`` replaced by token + lists, other text (and other braces) kept as :class:`Lit`. *section* given and empty (a preamble item) drops + ``{section}`` and ONE ``.`` after it; None means "not given" (headings), and ``{section}`` is then empty.""" + if section is not None and not section: + template = re.sub(r"\{section\}\.", "", template) + values: dict[str, Sequence[Token]] = {"n": n, "path": path, "section": section or ()} + out: list[Token] = [] + position = 0 + for match in _FIELD_RE.finditer(template): + if match.start() > position: + out.append(Lit(template[position:match.start()])) + out.extend(values[match.group(1)]) + position = match.end() + if position < len(template): + out.append(Lit(template[position:])) + return out + + +def extend_tokens(designation: Sequence[Token], piece: Sequence[Token], *, textual: bool) -> list[Token]: + """:func:`.numbering.extend` on tokens: *designation* followed by *piece*; the piece loses leading dots when + nothing precedes it or under the ``none`` scheme, where a space goes before it.""" + piece = list(piece) + if (not designation or textual) and piece and isinstance(piece[0], Lit): + piece[0] = Lit(piece[0].text.lstrip(".")) + if not designation: + return [token for token in piece if token != Lit("")] + if textual: + return [*designation, Lit(" "), *(token for token in piece if token != Lit(""))] + return [*designation, *piece] + + +def text_of(tokens: Sequence[Token]) -> str: + return "".join(token.text for token in tokens) + + +# --------------------------------------------------------------------------- +# lvlText +# --------------------------------------------------------------------------- + +#: A level text as parts: text, and the number of the level (from 0) a ``%k`` stands for. +LevelParts = tuple[str | int, ...] + +_PLACEHOLDER = re.compile(r"%([1-9])") + + +def level_text(label: str, refs: Sequence[str], level: int, *, section: Sequence[Token] | None, + textual: bool = False) -> LevelParts: + """The ``lvlText`` of level *level* of a list, from its ``label`` template, as parts. + + Only counters of the list's own levels can be shown (Word's ``%k`` names a level of the same abstractNum), so + ``{n}`` is this level's counter and ``{path}`` the designation over levels ``0..level``, each with its own ``ref`` + template (*refs*, one per level). *section* is ``None`` for headings (``{section}`` is empty) and ``()`` for the + foreign-free form of a list item or paragraph, the one a preamble item has: ``{section}`` and the dot after it + are dropped, which is the documented fallback where a label needs a section number Word cannot show there.""" + designation: list[Token] = [] + for j in range(level + 1): + piece = fill_tokens(refs[j], n=[Count(j, "")], section=section) + designation = extend_tokens(designation, piece, textual=textual) + tokens = fill_tokens(label, n=[Count(level, "")], path=designation, section=section) + parts: list[str | int] = [] + for token in tokens: + if isinstance(token, Count): + parts.append(token.owner) # type: ignore[arg-type] + elif parts and isinstance(parts[-1], str): + parts[-1] += token.text + else: + parts.append(token.text) + return tuple(parts) + + +def level_text_problem(parts: LevelParts) -> str | None: + """Why *parts* cannot be written as an ``lvlText``, or None. A literal ``%`` before a digit, or before a + counter, cannot be told from a placeholder; this happens where the pieces of a label are put together (a ref + ending in ``%`` before another that starts with a counter), not only inside one template.""" + text = "".join(f"%{part + 1}" if isinstance(part, int) else part for part in parts) + before_counter = any(isinstance(after, int) and isinstance(part, str) and part.endswith("%") + for part, after in zip(parts, parts[1:], strict=False)) + if before_counter or len(_PLACEHOLDER.findall(text)) != sum(isinstance(part, int) for part in parts): + return "a '%' meets a digit or a counter" + return None + + +def lvl_text(parts: LevelParts) -> str: + """*parts* in OOXML form: ``%k`` for level ``k-1``. Style validation (:func:`.docx_plan.docx_style_errors`) + refuses parts with a :func:`level_text_problem` before rendering, so one reaching here is a bug.""" + if level_text_problem(parts) is not None: + raise InternalError("a label that Word's level text cannot write reached the DOCX planner. This is a bug; " + "style validation refuses it.") + return "".join(f"%{part + 1}" if isinstance(part, int) else part for part in parts) + + +_SECTION = "section" + + +def shows_section(label: str, refs: Sequence[str], level: int, *, textual: bool = False) -> bool: + """True if the label of level *level* of a list in a section shows a counter of the section: it uses + ``{section}``, or ``{path}`` of a designation that starts from the section's. Judged on the filled-in label + (a counter of the section is in it), not on how the template is spelled.""" + section: list[Token] = [Count(_SECTION, "")] + designation: list[Token] = list(section) + for j in range(level + 1): + designation = extend_tokens(designation, fill_tokens(refs[j], n=[Count(j, "")], section=section), + textual=textual) + tokens = fill_tokens(label, n=[Count(level, "")], path=designation, section=section) + return any(isinstance(token, Count) and token.owner == _SECTION for token in tokens) + + +def split_level_text(text: str) -> list[str | int]: + """An ``lvlText`` as literal strings and level numbers (from 0), as :func:`lvl_text` writes it.""" + parts: list[str | int] = [] + position = 0 + for match in _PLACEHOLDER.finditer(text): + if match.start() > position: + parts.append(text[position:match.start()]) + parts.append(int(match.group(1)) - 1) + position = match.end() + if position < len(text): + parts.append(text[position:]) + return parts + + +# --------------------------------------------------------------------------- +# The simulator +# --------------------------------------------------------------------------- + + +class Step(NamedTuple): + """What Word gives one numbered paragraph.""" + + #: The counter of the paragraph's own level. + count: int + #: The level's own counter as written (``a``, ``IV``). + own: str + #: The full ``lvlText`` expansion, trailing period kept (``1.``, ``(a)``). + label: str + #: What ``REF \n`` returns: the label minus its trailing periods. + word_ref: str + #: The label as tokens, each counter with the unit (or ghost) that holds it. + tokens: tuple[Token, ...] + #: The counter of the paragraph's own level as LibreOffice counts it (-1: not modelled): it counts a level that + #: nothing numbered before a deeper one as its start value and goes on from it, where Word shows one less and + #: leaves it unused, so the next paragraph of that level is 2 there and 1 in Word. + lo_count: int = -1 + + +class WordNumbering: + """Word's counters for a set of abstractNums and nums. Register each with :meth:`add_abstract` and + :meth:`add_num` (an abstractNum's levels are read when a paragraph is stepped, so a level may be defined + after the abstractNum is added), then :meth:`step` paragraphs in document order. + + *skipped* is the model of a level that was never used before a deeper one: ``"lo"`` shows its start value and + counts on (what the renderer does for a skipped heading level, and LibreOffice shows); ``"word"`` shows + ``start - 1`` and does not advance it, which is what Word is expected to do. It is the default of every + abstractNum and can be set for each in :meth:`add_abstract`.""" + + def __init__(self, *, skipped: Literal["lo", "word"] = "lo") -> None: + self.skipped = skipped + self._skipped: dict[int, Literal["lo", "word"]] = {} + self._levels: dict[int, Sequence[LevelDef]] = {} + self._nums: dict[int, Num] = {} + self._count: dict[int, list[int]] = {} + self._lo: dict[int, list[int]] = {} + self._holder: dict[int, list[object | None]] = {} + self._used: set[tuple[int, int]] = set() + + def add_abstract(self, abstract_id: int, levels: Sequence[LevelDef], *, + skipped: Literal["lo", "word"] | None = None) -> None: + self._levels[abstract_id] = levels + self._skipped[abstract_id] = skipped or self.skipped + self._count[abstract_id] = [0] * 9 + self._lo[abstract_id] = [0] * 9 + self._holder[abstract_id] = [None] * 9 + + def add_num(self, num: Num) -> None: + self._nums[num.id] = num + + def _overrides(self, num_id: int) -> dict[int, int]: + return dict(self._nums[num_id].start_overrides) + + def peek(self, num_id: int, ilvl: int) -> int: + """The number the next paragraph at (*num_id*, *ilvl*) would get.""" + num = self._nums[num_id] + overrides = self._overrides(num_id) + if ilvl in overrides and (num_id, ilvl) not in self._used: + return overrides[ilvl] + return self._count[num.abstract_id][ilvl] + 1 + + def current(self, num_id: int, ilvl: int) -> tuple[int, object | None]: + """The counter of the level now, and the unit that last counted it (None if unused).""" + abstract_id = self._nums[num_id].abstract_id + return self._count[abstract_id][ilvl], self._holder[abstract_id][ilvl] + + def step(self, num_id: int, ilvl: int, owner: object) -> Step: + """Number the next paragraph at (*num_id*, *ilvl*), which *owner* (a unit id) is. ``numId`` 0 (no + numbering) counts nothing and restarts nothing; ``REF \\n`` of such a paragraph returns ``0``.""" + if num_id == 0: + return Step(0, "", "", "0", ()) + num = self._nums[num_id] + levels = self._levels[num.abstract_id] + count, holder = self._count[num.abstract_id], self._holder[num.abstract_id] + lo = self._lo[num.abstract_id] + overrides = self._overrides(num_id) + if ilvl in overrides and (num_id, ilvl) not in self._used: + self._used.add((num_id, ilvl)) + count[ilvl] = lo[ilvl] = overrides[ilvl] - 1 + count[ilvl] += 1 + lo[ilvl] += 1 + holder[ilvl] = owner + for deeper in range(ilvl + 1, 9): + count[deeper], holder[deeper] = 0, None + lo[deeper] = 0 + for above in range(ilvl): + if lo[above] == 0: + lo[above] = levels[above].start + skipped = self._skipped[num.abstract_id] + for above in range(ilvl): + if count[above] == 0: + holder[above] = Ghost(str(owner), above) + count[above] = levels[above].start if skipped == "lo" else levels[above].start - 1 + tokens: list[Token] = [] + for part in split_level_text(levels[ilvl].lvl_text): + if isinstance(part, int): + tokens.append(Count(holder[part], word_counter(count[part], levels[part].num_fmt))) + else: + tokens.append(Lit(part)) + if skipped == "word": + # Never-used levels show start - 1 and stay unused: the next heading of that level starts at `start`. + for above in range(ilvl): + if isinstance(holder[above], Ghost): + count[above], holder[above] = 0, None + label = text_of(tokens) + return Step(count[ilvl], word_counter(count[ilvl], levels[ilvl].num_fmt), label, + label.rstrip("."), tuple(tokens), lo[ilvl]) diff --git a/src/legaldown_render/resolve/docx_plan.py b/src/legaldown_render/resolve/docx_plan.py new file mode 100644 index 0000000..6b6ad28 --- /dev/null +++ b/src/legaldown_render/resolve/docx_plan.py @@ -0,0 +1,1768 @@ +"""The DOCX planner: from the render tree to the numbering Word will draw (docs/decisions/0008). + +``plan_docx(tree)`` is the one place that knows how LegalDown numbering becomes Word numbering; the DOCX writer +serialises its :class:`DocxPlan` and decides nothing (docs/decisions/0003). It is a pure function of the resolved +tree: the numbering structure the resolver put on every section, list item and top-level paragraph (``unit``, +``number``, ``parent_unit``, ``List.family``, ``RenderTree.formats``; §13.1, §13.2) is mapped to abstractNums, nums +and, for each unit, the (numId, ilvl) of its paragraph, and every placement is checked against the number the +resolver gave the unit, by simulating Word's counters (:mod:`.docx_numbering`). + +A plan holds the numbering definitions and the numbering of every unit, the bookmark of every anchor of the tree +(``anchors``, ``UnitPlan.bookmarks``; :mod:`.docx_bookmarks`), the fields of every cross-reference (``refs``: the +renderer's designation of the target covered by ``REF`` fields and the delimiters between them, with Word's value as +each field's cached result; :mod:`.docx_cover`) and the entries of the table of contents (``toc``, +``UnitPlan.toc_bookmark``), with the diagnostics for what Word will not show as the renderer does. +A label that needs the section's number is numbered on the heading abstractNum's level after the section's, and a list +nested in such an item on its reserved levels (§13.2); a later alternative that shares a number (§15.8) is a ``numId 0`` +paragraph whose number is a field of the unit that holds it (``UnitPlan.alt``); a unit Word would count off the +resolver's is restarted at a fixed value (``UnitPlan.ind`` is set for the units that carry an indent of their own). +:func:`docx_style_errors` is the part that runs on every DOCX render, before it (ADR 0006). + +**Data model.** Frozen data only; the planner assigns bookmark *names*, never ids (the writer allocates those); +names are valid Word bookmark names of at most 40 characters and unique ignoring case. All sequences are tuples, all +mappings are plain dicts built once in document order and never changed afterwards. +""" +from __future__ import annotations + +import re +from collections.abc import Callable, Mapping, Sequence +from dataclasses import dataclass, field, replace +from typing import Literal, NamedTuple + +from legaldown import Diagnostic + +from ..diagnostics import by_line +from ..style.model import LevelFormat, Style +from ..tree import ( + Blank, + Choice, + Code, + CrossRef, + DefinedTerm, + DraftingNote, + Emphasis, + FailureMarker, + HardBreak, + Image, + Inline, + Link, + List, + ListItem, + NumberingFormats, + Paragraph, + Quote, + RenderTree, + Section, + SoftBreak, + Strong, + TermRef, + Text, + Value, + has_visible_content, + iter_inlines, + iter_located_body_inlines, + iter_tree_inlines, + plain_text, +) +from ..urls import displayed_url, printable +from .docx_bookmarks import ATTACHMENTS_HEADING, BookmarkNames +from .docx_cover import DELIMITERS, Field, Shown, TextSwitch, Title, atoms, cover, suppressed, text_of, uncovered +from .docx_numbering import ( + NUM_FMT, + SAMPLE_COUNTER, + Count, + Ghost, + LevelParts, + Lit, + Step, + Token, + WordNumbering, + extend_tokens, + fill_tokens, + level_text, + level_text_problem, + lvl_text, + shows_section, + split_level_text, + word_counter, +) +from .numbering import format_counter, formats_of + +# --------------------------------------------------------------------------- +# The plan (frozen data) +# --------------------------------------------------------------------------- + + +@dataclass(frozen=True, slots=True) +class LevelDef: + """One ``w:lvl`` of an abstractNum.""" + + ilvl: int + #: ``decimal``, ``lowerLetter``, ``upperLetter``, ``lowerRoman``, ``upperRoman`` or ``bullet``. + num_fmt: str + #: In OOXML form (``%1.%2``, ``(%3)``, ``Article %1``, ``•``): ``%k`` is the counter of level ``k-1``. + lvl_text: str + start: int = 1 + #: ``Heading1``..``Heading5``, only on levels 0-4 of the heading abstractNum. + p_style: str | None = None + suff: Literal["tab", "space"] = "tab" + #: The level's own ``w:ind`` in twips. On the heading abstractNum this serves the heading; a unit that sits on + #: the level at another nesting depth carries its own indent (:attr:`UnitPlan.ind`). + left: int = 0 + hanging: int = 0 + + +@dataclass(frozen=True, slots=True) +class AbstractNum: + id: int # 0.., in order of first use (headings first) + #: ``detached``: the enumerated levels with each level's own counter alone, for a list nested where the enumerated list + #: would have to show the counter of a unit that is not on it (§13.2). ``enumerated+1`` to ``enumerated+3`` (and + #: ``renumber+k``, ``bullet+k``): Word has nine levels, so a list nested deeper takes the next abstractNum of its + #: family, whose level *j* is the family's level of depth ``9k + j`` with its own counter alone. + kind: Literal["headings", "enumerated", "detached", "renumber", "paragraphs", "bullet", + "enumerated+1", "enumerated+2", "enumerated+3", "renumber+1", "renumber+2", "renumber+3", + "bullet+1", "bullet+2", "bullet+3"] + #: Eight hex digits, deterministic (``0x4C440000 + id``). Equal in every LegalDown document, so Word may + #: merge lists on paste between two of them (cosmetic). + nsid: str + multi_level: Literal["multilevel", "hybridMultilevel"] + #: Always nine levels. + levels: tuple[LevelDef, ...] + + +@dataclass(frozen=True, slots=True) +class Num: + """One ``w:num``. ``nums[0]`` is the heading num (numId 1) only when headings are numbered.""" + + id: int # from 1 (numId 0 means "no numbering" and is never a Num) + abstract_id: int + #: ``(ilvl, value)``: the first number of the list at that level; the deeper levels restart by themselves when + #: that level counts. + start_overrides: tuple[tuple[int, int], ...] = () + + +@dataclass(frozen=True, slots=True) +class Literal_: + """Text of a reference or number that is not a counter: delimiters and the words of ``references.format``.""" + + text: str + + +@dataclass(frozen=True, slots=True) +class FieldSeg: + """A ``REF`` field with its cached result. Instruction: ``REF `` + ``\\n`` if *number* + ``\\t`` if + *suppress_text* + ``\\h`` if *hyperlink*. Without *number* it returns the bookmarked text (heading text under + the ``none`` scheme).""" + + bookmark: str + number: bool + suppress_text: bool + hyperlink: bool + #: What Word computes (the simulator's value). + cached: str + #: What LibreOffice computes (it ignores ``\t``); equal to *cached* unless *suppress_text*. For tests only. + cached_lo: str + + +Segment = Literal_ | FieldSeg + + +@dataclass(frozen=True, slots=True) +class AltNumber: + """The number of a later alternative that shares a number (§15.8): its paragraph has ``numId 0`` and starts + with these segments (the label as a cover of the holder's number: ``REF holder \\n \\h`` and the stripped + period), then a tab or space. Its indent is :attr:`UnitPlan.ind`.""" + + segments: tuple[Segment, ...] + suff: Literal["tab", "space"] + + +@dataclass(frozen=True, slots=True) +class UnitPlan: + """What the paragraph of one tree unit (section, list item, top-level paragraph) needs, key: ``unit``.""" + + unit: str + kind: Literal["section", "item", "paragraph"] + #: None: the writer writes no ``w:numPr`` (a heading takes its style's; nothing else is unnumbered). + num_id: int | None + ilvl: int | None + #: True: the writer MUST write ``w:numPr`` (``numId`` 0 for an alternative). + explicit: bool + #: ``(left, hanging)`` in twips, written as ``w:ind`` on the paragraph, for a unit whose nesting depth differs + #: from its level's natural depth (a nested list on the heading abstractNum, an alternative); else None. + ind: tuple[int, int] | None + alt: AltNumber | None + #: Word's number text for the unit (the full ``lvlText`` expansion, trailing period kept: ``1.``, ``(a)``). + label: str + #: What ``REF \n`` returns (trailing period dropped): the cached text of the unit's own field. + word_ref: str + #: Word's counter value (None for an alternative, a bullet and an unnumbered unit). + count: int | None + #: Why the unit is not on the plain path: ``resync`` (restarted at a fixed value), ``alternative`` (``numId`` 0), + #: ``reserved-level`` (a list nested in an item on the heading list, on one of its reserved levels), + #: ``restart-family`` / ``restart-section`` / ``restart-alternative`` / ``restart-depth`` (a nested list in a list of + #: another kind, in an item whose number carries the section, in an alternative, past Word's nine levels: a num of + #: its own that restarts by a fixed setting), ``label-fallback`` (the label is written without a number Word cannot + #: show there), ``clamped`` (nested past :data:`MAX_DEPTH`, in any family), ``skipped-level``. + cause: tuple[str, ...] + #: Names to wrap around the paragraph's title/first paragraph content: the bookmark of the unit's anchor when it + #: has one, else a hidden one when a reference field points at it, else none. + bookmarks: tuple[str, ...] + #: ``_Toc`` name of a heading listed in the contents. + toc_bookmark: str | None + #: True: *bookmarks* are points at the start of the paragraph, not around its text (no field returns that + #: text, and a field must never stand inside the bookmark it points at). + collapsed: bool = False + #: Targets of the ``{{ref:}}`` in this heading's title that are written as their text, not as fields, because + #: their fields would lead back into the title (see :meth:`_Planner.cyclic_titles`). + literal_refs: tuple[str, ...] = () + + +@dataclass(frozen=True, slots=True) +class RefPlan: + """The fields of every ``{{ref:}}`` to one unit (``CrossRef.target_unit``), wherever it stands.""" + + #: ``references.format`` literals around the cover of the designation. + segments: tuple[Segment, ...] + #: ``concat(cached) == CrossRef.text``. + exact: bool + #: No cover exists, so the reference is the renderer's text with no field: right when the file is opened, but + #: fixed (an edit in Word does not change it). The plan reports each (``render-docx-reference``). + fixed: bool = False + #: The text of the counters of heading levels the document skips (a Core error, ``heading-skip``, §4.1) that the cover + #: writes as literal text, since no unit holds those counters and so no field returns them; the rest is fields. + #: Empty when the cover has none. Reported like a fixed reference (``render-docx-reference``). + ghost: str = "" + + +@dataclass(frozen=True, slots=True) +class TocEntryPlan: + #: The ``ContentsEntry`` it is made from (``ContentsEntry.unit``: the section's unit, None for attachments). + unit: str | None + level: int # 1..5, the TOCn style + #: Word's number text of the heading incl. the trailing period; ``""`` for attachments. + number: str + sep: Literal["\t", " ", ""] + title: tuple[Inline, ...] # ContentsEntry.title, already unlinked + condition_label: str | None + bookmark: str # the ``_Toc`` name the entry's hyperlink and PAGEREF point to + cached_page: str = "" # always empty + + +@dataclass(frozen=True, slots=True) +class DocxPlan: + abstract_nums: tuple[AbstractNum, ...] + nums: tuple[Num, ...] + #: The num the heading styles' ``numPr`` points to; None under the ``none`` scheme. + heading_num_id: int | None + units: Mapping[str, UnitPlan] = field(default_factory=dict) + #: Every output anchor of the tree (units, ``DefinedTerm.anchor``, ``AttachmentPart.anchor``, + #: ``attachments_anchor``) -> bookmark name. + anchors: Mapping[str, str] = field(default_factory=dict) + #: ``CrossRef.target_unit`` -> its fields. + refs: Mapping[str, RefPlan] = field(default_factory=dict) + toc: tuple[TocEntryPlan, ...] = () + + +# --------------------------------------------------------------------------- +# Style checks that belong before rendering (ADR 0006) +# --------------------------------------------------------------------------- + +#: The longest ``lvlText`` Word accepts (taken from Word's limit of 255 characters on a level's text). +MAX_LVL_TEXT = 255 + +#: A literal ``%`` that ``lvlText`` could not tell from a placeholder once the label is filled in: before a digit, +#: or before a field that becomes ``%k``. +_AMBIGUOUS_PERCENT = re.compile(r"%(?=[0-9{])") + +#: The setting a kind of abstractNum is made from, for a message. +_SETTING = {"headings": "numbering.levels", "enumerated": "enumeration.levels", "detached": "enumeration.levels", + "renumber": "enumeration.ordered", "paragraphs": "paragraphs.format"} + + +def docx_style_errors(style: Style) -> list[str]: + """What in *style* Word's numbering cannot express, one message per problem naming the setting; empty when + the style is fine for DOCX. + + Word's ``lvlText`` has no way to write a ``%`` before a digit or a counter, whether it stands in one template + or meets a counter where a label and the refs it shows are put together, and a level's text is at most + :data:`MAX_LVL_TEXT` characters. These are problems of configuration, so a DOCX render calls this before + rendering (``style.docx.docx_settings``) and raises :class:`~legaldown_render.errors.StyleError` (ADR 0006).""" + problems: list[str] = [] + formats = formats_of(style) + templates: list[tuple[str, LevelFormat]] = [] + if formats.headings: + templates += [(f"numbering.levels[{i}]", f) for i, f in enumerate(style.numbering.levels[:5])] + if formats.enumerated: + templates += [(f"enumeration.levels[{i}]", f) for i, f in enumerate(style.enumeration.levels)] + if formats.paragraphs is not None: + templates.append(("paragraphs.format", formats.paragraphs)) + for path, level in templates: + for name, template in (("label", level.label), ("ref", level.ref)): + if _AMBIGUOUS_PERCENT.search(template): + problems.append(f"{path}.{name}: a '%' before a digit or a field cannot be written in a Word " + f"numbering level (DOCX); write it without the '%'") + if problems: + return problems + unwritable: list[tuple[str, int]] = [] + definitions = _definitions(formats, unwritable) + first_level: dict[str, int] = {} + for kind, level in unwritable: + first_level.setdefault(kind, level) + for kind, level in first_level.items(): + problems.append(f"{_SETTING[kind]}: a '%' meets a digit or a counter where the label and the refs of level " + f"{level + 1} are put together, which a Word numbering level cannot write (DOCX)") + for kind, levels in definitions.items(): + for level in levels: + if len(level.lvl_text) > MAX_LVL_TEXT: + problems.append(f"{_SETTING.get(kind.split('+')[0], kind)}: the numbering level {level.ilvl + 1} is longer than " + f"{MAX_LVL_TEXT} characters in Word (DOCX): shorten its label or the refs it shows") + return problems + + +# --------------------------------------------------------------------------- +# Level definitions +# --------------------------------------------------------------------------- + +_BULLETS = ("•", "–", "◦") +#: Twips: one centimetre, the step of the indent of items. +_STEP = 567 +#: Twips per character of a heading's widest label. +_CHARACTER = 113 +#: Word's levels in one list, and how many lists of them a nesting of items can use: a list nested deeper than nine +#: levels goes on in the next abstractNum of its family (``enumerated+1`` ...), up to this many levels in all; past them +#: the last level is used again and the lists restart by fixed values. +_LEVELS = 9 +MAX_DEPTH = 4 * _LEVELS +#: The width of the text of an A4 page with 25 mm margins, in twips: what the indents of nested lists are limited by when +#: the planner is not told the page (:func:`plan_docx`). +DEFAULT_TEXT_WIDTH = 9072 +#: The part of the text width an item's indent may reach: a list nested very deeply stops stepping in there and stays +#: readable on any page (A5 included). +_MAX_INDENT = 0.6 + + +def max_indent(text_width: int) -> int: + """The widest indent, in twips, a nested list steps in to on a page whose text is *text_width* twips wide.""" + return max(_STEP, int(text_width * _MAX_INDENT)) + + +def _level(ilvl: int, num_fmt: str, parts: LevelParts, *, p_style: str | None = None, + widths: Mapping[int, str] | None = None, unwritable: list[tuple[str, int]] | None = None, + kind: str = "", depth: int | None = None, max_left: int | None = None) -> LevelDef: + """One level. Headings (*widths*: the ``numFmt`` of each level a ``%k`` names) get an indent wide enough for + their widest label; items step one centimetre per level of nesting (*depth*, the level's own by default: a level of + a list that goes on past Word's nine levels is nested deeper). A label with a word in it (``Article {n}``) is + followed by a space, any other by a tab. With *unwritable*, parts a level text cannot write are noted there + (and written as nothing) instead of raising.""" + words = any(isinstance(part, str) and any(c.isalpha() for c in part) for part in parts) + depth = ilvl if depth is None else depth + if widths is None: + left, hanging = min(_STEP * (depth + 1), max_left or max_indent(DEFAULT_TEXT_WIDTH)), _STEP + else: + sample = sum(len(part) if isinstance(part, str) else len(SAMPLE_COUNTER[widths[part]]) for part in parts) + left = hanging = max(_STEP, _CHARACTER * (sample + 2)) + if unwritable is not None and level_text_problem(parts) is not None: + unwritable.append((kind, depth)) + parts = () + return LevelDef(ilvl, num_fmt, lvl_text(parts), p_style=p_style, suff="space" if words else "tab", + left=left, hanging=hanging) + + +def _neutral(ilvl: int) -> LevelDef: + """A level nothing uses: its own counter, decimal.""" + return LevelDef(ilvl, "decimal", f"%{ilvl + 1}", left=_STEP * (ilvl + 1), hanging=_STEP) + + +def _own(formats: NumberingFormats, ilvl: int, depth: int, textual: bool, unwritable: list[tuple[str, int]] | None, + max_left: int | None = None) -> LevelDef: + """Level *ilvl* of an enumerated list nested *depth* levels deep, with its own counter alone (a ``{path}`` shows + its own ``ref`` and nothing of the units it is nested in): the levels of the detached list and of the lists that go + on past Word's nine levels, where no unit of the list holds the levels above.""" + levels = formats.enumerated + here = levels[depth % len(levels)] + return _level(ilvl, NUM_FMT[here.counter], + level_text(here.label, [""] * ilvl + [here.ref], ilvl, section=(), textual=textual), + unwritable=unwritable, kind="enumerated", depth=depth, max_left=max_left) + + +def _definitions(formats: NumberingFormats, unwritable: list[tuple[str, int]] | None = None, + max_left: int | None = None) -> dict[str, list[LevelDef]]: + """The nine levels of every abstractNum *formats* needs, by kind. A label that needs a number of the section + (``{section}``, or ``{path}`` of an item or paragraph in one) is written without it, as for a preamble item: + Word shows only counters of the list itself in a level.""" + textual = formats.scheme == "none" + out: dict[str, list[LevelDef]] = {} + if formats.headings: + refs = [f.ref for f in formats.headings] + fmts = {j: NUM_FMT[f.counter] for j, f in enumerate(formats.headings)} + levels = [_level(k, fmts[k], level_text(f.label, refs, k, section=None), p_style=f"Heading{k + 1}", + widths=fmts, unwritable=unwritable, kind="headings") + for k, f in enumerate(formats.headings)] + out["headings"] = [*levels, *(_neutral(k) for k in range(5, 9))] + if formats.enumerated: + size = len(formats.enumerated) + refs = [formats.enumerated[j % size].ref for j in range(9)] + out["enumerated"] = [ + _level(d, NUM_FMT[formats.enumerated[d % size].counter], + level_text(formats.enumerated[d % size].label, refs, d, section=(), textual=textual), + unwritable=unwritable, kind="enumerated", max_left=max_left) + for d in range(9)] + # The same levels with nothing but their own counter (a ``{path}`` shows its own ``ref`` alone): where a list is + # nested in a unit that is not on the enumerated list, Word has no counter of the enclosing units to show. + out["detached"] = [_own(formats, d, d, textual, unwritable, max_left) for d in range(_LEVELS)] + for k in range(1, MAX_DEPTH // _LEVELS): + out[f"enumerated+{k}"] = [_own(formats, d, _LEVELS * k + d, textual, unwritable, max_left) + for d in range(_LEVELS)] + if formats.renumber is not None: + f = formats.renumber + for k in range(MAX_DEPTH // _LEVELS): + out["renumber" if k == 0 else f"renumber+{k}"] = [ + _level(d, NUM_FMT[f.counter], level_text(f.label, [f.ref] * 9, d, section=(), textual=textual), + unwritable=unwritable, kind="renumber", depth=_LEVELS * k + d, max_left=max_left) + for d in range(_LEVELS)] + if formats.paragraphs is not None: + f = formats.paragraphs + out["paragraphs"] = [ + _level(0, NUM_FMT[f.counter], level_text(f.label, [f.ref], 0, section=(), textual=textual), + unwritable=unwritable, kind="paragraphs"), + *(_neutral(k) for k in range(1, 9))] + limit = max_left or max_indent(DEFAULT_TEXT_WIDTH) + for k in range(MAX_DEPTH // _LEVELS): + out["bullet" if k == 0 else f"bullet+{k}"] = [ + LevelDef(d, "bullet", _BULLETS[(_LEVELS * k + d) % 3], left=min(_STEP * (_LEVELS * k + d + 1), limit), + hanging=_STEP) for d in range(_LEVELS)] + return out + + +def _reads(level: LevelDef, levels: Sequence[LevelDef]) -> str: + """*level* as it reads with every counter at 1, for a message.""" + return "".join(word_counter(1, levels[part].num_fmt) if isinstance(part, int) else part + for part in split_level_text(level.lvl_text)) + + +# --------------------------------------------------------------------------- +# The planner +# --------------------------------------------------------------------------- + + +def _shown(inlines: tuple[Inline, ...], ref_text: Callable[[CrossRef], str] | None = None) -> str: + """What the runs of *inlines* show in the document (``writers/docx/runs.py``): the text of each, a hard break as a + line break, a blank with its prompt. A heading's bookmark returns this, which is not always the heading's + designation (white space is collapsed in that). A cross-reference shows what *ref_text* says its fields give (its + own text by default).""" + out: list[str] = [] + for inline in inlines: + match inline: + case CrossRef() if ref_text is not None: + out.append(ref_text(inline)) + case Text(text=text) | Code(text=text) | CrossRef(text=text) | TermRef(text=text) | Value(text=text) \ + | FailureMarker(text=text): + out.append(text) + case SoftBreak(): + out.append(" ") + case HardBreak(): + out.append("\n") + case Blank(text=text, prompt=prompt): + out.append(text + (f" ({prompt})" if prompt else "")) + case Choice(options=options, separator=separator): + out.append("[" + separator.join(options) + "]") + case Image(children=children): + out.append(plain_text(children)) + case Link(href=href, children=children): + out.append(_shown(children, ref_text) if has_visible_content(children) else displayed_url(href)) + case Emphasis(children=children) | Strong(children=children) | DefinedTerm(children=children): + out.append(_shown(children, ref_text)) + return "".join(out) + + +def _components(successors: Mapping[str, Sequence[str]]) -> tuple[dict[str, int], list[list[str]]]: + """The strongly connected components of a directed graph (Tarjan, iterative, so that a chain of any length is + fine): the component of each node, and the components in the order they are completed, which is every node's + successors first.""" + index: dict[str, int] = {} + low: dict[str, int] = {} + on_stack: set[str] = set() + stack: list[str] = [] + component: dict[str, int] = {} + order: list[list[str]] = [] + nodes = list(dict.fromkeys([*successors, *(n for out in successors.values() for n in out)])) + for root in nodes: + if root in index: + continue + index[root] = low[root] = len(index) + stack.append(root) + on_stack.add(root) + work = [(root, iter(successors.get(root, ())))] + while work: + node, following = work[-1] + for nxt in following: + if nxt not in index: + index[nxt] = low[nxt] = len(index) + stack.append(nxt) + on_stack.add(nxt) + work.append((nxt, iter(successors.get(nxt, ())))) + break + if nxt in on_stack: + low[node] = min(low[node], index[nxt]) + else: + work.pop() + if work: + parent = work[-1][0] + low[parent] = min(low[parent], low[node]) + if low[node] == index[node]: + members = [] + while True: + member = stack.pop() + on_stack.discard(member) + component[member] = len(order) + members.append(member) + if member == node: + break + order.append(members) + return component, order + + +_Unit = Section | ListItem | Paragraph + + +class _Abstract: + """An abstractNum being built: the simulator reads its levels while the plan grows.""" + + def __init__(self, abstract_id: int, kind: str, levels: list[LevelDef]) -> None: + self.id, self.kind, self.levels = abstract_id, kind, levels + + +class _Placed(NamedTuple): + """Where a list item is numbered, for the lists nested in it.""" + + num: Num + abstract: _Abstract + + +class _Paragraphs(NamedTuple): + """Where the numbered paragraphs of one section go: their num, the abstractNum and level it is on, and whether + their label is the foreign-free one for want of a place where Word can show the section.""" + + num: Num + abstract: _Abstract + ilvl: int + fallback: bool + + +class _Alternative: + """A later alternative that shares a number, for the sections under it: they restart in a num of their own, made + when the first of them is numbered.""" + + def __init__(self, level: int) -> None: + self.level = level + self.num: Num | None = None + + +#: The owner of the counter of the unit a level text is being made for. +_SELF = object() + +#: What the list of a nested list that restarts by a fixed setting is called (``UnitPlan.cause``), and what is reported +#: about it (information: Word's own "Restart at 1", which cannot be off when the document is opened). +_RESTARTS: dict[str, tuple[tuple[str, ...], str]] = { + "restart-family": (("restart-family",), "A list nested in a list of another kind restarts by a fixed setting (Restart " + "at 1), not by structure: an item added above it in Word will not restart it."), + "restart-section": (("restart-section",), "A list nested in an item whose number carries the section's cannot " + "restart by structure here: it restarts by a fixed setting (Restart at 1), and an item " + "added above it in Word will not restart it."), + "restart-alternative": (("restart-alternative",), "A list nested in an alternative that shares a number restarts by a " + "fixed setting (Restart at 1), not by structure: an item added above it in Word will not " + "restart it."), + "resync": (("resync",), "A list that Word would continue from another list in the same item restarts by a fixed " + "setting (Restart at 1), not by structure."), + "restart-depth": (("restart-depth",), "A list nested past Word's nine levels goes on in a list of its own, which " + "restarts by a fixed setting (Restart at 1), not by structure: an item added above it in Word will " + "not restart it."), +} + + +class _Reference(NamedTuple): + """The ``{{ref:}}`` to one unit that the plan builds fields for: the unit, its section, what the renderer shows and + the first line it stands at (None outside the body).""" + + target: str + section: str | None + text: str + line: int | None + + +class _Built(NamedTuple): + """What the plan makes of one reference: its pieces (literal text and fields), whether they read exactly as the + renderer's text, what LibreOffice shows for them when that is not what Word shows (a field with ``\\t``), else + None, and whether it is fixed text for want of a cover.""" + + pieces: list[str | Field] + exact: bool + lo_text: str | None + fixed: bool + ghost: str = "" + + +class _Finding(NamedTuple): + """A reference that is fixed text (*kind* ``fixed``, *reason* why), has the number of a skipped heading level as + fixed text (``ghost``) or reads differently in Word (``differs``). References with the same *kind* and *group* + are one finding.""" + + kind: str + reason: str + found: _Reference + shown: str + group: str = "" + + +class _Drift: + """A run of units of one level that Word counts off from the resolver by the same amount.""" + + def __init__(self, node: _Unit, kind: str, step: Step, delta: int) -> None: + self.node, self.kind, self.step, self.delta, self.more = node, kind, step, delta, 0 + + +class _Planner: + def __init__(self, tree: RenderTree, file: str | None, text_switch: TextSwitch = "documented", + text_width: int = DEFAULT_TEXT_WIDTH) -> None: + self.tree, self.file, self.text_switch = tree, file or "", text_switch + self.max_left = max_indent(text_width) + self.formats = tree.formats + self.textual = tree.formats.scheme == "none" + self.diagnostics: list[Diagnostic] = [] + self.found: dict[tuple[str, str, str], int] = {} + self.sim = WordNumbering() + self.abstracts: list[_Abstract] = [] + self.nums: list[Num] = [] + self.units: dict[str, UnitPlan] = {} + self.parent: dict[str, str | None] = {} + self.section_level: dict[str, int] = {} + self.placed: dict[str, _Placed] = {} + self.paragraph_nums: dict[str, _Paragraphs] = {} + #: A later alternative that shares a number -> the unit it repeats and the suffix of its level; and every such + #: unit (items and paragraphs too) -> the unit that holds its number, for designations and fields. + self.alts: dict[str, tuple[str, Literal["tab", "space"]]] = {} + self.canon: dict[str, str] = {} + self.alternatives: list[_Alternative] = [] + #: The levels of the heading abstractNum a unit other than a heading has defined, and the units on it that + #: Word would number off the resolver's (reported once). + self.claimed: set[int] = set() + self.collisions: list[tuple[_Unit, str]] = [] + #: Items and paragraphs, and where the text of each on the heading list starts (twips), for its nested items. + self.item_units: set[str] = set() + self.text_left: dict[str, int] = {} + #: The level after the sections' fifth is kept for units of a level-5 section when the document has one and + #: a label that needs the section. + self.reserve5 = False + self.flagged: set[str] = set() + self.reported: set[tuple[str, object]] = set() + #: The labels Word draws differently for want of counters it cannot show, by what they have in common, as + #: (line, message) of each place; reported once for each group (:meth:`report_section_labels`). + self.label_findings: dict[tuple[object, ...], list[tuple[int | None, str]]] = {} + #: The units of lists whose number LibreOffice reads higher than Word does (:attr:`.docx_numbering.Step.lo_count`). + self.lo_counts: list[tuple[_Unit, Step]] = [] + self.drifts: list[_Drift] = [] + self.open_drift: dict[tuple[int, int], _Drift] = {} + self.heading_num: Num | None = None + self.bullet_nums: dict[str, Num] = {} + self.depth_reported = False + self.definitions = _definitions(self.formats, max_left=self.max_left) + self.by_kind: dict[str, _Abstract] = {} + #: Every unit's node, in document order; what Word numbers each as (:class:`Step`); and its designation as + #: the resolver wrote it, as tokens (§13.1-§13.3), each counter with the unit that holds it. + self.nodes: dict[str, _Unit] = {} + self.steps: dict[str, Step] = {} + self.desig: dict[str, list[Shown]] = {} + #: The parts of the number of the open section, one per level: the unit that holds it (a skipped level has a + #: :class:`Ghost`) and its value. + self.owners: list[tuple[object, int]] = [] + #: What a heading's bookmark returns (``REF bookmark \\h``): its title as the runs show it. + self.heading_text: dict[str, str] = {} + self.unreferable: list[_Finding] = [] + + # -- registry ------------------------------------------------------------------- + + def abstract(self, kind: str) -> _Abstract: + """The abstractNum of *kind*, made on first use (so ids follow the order of use). Only headings can skip a + level (the renderer counts it as 1, as LibreOffice does); in any other list a level that nothing numbers + is read as Word reads it, 0, so that a label that needs it is reported.""" + found = self.by_kind.get(kind) + if found is None: + found = _Abstract(len(self.abstracts), kind, self.definitions[kind]) + self.abstracts.append(found) + self.by_kind[kind] = found + self.sim.add_abstract(found.id, found.levels, skipped="lo" if kind == "headings" else "word") + return found + + def new_num(self, abstract: _Abstract, ilvl: int | None = None, first: int = 1) -> Num: + """A num on *abstract*; with *ilvl*, its list starts there at *first* (the deeper levels restart by + themselves when that level counts).""" + num = Num(len(self.nums) + 1, abstract.id, () if ilvl is None else ((ilvl, first),)) + self.nums.append(num) + self.sim.add_num(num) + return num + + def report(self, rule: str, level: str, message: str, line: int | None) -> None: + """Record a finding; the same one (rule, level, message) again is one, at the lowest line.""" + key = (rule, level, message) + index = self.found.get(key) + found = Diagnostic(rule=rule, level=level, message=message, line=line, file=self.file) + if index is None: + self.found[key] = len(self.diagnostics) + self.diagnostics.append(found) + else: + known = self.diagnostics[index] + if line is not None and (known.line is None or line < known.line): + self.diagnostics[index] = found + + # -- references, bookmarks and the table of contents ----------------------------- + + def words(self, unit: str) -> tuple[Shown, ...]: + """What ``REF \\n`` returns, as tokens: the number Word draws for the unit without its trailing periods + (:attr:`Step.word_ref` with the unit that holds each counter).""" + tokens = list(self.steps[unit].tokens) + while tokens and isinstance(tokens[-1], Lit): + stripped = tokens[-1].text.rstrip(".") + if stripped == tokens[-1].text: + break + if stripped: + tokens[-1] = Lit(stripped) + break + tokens.pop() + return tuple(tokens) + + def fields(self, designation: Sequence[Shown]) -> list[Field]: + """The fields that can stand for parts of *designation*: for each unit that holds a counter of it, ``REF \\n`` + of its bookmark, and ``REF \\n \\t`` where that leaves the counters bare (:func:`.docx_cover.suppressed`); for a + heading under the ``none`` scheme the bookmarked text. A later alternative takes its number from the unit that + holds it and is never a field's target (§15.8).""" + fields: list[Field] = [] + seen: set[str] = set() + for token in designation: + owner = None if isinstance(token, Lit) else token.owner + if not isinstance(owner, str) or owner in seen or owner not in self.units: + continue + seen.add(owner) + if self.units[owner].num_id == 0 or self.units[owner].alt is not None: + continue + if isinstance(token, Title): + fields.append(Field(owner, False, False, (Title(owner, self.heading_text.get(owner, token.text)),))) + elif owner in self.steps: + words = self.words(owner) + fields.append(Field(owner, True, False, words)) + bare = suppressed(words, self.text_switch) + if bare is not None and atoms(bare) != atoms(words): + fields.append(Field(owner, True, True, bare)) + return fields + + def why_uncovered(self, designation: Sequence[Shown], fields: Sequence[Field]) -> tuple[str, str]: + """Why no combination of *fields* and delimiters reads like *designation* (§13.1), and which references fail for + the same reason (so that they are one finding): every reference whose number carries the counters of other + units, and those whose number is drawn with the same shape (its counters as ``#``) around the counter. Two + reasons are told apart, since the style can answer each: + + * the number carries **delimiters** the reference leaves out (a label ``({n})`` under a ref ``{n}``, a prefix + ``No. ``): Word's ``\\t`` switch could drop them, but the style's ``docx.reference_text_switch`` does not take it + to (it is ``off``, or what Word does with them is not documented); + * the number carries the **counters of other units** that the reference writes itself before it (a label + ``{section}.{n}`` under a ref ``{n}``): Word returns the whole number or none of it.""" + wanted = atoms(designation) + _, owner = wanted[uncovered(designation, fields) or 0] + mine = {t.owner for t in designation if isinstance(t, Count)} + skips = any(isinstance(owner_, Ghost) for owner_ in mine) + for candidate in fields: + plain = candidate.number and not candidate.suppress_text + if not (plain and any(isinstance(t, Count) and t.owner == owner for t in candidate.tokens)): + continue + drawn = printable(text_of(candidate.tokens)) + shape = "".join("#" if isinstance(t, Count) else t.text for t in candidate.tokens) + if skips and any(isinstance(t, Count) and t.owner not in mine for t in candidate.tokens): + return ("a heading level is skipped before it (§4.1) and Word's counter for that level belongs to another " + "unit, so no field returns the number the renderer shows", "skipped") + if any(isinstance(t, Count) and t.owner != owner for t in candidate.tokens): + return (f"Word draws its number as '{drawn}', which holds the counters of other units that the reference " + f"writes itself before it (the section's, the enclosing items'), and a field returns the whole " + f"number or none of it", "counters") + around = "".join(t.text for t in candidate.tokens if isinstance(t, Lit)) + if self.text_switch == "off": + how = "Word's \\t switch, which could leave it out, is off (docx.reference_text_switch)" + elif {c for c in around if c in DELIMITERS} == {"."}: + how = ("Word is not documented to leave a period after letters out with its \\t switch " + "(docx.reference_text_switch: guess expects it to)") + else: + how = "Word is not documented to leave such punctuation out with its \\t switch" + return (f"Word draws its number as '{drawn}', with text or punctuation around the counter that the " + f"reference does not have, and {how}. The reference is made of fields when the level's `ref` has " + f"them too (put it in `ref`, for example `({{n}})`)", f"delimiters {shape}") + return "no combination of Word's fields reads like it", "other" + + def reference(self, found: _Reference) -> _Built | None: + """The fields of the reference *found*: the whole text it shows (the designation inside the text of + ``references.format``) covered by fields and delimiters, so that a field may stand for the words of the format + when the number it returns holds the same words (``Article {designation}`` and a label ``Article {n}``: one + plain ``REF``, no ``\\t``); None for a target the plan knows nothing of. A designation that cannot be covered is + written as the renderer's text, which is right at open and not a field (``fixed``), and noted for the + report, as is a cover that reads differently in Word for a reason no diagnostic of a unit already gives.""" + node = self.nodes.get(found.target) + if node is None or found.target not in self.units: + return None + numbered = isinstance(node, Section) or node.number is not None + designation = self.desig[found.target] if numbered else self.desig.get(found.section or "", []) + fields = self.fields(designation) + whole: list[Shown] = [] + parts = self.formats.reference.split("{designation}") + for index, literal in enumerate(parts): + if literal: + whole.append(Lit(literal)) + if index < len(parts) - 1: + whole.extend(designation) + pieces = cover(whole, fields) + if pieces is None: + reason, group = self.why_uncovered(designation, fields) + self.unreferable.append(_Finding("fixed", reason, found, found.text, group)) + return _Built([found.text] if found.text else [], True, None, True) + ghost = self.skipped_part(whole, pieces) + if ghost: + self.unreferable.append(_Finding("ghost", "", found, ghost)) + shown = "".join(piece if isinstance(piece, str) else text_of(piece.tokens) for piece in pieces) + if shown != found.text: + chain = {t.owner for t in designation if not isinstance(t, Lit) and isinstance(t.owner, str)} + if not chain & self.flagged and found.target not in self.flagged: + self.unreferable.append(_Finding("differs", "", found, shown)) + lo_text = None + if any(isinstance(piece, Field) and piece.suppress_text for piece in pieces): + lo_text = "".join(piece if isinstance(piece, str) else text_of(self.words(piece.owner) if piece.number + else piece.tokens) for piece in pieces) + return _Built(pieces, shown == found.text, lo_text, False, ghost) + + @staticmethod + def skipped_part(whole: Sequence[Shown], pieces: Sequence[str | Field]) -> str: + """The text of the counters of heading levels the document skips (a :class:`Ghost`: nothing holds them) that the + cover writes as literal text, in order; empty when it writes none (a field returns them, or the reference has + none).""" + ghosts: list[bool] = [] + for token in whole: + ghosts += [isinstance(token, Count) and isinstance(token.owner, Ghost)] * len(atoms([token])) + text, at = "", 0 + for piece in pieces: + size = len(piece) if isinstance(piece, str) else len(atoms(piece.tokens)) + if isinstance(piece, str): + text += "".join(character for character, ghost in zip(piece, ghosts[at:at + size], strict=True) if ghost) + at += size + return text + + def report_references(self) -> None: + """One finding for the references that fail in the same way (the first, by line, and how many more), as + for a run of units counted off by the same amount.""" + groups: dict[tuple[str, str], list[_Finding]] = {} + for finding in self.unreferable: + groups.setdefault((finding.kind, finding.group or finding.reason), []).append(finding) + for (kind, _), members in groups.items(): + members.sort(key=lambda m: (m.found.line is None, m.found.line or 0)) + first = members[0] + reason = first.reason + text, shown = printable(first.found.text), printable(first.shown) + count = len(members) - 1 + many = f"{count} more cross-references" if count > 1 else "1 more cross-reference" + more = f" {many} {'fail' if count > 1 else 'fails'} in the same way." if count else "" + if kind == "ghost": + more = f" {many} {'are' if count > 1 else 'is'} written the same way." if count else "" + message = (f"The cross-reference '{text}' holds the number of a heading level that the document skips " + f"(a heading level must follow the one above, §4.1), which no Word number returns: " + f"'{shown}' is written as text there and will not follow edits made in Word; the rest is " + f"made of fields.{more}") + elif kind == "fixed": + message = (f"The cross-reference '{text}' cannot be built from Word's numbering fields: {reason}. It is " + f"written as the text '{text}', which is right now but is not a field: it will not follow " + f"edits made in Word.{more}") + else: + message = (f"The cross-reference '{text}' will read '{shown}' in Word: a part of it is numbered or " + f"written differently there.{more}") + self.report("render-docx-reference", "warning", message, first.found.line) + + def cyclic_titles(self, built: dict[str, _Built]) -> tuple[dict[str, tuple[str, ...]], list[str]]: + """The cross-references in a heading's title that must be text, by heading unit, and the headings in the + order their texts can be worked out (those a title's fields return first). Under the ``none`` scheme a heading + is designated by its text, and a field for it (``REF bookmark \\h``) returns what its bookmark holds; a + reference in the title that leads back to the heading (directly, or through the titles of other headings) + would be a field inside the bookmark it points at, which Word cannot show (a self-reference) and LibreOffice + expands without end. Such a reference is written as its text and reported. The headings are the nodes of a + graph, an edge for each heading whose text a field in another's title returns; a reference in a title is + cyclic exactly when such a field's heading is in the title's own strongly connected component (the heading + itself included).""" + if not any(not p.number for made in built.values() for p in made.pieces if isinstance(p, Field)): + return {}, [] # no field returns the text of a heading: nothing can lead back into one + inside: dict[str, list[str]] = {} + for section in self.tree.sections: + if section.unit is not None: + targets = [i.target_unit for i in iter_inlines(section.title) + if isinstance(i, CrossRef) and i.target_unit in built] + inside[section.unit] = list(dict.fromkeys(t for t in targets if t is not None)) + owners = {target: tuple(dict.fromkeys(p.owner for p in made.pieces if isinstance(p, Field) and not p.number)) + for target, made in built.items()} + component, completed = _components({heading: list(dict.fromkeys(o for t in targets for o in owners[t])) + for heading, targets in inside.items()}) + found: dict[str, tuple[str, ...]] = {} + for heading, targets in inside.items(): + cyclic = tuple(t for t in targets if any(component[o] == component[heading] for o in owners[t])) + if cyclic: + found[heading] = cyclic + return found, [heading for members in completed for heading in members] + + def heading_texts(self, built: dict[str, _Built], cyclic: Mapping[str, tuple[str, ...]], order: list[str]) -> None: + """What a field that returns a heading's text gives, worked out from the runs the heading is written with + (§13.3): a reference in the title is the text of the fields it is written as (which return the text of other + headings, worked out first), or its own text where it is written as text.""" + sections = {section.unit: section for section in self.tree.sections if section.unit is not None} + for heading in order: + section = sections.get(heading) + if section is None: + continue + + def shown(ref: CrossRef, heading: str = heading) -> str: + made = built.get(ref.target_unit or "") + if made is None or ref.target_unit in cyclic.get(heading, ()): + return ref.text + return "".join(piece if isinstance(piece, str) else + text_of(piece.tokens) if piece.number else self.heading_text[piece.owner] + for piece in made.pieces) + + self.heading_text[heading] = _shown(section.title, shown) + + def references(self, found: Mapping[str, _Reference]) -> dict[str, _Built]: + built: dict[str, _Built] = {} + for target, reference in found.items(): + made = self.reference(reference) + if made is not None: + built[target] = made + return built + + def finish(self) -> tuple[dict[str, UnitPlan], dict[str, str], dict[str, RefPlan], tuple[TocEntryPlan, ...]]: + """The bookmarks, the cross-reference fields and the table of contents, which need the numbering of every + unit: the bookmark of every anchor (:class:`.docx_bookmarks.BookmarkNames`, given out in the order of the + document so that the names are the same on every run), a hidden one for a unit a field needs that has no + anchor, the fields of every ``{{ref:}}`` target, and the table of contents entries.""" + found: dict[str, _Reference] = {} + for inline in iter_tree_inlines(self.tree): + if isinstance(inline, CrossRef) and inline.target_unit is not None: + found.setdefault(inline.target_unit, _Reference(inline.target_unit, inline.section_unit, + inline.text, None)) + if found: + for inline, line in iter_located_body_inlines(self.tree): + if isinstance(inline, CrossRef) and inline.target_unit in found and line is not None: + known = found[inline.target_unit] + if known.line is None or line < known.line: + found[inline.target_unit] = known._replace(line=line) + built = self.references(found) + cyclic, order = self.cyclic_titles(built) + if order: + # a heading's text is made of the fields in its title: the first pass found which are text and in what + # order the texts can be worked out, and the references are made again from them + self.heading_texts(built, cyclic, order) + self.unreferable = [] + built = self.references(found) + self.report_references() + needed = {piece.owner for made in built.values() for piece in made.pieces if isinstance(piece, Field)} + needed |= {holder for holder, _ in self.alts.values()} + #: A bookmark holds its paragraph's text only if a field returns that text; any other is a point at the start + #: of the paragraph, so that no field ever stands inside the bookmark it points at. + enclosing = {piece.owner for made in built.values() for piece in made.pieces + if isinstance(piece, Field) and not piece.number} + word_only = [(found[target].text, made.lo_text) for target, made in built.items() if made.lo_text is not None] + if word_only: + first, lo = word_only[0] + more = f"; so do {len(word_only) - 1} more" if len(word_only) > 1 else "" + self.report("render-docx-reference-word-only", "info", + f"The cross-reference '{printable(first)}' leaves out the words of a number with Word's \\t " + f"switch, which LibreOffice ignores: there it reads '{printable(lo)}'{more}.", None) + for section in self.tree.sections: + if section.unit in cyclic: + where = f"line {section.line}" if section.line is not None else f"section {section.unit}" + self.report("render-docx-reference-cycle", "warning", + f"The heading '{printable(plain_text(section.title)[:40])}' ({where}) holds a " + f"cross-reference that leads back to the heading itself, which a field cannot show (the heading's text is what " + f"the reference returns): it is written as text, so it will not follow edits made in Word.", + section.line) + + names = BookmarkNames() + anchors: dict[str, str] = {} + bookmark: dict[str, str] = {} + units = dict(self.units) + for unit, node in self.nodes.items(): + anchor = node.anchor + if anchor: + name = names.anchor("ld_", anchor) + anchors.setdefault(anchor, name) + elif unit in needed: + name = names.hidden() + else: + continue + bookmark[unit] = name + units[unit] = replace(units[unit], bookmarks=(name,), collapsed=unit not in enclosing) + for unit, targets in cyclic.items(): + units[unit] = replace(units[unit], literal_refs=targets) + for unit, (holder, alt_suff) in self.alts.items(): + # The number of a later alternative is the number of the unit it repeats (§15.8). + shared = self.units[holder] + tail = shared.label[len(shared.word_ref):] + alt_segments: list[Segment] = [ + FieldSeg(bookmark[holder], True, False, True, shared.word_ref, shared.word_ref)] + units[unit] = replace(units[unit], alt=AltNumber( + (*alt_segments, *([Literal_(tail)] if tail else [])), alt_suff)) + for inline in iter_tree_inlines(self.tree): + if isinstance(inline, DefinedTerm) and inline.defining and inline.anchor and inline.anchor not in anchors: + anchors[inline.anchor] = names.anchor("lddef_", inline.anchor.removeprefix("def:")) + for attachment in self.tree.attachments: + if attachment.anchor and attachment.anchor not in anchors: + anchors[attachment.anchor] = names.anchor("ldatt_", attachment.anchor) + if self.tree.attachments_anchor and self.tree.attachments_anchor not in anchors: + anchors[self.tree.attachments_anchor] = names.fixed(ATTACHMENTS_HEADING) + + refs: dict[str, RefPlan] = {} + for target, made in built.items(): + segments: list[Segment] = [] + for piece in made.pieces: + if isinstance(piece, str): + segments.append(Literal_(piece)) + continue + cached = text_of(piece.tokens) + plain = text_of(self.words(piece.owner)) if piece.number else cached + segments.append(FieldSeg(bookmark[piece.owner], piece.number, piece.suppress_text, True, cached, + plain if piece.suppress_text else cached)) + refs[target] = RefPlan(tuple(segments), made.exact, made.fixed, made.ghost) + + toc: list[TocEntryPlan] = [] + for entry in self.tree.contents: + name = names.toc() + number = "" + sep: Literal["\t", " ", ""] = "" + mine = units.get(entry.unit) if entry.unit is not None else None + if mine is not None: + mine = units[mine.unit] = replace(mine, toc_bookmark=name) + suff: str | None = None + if mine.alt is not None: + number = "".join(piece.text if isinstance(piece, Literal_) else piece.cached + for piece in mine.alt.segments) + suff = mine.alt.suff + elif mine.ilvl is not None: + number, suff = mine.label, self.definitions["headings"][mine.ilvl].suff + if number: + sep = "\t" if suff == "tab" else " " + toc.append(TocEntryPlan(entry.unit, entry.level, number, sep, entry.title, entry.condition_label, name)) + return units, anchors, refs, tuple(toc) + + # -- the pass ------------------------------------------------------------------- + + def run(self) -> tuple[DocxPlan, list[Diagnostic]]: + if self.formats.headings: + self.heading_num = self.new_num(self.abstract("headings")) + formats = [f for f in (*self.formats.enumerated[:1], self.formats.paragraphs) if f is not None] + self.reserve5 = any(section.level == 5 for section in self.tree.sections) and any( + shows_section(f.label, [f.ref], 0) for f in formats) + for block in self.tree.preamble: + self.block(block, None) + for section in self.tree.sections: + self.section(section) + for block in section.blocks: + self.block(block, section) + self.report_section_labels() + self.report_libreoffice_counts() + for drift in self.drifts: + self.report_drift(drift) + self.report_collisions() + self.report_alternatives() + units, anchors, refs, toc = self.finish() + abstracts = tuple( + AbstractNum(a.id, a.kind, f"{0x4C440000 + a.id:08X}", # type: ignore[arg-type] + "multilevel" if a.kind == "headings" else "hybridMultilevel", tuple(a.levels)) + for a in self.abstracts) + heading_id = self.heading_num.id if self.heading_num else None + return (DocxPlan(abstracts, tuple(self.nums), heading_id, units=units, anchors=anchors, refs=refs, toc=toc), + by_line(self.diagnostics)) + + def block(self, block: object, section: Section | None, quoted: bool = False) -> None: + """Plan *block*; *quoted*: it stands in a quote or a drafting note (§13.2), whose items are numbered like any + other but, being built on the section whatever they are nested in, never ride the heading list.""" + match block: + case Paragraph(unit=str()) if section is not None: + self.paragraph(block, section) + case List(): + self.list_(block, section, quoted) + case Quote(blocks=blocks) | DraftingNote(blocks=blocks): + for child in blocks: + self.block(child, section, True) + + # -- what a unit gets ----------------------------------------------------------- + + def add(self, node: _Unit, kind: Literal["section", "item", "paragraph"], num: Num | None, ilvl: int | None, *, + explicit: bool, cause: tuple[str, ...] = (), step: Step | None = None, + ind: tuple[int, int] | None = None, holder: str | None = None) -> None: + """Record the plan of *node*. A later alternative (*holder*: the unit whose number it repeats) takes the + label of its holder and ``numId`` 0; its number is written as fields by :meth:`finish`.""" + assert node.unit is not None + self.nodes[node.unit] = node + num_id = num.id if num else None + label = word_ref = "" + count = None + if step is not None: + self.steps[node.unit] = step + label, word_ref, count = step.label, step.word_ref, step.count + if holder is not None: + num_id, label, word_ref = 0, self.units[holder].label, self.units[holder].word_ref + self.units[node.unit] = UnitPlan(node.unit, kind, num_id, ilvl, explicit, ind, None, label, word_ref, count, + cause, (), None) + + def ancestor_flagged(self, unit: str) -> bool: + parent = self.parent.get(unit) + while parent is not None: + if parent in self.flagged: + return True + parent = self.parent.get(parent) + return False + + def check(self, node: _Unit, kind: str, step: Step, counter: str, group: object, cause: tuple[str, ...], + key: tuple[int, int]) -> None: + """Compare what Word gives *node* with what the resolver gave it. Word's count must be the resolver's (a run + of units off by the same amount is one finding, :meth:`report_drift`); the counter must read the same + (letters past 26 and Roman numerals from 4000 do not); and the whole label must read the same unless a + diagnostic already says why it cannot (a label that needs the section, a clamped depth). A unit whose + ancestor already differs is not reported again. *counter* is the one the level Word uses is written in.""" + assert node.unit is not None and node.number is not None + fresh = not self.ancestor_flagged(node.unit) + expected = format_counter(node.number, counter) + word = kind[0].upper() + kind[1:] + if step.count != node.number: + self.flagged.add(node.unit) + if fresh: + delta = step.count - node.number + drift = self.open_drift.get(key) + if drift is not None and drift.delta == delta: + drift.more += 1 + else: + drift = self.open_drift[key] = _Drift(node, kind, step, delta) + self.drifts.append(drift) + return + self.open_drift.pop(key, None) + if step.own == expected: + if step.label == node.label or "label-fallback" in cause or "clamped" in cause: + return + rule = "render-docx-number-mismatch" + message = f"Word will number the {kind} '{printable(node.label or '')}' as '{printable(step.label)}'." + elif counter.endswith("roman") and node.number >= 4000: + rule = "render-docx-roman-overflow" + message = (f"{word} {node.number} is a Roman numeral of 4000 or more: the renderer shows digits " + f"('{printable(expected)}'), Word shows '{printable(step.own)}'.") + elif counter.endswith("alpha"): + rule = "render-docx-letter-overflow" + message = (f"{word} {node.number} ('{printable(expected)}') will show as '{printable(step.own)}' in " + f"Word: Word repeats the letter after 26 ('aa', 'bb'), the renderer counts 'aa', 'ab'. " + f"References to it and to its sub-items read the same way.") + else: + rule = "render-docx-number-mismatch" + message = f"Word will write the counter of the {kind} '{printable(expected)}' as '{printable(step.own)}'." + self.flagged.add(node.unit) + # An overflow is reported once for the list, heading level or section it starts in. + once = rule.endswith("overflow") + if fresh and not (once and (rule, group) in self.reported): + self.reported.add((rule, group)) + self.report(rule, "warning", message, node.line) + + def report_drift(self, drift: _Drift) -> None: + """One finding for a run of units that Word counts off by the same amount: the first, and how many follow.""" + node, step = drift.node, drift.step + message = (f"Word will number the {drift.kind} '{printable(node.label or '')}' as '{printable(step.label)}', " + f"{abs(drift.delta)} {'later' if drift.delta > 0 else 'earlier'} than the renderer") + if drift.more: + message += f"; so will the following {drift.more}" + self.report("render-docx-number-mismatch", "warning", message + ".", node.line) + + def fallback_label(self, level: LevelFormat, family: str, ilvl: int, kind: str, line: int | None) -> None: + """Report a label that shows the section's number where Word can show only the list's own counters.""" + levels = self.definitions[family] + self.report("render-docx-section-label", "warning", + f"The label '{printable(level.label)}' of {kind}s in a section shows the section's number, " + f"which Word can show only for headings: in Word it reads " + f"'{printable(_reads(levels[ilvl], levels))}'.", line) + + def detaches(self, depth: int, parent: str | None, placed: _Placed | None) -> bool: + """True if a list of the enumerated family at *depth* in *parent* goes on the detached abstractNum: it is nested + in an item on that abstractNum (its structure restarts it), or the enumerated level it would take names a + level that no unit of its enclosing chain holds, so that Word could show the counter of nothing, or of what + another list counted last. (The same condition as :meth:`numbered` reports a label whose counter Word cannot + show; here it is met before the list is placed.)""" + if placed is not None and placed.abstract.kind == "detached": + return True + named = {part for part in split_level_text(self.definitions["enumerated"][depth].lvl_text) + if isinstance(part, int) and part != depth} + enumerated = self.by_kind.get("enumerated") + held: set[int] = set() + above = parent + while above is not None: + unit = self.canon.get(above, above) + on = self.placed.get(unit) + if enumerated is not None and on is not None and on.abstract is enumerated \ + and self.units[unit].num_id not in (None, 0): + held.add(self.units[unit].ilvl) # type: ignore[arg-type] + above = self.parent.get(above) + return not named <= held + + def lost_counters(self, fmt: LevelFormat, section: Section | None, parent: str | None) -> list[object]: + """The owners of the counters the resolver's label of a unit of *fmt* shows besides its own, in order: what the + detached abstractNum leaves out.""" + owners = [t.owner for t in self.label_tokens(fmt, section, parent) + if isinstance(t, Count | Title) and t.owner is not _SELF] # a Title: the heading's text, under ``none`` + return list(dict.fromkeys(owners)) + + def describe(self, owners: Sequence[object]) -> str: + """What the units that hold *owners* are, for a message: the section, an enclosing item, an enclosing ordered + item.""" + names: list[str] = [] + if any(isinstance(owner, Ghost) or owner not in self.units or self.units[owner].kind == "section" # type: ignore[index] + for owner in owners): + names.append("section") + kinds = [self.placed[owner].abstract.kind.split("+")[0] # type: ignore[index] + for owner in owners if owner in self.placed] + if "renumber" in kinds: + names.append("ordered item") + if any(kind != "renumber" for kind in kinds): + names.append("item") + if not names: + return "an enclosing unit" + return "the enclosing " + (", ".join(names[:-1]) + " and " + names[-1] if len(names) > 1 else names[0]) + + def detached_label(self, fmt: LevelFormat, abstract: _Abstract, ilvl: int, kind: str, lost: Sequence[object], + placed: _Placed | None, first: ListItem) -> None: + """Report a list on an abstractNum that shows each level's own counter alone (the detached one, or the next of + its family past Word's nine levels) although the resolver's label shows the number of an enclosing unit: Word + draws the unit's own counter, and a reference to it is still made of fields.""" + levels = abstract.levels + if abstract.kind != "detached": + why = "the list is nested past Word's nine levels and goes on in a list of its own" + elif placed is None: + why = "the list stands outside the unit it belongs to" + elif placed.abstract.kind == "headings": + why = "the levels of the heading list kept for nested lists are used up or read otherwise" + else: + why = "the list is nested in a unit that is on another list" + self.section_label(("detached", fmt.label, self.describe(lost), why), + f"The label '{printable(fmt.label)}' of {kind}s shows the number of {self.describe(lost)}, " + f"which Word cannot show in this list ({why}): in Word it reads " + f"'{printable(_reads(levels[ilvl], levels))}'. A reference to such an item is still made of " + f"fields.", first.line) + + def section_label(self, group: tuple[object, ...], message: str, line: int | None) -> None: + """Note a label Word draws differently for want of counters it cannot show; the ones that have *group* in + common are one finding.""" + self.label_findings.setdefault(group, []).append((line, message)) + + def report_section_labels(self) -> None: + """One ``render-docx-section-label`` for each group: the first place by line, and how many more do the same + (as for the references that fail alike).""" + for members in self.label_findings.values(): + members.sort(key=lambda m: (m[0] is None, m[0] or 0)) + line, message = members[0] + if len(members) > 1: + message += f" {len(members) - 1} more {'places do' if len(members) > 2 else 'place does'} the same." + self.report("render-docx-section-label", "warning", message, line) + + def shared_counter(self, message: str, line: int | None) -> None: + self.report("render-docx-shared-counter", "warning", message, line) + + # -- units ---------------------------------------------------------------------- + + def section_designation(self, section: Section) -> list[Shown]: + """The designation of *section* as tokens, written as the resolver writes it (§13.1): the ``ref`` of every part + of its number after the one before; a part the document skipped is a :class:`Ghost` that no unit holds. Under + the ``none`` scheme it is the heading's text (§13.3), and empty when that is. A later alternative that shares + a number with another holds none of its own: its part is the counter of the unit it repeats (§15.8).""" + assert section.unit is not None + if self.textual: + text = section.designation or "" + return [Title(section.unit, text)] if text else [] + assert section.number is not None + del self.owners[section.level - 1:] + while len(self.owners) < section.level - 1: + self.owners.append((Ghost(section.unit, len(self.owners)), 1)) + self.owners.append((self.canon.get(section.unit, section.unit), section.number)) + designation: list[Shown] = [] + for part, (owner, value) in enumerate(self.owners): + level = self.formats.headings[min(part, len(self.formats.headings) - 1)] + designation = extend_tokens( + designation, fill_tokens(level.ref, n=[Count(owner, format_counter(value, level.counter))]), + textual=False) + return designation + + def section(self, section: Section) -> None: + unit = section.unit + assert unit is not None + self.parent[unit] = section.parent_unit + self.section_level[unit] = section.level + self.heading_text[unit] = _shown(section.title) + if self.heading_num is None or section.number is None: + self.desig[unit] = self.section_designation(section) + self.add(section, "section", None, None, explicit=False) + return + abstract = self.by_kind["headings"] + ilvl = section.level - 1 + parent = section.parent_unit + skipped = (parent is None and section.level > 1) or ( + parent is not None and self.section_level[parent] + 1 < section.level) + # The sections under a later alternative restart in a num of their own: the alternative counts nothing, so + # on the headings' num they would go on from the alternative it repeats (§15.8). + while self.alternatives and self.alternatives[-1].level >= section.level: + self.alternatives.pop() + num, explicit = self.heading_num, False + if self.alternatives: + context = self.alternatives[-1] + if context.num is None: + context.num = self.new_num(abstract, ilvl, section.number) + num, explicit = context.num, True + self.numbered(section, "section", num, abstract, ilvl, counter=self.formats.headings[ilvl].counter, + group=("heading", section.level), cause=("skipped-level",) if skipped else (), + explicit=explicit) + if unit in self.alts: + self.alternatives.append(_Alternative(section.level)) + self.desig[unit] = self.section_designation(section) + + def paragraph(self, paragraph: Paragraph, section: Section) -> None: + assert paragraph.unit is not None and section.unit is not None + self.parent[paragraph.unit] = paragraph.parent_unit + fmt = self.formats.paragraphs + base = self.desig[section.unit] + if paragraph.number is None or fmt is None: + self.desig[paragraph.unit] = base + self.add(paragraph, "paragraph", None, None, explicit=False) + return + shows = shows_section(fmt.label, [fmt.ref], 0, textual=self.textual) + entry = self.paragraph_nums.get(section.unit) + if entry is None: + slot = self.qualified(fmt, section) if shows else None + abstract, ilvl = slot or (self.abstract("paragraphs"), 0) + entry = self.paragraph_nums[section.unit] = _Paragraphs( + self.new_num(abstract, ilvl), abstract, ilvl, shows and slot is None) + cause: tuple[str, ...] = () + if entry.fallback: + cause = ("label-fallback",) + self.fallback_label(fmt, "paragraphs", 0, "paragraph", paragraph.line) + self.numbered(paragraph, "paragraph", entry.num, entry.abstract, entry.ilvl, counter=fmt.counter, + group=("paragraphs", section.unit), cause=cause) + piece = fill_tokens(fmt.ref, n=[self.counted(paragraph, fmt.counter)], section=base) + self.desig[paragraph.unit] = extend_tokens(base, piece, textual=self.textual) + + def counted(self, node: ListItem | Paragraph, counter: str) -> Count: + """The counter token of *node* in a designation: the unit's own, or, for a later alternative, the one of the + unit it repeats.""" + assert node.unit is not None and node.number is not None + return Count(self.canon.get(node.unit, node.unit), format_counter(node.number, counter)) + + # -- the heading list's free levels (§2.5) ---------------------------------------- + + def label_tokens(self, fmt: LevelFormat, section: Section | None, parent: str | None, *, designation: bool = False + ) -> list[Token]: + """The label of a unit of *fmt* directly in *parent* (a section or an item; None in the preamble), filled in as + the resolver does (§13.2), as tokens: the counter of the unit itself is :data:`_SELF`'s, every other counter is + that of the unit that holds it (the parts of the section's number and the items on the way). With + *designation*, the unit's designation (§13.1) in the same form.""" + section_des = self.desig[section.unit] if section is not None and section.unit is not None else [] + own = [Count(_SELF, "")] + path = extend_tokens(self.desig.get(parent or "", []), fill_tokens(fmt.ref, n=own, section=section_des), + textual=self.textual) + return path if designation else fill_tokens(fmt.label, n=own, path=path, section=section_des) + + def reads_as_designation(self, fmt: LevelFormat, section: Section, parent: str | None) -> bool: + """True if what Word draws for a unit of *fmt* in *parent*, the label with its trailing periods dropped (what + ``REF \\n`` returns), is the end of the unit's designation: a field for the unit then stands for that part of + every reference to it and to what is nested in it, with the counters of the enclosing units in it.""" + shown = self.label_tokens(fmt, section, parent) + while shown and isinstance(shown[-1], Lit): + stripped = shown[-1].text.rstrip(".") + if stripped: + shown[-1] = Lit(stripped) + break + shown.pop() + wanted, mine = atoms(self.label_tokens(fmt, section, parent, designation=True)), atoms(shown) + return len(mine) <= len(wanted) and wanted[len(wanted) - len(mine):] == mine + + def heading_level(self, owner: object) -> int | None: + """The heading list's level the counter of *owner* is drawn at, when it is a unit of the open section that is + numbered on the heading list (an item or a paragraph that rides it, or a nested item on one of its reserved + levels); None for any other owner.""" + found = self.placed.get(owner) if isinstance(owner, str) else None + if found is None or found.abstract.kind != "headings": + return None + unit = self.units[owner] # type: ignore[index] + return unit.ilvl if unit.num_id not in (None, 0) else None + + def candidate(self, fmt: LevelFormat, section: Section, parent: str | None, ilvl: int) -> LevelParts | None: + """The level text a unit of *fmt* in *parent* would have at level *ilvl* of the heading list: its label filled + in as the resolver does, with the counter of each part of the section's number as ``%j`` of the heading level + that holds it, the counter of an item on the way as ``%j`` of the level that item is on, and its own counter + at *ilvl*. None when a counter is of an owner that is not on the heading list or the text cannot be + written.""" + assert section.unit is not None + level_of = {owner: j for j, (owner, _) in enumerate(self.owners)} + parts: list[str | int] = [] + for token in self.label_tokens(fmt, section, parent): + if isinstance(token, Title): + return None + if isinstance(token, Count): + j = ilvl if token.owner is _SELF else level_of.get(token.owner, self.heading_level(token.owner)) + if j is None: + return None + parts.append(j) + elif parts and isinstance(parts[-1], str): + parts[-1] += token.text + else: + parts.append(token.text) + return None if level_text_problem(tuple(parts)) is not None else tuple(parts) + + def claim(self, ilvl: int, definition: LevelDef, *, section_level: bool) -> tuple[_Abstract, int] | None: + """The heading abstractNum at *ilvl* if the level reads as *definition*: a heading level when it is the same, + a free level (levels 6-9) when nothing claimed it or it is the same, which defines it. The level after the + sections' fifth, which only a level-5 section's units can use, is kept for them when the document has one.""" + abstract = self.by_kind["headings"] + current = abstract.levels[ilvl] + if ilvl <= 4 or ilvl in self.claimed: + same = (current.lvl_text, current.num_fmt) == (definition.lvl_text, definition.num_fmt) + return (abstract, ilvl) if same else None + if ilvl == 5 and self.reserve5 and not section_level: + return None + abstract.levels[ilvl] = definition + self.claimed.add(ilvl) + return abstract, ilvl + + def heading_slot(self, fmt: LevelFormat, section: Section, parent: str | None, ilvl: int, *, section_level: bool + ) -> tuple[_Abstract, int] | None: + """The heading list's level *ilvl* for the units of *fmt* in *parent*, if the label can be written there with + counters of the heading list alone and the level is free or reads the same. A level after a section's is as wide + as the widest of its labels, as a heading's is; a nested one steps in a centimetre from the item it is in, as + every nested list does (a label with the path in it is longer than that, and Word moves the text to its next tab + stop).""" + parts = self.candidate(fmt, section, parent, ilvl) + if parts is None: + return None + if not section_level and any(isinstance(part, int) and part != ilvl for part in parts) \ + and not self.reads_as_designation(fmt, section, parent): + return None # the counters it shows are not the ones its designation ends with: no field could stand for it + levels = self.by_kind["headings"].levels + definition = _level(ilvl, NUM_FMT[fmt.counter], parts, + widths={j: levels[j].num_fmt for j in range(9)} if section_level else None) + return self.claim(ilvl, definition, section_level=section_level) + + def qualified(self, fmt: LevelFormat, section: Section) -> tuple[_Abstract, int] | None: + """Where the units of *fmt* directly in *section* go when their label needs the section's number: the heading + list's level after the section's (``%1.%2`` after a level-1 section), where Word can show the section's + counters, if that level reads as the label does. None: the label has a foreign-free form of its own + (:meth:`fallback_label`). Under the ``none`` scheme the section's number is its heading text.""" + if self.textual or "headings" not in self.by_kind: + return None + return self.heading_slot(fmt, section, section.unit, section.level, section_level=True) + + def reserved(self, fmt: LevelFormat, depth: int, section: Section, parent: str) -> tuple[_Abstract, int] | None: + """Where a list nested *depth* levels below a unit on the heading list goes: the heading abstractNum's level + ``4 + depth``, defined on first use. Its text is the label as the resolver writes it, with every counter it + shows at the level of the heading list that holds it (the section's number, the item it is nested in, the + items between): a ``{path}`` label is live, and the list restarts by structure under every new item. None when + a counter it shows is not on the heading list, or the level reads otherwise for another list.""" + return self.heading_slot(fmt, section, parent, 4 + depth, section_level=False) + + # -- numbering a unit --------------------------------------------------------------- + + def indent(self, abstract: _Abstract, ilvl: int, unit: str, parent: str | None, *, alternative: bool + ) -> tuple[int, int] | None: + """The indent a unit carries itself, in twips ``(left, hanging)``. An item or paragraph on the heading abstractNum + (whose levels indent the headings) takes the width of the heading level it stands on; one that stands in an + item with an indent of its own, on any list, is one step in from the text of that item; a later alternative, + which has no numbering to take an indent from, takes its level's. Any other unit takes its level's, and the + plan sets nothing.""" + level = abstract.levels[ilvl] + if unit in self.item_units and parent in self.text_left: + left, hanging = min(self.text_left[parent] + _STEP, self.max_left), _STEP + elif unit in self.item_units and abstract.kind == "headings": + left, hanging = level.left, level.hanging + else: + return (level.left, level.hanging) if alternative else None + self.text_left[unit] = left + return left, hanging + + def numbered(self, node: _Unit, kind: Literal["section", "item", "paragraph"], num: Num, abstract: _Abstract, + ilvl: int, *, counter: str, group: object, cause: tuple[str, ...] = (), explicit: bool = True) -> Num: + """Number *node* on *num* at *ilvl* of *abstract* and check it against the resolver; returns the num it is on. + + * A later alternative whose number Word would not give (§15.8) takes ``numId`` 0 and the number of the unit + it repeats, as fields. + * A unit whose number Word would give differently (it goes on where the resolver restarts) is put on a num + that starts there, a fixed setting (§2.7). + * A label that shows a number of a unit outside the unit's own ancestors, which Word shows from the levels + of its list, is marked ``label-fallback``.""" + assert node.unit is not None and node.number is not None + unit = node.unit + word = {"section": "heading", "item": "list item", "paragraph": "paragraph"}[kind] + parent = node.parent_unit + if kind != "section": + self.item_units.add(unit) + peek = self.sim.peek(num.id, ilvl) + if node.shares_number_with is not None and peek != node.number: + # Word would count on, and the number is the one the level holds now: the unit that last counted the level + # holds it, and the alternative shows that unit's number (a unit that shares a number but finds another + # in the level, because a list of the heading list counted in between, is restarted below). + count, holder = self.sim.current(num.id, ilvl) + if count == node.number and isinstance(holder, str) and holder in self.units: + self.canon[unit] = holder + self.alts[unit] = (holder, abstract.levels[ilvl].suff) + self.add(node, kind, None, ilvl, explicit=True, cause=(*cause, "alternative"), holder=holder, + ind=self.indent(abstract, ilvl, unit, parent, alternative=True)) + return num + used = num + if peek != node.number: + used = self.new_num(abstract, ilvl, node.number) + cause, explicit = (*cause, "resync"), True + if abstract.kind == "headings": + self.collisions.append((node, word)) + else: + self.shared_counter( + "Word's counter for a level of this list was also used by another list in between; the numbers " + "are kept as the renderer shows them by restarting at a fixed number, which will not follow " + "items added or removed before it.", node.line) + step = self.sim.step(used.id, ilvl, unit) + if abstract.kind != "headings" and step.lo_count not in (-1, step.count): + self.lo_counts.append((node, step)) + if kind != "section": + # The counters of the unit's own chain: the units it stands under, and every part of the number of the + # section (a skipped heading level counts, as the ghost of the heading that skipped it). + chain: set[object] = {t.owner for t in self.desig.get(parent or "", ()) if isinstance(t, Count)} + above = parent + while above is not None: + chain.add(self.canon.get(above, above)) + above = self.parent.get(above) + if "label-fallback" not in cause and any( + isinstance(t, Count) and t.owner != unit and t.owner not in chain for t in step.tokens): + # The label shows the counter of a level of its list that no unit of its own chain holds: Word shows + # a level nothing numbers as 0, or the counter of whatever last counted it, and the resolver's label + # is that of the enclosing unit. Not left to Word's reading. + cause += ("label-fallback",) + skipped = any(isinstance(t, Count) and isinstance(t.owner, Ghost) for t in self.desig.get(parent or "", ())) + if skipped: + message = (f"The label of {word}s '{printable(node.label or '')}' holds the number of a heading level " + f"that the document skips (a heading level must follow the one above, §4.1): Word draws " + f"that level with the counter of another unit, so it reads '{printable(step.label)}' only " + f"as long as that counter is what the renderer counts, and a unit inserted there changes it.") + else: + message = (f"The label of {word}s '{printable(node.label or '')}' shows the number of an enclosing " + f"item that Word cannot show here (the list is not nested directly in an item of its own " + f"kind): in Word it reads '{printable(step.label)}'.") + self.section_label(("chain", word, skipped), message, node.line) + self.add(node, kind, used, ilvl, explicit=explicit, cause=cause, step=step, + ind=self.indent(abstract, ilvl, unit, parent, alternative=False)) + self.check(node, word, step, counter, group, cause, (abstract.id, ilvl)) + return used + + def restart_class(self, parent: str, placed: _Placed, abstract: _Abstract, *, deeper: bool = False) -> str: + """Why a list nested in the item *parent* does not take its num: the item is a later alternative (numbered by + no list), the list goes on past Word's nine levels (*deeper*), the item is on the heading list but the list + cannot be, it is on another kind of list, or Word would go on counting from another list in the same item.""" + if self.units[parent].num_id == 0: + return "restart-alternative" + if deeper: + return "restart-depth" + if placed.abstract is not abstract: + return "restart-section" if placed.abstract.kind == "headings" else "restart-family" + return "resync" + + def report_libreoffice_counts(self) -> None: + """One information for the units LibreOffice numbers higher than Word does. A level of a list that nothing + counted before a deeper item of the same list (an item nested in a unit of another list: its own level above is + never used here) is read by Word as unused, so the next unit of that level is 1; LibreOffice counts the + unused level as 1 and goes on from it, so that unit is 2 there. The plan is Word's, the file is right for it.""" + if not self.lo_counts: + return + node, step = self.lo_counts[0] + more = f", and {len(self.lo_counts) - 1} more" if len(self.lo_counts) > 1 else "" + self.report("render-docx-libreoffice-count", "info", + f"{len(self.lo_counts)} list item(s) are numbered as the renderer does in Word but one or more higher " + f"in LibreOffice, which counts a level that nothing in the list numbered before a deeper item as 1: " + f"the item '{printable(node.label or '')}' (the first{more}) reads '{step.own}' in Word and " + f"'{printable(self.in_libreoffice(node, step))}' there. Nothing in the file is wrong for Word.", + node.line) + + def in_libreoffice(self, node: _Unit, step: Step) -> str: + """The counter of *node* as LibreOffice reads it, in the format of its level.""" + assert node.unit is not None + unit = self.units[node.unit] + assert unit.ilvl is not None + abstract = next(a for a in self.abstracts if a.id == self.nums[(unit.num_id or 1) - 1].abstract_id) + return word_counter(step.lo_count, abstract.levels[unit.ilvl].num_fmt) + + def report_depth(self, depth: int, line: int | None) -> None: + """Report, once, a list nested past the levels the plan has (:data:`MAX_DEPTH`), of any family.""" + if not self.depth_reported: + self.depth_reported = True + self.report("render-docx-list-depth", "warning", + f"A list is nested {depth} levels deep: the plan has {MAX_DEPTH} levels of Word lists. Items from " + f"level {MAX_DEPTH + 1} on are numbered in the format of level {MAX_DEPTH} and restart by fixed " + f"values (a bullet has no number to restart).", line) + + def report_collisions(self) -> None: + """One finding for the units of the heading list that Word would number off the resolver's because they share + a counter with others of their section (§2.7): the first, and how many more. What an edit in Word does to + the fixed value depends on the unit: a sub-heading restarted after the clauses of its section keeps it, and + the headings after it read higher if it is deleted; a clause or paragraph does not follow what is inserted or + deleted before it.""" + if not self.collisions: + return + node, word = self.collisions[0] + assert node.unit is not None + designation = text_of(self.desig[node.unit]) + more = f" {len(self.collisions) - 1} more units do the same." if len(self.collisions) > 1 else "" + if word == "heading": + effect = ("The heading keeps that value when clauses or headings are inserted before it, and if it is " + "deleted in Word the headings after it read higher than the renderer's.") + else: + effect = ("The unit keeps that value: a clause or paragraph inserted or deleted before it in Word does not " + "move it.") + self.shared_counter( + f"The {word} '{printable(node.label or '')}' has the designation {printable(designation)}, which another " + f"unit of its section has too (in every output format their designations collide), and in Word they " + f"share one counter. The number is kept as the renderer shows it by restarting at a fixed value. " + f"{effect}{more}", node.line) + + def report_alternatives(self) -> None: + """One information for the document's alternatives that share a number: their numbers are fields of the first + alternative, which a lawyer can delete in Word (§15.8).""" + if not self.alts: + return + holder = next(iter(self.alts.values()))[0] + node = self.nodes[holder] + where = f", line {node.line}" if node.line is not None else "" + self.report("render-docx-alternative", "info", + f"{len(self.alts)} alternative(s) that share a number show it as a field of the first alternative " + f"('{printable(self.units[holder].label)}'{where}); deleting that paragraph in Word " + f"breaks their numbers and every reference to their shared identifier. Render the document " + f"with the condition values to produce the final one.", node.line) + + def list_(self, lst: List, section: Section | None, quoted: bool = False) -> None: + first = lst.items[0] + if lst.family == "bullet": + self.bullets(lst, section, quoted) + return + fmt = lst.format + assert fmt is not None + depth = lst.depth + # Word has nine levels in a list: a list nested deeper goes on in the next abstractNum of its family (levels + # 10-18 in ``enumerated+1``, and so on up to MAX_DEPTH), so that it keeps restarting by structure inside that + # abstractNum; past MAX_DEPTH the last level is used again. + reached = min(depth, MAX_DEPTH - 1) + chunk, rem = divmod(reached, _LEVELS) + levels = self.formats.enumerated if lst.family == "enumerated" else (fmt,) + counter = levels[reached % len(levels)].counter + parent = first.parent_unit + placed = self.placed.get(parent) if parent is not None else None + parent_live = placed is not None and self.units[parent].num_id not in (None, 0) + refs = ([f.ref for f in (self.formats.enumerated[j % len(self.formats.enumerated)] for j in range(depth + 1))] + if lst.family == "enumerated" else [fmt.ref] * (depth + 1)) + shows = section is not None and shows_section(fmt.label, refs, depth, textual=self.textual) + # A label that needs the section's number goes on the heading list's level after the section's; a list nested + # in a unit that is on the heading list goes on its reserved levels, so that Word keeps restarting it by + # structure, with the label's counters of the section and of the items on the way as levels of that list. + # Otherwise the family's own abstractNum, with a label Word can write; and where that abstractNum would have to + # show a counter of a unit that is not on it, the detached one, which shows each level's own counter alone. + slot = None + if lst.family == "enumerated" and section is not None and not quoted: + if depth == 0 and placed is None: + slot = self.qualified(fmt, section) if shows else None + elif placed is not None and placed.abstract.kind == "headings" and 1 <= depth <= 4: + assert parent is not None + slot = self.reserved(fmt, depth, section, parent) + detached = slot is None and lst.family == "enumerated" and chunk == 0 and self.detaches(depth, parent, placed) + if slot is not None: + abstract, ilvl = slot + elif detached: + abstract, ilvl = self.abstract("detached"), depth + else: + abstract, ilvl = self.abstract(lst.family if chunk == 0 else f"{lst.family}+{chunk}"), rem + cause: tuple[str, ...] = () + # Nested in an item: that item's num, if it is live, of the same kind and Word counts the list from where the + # resolver does (a sibling list continues, a list in a new item restarts); otherwise a num of its own. + if (placed is not None and parent_live and placed.abstract is abstract and depth < MAX_DEPTH + and self.sim.peek(placed.num.id, ilvl) == first.number): + num = placed.num + else: + num = self.new_num(abstract, ilvl, first.number or 1) + if depth >= MAX_DEPTH: + cause = ("clamped",) + self.report_depth(depth, first.line) + elif placed is not None and first.number > 1: + cause = ("resync",) + self.shared_counter( + "A list that goes on from an earlier list in the same item (the resolver counts them as one) " + "starts at a fixed number in Word: it will not follow items added or removed before it.", + first.line) + elif placed is not None: + cause, message = _RESTARTS[self.restart_class(parent, placed, abstract, deeper=chunk > 0 and rem == 0)] + self.report("render-docx-static-restart", "info", message, first.line) + if slot is not None and abstract.kind == "headings" and depth > 0: + cause += ("reserved-level",) + own_only = detached or chunk > 0 # the levels of these abstractNums show each level's own counter alone + lost = self.lost_counters(fmt, section, parent) if own_only else [] + fallback = bool(lost) if own_only else shows and slot is None + if own_only and lost: + self.detached_label(fmt, abstract, ilvl, "list item", lost, placed, first) + elif fallback: + self.fallback_label(fmt, lst.family, min(depth, 8), "list item", first.line) + parent_des = self.desig.get(parent, []) if parent is not None else [] + section_des = self.desig[section.unit] if section is not None and section.unit is not None else [] + for index, item in enumerate(lst.items): + assert item.unit is not None and item.number is not None + self.parent[item.unit] = item.parent_unit + item_cause = cause if index == 0 else () + if depth >= MAX_DEPTH and "clamped" not in item_cause: + item_cause += ("clamped",) + if slot is not None and depth > 0 and "reserved-level" not in item_cause: + item_cause += ("reserved-level",) + if fallback: + item_cause += ("label-fallback",) + used = self.numbered(item, "item", num, abstract, ilvl, counter=counter, group=("list", first.unit), + cause=item_cause) + piece = fill_tokens(fmt.ref, n=[self.counted(item, fmt.counter)], section=section_des) + self.desig[item.unit] = extend_tokens(parent_des, piece, textual=self.textual) + self.placed[item.unit] = _Placed(used, abstract) + for child in item.blocks: + self.block(child, section, quoted) + + def bullets(self, lst: List, section: Section | None, quoted: bool) -> None: + """Bullets count nothing, so every bullet list of the document at one stretch of nine levels is numbered by + one num; a list nested deeper goes on in the next abstractNum, as the other families do.""" + reached = min(lst.depth, MAX_DEPTH - 1) + chunk, ilvl = divmod(reached, _LEVELS) + kind = "bullet" if chunk == 0 else f"bullet+{chunk}" + abstract = self.abstract(kind) + if kind not in self.bullet_nums: + self.bullet_nums[kind] = self.new_num(abstract) + bullet_num = self.bullet_nums[kind] + parent = lst.items[0].parent_unit + cause: tuple[str, ...] = () + if lst.depth >= MAX_DEPTH: + cause = ("clamped",) + self.report_depth(lst.depth, lst.items[0].line) + for item in lst.items: + assert item.unit is not None + self.parent[item.unit] = item.parent_unit + self.nodes[item.unit] = item + self.desig[item.unit] = self.desig.get(parent, []) if parent is not None else [] + self.units[item.unit] = UnitPlan(item.unit, "item", bullet_num.id, ilvl, True, None, None, "", "", + None, cause, (), None) + self.placed[item.unit] = _Placed(bullet_num, abstract) + for child in item.blocks: + self.block(child, section, quoted) + + +def plan_docx(tree: RenderTree, *, file: str | None = None, text_switch: TextSwitch = "documented", + text_width: int = DEFAULT_TEXT_WIDTH) -> tuple[DocxPlan, list[Diagnostic]]: + """The numbering plan of *tree* for Word, and the diagnostics only planning can produce (all + ``render-docx-*``, in the order of their lines, naming *file*). Pure: the same tree and *text_switch* (the style's + ``docx.reference_text_switch``: what a reference may expect of Word's ``\\t`` switch, see + :func:`.docx_cover.suppressed`) and *text_width* (the page's, in twips: nested lists stop stepping in at 60% of it) + give an equal plan, and ``tree.formats`` carries the style's numbering. A style + Word's numbering cannot express (:func:`docx_style_errors`) is refused before rendering (ADR 0006), so it never + reaches here.""" + return _Planner(tree, file, text_switch, text_width).run() diff --git a/src/legaldown_render/resolve/numbering.py b/src/legaldown_render/resolve/numbering.py index de95a8e..fcfadbd 100644 --- a/src/legaldown_render/resolve/numbering.py +++ b/src/legaldown_render/resolve/numbering.py @@ -3,7 +3,8 @@ import re -from ..style.model import LevelFormat, Numbering +from ..style.model import LevelFormat, Numbering, Style +from ..tree import NumberingFormats _ROMAN = ( (1000, "m"), (900, "cm"), (500, "d"), (400, "cd"), (100, "c"), (90, "xc"), @@ -94,3 +95,17 @@ def extend(designation: str, piece: str, *, textual: bool) -> str: if textual: return f"{designation} {piece.lstrip('.')}" return designation + piece + + +def formats_of(style: Style) -> NumberingFormats: + """The numbering formats *style* gives (``RenderTree.formats``): those the resolver writes labels and + designations with, and the DOCX planner builds its numbering from.""" + enumeration = style.enumeration + return NumberingFormats( + scheme=style.numbering.scheme, + headings=() if style.numbering.scheme == "none" else heading_levels(style.numbering), + enumerated=tuple(enumeration.levels) if enumeration.enabled or enumeration.ordered == "enumerate" else (), + renumber=RENUMBERED if enumeration.ordered == "renumber" else None, + paragraphs=style.paragraphs.format if style.paragraphs.numbered else None, + reference=style.references.format, + ) diff --git a/src/legaldown_render/resolve/resolver.py b/src/legaldown_render/resolve/resolver.py index fd406b5..7b849b7 100644 --- a/src/legaldown_render/resolve/resolver.py +++ b/src/legaldown_render/resolve/resolver.py @@ -85,7 +85,9 @@ Inline, Link, List, + ListFamily, ListItem, + NumberingFormats, Paragraph, PartyInfo, Quote, @@ -105,7 +107,7 @@ unlinked, ) from ..urls import file_reference_problem, is_empty_path, is_link_target, printable -from .numbering import RENUMBERED, extend, fill, format_counter, heading_levels +from .numbering import RENUMBERED, extend, fill, format_counter, formats_of, heading_levels from .values import DURATION_UNITS, Formatter #: Definition anchors are prefixed with a character no identifier can hold, @@ -224,6 +226,9 @@ class _Target: enumerated: bool = True #: The numbered unit the anchor is on, for render-ref-ambiguous. slot: _Slot | None = None + #: The id (``Section.unit``, ``ListItem.unit``, ``Paragraph.unit``) of the + #: section, item or paragraph the anchor is on. + unit: str | None = None #: The section the anchor is in (a section's own index), None in the preamble. section_index: int | None = None #: For an item or paragraph: the (``ref`` template, counter text) of each @@ -315,6 +320,8 @@ def __init__( self._section_texts: list[str] = [] # The section being structured, for the targets it holds. self._section_index: int | None = None + # How many item and paragraph unit ids have been given out (see _unit). + self._units = 0 self.attachment_anchors: dict[str, str | None] = {} self.section_presences: list[Presence] = [] @@ -350,6 +357,8 @@ def run(self) -> RenderTree: title = resolve(self.tree.title) with self._at(self.key_line("subtitle")): subtitle = resolve(self.tree.subtitle) + effective_date = self._effective_date() + version = self._version() resolved = replace( self.tree, title=title, @@ -357,12 +366,15 @@ def run(self) -> RenderTree: locale=self.formatter.tag, preamble=preamble, sections=sections, - header=self._header(), + header=self._header(effective_date, version), + version=plain_text(version), + effective_date=plain_text(effective_date), sides=self._sides() if self.style.title_block.parties else (), attachments=attachments, attachments_label=self.labels.attachments or "", contents=self._contents(sections, attachments), contents_label=self.labels.contents or "", + formats=self._formats(), colon=self.labels.colon or "", attachments_anchor=ATTACHMENTS_ANCHOR if attachments else "", signatures=self._signatures(), @@ -383,6 +395,7 @@ def _structure_all(self, designations: list[str] | None) -> tuple[tuple[Section, """The preamble and the sections, numbered and labelled; under the ``none`` scheme with the section *designations* worked out by :meth:`_designate`, or else the headings' text alone.""" + self._units = 0 # the same ids on every pass: they follow document order sections = self._number_sections(designations) self._section_index = None preamble = tuple(self._structure(block, section=None, depth=0, presence=ALWAYS) for block in self.tree.preamble) @@ -393,6 +406,17 @@ def _structure_all(self, designations: list[str] | None) -> tuple[tuple[Section, self._section_index = None return tuple(out), preamble + def _formats(self) -> NumberingFormats: + """The numbering formats in effect (``RenderTree.formats``): those the + labels and designations were made with (:meth:`_number_sections`, + :meth:`_structure_list`, :meth:`_number_paragraph`).""" + return formats_of(self.style) + + def _unit(self) -> str: + """The unit id of the next item or paragraph, in document order.""" + self._units += 1 + return f"u{self._units}" + # -- designations under the none scheme (§13.3) ------------------------------ def _reads_references(self, sections: tuple[Section, ...]) -> bool: @@ -551,7 +575,7 @@ def _contents( return () entries = [ ContentsEntry(section.level, section.label, unlinked(section.title), section.anchor, - section.condition_label) + section.condition_label, section.unit) for section in sections if section.level <= settings.depth ] @@ -643,6 +667,8 @@ def _number_sections(self, designations: list[str] | None = None) -> list[Sectio """ levels = heading_levels(self.style.numbering) presences: dict[int, Presence] = {} # level -> presence of the open section + open_units: list[tuple[int, str]] = [] # (level, unit) of the sections a section is inside + first_unit: dict[str, str] = {} # section number -> unit of the first section that has it out: list[Section] = [] for index, (section, indexed) in enumerate(zip(self.tree.sections, self.result.index.sections, strict=True)): # The heading's own level, clamped to 1-5 (§4.1), decides which @@ -672,9 +698,19 @@ def _number_sections(self, designations: list[str] | None = None) -> list[Sectio # Sections with one number are alternatives (the validator's). slot = _Slot(_SECTIONS, indexed.number, presences[level], "section", section.identifier) self._record(designation, slot) - self._register(section.identifier, _Target(designation, anchor, slot=slot, section_index=index)) - out.append(replace(section, level=min(len(parts), 5), label=label, designation=designation, - anchor=anchor)) + unit = _section_unit(index) + self._register(section.identifier, _Target(designation, anchor, slot=slot, section_index=index, unit=unit)) + rendered = min(len(parts), 5) + # The section a section is under is the nearest one above it with + # a smaller number of parts: a skipped level is no section. + while open_units and open_units[-1][0] >= rendered: + open_units.pop() + parent = open_units[-1][1] if open_units else None + open_units.append((rendered, unit)) + first = first_unit.setdefault(indexed.number, unit) + out.append(replace(section, level=rendered, label=label, designation=designation, anchor=anchor, + unit=unit, number=None if self.textual else parts[-1], parent_unit=parent, + shares_number_with=first if first != unit and not self.textual else None)) return out def _presence(self, condition: str) -> Presence: @@ -730,8 +766,9 @@ def _structure_section(self, section: Section, presence: Presence) -> tuple[Bloc if isinstance(block, Paragraph) and block.top_level: # Alternative paragraphs share a number, as sections do (§15.8). own = self._own_presence(block.condition, presence) - number = paragraphs.next(block.anchor_id, own) - block = self._number_paragraph(block, section, paragraphs, number, + unit = self._unit() + number, shares = paragraphs.next(block.anchor_id, own, unit) + block = self._number_paragraph(block, section, paragraphs, number, unit, shares, presence if own is None else own) out.append(self._structure(block, section=section, depth=0, presence=presence)) return tuple(out) @@ -743,7 +780,7 @@ def _own_presence(self, condition: str, enclosing: Presence) -> Presence | None: return enclosing | own if own else None def _number_paragraph(self, block: Paragraph, section: Section, counter: _Counter, number: int, - presence: Presence) -> Paragraph: + unit: str, shares: str | None, presence: Presence) -> Paragraph: numbering = self.style.paragraphs base = section.designation or "" if numbering.numbered: @@ -753,14 +790,17 @@ def _number_paragraph(self, block: Paragraph, section: Section, counter: _Counte label = fill(fmt.label, n=n, section=base, path=designation) slot = _Slot(counter, number, presence, "paragraph", block.anchor_id) target = _Target(designation, None, slot=slot, section_index=self._section_index, - parts=((fmt.ref, n),)) + parts=((fmt.ref, n),), unit=unit) self._record(designation, slot) else: - label, target = None, _Target(base, None, enumerated=False, section_index=self._section_index) + label = None + target = _Target(base, None, enumerated=False, section_index=self._section_index, unit=unit) anchor = self._anchor(block.anchor_id) if block.anchor_id: self._register(block.anchor_id, replace(target, anchor=anchor)) - return replace(block, label=label, anchor=anchor) + return replace(block, label=label, anchor=anchor, unit=unit, parent_unit=section.unit, + number=number if numbering.numbered else None, + shares_number_with=shares if numbering.numbered else None) def _structure(self, block: Block, *, section: Section | None, depth: int, presence: Presence, quoted: bool = False) -> Block: @@ -769,7 +809,8 @@ def _structure(self, block: Block, *, section: Section | None, depth: int, prese match block: case List(): return self._structure_list(block, section=section, depth=depth, - parent=section.designation if section else "", presence=presence, + parent=section.designation if section else "", + parent_unit=section.unit if section else None, presence=presence, quoted=quoted) case Quote(blocks=blocks): return replace(block, blocks=tuple(self._structure(child, section=section, depth=depth, @@ -784,20 +825,25 @@ def _structure(self, block: Block, *, section: Section | None, depth: int, prese return block def _structure_list(self, block: List, *, section: Section | None, depth: int, parent: str, - presence: Presence, counters: dict[tuple[str, str] | None, _Counter] | None = None, + parent_unit: str | None, presence: Presence, + counters: dict[tuple[str, str] | None, _Counter] | None = None, quoted: bool = False, parts: tuple[tuple[str, str], ...] = ()) -> List: """*block* numbered under *parent*. Sibling lists nested in the same item share *counters*, one per way of writing a designation (counter style and reference form): a second list whose designations would - read like the first's goes on from it, so none repeats.""" + read like the first's goes on from it, so none repeats. *parent_unit* is + the unit of *parent*: the section, or the item the list is nested in.""" enumeration = self.style.enumeration fmt: LevelFormat | None + family: ListFamily = "enumerated" if block.ordered and enumeration.ordered == "renumber": - fmt = RENUMBERED + fmt, family = RENUMBERED, "renumber" elif block.ordered or enumeration.enabled: fmt = enumeration.levels[depth % len(enumeration.levels)] if enumeration.levels else None else: fmt = None + if fmt is None: + family = "bullet" base = section.designation if section else "" items: list[ListItem] = [] if counters is None: @@ -807,7 +853,8 @@ def _structure_list(self, block: List, *, section: Section | None, depth: int, p counter = counters.setdefault(key, _Counter(self._exclusive)) counter.new_list() for item in block.items: - index = counter.next(item.anchor_id, self._own_presence(item.condition, presence)) + unit = self._unit() + index, shares = counter.next(item.anchor_id, self._own_presence(item.condition, presence), unit) # An item's nested blocks are present only when the item is. inner = presence | self._presence(item.condition) if fmt is not None: @@ -815,27 +862,31 @@ def _structure_list(self, block: List, *, section: Section | None, depth: int, p designation = extend(parent, fill(fmt.ref, n=n, section=base), textual=self.textual) label = fill(fmt.label, n=n, section=base, path=designation) item_parts = (*parts, (fmt.ref, n)) - target = _Target(designation, None, section_index=self._section_index, parts=item_parts) + target = _Target(designation, None, section_index=self._section_index, parts=item_parts, + unit=unit) if not quoted: slot = _Slot(counter, index, inner, "item", item.anchor_id) target = replace(target, slot=slot) self._record(designation, slot) else: designation, label, item_parts = parent, None, parts - target = _Target(base, None, enumerated=False, section_index=self._section_index) + target = _Target(base, None, enumerated=False, section_index=self._section_index, unit=unit) anchor = self._anchor(item.anchor_id) if item.anchor_id and section is not None: self._register(item.anchor_id, replace(target, anchor=anchor)) siblings: dict[tuple[str, str] | None, _Counter] = {} children = tuple( - self._structure_list(child, section=section, depth=depth + 1, parent=designation, presence=inner, - counters=siblings, quoted=quoted, parts=item_parts) + self._structure_list(child, section=section, depth=depth + 1, parent=designation, + parent_unit=unit, presence=inner, counters=siblings, quoted=quoted, + parts=item_parts) if isinstance(child, List) else self._structure(child, section=section, depth=depth + 1, presence=inner, quoted=quoted) for child in item.blocks ) - items.append(replace(item, blocks=children, label=label, anchor=anchor)) - return List(block.ordered, tuple(items), enumerated=fmt is not None) + items.append(replace(item, blocks=children, label=label, anchor=anchor, unit=unit, + number=index if fmt is not None else None, parent_unit=parent_unit, + shares_number_with=shares if fmt is not None else None)) + return List(block.ordered, tuple(items), enumerated=fmt is not None, family=family, depth=depth, format=fmt) # -- blocks --------------------------------------------------------------- @@ -962,7 +1013,8 @@ def _ref(self, directive: Directive, target_id: str) -> Inline: designation = _compose(target.parts, self._cut_text(self._section_texts[target.section_index])) else: designation = self._cut_text(designation) - return CrossRef(self._reference_text(designation), target.anchor or "") + return CrossRef(self._reference_text(designation), target.anchor or "", target.unit, + None if target.section_index is None else _section_unit(target.section_index)) def _cut_text(self, text: str) -> str: """*text*, a heading's designation, as a reference shows it: at most @@ -1332,15 +1384,28 @@ def _plain_value(self, text: str, *key: Any) -> str: def _frontmatter(self, text: str) -> tuple[Inline, ...]: return self._resolve_inlines(plain_inlines(text)) - def _header(self) -> tuple[tuple[str, tuple[Inline, ...]], ...]: + def _effective_date(self) -> tuple[Inline, ...]: + """The effective date as the title block shows it (a valid date formatted, any + other text resolved), or nothing when the document has none.""" + if not self.metadata.effective_date: + return () + with self._at(self.key_line("effective_date")): + return self._date_or_text(self.metadata.effective_date) + + def _version(self) -> tuple[Inline, ...]: + """The version as the title block shows it, or nothing when the document has none.""" + if not self.metadata.version: + return () + with self._at(self.key_line("version")): + return self._frontmatter(self.metadata.version) + + def _header(self, effective_date: tuple[Inline, ...], + version: tuple[Inline, ...]) -> tuple[tuple[str, tuple[Inline, ...]], ...]: rows: list[tuple[str, tuple[Inline, ...]]] = [] - metadata = self.metadata - if self.style.title_block.effective_date and metadata.effective_date: - with self._at(self.key_line("effective_date")): - rows.append((self.labels.effective_date or "", self._date_or_text(metadata.effective_date))) - if self.style.title_block.version and metadata.version: - with self._at(self.key_line("version")): - rows.append((self.labels.version or "", self._frontmatter(metadata.version))) + if self.style.title_block.effective_date and effective_date: + rows.append((self.labels.effective_date or "", effective_date)) + if self.style.title_block.version and version: + rows.append((self.labels.version or "", version)) return tuple(rows) def _date_or_text(self, value: str) -> tuple[Inline, ...]: @@ -1557,6 +1622,11 @@ def _references(self) -> None: } +def _section_unit(index: int) -> str: + """The unit id of the section at *index* in document order (``Section.unit``).""" + return f"s{index + 1}" + + #: What numbers sections, in a unit's slot: the validator (its numbers). _SECTIONS = object() @@ -1565,14 +1635,18 @@ class _Counter: """Numbers sibling units in order (§13.2, §15.8). A unit joins the previous unit's number when it is an alternative to it: the same identifier, and a presence that excludes the presence of every unit - already holding that number (§15.4). Only adjacent units are compared, - as the validator does for sections.""" + already holding that number (§15.4). Only consecutive units *of this + counter* are compared, as the validator does for sections: for the + top-level paragraphs of a section, a list or table between two of them + does not separate them.""" def __init__(self, exclusive: Callable[[Presence, Presence], bool]) -> None: self.exclusive = exclusive self.count = 0 self.identifier = "" self.holders: list[Presence] = [] + # The unit that first held the current number (see next). + self.first_unit: str | None = None def new_list(self) -> None: """Go on counting in another list: its first unit is no alternative @@ -1580,17 +1654,20 @@ def new_list(self) -> None: self.identifier = "" self.holders = [] - def next(self, identifier: str, presence: Presence | None) -> int: - """The number for the next unit. *presence* is None for a unit with - no valid condition of its own, which is never an alternative.""" + def next(self, identifier: str, presence: Presence | None, unit: str) -> tuple[int, str | None]: + """The number for the next unit, *unit*, and the unit that first held + it when *unit* is a later alternative, else None. *presence* is None + for a unit with no valid condition of its own, which is never an + alternative.""" if (presence is not None and identifier and identifier == self.identifier and all(self.exclusive(holder, presence) for holder in self.holders)): self.holders.append(presence) - return self.count + return self.count, self.first_unit self.count += 1 self.identifier = identifier if presence is not None else "" self.holders = [presence] if presence is not None else [] - return self.count + self.first_unit = unit + return self.count, None class _Slot(NamedTuple): diff --git a/src/legaldown_render/style/__init__.py b/src/legaldown_render/style/__init__.py index bbee595..7f30c65 100644 --- a/src/legaldown_render/style/__init__.py +++ b/src/legaldown_render/style/__init__.py @@ -5,6 +5,7 @@ Attachments, Contents, Definitions, + Docx, Enumeration, HeadingStyle, Html, @@ -12,6 +13,7 @@ LevelFormat, Numbering, Page, + PageText, Paragraphs, Placeholders, References, @@ -28,6 +30,7 @@ "Attachments", "Contents", "Definitions", + "Docx", "Enumeration", "HeadingStyle", "Html", @@ -35,6 +38,7 @@ "LevelFormat", "Numbering", "Page", + "PageText", "Paragraphs", "Placeholders", "References", diff --git a/src/legaldown_render/style/docx.py b/src/legaldown_render/style/docx.py new file mode 100644 index 0000000..d3bc135 --- /dev/null +++ b/src/legaldown_render/style/docx.py @@ -0,0 +1,429 @@ +"""A style's presentation settings as the DOCX writer reads them: Word units. + +The style template speaks CSS (``25mm``, ``1.15em``, ``#1a1a1a``, +``'Georgia, serif'``), because the HTML writer passes the values on. Word has +twips, half-points and six-digit colours, so :func:`docx_settings` converts +every setting the DOCX writer uses, once, and reports **all** that it cannot +as one :class:`~..errors.StyleError` (docs/decisions/0006: a configuration +problem raises before rendering, and the render API calls it before the +document is touched). What it converts: + +* Lengths: ``mm``, ``cm``, ``in``, ``pt``, ``pc`` and ``px`` are absolute + (96 px to the inch, as CSS has it). ``em`` and ``rem`` are relative to + ``typography.font_size``, which itself is relative to the 12 pt (16 px) a + browser starts from when it is written in them. ``%``, ``ch`` and ``ex`` + depend on a layout or a font Word does not give the renderer, so they are an + error for DOCX (HTML takes them). +* ``page.size`` and ``page.margin``: paper names, one or two lengths, an + orientation; one to four margin lengths as CSS shorthand. +* Colours: ``#rgb``, ``#rrggbb`` (with or without an alpha digit pair, which Word + has no use for), ``rgb()``, ``hsl()``, and the sixteen basic names with + ``orange``; any other name is an error asking for a hexadecimal colour. +* The first family of ``typography.font_family``. +* The numbering labels, which Word's level text must be able to write (``resolve.docx_plan.docx_style_errors``). + +A message names the setting and what is wrong with it and never quotes the +value, so it is safe to show for a style from somebody else. +""" +from __future__ import annotations + +import colorsys +import math +import re +from dataclasses import dataclass +from typing import Literal + +from ..errors import StyleError +from ..resolve.docx_plan import docx_style_errors +from .model import Style + +#: A browser starts from 16 px; the size of text written in ``em`` or ``rem`` +#: before ``typography.font_size`` says otherwise. +BASE_FONT_POINTS = 12.0 + +_TWIPS_PER_POINT = 20 +#: Twips per unit of the absolute CSS lengths: 1 in = 72 pt = 96 px = 25.4 mm. +_ABSOLUTE = { + "pt": 20.0, + "pc": 240.0, + "px": 15.0, + "in": 1440.0, + "cm": 1440.0 / 2.54, + "mm": 1440.0 / 25.4, +} +_RELATIVE = ("em", "rem") +#: Units that cannot be a Word measure, as the message says. +_REFUSED = ("%", "ch", "ex") + +_LENGTH_RE = re.compile(r"^([+-]?(?:[0-9]+(?:\.[0-9]*)?|\.[0-9]+))(em|rem|ex|ch|pt|pc|px|mm|cm|in|%)?$") +_NUMBER_RE = re.compile(r"^[+-]?(?:[0-9]+(?:\.[0-9]*)?|\.[0-9]+)$") + +#: Paper sizes (width, height) in twips, portrait: ISO A and B in millimetres +#: (CSS's B is ISO B), the American ones in inches. +_PAPER = { + "a3": (16838, 23811), + "a4": (11906, 16838), + "a5": (8391, 11906), + "b4": (14173, 20013), + "b5": (9979, 14173), + "letter": (12240, 15840), + "legal": (12240, 20160), + "ledger": (15840, 24480), +} +_ORIENTATIONS = ("portrait", "landscape") + +#: The largest size Word accepts for a font (``ST_HpsMeasure``), in points. +_MAX_FONT_POINTS = 1638 +#: The largest body text: the title is 1.6 times it and must stay within :data:`_MAX_FONT_POINTS`. +_MAX_BODY_POINTS = 1023 +#: Word's limit for a page edge, a margin or a line (22 inches), in twips; and the least page edge. +_MAX_TWIPS = 31680 +_MIN_PAGE_TWIPS = 144 +#: The least width and height of the text area, so that no tab stop or text width is negative or absurd. +_MIN_TEXT_TWIPS = 720 +#: A number of more characters than this is no length (and is not converted: ``float`` of a +#: few hundred digits is infinity); a length over this many twips is refused before it is used. +_MAX_NUMBER_CHARS = 32 +_MAX_LENGTH_TWIPS = 10_000_000 + +_BASIC_COLORS = { + "black": "000000", "silver": "C0C0C0", "gray": "808080", "grey": "808080", "white": "FFFFFF", + "maroon": "800000", "red": "FF0000", "purple": "800080", "fuchsia": "FF00FF", "magenta": "FF00FF", + "green": "008000", "lime": "00FF00", "olive": "808000", "yellow": "FFFF00", "navy": "000080", + "blue": "0000FF", "teal": "008080", "aqua": "00FFFF", "cyan": "00FFFF", "orange": "FFA500", +} + +#: What the families CSS names generically are in a Word document. +_GENERIC_FAMILIES = { + "serif": "Times New Roman", "sans-serif": "Arial", "system-ui": "Arial", "ui-sans-serif": "Arial", + "ui-serif": "Times New Roman", "monospace": "Courier New", "ui-monospace": "Courier New", + "cursive": "Times New Roman", "fantasy": "Times New Roman", +} +_DEFAULT_FAMILY = "Times New Roman" + + +@dataclass(frozen=True, slots=True) +class PageGeometry: + """The page, in twips (1/1440 inch).""" + + width: int + height: int + top: int + right: int + bottom: int + left: int + #: Distance of the running header from the top edge, and of the footer from the bottom. + header: int + footer: int + landscape: bool + + @property + def text_width(self) -> int: + """The width between the margins: where a right tab stop goes.""" + return self.width - self.left - self.right + + +@dataclass(frozen=True, slots=True) +class HeadingFormat: + """One heading level's character and paragraph formatting.""" + + #: Half-points, the unit of ``w:sz``. + half_points: int + bold: bool + italic: bool + caps: bool + small_caps: bool + align: str + + +@dataclass(frozen=True, slots=True) +class DocxSettings: + """Every presentation setting the DOCX writer reads, in Word's units.""" + + page: PageGeometry + #: The first family of ``typography.font_family``. + font: str + #: Body text size, in half-points. + half_points: int + #: ``w:spacing/@w:line`` and its rule (``auto``: 240 is one line; ``atLeast``: twips). + line: int + line_rule: str + #: Six hexadecimal digits. + color: str + link_color: str + justify: bool + headings: dict[int, HeadingFormat] + update_fields_on_open: bool + #: ``off``, ``documented`` or ``guess``: what a cross-reference may expect of Word's ``\t`` switch. + reference_text_switch: Literal["off", "documented", "guess"] + + @property + def points(self) -> float: + return self.half_points / 2 + + +class _Problems: + """The problems found while converting, each naming its setting.""" + + def __init__(self) -> None: + self.items: list[str] = [] + + def add(self, key: str, problem: str) -> None: + message = f"{key}: {problem}" + if message not in self.items: + self.items.append(message) + + +def docx_settings(style: Style, *, trusted: bool = True) -> DocxSettings: + """*style*'s presentation settings in Word units. Raises + :class:`~..errors.StyleError` listing every setting the DOCX writer cannot + express (see the module docstring); *trusted* only says how the style is + named in it.""" + problems = _Problems() + typography = style.typography + + font_points = _font_points(typography.font_size, "typography.font_size", BASE_FONT_POINTS, problems, + _MAX_BODY_POINTS) + font_points = font_points if font_points is not None else 11.0 + half_points = round(font_points * 2) + + line, rule = _line_height(typography.line_height, font_points, problems) + page = _page(style.page.size, style.page.margin, font_points, problems) + color = _color(typography.color, "typography.color", problems) + link_color = _color(typography.link_color, "typography.link_color", problems) + + headings: dict[int, HeadingFormat] = {} + for level in range(1, 6): + heading = style.headings.get(level) + if heading is None: + continue + key = f"headings.{level}" + size = _font_points(heading.size, f"{key}.size", font_points, problems) + bold = _bold(heading.weight, f"{key}.weight", problems) + headings[level] = HeadingFormat( + half_points=round((size if size is not None else font_points) * 2), bold=bold, italic=heading.italic, + caps=heading.transform == "uppercase", small_caps=heading.transform == "small-caps", align=heading.align) + + problems.items.extend(docx_style_errors(style)) + if problems.items: + raise StyleError(f"'{style.name}'" if trusted else "'untrusted style'", problems.items) + return DocxSettings( + page=page, font=_family(typography.font_family), half_points=half_points, line=line, line_rule=rule, + color=color, link_color=link_color, justify=typography.justify, headings=headings, + update_fields_on_open=style.docx.update_fields_on_open, + reference_text_switch=style.docx.reference_text_switch) + + +# -- Lengths ---------------------------------------------------------------------- + + +def _length(value: str, key: str, em_points: float, problems: _Problems) -> int | None: + """*value*, a CSS length, in twips; None (with a problem) when Word cannot + have it. ``em`` and ``rem`` are *em_points* points.""" + match = _LENGTH_RE.match(value.strip().lower()) + if match is None: + problems.add(key, "is not a length the DOCX writer can read (a number and a unit such as 25mm, 11pt or 1.2em)") + return None + if len(match.group(1)) > _MAX_NUMBER_CHARS: + problems.add(key, "is a number too large for a Word document") + return None + number, unit = float(match.group(1)), match.group(2) + if unit is None: + if number != 0: + problems.add(key, "a length needs a unit (only 0 has none)") + return None + return 0 + if unit in _REFUSED: + problems.add(key, f"uses '{unit}', which a Word document cannot express; use pt, mm, cm, in, px, em or rem " + f"(HTML accepts it)") + return None + twips = number * (em_points * _TWIPS_PER_POINT if unit in _RELATIVE else _ABSOLUTE[unit]) + if not math.isfinite(twips) or abs(twips) > _MAX_LENGTH_TWIPS: + problems.add(key, "is a length too large for a Word document") + return None + return round(twips) + + +def _font_points(value: str, key: str, em_points: float, problems: _Problems, + maximum: int = _MAX_FONT_POINTS) -> float | None: + twips = _length(value, key, em_points, problems) + if twips is None: + return None + points = twips / _TWIPS_PER_POINT + if not 1 <= points <= maximum: + problems.add(key, f"must be between 1pt and {maximum}pt in a Word document") + return None + return points + + +def _line_height(value: str, font_points: float, problems: _Problems) -> tuple[int, str]: + """``w:spacing``'s ``line`` and ``lineRule``: ``normal`` is single spacing, a + number multiplies the font size (``auto``, 240 to the line), a length is + the least height of a line (``atLeast``: ``exact`` would cut off tall text).""" + text = value.strip() + if text == "normal": + return 240, "auto" + if _NUMBER_RE.match(text): + factor = float(text) + if not 0 < factor <= 100: # also false for infinity + problems.add("typography.line_height", "a number must be greater than 0 and at most 100") + return 240, "auto" + return round(factor * 240), "auto" + twips = _length(text, "typography.line_height", font_points, problems) + if twips is None or not 0 < twips <= _MAX_TWIPS: + if twips is not None: + problems.add("typography.line_height", f"must be greater than 0 and at most {_MAX_TWIPS // 20}pt") + return 240, "auto" + return twips, "atLeast" + + +def _page(size: str, margin: str, font_points: float, problems: _Problems) -> PageGeometry: + """The page in twips. Word's limits hold: an edge of 144 to 31680, a margin of 0 to + 31680, and margins that leave a text area of at least 720 in each direction, so a + tab stop at the text width can never be negative.""" + before = len(problems.items) + width, height = _page_size(size, font_points, problems) + top, right, bottom, left = _margins(margin, font_points, problems) + if len(problems.items) == before: # a value that is already wrong has said so + if not all(_MIN_PAGE_TWIPS <= edge <= _MAX_TWIPS for edge in (width, height)): + problems.add("page.size", f"each edge must be between {_MIN_PAGE_TWIPS / 20:g}pt and " + f"{_MAX_TWIPS // 20}pt in a Word document") + elif not all(0 <= edge <= _MAX_TWIPS for edge in (top, right, bottom, left)): + problems.add("page.margin", f"each margin must be between 0 and {_MAX_TWIPS // 20}pt in a Word document") + elif width - left - right < _MIN_TEXT_TWIPS or height - top - bottom < _MIN_TEXT_TWIPS: + problems.add("page.margin", "the margins leave no room for text on the page (page.size)") + return PageGeometry(width=width, height=height, top=top, right=right, bottom=bottom, left=left, + header=max(0, min(720, top // 2)), footer=max(0, min(720, bottom // 2)), + landscape=width > height) + + +def _page_size(value: str, font_points: float, problems: _Problems) -> tuple[int, int]: + """The page's width and height in twips. ``auto`` and an orientation alone + take A4; a paper name is portrait unless it says ``landscape``; one length + is a square, two are width and height. A page wider than it is tall is + landscape.""" + default = _PAPER["a4"] + tokens = value.lower().split() + if not 1 <= len(tokens) <= 2: + problems.add("page.size", "must be a paper size (A3 to A5, B4, B5, letter, legal, ledger), auto, an " + "orientation, or one or two lengths") + return default + if tokens == ["auto"]: + return default + orientation = tokens[-1] if tokens[-1] in _ORIENTATIONS else None + names = tokens[:-1] if orientation else tokens + if not names: + width, height = default + elif len(names) == 1 and names[0] in _PAPER: + width, height = _PAPER[names[0]] + else: + lengths = [_length(token, "page.size", font_points, problems) for token in names] + if None in lengths or any(length <= 0 for length in lengths if length is not None): + if None not in lengths: + problems.add("page.size", "lengths must be greater than 0") + return default + if orientation: + problems.add("page.size", "an orientation goes with a paper size, not with lengths") + width = lengths[0] or 0 + height = lengths[1] if len(lengths) > 1 else width + orientation = None # the lengths say which way the page lies + if (orientation == "landscape" and width < height) or (orientation == "portrait" and width > height): + width, height = height, width + return width, height + + +def _margins(value: str, font_points: float, problems: _Problems) -> tuple[int, int, int, int]: + """Top, right, bottom and left in twips, from one to four lengths as in the + CSS shorthand: what is missing repeats the opposite side.""" + tokens = value.split() + if not 1 <= len(tokens) <= 4: + problems.add("page.margin", "must be one to four lengths") + return (1417,) * 4 + lengths = [_length(token, "page.margin", font_points, problems) for token in tokens] + if None in lengths: + return (1417,) * 4 + top, right, bottom, left = (lengths + lengths[1:2] * (4 - len(lengths)) if len(lengths) == 3 + else lengths * (4 // len(lengths))) + if min(top, right, bottom, left) < 0: + problems.add("page.margin", "must not be negative") + return top, right, bottom, left + + +# -- Colours, weights, fonts -------------------------------------------------------- + + +def _color(value: str, key: str, problems: _Problems) -> str: + """*value*, a CSS colour, as six upper-case hexadecimal digits.""" + text = value.strip().lower() + if text.startswith("#"): + digits = text[1:] + if re.fullmatch(r"[0-9a-f]{3,4}", digits): + return "".join(char * 2 for char in digits[:3]).upper() + if re.fullmatch(r"(?:[0-9a-f]{6}|[0-9a-f]{8})", digits): + return digits[:6].upper() + elif text in _BASIC_COLORS: + return _BASIC_COLORS[text] + else: + function = re.fullmatch(r"(rgba?|hsla?)\(([0-9a-z.%,/ +-]{1,60})\)", text) + if function is not None: + parts = [part for part in re.split(r"[ ,/]+", function.group(2).strip()) if part] + if function.group(1).startswith("rgb") and len(parts) in (3, 4): + channels = [_channel(part) for part in parts[:3]] + if None not in channels: + return "".join(f"{channel:02X}" for channel in channels) # type: ignore[str-format] + elif function.group(1).startswith("hsl") and len(parts) in (3, 4): + rgb = _hsl(parts) + if rgb is not None: + return rgb + problems.add(key, "is a colour the DOCX writer cannot read; write it as #rrggbb (or #rgb, rgb(), hsl(), or one " + "of black, silver, gray, white, maroon, red, purple, fuchsia, green, lime, olive, yellow, " + "navy, blue, teal, aqua, orange)") + return "000000" + + +def _channel(part: str) -> int | None: + try: + number = float(part[:-1]) * 2.55 if part.endswith("%") else float(part) + except ValueError: + return None + return round(min(255.0, max(0.0, number))) if math.isfinite(number) else None + + +def _hsl(parts: list[str]) -> str | None: + try: + hue_degrees = float(parts[0].removesuffix("deg")) + if not (parts[1].endswith("%") and parts[2].endswith("%")): + return None + saturation = float(parts[1][:-1]) + lightness = float(parts[2][:-1]) + except ValueError: + return None + if not all(math.isfinite(number) for number in (hue_degrees, saturation, lightness)): + return None + hue = hue_degrees % 360 / 360 + saturation = min(100.0, max(0.0, saturation)) / 100 + lightness = min(100.0, max(0.0, lightness)) / 100 + red, green, blue = colorsys.hls_to_rgb(hue, lightness, saturation) + return f"{round(red * 255):02X}{round(green * 255):02X}{round(blue * 255):02X}" + + +def _bold(value: str, key: str, problems: _Problems) -> bool: + text = value.strip().lower() + if text in ("bold", "bolder"): + return True + if text in ("normal", "lighter"): + return False + if _NUMBER_RE.match(text): + return float(text) >= 600 + problems.add(key, "is not a weight the DOCX writer can read (normal, bold, or a number)") + return False + + +def _family(value: str) -> str: + """The first family of a CSS ``font-family`` list: its name, unquoted; a + generic family (``serif``, ``monospace``) as the font Word uses for it.""" + match = re.match(r"""\s*(?:"([^"]*)"|'([^']*)'|([^,]*))""", value) + name = next((group for group in (match.groups() if match else ()) if group is not None), "").strip() + if not name: + return _DEFAULT_FAMILY + return _GENERIC_FAMILIES.get(name.lower(), name) diff --git a/src/legaldown_render/style/loader.py b/src/legaldown_render/style/loader.py index ce2eda9..f4bbb57 100644 --- a/src/legaldown_render/style/loader.py +++ b/src/legaldown_render/style/loader.py @@ -209,7 +209,7 @@ def _load(style: Any, overrides: Mapping[str, Any] | None, trusted: bool) -> Sty expanded = _expand_dotted(overrides, trusted) _refuse_meta_overrides(expanded) data = _merge(data, expanded) - return _build(data, source, trusted) + return _build(data, source if trusted else _UNTRUSTED_SOURCE, trusted) def style_to_dict(style: Style) -> dict[str, Any]: diff --git a/src/legaldown_render/style/model.py b/src/legaldown_render/style/model.py index 05ab2e2..d0c4bb1 100644 --- a/src/legaldown_render/style/model.py +++ b/src/legaldown_render/style/model.py @@ -6,7 +6,7 @@ * the **semantic** part — numbering, enumeration, references, values, labels — drives resolution, so it applies identically to every output format; -* the **presentation** part — typography, headings, page, html — is read only +* the **presentation** part — typography, headings, page, html, docx — is read only by writers. Every field has a default, so the built-in ``default`` template is simply @@ -315,13 +315,53 @@ class Typography: justify: bool = False +@dataclass(frozen=True, slots=True) +class PageText: + """The text of one running header or footer: three templates, set at the + left margin, centred, and at the right margin of every page. Each + substitutes the fields in :data:`PAGE_TEXT` (``{page}``, ``{pages}``, + ``{title}``, ``{version}``, ``{date}``); an empty one shows nothing.""" + + left: str = "" + center: str = "" + right: str = "" + + def __bool__(self) -> bool: + return bool(self.left or self.center or self.right) + + @dataclass(frozen=True, slots=True) class Page: - """Print layout. Used by the HTML writer's print stylesheet now, and by - the PDF writer later.""" + """Print layout: the HTML writer's print stylesheet (``size`` and + ``margin``), the DOCX writer's page setup, headers and footers, and the PDF + writer's later. The HTML writer sets no running header or footer yet.""" size: str = css_field("page-size", "A4") margin: str = css_field("margin", "25mm") + #: Running header: empty by default. + header: PageText = field(default_factory=PageText) + #: Running footer: the page number and the page count, centred, by default. + footer: PageText = field(default_factory=lambda: PageText(center="{page} / {pages}")) + + +@dataclass(frozen=True, slots=True) +class Docx: + """Settings of the DOCX writer only (docs/style-templates.md).""" + + #: Ask Word to update every field when it opens the file (``w:updateFields``): + #: the cross-references, the table of contents and its page numbers. Word + #: shows one prompt for it. The fields hold the values the renderer computed + #: (their cached results) either way, so a viewer that never updates fields + #: shows the document correctly, only without the page numbers of the table + #: of contents. + update_fields_on_open: bool = True + #: What Word's ``\t`` switch (``REF \n \t``, which leaves the words of a number out) is taken to do when a + #: cross-reference is built from fields (docs/style-templates.md, "DOCX"). ``off``: never used, so a reference that + #: needs it is fixed text. ``documented``: only where Microsoft's documentation says what Word returns (words before the first counter + #: and after the last are left out, a delimiter between counters stays). ``guess``: also where Word is expected to do + #: what it is not documented to (a period after the letters that lead a number: ``No. 1``, ``Čl. 1``, and words between + #: counters); to be checked in Word. + reference_text_switch: Literal["off", "documented", "guess"] = "documented" @dataclass(frozen=True, slots=True) @@ -371,6 +411,7 @@ class Style: headings: dict[int, HeadingStyle] = field(default_factory=_default_headings) page: Page = field(default_factory=Page) html: Html = field(default_factory=Html) + docx: Docx = field(default_factory=Docx) # --------------------------------------------------------------------------- @@ -405,6 +446,9 @@ class TemplateFields: #: List items and paragraphs also see the containing section's designation. ITEM_LABEL = TemplateFields(frozenset({"n", "path", "section"})) ITEM_REF = TemplateFields(frozenset({"n", "section"}), frozenset({"n"}), "each item must tell itself apart (§13.3)") +#: A running header or footer sees the page counters and the document's title, +#: version and date; it requires nothing, as an empty one shows nothing. +PAGE_TEXT = TemplateFields(frozenset({"page", "pages", "title", "version", "date"})) REFERENCE_FORMAT = TemplateFields( frozenset({"designation"}), frozenset({"designation"}), "every reference would read the same (§13.3)" ) @@ -451,6 +495,9 @@ def template_settings(style: Style) -> Iterator[tuple[str, str, TemplateFields]] yield ("paragraphs.format.ref", style.paragraphs.format.ref, ITEM_REF if style.paragraphs.numbered else _unused(ITEM_REF)) yield "references.format", style.references.format, REFERENCE_FORMAT + for part in ("header", "footer"): + for side in ("left", "center", "right"): + yield f"page.{part}.{side}", getattr(getattr(style.page, part), side), PAGE_TEXT for item in fields(Labels): template = getattr(style.labels, item.name) if template is not None and item.name in LABEL_FIELDS: diff --git a/src/legaldown_render/tree.py b/src/legaldown_render/tree.py index a98051f..2fddfd8 100644 --- a/src/legaldown_render/tree.py +++ b/src/legaldown_render/tree.py @@ -14,10 +14,12 @@ from collections.abc import Callable, Iterator from dataclasses import dataclass, field, replace +from typing import Literal from legaldown.syntax import Directive from .errors import InternalError +from .style.model import LevelFormat # --------------------------------------------------------------------------- # Inlines @@ -96,10 +98,38 @@ class DefinitionSource: @dataclass(frozen=True, slots=True) class CrossRef: - """A resolved ``{{ref:}}``: display text and the output anchor it links to.""" + """A resolved ``{{ref:}}``: display text and the output anchor it links to. + + ``target_unit`` and ``section_unit`` name the units the reference points + at (see :class:`Section`), for a writer that builds the + reference from the numbers instead of using ``text``: + + * ``target_unit`` is the unit the identifier is on: a section, a list item + or a numbered paragraph. When alternatives share the identifier + (§15.4), it is the **first** of them, the one that keeps the anchor, as + ``target`` does. An item or paragraph the style does not number + (``number`` is ``None``) is still the target; ``text`` is then the + designation of its section (§6.3, ``ref-not-enumerated``). + * ``section_unit`` is the section that qualifies the designation: the + section holding the item or paragraph, and for a section target the + target itself (so it equals ``target_unit``). + * Under the ``none`` scheme (§13.3) both are set as above: ``text`` is the + section's heading text with the item or paragraph piece after it, so + the section's number is ``None`` and its title is what the designation + starts from. + + Both are ``None`` in a copy that no longer links (:func:`unlinked`, as in + the table of contents), which names no unit. A reference that does + not resolve is a :class:`FailureMarker`, not a ``CrossRef``, and so is a + ``{{ref:}}`` to an attachment id, which is a broken reference (§5.6); + ``{{attach:}}`` is a :class:`Value` of kind ``attach``, whose ``href`` is + ``"#"`` and the attachment's anchor. + """ text: str target: str + target_unit: str | None = None + section_unit: str | None = None @dataclass(frozen=True, slots=True) @@ -186,6 +216,16 @@ class Paragraph: label: str | None = None anchor: str | None = None condition_label: str | None = None + #: The numbering of a top-level paragraph of a section (see Section): + #: its unit id, counter value (None when the style does not number + #: paragraphs), the section's unit, and the first alternative it shares + #: its number with. Alternatives are the section's top-level paragraphs + #: only: a list or table between two of them does not separate them. + #: Other paragraphs are not units: all four are None. + unit: str | None = None + number: int | None = None + parent_unit: str | None = None + shares_number_with: str | None = None #: Its first line in the document's file, counted from 1 (§16.9), which the #: renderer's diagnostics name; None when the document has no source to #: name a line of. Not part of equality: the same content is equal @@ -202,16 +242,61 @@ class ListItem: label: str | None = None anchor: str | None = None condition_label: str | None = None + #: The item's numbering (see Section): its unit id, counter value (None in + #: a list that is not enumerated), the unit its designation is built on (the + #: section for an item of a list that is not nested in an item; the enclosing + #: item for a nested one; None in the preamble), and the first alternative + #: it shares its number with. The counter value does not always start at 1 + #: in a list: see :class:`List`. + unit: str | None = None + number: int | None = None + parent_unit: str | None = None + shares_number_with: str | None = None #: The item's marker line (see Paragraph.line). line: int | None = field(default=None, compare=False) +#: How a list's items are numbered (§13.2): ``renumber``, an ordered list +#: counted 1., 2., ... whatever its source says; ``enumerated``, items counted +#: in the format of their nesting depth (``enumeration.levels``); ``bullet``, +#: no numbers. +ListFamily = Literal["renumber", "enumerated", "bullet"] + + @dataclass(frozen=True, slots=True) class List: ordered: bool items: tuple[ListItem, ...] #: Set by the resolve stage: whether items carry generated labels. enumerated: bool = False + #: Set by the resolve stage: which numbering the list is under. It is + #: ``bullet`` exactly when ``enumerated`` is False. + family: ListFamily = "bullet" + #: Set by the resolve stage: the nesting depth the list's format was + #: chosen at, from 0 for a list that is not inside an item. It selects the + #: format and nothing else: a list in a quote or drafting note inside an + #: item is one level deeper than the item, yet its items' ``parent_unit`` + #: is the section. Where a list nests, follow ``parent_unit`` (an item, or + #: a section), not ``depth``. + depth: int = 0 + #: Set by the resolve stage: the format its items are numbered in (the + #: counter, ``label`` and ``ref`` templates of §13.2), None for ``bullet``. + #: It is the entry of :attr:`RenderTree.formats` that the list's family + #: and depth select, so that nobody has to select it again. + format: LevelFormat | None = None + + @property + def start(self) -> int | None: + """The number of the first item (``items[0].number``), None when the + list is not enumerated. It is 1 unless the list goes on from a sibling: + lists nested directly in one item share a counter per way of writing a + designation (counter style and ``ref`` template), so that a second + list after a paragraph, or a list of another family with the same + designations, continues the first's numbers ("(c)" after "(a)", "(b)") + and no designation is repeated. Top-level lists, and lists in quotes, + always start at 1. A list has no id of its own: name it by + ``items[0].unit``.""" + return self.items[0].number if self.items else None @dataclass(frozen=True, slots=True) @@ -264,7 +349,43 @@ class Rule: @dataclass(frozen=True, slots=True) class Section: """One heading and the blocks up to the next heading. Sections are kept - flat, in document order; ``level`` gives the hierarchy.""" + flat, in document order; ``level`` gives the hierarchy. + + **Unit numbering.** The resolve stage names the numbered units of the + document (sections, list items, and the top-level paragraphs of a section) + so that a writer can follow the structure behind the numbers without + working out a rule of §13 again. Four fields, the same on :class:`Section`, + :class:`ListItem` and :class:`Paragraph`: + + * ``unit``: an id that is unique in the document and the same on every + render of it: ``s`` for the k-th section (from 1), ``u`` for the + k-th item or paragraph in document order (the preamble first). It is not + the anchor (which alternatives share and some units lack) and carries no + meaning beyond identity. + * ``number``: the unit's own counter value, the one :attr:`RenderTree.formats` + writes with the unit's counter style: the last part of a section's number + ("2.1" gives 1), the position of an item in its list, of a paragraph in its + section. ``None`` for a unit that is not numbered: any section under the + ``none`` scheme, an item of a list that is not enumerated, a paragraph + when the style does not number paragraphs. + * ``parent_unit``: the unit whose designation this one's is built on: for a + section the nearest section above it with a smaller ``level`` (a skipped + level counts as 1 and is not a unit, yet that 1 is part of every + designation below it, and so of the ``{section}`` of an item or paragraph + there); for an item of a list that is not inside an item, and for a + paragraph, the section (None in the preamble); for a nested item, the + item it is nested in. A list in a quote or drafting note, even inside an + item, is built on the section. + * ``shares_number_with``: for a later alternative that shares its number + (§15.8, §15.4) the unit of the **first** unit that has that number, else + None. Items and paragraphs share when they have the same identifier and a + presence that excludes that of every unit already holding the number; + sections share by number (the validator's): the subsections of a later + alternative section repeat the numbers of the earlier one's, whatever + their identifiers. It is set only on a numbered unit; the first + alternative is a unit like any other, and each alternative has a + ``unit`` of its own. + """ #: The heading's level as written; after resolution, how many parts #: its number has (1 for "2", 2 for "2.1"), clamped to 1-5. @@ -279,6 +400,10 @@ class Section: designation: str | None = None anchor: str | None = None condition_label: str | None = None + unit: str | None = None + number: int | None = None + parent_unit: str | None = None + shares_number_with: str | None = None #: The heading's first line (see Paragraph.line). line: int | None = field(default=None, compare=False) @@ -325,6 +450,47 @@ class ContentsEntry: title: tuple[Inline, ...] anchor: str | None condition_label: str | None = None + #: The section's :attr:`Section.unit` for an entry that lists a section, so that a writer keys the entry on the + #: unit and does not count positions; None for the attachments heading and for an attachment. + unit: str | None = None + + +@dataclass(frozen=True, slots=True) +class NumberingFormats: + """The numbering formats in effect, after the style is resolved (§13.1, + §13.2): the same :class:`~legaldown_render.style.model.LevelFormat` the + resolver wrote the labels and designations with, so that a writer builds + its own numbering from the same templates. + + A level's ``counter`` styles its number, ``label`` is what shows next to the + unit, ``ref`` is its piece of a designation. ``{n}`` is the unit's own + number, ``{path}`` the designation up to and including the unit, and + ``{section}`` (items and paragraphs) the designation of the enclosing section + (nothing for a preamble item, and a ``.`` right after it goes too). A + designation is the parent's designation followed by the unit's ``ref`` + filled in; under the ``none`` scheme the heading text stands in for a + section's designation and a space goes before the piece. + """ + + #: The heading scheme: ``decimal``, ``legal-outline``, ``mixed`` or ``none``. + scheme: str = "decimal" + #: One format per heading level, 1 to 5 (§4.1); empty under the ``none`` + #: scheme, where headings are not numbered. The n-th part of a section's + #: number takes the n-th format. + headings: tuple[LevelFormat, ...] = () + #: The formats of ``enumerated`` lists by nesting depth; a list nested + #: deeper than there are formats goes back to the first (``depth % + #: len(enumerated)``). Empty when no list is enumerated this way. + enumerated: tuple[LevelFormat, ...] = () + #: The format of every ``renumber`` list, at any depth; None when ordered + #: lists are enumerated (``enumeration.ordered: enumerate``). + renumber: LevelFormat | None = None + #: The format of a section's top-level paragraphs; None when the style does + #: not number them. + paragraphs: LevelFormat | None = None + #: What a reference shows around the designation (``references.format``, + #: with ``{designation}`` for it). + reference: str = "{designation}" @dataclass(frozen=True, slots=True) @@ -355,6 +521,14 @@ class RenderTree: attachments_anchor: str = "" #: Between a generated label and its value ("Name: ..."), per language. colon: str = ": " + #: The numbering formats the labels and designations were made with. + formats: NumberingFormats = NumberingFormats() + #: The frontmatter's ``version`` and ``effective_date`` as plain text, exactly as + #: the title block shows them (directives resolved, a valid date formatted by the + #: style for the locale), whether or not the style shows them there: the + #: running header and footer of a paged format use them. Empty when absent. + version: str = "" + effective_date: str = "" # --------------------------------------------------------------------------- @@ -506,7 +680,7 @@ def unlinked(inlines: tuple[Inline, ...]) -> tuple[Inline, ...]: out.extend(unlinked(children)) continue case CrossRef(): - inline = replace(inline, target="") + inline = replace(inline, target="", target_unit=None, section_unit=None) case TermRef(): inline = replace(inline, target=None) case DefinedTerm(): diff --git a/src/legaldown_render/writers/__init__.py b/src/legaldown_render/writers/__init__.py index cf77336..8bd28e2 100644 --- a/src/legaldown_render/writers/__init__.py +++ b/src/legaldown_render/writers/__init__.py @@ -2,6 +2,7 @@ (docs/decisions/0003); it only lays out what resolution computed.""" from __future__ import annotations +from .docx import DocxWriter from .html import HtmlWriter, stylesheet from .text import TextWriter @@ -9,6 +10,7 @@ FORMATS: dict[str, tuple[str, ...]] = { "html": ("html", "htm"), "text": ("txt", "text"), + "docx": ("docx",), } @@ -30,4 +32,4 @@ def format_for_path(path: str) -> str | None: return next((name for name, extensions in FORMATS.items() if extension in extensions), None) -__all__ = ["FORMATS", "HtmlWriter", "TextWriter", "file_extension", "format_for_path", "stylesheet"] +__all__ = ["FORMATS", "DocxWriter", "HtmlWriter", "TextWriter", "file_extension", "format_for_path", "stylesheet"] diff --git a/src/legaldown_render/writers/docx/__init__.py b/src/legaldown_render/writers/docx/__init__.py new file mode 100644 index 0000000..aa175af --- /dev/null +++ b/src/legaldown_render/writers/docx/__init__.py @@ -0,0 +1,104 @@ +"""The DOCX writer (§13.6): a resolved tree as a WordprocessingML package. + +It writes OOXML directly, with the standard library only (``zipfile`` and +``xml.etree``): docs/decisions/0008-docx-word-native.md says why. Like every writer it +contains no LegalDown rules (docs/decisions/0003); it lays out what resolution computed +and emits no diagnostics. The package has the parts a Word document needs and nothing that +varies between two renders of one document. + +Layout of this package: + +* :mod:`.ooxml` -- namespaces, the sanitiser, element, run, field, bookmark and hyperlink builders; +* :mod:`.package` -- parts, relationships, content types and the deterministic ZIP; +* :mod:`.styles`, :mod:`.settings`, :mod:`.headers`, :mod:`.properties` -- the parts that come from the style template; +* :mod:`.numbering` -- ``numbering.xml``, from the plan's numbering definitions; +* :mod:`.body` and :mod:`.runs` -- the body of the document and its inlines, laid out as the plan says; +* :mod:`.matter` -- the front and back matter: header rows and parties, the table of contents, the attachments and + the signature blocks. +""" +from __future__ import annotations + +from collections.abc import Mapping + +from ...resolve.docx_plan import DocxPlan, plan_docx +from ...style import Style +from ...style.docx import DocxSettings, docx_settings +from ...tree import RenderTree +from . import package as pkg +from .body import BodyWriter, document_title +from .headers import build_footer, build_header +from .matter import MatterSettings +from .numbering import numbering_part +from .ooxml import Bookmarks, el, serialize +from .properties import build_app, build_core +from .settings import build_settings +from .styles import build_styles, language_tag + +__all__ = ["DocxWriter"] + + +class DocxWriter: + """Writes a resolved tree as a ``.docx`` file. + + *style* is the style template (the DOCX settings are converted from it, see + :func:`~legaldown_render.style.docx.docx_settings`, which raises + :class:`~legaldown_render.errors.StyleError` for one Word cannot express); + *generator* names the program in the document's properties. The running header and + footer take ``{title}``, ``{version}`` and ``{date}`` from the tree (its title, and the + ``version`` and ``effective_date`` the resolver formatted as the title block shows them). + """ + + extension = "docx" + media_type = "application/vnd.openxmlformats-officedocument.wordprocessingml.document" + + def __init__(self, style: Style, *, generator: str = "", settings: DocxSettings | None = None) -> None: + self.style = style + self.generator = generator or "legaldown-render" + self.settings = settings if settings is not None else docx_settings(style) + + def write(self, tree: RenderTree, plan: DocxPlan | None = None) -> bytes: + """The ``.docx`` file for *tree*, numbered, bookmarked and cross-referenced as *plan* says. Without + one the tree is planned here (:func:`~legaldown_render.resolve.docx_plan.plan_docx`), whose diagnostics + are then lost: :func:`~legaldown_render.render` plans first and reports them.""" + if plan is None: + plan = plan_docx(tree, text_switch=self.settings.reference_text_switch, + text_width=self.settings.page.text_width)[0] + settings = self.settings + language = language_tag(tree.language, self.style.locale) + title = document_title(tree) + values: Mapping[str, str] = {"title": title, "version": tree.version, "date": tree.effective_date} + text_width = settings.page.text_width + + package = pkg.Package() + package.relationships.add(pkg.REL_OFFICE_DOCUMENT, "word/document.xml") + package.relationships.add(pkg.REL_CORE_PROPERTIES, "docProps/core.xml") + package.relationships.add(pkg.REL_EXTENDED_PROPERTIES, "docProps/app.xml") + + rels = pkg.Relationships() + rels.add(pkg.REL_STYLES, "styles.xml") + rels.add(pkg.REL_NUMBERING, "numbering.xml") + rels.add(pkg.REL_SETTINGS, "settings.xml") + header_id = footer_id = None + page = self.style.page + if page.header: + header_id = rels.add(pkg.REL_HEADER, "header1.xml") + if page.footer: + footer_id = rels.add(pkg.REL_FOOTER, "footer1.xml") + + matter = MatterSettings(text_width, settings.points, self.style.contents.depth, self.style.attachments.separator) + body = BodyWriter(tree, plan, Bookmarks(), rels, text_width, settings.link_color, matter).body( + settings.page, header_id=header_id, footer_id=footer_id) + document = el("document", None, body) + package.add("docProps/core.xml", pkg.CT_CORE_PROPERTIES, build_core(title, language)) + package.add("docProps/app.xml", pkg.CT_EXTENDED_PROPERTIES, build_app(self.generator)) + package.add("word/document.xml", pkg.CT_DOCUMENT, serialize(document), rels) + package.add("word/styles.xml", pkg.CT_STYLES, build_styles(self.style, settings, language, + heading_num_id=plan.heading_num_id)) + package.add("word/numbering.xml", pkg.CT_NUMBERING, numbering_part(plan)) + package.add("word/settings.xml", pkg.CT_SETTINGS, + build_settings(update_fields=settings.update_fields_on_open)) + if header_id: + package.add("word/header1.xml", pkg.CT_HEADER, build_header(page.header, values, text_width)) + if footer_id: + package.add("word/footer1.xml", pkg.CT_FOOTER, build_footer(page.footer, values, text_width)) + return package.to_bytes() diff --git a/src/legaldown_render/writers/docx/body.py b/src/legaldown_render/writers/docx/body.py new file mode 100644 index 0000000..5ba8672 --- /dev/null +++ b/src/legaldown_render/writers/docx/body.py @@ -0,0 +1,389 @@ +"""The body of the document: ``w:body`` and its closing ``w:sectPr``. + +:class:`BodyWriter` lays out the resolved tree as the plan says (docs/decisions/0008). It decides +nothing: which paragraph carries which number, which bookmarks wrap it and what a cross-reference +is made of are the plan's (:mod:`~legaldown_render.resolve.docx_plan`), looked up by the unit id +the tree gives each section, list item and top-level paragraph: + +* a unit's paragraph gets ``w:numPr`` when its :class:`~legaldown_render.resolve.docx_plan.UnitPlan` + says it is ``explicit`` (a heading otherwise takes the numbering of its style), ``numId`` 0 and + the segments of its :class:`~legaldown_render.resolve.docx_plan.AltNumber` for a later + alternative that shares a number (§15.8), and ``w:ind`` where the plan sets one; +* its bookmarks wrap the title or first paragraph's content, not the condition label or the + alternative's number; +* a list item whose first block is not a paragraph is an empty numbered paragraph, so that the + number and the bookmarks have a paragraph to stand on, followed by its blocks; +* the other blocks of an item, and a quote's or a drafting note's, are plain paragraphs at the indent + of what they belong to. + +The title block, the table of contents, the attachments and the signature blocks are laid out by +:mod:`.matter`, in the order the HTML writer shows them: title, subtitle, header rows and parties, contents, +preamble, sections, attachments, signatures. +""" +from __future__ import annotations + +import xml.etree.ElementTree as ET +from collections.abc import Iterable, Iterator, Sequence +from dataclasses import dataclass, replace + +from ...errors import InternalError +from ...resolve.docx_plan import AltNumber, DocxPlan, UnitPlan +from ...style.docx import PageGeometry +from ...tree import ( + Block, + CodeBlock, + DraftingNote, + Inline, + List, + ListItem, + Paragraph, + Quote, + RenderTree, + Rule, + Section, + Table, + plain_text, +) +from .matter import MatterSettings, MatterWriter +from .ooxml import Bookmarks, el, p, tbl_pr, tc_pr, text_run, tr_pr, val +from .package import Relationships +from .runs import InlineWriter +from .styles import ( + CODE, + CODE_INDENT, + CONDITION_LABEL, + DRAFTING_NOTE, + DRAFTING_NOTE_LABEL, + NOTE_INDENT, + QUOTE, + QUOTE_INDENT, + SUBTITLE, + TABLE_GRID, + TITLE, + container_bar, + heading_style, +) + +#: The left indent the paragraph styles set, in twips: what a paragraph in that style has without an indent of its own. +_STYLE_LEFT = {QUOTE: QUOTE_INDENT, DRAFTING_NOTE: NOTE_INDENT} + +#: Limits on what a plan's indents can do to the page, in twips: Word does not take a left indent past 31680 +#: (22 inches), and a paragraph needs room to be read in; a column needs room for a word. +_MAX_INDENT = 31680 +_MIN_TEXT = 2268 +_MIN_COLUMN = 1134 + +_RULE_COLOR = "C8C8C8" +#: The space above and below the text of a table cell, in twips. +_CELL_PADDING = 40 + + +def section_properties(page: PageGeometry, *, header_id: str | None = None, footer_id: str | None = None) -> ET.Element: + """The ``w:sectPr`` of the one section: the running header and footer, by the + relationship ids of their parts, then the paper and the margins in twips. + ``w:pgSz`` of a landscape page has the long edge as its width, and says so + (``w:orient``), which Word needs to print it so.""" + return el( + "sectPr", None, + header_id and el("headerReference", {"type": "default", "r:id": header_id}), + footer_id and el("footerReference", {"type": "default", "r:id": footer_id}), + el("pgSz", {"w": page.width, "h": page.height, "orient": "landscape" if page.landscape else None}), + el("pgMar", {"top": page.top, "right": page.right, "bottom": page.bottom, "left": page.left, + "header": page.header, "footer": page.footer, "gutter": 0}), + ) + + +def document_title(tree: RenderTree) -> str: + """The document's title as plain text, trimmed: the one text the title paragraph, the + ``{title}`` of a header or footer and the document properties all show.""" + return plain_text(tree.title).strip() + + +@dataclass(frozen=True, slots=True) +class Context: + """Where a block is written: the paragraph style of a quote or drafting note it is in, how far + its paragraphs are indented, and how much the quotes and notes around it add to a numbered unit.""" + + #: The style of the plain paragraphs here (``Quote``, ``DraftingNote``), None at the top. + style: str | None = None + #: Where the text of a plain paragraph starts, in twips from the margin: the text of the list item + #: it continues, or what the enclosing quotes and notes indent. + left: int = 0 + #: What the quotes and notes around a list item add to its own indent (§13.2). + extra: int = 0 + + +class BodyWriter: + """Writes the body of one document: see the module docstring. + + *bookmarks* and *relationships* are the document's (the bookmark ids are unique across the package; + the relationships receive the external links); *text_width* is the width between the margins in + twips, which tables take; *matter* says what the front and back matter need of the style (it defaults + to what the built-in styles give).""" + + def __init__(self, tree: RenderTree, plan: DocxPlan, bookmarks: Bookmarks, relationships: Relationships, + text_width: int, link_color: str, matter: MatterSettings | None = None) -> None: + self.tree = tree + self.plan = plan + self.bookmarks = bookmarks + self.relationships = relationships + self.text_width = text_width + self.link_color = link_color + self.matter_settings = matter if matter is not None else MatterSettings(text_width) + abstracts = {abstract.id: abstract for abstract in plan.abstract_nums} + self._levels = {num.id: abstracts[num.abstract_id].levels for num in plan.nums if num.abstract_id in abstracts} + attachment_titles = {a.anchor: plain_text(a.title) for a in tree.attachments if a.anchor} + #: Not known until a first, dry run has written the document: see :meth:`body`. + self.inlines = InlineWriter(plan, relationships, bookmarks, None, attachment_titles, link_color) + self.matter = MatterWriter(tree, plan, self.inlines, bookmarks, self.matter_settings) + + def _left(self, left: int) -> int: + """*left* twips of indent, kept where a paragraph can still be read: not past Word's limit, and leaving + :data:`_MIN_TEXT` of the text width (a deeply nested list, or a plan's wide indent, cannot push it off the page).""" + return max(0, min(left, self.text_width - _MIN_TEXT, _MAX_INDENT)) + + # -- the document --------------------------------------------------------------------- + + def body(self, page: PageGeometry, *, header_id: str | None = None, footer_id: str | None = None) -> ET.Element: + """``w:body`` for the tree, closed by the section properties of *page*. + + The document is written twice. The first, dry run learns which bookmarks it writes (a name in the plan + is not enough: a defined term inside alt text or a link title is never written), and the second writes + it for real: a ``REF`` field to a name the document does not hold is then a bug in the plan + (:class:`~legaldown_render.errors.InternalError`), and a link or a term reference to one shows its text + without the link. Whatever :meth:`content` writes counts, so the attachments' bookmarks need no list of + their own.""" + dry = BodyWriter(self.tree, self.plan, Bookmarks(), Relationships(), self.text_width, self.link_color, + self.matter_settings) + dry.content() + self.inlines.placed = dry.bookmarks.names + return el("body", None, self.content(), section_properties(page, header_id=header_id, footer_id=footer_id)) + + def content(self) -> list[ET.Element]: + """The block-level content of the body, in the order the HTML writer shows it: the title and the subtitle, + the header rows and the parties, the table of contents, the preamble, the sections, the attachments and + the signature blocks.""" + tree = self.tree + content: list[ET.Element] = [] + title = document_title(tree) + if title: + content.append(p(text_run(title), style=TITLE)) + subtitle = plain_text(tree.subtitle).strip() + if subtitle: + content.append(p(text_run(subtitle), style=SUBTITLE)) + content += self.matter.title_block() + content += self.matter.contents() + content += self.blocks(tree.preamble, Context()) + for section in tree.sections: + content += self.section(section) + content += self.matter.attachments() + content += self.matter.signatures() + content = self._apart(content) + if content and content[-1].tag.endswith("}tbl"): + content.append(p()) # a table cannot be the last thing before the section properties + return content + + @staticmethod + def _apart(content: list[ET.Element]) -> list[ET.Element]: + """*content* with an empty paragraph between two tables that follow each other (Word merges them + otherwise), as :meth:`blocks` does inside a block.""" + out: list[ET.Element] = [] + for element in content: + if element.tag.endswith("}tbl") and out and out[-1].tag.endswith("}tbl"): + out.append(p()) + out.append(element) + return out + + def section(self, section: Section) -> list[ET.Element]: + """The heading paragraph of *section* (``heading N``: the resolver has clamped the level to + 1-5) and its blocks. The condition label (§15.8) is a run inside the heading, after the title, + so that Word's table of contents keeps it.""" + level = min(max(section.level, 1), 5) + unit = self.plan.units.get(section.unit or "") + title = self.inlines.title_runs(section.title, unit.literal_refs if unit is not None else ()) + heading = self._unit(section.unit, title, style=heading_style(level), + context=Context(), conditions=(section.condition_label,), after=True) + return [heading, *self.blocks(section.blocks, Context())] + + # -- blocks --------------------------------------------------------------------------- + + def blocks(self, blocks: Iterable[Block], context: Context) -> list[ET.Element]: + """The elements of *blocks*. Two tables in a row get an empty paragraph between them, which + Word needs to keep them apart.""" + content: list[ET.Element] = [] + for block in blocks: + for element in self.block(block, context): + if element.tag.endswith("}tbl") and content and content[-1].tag.endswith("}tbl"): + content.append(p()) + content.append(element) + return content + + def block(self, block: Block, context: Context) -> Iterator[ET.Element]: + match block: + case Paragraph(): + yield self._paragraph(block, context) + case List(items=items): + for item in items: + yield from self._item(item, context) + case Quote(blocks=blocks): + yield from self.blocks(blocks, replace(context, style=QUOTE, left=context.left + QUOTE_INDENT, + extra=context.extra + QUOTE_INDENT)) + case DraftingNote(blocks=blocks, label=label): + inner = replace(context, style=DRAFTING_NOTE, left=context.left + NOTE_INDENT, + extra=context.extra + NOTE_INDENT) + if label: + yield p(text_run(label, style=DRAFTING_NOTE_LABEL), style=DRAFTING_NOTE, + props=[self._plain_indent(inner)]) + yield from self.blocks(blocks, inner) + case CodeBlock(text=text): + # One paragraph per line: Word shades and keeps them together as one block. + # Inside a quote or a note the paragraph stays at the container's indent and keeps its bar. + bar = container_bar(context.style) if context.style else None + left = self._left(context.left if bar is not None else context.left + CODE_INDENT) + for line in text.rstrip("\n").split("\n"): + yield p(text_run(line), style=CODE, + props=[el("ind", {"left": left}) if context.left else None, + bar is not None and el("pBdr", None, bar)]) + case Table(): + yield from self._table(block, context) + case Rule(): + bar = container_bar(context.style) if context.style else None + yield p(style=context.style, props=[ + el("pBdr", None, bar, el("bottom", {"val": "single", "sz": 4, "space": 1, "color": _RULE_COLOR})), + self._plain_indent(context)]) + case _: + raise InternalError(f"a {type(block).__name__} reached the DOCX writer, which does not write it " + "(a bug in the DOCX writer)") + + def _plain_indent(self, context: Context) -> ET.Element | None: + """``w:ind`` for a plain paragraph in *context*, unless its style already indents so far.""" + left = self._left(context.left) + return el("ind", {"left": left}) if left != _STYLE_LEFT.get(context.style or "", 0) else None + + def _paragraph(self, block: Paragraph, context: Context) -> ET.Element: + content = self.inlines.runs(block.inlines) + if block.unit is not None: + return self._unit(block.unit, content, style=context.style, context=context, + conditions=(block.condition_label,)) + return p(self._conditions((block.condition_label,), content), content, style=context.style, + props=[self._plain_indent(context)]) + + def _item(self, item: ListItem, context: Context) -> Iterator[ET.Element]: + """A list item: its first paragraph is the numbered paragraph, or an empty numbered paragraph + stands for it when the item opens with another block (or has none).""" + first = item.blocks[0] if item.blocks and isinstance(item.blocks[0], Paragraph) else None + rest = item.blocks[1:] if first is not None else item.blocks + conditions = (item.condition_label, first.condition_label if first is not None else None) + content = self.inlines.runs(first.inlines) if first is not None else [] + yield self._unit(item.unit, content, style=context.style, context=context, conditions=conditions) + inner = replace(context, left=self._text_left(self.plan.units.get(item.unit or ""), context)) + yield from self.blocks(rest, inner) + + # -- numbered paragraphs -------------------------------------------------------------- + + def _indent(self, unit: UnitPlan) -> tuple[int, int] | None: + """``(left, hanging)`` of the unit's paragraph: its own indent from the plan, else the indent of its + level in the numbering definition; None when the paragraph has neither.""" + if unit.ind is not None: + return unit.ind + levels = self._levels.get(unit.num_id) if unit.num_id else None + if levels is None or unit.ilvl is None: + return None + return levels[unit.ilvl].left, levels[unit.ilvl].hanging + + def _text_left(self, unit: UnitPlan | None, context: Context) -> int: + """Where the text of a unit's paragraph starts: what its continuation blocks are indented to.""" + indent = self._indent(unit) if unit is not None else None + return self._left(context.left if indent is None else indent[0] + context.extra) + + def _unit(self, unit_id: str | None, content: Sequence[ET.Element], *, style: str | None, context: Context, + conditions: Sequence[str | None], after: bool = False) -> ET.Element: + """The paragraph of the unit *unit_id* (None, or a unit the plan does not know, has no numbering) with + *content*, the title or first paragraph's runs. The condition labels lead the paragraph, or follow the + content with *after* (a heading); the unit's bookmarks wrap the content alone, or, where the plan says they are + collapsed, stand as points at the start of the paragraph.""" + unit = self.plan.units.get(unit_id) if unit_id else None + props: list[ET.Element | None] = [] + number: list[ET.Element] = [] + names: list[str] = [] + points: list[str] = [] + if unit is not None: + if unit.explicit or unit.alt is not None: + props.append(self._numbering(unit)) + indent = self._indent(unit) + if indent is not None and (unit.ind is not None or context.extra): + left = self._left(indent[0] + context.extra) + hanging = min(indent[1], left) + props.append(el("ind", {"left": left, "hanging": hanging or None})) + if unit.alt is not None: + number = self._alternative(unit.alt) + (points if unit.collapsed else names).extend(unit.bookmarks) + names.extend([unit.toc_bookmark] if unit.toc_bookmark else []) + else: + props.append(self._plain_indent(context)) + wrapped = [*self.inlines.points(points), *(self.inlines.wrap(names, content) if names else content)] + labels = self._conditions(conditions, content, after=after) + return p(number, wrapped + labels if after else [*labels, *wrapped], style=style, props=props) + + def _numbering(self, unit: UnitPlan) -> ET.Element: + """``w:numPr`` of an explicit unit: its num and level, ``numId`` 0 (no numbering) for an alternative.""" + num_id = 0 if unit.alt is not None else unit.num_id + if num_id is None: + raise InternalError(f"the unit {unit.unit} must carry numbering but the plan gives no num " + "(a bug in the DOCX plan)") + return el("numPr", None, unit.ilvl is not None and val("ilvl", unit.ilvl), val("numId", num_id)) + + def _alternative(self, alt: AltNumber) -> list[ET.Element]: + """The number of a later alternative as text and fields (§15.8), then its tab or space.""" + separator = el("r", None, el("tab")) if alt.suff == "tab" else text_run(" ") + return [*self.inlines.segments(alt.segments), separator] + + def _conditions(self, labels: Sequence[str | None], content: Sequence[ET.Element], *, + after: bool = False) -> list[ET.Element]: + """The condition labels (§15.8) as ``ConditionLabel`` runs, each one apart from the text by a space + (none after an empty paragraph's labels).""" + runs: list[ET.Element] = [] + for label in filter(None, labels): + run = text_run(label, style=CONDITION_LABEL) + if after: + runs += [text_run(" "), run] + else: + runs += [run, text_run(" ")] if content else [run] + return runs + + # -- tables --------------------------------------------------------------------------- + + def _table(self, table: Table, context: Context) -> Iterator[ET.Element]: + """A table: ``TableGrid``, the header row repeated on every page (``w:tblHeader``), columns aligned as + the table says, and indented with the list item or quote it is in. A row shorter than the header is + completed with empty cells.""" + columns = max(len(table.header), *(len(row) for row in table.rows), 0) + if not columns: + return + left = self._left(context.left) + width = self.text_width - left + if width < columns * _MIN_COLUMN: # no room to indent: the columns come first + left, width = 0, self.text_width + grid = [width // columns] * columns + grid[-1] += width - sum(grid) + align = (*table.align, *[""] * columns)[:columns] + + def row(cells: tuple[tuple[Inline, ...], ...], *, header: bool = False) -> ET.Element: + padded = (*cells, *[()] * (columns - len(cells))) + return el("tr", None, tr_pr(header and el("tblHeader")), [ + el("tc", None, tc_pr(el("tcW", {"w": size, "type": "dxa"})), + p(self.inlines.runs(cell), props=[ + el("spacing", {"before": 0, "after": 0}), el("jc", {"val": alignment or "left"})])) + for cell, size, alignment in zip(padded, grid, align, strict=True)]) + + yield el( + "tbl", None, + tbl_pr(val("tblStyle", TABLE_GRID), el("tblW", {"w": width, "type": "dxa"}), + left > 0 and el("tblInd", {"w": left, "type": "dxa"}), + el("tblCellMar", None, el("top", {"w": _CELL_PADDING, "type": "dxa"}), + el("bottom", {"w": _CELL_PADDING, "type": "dxa"})), + el("tblLook", {"val": "0620", "firstRow": 1, "lastRow": 0, "firstColumn": 0, "lastColumn": 0, + "noHBand": 1, "noVBand": 1})), + el("tblGrid", None, [el("gridCol", {"w": size}) for size in grid]), + row(table.header, header=True) if table.header else None, + [row(cells) for cells in table.rows]) diff --git a/src/legaldown_render/writers/docx/headers.py b/src/legaldown_render/writers/docx/headers.py new file mode 100644 index 0000000..f46e25d --- /dev/null +++ b/src/legaldown_render/writers/docx/headers.py @@ -0,0 +1,74 @@ +"""Running headers and footers (``page.header``, ``page.footer``). + +Each is one paragraph in a ``w:hdr`` or ``w:ftr`` part. Its left, centre and right +templates are laid out with tab stops at the middle and at the right margin; a +paragraph with only one of them is simply aligned. ``{page}`` and ``{pages}`` are +``PAGE`` and ``NUMPAGES`` fields, which Word and LibreOffice evaluate while +laying the pages out (their cached result is a placeholder); ``{title}``, +``{version}`` and ``{date}`` are the document's own text, written as text. +""" +from __future__ import annotations + +import re +import xml.etree.ElementTree as ET +from collections.abc import Mapping + +from ...style import PageText +from .ooxml import complex_field, el, p, serialize, text_run +from .styles import FOOTER, HEADER + +#: ``{name}`` as the style loader reads a field (docs/style-templates.md). +_FIELD_RE = re.compile(r"\{([A-Za-z_][A-Za-z0-9_]*)\}") + +#: The page-number fields, by the template field that stands for them. +_FIELDS = {"page": "PAGE", "pages": "NUMPAGES"} + + +def template_runs(template: str, values: Mapping[str, str]) -> list[ET.Element]: + """The runs of *template*: its text, ``{page}`` and ``{pages}`` as fields + (cached as 1), and any other field written as its text from *values*. An + identifier in braces that *values* does not name stays as written; the loader + refuses such a template, so this is for a style built by hand.""" + runs: list[ET.Element] = [] + position = 0 + for match in _FIELD_RE.finditer(template): + if match.start() > position: + runs.append(text_run(template[position:match.start()])) + name = match.group(1) + if name in _FIELDS: + runs += complex_field(_FIELDS[name], [text_run("1")]) + else: + runs.append(text_run(values.get(name, match.group(0)))) + position = match.end() + if position < len(template): + runs.append(text_run(template[position:])) + return runs + + +def page_text_paragraph(text: PageText, values: Mapping[str, str], text_width: int, style: str) -> ET.Element: + """The paragraph for *text*: left, centre and right templates separated by tabs, + with tab stops at the middle and the right margin (*text_width* twips from the + left one). A single template is aligned instead, so it needs no tabs.""" + left, center, right = (template_runs(template, values) for template in (text.left, text.center, text.right)) + if not text.left and bool(text.center) != bool(text.right): + return p(left or center or right, style=style, + props=[el("jc", {"val": "center" if text.center else "right"})]) + content: list[ET.Element] = [*left] + tabs: list[ET.Element] = [] + if text.center: + content += [el("r", None, el("tab")), *center] + tabs.append(el("tab", {"val": "center", "pos": text_width // 2})) + if text.right: + content += [el("r", None, el("tab")), *right] + tabs.append(el("tab", {"val": "right", "pos": text_width})) + return p(content, style=style, props=[tabs and el("tabs", None, tabs)]) + + +def build_header(text: PageText, values: Mapping[str, str], text_width: int) -> bytes: + """A ``w:hdr`` part showing *text*.""" + return serialize(el("hdr", None, page_text_paragraph(text, values, text_width, HEADER))) + + +def build_footer(text: PageText, values: Mapping[str, str], text_width: int) -> bytes: + """A ``w:ftr`` part showing *text*.""" + return serialize(el("ftr", None, page_text_paragraph(text, values, text_width, FOOTER))) diff --git a/src/legaldown_render/writers/docx/matter.py b/src/legaldown_render/writers/docx/matter.py new file mode 100644 index 0000000..8ce5809 --- /dev/null +++ b/src/legaldown_render/writers/docx/matter.py @@ -0,0 +1,365 @@ +"""The front and back matter of the document (§13.8, §3.6, §13.6): the header rows and the parties, +the table of contents, the attachments and the signature blocks, laid out as the HTML writer shows them +and in the order it shows them. + +Like the rest of the writer (docs/decisions/0008) it holds no rule: the entries of the table of contents, +their numbers and their ``_Toc`` bookmarks are the plan's (:attr:`~legaldown_render.resolve.docx_plan.DocxPlan.toc`), +and what the tree says is what is written. + +* **Table of contents (§2.7 of the plan).** One complex field, ``TOC \\o "1-N" \\h \\z \\u``, whose cached + result is the planned entries: one ``TOC1`` to ``TOC5`` paragraph each, a hyperlink to the entry's bookmark + holding the number, its separator, the title, the condition label and a ``PAGEREF`` field whose cached page + is empty (Word fills it in when it updates the field). The field opens in the first entry's paragraph and + closes in a paragraph of its own after the last, as Word writes it. The attachments are listed by outline + level, not by a style list: the attachments heading and each attachment's title are paragraphs of styles + that carry one (``\\u``), so that no list separator (a comma or a semicolon, by locale) is in the field. The + contents label is a paragraph of the ``ContentsHeading`` style: no outline level, no numbering, so not listed. +* **Attachments.** The heading and each title are bookmarked: by the plan's name for the anchor, so that + ``{{attach:}}`` is a ``REF`` field or a link that resolves, and by the ``_Toc`` name of the contents entry. + ``attachments.separator`` is a page break before the part (a paragraph property: no empty paragraph is left + on the page before, and no blank page when the part falls at a page top) or a rule above each attachment. +* **Signature blocks.** Borderless tables (each party a cell, two to a row) kept on one page: a row does not split + and every paragraph but the last keeps with the next. +""" +from __future__ import annotations + +import xml.etree.ElementTree as ET +from collections.abc import Iterable, Sequence +from dataclasses import dataclass + +from ...errors import InternalError +from ...resolve.docx_plan import DocxPlan, TocEntryPlan +from ...tree import ( + AttachmentPart, + ContentsEntry, + Inline, + PartyInfo, + RenderTree, + SideInfo, + SignatureParty, + plain_text, +) +from .ooxml import ( + Bookmarks, + el, + field_end, + field_start, + hyperlink_internal, + p, + r, + tbl_pr, + tc_pr, + text_run, + tr_pr, + val, +) +from .runs import InlineWriter +from .styles import ( + ATTACHMENT_FILE, + ATTACHMENT_TITLE, + ATTACHMENTS_HEADING, + CONDITION_LABEL, + CONTENTS_HEADING, + PARTY_NAME, + PARTY_SIDE, + SIGNATURE_LINE, + SIGNATURE_SIDE, + SIGNATURE_TEXT, + TITLE_BLOCK_LABEL, + TITLE_BLOCK_TEXT, + toc_style, +) + +#: The share of the text width the label column of the header rows and party details takes, in percent. +_LABEL_SHARE = 30 +#: The gap between two cells, in twips, and the share of a signature cell the signing line leaves free, in percent. +_GUTTER = 283 +_LINE_FREE = 20 +#: Signature blocks in a row. +_SIGNATURE_COLUMNS = 2 +#: What a character of a number takes in an entry of the contents, and the gap after it, in em; where the text +#: starts after a number's tab stop is the longest number of its level (a guess for a font the writer does +#: not measure: Word recomputes the stops when it updates the field). +_NUMBER_EM = 0.7 +_NUMBER_GAP_EM = 0.6 + + +@dataclass(frozen=True, slots=True) +class MatterSettings: + """What the matter needs besides the tree and the plan: the page and the style's contents and attachments.""" + + #: The width between the margins, in twips: where the page numbers' right tab goes, and the width of tables. + text_width: int + #: The body font size in points (the width of a number in the contents is estimated from it). + points: float = 11.0 + #: ``contents.depth``: the deepest heading level the ``TOC`` field lists. + depth: int = 2 + #: ``attachments.separator``: ``page-break``, ``rule`` or ``none``. + separator: str = "page-break" + + +class MatterWriter: + """Writes the matter around the sections. *inlines* and *bookmarks* are the body writer's, so that + every bookmark written here counts for the links and references that point at it (the document is + written twice, :meth:`.body.BodyWriter.body`).""" + + def __init__(self, tree: RenderTree, plan: DocxPlan, inlines: InlineWriter, bookmarks: Bookmarks, + settings: MatterSettings) -> None: + self.tree = tree + self.plan = plan + self.inlines = inlines + self.bookmarks = bookmarks + self.settings = settings + if tree.contents and len(tree.contents) != len(plan.toc): + raise InternalError("the plan's table of contents does not hold the entries of the tree's " + "(a bug in the DOCX plan)") + #: The ``_Toc`` names of the contents entries that are not a section's (a section's is the plan's + #: ``UnitPlan.toc_bookmark``, written with its heading), in order: the attachments heading when the style + #: gives it a label, then one per attachment. Paired by position, not by anchor: an attachment with no id + #: or the same id as another (validator Errors, still rendered) has no anchor of its own. + self._toc_of = [planned.bookmark for entry, planned in zip(tree.contents, plan.toc, strict=False) + if entry.unit is None] + wanted = (1 if tree.attachments_label else 0) + len(tree.attachments) + if self._toc_of and len(self._toc_of) != wanted: + raise InternalError("the table of contents does not list the attachments heading and each attachment " + "(a bug in the DOCX plan)") + + # -- header rows and parties ---------------------------------------------------------- + + def title_block(self) -> list[ET.Element]: + """The header rows (label and value, as ``dt`` and ``dd``) and the parties: each side's label, then + each party's name and its details (§13.5).""" + tree = self.tree + content: list[ET.Element] = [] + if tree.header: + content.append(self._rows([(label, value) for label, value in tree.header])) + for side in tree.sides: + content += self._side(side) + return content + + def _side(self, side: SideInfo) -> list[ET.Element]: + content = [p(self.inlines.runs(side.label), style=PARTY_SIDE)] + for party in side.parties: + content += self._party(party) + return content + + def _party(self, party: PartyInfo) -> list[ET.Element]: + content = [p(self.inlines.runs(party.name), style=PARTY_NAME)] + if party.details: + content.append(self._rows(list(party.details))) + return content + + def _rows(self, rows: Sequence[tuple[str, tuple[Inline, ...]]]) -> ET.Element: + """A borderless table of label and value rows: the label in italics (a style blanked by the label + leaves its cell empty, as the ``dt`` is).""" + label = self.settings.text_width * _LABEL_SHARE // 100 + widths = (label, self.settings.text_width - label) + return _table(widths, [ + [[p(text_run(name, style=TITLE_BLOCK_LABEL), style=TITLE_BLOCK_TEXT)] if name + else [p(style=TITLE_BLOCK_TEXT)], + [p(self.inlines.runs(value), style=TITLE_BLOCK_TEXT)]] + for name, value in rows]) + + # -- table of contents ---------------------------------------------------------------- + + def contents(self) -> list[ET.Element]: + """The contents label and the ``TOC`` field with its cached entries; nothing when the tree lists + nothing (the HTML writer writes no contents then, label included).""" + tree = self.tree + if not tree.contents: + return [] + content: list[ET.Element] = [] + if tree.contents_label: + content.append(p(text_run(tree.contents_label), style=CONTENTS_HEADING)) + depth = max(1, min(self.settings.depth, 5)) + instruction = f'TOC \\o "1-{depth}" \\h \\z \\u' + stops = self._number_stops(tree.contents, depth) + for index, (entry, planned) in enumerate(zip(tree.contents, self.plan.toc, strict=True)): + level = self._level(entry, depth) + content.append(self._entry(planned, level, stops.get(level), + lead=field_start(instruction) if index == 0 else ())) + content.append(p(field_end())) + return content + + @staticmethod + def _level(entry: ContentsEntry, depth: int) -> int: + """The level Word lists *entry* at (``TOCn``): a section's own, and an attachment's outline level + clamped to the field's depth (below, the field does not list it at all), so that the cached result is the + one an update gives.""" + return entry.level if entry.unit is not None else max(1, min(entry.level, depth)) + + def _outline(self, level: int) -> ET.Element | None: + """The outline level of an attachments paragraph the contents lists at *level* (``\\u`` lists by it), never + deeper than the field's depth, which would drop it from the list. Where the contents exists and does not + list the attachments (``contents.attachments``), the paragraphs are body text, as their style's outline + level would list them; with no contents the style's level stands (the navigation pane).""" + if not self.tree.contents: + return None + if not any(entry.unit is None for entry in self.tree.contents): + return val("outlineLvl", 9) + return val("outlineLvl", max(0, min(level - 1, self.settings.depth - 1))) + + def _number_stops(self, entries: Iterable[ContentsEntry], depth: int) -> dict[int, int]: + """The position of the tab after the number, by level: left of the text of every entry of that level + whose number is followed by a tab, so that their titles line up.""" + stops: dict[int, int] = {} + em = self.settings.points * 20 + for entry, planned in zip(entries, self.plan.toc, strict=True): + if planned.sep != "\t" or not planned.number: + continue + level = self._level(entry, depth) + width = len(planned.number) * _NUMBER_EM * em + _NUMBER_GAP_EM * em + 240 * (level - 1) + stops[level] = min(max(stops.get(level, 0), round(width)), self.settings.text_width // 2) + return stops + + def _entry(self, planned: TocEntryPlan, level: int, stop: int | None, + lead: Iterable[ET.Element] = ()) -> ET.Element: + """One entry's paragraph: a hyperlink to its bookmark with the number, the title, the condition label, + and the page number as a ``PAGEREF`` field with an empty cached result. *lead* are runs before the + hyperlink (the opening of the ``TOC`` field, in the first entry).""" + name = self._placed(planned.bookmark) + runs: list[ET.Element] = [] + if planned.number: + runs.append(text_run(planned.number, props=[el("noProof")])) + runs.append(r(el("tab"), props=[el("noProof")]) if planned.sep == "\t" + else text_run(" ", props=[el("noProof")])) + runs += self.inlines.runs(planned.title) + if planned.condition_label: + runs += [text_run(" "), text_run(planned.condition_label, style=CONDITION_LABEL)] + runs.append(r(el("tab"), props=[el("noProof")])) + runs += [*field_start(f"PAGEREF {name} \\h"), field_end()] + tabs = [stop is not None and bool(planned.number) and planned.sep == "\t" and el("tab", {"val": "left", "pos": stop}), + el("tab", {"val": "right", "leader": "dot", "pos": self.settings.text_width})] + return p(*lead, hyperlink_internal(name, *runs), style=toc_style(level), props=[el("tabs", None, tabs)]) + + def _placed(self, name: str) -> str: + """*name*, a bookmark an entry points at: the document must write it (not known in the dry run).""" + placed = self.inlines.placed + if placed is not None and name.lower() not in placed: + raise InternalError(f"a table of contents entry points at the bookmark '{name}', which the document " + "does not contain (a bug in the DOCX plan)") + return name + + # -- attachments ---------------------------------------------------------------------- + + def attachments(self) -> list[ET.Element]: + """The attachments heading and each attachment's title, condition label and file (§13.8).""" + tree = self.tree + if not tree.attachments: + return [] + content: list[ET.Element] = [] + toc = list(self._toc_of) + separator = self.settings.separator + # The page break goes before the first thing of the part, whichever that is, and before every title after the + # first: the heading stays with the first attachment. + breaking = separator == "page-break" + level = 1 + if tree.attachments_label: + level = 2 + names = self._names(tree.attachments_anchor, toc.pop(0) if toc else None) + content.append(p(self.inlines.wrap(names, [text_run(tree.attachments_label)]) if names + else text_run(tree.attachments_label), + style=ATTACHMENTS_HEADING, + props=[breaking and el("pageBreakBefore"), + self._outline(1)])) + breaking = False + for attachment in tree.attachments: + content += self._attachment(attachment, level, toc.pop(0) if toc else None, page_break=breaking, + rule=separator == "rule") + breaking = separator == "page-break" + return content + + def _names(self, anchor: str | None, toc: str | None) -> list[str]: + """The bookmarks of an attachments paragraph: the plan's name for its *anchor*, unless the document already + holds a bookmark of that name (a second attachment with the same id, or an id that is a section's), and + the contents entry's ``_Toc`` name *toc*.""" + name = self.plan.anchors.get(anchor) if anchor else None + names = [name] if name is not None and name.lower() not in self.bookmarks.names else [] + return [*names, *([toc] if toc else [])] + + def _attachment(self, attachment: AttachmentPart, level: int, toc: str | None, *, page_break: bool, + rule: bool) -> list[ET.Element]: + names = self._names(attachment.anchor, toc) + title = self.inlines.runs(attachment.title) + shown = self.inlines.wrap(names, title) if names else title + if attachment.condition_label: + shown += [text_run(" "), text_run(attachment.condition_label, style=CONDITION_LABEL)] + content = [p(shown, style=ATTACHMENT_TITLE, props=[ + page_break and el("pageBreakBefore"), + rule and el("pBdr", None, el("top", {"val": "single", "sz": 4, "space": 8, "color": "C8C8C8"})), + self._outline(level)])] + if attachment.file: + content.append(p(text_run(attachment.file), style=ATTACHMENT_FILE)) + return content + + # -- signature blocks ----------------------------------------------------------------- + + def signatures(self) -> list[ET.Element]: + """The signature blocks (§3.6) as a borderless table, a party to a cell, two to a row, kept together.""" + tree = self.tree + if not tree.signatures: + return [] + columns = min(_SIGNATURE_COLUMNS, len(tree.signatures)) + cell = self.settings.text_width // _SIGNATURE_COLUMNS + cells = [self._signature(signature, cell) for signature in tree.signatures] + rows = [cells[start:start + columns] for start in range(0, len(cells), columns)] + for row in rows: + row += [[p(style=SIGNATURE_TEXT)] for _ in range(columns - len(row))] + # Keep the table on one page: every paragraph of it but the last keeps with the next. + for paragraph in [paragraph for row in rows for blocks in row for paragraph in blocks][:-1]: + _keep_next(paragraph) + return [_table((cell,) * columns, rows)] + + def _signature(self, signature: SignatureParty, cell: int) -> list[ET.Element]: + """One party's block: its side, its name, a signing line with the signatory's name and title (one blank + line when none is declared) for each representative, then the date and place to be written in.""" + labels, colon = self.tree.signature_labels, self.tree.colon + blocks = [p(self.inlines.runs(signature.side), style=SIGNATURE_SIDE), + p(self.inlines.runs(signature.name), style=SIGNATURE_TEXT)] + + def row(key: str, value: tuple[Inline, ...] = (), *, write_in: bool = False) -> list[ET.Element]: + # As the text writer: a blanked label leaves the value alone and a row with neither is left out. Date + # and place (*write_in*) are to be written in, so they keep a no-break space of the colon; a name and + # a title without a value lose all the space after the label. + shown = plain_text(value) + label = f"{labels.get(key, '')}{colon}" if labels.get(key) else "" + if not (label + shown).strip(): + return [] + lead = label if shown else label.rstrip(" " if write_in else None) + return [p(text_run(lead) if lead else None, self.inlines.runs(value), style=SIGNATURE_TEXT)] + + for name, title in signature.signatories or (((), ()),): + blocks.append(p(style=SIGNATURE_LINE, props=[ + el("ind", {"right": cell * _LINE_FREE // 100 + _GUTTER})])) + blocks += [*row("name", name), *row("title", title)] + blocks += [*row("date", write_in=True), *row("place", write_in=True)] + return blocks + + +_PPR = "{http://schemas.openxmlformats.org/wordprocessingml/2006/main}pPr" + + +def _keep_next(paragraph: ET.Element) -> None: + """Add ``w:keepNext`` to *paragraph*'s properties, in the place the schema gives it (right after the style).""" + properties = paragraph.find(_PPR) + if properties is None: + properties = el("pPr") + paragraph.insert(0, properties) + keep = el("keepNext") + properties.insert(1 if properties.find("{*}pStyle") is not None else 0, keep) + + +def _table(widths: Sequence[int], rows: Sequence[Sequence[Sequence[ET.Element]]]) -> ET.Element: + """A borderless table: one ``w:tc`` per entry of a row, holding its paragraphs, *widths* twips wide; + rows do not split across pages.""" + return el( + "tbl", None, + tbl_pr(el("tblW", {"w": sum(widths), "type": "dxa"}), el("tblLayout", {"type": "fixed"}), + el("tblCellMar", None, el("left", {"w": 0, "type": "dxa"}), el("right", {"w": _GUTTER, "type": "dxa"})), + el("tblLook", {"val": "0000", "firstRow": 0, "lastRow": 0, "firstColumn": 0, "lastColumn": 0, + "noHBand": 1, "noVBand": 1})), + el("tblGrid", None, [el("gridCol", {"w": width}) for width in widths]), + [el("tr", None, tr_pr(el("cantSplit")), + [el("tc", None, tc_pr(el("tcW", {"w": width, "type": "dxa"})), paragraphs) + for paragraphs, width in zip(row, widths, strict=True)]) + for row in rows]) diff --git a/src/legaldown_render/writers/docx/numbering.py b/src/legaldown_render/writers/docx/numbering.py new file mode 100644 index 0000000..dcd7127 --- /dev/null +++ b/src/legaldown_render/writers/docx/numbering.py @@ -0,0 +1,48 @@ +"""``word/numbering.xml`` from the plan: the abstract numbering definitions and their instances. + +The planner decides every one of them (:class:`~legaldown_render.resolve.docx_plan.AbstractNum`, +:class:`~legaldown_render.resolve.docx_plan.Num`); this module lays them out, in the order +CT_Numbering, CT_AbstractNum and CT_Lvl require (ISO/IEC 29500-1, 17.9), and decides nothing. +""" +from __future__ import annotations + +import xml.etree.ElementTree as ET + +from ...resolve.docx_plan import AbstractNum, DocxPlan, LevelDef, Num +from .ooxml import el, val +from .settings import build_numbering + + +def level(definition: LevelDef) -> ET.Element: + """One ``w:lvl``: start, number format, paragraph style, suffix, text, justification and indent, in schema order.""" + return el( + "lvl", {"ilvl": definition.ilvl}, + val("start", definition.start), + val("numFmt", definition.num_fmt), + definition.p_style is not None and val("pStyle", definition.p_style), + definition.suff == "space" and val("suff", "space"), + val("lvlText", definition.lvl_text), + val("lvlJc", "left"), + (definition.left > 0 or definition.hanging > 0) and el( + "pPr", None, el("ind", {"left": definition.left, "hanging": definition.hanging or None})), + ) + + +def abstract_num(abstract: AbstractNum) -> ET.Element: + """One ``w:abstractNum``: its list identifier, its kind of list and its levels.""" + return el("abstractNum", {"abstractNumId": abstract.id}, val("nsid", abstract.nsid.upper()), + val("multiLevelType", abstract.multi_level), [level(definition) for definition in abstract.levels]) + + +def num(instance: Num) -> ET.Element: + """One ``w:num``: the definition it uses and, for each level that starts elsewhere than at the + level's own start, a ``w:startOverride``.""" + return el("num", {"numId": instance.id}, val("abstractNumId", instance.abstract_id), + [el("lvlOverride", {"ilvl": ilvl}, val("startOverride", value)) + for ilvl, value in instance.start_overrides]) + + +def numbering_part(plan: DocxPlan) -> bytes: + """``word/numbering.xml`` for *plan*: its abstract numberings, then its nums, in the plan's order.""" + return build_numbering([abstract_num(abstract) for abstract in plan.abstract_nums], + [num(instance) for instance in plan.nums]) diff --git a/src/legaldown_render/writers/docx/ooxml.py b/src/legaldown_render/writers/docx/ooxml.py new file mode 100644 index 0000000..fdb09f1 --- /dev/null +++ b/src/legaldown_render/writers/docx/ooxml.py @@ -0,0 +1,387 @@ +"""Building blocks for WordprocessingML: namespaces, the sanitiser, element +builders, fields, bookmarks and hyperlinks. + +Everything in a DOCX goes through :func:`el`, and :func:`el` passes every +attribute value and text through :func:`_t`. That is the one place that makes +text from a document, a style or a command line safe for XML (docs/architecture.md, +Security), so a builder cannot forget it: ``xml.etree`` alone would write +``\\x01`` raw and a lone surrogate as ``�``, neither of which is XML, +and Word then reports unreadable content. + +The builders place children in the order the schema requires +(:func:`ppr`, :func:`rpr`), so callers pass properties in any order and the +output still validates (ISO/IEC 29500-4, CT_PPr and CT_RPr are sequences). +""" +from __future__ import annotations + +import re +import xml.etree.ElementTree as ET +from collections.abc import Iterable, Mapping +from typing import Any +from xml.sax.saxutils import escape, quoteattr + +from ...errors import InternalError + +W = "http://schemas.openxmlformats.org/wordprocessingml/2006/main" +R = "http://schemas.openxmlformats.org/officeDocument/2006/relationships" +XML = "http://www.w3.org/XML/1998/namespace" +CP = "http://schemas.openxmlformats.org/package/2006/metadata/core-properties" +DC = "http://purl.org/dc/elements/1.1/" +DCTERMS = "http://purl.org/dc/terms/" +DCMITYPE = "http://purl.org/dc/dcmitype/" +XSI = "http://www.w3.org/2001/XMLSchema-instance" +PACKAGE_RELATIONSHIPS = "http://schemas.openxmlformats.org/package/2006/relationships" +CONTENT_TYPES = "http://schemas.openxmlformats.org/package/2006/content-types" +EXTENDED_PROPERTIES = "http://schemas.openxmlformats.org/officeDocument/2006/extended-properties" + +#: The prefix of each namespace in the parts written here. :func:`serialize` uses this table, not +#: ``xml.etree``'s global registry, which any other library may change (``register_namespace``). +_PREFIXES = {"w": W, "r": R, "cp": CP, "dc": DC, "dcterms": DCTERMS, "dcmitype": DCMITYPE, "xsi": XSI} +_PREFIX_OF = {uri: prefix for prefix, uri in {**_PREFIXES, "xml": XML}.items()} + +#: Characters XML 1.0 cannot carry (C0 but TAB, LF and CR), and the ones Word +#: refuses or reads as something else: DEL and C1 (NEL included), the +#: noncharacters U+FFFE and U+FFFF, and lone surrogates (which cannot be encoded as UTF-8). +_INVALID = re.compile("[\x00-\x08\x0b\x0c\x0e-\x1f\x7f-\x9f\ud800-\udfff\ufffe\uffff]") +_BREAKS = re.compile(r"\r\n|[\r\n]") +_WHITESPACE = re.compile(r"[\t\r\n]") + + +def _t(value: str, *, breaks: bool = False) -> str: + """*value* made safe to write: the characters in :data:`_INVALID` are dropped. + TAB, LF and CR are kept only with *breaks*, for text that is split on them + (:func:`text_run` turns them into ``w:tab`` and ``w:br``); anywhere else (an + attribute, an instruction, a relationship) they become a space, as an XML + parser would read them in an attribute anyway.""" + value = _INVALID.sub("", value) + return value if breaks else _WHITESPACE.sub(" ", value) + + +def _qname(name: str) -> str: + """``{namespace}local`` for *name*: a name already in that form, ``prefix:local`` + for the prefixes used here, or a bare WordprocessingML name.""" + if name.startswith("{"): + return name + prefix, colon, local = name.partition(":") + if not colon: + return f"{{{W}}}{name}" + namespaces = {**_PREFIXES, "xml": XML} + if prefix not in namespaces: + raise InternalError(f"unknown XML prefix '{prefix}' (a bug in the DOCX writer)") + return f"{{{namespaces[prefix]}}}{local}" + + +def el(name: str, attrs: Mapping[str, Any] | None = None, *children: Any, text: str | None = None) -> ET.Element: + """An element named *name* (see :func:`_qname`: bare names are + WordprocessingML's) with *attrs* (a ``None`` value leaves the attribute out; an + ``int`` is written as is, and a ``bool`` as ``1``/``0``), *text*, and *children*. + A child may be an element, ``None`` or ``False`` (left out), or an iterable of those, so + optional parts and lists need no ``if``. Every string is passed through + :func:`_t`.""" + element = ET.Element(_qname(name)) + for key, value in (attrs or {}).items(): + if value is None: + continue + if isinstance(value, bool): + value = "1" if value else "0" + element.set(_qname(key), _t(str(value))) + if text is not None: + element.text = _t(text) + element.extend(_flatten(children)) + return element + + +def _flatten(children: Iterable[Any]) -> list[ET.Element]: + """The elements in *children*, which may nest iterables and hold ``None`` and + ``False`` (so ``condition and element`` can be a child).""" + flat: list[ET.Element] = [] + for child in children: + if child is None or child is False: + continue + if isinstance(child, ET.Element): + flat.append(child) + elif isinstance(child, str | bytes): + raise InternalError("text was passed where an element belongs (a bug in the DOCX writer)") + else: + flat.extend(_flatten(child)) + return flat + + +def val(name: str, value: Any = None) -> ET.Element: + """````, or ```` without a *value*: the shape + of most properties.""" + return el(name, {"val": value}) + + +_DECLARATION = b'\n' + + +def _name(tag: str) -> tuple[str, str]: + """The namespace and the local name of an ElementTree *tag* (``""`` for none).""" + if tag.startswith("{"): + namespace, _, local = tag[1:].partition("}") + return namespace, local + return "", tag + + +def _attribute(value: str) -> str: + """*value* as the text of a double-quoted attribute: sanitised, and escaped so that + nothing, white space included, changes when a parser normalises it.""" + return _t(value).replace("&", "&").replace("<", "<").replace(">", ">").replace('"', """) + + +def serialize(root: ET.Element) -> bytes: + """*root* as a UTF-8 XML part with the declaration Word writes. Prefixes come from this + module's own table, never from ``xml.etree``'s global registry, so what another library + registers cannot change the bytes. Attributes keep the order they were set in, the + namespaces are declared on the root sorted by prefix, and the same tree is always the + same bytes. Every text and attribute is sanitised here as well (:func:`_t`), whoever + built the element. Only ``text`` is written, as no part here has mixed content, so a + tail is refused rather than dropped. A carriage return in text is written as `` ``, + which a parser reads back as CR, where a raw CR would read as LF.""" + namespaces: set[str] = set() + for element in root.iter(): + if element.tail: + raise InternalError("mixed content is not written (a bug in the DOCX writer)") + namespaces.add(_name(element.tag)[0]) + namespaces.update(_name(key)[0] for key in element.attrib) + namespaces.discard("") + unknown = namespaces - _PREFIX_OF.keys() + if unknown: + raise InternalError("an element is in a namespace the DOCX writer has no prefix for (a bug in the writer)") + declarations = "".join(f' xmlns:{prefix}="{uri}"' + for prefix, uri in sorted((_PREFIX_OF[uri], uri) for uri in namespaces if uri != XML)) + + def qualified(tag: str) -> str: + namespace, local = _name(tag) + return f"{_PREFIX_OF[namespace]}:{local}" if namespace else local + + out: list[str] = [] + stack: list[tuple[ET.Element, bool]] = [(root, False)] + while stack: + element, closing = stack.pop() + name = qualified(element.tag) + if closing: + out.append(f"") + continue + attributes = "".join(f' {qualified(key)}="{_attribute(value)}"' for key, value in element.attrib.items()) + extra = declarations if element is root else "" + text = element.text + if text is None and len(element) == 0: + out.append(f"<{name}{extra}{attributes}/>") + continue + out.append(f"<{name}{extra}{attributes}>") + if text: + out.append(_t(text, breaks=True).replace("&", "&").replace("<", "<").replace(">", ">") + .replace("\r", " ")) + stack.append((element, True)) + stack.extend((child, False) for child in reversed(element)) + return _DECLARATION + "".join(out).encode("utf-8") + + +def flat_part(namespace: str, root: str, children: Iterable[tuple[str, Mapping[str, Any], str | None]]) -> bytes: + """A part whose elements are all in one *namespace*, written unprefixed with no + namespace on attributes (the package's content types and relationships, and the + extended properties), as ```` and one element per + ``(name, attributes, text)`` in *children*. ``xml.etree`` cannot write a default + namespace together with unqualified attributes, so this writes the text itself, + escaping every value (:func:`_t`, then XML's own escapes).""" + def attributes(values: Mapping[str, Any]) -> str: + return "".join(f" {name}={quoteattr(_t(str(value)))}" for name, value in values.items() if value is not None) + + lines = [] + for name, values, text in children: + lines.append(f"<{name}{attributes(values)}/>" if text is None + else f"<{name}{attributes(values)}>{escape(_t(text))}") + body = "".join(lines) + part = f"<{root} xmlns={quoteattr(namespace)}>{body}" if body else f"<{root} xmlns={quoteattr(namespace)}/>" + return _DECLARATION + part.encode("utf-8") + + +# --------------------------------------------------------------------------- +# Properties, in schema order +# --------------------------------------------------------------------------- + +#: CT_PPrBase, then the elements CT_PPr adds after it (ISO/IEC 29500-1, 17.3.1.26). +_PPR_ORDER = ( + "pStyle", "keepNext", "keepLines", "pageBreakBefore", "framePr", "widowControl", "numPr", + "suppressLineNumbers", "pBdr", "shd", "tabs", "suppressAutoHyphens", "kinsoku", "wordWrap", + "overflowPunct", "topLinePunct", "autoSpaceDE", "autoSpaceDN", "bidi", "adjustRightInd", "snapToGrid", + "spacing", "ind", "contextualSpacing", "mirrorIndents", "suppressOverlap", "jc", "textDirection", + "textAlignment", "textboxTightWrap", "outlineLvl", "divId", "cnfStyle", "rPr", "sectPr", "pPrChange", +) + +#: EG_RPrBase (17.3.2.28). +_RPR_ORDER = ( + "rStyle", "rFonts", "b", "bCs", "i", "iCs", "caps", "smallCaps", "strike", "dstrike", "outline", + "shadow", "emboss", "imprint", "noProof", "snapToGrid", "vanish", "webHidden", "color", "spacing", "w", + "kern", "position", "sz", "szCs", "highlight", "u", "effect", "bdr", "shd", "fitText", "vertAlign", + "rtl", "cs", "em", "lang", "eastAsianLayout", "specVanish", "oMath", +) + +#: CT_TblPrBase (17.4.60). +_TBLPR_ORDER = ( + "tblStyle", "tblpPr", "tblOverlap", "bidiVisual", "tblStyleRowBandSize", "tblStyleColBandSize", "tblW", + "jc", "tblCellSpacing", "tblInd", "tblBorders", "shd", "tblLayout", "tblCellMar", "tblLook", + "tblCaption", "tblDescription", +) + +#: CT_TcPr (17.4.70). +_TCPR_ORDER = ( + "cnfStyle", "tcW", "gridSpan", "hMerge", "vMerge", "tcBorders", "shd", "noWrap", "tcMar", "textDirection", + "tcFitText", "vAlign", "hideMark", +) + +#: CT_TrPr: a choice, in any order (17.4.82), kept in this order for stable output. +_TRPR_ORDER = ("cnfStyle", "divId", "gridBefore", "gridAfter", "wBefore", "wAfter", "cantSplit", "trHeight", + "tblHeader", "tblCellSpacing", "jc", "hidden") + + +def _ordered(name: str, order: tuple[str, ...], children: Iterable[Any]) -> ET.Element | None: + """The element *name* holding *children* sorted into *order* (by local name; + an unknown child is a bug), or None when there are none.""" + flat = _flatten(children) + if not flat: + return None + rank = {local: index for index, local in enumerate(order)} + for child in flat: + local = child.tag.rpartition("}")[2] + if local not in rank: + raise InternalError(f"'{local}' is not a child of w:{name} (a bug in the DOCX writer)") + flat.sort(key=lambda child: rank[child.tag.rpartition("}")[2]]) # stable: equal names keep their order + return el(name, None, flat) + + +def ppr(*children: Any) -> ET.Element | None: + """``w:pPr`` with *children* in schema order, or None when there are none.""" + return _ordered("pPr", _PPR_ORDER, children) + + +def rpr(*children: Any) -> ET.Element | None: + """``w:rPr`` with *children* in schema order, or None when there are none.""" + return _ordered("rPr", _RPR_ORDER, children) + + +def tbl_pr(*children: Any) -> ET.Element | None: + """``w:tblPr`` with *children* in schema order, or None when there are none.""" + return _ordered("tblPr", _TBLPR_ORDER, children) + + +def tc_pr(*children: Any) -> ET.Element | None: + """``w:tcPr`` with *children* in schema order, or None when there are none.""" + return _ordered("tcPr", _TCPR_ORDER, children) + + +def tr_pr(*children: Any) -> ET.Element | None: + """``w:trPr`` with *children* in a fixed order, or None when there are none.""" + return _ordered("trPr", _TRPR_ORDER, children) + + +# --------------------------------------------------------------------------- +# Paragraphs and runs +# --------------------------------------------------------------------------- + + +def p(*content: Any, style: str | None = None, props: Iterable[Any] = ()) -> ET.Element: + """A paragraph: *style* (a style id) and *props* (``w:pPr`` children, any + order) first, then *content* (runs, fields, bookmarks, hyperlinks).""" + return el("p", None, ppr(val("pStyle", style) if style else None, props), content) + + +def r(*content: Any, style: str | None = None, props: Iterable[Any] = ()) -> ET.Element: + """A run: *style* (a character style id) and *props* (``w:rPr`` children, + any order) first, then *content* (``w:t``, ``w:tab``, ``w:br``, field characters).""" + return el("r", None, rpr(val("rStyle", style) if style else None, props), content) + + +def text_run(text: str, *, style: str | None = None, props: Iterable[Any] = ()) -> ET.Element: + """A run showing *text*. TAB becomes ``w:tab`` and a line break (LF, CR, or CRLF) + ``w:br``, because neither can live in ``w:t``; everything else is + sanitised (:func:`_t`) and kept, spaces included (``xml:space="preserve"``).""" + content: list[ET.Element] = [] + for line_index, line in enumerate(_BREAKS.split(_t(text, breaks=True))): + if line_index: + content.append(el("br")) + for tab_index, piece in enumerate(line.split("\t")): + if tab_index: + content.append(el("tab")) + if piece: + content.append(el("t", {"xml:space": "preserve"}, text=piece)) + return r(content, style=style, props=props) + + +def field_start(instruction: str, *, props: Iterable[Any] = (), dirty: bool = False) -> list[ET.Element]: + """The runs that open a complex field: ``begin``, the instruction and ``separate``. The result runs + follow, then :func:`field_end`; a field that spans paragraphs (the table of contents) is written in + these parts.""" + def char(kind: str, **extra: Any) -> ET.Element: + return r(el("fldChar", {"fldCharType": kind, **extra}), props=props) + + instr = r(el("instrText", {"xml:space": "preserve"}, text=f" {_t(instruction).strip()} "), props=props) + return [char("begin", dirty=True if dirty else None), instr, char("separate")] + + +def field_end(*, props: Iterable[Any] = ()) -> ET.Element: + """The run that closes a complex field.""" + return r(el("fldChar", {"fldCharType": "end"}), props=props) + + +def complex_field(instruction: str, result: Iterable[ET.Element] = (), *, props: Iterable[Any] = (), + dirty: bool = False) -> list[ET.Element]: + """The runs of a complex field: ``begin``, the instruction, ``separate``, + the cached *result* runs, ``end``. The instruction is what Word evaluates + (``PAGE``, ``REF bookmark \\n \\h``); the result is what the renderer + computed for it, shown until Word updates the field. *props* format the + field's own runs; *dirty* asks Word to update this one field when the file + opens.""" + return [*field_start(instruction, props=props, dirty=dirty), *result, field_end(props=props)] + + +def hyperlink_external(relationship_id: str, *runs: ET.Element, tooltip: str | None = None) -> ET.Element: + """A hyperlink to an address, by the id :meth:`~.package.Relationships.add_external` gave it.""" + return el("hyperlink", {"r:id": relationship_id, "tooltip": tooltip, "history": True}, runs) + + +def hyperlink_internal(bookmark: str, *runs: ET.Element, tooltip: str | None = None) -> ET.Element: + """A hyperlink to a bookmark in the document.""" + return el("hyperlink", {"anchor": bookmark, "tooltip": tooltip, "history": True}, runs) + + +# --------------------------------------------------------------------------- +# Bookmarks +# --------------------------------------------------------------------------- + +#: A bookmark name: a letter (or ``_`` for a hidden one), then letters, digits and ``_``, +#: at most 40 characters (Word's limit). +_BOOKMARK_RE = re.compile(r"[A-Za-z_][A-Za-z0-9_]{0,39}") + + +class Bookmarks: + """Allocates bookmark ids, which must be unique across the document body, the + headers and the footers, and checks that names are valid and unique (Word + compares names without regard to case). One instance serves a whole package.""" + + def __init__(self) -> None: + self._next = 1 + self._names: set[str] = set() + + @property + def names(self) -> frozenset[str]: + """The names allocated so far, lower-cased (Word compares names without regard to case).""" + return frozenset(self._names) + + def pair(self, name: str) -> tuple[ET.Element, ET.Element]: + """The ``w:bookmarkStart`` and ``w:bookmarkEnd`` of a new bookmark *name*.""" + if not _BOOKMARK_RE.fullmatch(name): + raise InternalError("a bookmark name is not valid (a bug in the DOCX writer)") + if name.lower() in self._names: + raise InternalError("a bookmark name is used twice (a bug in the DOCX writer)") + self._names.add(name.lower()) + identifier = self._next + self._next += 1 + return el("bookmarkStart", {"id": identifier, "name": name}), el("bookmarkEnd", {"id": identifier}) + + def around(self, name: str, *content: Any) -> list[ET.Element]: + """*content* between a new bookmark's start and end, as a list for a paragraph.""" + start, end = self.pair(name) + return [start, *_flatten(content), end] diff --git a/src/legaldown_render/writers/docx/package.py b/src/legaldown_render/writers/docx/package.py new file mode 100644 index 0000000..073475e --- /dev/null +++ b/src/legaldown_render/writers/docx/package.py @@ -0,0 +1,150 @@ +"""The Open Packaging Conventions container: parts, relationships, content types +and the ZIP they are written to (ISO/IEC 29500-2). + +Everything here is deterministic: parts are written in the order they were added +(after ``[Content_Types].xml`` and ``_rels/.rels``), relationship ids are numbered +in the order the relationships were added, and the ZIP entries have a fixed time, +attributes and compression, so the same document is the same bytes. +""" +from __future__ import annotations + +import io +import re +import zipfile +from urllib.parse import quote + +from ...errors import InternalError +from ...urls import LINK_SCHEMES, is_link_target, url_scheme +from .ooxml import CONTENT_TYPES, PACKAGE_RELATIONSHIPS, _t, flat_part + +#: Relationship types (ISO/IEC 29500-1, 11.3 and 15.2). +REL_OFFICE_DOCUMENT = "http://schemas.openxmlformats.org/officeDocument/2006/relationships/officeDocument" +REL_CORE_PROPERTIES = "http://schemas.openxmlformats.org/package/2006/relationships/metadata/core-properties" +REL_EXTENDED_PROPERTIES = "http://schemas.openxmlformats.org/officeDocument/2006/relationships/extended-properties" +REL_STYLES = "http://schemas.openxmlformats.org/officeDocument/2006/relationships/styles" +REL_NUMBERING = "http://schemas.openxmlformats.org/officeDocument/2006/relationships/numbering" +REL_SETTINGS = "http://schemas.openxmlformats.org/officeDocument/2006/relationships/settings" +REL_HEADER = "http://schemas.openxmlformats.org/officeDocument/2006/relationships/header" +REL_FOOTER = "http://schemas.openxmlformats.org/officeDocument/2006/relationships/footer" +REL_HYPERLINK = "http://schemas.openxmlformats.org/officeDocument/2006/relationships/hyperlink" + +#: Content types of the parts this writer produces. +CT_RELATIONSHIPS = "application/vnd.openxmlformats-package.relationships+xml" +CT_XML = "application/xml" +CT_DOCUMENT = "application/vnd.openxmlformats-officedocument.wordprocessingml.document.main+xml" +CT_STYLES = "application/vnd.openxmlformats-officedocument.wordprocessingml.styles+xml" +CT_NUMBERING = "application/vnd.openxmlformats-officedocument.wordprocessingml.numbering+xml" +CT_SETTINGS = "application/vnd.openxmlformats-officedocument.wordprocessingml.settings+xml" +CT_HEADER = "application/vnd.openxmlformats-officedocument.wordprocessingml.header+xml" +CT_FOOTER = "application/vnd.openxmlformats-officedocument.wordprocessingml.footer+xml" +CT_CORE_PROPERTIES = "application/vnd.openxmlformats-package.core-properties+xml" +CT_EXTENDED_PROPERTIES = "application/vnd.openxmlformats-officedocument.extended-properties+xml" + +#: What may stand in a URI reference unescaped: RFC 3986's unreserved and reserved +#: characters, and ``%`` (an escape already there stays one). +_URI_SAFE = "!#$&'()*+,/:;=?@[]%-._~" +_URI_FORBIDDEN = re.compile(r"[\x00-\x1f\x7f-\x9f\ud800-\udfff]") + + +def external_target(url: str) -> str | None: + """*url* as the Target of an external relationship, which must be a valid URI + reference: what a URI cannot hold (a space, a quotation mark, ``<``, ``>``, a + backslash, a non-ASCII letter) is percent-encoded as UTF-8, an escape already + in it is kept. None when it cannot be made one: it is empty, or holds a control + character or a lone surrogate, which no encoding makes safe to hand to Word.""" + url = url.strip() + if not url or _URI_FORBIDDEN.search(url): + return None + return quote(url, safe=_URI_SAFE) + + +class Relationships: + """The relationships of one part, with ids ``rId1``, ``rId2``, ... in the order they were added.""" + + def __init__(self) -> None: + self._items: list[tuple[str, str, bool]] = [] + self._ids: dict[tuple[str, str, bool], str] = {} + + def add(self, relationship_type: str, target: str, *, external: bool = False) -> str: + """The id of the relationship to *target* (a part name relative to the owning + part, or with *external* an address, see :func:`external_target`), added if + it is not there yet.""" + key = (relationship_type, target, external) + if key not in self._ids: + self._items.append(key) + self._ids[key] = f"rId{len(self._items)}" + return self._ids[key] + + def add_external(self, url: str) -> str | None: + """The id of a hyperlink relationship to *url*, or None when *url* is not one the + document may link to: the caller then shows the link's text alone. It is the + HTML writer's rule (:func:`~legaldown_render.urls.is_link_target`) and more, as + Word, unlike a browser, follows what a relationship says without asking: the URL + needs a scheme from :data:`~legaldown_render.urls.LINK_SCHEMES` (no relative path, + and no ``//host``), has no backslash, and can be written as a URI + (:func:`external_target`).""" + stripped = url.strip() + scheme = url_scheme(stripped) + if scheme is None or scheme not in LINK_SCHEMES or not is_link_target(stripped) \ + or stripped.startswith("//") or "\\" in stripped: + return None + target = external_target(stripped) + return None if target is None else self.add(REL_HYPERLINK, target, external=True) + + def __bool__(self) -> bool: + return bool(self._items) + + def to_xml(self) -> bytes: + return flat_part(PACKAGE_RELATIONSHIPS, "Relationships", [ + ("Relationship", {"Id": f"rId{index}", "Type": relationship_type, "Target": target, + "TargetMode": "External" if external else None}, None) + for index, (relationship_type, target, external) in enumerate(self._items, 1)]) + + +class Package: + """A DOCX package under construction: parts by name (``/word/document.xml`` + without the slash), their relationships, and the ZIP they become.""" + + def __init__(self) -> None: + self._parts: dict[str, tuple[str, bytes]] = {} + #: The package's own relationships (``_rels/.rels``): the main document and the properties. + self.relationships = Relationships() + self._part_relationships: dict[str, Relationships] = {} + + def add(self, name: str, content_type: str, data: bytes, relationships: Relationships | None = None) -> None: + """Add the part *name* (such as ``word/styles.xml``) with its *content_type*, and + the *relationships* it is the source of, if it has any.""" + if name in self._parts: + raise InternalError(f"the part {name} is added twice (a bug in the DOCX writer)") + self._parts[name] = (content_type, data) + if relationships: + self._part_relationships[name] = relationships + + def _content_types(self) -> bytes: + return flat_part(CONTENT_TYPES, "Types", [ + ("Default", {"Extension": "rels", "ContentType": CT_RELATIONSHIPS}, None), + ("Default", {"Extension": "xml", "ContentType": CT_XML}, None), + *(("Override", {"PartName": f"/{name}", "ContentType": content_type}, None) + for name, (content_type, _) in self._parts.items())]) + + def to_bytes(self) -> bytes: + """The package as a ZIP: content types first, then the package relationships, then + every part with its own relationships beside it (``word/_rels/document.xml.rels``).""" + entries: list[tuple[str, bytes]] = [("[Content_Types].xml", self._content_types()), + ("_rels/.rels", self.relationships.to_xml())] + for name, (_, data) in self._parts.items(): + entries.append((name, data)) + if name in self._part_relationships: + folder, _, base = name.rpartition("/") + entries.append((f"{folder}/_rels/{base}.rels" if folder else f"_rels/{base}.rels", + self._part_relationships[name].to_xml())) + buffer = io.BytesIO() + with zipfile.ZipFile(buffer, "w") as archive: + for name, data in entries: + # A fixed date, system and mode; the compression the same on every platform. + info = zipfile.ZipInfo(_t(name), date_time=(1980, 1, 1, 0, 0, 0)) + info.compress_type = zipfile.ZIP_DEFLATED + info.create_system = 3 + info.external_attr = 0o100644 << 16 + archive.writestr(info, data, compresslevel=6) + return buffer.getvalue() diff --git a/src/legaldown_render/writers/docx/properties.py b/src/legaldown_render/writers/docx/properties.py new file mode 100644 index 0000000..1ff74e5 --- /dev/null +++ b/src/legaldown_render/writers/docx/properties.py @@ -0,0 +1,20 @@ +"""``docProps/core.xml`` and ``docProps/app.xml``: the document's properties. + +They carry the title and the language, and the program that wrote the file. They carry +no date and no author, so that the same document is the same bytes on any day (the +dates of a contract are its own, in its text).""" +from __future__ import annotations + +from .ooxml import EXTENDED_PROPERTIES, el, flat_part, serialize + + +def build_core(title: str, language: str) -> bytes: + """``docProps/core.xml``: ``dc:title`` (left out when the title is empty) and ``dc:language``.""" + return serialize(el("cp:coreProperties", None, + el("dc:title", text=title) if title.strip() else None, + el("dc:language", text=language))) + + +def build_app(application: str) -> bytes: + """``docProps/app.xml``: the name of the program, as Word shows it under Properties.""" + return flat_part(EXTENDED_PROPERTIES, "Properties", [("Application", {}, application)]) diff --git a/src/legaldown_render/writers/docx/runs.py b/src/legaldown_render/writers/docx/runs.py new file mode 100644 index 0000000..22e5660 --- /dev/null +++ b/src/legaldown_render/writers/docx/runs.py @@ -0,0 +1,272 @@ +"""Inlines as runs: text, formatting, fields, hyperlinks and bookmarks (§13.3 to §13.5, §8). + +:class:`InlineWriter` turns the resolved inlines of the tree into the content of a paragraph, and +the segments of the plan into runs. The plan says what a cross-reference is made of (literal text +and ``REF`` fields with their cached results) and which bookmark every anchor is; the writer +lays that out and decides nothing (docs/decisions/0003). It writes the cached result of a field +exactly as the plan gives it: Word computes the live value when it updates the field. +""" +from __future__ import annotations + +import xml.etree.ElementTree as ET +from collections.abc import Collection, Iterable, Mapping, Sequence +from dataclasses import dataclass, replace +from typing import Any + +from ...errors import InternalError +from ...resolve.docx_plan import DocxPlan, FieldSeg, Literal_, Segment +from ...tree import ( + Blank, + Choice, + Code, + CrossRef, + DefinedTerm, + Emphasis, + FailureMarker, + HardBreak, + Image, + Inline, + Link, + SoftBreak, + Strong, + TermRef, + Text, + Value, + has_visible_content, + plain_text, +) +from ...urls import displayed_url, is_link_target +from .ooxml import _BOOKMARK_RE, Bookmarks, complex_field, el, hyperlink_external, hyperlink_internal, r, text_run, val +from .package import Relationships +from .styles import ( + BLANK, + CHOICE, + DEFINED_TERM, + FAILURE_MARKER, + HYPERLINK, + INLINE_CODE, + TERM_REFERENCE, +) + + +@dataclass(frozen=True, slots=True) +class Format: + """The character formatting of the runs being written: a character style and the emphasis + that enclosing ``*``/``**`` add. *link_color* is set inside a link, where a run has one character + style and it is not ``Hyperlink``: the run then carries the link's colour and underline directly.""" + + style: str | None = None + bold: bool = False + italic: bool = False + link_color: str | None = None + + @property + def props(self) -> list[Any]: + """The run properties (``w:rPr`` children, in any order, :func:`~.ooxml.rpr` sorts them).""" + linked = self.link_color is not None and self.style != HYPERLINK + return [val("rStyle", self.style) if self.style else None, self.bold and [el("b"), el("bCs")], + self.italic and [el("i"), el("iCs")], linked and el("color", {"val": self.link_color}), + linked and el("u", {"val": "single"})] + + def styled(self, style: str) -> Format: + return replace(self, style=style) + + +#: The longest screen tip of a hyperlink, in characters (Word's limit). +_TOOLTIP = 255 + +#: No character formatting. +PLAIN = Format() + + +def reference_instruction(segment: FieldSeg) -> str: + """The instruction of *segment*: ``REF bookmark``, then ``\\n`` (the paragraph number), ``\\t`` (without + the text around the counter) and ``\\h`` (a link to the bookmark), as the segment says.""" + return " ".join(["REF", segment.bookmark, *(switch for switch, on in ( + ("\\n", segment.number), ("\\t", segment.suppress_text), ("\\h", segment.hyperlink)) if on)]) + + +class InlineWriter: + """Writes inlines for one document. + + *plan* names the bookmarks and the reference fields; *relationships* receives the + external hyperlinks (:meth:`~.package.Relationships.add_external`: a link the document may not + make stays text); *bookmarks* allocates the ids of the bookmarks written here; *placed* is the + lower-cased names of the bookmarks the document writes, so that no field or link points at one that is + not there (None: not known yet, as in the dry run that finds them out, :meth:`.body.BodyWriter.body`); + *attachment_titles* maps an attachment's anchor to its title as text; *link_color* is the colour of a + hyperlink (``w:color``), which runs inside a link carry when their own style is not ``Hyperlink``.""" + + def __init__(self, plan: DocxPlan, relationships: Relationships, bookmarks: Bookmarks, + placed: Collection[str] | None, attachment_titles: Mapping[str, str], link_color: str) -> None: + self.plan = plan + self.relationships = relationships + self.bookmarks = bookmarks + self.placed = placed + self.attachment_titles = attachment_titles + self.link_color = link_color + #: Targets of the cross-references that are written as their text (the plan's ``literal_refs``), while + #: a heading's title is written. + self._literal: frozenset[str] = frozenset() + + def _is_placed(self, name: str) -> bool: + return self.placed is None or name.lower() in self.placed + + def _reference_target(self, name: str) -> str: + """*name*, a bookmark a field points at: it must be a valid name and written by the document, or the plan + is wrong (Word would show "Error! Reference source not found").""" + if not _BOOKMARK_RE.fullmatch(name): + raise InternalError("a reference field names a bookmark that is not a valid name (a bug in the DOCX plan)") + if not self._is_placed(name): + raise InternalError(f"a reference field points at the bookmark '{name}', which the document does not " + "contain (a bug in the DOCX plan)") + return name + + # -- plan segments -------------------------------------------------------------------- + + def segments(self, segments: Iterable[Segment], fmt: Format = PLAIN) -> list[ET.Element]: + """The runs of *segments*: literal text, and each ``REF`` field with its cached result. + A field whose bookmark the document does not write, or that is not a bookmark name, is a bug in the plan.""" + runs: list[ET.Element] = [] + for segment in segments: + if isinstance(segment, Literal_): + if segment.text: + runs.append(text_run(segment.text, props=fmt.props)) + continue + self._reference_target(segment.bookmark) + runs += complex_field(reference_instruction(segment), + [text_run(segment.cached, props=fmt.props)] if segment.cached else [], + props=fmt.props) + return runs + + # -- inlines -------------------------------------------------------------------------- + + def runs(self, inlines: Iterable[Inline], fmt: Format = PLAIN, *, in_link: bool = False) -> list[ET.Element]: + """The content of a paragraph (runs, fields, hyperlinks, bookmarks) for *inlines*. *in_link* is + True inside an external link, where no further link is made: its text stays one link.""" + content: list[ET.Element] = [] + for inline in inlines: + content += self._inline(inline, fmt, in_link) + return content + + def title_runs(self, inlines: Iterable[Inline], literal: Collection[str] = ()) -> list[ET.Element]: + """The runs of a heading's title, in which a cross-reference to any of the units *literal* is its text + and no field (the plan says its field would lead back into the title).""" + self._literal = frozenset(literal) + try: + return self.runs(inlines) + finally: + self._literal = frozenset() + + def _inline(self, inline: Inline, fmt: Format, in_link: bool) -> list[ET.Element]: + match inline: + case Text(text=text): + return [text_run(text, props=fmt.props)] + case SoftBreak(): + return [text_run(" ", props=fmt.props)] + case HardBreak(): + return [r(el("br"), props=fmt.props)] + case Emphasis(children=children): + return self.runs(children, replace(fmt, italic=True), in_link=in_link) + case Strong(children=children): + return self.runs(children, replace(fmt, bold=True), in_link=in_link) + case Code(text=text): + return [text_run(text, props=fmt.styled(INLINE_CODE).props)] + case Link(): + return self._link(inline, fmt, in_link) + case Image(children=children): + # Alt text at conformance Level 2: no file is read (§2.3, §17.6). + alt = plain_text(children) + return [text_run(alt, props=fmt.props)] if alt else [] + case CrossRef(text=text, target_unit=unit): + reference = self.plan.refs.get(unit) if unit is not None and unit not in self._literal else None + if reference is None: + return [text_run(text, props=fmt.props)] + return self.segments(reference.segments, fmt) + case TermRef(text=text, target=target): + run = text_run(text, props=fmt.styled(TERM_REFERENCE).props) + name = self._internal(target) if target is not None and not in_link else None + return [hyperlink_internal(name, run)] if name else [run] + case DefinedTerm(children=children, anchor=anchor, defining=defining): + content = self.runs(children, fmt.styled(DEFINED_TERM), in_link=in_link) + name = self.plan.anchors.get(anchor) if anchor and defining else None + return self.bookmarks.around(name, content) if name else content + case Value(): + return self._value(inline, fmt, in_link) + case Blank(text=text, prompt=prompt): + runs = [text_run(text, props=fmt.styled(BLANK).props)] + if prompt: + runs.append(text_run(f" ({prompt})", props=replace(fmt, italic=True).props)) + return runs + case Choice(options=options, separator=separator): + return [text_run("[" + separator.join(options) + "]", props=fmt.styled(CHOICE).props)] + case FailureMarker(text=text): + return [text_run(text, props=fmt.styled(FAILURE_MARKER).props)] + case _: + raise InternalError(f"a {type(inline).__name__} reached the DOCX writer, which does not write it " + "(a bug: the tree is not resolved)") + + def _internal(self, anchor: str) -> str | None: + """The bookmark of the output *anchor*, when the document has it.""" + name = self.plan.anchors.get(anchor) + return name if name is not None and self._is_placed(name) else None + + def _linked(self, fmt: Format) -> Format: + """*fmt* inside a link: the ``Hyperlink`` style, and the link's colour for runs that take another style.""" + return replace(fmt, style=HYPERLINK, link_color=self.link_color) + + def _link_runs(self, link: Link, fmt: Format) -> list[ET.Element]: + """What a link shows (§8.7): its text, or its address when it has none, as HTML does. An image with no + alt text writes no run, so it shows the address too: a hyperlink is never empty.""" + shown = link.children if has_visible_content(link.children) else () + runs = self.runs(shown, fmt, in_link=True) + return runs or self.runs((Text(displayed_url(link.href)),), fmt, in_link=True) + + def _link(self, link: Link, fmt: Format, in_link: bool) -> list[ET.Element]: + """A link: external through a relationship, or to a bookmark for ``#anchor`` of this document. A link that + cannot be made one here shows what HTML shows: its text (address when it has none) when the URL is a link + at all (§8.7), else its children alone.""" + if in_link: + return self._link_runs(link, fmt) + tooltip = plain_text(link.title)[:_TOOLTIP] or None + if link.href.startswith("#"): + name = self._internal(link.href[1:]) + if name is not None: + return [hyperlink_internal(name, *self._link_runs(link, self._linked(fmt)), tooltip=tooltip)] + else: + identifier = self.relationships.add_external(link.href) + if identifier is not None: + return [hyperlink_external(identifier, *self._link_runs(link, self._linked(fmt)), tooltip=tooltip)] + if is_link_target(link.href): + return self._link_runs(link, fmt) + return self.runs(link.children, fmt) + + def _value(self, value: Value, fmt: Format, in_link: bool) -> list[ET.Element]: + """A field value as text. An ``attach`` value leads to its attachment: a ``REF`` field when the + text is the attachment's title (so Word keeps it equal to the title), else a link with the + text it shows (``label=``).""" + text = [text_run(value.text, props=fmt.props)] + href = value.href + if not href or in_link: + return text + linked = text_run(value.text, props=self._linked(fmt).props) + if href.startswith("#"): + name = self._internal(href[1:]) + if name is None: + return text + if self.attachment_titles.get(href[1:]) == value.text: + return complex_field(f"REF {self._reference_target(name)} \\h", text, props=fmt.props) + return [hyperlink_internal(name, linked)] + identifier = self.relationships.add_external(href) + return text if identifier is None else [hyperlink_external(identifier, linked)] + + # -- helpers for paragraphs ----------------------------------------------------------- + + def points(self, names: Sequence[str]) -> list[ET.Element]: + """Bookmarks that mark a point: each start is followed by its end.""" + return [element for name in names for element in self.bookmarks.pair(name)] + + def wrap(self, names: Sequence[str], content: Sequence[Any]) -> list[ET.Element]: + """*content* inside nested bookmarks, the first of *names* outermost.""" + pairs = [self.bookmarks.pair(name) for name in names] + return [*(start for start, _ in pairs), *content, *(end for _, end in reversed(pairs))] diff --git a/src/legaldown_render/writers/docx/settings.py b/src/legaldown_render/writers/docx/settings.py new file mode 100644 index 0000000..7859705 --- /dev/null +++ b/src/legaldown_render/writers/docx/settings.py @@ -0,0 +1,37 @@ +"""``word/settings.xml`` and ``word/numbering.xml``.""" +from __future__ import annotations + +import xml.etree.ElementTree as ET +from collections.abc import Iterable + +from .ooxml import el, serialize, val + +#: ``w:compat``'s mode 15 is Word 2013 and later; without it Word opens the file in Compatibility Mode. +COMPATIBILITY_MODE = 15 + + +def build_settings(*, update_fields: bool) -> bytes: + """``word/settings.xml``. *update_fields* writes ``w:updateFields``, so that Word + updates every field (cross-references, the table of contents and its page + numbers) when it opens the file, after asking once. + + The children are in the order CT_Settings requires: ``updateFields`` comes + after ``characterSpacingControl`` (and ``alwaysMergeEmptyNamespace``) and before + ``compat``. There are no revision save ids (``w:rsids``): they would make two + renders of one document differ and say nothing.""" + return serialize(el( + "settings", None, + val("defaultTabStop", 720), + val("characterSpacingControl", "doNotCompress"), + update_fields and val("updateFields", "true"), + el("compat", None, el("compatSetting", { + "name": "compatibilityMode", "uri": "http://schemas.microsoft.com/office/word", + "val": COMPATIBILITY_MODE})), + )) + + +def build_numbering(abstracts: Iterable[ET.Element] = (), nums: Iterable[ET.Element] = ()) -> bytes: + """``word/numbering.xml`` holding the ``w:abstractNum`` definitions, then the ``w:num`` + instances that CT_Numbering requires in that order. With none it is an + empty part: valid, and ready for the numbering definitions of headings, lists and paragraphs.""" + return serialize(el("numbering", None, abstracts, nums)) diff --git a/src/legaldown_render/writers/docx/styles.py b/src/legaldown_render/writers/docx/styles.py new file mode 100644 index 0000000..dae96a3 --- /dev/null +++ b/src/legaldown_render/writers/docx/styles.py @@ -0,0 +1,303 @@ +"""``word/styles.xml``: the Word styles a style template becomes (docs/style-templates.md). + +The document body only names these styles (``Heading1``, ``Quote``, ``DraftingNote``, +...), so a lawyer who changes a style in Word changes every paragraph that uses +it. The built-in styles keep the names Word knows them by (``heading 1``, +``toc 1``, ``Title``, ``Hyperlink``, ``Table Grid``), because Word's table of +contents, navigation pane and style gallery find them by name; the ones that are +the renderer's own are custom styles. +""" +from __future__ import annotations + +import re +import xml.etree.ElementTree as ET +from collections.abc import Iterable +from typing import Any + +from ...style import Style +from ...style.docx import DocxSettings, HeadingFormat +from .ooxml import el, ppr, rpr, serialize, val + +# Style ids. A paragraph or a run names a style by id; Word shows the name. +NORMAL = "Normal" +TITLE = "Title" +SUBTITLE = "Subtitle" +QUOTE = "Quote" +CONTENTS_HEADING = "ContentsHeading" +DRAFTING_NOTE = "DraftingNote" +DRAFTING_NOTE_LABEL = "DraftingNoteLabel" +CODE = "Code" +INLINE_CODE = "InlineCode" +TABLE_GRID = "TableGrid" +HYPERLINK = "Hyperlink" +HEADER = "Header" +FOOTER = "Footer" +DEFINED_TERM = "DefinedTerm" +TERM_REFERENCE = "TermReference" +BLANK = "Blank" +CHOICE = "Choice" +FAILURE_MARKER = "FailureMarker" +CONDITION_LABEL = "ConditionLabel" +TITLE_BLOCK_TEXT = "TitleBlockText" +TITLE_BLOCK_LABEL = "TitleBlockLabel" +PARTY_SIDE = "PartySide" +PARTY_NAME = "PartyName" +ATTACHMENTS_HEADING = "AttachmentsHeading" +ATTACHMENT_TITLE = "AttachmentTitle" +ATTACHMENT_FILE = "AttachmentFile" +SIGNATURE_SIDE = "SignatureSide" +SIGNATURE_TEXT = "SignatureText" +SIGNATURE_LINE = "SignatureLine" + + +def heading_style(level: int) -> str: + """The id of the style for heading *level* (1 to 5), named ``heading N``.""" + return f"Heading{level}" + + +def toc_style(level: int) -> str: + """The id of the table of contents' style for entries of *level* (1 to 5), named ``toc N``.""" + return f"TOC{level}" + + +#: The left indent in twips that ``Quote`` and ``DraftingNote`` set, which the body adds to the paragraphs +#: that carry an indent of their own (a list item in a quote). +QUOTE_INDENT = 567 +NOTE_INDENT = 227 +CODE_INDENT = 113 +#: The outline level (0-based) of the attachments heading and of an attachment's title: the levels the +#: contents lists them at, as the HTML contents does (the heading at 1, the titles under it at 2). +ATTACHMENTS_OUTLINE = 0 +ATTACHMENT_OUTLINE = 1 +#: The space above a signing line, in twips: room to sign in. +SIGNATURE_SPACE = 720 + +_MUTED = "666666" +_RULE = "C8C8C8" +_MONOSPACE = "Courier New" +#: Two of the HTML writer's marker colours, so the formats look alike. +_FAILURE_FILL, _FAILURE_TEXT = "FDE2E1", "8A1C14" +_CONDITION_FILL, _CONDITION_TEXT = "EFE9FB", "5B3FA0" +_NOTE_FILL, _NOTE_RULE = "FFF8E1", "D9A400" +_CODE_FILL = "F5F5F5" + +_LANGUAGE_RE = re.compile(r"[A-Za-z]{2,3}(?:-[A-Za-z0-9]{1,8}){0,3}") + + +def language_tag(*candidates: str | None) -> str: + """The first of *candidates* that is a language tag (``en``, ``cs-CZ``), else ``en-US``: + the document's ``language`` is free text to the renderer, and ``w:lang`` is read by Word's proofing.""" + return next((tag for tag in candidates if tag and _LANGUAGE_RE.fullmatch(tag)), "en-US") + + +def _fonts(name: str) -> ET.Element: + """The Latin font. The complex-script and East Asian fonts stay unset, for Word to choose by + script: they are not the Latin font's to give.""" + return el("rFonts", {"ascii": name, "hAnsi": name}) + + +def _size(half_points: int) -> list[ET.Element]: + return [val("sz", half_points), val("szCs", half_points)] + + +def _shade(fill: str) -> ET.Element: + return el("shd", {"val": "clear", "color": "auto", "fill": fill}) + + +def _border(side: str, color: str, size: int = 4, space: int = 0) -> ET.Element: + return el(side, {"val": "single", "sz": size, "space": space, "color": color}) + + +def container_bar(style_id: str) -> ET.Element | None: + """The left border that ``Quote`` and ``DraftingNote`` draw, for the paragraphs inside them that are not + in that style (code, rules), so that the bar runs unbroken; None for any other style.""" + return {QUOTE: lambda: _border("left", _RULE, 18, 10), + DRAFTING_NOTE: lambda: _border("left", _NOTE_RULE, 24, 8)}.get(style_id, lambda: None)() + + +def _style(style_id: str, name: str, *, kind: str = "paragraph", based_on: str | None = None, + next_style: str | None = None, priority: int | None = None, quick: bool = False, custom: bool = False, + hidden_until_used: bool = False, default: bool = False, props: Iterable[Any] = (), + run: Iterable[Any] = (), table: ET.Element | None = None, table_conditions: Iterable[Any] = (), + semi_hidden: bool = False) -> ET.Element: + """One ``w:style``, its children in the order CT_Style requires.""" + return el( + "style", {"type": kind, "default": True if default else None, "customStyle": True if custom else None, + "styleId": style_id}, + val("name", name), + based_on and val("basedOn", based_on), + next_style and val("next", next_style), + priority is not None and val("uiPriority", priority), + semi_hidden and el("semiHidden"), + hidden_until_used and el("unhideWhenUsed"), + quick and el("qFormat"), + ppr(props), + rpr(run), + table, + table_conditions, + ) + + +def _text_style(kind: str) -> list[ET.Element]: + """The run properties of a ``definitions.style`` or ``term_style`` value.""" + return { + "bold": [el("b"), el("bCs")], + "italic": [el("i"), el("iCs")], + "underline": [el("u", {"val": "single"})], + "small-caps": [el("smallCaps")], + }.get(kind, []) + + +def _heading_run(heading: HeadingFormat | None, settings: DocxSettings) -> list[Any]: + """The run properties of a heading level (None: a style that sets none: bold at the body size), for the + paragraph styles that look like a heading and are not one (so that no heading numbering reaches them).""" + if heading is None: + return [el("b"), el("bCs"), _size(settings.half_points)] + return [heading.bold and [el("b"), el("bCs")], heading.italic and [el("i"), el("iCs")], + heading.caps and el("caps"), heading.small_caps and el("smallCaps"), _size(heading.half_points)] + + +def build_styles(style: Style, settings: DocxSettings, language: str, *, heading_num_id: int | None = None) -> bytes: + """``word/styles.xml`` for *style*, converted to Word units as *settings*, in the + document *language* (``w:lang``). *heading_num_id* links ``heading 1`` to ``heading 5`` + to that numbering definition, one level each (``w:numPr`` in the style, matched by a + ``w:pStyle`` in the level).""" + font_points = settings.points + + def em(factor: float) -> int: + """Twips in *factor* times the body font size.""" + return round(factor * font_points * 20) + + def half_points(factor: float) -> int: + return max(2, round(factor * settings.half_points)) + + text_width = settings.page.text_width + defaults = el( + "docDefaults", + None, + el("rPrDefault", None, rpr(_fonts(settings.font), _size(settings.half_points), + el("lang", {"val": language}))), + el("pPrDefault"), + ) + + styles = [ + _style(NORMAL, "Normal", default=True, quick=True, + props=[el("spacing", {"after": em(0.6), "line": settings.line, "lineRule": settings.line_rule}), + el("jc", {"val": "both"}) if settings.justify else None], + run=[el("color", {"val": settings.color}), _size(settings.half_points)]), + _style("DefaultParagraphFont", "Default Paragraph Font", kind="character", default=True, priority=1, + semi_hidden=True, hidden_until_used=True), + _style("TableNormal", "Normal Table", kind="table", default=True, priority=99, semi_hidden=True, + hidden_until_used=True, + table=el("tblPr", None, el("tblInd", {"w": 0, "type": "dxa"}), + el("tblCellMar", None, *(el(side, {"w": width, "type": "dxa"}) + for side, width in (("top", 0), ("left", 108), + ("bottom", 0), ("right", 108)))))), + _style("NoList", "No List", kind="numbering", default=True, priority=99, semi_hidden=True, + hidden_until_used=True), + ] + + for level in range(1, 6): + heading = settings.headings.get(level) + if heading is None: + continue + styles.append(_style( + heading_style(level), f"heading {level}", based_on=NORMAL, next_style=NORMAL, priority=9, quick=True, + props=[el("keepNext"), el("keepLines"), + heading_num_id is not None and el("numPr", None, val("ilvl", level - 1), + val("numId", heading_num_id)), + el("spacing", {"before": em(1.2) if level == 1 else em(0.9), "after": em(0.4)}), + el("jc", {"val": heading.align}), val("outlineLvl", level - 1)], + run=[heading.bold and [el("b"), el("bCs")], heading.italic and [el("i"), el("iCs")], + heading.caps and el("caps"), heading.small_caps and el("smallCaps"), _size(heading.half_points)])) + + first = settings.headings.get(1) + second = settings.headings.get(2) or first + styles += [ + _style(TITLE, "Title", based_on=NORMAL, next_style=NORMAL, priority=10, quick=True, + props=[el("keepNext"), el("spacing", {"before": 0, "after": em(0.5)}), el("jc", {"val": "center"})], + run=[el("b"), el("bCs"), _size(half_points(1.6))]), + _style(SUBTITLE, "Subtitle", based_on=NORMAL, next_style=NORMAL, priority=11, quick=True, + props=[el("spacing", {"before": 0, "after": em(1)}), el("jc", {"val": "center"})], + run=[el("i"), el("iCs"), el("color", {"val": _MUTED}), _size(half_points(1.1))]), + _style(QUOTE, "Quote", based_on=NORMAL, next_style=NORMAL, priority=29, quick=True, + props=[el("pBdr", None, _border("left", _RULE, 18, 10)), + el("spacing", {"before": em(0.5), "after": em(0.5)}), el("ind", {"left": QUOTE_INDENT, "right": 567})]), + _style(CONTENTS_HEADING, "Contents Heading", based_on=NORMAL, next_style=NORMAL, priority=39, custom=True, + props=[el("keepNext"), el("spacing", {"before": em(1.2), "after": em(0.4)}), + el("jc", {"val": first.align if first else "left"})], + run=[(first is None or first.bold) and [el("b"), el("bCs")], first and first.caps and el("caps"), + first and first.small_caps and el("smallCaps"), + _size(first.half_points if first else settings.half_points)]), + _style(TITLE_BLOCK_TEXT, "Title Block Text", based_on=NORMAL, priority=41, custom=True, + props=[el("spacing", {"before": 0, "after": em(0.2)}), el("jc", {"val": "left"})]), + _style(PARTY_SIDE, "Party Side", based_on=NORMAL, next_style=NORMAL, priority=42, custom=True, + props=[el("keepNext"), el("spacing", {"before": em(1), "after": em(0.2)}), el("jc", {"val": "left"})], + run=[el("b"), el("bCs")]), + _style(PARTY_NAME, "Party Name", based_on=NORMAL, priority=43, custom=True, + props=[el("keepNext"), el("spacing", {"before": em(0.5), "after": em(0.2)}), el("jc", {"val": "left"})]), + _style(ATTACHMENTS_HEADING, "Attachments Heading", based_on=NORMAL, next_style=NORMAL, priority=44, + custom=True, props=[el("keepNext"), el("keepLines"), el("spacing", {"before": em(1.2), "after": em(0.4)}), + el("jc", {"val": first.align if first else "left"}), + val("outlineLvl", ATTACHMENTS_OUTLINE)], + run=_heading_run(first, settings)), + _style(ATTACHMENT_TITLE, "Attachment Title", based_on=NORMAL, next_style=NORMAL, priority=45, custom=True, + props=[el("keepNext"), el("keepLines"), el("spacing", {"before": em(0.9), "after": em(0.4)}), + el("jc", {"val": second.align if second else "left"}), val("outlineLvl", ATTACHMENT_OUTLINE)], + run=_heading_run(second, settings)), + _style(ATTACHMENT_FILE, "Attachment File", based_on=NORMAL, priority=46, custom=True, + run=[el("color", {"val": _MUTED})]), + _style(SIGNATURE_SIDE, "Signature Side", based_on=NORMAL, priority=47, custom=True, + props=[el("keepLines"), el("spacing", {"before": em(1.5), "after": em(0.2)}), el("jc", {"val": "left"})], + run=[el("b"), el("bCs")]), + _style(SIGNATURE_TEXT, "Signature Text", based_on=NORMAL, priority=48, custom=True, + props=[el("keepLines"), el("spacing", {"before": 0, "after": em(0.2)}), el("jc", {"val": "left"})]), + _style(SIGNATURE_LINE, "Signature Line", based_on=NORMAL, priority=49, custom=True, + props=[el("keepLines"), el("pBdr", None, _border("bottom", "auto", 4, 1)), + el("spacing", {"before": SIGNATURE_SPACE, "after": em(0.3)})]), + _style(DRAFTING_NOTE, "Drafting Note", based_on=NORMAL, priority=30, custom=True, + props=[el("keepLines"), el("pBdr", None, _border("left", _NOTE_RULE, 24, 8)), _shade(_NOTE_FILL), + el("spacing", {"before": em(0.4), "after": em(0.4)}), el("ind", {"left": NOTE_INDENT, "right": 113})]), + _style(CODE, "Code", based_on=NORMAL, priority=31, custom=True, + props=[_shade(_CODE_FILL), el("spacing", {"before": 0, "after": 0, "line": 240, "lineRule": "auto"}), + el("ind", {"left": CODE_INDENT, "right": 113}), el("jc", {"val": "left"})], + run=[_fonts(_MONOSPACE), _size(half_points(0.9))]), + _style(TABLE_GRID, "Table Grid", kind="table", based_on="TableNormal", priority=39, + table=el("tblPr", None, el("tblBorders", None, *(_border(side, _RULE, 4) + for side in ("top", "left", "bottom", "right", + "insideH", "insideV")))), + table_conditions=el("tblStylePr", {"type": "firstRow"}, rpr([el("b"), el("bCs")]))), + _style(HEADER, "header", based_on=NORMAL, priority=99, hidden_until_used=True, + props=[el("spacing", {"after": 0, "line": 240, "lineRule": "auto"})], + run=[el("color", {"val": _MUTED}), _size(half_points(0.9))]), + _style(FOOTER, "footer", based_on=NORMAL, priority=99, hidden_until_used=True, + props=[el("spacing", {"after": 0, "line": 240, "lineRule": "auto"})], + run=[el("color", {"val": _MUTED}), _size(half_points(0.9))]), + ] + for level in range(1, 6): + styles.append(_style( + toc_style(level), f"toc {level}", based_on=NORMAL, next_style=NORMAL, priority=39, hidden_until_used=True, + props=[el("tabs", None, el("tab", {"val": "right", "leader": "dot", "pos": text_width})), + el("spacing", {"after": em(0.2)}), el("ind", {"left": 240 * (level - 1)})])) + + styles += [ + _style(HYPERLINK, "Hyperlink", kind="character", priority=99, hidden_until_used=True, + run=[el("color", {"val": settings.link_color}), el("u", {"val": "single"})]), + _style(INLINE_CODE, "Inline Code", kind="character", priority=32, custom=True, + run=[_fonts(_MONOSPACE), _size(half_points(0.9))]), + _style(DEFINED_TERM, "Defined Term", kind="character", priority=33, custom=True, + run=_text_style(style.definitions.style)), + _style(TERM_REFERENCE, "Term Reference", kind="character", priority=34, custom=True, + run=_text_style(style.definitions.term_style)), + _style(BLANK, "Blank", kind="character", priority=35, custom=True, run=[el("noProof")]), + _style(CHOICE, "Choice", kind="character", priority=36, custom=True, run=[_shade(_CONDITION_FILL)]), + _style(FAILURE_MARKER, "Failure Marker", kind="character", priority=37, custom=True, + run=[el("color", {"val": _FAILURE_TEXT}), _shade(_FAILURE_FILL)]), + _style(CONDITION_LABEL, "Condition Label", kind="character", priority=38, custom=True, + run=[el("color", {"val": _CONDITION_TEXT}), _size(half_points(0.8)), _shade(_CONDITION_FILL)]), + _style(TITLE_BLOCK_LABEL, "Title Block Label", kind="character", priority=41, custom=True, + run=[el("i"), el("iCs")]), + _style(DRAFTING_NOTE_LABEL, "Drafting Note Label", kind="character", priority=40, custom=True, + run=[el("b"), el("bCs"), _size(half_points(0.9))]), + ] + return serialize(el("styles", None, defaults, styles)) diff --git a/tests/conformance/test_spec_corpus.py b/tests/conformance/test_spec_corpus.py index 32b94dd..807e6f2 100644 --- a/tests/conformance/test_spec_corpus.py +++ b/tests/conformance/test_spec_corpus.py @@ -54,6 +54,7 @@ def test_renders_without_crashing(path: Path) -> None: try: html = render(source).output render(source, format="text") + assert render(source, format="docx").binary is not None except DocumentError: assert "frontmatter-invalid-yaml" in str(path) or "frontmatter" in str(path) return diff --git a/tests/conftest.py b/tests/conftest.py index c01c435..0d6b241 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -61,6 +61,20 @@ def pytest_addoption(parser: pytest.Parser) -> None: help="rewrite the golden files from the current output instead of comparing") +def pytest_collection_modifyitems(config: pytest.Config, items: list[pytest.Item]) -> None: + """Leave the DOCX acceptance corpus (marker ``acceptance``) out of a plain ``pytest`` run, which stays as quick as it + was: it runs when a marker expression is given (``-m libreoffice``, as the DOCX job does), when + ``LEGALDOWN_ACCEPTANCE`` is set, or when a command-line argument is its file (``tests/test_docx_acceptance.py``, + with or without a ``::test``).""" + if config.getoption("markexpr") or os.environ.get("LEGALDOWN_ACCEPTANCE") or any( + Path(argument.split("::")[0]).name == "test_docx_acceptance.py" for argument in config.args): + return + dropped = [item for item in items if item.get_closest_marker("acceptance")] + if dropped: + config.hook.pytest_deselected(items=dropped) + items[:] = [item for item in items if item not in dropped] + + @pytest.fixture def update_golden(request: pytest.FixtureRequest) -> bool: return bool(request.config.getoption("--update-golden")) diff --git a/tests/docx_support.py b/tests/docx_support.py new file mode 100644 index 0000000..4ea86e6 --- /dev/null +++ b/tests/docx_support.py @@ -0,0 +1,265 @@ +"""Helpers for the DOCX tests: reading a package, validating its parts against the +ISO/IEC 29500-4 transitional schemas, and running LibreOffice over it. + +The schemas are not in the repository (tests/schemas/README.md): ``python tests/schemas/fetch.py`` +downloads them. The tests that need them, or LibreOffice, skip when they are missing, unless +``LEGALDOWN_REQUIRE_DOCX_TOOLS`` is set (the CI job sets it), when a missing tool is a failure. +""" +from __future__ import annotations + +import io +import json +import os +import posixpath +import shutil +import subprocess +import tempfile +import xml.etree.ElementTree as ET +import zipfile +from collections.abc import Callable +from functools import cache +from pathlib import Path + +import pytest + +ROOT = Path(__file__).parent +UNO_EXPORT = ROOT / "lo" / "uno_export.py" +UNO_EDIT = ROOT / "lo" / "uno_edit.py" + +W_NS = "http://schemas.openxmlformats.org/wordprocessingml/2006/main" +W = f"{{{W_NS}}}" + + +def tool_missing(reason: str) -> None: + """Skip, or fail when the environment promises the tools (CI): a job that sets + ``LEGALDOWN_REQUIRE_DOCX_TOOLS`` must not pass by skipping what it exists to run.""" + if os.environ.get("LEGALDOWN_REQUIRE_DOCX_TOOLS"): + pytest.fail(f"required tool missing: {reason}") + pytest.skip(reason) + + +# -- Reading a package ------------------------------------------------------------------- + + +def parts(data: bytes) -> dict[str, bytes]: + """Every entry of the ZIP *data*, by name, in archive order.""" + with zipfile.ZipFile(io.BytesIO(data)) as archive: + return {name: archive.read(name) for name in archive.namelist()} + + +def texts(root: ET.Element) -> list[str]: + """The text of every paragraph under *root*, tabs as TAB and breaks as LF (the tab stops + of a paragraph's properties are not text, so only the content of runs counts).""" + lines = [] + for paragraph in root.iter(f"{W}p"): + line = "" + for run in paragraph.iter(f"{W}r"): + for node in run: + if node.tag == f"{W}t": + line += node.text or "" + elif node.tag == f"{W}tab": + line += "\t" + elif node.tag == f"{W}br": + line += "\n" + lines.append(line) + return lines + + +# -- Schemas ----------------------------------------------------------------------------- + + +def schema_directory() -> Path | None: + candidate = Path(os.environ.get("LEGALDOWN_OOXML_SCHEMAS") or ROOT / "schemas" / "ooxml") + return candidate if (candidate / "ld-wml.xsd").is_file() else None + + +@cache +def _schemas() -> dict[str, object]: + from lxml import etree + + directory = schema_directory() + assert directory is not None + return {name: etree.XMLSchema(etree.parse(str(directory / file))) + for name, file in (("wml", "ld-wml.xsd"), ("types", "opc-contentTypes.xsd"), + ("rels", "opc-relationships.xsd"), + ("app", "shared-documentPropertiesExtended.xsd"))} + + +def require_schemas() -> None: + """Skip (or fail in CI) unless lxml and the schemas are there.""" + try: + import lxml # noqa: F401 + except ImportError: + tool_missing("lxml is not installed (a test dependency: pip install -e '.[dev]')") + if schema_directory() is None: + tool_missing("the OOXML schemas are not fetched (python tests/schemas/fetch.py)") + + +def schema_for(name: str) -> str | None: + """Which schema validates the part *name*, or None when none is available (core.xml: + its schema imports Dublin Core from the web).""" + if name == "[Content_Types].xml": + return "types" + if name.endswith(".rels"): + return "rels" + if name == "docProps/app.xml": + return "app" + if name.startswith("word/") and name.endswith(".xml"): + return "wml" + return None + + +def well_formed(data: bytes) -> object: + """The parsed XML of *data* (raising lxml's error if it is not well-formed), without + resolving entities or touching the network.""" + from lxml import etree + + return etree.fromstring(data, etree.XMLParser(resolve_entities=False, no_network=True, huge_tree=False)) + + +def validation_errors(data: bytes) -> list[str]: + """Everything wrong with the package *data*: a part that is not well-formed or does not + validate against its schema, a relationship to a part that is not there, a part that has + no content type. Empty when the package is sound.""" + from lxml import etree + + require_schemas() + problems: list[str] = [] + contents = parts(data) + for name, content in contents.items(): + try: + root = well_formed(content) + except etree.XMLSyntaxError as error: + problems.append(f"{name}: not well-formed: {error}") + continue + schema_name = schema_for(name) + if schema_name is not None: + schema = _schemas()[schema_name] + if not schema.validate(root): # type: ignore[attr-defined] + problems += [f"{name}: {entry.message} (line {entry.line})" + for entry in schema.error_log] # type: ignore[attr-defined] + problems += _package_problems(contents) + return problems + + +def _package_problems(contents: dict[str, bytes]) -> list[str]: + from lxml import etree + + problems: list[str] = [] + names = list(contents) + if names[:1] != ["[Content_Types].xml"]: + problems.append("[Content_Types].xml is not the first entry") + types = etree.fromstring(contents["[Content_Types].xml"]) + defaults = {entry.get("Extension") for entry in types if entry.tag.endswith("}Default")} + overrides = {entry.get("PartName") for entry in types if entry.tag.endswith("}Override")} + for name in names: + if name != "[Content_Types].xml" and f"/{name}" not in overrides and name.rpartition(".")[2] not in defaults: + problems.append(f"{name}: no content type") + for override in overrides: + if override.lstrip("/") not in contents: + problems.append(f"{override}: a content type for a part that is not there") + for name, content in contents.items(): + if not name.endswith(".rels"): + continue + owner_folder = posixpath.dirname(posixpath.dirname(name)) + for relationship in etree.fromstring(content): + if relationship.get("TargetMode") == "External": + continue + target = posixpath.normpath(posixpath.join(owner_folder, relationship.get("Target"))) + if target not in contents: + problems.append(f"{name}: the target {relationship.get('Target')} is not a part") + return problems + + +# -- LibreOffice --------------------------------------------------------------------------- + + +@cache +def system_python() -> str | None: + """A Python that can ``import uno`` (python3-uno installs it for the system's Python, not + for a virtual environment's), or None.""" + for candidate in ("/usr/bin/python3", shutil.which("python3"), shutil.which("python")): + if candidate and Path(candidate).is_file(): + done = subprocess.run([candidate, "-c", "import uno"], capture_output=True, check=False) + if done.returncode == 0: + return candidate + return None + + +def require_libreoffice() -> str: + """The Python to run the UNO script with, or skip (fail in CI) when LibreOffice or its Python bindings are missing.""" + if shutil.which("soffice") is None: + tool_missing("LibreOffice (soffice) is not installed") + python = system_python() + if python is None: + tool_missing("no Python with the uno module (python3-uno) was found") + assert python is not None + return python + + +class Exported: + """What LibreOffice says about a document once its fields are updated.""" + + def __init__(self, text: str, header: str, footer: str, pages: int) -> None: + self.text = text + self.header = header + self.footer = footer + self.pages = pages + + @property + def lines(self) -> list[str]: + return self.text.splitlines() + + +def repackage(data: bytes, name: str, change: Callable[[str], str]) -> bytes: + """The package *data* with the text of the part *name* changed by *change*: for a test that + the checks catch what they are meant to.""" + buffer = io.BytesIO() + with zipfile.ZipFile(buffer, "w") as archive: + for part, content in parts(data).items(): + archive.writestr(part, change(content.decode("utf-8")).encode("utf-8") if part == name else content) + return buffer.getvalue() + + +def libreoffice_export(documents: dict[str, bytes], *, update: bool = True) -> dict[str, Exported]: + """Open each document (name to DOCX bytes) in one LibreOffice instance, update its fields + and indexes (unless *update* is False: the fields as the file caches them), and return its + text, header, footer and page count.""" + python = require_libreoffice() + with tempfile.TemporaryDirectory(prefix="ld-lo-") as folder: + out = Path(folder, "out") + out.mkdir() + for name, data in documents.items(): + Path(folder, f"{name}.docx").write_bytes(data) + done = subprocess.run([python, str(UNO_EXPORT), *([] if update else ["--no-update"]), str(out), + *(str(Path(folder, f"{n}.docx")) for n in documents)], + capture_output=True, text=True, timeout=300 + 3 * len(documents), check=False) + if done.returncode != 0: + pytest.fail(f"LibreOffice export failed:\n{done.stdout}\n{done.stderr}") + result = {} + for name in documents: + text = Path(out, f"{name}.txt").read_text(encoding="utf-8-sig") + info = json.loads(Path(out, f"{name}.json").read_text(encoding="utf-8")) + result[name] = Exported(text, info["header"], info["footer"], info["pages"]) + return result + + +def libreoffice_edit(jobs: dict[str, tuple[bytes, list[tuple[str, ...]]]]) -> dict[str, str]: + """Open each document (name to DOCX bytes and edits) in one LibreOffice instance, insert a paragraph with the text + ``new`` before the first paragraph that holds ``match`` for each ``(match, new)`` (delete that paragraph when + ``new`` is None; put it on the list and level of the paragraph that holds ``like`` for ``(match, new, like)``), + update the fields and indexes, and return the text of each (tests/lo/uno_edit.py).""" + python = require_libreoffice() + with tempfile.TemporaryDirectory(prefix="ld-lo-") as folder: + out = Path(folder, "out") + out.mkdir() + specification = [] + for name, (data, edits) in jobs.items(): + Path(folder, f"{name}.docx").write_bytes(data) + specification.append({"name": name, "file": str(Path(folder, f"{name}.docx")), "edits": edits}) + Path(folder, "jobs.json").write_text(json.dumps(specification), encoding="utf-8") + done = subprocess.run([python, str(UNO_EDIT), str(out), str(Path(folder, "jobs.json"))], + capture_output=True, text=True, timeout=300, check=False) + if done.returncode != 0: + pytest.fail(f"LibreOffice edit failed:\n{done.stdout}\n{done.stderr}") + return {name: Path(out, f"{name}.txt").read_text(encoding="utf-8-sig") for name in jobs} diff --git a/tests/lo/uno_edit.py b/tests/lo/uno_edit.py new file mode 100644 index 0000000..5832d09 --- /dev/null +++ b/tests/lo/uno_edit.py @@ -0,0 +1,146 @@ +#!/usr/bin/python3 +"""Edit DOCX files in headless LibreOffice through UNO the way a lawyer does, update the fields, and write what +the document says: for each job, a paragraph is inserted before the first paragraph that holds some text (so that +it takes that paragraph's numbering and style, as pressing Enter at its start does), or that paragraph is deleted, +then every field and index is updated (the indexes too, so a table of contents lists what was inserted) and the body +is written as plain text (``JOB.txt``, UTF-8, one line per paragraph, read from the document model: tests/lo/uno_text.py). + +Run it with the system Python that has the ``uno`` module, **not** with the project's virtual environment: + + /usr/bin/python3 tests/lo/uno_edit.py OUTDIR JOBS.json + +``JOBS.json`` is a list of ``{"name": "job", "file": "/path/doc.docx", "edits": [["text of a paragraph", "new"], ...]}``; +an edit whose new text is ``null`` deletes the paragraph whose text is exactly that instead. An edit with a third entry, +``["text of a paragraph", "new", "text of another"]``, puts the new paragraph on the list and the level of the paragraph that +holds the third text, as pasting a copy of that item at this place does (a new paragraph otherwise takes the numbering of +the one it is inserted before). +One LibreOffice instance serves every job. Exit status 0 when every job was exported. +""" +from __future__ import annotations + +import contextlib +import json +import os +import shutil +import subprocess +import sys +import tempfile +import time +import uuid + +import uno # type: ignore[import-not-found] # noqa: I001 - provided by python3-uno, not by pip +from com.sun.star.beans import PropertyValue # type: ignore[import-not-found] +from com.sun.star.text.ControlCharacter import PARAGRAPH_BREAK # type: ignore[import-not-found] +from uno_text import body_text + + +def _value(name: str, value: object) -> PropertyValue: + prop = PropertyValue() + prop.Name, prop.Value = name, value + return prop + + +def _connect(pipe: str, process: subprocess.Popen[bytes]) -> object: + local = uno.getComponentContext() + resolver = local.ServiceManager.createInstanceWithContext("com.sun.star.bridge.UnoUrlResolver", local) + for _ in range(120): + if process.poll() is not None: + raise SystemExit("soffice exited before it accepted a connection") + try: + return resolver.resolve(f"uno:pipe,name={pipe};urp;StarOffice.ComponentContext") + except Exception: # not listening yet + time.sleep(0.5) + raise SystemExit("soffice did not accept a connection in time") + + +def _paragraph(document: object, match: str) -> object: + """The first paragraph of the body that holds *match*.""" + paragraphs = document.getText().createEnumeration() + while paragraphs.hasMoreElements(): + paragraph = paragraphs.nextElement() + if (paragraph.supportsService("com.sun.star.text.Paragraph") and not paragraph.ParaStyleName.startswith("Contents") + and match in paragraph.getString()): + return paragraph + raise SystemExit(f"no paragraph holds {match!r}") + + +def _edit(document: object, match: str, new: str | None, like: str | None = None) -> None: + """Insert a paragraph with the text *new* before the first paragraph that holds *match*, on the list and level of the + paragraph that holds *like* when that is given, or, when *new* is None, delete the first paragraph whose text is + exactly *match* (with its break, as Delete at its end does; the last paragraph of the document with the break before + it, so that no empty paragraph is left).""" + text = document.getText() + paragraphs = text.createEnumeration() + while paragraphs.hasMoreElements(): + paragraph = paragraphs.nextElement() + # A line of a table of contents holds the heading's words too: only the document's own paragraphs count. + if not paragraph.supportsService("com.sun.star.text.Paragraph") or paragraph.ParaStyleName.startswith("Contents"): + continue + if new is None: + if paragraph.getString() != match: + continue + cursor = text.createTextCursorByRange(paragraph.getStart()) + cursor.gotoEndOfParagraph(True) + if not cursor.goRight(1, True): # the last paragraph: take the break before it instead + cursor = text.createTextCursorByRange(paragraph.getStart()) + cursor.goLeft(1, False) + cursor.gotoEnd(True) + cursor.setString("") + return + if match in paragraph.getString(): + cursor = text.createTextCursorByRange(paragraph.getStart()) + text.insertControlCharacter(cursor, PARAGRAPH_BREAK, False) + cursor.goLeft(1, False) + cursor.setString(new) + if like is not None: + copied = _paragraph(document, like) + for name in ("NumberingRules", "ListId", "NumberingLevel"): + cursor.setPropertyValue(name, copied.getPropertyValue(name)) + return + raise SystemExit(f"no paragraph {'is' if new is None else 'holds'} {match!r}") + + +def edit(out_dir: str, jobs: list[dict]) -> None: + profile = tempfile.mkdtemp(prefix="lo-profile-") + pipe = f"legaldown_{uuid.uuid4().hex}" + process = subprocess.Popen( + ["soffice", "--headless", "--invisible", "--norestore", "--nologo", "--nodefault", + f"--accept=pipe,name={pipe};urp;", f"-env:UserInstallation=file://{profile}"], + stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) + desktop = None + try: + context = _connect(pipe, process) + desktop = context.ServiceManager.createInstanceWithContext("com.sun.star.frame.Desktop", context) + for job in jobs: + document = desktop.loadComponentFromURL(uno.systemPathToFileUrl(os.path.abspath(job["file"])), "_blank", 0, + (_value("Hidden", True),)) + if document is None: + raise SystemExit(f"LibreOffice could not open {job['file']}") + try: + for match, new, *like in job["edits"]: + _edit(document, match, new, *like) + for _ in range(2): # page numbers and indexes settle after the first layout + indexes = document.getDocumentIndexes() + for position in range(indexes.getCount()): + indexes.getByIndex(position).update() + document.getTextFields().refresh() + document.refresh() + with open(os.path.join(out_dir, f"{job['name']}.txt"), "w", encoding="utf-8", newline="") as handle: + handle.write(body_text(document)) + finally: + document.close(True) + finally: + if desktop is not None: + with contextlib.suppress(Exception): # already gone + desktop.terminate() + time.sleep(0.5) + process.kill() + process.wait() + shutil.rmtree(profile, ignore_errors=True) + + +if __name__ == "__main__": + if len(sys.argv) != 3: + raise SystemExit(__doc__) + with open(sys.argv[2], encoding="utf-8") as handle: + edit(sys.argv[1], json.load(handle)) diff --git a/tests/lo/uno_export.py b/tests/lo/uno_export.py new file mode 100644 index 0000000..9246526 --- /dev/null +++ b/tests/lo/uno_export.py @@ -0,0 +1,118 @@ +#!/usr/bin/python3 +"""Open DOCX files in headless LibreOffice through UNO, update every field and index, +and write what the document says: one ``NAME.txt`` per file (the body as plain text, +UTF-8, one line per paragraph, read from the document model: tests/lo/uno_text.py) and one +``NAME.json`` with the page count and the text of the running header and footer. + +Run it with the system Python that has the ``uno`` module (``/usr/bin/python3`` once +``python3-uno`` is installed), **not** with the project's virtual environment: + + /usr/bin/python3 tests/lo/uno_export.py [--no-update] OUTDIR FILE.docx [FILE.docx ...] + +The same LibreOffice instance opens every file, so a test module pays the start-up +once. Exit status 0 when every file was exported. + +With ``--no-update`` the fields are left as the file caches them: what the export says +is then what the writer wrote (a document whose cached results were made wrong on +purpose shows the wrong results there, and the right ones only after the update, which +is how a round trip proves that LibreOffice computed the numbers and did not read them). +""" +from __future__ import annotations + +import contextlib +import json +import os +import shutil +import subprocess +import sys +import tempfile +import time +import uuid + +import uno # type: ignore[import-not-found] # noqa: I001 - provided by python3-uno, not by pip +from com.sun.star.beans import PropertyValue # type: ignore[import-not-found] +from uno_text import body_text + + +def _value(name: str, value: object) -> PropertyValue: + prop = PropertyValue() + prop.Name, prop.Value = name, value + return prop + + +def _connect(pipe: str, process: subprocess.Popen[bytes]) -> object: + local = uno.getComponentContext() + resolver = local.ServiceManager.createInstanceWithContext("com.sun.star.bridge.UnoUrlResolver", local) + for _ in range(120): + if process.poll() is not None: + raise SystemExit("soffice exited before it accepted a connection") + try: + return resolver.resolve(f"uno:pipe,name={pipe};urp;StarOffice.ComponentContext") + except Exception: # not listening yet + time.sleep(0.5) + raise SystemExit("soffice did not accept a connection in time") + + +def _update(document: object) -> None: + """Update every index and field, twice: page numbers settle after the first layout.""" + for _ in range(2): + indexes = document.getDocumentIndexes() + for position in range(indexes.getCount()): + indexes.getByIndex(position).update() + document.getTextFields().refresh() + document.refresh() + + +def _page_texts(document: object) -> tuple[str, str]: + """The text of the first page style's header and footer ("" when it has none).""" + styles = document.getStyleFamilies().getByName("PageStyles") + page = styles.getByName(document.getCurrentController().getViewCursor().PageStyleName) + header = page.HeaderText.getString() if page.HeaderIsOn else "" + footer = page.FooterText.getString() if page.FooterIsOn else "" + return header, footer + + +def export(out_dir: str, files: list[str], *, update: bool = True) -> None: + profile = tempfile.mkdtemp(prefix="lo-profile-") + pipe = f"legaldown_{uuid.uuid4().hex}" + process = subprocess.Popen( + ["soffice", "--headless", "--invisible", "--norestore", "--nologo", "--nodefault", + f"--accept=pipe,name={pipe};urp;", f"-env:UserInstallation=file://{profile}"], + stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) + desktop = None + try: + context = _connect(pipe, process) + desktop = context.ServiceManager.createInstanceWithContext("com.sun.star.frame.Desktop", context) + for path in files: + path = os.path.abspath(path) + stem = os.path.splitext(os.path.basename(path))[0] + document = desktop.loadComponentFromURL(uno.systemPathToFileUrl(path), "_blank", 0, + (_value("Hidden", True),)) + if document is None: + raise SystemExit(f"LibreOffice could not open {path}") + try: + if update: + _update(document) + header, footer = _page_texts(document) + with open(os.path.join(out_dir, f"{stem}.txt"), "w", encoding="utf-8", newline="") as handle: + handle.write(body_text(document)) + pages = document.getCurrentController().PageCount + with open(os.path.join(out_dir, f"{stem}.json"), "w", encoding="utf-8") as handle: + json.dump({"header": header, "footer": footer, "pages": pages}, handle) + finally: + document.close(True) + finally: + if desktop is not None: + with contextlib.suppress(Exception): # already gone + desktop.terminate() + time.sleep(0.5) + process.kill() + process.wait() + shutil.rmtree(profile, ignore_errors=True) + + +if __name__ == "__main__": + arguments = [argument for argument in sys.argv[1:] if argument != "--no-update"] + if len(arguments) < 2: + raise SystemExit(__doc__) + export(arguments[0], arguments[1:], update="--no-update" not in sys.argv[1:]) diff --git a/tests/lo/uno_text.py b/tests/lo/uno_text.py new file mode 100644 index 0000000..18e5448 --- /dev/null +++ b/tests/lo/uno_text.py @@ -0,0 +1,57 @@ +"""What a LibreOffice document says, read from its model instead of from a text export filter. + +LibreOffice's plain-text export lays a table row out as one line (cells separated by tabs) in some versions +and as one line per cell paragraph in others, so a test that compared that text would depend on the version of +the LibreOffice that happens to be installed. The text here is made by walking the body the way the document +holds it, so it is the same under every version: one line per paragraph, a table's rows and cells in order (each +cell's paragraphs, and a table inside a cell, in place), a soft line break starting a new line, and, before the +text of a numbered paragraph, the label LibreOffice draws for it (``ListLabelString``, which is what its numbering +computes once the fields are updated) and one space. + +This module needs nothing from LibreOffice itself (it reads what the UNO objects answer), so the tests can +run it over stand-ins (tests/test_docx_lo_text.py). It is imported by tests/lo/uno_export.py and +tests/lo/uno_edit.py, which run under the system Python that has ``uno``. +""" +from __future__ import annotations + +import re + +PARAGRAPH = "com.sun.star.text.Paragraph" +TABLE = "com.sun.star.text.TextTable" + + +def _cell_order(name: str) -> tuple[int, int, str]: + """The reading order of a cell name ("A1", "B1", "A2", ..., "AA1"): by row, then by column.""" + match = re.fullmatch(r"([A-Za-z]+)(\d+)", name) + assert match is not None, name + return int(match[2]), len(match[1]), match[1] + + +def paragraph_lines(paragraph: object) -> list[str]: + """The lines of one paragraph: its text, split at the soft line breaks, the first one led by the list label + (and a space) when the paragraph shows one.""" + label = paragraph.ListLabelString # type: ignore[attr-defined] + text = paragraph.getString() # type: ignore[attr-defined] + lines = text.replace("\r\n", "\n").replace("\r", "\n").split("\n") + if label: + lines[0] = f"{label} {lines[0]}" + return lines + + +def collect(enumeration: object, lines: list[str]) -> None: + """Append the lines of every paragraph and table that *enumeration* (of a text or of a table cell) yields.""" + while enumeration.hasMoreElements(): # type: ignore[attr-defined] + element = enumeration.nextElement() # type: ignore[attr-defined] + if element.supportsService(TABLE): + for name in sorted(element.getCellNames(), key=_cell_order): + collect(element.getCellByName(name).createEnumeration(), lines) + elif element.supportsService(PARAGRAPH): + lines += paragraph_lines(element) + + +def body_text(document: object) -> str: + """The body of *document* as text: every line ends with a line feed, so that an empty last paragraph shows as + an empty last line, as in a plain-text file.""" + lines: list[str] = [] + collect(document.getText().createEnumeration(), lines) # type: ignore[attr-defined] + return "".join(f"{line}\n" for line in lines) diff --git a/tests/schemas/README.md b/tests/schemas/README.md new file mode 100644 index 0000000..727a77b --- /dev/null +++ b/tests/schemas/README.md @@ -0,0 +1,46 @@ +# OOXML schemas for the DOCX tests + +`tests/test_docx_validation.py` validates every part of the generated documents against the ISO/IEC +29500-4 (transitional) schemas. The schemas are **not stored in this repository**. Fetch them with + +``` +python tests/schemas/fetch.py # writes tests/schemas/ooxml/, which git ignores +``` + +(standard library only; `LEGALDOWN_OOXML_SCHEMAS=DIR` points the tests at another copy). The tests +skip when they are missing, and fail instead when `LEGALDOWN_REQUIRE_DOCX_TOOLS` is set, as the `docx` +job of `.github/workflows/ci.yml` does after fetching them. + +## What is fetched, and from where + +| Files | Source | Used for | +|---|---|---| +| `wml.xsd` and the schemas it imports (`dml-*.xsd`, `shared-*.xsd`, ...) | `OfficeOpenXML-XMLSchema-Transitional.zip` in [ECMA-376 5th edition, Part 4](https://ecma-international.org/wp-content/uploads/ECMA-376-4_5th_edition_december_2016.zip) (the transitional schemas of ISO/IEC 29500-4:2016) | `document.xml`, `styles.xml`, `numbering.xml`, `settings.xml`, headers and footers; `app.xml` (`shared-documentPropertiesExtended.xsd`) | +| `opc-contentTypes.xsd`, `opc-relationships.xsd` | `OpenPackagingConventions-XMLSchema.zip` in [ECMA-376 5th edition, Part 2](https://ecma-international.org/wp-content/uploads/ECMA-376-2_5th_edition_december_2021.zip) | `[Content_Types].xml` and every `.rels` | +| `xml.xsd` | [W3C](https://www.w3.org/2001/xml.xsd) | `xml:space` and `xml:lang`, which `wml.xsd` imports without saying where they are | +| `ld-wml.xsd` | written by `fetch.py` (ours, a few lines) | a wrapper that loads `xml.xsd` before `wml.xsd`, as libxml2 needs | + +Each download is checked against the SHA-256 recorded in `fetch.py`; a changed file stops the script. + +**Not validated:** `docProps/core.xml`. Its schema (`opc-coreProperties.xsd`) imports the Dublin Core +schemas from `dublincore.org` over the network. The tests check that part's shape directly +(tests/test_docx_package.py). Everything else is validated; the package itself (content types for +every part, every relationship target present, `[Content_Types].xml` first) is checked in +`tests/docx_support.py`. + +## Why they are fetched and not vendored + +Committing them would need a licence that allows redistribution. None is clear, so they are fetched: + +* The XSD files carry no licence or copyright header, and the standards' own text (the Part 4 and Part 2 PDFs + in the same downloads) contains no redistribution grant that could be extracted from it. +* The schemas are published, free of charge, by Ecma International (ECMA-376) and, as ISO/IEC 29500, by + ISO/IEC, whose webstore terms apply to that copy. Both are copyrighted standards texts. Ecma's general + practice is to allow copying with the notice retained, but that was not confirmed here for these files, and + the ISO/IEC terms differ. +* Another copy of the same schemas, bundled with a third-party tool whose licence forbids copying and + distributing it, was available to the author. It was neither used nor copied: only the files fetched from + the publishers are used. + +If the maintainers clear the licence (for example by asking Ecma), the files can be committed here and +`fetch.py` dropped; nothing else depends on where they live. diff --git a/tests/schemas/fetch.py b/tests/schemas/fetch.py new file mode 100644 index 0000000..f9240ca --- /dev/null +++ b/tests/schemas/fetch.py @@ -0,0 +1,73 @@ +"""Download the OOXML schemas the DOCX tests validate against. + +Usage: ``python tests/schemas/fetch.py [DIRECTORY]`` (default: ``tests/schemas/ooxml``, +which is ignored by git). Standard library only. See tests/schemas/README.md for what +is fetched and why it is not stored in the repository. + +Every download is checked against the SHA-256 it had when the schemas were last +reviewed; a different file stops the script, so a changed upstream is noticed rather +than trusted. +""" +from __future__ import annotations + +import hashlib +import io +import sys +import time +import urllib.request +import zipfile +from pathlib import Path + +#: (url, sha256 of the download, the inner zip to open, the files to take from it) +SOURCES = ( + ("https://ecma-international.org/wp-content/uploads/ECMA-376-4_5th_edition_december_2016.zip", + "bd25da1109f73762356596918bf5ff8b74a1331642dba5f1c1d1dfc6bed34ecd", + "OfficeOpenXML-XMLSchema-Transitional.zip", None), + ("https://ecma-international.org/wp-content/uploads/ECMA-376-2_5th_edition_december_2021.zip", + "1d489dc491168ea1f9e9a59063acc8dd5f02b4ad1d21aa7ec19ba9a58d020c70", + "OpenPackagingConventions-XMLSchema.zip", ("opc-contentTypes.xsd", "opc-relationships.xsd")), +) +XML_XSD = ("https://www.w3.org/2001/xml.xsd", "61960fb3131e38022caad5360e2f33a3382578ab3c80cd58bd74320ede61b20c") + +#: wml.xsd imports the XML namespace without saying where it is; libxml2 needs it loaded first. +WRAPPER = """ + + + + +""" + + +def download(url: str, sha256: str) -> bytes: + last: Exception | None = None + for attempt in range(4): + try: + with urllib.request.urlopen(url, timeout=120) as response: # noqa: S310 - fixed https URLs + data = response.read() + break + except OSError as error: + last = error + time.sleep(2 * (attempt + 1)) + else: + raise SystemExit(f"cannot download {url}: {last}") + found = hashlib.sha256(data).hexdigest() + if found != sha256: + raise SystemExit(f"{url}: expected SHA-256 {sha256}, got {found}; review the change before updating this script") + return data + + +def fetch(target: Path) -> None: + target.mkdir(parents=True, exist_ok=True) + for url, sha256, inner, wanted in SOURCES: + with zipfile.ZipFile(io.BytesIO(download(url, sha256))) as outer, \ + zipfile.ZipFile(io.BytesIO(outer.read(inner))) as schemas: + for name in schemas.namelist(): + if name.endswith(".xsd") and (wanted is None or name in wanted): + (target / Path(name).name).write_bytes(schemas.read(name)) + (target / "xml.xsd").write_bytes(download(*XML_XSD)) + (target / "ld-wml.xsd").write_text(WRAPPER, encoding="utf-8") + print(f"schemas written to {target}") + + +if __name__ == "__main__": + fetch(Path(sys.argv[1]) if len(sys.argv) > 1 else Path(__file__).parent / "ooxml") diff --git a/tests/test_cli_io.py b/tests/test_cli_io.py index a01d484..a5ff668 100644 --- a/tests/test_cli_io.py +++ b/tests/test_cli_io.py @@ -214,14 +214,14 @@ def test_the_extension_of_a_file_name(name: str, extension: str) -> None: @pytest.mark.parametrize(("name", "expected"), [ (".txt", "text"), ("x.txt.", "text"), ("x.html ", "html"), (".htm", "html"), ("x.unknown", None), + ("x.docx", "docx"), ("X.DOCX", "docx"), ("x.docx ", "docx"), (".docx", "docx"), ("x.doc", None), ]) def test_the_format_a_file_name_implies(name: str, expected: str | None) -> None: assert format_for_path(name) == expected -@pytest.mark.parametrize("name", ["x.docx", "x.pdf", "x.md", "x.DOCX", "x.doc", "x.odt", "x.rtf", "x.markdown", - "x.epub", "x.tex", "x.v2.Pdf", ".pdf", ".md", "x.pdf.", "x.docx ", "x.docx. .", - "dir.pdf", "x.MD"]) +@pytest.mark.parametrize("name", ["x.pdf", "x.md", "x.doc", "x.odt", "x.rtf", "x.markdown", + "x.epub", "x.tex", "x.v2.Pdf", ".pdf", ".md", "x.pdf.", "dir.pdf", "x.MD"]) def test_unknown_output_extension_without_format_is_a_usage_error( tmp_path: Path, capsys: pytest.CaptureFixture[str], name: str) -> None: target = tmp_path / name @@ -230,7 +230,7 @@ def test_unknown_output_extension_without_format_is_a_usage_error( assert exit_info.value.code == 2 err = capsys.readouterr().err assert f"cannot tell the output format from '{target}'" in err - assert "--format html or text" in err + assert "--format html, text or docx" in err assert not target.exists() assert list(tmp_path.iterdir()) == [] @@ -240,8 +240,8 @@ def test_unknown_output_extension_without_format_is_a_usage_error( _TRAILING = pytest.mark.skipif(sys.platform == "win32", reason="Windows strips trailing dots and spaces from names") -@pytest.mark.parametrize("name", ["x.docx", pytest.param("x.pdf.", marks=_TRAILING), ".md", - pytest.param("x.docx ", marks=_TRAILING)]) +@pytest.mark.parametrize("name", ["x.pdf", pytest.param("x.pdf.", marks=_TRAILING), ".md", + pytest.param("x.odt ", marks=_TRAILING)]) def test_explicit_format_is_honoured_whatever_the_extension(tmp_path: Path, name: str) -> None: target = tmp_path / name assert main([FEATURES, "-q", "-f", "text", "-o", str(target)]) == 0 @@ -280,4 +280,4 @@ def test_the_help_states_the_output_rule(capsys: pytest.CaptureFixture[str]) -> help_text = " ".join(capsys.readouterr().out.split()) assert "its extension sets the format" not in help_text assert "any other name, with or without an extension, writes HTML" in help_text - assert ".docx, .doc, .odt, .rtf, .pdf, .md, .markdown, .epub, .tex" in help_text + assert ".doc, .odt, .rtf, .pdf, .md, .markdown, .epub, .tex" in help_text diff --git a/tests/test_docx_acceptance.py b/tests/test_docx_acceptance.py new file mode 100644 index 0000000..f738736 --- /dev/null +++ b/tests/test_docx_acceptance.py @@ -0,0 +1,594 @@ +"""Acceptance of the DOCX writer: the measurable claims of ADR 0008, over the whole corpus and the three built-in styles. + +The corpus is that of ``test_docx_roundtrip.py`` (``tests/documents``, the specification's examples and valid +fixtures, and the documents written to stress the numbering); that module is the finer instrument (it makes the cached +results wrong on purpose and compares every number and reference unit by unit), and this one the coarse, complete one: + +* **Equivalence.** LibreOffice opens each document with its fields and indexes updated, and what it shows is what the + plain-text writer prints, line by line, **modulo the differences the formats have by nature** (:func:`normalise`: + borders for rules and signing lines, a table's cells, the markup of the plain-text layout) **and the gaps the plan + reports** (:data:`GAPS`: a number Word draws differently from the renderer's, with a ``render-docx-*`` diagnostic; + :data:`GAP_LINES` pins each line that differs, so that nothing else may). + The numbers, the cross-references, the entries of the contents, the attachments and the signature blocks are in that + text. The page numbers of the contents are checked on their own: they are positive, never decrease and never pass the + number of pages. +* **Live after edits.** In one LibreOffice session, for a handful of documents under each style, a heading, a list + item, a nested item and (under the continental style) a numbered paragraph are inserted, and a clause is deleted, + as a lawyer does (:mod:`tests/lo/uno_edit.py`); the fields and indexes are updated; the result is what the text + writer prints for the source with the same edits made, exactly: in the documents of the subset no number is fixed in a + way these edits reach. +* **Valid and deterministic.** Every part of every document validates against the ISO/IEC 29500 schemas, and rendering a + document twice gives the same bytes (that test has the marker ``acceptance`` alone). + +The LibreOffice tests are skipped when ``soffice`` or ``python3-uno`` is missing and fail instead when +``LEGALDOWN_REQUIRE_DOCX_TOOLS`` is set (``tests/docx_support.py``). The module is part of the DOCX job (``pytest -m +"libreoffice or ooxml_schema"``): ``conftest.py`` leaves it out of a run that names no marker, which keeps the default +``pytest`` as quick as it was. +""" +from __future__ import annotations + +import difflib +import functools +import re +import xml.etree.ElementTree as ET +from collections.abc import Iterator +from dataclasses import dataclass + +import pytest +from conftest import spec_root +from docx_support import ( + Exported, + W, + libreoffice_edit, + libreoffice_export, + parts, + require_libreoffice, + require_schemas, + tool_missing, + validation_errors, +) +from test_docx_roundtrip import EXPLAINING, corpus + +from legaldown_render import DocumentError, render +from legaldown_render.tree import ( + Code, + CrossRef, + DraftingNote, + Emphasis, + List, + ListItem, + Paragraph, + Quote, + RenderTree, + Section, + Strong, + Text, + iter_tree_inlines, +) +from legaldown_render.writers.text import inline_text + +pytestmark = pytest.mark.acceptance + +#: The built-in styles, with the table of contents on (down to the fifth level, so that every heading is in it). +STYLES: dict[str, dict] = { + "default": {"overrides": {"contents.enabled": True, "contents.depth": 5}}, + "outline": {"style": "outline", "overrides": {"contents.enabled": True, "contents.depth": 5}}, + "continental": {"style": "continental", "overrides": {"contents.enabled": True, "contents.depth": 5}}, +} + +#: The diagnostics that say Word (and so LibreOffice, which follows its numbering) draws a number differently from the +#: renderer's: a letter counter past 26, a Roman numeral from 4000, a list nested past Word's levels, a label that shows +#: counters Word's list cannot (``render-docx-section-label``), a number off by a count (``render-docx-number-mismatch``), +#: and the two readings of a reference that only LibreOffice has (``\t`` ignored, a level nothing numbered counted). +GAPS = EXPLAINING | {"render-docx-number-mismatch", "render-docx-reference-word-only", "render-docx-libreoffice-count"} + + +# -- The corpus --------------------------------------------------------------------------------------------------- + + +@dataclass +class Shown: + """One document under one style: what the text writer prints, what LibreOffice shows, and what the render said.""" + + name: str + style: str + tree: RenderTree + written: str + rules: frozenset[str] + data: bytes + shown: Exported | None = None + + +@functools.cache +def documents() -> dict[str, str]: + """The corpus. A run that promises its tools (CI) must have the specification's documents in it.""" + found = corpus() + if spec_root() is None: + tool_missing("no LegalDown specification checkout (set LEGALDOWN_SPEC_DIR)") + return found + + +def rendered(source: str, style: str) -> tuple[bytes, str, RenderTree, frozenset[str]] | None: + """The DOCX, the text, the tree and the rules reported for *source* under *style* (None when it is not a document).""" + try: + made = render(source, format="docx", **STYLES[style]) + except DocumentError: + return None + assert made.binary is not None + written = render(source, format="text", **STYLES[style]) + return made.binary, written.output, written.tree, frozenset(d.rule for d in made.diagnostics) + + +@functools.cache +def shown_corpus() -> dict[str, tuple[Shown, ...]]: + """Every document under every style, shown by LibreOffice with its fields and indexes updated: one session.""" + require_libreoffice() + cases: dict[str, list[Shown]] = {style: [] for style in STYLES} + for name, source in documents().items(): + for style in STYLES: + made = rendered(source, style) + if made is not None: + data, written, tree, rules = made + cases[style].append(Shown(name, style, tree, written, rules, data)) + everything = {f"d{index}": case for index, case in enumerate(case for group in cases.values() for case in group)} + exported = libreoffice_export({key: case.data for key, case in everything.items()}) + for key, case in everything.items(): + case.shown = exported[key] + return {style: tuple(group) for style, group in cases.items()} + + +# -- Comparing ----------------------------------------------------------------------------------------------------- + + +def squeeze(line: str) -> str: + return re.sub(r"\s+", " ", line).strip() + + +def blocks_of(tree: RenderTree, *, quoted: bool = True) -> Iterator[Section | Paragraph | ListItem | DraftingNote]: + """Every section, paragraph, list item and drafting note of the tree, in the order of the document; with *quoted* False + not those in a quote or a drafting note (which are neither numbered units nor anything a lawyer edits as a clause).""" + def walk(blocks: tuple) -> Iterator[Paragraph | ListItem | DraftingNote]: + for block in blocks: + match block: + case Paragraph(): + yield block + case DraftingNote(blocks=inner): + yield block + if quoted: + yield from walk(inner) + case List(): + for item in block.items: + yield item + yield from walk(item.blocks) + case Quote(blocks=inner): + if quoted: + yield from walk(inner) + + yield from walk(tree.preamble) + for section in tree.sections: + yield section + yield from walk(section.blocks) + + +def normalise(written: str, shown: str, tree: RenderTree) -> tuple[list[str], list[str], list[int]]: + """The lines of the plain-text writer's output *written* and of the text LibreOffice *shown*, made comparable, and the + page numbers of the contents. The formats differ by nature in these ways, each undone here and in no other: + + * a rule (``---``) and a signing line (``____``) are borders of paragraphs and cells in the DOCX, not text; + * a table is pipes and a separator row in the text and cells in the DOCX (tests/lo/uno_text.py reads one line a cell): a row is + compared as the words of its cells, however many lines they take, so that a text laid out by row would do as well; + * a quote's ``>`` is the indent of a style in the DOCX; + * a link's ```` after its text is the target of the hyperlink; + * a condition label (§15.8) and the label of a drafting note are in ``[ ]`` in the text and a styled run in the DOCX, + the label of a heading or an attachment is a line of its own in the text and a run of the title's paragraph in the + DOCX, and the label of a numbered paragraph stands before its number in the text and after it in the DOCX + (the number is drawn by Word before the paragraph); + * ``label: value`` rows of the title block and the parties are two cells in the DOCX (compared like a table row); + * the table of contents of LibreOffice ends its entries with a tab and a page number, which the text has not: they are + taken off and returned (their pages are not the renderer's to promise); + * white space is not compared. + """ + labels = {unit.condition_label for unit in blocks_of(tree) if getattr(unit, "condition_label", None)} + labels |= {unit.label for unit in blocks_of(tree) if isinstance(unit, DraftingNote) and unit.label} + labels |= {entry.condition_label for entry in tree.contents if entry.condition_label} + labels |= {part.condition_label for part in tree.attachments if part.condition_label} + titles = {(squeeze(" ".join(filter(None, (s.label, inline_text(s.title))))), s.condition_label) + for s in tree.sections if s.condition_label} + titles |= {(squeeze(inline_text(a.title)), a.condition_label) for a in tree.attachments if a.condition_label} + swaps = [(unit.condition_label, unit.label) for unit in blocks_of(tree) + if isinstance(unit, Paragraph) and unit.condition_label and unit.label] + rows = [*tree.header, *(row for side in tree.sides for party in side.parties for row in party.details)] + split = {squeeze(f"{label}{tree.colon}{inline_text(value)}"): [squeeze(label), squeeze(inline_text(value))] + for label, value in rows if label} + + #: (the text of a line, whether it is the cells of a table row or the label and value of a title-block row) + items: list[tuple[str, bool]] = [] + for raw in written.splitlines(): + line = raw.strip() + if re.fullmatch(r"-{3,}|_{3,}", line) or re.fullmatch(r"\|[:\-| ]+\|", line): + continue + if line.startswith("|") and line.endswith("|"): + cells = [squeeze(cell) for cell in re.split(r"(?\s]*>", "", re.sub(r"^(>\s?)+", "", line)) + for label in labels: + line = line.replace(f"[{label}]", label) + for condition, number in swaps: + line = line.replace(f"{condition} {number} ", f"{number} {condition} ") + line = squeeze(line) + if not line: + continue + if items and (items[-1][0], line) in titles: + items[-1] = (f"{items[-1][0]} {line}", False) + continue + items.append((" ".join(split[line]) if line in split else line, line in split)) + + raw_lines = shown.lstrip("\ufeff").splitlines() + pages: list[int] = [] + squeezed = [squeeze(raw) for raw in raw_lines] + if tree.contents_label and tree.contents_label in squeezed: + start = squeezed.index(tree.contents_label) + 1 + for position in range(start, min(start + len(tree.contents), len(raw_lines))): + entry = re.fullmatch(r"(.*)\t(\d+)\s*", raw_lines[position]) + if entry: + squeezed[position] = squeeze(entry[1]) + pages.append(int(entry[2])) + shown_lines = [line for line in squeezed if line] + + # A row that the text writer prints on one line is read from as many lines of LibreOffice's as it takes to say the + # same words (one a cell, as tests/lo/uno_text.py reads the document, but not by that alone). + written_lines: list[str] = [] + seen: list[str] = [] + cursor = 0 + for text, cells in items: + written_lines.append(text) + if cursor >= len(shown_lines): + continue + line = shown_lines[cursor] + cursor += 1 + while cells and line != text and text.startswith(line) and cursor < len(shown_lines): + line = squeeze(f"{line} {shown_lines[cursor]}") + cursor += 1 + seen.append(line) + seen += shown_lines[cursor:] + return written_lines, seen, pages + + +def difference(written: list[str], shown: list[str]) -> str: + return "\n".join(difflib.unified_diff(written, shown, "text writer", "LibreOffice", lineterm="", n=1)) + + +#: The lines that differ, by (style, document), where the plan reports a gap: what the text writer prints, then what LibreOffice +#: (which follows Word's numbering) shows. Nothing else may differ: not a line more, not a word of any length. +GAP_LINES: dict[tuple[str, str], list[tuple[str, str]]] = { + # letters past 26: Word repeats them (bb), the renderer counts on (ab) -- render-docx-letter-overflow + ("default", "stress/letters"): [ + ("(ab) item 28", "(bb) item 28"), ("(ac) item 29", "(cc) item 29"), ("(ad) item 30", "(dd) item 30"), + ("(ae) item 31", "(ee) item 31"), ("Ref 1(a) 1(z) 1(aa) 1(ab) 1(ae)", "Ref 1(a) 1(z) 1(aa) 1(bb) 1(ee)")], + ("outline", "stress/letters"): [ + ("(ab) item 28", "(bb) item 28"), ("(ac) item 29", "(cc) item 29"), ("(ad) item 30", "(dd) item 30"), + ("(ae) item 31", "(ee) item 31"), ("Ref I(a) I(z) I(aa) I(ab) I(ae)", "Ref I(a) I(z) I(aa) I(bb) I(ee)")], + # a list that cannot show the section's number draws its own counter alone -- render-docx-section-label + ("continental", "stress/qualified"): [("1.1 deeper", "1 deeper")], + ("continental", "stress/paths"): [("1.1 four", "1 four"), ("1.1 seven", "1 seven")], + ("continental", "stress/ordered paths"): [ + ("1.1 four", "1 four"), ("1.2 four b", "2 four b"), ("1.1 four c", "1 four c"), ("1.1 y", "1 y"), ("1.1 w", "1 w")], + ("continental", "stress/deep"): [ + ("1.1 level 3", "1 level 3"), ("1.1 level 6", "1 level 6"), ("1.1 level 9", "1 level 9"), + ("1.1 level 9 again", "1 level 9 again")], +} + + +def check(case: Shown) -> bool: + """Assert the equivalence of *case*; returns True if it holds exactly, False if it holds modulo the reported gap that + :data:`GAP_LINES` pins line by line (and that the render must report).""" + assert case.shown is not None + written, seen, pages = normalise(case.written, case.shown.text, case.tree) + if case.tree.contents: + assert pages, "the contents lists no page numbers" + assert pages == sorted(pages) and pages[0] >= 1 and pages[-1] <= case.shown.pages, (pages, case.shown.pages) + if written == seen: + assert (case.style, case.name) not in GAP_LINES, "the gap that was pinned is gone: remove it from GAP_LINES" + return True + assert case.rules & GAPS, f"the numbers differ and the render reports no gap:\n{difference(written, seen)}" + assert len(written) == len(seen), f"a gap changes lines, not their number:\n{difference(written, seen)}" + differing = [(a, b) for a, b in zip(written, seen, strict=True) if a != b] + assert differing == GAP_LINES.get((case.style, case.name)), f"not the pinned gap:\n{difference(written, seen)}" + return False + + +@pytest.mark.libreoffice +@pytest.mark.parametrize("style", STYLES) +def test_libreoffice_shows_every_document_as_the_text_writer_prints_it(style: str) -> None: + """Numbers, references, the contents, the attachments, the signatures: everything the plain-text writer prints is what + LibreOffice shows after it has updated the fields and indexes, except in the documents the plan reports a gap for.""" + for case in shown_corpus()[style]: + try: + check(case) + except AssertionError as error: + raise AssertionError(f"{style}/{case.name}: {error}") from error + + +@pytest.mark.libreoffice +def test_the_corpus_is_not_vacuous_and_every_gap_is_one_the_plan_reports() -> None: + """The comparison reaches what it is meant to: the specification's documents, a table of contents and signature + blocks, attachments, template constructs and references; and the documents that differ are few and are the + stress documents that ask for what Word draws differently (letters past 26, labels that need the section).""" + cases = [case for group in shown_corpus().values() for case in group] + assert len(cases) >= 3 * 30 + assert sum(case.name.startswith(("examples/", "fixtures/")) for case in cases) >= 3 * 18 + assert all(case.tree.contents for case in cases if case.tree.sections) + assert any(case.tree.signatures for case in cases) and any(case.tree.attachments for case in cases) + assert any(case.tree.is_template for case in cases) + assert sum(isinstance(i, CrossRef) for case in cases for i in iter_tree_inlines(case.tree)) > 100 + different = [case for case in cases if not check(case)] + assert {case.name for case in different} <= {f"stress/{name}" for name in ( + "letters", "qualified", "paths", "ordered paths", "deep")}, [(c.style, c.name) for c in different] + assert different, "the gaps the plan reports are not reached" + assert all(case.name.startswith("stress/") for case in different) + assert all(case.style != "default" or case.name == "stress/letters" for case in different) + + +@pytest.mark.libreoffice +def test_the_comparison_catches_a_wrong_number_a_missing_line_and_a_wrong_page_order() -> None: + """The check is not vacuous: a number LibreOffice draws differently, a line it leaves out, and a table of contents whose + pages run backwards are each a failure.""" + case = next(c for c in shown_corpus()["default"] if c.name == "documents/features.lgd") + assert case.shown is not None and check(case) + text = case.shown.text + assert "\n2. Services" in text + for broken in (text.replace("\n2. Services", "\n3. Services", 1), + text.replace("Business Day means", "", 1), + re.sub(r"\t(\d+)\n", lambda m: f"\t{9 - int(m[1])}\n", text, count=2)): + assert broken != text + mutated = Shown(case.name, case.style, case.tree, case.written, case.rules, case.data, Exported( + broken, case.shown.header, case.shown.footer, case.shown.pages)) + with pytest.raises(AssertionError): + check(mutated) + + +@pytest.mark.libreoffice +def test_a_table_row_on_one_line_reads_as_the_same_row_on_several() -> None: + """A table laid out as a line a cell (as the document is read) or a line a row (as some LibreOffice text filters write + it) is the same table, so both pass.""" + case = next(c for c in shown_corpus()["default"] if c.name == "documents/features.lgd") + assert case.shown is not None + text = case.shown.text + for cells in (["Service", "Fee", "Notes"], ["Hosting", "$500.00", "per month"], ["Support", "$250.75", "see 3"]): + stacked = "\n".join(cells) + assert stacked in text, cells + text = text.replace(stacked, "\t".join(cells)) + assert text != case.shown.text + assert check(Shown(case.name, case.style, case.tree, case.written, case.rules, case.data, + Exported(text, case.shown.header, case.shown.footer, case.shown.pages))) + + +# -- Edits --------------------------------------------------------------------------------------------------------- + + +@dataclass(frozen=True) +class Edit: + """What a lawyer does in Word, and the same thing in the source.""" + + kind: str + #: The text of the paragraph it goes before (UNO inserts before the first paragraph that holds it), or the whole text of + #: the paragraph that is deleted. + match: str + #: The text typed (None: the paragraph is deleted). + new: str | None + #: The line of the source it is made at (1-based), and what takes that line's place. + line: int + source: str + + +#: The documents the edits are made in: the two test documents, specification examples with lists, a template and +#: numbered clauses, and documents written for the numbering. +EDITED = ( + "documents/features.lgd", "documents/template.lgd", "examples/advanced/msa/master-service-agreement.lgd", + "examples/advanced/template/consulting-agreement-template.lgd", "examples/simple/nda/mutual-nda.lgd", + "examples/simple/policy/remote-work-policy.lgd", "fixtures/valid/anchors-and-refs.lgd", "stress/nested", + "stress/lists", "stress/alternatives", +) + +MARKER = re.compile(r"^(\s*(?:[-*+]|\d+[.)])\s+)") + + +def body_texts(data: bytes) -> list[str]: + """The text of every paragraph of the document that is not a line of the table of contents or its label.""" + root = ET.fromstring(parts(data)["word/document.xml"]) + found = [] + for paragraph in root.iter(f"{W}p"): + style = paragraph.find(f"{W}pPr/{W}pStyle") + if style is not None and re.fullmatch(r"TOC\d|ContentsHeading", style.get(f"{W}val", "")): + continue + found.append("".join(t.text or "" for t in paragraph.iter(f"{W}t"))) + return found + + +def plain(inlines: tuple) -> str | None: + """The text of *inlines* if it is words alone, possibly emphasised (a paragraph that holds a field, a value or a break has + text that changes or that Word keeps apart, which words do not find).""" + found = [] + for inline in inlines: + if isinstance(inline, Text | Code): + found.append(inline.text) + elif isinstance(inline, Emphasis | Strong): + inner = plain(inline.children) + if inner is None: + return None + found.append(inner) + else: + return None + return "".join(found) or None + + +def leading(inlines: tuple) -> str | None: + """The words a paragraph starts with, if it starts with some (up to 40 characters of its first text, which a field + later in the paragraph does not change): what UNO needs to find a paragraph to insert before.""" + first = inlines[0] if inlines else None + return first.text[:40].strip() if isinstance(first, Text) and len(first.text.strip()) >= 8 else None + + +def unique(texts: list[str], text: str) -> bool: + """Whether exactly one paragraph holds *text* (so that UNO, which takes the first that does, finds that one).""" + return len(text) >= 4 and sum(text in other for other in texts) == 1 + + +def edits_for(source: str, data: bytes, tree: RenderTree) -> list[Edit]: + """The edits the document allows: a heading, an item and a nested item inserted before the first that qualify, a + numbered paragraph where the style numbers them, and the last clause that is a single line nothing refers to deleted. + A target qualifies when it stands in the source on a line of its own, has no condition, is not a later alternative and + has text alone (so that its words find it).""" + lines = source.splitlines() + texts = body_texts(data) + referred = {i.target_unit for i in iter_tree_inlines(tree) if isinstance(i, CrossRef)} + found: dict[str, Edit] = {} + deletable: Edit | None = None + + def lead(line: int) -> str: + return lines[line - 1] if 0 < line <= len(lines) else "" + + for node in blocks_of(tree, quoted=False): + if getattr(node, "condition", "") or getattr(node, "shares_number_with", None) or node.line is None: + continue + if isinstance(node, Section): + text, row = plain(node.title) or leading(node.title), lead(node.line) + if text and "heading" not in found and re.match(rf"#{{{node.level}}} ", row) and unique(texts, text): + found["heading"] = Edit("heading", text, "ZZ heading", node.line, + f"{'#' * node.level} ZZ heading\n\n{row}") + elif isinstance(node, ListItem) and node.blocks and isinstance(node.blocks[0], Paragraph): + whole, row = plain(node.blocks[0].inlines), lead(node.line) + text = whole or leading(node.blocks[0].inlines) + marker = MARKER.match(row) + if not text or not marker or not unique(texts, text): + continue + nested = node.parent_unit is not None and any( + isinstance(other, ListItem) and other.unit == node.parent_unit for other in blocks_of(tree, quoted=False)) + kind = "nested item" if nested else "item" + if kind not in found: + found[kind] = Edit(kind, text, f"ZZ {kind}", node.line, f"{marker[1]}ZZ {kind}\n{row}") + single = whole is not None and texts.count(whole) == 1 and len(node.blocks) == 1 \ + and row[marker.end():].startswith(whole) + following = lead(node.line + 1) + if (single and node.unit not in referred + and (not following.strip() or MARKER.match(following) or following.startswith("#"))): + deletable = Edit("delete", whole, None, node.line, "") + elif isinstance(node, Paragraph) and node.top_level and node.label: + text, row = plain(node.inlines) or leading(node.inlines), lead(node.line) + if text and "paragraph" not in found and row.startswith(text[:12]) and unique(texts, text): + found["paragraph"] = Edit("paragraph", text, "ZZ paragraph", node.line, f"ZZ paragraph\n\n{row}") + chosen = list(found.values()) + if deletable is not None and all(deletable.line != edit.line for edit in chosen): + chosen.append(deletable) + return chosen + + +def made(source: str, edits: list[Edit]) -> str: + """*source* with the edits made, from the last line up so that the lines of the others stay where they are.""" + lines = source.splitlines() + for edit in sorted(edits, key=lambda e: e.line, reverse=True): + lines[edit.line - 1:edit.line] = edit.source.split("\n") if edit.source else [] + return "\n".join(lines) + "\n" + + +@dataclass +class Edited: + name: str + style: str + rules: frozenset[str] + edits: list[Edit] + after: RenderTree + written: str + #: The tree and the text of the document before the edits. + before: RenderTree + unedited: str + shown: str = "" + + +@functools.cache +def edited_corpus() -> dict[str, tuple[Edited, ...]]: + """The edited documents under each style, edited in LibreOffice and updated: one session for all of them.""" + require_libreoffice() + jobs: dict[str, tuple[bytes, list[tuple[str, ...]]]] = {} + cases: dict[str, list[Edited]] = {style: [] for style in STYLES} + keyed: dict[str, Edited] = {} + for name in EDITED: + source = documents()[name] + for style in STYLES: + made_ = rendered(source, style) + assert made_ is not None, name + data, unedited, tree, rules = made_ + edits = edits_for(source, data, tree) + if not edits: + continue + after = render(made(source, edits), format="text", **STYLES[style]) + case = Edited(name, style, rules, edits, after.tree, after.output, tree, unedited) + cases[style].append(case) + key = f"e{len(keyed)}" + keyed[key] = case + jobs[key] = (data, [(e.match, e.new) for e in edits]) + shown = libreoffice_edit(jobs) + for key, case in keyed.items(): + case.shown = shown[key] + return {style: tuple(group) for style, group in cases.items()} + + +@pytest.mark.libreoffice +@pytest.mark.parametrize("style", STYLES) +def test_the_document_follows_the_edits_a_lawyer_makes(style: str) -> None: + """After a heading, an item, a nested item and a numbered paragraph are inserted and a clause is deleted, and the fields + and indexes are updated, the document reads as the plain-text writer prints the source with the same edits: every number + and reference moved, the contents list the new heading. The text is equal, line for line, in every document: an edit + that reached a number fixed in the file (a restart by a fixed value) would show, and none does in this subset.""" + for case in edited_corpus()[style]: + written, seen, _ = normalise(case.written, case.shown, case.after) + assert written == seen, (f"{style}/{case.name}: not live after {[e.kind for e in case.edits]}:\n" + f"{difference(written, seen)}") + + +@pytest.mark.libreoffice +@pytest.mark.parametrize("style", STYLES) +def test_the_edits_move_the_numbers_after_them(style: str) -> None: + """The edits are not no-ops that a document with static numbers would pass: in every document, what the edits did not + type or delete reads differently afterwards (the headings after an inserted one are numbered one higher, and so on).""" + for case in edited_corpus()[style]: + before, _, _ = normalise(case.unedited, "", case.before) + after, _, _ = normalise(case.written, "", case.after) + typed = [line for line in after if "ZZ " in line] + deleted = {edit.match for edit in case.edits if edit.new is None} + rest = [line for line in after if "ZZ " not in line] + assert typed and rest != [line for line in before if not any(m in line for m in deleted)], (style, case.name) + + +@pytest.mark.libreoffice +def test_every_kind_of_edit_is_made_in_the_corpus() -> None: + """The subset is not vacuous: a heading, an item, a nested item and a deleted clause are edited under every style, and a + numbered paragraph under the continental style (the others do not number paragraphs).""" + for style, group in edited_corpus().items(): + kinds = {edit.kind for case in group for edit in case.edits} + assert {"heading", "item", "nested item", "delete"} <= kinds, (style, kinds) + assert ("paragraph" in kinds) == (style == "continental"), (style, kinds) + + +# -- The schemas and determinism ----------------------------------------------------------------------------------- + + +@pytest.mark.ooxml_schema +@pytest.mark.parametrize("style", STYLES) +def test_every_part_of_every_corpus_document_validates(style: str) -> None: + require_schemas() + for name, source in documents().items(): + made_ = rendered(source, style) + if made_ is not None: + assert validation_errors(made_[0]) == [], f"{style}/{name}" + + +@pytest.mark.parametrize("style", STYLES) +def test_rendering_a_corpus_document_twice_gives_the_same_bytes(style: str) -> None: + """Needs no tool but the corpus (marker ``acceptance`` alone: ``pytest -m acceptance`` runs it).""" + for name, source in documents().items(): + first, second = rendered(source, style), rendered(source, style) + assert (first is None) == (second is None), f"{style}/{name}" + if first is not None and second is not None: + assert first[0] == second[0], f"{style}/{name}: two renders differ" diff --git a/tests/test_docx_body.py b/tests/test_docx_body.py new file mode 100644 index 0000000..a32fff4 --- /dev/null +++ b/tests/test_docx_body.py @@ -0,0 +1,1125 @@ +"""The body of the DOCX, written as a hand-made :class:`DocxPlan` says: numbering definitions, the +numbering of each paragraph (``w:numPr``, alternatives, indents), bookmarks and ``REF`` fields with +their cached results, hyperlinks, every inline and every block, escaping, validity against the +schemas (marker ``ooxml_schema``) and what LibreOffice shows once fields are updated (marker +``libreoffice``). The plans are written by hand, so these tests need no planner; the writer adds no +rule of its own, and what a plan does not say it does not write.""" +from __future__ import annotations + +import xml.etree.ElementTree as ET +from collections.abc import Sequence +from typing import Any + +import pytest +from docx_support import W_NS, libreoffice_export, parts, require_libreoffice, validation_errors + +from legaldown_render import render +from legaldown_render.errors import InternalError +from legaldown_render.resolve.docx_plan import ( + AbstractNum, + AltNumber, + DocxPlan, + FieldSeg, + LevelDef, + Literal_, + Num, + RefPlan, + UnitPlan, +) +from legaldown_render.style import load_style +from legaldown_render.tree import ( + Blank, + Block, + Choice, + Code, + CodeBlock, + CrossRef, + DefinedTerm, + DraftingNote, + Emphasis, + FailureMarker, + HardBreak, + Image, + Inline, + Link, + List, + ListItem, + Paragraph, + Quote, + RenderTree, + Rule, + Section, + SoftBreak, + Strong, + Table, + TermRef, + Text, + Value, +) +from legaldown_render.writers import DocxWriter + +W = f"{{{W_NS}}}" + + +# -- Building trees and plans by hand -------------------------------------------------------------------------- + + +def tree(*blocks: Block, sections: Sequence[Section] = (), **settings: Any) -> RenderTree: + return RenderTree(title=(Text("T"),), subtitle=(), language="en", preamble=tuple(blocks), sections=tuple(sections), + **settings) + + +def para(*inlines: Inline | str, **fields: Any) -> Paragraph: + return Paragraph(tuple(Text(i) if isinstance(i, str) else i for i in inlines), **fields) + + +def item(*blocks: Block | str, unit: str | None = None, **fields: Any) -> ListItem: + return ListItem(tuple(para(b) if isinstance(b, str) else b for b in blocks), unit=unit, **fields) + + +def bullets(*items: ListItem, depth: int = 0) -> List: + return List(False, tuple(items), enumerated=True, family="enumerated", depth=depth) + + +def section(title: str, *blocks: Block, unit: str | None = None, level: int = 1, **fields: Any) -> Section: + return Section(level=level, title=(Text(title),), identifier="", condition="", blocks=tuple(blocks), unit=unit, + **fields) + + +def unit(unit_id: str, kind: str = "paragraph", *, num_id: int | None = None, ilvl: int | None = None, + explicit: bool = False, ind: tuple[int, int] | None = None, alt: AltNumber | None = None, + bookmarks: tuple[str, ...] = (), toc: str | None = None, collapsed: bool = False, + literal_refs: tuple[str, ...] = ()) -> UnitPlan: + return UnitPlan(unit=unit_id, kind=kind, num_id=num_id, ilvl=ilvl, explicit=explicit, ind=ind, alt=alt, # type: ignore[arg-type] + label="", word_ref="", count=None, cause=(), bookmarks=bookmarks, toc_bookmark=toc, + collapsed=collapsed, literal_refs=literal_refs) + + +def level(ilvl: int, fmt: str = "decimal", text: str | None = None, **fields: Any) -> LevelDef: + return LevelDef(ilvl, fmt, text if text is not None else f"%{ilvl + 1}.", **fields) + + +def abstract(identifier: int, kind: str = "enumerated", levels: Sequence[LevelDef] = ()) -> AbstractNum: + full = [*levels, *(level(i) for i in range(len(levels), 9))] + return AbstractNum(identifier, kind, f"{0x4C440000 + identifier:08X}", "hybridMultilevel", tuple(full)) # type: ignore[arg-type] + + +def field(bookmark: str, cached: str = "1", *, number: bool = True, suppress: bool = False, + hyperlink: bool = True) -> FieldSeg: + return FieldSeg(bookmark, number, suppress, hyperlink, cached, cached) + + +def plan(*units: UnitPlan, abstracts: Sequence[AbstractNum] = (), nums: Sequence[Num] = (), + heading: int | None = None, anchors: dict[str, str] | None = None, + refs: dict[str, tuple[Any, ...]] | None = None) -> DocxPlan: + return DocxPlan(abstract_nums=tuple(abstracts), nums=tuple(nums), heading_num_id=heading, + units={u.unit: u for u in units}, anchors=anchors or {}, + refs={key: RefPlan(tuple(segments), True) for key, segments in (refs or {}).items()}) + + +def build(document: RenderTree, the_plan: DocxPlan | None = None, **settings: Any) -> dict[str, bytes]: + data = DocxWriter(load_style("default", overrides=settings.get("overrides", {})), generator="test").write( + document, the_plan) + return parts(data) + + +def body(document: RenderTree, the_plan: DocxPlan | None = None, **settings: Any) -> list[ET.Element]: + """The children of ``w:body`` but the closing ``w:sectPr``, after the title paragraph.""" + root = ET.fromstring(build(document, the_plan, **settings)["word/document.xml"]) + children = list(root.find(f"{W}body")) # type: ignore[arg-type] + assert children[-1].tag == f"{W}sectPr" + assert children[0].find(f"{W}pPr/{W}pStyle").get(f"{W}val") == "Title" # type: ignore[union-attr] + return children[1:-1] + + +# -- Reading paragraphs ------------------------------------------------------------------------------------------ + + +def local(element: ET.Element) -> str: + return element.tag.removeprefix(W) + + +def pstyle(paragraph: ET.Element) -> str | None: + found = paragraph.find(f"{W}pPr/{W}pStyle") + return None if found is None else found.get(f"{W}val") + + +def numbering(paragraph: ET.Element) -> tuple[int | None, int] | None: + """``(ilvl, numId)`` of the paragraph's own ``w:numPr``.""" + found = paragraph.find(f"{W}pPr/{W}numPr") + if found is None: + return None + ilvl = found.find(f"{W}ilvl") + return (None if ilvl is None else int(ilvl.get(f"{W}val")), int(found.find(f"{W}numId").get(f"{W}val"))) # type: ignore[arg-type, union-attr] + + +def indent(paragraph: ET.Element) -> dict[str, int] | None: + found = paragraph.find(f"{W}pPr/{W}ind") + return None if found is None else {key.removeprefix(W): int(value) for key, value in found.attrib.items()} + + +def flow(paragraph: ET.Element) -> str: + """The paragraph's content in one line: text, ``→`` for a tab and ``↵`` for a break, a run's character style + and emphasis after it (``text@Style+b+i``), a field as ``{instruction|result}``, a bookmark as ``…`` + and a hyperlink as ``(link target: …)``.""" + names: dict[str, str] = {} + out: list[str] = [] + + def run(node: ET.Element) -> None: + pieces = [] + for child in node: + match local(child): + case "t": + pieces.append(child.text or "") + case "tab": + pieces.append("→") + case "br": + pieces.append("↵") + case "fldChar": + out.append({"begin": "{", "separate": "|", "end": "}"}[child.get(f"{W}fldCharType")]) # type: ignore[index] + case "instrText": + out.append((child.text or "").strip()) + if pieces: + suffix = "" + style = node.find(f"{W}rPr/{W}rStyle") + if style is not None: + suffix += "@" + style.get(f"{W}val") # type: ignore[operator] + for flag in ("b", "i"): + if node.find(f"{W}rPr/{W}{flag}") is not None: + suffix += f"+{flag}" + out.append("".join(pieces) + suffix) + + def walk(node: ET.Element) -> None: + for child in node: + match local(child): + case "r": + run(child) + case "bookmarkStart": + names[child.get(f"{W}id")] = child.get(f"{W}name") # type: ignore[assignment, index] + out.append(f"<{child.get(f'{W}name')}>") + case "bookmarkEnd": + out.append(f"") # type: ignore[index] + case "hyperlink": + target = child.get(f"{W}anchor") or child.get("{http://schemas.openxmlformats.org/officeDocument/2006/relationships}id") + out.append(f"(link {target}: ") + walk(child) + out.append(")") + + walk(paragraph) + return "".join(out) + + +# -- numbering.xml ----------------------------------------------------------------------------------------------- + + +def test_the_numbering_part_holds_each_abstract_numbering_and_each_instance_in_the_plans_order() -> None: + headings = abstract(0, "headings", [ + level(0, text="%1.", p_style="Heading1", left=567, hanging=567), + level(1, text="%1.%2", p_style="Heading2", left=680, hanging=680, suff="space"), + level(2, "lowerLetter", "(%3)", start=3)]) + bullet = abstract(1, "bullet", [LevelDef(0, "bullet", "•", left=567, hanging=360)]) + document = tree(para("x")) + the_plan = plan(abstracts=[headings, bullet], nums=[Num(1, 0), Num(2, 1, ((0, 1), (1, 1))), Num(3, 1, ((0, 4),))], + heading=1) + root = ET.fromstring(build(document, the_plan)["word/numbering.xml"]) + assert [local(child) for child in root] == ["abstractNum", "abstractNum", "num", "num", "num"] + first = root[0] + assert first.get(f"{W}abstractNumId") == "0" + assert [local(child) for child in first][:3] == ["nsid", "multiLevelType", "lvl"] + assert first.find(f"{W}nsid").get(f"{W}val") == "4C440000" # type: ignore[union-attr] + assert first.find(f"{W}multiLevelType").get(f"{W}val") == "hybridMultilevel" # type: ignore[union-attr] + assert len(first.findall(f"{W}lvl")) == 9 + lvl0, lvl1, lvl2 = first.findall(f"{W}lvl")[:3] + assert [local(child) for child in lvl0] == ["start", "numFmt", "pStyle", "lvlText", "lvlJc", "pPr"] + assert lvl0.get(f"{W}ilvl") == "0" + assert lvl0.find(f"{W}pStyle").get(f"{W}val") == "Heading1" # type: ignore[union-attr] + assert lvl0.find(f"{W}lvlText").get(f"{W}val") == "%1." # type: ignore[union-attr] + assert dict(lvl0.find(f"{W}pPr/{W}ind").attrib) == {f"{W}left": "567", f"{W}hanging": "567"} # type: ignore[union-attr] + assert [local(child) for child in lvl1] == ["start", "numFmt", "pStyle", "suff", "lvlText", "lvlJc", "pPr"] + assert lvl1.find(f"{W}suff").get(f"{W}val") == "space" # type: ignore[union-attr] + assert lvl2.find(f"{W}numFmt").get(f"{W}val") == "lowerLetter" + assert lvl2.find(f"{W}start").get(f"{W}val") == "3" # type: ignore[union-attr] + assert lvl2.find(f"{W}pPr") is None # a level with no indent writes none + assert root[1].find(f"{W}lvl/{W}numFmt").get(f"{W}val") == "bullet" # type: ignore[union-attr] + assert root[1].find(f"{W}lvl/{W}lvlText").get(f"{W}val") == "•" # type: ignore[union-attr] + nums = root.findall(f"{W}num") + assert [(n.get(f"{W}numId"), n.find(f"{W}abstractNumId").get(f"{W}val")) for n in nums] == [ # type: ignore[union-attr] + ("1", "0"), ("2", "1"), ("3", "1")] + assert [local(child) for child in nums[1]] == ["abstractNumId", "lvlOverride", "lvlOverride"] + assert [(o.get(f"{W}ilvl"), o.find(f"{W}startOverride").get(f"{W}val")) for o in nums[1].findall(f"{W}lvlOverride") # type: ignore[union-attr] + ] == [("0", "1"), ("1", "1")] + assert nums[0].find(f"{W}lvlOverride") is None + + +def test_the_headings_styles_are_linked_to_the_headings_num() -> None: + document = tree(sections=[section("A", unit="s1")]) + styles = ET.fromstring(build(document, plan(abstracts=[abstract(0, "headings")], nums=[Num(1, 0)], heading=1) + )["word/styles.xml"]) + by_id = {s.get(f"{W}styleId"): s for s in styles.findall(f"{W}style")} + for number in range(1, 6): + numpr = by_id[f"Heading{number}"].find(f"{W}pPr/{W}numPr") + assert (numpr.find(f"{W}ilvl").get(f"{W}val"), numpr.find(f"{W}numId").get(f"{W}val")) == (str(number - 1), "1") # type: ignore[union-attr] + plain = ET.fromstring(build(document, plan())["word/styles.xml"]) + assert plain.find(f".//{W}style[@{W}styleId='Heading1']/{W}pPr/{W}numPr") is None + + +def test_an_empty_plan_writes_an_empty_numbering_part() -> None: + assert list(ET.fromstring(build(tree(para("x")), plan())["word/numbering.xml"])) == [] + + +# -- The numbering of a paragraph -------------------------------------------------------------------------------- + + +def test_a_unit_writes_numpr_only_when_the_plan_says_it_is_explicit() -> None: + document = tree(bullets(item("a", unit="u1"), item("b", unit="u2")), + sections=[section("Live", unit="s1", level=2), section("Own", unit="s2")]) + the_plan = plan(unit("s1", "section", num_id=1, ilvl=1), unit("s2", "section", num_id=3, ilvl=0, explicit=True), + unit("u1", "item", num_id=2, ilvl=0, explicit=True), unit("u2", "item", num_id=2, ilvl=0), + abstracts=[abstract(0, "headings"), abstract(1)], + nums=[Num(1, 0), Num(2, 1), Num(3, 0, ((0, 5),))], heading=1) + children = body(document, the_plan) + items = [c for c in children if pstyle(c) is None] + heading_live, heading_own = (c for c in children if pstyle(c) and pstyle(c).startswith("Heading")) # type: ignore[union-attr] + assert (pstyle(heading_live), numbering(heading_live)) == ("Heading2", None) + assert (pstyle(heading_own), numbering(heading_own)) == ("Heading1", (0, 3)) + assert [numbering(i) for i in items] == [(0, 2), None] + + +def test_numpr_is_in_schema_order_among_the_other_properties() -> None: + document = tree(bullets(item("a", unit="u1"))) + the_plan = plan(unit("u1", "item", num_id=1, ilvl=2, explicit=True, ind=(1134, 567)), abstracts=[abstract(0)], + nums=[Num(1, 0)]) + (paragraph,) = body(document, the_plan) + assert [local(child) for child in paragraph.find(f"{W}pPr")] == ["numPr", "ind"] # type: ignore[union-attr] + assert [local(child) for child in paragraph.find(f"{W}pPr/{W}numPr")] == ["ilvl", "numId"] # type: ignore[union-attr] + assert indent(paragraph) == {"left": 1134, "hanging": 567} + + +def test_a_unit_without_a_level_the_plan_names_writes_just_the_num() -> None: + (paragraph,) = body(tree(bullets(item("a", unit="u1"))), plan(unit("u1", "item", num_id=1, explicit=True), + abstracts=[abstract(0)], nums=[Num(1, 0)])) + assert numbering(paragraph) == (None, 1) + + +def test_an_explicit_unit_with_no_num_is_a_bug_in_the_plan() -> None: + with pytest.raises(InternalError, match="plan"): + body(tree(bullets(item("a", unit="u1"))), plan(unit("u1", "item", explicit=True))) + + +def test_an_indent_is_written_where_the_plan_sets_one_and_a_hanging_of_zero_is_left_out() -> None: + document = tree(bullets(item("a", unit="u1"), item("b", unit="u2"), item("c", unit="u3"))) + the_plan = plan(unit("u1", "item", ind=(900, 0)), unit("u2", "item", ind=(900, 300)), unit("u3", "item")) + first, second, third = body(document, the_plan) + assert indent(first) == {"left": 900} + assert indent(second) == {"left": 900, "hanging": 300} + assert indent(third) is None + + +def test_the_level_indent_is_not_repeated_on_a_paragraph_that_lives_on_its_level() -> None: + document = tree(bullets(item("a", unit="u1"))) + the_plan = plan(unit("u1", "item", num_id=1, ilvl=1, explicit=True), + abstracts=[abstract(0, levels=[level(0), level(1, left=1134, hanging=567)])], nums=[Num(1, 0)]) + (paragraph,) = body(document, the_plan) + assert indent(paragraph) is None + + +def test_a_later_alternative_has_no_numbering_its_number_as_fields_and_a_tab() -> None: + alt = AltNumber((field("ld_first", "2"), Literal_(".")), "tab") + document = tree(sections=[section("Courts", unit="s1"), section("Arbitration", unit="s2")]) + the_plan = plan(unit("s1", "section", bookmarks=("ld_first",)), + unit("s2", "section", num_id=0, ilvl=0, explicit=True, ind=(567, 567), alt=alt, + bookmarks=("ld_second",))) + first, second = body(document, the_plan) + assert numbering(first) is None + assert numbering(second) == (0, 0) + assert indent(second) == {"left": 567, "hanging": 567} + assert flow(second) == "{REF ld_first \\n \\h|2}.→Arbitration" + + +def test_an_alternative_whose_plan_has_no_num_still_gets_num_zero() -> None: + alt = AltNumber((field("a", "1"),), "space") + document = tree(bullets(item("x", unit="u1")), para("a", unit="u2")) + the_plan = plan(unit("u1", bookmarks=("a",)), unit("u2", explicit=True, alt=alt)) + first, second = body(document, the_plan) + assert numbering(second) == (None, 0) + assert flow(second) == "{REF a \\n \\h|1} a" + + +# -- Bookmarks ---------------------------------------------------------------------------------------------- + + +def test_bookmarks_wrap_the_content_nested_with_the_first_outermost_and_the_toc_bookmark_innermost() -> None: + document = tree(sections=[section("Heading", unit="s1")]) + (heading,) = body(document, plan(unit("s1", "section", bookmarks=("ld_h", "_Ld1"), toc="_Toc100000001"))) + assert flow(heading) == "<_Ld1><_Toc100000001>Heading" + + +def test_a_collapsed_bookmark_is_a_point_at_the_start_of_the_paragraph_and_the_toc_bookmark_still_wraps() -> None: + document = tree(sections=[section("Heading", unit="s1")]) + (heading,) = body(document, plan(unit("s1", "section", bookmarks=("ld_h",), toc="_Toc1", collapsed=True))) + assert flow(heading) == "<_Toc1>Heading" + + +def test_a_cross_reference_in_a_title_that_the_plan_makes_literal_is_its_text_and_elsewhere_a_field() -> None: + reference = CrossRef("See B", "t", "s2") + other = CrossRef("C", "t", "s3") + document = tree(sections=[Section(1, (Text("A "), reference, Text(" and "), other), "", "", (), unit="s1"), + section("B", unit="s2"), section("C", unit="s3")]) + refs = {"s2": (field("b", "B", number=False),), "s3": (field("c", "C"),)} + units_ = (unit("s2", "section", bookmarks=("b",)), unit("s3", "section", bookmarks=("c",))) + literal = plan(unit("s1", "section", bookmarks=("a",), literal_refs=("s2",)), *units_, refs=refs) + assert flow(body(document, literal)[0]) == "A See B and {REF c \\n \\h|C}" + fields = plan(unit("s1", "section", bookmarks=("a",)), *units_, refs=refs) + assert flow(body(document, fields)[0]) == "A {REF b \\h|B} and {REF c \\n \\h|C}" + + +def test_bookmark_ids_are_unique_and_pair_each_start_with_its_end() -> None: + document = tree(bullets(item("a", unit="u1"), item("b", unit="u2")), sections=[section("H", unit="s1")]) + the_plan = plan(unit("s1", "section", bookmarks=("a1", "a2")), unit("u1", bookmarks=("b1",)), + unit("u2", bookmarks=("b2",), toc="_Toc1")) + root = ET.fromstring(build(document, the_plan)["word/document.xml"]) + starts = [(s.get(f"{W}id"), s.get(f"{W}name")) for s in root.iter(f"{W}bookmarkStart")] + ends = [e.get(f"{W}id") for e in root.iter(f"{W}bookmarkEnd")] + assert len({i for i, _ in starts}) == len(starts) == 5 + assert sorted(i for i, _ in starts) == sorted(ends) + assert [name for _, name in starts] == ["b1", "b2", "_Toc1", "a1", "a2"] + + +def test_condition_labels_and_an_alternatives_number_stay_outside_the_bookmarks() -> None: + alt = AltNumber((field("x", "1"),), "tab") + document = tree(para("Text", unit="u1", condition_label="if A"), bullets(item("held", unit="u2")), + sections=[section("Title", unit="s1", condition_label="if B")]) + the_plan = plan(unit("u1", bookmarks=("p",), alt=alt, explicit=True), unit("s1", "section", bookmarks=("h",)), + unit("u2", bookmarks=("x",))) + paragraph, _, heading = body(document, the_plan) + assert flow(paragraph) == "{REF x \\n \\h|1}→if A@ConditionLabel

Text

" + assert flow(heading) == "Title if B@ConditionLabel" + + +def test_an_item_that_opens_with_a_list_is_an_empty_numbered_paragraph_with_the_bookmarks_then_the_list() -> None: + nested = bullets(item("inner", unit="u2"), depth=1) + document = tree(bullets(item(nested, unit="u1", condition_label="if C"), item("plain", unit="u3"))) + the_plan = plan(unit("u1", "item", num_id=1, ilvl=0, explicit=True, bookmarks=("ld_one",)), + unit("u2", "item", num_id=1, ilvl=1, explicit=True, bookmarks=("ld_inner",)), + unit("u3", "item", num_id=1, ilvl=0, explicit=True), + abstracts=[abstract(0, levels=[level(0, left=567, hanging=567), level(1, left=1134, hanging=567)])], + nums=[Num(1, 0)]) + first, inner, last = body(document, the_plan) + assert (numbering(first), flow(first)) == ((0, 1), "if C@ConditionLabel") + assert (numbering(inner), flow(inner)) == ((1, 1), "inner") + assert (numbering(last), flow(last)) == ((0, 1), "plain") + + +@pytest.mark.parametrize("blocks", [(), (Quote((para("q"),)),), (CodeBlock("c"),)]) +def test_an_item_without_a_first_paragraph_is_an_empty_numbered_paragraph(blocks: tuple[Block, ...]) -> None: + document = tree(bullets(ListItem(blocks, unit="u1"))) + the_plan = plan(unit("u1", "item", num_id=1, ilvl=0, explicit=True, bookmarks=("ld_x",)), abstracts=[abstract(0)], + nums=[Num(1, 0)]) + first, *rest = body(document, the_plan) + assert (numbering(first), flow(first)) == ((0, 1), "") + assert len(rest) == len(blocks) + + +def test_the_blocks_after_an_items_first_paragraph_continue_at_its_indent_unnumbered() -> None: + document = tree(bullets(item("first", para("second"), CodeBlock("code"), bullets(item("n", unit="u2"), depth=1), + unit="u1"))) + the_plan = plan(unit("u1", "item", num_id=1, ilvl=0, explicit=True), + unit("u2", "item", num_id=1, ilvl=1, explicit=True), + abstracts=[abstract(0, levels=[level(0, left=567, hanging=567), level(1, left=1134, hanging=567)])], + nums=[Num(1, 0)]) + first, second, code, nested = body(document, the_plan) + assert numbering(second) is None and indent(second) == {"left": 567} and flow(second) == "second" + assert pstyle(code) == "Code" and indent(code) == {"left": 567 + 113} + assert numbering(nested) == (1, 1) + + +def test_continuation_follows_an_indent_the_plan_sets_over_the_levels() -> None: + document = tree(bullets(item("first", "more", unit="u1"))) + first, more = body(document, plan(unit("u1", "item", ind=(2000, 400)))) + assert indent(first) == {"left": 2000, "hanging": 400} + assert indent(more) == {"left": 2000} + + +def test_a_unit_the_plan_does_not_know_is_a_plain_paragraph() -> None: + document = tree(para("a", unit="u9"), bullets(item("b", unit="u8")), sections=[section("H", unit="s9")]) + paragraph, listed, heading = body(document, plan()) + assert [pstyle(paragraph), numbering(paragraph), flow(paragraph)] == [None, None, "a"] + assert [pstyle(listed), numbering(listed), flow(listed)] == [None, None, "b"] + assert [pstyle(heading), numbering(heading)] == ["Heading1", None] + + +# -- Cross-references and fields --------------------------------------------------------------------------------- + + +def test_a_cross_reference_is_its_literal_segments_and_ref_fields_with_the_cached_results() -> None: + reference = CrossRef("Clause 1.2(b)", "x", "u9", "s2") + document = tree(para("See ", reference, "."), sections=[section("H", unit="s1")]) + refs = {"u9": (Literal_("Clause "), field("ld_s", "1.2"), Literal_("("), field("_Ld1", "b", suppress=True), + Literal_(")"))} + the_plan = plan(unit("s1", "section", bookmarks=("ld_s", "_Ld1")), refs=refs) + paragraph, _ = body(document, the_plan) + assert flow(paragraph) == "See Clause {REF ld_s \\n \\h|1.2}({REF _Ld1 \\n \\t \\h|b})." + + +def test_the_switches_follow_the_segment_in_a_fixed_order() -> None: + cases = [(field("a"), "REF a \\n \\h"), (field("a", hyperlink=False), "REF a \\n"), + (field("a", number=False), "REF a \\h"), (field("a", number=False, hyperlink=False), "REF a"), + (field("a", suppress=True, hyperlink=False), "REF a \\n \\t"), + (field("a", suppress=True), "REF a \\n \\t \\h")] + for segment, instruction in cases: + document = tree(para(CrossRef("x", "t", "u9")), bullets(item("y", unit="u1"))) + paragraph = body(document, plan(unit("u1", bookmarks=("a",)), refs={"u9": (segment,)}))[0] + assert flow(paragraph) == "{" + instruction + "|1}" + + +def test_a_field_is_five_runs_the_cached_result_between_separate_and_end() -> None: + document = tree(para(CrossRef("x", "t", "u9")), bullets(item("y", unit="u1"))) + paragraph = body(document, plan(unit("u1", bookmarks=("a",)), refs={"u9": (field("a", "7"),)}))[0] + runs = paragraph.findall(f"{W}r") + kinds = [(r.find(f"{W}fldChar").get(f"{W}fldCharType") if r.find(f"{W}fldChar") is not None + else local(r[0])) for r in runs] + assert kinds == ["begin", "instrText", "separate", "t", "end"] + + +def test_a_cached_result_that_differs_from_the_renderers_text_is_written_as_the_plan_gives_it() -> None: + reference = CrossRef("1.1(bb)", "x", "u9") + document = tree(para(reference), bullets(item("y", unit="u1"))) + refs = {"u9": (field("a", "1.1(aa)"),)} + plan_ = DocxPlan((), (), None, {"u1": unit("u1", bookmarks=("a",))}, {}, {"u9": RefPlan(tuple(refs["u9"]), False)}) + assert flow(body(document, plan_)[0]) == "{REF a \\n \\h|1.1(aa)}" + + +def test_a_cross_reference_without_fields_shows_its_text() -> None: + document = tree(para("see ", CrossRef("1.2", "x", "u9"), " and ", CrossRef("3", "", None, None), " and ", + CrossRef("4", "x", "u7"))) + (paragraph,) = body(document, plan(refs={})) + assert flow(paragraph) == "see 1.2 and 3 and 4" + + +def test_a_field_to_a_bookmark_the_document_does_not_have_is_a_bug_in_the_plan() -> None: + document = tree(para(CrossRef("x", "t", "u9"))) + with pytest.raises(InternalError, match="'nowhere'"): + body(document, plan(refs={"u9": (field("nowhere"),)})) + + +def test_a_reference_inside_a_link_stays_a_field_and_text_stays_one_link() -> None: + link = Link("https://example.com/", (Text("see "), CrossRef("1", "x", "u9"), TermRef("t", "def:t", "bold"))) + document = tree(para(link), bullets(item("y", unit="u1"))) + the_plan = plan(unit("u1", bookmarks=("a",)), anchors={"def:t": "lddef_t"}, refs={"u9": (field("a", "1"),)}) + (paragraph,) = body(document, the_plan)[:1] + assert flow(paragraph) == "(link rId5: see @Hyperlink{REF a \\n \\h|1@Hyperlink}t@TermReference)" + + +# -- Terms, definitions, attachments, markers ----------------------------------------------------------------- + + +def defined(anchor: str | None = "def:t", *, defining: bool = True) -> DefinedTerm: + return DefinedTerm((Text("Term"),), anchor, "bold", defining) + + +def test_a_defined_term_is_a_bookmark_around_text_in_the_defined_term_style() -> None: + document = tree(para("A ", defined(), " is")) + (paragraph,) = body(document, plan(anchors={"def:t": "lddef_t"})) + assert flow(paragraph) == "A Term@DefinedTerm is" + + +@pytest.mark.parametrize(("term", "anchors"), [(defined(None), {}), (defined(defining=False), {"def:t": "lddef_t"}), + (defined(), {})]) +def test_a_term_that_defines_nothing_here_or_has_no_bookmark_is_styled_text(term: DefinedTerm, + anchors: dict[str, str]) -> None: + (paragraph,) = body(tree(para(term)), plan(anchors=anchors)) + assert flow(paragraph) == "Term@DefinedTerm" + + +def test_a_term_reference_is_an_internal_hyperlink_to_the_definition_with_the_term_as_text() -> None: + document = tree(para("the ", TermRef("terms", "def:t", "bold"), " apply"), para(defined())) + first, _ = body(document, plan(anchors={"def:t": "lddef_t"})) + assert flow(first) == "the (link lddef_t: terms@TermReference) apply" + link = first.find(f"{W}hyperlink") + assert link.get(f"{W}anchor") == "lddef_t" and link.get(f"{W}history") == "1" # type: ignore[union-attr] + + +@pytest.mark.parametrize("term", [TermRef("a", None, "bold"), TermRef("a", "def:other", "bold"), + TermRef("a", "def:t", "bold")]) +def test_a_term_reference_with_no_definition_in_the_document_is_styled_text(term: TermRef) -> None: + (paragraph,) = body(tree(para(term)), plan(anchors={"def:t": "lddef_t"})) + assert flow(paragraph) == "a@TermReference" + + +def attachment_tree(value: Value, titles: Sequence[str] = ("Schedule A",)) -> RenderTree: + from legaldown_render.tree import AttachmentPart + + attachments = tuple(AttachmentPart(f"a{i}", (Text(title),), "f.lgd", f"att-{i}") for i, title in enumerate(titles)) + return tree(para("see ", value), attachments=attachments) + + +def test_an_attachment_value_is_a_field_when_it_is_the_title_else_a_link_with_its_text_and_text_when_unplaced() -> None: + def paragraph(value: Value, *, placed: bool = True) -> str: + # The attachment's title is written with its bookmark; a plan that names no bookmark for it leaves the text. + the_plan = plan(anchors={"att-0": "ldatt_att_0"} if placed else {}) + document = tree(para("see ", value), attachments=attachment_tree(value).attachments) + return flow(body(document, the_plan)[0]) + + assert paragraph(Value("attach", "Schedule A", "#att-0")) == "see {REF ldatt_att_0 \\h|Schedule A}" + assert paragraph(Value("attach", "the schedule", "#att-0")) == "see (link ldatt_att_0: the schedule@Hyperlink)" + assert paragraph(Value("attach", "Schedule A", "#att-0"), placed=False) == "see Schedule A" + assert paragraph(Value("attach", "x", "#att-9")) == "see x" + assert paragraph(Value("attach", "x", None)) == "see x" + + +def test_a_value_that_links_to_a_file_is_an_external_hyperlink_when_the_document_may_link_there() -> None: + document = tree(para(Value("attach", "price list", "https://example.com/p.pdf"), " ", + Value("attach", "other", "files/p.pdf"), " ", Value("date", "1 May", None))) + (paragraph,) = body(document, plan()) + assert flow(paragraph) == "(link rId5: price list@Hyperlink) other 1 May" + + +def test_failure_blank_and_choice_are_runs_in_their_styles() -> None: + document = tree(para(FailureMarker("[BROKEN REF: x]"), " ", Blank("b", "____", "your name"), " ", Blank("c", "[TBD]"), + " ", Choice("q", ("one", "two"), " | "))) + (paragraph,) = body(document, plan()) + assert flow(paragraph) == "[BROKEN REF: x]@FailureMarker ____@Blank (your name)+i [TBD]@Blank [one | two]@Choice" + + +# -- Inlines ---------------------------------------------------------------------------------------------------- + + +def test_emphasis_strong_and_code_are_runs_and_nest() -> None: + document = tree(para("a ", Emphasis((Text("e"), Strong((Text("both"),)))), Strong((Text("s"),)), Code("c"), + Emphasis((Code("ic"),)), SoftBreak(), "z", HardBreak(), "y")) + (paragraph,) = body(document, plan()) + assert flow(paragraph) == "a e+i" "both+b+i" "s+b" "c@InlineCode" "ic@InlineCode+i" " z↵y" + + +def test_run_properties_are_in_schema_order() -> None: + document = tree(para(Strong((Emphasis((Code("x"),)),)))) + run = body(document, plan())[0].find(f"{W}r") + assert [local(child) for child in run.find(f"{W}rPr")] == ["rStyle", "b", "bCs", "i", "iCs"] # type: ignore[union-attr] + + +def test_an_external_link_is_a_hyperlink_by_relationship_and_a_refused_one_is_its_text() -> None: + link = Link("https://example.com/a b", (Text("site"),), (Text("tip"),)) + refused = Link("javascript:alert(1)", (Text("bad"),)) + document = tree(para(link, " ", refused, " ", Link("https://example.com/a b", (Text("again"),)))) + contents = build(document, plan()) + paragraph = [c for c in ET.fromstring(contents["word/document.xml"]).find(f"{W}body") if pstyle(c) is None][0] # type: ignore[union-attr] + assert flow(paragraph) == "(link rId5: site@Hyperlink) bad (link rId5: again@Hyperlink)" + rels = ET.fromstring(contents["word/_rels/document.xml.rels"]) + external = [r for r in rels if r.get("TargetMode") == "External"] + assert [(r.get("Id"), r.get("Target")) for r in external] == [("rId5", "https://example.com/a%20b")] + assert paragraph.find(f"{W}hyperlink").get(f"{W}tooltip") == "tip" # type: ignore[union-attr] + + +def test_a_link_with_no_text_shows_its_address_and_emphasis_inside_it_keeps_the_link_style() -> None: + document = tree(para(Link("mailto:a@example.com", ()), " ", Link("https://e.com/", (Emphasis((Text("em"),)),)))) + (paragraph,) = body(document, plan()) + assert flow(paragraph) == "(link rId5: mailto:a@example.com@Hyperlink) (link rId6: em@Hyperlink+i)" + + +def test_an_internal_link_to_an_anchor_of_the_document_is_a_hyperlink_to_its_bookmark() -> None: + document = tree(para(Link("#clause", (Text("here"),)), " ", Link("#missing", (Text("there"),))), + bullets(item("x", unit="u1"))) + first, _ = body(document, plan(unit("u1", bookmarks=("ld_clause",)), anchors={"clause": "ld_clause"})) + assert flow(first) == "(link ld_clause: here@Hyperlink) there" + + +def test_an_image_is_its_alt_text() -> None: + (paragraph,) = body(tree(para("a", Image("x.png", (Text("a chart"),)), Image("y.png", ()))), plan()) + assert flow(paragraph) == "aa chart" + + +def test_text_with_tabs_and_line_breaks_becomes_tab_and_break_elements() -> None: + (paragraph,) = body(tree(para("a\tb\nc\r\nd")), plan()) + assert flow(paragraph) == "a→b↵c↵d" + + +def test_a_space_at_either_end_of_a_run_is_kept() -> None: + (paragraph,) = body(tree(para(" a ")), plan()) + assert paragraph.find(f"{W}r/{W}t").get("{http://www.w3.org/XML/1998/namespace}space") == "preserve" # type: ignore[union-attr] + + +# -- Blocks ----------------------------------------------------------------------------------------------------- + + +def test_a_quote_is_paragraphs_in_the_quote_style_and_a_note_has_its_label_run() -> None: + document = tree(Quote((para("q1"), para("q2"))), DraftingNote((para("n"),), "Drafting note"), + DraftingNote((para("m"),), None)) + q1, q2, label, note, bare = body(document, plan()) + assert [(pstyle(p), flow(p)) for p in (q1, q2, label, note, bare)] == [ + ("Quote", "q1"), ("Quote", "q2"), ("DraftingNote", "Drafting note@DraftingNoteLabel"), + ("DraftingNote", "n"), ("DraftingNote", "m")] + assert all(indent(p) is None for p in (q1, q2, label, note, bare)) + + +def test_a_list_in_a_quote_adds_the_quotes_indent_to_its_own_and_its_continuation() -> None: + inner = bullets(item("a", "more", unit="u1")) + document = tree(Quote((para("lead"), inner)), DraftingNote((inner,), "N")) + the_plan = plan(unit("u1", "item", num_id=1, ilvl=0, explicit=True), abstracts=[abstract(0, levels=[ + level(0, left=567, hanging=567)])], nums=[Num(1, 0)]) + lead, a, more, label, a2, more2 = body(document, the_plan) + assert (pstyle(a), numbering(a), indent(a)) == ("Quote", (0, 1), {"left": 567 + 567, "hanging": 567}) + assert (pstyle(more), numbering(more), indent(more)) == ("Quote", None, {"left": 567 + 567}) + assert indent(a2) == {"left": 567 + 227, "hanging": 567} and indent(more2) == {"left": 567 + 227} + assert pstyle(a2) == pstyle(more2) == "DraftingNote" + + +def test_a_quote_inside_a_note_carries_both_indents() -> None: + document = tree(DraftingNote((Quote((para("q"),)), Rule()), None), Quote((DraftingNote((para("n"),), None),))) + q, rule, n = body(document, plan()) + assert (pstyle(q), indent(q)) == ("Quote", {"left": 227 + 567}) + assert (pstyle(rule), indent(rule)) == ("DraftingNote", None) + assert (pstyle(n), indent(n)) == ("DraftingNote", {"left": 567 + 227}) + + +def test_a_code_block_is_one_paragraph_per_line_in_the_code_style() -> None: + document = tree(CodeBlock("a\n b\n\n\tc\n"), CodeBlock("")) + *lines, empty = body(document, plan()) + assert [(pstyle(p), flow(p)) for p in lines] == [("Code", "a"), ("Code", " b"), ("Code", ""), ("Code", "→c")] + assert pstyle(empty) == "Code" + + +def test_a_rule_is_a_bottom_border() -> None: + (rule,) = body(tree(Rule()), plan()) + border = rule.find(f"{W}pPr/{W}pBdr/{W}bottom") + assert border is not None and border.get(f"{W}val") == "single" + assert flow(rule) == "" + + +def table(*rows: Sequence[str], header: Sequence[str] = ("H1", "H2"), align: tuple[str, ...] = ()) -> Table: + def cells(row: Sequence[str]) -> tuple[tuple[Inline, ...], ...]: + return tuple((Text(cell),) for cell in row) + + return Table(cells(header), tuple(cells(row) for row in rows), align) + + +def test_a_table_is_a_grid_with_a_repeated_header_row_and_aligned_columns() -> None: + document = tree(para("before"), table(("a", "b"), ("c", "d"), align=("right", "center")), para("after")) + before, tbl, after = body(document, plan()) + assert local(tbl) == "tbl" and flow(before) == "before" and flow(after) == "after" + assert [local(child) for child in tbl] == ["tblPr", "tblGrid", "tr", "tr", "tr"] + props = tbl.find(f"{W}tblPr") + assert props.find(f"{W}tblStyle").get(f"{W}val") == "TableGrid" # type: ignore[union-attr] + assert [local(child) for child in props] == ["tblStyle", "tblW", "tblCellMar", "tblLook"] # type: ignore[arg-type] + widths = [int(g.get(f"{W}w")) for g in tbl.findall(f"{W}tblGrid/{W}gridCol")] # type: ignore[arg-type] + assert len(widths) == 2 and abs(widths[0] - widths[1]) <= 1 + assert int(props.find(f"{W}tblW").get(f"{W}w")) == sum(widths) # type: ignore[union-attr, arg-type] + rows = tbl.findall(f"{W}tr") + assert rows[0].find(f"{W}trPr/{W}tblHeader") is not None + assert all(r.find(f"{W}trPr") is None for r in rows[1:]) + cells = [[(flow(c.find(f"{W}p")), c.find(f"{W}p/{W}pPr/{W}jc").get(f"{W}val")) for c in r.findall(f"{W}tc")] # type: ignore[arg-type, union-attr] + for r in rows] + assert cells == [[("H1", "right"), ("H2", "center")], [("a", "right"), ("b", "center")], + [("c", "right"), ("d", "center")]] + assert [int(c.find(f"{W}tcPr/{W}tcW").get(f"{W}w")) for c in rows[0].findall(f"{W}tc")] == widths # type: ignore[arg-type, union-attr] + + +def test_a_cell_holds_inlines_and_a_short_row_is_completed_with_empty_cells() -> None: + document = tree(Table(((Text("A"),), (Text("B"),), (Text("C"),)), + (((Strong((Text("x"),)),), (TermRef("t", None, "bold"),)),), ("", "left"))) + (tbl,) = [c for c in body(document, plan()) if local(c) == "tbl"] + row = tbl.findall(f"{W}tr")[1] + assert [flow(c.find(f"{W}p")) for c in row.findall(f"{W}tc")] == ["x+b", "t@TermReference", ""] + assert len(tbl.findall(f"{W}tblGrid/{W}gridCol")) == 3 + assert [c.find(f"{W}p/{W}pPr/{W}jc").get(f"{W}val") for c in row.findall(f"{W}tc")] == ["left"] * 3 # type: ignore[union-attr] + + +def test_two_tables_in_a_row_and_a_table_at_the_end_get_an_empty_paragraph_after_or_between() -> None: + first, between, second, last = body(tree(table(), table()), plan()) + assert [local(first), local(between), local(second), local(last)] == ["tbl", "p", "tbl", "p"] + assert flow(between) == flow(last) == "" + assert [local(c) for c in body(tree(para("x"), table()), plan())] == ["p", "tbl", "p"] + assert [local(c) for c in body(tree(table(), para("x")), plan())] == ["tbl", "p"] + + +def test_a_table_in_a_list_item_or_quote_is_indented_and_narrower() -> None: + document = tree(bullets(item("a", table(), unit="u1")), Quote((table(),))) + the_plan = plan(unit("u1", "item", ind=(1000, 0))) + _, in_item, in_quote = (c for c in body(document, the_plan) if local(c) in ("tbl", "p") and (local(c) == "tbl" or flow(c) == "a")) + for tbl, left in ((in_item, 1000), (in_quote, 567)): + props = tbl.find(f"{W}tblPr") + assert int(props.find(f"{W}tblInd").get(f"{W}w")) == left # type: ignore[union-attr, arg-type] + full = int(ET.fromstring(build(tree(table()), plan())["word/document.xml"]).find( # type: ignore[arg-type] + f".//{W}tblW").get(f"{W}w")) + assert int(props.find(f"{W}tblW").get(f"{W}w")) == full - left # type: ignore[union-attr, arg-type] + + +def test_a_table_with_no_columns_is_left_out() -> None: + assert body(tree(Table((), (), ()), para("x")), plan()) and len(body(tree(Table((), (), ())), plan())) == 0 + + +# -- Headings and condition labels ------------------------------------------------------------------------------ + + +def test_headings_use_the_heading_styles_and_are_clamped_to_five() -> None: + document = tree(sections=[section("a", level=n, unit=f"s{n}") for n in (0, 1, 3, 5, 9)]) + assert [pstyle(h) for h in body(document, plan())] == ["Heading1", "Heading1", "Heading3", "Heading5", "Heading5"] + + +def test_a_condition_label_leads_a_paragraph_and_an_item_and_follows_a_heading_title() -> None: + document = tree(para("p", condition_label="if A"), para("plain"), + bullets(item(para("first", condition_label="inner"), "second", unit="u1", + condition_label="if B")), sections=[section("H", condition_label="if C", unit="s1")]) + the_plan = plan(unit("u1", "item", ind=(567, 0))) + p, plain, item_paragraph, second, heading = body(document, the_plan) + assert flow(p) == "if A@ConditionLabel p" and flow(plain) == "plain" + assert flow(item_paragraph) == "if B@ConditionLabel inner@ConditionLabel first" + assert flow(heading) == "H if C@ConditionLabel" + + +# -- Bookmarks the document does not write, indents and widths, links with no text ---------------------------- + + +def test_a_bookmark_that_is_only_in_text_the_document_never_writes_is_not_a_target() -> None: + """A defined term inside alt text or a link's title is plan, not document: a reference to it has no target.""" + term = DefinedTerm((Text("Term"),), "def:t", "bold") + document = tree(para(Image("i.png", (Text("alt "), term)), TermRef("Term", "def:t", "bold"), + Link("https://e.com/", (Text("l"),), (Text("tip "), term)), TermRef("Term", "def:t", "bold"))) + (paragraph,) = body(document, plan(anchors={"def:t": "lddef_t"})) + assert flow(paragraph) == "alt TermTerm@TermReference(link rId5: l@Hyperlink)Term@TermReference" + assert not list(paragraph.iter(f"{W}bookmarkStart")) + with pytest.raises(InternalError, match="lddef_t"): + body(tree(para(Image("i.png", (term,)), CrossRef("x", "t", "u9"))), + plan(anchors={"def:t": "lddef_t"}, refs={"u9": (field("lddef_t"),)})) + + +def test_a_term_reference_before_its_definition_still_links_and_a_reference_after_it_too() -> None: + document = tree(para(TermRef("T", "def:t", "bold"), DefinedTerm((Text("Term"),), "def:t", "bold"), + CrossRef("x", "t", "u9"))) + (paragraph,) = body(document, plan(anchors={"def:t": "lddef_t"}, refs={"u9": (field("lddef_t", number=False),)})) + assert flow(paragraph) == "(link lddef_t: T@TermReference)Term@DefinedTerm{REF lddef_t \\h|1}" + + +def test_a_field_may_not_name_what_is_not_a_bookmark() -> None: + for name in ("bad name", "1x", "a" * 41, "a\\n"): + with pytest.raises(InternalError, match="valid"): + body(tree(para(CrossRef("x", "t", "u9"))), plan(refs={"u9": (field(name),)})) + + +def test_every_hyperlink_has_content_and_a_link_that_is_not_made_still_shows_what_html_shows() -> None: + nothing = Link("https://e.com/", (Image("p.png", ()),)) + document = tree(para(Link("docs/a.pdf", ()), " ", Link("#nope", ()), " ", nothing, " ", + Link("javascript:x", (Text("bad"),)), Link("javascript:x", ()))) + (paragraph,) = body(document, plan()) + assert flow(paragraph) == "docs/a.pdf #nope (link rId5: https://e.com/@Hyperlink) bad" + assert all(len(link) for link in paragraph.iter(f"{W}hyperlink")) + + +def test_a_tooltip_is_cut_to_255_characters() -> None: + (paragraph,) = body(tree(para(Link("https://e.com/", (Text("l"),), (Text("t" * 300),)))), plan()) + assert len(paragraph.find(f"{W}hyperlink").get(f"{W}tooltip")) == 255 # type: ignore[union-attr, arg-type] + + +def test_runs_in_a_link_that_take_another_style_keep_the_links_colour_and_underline() -> None: + link = Link("https://e.com/", (Code("c"), TermRef("t", None, "bold"), Blank("b", "__"), Text("x"))) + (paragraph,) = body(tree(para(link)), plan()) + runs = paragraph.find(f"{W}hyperlink").findall(f"{W}r") # type: ignore[union-attr] + styles = [r.find(f"{W}rPr/{W}rStyle").get(f"{W}val") for r in runs] # type: ignore[union-attr] + assert styles == ["InlineCode", "TermReference", "Blank", "Hyperlink"] + for run in runs[:3]: + assert run.find(f"{W}rPr/{W}u").get(f"{W}val") == "single" # type: ignore[union-attr] + assert run.find(f"{W}rPr/{W}color") is not None + assert runs[3].find(f"{W}rPr/{W}u") is None # the Hyperlink style has them + + +def test_an_alternative_is_never_numbered_by_a_heading_style() -> None: + """The plan sets numId 0 for an alternative; a heading that names an alternative but is not marked explicit still + gets it, or the heading style's own numbering would count it.""" + alt = AltNumber((field("a", "1"),), "tab") + document = tree(bullets(item("x", unit="u1")), sections=[section("H", unit="s1")]) + the_plan = plan(unit("u1", bookmarks=("a",)), unit("s1", "section", alt=alt, ind=(567, 567))) + _, heading = body(document, the_plan) + assert numbering(heading) == (None, 0) + + +def test_an_inline_or_block_the_writer_does_not_know_is_a_bug() -> None: + from legaldown_render.tree import DirectiveSource + + with pytest.raises(InternalError, match="DirectiveSource"): + body(tree(para(DirectiveSource(None))), plan()) # type: ignore[arg-type] + with pytest.raises(InternalError, match="int"): + body(tree(42), plan()) # type: ignore[arg-type] + + +def test_code_and_rules_in_a_quote_or_a_note_keep_the_bar_of_the_container() -> None: + document = tree(Quote((para("q"), CodeBlock("c"), Rule())), DraftingNote((CodeBlock("n"), Rule()), None), Rule()) + q, code, rule, note_code, note_rule, plain_rule = body(document, plan()) + assert code.find(f"{W}pPr/{W}pBdr/{W}left").get(f"{W}color") == "C8C8C8" # type: ignore[union-attr] + assert indent(code) == {"left": 567} + assert [local(c) for c in rule.find(f"{W}pPr/{W}pBdr")] == ["left", "bottom"] # type: ignore[arg-type, union-attr] + assert note_code.find(f"{W}pPr/{W}pBdr/{W}left").get(f"{W}color") == "D9A400" # type: ignore[union-attr] + assert [local(c) for c in note_rule.find(f"{W}pPr/{W}pBdr")] == ["left", "bottom"] # type: ignore[arg-type, union-attr] + assert [local(c) for c in plain_rule.find(f"{W}pPr/{W}pBdr")] == ["bottom"] # type: ignore[arg-type, union-attr] + + +def test_the_table_look_says_what_its_flags_say() -> None: + (tbl, _) = body(tree(table()), plan()) + look = tbl.find(f"{W}tblPr/{W}tblLook") + flags = {"firstRow": 0x20, "lastRow": 0x40, "firstColumn": 0x80, "noHBand": 0x200, "noVBand": 0x400} + assert int(look.get(f"{W}val"), 16) == sum(bit for name, bit in flags.items() if look.get(f"{W}{name}") == "1") # type: ignore[union-attr, arg-type] + + +TEXT_WIDTH = 9072 # the default style's page: A4 with 2.5 cm margins + + +def lefts(root: ET.Element) -> list[int]: + return [int(e.get(f"{W}left")) for e in root.iter(f"{W}ind") if e.get(f"{W}left")] # type: ignore[arg-type] + + +def test_a_wide_indent_in_the_plan_cannot_push_the_text_off_the_page_or_squeeze_a_table() -> None: + document = tree(bullets(item("a", table(header=("x", "y", "z")), unit="u1"), item("b", "c", unit="u2"))) + the_plan = plan(unit("u1", "item", ind=(9000, 567)), unit("u2", "item", ind=(99999, 99999))) + result = body(document, the_plan) + root = ET.Element("x") + root.extend(result) + assert all(left <= TEXT_WIDTH - 2268 for left in lefts(root)) + assert max(lefts(root)) == TEXT_WIDTH - 2268 + assert all(int(h.get(f"{W}hanging")) <= int(h.get(f"{W}left")) for h in root.iter(f"{W}ind") if h.get(f"{W}hanging")) # type: ignore[arg-type] + tbl = next(c for c in result if local(c) == "tbl") + assert tbl.find(f"{W}tblPr/{W}tblInd") is None # three columns do not fit beside it: the indent goes + widths = [int(g.get(f"{W}w")) for g in tbl.findall(f"{W}tblGrid/{W}gridCol")] # type: ignore[arg-type] + assert min(widths) >= 1134 or sum(widths) == TEXT_WIDTH + + +def test_a_table_beside_an_indent_keeps_at_least_a_column_width_per_column() -> None: + wide = Table(tuple((Text(c),) for c in "abc"), (), ()) + for left in (0, 1000, 5000, 8000): + document = tree(bullets(item("a", wide, unit="u1"))) + tbl = next(c for c in body(document, plan(unit("u1", "item", ind=(left, 0)))) if local(c) == "tbl") + widths = [int(g.get(f"{W}w")) for g in tbl.findall(f"{W}tblGrid/{W}gridCol")] # type: ignore[arg-type] + assert min(widths) >= 1134 and sum(widths) <= TEXT_WIDTH + + +def test_a_deeply_indented_list_stays_on_the_page() -> None: + inner: Block = bullets(item("deepest", unit="u70"), depth=70) + for depth in range(69, -1, -1): + inner = bullets(item(f"d{depth}", inner, unit=f"u{depth}"), depth=depth) + document = tree(inner) + indents = [unit(f"u{depth}", "item", ind=((depth + 1) * 567, 0)) for depth in range(71)] + data = DocxWriter(load_style("default")).write(document, plan(*indents)) + root = ET.fromstring(parts(data)["word/document.xml"]) + assert max(lefts(root)) <= min(31680, TEXT_WIDTH - 2268) + + +# -- Whole documents -------------------------------------------------------------------------------------------- + + +def test_the_title_and_the_subtitle_come_first_and_the_section_properties_last() -> None: + document = RenderTree(title=(Text("My Title"),), subtitle=(Text("Sub"),), language="en", preamble=(para("x"),), + sections=()) + root = ET.fromstring(build(document, plan())["word/document.xml"]) + children = list(root.find(f"{W}body")) # type: ignore[arg-type] + assert [(pstyle(c), flow(c)) for c in children[:-1]] == [("Title", "My Title"), ("Subtitle", "Sub"), (None, "x")] + assert local(children[-1]) == "sectPr" + + +def test_a_rendered_document_is_numbered_cross_referenced_and_bookmarked_as_the_planner_says() -> None: + result = render("---\ntitle: T\n---\n\n# A\n\n- a {#x}\n - b\n\nSee {{ref: x}}.\n", format="docx") + contents = parts(result.binary) # type: ignore[arg-type] + assert [child.tag.removeprefix(W) for child in ET.fromstring(contents["word/numbering.xml"])] == [ + "abstractNum", "abstractNum", "num", "num"] + root = ET.fromstring(contents["word/document.xml"]) + assert [flow(p) for p in root.iter(f"{W}p")] == [ + "T", "A", "a", "b", "See {REF ld_a \\n \\h|1}{REF ld_x \\n \\h|(a)}."] + assert [indent(p) for p in root.iter(f"{W}p")] == [None] * 5 + + +def test_the_same_plan_and_tree_are_the_same_bytes() -> None: + document = tree(bullets(item("a", unit="u1")), para(CrossRef("x", "t", "u9"))) + the_plan = plan(unit("u1", "item", num_id=1, ilvl=0, explicit=True, bookmarks=("a",)), abstracts=[abstract(0)], + nums=[Num(1, 0)], refs={"u9": (field("a"),)}) + assert build(document, the_plan) == build(document, the_plan) + + +# -- Escaping --------------------------------------------------------------------------------------------------- + + +EVIL = "x\x00\x01\x08\x0b\x1f\x7f\x85\ud800\ufffe<&>]]>\"'y" +CLEAN = "x<&>]]>\"'y" + + +def test_hostile_text_in_every_inline_and_block_is_dropped_by_the_sanitiser_and_the_part_stays_well_formed() -> None: + document = tree( + para(EVIL, Emphasis((Text(EVIL),)), Code(EVIL), Link("https://e.com/" + EVIL, (Text(EVIL),), (Text(EVIL),)), + Image("i", (Text(EVIL),)), CrossRef(EVIL, "t", "u9"), TermRef(EVIL, "def:t", "bold"), + DefinedTerm((Text(EVIL),), "def:t", "bold"), Value("attach", EVIL, "#att"), Blank("b", EVIL, EVIL), + Choice(EVIL, (EVIL, EVIL), EVIL), FailureMarker(EVIL), condition_label=EVIL, unit="u1"), + Quote((para(EVIL),)), DraftingNote((para(EVIL),), EVIL), CodeBlock(EVIL), table((EVIL, EVIL), header=(EVIL, EVIL)), + sections=[section(EVIL, unit="s1", condition_label=EVIL)]) + the_plan = plan(unit("u1", bookmarks=("a",)), anchors={"def:t": "lddef_t", "att": "a"}, + refs={"u9": (Literal_(EVIL), field("a", EVIL))}) + contents = build(document, the_plan) + for data in contents.values(): + ET.fromstring(data) + data.decode("utf-8") + text = contents["word/document.xml"].decode("utf-8") + for dropped in ("\x00", "\x01", "\x08", "\x0b", "\x1f", "\x7f", "\x85", "\ufffe"): + assert dropped not in text + root = ET.fromstring(contents["word/document.xml"]) + assert CLEAN in "".join(t.text or "" for t in root.iter(f"{W}t")) + # A field's instruction is the bookmark name and the switches, whatever the text around it says. + assert {(i.text or "").strip() for i in root.iter(f"{W}instrText")} == {"REF a \\n \\h"} + + +# -- Validity --------------------------------------------------------------------------------------------------- + +SOURCE = """--- +legaldown: '0.2' +title: Body +--- + +# Alpha {#alpha} + +Intro, see {{ref: nb}} and {{ref: beta}}. + +## Beta {#beta} + +- item one {#one} + - nested a + - nested b {#nb} + + Continued. +- item two +- - only nested +- item four + +# Gamma + +Last, see {{ref: alpha}}. +""" + + +def planned_document() -> tuple[RenderTree, DocxPlan]: + """The tree of :data:`SOURCE` and a plan written by hand for it: numbered headings, a two-level list on its own + numbering, and the references of the sentences as fields whose cached results are wrong on purpose.""" + document = render(SOURCE, format="text").tree + units = {s.unit: s for s in document.sections} + assert [s for s in units] == ["s1", "s2", "s3"] + headings = abstract(0, "headings", [ + level(0, text="%1.", p_style="Heading1", left=567, hanging=567), + level(1, text="%1.%2", p_style="Heading2", left=567, hanging=567)]) + items = abstract(1, "enumerated", [level(0, "lowerLetter", "(%1)", left=567, hanging=567), + level(1, "lowerRoman", "(%2)", left=1134, hanging=567)]) + nums = [Num(1, 0), Num(2, 1, ((0, 1), (1, 1)))] + list_units = {"u2": ("(a)", 0, ("ld_one",)), "u3": ("(i)", 1, ()), "u4": ("(ii)", 1, ("ld_nb",)), + "u5": ("(b)", 0, ()), "u6": ("(c)", 0, ()), "u7": ("(i)", 1, ()), "u8": ("(d)", 0, ())} + planned = [unit("s1", "section", bookmarks=("ld_alpha",), toc="_Toc100000001"), + unit("s2", "section", bookmarks=("ld_beta",), toc="_Toc100000002"), + unit("s3", "section", toc="_Toc100000003"), + unit("u1", "paragraph"), unit("u9", "paragraph")] + planned += [unit(u, "item", num_id=2, ilvl=ilvl, explicit=True, bookmarks=marks) + for u, (_, ilvl, marks) in list_units.items()] + return document, plan( + *planned, abstracts=[headings, items], nums=nums, heading=1, + anchors={"alpha": "ld_alpha", "beta": "ld_beta", "one": "ld_one", "nb": "ld_nb"}, + refs={"u4": (field("ld_beta", "XXX"), field("ld_one", "XXX"), field("ld_nb", "XXX")), + "s2": (field("ld_beta", "XXX"),), "s1": (Literal_("Section "), field("ld_alpha", "XXX"))}) + + +def planned_data() -> bytes: + document, the_plan = planned_document() + return DocxWriter(load_style("default"), generator="test").write(document, the_plan) + + +@pytest.mark.ooxml_schema +def test_every_hand_planned_document_validates_against_the_schemas() -> None: + from docx_support import require_schemas + + require_schemas() + documents = [planned_data(), build_hostile(), build_everything()] + for data in documents: + assert validation_errors(data) == [] + + +def build_hostile() -> bytes: + document = tree( + para(EVIL, Emphasis((Text(EVIL),)), Link("https://e.com/x", (Text(EVIL),), (Text(EVIL),)), Image("i", (Text(EVIL),)), + FailureMarker(EVIL), Blank("b", EVIL, EVIL), condition_label=EVIL), + table((EVIL, EVIL), header=(EVIL, EVIL)), sections=[section(EVIL, unit="s1")]) + return DocxWriter(load_style("default")).write(document, plan()) + + +def build_everything() -> bytes: + """Every block and inline kind, a table in a list in a quote, and every kind of unit the plan can describe.""" + alt = AltNumber((field("ld_a", "1"), Literal_(".")), "space") + nested = bullets(item("inner", table(("a", "b")), unit="u3"), depth=1) + document = tree( + para("Intro ", defined(), " ", TermRef("term", "def:t", "bold"), " ", CrossRef("1", "a", "s1"), " ", + Link("https://e.com/", (Emphasis((Strong((Text("l"),)),)),)), Image("i", (Text("alt"),)), + Blank("b", "___", "p"), Choice("q", ("o", "p"), "/"), FailureMarker("[X]"), Code("c"), SoftBreak(), + HardBreak(), Value("attach", "S", "#att-0"), condition_label="if A", unit="u1"), + Quote((para("q"), bullets(item("x", nested, unit="u2")))), + DraftingNote((para("n"), CodeBlock("a\nb"), Rule(), table(("1", "2"))), "Note"), + table(("a", "b"), align=("left", "right")), table(("c", "d")), + bullets(ListItem((nested,), unit="u4")), + sections=[section("One", para("alt", unit="u5"), unit="s1"), section("Two", unit="s2", condition_label="if B")], + attachments=attachment_tree(Value("attach", "S", "#att-0")).attachments) + the_plan = plan( + unit("s1", "section", bookmarks=("ld_a",), toc="_Toc1"), + unit("s2", "section", num_id=0, ilvl=0, explicit=True, ind=(567, 567), alt=alt), + unit("u1"), unit("u2", "item", num_id=1, ilvl=0, explicit=True), + unit("u3", "item", num_id=1, ilvl=1, explicit=True), unit("u4", "item", num_id=1, ilvl=0, explicit=True), + unit("u5", "paragraph", num_id=2, ilvl=0, explicit=True), + abstracts=[abstract(0, "headings", [level(0, p_style="Heading1")]), abstract(1)], nums=[Num(1, 1), Num(2, 1, ((0, 1),))], + heading=None, anchors={"def:t": "lddef_t", "att-0": "ldatt_att_0"}, refs={"s1": (field("ld_a", "1"),)}) + return DocxWriter(load_style("default")).write(document, the_plan) + + +# -- LibreOffice ------------------------------------------------------------------------------------------------ + + +@pytest.mark.libreoffice +def test_libreoffice_numbers_the_hand_planned_document_and_updates_every_reference() -> None: + """The cached results are wrong on purpose ('XXX'): what LibreOffice shows after updating fields is what the + numbering definitions and the REF fields compute, which is what a lawyer sees after inserting a clause.""" + require_libreoffice() + shown = libreoffice_export({"planned": planned_data()})["planned"] + lines = [" ".join(line.split()) for line in shown.lines if line.strip()] + assert lines == [ + "Body", + "1. Alpha", + "Intro, see 1.1(a)(ii) and 1.1.", + "1.1 Beta", + "(a) item one", + "(i) nested a", + "(ii) nested b", + "Continued.", + "(b) item two", + "(c)", + "(i) only nested", + "(d) item four", + "2. Gamma", + "Last, see Section 1.", + ] + + +def alternative_data() -> bytes: + """A heading, a later alternative that shares its number (numId 0 and the number as a REF field, cached wrong), a + paragraph with a hyperlink, a table and a closing paragraph.""" + link = Link("https://example.com/", (Text("site"),)) + document = tree(sections=[ + section("One", unit="s1"), + section("Two", para("See ", link, "."), table(("a", "b"), ("c", "d")), para("End"), unit="s2")]) + the_plan = plan( + unit("s1", "section", bookmarks=("ld_one",)), + unit("s2", "section", num_id=0, ilvl=0, explicit=True, ind=(567, 567), + alt=AltNumber((field("ld_one", "XXX"), Literal_(".")), "tab")), + abstracts=[abstract(0, "headings", [level(0, text="%1.", p_style="Heading1", left=567, hanging=567)])], + nums=[Num(1, 0)], heading=1) + return DocxWriter(load_style("default"), generator="test").write(document, the_plan) + + +@pytest.mark.ooxml_schema +def test_the_alternative_document_validates() -> None: + from docx_support import require_schemas + + require_schemas() + assert validation_errors(alternative_data()) == [] + + +@pytest.mark.libreoffice +def test_libreoffice_shows_an_alternatives_number_as_its_holders_a_table_and_a_hyperlinks_text() -> None: + require_libreoffice() + shown = libreoffice_export({"alt": alternative_data()})["alt"] + lines = [" ".join(line.split()) for line in shown.lines if line.strip()] + # "Two" is not counted: it shows the number of "One" as a field, not a counter of its own. + assert lines == ["T", "1. One", "1. Two", "See site.", "H1", "H2", "a", "b", "c", "d", "End"] diff --git a/tests/test_docx_escaping.py b/tests/test_docx_escaping.py new file mode 100644 index 0000000..62cf22f --- /dev/null +++ b/tests/test_docx_escaping.py @@ -0,0 +1,94 @@ +"""Nothing a document, a style or the command line supplies makes the DOCX malformed: every part +of the package is well-formed XML for any text, with the characters XML cannot carry (C0 +controls, DEL, C1, U+FFFE and U+FFFF, lone surrogates) dropped by the one sanitiser +(docs/architecture.md, Security). ``xml.etree`` alone would write a control character raw and a +lone surrogate as a character reference, and Word then reports unreadable content.""" +from __future__ import annotations + +import random +import xml.etree.ElementTree as ET + +import pytest +import yaml +from docx_support import parts +from test_escaping import ESC, EVIL + +from legaldown_render import render + +#: Characters that break XML, text, or Word's reading of it. +HOSTILE = [chr(code) for code in (*range(0x20), 0x7F, 0x80, 0x85, 0x9F, 0xD800, 0xDBFF, 0xDC00, 0xDFFF, 0xFFFE, 0xFFFF, + 0x2028, 0x2029, 0x202E, 0x200B, 0xFEFF)] +MARKUP = list("<>&\"'`[]()\\*_#|~$%@!") + ["]]>", " str: + return "".join(rnd.choice(ALPHABET) for _ in range(length)) + + +def document(rnd: random.Random) -> str: + front = yaml.safe_dump({"legaldown": "0.2", "title": soup(rnd, 25), "subtitle": soup(rnd, 25), + "version": soup(rnd, 8), "effective_date": soup(rnd, 8)}, allow_unicode=True) + return (f"---\n{front}---\n\n{soup(rnd, 40)}\n\n# {soup(rnd, 20)}\n\n{soup(rnd, 80)}\n\n## {soup(rnd, 15)}\n\n" + f"- {soup(rnd, 30)}\n - {soup(rnd, 30)}\n\n> {soup(rnd, 30)}\n\n```\n{soup(rnd, 50)}\n```\n\n" + f"| {soup(rnd, 5)} | {soup(rnd, 5)} |\n|---|---|\n| {soup(rnd, 5)} | {soup(rnd, 5)} |\n\n" + f"[{soup(rnd, 8)}](https://example.com/{soup(rnd, 8)})\n") + + +def assert_well_formed(data: bytes) -> None: + for name, content in parts(data).items(): + try: + ET.fromstring(content) + content.decode("utf-8") + except (ET.ParseError, UnicodeDecodeError) as error: + pytest.fail(f"{name} is not well-formed UTF-8 XML: {error}") + + +def test_the_premise_unsanitised_xml_would_be_malformed() -> None: + element = ET.Element("t") + element.text = "a\x01b" + with pytest.raises(ET.ParseError): + ET.fromstring(ET.tostring(element)) + element.text = "a\ud800b" + with pytest.raises((UnicodeEncodeError, ET.ParseError)): + ET.fromstring(ET.tostring(element, encoding="utf-8")) + + +@pytest.mark.parametrize("seed", range(40)) +def test_every_part_is_well_formed_for_any_text(seed: int) -> None: + rnd = random.Random(seed) + overrides = {key: soup(rnd, 12) + "{page}" + soup(rnd, 4) + "{pages}{title}" for key in FREE_TEXT_SETTINGS} + result = render(document(rnd), format="docx", overrides=overrides) + assert result.binary is not None + assert_well_formed(result.binary) + + +@pytest.mark.parametrize("hostile", [ESC, EVIL, "\x1b[2J\x07\x00", "\ud800", "a\ud800b\udfffc", "\ufffe\uffff", "]]>", + "", "�", "\x85
"]) +def test_a_hostile_string_in_each_place_is_harmless(hostile: str) -> None: + quoted = yaml.safe_dump(hostile, allow_unicode=True).rstrip("\n").removesuffix("\n...").rstrip("\n") + source = (f"---\nlegaldown: '0.2'\ntitle: {quoted}\nsubtitle: {quoted}\n---\n\n# {hostile}\n\n{hostile} " + f"[{hostile}]()\n\n```\n{hostile}\n```\n\n| {hostile} |\n|---|\n| {hostile} |\n") + overrides = {key: hostile + "{page}" for key in FREE_TEXT_SETTINGS} + assert_well_formed(render(source, format="docx", overrides=overrides).binary or b"") + + +def test_nothing_from_the_document_is_left_as_a_control_character() -> None: + source = "---\nlegaldown: '0.2'\ntitle: 'a\\x01b'\n---\n\n# H\x01\x08\x0b\x0c\x1f\x7f\x85x\n\nbody\x01\n" + contents = parts(render(source, format="docx").binary or b"") + for name, content in contents.items(): + text = content.decode("utf-8") + assert not any(ord(c) < 0x20 and c not in "\t\n\r" or 0x7F <= ord(c) <= 0x9F for c in text), name + assert "\ud800" not in text, name + + +def test_a_header_template_with_a_control_character_is_sanitised_not_refused() -> None: + contents = parts(render("---\ntitle: T\n---\n# A\n", format="docx", + overrides={"page.footer.center": "a\x00b\x1bc{page}"}).binary or b"") + footer = ET.fromstring(contents["word/footer1.xml"]) + assert "abc" in "".join(node.text or "" for node in footer.iter() if node.tag.endswith("}t")) diff --git a/tests/test_docx_libreoffice.py b/tests/test_docx_libreoffice.py new file mode 100644 index 0000000..482364b --- /dev/null +++ b/tests/test_docx_libreoffice.py @@ -0,0 +1,183 @@ +"""LibreOffice as the oracle: each document is opened headless through UNO (tests/lo/uno_export.py, +run by the system Python that has ``uno``), its fields and indexes are updated, and what it shows is +compared with what was meant: the words in order, the running header and footer, the page count. +Skipped when ``soffice`` or ``python3-uno`` is missing (it is not on Windows, and not in the test +jobs that do not install it); a failure instead when ``LEGALDOWN_REQUIRE_DOCX_TOOLS`` is set, as in +the CI job that installs them (``pytest -m libreoffice``).""" +from __future__ import annotations + +import re +import xml.etree.ElementTree as ET + +import pytest +from conftest import DOCUMENTS +from docx_support import Exported, libreoffice_export, parts, require_libreoffice, texts + +from legaldown_render import render +from legaldown_render.resolve.docx_plan import plan_docx + +pytestmark = pytest.mark.libreoffice + +ROUND_TRIP = """--- +legaldown: '0.2' +title: Round Trip +subtitle: Sub +language: cs +version: '1.2' +effective_date: 2026-03-04 +--- + +Preamble ž€. + +# First + +Body with *emphasis*, **strong** and `code`.\\ +Second line. + +## Second + +- item one +- item two + +> A quote + +``` +code line 1 + code line 2 +``` + +| H1 | H2 | +|----|----| +| a | b | +""" + +LONG = "---\ntitle: Long\n---\n\n" + "".join(f"# Heading {n}\n\nParagraph {n} " + "word " * 120 + "\n\n" for n in range(40)) + +#: A few lines and three attachments, each of which starts a page (``attachments.separator: page-break``, the default): +#: how many pages it has is the layout's business, but never fewer than the breaks make, and in the order of the document. +PAGED = """--- +legaldown: '0.2' +title: Paged +attachments: + - {id: alpha, title: Alpha Schedule} + - {id: beta, title: Beta Schedule} + - {id: gamma, title: Gamma Schedule} +--- + +# One + +See {{attach: gamma}}. +""" + +HOSTILE = "---\nlegaldown: '0.2'\ntitle: \"a\\x01b\"\n---\n\n# H\x01\x08\x0b\x0c\x1f\x7f\x85x\n\nbody\x01 <&> ]]>\n" + + +def docx(source: str, **settings: object) -> bytes: + result = render(source, format="docx", **settings) + assert result.binary is not None + return result.binary + + +def without_number(line: str, labels: list[str]) -> str: + """*line* without the number LibreOffice drew before the text of a numbered paragraph (one of *labels*, + the numbers Word draws, with any indent of a nested item).""" + text = line.lstrip() + for label in labels: + if text.startswith(f"{label} "): + return text.removeprefix(f"{label} ") + return line + + +def expected_lines(data: bytes) -> list[str]: + """What the paragraphs of the document part say, one line per line, without blank ones.""" + lines = [] + for paragraph in texts(ET.fromstring(parts(data)["word/document.xml"])): + lines += paragraph.split("\n") + return [line for line in lines if line] + + +@pytest.fixture(scope="module") +def shown() -> dict[str, Exported]: + require_libreoffice() + documents = { + "round_trip": docx(ROUND_TRIP, overrides={"page.header.left": "{title}", "page.header.center": "{date}", + "page.header.right": "v{version}, p{page}"}), + "features": docx((DOCUMENTS / "features.lgd").read_text(encoding="utf-8")), + "template": docx((DOCUMENTS / "template.lgd").read_text(encoding="utf-8"), style="continental"), + "long": docx(LONG), + "paged": docx(PAGED, overrides={"contents.enabled": True, "page.footer": {"left": "{page}", "center": "", "right": "{pages}"}}), + "hostile": docx(HOSTILE), + "no_footer": docx(ROUND_TRIP, overrides={"page.footer": {"center": ""}}), + "landscape": docx(ROUND_TRIP, overrides={"page.size": "A4 landscape", "page.footer.right": "{page}"}), + } + return libreoffice_export(documents) + + +def test_the_words_come_back_in_order(shown: dict[str, Exported]) -> None: + # the numbers are drawn by LibreOffice from the numbering definitions (test_docx_roundtrip.py compares them) + assert shown["round_trip"].lines == [ + "Round Trip", "Sub", "Datum účinnosti", "4. března 2026", "Preamble ž€.", "1. First", "Body with emphasis, strong and code.", "Second line.", + "1.1 Second", "(a) item one", "(b) item two", "A quote", "code line 1", " code line 2", "H1", "H2", + "a", "b", ""] + + +@pytest.mark.parametrize("name", ["features", "template"]) +def test_the_test_documents_read_as_their_paragraphs(shown: dict[str, Exported], name: str) -> None: + settings = {"style": "continental"} if name == "template" else {} + source = (DOCUMENTS / f"{name}.lgd").read_text(encoding="utf-8") + labels = sorted({unit.label for unit in plan_docx(render(source, format="text", **settings).tree)[0].units.values() + if unit.label}, key=len, reverse=True) + words = [without_number(line, labels) for line in shown[name].lines if line] + assert words == expected_lines(docx(source, **settings)) + + +def test_the_header_shows_the_templates_and_the_page_field(shown: dict[str, Exported]) -> None: + assert shown["round_trip"].header == "Round Trip\t4. března 2026\tv1.2, p1" + + +def test_the_default_footer_counts_the_pages(shown: dict[str, Exported]) -> None: + footer = shown["round_trip"].footer + match = re.fullmatch(r"1 / (\d+)", footer) # a few lines: one page in any font + assert match is not None, footer + assert int(match.group(1)) == shown["round_trip"].pages == 1 + assert shown["round_trip"].header.count("\t") == 2 + + +def test_the_page_count_field_follows_the_layout(shown: dict[str, Exported]) -> None: + """How many pages a text takes depends on the fonts of the machine, so the counts are compared with each other: the + long document has more pages than the short one, and its footer reads the first page and the number of pages the + layout has.""" + long, short = shown["long"], shown["round_trip"] + assert long.pages > short.pages + match = re.fullmatch(r"(\d+) / (\d+)", long.footer) + assert match is not None, long.footer + assert int(match.group(2)) == long.pages + assert 1 <= int(match.group(1)) <= long.pages + + +def test_each_attachment_starts_a_page_and_the_contents_give_their_pages_in_order(shown: dict[str, Exported]) -> None: + """The page breaks hold in any font: three attachments are on three different pages, in the order of the document, the + first not on the first page, and the footer's page count is the layout's.""" + paged = shown["paged"] + pages = [int(m[1]) for title in ("Alpha", "Beta", "Gamma") + for m in [re.search(rf"^\s*{title} Schedule\t(\d+)$", paged.text, re.M)] if m] + assert len(pages) == 3, paged.text + assert 2 <= pages[0] < pages[1] < pages[2] <= paged.pages + left, right = paged.footer.split("\t")[0], paged.footer.split("\t")[-1] + assert 1 <= int(left) <= paged.pages and right == str(paged.pages), paged.footer # {page} is that of the view's cursor + + +def test_no_footer_means_none(shown: dict[str, Exported]) -> None: + assert shown["no_footer"].footer == "" + + +def test_a_landscape_page_with_a_right_aligned_page_number_opens(shown: dict[str, Exported]) -> None: + assert shown["landscape"].footer == "\t1 / 1\t1" + assert shown["landscape"].lines == shown["round_trip"].lines + + +def test_hostile_text_opens_with_the_unwritable_characters_dropped(shown: dict[str, Exported]) -> None: + lines = shown["hostile"].lines + assert lines[0] == "ab" + assert lines[1] == "1. Hx" + assert lines[2] == "body <&> ]]>" diff --git a/tests/test_docx_lo_text.py b/tests/test_docx_lo_text.py new file mode 100644 index 0000000..31f1184 --- /dev/null +++ b/tests/test_docx_lo_text.py @@ -0,0 +1,91 @@ +"""The text tests/lo/uno_text.py reads from a LibreOffice document model: one line per paragraph, whatever the +version of LibreOffice. It is run here over stand-ins for the UNO objects, so no LibreOffice is needed: the +text export filter, which lays a table row out differently in different versions, is not involved at all.""" +from __future__ import annotations + +import importlib.util +from pathlib import Path + +spec = importlib.util.spec_from_file_location("uno_text", Path(__file__).parent / "lo" / "uno_text.py") +assert spec is not None and spec.loader is not None +uno_text = importlib.util.module_from_spec(spec) +spec.loader.exec_module(uno_text) + + +class Enumeration: + def __init__(self, items: list[object]) -> None: + self.items = list(items) + + def hasMoreElements(self) -> bool: # noqa: N802 - the UNO name + return bool(self.items) + + def nextElement(self) -> object: # noqa: N802 + return self.items.pop(0) + + +class Paragraph: + def __init__(self, text: str, label: str = "") -> None: + self.text, self.ListLabelString = text, label + + def supportsService(self, name: str) -> bool: # noqa: N802 + return name == uno_text.PARAGRAPH + + def getString(self) -> str: # noqa: N802 + return self.text + + +class Cell: + def __init__(self, *items: object) -> None: + self.items = items + + def createEnumeration(self) -> Enumeration: # noqa: N802 + return Enumeration(list(self.items)) + + +class Table: + def __init__(self, cells: dict[str, Cell]) -> None: + self.cells = cells + + def supportsService(self, name: str) -> bool: # noqa: N802 + return name == uno_text.TABLE + + def getCellNames(self) -> list[str]: # noqa: N802 + return list(self.cells) + + def getCellByName(self, name: str) -> Cell: # noqa: N802 + return self.cells[name] + + +class Document: + def __init__(self, *items: object) -> None: + self.items = items + + def getText(self) -> Document: # noqa: N802 + return self + + def createEnumeration(self) -> Enumeration: # noqa: N802 + return Enumeration(list(self.items)) + + +def test_a_table_row_is_one_line_per_cell_paragraph_in_reading_order() -> None: + # the cell names in the order a version of LibreOffice may list them: the text is by row, then by column + table = Table({"B2": Cell(Paragraph("d")), "A1": Cell(Paragraph("a")), "B1": Cell(Paragraph("b"), Paragraph("b2")), + "A2": Cell(Paragraph("c"))}) + document = Document(Paragraph("before"), table, Paragraph("")) + assert uno_text.body_text(document) == "before\na\nb\nb2\nc\nd\n\n" + + +def test_a_table_inside_a_cell_is_read_in_place_and_wide_tables_by_column_length() -> None: + inner = Table({"A1": Cell(Paragraph("x")), "B1": Cell(Paragraph("y"))}) + names = ["A1", "B1", "Z1", "AA1", "A2"] + outer = Table({name: Cell(inner if name == "B1" else Paragraph(name)) for name in reversed(names)}) + assert uno_text.body_text(Document(outer)) == "A1\nx\ny\nZ1\nAA1\nA2\n" + + +def test_a_numbered_paragraph_leads_with_its_label_and_a_space_and_a_line_break_starts_a_line() -> None: + document = Document(Paragraph("First", "1."), Paragraph("one\ntwo\tthree", "(a)"), Paragraph("plain\r\nnext")) + assert uno_text.body_text(document) == "1. First\n(a) one\ntwo\tthree\nplain\nnext\n" + + +def test_an_empty_document_has_no_text() -> None: + assert uno_text.body_text(Document()) == "" diff --git a/tests/test_docx_matter.py b/tests/test_docx_matter.py new file mode 100644 index 0000000..6d3b289 --- /dev/null +++ b/tests/test_docx_matter.py @@ -0,0 +1,968 @@ +"""The front and back matter of the DOCX (§13.6, §13.8, §3.6): the header rows and the parties, the table of +contents, the attachments and the signature blocks. Their XML (no tool needed), the validity of documents +that hold every part, hostile text included (marker ``ooxml_schema``), and what LibreOffice shows once its +fields and indexes are updated (marker ``libreoffice``): the table of contents lists the HTML contents' +entries with page numbers, a reference to an attachment resolves, the attachments are paged as configured, +and the document reads in the order the plain-text writer prints it.""" +from __future__ import annotations + +import random +import re +import xml.etree.ElementTree as ET +from typing import Any + +import pytest +import yaml +from conftest import DOCUMENTS +from docx_support import ( + W_NS, + Exported, + libreoffice_edit, + libreoffice_export, + parts, + repackage, + require_libreoffice, + require_schemas, + texts, + validation_errors, +) +from test_docx_escaping import ALPHABET, assert_well_formed, soup + +from legaldown_render import render +from legaldown_render.errors import InternalError +from legaldown_render.resolve.docx_plan import DocxPlan, plan_docx +from legaldown_render.style import load_style +from legaldown_render.style.docx import docx_settings +from legaldown_render.tree import plain_text +from legaldown_render.writers import DocxWriter + +W = f"{{{W_NS}}}" + +FRONT = """--- +legaldown: '0.2' +title: Master Agreement +subtitle: Every part of the matter +version: '2.1' +effective_date: 2026-03-04 +language: en +questions: + forum: + type: choice + prompt: How are disputes resolved? + choices: + courts: State courts + arbitration: Arbitration +sides: + - name: sellers + label: Seller + parties: + - name: acme + type: legal_entity + legal_name: Acme Corporation + identification_number: CZ-123 + address: 1 Main Street + representatives: + - name: John Smith + title: Chief Executive Officer + - name: Jane Roe + title: Director + - name: buyers + label: Buyer + parties: + - name: beta + type: legal_entity + legal_name: Beta Industries Inc. + - name: gamma + type: legal_entity + legal_name: Gamma Ltd. + representatives: + - name: Pat Doe + title: Proxy +attachments: + - id: schedule-a + title: "Schedule A: Services" + file: files/services.lgd + - id: price-list + title: Price List + when: forum:courts + - id: terms + title: Terms +--- + +Preamble text. + +# Services {#services} + +The Seller provides {{attach: schedule-a}}, see {{attach: price-list, label=the prices}}. + +## Delivery + +Text of the delivery. + +# Fees {#fees} + +Fees are in {{attach: terms}}. + +## Payment {#payment} + +Net 30. + +### Late payment + +Interest applies. +""" + + +def docx(source: str = FRONT, **settings: Any) -> bytes: + result = render(source, format="docx", **settings) + assert result.binary is not None + return result.binary + + +def document_xml(data: bytes) -> ET.Element: + return ET.fromstring(parts(data)["word/document.xml"]) + + +def body(data: bytes) -> list[ET.Element]: + return list(document_xml(data).find(f"{W}body")) # type: ignore[arg-type] + + +def style_of(paragraph: ET.Element) -> str | None: + node = paragraph.find(f"{W}pPr/{W}pStyle") + return None if node is None else node.get(f"{W}val") + + +def paragraphs(data: bytes, style: str) -> list[ET.Element]: + return [p for p in document_xml(data).iter(f"{W}p") if style_of(p) == style] + + +def text_of(element: ET.Element) -> str: + """The text of a paragraph (a tab as TAB), or of any other element: its runs' text.""" + if element.tag in (f"{W}p", f"{W}tc"): + return "\n".join(texts(element)) if element.tag == f"{W}tc" else texts(element)[0] + return "".join({f"{W}t": lambda n: n.text or "", f"{W}tab": lambda n: "\t"}.get(n.tag, lambda n: "")(n) + for n in element.iter()) + + +def instructions(root: ET.Element) -> list[str]: + return [(node.text or "").strip() for node in root.iter(f"{W}instrText")] + + +def bookmark_names(root: ET.Element) -> list[str]: + return [node.get(f"{W}name") for node in root.iter(f"{W}bookmarkStart")] # type: ignore[misc] + + +def tables(data: bytes) -> list[ET.Element]: + return list(document_xml(data).iter(f"{W}tbl")) + + +def contents_style(**overrides: Any) -> dict[str, Any]: + return {"overrides": {"contents.enabled": True, **overrides}} + + +# -- Order ----------------------------------------------------------------------------------------------------- + + +def test_the_matter_is_written_in_the_order_the_html_shows_it() -> None: + data = docx(**contents_style()) + styles = [style_of(c) if c.tag == f"{W}p" else "table" for c in body(data) if c.tag != f"{W}sectPr"] + order = [name for name in styles if name in ("Title", "Subtitle", "table", "PartySide", "ContentsHeading", "TOC1", + "Heading1", "AttachmentsHeading", "AttachmentTitle", + "SignatureSide")] + first = {} + for index, name in enumerate(order): + first.setdefault(name, index) + sequence = ["Title", "Subtitle", "table", "PartySide", "ContentsHeading", "TOC1", "Heading1", + "AttachmentsHeading", "AttachmentTitle"] + assert [first[name] for name in sequence] == sorted(first[name] for name in sequence) + assert order[-1] == "table" # the signature table, after the attachments + assert max(i for i, name in enumerate(order) if name == "AttachmentTitle") < len(order) - 1 + + +# -- Header rows and parties ----------------------------------------------------------------------------------- + + +def test_the_header_rows_are_a_borderless_table_of_label_and_value() -> None: + data = docx() + header = tables(data)[0] + rows = [[text_of(cell) for cell in row.iter(f"{W}tc")] for row in header.iter(f"{W}tr")] + assert rows == [["Effective date", "March 4, 2026"], ["Version", "2.1"]] or rows[0] == ["Effective date", + "March 4, 2026"] + assert header.find(f"{W}tblPr/{W}tblBorders") is None and header.find(f"{W}tblPr/{W}tblStyle") is None + label = header.find(f"{W}tr/{W}tc/{W}p/{W}r/{W}rPr/{W}rStyle") + assert label is not None and label.get(f"{W}val") == "TitleBlockLabel" + + +def test_the_parties_follow_their_side_with_their_details() -> None: + data = docx() + sides = [text_of(p) for p in paragraphs(data, "PartySide")] + assert sides == ["Seller", "Buyer"] + assert [text_of(p) for p in paragraphs(data, "PartyName")] == ["Acme Corporation", "Beta Industries Inc.", + "Gamma Ltd."] + details = tables(data)[1] + rows = [[text_of(cell) for cell in row.iter(f"{W}tc")] for row in details.iter(f"{W}tr")] + assert ["Identification No.", "CZ-123"] in rows and ["Address", "1 Main Street"] in rows + assert any(row[0] == "Represented by" and "John Smith" in row[1] for row in rows) + + +def test_no_header_rows_no_parties_means_no_table() -> None: + data = docx("---\ntitle: T\n---\n# A\n", overrides={"title_block.effective_date": False}) + assert tables(data) == [] + + +def test_the_style_can_hide_the_parties() -> None: + data = docx(overrides={"title_block.parties": False}) + assert paragraphs(data, "PartySide") == [] + + +def test_version_and_date_are_the_rows_the_title_block_shows() -> None: + data = docx(overrides={"title_block.version": True}) + rows = [[text_of(cell) for cell in row.iter(f"{W}tc")] for row in tables(data)[0].iter(f"{W}tr")] + assert rows == [["Effective date", "March 4, 2026"], ["Version", "2.1"]] + + +def test_two_tables_in_a_row_are_kept_apart() -> None: + source = "---\ntitle: T\neffective_date: 2026-01-02\n---\n\n| a | b |\n|---|---|\n| c | d |\n" + elements = [c for c in body(docx(source)) if c.tag != f"{W}sectPr"] + tags = [c.tag.removeprefix(W) for c in elements] + assert tags[:5] == ["p", "tbl", "p", "tbl", "p"] + + +# -- Table of contents ----------------------------------------------------------------------------------------- + + +def toc_paragraphs(data: bytes) -> list[ET.Element]: + return [p for p in document_xml(data).iter(f"{W}p") if (style_of(p) or "").startswith("TOC")] + + +def test_the_contents_label_is_a_style_without_numbers_or_outline() -> None: + data = docx(**contents_style()) + [label] = paragraphs(data, "ContentsHeading") + assert text_of(label) == "Contents" + assert label.find(f"{W}pPr/{W}numPr") is None and label.find(f"{W}pPr/{W}outlineLvl") is None + styles = ET.fromstring(parts(data)["word/styles.xml"]) + heading = next(s for s in styles.iter(f"{W}style") if s.get(f"{W}styleId") == "ContentsHeading") + assert heading.find(f"{W}pPr/{W}outlineLvl") is None and heading.find(f"{W}pPr/{W}numPr") is None + assert heading.find(f"{W}name").get(f"{W}val") == "Contents Heading" + + +def test_the_toc_field_is_one_complex_field_over_the_entries() -> None: + data = docx(**contents_style()) + root = document_xml(data) + toc = [i for i in instructions(root) if i.startswith("TOC")] + assert toc == ['TOC \\o "1-2" \\h \\z \\u'] + kinds = [node.get(f"{W}fldCharType") for node in root.iter(f"{W}fldChar")] + assert kinds.count("begin") == kinds.count("separate") == kinds.count("end") + # the TOC field opens in the first entry and closes in a paragraph of its own after the last + elements = list(root.find(f"{W}body")) # type: ignore[arg-type] + entries = [p for p in elements if (style_of(p) or "").startswith("TOC")] + first_kinds = [n.get(f"{W}fldCharType") for n in entries[0].iter(f"{W}fldChar")] + assert first_kinds[:2] == ["begin", "separate"] + last = elements[elements.index(entries[-1]) + 1] + assert text_of(last) == "" and [n.get(f"{W}fldCharType") for n in last.iter(f"{W}fldChar")] == ["end"] + + +@pytest.mark.parametrize("depth", [1, 2, 3, 5]) +def test_the_depth_is_the_style_s(depth: int) -> None: + data = docx(**contents_style(**{"contents.depth": depth})) + assert f'TOC \\o "1-{depth}" \\h \\z \\u' in instructions(document_xml(data)) + + +def test_the_cached_entries_are_the_planned_ones() -> None: + result = render(FRONT, format="docx", **contents_style()) + assert result.binary is not None + plan = plan_docx(result.tree)[0] + entries = toc_paragraphs(result.binary) + assert len(entries) == len(plan.toc) == len(result.tree.contents) + for paragraph, planned, entry in zip(entries, plan.toc, result.tree.contents, strict=True): + assert style_of(paragraph) == f"TOC{planned.level}" + link = paragraph.find(f"{W}hyperlink") + assert link is not None and link.get(f"{W}anchor") == planned.bookmark + shown = text_of(paragraph) + expected = (planned.number + planned.sep + plain_text(planned.title) if planned.number + else plain_text(planned.title)) + (f" {planned.condition_label}" if planned.condition_label else "") + assert shown == expected + "\t" + assert entry.condition_label == planned.condition_label + + +def test_each_entry_links_and_refers_to_a_bookmark_the_document_has() -> None: + data = docx(**contents_style()) + root = document_xml(data) + names = set(bookmark_names(root)) + entries = toc_paragraphs(data) + anchors = [p.find(f"{W}hyperlink").get(f"{W}anchor") for p in entries] # type: ignore[union-attr] + pagerefs = [i.split()[1] for i in instructions(root) if i.startswith("PAGEREF")] + assert anchors == pagerefs and all(a.startswith("_Toc") for a in anchors) + assert set(anchors) <= names and len(set(anchors)) == len(anchors) + assert all(i.endswith("\\h") for i in instructions(root) if i.startswith("PAGEREF")) + for paragraph in entries: # the cached page is empty: separate and end follow each other + kinds = [n.get(f"{W}fldCharType") for n in paragraph.iter(f"{W}fldChar")] + assert kinds[-3:] == ["begin", "separate", "end"] + assert paragraph.find(f"{W}hyperlink").find(f"{W}r/{W}fldChar[@{W}fldCharType='end']") is not None # type: ignore[union-attr] + + +def test_each_entry_has_a_dotted_right_tab_at_the_text_width() -> None: + data = docx(**contents_style()) + text_width = docx_settings(load_style("default")).page.text_width + for paragraph in toc_paragraphs(data): + right = [t for t in paragraph.findall(f"{W}pPr/{W}tabs/{W}tab") if t.get(f"{W}val") == "right"] + assert [(t.get(f"{W}leader"), t.get(f"{W}pos")) for t in right] == [("dot", str(text_width))] + styles = ET.fromstring(parts(data)["word/styles.xml"]) + toc1 = next(s for s in styles.iter(f"{W}style") if s.get(f"{W}styleId") == "TOC1") + tab = toc1.find(f"{W}pPr/{W}tabs/{W}tab") + assert (tab.get(f"{W}val"), tab.get(f"{W}leader"), tab.get(f"{W}pos")) == ("right", "dot", str(text_width)) + + +def test_a_number_is_followed_by_the_tab_the_heading_has_and_aligned_per_level() -> None: + data = docx(**contents_style()) + stops: dict[str, set[str]] = {} + for paragraph in toc_paragraphs(data): + left = [t.get(f"{W}pos") for t in paragraph.findall(f"{W}pPr/{W}tabs/{W}tab") if t.get(f"{W}val") == "left"] + link = paragraph.find(f"{W}hyperlink") + runs = list(link) # type: ignore[arg-type] + has_number = (style_of(paragraph) in ("TOC1", "TOC2")) and text_of(paragraph)[:1].isdigit() + if has_number: + assert runs[1].find(f"{W}tab") is not None and len(left) == 1 + stops.setdefault(style_of(paragraph), set()).add(left[0]) # type: ignore[arg-type] + else: + assert not left + assert all(len(stop) == 1 for stop in stops.values()) + + +def test_a_condition_label_is_a_run_after_the_title() -> None: + source = (DOCUMENTS / "template.lgd").read_text(encoding="utf-8") + data = docx(source, style="continental", overrides={"contents.enabled": True}) + result = render(source, format="text", style="continental", overrides={"contents.enabled": True}) + labelled = [e for e in result.tree.contents if e.condition_label] + assert labelled + runs = [r for p in toc_paragraphs(data) for r in p.iter(f"{W}r") + if r.find(f"{W}rPr/{W}rStyle") is not None and r.find(f"{W}rPr/{W}rStyle").get(f"{W}val") == "ConditionLabel"] # type: ignore[union-attr] + assert [text_of(r) for r in runs] == [e.condition_label for e in labelled] + + +def test_no_contents_no_field_and_no_entries() -> None: + data = docx() + root = document_xml(data) + assert not any(i.startswith(("TOC", "PAGEREF")) for i in instructions(root)) + assert toc_paragraphs(data) == [] and paragraphs(data, "ContentsHeading") == [] + assert not [n for n in bookmark_names(root) if n.startswith("_Toc")] + + +def test_headings_carry_the_toc_bookmarks_when_listed() -> None: + data = docx(**contents_style()) + root = document_xml(data) + names = bookmark_names(root) + assert len([n for n in names if n.startswith("_Toc")]) == len(toc_paragraphs(data)) + + +def test_a_contents_with_nothing_to_list_writes_nothing() -> None: + data = docx("---\ntitle: T\n---\n\nJust text.\n", **contents_style()) + assert toc_paragraphs(data) == [] and paragraphs(data, "ContentsHeading") == [] + + +def test_a_blanked_contents_label_leaves_the_field() -> None: + data = docx(**contents_style(**{"labels.contents": ""})) + assert paragraphs(data, "ContentsHeading") == [] and toc_paragraphs(data) + + +def test_the_planned_entries_and_the_tree_must_agree() -> None: + result = render(FRONT, format="docx", **contents_style()) + plan = plan_docx(result.tree)[0] + broken = DocxPlan(plan.abstract_nums, plan.nums, plan.heading_num_id, units=plan.units, anchors=plan.anchors, + refs=plan.refs, toc=plan.toc[:-1]) + with pytest.raises(InternalError): + DocxWriter(load_style("default", overrides={"contents.enabled": True})).write(result.tree, broken) + + +def test_an_entry_pointing_at_a_bookmark_the_document_lacks_is_a_bug() -> None: + result = render(FRONT, format="docx", **contents_style()) + plan = plan_docx(result.tree)[0] + units = {key: (unit if unit.toc_bookmark is None else _without_toc(unit)) for key, unit in plan.units.items()} + broken = DocxPlan(plan.abstract_nums, plan.nums, plan.heading_num_id, units=units, anchors=plan.anchors, + refs=plan.refs, toc=plan.toc) + with pytest.raises(InternalError): + DocxWriter(load_style("default", overrides={"contents.enabled": True})).write(result.tree, broken) + + +def _without_toc(unit: Any) -> Any: + from dataclasses import replace + + return replace(unit, toc_bookmark=None) + + +# -- Attachments in the contents: outline levels ---------------------------------------------------------------- + + +def outline(paragraph: ET.Element) -> str | None: + node = paragraph.find(f"{W}pPr/{W}outlineLvl") + return None if node is None else node.get(f"{W}val") + + +def test_the_attachments_are_listed_by_outline_level_not_by_style_list() -> None: + data = docx(**contents_style()) + assert not any("\\t" in i for i in instructions(document_xml(data))) + [heading] = paragraphs(data, "AttachmentsHeading") + titles = paragraphs(data, "AttachmentTitle") + assert outline(heading) == "0" and {outline(t) for t in titles} == {"1"} + styles = ET.fromstring(parts(data)["word/styles.xml"]) + by_id = {s.get(f"{W}styleId"): s for s in styles.iter(f"{W}style")} + assert by_id["AttachmentsHeading"].find(f"{W}pPr/{W}outlineLvl").get(f"{W}val") == "0" + assert by_id["AttachmentTitle"].find(f"{W}pPr/{W}outlineLvl").get(f"{W}val") == "1" + for name in ("AttachmentsHeading", "AttachmentTitle"): # no heading numbering reaches them + assert by_id[name].find(f"{W}pPr/{W}numPr") is None and by_id[name].find(f"{W}basedOn").get(f"{W}val") == "Normal" + + +def test_every_outline_level_the_contents_lists_is_what_the_html_lists() -> None: + """The ``\\o`` range and the outline levels select exactly the entries of the tree's contents.""" + for depth in (1, 2, 3): + result = render(FRONT, format="docx", **contents_style(**{"contents.depth": depth})) + assert result.binary is not None + root = document_xml(result.binary) + listed = [] + for paragraph in root.iter(f"{W}p"): + style = style_of(paragraph) or "" + level = None + if style.startswith("Heading"): + level = int(style.removeprefix("Heading")) - 1 + elif outline(paragraph) is not None: + level = int(outline(paragraph)) # type: ignore[arg-type] + if level is not None and level < depth: + listed.append(text_of(paragraph)) + expected = [" ".join(filter(None, (plain_text(e.title), e.condition_label))) for e in result.tree.contents] + assert listed == expected + + +def test_attachments_not_listed_are_body_text_for_the_contents() -> None: + data = docx(**contents_style(**{"contents.attachments": False})) + assert {outline(p) for p in paragraphs(data, "AttachmentTitle")} == {"9"} + assert outline(paragraphs(data, "AttachmentsHeading")[0]) == "9" + assert not [p for p in toc_paragraphs(data) if "Schedule A" in text_of(p)] + + +def test_a_depth_of_one_keeps_the_attachments_in_the_list() -> None: + data = docx(**contents_style(**{"contents.depth": 1})) + assert {outline(p) for p in paragraphs(data, "AttachmentTitle")} == {"0"} + assert [style_of(p) for p in toc_paragraphs(data)].count("TOC1") == len(toc_paragraphs(data)) + + +def test_a_blanked_attachments_label_lists_the_titles_at_the_first_level() -> None: + data = docx(**contents_style(**{"labels.attachments": ""})) + assert paragraphs(data, "AttachmentsHeading") == [] + assert {outline(p) for p in paragraphs(data, "AttachmentTitle")} == {"0"} + assert [text_of(p) for p in toc_paragraphs(data)][-3:] == [ + "Schedule A: Services\t", "Price List " + "Only if: forum:courts\t", "Terms\t"] + + +# -- Attachments ----------------------------------------------------------------------------------------------- + + +def page_break(paragraph: ET.Element) -> bool: + return paragraph.find(f"{W}pPr/{W}pageBreakBefore") is not None + + +def test_the_attachments_are_a_heading_and_titles_with_their_files_and_conditions() -> None: + data = docx() + [heading] = paragraphs(data, "AttachmentsHeading") + assert text_of(heading) == "Attachments" + titles = [text_of(p) for p in paragraphs(data, "AttachmentTitle")] + assert titles == ["Schedule A: Services", "Price List Only if: forum:courts", "Terms"] + assert [text_of(p) for p in paragraphs(data, "AttachmentFile")] == ["File: files/services.lgd"] + label = [r for p in paragraphs(data, "AttachmentTitle") for r in p.iter(f"{W}r") + if r.find(f"{W}rPr/{W}rStyle") is not None] + assert [text_of(r) for r in label] == ["Only if: forum:courts"] + + +def test_each_attachment_is_a_bookmark_and_an_attach_reference_resolves_to_it() -> None: + result = render(FRONT, format="docx", **contents_style()) + assert result.binary is not None + root = document_xml(result.binary) + plan = plan_docx(result.tree)[0] + names = bookmark_names(root) + for anchor in (a.anchor for a in result.tree.attachments): + assert plan.anchors[anchor] in names + assert plan.anchors[result.tree.attachments_anchor] in names + refs = [i for i in instructions(root) if i.startswith("REF ldatt_")] + assert refs and all(i.split()[1] in names for i in refs) + links = [h.get(f"{W}anchor") for h in root.iter(f"{W}hyperlink") if (h.get(f"{W}anchor") or "").startswith("ldatt_")] + assert links and all(a in names for a in links) + assert len(names) == len(set(names)) + # the bookmark holds the title but not the condition label + [paragraph] = [p for p in paragraphs(result.binary, "AttachmentTitle") if "Price List" in text_of(p)] + start = next(e for e in paragraph if e.tag == f"{W}bookmarkStart" and e.get(f"{W}name").startswith("ldatt_")) # type: ignore[union-attr] + inside = [] + for element in list(paragraph)[list(paragraph).index(start) + 1:]: + if element.tag == f"{W}bookmarkEnd" and element.get(f"{W}id") == start.get(f"{W}id"): + break + inside.append(element) + assert "".join(text_of(e) for e in inside) == "Price List" + + +def test_the_attachments_are_bookmarked_without_a_contents_too() -> None: + data = docx() + names = bookmark_names(document_xml(data)) + assert {"ldatt_schedule_a", "ldatt_price_list", "ldatt_terms", "_LdAttachments"} <= set(names) + assert not [n for n in names if n.startswith("_Toc")] + + +def test_page_break_separator_breaks_before_the_part_and_each_attachment() -> None: + data = docx() # the default style + [heading] = paragraphs(data, "AttachmentsHeading") + titles = paragraphs(data, "AttachmentTitle") + assert page_break(heading) and [page_break(t) for t in titles] == [False, True, True] + assert "w:br w:type" not in parts(data)["word/document.xml"].decode("utf-8") + + +def test_page_break_without_a_label_breaks_before_the_first_title() -> None: + data = docx(overrides={"labels.attachments": ""}) + titles = paragraphs(data, "AttachmentTitle") + assert [page_break(t) for t in titles] == [True, True, True] + + +def test_rule_separator_draws_a_line_above_each_attachment() -> None: + data = docx(overrides={"attachments.separator": "rule"}) + titles = paragraphs(data, "AttachmentTitle") + assert all(t.find(f"{W}pPr/{W}pBdr/{W}top") is not None for t in titles) + assert not any(page_break(p) for p in titles + paragraphs(data, "AttachmentsHeading")) + + +def test_no_separator_draws_nothing() -> None: + data = docx(overrides={"attachments.separator": "none"}) + for p in paragraphs(data, "AttachmentTitle") + paragraphs(data, "AttachmentsHeading"): + assert not page_break(p) and p.find(f"{W}pPr/{W}pBdr") is None + + +def test_omitted_attachments_write_nothing() -> None: + data = docx(overrides={"attachments.render": "omit"}) + assert paragraphs(data, "AttachmentsHeading") == [] and paragraphs(data, "AttachmentTitle") == [] + + +def test_the_attachments_listed_in_the_contents_carry_its_bookmarks() -> None: + result = render(FRONT, format="docx", **contents_style()) + assert result.binary is not None + plan = plan_docx(result.tree)[0] + unitless = [t.bookmark for e, t in zip(result.tree.contents, plan.toc, strict=True) if e.unit is None] + root = document_xml(result.binary) + heading, *titles = [*paragraphs(result.binary, "AttachmentsHeading"), *paragraphs(result.binary, "AttachmentTitle")] + for paragraph, name in zip([heading, *titles], unitless, strict=True): + assert name in [b.get(f"{W}name") for b in paragraph.iter(f"{W}bookmarkStart")] + assert len(set(bookmark_names(root))) == len(bookmark_names(root)) + + +#: What the attachments say, and how many there are: two with no id, two with the same id, and one whose id is the +#: identifier of a heading (``# A {#a}`` below; ``attachment-id-collision``). +BAD_ATTACHMENTS = { + "no ids": (" - {title: NoId, file: a.lgd}\n - {title: NoId2, file: b.lgd}\n", 2), + "a duplicate id": (" - {id: x, title: A}\n - {id: x, title: B}\n", 2), + "an id that is a section's": (" - {id: a, title: A}\n", 1), +} + + +@pytest.mark.parametrize("which", BAD_ATTACHMENTS) +@pytest.mark.parametrize("overrides", [{}, {"labels.attachments": ""}, {"contents.depth": 1}]) +def test_attachments_without_a_usable_id_are_still_written_and_listed(which: str, overrides: dict[str, Any]) -> None: + """Attachments with no id, or an id another attachment or a heading has (validator Errors: the document still + renders), have no anchor of their own: the contents lists each by position, and no bookmark is written twice.""" + yaml, count = BAD_ATTACHMENTS[which] + source = ("---\nlegaldown: '0.2'\ntitle: T\nattachments:\n" + yaml + + "---\n\n# A {#a}\n\nSee {{attach: x}}.\n") + result = render(source, format="docx", overrides={"contents.enabled": True, **overrides}) + assert result.binary is not None + root = document_xml(result.binary) + names = bookmark_names(root) + assert len(names) == len({n.lower() for n in names}) + assert len(paragraphs(result.binary, "AttachmentTitle")) == count + toc = [text_of(p).rstrip("\t") for p in toc_paragraphs(result.binary)] + assert toc[-count:] == [text_of(p) for p in paragraphs(result.binary, "AttachmentTitle")] + anchors = [p.find(f"{W}hyperlink").get(f"{W}anchor") for p in toc_paragraphs(result.binary)] # type: ignore[union-attr] + assert set(anchors) <= set(names) + if schema_available(): + assert validation_errors(result.binary) == [] + + +def schema_available() -> bool: + from docx_support import schema_directory + + try: + import lxml # noqa: F401 + except ImportError: + return False + return schema_directory() is not None + + +# -- Signature blocks ------------------------------------------------------------------------------------------ + + +def signature_table(data: bytes) -> ET.Element: + return next(t for t in reversed(tables(data)) if any(style_of(p) == "SignatureSide" for p in t.iter(f"{W}p"))) + + +def test_the_signatures_are_a_borderless_table_two_parties_to_a_row() -> None: + data = docx() + table = signature_table(data) + assert table.find(f"{W}tblPr/{W}tblBorders") is None + rows = table.findall(f"{W}tr") + assert [len(row.findall(f"{W}tc")) for row in rows] == [2, 2] + cells = [c for row in rows for c in row.findall(f"{W}tc")] + sides = [text_of(c.find(f"{W}p")) for c in cells] # type: ignore[arg-type] + assert sides == ["Seller", "Buyer", "Buyer", ""] + assert [text_of(c.findall(f"{W}p")[1]) for c in cells[:3]] == ["Acme Corporation", "Beta Industries Inc.", "Gamma Ltd."] + + +def test_each_signatory_has_a_line_a_name_and_a_title() -> None: + data = docx() + acme = signature_table(data).find(f"{W}tr/{W}tc") + lines = [(style_of(p), text_of(p)) for p in acme.findall(f"{W}p")] # type: ignore[union-attr] + assert lines == [ + ("SignatureSide", "Seller"), ("SignatureText", "Acme Corporation"), + ("SignatureLine", ""), ("SignatureText", "Name: John Smith"), ("SignatureText", "Title: Chief Executive Officer"), + ("SignatureLine", ""), ("SignatureText", "Name: Jane Roe"), ("SignatureText", "Title: Director"), + ("SignatureText", "Date:"), ("SignatureText", "Place:")] + line = acme.find(f"{W}p[3]") # type: ignore[union-attr] + assert line.find(f"{W}pPr/{W}ind") is not None # type: ignore[union-attr] + styles = ET.fromstring(parts(data)["word/styles.xml"]) + style = next(s for s in styles.iter(f"{W}style") if s.get(f"{W}styleId") == "SignatureLine") + assert style.find(f"{W}pPr/{W}pBdr/{W}bottom") is not None + + +def test_a_party_without_representatives_has_one_blank_signing_line() -> None: + data = docx() + beta = signature_table(data).findall(f"{W}tr")[0].findall(f"{W}tc")[1] + lines = [(style_of(p), text_of(p)) for p in beta.findall(f"{W}p")] + assert lines == [("SignatureSide", "Buyer"), ("SignatureText", "Beta Industries Inc."), ("SignatureLine", ""), + ("SignatureText", "Name:"), ("SignatureText", "Title:"), ("SignatureText", "Date:"), + ("SignatureText", "Place:")] + + +def test_the_signatures_stay_together() -> None: + data = docx() + table = signature_table(data) + assert all(row.find(f"{W}trPr/{W}cantSplit") is not None for row in table.findall(f"{W}tr")) + paragraphs_in = list(table.iter(f"{W}p")) + kept = [p.find(f"{W}pPr/{W}keepNext") is not None for p in paragraphs_in] + assert all(kept[:-1]) and not kept[-1] + + +def test_one_party_is_one_cell() -> None: + source = ("---\nlegaldown: '0.2'\ntitle: T\nsides:\n - name: a\n parties:\n - name: p\n type: legal_entity\n" + " legal_name: Only One\n---\n\n# A\n") + table = signature_table(docx(source)) + assert [len(r.findall(f"{W}tc")) for r in table.findall(f"{W}tr")] == [1] + + +def test_no_signatures_when_the_style_or_the_document_says_so() -> None: + assert paragraphs(docx(overrides={"signatures.enabled": False}), "SignatureSide") == [] + off = FRONT.replace("language: en\n", "language: en\ninclude_signatures: false\n", 1) + assert paragraphs(docx(off), "SignatureSide") == [] + + +def test_a_signature_label_blanked_by_the_style_leaves_the_value() -> None: + data = docx(overrides={"labels.signature_name": "", "labels.signature_title": "", "labels.signature_place": ""}) + acme = signature_table(data).find(f"{W}tr/{W}tc") + lines = [text_of(p) for p in acme.findall(f"{W}p")] # type: ignore[union-attr] + assert "John Smith" in lines and "Chief Executive Officer" in lines and "Place:" not in lines and "Date:" in lines + + +def test_the_document_ends_with_a_paragraph_after_the_signature_table() -> None: + elements = [c for c in body(docx()) if c.tag != f"{W}sectPr"] + assert elements[-1].tag == f"{W}p" and elements[-2].tag == f"{W}tbl" + + +# -- The interim warning is gone ------------------------------------------------------------------------------- + + +def test_a_docx_render_reports_what_the_text_render_reports() -> None: + document = (DOCUMENTS / "features.lgd").read_text(encoding="utf-8") + docx_rules = [d.rule for d in render(document, format="docx").diagnostics] + assert "render-docx-interim" not in docx_rules + text = [d.rule for d in render(document, format="text").diagnostics] + assert [r for r in docx_rules if not r.startswith("render-docx")] == text + + +# -- Schema validity ------------------------------------------------------------------------------------------- + + +def hostile_document(rnd: random.Random) -> str: + def party(index: int) -> dict[str, Any]: + return {"name": f"p{index}", "type": "legal_entity", "legal_name": soup(rnd, 12) + "x", + "address": soup(rnd, 10), + "representatives": [{"name": soup(rnd, 8) + "n", "title": soup(rnd, 8)} for _ in range(rnd.randint(0, 2))]} + + front = {"legaldown": "0.2", "title": soup(rnd, 20), "subtitle": soup(rnd, 20), "version": soup(rnd, 6), + "effective_date": "2026-01-02", + "sides": [{"name": f"s{i}", "label": soup(rnd, 8) + "l", "parties": [party(i * 3 + j) for j in range(2)]} + for i in range(2)], + "attachments": [{"id": f"a{i}", "title": soup(rnd, 15) + "t", "file": soup(rnd, 10) + "f"} + for i in range(3)] + [{"title": soup(rnd, 6) + "n"}, {"id": "a1", "title": soup(rnd, 6) + "d"}, + {"id": "h1", "title": soup(rnd, 6) + "s"}]} + return (f"---\n{yaml.safe_dump(front, allow_unicode=True)}---\n\n{soup(rnd, 20)}\n\n# {soup(rnd, 15)} {{#h1}}\n\n" + f"{soup(rnd, 30)} {{{{attach: a1}}}}\n\n## {soup(rnd, 10)}\n\ntext\n") + + +@pytest.mark.ooxml_schema +@pytest.mark.parametrize("settings", [ + {}, {"overrides": {"contents.enabled": True}}, + {"overrides": {"contents.enabled": True, "contents.depth": 5, "attachments.separator": "rule"}}, + {"overrides": {"contents.enabled": True, "contents.attachments": False, "attachments.separator": "none"}}, + {"overrides": {"contents.enabled": True, "labels.attachments": "", "labels.contents": ""}}, + {"overrides": {"title_block.version": True, "page.size": "A4 landscape"}}, + {"style": "continental", "overrides": {"contents.enabled": True}}, + {"style": "outline", "overrides": {"contents.enabled": True, "contents.depth": 3}}, + {"style": "continental", "overrides": {"numbering.scheme": "none", "contents.enabled": True}}, +]) +def test_a_document_with_every_part_validates(settings: dict[str, Any]) -> None: + require_schemas() + assert validation_errors(docx(**settings)) == [] + + +@pytest.mark.ooxml_schema +@pytest.mark.parametrize("name", ["features", "template"]) +@pytest.mark.parametrize("style", ["default", "continental", "outline"]) +def test_the_test_documents_validate_with_a_contents(name: str, style: str) -> None: + require_schemas() + source = (DOCUMENTS / f"{name}.lgd").read_text(encoding="utf-8") + assert validation_errors(docx(source, style=style, overrides={"contents.enabled": True, "contents.depth": 5})) == [] + + +@pytest.mark.ooxml_schema +@pytest.mark.parametrize("seed", range(12)) +def test_hostile_text_in_every_part_of_the_matter_validates(seed: int) -> None: + require_schemas() + rnd = random.Random(7000 + seed) + data = docx(hostile_document(rnd), overrides={"contents.enabled": True, + "labels.contents": soup(rnd, 6), "labels.attachments": soup(rnd, 6), + "labels.colon": soup(rnd, 3) + ":"}) + assert_well_formed(data) + assert validation_errors(data) == [] + + +def test_hostile_front_matter_is_well_formed() -> None: + for seed in range(20): + rnd = random.Random(seed) + assert_well_formed(docx(hostile_document(rnd), overrides={"contents.enabled": True})) + assert ALPHABET + + +# -- LibreOffice ----------------------------------------------------------------------------------------------- + +def toc_lines(exported: Exported) -> list[str]: + """The entries of the contents LibreOffice drew, as ``number title`` (the page number and the tabs left + out), and the page numbers.""" + lines = [] + for line in exported.text.splitlines(): + match = re.fullmatch(r"(.*)\t(\d+)", line) + if match: + lines.append(" ".join(match.group(1).split())) + return lines + + +def toc_pages(exported: Exported) -> list[int]: + return [int(m.group(1)) for line in exported.text.splitlines() if (m := re.fullmatch(r".*\t(\d+)", line))] + + +def expected_entries(source: str, **settings: Any) -> list[str]: + tree = render(source, format="text", **settings).tree + return [" ".join(" ".join(filter(None, (e.label, plain_text(e.title), + e.condition_label))).split()) for e in tree.contents] + + +LONG = ("---\nlegaldown: '0.2'\ntitle: Long\nattachments:\n - id: one\n title: First\n - id: two\n title: Second\n" + " - id: three\n title: Third\n---\n\n" + "".join(f"# Heading {n}\n\nParagraph {n} " + "word " * 100 + "\n\n" + for n in range(6)) + "See {{attach: two}}.\n") + + +@pytest.fixture(scope="module") +def shown() -> dict[str, Exported]: + require_libreoffice() + features = (DOCUMENTS / "features.lgd").read_text(encoding="utf-8") + template = (DOCUMENTS / "template.lgd").read_text(encoding="utf-8") + contents = {"contents.enabled": True, "contents.depth": 3} + documents = { + "master": docx(FRONT, overrides=contents), + "depth1": docx(FRONT, overrides={"contents.enabled": True, "contents.depth": 1}), + "features": docx(features, overrides=contents), + "template": docx(template, style="continental", overrides=contents), + "long_break": docx(LONG, overrides={"contents.enabled": True}), + "long_rule": docx(LONG, overrides={"contents.enabled": True, "attachments.separator": "rule"}), + "long_none": docx(LONG, overrides={"contents.enabled": True, "attachments.separator": "none"}), + } + # The cached title of the attachment reference is wrong on purpose: LibreOffice must compute it. + wrong = repackage(docx(LONG), "word/document.xml", + lambda text: re.sub(r"(REF ldatt_two [^<]*" + r")Second", r"\1WRONG", text)) + assert b"WRONG" in parts(wrong)["word/document.xml"] + documents["wrong_cache"] = wrong + return libreoffice_export(documents) + + +@pytest.mark.libreoffice +def test_the_contents_lists_exactly_the_html_entries(shown: dict[str, Exported]) -> None: + features = (DOCUMENTS / "features.lgd").read_text(encoding="utf-8") + assert toc_lines(shown["features"]) == expected_entries(features, overrides={"contents.enabled": True, + "contents.depth": 3}) + assert toc_lines(shown["master"]) == expected_entries(FRONT, overrides={"contents.enabled": True, + "contents.depth": 3}) + assert toc_lines(shown["depth1"]) == expected_entries(FRONT, overrides={"contents.enabled": True, + "contents.depth": 1}) + # An alternative that shares a number (§15.8) shows the first one's number in the contents too, so every entry of + # the template, the alternatives' included, is the HTML's. + template = (DOCUMENTS / "template.lgd").read_text(encoding="utf-8") + settings = {"style": "continental", "overrides": {"contents.enabled": True, "contents.depth": 3}} + result = render(template, format="text", **settings) + assert any(section.shares_number_with for section in result.tree.sections) + assert toc_lines(shown["template"]) == expected_entries(template, **settings) + + +@pytest.mark.libreoffice +def test_page_numbers_appear_after_the_update(shown: dict[str, Exported]) -> None: + pages = toc_pages(shown["master"]) + assert len(pages) == len(toc_lines(shown["master"])) and all(p >= 1 for p in pages) + assert pages == sorted(pages) + assert max(pages) <= shown["master"].pages + # What holds in any font: the long document's six headings (a fifth of a page of text each) run in order from page 1, + # and the attachments, which each start a page, come after them and end the document. + long = shown["long_break"] + by_title = dict(zip(toc_lines(long), toc_pages(long), strict=True)) + headings = [by_title[f"{n + 1}. Heading {n}"] for n in range(6)] + assert headings[0] == 1 and all(0 <= b - a <= 1 for a, b in zip(headings, headings[1:], strict=False)) + assert by_title["Attachments"] > headings[-1] + assert by_title["Third"] == long.pages == by_title["Attachments"] + 2 + + +@pytest.mark.libreoffice +def test_the_cached_contents_reads_right_before_any_update() -> None: + """The entries in the file are what the HTML shows, so a viewer that never updates fields shows them (without page + numbers, which Word computes).""" + require_libreoffice() + settings = {"overrides": {"contents.enabled": True, "contents.depth": 3}} + template = (DOCUMENTS / "features.lgd").read_text(encoding="utf-8") + for source in (FRONT, template): + cached = libreoffice_export({"cached": docx(source, **settings)}, update=False)["cached"] + lines = cached.text.splitlines() + start = lines.index("Contents") + 1 + entries = lines[start:lines.index("", start)] + assert [" ".join(line.split()) for line in entries] == expected_entries(source, **settings) + + +@pytest.mark.libreoffice +def test_the_contents_is_not_one_of_its_own_entries(shown: dict[str, Exported]) -> None: + assert "Contents" not in toc_lines(shown["master"]) + + +@pytest.mark.libreoffice +def test_an_attachment_reference_resolves_to_the_title_not_the_cache(shown: dict[str, Exported]) -> None: + assert "See Second." in shown["wrong_cache"].text and "WRONG" not in shown["wrong_cache"].text + assert "Schedule A: Services" in shown["master"].text # the {{attach:}} of FRONT, a field to the title + + +@pytest.mark.libreoffice +def test_a_page_break_separator_puts_each_attachment_on_its_own_page(shown: dict[str, Exported]) -> None: + pages = toc_pages(shown["long_break"]) + entries = toc_lines(shown["long_break"]) + by_title = dict(zip(entries, pages, strict=True)) + assert by_title["Attachments"] == by_title["First"] # the heading stays with the first attachment + assert by_title["First"] > by_title["6. Heading 5"] + assert by_title["First"] < by_title["Second"] < by_title["Third"] + assert shown["long_break"].pages == by_title["Third"] + + +@pytest.mark.libreoffice +def test_rule_and_none_keep_the_attachments_on_the_pages_they_fall_on(shown: dict[str, Exported]) -> None: + for name in ("long_rule", "long_none"): + by_title = dict(zip(toc_lines(shown[name]), toc_pages(shown[name]), strict=True)) + # a few lines each: no break between them, so at most the bottom of one page and the top of the next + assert 0 <= by_title["Third"] - by_title["First"] <= 1 and by_title["First"] <= by_title["Second"] <= by_title["Third"] + assert shown["long_break"].pages >= shown["long_rule"].pages + 2 + + +def words(text: str) -> list[str]: + """The words of *text* in order (the plain-text writer's ```` after a link left out: a hyperlink is not text).""" + return re.findall(r"[^\W_]+", re.sub(r"<(?:https?|mailto|tel):[^>]*>", "", text).lower()) + + +def lo_words(exported: Exported) -> list[str]: + lines = [] + for line in exported.text.splitlines(): + lines.append(re.sub(r"\t\d+$", "", line)) + return words("\n".join(lines)) + + +@pytest.mark.libreoffice +@pytest.mark.parametrize("name", ["master", "features"]) +def test_the_whole_document_reads_in_the_order_the_text_writer_prints_it(shown: dict[str, Exported], name: str) -> None: + contents = {"contents.enabled": True, "contents.depth": 3} + source = FRONT if name == "master" else (DOCUMENTS / "features.lgd").read_text(encoding="utf-8") + text = render(source, format="text", overrides=contents).output + assert lo_words(shown[name]) == words(text) + + +@pytest.mark.libreoffice +def test_the_template_reads_in_the_text_writers_order_numbers_included(shown: dict[str, Exported]) -> None: + source = (DOCUMENTS / "template.lgd").read_text(encoding="utf-8") + text = render(source, format="text", style="continental", overrides={"contents.enabled": True, + "contents.depth": 3}).output + expected, found = words(text), lo_words(shown["template"]) + assert found == expected + + +@pytest.mark.libreoffice +def test_a_heading_inserted_in_the_editor_is_listed_once_the_contents_is_updated() -> None: + """The contents is live: the heading a lawyer adds (Enter at the start of a heading) is an entry, and the numbers + after it move, as they do in the contents of the document written with it.""" + require_libreoffice() + contents = {"contents.enabled": True, "contents.depth": 3} + said = libreoffice_edit({"insert": (docx(FRONT, overrides=contents), [("Payment", "Inserted")])})["insert"] + edited = FRONT.replace("## Payment {#payment}", "## Inserted\n\n## Payment {#payment}") + lines = [" ".join(m.group(1).split()) for line in said.splitlines() if (m := re.fullmatch(r"(.*)\t(\d+)", line))] + assert lines == expected_entries(edited, overrides=contents) + assert "2.1 Inserted" in lines and "2.2 Payment" in lines + + +#: SHA-256 of the document part and the styles part of :data:`FRONT` with a contents (uncompressed: the ZIP's bytes +#: depend on the zlib). When the matter changes on purpose, run this test, review the change, and replace the digests. +MATTER_GOLDEN = { + "word/document.xml": "d67511803cd53640fbd96c3c2f6531cc5cf649158c7f56a0d0681db4da5c1b98", + "word/styles.xml": "458e86ca2ad045ce3a714a7e5ba7f3c714e8ab41edf35afb754611046c082d98", +} + + +def test_the_matter_of_a_fixed_document_is_what_it_was() -> None: + import hashlib + + contents = parts(DocxWriter(load_style("default", overrides={"contents.enabled": True}), generator="golden") + .write(render(FRONT, format="text", overrides={"contents.enabled": True}).tree)) + digests = {name: hashlib.sha256(contents[name]).hexdigest() for name in MATTER_GOLDEN} + assert digests == MATTER_GOLDEN, "\n".join(f' "{name}": "{digest}",' for name, digest in digests.items()) + + +def used_styles(data: bytes) -> dict[str, set[str]]: + """The paragraph, character and table styles the parts of the document use, by kind.""" + used: dict[str, set[str]] = {"pStyle": set(), "rStyle": set(), "tblStyle": set()} + for name, content in parts(data).items(): + if name.startswith("word/") and name.endswith(".xml") and name != "word/styles.xml": + for node in ET.fromstring(content).iter(): + local = node.tag.removeprefix(W) + if local in used: + used[local].add(node.get(f"{W}val")) # type: ignore[arg-type] + return used + + +@pytest.mark.parametrize("settings", [ + {"overrides": {"contents.enabled": True, "contents.depth": 5}}, + {"style": "continental", "overrides": {"contents.enabled": True, "title_block.version": True}}, +]) +@pytest.mark.parametrize("name", ["master", "features", "template"]) +def test_every_style_a_document_uses_is_defined(name: str, settings: dict[str, Any]) -> None: + source = FRONT if name == "master" else (DOCUMENTS / f"{name}.lgd").read_text(encoding="utf-8") + data = docx(source, **settings) + defined = {s.get(f"{W}styleId"): s.get(f"{W}type") for s in ET.fromstring(parts(data)["word/styles.xml"]).iter(f"{W}style")} + used = used_styles(data) + assert used["pStyle"] and used["rStyle"] + for kind, expected in (("pStyle", "paragraph"), ("rStyle", "character"), ("tblStyle", "table")): + for style in used[kind]: + assert defined.get(style) == expected, f"{kind} {style} is not a defined {expected} style" + + +def test_the_title_block_labels_are_italic() -> None: + styles = {s.get(f"{W}styleId"): s for s in ET.fromstring(parts(docx())["word/styles.xml"]).iter(f"{W}style")} + label = styles["TitleBlockLabel"] + assert label.get(f"{W}type") == "character" and label.find(f"{W}rPr/{W}i") is not None diff --git a/tests/test_docx_package.py b/tests/test_docx_package.py new file mode 100644 index 0000000..6175862 --- /dev/null +++ b/tests/test_docx_package.py @@ -0,0 +1,868 @@ +"""The OOXML package the DOCX writer produces: its parts, relationships and content types, +the settings, styles, headers and footers, properties and page setup generated from the +style template, determinism, and the builders later work reuses. Schema validity is in +test_docx_validation.py, LibreOffice in test_docx_libreoffice.py; this module needs neither.""" +from __future__ import annotations + +import hashlib +import io +import sys +import xml.etree.ElementTree as ET +import zipfile +from pathlib import Path + +import pytest +from conftest import DOCUMENTS, FEATURES, FRONT, run_cli +from docx_support import W_NS, parts, texts + +from legaldown_render import RenderOptions, render, render_file +from legaldown_render.cli import main +from legaldown_render.errors import InternalError +from legaldown_render.tree import plain_text +from legaldown_render.writers import FORMATS, DocxWriter, format_for_path +from legaldown_render.writers.docx import ooxml, package, styles +from legaldown_render.writers.docx.headers import template_runs + +W = f"{{{W_NS}}}" +SOURCE = ("---\nlegaldown: '0.2'\ntitle: Services Agreement\nsubtitle: A subtitle\nlanguage: cs\nversion: '2.1'\n" + "effective_date: 2026-03-04\n---\n\nPreamble text.\n\n# First {#first}\n\nBody *one*.\n\n## Nested\n\n" + "- item a\n- item b\n\n> quoted\n\n```\ncode 1\ncode 2\n```\n\n| A | B |\n|---|---|\n| 1 | 2 |\n\n---\n") + + +def docx(source: str = SOURCE, **settings: object) -> dict[str, bytes]: + result = render(source, format="docx", **settings) + assert result.binary is not None + return parts(result.binary) + + +def xml(part: bytes) -> ET.Element: + return ET.fromstring(part) + + +# -- The package --------------------------------------------------------------------------- + + +def test_a_package_has_the_parts_a_word_document_needs_in_a_fixed_order() -> None: + assert list(docx()) == [ + "[Content_Types].xml", "_rels/.rels", "docProps/core.xml", "docProps/app.xml", "word/document.xml", + "word/_rels/document.xml.rels", "word/styles.xml", "word/numbering.xml", "word/settings.xml", + "word/footer1.xml"] + + +def test_content_types_name_every_part() -> None: + contents = docx() + root = xml(contents["[Content_Types].xml"]) + ns = "{http://schemas.openxmlformats.org/package/2006/content-types}" + defaults = {entry.get("Extension"): entry.get("ContentType") for entry in root.iter(f"{ns}Default")} + overrides = {entry.get("PartName"): entry.get("ContentType") for entry in root.iter(f"{ns}Override")} + assert defaults == {"rels": "application/vnd.openxmlformats-package.relationships+xml", + "xml": "application/xml"} + assert overrides == { + "/docProps/core.xml": "application/vnd.openxmlformats-package.core-properties+xml", + "/docProps/app.xml": "application/vnd.openxmlformats-officedocument.extended-properties+xml", + "/word/document.xml": "application/vnd.openxmlformats-officedocument.wordprocessingml.document.main+xml", + "/word/styles.xml": "application/vnd.openxmlformats-officedocument.wordprocessingml.styles+xml", + "/word/numbering.xml": "application/vnd.openxmlformats-officedocument.wordprocessingml.numbering+xml", + "/word/settings.xml": "application/vnd.openxmlformats-officedocument.wordprocessingml.settings+xml", + "/word/footer1.xml": "application/vnd.openxmlformats-officedocument.wordprocessingml.footer+xml", + } + + +def test_the_relationships_point_at_the_parts() -> None: + contents = docx() + ns = "{http://schemas.openxmlformats.org/package/2006/relationships}" + package_rels = [(r.get("Id"), r.get("Type").rsplit("/", 1)[1], r.get("Target")) + for r in xml(contents["_rels/.rels"]).iter(f"{ns}Relationship")] + assert package_rels == [("rId1", "officeDocument", "word/document.xml"), + ("rId2", "core-properties", "docProps/core.xml"), + ("rId3", "extended-properties", "docProps/app.xml")] + document_rels = [(r.get("Id"), r.get("Type").rsplit("/", 1)[1], r.get("Target")) + for r in xml(contents["word/_rels/document.xml.rels"]).iter(f"{ns}Relationship")] + assert document_rels == [("rId1", "styles", "styles.xml"), ("rId2", "numbering", "numbering.xml"), + ("rId3", "settings", "settings.xml"), ("rId4", "footer", "footer1.xml")] + + +def test_every_part_is_utf8_xml_with_a_declaration() -> None: + for name, content in docx().items(): + assert content.startswith(b'\n'), name + xml(content) # well-formed + + +def test_the_zip_is_deterministic_and_has_no_timestamps_or_platform_bits() -> None: + first = render(SOURCE, format="docx").binary + second = render(SOURCE, format="docx").binary + assert first == second + assert first is not None + with zipfile.ZipFile(io.BytesIO(first)) as archive: + assert archive.testzip() is None + for info in archive.infolist(): + assert info.date_time == (1980, 1, 1, 0, 0, 0) + assert info.compress_type == zipfile.ZIP_DEFLATED + assert (info.create_system, info.external_attr >> 16, info.extra, info.comment) == (3, 0o100644, b"", b"") + + +def test_a_different_document_is_a_different_file() -> None: + assert render(SOURCE, format="docx").binary != render(SOURCE.replace("First", "Second"), format="docx").binary + + +# -- settings.xml ------------------------------------------------------------------------ + + +def test_settings_have_compatibility_mode_15_and_no_revision_ids() -> None: + settings = xml(docx()["word/settings.xml"]) + modes = [(c.get(f"{W}name"), c.get(f"{W}val")) for c in settings.iter(f"{W}compatSetting")] + assert modes == [("compatibilityMode", "15")] + assert b"rsid" not in docx()["word/settings.xml"] + + +def test_update_fields_is_in_schema_order_after_the_character_spacing_and_before_compat() -> None: + settings = xml(docx()["word/settings.xml"]) + assert [child.tag.removeprefix(W) for child in settings] == [ + "defaultTabStop", "characterSpacingControl", "updateFields", "compat"] + assert settings.find(f"{W}updateFields").get(f"{W}val") == "true" + + +def test_update_fields_on_open_can_be_turned_off() -> None: + settings = xml(docx(overrides={"docx.update_fields_on_open": False})["word/settings.xml"]) + assert settings.find(f"{W}updateFields") is None + assert settings.find(f"{W}compat") is not None + + +def test_the_numbering_part_holds_the_plans_definitions() -> None: + assert [child.tag.removeprefix(W) for child in xml(docx()["word/numbering.xml"])] == [ + "abstractNum", "abstractNum", "num", "num"] + from legaldown_render.writers.docx.settings import build_numbering + + filled = xml(build_numbering([ooxml.el("abstractNum", {"abstractNumId": 0})], [ooxml.el("num", {"numId": 1})])) + assert [child.tag.removeprefix(W) for child in filled] == ["abstractNum", "num"] + + +# -- styles.xml ---------------------------------------------------------------------------- + + +def style_map(contents: dict[str, bytes]) -> dict[str, ET.Element]: + return {s.get(f"{W}styleId"): s for s in xml(contents["word/styles.xml"]).iter(f"{W}style")} + + +def test_the_styles_the_body_and_word_need_exist_under_the_names_word_knows() -> None: + by_id = style_map(docx()) + names = {style_id: s.find(f"{W}name").get(f"{W}val") for style_id, s in by_id.items()} + assert names == { + "Normal": "Normal", "DefaultParagraphFont": "Default Paragraph Font", "TableNormal": "Normal Table", + "NoList": "No List", "Heading1": "heading 1", "Heading2": "heading 2", "Heading3": "heading 3", + "Heading4": "heading 4", "Heading5": "heading 5", "Title": "Title", "Subtitle": "Subtitle", + "Quote": "Quote", "ContentsHeading": "Contents Heading", "DraftingNote": "Drafting Note", "Code": "Code", + "TableGrid": "Table Grid", "Header": "header", "Footer": "footer", "TOC1": "toc 1", "TOC2": "toc 2", + "TOC3": "toc 3", "TOC4": "toc 4", "TOC5": "toc 5", "Hyperlink": "Hyperlink", "InlineCode": "Inline Code", + "DefinedTerm": "Defined Term", "TermReference": "Term Reference", "Blank": "Blank", "Choice": "Choice", + "FailureMarker": "Failure Marker", "ConditionLabel": "Condition Label", + "DraftingNoteLabel": "Drafting Note Label", "TitleBlockText": "Title Block Text", "TitleBlockLabel": "Title Block Label", + "PartySide": "Party Side", "PartyName": "Party Name", "AttachmentsHeading": "Attachments Heading", + "AttachmentTitle": "Attachment Title", "AttachmentFile": "Attachment File", + "SignatureSide": "Signature Side", "SignatureText": "Signature Text", "SignatureLine": "Signature Line"} + assert by_id["Normal"].get(f"{W}default") == "1" + assert by_id["TableGrid"].get(f"{W}type") == "table" + assert by_id["Hyperlink"].get(f"{W}type") == "character" + assert by_id["ContentsHeading"].get(f"{W}customStyle") == "1" + assert by_id["Heading1"].get(f"{W}customStyle") is None + + +def test_headings_are_outline_levels_zero_to_four_numbered_by_the_heading_list() -> None: + by_id = style_map(docx()) + for level in range(1, 6): + heading = by_id[f"Heading{level}"] + assert heading.find(f"{W}pPr/{W}outlineLvl").get(f"{W}val") == str(level - 1) + assert heading.find(f"{W}basedOn").get(f"{W}val") == "Normal" + numbering = heading.find(f"{W}pPr/{W}numPr") + assert (numbering.find(f"{W}ilvl").get(f"{W}val"), numbering.find(f"{W}numId").get(f"{W}val")) == ( + str(level - 1), "1") + assert all(style.find(f"{W}pPr/{W}numPr") is None for key, style in by_id.items() if not key.startswith("Heading")) + assert by_id["ContentsHeading"].find(f"{W}pPr/{W}outlineLvl") is None # in no table of contents + + +def test_headings_link_to_a_numbering_definition_when_given_one() -> None: + from legaldown_render.style import load_style + from legaldown_render.style.docx import docx_settings + + style = load_style("default") + root = xml(styles.build_styles(style, docx_settings(style), "en", heading_num_id=7)) + for style_element in root.iter(f"{W}style"): + style_id = style_element.get(f"{W}styleId") + num = style_element.find(f"{W}pPr/{W}numPr") + if style_id.startswith("Heading"): + level = int(style_id.removeprefix("Heading")) - 1 + assert (num.find(f"{W}ilvl").get(f"{W}val"), num.find(f"{W}numId").get(f"{W}val")) == (str(level), "7") + else: + assert num is None + + +def test_headings_follow_the_style_template() -> None: + by_id = style_map(docx(style="continental")) + heading = by_id["Heading1"] + assert heading.find(f"{W}pPr/{W}jc").get(f"{W}val") == "center" + assert heading.find(f"{W}rPr/{W}caps") is None # continental: transform none + assert heading.find(f"{W}rPr/{W}sz").get(f"{W}val") == "24" # 1.1em of 11pt, rounded to half-points + default = style_map(docx())["Heading1"] + assert default.find(f"{W}rPr/{W}caps") is not None and default.find(f"{W}rPr/{W}b") is not None + + +def test_normal_carries_size_colour_justification_and_line_height() -> None: + normal = style_map(docx(overrides={"typography.justify": True, "typography.font_size": "12pt", + "typography.line_height": "1.25", "typography.color": "#112233"}))["Normal"] + assert normal.find(f"{W}rPr/{W}sz").get(f"{W}val") == "24" + assert normal.find(f"{W}rPr/{W}color").get(f"{W}val") == "112233" + assert normal.find(f"{W}pPr/{W}jc").get(f"{W}val") == "both" + spacing = normal.find(f"{W}pPr/{W}spacing") + assert (spacing.get(f"{W}line"), spacing.get(f"{W}lineRule")) == ("300", "auto") + + +def test_doc_defaults_have_the_language_and_the_first_font_family() -> None: + root = xml(docx()["word/styles.xml"]) + run = root.find(f"{W}docDefaults/{W}rPrDefault/{W}rPr") + fonts = run.find(f"{W}rFonts") + # Only the Latin font: complex-script and East Asian fonts are not the Latin font's to give. + assert {key.removeprefix(W): value for key, value in fonts.attrib.items()} == {"ascii": "Georgia", "hAnsi": "Georgia"} + lang = run.find(f"{W}lang") + assert {key.removeprefix(W): value for key, value in lang.attrib.items()} == {"val": "cs"} # the document's language + english = xml(docx(FRONT + "# A\n")["word/styles.xml"]).find(f"{W}docDefaults/{W}rPrDefault/{W}rPr/{W}lang") + assert english.get(f"{W}val") == "en" + + +def test_a_language_that_is_not_a_tag_falls_back_to_the_locale_then_en_us() -> None: + assert styles.language_tag("cs-CZ", "en") == "cs-CZ" + assert styles.language_tag("not a tag", "de-DE") == "de-DE" + assert styles.language_tag("", None) == "en-US" + assert styles.language_tag(None, "") == "en-US" + + +def test_term_styles_follow_the_definitions_settings() -> None: + by_id = style_map(docx(overrides={"definitions.style": "italic", "definitions.term_style": "underline"})) + assert by_id["DefinedTerm"].find(f"{W}rPr/{W}i") is not None + assert by_id["TermReference"].find(f"{W}rPr/{W}u").get(f"{W}val") == "single" + plain = style_map(docx(overrides={"definitions.style": "plain"}))["DefinedTerm"] + assert plain.find(f"{W}rPr") is None + + +def test_the_hyperlink_and_toc_styles_use_the_link_colour_and_the_text_width() -> None: + by_id = style_map(docx(overrides={"typography.link_color": "#ff0000", "page.margin": "20mm"})) + assert by_id["Hyperlink"].find(f"{W}rPr/{W}color").get(f"{W}val") == "FF0000" + tab = by_id["TOC2"].find(f"{W}pPr/{W}tabs/{W}tab") + assert (tab.get(f"{W}val"), tab.get(f"{W}leader"), tab.get(f"{W}pos")) == ("right", "dot", str(11906 - 2 * 1134)) + assert by_id["TOC3"].find(f"{W}pPr/{W}ind").get(f"{W}left") == "480" + + +# -- Page setup, header and footer --------------------------------------------------------- + + +def section(contents: dict[str, bytes]) -> ET.Element: + return xml(contents["word/document.xml"]).find(f"{W}body/{W}sectPr") + + +def test_the_page_setup_is_in_twips() -> None: + sect = section(docx(overrides={"page.size": "letter", "page.margin": "1in 2in"})) + assert {k.removeprefix(W): v for k, v in sect.find(f"{W}pgSz").attrib.items()} == {"w": "12240", "h": "15840"} + margins = {k.removeprefix(W): v for k, v in sect.find(f"{W}pgMar").attrib.items()} + assert margins == {"top": "1440", "right": "2880", "bottom": "1440", "left": "2880", "header": "720", + "footer": "720", "gutter": "0"} + + +def test_landscape_has_the_long_edge_as_width_and_says_so() -> None: + size = section(docx(overrides={"page.size": "A4 landscape"})).find(f"{W}pgSz") + assert {k.removeprefix(W): v for k, v in size.attrib.items()} == {"w": "16838", "h": "11906", "orient": "landscape"} + + +def test_the_section_references_the_footer_and_there_is_no_header_by_default() -> None: + contents = docx() + sect = section(contents) + assert sect.find(f"{W}headerReference") is None + reference = sect.find(f"{W}footerReference") + assert reference.get(f"{W}type") == "default" + assert reference.get("{http://schemas.openxmlformats.org/officeDocument/2006/relationships}id") == "rId4" + assert [child.tag.removeprefix(W) for child in sect] == ["footerReference", "pgSz", "pgMar"] + + +def test_a_document_without_header_and_footer_has_no_such_parts() -> None: + contents = docx(overrides={"page.footer": {"center": ""}}) + assert not [name for name in contents if "header" in name or "footer" in name] + assert section(contents).find(f"{W}footerReference") is None + + +def test_the_default_footer_is_a_page_and_pages_field_centred() -> None: + footer = xml(docx()["word/footer1.xml"]) + (paragraph,) = footer.iter(f"{W}p") + assert paragraph.find(f"{W}pPr/{W}pStyle").get(f"{W}val") == "Footer" + assert paragraph.find(f"{W}pPr/{W}jc").get(f"{W}val") == "center" + instructions = [t.text for t in paragraph.iter(f"{W}instrText")] + assert instructions == [" PAGE ", " NUMPAGES "] + kinds = [c.get(f"{W}fldCharType") for c in paragraph.iter(f"{W}fldChar")] + assert kinds == ["begin", "separate", "end"] * 2 + assert texts(footer) == ["1 / 1"] # the cached results and the literal between them + + +def header_footer(**templates: str) -> dict[str, bytes]: + return docx(overrides={f"page.{key.replace('_', '.')}": value for key, value in templates.items()}) + + +def test_three_templates_are_set_with_tab_stops_at_the_middle_and_right_margin() -> None: + contents = header_footer(header_left="{title}", header_center="{date}", header_right="v{version}, {page}") + header = xml(contents["word/header1.xml"]) + (paragraph,) = header.iter(f"{W}p") + assert texts(header) == ["Services Agreement\t4. března 2026\tv2.1, 1"] + width = 11906 - 2 * 1417 + tabs = [(t.get(f"{W}val"), t.get(f"{W}pos")) for t in paragraph.iter(f"{W}tab") if t.get(f"{W}pos")] + assert tabs == [("center", str(width // 2)), ("right", str(width))] + assert paragraph.find(f"{W}pPr/{W}pStyle").get(f"{W}val") == "Header" + assert section(contents).find(f"{W}headerReference").get( + "{http://schemas.openxmlformats.org/officeDocument/2006/relationships}id") == "rId4" + + +@pytest.mark.parametrize(("templates", "line", "alignment", "stops"), [ + ({"header_left": "L"}, "L", None, []), + ({"header_center": "C"}, "C", "center", []), + ({"header_right": "R"}, "R", "right", []), + ({"header_left": "L", "header_center": "C"}, "L\tC", None, ["center"]), + ({"header_left": "L", "header_right": "R"}, "L\tR", None, ["right"]), + ({"header_center": "C", "header_right": "R"}, "\tC\tR", None, ["center", "right"]), + ({"header_left": "L", "header_center": "C", "header_right": "R"}, "L\tC\tR", None, ["center", "right"]), +]) +def test_how_the_templates_are_laid_out(templates: dict[str, str], line: str, alignment: str | None, + stops: list[str]) -> None: + header = xml(header_footer(**templates)["word/header1.xml"]) + assert texts(header) == [line] + jc = header.find(f".//{W}pPr/{W}jc") + assert (jc.get(f"{W}val") if jc is not None else None) == alignment + assert [t.get(f"{W}val") for t in header.iter(f"{W}tab") if t.get(f"{W}pos")] == stops + + +def test_title_version_and_date_are_text_and_a_missing_one_is_empty() -> None: + footer = xml(header_footer(footer_center="{title}|{version}|{date}")["word/footer1.xml"]) + assert texts(footer) == ["Services Agreement|2.1|4. března 2026"] + bare = xml(docx(FRONT + "# A\n", overrides={"page.footer.center": "[{version}][{date}]"})["word/footer1.xml"]) + assert texts(bare) == ["[][]"] + + +def test_the_date_follows_the_style_and_the_locale() -> None: + footer = xml(docx(overrides={"page.footer.center": "{date}", "values.date": "short", "locale": "cs-CZ"}) + ["word/footer1.xml"]) + assert texts(footer) == ["04.03.26"] + + +def test_a_version_or_date_that_holds_directives_reads_as_the_title_block_shows_it() -> None: + source = "---\ntitle: T\nversion: 'v {{date: 2026-03-04}} {{placeholder: v}}'\neffective_date: '{{date: 2026-01-02}}'\n---\n# A\n" + overrides = {"page.footer.center": "[{version}|{date}]", "title_block.version": True} + result = render(source, format="docx", overrides=overrides) + footer = xml(parts(result.binary)["word/footer1.xml"]) + assert texts(footer) == ["[v March 4, 2026 [_____]|January 2, 2026]"] + shown = {label: plain_text(value) for label, value in result.tree.header} + assert shown["Version"] == "v March 4, 2026 [_____]" # the same text the title block has + + +def test_the_tree_carries_version_and_date_whether_or_not_the_title_block_shows_them() -> None: + tree = render(SOURCE, format="text", overrides={"title_block.version": False, + "title_block.effective_date": False}).tree + assert (tree.version, tree.effective_date, tree.header) == ("2.1", "4. března 2026", ()) + bare = render(FRONT + "# A\n", format="text").tree + assert (bare.version, bare.effective_date) == ("", "") + + +def test_the_title_is_the_same_text_in_the_body_the_properties_and_the_footer() -> None: + source = "---\ntitle: ' A {{date: 2026-03-04}} B '\n---\n# A\n" + contents = docx(source, overrides={"page.footer.center": "{title}"}) + expected = "A March 4, 2026 B" + assert texts(xml(contents["word/document.xml"]))[0] == expected + assert texts(xml(contents["word/footer1.xml"])) == [expected] + assert xml(contents["docProps/core.xml"])[0].text == expected + + +def test_a_field_name_the_writer_does_not_know_stays_as_written() -> None: + runs = template_runs("{page} of {pages}: {nope}", {"title": "x"}) + root = ET.Element("p") + root.extend(runs) + assert "".join(t.text for t in root.iter(f"{W}t")) == "1 of 1: {nope}" + + +# -- Properties -------------------------------------------------------------------------- + + +def test_core_properties_have_the_title_and_language_and_no_timestamps() -> None: + core = xml(docx()["docProps/core.xml"]) + assert [(child.tag.rsplit("}", 1)[1], child.text) for child in core] == [ + ("title", "Services Agreement"), ("language", "cs")] + raw = docx()["docProps/core.xml"].decode("utf-8") + assert "created" not in raw and "modified" not in raw and "creator" not in raw + + +def test_core_properties_leave_out_an_empty_title() -> None: + core = xml(docx("---\ntitle: ''\n---\n# A\n")["docProps/core.xml"]) + assert [child.tag.rsplit("}", 1)[1] for child in core] == ["language"] + + +def test_app_properties_name_the_program() -> None: + app = xml(docx()["docProps/app.xml"]) + assert app.tag == "{http://schemas.openxmlformats.org/officeDocument/2006/extended-properties}Properties" + assert app[0].text.startswith("legaldown-render ") + + +# -- The body without a plan --------------------------------------------------------------- +# (the body writer itself is in test_docx_body.py) + + +def test_the_body_without_a_plan_has_the_title_headings_and_text_and_no_numbers() -> None: + contents = docx() + body = xml(contents["word/document.xml"]).find(f"{W}body") + styles_used = [(p.find(f"{W}pPr/{W}pStyle").get(f"{W}val") if p.find(f"{W}pPr/{W}pStyle") is not None else None, + texts(p)[0]) for p in body.iter(f"{W}p")] + # the header row of the title block (effective date, in Czech) comes between the subtitle and the preamble + assert styles_used[:7] == [("Title", "Services Agreement"), ("Subtitle", "A subtitle"), + ("TitleBlockText", "Datum účinnosti"), ("TitleBlockText", "4. března 2026"), + (None, "Preamble text."), ("Heading1", "First"), (None, "Body one.")] + assert ("Heading2", "Nested") in styles_used + assert (None, "item a") in styles_used and ("Quote", "quoted") in styles_used + assert [text for style, text in styles_used if style == "Code"] == ["code 1", "code 2"] + # The cells of the table, then the paragraph of the rule that ends the document. + assert [text for _, text in styles_used][-5:] == ["A", "B", "1", "2", ""] + assert all(not text.startswith(("1.", "(a)")) for _, text in styles_used) + + +def test_a_heading_level_is_clamped_to_the_five_word_styles() -> None: + source = FRONT + "# 1\n\n## 2\n\n### 3\n\n#### 4\n\n##### 5\n" + used = [p.find(f"{W}pPr/{W}pStyle").get(f"{W}val") + for p in xml(docx(source)["word/document.xml"]).iter(f"{W}p") if p.find(f"{W}pPr/{W}pStyle") is not None] + assert used == ["Title", "Heading1", "Heading2", "Heading3", "Heading4", "Heading5"] + + +def test_a_code_block_is_one_paragraph_per_line_and_a_hard_break_is_a_break() -> None: + document = xml(docx(FRONT + "# A\n\nline one\\\nline two\n")["word/document.xml"]) + assert "line one\nline two" in texts(document) + + +# -- Sanitising text ----------------------------------------------------------------------- + + +def test_the_sanitiser_drops_what_xml_cannot_carry() -> None: + assert ooxml._t("a\x00b\x01c\x08d\x0be\x0cf\x1fg\x7fh\x85i\x9fj\ufffek\uffffl\ud800m\udfffn") == "abcdefghijklmn" + assert ooxml._t("keep ž € \U0001f600 
 ") == "keep ž € \U0001f600 
 " + assert ooxml._t("a\tb\nc\rd") == "a b c d" + assert ooxml._t("a\tb\nc\rd", breaks=True) == "a\tb\nc\rd" + + +def test_text_runs_turn_tab_and_line_breaks_into_elements() -> None: + run = ooxml.text_run("a\tb\nc\r\nd\re f") + assert [(child.tag.removeprefix(W), child.text) for child in run] == [ + ("t", "a"), ("tab", None), ("t", "b"), ("br", None), ("t", "c"), ("br", None), ("t", "d"), ("br", None), + ("t", "e f")] + assert all(t.get("{http://www.w3.org/XML/1998/namespace}space") == "preserve" for t in run.iter(f"{W}t")) + assert list(ooxml.text_run("")) == [] + assert [child.tag.removeprefix(W) for child in ooxml.text_run("\x00\x01")] == [] + + +def test_every_attribute_and_text_goes_through_the_sanitiser() -> None: + element = ooxml.el("bookmarkStart", {"name": "a\x01b", "id": 3, "flag": True, "off": False, "none": None}, + text="x\x02y") + assert element.attrib == {f"{W}name": "ab", f"{W}id": "3", f"{W}flag": "1", f"{W}off": "0"} + assert element.text == "xy" + assert ET.tostring(ooxml.el("t", text="a\ud800b"), encoding="utf-8") # encodable: no lone surrogate survives + + +def test_serialisation_is_valid_utf8_for_any_text() -> None: + hostile = "".join(chr(code) for code in (*range(0x20), 0x7F, 0x85, 0xD800, 0xDFFF, 0xFFFE, 0xFFFF)) + "<&>\"'" + data = ooxml.serialize(ooxml.el("p", {"x": hostile}, ooxml.text_run(hostile), ooxml.el("instrText", text=hostile))) + ET.fromstring(data.decode("utf-8")) + + +def test_properties_are_sorted_into_schema_order() -> None: + properties = ooxml.ppr(ooxml.val("jc", "left"), ooxml.val("outlineLvl", 0), ooxml.val("pStyle", "X"), + ooxml.el("spacing"), ooxml.el("keepNext"), ooxml.el("numPr"), ooxml.el("ind")) + assert [child.tag.removeprefix(W) for child in properties] == [ + "pStyle", "keepNext", "numPr", "spacing", "ind", "jc", "outlineLvl"] + run = ooxml.rpr(ooxml.val("sz", 2), ooxml.el("b"), ooxml.val("rStyle", "S"), ooxml.val("color", "000000"), + ooxml.el("i")) + assert [child.tag.removeprefix(W) for child in run] == ["rStyle", "b", "i", "color", "sz"] + assert ooxml.ppr(None, [None, False]) is None + with pytest.raises(InternalError): + ooxml.ppr(ooxml.el("notAChild")) + + +def test_a_paragraph_puts_its_properties_before_its_content() -> None: + paragraph = ooxml.p(ooxml.text_run("x"), style="Quote", props=[ooxml.el("keepNext")]) + assert [child.tag.removeprefix(W) for child in paragraph] == ["pPr", "r"] + assert [child.tag.removeprefix(W) for child in paragraph[0]] == ["pStyle", "keepNext"] + + +# -- Fields, bookmarks, hyperlinks --------------------------------------------------------- + + +def test_a_complex_field_is_begin_instruction_separate_result_end() -> None: + runs = ooxml.complex_field("REF ld_x \\n \\h", [ooxml.text_run("2.1")], dirty=True) + root = ET.Element("p") + root.extend(runs) + chars = [(c.get(f"{W}fldCharType"), c.get(f"{W}dirty")) for c in root.iter(f"{W}fldChar")] + assert chars == [("begin", "1"), ("separate", None), ("end", None)] + assert [t.text for t in root.iter(f"{W}instrText")] == [" REF ld_x \\n \\h "] + assert [child[0].tag.removeprefix(W) if len(child) else None for child in runs] == [ + "fldChar", "instrText", "fldChar", "t", "fldChar"] + empty = ooxml.complex_field("PAGEREF x") + assert len(empty) == 4 # no cached result + + +def test_a_field_instruction_is_sanitised_and_kept_on_one_line() -> None: + runs = ooxml.complex_field("REF a\x01b\nc") + assert runs[1].find(f"{W}instrText").text == " REF ab c " + + +def test_bookmark_ids_are_unique_across_the_package_and_names_are_checked() -> None: + bookmarks = ooxml.Bookmarks() + first_start, first_end = bookmarks.pair("ld_first") + second = bookmarks.around("_Ld2", ooxml.text_run("x")) + assert first_start.get(f"{W}id") == first_end.get(f"{W}id") == "1" + assert [c.tag.removeprefix(W) for c in second] == ["bookmarkStart", "r", "bookmarkEnd"] + assert second[0].get(f"{W}id") == second[2].get(f"{W}id") == "2" + for bad in ("", "1abc", "a-b", "a b", "x" * 41, "ž"): + with pytest.raises(InternalError): + bookmarks.pair(bad) + with pytest.raises(InternalError): + bookmarks.pair("LD_FIRST") # Word compares names without regard to case + assert bookmarks.pair("x" * 40) + + +def test_hyperlinks_are_internal_by_anchor_or_external_by_relationship() -> None: + relationships = package.Relationships() + first = relationships.add_external("https://example.com/a b?q=ž") + again = relationships.add_external("https://example.com/a b?q=ž") + other = relationships.add_external("mailto:x@example.com") + assert (first, again, other) == ("rId1", "rId1", "rId2") + external = ooxml.hyperlink_external(first, ooxml.text_run("here")) + assert external.get("{http://schemas.openxmlformats.org/officeDocument/2006/relationships}id") == "rId1" + internal = ooxml.hyperlink_internal("ld_x", ooxml.text_run("there"), tooltip="t") + assert (internal.get(f"{W}anchor"), internal.get(f"{W}tooltip")) == ("ld_x", "t") + listed = ET.fromstring(relationships.to_xml()) + assert [(r.get("Id"), r.get("Target"), r.get("TargetMode")) for r in listed] == [ + ("rId1", "https://example.com/a%20b?q=%C5%BE", "External"), ("rId2", "mailto:x@example.com", "External")] + + +@pytest.mark.parametrize(("url", "expected"), [ + ("https://example.com/a?b=c&d=e#f", "https://example.com/a?b=c&d=e#f"), + ("https://example.com/a b", "https://example.com/a%20b"), + ("https://example.com/%20already", "https://example.com/%20already"), + ("https://example.com/ž", "https://example.com/%C5%BE"), + ('https://example.com/"<>^`{|}', "https://example.com/%22%3C%3E%5E%60%7B%7C%7D"), + ("https://e/a\\b", "https://e/a%5Cb"), + (" https://example.com ", "https://example.com"), + ("", None), (" ", None), ("https://e/\x01", None), ("https://e/\x7f", None), ("https://e/a\nb", None), + ("https://e/\ud800", None), +]) +def test_a_relationship_target_is_a_valid_uri_or_none(url: str, expected: str | None) -> None: + assert package.external_target(url) == expected + if "\\" not in url: + assert package.Relationships().add_external(url) == (None if expected is None else "rId1") + + +@pytest.mark.parametrize("url", [ + "javascript:alert(1)", "JaVaScRiPt:alert(1)", "java\tscript:alert(1)", "file:///etc/passwd", "ms-msdt:/id PCWDiagnostic", + "ms-word:ofe|u|https://x/a.docx", "data:text/html,x", "vbscript:x", "//evil.example/x", "///x", "\\\\host\\share", + "\\host", "https://e/a\\b", "relative/path", "../up", "#fragment", "?q=1", "C:\\x", "localhost:8080", "", +]) +def test_a_hyperlink_relationship_is_refused_for_what_is_not_a_link(url: str) -> None: + """The HTML writer's rule (is_link_target), and more: no relative target and no //host, + since Word follows a relationship without a base to read it against.""" + assert package.Relationships().add_external(url) is None + + +@pytest.mark.parametrize("url", ["https://example.com", "http://e/x", "mailto:a@example.com", "tel:+420123456789", + "ftp://e/f", "urn:isbn:0451450523", "sms:+1555"]) +def test_a_hyperlink_relationship_is_made_for_a_link_scheme(url: str) -> None: + assert package.Relationships().add_external(url) == "rId1" + + +def test_relationships_are_found_in_constant_time_and_keep_their_order() -> None: + relationships = package.Relationships() + ids = [relationships.add_external(f"https://example.com/{n}") for n in range(2000)] + assert ids == [f"rId{n + 1}" for n in range(2000)] + assert relationships.add_external("https://example.com/1999") == "rId2000" + assert relationships.add(package.REL_STYLES, "styles.xml") == "rId2001" + + +def test_a_relationship_value_is_escaped_in_the_part() -> None: + relationships = package.Relationships() + relationships.add("http://x/\"&<", "a&b\"c", external=True) + (entry,) = ET.fromstring(relationships.to_xml()) + assert (entry.get("Type"), entry.get("Target")) == ("http://x/\"&<", "a&b\"c") + + +def test_a_part_added_twice_is_a_bug() -> None: + bundle = package.Package() + bundle.add("word/x.xml", package.CT_XML, b"") + with pytest.raises(InternalError): + bundle.add("word/x.xml", package.CT_XML, b"") + + +# -- The API and the command line ------------------------------------------------------------ + + +def test_docx_is_a_format_selected_by_its_extension() -> None: + assert FORMATS["docx"] == ("docx",) + assert format_for_path("contract.docx") == "docx" + assert format_for_path("CONTRACT.DOCX") == "docx" + + +def test_the_result_carries_the_bytes_and_an_empty_output() -> None: + result = render(SOURCE, RenderOptions(format="docx")) + assert result.format == "docx" + assert result.output == "" + assert isinstance(result.binary, bytes) and result.binary.startswith(b"PK") + assert result.ok + assert render(SOURCE, format="html").binary is None and render(SOURCE, format="text").binary is None + + +def test_render_file_renders_docx(tmp_path: Path) -> None: + source = tmp_path / "contract.lgd" + source.write_text(SOURCE, encoding="utf-8") + assert render_file(source, format="docx").binary == render(SOURCE, format="docx").binary + + +def test_diagnostics_are_the_other_formats_plus_the_planners() -> None: + document = (DOCUMENTS / "features.lgd").read_text(encoding="utf-8") + docx_result = render(document, format="docx").diagnostics + assert [d for d in docx_result if not d.rule.startswith("render-docx")] == render(document, format="text").diagnostics + assert "render-docx-interim" not in {d.rule for d in docx_result} + + +def test_a_clean_document_has_no_docx_diagnostics_and_does_not_refuse_under_strict() -> None: + result = render(FRONT + "# A\n", format="docx", strict=True, filename="d.lgd") + assert [d for d in result.diagnostics if d.rule.startswith("render-docx")] == [] + assert result.ok + + +def test_the_planners_findings_follow_the_renderers_in_the_order_of_their_lines_with_the_documents_file() -> None: + source = FRONT + "# A {#a}\n\n- x {#x}\n\nSee {{ref: nowhere}}.\n\n" + "- r\n" * 30 + result = render(source, format="docx", filename="deal.lgd") + text = render(source, format="text", filename="deal.lgd").diagnostics + assert [d.rule for d in result.diagnostics] == [ + *(d.rule for d in text), "render-docx-letter-overflow"] + assert result.diagnostics[:len(text)] == text and {d.file for d in result.diagnostics} == {"deal.lgd"} + assert "ref-broken" in {d.rule for d in text} + + +def test_the_planners_findings_are_ordered_with_the_renderers_own_by_line() -> None: + source = FRONT + "# A\n\n" + "- r\n" * 30 + "\nA [link](javascript:alert) here.\n" + result = render(source, format="docx") + found = [(d.rule, d.line) for d in result.diagnostics if d.rule.startswith("render-")] + assert [rule for rule, _ in found] == ["render-docx-letter-overflow", "render-link-refused"] + assert found[0][1] < found[1][1] + + +def test_the_writer_gets_the_plan_the_render_reported_on() -> None: + result = render(FRONT + "# A {#a}\n\n- x {#x}\n\nSee {{ref: x}}.\n", format="docx") + assert result.binary is not None + document = parts(result.binary)["word/document.xml"].decode("utf-8") + assert document.count("REF ld_a \\n \\h") == 1 and document.count("REF ld_x \\n \\h") == 1 + assert 'w:name="ld_x"' in document + + +def test_the_docx_writer_can_be_used_directly() -> None: + from legaldown_render.style import load_style + + result = render(SOURCE, format="text") + data = DocxWriter(load_style("default"), generator="test 1").write(result.tree) + assert parts(data)["docProps/app.xml"].count(b"test 1") == 1 + + +def test_the_cli_writes_a_docx_file(tmp_path: Path) -> None: + target = tmp_path / "out.docx" + assert main([FEATURES, "-q", "-o", str(target)]) == 0 + assert target.read_bytes() == render((DOCUMENTS / "features.lgd").read_text(encoding="utf-8"), + format="docx").binary + other = tmp_path / "other.bin" + assert main([FEATURES, "-q", "-f", "docx", "-o", str(other)]) == 0 + assert other.read_bytes() == target.read_bytes() + + +def test_an_uppercase_extension_selects_docx(tmp_path: Path) -> None: + target = tmp_path / "OUT.DOCX" + assert main([FEATURES, "-q", "-o", str(target)]) == 0 + assert target.read_bytes().startswith(b"PK") + + +def test_docx_to_standard_output_with_dash_writes_bytes() -> None: + expected = render((DOCUMENTS / "features.lgd").read_text(encoding="utf-8"), format="docx").binary + done = run_cli(FEATURES, "-q", "-f", "docx", "-o", "-") + assert done.returncode == 0 and done.stderr == b"" + assert done.stdout.startswith(b"PK") and done.stdout == expected + + +def test_docx_without_output_goes_to_standard_output_when_that_is_not_a_terminal() -> None: + done = run_cli(FEATURES, "-q", "-f", "docx") # the test's standard output is a pipe + assert done.returncode == 0 and done.stderr == b"" + assert done.stdout.startswith(b"PK") + assert done.stdout == run_cli(FEATURES, "-q", "-f", "docx", "-o", "-").stdout + + +def test_fragment_is_a_usage_error_for_docx(tmp_path: Path) -> None: + done = run_cli(FEATURES, "-q", "-f", "docx", "--fragment", "-o", str(tmp_path / "x.docx")) + assert done.returncode == 2 and b"--fragment applies to HTML only" in done.stderr + assert list(tmp_path.iterdir()) == [] + + +def test_docx_without_output_is_refused_on_a_terminal(monkeypatch: pytest.MonkeyPatch, + capsys: pytest.CaptureFixture[str]) -> None: + monkeypatch.setattr(sys.stdout, "isatty", lambda: True) + with pytest.raises(SystemExit) as exit_info: + main([FEATURES, "-q", "-f", "docx"]) + assert exit_info.value.code == 2 + assert "refusing to write a DOCX file to a terminal" in capsys.readouterr().err + + +def test_a_standard_output_without_isatty_is_not_a_terminal(monkeypatch: pytest.MonkeyPatch, + capsysbinary: pytest.CaptureFixture[bytes]) -> None: + class Plain: # no isatty, no reconfigure: some embedders replace the stream so + buffer = io.BytesIO() + + def write(self, text: str) -> int: + return len(text) + + def flush(self) -> None: + pass + + monkeypatch.setattr(sys, "stdout", Plain()) + assert main([FEATURES, "-q", "-f", "docx"]) == 0 + assert Plain.buffer.getvalue().startswith(b"PK") + + +def test_docx_is_not_written_to_a_terminal(monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str]) -> None: + monkeypatch.setattr(sys.stdout, "isatty", lambda: True) + with pytest.raises(SystemExit) as exit_info: + main([FEATURES, "-q", "-f", "docx", "-o", "-"]) + assert exit_info.value.code == 2 + assert "refusing to write a DOCX file to a terminal" in capsys.readouterr().err + + +def test_a_style_word_cannot_express_exits_with_status_2_and_writes_no_file(tmp_path: Path) -> None: + target = tmp_path / "out.docx" + done = run_cli(FEATURES, "-q", "-o", str(target), "--set", "page.margin=5%") + assert done.returncode == 2 + assert b"page.margin: uses '%'" in done.stderr + assert not target.exists() + assert run_cli(FEATURES, "-q", "-o", str(tmp_path / "out.html"), "--set", "page.margin=5%").returncode == 0 + + +def test_the_cli_cannot_write_a_docx_into_a_missing_folder(tmp_path: Path) -> None: + done = run_cli(FEATURES, "-q", "-o", str(tmp_path / "no" / "out.docx")) + assert done.returncode == 1 and b"cannot write" in done.stderr + + +def test_text_and_html_to_standard_output_with_dash_still_work_and_name_no_file(tmp_path: Path) -> None: + done = run_cli(FEATURES, "-q", "-f", "text", "-o", "-", cwd=tmp_path) + assert done.returncode == 0 and done.stdout.startswith(b"Services Agreement") + assert list(tmp_path.iterdir()) == [] + + +def test_an_untrusted_render_makes_a_docx_too(tmp_path: Path) -> None: + result = render(SOURCE, format="docx", untrusted=True, document_root=tmp_path) + assert result.binary is not None and result.ok + + +# -- Output that does not depend on anything outside the writer ------------------------------------- + + +def test_the_output_does_not_depend_on_the_global_namespace_registry() -> None: + expected = render(SOURCE, format="docx").binary + saved = dict(ET._namespace_map) # type: ignore[attr-defined] + try: + for uri in (W_NS, "http://schemas.openxmlformats.org/officeDocument/2006/relationships", + "http://schemas.openxmlformats.org/package/2006/metadata/core-properties", + "http://purl.org/dc/elements/1.1/", "http://www.w3.org/XML/1998/namespace"): + ET.register_namespace("host", uri) # another library's registration + assert render(SOURCE, format="docx").binary == expected + finally: + ET._namespace_map.clear() # type: ignore[attr-defined] + ET._namespace_map.update(saved) # type: ignore[attr-defined] + + +def test_importing_the_writer_registers_no_namespace() -> None: + code = ("import xml.etree.ElementTree as ET\nbefore = dict(ET._namespace_map)\n" + "import legaldown_render.writers.docx\nassert ET._namespace_map == before\n") + assert run_python(code).returncode == 0 + + +def run_python(code: str): # noqa: ANN201 - a test helper + import os + import subprocess + + env = dict(os.environ, PYTHONPATH=str(Path(__file__).resolve().parents[1] / "src")) + return subprocess.run([sys.executable, "-c", code], capture_output=True, env=env, check=False) + + +def test_the_serialiser_sanitises_whoever_built_the_element() -> None: + root = ET.Element(f"{W}t", {f"{W}val": "a\x01b\n\"<&>", "plain": "x"}) + root.text = "a\x00b\ud800c&<>\t" + data = ooxml.serialize(root).decode("utf-8") + assert data.endswith('abc&<>\t') + with pytest.raises(InternalError): + ooxml.serialize(ET.Element("{urn:other}x")) + + +def test_a_carriage_return_in_text_is_written_as_a_character_reference() -> None: + root = ET.Element(f"{W}t") + root.text = "a\rb" + data = ooxml.serialize(root).decode("utf-8") + assert " " in data and "\r" not in data + assert ET.fromstring(data).text == "a\rb" + + +def test_an_element_with_a_tail_is_refused_rather_than_dropped() -> None: + root = ET.Element(f"{W}p") + ET.SubElement(root, f"{W}r").tail = "text" + with pytest.raises(InternalError, match="mixed content is not written"): + ooxml.serialize(root) + + +def test_a_text_passed_as_a_child_is_a_bug_not_a_recursion_error() -> None: + for bad in ("text", b"bytes", ["ok", ooxml.el("x")]): + with pytest.raises(InternalError): + ooxml.el("p", None, bad) + assert [c.tag for c in ooxml.el("p", None, [None, False, [ooxml.el("x")]])] == [f"{W}x"] + assert ooxml.p(style="").find(f"{W}pPr") is None + + +def test_names_and_languages_must_match_in_full() -> None: + bookmarks = ooxml.Bookmarks() + with pytest.raises(InternalError): + bookmarks.pair("ld_x\n") + assert styles.language_tag("cs\n", "en") == "en" + assert styles.language_tag("cs-CZ\n", None) == "en-US" + + +def test_code_is_left_aligned_even_when_the_body_is_justified() -> None: + by_id = style_map(docx(overrides={"typography.justify": True})) + assert by_id["Normal"].find(f"{W}pPr/{W}jc").get(f"{W}val") == "both" + assert by_id["Code"].find(f"{W}pPr/{W}jc").get(f"{W}val") == "left" + + +#: SHA-256 of each part of the fixed document below, uncompressed. The ZIP's own bytes depend on the +#: zlib that compressed them (a different zlib may write different, equivalent, bytes); these do not. +#: When a part changes on purpose, run this test, review the change, and replace the digest. +GOLDEN = { + '[Content_Types].xml': '0edc3b54ddf02325a53d17a23256a46b50f782562ff0319d5c48b73ceb50d29b', + '_rels/.rels': 'd6f21829c0081670bdb417d3e3f644b81134a121e074fb153c7686faaa8b167b', + 'docProps/core.xml': '159ed0e537106ab29739fd357a5f6d89a2c0b0154daefd4c13d878bd16dff47e', + 'docProps/app.xml': 'b285204d159fd58da80b6a0242cef523448f9c3a598b357b15faf4627160dcc3', + 'word/document.xml': '39cbe0403104ccefefb9b936e3e432b266d651e54628873ff2d71a57dde798c8', + 'word/_rels/document.xml.rels': 'ada136c2e7d05a317579c9503c1f0dd8dfaa8f0cce687df86fe59e36d5fa5d57', + 'word/styles.xml': 'eec93405688edd872f0c341deeea171d8c5b72a6c83aeea3ff8cd01646c78bc1', + 'word/numbering.xml': '368f179b5e26a34c1f23db597966dfd96b43dc96801a9a7ded728c379db07f59', + 'word/settings.xml': 'b3c72bc5720862c1a9c1449d89903673777f30df6ff38ceb23da73e21cad22e1', + 'word/footer1.xml': 'e693781f92e5a7ccf747fd2ed3facb627e448f389b021882cac589d746aa709b', +} + + +def part_digests() -> dict[str, str]: + from legaldown_render.style import load_style + + tree = render(SOURCE, format="text").tree + data = DocxWriter(load_style("default"), generator="golden").write(tree) + return {name: hashlib.sha256(content).hexdigest() for name, content in parts(data).items()} + + +def test_the_parts_of_a_fixed_document_are_what_they_were() -> None: + digests = part_digests() + assert digests == GOLDEN, "\n".join(f" {name!r}: {digest!r}," for name, digest in digests.items()) diff --git a/tests/test_docx_plan.py b/tests/test_docx_plan.py new file mode 100644 index 0000000..a2a7cb4 --- /dev/null +++ b/tests/test_docx_plan.py @@ -0,0 +1,2823 @@ +"""The DOCX plan (docs/architecture.md, "The DOCX planner"): which abstractNums and nums a document needs, +which (numId, ilvl) each section, list item and numbered paragraph takes, and that Word, simulated, numbers every unit +as the resolver did, or a diagnostic says why not; and, from the numbering, the fields of every cross-reference, the +bookmarks and the table of contents. Everything is checked on the plan's data; no file is written (the LibreOffice +round trip is in test_docx_roundtrip.py).""" +from __future__ import annotations + +import functools +import io +import random +import re +import time +import xml.etree.ElementTree as ET +import zipfile +from collections.abc import Iterator, Sequence +from pathlib import Path + +import pytest +from conftest import DOCUMENTS, FRONT, spec_root +from docx_support import W, parts + +from legaldown_render import DocumentError, render +from legaldown_render.errors import StyleError +from legaldown_render.resolve.docx_bookmarks import BookmarkNames +from legaldown_render.resolve.docx_cover import Field, Title, cover, suppressed, uncovered +from legaldown_render.resolve.docx_numbering import NUM_FMT, Count, Ghost, Lit, Step, word_counter +from legaldown_render.resolve.docx_plan import ( + DocxPlan, + FieldSeg, + Literal_, + Segment, + _components, + _Planner, + docx_style_errors, + max_indent, + plan_docx, +) +from legaldown_render.resolve.numbering import format_counter, formats_of +from legaldown_render.style import load_style +from legaldown_render.style.docx import docx_settings +from legaldown_render.tree import ( + CrossRef, + DraftingNote, + List, + ListItem, + Paragraph, + Quote, + RenderTree, + Section, + iter_tree_inlines, +) +from legaldown_render.writers.docx import DocxWriter + +NONE = {"numbering.scheme": "none"} +PARAGRAPHS = {"paragraphs.numbered": True} +ALTERNATIVES = FRONT.replace("title: T", ( + "title: T\nquestions:\n forum:\n type: choice\n choices:\n courts: Courts\n arb: Arbitration\n" + " expert: Expert")) +#: The answers of the question of :data:`ALTERNATIVES`: any two exclude each other, so units that have the same +#: identifier and one answer each are alternatives (§15.4), up to three ways. +FORUMS = ("courts", "arb", "expert") + +#: The styles every document is planned under: the built-in ones, and the overrides that change which lists exist +#: and which labels need the section. +VARIANTS: dict[str, dict] = { + "default": {}, + "outline": {"style": "outline"}, + "continental": {"style": "continental"}, + "mixed": {"overrides": {"numbering.scheme": "mixed"}}, + "none": {"overrides": NONE}, + "paragraphs": {"overrides": PARAGRAPHS}, + "continental-none": {"style": "continental", "overrides": NONE}, + "enumerate-ordered": {"overrides": {"enumeration.ordered": "enumerate"}}, + "bullets": {"overrides": {"enumeration.enabled": False}}, + # labels with words and symbols, a label that shows the path of an item, and a paragraph number with a symbol + "article": {"overrides": {"numbering.levels": [ + {"counter": "upper-roman", "label": "Article {n}", "ref": "{n}"}, + {"counter": "decimal", "label": "{path}", "ref": ".{n}"}, + {"counter": "lower-alpha", "label": "({n})", "ref": "({n})"}]}}, + "section-sign": {"overrides": {"enumeration.levels": [ + {"counter": "decimal", "label": "\u00a7 {n}", "ref": "{n}"}, + {"counter": "lower-alpha", "label": "{n})", "ref": "({n})"}, + {"counter": "lower-roman", "label": "({n})", "ref": "({n})"}]}}, + "path-items": {"overrides": {"enumeration.levels": [ + {"counter": "lower-alpha", "label": "({n})", "ref": "({n})"}, + {"counter": "decimal", "label": "{path}", "ref": "({n})"}, + {"counter": "lower-roman", "label": "{n}.", "ref": "({n})"}]}}, + "path-items-continental": {"style": "continental", "overrides": {"enumeration.levels": [ + {"counter": "lower-alpha", "label": "({n})", "ref": "({n})"}, + {"counter": "decimal", "label": "{path}", "ref": "({n})"}, + {"counter": "lower-roman", "label": "{n}.", "ref": "({n})"}]}}, + "paragraph-sign": {"overrides": {"paragraphs.numbered": True, "paragraphs.format": { + "counter": "decimal", "label": "\u00b6 {n}", "ref": "{n}"}}}, +} + + +def planned(body: str, front: str = FRONT, **settings: object): # noqa: ANN201 + result = render(front + body, format="text", **settings) + plan, diagnostics = plan_docx(result.tree) + return result.tree, plan, diagnostics + + +def rules(diagnostics: list) -> set[str]: # noqa: ANN001 + return {d.rule for d in diagnostics} + + +# -- A walk of the units ---------------------------------------------------------- + + +def _blocks(blocks: tuple) -> Iterator[Section | ListItem | Paragraph]: + for block in blocks: + match block: + case Paragraph() if block.unit is not None: + yield block + case List(): + for item in block.items: + yield item + yield from _blocks(item.blocks) + case Quote(blocks=inner) | DraftingNote(blocks=inner): + yield from _blocks(inner) + + +def units(tree: RenderTree) -> list[Section | ListItem | Paragraph]: + found = list(_blocks(tree.preamble)) + for section in tree.sections: + found.append(section) + found.extend(_blocks(section.blocks)) + return found + + +def lists(tree: RenderTree) -> list[List]: + found: list[List] = [] + + def walk(blocks: tuple) -> None: + for block in blocks: + match block: + case List(): + found.append(block) + for item in block.items: + walk(item.blocks) + case Quote(blocks=inner) | DraftingNote(blocks=inner): + walk(inner) + + walk(tree.preamble) + for section in tree.sections: + walk(section.blocks) + return found + + +def items(tree: RenderTree) -> list[ListItem]: + """Every list item, in document order (unit ids also count paragraphs, so tests ask for items by position).""" + return [u for u in units(tree) if isinstance(u, ListItem)] + + +def shown(plan: DocxPlan, *nodes: ListItem) -> list[tuple]: + """(numId, ilvl, label) of each node.""" + return [(plan.units[n.unit].num_id, plan.units[n.unit].ilvl, plan.units[n.unit].label) for n in nodes] + + +# -- The abstractNums --------------------------------------------------------------- + + +def test_the_default_style_needs_a_heading_a_list_and_a_renumber_abstract_num() -> None: + _, plan, diagnostics = planned("# A\n\n- x\n\n1. y\n") + assert [(a.id, a.kind, a.nsid, a.multi_level) for a in plan.abstract_nums] == [ + (0, "headings", "4C440000", "multilevel"), + (1, "enumerated", "4C440001", "hybridMultilevel"), + (2, "renumber", "4C440002", "hybridMultilevel")] + assert all(len(a.levels) == 9 for a in plan.abstract_nums) + assert plan.heading_num_id == 1 and plan.nums[0].id == 1 and plan.nums[0].start_overrides == () + assert diagnostics == [] + + +def test_abstract_nums_are_made_in_the_order_of_use_and_only_when_used() -> None: + _, plan, _ = planned("# A\n\n1. y\n\n- x\n") + assert [a.kind for a in plan.abstract_nums] == ["headings", "renumber", "enumerated"] + _, plan, _ = planned("Text.\n") + assert [a.kind for a in plan.abstract_nums] == ["headings"] + + +def test_heading_levels_follow_the_scheme() -> None: + def texts(**settings: object) -> list[tuple[str, str]]: + _, plan, _ = planned("# A\n", **settings) + return [(level.num_fmt, level.lvl_text) for level in plan.abstract_nums[0].levels[:5]] + + assert texts() == [("decimal", "%1."), ("decimal", "%1.%2"), ("decimal", "%1.%2.%3"), + ("decimal", "%1.%2.%3.%4"), ("decimal", "%1.%2.%3.%4.%5")] + assert texts(style="outline") == [("upperRoman", "%1."), ("upperLetter", "%2."), ("decimal", "%3."), + ("lowerLetter", "%4."), ("lowerRoman", "(%5)")] + assert texts(overrides={"numbering.scheme": "mixed"}) == [ + ("decimal", "%1."), ("lowerLetter", "(%2)"), ("lowerRoman", "(%3)"), ("upperLetter", "(%4)"), + ("decimal", "(%5)")] + assert texts(style="continental")[:2] == [("decimal", "%1."), ("decimal", "%1.%2")] + + +def test_heading_levels_are_linked_to_the_heading_styles_and_unused_levels_are_neutral() -> None: + _, plan, _ = planned("# A\n") + levels = plan.abstract_nums[0].levels + assert [level.p_style for level in levels] == [f"Heading{k}" for k in range(1, 6)] + [None] * 4 + assert [(level.num_fmt, level.lvl_text) for level in levels[5:]] == [("decimal", f"%{k}") for k in range(6, 10)] + + +def test_a_custom_heading_label_is_a_level_text_and_a_word_label_is_followed_by_a_space() -> None: + levels = [{"counter": "upper-roman", "label": "Article {n}", "ref": "{n}"}] + _, plan, _ = planned("# A\n\n## B\n", overrides={"numbering.levels": levels}) + first, second = plan.abstract_nums[0].levels[:2] + assert (first.num_fmt, first.lvl_text, first.suff) == ("upperRoman", "Article %1", "space") + assert (second.lvl_text, second.suff) == ("%1.%2", "tab") + assert plan.units["s1"].label == "Article I" + + +def test_level_indents_step_by_level_and_headings_are_as_wide_as_their_widest_label() -> None: + _, plan, _ = planned("# A\n\n- x\n") + headings, items = plan.abstract_nums[0].levels, plan.abstract_nums[1].levels + assert [(level.left, level.hanging) for level in items[:3]] == [(567, 567), (1134, 567), (1701, 567)] + assert headings[0].left == headings[0].hanging == 567 + assert headings[4].left == headings[4].hanging > headings[1].left > headings[0].left + + +def test_the_none_scheme_has_no_heading_abstract_num_and_sections_are_unnumbered() -> None: + tree, plan, _ = planned("# A\n\n- x\n", overrides=NONE) + assert plan.heading_num_id is None + assert [a.kind for a in plan.abstract_nums] == ["enumerated"] + assert plan.units["s1"].num_id is None and plan.units["s1"].explicit is False and plan.units["s1"].count is None + assert plan.nums[0].abstract_id == 0 + + +# -- Headings ------------------------------------------------------------------------ + + +def test_headings_take_the_style_num_at_their_level() -> None: + tree, plan, _ = planned("# A\n\n## B\n\n### C\n\n# D\n") + assert [(u.num_id, u.ilvl, u.explicit, u.label, u.word_ref, u.count) for u in map(plan.units.get, ("s1", "s2", "s3", "s4"))] == [ + (1, 0, False, "1.", "1", 1), (1, 1, False, "1.1", "1.1", 1), (1, 2, False, "1.1.1", "1.1.1", 1), + (1, 0, False, "2.", "2", 2)] + + +def test_a_skipped_heading_level_is_numbered_as_the_resolver_does() -> None: + # heading-skip is a Core Error of the document: the plan says nothing more, and notes the cause on the unit. + tree, plan, diagnostics = planned("# A\n\n### B\n\n## C\n") + assert [plan.units[s].label for s in ("s1", "s2", "s3")] == ["1.", "1.1.1", "1.2"] + assert plan.units["s2"].cause == ("skipped-level",) and plan.units["s3"].cause == () + assert diagnostics == [] + + +def test_a_document_that_starts_below_level_one_skips_nothing() -> None: + _, plan, diagnostics = planned("## A\n\n## B\n") + assert [plan.units[s].label for s in ("s1", "s2")] == ["1.", "2."] and diagnostics == [] + + +# -- Lists --------------------------------------------------------------------------- + +NESTED = """# A + +- one + - nested a + - nested b + - deep +- two +- three + +1. first +1. second +""" + + +def test_every_top_level_list_has_a_num_that_starts_it_at_one() -> None: + tree, plan, _ = planned(NESTED) + items = [u for u in units(tree) if isinstance(u, ListItem)] + first_list, ordered = plan.units[items[0].unit], plan.units[items[-2].unit] + assert (first_list.num_id, first_list.ilvl, first_list.explicit, first_list.label) == (2, 0, True, "(a)") + assert (ordered.num_id, ordered.ilvl, ordered.label) == (3, 0, "1.") + assert plan.nums[1].start_overrides == ((0, 1),) + assert plan.nums[2].abstract_id == 2 + + +def test_two_lists_in_a_section_restart_each_at_its_own_num() -> None: + tree, plan, _ = planned("# A\n\n- x\n- y\n\nText.\n\n- z\n") + assert [(n, label) for n, _, label in shown(plan, *items(tree))] == [(2, "(a)"), (2, "(b)"), (3, "(a)")] + + +def test_a_nested_list_of_the_same_kind_uses_a_deeper_level_of_the_parents_num() -> None: + tree, plan, diagnostics = planned(NESTED) + assert shown(plan, *items(tree)[:6]) == [ + (2, 0, "(a)"), (2, 1, "(i)"), (2, 1, "(ii)"), (2, 2, "(A)"), (2, 0, "(b)"), (2, 0, "(c)")] + assert len(plan.nums) == 3 and diagnostics == [] + + +def test_a_list_in_a_new_item_restarts_by_structure_and_needs_no_override() -> None: + tree, plan, _ = planned("# A\n\n- a\n - x\n- b\n - y\n") + _, x, _, y = items(tree) + assert shown(plan, x, y) == [(2, 1, "(i)"), (2, 1, "(i)")] and plan.units[y.unit].cause == () + + +def test_sibling_lists_in_one_item_share_the_num_so_that_the_second_continues() -> None: + body = "# A\n\n- a\n - x\n - y\n\n Between.\n\n - z\n" + tree, plan, diagnostics = planned(body) + nested = items(tree)[1:] + assert [i.number for i in nested] == [1, 2, 3] # the resolver goes on with the second list + assert shown(plan, *nested) == [(2, 1, "(i)"), (2, 1, "(ii)"), (2, 1, "(iii)")] + assert len(plan.nums) == 2 and diagnostics == [] + + +def test_a_sibling_list_of_another_kind_that_continues_gets_a_num_starting_at_its_number() -> None: + # The nested ordered list and the second nested ordered list share one counter in the resolver: the second + # list's first item is 3. Another kind of list in between puts them on different nums. + body = "# A\n\n- a\n 1. x\n 1. y\n\n Between.\n\n - w\n\n 1. z\n" + tree, plan, diagnostics = planned(body) + ordered = [u for u in units(tree) if isinstance(u, ListItem) and u.label and u.label.endswith(".")] + assert [item.number for item in ordered] == [1, 2, 3] + assert [plan.units[item.unit].label for item in ordered] == ["1.", "2.", "3."] + # Another kind of list in between: the second ordered list cannot go on from the first by structure, so its num + # starts at its number, 3: a fixed start that will not follow items added before it, which is a warning (the + # first ordered list restarts at 1, which is information only). + last = plan.units[ordered[-1].unit] + assert last.num_id != plan.units[ordered[0].unit].num_id and last.cause == ("resync",) + assert plan.nums[last.num_id - 1].start_overrides == ((1, 3),) + assert [(d.rule, d.level) for d in diagnostics] == [ + ("render-docx-static-restart", "info"), ("render-docx-shared-counter", "warning")] + + +def test_a_nested_list_of_another_kind_has_a_num_of_its_own_and_is_reported_as_information() -> None: + tree, plan, diagnostics = planned("# A\n\n- a\n 1. x\n 1. y\n- b\n 1. z\n") + _, x, y, _, z = items(tree) + assert shown(plan, x, y, z) == [(3, 1, "1."), (3, 1, "2."), (4, 1, "1.")] + assert plan.units[x.unit].cause == ("restart-family",) + assert plan.nums[2].start_overrides == ((1, 1),) and plan.nums[2].abstract_id == 2 + assert [(d.rule, d.level) for d in diagnostics] == [("render-docx-static-restart", "info")] + assert diagnostics[0].line == tree.sections[0].blocks[0].items[0].blocks[1].items[0].line + + +def test_a_list_in_a_quote_inside_an_item_is_numbered_from_the_section_with_a_num_of_its_own() -> None: + body = "# A\n\n- a\n - x\n\n > - quoted\n > - quoted two\n\n - y\n- b\n" + tree, plan, diagnostics = planned(body) + _, x, quoted, quoted_too, y, _ = items(tree) + assert [quoted.parent_unit, quoted_too.parent_unit] == ["s1", "s1"] # built on the section, not on the item + # a num of its own, at the level its depth gives: fresh counter, as the resolver has + assert shown(plan, quoted, quoted_too) == [(3, 1, "(i)"), (3, 1, "(ii)")] + # The list after the quote goes on from the first nested list (the resolver counts them as siblings), but + # Word's counter of that level was used by the quote's list: it starts again where the resolver is, by a + # fixed setting. + assert shown(plan, x, y) == [(2, 1, "(i)"), (4, 1, "(ii)")] + assert plan.nums[3].start_overrides == ((1, 2),) and plan.units[y.unit].cause == ("resync",) + assert [(d.rule, d.level) for d in diagnostics] == [("render-docx-shared-counter", "warning")] + + +def test_bullets_count_nothing_so_one_num_serves_every_bullet_list() -> None: + body = "# A\n\n- a\n - b\n\n1. o\n - c\n\n# B\n\n- d\n" + tree, plan, diagnostics = planned(body, overrides={"enumeration.enabled": False}) + first, second, _, third, fourth = items(tree) + assert [a.kind for a in plan.abstract_nums] == ["headings", "bullet", "renumber"] + bullet = plan.abstract_nums[1] + assert shown(plan, first, second, third, fourth)[:2] == [(2, 0, ""), (2, 1, "")] + assert {plan.units[n.unit].num_id for n in (first, second, third, fourth)} == {2} + assert all(num.start_overrides == () for num in plan.nums if num.abstract_id == bullet.id) + assert plan.units[first.unit].count is None + assert [level.lvl_text for level in bullet.levels[:4]] == ["\u2022", "\u2013", "\u25e6", "\u2022"] + assert diagnostics == [] # bullets under an ordered list need no restart and no report + + +def test_ordered_lists_that_are_enumerated_take_the_enumeration_levels() -> None: + _, plan, _ = planned("# A\n\n1. x\n 1. y\n", overrides={"enumeration.ordered": "enumerate"}) + assert [a.kind for a in plan.abstract_nums] == ["headings", "enumerated"] + assert [plan.units[u].label for u in ("u1", "u2")] == ["(a)", "(i)"] + + +def test_an_item_in_the_preamble_is_numbered_without_a_section() -> None: + _, plan, _ = planned("- a\n- b\n\n# A\n", style="continental") + assert [(plan.units[u].label, plan.units[u].num_id) for u in ("u1", "u2")] == [("1", 2), ("2", 2)] + + +def test_items_in_a_quote_or_a_drafting_note_are_numbered_like_any_item() -> None: + tree, plan, _ = planned("# A\n\n> - quoted\n\n> [!DRAFTING]\n> - noted\n") + assert [(n, label) for n, _, label in shown(plan, *items(tree))] == [(2, "(a)"), (3, "(a)")] + + +def test_a_list_deeper_than_word_has_levels_goes_on_in_the_next_abstract_num_and_restarts_by_a_fixed_setting() -> None: + body = "# A\n\n" + "".join(" " * i + f"- level {i}\n" for i in range(11)) + tree, plan, diagnostics = planned(body) + deep = items(tree) + assert [plan.units[i.unit].ilvl for i in deep] == [0, 1, 2, 3, 4, 5, 6, 7, 8, 0, 1] + assert [a.kind for a in plan.abstract_nums] == ["headings", "enumerated", "enumerated+1"] + # the resolver's cycle goes on, and Word draws it: level 10 is the first level of the next list, in its format + assert [plan.units[i.unit].label for i in deep] == [i.label for i in deep] + assert [plan.units[i.unit].cause for i in deep[8:]] == [(), ("restart-depth",), ()] + # the list at level 10 is numbered by a num of its own, and the list in it by structure + assert plan.units[deep[9].unit].num_id != plan.units[deep[8].unit].num_id == 2 + assert plan.units[deep[10].unit].num_id == plan.units[deep[9].unit].num_id + assert plan.nums[plan.units[deep[9].unit].num_id - 1].start_overrides == ((0, 1),) + [found] = diagnostics + assert (found.rule, found.level, found.line) == ("render-docx-static-restart", "info", deep[9].line) + assert "past Word's nine levels" in found.message and "Restart at 1" in found.message + + +def test_items_of_lists_past_level_nine_keep_their_numbers_and_the_list_at_level_nine_goes_on() -> None: + body = "# A\n\n" + "".join(" " * i + f"- level {i}\n" for i in range(9)) + body += "".join(" " * 9 + f"- nine {k}\n" for k in "abc") + body += "".join(" " * 10 + f"- ten {k}\n" for k in "ab") + body += " " * 8 + "- level 8 again\n" + tree, plan, diagnostics = planned(body) + found = items(tree) + nine, ten, again = found[9:12], found[12:14], found[14] + # each item has Word's counter of its own level and the resolver's label + assert [(plan.units[i.unit].ilvl, plan.units[i.unit].count) for i in (*nine, *ten)] == [ + (0, 1), (0, 2), (0, 3), (1, 1), (1, 2)] + assert [plan.units[i.unit].label for i in (*nine, *ten)] == ["(a)", "(b)", "(c)", "(i)", "(ii)"] + assert [i.label for i in (*nine, *ten)] == ["(a)", "(b)", "(c)", "(i)", "(ii)"] + # the sibling at level nine is on the list of level nine: the lists past it have their own, so it needs no restart + again_plan = plan.units[again.unit] + assert (again.number, again_plan.count, again_plan.cause) == (2, 2, ()) and again_plan.ilvl == 8 + assert plan.units[again.unit].num_id == plan.units[found[8].unit].num_id + assert rules(diagnostics) == {"render-docx-static-restart"} + + +def test_the_abstract_nums_past_nine_levels_have_the_families_formats_of_the_depth_they_stand_for() -> None: + levels = [{"counter": "decimal", "label": "{n}.", "ref": "{n}"}, + {"counter": "lower-alpha", "label": "({n})", "ref": "({n})"}, + {"counter": "lower-roman", "label": "{n})", "ref": "({n})"}, + {"counter": "upper-alpha", "label": "[{n}]", "ref": "[{n}]"}] + body = "# A\n\n" + "".join(" " * i + f"- level {i}\n" for i in range(20)) + _, plan, _ = planned(body, overrides={"enumeration.levels": levels}) + by_kind = {a.kind: a for a in plan.abstract_nums} + assert list(by_kind) == ["headings", "enumerated", "enumerated+1", "enumerated+2"] + # depth 9 is the second level of the cycle of four, depth 10 the third, ...; each level shows its own counter alone + assert [(level.num_fmt, level.lvl_text) for level in by_kind["enumerated+1"].levels[:5]] == [ + ("lowerLetter", "(%1)"), ("lowerRoman", "%2)"), ("upperLetter", "[%3]"), ("decimal", "%4."), + ("lowerLetter", "(%5)")] + assert [(level.num_fmt, level.lvl_text) for level in by_kind["enumerated+2"].levels[:3]] == [ + ("lowerRoman", "%1)"), ("upperLetter", "[%2]"), ("decimal", "%3.")] # depth 18, 19, 20 + # the indent stops stepping in at 60% of the text width of an A4 page (9072 twips), which level 9 has nearly reached + assert [level.left for level in by_kind["enumerated"].levels[6:]] == [567 * 7, 567 * 8, 567 * 9] + assert [level.left for level in by_kind["enumerated+1"].levels[:5]] == [max_indent(9072)] * 5 == [5443] * 5 + assert all(len(a.levels) == 9 for a in plan.abstract_nums) + + +def test_the_indent_of_a_deep_list_is_limited_by_the_width_of_the_page_it_is_on() -> None: + body = "# A\n\n" + "".join(" " * i + f"- level {i}\n" for i in range(14)) + "\n- bullet\n" + "".join( + " " * (i + 1) + "- bullet\n" for i in range(12)) + result = render(FRONT + body, format="text") + for width in (5557, 14000): # A5 with 25 mm margins; a wide page + plan, _ = plan_docx(result.tree, text_width=width) + lefts = [level.left for a in plan.abstract_nums if a.kind != "headings" for level in a.levels] + assert max(lefts) == max_indent(width) == int(width * 0.6) + # the indents of the items that carry their own (those on the heading list and nested in them) stop there too + clauses = render(FRONT + body, format="text", style="continental") + plan, _ = plan_docx(clauses.tree, text_width=5557) + assert max(unit.ind[0] for unit in plan.units.values() if unit.ind) == max_indent(5557) + # the render plans for the page of the document: on A5 no indent of the file goes past 60% of the text + data = render(FRONT + body, format="docx", style="continental", overrides={"page.size": "A5"}).binary + assert data is not None + document = ET.fromstring(parts(data)["word/document.xml"]) + text_width = 8391 - 2 * 1417 + lefts = [int(ind.get(f"{W}left")) for ind in document.iter(f"{W}ind") if ind.get(f"{W}left")] + assert max(lefts) > 3000 and max(lefts) <= int(text_width * 0.6) + 1 + + +def test_ordered_lists_go_on_past_nine_levels_too() -> None: + body = "# A\n\n" + "".join(" " * i + f"{1}. level {i}\n" for i in range(11)) + tree, plan, diagnostics = planned(body) + deep = items(tree) + assert [a.kind for a in plan.abstract_nums] == ["headings", "renumber", "renumber+1"] + assert [plan.units[i.unit].ilvl for i in deep[8:]] == [8, 0, 1] + assert [plan.units[i.unit].label for i in deep] == [i.label for i in deep] + assert [plan.units[i.unit].cause for i in deep[8:]] == [(), ("restart-depth",), ()] + assert rules(diagnostics) == {"render-docx-static-restart"} + + +def test_bullets_go_on_past_nine_levels_in_further_abstract_nums_too() -> None: + body = "# A\n\n" + "".join(" " * i + f"- bullet {i}\n" for i in range(12)) + tree, plan, diagnostics = planned(body, overrides={"enumeration.enabled": False}) + deep = items(tree) + assert [a.kind for a in plan.abstract_nums] == ["headings", "bullet", "bullet+1"] + assert [plan.units[i.unit].ilvl for i in deep] == [0, 1, 2, 3, 4, 5, 6, 7, 8, 0, 1, 2] + assert len({plan.units[i.unit].num_id for i in deep[:9]}) == 1 == len({plan.units[i.unit].num_id for i in deep[9:]}) + assert plan.units[deep[8].unit].num_id != plan.units[deep[9].unit].num_id + assert all(plan.units[i.unit].cause == () for i in deep) and diagnostics == [] + # the glyph goes on cycling, as it would at that depth + bullets = {a.kind: a for a in plan.abstract_nums} + assert [bullets["bullet"].levels[8].lvl_text, bullets["bullet+1"].levels[0].lvl_text] == ["◦", "•"] + + +def test_a_list_of_any_family_nested_past_the_levels_the_plan_has_is_reported_once() -> None: + for marker in ("-", "1."): + width = 3 if marker == "1." else 2 + body = "# A\n\n" + "".join(" " * (width * i) + f"{marker} level {i}\n" for i in range(38)) + _, plan, diagnostics = planned(body, overrides={"enumeration.enabled": marker == "1."}) + assert [d.rule for d in diagnostics if d.rule == "render-docx-list-depth"] == ["render-docx-list-depth"] + assert any("clamped" in unit.cause for unit in plan.units.values()) + + +def test_a_list_nested_past_the_levels_the_plan_has_is_clamped_and_reported_once() -> None: + body = "# A\n\n" + "".join(" " * i + f"- level {i}\n" for i in range(38)) + tree, plan, diagnostics = planned(body) + deep = items(tree) + assert [a.kind for a in plan.abstract_nums] == ["headings", "enumerated", "enumerated+1", "enumerated+2", + "enumerated+3"] + assert [plan.units[i.unit].ilvl for i in deep[34:]] == [7, 8, 8, 8] + assert [plan.units[i.unit].cause for i in deep[35:]] == [(), ("clamped",), ("clamped",)] + # each clamped list restarts on a num of its own: Word's level 9 of the last list would count on otherwise + assert len({plan.units[i.unit].num_id for i in deep[35:]}) == 3 + assert all(plan.units[i.unit].count == 1 for i in deep[35:]) + [found] = [d for d in diagnostics if d.rule == "render-docx-list-depth"] + assert found.level == "warning" and found.line == deep[36].line and "36" in found.message + + +def test_a_list_that_goes_on_from_a_sibling_with_a_fixed_start_is_a_warning_not_a_restart() -> None: + _, _, diagnostics = planned("# A\n\n- a\n 1. x\n\n Between.\n\n - w\n\n 1. z\n") + assert [(d.rule, d.level) for d in diagnostics] == [ + ("render-docx-static-restart", "info"), ("render-docx-shared-counter", "warning")] + assert "Restart at 1" not in diagnostics[1].message + + +# -- Paragraphs ---------------------------------------------------------------------- + + +def test_numbered_paragraphs_have_a_num_per_section_that_starts_at_one() -> None: + body = "# A\n\nOne.\n\nTwo.\n\n# B\n\nThree.\n" + tree, plan, diagnostics = planned(body, style="continental") + paragraphs = [u for u in units(tree) if isinstance(u, Paragraph)] + assert [(plan.units[p.unit].num_id, plan.units[p.unit].ilvl, plan.units[p.unit].label) for p in paragraphs] == [ + (2, 0, "(1)"), (2, 0, "(2)"), (3, 0, "(1)")] + assert plan.abstract_nums[1].kind == "paragraphs" and plan.abstract_nums[1].levels[0].lvl_text == "(%1)" + assert diagnostics == [] + + +def test_paragraphs_are_not_numbered_unless_the_style_numbers_them() -> None: + _, plan, _ = planned("# A\n\nOne.\n") + assert plan.units["u1"].num_id is None and plan.units["u1"].count is None and plan.units["u1"].label == "" + + +# -- Labels that need the section ----------------------------------------------------- + + +# -- Labels that need the section: on the heading list (§13.2) -------------------------------------------------- + +CONTINENTAL_ITEMS = "# A\n\n- x\n - y\n - z\n - w\n- v\n\n## B\n\n- u\n" + + +def test_an_item_whose_label_needs_the_section_rides_the_heading_list_one_level_below_the_section() -> None: + tree, plan, diagnostics = planned(CONTINENTAL_ITEMS, style="continental") + x, y, z, w, v, u = items(tree) + # the heading num, at the level after the section's (%1.%2 after "1."), one num for the list with an override + assert shown(plan, x, v) == [(2, 1, "1.1"), (2, 1, "1.2")] + assert plan.nums[1].abstract_id == plan.heading_num_id - 1 == 0 and plan.nums[1].start_overrides == ((1, 1),) + assert [a.kind for a in plan.abstract_nums] == ["headings"] # no abstractNum of the items' own is made + assert plan.units[x.unit].cause == () and not plan.units[x.unit].alt + assert [x.label, v.label, u.label] == ["1.1", "1.2", "1.1.1"] + # an item under a level-2 section is on the level after that: %1.%2.%3 + assert shown(plan, u) == [(4, 2, "1.1.1")] + assert plan.abstract_nums[0].levels[2].lvl_text == "%1.%2.%3" + # the items are no headings: no heading style and a name of their own for their indent + assert plan.units[x.unit].ind == (plan.abstract_nums[0].levels[1].left,) * 2 + + +def test_the_heading_levels_an_item_stands_on_are_not_changed_by_it() -> None: + _, plain, _ = planned("# A\n\n## B\n\n### C\n", style="continental") + _, with_items, _ = planned(CONTINENTAL_ITEMS, style="continental") + assert plain.abstract_nums[0].levels[:5] == with_items.abstract_nums[0].levels[:5] + + +def test_a_sub_heading_after_the_items_of_its_section_is_restarted_where_the_resolver_is() -> None: + tree, plan, diagnostics = planned(CONTINENTAL_ITEMS, style="continental") + b = tree.sections[1] + # Word's counter of the level is shared by the sub-heading and the items: 1.3 where the resolver has 1.1 again + assert (b.label, plan.units[b.unit].label, plan.units[b.unit].cause) == ("1.1", "1.1", ("resync",)) + assert plan.units[b.unit].explicit # a num of its own: the paragraph has to carry it + assert plan.units[b.unit].explicit # a num of its own: the paragraph must carry it + assert plan.nums[plan.units[b.unit].num_id - 1].start_overrides == ((1, 1),) + [found] = [d for d in diagnostics if d.rule == "render-docx-shared-counter"] + assert found.level == "warning" and found.line == b.line + assert "every output format" in found.message and "designation 1.1" in found.message + assert "render-docx-number-mismatch" not in rules(diagnostics) and "render-docx-section-label" not in rules(diagnostics) + + +def test_a_resync_is_reported_once_with_the_number_of_places_that_do_the_same() -> None: + body = "".join(f"# S{i}\n\n- a\n\n## T{i}\n\n" for i in range(5)) + _, _, diagnostics = planned(body, style="continental") + [found] = [d for d in diagnostics if d.rule == "render-docx-shared-counter"] + assert found.message.endswith("4 more units do the same.") + + +def test_a_list_nested_in_an_item_on_the_heading_list_rides_the_levels_reserved_for_it() -> None: + tree, plan, diagnostics = planned(CONTINENTAL_ITEMS, style="continental") + x, y, z, w, v, u = items(tree) + # depth 1 and 2: levels 6 and 7 of the heading abstractNum, the num of the parent and no override + assert shown(plan, y, z, w) == [(2, 5, "a)"), (2, 5, "b)"), (2, 6, "i)")] + assert [plan.units[i.unit].cause for i in (y, w)] == [("reserved-level",), ("reserved-level",)] + levels = plan.abstract_nums[0].levels + assert [(levels[i].num_fmt, levels[i].lvl_text) for i in (5, 6)] == [("lowerLetter", "%6)"), ("lowerRoman", "%7)")] + assert [(levels[i].num_fmt, levels[i].lvl_text) for i in (7, 8)] == [("decimal", "%8"), ("decimal", "%9")] # unused + assert len(plan.nums) == 4 and plan.nums[1].start_overrides == ((1, 1),) + # one step in from the text of the item it is nested in, as the items of any list are + assert plan.units[y.unit].ind == (plan.units[x.unit].ind[0] + 567, 567) + assert plan.units[w.unit].ind == (plan.units[x.unit].ind[0] + 1134, 567) + # the item after the nested ones goes on, and a new item's list restarts by structure + assert plan.units[v.unit].count == 2 and diagnostics[0].rule == "render-docx-shared-counter" + + +def test_a_nested_list_under_a_new_item_on_the_heading_list_restarts_by_structure() -> None: + tree, plan, diagnostics = planned("# A\n\n- a\n - x\n- b\n - y\n - z\n", style="continental") + _, x, _, y, z = items(tree) + assert shown(plan, x, y, z) == [(2, 5, "a)"), (2, 5, "a)"), (2, 5, "b)")] and diagnostics == [] + assert len(plan.nums) == 2 and plan.units[y.unit].cause == ("reserved-level",) + + +def test_siblings_nested_in_one_item_on_the_heading_list_go_on_counting() -> None: + body = "# A\n\n- a\n - x\n - y\n\n Between.\n\n - z\n" + tree, plan, diagnostics = planned(body, style="continental") + assert [(plan.units[i.unit].label, plan.units[i.unit].num_id) for i in items(tree)[1:]] == [ + ("a)", 2), ("b)", 2), ("c)", 2)] + assert len(plan.nums) == 2 and diagnostics == [] + + +def test_a_reserved_level_is_defined_on_first_use_and_only_when_it_is_used() -> None: + _, plain, _ = planned("# A\n\n- x\n", style="continental") + assert [(level.lvl_text, level.num_fmt) for level in plain.abstract_nums[0].levels[5:]] == [ + (f"%{k}", "decimal") for k in range(6, 10)] + _, nested, _ = planned("# A\n\n- x\n - y\n - z\n", style="continental") + assert [level.lvl_text for level in nested.abstract_nums[0].levels[5:]] == ["%6)", "%7)", "%8", "%9"] + + +def test_a_reserved_level_is_shared_by_every_list_that_reads_as_it_does() -> None: + tree, plan, diagnostics = planned("# A\n\n- x\n - y\n\n# B\n\n- z\n - w\n", style="continental") + _, y, _, w = items(tree) + assert (plan.units[y.unit].ilvl, plan.units[w.unit].ilvl) == (5, 5) and diagnostics == [] + assert [level.lvl_text for level in plan.abstract_nums[0].levels[5:7]] == ["%6)", "%7"] + + +def test_a_nested_label_that_shows_the_section_does_not_ride_the_reserved_levels() -> None: + # depth 3 of the continental style is the first level's label again, {section}.{n} + body = "# A\n\n- x\n - y\n - z\n - w\n - v\n" + tree, plan, diagnostics = planned(body, style="continental") + x, y, z, w, v = items(tree) + assert [plan.units[i.unit].ilvl for i in (x, y, z)] == [1, 5, 6] + assert plan.units[w.unit].ilvl == 3 and plan.units[w.unit].cause == ("restart-section", "label-fallback") + assert plan.units[w.unit].label == "1" and w.label == "1.1" + # it is one step in from the item it is nested in, on whatever list it is + assert plan.units[w.unit].ind == (plan.units[z.unit].ind[0] + 567, 567) + # the list after it is on the family's abstractNum too, under a parent that is not on the heading list + assert [a.kind for a in plan.abstract_nums] == ["headings", "enumerated"] + assert plan.units[v.unit].ilvl == 4 and plan.units[v.unit].cause == () + assert [(d.rule, d.level) for d in diagnostics] == [ + ("render-docx-static-restart", "info"), ("render-docx-section-label", "warning")] + + +FIVE_LEVELS = {"enumeration.levels": [ + {"counter": "decimal", "label": "{section}.{n}", "ref": ".{n}"}, + {"counter": "lower-alpha", "label": "{n})", "ref": "({n})"}, + {"counter": "lower-roman", "label": "{n})", "ref": "({n})"}, + {"counter": "upper-alpha", "label": "({n})", "ref": "({n})"}, + {"counter": "decimal", "label": "[{n}]", "ref": "[{n}]"}, + {"counter": "lower-alpha", "label": "{n}.", "ref": "{n}"}]} + + +def test_a_chain_of_four_lists_in_a_clause_rides_all_four_reserved_levels_and_the_fifth_falls_back() -> None: + body = "# A\n\n" + "".join(" " * i + f"- level {i}\n" for i in range(6)) + tree, plan, diagnostics = planned(body, style="continental", overrides=FIVE_LEVELS) + found = items(tree) + assert [plan.units[i.unit].ilvl for i in found[:5]] == [1, 5, 6, 7, 8] + assert [plan.units[i.unit].label for i in found[:5]] == ["1.1", "a)", "i)", "(A)", "[1]"] + assert [i.label for i in found[:5]] == ["1.1", "a)", "i)", "(A)", "[1]"] + assert len({plan.units[i.unit].num_id for i in found[:5]}) == 1 # the clause's num, no override of their own + assert [level.lvl_text for level in plan.abstract_nums[0].levels[5:]] == ["%6)", "%7)", "(%8)", "[%9]"] + assert [plan.units[i.unit].cause for i in found[1:5]] == [("reserved-level",)] * 4 + # depth 5 is past the reserved levels: the family's own abstractNum, restarted by a fixed setting + assert plan.units[found[5].unit].cause == ("restart-section",) and plan.units[found[5].unit].ilvl == 5 + assert rules(diagnostics) == {"render-docx-static-restart"} + + +def test_a_nested_list_before_a_level_5_section_falls_back_without_a_section_label_finding() -> None: + # The level after the fifth heading level is kept for the units of the level-5 section, so the nested list of an + # earlier section cannot have it: it keeps the family's abstractNum, restarted by a fixed setting (information), + # and its label, which has no section in it, is not reported. + body = "# A\n\n- x\n - y\n - z\n\n## B\n\n### C\n\n#### D\n\n##### E\n\n- five\n" + tree, plan, diagnostics = planned(body, style="continental") + x, y, z, five = items(tree) + assert plan.units[x.unit].ilvl == 1 and plan.units[five.unit].ilvl == 5 + assert plan.units[y.unit].cause == ("restart-section",) and plan.units[y.unit].ilvl == 1 + assert plan.units[z.unit].cause == () and plan.units[z.unit].ilvl == 2 + assert rules(diagnostics) == {"render-docx-static-restart", "render-docx-shared-counter"} # B follows the clause + assert plan.abstract_nums[0].levels[5].lvl_text == "%1.%2.%3.%4.%5.%6" + # without the level-5 section the same lists ride the reserved levels + tree, plan, diagnostics = planned("# A\n\n- x\n - y\n - z\n", style="continental") + assert [plan.units[i.unit].ilvl for i in items(tree)] == [1, 5, 6] and diagnostics == [] + + + +def test_an_ordered_list_nested_in_an_item_on_the_heading_list_keeps_its_own_numbering() -> None: + tree, plan, diagnostics = planned("# A\n\n- x\n 1. y\n 1. z\n", style="continental") + x, y, z = items(tree) + assert [a.kind for a in plan.abstract_nums] == ["headings", "renumber"] + assert shown(plan, y, z) == [(3, 1, "1."), (3, 1, "2.")] and plan.units[y.unit].cause == ("restart-section",) + assert plan.units[y.unit].ind == (plan.units[x.unit].ind[0] + 567, 567) + [found] = diagnostics + assert (found.rule, found.level) == ("render-docx-static-restart", "info") and "carries the section" in found.message + + +def test_the_level_after_the_fifth_is_kept_for_level_5_sections_when_the_document_has_one() -> None: + body = "# A\n\n## B\n\n### C\n\n#### D\n\n##### E\n\n- five\n - nested\n- six\n" + tree, plan, diagnostics = planned(body, style="continental") + five, nested, six = items(tree) + assert shown(plan, five, six) == [(2, 5, "1.1.1.1.1.1"), (2, 5, "1.1.1.1.1.2")] + assert plan.abstract_nums[0].levels[5].lvl_text == "%1.%2.%3.%4.%5.%6" + # the nested list falls back: its slot is the level-5 section's + assert plan.units[nested.unit].cause == ("restart-section",) and plan.units[nested.unit].ilvl == 1 + # a document without a level-5 section lets the nested list have it + _, other, _ = planned("# A\n\n- x\n - y\n", style="continental") + assert other.abstract_nums[0].levels[5].lvl_text == "%6)" + + +def test_a_level_a_different_label_has_taken_is_not_shared() -> None: + # the numbered paragraphs under a level-5 section need another text on the level after the section's than the + # items: the second kind falls back, and says so + body = "# A\n\n## B\n\n### C\n\n#### D\n\n##### E\n\n- five\n\nPara.\n" + levels = {"paragraphs.numbered": True, "paragraphs.format": {"counter": "decimal", "label": "{section}-{n}", + "ref": "-{n}"}} + tree, plan, diagnostics = planned(body, style="continental", overrides=levels) + [paragraph] = [u for u in units(tree) if isinstance(u, Paragraph)] + assert plan.units[paragraph.unit].cause == ("label-fallback",) and plan.units[paragraph.unit].label == "-1" + assert plan.units[items(tree)[0].unit].ilvl == 5 + assert rules(diagnostics) == {"render-docx-section-label"} + + +def test_a_label_the_heading_list_cannot_show_is_written_without_the_section_and_reported() -> None: + levels = [{"counter": "decimal", "label": "{section}.{n}", "ref": ".{n}"}, + {"counter": "lower-alpha", "label": "{n})", "ref": "({n})"}] + tree, plan, diagnostics = planned("# A\n\n- x\n - y\n", style="outline", overrides={"enumeration.levels": levels}) + x, y = items(tree) + assert (x.label, plan.units[x.unit].label, plan.units[x.unit].cause) == ("I.1", "1", ("label-fallback",)) + assert [a.kind for a in plan.abstract_nums] == ["headings", "enumerated"] and plan.units[x.unit].ind is None + [found] = diagnostics + assert found.rule == "render-docx-section-label" and found.level == "warning" and "'1'" in found.message + assert found.line == x.line + + +def test_under_the_none_scheme_the_label_that_needs_the_section_is_written_without_it() -> None: + tree, plan, diagnostics = planned("# A\n\n- x\n", style="continental", overrides=NONE) + [x] = items(tree) + assert (x.label, plan.units[x.unit].label, plan.units[x.unit].cause) == ("A.1", "1", ("label-fallback",)) + assert rules(diagnostics) == {"render-docx-section-label"} + + +def test_the_items_of_a_quote_do_not_ride_the_heading_list() -> None: + tree, plan, diagnostics = planned("# A\n\n> - quoted\n\n- real\n", style="continental") + quoted, real = items(tree) + assert plan.units[quoted.unit].cause == ("label-fallback",) and plan.units[quoted.unit].label == "1" + assert shown(plan, real)[0][1:] == (1, "1.1") + assert rules(diagnostics) == {"render-docx-section-label"} + + +def test_numbered_paragraphs_whose_label_needs_the_section_ride_the_heading_list_too() -> None: + tree, plan, diagnostics = planned("# A\n\nOne.\n\nTwo.\n\n## B\n\nThree.\n", overrides=PARAGRAPHS) + one, two, three = [u for u in units(tree) if isinstance(u, Paragraph)] + assert [(plan.units[p.unit].num_id, plan.units[p.unit].ilvl, plan.units[p.unit].label) for p in (one, two, three)] \ + == [(2, 1, "1.1"), (2, 1, "1.2"), (4, 2, "1.1.1")] + assert [a.kind for a in plan.abstract_nums] == ["headings"] + # B's own number goes on from Word's counter of the level unless restarted + assert plan.units["s2"].cause == ("resync",) and rules(diagnostics) == {"render-docx-shared-counter"} + + +def test_items_in_the_middle_of_numbered_paragraphs_share_their_counter_and_the_paragraph_after_is_restarted() -> None: + # continental's paragraphs are (1), a list of their own; made {section}.{n} they share the level with the items + levels = {"paragraphs.numbered": True, "paragraphs.format": {"counter": "decimal", "label": "{section}.{n}", + "ref": ".{n}"}} + tree, plan, diagnostics = planned("# A\n\nOne.\n\n- x\n- y\n\nTwo.\n", style="continental", overrides=levels) + one, x, y, two = units(tree)[1:] + assert [(plan.units[u.unit].label, plan.units[u.unit].cause) for u in (one, x, y, two)] == [ + ("1.1", ()), ("1.1", ()), ("1.2", ()), ("1.2", ("resync",))] # the renderer counts them apart + assert plan.nums[plan.units[two.unit].num_id - 1].start_overrides == ((1, 2),) + assert rules(diagnostics) == {"render-docx-shared-counter"} + + +def test_an_item_nested_under_a_foreign_parent_reports_a_section_label_once() -> None: + levels = [{"counter": "lower-alpha", "label": "({n})", "ref": "({n})"}, + {"counter": "decimal", "label": "{path}", "ref": "({n})"}] + tree, plan, diagnostics = planned("# A\n\n1. a\n - x\n", overrides={"enumeration.levels": levels}) + _, nested = items(tree) + assert plan.units[nested.unit].cause == ("restart-family", "label-fallback") + assert [d.rule for d in diagnostics].count("render-docx-section-label") == 1 + + +def test_a_quote_inside_a_preamble_item_with_a_path_label_does_not_show_the_counter_of_a_list_it_is_not_in() -> None: + # the quote is built on no item, so its path is its own counter; the enumerated level would show the counter of + # whatever the level above counted last ((a)(1)), so the list takes the detached levels, which draw it as it is + levels = [{"counter": "lower-alpha", "label": "({n})", "ref": "({n})"}, + {"counter": "decimal", "label": "{path}", "ref": "({n})"}] + tree, plan, diagnostics = planned("- a\n\n > - quoted\n\n# A\n", overrides={"enumeration.levels": levels}) + _, quoted = items(tree) + assert quoted.parent_unit is None and quoted.label == "(1)" + assert plan.units[quoted.unit].cause == () and plan.units[quoted.unit].label == "(1)" + assert [a.kind for a in plan.abstract_nums] == ["headings", "enumerated", "detached"] + assert diagnostics == [] + + +def test_a_preamble_item_needs_no_section_and_is_not_reported() -> None: + _, plan, diagnostics = planned("- x\n\n# A\n", style="continental") + assert plan.units["u1"].label == "1" and diagnostics == [] + + +# -- Labels that show the path: the reserved levels of the heading list, and the detached levels ---------------- + +PATH_ITEMS = {"style": "continental", "overrides": {"enumeration.levels": [ + {"counter": "decimal", "label": "{section}.{n}", "ref": ".{n}"}, + {"counter": "lower-alpha", "label": "{path}", "ref": "({n})"}, + {"counter": "lower-roman", "label": "{path}", "ref": "({n})"}, + {"counter": "upper-alpha", "label": "{path}", "ref": "({n})"}, + {"counter": "decimal", "label": "{path}", "ref": "({n})"}, + {"counter": "lower-alpha", "label": "{path}", "ref": "({n})"}]}} + + +def refs_of(tree: RenderTree, plan: DocxPlan) -> list[tuple[str, str]]: + """Each reference's text and its fields, as ``show`` writes them.""" + return [(i.text, show(plan, plan.refs[i.target_unit].segments)) + for i in iter_tree_inlines(tree) if isinstance(i, CrossRef)] + + +def test_a_path_label_nested_in_a_clause_rides_the_reserved_levels_with_the_counters_of_its_chain() -> None: + body = ("# A {#a}\n\n- one {#i1}\n - two {#i2}\n - three {#i3}\n- uno {#j1}\n - dos {#j2}\n\n" + "Refs: {{ref: i1}} {{ref: i2}} {{ref: i3}} {{ref: j2}}\n") + tree, plan, diagnostics = planned(body, **PATH_ITEMS) + one, two, three, uno, dos = items(tree)[:5] + # the label as the renderer writes it, drawn by Word: the section's counter and the clause's are levels of the list + assert [(plan.units[i.unit].label, i.label) for i in (one, two, three, uno, dos)] == [ + (label, label) for label in ("1.1", "1.1(a)", "1.1(a)(i)", "1.2", "1.2(a)")] + assert [plan.units[i.unit].ilvl for i in (one, two, three)] == [1, 5, 6] + assert [level.lvl_text for level in plan.abstract_nums[0].levels[5:7]] == ["%1.%2(%6)", "%1.%2(%6)(%7)"] + assert [plan.units[i.unit].cause for i in (two, three, dos)] == [("reserved-level",)] * 3 + # the clause's num: no override for the nested lists, which restart by structure under every new clause + assert len({plan.units[i.unit].num_id for i in (one, two, three, uno, dos)}) == 1 + assert [n for n in plan.nums if n.abstract_id == 0] == [plan.nums[0], plan.nums[1]] # the heading num, the clause's + assert diagnostics == [] + # a field for the label is the whole of the reference + assert refs_of(tree, plan) == [("1.1", "REF i1"), ("1.1(a)", "REF i2"), ("1.1(a)(i)", "REF i3"), + ("1.2(a)", "REF j2")] + + +def test_a_path_list_in_a_section_of_another_level_cannot_share_the_reserved_level_and_is_detached() -> None: + # under a level-2 section the clause is on the level after it: the reserved level would read %1.%2.%3(%6), which + # the first list has taken as %1.%2(%6) + body = ("# A {#a}\n\n- one {#i1}\n - two {#i2}\n\n## B {#b}\n\n- bee {#k1}\n - kay {#k2}\n - kay two {#k3}\n\n" + "Refs: {{ref: i2}} {{ref: k2}} {{ref: k3}}\n") + tree, plan, diagnostics = planned(body, **PATH_ITEMS) + _, two, _, kay, kay_two = [i for i in items(tree) if i.number is not None][:5] + assert plan.units[two.unit].cause == ("reserved-level",) and plan.units[two.unit].label == "1.1(a)" + assert [(plan.units[i.unit].label, i.label) for i in (kay, kay_two)] == [("(a)", "1.1.1(a)"), ("(b)", "1.1.1(b)")] + assert plan.units[kay.unit].cause == ("restart-section", "label-fallback") + assert [a.kind for a in plan.abstract_nums if a.kind == "detached"] == ["detached"] + assert plan.units[kay.unit].ilvl == 1 and plan.abstract_nums[plan.nums[plan.units[kay.unit].num_id - 1].abstract_id].kind == "detached" + # the label is shorter in Word, and the references to it are fields all the same + assert refs_of(tree, plan) == [("1.1(a)", "REF i2"), ("1.1.1(a)", "REF k1 + REF k2"), + ("1.1.1(b)", "REF k1 + REF k3")] + [restart] = [d for d in diagnostics if d.rule == "render-docx-static-restart"] + [lost] = [d for d in diagnostics if d.rule == "render-docx-section-label"] + assert restart.level == "info" and "carries the section" in restart.message + assert lost.level == "warning" and lost.line == kay.line + assert "shows the number of the enclosing section and item" in lost.message and "it reads '(a)'" in lost.message + assert "kept for nested lists are used up or read otherwise" in lost.message + + +def test_a_path_list_nested_past_the_reserved_levels_is_detached() -> None: + body = "# A\n\n" + "".join(" " * i + f"- level {i} {{#l{i}}}\n" for i in range(7)) + body += "\nRefs: " + " ".join(f"{{{{ref: l{i}}}}}" for i in range(7)) + "\n" + tree, plan, diagnostics = planned(body, **PATH_ITEMS) + found = items(tree)[:7] + assert [plan.units[i.unit].ilvl for i in found] == [1, 5, 6, 7, 8, 5, 6] + assert [plan.units[i.unit].label for i in found[:5]] == [i.label for i in found[:5]] + assert [level.lvl_text for level in plan.abstract_nums[0].levels[5:]] == [ + "%1.%2(%6)", "%1.%2(%6)(%7)", "%1.%2(%6)(%7)(%8)", "%1.%2(%6)(%7)(%8)(%9)"] + # depth 5 and the list under it: the detached levels, one num for both, restarted by structure inside + assert [plan.units[i.unit].cause for i in found[5:]] == [("restart-section", "label-fallback"), ("label-fallback",)] + assert plan.units[found[5].unit].num_id == plan.units[found[6].unit].num_id + assert [plan.units[i.unit].label for i in found[5:]] == ["(a)", "1"] + assert [text for text, _ in refs_of(tree, plan)][-2:] == ["1.1(a)(i)(A)(1)(a)", "1.1(a)(i)(A)(1)(a).1"] + assert all(plan.refs[i.target_unit].exact and not plan.refs[i.target_unit].fixed + for i in iter_tree_inlines(tree) if isinstance(i, CrossRef)) + assert [d.rule for d in diagnostics] == ["render-docx-static-restart", "render-docx-section-label", + "render-docx-section-label"] + + +ORDERED_PARENT = {"overrides": {"enumeration.levels": [ + {"counter": "lower-alpha", "label": "({n})", "ref": "({n})"}, + {"counter": "decimal", "label": "({n})", "ref": "({n})"}, + {"counter": "lower-roman", "label": "{n}", "ref": "({n})"}, + {"counter": "upper-alpha", "label": "{path}", "ref": "({n})"}]}} + + +def test_a_path_list_below_an_ordered_item_is_detached_and_its_references_are_fields() -> None: + body = ("# A {#a}\n\n- one {#i1}\n 1. ordered {#o}\n - three {#i3}\n - four {#i4}\n - four b {#i5}\n" + " - three b {#i6}\n - four c {#i7}\n\nRefs: {{ref: i3}} {{ref: i4}} {{ref: i5}} {{ref: i7}}\n") + tree, plan, diagnostics = planned(body, **ORDERED_PARENT) + found = {i.unit: i for i in items(tree)} + i4, i5, i7 = (found[u] for u in ("u4", "u5", "u7")) + assert [i.label for i in (i4, i5, i7)] == ["1(a)(1)(i)(A)", "1(a)(1)(i)(B)", "1(a)(1)(ii)(A)"] + # Word draws the level's own counter: the enumerated level would read the (never used) levels above as 0 or as + # the counter of another list + assert [plan.units[i.unit].label for i in (i4, i5, i7)] == ["(A)", "(B)", "(A)"] + assert [plan.units[i.unit].cause for i in (i4, i5, i7)] == [ + ("restart-family", "label-fallback"), ("label-fallback",), ("restart-family", "label-fallback")] + assert plan.units[i4.unit].num_id == plan.units[i5.unit].num_id != plan.units[i7.unit].num_id + assert [(a.kind, a.levels[3].lvl_text) for a in plan.abstract_nums][-1] == ("detached", "(%4)") + assert [n.start_overrides for n in plan.nums if n.abstract_id == plan.abstract_nums[-1].id] == [((3, 1),)] * 2 + assert refs_of(tree, plan) == [ + ("1(a)(1)(i)", 'REF a + REF i1 + "(" + REF o + ")(" + REF i3 + ")"'), + ("1(a)(1)(i)(A)", 'REF a + REF i1 + "(" + REF o + ")(" + REF i3 + ")" + REF i4'), + ("1(a)(1)(i)(B)", 'REF a + REF i1 + "(" + REF o + ")(" + REF i3 + ")" + REF i5'), + ("1(a)(1)(ii)(A)", 'REF a + REF i1 + "(" + REF o + ")(" + REF i6 + ")" + REF i7')] + [lost] = [d for d in diagnostics if d.rule == "render-docx-section-label"] + assert "shows the number of the enclosing section, ordered item and item" in lost.message + assert "it reads '(A)'" in lost.message and "the list is nested in a unit that is on another list" in lost.message + assert "section's number" not in lost.message + + +def test_lists_nested_in_a_detached_item_stay_on_the_detached_levels_and_restart_by_structure() -> None: + body = ("# A\n\n- one\n 1. ordered\n - three\n - four\n - five\n - five b\n" + " - four b\n - five c\n") + tree, plan, diagnostics = planned(body, **ORDERED_PARENT) + four, five, five_b, four_b, five_c = items(tree)[3:8] + kinds = {a.id: a.kind for a in plan.abstract_nums} + assert [kinds[plan.nums[plan.units[i.unit].num_id - 1].abstract_id] for i in (four, five, five_b, four_b, five_c)] \ + == ["detached"] * 5 + assert len({plan.units[i.unit].num_id for i in (four, five, five_b, four_b, five_c)}) == 1 # one num, no override + assert [(plan.units[i.unit].ilvl, plan.units[i.unit].label) for i in (five, five_b, five_c)] == [ + (4, "(a)"), (4, "(b)"), (4, "(a)")] # the list under the second "four" starts again, by structure + assert [plan.units[i.unit].cause for i in (five, five_b, five_c)] == [()] * 3 + + +def test_the_detached_levels_hold_each_levels_own_counter_alone_and_exist_only_when_used() -> None: + _, plain, _ = planned("# A\n\n- x\n - y\n") + assert "detached" not in [a.kind for a in plain.abstract_nums] + _, plan, _ = planned("# A\n\n- x\n 1. y\n - z\n - w\n", **ORDERED_PARENT) + [detached] = [a for a in plan.abstract_nums if a.kind == "detached"] + enumerated = next(a for a in plan.abstract_nums if a.kind == "enumerated") + assert [(level.num_fmt, level.lvl_text) for level in detached.levels[:4]] == [ + ("lowerLetter", "(%1)"), ("decimal", "(%2)"), ("lowerRoman", "%3"), ("upperLetter", "(%4)")] + assert enumerated.levels[3].lvl_text == "(%1)(%2)(%3)(%4)" # the enumerated list shows the whole path + assert detached.multi_level == "hybridMultilevel" and len(detached.levels) == 9 + + +def test_a_label_whose_counters_are_not_the_end_of_its_designation_keeps_the_level_that_shows_less() -> None: + # {section}.{n} at depth 3 of the continental style would show the section's counter before the item's, but the + # designation of the item ends with ".1": no field for the label could stand for it, so the label is written + # without the section (reported) and the reference is made of fields + body = "# A\n\n- x {#x}\n - y {#y}\n - z {#z}\n - w {#w}\n\nRef {{ref: w}}\n" + tree, plan, diagnostics = planned(body, style="continental") + [(text, fields)] = refs_of(tree, plan) + assert text == "1.1(a)(i).1" and fields == 'REF x + "(" + REF y + "(" + REF z + "." + REF w' + assert plan.units["u4"].cause == ("restart-section", "label-fallback") and plan.units["u4"].label == "1" + + +# -- Counters Word writes differently -------------------------------------------------- + + +def _items(n: int) -> str: + return "# A\n\n" + "".join(f"- item {i}\n" for i in range(1, n + 1)) + + +def test_letters_past_twenty_six_are_reported_once_for_the_list() -> None: + tree, plan, diagnostics = planned(_items(60)) + items = [u for u in units(tree) if isinstance(u, ListItem)] + assert plan.units[items[26].unit].label == "(aa)" and plan.units[items[27].unit].label == "(bb)" + assert items[27].label == "(ab)" + [found] = diagnostics + assert (found.rule, found.level, found.line) == ("render-docx-letter-overflow", "warning", items[27].line) + assert "'ab'" in found.message and "'bb'" in found.message + assert plan.units[items[27].unit].count == 28 + + +def test_an_overflow_that_reads_the_same_in_two_lists_is_one_finding_at_the_first() -> None: + tree, _, diagnostics = planned(_items(30) + "\nText.\n\n" + _items(30)[5:]) + [found] = diagnostics + assert found.rule == "render-docx-letter-overflow" and found.line == items(tree)[27].line + + +def test_roman_numerals_from_4000_are_reported() -> None: + levels = [{"counter": "upper-roman", "label": "{n}.", "ref": "{n}"}] + body = "".join(f"# H{i}\n\n" for i in range(1, 4)) + tree, plan, diagnostics = planned(body, overrides={"numbering.levels": levels}) + assert diagnostics == [] + levels = [{"counter": "lower-roman", "label": "({n})", "ref": "({n})"}] * 3 + many = "".join(f"- i{i}\n" for i in range(1, 4002)) + tree, plan, diagnostics = planned("# A\n\n" + many, overrides={"enumeration.levels": levels}) + items = [u for u in units(tree) if isinstance(u, ListItem)] + assert (items[3998].label, plan.units[items[3998].unit].label) == ("(mmmcmxcix)", "(mmmcmxcix)") + assert (items[3999].label, plan.units[items[3999].unit].label) == ("(4000)", "(mmmm)") + [found] = [d for d in diagnostics if d.rule == "render-docx-roman-overflow"] + assert found.line == items[3999].line and "'4000'" in found.message and "'mmmm'" in found.message + + +# -- Alternatives that share a number (§15.8) ------------------------------------------------------------------- + +SECTIONS = ("# Intro\n\n# D {#d when=forum:courts}\n\n## Sub {#s1}\n\n## Sub two {#s2}\n\n# D {#d when=forum:arb}\n\n" + "## Sub alt\n\n- q\n\n# D {#d when=forum:expert}\n\n## Sub exp\n\n## Sub exp two\n\n## Sub exp three\n\n" + "# Last\n\n## L1\n") + + +def segments_of(plan: DocxPlan, unit: str) -> str: + alt = plan.units[unit].alt + assert alt is not None + return show(plan, alt.segments) + + +def test_a_later_alternative_section_has_no_number_of_its_own_and_shows_the_holders_as_a_field() -> None: + body = "# Disputes {#d when=forum:courts}\n\n# Disputes {#d when=forum:arb}\n\n# Last\n" + tree, plan, diagnostics = planned(body, ALTERNATIVES) + assert tree.sections[1].shares_number_with == "s1" + first, second, last = (plan.units[s] for s in ("s1", "s2", "s3")) + assert (second.num_id, second.ilvl, second.explicit, second.count, second.cause) == (0, 0, True, None, ("alternative",)) + assert (second.label, second.word_ref, second.ind) == ("1.", "1", (567, 567)) + assert (first.label, last.label, last.count) == ("1.", "2.", 2) and last.num_id == first.num_id + assert segments_of(plan, "s2") == 'REF d + "."' and second.alt.suff == "tab" + segment, period = second.alt.segments + assert (segment.number, segment.hyperlink, segment.suppress_text, segment.cached) == (True, True, False, "1") + assert period == Literal_(".") and segment.bookmark == plan.units["s1"].bookmarks[0] + assert rules(diagnostics) == {"render-docx-alternative"} + + +def test_the_text_after_the_number_of_a_later_alternative_is_the_label_beyond_what_the_field_returns() -> None: + body = "# D {#d when=forum:courts}\n\n# D {#d when=forum:arb}\n" + _, plan, _ = planned(body, ALTERNATIVES, overrides={"numbering.scheme": "legal-outline"}) + assert (show(plan, plan.units["s2"].alt.segments), plan.units["s2"].label) == ('REF d + "."', "I.") + # REF \n drops periods only: a label that ends otherwise is all the field's + _, plan, _ = planned(body, ALTERNATIVES, overrides={"numbering.scheme": "mixed"}) + assert (show(plan, plan.units["s2"].alt.segments), plan.units["s2"].label) == ('REF d + "."', "1.") + _, plan, _ = planned("# A\n\n- a {#i when=forum:courts}\n- b {#i when=forum:arb}\n", ALTERNATIVES) + assert (show(plan, plan.units["u2"].alt.segments), plan.units["u2"].label) == ("REF i", "(a)") + + +def test_a_reference_to_the_shared_identifier_is_the_first_alternatives() -> None: + body = "# D {#d when=forum:courts}\n\n# D {#d when=forum:arb}\n\nSee {{ref: d}}.\n" + tree, plan, _ = planned(body, ALTERNATIVES) + assert list(plan.refs) == ["s1"] and show(plan, plan.refs["s1"].segments) == "REF d" + assert plan.units["s2"].bookmarks == () # the holder carries the bookmark, the alternative has none of its own + + +def test_the_holder_of_an_alternatives_number_has_a_bookmark_even_when_nothing_refers_to_it() -> None: + _, plan, _ = planned("# D\n\n- a {#i when=forum:courts}\n- b {#i when=forum:arb}\n", ALTERNATIVES) + [segment] = plan.units["u2"].alt.segments + assert plan.units["u1"].bookmarks == (segment.bookmark,) and segment.bookmark.startswith("ld_") + _, plan, _ = planned("# D\n\n- a {when=forum:courts}\n- b {when=forum:arb}\n", ALTERNATIVES) + assert plan.units["u2"].alt is None # no identifier: no alternative + _, plan, _ = planned("# D {when=forum:courts}\n\n# D2 {when=forum:arb}\n", ALTERNATIVES) + assert plan.units["s2"].alt is None + + +def test_the_sections_under_a_later_alternative_start_in_a_num_of_their_own() -> None: + tree, plan, diagnostics = planned(SECTIONS, ALTERNATIVES) + sub, sub_two, alt, sub_alt, alt_three, exp, exp_two, exp_three, last, l1 = ( + plan.units[s] for s in ("s3", "s4", "s5", "s6", "s7", "s8", "s9", "s10", "s11", "s12")) + assert [s.label for s in (sub, sub_two, sub_alt, exp, exp_two, exp_three, last, l1)] == [ + "2.1", "2.2", "2.1", "2.1", "2.2", "2.3", "3.", "3.1"] + assert (alt.num_id, alt_three.num_id) == (0, 0) and alt.label == alt_three.label == "2." + # both holders' numbers are the first alternative's, the field points at it + assert segments_of(plan, "s5") == segments_of(plan, "s7") == 'REF d + "."' + # the first alternative's children are on the headings' num; each later one's on a fresh num that starts at its level + assert {sub.num_id, sub_two.num_id, last.num_id} == {1} and sub_alt.num_id != 1 and exp.num_id not in (1, sub_alt.num_id) + assert (sub_alt.explicit, exp.explicit, sub.explicit) == (True, True, False) + assert [plan.nums[n - 1].start_overrides for n in (sub_alt.num_id, exp.num_id)] == [((1, 1),), ((1, 1),)] + assert exp.num_id == exp_two.num_id == exp_three.num_id + # the section after the alternatives goes back to the headings' num: it goes on from the holder's counter + assert (last.count, l1.count) == (3, 1) and l1.num_id == 1 and last.explicit is False + assert all(d.level == "info" for d in diagnostics) + + +def test_an_alternative_that_is_not_the_first_to_be_numbered_takes_no_num_for_its_children_until_they_come() -> None: + _, plan, _ = planned("# D {#d when=forum:courts}\n\n# D {#d when=forum:arb}\n\n# E\n", ALTERNATIVES) + assert [n.id for n in plan.nums] == [1] # no num made for sections that never come + + +def test_an_alternative_below_a_later_alternative_whose_number_is_live_there_is_numbered_by_word() -> None: + body = ("# D {#d when=forum:courts}\n\n## E {#e when=forum:courts}\n\n### F {#f}\n\n" + "# D {#d when=forum:arb}\n\n## E {#e when=forum:arb}\n\n### F2\n\n## G\n\n### H\n") + tree, plan, _ = planned(body, ALTERNATIVES) + assert [(plan.units[s.unit].label, plan.units[s.unit].num_id, bool(plan.units[s.unit].alt)) for s in tree.sections] == [ + ("1.", 1, False), ("1.1", 1, False), ("1.1.1", 1, False), ("1.", 0, True), ("1.1", 2, False), + ("1.1.1", 2, False), ("1.2", 2, False), ("1.2.1", 2, False)] + # the alternative's first section starts the fresh num with the number it shares: no field is needed for it + assert plan.units["s5"].explicit and plan.nums[1].start_overrides == ((1, 1),) + + +def test_an_item_that_repeats_the_number_of_the_one_before_is_numbered_by_a_field() -> None: + body = "# S\n\n- a {#i when=forum:courts}\n- b {#i when=forum:arb}\n- c\n" + tree, plan, diagnostics = planned(body, ALTERNATIVES) + a, b, c = items(tree) + assert (b.shares_number_with, plan.units[b.unit].num_id, plan.units[b.unit].label) == (a.unit, 0, "(a)") + assert (plan.units[b.unit].ilvl, plan.units[b.unit].ind, plan.units[b.unit].cause) == (0, (567, 567), ("alternative",)) + assert shown(plan, a, c) == [(2, 0, "(a)"), (2, 0, "(b)")] and plan.units[c.unit].count == 2 + assert segments_of(plan, b.unit) == "REF i" and rules(diagnostics) == {"render-docx-alternative"} + + +def test_the_children_of_a_later_alternative_item_get_a_num_of_their_own_and_it_is_reported_as_information() -> None: + body = "# S\n\n- a {#i when=forum:courts}\n - x\n- b {#i when=forum:arb}\n - y\n - z\n- c\n" + tree, plan, diagnostics = planned(body, ALTERNATIVES) + _, x, _, y, z, _ = items(tree) + assert shown(plan, x, y, z) == [(2, 1, "(i)"), (3, 1, "(i)"), (3, 1, "(ii)")] + assert plan.units[y.unit].cause == ("restart-alternative",) and plan.units[z.unit].cause == () + assert plan.nums[2].start_overrides == ((1, 1),) + found = [d for d in diagnostics if d.rule == "render-docx-static-restart"][0] + assert (found.rule, found.level, found.line) == ("render-docx-static-restart", "info", y.line) + assert "alternative" in found.message and rules(diagnostics) == {"render-docx-static-restart", "render-docx-alternative"} + + +def test_a_paragraph_that_repeats_the_number_of_the_one_before_is_numbered_by_a_field() -> None: + body = "# S\n\nOne. {#p when=forum:courts}\n\nOne arb. {#p when=forum:arb}\n\nTwo.\n" + tree, plan, diagnostics = planned(body, ALTERNATIVES, overrides=PARAGRAPHS) + one, arb, two = [u for u in units(tree) if isinstance(u, Paragraph)] + assert [(plan.units[u.unit].num_id, plan.units[u.unit].label) for u in (one, arb, two)] == [ + (2, "1.1"), (0, "1.1"), (2, "1.2")] + assert plan.units[arb.unit].ilvl == 1 and plan.units[two.unit].count == 2 + assert rules(diagnostics) == {"render-docx-alternative"} + + +def test_alternatives_on_the_heading_list_carry_the_indent_of_the_unit_not_of_the_heading() -> None: + body = "# S\n\n- a {#i when=forum:courts}\n - n\n- b {#i when=forum:arb}\n - m\n" + tree, plan, diagnostics = planned(body, ALTERNATIVES, style="continental") + a, n, b, m = items(tree) + heading_level = plan.abstract_nums[0].levels[1] + assert plan.units[b.unit].num_id == 0 and plan.units[b.unit].ilvl == 1 + assert plan.units[b.unit].ind == (heading_level.left, heading_level.hanging) == plan.units[a.unit].ind + # the children of the alternative restart on a fresh num of the heading list's reserved level + assert (plan.units[m.unit].ilvl, plan.units[m.unit].cause) == (5, ("restart-alternative", "reserved-level")) + assert plan.units[m.unit].num_id not in (plan.units[n.unit].num_id, 1) + assert plan.nums[plan.units[m.unit].num_id - 1].start_overrides == ((5, 1),) + + +def test_a_designation_below_an_alternative_is_made_of_the_holders_counters() -> None: + body = ("# D {#d when=forum:courts}\n\n- a {#a when=forum:courts}\n- b {#a when=forum:arb}\n - deep {#deep}\n\n" + "# D {#d when=forum:arb}\n\n- z {#z}\n\nSee {{ref: deep}} {{ref: z}}.\n") + tree, plan, diagnostics = planned(body, ALTERNATIVES, style="continental") + texts = {i.target_unit: i.text for i in iter_tree_inlines(tree) if isinstance(i, CrossRef)} + assert [(texts[t], show(plan, plan.refs[t].segments)) for t in plan.refs] == [ + ("1.1(a)", 'REF a + "(" + REF deep'), ("1.1", "REF z")] + assert all(r.exact and not r.fixed for r in plan.refs.values()) + assert "render-docx-reference" not in rules(diagnostics) + + +def test_a_unit_that_shares_a_number_but_finds_another_count_in_the_level_is_restarted_not_made_an_alternative() -> None: + # Two clauses of the later alternative's sub-section counted the level of the heading list between: the number the + # sub-sub-section shares is not the one the level holds, so Word's counter is restarted at it instead of a field + # standing for a unit that is not where it is. + body = ("# D {#d when=forum:courts}\n\n## Sub {#s}\n\n### Deeper {#t}\n\n# D {#d when=forum:arb}\n\n" + "## Sub alt\n\n- clause one\n- clause two\n\n### Deeper alt\n") + tree, plan, diagnostics = planned(body, ALTERNATIVES, style="continental") + deeper = tree.sections[5] + assert deeper.shares_number_with == "s3" + mine = plan.units[deeper.unit] + assert (mine.alt, mine.cause, mine.label, mine.count, mine.explicit) == (None, ("resync",), "1.1.1", 1, True) + assert mine.num_id not in (0, None) and plan.nums[mine.num_id - 1].start_overrides == ((2, 1),) + assert "render-docx-shared-counter" in rules(diagnostics) + + +def test_the_holder_is_the_unit_that_holds_the_level_now_not_always_the_first_alternative() -> None: + # A clause on the same level of the heading list counted between the two paragraphs that share a number: the level + # holds the clause's 1.1 now, and the later alternative shows that (the designations below it are made of the + # counters the level holds), so it refers to the clause. Nothing is off: the text is the same. + levels = {"paragraphs.numbered": True, "paragraphs.format": {"counter": "decimal", "label": "{section}.{n}", + "ref": ".{n}"}} + body = "# S\n\nPara. {#p when=forum:courts}\n\n- clause {#i}\n\nPara arb. {#p when=forum:arb}\n\nAfter.\n" + tree, plan, diagnostics = planned(body, ALTERNATIVES, style="continental", overrides=levels) + first, clause, later, after = [u for u in units(tree)][1:] + assert plan.units[later.unit].num_id == 0 and plan.units[later.unit].label == "1.1" + assert plan.units[later.unit].alt.segments[0].bookmark == plan.units[clause.unit].bookmarks[0] + assert plan.units[first.unit].bookmarks == ("ld_p",) and plan.units[after.unit].label == "1.2" + assert "render-docx-number-mismatch" not in rules(diagnostics) + + +def test_the_alternatives_of_a_document_are_one_information_naming_the_first() -> None: + body = ("# D {#d when=forum:courts}\n\n# D {#d when=forum:arb}\n\n# E {#e when=forum:courts}\n\n" + "# E {#e when=forum:arb}\n\n# E {#e when=forum:expert}\n") + tree, plan, diagnostics = planned(body, ALTERNATIVES) + [found] = [d for d in diagnostics if d.rule == "render-docx-alternative"] + assert found.level == "info" and found.line == tree.sections[0].line + assert "3 alternative(s)" in found.message and "'1.'" in found.message and "deleting that paragraph" in found.message + assert f"line {tree.sections[0].line}" in found.message and "condition values" in found.message + _, _, diagnostics = planned("# A\n\n# B\n", ALTERNATIVES) + assert "render-docx-alternative" not in rules(diagnostics) + + +def test_the_message_about_a_restarted_unit_says_what_an_edit_does_to_it() -> None: + _, _, diagnostics = planned(CONTINENTAL_ITEMS, style="continental") + [heading] = [d for d in diagnostics if d.rule == "render-docx-shared-counter"] + assert "if it is deleted in Word the headings after it read higher" in heading.message + levels = {"paragraphs.numbered": True, "paragraphs.format": {"counter": "decimal", "label": "{section}.{n}", + "ref": ".{n}"}} + _, _, diagnostics = planned("# A\n\nOne.\n\n- x\n- y\n\nTwo.\n", style="continental", overrides=levels) + [paragraph] = [d for d in diagnostics if d.rule == "render-docx-shared-counter"] + assert "inserted or deleted before it in Word does not move it" in paragraph.message + + +def test_the_contents_show_the_number_the_alternative_shares() -> None: + body = "# D {#d when=forum:courts}\n\n## S\n\n# D {#d when=forum:arb}\n\n## S2\n\n# Last\n" + _, plan, _ = planned(body, ALTERNATIVES, overrides={"contents.enabled": True, "contents.depth": 2}) + assert [(e.unit, e.number, e.sep) for e in plan.toc] == [ + ("s1", "1.", "\t"), ("s2", "1.1", "\t"), ("s3", "1.", "\t"), ("s4", "1.1", "\t"), ("s5", "2.", "\t")] + assert plan.units["s3"].toc_bookmark == plan.toc[2].bookmark + + +def test_the_items_on_the_heading_list_are_not_in_the_contents() -> None: + body = "# A\n\n- x\n - y\n\n## B\n\n- z\n" + tree, plan, _ = planned(body, style="continental", overrides={"contents.enabled": True, "contents.depth": 5}) + assert [e.unit for e in plan.toc] == ["s1", "s2"] + assert all(plan.units[i.unit].toc_bookmark is None for i in items(tree)) + + +def test_an_alternative_that_starts_a_section_with_its_own_shared_number_repeats_it_by_a_live_counter() -> None: + # the children of a later alternative section repeat the numbers of the earlier one's: they are numbered by Word + # (on the fresh num) and are not alternatives themselves + tree, plan, _ = planned(SECTIONS, ALTERNATIVES) + for unit in ("s6", "s8", "s9"): + assert tree.sections[int(unit[1:]) - 1].shares_number_with in ("s3", "s4", None) + assert plan.units[unit].count is not None and not plan.units[unit].alt + + +def test_drift_is_reported_once_for_a_run_of_units_counted_off_by_the_same_amount() -> None: + # not a finding the planner makes for a document it can place (a unit Word would number differently is restarted + # or made a field), but the catch-all that tells of a bug in the placement: it groups runs + tree, _, _ = planned("# A\n\n" + "".join(f"- i{n}\n" for n in range(6))) + planner = _Planner(tree, None) + nodes = items(tree) + for node in nodes: + step = Step(node.number + 1, "", "", "", ()) + planner.check(node, "list item", step, "decimal", ("list", "u1"), (), (0, 0)) + for drift in planner.drifts: + planner.report_drift(drift) + [found] = planner.diagnostics + assert found.rule == "render-docx-number-mismatch" and found.line == nodes[0].line + assert "1 later than the renderer" in found.message and "the following 5" in found.message + + +def test_a_different_drift_is_a_finding_of_its_own() -> None: + tree, _, _ = planned("# A\n\n" + "".join(f"- i{n}\n" for n in range(4))) + planner = _Planner(tree, None) + for node, delta in zip(items(tree), (1, 1, 2, 2), strict=True): + planner.check(node, "list item", Step(node.number + delta, "", "", "", ()), "decimal", ("list", "u1"), (), (0, 0)) + for drift in planner.drifts: + planner.report_drift(drift) + assert len(planner.diagnostics) == 2 and "1 later" in planner.diagnostics[0].message + assert "2 later" in planner.diagnostics[1].message + + +# -- Diagnostics: lines, files, order, repeats -------------------------------------------- + + +def test_diagnostics_name_the_file_and_the_line_and_come_in_line_order() -> None: + result = render(FRONT + "# A\n\n- x\n 1. y\n\n- p\n- q\n" + "- r\n" * 30, format="text") + plan, diagnostics = plan_docx(result.tree, file="deal.lgd") + assert [(d.rule, d.file) for d in diagnostics] == [ + ("render-docx-static-restart", "deal.lgd"), ("render-docx-letter-overflow", "deal.lgd")] + lines = [d.line for d in diagnostics] + assert lines == sorted(lines) and None not in lines + _, anonymous = plan_docx(result.tree) + assert {d.file for d in anonymous} == {""} + + +def test_the_same_finding_is_reported_once() -> None: + _, _, diagnostics = planned("# A\n\n- a\n 1. x\n- b\n 1. y\n- c\n 1. z\n") + assert [d.rule for d in diagnostics] == ["render-docx-static-restart"] + + +def test_untrusted_text_in_a_message_is_escaped() -> None: + overrides = {"enumeration.levels": [{"counter": "lower-alpha", "label": "{section}\u0007{n}", "ref": "({n})"}]} + _, _, diagnostics = planned("# A\n\n- x\n", overrides=overrides) + [found] = diagnostics + assert found.rule == "render-docx-section-label" and "\u0007" not in found.message + + +# -- Determinism and the plan's data ------------------------------------------------------- + + +def test_the_same_tree_gives_an_equal_plan_and_the_plan_is_frozen() -> None: + result = render(FRONT + NESTED, format="text", style="continental") + first = plan_docx(result.tree) + again = plan_docx(result.tree) + assert first[0] == again[0] and first[1] == again[1] + with pytest.raises(AttributeError): + first[0].heading_num_id = 5 # type: ignore[misc] + assert list(first[0].units) == [u.unit for u in units(result.tree)] + assert first[0].refs == {} and first[0].toc == () and first[0].anchors == {"a": "ld_a"} + + +def test_unit_plans_carry_the_whole_plan_for_a_unit() -> None: + _, plan, _ = planned("# A\n\n- x\n") + unit = plan.units["u1"] + assert (unit.kind, unit.ind, unit.alt, unit.bookmarks, unit.toc_bookmark) == ("item", None, None, (), None) + assert plan.units["s1"].kind == "section" + + +# -- Style checks (ADR 0006) ---------------------------------------------------------------- + + +def test_formats_of_a_style_are_the_formats_in_the_tree() -> None: + for settings in VARIANTS.values(): + result = render(FRONT + "# A\n", format="text", **settings) + assert formats_of(result.style) == result.tree.formats + + +def test_the_built_in_styles_are_fine_for_docx() -> None: + for name in ("default", "outline", "continental"): + assert docx_style_errors(load_style(name)) == [] + + +@pytest.mark.parametrize(("key", "value"), [ + ("numbering.levels", [{"counter": "decimal", "label": "%1 {n}", "ref": "{n}"}]), + ("numbering.levels", [{"counter": "decimal", "label": "%{n}", "ref": "{n}"}]), + ("numbering.levels", [{"counter": "decimal", "label": "{n}", "ref": "%2{n}"}]), + ("enumeration.levels", [{"counter": "decimal", "label": "{n}%9", "ref": "{n}"}]), + ("paragraphs.format", {"counter": "decimal", "label": "%3{n}", "ref": "{n}"}), +]) +def test_a_percent_that_lvl_text_cannot_write_is_a_style_error(key: str, value: object) -> None: + overrides = {key: value, "paragraphs.numbered": True} + style = load_style(overrides=overrides) + problems = docx_style_errors(style) + assert len(problems) == 1 and key.split(".")[0] in problems[0] and "'%'" in problems[0] + with pytest.raises(StyleError) as caught: # the DOCX settings step refuses it before anything is rendered + docx_settings(style) + assert caught.value.problems == problems + + +@pytest.mark.parametrize(("levels", "setting"), [ + # a ref ending in '%' meets the counter that starts the next level's ref once the label is put together + ([{"counter": "decimal", "label": "{path}", "ref": "{n}%"}, {"counter": "decimal", "label": "{path}", "ref": "{n}"}], + "numbering.levels"), + ([{"counter": "decimal", "label": "{path}", "ref": "{n}%"}, {"counter": "decimal", "label": "{path}", "ref": "5{n}"}], + "numbering.levels"), +]) +def test_a_percent_that_meets_a_counter_where_the_label_is_put_together_is_a_style_error( + levels: list[dict], setting: str) -> None: + style = load_style(overrides={"numbering.levels": levels}) + [problem] = docx_style_errors(style) + assert problem.startswith(setting) and "put together" in problem + with pytest.raises(StyleError): + docx_settings(style) + + +def test_a_percent_that_cannot_be_mistaken_is_fine() -> None: + levels = [{"counter": "decimal", "label": "{n}% of", "ref": "{n}"}] + assert docx_style_errors(load_style(overrides={"numbering.levels": levels})) == [] + tree, plan, _ = planned("# A\n", overrides={"numbering.levels": levels}) + assert plan.abstract_nums[0].levels[0].lvl_text == "%1% of" + + +def test_settings_that_are_off_are_not_checked() -> None: + overrides = {"enumeration.enabled": False, "enumeration.levels": [ + {"counter": "decimal", "label": "%1{n}", "ref": "{n}"}]} + assert docx_style_errors(load_style(overrides=overrides)) == [] + assert docx_style_errors(load_style(overrides={"numbering.scheme": "none"})) == [] + + +def test_a_level_text_longer_than_word_allows_is_a_style_error() -> None: + long = "x" * 260 + style = load_style(overrides={"numbering.levels": [{"counter": "decimal", "label": long + "{n}", "ref": "{n}"}]}) + [problem] = docx_style_errors(style) + assert problem.startswith("numbering.levels") and "255" in problem + with pytest.raises(StyleError): + docx_settings(style) + + +# -- Labels that show what Word cannot, and runs of units that Word counts off ------------------------ + + +def test_what_a_label_shows_is_judged_on_the_filled_label_not_on_its_spelling() -> None: + # `{path}` of an item in a section starts from the section's number: it shows it. In the preamble it does not. + levels = [{"counter": "lower-alpha", "label": "({n})", "ref": "({n})"}, + {"counter": "decimal", "label": "{path}", "ref": "({n})"}] + tree, plan, diagnostics = planned("# A\n\n- a\n - x\n - y\n- b\n", overrides={"enumeration.levels": levels}) + first, nested, _, _ = items(tree)[:4] + assert plan.units[first.unit].cause == () and plan.units[nested.unit].cause == ("label-fallback",) + [found] = diagnostics + # the reading is the one at the list's own depth: the level-2 text without the section ("(%1)(%2)" has only + # the list's own counters), not the level-1 text + assert found.rule == "render-docx-section-label" and "'(a)(1)'" in found.message + tree, plan, diagnostics = planned("- a\n - x\n\n# A\n", overrides={"enumeration.levels": levels}) + assert diagnostics == [] and plan.units[items(tree)[1].unit].cause == () + + +def test_a_label_that_needs_a_level_nothing_numbers_is_not_left_to_word() -> None: + # An ordered list's items are numbered by another abstractNum than the enumerated item nested in one: the level + # above the nested item is never used, which Word shows as 0, not 1. + levels = [{"counter": "lower-alpha", "label": "({n})", "ref": "({n})"}, + {"counter": "decimal", "label": "{path}", "ref": "({n})"}] + tree, plan, diagnostics = planned("1. a\n - x\n\n# A\n", overrides={"enumeration.levels": levels}) + _, nested = items(tree) + assert plan.units[nested.unit].cause == ("restart-family", "label-fallback") + assert plan.units[nested.unit].label == "(1)" # not "()(1)": the detached levels draw their own counter alone + assert [a.kind for a in plan.abstract_nums] == ["headings", "renumber", "detached"] + assert {d.rule for d in diagnostics} == {"render-docx-static-restart", "render-docx-section-label"} + + +# -- Word's counters against LibreOffice's, observed ----------------------------------------------------- + +#: What LibreOffice 24.2 writes for these values, from a document with one list per value (it follows the OOXML +#: definition, which Word documents too): letters repeat after 26, Roman numerals go on with `m` from 4000. +LIBREOFFICE_COUNTERS = { + "lowerLetter": {1: "a", 2: "b", 25: "y", 26: "z", 27: "aa", 28: "bb", 29: "cc", 52: "zz", 53: "aaa", 54: "bbb", + 55: "ccc", 78: "zzz", 79: "aaaa", 100: "vvvv"}, + "upperLetter": {27: "AA", 28: "BB", 52: "ZZ", 53: "AAA", 100: "VVVV"}, + "lowerRoman": {1: "i", 28: "xxviii", 79: "lxxix", 100: "c", 3999: "mmmcmxcix", 4000: "mmmm"}, + "upperRoman": {29: "XXIX", 78: "LXXVIII", 3999: "MMMCMXCIX", 4000: "MMMM"}, + "decimal": {1: "1", 26: "26", 4000: "4000"}, +} + + +@pytest.mark.parametrize("num_fmt", LIBREOFFICE_COUNTERS) +def test_word_counters_are_the_ones_libreoffice_wrote(num_fmt: str) -> None: + for value, written in LIBREOFFICE_COUNTERS[num_fmt].items(): + assert word_counter(value, num_fmt) == written + + +# -- Word numbers every unit as the resolver did, or a diagnostic says why not --------------------- + +def numbered(tree: RenderTree) -> Iterator[tuple[Section | ListItem | Paragraph, str]]: + """Every numbered unit with the counter the resolver wrote its number in.""" + formats = tree.formats + + def blocks(found: tuple) -> Iterator[tuple[Section | ListItem | Paragraph, str]]: + for block in found: + match block: + case Paragraph() if block.number is not None and formats.paragraphs is not None: + yield block, formats.paragraphs.counter + case List() if block.format is not None: + for item in block.items: + yield item, block.format.counter + yield from blocks(item.blocks) + case List(): + for item in block.items: + yield from blocks(item.blocks) + case Quote(blocks=inner) | DraftingNote(blocks=inner): + yield from blocks(inner) + + yield from blocks(tree.preamble) + for section in tree.sections: + if section.number is not None: + yield section, formats.headings[section.level - 1].counter + yield from blocks(section.blocks) + + +@functools.cache +def planned_run(source: str, variant: str) -> tuple: + """*source* rendered under *variant* and planned: (result, plan, diagnostics). The property tests of the numbering + and of the references look at the same documents, so each is rendered and planned once.""" + result = render(source, format="text", **VARIANTS[variant]) + plan, diagnostics = plan_docx(result.tree) + return result, plan, diagnostics + + +def assert_planned_as_resolved(result, plan, diagnostics) -> None: # noqa: ANN001 + """The property, unit by unit: Word's count is the resolver's number, and Word's label is the resolver's label, + unless that unit's own counter reads differently in Word (a letter past 26, a Roman numeral from 4000: its label + and those of its descendants may differ), its label cannot show what the resolver's does (the plan says so on the + unit), or its depth is past Word's. A later alternative that shares a number has no counter of its own (``numId`` 0, + its number the holder's, as fields) and reads as the resolver's label. There is no other exception, and a + diagnostic is reported exactly where there is one.""" + tree = result.tree + found = rules(diagnostics) + # A document that skips a heading level (a Core Error, which the renderer still draws) is the one exception: Word + # is expected to read the level that was skipped as 0 where the renderer and LibreOffice count it as 1, and the + # catch-all ``render-docx-number-mismatch`` may then report units Word counts off. + skips = any("skipped-level" in unit.cause for unit in plan.units.values()) + overflows: dict[str, str] = {} # unit -> the rule its counter reads differently under (itself or an ancestor) + parents = {unit.unit: unit.parent_unit for unit in units(tree)} + own_rules: set[str] = set() + for node, counter in numbered(tree): + mine = plan.units[node.unit] + clamped = "clamped" in mine.cause + alternative = node.shares_number_with is not None and mine.num_id == 0 + if alternative: + assert mine.count is None and "alternative" in mine.cause and mine.explicit and mine.ind is not None + assert mine.ilvl is not None and mine.label.rstrip(".") == mine.word_ref + elif not skips: + assert mine.count == node.number, (node.unit, mine.count, node.number) + own = "" + if not clamped and word_counter(node.number, NUM_FMT[counter]) != format_counter(node.number, counter): + own = "render-docx-roman-overflow" if counter.endswith("roman") else "render-docx-letter-overflow" + inherited = overflows.get(parents[node.unit] or "", "") or "" + overflows[node.unit] = own or inherited + if not (own or inherited or clamped or "label-fallback" in mine.cause): + assert mine.label == node.label, (node.unit, mine.label, node.label) + assert mine.cause.count("label-fallback") <= 1, (node.unit, mine.cause) + for cause, rule in (("label-fallback", "render-docx-section-label"), ("clamped", "render-docx-list-depth"), + ("resync", "render-docx-shared-counter")): + if cause in mine.cause: + assert rule in found, (node.unit, cause) + own_rules.update([own] if own else []) + assert found & {"render-docx-roman-overflow", "render-docx-letter-overflow"} == own_rules + assert skips or "render-docx-number-mismatch" not in found, "a placement is wrong: " + "; ".join( + d.message for d in diagnostics if d.rule == "render-docx-number-mismatch") + assert ("render-docx-alternative" in found) == any(u.alt is not None for u in plan.units.values()) + assert all(d.rule.startswith("render-docx-") for d in diagnostics) + assert {u.unit for u in units(tree)} == set(plan.units) + # Every num with an override is used, by a unit whose level the override names. + first: dict[int, str] = {} + for unit in units(tree): + mine = plan.units[unit.unit] + if mine.num_id: + first.setdefault(mine.num_id, unit.unit) + for num in plan.nums: + assert num.id in first, f"num {num.id} is never used" + if num.start_overrides: + assert plan.units[first[num.id]].ilvl == num.start_overrides[0][0] + # A unit carries an indent of its own where its level's serves another (a unit that is no heading on the heading + # list), where it has no level to take it from (a later alternative), and where it is nested in a unit that does + # (one step in from it, whatever list it is on). + headings = next((a.id for a in plan.abstract_nums if a.kind == "headings"), None) + nums = {n.id: n.abstract_id for n in plan.nums} + by_bookmark = {name: u.unit for u in plan.units.values() for name in u.bookmarks} + indented: set[str] = set() # the units that stand where an indent of their own is the one their text starts at + for unit in units(tree): + mine = plan.units[unit.unit] + on_headings = mine.num_id is not None and nums.get(mine.num_id) == headings and not isinstance(unit, Section) + if mine.alt is not None: # a later alternative stands where the unit it repeats does + on_headings = by_bookmark[mine.alt.segments[0].bookmark] in indented + if not isinstance(unit, Section) and (on_headings or unit.parent_unit in indented): + indented.add(unit.unit) + assert (mine.ind is not None) == (unit.unit in indented or mine.num_id == 0), (unit.unit, mine.ind, mine.num_id) + assert (mine.alt is not None) == (mine.num_id == 0) + + +@pytest.mark.parametrize("variant", VARIANTS) +@pytest.mark.parametrize("name", ["features.lgd", "template.lgd"]) +def test_the_test_documents_are_planned_as_resolved(name: str, variant: str) -> None: + source = (DOCUMENTS / name).read_text(encoding="utf-8") + assert_planned_as_resolved(*planned_run(source, variant)) + + +SPEC = spec_root() + + +def _spec_documents() -> list[Path]: + if SPEC is None: + return [] + return sorted((SPEC / "examples").rglob("*.lgd")) + sorted((SPEC / "fixtures" / "valid").rglob("*.lgd")) + + +@pytest.mark.conformance +@pytest.mark.skipif(SPEC is None, reason="LegalDown specification checkout not found") +@pytest.mark.parametrize("variant", VARIANTS) +def test_the_specifications_documents_are_planned_as_resolved(variant: str) -> None: + for path in _spec_documents(): + try: + run = planned_run(path.read_text(encoding="utf-8"), variant) + except DocumentError: + continue + try: + assert_planned_as_resolved(*run) + except AssertionError as error: + raise AssertionError(f"{path.relative_to(SPEC)}: {error}") from error + + +# -- Random documents --------------------------------------------------------------------------- + + +def _random_list(rng: random.Random, depth: int, indent: str, limit: int) -> list[str]: + ordered = rng.random() < 0.3 + marker = "1. " if ordered else "- " + lines: list[str] = [] + for index in range(rng.choice([1, 2, 2, 3, 5, 30]) if depth == 0 else rng.randint(1, 3)): + lines.append(f"{indent}{marker}item {depth}.{index}") + inner = indent + " " * len(marker) + chain = limit > 4 and index == 0 # a deep list is a chain: only its first item nests, or it would explode + if depth < limit and (chain or (limit <= 4 and rng.random() < 0.45)): + lines.extend(_random_list(rng, depth + 1, inner, limit)) + if rng.random() < 0.3: # a sibling list after text: it goes on from the first + lines += ["", f"{inner}Between.", ""] + lines.extend(_random_list(rng, depth + 1, inner, limit)) + if depth < 3 and rng.random() < 0.12: + lines += ["", f"{inner}> - quoted", f"{inner}> - quoted too", ""] + return lines + [""] + + +def _random_document(rng: random.Random) -> str: + out: list[str] = [] + level = 1 + deep = rng.random() < 0.12 # a list nested past the nine levels Word has + for _ in range(rng.randint(1, 7)): + move = rng.random() + level = max(1, min(5, level + (1 if move < 0.4 else -1 if move < 0.55 else 2 if move < 0.6 else 0))) + out.append("#" * level + " Heading\n") + for _ in range(rng.randint(0, 3)): + kind = rng.random() + if kind < 0.45: + out.extend(_random_list(rng, 0, "", 11 if deep else 4)) + elif kind < 0.7: + out.append("A paragraph.\n") + elif kind < 0.85: + out.append("> quoted\n") + else: + out.append("> [!DRAFTING]\n> - a note\n> - and another\n") + if rng.random() < 0.3: + out.insert(0, "- preamble item\n - nested\n") + return "\n".join(out) + + +def _alternative_list(rng: random.Random, index: int, indent: str, depth: int) -> list[str]: + """A list whose first item has two or three alternatives, some with lists of their own (children that differ in + number between the alternatives), among other items.""" + lines: list[str] = [] + for way in range(rng.choice([2, 3])): + lines.append(f"{indent}- item {way} {{#i{index}d{depth} when=forum:{FORUMS[way]}}}") + if depth < 3 and rng.random() < 0.6: + lines.extend(_random_list(rng, depth + 1, indent + " ", 2)) + if depth < 2 and rng.random() < 0.3: + lines.extend(_alternative_list(rng, index, indent + " ", depth + 1)) + lines.append(f"{indent}- after") + if depth < 3 and rng.random() < 0.5: + lines.extend(_random_list(rng, depth + 1, indent + " ", 2)) + return lines + [""] + + +def _random_alternatives(rng: random.Random) -> str: + """Sections, items and paragraphs that share a number, in twos and threes, with children of their own and with + other units around them.""" + out: list[str] = [] + for index in range(rng.randint(2, 6)): + out.append(f"# Section {index} {{#h{index}}}\n") + if rng.random() < 0.5: + ways = rng.choice([2, 3]) + out.append("\n\n".join(f"Para {way}. {{#p{index} when=forum:{FORUMS[way]}}}" for way in range(ways)) + + "\n\nAfter.\n") + if rng.random() < 0.6: + out.append("\n".join(_alternative_list(rng, index, "", 0))) + if rng.random() < 0.5: + for way in range(rng.choice([2, 3])): + out.append(f"# Disputes {{#d{index} when=forum:{FORUMS[way]}}}\n") + for sub in range(rng.randint(0, 3)): + out.append(f"## Sub {way}.{sub} {{#s{index}w{way}n{sub}}}\n") + if rng.random() < 0.4: + out.extend(_random_list(rng, 0, "", 2)) + if rng.random() < 0.3: + out.append(f"### Deeper {{#t{index}w{way}n{sub}}}\n") + if rng.random() < 0.4: + out.append("Text.\n\n- an item\n") + return "\n".join(out) + + +#: The random documents: one set for the numbering and the references. Each has an anchor on every unit and a +#: paragraph that refers to up to 40 of them (a different part of them in each document, so that every unit is +#: referred to by some document), and each variant plans its own sample of them, once. +RANDOM_DOCUMENTS = 36 +ALTERNATIVE_DOCUMENTS = 24 +PER_VARIANT = 9 + + +@functools.cache +def _random_sources() -> tuple[str, ...]: + rng = random.Random(20261028) + return tuple(with_references(with_anchors(_random_document(rng)), shift=index) for index in range(RANDOM_DOCUMENTS)) + + +@functools.cache +def _alternative_sources() -> tuple[str, ...]: + rng = random.Random(7) + return tuple(ALTERNATIVES + with_references(_random_alternatives(rng), shift=index) + for index in range(ALTERNATIVE_DOCUMENTS)) + + +def sample(variant: str, count: int, sources: tuple[str, ...]) -> list[str]: + """*count* of *sources*, a different choice for each variant.""" + return [sources[i] for i in sorted(random.Random(variant).sample(range(len(sources)), count))] + + +@functools.cache +def random_runs(variant: str) -> tuple: + """(source, result, plan, diagnostics) of the random documents planned under *variant*.""" + return tuple((source, *planned_run(FRONT + source, variant)) + for source in sample(variant, PER_VARIANT, _random_sources())) + + +@functools.cache +def alternative_runs(variant: str) -> tuple: + return tuple((source, *planned_run(source, variant)) + for source in sample(variant, PER_VARIANT, _alternative_sources())) + + +@pytest.mark.parametrize("variant", VARIANTS) +def test_random_documents_are_planned_as_resolved(variant: str) -> None: + for source, result, plan, diagnostics in random_runs(variant): + try: + assert_planned_as_resolved(result, plan, diagnostics) + except AssertionError as error: + raise AssertionError(f"{error}\n--- document ---\n{source}") from error + + +@pytest.mark.parametrize("variant", VARIANTS) +def test_random_alternatives_are_planned_as_resolved(variant: str) -> None: + for source, result, plan, diagnostics in alternative_runs(variant): + try: + assert_planned_as_resolved(result, plan, diagnostics) + except AssertionError as error: + raise AssertionError(f"{error}\n--- document ---\n{source}") from error + + +def test_planning_twice_gives_the_same_plan() -> None: + for variant in ("default", "continental", "none"): + for _, result, plan, diagnostics in random_runs(variant)[:4]: + assert plan_docx(result.tree) == (plan, diagnostics) + + +def test_the_random_documents_reach_every_case_the_property_is_about() -> None: + seen_rules: set[str] = set() + seen_causes: set[str] = set() + depth = 0 + for variant in VARIANTS: + for _, result, plan, diagnostics in random_runs(variant): + seen_rules |= rules(diagnostics) + seen_causes |= {cause for unit in plan.units.values() for cause in unit.cause} + depth = max(depth, max((lst.depth for lst in lists(result.tree)), default=0)) + for variant in VARIANTS: + for _, _, plan, diagnostics in alternative_runs(variant): + seen_rules |= rules(diagnostics) + seen_causes |= {cause for unit in plan.units.values() for cause in unit.cause} + assert {"render-docx-letter-overflow", "render-docx-section-label", "render-docx-static-restart", + "render-docx-shared-counter"} <= seen_rules + assert {"restart-family", "restart-section", "restart-alternative", "restart-depth", "resync", "label-fallback", + "skipped-level", "alternative", "reserved-level"} <= seen_causes + assert depth >= 10 + + +def test_documents_of_one_size_refer_to_different_units_so_that_more_of_them_are_referred_to() -> None: + groups: dict[int, list[set[str]]] = {} + for source in _random_sources(): + stride = -(-len(re.findall(r"\{#([a-z][a-z0-9-]*)", source)) // 40) or 1 + groups.setdefault(stride, []).append(set(re.findall(r"\{\{ref: ([a-z][a-z0-9-]*)", source))) + spread = [members for stride, members in groups.items() if stride > 1 and len(members) > 1] + assert spread + for members in spread: + assert len(set().union(*members)) > max(len(refs) for refs in members) + + +@pytest.mark.parametrize("variant", VARIANTS) +def test_a_reference_is_fixed_text_only_in_a_document_that_skips_a_heading_level(variant: str) -> None: + """Every reference of a document without errors is made of fields and the delimiters between them, whatever the + numbering style and the depth of its lists: the nesting past Word's nine levels has levels of its own, a path label + is live, and the words of a label that the format repeats are one field. What cannot be a field is the number of a + heading level the document skips (a Core error: no unit holds it): it is text, reported, and in such a document + the counters of the units Word draws differently are text too.""" + for source, _, plan, _ in (*random_runs(variant), *alternative_runs(variant)): + skipped = any("skipped-level" in unit.cause for unit in plan.units.values()) + if not skipped: + assert not [t for t, r in plan.refs.items() if r.fixed or r.ghost], source + + +#: Label forms a style can write that the built-in ones do not (and the reason a reference to them may stay fixed text: +#: ``delimiters`` around the counter that Word's ``\\t`` switch is not documented to leave out, ``counters`` of other +#: units that the label shows and the reference writes itself, ``period`` a prefix like ``Čl. `` that only the switch's +#: ``guess`` setting leaves out; None: all live). +EXTRA_VARIANTS: dict[str, tuple[dict, str | None]] = { + "section-path-word": ({"overrides": {"numbering.levels": [ + {"counter": "upper-roman", "label": "Article {n}", "ref": "{n}"}, + {"counter": "decimal", "label": "Section {path}", "ref": ".{n}"}, + {"counter": "lower-alpha", "label": "({n})", "ref": "({n})"}], "references.format": "Section {designation}"}}, None), + "path-all": ({"overrides": {"enumeration.levels": [ + {"counter": "decimal", "label": "{path}", "ref": ".{n}"}, {"counter": "lower-alpha", "label": "{path}", "ref": "({n})"}, + {"counter": "lower-roman", "label": "{path}", "ref": "({n})"}]}}, None), + "items-path-continental-sec": ({"style": "continental", "overrides": {"enumeration.levels": [ + {"counter": "decimal", "label": "{section}.{n}", "ref": ".{n}"}, + {"counter": "lower-alpha", "label": "{path}", "ref": "({n})"}, + {"counter": "lower-roman", "label": "{path}", "ref": "({n})"}]}}, None), + "outline-items-with-section": ({"style": "outline", "overrides": {"enumeration.levels": [ + {"counter": "decimal", "label": "{section}.{n}", "ref": ".{n}"}, + {"counter": "lower-alpha", "label": "{n})", "ref": "({n})"}]}}, None), + "heading-path-slash": ({"overrides": {"numbering.levels": [ + {"counter": "decimal", "label": "{n}.", "ref": "{n}"}, {"counter": "decimal", "label": "{path}", "ref": "/{n}"}]}}, None), + "outline-paragraph-path": ({"style": "outline", "overrides": {"paragraphs.numbered": True, "paragraphs.format": { + "counter": "decimal", "label": "{path}", "ref": ".{n}"}}}, None), + "continental-paragraph-dash": ({"style": "continental", "overrides": {"paragraphs.numbered": True, "paragraphs.format": { + "counter": "decimal", "label": "{section}-{n}", "ref": "-{n}"}}}, None), + "parenthesis-label-bare-ref": ({"overrides": {"enumeration.levels": [ + {"counter": "lower-alpha", "label": "({n})", "ref": "{n}"}, {"counter": "lower-roman", "label": "({n})", "ref": "{n}"}, + {"counter": "upper-alpha", "label": "({n})", "ref": "{n}"}]}}, "delimiters"), + "colon-label": ({"overrides": {"enumeration.levels": [ + {"counter": "decimal", "label": "{n}:", "ref": "{n}"}, {"counter": "lower-alpha", "label": "{n}:", "ref": "{n}"}]}}, + "delimiters"), + "section-in-label-not-in-ref": ({"overrides": {"enumeration.levels": [ + {"counter": "decimal", "label": "{section}.{n}", "ref": "{n}"}, {"counter": "lower-alpha", "label": "({n})", + "ref": "({n})"}]}}, "counters"), + "abbreviation-with-a-period": ({"overrides": {"numbering.levels": [ + {"counter": "decimal", "label": "Čl. {n}", "ref": "{n}"}, {"counter": "decimal", "label": "{path}", "ref": ".{n}"}], + "references.format": "čl. {designation}"}}, "period"), +} + + +@functools.cache +def extra_runs(name: str, switch: str) -> tuple: + """(source, plan, diagnostics) of a sample of the random documents without a skipped heading level, under the label + forms of :data:`EXTRA_VARIANTS` and a setting of the switch.""" + runs = [] + for source in (*sample(name, 8, _random_sources()), *sample(name, 2, _alternative_sources())): + prefix = "" if source.startswith("---") else FRONT + result = render(prefix + source, format="text", **EXTRA_VARIANTS[name][0]) + plan, diagnostics = plan_docx(result.tree, text_switch=switch) # type: ignore[arg-type] + if not any("skipped-level" in unit.cause for unit in plan.units.values()): + runs.append((source, plan, diagnostics)) + return tuple(runs) + + +@pytest.mark.parametrize("name", EXTRA_VARIANTS) +def test_every_label_form_is_live_except_where_the_reason_a_reference_stays_fixed_says_so(name: str) -> None: + reason = EXTRA_VARIANTS[name][1] + runs = extra_runs(name, "documented") + assert runs + fixed = sum(r.fixed for _, plan, _ in runs for r in plan.refs.values()) + if reason is None: + assert fixed == 0 + return + assert fixed > 0 + findings = [d.message for _, _, ds in runs for d in ds if d.rule == "render-docx-reference" and "not a field" in d.message] + phrase = {"delimiters": "text or punctuation around the counter", "counters": "holds the counters", + "period": "a period after letters"}[reason] + assert findings and all(phrase in m for m in findings) + # the setting that guesses is the one that lets a prefix with a period be left out, and no other reason is its business + guessed = sum(r.fixed for _, plan, _ in extra_runs(name, "guess") for r in plan.refs.values()) + assert (guessed == 0) == (reason == "period") + + +def test_each_variant_plans_its_own_sample_of_the_random_documents() -> None: + samples = {tuple(sample(variant, PER_VARIANT, _random_sources())) for variant in VARIANTS} + assert len(samples) == len(VARIANTS) + + +# ===================================================================================================== +# Cross-references, bookmarks and the table of contents (docs/architecture.md, "The DOCX planner") +# ===================================================================================================== + + +def show(plan: DocxPlan, segments: Sequence[Segment]) -> str: + """*segments* as one line: literal text in quotes, a field as ``REF`` and the anchor of the bookmark it points at, + ``\\t`` after it when it suppresses the text around the counter, ``\\h`` for the text of a heading.""" + anchors = {name: anchor for anchor, name in plan.anchors.items()} + pieces = [] + for segment in segments: + if isinstance(segment, Literal_): + pieces.append(f'"{segment.text}"') + else: + pieces.append(f"REF {anchors[segment.bookmark]}" + (" \\t" if segment.suppress_text else "") + + ("" if segment.number else " \\h")) + return " + ".join(pieces) + + +def cached(segments: Sequence[Segment]) -> str: + return "".join(s.text if isinstance(s, Literal_) else s.cached for s in segments) + + +#: A document that has every kind of target; the references at the end are the ones the tests read. +TARGETS = """# Alpha {#alpha} + +Intro paragraph one. {#p1} + +- first item {#i1} +- second item {#i2} + - nested one {#n1} + - nested two {#n2} + - deep one {#d1} +- third item {#i3} + +## Beta {#beta} + +### Gamma {#gamma} + +1. ordered one {#o1} +2. ordered two {#o2} + - bullet in ordered {#b1} + +#### Delta {#delta} + +##### Epsilon {#eps} + +- eps item {#ei} + +# Second {#second} + +- a {#sa} +""" +REFERENCES = "\n" + " ".join(f"{{{{ref: {anchor}}}}}" for anchor in ( + "alpha", "p1", "i1", "n2", "d1", "beta", "gamma", "o2", "b1", "delta", "eps", "ei", "second", "sa", "i3")) + +ARTICLE = {"numbering.levels": [{"counter": "upper-roman", "label": "Article {n}", "ref": "{n}"}]} +SECTION_SIGN = {"numbering.levels": [{"counter": "decimal", "label": "§ {n}", "ref": "{n}"}], + "references.format": "§ {designation}"} +CL_PERIOD = {"numbering.levels": [ + {"counter": "decimal", "label": "Čl. {n}", "ref": "{n}"}, {"counter": "decimal", "label": "{path}", "ref": ".{n}"}], + "references.format": "Čl. {designation}"} +SECTION_WORD = {"numbering.levels": [ + {"counter": "upper-roman", "label": "Article {n}", "ref": "{n}"}, + {"counter": "decimal", "label": "Section {path}", "ref": ".{n}"}], "references.format": "Section {designation}"} + + +def covers(variant: str, text_switch: str = "documented", **settings: object) -> dict[str, tuple[str, str]]: + """Anchor of each target to (what the renderer shows, the plan's cover), for the document of :data:`TARGETS`.""" + result = render(FRONT + TARGETS + REFERENCES, format="text", **settings) + plan, _ = plan_docx(result.tree, text_switch=text_switch) # type: ignore[arg-type] + units_of = {n.unit: n for n in units(result.tree)} + found = {} + for inline in iter_tree_inlines(result.tree): + if isinstance(inline, CrossRef): + anchor = units_of[inline.target_unit].anchor + found[anchor] = (inline.text, show(plan, plan.refs[inline.target_unit].segments)) + return found + + +# What each style's references are made of: fields wherever Word draws the number, literal text only for the +# delimiters between them. A field for the target alone covers a ``{path}`` label (``1.1.1``), one field per +# level covers a ``{n}`` label (``I.A.1``). +COVERS = [ + # default: decimal headings, (a) (i) (A) items, 1.1.1(2) for an ordered item in a section + ("default", {}, "alpha", "1", "REF alpha"), + ("default", {}, "i1", "1(a)", "REF alpha + REF i1"), + ("default", {}, "d1", "1(b)(ii)(A)", "REF alpha + REF i2 + REF n2 + REF d1"), + ("default", {}, "gamma", "1.1.1", "REF gamma"), + ("default", {}, "o2", "1.1.1(2)", 'REF gamma + "(" + REF o2 + ")"'), + ("default", {}, "eps", "1.1.1.1.1", "REF eps"), + ("default", {}, "ei", "1.1.1.1.1(a)", "REF eps + REF ei"), + # outline: every part of the number is a counter of its own level + ("outline", {"style": "outline"}, "alpha", "I", "REF alpha"), + ("outline", {"style": "outline"}, "beta", "I.A", 'REF alpha + "." + REF beta'), + ("outline", {"style": "outline"}, "eps", "I.A.1.a(i)", + 'REF alpha + "." + REF beta + "." + REF gamma + "." + REF delta + REF eps'), + ("outline", {"style": "outline"}, "i3", "I(c)", "REF alpha + REF i3"), + # mixed: 1(a)(i) + ("mixed", {"overrides": {"numbering.scheme": "mixed"}}, "gamma", "1(a)(i)", "REF alpha + REF beta + REF gamma"), + ("mixed", {"overrides": {"numbering.scheme": "mixed"}}, "o2", "1(a)(i)(2)", + 'REF alpha + REF beta + REF gamma + "(" + REF o2 + ")"'), + # continental: an item rides the heading list's next level (%1.%2), so one field is its whole number; an item + # nested in an item is written "b)" and referred to as "(b)" + ("continental", {"style": "continental"}, "i1", "1.1", "REF i1"), + ("continental", {"style": "continental"}, "n2", "1.2(b)", 'REF i2 + "(" + REF n2'), + ("continental", {"style": "continental"}, "d1", "1.2(b)(i)", 'REF i2 + "(" + REF n2 + "(" + REF d1'), + ("continental", {"style": "continental"}, "p1", "1(1)", "REF alpha + REF p1"), + ("continental", {"style": "continental"}, "gamma", "1.1.1", "REF gamma"), + # the none scheme: the text of the heading, then the pieces after a space + ("none", {"overrides": {"numbering.scheme": "none"}}, "alpha", "Alpha", "REF alpha \\h"), + ("none", {"overrides": {"numbering.scheme": "none"}}, "d1", "Alpha (b) (ii) (A)", + 'REF alpha \\h + " " + REF i2 + " " + REF n2 + " " + REF d1'), + ("none", {"overrides": {"numbering.scheme": "none"}}, "o2", "Gamma (2)", 'REF gamma \\h + " (" + REF o2 + ")"'), + # words and symbols in a label that the reference leaves out: \t drops them, the format puts them back + ("article", {"overrides": {**ARTICLE, "references.format": "Article {designation}"}}, "alpha", "Article I", "REF alpha"), + ("article", {"overrides": {**ARTICLE, "references.format": "Article {designation}"}}, "i1", "Article I(a)", + "REF alpha + REF i1"), + ("article", {"overrides": {**ARTICLE, "references.format": "Article {designation}"}}, "beta", "Article I.1", + '"Article " + REF beta'), + ("section-sign", {"overrides": SECTION_SIGN}, "d1", "§ 1(b)(ii)(A)", "REF alpha + REF i2 + REF n2 + REF d1"), + ("section-sign", {"overrides": SECTION_SIGN}, "gamma", "§ 1.1.1", '"§ " + REF gamma'), + # the words of the format are the number's own words: a field for the whole number stands for them too + ("cl-period", {"overrides": CL_PERIOD}, "i1", "Čl. 1(a)", "REF alpha + REF i1"), + ("cl-period", {"overrides": CL_PERIOD}, "beta", "Čl. 1.1", '"Čl. " + REF beta'), + ("section-word", {"overrides": SECTION_WORD}, "beta", "Section I.1", "REF beta"), + ("section-word", {"overrides": SECTION_WORD}, "gamma", "Section I.1.1", '"Section " + REF gamma'), + # a label that says another word than the format: the words are left out of the number with \t and the format's + # put back + ("section-word", {"overrides": SECTION_WORD}, "alpha", "Section I", '"Section " + REF alpha \\t'), + ("section-word", {"overrides": SECTION_WORD}, "i1", "Section I(a)", '"Section " + REF alpha \\t + REF i1'), +] + + +@pytest.mark.parametrize(("variant", "settings", "anchor", "text", "expected"), COVERS) +def test_a_reference_is_the_renderers_designation_covered_by_fields_and_delimiters( + variant: str, settings: dict, anchor: str, text: str, expected: str) -> None: + assert covers(variant, **settings)[anchor] == (text, expected) + + +def test_a_cover_reads_as_the_renderer_does_and_a_field_is_cached_as_word_draws_it() -> None: + result = render(FRONT + TARGETS + REFERENCES, format="text", style="outline") + plan, diagnostics = plan_docx(result.tree) + for inline in iter_tree_inlines(result.tree): + if isinstance(inline, CrossRef): + reference = plan.refs[inline.target_unit] + assert reference.exact and cached(reference.segments) == inline.text + for segment in reference.segments: + if isinstance(segment, FieldSeg): + assert segment.number and segment.hyperlink and not segment.suppress_text + assert segment.cached_lo == segment.cached + # a section's own field returns its number without the period of its label + assert plan.units["s1"].label == "I." and plan.units["s1"].word_ref == "I" + assert next(s for s in plan.refs["s1"].segments if isinstance(s, FieldSeg)).cached == "I" + assert diagnostics and all(d.rule == "render-docx-static-restart" for d in diagnostics) + + +def test_a_reference_to_an_item_in_another_section_is_built_from_the_chain_of_its_own_section() -> None: + body = "# One {#one}\n\n## Inner {#inner}\n\n- deep {#deep}\n - deeper {#deeper}\n\n# Two {#two}\n\n" \ + "See {{ref: deeper}} and {{ref: inner}}, {{ref: one}}.\n" + tree, plan, diagnostics = planned(body, style="outline") + refs = {i.text: plan.refs[i.target_unit] for i in iter_tree_inlines(tree) if isinstance(i, CrossRef)} + assert [(text, show(plan, r.segments)) for text, r in refs.items()] == [ + ("I.A(a)(i)", 'REF one + "." + REF inner + REF deep + REF deeper'), + ("I.A", 'REF one + "." + REF inner'), ("I", "REF one")] + assert all(r.exact for r in refs.values()) and not [d for d in diagnostics if "reference" in d.rule] + + +def test_a_reference_to_a_unit_the_style_does_not_number_shows_the_designation_of_its_section() -> None: + body = "# Alpha {#alpha}\n\n## Beta {#beta}\n\n- bullet {#bullet}\n\nSee {{ref: bullet}}.\n" + tree, plan, diagnostics = planned(body, overrides={"enumeration.enabled": False}) + [reference] = [i for i in iter_tree_inlines(tree) if isinstance(i, CrossRef)] + assert reference.text == "1.1" and reference.target_unit != reference.section_unit + assert show(plan, plan.refs[reference.target_unit].segments) == "REF beta" + assert plan.refs[reference.target_unit].exact and cached(plan.refs[reference.target_unit].segments) == "1.1" + # the bullet itself is a target nobody's field points at: it keeps its own bookmark for links + assert plan.units[reference.target_unit].bookmarks == ("ld_bullet",) + + +def test_references_format_wraps_every_occurrence_of_the_designation() -> None: + body = "# Alpha {#alpha}\n\n- x {#x}\n\nSee {{ref: alpha}} {{ref: x}}.\n" + for fmt, expected in (("Section {designation} hereof", ["Section 1 hereof", "Section 1(a) hereof"]), + ("{designation} and {designation}", ["1 and 1", "1(a) and 1(a)"]), + ("[{designation}]", ["[1]", "[1(a)]"])): + tree, plan, _ = planned(body, overrides={"references.format": fmt}) + found = [i for i in iter_tree_inlines(tree) if isinstance(i, CrossRef)] + assert [i.text for i in found] == expected + assert [cached(plan.refs[i.target_unit].segments) for i in found] == expected + assert all(plan.refs[i.target_unit].exact for i in found) + tree, plan, _ = planned(body, overrides={"references.format": "{designation} and {designation}"}) + [first, second] = [plan.refs[i.target_unit] for i in iter_tree_inlines(tree) if isinstance(i, CrossRef)] + assert show(plan, first.segments) == 'REF alpha + " and " + REF alpha' + assert show(plan, second.segments) == 'REF alpha + REF x + " and " + REF alpha + REF x' + + +def test_a_reference_to_the_same_target_is_planned_once() -> None: + tree, plan, _ = planned("# A {#a}\n\n{{ref: a}} {{ref: a}} and again {{ref: a}}.\n") + assert list(plan.refs) == ["s1"] + + +def test_the_fields_of_a_unit_that_is_not_a_target_have_a_bookmark_of_their_own() -> None: + """Every unit a field points at has a bookmark: its anchor's, or a hidden one when it has no anchor.""" + tree, plan, _ = planned("# A\n\n- x\n - y {#y}\n\nSee {{ref: y}}.\n") + assert plan.units["u2"].bookmarks == ("ld_y",) + assert plan.units["u1"].bookmarks == ("_Ld1",) and plan.units["s1"].bookmarks == ("ld_a",) + assert plan.units["s1"].bookmarks[0] in plan.anchors.values() and "_Ld1" not in plan.anchors.values() + [reference] = plan.refs.values() + assert [s.bookmark for s in reference.segments] == ["ld_a", "_Ld1", "ld_y"] + + +# -- Why a reference cannot be built from fields ------------------------------------------------------------- + + +def references_of(tree: RenderTree, plan: DocxPlan) -> list[tuple[str, str, bool]]: + return [(i.text, show(plan, plan.refs[i.target_unit].segments), plan.refs[i.target_unit].exact) + for i in iter_tree_inlines(tree) if isinstance(i, CrossRef)] + + +def test_a_skipped_heading_level_is_the_one_part_of_a_reference_that_is_written_as_text() -> None: + body = "# A {#a}\n\n### B {#b}\n\nSee {{ref: b}}.\n" + tree, plan, diagnostics = planned(body, style="outline") + # no unit holds the number of the level the document skips (an error, §4.1), so it is text between two fields + assert references_of(tree, plan) == [("I.A.1", 'REF a + ".A." + REF b', True)] + assert plan.refs["s2"].ghost == "A" and not plan.refs["s2"].fixed + [found] = [d for d in diagnostics if d.rule == "render-docx-reference"] + assert found.level == "warning" and found.line == tree.sections[1].blocks[0].line + assert found.message.startswith("The cross-reference 'I.A.1' holds the number of a heading level that the document skips") + assert "'A' is written as text there and will not follow edits made in Word; the rest is made of fields" in found.message + # the decimal scheme writes the whole number in the heading's own field, so nothing is missing there + tree, plan, diagnostics = planned(body) + assert references_of(tree, plan) == [("1.1.1", "REF b", True)] and not diagnostics + assert not plan.refs["s2"].fixed and not plan.refs["s2"].ghost + + +def test_text_around_a_counter_that_word_cannot_drop_leaves_the_reference_as_one_field() -> None: + body = "# A\n\n- x {#x}\n\nSee {{ref: x}}.\n" + for label, shown in (("({n})", "(1)"), ("{n}:", "1:"), ("No. {n}", "No. 1")): + overrides = {"enumeration.levels": [{"counter": "decimal", "label": label, "ref": "{n}"}]} + tree, plan, diagnostics = planned(body, overrides=overrides) + [(text, cover_, exact)] = references_of(tree, plan) + assert (text, cover_, exact) == ("11", '"11"', True) and plan.refs[next(iter(plan.refs))].fixed + [found] = [d for d in diagnostics if d.rule == "render-docx-reference"] + assert f"Word draws its number as '{shown}'" in found.message and "written as the text '11'" in found.message + + +def test_a_cover_that_reads_differently_in_word_is_reported_unless_a_unit_already_says_why() -> None: + many = "\n".join(f"- i{i} {{#m{i}}}" for i in range(1, 30)) + tree, plan, diagnostics = planned(f"# A {{#a}}\n\n{many}\n\nSee {{{{ref: m28}}}} {{{{ref: m1}}}}.\n") + # the letter counter past 26 is a finding of its own: the references to it are not reported again + assert references_of(tree, plan) == [("1(ab)", "REF a + REF m28", False), ("1(a)", "REF a + REF m1", True)] + assert [d.rule for d in diagnostics] == ["render-docx-letter-overflow"] + assert cached(plan.refs[tree.sections[0].blocks[0].items[27].unit].segments) == "1(bb)" + + +def test_a_reference_whose_text_the_renderer_cut_is_reported() -> None: + title = "word " * 300 + body = f"# {title.strip()} {{#long}}\n\nSee {{{{ref: long}}}}.\n" + tree, plan, diagnostics = planned(body, overrides=NONE) + [reference] = [i for i in iter_tree_inlines(tree) if isinstance(i, CrossRef)] + assert len(reference.text) < len(title) - 100 + assert not plan.refs["s1"].exact + [found] = [d for d in diagnostics if d.rule == "render-docx-reference"] + assert "will read" in found.message and "numbered or written differently" in found.message + + +def test_a_cover_with_the_switch_word_ignores_in_libreoffice_is_an_information_for_the_document() -> None: + body = "# A {#a}\n\n# B {#b}\n\nSee {{ref: a}} {{ref: b}}.\n" + settings = {"overrides": {**ARTICLE, "references.format": "Art. {designation}"}} + tree, plan, diagnostics = planned(body, **settings) + first, second = (plan.refs[i.target_unit].segments for i in iter_tree_inlines(tree) if isinstance(i, CrossRef)) + assert [(s.suppress_text, s.cached, s.cached_lo) for s in first if isinstance(s, FieldSeg)] == [(True, "I", "Article I")] + assert [(s.suppress_text, s.cached, s.cached_lo) for s in second if isinstance(s, FieldSeg)] == [(True, "II", "Article II")] + [found] = [d for d in diagnostics if d.rule == "render-docx-reference-word-only"] + assert found.level == "info" and found.line is None and "'Art. I'" in found.message + assert "reads 'Art. Article I'" in found.message and "so do 1 more" in found.message + assert all(plan.refs[i].exact for i in plan.refs) + + +def test_the_words_of_the_format_that_the_number_holds_too_need_no_switch() -> None: + """``Article {designation}`` and a label ``Article {n}``: the plain ``REF`` returns ``Article I``, which is the + whole reference, so there is no ``\\t`` and nothing for LibreOffice to read differently.""" + body = "# A {#a}\n\n# B {#b}\n\nSee {{ref: a}} {{ref: b}}.\n" + tree, plan, diagnostics = planned(body, overrides={**ARTICLE, "references.format": "Article {designation}"}) + for reference in (plan.refs[i.target_unit] for i in iter_tree_inlines(tree) if isinstance(i, CrossRef)): + [segment] = reference.segments + assert isinstance(segment, FieldSeg) and not segment.suppress_text and segment.cached == segment.cached_lo + assert reference.exact + assert [segment.cached for i in iter_tree_inlines(tree) if isinstance(i, CrossRef) + for segment in plan.refs[i.target_unit].segments] == ["Article I", "Article II"] + assert diagnostics == [] + + +#: A label that says another word than the format (the words go with ``\t``, the number between its counters stays) and a +#: label that leads with an abbreviation that has a period. +SECTION_SHORT = {"overrides": {**SECTION_WORD, "references.format": "Sec. {designation}"}} +CL_LOWER = {"overrides": {**CL_PERIOD, "references.format": "čl. {designation}"}} + + +@pytest.mark.parametrize(("switch", "settings", "anchor", "expected"), [ + # Microsoft's example, "Section 1.01" gives "1.01": words go, the delimiter between the counters stays + ("off", SECTION_SHORT, "beta", ("Sec. I.1", '"Sec. I.1"')), + ("documented", SECTION_SHORT, "beta", ("Sec. I.1", '"Sec. " + REF beta \\t')), + ("guess", SECTION_SHORT, "beta", ("Sec. I.1", '"Sec. " + REF beta \\t')), + ("documented", SECTION_SHORT, "i1", ("Sec. I(a)", '"Sec. " + REF alpha \\t + REF i1')), + # a period after letters ("Čl. 1") is not documented: only the setting that guesses uses it + ("off", CL_LOWER, "alpha", ("čl. 1", '"čl. 1"')), + ("documented", CL_LOWER, "alpha", ("čl. 1", '"čl. 1"')), + ("guess", CL_LOWER, "alpha", ("čl. 1", '"čl. " + REF alpha \\t')), + ("guess", CL_LOWER, "i1", ("čl. 1(a)", '"čl. " + REF alpha \\t + REF i1')), + # what the cover does without the switch does not depend on the setting + ("off", CL_LOWER, "beta", ("čl. 1.1", '"čl. " + REF beta')), + ("guess", CL_LOWER, "beta", ("čl. 1.1", '"čl. " + REF beta')), +]) +def test_the_text_switch_setting_decides_which_references_use_the_switch( + switch: str, settings: dict, anchor: str, expected: tuple[str, str]) -> None: + assert covers("x", switch, **settings)[anchor] == expected + + +def test_a_reference_that_needs_a_switch_the_setting_does_not_allow_is_fixed_text_and_reported() -> None: + body = "# A {#a}\n\nSee {{ref: a}}.\n" + for switch, fixed in (("off", True), ("documented", True), ("guess", False)): + result = render(FRONT + body, format="text", **CL_LOWER) + plan, diagnostics = plan_docx(result.tree, text_switch=switch) # type: ignore[arg-type] + assert plan.refs["s1"].fixed is fixed + assert any(d.rule == "render-docx-reference" and "not a field" in d.message for d in diagnostics) is fixed + assert any(d.rule == "render-docx-reference-word-only" for d in diagnostics) is not fixed + # off: no field of the document has the switch, whatever the labels + result = render(FRONT + TARGETS + REFERENCES, format="text", **SECTION_SHORT) + plan, diagnostics = plan_docx(result.tree, text_switch="off") + assert not any(isinstance(s, FieldSeg) and s.suppress_text for r in plan.refs.values() for s in r.segments) + assert "render-docx-reference-word-only" not in rules(diagnostics) + assert any(r.fixed for r in plan.refs.values()) + + +def test_the_rendered_file_has_the_switch_only_where_the_style_allows_it() -> None: + body = "# A {#a}\n\nSee {{ref: a}}.\n" + texts = {} + for switch in ("off", "documented", "guess"): + result = render(FRONT + body, format="docx", overrides={**CL_LOWER["overrides"], "docx.reference_text_switch": switch}) + assert result.binary is not None + texts[switch] = zipfile.ZipFile(io.BytesIO(result.binary)).read("word/document.xml").decode("utf-8") + assert ["\\t" in texts[switch] for switch in texts] == [False, False, True] + + +def test_a_writer_that_plans_the_tree_itself_plans_it_with_the_styles_switch() -> None: + result = render(FRONT + "# A {#a}\n\nSee {{ref: a}}.\n", format="text", + overrides={**CL_LOWER["overrides"], "docx.reference_text_switch": "guess"}) + data = DocxWriter(result.style).write(result.tree) + assert "\\t" in zipfile.ZipFile(io.BytesIO(data)).read("word/document.xml").decode("utf-8") + other = render(FRONT + "# A {#a}\n\nSee {{ref: a}}.\n", format="text", **CL_LOWER) + data = DocxWriter(other.style).write(other.tree) + assert "\\t" not in zipfile.ZipFile(io.BytesIO(data)).read("word/document.xml").decode("utf-8") + + +def test_reference_findings_name_the_file_and_the_first_line_of_a_repeated_reference() -> None: + body = "# A {#a}\n\n### B {#b}\n\nFirst {{ref: b}}.\n\nSecond {{ref: b}}.\n" + result = render(FRONT + body, format="text", style="outline") + _, diagnostics = plan_docx(result.tree, file="deal.lgd") + [found] = [d for d in diagnostics if d.rule == "render-docx-reference"] + assert found.file == "deal.lgd" and found.line == result.tree.sections[1].blocks[0].line + + +def test_untrusted_text_in_a_reference_message_is_escaped() -> None: + body = "# A\n\n### B {#b}\n\nSee {{ref: b}}.\n" + levels = [{"counter": "decimal", "label": "{n}\u0007", "ref": "{n}\u0007"}, {"counter": "decimal", "label": "{n}", + "ref": "{n}"}] + _, _, diagnostics = planned(body, overrides={"numbering.levels": levels}) + assert all("\u0007" not in d.message for d in diagnostics) + + +# -- Bookmarks that never hold a field, and references that would ----------------------------------------------- + + +def test_a_bookmark_is_a_point_unless_a_field_returns_the_text_it_holds() -> None: + body = "# A {#a}\n\n- x {#x}\n\nSee {{ref: x}} and {{ref: a}}.\n" + _, plan, _ = planned(body) + assert plan.units["s1"].collapsed and plan.units["u1"].collapsed # numbers only: \n fields + _, plan, _ = planned(body, overrides=NONE) + # the heading's text is what the field for it returns: its bookmark holds it; the item's is a point + assert not plan.units["s1"].collapsed and plan.units["u1"].collapsed + + +def test_a_heading_that_refers_to_itself_or_in_a_circle_has_those_references_written_as_text() -> None: + body = ("# A {{ref: b}} {#a}\n\n# B {{ref: a}} {#b}\n\n# C {{ref: c}} {#c}\n\n# D {{ref: e}} {#d}\n\n" + "# E {#e}\n\nSee {{ref: a}}.\n") + tree, plan, diagnostics = planned(body, overrides=NONE) + assert [(u, plan.units[u].literal_refs) for u in ("s1", "s2", "s3", "s4", "s5")] == [ + ("s1", ("s2",)), ("s2", ("s1",)), ("s3", ("s3",)), ("s4", ()), ("s5", ())] # D's reference to E is no cycle + found = [d for d in diagnostics if d.rule == "render-docx-reference-cycle"] + assert [(d.level, d.line) for d in found] == [("warning", tree.sections[i].line) for i in range(3)] + assert "written as text" in found[0].message and "will not follow edits" in found[0].message + # outside a heading, and where the number is what is referred to, nothing is a cycle + _, plan, diagnostics = planned(body) + assert not any(u.literal_refs for u in plan.units.values()) + assert "render-docx-reference-cycle" not in rules(diagnostics) + + +def test_a_reference_in_a_heading_to_a_unit_inside_another_heading_is_still_a_field() -> None: + body = "# A {{ref: x}} {#a}\n\n# B {#b}\n\n- x {#x}\n\n# C {{ref: b}} {#c}\n" + _, plan, diagnostics = planned(body, overrides=NONE) + assert not any(u.literal_refs for u in plan.units.values()) + assert "render-docx-reference-cycle" not in rules(diagnostics) + + +def test_two_cyclic_headings_with_the_same_text_are_both_reported() -> None: + body = "# Same {{ref: a}} {#a}\n\n# Same {{ref: b}} {#b}\n\n# Same {{ref: c}} {#c}\n" + tree, plan, diagnostics = planned(body, overrides=NONE) + found = [d for d in diagnostics if d.rule == "render-docx-reference-cycle"] + assert [d.line for d in found] == [section.line for section in tree.sections] + assert all(f"line {d.line}" in d.message for d in found) + + +def test_the_headings_that_lead_to_each_other_are_found_by_components_not_by_walking_from_each() -> None: + # a ring of three with a tail into it and a chain out of it: only the ring is cyclic, whatever the order + successors = {"a": ["b"], "b": ["c"], "c": ["a", "d"], "t": ["a"], "d": ["e"], "e": []} + component, order = _components(successors) + assert component["a"] == component["b"] == component["c"] + assert len({component[n] for n in "abctde"}) == 4 + assert [sorted(members) for members in order if len(members) > 1] == [["a", "b", "c"]] + # each component comes after those it leads to + assert order.index(["e"]) < order.index(["d"]) < order.index(next(m for m in order if "a" in m)) + assert order.index(["t"]) == len(order) - 1 + + +def test_the_components_of_a_graph_agree_with_what_each_node_can_reach() -> None: + rng = random.Random(3) + for _ in range(200): + size = rng.randint(1, 9) + names = [f"n{i}" for i in range(size)] + graph = {n: [m for m in names if rng.random() < 0.25] for n in names} + component, _ = _components(graph) + + def reaches(start: str, goal: str, graph: dict[str, list[str]] = graph) -> bool: + seen, pending = set(), list(graph[start]) + while pending: + node = pending.pop() + if node == goal: + return True + if node not in seen: + seen.add(node) + pending += graph[node] + return False + + for a in names: + for b in names: + assert (component[a] == component[b]) == (a == b or (reaches(a, b) and reaches(b, a))) + + +def test_a_very_long_chain_of_components_does_not_hit_the_recursion_limit() -> None: + chain = {f"n{i}": [f"n{i + 1}"] for i in range(50_000)} + component, order = _components(chain) + assert len(order) == 50_001 and len(set(component.values())) == 50_001 + ring = {f"n{i}": [f"n{(i + 1) % 50_000}"] for i in range(50_000)} + _, order = _components(ring) + assert len(order) == 1 and len(order[0]) == 50_000 + + +def _chain(count: int, close: bool) -> str: + """Headings that each refer to the one before (and the first to the last, with *close*): under the none scheme + each reference is a field for the heading's text.""" + parts = [f"# H0 {{{{ref: h{count - 1}}}}} {{#h0}}\n" if close else "# H0 {#h0}\n"] + parts += [f"# H{i} see {{{{ref: h{i - 1}}}}} {{#h{i}}}\n" for i in range(1, count)] + return "\n".join(parts) + + +@pytest.mark.parametrize("close", [False, True]) +def test_the_headings_of_a_long_chain_of_references_are_planned_in_linear_time(close: bool) -> None: + def seconds(count: int) -> float: + tree = render(FRONT + _chain(count, close), format="text", overrides=NONE).tree + best = float("inf") + for _ in range(2): + started = time.perf_counter() + plan_docx(tree) + best = min(best, time.perf_counter() - started) + return best + + small, large = seconds(500), seconds(4000) + assert large < 30 * max(small, 0.01), (small, large) # eight times the headings: a walk from each would be sixty-four + assert large < 2.0 + + +def test_a_ring_of_references_in_titles_is_all_text_and_a_chain_all_fields() -> None: + _, ring, _ = planned(_chain(30, True), overrides=NONE) + assert all(ring.units[f"s{i}"].literal_refs for i in range(1, 31)) + _, chain, _ = planned(_chain(30, False), overrides=NONE) + assert not any(unit.literal_refs for unit in chain.units.values()) + + +def test_the_text_a_heading_field_returns_is_made_of_the_fields_in_the_title_it_stands_for() -> None: + # A's title refers to the item under it (written as text: the item's designation starts with A's own text); B's + # refers to A, a field for A's text, which includes the text A's title has for that reference. + body = "# A {{ref: i}} {#a}\n\n- x {#i}\n\n# B {{ref: a}} {#b}\n\nRefs {{ref: a}} {{ref: b}} {{ref: i}}\n" + tree, plan, _ = planned(body, overrides=NONE) + a, b = tree.sections + assert plan.units[a.unit].literal_refs == ("u1",) and plan.units[b.unit].literal_refs == () + texts = {t: cached(plan.refs[t].segments) for t in plan.refs} + title_of_a = next(i.text for i in iter_tree_inlines(tree) if isinstance(i, CrossRef) and i.target_unit == "u1") + # B's heading text is "B " and what the field for A gives, which is A's text: "A " and the reference as text + assert texts[a.unit] == "A " + title_of_a + assert texts[b.unit] == "B " + texts[a.unit] + + +# -- What the bookmark returns is what the runs show --------------------------------------------------------------- + + +def test_a_heading_field_returns_the_text_of_the_runs_not_the_renderers_collapsed_designation() -> None: + tree, plan, diagnostics = planned("# The {{def: foo}} clause {#c}\n\nSee {{ref: c}}.\n", overrides=NONE) + [reference] = [i for i in iter_tree_inlines(tree) if isinstance(i, CrossRef)] + assert reference.text == "The clause" # the renderer collapses the white space of a designation + [segment] = plan.refs[reference.target_unit].segments + assert segment.cached == "The clause" and not plan.refs[reference.target_unit].exact # what the runs show + [found] = [d for d in diagnostics if d.rule == "render-docx-reference"] + assert "'The clause'" in found.message and "'The clause'" in found.message + + +def test_the_text_of_the_runs_has_a_blanks_prompt_and_a_hard_break() -> None: + from legaldown_render.resolve.docx_plan import _shown + from legaldown_render.tree import Blank, Emphasis, HardBreak, SoftBreak, Text + + assert _shown((Text("a"), SoftBreak(), Emphasis((Text("b"),)), HardBreak(), Blank("x", "[_]", "How much"), + Blank("y", "[_]"))) == "a b\n[_] (How much)[_]" + + +# -- One finding for references that fail alike ---------------------------------------------------------------------- + + +def test_references_that_fail_in_the_same_way_are_one_finding_naming_the_first_and_how_many_more() -> None: + body = "# A {#a}\n\n### B {#b}\n\n### C {#c}\n\n# D {#d}\n\n### E {#e}\n\n{{ref: b}} {{ref: c}} {{ref: e}}\n" + tree, plan, diagnostics = planned(body, style="outline") + found = [d for d in diagnostics if d.rule == "render-docx-reference"] + assert len(found) == 1 and found[0].message.startswith("The cross-reference 'I.A.1' holds the number") + assert found[0].message.endswith(" 2 more cross-references are written the same way.") + assert all(plan.refs[t].ghost for t in ("s2", "s3", "s5")) + assert found[0].line == tree.sections[-1].blocks[-1].line # the lowest line of the three: all in one paragraph + + +def test_references_that_are_fixed_for_the_same_reason_are_one_finding() -> None: + levels = [{"counter": "lower-alpha", "label": "({n})", "ref": "{n}"}] + tree, plan, diagnostics = planned("# A\n\n- x {#x}\n- y {#y}\n- z {#z}\n\n{{ref: x}} {{ref: y}} {{ref: z}}\n", + overrides={"enumeration.levels": levels}) + [only] = [d for d in diagnostics if d.rule == "render-docx-reference"] + assert only.message.endswith(" 2 more cross-references fail in the same way.") + assert all(r.fixed for r in plan.refs.values()) and not any(r.ghost for r in plan.refs.values()) + tree, plan, diagnostics = planned("# A\n\n- x {#x}\n- y {#y}\n\n{{ref: x}} {{ref: y}}\n", + overrides={"enumeration.levels": levels}) + [only] = [d for d in diagnostics if d.rule == "render-docx-reference"] + assert only.message.endswith(" 1 more cross-reference fails in the same way.") + + +@pytest.mark.parametrize(("levels", "switch", "phrases"), [ + # delimiters in the label that the reference leaves out + ([{"counter": "lower-alpha", "label": "({n})", "ref": "{n}"}], "documented", + ["Word draws its number as '(a)'", "text or punctuation around the counter that the reference does not have", + "Word is not documented to leave such punctuation out with its \\t switch", + "put it in `ref`, for example `({n})`"]), + ([{"counter": "decimal", "label": "{n}:", "ref": "{n}"}], "guess", + ["Word draws its number as '1:'", "not documented to leave such punctuation out"]), + # an abbreviation with a period: the setting that guesses leaves it out + ([{"counter": "decimal", "label": "No. {n}", "ref": "{n}"}], "documented", + ["Word draws its number as 'No. 1'", "not documented to leave a period after letters out", + "docx.reference_text_switch: guess expects it to"]), + ([{"counter": "decimal", "label": "No. {n}", "ref": "{n}"}], "off", + ["Word draws its number as 'No. 1'", "Word's \\t switch, which could leave it out, is off (docx.reference_text_switch)"]), + # a label that carries the counter of the section, which the reference writes itself + ([{"counter": "decimal", "label": "{section}.{n}", "ref": "{n}"}], "documented", + ["Word draws its number as '1.1'", "holds the counters of other units that the reference writes itself before it", + "a field returns the whole number or none of it"]), +]) +def test_the_reason_a_reference_is_fixed_names_what_the_style_could_change( + levels: list, switch: str, phrases: list[str]) -> None: + result = render(FRONT + "# A\n\n- x {#x}\n\nSee {{ref: x}}.\n", format="text", overrides={"enumeration.levels": levels}) + plan, diagnostics = plan_docx(result.tree, text_switch=switch) # type: ignore[arg-type] + [found] = [d for d in diagnostics if d.rule == "render-docx-reference"] + assert all(phrase in found.message for phrase in phrases), found.message + assert "written as the text" in found.message and "not a field" in found.message + assert all(r.fixed for r in plan.refs.values()) + + +def test_a_reference_below_a_skipped_level_whose_number_Word_draws_with_another_units_counter_says_so() -> None: + # the paragraph of the section after the skip is numbered 1.1.1.1 by Word too, but through the counter of the + # paragraph before the skip (level 2 of the heading list), not of the level the document skips + body = ("# Heading\n\nA paragraph. {#a1}\n\n### Heading {#h3}\n\nA paragraph. {#a22}\n\nA third. {#a23}\n\n" + "Refs: {{ref: a1}} {{ref: a22}}\n") + tree, plan, diagnostics = planned(body, overrides=PARAGRAPHS) + [reference] = [d for d in diagnostics if d.rule == "render-docx-reference"] + assert "a heading level is skipped before it (§4.1) and Word's counter for that level belongs to another unit" in reference.message + assert "style" not in reference.message and "label" not in reference.message + [label] = [d for d in diagnostics if d.rule == "render-docx-section-label"] + # what Word draws is not stated as if it were different from the renderer's: it is the same text, from another counter + assert "holds the number of a heading level that the document skips" in label.message + assert "another unit" in label.message and "only as long as that counter" in label.message + assert label.message.endswith(" 2 more places do the same.") # the three paragraphs, one finding + + +def test_a_level_nothing_counted_before_a_deeper_item_is_read_higher_by_libreoffice_and_the_plan_says_so() -> None: + # c is an ordered item (another list), d nested in it is on the enumerated list's level 4 with the level 3 above it + # never numbered there; e and f are the first items of that level 3: 1 and 2 in Word, 2 and 3 in LibreOffice + tree, plan, diagnostics = planned("# A\n\n- a\n - b\n 1. c\n - d\n - e\n - f\n") + e, f = [i for i in items(tree) if i.label in ("(A)", "(B)")] + assert [plan.units[i.unit].count for i in (e, f)] == [1, 2] + [found] = [d for d in diagnostics if d.rule == "render-docx-libreoffice-count"] + assert found.level == "info" and found.line == e.line + assert "the item '(A)' (the first, and 1 more) reads 'A' in Word and 'B' there" in found.message + assert "Nothing in the file is wrong for Word" in found.message + # without the deeper item first, nothing differs + _, _, diagnostics = planned("# A\n\n- a\n - b\n 1. c\n - e\n - f\n") + assert "render-docx-libreoffice-count" not in rules(diagnostics) + + +def test_the_labels_of_the_lists_of_one_chain_that_lose_counters_are_one_finding() -> None: + body = ("# A\n\n- one\n 1. ordered\n - three\n - four\n - three b\n - four c\n" + " 1. ordered two\n - x\n - y\n") + _, plan, diagnostics = planned(body, **ORDERED_PARENT) + found = [d for d in diagnostics if d.rule == "render-docx-section-label"] + assert len(found) == 1 and found[0].message.endswith(" 2 more places do the same.") + assert sum(next(a.kind for a in plan.abstract_nums if a.id == n.abstract_id) == "detached" + for n in plan.nums) == 3 + + +def test_every_reference_that_carries_the_counters_of_other_units_is_one_finding_whatever_its_shape() -> None: + levels = [{"counter": "decimal", "label": "{section}.{n}", "ref": "{n}"}] + body = "# A\n\n- x {#x}\n\n## B\n\n- y {#y}\n\nSee {{ref: x}} {{ref: y}}.\n" + _, plan, diagnostics = planned(body, overrides={"enumeration.levels": levels}) + found = [d for d in diagnostics if d.rule == "render-docx-reference"] + assert len(found) == 1 and found[0].message.endswith(" 1 more cross-reference fails in the same way.") + assert all(r.fixed for r in plan.refs.values()) + + +# -- The cover on its own ---------------------------------------------------------------------------------- + + +def field(owner: str, *tokens: object, number: bool = True, suppress: bool = False) -> Field: + return Field(owner, number, suppress, tuple(tokens)) # type: ignore[arg-type] + + +def test_fewest_fields_win_then_fewest_switches_then_fewest_literals() -> None: + designation = [Count("a", "1"), Lit("."), Count("b", "2")] + whole = field("b", Count("a", "1"), Lit("."), Count("b", "2")) + parts = [field("a", Count("a", "1")), field("b", Count("b", "2"))] + assert cover(designation, [*parts, whole]) == [whole] + assert cover(designation, parts) == [parts[0], ".", parts[1]] + plain = field("a", Lit("Article "), Count("a", "I")) + bare = field("a", Count("a", "I"), suppress=True) + assert cover([Count("a", "I")], [plain, bare]) == [bare] + assert cover([Lit("Article "), Count("a", "I")], [plain, bare]) == [plain] # the words are in the field + assert cover([Lit("Article "), Count("a", "I")], [bare]) == ["Article ", bare] # or come from the literal + assert cover([], []) == [] + + +def test_a_literal_stands_for_delimiters_only_never_for_a_counter_or_a_title() -> None: + assert cover([Count("a", "1")], []) is None + assert cover([Title("a", "Alpha")], []) is None + assert cover([Lit("(")], []) == ["("] + assert cover([Title("a", "Alpha")], [field("a", Title("a", "Alpha"), number=False)]) is not None + + +def test_a_field_fits_where_the_counters_belong_to_the_same_units_whatever_their_text() -> None: + designation = [Count("a", "ab")] + assert cover(designation, [field("a", Count("a", "bb"))]) == [field("a", Count("a", "bb"))] + assert cover(designation, [field("a", Count("other", "ab"))]) is None + + +def test_the_atom_nothing_can_stand_for_is_found() -> None: + designation = [Count("a", "I"), Lit("."), Count("c", "1"), Lit("."), Count("b", "1")] + fields = [field("a", Count("a", "I")), field("b", Count("b", "1"))] + assert cover(designation, fields) is None and uncovered(designation, fields) == 2 + assert uncovered([Count("a", "I")], fields) is None + + +def test_the_counter_of_a_skipped_heading_level_is_literal_text_since_no_unit_holds_it() -> None: + ghost = Count(Ghost("s2", 1), "A") + designation = [Count("a", "I"), Lit("."), ghost, Lit("."), Count("b", "1")] + fields = [field("a", Count("a", "I")), field("b", Count("b", "1"))] + assert cover(designation, fields) == [fields[0], ".A.", fields[1]] + # a field that returns the ghost's text covers it too (the decimal scheme's number holds every level's counter) + whole = field("b", Count("a", "1"), Lit("."), Count(Ghost("s2", 1), "1"), Lit("."), Count("b", "1")) + assert cover([Count("a", "1"), Lit("."), Count(Ghost("s2", 1), "1"), Lit("."), Count("b", "1")], [whole]) == [whole] + assert uncovered(designation, fields) is None + # any other counter that no field returns is still not literal text + assert cover([Count("a", "I"), Lit("."), Count("c", "A")], fields) is None + + +def test_the_switch_that_drops_text_is_modelled_only_where_its_result_is_known() -> None: + one, two = Count("a", "1"), Count("b", "2") + # the words around a number go (Microsoft: "Section 1.01" gives "1.01"), the delimiter between its counters stays + assert suppressed([Lit("Article "), Count("a", "I")]) == (Count("a", "I"),) + assert suppressed([Lit("§ "), Count("a", "3")]) == (Count("a", "3"),) + assert suppressed([Lit("Section "), one, Lit("."), two]) == (one, Lit("."), two) + assert suppressed([one, Lit("."), two]) == (one, Lit("."), two) # nothing to drop: no field of its own is needed + assert suppressed([Lit("Part "), one, Lit(" of "), two]) is None # words between counters: only `guess` drops them + # a delimiter around the counters, or mixed with words between them, is not documented + assert suppressed([Lit("("), Count("a", "a"), Lit(")")]) is None + assert suppressed([Count("a", "a"), Lit(")")]) is None + assert suppressed([one, Lit(":")]) is None + assert suppressed([Lit("No. "), one]) is None + assert suppressed([one, Lit(" to."), two]) is None + assert suppressed([Lit("Article")]) is None and suppressed([]) is None + + +def test_the_switch_setting_says_what_word_may_be_expected_to_do() -> None: + one, two = Count("a", "1"), Count("b", "2") + # off: never + assert suppressed([Lit("Article "), Count("a", "I")], "off") is None + assert suppressed([Lit("Section "), one, Lit("."), two], "off") is None + # guess: a period after the letters that lead a number goes too; no other delimiter does, and only in leading text + assert suppressed([Lit("No. "), one], "guess") == (one,) + assert suppressed([Lit("Čl. "), one], "guess") == (one,) + assert suppressed([Lit("Art. "), one, Lit("."), two], "guess") == (one, Lit("."), two) + assert suppressed([Lit("("), one], "guess") is None and suppressed([Lit("1. "), one], "guess") is None + assert suppressed([one, Lit(" Art.")], "guess") is None and suppressed([Lit("No: "), one], "guess") is None + assert suppressed([Lit("Article "), Count("a", "I")], "guess") == (Count("a", "I"),) + assert suppressed([Lit("Part "), one, Lit(" of "), two], "guess") == (one, two) # words between counters go too + assert suppressed([one, Lit(" to."), two], "guess") is None # a mixture with a delimiter never does + + +# -- Bookmarks ---------------------------------------------------------------------------------------------- + + +def test_bookmark_names_have_a_prefix_for_what_they_mark_and_write_dashes_as_underscores() -> None: + names = BookmarkNames() + assert names.anchor("ld_", "delay-cause") == "ld_delay_cause" + assert names.anchor("lddef_", "business-day") == "lddef_business_day" + assert names.anchor("ldatt_", "schedule-a") == "ldatt_schedule_a" + assert names.fixed("_LdAttachments") == "_LdAttachments" + assert names.toc() == "_Toc100000001" and names.toc() == "_Toc100000002" + + +def test_a_name_that_cannot_be_used_is_a_hidden_one_counted_in_the_order_they_are_given() -> None: + names = BookmarkNames() + assert names.anchor("ld_", "a-b") == "ld_a_b" + assert names.anchor("ld_", "a_b") == "_Ld1" # the same name once the dash is an underscore: not lossless + assert names.anchor("ld_", "A-B") == "_Ld2" # the same ignoring case + assert names.anchor("ld_", "clanek-č") == "_Ld3" # a letter a name cannot hold + assert names.anchor("ld_", "x" * 37) == "ld_" + "x" * 37 # 40 characters + assert names.anchor("ld_", "y" * 38) == "_Ld4" # 41 + assert names.hidden() == "_Ld5" + + +def test_hidden_names_are_numbered_in_document_order_across_anchors_and_units() -> None: + tree, plan, _ = planned("# A\n\n- x\n - y {#y}\n\nSee {{ref: y}} and " + "{{ref: " + "z" * 50 + "}}.\n") + assert plan.units["u1"].bookmarks == ("_Ld1",) + + +def test_an_anchor_too_long_for_a_name_is_hidden_and_still_found_by_the_plan() -> None: + anchor = "a-very-long-identifier-that-goes-on-and-on-past-forty" + tree, plan, _ = planned(f"# A {{#{anchor}}}\n\n{{{{ref: {anchor}}}}}\n") + assert plan.anchors == {anchor: "_Ld1"} and plan.units["s1"].bookmarks == ("_Ld1",) + assert [s.bookmark for s in plan.refs["s1"].segments if isinstance(s, FieldSeg)] == ["_Ld1"] + + +def test_the_names_do_not_depend_on_what_was_rendered_before() -> None: + body = "# A {#a-b}\n\n- x {#x}\n\nSee {{ref: x}}.\n" + first = planned(body)[1] + render(FRONT + "# Other {#a}\n", format="text") + assert planned(body)[1] == first + + +def test_definitions_and_attachments_have_bookmarks_too() -> None: + result = render((DOCUMENTS / "features.lgd").read_text(encoding="utf-8"), format="text") + plan, _ = plan_docx(result.tree) + assert {a: n for a, n in plan.anchors.items() if not n.startswith("ld_")} == { + "def:agreement": "lddef_agreement", "def:services": "lddef_services", "def:business-day": "lddef_business_day", + "schedule-a": "ldatt_schedule_a", "exhibit-1": "ldatt_exhibit_1", "ld:attachments": "_LdAttachments"} + assert plan.anchors["services"] == "ld_services" and plan.anchors["delay-cause"] == "ld_delay_cause" + assert all(plan.units[unit.unit].bookmarks == (plan.anchors[unit.anchor],) + for unit in units(result.tree) if unit.anchor) + + +def test_every_name_is_valid_and_unique_ignoring_case() -> None: + for name in ("features.lgd", "template.lgd"): + for settings in VARIANTS.values(): + result = render((DOCUMENTS / name).read_text(encoding="utf-8"), format="text", **settings) + plan, _ = plan_docx(result.tree) + assert_names_are_sound(plan) + + +def assert_names_are_sound(plan: DocxPlan) -> None: + names = [*(n for u in plan.units.values() for n in u.bookmarks), *plan.anchors.values(), + *(t.bookmark for t in plan.toc)] + names = list(dict.fromkeys(names)) # an anchor's name is the first of its unit's + assert all(re.fullmatch(r"[A-Za-z_][A-Za-z0-9_]{0,39}", n) for n in names), names + assert len({n.lower() for n in names}) == len(names), names + + +# -- The table of contents --------------------------------------------------------------------------------------- + + +ATTACHED = """--- +title: T +attachments: + - id: schedule-a + title: Schedule A + file: a.lgd + - id: exhibit-1 + title: Exhibit 1 + file: b.pdf +--- +""" + + +def test_a_contents_entry_names_the_section_it_lists() -> None: + result = render(ATTACHED + "# A\n\n## B\n\n### C\n\n# D\n", format="text", + overrides={"contents.enabled": True, "contents.depth": 2}) + assert [(e.level, e.unit) for e in result.tree.contents] == [ + (1, "s1"), (2, "s2"), (1, "s4"), (1, None), (2, None), (2, None)] + result = render(ATTACHED + "# A\n", format="text") + assert result.tree.contents == () + + +def test_the_contents_come_from_the_numbers_word_draws() -> None: + body = "# Alpha\n\n## Beta\n\n### Gamma\n\n# Delta\n" + tree, plan, _ = planned(body, ATTACHED, overrides={"contents.enabled": True, "contents.depth": 2}) + assert [(e.unit, e.level, e.number, e.sep, e.condition_label, e.cached_page) for e in plan.toc] == [ + ("s1", 1, "1.", "\t", None, ""), ("s2", 2, "1.1", "\t", None, ""), ("s4", 1, "2.", "\t", None, ""), + (None, 1, "", "", None, ""), (None, 2, "", "", None, ""), (None, 2, "", "", None, "")] + assert [e.title for e in plan.toc[:3]] == [tree.sections[0].title, tree.sections[1].title, tree.sections[3].title] + assert [e.bookmark for e in plan.toc] == [f"_Toc10000000{k}" for k in range(1, 7)] + assert [plan.units[s].toc_bookmark for s in ("s1", "s2", "s3", "s4")] == [ + "_Toc100000001", "_Toc100000002", None, "_Toc100000003"] + assert [e.title for e in plan.toc[3:]] == [tree.contents[3].title, tree.contents[4].title, tree.contents[5].title] + assert_names_are_sound(plan) + + +def test_a_contents_entry_has_the_separator_the_level_has_and_the_number_of_the_style() -> None: + body = "# Alpha\n\n## Beta\n" + _, plan, _ = planned(body, overrides={"contents.enabled": True, **ARTICLE}) + assert [(e.number, e.sep) for e in plan.toc] == [("Article I", " "), ("I.1", "\t")] + _, plan, _ = planned(body, overrides={"contents.enabled": True, "numbering.scheme": "legal-outline"}) + assert [(e.number, e.sep) for e in plan.toc] == [("I.", "\t"), ("A.", "\t")] + _, plan, _ = planned(body, overrides={"contents.enabled": True, **NONE}) + assert [(e.number, e.sep) for e in plan.toc] == [("", ""), ("", "")] + _, plan, _ = planned(body, overrides={"contents.enabled": True, "contents.attachments": False}, front=ATTACHED) + assert [e.unit for e in plan.toc] == ["s1"] * 0 + ["s1", "s2"] + + +def test_the_contents_carry_the_condition_label_of_a_conditional_heading() -> None: + _, plan, _ = planned("# A {when=forum:courts}\n\n# B\n", ALTERNATIVES, overrides={"contents.enabled": True}) + assert [e.condition_label for e in plan.toc] == ["Only if: forum:courts", None] + + +def test_there_are_no_entries_unless_the_style_asks_for_contents() -> None: + _, plan, _ = planned("# A\n", ATTACHED) + assert plan.toc == () and plan.units["s1"].toc_bookmark is None + + +# -- The property: the plan's references read as the renderer wrote them, or a diagnostic says why not --------- + + +def with_anchors(source: str) -> str: + """*source* with an anchor on every heading, item, quoted item and plain paragraph the random documents hold.""" + out, count = [], 0 + for line in source.splitlines(): + if re.match(r"^\s*(#+ Heading|[-]|\d+\.) ", line) or line == "A paragraph." or re.match(r"^\s*> - ", line): + count += 1 + line += f" {{#a{count}}}" + out.append(line) + return "\n".join(out) + + +def with_references(source: str, shift: int = 0) -> str: + """*source* with a paragraph at its end that refers to the anchors in it (at most 40, evenly spread, starting at + the *shift*-th: resolving a reference costs more than numbering a unit, and documents of different shifts refer + to different units).""" + anchors = list(dict.fromkeys(re.findall(r"\{#([a-z][a-z0-9-]*)", source))) + stride = -(-len(anchors) // 40) or 1 + anchors = anchors[shift % stride::stride] + return source + "\n\nRefs: " + " ".join(f"{{{{ref: {anchor}}}}}" for anchor in anchors) + "\n" + + +def literal_text_of_a_cover(tree: RenderTree, segments: Sequence[Segment]) -> str: + """The literal text of *segments* without the text of ``references.format`` around the designation. A field may + stand for that text (a number that holds the same words: ``Article I`` for ``Article {designation}``): it is then + in the cached text of the fields.""" + text = "".join(s.text for s in segments if isinstance(s, Literal_)) + fields = "".join(s.cached for s in segments if isinstance(s, FieldSeg)) + for part in tree.formats.reference.split("{designation}"): + if part and part not in text: + assert part in fields, (part, text, fields) + continue + text = text.replace(part, "", 1) + return text + + +def assert_references_as_resolved(result, plan, diagnostics) -> None: # noqa: ANN001 + """For every reference: it is a cover of fields and delimiters, with no counter written as text; and it reads as + the renderer's text (``exact``, and the cached results say so), or the plan says why not: a unit on the way is + numbered differently in Word (a diagnostic of the numbering says so, as the property of the numbering above has + it), or a ``render-docx-reference`` finding names it. A reference that has no cover is the renderer's text, marked + ``fixed`` and reported as such. Every field points at a bookmark the plan makes, and the names are sound.""" + tree = result.tree + assert_names_are_sound(plan) + nodes = {u.unit: u for u in units(tree)} + counters = {node.unit: counter for node, counter in numbered(tree)} + marked = {n for u in plan.units.values() for n in u.bookmarks} + reported = [d for d in diagnostics if d.rule == "render-docx-reference"] + texts = {i.target_unit: i.text for i in iter_tree_inlines(tree) if isinstance(i, CrossRef) and i.target_unit} + assert set(plan.refs) == set(texts) + # The ``ref`` templates write delimiters only: any letter or digit that is literal text in a cover is a counter + # that should have been a field. + templates = [f.ref for f in (*tree.formats.headings, *tree.formats.enumerated, tree.formats.renumber, + tree.formats.paragraphs) if f is not None] + delimiters_only = not any(c.isalnum() for t in templates for c in re.sub(r"\{(n|path|section)\}", "", t)) + # The one counter a cover may write as text is that of a heading level the document skips (a Core error), which no + # unit holds, and the plan says which it wrote. + skipping = any("skipped-level" in u.cause for u in plan.units.values()) + needing_a_finding = 0 + fixed = ghosts = 0 + for target, text in texts.items(): + reference = plan.refs[target] + assert all(s.bookmark in marked for s in reference.segments if isinstance(s, FieldSeg)) + assert reference.exact == (cached(reference.segments) == text) + for segment in reference.segments: + if isinstance(segment, FieldSeg) and segment.number and not segment.suppress_text: + assert segment.cached == plan.units[next(u for u, p in plan.units.items() + if segment.bookmark in p.bookmarks)].word_ref + if reference.fixed: + fixed += 1 + assert reference.segments == ((Literal_(text),) if text else ()) and reference.exact + continue + assert not reference.ghost or skipping, (target, text) + if delimiters_only: + # the letters and digits written as text are exactly the counters the plan says it wrote as text + written = "".join(c for c in literal_text_of_a_cover(tree, reference.segments) if c.isalnum()) + assert written == "".join(c for c in reference.ghost if c.isalnum()), ( + target, text, show(plan, reference.segments)) + ghosts += bool(reference.ghost) + assert any(isinstance(s, FieldSeg) for s in reference.segments) or not text, (target, text) + if reference.exact: + continue + chain, unit = [], target + while unit is not None: + chain.append(unit) + unit = nodes[unit].parent_unit + differs = False + for unit in chain: + mine, node = plan.units[unit], nodes[unit] + if node.number is not None and mine.count is not None: + differs |= mine.count != node.number or ( + unit in counters and word_counter(node.number, NUM_FMT[counters[unit]]) + != format_counter(node.number, counters[unit])) + needing_a_finding += not differs + # Every fixed text and every other difference is in a finding, named or counted among "N more". + counted = 0 + for finding in reported: + more = re.search(r" (\d+) more cross-references? (?:fails? in|are written|is written) the same way\.$", + finding.message) + counted += 1 + (int(more.group(1)) if more else 0) + assert needing_a_finding + fixed + ghosts <= counted <= len(texts) + if fixed: + assert any("not a field" in d.message for d in reported) + if ghosts: + assert any("heading level that the document skips" in d.message for d in reported) + + +@pytest.mark.parametrize("variant", VARIANTS) +@pytest.mark.parametrize("name", ["features.lgd", "template.lgd"]) +def test_the_test_documents_references_are_planned_as_resolved(name: str, variant: str) -> None: + source = (DOCUMENTS / name).read_text(encoding="utf-8") + assert_references_as_resolved(*planned_run(source, variant)) + + +@pytest.mark.conformance +@pytest.mark.skipif(SPEC is None, reason="LegalDown specification checkout not found") +@pytest.mark.parametrize("variant", VARIANTS) +def test_the_specifications_documents_references_are_planned_as_resolved(variant: str) -> None: + for path in _spec_documents(): + try: + run = planned_run(path.read_text(encoding="utf-8"), variant) + except DocumentError: + continue + try: + assert_references_as_resolved(*run) + except AssertionError as error: + raise AssertionError(f"{path.relative_to(SPEC)}: {error}") from error + + +@pytest.mark.parametrize("variant", VARIANTS) +def test_random_documents_references_are_planned_as_resolved(variant: str) -> None: + for source, result, plan, diagnostics in random_runs(variant): + try: + assert_references_as_resolved(result, plan, diagnostics) + except AssertionError as error: + raise AssertionError(f"{error}\n--- document ---\n{source}") from error + + +@pytest.mark.parametrize("variant", VARIANTS) +def test_random_alternatives_references_are_planned_as_resolved(variant: str) -> None: + for source, result, plan, diagnostics in alternative_runs(variant): + try: + assert_references_as_resolved(result, plan, diagnostics) + except AssertionError as error: + raise AssertionError(f"{error}\n--- document ---\n{source}") from error + + +def test_the_random_documents_reach_every_kind_of_cover() -> None: + shapes: set[str] = set() + inexact = switches = 0 + for variant in VARIANTS: + for _, _, plan, _ in random_runs(variant): + for reference in plan.refs.values(): + shapes.add("".join("L" if isinstance(s, Literal_) else "F" for s in reference.segments)) + inexact += not reference.exact + switches += any(isinstance(s, FieldSeg) and s.suppress_text for s in reference.segments) + assert {"F", "FF", "FFF", "FFFF", "FLF", "FLFL", "FLFLF"} <= shapes + assert inexact > 0 and switches > 0 diff --git a/tests/test_docx_roundtrip.py b/tests/test_docx_roundtrip.py new file mode 100644 index 0000000..5099763 --- /dev/null +++ b/tests/test_docx_roundtrip.py @@ -0,0 +1,1218 @@ +"""LibreOffice as the oracle of the numbering and the cross-references (docs/architecture.md, "The DOCX planner"). + +Every document of the test corpus (the test documents, the specification's examples and valid fixtures, and +documents written to stress the numbering) is rendered to DOCX under the default, outline, continental and mixed +schemes and under ``none`` (where a reference is a field for the text of a heading; the specification's documents +take turns, each under half of them), then changed so that the file cannot be right by accident: + +* every cached result of a field is replaced by ``XXX``, so that what LibreOffice shows after it has updated the + fields is what it computed from the numbering definitions and the bookmarks, not what the writer cached; +* a marker ``⟦u3⟧`` leads the paragraph of every numbered unit, so that the number LibreOffice drew before it can be + read from the exported text, and a paragraph per ``{{ref:}}`` target is added, made of exactly the fields the plan + gives that reference, preceded by ``⟦ref u3⟧``. + +Every body paragraph's text must also be what the file cached for it, and a second round trip makes the edits a +lawyer makes (an item, a heading and an ordered item inserted before the targets, in LibreOffice) and updates the +fields: every reference and number must read as the renderer writes them for the edited source. The table of +contents is not part of this comparison (its entries are in ``test_docx_matter.py`` and ``test_docx_acceptance.py``, +which also compares whole documents with the plain-text writer's). + +The exported text is then compared with what the renderer wrote (the labels and designations that the text writer +prints), **modulo the gaps the plan documents**, each of which the plan reports with a diagnostic: + +* a unit whose number Word draws differently from the renderer's (a letter counter past 26, a label that needs the + section's number where the heading list cannot show it, a list deeper than Word's nine levels): LibreOffice must + agree with the plan (the simulator), and the plan must say so; a later alternative that shares a number has no number + of its own (``numId`` 0) and shows the holder's as a field, which LibreOffice updates to the renderer's; +* ``\\t`` is ignored by LibreOffice (it is Word's switch), so a reference built with it reads there as + ``FieldSeg.cached_lo`` says; +* LibreOffice fills a skipped heading level with its start value, which is what the simulator models for headings. + +Skipped when LibreOffice or its Python bindings are missing, a failure when ``LEGALDOWN_REQUIRE_DOCX_TOOLS`` is set. +""" +from __future__ import annotations + +import difflib +import functools +import re +import xml.etree.ElementTree as ET +from dataclasses import dataclass, replace + +import pytest +from conftest import DOCUMENTS, FRONT, spec_root +from docx_support import Exported, W, libreoffice_edit, libreoffice_export, parts, repackage, require_libreoffice +from test_docx_plan import ALTERNATIVES, ORDERED_PARENT, PATH_ITEMS, units + +from legaldown_render import DocumentError, render +from legaldown_render.resolve.docx_plan import DocxPlan, FieldSeg, Literal_, plan_docx +from legaldown_render.tree import CrossRef, ListItem, Paragraph, RenderTree, Section, iter_tree_inlines, plain_text +from legaldown_render.writers.docx.ooxml import Bookmarks, p, text_run +from legaldown_render.writers.docx.package import Relationships +from legaldown_render.writers.docx.runs import InlineWriter + +pytestmark = pytest.mark.libreoffice + +VARIANTS: dict[str, dict] = { + "default": {}, + "outline": {"style": "outline"}, + "continental": {"style": "continental"}, + "mixed": {"overrides": {"numbering.scheme": "mixed"}}, + "none": {"overrides": {"numbering.scheme": "none"}}, +} + +#: Documents that stress the numbering and the references, written for this test. +STRESS: dict[str, str] = { + "nested": FRONT + """ +# Alpha {#alpha} + +Intro paragraph one. {#p1} + +Intro paragraph two. {#p2} + +- first item {#i1} +- second item {#i2} + - nested one {#n1} + - nested two {#n2} + - deep one {#d1} +- third item {#i3} + +## Beta {#beta} + +### Gamma {#gamma} + +1. ordered one {#o1} +2. ordered two {#o2} + - bullet in ordered {#b1} + +#### Delta {#delta} + +##### Epsilon {#eps} + +- eps item {#ei} + +# Second {#second} + +- a {#sa} + +Refs: {{ref: alpha}} {{ref: p1}} {{ref: i1}} {{ref: n2}} {{ref: d1}} {{ref: beta}} {{ref: gamma}} {{ref: o2}} +{{ref: b1}} {{ref: delta}} {{ref: eps}} {{ref: ei}} {{ref: second}} {{ref: sa}} {{ref: i3}} {{ref: p2}} +""", + "lists": FRONT + """ +# A {#a} + +- x {#x} + - n1 {#n1} + - n2 {#n2} + + Text between. + + - n3 {#n3} + 1. o1 {#o1} + 1. o2 {#o2} + > - quoted {#q1} + > - quoted two +- y {#y} + 1. oy {#oy} + - ny {#ny} + +Para. + +- z {#z} + +Ref {{ref: n3}} {{ref: o2}} {{ref: oy}} {{ref: ny}} {{ref: z}} {{ref: q1}} +""", + "preamble": FRONT + """- pre one {#p1} +- pre two {#p2} + - pre nested {#pn} + +# A {#a} + +- in {#in1} + +Ref {{ref: p1}} {{ref: pn}} {{ref: in1}} +""", + "letters": FRONT + "# A {#a}\n\n" + "\n".join(f"- item {i} {{#m{i}}}" for i in range(1, 32)) + ( + "\n\nRef {{ref: m1}} {{ref: m26}} {{ref: m27}} {{ref: m28}} {{ref: m31}}\n"), + "level five": FRONT + """ +# A + +## B + +### C + +#### D + +##### E {#e} + +- five item {#fi} +- five item two {#fj} + +Para {#pp} + +Ref {{ref: fi}} {{ref: fj}} {{ref: e}} {{ref: pp}} +""", + "skipped": FRONT + """ +# One {#one} + +### Deep {#deep} + +## Two {#two} + +#### Deeper {#deeper} + +Ref: {{ref: deep}} {{ref: two}} {{ref: deeper}} +""", + "helpers": FRONT + """ +# A {#a} + +- unanchored + - also unanchored + - anchored {#deep} + +- unanchored ordered + 1. unanchored nested + 1. anchored too {#too} + +Ref {{ref: deep}} {{ref: too}} +""", + "cycles": FRONT + """ +# A {{ref: b}} {#a} + +# B {{ref: a}} {#b} + +# C {{ref: c}} {#c} + +# D {{ref: e}} {#d} + +# E plain {#e} + +- item in {{ref: it}} {#it} + +See {{ref: a}} {{ref: b}} {{ref: c}} {{ref: d}} {{ref: it}}. +""", + "alternatives": ALTERNATIVES + """ +# Intro {#intro} + +# Disputes {#d when=forum:courts} + +## Sub one {#s1} + +- courts item {#ci when=forum:courts} +- arbitration item {#ci when=forum:arb} + - nested under alt {#an} +- common item {#cm} + +# Disputes {#d when=forum:arb} + +## Sub alt {#sa} + +# Last {#last} + +Ref {{ref: d}} {{ref: s1}} {{ref: sa}} {{ref: ci}} {{ref: an}} {{ref: cm}} {{ref: last}} +""", + "alternatives in threes": ALTERNATIVES + """ +# Intro {#intro} + +Para one. {#pc when=forum:courts} + +Para arbitration. {#pc when=forum:arb} + +Para expert. {#pc when=forum:expert} + +Para next. {#pn} + +# Disputes {#d when=forum:courts} + +## Sub one {#s1} + +## Sub two {#s2} + +# Disputes {#d when=forum:arb} + +## Sub alt {#sa} + +- quoted item {#q} + +# Disputes {#d when=forum:expert} + +## Expert one {#e1} + +## Expert two {#e2} + +## Expert three {#e3} + +# Items {#items} + +- courts {#ci when=forum:courts} + - courts child {#cc} +- arbitration {#ci when=forum:arb} + - arbitration child one {#ac1} + - arbitration child two {#ac2} +- expert {#ci when=forum:expert} +- common {#cm} + - common child {#cmc} + +# Last {#last} + +## Last one {#l1} + +Ref {{ref: d}} {{ref: s1}} {{ref: s2}} {{ref: sa}} {{ref: q}} {{ref: e1}} {{ref: e3}} {{ref: ci}} {{ref: cc}} +{{ref: ac1}} {{ref: ac2}} {{ref: cm}} {{ref: cmc}} {{ref: pc}} {{ref: pn}} {{ref: last}} {{ref: l1}} +""", + "nested titles": FRONT + """ +# A {{ref: i}} {#a} + +- x {#i} + +# B {{ref: a}} {#b} + +# C {{ref: b}} and {{ref: a}} {#c} + +Refs {{ref: a}} {{ref: b}} {{ref: c}} {{ref: i}} +""", + "qualified": FRONT + """ +# Alpha {#alpha} + +Intro. {#p1} + +- first {#i1} + - nested one {#n1} + - nested two {#n2} + - deep {#d1} + - deeper {#d2} + - deepest {#d3} +- second {#i2} + 1. ordered {#o1} + 1. ordered two {#o2} + +Between. + +- third {#i3} + +## Beta {#beta} + +- beta item {#bi} + - beta nested {#bn} + +### Gamma {#gamma} + +#### Delta {#delta} + +##### Epsilon {#eps} + +- five item {#ei} + - five nested {#en} +- five two {#ej} + +# Second {#second} + +- x {#sx} + - y {#sy} + +Ref {{ref: i1}} {{ref: n2}} {{ref: d3}} {{ref: o2}} {{ref: i3}} {{ref: beta}} {{ref: bn}} {{ref: ei}} {{ref: en}} +{{ref: ej}} {{ref: sy}} {{ref: p1}} +""", +} + +#: Labels that show the path of an item (`{path}`): in a clause (the reserved levels of the heading list carry the clause's +#: and the section's counters), in a section of another level (the reserved level reads otherwise: detached levels), past +#: the reserved levels, and below an ordered item. +STRESS["paths"] = FRONT + """ +# Alpha {#alpha} + +- one {#i1} + - two {#i2} + - three {#i3} + - four {#i4} + - five {#i5} + - six {#i6} + - seven {#i7} +- uno {#j1} + - dos {#j2} + - tres {#j3} + +## Beta {#beta} + +- bee {#k1} + - kay {#k2} + - kay two {#k3} +- bee two {#k4} + - kay three {#k5} + +# Second {#second} + +- x {#x1} + - y {#y1} + +Refs: {{ref: i1}} {{ref: i2}} {{ref: i3}} {{ref: i4}} {{ref: i5}} {{ref: i6}} {{ref: i7}} {{ref: j2}} {{ref: j3}} +{{ref: k2}} {{ref: k3}} {{ref: k5}} {{ref: y1}} +""" +STRESS["ordered paths"] = FRONT + """ +# Alpha {#alpha} + +- one {#i1} + 1. ordered {#o} + - three {#i3} + - four {#i4} + - five {#i5} + - five b {#i6} + - four b {#i7} + - three b {#i8} + - four c {#i9} + 1. ordered two {#o2} + - x {#x} + - y {#y} +- two {#t} + 1. o3 {#o3} + - z {#z} + - w {#w} + +Refs: {{ref: i3}} {{ref: i4}} {{ref: i5}} {{ref: i6}} {{ref: i7}} {{ref: i9}} {{ref: y}} {{ref: w}} +""" + +#: Lists nested past Word's nine levels: each nine levels are a list of their own, restarted at a fixed value where they +#: begin, and numbered by structure inside. +STRESS["deep"] = FRONT + "# A {#a}\n\n" + "".join( + " " * d + f"- level {d} {{#l{d}}}\n" for d in range(12)) + " " * 8 + "- level 8 again {#l8b}\n" + ( + " " * 9 + "- level 9 again {#l9b}\n") + "\nRefs: " + " ".join( + f"{{{{ref: {anchor}}}}}" for anchor in ("l0", "l7", "l8", "l9", "l10", "l11", "l8b", "l9b")) + "\n" + +#: The stress documents that are shown under some variants only (each costs LibreOffice time): the continental style, where +#: the items ride the heading list, the default, and the default with numbered paragraphs. +FEW_VARIANTS: dict[str, tuple[str, ...]] = { + **{name: ("continental", "default", "paragraphs") for name in ("alternatives in threes", "nested titles", "qualified")}, + "paths": ("continental", "default"), "ordered paths": ("default",), "deep": ("continental", "default", "outline")} + +#: The variants that show some stress documents only. +ONLY_DOCS: dict[str, tuple[str, ...]] = { + "five levels": ("qualified",), "shared level": ("qualified", "nested"), "path items": ("paths", "lists", "deep"), + "ordered parent": ("ordered paths", "lists"), "section-short": ("nested", "qualified"), + "cl-guess": ("nested", "qualified")} + +#: The settings that give the numbering words and symbols: a field that returns the same words as the reference's format +#: stands for them, otherwise Word's ``\t`` switch drops them from the number. +WORDS: dict[str, dict] = { + "article": {"overrides": {"numbering.levels": [ + {"counter": "upper-roman", "label": "Article {n}", "ref": "{n}"}], "references.format": "Article {designation}"}}, + "section-sign": {"overrides": {"numbering.levels": [ + {"counter": "decimal", "label": "§ {n}", "ref": "{n}"}], "references.format": "§ {designation}"}}, + # the format says a word the label does not: Word's \t leaves the label's out and the format's is put back + "article-short": {"overrides": {"numbering.levels": [ + {"counter": "upper-roman", "label": "Article {n}", "ref": "{n}"}], "references.format": "Art. {designation}"}}, + # a number with a delimiter between its counters ("Section 1.1") that the format words differently: Word's \t keeps it + "section-short": {"overrides": {"numbering.levels": [ + {"counter": "upper-roman", "label": "Article {n}", "ref": "{n}"}, + {"counter": "decimal", "label": "Section {path}", "ref": ".{n}"}], "references.format": "Sec. {designation}"}}, + # a period after letters ("Čl. 1") that only the setting that guesses lets the switch drop + "cl-guess": {"overrides": {"numbering.levels": [ + {"counter": "decimal", "label": "Čl. {n}", "ref": "{n}"}, {"counter": "decimal", "label": "{path}", "ref": ".{n}"}], + "references.format": "čl. {designation}", "docx.reference_text_switch": "guess"}}, + "paragraphs": {"overrides": {"paragraphs.numbered": True}}, + # the items and the paragraphs of a section share the heading list's level: the numbers are restarted where the + # resolver counts them apart + # six levels of enumeration with no section in them but the first: a chain of four lists nested in a clause rides all + # four reserved levels of the heading list + "five levels": {"style": "continental", "overrides": {"enumeration.levels": [ + {"counter": "decimal", "label": "{section}.{n}", "ref": ".{n}"}, + {"counter": "lower-alpha", "label": "{n})", "ref": "({n})"}, + {"counter": "lower-roman", "label": "{n})", "ref": "({n})"}, + {"counter": "upper-alpha", "label": "({n})", "ref": "({n})"}, + {"counter": "decimal", "label": "[{n}]", "ref": "[{n}]"}, + {"counter": "lower-alpha", "label": "{n}.", "ref": "{n}"}]}}, + "shared level": {"style": "continental", "overrides": {"paragraphs.numbered": True, "paragraphs.format": { + "counter": "decimal", "label": "{section}.{n}", "ref": ".{n}"}}}, + "path items": PATH_ITEMS, + "ordered parent": ORDERED_PARENT, +} + +#: What a diagnostic of the plan has to say when Word numbers a unit differently from the renderer. +EXPLAINING = {"render-docx-section-label", "render-docx-letter-overflow", "render-docx-roman-overflow", + "render-docx-list-depth"} + +WRONG = "XXX" +OPEN, CLOSE = "⟦", "⟧" +MARKED = re.compile(rf"^(?P.*?){OPEN}(?P[^{CLOSE}]+){CLOSE}(?P.*)$") + + +# -- Making the files --------------------------------------------------------------------------------------------- + + +def _is_field_part(run: ET.Element) -> str | None: + """``begin``, ``separate`` or ``end`` for a run that holds that field character, else None.""" + mark = run.find(f"{W}fldChar") + return None if mark is None else mark.get(f"{W}fldCharType") + + +def seed_wrong_results(root: ET.Element, wrong: str = WRONG) -> int: + """Replace the cached result of every complex field under *root* by *wrong*; returns how many there are.""" + count = 0 + for paragraph in root.iter(f"{W}p"): + state = None + first: ET.Element | None = None + for run in list(paragraph.iter(f"{W}r")): + part = _is_field_part(run) + if part == "begin": + state, first = "instruction", None + count += 1 + elif part == "separate": + state = "result" + elif part == "end": + if state == "result" and first is None: + spare = ET.Element(f"{W}r") + ET.SubElement(spare, f"{W}t").text = wrong + run_list = list(paragraph) + paragraph.insert(run_list.index(run), spare) + state = None + elif state == "result": + text = run.find(f"{W}t") + if text is None: + continue + if first is None: + text.text, first = wrong, run + else: + paragraph.remove(run) + return count + + +def marked_paragraphs(root: ET.Element, tree: RenderTree, plan: DocxPlan) -> list[Section | ListItem | Paragraph]: + """The units whose paragraph carries a number: every heading and every unit the plan says is explicit, in the + order of the document, which is the order of the paragraphs of the body that carry a heading style or ``w:numPr``.""" + expected = [unit for unit in units(tree) if isinstance(unit, Section) or plan.units[unit.unit].explicit] + body = root.find(f"{W}body") + assert body is not None + found = [] + for paragraph in body.findall(f"{W}p"): + style = paragraph.find(f"{W}pPr/{W}pStyle") + heading = style is not None and re.fullmatch(r"Heading[1-5]", style.get(f"{W}val", "")) is not None + if heading or paragraph.find(f"{W}pPr/{W}numPr") is not None: + found.append(paragraph) + assert len(found) == len(expected), (len(found), len(expected)) + for paragraph, unit in zip(found, expected, strict=True): + marker = ET.Element(f"{W}r") + ET.SubElement(marker, f"{W}t").text = f"{OPEN}{unit.unit}{CLOSE}" + position = 1 if paragraph.find(f"{W}pPr") is not None else 0 + paragraph.insert(position, marker) + return expected + + +def reference_lines(root: ET.Element, plan: DocxPlan) -> None: + """Add a paragraph for every reference target of the plan, made of the fields of that reference.""" + body = root.find(f"{W}body") + assert body is not None + writer = InlineWriter(plan, Relationships(), Bookmarks(), None, {}, "0563C1") + end = len(body) - 1 # before the section properties + for offset, (target, reference) in enumerate(plan.refs.items()): + line = p(text_run(f"{OPEN}ref {target}{CLOSE}"), writer.segments(reference.segments)) + body.insert(end + offset, line) + + +def prepared(data: bytes, tree: RenderTree, plan: DocxPlan) -> bytes: + """*data* with the results of its fields made wrong, its unit paragraphs marked and a line per reference.""" + def change(document: str) -> str: + ET.register_namespace("w", W.strip("{}")) + ET.register_namespace("r", "http://schemas.openxmlformats.org/officeDocument/2006/relationships") + root = ET.fromstring(document.encode("utf-8")) + seed_wrong_results(root) + marked_paragraphs(root, tree, plan) + reference_lines(root, plan) + return ET.tostring(root, encoding="unicode", xml_declaration=False) + + return repackage(data, "word/document.xml", change) + + +@dataclass +class Case: + variant: str + name: str + tree: RenderTree + plan: DocxPlan + rules: set[str] + original: bytes + data: bytes + shown: Exported | None = None + + +def corpus() -> dict[str, str]: + """The documents: tests/documents, the specification's examples and valid fixtures, and the stress documents.""" + found = {f"documents/{path.name}": path.read_text(encoding="utf-8") for path in sorted(DOCUMENTS.glob("*.lgd"))} + spec = spec_root() + if spec is not None: + for path in sorted((spec / "examples").rglob("*.lgd")) + sorted((spec / "fixtures" / "valid").rglob("*.lgd")): + found[str(path.relative_to(spec))] = path.read_text(encoding="utf-8") + found.update({f"stress/{name}": source for name, source in STRESS.items()}) + return found + + +def make_case(variant: str, name: str, source: str) -> Case | None: + try: + result = render(source, format="docx", **{**VARIANTS, **WORDS}[variant]) + except DocumentError: + return None + assert result.binary is not None + plan, diagnostics = plan_docx(result.tree, text_switch=result.style.docx.reference_text_switch) + return Case(variant, name, result.tree, plan, {d.rule for d in diagnostics}, result.binary, + prepared(result.binary, result.tree, plan)) + + +@functools.cache +def round_trips() -> dict[str, tuple[Case, ...]]: + """Every document under every variant, rendered, made wrong and marked, and shown by LibreOffice: one LibreOffice + instance for all of them.""" + require_libreoffice() + cases: dict[str, list[Case]] = {variant: [] for variant in (*VARIANTS, *WORDS)} + for offset, variant in enumerate(cases): + for index, (name, source) in enumerate(corpus().items()): + if variant in WORDS and not name.startswith("stress/"): + continue + if variant in ONLY_DOCS and name.removeprefix("stress/") not in ONLY_DOCS[variant]: + continue + if (variant not in ONLY_DOCS and name.removeprefix("stress/") in FEW_VARIANTS + and variant not in FEW_VARIANTS[name.removeprefix("stress/")]): + continue + if name.startswith(("examples/", "fixtures/")) and (index + offset) % 2: + continue # the specification's documents take turns under the schemes: each is shown under half + case = make_case(variant, name, source) + if case is not None: + cases[variant].append(case) + everything = {f"d{index}": case for index, case in enumerate(case for group in cases.values() for case in group)} + shown = libreoffice_export({key: case.data for key, case in everything.items()}) + for key, case in everything.items(): + case.shown = shown[key] + return {variant: tuple(group) for variant, group in cases.items()} + + +def round_trip(variant: str) -> tuple[Case, ...]: + return round_trips()[variant] + + +# -- Reading what LibreOffice says ---------------------------------------------------------------------------------- + + +def marked_lines(shown: Exported) -> dict[str, tuple[str, str]]: + """Tag to (the number LibreOffice drew before the marker, the text after it), for every marked line.""" + found: dict[str, tuple[str, str]] = {} + for line in shown.text.lstrip("\ufeff").splitlines(): + match = MARKED.match(line) + if match: + found[match["tag"]] = (match["number"].strip(), match["rest"].strip()) + return found + + +def lo_reference(segments: tuple) -> str: + """What LibreOffice shows for *segments*: the cached results, but for a field written with ``\\t`` (which it + ignores) the result of the same field without it.""" + return "".join(s.text if isinstance(s, Literal_) else s.cached_lo for s in segments) + + +def renderer_references(tree: RenderTree) -> dict[str, str]: + return {i.target_unit: i.text for i in iter_tree_inlines(tree) if isinstance(i, CrossRef) and i.target_unit} + + +def squeeze(line: str) -> str: + return re.sub(r"\s+", " ", line).strip() + + +def cached_lines(data: bytes, plan: DocxPlan) -> list[str]: + """What the paragraphs of the file say, as the writer cached it, one entry per line without the blank ones. A field + written with ``\\t`` is what LibreOffice shows for it (it ignores the switch), by ``FieldSeg.cached_lo``.""" + lo = {(s.bookmark, s.suppress_text): s.cached_lo for r in plan.refs.values() for s in r.segments + if isinstance(s, FieldSeg)} + root = ET.fromstring(parts(data)["word/document.xml"]) + lines: list[str] = [] + for paragraph in root.iter(f"{W}p"): + text, state, instruction, replacement = "", None, "", None + for run in paragraph.iter(f"{W}r"): + mark = _is_field_part(run) + if mark == "begin": + state, instruction, replacement = "instruction", "", None + elif mark == "separate": + state = "result" + switches = instruction.split() + if "\\t" in switches: + replacement = lo.get((switches[1], True)) + elif mark == "end": + text += replacement or "" + state = None + elif state == "instruction": + instruction += "".join(node.text or "" for node in run.findall(f"{W}instrText")) + elif state == "result" and replacement is not None: + continue + else: + for node in run: + text += (node.text or "") if node.tag == f"{W}t" else "\t" if node.tag == f"{W}tab" else ( + "\n" if node.tag == f"{W}br" else "") + lines += [squeeze(line) for line in text.split("\n") if squeeze(line)] + return lines + + +def shown_lines(shown: Exported) -> list[str]: + """What LibreOffice says, line by line, without the blank ones, the marker lines of the references and the + markers and numbers of the numbered paragraphs.""" + lines = [] + for line in shown.text.lstrip("\ufeff").splitlines(): + match = MARKED.match(line) + if match and match["tag"].startswith("ref "): + continue + text = squeeze(match["rest"] if match else line) + if text: + lines.append(text) + return lines + + +def check(case: Case) -> None: + """The property: every number LibreOffice draws is the renderer's, or the plan's where the plan says Word differs + (with a diagnostic), every reference reads as the renderer wrote it, or as the plan says, and every body paragraph + says what the file cached for it. The table of contents is compared elsewhere (``test_docx_matter.py``).""" + assert case.shown is not None + lines = marked_lines(case.shown) + excused = False + for node in units(case.tree): + mine = case.plan.units[node.unit] + if not (isinstance(node, Section) or mine.explicit): + continue + number, text = lines[node.unit] + label = node.label or "" + if mine.num_id is None: + assert number == "", (node.unit, number) + elif mine.num_id == 0: # a later alternative: no number of its own, the holder's as fields, which LibreOffice + # updates to what the renderer wrote + assert number == "" and text.startswith(label), (node.unit, "LibreOffice", number, text, "renderer", label) + assert text.startswith(mine.label), (node.unit, "LibreOffice", text, "plan", mine.label) + elif mine.count is None: # a bullet: LibreOffice draws its own symbol + assert number != "", node.unit + else: + assert number == mine.label, (node.unit, "LibreOffice", number, "plan", mine.label) + if mine.label != label: + excused = True + else: + assert number == label, (node.unit, number, label) + if excused: + assert case.rules & EXPLAINING, "the plan says Word numbers a unit differently but reports nothing" + shown_by_renderer = renderer_references(case.tree) + for target, reference in case.plan.refs.items(): + _, text = lines[f"ref {target}"] + assert text == lo_reference(reference.segments), (target, "LibreOffice", text, "plan", reference.segments) + suppressing = any(isinstance(s, FieldSeg) and s.suppress_text for s in reference.segments) + if reference.exact and not suppressing: + assert text == shown_by_renderer[target], (target, text, shown_by_renderer[target]) + cached, said = cached_lines(case.original, case.plan), shown_lines(case.shown) + assert said == cached, "\n".join(difflib.unified_diff(cached, said, "cached", "LibreOffice", lineterm="", n=1)) + + +@pytest.mark.parametrize("variant", list(VARIANTS) + list(WORDS)) +def test_libreoffice_draws_the_numbers_and_reads_the_references_as_the_renderer_wrote_them(variant: str) -> None: + cases = round_trip(variant) + assert cases + for case in cases: + try: + check(case) + except AssertionError as error: + raise AssertionError(f"{variant}/{case.name}: {error}") from error + + +def test_the_round_trip_reaches_numbers_references_and_every_gap() -> None: + """The corpus is not vacuous: it numbers headings, items and paragraphs, it has references of every kind, and it + reaches each documented gap (``\\t``, an overflowing letter counter, a label that needs the section).""" + seen_rules: set[str] = set() + seen_causes: set[str] = set() + references = suppressing = numbered = hidden = 0 + for variant in list(VARIANTS) + list(WORDS): + for case in round_trip(variant): + seen_rules |= case.rules + seen_causes |= {cause for unit in case.plan.units.values() for cause in unit.cause} + references += len(case.plan.refs) + numbered += sum(1 for unit in case.plan.units.values() if unit.count is not None) + suppressing += sum(any(isinstance(s, FieldSeg) and s.suppress_text for s in r.segments) + for r in case.plan.refs.values()) + hidden += sum(isinstance(s, FieldSeg) and s.bookmark.startswith("_Ld") + for r in case.plan.refs.values() for s in r.segments) + assert numbered > 200 and references > 100 and suppressing > 0 and hidden > 0 + assert {"render-docx-letter-overflow", "render-docx-section-label", "render-docx-shared-counter"} <= seen_rules + assert "render-docx-number-mismatch" not in seen_rules + assert {"alternative", "reserved-level", "resync", "restart-alternative", "restart-section"} <= seen_causes + + +def test_the_cached_results_are_wrong_in_the_file_and_libreoffice_does_not_read_them() -> None: + """The seed works: the file holds ``XXX`` as the result of every reference field, and LibreOffice shows the right + results all the same, even before the fields are updated, because it computes them from the numbering and the + bookmarks when it opens the file (so that what the comparison above checks is its computation, not the cache).""" + document = next(case for case in round_trip("default") if case.name == "stress/nested") + written = parts(document.data)["word/document.xml"].decode("utf-8") + assert written.count(f">{WRONG}<") >= len(document.plan.refs) > 10 + unupdated = libreoffice_export({"nested": document.data}, update=False)["nested"] + for shown in (document.shown, unupdated): + lines = marked_lines(shown) # type: ignore[arg-type] + assert not any(WRONG in text for _, text in lines.values()) + check(replace(document, shown=shown)) + + +# -- Edits in LibreOffice, then the fields updated ------------------------------------------------------------------- + + +EDITED = FRONT + """ +# Alpha {#alpha} + +Para {#p1} + +- first {#i1} +- second item {#i2} + - nested one {#n1} + - nested two {#n2} +- third {#i3} + +## Beta {#beta} + +1. ord one {#o1} +2. ord two {#o2} + +# Gamma {#gamma} + +- g item {#gi} + +Refs: {{ref: alpha}} {{ref: p1}} {{ref: i1}} {{ref: i2}} {{ref: n1}} {{ref: n2}} {{ref: i3}} {{ref: beta}} {{ref: o2}} +{{ref: gamma}} {{ref: gi}} +""" + +#: What a lawyer does in Word: press Enter at the start of a paragraph and type (an item, a heading, an ordered item). +#: Each is (the text of the paragraph it goes before, what is typed, the source line it goes before, the new line). +INSERTIONS = [ + ("second item", "NEW item", "- second item {#i2}", "- NEW item"), + ("Gamma", "NEWH", "# Gamma {#gamma}", "# NEWH\n"), + ("ord two", "NEW ord", "2. ord two {#o2}", "1. NEW ord"), +] + +EDIT_VARIANTS = {name: VARIANTS[name] for name in ("default", "outline", "continental", "none")} + + +def after_the_edits() -> str: + source = EDITED + for _, _, before, new in INSERTIONS: + source = source.replace(before, f"{new}\n{before}") + return source + + +@functools.cache +def edited() -> dict[str, tuple[str, str]]: + """Variant to (what LibreOffice says after the insertions and the update, what the text writer prints for the + edited source).""" + require_libreoffice() + jobs = {} + for variant, settings in EDIT_VARIANTS.items(): + jobs[variant] = (render(EDITED, format="docx", **settings).binary, [(m, n) for m, n, _, _ in INSERTIONS]) + said = libreoffice_edit(jobs) # type: ignore[arg-type] + return {variant: (said[variant], render(after_the_edits(), format="text", **EDIT_VARIANTS[variant]).output) + for variant in EDIT_VARIANTS} + + +@pytest.mark.parametrize("variant", EDIT_VARIANTS) +def test_the_references_follow_the_insertions_a_lawyer_makes(variant: str) -> None: + """The point of the whole design: after items and headings are inserted before the targets and the fields are + updated, every reference reads as the renderer writes it for the edited document (a reference that was text + would still say what it said before).""" + said, written = edited()[variant] + refs = [squeeze(line) for line in said.splitlines() if "Refs:" in line] + expected = [squeeze(line) for line in written.splitlines() if "Refs:" in line] + assert refs == expected and refs + before = render(EDITED, format="text", **EDIT_VARIANTS[variant]).output + assert [squeeze(line) for line in before.splitlines() if "Refs:" in line] != expected # the edits moved them + + +@pytest.mark.parametrize("variant", ["default", "outline", "continental", "none"]) +def test_the_numbers_follow_the_insertions_a_lawyer_makes(variant: str) -> None: + said, written = edited()[variant] + assert [squeeze(line) for line in said.lstrip("\ufeff").splitlines() if squeeze(line)] == [ + squeeze(line) for line in written.splitlines() if squeeze(line)] + + +# -- Clauses and nested items inserted under the continental style -------------------------------------------------- + + +CLAUSES = FRONT + """ +# Alpha {#alpha} + +- clause one {#c1} + - nested of one {#n1} +- clause two {#c2} + - nested of two {#n2} + - deep of two {#d2} + +Refs: {{ref: c1}} {{ref: n1}} {{ref: c2}} {{ref: n2}} {{ref: d2}} +""" + +#: A clause and a nested item inserted before the second clause and its nested item. +CLAUSE_EDITS = [ + ("clause two", "NEW clause", "- clause two {#c2}", "- NEW clause"), + ("nested of two", "NEW nested", " - nested of two {#n2}", " - NEW nested"), +] + +#: A heading section whose sub-headings follow numbered clauses: Word's counter of the level is shared with the clauses, +#: so the first sub-heading is restarted at the number the renderer gives it (a fixed value). +RESYNCED = FRONT + """ +# Alpha {#alpha} + +- clause one {#c1} +- spare clause +- clause two {#c2} + +## Beta {#beta} + +## Gamma {#gamma} + +Refs: {{ref: c1}} {{ref: c2}} {{ref: beta}} {{ref: gamma}} +""" + + +def applied(source: str, edits: list[tuple[str, str]]) -> str: + """*source* with each new line put before the line it goes before.""" + for before, new in edits: + source = source.replace(before, f"{new}\n{before}", 1) + return source + + +@functools.cache +def clauses() -> dict[str, str]: + """What LibreOffice says about the clause document after the two insertions, under the continental style and under + the default (whose nested lists are the family's own).""" + require_libreoffice() + edits = [(match, new) for match, new, _, _ in CLAUSE_EDITS] + jobs = {style: (render(CLAUSES, format="docx", **settings).binary, edits) + for style, settings in {"continental": {"style": "continental"}, "default": {}}.items()} + return libreoffice_edit(jobs) # type: ignore[arg-type] + + +def lines_of(text: str) -> list[str]: + return [squeeze(line) for line in text.lstrip("\ufeff").splitlines() if squeeze(line)] + + +@pytest.mark.parametrize("style", ["continental", "default"]) +def test_a_clause_and_a_nested_item_inserted_in_word_are_numbered_as_the_renderer_numbers_them(style: str) -> None: + settings = {"style": "continental"} if style == "continental" else {} + after = render(applied(CLAUSES, [(before, new) for _, _, before, new in CLAUSE_EDITS]), format="text", **settings) + said = lines_of(clauses()[style]) + assert said == lines_of(after.output) + # the item inserted into a list of nested items starts it: "a)" under the continental style, "(i)" under the default + first, second = ("a)", "b)") if style == "continental" else ("(i)", "(ii)") + assert [line for line in said if line.endswith(("NEW nested", "nested of two"))] == [ + f"{first} NEW nested", f"{second} nested of two"] + + +def test_the_clause_inserted_under_the_continental_style_takes_the_next_number_of_the_heading_list() -> None: + said = lines_of(clauses()["continental"]) + assert [line for line in said if line.endswith(("NEW clause", "clause two"))] == ["1.2 NEW clause", "1.3 clause two"] + assert [line for line in said if line.startswith("(1) Refs:")] == ["(1) Refs: 1.1 1.1(a) 1.3 1.3(b) 1.3(b)(i)"] + + +# -- A sub-heading Word restarts at a fixed number: the limit of what an edit keeps live ------------------------------- + + +@functools.cache +def resynced() -> dict[str, str]: + require_libreoffice() + data = render(RESYNCED, format="docx", style="continental").binary + jobs = {"original": (data, []), "clause deleted": (data, [("spare clause", None)]), + "clause inserted": (data, [("spare clause", "NEW clause")]), + "heading deleted": (data, [("Beta", None)]), "heading inserted": (data, [("Beta", "NEWH")])} + return libreoffice_edit(jobs) # type: ignore[arg-type] + + +def test_a_sub_heading_after_clauses_is_restarted_by_a_fixed_value_and_the_plan_says_so() -> None: + result = render(RESYNCED, format="docx", style="continental") + plan, diagnostics = plan_docx(result.tree) + assert plan.units["s2"].cause == ("resync",) + [found] = [d for d in diagnostics if d.rule == "render-docx-shared-counter"] + assert found.level == "warning" + assert lines_of(resynced()["original"]) == lines_of(render(RESYNCED, format="text", style="continental").output) + + +def test_a_clause_inserted_or_deleted_before_the_restarted_sub_heading_leaves_it_and_what_follows_right() -> None: + sources = {"clause deleted": RESYNCED.replace("- spare clause\n", ""), + "clause inserted": applied(RESYNCED, [("- spare clause", "- NEW clause")])} + for name, source in sources.items(): + assert lines_of(resynced()[name]) == lines_of(render(source, format="text", style="continental").output), name + + +def test_a_sub_heading_inserted_before_the_restarted_one_takes_its_num_and_everything_follows() -> None: + source = applied(RESYNCED, [("## Beta", "## NEWH\n")]) + assert lines_of(resynced()["heading inserted"]) == lines_of( + render(source, format="text", style="continental").output) + + +def test_deleting_the_restarted_sub_heading_is_the_documented_limit() -> None: + """The heading that holds the fixed value is gone: the headings after it go on from the counter the clauses of the + section used, so Gamma reads 1.4 where the renderer's document says 1.1 (the plan's warning says so). The + clauses before it are right and the broken reference to the deleted heading is Word's.""" + _, diagnostics = plan_docx(render(RESYNCED, format="docx", style="continental").tree) + [found] = [d for d in diagnostics if d.rule == "render-docx-shared-counter"] + assert "if it is deleted in Word the headings after it read higher" in found.message + said = lines_of(resynced()["heading deleted"]) + assert [line for line in said if "clause" in line and not line.startswith("(")] == [ + "1.1 clause one", "1.2 spare clause", "1.3 clause two"] + assert "1.4 Gamma" in said and "1.1 Gamma" in lines_of(render( + RESYNCED.replace("## Beta {#beta}\n\n", ""), format="text", style="continental").output) + + +@functools.cache +def shared_level() -> dict[str, str]: + require_libreoffice() + levels = {"paragraphs.numbered": True, "paragraphs.format": {"counter": "decimal", "label": "{section}.{n}", + "ref": ".{n}"}} + data = render(PARAGRAPHS_AND_CLAUSES, format="docx", style="continental", overrides=levels).binary + return libreoffice_edit({"original": (data, []), "inserted": (data, [("One.", "NEWP")])}) # type: ignore[arg-type] + + +PARAGRAPHS_AND_CLAUSES = FRONT + "# A\n\nOne.\n\n- x\n- y\n\nTwo.\n" + + +def test_a_paragraph_restarted_after_clauses_does_not_follow_a_paragraph_inserted_before_it() -> None: + levels = {"paragraphs.numbered": True, "paragraphs.format": {"counter": "decimal", "label": "{section}.{n}", + "ref": ".{n}"}} + assert lines_of(shared_level()["original"]) == lines_of(render( + PARAGRAPHS_AND_CLAUSES, format="text", style="continental", overrides=levels).output) + plan, diagnostics = plan_docx(render(PARAGRAPHS_AND_CLAUSES, format="docx", style="continental", + overrides=levels).tree) + [found] = [d for d in diagnostics if d.rule == "render-docx-shared-counter"] + assert "inserted or deleted before it in Word does not move it" in found.message + # the renderer would number the new paragraph 1.1, One 1.2 and Two 1.3; Two holds its fixed value + said = lines_of(shared_level()["inserted"]) + assert [line for line in said if line.endswith(("NEWP", "One.", "Two."))] == ["1.1 NEWP", "1.2 One.", "1.2 Two."] + + +# -- The contents: the headings with the numbers Word draws, and none of the units on the heading list ------------------ + +CONTENTS = ALTERNATIVES + """ +# Alpha {#alpha} + +- item one {#i1} + - nested {#n1} +- item two {#i2} + +## Beta {#beta} + +- beta item + +# Disputes {#d when=forum:courts} + +- courts item + +## Sub + +# Disputes {#d when=forum:arb} + +## Sub alt + +# Last +""" + + +@functools.cache +def contents() -> tuple[DocxPlan, tuple[str, ...], str]: + """The plan, the entries of the contents LibreOffice builds from the ``TOC`` field the writer wrote (the lines that + end in a page number), and the rest of what it shows.""" + require_libreoffice() + result = render(CONTENTS, format="docx", style="continental", overrides={"contents.enabled": True, + "contents.depth": 5}) + assert result.binary is not None + plan, _ = plan_docx(result.tree) + lines = libreoffice_export({"contents": result.binary})["contents"].text.lstrip("\ufeff").splitlines() + entries = tuple(line for line in lines if re.fullmatch(r".*\t\d+", line)) + return plan, entries, "\n".join(line for line in lines if line not in entries) + + +def test_the_contents_list_the_headings_as_the_plan_does_and_none_of_the_items_on_the_heading_list() -> None: + plan, entries, body = contents() + expected = [squeeze(f"{entry.number} {plain_text(entry.title)} {entry.condition_label or ''}") for entry in plan.toc] + said = [re.sub(r"\s+\d+$", "", squeeze(line)) for line in entries] + assert said == expected + assert len(plan.toc) == 7 + assert [e.number for e in plan.toc] == ["1.", "1.1", "2.", "2.1", "2.", "2.1", "3."] # the alternative's is shared + assert not any(word in line for line in entries for word in ("item", "nested")) + assert "1.1 item one" in body and "a) nested" in body # the items are in the body: on the heading list, not in it + + +# -- Labels that show the path: live on the reserved levels, detached where they cannot be --------------------------------- + +PATH_CLAUSES = FRONT + """ +# Alpha {#alpha} + +- clause one {#c1} + - nested of one {#n1} + - deep of one {#d1} +- clause two {#c2} + - nested of two {#n2} + - deep of two {#d2} + +Refs: {{ref: c1}} {{ref: n1}} {{ref: d1}} {{ref: c2}} {{ref: n2}} {{ref: d2}} +""" + +#: A clause, a nested item and a deeper one inserted before the second clause and what is under it. +PATH_EDITS = [ + ("clause two", "NEW clause", "- clause two {#c2}", "- NEW clause"), + ("nested of two", "NEW nested", " - nested of two {#n2}", " - NEW nested"), + ("deep of two", "NEW deep", " - deep of two {#d2}", " - NEW deep"), +] + + +def path_clauses() -> str: + return later_edits()["paths"] + + +def test_a_path_label_on_the_reserved_levels_follows_the_items_a_lawyer_inserts_and_so_do_its_references() -> None: + """The label and every reference to it are live: after a clause, a nested item and a deeper one are inserted in + the editor and the fields are updated, the numbers (with the section's and the clause's counters in them) and the + references read as the renderer writes them for the edited source.""" + source = applied(PATH_CLAUSES, [(before, new) for _, _, before, new in PATH_EDITS]) + said = lines_of(path_clauses()) + assert said == lines_of(render(source, format="text", **PATH_ITEMS).output) + assert {"1.3(a) NEW nested", "1.3(b) nested of two", "1.3(b)(i) NEW deep", "1.3(b)(ii) deep of two"} <= set(said) + assert [line for line in said if line.startswith("(1) Refs")] == [ + "(1) Refs: 1.1 1.1(a) 1.1(a)(i) 1.3 1.3(b) 1.3(b)(ii)"] + + +ORDERED_PATHS = FRONT + """ +# Alpha {#alpha} + +- one {#i1} + 1. ordered {#o} + - three {#i3} + - four {#i4} + - four b {#i5} + - three b {#i6} + - four c {#i7} + +Refs: {{ref: i4}} {{ref: i5}} {{ref: i7}} +""" + + +def ordered_paths() -> dict[str, str]: + """The ordered-parent document as it is, with a nested item inserted in the detached list, and with a new parent whose + nested item is a copy of one of the detached list's (Word's paste keeps the list).""" + return {name: text for name, text in later_edits().items() if name in ("original", "nested inserted", "new parent")} + + +def test_a_detached_list_draws_its_own_counter_and_its_references_are_live() -> None: + said = lines_of(ordered_paths()["original"]) + assert [line for line in said if line.endswith(("four", "four b", "four c"))] == ["(A) four", "(B) four b", "(A) four c"] + assert "Refs: 1(a)(1)(i)(A) 1(a)(1)(i)(B) 1(a)(1)(ii)(A)" in said[-1] + # an item inserted in the detached list renumbers what follows, in the list and in the references to it + after = lines_of(ordered_paths()["nested inserted"]) + assert [line for line in after if line.endswith(("four", "NEW four", "four b", "four c"))] == [ + "(A) four", "(B) NEW four", "(C) four b", "(A) four c"] + assert "Refs: 1(a)(1)(i)(A) 1(a)(1)(i)(C) 1(a)(1)(ii)(A)" in after[-1] + + +def test_a_new_parent_whose_nested_item_is_copied_from_a_detached_list_goes_on_counting_there() -> None: + """The documented limit of the detached levels: a nested list restarts at a fixed value under the parent that has it, + so a list that is new in Word, made by copying an item of another, goes on from that one (the renderer would start + it again at ``(A)``), while the list that was restarted keeps its start.""" + said = lines_of(ordered_paths()["new parent"]) + assert [line for line in said if line.endswith(("four b", "NEW copy", "four c"))] == [ + "(B) four b", "(C) NEW copy", "(A) four c"] + assert "ii NEW three" in said and "iii three b" in said + assert "Refs: 1(a)(1)(i)(A) 1(a)(1)(i)(B) 1(a)(1)(iii)(A)" in said[-1] + + +# -- Lists nested past Word's nine levels --------------------------------------------------------------------------------- + +DEEP = FRONT + "# A {#a}\n\n" + "".join(" " * d + f"- level {d} {{#l{d}}}\n" for d in range(12)) + ( + "\nRefs: {{ref: l8}} {{ref: l9}} {{ref: l10}} {{ref: l11}}\n") + +#: An item inserted before the first of the list at level 9, before the one at level 10, and before the one at level 8. +DEEP_EDITS = [ + ("level 9", "NEW nine", " - level 9 {#l9}", " - NEW nine"), + ("level 10", "NEW ten", " - level 10 {#l10}", " - NEW ten"), + ("level 8", "NEW eight", " - level 8 {#l8}", " - NEW eight"), +] + + +def deep_lists() -> str: + return later_edits()["deep"] + + +def test_items_nested_past_nine_levels_are_numbered_by_the_list_they_go_on_in_and_follow_edits() -> None: + """The lists nested past Word's nine levels keep restarting by structure inside the list they go on in, so an item + inserted at level 8, 9 or 10 renumbers what follows and every reference to it, as the renderer does.""" + source = applied(DEEP, [(before, new) for _, _, before, new in DEEP_EDITS]) + assert lines_of(deep_lists()) == lines_of(render(source, format="text").output) + said = lines_of(deep_lists()) + assert [line for line in said if line.endswith(("NEW eight", "level 8", "NEW nine", "level 9", "NEW ten", + "level 10", "level 11"))] == [ + "(A) NEW eight", "(B) level 8", "(a) NEW nine", "(b) level 9", "(i) NEW ten", "(ii) level 10", "(A) level 11"] + + +# -- A skipped heading level: the one part of a reference that is text ---------------------------------------------------- + +SKIPPED = FRONT + """ +# One {#one} + +### Deep {#deep} + +- item {#i} + +# Two {#two} + +### Deeper {#deeper} + +Refs: {{ref: one}} {{ref: deep}} {{ref: i}} {{ref: two}} {{ref: deeper}} +""" + + +def skipped() -> tuple[str, str]: + """What LibreOffice says about the document that skips a heading level (outline numbering) after a heading is inserted + before its first section, and what the text writer prints for the edited source.""" + edited_source = applied(SKIPPED, [("# One {#one}", "# NEWH\n")]) + return later_edits()["skipped"], render(edited_source, format="text", style="outline").output + + +@functools.cache +def later_edits() -> dict[str, str]: + """What LibreOffice says about the documents of the sections above after their edits, all in one LibreOffice session + (starting it is what costs): path labels in a clause, the ordered-parent list as it is, with a nested item inserted + and with a new parent whose nested item is a copy, the list past nine levels, and the document that skips a level.""" + require_libreoffice() + + def docx(source: str, **settings: object) -> bytes: + data = render(source, format="docx", **settings).binary + assert data is not None + return data + + ordered = docx(ORDERED_PATHS, **ORDERED_PARENT) + return libreoffice_edit({ # type: ignore[arg-type] + "paths": (docx(PATH_CLAUSES, **PATH_ITEMS), [(match, new) for match, new, _, _ in PATH_EDITS]), + "original": (ordered, []), + "nested inserted": (ordered, [("four b", "NEW four")]), + "new parent": (ordered, [("three b", "NEW three"), ("three b", "NEW copy", "four b")]), + "deep": (docx(DEEP), [(match, new) for match, new, _, _ in DEEP_EDITS]), + "skipped": (docx(SKIPPED, style="outline"), [("One", "NEWH")])}) + + +def test_the_live_parts_of_a_reference_below_a_skipped_level_follow_a_heading_inserted_before_it() -> None: + """The skipped level has no number of its own, so its part of the reference is text (``A``) between two fields; the + fields are updated by the edit, so ``I.A.1`` becomes ``II.A.1`` with the text kept, as the renderer writes it.""" + said, written = skipped() + assert lines_of(said) == lines_of(written) + assert [line for line in lines_of(said) if line.startswith("Refs")] == ["Refs: II II.A.1 II.A.1(a) III III.A.1"] + result = render(SKIPPED, format="docx", style="outline") + plan, diagnostics = plan_docx(result.tree) + assert [r.ghost for r in plan.refs.values()] == ["", "A", "A", "", "A"] + assert not any(r.fixed for r in plan.refs.values()) + assert any(d.rule == "render-docx-reference" and "heading level that the document skips" in d.message + for d in diagnostics) + + +# -- A level that nothing counted before a deeper item: where LibreOffice and Word differ ----------------------------------- + +UNCOUNTED = FRONT + "# A\n\n- a\n - b\n 1. c\n - d\n - e\n - f\n" + + +def test_libreoffice_counts_the_level_nothing_numbered_before_a_deeper_item_and_the_plan_reports_it() -> None: + """The one place LibreOffice's numbers differ from the plan's for a reason of Word's model, not of the file: the items + ``e`` and ``f`` are the first of their level in their list, which Word numbers (A), (B) and LibreOffice (B), (C) + (it counts the level above ``d`` as one). The plan is Word's and reports the difference.""" + require_libreoffice() + result = render(UNCOUNTED, format="docx") + plan, diagnostics = plan_docx(result.tree) + said = lines_of(libreoffice_export({"uncounted": result.binary})["uncounted"].text) # type: ignore[arg-type] + assert [line for line in said if line.endswith((" e", " f"))] == ["(B) e", "(C) f"] + assert [plan.units[u.unit].label for u in units(result.tree) if plan.units[u.unit].count and u.unit in ("u5", "u6")] \ + == ["(A)", "(B)"] + assert any(d.rule == "render-docx-libreoffice-count" and d.level == "info" for d in diagnostics) diff --git a/tests/test_docx_simulator.py b/tests/test_docx_simulator.py new file mode 100644 index 0000000..954a17f --- /dev/null +++ b/tests/test_docx_simulator.py @@ -0,0 +1,275 @@ +"""Word's list numbering as the DOCX planner models it (docs/architecture.md, "The DOCX planner"): counters as Word +writes them, label templates as tokens (the same text the resolver fills in, with the owner of each counter kept), +the ``lvlText`` of a level, and the simulator that gives each paragraph of a list its number.""" +from __future__ import annotations + +import pytest + +from legaldown_render.errors import InternalError +from legaldown_render.resolve.docx_numbering import ( + Count, + Ghost, + Lit, + WordNumbering, + extend_tokens, + fill_tokens, + level_text, + level_text_problem, + lvl_text, + shows_section, + split_level_text, + text_of, + word_counter, +) +from legaldown_render.resolve.docx_plan import AbstractNum, LevelDef, Num +from legaldown_render.resolve.numbering import extend, fill, format_counter + +# -- Counters --------------------------------------------------------------------- + + +@pytest.mark.parametrize(("value", "num_fmt", "expected"), [ + (1, "decimal", "1"), (4000, "decimal", "4000"), + (1, "lowerLetter", "a"), (26, "lowerLetter", "z"), (27, "lowerLetter", "aa"), (28, "lowerLetter", "bb"), + (52, "lowerLetter", "zz"), (53, "lowerLetter", "aaa"), (54, "upperLetter", "BBB"), + (4, "lowerRoman", "iv"), (14, "upperRoman", "XIV"), (3999, "upperRoman", "MMMCMXCIX"), + (4000, "lowerRoman", "mmmm"), (4999, "lowerRoman", "mmmmcmxcix"), + (0, "decimal", "0"), (0, "lowerLetter", ""), +]) +def test_word_counters(value: int, num_fmt: str, expected: str) -> None: + assert word_counter(value, num_fmt) == expected + + +@pytest.mark.parametrize(("counter", "num_fmt"), [ + ("decimal", "decimal"), ("lower-alpha", "lowerLetter"), ("upper-alpha", "upperLetter"), + ("lower-roman", "lowerRoman"), ("upper-roman", "upperRoman")]) +def test_word_and_renderer_counters_agree_where_word_defines_them(counter: str, num_fmt: str) -> None: + limit = 4000 if counter.endswith("roman") else 27 if counter.endswith("alpha") else 5000 + for value in range(1, limit): + assert word_counter(value, num_fmt) == format_counter(value, counter) + + +def test_word_repeats_letters_where_the_renderer_counts_on() -> None: + assert [word_counter(v, "lowerLetter") for v in (27, 28, 29)] == ["aa", "bb", "cc"] + assert [format_counter(v, "lower-alpha") for v in (27, 28, 29)] == ["aa", "ab", "ac"] + assert word_counter(4000, "upperRoman") == "MMMM" and format_counter(4000, "upper-roman") == "4000" + + +# -- Token strings ---------------------------------------------------------------- + + +@pytest.mark.parametrize(("template", "section"), [ + ("{n}.", None), ("({n})", None), ("{section}.{n}", "5"), ("{section}.{n}", ""), ("{section}..{n}", ""), + ("({section}){n}", ""), ("{path}.{n}", "2"), ("a {other} {n}", "1"), ("{n}{n}", None)]) +def test_filling_tokens_reads_as_filling_the_template(template: str, section: str | None) -> None: + tokens = fill_tokens(template, n=[Count("n", "b")], path=[Lit("2.1")], + section=None if section is None else [Lit(section)] if section else []) + kwargs = {} if section is None else {"section": section} + assert text_of(tokens) == fill(template, n="b", path="2.1", **kwargs) + + +@pytest.mark.parametrize(("designation", "piece", "textual"), [ + ("", ".1", False), ("2.1", "(b)", False), ("Termination", "(a)", True), ("", ".1", True), ("A", ".1", True)]) +def test_extending_tokens_reads_as_extending_the_designation(designation: str, piece: str, textual: bool) -> None: + tokens = extend_tokens([Lit(designation)] if designation else [], [Lit(piece)], textual=textual) + assert text_of(tokens) == extend(designation, piece, textual=textual) + + +def test_tokens_keep_the_owner_of_each_counter() -> None: + tokens = fill_tokens("{path}({n})", n=[Count("u2", "b")], path=[Count("s1", "1"), Lit(".")]) + assert tokens == [Count("s1", "1"), Lit("."), Lit("("), Count("u2", "b"), Lit(")")] + + +# -- lvlText ---------------------------------------------------------------------- + + +def test_a_level_shows_only_counters_of_its_own_list() -> None: + refs = ["{n}", ".{n}", ".{n}"] + assert lvl_text(level_text("{path}.", refs, 0, section=None)) == "%1." + assert lvl_text(level_text("{path}", refs, 2, section=None)) == "%1.%2.%3" + assert lvl_text(level_text("({n})", ["({n})"] * 3, 1, section=())) == "(%2)" + assert lvl_text(level_text("Article {n}", ["{n}"], 0, section=None)) == "Article %1" + + +def test_a_label_that_needs_the_section_is_written_as_in_the_preamble() -> None: + # Word cannot show the section's counter in a list level: the label loses the field and the dot after it. + assert lvl_text(level_text("{section}.{n}", [".{n}"], 0, section=())) == "%1" + assert lvl_text(level_text("{path}", [".{n}", "({n})"], 1, section=())) == "%1(%2)" + + +def test_literal_percent_before_a_digit_cannot_be_written() -> None: + assert lvl_text(level_text("100% {n}", ["{n}"], 0, section=None)) == "100% %1" + with pytest.raises(InternalError): + lvl_text(level_text("%1 {n}", ["{n}"], 0, section=None)) + with pytest.raises(InternalError): + lvl_text(level_text("%{n}", ["{n}"], 0, section=None)) + + +def test_a_percent_that_meets_a_counter_where_pieces_are_put_together_is_a_problem() -> None: + parts = level_text("{path}", ["{n}%", "{n}"], 1, section=None) + assert parts == (0, "%", 1) and level_text_problem(parts) is not None + assert level_text_problem(level_text("{path}", ["{n}%", " {n}"], 1, section=None)) is None + assert level_text_problem(("5%", 0)) is not None + with pytest.raises(InternalError): + lvl_text(parts) + + +def test_a_label_shows_the_section_when_the_filled_label_has_its_counter() -> None: + refs = ["({n})", "({n})"] + assert shows_section("{section}.{n}", [".{n}"], 0) + assert shows_section("{path}", refs, 1) and shows_section("{path}", refs, 0) + assert not shows_section("({n})", refs, 1) and not shows_section("{n})", refs, 0) + assert shows_section("({section}) {n}", refs, 0) + # a ref that holds the section puts it in every `{path}` below, not in a label that shows only its own counter + assert shows_section("{path}", ["{section}{n}", "({n})"], 1) + assert not shows_section("({n})", ["{section}{n}", "({n})"], 1) + assert shows_section("{section}.{n}", [".{n}"], 0, textual=True) + + +def test_split_level_text() -> None: + assert split_level_text("%1.%2(%10)x") == [0, ".", 1, "(", 0, "0)x"] + assert split_level_text("plain") == ["plain"] + + +# -- The simulator ---------------------------------------------------------------- + + +def _levels(*specs: tuple[str, str]) -> list[LevelDef]: + levels = [LevelDef(i, fmt, text) for i, (fmt, text) in enumerate(specs)] + return levels + [LevelDef(i, "decimal", f"%{i + 1}") for i in range(len(levels), 9)] + + +HEADINGS = _levels(("decimal", "%1."), ("decimal", "%1.%2"), ("decimal", "%1.%2.%3")) +ITEMS = _levels(("lowerLetter", "(%1)"), ("lowerRoman", "(%2)"), ("upperLetter", "(%3)")) + + +def numbering(*nums: Num, skipped: str = "lo") -> WordNumbering: + sim = WordNumbering(skipped=skipped) # type: ignore[arg-type] + sim.add_abstract(0, HEADINGS) + sim.add_abstract(1, ITEMS) + for num in nums: + sim.add_num(num) + return sim + + +def test_counters_count_and_deeper_levels_restart() -> None: + sim = numbering(Num(1, 0)) + labels = [sim.step(1, i, f"u{k}").label for k, i in enumerate([0, 1, 1, 2, 0, 1])] + assert labels == ["1.", "1.1", "1.2", "1.2.1", "2.", "2.1"] + + +def test_a_step_says_what_ref_gives_and_which_unit_holds_each_counter() -> None: + sim = numbering(Num(1, 0)) + sim.step(1, 0, "s1") + step = sim.step(1, 1, "s2") + assert (step.count, step.own, step.label, step.word_ref) == (1, "1", "1.1", "1.1") + assert step.tokens == (Count("s1", "1"), Lit("."), Count("s2", "1")) + assert sim.step(1, 0, "s3").word_ref == "2" and sim.step(1, 0, "s4").label == "3." + + +def test_a_start_override_restarts_a_level_at_its_first_use_only() -> None: + sim = numbering(Num(1, 1), Num(2, 1, ((0, 1), (1, 1))), Num(3, 1, ((0, 3), (1, 1)))) + assert [sim.step(1, 0, "a").label, sim.step(1, 0, "b").label] == ["(a)", "(b)"] + assert sim.peek(2, 0) == 1 + assert [sim.step(2, 0, "c").label, sim.step(2, 0, "d").label] == ["(a)", "(b)"] + assert sim.peek(3, 0) == 3 + assert [sim.step(3, 0, "e").label, sim.step(3, 0, "f").label] == ["(c)", "(d)"] + + +def test_counters_are_shared_by_every_num_of_an_abstract_num() -> None: + # A num without an override goes on from the counter of the abstractNum; one with an override restarts the level + # while the levels above keep their values. + sim = numbering(Num(1, 0), Num(2, 0, ((1, 1), (2, 1)))) + sim.step(1, 0, "s1") + sim.step(1, 1, "s2") + sim.step(1, 1, "s3") + assert sim.step(2, 1, "s4").label == "1.1" + assert sim.step(1, 1, "s5").label == "1.2" + assert sim.step(2, 0, "s6").label == "2." + + +def test_a_paragraph_without_numbering_counts_nothing_and_restarts_nothing() -> None: + sim = numbering(Num(1, 0)) + sim.step(1, 0, "s1") + sim.step(1, 1, "s2") + sim.step(1, 1, "s3") + step = sim.step(0, 0, "alternative") + assert (step.count, step.label, step.word_ref, step.tokens) == (0, "", "0", ()) + assert sim.current(1, 0) == (1, "s1") and sim.current(1, 1) == (2, "s3") + assert sim.step(1, 1, "s4").label == "1.3" and sim.step(1, 0, "s5").label == "2." + + +def test_peek_and_current_do_not_count() -> None: + sim = numbering(Num(1, 1)) + assert (sim.peek(1, 0), sim.current(1, 0)) == (1, (0, None)) + sim.step(1, 0, "u1") + assert (sim.peek(1, 0), sim.current(1, 0)) == (2, (1, "u1")) + assert sim.peek(1, 1) == 1 + + +def test_a_skipped_level_reads_as_its_start_value_and_counts_on() -> None: + sim = numbering(Num(1, 0)) + sim.step(1, 0, "s1") + deep = sim.step(1, 2, "s2") + assert deep.label == "1.1.1" + assert deep.tokens[2] == Count(Ghost("s2", 1), "1") + assert sim.step(1, 1, "s3").label == "1.2" + + +def test_in_the_word_model_a_skipped_level_reads_zero_and_is_not_advanced() -> None: + sim = numbering(Num(1, 0), skipped="word") + sim.step(1, 0, "s1") + assert sim.step(1, 2, "s2").label == "1.0.1" + assert sim.step(1, 1, "s3").label == "1.1" + + +def test_each_abstract_num_has_its_own_model_of_a_level_that_nothing_numbered() -> None: + sim = WordNumbering() + sim.add_abstract(0, HEADINGS) + sim.add_abstract(1, _levels(("lowerLetter", "(%1)"), ("decimal", "%1.%2")), skipped="word") + sim.add_num(Num(1, 0)) + sim.add_num(Num(2, 1)) + assert sim.step(1, 1, "s1").label == "1.1" # the renderer's and LibreOffice's rule + assert sim.step(2, 1, "u1").label == ".1" # Word's: a letter counter at 0 is nothing + + +def test_libreoffice_counts_a_level_nothing_numbered_before_a_deeper_one_and_goes_on_from_it() -> None: + """The count the plan gives is Word's (the unused level stays unused, so the next unit of that level is 1); + ``Step.lo_count`` is LibreOffice's (the unused level reads as its start and counts, so the next unit is 2).""" + sim = numbering(Num(1, 1), skipped="word") + first = sim.step(1, 2, "u1") # level 3 first: levels 1 and 2 were never numbered in this list + assert (first.count, first.lo_count) == (1, 1) + second = sim.step(1, 1, "u2") + assert (second.count, second.lo_count) == (1, 2) and second.label == "(i)" + third = sim.step(1, 1, "u3") + assert (third.count, third.lo_count) == (2, 3) + # a level that was numbered is counted the same in both + fresh = numbering(Num(1, 1), skipped="word") + assert [(step.count, step.lo_count) for step in (fresh.step(1, 0, "a"), fresh.step(1, 1, "b"), fresh.step(1, 1, "c"))] \ + == [(1, 1), (1, 1), (2, 2)] + + +def test_the_first_paragraph_of_a_list_deeper_than_the_level_it_starts_at_fills_the_levels_above() -> None: + sim = numbering(Num(1, 1, ((1, 1),))) + assert sim.step(1, 1, "u1").label == "(i)" + + +def test_letters_past_twenty_six_repeat_in_the_word_model() -> None: + sim = numbering(Num(1, 1)) + labels = [sim.step(1, 0, f"u{i}").own for i in range(1, 30)] + assert labels[25:] == ["z", "aa", "bb", "cc"] + + +def test_an_abstract_num_may_gain_its_levels_after_it_is_added() -> None: + levels = _levels(("decimal", "%1.")) + sim = WordNumbering() + sim.add_abstract(0, levels) + sim.add_num(Num(1, 0)) + levels[1] = LevelDef(1, "lowerLetter", "%2)") + sim.step(1, 0, "a") + assert sim.step(1, 1, "b").label == "a)" + + +def test_abstract_num_is_frozen_data() -> None: + abstract = AbstractNum(0, "headings", "4C440000", "multilevel", tuple(HEADINGS)) + with pytest.raises(AttributeError): + abstract.id = 1 # type: ignore[misc] diff --git a/tests/test_docx_special.py b/tests/test_docx_special.py new file mode 100644 index 0000000..c2f85ed --- /dev/null +++ b/tests/test_docx_special.py @@ -0,0 +1,206 @@ +"""What the planner's hard cases become in the file (docs/architecture.md, "The DOCX planner"): the heading list's levels +that hold items, the reserved levels of the lists nested in them (with the path of the item in their text), the detached +levels, the abstractNums that go on past Word's nine levels, and the later alternatives that share a number, as +numbering definitions, ``w:numPr``, ``w:ind`` and ``REF`` fields; and that every document that has them validates +against the ISO/IEC 29500-4 schemas (marker ``ooxml_schema``). The numbers as Word's simulator and LibreOffice draw them +are checked in test_docx_plan.py and test_docx_roundtrip.py.""" +from __future__ import annotations + +import re +import xml.etree.ElementTree as ET + +import pytest +from conftest import FRONT +from docx_support import W, parts, require_schemas, validation_errors +from test_docx_plan import ALTERNATIVES, ORDERED_PARENT, PATH_ITEMS +from test_docx_roundtrip import STRESS + +from legaldown_render import render + + +def document(source: str, **settings: object) -> dict[str, ET.Element]: + result = render(source, format="docx", **settings) + assert result.binary is not None + return {name: ET.fromstring(content) for name, content in parts(result.binary).items() + if name in ("word/document.xml", "word/numbering.xml")} + + +def body_paragraphs(root: ET.Element) -> list[ET.Element]: + body = root.find(f"{W}body") + assert body is not None + return body.findall(f"{W}p") + + +def text_of(paragraph: ET.Element) -> str: + return "".join(node.text or "" for node in paragraph.iter(f"{W}t")) + + +def paragraph_with(root: ET.Element, text: str) -> ET.Element: + [found] = [p for p in body_paragraphs(root) if text_of(p).endswith(text)] + return found + + +def numbering_of(paragraph: ET.Element) -> tuple[int | None, int | None]: + ilvl = paragraph.find(f"{W}pPr/{W}numPr/{W}ilvl") + num = paragraph.find(f"{W}pPr/{W}numPr/{W}numId") + return (None if ilvl is None else int(ilvl.get(f"{W}val")), # type: ignore[arg-type] + None if num is None else int(num.get(f"{W}val"))) # type: ignore[arg-type] + + +def indent(paragraph: ET.Element) -> tuple[int, int] | None: + found = paragraph.find(f"{W}pPr/{W}ind") + return None if found is None else (int(found.get(f"{W}left")), int(found.get(f"{W}hanging", "0"))) # type: ignore[arg-type] + + +def levels(numbering: ET.Element, abstract_id: int) -> list[ET.Element]: + [abstract] = [a for a in numbering.findall(f"{W}abstractNum") if a.get(f"{W}abstractNumId") == str(abstract_id)] + return abstract.findall(f"{W}lvl") + + +def instructions(paragraph: ET.Element) -> list[str]: + return [n.text.strip() for n in paragraph.iter(f"{W}instrText") if n.text] + + +CONTINENTAL = FRONT + "# Alpha\n\n- clause one {#c1}\n - nested a\n - nested b\n - deeper\n- clause two\n" + + +def test_an_item_on_the_heading_list_is_numbered_at_the_level_after_its_section_with_an_indent_of_its_own() -> None: + parts_ = document(CONTINENTAL, style="continental") + one, two = paragraph_with(parts_["word/document.xml"], "clause one"), paragraph_with( + parts_["word/document.xml"], "clause two") + assert numbering_of(one) == numbering_of(two) == (1, 2) + assert one.find(f"{W}pPr/{W}pStyle") is None # no heading style: it is no heading + assert indent(one) == indent(two) is not None + level = levels(parts_["word/numbering.xml"], 0)[1] + assert level.find(f"{W}lvlText").get(f"{W}val") == "%1.%2" and level.find(f"{W}pStyle").get(f"{W}val") == "Heading2" + # the num of the list restarts the level at one, as every top-level list does + [num] = [n for n in parts_["word/numbering.xml"].findall(f"{W}num") if n.get(f"{W}numId") == "2"] + override = num.find(f"{W}lvlOverride") + assert override.get(f"{W}ilvl") == "1" and override.find(f"{W}startOverride").get(f"{W}val") == "1" # type: ignore[union-attr] + + +def test_a_nested_item_uses_a_reserved_level_of_the_same_num_with_the_indent_one_step_in() -> None: + parts_ = document(CONTINENTAL, style="continental") + root = parts_["word/document.xml"] + one, a, b, deeper = (paragraph_with(root, t) for t in ("clause one", "nested a", "nested b", "deeper")) + assert [numbering_of(p) for p in (a, b, deeper)] == [(5, 2), (5, 2), (6, 2)] # the parent's num, no override of its own + assert indent(a) == (indent(one)[0] + 567, 567) and indent(deeper) == (indent(one)[0] + 1134, 567) # type: ignore[index] + levels_ = levels(parts_["word/numbering.xml"], 0) + assert [(lvl.find(f"{W}numFmt").get(f"{W}val"), lvl.find(f"{W}lvlText").get(f"{W}val")) for lvl in levels_[5:7]] == [ + ("lowerLetter", "%6)"), ("lowerRoman", "%7)")] + assert [n.get(f"{W}numId") for n in parts_["word/numbering.xml"].findall(f"{W}num")] == ["1", "2"] + + +SHARED = ALTERNATIVES + """ +# Disputes {#d when=forum:courts} + +## Sub one {#s1} + +# Disputes {#d when=forum:arb} + +## Sub alt {#sa} + +- courts item {#ci when=forum:courts} +- arbitration item {#ci when=forum:arb} + - nested under it {#an} +- after +""" + + +def test_a_later_alternative_has_numid_zero_an_indent_of_its_own_and_the_number_of_the_holder_as_a_field() -> None: + parts_ = document(SHARED) + root = parts_["word/document.xml"] + heading = [p for p in body_paragraphs(root) if text_of(p).endswith("Disputes Only if: forum:arb")][0] + assert numbering_of(heading) == (0, 0) and indent(heading) == (567, 567) + assert heading.find(f"{W}pPr/{W}pStyle").get(f"{W}val") == "Heading1" # type: ignore[union-attr] + # the number: a REF field for the holder (with \n and \h), the period as text, then a tab, then the title + fields = instructions(heading) + assert len(fields) == 1 and re.fullmatch(r"REF \S+ \\n \\h", fields[0]) + assert text_of(heading).startswith("1.") # the cached result "1" and the period + runs = [r for r in heading.findall(f"{W}r")] + assert any(r.find(f"{W}tab") is not None for r in runs) + # the holder carries the bookmark the field points at + name = fields[0].split()[1] + assert name in {b.get(f"{W}name") for b in root.iter(f"{W}bookmarkStart")} + item = paragraph_with(root, "arbitration item") + [field] = instructions(item) + assert numbering_of(item) == (0, 0) and re.fullmatch(r"REF \S+ \\n \\h", field) + assert field.split()[1] in {b.get(f"{W}name") for b in root.iter(f"{W}bookmarkStart")} + # the first alternative and the section that follows are numbered by the list as usual + assert numbering_of(paragraph_with(root, "courts item")) == (0, 3) + assert numbering_of(paragraph_with(root, "after")) == (0, 3) + + +def test_the_children_of_a_later_alternative_are_numbered_by_a_num_of_their_own() -> None: + parts_ = document(SHARED) + root, numbering = parts_["word/document.xml"], parts_["word/numbering.xml"] + sub_alt, nested = paragraph_with(root, "Sub alt"), paragraph_with(root, "nested under it") + (_, sub_num), (_, nested_num) = numbering_of(sub_alt), numbering_of(nested) + assert sub_num == 2 and nested_num == 4 # the sections after the alternative; the list nested under the alternative item + for num_id, level in ((sub_num, "1"), (nested_num, "1")): + [num] = [n for n in numbering.findall(f"{W}num") if n.get(f"{W}numId") == str(num_id)] + assert num.find(f"{W}lvlOverride").get(f"{W}ilvl") == level # type: ignore[union-attr] + + +@pytest.mark.ooxml_schema +@pytest.mark.parametrize("name", ["alternatives", "alternatives in threes", "qualified", "level five", "nested titles"]) +@pytest.mark.parametrize("settings", [{"style": "continental"}, {}, {"overrides": {"numbering.scheme": "none"}}, + {"style": "continental", "overrides": {"paragraphs.numbered": True}}]) +def test_documents_with_alternatives_and_items_on_the_heading_list_validate(name: str, settings: dict) -> None: + require_schemas() + result = render(STRESS[name], format="docx", **settings) + assert result.binary is not None + assert validation_errors(result.binary) == [] + + +def lvl_texts(numbering: ET.Element, abstract_id: int) -> list[str]: + return [lvl.find(f"{W}lvlText").get(f"{W}val") for lvl in levels(numbering, abstract_id)] # type: ignore[misc, union-attr] + + +def test_a_path_label_on_the_reserved_levels_is_a_level_text_with_the_counters_of_its_chain() -> None: + source = FRONT + "# A\n\n- one\n - two {#two}\n - three {#three}\n- uno\n - dos\n" + parts_ = document(source, **PATH_ITEMS) + root, numbering = parts_["word/document.xml"], parts_["word/numbering.xml"] + assert lvl_texts(numbering, 0)[5:7] == ["%1.%2(%6)", "%1.%2(%6)(%7)"] + assert [numbering_of(paragraph_with(root, t)) for t in ("two", "three", "dos")] == [(5, 2), (6, 2), (5, 2)] + # the parent's num, no override of its own, and no abstractNum besides the heading list's + assert len(numbering.findall(f"{W}abstractNum")) == 1 + assert [n.get(f"{W}numId") for n in numbering.findall(f"{W}num")] == ["1", "2"] + + +def test_a_detached_abstract_num_holds_each_levels_own_counter_and_a_list_in_it_has_a_start_override() -> None: + source = FRONT + "# A\n\n- one\n 1. ordered\n - three\n - four {#four}\n - four b\n" + parts_ = document(source, **ORDERED_PARENT) + root, numbering = parts_["word/document.xml"], parts_["word/numbering.xml"] + abstracts = numbering.findall(f"{W}abstractNum") + assert len(abstracts) == 4 # headings, enumerated, renumber, detached + assert lvl_texts(numbering, 1)[3] == "(%1)(%2)(%3)(%4)" and lvl_texts(numbering, 3)[:4] == ["(%1)", "(%2)", "%3", "(%4)"] + four = paragraph_with(root, "four") + ilvl, num_id = numbering_of(four) + assert ilvl == 3 + [num] = [n for n in numbering.findall(f"{W}num") if n.get(f"{W}numId") == str(num_id)] + assert num.find(f"{W}abstractNumId").get(f"{W}val") == "3" # type: ignore[union-attr] + assert num.find(f"{W}lvlOverride/{W}startOverride").get(f"{W}val") == "1" # type: ignore[union-attr] + assert numbering_of(paragraph_with(root, "four b"))[1] == num_id + + +def test_a_list_nested_past_nine_levels_goes_on_in_the_next_abstract_num() -> None: + source = FRONT + "# A\n\n" + "".join(" " * d + f"- level {d}\n" for d in range(11)) + parts_ = document(source) + root, numbering = parts_["word/document.xml"], parts_["word/numbering.xml"] + kinds = len(numbering.findall(f"{W}abstractNum")) + assert kinds == 3 # headings, enumerated, enumerated+1 + assert [numbering_of(paragraph_with(root, f"level {d}")) for d in (8, 9, 10)] == [(8, 2), (0, 3), (1, 3)] + assert lvl_texts(numbering, 2)[:2] == ["(%1)", "(%2)"] + + +@pytest.mark.ooxml_schema +@pytest.mark.parametrize(("name", "settings"), [ + ("paths", PATH_ITEMS), ("paths", {}), ("ordered paths", ORDERED_PARENT), ("deep", {}), ("deep", PATH_ITEMS), + ("skipped", {"style": "outline"}), ("nested", {"overrides": {"docx.reference_text_switch": "guess"}}), + ("nested", {"overrides": {"docx.reference_text_switch": "off"}})]) +def test_documents_with_detached_levels_deep_lists_and_the_switch_validate(name: str, settings: dict) -> None: + require_schemas() + result = render(STRESS[name], format="docx", **settings) + assert result.binary is not None + assert validation_errors(result.binary) == [] diff --git a/tests/test_docx_style.py b/tests/test_docx_style.py new file mode 100644 index 0000000..3ed1904 --- /dev/null +++ b/tests/test_docx_style.py @@ -0,0 +1,335 @@ +"""The style settings the DOCX writer adds: ``page.header``, ``page.footer`` and +``docx.update_fields_on_open`` (docs/style-templates.md), and how a style's CSS-valued +settings become Word's units, with the problems only DOCX has raised as a +:class:`StyleError` before anything is rendered (docs/decisions/0006, 0008).""" +from __future__ import annotations + +import pytest +from conftest import FRONT + +from legaldown_render import RenderOptions, StyleError, dump_style, load_style, render +from legaldown_render.style import Docx, Page, PageText, parse_override +from legaldown_render.style.docx import docx_settings + +# -- page.header and page.footer --------------------------------------------------------- + + +def test_the_default_footer_is_the_page_number_and_count_and_the_header_is_empty() -> None: + page = load_style("default").page + assert page.footer == PageText(center="{page} / {pages}") + assert page.header == PageText() + assert not page.header + assert page.footer + + +def test_header_and_footer_are_set_by_side_in_a_file_and_by_a_dotted_override() -> None: + style = load_style("default", overrides={"page.header.left": "{title}", "page.header.right": "{version}, {date}", + "page.footer": {"center": "", "right": "{page}"}}) + assert style.page.header == PageText(left="{title}", right="{version}, {date}") + assert style.page.footer == PageText(right="{page}") + + +def test_every_field_is_accepted_in_every_position() -> None: + template = "{page} {pages} {title} {version} {date}" + overrides = {f"page.{part}.{side}": template for part in ("header", "footer") for side in ("left", "center", "right")} + assert load_style("default", overrides=overrides).page.header.center == template + + +@pytest.mark.parametrize("key", ["page.header.left", "page.header.center", "page.header.right", + "page.footer.left", "page.footer.center", "page.footer.right"]) +def test_an_unknown_field_is_a_style_error_naming_the_setting(key: str) -> None: + with pytest.raises(StyleError) as caught: + load_style("default", overrides={key: "{pgae}"}) + assert any(problem.startswith(f"{key}: unknown field {{pgae}}") for problem in caught.value.problems) + assert "{date}, {page}, {pages}, {title}, {version}" in str(caught.value) + + +def test_braces_that_are_not_a_field_are_text() -> None: + style = load_style("default", overrides={"page.footer.center": "{} {1} {a b}"}) + assert style.page.footer.center == "{} {1} {a b}" + + +def test_a_header_text_is_a_free_text_setting_with_the_untrusted_caps() -> None: + assert load_style("default", overrides={"page.header.left": "x" * 200}, trusted=False).page.header.left + with pytest.raises(StyleError, match="page.header.left: is longer than 200 characters"): + load_style("default", overrides={"page.header.left": "x" * 201}, trusted=False) + with pytest.raises(StyleError, match="page.footer.right: must not contain NUL or a lone surrogate"): + load_style("default", overrides={"page.footer.right": "a\x00b"}, trusted=False) + + +def test_an_untrusted_style_does_not_quote_its_unknown_field() -> None: + with pytest.raises(StyleError) as caught: + load_style("default", overrides={"page.header.left": "{secret}"}, trusted=False) + assert "secret" not in str(caught.value) + assert "unknown field a {field}" in str(caught.value) + + +def test_a_header_is_checked_as_text_on_the_command_line() -> None: + assert parse_override("page.header.left", "{title}") == "{title}" + assert parse_override("page.footer.center", "'{page}'") == "{page}" + + +def test_print_style_writes_header_and_footer_and_loads_again() -> None: + dumped = dump_style(load_style("default", overrides={"page.header.center": "{title}"})) + assert "header:" in dumped and "footer:" in dumped and "update_fields_on_open: true" in dumped + assert load_style(dumped).page.header.center == "{title}" + + +def test_a_page_object_built_by_hand_is_checked_like_data() -> None: + from legaldown_render.style import Style + + with pytest.raises(StyleError, match="page.footer.center: unknown field"): + load_style(Style(page=Page(footer=PageText(center="{nope}")))) + + +# -- docx.update_fields_on_open -------------------------------------------------------- + + +def test_fields_are_updated_on_open_by_default_and_the_setting_is_a_boolean() -> None: + assert load_style("default").docx == Docx(update_fields_on_open=True) + assert load_style("default", overrides={"docx.update_fields_on_open": False}).docx.update_fields_on_open is False + assert parse_override("docx.update_fields_on_open", "false") is False + with pytest.raises(StyleError, match="docx.update_fields_on_open: must be true or false"): + load_style("default", overrides={"docx.update_fields_on_open": "maybe"}) + + +# -- docx.reference_text_switch ----------------------------------------------------------- + + +def test_the_text_switch_is_documented_by_default_and_the_setting_is_one_of_three_words() -> None: + assert load_style("default").docx == Docx(update_fields_on_open=True, reference_text_switch="documented") + for word in ("off", "documented", "guess"): + style = load_style("default", overrides={"docx.reference_text_switch": word}) + assert style.docx.reference_text_switch == word + assert docx_settings(style).reference_text_switch == word + assert parse_override("docx.reference_text_switch", word) == word + with pytest.raises(StyleError, match="docx.reference_text_switch: must be one of off, documented, guess"): + load_style("default", overrides={"docx.reference_text_switch": "always"}) + with pytest.raises(StyleError, match="docx.reference_text_switch: must be one of"): + load_style("default", overrides={"docx.reference_text_switch": True}) + assert "reference_text_switch: documented" in dump_style(load_style("default")) + assert load_style(dump_style(load_style("default", overrides={"docx.reference_text_switch": "guess"})) + ).docx.reference_text_switch == "guess" + + +def test_an_untrusted_style_does_not_quote_the_value_of_the_text_switch() -> None: + with pytest.raises(StyleError) as caught: + load_style("default", overrides={"docx.reference_text_switch": "secret"}, trusted=False) + assert "secret" not in str(caught.value) and "must be one of off, documented, guess" in str(caught.value) + + +def test_an_unknown_docx_key_is_a_style_error_with_a_suggestion() -> None: + with pytest.raises(StyleError, match=r"docx.update_field_on_open: unknown key \(did you mean 'update_fields_on_open'"): + load_style("default", overrides={"docx.update_field_on_open": True}) + + +# -- Lengths, colours, fonts: CSS in, Word's units out ----------------------------------- + + +def settings(**overrides: object): # noqa: ANN201 - a test helper + return docx_settings(load_style("default", overrides=overrides)) + + +def test_the_default_style_in_word_units() -> None: + word = settings() + assert (word.page.width, word.page.height, word.page.landscape) == (11906, 16838, False) + assert (word.page.top, word.page.right, word.page.bottom, word.page.left) == (1417,) * 4 + assert word.page.text_width == 11906 - 2 * 1417 + assert (word.font, word.half_points, word.color, word.link_color) == ("Georgia", 22, "1A1A1A", "1F4E8C") + assert (word.line, word.line_rule) == (360, "auto") + assert word.update_fields_on_open is True + assert word.headings[1].half_points == 25 and word.headings[1].caps and word.headings[1].bold + assert word.headings[3].italic and not word.headings[5].bold + + +@pytest.mark.parametrize(("margin", "expected"), [ + ("1in", (1440,) * 4), ("10mm", (567,) * 4), ("2cm 1cm", (1134, 567, 1134, 567)), + ("1cm 2cm 3cm", (567, 1134, 1701, 1134)), ("1cm 2cm 3cm 4cm", (567, 1134, 1701, 2268)), + ("36pt", (720,) * 4), ("1pc", (240,) * 4), ("96px", (1440,) * 4), ("0", (0,) * 4), + ("1em", (220,) * 4), ("2rem", (440,) * 4), +]) +def test_margins_are_css_shorthand_in_twips(margin: str, expected: tuple[int, ...]) -> None: + page = settings(**{"page.margin": margin}).page + assert (page.top, page.right, page.bottom, page.left) == expected + + +def test_em_and_rem_follow_the_font_size() -> None: + page = settings(**{"page.margin": "2em", "typography.font_size": "10pt"}).page + assert page.top == 400 + big = settings(**{"typography.font_size": "2em"}) + assert big.half_points == 48 # 2em of the browser's 12pt + assert settings(**{"typography.font_size": "1rem"}).half_points == 24 + assert settings(**{"headings.2.size": "2em", "typography.font_size": "10pt"}).headings[2].half_points == 40 + + +@pytest.mark.parametrize(("size", "expected"), [ + ("A4", (11906, 16838, False)), ("a3", (16838, 23811, False)), ("A5", (8391, 11906, False)), + ("letter", (12240, 15840, False)), ("Legal", (12240, 20160, False)), ("B5", (9979, 14173, False)), + ("A4 landscape", (16838, 11906, True)), ("landscape", (16838, 11906, True)), ("auto", (11906, 16838, False)), + ("letter portrait", (12240, 15840, False)), ("210mm 297mm", (11906, 16838, False)), + ("297mm 210mm", (16838, 11906, True)), ("10cm", (5669, 5669, False)), +]) +def test_page_sizes(size: str, expected: tuple[int, int, bool]) -> None: + page = settings(**{"page.size": size}).page + assert (page.width, page.height, page.landscape) == expected + + +@pytest.mark.parametrize(("height", "line", "rule"), [ + ("normal", 240, "auto"), ("1", 240, "auto"), ("1.5", 360, "auto"), ("2", 480, "auto"), + ("18pt", 360, "atLeast"), ("1.5em", 330, "atLeast"), +]) +def test_line_height(height: str, line: int, rule: str) -> None: + word = settings(**{"typography.line_height": height}) + assert (word.line, word.line_rule) == (line, rule) + + +@pytest.mark.parametrize(("color", "expected"), [ + ("#1a1a1a", "1A1A1A"), ("#abc", "AABBCC"), ("#abcd", "AABBCC"), ("#11223344", "112233"), ("red", "FF0000"), + ("Navy", "000080"), ("rgb(255, 0, 128)", "FF0080"), ("rgb(100% 0% 0%)", "FF0000"), ("rgba(0,0,255,0.5)", "0000FF"), + ("hsl(120, 100%, 25%)", "008000"), ("hsl(0 0% 100%)", "FFFFFF"), +]) +def test_colours(color: str, expected: str) -> None: + assert settings(**{"typography.color": color}).color == expected + + +@pytest.mark.parametrize(("family", "expected"), [ + ('Georgia, "Times New Roman", serif', "Georgia"), ('"Times New Roman", serif', "Times New Roman"), + ("'Open Sans', sans-serif", "Open Sans"), ("serif", "Times New Roman"), ("sans-serif", "Arial"), + ("monospace", "Courier New"), ("Calibri", "Calibri"), ("", "Times New Roman"), +]) +def test_the_first_font_family(family: str, expected: str) -> None: + assert settings(**{"typography.font_family": family}).font == expected + + +@pytest.mark.parametrize(("weight", "bold"), [("700", True), ("bold", True), ("600", True), ("400", False), + ("normal", False), ("lighter", False), ("bolder", True)]) +def test_heading_weight(weight: str, bold: bool) -> None: + assert settings(**{"headings.2.weight": weight}).headings[2].bold is bold + + +def test_uppercase_and_small_caps_and_alignment() -> None: + word = settings(**{"headings.2.transform": "small-caps", "headings.2.align": "center"}) + assert word.headings[2].small_caps and not word.headings[2].caps and word.headings[2].align == "center" + + +# -- What Word cannot express is a StyleError for DOCX only -------------------------------- + + +@pytest.mark.parametrize("unit", ["%", "ch", "ex"]) +@pytest.mark.parametrize("key", ["page.margin", "page.size", "typography.font_size", "typography.line_height", + "headings.1.size"]) +def test_relative_units_word_cannot_express_are_a_style_error_for_docx(key: str, unit: str) -> None: + style = load_style("default", overrides={key: f"5{unit}"}) # HTML takes any of them + with pytest.raises(StyleError) as caught: + docx_settings(style) + assert any(problem.startswith(f"{key}: uses '{unit}'") for problem in caught.value.problems) + assert render(FRONT + "# A\n", RenderOptions(style=style)).output.startswith("") + + +def test_the_render_api_refuses_before_rendering_and_html_still_renders() -> None: + options = {"page.margin": "5%", "typography.color": "var(--ink)", "typography.font_size": "3ch"} + with pytest.raises(StyleError) as caught: + render(FRONT + "# A\n", format="docx", overrides=options) + assert {problem.split(":")[0] for problem in caught.value.problems} == { + "typography.font_size", "page.margin", "typography.color"} + assert render(FRONT + "# A\n", format="html", overrides=options).ok + + +@pytest.mark.parametrize(("key", "value", "fragment"), [ + ("page.margin", "1cm 2cm 3cm 4cm 5cm", "must be one to four lengths"), + ("page.margin", "auto", "not a length the DOCX writer can read"), + ("page.margin", "-1cm", "must not be negative"), + ("page.margin", "5", "needs a unit"), + ("page.size", "A4 A3", "not a length"), + ("page.size", "210mm landscape", "orientation goes with a paper size"), + ("page.size", "0 10mm", "greater than 0"), + ("typography.font_size", "0", "between 1pt and 1023pt"), + ("typography.font_size", "2000pt", "between 1pt and 1023pt"), + ("headings.1.size", "2000pt", "between 1pt and 1638pt"), + ("typography.line_height", "0", "greater than 0"), + ("typography.line_height", "calc(1 + 1)", "not a length"), + ("typography.color", "hsl(", "cannot read"), + ("typography.link_color", "rebeccapurple", "write it as #rrggbb"), + ("headings.1.weight", "heavy", "not a weight"), +]) +def test_other_values_word_cannot_read_are_named(key: str, value: str, fragment: str) -> None: + with pytest.raises(StyleError) as caught: + docx_settings(load_style("default", overrides={key: value})) + assert any(problem.startswith(f"{key}: ") and fragment in problem for problem in caught.value.problems), \ + caught.value.problems + + +def test_every_problem_is_reported_together_and_an_untrusted_style_is_not_quoted() -> None: + style = load_style("default", overrides={"page.margin": "5%", "typography.font_size": "9ch"}, trusted=False) + with pytest.raises(StyleError) as caught: + docx_settings(style, trusted=False) + assert len(caught.value.problems) == 2 + assert caught.value.source == "'untrusted style'" + assert "5%" not in str(caught.value) and "9ch" not in str(caught.value) + + +def test_the_built_in_styles_convert() -> None: + for name in ("default", "outline", "continental"): + docx_settings(load_style(name)) + + +# -- Numeric extremes: a StyleError, never a crash and never a value Word or the schema refuses --------------- + +DIGITS = ["1" + "0" * 40, "9" * 30, "9" * 33, "9" * 100, "9" * 400, "9" * 5000, "1" + "0" * 40 + "." + "9" * 40] +LENGTH_KEYS = ["page.size", "page.margin", "typography.font_size", "typography.line_height", "headings.1.size"] + + +@pytest.mark.parametrize("unit", ["mm", "pt", "in", "em", "px"]) +@pytest.mark.parametrize("digits", DIGITS) +@pytest.mark.parametrize("key", LENGTH_KEYS) +def test_an_enormous_length_is_a_style_error_that_does_not_quote_it(key: str, digits: str, unit: str) -> None: + for trusted in (True, False) if len(digits) < 150 else (True,): # an untrusted text is at most 200 characters + style = load_style("default", overrides={key: digits + unit}, trusted=trusted) + with pytest.raises(StyleError) as caught: + docx_settings(style, trusted=trusted) + assert digits[:12] not in str(caught.value) + with pytest.raises(StyleError): + render(FRONT + "# A\n", format="docx", overrides={key: digits + unit}) + + +@pytest.mark.parametrize("value", ["inf", "nan", "1e999", "9" * 400, "-" + "9" * 400]) +@pytest.mark.parametrize("function", ["rgb({0}, 0, 0)", "rgb(0 {0} 0)", "rgba(0,0,{0},1)", "hsl({0} 50% 50%)", + "hsl(10 {0}% 50%)", "hsl(10 50% {0}%)"]) +def test_a_colour_component_that_is_not_a_finite_number_is_a_style_error(function: str, value: str) -> None: + with pytest.raises(StyleError, match="typography.color: is a colour the DOCX writer cannot read"): + docx_settings(load_style("default", overrides={"typography.color": function.format(value)})) + + +@pytest.mark.parametrize(("key", "value", "fragment"), [ + ("page.size", "7pt", "each edge must be between 7.2pt and 1584pt"), + ("page.size", "1600pt 1600pt", "each edge must be between"), + ("page.size", "5pt 20in", "each edge must be between"), + ("page.margin", "1600pt", "each margin must be between 0 and 1584pt"), + ("page.margin", "50mm 100mm", "leave no room for text"), + ("page.margin", "150mm 0", "leave no room for text"), + ("page.margin", "105mm", "leave no room for text"), + ("typography.line_height", "1600pt", "at most 1584pt"), + ("typography.font_size", "1024pt", "between 1pt and 1023pt"), +]) +def test_values_outside_word_s_limits_are_named(key: str, value: str, fragment: str) -> None: + with pytest.raises(StyleError) as caught: + docx_settings(load_style("default", overrides={key: value})) + assert any(problem.startswith(f"{key}: ") and fragment in problem for problem in caught.value.problems), \ + caught.value.problems + + +def test_the_largest_values_that_fit_are_accepted_and_nothing_is_negative_or_overflows() -> None: + word = docx_settings(load_style("default", overrides={ + "page.size": "22in 22in", "page.margin": "0", "typography.font_size": "1023pt", + "typography.line_height": "1584pt", "headings.1.size": "1638pt"})) + assert (word.page.width, word.page.height, word.page.text_width) == (31680, 31680, 31680) + assert word.half_points == 2046 and round(word.half_points * 1.6) <= 3276 + narrow = docx_settings(load_style("default", overrides={"page.size": "1.5in 3in", "page.margin": "0.5in"})) + assert narrow.page.text_width == 720 and narrow.page.header >= 0 + + +@pytest.mark.parametrize("value", ["25MM", "25Mm", "1IN", "72PT", "6PC", "96PX", "1EM", "1.5REM"]) +def test_units_are_not_case_sensitive(value: str) -> None: + assert docx_settings(load_style("default", overrides={"page.margin": value})).page.top > 0 + with pytest.raises(StyleError, match="uses '%'"): + docx_settings(load_style("default", overrides={"page.margin": "5%"})) diff --git a/tests/test_docx_validation.py b/tests/test_docx_validation.py new file mode 100644 index 0000000..78a7d93 --- /dev/null +++ b/tests/test_docx_validation.py @@ -0,0 +1,172 @@ +"""Every part of every generated document validates against the ISO/IEC 29500-4 transitional +schemas (WordprocessingML, content types, relationships, extended properties), and the package +is sound: each relationship targets a part, each part has a content type. The schemas are +fetched, not stored (tests/schemas/README.md); the tests skip without them, and fail instead when +``LEGALDOWN_REQUIRE_DOCX_TOOLS`` is set, as in CI.""" +from __future__ import annotations + +import random + +import pytest +from conftest import DOCUMENTS, spec_root +from docx_support import parts, repackage, require_schemas, schema_for, tool_missing, validation_errors +from test_docx_escaping import FREE_TEXT_SETTINGS, document + +from legaldown_render import DocumentError, render + +pytestmark = pytest.mark.ooxml_schema + +DOCUMENT_NAMES = ("features", "template") +STYLES = ("default", "continental", "outline") + + +def make(source: str, **settings: object) -> bytes: + result = render(source, format="docx", **settings) + assert result.binary is not None + return result.binary + + +@pytest.fixture(autouse=True) +def _schemas() -> None: + require_schemas() + + +@pytest.mark.parametrize("style", STYLES) +@pytest.mark.parametrize("name", DOCUMENT_NAMES) +def test_the_test_documents_validate_under_every_built_in_style(name: str, style: str) -> None: + source = (DOCUMENTS / f"{name}.lgd").read_text(encoding="utf-8") + assert validation_errors(make(source, style=style)) == [] + + +def test_every_xml_part_has_a_schema_except_the_core_properties() -> None: + names = parts(make("---\ntitle: T\n---\n# A\n")) + assert {name: schema_for(name) for name in names} == { + "[Content_Types].xml": "types", "_rels/.rels": "rels", "docProps/core.xml": None, "docProps/app.xml": "app", + "word/document.xml": "wml", "word/_rels/document.xml.rels": "rels", "word/styles.xml": "wml", + "word/numbering.xml": "wml", "word/settings.xml": "wml", "word/footer1.xml": "wml"} + + +@pytest.mark.parametrize("overrides", [ + {}, + {"page.footer": {"center": ""}}, + {"page.header.left": "{title}", "page.header.center": "{date}", "page.header.right": "{page}/{pages} {version}"}, + {"page.header.center": "C", "page.footer": {"left": "L", "center": "", "right": "R"}}, + {"page.header.right": "R", "page.footer.center": ""}, + {"docx.update_fields_on_open": False}, + {"page.size": "A4 landscape", "page.margin": "1in 2in 3in 4in"}, + {"page.size": "letter", "page.margin": "0"}, + {"page.size": "10cm 20cm", "typography.font_size": "9pt", "typography.line_height": "14pt"}, + {"typography.justify": True, "typography.color": "rgb(10, 20, 30)", "typography.link_color": "hsl(200, 50%, 40%)"}, + {"definitions.style": "small-caps", "definitions.term_style": "underline"}, + {"headings": {1: {"transform": "small-caps", "align": "center", "italic": True, "weight": "400"}}}, + {"typography.font_family": "'Open Sans', sans-serif"}, + {"page.size": "22in 22in", "page.margin": "0", "typography.font_size": "1023pt", "typography.line_height": "1584pt", + "headings.1.size": "1638pt"}, + {"page.size": "1.5in 3in", "page.margin": "0.5in", "typography.font_size": "1pt"}, + {"page.margin": "25MM 1IN", "typography.font_size": "12PT"}, +]) +def test_the_settings_that_shape_the_parts_keep_them_valid(overrides: dict) -> None: + source = ("---\nlegaldown: '0.2'\ntitle: Agreement\nversion: '1'\neffective_date: 2026-01-02\n---\n\n" + "# One\n\ntext\n\n## Two\n\ntext\n") + assert validation_errors(make(source, overrides=overrides)) == [] + + +def test_an_empty_document_validates() -> None: + assert validation_errors(make("---\ntitle: T\n---\n")) == [] + + +def moved(text: str, first: str, second: str) -> str: + """*text* with the element that starts with *first* moved after the one that starts with *second* + (both self-closing, *first* the earlier).""" + start = text.index(first) + end = text.index("/>", start) + 2 + element = text[start:end] + after = text.index("/>", text.index(second)) + 2 + return text[:start] + text[end:after] + element + text[after:] + + +def test_the_schema_check_catches_a_wrong_order_in_the_settings() -> None: + """The validation is not vacuous: ``updateFields`` after ``compat`` (the order both prototypes + had) is rejected.""" + original = make("---\ntitle: T\n---\n# A\n") + settings = parts(original)["word/settings.xml"].decode("utf-8") + element = '' + assert element in settings + broken = repackage(original, "word/settings.xml", + lambda text: text.replace(element, "").replace("", element + "")) + assert any("updateFields" in problem for problem in validation_errors(broken)) + + +def test_the_schema_check_catches_an_element_out_of_order_in_a_style_and_in_the_section() -> None: + original = make("---\ntitle: T\n---\n# A\n") + assert validation_errors(original) == [] + style = repackage(original, "word/styles.xml", + lambda text: text.replace('', + '', 1)) + assert any("styles.xml" in problem for problem in validation_errors(style)) + section = repackage(original, "word/document.xml", lambda text: moved(text, " None: + rnd = random.Random(1000 + seed) + overrides = {key: "".join(rnd.choice("ab \x01\x00\ud800<&ž") for _ in range(10)) + "{page}" for key in FREE_TEXT_SETTINGS} + assert validation_errors(make(document(rnd), overrides=overrides)) == [] + + +UNITS = ["mm", "cm", "in", "pt", "pc", "px", "em", "rem", "%", "ch", "ex", "", "MM"] + + +def random_length(rnd: random.Random) -> str: + """A number as small, as large or as odd as a style might write it.""" + whole = rnd.choice(["0", str(rnd.randint(0, 40)), str(rnd.randint(0, 100000)), "9" * rnd.randint(1, 45), + "1" + "0" * rnd.randint(1, 400)]) + fraction = rnd.choice(["", "", "." + "".join(rnd.choice("0123456789") for _ in range(rnd.randint(1, 40)))]) + return f"{whole}{fraction}{rnd.choice(UNITS)}" + + +@pytest.mark.parametrize("seed", range(60)) +def test_random_lengths_are_a_style_error_or_a_valid_document(seed: int) -> None: + from legaldown_render import StyleError + + rnd = random.Random(5000 + seed) + keys = ["page.size", "page.margin", "typography.font_size", "typography.line_height", "headings.1.size", + "headings.3.size"] + overrides = {key: " ".join(random_length(rnd) for _ in range(rnd.randint(1, 4))) + for key in rnd.sample(keys, rnd.randint(1, len(keys)))} + try: + data = make("---\ntitle: T\n---\n# A\n\ntext\n", overrides=overrides) + except StyleError as error: + assert not any(value.split()[0][:12] in str(error) for value in overrides.values() if len(value.split()[0]) > 12) + return + assert validation_errors(data) == [] + + +# -- The specification's own documents --------------------------------------------------------------- + +SPEC = spec_root() +SPEC_FILES = sorted(path for folder in ("examples", "fixtures") for path in (SPEC / folder).rglob("*.lgd")) if SPEC else [] + + +def test_the_specification_corpus_is_available() -> None: + """A run that promises its tools (CI) must have the corpus it validates.""" + if SPEC is None: + tool_missing("no LegalDown specification checkout (set LEGALDOWN_SPEC_DIR)") + assert SPEC_FILES + + +@pytest.mark.parametrize("style", ["default", "outline"]) +@pytest.mark.parametrize("path", SPEC_FILES, ids=lambda path: str(path.relative_to(SPEC)) if SPEC else str(path)) +def test_every_specification_document_validates(path: object, style: str) -> None: + source = path.read_text(encoding="utf-8") # type: ignore[attr-defined] + try: + data = make(source, style=style) + except DocumentError: + pytest.skip("not a document (invalid frontmatter)") + assert validation_errors(data) == [] diff --git a/tests/test_docx_wordcheck.py b/tests/test_docx_wordcheck.py new file mode 100644 index 0000000..445dcf6 --- /dev/null +++ b/tests/test_docx_wordcheck.py @@ -0,0 +1,108 @@ +"""The generator of the Word check document (tests/word_check.py, .github/PUBLISHING.md item 8 and 9): that it makes a +valid package whose rows carry what the planner assumes, from the planner's own code, and that LibreOffice (which does +not read ``\\t``) computes the rows' plain ``REF \\n`` fields as the planner caches them.""" +from __future__ import annotations + +import re +from pathlib import Path + +import pytest +from docx_support import libreoffice_export, parts, require_libreoffice, require_schemas, validation_errors +from word_check import ROWS, build, main + +from legaldown_render import render +from legaldown_render.resolve.docx_cover import suppressed +from legaldown_render.resolve.docx_numbering import Count, Lit +from legaldown_render.resolve.docx_plan import plan_docx + +REPOSITORY = Path(__file__).resolve().parents[1] +CHECKLIST = REPOSITORY / ".github" / "PUBLISHING.md" + + +def test_the_document_is_a_package_that_holds_every_row_and_the_assumption_beside_it() -> None: + contents = parts(build()) + assert {"[Content_Types].xml", "word/document.xml", "word/numbering.xml", "word/styles.xml"} <= set(contents) + text = re.sub(r"<[^>]+>", "", contents["word/document.xml"].decode("utf-8")) + for number, row in enumerate(ROWS, 1): + assert f"Row {number}, {row.title}" in text + # the planner's assumption is what its own code says, per setting of docx.reference_text_switch + assert "Row 6, an abbreviation with a period: No. %1: drawn 'No. 1'" in text + assert "documented: the switch is not used; guess: '1'" in text # No. 1: only the guess drops the period + assert "documented: 'I.1'; guess: 'I.1'" in text # Microsoft's example with a roman counter + assert "Part %1 of %2" in text and "documented: the switch is not used; guess: '11'" in text # words between counters + assert contents["word/document.xml"].count(b"REF t3 \\n \\t \\h") == 1 + assert suppressed([Lit("Section "), Count("a", "1"), Lit("."), Count("b", "1")]) is not None + + +def test_the_document_is_deterministic_and_the_command_writes_it(tmp_path) -> None: # noqa: ANN001 + assert build() == build() + target = tmp_path / "folder" / "check.docx" + assert main(["word_check.py", str(target)]) == 0 and target.read_bytes() == build() + assert main(["word_check.py"]) == 2 + + +@pytest.mark.ooxml_schema +def test_the_document_validates_against_the_schemas() -> None: + require_schemas() + assert validation_errors(build()) == [] + + +@pytest.mark.libreoffice +def test_libreoffice_computes_the_plain_fields_as_the_planner_caches_them() -> None: + require_libreoffice() + said = libreoffice_export({"check": build()})["check"].text + lines = [line for line in said.splitlines() if "REF \\n gives" in line] + assert len(lines) == len(ROWS) + 6 + 2 # the rows, the nested items of part B, the two headings of part C + for line in lines: + computed = re.search(r"REF \\n gives (.*?)(?:\. REF \\n \\t|$)", line) + assumed = re.search(r"\[planner: (.*?)\]", line) or re.search(r"The planner assumes REF \\n gives '(.*?)';", line) + assert computed and assumed, line + assert computed.group(1).rstrip(".") == assumed.group(1).rstrip("."), line + + +# -- The checklist's own snippet ----------------------------------------------------------------------------------- + + +def checklist_snippet() -> str: + """The Python the checklist gives for making the documents to open in Word.""" + if not CHECKLIST.is_file(): + pytest.skip("the checklist is not part of this distribution") + section = CHECKLIST.read_text(encoding="utf-8").split("## Checking the DOCX in Word", 1)[1] + found = re.search(r"```python\n(.*?)```", section, re.S) + assert found is not None + return found.group(1) + + +def test_the_snippet_of_the_checklist_makes_the_documents_it_names(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + """What the checklist tells a maintainer to run still runs, and makes every file its items name: the three styles and the + settings variants of both golden documents (the template has no attachments, so none of the variants about them), and + the document of the level nothing numbered.""" + monkeypatch.chdir(REPOSITORY) + exec(compile(checklist_snippet().replace('Path("word-check")', f"Path({str(tmp_path)!r})"), "PUBLISHING.md", "exec"), {}) + made = sorted(path.name for path in tmp_path.iterdir()) + styles = ("default", "continental", "outline") + variants = ("attachments-not-listed", "depth-1", "no-attachments-label", "separator-none", "separator-rule", + "scheme-none", "paragraphs-numbered") + about_attachments = {"attachments-not-listed", "no-attachments-label", "separator-none", "separator-rule"} + expected = [f"features.{name}.docx" for name in (*styles, *variants)] + expected += [f"template.{name}.docx" for name in (*styles, *(v for v in variants if v not in about_attachments))] + expected.append("uncounted.default.docx") + assert made == sorted(expected) + checklist = CHECKLIST.read_text(encoding="utf-8") + assert all(name in checklist for name in (*variants, "uncounted.default.docx")) + assert all(parts((tmp_path / name).read_bytes())["word/document.xml"] for name in made) + + +def test_the_document_of_the_level_nothing_numbered_is_the_one_the_checklist_describes( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + """Items ``e`` and ``f`` are numbered ``(A)`` and ``(B)`` as the checklist says Word is expected to number them, with the + information that LibreOffice (``(B)``, ``(C)``) does not (tests/test_docx_roundtrip.py shows what it does).""" + monkeypatch.chdir(REPOSITORY) + code = checklist_snippet().replace('Path("word-check")', f"Path({str(tmp_path)!r})") + namespace: dict = {} + exec(compile(code, "PUBLISHING.md", "exec"), namespace) + result = render(namespace["uncounted"], format="docx") + plan, diagnostics = plan_docx(result.tree) + items = [unit for unit in plan.units.values() if unit.label in ("(A)", "(B)")] + assert [unit.label for unit in items] == ["(A)", "(B)"] + assert any(d.rule == "render-docx-libreoffice-count" and d.level == "info" for d in diagnostics) diff --git a/tests/test_errors.py b/tests/test_errors.py index 1968390..f446398 100644 --- a/tests/test_errors.py +++ b/tests/test_errors.py @@ -34,9 +34,9 @@ def test_unknown_format_raises_options_error() -> None: with pytest.raises(OptionsError, match="unknown output format 'pdf'"): render("# x\n", RenderOptions(format="pdf")) with pytest.raises(OptionsError): - render("# x\n", format="docx") + render("# x\n", format="odt") with pytest.raises(RenderError): - render("# x\n", format="docx") + render("# x\n", format="pdf") def test_invalid_style_through_the_api_is_a_render_error() -> None: diff --git a/tests/test_numbering_structure.py b/tests/test_numbering_structure.py new file mode 100644 index 0000000..e465d6d --- /dev/null +++ b/tests/test_numbering_structure.py @@ -0,0 +1,935 @@ +"""The numbering structure the render tree carries (§13.1–§13.3, §15.4, §15.8): the formats in +effect, and for every section, list item and top-level paragraph its unit id, counter value, +parent and the alternative it shares its number with, so that a writer can build numbering +of its own without working out a rule of §13 again. Every value is checked against what the +resolver wrote as labels and designations: they must be derivable from the exposed +structure alone, under each built-in style, over the specification's corpus too.""" +from __future__ import annotations + +from collections.abc import Iterator +from dataclasses import dataclass +from pathlib import Path + +import pytest +from conftest import DOCUMENTS, FRONT, spec_root + +from legaldown_render import DocumentError, render +from legaldown_render.resolve.numbering import extend, fill, format_counter +from legaldown_render.style.model import LevelFormat +from legaldown_render.tree import ( + CrossRef, + DraftingNote, + FailureMarker, + List, + ListItem, + Paragraph, + Quote, + RenderTree, + Section, + Value, + iter_tree_inlines, +) + +NONE = {"numbering.scheme": "none"} +THREE_WAY = FRONT.replace("title: T", ( + "title: T\nquestions:\n q:\n type: choice\n choices:\n a: A\n b: B\n c: C")) +ALTERNATIVES = FRONT.replace("title: T", ( + "title: T\nquestions:\n forum:\n type: choice\n choices:\n courts: Courts\n arb: Arbitration")) +ATTACHED = FRONT.replace("title: T", "title: T\nattachments:\n - id: att\n title: Schedule\n file: att.lgd") + + +def tree_of(body: str, front: str = FRONT, **settings: object) -> RenderTree: + return render(front + body, format="text", **settings).tree + + +# -- A walk of the units, in document order --------------------------------------- + + +@dataclass(frozen=True) +class Found: + node: Section | ListItem | Paragraph + #: The section it is in; None in the preamble (and for sections themselves). + section: Section | None + #: The list an item is in. + owner: List | None = None + + @property + def kind(self) -> str: + return {Section: "section", ListItem: "item", Paragraph: "paragraph"}[type(self.node)] + + +def _blocks(blocks: tuple, section: Section | None) -> Iterator[Found]: + for block in blocks: + match block: + case Paragraph() if block.unit is not None: + yield Found(block, section) + case List(): + for item in block.items: + yield Found(item, section, block) + yield from _blocks(item.blocks, section) + case Quote(blocks=inner) | DraftingNote(blocks=inner): + yield from _blocks(inner, section) + + +def walk(tree: RenderTree) -> list[Found]: + """Every unit: the preamble's, then each section and what it holds.""" + found = list(_blocks(tree.preamble, None)) + for section in tree.sections: + found.append(Found(section, None)) + found.extend(_blocks(section.blocks, section)) + return found + + +def units(tree: RenderTree) -> dict[str, Found]: + return {found.node.unit: found for found in walk(tree)} + + +def sections(tree: RenderTree) -> list[Section]: + return list(tree.sections) + + +def items(tree: RenderTree) -> list[ListItem]: + return [found.node for found in walk(tree) if isinstance(found.node, ListItem)] + + +def paragraphs(tree: RenderTree) -> list[Paragraph]: + return [found.node for found in walk(tree) if isinstance(found.node, Paragraph)] + + +def lists(tree: RenderTree) -> list[List]: + seen: list[List] = [] + for found in walk(tree): + if found.owner is not None and found.owner not in seen: + seen.append(found.owner) + return seen + + +def refs(tree: RenderTree) -> list[CrossRef]: + return [inline for inline in iter_tree_inlines(tree) if isinstance(inline, CrossRef)] + + +def structure(tree: RenderTree) -> list[tuple]: + """What the tree says of each unit, for comparing two trees.""" + return [(f.kind, f.node.unit, f.node.number, f.node.parent_unit, f.node.shares_number_with) for f in walk(tree)] + + +# -- The formats in effect -------------------------------------------------------- + + +def test_the_default_formats() -> None: + formats = tree_of("# A\n").formats + assert formats.scheme == "decimal" + assert [(f.counter, f.label, f.ref) for f in formats.headings] == ( + [("decimal", "{path}.", "{n}")] + [("decimal", "{path}", ".{n}")] * 4) + assert [(f.counter, f.label, f.ref) for f in formats.enumerated] == [ + ("lower-alpha", "({n})", "({n})"), ("lower-roman", "({n})", "({n})"), ("upper-alpha", "({n})", "({n})")] + assert (formats.renumber.counter, formats.renumber.label, formats.renumber.ref) == ("decimal", "{n}.", "({n})") + assert formats.paragraphs is None + assert formats.reference == "{designation}" + + +def test_formats_follow_the_style_after_it_is_resolved() -> None: + outline = tree_of("# A\n", style="outline").formats + assert outline.scheme == "legal-outline" + assert [f.counter for f in outline.headings] == [ + "upper-roman", "upper-alpha", "decimal", "lower-alpha", "lower-roman"] + continental = tree_of("# A\n", style="continental").formats + # The style names the first heading format only; the rest are the scheme's. + assert [(f.label, f.ref) for f in continental.headings] == [("{n}.", "{n}")] + [("{path}", ".{n}")] * 4 + assert continental.enumerated[0].label == "{section}.{n}" + assert continental.paragraphs is not None + assert (continental.paragraphs.label, continental.paragraphs.ref) == ("({n})", "({n})") + mixed = tree_of("# A\n", overrides={"numbering.scheme": "mixed"}).formats + assert mixed.scheme == "mixed" + assert [f.label for f in mixed.headings] == ["{n}.", "({n})", "({n})", "({n})", "({n})"] + + +def test_a_style_can_replace_heading_levels_and_references() -> None: + formats = tree_of("# A\n", overrides={ + "numbering.levels": [{"counter": "upper-alpha", "label": "Article {n}", "ref": "{n}"}], + "references.format": "Section {designation}"}).formats + assert formats.headings[0].label == "Article {n}" + assert formats.headings[1].ref == ".{n}" + assert formats.reference == "Section {designation}" + + +def test_the_none_scheme_numbers_no_headings_but_still_formats_the_rest() -> None: + formats = tree_of("# A\n", overrides=NONE).formats + assert formats.scheme == "none" + assert formats.headings == () + assert formats.enumerated and formats.renumber is not None + + +def test_ordered_lists_may_be_enumerated_instead_of_renumbered() -> None: + formats = tree_of("# A\n", overrides={"enumeration.ordered": "enumerate"}).formats + assert formats.renumber is None + assert formats.enumerated + + +def test_formats_of_a_list_family_that_is_off_are_left_out() -> None: + formats = tree_of("# A\n", overrides={"enumeration.enabled": False}).formats + assert formats.enumerated == () + assert formats.renumber is not None + both = tree_of("# A\n", overrides={"enumeration.enabled": False, "enumeration.ordered": "enumerate"}).formats + assert both.enumerated # ordered lists still use the levels + + +# -- Headings ---------------------------------------------------------------------- + + +def test_sections_have_ids_in_document_order_and_their_own_counter() -> None: + tree = tree_of("# A\n\n## B\n\n## C\n\n### D\n\n# E\n\n## F\n") + assert [(s.unit, s.level, s.number, s.parent_unit, s.designation) for s in tree.sections] == [ + ("s1", 1, 1, None, "1"), ("s2", 2, 1, "s1", "1.1"), ("s3", 2, 2, "s1", "1.2"), + ("s4", 3, 1, "s3", "1.2.1"), ("s5", 1, 2, None, "2"), ("s6", 2, 1, "s5", "2.1")] + assert all(s.shares_number_with is None for s in tree.sections) + + +def test_outline_sections_count_in_their_own_styles() -> None: + tree = tree_of("# A\n\n## B\n\n### C\n\n#### D\n\n##### E\n\n## F\n", style="outline") + assert [(s.designation, s.number) for s in tree.sections] == [ + ("I", 1), ("I.A", 1), ("I.A.1", 1), ("I.A.1.a", 1), ("I.A.1.a(i)", 1), ("I.B", 2)] + assert [s.parent_unit for s in tree.sections] == [None, "s1", "s2", "s3", "s4", "s1"] + + +def test_continental_headings_are_decimal_with_a_dotted_label() -> None: + tree = tree_of("# A\n\n## B\n", style="continental") + assert [(s.label, s.number) for s in tree.sections] == [("1.", 1), ("1.1", 1)] + + +def test_a_skipped_level_counts_as_one_and_is_no_section() -> None: + # heading-skip is an Error, but the document still renders, numbered as the validator numbers it. + tree = tree_of("# A\n\n### B\n\n## C\n\n#### D\n\n### E\n") + assert [(s.unit, s.level, s.number, s.parent_unit, s.designation) for s in tree.sections] == [ + ("s1", 1, 1, None, "1"), + ("s2", 3, 1, "s1", "1.1.1"), # level 2 skipped: it is 1, and has no unit + ("s3", 2, 2, "s1", "1.2"), + ("s4", 4, 1, "s3", "1.2.1.1"), + ("s5", 3, 2, "s3", "1.2.2")] + + +def test_a_document_that_starts_below_level_one_is_numbered_from_the_top() -> None: + # As the validator numbers it: the first heading is the first section, whatever its marks. + tree = tree_of("## A\n\n## B\n\n### C\n") + assert [(s.level, s.number, s.parent_unit, s.designation) for s in tree.sections] == [ + (1, 1, None, "1"), (1, 2, None, "2"), (2, 1, "s2", "2.1")] + + +def test_sections_are_unnumbered_under_the_none_scheme_but_keep_their_structure() -> None: + tree = tree_of("# A\n\n## B\n\n# C\n", overrides=NONE) + assert [(s.unit, s.number, s.parent_unit, s.label, s.designation) for s in tree.sections] == [ + ("s1", None, None, None, "A"), ("s2", None, "s1", None, "B"), ("s3", None, None, None, "C")] + + +def test_alternative_sections_share_a_number_and_name_the_first() -> None: + body = ("# Disputes {#d when=forum:courts}\n\n## Sub one\n\n## Sub two\n\n" + "# Disputes {#d when=forum:arb}\n\n## Sub alt {#sub-alt}\n\n# Last\n") + tree = tree_of(body, ALTERNATIVES) + assert [(s.unit, s.designation, s.number, s.parent_unit, s.shares_number_with) for s in tree.sections] == [ + ("s1", "1", 1, None, None), + ("s2", "1.1", 1, "s1", None), + ("s3", "1.2", 2, "s1", None), + ("s4", "1", 1, None, "s1"), + ("s5", "1.1", 1, "s4", "s2"), # the number is shared, not the parent: it is the later alternative's child + ("s6", "2", 2, None, None)] + + +def test_three_alternative_sections_all_name_the_first() -> None: + body = ("# D {#d when=q:a}\n\n## s\n\n# D {#d when=q:b}\n\n## s2\n\n## s3\n\n" + "# D {#d when=q:c}\n\n## s4\n\n## s5\n\n## s6\n") + tree = tree_of(body, THREE_WAY) + assert [(s.unit, s.designation, s.parent_unit, s.shares_number_with) for s in tree.sections] == [ + ("s1", "1", None, None), ("s2", "1.1", "s1", None), + ("s3", "1", None, "s1"), ("s4", "1.1", "s3", "s2"), ("s5", "1.2", "s3", None), + ("s6", "1", None, "s1"), ("s7", "1.1", "s6", "s2"), ("s8", "1.2", "s6", "s5"), ("s9", "1.3", "s6", None)] + + +def test_a_later_alternative_may_have_more_subsections_and_the_extra_ones_are_first_holders() -> None: + body = ("# D {#d when=forum:courts}\n\n## One\n\n# D {#d when=forum:arb}\n\n## One alt\n\n## Extra\n\n" + "### Deep\n\n# After\n") + tree = tree_of(body, ALTERNATIVES) + assert [(s.designation, s.shares_number_with) for s in tree.sections] == [ + ("1", None), ("1.1", None), ("1", "s1"), ("1.1", "s2"), ("1.2", None), ("1.2.1", None), ("2", None)] + assert [s.parent_unit for s in tree.sections] == [None, "s1", None, "s3", "s3", "s5", None] + + +def test_a_heading_below_level_five_is_numbered_as_level_five() -> None: + tree = tree_of("# A\n\n## B\n\n### C\n\n#### D\n\n##### E\n\n###### F\n\n## G\n") + assert [(s.level, s.number, s.parent_unit, s.designation) for s in tree.sections] == [ + (1, 1, None, "1"), (2, 1, "s1", "1.1"), (3, 1, "s2", "1.1.1"), (4, 1, "s3", "1.1.1.1"), + (5, 1, "s4", "1.1.1.1.1"), (5, 2, "s4", "1.1.1.1.2"), (2, 2, "s1", "1.2")] + + +def test_alternative_sections_under_the_none_scheme_share_nothing() -> None: + body = "# Disputes {#d when=forum:courts}\n\n# Disputes {#d when=forum:arb}\n" + tree = tree_of(body, ALTERNATIVES, overrides=NONE) + assert [s.shares_number_with for s in tree.sections] == [None, None] + + +def test_unit_ids_do_not_depend_on_the_anchor() -> None: + body = "# A {#custom}\n\n# B\n\n# C {#custom-two}\n" + tree = tree_of(body) + assert [s.unit for s in tree.sections] == ["s1", "s2", "s3"] + assert [s.anchor for s in tree.sections] == ["custom", "b", "custom-two"] + + +# -- List items -------------------------------------------------------------------- + +NESTED = """# A + +- one {#one} +- two + - nested a {#na} + - nested b + - deep +- three + +1. first +1. second {#second} +""" + + +def test_nested_enumerated_items_carry_number_parent_and_family() -> None: + tree = tree_of(NESTED) + assert [(i.unit, i.label, i.number, i.parent_unit) for i in items(tree)] == [ + ("u1", "(a)", 1, "s1"), + ("u2", "(b)", 2, "s1"), + ("u3", "(i)", 1, "u2"), + ("u4", "(ii)", 2, "u2"), + ("u5", "(A)", 1, "u4"), + ("u6", "(c)", 3, "s1"), + ("u7", "1.", 1, "s1"), + ("u8", "2.", 2, "s1")] + formats = tree.formats + assert [(f.family, f.depth, f.enumerated) for f in lists(tree)] == [ + ("enumerated", 0, True), ("enumerated", 1, True), ("enumerated", 2, True), ("renumber", 0, True)] + assert [f.format for f in lists(tree)] == [*formats.enumerated, formats.renumber] + + +def test_families_mix_by_depth_and_the_enumerated_formats_cycle() -> None: + body = "# A\n\n1. r1\n - e1\n 1. r2\n - e2\n - e3\n - e4\n 1. r3\n" + tree = tree_of(body) + formats = tree.formats + assert [(f.family, f.depth) for f in lists(tree)] == [ + ("renumber", 0), ("enumerated", 1), ("renumber", 2), ("enumerated", 3), ("enumerated", 4), + ("enumerated", 5), ("renumber", 6)] + assert [f.format for f in lists(tree)] == [ + formats.renumber, formats.enumerated[1], formats.renumber, formats.enumerated[0], formats.enumerated[1], + formats.enumerated[2], formats.renumber] + assert [i.label for i in items(tree)] == ["1.", "(i)", "1.", "(a)", "(i)", "(A)", "1."] + assert [i.parent_unit for i in items(tree)] == ["s1", "u1", "u2", "u3", "u4", "u5", "u6"] + + +def test_a_list_nested_deeper_than_the_formats_goes_back_to_the_first() -> None: + body = "# A\n\n- 1\n - 2\n - 3\n - 4\n - 5\n" + tree = tree_of(body) + assert [(f.depth, f.format) for f in lists(tree)] == [ + (0, tree.formats.enumerated[0]), (1, tree.formats.enumerated[1]), (2, tree.formats.enumerated[2]), + (3, tree.formats.enumerated[0]), (4, tree.formats.enumerated[1])] + assert [i.label for i in items(tree)] == ["(a)", "(i)", "(A)", "(a)", "(i)"] + + +def test_an_ordered_list_nested_in_an_unordered_one_is_renumbered_from_its_parent() -> None: + tree = tree_of("# A\n\n- one\n 1. x\n 1. y\n") + assert [(f.family, f.depth) for f in lists(tree)] == [("enumerated", 0), ("renumber", 1)] + assert [(i.label, i.number, i.parent_unit) for i in items(tree)] == [ + ("(a)", 1, "s1"), ("1.", 1, "u1"), ("2.", 2, "u1")] + + +def test_ordered_lists_enumerated_by_depth_when_the_style_says_so() -> None: + tree = tree_of("# A\n\n1. x\n 1. y\n", overrides={"enumeration.ordered": "enumerate"}) + assert [(f.family, f.format) for f in lists(tree)] == [ + ("enumerated", tree.formats.enumerated[0]), ("enumerated", tree.formats.enumerated[1])] + assert [i.label for i in items(tree)] == ["(a)", "(i)"] + + +def test_bullets_are_units_without_numbers() -> None: + tree = tree_of("# A\n\n- x {#x}\n- y\n - z\n\n1. n\n", overrides={"enumeration.enabled": False}) + assert [(f.family, f.format, f.enumerated) for f in lists(tree)] == [ + ("bullet", None, False), ("bullet", None, False), ("renumber", tree.formats.renumber, True)] + assert [(i.unit, i.number, i.parent_unit, i.shares_number_with, i.label) for i in items(tree)] == [ + ("u1", None, "s1", None, None), ("u2", None, "s1", None, None), ("u3", None, "u2", None, None), + ("u4", 1, "s1", None, "1.")] + + +def test_a_list_without_enumeration_levels_is_bullets() -> None: + tree = tree_of("# A\n\n- x\n", overrides={"enumeration.levels": []}) + assert [(f.family, f.format) for f in lists(tree)] == [("bullet", None)] + assert items(tree)[0].number is None + + +def test_section_qualified_items_are_built_on_their_section() -> None: + tree = tree_of("# A\n\n- x\n - y\n- z\n\n# B\n\n- w\n", style="continental") + assert [(i.label, i.number, i.parent_unit) for i in items(tree)] == [ + ("1.1", 1, "s1"), ("a)", 1, "u1"), ("1.2", 2, "s1"), ("2.1", 1, "s2")] + top = lists(tree)[0] + assert top.format is not None + assert "{section}" in top.format.label and top.format.ref == ".{n}" + assert "{section}" not in tree.formats.enumerated[1].label + + +def test_items_of_a_list_in_a_quote_are_built_on_the_section() -> None: + tree = tree_of("# A\n\n> - q1\n> - q2\n> - q3\n\n- real\n") + assert [(i.unit, i.label, i.parent_unit) for i in items(tree)] == [ + ("u1", "(a)", "s1"), ("u2", "(b)", "s1"), ("u3", "(i)", "u2"), ("u4", "(a)", "s1")] + + +def test_the_preamble_has_items_without_a_parent() -> None: + tree = tree_of("- p1\n- p2\n - p3\n\n# A\n\n- x\n") + assert [(i.unit, i.label, i.parent_unit) for i in items(tree)] == [ + ("u1", "(a)", None), ("u2", "(b)", None), ("u3", "(i)", "u2"), ("u4", "(a)", "s1")] + + +def test_alternative_items_share_a_number() -> None: + body = ("# A\n\n- first\n- by courts {#i when=forum:courts}\n - inner\n" + "- by arbitration {#i when=forum:arb}\n - inner alt\n- last\n") + tree = tree_of(body, ALTERNATIVES) + assert [(i.unit, i.number, i.parent_unit, i.shares_number_with) for i in items(tree)] == [ + ("u1", 1, "s1", None), + ("u2", 2, "s1", None), + ("u3", 1, "u2", None), + ("u4", 2, "s1", "u2"), + ("u5", 1, "u4", None), # the alternative's own children are its own: they count from 1 + ("u6", 3, "s1", None)] + + +def test_three_alternatives_all_name_the_first() -> None: + tree = tree_of("# A\n\n- x\n- one {#i when=q:a}\n- two {#i when=q:b}\n- three {#i when=q:c}\n- y\n", THREE_WAY) + assert [(i.number, i.shares_number_with) for i in items(tree)] == [ + (1, None), (2, None), (2, "u2"), (2, "u2"), (3, None)] + + +def test_units_that_can_appear_together_do_not_share_a_number() -> None: + body = "# A\n\n- one {#i when=forum:courts}\n- two {#i when=forum:courts}\n" + assert [(i.number, i.shares_number_with) for i in items(tree_of(body, ALTERNATIVES))] == [(1, None), (2, None)] + + +def test_only_adjacent_alternatives_share_a_number_and_a_list_never_shares_with_the_one_before() -> None: + body = "# A\n\n- a {#i when=forum:courts}\n\ntext\n\n- b {#i when=forum:arb}\n" + assert [(i.number, i.shares_number_with) for i in items(tree_of(body, ALTERNATIVES))] == [(1, None), (1, None)] + + +def test_sibling_lists_in_one_item_go_on_counting() -> None: + """Lists nested directly in one item share a counter per way of writing a designation, so no + designation repeats: a list's first number is items[0].number (List.start), not always 1.""" + tree = tree_of("# A\n\n- x\n - a\n - b\n\n text\n\n - c\n - d\n") + assert [(i.label, i.number, i.parent_unit) for i in items(tree)[1:]] == [ + ("(i)", 1, "u1"), ("(ii)", 2, "u1"), ("(iii)", 3, "u1"), ("(iv)", 4, "u1")] + assert [(f.depth, f.start) for f in lists(tree)] == [(0, 1), (1, 1), (1, 3)] + + +def test_a_sibling_list_of_another_family_with_the_same_designations_goes_on_too() -> None: + # A renumbered list is "(n)" in a reference, so is an enumerated one whose ref is "({n})" in decimal. + levels = [{"counter": "decimal", "label": "{n}.", "ref": "({n})"}] * 2 + tree = tree_of("# A\n\n- x\n - a\n\n text\n\n 1. p\n 1. q\n", overrides={"enumeration.levels": levels}) + assert [(f.family, f.start) for f in lists(tree)] == [("enumerated", 1), ("enumerated", 1), ("renumber", 2)] + assert [i.label for i in items(tree)[1:]] == ["1.", "2.", "3."] + # With other designations the lists are independent. + other = tree_of("# A\n\n- x\n - a\n\n text\n\n 1. p\n 1. q\n") + assert [(f.family, f.start) for f in lists(other)] == [("enumerated", 1), ("enumerated", 1), ("renumber", 1)] + + +def test_top_level_lists_and_lists_in_quotes_start_at_one() -> None: + tree = tree_of("# A\n\n- x\n\ntext\n\n- y\n\n> - q\n\n> text\n\n> - r\n") + assert [f.start for f in lists(tree)] == [1, 1, 1, 1] + + +def test_alternatives_never_join_across_sibling_lists() -> None: + body = "# A\n\n- x\n - a {#i when=forum:courts}\n\n text\n\n - b {#i when=forum:arb}\n" + tree = tree_of(body, ALTERNATIVES) + assert [(i.number, i.shares_number_with) for i in items(tree)[1:]] == [(1, None), (2, None)] + # Within one list they do. + body = "# A\n\n- x\n - a {#i when=forum:courts}\n - b {#i when=forum:arb}\n" + assert [(i.number, i.shares_number_with) for i in items(tree_of(body, ALTERNATIVES))[1:]] == [(1, None), (1, "u2")] + + +def test_a_list_in_a_quote_or_drafting_note_inside_an_item_is_built_on_the_section() -> None: + body = ("# A\n\n- x\n > [!DRAFTING]\n > - n1\n > - n2\n\n > - q1\n\n - nested\n\n" + "> [!DRAFTING]\n> - top\n") + tree = tree_of(body) + assert [(i.unit, i.label, i.parent_unit) for i in items(tree)] == [ + ("u1", "(a)", "s1"), + ("u2", "(i)", "s1"), ("u3", "(ii)", "s1"), # depth 1 gives the format; the parent is the section + ("u4", "(i)", "s1"), + ("u5", "(i)", "u1"), # a list nested in the item itself is built on it + ("u6", "(a)", "s1")] + assert [(f.depth, f.start, f.items[0].parent_unit) for f in lists(tree)] == [ + (0, 1, "s1"), (1, 1, "s1"), (1, 1, "s1"), (1, 1, "u1"), (0, 1, "s1")] + assert [f.items[0].unit for f in lists(tree)] == ["u1", "u2", "u4", "u5", "u6"] # a list has no id: its first item + + +# -- Paragraphs -------------------------------------------------------------------- + + +def test_numbered_paragraphs_carry_number_and_section() -> None: + body = "Preamble.\n\n# A\n\nOne.\n\nTwo. {#two}\n\n- item\n\n Continued.\n\n# B\n\nThree.\n" + tree = tree_of(body, overrides={"paragraphs.numbered": True}) + assert [(p.unit, p.label, p.number, p.parent_unit) for p in paragraphs(tree)] == [ + ("u1", "1.1", 1, "s1"), ("u2", "1.2", 2, "s1"), ("u4", "2.1", 1, "s2")] + # The preamble's paragraph and the item's continuation are not units. + assert tree.preamble[0].unit is None and tree.preamble[0].number is None + inner = items(tree)[0].blocks[1] + assert isinstance(inner, Paragraph) + assert (inner.unit, inner.number, inner.parent_unit) == (None, None, None) + assert tree.formats.paragraphs is not None + assert tree.formats.paragraphs.label == "{section}.{n}" + + +def test_paragraphs_are_units_without_numbers_when_the_style_does_not_number_them() -> None: + tree = tree_of("# A\n\nOne. {#one}\n\nTwo.\n") + assert [(p.unit, p.number, p.parent_unit, p.label) for p in paragraphs(tree)] == [ + ("u1", None, "s1", None), ("u2", None, "s1", None)] + assert tree.formats.paragraphs is None + + +def test_alternative_paragraphs_share_a_number() -> None: + body = "# A\n\nIntro.\n\nCourts. {#p when=forum:courts}\n\nArbitration. {#p when=forum:arb}\n\nAfter.\n" + tree = tree_of(body, ALTERNATIVES, overrides={"paragraphs.numbered": True}) + assert [(p.unit, p.label, p.number, p.shares_number_with) for p in paragraphs(tree)] == [ + ("u1", "1.1", 1, None), ("u2", "1.2", 2, None), ("u3", "1.2", 2, "u2"), ("u4", "1.3", 3, None)] + + +def test_alternative_paragraphs_need_not_be_adjacent_blocks() -> None: + """Only the top-level paragraphs are counted together: a list or a table between two of them does not + keep them from sharing a number, and a paragraph does.""" + body = ("# A\n\nCourts. {#p when=forum:courts}\n\n- between\n\n| a |\n|---|\n| b |\n\n" + "Arbitration. {#p when=forum:arb}\n\nMid.\n\nCourts again. {#q when=forum:courts}\n\n" + "Mid.\n\nArbitration again. {#q when=forum:arb}\n") + tree = tree_of(body, ALTERNATIVES, overrides={"paragraphs.numbered": True}) + assert [(p.unit, p.number, p.shares_number_with) for p in paragraphs(tree)] == [ + ("u1", 1, None), ("u3", 1, "u1"), ("u4", 2, None), ("u5", 3, None), ("u6", 4, None), ("u7", 5, None)] + + +def test_three_alternative_paragraphs_all_name_the_first() -> None: + tree = tree_of("# A\n\nx {#p when=q:a}\n\n- l\n\ny {#p when=q:b}\n\nz {#p when=q:c}\n\nlast\n", + THREE_WAY, overrides={"paragraphs.numbered": True}) + assert [(p.number, p.shares_number_with) for p in paragraphs(tree)] == [ + (1, None), (1, "u1"), (1, "u1"), (2, None)] + + +def test_alternative_paragraphs_share_nothing_where_none_are_numbered() -> None: + body = "# A\n\nCourts. {#p when=forum:courts}\n\nArbitration. {#p when=forum:arb}\n" + assert [p.shares_number_with for p in paragraphs(tree_of(body, ALTERNATIVES))] == [None, None] + + +def test_paragraph_numbers_under_the_none_scheme() -> None: + tree = tree_of("# Heading\n\nOne.\n\nTwo.\n", overrides={**NONE, "paragraphs.numbered": True}) + assert [(p.label, p.number, p.parent_unit) for p in paragraphs(tree)] == [("Heading.1", 1, "s1"), ("Heading.2", 2, "s1")] + + +def test_paragraph_numbering_style_continental() -> None: + tree = tree_of("# A\n\nOne.\n\nTwo.\n", style="continental") + assert [(p.label, p.number) for p in paragraphs(tree)] == [("(1)", 1), ("(2)", 2)] + + +# -- Cross-references -------------------------------------------------------------- + +REFERENCES = """# Services {#svc} + +Intro. {#intro} + +- alpha {#alpha} + - beta {#beta} + +# Other {#other} + +See {{ref: svc}}, {{ref: alpha}}, {{ref: beta}}, {{ref: intro}}, {{ref: other}}. +""" + + +def test_a_reference_names_its_target_and_its_section() -> None: + tree = tree_of(REFERENCES, overrides={"paragraphs.numbered": True}) + assert [(r.text, r.target, r.target_unit, r.section_unit) for r in refs(tree)] == [ + ("1", "svc", "s1", "s1"), + ("1(a)", "alpha", "u2", "s1"), + ("1(a)(i)", "beta", "u3", "s1"), + ("1.1", "intro", "u1", "s1"), + ("2", "other", "s2", "s2")] + found = units(tree) + for ref in refs(tree): + assert found[ref.target_unit].node.anchor == ref.target + + +def test_a_reference_to_a_unit_the_style_does_not_number_names_the_unit_and_reads_as_its_section() -> None: + result = render(FRONT + REFERENCES, format="text") + assert "ref-not-enumerated" in {d.rule for d in result.diagnostics} + found = units(result.tree) + by_target = {r.target: r for r in refs(result.tree)} + intro = by_target["intro"] + assert (intro.text, intro.target_unit, intro.section_unit) == ("1", "u1", "s1") + assert found["u1"].node.number is None + bullets = render(FRONT + REFERENCES, format="text", overrides={"enumeration.enabled": False}) + beta = {r.target: r for r in refs(bullets.tree)}["beta"] + assert (beta.text, beta.target_unit, beta.section_unit) == ("1", "u3", "s1") + + +def test_a_reference_to_an_anchor_shared_by_alternatives_names_the_first() -> None: + body = ("# A\n\n- plain\n- by courts {#i when=forum:courts}\n- by arbitration {#i when=forum:arb}\n\n" + "Courts. {#p when=forum:courts}\n\nArb. {#p when=forum:arb}\n\n" + "# D {#d when=forum:courts}\n\n# D {#d when=forum:arb}\n\n" + "See {{ref: i}}, {{ref: p}}, {{ref: d}}.\n") + tree = tree_of(body, ALTERNATIVES, overrides={"paragraphs.numbered": True}) + assert [(r.text, r.target, r.target_unit, r.section_unit) for r in refs(tree)] == [ + ("1(b)", "i", "u2", "s1"), ("1.1", "p", "u4", "s1"), ("2", "d", "s2", "s2")] + shared = [f.node for f in walk(tree) if f.node.shares_number_with] + assert [n.unit for n in shared] == ["u3", "u5", "s3"] + assert {n.shares_number_with for n in shared} == {"u2", "u4", "s2"} + + +def test_a_reference_in_an_alternative_to_its_own_shared_number() -> None: + body = "# A\n\n- x {#i when=forum:courts}\n- y {#i when=forum:arb}\n\nSee {{ref: i}}.\n" + tree = tree_of(body, ALTERNATIVES) + assert [(r.target_unit, r.section_unit) for r in refs(tree)] == [("u1", "s1")] + + +def test_references_under_the_none_scheme_name_the_same_units() -> None: + tree = tree_of(REFERENCES, overrides={**NONE, "paragraphs.numbered": True}) + assert [(r.text, r.target_unit, r.section_unit) for r in refs(tree)] == [ + ("Services", "s1", "s1"), + ("Services (a)", "u2", "s1"), + ("Services (a) (i)", "u3", "s1"), + ("Services 1", "u1", "s1"), + ("Other", "s2", "s2")] + assert all(tree.sections[0].number is None for _ in refs(tree)) + + +def test_references_that_do_not_resolve_are_failure_markers_not_cross_references() -> None: + body = "# A\n\nSee {{ref: missing}}, {{ref: att}}, {{ref: }}, {{attach: att}}.\n" + tree = tree_of(body, ATTACHED) + assert refs(tree) == [] + markers = [i.text for i in iter_tree_inlines(tree) if isinstance(i, FailureMarker)] + assert markers[:2] == ["[BROKEN REF: missing]", "[BROKEN REF: att]"] + + +def test_an_attachment_is_reached_by_attach_whose_link_is_the_placeholders_anchor() -> None: + tree = tree_of("# A\n\nSee {{attach: att}}.\n", ATTACHED) + value = next(i for i in iter_tree_inlines(tree) if isinstance(i, Value) and i.kind == "attach") + assert value.href == "#" + tree.attachments[0].anchor + assert refs(tree) == [] + + +def test_a_copy_that_no_longer_links_names_no_unit() -> None: + body = "# See {{ref: b}} {#a}\n\n# Target {#b}\n" + tree = tree_of(body, overrides={"contents.enabled": True}) + in_heading = [i for i in tree.sections[0].title if isinstance(i, CrossRef)] + assert [(r.target_unit, r.section_unit) for r in in_heading] == [("s2", "s2")] + contents = [i for entry in tree.contents for i in entry.title if isinstance(i, CrossRef)] + assert [(r.text, r.target, r.target_unit, r.section_unit) for r in contents] == [("2", "", None, None)] + + +# -- Stability --------------------------------------------------------------------- + + +def test_the_same_document_gets_the_same_structure_every_time() -> None: + first = tree_of(NESTED + "\n# B\n\nText.\n", style="continental") + second = tree_of(NESTED + "\n# B\n\nText.\n", style="continental") + assert first == second + assert structure(first) == structure(second) + + +def test_unit_ids_are_unique_and_follow_document_order() -> None: + tree = tree_of("- pre\n\n" + NESTED + "\n## B\n\nText.\n\n> - quoted\n\n# C\n\n- x\n", + overrides={"paragraphs.numbered": True}) + ids = [f.node.unit for f in walk(tree)] + assert len(ids) == len(set(ids)) + numeric = [int(unit[1:]) for unit in ids if unit.startswith("u")] + assert numeric == sorted(numeric) == list(range(1, len(numeric) + 1)) + assert [u for u in ids if u.startswith("s")] == ["s1", "s2", "s3"] + + +def test_unit_ids_are_the_same_when_designations_are_worked_out_again() -> None: + """Under the none scheme a heading that holds a reference is designated by it, and the sections, + items and paragraphs are then numbered a second time (docs/architecture.md): the ids, numbers + and parents of that second pass are those a document without the reference gets.""" + items_and_paragraphs = "- x {#it}\n - y\n\nOne. {#para}\n\n- z\n" + with_reference = (f"# About {{{{ref: ov}}}} {{#h}}\n\n{items_and_paragraphs}\n# Overview {{#ov}}\n\n" + f"{items_and_paragraphs}\nSee {{{{ref: it}}}}, {{{{ref: para}}}}, {{{{ref: h}}}}.\n") + without = with_reference.replace("About {{ref: ov}}", "About Overview") + overrides = {**NONE, "paragraphs.numbered": True} + again = tree_of(with_reference, overrides=overrides) + once = tree_of(without, overrides=overrides) + assert again.sections[0].designation == "About Overview" # it was designated a second time + assert structure(again) == structure(once) + found = units(again) + assert all(r.target_unit in found and r.section_unit in found for r in refs(again)) + # The first "it" and "para" are the section About's (u1 u2, u3 and u4 are its item, nested item, + # paragraph and item), and the last section's are not the targets: the identifiers are the first's. + assert {r.target: r.target_unit for r in refs(again)} == {"ov": "s2", "it": "u1", "para": "u3", "h": "s1"} + + +# -- The structure is enough to write every label and reference again -------------- + + +class Recomputed: + """Labels and designations worked out again from what the tree exposes: the unit ids, numbers, + parents, list formats and ``RenderTree.formats``.""" + + def __init__(self, tree: RenderTree) -> None: + self.tree = tree + self.found = units(tree) + self.none = tree.formats.scheme == "none" + self.memo: dict[str, str] = {} + + def parts(self, unit: str) -> list[int]: + """The number of a section, part by part: a skipped level counts as 1.""" + chain: list[Section] = [] + current: str | None = unit + while current is not None: + section = self.found[current].node + assert isinstance(section, Section) + chain.append(section) + current = section.parent_unit + out: list[int] = [] + previous = 0 + for section in reversed(chain): + assert section.number is not None + out += [1] * (section.level - previous - 1) + [section.number] + previous = section.level + return out + + def section_designation(self, section: Section) -> str: + if self.none: + return section.designation or "" # the heading's text: the tree's own + designation = "" + levels = self.tree.formats.headings + for position, number in enumerate(self.parts(section.unit)): + fmt = levels[min(position, len(levels) - 1)] + designation = extend(designation, fill(fmt.ref, n=format_counter(number, fmt.counter)), textual=False) + return designation + + def section_label(self, section: Section) -> str | None: + if self.none: + return None + parts = self.parts(section.unit) + levels = self.tree.formats.headings + fmt = levels[min(len(parts), len(levels)) - 1] + return fill(fmt.label, n=format_counter(parts[-1], fmt.counter), path=self.section_designation(section)) + + def designation(self, unit: str | None) -> str: + """What a reference to *unit* builds on: its designation, which an unnumbered unit has as its parent's.""" + if unit is None: + return "" + if unit not in self.memo: + self.memo[unit] = self._designation(unit) + return self.memo[unit] + + def _format(self, found: Found): + if isinstance(found.node, Paragraph): + return self.tree.formats.paragraphs + assert found.owner is not None + return found.owner.format + + def _designation(self, unit: str) -> str: + found = self.found[unit] + node = found.node + if isinstance(node, Section): + return self.section_designation(node) + if node.number is None: + return self.designation(node.parent_unit) + fmt = self._format(found) + base = found.section.designation if found.section else "" + return extend(self.designation(node.parent_unit), + fill(fmt.ref, n=format_counter(node.number, fmt.counter), section=base or ""), + textual=self.none) + + def label(self, unit: str) -> str | None: + found = self.found[unit] + node = found.node + if isinstance(node, Section): + return self.section_label(node) + if node.number is None: + return None + fmt = self._format(found) + base = found.section.designation if found.section else "" + return fill(fmt.label, n=format_counter(node.number, fmt.counter), section=base or "", + path=self.designation(unit)) + + def reference(self, ref: CrossRef) -> str: + assert ref.target_unit is not None and ref.section_unit is not None + target = self.found[ref.target_unit].node + shown = self.designation(ref.target_unit if target.number is not None else ref.section_unit) + return self.tree.formats.reference.replace("{designation}", shown) + + +def check(tree: RenderTree) -> None: + """Every invariant of the tree's numbering structure.""" + order = walk(tree) + ids = [f.node.unit for f in order] + assert None not in ids + assert len(ids) == len(set(ids)), "unit ids are unique" + position = {unit: index for index, unit in enumerate(ids)} + recompute = Recomputed(tree) + formats = tree.formats + for found in order: + node = found.node + unit = node.unit + assert unit is not None + if node.parent_unit is not None: + assert node.parent_unit in position and position[node.parent_unit] < position[unit] + if node.shares_number_with is not None: + first = recompute.found[node.shares_number_with].node + assert position[node.shares_number_with] < position[unit] + assert node.number is not None and first.number == node.number and first.shares_number_with is None + # An alternative stands where its first does: same kind, same parent, same label. + assert type(first) is type(node) and first.label == node.label, unit + if not isinstance(node, Section): + assert first.parent_unit == node.parent_unit, unit + if isinstance(node, Section): + assert node.label == recompute.label(unit), unit + assert node.designation == recompute.designation(unit), unit + assert (node.number is None) == (formats.scheme == "none") + assert node.parent_unit is None or recompute.found[node.parent_unit].node.level < node.level + continue + assert node.label == recompute.label(unit), unit + if isinstance(node, ListItem): + assert found.owner is not None + assert (node.number is None) == (found.owner.format is None) == (not found.owner.enumerated) + else: + assert (node.number is None) == (formats.paragraphs is None) + assert node.parent_unit == (found.section.unit if found.section else None) + for block in lists(tree): + expected = (formats.renumber if block.family == "renumber" + else formats.enumerated[block.depth % len(formats.enumerated)] if block.family == "enumerated" + else None) + assert block.format == expected + assert (block.family == "bullet") == (not block.enumerated) + for ref in refs(tree): + if ref.target_unit is None: + assert ref.section_unit is None and ref.target == "" # a copy that no longer links + continue + assert ref.target_unit in position and ref.section_unit in position + assert recompute.found[ref.target_unit].node.anchor == ref.target or ref.target == "" + section = ref.section_unit + target_found = recompute.found[ref.target_unit] + assert section == (target_found.node.unit if isinstance(target_found.node, Section) + else target_found.section.unit) + assert ref.text == recompute.reference(ref), (ref.text, ref.target) + check_counting(tree, recompute.found) + + +def _counts(group: list, found: dict[str, Found], first: int, what: str) -> None: + """Consecutive numbered units of *group* count up by one, or share the number of an earlier one.""" + previous = first - 1 + for node in group: + if node.number is None: + continue + shared = node.shares_number_with + assert node.number == previous + 1 or (shared is not None and node.number == found[shared].node.number), ( + what, node.unit, node.number, previous) + previous = node.number + + +def check_counting(tree: RenderTree, found: dict[str, Found]) -> None: + """Numbers start where they should and go on without a gap: sections of one parent, paragraphs of one + section, and the items of a list, which goes on from an earlier sibling list in the same item when + the two write their designations alike.""" + by_parent: dict[str | None, list] = {} + for section in tree.sections: + by_parent.setdefault(section.parent_unit, []).append(section) + for parent, group in by_parent.items(): + _counts(group, found, 1, f"sections under {parent}") + in_section: dict[str, list] = {} + for paragraph in paragraphs(tree): + in_section.setdefault(paragraph.parent_unit, []).append(paragraph) + for parent, group in in_section.items(): + _counts(group, found, 1, f"paragraphs of {parent}") + last: dict[tuple, int] = {} # (item, counter, ref) -> the last number of the sibling lists so far + for block in lists(tree): + if block.format is None: + continue + parent = block.items[0].parent_unit + under_item = parent is not None and isinstance(found[parent].node, ListItem) + key = (parent, block.format.counter, block.format.ref) + first = last.get(key, 0) + 1 if under_item else 1 + _counts(block.items, found, first, f"items of the list at {block.items[0].unit}") + assert block.start == block.items[0].number + last[key] = block.items[-1].number + + +def path_labelled(style: str, overrides: dict) -> dict: + """*overrides* with the label of every list level and of paragraphs written as ``{path}``, so that + each label is the unit's whole designation and comparing labels compares designations.""" + formats = tree_of("# A\n", style=style, overrides=overrides).formats + out = dict(overrides) + if formats.enumerated: + out["enumeration.levels"] = [{"counter": f.counter, "label": "{path}", "ref": f.ref} + for f in formats.enumerated] + paragraph = formats.paragraphs or LevelFormat("decimal", "{section}.{n}", ".{n}") + out["paragraphs.numbered"] = True + out["paragraphs.format"] = {"counter": paragraph.counter, "label": "{path}", "ref": paragraph.ref} + return out + + +def variants(style: str, overrides: dict) -> list[dict]: + """The settings to check a style under: as they are, and with designations as labels.""" + return [overrides, path_labelled(style, overrides)] + + +STYLES = [("default", {}), ("outline", {}), ("continental", {}), + ("default", NONE), ("continental", {**NONE, "paragraphs.numbered": True}), + ("default", {"paragraphs.numbered": True, "enumeration.ordered": "enumerate"}), + ("default", {"enumeration.enabled": False}), + ("outline", {"references.format": "Section {designation}"})] +STYLE_IDS = ["default", "outline", "continental", "none", "continental-none", "enumerate", "bullets", "reference-format"] + + +@pytest.mark.parametrize(("style", "overrides"), STYLES, ids=STYLE_IDS) +def test_the_corpus_of_the_tests_is_numbered_as_its_structure_says(style: str, overrides: dict) -> None: + for name in ("features.lgd", "template.lgd"): + for variant in variants(style, overrides): + check(render((DOCUMENTS / name).read_text(encoding="utf-8"), format="text", style=style, + overrides=variant).tree) + + +@pytest.mark.parametrize(("body", "overrides"), [ + (NESTED, {}), (REFERENCES, {}), (REFERENCES, {"paragraphs.numbered": True}), (REFERENCES, NONE), + ("# A\n\n### B\n\n## C\n\n#### D\n\n### E\n\n- x {#x}\n\nSee {{ref: x}}, {{ref: e}}.\n", {}), + ("# A\n\n### B\n\n## C\n\n#### D\n\n### E\n\n- x {#x}\n\nSee {{ref: x}}.\n", {"numbering.scheme": "mixed"}), + ("# A\n\n> - q\n\n- x\n > - y\n", {}), + ("- p\n\n# A\n", {}), + ("# A\n\n- x\n - a\n - b\n\n text\n\n 1. c\n 1. d\n\n - e\n\n> [!DRAFTING]\n> - n\n", {}), + ("# A\n\n1. r\n - e1\n 1. r2\n - e2\n - e3\n - e4\n", {}), +], ids=["nested", "refs", "refs-paragraphs", "refs-none", "skips", "skips-mixed", "quotes", "preamble", + "sibling-lists", "mixed-families"]) +@pytest.mark.parametrize("style", ["default", "outline", "continental"]) +def test_small_documents_are_numbered_as_their_structure_says(body: str, overrides: dict, style: str) -> None: + for variant in variants(style, overrides): + check(tree_of(body, style=style, overrides=variant)) + + +def test_alternatives_are_numbered_as_their_structure_says() -> None: + body = ("# Disputes {#d when=forum:courts}\n\n## Sub one\n\n- a {#i when=forum:courts}\n- b {#i when=forum:arb}\n\n" + "# Disputes {#d when=forum:arb}\n\n## Sub alt {#sub-alt}\n\nOne. {#p when=forum:arb}\n\n" + "Two. {#p when=forum:courts}\n\nSee {{ref: d}}, {{ref: i}}, {{ref: p}}, {{ref: sub-alt}}.\n") + siblings = ("# A\n\n- x\n - a {#i when=forum:courts}\n\n text\n\n - b {#i when=forum:arb}\n - c {#i when=forum:courts}\n" + " - d {#i when=forum:arb}\n") + for style, overrides in STYLES: + for variant in variants(style, {**overrides, "paragraphs.numbered": True}): + check(tree_of(body, ALTERNATIVES, style=style, overrides=variant)) + check(tree_of(siblings, ALTERNATIVES, style=style, overrides=variant)) + + +# -- The specification's corpus ---------------------------------------------------- + +SPEC = spec_root() +CORPUS: list[Path] = sorted([*(SPEC / "examples").rglob("*.lgd"), *(SPEC / "fixtures").rglob("*.lgd")]) if SPEC else [] + + +@pytest.mark.conformance +@pytest.mark.skipif(SPEC is None, reason="LegalDown specification checkout not found") +@pytest.mark.parametrize(("style", "overrides"), STYLES[:5], ids=STYLE_IDS[:5]) +def test_the_specifications_corpus_is_numbered_as_its_structure_says(style: str, overrides: dict) -> None: + assert CORPUS, "no documents found in the specification checkout" + rendered = 0 + for path in CORPUS: + for variant in variants(style, overrides): + try: + result = render(path.read_text(encoding="utf-8"), format="text", style=style, overrides=variant) + except DocumentError: + continue + try: + check(result.tree) + except AssertionError as error: # name the document + raise AssertionError(f"{path.relative_to(SPEC)}: {error}") from error + rendered += 1 + assert rendered > 200 diff --git a/tests/test_untrusted.py b/tests/test_untrusted.py index 9ae0ce7..c7d5f30 100644 --- a/tests/test_untrusted.py +++ b/tests/test_untrusted.py @@ -387,6 +387,11 @@ def test_an_override_that_cannot_hold_keys_is_not_quoted() -> None: assert SECRET in str(caught.value) +def test_an_untrusted_default_style_is_named_like_any_other_untrusted_style() -> None: + assert refused("default", {"page.margin": "9" * 400 + "mm"}).source == "'untrusted style'" + assert refused(None, {"page.margin": "9" * 400 + "mm"}).source == "'untrusted style'" + + def test_the_style_object_name_is_not_quoted() -> None: error = refused(Style(name=SECRET, typography=Typography(font_size=SECRET))) assert SECRET not in str(error) diff --git a/tests/word_check.py b/tests/word_check.py new file mode 100644 index 0000000..6a492da --- /dev/null +++ b/tests/word_check.py @@ -0,0 +1,222 @@ +#!/usr/bin/env python3 +"""The document for the Word checks of what the DOCX planner assumes (.github/PUBLISHING.md, "Checking the DOCX in +Word", items 8 and 9): what Word's ``REF \\n \\t`` returns for the label forms the plan relies on, whether a level text +of the heading list that names the levels around a never-used one (``%1.%2(%6)`` with levels 3 to 5 unused) draws and +restarts as the plan models it, and what a skipped heading level draws. + +Run it with the project's environment, from the repository root:: + + python tests/word_check.py word-check/numbering-assumptions.docx + +Open the file in Word, update every field (Ctrl+A, F9), and compare each line with its last column: the number Word +draws, ``REF \\n`` and ``REF \\n \\t`` as Word computes them, against what the planner assumes under each setting of +``docx.reference_text_switch``. Every field holds, until it is updated, the planner's own value. Anything that differs +is a finding for the planner (report it, and set ``docx.reference_text_switch`` to what your Word does). + +The planner's side comes from the planner's own code (the simulator of Word's counters, ``suppressed``), never from a +table written here, so a change to the model shows in the file. +""" +from __future__ import annotations + +import sys +import xml.etree.ElementTree as ET +from collections.abc import Sequence +from dataclasses import dataclass +from pathlib import Path + +from legaldown_render.resolve.docx_cover import suppressed +from legaldown_render.resolve.docx_numbering import Count, Lit, WordNumbering +from legaldown_render.resolve.docx_plan import AbstractNum, LevelDef, Num +from legaldown_render.style import load_style +from legaldown_render.style.docx import docx_settings +from legaldown_render.writers.docx import package as pkg +from legaldown_render.writers.docx.numbering import abstract_num, num +from legaldown_render.writers.docx.ooxml import Bookmarks, complex_field, el, p, serialize, text_run, val +from legaldown_render.writers.docx.properties import build_app, build_core +from legaldown_render.writers.docx.settings import build_numbering, build_settings +from legaldown_render.writers.docx.styles import build_styles + +_STEP = 567 + + +def _level(ilvl: int, num_fmt: str, text: str) -> LevelDef: + """A level of a list of the check document. None is linked to a paragraph style: the document's own titles are + ``Heading`` paragraphs that Word must not number, and the numbered lines carry ``w:numPr`` themselves.""" + return LevelDef(ilvl, num_fmt, text, left=_STEP * (ilvl + 1), hanging=_STEP) + + +def _neutral(ilvl: int) -> LevelDef: + return _level(ilvl, "decimal", f"%{ilvl + 1}") + + +@dataclass(frozen=True) +class Row: + """One label form: the levels of its own list, and the level whose paragraph is referred to (the paragraphs of the + levels above it come first).""" + + title: str + levels: tuple[LevelDef, ...] + target: int + + +#: The label forms the cover can meet. Documented: Microsoft's example, ``Section 1.01`` gives ``1.01``. +ROWS = ( + Row("roman counter with a word: Article %1", (_level(0, "upperRoman", "Article %1"),), 0), + Row("section sign: § %1", (_level(0, "decimal", "§ %1"),), 0), + Row("Microsoft's example: Section %1.%2", (_level(0, "decimal", "%1"), _level(1, "decimal", "Section %1.%2")), 1), + Row("word and roman counter before a decimal: Section %1.%2 with a roman %1", + (_level(0, "upperRoman", "%1"), _level(1, "decimal", "Section %1.%2")), 1), + Row("an abbreviation with a period: Čl. %1", (_level(0, "decimal", "Čl. %1"),), 0), + Row("an abbreviation with a period: No. %1", (_level(0, "decimal", "No. %1"),), 0), + Row("parentheses around a letter: (%1)", (_level(0, "lowerLetter", "(%1)"),), 0), + Row("a parenthesis after a letter: %1)", (_level(0, "lowerLetter", "%1)"),), 0), + Row("a colon after a number: %1:", (_level(0, "decimal", "%1:"),), 0), + Row("a path with a parenthesis: %1.%2(%3)", + (_level(0, "decimal", "%1"), _level(1, "decimal", "%1.%2"), _level(2, "lowerLetter", "%1.%2(%3)")), 2), + Row("words between the counters: Part %1 of %2 (only the guess drops them)", + (_level(0, "decimal", "%1"), _level(1, "decimal", "Part %1 of %2")), 1), +) + +#: The levels of the heading list as the planner writes them for a style with items on the heading list's second level +#: and a ``{path}`` list nested in them on the reserved level 6: levels 3 to 5 are never used. +HEADING_LIST = ( + _level(0, "decimal", "%1."), _level(1, "decimal", "%1.%2"), _level(2, "decimal", "%1.%2.%3"), + _level(3, "decimal", "%1.%2.%3.%4"), _level(4, "decimal", "%1.%2.%3.%4.%5"), + _level(5, "lowerLetter", "%1.%2(%6)"), _level(6, "lowerRoman", "%1.%2(%6)(%7)"), _neutral(7), _neutral(8)) +SKIPPING_LIST = ( + _level(0, "decimal", "%1."), _level(1, "decimal", "%1.%2"), + _level(2, "decimal", "%1.%2.%3"), *(_neutral(k) for k in range(3, 9))) + + +@dataclass +class _Document: + """What the check document is made of: paragraphs, and the numbering definitions they are numbered by.""" + + body: list[ET.Element] + abstracts: list[ET.Element] + nums: list[ET.Element] + bookmarks: Bookmarks + + def list_(self, levels: Sequence[LevelDef], *, overrides: Sequence[tuple[int, int]] = ()) -> tuple[int, WordNumbering]: + """A new abstractNum with a num; returns the numId and the simulator of Word's counters for it.""" + padded = [*levels, *(_neutral(k) for k in range(len(levels), 9))] + abstract = AbstractNum(len(self.abstracts), "enumerated", f"{0x4C440100 + len(self.abstracts):08X}", + "multilevel", tuple(padded)) + instance = Num(len(self.nums) + 1, abstract.id, tuple(overrides)) + self.abstracts.append(abstract_num(abstract)) + self.nums.append(num(instance)) + simulator = WordNumbering(skipped="lo") + simulator.add_abstract(abstract.id, padded) + simulator.add_num(instance) + return instance.id, simulator + + def line(self, *content: object, style: str | None = None, numbering: tuple[int, int] | None = None) -> None: + props = [el("numPr", None, val("ilvl", numbering[1]), val("numId", numbering[0]))] if numbering else [] + self.body.append(p(*content, style=style, props=props)) + + def text(self, text: str, *, style: str | None = None) -> None: + self.line(text_run(text), style=style) + + +def _field(instruction: str, cached: str) -> list[ET.Element]: + return complex_field(instruction, [text_run(cached)]) + + +def _assumed(tokens: Sequence[Lit | Count]) -> str: + """What the planner assumes ``REF \\n \\t`` returns, under each setting of ``docx.reference_text_switch``.""" + said = [] + for mode in ("documented", "guess"): + kept = suppressed(tokens, mode) # type: ignore[arg-type] + said.append(f"{mode}: " + ("the switch is not used" if kept is None else f"'{''.join(t.text for t in kept)}'")) + return "; ".join(said) + + +def _words(tokens: Sequence[Lit | Count]) -> str: + """What ``REF \\n`` returns: the number's text without its trailing periods.""" + return "".join(t.text for t in tokens).rstrip(".") + + +def build() -> bytes: + """The check document as a ``.docx``.""" + doc = _Document([], [], [], Bookmarks()) + doc.text("Word check of the planner's assumptions about numbering", style="Title") + doc.text("Update every field (Ctrl+A, then F9). Each row gives the number as Word draws it, the field REF \\n and " + "the field REF \\n \\t as Word computes them, and the planner's assumption. The fields hold the planner's " + "values until they are updated: report every field that changes, and every row whose number is not the " + "one shown as 'drawn'.") + doc.text("A. The switch \\t on each label form (docx.reference_text_switch)", style="Heading1") + for number, row in enumerate(ROWS, 1): + num_id, simulator = doc.list_(row.levels) + step = None + for ilvl in range(row.target + 1): + step = simulator.step(num_id, ilvl, f"row{number}-{ilvl}") + name = f"t{number}" + runs = text_run(f"target of row {number}") if ilvl == row.target else text_run(f"(parent {ilvl})") + doc.line(*(doc.bookmarks.around(name, runs) if ilvl == row.target else [runs]), + numbering=(num_id, ilvl)) + assert step is not None + tokens = list(step.tokens) + doc.line(text_run(f"Row {number}, {row.title}: drawn '{step.label}'. REF \\n gives "), + *_field(f"REF t{number} \\n \\h", _words(tokens)), text_run(". REF \\n \\t gives "), + *_field(f"REF t{number} \\n \\t \\h", ("".join(t.text for t in (suppressed(tokens, "guess") or tokens)))), + text_run(f". The planner assumes REF \\n gives '{_words(tokens)}'; for REF \\n \\t, {_assumed(tokens)}.")) + doc.text("B. A level text that names the levels around three that are never used", style="Heading1") + doc.text("The heading list below has levels 3 to 5 that nothing numbers, and a nested level 6 whose text is " + "%1.%2(%6). The nested items must read 1.1(a), 1.1(b), then 1.2(a) under the next item.") + num_id, simulator = doc.list_(HEADING_LIST, overrides=((1, 1),)) + for name, ilvl, text in (("r_head", 0, "heading"), ("r_item1", 1, "item 1"), ("r_n1", 5, "nested a"), + ("r_n2", 5, "nested b"), ("r_item2", 1, "item 2"), ("r_n3", 5, "nested under item 2")): + step = simulator.step(num_id, ilvl, name) + doc.line(*doc.bookmarks.around(name, text_run(text)), text_run(f" [planner: {step.label}] REF \\n gives "), + *_field(f"REF {name} \\n \\h", step.word_ref), numbering=(num_id, ilvl)) + doc.text("C. A heading level the document skips", style="Heading1") + doc.text("A level-1 heading followed by a level-3 one. The renderer, and LibreOffice, count the skipped level as 1 " + "(the planner caches that reading, and writes the skipped level's part of a reference as text); Word is " + "expected to draw 0 for it. Record what Word draws.") + heading_id, heading_sim = doc.list_(SKIPPING_LIST) + for name, ilvl, style, text in (("s_one", 0, "Heading1", "First heading"), ("s_deep", 2, "Heading3", "Third level")): + step = heading_sim.step(heading_id, ilvl, name) + doc.line(*doc.bookmarks.around(name, text_run(text)), style=style, numbering=(heading_id, ilvl)) + doc.line(text_run(f"[planner: {step.label}] REF \\n gives "), *_field(f"REF {name} \\n \\h", step.word_ref)) + return _package(doc) + + +def _package(doc: _Document) -> bytes: + style = load_style("default") + settings = docx_settings(style) + package = pkg.Package() + package.relationships.add(pkg.REL_OFFICE_DOCUMENT, "word/document.xml") + package.relationships.add(pkg.REL_CORE_PROPERTIES, "docProps/core.xml") + package.relationships.add(pkg.REL_EXTENDED_PROPERTIES, "docProps/app.xml") + rels = pkg.Relationships() + rels.add(pkg.REL_STYLES, "styles.xml") + rels.add(pkg.REL_NUMBERING, "numbering.xml") + rels.add(pkg.REL_SETTINGS, "settings.xml") + page = settings.page + section = el("sectPr", None, + el("pgSz", {"w": page.width, "h": page.height}), + el("pgMar", {"top": page.top, "right": page.right, "bottom": page.bottom, "left": page.left, + "header": page.header, "footer": page.footer, "gutter": 0})) + package.add("docProps/core.xml", pkg.CT_CORE_PROPERTIES, build_core("Word check of the numbering assumptions", "en")) + package.add("docProps/app.xml", pkg.CT_EXTENDED_PROPERTIES, build_app("legaldown-render word_check")) + package.add("word/document.xml", pkg.CT_DOCUMENT, serialize(el("document", None, el("body", None, doc.body, section))), + rels) + package.add("word/styles.xml", pkg.CT_STYLES, build_styles(style, settings, "en", heading_num_id=None)) + package.add("word/numbering.xml", pkg.CT_NUMBERING, build_numbering(doc.abstracts, doc.nums)) + package.add("word/settings.xml", pkg.CT_SETTINGS, build_settings(update_fields=True)) + return package.to_bytes() + + +def main(argv: Sequence[str]) -> int: + if len(argv) != 2: + print(__doc__) + return 2 + path = Path(argv[1]) + path.parent.mkdir(parents=True, exist_ok=True) + path.write_bytes(build()) + print(f"wrote {path}") + return 0 + + +if __name__ == "__main__": + sys.exit(main(sys.argv))