implement: Save every run beside its zip (t46)

Author: peterbjohnson

README.md

@@ -110,6 +110,10 @@ the fields are route A's, and each one is flagged or is route A's word for it. A
 run that fails counts route A's fields in the same way, and adds a `route B failed` line
 naming pandoc's message; the set is route A's reading alone.
 
+The run is saved beside the zip. `OUT/report.txt` holds the printed lines, `OUT/reply-a.json`
+route A's reply, `OUT/reply-b.json` route B's, and `OUT/flags.json` one entry per flagged
+field. A run with no filter, and a run whose filter failed, writes no `reply-b.json`.
+
 `convert` exits 1 where a named file is not there, and where Mathpix, the model or
 pandoc failed, and 0 otherwise. A flagged
 field does not change the exit code: the zip is written whatever the flags say, and a

in2lambda_agent/routes.py

@@ -24,7 +24,7 @@
 import json
 import re
 import subprocess
-from dataclasses import dataclass, field
+from dataclasses import asdict, dataclass, field
 from pathlib import Path
 from typing import Any, Callable, Optional
 
@@ -415,6 +415,9 @@ class Converted:
     # twice saves this reply and passes it back as `convert`'s `route_a`; a
     # second call to the model returns different wording.
     route_a: Reply_ = field(default_factory=list)
+    # Route B's reply, the filter's two runs merged and before reconciling, and None
+    # where no filter was given or the filter run failed.
+    route_b: Optional[Reply_] = None
     tokens: int = 0
     # The counts of the reconciliation, zero where route B did not run.
     fields: int = 0
@@ -526,6 +529,12 @@ def convert(
     `on_stage`, where it is given, is called with a name and a message as each step
     finishes - `ocr`, `route A`, `route B`, `fields`, `build` - so that a caller watching
     a run shows each line as the step ends rather than the report at the end of it.
+
+    The run is saved in `out_dir` beside the zip: `reply-a.json` as route A answers,
+    `reply-b.json` as the filter runs, and `flags.json` and `report.txt` after the build.
+    Each file is written as soon as its content exists, so that a step that fails keeps
+    the replies of the steps before it. Where no filter was given, or the filter run
+    failed, no `reply-b.json` is written.
     """
     settings = settings or load_settings()
     backend = backend or choose_backend(settings)
@@ -534,6 +543,10 @@ def said(stage: str, message: str) -> None:
         if on_stage is not None:
             on_stage(stage, message)
 
+    out_dir = Path(out_dir)
+    # The build writes the zip here at the end of the run; the replies are written
+    # into the same directory as each route answers, before the build makes it.
+    out_dir.mkdir(parents=True, exist_ok=True)
     read = [f"{Path(d).name}: {_read_as(d, cache_dir)}" for d in (document, solutions) if d is not None]
     markdown, images = markdown_of(document, cache_dir, settings)
     solutions_md = markdown_of(solutions, cache_dir, settings)[0] if solutions else None
@@ -546,7 +559,9 @@ def said(stage: str, message: str) -> None:
     else:
         tokens = 0
         said("route A", "the reply given, no call made")
+    (out_dir / "reply-a.json").write_text(json.dumps(route_a, indent=2), encoding="utf-8")
     reply = route_a
+    route_b = None
     counts, error = (0, 0, 0, 0), None
     flags = [Flag(k, fields(reply)[k], "", NOT_VERBATIM) for k in not_verbatim(reply, source)]
     if lua is None:
@@ -561,6 +576,8 @@ def said(stage: str, message: str) -> None:
             error = (stderr.decode("utf-8", "replace") if stderr else str(problem)).strip()
             said("route B", f"failed: {error}")
         else:
+            route_b = other
+            (out_dir / "reply-b.json").write_text(json.dumps(other, indent=2), encoding="utf-8")
             reconciled = reconcile(reply, other, source, backend)
             reply, flags = reconciled.fields, reconciled.flags
             # The adjudication call is the document's second call, so its tokens
@@ -576,13 +593,19 @@ def said(stage: str, message: str) -> None:
             flags.append(Flag(k, fields(reply)[k], "", STRAY_MINUS))
     result = Converted(
         set=to_set(reply, name=name, directory=images), zip_path=None, flags=flags,
-        reply=reply, route_a=route_a, tokens=tokens,
+        reply=reply, route_a=route_a, route_b=route_b, tokens=tokens,
         fields=counts[0], agreed=counts[1], defaulted=counts[2], adjudicated=counts[3],
         route_b_error=error,
     )
     said("fields", result.counted())
     result.zip_path = build(result.set, out_dir)
     said("build", str(result.zip_path))
+    (out_dir / "flags.json").write_text(
+        json.dumps([asdict(one) for one in result.flags], indent=2), encoding="utf-8"
+    )
+    # The report names the zip, so it is written after the build, and holds the lines
+    # the command prints.
+    (out_dir / "report.txt").write_text("\n".join(result.report()) + "\n", encoding="utf-8")
     return result
 
 

tests/test_routes.py

@@ -7,6 +7,7 @@
 """
 
 import copy
+import dataclasses
 import difflib
 import json
 import os
@@ -464,6 +465,66 @@ def test_a_sheet_whose_filter_run_fails_keeps_its_route_a_reply(tmp_path):
     assert result.route_b_error
     assert result.reply == SHEET_DIRECT
     assert (result.fields, result.agreed, result.flags) == (0, 0, [])
+    # Route B wrote no reply, so there is no reply-b.json; the report says why.
+    assert not (tmp_path / "out" / "reply-b.json").exists()
+    assert "route B   failed:" in (tmp_path / "out" / "report.txt").read_text()
+
+
+# --- what a run leaves in the out directory ------------------------------------------------
+
+
+def test_a_run_leaves_its_report_and_replies_beside_the_zip(tmp_path):
+    out = tmp_path / "out"
+    result = routes.convert(
+        ME2 / "questions.md",
+        solutions=ME2 / "solutions.md",
+        out_dir=out,
+        backend=FakeBackend(json.dumps(REPLY)),
+        settings=Settings(),
+    )
+
+    assert json.loads((out / "reply-a.json").read_text()) == REPLY == result.route_a
+    assert json.loads((out / "flags.json").read_text()) == [
+        dataclasses.asdict(f) for f in result.flags
+    ]
+    assert (out / "report.txt").read_text() == "\n".join(result.report()) + "\n"
+    assert "route B did not run" in (out / "report.txt").read_text()
+    # No filter ran, so route B has no reply to save.
+    assert not (out / "reply-b.json").exists()
+
+
+@pytest.mark.skipif(shutil.which("pandoc") is None, reason="pandoc")
+def test_a_run_with_a_filter_leaves_route_bs_reply_too(tmp_path):
+    out = tmp_path / "out"
+    lua = FIXTURES / "pair-filter.lua"
+    backend = FakeBackend(
+        json.dumps(PAIRED_DIRECT),
+        json.dumps([{"field": "q2.main_text", "choice": "A", "reason": "B carries the parts too"}]),
+    )
+
+    result = routes.convert(
+        FIXTURES / "paired.md",
+        solutions=FIXTURES / "paired_solutions.md",
+        out_dir=out,
+        backend=backend,
+        settings=Settings(),
+        lua=lua,
+        name="paired",
+    )
+
+    # Route B's reply is the two filter runs merged, as convert merges them, and is
+    # saved before reconciling changes any field.
+    other = routes.merge(
+        routes.run_filter(lua, FIXTURES / "paired.md"),
+        routes.run_filter(lua, FIXTURES / "paired_solutions.md", role="solutions"),
+    )
+    assert result.route_b == other
+    assert json.loads((out / "reply-b.json").read_text()) == other
+    assert json.loads((out / "reply-a.json").read_text()) == PAIRED_DIRECT
+    assert json.loads((out / "flags.json").read_text()) == [
+        dataclasses.asdict(f) for f in result.flags
+    ]
+    assert (out / "report.txt").read_text() == "\n".join(result.report()) + "\n"
 
 
 # --- the report of one document -------------------------------------------------------------
@@ -604,3 +665,10 @@ def test_the_me2_pair_converts_from_the_command_line(tmp_path, capsys):
         "flag      q3.p1.worked_solution",
     ]
     assert printed.splitlines()[-1].startswith(f"build     {tmp_path / 'out'}")
+    # The run is saved beside the zip: no filter ran, so there is no reply-b.json.
+    out = tmp_path / "out"
+    report = (out / "report.txt").read_text()
+    assert report == "\n".join(printed.splitlines()[1:]) + "\n"
+    assert json.loads((out / "reply-a.json").read_text())
+    assert json.loads((out / "flags.json").read_text())
+    assert not (out / "reply-b.json").exists()