diff --git a/.gitignore b/.gitignore index 8a51ff9..497518a 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,7 @@ .env ExampleContents/ +# Written from colleagues' documents, like ExampleContents itself. +corpus-specs/ .workbench/ __pycache__/ *.egg-info/ diff --git a/README.md b/README.md index b22d333..0d31e93 100644 --- a/README.md +++ b/README.md @@ -115,7 +115,8 @@ prints, and the in2lambda function each stage calls. What the checks find — a block of the source in no field, two fields from the same lines, a gap in the numbering, a part nothing answers — returns to the model as a fixing round, with in2lambda's draft commands as its tools: `mark ignore`, -`question add`, `part add`, `question solution`, `split block`, and `field replace` for +`question add`, `part add`, `question solution`, `part solution`, `split block`, +`field set` for a field that is empty or took the wrong lines, and `field replace` for wording that no range of the source gives. The model writes a field by naming where its text is in the source rather than by typing the text out. in2lambda records each command in the draft's log with the layer of the field it wrote, so a reader can read diff --git a/docs/how-it-works.md b/docs/how-it-works.md index a0fdf74..66a177f 100644 --- a/docs/how-it-works.md +++ b/docs/how-it-works.md @@ -215,7 +215,7 @@ Each backend limits a call differently: | --- | --- | --- | | Spec | the spec system prompt, and the frozen source as `in2lambda.source.show` prints it | `in2lambda-spec.yaml`, and nothing else | | Spec rewrite | the same, with the errors of the last report appended to the prompt | `in2lambda-spec.yaml`, and nothing else | -| Fixing round | the fixing system prompt, the frozen source, every finding of the report, and a reviewer's note where there is one | the six draft commands, and nothing else | +| Fixing round | the fixing system prompt, the frozen source, every finding of the report, and a reviewer's note where there is one | the eight draft commands, and nothing else | The spec call has no tools. Its reply is the YAML of a spec, past a code fence where the model wrote one. The agent refuses a reply that is not YAML, a reply that is not a @@ -226,8 +226,9 @@ The spec rewrite is the same call with the report's errors in the prompt. It run per run, before any fixing round, where the run reused a saved spec and the checks fault the draft. It writes layer 1 fields, and it is not one of the `--rounds`. -The fixing round's tools are the six in2lambda draft commands: `mark ignore`, -`question add`, `part add`, `question solution`, `field replace` and `split block`. +The fixing round's tools are the eight in2lambda draft commands: `mark ignore`, +`question add`, `part add`, `question solution`, `part solution`, `field replace`, +`field set` and `split block`. in2lambda writes the field, records the command in the draft's log as `in2lambda-agent` and decides the layer. A command in2lambda refuses returns its refusal to the model as the tool's result, and the round continues. A field's text is named by block id or line diff --git a/in2lambda_agent/fix.py b/in2lambda_agent/fix.py index 2f858a7..1e2bbe1 100644 --- a/in2lambda_agent/fix.py +++ b/in2lambda_agent/fix.py @@ -78,6 +78,10 @@ part_add a lettered part of a question that is written already. question_solution a question's worked solution, wherever it is written. It answers every part of that question that has none of its own. + part_solution one part's own worked solution, from the lines under it on + the sheet. Use it where each part is answered where it + stands, and question_solution where one solution answers + the whole question. split_block cuts a block in two at a line, so that each half can be named: `b7` split at 14 becomes `b7a` and `b7b`. Use it when one block holds two things — a question and its first part @@ -90,6 +94,12 @@ place is at most {LITERAL_MAX} characters, as a literal is. A field whose text the source does not hold is left as a finding. + field_set quotes other lines into a field that is written already, + for a field that is empty or that took the wrong lines. It + writes the field again from the lines you name and drops + the ones it held, which are then in no field until you + answer for them. It has no literal: what it writes is in + the source. Lines that are already in a field cannot be put in another one. A command naming \ them is refused, and so is one naming a block that is not there; either way you \ @@ -110,6 +120,11 @@ "description": "The question to add to, by the key of its text: q2.", } +_PART = { + "type": "string", + "description": "The part to answer, by the key of its text: q1.p2.", +} + # Every command that fills a field takes one or the other of these, and in2lambda # refuses both at once, so neither is required and the model is told which to use. _WHERE = { @@ -144,6 +159,15 @@ "not write one." ), "split block": "Cut one block in two at a line, so each half can be named.", + "field set": ( + "Quote other lines of the source into a field already written, for a " + "field that is empty or that took the wrong lines. The lines it held " + "are dropped, and are then in no field." + ), + "part solution": ( + "Give one part the worked solution written under it. Where one " + "solution answers the whole question, question solution writes it." + ), } _PARAMETERS: dict[str, dict[str, Any]] = { @@ -204,6 +228,31 @@ }, "required": ["block", "at"], }, + # No `literal`: this command quotes lines into a field and in2lambda takes + # nothing else, so the wording no range of the source holds is `field + # replace`'s to repair once the field has been set. + "field set": { + "type": "object", + "properties": { + "field": { + "type": "string", + "description": "The field to write again, by its key: q1.text.", + }, + "text": { + "type": "string", + "description": ( + "Where the field's text is in the frozen source: a block " + "id, b7, or lines, s13 or s13:14." + ), + }, + }, + "required": ["field", "text"], + }, + "part solution": { + "type": "object", + "properties": {"part": _PART, **_WHERE}, + "required": ["part"], + }, } @@ -412,7 +461,7 @@ def _writes_field(draft: Path, name: str, args: dict[str, Any]) -> Optional[str] def _subject(call: ToolCall) -> str: """What one command was about, for the stage line: a block, question or field.""" - for name in ("block", "question", "field", "text", "literal"): + for name in ("block", "question", "part", "field", "text", "literal"): if name in call.arguments: return str(call.arguments[name]) return "" diff --git a/in2lambda_agent/package.py b/in2lambda_agent/package.py index 7778aec..b1af232 100644 --- a/in2lambda_agent/package.py +++ b/in2lambda_agent/package.py @@ -59,6 +59,8 @@ "question solution", "field replace", "split block", + "field set", + "part solution", ) """The draft commands a report is fixed with, named as the log names them.""" diff --git a/poetry.lock b/poetry.lock index 0216df5..d6d0e00 100644 --- a/poetry.lock +++ b/poetry.lock @@ -511,7 +511,7 @@ all = ["coverage (>=7.10.0)", "hypothesis (>=6.141.1)", "mypy (>=1.11.2)", "pyte [[package]] name = "in2lambda" -version = "2.0.0" +version = "1.1.0" description = "Converts content ready for import into Lambda Feedback" optional = false python-versions = "^3.10" @@ -532,7 +532,7 @@ convert = ["panflute (>=2.3.1,<3.0.0)", "pyyaml (>=6.0,<7.0)"] type = "git" url = "https://github.com/lambda-feedback/in2lambda.git" reference = "dev" -resolved_reference = "6699c53163eace1b02ecd15106612a94d95de4f7" +resolved_reference = "a76603dc3d21847c687caf558255941aa966a5fa" [[package]] name = "iniconfig" diff --git a/tests/test_fix.py b/tests/test_fix.py index dd254e3..b7d6adb 100644 --- a/tests/test_fix.py +++ b/tests/test_fix.py @@ -49,6 +49,8 @@ def test_the_tools_are_the_packages_draft_commands(draft): "question_solution", "field_replace", "split_block", + "field_set", + "part_solution", ] @@ -65,6 +67,7 @@ def test_the_tools_are_the_packages_draft_commands(draft): "q1.text", ), ("split_block", {"block": "b7", "at": 14}, "b7a and b7b"), + ("field_set", {"field": "q1.text", "text": "s22"}, "q1.text"), ], ) def test_each_tool_runs_its_command_and_in2lambda_logs_it( @@ -82,6 +85,72 @@ def test_each_tool_runs_its_command_and_in2lambda_logs_it( } +def test_a_field_set_writes_a_written_field_from_the_lines_it_names(draft): + # The finding it answers: a field that is empty, or that took the wrong + # lines. What the field held before is dropped, so the lines it came from + # are in no field and the next round is told about them. + before = json.loads(draft.read_text())["fields"]["q1.text"]["ranges"] + + result = run(draft, "field_set", {"field": "q1.text", "text": "s22"}) + written = json.loads(draft.read_text())["fields"]["q1.text"] + + assert result == "field set wrote q1.text" + assert before == [[5, 5]] + assert (written["ranges"], written["layer"], written["edited"]) == ( + [[22, 22]], + 3, + False, + ) + + +def test_a_part_solution_answers_one_part_rather_than_the_question(draft): + # A sheet that writes a solution under each part: the part is added from + # the lines holding it, and then answered on its own. + run(draft, "part_add", {"question": "q1", "text": "b7"}) + + result = run(draft, "part_solution", {"part": "q1.p3", "text": "s22"}) + written = json.loads(draft.read_text())["fields"]["q1.p3.solution"] + + assert result == "part solution wrote q1.p3.solution" + assert (written["ranges"], written["layer"], written["edited"]) == ( + [[22, 22]], + 3, + False, + ) + assert package.command_log(draft)[-1] == { + "command": "part solution", + "args": {"part": "q1.p3", "text": "s22"}, + "by": package.BY, + } + + +def test_a_part_solution_typed_out_is_layer_4(draft): + run(draft, "part_add", {"question": "q1", "text": "b7"}) + + result = run(draft, "part_solution", {"part": "q1.p3", "literal": "Term by term."}) + written = json.loads(draft.read_text())["fields"]["q1.p3.solution"] + + assert result == "part solution wrote q1.p3.solution" + assert (written["layer"], written["edited"]) == (4, True) + + +def test_a_part_solution_longer_than_a_repair_is_refused_before_it_is_written(draft): + typed = "x" * (fix.LITERAL_MAX + 1) + run(draft, "part_add", {"question": "q1", "text": "b7"}) + + result = run(draft, "part_solution", {"part": "q1.p3", "literal": typed}) + + assert result.startswith("part solution was refused: ") + assert f"literal is {len(typed)} characters" in result + assert commands(draft) == ["spec run", "part add"] + + +def test_the_system_prompt_names_the_two_commands_that_write_a_written_field(): + assert "part_solution one part's own worked solution" in fix.SYSTEM + assert "field_set quotes other lines into a field" in fix.SYSTEM + assert "It has no literal" in fix.SYSTEM + + @pytest.mark.parametrize( "arguments, layer, edited", [ @@ -294,10 +363,13 @@ def test_every_tool_that_types_says_how_little_it_may_type(draft): one for one in fix.tools(draft) if "literal" in one.parameters["properties"] ] + # `field_set` is not among them: in2lambda takes no literal for it, so a + # field it writes says what the source says and nothing else. assert [one.name for one in typing] == [ "question_add", "part_add", "question_solution", + "part_solution", ] assert all( one.parameters["properties"]["literal"]["maxLength"] == fix.LITERAL_MAX @@ -390,6 +462,10 @@ def test_the_halves_of_a_split_block_are_shown_to_the_next_round(draft): ], "2 commands (mark ignore b9, question solution q2)", ), + ( + [("part_solution", {"part": "q1.p3", "text": "s22"})], + "1 command (part solution q1.p3)", + ), ], ) def test_the_summary_names_what_each_command_was_about(calls, expected):