Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
14 changes: 10 additions & 4 deletions agentdescent/diagnose.py
Original file line number Diff line number Diff line change
Expand Up @@ -191,7 +191,8 @@ def diagnose_payload(run_id: str, *, store: Optional[str] = None) -> DiagnosisRe
run_id, "cancelled", "the run was cancelled before it finished",
evidence={"stop_reason": payload.get("stop_reason"),
"outcomes": payload.get("outcomes") or {}},
recommendations=("resume the run if the answer is still wanted",))
recommendations=("resume it if the answer is still wanted: "
f"`agentdescent resume {run_id}`",))
if payload.get("error"):
evidence = {"error": payload["error"]}
if payload.get("propose_error"):
Expand All @@ -205,7 +206,8 @@ def diagnose_payload(run_id: str, *, store: Optional[str] = None) -> DiagnosisRe
run_id, "running", "the run is still running; there is no verdict yet",
evidence={"stop_reason": payload.get("stop_reason"),
"history": payload.get("history") or []},
recommendations=("use `watch`/`status` and diagnose again when done",))
recommendations=(f"follow it with `agentdescent watch {run_id}`, then "
f"`agentdescent diagnose {run_id}` when it ends",))

final = payload.get("final_reward")
final_value = float(final) if final is not None else None
Expand All @@ -228,7 +230,10 @@ def diagnose_payload(run_id: str, *, store: Optional[str] = None) -> DiagnosisRe
f"held-out reward is {final_value:.3f}",
evidence={"final_reward": final_value,
"outcomes": outcomes},
recommendations=("review and apply the artifact",))
recommendations=(
f"review the diff: `agentdescent show {run_id}`",
f"preview the write: `agentdescent apply {run_id} "
"--dry-run`, then rerun it without --dry-run"))

perr = payload.get("propose_errors") or 0
if perr and not committed:
Expand Down Expand Up @@ -273,7 +278,8 @@ def diagnose_payload(run_id: str, *, store: Optional[str] = None) -> DiagnosisRe
"final_reward": final_value, "outcomes": outcomes,
"committed": committed},
recommendations=("raise rounds or n_workers for another attempt",
"inspect the remaining failures in `show --json`"))
"inspect the remaining failures in "
f"`agentdescent show {run_id} --json`"))

if rejected:
top = rejected[-1]
Expand Down
2 changes: 2 additions & 0 deletions changelog.d/898-diagnose-next-commands.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@
Fixed
`agentdescent diagnose` now names the runnable next command with the resolved run id for `solved` (`show <id>`, `apply <id> --dry-run`), `cancelled` (`resume <id>`), `running` (`watch <id>`) and `partial-improvement` (`show <id> --json`), instead of prose such as "review and apply the artifact" (#898).
8 changes: 5 additions & 3 deletions docs/diagnose.md
Original file line number Diff line number Diff line change
Expand Up @@ -14,10 +14,10 @@ The verdicts are deliberately few and ordered so the strongest evidence wins:
| Verdict | Meaning | First thing to try |
|---|---|---|
| `seed-already-solved` | the seed already scored the target before any round; nothing was learned | add cases the seed gets wrong, a stricter score, or a higher `target_reward` |
| `solved` | the target was reached | review and apply |
| `solved` | the target was reached | `agentdescent show <run_id>`, then `apply <run_id> --dry-run` |
| `partial-improvement` | candidates committed and held-out reward rose above the seed's, still below target | more rounds/workers |
| `running` | the run has not finished | `status` / `watch`, then diagnose again |
| `cancelled` | the run was cancelled | resume if the answer is still wanted |
| `running` | the run has not finished | `agentdescent watch <run_id>`, then diagnose again |
| `cancelled` | the run was cancelled | `agentdescent resume <run_id>` if the answer is still wanted |
| `run-failed` | the run reported an error | `log.txt` / `status.json`; include `propose_error` when present |
| `reflector-error` | reflector calls failed and nothing committed | model/endpoint/key (`doctor`) |
| `scoring-mismatch` | untruncated samples' output equals gold but reward is low | inspect the scorer |
Expand All @@ -36,6 +36,8 @@ The verdicts are deliberately few and ordered so the strongest evidence wins:
(a named algorithm's `dgm-select` / `gepa-select` parent picks commit every
round) fall through to the later verdicts (#756).
- Running and cancelled runs are named instead of being diagnosed as finished.
- Next steps for `solved`, `running`, `cancelled` and `partial-improvement`
name the runnable command with the resolved run id (also for `latest`) (#898).
- `diagnose_payload` calls `show_payload(..., diff=False)`, so it does not build
an artifact diff it would throw away.

Expand Down
27 changes: 27 additions & 0 deletions tests/test_diagnose.py
Original file line number Diff line number Diff line change
Expand Up @@ -178,3 +178,30 @@ def test_a_real_gain_names_both_rewards(monkeypatch):
assert report.verdict == "partial-improvement"
assert "0.250 -> 0.750" in report.summary
assert report.evidence["initial_reward"] == 0.25


@pytest.mark.parametrize("payload,verdict,commands", [
({"run_id": "20260101-000000-abc123", "final_reward": 1.0,
"outcomes": {"committed": 1}},
"solved", ["agentdescent show 20260101-000000-abc123",
"agentdescent apply 20260101-000000-abc123 --dry-run"]),
({"run_id": "20260101-000000-abc123", "state": "cancelled"},
"cancelled", ["agentdescent resume 20260101-000000-abc123"]),
({"run_id": "20260101-000000-abc123", "state": "running"},
"running", ["agentdescent watch 20260101-000000-abc123",
"agentdescent diagnose 20260101-000000-abc123"]),
({"run_id": "20260101-000000-abc123", "final_reward": 0.5,
"initial_reward": 0.0, "outcomes": {"committed": 1}},
"partial-improvement",
["agentdescent show 20260101-000000-abc123 --json"]),
])
def test_next_steps_name_the_resolved_run_id(monkeypatch, payload, verdict,
commands):
# Asked as `latest`; the next step must carry the id show resolved (#898).
_patch(monkeypatch, payload)
report = diagnose_payload("latest")
assert report.verdict == verdict
text = " ".join(report.recommendations)
for command in commands:
assert command in text
assert "latest" not in text
Loading