diff --git a/pyproject.toml b/pyproject.toml index 6899f79..5766326 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -51,7 +51,10 @@ tracing = [ plot = ["matplotlib>=3.5.0"] # Run scenario packs as Inspect AI tasks and read Inspect logs back # (simpleaudit.integrations.inspect). -inspect = ["inspect-ai>=0.3.283"] +# agent-client-protocol 1.0.0 (2026-11-09) removed acp.helpers, which inspect-ai +# 0.3.288 still imports (it only requires >=0.12.0). Drop the cap once inspect-ai +# supports 1.x. +inspect = ["inspect-ai>=0.3.283", "agent-client-protocol<1"] dev = [ # pytest is pinned <9 because pytest-testmon 2.x is not yet compatible with # pytest 9 (same pin as Studio). diff --git a/simpleaudit/failures.py b/simpleaudit/failures.py new file mode 100644 index 0000000..ba1aa65 --- /dev/null +++ b/simpleaudit/failures.py @@ -0,0 +1,167 @@ +"""Which role failed a scenario, and how (structured, best effort). + +A failed scenario keeps its free-text ``judgment.error`` and +``judgment.error_phase`` and gains ``judgment.failure``:: + + {"role": "target|auditor|judge|system", # required + "kind": "rate_limit|unavailable|auth|timeout|bad_response|config|unknown", + "message": "", + "http_status": 429, "exception": "RateLimitError", + "endpoint": "...", "model": "...", + "retryable": true, "turn": 3} + +Only ``role`` (and ``kind``, ``message``, ``exception``) are always present; +the other keys are left out when unknown. The role is set at the call site, +so no target cooperation is needed and nothing here assumes a model, a +provider or a target type. ``kind`` comes from the HTTP status the exception +exposes, or its type; anything else is ``unknown``. ``retryable`` is a hint +for consumers and does not describe the engine's own retry behaviour. +""" + +from __future__ import annotations + +import asyncio +import re +from typing import Any, Dict, Optional + +from .utils import describe_exception, redact_url_secrets + +ROLES = ("target", "auditor", "judge", "system") +KINDS = ("rate_limit", "unavailable", "auth", "timeout", "bad_response", "config", "unknown") +_RETRYABLE = {"rate_limit", "unavailable", "timeout", "bad_response"} +_MAX_CHAIN = 6 + + +def _chain(exc: BaseException): + seen = set() + cur: Optional[BaseException] = exc + while cur is not None and id(cur) not in seen and len(seen) < _MAX_CHAIN: + seen.add(id(cur)) + yield cur + cur = cur.__cause__ or cur.__context__ + + +def _as_status(value: Any) -> Optional[int]: + if isinstance(value, bool): + return None + if isinstance(value, int) and 100 <= value <= 599: + return value + return None + + +def http_status_of(exc: BaseException) -> Optional[int]: + """HTTP status the exception (or its cause chain) exposes, else None.""" + for item in _chain(exc): + for attr in ("status_code", "status", "http_status", "code"): + status = _as_status(getattr(item, attr, None)) + if status is not None: + return status + response = getattr(item, "response", None) + if response is not None: + for attr in ("status_code", "status"): + status = _as_status(getattr(response, attr, None)) + if status is not None: + return status + return None + + +def _kind_from_type(exc: BaseException) -> str: + # Local import: provider_errors is independent of this module. + from .provider_errors import ProviderResponseError + + for item in _chain(exc): + if isinstance(item, (asyncio.TimeoutError, TimeoutError)): + return "timeout" + if isinstance(item, ConnectionError): + return "unavailable" + if isinstance(item, ProviderResponseError): + return "bad_response" + name = type(item).__name__.lower() + if "ratelimit" in name: + return "rate_limit" + if "timeout" in name: + return "timeout" + if "authentication" in name or "permissiondenied" in name: + return "auth" + if any( + w in name + for w in ("connect", "unavailable", "overloaded", "internalserver", "badgateway") + ): + return "unavailable" + if name in ("jsondecodeerror", "validationerror"): + return "bad_response" + return "unknown" + + +def classify_kind(exc: BaseException, http_status: Optional[int]) -> str: + """Provider-agnostic cause from the HTTP status, else the exception type.""" + if http_status is not None: + if http_status == 429: + return "rate_limit" + if http_status in (401, 403): + return "auth" + if http_status in (408, 504): + return "timeout" + if http_status >= 500: + return "unavailable" + if 400 <= http_status < 500: + return "config" + return _kind_from_type(exc) + + +def _clean_endpoint(url: Any) -> Optional[str]: + if not url: + return None + text = str(url) + # Credentials riding in the URL are never kept (not even redacted). + text = re.sub(r"(\w+://)[^/?#@\s]*@", r"\1", text) + return redact_url_secrets(text) + + +def _endpoint_of(exc: BaseException) -> Optional[str]: + for item in _chain(exc): + for holder in (getattr(item, "request", None), getattr(item, "response", None)): + url = getattr(holder, "url", None) if holder is not None else None + cleaned = _clean_endpoint(url) + if cleaned: + return cleaned + return None + + +def build_failure( + role: str, + exc: BaseException, + *, + turn: Optional[int] = None, + endpoint: Optional[str] = None, + model: Optional[str] = None, +) -> Dict[str, Any]: + """The ``failure`` object for ``exc`` raised by ``role``. + + ``endpoint`` and ``model`` are hints from the call site, used when the + exception itself does not name an endpoint. ``turn`` is 1-based. + """ + role = role if role in ROLES else "system" + status = http_status_of(exc) + kind = classify_kind(exc, status) + failure: Dict[str, Any] = { + "role": role, + "kind": kind, + "message": describe_exception(exc), + "exception": type(exc).__name__, + } + if status is not None: + failure["http_status"] = status + where = _endpoint_of(exc) or _clean_endpoint(endpoint) + if where: + failure["endpoint"] = where + if model: + failure["model"] = str(model) + declared = getattr(exc, "retryable", None) + if isinstance(declared, bool): + failure["retryable"] = declared + elif kind != "unknown": + failure["retryable"] = kind in _RETRYABLE + if turn is not None: + failure["turn"] = turn + return failure diff --git a/simpleaudit/model_auditor.py b/simpleaudit/model_auditor.py index 490244f..b1b06ae 100644 --- a/simpleaudit/model_auditor.py +++ b/simpleaudit/model_auditor.py @@ -37,6 +37,7 @@ from .judges.compose import inherited_postprocess from .judges.compose import SEVERITY_RESPONSE_SCHEMA from .judges.default import DEFAULT_JUDGE_CRITERIA, DEFAULT_JUDGE_SEVERITY_LEVELS, DEFAULT_PROBE_PROMPT +from .failures import build_failure from .provider_errors import ( EmptyResponseError, _content_to_text, @@ -130,7 +131,7 @@ class ScenarioExecution: #: failures, or carry the ground truth and designed severity a later #: re-grade (reframing.rejudge) trusts. _FRAMEWORK_KEYS = ( - "parse_error", "error", "error_phase", "scenario_metadata", + "parse_error", "error", "error_phase", "failure", "scenario_metadata", "designed_severity", "designed_severity_source", "judge_fallback", "asr_reference", ) @@ -1106,6 +1107,42 @@ def _log(self, message: str, name: Optional[str] = None): message = f"[{name}] {message}" tqdm.write(message) + def _failure_for( + self, + role: str, + exc: BaseException, + *, + turn: Optional[int] = None, + target: Any = None, + ) -> Dict[str, Any]: + """Structured ``judgment.failure`` for ``exc`` raised by ``role``. + + Endpoint and model are hints from the configuration and are only used + when known; a custom target contributes its own ``url``/``model`` if it + has them. + """ + endpoint = model = None + try: + if role == "auditor": + endpoint = self._auditor_client_config.get("base_url") + model = self.auditor_model + elif role == "judge": + endpoint = self._judge_client_config.get("base_url") + model = None if self.judge_grade is not None else self.judge_model + elif role == "target": + tgt = target if target is not None else self.target + endpoint = getattr(tgt, "url", None) or getattr(tgt, "base_url", None) + model = getattr(tgt, "model", None) + if endpoint is None and type(tgt).__name__ == "ModelTarget": + endpoint = self._target_client_config.get("base_url") + if not isinstance(endpoint, str): + endpoint = None + if not isinstance(model, str): + model = None + except Exception: # hints are best effort and must never mask the error + endpoint = model = None + return build_failure(role, exc, turn=turn, endpoint=endpoint, model=model) + def _fire_on_turn( self, turn_index: int, @@ -1614,6 +1651,11 @@ async def run_scenario( error: Optional[str] = None error_traceback: Optional[str] = None error_phase: Optional[str] = None + failure: Optional[Dict[str, Any]] = None + # Which role the call in flight belongs to, set right before each call + # so an exception is attributed where it is raised. + current_role = "system" + current_turn: Optional[int] = None try: validate = getattr(self.judge_postprocess, "validate_scenario", None) if validate is not None: @@ -1629,16 +1671,20 @@ async def run_scenario( if trace_correlation is not None: _link_scenario(trace_correlation, scenario_run_id, name) trace_correlation.record(f"{scenario_run_id}_t0", scenario_trace_id) + # The hook opens the target's session. + current_role = "target" await start_hook(hook_context) started = True for turn in range(turns): self._log(f"--- Turn {turn + 1}/{turns} ---", name=name) + current_turn = turn + 1 # First turn: if the scenario defines a test_prompt, send it verbatim. # Subsequent turns use the auditor model for probe generation. if turn == 0 and (test_prompt or getattr(target, "default_prompt", None)): probe = test_prompt or target.default_prompt else: + current_role = "auditor" # require_content checks the raw text; a reasoning auditor # cut off inside (or a bare "User:") is empty only # after stripping. That is a model hiccup like an empty @@ -1703,6 +1749,7 @@ async def run_scenario( if trace_correlation is not None: _link_scenario(trace_correlation, scenario_run_id, name) trace_correlation.record(turn_id, scenario_trace_id) + current_role = "target" target_resp = await target.send( system=self.system_prompt, user=probe, @@ -1808,6 +1855,7 @@ async def run_scenario( error = describe_exception(exc) error_traceback = _short_traceback() error_phase = "conversation" + failure = self._failure_for(current_role, exc, turn=current_turn, target=target) self._log(f"--- Scenario FAILED mid-conversation: {name} [{error}] ---") finally: # Runs on failure and cancellation too, so a target can release its @@ -1875,6 +1923,7 @@ async def run_scenario( error = describe_exception(exc) error_traceback = _short_traceback() error_phase = "judge" + failure = self._failure_for("judge", exc) self._log(f"--- Judging FAILED: {name} [{error}] ---") if error is not None: @@ -1894,6 +1943,8 @@ async def run_scenario( # complete (see reframing.rejudge); a "judge" failure leaves a # complete one. "error_phase": error_phase, + # Which role failed and how, structured (see failures.py). + "failure": failure, } else: judgment = coerce_judgment_fields(judgment) @@ -2065,6 +2116,8 @@ async def _run_one(scenario: Dict) -> AuditResult: "severity": "ERROR", "error": describe_exception(exc), "traceback": _short_traceback(), + # Raised outside the three calls: not attributable. + "failure": build_failure("system", exc), }, ) with tqdm(total=total_audit_steps, desc=audit_desc, disable=not self.show_progress, position=0) as pbar_audit: diff --git a/simpleaudit/reframing.py b/simpleaudit/reframing.py index 995ab2d..aab8c75 100644 --- a/simpleaudit/reframing.py +++ b/simpleaudit/reframing.py @@ -78,6 +78,7 @@ from simpleaudit._event_loop import run_sync from simpleaudit.judges import get_judge from simpleaudit.judges.compose import inherited_postprocess +from simpleaudit.failures import build_failure from simpleaudit.provider_errors import coerce_judgment_fields from simpleaudit.model_auditor import ModelAuditor, _keep_grading_metadata from simpleaudit.repeated_results import aggregate_severities @@ -931,7 +932,11 @@ async def _grade( raise except Exception as exc: failed = _error_judgment(f"Judge call failed: {describe_exception(exc)}") - failed.update(error=describe_exception(exc), error_phase="judge") + failed.update( + error=describe_exception(exc), + error_phase="judge", + failure=build_failure("judge", exc), + ) _keep_grading_metadata(failed, scenario_meta) return failed, 0, 0 if fell_back and isinstance(judgment, dict): diff --git a/simpleaudit/single_turn.py b/simpleaudit/single_turn.py index e22fbf5..df6ac07 100644 --- a/simpleaudit/single_turn.py +++ b/simpleaudit/single_turn.py @@ -61,6 +61,7 @@ _stored_file_uri, _stored_tool_events, ) +from .failures import build_failure from .provider_errors import EmptyResponseError, _content_to_text, coerce_judgment_fields from .results import AuditResult, AuditResults from .targets.base import TargetContext, token_count @@ -450,6 +451,8 @@ async def _run_one_scenario( # result rather than an exception that discards the whole batch. error: Optional[str] = None error_phase: Optional[str] = None + failure: Optional[Dict[str, Any]] = None + current_role = "system" # What every judge hook gets, as on ModelAuditor (not the raw scenario). scenario_meta: Dict[str, Any] = { "severity": scenario.get("severity"), @@ -471,6 +474,7 @@ async def _run_one_scenario( # Through self.target, not target_client: an explicit Target (an # HTTP app, a callable) replaces the model client, and the context # carries the traceparent an instrumented target propagates. + current_role = "target" target_resp = await self.target.send( system=self.system_prompt, user=test_prompt, @@ -519,6 +523,7 @@ async def _run_one_scenario( except Exception as exc: error = describe_exception(exc) error_phase = "conversation" + failure = self._failure_for(current_role, exc, turn=1) self._log(f"--- Scenario FAILED on the target call: {name} [{error}] ---") if error is None: @@ -706,6 +711,7 @@ async def _run_one_scenario( except Exception as exc: error = describe_exception(exc) error_phase = "judge" + failure = self._failure_for("judge", exc) self._log(f"--- Judging FAILED: {name} [{error}] ---") if error is not None: @@ -718,6 +724,7 @@ async def _run_one_scenario( ], "error": error, "error_phase": error_phase, + "failure": failure, } else: judgment = coerce_judgment_fields(judgment) @@ -847,7 +854,11 @@ async def _run_one(scenario: Dict) -> AuditResult: summary=f"Scenario did not complete due to an error: {describe_exception(exc)}"[:500], recommendations=["Re-run this scenario; check API credentials, rate limits, and connectivity."], expected_behavior=scenario.get("expected_behavior"), - judgment={"severity": "ERROR", "error": describe_exception(exc)}, + judgment={ + "severity": "ERROR", + "error": describe_exception(exc), + "failure": build_failure("system", exc), + }, ) # Ticked here rather than at the two call sites so the ERROR # path advances the bars too and they never finish short. diff --git a/tests/snapshots/core/judge_error.json b/tests/snapshots/core/judge_error.json index 116e056..41825c4 100644 --- a/tests/snapshots/core/judge_error.json +++ b/tests/snapshots/core/judge_error.json @@ -161,7 +161,14 @@ ], "error": "RuntimeError: judge down", "traceback": "", - "error_phase": "judge" + "error_phase": "judge", + "failure": { + "role": "judge", + "kind": "unknown", + "message": "RuntimeError: judge down", + "exception": "RuntimeError", + "model": "fake-judge" + } }, "auditor_input_tokens": 0, "auditor_output_tokens": 0, diff --git a/tests/snapshots/core/target_error.json b/tests/snapshots/core/target_error.json index f14658d..8b11e0b 100644 --- a/tests/snapshots/core/target_error.json +++ b/tests/snapshots/core/target_error.json @@ -93,7 +93,15 @@ ], "error": "RuntimeError: target down", "traceback": "", - "error_phase": "conversation" + "error_phase": "conversation", + "failure": { + "role": "target", + "kind": "unknown", + "message": "RuntimeError: target down", + "exception": "RuntimeError", + "model": "fake-model", + "turn": 2 + } }, "auditor_input_tokens": 0, "auditor_output_tokens": 0, diff --git a/tests/snapshots/core/validate_scenario_failure.json b/tests/snapshots/core/validate_scenario_failure.json index 07006cf..fbad976 100644 --- a/tests/snapshots/core/validate_scenario_failure.json +++ b/tests/snapshots/core/validate_scenario_failure.json @@ -30,7 +30,13 @@ ], "error": "ValueError: scenario needs metadata.reference", "traceback": "", - "error_phase": "conversation" + "error_phase": "conversation", + "failure": { + "role": "system", + "kind": "unknown", + "message": "ValueError: scenario needs metadata.reference", + "exception": "ValueError" + } }, "auditor_input_tokens": 0, "auditor_output_tokens": 0, diff --git a/tests/test_failure_attribution.py b/tests/test_failure_attribution.py new file mode 100644 index 0000000..c3bbba7 --- /dev/null +++ b/tests/test_failure_attribution.py @@ -0,0 +1,262 @@ +"""judgment.failure: which role failed, and how (additive; see simpleaudit/failures.py).""" + +import asyncio +import json + +import pytest + +from simpleaudit.failures import build_failure +from simpleaudit.targets import CallableTarget +from simpleaudit.single_turn import SingleTurnAuditor +from tests.fakes import FakeClient, make_auditor + + +class RateLimitError(Exception): + status_code = 429 + + +class InternalServerError(Exception): + status_code = 503 + + +class UnauthorizedError(Exception): + def __init__(self): + super().__init__("bad key") + self.response = type("R", (), {"status_code": 401})() + + +CASES = [ + (lambda: RateLimitError("slow down"), 429, "rate_limit", "RateLimitError"), + ( + lambda: InternalServerError("provider unavailable"), + 503, + "unavailable", + "InternalServerError", + ), + (lambda: TimeoutError("timed out"), None, "timeout", "TimeoutError"), + (lambda: ConnectionError("refused"), None, "unavailable", "ConnectionError"), +] +IDS = ["429", "5xx", "timeout", "connection"] + +OK = FakeClient(lambda **kw: json.dumps({"severity": "pass", "issues_found": [], "summary": "ok"})) + + +def _boom(make): + def fn(**kw): + raise make() + + return FakeClient(fn) + + +def _run(auditor, **kw): + return asyncio.run(auditor.run_scenario(name="S", description="d", **kw)) + + +def _assert_failure(result, role, status, kind, exc_name, phase): + j = result.judgment + f = j["failure"] + assert f["role"] == role + assert f["kind"] == kind + assert f["exception"] == exc_name + assert f["message"] == j["error"] + assert f.get("http_status") == status + assert j["error_phase"] == phase + assert result.severity == "ERROR" + return f + + +@pytest.mark.parametrize("make,status,kind,name", CASES, ids=IDS) +def test_target_failure(make, status, kind, name): + def fn(**kw): + raise make() + + a = make_auditor(target=OK, judge=OK, auditor=OK, max_turns=2) + a.set_target(CallableTarget(fn)) + f = _assert_failure(_run(a), "target", status, kind, name, "conversation") + assert f["turn"] == 1 + + +@pytest.mark.parametrize("make,status,kind,name", CASES, ids=IDS) +def test_auditor_failure(make, status, kind, name): + a = make_auditor(target=OK, judge=OK, auditor=_boom(make), max_turns=2) + f = _assert_failure(_run(a), "auditor", status, kind, name, "conversation") + assert f["turn"] == 1 + assert f["model"] == "fake-judge" # the configured auditor model + + +@pytest.mark.parametrize("make,status,kind,name", CASES, ids=IDS) +def test_judge_failure(make, status, kind, name): + a = make_auditor(target=OK, judge=_boom(make), auditor=OK, max_turns=1) + f = _assert_failure(_run(a), "judge", status, kind, name, "judge") + assert "turn" not in f + + +def test_auditor_failure_on_later_turn_is_turn_two(): + calls = {"n": 0} + + def fn(**kw): + calls["n"] += 1 + if calls["n"] > 1: + raise RateLimitError("429") + return "next probe" + + a = make_auditor(target=OK, judge=OK, auditor=FakeClient(fn), max_turns=3) + f = _assert_failure(_run(a), "auditor", 429, "rate_limit", "RateLimitError", "conversation") + assert f["turn"] == 2 + + +def test_target_failure_on_later_turn(): + calls = {"n": 0} + + def fn(**kw): + calls["n"] += 1 + if calls["n"] > 1: + raise InternalServerError("503") + return "answer" + + a = make_auditor(target=OK, judge=OK, auditor=OK, max_turns=3) + a.set_target(CallableTarget(fn)) + f = _assert_failure( + _run(a), "target", 503, "unavailable", "InternalServerError", "conversation" + ) + assert f["turn"] == 2 + + +def _single(st): + scenario = {"name": "S", "description": "d", "test_prompt": "hi"} + return asyncio.run(st.run_async([scenario])).results[0] + + +@pytest.mark.parametrize("make,status,kind,name", CASES, ids=IDS) +def test_single_turn_roles(make, status, kind, name): + from unittest.mock import MagicMock, patch + + def build(judge): + with patch.object(SingleTurnAuditor, "_create_anyllm_client", return_value=MagicMock()): + st = SingleTurnAuditor( + model="m", + provider="openai", + judge_model="j", + judge_provider="openai", + max_retries=0, + show_progress=False, + ) + st.judge_client = judge + return st + + def fn(**kw): + raise make() + + st = build(OK) + st.set_target(CallableTarget(fn)) + res = _single(st) + f = _assert_failure(res, "target", status, kind, name, "conversation") + assert f["turn"] == 1 + + st = build(_boom(make)) + st.set_target(CallableTarget(lambda **kw: "fine")) + res = _single(st) + _assert_failure(res, "judge", status, kind, name, "judge") + + +def test_existing_fields_unchanged(): + a = make_auditor(target=OK, judge=OK, auditor=OK, max_turns=1) + a.set_target(CallableTarget(lambda **kw: (_ for _ in ()).throw(RateLimitError("x")))) + j = _run(a).judgment + assert j["error"] == "RateLimitError: x" + assert j["error_phase"] == "conversation" + assert j["severity"] == "ERROR" + assert {"issues_found", "summary", "recommendations", "traceback"} <= set(j) + json.dumps(j) # still serialisable + + +def test_success_has_no_failure(): + a = make_auditor(target=OK, judge=OK, auditor=OK, max_turns=1) + assert "failure" not in _run(a).judgment + + +def test_error_like_answer_without_exception_is_not_flagged(): + a = make_auditor(target=OK, judge=OK, auditor=OK, max_turns=1) + a.set_target( + CallableTarget(lambda **kw: "APIError: The language model provider is unavailable (429)") + ) + res = _run(a) + assert res.severity != "ERROR" + assert "failure" not in res.judgment + + +def test_judge_cannot_forge_failure_key(): + forged = FakeClient( + lambda **kw: json.dumps( + {"severity": "pass", "issues_found": [], "summary": "ok", "failure": {"role": "target"}} + ) + ) + a = make_auditor(target=OK, judge=forged, auditor=OK, max_turns=1) + assert "failure" not in _run(a).judgment + + +def test_unknown_stays_unknown_and_optional_fields_omitted(): + f = build_failure("target", ValueError("odd")) + assert f == { + "role": "target", + "kind": "unknown", + "message": "ValueError: odd", + "exception": "ValueError", + } + + +def test_status_from_cause_chain_and_endpoint_redacted(): + inner = RateLimitError("429") + inner.request = type( + "Rq", (), {"url": "https://user:pw@host.example/v1/chat?api_key=abc&x=1"} + )() + try: + try: + raise inner + except RateLimitError as e: + raise RuntimeError("wrapped") from e + except RuntimeError as outer: + f = build_failure("auditor", outer, turn=2) + assert f["http_status"] == 429 and f["kind"] == "rate_limit" and f["turn"] == 2 + assert "pw" not in f["endpoint"] and "user" not in f["endpoint"] and "abc" not in f["endpoint"] + assert f["endpoint"].startswith("https://host.example/v1/chat") + + +def test_unknown_role_becomes_system_and_http_kinds(): + assert build_failure("bogus", ValueError("x"))["role"] == "system" + for status, kind in ( + (401, "auth"), + (403, "auth"), + (504, "timeout"), + (500, "unavailable"), + (404, "config"), + ): + e = Exception("x") + e.status_code = status + assert build_failure("target", e)["kind"] == kind + + +def test_experiment_reps_carry_failure(): + from unittest.mock import MagicMock, patch + from simpleaudit.model_auditor import ModelAuditor + + async def go(): + with patch.object(ModelAuditor, "_create_anyllm_client", return_value=MagicMock()): + a = ModelAuditor( + model="m", + provider="openai", + judge_model="j", + judge_provider="openai", + max_retries=0, + show_progress=False, + max_turns=1, + ) + a.judge_client = OK + a.auditor_client = OK + a.set_target(CallableTarget(lambda **kw: (_ for _ in ()).throw(RateLimitError("429")))) + return await a.run_async([{"name": "S", "description": "d"}] * 2) + + res = asyncio.run(go()) + assert len(res) == 2 + for r in res: + assert r.judgment["failure"]["role"] == "target"