diff --git a/skillscope/cli.py b/skillscope/cli.py index bc9c406..423c9c7 100644 --- a/skillscope/cli.py +++ b/skillscope/cli.py @@ -84,7 +84,7 @@ ) from . import selection as select_module from . import harness as harnesses -from .agent import check_api_reachable +from .agent import _safe_print, check_api_reachable # The engines a graded run can be driven by. # claude-code -- the real CLI in a sandbox, via inspect_swe (Linux only) @@ -221,8 +221,8 @@ def _write_report(summary: dict, report: str, args: argparse.Namespace, label: s output.write_text(json.dumps(summary, indent=2), encoding="utf-8") print() - print(report) - print(f"[evals] JSON report: {output}") + _safe_print(report) + _safe_print(f"[evals] JSON report: {output}") summary_path = args.summary or os.environ.get("GITHUB_STEP_SUMMARY", "") if summary_path: diff --git a/skillscope/engine/behavioral.py b/skillscope/engine/behavioral.py index 06ca60f..e3c9026 100644 --- a/skillscope/engine/behavioral.py +++ b/skillscope/engine/behavioral.py @@ -311,7 +311,7 @@ def run( outcomes.extend(_failed(skill, skill_cases, deadline.active().message())) continue - print(f"[behavioral] {skill}: {len(skill_cases)} case(s)", flush=True) + agent._safe_print(f"[behavioral] {skill}: {len(skill_cases)} case(s)") try: logs = run_eval( inspect_eval, @@ -354,10 +354,9 @@ def run( for outcome in outcomes: passed = sum(1 for c in outcome.checks if c["passed"]) - print( + agent._safe_print( f" [{'PASS' if outcome.passed else 'FAIL'}] {outcome.id}: " f"{passed}/{len(outcome.checks)} checks in {outcome.elapsed_s}s" + (f" -- {outcome.error}" if outcome.error else ""), - flush=True, ) return outcomes diff --git a/tests/test_skillscope.py b/tests/test_skillscope.py index e8fe780..b8f590b 100644 --- a/tests/test_skillscope.py +++ b/tests/test_skillscope.py @@ -3682,6 +3682,37 @@ def summary(self, **meta): } return behavior.summarize([], {**base, **meta}) + def test_case_error_survives_a_cp1252_console(self) -> None: + error = "runner returned an invalid character: \ufffd" + cases = [ + datasets.Case(name, "check", "alpha", True, files_exist=["out"]) + for name in ("first", "second") + ] + with tempfile.TemporaryDirectory() as tmp: + output = Path(tmp) / "result.json" + summary = Path(tmp) / "summary.md" + with ( + io.TextIOWrapper(io.BytesIO(), encoding="cp1252") as console, + contextlib.redirect_stdout(console), + mock.patch.object(cli, "_prepare_graded_run", return_value=["alpha"]), + mock.patch.object(datasets, "load_dataset", return_value=cases), + mock.patch.object(engine_no_sandbox, "require_local"), + mock.patch.object(engine_sandbox, "require_provider"), + mock.patch.object(engine_behavioral, "build_task", side_effect=SystemExit(error)), + ): + code = cli.main([ + "behavioral", "--engine", "claude-code-no-sandbox", "--model", "mockllm/model", + "--output", str(output), "--summary", str(summary), + ]) + console.flush() + text = console.buffer.getvalue().decode("cp1252") + self.assertEqual(code, 1) + for case in cases: + self.assertIn(f"[FAIL] {case.id}", text) + self.assertEqual([case["error"] for case in json.loads(output.read_text())["cases"]], + [error, error]) + self.assertIn(error, summary.read_text(encoding="utf-8")) + def test_it_renders_for_a_sandboxed_run(self) -> None: out = behavior.render_markdown( self.summary(sandbox="podman", sandbox_isolated=True)