Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 3 additions & 3 deletions skillscope/cli.py
Original file line number Diff line number Diff line change
Expand Up @@ -84,7 +84,7 @@
)
from . import selection as select_module
from . import harness as harnesses
from .agent import check_api_reachable
from .agent import _safe_print, check_api_reachable

# The engines a graded run can be driven by.
# claude-code -- the real CLI in a sandbox, via inspect_swe (Linux only)
Expand Down Expand Up @@ -221,8 +221,8 @@ def _write_report(summary: dict, report: str, args: argparse.Namespace, label: s
output.write_text(json.dumps(summary, indent=2), encoding="utf-8")

print()
print(report)
print(f"[evals] JSON report: {output}")
_safe_print(report)
_safe_print(f"[evals] JSON report: {output}")

summary_path = args.summary or os.environ.get("GITHUB_STEP_SUMMARY", "")
if summary_path:
Expand Down
5 changes: 2 additions & 3 deletions skillscope/engine/behavioral.py
Original file line number Diff line number Diff line change
Expand Up @@ -311,7 +311,7 @@ def run(
outcomes.extend(_failed(skill, skill_cases, deadline.active().message()))
continue

print(f"[behavioral] {skill}: {len(skill_cases)} case(s)", flush=True)
agent._safe_print(f"[behavioral] {skill}: {len(skill_cases)} case(s)")
try:
logs = run_eval(
inspect_eval,
Expand Down Expand Up @@ -354,10 +354,9 @@ def run(

for outcome in outcomes:
passed = sum(1 for c in outcome.checks if c["passed"])
print(
agent._safe_print(
f" [{'PASS' if outcome.passed else 'FAIL'}] {outcome.id}: "
f"{passed}/{len(outcome.checks)} checks in {outcome.elapsed_s}s"
+ (f" -- {outcome.error}" if outcome.error else ""),
flush=True,
)
return outcomes
31 changes: 31 additions & 0 deletions tests/test_skillscope.py
Original file line number Diff line number Diff line change
Expand Up @@ -3682,6 +3682,37 @@ def summary(self, **meta):
}
return behavior.summarize([], {**base, **meta})

def test_case_error_survives_a_cp1252_console(self) -> None:
error = "runner returned an invalid character: \ufffd"
cases = [
datasets.Case(name, "check", "alpha", True, files_exist=["out"])
for name in ("first", "second")
]
with tempfile.TemporaryDirectory() as tmp:
output = Path(tmp) / "result.json"
summary = Path(tmp) / "summary.md"
with (
io.TextIOWrapper(io.BytesIO(), encoding="cp1252") as console,
contextlib.redirect_stdout(console),
mock.patch.object(cli, "_prepare_graded_run", return_value=["alpha"]),
mock.patch.object(datasets, "load_dataset", return_value=cases),
mock.patch.object(engine_no_sandbox, "require_local"),
mock.patch.object(engine_sandbox, "require_provider"),
mock.patch.object(engine_behavioral, "build_task", side_effect=SystemExit(error)),
):
code = cli.main([
"behavioral", "--engine", "claude-code-no-sandbox", "--model", "mockllm/model",
"--output", str(output), "--summary", str(summary),
])
console.flush()
text = console.buffer.getvalue().decode("cp1252")
self.assertEqual(code, 1)
for case in cases:
self.assertIn(f"[FAIL] {case.id}", text)
self.assertEqual([case["error"] for case in json.loads(output.read_text())["cases"]],
[error, error])
self.assertIn(error, summary.read_text(encoding="utf-8"))

def test_it_renders_for_a_sandboxed_run(self) -> None:
out = behavior.render_markdown(
self.summary(sandbox="podman", sandbox_isolated=True)
Expand Down