Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
44 changes: 44 additions & 0 deletions .github/workflows/selftest.yml
Original file line number Diff line number Diff line change
Expand Up @@ -119,6 +119,40 @@ jobs:
]
}
EOF
# A skill that declares a prerequisite: `install` builds a file that
# `check` then finds through the env the agent is handed.
mkdir -p fixture/prereq-skill/evals
cat > fixture/prereq-skill/SKILL.md <<'EOF'
---
name: prereq-skill
description: Needs a tool its container does not ship, for a test that provides it.
---
EOF
# Below evals/, so the copy has to create the directory too.
mkdir -p fixture/prereq-skill/evals/tools
cat > fixture/prereq-skill/evals/tools/install.sh <<'EOF'
echo built > "$SKILL_CACHE_DIR/tool.txt"
EOF
cat > fixture/prereq-skill/evals/machine.yml <<'EOF'
prerequisites:
- name: tool
check: [sh, -c, 'test -f "${PREREQ_TOOL:-/nonexistent}"']
install: [sh, tools/install.sh]
env:
PREREQ_TOOL: "${SKILL_CACHE_DIR}/tool.txt"
EOF
cat > fixture/prereq-skill/evals/evals.json <<'EOF'
{
"evaluations": [
{"id": "prereq-a", "skill_should_trigger": true, "prompt": "use the tool",
"files_exist": ["prerequisites-ok.txt"]},
{"id": "prereq-b", "skill_should_trigger": true, "prompt": "another way to ask"},
{"id": "prereq-c", "skill_should_trigger": true, "prompt": "a third phrasing"},
{"id": "prereq-d", "skill_should_trigger": false, "prompt": "something adjacent"},
{"id": "prereq-e", "skill_should_trigger": false, "prompt": "something else entirely"}
]
}
EOF

# No key, no model, no agent: a stub solver stands in for the agent so
# only the machinery beneath it can pass the case. That is what makes
Expand All @@ -133,6 +167,16 @@ jobs:
python tools/sandbox_smoketest.py fixture --skill demo-skill
--expect-sandbox ${{ matrix.os == 'windows-latest' && 'local' || 'docker' }}

# The `claude-code` leg's prerequisite step in a real container: the
# checks, the install, the copied script and its removal, and the env
# reaching the agent factory. Linux only, because only a container has a
# guest for those to happen in.
- name: Provide a skill's prerequisites in the container
if: matrix.os == 'ubuntu-latest'
run: >
python tools/sandbox_smoketest.py fixture --skill prereq-skill
--expect-sandbox docker --prerequisites

action:
name: Action against a throwaway repo
runs-on: ubuntu-latest
Expand Down
30 changes: 26 additions & 4 deletions .github/workflows/skill-evals.yml
Original file line number Diff line number Diff line change
Expand Up @@ -697,7 +697,9 @@ jobs:
# * testable change -> pass iff every leg that ran passed.
#
# A leg held back by a missing gate label is warned about, not failed: a pull
# request should not be blocked by a test it deliberately did not request.
# request should not be blocked by a test it deliberately did not request. A
# skill held back because its machine.yml could not be planned is failed:
# nobody chose not to run it.
results:
name: results
needs: [discover, routing, behavior, behavior-scoped]
Expand All @@ -719,12 +721,17 @@ jobs:
SKIPPED: ${{ needs.discover.outputs.skipped }}
HARNESS: ${{ job.workflow_ref }}
run: |
import json
import os
import sys

def value(name: str) -> str:
return os.environ.get(name, "")

def annotation(text: str) -> str:
# A workflow command ends at the first newline.
return text.replace("%", "%25").replace("\r", "%0D").replace("\n", "%0A")

print(f"harness: {value('HARNESS')}")
print(f"discover: {value('DISCOVER')}")
print(f"routing: {value('ROUTING')} (requested: {value('ROUTING_WANTED')})")
Expand All @@ -740,12 +747,23 @@ jobs:
sys.exit(f"The discover job did not succeed ({value('DISCOVER')}).")

# Say loudly when a change went out without running on real hardware.
skipped = value("SKIPPED").strip()
if skipped and skipped != "[]":
held = json.loads(value("SKIPPED").strip() or "[]")
gated = [item for item in held if "error" not in item]
if gated:
print(
"::warning::These skills' behavioral cases were held back for a "
f"missing gate label and did not run: {skipped}"
f"missing gate label and did not run: {json.dumps(gated)}"
)
# `select` holds back a skill whose machine.yml it could not plan, and
# plans the rest. In this workflow that cannot reach here: discover's
# structural step checks every skill's machine.yml first, more
# strictly than `select`, and a failure there stops the job before
# `select` runs, so the check above has already exited. This is a
# backstop for if the two ever check different skills, so that an
# error entry still fails the run instead of passing silently.
unplanned = [item for item in held if "error" in item]
for item in unplanned:
print(f"::error::{annotation(item['skill'] + ': ' + item['error'])}")

legs = (
("Routing", "ROUTING_WANTED", "ROUTING"),
Expand All @@ -759,6 +777,10 @@ jobs:
for label, _, result in requested
if value(result) != "success"
]
failures += [
f"{item['skill']}'s behavioral cases could not be planned and did not run."
for item in unplanned
]
if failures:
sys.exit("\n".join(failures))

Expand Down
5 changes: 3 additions & 2 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -95,8 +95,9 @@ python -m pip install -e .
python -m unittest discover -s tests -t .
```

Runtime dependencies are PyYAML for frontmatter, `inspect-ai` for the eval
framework both engines are built on, and `anthropic` for the model client.
Runtime dependencies are PyYAML for frontmatter and `evals/machine.yml`,
`inspect-ai` for the eval framework both engines are built on, and `anthropic`
for the model client.
`inspect-ai` became required when the last stdlib-only engine was removed; the
test suite still runs on the standard library alone, and builds a throwaway
repo in a temp directory rather than reading the tree it runs in.
Expand Down
123 changes: 113 additions & 10 deletions docs/authoring-evals.md
Original file line number Diff line number Diff line change
Expand Up @@ -129,8 +129,10 @@ Two optional files sit beside the dataset.

### `evals/machine.yml`

Only for a skill whose behavioral cases cannot run on the everyday runners on
every platform. Both keys are optional:
Only for a skill whose behavioral cases need something the everyday runners
do not give them: a narrower set of platforms, particular hardware, a sandbox of
its own (`sandbox:`, see [usage.md](usage.md#where-a-sandboxed-run-is-sandboxed)),
or [prerequisites](#prerequisites). Every key is optional:

```yaml
os: [Linux] # defaults to whatever platforms the repo runs on
Expand All @@ -146,6 +148,105 @@ moves the leg onto that pool.
Most skills that need this file need only `os: [Linux]`, to drop a Windows leg
that would just exercise the failure path of Linux-only tooling.

#### Prerequisites

A skill whose behavioral cases need something the runner may not have, such as
a library to build or a package to install, declares it here once. Every engine
applies the same declaration where its agent runs. Routing never does: it runs
no case to completion.

```yaml
os: [Linux]
prerequisites:
- name: rocprof-trace-decoder
check: [python3, -c, "import elftools, rocprof_trace_decoder"]
install: [sh, install-decoder.sh] # beside this file
env:
PYTHONPATH: "${SKILL_CACHE_DIR}/decoder/python"
ROCPROF_TRACE_DECODER_LIB: "${SKILL_CACHE_DIR}/decoder/lib/librocprof-trace-decoder.so"
```

- **The order is check, then install, then check again.**
- `check` runs first, without this prerequisite's `env`. If it exits 0, the
prerequisite is already there: nothing is installed and its `env` is not
applied, because whatever provides it already says where it is. Write
`check` so it exits 0 only when the agent would work without this `env`.
The log names which were satisfied by the environment and which were installed.
- Otherwise `install` runs, then `check` runs again with `env` applied. If it
still fails, the skill's cases are reported as failed, with the step's
output.
- Prerequisites run in order, and each one's steps see the `env` of the ones
before it that had to be installed.
- **When it runs depends on the engine.**
- The default engine runs it once per skill, on the host, before the first
case.
- `--engine claude-code` runs it in each case's container, before the agent
starts, so the container needs whatever `install` uses: network access for
a download, which the default sandbox does not have, and a user that can
write to `/opt/skillscope`.
- So the same `check` and `install` may run once or many times, and on a
machine that already has the prerequisite or one that does not. Write
them to give the same result either way.
- **Commands are argv lists.** A command may also be a mapping per platform,
for example `{linux: [sh, x.sh], windows: [pwsh, x.ps1]}`. A container
always counts as `linux`, and the structural check wants a command for every
platform in `os`. An `os` label that is not linux, windows, or macos is
reported too, when this skill has prerequisites: a typo would otherwise skip
that check. On the host, steps run in this directory. In a container
they run in a scratch directory holding only the files they name.
- **What a container gets.** Under `claude-code`, an argv item of `check` or
`install` that is a path to a file in this directory or below it, like
`install-decoder.sh` above, is copied into the container for the step, at the
same relative path, and removed before the agent starts. A file mentioned
only inside a string, as in `sh -c "..."`, is not. A copied file is not
executable there, so run a script through its interpreter (`[sh, install.sh]`,
not `[./install.sh]`). A step that names `evals.json` or `extended_evals.json`
is refused, and the skill's cases fail, rather than carry the answer key into
the agent's room.
- **No step has a time limit of its own.** On the host only `--timeout` stops
a hung install. It stops the processes the install started as well, whether
it stops the run gracefully or kills it outright. In a container the case's
own time limit does, which `--timeout` sets; without one, nothing does. A
host step also waits for everything holding its output open, so a step that
leaves a server running in the background does not finish.
- **Steps see two variables, and `env` values may refer to these two only.**
- `SKILL_SOURCE_DIR` is the source tree the skill came from, resolved the
way `sources.resolve` does (below). It is host only: a container has no
copy of it, and on the host it is unset if no tree can be found. An `env`
value naming it fails before anything is installed when it is missing.
- `SKILL_CACHE_DIR` is scratch space for what `install` builds. On the host
it is removed after the skill's last case. In a container it is
`/opt/skillscope/cache`. That path is provisional until a skill's image
is built against it.
- **An `env` value replaces the variable, it does not extend it.** Any
other `$` in a value is refused. `PATH` cannot be set here: it would
replace the agent's search path, and everything the agent starts would see
only that value. Nor can `ANTHROPIC_*`, `CLAUDE_*`, `COPILOT_*` or
`IS_SANDBOX`, which the agent's harness and model client read for their
own configuration, or `SKILL_SOURCE_DIR` and `SKILL_CACHE_DIR`, which
skillscope sets for the steps itself. Those are the only reserved names.
`GH_TOKEN` and `GITHUB_TOKEN` can be set, for a skill whose own tools need
them, but Copilot logs in with them too: on the host, a value set here
replaces the runner's for that skill's agent. Two prerequisites cannot
set the same variable, and one cannot set two names that differ only in
case. The reserved names and the duplicate rule ignore case, as Windows
does: `path` is refused like `PATH`, and `X` and `x` count as the same
variable. A name is ASCII letters, digits and underscores, not starting
with a digit, and a value holds no NUL.
- **Installing once per image instead of once per case.** Under `claude-code`,
build your `sandbox:` image so that `check` passes in it without this
prerequisite's `env`: run the same `install` at `SKILL_CACHE_DIR=/opt/skillscope/cache`
and set the same `env` in the image, or install wherever the image's own
environment already finds it. A source tree the install needs has to be put
into the image as well.
- **What it sets stays with its skill.** The `env` reaches only its own skill's
agent, and `SKILL_CACHE_DIR` is removed after the skill. Anything `install`
writes elsewhere on the host, such as a pip cache, is not cleaned up.
- **Steps are trusted like the rest of the skill.** On the host they run with
the runner's whole environment, credentials included, as `evals/hooks.py`
does; in a container they see only the container's. A failing step's output
ends up in the report.

### `evals/hooks.py`

Setup a dataset cannot express: cloning a repo, tearing down a container,
Expand Down Expand Up @@ -186,15 +287,17 @@ def setup_session(cache_dir): ... # NOT supported
def check(run, case, ctx): ... # NOT supported
```

`check` was handed the retired engine's `Run` object; the engines here have
inspect's `TaskState`, which is a different thing with different attributes.
`setup_session` returned `{name: value}` pairs substituted into prompts as
`{placeholders}`, and prompts are built before any hook runs -- so honouring it
would mean constructing every case after the hook rather than before. A `setup`
that *returns* placeholders is refused at runtime for the same reason.
`setup_session` is replaced by [prerequisites](#prerequisites). It ran once,
on the host, and anything it set reached the agent only through the host's
environment, so a skill that also ran under `claude-code` had to describe its
environment a second time for the container, and the two descriptions
drifted. Its `{placeholder}` return value has no replacement: nothing used it.
Put such values in the dataset instead.

If you were relying on either, move the work into `setup`/`teardown`, or put
the value in the dataset.
`check` was handed the retired engine's `Run` object. The engines here have
inspect's `TaskState`, which is a different thing with different attributes.
Move what it asserted into the dataset. A `setup` that *returns* placeholders
is refused at runtime, because prompts are built before any hook runs.

Keep prompts and expectations in the dataset even when you use hooks, so what
is being asserted stays readable without opening Python.
Expand Down
4 changes: 3 additions & 1 deletion docs/usage.md
Original file line number Diff line number Diff line change
Expand Up @@ -347,7 +347,9 @@ membership.
`sandbox: compose.yaml` in its `evals/machine.yml`, resolved beside it. Skills get a container with
no network by default; one that installs a server or pulls a model cannot run
that way and says so. Selecting a provider does not discard what a skill asked
for -- the compose file rides along.
for -- the compose file rides along. A skill that needs a tool installed first
declares it as [prerequisites](authoring-evals.md#prerequisites) in the same
file, and an install that downloads anything needs that egress too.

[`examples/skill-with-a-device/evals/`](../examples/skill-with-a-device/evals)
is the pair, worked through: a `machine.yml` that asks for GPU runners and
Expand Down
3 changes: 2 additions & 1 deletion pyproject.toml
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,8 @@ authors = [{ name = "Advanced Micro Devices, Inc." }]
#
# `anthropic` is listed explicitly: inspect-ai treats every model provider as
# optional, so installing it alone gets you a harness that cannot reach a model.
# PyYAML reads the optional evals/machine.yml, which only CI planning touches.
# PyYAML reads the optional evals/machine.yml: CI planning, and a behavioral
# run reading each skill's prerequisites.
#
# `agent-client-protocol` is inspect-ai's own dependency, bounded here because its
# 1.0.0 dropped `acp.helpers`, which inspect-ai imports whenever `eval()` runs
Expand Down
13 changes: 7 additions & 6 deletions skillscope/cli.py
Original file line number Diff line number Diff line change
Expand Up @@ -563,10 +563,11 @@ def _require_hook_support(engine: str, skills: list[str], command: str) -> None:
" They would be skipped silently, and the cases graded as though "
"they had run.\n"
" `setup` and `teardown` are supported. `check` needs the legacy "
"engine's Run object, and `setup_session` returns template variables "
"that are substituted before any hook runs -- neither has an "
"equivalent here.\n"
" Move what they do into the dataset, or into setup/teardown."
"engine's Run object, which has no equivalent here; move what it "
"asserts into the dataset.\n"
" `setup_session` is replaced by `prerequisites:` in "
"evals/machine.yml: a check, an install and the env the agent needs, "
"applied the same way on every engine. See docs/authoring-evals.md."
)


Expand Down Expand Up @@ -826,8 +827,8 @@ def cmd_behavioral(args: argparse.Namespace) -> int:
gradable,
engine_models.resolve(args.model),
args.effort,
solver_factory=lambda d: no_sandbox.harness_solver(
subject, args.model, args.effort, d
solver_factory=no_sandbox.behavioral_solver_factory(
args.model, args.effort, harness=subject
),
)
else:
Expand Down
Loading
Loading