feat(sdk): run experiments on datasets and send scores

.github/workflows/ci.yml

@@ -16,7 +16,7 @@ jobs:
         with:
           persist-credentials: false
       - run: bash .github/scripts/release_test.sh
-      - run: shellcheck .github/scripts/*.sh
+      - run: shellcheck .github/scripts/*.sh actions/experiment/*.sh
 
   typescript:
     runs-on: ubuntu-latest

actions/experiment/README.md

@@ -0,0 +1,103 @@
+# telemetry.dev experiment action
+
+This composite action runs your eval script in a GitHub Actions workflow. The script calls
+`runExperiment` (`@telemetry-dev/sdk`) or `run_experiment` (`telemetry-dev`). The action writes the
+result to the job summary and to a comment on the pull request. The job fails when the check of the
+pass marks fails, when the experiment does not complete, or when the script exits with an error.
+
+```yaml
+on: pull_request
+
+permissions:
+  contents: read
+  pull-requests: write
+
+jobs:
+  evals:
+    # A pull request from a fork gets no secrets, so the job runs only for branches of this
+    # repository.
+    if: github.event.pull_request.head.repo.full_name == github.repository
+    runs-on: ubuntu-latest
+    env:
+      TELEMETRY_DEV_API_KEY: ${{ secrets.TELEMETRY_DEV_API_KEY }}
+    steps:
+      - uses: actions/checkout@v5
+      - uses: actions/setup-node@v5
+        with:
+          node-version: 24
+      # Installs tsx too, which runs the .ts script.
+      - run: npm ci
+      - uses: telemetry-dev/telemetry/actions/experiment@main
+        with:
+          dataset: support-golden
+          script: evals/reply.eval.ts
+          min-scores: correctness=0.9
+```
+
+`evals/reply.eval.ts` runs the experiment:
+
+```ts
+import { init, runExperiment } from "@telemetry-dev/sdk";
+
+init();
+
+await runExperiment({
+  dataset: "support-golden",
+  task: (item) => replyToCustomer(item.input),
+  evaluators: ["correctness"],
+});
+```
+
+Pin the action to a commit SHA of this repository for a run that does not change when `main`
+changes.
+
+## Inputs
+
+| Input               | Description                                                                                                                                                                                                                                                                                                           |
+| ------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
+| `script`            | The eval script. A `.ts`, `.mts`, `.cts`, or `.tsx` script runs with the `tsx` of the project, from `node_modules/.bin` in the working directory or a directory above it, a `.js`, `.mjs`, or `.cjs` script with `node`, and a `.py` script with the `python` (or `python3`) of the PATH. Give `script` or `command`. |
+| `command`           | A shell command that runs the eval script, for example `uv run python evals/reply_eval.py` or `pnpm eval`.                                                                                                                                                                                                            |
+| `dataset`           | The dataset to run on. The action puts it in `TELEMETRY_DEV_EXPERIMENT_DATASET`. When the script also names a dataset, the two must be the same.                                                                                                                                                                      |
+| `min-scores`        | Pass marks, as `name=value` pairs that commas or new lines separate. A pass rate is 0 to 1, and a mean is on the scale of its score. The action puts them in `TELEMETRY_DEV_EXPERIMENT_MIN_SCORES`, and they win over the `minScores` of the script for the same name.                                                |
+| `api-key`           | A telemetry.dev API key with the `datasets:read` and `experiments:write` scopes, for all environments or for `experiment`. Default: the `TELEMETRY_DEV_API_KEY` variable of the job.                                                                                                                                  |
+| `base-url`          | The telemetry.dev API. Default: `https://api.telemetry.dev`.                                                                                                                                                                                                                                                          |
+| `working-directory` | The directory that the script or the command runs in. Default: `.`.                                                                                                                                                                                                                                                   |
+| `comment`           | Set to `false` to write the job summary only.                                                                                                                                                                                                                                                                         |
+| `github-token`      | The token that writes the comment. Default: `github.token`, which needs the `pull-requests: write` permission.                                                                                                                                                                                                        |
+
+The action sets `TELEMETRY_DEV_ENVIRONMENT` to `experiment` when the job does not set it, so the
+traces of the items stay out of your production pages. Thus the key must be for all environments
+or for `experiment`. A key for another environment, for example your production key, cannot send
+the traces of the items: the experiment then completes with its outputs, but with no traces and no
+cost. `CI` is `true` in GitHub Actions, thus the SDK runs in CI mode: it records the repository,
+the commit, the run, and the pull request with the experiment, and it exits with an error when the
+check fails.
+
+The action does not download packages. Add `tsx` to the `devDependencies` of the project for a
+`.ts` script, or give a `command`.
+
+The outputs, the errors, and the scorer reasons of the items go to telemetry.dev and stay with the
+experiment. The capture options of the SDK and the capture setting of the environment in the
+project do not apply to them.
+
+## Outputs
+
+| Output         | Description                                                           |
+| -------------- | --------------------------------------------------------------------- |
+| `check-status` | `passed`, `failed`, or `none`, for the last experiment of the script. |
+| `url`          | The page of the last experiment in telemetry.dev.                     |
+
+## The comment
+
+The action adds one comment to the pull request for each workflow, job, and `dataset`, `script`, or
+`command`. A later push changes that comment. The action changes only a comment that its token
+wrote: with a copy of the comment by someone else, it adds a new comment. When the token cannot
+write comments, the action logs a warning, and the job summary still has the result.
+
+A pull request from a fork gets no secrets, thus no API key, and the action stops with the error
+"Set the api-key input or the TELEMETRY_DEV_API_KEY variable." The `if` of the example above skips
+the job for those pull requests.
+
+The comment and the summary show the name of each experiment, its status, the count of the items
+that ran, and the value, the change from the baseline, and the pass mark of each score. The SDK
+writes the result for the action to a file in `RUNNER_TEMP` with mode `0600`.

actions/experiment/action.yml

@@ -0,0 +1,82 @@
+name: telemetry.dev experiment
+description: Run an experiment of your code on a telemetry.dev dataset, check its scores, and post the result on the pull request.
+author: telemetry.dev
+branding:
+  icon: check-circle
+  color: gray-dark
+
+inputs:
+  script:
+    description: The eval script, which calls runExperiment or run_experiment. A .ts script runs with the tsx in the node_modules of the project, a .js script with node, and a .py script with python. Give script or command.
+    required: false
+  command:
+    description: A shell command that runs the eval script, for example `uv run python evals/reply_eval.py`. Give script or command.
+    required: false
+  dataset:
+    description: The dataset to run on. When the script also names a dataset, the two must be the same.
+    required: false
+  min-scores:
+    description: Pass marks, as name=value pairs that commas or new lines separate, for example `correctness=0.9`. A pass rate is 0 to 1. They win over the minScores of the script for the same name.
+    required: false
+  api-key:
+    description: A telemetry.dev API key with the datasets:read and experiments:write scopes. Default the TELEMETRY_DEV_API_KEY variable of the job.
+    required: false
+  base-url:
+    description: The telemetry.dev API. Default https://api.telemetry.dev.
+    required: false
+  working-directory:
+    description: The directory that the script or the command runs in.
+    required: false
+    default: "."
+  comment:
+    description: Set to false to write the job summary only, with no comment on the pull request.
+    required: false
+    default: "true"
+  github-token:
+    description: The token that writes the comment. It needs the pull-requests write permission.
+    required: false
+    default: ${{ github.token }}
+
+outputs:
+  check-status:
+    description: passed, failed, or none, for the last experiment of the script.
+    value: ${{ steps.report.outputs.check-status }}
+  url:
+    description: The page of the last experiment in telemetry.dev.
+    value: ${{ steps.report.outputs.url }}
+
+runs:
+  using: composite
+  steps:
+    - id: run
+      shell: bash
+      working-directory: ${{ inputs.working-directory }}
+      env:
+        INPUT_SCRIPT: ${{ inputs.script }}
+        INPUT_COMMAND: ${{ inputs.command }}
+        INPUT_DATASET: ${{ inputs.dataset }}
+        INPUT_MIN_SCORES: ${{ inputs.min-scores }}
+        INPUT_API_KEY: ${{ inputs.api-key }}
+        INPUT_BASE_URL: ${{ inputs.base-url }}
+      run: bash "$GITHUB_ACTION_PATH/run.sh"
+
+    - id: report
+      if: always()
+      shell: bash
+      env:
+        TELEMETRY_DEV_EXPERIMENT_RESULTS: ${{ steps.run.outputs.results }}
+        EXPERIMENT_EXIT_CODE: ${{ steps.run.outputs.exit-code }}
+        INPUT_DATASET: ${{ inputs.dataset }}
+        INPUT_SCRIPT: ${{ inputs.script }}
+        INPUT_COMMAND: ${{ inputs.command }}
+        INPUT_COMMENT: ${{ inputs.comment }}
+        GITHUB_TOKEN: ${{ inputs.github-token }}
+      run: node "$GITHUB_ACTION_PATH/report.mjs"
+
+    - if: always() && steps.run.outputs.exit-code != '0'
+      shell: bash
+      env:
+        EXIT_CODE: ${{ steps.run.outputs.exit-code }}
+      run: |
+        echo "::error title=telemetry.dev experiment::The experiment script exited with code ${EXIT_CODE:-unknown}."
+        exit "${EXIT_CODE:-1}"