diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ed6093f9..d56afe09 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -16,7 +16,7 @@ jobs: with: persist-credentials: false - run: bash .github/scripts/release_test.sh - - run: shellcheck .github/scripts/*.sh + - run: shellcheck .github/scripts/*.sh actions/experiment/*.sh typescript: runs-on: ubuntu-latest diff --git a/actions/experiment/README.md b/actions/experiment/README.md new file mode 100644 index 00000000..431f2c00 --- /dev/null +++ b/actions/experiment/README.md @@ -0,0 +1,103 @@ +# telemetry.dev experiment action + +This composite action runs your eval script in a GitHub Actions workflow. The script calls +`runExperiment` (`@telemetry-dev/sdk`) or `run_experiment` (`telemetry-dev`). The action writes the +result to the job summary and to a comment on the pull request. The job fails when the check of the +pass marks fails, when the experiment does not complete, or when the script exits with an error. + +```yaml +on: pull_request + +permissions: + contents: read + pull-requests: write + +jobs: + evals: + # A pull request from a fork gets no secrets, so the job runs only for branches of this + # repository. + if: github.event.pull_request.head.repo.full_name == github.repository + runs-on: ubuntu-latest + env: + TELEMETRY_DEV_API_KEY: ${{ secrets.TELEMETRY_DEV_API_KEY }} + steps: + - uses: actions/checkout@v5 + - uses: actions/setup-node@v5 + with: + node-version: 24 + # Installs tsx too, which runs the .ts script. + - run: npm ci + - uses: telemetry-dev/telemetry/actions/experiment@main + with: + dataset: support-golden + script: evals/reply.eval.ts + min-scores: correctness=0.9 +``` + +`evals/reply.eval.ts` runs the experiment: + +```ts +import { init, runExperiment } from "@telemetry-dev/sdk"; + +init(); + +await runExperiment({ + dataset: "support-golden", + task: (item) => replyToCustomer(item.input), + evaluators: ["correctness"], +}); +``` + +Pin the action to a commit SHA of this repository for a run that does not change when `main` +changes. + +## Inputs + +| Input | Description | +| ------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `script` | The eval script. A `.ts`, `.mts`, `.cts`, or `.tsx` script runs with the `tsx` of the project, from `node_modules/.bin` in the working directory or a directory above it, a `.js`, `.mjs`, or `.cjs` script with `node`, and a `.py` script with the `python` (or `python3`) of the PATH. Give `script` or `command`. | +| `command` | A shell command that runs the eval script, for example `uv run python evals/reply_eval.py` or `pnpm eval`. | +| `dataset` | The dataset to run on. The action puts it in `TELEMETRY_DEV_EXPERIMENT_DATASET`. When the script also names a dataset, the two must be the same. | +| `min-scores` | Pass marks, as `name=value` pairs that commas or new lines separate. A pass rate is 0 to 1, and a mean is on the scale of its score. The action puts them in `TELEMETRY_DEV_EXPERIMENT_MIN_SCORES`, and they win over the `minScores` of the script for the same name. | +| `api-key` | A telemetry.dev API key with the `datasets:read` and `experiments:write` scopes, for all environments or for `experiment`. Default: the `TELEMETRY_DEV_API_KEY` variable of the job. | +| `base-url` | The telemetry.dev API. Default: `https://api.telemetry.dev`. | +| `working-directory` | The directory that the script or the command runs in. Default: `.`. | +| `comment` | Set to `false` to write the job summary only. | +| `github-token` | The token that writes the comment. Default: `github.token`, which needs the `pull-requests: write` permission. | + +The action sets `TELEMETRY_DEV_ENVIRONMENT` to `experiment` when the job does not set it, so the +traces of the items stay out of your production pages. Thus the key must be for all environments +or for `experiment`. A key for another environment, for example your production key, cannot send +the traces of the items: the experiment then completes with its outputs, but with no traces and no +cost. `CI` is `true` in GitHub Actions, thus the SDK runs in CI mode: it records the repository, +the commit, the run, and the pull request with the experiment, and it exits with an error when the +check fails. + +The action does not download packages. Add `tsx` to the `devDependencies` of the project for a +`.ts` script, or give a `command`. + +The outputs, the errors, and the scorer reasons of the items go to telemetry.dev and stay with the +experiment. The capture options of the SDK and the capture setting of the environment in the +project do not apply to them. + +## Outputs + +| Output | Description | +| -------------- | --------------------------------------------------------------------- | +| `check-status` | `passed`, `failed`, or `none`, for the last experiment of the script. | +| `url` | The page of the last experiment in telemetry.dev. | + +## The comment + +The action adds one comment to the pull request for each workflow, job, and `dataset`, `script`, or +`command`. A later push changes that comment. The action changes only a comment that its token +wrote: with a copy of the comment by someone else, it adds a new comment. When the token cannot +write comments, the action logs a warning, and the job summary still has the result. + +A pull request from a fork gets no secrets, thus no API key, and the action stops with the error +"Set the api-key input or the TELEMETRY_DEV_API_KEY variable." The `if` of the example above skips +the job for those pull requests. + +The comment and the summary show the name of each experiment, its status, the count of the items +that ran, and the value, the change from the baseline, and the pass mark of each score. The SDK +writes the result for the action to a file in `RUNNER_TEMP` with mode `0600`. diff --git a/actions/experiment/action.yml b/actions/experiment/action.yml new file mode 100644 index 00000000..5cc5fa67 --- /dev/null +++ b/actions/experiment/action.yml @@ -0,0 +1,82 @@ +name: telemetry.dev experiment +description: Run an experiment of your code on a telemetry.dev dataset, check its scores, and post the result on the pull request. +author: telemetry.dev +branding: + icon: check-circle + color: gray-dark + +inputs: + script: + description: The eval script, which calls runExperiment or run_experiment. A .ts script runs with the tsx in the node_modules of the project, a .js script with node, and a .py script with python. Give script or command. + required: false + command: + description: A shell command that runs the eval script, for example `uv run python evals/reply_eval.py`. Give script or command. + required: false + dataset: + description: The dataset to run on. When the script also names a dataset, the two must be the same. + required: false + min-scores: + description: Pass marks, as name=value pairs that commas or new lines separate, for example `correctness=0.9`. A pass rate is 0 to 1. They win over the minScores of the script for the same name. + required: false + api-key: + description: A telemetry.dev API key with the datasets:read and experiments:write scopes. Default the TELEMETRY_DEV_API_KEY variable of the job. + required: false + base-url: + description: The telemetry.dev API. Default https://api.telemetry.dev. + required: false + working-directory: + description: The directory that the script or the command runs in. + required: false + default: "." + comment: + description: Set to false to write the job summary only, with no comment on the pull request. + required: false + default: "true" + github-token: + description: The token that writes the comment. It needs the pull-requests write permission. + required: false + default: ${{ github.token }} + +outputs: + check-status: + description: passed, failed, or none, for the last experiment of the script. + value: ${{ steps.report.outputs.check-status }} + url: + description: The page of the last experiment in telemetry.dev. + value: ${{ steps.report.outputs.url }} + +runs: + using: composite + steps: + - id: run + shell: bash + working-directory: ${{ inputs.working-directory }} + env: + INPUT_SCRIPT: ${{ inputs.script }} + INPUT_COMMAND: ${{ inputs.command }} + INPUT_DATASET: ${{ inputs.dataset }} + INPUT_MIN_SCORES: ${{ inputs.min-scores }} + INPUT_API_KEY: ${{ inputs.api-key }} + INPUT_BASE_URL: ${{ inputs.base-url }} + run: bash "$GITHUB_ACTION_PATH/run.sh" + + - id: report + if: always() + shell: bash + env: + TELEMETRY_DEV_EXPERIMENT_RESULTS: ${{ steps.run.outputs.results }} + EXPERIMENT_EXIT_CODE: ${{ steps.run.outputs.exit-code }} + INPUT_DATASET: ${{ inputs.dataset }} + INPUT_SCRIPT: ${{ inputs.script }} + INPUT_COMMAND: ${{ inputs.command }} + INPUT_COMMENT: ${{ inputs.comment }} + GITHUB_TOKEN: ${{ inputs.github-token }} + run: node "$GITHUB_ACTION_PATH/report.mjs" + + - if: always() && steps.run.outputs.exit-code != '0' + shell: bash + env: + EXIT_CODE: ${{ steps.run.outputs.exit-code }} + run: | + echo "::error title=telemetry.dev experiment::The experiment script exited with code ${EXIT_CODE:-unknown}." + exit "${EXIT_CODE:-1}" diff --git a/actions/experiment/report.mjs b/actions/experiment/report.mjs new file mode 100644 index 00000000..ff94492b --- /dev/null +++ b/actions/experiment/report.mjs @@ -0,0 +1,236 @@ +// Writes the job summary of the experiment action, and the comment with the result on the pull +// request. runExperiment and run_experiment add one line of JSON to the results file for each +// experiment: the body of GET /v1/experiments/{id}. +import { createHash } from "node:crypto"; +import { appendFileSync, readFileSync } from "node:fs"; + +const RATE_KINDS = new Set(["pass_fail", "up_down"]); +/** Pages of 100 comments that the action reads to find its comment from an earlier push. */ +const MAX_COMMENT_PAGES = 10; +const STATUS = { completed: "Completed", failed: "Failed", stopped: "Stopped" }; + +const env = process.env; +const exitCode = Number.parseInt(env.EXPERIMENT_EXIT_CODE ?? "", 10); +const results = readResults(env.TELEMETRY_DEV_EXPERIMENT_RESULTS); +// The comment of this job and input, so that a new push changes it and a second step adds its own. +const key = createHash("sha256") + .update( + [ + env.GITHUB_WORKFLOW, + env.GITHUB_JOB, + env.INPUT_DATASET, + env.INPUT_SCRIPT, + env.INPUT_COMMAND, + ].join("\0"), + ) + .digest("hex") + .slice(0, 12); +const marker = ``; +const report = render(results, exitCode); + +if (env.GITHUB_STEP_SUMMARY) appendFileSync(env.GITHUB_STEP_SUMMARY, `${report}\n`); + +const last = results.at(-1); + +if (env.GITHUB_OUTPUT) { + appendFileSync( + env.GITHUB_OUTPUT, + `check-status=${last?.checkStatus ?? "none"}\nurl=${last ? (httpsUrl(last.url) ?? "") : ""}\n`, + ); +} + +if (env.INPUT_COMMENT !== "false") { + await comment(`${marker}\n${report}`).catch((error) => { + // A pull request from a fork gets a token that cannot write comments. + console.log( + `::warning title=telemetry.dev experiment::The comment on the pull request failed: ${error.message}`, + ); + }); +} + +function readResults(path) { + if (!path) return []; + + let text; + + try { + text = readFileSync(path, "utf8"); + } catch { + return []; + } + + return text + .split("\n") + .filter((line) => line.trim() !== "") + .flatMap((line) => { + try { + const result = JSON.parse(line); + + return typeof result === "object" && result !== null && typeof result.name === "string" + ? [result] + : []; + } catch { + return []; + } + }); +} + +function render(experiments, code) { + const lines = ["### telemetry.dev experiment", ""]; + + if (experiments.length === 0) { + lines.push( + `The experiment script exited with code ${Number.isNaN(code) ? "unknown" : code} before it wrote a result. The log of the step has the error.`, + ); + } + + for (const experiment of experiments) { + const url = httpsUrl(experiment.url); + const name = url ? `[${text(experiment.name)}](${url})` : text(experiment.name); + const total = Number(experiment.itemsTotal) || 0; + const failed = Number(experiment.itemsFailed) || 0; + const ran = Math.max((Number(experiment.itemsDone) || 0) - failed, 0); + + lines.push( + `**${name}**: ${outcome(experiment)}. ${ran} of ${total} items ran${failed > 0 ? `, ${failed} failed` : ""}.`, + ); + + if (typeof experiment.error === "string" && experiment.error !== "") { + lines.push("", `> ${text(experiment.error)}`); + } + + const scores = Array.isArray(experiment.scores) ? experiment.scores : []; + + if (scores.length > 0) { + lines.push( + "", + "| Score | Value | Change | Pass mark | Result |", + "| --- | --- | --- | --- | --- |", + ); + + for (const score of scores) { + lines.push( + `| ${text(score.name)} | ${value(score.kind, score.value)} | ${change(score.kind, score.delta)} | ${value(score.kind, score.minScore)} | ${score.passed === true ? "Passed" : score.passed === false ? "Failed" : "—"} |`, + ); + } + } + + lines.push(""); + } + + if (code !== 0 && experiments.length > 0) { + lines.push( + `The experiment script exited with code ${Number.isNaN(code) ? "unknown" : code}.`, + "", + ); + } + + return lines.join("\n").trimEnd(); +} + +function outcome(experiment) { + if (experiment.status !== "completed") + return STATUS[experiment.status] ?? text(String(experiment.status)); + + if (experiment.checkStatus === "failed") return "Check failed"; + + return experiment.checkStatus === "passed" ? "Check passed" : "Completed"; +} + +function value(kind, number) { + if (typeof number !== "number") return "—"; + + return RATE_KINDS.has(kind) + ? `${Math.round(number * 100)}%` + : String(Math.round(number * 100) / 100); +} + +function change(kind, delta) { + if (typeof delta !== "number") return "—"; + + if (delta === 0) return "0"; + + const size = RATE_KINDS.has(kind) ? `${Math.abs(delta)} pts` : String(Math.abs(delta)); + + return `${delta > 0 ? "↑" : "↓"} ${size}`; +} + +/** Text that cannot break the table or add HTML to the comment. */ +function text(value) { + return String(value) + .replace(/[\r\n]+/g, " ") + .replace(/&/g, "&") + .replace(//g, ">") + .replace(/\|/g, "\\|") + .replace(/([[\]()*_`])/g, "\\$1"); +} + +function httpsUrl(url) { + return typeof url === "string" && /^https:\/\/[^\s()<>]+$/.test(url) ? url : undefined; +} + +async function comment(body) { + const pull = pullRequestNumber(); + + if (pull === undefined || !env.GITHUB_TOKEN || !env.GITHUB_REPOSITORY) return; + + const api = (env.GITHUB_API_URL ?? "https://api.github.com").replace(/\/+$/, ""); + const issue = `${api}/repos/${env.GITHUB_REPOSITORY}/issues`; + // Anyone can paste the marker into a comment, so the action changes only a comment that its token + // wrote. A user token knows its login. The GITHUB_TOKEN and app tokens cannot read /user, and + // they write as a bot. + const login = await github("GET", `${api}/user`).then( + (user) => (typeof user?.login === "string" ? user.login : undefined), + () => undefined, + ); + const ours = (item) => + login === undefined ? item.user?.type === "Bot" : item.user?.login === login; + let existing; + + for (let page = 1; page <= MAX_COMMENT_PAGES && existing === undefined; page++) { + const comments = await github("GET", `${issue}/${pull}/comments?per_page=100&page=${page}`); + + existing = comments.find( + (item) => typeof item.body === "string" && item.body.startsWith(marker) && ours(item), + ); + + if (comments.length < 100) break; + } + + if (existing) await github("PATCH", `${issue}/comments/${existing.id}`, { body }); + else await github("POST", `${issue}/${pull}/comments`, { body }); +} + +function pullRequestNumber() { + if (!env.GITHUB_EVENT_PATH) return undefined; + + try { + const number = JSON.parse(readFileSync(env.GITHUB_EVENT_PATH, "utf8")).pull_request?.number; + + return Number.isSafeInteger(number) && number > 0 ? number : undefined; + } catch { + return undefined; + } +} + +async function github(method, url, body) { + const headers = { + accept: "application/vnd.github+json", + authorization: `Bearer ${env.GITHUB_TOKEN}`, + "x-github-api-version": "2022-11-28", + }; + const res = await fetch(url, { + method, + ...(body + ? { headers: { ...headers, "content-type": "application/json" }, body: JSON.stringify(body) } + : { headers }), + redirect: "error", + signal: AbortSignal.timeout(15_000), + }); + + if (!res.ok) + throw new Error(`GitHub returned ${res.status} for ${method} ${new URL(url).pathname}`); + + return res.json(); +} diff --git a/actions/experiment/run.sh b/actions/experiment/run.sh new file mode 100755 index 00000000..86cf7221 --- /dev/null +++ b/actions/experiment/run.sh @@ -0,0 +1,66 @@ +#!/usr/bin/env bash +# Runs the experiment script of the experiment action. A failed script does not fail this step: +# the report step reads the exit code from the output exit-code, and the last step exits with it. +set -uo pipefail + +script=${INPUT_SCRIPT:-} +command=${INPUT_COMMAND:-} + +fail() { + echo "::error title=telemetry.dev experiment::$1" + echo "exit-code=2" >>"$GITHUB_OUTPUT" + exit 0 +} + +if [ -n "$script" ] && [ -n "$command" ]; then fail "Give script or command, not the two."; fi +if [ -z "$script" ] && [ -z "$command" ]; then fail "Give the script or the command that runs the experiment."; fi + +if [ -n "${INPUT_API_KEY:-}" ]; then + echo "::add-mask::$INPUT_API_KEY" + export TELEMETRY_DEV_API_KEY=$INPUT_API_KEY +fi +if [ -z "${TELEMETRY_DEV_API_KEY:-}" ]; then fail "Set the api-key input or the TELEMETRY_DEV_API_KEY variable."; fi +if [ -n "${INPUT_BASE_URL:-}" ]; then export TELEMETRY_DEV_BASE_URL=$INPUT_BASE_URL; fi +if [ -n "${INPUT_DATASET:-}" ]; then export TELEMETRY_DEV_EXPERIMENT_DATASET=$INPUT_DATASET; fi +if [ -n "${INPUT_MIN_SCORES:-}" ]; then export TELEMETRY_DEV_EXPERIMENT_MIN_SCORES=$INPUT_MIN_SCORES; fi +# The traces of the items stay out of production unless the job names an environment. +export TELEMETRY_DEV_ENVIRONMENT=${TELEMETRY_DEV_ENVIRONMENT:-experiment} + +# mktemp makes the file with mode 0600, in a directory of this job. +results=$(mktemp "${RUNNER_TEMP:-/tmp}/telemetry-dev-experiment.XXXXXX") || fail "Could not make the results file." +echo "results=$results" >>"$GITHUB_OUTPUT" +export TELEMETRY_DEV_EXPERIMENT_RESULTS=$results + +# The tsx of the project, from node_modules/.bin here or in a directory above. The action does not +# download one: a package from npm at run time would run with the API key in its environment. +project_tsx() { + dir=$PWD + while :; do + if [ -x "$dir/node_modules/.bin/tsx" ]; then + echo "$dir/node_modules/.bin/tsx" + return 0 + fi + [ "$dir" = / ] && return 1 + dir=$(dirname "$dir") + done +} + +code=0 +if [ -n "$command" ]; then + bash -c "$command" || code=$? +else + case "$script" in + *.ts | *.mts | *.cts | *.tsx) + tsx=$(project_tsx) || fail "No tsx in node_modules. Add tsx to the devDependencies of the project and install them, or give a command." + "$tsx" "$script" || code=$? + ;; + *.js | *.mjs | *.cjs) node "$script" || code=$? ;; + *.py) + python=$(command -v python || command -v python3) || fail "No python on the PATH. Use actions/setup-python, or give a command." + "$python" "$script" || code=$? + ;; + *) fail "The script must end in .ts, .js, or .py. For another runner, give a command." ;; + esac +fi + +echo "exit-code=$code" >>"$GITHUB_OUTPUT" diff --git a/docs/sdk-conformance.md b/docs/sdk-conformance.md index 09583dcd..483a8c43 100644 --- a/docs/sdk-conformance.md +++ b/docs/sdk-conformance.md @@ -44,6 +44,10 @@ repository, not this SDK workspace. Paths below refer to that product repository | `prompt.version` | `td.prompt.version` (int) | omitted for a fallback prompt (`version` null / None) | | `prompt.label` | `td.prompt.label` | only for a fetch by label; a fetch without label or version is the `production` label | | variables of the last `prompt.compile()` before `withPrompt` / `with_prompt` starts, or in its scope | `td.prompt.variables` (JSON object) | mask → JSON stringify → truncate; gated by the effective captureInput of the span | +| `runExperiment` / `run_experiment` → each span and log record of an item run | `td.experiment.id` | the id that `POST /v1/experiments` returns; set like a propagated attribute, so the provider integration spans get it | +| the item of the run | `td.dataset.item_id` | the `id` of the dataset item: its key, the same across dataset versions | +| the item run | `gen_ai.conversation.id = "::"` | one session and one trace for each run; replaces an ambient or process session, and the run drops the active span | +| the root span of an item run | name `experiment `, `gen_ai.operation.name = "function"` | `input` is the item input; `output` is the task result; gated by capture flags | `SpanHandle.serializeContent(value, key)` / `SpanHandle.serialize_content(value, key)` returns what `input` / `output` would emit for `value` under `key`: mask → JSON stringify → truncate. It @@ -178,6 +182,106 @@ cannot tell a concurrent task from an awaited helper, so in TypeScript all spans its last `compile()`; concurrent tasks that compile their own variables call `withPrompt` in each task. +## Experiments and scores + +`runExperiment(options)` / `run_experiment(...)` (and `run_experiment_async`) and `score(options)` / +`score(...)` (and `score_async`) call the Evals routes on `{baseUrl}`, with the API key, the base +URL, and the empty-value rules of getPrompt. The requests have the `Authorization: Bearer ` +header, the SDK header, and a JSON body. Python also sends +`User-Agent: telemetry-dev-python/`, because bot filters refuse the default +`Python-urllib/3.x`. The body is the JSON of `JSON.stringify`: a lone surrogate goes as a +`\uXXXX` escape. The TypeScript SDK calls `fetch` as a plain function, because Workers and +browsers reject a call whose `this` is another object. The redirect rule of getPrompt applies. An +error has the code of the `{ error: { code, message } }` body, or `http_`, or +`missing_api_key`, `timeout`, `network_error`, or `invalid_response`. + +An experiment run sends these requests in this sequence: + +1. `GET /v1/datasets/{name}?version=&split=&cursor=`. The first page names the `datasetVersion` + option, or no version. Each next page names the `version` of the first page and its + `nextCursor`. A page with a different version, a cursor that repeats, a cursor with no items, + or more than 10,000 items in all is `invalid_response`. A first page with no items is + `dataset_empty`, and no experiment starts. +2. `POST /v1/experiments` with `name`, `dataset`, `datasetVersion` (the version of the first + page), `split`, `evaluators`, `minScores`, `runsPerItem`, `ci`, and `metadata`. Keys without a + value are left out. +3. `POST /v1/experiments/{id}/items` with at most 100 item runs and at most 4 MiB of JSON. An item + run has `itemId`, `runIndex`, `output` (a string as it is, other values as `JSON.stringify` JSON, + `null` for no value and for a value that JSON cannot hold, such as an object that holds itself, + cut to 100,000 UTF-16 units), `traceId` (when the span records), `latencyMs`, and, for a task + that failed, `error` (`: `, cut to 4,000 UTF-16 units) or, for a task that + succeeded, `scores`. Python sends a lone surrogate of an output or an error as U+FFFD. Each run + gets its own deep copy of the item, so a task that changes its input does not change the next + run. The posts go one at a time, in order. A post or a page fetch that fails stops the run: no + item starts after it. When the running items end, the SDK sends the request of step 4, sends its + failure to `onError` / `on_error`, and fails with the first error. The server then ends the + experiment with the item runs that it has, or fails it when it has none. Python does the same + after `KeyboardInterrupt` and cancellation. +4. `POST /v1/experiments/{id}/complete`, after `flush()`. +5. `GET /v1/experiments/{id}`, after 1, 2, 4, and then each 5 seconds, until the `status` is + `completed`, `failed`, or `stopped`, for 30 minutes at most (`timeout`). + +Retries: the GET requests, the item posts, and the complete request try 3 times in all after a +timeout, a network error, a 429, or a 5xx. The server skips an item run that it has. The POST that +makes the experiment tries again only after a 429, which the rate limit answers before any work: the +server answers 503 when its database work times out, which can be after the commit. A retry waits +for the `Retry-After` of a 429 or 503 (seconds or an HTTP date), at most 60 seconds, or else 1 +second and then 2 seconds. `POST /v1/scores` makes one request, with no retry. The TypeScript +request timeouts abort the request: 30 seconds for the experiment requests and 10 seconds for a +score. The Python timeouts apply to each socket operation of urllib. + +Options and limits: `runsPerItem` / `runs_per_item` is an integer from 1 to 5 (default 1). +`concurrency` is an integer from 1 to 100 (default 4). `evaluators` has at most 20 trimmed names +of 1 to 64 UTF-16 units. `scorers` has at most 20 scorers with trimmed names of 1 to 64 UTF-16 +units, and no scorer has the name `user_feedback` or the name of an evaluator of the run in a +different case. `minScores` / `min_scores` has at most 20 trimmed names, with finite numbers. A +Python bool is not an integer or a number for these options. Option errors are `TypeError` and +`RangeError` (TypeScript) or `TypeError` and `ValueError` (Python), before any request. + +Scorers run after the span of the item ends, one after the other. A result of `true` or `false` +is `{ value: 1, label: "pass" }` or `{ value: 0, label: "fail" }`. A number is the `value`. A +string is the trimmed `label`. An object or mapping gives `value` (a boolean there is a pass or a +fail), the trimmed `label`, and `reason` (cut to 4,000 UTF-16 units). `null` / `None` gives no +score. A result that the score rules below reject, or a scorer that throws, gives no score and +sends an error with the code `scorer_failed` to `onError` / `on_error`. + +Score rules, for `score` and for scorer results: the trimmed name has 1 to 64 UTF-16 units. A +value is a finite number. A label has 1 to 64 UTF-16 units after the trim. A score has a value, a +label, or the two. `user_feedback` has the value 0 or 1 and no label. The label `pass` goes with no +value or 1, and `fail` with no value or 0. For `score`: a `traceId` is 32 lowercase hex digits and +not all zeros, a `spanId` is 16 lowercase hex digits and not all zeros and needs a `traceId`, a +`sessionId` has 1 to 1,024 UTF-16 units, and a `comment` has at most 4,000. A score needs a +`traceId` or a `sessionId`. A score that breaks a rule is `invalid_score`, with no request. `score` +never throws: it returns the id, or no value after it sends the `ScoreError` to `onError` / +`on_error`. + +The result of step 5 needs `id`, `name`, `status`, `url`, `itemsTotal`, `itemsDone`, `scores`, +and `checkStatus`, else it is `invalid_response`: `checkStatus` decides CI mode, so a missing one +does not read as no check. A field of a score that can be null can also be missing. + +CI mode is on when the `ci` option is true, or with no option when `CI` is `true` or `1`. Then: + +- In GitHub Actions (`GITHUB_ACTIONS=true` and a `GITHUB_REPOSITORY` of `owner/name`, at most 200 + characters), `ci` is `{ provider: "github_actions", repository, commit, prNumber, prTitle, prUrl, +runUrl }`. The commit is the `pull_request.head.sha` of `GITHUB_EVENT_PATH`, else `GITHUB_SHA`, + and must be 7 to 64 hex digits. The title is cut to 256 UTF-16 units, with no half of a + surrogate pair at the end, because Postgres rejects a lone surrogate in jsonb. A URL must be + `https://` with no space, at most 2,048 UTF-16 units. The run URL is + `//actions/runs/`. Keys without a value are left + out. +- After the result, a status other than `completed` is `experiment_failed`, and a `checkStatus` of + `failed` is `check_failed`. The error has the `result`. + +The experiment action sets these variables: `TELEMETRY_DEV_EXPERIMENT_DATASET` is the dataset when +the `dataset` option has no value, and a different `dataset` option is a `TypeError`. +`TELEMETRY_DEV_EXPERIMENT_MIN_SCORES` holds `name=value` pairs that commas or new lines separate. +The value must match `-?\d+(\.\d+)?` (ASCII digits), else it is a `TypeError`. These marks win over +the `minScores` option for the same name. When `TELEMETRY_DEV_EXPERIMENT_RESULTS` names a file, the +SDK adds the JSON of the `GET /v1/experiments/{id}` body to it as one line. + +The SDKs print one line when the experiment starts and the result when it ends. A run with no +enabled client, or in an environment other than `experiment`, logs a warning. + ## The shared scenario Both e2e runners emit exactly this (under diff --git a/packages/sdk/README.md b/packages/sdk/README.md index afd5ef13..0edb8e2e 100644 --- a/packages/sdk/README.md +++ b/packages/sdk/README.md @@ -182,6 +182,120 @@ and TanStack AI integrations, and from other OpenTelemetry tracers, do not get t A model call outside `withPrompt` does not get the attributes, even when it sends messages from `compile()`; with input capture on, the call still records those messages as its input. +## Experiments + +`runExperiment` runs your code on each item of a dataset that you manage in telemetry.dev, and +records the outputs and the scores as an experiment. The experiment page compares it with the +baseline of the dataset. + +```ts +import { init, runExperiment } from "@telemetry-dev/sdk"; + +init({ environment: "experiment" }); + +const result = await runExperiment({ + name: "reply-v13-gpt-4o", + dataset: "support-golden", + task: (item) => replyToCustomer(item.input), + evaluators: ["correctness", "helpfulness"], + scorers: { short: (output) => String(output).length < 600 }, + minScores: { correctness: 0.9 }, +}); +``` + +- runExperiment reads the newest version of the dataset, or `datasetVersion`, 100 items at a time. + With `split`, it runs only the items of that split. Then it makes the experiment, with a name + that telemetry.dev picks when you give no `name`. +- `task(item)` gets the item: `id`, `number`, `input`, `expectedOutput`, `metadata`, `tags`, + `split`, and `runIndex`. Its return value, or the value of its promise, is the output. A string + stays as it is, and a different value becomes JSON, cut to 100,000 characters. A value that JSON + cannot hold, such as an object that holds itself, gives no output. A task that throws fails only + its item, and the item records the name and the message of the error. +- `concurrency` (1 to 100, default 4) items run at the same time. With `runsPerItem` (default 1, + at most 5), each item runs more than one time. Each run gets its own copy of the item, so a task + that changes its input does not change the next run. The pass rate and the mean of each score + use all runs. +- Each item run is a trace of its own, with the span `experiment `. The SDK puts + `td.experiment.id` and `td.dataset.item_id` on each span of the run, also on the spans of the + provider integrations. Each run also gets its own session id, + `::`, as `gen_ai.conversation.id`. Spans from the AI SDK and + TanStack AI integrations do not get these attributes. +- Call `init({ environment: "experiment" })` before the run. Pages, metrics, and alerts leave out + the `experiment` environment by default. In a different environment, runExperiment logs a + warning. Without a client, the items record no traces and no cost. +- `evaluators` names evaluators of the project: LLM judges, code checks, and rules. telemetry.dev + runs them on the outputs. `scorers` run on your machine, after the span of the item ends. A + scorer returns `true` or `false` for a pass or a fail, a number, a label, or an object with + `value`, `label`, and `reason`. A scorer that throws, or returns a score that is not valid, + gives no score for that item and reports a `scorer_failed` error through `onError`. A scorer + cannot have the name of an evaluator of the run, or the name `user_feedback`. +- runExperiment sends the outputs in batches of 100 items at most, and then waits up to 30 + minutes for the server evaluators. Then it prints the scores and returns the result: `status`, + `checkStatus`, item counts, `scores` with the value, the baseline value, the change, and the + pass mark of each score, and the `url` of the experiment page. +- `minScores` sets a pass mark for each score name: the lowest pass rate (0 to 1) of a pass/fail + score, or the lowest mean of a number score. A score under its mark fails the check. +- A failed request rejects with an `ExperimentError` whose `code` tells why, for example + `dataset_not_found`, `dataset_empty`, `evaluator_not_found`, `insufficient_scope`, + `missing_api_key`, or `timeout`. A read of the dataset or the result, a post of outputs, and the + completion go again up to 2 times after a timeout, a network error, a 429, or a 5xx. The server + skips an item run that it has. The request that makes the experiment goes again only after a + 429, thus a lost response or a 503 cannot make a second experiment. When the run stops after it + made the experiment, runExperiment sends the completion before it rejects: telemetry.dev then + ends the experiment with the item runs that it got, or fails it when it got none. +- The key needs the `datasets:read` and `experiments:write` scopes. runExperiment uses the same + API key, base URL, and `fetch` as getPrompt, or its own `apiKey`, `baseUrl`, and `fetch` + options. With `environment: "experiment"`, the key of the telemetry client must be for all + environments or for `experiment`: a key for another environment cannot send the traces of the + items, and the items then have no trace and no cost. + +The outputs, the errors of failed tasks, the scorer reasons, and `metadata` go to telemetry.dev +and stay with the experiment, because they are its result. `captureOutput`, `mask`, and the +capture setting of the environment in the project do not apply to them. They apply only to the +spans of the run. + +### CI mode + +When the `CI` variable is `true` or `1`, or with `ci: true`, runExperiment rejects when the check +fails (`check_failed`) or when the experiment does not complete (`experiment_failed`), so the +process exits with an error. The error has the `result`. In GitHub Actions, the experiment also +records the repository, the commit, the run URL, and the number, the title, and the URL of the pull +request. The commit of a pull request is its head commit. + +The [experiment action](../../actions/experiment) runs your eval script in a workflow, and posts +the result on the pull request. It gives runExperiment the `dataset` input in +`TELEMETRY_DEV_EXPERIMENT_DATASET`, which must agree with the `dataset` of the script, and the +`min-scores` input in `TELEMETRY_DEV_EXPERIMENT_MIN_SCORES`, which wins over `minScores` for the +same name. + +## Scores + +`score` records a score on a trace, a span, or a session, for example the thumbs up or down of a +user on a reply: + +```ts +import { score, startActiveSpan } from "@telemetry-dev/sdk"; + +// When the app replies, keep the trace id of the reply with the message: +const reply = await startActiveSpan("reply", async (span) => ({ + text: await replyToCustomer(message), + traceId: span.traceId, +})); + +// Later, when the user clicks thumbs down: +await score({ name: "user_feedback", traceId: reply.traceId, value: 0, comment: "Wrong amount" }); +``` + +- Give `traceId`, `sessionId`, or the two, and a `spanId` of the trace for a score on one span. + Give `value`, `label`, or the two. `user_feedback` takes the value 1 for up or 0 for down, and + no label. A pass/fail score has the label `pass` with the value 1, or `fail` with 0. +- Like the telemetry functions, score never rejects. It resolves with the id of the score, or with + `undefined` after it reports a `ScoreError` through `onError`. It checks the score before it + sends it (`invalid_score`). The trace id of a span from a client that is not initialized is all + zeros, and score reports it as `invalid_score`. +- score makes one request, with no retry, and waits 10 seconds at most. The key needs the + `scores:write` scope. A browser must not hold the key, so call score from your server. + ## API - `init(options?)` — create the client (isolated OTel TracerProvider by default). @@ -203,6 +317,9 @@ A model call outside `withPrompt` does not get the attributes, even when it send `gen_ai.conversation.id`, and `td.metadata.*` on every span and log record in scope. - `getPrompt(name, options?)` — fetch a prompt by label or version; see [Prompts](#prompts). - `withPrompt(prompt, fn)` — record the prompt on each model call in `fn`. +- `runExperiment(options)` — run your code on a dataset as an experiment; see + [Experiments](#experiments). +- `score(options)` — record a score on a trace, a span, or a session; see [Scores](#scores). - `log(message, { level, eventName, attributes }?)` — log record correlated to the active trace. - `getTraceparent()` — W3C traceparent of the active span (pass to other services; accept it via `startSpan(name, { parent })`). diff --git a/packages/sdk/src/api.ts b/packages/sdk/src/api.ts new file mode 100644 index 00000000..dd7fac66 --- /dev/null +++ b/packages/sdk/src/api.ts @@ -0,0 +1,203 @@ +import { DEFAULT_BASE_URL, resolveEnv } from "@telemetry-dev/otel"; + +import { currentClient } from "./client.ts"; + +/** Where the calls to the telemetry.dev API go, and with which key. */ +export interface ApiOptions { + /** API key. Falls back to the initialized client, then `TELEMETRY_DEV_API_KEY`. */ + apiKey?: string; + /** Falls back to the initialized client, then `TELEMETRY_DEV_BASE_URL`, then `https://api.telemetry.dev`. */ + baseUrl?: string; + /** Falls back to the initialized client's `fetch`, then `globalThis.fetch`. */ + fetch?: typeof fetch; +} + +export interface Api { + apiKey: string | undefined; + baseUrl: string; + sdkName: string; + fetchImpl: typeof fetch; + onError: ((cause: Error) => void) | undefined; +} + +/** An error class with the shape of `PromptError`: a code, a message, and the HTTP status. */ +export type ApiErrorType = new ( + code: string, + message: string, + status?: number, + options?: { cause?: unknown }, +) => E; + +interface RequestOptions { + body?: unknown; + query?: Record; + timeoutMs: number; + /** + * The server takes a repeat of this request at most once, so a retry after a timeout, a network + * error, or a 5xx is safe. Other requests try again only after a 429, which the rate limit + * answers before the server does any work. A 503 can come after the work committed. + */ + idempotent: boolean; + /** Default 3. */ + attempts?: number; +} + +type Attempt = + | { ok: true; body: unknown } + | { ok: false; error: Error; status?: number; retryAfterMs?: number }; + +/** A longer Retry-After waits only this long: the API counts requests by the minute. */ +const MAX_RETRY_AFTER_MS = 60_000; +/** IMF-fixdate, the HTTP date that servers send: `Sun, 06 Nov 1994 08:49:37 GMT`. */ +const HTTP_DATE = /^[A-Z][a-z]{2}, \d{2} [A-Z][a-z]{2} \d{4} \d{2}:\d{2}:\d{2} GMT$/; + +/** The key, base URL, and fetch of a call: its options, then the initialized client, then the env. */ +export function resolveApi(options: ApiOptions): Api { + const config = currentClient().core?.config; + const env = resolveEnv(); + + return { + apiKey: options.apiKey ?? config?.apiKey ?? env.TELEMETRY_DEV_API_KEY, + // An empty value counts as unset, as in the Python SDK. + baseUrl: ( + options.baseUrl || + config?.baseUrl || + env.TELEMETRY_DEV_BASE_URL || + DEFAULT_BASE_URL + ).replace(/\/+$/, ""), + sdkName: config?.sdkName ?? "@telemetry-dev/sdk", + fetchImpl: options.fetch ?? config?.fetchImpl ?? globalThis.fetch, + onError: config?.onError, + }; +} + +/** + * Sends one JSON request to the telemetry.dev API and returns the JSON of the response. A response + * that is not 2xx fails with the `{ error: { code, message } }` of its body, or `http_`. + * A failure that `options.idempotent` allows tries again, after the Retry-After of the response + * or after 1 and then 2 seconds. + */ +export async function request( + api: Api & { apiKey: string }, + ErrorType: ApiErrorType, + method: "GET" | "POST", + path: string, + options: RequestOptions, +): Promise { + const url = new URL(`${api.baseUrl}${path}`); + + for (const [key, value] of Object.entries(options.query ?? {})) { + if (value !== undefined) url.searchParams.set(key, value); + } + + // Outside send(), so that a value JSON cannot hold fails as itself, not as a network error. + const body = options.body === undefined ? undefined : JSON.stringify(options.body); + const attempts = options.attempts ?? 3; + + for (let attempt = 1; ; attempt++) { + const result = await send(api, ErrorType, method, url, body, options.timeoutMs); + + if (result.ok) return result.body; + + if (attempt >= attempts || !retryable(result.status, options.idempotent)) throw result.error; + + await sleep(Math.min(result.retryAfterMs ?? 1000 * attempt, MAX_RETRY_AFTER_MS)); + } +} + +/** The wait that the Retry-After of a 429 or a 503 asks for, in seconds or as an HTTP date. */ +export function retryAfterMs(res: Response): number | undefined { + if (res.status !== 429 && res.status !== 503) return undefined; + + const value = res.headers.get("retry-after")?.trim(); + + if (!value) return undefined; + + let ms: number; + + if (/^\d+$/.test(value)) ms = Number(value) * 1000; + else if (HTTP_DATE.test(value)) ms = Date.parse(value) - Date.now(); + else return undefined; + + return ms > 0 ? ms : undefined; +} + +export function sleep(ms: number): Promise { + return new Promise((resolve) => setTimeout(resolve, ms)); +} + +export function isRecord(value: unknown): value is Record { + return typeof value === "object" && value !== null && !Array.isArray(value); +} + +async function send( + api: Api & { apiKey: string }, + ErrorType: ApiErrorType, + method: "GET" | "POST", + url: URL, + body: string | undefined, + timeoutMs: number, +): Promise { + const controller = new AbortController(); + const timer = setTimeout(() => controller.abort(), timeoutMs); + + // A plain call, not a method call: Workers and browsers reject a fetch whose `this` is not the + // global object. + const { fetchImpl } = api; + + try { + const res = await fetchImpl(url, { + method, + headers: { + authorization: `Bearer ${api.apiKey}`, + "x-telemetry-dev-sdk": api.sdkName, + ...(body === undefined ? {} : { "content-type": "application/json" }), + }, + body, + signal: controller.signal, + // A redirect to another host must not get the API key, so a 3xx fails below like any status + // that is not OK. Cloudflare Workers reject redirect: "error". + redirect: "manual", + }); + + const json: unknown = await res.json().catch(() => undefined); + + if (res.ok) return { ok: true, body: json }; + + const error = isRecord(json) && isRecord(json.error) ? json.error : {}; + + return { + ok: false, + status: res.status, + retryAfterMs: retryAfterMs(res), + error: new ErrorType( + typeof error.code === "string" ? error.code : `http_${res.status}`, + typeof error.message === "string" ? error.message : `telemetry.dev returned ${res.status}`, + res.status, + ), + }; + } catch (error) { + // The timeout can also cut the body, which then reads as an invalid response. + if (controller.signal.aborted) { + return { + ok: false, + error: new ErrorType("timeout", `${method} ${url.pathname} took more than ${timeoutMs} ms`), + }; + } + + return { + ok: false, + error: new ErrorType("network_error", "Could not reach telemetry.dev", undefined, { + cause: error, + }), + }; + } finally { + clearTimeout(timer); + } +} + +function retryable(status: number | undefined, idempotent: boolean): boolean { + if (status === 429) return true; + + return idempotent && (status === undefined || status >= 500); +} diff --git a/packages/sdk/src/ci.ts b/packages/sdk/src/ci.ts new file mode 100644 index 00000000..33f13e99 --- /dev/null +++ b/packages/sdk/src/ci.ts @@ -0,0 +1,172 @@ +import type * as fs from "node:fs"; + +import { reportError } from "@telemetry-dev/otel"; + +/** The pull request and the workflow run of an experiment that runs in GitHub Actions. */ +export interface CiRun { + provider: "github_actions"; + /** `owner/name`. */ + repository: string; + commit: string; + prNumber?: number; + prTitle?: string; + prUrl?: string; + runUrl?: string; +} + +type Env = Record; + +type RuntimeGlobal = typeof globalThis & { + process?: { getBuiltinModule?: (id: string) => unknown }; +}; + +const REPOSITORY = /^[A-Za-z0-9_.-]+\/[A-Za-z0-9_.-]+$/; +const COMMIT = /^[0-9a-f]{7,64}$/i; +const HTTPS_URL = /^https:\/\/\S+$/; +/** A plain decimal, so that the TypeScript and Python SDKs read the same marks. */ +const DECIMAL = /^-?\d+(\.\d+)?$/; +const MAX_REPOSITORY_LENGTH = 200; +const MAX_URL_LENGTH = 2_048; +const MAX_PR_TITLE_LENGTH = 256; + +/** CI mode is on when `CI` is `true` or `1`, as CI services set it. */ +export function inCi(env: Env): boolean { + return env.CI === "true" || env.CI === "1"; +} + +/** + * The run that GitHub Actions describes in its variables, or undefined outside GitHub Actions. On a + * pull request the commit is the head of the pull request: `GITHUB_SHA` is then a merge commit + * that nobody pushed. + */ +export function githubRun(env: Env): CiRun | undefined { + const repository = env.GITHUB_REPOSITORY; + + if ( + env.GITHUB_ACTIONS !== "true" || + !repository || + repository.length > MAX_REPOSITORY_LENGTH || + !REPOSITORY.test(repository) + ) { + return undefined; + } + + const pull = pullRequest(env.GITHUB_EVENT_PATH); + const commit = pull?.commit ?? env.GITHUB_SHA; + + if (!commit || !COMMIT.test(commit)) return undefined; + + const server = (env.GITHUB_SERVER_URL ?? "https://github.com").replace(/\/+$/, ""); + const runId = env.GITHUB_RUN_ID; + + return { + provider: "github_actions", + repository, + commit, + prNumber: pull?.number, + prTitle: pull?.title, + prUrl: pull?.url, + runUrl: + runId && /^\d+$/.test(runId) + ? httpsUrl(`${server}/${repository}/actions/runs/${runId}`) + : undefined, + }; +} + +/** + * The pass marks in `TELEMETRY_DEV_EXPERIMENT_MIN_SCORES`, which the experiment action sets from + * its `min-scores` input: `name=value` pairs, separated by commas or new lines. + */ +export function minScoresFromEnv(env: Env): Record { + // Pairs, so that the name __proto__ becomes a key and not the prototype of the object. + const scores: [string, number][] = []; + + for (const pair of (env.TELEMETRY_DEV_EXPERIMENT_MIN_SCORES ?? "").split(/[,\n]/)) { + if (pair.trim() === "") continue; + const [name, value, ...rest] = pair.split("=").map((part) => part.trim()); + + if (!name || !value || rest.length > 0 || !DECIMAL.test(value)) { + throw new TypeError( + `TELEMETRY_DEV_EXPERIMENT_MIN_SCORES: "${pair.trim()}" is not a name=number pair`, + ); + } + + scores.push([name, Number(value)]); + } + + return Object.fromEntries(scores); +} + +/** + * Adds the result to the file in `TELEMETRY_DEV_EXPERIMENT_RESULTS` as one line of JSON. The + * experiment action reads the file to write the job summary and the comment on the pull request. + */ +export function writeResult(env: Env, result: unknown, onError?: (cause: Error) => void): void { + const path = env.TELEMETRY_DEV_EXPERIMENT_RESULTS; + + if (!path) return; + + try { + const fs = nodeFs(); + + if (!fs) throw new Error("This runtime cannot write TELEMETRY_DEV_EXPERIMENT_RESULTS"); + + fs.appendFileSync(path, `${JSON.stringify(result)}\n`, { mode: 0o600 }); + } catch (error) { + reportError(onError, error); + } +} + +function pullRequest( + eventPath: string | undefined, +): { number?: number; title?: string; url?: string; commit?: string } | undefined { + if (!eventPath) return undefined; + + let event: unknown; + + try { + event = JSON.parse(nodeFs()?.readFileSync(eventPath, "utf8") ?? "null"); + } catch { + return undefined; + } + + const pull = (event as { pull_request?: Record } | null)?.pull_request; + + if (typeof pull !== "object" || pull === null) return undefined; + + const head = pull.head as { sha?: unknown } | undefined; + + return { + number: + Number.isSafeInteger(pull.number) && (pull.number as number) > 0 + ? (pull.number as number) + : undefined, + title: typeof pull.title === "string" ? cut(pull.title, MAX_PR_TITLE_LENGTH) : undefined, + url: typeof pull.html_url === "string" ? httpsUrl(pull.html_url) : undefined, + commit: typeof head?.sha === "string" ? head.sha : undefined, + }; +} + +/** + * `text` cut to `max` UTF-16 units, with no half of a surrogate pair at the end: the API stores the + * run as jsonb, and Postgres rejects a lone surrogate there. + */ +function cut(text: string, max: number): string { + if (text.length <= max) return text; + + const last = text.charCodeAt(max - 1); + + return text.slice(0, last >= 0xd800 && last <= 0xdbff ? max - 1 : max); +} + +function httpsUrl(url: string): string | undefined { + return url.length <= MAX_URL_LENGTH && HTTPS_URL.test(url) ? url : undefined; +} + +// Read at call time, with no static `node:` import, so that the package still loads in runtimes +// without a file system. +function nodeFs(): typeof fs | undefined { + return (globalThis as RuntimeGlobal).process?.getBuiltinModule?.("node:fs") as + | typeof fs + | undefined; +} diff --git a/packages/sdk/src/experiments.ts b/packages/sdk/src/experiments.ts new file mode 100644 index 00000000..9bcc963b --- /dev/null +++ b/packages/sdk/src/experiments.ts @@ -0,0 +1,972 @@ +import { type Context, trace } from "@opentelemetry/api"; +import { + activeContext, + diag, + jsonAttr, + PROPAGATED_KEY, + propagatedFromContext, + reportError, + resolveEnv, +} from "@telemetry-dev/otel"; + +import { type Api, type ApiOptions, isRecord, request, resolveApi, sleep } from "./api.ts"; +import { truncate } from "./capture.ts"; +import { type CiRun, githubRun, inCi, minScoresFromEnv, writeResult } from "./ci.ts"; +import { currentClient, flush } from "./client.ts"; +import { checkValue, MAX_REASON_LENGTH, MAX_SCORE_NAME_LENGTH, USER_FEEDBACK } from "./scores.ts"; +import { type SpanHandle, startActiveSpan } from "./span.ts"; + +/** An item of a dataset version, as `GET /v1/datasets/{name}` returns it. */ +export interface DatasetItem { + /** The key of the item. It stays the same across dataset versions. */ + readonly id: string; + readonly number: number; + readonly input: Readonly>; + readonly expectedOutput: string | null; + readonly metadata: Readonly>; + readonly tags: readonly string[]; + readonly split: string | null; +} + +/** One run of a dataset item. With `runsPerItem` above 1, an item runs more than one time. */ +export interface ExperimentItem extends DatasetItem { + /** 0 to `runsPerItem` - 1. */ + readonly runIndex: number; +} + +/** + * What a scorer returns. `true` and `false` are a pass and a fail, a number is the value, and a + * string is the label. Null or undefined gives no score. + */ +export type ScorerResult = + | boolean + | number + | string + | { value?: number | boolean; label?: string; reason?: string } + | null + | undefined; + +/** Scores the output of an item on your machine. The score has the name of the scorer. */ +export type Scorer = ( + output: unknown, + item: ExperimentItem, +) => ScorerResult | Promise; + +export interface RunExperimentOptions extends ApiOptions { + /** Runs the code under test on one item. Its return value is the output of the item. */ + task: (item: ExperimentItem) => unknown; + /** + * The name of the dataset. Default: the `dataset` input of the experiment action, which it + * sets in `TELEMETRY_DEV_EXPERIMENT_DATASET`. When both are set, they must be the same. + */ + dataset?: string; + /** Unique in the project. telemetry.dev picks a name when it is not set. */ + name?: string; + /** Default: the newest version. */ + datasetVersion?: number; + /** Run only the items of this split. */ + split?: string; + /** Evaluators of the project that score the outputs on the server, by name. */ + evaluators?: readonly string[]; + /** Scorers that run here, by score name. */ + scorers?: Readonly>; + /** + * Pass marks by score name: the lowest pass rate (0 to 1) of a pass/fail score, or the lowest + * mean of a number score. The `min-scores` input of the experiment action adds to them and wins + * for the same name. + */ + minScores?: Readonly>; + /** 1 to 5. Default 1. */ + runsPerItem?: number; + /** Item runs at the same time, 1 to 100. Default 4. */ + concurrency?: number; + metadata?: Readonly>; + /** + * Records the GitHub Actions run, and rejects when the check fails or the experiment does not + * complete. Default: on when the `CI` variable is `true` or `1`. + */ + ci?: boolean; +} + +export interface ExperimentScore { + readonly name: string; + /** `pass_fail`, `number`, `labels`, `a_b_tie`, or `up_down`. Null before the score has values. */ + readonly kind: string | null; + /** The pass rate (0 to 1) of a pass/fail or up/down score, or the mean of a number score. */ + readonly value: number | null; + readonly passCount: number; + readonly failCount: number; + /** The value in the experiment that this one compares with, by default the dataset baseline. */ + readonly baselineValue: number | null; + /** Whole points of a pass rate, or the change of a mean to one decimal. */ + readonly delta: number | null; + readonly minScore: number | null; + /** Null when the score has no pass mark. */ + readonly passed: boolean | null; +} + +export interface ExperimentResult { + readonly id: string; + readonly name: string; + /** `completed`, `failed`, or `stopped`. */ + readonly status: string; + /** Null when the experiment has no pass marks. */ + readonly checkStatus: "passed" | "failed" | null; + readonly itemsTotal: number; + readonly itemsDone: number; + readonly itemsFailed: number; + /** Why the experiment failed. */ + readonly error: string | null; + readonly scores: readonly ExperimentScore[]; + readonly url: string; +} + +/** + * Why runExperiment failed. `code` is the code of the API error (`dataset_not_found`, + * `insufficient_scope`, `invalid_api_key`, …), or `missing_api_key`, `timeout`, `network_error`, + * `invalid_response`, or `dataset_empty`. In CI mode it is also `check_failed` or + * `experiment_failed`, and `result` holds the result. A response with a different status and no + * API error gives `http_`. A scorer that throws or returns a value that is not a score + * reports a `scorer_failed` error through `onError`. + */ +export class ExperimentError extends Error { + override readonly name = "ExperimentError"; + readonly result?: ExperimentResult; + + constructor( + readonly code: string, + message: string, + readonly status?: number, + options?: { cause?: unknown; result?: ExperimentResult }, + ) { + super(message, options); + this.result = options?.result; + } +} + +interface Run { + dataset: string; + task: (item: ExperimentItem) => unknown; + name?: string; + datasetVersion?: number; + split?: string; + evaluators?: readonly string[]; + scorers: [string, Scorer][]; + minScores?: Record; + runsPerItem: number; + concurrency: number; + metadata?: Readonly>; + ci: CiRun | undefined; + inCi: boolean; +} + +interface Created { + id: string; + name: string; + itemsTotal: number; + url: string; +} + +interface DatasetPage { + version: number; + items: DatasetItem[]; + nextCursor: string | null; +} + +interface ItemScore { + name: string; + value?: number; + label?: string; + reason?: string; +} + +interface ItemResult { + itemId: string; + runIndex: number; + output: string | null; + traceId?: string; + latencyMs: number; + error?: string; + scores?: ItemScore[]; +} + +type Authed = Api & { apiKey: string }; + +/** The `experiment` environment, which pages, metrics, and alerts leave out by default. */ +const EXPERIMENT_ENVIRONMENT = "experiment"; +const MAX_RUNS_PER_ITEM = 5; +const MAX_CONCURRENCY = 100; +/** Server evaluators of one experiment, and scorers: an item takes at most 20 scores. */ +const MAX_EVALUATORS = 20; +const MAX_MIN_SCORES = 20; +const MAX_ERROR_LENGTH = 4_000; +const MAX_OUTPUT_LENGTH = 100_000; +const MAX_DATASET_ITEMS = 10_000; +/** Items in one `POST /v1/experiments/{id}/items`. */ +const MAX_BATCH_ITEMS = 100; +/** Bytes of JSON in one post of items, well under the 16 MiB body limit of the API. */ +const MAX_BATCH_BYTES = 4 * 1024 * 1024; +const REQUEST_TIMEOUT_MS = 30_000; +/** How long runExperiment waits for the server evaluators after the last item. */ +const MAX_WAIT_MS = 30 * 60_000; +const MAX_POLL_INTERVAL_MS = 5_000; +const FINISHED = new Set(["completed", "failed", "stopped"]); +const RATE_KINDS = new Set(["pass_fail", "up_down"]); + +/** + * Runs `task` on each item of a dataset and records the result as an experiment in telemetry.dev. + * Each item run is its own trace, with `td.experiment.id` and `td.dataset.item_id` on each span. + * The outputs go to telemetry.dev in batches of at most 100, with the scores of `scorers`. Then + * runExperiment waits for the `evaluators` of the server, prints the result, and returns it. + * + * The run cannot start without its dataset, so runExperiment rejects with an `ExperimentError` + * when a request fails. When the run stops after it made the experiment, it first sends the + * complete request, so that telemetry.dev ends the experiment with the item runs that it has. A + * task that throws fails only its item. In CI mode, runExperiment rejects when the check fails, so + * the process exits with an error. + */ +export async function runExperiment(options: RunExperimentOptions): Promise { + const env = resolveEnv(); + const run = checkOptions(options, env); + const resolved = resolveApi(options); + + if (!resolved.apiKey) { + throw new ExperimentError( + "missing_api_key", + "No API key: pass apiKey or set TELEMETRY_DEV_API_KEY", + ); + } + + const api = { ...resolved, apiKey: resolved.apiKey }; + warnAboutClient(); + + const first = await fetchPage(api, run, run.datasetVersion, undefined); + + if (first.items.length === 0) { + throw new ExperimentError( + "dataset_empty", + `The dataset ${run.dataset} has no items${run.split === undefined ? "" : ` in the split ${run.split}`}`, + ); + } + + const created = parseCreated( + await request(api, ExperimentError, "POST", "/v1/experiments", { + body: { + name: run.name, + dataset: run.dataset, + datasetVersion: first.version, + split: run.split, + evaluators: run.evaluators, + minScores: run.minScores, + runsPerItem: run.runsPerItem, + ci: run.ci, + metadata: run.metadata, + }, + timeoutMs: REQUEST_TIMEOUT_MS, + // A repeat after a lost response would start a second experiment. + idempotent: false, + }), + ); + + try { + console.log( + `Experiment ${created.name}: ${created.itemsTotal / run.runsPerItem} items` + + (run.runsPerItem > 1 ? `, ${run.runsPerItem} runs each` : "") + + `. ${created.url}`, + ); + await runItems(api, run, created, itemRuns(api, run, first)); + } catch (error) { + // Without the complete, the experiment would stay running: the server ends it with the item + // runs that it has, or fails it when it has none. + await complete(api, created.id).catch((cause: unknown) => reportError(api.onError, cause)); + + throw error; + } + + // The spans of the items reach ingest before the server reads their cost. + await flush(); + await complete(api, created.id); + + const { result, body } = await waitForResult(api, created.id); + writeResult(env, body, api.onError); + console.log(describe(result)); + + if (run.inCi) checkResult(result); + + return result; +} + +function checkOptions(options: RunExperimentOptions, env: Record): Run { + if (typeof options.task !== "function") throw new TypeError("task must be a function"); + + for (const [key, value] of [ + ["dataset", options.dataset], + ["name", options.name], + ["split", options.split], + ] as const) { + if (value !== undefined && (typeof value !== "string" || value.length === 0)) { + throw new TypeError(`${key} must be a non-empty string`); + } + } + + const actionDataset = env.TELEMETRY_DEV_EXPERIMENT_DATASET || undefined; + const dataset = options.dataset ?? actionDataset; + + if (dataset === undefined) { + throw new TypeError("Pass dataset, or set the dataset input of the experiment action"); + } + + if (actionDataset !== undefined && actionDataset !== dataset) { + throw new TypeError( + `dataset is ${dataset}, but the experiment action names ${actionDataset}. Name one dataset.`, + ); + } + + const { datasetVersion } = options; + + if ( + datasetVersion !== undefined && + !(Number.isSafeInteger(datasetVersion) && datasetVersion >= 1) + ) { + throw new RangeError("datasetVersion must be a positive integer"); + } + + const evaluators = options.evaluators; + + if ( + evaluators !== undefined && + (!Array.isArray(evaluators) || + evaluators.length > MAX_EVALUATORS || + !evaluators.every((name) => typeof name === "string" && validScoreName(name))) + ) { + throw new TypeError(`evaluators must be a list of at most ${MAX_EVALUATORS} evaluator names`); + } + + const scorers = Object.entries(options.scorers ?? {}); + const evaluatorNames = new Set(evaluators?.map((name) => name.toLowerCase())); + + if (scorers.length > MAX_EVALUATORS) { + throw new TypeError(`scorers can have at most ${MAX_EVALUATORS} scorers`); + } + + for (const [name, scorer] of scorers) { + if (!validScoreName(name) || typeof scorer !== "function") { + throw new TypeError( + `scorers must map score names of 1 to ${MAX_SCORE_NAME_LENGTH} characters to functions`, + ); + } + + if (name === USER_FEEDBACK) { + throw new TypeError( + `${USER_FEEDBACK} is the score of user feedback. Name the scorer another way.`, + ); + } + + if (evaluatorNames.has(name.toLowerCase())) { + throw new TypeError(`The scorer ${name} has the name of an evaluator. Rename one of them.`); + } + } + + const minScores = { ...options.minScores, ...minScoresFromEnv(env) }; + + if ( + Object.keys(minScores).length > MAX_MIN_SCORES || + !Object.entries(minScores).every( + ([name, value]) => validScoreName(name) && Number.isFinite(value), + ) + ) { + throw new TypeError(`minScores must map at most ${MAX_MIN_SCORES} score names to numbers`); + } + + const runsPerItem = options.runsPerItem ?? 1; + + if (!Number.isInteger(runsPerItem) || runsPerItem < 1 || runsPerItem > MAX_RUNS_PER_ITEM) { + throw new RangeError(`runsPerItem must be an integer from 1 to ${MAX_RUNS_PER_ITEM}`); + } + + const concurrency = options.concurrency ?? 4; + + if (!Number.isInteger(concurrency) || concurrency < 1 || concurrency > MAX_CONCURRENCY) { + throw new RangeError(`concurrency must be an integer from 1 to ${MAX_CONCURRENCY}`); + } + + if (options.metadata !== undefined && !isRecord(options.metadata)) { + throw new TypeError("metadata must be an object"); + } + + if (options.ci !== undefined && typeof options.ci !== "boolean") { + throw new TypeError("ci must be a boolean"); + } + + const ci = options.ci ?? inCi(env); + + return { + dataset, + task: options.task, + name: options.name, + datasetVersion, + split: options.split, + evaluators, + scorers, + minScores: Object.keys(minScores).length > 0 ? minScores : undefined, + runsPerItem, + concurrency, + metadata: options.metadata, + ci: ci ? githubRun(env) : undefined, + inCi: ci, + }; +} + +/** A trimmed name of 1 to 64 characters, as the API stores score names. */ +function validScoreName(name: string): boolean { + return name.length > 0 && name.length <= MAX_SCORE_NAME_LENGTH && name.trim() === name; +} + +function warnAboutClient(): void { + const environment = currentClient().core?.config.environment; + + if (environment === undefined) { + diag.warn( + 'runExperiment: no telemetry client, so the items record no traces. Call init({ environment: "experiment" }) first.', + ); + } else if (environment.trim().toLowerCase() !== EXPERIMENT_ENVIRONMENT) { + diag.warn( + `runExperiment: the traces of the items go to the environment ${environment}. Call init({ environment: "experiment" }) to keep them out of it.`, + ); + } +} + +/** + * The runs of each item of the dataset version, page by page, so that only one page is in memory. + * Workers share the generator: an async generator answers each `next()` in turn. + */ +async function* itemRuns( + api: Authed, + run: Run, + first: DatasetPage, +): AsyncGenerator { + let page = first; + let count = 0; + + for (;;) { + count += page.items.length; + + if (count > MAX_DATASET_ITEMS) { + throw new ExperimentError( + "invalid_response", + `telemetry.dev returned more than ${MAX_DATASET_ITEMS} items`, + ); + } + + for (const item of page.items) { + // A copy for each run, so that a task that changes its input does not change the next run. + for (let runIndex = 0; runIndex < run.runsPerItem; runIndex++) { + yield { ...structuredClone(item), runIndex }; + } + } + + if (page.nextCursor === null) return; + + page = await fetchPage(api, run, page.version, page.nextCursor); + } +} + +/** One page of a dataset version. A page after the first names the version of the first. */ +async function fetchPage( + api: Authed, + run: Run, + version: number | undefined, + cursor: string | undefined, +): Promise { + const page = parseDatasetPage( + await request(api, ExperimentError, "GET", `/v1/datasets/${encodeURIComponent(run.dataset)}`, { + query: { version: version?.toString(), split: run.split, cursor }, + timeoutMs: REQUEST_TIMEOUT_MS, + idempotent: true, + }), + ); + + if ( + !page || + (version !== undefined && page.version !== version) || + (page.nextCursor !== null && (page.items.length === 0 || page.nextCursor === cursor)) + ) { + throw new ExperimentError( + "invalid_response", + "telemetry.dev returned a dataset it cannot read", + ); + } + + return page; +} + +/** + * Runs the items, `concurrency` at a time, and posts each result. A post or a page fetch that + * fails stops the run: no new item starts, and runItems rejects when the running items end. + */ +async function runItems( + api: Authed, + run: Run, + experiment: Created, + runs: AsyncGenerator, +): Promise { + const batch = itemBatch((results) => + request( + api, + ExperimentError, + "POST", + `/v1/experiments/${encodeURIComponent(experiment.id)}/items`, + // The server skips an item run that it has, so a repeat is safe. + { body: { items: results }, timeoutMs: REQUEST_TIMEOUT_MS, idempotent: true }, + ), + ); + + let failure: { error: unknown } | undefined; + + const worker = async () => { + while (failure === undefined) { + const next = await runs.next().catch((error: unknown) => { + failure ??= { error }; + + return { done: true as const, value: undefined }; + }); + + if (next.done) return; + + await runItem(run, experiment, next.value, api.onError) + .then((result) => batch.add(result)) + .catch((error: unknown) => { + failure ??= { error }; + }); + } + }; + + await Promise.all(Array.from({ length: run.concurrency }, worker)); + + if (failure === undefined) { + await batch.flush().catch((error: unknown) => { + failure = { error }; + }); + } + + if (failure !== undefined) { + await runs.return(undefined); + + throw failure.error; + } +} + +/** Collects item results and posts them in order, one post at a time. */ +function itemBatch(post: (results: ItemResult[]) => Promise) { + let pending: ItemResult[] = []; + let bytes = 0; + let queue: Promise = Promise.resolve(); + + const send = async () => { + const results = pending; + pending = []; + bytes = 0; + await post(results); + }; + + return { + add(result: ItemResult): Promise { + const size = new TextEncoder().encode(JSON.stringify(result)).length; + + queue = queue.then(async () => { + if (pending.length > 0 && bytes + size > MAX_BATCH_BYTES) await send(); + pending.push(result); + bytes += size; + + if (pending.length >= MAX_BATCH_ITEMS) await send(); + }); + + return queue; + }, + flush(): Promise { + queue = queue.then(() => (pending.length > 0 ? send() : undefined)); + + return queue; + }, + }; +} + +async function runItem( + run: Run, + experiment: Created, + item: ExperimentItem, + onError: ((cause: Error) => void) | undefined, +): Promise { + const started = performance.now(); + + const outcome = await startActiveSpan( + `experiment ${experiment.name}`, + { parent: itemContext(experiment.id, item), input: item.input }, + async (span): Promise<{ output?: unknown; error?: unknown; traceId?: string }> => { + try { + const output: unknown = await run.task(item); + span.update({ output }); + + return { output, traceId: traceIdOf(span) }; + } catch (error) { + span.update({ error }); + + return { + error: error ?? new Error("The task threw a nullish value"), + traceId: traceIdOf(span), + }; + } + }, + ); + + const base = { + itemId: item.id, + runIndex: item.runIndex, + traceId: outcome.traceId, + latencyMs: Math.round(performance.now() - started), + }; + + if ("error" in outcome) { + return { ...base, output: null, error: truncate(errorText(outcome.error), MAX_ERROR_LENGTH) }; + } + + const scores = await runScorers(run, outcome.output, item, onError); + + return { + ...base, + output: outputText(outcome.output), + scores: scores.length > 0 ? scores : undefined, + }; +} + +/** + * The context of one item run. It drops the active span, so that the run starts a trace of its + * own, and gives the run a session id of its own: an ambient session or `sessionMode: "process"` + * would otherwise put all runs in one session trace. + */ +function itemContext(experimentId: string, item: ExperimentItem): Context { + const base = trace.deleteSpan(activeContext()); + + return base.setValue(PROPAGATED_KEY, { + ...propagatedFromContext(base), + "td.experiment.id": experimentId, + "td.dataset.item_id": item.id, + "gen_ai.conversation.id": `${experimentId}:${item.id}:${item.runIndex}`, + }); +} + +function traceIdOf(span: SpanHandle): string | undefined { + return span.isRecording ? span.traceId : undefined; +} + +async function runScorers( + run: Run, + output: unknown, + item: ExperimentItem, + onError: ((cause: Error) => void) | undefined, +): Promise { + const scores: ItemScore[] = []; + + // After the span of the item ends, so that the model calls of a scorer do not count in the + // cost of the item. + for (const [name, scorer] of run.scorers) { + try { + const score = toScore(name, await scorer(output, item)); + + if (score) scores.push(score); + } catch (error) { + reportError( + onError, + new ExperimentError( + "scorer_failed", + `The scorer ${name} failed on item ${item.number}: ${errorText(error)}`, + undefined, + { cause: error }, + ), + ); + } + } + + return scores; +} + +/** The score of a scorer result. It throws for a result that the server would reject. */ +function toScore(name: string, result: ScorerResult): ItemScore | undefined { + if (result === null || result === undefined) return undefined; + + let score: ItemScore; + + if (typeof result === "boolean") score = { name, ...passFail(result) }; + else if (typeof result === "number") score = { name, value: result }; + else if (typeof result === "string") score = { name, label: result.trim() }; + else if (isRecord(result)) { + const { value, label, reason } = result as Record; + + for (const [key, field] of [ + ["label", label], + ["reason", reason], + ] as const) { + if (field !== undefined && typeof field !== "string") { + throw new TypeError(`returned a ${key} that is not a string`); + } + } + + score = { + name, + ...(typeof value === "boolean" ? passFail(value) : { value: value as number | undefined }), + ...(label === undefined ? {} : { label: (label as string).trim() }), + reason: reason === undefined ? undefined : truncate(reason as string, MAX_REASON_LENGTH), + }; + } else { + throw new TypeError(`returned a ${typeof result}`); + } + + const problem = checkValue(name, score.value, score.label); + + if (problem) throw new TypeError(`returned a score that is not valid: ${problem}`); + + return score; +} + +function passFail(pass: boolean): { value: number; label: string } { + return pass ? { value: 1, label: "pass" } : { value: 0, label: "fail" }; +} + +function outputText(output: unknown): string | null { + const text = output === null || output === undefined ? undefined : jsonAttr(output); + + return text === undefined ? null : truncate(text, MAX_OUTPUT_LENGTH); +} + +function errorText(error: unknown): string { + try { + return error instanceof Error ? `${error.name}: ${error.message}` : String(error); + } catch { + return "A value that cannot be converted to a string was thrown"; + } +} + +function complete(api: Authed, id: string): Promise { + return request( + api, + ExperimentError, + "POST", + `/v1/experiments/${encodeURIComponent(id)}/complete`, + { timeoutMs: REQUEST_TIMEOUT_MS, idempotent: true }, + ); +} + +/** Waits until the server evaluators finish and the experiment ends. */ +async function waitForResult( + api: Authed, + id: string, +): Promise<{ result: ExperimentResult; body: unknown }> { + const deadline = Date.now() + MAX_WAIT_MS; + + for (let interval = 1000; ; interval = Math.min(interval * 2, MAX_POLL_INTERVAL_MS)) { + const body = await request( + api, + ExperimentError, + "GET", + `/v1/experiments/${encodeURIComponent(id)}`, + { timeoutMs: REQUEST_TIMEOUT_MS, idempotent: true }, + ); + + const result = parseResult(body); + + if (FINISHED.has(result.status)) return { result, body }; + + if (Date.now() + interval > deadline) { + throw new ExperimentError( + "timeout", + `The experiment ${result.name} did not end in ${MAX_WAIT_MS / 60_000} minutes. ${result.url}`, + undefined, + { result }, + ); + } + + await sleep(interval); + } +} + +/** In CI mode, a failed check or an experiment that did not complete fails the run. */ +function checkResult(result: ExperimentResult): void { + if (result.status !== "completed") { + throw new ExperimentError( + "experiment_failed", + `The experiment ${result.name} ${result.status}${result.error ? `: ${result.error}` : ""}. ${result.url}`, + undefined, + { result }, + ); + } + + if (result.checkStatus !== "failed") return; + + const failed = result.scores + .filter((score) => score.passed === false) + .map( + (score) => + `${score.name} is ${formatScore(score.kind, score.value)}, under the pass mark ${formatScore(score.kind, score.minScore)}`, + ); + + throw new ExperimentError( + "check_failed", + `The check of ${result.name} failed${failed.length > 0 ? `: ${failed.join("; ")}` : ""}. ${result.url}`, + undefined, + { result }, + ); +} + +function describe(result: ExperimentResult): string { + const outcome = + result.status !== "completed" + ? `${result.status}${result.error ? `: ${result.error}` : ""}` + : result.checkStatus === null + ? "completed" + : `completed. The check ${result.checkStatus}`; + + return [ + `Experiment ${result.name} ${outcome}. ${result.url}`, + ...result.scores.map( + (score) => + ` ${score.name}: ${formatScore(score.kind, score.value)}` + + (score.minScore === null ? "" : ` (pass mark ${formatScore(score.kind, score.minScore)})`), + ), + ].join("\n"); +} + +function formatScore(kind: string | null, value: number | null): string { + if (value === null) return "no score"; + + return kind !== null && RATE_KINDS.has(kind) + ? `${Math.round(value * 100)}%` + : String(Math.round(value * 100) / 100); +} + +function parseCreated(body: unknown): Created { + if ( + !isRecord(body) || + typeof body.id !== "string" || + typeof body.name !== "string" || + typeof body.url !== "string" || + !Number.isSafeInteger(body.itemsTotal) + ) { + throw new ExperimentError( + "invalid_response", + "telemetry.dev returned an experiment it cannot read", + ); + } + + return { id: body.id, name: body.name, itemsTotal: body.itemsTotal as number, url: body.url }; +} + +function parseDatasetPage(body: unknown): DatasetPage | undefined { + if (!isRecord(body) || !isRecord(body.dataset) || !Array.isArray(body.items)) return undefined; + + const { version } = body.dataset; + const nextCursor = body.nextCursor ?? null; + + if (!Number.isSafeInteger(version) || (version as number) < 0) return undefined; + + if (nextCursor !== null && typeof nextCursor !== "string") return undefined; + + const items: DatasetItem[] = []; + + for (const item of body.items) { + if ( + !isRecord(item) || + typeof item.id !== "string" || + !Number.isSafeInteger(item.number) || + !isRecord(item.input) || + (item.expectedOutput !== null && typeof item.expectedOutput !== "string") || + !isRecord(item.metadata) || + !Array.isArray(item.tags) || + !item.tags.every((tag) => typeof tag === "string") || + (item.split !== null && typeof item.split !== "string") + ) { + return undefined; + } + + items.push({ + id: item.id, + number: item.number as number, + input: item.input, + expectedOutput: item.expectedOutput, + metadata: item.metadata, + tags: item.tags as string[], + split: item.split, + }); + } + + return { version: version as number, items, nextCursor }; +} + +function parseResult(body: unknown): ExperimentResult { + const scores = isRecord(body) && Array.isArray(body.scores) ? body.scores.map(parseScore) : []; + + if ( + !isRecord(body) || + typeof body.id !== "string" || + typeof body.name !== "string" || + typeof body.status !== "string" || + typeof body.url !== "string" || + // Required: it decides the result of CI mode, so a missing one must not read as no check. + !( + body.checkStatus === null || + body.checkStatus === "passed" || + body.checkStatus === "failed" + ) || + typeof body.itemsTotal !== "number" || + typeof body.itemsDone !== "number" || + !(body.itemsFailed === undefined || typeof body.itemsFailed === "number") || + !(body.error === undefined || body.error === null || typeof body.error === "string") || + !Array.isArray(body.scores) || + scores.some((score) => score === undefined) + ) { + throw new ExperimentError("invalid_response", "telemetry.dev returned a result it cannot read"); + } + + return { + id: body.id, + name: body.name, + status: body.status, + checkStatus: body.checkStatus, + itemsTotal: body.itemsTotal, + itemsDone: body.itemsDone, + itemsFailed: body.itemsFailed ?? 0, + error: body.error ?? null, + scores: scores as ExperimentScore[], + url: body.url, + }; +} + +/** A score of the result. A field that can be null can also be missing. */ +function parseScore(raw: unknown): ExperimentScore | undefined { + if (!isRecord(raw)) return undefined; + + const score = { + name: raw.name, + kind: raw.kind ?? null, + value: raw.value ?? null, + passCount: raw.passCount, + failCount: raw.failCount, + baselineValue: raw.baselineValue ?? null, + delta: raw.delta ?? null, + minScore: raw.minScore ?? null, + passed: raw.passed ?? null, + }; + const numbers = [score.value, score.baselineValue, score.delta, score.minScore]; + + if ( + typeof score.name !== "string" || + !(score.kind === null || typeof score.kind === "string") || + typeof score.passCount !== "number" || + typeof score.failCount !== "number" || + !numbers.every((field) => field === null || typeof field === "number") || + !(score.passed === null || typeof score.passed === "boolean") + ) { + return undefined; + } + + return score as ExperimentScore; +} diff --git a/packages/sdk/src/index.ts b/packages/sdk/src/index.ts index 2fc33431..2c81aee2 100644 --- a/packages/sdk/src/index.ts +++ b/packages/sdk/src/index.ts @@ -46,6 +46,20 @@ export { log, type LogOptions } from "./logs.ts"; export { observe, type ObserveOptions } from "./observe.ts"; +export { + type DatasetItem, + ExperimentError, + type ExperimentItem, + type ExperimentResult, + type ExperimentScore, + runExperiment, + type RunExperimentOptions, + type Scorer, + type ScorerResult, +} from "./experiments.ts"; + +export { score, ScoreError, type ScoreOptions } from "./scores.ts"; + export { type ChatMessage, type ChatPrompt, diff --git a/packages/sdk/src/prompts.ts b/packages/sdk/src/prompts.ts index 2e27b170..2787b294 100644 --- a/packages/sdk/src/prompts.ts +++ b/packages/sdk/src/prompts.ts @@ -1,16 +1,15 @@ import { type Attributes, type Context, createContextKey } from "@opentelemetry/api"; import { activeContext, - DEFAULT_BASE_URL, jsonAttr, omitUndefined, reportError, - resolveEnv, sessionIdOf, sha256, withContext, } from "@telemetry-dev/otel"; +import { isRecord, resolveApi, retryAfterMs } from "./api.ts"; import type { FieldCaptureConfig } from "./attrs.ts"; import { prepareCaptureValue } from "./capture.ts"; import { currentClient } from "./client.ts"; @@ -199,8 +198,6 @@ const MAX_RETRY_AFTER_MS = 3_600_000; /** How long a failed fetch with no copy answers calls, so that they do not each fetch again. */ const ERROR_TTL_MS = 10_000; const MAX_CACHE_ENTRIES = 1_000; -/** IMF-fixdate, the HTTP date that servers send: `Sun, 06 Nov 1994 08:49:37 GMT`. */ -const HTTP_DATE = /^[A-Z][a-z]{2}, \d{2} [A-Z][a-z]{2} \d{4} \d{2}:\d{2}:\d{2} GMT$/; const VARIABLE = /\{\{\s*([A-Za-z_][A-Za-z0-9_]*)\s*\}\}/g; const MODEL_CALL_OPERATIONS = new Set(["chat", "text_completion", "generate_content"]); const PROMPT_KEY = createContextKey("telemetry.dev prompt"); @@ -228,20 +225,8 @@ export async function getPrompt(name: string, options: GetPromptOptions = {}): P const type = options.type ?? "chat"; checkOptions(name, options, type); - const core = currentClient().core; - const env = resolveEnv(); - const apiKey = options.apiKey ?? core?.config.apiKey ?? env.TELEMETRY_DEV_API_KEY; - - // An empty value counts as unset, as in the Python SDK. - const baseUrl = ( - options.baseUrl || - core?.config.baseUrl || - env.TELEMETRY_DEV_BASE_URL || - DEFAULT_BASE_URL - ).replace(/\/+$/, ""); - + const { apiKey, baseUrl, sdkName, fetchImpl, onError } = resolveApi(options); const label = options.version === undefined ? (options.label ?? "production") : null; - const onError = core?.config.onError; let response: PromptResponse; try { @@ -269,11 +254,8 @@ export async function getPrompt(name: string, options: GetPromptOptions = {}): P () => fetchPrompt( url, - { - authorization: `Bearer ${apiKey}`, - "x-telemetry-dev-sdk": core?.config.sdkName ?? "@telemetry-dev/sdk", - }, - options.fetch ?? core?.config.fetchImpl ?? globalThis.fetch, + { authorization: `Bearer ${apiKey}`, "x-telemetry-dev-sdk": sdkName }, + fetchImpl, ), onError, ); @@ -492,7 +474,7 @@ async function fetchPrompt( ); const delay = retryAfterMs(res); - if (delay !== undefined) retryAfter.set(failure, delay); + if (delay !== undefined) retryAfter.set(failure, Math.min(delay, MAX_RETRY_AFTER_MS)); throw failure; } @@ -616,23 +598,6 @@ function keyFingerprint(apiKey: string): string { return Array.from(digest, (byte) => byte.toString(16).padStart(2, "0")).join(""); } -/** The Retry-After of a 429 or 503 response, as seconds or an IMF-fixdate HTTP date, capped. */ -function retryAfterMs(res: Response): number | undefined { - if (res.status !== 429 && res.status !== 503) return undefined; - - const value = res.headers.get("retry-after")?.trim(); - - if (!value) return undefined; - - let ms: number; - - if (/^\d+$/.test(value)) ms = Number(value) * 1000; - else if (HTTP_DATE.test(value)) ms = Date.parse(value) - Date.now(); - else return undefined; - - return ms > 0 ? Math.min(ms, MAX_RETRY_AFTER_MS) : undefined; -} - function parseResponse(body: unknown): PromptResponse | undefined { if (!isRecord(body)) return undefined; const { name, type, label, version, rollout } = body; @@ -724,10 +689,6 @@ function isVersion(value: unknown): value is number { return Number.isSafeInteger(value) && (value as number) >= 1; } -function isRecord(value: unknown): value is Record { - return typeof value === "object" && value !== null && !Array.isArray(value); -} - // Prompt objects share the cached response, so an app cannot change the cache through them. function deepFreeze(value: T): T { if (typeof value === "object" && value !== null && !Object.isFrozen(value)) { diff --git a/packages/sdk/src/scores.ts b/packages/sdk/src/scores.ts new file mode 100644 index 00000000..bb06d763 --- /dev/null +++ b/packages/sdk/src/scores.ts @@ -0,0 +1,172 @@ +import { reportError } from "@telemetry-dev/otel"; + +import { type ApiOptions, isRecord, request, resolveApi } from "./api.ts"; + +export interface ScoreOptions extends ApiOptions { + /** The score name. `user_feedback` is a thumbs up (1) or down (0) from a user of your app. */ + name: string; + /** The trace that the score is on. Pass `traceId`, `sessionId`, or both. */ + traceId?: string; + /** A span of the trace, for a score on one span. */ + spanId?: string; + sessionId?: string; + /** Pass `value`, `label`, or both. */ + value?: number; + label?: string; + comment?: string; +} + +/** + * Why score failed. `code` is the code of the API error (`insufficient_scope`, `invalid_api_key`, + * …), or `missing_api_key`, `invalid_score`, `timeout`, `network_error`, or `invalid_response`. A + * response with a different status and no API error gives `http_`. + */ +export class ScoreError extends Error { + override readonly name = "ScoreError"; + + constructor( + readonly code: string, + message: string, + readonly status?: number, + options?: { cause?: unknown }, + ) { + super(message, options); + } +} + +export const USER_FEEDBACK = "user_feedback"; +export const MAX_SCORE_NAME_LENGTH = 64; +export const MAX_LABEL_LENGTH = 64; +export const MAX_REASON_LENGTH = 4_000; +const MAX_SESSION_ID_LENGTH = 1_024; +const TRACE_ID = /^(?!0+$)[0-9a-f]{32}$/; +const SPAN_ID = /^(?!0+$)[0-9a-f]{16}$/; +const TIMEOUT_MS = 10_000; + +/** + * Records a score on a trace, a span, or a session, for example the feedback of a user on a + * reply. Like the telemetry functions, score never rejects: it resolves with the id of the score, + * or with undefined after it reports the error through `onError`. It makes one request, with no + * retry, so that a call in a request handler ends in 10 seconds at most. The trace id of a span + * from a client that is not initialized is all zeros, and score reports it as `invalid_score`. + */ +export async function score(options: ScoreOptions): Promise { + // A caller without types can pass no options, and score must not reject then either. + const fields: Partial = isRecord(options) ? options : {}; + const api = resolveApi(fields); + + try { + const name = typeof fields.name === "string" ? fields.name.trim() : fields.name; + const label = typeof fields.label === "string" ? fields.label.trim() : fields.label; + const problem = checkScore({ ...fields, name, label }); + + if (problem) throw new ScoreError("invalid_score", problem); + + if (!api.apiKey) { + throw new ScoreError( + "missing_api_key", + "No API key: pass apiKey or set TELEMETRY_DEV_API_KEY", + ); + } + + const body = await request({ ...api, apiKey: api.apiKey }, ScoreError, "POST", "/v1/scores", { + body: { + traceId: fields.traceId, + spanId: fields.spanId, + sessionId: fields.sessionId, + name, + value: fields.value, + label, + comment: fields.comment, + }, + timeoutMs: TIMEOUT_MS, + idempotent: false, + attempts: 1, + }); + + if (!isRecord(body) || typeof body.id !== "string") { + throw new ScoreError("invalid_response", "telemetry.dev returned a score it cannot read"); + } + + return body.id; + } catch (error) { + reportError(api.onError, error); + + return undefined; + } +} + +/** + * Why the server would reject the value and the label of a score, or undefined. A pass/fail score + * has the label `pass` with the value 1, or `fail` with 0. + */ +export function checkValue( + name: string, + value: number | undefined, + label: string | undefined, +): string | undefined { + if (value !== undefined && !(typeof value === "number" && Number.isFinite(value))) { + return "value must be a finite number"; + } + + if ( + label !== undefined && + (typeof label !== "string" || label.length === 0 || label.length > MAX_LABEL_LENGTH) + ) { + return `label must be a string of 1 to ${MAX_LABEL_LENGTH} characters`; + } + + if (value === undefined && label === undefined) return "Pass a value or a label"; + + if (name === USER_FEEDBACK && !(label === undefined && (value === 0 || value === 1))) { + return `${USER_FEEDBACK} takes the value 1 for up or 0 for down, and no label`; + } + + if (label === "pass" && value !== undefined && value !== 1) { + return "The label pass goes with the value 1"; + } + + if (label === "fail" && value !== undefined && value !== 0) { + return "The label fail goes with the value 0"; + } + + return undefined; +} + +function checkScore(fields: Partial): string | undefined { + const { name, traceId, spanId, sessionId, value, label, comment } = fields; + + if (typeof name !== "string" || name.length === 0 || name.length > MAX_SCORE_NAME_LENGTH) { + return `name must be a string of 1 to ${MAX_SCORE_NAME_LENGTH} characters`; + } + + if (traceId === undefined && sessionId === undefined) return "Pass traceId or sessionId"; + + if (traceId !== undefined && !(typeof traceId === "string" && TRACE_ID.test(traceId))) { + return "traceId must be 32 lowercase hex digits and not all zeros"; + } + + if (spanId !== undefined && !(typeof spanId === "string" && SPAN_ID.test(spanId))) { + return "spanId must be 16 lowercase hex digits and not all zeros"; + } + + if (spanId !== undefined && traceId === undefined) return "Pass the traceId of the span"; + + if ( + sessionId !== undefined && + (typeof sessionId !== "string" || + sessionId.length === 0 || + sessionId.length > MAX_SESSION_ID_LENGTH) + ) { + return `sessionId must be a string of 1 to ${MAX_SESSION_ID_LENGTH} characters`; + } + + if ( + comment !== undefined && + (typeof comment !== "string" || comment.length > MAX_REASON_LENGTH) + ) { + return `comment must be a string of at most ${MAX_REASON_LENGTH} characters`; + } + + return checkValue(name, value, label); +} diff --git a/packages/sdk/tests/experiment-action.test.ts b/packages/sdk/tests/experiment-action.test.ts new file mode 100644 index 00000000..eb8bd996 --- /dev/null +++ b/packages/sdk/tests/experiment-action.test.ts @@ -0,0 +1,413 @@ +import { spawn, spawnSync } from "node:child_process"; +import { mkdirSync, mkdtempSync, readFileSync, statSync, writeFileSync } from "node:fs"; +import { createServer, type IncomingMessage } from "node:http"; +import type { AddressInfo } from "node:net"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { fileURLToPath } from "node:url"; + +import { afterEach, expect, test } from "vitest"; + +// The experiment action: run.sh runs the eval script, and report.mjs writes the job summary and +// the comment on the pull request. +const action = fileURLToPath(new URL("../../../actions/experiment/", import.meta.url)); +const closers: (() => void)[] = []; + +afterEach(() => { + for (const close of closers.splice(0)) close(); +}); + +function workdir(): string { + return mkdtempSync(join(tmpdir(), "td-action-")); +} + +function outputs(path: string): Record { + return Object.fromEntries( + readFileSync(path, "utf8") + .split("\n") + .filter(Boolean) + .map((line) => line.split(/=(.*)/s).slice(0, 2) as [string, string]), + ); +} + +function runScript(env: Record, files: Record = {}) { + const dir = workdir(); + const output = join(dir, "output"); + writeFileSync(output, ""); + + for (const [name, content] of Object.entries(files)) writeFileSync(join(dir, name), content); + + const run = spawnSync("bash", [join(action, "run.sh")], { + cwd: dir, + encoding: "utf8", + env: { PATH: process.env.PATH, RUNNER_TEMP: dir, GITHUB_OUTPUT: output, ...env }, + }); + + return { dir, run, outputs: outputs(output) }; +} + +const result = { + id: "6f9619ff-8b86-4011-b42d-00c04fc964ff", + name: "reply-v13-gpt-4o", + status: "completed", + checkStatus: "failed", + itemsTotal: 120, + itemsDone: 120, + itemsFailed: 2, + error: null, + scores: [ + { + name: "correctness", + kind: "pass_fail", + value: 0.85, + passCount: 100, + failCount: 18, + baselineValue: 0.88, + delta: -3, + minScore: 0.9, + passed: false, + }, + { + name: "helpfulness", + kind: "number", + value: 4.436, + passCount: 0, + failCount: 0, + baselineValue: 4.1, + delta: 0.3, + minScore: null, + passed: null, + }, + ], + url: "https://telemetry.dev/o/org/p/project/evals/experiments/6f9619ff-8b86-4011-b42d-00c04fc964ff", +}; + +const summary = `### telemetry.dev experiment + +**[reply-v13-gpt-4o](https://telemetry.dev/o/org/p/project/evals/experiments/6f9619ff-8b86-4011-b42d-00c04fc964ff)**: Check failed. 118 of 120 items ran, 2 failed. + +| Score | Value | Change | Pass mark | Result | +| --- | --- | --- | --- | --- | +| correctness | 85% | ↓ 3 pts | 90% | Failed | +| helpfulness | 4.44 | ↑ 0.3 | — | — | + +The experiment script exited with code 1.`; + +interface GithubRequest { + method: string; + url: string; + body: unknown; +} + +interface GithubComment { + id: number; + body: string; + user: { login: string; type: "Bot" | "User" }; +} + +const BOT = { login: "github-actions[bot]", type: "Bot" } as const; + +/** + * The comment routes of the GitHub REST API for one pull request. Without `user`, GET /user fails + * as it does for the GITHUB_TOKEN. + */ +async function fakeGithub( + options: { comments?: GithubComment[]; status?: number; user?: { login: string } } = {}, +) { + const requests: GithubRequest[] = []; + const server = createServer(async (req: IncomingMessage, res) => { + let raw = ""; + for await (const chunk of req) raw += chunk; + requests.push({ method: req.method!, url: req.url!, body: raw ? JSON.parse(raw) : undefined }); + + if (req.url === "/user") { + res.writeHead(options.user ? 200 : 403, { "content-type": "application/json" }); + res.end( + JSON.stringify(options.user ?? { message: "Resource not accessible by integration" }), + ); + + return; + } + + res.writeHead(options.status ?? 200, { "content-type": "application/json" }); + res.end(JSON.stringify(req.method === "GET" ? (options.comments ?? []) : { id: 9 })); + }); + await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve)); + closers.push(() => server.close()); + + return { url: `http://127.0.0.1:${(server.address() as AddressInfo).port}`, requests }; +} + +async function report(env: Record, lines: unknown[]) { + const dir = workdir(); + const results = join(dir, "results.jsonl"); + const stepSummary = join(dir, "summary.md"); + const output = join(dir, "output"); + const event = join(dir, "event.json"); + writeFileSync(results, lines.map((line) => `${JSON.stringify(line)}\n`).join("")); + writeFileSync(stepSummary, ""); + writeFileSync(output, ""); + writeFileSync(event, JSON.stringify({ pull_request: { number: 42 } })); + + const child = spawn("node", [join(action, "report.mjs")], { + env: { + PATH: process.env.PATH, + TELEMETRY_DEV_EXPERIMENT_RESULTS: results, + GITHUB_STEP_SUMMARY: stepSummary, + GITHUB_OUTPUT: output, + GITHUB_EVENT_PATH: event, + GITHUB_REPOSITORY: "acme/support", + GITHUB_TOKEN: "ghs_test", + GITHUB_WORKFLOW: "evals", + GITHUB_JOB: "reply", + INPUT_DATASET: "support-golden", + EXPERIMENT_EXIT_CODE: "1", + ...env, + }, + }); + let stdout = ""; + child.stdout.on("data", (chunk) => (stdout += chunk)); + const code = await new Promise((resolve) => child.on("close", resolve)); + + return { code, stdout, summary: readFileSync(stepSummary, "utf8"), outputs: outputs(output) }; +} + +test("run.sh passes the inputs to the script and records its exit code without failing", () => { + const { + dir, + run, + outputs: written, + } = runScript({ + INPUT_COMMAND: `node -e 'const fs = require("node:fs"); fs.writeFileSync("env.json", JSON.stringify(process.env)); process.exit(3)'`, + INPUT_API_KEY: "td_live_secret", + INPUT_DATASET: "support-golden", + INPUT_MIN_SCORES: "correctness=0.9", + INPUT_BASE_URL: "https://ingest.test", + }); + + expect(run.status).toBe(0); + expect(run.stdout).toContain("::add-mask::td_live_secret"); + expect(written["exit-code"]).toBe("3"); + + const env = JSON.parse(readFileSync(join(dir, "env.json"), "utf8")); + expect(env).toMatchObject({ + TELEMETRY_DEV_API_KEY: "td_live_secret", + TELEMETRY_DEV_BASE_URL: "https://ingest.test", + TELEMETRY_DEV_EXPERIMENT_DATASET: "support-golden", + TELEMETRY_DEV_EXPERIMENT_MIN_SCORES: "correctness=0.9", + TELEMETRY_DEV_ENVIRONMENT: "experiment", + TELEMETRY_DEV_EXPERIMENT_RESULTS: written.results, + }); + expect(written.results!.startsWith(dir)).toBe(true); + expect(statSync(written.results!).mode & 0o777).toBe(0o600); +}); + +test("run.sh runs a .ts script with the tsx of the project, from a directory above", () => { + const project = workdir(); + const evals = join(project, "evals"); + const output = join(project, "output"); + mkdirSync(join(project, "node_modules", ".bin"), { recursive: true }); + mkdirSync(evals); + writeFileSync(output, ""); + // A stand-in for the tsx of the project that records the script it gets. + writeFileSync( + join(project, "node_modules", ".bin", "tsx"), + `#!/bin/sh\necho "$1" > "${join(project, "ran")}"\n`, + { mode: 0o755 }, + ); + + const run = spawnSync("bash", [join(action, "run.sh")], { + cwd: evals, + encoding: "utf8", + env: { + PATH: process.env.PATH, + RUNNER_TEMP: project, + GITHUB_OUTPUT: output, + INPUT_SCRIPT: "reply.eval.ts", + TELEMETRY_DEV_API_KEY: "td_live_job", + }, + }); + + expect(run.status).toBe(0); + expect(readFileSync(join(project, "ran"), "utf8")).toBe("reply.eval.ts\n"); + expect(outputs(output)["exit-code"]).toBe("0"); +}); + +test("run.sh keeps the environment of the job, and runs a .js script with node", () => { + const { run, outputs: written } = runScript( + { + INPUT_SCRIPT: "print-env.mjs", + TELEMETRY_DEV_API_KEY: "td_live_job", + TELEMETRY_DEV_ENVIRONMENT: "staging", + }, + { "print-env.mjs": "console.log(JSON.stringify(process.env));\n" }, + ); + + expect(written["exit-code"]).toBe("0"); + expect(JSON.parse(run.stdout)).toMatchObject({ + TELEMETRY_DEV_API_KEY: "td_live_job", + TELEMETRY_DEV_ENVIRONMENT: "staging", + }); +}); + +test.each([ + [ + "script and command", + { INPUT_SCRIPT: "a.ts", INPUT_COMMAND: "true", TELEMETRY_DEV_API_KEY: "k" }, + "Give script or command, not the two.", + ], + [ + "no script and no command", + { TELEMETRY_DEV_API_KEY: "k" }, + "Give the script or the command that runs the experiment.", + ], + [ + "no API key", + { INPUT_COMMAND: "true" }, + "Set the api-key input or the TELEMETRY_DEV_API_KEY variable.", + ], + [ + "a script of another type", + { INPUT_SCRIPT: "eval.rb", TELEMETRY_DEV_API_KEY: "k" }, + "The script must end in .ts, .js, or .py. For another runner, give a command.", + ], + [ + "a .ts script in a project without tsx", + { INPUT_SCRIPT: "reply.eval.ts", TELEMETRY_DEV_API_KEY: "k" }, + "No tsx in node_modules. Add tsx to the devDependencies of the project and install them, or give a command.", + ], +])("run.sh stops with an error for %s", (_, env, message) => { + const { run, outputs: written } = runScript(env); + + expect(run.stdout).toContain(`::error title=telemetry.dev experiment::${message}`); + expect(written["exit-code"]).toBe("2"); +}); + +test("report.mjs writes the summary, posts it on the pull request, and sets the outputs", async () => { + const github = await fakeGithub(); + + const { + code, + summary: written, + outputs: set, + } = await report({ GITHUB_API_URL: github.url }, [result]); + + expect(code).toBe(0); + expect(written).toBe(`${summary}\n`); + expect(set).toEqual({ "check-status": "failed", url: result.url }); + expect(github.requests.map((request) => `${request.method} ${request.url}`)).toEqual([ + "GET /user", + "GET /repos/acme/support/issues/42/comments?per_page=100&page=1", + "POST /repos/acme/support/issues/42/comments", + ]); + expect(github.requests[2]!.body).toEqual({ + body: expect.stringMatching( + /^\n### telemetry\.dev experiment\n/, + ), + }); + expect((github.requests[2]!.body as { body: string }).body.endsWith(summary)).toBe(true); +}); + +/** The body of the comment that the action posts for the result. */ +async function commentBody(env: Record = {}): Promise { + const github = await fakeGithub(); + await report({ GITHUB_API_URL: github.url, ...env }, [result]); + + return (github.requests.find((request) => request.method === "POST")!.body as { body: string }) + .body; +} + +function changes(github: Awaited>): string[] { + return github.requests + .filter((request) => request.method !== "GET") + .map((request) => `${request.method} ${request.url}`); +} + +test("report.mjs changes its comment from an earlier push, and a step with another dataset adds its own", async () => { + const own = { id: 8, body: await commentBody(), user: BOT }; + + const other = await fakeGithub({ comments: [own] }); + await report({ GITHUB_API_URL: other.url, INPUT_DATASET: "billing-golden" }, [result]); + + const again = await fakeGithub({ + comments: [{ id: 7, body: "Looks good", user: { login: "dana", type: "User" } }, own], + }); + await report({ GITHUB_API_URL: again.url }, [result]); + + expect(changes(other)).toEqual(["POST /repos/acme/support/issues/42/comments"]); + expect(changes(again)).toEqual(["PATCH /repos/acme/support/issues/comments/8"]); +}); + +test.each([ + ["a person pasted under the GITHUB_TOKEN", undefined, { login: "mallory", type: "User" }], + ["another bot wrote under a user token", { login: "dana" }, BOT], +] as const)( + "report.mjs does not change a comment with its marker that %s", + async (_, user, author) => { + const github = await fakeGithub({ + user, + comments: [{ id: 7, body: await commentBody(), user: author }], + }); + + await report({ GITHUB_API_URL: github.url }, [result]); + + expect(changes(github)).toEqual(["POST /repos/acme/support/issues/42/comments"]); + }, +); + +test("report.mjs changes the comment of the user of a user token", async () => { + const github = await fakeGithub({ + user: { login: "dana" }, + comments: [{ id: 7, body: await commentBody(), user: { login: "dana", type: "User" } }], + }); + + await report({ GITHUB_API_URL: github.url }, [result]); + + expect(changes(github)).toEqual(["PATCH /repos/acme/support/issues/comments/7"]); +}); + +test("report.mjs warns and succeeds when the token cannot write the comment", async () => { + const github = await fakeGithub({ status: 403 }); + + const { code, stdout } = await report({ GITHUB_API_URL: github.url }, [result]); + + expect(code).toBe(0); + expect(stdout).toContain( + "::warning title=telemetry.dev experiment::The comment on the pull request failed: GitHub returned 403", + ); +}); + +test("report.mjs reports a script that ended before it wrote a result, and comment: false posts nothing", async () => { + const github = await fakeGithub(); + + const { summary: written, outputs: set } = await report( + { GITHUB_API_URL: github.url, INPUT_COMMENT: "false", EXPERIMENT_EXIT_CODE: "1" }, + [], + ); + + expect(github.requests).toEqual([]); + + expect(written).toBe( + "### telemetry.dev experiment\n\nThe experiment script exited with code 1 before it wrote a result. The log of the step has the error.\n", + ); + expect(set).toEqual({ "check-status": "none", url: "" }); +}); + +test("report.mjs escapes names, so that they cannot break the table or add HTML", async () => { + const github = await fakeGithub(); + const { summary: written } = await report( + { GITHUB_API_URL: github.url, INPUT_COMMENT: "false", EXPERIMENT_EXIT_CODE: "0" }, + [ + { + ...result, + name: "|*bold*", + url: "javascript:alert(1)", + scores: [{ ...result.scores[0], name: "a|b\nc" }], + }, + ], + ); + + expect(written).toContain("**<img src=x>\\|\\*bold\\***: Check failed."); + expect(written).toContain("| a\\|b c | 85% |"); + expect(written).not.toContain("javascript:"); +}); diff --git a/packages/sdk/tests/experiments.test.ts b/packages/sdk/tests/experiments.test.ts new file mode 100644 index 00000000..5f555e35 --- /dev/null +++ b/packages/sdk/tests/experiments.test.ts @@ -0,0 +1,985 @@ +import { mkdtempSync, readFileSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; + +import { afterEach, beforeEach, expect, test, vi } from "vitest"; + +import { + type ExperimentItem, + ExperimentError, + log, + propagateAttributes, + runExperiment, + type RunExperimentOptions, + shutdown, + startActiveSpan, + startSpan, +} from "../src/index.ts"; +import { setup } from "./helpers.ts"; + +interface Request { + method: string; + url: URL; + body: any; + signal: AbortSignal | undefined; +} + +const EXPERIMENT_ID = "6f9619ff-8b86-4011-b42d-00c04fc964ff"; +const URL_OF_EXPERIMENT = `https://telemetry.dev/o/org/p/project/evals/experiments/${EXPERIMENT_ID}`; + +function itemId(number: number): string { + return `00000000-0000-4000-8000-${String(number).padStart(12, "0")}`; +} + +function datasetItem(number: number, split: string | null = null) { + return { + id: itemId(number), + number, + input: { question: `q${number}` } as Record, + expectedOutput: `a${number}`, + metadata: { source: "test" }, + tags: ["golden"], + split, + }; +} + +function result(overrides: Record = {}): Record { + return { + id: EXPERIMENT_ID, + name: "support-golden-v3", + status: "completed", + checkStatus: null, + itemsTotal: 2, + itemsDone: 2, + itemsFailed: 0, + error: null, + scores: [], + url: URL_OF_EXPERIMENT, + ...overrides, + }; +} + +/** + * The Evals routes of telemetry.dev, in memory: pages of a dataset version with a cursor, an + * experiment that takes item posts of at most 100 and skips repeats, and its result. + */ +function fakeApi( + options: { + items?: ReturnType[]; + pageSize?: number; + results?: Record[]; + override?: (request: Request) => Response | Promise | undefined; + } = {}, +) { + const items = options.items ?? [datasetItem(1), datasetItem(2)]; + const pageSize = options.pageSize ?? 100; + const results = options.results ?? [result()]; + const requests: Request[] = []; + const stored = new Set(); + + const fetch = vi.fn(async (input: string | URL, init?: RequestInit) => { + const url = new URL(input); + const request: Request = { + method: init?.method ?? "GET", + url, + body: typeof init?.body === "string" ? JSON.parse(init.body) : undefined, + signal: init?.signal ?? undefined, + }; + requests.push(request); + + const overridden = options.override?.(request); + + if (overridden) return overridden; + + const route = `${request.method} ${url.pathname}`; + + if (route === "GET /v1/datasets/support-golden") { + const split = url.searchParams.get("split"); + const start = Number(url.searchParams.get("cursor") ?? 0); + const matching = items.filter((item) => split === null || item.split === split); + + return Response.json({ + dataset: { + id: "dataset-1", + name: "support-golden", + version: Number(url.searchParams.get("version") ?? 3), + }, + items: matching.slice(start, start + pageSize), + nextCursor: start + pageSize < matching.length ? String(start + pageSize) : null, + }); + } + + if (route === "POST /v1/experiments") { + const count = items.filter( + (item) => request.body.split === undefined || item.split === request.body.split, + ).length; + + return Response.json( + { + id: EXPERIMENT_ID, + name: request.body.name ?? "support-golden-v3", + datasetVersion: request.body.datasetVersion, + itemsTotal: count * (request.body.runsPerItem ?? 1), + url: URL_OF_EXPERIMENT, + }, + { status: 201 }, + ); + } + + if (route === `POST /v1/experiments/${EXPERIMENT_ID}/items`) { + if (request.body.items.length > 100) { + return Response.json( + { error: { code: "invalid_request", message: "at most 100 items" } }, + { status: 400 }, + ); + } + + let accepted = 0; + + for (const run of request.body.items) { + const key = `${run.itemId}:${run.runIndex}`; + + if (!stored.has(key)) accepted += 1; + stored.add(key); + } + + return Response.json({ accepted, duplicates: request.body.items.length - accepted }); + } + + if (route === `POST /v1/experiments/${EXPERIMENT_ID}/complete`) { + return Response.json({ id: EXPERIMENT_ID, status: "running" }); + } + + if (route === `GET /v1/experiments/${EXPERIMENT_ID}`) { + return Response.json(results.length > 1 ? results.shift() : results[0]); + } + + return Response.json({ error: { code: "not_found", message: route } }, { status: 404 }); + }); + + const posts = () => + requests + .filter((request) => request.url.pathname.endsWith("/items")) + .map((request) => request.body.items); + + return { fetch: fetch as unknown as typeof globalThis.fetch, requests, posts }; +} + +function routes(requests: Request[]): string[] { + return requests.map((request) => `${request.method} ${request.url.pathname}`); +} + +function experiment( + api: ReturnType, + options: Partial = {}, +): ReturnType { + return runExperiment({ + apiKey: "td_live_test", + baseUrl: "https://ingest.test", + fetch: api.fetch, + dataset: "support-golden", + task: (item) => `answer to ${String(item.input.question)}`, + ...options, + }); +} + +beforeEach(() => { + for (const name of [ + "TELEMETRY_DEV_API_KEY", + "TELEMETRY_DEV_BASE_URL", + "TELEMETRY_DEV_EXPERIMENT_DATASET", + "TELEMETRY_DEV_EXPERIMENT_MIN_SCORES", + "TELEMETRY_DEV_EXPERIMENT_RESULTS", + "CI", + "GITHUB_ACTIONS", + "GITHUB_REPOSITORY", + "GITHUB_SHA", + "GITHUB_EVENT_PATH", + "GITHUB_RUN_ID", + "GITHUB_SERVER_URL", + ]) { + vi.stubEnv(name, undefined); + } + + vi.spyOn(console, "log").mockImplementation(() => {}); +}); + +afterEach(async () => { + await shutdown(); + vi.unstubAllEnvs(); + vi.restoreAllMocks(); +}); + +test("runs each item of each page, posts batches of 100 on one version, then completes", async () => { + const api = fakeApi({ + items: Array.from({ length: 75 }, (_, index) => datasetItem(index + 1)), + pageSize: 50, + results: [result({ itemsTotal: 150, itemsDone: 146, itemsFailed: 4 })], + }); + + const outcome = await experiment(api, { runsPerItem: 2, evaluators: ["Correctness"] }); + + const sent = routes(api.requests); + expect(sent.slice(0, 2)).toEqual(["GET /v1/datasets/support-golden", "POST /v1/experiments"]); + // The second page and the first post of items run at the same time, in either order. + expect(sent.slice(2, -2).sort()).toEqual([ + "GET /v1/datasets/support-golden", + `POST /v1/experiments/${EXPERIMENT_ID}/items`, + `POST /v1/experiments/${EXPERIMENT_ID}/items`, + ]); + expect(sent.slice(-2)).toEqual([ + `POST /v1/experiments/${EXPERIMENT_ID}/complete`, + `GET /v1/experiments/${EXPERIMENT_ID}`, + ]); + + // The first page names no version; the next one asks for the version of the first. + const pages = api.requests.filter((request) => request.url.pathname.startsWith("/v1/datasets/")); + expect(Object.fromEntries(pages[0]!.url.searchParams)).toEqual({}); + expect(Object.fromEntries(pages[1]!.url.searchParams)).toEqual({ version: "3", cursor: "50" }); + + expect(api.requests[1]!.body).toEqual({ + dataset: "support-golden", + datasetVersion: 3, + evaluators: ["Correctness"], + runsPerItem: 2, + }); + + const posts = api.posts(); + expect(posts.map((items) => items.length)).toEqual([100, 50]); + + const runs = posts + .flat() + .map((run: { itemId: string; runIndex: number }) => `${run.itemId}:${run.runIndex}`); + expect(new Set(runs).size).toBe(150); + expect(runs).toContain(`${itemId(75)}:1`); + + expect(posts[0]![0]).toEqual({ + itemId: itemId(1), + runIndex: 0, + output: "answer to q1", + latencyMs: expect.any(Number), + }); + + expect(outcome).toEqual({ + id: EXPERIMENT_ID, + name: "support-golden-v3", + status: "completed", + checkStatus: null, + itemsTotal: 150, + itemsDone: 146, + itemsFailed: 4, + error: null, + scores: [], + url: URL_OF_EXPERIMENT, + }); +}); + +test("each item run is its own trace, and each span of the task carries the experiment and the item", async () => { + const { spans, logs } = setup({ environment: "experiment", sessionMode: "process" }); + const api = fakeApi(); + + // An active span, an ambient session, and the process session would each put the runs in one + // trace. + await propagateAttributes({ sessionId: "ambient-session", userId: "u-1" }, () => + startActiveSpan("eval script", () => + experiment(api, { + runsPerItem: 2, + task: (item) => { + startSpan("chat gpt-4o", { type: "generation" }).end(); + log("replied", { attributes: { number: item.number } }); + + return { reply: `r${item.number}` }; + }, + }), + ), + ); + + const posted = api.posts().flat(); + const finished = spans.getFinishedSpans(); + const chats = finished.filter((span) => span.name === "chat gpt-4o"); + const roots = finished.filter((span) => span.name === "experiment support-golden-v3"); + + expect(chats).toHaveLength(4); + expect(roots).toHaveLength(4); + expect(new Set(roots.map((span) => span.spanContext().traceId)).size).toBe(4); + expect(roots.map((span) => span.parentSpanContext?.isRemote)).toEqual([true, true, true, true]); + + for (const run of posted) { + expect(run.traceId).toMatch(/^[0-9a-f]{32}$/); + const root = roots.find((span) => span.spanContext().traceId === run.traceId)!; + const chat = chats.find((span) => span.spanContext().traceId === run.traceId)!; + + expect(chat.parentSpanContext?.spanId).toBe(root.spanContext().spanId); + + for (const span of [root, chat]) { + expect(span.attributes).toMatchObject({ + "td.experiment.id": EXPERIMENT_ID, + "td.dataset.item_id": run.itemId, + "gen_ai.conversation.id": `${EXPERIMENT_ID}:${run.itemId}:${run.runIndex}`, + "user.id": "u-1", + }); + } + } + + const records = logs.getFinishedLogRecords(); + expect(records).toHaveLength(4); + + for (const record of records) { + const run = posted.find( + (candidate: { traceId: string }) => candidate.traceId === record.spanContext?.traceId, + ); + + expect(record.attributes).toMatchObject({ + "td.experiment.id": EXPERIMENT_ID, + "td.dataset.item_id": run.itemId, + }); + } + + expect(posted.map((run: { output: string }) => run.output).sort()).toEqual([ + '{"reply":"r1"}', + '{"reply":"r1"}', + '{"reply":"r2"}', + '{"reply":"r2"}', + ]); +}); + +test("runs at most `concurrency` tasks at a time", async () => { + const api = fakeApi({ items: Array.from({ length: 12 }, (_, index) => datasetItem(index + 1)) }); + let running = 0; + let most = 0; + + await experiment(api, { + concurrency: 3, + task: async () => { + running += 1; + most = Math.max(most, running); + await new Promise((resolve) => setTimeout(resolve, 5)); + running -= 1; + + return "ok"; + }, + }); + + expect(most).toBe(3); + expect(api.posts().flat()).toHaveLength(12); +}); + +test("a task that throws fails only its item", async () => { + const api = fakeApi(); + + await experiment(api, { + task: (item) => { + if (item.number === 1) throw new RangeError("no reply"); + + return "fine"; + }, + }); + + const [failed, passed] = api.posts().flat(); + + expect(failed).toMatchObject({ itemId: itemId(1), output: null, error: "RangeError: no reply" }); + expect(failed).not.toHaveProperty("scores"); + expect(passed).toMatchObject({ itemId: itemId(2), output: "fine" }); + expect(passed).not.toHaveProperty("error"); +}); + +test("an output or an error that cannot be encoded does not stop the run", async () => { + const api = fakeApi({ items: [datasetItem(1), datasetItem(2), datasetItem(3)] }); + const cycle: Record = {}; + cycle.self = cycle; + const unreadable = Object.defineProperty(new Error(), "message", { + get() { + throw new Error("no message"); + }, + }); + + const outcome = await experiment(api, { + task: (item) => { + if (item.number === 3) throw unreadable; + + return item.number === 1 ? cycle : 10n; + }, + // A lone surrogate, which UTF-8 cannot hold. + scorers: { tone: () => "\ud83d" }, + }); + + expect(outcome.status).toBe("completed"); + expect( + api + .posts() + .flat() + .sort((a: { itemId: string }, b: { itemId: string }) => a.itemId.localeCompare(b.itemId)) + .map((run: { output: unknown; error?: unknown; scores?: unknown }) => [ + run.output, + run.error, + run.scores, + ]), + ).toEqual([ + [null, undefined, [{ name: "tone", label: "\ud83d" }]], + [null, undefined, [{ name: "tone", label: "\ud83d" }]], + [null, "A value that cannot be converted to a string was thrown", undefined], + ]); + expect(routes(api.requests)).toContain(`POST /v1/experiments/${EXPERIMENT_ID}/complete`); +}); + +test("each run of an item starts from the dataset input, also when a task changes it", async () => { + const api = fakeApi({ + items: [{ ...datasetItem(1), input: { messages: [{ role: "user", content: "Hi" }] } }], + }); + const seen: number[] = []; + + await experiment(api, { + runsPerItem: 3, + concurrency: 1, + task: (item) => { + const messages = item.input.messages as unknown[]; + seen.push(messages.length); + messages.push({ role: "assistant", content: "Hello" }); + + return "ok"; + }, + }); + + expect(seen).toEqual([1, 1, 1]); +}); + +test("scorers give pass/fail, numbers, labels, and reasons, and a bad result drops only its score", async () => { + const errors: Error[] = []; + setup({ environment: "experiment", onError: (error) => errors.push(error) }); + const api = fakeApi({ items: [datasetItem(1)] }); + + await experiment(api, { + scorers: { + exact: (output, item) => output === `answer to ${String(item.input.question)}`, + polite: () => false, + length: (output) => String(output).length, + tone: () => " friendly ", + judged: () => ({ value: 0.25, reason: "too short" }), + none: () => null, + thrown: () => { + throw new Error("judge down"); + }, + mismatched: () => ({ value: 0.5, label: "pass" }), + }, + }); + + expect(api.posts()[0]![0].scores).toEqual([ + { name: "exact", value: 1, label: "pass" }, + { name: "polite", value: 0, label: "fail" }, + { name: "length", value: 12 }, + { name: "tone", label: "friendly" }, + { name: "judged", value: 0.25, reason: "too short" }, + ]); + + expect(errors.map((error) => [(error as ExperimentError).code, error.message])).toEqual([ + ["scorer_failed", "The scorer thrown failed on item 1: Error: judge down"], + [ + "scorer_failed", + "The scorer mismatched failed on item 1: TypeError: returned a score that is not valid: The label pass goes with the value 1", + ], + ]); +}); + +test("a failed post of items stops new items, completes the experiment, and rejects with the API error", async () => { + const api = fakeApi({ + items: Array.from({ length: 300 }, (_, index) => datasetItem(index + 1)), + override: (request) => + request.url.pathname.endsWith("/items") + ? Response.json( + { error: { code: "item_not_found", message: "support-golden v3 has no item" } }, + { status: 422 }, + ) + : undefined, + }); + let started = 0; + + const error = await experiment(api, { + concurrency: 1, + task: () => { + started += 1; + + return "ok"; + }, + }).catch((caught: unknown) => caught); + + expect(error).toBeInstanceOf(ExperimentError); + expect(error).toMatchObject({ code: "item_not_found", status: 422 }); + // The one worker waits for the post of the first 100, which fails, so no item starts after it. + expect(started).toBe(100); + // The complete ends the experiment, which would else stay running. + expect(routes(api.requests).slice(-2)).toEqual([ + `POST /v1/experiments/${EXPERIMENT_ID}/items`, + `POST /v1/experiments/${EXPERIMENT_ID}/complete`, + ]); +}); + +test("a complete that fails after a failed run goes to onError, and the run rejects with the first error", async () => { + const errors: Error[] = []; + setup({ environment: "experiment", onError: (error) => errors.push(error) }); + const api = fakeApi({ + override: (request) => + request.url.pathname.endsWith("/items") || request.url.pathname.endsWith("/complete") + ? Response.json( + { error: { code: "invalid_api_key", message: "The API key was revoked" } }, + { status: 401 }, + ) + : undefined, + }); + + await expect(experiment(api)).rejects.toMatchObject({ + code: "invalid_api_key", + message: "The API key was revoked", + }); + expect(routes(api.requests).filter((route) => route.endsWith("/complete"))).toHaveLength(1); + expect(errors.map((error) => (error as ExperimentError).code)).toEqual(["invalid_api_key"]); +}); + +test("an item post that fails with 500 goes again, and the server skips the runs it has", async () => { + let failures = 1; + const api = fakeApi({ + override: (request) => + request.url.pathname.endsWith("/items") && failures-- > 0 + ? new Response("upstream", { status: 500 }) + : undefined, + }); + + await experiment(api); + + expect(api.posts()).toHaveLength(2); + expect(api.posts()[1]).toEqual(api.posts()[0]); +}); + +test("a post of items holds at most 4 MiB of JSON", async () => { + const api = fakeApi({ items: Array.from({ length: 20 }, (_, index) => datasetItem(index + 1)) }); + + // 100,000 characters of 3 UTF-8 bytes: about 300 KB of JSON for each item run. + await experiment(api, { task: () => "中".repeat(100_000) }); + + expect(api.posts().map((items) => items.length)).toEqual([13, 7]); +}); + +test.each([ + ["10,000 items in pages run", 10_000, undefined], + ["10,001 items in pages stop the run", 10_001, "invalid_response"], +])("%s", async (_, count, code) => { + const api = fakeApi({ + items: Array.from({ length: count }, (_, index) => datasetItem(index + 1)), + pageSize: 5_000, + }); + + const outcome = await experiment(api, { concurrency: 20, task: () => "ok" }).catch( + (caught: unknown) => caught, + ); + + expect((outcome as ExperimentError).code).toBe(code); + expect(api.posts().flat()).toHaveLength(10_000); + expect(routes(api.requests).filter((route) => route.endsWith("/complete"))).toHaveLength(1); +}); + +test.each([ + ["a dataset page", "GET /v1/datasets/support-golden", "timeout", 2, undefined], + ["an item post", `POST /v1/experiments/${EXPERIMENT_ID}/items`, "network", 2, undefined], + ["the create", "POST /v1/experiments", "timeout", 1, "timeout"], + ["the create", "POST /v1/experiments", "network", 1, "network_error"], +])("a %s %s, after a %s, sends %i times in all", async (_, route, failure, sends, code) => { + vi.useFakeTimers(); + + try { + let failures = 1; + const api = fakeApi({ + override: (request) => { + if (`${request.method} ${request.url.pathname}` !== route || failures-- <= 0) { + return undefined; + } + + if (failure === "network") return Promise.reject(new TypeError("fetch failed")); + + // A server that never answers: the request ends when its timeout aborts it. + return new Promise((_, reject) => + request.signal!.addEventListener("abort", () => reject(request.signal!.reason)), + ); + }, + }); + + const outcome = experiment(api).catch((caught: unknown) => caught); + await vi.advanceTimersByTimeAsync(60_000); + + expect(((await outcome) as ExperimentError).code).toBe(code); + expect(routes(api.requests).filter((sent) => sent === route)).toHaveLength(sends); + } finally { + vi.useRealTimers(); + } +}); + +test.each([ + ["500", () => new Response("upstream", { status: 500 }), "http_500"], + // The server answers 503 when its database work times out, which can be after the commit. + [ + "503", + () => + Response.json( + { error: { code: "evals_unavailable", message: "Evals are unavailable right now" } }, + { status: 503 }, + ), + "evals_unavailable", + ], +])( + "a create that fails with %s does not go again, so it cannot start two experiments", + async (status, reply, code) => { + const api = fakeApi({ + override: (request) => + request.method === "POST" && request.url.pathname === "/v1/experiments" + ? reply() + : undefined, + }); + + await expect(experiment(api)).rejects.toMatchObject({ code, status: Number(status) }); + expect(routes(api.requests)).toEqual([ + "GET /v1/datasets/support-golden", + "POST /v1/experiments", + ]); + }, +); + +test("a create that gets a 429 goes again after its Retry-After", async () => { + let limited = 1; + const api = fakeApi({ + override: (request) => + request.method === "POST" && request.url.pathname === "/v1/experiments" && limited-- > 0 + ? Response.json( + { error: { code: "rate_limited", message: "Too many requests for this project" } }, + { status: 429, headers: { "retry-after": "2" } }, + ) + : undefined, + }); + const started = Date.now(); + + await experiment(api); + + expect(routes(api.requests).filter((route) => route === "POST /v1/experiments")).toHaveLength(2); + // Without the header, the first retry waits 1 second. + expect(Date.now() - started).toBeGreaterThanOrEqual(2_000); +}); + +test("waits until the server evaluators end the experiment", async () => { + const api = fakeApi({ results: [result({ status: "running", itemsDone: 1 }), result()] }); + + const outcome = await experiment(api); + + expect(outcome.status).toBe("completed"); + expect( + routes(api.requests).filter((route) => route.startsWith("GET /v1/experiments/")), + ).toHaveLength(2); +}); + +test("gives up when the experiment does not end in 30 minutes, with the result", async () => { + vi.useFakeTimers(); + + try { + const polls: number[] = []; + const api = fakeApi({ + override: (request) => { + if (request.method === "GET" && request.url.pathname.startsWith("/v1/experiments/")) { + polls.push(Date.now()); + + return Response.json(result({ status: "running", itemsDone: 1 })); + } + + return undefined; + }, + }); + + const outcome = experiment(api).catch((caught: unknown) => caught); + await vi.advanceTimersByTimeAsync(31 * 60_000); + const error = (await outcome) as ExperimentError; + + expect(error.code).toBe("timeout"); + expect(error.result?.status).toBe("running"); + // The last poll is the one whose next wait of 5 seconds would end past the 30 minutes. + const waited = polls.at(-1)! - polls[0]!; + expect(waited).toBeGreaterThan(30 * 60_000 - 5_000); + expect(waited).toBeLessThanOrEqual(30 * 60_000); + } finally { + vi.useRealTimers(); + } +}); + +test.each([ + [ + "the API error", + { error: { code: "dataset_not_found", message: "No dataset named support-golden" } }, + 404, + "dataset_not_found", + ], + [ + "a page that cannot be read", + { dataset: { version: 1 }, items: [{ id: 1 }] }, + 200, + "invalid_response", + ], + [ + "an empty dataset", + { dataset: { id: "d", name: "support-golden", version: 0 }, items: [], nextCursor: null }, + 200, + "dataset_empty", + ], +])("rejects with %s before it creates an experiment", async (_, body, status, code) => { + const api = fakeApi({ + override: (request) => + request.url.pathname.startsWith("/v1/datasets/") + ? Response.json(body, { status }) + : undefined, + }); + + await expect(experiment(api)).rejects.toMatchObject({ code }); + expect(routes(api.requests)).toEqual(["GET /v1/datasets/support-golden"]); +}); + +test("a cursor that repeats is an invalid response, not an endless fetch", async () => { + const api = fakeApi({ + override: (request) => + request.url.pathname.startsWith("/v1/datasets/") + ? Response.json({ + dataset: { id: "d", name: "support-golden", version: 3 }, + items: [datasetItem(1)], + nextCursor: "same", + }) + : undefined, + }); + + await expect(experiment(api)).rejects.toMatchObject({ code: "invalid_response" }); + expect( + routes(api.requests).filter((route) => route.startsWith("GET /v1/datasets/")), + ).toHaveLength(2); +}); + +const failedCheck = result({ + checkStatus: "failed", + scores: [ + { + name: "correctness", + kind: "pass_fail", + value: 0.85, + passCount: 17, + failCount: 3, + baselineValue: 0.9, + delta: -5, + minScore: 0.9, + passed: false, + }, + { + name: "length", + kind: null, + value: null, + passCount: 0, + failCount: 0, + baselineValue: null, + delta: null, + minScore: null, + passed: null, + }, + ], +}); + +test("in CI mode a failed check rejects with the result", async () => { + vi.stubEnv("CI", "true"); + const api = fakeApi({ results: [failedCheck] }); + + const error = (await experiment(api, { minScores: { correctness: 0.9 } }).catch( + (caught: unknown) => caught, + )) as ExperimentError; + + expect(error).toBeInstanceOf(ExperimentError); + expect(error.code).toBe("check_failed"); + expect(error.message).toBe( + `The check of support-golden-v3 failed: correctness is 85%, under the pass mark 90%. ${URL_OF_EXPERIMENT}`, + ); + expect(error.result?.scores[1]?.kind).toBeNull(); +}); + +test.each([ + ["CI is off", {}, undefined], + ["ci: false wins over CI", { ci: false }, "true"], +])("a failed check resolves when %s", async (_, options, ci) => { + if (ci) vi.stubEnv("CI", ci); + const api = fakeApi({ results: [failedCheck] }); + + await expect(experiment(api, options)).resolves.toMatchObject({ checkStatus: "failed" }); +}); + +test("a result score that leaves out its null fields reads them as null", async () => { + const api = fakeApi({ + results: [result({ scores: [{ name: "length", passCount: 0, failCount: 0 }] })], + }); + + const outcome = await experiment(api); + + expect(outcome.scores).toEqual([ + { + name: "length", + kind: null, + value: null, + passCount: 0, + failCount: 0, + baselineValue: null, + delta: null, + minScore: null, + passed: null, + }, + ]); +}); + +test("a result without checkStatus is an invalid response, not a run with no check", async () => { + const { checkStatus: _, ...missing } = failedCheck; + const api = fakeApi({ results: [missing] }); + + await expect(experiment(api)).rejects.toMatchObject({ code: "invalid_response" }); +}); + +test("in CI mode an experiment that fails rejects with its error", async () => { + vi.stubEnv("CI", "1"); + const api = fakeApi({ + results: [result({ status: "failed", error: "0 of 2 items ran. Each one failed." })], + }); + + await expect(experiment(api)).rejects.toMatchObject({ + code: "experiment_failed", + message: `The experiment support-golden-v3 failed: 0 of 2 items ran. Each one failed.. ${URL_OF_EXPERIMENT}`, + }); +}); + +test("in GitHub Actions the experiment records the pull request, its head commit, and the run", async () => { + const dir = mkdtempSync(join(tmpdir(), "td-experiment-")); + const event = join(dir, "event.json"); + const results = join(dir, "results.jsonl"); + writeFileSync( + event, + JSON.stringify({ + pull_request: { + number: 42, + title: "Shorter replies", + html_url: "https://github.com/acme/support/pull/42", + head: { sha: "a1b2c3d4e5f6a1b2c3d4e5f6a1b2c3d4e5f6a1b2" }, + }, + }), + ); + writeFileSync(results, ""); + + vi.stubEnv("CI", "true"); + vi.stubEnv("GITHUB_ACTIONS", "true"); + vi.stubEnv("GITHUB_REPOSITORY", "acme/support"); + vi.stubEnv("GITHUB_SHA", "ffffffffffffffffffffffffffffffffffffffff"); + vi.stubEnv("GITHUB_EVENT_PATH", event); + vi.stubEnv("GITHUB_RUN_ID", "991"); + vi.stubEnv("TELEMETRY_DEV_EXPERIMENT_DATASET", "support-golden"); + vi.stubEnv("TELEMETRY_DEV_EXPERIMENT_MIN_SCORES", "correctness=0.9\nlength = 12.5,__proto__=1"); + vi.stubEnv("TELEMETRY_DEV_EXPERIMENT_RESULTS", results); + const api = fakeApi(); + + await experiment(api, { dataset: undefined, minScores: { correctness: 0.5, tone: 2 } }); + + expect(api.requests[1]!.body).toMatchObject({ + dataset: "support-golden", + minScores: JSON.parse('{"correctness":0.9,"tone":2,"length":12.5,"__proto__":1}'), + ci: { + provider: "github_actions", + repository: "acme/support", + commit: "a1b2c3d4e5f6a1b2c3d4e5f6a1b2c3d4e5f6a1b2", + prNumber: 42, + prTitle: "Shorter replies", + prUrl: "https://github.com/acme/support/pull/42", + runUrl: "https://github.com/acme/support/actions/runs/991", + }, + }); + expect(readFileSync(results, "utf8")).toBe(`${JSON.stringify(result())}\n`); +}); + +test("a pull request title is cut to 256 UTF-16 units without half of an emoji", async () => { + const dir = mkdtempSync(join(tmpdir(), "td-experiment-")); + const event = join(dir, "event.json"); + // GitHub counts the emoji as one character, thus the title fits its limit of 256. + writeFileSync(event, JSON.stringify({ pull_request: { title: `${"a".repeat(255)}😀` } })); + vi.stubEnv("CI", "true"); + vi.stubEnv("GITHUB_ACTIONS", "true"); + vi.stubEnv("GITHUB_REPOSITORY", "acme/support"); + vi.stubEnv("GITHUB_SHA", "a1b2c3d"); + vi.stubEnv("GITHUB_EVENT_PATH", event); + const api = fakeApi(); + + await experiment(api); + + expect(api.requests[1]!.body.ci.prTitle).toBe("a".repeat(255)); +}); + +test.each([ + [ + "no dataset", + {}, + { dataset: undefined }, + "Pass dataset, or set the dataset input of the experiment action", + ], + [ + "a dataset that the action does not name", + { TELEMETRY_DEV_EXPERIMENT_DATASET: "support-v2" }, + {}, + "dataset is support-golden, but the experiment action names support-v2. Name one dataset.", + ], + [ + "min-scores that do not parse", + { TELEMETRY_DEV_EXPERIMENT_MIN_SCORES: "correctness=0x10" }, + {}, + 'TELEMETRY_DEV_EXPERIMENT_MIN_SCORES: "correctness=0x10" is not a name=number pair', + ], + ["runsPerItem 6", {}, { runsPerItem: 6 }, "runsPerItem must be an integer from 1 to 5"], + ["concurrency 0", {}, { concurrency: 0 }, "concurrency must be an integer from 1 to 100"], + [ + "a scorer named user_feedback", + {}, + { scorers: { user_feedback: () => 1 } }, + "user_feedback is the score of user feedback. Name the scorer another way.", + ], + [ + "a scorer with the name of an evaluator", + {}, + { evaluators: ["correctness"], scorers: { Correctness: () => true } }, + "The scorer Correctness has the name of an evaluator. Rename one of them.", + ], +])("rejects %s before any request", async (_, env, options, message) => { + for (const [name, value] of Object.entries(env)) vi.stubEnv(name, value); + const api = fakeApi(); + + await expect(experiment(api, options as Partial)).rejects.toThrow(message); + expect(api.requests).toEqual([]); +}); + +test("without an API key it rejects with missing_api_key and sends nothing", async () => { + const api = fakeApi(); + + await expect(experiment(api, { apiKey: undefined })).rejects.toMatchObject({ + code: "missing_api_key", + }); + expect(api.requests).toEqual([]); +}); + +test("the task gets each run of an item of the split, with its run index", async () => { + const seen: ExperimentItem[] = []; + const api = fakeApi({ items: [datasetItem(7, "test")] }); + + await experiment(api, { + split: "test", + runsPerItem: 2, + concurrency: 1, + task: (item) => { + seen.push(item); + + return "ok"; + }, + }); + + expect(seen).toEqual([ + { ...datasetItem(7, "test"), runIndex: 0 }, + { ...datasetItem(7, "test"), runIndex: 1 }, + ]); + expect(api.requests[0]!.url.searchParams.get("split")).toBe("test"); + expect(api.requests[1]!.body.split).toBe("test"); +}); diff --git a/packages/sdk/tests/scores.test.ts b/packages/sdk/tests/scores.test.ts new file mode 100644 index 00000000..9b9dccc4 --- /dev/null +++ b/packages/sdk/tests/scores.test.ts @@ -0,0 +1,245 @@ +import { createServer, type RequestListener } from "node:http"; +import type { AddressInfo } from "node:net"; + +import { afterEach, beforeEach, expect, test, vi } from "vitest"; + +import { score, ScoreError, type ScoreOptions, shutdown } from "../src/index.ts"; +import { setup } from "./helpers.ts"; + +const TRACE_ID = "4bf92f3577b34da6a3ce929d0e0e4736"; +const SPAN_ID = "00f067aa0ba902b7"; + +function fakeFetch(reply: () => Response) { + const calls: { url: URL; init: RequestInit | undefined }[] = []; + + const fetch = vi.fn(async (input: string | URL, init?: RequestInit) => { + calls.push({ url: new URL(input), init }); + + return reply(); + }); + + return { fetch: fetch as unknown as typeof globalThis.fetch, calls }; +} + +async function serve(handler: RequestListener): Promise<{ url: string; close: () => void }> { + const server = createServer(handler); + await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve)); + const { port } = server.address() as AddressInfo; + + return { + url: `http://127.0.0.1:${port}`, + close: () => { + server.closeAllConnections(); + server.close(); + }, + }; +} + +function errorsOf() { + const errors: ScoreError[] = []; + setup({ onError: (error) => errors.push(error as ScoreError) }); + + return errors; +} + +beforeEach(() => { + vi.stubEnv("TELEMETRY_DEV_API_KEY", undefined); + vi.stubEnv("TELEMETRY_DEV_BASE_URL", undefined); +}); + +afterEach(async () => { + await shutdown(); + vi.unstubAllEnvs(); + vi.restoreAllMocks(); +}); + +test("posts the score with the key and base URL of init, and resolves with its id", async () => { + const { fetch, calls } = fakeFetch(() => Response.json({ id: "score-1" }, { status: 202 })); + setup({ apiKey: "td_live_key", baseUrl: "https://ingest.test/", fetch }); + + const id = await score({ + name: " user_feedback ", + traceId: TRACE_ID, + spanId: SPAN_ID, + value: 0, + comment: "Wrong refund amount", + }); + + expect(id).toBe("score-1"); + expect(calls).toHaveLength(1); + expect(calls[0]!.url.href).toBe("https://ingest.test/v1/scores"); + expect(calls[0]!.init).toMatchObject({ + method: "POST", + headers: { + authorization: "Bearer td_live_key", + "x-telemetry-dev-sdk": "@telemetry-dev/sdk", + "content-type": "application/json", + }, + }); + expect(JSON.parse(calls[0]!.init!.body as string)).toEqual({ + name: "user_feedback", + traceId: TRACE_ID, + spanId: SPAN_ID, + value: 0, + comment: "Wrong refund amount", + }); +}); + +test("calls the global fetch as a plain function, as Workers and browsers need", async () => { + vi.stubEnv("TELEMETRY_DEV_API_KEY", "td_live_key"); + // A strict function called as a method of an object gets that object as `this`. + vi.stubGlobal("fetch", function (this: unknown) { + if (this !== undefined && this !== globalThis) throw new TypeError("Illegal invocation"); + + return Promise.resolve(Response.json({ id: "score-3" }, { status: 202 })); + }); + + try { + await expect(score({ name: "helpful", traceId: TRACE_ID, value: 1 })).resolves.toBe("score-3"); + } finally { + vi.unstubAllGlobals(); + } +}); + +test("score with no options reports invalid_score and resolves undefined", async () => { + const errors = errorsOf(); + + await expect((score as (options?: ScoreOptions) => Promise)()).resolves.toBeUndefined(); + expect(errors.map((error) => [error.code, error.message])).toEqual([ + ["invalid_score", "name must be a string of 1 to 64 characters"], + ]); +}); + +test("a session score needs no trace, and a label goes trimmed", async () => { + const { fetch, calls } = fakeFetch(() => Response.json({ id: "score-2" }, { status: 202 })); + + await expect( + score({ apiKey: "td_live_key", fetch, name: "resolved", sessionId: "chat-9", label: " yes " }), + ).resolves.toBe("score-2"); + expect(JSON.parse(calls[0]!.init!.body as string)).toEqual({ + name: "resolved", + sessionId: "chat-9", + label: "yes", + }); +}); + +test.each<[string, Partial, string]>([ + ["no trace and no session", { traceId: undefined }, "Pass traceId or sessionId"], + [ + "the all-zero trace id of a span that did not record", + { traceId: "0".repeat(32) }, + "traceId must be 32 lowercase hex digits and not all zeros", + ], + [ + "an uppercase trace id", + { traceId: TRACE_ID.toUpperCase() }, + "traceId must be 32 lowercase hex digits and not all zeros", + ], + [ + "a span without its trace", + { traceId: undefined, sessionId: "s", spanId: SPAN_ID }, + "Pass the traceId of the span", + ], + ["no value and no label", { value: undefined }, "Pass a value or a label"], + ["a value that is not finite", { value: Number.NaN }, "value must be a finite number"], + [ + "a label for user feedback", + { name: "user_feedback", value: 1, label: "up" }, + "user_feedback takes the value 1 for up or 0 for down, and no label", + ], + [ + "user feedback of 0.5", + { name: "user_feedback", value: 0.5 }, + "user_feedback takes the value 1 for up or 0 for down, and no label", + ], + ["a pass of 0", { value: 0, label: "pass" }, "The label pass goes with the value 1"], + ["a fail of 1", { value: 1, label: "fail" }, "The label fail goes with the value 0"], + ["a blank name", { name: " " }, "name must be a string of 1 to 64 characters"], + [ + "a comment of 4,001 characters", + { comment: "x".repeat(4_001) }, + "comment must be a string of at most 4000 characters", + ], +])("reports invalid_score for %s and sends nothing", async (_, options, message) => { + const errors = errorsOf(); + const { fetch, calls } = fakeFetch(() => Response.json({ id: "never" })); + + await expect( + score({ + apiKey: "td_live_key", + fetch, + name: "helpful", + traceId: TRACE_ID, + value: 1, + ...options, + }), + ).resolves.toBeUndefined(); + expect(calls).toEqual([]); + expect(errors.map((error) => [error.code, error.message])).toEqual([["invalid_score", message]]); +}); + +test.each([ + [ + "the API error", + () => + Response.json( + { error: { code: "insufficient_scope", message: "This API key cannot send scores" } }, + { status: 403 }, + ), + ["insufficient_scope", 403], + ], + ["a status with no API error", () => new Response("down", { status: 503 }), ["http_503", 503]], + ["a body with no id", () => Response.json({}, { status: 202 }), ["invalid_response", undefined]], +])("reports %s, tries once, and resolves undefined", async (_, reply, [code, status]) => { + const errors = errorsOf(); + const { fetch, calls } = fakeFetch(reply); + + await expect( + score({ apiKey: "td_live_key", fetch, name: "helpful", traceId: TRACE_ID, value: 1 }), + ).resolves.toBeUndefined(); + expect(calls).toHaveLength(1); + expect(errors.map((error) => [error.code, error.status])).toEqual([[code, status]]); +}); + +test("without an API key it reports missing_api_key", async () => { + const errors = errorsOf(); + const { fetch, calls } = fakeFetch(() => Response.json({ id: "never" })); + + await score({ apiKey: "", fetch, name: "helpful", traceId: TRACE_ID, value: 1 }); + + expect(calls).toEqual([]); + expect(errors.map((error) => error.code)).toEqual(["missing_api_key"]); +}); + +test("does not follow a redirect, so the key stays with the API host", async () => { + const errors = errorsOf(); + let reached = 0; + const other = await serve((_, res) => { + reached += 1; + res.end(JSON.stringify({ id: "stolen" })); + }); + const api = await serve((_, res) => { + res.writeHead(307, { location: `${other.url}/v1/scores` }); + res.end(); + }); + + try { + await expect( + // The real fetch: setup() gives the client a stub. + score({ + apiKey: "td_live_key", + baseUrl: api.url, + fetch: globalThis.fetch, + name: "helpful", + traceId: TRACE_ID, + value: 1, + }), + ).resolves.toBeUndefined(); + } finally { + api.close(); + other.close(); + } + + expect(reached).toBe(0); + expect(errors.map((error) => [error.code, error.status])).toEqual([["http_307", 307]]); +}); diff --git a/sdks/python/README.md b/sdks/python/README.md index b785d104..bc53f67e 100644 --- a/sdks/python/README.md +++ b/sdks/python/README.md @@ -89,6 +89,8 @@ later `init()` generates a new ID. It is not persisted across processes. | `get_traceparent() -> str \| None` | W3C traceparent of the current context. | | `get_prompt(name, *, label=, version=, type="chat", cache_ttl_seconds=60, fallback=, api_key=, base_url=)` / `get_prompt_async(...)` | Fetch a prompt by label or version; see [Prompts](#prompts). | | `with_prompt(prompt)` | Context manager recording the prompt on each model call started inside. | +| `run_experiment(*, task, dataset=, name=, dataset_version=, split=, evaluators=, scorers=, min_scores=, runs_per_item=1, concurrency=4, metadata=, ci=, api_key=, base_url=)` / `run_experiment_async(...)` | Run your code on a dataset as an experiment; see [Experiments](#experiments). | +| `score(*, name, trace_id=, span_id=, session_id=, value=, label=, comment=, api_key=, base_url=)` / `score_async(...)` | Record a score on a trace, a span, or a session; see [Scores](#scores). | | `flush(timeout_s=10.0)` / `shutdown(timeout_s=10.0)` | Force-flush / tear down traces + logs + metrics. Shutdown also runs atexit unless `disable_atexit=True`. | | `TelemetrySpanProcessor` / `telemetry_dev.otel.create_telemetry_span_exporter` | Bring-your-own-OTel helpers (below). | | `MaskContext`, `Usage`, `SpanHandle`, `Client`, `NOT_GIVEN` | Supporting types. | @@ -255,6 +257,123 @@ other OpenTelemetry tracers do not get the attributes. A model call outside `wit get the attributes, even when it sends messages from `compile()`; with input capture on, the call still records those messages as its input. +## Experiments + +`run_experiment` runs your code on each item of a dataset that you manage in telemetry.dev, and +records the outputs and the scores as an experiment. The experiment page compares it with the +baseline of the dataset. + +```python +import telemetry_dev + +telemetry_dev.init(environment="experiment") + +result = telemetry_dev.run_experiment( + name="reply-v13-gpt-4o", + dataset="support-golden", + task=lambda item: reply_to_customer(item.input), + evaluators=["correctness", "helpfulness"], + scorers={"short": lambda output, item: len(output) < 600}, + min_scores={"correctness": 0.9}, +) +``` + +- `run_experiment` reads the newest version of the dataset, or `dataset_version`, 100 items at a + time. With `split`, it runs only the items of that split. Then it makes the experiment, with a + name that telemetry.dev picks when you give no `name`. +- `task(item)` gets an `ExperimentItem`: `id`, `number`, `input`, `expected_output`, `metadata`, + `tags`, `split`, and `run_index`. Its return value is the output. A string stays as it is, and a + different value becomes JSON, cut to 100,000 UTF-16 units. A value that JSON cannot hold, such + as a dict that holds itself, gives no output. A task that raises fails only its item, and the + item records the type and the message of the exception. +- `concurrency` (1 to 100, default 4) threads run the items, each in a copy of the context of the + caller, so `propagate_attributes` around the call applies to each item. With `runs_per_item` + (default 1, at most 5), each item runs more than one time. Each run gets its own copy of the + item, so a task that changes its input does not change the next run. The pass rate and the mean + of each score use all runs. +- `run_experiment_async` takes async tasks and scorers. `concurrency` asyncio tasks run the items, + and the requests go in worker threads. A task that is not async runs on the event loop, so use + `run_experiment` for it. `run_experiment` raises `TypeError` for an async task or scorer. +- Each item run is a trace of its own, with the span `experiment `. The SDK puts + `td.experiment.id` and `td.dataset.item_id` on each span of the run, also on the spans of the + provider integrations. Each run also gets its own session id, + `::`, as `gen_ai.conversation.id`. +- Call `init(environment="experiment")` before the run. Pages, metrics, and alerts leave out the + `experiment` environment by default. In a different environment, `run_experiment` logs a + warning. Without an enabled client, the items record no traces and no cost. +- `evaluators` names evaluators of the project: LLM judges, code checks, and rules. telemetry.dev + runs them on the outputs. `scorers` run on your machine, after the span of the item ends. A + scorer returns `True` or `False` for a pass or a fail, a number, a label, or a mapping with + `value`, `label`, and `reason`. A scorer that raises, or returns a score that is not valid, gives + no score for that item and sends a `scorer_failed` error to `on_error`. A scorer cannot have the + name of an evaluator of the run, or the name `user_feedback`. +- `run_experiment` sends the outputs in batches of 100 items at most, and then waits up to 30 + minutes for the server evaluators. Then it prints the scores and returns an `ExperimentResult`: + `status`, `check_status`, item counts, `scores` with the value, the baseline value, the change, + and the pass mark of each score, and the `url` of the experiment page. +- `min_scores` sets a pass mark for each score name: the lowest pass rate (0 to 1) of a pass/fail + score, or the lowest mean of a number score. A score under its mark fails the check. +- A failed request raises `ExperimentError` whose `code` tells why, for example + `dataset_not_found`, `dataset_empty`, `evaluator_not_found`, `insufficient_scope`, + `missing_api_key`, or `timeout`. A read of the dataset or the result, a post of outputs, and the + completion go again up to 2 times after a timeout, a network error, a 429, or a 5xx. The server + skips an item run that it has. The request that makes the experiment goes again only after a + 429, thus a lost response or a 503 cannot make a second experiment. When the run stops after it + made the experiment, also after Ctrl-C or a cancellation, `run_experiment` sends the completion + before it raises: telemetry.dev then ends the experiment with the item runs that it got, or fails + it when it got none. +- The key needs the `datasets:read` and `experiments:write` scopes. `run_experiment` uses the same + API key and base URL as `get_prompt`, or its own `api_key` and `base_url`. With + `environment="experiment"`, the key of the telemetry client must be for all environments or for + `experiment`: a key for another environment cannot send the traces of the items, and the items + then have no trace and no cost. + +The outputs, the exceptions of failed tasks, the scorer reasons, and `metadata` go to telemetry.dev +and stay with the experiment, because they are its result. `capture_output`, `mask`, and the +capture setting of the environment in the project do not apply to them. They apply only to the +spans of the run. + +### CI mode + +When the `CI` variable is `true` or `1`, or with `ci=True`, `run_experiment` raises when the check +fails (`check_failed`) or when the experiment does not complete (`experiment_failed`), so the +process exits with an error. The error has the `result`. In GitHub Actions, the experiment also +records the repository, the commit, the run URL, and the number, the title, and the URL of the pull +request. The commit of a pull request is its head commit. + +The [experiment action](../../actions/experiment) runs your eval script in a workflow, and posts +the result on the pull request. It gives `run_experiment` the `dataset` input in +`TELEMETRY_DEV_EXPERIMENT_DATASET`, which must agree with the `dataset` of the script, and the +`min-scores` input in `TELEMETRY_DEV_EXPERIMENT_MIN_SCORES`, which wins over `min_scores` for the +same name. + +## Scores + +`score` records a score on a trace, a span, or a session, for example the thumbs up or down of a +user on a reply: + +```python +import telemetry_dev + +# When the app replies, keep the trace id of the reply with the message: +with telemetry_dev.start_span("reply") as span: + text = reply_to_customer(message) + trace_id = f"{span.span.get_span_context().trace_id:032x}" + +# Later, when the user clicks thumbs down: +telemetry_dev.score(name="user_feedback", trace_id=trace_id, value=0, comment="Wrong amount") +``` + +- Give `trace_id`, `session_id`, or the two, and a `span_id` of the trace for a score on one span. + Give `value`, `label`, or the two. `user_feedback` takes the value 1 for up or 0 for down, and + no label. A pass/fail score has the label `pass` with the value 1, or `fail` with 0. +- Like the telemetry functions, `score` never raises. It returns the id of the score, or `None` + after it sends a `ScoreError` to `on_error`. It checks the score before it sends it + (`invalid_score`). The trace id of a span from a client that is not enabled is all zeros, and + `score` reports it as `invalid_score`. +- `score` makes one request, with no retry. `score_async` runs it in a worker thread. The key + needs the `scores:write` scope. + ## Auto-metrics Ended spans automatically record histograms (DELTA temporality, exported every 60s): diff --git a/sdks/python/src/telemetry_dev/__init__.py b/sdks/python/src/telemetry_dev/__init__.py index 9ed7aabf..bbcb2307 100644 --- a/sdks/python/src/telemetry_dev/__init__.py +++ b/sdks/python/src/telemetry_dev/__init__.py @@ -17,6 +17,18 @@ from ._config import SDK_VERSION as __version__ from ._config import LogLevelOption from ._context import get_traceparent, propagate_attributes +from ._experiments import ( + AsyncScorer, + DatasetItem, + ExperimentError, + ExperimentItem, + ExperimentResult, + ExperimentScore, + Scorer, + ScorerResult, + run_experiment, + run_experiment_async, +) from ._logs import log from ._observe import observe from ._prompts import ( @@ -29,6 +41,7 @@ get_prompt_async, with_prompt, ) +from ._scores import ScoreError, score, score_async from ._semconv import LogLevel, SpanType from ._serialize import AttributeValue, Mask, MaskContext from ._spans import ( @@ -42,10 +55,16 @@ __all__ = [ "NOT_GIVEN", + "AsyncScorer", "AttributeValue", "CaptureBudget", "ChatPrompt", "Client", + "DatasetItem", + "ExperimentError", + "ExperimentItem", + "ExperimentResult", + "ExperimentScore", "LogLevel", "LogLevelOption", "Mask", @@ -53,6 +72,9 @@ "Prompt", "PromptError", "PromptModel", + "ScoreError", + "Scorer", + "ScorerResult", "SpanHandle", "SpanType", "TelemetrySpanProcessor", @@ -68,6 +90,10 @@ "log", "observe", "propagate_attributes", + "run_experiment", + "run_experiment_async", + "score", + "score_async", "shutdown", "start_span", "update_current_span", diff --git a/sdks/python/src/telemetry_dev/_api.py b/sdks/python/src/telemetry_dev/_api.py new file mode 100644 index 00000000..5dc30652 --- /dev/null +++ b/sdks/python/src/telemetry_dev/_api.py @@ -0,0 +1,203 @@ +"""Calls to the telemetry.dev API: the key and base URL, one JSON request, and its errors.""" + +from __future__ import annotations + +import http.client +import json +import re +import time +import urllib.error +import urllib.parse +import urllib.request +from collections.abc import Callable, Mapping +from dataclasses import dataclass +from datetime import datetime, timezone +from email.message import Message +from email.utils import parsedate_to_datetime +from typing import IO, Any, Literal, TypeVar, cast + +from ._client import get_client +from ._config import SDK_VERSION, resolve_config +from ._serialize import json_dumps + +# A longer Retry-After waits only this long: the API counts requests by the minute. +_MAX_RETRY_AFTER_S = 60.0 +# IMF-fixdate, the HTTP date that servers send: Sun, 06 Nov 1994 08:49:37 GMT. +_HTTP_DATE = re.compile(r"[A-Z][a-z]{2}, \d{2} [A-Z][a-z]{2} \d{4} \d{2}:\d{2}:\d{2} GMT", re.ASCII) + + +# An error class with the shape of PromptError: a code, a message, and the status. +E = TypeVar("E", bound=Exception) + + +@dataclass(frozen=True) +class Api: + api_key: str | None + base_url: str + on_error: Callable[[BaseException], None] | None + + +def resolve_api(api_key: str | None, base_url: str | None) -> Api: + """The key and base URL of a call: its arguments, then the enabled client, then the env.""" + client = get_client() + enabled = client if client is not None and client.enabled else None + config = enabled.config if enabled is not None else resolve_config() + return Api( + api_key=api_key if api_key is not None else config.api_key, + base_url=(base_url or config.base_url).rstrip("/"), + on_error=enabled.on_error if enabled is not None else None, + ) + + +@dataclass(frozen=True) +class _Failure: + error: Exception + status: int | None + retry_after: float | None + + +def request( + api: Api, + error_type: Callable[[str, str, int | None], E], + method: Literal["GET", "POST"], + path: str, + *, + timeout_s: float, + idempotent: bool, + body: object = None, + query: Mapping[str, str | None] | None = None, + attempts: int = 3, +) -> Any: + """Send one JSON request and return the JSON of the response. + + A response that is not 2xx raises the ``{"error": {"code", "message"}}`` of its body, or + ``http_``. With ``idempotent``, the server takes a repeat at most once, so a timeout, a + network error, or a 5xx tries again. Other requests try again only after a 429, which the rate + limit answers before the server does any work: a 503 can come after the work committed. A + retry waits for the Retry-After of the response, or for 1 and then 2 seconds. The timeout + applies to each socket operation of urllib. + """ + url = f"{api.base_url}{path}" + params = {key: value for key, value in (query or {}).items() if value is not None} + if params: + url = f"{url}?{urllib.parse.urlencode(params)}" + headers = auth_headers(api.api_key) + data = None + if body is not None: + # The JSON of JSON.stringify, which escapes a lone surrogate that UTF-8 cannot hold. + data = json_dumps(body, default=None).encode() + headers["Content-Type"] = "application/json" + attempt = 1 + while True: + result = _send(error_type, method, url, path, data, headers, timeout_s) + if not isinstance(result, _Failure): + return result + if attempt >= attempts or not _retryable(result.status, idempotent): + raise result.error + time.sleep(min(result.retry_after or float(attempt), _MAX_RETRY_AFTER_S)) + attempt += 1 + + +def auth_headers(api_key: str | None) -> dict[str, str]: + return { + "Authorization": f"Bearer {api_key}", + "X-Telemetry-Dev-SDK": "telemetry-dev", + # urllib's default User-Agent, Python-urllib/3.x, is refused by bot filters such as + # Cloudflare's Browser Integrity Check. + "User-Agent": f"telemetry-dev-python/{SDK_VERSION}", + } + + +class NoRedirect(urllib.request.HTTPRedirectHandler): + # A redirect to another host must not get the API key, so a 3xx fails like any status that is + # not 2xx. + def redirect_request( + self, + req: urllib.request.Request, + fp: IO[bytes], + code: int, + msg: str, + headers: Message, + newurl: str, + ) -> urllib.request.Request | None: + return None + + +opener = urllib.request.build_opener(NoRedirect) + + +def api_error(error_type: Callable[[str, str, int | None], E], status: int, body: bytes) -> E: + """The error of a response that is not 2xx: its API code and message, or ``http_``.""" + try: + error = json.loads(body).get("error") + except (ValueError, AttributeError): + error = None + fields = cast("dict[str, Any]", error) if isinstance(error, dict) else {} + code = fields.get("code") + message = fields.get("message") + return error_type( + code if isinstance(code, str) else f"http_{status}", + message if isinstance(message, str) else f"telemetry.dev returned {status}", + status, + ) + + +def retry_after(status: int, value: str | None) -> float | None: + """The seconds that the Retry-After of a 429 or a 503 asks for: seconds or an HTTP date.""" + value = (value or "").strip() + if status not in (429, 503) or not value: + return None + if value.isascii() and value.isdigit(): + seconds = float(value) + elif _HTTP_DATE.fullmatch(value): + try: + date = parsedate_to_datetime(value) + except (TypeError, ValueError): + return None + seconds = (date - datetime.now(timezone.utc)).total_seconds() + else: + return None + return seconds if seconds > 0 else None + + +def _send( + error_type: Callable[[str, str, int | None], E], + method: str, + url: str, + path: str, + data: bytes | None, + headers: dict[str, str], + timeout_s: float, +) -> Any: + request = urllib.request.Request(url, data=data, headers=headers, method=method) + try: + with opener.open(request, timeout=timeout_s) as response: + body: bytes = response.read() + except urllib.error.HTTPError as exc: + with exc: + return _Failure( + api_error(error_type, exc.code, exc.read()), + exc.code, + retry_after(exc.code, exc.headers.get("Retry-After")), + ) + except (urllib.error.URLError, OSError, http.client.HTTPException) as exc: + timed_out = isinstance(exc, TimeoutError) or isinstance( + getattr(exc, "reason", None), TimeoutError + ) + error = ( + error_type("timeout", f"{method} {path} took more than {timeout_s:g} s", None) + if timed_out + else error_type("network_error", "Could not reach telemetry.dev", None) + ) + error.__cause__ = exc + return _Failure(error, None, None) + try: + return json.loads(body) + except ValueError: + return None + + +def _retryable(status: int | None, idempotent: bool) -> bool: + if status == 429: + return True + return idempotent and (status is None or status >= 500) diff --git a/sdks/python/src/telemetry_dev/_ci.py b/sdks/python/src/telemetry_dev/_ci.py new file mode 100644 index 00000000..e27701f1 --- /dev/null +++ b/sdks/python/src/telemetry_dev/_ci.py @@ -0,0 +1,136 @@ +"""The GitHub Actions run of an experiment, and the variables of the experiment action.""" + +from __future__ import annotations + +import json +import os +import re +from collections.abc import Callable, Mapping +from typing import Any, cast + +from ._config import report_error +from ._serialize import utf16_length + +_REPOSITORY = re.compile(r"[A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+") +_COMMIT = re.compile(r"[0-9a-fA-F]{7,64}") +_HTTPS_URL = re.compile(r"https://\S+") +# A plain decimal, so that the TypeScript and Python SDKs read the same marks. +_DECIMAL = re.compile(r"-?\d+(\.\d+)?", re.ASCII) +_RUN_ID = re.compile(r"\d+", re.ASCII) +_MAX_REPOSITORY_LENGTH = 200 +_MAX_URL_LENGTH = 2048 +_MAX_PR_TITLE_LENGTH = 256 + + +def in_ci(env: Mapping[str, str]) -> bool: + """CI mode is on when ``CI`` is ``true`` or ``1``, as CI services set it.""" + return env.get("CI") in ("true", "1") + + +def github_run(env: Mapping[str, str]) -> dict[str, Any] | None: + """The run that GitHub Actions describes in its variables, or None outside GitHub Actions. + + On a pull request the commit is the head of the pull request: ``GITHUB_SHA`` is then a merge + commit that nobody pushed. + """ + repository = env.get("GITHUB_REPOSITORY") or "" + if ( + env.get("GITHUB_ACTIONS") != "true" + or len(repository) > _MAX_REPOSITORY_LENGTH + or not _REPOSITORY.fullmatch(repository) + ): + return None + pull = _pull_request(env.get("GITHUB_EVENT_PATH")) + commit = pull.get("commit") or env.get("GITHUB_SHA") + if not isinstance(commit, str) or not _COMMIT.fullmatch(commit): + return None + server = (env.get("GITHUB_SERVER_URL") or "https://github.com").rstrip("/") + run_id = env.get("GITHUB_RUN_ID") or "" + run: dict[str, Any] = { + "provider": "github_actions", + "repository": repository, + "commit": commit, + "prNumber": pull.get("number"), + "prTitle": pull.get("title"), + "prUrl": pull.get("url"), + "runUrl": _https_url(f"{server}/{repository}/actions/runs/{run_id}") + if _RUN_ID.fullmatch(run_id) + else None, + } + return {key: value for key, value in run.items() if value is not None} + + +def min_scores_from_env(env: Mapping[str, str]) -> dict[str, float]: + """The pass marks in ``TELEMETRY_DEV_EXPERIMENT_MIN_SCORES``, which the experiment action + sets from its ``min-scores`` input: ``name=value`` pairs, separated by commas or new lines.""" + scores: dict[str, float] = {} + for pair in re.split(r"[,\n]", env.get("TELEMETRY_DEV_EXPERIMENT_MIN_SCORES") or ""): + if not pair.strip(): + continue + parts = [part.strip() for part in pair.split("=")] + if len(parts) != 2 or not parts[0] or not _DECIMAL.fullmatch(parts[1]): + raise TypeError( + f'TELEMETRY_DEV_EXPERIMENT_MIN_SCORES: "{pair.strip()}" is not a name=number pair' + ) + scores[parts[0]] = float(parts[1]) + return scores + + +def write_result( + env: Mapping[str, str], + result: object, + on_error: Callable[[BaseException], None] | None, +) -> None: + """Add the result to the file in ``TELEMETRY_DEV_EXPERIMENT_RESULTS`` as one line of JSON. + The experiment action reads the file to write the job summary and the pull request comment.""" + path = env.get("TELEMETRY_DEV_EXPERIMENT_RESULTS") + if not path: + return + line = json.dumps(result, ensure_ascii=False, separators=(",", ":")) + "\n" + try: + fd = os.open(path, os.O_WRONLY | os.O_APPEND | os.O_CREAT, 0o600) + with os.fdopen(fd, "a", encoding="utf-8") as file: + file.write(line) + except OSError as exc: + report_error(on_error, "writing TELEMETRY_DEV_EXPERIMENT_RESULTS failed", exc) + + +def _pull_request(event_path: str | None) -> dict[str, Any]: + if not event_path: + return {} + try: + with open(event_path, encoding="utf-8") as file: + event = json.load(file) + except (OSError, ValueError): + return {} + pull = cast("dict[str, Any]", event).get("pull_request") if isinstance(event, dict) else None + if not isinstance(pull, dict): + return {} + fields = cast("dict[str, Any]", pull) + number = fields.get("number") + title = fields.get("title") + url = fields.get("html_url") + head = fields.get("head") + sha = cast("dict[str, Any]", head).get("sha") if isinstance(head, dict) else None + return { + "number": number + if isinstance(number, int) and not isinstance(number, bool) and number > 0 + else None, + "title": _cut(title, _MAX_PR_TITLE_LENGTH) if isinstance(title, str) else None, + "url": _https_url(url) if isinstance(url, str) else None, + "commit": sha if isinstance(sha, str) else None, + } + + +def _https_url(url: str) -> str | None: + return url if utf16_length(url) <= _MAX_URL_LENGTH and _HTTPS_URL.fullmatch(url) else None + + +def _cut(text: str, max_units: int) -> str: + """``text`` cut to ``max_units`` UTF-16 units, the length that the API counts, with no half of + a surrogate pair at the end: the API stores the run as jsonb, and Postgres rejects a lone + surrogate there.""" + encoded = text.encode("utf-16-le", "surrogatepass") + if len(encoded) <= max_units * 2: + return text + return encoded[: max_units * 2].decode("utf-16-le", "ignore") diff --git a/sdks/python/src/telemetry_dev/_experiments.py b/sdks/python/src/telemetry_dev/_experiments.py new file mode 100644 index 00000000..303dc89b --- /dev/null +++ b/sdks/python/src/telemetry_dev/_experiments.py @@ -0,0 +1,1138 @@ +"""Experiments of your code: run a task on each item of a dataset and record the result.""" + +from __future__ import annotations + +import asyncio +import contextvars +import copy +import inspect +import math +import os +import threading +import time +import urllib.parse +from collections.abc import Awaitable, Callable, Generator, Mapping, Sequence +from concurrent.futures import ThreadPoolExecutor +from dataclasses import dataclass +from time import perf_counter +from typing import Any, Literal, TypeAlias, TypeGuard, cast + +from opentelemetry import context as otel_context +from opentelemetry import trace +from opentelemetry.context import Context + +from ._api import Api, request, resolve_api +from ._ci import github_run, in_ci, min_scores_from_env, write_result +from ._client import flush, get_client +from ._config import logger, report_error +from ._context import context_with_propagated_attributes, propagated_attributes +from ._scores import ( + MAX_REASON_LENGTH, + MAX_SCORE_NAME_LENGTH, + USER_FEEDBACK, + check_value, + is_finite, +) +from ._semconv import ATTR_DATASET_ITEM_ID, ATTR_EXPERIMENT_ID, ATTR_SESSION_ID +from ._serialize import json_dumps, truncate, utf16_length +from ._spans import SpanHandle, start_span + +# The experiment environment, which pages, metrics, and alerts leave out by default. +_EXPERIMENT_ENVIRONMENT = "experiment" +_MAX_RUNS_PER_ITEM = 5 +_MAX_CONCURRENCY = 100 +# Server evaluators of one experiment, and scorers: an item takes at most 20 scores. +_MAX_EVALUATORS = 20 +_MAX_MIN_SCORES = 20 +_MAX_ERROR_LENGTH = 4000 +_MAX_OUTPUT_LENGTH = 100_000 +_MAX_DATASET_ITEMS = 10_000 +# Items in one POST /v1/experiments/{id}/items. +_MAX_BATCH_ITEMS = 100 +# Bytes of JSON in one post of items, well under the 16 MiB body limit of the API. +_MAX_BATCH_BYTES = 4 * 1024 * 1024 +_REQUEST_TIMEOUT_S = 30.0 +# How long run_experiment waits for the server evaluators after the last item. +_MAX_WAIT_S = 30 * 60.0 +_MAX_POLL_INTERVAL_S = 5.0 +_FINISHED = frozenset({"completed", "failed", "stopped"}) +_RATE_KINDS = frozenset({"pass_fail", "up_down"}) + + +@dataclass(frozen=True) +class DatasetItem: + """An item of a dataset version, as ``GET /v1/datasets/{name}`` returns it. ``id`` is the key + of the item: it stays the same across dataset versions.""" + + id: str + number: int + input: Mapping[str, Any] + expected_output: str | None + metadata: Mapping[str, Any] + tags: tuple[str, ...] + split: str | None + + +@dataclass(frozen=True) +class ExperimentItem(DatasetItem): + """One run of a dataset item, ``run_index`` 0 to ``runs_per_item`` - 1.""" + + run_index: int + + +ScorerResult: TypeAlias = "bool | int | float | str | Mapping[str, Any] | None" +"""What a scorer returns. True and False are a pass and a fail, a number is the value, and a +string is the label. A mapping can have ``value``, ``label``, and ``reason``. None gives no +score.""" + +Scorer: TypeAlias = "Callable[[Any, ExperimentItem], ScorerResult]" +"""Scores the output of an item on your machine. The score has the name of the scorer.""" + +AsyncScorer: TypeAlias = "Callable[[Any, ExperimentItem], ScorerResult | Awaitable[ScorerResult]]" +"""A scorer of ``run_experiment_async``, which can be async.""" + + +@dataclass(frozen=True) +class ExperimentScore: + """One score name of an experiment. ``value`` is the pass rate (0 to 1) of a pass/fail or + up/down score, or the mean of a number score. ``kind`` is None before the score has values. + ``baseline_value`` is the value in the experiment that this one compares with, by default the + dataset baseline. ``delta`` is whole points of a pass rate, or the change of a mean to one + decimal. ``passed`` is None when the score has no pass mark.""" + + name: str + kind: str | None + value: float | None + pass_count: int + fail_count: int + baseline_value: float | None + delta: float | None + min_score: float | None + passed: bool | None + + +@dataclass(frozen=True) +class ExperimentResult: + """The result of an experiment. ``status`` is ``completed``, ``failed``, or ``stopped``. + ``check_status`` is None when the experiment has no pass marks. ``error`` tells why the + experiment failed.""" + + id: str + name: str + status: str + check_status: Literal["passed", "failed"] | None + items_total: int + items_done: int + items_failed: int + error: str | None + scores: tuple[ExperimentScore, ...] + url: str + + +class ExperimentError(Exception): + """Why run_experiment failed. ``code`` is the code of the API error (``dataset_not_found``, + ``insufficient_scope``, ``invalid_api_key``, ...), or ``missing_api_key``, ``timeout``, + ``network_error``, ``invalid_response``, or ``dataset_empty``. In CI mode it is also + ``check_failed`` or ``experiment_failed``, and ``result`` holds the result. A response with a + different status and no API error gives ``http_``. A scorer that raises or returns a + value that is not a score sends a ``scorer_failed`` error to ``on_error``.""" + + def __init__( + self, + code: str, + message: str, + status: int | None = None, + *, + result: ExperimentResult | None = None, + ) -> None: + super().__init__(message) + self.code = code + self.message = message + self.status = status + self.result = result + + +def run_experiment( + *, + task: Callable[[ExperimentItem], Any], + dataset: str | None = None, + name: str | None = None, + dataset_version: int | None = None, + split: str | None = None, + evaluators: Sequence[str] | None = None, + scorers: Mapping[str, Scorer] | None = None, + min_scores: Mapping[str, float] | None = None, + runs_per_item: int = 1, + concurrency: int = 4, + metadata: Mapping[str, Any] | None = None, + ci: bool | None = None, + api_key: str | None = None, + base_url: str | None = None, +) -> ExperimentResult: + """Run ``task`` on each item of a dataset and record the result as an experiment. + + Each item run is its own trace, with ``td.experiment.id`` and ``td.dataset.item_id`` on each + span. The outputs go to telemetry.dev in batches of at most 100, with the scores of + ``scorers``. Then run_experiment waits for the ``evaluators`` of the server, prints the result, + and returns it. ``concurrency`` (1 to 100, default 4) threads run the items, each in a copy of + the context of the caller. + + ``dataset`` defaults to the ``dataset`` input of the experiment action, which it sets in + ``TELEMETRY_DEV_EXPERIMENT_DATASET``. When the two are set, they must be the same. + ``min_scores`` holds pass marks by score name: the lowest pass rate (0 to 1) of a pass/fail + score, or the lowest mean of a number score. The ``min-scores`` input of the action adds to + them and wins for the same name. ``runs_per_item`` is 1 to 5. + + The run cannot start without its dataset, so run_experiment raises ``ExperimentError`` when a + request fails. When the run stops after it made the experiment, it first sends the complete + request, so that telemetry.dev ends the experiment with the item runs that it has. A task that + raises fails only its item. In CI mode (``ci``, default on when ``CI`` is ``true`` or ``1``), + it records the GitHub Actions run, and it raises when the check fails or the experiment does + not complete, so the process exits with an error. + """ + run, api = _prepare( + task, + dataset, + name, + dataset_version, + split, + evaluators, + scorers, + min_scores, + runs_per_item, + concurrency, + metadata, + ci, + api_key, + base_url, + asynchronous=False, + ) + runs, created = _start(api, run) + try: + _announce(run, created) + _run_items(api, run, created, runs) + except BaseException: + _complete_after_failure(api, created) + raise + return _finish(api, run, created) + + +async def run_experiment_async( + *, + task: Callable[[ExperimentItem], Any], + dataset: str | None = None, + name: str | None = None, + dataset_version: int | None = None, + split: str | None = None, + evaluators: Sequence[str] | None = None, + scorers: Mapping[str, AsyncScorer] | None = None, + min_scores: Mapping[str, float] | None = None, + runs_per_item: int = 1, + concurrency: int = 4, + metadata: Mapping[str, Any] | None = None, + ci: bool | None = None, + api_key: str | None = None, + base_url: str | None = None, +) -> ExperimentResult: + """``run_experiment`` for async tasks and scorers. ``concurrency`` asyncio tasks run the + items, and the requests go in worker threads. A task that is not async runs on the event + loop, so it blocks the other items while it runs: use ``run_experiment`` for it.""" + run, api = _prepare( + task, + dataset, + name, + dataset_version, + split, + evaluators, + scorers, + min_scores, + runs_per_item, + concurrency, + metadata, + ci, + api_key, + base_url, + asynchronous=True, + ) + runs, created = await asyncio.to_thread(_start, api, run) + try: + _announce(run, created) + await _run_items_async(api, run, created, runs) + except BaseException: + await asyncio.to_thread(_complete_after_failure, api, created) + raise + return await asyncio.to_thread(_finish, api, run, created) + + +@dataclass(frozen=True) +class _Run: + dataset: str + task: Callable[[ExperimentItem], Any] + name: str | None + dataset_version: int | None + split: str | None + evaluators: list[str] | None + scorers: list[tuple[str, Callable[[Any, ExperimentItem], Any]]] + min_scores: dict[str, float] | None + runs_per_item: int + concurrency: int + metadata: dict[str, Any] | None + ci: dict[str, Any] | None + in_ci: bool + + +@dataclass(frozen=True) +class _Created: + id: str + name: str + items_total: int + url: str + + +@dataclass(frozen=True) +class _Page: + version: int + items: list[DatasetItem] + next_cursor: str | None + + +def _prepare( + task: object, + dataset: object, + name: object, + dataset_version: object, + split: object, + evaluators: object, + scorers: object, + min_scores: object, + runs_per_item: object, + concurrency: object, + metadata: object, + ci: object, + api_key: str | None, + base_url: str | None, + *, + asynchronous: bool, +) -> tuple[_Run, Api]: + env = os.environ + if not callable(task): + raise TypeError("task must be a function") + if not asynchronous and inspect.iscoroutinefunction(task): + raise TypeError("task is async: use run_experiment_async") + for key, value in (("dataset", dataset), ("name", name), ("split", split)): + if value is not None and (not isinstance(value, str) or not value): + raise TypeError(f"{key} must be a non-empty string") + action_dataset = env.get("TELEMETRY_DEV_EXPERIMENT_DATASET") or None + chosen = cast("str | None", dataset) if dataset is not None else action_dataset + if chosen is None: + raise TypeError("Pass dataset, or set the dataset input of the experiment action") + if action_dataset is not None and action_dataset != chosen: + raise TypeError( + f"dataset is {chosen}, but the experiment action names {action_dataset}. " + "Name one dataset." + ) + if dataset_version is not None and not (_is_int(dataset_version) and dataset_version >= 1): + raise ValueError("dataset_version must be a positive integer") + + evaluator_names: list[str] | None = None + if evaluators is not None: + names = ( + list(cast("Sequence[object]", evaluators)) + if isinstance(evaluators, Sequence) and not isinstance(evaluators, str) + else None + ) + if ( + names is None + or len(names) > _MAX_EVALUATORS + or not all(isinstance(n, str) and _valid_score_name(n) for n in names) + ): + raise TypeError( + f"evaluators must be a list of at most {_MAX_EVALUATORS} evaluator names" + ) + evaluator_names = cast("list[str]", names) + + if scorers is not None and not isinstance(scorers, Mapping): + raise TypeError("scorers must be a mapping of score names to functions") + scorer_list = list(cast("Mapping[object, object]", scorers or {}).items()) + if len(scorer_list) > _MAX_EVALUATORS: + raise TypeError(f"scorers can have at most {_MAX_EVALUATORS} scorers") + folded = {n.lower() for n in evaluator_names or []} + for scorer_name, scorer in scorer_list: + if ( + not isinstance(scorer_name, str) + or not _valid_score_name(scorer_name) + or not callable(scorer) + ): + raise TypeError( + f"scorers must map score names of 1 to {MAX_SCORE_NAME_LENGTH} characters " + "to functions" + ) + if not asynchronous and inspect.iscoroutinefunction(scorer): + raise TypeError(f"The scorer {scorer_name} is async: use run_experiment_async") + if scorer_name == USER_FEEDBACK: + raise TypeError( + f"{USER_FEEDBACK} is the score of user feedback. Name the scorer another way." + ) + if scorer_name.lower() in folded: + raise TypeError( + f"The scorer {scorer_name} has the name of an evaluator. Rename one of them." + ) + + if min_scores is not None and not isinstance(min_scores, Mapping): + raise TypeError(f"min_scores must map at most {_MAX_MIN_SCORES} score names to numbers") + marks: dict[object, object] = { + **cast("Mapping[object, object]", min_scores or {}), + **min_scores_from_env(env), + } + if len(marks) > _MAX_MIN_SCORES or not all( + isinstance(n, str) and _valid_score_name(n) and is_finite(v) for n, v in marks.items() + ): + raise TypeError(f"min_scores must map at most {_MAX_MIN_SCORES} score names to numbers") + + if not _is_int(runs_per_item) or not 1 <= runs_per_item <= _MAX_RUNS_PER_ITEM: + raise ValueError(f"runs_per_item must be an integer from 1 to {_MAX_RUNS_PER_ITEM}") + if not _is_int(concurrency) or not 1 <= concurrency <= _MAX_CONCURRENCY: + raise ValueError(f"concurrency must be an integer from 1 to {_MAX_CONCURRENCY}") + if metadata is not None and not isinstance(metadata, Mapping): + raise TypeError("metadata must be a mapping") + if ci is not None and not isinstance(ci, bool): + raise TypeError("ci must be a bool") + ci_mode = in_ci(env) if ci is None else ci + + api = resolve_api(api_key, base_url) + if not api.api_key: + raise ExperimentError( + "missing_api_key", "No API key: pass api_key or set TELEMETRY_DEV_API_KEY" + ) + run = _Run( + dataset=chosen, + task=task, + name=cast("str | None", name), + dataset_version=dataset_version, + split=cast("str | None", split), + evaluators=evaluator_names, + scorers=cast("list[tuple[str, Callable[[Any, ExperimentItem], Any]]]", scorer_list), + min_scores={str(n): float(cast("float", v)) for n, v in marks.items()} or None, + runs_per_item=runs_per_item, + concurrency=concurrency, + metadata=dict(cast("Mapping[str, Any]", metadata)) if metadata is not None else None, + ci=github_run(env) if ci_mode else None, + in_ci=ci_mode, + ) + return run, api + + +def _valid_score_name(name: str) -> bool: + """A trimmed name of 1 to 64 characters, as the API stores score names.""" + return 0 < utf16_length(name) <= MAX_SCORE_NAME_LENGTH and name.strip() == name + + +def _is_int(value: object) -> TypeGuard[int]: + return isinstance(value, int) and not isinstance(value, bool) + + +def _warn_about_client() -> None: + client = get_client() + environment = client.config.environment if client is not None and client.enabled else None + if environment is None: + logger.warning( + "run_experiment: no telemetry client, so the items record no traces. " + 'Call init(environment="experiment") first.' + ) + elif environment.strip().lower() != _EXPERIMENT_ENVIRONMENT: + logger.warning( + "run_experiment: the traces of the items go to the environment %s. " + 'Call init(environment="experiment") to keep them out of it.', + environment, + ) + + +def _start(api: Api, run: _Run) -> tuple[_Runs, _Created]: + _warn_about_client() + first = _fetch_page(api, run, run.dataset_version, None) + if not first.items: + scope = "" if run.split is None else f" in the split {run.split}" + raise ExperimentError("dataset_empty", f"The dataset {run.dataset} has no items{scope}") + body = { + "name": run.name, + "dataset": run.dataset, + "datasetVersion": first.version, + "split": run.split, + "evaluators": run.evaluators, + "minScores": run.min_scores, + "runsPerItem": run.runs_per_item, + "ci": run.ci, + "metadata": run.metadata, + } + created = _parse_created( + request( + api, + ExperimentError, + "POST", + "/v1/experiments", + body={key: value for key, value in body.items() if value is not None}, + timeout_s=_REQUEST_TIMEOUT_S, + # A repeat after a lost response would start a second experiment. + idempotent=False, + ) + ) + return _Runs(api, run, first), created + + +def _announce(run: _Run, created: _Created) -> None: + each = f", {run.runs_per_item} runs each" if run.runs_per_item > 1 else "" + print( + f"Experiment {created.name}: {created.items_total // run.runs_per_item} items{each}. " + f"{created.url}", + flush=True, + ) + + +def _complete_after_failure(api: Api, created: _Created) -> None: + """Without the complete, the experiment would stay running: the server ends it with the item + runs that it has, or fails it when it has none.""" + try: + _complete(api, created) + except Exception as exc: + report_error(api.on_error, "completing the experiment after a failed run failed", exc) + + +class _Runs: + """The runs of each item of the dataset version, page by page, so that only one page is in + memory. A lock lets the workers share the generator.""" + + def __init__(self, api: Api, run: _Run, first: _Page) -> None: + self._lock = threading.Lock() + self._runs = _item_runs(api, run, first) + + def next(self) -> ExperimentItem | None: + with self._lock: + return next(self._runs, None) + + def close(self) -> None: + with self._lock: + self._runs.close() + + +def _item_runs(api: Api, run: _Run, first: _Page) -> Generator[ExperimentItem]: + page = first + count = 0 + while True: + count += len(page.items) + if count > _MAX_DATASET_ITEMS: + raise ExperimentError( + "invalid_response", f"telemetry.dev returned more than {_MAX_DATASET_ITEMS} items" + ) + for item in page.items: + # A copy for each run, so that a task that changes its input does not change the next + # run. + for run_index in range(run.runs_per_item): + yield ExperimentItem(**copy.deepcopy(vars(item)), run_index=run_index) + if page.next_cursor is None: + return + page = _fetch_page(api, run, page.version, page.next_cursor) + + +def _fetch_page(api: Api, run: _Run, version: int | None, cursor: str | None) -> _Page: + """One page of a dataset version. A page after the first names the version of the first.""" + page = _parse_page( + request( + api, + ExperimentError, + "GET", + f"/v1/datasets/{urllib.parse.quote(run.dataset, safe='')}", + query={ + "version": None if version is None else str(version), + "split": run.split, + "cursor": cursor, + }, + timeout_s=_REQUEST_TIMEOUT_S, + idempotent=True, + ) + ) + if ( + page is None + or (version is not None and page.version != version) + or (page.next_cursor is not None and (not page.items or page.next_cursor == cursor)) + ): + raise ExperimentError("invalid_response", "telemetry.dev returned a dataset it cannot read") + return page + + +class _Batch: + """Collects item results and posts them in order, one post at a time.""" + + def __init__(self, post: Callable[[list[dict[str, Any]]], object]) -> None: + self._post = post + self._pending: list[dict[str, Any]] = [] + self._bytes = 0 + self._lock = threading.Lock() + + def add(self, result: dict[str, Any]) -> None: + size = len(json_dumps(result, default=None).encode()) + with self._lock: + if self._pending and self._bytes + size > _MAX_BATCH_BYTES: + self._send() + self._pending.append(result) + self._bytes += size + if len(self._pending) >= _MAX_BATCH_ITEMS: + self._send() + + def flush(self) -> None: + with self._lock: + if self._pending: + self._send() + + def _send(self) -> None: + results = self._pending + self._pending = [] + self._bytes = 0 + self._post(results) + + +def _batch(api: Api, created: _Created) -> _Batch: + path = f"/v1/experiments/{urllib.parse.quote(created.id, safe='')}/items" + return _Batch( + lambda results: request( + api, + ExperimentError, + "POST", + path, + body={"items": results}, + timeout_s=_REQUEST_TIMEOUT_S, + # The server skips an item run that it has, so a repeat is safe. + idempotent=True, + ) + ) + + +def _run_items(api: Api, run: _Run, created: _Created, runs: _Runs) -> None: + """Run the items, ``concurrency`` at a time, and post each result. A post or a page fetch + that fails stops the run: no new item starts, and _run_items raises when the running items + end.""" + batch = _batch(api, created) + failures: list[BaseException] = [] + caller = contextvars.copy_context() + + def worker() -> None: + while not failures: + try: + item = runs.next() + if item is None: + return + # A copy for each item: one Context cannot run in two threads at the same time. + batch.add(caller.copy().run(_run_item, run, created, item, api.on_error)) + except Exception as exc: + failures.append(exc) + + with ThreadPoolExecutor(run.concurrency, thread_name_prefix="telemetry-dev-experiment") as pool: + futures = [pool.submit(worker) for _ in range(run.concurrency)] + try: + for future in futures: + future.result() + except BaseException as exc: + # Stops the workers, for example after Ctrl-C. + failures.append(exc) + raise + _end_items(batch, runs, failures) + + +async def _run_items_async(api: Api, run: _Run, created: _Created, runs: _Runs) -> None: + batch = _batch(api, created) + failures: list[BaseException] = [] + + async def worker() -> None: + while not failures: + try: + item = await asyncio.to_thread(runs.next) + if item is None: + return + result = await _run_item_async(run, created, item, api.on_error) + await asyncio.to_thread(batch.add, result) + except Exception as exc: + failures.append(exc) + + # gather runs each worker in its own task, with its own copy of the context. + await asyncio.gather(*(worker() for _ in range(run.concurrency))) + await asyncio.to_thread(_end_items, batch, runs, failures) + + +def _end_items(batch: _Batch, runs: _Runs, failures: list[BaseException]) -> None: + if not failures: + try: + batch.flush() + except Exception as exc: + failures.append(exc) + if failures: + runs.close() + raise failures[0] + + +def _run_item( + run: _Run, + experiment: _Created, + item: ExperimentItem, + on_error: Callable[[BaseException], None] | None, +) -> dict[str, Any]: + started = perf_counter() + output: Any = None + error: Exception | None = None + with start_span( + f"experiment {experiment.name}", + input=dict(item.input), + parent=_item_context(experiment.id, item), + ) as span: + try: + output = run.task(item) + if inspect.isawaitable(output): + _close(output) + raise TypeError("task returned an awaitable: use run_experiment_async") + span.update(output=output) + except Exception as exc: + error = exc + span.update(error=exc) + trace_id = _trace_id(span) + result = _item_result(item, trace_id, started, output, error) + if error is None: + _add_scores( + result, [_score(name, scorer, output, item, on_error) for name, scorer in run.scorers] + ) + return result + + +async def _run_item_async( + run: _Run, + experiment: _Created, + item: ExperimentItem, + on_error: Callable[[BaseException], None] | None, +) -> dict[str, Any]: + started = perf_counter() + output: Any = None + error: Exception | None = None + with start_span( + f"experiment {experiment.name}", + input=dict(item.input), + parent=_item_context(experiment.id, item), + ) as span: + try: + output = run.task(item) + if inspect.isawaitable(output): + output = await output + span.update(output=output) + except Exception as exc: + error = exc + span.update(error=exc) + trace_id = _trace_id(span) + result = _item_result(item, trace_id, started, output, error) + if error is None: + scores: list[dict[str, Any] | None] = [] + for name, scorer in run.scorers: + try: + value = scorer(output, item) + if inspect.isawaitable(value): + value = await value + scores.append(_to_score(name, value)) + except Exception as exc: + _scorer_failed(on_error, name, item, exc) + scores.append(None) + _add_scores(result, scores) + return result + + +def _item_context(experiment_id: str, item: ExperimentItem) -> Context: + """The context of one item run. It drops the active span, so that the run starts a trace of + its own, and gives the run a session id of its own: an ambient session or + ``session_mode="process"`` would otherwise put all runs in one session trace.""" + base = trace.set_span_in_context(trace.INVALID_SPAN, otel_context.get_current()) + return context_with_propagated_attributes( + { + **propagated_attributes(base), + ATTR_EXPERIMENT_ID: experiment_id, + ATTR_DATASET_ITEM_ID: item.id, + ATTR_SESSION_ID: f"{experiment_id}:{item.id}:{item.run_index}", + }, + base, + ) + + +def _trace_id(span: SpanHandle) -> str | None: + context = span.span.get_span_context() + return f"{context.trace_id:032x}" if span.span.is_recording() and context.is_valid else None + + +def _item_result( + item: ExperimentItem, + trace_id: str | None, + started: float, + output: Any, + error: Exception | None, +) -> dict[str, Any]: + result: dict[str, Any] = { + "itemId": item.id, + "runIndex": item.run_index, + "output": None if error is not None else _output_text(output), + "latencyMs": round((perf_counter() - started) * 1000), + } + if trace_id is not None: + result["traceId"] = trace_id + if error is not None: + result["error"] = truncate(_error_text(error), _MAX_ERROR_LENGTH) + return result + + +def _add_scores(result: dict[str, Any], scores: list[dict[str, Any] | None]) -> None: + kept = [score for score in scores if score is not None] + if kept: + result["scores"] = kept + + +def _score( + name: str, + scorer: Callable[[Any, ExperimentItem], Any], + output: Any, + item: ExperimentItem, + on_error: Callable[[BaseException], None] | None, +) -> dict[str, Any] | None: + # After the span of the item ends, so that the model calls of a scorer do not count in the + # cost of the item. + try: + value = scorer(output, item) + if inspect.isawaitable(value): + _close(value) + raise TypeError("returned an awaitable: use run_experiment_async") + return _to_score(name, value) + except Exception as exc: + _scorer_failed(on_error, name, item, exc) + return None + + +def _scorer_failed( + on_error: Callable[[BaseException], None] | None, + name: str, + item: ExperimentItem, + exc: Exception, +) -> None: + error = ExperimentError( + "scorer_failed", f"The scorer {name} failed on item {item.number}: {_error_text(exc)}" + ) + error.__cause__ = exc + report_error(on_error, "a scorer failed", error) + + +def _to_score(name: str, result: object) -> dict[str, Any] | None: + """The score of a scorer result. It raises for a result that the server would reject.""" + if result is None: + return None + score: dict[str, Any] = {"name": name} + if isinstance(result, bool): + score.update(_pass_fail(result)) + elif isinstance(result, (int, float)): + score["value"] = result + elif isinstance(result, str): + score["label"] = result.strip() + elif isinstance(result, Mapping): + fields = cast("Mapping[str, object]", result) + value, label, reason = fields.get("value"), fields.get("label"), fields.get("reason") + for key, text in (("label", label), ("reason", reason)): + if text is not None and not isinstance(text, str): + raise TypeError(f"returned a {key} that is not a string") + if isinstance(value, bool): + score.update(_pass_fail(value)) + elif value is not None: + score["value"] = value + if isinstance(label, str): + score["label"] = label.strip() + if isinstance(reason, str): + score["reason"] = truncate(reason, MAX_REASON_LENGTH) + else: + raise TypeError(f"returned a {type(result).__name__}") + problem = check_value(name, score.get("value"), score.get("label")) + if problem is not None: + raise TypeError(f"returned a score that is not valid: {problem}") + return score + + +def _pass_fail(passed: bool) -> dict[str, Any]: + return {"value": 1, "label": "pass"} if passed else {"value": 0, "label": "fail"} + + +def _output_text(output: Any) -> str | None: + """The output as text. An output that JSON cannot hold, such as a dict that holds itself, is + None.""" + if output is None: + return None + try: + text = output if isinstance(output, str) else json_dumps(output) + except Exception: + return None + return truncate(_well_formed(text), _MAX_OUTPUT_LENGTH) + + +def _error_text(error: BaseException) -> str: + try: + text = f"{type(error).__name__}: {error}" + except Exception: + # A __str__ that raises. + return type(error).__name__ + return _well_formed(text) + + +def _well_formed(text: str) -> str: + # truncate fails on a lone surrogate, which a str can hold, so each one becomes U+FFFD. + return text.encode("utf-16-le", "surrogatepass").decode("utf-16-le", "replace") + + +def _close(awaitable: object) -> None: + # A coroutine that never runs warns when it is collected. + close = getattr(awaitable, "close", None) + if callable(close): + close() + + +def _finish(api: Api, run: _Run, created: _Created) -> ExperimentResult: + # The spans of the items reach ingest before the server reads their cost. + flush() + _complete(api, created) + result, body = _wait_for_result(api, created.id) + write_result(os.environ, body, api.on_error) + print(_describe(result), flush=True) + if run.in_ci: + _check_result(result) + return result + + +def _complete(api: Api, created: _Created) -> None: + request( + api, + ExperimentError, + "POST", + f"/v1/experiments/{urllib.parse.quote(created.id, safe='')}/complete", + timeout_s=_REQUEST_TIMEOUT_S, + idempotent=True, + ) + + +def _wait_for_result(api: Api, experiment_id: str) -> tuple[ExperimentResult, object]: + """Wait until the server evaluators finish and the experiment ends.""" + deadline = time.monotonic() + _MAX_WAIT_S + interval = 1.0 + while True: + body = request( + api, + ExperimentError, + "GET", + f"/v1/experiments/{urllib.parse.quote(experiment_id, safe='')}", + timeout_s=_REQUEST_TIMEOUT_S, + idempotent=True, + ) + result = _parse_result(body) + if result.status in _FINISHED: + return result, body + if time.monotonic() + interval > deadline: + raise ExperimentError( + "timeout", + f"The experiment {result.name} did not end in {_MAX_WAIT_S / 60:g} minutes. " + f"{result.url}", + result=result, + ) + time.sleep(interval) + interval = min(interval * 2, _MAX_POLL_INTERVAL_S) + + +def _check_result(result: ExperimentResult) -> None: + """In CI mode, a failed check or an experiment that did not complete fails the run.""" + if result.status != "completed": + reason = f": {result.error}" if result.error else "" + raise ExperimentError( + "experiment_failed", + f"The experiment {result.name} {result.status}{reason}. {result.url}", + result=result, + ) + if result.check_status != "failed": + return + failed = [ + f"{score.name} is {_format(score.kind, score.value)}, under the pass mark " + f"{_format(score.kind, score.min_score)}" + for score in result.scores + if score.passed is False + ] + detail = f": {'; '.join(failed)}" if failed else "" + raise ExperimentError( + "check_failed", + f"The check of {result.name} failed{detail}. {result.url}", + result=result, + ) + + +def _describe(result: ExperimentResult) -> str: + if result.status != "completed": + outcome = result.status + (f": {result.error}" if result.error else "") + elif result.check_status is None: + outcome = "completed" + else: + outcome = f"completed. The check {result.check_status}" + lines = [f"Experiment {result.name} {outcome}. {result.url}"] + for score in result.scores: + mark = ( + "" + if score.min_score is None + else f" (pass mark {_format(score.kind, score.min_score)})" + ) + lines.append(f" {score.name}: {_format(score.kind, score.value)}{mark}") + return "\n".join(lines) + + +def _format(kind: str | None, value: float | None) -> str: + if value is None: + return "no score" + if kind in _RATE_KINDS: + return f"{_js_round(value * 100)}%" + rounded = _js_round(value * 100) / 100 + return str(int(rounded)) if rounded.is_integer() else repr(rounded) + + +def _js_round(value: float) -> int: + # Math.round of JavaScript, which rounds a half up. round() of Python rounds it to even. + return math.floor(value + 0.5) + + +def _parse_created(body: object) -> _Created: + fields = cast("dict[str, Any]", body) if isinstance(body, dict) else {} + created_id, name, url = fields.get("id"), fields.get("name"), fields.get("url") + items_total = fields.get("itemsTotal") + if ( + not isinstance(created_id, str) + or not isinstance(name, str) + or not isinstance(url, str) + or not _is_int(items_total) + ): + raise ExperimentError( + "invalid_response", "telemetry.dev returned an experiment it cannot read" + ) + return _Created(created_id, name, items_total, url) + + +def _parse_page(body: object) -> _Page | None: + if not isinstance(body, dict): + return None + fields = cast("dict[str, Any]", body) + dataset, items = fields.get("dataset"), fields.get("items") + next_cursor = fields.get("nextCursor") + if not isinstance(dataset, dict) or not isinstance(items, list): + return None + version = cast("dict[str, Any]", dataset).get("version") + if not _is_int(version) or version < 0: + return None + if next_cursor is not None and not isinstance(next_cursor, str): + return None + parsed: list[DatasetItem] = [] + for raw in cast("list[object]", items): + item = _parse_item(raw) + if item is None: + return None + parsed.append(item) + return _Page(version, parsed, next_cursor) + + +def _parse_item(raw: object) -> DatasetItem | None: + if not isinstance(raw, dict): + return None + item = cast("dict[str, Any]", raw) + item_id, number, tags = item.get("id"), item.get("number"), item.get("tags") + item_input, metadata = item.get("input"), item.get("metadata") + expected, split = item.get("expectedOutput"), item.get("split") + if ( + not isinstance(item_id, str) + or not _is_int(number) + or not isinstance(item_input, dict) + or not (expected is None or isinstance(expected, str)) + or not isinstance(metadata, dict) + or not isinstance(tags, list) + or not all(isinstance(tag, str) for tag in cast("list[object]", tags)) + or not (split is None or isinstance(split, str)) + ): + return None + return DatasetItem( + id=item_id, + number=number, + input=cast("dict[str, Any]", item_input), + expected_output=expected, + metadata=cast("dict[str, Any]", metadata), + tags=tuple(cast("list[str]", tags)), + split=split, + ) + + +def _parse_result(body: object) -> ExperimentResult: + fields = cast("dict[str, Any]", body) if isinstance(body, dict) else {} + raw_scores = fields.get("scores") + scores = ( + [_parse_score(score) for score in cast("list[object]", raw_scores)] + if isinstance(raw_scores, list) + else None + ) + # Required: it decides the result of CI mode, so a missing one must not read as no check. + check_status = fields.get("checkStatus", "missing") + items_failed = fields.get("itemsFailed", 0) + error = fields.get("error") + if ( + not isinstance(fields.get("id"), str) + or not isinstance(fields.get("name"), str) + or not isinstance(fields.get("status"), str) + or not isinstance(fields.get("url"), str) + or check_status not in (None, "passed", "failed") + or not _is_number(fields.get("itemsTotal")) + or not _is_number(fields.get("itemsDone")) + or not _is_number(items_failed) + or not (error is None or isinstance(error, str)) + or scores is None + or any(score is None for score in scores) + ): + raise ExperimentError("invalid_response", "telemetry.dev returned a result it cannot read") + return ExperimentResult( + id=fields["id"], + name=fields["name"], + status=fields["status"], + check_status=check_status, + items_total=int(fields["itemsTotal"]), + items_done=int(fields["itemsDone"]), + items_failed=int(items_failed), + error=error, + scores=tuple(cast("list[ExperimentScore]", scores)), + url=fields["url"], + ) + + +def _parse_score(raw: object) -> ExperimentScore | None: + """A score of the result. A field that can be None can also be missing.""" + if not isinstance(raw, dict): + return None + score = cast("dict[str, Any]", raw) + name, kind, passed = score.get("name"), score.get("kind"), score.get("passed") + pass_count, fail_count = score.get("passCount"), score.get("failCount") + nullable = [score.get(key) for key in ("value", "baselineValue", "delta", "minScore")] + if ( + not isinstance(name, str) + or not (kind is None or isinstance(kind, str)) + or not _is_number(pass_count) + or not _is_number(fail_count) + or not all(value is None or _is_number(value) for value in nullable) + or not (passed is None or isinstance(passed, bool)) + ): + return None + value, baseline_value, delta, min_score = nullable + return ExperimentScore( + name=name, + kind=kind, + value=value, + pass_count=int(cast("float", pass_count)), + fail_count=int(cast("float", fail_count)), + baseline_value=baseline_value, + delta=delta, + min_score=min_score, + passed=passed, + ) + + +def _is_number(value: object) -> bool: + return isinstance(value, (int, float)) and not isinstance(value, bool) diff --git a/sdks/python/src/telemetry_dev/_prompts.py b/sdks/python/src/telemetry_dev/_prompts.py index 96737403..4408852e 100644 --- a/sdks/python/src/telemetry_dev/_prompts.py +++ b/sdks/python/src/telemetry_dev/_prompts.py @@ -17,18 +17,16 @@ from contextlib import contextmanager from contextvars import ContextVar from dataclasses import dataclass, field -from datetime import datetime, timezone -from email.message import Message -from email.utils import parsedate_to_datetime from time import monotonic -from typing import IO, Any, Literal, TypeAlias, TypedDict, cast, overload +from typing import Any, Literal, TypeAlias, TypedDict, cast, overload from weakref import WeakKeyDictionary from opentelemetry import context as otel_context from opentelemetry.context import Context +from ._api import NoRedirect, api_error, auth_headers, resolve_api, retry_after from ._client import Client, get_client -from ._config import SDK_VERSION, report_error, resolve_config +from ._config import report_error from ._context import propagated_attributes from ._semconv import ( ATTR_PROMPT_LABEL, @@ -45,8 +43,6 @@ # The longest Retry-After that get_prompt honors, so that a bad header cannot stop fetches. _MAX_RETRY_AFTER_S = 3600.0 _MAX_CACHE_ENTRIES = 1000 -# IMF-fixdate, the HTTP date that servers send: Sun, 06 Nov 1994 08:49:37 GMT. -_HTTP_DATE = re.compile(r"[A-Z][a-z]{2}, \d{2} [A-Z][a-z]{2} \d{4} \d{2}:\d{2}:\d{2} GMT", re.ASCII) _VARIABLE = re.compile(r"\{\{\s*([A-Za-z_][A-Za-z0-9_]*)\s*\}\}") _MODEL_CALL_OPERATIONS = frozenset({"chat", "text_completion", "generate_content"}) _PROMPT_KEY = otel_context.create_key("telemetry_dev.prompt") @@ -310,11 +306,11 @@ def _get_prompt( _check(name, type, label, version, cache_ttl_seconds, fallback) client = get_client() enabled = client if client is not None and client.enabled else None - config = enabled.config if enabled is not None else resolve_config() - key = api_key if api_key is not None else config.api_key - url = (base_url or config.base_url).rstrip("/") + api = resolve_api(api_key, base_url) + key = api.api_key + url = api.base_url asked_label = (label if label is not None else "production") if version is None else None - on_error = enabled.on_error if enabled is not None else None + on_error = api.on_error try: if not key: @@ -336,13 +332,7 @@ def _get_prompt( cache_ttl_seconds, lambda: _fetch( f"{url}/v1/prompts?{urllib.parse.urlencode(query)}", - { - "Authorization": f"Bearer {key}", - "X-Telemetry-Dev-SDK": "telemetry-dev", - # urllib's default User-Agent, Python-urllib/3.x, is refused by bot filters - # such as Cloudflare's Browser Integrity Check. - "User-Agent": f"telemetry-dev-python/{SDK_VERSION}", - }, + auth_headers(key), ), on_error, ) @@ -535,21 +525,6 @@ def _refresh( done.set() -class _NoRedirect(urllib.request.HTTPRedirectHandler): - # A redirect to another host must not get the API key, so a 3xx fails like any status that is - # not 2xx. - def redirect_request( - self, - req: urllib.request.Request, - fp: IO[bytes], - code: int, - msg: str, - headers: Message, - newurl: str, - ) -> urllib.request.Request | None: - return None - - class _Deadline: """Ends one prompt fetch after ``_FETCH_TIMEOUT_S``. The timeout of urllib applies to each socket operation, so a server that sends a byte now and then could keep a fetch reading forever, @@ -648,12 +623,12 @@ def do_open(self, http_class: Any, req: urllib.request.Request, **http_conn_args return super().do_open(_on_deadline(http_class, req), req, **http_conn_args) -_opener = urllib.request.build_opener(_NoRedirect, _HTTPHandler, _HTTPSHandler) +_opener = urllib.request.build_opener(NoRedirect, _HTTPHandler, _HTTPSHandler) def _fetch(url: str, headers: dict[str, str]) -> dict[str, Any]: deadline = _Deadline() - retry_after: str | None = None + retry_header: str | None = None try: try: with _opener.open( @@ -664,7 +639,7 @@ def _fetch(url: str, headers: dict[str, str]) -> dict[str, Any]: except urllib.error.HTTPError as exc: with exc: status, body = exc.code, exc.read() - retry_after = exc.headers.get("Retry-After") + retry_header = exc.headers.get("Retry-After") except Exception as exc: if ( deadline.expired @@ -681,9 +656,10 @@ def _fetch(url: str, headers: dict[str, str]) -> dict[str, Any]: if deadline.expired: raise _timeout() if not 200 <= status < 300: - error = _api_error(status, body) - if status in (429, 503): - error._retry_after = _retry_after(retry_after) # pyright: ignore[reportPrivateUsage] + error = api_error(PromptError, status, body) + wait = retry_after(status, retry_header) + if wait is not None: + error._retry_after = min(wait, _MAX_RETRY_AFTER_S) # pyright: ignore[reportPrivateUsage] raise error try: parsed = _parse_response(json.loads(body)) @@ -698,40 +674,6 @@ def _timeout() -> PromptError: return PromptError("timeout", f"Fetching the prompt took more than {_FETCH_TIMEOUT_S:g} s") -def _api_error(status: int, body: bytes) -> PromptError: - try: - error = json.loads(body).get("error") - except (ValueError, AttributeError): - error = None - fields = cast("dict[str, Any]", error) if isinstance(error, dict) else {} - code = fields.get("code") - message = fields.get("message") - return PromptError( - code if isinstance(code, str) else f"http_{status}", - message if isinstance(message, str) else f"telemetry.dev returned {status}", - status, - ) - - -def _retry_after(value: str | None) -> float | None: - """Seconds to wait from a Retry-After header of seconds or an IMF-fixdate HTTP date - (``Sun, 06 Nov 1994 08:49:37 GMT``), capped.""" - value = (value or "").strip() - if not value: - return None - if value.isascii() and value.isdigit(): - seconds = float(value) - elif _HTTP_DATE.fullmatch(value): - try: - date = parsedate_to_datetime(value) - except (TypeError, ValueError): - return None - seconds = (date - datetime.now(timezone.utc)).total_seconds() - else: - return None - return min(seconds, _MAX_RETRY_AFTER_S) if seconds > 0 else None - - def _parse_response(data: object) -> dict[str, Any] | None: if not isinstance(data, dict): return None diff --git a/sdks/python/src/telemetry_dev/_scores.py b/sdks/python/src/telemetry_dev/_scores.py new file mode 100644 index 00000000..f5dbee3f --- /dev/null +++ b/sdks/python/src/telemetry_dev/_scores.py @@ -0,0 +1,181 @@ +"""Scores on traces, spans, and sessions, as ``POST /v1/scores`` takes them.""" + +from __future__ import annotations + +import asyncio +import math +import re +from typing import Any, cast + +from ._api import request, resolve_api +from ._config import report_error +from ._serialize import utf16_length + +USER_FEEDBACK = "user_feedback" +MAX_SCORE_NAME_LENGTH = 64 +MAX_LABEL_LENGTH = 64 +MAX_REASON_LENGTH = 4000 +_MAX_SESSION_ID_LENGTH = 1024 +_TRACE_ID = re.compile(r"(?!0+$)[0-9a-f]{32}") +_SPAN_ID = re.compile(r"(?!0+$)[0-9a-f]{16}") +_TIMEOUT_S = 10.0 + + +class ScoreError(Exception): + """Why score failed. ``code`` is the code of the API error (``insufficient_scope``, + ``invalid_api_key``, ...), or ``missing_api_key``, ``invalid_score``, ``timeout``, + ``network_error``, or ``invalid_response``. A response with a different status and no API + error gives ``http_``. ``status`` is the HTTP status of the response.""" + + def __init__(self, code: str, message: str, status: int | None = None) -> None: + super().__init__(message) + self.code = code + self.message = message + self.status = status + + +def score( + *, + name: str, + trace_id: str | None = None, + span_id: str | None = None, + session_id: str | None = None, + value: float | None = None, + label: str | None = None, + comment: str | None = None, + api_key: str | None = None, + base_url: str | None = None, +) -> str | None: + """Record a score on a trace, a span, or a session, for example the feedback of a user on a + reply. ``user_feedback`` is a thumbs up (1) or down (0). Pass ``trace_id``, ``session_id``, or + the two, and ``value``, ``label``, or the two. + + Like the telemetry functions, score never raises: it returns the id of the score, or None + after it sends the error to ``on_error``. It makes one request, with no retry, so that a call + in a request handler waits 10 seconds at most for each socket operation. The trace id of a + span from a client that is not enabled is all zeros, and score reports it as + ``invalid_score``. + """ + api = resolve_api(api_key, base_url) + try: + problem = _check_score(name, trace_id, span_id, session_id, value, label, comment) + if problem is not None: + raise ScoreError("invalid_score", problem) + if not api.api_key: + raise ScoreError( + "missing_api_key", "No API key: pass api_key or set TELEMETRY_DEV_API_KEY" + ) + fields = { + "traceId": trace_id, + "spanId": span_id, + "sessionId": session_id, + "name": name.strip(), + "value": value, + "label": None if label is None else label.strip(), + "comment": comment, + } + body = request( + api, + ScoreError, + "POST", + "/v1/scores", + body={key: field for key, field in fields.items() if field is not None}, + timeout_s=_TIMEOUT_S, + idempotent=False, + attempts=1, + ) + score_id = cast("dict[str, Any]", body).get("id") if isinstance(body, dict) else None + if not isinstance(score_id, str): + raise ScoreError("invalid_response", "telemetry.dev returned a score it cannot read") + return score_id + except Exception as exc: + report_error(api.on_error, "sending the score failed", exc) + return None + + +async def score_async( + *, + name: str, + trace_id: str | None = None, + span_id: str | None = None, + session_id: str | None = None, + value: float | None = None, + label: str | None = None, + comment: str | None = None, + api_key: str | None = None, + base_url: str | None = None, +) -> str | None: + """``score`` in a worker thread, so that the request does not block the event loop.""" + return await asyncio.to_thread( + score, + name=name, + trace_id=trace_id, + span_id=span_id, + session_id=session_id, + value=value, + label=label, + comment=comment, + api_key=api_key, + base_url=base_url, + ) + + +def check_value(name: str, value: object, label: object) -> str | None: + """Why the server would reject the value and the label of a score, or None. A pass/fail score + has the label ``pass`` with the value 1, or ``fail`` with 0.""" + if value is not None and not is_finite(value): + return "value must be a finite number" + if label is not None and not ( + isinstance(label, str) and 0 < utf16_length(label) <= MAX_LABEL_LENGTH + ): + return f"label must be a string of 1 to {MAX_LABEL_LENGTH} characters" + if value is None and label is None: + return "Pass a value or a label" + if name == USER_FEEDBACK and not (label is None and value in (0, 1)): + return f"{USER_FEEDBACK} takes the value 1 for up or 0 for down, and no label" + if label == "pass" and value is not None and value != 1: + return "The label pass goes with the value 1" + if label == "fail" and value is not None and value != 0: + return "The label fail goes with the value 0" + return None + + +def _check_score( + name: object, + trace_id: object, + span_id: object, + session_id: object, + value: object, + label: object, + comment: object, +) -> str | None: + if not isinstance(name, str) or not 0 < utf16_length(name.strip()) <= MAX_SCORE_NAME_LENGTH: + return f"name must be a string of 1 to {MAX_SCORE_NAME_LENGTH} characters" + if trace_id is None and session_id is None: + return "Pass trace_id or session_id" + if trace_id is not None and not (isinstance(trace_id, str) and _TRACE_ID.fullmatch(trace_id)): + return "trace_id must be 32 lowercase hex digits and not all zeros" + if span_id is not None and not (isinstance(span_id, str) and _SPAN_ID.fullmatch(span_id)): + return "span_id must be 16 lowercase hex digits and not all zeros" + if span_id is not None and trace_id is None: + return "Pass the trace_id of the span" + if session_id is not None and not ( + isinstance(session_id, str) and 0 < utf16_length(session_id) <= _MAX_SESSION_ID_LENGTH + ): + return f"session_id must be a string of 1 to {_MAX_SESSION_ID_LENGTH} characters" + if comment is not None and not ( + isinstance(comment, str) and utf16_length(comment) <= MAX_REASON_LENGTH + ): + return f"comment must be a string of at most {MAX_REASON_LENGTH} characters" + return check_value(name.strip(), value, label.strip() if isinstance(label, str) else label) + + +def is_finite(value: object) -> bool: + # True and False are ints in Python, but JSON sends them as booleans, which are not numbers. + if isinstance(value, bool) or not isinstance(value, (int, float)): + return False + try: + return math.isfinite(value) + except OverflowError: + # An int too large for a float, which JavaScript reads as Infinity. + return False diff --git a/sdks/python/src/telemetry_dev/_semconv.py b/sdks/python/src/telemetry_dev/_semconv.py index 69553c02..7d3041b3 100644 --- a/sdks/python/src/telemetry_dev/_semconv.py +++ b/sdks/python/src/telemetry_dev/_semconv.py @@ -88,6 +88,8 @@ ATTR_PROMPT_VERSION = "td.prompt.version" ATTR_PROMPT_LABEL = "td.prompt.label" ATTR_PROMPT_VARIABLES = "td.prompt.variables" +ATTR_EXPERIMENT_ID = "td.experiment.id" +ATTR_DATASET_ITEM_ID = "td.dataset.item_id" USAGE_ATTRS: dict[str, str] = { "input_tokens": "gen_ai.usage.input_tokens", diff --git a/sdks/python/src/telemetry_dev/_serialize.py b/sdks/python/src/telemetry_dev/_serialize.py index dcf13213..f69c4a67 100644 --- a/sdks/python/src/telemetry_dev/_serialize.py +++ b/sdks/python/src/telemetry_dev/_serialize.py @@ -30,6 +30,11 @@ class MaskContext: Mask = Callable[[Any, MaskContext], Any] +def utf16_length(text: str) -> int: + """The length of ``text`` in UTF-16 units, as JavaScript and the telemetry.dev API count it.""" + return len(text.encode("utf-16-le", "surrogatepass")) // 2 + + def truncate(text: str, max_len: int, *, keep_end: bool = False) -> str: """Cap ``text`` at ``max_len`` UTF-16 units, marking the cut. diff --git a/sdks/python/tests/fake_api.py b/sdks/python/tests/fake_api.py new file mode 100644 index 00000000..d86f4cd0 --- /dev/null +++ b/sdks/python/tests/fake_api.py @@ -0,0 +1,221 @@ +"""The Evals routes of telemetry.dev on a local HTTP server: pages of a dataset version with a +cursor, an experiment that takes item posts of at most 100 and skips repeats, and its result.""" + +from __future__ import annotations + +import json +import sys +import threading +import time +from collections.abc import Callable, Generator +from contextlib import contextmanager +from dataclasses import dataclass, field +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from typing import Any +from urllib.parse import parse_qsl, urlsplit + +EXPERIMENT_ID = "6f9619ff-8b86-4011-b42d-00c04fc964ff" +EXPERIMENT_URL = f"https://telemetry.dev/o/org/p/project/evals/experiments/{EXPERIMENT_ID}" + + +def item_id(number: int) -> str: + return f"00000000-0000-4000-8000-{number:012d}" + + +def dataset_item(number: int, split: str | None = None) -> dict[str, Any]: + return { + "id": item_id(number), + "number": number, + "input": {"question": f"q{number}"}, + "expectedOutput": f"a{number}", + "metadata": {"source": "test"}, + "tags": ["golden"], + "split": split, + } + + +def result(**overrides: Any) -> dict[str, Any]: + body: dict[str, Any] = { + "id": EXPERIMENT_ID, + "name": "support-golden-v3", + "status": "completed", + "checkStatus": None, + "itemsTotal": 2, + "itemsDone": 2, + "itemsFailed": 0, + "error": None, + "scores": [], + "url": EXPERIMENT_URL, + } + body.update(overrides) + return body + + +@dataclass +class ApiRequest: + method: str + path: str + query: dict[str, str] + headers: dict[str, str] + body: Any + + @property + def route(self) -> str: + return f"{self.method} {self.path}" + + +# (status, JSON body, headers), or None to answer as the API does. +Reply = tuple[int, object, dict[str, str]] + + +@dataclass(frozen=True) +class Drop: + """Closes the connection with no answer, as a network failure does.""" + + +@dataclass(frozen=True) +class Late: + """Answers after a wait, as a server that is slower than the timeout of the client.""" + + seconds: float + reply: Reply + + +@dataclass +class FakeApi: + url: str + items: list[dict[str, Any]] = field(default_factory=lambda: [dataset_item(1), dataset_item(2)]) + page_size: int = 100 + results: list[dict[str, Any]] = field(default_factory=lambda: [result()]) + override: Callable[[ApiRequest], Reply | Drop | Late | None] | None = None + requests: list[ApiRequest] = field(default_factory=lambda: []) + stored: set[str] = field(default_factory=lambda: set[str]()) + lock: threading.Lock = field(default_factory=threading.Lock) + + def routes(self) -> list[str]: + return [request.route for request in self.requests] + + def posts(self) -> list[list[dict[str, Any]]]: + return [ + request.body["items"] for request in self.requests if request.path.endswith("/items") + ] + + def answer(self, request: ApiRequest) -> Reply | Drop | Late: + with self.lock: + self.requests.append(request) + overridden = self.override(request) if self.override else None + if overridden is not None: + return overridden + return self._route(request) + + def _route(self, request: ApiRequest) -> Reply: + if request.route == "GET /v1/datasets/support-golden": + split = request.query.get("split") + start = int(request.query.get("cursor", "0")) + matching = [item for item in self.items if split is None or item["split"] == split] + more = start + self.page_size < len(matching) + return ( + 200, + { + "dataset": { + "id": "dataset-1", + "name": "support-golden", + "version": int(request.query.get("version", "3")), + }, + "items": matching[start : start + self.page_size], + "nextCursor": str(start + self.page_size) if more else None, + }, + {}, + ) + if request.route == "POST /v1/experiments": + split = request.body.get("split") + count = len([item for item in self.items if split is None or item["split"] == split]) + return ( + 201, + { + "id": EXPERIMENT_ID, + "name": request.body.get("name", "support-golden-v3"), + "datasetVersion": request.body.get("datasetVersion"), + "itemsTotal": count * request.body.get("runsPerItem", 1), + "url": EXPERIMENT_URL, + }, + {}, + ) + if request.route == f"POST /v1/experiments/{EXPERIMENT_ID}/items": + runs = request.body["items"] + if len(runs) > 100: + return 400, {"error": {"code": "invalid_request", "message": "at most 100"}}, {} + keys = [f"{run['itemId']}:{run.get('runIndex', 0)}" for run in runs] + accepted = len([key for key in keys if key not in self.stored]) + self.stored.update(keys) + return 200, {"accepted": accepted, "duplicates": len(runs) - accepted}, {} + if request.route == f"POST /v1/experiments/{EXPERIMENT_ID}/complete": + return 200, {"id": EXPERIMENT_ID, "status": "running"}, {} + if request.route == f"GET /v1/experiments/{EXPERIMENT_ID}": + return 200, self.results.pop(0) if len(self.results) > 1 else self.results[0], {} + return 404, {"error": {"code": "not_found", "message": request.route}}, {} + + +@contextmanager +def fake_api(**options: Any) -> Generator[FakeApi]: + api: FakeApi | None = None + + class Handler(BaseHTTPRequestHandler): + def do_GET(self) -> None: + self._answer() + + def do_POST(self) -> None: + self._answer() + + def _answer(self) -> None: + assert api is not None + length = int(self.headers.get("content-length") or 0) + raw = self.rfile.read(length) if length else b"" + target = urlsplit(self.path) + reply = api.answer( + ApiRequest( + method=self.command, + path=target.path, + query=dict(parse_qsl(target.query)), + headers={key.lower(): value for key, value in self.headers.items()}, + body=json.loads(raw) if raw else None, + ) + ) + if isinstance(reply, Drop): + self.close_connection = True + return + if isinstance(reply, Late): + time.sleep(reply.seconds) + reply = reply.reply + status, body, headers = reply + data = body if isinstance(body, bytes) else json.dumps(body).encode() + self.send_response(status) + self.send_header("content-type", "application/json") + self.send_header("content-length", str(len(data))) + for key, value in headers.items(): + self.send_header(key, value) + self.end_headers() + self.wfile.write(data) + + def log_message(self, format: str, *args: object) -> None: + pass + + class Server(ThreadingHTTPServer): + def handle_error(self, request: Any, client_address: Any) -> None: + # The answer of a Late reply can go to a client that has stopped waiting. + if not isinstance(sys.exc_info()[1], (BrokenPipeError, ConnectionResetError)): + super().handle_error(request, client_address) + + server = Server(("127.0.0.1", 0), Handler) + thread = threading.Thread( + target=server.serve_forever, kwargs={"poll_interval": 0.01}, daemon=True + ) + host, port = server.server_address[0], server.server_address[1] + api = FakeApi(url=f"http://{host}:{port}", **options) + thread.start() + try: + yield api + finally: + server.shutdown() + server.server_close() + thread.join(timeout=5) diff --git a/sdks/python/tests/test_experiments.py b/sdks/python/tests/test_experiments.py new file mode 100644 index 00000000..8668f3c0 --- /dev/null +++ b/sdks/python/tests/test_experiments.py @@ -0,0 +1,1012 @@ +from __future__ import annotations + +import asyncio +import json +import threading +import time +from collections.abc import Generator +from pathlib import Path +from typing import Any + +import pytest + +import telemetry_dev +import telemetry_dev._experiments as experiments_module +from telemetry_dev import ( + ExperimentError, + ExperimentItem, + ExperimentResult, + ExperimentScore, + Scorer, + log, + propagate_attributes, + run_experiment, + run_experiment_async, + start_span, +) + +from .conftest import MakeClient +from .fake_api import ( + EXPERIMENT_ID, + EXPERIMENT_URL, + ApiRequest, + Drop, + FakeApi, + Late, + Reply, + dataset_item, + fake_api, + item_id, + result, +) + +ENV_VARS = ( + "TELEMETRY_DEV_EXPERIMENT_DATASET", + "TELEMETRY_DEV_EXPERIMENT_MIN_SCORES", + "TELEMETRY_DEV_EXPERIMENT_RESULTS", + "CI", + "GITHUB_ACTIONS", + "GITHUB_REPOSITORY", + "GITHUB_SHA", + "GITHUB_EVENT_PATH", + "GITHUB_RUN_ID", + "GITHUB_SERVER_URL", +) + + +@pytest.fixture(autouse=True) +def clear_ci(monkeypatch: pytest.MonkeyPatch) -> None: + # GitHub Actions sets CI and GITHUB_* for this suite too. + for var in ENV_VARS: + monkeypatch.delenv(var, raising=False) + + +@pytest.fixture +def api() -> Generator[FakeApi]: + with fake_api() as server: + yield server + + +def answer(item: ExperimentItem) -> str: + return f"answer to {item.input['question']}" + + +def experiment(api: FakeApi, **options: Any) -> ExperimentResult: + arguments: dict[str, Any] = { + "api_key": "td_live_test", + "base_url": api.url, + "dataset": "support-golden", + "task": answer, + } + arguments.update(options) + return run_experiment(**arguments) + + +def answer_with(status: int, body: object, headers: dict[str, str] | None = None) -> Reply: + return status, body, headers or {} + + +def test_runs_each_item_of_each_page_posts_batches_of_100_on_one_version_then_completes() -> None: + with fake_api( + items=[dataset_item(number) for number in range(1, 76)], + page_size=50, + results=[result(itemsTotal=150, itemsDone=146, itemsFailed=4)], + ) as api: + outcome = experiment(api, runs_per_item=2, evaluators=["Correctness"]) + + routes = api.routes() + assert routes[:2] == ["GET /v1/datasets/support-golden", "POST /v1/experiments"] + # The second page and the first post of items run at the same time, in either order. + assert sorted(routes[2:-2]) == [ + "GET /v1/datasets/support-golden", + f"POST /v1/experiments/{EXPERIMENT_ID}/items", + f"POST /v1/experiments/{EXPERIMENT_ID}/items", + ] + assert routes[-2:] == [ + f"POST /v1/experiments/{EXPERIMENT_ID}/complete", + f"GET /v1/experiments/{EXPERIMENT_ID}", + ] + # The first page names no version; the next one asks for the version of the first. + pages = [request for request in api.requests if request.path.startswith("/v1/datasets/")] + assert pages[0].query == {} + assert pages[1].query == {"version": "3", "cursor": "50"} + assert api.requests[1].body == { + "dataset": "support-golden", + "datasetVersion": 3, + "evaluators": ["Correctness"], + "runsPerItem": 2, + } + assert api.requests[1].headers["authorization"] == "Bearer td_live_test" + assert api.requests[1].headers["x-telemetry-dev-sdk"] == "telemetry-dev" + # Cloudflare refuses the default Python-urllib User-Agent. + assert api.requests[1].headers["user-agent"] == ( + f"telemetry-dev-python/{telemetry_dev.__version__}" + ) + + posts = api.posts() + assert [len(items) for items in posts] == [100, 50] + runs = [f"{run['itemId']}:{run['runIndex']}" for items in posts for run in items] + assert len(set(runs)) == 150 + assert f"{item_id(75)}:1" in runs + first = posts[0][0] + assert isinstance(first.pop("latencyMs"), int) + assert first == {"itemId": item_id(1), "runIndex": 0, "output": "answer to q1"} + + assert outcome == ExperimentResult( + id=EXPERIMENT_ID, + name="support-golden-v3", + status="completed", + check_status=None, + items_total=150, + items_done=146, + items_failed=4, + error=None, + scores=(), + url=EXPERIMENT_URL, + ) + + +def test_each_item_run_is_its_own_trace_and_each_span_of_the_task_carries_the_item( + make: MakeClient, api: FakeApi +) -> None: + # The session parent of a root span needs the key. + memory = make(api_key="td_live_test", environment="experiment", session_mode="process") + + def task(item: ExperimentItem) -> dict[str, str]: + with start_span("chat gpt-4o", type="generation"): + pass + log("replied") + return {"reply": f"r{item.number}"} + + # An active span, an ambient session, and the process session would each put the runs in + # one trace. + with ( + propagate_attributes(session_id="ambient-session", user_id="u-1"), + start_span("eval script"), + ): + experiment(api, runs_per_item=2, task=task) + + finished = memory.span_exporter.get_finished_spans() + chats = [span for span in finished if span.name == "chat gpt-4o"] + roots = [span for span in finished if span.name == "experiment support-golden-v3"] + assert len(chats) == 4 + assert len(roots) == 4 + + def trace_of(span: Any) -> str: + return f"{span.context.trace_id:032x}" + + assert len({trace_of(span) for span in roots}) == 4 + assert [span.parent is not None and span.parent.is_remote for span in roots] == [True] * 4 + posted = [run for items in api.posts() for run in items] + for run in posted: + root = next(span for span in roots if trace_of(span) == run["traceId"]) + chat = next(span for span in chats if trace_of(span) == run["traceId"]) + assert chat.parent is not None + assert chat.parent.span_id == root.context.span_id + for span in (root, chat): + attributes = dict(span.attributes or {}) + assert attributes["td.experiment.id"] == EXPERIMENT_ID + assert attributes["td.dataset.item_id"] == run["itemId"] + assert attributes["gen_ai.conversation.id"] == ( + f"{EXPERIMENT_ID}:{run['itemId']}:{run['runIndex']}" + ) + assert attributes["user.id"] == "u-1" + records = memory.log_exporter.get_finished_logs() + assert len(records) == 4 + for record in records: + trace_id = f"{record.log_record.trace_id:032x}" + run = next(run for run in posted if run["traceId"] == trace_id) + attributes = dict(record.log_record.attributes or {}) + assert attributes["td.experiment.id"] == EXPERIMENT_ID + assert attributes["td.dataset.item_id"] == run["itemId"] + assert sorted(run["output"] for run in posted) == [ + '{"reply":"r1"}', + '{"reply":"r1"}', + '{"reply":"r2"}', + '{"reply":"r2"}', + ] + + +def test_runs_at_most_concurrency_tasks_at_a_time() -> None: + lock = threading.Lock() + running = 0 + most = 0 + + def task(item: ExperimentItem) -> str: + nonlocal running, most + with lock: + running += 1 + most = max(most, running) + time.sleep(0.05) + with lock: + running -= 1 + return "ok" + + with fake_api(items=[dataset_item(number) for number in range(1, 13)]) as api: + experiment(api, concurrency=3, task=task) + + assert most == 3 + assert len([run for items in api.posts() for run in items]) == 12 + + +def test_a_task_that_raises_fails_only_its_item(api: FakeApi) -> None: + def task(item: ExperimentItem) -> str: + if item.number == 1: + raise ValueError("no reply") + return "fine" + + experiment(api, task=task) + + failed, passed = [run for items in api.posts() for run in items] + assert failed["output"] is None + assert failed["error"] == "ValueError: no reply" + assert "scores" not in failed + assert passed["output"] == "fine" + assert "error" not in passed + + +def test_scorers_give_scores_and_a_bad_result_drops_only_its_score(make: MakeClient) -> None: + errors: list[BaseException] = [] + make(environment="experiment", on_error=errors.append) + + def thrown(output: Any, item: ExperimentItem) -> None: + raise RuntimeError("judge down") + + scorers: dict[str, Scorer] = { + "exact": lambda output, item: output == f"answer to {item.input['question']}", + "polite": lambda output, item: False, + "length": lambda output, item: len(output), + "tone": lambda output, item: " friendly ", + "judged": lambda output, item: {"value": 0.25, "reason": "too short"}, + "none": lambda output, item: None, + "thrown": thrown, + "mismatched": lambda output, item: {"value": 0.5, "label": "pass"}, + # 33 emoji are 33 characters in Python but 66 UTF-16 units for the API. + "wide": lambda output, item: "\U0001f600" * 33, + } + + with fake_api(items=[dataset_item(1)]) as api: + experiment(api, scorers=scorers) + + assert api.posts()[0][0]["scores"] == [ + {"name": "exact", "value": 1, "label": "pass"}, + {"name": "polite", "value": 0, "label": "fail"}, + {"name": "length", "value": 12}, + {"name": "tone", "label": "friendly"}, + {"name": "judged", "value": 0.25, "reason": "too short"}, + ] + assert [(getattr(error, "code", None), str(error)) for error in errors] == [ + ("scorer_failed", "The scorer thrown failed on item 1: RuntimeError: judge down"), + ( + "scorer_failed", + "The scorer mismatched failed on item 1: TypeError: returned a score that is not " + "valid: The label pass goes with the value 1", + ), + ( + "scorer_failed", + "The scorer wide failed on item 1: TypeError: returned a score that is not valid: " + "label must be a string of 1 to 64 characters", + ), + ] + + +def test_an_output_is_cut_to_100000_utf16_units(api: FakeApi) -> None: + def task(item: ExperimentItem) -> str: + return "\U0001f600" * 60_000 + + experiment(api, task=task) + + output = api.posts()[0][0]["output"] + assert len(output.encode("utf-16-le")) // 2 <= 100_000 + assert output.endswith("...[truncated]") + + +def run_both(api: FakeApi, asynchronous: bool, **options: Any) -> ExperimentResult: + """``experiment`` through run_experiment or run_experiment_async.""" + if not asynchronous: + return experiment(api, **options) + arguments: dict[str, Any] = { + "api_key": "td_live_test", + "base_url": api.url, + "dataset": "support-golden", + "task": answer, + } + arguments.update(options) + return asyncio.run(run_experiment_async(**arguments)) + + +def unknown_item(request: ApiRequest) -> Reply | None: + if request.path.endswith("/items"): + return answer_with( + 422, {"error": {"code": "item_not_found", "message": "support-golden v3 has no item"}} + ) + return None + + +@pytest.mark.parametrize("asynchronous", [False, True], ids=["sync", "async"]) +def test_a_failed_post_of_items_stops_new_items_completes_the_experiment_and_raises( + asynchronous: bool, +) -> None: + started = 0 + + def task(item: ExperimentItem) -> str: + nonlocal started + started += 1 + return "ok" + + with fake_api(items=[dataset_item(n) for n in range(1, 301)], override=unknown_item) as api: + with pytest.raises(ExperimentError) as caught: + run_both(api, asynchronous, concurrency=1, task=task) + + assert (caught.value.code, caught.value.status) == ("item_not_found", 422) + # The one worker waits for the post of the first 100, which fails, so no item starts after. + assert started == 100 + # The complete ends the experiment, which would else stay running. + assert api.routes()[-2:] == [ + f"POST /v1/experiments/{EXPERIMENT_ID}/items", + f"POST /v1/experiments/{EXPERIMENT_ID}/complete", + ] + + +def test_a_complete_that_fails_after_a_failed_run_goes_to_on_error(make: MakeClient) -> None: + errors: list[BaseException] = [] + make(environment="experiment", on_error=errors.append) + + def revoked(request: ApiRequest) -> Reply | None: + if request.path.endswith(("/items", "/complete")): + return answer_with( + 401, {"error": {"code": "invalid_api_key", "message": "The API key was revoked"}} + ) + return None + + with fake_api(override=revoked) as api: + with pytest.raises(ExperimentError) as caught: + experiment(api) + + assert (caught.value.code, str(caught.value)) == ("invalid_api_key", "The API key was revoked") + assert api.routes().count(f"POST /v1/experiments/{EXPERIMENT_ID}/complete") == 1 + assert [getattr(error, "code", None) for error in errors] == ["invalid_api_key"] + + +def test_an_interrupted_run_completes_the_experiment(api: FakeApi) -> None: + def task(item: ExperimentItem) -> str: + raise KeyboardInterrupt + + with pytest.raises(KeyboardInterrupt): + experiment(api, task=task) + + assert api.routes()[-1] == f"POST /v1/experiments/{EXPERIMENT_ID}/complete" + + +class UnreadableError(Exception): + def __str__(self) -> str: + raise RuntimeError("no message") + + +@pytest.mark.parametrize("asynchronous", [False, True], ids=["sync", "async"]) +def test_an_output_or_an_error_that_cannot_be_encoded_does_not_stop_the_run( + asynchronous: bool, +) -> None: + cycle: dict[str, Any] = {} + cycle["self"] = cycle + + def task(item: ExperimentItem) -> object: + if item.number == 3: + raise UnreadableError + # A lone surrogate, which UTF-8 cannot hold, becomes U+FFFD. + return cycle if item.number == 1 else "\ud83d" + + def tone(output: Any, item: ExperimentItem) -> str: + return "\ud83d" + + with fake_api(items=[dataset_item(1), dataset_item(2), dataset_item(3)]) as api: + outcome = run_both(api, asynchronous, task=task, scorers={"tone": tone}) + + assert outcome.status == "completed" + posted = sorted((run for items in api.posts() for run in items), key=lambda run: run["itemId"]) + tones = [{"name": "tone", "label": "\ud83d"}] + assert [(run["output"], run.get("error"), run.get("scores")) for run in posted] == [ + (None, None, tones), + ("\ufffd", None, tones), + (None, "UnreadableError", None), + ] + assert f"POST /v1/experiments/{EXPERIMENT_ID}/complete" in api.routes() + + +def test_each_run_of_an_item_starts_from_the_dataset_input() -> None: + seen: list[int] = [] + + def task(item: ExperimentItem) -> str: + messages = item.input["messages"] + seen.append(len(messages)) + messages.append({"role": "assistant", "content": "Hello"}) + return "ok" + + item = {**dataset_item(1), "input": {"messages": [{"role": "user", "content": "Hi"}]}} + with fake_api(items=[item]) as api: + experiment(api, runs_per_item=3, concurrency=1, task=task) + + assert seen == [1, 1, 1] + + +def test_a_post_of_items_holds_at_most_4_mib_of_json() -> None: + def task(item: ExperimentItem) -> str: + # 100,000 characters of 3 UTF-8 bytes: about 300 KB of JSON for each item run. + return "\u4e2d" * 100_000 + + with fake_api(items=[dataset_item(number) for number in range(1, 21)]) as api: + experiment(api, task=task) + + assert [len(items) for items in api.posts()] == [13, 7] + + +@pytest.mark.parametrize( + ("count", "code"), + [(10_000, None), (10_001, "invalid_response")], + ids=["10,000 items in pages run", "10,001 items in pages stop the run"], +) +def test_a_dataset_has_at_most_10000_items(count: int, code: str | None) -> None: + items = [dataset_item(number) for number in range(1, count + 1)] + with fake_api(items=items, page_size=5_000) as api: + try: + experiment(api, concurrency=20) + error = None + except ExperimentError as exc: + error = exc.code + + assert error == code + assert len([run for items in api.posts() for run in items]) == 10_000 + assert api.routes().count(f"POST /v1/experiments/{EXPERIMENT_ID}/complete") == 1 + + +@pytest.mark.parametrize( + ("route", "failure", "sends", "code"), + [ + ("GET /v1/datasets/support-golden", "timeout", 2, None), + (f"POST /v1/experiments/{EXPERIMENT_ID}/items", "network", 2, None), + ("POST /v1/experiments", "timeout", 1, "timeout"), + ("POST /v1/experiments", "network", 1, "network_error"), + ], + ids=[ + "a timeout of a dataset page goes again", + "a network error of an item post goes again", + "a timeout of the create does not go again", + "a network error of the create does not go again", + ], +) +def test_a_request_after_a_timeout_or_a_network_error( + monkeypatch: pytest.MonkeyPatch, route: str, failure: str, sends: int, code: str | None +) -> None: + monkeypatch.setattr(experiments_module, "_REQUEST_TIMEOUT_S", 0.2) + failures = [1] + + def fail_once(request: ApiRequest) -> Reply | Drop | Late | None: + if request.route != route or not failures: + return None + failures.pop() + return Drop() if failure == "network" else Late(1.0, answer_with(500, {})) + + with fake_api(override=fail_once) as api: + try: + experiment(api) + error = None + except ExperimentError as exc: + error = exc.code + + assert error == code + assert api.routes().count(route) == sends + + +def test_an_item_post_that_fails_with_500_goes_again() -> None: + failures = [1] + + def flaky(request: ApiRequest) -> Reply | None: + if request.path.endswith("/items") and failures: + failures.pop() + return answer_with(500, {}) + return None + + with fake_api(override=flaky) as api: + experiment(api) + + assert len(api.posts()) == 2 + assert api.posts()[1] == api.posts()[0] + + +@pytest.mark.parametrize( + ("reply", "code"), + [ + (answer_with(500, {}), "http_500"), + # The server answers 503 when its database work times out, which can be after the commit. + ( + answer_with( + 503, + {"error": {"code": "evals_unavailable", "message": "Evals are unavailable"}}, + ), + "evals_unavailable", + ), + ], + ids=["500", "503"], +) +def test_a_create_that_fails_does_not_go_again(reply: Reply, code: str) -> None: + def broken(request: ApiRequest) -> Reply | None: + return reply if request.route == "POST /v1/experiments" else None + + with fake_api(override=broken) as api: + with pytest.raises(ExperimentError) as caught: + experiment(api) + + assert (caught.value.code, caught.value.status) == (code, reply[0]) + assert api.routes() == ["GET /v1/datasets/support-golden", "POST /v1/experiments"] + + +def test_a_create_that_gets_a_429_goes_again_after_its_retry_after() -> None: + limited = [1] + + def limit(request: ApiRequest) -> Reply | None: + if request.route == "POST /v1/experiments" and limited: + limited.pop() + return answer_with( + 429, + {"error": {"code": "rate_limited", "message": "Too many requests"}}, + {"retry-after": "2"}, + ) + return None + + with fake_api(override=limit) as api: + started = time.monotonic() + experiment(api) + + assert api.routes().count("POST /v1/experiments") == 2 + # Without the header, the first retry waits 1 second. + assert time.monotonic() - started >= 2 + + +def test_waits_until_the_server_evaluators_end_the_experiment() -> None: + with fake_api(results=[result(status="running", itemsDone=1), result()]) as api: + outcome = experiment(api) + + assert outcome.status == "completed" + assert api.routes().count(f"GET /v1/experiments/{EXPERIMENT_ID}") == 2 + + +class FakeClock: + def __init__(self) -> None: + self.now = 0.0 + + def monotonic(self) -> float: + return self.now + + def sleep(self, seconds: float) -> None: + self.now += seconds + + +def test_gives_up_when_the_experiment_does_not_end_in_30_minutes( + monkeypatch: pytest.MonkeyPatch, +) -> None: + clock = FakeClock() + monkeypatch.setattr(experiments_module, "time", clock) + polls: list[float] = [] + + def running(request: ApiRequest) -> Reply | None: + if request.route == f"GET /v1/experiments/{EXPERIMENT_ID}": + polls.append(clock.now) + return answer_with(200, result(status="running", itemsDone=1)) + return None + + with fake_api(override=running) as api: + with pytest.raises(ExperimentError) as caught: + experiment(api) + + assert caught.value.code == "timeout" + assert caught.value.result is not None + assert caught.value.result.status == "running" + # The last poll is the one whose next wait of 5 seconds would end past the 30 minutes. + assert 30 * 60 - 5 < polls[-1] - polls[0] <= 30 * 60 + + +@pytest.mark.parametrize( + ("status", "body", "code"), + [ + ( + 404, + {"error": {"code": "dataset_not_found", "message": "No dataset"}}, + "dataset_not_found", + ), + (200, {"dataset": {"version": 1}, "items": [{"id": 1}]}, "invalid_response"), + ( + 200, + {"dataset": {"id": "d", "name": "support-golden", "version": 0}, "items": []}, + "dataset_empty", + ), + ], + ids=["the API error", "a page that cannot be read", "an empty dataset"], +) +def test_raises_before_it_creates_an_experiment(status: int, body: object, code: str) -> None: + def page(request: ApiRequest) -> Reply | None: + return answer_with(status, body) if request.path.startswith("/v1/datasets/") else None + + with fake_api(override=page) as api: + with pytest.raises(ExperimentError) as caught: + experiment(api) + + assert caught.value.code == code + assert api.routes() == ["GET /v1/datasets/support-golden"] + + +def test_a_cursor_that_repeats_is_an_invalid_response() -> None: + def same(request: ApiRequest) -> Reply | None: + if not request.path.startswith("/v1/datasets/"): + return None + return answer_with( + 200, + { + "dataset": {"id": "d", "name": "support-golden", "version": 3}, + "items": [dataset_item(1)], + "nextCursor": "same", + }, + ) + + with fake_api(override=same) as api: + with pytest.raises(ExperimentError) as caught: + experiment(api) + + assert caught.value.code == "invalid_response" + assert api.routes().count("GET /v1/datasets/support-golden") == 2 + + +FAILED_CHECK = result( + checkStatus="failed", + scores=[ + { + "name": "correctness", + "kind": "pass_fail", + "value": 0.85, + "passCount": 17, + "failCount": 3, + "baselineValue": 0.9, + "delta": -5, + "minScore": 0.9, + "passed": False, + }, + { + "name": "length", + "kind": None, + "value": None, + "passCount": 0, + "failCount": 0, + "baselineValue": None, + "delta": None, + "minScore": None, + "passed": None, + }, + ], +) + + +def test_in_ci_mode_a_failed_check_raises_with_the_result(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv("CI", "true") + with fake_api(results=[FAILED_CHECK]) as api: + with pytest.raises(ExperimentError) as caught: + experiment(api, min_scores={"correctness": 0.9}) + + assert caught.value.code == "check_failed" + assert str(caught.value) == ( + "The check of support-golden-v3 failed: correctness is 85%, under the pass mark 90%. " + f"{EXPERIMENT_URL}" + ) + assert caught.value.result is not None + assert caught.value.result.scores[1] == ExperimentScore( + name="length", + kind=None, + value=None, + pass_count=0, + fail_count=0, + baseline_value=None, + delta=None, + min_score=None, + passed=None, + ) + + +@pytest.mark.parametrize( + ("options", "ci"), + [({}, None), ({"ci": False}, "true")], + ids=["CI is off", "ci=False wins over CI"], +) +def test_a_failed_check_returns_when( + monkeypatch: pytest.MonkeyPatch, options: dict[str, Any], ci: str | None +) -> None: + if ci is not None: + monkeypatch.setenv("CI", ci) + with fake_api(results=[FAILED_CHECK]) as api: + assert experiment(api, **options).check_status == "failed" + + +def test_a_result_score_that_leaves_out_its_null_fields_reads_them_as_none() -> None: + scores = [{"name": "length", "passCount": 0, "failCount": 0}] + with fake_api(results=[result(scores=scores)]) as api: + outcome = experiment(api) + + assert outcome.scores == ( + ExperimentScore( + name="length", + kind=None, + value=None, + pass_count=0, + fail_count=0, + baseline_value=None, + delta=None, + min_score=None, + passed=None, + ), + ) + + +def test_a_result_without_check_status_is_an_invalid_response() -> None: + missing = {key: value for key, value in FAILED_CHECK.items() if key != "checkStatus"} + with fake_api(results=[missing]) as api: + with pytest.raises(ExperimentError) as caught: + experiment(api) + + assert caught.value.code == "invalid_response" + + +def test_in_ci_mode_an_experiment_that_fails_raises_with_its_error( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setenv("CI", "1") + with fake_api( + results=[result(status="failed", error="0 of 2 items ran. Each one failed.")] + ) as api: + with pytest.raises(ExperimentError) as caught: + experiment(api) + + assert caught.value.code == "experiment_failed" + assert str(caught.value) == ( + "The experiment support-golden-v3 failed: 0 of 2 items ran. Each one failed.. " + f"{EXPERIMENT_URL}" + ) + + +def test_in_github_actions_it_records_the_pull_request_its_head_commit_and_the_run( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path, api: FakeApi +) -> None: + event = tmp_path / "event.json" + results = tmp_path / "results.jsonl" + event.write_text( + json.dumps( + { + "pull_request": { + "number": 42, + "title": "Shorter replies", + "html_url": "https://github.com/acme/support/pull/42", + "head": {"sha": "a1b2c3d4e5f6a1b2c3d4e5f6a1b2c3d4e5f6a1b2"}, + } + } + ) + ) + for name, value in { + "CI": "true", + "GITHUB_ACTIONS": "true", + "GITHUB_REPOSITORY": "acme/support", + "GITHUB_SHA": "f" * 40, + "GITHUB_EVENT_PATH": str(event), + "GITHUB_RUN_ID": "991", + "TELEMETRY_DEV_EXPERIMENT_DATASET": "support-golden", + "TELEMETRY_DEV_EXPERIMENT_MIN_SCORES": "correctness=0.9\nlength = 12.5", + "TELEMETRY_DEV_EXPERIMENT_RESULTS": str(results), + }.items(): + monkeypatch.setenv(name, value) + + experiment(api, dataset=None, min_scores={"correctness": 0.5, "tone": 2}) + + create = api.requests[1].body + assert create["dataset"] == "support-golden" + assert create["minScores"] == {"correctness": 0.9, "length": 12.5, "tone": 2} + assert create["ci"] == { + "provider": "github_actions", + "repository": "acme/support", + "commit": "a1b2c3d4e5f6a1b2c3d4e5f6a1b2c3d4e5f6a1b2", + "prNumber": 42, + "prTitle": "Shorter replies", + "prUrl": "https://github.com/acme/support/pull/42", + "runUrl": "https://github.com/acme/support/actions/runs/991", + } + lines = results.read_text().splitlines() + assert [json.loads(line) for line in lines] == [result()] + assert results.stat().st_mode & 0o777 == 0o600 + + +@pytest.mark.parametrize( + ("title", "sent"), + [ + ("\U0001f600" * 200, "\U0001f600" * 128), + # GitHub counts the emoji as one character, thus the title fits its limit of 256. + ("a" * 255 + "\U0001f600", "a" * 255), + ], + ids=["200 emoji", "255 letters and an emoji"], +) +def test_a_pull_request_title_is_cut_to_256_utf16_units_without_half_of_an_emoji( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path, api: FakeApi, title: str, sent: str +) -> None: + event = tmp_path / "event.json" + event.write_text(json.dumps({"pull_request": {"title": title}})) + for name, value in { + "CI": "true", + "GITHUB_ACTIONS": "true", + "GITHUB_REPOSITORY": "acme/support", + "GITHUB_SHA": "a1b2c3d", + "GITHUB_EVENT_PATH": str(event), + }.items(): + monkeypatch.setenv(name, value) + + experiment(api) + + assert api.requests[1].body["ci"]["prTitle"] == sent + + +def one(output: Any, item: ExperimentItem) -> int: + return 1 + + +REJECTED: list[tuple[dict[str, str], dict[str, Any], type[Exception], str]] = [ + ( + {}, + {"dataset": None}, + TypeError, + "Pass dataset, or set the dataset input of the experiment action", + ), + ( + {"TELEMETRY_DEV_EXPERIMENT_DATASET": "support-v2"}, + {}, + TypeError, + "dataset is support-golden, but the experiment action names support-v2. Name one dataset.", + ), + ( + {"TELEMETRY_DEV_EXPERIMENT_MIN_SCORES": "correctness=1_000"}, + {}, + TypeError, + 'TELEMETRY_DEV_EXPERIMENT_MIN_SCORES: "correctness=1_000" is not a name=number pair', + ), + ({}, {"runs_per_item": 6}, ValueError, "runs_per_item must be an integer from 1 to 5"), + ({}, {"runs_per_item": True}, ValueError, "runs_per_item must be an integer from 1 to 5"), + ({}, {"concurrency": 0}, ValueError, "concurrency must be an integer from 1 to 100"), + ( + {}, + {"scorers": {"user_feedback": one}}, + TypeError, + "user_feedback is the score of user feedback. Name the scorer another way.", + ), + ( + {}, + {"evaluators": ["correctness"], "scorers": {"Correctness": one}}, + TypeError, + "The scorer Correctness has the name of an evaluator. Rename one of them.", + ), + ({}, {"evaluators": "correctness"}, TypeError, "evaluators must be a list of at most"), +] + + +@pytest.mark.parametrize( + ("env", "options", "error", "message"), + REJECTED, + ids=[ + "no dataset", + "a dataset that the action does not name", + "min-scores that do not parse", + "runs_per_item 6", + "runs_per_item True", + "concurrency 0", + "a scorer named user_feedback", + "a scorer with the name of an evaluator", + "evaluators as one string", + ], +) +def test_raises_before_any_request( + monkeypatch: pytest.MonkeyPatch, + api: FakeApi, + env: dict[str, str], + options: dict[str, Any], + error: type[Exception], + message: str, +) -> None: + for name, value in env.items(): + monkeypatch.setenv(name, value) + + with pytest.raises(error) as caught: + experiment(api, **options) + + assert str(caught.value).startswith(message) + assert api.requests == [] + + +def test_without_an_api_key_it_raises_missing_api_key(api: FakeApi) -> None: + with pytest.raises(ExperimentError) as caught: + experiment(api, api_key="") + + assert caught.value.code == "missing_api_key" + assert api.requests == [] + + +def test_the_task_gets_each_run_of_an_item_of_the_split_with_its_run_index() -> None: + seen: list[ExperimentItem] = [] + + def task(item: ExperimentItem) -> str: + seen.append(item) + return "ok" + + with fake_api(items=[dataset_item(7, "test")]) as api: + experiment(api, split="test", runs_per_item=2, concurrency=1, task=task) + + assert seen == [ + ExperimentItem( + id=item_id(7), + number=7, + input={"question": "q7"}, + expected_output="a7", + metadata={"source": "test"}, + tags=("golden",), + split="test", + run_index=index, + ) + for index in (0, 1) + ] + assert api.requests[0].query["split"] == "test" + assert api.requests[1].body["split"] == "test" + + +def test_run_experiment_rejects_an_async_task(api: FakeApi) -> None: + async def task(item: ExperimentItem) -> str: + return "ok" + + with pytest.raises(TypeError, match="task is async: use run_experiment_async"): + experiment(api, task=task) + assert api.requests == [] + + +async def test_run_experiment_async_runs_async_tasks_and_scorers_at_the_same_time( + make: MakeClient, +) -> None: + memory = make(environment="experiment") + running = 0 + most = 0 + + async def task(item: ExperimentItem) -> str: + nonlocal running, most + running += 1 + most = max(most, running) + with start_span("chat gpt-4o", type="generation"): + await asyncio.sleep(0.05) + running -= 1 + return f"r{item.number}" + + async def exact(output: Any, item: ExperimentItem) -> bool: + await asyncio.sleep(0) + return output == f"r{item.number}" + + with fake_api(items=[dataset_item(number) for number in range(1, 9)]) as api: + outcome = await run_experiment_async( + api_key="td_live_test", + base_url=api.url, + dataset="support-golden", + task=task, + scorers={"exact": exact}, + concurrency=4, + ) + + assert outcome.status == "completed" + assert most == 4 + posted = sorted( + (run for items in api.posts() for run in items), key=lambda run: str(run["itemId"]) + ) + assert [run["scores"] for run in posted] == [ + [{"name": "exact", "value": 1, "label": "pass"}] + ] * 8 + chats = [ + span for span in memory.span_exporter.get_finished_spans() if span.name == "chat gpt-4o" + ] + by_trace = {f"{span.context.trace_id:032x}": span for span in chats} + for run in posted: + attributes = dict(by_trace[run["traceId"]].attributes or {}) + assert attributes["td.dataset.item_id"] == run["itemId"] diff --git a/sdks/python/tests/test_scores.py b/sdks/python/tests/test_scores.py new file mode 100644 index 00000000..d72d0d2e --- /dev/null +++ b/sdks/python/tests/test_scores.py @@ -0,0 +1,206 @@ +from __future__ import annotations + +from collections.abc import Generator +from typing import Any + +import pytest + +import telemetry_dev +from telemetry_dev import ScoreError, score, score_async + +from .conftest import MakeClient +from .fake_api import ApiRequest, FakeApi, Reply, fake_api + +TRACE_ID = "4bf92f3577b34da6a3ce929d0e0e4736" +SPAN_ID = "00f067aa0ba902b7" + + +def replying(reply: Reply) -> Generator[FakeApi]: + def answer(request: ApiRequest) -> Reply: + return reply + + with fake_api(override=answer) as api: + yield api + + +@pytest.fixture +def accepted() -> Generator[FakeApi]: + yield from replying((202, {"id": "score-1"}, {})) + + +@pytest.fixture +def errors(make: MakeClient) -> list[ScoreError]: + caught: list[ScoreError] = [] + make(on_error=caught.append) + return caught + + +def test_posts_the_score_with_the_key_and_base_url_of_init_and_returns_its_id( + make: MakeClient, accepted: FakeApi +) -> None: + make(api_key="td_live_key", base_url=f"{accepted.url}/") + + assert ( + score( + name=" user_feedback ", + trace_id=TRACE_ID, + span_id=SPAN_ID, + value=0, + comment="Wrong refund amount", + ) + == "score-1" + ) + + [request] = accepted.requests + assert request.route == "POST /v1/scores" + assert request.headers["authorization"] == "Bearer td_live_key" + assert request.headers["x-telemetry-dev-sdk"] == "telemetry-dev" + assert request.headers["user-agent"] == f"telemetry-dev-python/{telemetry_dev.__version__}" + assert request.headers["content-type"] == "application/json" + assert request.body == { + "name": "user_feedback", + "traceId": TRACE_ID, + "spanId": SPAN_ID, + "value": 0, + "comment": "Wrong refund amount", + } + + +async def test_a_session_score_needs_no_trace_and_a_label_goes_trimmed(accepted: FakeApi) -> None: + returned = await score_async( + api_key="td_live_key", + base_url=accepted.url, + name="resolved", + session_id="chat-9", + label=" yes ", + ) + + assert returned == "score-1" + assert accepted.requests[0].body == {"name": "resolved", "sessionId": "chat-9", "label": "yes"} + + +@pytest.mark.parametrize( + ("options", "message"), + [ + ({"trace_id": None}, "Pass trace_id or session_id"), + ( + {"trace_id": "0" * 32}, + "trace_id must be 32 lowercase hex digits and not all zeros", + ), + ( + {"trace_id": TRACE_ID.upper()}, + "trace_id must be 32 lowercase hex digits and not all zeros", + ), + ( + {"trace_id": None, "session_id": "s", "span_id": SPAN_ID}, + "Pass the trace_id of the span", + ), + ({"value": None}, "Pass a value or a label"), + ({"value": float("nan")}, "value must be a finite number"), + ({"value": True}, "value must be a finite number"), + ({"value": 10**400}, "value must be a finite number"), + ( + {"name": "user_feedback", "value": 1, "label": "up"}, + "user_feedback takes the value 1 for up or 0 for down, and no label", + ), + ( + {"name": "user_feedback", "value": 0.5}, + "user_feedback takes the value 1 for up or 0 for down, and no label", + ), + ({"value": 0, "label": "pass"}, "The label pass goes with the value 1"), + ({"value": 1, "label": "fail"}, "The label fail goes with the value 0"), + ({"name": " "}, "name must be a string of 1 to 64 characters"), + # 33 emoji are 33 characters in Python but 66 UTF-16 units for the API. + ({"name": "\U0001f600" * 33}, "name must be a string of 1 to 64 characters"), + ({"comment": "x" * 4001}, "comment must be a string of at most 4000 characters"), + ], + ids=[ + "no trace and no session", + "the all-zero trace id of a span that did not record", + "an uppercase trace id", + "a span without its trace", + "no value and no label", + "a value that is not finite", + "a bool value", + "an int too large for a float", + "a label for user feedback", + "user feedback of 0.5", + "a pass of 0", + "a fail of 1", + "a blank name", + "a name of 66 UTF-16 units", + "a comment of 4001 characters", + ], +) +def test_reports_invalid_score_and_sends_nothing( + errors: list[ScoreError], accepted: FakeApi, options: dict[str, Any], message: str +) -> None: + arguments: dict[str, Any] = { + "api_key": "td_live_key", + "base_url": accepted.url, + "name": "helpful", + "trace_id": TRACE_ID, + "value": 1, + } + arguments.update(options) + + assert score(**arguments) is None + assert accepted.requests == [] + assert [(error.code, error.message) for error in errors] == [("invalid_score", message)] + + +FAILURES: list[tuple[Reply, str, int | None]] = [ + ( + (403, {"error": {"code": "insufficient_scope", "message": "No scores"}}, {}), + "insufficient_scope", + 403, + ), + ((503, {}, {"retry-after": "1"}), "http_503", 503), + ((202, {}, {}), "invalid_response", None), +] + + +@pytest.mark.parametrize( + ("reply", "code", "status"), + FAILURES, + ids=["the API error", "a status with no API error", "a body with no id"], +) +def test_reports_the_failure_tries_once_and_returns_none( + errors: list[ScoreError], reply: Reply, code: str, status: int | None +) -> None: + for api in replying(reply): + assert ( + score( + api_key="td_live_key", base_url=api.url, name="helpful", trace_id=TRACE_ID, value=1 + ) + is None + ) + assert len(api.requests) == 1 + + assert [(error.code, error.status) for error in errors] == [(code, status)] + + +def test_without_an_api_key_it_reports_missing_api_key( + errors: list[ScoreError], accepted: FakeApi +) -> None: + score(api_key="", base_url=accepted.url, name="helpful", trace_id=TRACE_ID, value=1) + + assert accepted.requests == [] + assert [error.code for error in errors] == ["missing_api_key"] + + +def test_does_not_follow_a_redirect_so_the_key_stays_with_the_api_host( + errors: list[ScoreError], +) -> None: + with fake_api() as other: + + def redirect(request: ApiRequest) -> Reply: + return 307, {}, {"location": f"{other.url}/v1/scores"} + + with fake_api(override=redirect) as api: + score( + api_key="td_live_key", base_url=api.url, name="helpful", trace_id=TRACE_ID, value=1 + ) + + assert other.requests == [] + assert [(error.code, error.status) for error in errors] == [("http_307", 307)]