From 2f479a57767a13cdad892dc3aafa3ca0b736bfa7 Mon Sep 17 00:00:00 2001 From: ephraimduncan Date: Thu, 29 Oct 2026 23:01:41 +0000 Subject: [PATCH 01/10] feat(sdk): add runExperiment and score runExperiment gets a dataset version page by page and makes the experiment. It runs the task on each item, with concurrency and runs for each item. It sends the outputs and the local scores in batches of at most 100. Then it completes the experiment, waits for the server evaluators, and returns the summary. Each item run has its own trace. Each span of the run has td.experiment.id and td.dataset.item_id. In CI mode, runExperiment records the GitHub Actions run, and it rejects when the check is not successful. score sends a score on a trace, a span, or a session to /v1/scores. It never rejects. It sends its errors to onError. getPrompt and the new calls now use one function for the key, the base URL, and the Retry-After header. --- packages/sdk/src/api.ts | 199 +++++ packages/sdk/src/ci.ts | 159 ++++ packages/sdk/src/experiments.ts | 958 +++++++++++++++++++++++++ packages/sdk/src/index.ts | 14 + packages/sdk/src/prompts.ts | 48 +- packages/sdk/src/scores.ts | 170 +++++ packages/sdk/tests/experiments.test.ts | 723 +++++++++++++++++++ packages/sdk/tests/scores.test.ts | 220 ++++++ 8 files changed, 2448 insertions(+), 43 deletions(-) create mode 100644 packages/sdk/src/api.ts create mode 100644 packages/sdk/src/ci.ts create mode 100644 packages/sdk/src/experiments.ts create mode 100644 packages/sdk/src/scores.ts create mode 100644 packages/sdk/tests/experiments.test.ts create mode 100644 packages/sdk/tests/scores.test.ts diff --git a/packages/sdk/src/api.ts b/packages/sdk/src/api.ts new file mode 100644 index 0000000..8274701 --- /dev/null +++ b/packages/sdk/src/api.ts @@ -0,0 +1,199 @@ +import { DEFAULT_BASE_URL, resolveEnv } from "@telemetry-dev/otel"; + +import { currentClient } from "./client.ts"; + +/** Where the calls to the telemetry.dev API go, and with which key. */ +export interface ApiOptions { + /** API key. Falls back to the initialized client, then `TELEMETRY_DEV_API_KEY`. */ + apiKey?: string; + /** Falls back to the initialized client, then `TELEMETRY_DEV_BASE_URL`, then `https://ingest.telemetry.dev`. */ + baseUrl?: string; + /** Falls back to the initialized client's `fetch`, then `globalThis.fetch`. */ + fetch?: typeof fetch; +} + +export interface Api { + apiKey: string | undefined; + baseUrl: string; + sdkName: string; + fetchImpl: typeof fetch; + onError: ((cause: Error) => void) | undefined; +} + +/** An error class with the shape of `PromptError`: a code, a message, and the HTTP status. */ +export type ApiErrorType = new ( + code: string, + message: string, + status?: number, + options?: { cause?: unknown }, +) => E; + +interface RequestOptions { + body?: unknown; + query?: Record; + timeoutMs: number; + /** + * The server takes a repeat of this request at most once, so a retry after a timeout, a network + * error, or a 5xx is safe. Other requests try again only after a 429 or a 503: the server did + * not take the request then. + */ + idempotent: boolean; + /** Default 3. */ + attempts?: number; +} + +type Attempt = + | { ok: true; body: unknown } + | { ok: false; error: Error; status?: number; retryAfterMs?: number }; + +/** A longer Retry-After waits only this long: the API counts requests by the minute. */ +const MAX_RETRY_AFTER_MS = 60_000; +/** IMF-fixdate, the HTTP date that servers send: `Sun, 06 Nov 1994 08:49:37 GMT`. */ +const HTTP_DATE = /^[A-Z][a-z]{2}, \d{2} [A-Z][a-z]{2} \d{4} \d{2}:\d{2}:\d{2} GMT$/; + +/** The key, base URL, and fetch of a call: its options, then the initialized client, then the env. */ +export function resolveApi(options: ApiOptions): Api { + const config = currentClient().core?.config; + const env = resolveEnv(); + + return { + apiKey: options.apiKey ?? config?.apiKey ?? env.TELEMETRY_DEV_API_KEY, + // An empty value counts as unset, as in the Python SDK. + baseUrl: ( + options.baseUrl || + config?.baseUrl || + env.TELEMETRY_DEV_BASE_URL || + DEFAULT_BASE_URL + ).replace(/\/+$/, ""), + sdkName: config?.sdkName ?? "@telemetry-dev/sdk", + fetchImpl: options.fetch ?? config?.fetchImpl ?? globalThis.fetch, + onError: config?.onError, + }; +} + +/** + * Sends one JSON request to the telemetry.dev API and returns the JSON of the response. A response + * that is not 2xx fails with the `{ error: { code, message } }` of its body, or `http_`. + * A failure that `options.idempotent` allows tries again, after the Retry-After of the response + * or after 1 and then 2 seconds. + */ +export async function request( + api: Api & { apiKey: string }, + ErrorType: ApiErrorType, + method: "GET" | "POST", + path: string, + options: RequestOptions, +): Promise { + const url = new URL(`${api.baseUrl}${path}`); + + for (const [key, value] of Object.entries(options.query ?? {})) { + if (value !== undefined) url.searchParams.set(key, value); + } + + // Outside send(), so that a value JSON cannot hold fails as itself, not as a network error. + const body = options.body === undefined ? undefined : JSON.stringify(options.body); + const attempts = options.attempts ?? 3; + + for (let attempt = 1; ; attempt++) { + const result = await send(api, ErrorType, method, url, body, options.timeoutMs); + + if (result.ok) return result.body; + + if (attempt >= attempts || !retryable(result.status, options.idempotent)) throw result.error; + + await sleep(Math.min(result.retryAfterMs ?? 1000 * attempt, MAX_RETRY_AFTER_MS)); + } +} + +/** The wait that the Retry-After of a 429 or a 503 asks for, in seconds or as an HTTP date. */ +export function retryAfterMs(res: Response): number | undefined { + if (res.status !== 429 && res.status !== 503) return undefined; + + const value = res.headers.get("retry-after")?.trim(); + + if (!value) return undefined; + + let ms: number; + + if (/^\d+$/.test(value)) ms = Number(value) * 1000; + else if (HTTP_DATE.test(value)) ms = Date.parse(value) - Date.now(); + else return undefined; + + return ms > 0 ? ms : undefined; +} + +export function sleep(ms: number): Promise { + return new Promise((resolve) => setTimeout(resolve, ms)); +} + +export function isRecord(value: unknown): value is Record { + return typeof value === "object" && value !== null && !Array.isArray(value); +} + +async function send( + api: Api & { apiKey: string }, + ErrorType: ApiErrorType, + method: "GET" | "POST", + url: URL, + body: string | undefined, + timeoutMs: number, +): Promise { + const controller = new AbortController(); + const timer = setTimeout(() => controller.abort(), timeoutMs); + + try { + const res = await api.fetchImpl(url, { + method, + headers: { + authorization: `Bearer ${api.apiKey}`, + "x-telemetry-dev-sdk": api.sdkName, + ...(body === undefined ? {} : { "content-type": "application/json" }), + }, + body, + signal: controller.signal, + // A redirect to another host must not get the API key, so a 3xx fails below like any status + // that is not OK. Cloudflare Workers reject redirect: "error". + redirect: "manual", + }); + + const json: unknown = await res.json().catch(() => undefined); + + if (res.ok) return { ok: true, body: json }; + + const error = isRecord(json) && isRecord(json.error) ? json.error : {}; + + return { + ok: false, + status: res.status, + retryAfterMs: retryAfterMs(res), + error: new ErrorType( + typeof error.code === "string" ? error.code : `http_${res.status}`, + typeof error.message === "string" ? error.message : `telemetry.dev returned ${res.status}`, + res.status, + ), + }; + } catch (error) { + // The timeout can also cut the body, which then reads as an invalid response. + if (controller.signal.aborted) { + return { + ok: false, + error: new ErrorType("timeout", `${method} ${url.pathname} took more than ${timeoutMs} ms`), + }; + } + + return { + ok: false, + error: new ErrorType("network_error", "Could not reach telemetry.dev", undefined, { + cause: error, + }), + }; + } finally { + clearTimeout(timer); + } +} + +function retryable(status: number | undefined, idempotent: boolean): boolean { + if (status === 429 || status === 503) return true; + + return idempotent && (status === undefined || status >= 500); +} diff --git a/packages/sdk/src/ci.ts b/packages/sdk/src/ci.ts new file mode 100644 index 0000000..45eb20c --- /dev/null +++ b/packages/sdk/src/ci.ts @@ -0,0 +1,159 @@ +import type * as fs from "node:fs"; + +import { reportError } from "@telemetry-dev/otel"; + +/** The pull request and the workflow run of an experiment that runs in GitHub Actions. */ +export interface CiRun { + provider: "github_actions"; + /** `owner/name`. */ + repository: string; + commit: string; + prNumber?: number; + prTitle?: string; + prUrl?: string; + runUrl?: string; +} + +type Env = Record; + +type RuntimeGlobal = typeof globalThis & { + process?: { getBuiltinModule?: (id: string) => unknown }; +}; + +const REPOSITORY = /^[A-Za-z0-9_.-]+\/[A-Za-z0-9_.-]+$/; +const COMMIT = /^[0-9a-f]{7,64}$/i; +const HTTPS_URL = /^https:\/\/\S+$/; +/** A plain decimal, so that the TypeScript and Python SDKs read the same marks. */ +const DECIMAL = /^-?\d+(\.\d+)?$/; +const MAX_REPOSITORY_LENGTH = 200; +const MAX_URL_LENGTH = 2_048; +const MAX_PR_TITLE_LENGTH = 256; + +/** CI mode is on when `CI` is `true` or `1`, as CI services set it. */ +export function inCi(env: Env): boolean { + return env.CI === "true" || env.CI === "1"; +} + +/** + * The run that GitHub Actions describes in its variables, or undefined outside GitHub Actions. On a + * pull request the commit is the head of the pull request: `GITHUB_SHA` is then a merge commit + * that nobody pushed. + */ +export function githubRun(env: Env): CiRun | undefined { + const repository = env.GITHUB_REPOSITORY; + + if ( + env.GITHUB_ACTIONS !== "true" || + !repository || + repository.length > MAX_REPOSITORY_LENGTH || + !REPOSITORY.test(repository) + ) { + return undefined; + } + + const pull = pullRequest(env.GITHUB_EVENT_PATH); + const commit = pull?.commit ?? env.GITHUB_SHA; + + if (!commit || !COMMIT.test(commit)) return undefined; + + const server = (env.GITHUB_SERVER_URL ?? "https://github.com").replace(/\/+$/, ""); + const runId = env.GITHUB_RUN_ID; + + return { + provider: "github_actions", + repository, + commit, + prNumber: pull?.number, + prTitle: pull?.title, + prUrl: pull?.url, + runUrl: + runId && /^\d+$/.test(runId) + ? httpsUrl(`${server}/${repository}/actions/runs/${runId}`) + : undefined, + }; +} + +/** + * The pass marks in `TELEMETRY_DEV_EXPERIMENT_MIN_SCORES`, which the experiment action sets from + * its `min-scores` input: `name=value` pairs, separated by commas or new lines. + */ +export function minScoresFromEnv(env: Env): Record { + const scores: Record = {}; + + for (const pair of (env.TELEMETRY_DEV_EXPERIMENT_MIN_SCORES ?? "").split(/[,\n]/)) { + if (pair.trim() === "") continue; + const [name, value, ...rest] = pair.split("=").map((part) => part.trim()); + + if (!name || !value || rest.length > 0 || !DECIMAL.test(value)) { + throw new TypeError( + `TELEMETRY_DEV_EXPERIMENT_MIN_SCORES: "${pair.trim()}" is not a name=number pair`, + ); + } + + scores[name] = Number(value); + } + + return scores; +} + +/** + * Adds the result to the file in `TELEMETRY_DEV_EXPERIMENT_RESULTS` as one line of JSON. The + * experiment action reads the file to write the job summary and the comment on the pull request. + */ +export function writeResult(env: Env, result: unknown, onError?: (cause: Error) => void): void { + const path = env.TELEMETRY_DEV_EXPERIMENT_RESULTS; + + if (!path) return; + + try { + const fs = nodeFs(); + + if (!fs) throw new Error("This runtime cannot write TELEMETRY_DEV_EXPERIMENT_RESULTS"); + + fs.appendFileSync(path, `${JSON.stringify(result)}\n`, { mode: 0o600 }); + } catch (error) { + reportError(onError, error); + } +} + +function pullRequest( + eventPath: string | undefined, +): { number?: number; title?: string; url?: string; commit?: string } | undefined { + if (!eventPath) return undefined; + + let event: unknown; + + try { + event = JSON.parse(nodeFs()?.readFileSync(eventPath, "utf8") ?? "null"); + } catch { + return undefined; + } + + const pull = (event as { pull_request?: Record } | null)?.pull_request; + + if (typeof pull !== "object" || pull === null) return undefined; + + const head = pull.head as { sha?: unknown } | undefined; + + return { + number: + Number.isSafeInteger(pull.number) && (pull.number as number) > 0 + ? (pull.number as number) + : undefined, + title: typeof pull.title === "string" ? pull.title.slice(0, MAX_PR_TITLE_LENGTH) : undefined, + url: typeof pull.html_url === "string" ? httpsUrl(pull.html_url) : undefined, + commit: typeof head?.sha === "string" ? head.sha : undefined, + }; +} + +function httpsUrl(url: string): string | undefined { + return url.length <= MAX_URL_LENGTH && HTTPS_URL.test(url) ? url : undefined; +} + +// Read at call time, with no static `node:` import, so that the package still loads in runtimes +// without a file system. +function nodeFs(): typeof fs | undefined { + return (globalThis as RuntimeGlobal).process?.getBuiltinModule?.("node:fs") as + | typeof fs + | undefined; +} diff --git a/packages/sdk/src/experiments.ts b/packages/sdk/src/experiments.ts new file mode 100644 index 0000000..bb90d0d --- /dev/null +++ b/packages/sdk/src/experiments.ts @@ -0,0 +1,958 @@ +import { type Context, trace } from "@opentelemetry/api"; +import { + activeContext, + diag, + jsonAttr, + PROPAGATED_KEY, + propagatedFromContext, + reportError, + resolveEnv, +} from "@telemetry-dev/otel"; + +import { type Api, type ApiOptions, isRecord, request, resolveApi, sleep } from "./api.ts"; +import { truncate } from "./capture.ts"; +import { type CiRun, githubRun, inCi, minScoresFromEnv, writeResult } from "./ci.ts"; +import { currentClient, flush } from "./client.ts"; +import { checkValue, MAX_REASON_LENGTH, MAX_SCORE_NAME_LENGTH, USER_FEEDBACK } from "./scores.ts"; +import { type SpanHandle, startActiveSpan } from "./span.ts"; + +/** An item of a dataset version, as `GET /v1/datasets/{name}` returns it. */ +export interface DatasetItem { + /** The key of the item. It stays the same across dataset versions. */ + readonly id: string; + readonly number: number; + readonly input: Readonly>; + readonly expectedOutput: string | null; + readonly metadata: Readonly>; + readonly tags: readonly string[]; + readonly split: string | null; +} + +/** One run of a dataset item. With `runsPerItem` above 1, an item runs more than one time. */ +export interface ExperimentItem extends DatasetItem { + /** 0 to `runsPerItem` - 1. */ + readonly runIndex: number; +} + +/** + * What a scorer returns. `true` and `false` are a pass and a fail, a number is the value, and a + * string is the label. Null or undefined gives no score. + */ +export type ScorerResult = + | boolean + | number + | string + | { value?: number | boolean; label?: string; reason?: string } + | null + | undefined; + +/** Scores the output of an item on your machine. The score has the name of the scorer. */ +export type Scorer = ( + output: unknown, + item: ExperimentItem, +) => ScorerResult | Promise; + +export interface RunExperimentOptions extends ApiOptions { + /** Runs the code under test on one item. Its return value is the output of the item. */ + task: (item: ExperimentItem) => unknown; + /** + * The name of the dataset. Default: the `dataset` input of the experiment action, which it + * sets in `TELEMETRY_DEV_EXPERIMENT_DATASET`. When both are set, they must be the same. + */ + dataset?: string; + /** Unique in the project. telemetry.dev picks a name when it is not set. */ + name?: string; + /** Default: the newest version. */ + datasetVersion?: number; + /** Run only the items of this split. */ + split?: string; + /** Evaluators of the project that score the outputs on the server, by name. */ + evaluators?: readonly string[]; + /** Scorers that run here, by score name. */ + scorers?: Readonly>; + /** + * Pass marks by score name: the lowest pass rate (0 to 1) of a pass/fail score, or the lowest + * mean of a number score. The `min-scores` input of the experiment action adds to them and wins + * for the same name. + */ + minScores?: Readonly>; + /** 1 to 5. Default 1. */ + runsPerItem?: number; + /** Item runs at the same time, 1 to 100. Default 4. */ + concurrency?: number; + metadata?: Readonly>; + /** + * Records the GitHub Actions run, and rejects when the check fails or the experiment does not + * complete. Default: on when the `CI` variable is `true` or `1`. + */ + ci?: boolean; +} + +export interface ExperimentScore { + readonly name: string; + /** `pass_fail`, `number`, `labels`, `a_b_tie`, or `up_down`. Null before the score has values. */ + readonly kind: string | null; + /** The pass rate (0 to 1) of a pass/fail or up/down score, or the mean of a number score. */ + readonly value: number | null; + readonly passCount: number; + readonly failCount: number; + /** The value in the experiment that this one compares with, by default the dataset baseline. */ + readonly baselineValue: number | null; + /** Whole points of a pass rate, or the change of a mean to one decimal. */ + readonly delta: number | null; + readonly minScore: number | null; + /** Null when the score has no pass mark. */ + readonly passed: boolean | null; +} + +export interface ExperimentResult { + readonly id: string; + readonly name: string; + /** `completed`, `failed`, or `stopped`. */ + readonly status: string; + /** Null when the experiment has no pass marks. */ + readonly checkStatus: "passed" | "failed" | null; + readonly itemsTotal: number; + readonly itemsDone: number; + readonly itemsFailed: number; + /** Why the experiment failed. */ + readonly error: string | null; + readonly scores: readonly ExperimentScore[]; + readonly url: string; +} + +/** + * Why runExperiment failed. `code` is the code of the API error (`dataset_not_found`, + * `insufficient_scope`, `invalid_api_key`, …), or `missing_api_key`, `timeout`, `network_error`, + * `invalid_response`, or `dataset_empty`. In CI mode it is also `check_failed` or + * `experiment_failed`, and `result` holds the result. A response with a different status and no + * API error gives `http_`. A scorer that throws or returns a value that is not a score + * reports a `scorer_failed` error through `onError`. + */ +export class ExperimentError extends Error { + override readonly name = "ExperimentError"; + readonly result?: ExperimentResult; + + constructor( + readonly code: string, + message: string, + readonly status?: number, + options?: { cause?: unknown; result?: ExperimentResult }, + ) { + super(message, options); + this.result = options?.result; + } +} + +interface Run { + dataset: string; + task: (item: ExperimentItem) => unknown; + name?: string; + datasetVersion?: number; + split?: string; + evaluators?: readonly string[]; + scorers: [string, Scorer][]; + minScores?: Record; + runsPerItem: number; + concurrency: number; + metadata?: Readonly>; + ci: CiRun | undefined; + inCi: boolean; +} + +interface Created { + id: string; + name: string; + itemsTotal: number; + url: string; +} + +interface DatasetPage { + version: number; + items: DatasetItem[]; + nextCursor: string | null; +} + +interface ItemScore { + name: string; + value?: number; + label?: string; + reason?: string; +} + +interface ItemResult { + itemId: string; + runIndex: number; + output: string | null; + traceId?: string; + latencyMs: number; + error?: string; + scores?: ItemScore[]; +} + +type Authed = Api & { apiKey: string }; + +/** The `experiment` environment, which pages, metrics, and alerts leave out by default. */ +const EXPERIMENT_ENVIRONMENT = "experiment"; +const MAX_RUNS_PER_ITEM = 5; +const MAX_CONCURRENCY = 100; +/** Server evaluators of one experiment, and scorers: an item takes at most 20 scores. */ +const MAX_EVALUATORS = 20; +const MAX_MIN_SCORES = 20; +const MAX_ERROR_LENGTH = 4_000; +const MAX_OUTPUT_LENGTH = 100_000; +const MAX_DATASET_ITEMS = 10_000; +/** Items in one `POST /v1/experiments/{id}/items`. */ +const MAX_BATCH_ITEMS = 100; +/** Bytes of JSON in one post of items, well under the 16 MiB body limit of the API. */ +const MAX_BATCH_BYTES = 4 * 1024 * 1024; +const REQUEST_TIMEOUT_MS = 30_000; +/** How long runExperiment waits for the server evaluators after the last item. */ +const MAX_WAIT_MS = 30 * 60_000; +const MAX_POLL_INTERVAL_MS = 5_000; +const FINISHED = new Set(["completed", "failed", "stopped"]); +const RATE_KINDS = new Set(["pass_fail", "up_down"]); + +/** + * Runs `task` on each item of a dataset and records the result as an experiment in telemetry.dev. + * Each item run is its own trace, with `td.experiment.id` and `td.dataset.item_id` on each span. + * The outputs go to telemetry.dev in batches of at most 100, with the scores of `scorers`. Then + * runExperiment waits for the `evaluators` of the server, prints the result, and returns it. + * + * The run cannot start without its dataset, so runExperiment rejects with an `ExperimentError` + * when a request fails. A task that throws fails only its item. In CI mode, runExperiment rejects + * when the check fails, so the process exits with an error. + */ +export async function runExperiment(options: RunExperimentOptions): Promise { + const env = resolveEnv(); + const run = checkOptions(options, env); + const resolved = resolveApi(options); + + if (!resolved.apiKey) { + throw new ExperimentError( + "missing_api_key", + "No API key: pass apiKey or set TELEMETRY_DEV_API_KEY", + ); + } + + const api = { ...resolved, apiKey: resolved.apiKey }; + warnAboutClient(); + + const first = await fetchPage(api, run, run.datasetVersion, undefined); + + if (first.items.length === 0) { + throw new ExperimentError( + "dataset_empty", + `The dataset ${run.dataset} has no items${run.split === undefined ? "" : ` in the split ${run.split}`}`, + ); + } + + const created = parseCreated( + await request(api, ExperimentError, "POST", "/v1/experiments", { + body: { + name: run.name, + dataset: run.dataset, + datasetVersion: first.version, + split: run.split, + evaluators: run.evaluators, + minScores: run.minScores, + runsPerItem: run.runsPerItem, + ci: run.ci, + metadata: run.metadata, + }, + timeoutMs: REQUEST_TIMEOUT_MS, + // A repeat after a lost response would start a second experiment. + idempotent: false, + }), + ); + + console.log( + `Experiment ${created.name}: ${created.itemsTotal / run.runsPerItem} items` + + (run.runsPerItem > 1 ? `, ${run.runsPerItem} runs each` : "") + + `. ${created.url}`, + ); + + await runItems(api, run, created, itemRuns(api, run, first)); + // The spans of the items reach ingest before the server reads their cost. + await flush(); + + await request( + api, + ExperimentError, + "POST", + `/v1/experiments/${encodeURIComponent(created.id)}/complete`, + { timeoutMs: REQUEST_TIMEOUT_MS, idempotent: true }, + ); + + const { result, body } = await waitForResult(api, created.id); + writeResult(env, body, api.onError); + console.log(describe(result)); + + if (run.inCi) checkResult(result); + + return result; +} + +function checkOptions(options: RunExperimentOptions, env: Record): Run { + if (typeof options.task !== "function") throw new TypeError("task must be a function"); + + for (const [key, value] of [ + ["dataset", options.dataset], + ["name", options.name], + ["split", options.split], + ] as const) { + if (value !== undefined && (typeof value !== "string" || value.length === 0)) { + throw new TypeError(`${key} must be a non-empty string`); + } + } + + const actionDataset = env.TELEMETRY_DEV_EXPERIMENT_DATASET || undefined; + const dataset = options.dataset ?? actionDataset; + + if (dataset === undefined) { + throw new TypeError("Pass dataset, or set the dataset input of the experiment action"); + } + + if (actionDataset !== undefined && actionDataset !== dataset) { + throw new TypeError( + `dataset is ${dataset}, but the experiment action names ${actionDataset}. Name one dataset.`, + ); + } + + const { datasetVersion } = options; + + if ( + datasetVersion !== undefined && + !(Number.isSafeInteger(datasetVersion) && datasetVersion >= 1) + ) { + throw new RangeError("datasetVersion must be a positive integer"); + } + + const evaluators = options.evaluators; + + if ( + evaluators !== undefined && + (!Array.isArray(evaluators) || + evaluators.length > MAX_EVALUATORS || + !evaluators.every((name) => typeof name === "string" && validScoreName(name))) + ) { + throw new TypeError(`evaluators must be a list of at most ${MAX_EVALUATORS} evaluator names`); + } + + const scorers = Object.entries(options.scorers ?? {}); + const evaluatorNames = new Set(evaluators?.map((name) => name.toLowerCase())); + + if (scorers.length > MAX_EVALUATORS) { + throw new TypeError(`scorers can have at most ${MAX_EVALUATORS} scorers`); + } + + for (const [name, scorer] of scorers) { + if (!validScoreName(name) || typeof scorer !== "function") { + throw new TypeError( + `scorers must map score names of 1 to ${MAX_SCORE_NAME_LENGTH} characters to functions`, + ); + } + + if (name === USER_FEEDBACK) { + throw new TypeError( + `${USER_FEEDBACK} is the score of user feedback. Name the scorer another way.`, + ); + } + + if (evaluatorNames.has(name.toLowerCase())) { + throw new TypeError(`The scorer ${name} has the name of an evaluator. Rename one of them.`); + } + } + + const minScores = { ...options.minScores, ...minScoresFromEnv(env) }; + + if ( + Object.keys(minScores).length > MAX_MIN_SCORES || + !Object.entries(minScores).every( + ([name, value]) => validScoreName(name) && Number.isFinite(value), + ) + ) { + throw new TypeError(`minScores must map at most ${MAX_MIN_SCORES} score names to numbers`); + } + + const runsPerItem = options.runsPerItem ?? 1; + + if (!Number.isInteger(runsPerItem) || runsPerItem < 1 || runsPerItem > MAX_RUNS_PER_ITEM) { + throw new RangeError(`runsPerItem must be an integer from 1 to ${MAX_RUNS_PER_ITEM}`); + } + + const concurrency = options.concurrency ?? 4; + + if (!Number.isInteger(concurrency) || concurrency < 1 || concurrency > MAX_CONCURRENCY) { + throw new RangeError(`concurrency must be an integer from 1 to ${MAX_CONCURRENCY}`); + } + + if (options.metadata !== undefined && !isRecord(options.metadata)) { + throw new TypeError("metadata must be an object"); + } + + if (options.ci !== undefined && typeof options.ci !== "boolean") { + throw new TypeError("ci must be a boolean"); + } + + const ci = options.ci ?? inCi(env); + + return { + dataset, + task: options.task, + name: options.name, + datasetVersion, + split: options.split, + evaluators, + scorers, + minScores: Object.keys(minScores).length > 0 ? minScores : undefined, + runsPerItem, + concurrency, + metadata: options.metadata, + ci: ci ? githubRun(env) : undefined, + inCi: ci, + }; +} + +/** A trimmed name of 1 to 64 characters, as the API stores score names. */ +function validScoreName(name: string): boolean { + return name.length > 0 && name.length <= MAX_SCORE_NAME_LENGTH && name.trim() === name; +} + +function warnAboutClient(): void { + const environment = currentClient().core?.config.environment; + + if (environment === undefined) { + diag.warn( + 'runExperiment: no telemetry client, so the items record no traces. Call init({ environment: "experiment" }) first.', + ); + } else if (environment.trim().toLowerCase() !== EXPERIMENT_ENVIRONMENT) { + diag.warn( + `runExperiment: the traces of the items go to the environment ${environment}. Call init({ environment: "experiment" }) to keep them out of it.`, + ); + } +} + +/** + * The runs of each item of the dataset version, page by page, so that only one page is in memory. + * Workers share the generator: an async generator answers each `next()` in turn. + */ +async function* itemRuns( + api: Authed, + run: Run, + first: DatasetPage, +): AsyncGenerator { + let page = first; + let count = 0; + + for (;;) { + count += page.items.length; + + if (count > MAX_DATASET_ITEMS) { + throw new ExperimentError( + "invalid_response", + `telemetry.dev returned more than ${MAX_DATASET_ITEMS} items`, + ); + } + + for (const item of page.items) { + for (let runIndex = 0; runIndex < run.runsPerItem; runIndex++) yield { ...item, runIndex }; + } + + if (page.nextCursor === null) return; + + page = await fetchPage(api, run, page.version, page.nextCursor); + } +} + +/** One page of a dataset version. A page after the first names the version of the first. */ +async function fetchPage( + api: Authed, + run: Run, + version: number | undefined, + cursor: string | undefined, +): Promise { + const page = parseDatasetPage( + await request(api, ExperimentError, "GET", `/v1/datasets/${encodeURIComponent(run.dataset)}`, { + query: { version: version?.toString(), split: run.split, cursor }, + timeoutMs: REQUEST_TIMEOUT_MS, + idempotent: true, + }), + ); + + if ( + !page || + (version !== undefined && page.version !== version) || + (page.nextCursor !== null && (page.items.length === 0 || page.nextCursor === cursor)) + ) { + throw new ExperimentError( + "invalid_response", + "telemetry.dev returned a dataset it cannot read", + ); + } + + return page; +} + +/** + * Runs the items, `concurrency` at a time, and posts each result. A post or a page fetch that + * fails stops the run: no new item starts, and runExperiment rejects when the running items end. + */ +async function runItems( + api: Authed, + run: Run, + experiment: Created, + runs: AsyncGenerator, +): Promise { + const batch = itemBatch((results) => + request( + api, + ExperimentError, + "POST", + `/v1/experiments/${encodeURIComponent(experiment.id)}/items`, + // The server skips an item run that it has, so a repeat is safe. + { body: { items: results }, timeoutMs: REQUEST_TIMEOUT_MS, idempotent: true }, + ), + ); + + let failure: { error: unknown } | undefined; + + const worker = async () => { + while (failure === undefined) { + const next = await runs.next().catch((error: unknown) => { + failure ??= { error }; + + return { done: true as const, value: undefined }; + }); + + if (next.done) return; + + const result = await runItem(run, experiment, next.value, api.onError); + + await batch.add(result).catch((error: unknown) => { + failure ??= { error }; + }); + } + }; + + await Promise.all(Array.from({ length: run.concurrency }, worker)); + + if (failure === undefined) { + await batch.flush().catch((error: unknown) => { + failure = { error }; + }); + } + + if (failure !== undefined) { + await runs.return(undefined); + + throw failure.error; + } +} + +/** Collects item results and posts them in order, one post at a time. */ +function itemBatch(post: (results: ItemResult[]) => Promise) { + let pending: ItemResult[] = []; + let bytes = 0; + let queue: Promise = Promise.resolve(); + + const send = async () => { + const results = pending; + pending = []; + bytes = 0; + await post(results); + }; + + return { + add(result: ItemResult): Promise { + const size = new TextEncoder().encode(JSON.stringify(result)).length; + + queue = queue.then(async () => { + if (pending.length > 0 && bytes + size > MAX_BATCH_BYTES) await send(); + pending.push(result); + bytes += size; + + if (pending.length >= MAX_BATCH_ITEMS) await send(); + }); + + return queue; + }, + flush(): Promise { + queue = queue.then(() => (pending.length > 0 ? send() : undefined)); + + return queue; + }, + }; +} + +async function runItem( + run: Run, + experiment: Created, + item: ExperimentItem, + onError: ((cause: Error) => void) | undefined, +): Promise { + const started = performance.now(); + + const outcome = await startActiveSpan( + `experiment ${experiment.name}`, + { parent: itemContext(experiment.id, item), input: item.input }, + async (span): Promise<{ output?: unknown; error?: unknown; traceId?: string }> => { + try { + const output: unknown = await run.task(item); + span.update({ output }); + + return { output, traceId: traceIdOf(span) }; + } catch (error) { + span.update({ error }); + + return { + error: error ?? new Error("The task threw a nullish value"), + traceId: traceIdOf(span), + }; + } + }, + ); + + const base = { + itemId: item.id, + runIndex: item.runIndex, + traceId: outcome.traceId, + latencyMs: Math.round(performance.now() - started), + }; + + if ("error" in outcome) { + return { ...base, output: null, error: truncate(errorText(outcome.error), MAX_ERROR_LENGTH) }; + } + + const scores = await runScorers(run, outcome.output, item, onError); + + return { + ...base, + output: outputText(outcome.output), + scores: scores.length > 0 ? scores : undefined, + }; +} + +/** + * The context of one item run. It drops the active span, so that the run starts a trace of its + * own, and gives the run a session id of its own: an ambient session or `sessionMode: "process"` + * would otherwise put all runs in one session trace. + */ +function itemContext(experimentId: string, item: ExperimentItem): Context { + const base = trace.deleteSpan(activeContext()); + + return base.setValue(PROPAGATED_KEY, { + ...propagatedFromContext(base), + "td.experiment.id": experimentId, + "td.dataset.item_id": item.id, + "gen_ai.conversation.id": `${experimentId}:${item.id}:${item.runIndex}`, + }); +} + +function traceIdOf(span: SpanHandle): string | undefined { + return span.isRecording ? span.traceId : undefined; +} + +async function runScorers( + run: Run, + output: unknown, + item: ExperimentItem, + onError: ((cause: Error) => void) | undefined, +): Promise { + const scores: ItemScore[] = []; + + // After the span of the item ends, so that the model calls of a scorer do not count in the + // cost of the item. + for (const [name, scorer] of run.scorers) { + try { + const score = toScore(name, await scorer(output, item)); + + if (score) scores.push(score); + } catch (error) { + reportError( + onError, + new ExperimentError( + "scorer_failed", + `The scorer ${name} failed on item ${item.number}: ${errorText(error)}`, + undefined, + { cause: error }, + ), + ); + } + } + + return scores; +} + +/** The score of a scorer result. It throws for a result that the server would reject. */ +function toScore(name: string, result: ScorerResult): ItemScore | undefined { + if (result === null || result === undefined) return undefined; + + let score: ItemScore; + + if (typeof result === "boolean") score = { name, ...passFail(result) }; + else if (typeof result === "number") score = { name, value: result }; + else if (typeof result === "string") score = { name, label: result.trim() }; + else if (isRecord(result)) { + const { value, label, reason } = result as Record; + + for (const [key, field] of [ + ["label", label], + ["reason", reason], + ] as const) { + if (field !== undefined && typeof field !== "string") { + throw new TypeError(`returned a ${key} that is not a string`); + } + } + + score = { + name, + ...(typeof value === "boolean" ? passFail(value) : { value: value as number | undefined }), + ...(label === undefined ? {} : { label: (label as string).trim() }), + reason: reason === undefined ? undefined : truncate(reason as string, MAX_REASON_LENGTH), + }; + } else { + throw new TypeError(`returned a ${typeof result}`); + } + + const problem = checkValue(name, score.value, score.label); + + if (problem) throw new TypeError(`returned a score that is not valid: ${problem}`); + + return score; +} + +function passFail(pass: boolean): { value: number; label: string } { + return pass ? { value: 1, label: "pass" } : { value: 0, label: "fail" }; +} + +function outputText(output: unknown): string | null { + const text = output === null || output === undefined ? undefined : jsonAttr(output); + + return text === undefined ? null : truncate(text, MAX_OUTPUT_LENGTH); +} + +function errorText(error: unknown): string { + if (error instanceof Error) return `${error.name}: ${error.message}`; + + try { + return String(error); + } catch { + return "A value that cannot be converted to a string was thrown"; + } +} + +/** Waits until the server evaluators finish and the experiment ends. */ +async function waitForResult( + api: Authed, + id: string, +): Promise<{ result: ExperimentResult; body: unknown }> { + const deadline = Date.now() + MAX_WAIT_MS; + + for (let interval = 1000; ; interval = Math.min(interval * 2, MAX_POLL_INTERVAL_MS)) { + const body = await request( + api, + ExperimentError, + "GET", + `/v1/experiments/${encodeURIComponent(id)}`, + { timeoutMs: REQUEST_TIMEOUT_MS, idempotent: true }, + ); + + const result = parseResult(body); + + if (FINISHED.has(result.status)) return { result, body }; + + if (Date.now() + interval > deadline) { + throw new ExperimentError( + "timeout", + `The experiment ${result.name} did not end in ${MAX_WAIT_MS / 60_000} minutes. ${result.url}`, + undefined, + { result }, + ); + } + + await sleep(interval); + } +} + +/** In CI mode, a failed check or an experiment that did not complete fails the run. */ +function checkResult(result: ExperimentResult): void { + if (result.status !== "completed") { + throw new ExperimentError( + "experiment_failed", + `The experiment ${result.name} ${result.status}${result.error ? `: ${result.error}` : ""}. ${result.url}`, + undefined, + { result }, + ); + } + + if (result.checkStatus !== "failed") return; + + const failed = result.scores + .filter((score) => score.passed === false) + .map( + (score) => + `${score.name} is ${formatScore(score.kind, score.value)}, under the pass mark ${formatScore(score.kind, score.minScore)}`, + ); + + throw new ExperimentError( + "check_failed", + `The check of ${result.name} failed${failed.length > 0 ? `: ${failed.join("; ")}` : ""}. ${result.url}`, + undefined, + { result }, + ); +} + +function describe(result: ExperimentResult): string { + const outcome = + result.status !== "completed" + ? `${result.status}${result.error ? `: ${result.error}` : ""}` + : result.checkStatus === null + ? "completed" + : `completed. The check ${result.checkStatus}`; + + return [ + `Experiment ${result.name} ${outcome}. ${result.url}`, + ...result.scores.map( + (score) => + ` ${score.name}: ${formatScore(score.kind, score.value)}` + + (score.minScore === null ? "" : ` (pass mark ${formatScore(score.kind, score.minScore)})`), + ), + ].join("\n"); +} + +function formatScore(kind: string | null, value: number | null): string { + if (value === null) return "no score"; + + return kind !== null && RATE_KINDS.has(kind) + ? `${Math.round(value * 100)}%` + : String(Number(value.toFixed(2))); +} + +function parseCreated(body: unknown): Created { + if ( + !isRecord(body) || + typeof body.id !== "string" || + typeof body.name !== "string" || + typeof body.url !== "string" || + !Number.isSafeInteger(body.itemsTotal) + ) { + throw new ExperimentError( + "invalid_response", + "telemetry.dev returned an experiment it cannot read", + ); + } + + return { id: body.id, name: body.name, itemsTotal: body.itemsTotal as number, url: body.url }; +} + +function parseDatasetPage(body: unknown): DatasetPage | undefined { + if (!isRecord(body) || !isRecord(body.dataset) || !Array.isArray(body.items)) return undefined; + + const { version } = body.dataset; + const nextCursor = body.nextCursor ?? null; + + if (!Number.isSafeInteger(version) || (version as number) < 0) return undefined; + + if (nextCursor !== null && typeof nextCursor !== "string") return undefined; + + const items: DatasetItem[] = []; + + for (const item of body.items) { + if ( + !isRecord(item) || + typeof item.id !== "string" || + !Number.isSafeInteger(item.number) || + !isRecord(item.input) || + (item.expectedOutput !== null && typeof item.expectedOutput !== "string") || + !isRecord(item.metadata) || + !Array.isArray(item.tags) || + !item.tags.every((tag) => typeof tag === "string") || + (item.split !== null && typeof item.split !== "string") + ) { + return undefined; + } + + items.push({ + id: item.id, + number: item.number as number, + input: item.input, + expectedOutput: item.expectedOutput, + metadata: item.metadata, + tags: item.tags as string[], + split: item.split, + }); + } + + return { version: version as number, items, nextCursor }; +} + +function parseResult(body: unknown): ExperimentResult { + const scores = isRecord(body) && Array.isArray(body.scores) ? body.scores.map(parseScore) : []; + + if ( + !isRecord(body) || + typeof body.id !== "string" || + typeof body.name !== "string" || + typeof body.status !== "string" || + typeof body.url !== "string" || + !( + body.checkStatus === null || + body.checkStatus === "passed" || + body.checkStatus === "failed" + ) || + typeof body.itemsTotal !== "number" || + typeof body.itemsDone !== "number" || + !(body.itemsFailed === undefined || typeof body.itemsFailed === "number") || + !(body.error === undefined || body.error === null || typeof body.error === "string") || + !Array.isArray(body.scores) || + scores.some((score) => score === undefined) + ) { + throw new ExperimentError("invalid_response", "telemetry.dev returned a result it cannot read"); + } + + return { + id: body.id, + name: body.name, + status: body.status, + checkStatus: body.checkStatus, + itemsTotal: body.itemsTotal, + itemsDone: body.itemsDone, + itemsFailed: body.itemsFailed ?? 0, + error: body.error ?? null, + scores: scores as ExperimentScore[], + url: body.url, + }; +} + +function parseScore(value: unknown): ExperimentScore | undefined { + if ( + !isRecord(value) || + typeof value.name !== "string" || + !(value.kind === null || typeof value.kind === "string") || + typeof value.passCount !== "number" || + typeof value.failCount !== "number" + ) { + return undefined; + } + + const nullable = ["value", "baselineValue", "delta", "minScore"] as const; + + if (!nullable.every((key) => value[key] === null || typeof value[key] === "number")) { + return undefined; + } + + if (value.passed !== null && typeof value.passed !== "boolean") return undefined; + + return { + name: value.name, + kind: value.kind, + value: value.value as number | null, + passCount: value.passCount, + failCount: value.failCount, + baselineValue: value.baselineValue as number | null, + delta: value.delta as number | null, + minScore: value.minScore as number | null, + passed: value.passed, + }; +} diff --git a/packages/sdk/src/index.ts b/packages/sdk/src/index.ts index 2fc3343..2c81aee 100644 --- a/packages/sdk/src/index.ts +++ b/packages/sdk/src/index.ts @@ -46,6 +46,20 @@ export { log, type LogOptions } from "./logs.ts"; export { observe, type ObserveOptions } from "./observe.ts"; +export { + type DatasetItem, + ExperimentError, + type ExperimentItem, + type ExperimentResult, + type ExperimentScore, + runExperiment, + type RunExperimentOptions, + type Scorer, + type ScorerResult, +} from "./experiments.ts"; + +export { score, ScoreError, type ScoreOptions } from "./scores.ts"; + export { type ChatMessage, type ChatPrompt, diff --git a/packages/sdk/src/prompts.ts b/packages/sdk/src/prompts.ts index 2e27b17..7b4612d 100644 --- a/packages/sdk/src/prompts.ts +++ b/packages/sdk/src/prompts.ts @@ -1,16 +1,15 @@ import { type Attributes, type Context, createContextKey } from "@opentelemetry/api"; import { activeContext, - DEFAULT_BASE_URL, jsonAttr, omitUndefined, reportError, - resolveEnv, sessionIdOf, sha256, withContext, } from "@telemetry-dev/otel"; +import { isRecord, resolveApi, retryAfterMs } from "./api.ts"; import type { FieldCaptureConfig } from "./attrs.ts"; import { prepareCaptureValue } from "./capture.ts"; import { currentClient } from "./client.ts"; @@ -199,8 +198,6 @@ const MAX_RETRY_AFTER_MS = 3_600_000; /** How long a failed fetch with no copy answers calls, so that they do not each fetch again. */ const ERROR_TTL_MS = 10_000; const MAX_CACHE_ENTRIES = 1_000; -/** IMF-fixdate, the HTTP date that servers send: `Sun, 06 Nov 1994 08:49:37 GMT`. */ -const HTTP_DATE = /^[A-Z][a-z]{2}, \d{2} [A-Z][a-z]{2} \d{4} \d{2}:\d{2}:\d{2} GMT$/; const VARIABLE = /\{\{\s*([A-Za-z_][A-Za-z0-9_]*)\s*\}\}/g; const MODEL_CALL_OPERATIONS = new Set(["chat", "text_completion", "generate_content"]); const PROMPT_KEY = createContextKey("telemetry.dev prompt"); @@ -228,20 +225,8 @@ export async function getPrompt(name: string, options: GetPromptOptions = {}): P const type = options.type ?? "chat"; checkOptions(name, options, type); - const core = currentClient().core; - const env = resolveEnv(); - const apiKey = options.apiKey ?? core?.config.apiKey ?? env.TELEMETRY_DEV_API_KEY; - - // An empty value counts as unset, as in the Python SDK. - const baseUrl = ( - options.baseUrl || - core?.config.baseUrl || - env.TELEMETRY_DEV_BASE_URL || - DEFAULT_BASE_URL - ).replace(/\/+$/, ""); - + const { apiKey, baseUrl, sdkName, fetchImpl, onError } = resolveApi(options); const label = options.version === undefined ? (options.label ?? "production") : null; - const onError = core?.config.onError; let response: PromptResponse; try { @@ -269,11 +254,8 @@ export async function getPrompt(name: string, options: GetPromptOptions = {}): P () => fetchPrompt( url, - { - authorization: `Bearer ${apiKey}`, - "x-telemetry-dev-sdk": core?.config.sdkName ?? "@telemetry-dev/sdk", - }, - options.fetch ?? core?.config.fetchImpl ?? globalThis.fetch, + { authorization: `Bearer ${apiKey}`, "x-telemetry-dev-sdk": sdkName }, + fetchImpl, ), onError, ); @@ -492,7 +474,7 @@ async function fetchPrompt( ); const delay = retryAfterMs(res); - if (delay !== undefined) retryAfter.set(failure, delay); + if (delay !== undefined) retryAfter.set(failure, Math.min(delay, MAX_RETRY_AFTER_MS)); throw failure; } @@ -617,22 +599,6 @@ function keyFingerprint(apiKey: string): string { } /** The Retry-After of a 429 or 503 response, as seconds or an IMF-fixdate HTTP date, capped. */ -function retryAfterMs(res: Response): number | undefined { - if (res.status !== 429 && res.status !== 503) return undefined; - - const value = res.headers.get("retry-after")?.trim(); - - if (!value) return undefined; - - let ms: number; - - if (/^\d+$/.test(value)) ms = Number(value) * 1000; - else if (HTTP_DATE.test(value)) ms = Date.parse(value) - Date.now(); - else return undefined; - - return ms > 0 ? Math.min(ms, MAX_RETRY_AFTER_MS) : undefined; -} - function parseResponse(body: unknown): PromptResponse | undefined { if (!isRecord(body)) return undefined; const { name, type, label, version, rollout } = body; @@ -724,10 +690,6 @@ function isVersion(value: unknown): value is number { return Number.isSafeInteger(value) && (value as number) >= 1; } -function isRecord(value: unknown): value is Record { - return typeof value === "object" && value !== null && !Array.isArray(value); -} - // Prompt objects share the cached response, so an app cannot change the cache through them. function deepFreeze(value: T): T { if (typeof value === "object" && value !== null && !Object.isFrozen(value)) { diff --git a/packages/sdk/src/scores.ts b/packages/sdk/src/scores.ts new file mode 100644 index 0000000..e600c2b --- /dev/null +++ b/packages/sdk/src/scores.ts @@ -0,0 +1,170 @@ +import { reportError } from "@telemetry-dev/otel"; + +import { type ApiOptions, isRecord, request, resolveApi } from "./api.ts"; + +export interface ScoreOptions extends ApiOptions { + /** The score name. `user_feedback` is a thumbs up (1) or down (0) from a user of your app. */ + name: string; + /** The trace that the score is on. Pass `traceId`, `sessionId`, or both. */ + traceId?: string; + /** A span of the trace, for a score on one span. */ + spanId?: string; + sessionId?: string; + /** Pass `value`, `label`, or both. */ + value?: number; + label?: string; + comment?: string; +} + +/** + * Why score failed. `code` is the code of the API error (`insufficient_scope`, `invalid_api_key`, + * …), or `missing_api_key`, `invalid_score`, `timeout`, `network_error`, or `invalid_response`. A + * response with a different status and no API error gives `http_`. + */ +export class ScoreError extends Error { + override readonly name = "ScoreError"; + + constructor( + readonly code: string, + message: string, + readonly status?: number, + options?: { cause?: unknown }, + ) { + super(message, options); + } +} + +export const USER_FEEDBACK = "user_feedback"; +export const MAX_SCORE_NAME_LENGTH = 64; +export const MAX_LABEL_LENGTH = 64; +export const MAX_REASON_LENGTH = 4_000; +const MAX_SESSION_ID_LENGTH = 1_024; +const TRACE_ID = /^(?!0+$)[0-9a-f]{32}$/; +const SPAN_ID = /^(?!0+$)[0-9a-f]{16}$/; +const TIMEOUT_MS = 10_000; + +/** + * Records a score on a trace, a span, or a session, for example the feedback of a user on a + * reply. Like the telemetry functions, score never rejects: it resolves with the id of the score, + * or with undefined after it reports the error through `onError`. It makes one request, with no + * retry, so that a call in a request handler ends in 10 seconds at most. The trace id of a span + * from a client that is not initialized is all zeros, and score reports it as `invalid_score`. + */ +export async function score(options: ScoreOptions): Promise { + const api = resolveApi(options); + + try { + const name = typeof options.name === "string" ? options.name.trim() : options.name; + const label = typeof options.label === "string" ? options.label.trim() : options.label; + const problem = checkScore({ ...options, name, label }); + + if (problem) throw new ScoreError("invalid_score", problem); + + if (!api.apiKey) { + throw new ScoreError( + "missing_api_key", + "No API key: pass apiKey or set TELEMETRY_DEV_API_KEY", + ); + } + + const body = await request({ ...api, apiKey: api.apiKey }, ScoreError, "POST", "/v1/scores", { + body: { + traceId: options.traceId, + spanId: options.spanId, + sessionId: options.sessionId, + name, + value: options.value, + label, + comment: options.comment, + }, + timeoutMs: TIMEOUT_MS, + idempotent: false, + attempts: 1, + }); + + if (!isRecord(body) || typeof body.id !== "string") { + throw new ScoreError("invalid_response", "telemetry.dev returned a score it cannot read"); + } + + return body.id; + } catch (error) { + reportError(api.onError, error); + + return undefined; + } +} + +/** + * Why the server would reject the value and the label of a score, or undefined. A pass/fail score + * has the label `pass` with the value 1, or `fail` with 0. + */ +export function checkValue( + name: string, + value: number | undefined, + label: string | undefined, +): string | undefined { + if (value !== undefined && !(typeof value === "number" && Number.isFinite(value))) { + return "value must be a finite number"; + } + + if ( + label !== undefined && + (typeof label !== "string" || label.length === 0 || label.length > MAX_LABEL_LENGTH) + ) { + return `label must be a string of 1 to ${MAX_LABEL_LENGTH} characters`; + } + + if (value === undefined && label === undefined) return "Pass a value or a label"; + + if (name === USER_FEEDBACK && !(label === undefined && (value === 0 || value === 1))) { + return `${USER_FEEDBACK} takes the value 1 for up or 0 for down, and no label`; + } + + if (label === "pass" && value !== undefined && value !== 1) { + return "The label pass goes with the value 1"; + } + + if (label === "fail" && value !== undefined && value !== 0) { + return "The label fail goes with the value 0"; + } + + return undefined; +} + +function checkScore(options: ScoreOptions): string | undefined { + const { name, traceId, spanId, sessionId, value, label, comment } = options; + + if (typeof name !== "string" || name.length === 0 || name.length > MAX_SCORE_NAME_LENGTH) { + return `name must be a string of 1 to ${MAX_SCORE_NAME_LENGTH} characters`; + } + + if (traceId === undefined && sessionId === undefined) return "Pass traceId or sessionId"; + + if (traceId !== undefined && !(typeof traceId === "string" && TRACE_ID.test(traceId))) { + return "traceId must be 32 lowercase hex digits and not all zeros"; + } + + if (spanId !== undefined && !(typeof spanId === "string" && SPAN_ID.test(spanId))) { + return "spanId must be 16 lowercase hex digits and not all zeros"; + } + + if (spanId !== undefined && traceId === undefined) return "Pass the traceId of the span"; + + if ( + sessionId !== undefined && + (typeof sessionId !== "string" || + sessionId.length === 0 || + sessionId.length > MAX_SESSION_ID_LENGTH) + ) { + return `sessionId must be a string of 1 to ${MAX_SESSION_ID_LENGTH} characters`; + } + + if ( + comment !== undefined && + (typeof comment !== "string" || comment.length > MAX_REASON_LENGTH) + ) { + return `comment must be a string of at most ${MAX_REASON_LENGTH} characters`; + } + + return checkValue(name, value, label); +} diff --git a/packages/sdk/tests/experiments.test.ts b/packages/sdk/tests/experiments.test.ts new file mode 100644 index 0000000..eae10e2 --- /dev/null +++ b/packages/sdk/tests/experiments.test.ts @@ -0,0 +1,723 @@ +import { mkdtempSync, readFileSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; + +import { afterEach, beforeEach, expect, test, vi } from "vitest"; + +import { + type ExperimentItem, + ExperimentError, + propagateAttributes, + runExperiment, + type RunExperimentOptions, + shutdown, + startActiveSpan, + startSpan, +} from "../src/index.ts"; +import { setup } from "./helpers.ts"; + +interface Request { + method: string; + url: URL; + body: any; +} + +const EXPERIMENT_ID = "6f9619ff-8b86-4011-b42d-00c04fc964ff"; +const URL_OF_EXPERIMENT = `https://telemetry.dev/o/org/p/project/evals/experiments/${EXPERIMENT_ID}`; + +function itemId(number: number): string { + return `00000000-0000-4000-8000-${String(number).padStart(12, "0")}`; +} + +function datasetItem(number: number, split: string | null = null) { + return { + id: itemId(number), + number, + input: { question: `q${number}` }, + expectedOutput: `a${number}`, + metadata: { source: "test" }, + tags: ["golden"], + split, + }; +} + +function result(overrides: Record = {}): Record { + return { + id: EXPERIMENT_ID, + name: "support-golden-v3", + status: "completed", + checkStatus: null, + itemsTotal: 2, + itemsDone: 2, + itemsFailed: 0, + error: null, + scores: [], + url: URL_OF_EXPERIMENT, + ...overrides, + }; +} + +/** + * The Evals routes of telemetry.dev, in memory: pages of a dataset version with a cursor, an + * experiment that takes item posts of at most 100 and skips repeats, and its result. + */ +function fakeApi( + options: { + items?: ReturnType[]; + pageSize?: number; + results?: Record[]; + override?: (request: Request) => Response | undefined; + } = {}, +) { + const items = options.items ?? [datasetItem(1), datasetItem(2)]; + const pageSize = options.pageSize ?? 100; + const results = options.results ?? [result()]; + const requests: Request[] = []; + const stored = new Set(); + + const fetch = vi.fn(async (input: string | URL, init?: RequestInit) => { + const url = new URL(input); + const request: Request = { + method: init?.method ?? "GET", + url, + body: typeof init?.body === "string" ? JSON.parse(init.body) : undefined, + }; + requests.push(request); + + const overridden = options.override?.(request); + + if (overridden) return overridden; + + const route = `${request.method} ${url.pathname}`; + + if (route === "GET /v1/datasets/support-golden") { + const split = url.searchParams.get("split"); + const start = Number(url.searchParams.get("cursor") ?? 0); + const matching = items.filter((item) => split === null || item.split === split); + + return Response.json({ + dataset: { + id: "dataset-1", + name: "support-golden", + version: Number(url.searchParams.get("version") ?? 3), + }, + items: matching.slice(start, start + pageSize), + nextCursor: start + pageSize < matching.length ? String(start + pageSize) : null, + }); + } + + if (route === "POST /v1/experiments") { + const count = items.filter( + (item) => request.body.split === undefined || item.split === request.body.split, + ).length; + + return Response.json( + { + id: EXPERIMENT_ID, + name: request.body.name ?? "support-golden-v3", + datasetVersion: request.body.datasetVersion, + itemsTotal: count * (request.body.runsPerItem ?? 1), + url: URL_OF_EXPERIMENT, + }, + { status: 201 }, + ); + } + + if (route === `POST /v1/experiments/${EXPERIMENT_ID}/items`) { + if (request.body.items.length > 100) { + return Response.json( + { error: { code: "invalid_request", message: "at most 100 items" } }, + { status: 400 }, + ); + } + + let accepted = 0; + + for (const run of request.body.items) { + const key = `${run.itemId}:${run.runIndex}`; + + if (!stored.has(key)) accepted += 1; + stored.add(key); + } + + return Response.json({ accepted, duplicates: request.body.items.length - accepted }); + } + + if (route === `POST /v1/experiments/${EXPERIMENT_ID}/complete`) { + return Response.json({ id: EXPERIMENT_ID, status: "running" }); + } + + if (route === `GET /v1/experiments/${EXPERIMENT_ID}`) { + return Response.json(results.length > 1 ? results.shift() : results[0]); + } + + return Response.json({ error: { code: "not_found", message: route } }, { status: 404 }); + }); + + const posts = () => + requests + .filter((request) => request.url.pathname.endsWith("/items")) + .map((request) => request.body.items); + + return { fetch: fetch as unknown as typeof globalThis.fetch, requests, posts }; +} + +function routes(requests: Request[]): string[] { + return requests.map((request) => `${request.method} ${request.url.pathname}`); +} + +function experiment( + api: ReturnType, + options: Partial = {}, +): ReturnType { + return runExperiment({ + apiKey: "td_live_test", + baseUrl: "https://ingest.test", + fetch: api.fetch, + dataset: "support-golden", + task: (item) => `answer to ${String(item.input.question)}`, + ...options, + }); +} + +beforeEach(() => { + for (const name of [ + "TELEMETRY_DEV_API_KEY", + "TELEMETRY_DEV_BASE_URL", + "TELEMETRY_DEV_EXPERIMENT_DATASET", + "TELEMETRY_DEV_EXPERIMENT_MIN_SCORES", + "TELEMETRY_DEV_EXPERIMENT_RESULTS", + "CI", + "GITHUB_ACTIONS", + "GITHUB_REPOSITORY", + "GITHUB_SHA", + "GITHUB_EVENT_PATH", + "GITHUB_RUN_ID", + "GITHUB_SERVER_URL", + ]) { + vi.stubEnv(name, undefined); + } + + vi.spyOn(console, "log").mockImplementation(() => {}); +}); + +afterEach(async () => { + await shutdown(); + vi.unstubAllEnvs(); + vi.restoreAllMocks(); +}); + +test("runs each item of each page, posts batches of 100 on one version, then completes", async () => { + const api = fakeApi({ + items: Array.from({ length: 75 }, (_, index) => datasetItem(index + 1)), + pageSize: 50, + results: [result({ itemsTotal: 150, itemsDone: 146, itemsFailed: 4 })], + }); + + const outcome = await experiment(api, { runsPerItem: 2, evaluators: ["Correctness"] }); + + expect(routes(api.requests)).toEqual([ + "GET /v1/datasets/support-golden", + "POST /v1/experiments", + "GET /v1/datasets/support-golden", + `POST /v1/experiments/${EXPERIMENT_ID}/items`, + `POST /v1/experiments/${EXPERIMENT_ID}/items`, + `POST /v1/experiments/${EXPERIMENT_ID}/complete`, + `GET /v1/experiments/${EXPERIMENT_ID}`, + ]); + + // The first page names no version; the next one asks for the version of the first. + expect(Object.fromEntries(api.requests[0]!.url.searchParams)).toEqual({}); + expect(Object.fromEntries(api.requests[2]!.url.searchParams)).toEqual({ + version: "3", + cursor: "50", + }); + + expect(api.requests[1]!.body).toEqual({ + dataset: "support-golden", + datasetVersion: 3, + evaluators: ["Correctness"], + runsPerItem: 2, + }); + + const posts = api.posts(); + expect(posts.map((items) => items.length)).toEqual([100, 50]); + + const runs = posts + .flat() + .map((run: { itemId: string; runIndex: number }) => `${run.itemId}:${run.runIndex}`); + expect(new Set(runs).size).toBe(150); + expect(runs).toContain(`${itemId(75)}:1`); + + expect(posts[0]![0]).toEqual({ + itemId: itemId(1), + runIndex: 0, + output: "answer to q1", + latencyMs: expect.any(Number), + }); + + expect(outcome).toEqual({ + id: EXPERIMENT_ID, + name: "support-golden-v3", + status: "completed", + checkStatus: null, + itemsTotal: 150, + itemsDone: 146, + itemsFailed: 4, + error: null, + scores: [], + url: URL_OF_EXPERIMENT, + }); +}); + +test("each item run is its own trace, and each span of the task carries the experiment and the item", async () => { + const { spans } = setup({ environment: "experiment", sessionMode: "process" }); + const api = fakeApi(); + + // An active span, an ambient session, and the process session would each put the runs in one + // trace. + await propagateAttributes({ sessionId: "ambient-session", userId: "u-1" }, () => + startActiveSpan("eval script", () => + experiment(api, { + runsPerItem: 2, + task: (item) => { + startSpan("chat gpt-4o", { type: "generation" }).end(); + + return { reply: `r${item.number}` }; + }, + }), + ), + ); + + const posted = api.posts().flat(); + const finished = spans.getFinishedSpans(); + const chats = finished.filter((span) => span.name === "chat gpt-4o"); + const roots = finished.filter((span) => span.name === "experiment support-golden-v3"); + + expect(chats).toHaveLength(4); + expect(roots).toHaveLength(4); + expect(new Set(roots.map((span) => span.spanContext().traceId)).size).toBe(4); + expect(roots.map((span) => span.parentSpanContext?.isRemote)).toEqual([true, true, true, true]); + + for (const run of posted) { + expect(run.traceId).toMatch(/^[0-9a-f]{32}$/); + const root = roots.find((span) => span.spanContext().traceId === run.traceId)!; + const chat = chats.find((span) => span.spanContext().traceId === run.traceId)!; + + expect(chat.parentSpanContext?.spanId).toBe(root.spanContext().spanId); + + for (const span of [root, chat]) { + expect(span.attributes).toMatchObject({ + "td.experiment.id": EXPERIMENT_ID, + "td.dataset.item_id": run.itemId, + "gen_ai.conversation.id": `${EXPERIMENT_ID}:${run.itemId}:${run.runIndex}`, + "user.id": "u-1", + }); + } + } + + expect(posted.map((run: { output: string }) => run.output).sort()).toEqual([ + '{"reply":"r1"}', + '{"reply":"r1"}', + '{"reply":"r2"}', + '{"reply":"r2"}', + ]); +}); + +test("runs at most `concurrency` tasks at a time", async () => { + const api = fakeApi({ items: Array.from({ length: 12 }, (_, index) => datasetItem(index + 1)) }); + let running = 0; + let most = 0; + + await experiment(api, { + concurrency: 3, + task: async () => { + running += 1; + most = Math.max(most, running); + await new Promise((resolve) => setTimeout(resolve, 5)); + running -= 1; + + return "ok"; + }, + }); + + expect(most).toBe(3); + expect(api.posts().flat()).toHaveLength(12); +}); + +test("a task that throws fails only its item", async () => { + const api = fakeApi(); + + await experiment(api, { + task: (item) => { + if (item.number === 1) throw new RangeError("no reply"); + + return "fine"; + }, + }); + + const [failed, passed] = api.posts().flat(); + + expect(failed).toMatchObject({ itemId: itemId(1), output: null, error: "RangeError: no reply" }); + expect(failed).not.toHaveProperty("scores"); + expect(passed).toMatchObject({ itemId: itemId(2), output: "fine" }); + expect(passed).not.toHaveProperty("error"); +}); + +test("scorers give pass/fail, numbers, labels, and reasons, and a bad result drops only its score", async () => { + const errors: Error[] = []; + setup({ environment: "experiment", onError: (error) => errors.push(error) }); + const api = fakeApi({ items: [datasetItem(1)] }); + + await experiment(api, { + scorers: { + exact: (output, item) => output === `answer to ${String(item.input.question)}`, + polite: () => false, + length: (output) => String(output).length, + tone: () => " friendly ", + judged: () => ({ value: 0.25, reason: "too short" }), + none: () => null, + thrown: () => { + throw new Error("judge down"); + }, + mismatched: () => ({ value: 0.5, label: "pass" }), + }, + }); + + expect(api.posts()[0]![0].scores).toEqual([ + { name: "exact", value: 1, label: "pass" }, + { name: "polite", value: 0, label: "fail" }, + { name: "length", value: 12 }, + { name: "tone", label: "friendly" }, + { name: "judged", value: 0.25, reason: "too short" }, + ]); + + expect(errors.map((error) => [(error as ExperimentError).code, error.message])).toEqual([ + ["scorer_failed", "The scorer thrown failed on item 1: Error: judge down"], + [ + "scorer_failed", + "The scorer mismatched failed on item 1: TypeError: returned a score that is not valid: The label pass goes with the value 1", + ], + ]); +}); + +test("a failed post of items stops new items, never completes, and rejects with the API error", async () => { + const api = fakeApi({ + items: Array.from({ length: 300 }, (_, index) => datasetItem(index + 1)), + override: (request) => + request.url.pathname.endsWith("/items") + ? Response.json( + { error: { code: "experiment_ended", message: "support-golden-v3 has ended." } }, + { status: 409 }, + ) + : undefined, + }); + let started = 0; + + const error = await experiment(api, { + concurrency: 1, + task: () => { + started += 1; + + return "ok"; + }, + }).catch((caught: unknown) => caught); + + expect(error).toBeInstanceOf(ExperimentError); + expect(error).toMatchObject({ code: "experiment_ended", status: 409 }); + // The one worker waits for the post of the first 100, which fails, so no item starts after it. + expect(started).toBe(100); + expect(routes(api.requests)).not.toContain(`POST /v1/experiments/${EXPERIMENT_ID}/complete`); +}); + +test("an item post that fails with 500 goes again, and the server skips the runs it has", async () => { + let failures = 1; + const api = fakeApi({ + override: (request) => + request.url.pathname.endsWith("/items") && failures-- > 0 + ? new Response("upstream", { status: 500 }) + : undefined, + }); + + await experiment(api); + + expect(api.posts()).toHaveLength(2); + expect(api.posts()[1]).toEqual(api.posts()[0]); +}); + +test("a create that fails with 500 does not go again, so it cannot start two experiments", async () => { + const api = fakeApi({ + override: (request) => + request.method === "POST" && request.url.pathname === "/v1/experiments" + ? new Response("upstream", { status: 500 }) + : undefined, + }); + + await expect(experiment(api)).rejects.toMatchObject({ code: "http_500", status: 500 }); + expect(routes(api.requests).filter((route) => route === "POST /v1/experiments")).toHaveLength(1); +}); + +test("a create that gets a 429 goes again after its Retry-After", async () => { + let limited = 1; + const api = fakeApi({ + override: (request) => + request.method === "POST" && request.url.pathname === "/v1/experiments" && limited-- > 0 + ? Response.json( + { error: { code: "rate_limited", message: "Too many requests for this project" } }, + { status: 429, headers: { "retry-after": "2" } }, + ) + : undefined, + }); + const started = Date.now(); + + await experiment(api); + + expect(routes(api.requests).filter((route) => route === "POST /v1/experiments")).toHaveLength(2); + // Without the header, the first retry waits 1 second. + expect(Date.now() - started).toBeGreaterThanOrEqual(2_000); +}); + +test("waits until the server evaluators end the experiment", async () => { + const api = fakeApi({ results: [result({ status: "running", itemsDone: 1 }), result()] }); + + const outcome = await experiment(api); + + expect(outcome.status).toBe("completed"); + expect( + routes(api.requests).filter((route) => route.startsWith("GET /v1/experiments/")), + ).toHaveLength(2); +}); + +test.each([ + [ + "the API error", + { error: { code: "dataset_not_found", message: "No dataset named support-golden" } }, + 404, + "dataset_not_found", + ], + [ + "a page that cannot be read", + { dataset: { version: 1 }, items: [{ id: 1 }] }, + 200, + "invalid_response", + ], + [ + "an empty dataset", + { dataset: { id: "d", name: "support-golden", version: 0 }, items: [], nextCursor: null }, + 200, + "dataset_empty", + ], +])("rejects with %s before it creates an experiment", async (_, body, status, code) => { + const api = fakeApi({ + override: (request) => + request.url.pathname.startsWith("/v1/datasets/") + ? Response.json(body, { status }) + : undefined, + }); + + await expect(experiment(api)).rejects.toMatchObject({ code }); + expect(routes(api.requests)).toEqual(["GET /v1/datasets/support-golden"]); +}); + +test("a cursor that repeats is an invalid response, not an endless fetch", async () => { + const api = fakeApi({ + override: (request) => + request.url.pathname.startsWith("/v1/datasets/") + ? Response.json({ + dataset: { id: "d", name: "support-golden", version: 3 }, + items: [datasetItem(1)], + nextCursor: "same", + }) + : undefined, + }); + + await expect(experiment(api)).rejects.toMatchObject({ code: "invalid_response" }); + expect( + routes(api.requests).filter((route) => route.startsWith("GET /v1/datasets/")), + ).toHaveLength(2); +}); + +const failedCheck = result({ + checkStatus: "failed", + scores: [ + { + name: "correctness", + kind: "pass_fail", + value: 0.85, + passCount: 17, + failCount: 3, + baselineValue: 0.9, + delta: -5, + minScore: 0.9, + passed: false, + }, + { + name: "length", + kind: null, + value: null, + passCount: 0, + failCount: 0, + baselineValue: null, + delta: null, + minScore: null, + passed: null, + }, + ], +}); + +test("in CI mode a failed check rejects with the result", async () => { + vi.stubEnv("CI", "true"); + const api = fakeApi({ results: [failedCheck] }); + + const error = (await experiment(api, { minScores: { correctness: 0.9 } }).catch( + (caught: unknown) => caught, + )) as ExperimentError; + + expect(error).toBeInstanceOf(ExperimentError); + expect(error.code).toBe("check_failed"); + expect(error.message).toBe( + `The check of support-golden-v3 failed: correctness is 85%, under the pass mark 90%. ${URL_OF_EXPERIMENT}`, + ); + expect(error.result?.scores[1]?.kind).toBeNull(); +}); + +test.each([ + ["CI is off", {}, undefined], + ["ci: false wins over CI", { ci: false }, "true"], +])("a failed check resolves when %s", async (_, options, ci) => { + if (ci) vi.stubEnv("CI", ci); + const api = fakeApi({ results: [failedCheck] }); + + await expect(experiment(api, options)).resolves.toMatchObject({ checkStatus: "failed" }); +}); + +test("in CI mode an experiment that fails rejects with its error", async () => { + vi.stubEnv("CI", "1"); + const api = fakeApi({ + results: [result({ status: "failed", error: "0 of 2 items ran. Each one failed." })], + }); + + await expect(experiment(api)).rejects.toMatchObject({ + code: "experiment_failed", + message: `The experiment support-golden-v3 failed: 0 of 2 items ran. Each one failed.. ${URL_OF_EXPERIMENT}`, + }); +}); + +test("in GitHub Actions the experiment records the pull request, its head commit, and the run", async () => { + const dir = mkdtempSync(join(tmpdir(), "td-experiment-")); + const event = join(dir, "event.json"); + const results = join(dir, "results.jsonl"); + writeFileSync( + event, + JSON.stringify({ + pull_request: { + number: 42, + title: "Shorter replies", + html_url: "https://github.com/acme/support/pull/42", + head: { sha: "a1b2c3d4e5f6a1b2c3d4e5f6a1b2c3d4e5f6a1b2" }, + }, + }), + ); + writeFileSync(results, ""); + + vi.stubEnv("CI", "true"); + vi.stubEnv("GITHUB_ACTIONS", "true"); + vi.stubEnv("GITHUB_REPOSITORY", "acme/support"); + vi.stubEnv("GITHUB_SHA", "ffffffffffffffffffffffffffffffffffffffff"); + vi.stubEnv("GITHUB_EVENT_PATH", event); + vi.stubEnv("GITHUB_RUN_ID", "991"); + vi.stubEnv("TELEMETRY_DEV_EXPERIMENT_DATASET", "support-golden"); + vi.stubEnv("TELEMETRY_DEV_EXPERIMENT_MIN_SCORES", "correctness=0.9\nlength = 12.5"); + vi.stubEnv("TELEMETRY_DEV_EXPERIMENT_RESULTS", results); + const api = fakeApi(); + + await experiment(api, { dataset: undefined, minScores: { correctness: 0.5, tone: 2 } }); + + expect(api.requests[1]!.body).toMatchObject({ + dataset: "support-golden", + minScores: { correctness: 0.9, length: 12.5, tone: 2 }, + ci: { + provider: "github_actions", + repository: "acme/support", + commit: "a1b2c3d4e5f6a1b2c3d4e5f6a1b2c3d4e5f6a1b2", + prNumber: 42, + prTitle: "Shorter replies", + prUrl: "https://github.com/acme/support/pull/42", + runUrl: "https://github.com/acme/support/actions/runs/991", + }, + }); + expect(readFileSync(results, "utf8")).toBe(`${JSON.stringify(result())}\n`); +}); + +test.each([ + [ + "no dataset", + {}, + { dataset: undefined }, + "Pass dataset, or set the dataset input of the experiment action", + ], + [ + "a dataset that the action does not name", + { TELEMETRY_DEV_EXPERIMENT_DATASET: "support-v2" }, + {}, + "dataset is support-golden, but the experiment action names support-v2. Name one dataset.", + ], + [ + "min-scores that do not parse", + { TELEMETRY_DEV_EXPERIMENT_MIN_SCORES: "correctness=0x10" }, + {}, + 'TELEMETRY_DEV_EXPERIMENT_MIN_SCORES: "correctness=0x10" is not a name=number pair', + ], + ["runsPerItem 6", {}, { runsPerItem: 6 }, "runsPerItem must be an integer from 1 to 5"], + ["concurrency 0", {}, { concurrency: 0 }, "concurrency must be an integer from 1 to 100"], + [ + "a scorer named user_feedback", + {}, + { scorers: { user_feedback: () => 1 } }, + "user_feedback is the score of user feedback. Name the scorer another way.", + ], + [ + "a scorer with the name of an evaluator", + {}, + { evaluators: ["correctness"], scorers: { Correctness: () => true } }, + "The scorer Correctness has the name of an evaluator. Rename one of them.", + ], +])("rejects %s before any request", async (_, env, options, message) => { + for (const [name, value] of Object.entries(env)) vi.stubEnv(name, value); + const api = fakeApi(); + + await expect(experiment(api, options as Partial)).rejects.toThrow(message); + expect(api.requests).toEqual([]); +}); + +test("without an API key it rejects with missing_api_key and sends nothing", async () => { + const api = fakeApi(); + + await expect(experiment(api, { apiKey: undefined })).rejects.toMatchObject({ + code: "missing_api_key", + }); + expect(api.requests).toEqual([]); +}); + +test("the task gets each run of an item of the split, with its run index", async () => { + const seen: ExperimentItem[] = []; + const api = fakeApi({ items: [datasetItem(7, "test")] }); + + await experiment(api, { + split: "test", + runsPerItem: 2, + concurrency: 1, + task: (item) => { + seen.push(item); + + return "ok"; + }, + }); + + expect(seen).toEqual([ + { ...datasetItem(7, "test"), runIndex: 0 }, + { ...datasetItem(7, "test"), runIndex: 1 }, + ]); + expect(api.requests[0]!.url.searchParams.get("split")).toBe("test"); + expect(api.requests[1]!.body.split).toBe("test"); +}); diff --git a/packages/sdk/tests/scores.test.ts b/packages/sdk/tests/scores.test.ts new file mode 100644 index 0000000..d740b40 --- /dev/null +++ b/packages/sdk/tests/scores.test.ts @@ -0,0 +1,220 @@ +import { createServer, type RequestListener } from "node:http"; +import type { AddressInfo } from "node:net"; + +import { afterEach, beforeEach, expect, test, vi } from "vitest"; + +import { score, ScoreError, type ScoreOptions, shutdown } from "../src/index.ts"; +import { setup } from "./helpers.ts"; + +const TRACE_ID = "4bf92f3577b34da6a3ce929d0e0e4736"; +const SPAN_ID = "00f067aa0ba902b7"; + +function fakeFetch(reply: () => Response) { + const calls: { url: URL; init: RequestInit | undefined }[] = []; + + const fetch = vi.fn(async (input: string | URL, init?: RequestInit) => { + calls.push({ url: new URL(input), init }); + + return reply(); + }); + + return { fetch: fetch as unknown as typeof globalThis.fetch, calls }; +} + +async function serve(handler: RequestListener): Promise<{ url: string; close: () => void }> { + const server = createServer(handler); + await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve)); + const { port } = server.address() as AddressInfo; + + return { + url: `http://127.0.0.1:${port}`, + close: () => { + server.closeAllConnections(); + server.close(); + }, + }; +} + +function errorsOf() { + const errors: ScoreError[] = []; + setup({ onError: (error) => errors.push(error as ScoreError) }); + + return errors; +} + +beforeEach(() => { + vi.stubEnv("TELEMETRY_DEV_API_KEY", undefined); + vi.stubEnv("TELEMETRY_DEV_BASE_URL", undefined); +}); + +afterEach(async () => { + await shutdown(); + vi.unstubAllEnvs(); + vi.restoreAllMocks(); +}); + +test("posts the score with the key and base URL of init, and resolves with its id", async () => { + const { fetch, calls } = fakeFetch(() => Response.json({ id: "score-1" }, { status: 202 })); + setup({ apiKey: "td_live_key", baseUrl: "https://ingest.test/", fetch }); + + const id = await score({ + name: " user_feedback ", + traceId: TRACE_ID, + spanId: SPAN_ID, + value: 0, + comment: "Wrong refund amount", + }); + + expect(id).toBe("score-1"); + expect(calls).toHaveLength(1); + expect(calls[0]!.url.href).toBe("https://ingest.test/v1/scores"); + expect(calls[0]!.init).toMatchObject({ + method: "POST", + headers: { + authorization: "Bearer td_live_key", + "x-telemetry-dev-sdk": "@telemetry-dev/sdk", + "content-type": "application/json", + }, + }); + expect(JSON.parse(calls[0]!.init!.body as string)).toEqual({ + name: "user_feedback", + traceId: TRACE_ID, + spanId: SPAN_ID, + value: 0, + comment: "Wrong refund amount", + }); +}); + +test("a session score needs no trace, and a label goes trimmed", async () => { + const { fetch, calls } = fakeFetch(() => Response.json({ id: "score-2" }, { status: 202 })); + + await expect( + score({ apiKey: "td_live_key", fetch, name: "resolved", sessionId: "chat-9", label: " yes " }), + ).resolves.toBe("score-2"); + expect(JSON.parse(calls[0]!.init!.body as string)).toEqual({ + name: "resolved", + sessionId: "chat-9", + label: "yes", + }); +}); + +test.each<[string, Partial, string]>([ + ["no trace and no session", { traceId: undefined }, "Pass traceId or sessionId"], + [ + "the all-zero trace id of a span that did not record", + { traceId: "0".repeat(32) }, + "traceId must be 32 lowercase hex digits and not all zeros", + ], + [ + "an uppercase trace id", + { traceId: TRACE_ID.toUpperCase() }, + "traceId must be 32 lowercase hex digits and not all zeros", + ], + [ + "a span without its trace", + { traceId: undefined, sessionId: "s", spanId: SPAN_ID }, + "Pass the traceId of the span", + ], + ["no value and no label", { value: undefined }, "Pass a value or a label"], + ["a value that is not finite", { value: Number.NaN }, "value must be a finite number"], + [ + "a label for user feedback", + { name: "user_feedback", value: 1, label: "up" }, + "user_feedback takes the value 1 for up or 0 for down, and no label", + ], + [ + "user feedback of 0.5", + { name: "user_feedback", value: 0.5 }, + "user_feedback takes the value 1 for up or 0 for down, and no label", + ], + ["a pass of 0", { value: 0, label: "pass" }, "The label pass goes with the value 1"], + ["a fail of 1", { value: 1, label: "fail" }, "The label fail goes with the value 0"], + ["a blank name", { name: " " }, "name must be a string of 1 to 64 characters"], + [ + "a comment of 4,001 characters", + { comment: "x".repeat(4_001) }, + "comment must be a string of at most 4000 characters", + ], +])("reports invalid_score for %s and sends nothing", async (_, options, message) => { + const errors = errorsOf(); + const { fetch, calls } = fakeFetch(() => Response.json({ id: "never" })); + + await expect( + score({ + apiKey: "td_live_key", + fetch, + name: "helpful", + traceId: TRACE_ID, + value: 1, + ...options, + }), + ).resolves.toBeUndefined(); + expect(calls).toEqual([]); + expect(errors.map((error) => [error.code, error.message])).toEqual([["invalid_score", message]]); +}); + +test.each([ + [ + "the API error", + () => + Response.json( + { error: { code: "insufficient_scope", message: "This API key cannot send scores" } }, + { status: 403 }, + ), + ["insufficient_scope", 403], + ], + ["a status with no API error", () => new Response("down", { status: 503 }), ["http_503", 503]], + ["a body with no id", () => Response.json({}, { status: 202 }), ["invalid_response", undefined]], +])("reports %s, tries once, and resolves undefined", async (_, reply, [code, status]) => { + const errors = errorsOf(); + const { fetch, calls } = fakeFetch(reply); + + await expect( + score({ apiKey: "td_live_key", fetch, name: "helpful", traceId: TRACE_ID, value: 1 }), + ).resolves.toBeUndefined(); + expect(calls).toHaveLength(1); + expect(errors.map((error) => [error.code, error.status])).toEqual([[code, status]]); +}); + +test("without an API key it reports missing_api_key", async () => { + const errors = errorsOf(); + const { fetch, calls } = fakeFetch(() => Response.json({ id: "never" })); + + await score({ apiKey: "", fetch, name: "helpful", traceId: TRACE_ID, value: 1 }); + + expect(calls).toEqual([]); + expect(errors.map((error) => error.code)).toEqual(["missing_api_key"]); +}); + +test("does not follow a redirect, so the key stays with the API host", async () => { + const errors = errorsOf(); + let reached = 0; + const other = await serve((_, res) => { + reached += 1; + res.end(JSON.stringify({ id: "stolen" })); + }); + const api = await serve((_, res) => { + res.writeHead(307, { location: `${other.url}/v1/scores` }); + res.end(); + }); + + try { + await expect( + // The real fetch: setup() gives the client a stub. + score({ + apiKey: "td_live_key", + baseUrl: api.url, + fetch: globalThis.fetch, + name: "helpful", + traceId: TRACE_ID, + value: 1, + }), + ).resolves.toBeUndefined(); + } finally { + api.close(); + other.close(); + } + + expect(reached).toBe(0); + expect(errors.map((error) => [error.code, error.status])).toEqual([["http_307", 307]]); +}); From e44cf6022fc0e1938f3a11efaf97b49441ae574d Mon Sep 17 00:00:00 2001 From: ephraimduncan Date: Thu, 29 Oct 2026 23:27:41 +0000 Subject: [PATCH 02/10] fix(sdk): keep each min score and print scores as the python sdk does minScoresFromEnv now makes its object from pairs. Before, the name __proto__ set the prototype of the object, and its mark went away with no error. A number score now prints as Math.round(value * 100) / 100, a rule that the Python SDK can copy. The tests now also cover the experiment attributes on log records and the request order that the code holds. --- packages/sdk/src/ci.ts | 7 +++-- packages/sdk/src/experiments.ts | 2 +- packages/sdk/tests/experiments.test.ts | 39 ++++++++++++++++++-------- 3 files changed, 33 insertions(+), 15 deletions(-) diff --git a/packages/sdk/src/ci.ts b/packages/sdk/src/ci.ts index 45eb20c..3d7603e 100644 --- a/packages/sdk/src/ci.ts +++ b/packages/sdk/src/ci.ts @@ -78,7 +78,8 @@ export function githubRun(env: Env): CiRun | undefined { * its `min-scores` input: `name=value` pairs, separated by commas or new lines. */ export function minScoresFromEnv(env: Env): Record { - const scores: Record = {}; + // Pairs, so that the name __proto__ becomes a key and not the prototype of the object. + const scores: [string, number][] = []; for (const pair of (env.TELEMETRY_DEV_EXPERIMENT_MIN_SCORES ?? "").split(/[,\n]/)) { if (pair.trim() === "") continue; @@ -90,10 +91,10 @@ export function minScoresFromEnv(env: Env): Record { ); } - scores[name] = Number(value); + scores.push([name, Number(value)]); } - return scores; + return Object.fromEntries(scores); } /** diff --git a/packages/sdk/src/experiments.ts b/packages/sdk/src/experiments.ts index bb90d0d..f3d6d4b 100644 --- a/packages/sdk/src/experiments.ts +++ b/packages/sdk/src/experiments.ts @@ -826,7 +826,7 @@ function formatScore(kind: string | null, value: number | null): string { return kind !== null && RATE_KINDS.has(kind) ? `${Math.round(value * 100)}%` - : String(Number(value.toFixed(2))); + : String(Math.round(value * 100) / 100); } function parseCreated(body: unknown): Created { diff --git a/packages/sdk/tests/experiments.test.ts b/packages/sdk/tests/experiments.test.ts index eae10e2..daa3214 100644 --- a/packages/sdk/tests/experiments.test.ts +++ b/packages/sdk/tests/experiments.test.ts @@ -7,6 +7,7 @@ import { afterEach, beforeEach, expect, test, vi } from "vitest"; import { type ExperimentItem, ExperimentError, + log, propagateAttributes, runExperiment, type RunExperimentOptions, @@ -216,22 +217,23 @@ test("runs each item of each page, posts batches of 100 on one version, then com const outcome = await experiment(api, { runsPerItem: 2, evaluators: ["Correctness"] }); - expect(routes(api.requests)).toEqual([ - "GET /v1/datasets/support-golden", - "POST /v1/experiments", + const sent = routes(api.requests); + expect(sent.slice(0, 2)).toEqual(["GET /v1/datasets/support-golden", "POST /v1/experiments"]); + // The second page and the first post of items run at the same time, in either order. + expect(sent.slice(2, -2).sort()).toEqual([ "GET /v1/datasets/support-golden", `POST /v1/experiments/${EXPERIMENT_ID}/items`, `POST /v1/experiments/${EXPERIMENT_ID}/items`, + ]); + expect(sent.slice(-2)).toEqual([ `POST /v1/experiments/${EXPERIMENT_ID}/complete`, `GET /v1/experiments/${EXPERIMENT_ID}`, ]); // The first page names no version; the next one asks for the version of the first. - expect(Object.fromEntries(api.requests[0]!.url.searchParams)).toEqual({}); - expect(Object.fromEntries(api.requests[2]!.url.searchParams)).toEqual({ - version: "3", - cursor: "50", - }); + const pages = api.requests.filter((request) => request.url.pathname.startsWith("/v1/datasets/")); + expect(Object.fromEntries(pages[0]!.url.searchParams)).toEqual({}); + expect(Object.fromEntries(pages[1]!.url.searchParams)).toEqual({ version: "3", cursor: "50" }); expect(api.requests[1]!.body).toEqual({ dataset: "support-golden", @@ -271,7 +273,7 @@ test("runs each item of each page, posts batches of 100 on one version, then com }); test("each item run is its own trace, and each span of the task carries the experiment and the item", async () => { - const { spans } = setup({ environment: "experiment", sessionMode: "process" }); + const { spans, logs } = setup({ environment: "experiment", sessionMode: "process" }); const api = fakeApi(); // An active span, an ambient session, and the process session would each put the runs in one @@ -282,6 +284,7 @@ test("each item run is its own trace, and each span of the task carries the expe runsPerItem: 2, task: (item) => { startSpan("chat gpt-4o", { type: "generation" }).end(); + log("replied", { attributes: { number: item.number } }); return { reply: `r${item.number}` }; }, @@ -316,6 +319,20 @@ test("each item run is its own trace, and each span of the task carries the expe } } + const records = logs.getFinishedLogRecords(); + expect(records).toHaveLength(4); + + for (const record of records) { + const run = posted.find( + (candidate: { traceId: string }) => candidate.traceId === record.spanContext?.traceId, + ); + + expect(record.attributes).toMatchObject({ + "td.experiment.id": EXPERIMENT_ID, + "td.dataset.item_id": run.itemId, + }); + } + expect(posted.map((run: { output: string }) => run.output).sort()).toEqual([ '{"reply":"r1"}', '{"reply":"r1"}', @@ -627,7 +644,7 @@ test("in GitHub Actions the experiment records the pull request, its head commit vi.stubEnv("GITHUB_EVENT_PATH", event); vi.stubEnv("GITHUB_RUN_ID", "991"); vi.stubEnv("TELEMETRY_DEV_EXPERIMENT_DATASET", "support-golden"); - vi.stubEnv("TELEMETRY_DEV_EXPERIMENT_MIN_SCORES", "correctness=0.9\nlength = 12.5"); + vi.stubEnv("TELEMETRY_DEV_EXPERIMENT_MIN_SCORES", "correctness=0.9\nlength = 12.5,__proto__=1"); vi.stubEnv("TELEMETRY_DEV_EXPERIMENT_RESULTS", results); const api = fakeApi(); @@ -635,7 +652,7 @@ test("in GitHub Actions the experiment records the pull request, its head commit expect(api.requests[1]!.body).toMatchObject({ dataset: "support-golden", - minScores: { correctness: 0.9, length: 12.5, tone: 2 }, + minScores: JSON.parse('{"correctness":0.9,"tone":2,"length":12.5,"__proto__":1}'), ci: { provider: "github_actions", repository: "acme/support", From b8dc041c3907beb0bf5647b928c3c8a180e57525 Mon Sep 17 00:00:00 2001 From: ephraimduncan Date: Thu, 29 Oct 2026 23:27:50 +0000 Subject: [PATCH 03/10] feat(sdk-python): add run_experiment and score run_experiment and run_experiment_async are the Python twins of runExperiment, with the same options, defaults, errors, and requests. The sync call runs the items in threads, each in a copy of the context of the caller. The async call runs them as asyncio tasks. The two read the dataset page by page and post the outputs in batches of at most 100. score and score_async send a score to /v1/scores. They never raise and send their errors to on_error. The new _api module holds the key and base URL, one JSON request with its retry rules, the redirect guard, and the Retry-After parser. The prompt code now uses it too. The checks count lengths in UTF-16 units, as the API does. --- sdks/python/src/telemetry_dev/__init__.py | 26 + sdks/python/src/telemetry_dev/_api.py | 190 +++ sdks/python/src/telemetry_dev/_ci.py | 134 ++ sdks/python/src/telemetry_dev/_experiments.py | 1100 +++++++++++++++++ sdks/python/src/telemetry_dev/_prompts.py | 80 +- sdks/python/src/telemetry_dev/_scores.py | 181 +++ sdks/python/src/telemetry_dev/_semconv.py | 2 + sdks/python/src/telemetry_dev/_serialize.py | 5 + sdks/python/tests/fake_api.py | 193 +++ sdks/python/tests/test_experiments.py | 746 +++++++++++ sdks/python/tests/test_scores.py | 204 +++ 11 files changed, 2795 insertions(+), 66 deletions(-) create mode 100644 sdks/python/src/telemetry_dev/_api.py create mode 100644 sdks/python/src/telemetry_dev/_ci.py create mode 100644 sdks/python/src/telemetry_dev/_experiments.py create mode 100644 sdks/python/src/telemetry_dev/_scores.py create mode 100644 sdks/python/tests/fake_api.py create mode 100644 sdks/python/tests/test_experiments.py create mode 100644 sdks/python/tests/test_scores.py diff --git a/sdks/python/src/telemetry_dev/__init__.py b/sdks/python/src/telemetry_dev/__init__.py index 9ed7aab..bbcb230 100644 --- a/sdks/python/src/telemetry_dev/__init__.py +++ b/sdks/python/src/telemetry_dev/__init__.py @@ -17,6 +17,18 @@ from ._config import SDK_VERSION as __version__ from ._config import LogLevelOption from ._context import get_traceparent, propagate_attributes +from ._experiments import ( + AsyncScorer, + DatasetItem, + ExperimentError, + ExperimentItem, + ExperimentResult, + ExperimentScore, + Scorer, + ScorerResult, + run_experiment, + run_experiment_async, +) from ._logs import log from ._observe import observe from ._prompts import ( @@ -29,6 +41,7 @@ get_prompt_async, with_prompt, ) +from ._scores import ScoreError, score, score_async from ._semconv import LogLevel, SpanType from ._serialize import AttributeValue, Mask, MaskContext from ._spans import ( @@ -42,10 +55,16 @@ __all__ = [ "NOT_GIVEN", + "AsyncScorer", "AttributeValue", "CaptureBudget", "ChatPrompt", "Client", + "DatasetItem", + "ExperimentError", + "ExperimentItem", + "ExperimentResult", + "ExperimentScore", "LogLevel", "LogLevelOption", "Mask", @@ -53,6 +72,9 @@ "Prompt", "PromptError", "PromptModel", + "ScoreError", + "Scorer", + "ScorerResult", "SpanHandle", "SpanType", "TelemetrySpanProcessor", @@ -68,6 +90,10 @@ "log", "observe", "propagate_attributes", + "run_experiment", + "run_experiment_async", + "score", + "score_async", "shutdown", "start_span", "update_current_span", diff --git a/sdks/python/src/telemetry_dev/_api.py b/sdks/python/src/telemetry_dev/_api.py new file mode 100644 index 0000000..4f8e042 --- /dev/null +++ b/sdks/python/src/telemetry_dev/_api.py @@ -0,0 +1,190 @@ +"""Calls to the telemetry.dev API: the key and base URL, one JSON request, and its errors.""" + +from __future__ import annotations + +import http.client +import json +import re +import time +import urllib.error +import urllib.parse +import urllib.request +from collections.abc import Callable, Mapping +from dataclasses import dataclass +from datetime import datetime, timezone +from email.message import Message +from email.utils import parsedate_to_datetime +from typing import IO, Any, Literal, TypeVar, cast + +from ._client import get_client +from ._config import resolve_config + +# A longer Retry-After waits only this long: the API counts requests by the minute. +_MAX_RETRY_AFTER_S = 60.0 +# IMF-fixdate, the HTTP date that servers send: Sun, 06 Nov 1994 08:49:37 GMT. +_HTTP_DATE = re.compile(r"[A-Z][a-z]{2}, \d{2} [A-Z][a-z]{2} \d{4} \d{2}:\d{2}:\d{2} GMT", re.ASCII) + + +# An error class with the shape of PromptError: a code, a message, and the status. +E = TypeVar("E", bound=Exception) + + +@dataclass(frozen=True) +class Api: + api_key: str | None + base_url: str + on_error: Callable[[BaseException], None] | None + + +def resolve_api(api_key: str | None, base_url: str | None) -> Api: + """The key and base URL of a call: its arguments, then the enabled client, then the env.""" + client = get_client() + enabled = client if client is not None and client.enabled else None + config = enabled.config if enabled is not None else resolve_config() + return Api( + api_key=api_key if api_key is not None else config.api_key, + base_url=(base_url or config.base_url).rstrip("/"), + on_error=enabled.on_error if enabled is not None else None, + ) + + +@dataclass(frozen=True) +class _Failure: + error: Exception + status: int | None + retry_after: float | None + + +def request( + api: Api, + error_type: Callable[[str, str, int | None], E], + method: Literal["GET", "POST"], + path: str, + *, + timeout_s: float, + idempotent: bool, + body: object = None, + query: Mapping[str, str | None] | None = None, + attempts: int = 3, +) -> Any: + """Send one JSON request and return the JSON of the response. + + A response that is not 2xx raises the ``{"error": {"code", "message"}}`` of its body, or + ``http_``. With ``idempotent``, the server takes a repeat at most once, so a timeout, a + network error, or a 5xx tries again. Other requests try again only after a 429 or a 503: the + server did not take the request then. A retry waits for the Retry-After of the response, or + for 1 and then 2 seconds. The timeout applies to each socket operation of urllib. + """ + url = f"{api.base_url}{path}" + params = {key: value for key, value in (query or {}).items() if value is not None} + if params: + url = f"{url}?{urllib.parse.urlencode(params)}" + headers = {"Authorization": f"Bearer {api.api_key}", "X-Telemetry-Dev-SDK": "telemetry-dev"} + data = None + if body is not None: + data = json.dumps(body, ensure_ascii=False, allow_nan=False).encode() + headers["Content-Type"] = "application/json" + attempt = 1 + while True: + result = _send(error_type, method, url, path, data, headers, timeout_s) + if not isinstance(result, _Failure): + return result + if attempt >= attempts or not _retryable(result.status, idempotent): + raise result.error + time.sleep(min(result.retry_after or float(attempt), _MAX_RETRY_AFTER_S)) + attempt += 1 + + +class NoRedirect(urllib.request.HTTPRedirectHandler): + # A redirect to another host must not get the API key, so a 3xx fails like any status that is + # not 2xx. + def redirect_request( + self, + req: urllib.request.Request, + fp: IO[bytes], + code: int, + msg: str, + headers: Message, + newurl: str, + ) -> urllib.request.Request | None: + return None + + +opener = urllib.request.build_opener(NoRedirect) + + +def api_error(error_type: Callable[[str, str, int | None], E], status: int, body: bytes) -> E: + """The error of a response that is not 2xx: its API code and message, or ``http_``.""" + try: + error = json.loads(body).get("error") + except (ValueError, AttributeError): + error = None + fields = cast("dict[str, Any]", error) if isinstance(error, dict) else {} + code = fields.get("code") + message = fields.get("message") + return error_type( + code if isinstance(code, str) else f"http_{status}", + message if isinstance(message, str) else f"telemetry.dev returned {status}", + status, + ) + + +def retry_after(status: int, value: str | None) -> float | None: + """The seconds that the Retry-After of a 429 or a 503 asks for: seconds or an HTTP date.""" + value = (value or "").strip() + if status not in (429, 503) or not value: + return None + if value.isascii() and value.isdigit(): + seconds = float(value) + elif _HTTP_DATE.fullmatch(value): + try: + date = parsedate_to_datetime(value) + except (TypeError, ValueError): + return None + seconds = (date - datetime.now(timezone.utc)).total_seconds() + else: + return None + return seconds if seconds > 0 else None + + +def _send( + error_type: Callable[[str, str, int | None], E], + method: str, + url: str, + path: str, + data: bytes | None, + headers: dict[str, str], + timeout_s: float, +) -> Any: + request = urllib.request.Request(url, data=data, headers=headers, method=method) + try: + with opener.open(request, timeout=timeout_s) as response: + body: bytes = response.read() + except urllib.error.HTTPError as exc: + with exc: + return _Failure( + api_error(error_type, exc.code, exc.read()), + exc.code, + retry_after(exc.code, exc.headers.get("Retry-After")), + ) + except (urllib.error.URLError, OSError, http.client.HTTPException) as exc: + timed_out = isinstance(exc, TimeoutError) or isinstance( + getattr(exc, "reason", None), TimeoutError + ) + error = ( + error_type("timeout", f"{method} {path} took more than {timeout_s:g} s", None) + if timed_out + else error_type("network_error", "Could not reach telemetry.dev", None) + ) + error.__cause__ = exc + return _Failure(error, None, None) + try: + return json.loads(body) + except ValueError: + return None + + +def _retryable(status: int | None, idempotent: bool) -> bool: + if status in (429, 503): + return True + return idempotent and (status is None or status >= 500) diff --git a/sdks/python/src/telemetry_dev/_ci.py b/sdks/python/src/telemetry_dev/_ci.py new file mode 100644 index 0000000..4b6cd00 --- /dev/null +++ b/sdks/python/src/telemetry_dev/_ci.py @@ -0,0 +1,134 @@ +"""The GitHub Actions run of an experiment, and the variables of the experiment action.""" + +from __future__ import annotations + +import json +import os +import re +from collections.abc import Callable, Mapping +from typing import Any, cast + +from ._config import report_error +from ._serialize import utf16_length + +_REPOSITORY = re.compile(r"[A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+") +_COMMIT = re.compile(r"[0-9a-fA-F]{7,64}") +_HTTPS_URL = re.compile(r"https://\S+") +# A plain decimal, so that the TypeScript and Python SDKs read the same marks. +_DECIMAL = re.compile(r"-?\d+(\.\d+)?", re.ASCII) +_RUN_ID = re.compile(r"\d+", re.ASCII) +_MAX_REPOSITORY_LENGTH = 200 +_MAX_URL_LENGTH = 2048 +_MAX_PR_TITLE_LENGTH = 256 + + +def in_ci(env: Mapping[str, str]) -> bool: + """CI mode is on when ``CI`` is ``true`` or ``1``, as CI services set it.""" + return env.get("CI") in ("true", "1") + + +def github_run(env: Mapping[str, str]) -> dict[str, Any] | None: + """The run that GitHub Actions describes in its variables, or None outside GitHub Actions. + + On a pull request the commit is the head of the pull request: ``GITHUB_SHA`` is then a merge + commit that nobody pushed. + """ + repository = env.get("GITHUB_REPOSITORY") or "" + if ( + env.get("GITHUB_ACTIONS") != "true" + or len(repository) > _MAX_REPOSITORY_LENGTH + or not _REPOSITORY.fullmatch(repository) + ): + return None + pull = _pull_request(env.get("GITHUB_EVENT_PATH")) + commit = pull.get("commit") or env.get("GITHUB_SHA") + if not isinstance(commit, str) or not _COMMIT.fullmatch(commit): + return None + server = (env.get("GITHUB_SERVER_URL") or "https://github.com").rstrip("/") + run_id = env.get("GITHUB_RUN_ID") or "" + run: dict[str, Any] = { + "provider": "github_actions", + "repository": repository, + "commit": commit, + "prNumber": pull.get("number"), + "prTitle": pull.get("title"), + "prUrl": pull.get("url"), + "runUrl": _https_url(f"{server}/{repository}/actions/runs/{run_id}") + if _RUN_ID.fullmatch(run_id) + else None, + } + return {key: value for key, value in run.items() if value is not None} + + +def min_scores_from_env(env: Mapping[str, str]) -> dict[str, float]: + """The pass marks in ``TELEMETRY_DEV_EXPERIMENT_MIN_SCORES``, which the experiment action + sets from its ``min-scores`` input: ``name=value`` pairs, separated by commas or new lines.""" + scores: dict[str, float] = {} + for pair in re.split(r"[,\n]", env.get("TELEMETRY_DEV_EXPERIMENT_MIN_SCORES") or ""): + if not pair.strip(): + continue + parts = [part.strip() for part in pair.split("=")] + if len(parts) != 2 or not parts[0] or not _DECIMAL.fullmatch(parts[1]): + raise TypeError( + f'TELEMETRY_DEV_EXPERIMENT_MIN_SCORES: "{pair.strip()}" is not a name=number pair' + ) + scores[parts[0]] = float(parts[1]) + return scores + + +def write_result( + env: Mapping[str, str], + result: object, + on_error: Callable[[BaseException], None] | None, +) -> None: + """Add the result to the file in ``TELEMETRY_DEV_EXPERIMENT_RESULTS`` as one line of JSON. + The experiment action reads the file to write the job summary and the pull request comment.""" + path = env.get("TELEMETRY_DEV_EXPERIMENT_RESULTS") + if not path: + return + line = json.dumps(result, ensure_ascii=False, separators=(",", ":")) + "\n" + try: + fd = os.open(path, os.O_WRONLY | os.O_APPEND | os.O_CREAT, 0o600) + with os.fdopen(fd, "a", encoding="utf-8") as file: + file.write(line) + except OSError as exc: + report_error(on_error, "writing TELEMETRY_DEV_EXPERIMENT_RESULTS failed", exc) + + +def _pull_request(event_path: str | None) -> dict[str, Any]: + if not event_path: + return {} + try: + with open(event_path, encoding="utf-8") as file: + event = json.load(file) + except (OSError, ValueError): + return {} + pull = cast("dict[str, Any]", event).get("pull_request") if isinstance(event, dict) else None + if not isinstance(pull, dict): + return {} + fields = cast("dict[str, Any]", pull) + number = fields.get("number") + title = fields.get("title") + url = fields.get("html_url") + head = fields.get("head") + sha = cast("dict[str, Any]", head).get("sha") if isinstance(head, dict) else None + return { + "number": number + if isinstance(number, int) and not isinstance(number, bool) and number > 0 + else None, + "title": _cut(title, _MAX_PR_TITLE_LENGTH) if isinstance(title, str) else None, + "url": _https_url(url) if isinstance(url, str) else None, + "commit": sha if isinstance(sha, str) else None, + } + + +def _https_url(url: str) -> str | None: + return url if utf16_length(url) <= _MAX_URL_LENGTH and _HTTPS_URL.fullmatch(url) else None + + +def _cut(text: str, max_units: int) -> str: + """``text`` cut to ``max_units`` UTF-16 units, the length that the API counts.""" + encoded = text.encode("utf-16-le", "surrogatepass") + if len(encoded) <= max_units * 2: + return text + return encoded[: max_units * 2].decode("utf-16-le", "ignore") diff --git a/sdks/python/src/telemetry_dev/_experiments.py b/sdks/python/src/telemetry_dev/_experiments.py new file mode 100644 index 0000000..b4e031d --- /dev/null +++ b/sdks/python/src/telemetry_dev/_experiments.py @@ -0,0 +1,1100 @@ +"""Experiments of your code: run a task on each item of a dataset and record the result.""" + +from __future__ import annotations + +import asyncio +import contextvars +import inspect +import json +import math +import os +import threading +import time +import urllib.parse +from collections.abc import Awaitable, Callable, Generator, Mapping, Sequence +from concurrent.futures import ThreadPoolExecutor +from dataclasses import dataclass +from time import perf_counter +from typing import Any, Literal, TypeAlias, TypeGuard, cast + +from opentelemetry import context as otel_context +from opentelemetry import trace +from opentelemetry.context import Context + +from ._api import Api, request, resolve_api +from ._ci import github_run, in_ci, min_scores_from_env, write_result +from ._client import flush, get_client +from ._config import logger, report_error +from ._context import context_with_propagated_attributes, propagated_attributes +from ._scores import ( + MAX_REASON_LENGTH, + MAX_SCORE_NAME_LENGTH, + USER_FEEDBACK, + check_value, + is_finite, +) +from ._semconv import ATTR_DATASET_ITEM_ID, ATTR_EXPERIMENT_ID, ATTR_SESSION_ID +from ._serialize import json_dumps, truncate, utf16_length +from ._spans import SpanHandle, start_span + +# The experiment environment, which pages, metrics, and alerts leave out by default. +_EXPERIMENT_ENVIRONMENT = "experiment" +_MAX_RUNS_PER_ITEM = 5 +_MAX_CONCURRENCY = 100 +# Server evaluators of one experiment, and scorers: an item takes at most 20 scores. +_MAX_EVALUATORS = 20 +_MAX_MIN_SCORES = 20 +_MAX_ERROR_LENGTH = 4000 +_MAX_OUTPUT_LENGTH = 100_000 +_MAX_DATASET_ITEMS = 10_000 +# Items in one POST /v1/experiments/{id}/items. +_MAX_BATCH_ITEMS = 100 +# Bytes of JSON in one post of items, well under the 16 MiB body limit of the API. +_MAX_BATCH_BYTES = 4 * 1024 * 1024 +_REQUEST_TIMEOUT_S = 30.0 +# How long run_experiment waits for the server evaluators after the last item. +_MAX_WAIT_S = 30 * 60.0 +_MAX_POLL_INTERVAL_S = 5.0 +_FINISHED = frozenset({"completed", "failed", "stopped"}) +_RATE_KINDS = frozenset({"pass_fail", "up_down"}) + + +@dataclass(frozen=True) +class DatasetItem: + """An item of a dataset version, as ``GET /v1/datasets/{name}`` returns it. ``id`` is the key + of the item: it stays the same across dataset versions.""" + + id: str + number: int + input: Mapping[str, Any] + expected_output: str | None + metadata: Mapping[str, Any] + tags: tuple[str, ...] + split: str | None + + +@dataclass(frozen=True) +class ExperimentItem(DatasetItem): + """One run of a dataset item, ``run_index`` 0 to ``runs_per_item`` - 1.""" + + run_index: int + + +ScorerResult: TypeAlias = "bool | int | float | str | Mapping[str, Any] | None" +"""What a scorer returns. True and False are a pass and a fail, a number is the value, and a +string is the label. A mapping can have ``value``, ``label``, and ``reason``. None gives no +score.""" + +Scorer: TypeAlias = "Callable[[Any, ExperimentItem], ScorerResult]" +"""Scores the output of an item on your machine. The score has the name of the scorer.""" + +AsyncScorer: TypeAlias = "Callable[[Any, ExperimentItem], ScorerResult | Awaitable[ScorerResult]]" +"""A scorer of ``run_experiment_async``, which can be async.""" + + +@dataclass(frozen=True) +class ExperimentScore: + """One score name of an experiment. ``value`` is the pass rate (0 to 1) of a pass/fail or + up/down score, or the mean of a number score. ``kind`` is None before the score has values. + ``baseline_value`` is the value in the experiment that this one compares with, by default the + dataset baseline. ``delta`` is whole points of a pass rate, or the change of a mean to one + decimal. ``passed`` is None when the score has no pass mark.""" + + name: str + kind: str | None + value: float | None + pass_count: int + fail_count: int + baseline_value: float | None + delta: float | None + min_score: float | None + passed: bool | None + + +@dataclass(frozen=True) +class ExperimentResult: + """The result of an experiment. ``status`` is ``completed``, ``failed``, or ``stopped``. + ``check_status`` is None when the experiment has no pass marks. ``error`` tells why the + experiment failed.""" + + id: str + name: str + status: str + check_status: Literal["passed", "failed"] | None + items_total: int + items_done: int + items_failed: int + error: str | None + scores: tuple[ExperimentScore, ...] + url: str + + +class ExperimentError(Exception): + """Why run_experiment failed. ``code`` is the code of the API error (``dataset_not_found``, + ``insufficient_scope``, ``invalid_api_key``, ...), or ``missing_api_key``, ``timeout``, + ``network_error``, ``invalid_response``, or ``dataset_empty``. In CI mode it is also + ``check_failed`` or ``experiment_failed``, and ``result`` holds the result. A response with a + different status and no API error gives ``http_``. A scorer that raises or returns a + value that is not a score sends a ``scorer_failed`` error to ``on_error``.""" + + def __init__( + self, + code: str, + message: str, + status: int | None = None, + *, + result: ExperimentResult | None = None, + ) -> None: + super().__init__(message) + self.code = code + self.message = message + self.status = status + self.result = result + + +def run_experiment( + *, + task: Callable[[ExperimentItem], Any], + dataset: str | None = None, + name: str | None = None, + dataset_version: int | None = None, + split: str | None = None, + evaluators: Sequence[str] | None = None, + scorers: Mapping[str, Scorer] | None = None, + min_scores: Mapping[str, float] | None = None, + runs_per_item: int = 1, + concurrency: int = 4, + metadata: Mapping[str, Any] | None = None, + ci: bool | None = None, + api_key: str | None = None, + base_url: str | None = None, +) -> ExperimentResult: + """Run ``task`` on each item of a dataset and record the result as an experiment. + + Each item run is its own trace, with ``td.experiment.id`` and ``td.dataset.item_id`` on each + span. The outputs go to telemetry.dev in batches of at most 100, with the scores of + ``scorers``. Then run_experiment waits for the ``evaluators`` of the server, prints the result, + and returns it. ``concurrency`` (1 to 100, default 4) threads run the items, each in a copy of + the context of the caller. + + ``dataset`` defaults to the ``dataset`` input of the experiment action, which it sets in + ``TELEMETRY_DEV_EXPERIMENT_DATASET``. When the two are set, they must be the same. + ``min_scores`` holds pass marks by score name: the lowest pass rate (0 to 1) of a pass/fail + score, or the lowest mean of a number score. The ``min-scores`` input of the action adds to + them and wins for the same name. ``runs_per_item`` is 1 to 5. + + The run cannot start without its dataset, so run_experiment raises ``ExperimentError`` when a + request fails. A task that raises fails only its item. In CI mode (``ci``, default on when + ``CI`` is ``true`` or ``1``), it records the GitHub Actions run, and it raises when the check + fails or the experiment does not complete, so the process exits with an error. + """ + run, api = _prepare( + task, + dataset, + name, + dataset_version, + split, + evaluators, + scorers, + min_scores, + runs_per_item, + concurrency, + metadata, + ci, + api_key, + base_url, + asynchronous=False, + ) + runs, created = _start(api, run) + _run_items(api, run, created, runs) + return _finish(api, run, created) + + +async def run_experiment_async( + *, + task: Callable[[ExperimentItem], Any], + dataset: str | None = None, + name: str | None = None, + dataset_version: int | None = None, + split: str | None = None, + evaluators: Sequence[str] | None = None, + scorers: Mapping[str, AsyncScorer] | None = None, + min_scores: Mapping[str, float] | None = None, + runs_per_item: int = 1, + concurrency: int = 4, + metadata: Mapping[str, Any] | None = None, + ci: bool | None = None, + api_key: str | None = None, + base_url: str | None = None, +) -> ExperimentResult: + """``run_experiment`` for async tasks and scorers. ``concurrency`` asyncio tasks run the + items, and the requests go in worker threads. A task that is not async runs on the event + loop, so it blocks the other items while it runs: use ``run_experiment`` for it.""" + run, api = _prepare( + task, + dataset, + name, + dataset_version, + split, + evaluators, + scorers, + min_scores, + runs_per_item, + concurrency, + metadata, + ci, + api_key, + base_url, + asynchronous=True, + ) + runs, created = await asyncio.to_thread(_start, api, run) + await _run_items_async(api, run, created, runs) + return await asyncio.to_thread(_finish, api, run, created) + + +@dataclass(frozen=True) +class _Run: + dataset: str + task: Callable[[ExperimentItem], Any] + name: str | None + dataset_version: int | None + split: str | None + evaluators: list[str] | None + scorers: list[tuple[str, Callable[[Any, ExperimentItem], Any]]] + min_scores: dict[str, float] | None + runs_per_item: int + concurrency: int + metadata: dict[str, Any] | None + ci: dict[str, Any] | None + in_ci: bool + + +@dataclass(frozen=True) +class _Created: + id: str + name: str + items_total: int + url: str + + +@dataclass(frozen=True) +class _Page: + version: int + items: list[DatasetItem] + next_cursor: str | None + + +def _prepare( + task: object, + dataset: object, + name: object, + dataset_version: object, + split: object, + evaluators: object, + scorers: object, + min_scores: object, + runs_per_item: object, + concurrency: object, + metadata: object, + ci: object, + api_key: str | None, + base_url: str | None, + *, + asynchronous: bool, +) -> tuple[_Run, Api]: + env = os.environ + if not callable(task): + raise TypeError("task must be a function") + if not asynchronous and inspect.iscoroutinefunction(task): + raise TypeError("task is async: use run_experiment_async") + for key, value in (("dataset", dataset), ("name", name), ("split", split)): + if value is not None and (not isinstance(value, str) or not value): + raise TypeError(f"{key} must be a non-empty string") + action_dataset = env.get("TELEMETRY_DEV_EXPERIMENT_DATASET") or None + chosen = cast("str | None", dataset) if dataset is not None else action_dataset + if chosen is None: + raise TypeError("Pass dataset, or set the dataset input of the experiment action") + if action_dataset is not None and action_dataset != chosen: + raise TypeError( + f"dataset is {chosen}, but the experiment action names {action_dataset}. " + "Name one dataset." + ) + if dataset_version is not None and not (_is_int(dataset_version) and dataset_version >= 1): + raise ValueError("dataset_version must be a positive integer") + + evaluator_names: list[str] | None = None + if evaluators is not None: + names = ( + list(cast("Sequence[object]", evaluators)) + if isinstance(evaluators, Sequence) and not isinstance(evaluators, str) + else None + ) + if ( + names is None + or len(names) > _MAX_EVALUATORS + or not all(isinstance(n, str) and _valid_score_name(n) for n in names) + ): + raise TypeError( + f"evaluators must be a list of at most {_MAX_EVALUATORS} evaluator names" + ) + evaluator_names = cast("list[str]", names) + + if scorers is not None and not isinstance(scorers, Mapping): + raise TypeError("scorers must be a mapping of score names to functions") + scorer_list = list(cast("Mapping[object, object]", scorers or {}).items()) + if len(scorer_list) > _MAX_EVALUATORS: + raise TypeError(f"scorers can have at most {_MAX_EVALUATORS} scorers") + folded = {n.lower() for n in evaluator_names or []} + for scorer_name, scorer in scorer_list: + if ( + not isinstance(scorer_name, str) + or not _valid_score_name(scorer_name) + or not callable(scorer) + ): + raise TypeError( + f"scorers must map score names of 1 to {MAX_SCORE_NAME_LENGTH} characters " + "to functions" + ) + if not asynchronous and inspect.iscoroutinefunction(scorer): + raise TypeError(f"The scorer {scorer_name} is async: use run_experiment_async") + if scorer_name == USER_FEEDBACK: + raise TypeError( + f"{USER_FEEDBACK} is the score of user feedback. Name the scorer another way." + ) + if scorer_name.lower() in folded: + raise TypeError( + f"The scorer {scorer_name} has the name of an evaluator. Rename one of them." + ) + + if min_scores is not None and not isinstance(min_scores, Mapping): + raise TypeError(f"min_scores must map at most {_MAX_MIN_SCORES} score names to numbers") + marks: dict[object, object] = { + **cast("Mapping[object, object]", min_scores or {}), + **min_scores_from_env(env), + } + if len(marks) > _MAX_MIN_SCORES or not all( + isinstance(n, str) and _valid_score_name(n) and is_finite(v) for n, v in marks.items() + ): + raise TypeError(f"min_scores must map at most {_MAX_MIN_SCORES} score names to numbers") + + if not _is_int(runs_per_item) or not 1 <= runs_per_item <= _MAX_RUNS_PER_ITEM: + raise ValueError(f"runs_per_item must be an integer from 1 to {_MAX_RUNS_PER_ITEM}") + if not _is_int(concurrency) or not 1 <= concurrency <= _MAX_CONCURRENCY: + raise ValueError(f"concurrency must be an integer from 1 to {_MAX_CONCURRENCY}") + if metadata is not None and not isinstance(metadata, Mapping): + raise TypeError("metadata must be a mapping") + if ci is not None and not isinstance(ci, bool): + raise TypeError("ci must be a bool") + ci_mode = in_ci(env) if ci is None else ci + + api = resolve_api(api_key, base_url) + if not api.api_key: + raise ExperimentError( + "missing_api_key", "No API key: pass api_key or set TELEMETRY_DEV_API_KEY" + ) + run = _Run( + dataset=chosen, + task=task, + name=cast("str | None", name), + dataset_version=dataset_version, + split=cast("str | None", split), + evaluators=evaluator_names, + scorers=cast("list[tuple[str, Callable[[Any, ExperimentItem], Any]]]", scorer_list), + min_scores={str(n): float(cast("float", v)) for n, v in marks.items()} or None, + runs_per_item=runs_per_item, + concurrency=concurrency, + metadata=dict(cast("Mapping[str, Any]", metadata)) if metadata is not None else None, + ci=github_run(env) if ci_mode else None, + in_ci=ci_mode, + ) + return run, api + + +def _valid_score_name(name: str) -> bool: + """A trimmed name of 1 to 64 characters, as the API stores score names.""" + return 0 < utf16_length(name) <= MAX_SCORE_NAME_LENGTH and name.strip() == name + + +def _is_int(value: object) -> TypeGuard[int]: + return isinstance(value, int) and not isinstance(value, bool) + + +def _warn_about_client() -> None: + client = get_client() + environment = client.config.environment if client is not None and client.enabled else None + if environment is None: + logger.warning( + "run_experiment: no telemetry client, so the items record no traces. " + 'Call init(environment="experiment") first.' + ) + elif environment.strip().lower() != _EXPERIMENT_ENVIRONMENT: + logger.warning( + "run_experiment: the traces of the items go to the environment %s. " + 'Call init(environment="experiment") to keep them out of it.', + environment, + ) + + +def _start(api: Api, run: _Run) -> tuple[_Runs, _Created]: + _warn_about_client() + first = _fetch_page(api, run, run.dataset_version, None) + if not first.items: + scope = "" if run.split is None else f" in the split {run.split}" + raise ExperimentError("dataset_empty", f"The dataset {run.dataset} has no items{scope}") + body = { + "name": run.name, + "dataset": run.dataset, + "datasetVersion": first.version, + "split": run.split, + "evaluators": run.evaluators, + "minScores": run.min_scores, + "runsPerItem": run.runs_per_item, + "ci": run.ci, + "metadata": run.metadata, + } + created = _parse_created( + request( + api, + ExperimentError, + "POST", + "/v1/experiments", + body={key: value for key, value in body.items() if value is not None}, + timeout_s=_REQUEST_TIMEOUT_S, + # A repeat after a lost response would start a second experiment. + idempotent=False, + ) + ) + each = f", {run.runs_per_item} runs each" if run.runs_per_item > 1 else "" + print( + f"Experiment {created.name}: {created.items_total // run.runs_per_item} items{each}. " + f"{created.url}", + flush=True, + ) + return _Runs(api, run, first), created + + +class _Runs: + """The runs of each item of the dataset version, page by page, so that only one page is in + memory. A lock lets the workers share the generator.""" + + def __init__(self, api: Api, run: _Run, first: _Page) -> None: + self._lock = threading.Lock() + self._runs = _item_runs(api, run, first) + + def next(self) -> ExperimentItem | None: + with self._lock: + return next(self._runs, None) + + def close(self) -> None: + with self._lock: + self._runs.close() + + +def _item_runs(api: Api, run: _Run, first: _Page) -> Generator[ExperimentItem]: + page = first + count = 0 + while True: + count += len(page.items) + if count > _MAX_DATASET_ITEMS: + raise ExperimentError( + "invalid_response", f"telemetry.dev returned more than {_MAX_DATASET_ITEMS} items" + ) + for item in page.items: + for run_index in range(run.runs_per_item): + yield ExperimentItem(**vars(item), run_index=run_index) + if page.next_cursor is None: + return + page = _fetch_page(api, run, page.version, page.next_cursor) + + +def _fetch_page(api: Api, run: _Run, version: int | None, cursor: str | None) -> _Page: + """One page of a dataset version. A page after the first names the version of the first.""" + page = _parse_page( + request( + api, + ExperimentError, + "GET", + f"/v1/datasets/{urllib.parse.quote(run.dataset, safe='')}", + query={ + "version": None if version is None else str(version), + "split": run.split, + "cursor": cursor, + }, + timeout_s=_REQUEST_TIMEOUT_S, + idempotent=True, + ) + ) + if ( + page is None + or (version is not None and page.version != version) + or (page.next_cursor is not None and (not page.items or page.next_cursor == cursor)) + ): + raise ExperimentError("invalid_response", "telemetry.dev returned a dataset it cannot read") + return page + + +class _Batch: + """Collects item results and posts them in order, one post at a time.""" + + def __init__(self, post: Callable[[list[dict[str, Any]]], object]) -> None: + self._post = post + self._pending: list[dict[str, Any]] = [] + self._bytes = 0 + self._lock = threading.Lock() + + def add(self, result: dict[str, Any]) -> None: + size = len(json.dumps(result, ensure_ascii=False).encode()) + with self._lock: + if self._pending and self._bytes + size > _MAX_BATCH_BYTES: + self._send() + self._pending.append(result) + self._bytes += size + if len(self._pending) >= _MAX_BATCH_ITEMS: + self._send() + + def flush(self) -> None: + with self._lock: + if self._pending: + self._send() + + def _send(self) -> None: + results = self._pending + self._pending = [] + self._bytes = 0 + self._post(results) + + +def _batch(api: Api, created: _Created) -> _Batch: + path = f"/v1/experiments/{urllib.parse.quote(created.id, safe='')}/items" + return _Batch( + lambda results: request( + api, + ExperimentError, + "POST", + path, + body={"items": results}, + timeout_s=_REQUEST_TIMEOUT_S, + # The server skips an item run that it has, so a repeat is safe. + idempotent=True, + ) + ) + + +def _run_items(api: Api, run: _Run, created: _Created, runs: _Runs) -> None: + """Run the items, ``concurrency`` at a time, and post each result. A post or a page fetch + that fails stops the run: no new item starts, and run_experiment raises when the running + items end.""" + batch = _batch(api, created) + failures: list[BaseException] = [] + caller = contextvars.copy_context() + + def worker() -> None: + while not failures: + try: + item = runs.next() + except Exception as exc: + failures.append(exc) + return + if item is None: + return + # A copy for each item: one Context cannot run in two threads at the same time. + result = caller.copy().run(_run_item, run, created, item, api.on_error) + try: + batch.add(result) + except Exception as exc: + failures.append(exc) + + with ThreadPoolExecutor(run.concurrency, thread_name_prefix="telemetry-dev-experiment") as pool: + futures = [pool.submit(worker) for _ in range(run.concurrency)] + try: + for future in futures: + future.result() + except BaseException as exc: + # Stops the workers, for example after Ctrl-C. + failures.append(exc) + raise + _end_items(batch, runs, failures) + + +async def _run_items_async(api: Api, run: _Run, created: _Created, runs: _Runs) -> None: + batch = _batch(api, created) + failures: list[BaseException] = [] + + async def worker() -> None: + while not failures: + try: + item = await asyncio.to_thread(runs.next) + except Exception as exc: + failures.append(exc) + return + if item is None: + return + result = await _run_item_async(run, created, item, api.on_error) + try: + await asyncio.to_thread(batch.add, result) + except Exception as exc: + failures.append(exc) + + # gather runs each worker in its own task, with its own copy of the context. + await asyncio.gather(*(worker() for _ in range(run.concurrency))) + await asyncio.to_thread(_end_items, batch, runs, failures) + + +def _end_items(batch: _Batch, runs: _Runs, failures: list[BaseException]) -> None: + if not failures: + try: + batch.flush() + except Exception as exc: + failures.append(exc) + if failures: + runs.close() + raise failures[0] + + +def _run_item( + run: _Run, + experiment: _Created, + item: ExperimentItem, + on_error: Callable[[BaseException], None] | None, +) -> dict[str, Any]: + started = perf_counter() + output: Any = None + error: Exception | None = None + with start_span( + f"experiment {experiment.name}", + input=dict(item.input), + parent=_item_context(experiment.id, item), + ) as span: + try: + output = run.task(item) + if inspect.isawaitable(output): + _close(output) + raise TypeError("task returned an awaitable: use run_experiment_async") + span.update(output=output) + except Exception as exc: + error = exc + span.update(error=exc) + trace_id = _trace_id(span) + result = _item_result(item, trace_id, started, output, error) + if error is None: + _add_scores( + result, [_score(name, scorer, output, item, on_error) for name, scorer in run.scorers] + ) + return result + + +async def _run_item_async( + run: _Run, + experiment: _Created, + item: ExperimentItem, + on_error: Callable[[BaseException], None] | None, +) -> dict[str, Any]: + started = perf_counter() + output: Any = None + error: Exception | None = None + with start_span( + f"experiment {experiment.name}", + input=dict(item.input), + parent=_item_context(experiment.id, item), + ) as span: + try: + output = run.task(item) + if inspect.isawaitable(output): + output = await output + span.update(output=output) + except Exception as exc: + error = exc + span.update(error=exc) + trace_id = _trace_id(span) + result = _item_result(item, trace_id, started, output, error) + if error is None: + scores: list[dict[str, Any] | None] = [] + for name, scorer in run.scorers: + try: + value = scorer(output, item) + if inspect.isawaitable(value): + value = await value + scores.append(_to_score(name, value)) + except Exception as exc: + _scorer_failed(on_error, name, item, exc) + scores.append(None) + _add_scores(result, scores) + return result + + +def _item_context(experiment_id: str, item: ExperimentItem) -> Context: + """The context of one item run. It drops the active span, so that the run starts a trace of + its own, and gives the run a session id of its own: an ambient session or + ``session_mode="process"`` would otherwise put all runs in one session trace.""" + base = trace.set_span_in_context(trace.INVALID_SPAN, otel_context.get_current()) + return context_with_propagated_attributes( + { + **propagated_attributes(base), + ATTR_EXPERIMENT_ID: experiment_id, + ATTR_DATASET_ITEM_ID: item.id, + ATTR_SESSION_ID: f"{experiment_id}:{item.id}:{item.run_index}", + }, + base, + ) + + +def _trace_id(span: SpanHandle) -> str | None: + context = span.span.get_span_context() + return f"{context.trace_id:032x}" if span.span.is_recording() and context.is_valid else None + + +def _item_result( + item: ExperimentItem, + trace_id: str | None, + started: float, + output: Any, + error: Exception | None, +) -> dict[str, Any]: + result: dict[str, Any] = { + "itemId": item.id, + "runIndex": item.run_index, + "output": None if error is not None else _output_text(output), + "latencyMs": round((perf_counter() - started) * 1000), + } + if trace_id is not None: + result["traceId"] = trace_id + if error is not None: + result["error"] = truncate(_error_text(error), _MAX_ERROR_LENGTH) + return result + + +def _add_scores(result: dict[str, Any], scores: list[dict[str, Any] | None]) -> None: + kept = [score for score in scores if score is not None] + if kept: + result["scores"] = kept + + +def _score( + name: str, + scorer: Callable[[Any, ExperimentItem], Any], + output: Any, + item: ExperimentItem, + on_error: Callable[[BaseException], None] | None, +) -> dict[str, Any] | None: + # After the span of the item ends, so that the model calls of a scorer do not count in the + # cost of the item. + try: + value = scorer(output, item) + if inspect.isawaitable(value): + _close(value) + raise TypeError("returned an awaitable: use run_experiment_async") + return _to_score(name, value) + except Exception as exc: + _scorer_failed(on_error, name, item, exc) + return None + + +def _scorer_failed( + on_error: Callable[[BaseException], None] | None, + name: str, + item: ExperimentItem, + exc: Exception, +) -> None: + error = ExperimentError( + "scorer_failed", f"The scorer {name} failed on item {item.number}: {_error_text(exc)}" + ) + error.__cause__ = exc + report_error(on_error, "a scorer failed", error) + + +def _to_score(name: str, result: object) -> dict[str, Any] | None: + """The score of a scorer result. It raises for a result that the server would reject.""" + if result is None: + return None + score: dict[str, Any] = {"name": name} + if isinstance(result, bool): + score.update(_pass_fail(result)) + elif isinstance(result, (int, float)): + score["value"] = result + elif isinstance(result, str): + score["label"] = result.strip() + elif isinstance(result, Mapping): + fields = cast("Mapping[str, object]", result) + value, label, reason = fields.get("value"), fields.get("label"), fields.get("reason") + for key, text in (("label", label), ("reason", reason)): + if text is not None and not isinstance(text, str): + raise TypeError(f"returned a {key} that is not a string") + if isinstance(value, bool): + score.update(_pass_fail(value)) + elif value is not None: + score["value"] = value + if isinstance(label, str): + score["label"] = label.strip() + if isinstance(reason, str): + score["reason"] = truncate(reason, MAX_REASON_LENGTH) + else: + raise TypeError(f"returned a {type(result).__name__}") + problem = check_value(name, score.get("value"), score.get("label")) + if problem is not None: + raise TypeError(f"returned a score that is not valid: {problem}") + return score + + +def _pass_fail(passed: bool) -> dict[str, Any]: + return {"value": 1, "label": "pass"} if passed else {"value": 0, "label": "fail"} + + +def _output_text(output: Any) -> str | None: + if output is None: + return None + text = output if isinstance(output, str) else json_dumps(output) + return truncate(text, _MAX_OUTPUT_LENGTH) + + +def _error_text(error: BaseException) -> str: + return f"{type(error).__name__}: {error}" + + +def _close(awaitable: object) -> None: + # A coroutine that never runs warns when it is collected. + close = getattr(awaitable, "close", None) + if callable(close): + close() + + +def _finish(api: Api, run: _Run, created: _Created) -> ExperimentResult: + # The spans of the items reach ingest before the server reads their cost. + flush() + request( + api, + ExperimentError, + "POST", + f"/v1/experiments/{urllib.parse.quote(created.id, safe='')}/complete", + timeout_s=_REQUEST_TIMEOUT_S, + idempotent=True, + ) + result, body = _wait_for_result(api, created.id) + write_result(os.environ, body, api.on_error) + print(_describe(result), flush=True) + if run.in_ci: + _check_result(result) + return result + + +def _wait_for_result(api: Api, experiment_id: str) -> tuple[ExperimentResult, object]: + """Wait until the server evaluators finish and the experiment ends.""" + deadline = time.monotonic() + _MAX_WAIT_S + interval = 1.0 + while True: + body = request( + api, + ExperimentError, + "GET", + f"/v1/experiments/{urllib.parse.quote(experiment_id, safe='')}", + timeout_s=_REQUEST_TIMEOUT_S, + idempotent=True, + ) + result = _parse_result(body) + if result.status in _FINISHED: + return result, body + if time.monotonic() + interval > deadline: + raise ExperimentError( + "timeout", + f"The experiment {result.name} did not end in {_MAX_WAIT_S / 60:g} minutes. " + f"{result.url}", + result=result, + ) + time.sleep(interval) + interval = min(interval * 2, _MAX_POLL_INTERVAL_S) + + +def _check_result(result: ExperimentResult) -> None: + """In CI mode, a failed check or an experiment that did not complete fails the run.""" + if result.status != "completed": + reason = f": {result.error}" if result.error else "" + raise ExperimentError( + "experiment_failed", + f"The experiment {result.name} {result.status}{reason}. {result.url}", + result=result, + ) + if result.check_status != "failed": + return + failed = [ + f"{score.name} is {_format(score.kind, score.value)}, under the pass mark " + f"{_format(score.kind, score.min_score)}" + for score in result.scores + if score.passed is False + ] + detail = f": {'; '.join(failed)}" if failed else "" + raise ExperimentError( + "check_failed", + f"The check of {result.name} failed{detail}. {result.url}", + result=result, + ) + + +def _describe(result: ExperimentResult) -> str: + if result.status != "completed": + outcome = result.status + (f": {result.error}" if result.error else "") + elif result.check_status is None: + outcome = "completed" + else: + outcome = f"completed. The check {result.check_status}" + lines = [f"Experiment {result.name} {outcome}. {result.url}"] + for score in result.scores: + mark = ( + "" + if score.min_score is None + else f" (pass mark {_format(score.kind, score.min_score)})" + ) + lines.append(f" {score.name}: {_format(score.kind, score.value)}{mark}") + return "\n".join(lines) + + +def _format(kind: str | None, value: float | None) -> str: + if value is None: + return "no score" + if kind in _RATE_KINDS: + return f"{_js_round(value * 100)}%" + rounded = _js_round(value * 100) / 100 + return str(int(rounded)) if rounded.is_integer() else repr(rounded) + + +def _js_round(value: float) -> int: + # Math.round of JavaScript, which rounds a half up. round() of Python rounds it to even. + return math.floor(value + 0.5) + + +def _parse_created(body: object) -> _Created: + fields = cast("dict[str, Any]", body) if isinstance(body, dict) else {} + created_id, name, url = fields.get("id"), fields.get("name"), fields.get("url") + items_total = fields.get("itemsTotal") + if ( + not isinstance(created_id, str) + or not isinstance(name, str) + or not isinstance(url, str) + or not _is_int(items_total) + ): + raise ExperimentError( + "invalid_response", "telemetry.dev returned an experiment it cannot read" + ) + return _Created(created_id, name, items_total, url) + + +def _parse_page(body: object) -> _Page | None: + if not isinstance(body, dict): + return None + fields = cast("dict[str, Any]", body) + dataset, items = fields.get("dataset"), fields.get("items") + next_cursor = fields.get("nextCursor") + if not isinstance(dataset, dict) or not isinstance(items, list): + return None + version = cast("dict[str, Any]", dataset).get("version") + if not _is_int(version) or version < 0: + return None + if next_cursor is not None and not isinstance(next_cursor, str): + return None + parsed: list[DatasetItem] = [] + for raw in cast("list[object]", items): + item = _parse_item(raw) + if item is None: + return None + parsed.append(item) + return _Page(version, parsed, next_cursor) + + +def _parse_item(raw: object) -> DatasetItem | None: + if not isinstance(raw, dict): + return None + item = cast("dict[str, Any]", raw) + item_id, number, tags = item.get("id"), item.get("number"), item.get("tags") + item_input, metadata = item.get("input"), item.get("metadata") + expected, split = item.get("expectedOutput"), item.get("split") + if ( + not isinstance(item_id, str) + or not _is_int(number) + or not isinstance(item_input, dict) + or not (expected is None or isinstance(expected, str)) + or not isinstance(metadata, dict) + or not isinstance(tags, list) + or not all(isinstance(tag, str) for tag in cast("list[object]", tags)) + or not (split is None or isinstance(split, str)) + ): + return None + return DatasetItem( + id=item_id, + number=number, + input=cast("dict[str, Any]", item_input), + expected_output=expected, + metadata=cast("dict[str, Any]", metadata), + tags=tuple(cast("list[str]", tags)), + split=split, + ) + + +def _parse_result(body: object) -> ExperimentResult: + fields = cast("dict[str, Any]", body) if isinstance(body, dict) else {} + raw_scores = fields.get("scores") + scores = ( + [_parse_score(score) for score in cast("list[object]", raw_scores)] + if isinstance(raw_scores, list) + else None + ) + check_status = fields.get("checkStatus", "missing") + items_failed = fields.get("itemsFailed", 0) + error = fields.get("error") + if ( + not isinstance(fields.get("id"), str) + or not isinstance(fields.get("name"), str) + or not isinstance(fields.get("status"), str) + or not isinstance(fields.get("url"), str) + or check_status not in (None, "passed", "failed") + or not _is_number(fields.get("itemsTotal")) + or not _is_number(fields.get("itemsDone")) + or not _is_number(items_failed) + or not (error is None or isinstance(error, str)) + or scores is None + or any(score is None for score in scores) + ): + raise ExperimentError("invalid_response", "telemetry.dev returned a result it cannot read") + return ExperimentResult( + id=fields["id"], + name=fields["name"], + status=fields["status"], + check_status=check_status, + items_total=int(fields["itemsTotal"]), + items_done=int(fields["itemsDone"]), + items_failed=int(items_failed), + error=error, + scores=tuple(cast("list[ExperimentScore]", scores)), + url=fields["url"], + ) + + +def _parse_score(raw: object) -> ExperimentScore | None: + if not isinstance(raw, dict): + return None + score = cast("dict[str, Any]", raw) + name, kind, passed = score.get("name"), score.get("kind"), score.get("passed") + pass_count, fail_count = score.get("passCount"), score.get("failCount") + nullable = [score.get(key) for key in ("value", "baselineValue", "delta", "minScore")] + if ( + not isinstance(name, str) + or not (kind is None or isinstance(kind, str)) + or not _is_number(pass_count) + or not _is_number(fail_count) + or not all(value is None or _is_number(value) for value in nullable) + or not (passed is None or isinstance(passed, bool)) + ): + return None + value, baseline_value, delta, min_score = nullable + return ExperimentScore( + name=name, + kind=kind, + value=value, + pass_count=int(cast("float", pass_count)), + fail_count=int(cast("float", fail_count)), + baseline_value=baseline_value, + delta=delta, + min_score=min_score, + passed=passed, + ) + + +def _is_number(value: object) -> bool: + return isinstance(value, (int, float)) and not isinstance(value, bool) diff --git a/sdks/python/src/telemetry_dev/_prompts.py b/sdks/python/src/telemetry_dev/_prompts.py index 9673740..a7f3aff 100644 --- a/sdks/python/src/telemetry_dev/_prompts.py +++ b/sdks/python/src/telemetry_dev/_prompts.py @@ -17,18 +17,16 @@ from contextlib import contextmanager from contextvars import ContextVar from dataclasses import dataclass, field -from datetime import datetime, timezone -from email.message import Message -from email.utils import parsedate_to_datetime from time import monotonic -from typing import IO, Any, Literal, TypeAlias, TypedDict, cast, overload +from typing import Any, Literal, TypeAlias, TypedDict, cast, overload from weakref import WeakKeyDictionary from opentelemetry import context as otel_context from opentelemetry.context import Context +from ._api import NoRedirect, api_error, resolve_api, retry_after from ._client import Client, get_client -from ._config import SDK_VERSION, report_error, resolve_config +from ._config import SDK_VERSION, report_error from ._context import propagated_attributes from ._semconv import ( ATTR_PROMPT_LABEL, @@ -45,8 +43,6 @@ # The longest Retry-After that get_prompt honors, so that a bad header cannot stop fetches. _MAX_RETRY_AFTER_S = 3600.0 _MAX_CACHE_ENTRIES = 1000 -# IMF-fixdate, the HTTP date that servers send: Sun, 06 Nov 1994 08:49:37 GMT. -_HTTP_DATE = re.compile(r"[A-Z][a-z]{2}, \d{2} [A-Z][a-z]{2} \d{4} \d{2}:\d{2}:\d{2} GMT", re.ASCII) _VARIABLE = re.compile(r"\{\{\s*([A-Za-z_][A-Za-z0-9_]*)\s*\}\}") _MODEL_CALL_OPERATIONS = frozenset({"chat", "text_completion", "generate_content"}) _PROMPT_KEY = otel_context.create_key("telemetry_dev.prompt") @@ -310,11 +306,11 @@ def _get_prompt( _check(name, type, label, version, cache_ttl_seconds, fallback) client = get_client() enabled = client if client is not None and client.enabled else None - config = enabled.config if enabled is not None else resolve_config() - key = api_key if api_key is not None else config.api_key - url = (base_url or config.base_url).rstrip("/") + api = resolve_api(api_key, base_url) + key = api.api_key + url = api.base_url asked_label = (label if label is not None else "production") if version is None else None - on_error = enabled.on_error if enabled is not None else None + on_error = api.on_error try: if not key: @@ -535,21 +531,6 @@ def _refresh( done.set() -class _NoRedirect(urllib.request.HTTPRedirectHandler): - # A redirect to another host must not get the API key, so a 3xx fails like any status that is - # not 2xx. - def redirect_request( - self, - req: urllib.request.Request, - fp: IO[bytes], - code: int, - msg: str, - headers: Message, - newurl: str, - ) -> urllib.request.Request | None: - return None - - class _Deadline: """Ends one prompt fetch after ``_FETCH_TIMEOUT_S``. The timeout of urllib applies to each socket operation, so a server that sends a byte now and then could keep a fetch reading forever, @@ -648,12 +629,12 @@ def do_open(self, http_class: Any, req: urllib.request.Request, **http_conn_args return super().do_open(_on_deadline(http_class, req), req, **http_conn_args) -_opener = urllib.request.build_opener(_NoRedirect, _HTTPHandler, _HTTPSHandler) +_opener = urllib.request.build_opener(NoRedirect, _HTTPHandler, _HTTPSHandler) def _fetch(url: str, headers: dict[str, str]) -> dict[str, Any]: deadline = _Deadline() - retry_after: str | None = None + retry_header: str | None = None try: try: with _opener.open( @@ -664,7 +645,7 @@ def _fetch(url: str, headers: dict[str, str]) -> dict[str, Any]: except urllib.error.HTTPError as exc: with exc: status, body = exc.code, exc.read() - retry_after = exc.headers.get("Retry-After") + retry_header = exc.headers.get("Retry-After") except Exception as exc: if ( deadline.expired @@ -681,9 +662,10 @@ def _fetch(url: str, headers: dict[str, str]) -> dict[str, Any]: if deadline.expired: raise _timeout() if not 200 <= status < 300: - error = _api_error(status, body) - if status in (429, 503): - error._retry_after = _retry_after(retry_after) # pyright: ignore[reportPrivateUsage] + error = api_error(PromptError, status, body) + wait = retry_after(status, retry_header) + if wait is not None: + error._retry_after = min(wait, _MAX_RETRY_AFTER_S) # pyright: ignore[reportPrivateUsage] raise error try: parsed = _parse_response(json.loads(body)) @@ -698,40 +680,6 @@ def _timeout() -> PromptError: return PromptError("timeout", f"Fetching the prompt took more than {_FETCH_TIMEOUT_S:g} s") -def _api_error(status: int, body: bytes) -> PromptError: - try: - error = json.loads(body).get("error") - except (ValueError, AttributeError): - error = None - fields = cast("dict[str, Any]", error) if isinstance(error, dict) else {} - code = fields.get("code") - message = fields.get("message") - return PromptError( - code if isinstance(code, str) else f"http_{status}", - message if isinstance(message, str) else f"telemetry.dev returned {status}", - status, - ) - - -def _retry_after(value: str | None) -> float | None: - """Seconds to wait from a Retry-After header of seconds or an IMF-fixdate HTTP date - (``Sun, 06 Nov 1994 08:49:37 GMT``), capped.""" - value = (value or "").strip() - if not value: - return None - if value.isascii() and value.isdigit(): - seconds = float(value) - elif _HTTP_DATE.fullmatch(value): - try: - date = parsedate_to_datetime(value) - except (TypeError, ValueError): - return None - seconds = (date - datetime.now(timezone.utc)).total_seconds() - else: - return None - return min(seconds, _MAX_RETRY_AFTER_S) if seconds > 0 else None - - def _parse_response(data: object) -> dict[str, Any] | None: if not isinstance(data, dict): return None diff --git a/sdks/python/src/telemetry_dev/_scores.py b/sdks/python/src/telemetry_dev/_scores.py new file mode 100644 index 0000000..f5dbee3 --- /dev/null +++ b/sdks/python/src/telemetry_dev/_scores.py @@ -0,0 +1,181 @@ +"""Scores on traces, spans, and sessions, as ``POST /v1/scores`` takes them.""" + +from __future__ import annotations + +import asyncio +import math +import re +from typing import Any, cast + +from ._api import request, resolve_api +from ._config import report_error +from ._serialize import utf16_length + +USER_FEEDBACK = "user_feedback" +MAX_SCORE_NAME_LENGTH = 64 +MAX_LABEL_LENGTH = 64 +MAX_REASON_LENGTH = 4000 +_MAX_SESSION_ID_LENGTH = 1024 +_TRACE_ID = re.compile(r"(?!0+$)[0-9a-f]{32}") +_SPAN_ID = re.compile(r"(?!0+$)[0-9a-f]{16}") +_TIMEOUT_S = 10.0 + + +class ScoreError(Exception): + """Why score failed. ``code`` is the code of the API error (``insufficient_scope``, + ``invalid_api_key``, ...), or ``missing_api_key``, ``invalid_score``, ``timeout``, + ``network_error``, or ``invalid_response``. A response with a different status and no API + error gives ``http_``. ``status`` is the HTTP status of the response.""" + + def __init__(self, code: str, message: str, status: int | None = None) -> None: + super().__init__(message) + self.code = code + self.message = message + self.status = status + + +def score( + *, + name: str, + trace_id: str | None = None, + span_id: str | None = None, + session_id: str | None = None, + value: float | None = None, + label: str | None = None, + comment: str | None = None, + api_key: str | None = None, + base_url: str | None = None, +) -> str | None: + """Record a score on a trace, a span, or a session, for example the feedback of a user on a + reply. ``user_feedback`` is a thumbs up (1) or down (0). Pass ``trace_id``, ``session_id``, or + the two, and ``value``, ``label``, or the two. + + Like the telemetry functions, score never raises: it returns the id of the score, or None + after it sends the error to ``on_error``. It makes one request, with no retry, so that a call + in a request handler waits 10 seconds at most for each socket operation. The trace id of a + span from a client that is not enabled is all zeros, and score reports it as + ``invalid_score``. + """ + api = resolve_api(api_key, base_url) + try: + problem = _check_score(name, trace_id, span_id, session_id, value, label, comment) + if problem is not None: + raise ScoreError("invalid_score", problem) + if not api.api_key: + raise ScoreError( + "missing_api_key", "No API key: pass api_key or set TELEMETRY_DEV_API_KEY" + ) + fields = { + "traceId": trace_id, + "spanId": span_id, + "sessionId": session_id, + "name": name.strip(), + "value": value, + "label": None if label is None else label.strip(), + "comment": comment, + } + body = request( + api, + ScoreError, + "POST", + "/v1/scores", + body={key: field for key, field in fields.items() if field is not None}, + timeout_s=_TIMEOUT_S, + idempotent=False, + attempts=1, + ) + score_id = cast("dict[str, Any]", body).get("id") if isinstance(body, dict) else None + if not isinstance(score_id, str): + raise ScoreError("invalid_response", "telemetry.dev returned a score it cannot read") + return score_id + except Exception as exc: + report_error(api.on_error, "sending the score failed", exc) + return None + + +async def score_async( + *, + name: str, + trace_id: str | None = None, + span_id: str | None = None, + session_id: str | None = None, + value: float | None = None, + label: str | None = None, + comment: str | None = None, + api_key: str | None = None, + base_url: str | None = None, +) -> str | None: + """``score`` in a worker thread, so that the request does not block the event loop.""" + return await asyncio.to_thread( + score, + name=name, + trace_id=trace_id, + span_id=span_id, + session_id=session_id, + value=value, + label=label, + comment=comment, + api_key=api_key, + base_url=base_url, + ) + + +def check_value(name: str, value: object, label: object) -> str | None: + """Why the server would reject the value and the label of a score, or None. A pass/fail score + has the label ``pass`` with the value 1, or ``fail`` with 0.""" + if value is not None and not is_finite(value): + return "value must be a finite number" + if label is not None and not ( + isinstance(label, str) and 0 < utf16_length(label) <= MAX_LABEL_LENGTH + ): + return f"label must be a string of 1 to {MAX_LABEL_LENGTH} characters" + if value is None and label is None: + return "Pass a value or a label" + if name == USER_FEEDBACK and not (label is None and value in (0, 1)): + return f"{USER_FEEDBACK} takes the value 1 for up or 0 for down, and no label" + if label == "pass" and value is not None and value != 1: + return "The label pass goes with the value 1" + if label == "fail" and value is not None and value != 0: + return "The label fail goes with the value 0" + return None + + +def _check_score( + name: object, + trace_id: object, + span_id: object, + session_id: object, + value: object, + label: object, + comment: object, +) -> str | None: + if not isinstance(name, str) or not 0 < utf16_length(name.strip()) <= MAX_SCORE_NAME_LENGTH: + return f"name must be a string of 1 to {MAX_SCORE_NAME_LENGTH} characters" + if trace_id is None and session_id is None: + return "Pass trace_id or session_id" + if trace_id is not None and not (isinstance(trace_id, str) and _TRACE_ID.fullmatch(trace_id)): + return "trace_id must be 32 lowercase hex digits and not all zeros" + if span_id is not None and not (isinstance(span_id, str) and _SPAN_ID.fullmatch(span_id)): + return "span_id must be 16 lowercase hex digits and not all zeros" + if span_id is not None and trace_id is None: + return "Pass the trace_id of the span" + if session_id is not None and not ( + isinstance(session_id, str) and 0 < utf16_length(session_id) <= _MAX_SESSION_ID_LENGTH + ): + return f"session_id must be a string of 1 to {_MAX_SESSION_ID_LENGTH} characters" + if comment is not None and not ( + isinstance(comment, str) and utf16_length(comment) <= MAX_REASON_LENGTH + ): + return f"comment must be a string of at most {MAX_REASON_LENGTH} characters" + return check_value(name.strip(), value, label.strip() if isinstance(label, str) else label) + + +def is_finite(value: object) -> bool: + # True and False are ints in Python, but JSON sends them as booleans, which are not numbers. + if isinstance(value, bool) or not isinstance(value, (int, float)): + return False + try: + return math.isfinite(value) + except OverflowError: + # An int too large for a float, which JavaScript reads as Infinity. + return False diff --git a/sdks/python/src/telemetry_dev/_semconv.py b/sdks/python/src/telemetry_dev/_semconv.py index 69553c0..7d3041b 100644 --- a/sdks/python/src/telemetry_dev/_semconv.py +++ b/sdks/python/src/telemetry_dev/_semconv.py @@ -88,6 +88,8 @@ ATTR_PROMPT_VERSION = "td.prompt.version" ATTR_PROMPT_LABEL = "td.prompt.label" ATTR_PROMPT_VARIABLES = "td.prompt.variables" +ATTR_EXPERIMENT_ID = "td.experiment.id" +ATTR_DATASET_ITEM_ID = "td.dataset.item_id" USAGE_ATTRS: dict[str, str] = { "input_tokens": "gen_ai.usage.input_tokens", diff --git a/sdks/python/src/telemetry_dev/_serialize.py b/sdks/python/src/telemetry_dev/_serialize.py index dcf1321..f69c4a6 100644 --- a/sdks/python/src/telemetry_dev/_serialize.py +++ b/sdks/python/src/telemetry_dev/_serialize.py @@ -30,6 +30,11 @@ class MaskContext: Mask = Callable[[Any, MaskContext], Any] +def utf16_length(text: str) -> int: + """The length of ``text`` in UTF-16 units, as JavaScript and the telemetry.dev API count it.""" + return len(text.encode("utf-16-le", "surrogatepass")) // 2 + + def truncate(text: str, max_len: int, *, keep_end: bool = False) -> str: """Cap ``text`` at ``max_len`` UTF-16 units, marking the cut. diff --git a/sdks/python/tests/fake_api.py b/sdks/python/tests/fake_api.py new file mode 100644 index 0000000..7df2e7a --- /dev/null +++ b/sdks/python/tests/fake_api.py @@ -0,0 +1,193 @@ +"""The Evals routes of telemetry.dev on a local HTTP server: pages of a dataset version with a +cursor, an experiment that takes item posts of at most 100 and skips repeats, and its result.""" + +from __future__ import annotations + +import json +import threading +from collections.abc import Callable, Generator +from contextlib import contextmanager +from dataclasses import dataclass, field +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from typing import Any +from urllib.parse import parse_qsl, urlsplit + +EXPERIMENT_ID = "6f9619ff-8b86-4011-b42d-00c04fc964ff" +EXPERIMENT_URL = f"https://telemetry.dev/o/org/p/project/evals/experiments/{EXPERIMENT_ID}" + + +def item_id(number: int) -> str: + return f"00000000-0000-4000-8000-{number:012d}" + + +def dataset_item(number: int, split: str | None = None) -> dict[str, Any]: + return { + "id": item_id(number), + "number": number, + "input": {"question": f"q{number}"}, + "expectedOutput": f"a{number}", + "metadata": {"source": "test"}, + "tags": ["golden"], + "split": split, + } + + +def result(**overrides: Any) -> dict[str, Any]: + body: dict[str, Any] = { + "id": EXPERIMENT_ID, + "name": "support-golden-v3", + "status": "completed", + "checkStatus": None, + "itemsTotal": 2, + "itemsDone": 2, + "itemsFailed": 0, + "error": None, + "scores": [], + "url": EXPERIMENT_URL, + } + body.update(overrides) + return body + + +@dataclass +class ApiRequest: + method: str + path: str + query: dict[str, str] + headers: dict[str, str] + body: Any + + @property + def route(self) -> str: + return f"{self.method} {self.path}" + + +# (status, JSON body, headers), or None to answer as the API does. +Reply = tuple[int, object, dict[str, str]] + + +@dataclass +class FakeApi: + url: str + items: list[dict[str, Any]] = field(default_factory=lambda: [dataset_item(1), dataset_item(2)]) + page_size: int = 100 + results: list[dict[str, Any]] = field(default_factory=lambda: [result()]) + override: Callable[[ApiRequest], Reply | None] | None = None + requests: list[ApiRequest] = field(default_factory=lambda: []) + stored: set[str] = field(default_factory=lambda: set[str]()) + lock: threading.Lock = field(default_factory=threading.Lock) + + def routes(self) -> list[str]: + return [request.route for request in self.requests] + + def posts(self) -> list[list[dict[str, Any]]]: + return [ + request.body["items"] for request in self.requests if request.path.endswith("/items") + ] + + def answer(self, request: ApiRequest) -> Reply: + with self.lock: + self.requests.append(request) + overridden = self.override(request) if self.override else None + if overridden is not None: + return overridden + return self._route(request) + + def _route(self, request: ApiRequest) -> Reply: + if request.route == "GET /v1/datasets/support-golden": + split = request.query.get("split") + start = int(request.query.get("cursor", "0")) + matching = [item for item in self.items if split is None or item["split"] == split] + more = start + self.page_size < len(matching) + return ( + 200, + { + "dataset": { + "id": "dataset-1", + "name": "support-golden", + "version": int(request.query.get("version", "3")), + }, + "items": matching[start : start + self.page_size], + "nextCursor": str(start + self.page_size) if more else None, + }, + {}, + ) + if request.route == "POST /v1/experiments": + split = request.body.get("split") + count = len([item for item in self.items if split is None or item["split"] == split]) + return ( + 201, + { + "id": EXPERIMENT_ID, + "name": request.body.get("name", "support-golden-v3"), + "datasetVersion": request.body.get("datasetVersion"), + "itemsTotal": count * request.body.get("runsPerItem", 1), + "url": EXPERIMENT_URL, + }, + {}, + ) + if request.route == f"POST /v1/experiments/{EXPERIMENT_ID}/items": + runs = request.body["items"] + if len(runs) > 100: + return 400, {"error": {"code": "invalid_request", "message": "at most 100"}}, {} + keys = [f"{run['itemId']}:{run.get('runIndex', 0)}" for run in runs] + accepted = len([key for key in keys if key not in self.stored]) + self.stored.update(keys) + return 200, {"accepted": accepted, "duplicates": len(runs) - accepted}, {} + if request.route == f"POST /v1/experiments/{EXPERIMENT_ID}/complete": + return 200, {"id": EXPERIMENT_ID, "status": "running"}, {} + if request.route == f"GET /v1/experiments/{EXPERIMENT_ID}": + return 200, self.results.pop(0) if len(self.results) > 1 else self.results[0], {} + return 404, {"error": {"code": "not_found", "message": request.route}}, {} + + +@contextmanager +def fake_api(**options: Any) -> Generator[FakeApi]: + api: FakeApi | None = None + + class Handler(BaseHTTPRequestHandler): + def do_GET(self) -> None: + self._answer() + + def do_POST(self) -> None: + self._answer() + + def _answer(self) -> None: + assert api is not None + length = int(self.headers.get("content-length") or 0) + raw = self.rfile.read(length) if length else b"" + target = urlsplit(self.path) + status, body, headers = api.answer( + ApiRequest( + method=self.command, + path=target.path, + query=dict(parse_qsl(target.query)), + headers={key.lower(): value for key, value in self.headers.items()}, + body=json.loads(raw) if raw else None, + ) + ) + data = body if isinstance(body, bytes) else json.dumps(body).encode() + self.send_response(status) + self.send_header("content-type", "application/json") + self.send_header("content-length", str(len(data))) + for key, value in headers.items(): + self.send_header(key, value) + self.end_headers() + self.wfile.write(data) + + def log_message(self, format: str, *args: object) -> None: + pass + + server = ThreadingHTTPServer(("127.0.0.1", 0), Handler) + thread = threading.Thread( + target=server.serve_forever, kwargs={"poll_interval": 0.01}, daemon=True + ) + host, port = server.server_address[0], server.server_address[1] + api = FakeApi(url=f"http://{host}:{port}", **options) + thread.start() + try: + yield api + finally: + server.shutdown() + server.server_close() + thread.join(timeout=5) diff --git a/sdks/python/tests/test_experiments.py b/sdks/python/tests/test_experiments.py new file mode 100644 index 0000000..a0b1807 --- /dev/null +++ b/sdks/python/tests/test_experiments.py @@ -0,0 +1,746 @@ +from __future__ import annotations + +import asyncio +import json +import threading +import time +from collections.abc import Generator +from pathlib import Path +from typing import Any + +import pytest + +from telemetry_dev import ( + ExperimentError, + ExperimentItem, + ExperimentResult, + ExperimentScore, + Scorer, + log, + propagate_attributes, + run_experiment, + run_experiment_async, + start_span, +) + +from .conftest import MakeClient +from .fake_api import ( + EXPERIMENT_ID, + EXPERIMENT_URL, + ApiRequest, + FakeApi, + Reply, + dataset_item, + fake_api, + item_id, + result, +) + +ENV_VARS = ( + "TELEMETRY_DEV_EXPERIMENT_DATASET", + "TELEMETRY_DEV_EXPERIMENT_MIN_SCORES", + "TELEMETRY_DEV_EXPERIMENT_RESULTS", + "CI", + "GITHUB_ACTIONS", + "GITHUB_REPOSITORY", + "GITHUB_SHA", + "GITHUB_EVENT_PATH", + "GITHUB_RUN_ID", + "GITHUB_SERVER_URL", +) + + +@pytest.fixture(autouse=True) +def clear_ci(monkeypatch: pytest.MonkeyPatch) -> None: + # GitHub Actions sets CI and GITHUB_* for this suite too. + for var in ENV_VARS: + monkeypatch.delenv(var, raising=False) + + +@pytest.fixture +def api() -> Generator[FakeApi]: + with fake_api() as server: + yield server + + +def answer(item: ExperimentItem) -> str: + return f"answer to {item.input['question']}" + + +def experiment(api: FakeApi, **options: Any) -> ExperimentResult: + arguments: dict[str, Any] = { + "api_key": "td_live_test", + "base_url": api.url, + "dataset": "support-golden", + "task": answer, + } + arguments.update(options) + return run_experiment(**arguments) + + +def answer_with(status: int, body: object, headers: dict[str, str] | None = None) -> Reply: + return status, body, headers or {} + + +def test_runs_each_item_of_each_page_posts_batches_of_100_on_one_version_then_completes() -> None: + with fake_api( + items=[dataset_item(number) for number in range(1, 76)], + page_size=50, + results=[result(itemsTotal=150, itemsDone=146, itemsFailed=4)], + ) as api: + outcome = experiment(api, runs_per_item=2, evaluators=["Correctness"]) + + routes = api.routes() + assert routes[:2] == ["GET /v1/datasets/support-golden", "POST /v1/experiments"] + # The second page and the first post of items run at the same time, in either order. + assert sorted(routes[2:-2]) == [ + "GET /v1/datasets/support-golden", + f"POST /v1/experiments/{EXPERIMENT_ID}/items", + f"POST /v1/experiments/{EXPERIMENT_ID}/items", + ] + assert routes[-2:] == [ + f"POST /v1/experiments/{EXPERIMENT_ID}/complete", + f"GET /v1/experiments/{EXPERIMENT_ID}", + ] + # The first page names no version; the next one asks for the version of the first. + pages = [request for request in api.requests if request.path.startswith("/v1/datasets/")] + assert pages[0].query == {} + assert pages[1].query == {"version": "3", "cursor": "50"} + assert api.requests[1].body == { + "dataset": "support-golden", + "datasetVersion": 3, + "evaluators": ["Correctness"], + "runsPerItem": 2, + } + assert api.requests[1].headers["authorization"] == "Bearer td_live_test" + assert api.requests[1].headers["x-telemetry-dev-sdk"] == "telemetry-dev" + + posts = api.posts() + assert [len(items) for items in posts] == [100, 50] + runs = [f"{run['itemId']}:{run['runIndex']}" for items in posts for run in items] + assert len(set(runs)) == 150 + assert f"{item_id(75)}:1" in runs + first = posts[0][0] + assert isinstance(first.pop("latencyMs"), int) + assert first == {"itemId": item_id(1), "runIndex": 0, "output": "answer to q1"} + + assert outcome == ExperimentResult( + id=EXPERIMENT_ID, + name="support-golden-v3", + status="completed", + check_status=None, + items_total=150, + items_done=146, + items_failed=4, + error=None, + scores=(), + url=EXPERIMENT_URL, + ) + + +def test_each_item_run_is_its_own_trace_and_each_span_of_the_task_carries_the_item( + make: MakeClient, api: FakeApi +) -> None: + # The session parent of a root span needs the key. + memory = make(api_key="td_live_test", environment="experiment", session_mode="process") + + def task(item: ExperimentItem) -> dict[str, str]: + with start_span("chat gpt-4o", type="generation"): + pass + log("replied") + return {"reply": f"r{item.number}"} + + # An active span, an ambient session, and the process session would each put the runs in + # one trace. + with ( + propagate_attributes(session_id="ambient-session", user_id="u-1"), + start_span("eval script"), + ): + experiment(api, runs_per_item=2, task=task) + + finished = memory.span_exporter.get_finished_spans() + chats = [span for span in finished if span.name == "chat gpt-4o"] + roots = [span for span in finished if span.name == "experiment support-golden-v3"] + assert len(chats) == 4 + assert len(roots) == 4 + + def trace_of(span: Any) -> str: + return f"{span.context.trace_id:032x}" + + assert len({trace_of(span) for span in roots}) == 4 + assert [span.parent is not None and span.parent.is_remote for span in roots] == [True] * 4 + posted = [run for items in api.posts() for run in items] + for run in posted: + root = next(span for span in roots if trace_of(span) == run["traceId"]) + chat = next(span for span in chats if trace_of(span) == run["traceId"]) + assert chat.parent is not None + assert chat.parent.span_id == root.context.span_id + for span in (root, chat): + attributes = dict(span.attributes or {}) + assert attributes["td.experiment.id"] == EXPERIMENT_ID + assert attributes["td.dataset.item_id"] == run["itemId"] + assert attributes["gen_ai.conversation.id"] == ( + f"{EXPERIMENT_ID}:{run['itemId']}:{run['runIndex']}" + ) + assert attributes["user.id"] == "u-1" + records = memory.log_exporter.get_finished_logs() + assert len(records) == 4 + for record in records: + trace_id = f"{record.log_record.trace_id:032x}" + run = next(run for run in posted if run["traceId"] == trace_id) + attributes = dict(record.log_record.attributes or {}) + assert attributes["td.experiment.id"] == EXPERIMENT_ID + assert attributes["td.dataset.item_id"] == run["itemId"] + assert sorted(run["output"] for run in posted) == [ + '{"reply":"r1"}', + '{"reply":"r1"}', + '{"reply":"r2"}', + '{"reply":"r2"}', + ] + + +def test_runs_at_most_concurrency_tasks_at_a_time() -> None: + lock = threading.Lock() + running = 0 + most = 0 + + def task(item: ExperimentItem) -> str: + nonlocal running, most + with lock: + running += 1 + most = max(most, running) + time.sleep(0.05) + with lock: + running -= 1 + return "ok" + + with fake_api(items=[dataset_item(number) for number in range(1, 13)]) as api: + experiment(api, concurrency=3, task=task) + + assert most == 3 + assert len([run for items in api.posts() for run in items]) == 12 + + +def test_a_task_that_raises_fails_only_its_item(api: FakeApi) -> None: + def task(item: ExperimentItem) -> str: + if item.number == 1: + raise ValueError("no reply") + return "fine" + + experiment(api, task=task) + + failed, passed = [run for items in api.posts() for run in items] + assert failed["output"] is None + assert failed["error"] == "ValueError: no reply" + assert "scores" not in failed + assert passed["output"] == "fine" + assert "error" not in passed + + +def test_scorers_give_scores_and_a_bad_result_drops_only_its_score(make: MakeClient) -> None: + errors: list[BaseException] = [] + make(environment="experiment", on_error=errors.append) + + def thrown(output: Any, item: ExperimentItem) -> None: + raise RuntimeError("judge down") + + scorers: dict[str, Scorer] = { + "exact": lambda output, item: output == f"answer to {item.input['question']}", + "polite": lambda output, item: False, + "length": lambda output, item: len(output), + "tone": lambda output, item: " friendly ", + "judged": lambda output, item: {"value": 0.25, "reason": "too short"}, + "none": lambda output, item: None, + "thrown": thrown, + "mismatched": lambda output, item: {"value": 0.5, "label": "pass"}, + # 33 emoji are 33 characters in Python but 66 UTF-16 units for the API. + "wide": lambda output, item: "\U0001f600" * 33, + } + + with fake_api(items=[dataset_item(1)]) as api: + experiment(api, scorers=scorers) + + assert api.posts()[0][0]["scores"] == [ + {"name": "exact", "value": 1, "label": "pass"}, + {"name": "polite", "value": 0, "label": "fail"}, + {"name": "length", "value": 12}, + {"name": "tone", "label": "friendly"}, + {"name": "judged", "value": 0.25, "reason": "too short"}, + ] + assert [(getattr(error, "code", None), str(error)) for error in errors] == [ + ("scorer_failed", "The scorer thrown failed on item 1: RuntimeError: judge down"), + ( + "scorer_failed", + "The scorer mismatched failed on item 1: TypeError: returned a score that is not " + "valid: The label pass goes with the value 1", + ), + ( + "scorer_failed", + "The scorer wide failed on item 1: TypeError: returned a score that is not valid: " + "label must be a string of 1 to 64 characters", + ), + ] + + +def test_an_output_is_cut_to_100000_utf16_units(api: FakeApi) -> None: + def task(item: ExperimentItem) -> str: + return "\U0001f600" * 60_000 + + experiment(api, task=task) + + output = api.posts()[0][0]["output"] + assert len(output.encode("utf-16-le")) // 2 <= 100_000 + assert output.endswith("...[truncated]") + + +def test_a_failed_post_of_items_stops_new_items_never_completes_and_raises() -> None: + started = 0 + + def task(item: ExperimentItem) -> str: + nonlocal started + started += 1 + return "ok" + + def ended(request: ApiRequest) -> Reply | None: + if request.path.endswith("/items"): + return answer_with( + 409, + {"error": {"code": "experiment_ended", "message": "support-golden-v3 has ended."}}, + ) + return None + + with fake_api(items=[dataset_item(n) for n in range(1, 301)], override=ended) as api: + with pytest.raises(ExperimentError) as caught: + experiment(api, concurrency=1, task=task) + + assert (caught.value.code, caught.value.status) == ("experiment_ended", 409) + # The one worker waits for the post of the first 100, which fails, so no item starts after. + assert started == 100 + assert f"POST /v1/experiments/{EXPERIMENT_ID}/complete" not in api.routes() + + +def test_an_item_post_that_fails_with_500_goes_again() -> None: + failures = [1] + + def flaky(request: ApiRequest) -> Reply | None: + if request.path.endswith("/items") and failures: + failures.pop() + return answer_with(500, {}) + return None + + with fake_api(override=flaky) as api: + experiment(api) + + assert len(api.posts()) == 2 + assert api.posts()[1] == api.posts()[0] + + +def test_a_create_that_fails_with_500_does_not_go_again() -> None: + def broken(request: ApiRequest) -> Reply | None: + return answer_with(500, {}) if request.route == "POST /v1/experiments" else None + + with fake_api(override=broken) as api: + with pytest.raises(ExperimentError) as caught: + experiment(api) + + assert (caught.value.code, caught.value.status) == ("http_500", 500) + assert api.routes().count("POST /v1/experiments") == 1 + + +def test_a_create_that_gets_a_429_goes_again_after_its_retry_after() -> None: + limited = [1] + + def limit(request: ApiRequest) -> Reply | None: + if request.route == "POST /v1/experiments" and limited: + limited.pop() + return answer_with( + 429, + {"error": {"code": "rate_limited", "message": "Too many requests"}}, + {"retry-after": "2"}, + ) + return None + + with fake_api(override=limit) as api: + started = time.monotonic() + experiment(api) + + assert api.routes().count("POST /v1/experiments") == 2 + # Without the header, the first retry waits 1 second. + assert time.monotonic() - started >= 2 + + +def test_waits_until_the_server_evaluators_end_the_experiment() -> None: + with fake_api(results=[result(status="running", itemsDone=1), result()]) as api: + outcome = experiment(api) + + assert outcome.status == "completed" + assert api.routes().count(f"GET /v1/experiments/{EXPERIMENT_ID}") == 2 + + +@pytest.mark.parametrize( + ("status", "body", "code"), + [ + ( + 404, + {"error": {"code": "dataset_not_found", "message": "No dataset"}}, + "dataset_not_found", + ), + (200, {"dataset": {"version": 1}, "items": [{"id": 1}]}, "invalid_response"), + ( + 200, + {"dataset": {"id": "d", "name": "support-golden", "version": 0}, "items": []}, + "dataset_empty", + ), + ], + ids=["the API error", "a page that cannot be read", "an empty dataset"], +) +def test_raises_before_it_creates_an_experiment(status: int, body: object, code: str) -> None: + def page(request: ApiRequest) -> Reply | None: + return answer_with(status, body) if request.path.startswith("/v1/datasets/") else None + + with fake_api(override=page) as api: + with pytest.raises(ExperimentError) as caught: + experiment(api) + + assert caught.value.code == code + assert api.routes() == ["GET /v1/datasets/support-golden"] + + +def test_a_cursor_that_repeats_is_an_invalid_response() -> None: + def same(request: ApiRequest) -> Reply | None: + if not request.path.startswith("/v1/datasets/"): + return None + return answer_with( + 200, + { + "dataset": {"id": "d", "name": "support-golden", "version": 3}, + "items": [dataset_item(1)], + "nextCursor": "same", + }, + ) + + with fake_api(override=same) as api: + with pytest.raises(ExperimentError) as caught: + experiment(api) + + assert caught.value.code == "invalid_response" + assert api.routes().count("GET /v1/datasets/support-golden") == 2 + + +FAILED_CHECK = result( + checkStatus="failed", + scores=[ + { + "name": "correctness", + "kind": "pass_fail", + "value": 0.85, + "passCount": 17, + "failCount": 3, + "baselineValue": 0.9, + "delta": -5, + "minScore": 0.9, + "passed": False, + }, + { + "name": "length", + "kind": None, + "value": None, + "passCount": 0, + "failCount": 0, + "baselineValue": None, + "delta": None, + "minScore": None, + "passed": None, + }, + ], +) + + +def test_in_ci_mode_a_failed_check_raises_with_the_result(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv("CI", "true") + with fake_api(results=[FAILED_CHECK]) as api: + with pytest.raises(ExperimentError) as caught: + experiment(api, min_scores={"correctness": 0.9}) + + assert caught.value.code == "check_failed" + assert str(caught.value) == ( + "The check of support-golden-v3 failed: correctness is 85%, under the pass mark 90%. " + f"{EXPERIMENT_URL}" + ) + assert caught.value.result is not None + assert caught.value.result.scores[1] == ExperimentScore( + name="length", + kind=None, + value=None, + pass_count=0, + fail_count=0, + baseline_value=None, + delta=None, + min_score=None, + passed=None, + ) + + +@pytest.mark.parametrize( + ("options", "ci"), + [({}, None), ({"ci": False}, "true")], + ids=["CI is off", "ci=False wins over CI"], +) +def test_a_failed_check_returns_when( + monkeypatch: pytest.MonkeyPatch, options: dict[str, Any], ci: str | None +) -> None: + if ci is not None: + monkeypatch.setenv("CI", ci) + with fake_api(results=[FAILED_CHECK]) as api: + assert experiment(api, **options).check_status == "failed" + + +def test_in_ci_mode_an_experiment_that_fails_raises_with_its_error( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setenv("CI", "1") + with fake_api( + results=[result(status="failed", error="0 of 2 items ran. Each one failed.")] + ) as api: + with pytest.raises(ExperimentError) as caught: + experiment(api) + + assert caught.value.code == "experiment_failed" + assert str(caught.value) == ( + "The experiment support-golden-v3 failed: 0 of 2 items ran. Each one failed.. " + f"{EXPERIMENT_URL}" + ) + + +def test_in_github_actions_it_records_the_pull_request_its_head_commit_and_the_run( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path, api: FakeApi +) -> None: + event = tmp_path / "event.json" + results = tmp_path / "results.jsonl" + event.write_text( + json.dumps( + { + "pull_request": { + "number": 42, + "title": "Shorter replies", + "html_url": "https://github.com/acme/support/pull/42", + "head": {"sha": "a1b2c3d4e5f6a1b2c3d4e5f6a1b2c3d4e5f6a1b2"}, + } + } + ) + ) + for name, value in { + "CI": "true", + "GITHUB_ACTIONS": "true", + "GITHUB_REPOSITORY": "acme/support", + "GITHUB_SHA": "f" * 40, + "GITHUB_EVENT_PATH": str(event), + "GITHUB_RUN_ID": "991", + "TELEMETRY_DEV_EXPERIMENT_DATASET": "support-golden", + "TELEMETRY_DEV_EXPERIMENT_MIN_SCORES": "correctness=0.9\nlength = 12.5", + "TELEMETRY_DEV_EXPERIMENT_RESULTS": str(results), + }.items(): + monkeypatch.setenv(name, value) + + experiment(api, dataset=None, min_scores={"correctness": 0.5, "tone": 2}) + + create = api.requests[1].body + assert create["dataset"] == "support-golden" + assert create["minScores"] == {"correctness": 0.9, "length": 12.5, "tone": 2} + assert create["ci"] == { + "provider": "github_actions", + "repository": "acme/support", + "commit": "a1b2c3d4e5f6a1b2c3d4e5f6a1b2c3d4e5f6a1b2", + "prNumber": 42, + "prTitle": "Shorter replies", + "prUrl": "https://github.com/acme/support/pull/42", + "runUrl": "https://github.com/acme/support/actions/runs/991", + } + lines = results.read_text().splitlines() + assert [json.loads(line) for line in lines] == [result()] + assert results.stat().st_mode & 0o777 == 0o600 + + +def test_a_pull_request_title_is_cut_to_256_utf16_units( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path, api: FakeApi +) -> None: + event = tmp_path / "event.json" + event.write_text(json.dumps({"pull_request": {"title": "\U0001f600" * 200}})) + for name, value in { + "CI": "true", + "GITHUB_ACTIONS": "true", + "GITHUB_REPOSITORY": "acme/support", + "GITHUB_SHA": "a1b2c3d", + "GITHUB_EVENT_PATH": str(event), + }.items(): + monkeypatch.setenv(name, value) + + experiment(api) + + assert api.requests[1].body["ci"]["prTitle"] == "\U0001f600" * 128 + + +def one(output: Any, item: ExperimentItem) -> int: + return 1 + + +REJECTED: list[tuple[dict[str, str], dict[str, Any], type[Exception], str]] = [ + ( + {}, + {"dataset": None}, + TypeError, + "Pass dataset, or set the dataset input of the experiment action", + ), + ( + {"TELEMETRY_DEV_EXPERIMENT_DATASET": "support-v2"}, + {}, + TypeError, + "dataset is support-golden, but the experiment action names support-v2. Name one dataset.", + ), + ( + {"TELEMETRY_DEV_EXPERIMENT_MIN_SCORES": "correctness=1_000"}, + {}, + TypeError, + 'TELEMETRY_DEV_EXPERIMENT_MIN_SCORES: "correctness=1_000" is not a name=number pair', + ), + ({}, {"runs_per_item": 6}, ValueError, "runs_per_item must be an integer from 1 to 5"), + ({}, {"runs_per_item": True}, ValueError, "runs_per_item must be an integer from 1 to 5"), + ({}, {"concurrency": 0}, ValueError, "concurrency must be an integer from 1 to 100"), + ( + {}, + {"scorers": {"user_feedback": one}}, + TypeError, + "user_feedback is the score of user feedback. Name the scorer another way.", + ), + ( + {}, + {"evaluators": ["correctness"], "scorers": {"Correctness": one}}, + TypeError, + "The scorer Correctness has the name of an evaluator. Rename one of them.", + ), + ({}, {"evaluators": "correctness"}, TypeError, "evaluators must be a list of at most"), +] + + +@pytest.mark.parametrize( + ("env", "options", "error", "message"), + REJECTED, + ids=[ + "no dataset", + "a dataset that the action does not name", + "min-scores that do not parse", + "runs_per_item 6", + "runs_per_item True", + "concurrency 0", + "a scorer named user_feedback", + "a scorer with the name of an evaluator", + "evaluators as one string", + ], +) +def test_raises_before_any_request( + monkeypatch: pytest.MonkeyPatch, + api: FakeApi, + env: dict[str, str], + options: dict[str, Any], + error: type[Exception], + message: str, +) -> None: + for name, value in env.items(): + monkeypatch.setenv(name, value) + + with pytest.raises(error) as caught: + experiment(api, **options) + + assert str(caught.value).startswith(message) + assert api.requests == [] + + +def test_without_an_api_key_it_raises_missing_api_key(api: FakeApi) -> None: + with pytest.raises(ExperimentError) as caught: + experiment(api, api_key="") + + assert caught.value.code == "missing_api_key" + assert api.requests == [] + + +def test_the_task_gets_each_run_of_an_item_of_the_split_with_its_run_index() -> None: + seen: list[ExperimentItem] = [] + + def task(item: ExperimentItem) -> str: + seen.append(item) + return "ok" + + with fake_api(items=[dataset_item(7, "test")]) as api: + experiment(api, split="test", runs_per_item=2, concurrency=1, task=task) + + assert seen == [ + ExperimentItem( + id=item_id(7), + number=7, + input={"question": "q7"}, + expected_output="a7", + metadata={"source": "test"}, + tags=("golden",), + split="test", + run_index=index, + ) + for index in (0, 1) + ] + assert api.requests[0].query["split"] == "test" + assert api.requests[1].body["split"] == "test" + + +def test_run_experiment_rejects_an_async_task(api: FakeApi) -> None: + async def task(item: ExperimentItem) -> str: + return "ok" + + with pytest.raises(TypeError, match="task is async: use run_experiment_async"): + experiment(api, task=task) + assert api.requests == [] + + +async def test_run_experiment_async_runs_async_tasks_and_scorers_at_the_same_time( + make: MakeClient, +) -> None: + memory = make(environment="experiment") + running = 0 + most = 0 + + async def task(item: ExperimentItem) -> str: + nonlocal running, most + running += 1 + most = max(most, running) + with start_span("chat gpt-4o", type="generation"): + await asyncio.sleep(0.05) + running -= 1 + return f"r{item.number}" + + async def exact(output: Any, item: ExperimentItem) -> bool: + await asyncio.sleep(0) + return output == f"r{item.number}" + + with fake_api(items=[dataset_item(number) for number in range(1, 9)]) as api: + outcome = await run_experiment_async( + api_key="td_live_test", + base_url=api.url, + dataset="support-golden", + task=task, + scorers={"exact": exact}, + concurrency=4, + ) + + assert outcome.status == "completed" + assert most == 4 + posted = sorted( + (run for items in api.posts() for run in items), key=lambda run: str(run["itemId"]) + ) + assert [run["scores"] for run in posted] == [ + [{"name": "exact", "value": 1, "label": "pass"}] + ] * 8 + chats = [ + span for span in memory.span_exporter.get_finished_spans() if span.name == "chat gpt-4o" + ] + by_trace = {f"{span.context.trace_id:032x}": span for span in chats} + for run in posted: + attributes = dict(by_trace[run["traceId"]].attributes or {}) + assert attributes["td.dataset.item_id"] == run["itemId"] diff --git a/sdks/python/tests/test_scores.py b/sdks/python/tests/test_scores.py new file mode 100644 index 0000000..44d28de --- /dev/null +++ b/sdks/python/tests/test_scores.py @@ -0,0 +1,204 @@ +from __future__ import annotations + +from collections.abc import Generator +from typing import Any + +import pytest + +from telemetry_dev import ScoreError, score, score_async + +from .conftest import MakeClient +from .fake_api import ApiRequest, FakeApi, Reply, fake_api + +TRACE_ID = "4bf92f3577b34da6a3ce929d0e0e4736" +SPAN_ID = "00f067aa0ba902b7" + + +def replying(reply: Reply) -> Generator[FakeApi]: + def answer(request: ApiRequest) -> Reply: + return reply + + with fake_api(override=answer) as api: + yield api + + +@pytest.fixture +def accepted() -> Generator[FakeApi]: + yield from replying((202, {"id": "score-1"}, {})) + + +@pytest.fixture +def errors(make: MakeClient) -> list[ScoreError]: + caught: list[ScoreError] = [] + make(on_error=caught.append) + return caught + + +def test_posts_the_score_with_the_key_and_base_url_of_init_and_returns_its_id( + make: MakeClient, accepted: FakeApi +) -> None: + make(api_key="td_live_key", base_url=f"{accepted.url}/") + + assert ( + score( + name=" user_feedback ", + trace_id=TRACE_ID, + span_id=SPAN_ID, + value=0, + comment="Wrong refund amount", + ) + == "score-1" + ) + + [request] = accepted.requests + assert request.route == "POST /v1/scores" + assert request.headers["authorization"] == "Bearer td_live_key" + assert request.headers["x-telemetry-dev-sdk"] == "telemetry-dev" + assert request.headers["content-type"] == "application/json" + assert request.body == { + "name": "user_feedback", + "traceId": TRACE_ID, + "spanId": SPAN_ID, + "value": 0, + "comment": "Wrong refund amount", + } + + +async def test_a_session_score_needs_no_trace_and_a_label_goes_trimmed(accepted: FakeApi) -> None: + returned = await score_async( + api_key="td_live_key", + base_url=accepted.url, + name="resolved", + session_id="chat-9", + label=" yes ", + ) + + assert returned == "score-1" + assert accepted.requests[0].body == {"name": "resolved", "sessionId": "chat-9", "label": "yes"} + + +@pytest.mark.parametrize( + ("options", "message"), + [ + ({"trace_id": None}, "Pass trace_id or session_id"), + ( + {"trace_id": "0" * 32}, + "trace_id must be 32 lowercase hex digits and not all zeros", + ), + ( + {"trace_id": TRACE_ID.upper()}, + "trace_id must be 32 lowercase hex digits and not all zeros", + ), + ( + {"trace_id": None, "session_id": "s", "span_id": SPAN_ID}, + "Pass the trace_id of the span", + ), + ({"value": None}, "Pass a value or a label"), + ({"value": float("nan")}, "value must be a finite number"), + ({"value": True}, "value must be a finite number"), + ({"value": 10**400}, "value must be a finite number"), + ( + {"name": "user_feedback", "value": 1, "label": "up"}, + "user_feedback takes the value 1 for up or 0 for down, and no label", + ), + ( + {"name": "user_feedback", "value": 0.5}, + "user_feedback takes the value 1 for up or 0 for down, and no label", + ), + ({"value": 0, "label": "pass"}, "The label pass goes with the value 1"), + ({"value": 1, "label": "fail"}, "The label fail goes with the value 0"), + ({"name": " "}, "name must be a string of 1 to 64 characters"), + # 33 emoji are 33 characters in Python but 66 UTF-16 units for the API. + ({"name": "\U0001f600" * 33}, "name must be a string of 1 to 64 characters"), + ({"comment": "x" * 4001}, "comment must be a string of at most 4000 characters"), + ], + ids=[ + "no trace and no session", + "the all-zero trace id of a span that did not record", + "an uppercase trace id", + "a span without its trace", + "no value and no label", + "a value that is not finite", + "a bool value", + "an int too large for a float", + "a label for user feedback", + "user feedback of 0.5", + "a pass of 0", + "a fail of 1", + "a blank name", + "a name of 66 UTF-16 units", + "a comment of 4001 characters", + ], +) +def test_reports_invalid_score_and_sends_nothing( + errors: list[ScoreError], accepted: FakeApi, options: dict[str, Any], message: str +) -> None: + arguments: dict[str, Any] = { + "api_key": "td_live_key", + "base_url": accepted.url, + "name": "helpful", + "trace_id": TRACE_ID, + "value": 1, + } + arguments.update(options) + + assert score(**arguments) is None + assert accepted.requests == [] + assert [(error.code, error.message) for error in errors] == [("invalid_score", message)] + + +FAILURES: list[tuple[Reply, str, int | None]] = [ + ( + (403, {"error": {"code": "insufficient_scope", "message": "No scores"}}, {}), + "insufficient_scope", + 403, + ), + ((503, {}, {"retry-after": "1"}), "http_503", 503), + ((202, {}, {}), "invalid_response", None), +] + + +@pytest.mark.parametrize( + ("reply", "code", "status"), + FAILURES, + ids=["the API error", "a status with no API error", "a body with no id"], +) +def test_reports_the_failure_tries_once_and_returns_none( + errors: list[ScoreError], reply: Reply, code: str, status: int | None +) -> None: + for api in replying(reply): + assert ( + score( + api_key="td_live_key", base_url=api.url, name="helpful", trace_id=TRACE_ID, value=1 + ) + is None + ) + assert len(api.requests) == 1 + + assert [(error.code, error.status) for error in errors] == [(code, status)] + + +def test_without_an_api_key_it_reports_missing_api_key( + errors: list[ScoreError], accepted: FakeApi +) -> None: + score(api_key="", base_url=accepted.url, name="helpful", trace_id=TRACE_ID, value=1) + + assert accepted.requests == [] + assert [error.code for error in errors] == ["missing_api_key"] + + +def test_does_not_follow_a_redirect_so_the_key_stays_with_the_api_host( + errors: list[ScoreError], +) -> None: + with fake_api() as other: + + def redirect(request: ApiRequest) -> Reply: + return 307, {}, {"location": f"{other.url}/v1/scores"} + + with fake_api(override=redirect) as api: + score( + api_key="td_live_key", base_url=api.url, name="helpful", trace_id=TRACE_ID, value=1 + ) + + assert other.requests == [] + assert [(error.code, error.status) for error in errors] == [("http_307", 307)] From 968a45e0eeff183d2409c0336c6328ac07a2e36d Mon Sep 17 00:00:00 2001 From: ephraimduncan Date: Thu, 29 Oct 2026 23:27:57 +0000 Subject: [PATCH 04/10] feat(sdk): add the experiment action for pull requests actions/experiment is a composite action. It runs the eval script of the user, a .ts, .js, or .py file or a command, with the dataset and the min-scores inputs in the variables that the SDKs read. Then it writes the result to the job summary and to one comment on the pull request, and it fails the job when the script exits with an error. The inputs go to the scripts through env, never into a run line. The results file has mode 0600, and the comment escapes the names. A token that cannot write the comment gives a warning. CI now runs shellcheck on the action script. --- .github/workflows/ci.yml | 2 +- actions/experiment/README.md | 84 +++++ actions/experiment/action.yml | 82 +++++ actions/experiment/report.mjs | 227 +++++++++++++ actions/experiment/run.sh | 49 +++ packages/sdk/tests/experiment-action.test.ts | 315 +++++++++++++++++++ 6 files changed, 758 insertions(+), 1 deletion(-) create mode 100644 actions/experiment/README.md create mode 100644 actions/experiment/action.yml create mode 100644 actions/experiment/report.mjs create mode 100755 actions/experiment/run.sh create mode 100644 packages/sdk/tests/experiment-action.test.ts diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ed6093f..d56afe0 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -16,7 +16,7 @@ jobs: with: persist-credentials: false - run: bash .github/scripts/release_test.sh - - run: shellcheck .github/scripts/*.sh + - run: shellcheck .github/scripts/*.sh actions/experiment/*.sh typescript: runs-on: ubuntu-latest diff --git a/actions/experiment/README.md b/actions/experiment/README.md new file mode 100644 index 0000000..77f5a72 --- /dev/null +++ b/actions/experiment/README.md @@ -0,0 +1,84 @@ +# telemetry.dev experiment action + +This composite action runs your eval script in a GitHub Actions workflow. The script calls +`runExperiment` (`@telemetry-dev/sdk`) or `run_experiment` (`telemetry-dev`). The action writes the +result to the job summary and to a comment on the pull request. The job fails when the check of the +pass marks fails, when the experiment does not complete, or when the script exits with an error. + +```yaml +on: pull_request + +permissions: + contents: read + pull-requests: write + +jobs: + evals: + runs-on: ubuntu-latest + env: + TELEMETRY_DEV_API_KEY: ${{ secrets.TELEMETRY_DEV_API_KEY }} + steps: + - uses: actions/checkout@v5 + - uses: actions/setup-node@v5 + with: + node-version: 24 + - run: npm ci + - uses: telemetry-dev/telemetry/actions/experiment@main + with: + dataset: support-golden + script: evals/reply.eval.ts + min-scores: correctness=0.9 +``` + +`evals/reply.eval.ts` runs the experiment: + +```ts +import { init, runExperiment } from "@telemetry-dev/sdk"; + +init(); + +await runExperiment({ + dataset: "support-golden", + task: (item) => replyToCustomer(item.input), + evaluators: ["correctness"], +}); +``` + +Pin the action to a commit SHA of this repository for a run that does not change when `main` +changes. + +## Inputs + +| Input | Description | +| ------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `script` | The eval script. A `.ts`, `.mts`, `.cts`, or `.tsx` script runs with `npx --yes tsx`, a `.js`, `.mjs`, or `.cjs` script with `node`, and a `.py` script with the `python` (or `python3`) of the PATH. Give `script` or `command`. | +| `command` | A shell command that runs the eval script, for example `uv run python evals/reply_eval.py` or `pnpm eval`. | +| `dataset` | The dataset to run on. The action puts it in `TELEMETRY_DEV_EXPERIMENT_DATASET`. When the script also names a dataset, the two must be the same. | +| `min-scores` | Pass marks, as `name=value` pairs that commas or new lines separate. A pass rate is 0 to 1, and a mean is on the scale of its score. The action puts them in `TELEMETRY_DEV_EXPERIMENT_MIN_SCORES`, and they win over the `minScores` of the script for the same name. | +| `api-key` | A telemetry.dev API key with the `datasets:read` and `experiments:write` scopes. Default: the `TELEMETRY_DEV_API_KEY` variable of the job. | +| `base-url` | The telemetry.dev API. Default: `https://ingest.telemetry.dev`. | +| `working-directory` | The directory that the script or the command runs in. Default: `.`. | +| `comment` | Set to `false` to write the job summary only. | +| `github-token` | The token that writes the comment. Default: `github.token`, which needs the `pull-requests: write` permission. | + +The action sets `TELEMETRY_DEV_ENVIRONMENT` to `experiment` when the job does not set it, so the +traces of the items stay out of your production pages. `CI` is `true` in GitHub Actions, thus the +SDK runs in CI mode: it records the repository, the commit, the run, and the pull request with +the experiment, and it exits with an error when the check fails. + +## Outputs + +| Output | Description | +| -------------- | --------------------------------------------------------------------- | +| `check-status` | `passed`, `failed`, or `none`, for the last experiment of the script. | +| `url` | The page of the last experiment in telemetry.dev. | + +## The comment + +The action adds one comment to the pull request for each workflow, job, and `dataset`, `script`, or +`command`. A later push changes that comment. A pull request from a fork gets a token that cannot +write comments: the action then logs a warning, and the job summary still has the result. + +The comment and the summary show the name of each experiment, its status, the count of the items +that ran, and the value, the change from the baseline, and the pass mark of each score. The SDK +writes the result for the action to a file in `RUNNER_TEMP` with mode `0600`. diff --git a/actions/experiment/action.yml b/actions/experiment/action.yml new file mode 100644 index 0000000..19a774c --- /dev/null +++ b/actions/experiment/action.yml @@ -0,0 +1,82 @@ +name: telemetry.dev experiment +description: Run an experiment of your code on a telemetry.dev dataset, check its scores, and post the result on the pull request. +author: telemetry.dev +branding: + icon: check-circle + color: gray-dark + +inputs: + script: + description: The eval script, which calls runExperiment or run_experiment. A .ts script runs with tsx, a .js script with node, and a .py script with python. Give script or command. + required: false + command: + description: A shell command that runs the eval script, for example `uv run python evals/reply_eval.py`. Give script or command. + required: false + dataset: + description: The dataset to run on. When the script also names a dataset, the two must be the same. + required: false + min-scores: + description: Pass marks, as name=value pairs that commas or new lines separate, for example `correctness=0.9`. A pass rate is 0 to 1. They win over the minScores of the script for the same name. + required: false + api-key: + description: A telemetry.dev API key with the datasets:read and experiments:write scopes. Default the TELEMETRY_DEV_API_KEY variable of the job. + required: false + base-url: + description: The telemetry.dev API. Default https://ingest.telemetry.dev. + required: false + working-directory: + description: The directory that the script or the command runs in. + required: false + default: "." + comment: + description: Set to false to write the job summary only, with no comment on the pull request. + required: false + default: "true" + github-token: + description: The token that writes the comment. It needs the pull-requests write permission. + required: false + default: ${{ github.token }} + +outputs: + check-status: + description: passed, failed, or none, for the last experiment of the script. + value: ${{ steps.report.outputs.check-status }} + url: + description: The page of the last experiment in telemetry.dev. + value: ${{ steps.report.outputs.url }} + +runs: + using: composite + steps: + - id: run + shell: bash + working-directory: ${{ inputs.working-directory }} + env: + INPUT_SCRIPT: ${{ inputs.script }} + INPUT_COMMAND: ${{ inputs.command }} + INPUT_DATASET: ${{ inputs.dataset }} + INPUT_MIN_SCORES: ${{ inputs.min-scores }} + INPUT_API_KEY: ${{ inputs.api-key }} + INPUT_BASE_URL: ${{ inputs.base-url }} + run: bash "$GITHUB_ACTION_PATH/run.sh" + + - id: report + if: always() + shell: bash + env: + TELEMETRY_DEV_EXPERIMENT_RESULTS: ${{ steps.run.outputs.results }} + EXPERIMENT_EXIT_CODE: ${{ steps.run.outputs.exit-code }} + INPUT_DATASET: ${{ inputs.dataset }} + INPUT_SCRIPT: ${{ inputs.script }} + INPUT_COMMAND: ${{ inputs.command }} + INPUT_COMMENT: ${{ inputs.comment }} + GITHUB_TOKEN: ${{ inputs.github-token }} + run: node "$GITHUB_ACTION_PATH/report.mjs" + + - if: always() && steps.run.outputs.exit-code != '0' + shell: bash + env: + EXIT_CODE: ${{ steps.run.outputs.exit-code }} + run: | + echo "::error title=telemetry.dev experiment::The experiment script exited with code ${EXIT_CODE:-unknown}." + exit "${EXIT_CODE:-1}" diff --git a/actions/experiment/report.mjs b/actions/experiment/report.mjs new file mode 100644 index 0000000..67ce615 --- /dev/null +++ b/actions/experiment/report.mjs @@ -0,0 +1,227 @@ +// Writes the job summary of the experiment action, and the comment with the result on the pull +// request. runExperiment and run_experiment add one line of JSON to the results file for each +// experiment: the body of GET /v1/experiments/{id}. +import { createHash } from "node:crypto"; +import { appendFileSync, readFileSync } from "node:fs"; + +const RATE_KINDS = new Set(["pass_fail", "up_down"]); +/** Pages of 100 comments that the action reads to find its comment from an earlier push. */ +const MAX_COMMENT_PAGES = 10; +const STATUS = { completed: "Completed", failed: "Failed", stopped: "Stopped" }; + +const env = process.env; +const exitCode = Number.parseInt(env.EXPERIMENT_EXIT_CODE ?? "", 10); +const results = readResults(env.TELEMETRY_DEV_EXPERIMENT_RESULTS); +// The comment of this job and input, so that a new push changes it and a second step adds its own. +const key = createHash("sha256") + .update( + [ + env.GITHUB_WORKFLOW, + env.GITHUB_JOB, + env.INPUT_DATASET, + env.INPUT_SCRIPT, + env.INPUT_COMMAND, + ].join("\0"), + ) + .digest("hex") + .slice(0, 12); +const marker = ``; +const report = render(results, exitCode); + +if (env.GITHUB_STEP_SUMMARY) appendFileSync(env.GITHUB_STEP_SUMMARY, `${report}\n`); + +const last = results.at(-1); + +if (env.GITHUB_OUTPUT) { + appendFileSync( + env.GITHUB_OUTPUT, + `check-status=${last?.checkStatus ?? "none"}\nurl=${last ? (httpsUrl(last.url) ?? "") : ""}\n`, + ); +} + +if (env.INPUT_COMMENT !== "false") { + await comment(`${marker}\n${report}`).catch((error) => { + // A pull request from a fork gets a token that cannot write comments. + console.log( + `::warning title=telemetry.dev experiment::The comment on the pull request failed: ${error.message}`, + ); + }); +} + +function readResults(path) { + if (!path) return []; + + let text; + + try { + text = readFileSync(path, "utf8"); + } catch { + return []; + } + + return text + .split("\n") + .filter((line) => line.trim() !== "") + .flatMap((line) => { + try { + const result = JSON.parse(line); + + return typeof result === "object" && result !== null && typeof result.name === "string" + ? [result] + : []; + } catch { + return []; + } + }); +} + +function render(experiments, code) { + const lines = ["### telemetry.dev experiment", ""]; + + if (experiments.length === 0) { + lines.push( + `The experiment script exited with code ${Number.isNaN(code) ? "unknown" : code} before it wrote a result. The log of the step has the error.`, + ); + } + + for (const experiment of experiments) { + const url = httpsUrl(experiment.url); + const name = url ? `[${text(experiment.name)}](${url})` : text(experiment.name); + const total = Number(experiment.itemsTotal) || 0; + const failed = Number(experiment.itemsFailed) || 0; + const ran = Math.max((Number(experiment.itemsDone) || 0) - failed, 0); + + lines.push( + `**${name}**: ${outcome(experiment)}. ${ran} of ${total} items ran${failed > 0 ? `, ${failed} failed` : ""}.`, + ); + + if (typeof experiment.error === "string" && experiment.error !== "") { + lines.push("", `> ${text(experiment.error)}`); + } + + const scores = Array.isArray(experiment.scores) ? experiment.scores : []; + + if (scores.length > 0) { + lines.push( + "", + "| Score | Value | Change | Pass mark | Result |", + "| --- | --- | --- | --- | --- |", + ); + + for (const score of scores) { + lines.push( + `| ${text(score.name)} | ${value(score.kind, score.value)} | ${change(score.kind, score.delta)} | ${value(score.kind, score.minScore)} | ${score.passed === true ? "Passed" : score.passed === false ? "Failed" : "—"} |`, + ); + } + } + + lines.push(""); + } + + if (code !== 0 && experiments.length > 0) { + lines.push( + `The experiment script exited with code ${Number.isNaN(code) ? "unknown" : code}.`, + "", + ); + } + + return lines.join("\n").trimEnd(); +} + +function outcome(experiment) { + if (experiment.status !== "completed") + return STATUS[experiment.status] ?? text(String(experiment.status)); + + if (experiment.checkStatus === "failed") return "Check failed"; + + return experiment.checkStatus === "passed" ? "Check passed" : "Completed"; +} + +function value(kind, number) { + if (typeof number !== "number") return "—"; + + return RATE_KINDS.has(kind) + ? `${Math.round(number * 100)}%` + : String(Math.round(number * 100) / 100); +} + +function change(kind, delta) { + if (typeof delta !== "number") return "—"; + + if (delta === 0) return "0"; + + const size = RATE_KINDS.has(kind) ? `${Math.abs(delta)} pts` : String(Math.abs(delta)); + + return `${delta > 0 ? "↑" : "↓"} ${size}`; +} + +/** Text that cannot break the table or add HTML to the comment. */ +function text(value) { + return String(value) + .replace(/[\r\n]+/g, " ") + .replace(/&/g, "&") + .replace(//g, ">") + .replace(/\|/g, "\\|") + .replace(/([[\]()*_`])/g, "\\$1"); +} + +function httpsUrl(url) { + return typeof url === "string" && /^https:\/\/[^\s()<>]+$/.test(url) ? url : undefined; +} + +async function comment(body) { + const pull = pullRequestNumber(); + + if (pull === undefined || !env.GITHUB_TOKEN || !env.GITHUB_REPOSITORY) return; + + const api = (env.GITHUB_API_URL ?? "https://api.github.com").replace(/\/+$/, ""); + const issue = `${api}/repos/${env.GITHUB_REPOSITORY}/issues`; + let existing; + + for (let page = 1; page <= MAX_COMMENT_PAGES && existing === undefined; page++) { + const comments = await github("GET", `${issue}/${pull}/comments?per_page=100&page=${page}`); + + existing = comments.find( + (item) => typeof item.body === "string" && item.body.startsWith(marker), + ); + + if (comments.length < 100) break; + } + + if (existing) await github("PATCH", `${issue}/comments/${existing.id}`, { body }); + else await github("POST", `${issue}/${pull}/comments`, { body }); +} + +function pullRequestNumber() { + if (!env.GITHUB_EVENT_PATH) return undefined; + + try { + const number = JSON.parse(readFileSync(env.GITHUB_EVENT_PATH, "utf8")).pull_request?.number; + + return Number.isSafeInteger(number) && number > 0 ? number : undefined; + } catch { + return undefined; + } +} + +async function github(method, url, body) { + const headers = { + accept: "application/vnd.github+json", + authorization: `Bearer ${env.GITHUB_TOKEN}`, + "x-github-api-version": "2022-11-28", + }; + const res = await fetch(url, { + method, + ...(body + ? { headers: { ...headers, "content-type": "application/json" }, body: JSON.stringify(body) } + : { headers }), + redirect: "error", + signal: AbortSignal.timeout(15_000), + }); + + if (!res.ok) + throw new Error(`GitHub returned ${res.status} for ${method} ${new URL(url).pathname}`); + + return res.json(); +} diff --git a/actions/experiment/run.sh b/actions/experiment/run.sh new file mode 100755 index 0000000..9d6f1c2 --- /dev/null +++ b/actions/experiment/run.sh @@ -0,0 +1,49 @@ +#!/usr/bin/env bash +# Runs the experiment script of the experiment action. A failed script does not fail this step: +# the report step reads the exit code from the output exit-code, and the last step exits with it. +set -uo pipefail + +script=${INPUT_SCRIPT:-} +command=${INPUT_COMMAND:-} + +fail() { + echo "::error title=telemetry.dev experiment::$1" + echo "exit-code=2" >>"$GITHUB_OUTPUT" + exit 0 +} + +if [ -n "$script" ] && [ -n "$command" ]; then fail "Give script or command, not the two."; fi +if [ -z "$script" ] && [ -z "$command" ]; then fail "Give the script or the command that runs the experiment."; fi + +if [ -n "${INPUT_API_KEY:-}" ]; then + echo "::add-mask::$INPUT_API_KEY" + export TELEMETRY_DEV_API_KEY=$INPUT_API_KEY +fi +if [ -z "${TELEMETRY_DEV_API_KEY:-}" ]; then fail "Set the api-key input or the TELEMETRY_DEV_API_KEY variable."; fi +if [ -n "${INPUT_BASE_URL:-}" ]; then export TELEMETRY_DEV_BASE_URL=$INPUT_BASE_URL; fi +if [ -n "${INPUT_DATASET:-}" ]; then export TELEMETRY_DEV_EXPERIMENT_DATASET=$INPUT_DATASET; fi +if [ -n "${INPUT_MIN_SCORES:-}" ]; then export TELEMETRY_DEV_EXPERIMENT_MIN_SCORES=$INPUT_MIN_SCORES; fi +# The traces of the items stay out of production unless the job names an environment. +export TELEMETRY_DEV_ENVIRONMENT=${TELEMETRY_DEV_ENVIRONMENT:-experiment} + +# mktemp makes the file with mode 0600, in a directory of this job. +results=$(mktemp "${RUNNER_TEMP:-/tmp}/telemetry-dev-experiment.XXXXXX") || fail "Could not make the results file." +echo "results=$results" >>"$GITHUB_OUTPUT" +export TELEMETRY_DEV_EXPERIMENT_RESULTS=$results + +code=0 +if [ -n "$command" ]; then + bash -c "$command" || code=$? +else + case "$script" in + *.ts | *.mts | *.cts | *.tsx) npx --yes tsx "$script" || code=$? ;; + *.js | *.mjs | *.cjs) node "$script" || code=$? ;; + *.py) + python=$(command -v python || command -v python3) || fail "No python on the PATH. Use actions/setup-python, or give a command." + "$python" "$script" || code=$? + ;; + *) fail "The script must end in .ts, .js, or .py. For another runner, give a command." ;; + esac +fi + +echo "exit-code=$code" >>"$GITHUB_OUTPUT" diff --git a/packages/sdk/tests/experiment-action.test.ts b/packages/sdk/tests/experiment-action.test.ts new file mode 100644 index 0000000..cb7be99 --- /dev/null +++ b/packages/sdk/tests/experiment-action.test.ts @@ -0,0 +1,315 @@ +import { spawn, spawnSync } from "node:child_process"; +import { mkdtempSync, readFileSync, statSync, writeFileSync } from "node:fs"; +import { createServer, type IncomingMessage } from "node:http"; +import type { AddressInfo } from "node:net"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { fileURLToPath } from "node:url"; + +import { afterEach, expect, test } from "vitest"; + +// The experiment action: run.sh runs the eval script, and report.mjs writes the job summary and +// the comment on the pull request. +const action = fileURLToPath(new URL("../../../actions/experiment/", import.meta.url)); +const closers: (() => void)[] = []; + +afterEach(() => { + for (const close of closers.splice(0)) close(); +}); + +function workdir(): string { + return mkdtempSync(join(tmpdir(), "td-action-")); +} + +function outputs(path: string): Record { + return Object.fromEntries( + readFileSync(path, "utf8") + .split("\n") + .filter(Boolean) + .map((line) => line.split(/=(.*)/s).slice(0, 2) as [string, string]), + ); +} + +function runScript(env: Record, files: Record = {}) { + const dir = workdir(); + const output = join(dir, "output"); + writeFileSync(output, ""); + + for (const [name, content] of Object.entries(files)) writeFileSync(join(dir, name), content); + + const run = spawnSync("bash", [join(action, "run.sh")], { + cwd: dir, + encoding: "utf8", + env: { PATH: process.env.PATH, RUNNER_TEMP: dir, GITHUB_OUTPUT: output, ...env }, + }); + + return { dir, run, outputs: outputs(output) }; +} + +const result = { + id: "6f9619ff-8b86-4011-b42d-00c04fc964ff", + name: "reply-v13-gpt-4o", + status: "completed", + checkStatus: "failed", + itemsTotal: 120, + itemsDone: 120, + itemsFailed: 2, + error: null, + scores: [ + { + name: "correctness", + kind: "pass_fail", + value: 0.85, + passCount: 100, + failCount: 18, + baselineValue: 0.88, + delta: -3, + minScore: 0.9, + passed: false, + }, + { + name: "helpfulness", + kind: "number", + value: 4.436, + passCount: 0, + failCount: 0, + baselineValue: 4.1, + delta: 0.3, + minScore: null, + passed: null, + }, + ], + url: "https://telemetry.dev/o/org/p/project/evals/experiments/6f9619ff-8b86-4011-b42d-00c04fc964ff", +}; + +const summary = `### telemetry.dev experiment + +**[reply-v13-gpt-4o](https://telemetry.dev/o/org/p/project/evals/experiments/6f9619ff-8b86-4011-b42d-00c04fc964ff)**: Check failed. 118 of 120 items ran, 2 failed. + +| Score | Value | Change | Pass mark | Result | +| --- | --- | --- | --- | --- | +| correctness | 85% | ↓ 3 pts | 90% | Failed | +| helpfulness | 4.44 | ↑ 0.3 | — | — | + +The experiment script exited with code 1.`; + +interface GithubRequest { + method: string; + url: string; + body: unknown; +} + +/** The comment routes of the GitHub REST API for one pull request. */ +async function fakeGithub( + options: { comments?: { id: number; body: string }[]; status?: number } = {}, +) { + const requests: GithubRequest[] = []; + const server = createServer(async (req: IncomingMessage, res) => { + let raw = ""; + for await (const chunk of req) raw += chunk; + requests.push({ method: req.method!, url: req.url!, body: raw ? JSON.parse(raw) : undefined }); + res.writeHead(options.status ?? 200, { "content-type": "application/json" }); + res.end(JSON.stringify(req.method === "GET" ? (options.comments ?? []) : { id: 9 })); + }); + await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve)); + closers.push(() => server.close()); + + return { url: `http://127.0.0.1:${(server.address() as AddressInfo).port}`, requests }; +} + +async function report(env: Record, lines: unknown[]) { + const dir = workdir(); + const results = join(dir, "results.jsonl"); + const stepSummary = join(dir, "summary.md"); + const output = join(dir, "output"); + const event = join(dir, "event.json"); + writeFileSync(results, lines.map((line) => `${JSON.stringify(line)}\n`).join("")); + writeFileSync(stepSummary, ""); + writeFileSync(output, ""); + writeFileSync(event, JSON.stringify({ pull_request: { number: 42 } })); + + const child = spawn("node", [join(action, "report.mjs")], { + env: { + PATH: process.env.PATH, + TELEMETRY_DEV_EXPERIMENT_RESULTS: results, + GITHUB_STEP_SUMMARY: stepSummary, + GITHUB_OUTPUT: output, + GITHUB_EVENT_PATH: event, + GITHUB_REPOSITORY: "acme/support", + GITHUB_TOKEN: "ghs_test", + GITHUB_WORKFLOW: "evals", + GITHUB_JOB: "reply", + INPUT_DATASET: "support-golden", + EXPERIMENT_EXIT_CODE: "1", + ...env, + }, + }); + let stdout = ""; + child.stdout.on("data", (chunk) => (stdout += chunk)); + const code = await new Promise((resolve) => child.on("close", resolve)); + + return { code, stdout, summary: readFileSync(stepSummary, "utf8"), outputs: outputs(output) }; +} + +test("run.sh passes the inputs to the script and records its exit code without failing", () => { + const { + dir, + run, + outputs: written, + } = runScript({ + INPUT_COMMAND: `node -e 'const fs = require("node:fs"); fs.writeFileSync("env.json", JSON.stringify(process.env)); process.exit(3)'`, + INPUT_API_KEY: "td_live_secret", + INPUT_DATASET: "support-golden", + INPUT_MIN_SCORES: "correctness=0.9", + INPUT_BASE_URL: "https://ingest.test", + }); + + expect(run.status).toBe(0); + expect(run.stdout).toContain("::add-mask::td_live_secret"); + expect(written["exit-code"]).toBe("3"); + + const env = JSON.parse(readFileSync(join(dir, "env.json"), "utf8")); + expect(env).toMatchObject({ + TELEMETRY_DEV_API_KEY: "td_live_secret", + TELEMETRY_DEV_BASE_URL: "https://ingest.test", + TELEMETRY_DEV_EXPERIMENT_DATASET: "support-golden", + TELEMETRY_DEV_EXPERIMENT_MIN_SCORES: "correctness=0.9", + TELEMETRY_DEV_ENVIRONMENT: "experiment", + TELEMETRY_DEV_EXPERIMENT_RESULTS: written.results, + }); + expect(written.results!.startsWith(dir)).toBe(true); + expect(statSync(written.results!).mode & 0o777).toBe(0o600); +}); + +test("run.sh keeps the environment of the job, and runs a .js script with node", () => { + const { run, outputs: written } = runScript( + { + INPUT_SCRIPT: "print-env.mjs", + TELEMETRY_DEV_API_KEY: "td_live_job", + TELEMETRY_DEV_ENVIRONMENT: "staging", + }, + { "print-env.mjs": "console.log(JSON.stringify(process.env));\n" }, + ); + + expect(written["exit-code"]).toBe("0"); + expect(JSON.parse(run.stdout)).toMatchObject({ + TELEMETRY_DEV_API_KEY: "td_live_job", + TELEMETRY_DEV_ENVIRONMENT: "staging", + }); +}); + +test.each([ + [ + "script and command", + { INPUT_SCRIPT: "a.ts", INPUT_COMMAND: "true", TELEMETRY_DEV_API_KEY: "k" }, + "Give script or command, not the two.", + ], + [ + "no script and no command", + { TELEMETRY_DEV_API_KEY: "k" }, + "Give the script or the command that runs the experiment.", + ], + [ + "no API key", + { INPUT_COMMAND: "true" }, + "Set the api-key input or the TELEMETRY_DEV_API_KEY variable.", + ], + [ + "a script of another type", + { INPUT_SCRIPT: "eval.rb", TELEMETRY_DEV_API_KEY: "k" }, + "The script must end in .ts, .js, or .py. For another runner, give a command.", + ], +])("run.sh stops with an error for %s", (_, env, message) => { + const { run, outputs: written } = runScript(env); + + expect(run.stdout).toContain(`::error title=telemetry.dev experiment::${message}`); + expect(written["exit-code"]).toBe("2"); +}); + +test("report.mjs writes the summary, posts it on the pull request, and sets the outputs", async () => { + const github = await fakeGithub(); + + const { + code, + summary: written, + outputs: set, + } = await report({ GITHUB_API_URL: github.url }, [result]); + + expect(code).toBe(0); + expect(written).toBe(`${summary}\n`); + expect(set).toEqual({ "check-status": "failed", url: result.url }); + expect(github.requests.map((request) => `${request.method} ${request.url}`)).toEqual([ + "GET /repos/acme/support/issues/42/comments?per_page=100&page=1", + "POST /repos/acme/support/issues/42/comments", + ]); + expect(github.requests[1]!.body).toEqual({ + body: expect.stringMatching( + /^\n### telemetry\.dev experiment\n/, + ), + }); + expect((github.requests[1]!.body as { body: string }).body.endsWith(summary)).toBe(true); +}); + +test("report.mjs changes its comment from an earlier push, and a step with another dataset adds its own", async () => { + const first = await fakeGithub(); + await report({ GITHUB_API_URL: first.url }, [result]); + const own = { id: 8, body: (first.requests[1]!.body as { body: string }).body }; + + const other = await fakeGithub({ comments: [own] }); + await report({ GITHUB_API_URL: other.url, INPUT_DATASET: "billing-golden" }, [result]); + + const again = await fakeGithub({ comments: [{ id: 7, body: "Looks good" }, own] }); + await report({ GITHUB_API_URL: again.url }, [result]); + + expect(other.requests.map((request) => request.method)).toEqual(["GET", "POST"]); + expect(again.requests.map((request) => `${request.method} ${request.url}`)).toEqual([ + "GET /repos/acme/support/issues/42/comments?per_page=100&page=1", + "PATCH /repos/acme/support/issues/comments/8", + ]); +}); + +test("report.mjs warns and succeeds when the token cannot write the comment", async () => { + const github = await fakeGithub({ status: 403 }); + + const { code, stdout } = await report({ GITHUB_API_URL: github.url }, [result]); + + expect(code).toBe(0); + expect(stdout).toContain( + "::warning title=telemetry.dev experiment::The comment on the pull request failed: GitHub returned 403", + ); +}); + +test("report.mjs reports a script that ended before it wrote a result, and comment: false posts nothing", async () => { + const github = await fakeGithub(); + + const { summary: written, outputs: set } = await report( + { GITHUB_API_URL: github.url, INPUT_COMMENT: "false", EXPERIMENT_EXIT_CODE: "1" }, + [], + ); + + expect(github.requests).toEqual([]); + + expect(written).toBe( + "### telemetry.dev experiment\n\nThe experiment script exited with code 1 before it wrote a result. The log of the step has the error.\n", + ); + expect(set).toEqual({ "check-status": "none", url: "" }); +}); + +test("report.mjs escapes names, so that they cannot break the table or add HTML", async () => { + const github = await fakeGithub(); + const { summary: written } = await report( + { GITHUB_API_URL: github.url, INPUT_COMMENT: "false", EXPERIMENT_EXIT_CODE: "0" }, + [ + { + ...result, + name: "|*bold*", + url: "javascript:alert(1)", + scores: [{ ...result.scores[0], name: "a|b\nc" }], + }, + ], + ); + + expect(written).toContain("**<img src=x>\\|\\*bold\\***: Check failed."); + expect(written).toContain("| a\\|b c | 85% |"); + expect(written).not.toContain("javascript:"); +}); From 473a5293b699201632b35acc1d3a868807eed7c8 Mon Sep 17 00:00:00 2001 From: ephraimduncan Date: Thu, 29 Oct 2026 23:28:10 +0000 Subject: [PATCH 05/10] docs(sdk): document experiments, scores, and the experiment action The READMEs of the two core SDKs get the Experiments and the Scores sections. They tell which data goes to telemetry.dev when output capture is off: the outputs, the task errors, the scorer reasons, and the metadata. The conformance document gets the experiment attributes and the wire contract of the two SDKs: the requests, the retry rules, the limits, the score rules, CI mode, and the variables of the action. --- docs/sdk-conformance.md | 88 +++++++++++++++++++++++++++++++ packages/sdk/README.md | 110 +++++++++++++++++++++++++++++++++++++++ sdks/python/README.md | 111 ++++++++++++++++++++++++++++++++++++++++ 3 files changed, 309 insertions(+) diff --git a/docs/sdk-conformance.md b/docs/sdk-conformance.md index 09583dc..e81153b 100644 --- a/docs/sdk-conformance.md +++ b/docs/sdk-conformance.md @@ -44,6 +44,10 @@ repository, not this SDK workspace. Paths below refer to that product repository | `prompt.version` | `td.prompt.version` (int) | omitted for a fallback prompt (`version` null / None) | | `prompt.label` | `td.prompt.label` | only for a fetch by label; a fetch without label or version is the `production` label | | variables of the last `prompt.compile()` before `withPrompt` / `with_prompt` starts, or in its scope | `td.prompt.variables` (JSON object) | mask → JSON stringify → truncate; gated by the effective captureInput of the span | +| `runExperiment` / `run_experiment` → each span and log record of an item run | `td.experiment.id` | the id that `POST /v1/experiments` returns; set like a propagated attribute, so the provider integration spans get it | +| the item of the run | `td.dataset.item_id` | the `id` of the dataset item: its key, the same across dataset versions | +| the item run | `gen_ai.conversation.id = "::"` | one session and one trace for each run; replaces an ambient or process session, and the run drops the active span | +| the root span of an item run | name `experiment `, `gen_ai.operation.name = "function"` | `input` is the item input; `output` is the task result; gated by capture flags | `SpanHandle.serializeContent(value, key)` / `SpanHandle.serialize_content(value, key)` returns what `input` / `output` would emit for `value` under `key`: mask → JSON stringify → truncate. It @@ -178,6 +182,90 @@ cannot tell a concurrent task from an awaited helper, so in TypeScript all spans its last `compile()`; concurrent tasks that compile their own variables call `withPrompt` in each task. +## Experiments and scores + +`runExperiment(options)` / `run_experiment(...)` (and `run_experiment_async`) and `score(options)` +/ `score(...)` (and `score_async`) call the Evals routes on `{baseUrl}`, with the API key, the base +URL, and the empty-value rules of getPrompt. The requests have the `Authorization: Bearer ` +header, the SDK header, and a JSON body. The redirect rule of getPrompt applies. An error has the +code of the `{ error: { code, message } }` body, or `http_`, or `missing_api_key`, +`timeout`, `network_error`, or `invalid_response`. + +An experiment run sends these requests in this sequence: + +1. `GET /v1/datasets/{name}?version=&split=&cursor=`. The first page names the `datasetVersion` + option, or no version. Each next page names the `version` of the first page and its + `nextCursor`. A page with a different version, a cursor that repeats, a cursor with no items, + or more than 10,000 items in all is `invalid_response`. A first page with no items is + `dataset_empty`, and no experiment starts. +2. `POST /v1/experiments` with `name`, `dataset`, `datasetVersion` (the version of the first + page), `split`, `evaluators`, `minScores`, `runsPerItem`, `ci`, and `metadata`. Keys without a + value are left out. +3. `POST /v1/experiments/{id}/items` with at most 100 item runs and at most 4 MiB of JSON. An item + run has `itemId`, `runIndex`, `output` (a string as it is, other values as `JSON.stringify` JSON, + `null` for no value, cut to 100,000 UTF-16 units), `traceId` (when the span records), + `latencyMs`, and, for a task that failed, `error` (`: `, cut to 4,000 UTF-16 + units) or, for a task that succeeded, `scores`. The posts go one at a time, in order. A post + that fails stops the run: no item starts after it, and the run fails when the running items end. +4. `POST /v1/experiments/{id}/complete`, after `flush()`. +5. `GET /v1/experiments/{id}`, after 1, 2, 4, and then each 5 seconds, until the `status` is + `completed`, `failed`, or `stopped`, for 30 minutes at most (`timeout`). + +Retries: the GET requests, the item posts, and the complete request try 3 times in all after a +timeout, a network error, a 429, or a 5xx. The server skips an item run that it has. The POST that +makes the experiment tries again only after a 429 or a 503. A retry waits for the `Retry-After` of +a 429 or 503 (seconds or an HTTP date), at most 60 seconds, or else 1 second and then 2 seconds. +`POST /v1/scores` makes one request, with no retry. The TypeScript request timeouts abort the +request: 30 seconds for the experiment requests and 10 seconds for a score. The Python timeouts +apply to each socket operation of urllib. + +Options and limits: `runsPerItem` / `runs_per_item` is an integer from 1 to 5 (default 1). +`concurrency` is an integer from 1 to 100 (default 4). `evaluators` has at most 20 trimmed names +of 1 to 64 UTF-16 units. `scorers` has at most 20 scorers with trimmed names of 1 to 64 UTF-16 +units, and no scorer has the name `user_feedback` or the name of an evaluator of the run in a +different case. `minScores` / `min_scores` has at most 20 trimmed names, with finite numbers. A +Python bool is not an integer or a number for these options. Option errors are `TypeError` and +`RangeError` (TypeScript) or `TypeError` and `ValueError` (Python), before any request. + +Scorers run after the span of the item ends, one after the other. A result of `true` or `false` +is `{ value: 1, label: "pass" }` or `{ value: 0, label: "fail" }`. A number is the `value`. A +string is the trimmed `label`. An object or mapping gives `value` (a boolean there is a pass or a +fail), the trimmed `label`, and `reason` (cut to 4,000 UTF-16 units). `null` / `None` gives no +score. A result that the score rules below reject, or a scorer that throws, gives no score and +sends an error with the code `scorer_failed` to `onError` / `on_error`. + +Score rules, for `score` and for scorer results: the trimmed name has 1 to 64 UTF-16 units. A +value is a finite number. A label has 1 to 64 UTF-16 units after the trim. A score has a value, a +label, or the two. `user_feedback` has the value 0 or 1 and no label. The label `pass` goes with no +value or 1, and `fail` with no value or 0. For `score`: a `traceId` is 32 lowercase hex digits and +not all zeros, a `spanId` is 16 lowercase hex digits and not all zeros and needs a `traceId`, a +`sessionId` has 1 to 1,024 UTF-16 units, and a `comment` has at most 4,000. A score needs a +`traceId` or a `sessionId`. A score that breaks a rule is `invalid_score`, with no request. `score` +never throws: it returns the id, or no value after it sends the `ScoreError` to `onError` / +`on_error`. + +CI mode is on when the `ci` option is true, or with no option when `CI` is `true` or `1`. Then: + +- In GitHub Actions (`GITHUB_ACTIONS=true` and a `GITHUB_REPOSITORY` of `owner/name`, at most 200 + characters), `ci` is `{ provider: "github_actions", repository, commit, prNumber, prTitle, prUrl, +runUrl }`. The commit is the `pull_request.head.sha` of `GITHUB_EVENT_PATH`, else `GITHUB_SHA`, + and must be 7 to 64 hex digits. The title is cut to 256 UTF-16 units. A URL must be `https://` + with no space, at most 2,048 UTF-16 units. The run URL is + `//actions/runs/`. Keys without a value are left + out. +- After the result, a status other than `completed` is `experiment_failed`, and a `checkStatus` of + `failed` is `check_failed`. The error has the `result`. + +The experiment action sets these variables: `TELEMETRY_DEV_EXPERIMENT_DATASET` is the dataset when +the `dataset` option has no value, and a different `dataset` option is a `TypeError`. +`TELEMETRY_DEV_EXPERIMENT_MIN_SCORES` holds `name=value` pairs that commas or new lines separate. +The value must match `-?\d+(\.\d+)?` (ASCII digits), else it is a `TypeError`. These marks win over +the `minScores` option for the same name. When `TELEMETRY_DEV_EXPERIMENT_RESULTS` names a file, the +SDK adds the JSON of the `GET /v1/experiments/{id}` body to it as one line. + +The SDKs print one line when the experiment starts and the result when it ends. A run with no +enabled client, or in an environment other than `experiment`, logs a warning. + ## The shared scenario Both e2e runners emit exactly this (under diff --git a/packages/sdk/README.md b/packages/sdk/README.md index afd5ef1..c60c5ec 100644 --- a/packages/sdk/README.md +++ b/packages/sdk/README.md @@ -182,6 +182,113 @@ and TanStack AI integrations, and from other OpenTelemetry tracers, do not get t A model call outside `withPrompt` does not get the attributes, even when it sends messages from `compile()`; with input capture on, the call still records those messages as its input. +## Experiments + +`runExperiment` runs your code on each item of a dataset that you manage in telemetry.dev, and +records the outputs and the scores as an experiment. The experiment page compares it with the +baseline of the dataset. + +```ts +import { init, runExperiment } from "@telemetry-dev/sdk"; + +init({ environment: "experiment" }); + +const result = await runExperiment({ + name: "reply-v13-gpt-4o", + dataset: "support-golden", + task: (item) => replyToCustomer(item.input), + evaluators: ["correctness", "helpfulness"], + scorers: { short: (output) => String(output).length < 600 }, + minScores: { correctness: 0.9 }, +}); +``` + +- runExperiment reads the newest version of the dataset, or `datasetVersion`, 100 items at a time. + With `split`, it runs only the items of that split. Then it makes the experiment, with a name + that telemetry.dev picks when you give no `name`. +- `task(item)` gets the item: `id`, `number`, `input`, `expectedOutput`, `metadata`, `tags`, + `split`, and `runIndex`. Its return value, or the value of its promise, is the output. A string + stays as it is, and a different value becomes JSON, cut to 100,000 characters. A task that + throws fails only its item, and the item records the name and the message of the error. +- `concurrency` (1 to 100, default 4) items run at the same time. `runsPerItem` (1 to 5, default + 1. runs each item more than one time. The pass rate and the mean of each score use all runs. +- Each item run is a trace of its own, with the span `experiment `. The SDK puts + `td.experiment.id` and `td.dataset.item_id` on each span of the run, also on the spans of the + provider integrations. Each run also gets its own session id, `::`, as `gen_ai.conversation.id`. Spans from the AI SDK and TanStack AI integrations do not + get these attributes. +- Call `init({ environment: "experiment" })` before the run. Pages, metrics, and alerts leave out + the `experiment` environment by default. In a different environment, runExperiment logs a + warning. Without a client, the items record no traces and no cost. +- `evaluators` names evaluators of the project: LLM judges, code checks, and rules. telemetry.dev + runs them on the outputs. `scorers` run on your machine, after the span of the item ends. A + scorer returns `true` or `false` for a pass or a fail, a number, a label, or an object with + `value`, `label`, and `reason`. A scorer that throws, or returns a score that is not valid, + gives no score for that item and reports a `scorer_failed` error through `onError`. A scorer + cannot have the name of an evaluator of the run, or the name `user_feedback`. +- runExperiment sends the outputs in batches of 100 items at most, and then waits up to 30 + minutes for the server evaluators. Then it prints the scores and returns the result: `status`, + `checkStatus`, item counts, `scores` with the value, the baseline value, the change, and the + pass mark of each score, and the `url` of the experiment page. +- `minScores` sets a pass mark for each score name: the lowest pass rate (0 to 1) of a pass/fail + score, or the lowest mean of a number score. A score under its mark fails the check. +- A failed request rejects with an `ExperimentError` whose `code` tells why, for example + `dataset_not_found`, `dataset_empty`, `evaluator_not_found`, `insufficient_scope`, + `missing_api_key`, or `timeout`. A read of the dataset or the result, a post of outputs, and the + completion go again up to 2 times after a timeout, a network error, a 429, or a 5xx. The server + skips an item run that it has. The request that makes the experiment goes again only after a 429 + or a 503, thus a lost response cannot make a second experiment. If the run stops before the end, + the experiment does not complete in telemetry.dev. +- The key needs the `datasets:read` and `experiments:write` scopes. runExperiment uses the same + API key, base URL, and `fetch` as getPrompt, or its own `apiKey`, `baseUrl`, and `fetch` + options. + +The outputs, the errors of failed tasks, the scorer reasons, and `metadata` go to telemetry.dev +also when `captureOutput` is off, because they are the result of the experiment. The capture +options and `mask` apply only to the spans of the run. + +### CI mode + +When the `CI` variable is `true` or `1`, or with `ci: true`, runExperiment rejects when the check +fails (`check_failed`) or when the experiment does not complete (`experiment_failed`), so the +process exits with an error. The error has the `result`. In GitHub Actions, the experiment also +records the repository, the commit, the run URL, and the number, the title, and the URL of the pull +request. The commit of a pull request is its head commit. + +The [experiment action](../../actions/experiment) runs your eval script in a workflow, and posts +the result on the pull request. It gives runExperiment the `dataset` input in +`TELEMETRY_DEV_EXPERIMENT_DATASET`, which must agree with the `dataset` of the script, and the +`min-scores` input in `TELEMETRY_DEV_EXPERIMENT_MIN_SCORES`, which wins over `minScores` for the +same name. + +## Scores + +`score` records a score on a trace, a span, or a session, for example the thumbs up or down of a +user on a reply: + +```ts +import { score, startActiveSpan } from "@telemetry-dev/sdk"; + +// When the app replies, keep the trace id of the reply with the message: +const reply = await startActiveSpan("reply", async (span) => ({ + text: await replyToCustomer(message), + traceId: span.traceId, +})); + +// Later, when the user clicks thumbs down: +await score({ name: "user_feedback", traceId: reply.traceId, value: 0, comment: "Wrong amount" }); +``` + +- Give `traceId`, `sessionId`, or the two, and a `spanId` of the trace for a score on one span. + Give `value`, `label`, or the two. `user_feedback` takes the value 1 for up or 0 for down, and + no label. A pass/fail score has the label `pass` with the value 1, or `fail` with 0. +- Like the telemetry functions, score never rejects. It resolves with the id of the score, or with + `undefined` after it reports a `ScoreError` through `onError`. It checks the score before it + sends it (`invalid_score`). The trace id of a span from a client that is not initialized is all + zeros, and score reports it as `invalid_score`. +- score makes one request, with no retry, and waits 10 seconds at most. The key needs the + `scores:write` scope. A browser must not hold the key, so call score from your server. + ## API - `init(options?)` — create the client (isolated OTel TracerProvider by default). @@ -203,6 +310,9 @@ A model call outside `withPrompt` does not get the attributes, even when it send `gen_ai.conversation.id`, and `td.metadata.*` on every span and log record in scope. - `getPrompt(name, options?)` — fetch a prompt by label or version; see [Prompts](#prompts). - `withPrompt(prompt, fn)` — record the prompt on each model call in `fn`. +- `runExperiment(options)` — run your code on a dataset as an experiment; see + [Experiments](#experiments). +- `score(options)` — record a score on a trace, a span, or a session; see [Scores](#scores). - `log(message, { level, eventName, attributes }?)` — log record correlated to the active trace. - `getTraceparent()` — W3C traceparent of the active span (pass to other services; accept it via `startSpan(name, { parent })`). diff --git a/sdks/python/README.md b/sdks/python/README.md index b785d10..f7696a1 100644 --- a/sdks/python/README.md +++ b/sdks/python/README.md @@ -89,6 +89,8 @@ later `init()` generates a new ID. It is not persisted across processes. | `get_traceparent() -> str \| None` | W3C traceparent of the current context. | | `get_prompt(name, *, label=, version=, type="chat", cache_ttl_seconds=60, fallback=, api_key=, base_url=)` / `get_prompt_async(...)` | Fetch a prompt by label or version; see [Prompts](#prompts). | | `with_prompt(prompt)` | Context manager recording the prompt on each model call started inside. | +| `run_experiment(*, task, dataset=, name=, dataset_version=, split=, evaluators=, scorers=, min_scores=, runs_per_item=1, concurrency=4, metadata=, ci=, api_key=, base_url=)` / `run_experiment_async(...)` | Run your code on a dataset as an experiment; see [Experiments](#experiments). | +| `score(*, name, trace_id=, span_id=, session_id=, value=, label=, comment=, api_key=, base_url=)` / `score_async(...)` | Record a score on a trace, a span, or a session; see [Scores](#scores). | | `flush(timeout_s=10.0)` / `shutdown(timeout_s=10.0)` | Force-flush / tear down traces + logs + metrics. Shutdown also runs atexit unless `disable_atexit=True`. | | `TelemetrySpanProcessor` / `telemetry_dev.otel.create_telemetry_span_exporter` | Bring-your-own-OTel helpers (below). | | `MaskContext`, `Usage`, `SpanHandle`, `Client`, `NOT_GIVEN` | Supporting types. | @@ -255,6 +257,115 @@ other OpenTelemetry tracers do not get the attributes. A model call outside `wit get the attributes, even when it sends messages from `compile()`; with input capture on, the call still records those messages as its input. +## Experiments + +`run_experiment` runs your code on each item of a dataset that you manage in telemetry.dev, and +records the outputs and the scores as an experiment. The experiment page compares it with the +baseline of the dataset. + +```python +import telemetry_dev + +telemetry_dev.init(environment="experiment") + +result = telemetry_dev.run_experiment( + name="reply-v13-gpt-4o", + dataset="support-golden", + task=lambda item: reply_to_customer(item.input), + evaluators=["correctness", "helpfulness"], + scorers={"short": lambda output, item: len(output) < 600}, + min_scores={"correctness": 0.9}, +) +``` + +- `run_experiment` reads the newest version of the dataset, or `dataset_version`, 100 items at a + time. With `split`, it runs only the items of that split. Then it makes the experiment, with a + name that telemetry.dev picks when you give no `name`. +- `task(item)` gets an `ExperimentItem`: `id`, `number`, `input`, `expected_output`, `metadata`, + `tags`, `split`, and `run_index`. Its return value is the output. A string stays as it is, and a + different value becomes JSON, cut to 100,000 UTF-16 units. A task that raises fails only its + item, and the item records the type and the message of the exception. +- `concurrency` (1 to 100, default 4) threads run the items, each in a copy of the context of the + caller, so `propagate_attributes` around the call applies to each item. `runs_per_item` (1 to 5, + default 1) runs each item more than one time. The pass rate and the mean of each score use all + runs. +- `run_experiment_async` takes async tasks and scorers. `concurrency` asyncio tasks run the items, + and the requests go in worker threads. A task that is not async runs on the event loop, so use + `run_experiment` for it. `run_experiment` raises `TypeError` for an async task or scorer. +- Each item run is a trace of its own, with the span `experiment `. The SDK puts + `td.experiment.id` and `td.dataset.item_id` on each span of the run, also on the spans of the + provider integrations. Each run also gets its own session id, `::`, as `gen_ai.conversation.id`. +- Call `init(environment="experiment")` before the run. Pages, metrics, and alerts leave out the + `experiment` environment by default. In a different environment, `run_experiment` logs a + warning. Without an enabled client, the items record no traces and no cost. +- `evaluators` names evaluators of the project: LLM judges, code checks, and rules. telemetry.dev + runs them on the outputs. `scorers` run on your machine, after the span of the item ends. A + scorer returns `True` or `False` for a pass or a fail, a number, a label, or a mapping with + `value`, `label`, and `reason`. A scorer that raises, or returns a score that is not valid, gives + no score for that item and sends a `scorer_failed` error to `on_error`. A scorer cannot have the + name of an evaluator of the run, or the name `user_feedback`. +- `run_experiment` sends the outputs in batches of 100 items at most, and then waits up to 30 + minutes for the server evaluators. Then it prints the scores and returns an `ExperimentResult`: + `status`, `check_status`, item counts, `scores` with the value, the baseline value, the change, + and the pass mark of each score, and the `url` of the experiment page. +- `min_scores` sets a pass mark for each score name: the lowest pass rate (0 to 1) of a pass/fail + score, or the lowest mean of a number score. A score under its mark fails the check. +- A failed request raises `ExperimentError` whose `code` tells why, for example + `dataset_not_found`, `dataset_empty`, `evaluator_not_found`, `insufficient_scope`, + `missing_api_key`, or `timeout`. A read of the dataset or the result, a post of outputs, and the + completion go again up to 2 times after a timeout, a network error, a 429, or a 5xx. The server + skips an item run that it has. The request that makes the experiment goes again only after a 429 + or a 503, thus a lost response cannot make a second experiment. If the run stops before the end, + the experiment does not complete in telemetry.dev. +- The key needs the `datasets:read` and `experiments:write` scopes. `run_experiment` uses the same + API key and base URL as `get_prompt`, or its own `api_key` and `base_url`. + +The outputs, the exceptions of failed tasks, the scorer reasons, and `metadata` go to telemetry.dev +also when `capture_output` is off, because they are the result of the experiment. The capture +options and `mask` apply only to the spans of the run. + +### CI mode + +When the `CI` variable is `true` or `1`, or with `ci=True`, `run_experiment` raises when the check +fails (`check_failed`) or when the experiment does not complete (`experiment_failed`), so the +process exits with an error. The error has the `result`. In GitHub Actions, the experiment also +records the repository, the commit, the run URL, and the number, the title, and the URL of the pull +request. The commit of a pull request is its head commit. + +The [experiment action](../../actions/experiment) runs your eval script in a workflow, and posts +the result on the pull request. It gives `run_experiment` the `dataset` input in +`TELEMETRY_DEV_EXPERIMENT_DATASET`, which must agree with the `dataset` of the script, and the +`min-scores` input in `TELEMETRY_DEV_EXPERIMENT_MIN_SCORES`, which wins over `min_scores` for the +same name. + +## Scores + +`score` records a score on a trace, a span, or a session, for example the thumbs up or down of a +user on a reply: + +```python +import telemetry_dev + +# When the app replies, keep the trace id of the reply with the message: +with telemetry_dev.start_span("reply") as span: + text = reply_to_customer(message) + trace_id = f"{span.span.get_span_context().trace_id:032x}" + +# Later, when the user clicks thumbs down: +telemetry_dev.score(name="user_feedback", trace_id=trace_id, value=0, comment="Wrong amount") +``` + +- Give `trace_id`, `session_id`, or the two, and a `span_id` of the trace for a score on one span. + Give `value`, `label`, or the two. `user_feedback` takes the value 1 for up or 0 for down, and + no label. A pass/fail score has the label `pass` with the value 1, or `fail` with 0. +- Like the telemetry functions, `score` never raises. It returns the id of the score, or `None` + after it sends a `ScoreError` to `on_error`. It checks the score before it sends it + (`invalid_score`). The trace id of a span from a client that is not enabled is all zeros, and + `score` reports it as `invalid_score`. +- `score` makes one request, with no retry. `score_async` runs it in a worker thread. The key + needs the `scores:write` scope. + ## Auto-metrics Ended spans automatically record histograms (DELTA temporality, exported every 60s): From 1d8ee43268e3acb782e443c8146558987e1887ef Mon Sep 17 00:00:00 2001 From: ephraimduncan Date: Fri, 30 Oct 2026 00:33:26 +0000 Subject: [PATCH 06/10] fix(sdk): complete the experiment after a run stops send() called fetch as a method of the Api object. Workers and browsers reject a fetch call that has a different `this` object. Thus score and runExperiment did not operate there without a `fetch` option. send() calls fetch as a function, not as a method. When a run stops after runExperiment makes the experiment, it sends the complete request before it rejects. Examples are an item post or a page fetch that is not successful, and more than 10,000 items. The server then completes the experiment with its item runs, or sets the status to failed when it has none. An error of that request goes to onError, and runExperiment rejects with the first error. The create request tries again only after a 429. The server sends a 503 when its database work is longer than its time limit. That 503 can come after the commit, and a retry then makes a second experiment. Each run of an item gets its own copy of the item. Thus a task that changes its input does not change the next run. The SDK cuts a pull request title without half of a surrogate pair, because Postgres rejects a lone surrogate in jsonb. A score field that can be null can also be missing, as in the Python SDK. A result without checkStatus stays an invalid_response. score() without options reports invalid_score and does not reject. New tests do checks of the 4 MiB batch limit, the 10,000-item limit, the 30-minute wait, and the retry after a timeout or a network error. --- packages/sdk/README.md | 33 +-- packages/sdk/src/api.ts | 12 +- packages/sdk/src/ci.ts | 14 +- packages/sdk/src/experiments.ts | 116 ++++++----- packages/sdk/src/prompts.ts | 1 - packages/sdk/src/scores.ts | 24 ++- packages/sdk/tests/experiments.test.ts | 273 +++++++++++++++++++++++-- packages/sdk/tests/scores.test.ts | 25 +++ 8 files changed, 403 insertions(+), 95 deletions(-) diff --git a/packages/sdk/README.md b/packages/sdk/README.md index c60c5ec..0edb8e2 100644 --- a/packages/sdk/README.md +++ b/packages/sdk/README.md @@ -208,15 +208,18 @@ const result = await runExperiment({ that telemetry.dev picks when you give no `name`. - `task(item)` gets the item: `id`, `number`, `input`, `expectedOutput`, `metadata`, `tags`, `split`, and `runIndex`. Its return value, or the value of its promise, is the output. A string - stays as it is, and a different value becomes JSON, cut to 100,000 characters. A task that - throws fails only its item, and the item records the name and the message of the error. -- `concurrency` (1 to 100, default 4) items run at the same time. `runsPerItem` (1 to 5, default - 1. runs each item more than one time. The pass rate and the mean of each score use all runs. + stays as it is, and a different value becomes JSON, cut to 100,000 characters. A value that JSON + cannot hold, such as an object that holds itself, gives no output. A task that throws fails only + its item, and the item records the name and the message of the error. +- `concurrency` (1 to 100, default 4) items run at the same time. With `runsPerItem` (default 1, + at most 5), each item runs more than one time. Each run gets its own copy of the item, so a task + that changes its input does not change the next run. The pass rate and the mean of each score + use all runs. - Each item run is a trace of its own, with the span `experiment `. The SDK puts `td.experiment.id` and `td.dataset.item_id` on each span of the run, also on the spans of the - provider integrations. Each run also gets its own session id, `::`, as `gen_ai.conversation.id`. Spans from the AI SDK and TanStack AI integrations do not - get these attributes. + provider integrations. Each run also gets its own session id, + `::`, as `gen_ai.conversation.id`. Spans from the AI SDK and + TanStack AI integrations do not get these attributes. - Call `init({ environment: "experiment" })` before the run. Pages, metrics, and alerts leave out the `experiment` environment by default. In a different environment, runExperiment logs a warning. Without a client, the items record no traces and no cost. @@ -236,16 +239,20 @@ index>`, as `gen_ai.conversation.id`. Spans from the AI SDK and TanStack AI inte `dataset_not_found`, `dataset_empty`, `evaluator_not_found`, `insufficient_scope`, `missing_api_key`, or `timeout`. A read of the dataset or the result, a post of outputs, and the completion go again up to 2 times after a timeout, a network error, a 429, or a 5xx. The server - skips an item run that it has. The request that makes the experiment goes again only after a 429 - or a 503, thus a lost response cannot make a second experiment. If the run stops before the end, - the experiment does not complete in telemetry.dev. + skips an item run that it has. The request that makes the experiment goes again only after a + 429, thus a lost response or a 503 cannot make a second experiment. When the run stops after it + made the experiment, runExperiment sends the completion before it rejects: telemetry.dev then + ends the experiment with the item runs that it got, or fails it when it got none. - The key needs the `datasets:read` and `experiments:write` scopes. runExperiment uses the same API key, base URL, and `fetch` as getPrompt, or its own `apiKey`, `baseUrl`, and `fetch` - options. + options. With `environment: "experiment"`, the key of the telemetry client must be for all + environments or for `experiment`: a key for another environment cannot send the traces of the + items, and the items then have no trace and no cost. The outputs, the errors of failed tasks, the scorer reasons, and `metadata` go to telemetry.dev -also when `captureOutput` is off, because they are the result of the experiment. The capture -options and `mask` apply only to the spans of the run. +and stay with the experiment, because they are its result. `captureOutput`, `mask`, and the +capture setting of the environment in the project do not apply to them. They apply only to the +spans of the run. ### CI mode diff --git a/packages/sdk/src/api.ts b/packages/sdk/src/api.ts index 8274701..573f780 100644 --- a/packages/sdk/src/api.ts +++ b/packages/sdk/src/api.ts @@ -34,8 +34,8 @@ interface RequestOptions { timeoutMs: number; /** * The server takes a repeat of this request at most once, so a retry after a timeout, a network - * error, or a 5xx is safe. Other requests try again only after a 429 or a 503: the server did - * not take the request then. + * error, or a 5xx is safe. Other requests try again only after a 429, which the rate limit + * answers before the server does any work. A 503 can come after the work committed. */ idempotent: boolean; /** Default 3. */ @@ -141,8 +141,12 @@ async function send( const controller = new AbortController(); const timer = setTimeout(() => controller.abort(), timeoutMs); + // A plain call, not a method call: Workers and browsers reject a fetch whose `this` is not the + // global object. + const { fetchImpl } = api; + try { - const res = await api.fetchImpl(url, { + const res = await fetchImpl(url, { method, headers: { authorization: `Bearer ${api.apiKey}`, @@ -193,7 +197,7 @@ async function send( } function retryable(status: number | undefined, idempotent: boolean): boolean { - if (status === 429 || status === 503) return true; + if (status === 429) return true; return idempotent && (status === undefined || status >= 500); } diff --git a/packages/sdk/src/ci.ts b/packages/sdk/src/ci.ts index 3d7603e..33f13e9 100644 --- a/packages/sdk/src/ci.ts +++ b/packages/sdk/src/ci.ts @@ -141,12 +141,24 @@ function pullRequest( Number.isSafeInteger(pull.number) && (pull.number as number) > 0 ? (pull.number as number) : undefined, - title: typeof pull.title === "string" ? pull.title.slice(0, MAX_PR_TITLE_LENGTH) : undefined, + title: typeof pull.title === "string" ? cut(pull.title, MAX_PR_TITLE_LENGTH) : undefined, url: typeof pull.html_url === "string" ? httpsUrl(pull.html_url) : undefined, commit: typeof head?.sha === "string" ? head.sha : undefined, }; } +/** + * `text` cut to `max` UTF-16 units, with no half of a surrogate pair at the end: the API stores the + * run as jsonb, and Postgres rejects a lone surrogate there. + */ +function cut(text: string, max: number): string { + if (text.length <= max) return text; + + const last = text.charCodeAt(max - 1); + + return text.slice(0, last >= 0xd800 && last <= 0xdbff ? max - 1 : max); +} + function httpsUrl(url: string): string | undefined { return url.length <= MAX_URL_LENGTH && HTTPS_URL.test(url) ? url : undefined; } diff --git a/packages/sdk/src/experiments.ts b/packages/sdk/src/experiments.ts index f3d6d4b..9bcc963 100644 --- a/packages/sdk/src/experiments.ts +++ b/packages/sdk/src/experiments.ts @@ -220,8 +220,10 @@ const RATE_KINDS = new Set(["pass_fail", "up_down"]); * runExperiment waits for the `evaluators` of the server, prints the result, and returns it. * * The run cannot start without its dataset, so runExperiment rejects with an `ExperimentError` - * when a request fails. A task that throws fails only its item. In CI mode, runExperiment rejects - * when the check fails, so the process exits with an error. + * when a request fails. When the run stops after it made the experiment, it first sends the + * complete request, so that telemetry.dev ends the experiment with the item runs that it has. A + * task that throws fails only its item. In CI mode, runExperiment rejects when the check fails, so + * the process exits with an error. */ export async function runExperiment(options: RunExperimentOptions): Promise { const env = resolveEnv(); @@ -266,23 +268,24 @@ export async function runExperiment(options: RunExperimentOptions): Promise 1 ? `, ${run.runsPerItem} runs each` : "") + - `. ${created.url}`, - ); + try { + console.log( + `Experiment ${created.name}: ${created.itemsTotal / run.runsPerItem} items` + + (run.runsPerItem > 1 ? `, ${run.runsPerItem} runs each` : "") + + `. ${created.url}`, + ); + await runItems(api, run, created, itemRuns(api, run, first)); + } catch (error) { + // Without the complete, the experiment would stay running: the server ends it with the item + // runs that it has, or fails it when it has none. + await complete(api, created.id).catch((cause: unknown) => reportError(api.onError, cause)); + + throw error; + } - await runItems(api, run, created, itemRuns(api, run, first)); // The spans of the items reach ingest before the server reads their cost. await flush(); - - await request( - api, - ExperimentError, - "POST", - `/v1/experiments/${encodeURIComponent(created.id)}/complete`, - { timeoutMs: REQUEST_TIMEOUT_MS, idempotent: true }, - ); + await complete(api, created.id); const { result, body } = await waitForResult(api, created.id); writeResult(env, body, api.onError); @@ -456,7 +459,10 @@ async function* itemRuns( } for (const item of page.items) { - for (let runIndex = 0; runIndex < run.runsPerItem; runIndex++) yield { ...item, runIndex }; + // A copy for each run, so that a task that changes its input does not change the next run. + for (let runIndex = 0; runIndex < run.runsPerItem; runIndex++) { + yield { ...structuredClone(item), runIndex }; + } } if (page.nextCursor === null) return; @@ -496,7 +502,7 @@ async function fetchPage( /** * Runs the items, `concurrency` at a time, and posts each result. A post or a page fetch that - * fails stops the run: no new item starts, and runExperiment rejects when the running items end. + * fails stops the run: no new item starts, and runItems rejects when the running items end. */ async function runItems( api: Authed, @@ -527,11 +533,11 @@ async function runItems( if (next.done) return; - const result = await runItem(run, experiment, next.value, api.onError); - - await batch.add(result).catch((error: unknown) => { - failure ??= { error }; - }); + await runItem(run, experiment, next.value, api.onError) + .then((result) => batch.add(result)) + .catch((error: unknown) => { + failure ??= { error }; + }); } }; @@ -733,15 +739,23 @@ function outputText(output: unknown): string | null { } function errorText(error: unknown): string { - if (error instanceof Error) return `${error.name}: ${error.message}`; - try { - return String(error); + return error instanceof Error ? `${error.name}: ${error.message}` : String(error); } catch { return "A value that cannot be converted to a string was thrown"; } } +function complete(api: Authed, id: string): Promise { + return request( + api, + ExperimentError, + "POST", + `/v1/experiments/${encodeURIComponent(id)}/complete`, + { timeoutMs: REQUEST_TIMEOUT_MS, idempotent: true }, + ); +} + /** Waits until the server evaluators finish and the experiment ends. */ async function waitForResult( api: Authed, @@ -896,6 +910,7 @@ function parseResult(body: unknown): ExperimentResult { typeof body.name !== "string" || typeof body.status !== "string" || typeof body.url !== "string" || + // Required: it decides the result of CI mode, so a missing one must not read as no check. !( body.checkStatus === null || body.checkStatus === "passed" || @@ -925,34 +940,33 @@ function parseResult(body: unknown): ExperimentResult { }; } -function parseScore(value: unknown): ExperimentScore | undefined { +/** A score of the result. A field that can be null can also be missing. */ +function parseScore(raw: unknown): ExperimentScore | undefined { + if (!isRecord(raw)) return undefined; + + const score = { + name: raw.name, + kind: raw.kind ?? null, + value: raw.value ?? null, + passCount: raw.passCount, + failCount: raw.failCount, + baselineValue: raw.baselineValue ?? null, + delta: raw.delta ?? null, + minScore: raw.minScore ?? null, + passed: raw.passed ?? null, + }; + const numbers = [score.value, score.baselineValue, score.delta, score.minScore]; + if ( - !isRecord(value) || - typeof value.name !== "string" || - !(value.kind === null || typeof value.kind === "string") || - typeof value.passCount !== "number" || - typeof value.failCount !== "number" + typeof score.name !== "string" || + !(score.kind === null || typeof score.kind === "string") || + typeof score.passCount !== "number" || + typeof score.failCount !== "number" || + !numbers.every((field) => field === null || typeof field === "number") || + !(score.passed === null || typeof score.passed === "boolean") ) { return undefined; } - const nullable = ["value", "baselineValue", "delta", "minScore"] as const; - - if (!nullable.every((key) => value[key] === null || typeof value[key] === "number")) { - return undefined; - } - - if (value.passed !== null && typeof value.passed !== "boolean") return undefined; - - return { - name: value.name, - kind: value.kind, - value: value.value as number | null, - passCount: value.passCount, - failCount: value.failCount, - baselineValue: value.baselineValue as number | null, - delta: value.delta as number | null, - minScore: value.minScore as number | null, - passed: value.passed, - }; + return score as ExperimentScore; } diff --git a/packages/sdk/src/prompts.ts b/packages/sdk/src/prompts.ts index 7b4612d..2787b29 100644 --- a/packages/sdk/src/prompts.ts +++ b/packages/sdk/src/prompts.ts @@ -598,7 +598,6 @@ function keyFingerprint(apiKey: string): string { return Array.from(digest, (byte) => byte.toString(16).padStart(2, "0")).join(""); } -/** The Retry-After of a 429 or 503 response, as seconds or an IMF-fixdate HTTP date, capped. */ function parseResponse(body: unknown): PromptResponse | undefined { if (!isRecord(body)) return undefined; const { name, type, label, version, rollout } = body; diff --git a/packages/sdk/src/scores.ts b/packages/sdk/src/scores.ts index e600c2b..bb06d76 100644 --- a/packages/sdk/src/scores.ts +++ b/packages/sdk/src/scores.ts @@ -51,12 +51,14 @@ const TIMEOUT_MS = 10_000; * from a client that is not initialized is all zeros, and score reports it as `invalid_score`. */ export async function score(options: ScoreOptions): Promise { - const api = resolveApi(options); + // A caller without types can pass no options, and score must not reject then either. + const fields: Partial = isRecord(options) ? options : {}; + const api = resolveApi(fields); try { - const name = typeof options.name === "string" ? options.name.trim() : options.name; - const label = typeof options.label === "string" ? options.label.trim() : options.label; - const problem = checkScore({ ...options, name, label }); + const name = typeof fields.name === "string" ? fields.name.trim() : fields.name; + const label = typeof fields.label === "string" ? fields.label.trim() : fields.label; + const problem = checkScore({ ...fields, name, label }); if (problem) throw new ScoreError("invalid_score", problem); @@ -69,13 +71,13 @@ export async function score(options: ScoreOptions): Promise const body = await request({ ...api, apiKey: api.apiKey }, ScoreError, "POST", "/v1/scores", { body: { - traceId: options.traceId, - spanId: options.spanId, - sessionId: options.sessionId, + traceId: fields.traceId, + spanId: fields.spanId, + sessionId: fields.sessionId, name, - value: options.value, + value: fields.value, label, - comment: options.comment, + comment: fields.comment, }, timeoutMs: TIMEOUT_MS, idempotent: false, @@ -131,8 +133,8 @@ export function checkValue( return undefined; } -function checkScore(options: ScoreOptions): string | undefined { - const { name, traceId, spanId, sessionId, value, label, comment } = options; +function checkScore(fields: Partial): string | undefined { + const { name, traceId, spanId, sessionId, value, label, comment } = fields; if (typeof name !== "string" || name.length === 0 || name.length > MAX_SCORE_NAME_LENGTH) { return `name must be a string of 1 to ${MAX_SCORE_NAME_LENGTH} characters`; diff --git a/packages/sdk/tests/experiments.test.ts b/packages/sdk/tests/experiments.test.ts index daa3214..5f555e3 100644 --- a/packages/sdk/tests/experiments.test.ts +++ b/packages/sdk/tests/experiments.test.ts @@ -21,6 +21,7 @@ interface Request { method: string; url: URL; body: any; + signal: AbortSignal | undefined; } const EXPERIMENT_ID = "6f9619ff-8b86-4011-b42d-00c04fc964ff"; @@ -34,7 +35,7 @@ function datasetItem(number: number, split: string | null = null) { return { id: itemId(number), number, - input: { question: `q${number}` }, + input: { question: `q${number}` } as Record, expectedOutput: `a${number}`, metadata: { source: "test" }, tags: ["golden"], @@ -67,7 +68,7 @@ function fakeApi( items?: ReturnType[]; pageSize?: number; results?: Record[]; - override?: (request: Request) => Response | undefined; + override?: (request: Request) => Response | Promise | undefined; } = {}, ) { const items = options.items ?? [datasetItem(1), datasetItem(2)]; @@ -82,6 +83,7 @@ function fakeApi( method: init?.method ?? "GET", url, body: typeof init?.body === "string" ? JSON.parse(init.body) : undefined, + signal: init?.signal ?? undefined, }; requests.push(request); @@ -381,6 +383,66 @@ test("a task that throws fails only its item", async () => { expect(passed).not.toHaveProperty("error"); }); +test("an output or an error that cannot be encoded does not stop the run", async () => { + const api = fakeApi({ items: [datasetItem(1), datasetItem(2), datasetItem(3)] }); + const cycle: Record = {}; + cycle.self = cycle; + const unreadable = Object.defineProperty(new Error(), "message", { + get() { + throw new Error("no message"); + }, + }); + + const outcome = await experiment(api, { + task: (item) => { + if (item.number === 3) throw unreadable; + + return item.number === 1 ? cycle : 10n; + }, + // A lone surrogate, which UTF-8 cannot hold. + scorers: { tone: () => "\ud83d" }, + }); + + expect(outcome.status).toBe("completed"); + expect( + api + .posts() + .flat() + .sort((a: { itemId: string }, b: { itemId: string }) => a.itemId.localeCompare(b.itemId)) + .map((run: { output: unknown; error?: unknown; scores?: unknown }) => [ + run.output, + run.error, + run.scores, + ]), + ).toEqual([ + [null, undefined, [{ name: "tone", label: "\ud83d" }]], + [null, undefined, [{ name: "tone", label: "\ud83d" }]], + [null, "A value that cannot be converted to a string was thrown", undefined], + ]); + expect(routes(api.requests)).toContain(`POST /v1/experiments/${EXPERIMENT_ID}/complete`); +}); + +test("each run of an item starts from the dataset input, also when a task changes it", async () => { + const api = fakeApi({ + items: [{ ...datasetItem(1), input: { messages: [{ role: "user", content: "Hi" }] } }], + }); + const seen: number[] = []; + + await experiment(api, { + runsPerItem: 3, + concurrency: 1, + task: (item) => { + const messages = item.input.messages as unknown[]; + seen.push(messages.length); + messages.push({ role: "assistant", content: "Hello" }); + + return "ok"; + }, + }); + + expect(seen).toEqual([1, 1, 1]); +}); + test("scorers give pass/fail, numbers, labels, and reasons, and a bad result drops only its score", async () => { const errors: Error[] = []; setup({ environment: "experiment", onError: (error) => errors.push(error) }); @@ -418,14 +480,14 @@ test("scorers give pass/fail, numbers, labels, and reasons, and a bad result dro ]); }); -test("a failed post of items stops new items, never completes, and rejects with the API error", async () => { +test("a failed post of items stops new items, completes the experiment, and rejects with the API error", async () => { const api = fakeApi({ items: Array.from({ length: 300 }, (_, index) => datasetItem(index + 1)), override: (request) => request.url.pathname.endsWith("/items") ? Response.json( - { error: { code: "experiment_ended", message: "support-golden-v3 has ended." } }, - { status: 409 }, + { error: { code: "item_not_found", message: "support-golden v3 has no item" } }, + { status: 422 }, ) : undefined, }); @@ -441,10 +503,35 @@ test("a failed post of items stops new items, never completes, and rejects with }).catch((caught: unknown) => caught); expect(error).toBeInstanceOf(ExperimentError); - expect(error).toMatchObject({ code: "experiment_ended", status: 409 }); + expect(error).toMatchObject({ code: "item_not_found", status: 422 }); // The one worker waits for the post of the first 100, which fails, so no item starts after it. expect(started).toBe(100); - expect(routes(api.requests)).not.toContain(`POST /v1/experiments/${EXPERIMENT_ID}/complete`); + // The complete ends the experiment, which would else stay running. + expect(routes(api.requests).slice(-2)).toEqual([ + `POST /v1/experiments/${EXPERIMENT_ID}/items`, + `POST /v1/experiments/${EXPERIMENT_ID}/complete`, + ]); +}); + +test("a complete that fails after a failed run goes to onError, and the run rejects with the first error", async () => { + const errors: Error[] = []; + setup({ environment: "experiment", onError: (error) => errors.push(error) }); + const api = fakeApi({ + override: (request) => + request.url.pathname.endsWith("/items") || request.url.pathname.endsWith("/complete") + ? Response.json( + { error: { code: "invalid_api_key", message: "The API key was revoked" } }, + { status: 401 }, + ) + : undefined, + }); + + await expect(experiment(api)).rejects.toMatchObject({ + code: "invalid_api_key", + message: "The API key was revoked", + }); + expect(routes(api.requests).filter((route) => route.endsWith("/complete"))).toHaveLength(1); + expect(errors.map((error) => (error as ExperimentError).code)).toEqual(["invalid_api_key"]); }); test("an item post that fails with 500 goes again, and the server skips the runs it has", async () => { @@ -462,18 +549,98 @@ test("an item post that fails with 500 goes again, and the server skips the runs expect(api.posts()[1]).toEqual(api.posts()[0]); }); -test("a create that fails with 500 does not go again, so it cannot start two experiments", async () => { +test("a post of items holds at most 4 MiB of JSON", async () => { + const api = fakeApi({ items: Array.from({ length: 20 }, (_, index) => datasetItem(index + 1)) }); + + // 100,000 characters of 3 UTF-8 bytes: about 300 KB of JSON for each item run. + await experiment(api, { task: () => "中".repeat(100_000) }); + + expect(api.posts().map((items) => items.length)).toEqual([13, 7]); +}); + +test.each([ + ["10,000 items in pages run", 10_000, undefined], + ["10,001 items in pages stop the run", 10_001, "invalid_response"], +])("%s", async (_, count, code) => { const api = fakeApi({ - override: (request) => - request.method === "POST" && request.url.pathname === "/v1/experiments" - ? new Response("upstream", { status: 500 }) - : undefined, + items: Array.from({ length: count }, (_, index) => datasetItem(index + 1)), + pageSize: 5_000, }); - await expect(experiment(api)).rejects.toMatchObject({ code: "http_500", status: 500 }); - expect(routes(api.requests).filter((route) => route === "POST /v1/experiments")).toHaveLength(1); + const outcome = await experiment(api, { concurrency: 20, task: () => "ok" }).catch( + (caught: unknown) => caught, + ); + + expect((outcome as ExperimentError).code).toBe(code); + expect(api.posts().flat()).toHaveLength(10_000); + expect(routes(api.requests).filter((route) => route.endsWith("/complete"))).toHaveLength(1); +}); + +test.each([ + ["a dataset page", "GET /v1/datasets/support-golden", "timeout", 2, undefined], + ["an item post", `POST /v1/experiments/${EXPERIMENT_ID}/items`, "network", 2, undefined], + ["the create", "POST /v1/experiments", "timeout", 1, "timeout"], + ["the create", "POST /v1/experiments", "network", 1, "network_error"], +])("a %s %s, after a %s, sends %i times in all", async (_, route, failure, sends, code) => { + vi.useFakeTimers(); + + try { + let failures = 1; + const api = fakeApi({ + override: (request) => { + if (`${request.method} ${request.url.pathname}` !== route || failures-- <= 0) { + return undefined; + } + + if (failure === "network") return Promise.reject(new TypeError("fetch failed")); + + // A server that never answers: the request ends when its timeout aborts it. + return new Promise((_, reject) => + request.signal!.addEventListener("abort", () => reject(request.signal!.reason)), + ); + }, + }); + + const outcome = experiment(api).catch((caught: unknown) => caught); + await vi.advanceTimersByTimeAsync(60_000); + + expect(((await outcome) as ExperimentError).code).toBe(code); + expect(routes(api.requests).filter((sent) => sent === route)).toHaveLength(sends); + } finally { + vi.useRealTimers(); + } }); +test.each([ + ["500", () => new Response("upstream", { status: 500 }), "http_500"], + // The server answers 503 when its database work times out, which can be after the commit. + [ + "503", + () => + Response.json( + { error: { code: "evals_unavailable", message: "Evals are unavailable right now" } }, + { status: 503 }, + ), + "evals_unavailable", + ], +])( + "a create that fails with %s does not go again, so it cannot start two experiments", + async (status, reply, code) => { + const api = fakeApi({ + override: (request) => + request.method === "POST" && request.url.pathname === "/v1/experiments" + ? reply() + : undefined, + }); + + await expect(experiment(api)).rejects.toMatchObject({ code, status: Number(status) }); + expect(routes(api.requests)).toEqual([ + "GET /v1/datasets/support-golden", + "POST /v1/experiments", + ]); + }, +); + test("a create that gets a 429 goes again after its Retry-After", async () => { let limited = 1; const api = fakeApi({ @@ -505,6 +672,38 @@ test("waits until the server evaluators end the experiment", async () => { ).toHaveLength(2); }); +test("gives up when the experiment does not end in 30 minutes, with the result", async () => { + vi.useFakeTimers(); + + try { + const polls: number[] = []; + const api = fakeApi({ + override: (request) => { + if (request.method === "GET" && request.url.pathname.startsWith("/v1/experiments/")) { + polls.push(Date.now()); + + return Response.json(result({ status: "running", itemsDone: 1 })); + } + + return undefined; + }, + }); + + const outcome = experiment(api).catch((caught: unknown) => caught); + await vi.advanceTimersByTimeAsync(31 * 60_000); + const error = (await outcome) as ExperimentError; + + expect(error.code).toBe("timeout"); + expect(error.result?.status).toBe("running"); + // The last poll is the one whose next wait of 5 seconds would end past the 30 minutes. + const waited = polls.at(-1)! - polls[0]!; + expect(waited).toBeGreaterThan(30 * 60_000 - 5_000); + expect(waited).toBeLessThanOrEqual(30 * 60_000); + } finally { + vi.useRealTimers(); + } +}); + test.each([ [ "the API error", @@ -608,6 +807,35 @@ test.each([ await expect(experiment(api, options)).resolves.toMatchObject({ checkStatus: "failed" }); }); +test("a result score that leaves out its null fields reads them as null", async () => { + const api = fakeApi({ + results: [result({ scores: [{ name: "length", passCount: 0, failCount: 0 }] })], + }); + + const outcome = await experiment(api); + + expect(outcome.scores).toEqual([ + { + name: "length", + kind: null, + value: null, + passCount: 0, + failCount: 0, + baselineValue: null, + delta: null, + minScore: null, + passed: null, + }, + ]); +}); + +test("a result without checkStatus is an invalid response, not a run with no check", async () => { + const { checkStatus: _, ...missing } = failedCheck; + const api = fakeApi({ results: [missing] }); + + await expect(experiment(api)).rejects.toMatchObject({ code: "invalid_response" }); +}); + test("in CI mode an experiment that fails rejects with its error", async () => { vi.stubEnv("CI", "1"); const api = fakeApi({ @@ -666,6 +894,23 @@ test("in GitHub Actions the experiment records the pull request, its head commit expect(readFileSync(results, "utf8")).toBe(`${JSON.stringify(result())}\n`); }); +test("a pull request title is cut to 256 UTF-16 units without half of an emoji", async () => { + const dir = mkdtempSync(join(tmpdir(), "td-experiment-")); + const event = join(dir, "event.json"); + // GitHub counts the emoji as one character, thus the title fits its limit of 256. + writeFileSync(event, JSON.stringify({ pull_request: { title: `${"a".repeat(255)}😀` } })); + vi.stubEnv("CI", "true"); + vi.stubEnv("GITHUB_ACTIONS", "true"); + vi.stubEnv("GITHUB_REPOSITORY", "acme/support"); + vi.stubEnv("GITHUB_SHA", "a1b2c3d"); + vi.stubEnv("GITHUB_EVENT_PATH", event); + const api = fakeApi(); + + await experiment(api); + + expect(api.requests[1]!.body.ci.prTitle).toBe("a".repeat(255)); +}); + test.each([ [ "no dataset", diff --git a/packages/sdk/tests/scores.test.ts b/packages/sdk/tests/scores.test.ts index d740b40..9b9dccc 100644 --- a/packages/sdk/tests/scores.test.ts +++ b/packages/sdk/tests/scores.test.ts @@ -85,6 +85,31 @@ test("posts the score with the key and base URL of init, and resolves with its i }); }); +test("calls the global fetch as a plain function, as Workers and browsers need", async () => { + vi.stubEnv("TELEMETRY_DEV_API_KEY", "td_live_key"); + // A strict function called as a method of an object gets that object as `this`. + vi.stubGlobal("fetch", function (this: unknown) { + if (this !== undefined && this !== globalThis) throw new TypeError("Illegal invocation"); + + return Promise.resolve(Response.json({ id: "score-3" }, { status: 202 })); + }); + + try { + await expect(score({ name: "helpful", traceId: TRACE_ID, value: 1 })).resolves.toBe("score-3"); + } finally { + vi.unstubAllGlobals(); + } +}); + +test("score with no options reports invalid_score and resolves undefined", async () => { + const errors = errorsOf(); + + await expect((score as (options?: ScoreOptions) => Promise)()).resolves.toBeUndefined(); + expect(errors.map((error) => [error.code, error.message])).toEqual([ + ["invalid_score", "name must be a string of 1 to 64 characters"], + ]); +}); + test("a session score needs no trace, and a label goes trimmed", async () => { const { fetch, calls } = fakeFetch(() => Response.json({ id: "score-2" }, { status: 202 })); From bf7844f97df44c6d67aba99cd22dc10209de641d Mon Sep 17 00:00:00 2001 From: ephraimduncan Date: Fri, 30 Oct 2026 00:33:41 +0000 Subject: [PATCH 07/10] fix(sdk-python): complete the experiment after a run stops An output that JSON cannot hold stopped the full run outside the guard of its item. An example is a dict that holds itself. An exception with a __str__ that raises did the same. The SDK sends None as the output, as the TypeScript SDK does, and the name of the type as the error. A lone surrogate in an output or an error becomes U+FFFD. The request body is the JSON of JSON.stringify, which escapes a lone surrogate in a label. When a run stops after run_experiment makes the experiment, it sends the complete request before it raises. This also occurs after Ctrl-C and after a cancellation. An error of that request goes to on_error. An error of an item run stops all workers. The create request tries again only after a 429, as in the TypeScript SDK. Each run of an item gets a deep copy of the item. The requests for experiments, scores, and prompts send the User-Agent telemetry-dev-python/, because Cloudflare rejects Python-urllib. New tests do checks of the 4 MiB batch limit, the 10,000-item limit, the 30-minute wait, and the retry after a timeout or a dropped connection. --- sdks/python/README.md | 34 +- sdks/python/src/telemetry_dev/_api.py | 27 +- sdks/python/src/telemetry_dev/_ci.py | 4 +- sdks/python/src/telemetry_dev/_experiments.py | 110 ++++--- sdks/python/src/telemetry_dev/_prompts.py | 12 +- sdks/python/tests/fake_api.py | 36 ++- sdks/python/tests/test_experiments.py | 304 ++++++++++++++++-- sdks/python/tests/test_scores.py | 2 + 8 files changed, 440 insertions(+), 89 deletions(-) diff --git a/sdks/python/README.md b/sdks/python/README.md index f7696a1..bc53f67 100644 --- a/sdks/python/README.md +++ b/sdks/python/README.md @@ -283,19 +283,21 @@ result = telemetry_dev.run_experiment( name that telemetry.dev picks when you give no `name`. - `task(item)` gets an `ExperimentItem`: `id`, `number`, `input`, `expected_output`, `metadata`, `tags`, `split`, and `run_index`. Its return value is the output. A string stays as it is, and a - different value becomes JSON, cut to 100,000 UTF-16 units. A task that raises fails only its - item, and the item records the type and the message of the exception. + different value becomes JSON, cut to 100,000 UTF-16 units. A value that JSON cannot hold, such + as a dict that holds itself, gives no output. A task that raises fails only its item, and the + item records the type and the message of the exception. - `concurrency` (1 to 100, default 4) threads run the items, each in a copy of the context of the - caller, so `propagate_attributes` around the call applies to each item. `runs_per_item` (1 to 5, - default 1) runs each item more than one time. The pass rate and the mean of each score use all - runs. + caller, so `propagate_attributes` around the call applies to each item. With `runs_per_item` + (default 1, at most 5), each item runs more than one time. Each run gets its own copy of the + item, so a task that changes its input does not change the next run. The pass rate and the mean + of each score use all runs. - `run_experiment_async` takes async tasks and scorers. `concurrency` asyncio tasks run the items, and the requests go in worker threads. A task that is not async runs on the event loop, so use `run_experiment` for it. `run_experiment` raises `TypeError` for an async task or scorer. - Each item run is a trace of its own, with the span `experiment `. The SDK puts `td.experiment.id` and `td.dataset.item_id` on each span of the run, also on the spans of the - provider integrations. Each run also gets its own session id, `::`, as `gen_ai.conversation.id`. + provider integrations. Each run also gets its own session id, + `::`, as `gen_ai.conversation.id`. - Call `init(environment="experiment")` before the run. Pages, metrics, and alerts leave out the `experiment` environment by default. In a different environment, `run_experiment` logs a warning. Without an enabled client, the items record no traces and no cost. @@ -315,15 +317,21 @@ result = telemetry_dev.run_experiment( `dataset_not_found`, `dataset_empty`, `evaluator_not_found`, `insufficient_scope`, `missing_api_key`, or `timeout`. A read of the dataset or the result, a post of outputs, and the completion go again up to 2 times after a timeout, a network error, a 429, or a 5xx. The server - skips an item run that it has. The request that makes the experiment goes again only after a 429 - or a 503, thus a lost response cannot make a second experiment. If the run stops before the end, - the experiment does not complete in telemetry.dev. + skips an item run that it has. The request that makes the experiment goes again only after a + 429, thus a lost response or a 503 cannot make a second experiment. When the run stops after it + made the experiment, also after Ctrl-C or a cancellation, `run_experiment` sends the completion + before it raises: telemetry.dev then ends the experiment with the item runs that it got, or fails + it when it got none. - The key needs the `datasets:read` and `experiments:write` scopes. `run_experiment` uses the same - API key and base URL as `get_prompt`, or its own `api_key` and `base_url`. + API key and base URL as `get_prompt`, or its own `api_key` and `base_url`. With + `environment="experiment"`, the key of the telemetry client must be for all environments or for + `experiment`: a key for another environment cannot send the traces of the items, and the items + then have no trace and no cost. The outputs, the exceptions of failed tasks, the scorer reasons, and `metadata` go to telemetry.dev -also when `capture_output` is off, because they are the result of the experiment. The capture -options and `mask` apply only to the spans of the run. +and stay with the experiment, because they are its result. `capture_output`, `mask`, and the +capture setting of the environment in the project do not apply to them. They apply only to the +spans of the run. ### CI mode diff --git a/sdks/python/src/telemetry_dev/_api.py b/sdks/python/src/telemetry_dev/_api.py index 4f8e042..5dc3065 100644 --- a/sdks/python/src/telemetry_dev/_api.py +++ b/sdks/python/src/telemetry_dev/_api.py @@ -17,7 +17,8 @@ from typing import IO, Any, Literal, TypeVar, cast from ._client import get_client -from ._config import resolve_config +from ._config import SDK_VERSION, resolve_config +from ._serialize import json_dumps # A longer Retry-After waits only this long: the API counts requests by the minute. _MAX_RETRY_AFTER_S = 60.0 @@ -71,18 +72,20 @@ def request( A response that is not 2xx raises the ``{"error": {"code", "message"}}`` of its body, or ``http_``. With ``idempotent``, the server takes a repeat at most once, so a timeout, a - network error, or a 5xx tries again. Other requests try again only after a 429 or a 503: the - server did not take the request then. A retry waits for the Retry-After of the response, or - for 1 and then 2 seconds. The timeout applies to each socket operation of urllib. + network error, or a 5xx tries again. Other requests try again only after a 429, which the rate + limit answers before the server does any work: a 503 can come after the work committed. A + retry waits for the Retry-After of the response, or for 1 and then 2 seconds. The timeout + applies to each socket operation of urllib. """ url = f"{api.base_url}{path}" params = {key: value for key, value in (query or {}).items() if value is not None} if params: url = f"{url}?{urllib.parse.urlencode(params)}" - headers = {"Authorization": f"Bearer {api.api_key}", "X-Telemetry-Dev-SDK": "telemetry-dev"} + headers = auth_headers(api.api_key) data = None if body is not None: - data = json.dumps(body, ensure_ascii=False, allow_nan=False).encode() + # The JSON of JSON.stringify, which escapes a lone surrogate that UTF-8 cannot hold. + data = json_dumps(body, default=None).encode() headers["Content-Type"] = "application/json" attempt = 1 while True: @@ -95,6 +98,16 @@ def request( attempt += 1 +def auth_headers(api_key: str | None) -> dict[str, str]: + return { + "Authorization": f"Bearer {api_key}", + "X-Telemetry-Dev-SDK": "telemetry-dev", + # urllib's default User-Agent, Python-urllib/3.x, is refused by bot filters such as + # Cloudflare's Browser Integrity Check. + "User-Agent": f"telemetry-dev-python/{SDK_VERSION}", + } + + class NoRedirect(urllib.request.HTTPRedirectHandler): # A redirect to another host must not get the API key, so a 3xx fails like any status that is # not 2xx. @@ -185,6 +198,6 @@ def _send( def _retryable(status: int | None, idempotent: bool) -> bool: - if status in (429, 503): + if status == 429: return True return idempotent and (status is None or status >= 500) diff --git a/sdks/python/src/telemetry_dev/_ci.py b/sdks/python/src/telemetry_dev/_ci.py index 4b6cd00..e27701f 100644 --- a/sdks/python/src/telemetry_dev/_ci.py +++ b/sdks/python/src/telemetry_dev/_ci.py @@ -127,7 +127,9 @@ def _https_url(url: str) -> str | None: def _cut(text: str, max_units: int) -> str: - """``text`` cut to ``max_units`` UTF-16 units, the length that the API counts.""" + """``text`` cut to ``max_units`` UTF-16 units, the length that the API counts, with no half of + a surrogate pair at the end: the API stores the run as jsonb, and Postgres rejects a lone + surrogate there.""" encoded = text.encode("utf-16-le", "surrogatepass") if len(encoded) <= max_units * 2: return text diff --git a/sdks/python/src/telemetry_dev/_experiments.py b/sdks/python/src/telemetry_dev/_experiments.py index b4e031d..303dc89 100644 --- a/sdks/python/src/telemetry_dev/_experiments.py +++ b/sdks/python/src/telemetry_dev/_experiments.py @@ -4,8 +4,8 @@ import asyncio import contextvars +import copy import inspect -import json import math import os import threading @@ -184,9 +184,11 @@ def run_experiment( them and wins for the same name. ``runs_per_item`` is 1 to 5. The run cannot start without its dataset, so run_experiment raises ``ExperimentError`` when a - request fails. A task that raises fails only its item. In CI mode (``ci``, default on when - ``CI`` is ``true`` or ``1``), it records the GitHub Actions run, and it raises when the check - fails or the experiment does not complete, so the process exits with an error. + request fails. When the run stops after it made the experiment, it first sends the complete + request, so that telemetry.dev ends the experiment with the item runs that it has. A task that + raises fails only its item. In CI mode (``ci``, default on when ``CI`` is ``true`` or ``1``), + it records the GitHub Actions run, and it raises when the check fails or the experiment does + not complete, so the process exits with an error. """ run, api = _prepare( task, @@ -206,7 +208,12 @@ def run_experiment( asynchronous=False, ) runs, created = _start(api, run) - _run_items(api, run, created, runs) + try: + _announce(run, created) + _run_items(api, run, created, runs) + except BaseException: + _complete_after_failure(api, created) + raise return _finish(api, run, created) @@ -248,7 +255,12 @@ async def run_experiment_async( asynchronous=True, ) runs, created = await asyncio.to_thread(_start, api, run) - await _run_items_async(api, run, created, runs) + try: + _announce(run, created) + await _run_items_async(api, run, created, runs) + except BaseException: + await asyncio.to_thread(_complete_after_failure, api, created) + raise return await asyncio.to_thread(_finish, api, run, created) @@ -464,13 +476,25 @@ def _start(api: Api, run: _Run) -> tuple[_Runs, _Created]: idempotent=False, ) ) + return _Runs(api, run, first), created + + +def _announce(run: _Run, created: _Created) -> None: each = f", {run.runs_per_item} runs each" if run.runs_per_item > 1 else "" print( f"Experiment {created.name}: {created.items_total // run.runs_per_item} items{each}. " f"{created.url}", flush=True, ) - return _Runs(api, run, first), created + + +def _complete_after_failure(api: Api, created: _Created) -> None: + """Without the complete, the experiment would stay running: the server ends it with the item + runs that it has, or fails it when it has none.""" + try: + _complete(api, created) + except Exception as exc: + report_error(api.on_error, "completing the experiment after a failed run failed", exc) class _Runs: @@ -500,8 +524,10 @@ def _item_runs(api: Api, run: _Run, first: _Page) -> Generator[ExperimentItem]: "invalid_response", f"telemetry.dev returned more than {_MAX_DATASET_ITEMS} items" ) for item in page.items: + # A copy for each run, so that a task that changes its input does not change the next + # run. for run_index in range(run.runs_per_item): - yield ExperimentItem(**vars(item), run_index=run_index) + yield ExperimentItem(**copy.deepcopy(vars(item)), run_index=run_index) if page.next_cursor is None: return page = _fetch_page(api, run, page.version, page.next_cursor) @@ -543,7 +569,7 @@ def __init__(self, post: Callable[[list[dict[str, Any]]], object]) -> None: self._lock = threading.Lock() def add(self, result: dict[str, Any]) -> None: - size = len(json.dumps(result, ensure_ascii=False).encode()) + size = len(json_dumps(result, default=None).encode()) with self._lock: if self._pending and self._bytes + size > _MAX_BATCH_BYTES: self._send() @@ -582,8 +608,8 @@ def _batch(api: Api, created: _Created) -> _Batch: def _run_items(api: Api, run: _Run, created: _Created, runs: _Runs) -> None: """Run the items, ``concurrency`` at a time, and post each result. A post or a page fetch - that fails stops the run: no new item starts, and run_experiment raises when the running - items end.""" + that fails stops the run: no new item starts, and _run_items raises when the running items + end.""" batch = _batch(api, created) failures: list[BaseException] = [] caller = contextvars.copy_context() @@ -592,15 +618,10 @@ def worker() -> None: while not failures: try: item = runs.next() - except Exception as exc: - failures.append(exc) - return - if item is None: - return - # A copy for each item: one Context cannot run in two threads at the same time. - result = caller.copy().run(_run_item, run, created, item, api.on_error) - try: - batch.add(result) + if item is None: + return + # A copy for each item: one Context cannot run in two threads at the same time. + batch.add(caller.copy().run(_run_item, run, created, item, api.on_error)) except Exception as exc: failures.append(exc) @@ -624,13 +645,9 @@ async def worker() -> None: while not failures: try: item = await asyncio.to_thread(runs.next) - except Exception as exc: - failures.append(exc) - return - if item is None: - return - result = await _run_item_async(run, created, item, api.on_error) - try: + if item is None: + return + result = await _run_item_async(run, created, item, api.on_error) await asyncio.to_thread(batch.add, result) except Exception as exc: failures.append(exc) @@ -840,14 +857,29 @@ def _pass_fail(passed: bool) -> dict[str, Any]: def _output_text(output: Any) -> str | None: + """The output as text. An output that JSON cannot hold, such as a dict that holds itself, is + None.""" if output is None: return None - text = output if isinstance(output, str) else json_dumps(output) - return truncate(text, _MAX_OUTPUT_LENGTH) + try: + text = output if isinstance(output, str) else json_dumps(output) + except Exception: + return None + return truncate(_well_formed(text), _MAX_OUTPUT_LENGTH) def _error_text(error: BaseException) -> str: - return f"{type(error).__name__}: {error}" + try: + text = f"{type(error).__name__}: {error}" + except Exception: + # A __str__ that raises. + return type(error).__name__ + return _well_formed(text) + + +def _well_formed(text: str) -> str: + # truncate fails on a lone surrogate, which a str can hold, so each one becomes U+FFFD. + return text.encode("utf-16-le", "surrogatepass").decode("utf-16-le", "replace") def _close(awaitable: object) -> None: @@ -860,6 +892,16 @@ def _close(awaitable: object) -> None: def _finish(api: Api, run: _Run, created: _Created) -> ExperimentResult: # The spans of the items reach ingest before the server reads their cost. flush() + _complete(api, created) + result, body = _wait_for_result(api, created.id) + write_result(os.environ, body, api.on_error) + print(_describe(result), flush=True) + if run.in_ci: + _check_result(result) + return result + + +def _complete(api: Api, created: _Created) -> None: request( api, ExperimentError, @@ -868,12 +910,6 @@ def _finish(api: Api, run: _Run, created: _Created) -> ExperimentResult: timeout_s=_REQUEST_TIMEOUT_S, idempotent=True, ) - result, body = _wait_for_result(api, created.id) - write_result(os.environ, body, api.on_error) - print(_describe(result), flush=True) - if run.in_ci: - _check_result(result) - return result def _wait_for_result(api: Api, experiment_id: str) -> tuple[ExperimentResult, object]: @@ -1035,6 +1071,7 @@ def _parse_result(body: object) -> ExperimentResult: if isinstance(raw_scores, list) else None ) + # Required: it decides the result of CI mode, so a missing one must not read as no check. check_status = fields.get("checkStatus", "missing") items_failed = fields.get("itemsFailed", 0) error = fields.get("error") @@ -1067,6 +1104,7 @@ def _parse_result(body: object) -> ExperimentResult: def _parse_score(raw: object) -> ExperimentScore | None: + """A score of the result. A field that can be None can also be missing.""" if not isinstance(raw, dict): return None score = cast("dict[str, Any]", raw) diff --git a/sdks/python/src/telemetry_dev/_prompts.py b/sdks/python/src/telemetry_dev/_prompts.py index a7f3aff..4408852 100644 --- a/sdks/python/src/telemetry_dev/_prompts.py +++ b/sdks/python/src/telemetry_dev/_prompts.py @@ -24,9 +24,9 @@ from opentelemetry import context as otel_context from opentelemetry.context import Context -from ._api import NoRedirect, api_error, resolve_api, retry_after +from ._api import NoRedirect, api_error, auth_headers, resolve_api, retry_after from ._client import Client, get_client -from ._config import SDK_VERSION, report_error +from ._config import report_error from ._context import propagated_attributes from ._semconv import ( ATTR_PROMPT_LABEL, @@ -332,13 +332,7 @@ def _get_prompt( cache_ttl_seconds, lambda: _fetch( f"{url}/v1/prompts?{urllib.parse.urlencode(query)}", - { - "Authorization": f"Bearer {key}", - "X-Telemetry-Dev-SDK": "telemetry-dev", - # urllib's default User-Agent, Python-urllib/3.x, is refused by bot filters - # such as Cloudflare's Browser Integrity Check. - "User-Agent": f"telemetry-dev-python/{SDK_VERSION}", - }, + auth_headers(key), ), on_error, ) diff --git a/sdks/python/tests/fake_api.py b/sdks/python/tests/fake_api.py index 7df2e7a..d86f4cd 100644 --- a/sdks/python/tests/fake_api.py +++ b/sdks/python/tests/fake_api.py @@ -4,7 +4,9 @@ from __future__ import annotations import json +import sys import threading +import time from collections.abc import Callable, Generator from contextlib import contextmanager from dataclasses import dataclass, field @@ -66,13 +68,26 @@ def route(self) -> str: Reply = tuple[int, object, dict[str, str]] +@dataclass(frozen=True) +class Drop: + """Closes the connection with no answer, as a network failure does.""" + + +@dataclass(frozen=True) +class Late: + """Answers after a wait, as a server that is slower than the timeout of the client.""" + + seconds: float + reply: Reply + + @dataclass class FakeApi: url: str items: list[dict[str, Any]] = field(default_factory=lambda: [dataset_item(1), dataset_item(2)]) page_size: int = 100 results: list[dict[str, Any]] = field(default_factory=lambda: [result()]) - override: Callable[[ApiRequest], Reply | None] | None = None + override: Callable[[ApiRequest], Reply | Drop | Late | None] | None = None requests: list[ApiRequest] = field(default_factory=lambda: []) stored: set[str] = field(default_factory=lambda: set[str]()) lock: threading.Lock = field(default_factory=threading.Lock) @@ -85,7 +100,7 @@ def posts(self) -> list[list[dict[str, Any]]]: request.body["items"] for request in self.requests if request.path.endswith("/items") ] - def answer(self, request: ApiRequest) -> Reply: + def answer(self, request: ApiRequest) -> Reply | Drop | Late: with self.lock: self.requests.append(request) overridden = self.override(request) if self.override else None @@ -157,7 +172,7 @@ def _answer(self) -> None: length = int(self.headers.get("content-length") or 0) raw = self.rfile.read(length) if length else b"" target = urlsplit(self.path) - status, body, headers = api.answer( + reply = api.answer( ApiRequest( method=self.command, path=target.path, @@ -166,6 +181,13 @@ def _answer(self) -> None: body=json.loads(raw) if raw else None, ) ) + if isinstance(reply, Drop): + self.close_connection = True + return + if isinstance(reply, Late): + time.sleep(reply.seconds) + reply = reply.reply + status, body, headers = reply data = body if isinstance(body, bytes) else json.dumps(body).encode() self.send_response(status) self.send_header("content-type", "application/json") @@ -178,7 +200,13 @@ def _answer(self) -> None: def log_message(self, format: str, *args: object) -> None: pass - server = ThreadingHTTPServer(("127.0.0.1", 0), Handler) + class Server(ThreadingHTTPServer): + def handle_error(self, request: Any, client_address: Any) -> None: + # The answer of a Late reply can go to a client that has stopped waiting. + if not isinstance(sys.exc_info()[1], (BrokenPipeError, ConnectionResetError)): + super().handle_error(request, client_address) + + server = Server(("127.0.0.1", 0), Handler) thread = threading.Thread( target=server.serve_forever, kwargs={"poll_interval": 0.01}, daemon=True ) diff --git a/sdks/python/tests/test_experiments.py b/sdks/python/tests/test_experiments.py index a0b1807..8668f3c 100644 --- a/sdks/python/tests/test_experiments.py +++ b/sdks/python/tests/test_experiments.py @@ -10,6 +10,8 @@ import pytest +import telemetry_dev +import telemetry_dev._experiments as experiments_module from telemetry_dev import ( ExperimentError, ExperimentItem, @@ -28,7 +30,9 @@ EXPERIMENT_ID, EXPERIMENT_URL, ApiRequest, + Drop, FakeApi, + Late, Reply, dataset_item, fake_api, @@ -114,6 +118,10 @@ def test_runs_each_item_of_each_page_posts_batches_of_100_on_one_version_then_co } assert api.requests[1].headers["authorization"] == "Bearer td_live_test" assert api.requests[1].headers["x-telemetry-dev-sdk"] == "telemetry-dev" + # Cloudflare refuses the default Python-urllib User-Agent. + assert api.requests[1].headers["user-agent"] == ( + f"telemetry-dev-python/{telemetry_dev.__version__}" + ) posts = api.posts() assert [len(items) for items in posts] == [100, 50] @@ -293,7 +301,32 @@ def task(item: ExperimentItem) -> str: assert output.endswith("...[truncated]") -def test_a_failed_post_of_items_stops_new_items_never_completes_and_raises() -> None: +def run_both(api: FakeApi, asynchronous: bool, **options: Any) -> ExperimentResult: + """``experiment`` through run_experiment or run_experiment_async.""" + if not asynchronous: + return experiment(api, **options) + arguments: dict[str, Any] = { + "api_key": "td_live_test", + "base_url": api.url, + "dataset": "support-golden", + "task": answer, + } + arguments.update(options) + return asyncio.run(run_experiment_async(**arguments)) + + +def unknown_item(request: ApiRequest) -> Reply | None: + if request.path.endswith("/items"): + return answer_with( + 422, {"error": {"code": "item_not_found", "message": "support-golden v3 has no item"}} + ) + return None + + +@pytest.mark.parametrize("asynchronous", [False, True], ids=["sync", "async"]) +def test_a_failed_post_of_items_stops_new_items_completes_the_experiment_and_raises( + asynchronous: bool, +) -> None: started = 0 def task(item: ExperimentItem) -> str: @@ -301,22 +334,167 @@ def task(item: ExperimentItem) -> str: started += 1 return "ok" - def ended(request: ApiRequest) -> Reply | None: - if request.path.endswith("/items"): + with fake_api(items=[dataset_item(n) for n in range(1, 301)], override=unknown_item) as api: + with pytest.raises(ExperimentError) as caught: + run_both(api, asynchronous, concurrency=1, task=task) + + assert (caught.value.code, caught.value.status) == ("item_not_found", 422) + # The one worker waits for the post of the first 100, which fails, so no item starts after. + assert started == 100 + # The complete ends the experiment, which would else stay running. + assert api.routes()[-2:] == [ + f"POST /v1/experiments/{EXPERIMENT_ID}/items", + f"POST /v1/experiments/{EXPERIMENT_ID}/complete", + ] + + +def test_a_complete_that_fails_after_a_failed_run_goes_to_on_error(make: MakeClient) -> None: + errors: list[BaseException] = [] + make(environment="experiment", on_error=errors.append) + + def revoked(request: ApiRequest) -> Reply | None: + if request.path.endswith(("/items", "/complete")): return answer_with( - 409, - {"error": {"code": "experiment_ended", "message": "support-golden-v3 has ended."}}, + 401, {"error": {"code": "invalid_api_key", "message": "The API key was revoked"}} ) return None - with fake_api(items=[dataset_item(n) for n in range(1, 301)], override=ended) as api: + with fake_api(override=revoked) as api: with pytest.raises(ExperimentError) as caught: - experiment(api, concurrency=1, task=task) + experiment(api) - assert (caught.value.code, caught.value.status) == ("experiment_ended", 409) - # The one worker waits for the post of the first 100, which fails, so no item starts after. - assert started == 100 - assert f"POST /v1/experiments/{EXPERIMENT_ID}/complete" not in api.routes() + assert (caught.value.code, str(caught.value)) == ("invalid_api_key", "The API key was revoked") + assert api.routes().count(f"POST /v1/experiments/{EXPERIMENT_ID}/complete") == 1 + assert [getattr(error, "code", None) for error in errors] == ["invalid_api_key"] + + +def test_an_interrupted_run_completes_the_experiment(api: FakeApi) -> None: + def task(item: ExperimentItem) -> str: + raise KeyboardInterrupt + + with pytest.raises(KeyboardInterrupt): + experiment(api, task=task) + + assert api.routes()[-1] == f"POST /v1/experiments/{EXPERIMENT_ID}/complete" + + +class UnreadableError(Exception): + def __str__(self) -> str: + raise RuntimeError("no message") + + +@pytest.mark.parametrize("asynchronous", [False, True], ids=["sync", "async"]) +def test_an_output_or_an_error_that_cannot_be_encoded_does_not_stop_the_run( + asynchronous: bool, +) -> None: + cycle: dict[str, Any] = {} + cycle["self"] = cycle + + def task(item: ExperimentItem) -> object: + if item.number == 3: + raise UnreadableError + # A lone surrogate, which UTF-8 cannot hold, becomes U+FFFD. + return cycle if item.number == 1 else "\ud83d" + + def tone(output: Any, item: ExperimentItem) -> str: + return "\ud83d" + + with fake_api(items=[dataset_item(1), dataset_item(2), dataset_item(3)]) as api: + outcome = run_both(api, asynchronous, task=task, scorers={"tone": tone}) + + assert outcome.status == "completed" + posted = sorted((run for items in api.posts() for run in items), key=lambda run: run["itemId"]) + tones = [{"name": "tone", "label": "\ud83d"}] + assert [(run["output"], run.get("error"), run.get("scores")) for run in posted] == [ + (None, None, tones), + ("\ufffd", None, tones), + (None, "UnreadableError", None), + ] + assert f"POST /v1/experiments/{EXPERIMENT_ID}/complete" in api.routes() + + +def test_each_run_of_an_item_starts_from_the_dataset_input() -> None: + seen: list[int] = [] + + def task(item: ExperimentItem) -> str: + messages = item.input["messages"] + seen.append(len(messages)) + messages.append({"role": "assistant", "content": "Hello"}) + return "ok" + + item = {**dataset_item(1), "input": {"messages": [{"role": "user", "content": "Hi"}]}} + with fake_api(items=[item]) as api: + experiment(api, runs_per_item=3, concurrency=1, task=task) + + assert seen == [1, 1, 1] + + +def test_a_post_of_items_holds_at_most_4_mib_of_json() -> None: + def task(item: ExperimentItem) -> str: + # 100,000 characters of 3 UTF-8 bytes: about 300 KB of JSON for each item run. + return "\u4e2d" * 100_000 + + with fake_api(items=[dataset_item(number) for number in range(1, 21)]) as api: + experiment(api, task=task) + + assert [len(items) for items in api.posts()] == [13, 7] + + +@pytest.mark.parametrize( + ("count", "code"), + [(10_000, None), (10_001, "invalid_response")], + ids=["10,000 items in pages run", "10,001 items in pages stop the run"], +) +def test_a_dataset_has_at_most_10000_items(count: int, code: str | None) -> None: + items = [dataset_item(number) for number in range(1, count + 1)] + with fake_api(items=items, page_size=5_000) as api: + try: + experiment(api, concurrency=20) + error = None + except ExperimentError as exc: + error = exc.code + + assert error == code + assert len([run for items in api.posts() for run in items]) == 10_000 + assert api.routes().count(f"POST /v1/experiments/{EXPERIMENT_ID}/complete") == 1 + + +@pytest.mark.parametrize( + ("route", "failure", "sends", "code"), + [ + ("GET /v1/datasets/support-golden", "timeout", 2, None), + (f"POST /v1/experiments/{EXPERIMENT_ID}/items", "network", 2, None), + ("POST /v1/experiments", "timeout", 1, "timeout"), + ("POST /v1/experiments", "network", 1, "network_error"), + ], + ids=[ + "a timeout of a dataset page goes again", + "a network error of an item post goes again", + "a timeout of the create does not go again", + "a network error of the create does not go again", + ], +) +def test_a_request_after_a_timeout_or_a_network_error( + monkeypatch: pytest.MonkeyPatch, route: str, failure: str, sends: int, code: str | None +) -> None: + monkeypatch.setattr(experiments_module, "_REQUEST_TIMEOUT_S", 0.2) + failures = [1] + + def fail_once(request: ApiRequest) -> Reply | Drop | Late | None: + if request.route != route or not failures: + return None + failures.pop() + return Drop() if failure == "network" else Late(1.0, answer_with(500, {})) + + with fake_api(override=fail_once) as api: + try: + experiment(api) + error = None + except ExperimentError as exc: + error = exc.code + + assert error == code + assert api.routes().count(route) == sends def test_an_item_post_that_fails_with_500_goes_again() -> None: @@ -335,16 +513,31 @@ def flaky(request: ApiRequest) -> Reply | None: assert api.posts()[1] == api.posts()[0] -def test_a_create_that_fails_with_500_does_not_go_again() -> None: +@pytest.mark.parametrize( + ("reply", "code"), + [ + (answer_with(500, {}), "http_500"), + # The server answers 503 when its database work times out, which can be after the commit. + ( + answer_with( + 503, + {"error": {"code": "evals_unavailable", "message": "Evals are unavailable"}}, + ), + "evals_unavailable", + ), + ], + ids=["500", "503"], +) +def test_a_create_that_fails_does_not_go_again(reply: Reply, code: str) -> None: def broken(request: ApiRequest) -> Reply | None: - return answer_with(500, {}) if request.route == "POST /v1/experiments" else None + return reply if request.route == "POST /v1/experiments" else None with fake_api(override=broken) as api: with pytest.raises(ExperimentError) as caught: experiment(api) - assert (caught.value.code, caught.value.status) == ("http_500", 500) - assert api.routes().count("POST /v1/experiments") == 1 + assert (caught.value.code, caught.value.status) == (code, reply[0]) + assert api.routes() == ["GET /v1/datasets/support-golden", "POST /v1/experiments"] def test_a_create_that_gets_a_429_goes_again_after_its_retry_after() -> None: @@ -377,6 +570,41 @@ def test_waits_until_the_server_evaluators_end_the_experiment() -> None: assert api.routes().count(f"GET /v1/experiments/{EXPERIMENT_ID}") == 2 +class FakeClock: + def __init__(self) -> None: + self.now = 0.0 + + def monotonic(self) -> float: + return self.now + + def sleep(self, seconds: float) -> None: + self.now += seconds + + +def test_gives_up_when_the_experiment_does_not_end_in_30_minutes( + monkeypatch: pytest.MonkeyPatch, +) -> None: + clock = FakeClock() + monkeypatch.setattr(experiments_module, "time", clock) + polls: list[float] = [] + + def running(request: ApiRequest) -> Reply | None: + if request.route == f"GET /v1/experiments/{EXPERIMENT_ID}": + polls.append(clock.now) + return answer_with(200, result(status="running", itemsDone=1)) + return None + + with fake_api(override=running) as api: + with pytest.raises(ExperimentError) as caught: + experiment(api) + + assert caught.value.code == "timeout" + assert caught.value.result is not None + assert caught.value.result.status == "running" + # The last poll is the one whose next wait of 5 seconds would end past the 30 minutes. + assert 30 * 60 - 5 < polls[-1] - polls[0] <= 30 * 60 + + @pytest.mark.parametrize( ("status", "body", "code"), [ @@ -495,6 +723,35 @@ def test_a_failed_check_returns_when( assert experiment(api, **options).check_status == "failed" +def test_a_result_score_that_leaves_out_its_null_fields_reads_them_as_none() -> None: + scores = [{"name": "length", "passCount": 0, "failCount": 0}] + with fake_api(results=[result(scores=scores)]) as api: + outcome = experiment(api) + + assert outcome.scores == ( + ExperimentScore( + name="length", + kind=None, + value=None, + pass_count=0, + fail_count=0, + baseline_value=None, + delta=None, + min_score=None, + passed=None, + ), + ) + + +def test_a_result_without_check_status_is_an_invalid_response() -> None: + missing = {key: value for key, value in FAILED_CHECK.items() if key != "checkStatus"} + with fake_api(results=[missing]) as api: + with pytest.raises(ExperimentError) as caught: + experiment(api) + + assert caught.value.code == "invalid_response" + + def test_in_ci_mode_an_experiment_that_fails_raises_with_its_error( monkeypatch: pytest.MonkeyPatch, ) -> None: @@ -561,11 +818,20 @@ def test_in_github_actions_it_records_the_pull_request_its_head_commit_and_the_r assert results.stat().st_mode & 0o777 == 0o600 -def test_a_pull_request_title_is_cut_to_256_utf16_units( - monkeypatch: pytest.MonkeyPatch, tmp_path: Path, api: FakeApi +@pytest.mark.parametrize( + ("title", "sent"), + [ + ("\U0001f600" * 200, "\U0001f600" * 128), + # GitHub counts the emoji as one character, thus the title fits its limit of 256. + ("a" * 255 + "\U0001f600", "a" * 255), + ], + ids=["200 emoji", "255 letters and an emoji"], +) +def test_a_pull_request_title_is_cut_to_256_utf16_units_without_half_of_an_emoji( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path, api: FakeApi, title: str, sent: str ) -> None: event = tmp_path / "event.json" - event.write_text(json.dumps({"pull_request": {"title": "\U0001f600" * 200}})) + event.write_text(json.dumps({"pull_request": {"title": title}})) for name, value in { "CI": "true", "GITHUB_ACTIONS": "true", @@ -577,7 +843,7 @@ def test_a_pull_request_title_is_cut_to_256_utf16_units( experiment(api) - assert api.requests[1].body["ci"]["prTitle"] == "\U0001f600" * 128 + assert api.requests[1].body["ci"]["prTitle"] == sent def one(output: Any, item: ExperimentItem) -> int: diff --git a/sdks/python/tests/test_scores.py b/sdks/python/tests/test_scores.py index 44d28de..d72d0d2 100644 --- a/sdks/python/tests/test_scores.py +++ b/sdks/python/tests/test_scores.py @@ -5,6 +5,7 @@ import pytest +import telemetry_dev from telemetry_dev import ScoreError, score, score_async from .conftest import MakeClient @@ -54,6 +55,7 @@ def test_posts_the_score_with_the_key_and_base_url_of_init_and_returns_its_id( assert request.route == "POST /v1/scores" assert request.headers["authorization"] == "Bearer td_live_key" assert request.headers["x-telemetry-dev-sdk"] == "telemetry-dev" + assert request.headers["user-agent"] == f"telemetry-dev-python/{telemetry_dev.__version__}" assert request.headers["content-type"] == "application/json" assert request.body == { "name": "user_feedback", From ae0ed556f68d1952cbe33ca8f93f29a429fab5a4 Mon Sep 17 00:00:00 2001 From: ephraimduncan Date: Fri, 30 Oct 2026 00:33:42 +0000 Subject: [PATCH 08/10] fix(sdk): use the tsx of the project and change only our comment The action ran a .ts script with npx --yes tsx. npx then downloads the newest tsx from npm, in a step that has the API key. The action uses the tsx in node_modules/.bin of the working directory or of a directory above it. If it finds no tsx, it stops with a clear error. report.mjs used each comment that started with its marker, thus it tried to change a pasted copy. It changes only a comment of its token. For a user token, that is a comment of its login. The GITHUB_TOKEN and app tokens cannot read /user, and they write as a bot. The README tells that the key must be for all environments or for experiment. It tells that the capture settings do not apply to item outputs, and that a pull request from a fork gets no API key. The example does not run the job for those pull requests. --- actions/experiment/README.md | 51 +++++--- actions/experiment/action.yml | 2 +- actions/experiment/report.mjs | 11 +- actions/experiment/run.sh | 19 ++- packages/sdk/tests/experiment-action.test.ts | 126 ++++++++++++++++--- 5 files changed, 176 insertions(+), 33 deletions(-) diff --git a/actions/experiment/README.md b/actions/experiment/README.md index 77f5a72..7b851fd 100644 --- a/actions/experiment/README.md +++ b/actions/experiment/README.md @@ -14,6 +14,9 @@ permissions: jobs: evals: + # A pull request from a fork gets no secrets, so the job runs only for branches of this + # repository. + if: github.event.pull_request.head.repo.full_name == github.repository runs-on: ubuntu-latest env: TELEMETRY_DEV_API_KEY: ${{ secrets.TELEMETRY_DEV_API_KEY }} @@ -22,6 +25,7 @@ jobs: - uses: actions/setup-node@v5 with: node-version: 24 + # Installs tsx too, which runs the .ts script. - run: npm ci - uses: telemetry-dev/telemetry/actions/experiment@main with: @@ -49,22 +53,32 @@ changes. ## Inputs -| Input | Description | -| ------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| `script` | The eval script. A `.ts`, `.mts`, `.cts`, or `.tsx` script runs with `npx --yes tsx`, a `.js`, `.mjs`, or `.cjs` script with `node`, and a `.py` script with the `python` (or `python3`) of the PATH. Give `script` or `command`. | -| `command` | A shell command that runs the eval script, for example `uv run python evals/reply_eval.py` or `pnpm eval`. | -| `dataset` | The dataset to run on. The action puts it in `TELEMETRY_DEV_EXPERIMENT_DATASET`. When the script also names a dataset, the two must be the same. | -| `min-scores` | Pass marks, as `name=value` pairs that commas or new lines separate. A pass rate is 0 to 1, and a mean is on the scale of its score. The action puts them in `TELEMETRY_DEV_EXPERIMENT_MIN_SCORES`, and they win over the `minScores` of the script for the same name. | -| `api-key` | A telemetry.dev API key with the `datasets:read` and `experiments:write` scopes. Default: the `TELEMETRY_DEV_API_KEY` variable of the job. | -| `base-url` | The telemetry.dev API. Default: `https://ingest.telemetry.dev`. | -| `working-directory` | The directory that the script or the command runs in. Default: `.`. | -| `comment` | Set to `false` to write the job summary only. | -| `github-token` | The token that writes the comment. Default: `github.token`, which needs the `pull-requests: write` permission. | +| Input | Description | +| ------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `script` | The eval script. A `.ts`, `.mts`, `.cts`, or `.tsx` script runs with the `tsx` of the project, from `node_modules/.bin` in the working directory or a directory above it, a `.js`, `.mjs`, or `.cjs` script with `node`, and a `.py` script with the `python` (or `python3`) of the PATH. Give `script` or `command`. | +| `command` | A shell command that runs the eval script, for example `uv run python evals/reply_eval.py` or `pnpm eval`. | +| `dataset` | The dataset to run on. The action puts it in `TELEMETRY_DEV_EXPERIMENT_DATASET`. When the script also names a dataset, the two must be the same. | +| `min-scores` | Pass marks, as `name=value` pairs that commas or new lines separate. A pass rate is 0 to 1, and a mean is on the scale of its score. The action puts them in `TELEMETRY_DEV_EXPERIMENT_MIN_SCORES`, and they win over the `minScores` of the script for the same name. | +| `api-key` | A telemetry.dev API key with the `datasets:read` and `experiments:write` scopes, for all environments or for `experiment`. Default: the `TELEMETRY_DEV_API_KEY` variable of the job. | +| `base-url` | The telemetry.dev API. Default: `https://ingest.telemetry.dev`. | +| `working-directory` | The directory that the script or the command runs in. Default: `.`. | +| `comment` | Set to `false` to write the job summary only. | +| `github-token` | The token that writes the comment. Default: `github.token`, which needs the `pull-requests: write` permission. | The action sets `TELEMETRY_DEV_ENVIRONMENT` to `experiment` when the job does not set it, so the -traces of the items stay out of your production pages. `CI` is `true` in GitHub Actions, thus the -SDK runs in CI mode: it records the repository, the commit, the run, and the pull request with -the experiment, and it exits with an error when the check fails. +traces of the items stay out of your production pages. Thus the key must be for all environments +or for `experiment`. A key for another environment, for example your production key, cannot send +the traces of the items: the experiment then completes with its outputs, but with no traces and no +cost. `CI` is `true` in GitHub Actions, thus the SDK runs in CI mode: it records the repository, +the commit, the run, and the pull request with the experiment, and it exits with an error when the +check fails. + +The action does not download packages. Add `tsx` to the `devDependencies` of the project for a +`.ts` script, or give a `command`. + +The outputs, the errors, and the scorer reasons of the items go to telemetry.dev and stay with the +experiment. The capture options of the SDK and the capture setting of the environment in the +project do not apply to them. ## Outputs @@ -76,8 +90,13 @@ the experiment, and it exits with an error when the check fails. ## The comment The action adds one comment to the pull request for each workflow, job, and `dataset`, `script`, or -`command`. A later push changes that comment. A pull request from a fork gets a token that cannot -write comments: the action then logs a warning, and the job summary still has the result. +`command`. A later push changes that comment. The action changes only a comment that its token +wrote: with a copy of the comment by someone else, it adds a new comment. When the token cannot +write comments, the action logs a warning, and the job summary still has the result. + +A pull request from a fork gets no secrets, thus no API key, and the action stops with the error +"Set the api-key input or the TELEMETRY_DEV_API_KEY variable." The `if` of the example above skips +the job for those pull requests. The comment and the summary show the name of each experiment, its status, the count of the items that ran, and the value, the change from the baseline, and the pass mark of each score. The SDK diff --git a/actions/experiment/action.yml b/actions/experiment/action.yml index 19a774c..edde3ec 100644 --- a/actions/experiment/action.yml +++ b/actions/experiment/action.yml @@ -7,7 +7,7 @@ branding: inputs: script: - description: The eval script, which calls runExperiment or run_experiment. A .ts script runs with tsx, a .js script with node, and a .py script with python. Give script or command. + description: The eval script, which calls runExperiment or run_experiment. A .ts script runs with the tsx in the node_modules of the project, a .js script with node, and a .py script with python. Give script or command. required: false command: description: A shell command that runs the eval script, for example `uv run python evals/reply_eval.py`. Give script or command. diff --git a/actions/experiment/report.mjs b/actions/experiment/report.mjs index 67ce615..ff94492 100644 --- a/actions/experiment/report.mjs +++ b/actions/experiment/report.mjs @@ -177,13 +177,22 @@ async function comment(body) { const api = (env.GITHUB_API_URL ?? "https://api.github.com").replace(/\/+$/, ""); const issue = `${api}/repos/${env.GITHUB_REPOSITORY}/issues`; + // Anyone can paste the marker into a comment, so the action changes only a comment that its token + // wrote. A user token knows its login. The GITHUB_TOKEN and app tokens cannot read /user, and + // they write as a bot. + const login = await github("GET", `${api}/user`).then( + (user) => (typeof user?.login === "string" ? user.login : undefined), + () => undefined, + ); + const ours = (item) => + login === undefined ? item.user?.type === "Bot" : item.user?.login === login; let existing; for (let page = 1; page <= MAX_COMMENT_PAGES && existing === undefined; page++) { const comments = await github("GET", `${issue}/${pull}/comments?per_page=100&page=${page}`); existing = comments.find( - (item) => typeof item.body === "string" && item.body.startsWith(marker), + (item) => typeof item.body === "string" && item.body.startsWith(marker) && ours(item), ); if (comments.length < 100) break; diff --git a/actions/experiment/run.sh b/actions/experiment/run.sh index 9d6f1c2..86cf722 100755 --- a/actions/experiment/run.sh +++ b/actions/experiment/run.sh @@ -31,12 +31,29 @@ results=$(mktemp "${RUNNER_TEMP:-/tmp}/telemetry-dev-experiment.XXXXXX") || fail echo "results=$results" >>"$GITHUB_OUTPUT" export TELEMETRY_DEV_EXPERIMENT_RESULTS=$results +# The tsx of the project, from node_modules/.bin here or in a directory above. The action does not +# download one: a package from npm at run time would run with the API key in its environment. +project_tsx() { + dir=$PWD + while :; do + if [ -x "$dir/node_modules/.bin/tsx" ]; then + echo "$dir/node_modules/.bin/tsx" + return 0 + fi + [ "$dir" = / ] && return 1 + dir=$(dirname "$dir") + done +} + code=0 if [ -n "$command" ]; then bash -c "$command" || code=$? else case "$script" in - *.ts | *.mts | *.cts | *.tsx) npx --yes tsx "$script" || code=$? ;; + *.ts | *.mts | *.cts | *.tsx) + tsx=$(project_tsx) || fail "No tsx in node_modules. Add tsx to the devDependencies of the project and install them, or give a command." + "$tsx" "$script" || code=$? + ;; *.js | *.mjs | *.cjs) node "$script" || code=$? ;; *.py) python=$(command -v python || command -v python3) || fail "No python on the PATH. Use actions/setup-python, or give a command." diff --git a/packages/sdk/tests/experiment-action.test.ts b/packages/sdk/tests/experiment-action.test.ts index cb7be99..eb8bd99 100644 --- a/packages/sdk/tests/experiment-action.test.ts +++ b/packages/sdk/tests/experiment-action.test.ts @@ -1,5 +1,5 @@ import { spawn, spawnSync } from "node:child_process"; -import { mkdtempSync, readFileSync, statSync, writeFileSync } from "node:fs"; +import { mkdirSync, mkdtempSync, readFileSync, statSync, writeFileSync } from "node:fs"; import { createServer, type IncomingMessage } from "node:http"; import type { AddressInfo } from "node:net"; import { tmpdir } from "node:os"; @@ -99,15 +99,36 @@ interface GithubRequest { body: unknown; } -/** The comment routes of the GitHub REST API for one pull request. */ +interface GithubComment { + id: number; + body: string; + user: { login: string; type: "Bot" | "User" }; +} + +const BOT = { login: "github-actions[bot]", type: "Bot" } as const; + +/** + * The comment routes of the GitHub REST API for one pull request. Without `user`, GET /user fails + * as it does for the GITHUB_TOKEN. + */ async function fakeGithub( - options: { comments?: { id: number; body: string }[]; status?: number } = {}, + options: { comments?: GithubComment[]; status?: number; user?: { login: string } } = {}, ) { const requests: GithubRequest[] = []; const server = createServer(async (req: IncomingMessage, res) => { let raw = ""; for await (const chunk of req) raw += chunk; requests.push({ method: req.method!, url: req.url!, body: raw ? JSON.parse(raw) : undefined }); + + if (req.url === "/user") { + res.writeHead(options.user ? 200 : 403, { "content-type": "application/json" }); + res.end( + JSON.stringify(options.user ?? { message: "Resource not accessible by integration" }), + ); + + return; + } + res.writeHead(options.status ?? 200, { "content-type": "application/json" }); res.end(JSON.stringify(req.method === "GET" ? (options.comments ?? []) : { id: 9 })); }); @@ -181,6 +202,37 @@ test("run.sh passes the inputs to the script and records its exit code without f expect(statSync(written.results!).mode & 0o777).toBe(0o600); }); +test("run.sh runs a .ts script with the tsx of the project, from a directory above", () => { + const project = workdir(); + const evals = join(project, "evals"); + const output = join(project, "output"); + mkdirSync(join(project, "node_modules", ".bin"), { recursive: true }); + mkdirSync(evals); + writeFileSync(output, ""); + // A stand-in for the tsx of the project that records the script it gets. + writeFileSync( + join(project, "node_modules", ".bin", "tsx"), + `#!/bin/sh\necho "$1" > "${join(project, "ran")}"\n`, + { mode: 0o755 }, + ); + + const run = spawnSync("bash", [join(action, "run.sh")], { + cwd: evals, + encoding: "utf8", + env: { + PATH: process.env.PATH, + RUNNER_TEMP: project, + GITHUB_OUTPUT: output, + INPUT_SCRIPT: "reply.eval.ts", + TELEMETRY_DEV_API_KEY: "td_live_job", + }, + }); + + expect(run.status).toBe(0); + expect(readFileSync(join(project, "ran"), "utf8")).toBe("reply.eval.ts\n"); + expect(outputs(output)["exit-code"]).toBe("0"); +}); + test("run.sh keeps the environment of the job, and runs a .js script with node", () => { const { run, outputs: written } = runScript( { @@ -219,6 +271,11 @@ test.each([ { INPUT_SCRIPT: "eval.rb", TELEMETRY_DEV_API_KEY: "k" }, "The script must end in .ts, .js, or .py. For another runner, give a command.", ], + [ + "a .ts script in a project without tsx", + { INPUT_SCRIPT: "reply.eval.ts", TELEMETRY_DEV_API_KEY: "k" }, + "No tsx in node_modules. Add tsx to the devDependencies of the project and install them, or give a command.", + ], ])("run.sh stops with an error for %s", (_, env, message) => { const { run, outputs: written } = runScript(env); @@ -239,33 +296,74 @@ test("report.mjs writes the summary, posts it on the pull request, and sets the expect(written).toBe(`${summary}\n`); expect(set).toEqual({ "check-status": "failed", url: result.url }); expect(github.requests.map((request) => `${request.method} ${request.url}`)).toEqual([ + "GET /user", "GET /repos/acme/support/issues/42/comments?per_page=100&page=1", "POST /repos/acme/support/issues/42/comments", ]); - expect(github.requests[1]!.body).toEqual({ + expect(github.requests[2]!.body).toEqual({ body: expect.stringMatching( /^\n### telemetry\.dev experiment\n/, ), }); - expect((github.requests[1]!.body as { body: string }).body.endsWith(summary)).toBe(true); + expect((github.requests[2]!.body as { body: string }).body.endsWith(summary)).toBe(true); }); +/** The body of the comment that the action posts for the result. */ +async function commentBody(env: Record = {}): Promise { + const github = await fakeGithub(); + await report({ GITHUB_API_URL: github.url, ...env }, [result]); + + return (github.requests.find((request) => request.method === "POST")!.body as { body: string }) + .body; +} + +function changes(github: Awaited>): string[] { + return github.requests + .filter((request) => request.method !== "GET") + .map((request) => `${request.method} ${request.url}`); +} + test("report.mjs changes its comment from an earlier push, and a step with another dataset adds its own", async () => { - const first = await fakeGithub(); - await report({ GITHUB_API_URL: first.url }, [result]); - const own = { id: 8, body: (first.requests[1]!.body as { body: string }).body }; + const own = { id: 8, body: await commentBody(), user: BOT }; const other = await fakeGithub({ comments: [own] }); await report({ GITHUB_API_URL: other.url, INPUT_DATASET: "billing-golden" }, [result]); - const again = await fakeGithub({ comments: [{ id: 7, body: "Looks good" }, own] }); + const again = await fakeGithub({ + comments: [{ id: 7, body: "Looks good", user: { login: "dana", type: "User" } }, own], + }); await report({ GITHUB_API_URL: again.url }, [result]); - expect(other.requests.map((request) => request.method)).toEqual(["GET", "POST"]); - expect(again.requests.map((request) => `${request.method} ${request.url}`)).toEqual([ - "GET /repos/acme/support/issues/42/comments?per_page=100&page=1", - "PATCH /repos/acme/support/issues/comments/8", - ]); + expect(changes(other)).toEqual(["POST /repos/acme/support/issues/42/comments"]); + expect(changes(again)).toEqual(["PATCH /repos/acme/support/issues/comments/8"]); +}); + +test.each([ + ["a person pasted under the GITHUB_TOKEN", undefined, { login: "mallory", type: "User" }], + ["another bot wrote under a user token", { login: "dana" }, BOT], +] as const)( + "report.mjs does not change a comment with its marker that %s", + async (_, user, author) => { + const github = await fakeGithub({ + user, + comments: [{ id: 7, body: await commentBody(), user: author }], + }); + + await report({ GITHUB_API_URL: github.url }, [result]); + + expect(changes(github)).toEqual(["POST /repos/acme/support/issues/42/comments"]); + }, +); + +test("report.mjs changes the comment of the user of a user token", async () => { + const github = await fakeGithub({ + user: { login: "dana" }, + comments: [{ id: 7, body: await commentBody(), user: { login: "dana", type: "User" } }], + }); + + await report({ GITHUB_API_URL: github.url }, [result]); + + expect(changes(github)).toEqual(["PATCH /repos/acme/support/issues/comments/7"]); }); test("report.mjs warns and succeeds when the token cannot write the comment", async () => { From fad2f001dc0574a5ece53efc1ca0025e311e0d4f Mon Sep 17 00:00:00 2001 From: ephraimduncan Date: Fri, 30 Oct 2026 00:33:43 +0000 Subject: [PATCH 09/10] docs(sdk): tell how a stopped experiment run completes The conformance document gives the complete request after a run stops, and the create retry only after a 429. It also gives the copy of the item for each run and the outputs that JSON cannot hold. It gives the Python User-Agent, the fetch call as a function, the fields that a result must have, and the cut of the title. --- docs/sdk-conformance.md | 48 +++++++++++++++++++++++++++-------------- 1 file changed, 32 insertions(+), 16 deletions(-) diff --git a/docs/sdk-conformance.md b/docs/sdk-conformance.md index e81153b..483a8c4 100644 --- a/docs/sdk-conformance.md +++ b/docs/sdk-conformance.md @@ -184,12 +184,16 @@ task. ## Experiments and scores -`runExperiment(options)` / `run_experiment(...)` (and `run_experiment_async`) and `score(options)` -/ `score(...)` (and `score_async`) call the Evals routes on `{baseUrl}`, with the API key, the base +`runExperiment(options)` / `run_experiment(...)` (and `run_experiment_async`) and `score(options)` / +`score(...)` (and `score_async`) call the Evals routes on `{baseUrl}`, with the API key, the base URL, and the empty-value rules of getPrompt. The requests have the `Authorization: Bearer ` -header, the SDK header, and a JSON body. The redirect rule of getPrompt applies. An error has the -code of the `{ error: { code, message } }` body, or `http_`, or `missing_api_key`, -`timeout`, `network_error`, or `invalid_response`. +header, the SDK header, and a JSON body. Python also sends +`User-Agent: telemetry-dev-python/`, because bot filters refuse the default +`Python-urllib/3.x`. The body is the JSON of `JSON.stringify`: a lone surrogate goes as a +`\uXXXX` escape. The TypeScript SDK calls `fetch` as a plain function, because Workers and +browsers reject a call whose `this` is another object. The redirect rule of getPrompt applies. An +error has the code of the `{ error: { code, message } }` body, or `http_`, or +`missing_api_key`, `timeout`, `network_error`, or `invalid_response`. An experiment run sends these requests in this sequence: @@ -203,21 +207,28 @@ An experiment run sends these requests in this sequence: value are left out. 3. `POST /v1/experiments/{id}/items` with at most 100 item runs and at most 4 MiB of JSON. An item run has `itemId`, `runIndex`, `output` (a string as it is, other values as `JSON.stringify` JSON, - `null` for no value, cut to 100,000 UTF-16 units), `traceId` (when the span records), - `latencyMs`, and, for a task that failed, `error` (`: `, cut to 4,000 UTF-16 - units) or, for a task that succeeded, `scores`. The posts go one at a time, in order. A post - that fails stops the run: no item starts after it, and the run fails when the running items end. + `null` for no value and for a value that JSON cannot hold, such as an object that holds itself, + cut to 100,000 UTF-16 units), `traceId` (when the span records), `latencyMs`, and, for a task + that failed, `error` (`: `, cut to 4,000 UTF-16 units) or, for a task that + succeeded, `scores`. Python sends a lone surrogate of an output or an error as U+FFFD. Each run + gets its own deep copy of the item, so a task that changes its input does not change the next + run. The posts go one at a time, in order. A post or a page fetch that fails stops the run: no + item starts after it. When the running items end, the SDK sends the request of step 4, sends its + failure to `onError` / `on_error`, and fails with the first error. The server then ends the + experiment with the item runs that it has, or fails it when it has none. Python does the same + after `KeyboardInterrupt` and cancellation. 4. `POST /v1/experiments/{id}/complete`, after `flush()`. 5. `GET /v1/experiments/{id}`, after 1, 2, 4, and then each 5 seconds, until the `status` is `completed`, `failed`, or `stopped`, for 30 minutes at most (`timeout`). Retries: the GET requests, the item posts, and the complete request try 3 times in all after a timeout, a network error, a 429, or a 5xx. The server skips an item run that it has. The POST that -makes the experiment tries again only after a 429 or a 503. A retry waits for the `Retry-After` of -a 429 or 503 (seconds or an HTTP date), at most 60 seconds, or else 1 second and then 2 seconds. -`POST /v1/scores` makes one request, with no retry. The TypeScript request timeouts abort the -request: 30 seconds for the experiment requests and 10 seconds for a score. The Python timeouts -apply to each socket operation of urllib. +makes the experiment tries again only after a 429, which the rate limit answers before any work: the +server answers 503 when its database work times out, which can be after the commit. A retry waits +for the `Retry-After` of a 429 or 503 (seconds or an HTTP date), at most 60 seconds, or else 1 +second and then 2 seconds. `POST /v1/scores` makes one request, with no retry. The TypeScript +request timeouts abort the request: 30 seconds for the experiment requests and 10 seconds for a +score. The Python timeouts apply to each socket operation of urllib. Options and limits: `runsPerItem` / `runs_per_item` is an integer from 1 to 5 (default 1). `concurrency` is an integer from 1 to 100 (default 4). `evaluators` has at most 20 trimmed names @@ -244,13 +255,18 @@ not all zeros, a `spanId` is 16 lowercase hex digits and not all zeros and needs never throws: it returns the id, or no value after it sends the `ScoreError` to `onError` / `on_error`. +The result of step 5 needs `id`, `name`, `status`, `url`, `itemsTotal`, `itemsDone`, `scores`, +and `checkStatus`, else it is `invalid_response`: `checkStatus` decides CI mode, so a missing one +does not read as no check. A field of a score that can be null can also be missing. + CI mode is on when the `ci` option is true, or with no option when `CI` is `true` or `1`. Then: - In GitHub Actions (`GITHUB_ACTIONS=true` and a `GITHUB_REPOSITORY` of `owner/name`, at most 200 characters), `ci` is `{ provider: "github_actions", repository, commit, prNumber, prTitle, prUrl, runUrl }`. The commit is the `pull_request.head.sha` of `GITHUB_EVENT_PATH`, else `GITHUB_SHA`, - and must be 7 to 64 hex digits. The title is cut to 256 UTF-16 units. A URL must be `https://` - with no space, at most 2,048 UTF-16 units. The run URL is + and must be 7 to 64 hex digits. The title is cut to 256 UTF-16 units, with no half of a + surrogate pair at the end, because Postgres rejects a lone surrogate in jsonb. A URL must be + `https://` with no space, at most 2,048 UTF-16 units. The run URL is `//actions/runs/`. Keys without a value are left out. - After the result, a status other than `completed` is `experiment_failed`, and a `checkStatus` of From 85fa4f14b95194aa59c4ebca7a4fe8062d6bf82d Mon Sep 17 00:00:00 2001 From: ephraimduncan Date: Fri, 30 Oct 2026 01:01:35 +0000 Subject: [PATCH 10/10] docs(sdk): give api.telemetry.dev as the default base url The base-url input of the experiment action and the baseUrl comment in api.ts gave the old default, https://ingest.telemetry.dev. The default base URL of the SDKs changed to https://api.telemetry.dev in #109. --- actions/experiment/README.md | 2 +- actions/experiment/action.yml | 2 +- packages/sdk/src/api.ts | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/actions/experiment/README.md b/actions/experiment/README.md index 7b851fd..431f2c0 100644 --- a/actions/experiment/README.md +++ b/actions/experiment/README.md @@ -60,7 +60,7 @@ changes. | `dataset` | The dataset to run on. The action puts it in `TELEMETRY_DEV_EXPERIMENT_DATASET`. When the script also names a dataset, the two must be the same. | | `min-scores` | Pass marks, as `name=value` pairs that commas or new lines separate. A pass rate is 0 to 1, and a mean is on the scale of its score. The action puts them in `TELEMETRY_DEV_EXPERIMENT_MIN_SCORES`, and they win over the `minScores` of the script for the same name. | | `api-key` | A telemetry.dev API key with the `datasets:read` and `experiments:write` scopes, for all environments or for `experiment`. Default: the `TELEMETRY_DEV_API_KEY` variable of the job. | -| `base-url` | The telemetry.dev API. Default: `https://ingest.telemetry.dev`. | +| `base-url` | The telemetry.dev API. Default: `https://api.telemetry.dev`. | | `working-directory` | The directory that the script or the command runs in. Default: `.`. | | `comment` | Set to `false` to write the job summary only. | | `github-token` | The token that writes the comment. Default: `github.token`, which needs the `pull-requests: write` permission. | diff --git a/actions/experiment/action.yml b/actions/experiment/action.yml index edde3ec..5cc5fa6 100644 --- a/actions/experiment/action.yml +++ b/actions/experiment/action.yml @@ -22,7 +22,7 @@ inputs: description: A telemetry.dev API key with the datasets:read and experiments:write scopes. Default the TELEMETRY_DEV_API_KEY variable of the job. required: false base-url: - description: The telemetry.dev API. Default https://ingest.telemetry.dev. + description: The telemetry.dev API. Default https://api.telemetry.dev. required: false working-directory: description: The directory that the script or the command runs in. diff --git a/packages/sdk/src/api.ts b/packages/sdk/src/api.ts index 573f780..dd7fac6 100644 --- a/packages/sdk/src/api.ts +++ b/packages/sdk/src/api.ts @@ -6,7 +6,7 @@ import { currentClient } from "./client.ts"; export interface ApiOptions { /** API key. Falls back to the initialized client, then `TELEMETRY_DEV_API_KEY`. */ apiKey?: string; - /** Falls back to the initialized client, then `TELEMETRY_DEV_BASE_URL`, then `https://ingest.telemetry.dev`. */ + /** Falls back to the initialized client, then `TELEMETRY_DEV_BASE_URL`, then `https://api.telemetry.dev`. */ baseUrl?: string; /** Falls back to the initialized client's `fetch`, then `globalThis.fetch`. */ fetch?: typeof fetch;