diff --git a/.github/workflows/bench.yml b/.github/workflows/bench.yml new file mode 100644 index 0000000..458a8ed --- /dev/null +++ b/.github/workflows/bench.yml @@ -0,0 +1,98 @@ +name: Bench + +# Runs only when a maintainer adds the `bench` label to a PR. Remove and re-add +# the label to run it again after new commits. +on: + pull_request: + types: [labeled] + +permissions: + contents: read + +# The label name is in the group so adding some other label to the PR can't +# cancel a running bench. +concurrency: + group: bench-${{ github.event.pull_request.number }}-${{ github.event.label.name }} + cancel-in-progress: true + +jobs: + bench: + name: Output and memory check + if: github.event.label.name == 'bench' + runs-on: blacksmith-6vcpu-macos-latest + timeout-minutes: 45 + steps: + - name: Checkout + uses: actions/checkout@v4 + with: + fetch-depth: 0 + persist-credentials: false + + - name: Install Rust + uses: dtolnay/rust-toolchain@stable + + - name: Install Bun + uses: oven-sh/setup-bun@v2 + + - name: Check out the base branch + run: | + echo "BASE_DIR=$RUNNER_TEMP/wrec-base" >> "$GITHUB_ENV" + git worktree add --detach "$RUNNER_TEMP/wrec-base" "${{ github.event.pull_request.base.sha }}" + + - name: Cache Cargo + uses: actions/cache@v4 + with: + path: | + ~/.cargo/registry + ~/.cargo/git + target + ${{ env.BASE_DIR }}/target + key: ${{ runner.os }}-cargo-bench-${{ hashFiles('Cargo.lock') }} + restore-keys: | + ${{ runner.os }}-cargo-bench- + + - name: Build base + working-directory: ${{ env.BASE_DIR }} + run: cargo build --release -p cli -p daemon + + - name: Build PR + run: cargo build --release -p cli -p daemon + + # The VM has SIP off, so TCC.db is writable. macOS checks the runner + # process that launched the recorder, so the runner itself needs the grant. + - name: Grant Screen Recording + run: | + now=$(date +%s) + for c in \ + "$PWD/target/release/wrec" "$PWD/target/release/daemon" \ + "$BASE_DIR/target/release/wrec" "$BASE_DIR/target/release/daemon" \ + $(find "$PWD/target/release/build" "$BASE_DIR/target/release/build" -name capture-engine -type f) \ + "$(command -v bun)" "$HOME/bin/Runner.Listener" "$HOME/bin/Runner.Worker"; do + for db in "$HOME/Library/Application Support/com.apple.TCC/TCC.db" "/Library/Application Support/com.apple.TCC/TCC.db"; do + sudo sqlite3 "$db" "insert or replace into access (service,client,client_type,auth_value,auth_reason,auth_version,indirect_object_identifier,flags,last_modified) values ('kTCCServiceScreenCapture','$c',1,2,4,1,'UNUSED',0,$now);" + done + done + sudo killall tccd || true + sleep 2 + + - name: Bench PR against base + run: | + set -o pipefail + bun run bench ci --wrec target/release/wrec --against "$BASE_DIR/target/release/wrec" 2>&1 | tee /tmp/bench.log + + - name: Summary + if: always() + run: | + { + echo '```' + sed -n '/^[a-z0-9-]*: \(pass\|fail\|inconclusive\)$/,$p' /tmp/bench.log + echo '```' + } >> "$GITHUB_STEP_SUMMARY" + + - name: Upload results + if: always() + uses: actions/upload-artifact@v4 + with: + name: bench-${{ github.event.pull_request.number }} + path: benchmarks/runs/ + if-no-files-found: ignore diff --git a/benchmarks/README.md b/benchmarks/README.md index fe575f4..c4220e1 100644 --- a/benchmarks/README.md +++ b/benchmarks/README.md @@ -43,6 +43,22 @@ cd docs && bun run dev # then open localhost:4321/benchmarks The smoke suite runs one balanced HEVC recording against the stimulus window and does not apply release gates. It is meant for local sanity checks and CI plumbing. +## PR Check + +Add the `bench` label to a PR to run `.github/workflows/bench.yml`. It builds +the PR and its base branch on a Blacksmith macOS runner and runs: + +```sh +bun run bench ci --wrec target/release/wrec --against /path/to/base/wrec +``` + +The `ci` suite records balanced 1080p HEVC and H.264 for 5 s, three times per +build. It fails if a recording breaks (exit code, decode, codec, dimensions, +timestamp order) or if peak memory grows more than 2% and 1 MB over the base. +It skips fps, CPU, and latency gates, because the runner is a VM and those +numbers only measure the VM. To run it again after new commits, remove the +label and add it back. + ## Release Gate Before creating a version tag, run the local release check from a clean checkout diff --git a/benchmarks/src/benchmark-values.test.ts b/benchmarks/src/benchmark-values.test.ts index 3fb90d3..82173ff 100644 --- a/benchmarks/src/benchmark-values.test.ts +++ b/benchmarks/src/benchmark-values.test.ts @@ -21,6 +21,11 @@ describe("benchmarkExitCode", () => { expect(benchmarkExitCode("release", "inconclusive")).toBe(1); }); + test("fails the PR check when it does not pass", () => { + expect(benchmarkExitCode("ci", "pass")).toBe(0); + expect(benchmarkExitCode("ci", "fail")).toBe(1); + }); + test("keeps smoke runs informational", () => { expect(benchmarkExitCode("smoke", "pass")).toBe(0); expect(benchmarkExitCode("smoke", "fail")).toBe(0); diff --git a/benchmarks/src/benchmark-values.ts b/benchmarks/src/benchmark-values.ts index 80a0e14..a70f073 100644 --- a/benchmarks/src/benchmark-values.ts +++ b/benchmarks/src/benchmark-values.ts @@ -9,4 +9,4 @@ export const nullableNumber = (value: unknown) => { }; export const benchmarkExitCode = (suite: SuiteName, status: OverallStatus) => - suite === "release" && status !== "pass" ? 1 : 0; + suite !== "smoke" && status !== "pass" ? 1 : 0; diff --git a/benchmarks/src/gates.test.ts b/benchmarks/src/gates.test.ts index e9fe3bf..7dfdc0f 100644 --- a/benchmarks/src/gates.test.ts +++ b/benchmarks/src/gates.test.ts @@ -1,5 +1,6 @@ import { describe, expect, test } from "bun:test"; import { + evaluateCiGates, evaluateProfileGates, type ProfileSpec, type RunResult, @@ -160,3 +161,72 @@ describe("release gate hardening", () => { expect(status("regression_observed_fps")).toBe("pass"); }); }); + +describe("ci gates", () => { + const mb = 1024 * 1024; + const withRss = ( + rep: number, + variant: "candidate" | "reference", + rssBytes: number, + ) => { + const base = run(rep, 500, 100); + return { + ...base, + id: `${variant}-${rep}`, + variant, + processSummary: { ...base.processSummary, maxTotalRssBytes: rssBytes }, + } satisfies RunResult; + }; + const reference = [1, 2, 3].map((rep) => withRss(rep, "reference", 46 * mb)); + const status = (gates: ReturnType, name: string) => + gates.find((gate) => gate.name === name)?.status; + + test("passes memory within VM noise and fails a real increase", () => { + const same = [1, 2, 3].map((rep) => withRss(rep, "candidate", 46.1 * mb)); + const grown = [1, 2, 3].map((rep) => withRss(rep, "candidate", 48 * mb)); + const dims = { width: 1920, height: 1080 }; + + expect( + status( + evaluateCiGates(profile, dims, same, reference), + "regression_max_rss", + ), + ).toBe("pass"); + expect( + status( + evaluateCiGates(profile, dims, grown, reference), + "regression_max_rss", + ), + ).toBe("fail"); + }); + + test("fails a recording with the wrong codec", () => { + const h264 = [1, 2, 3].map((rep) => { + const base = withRss(rep, "candidate", 46 * mb); + return { ...base, decode: { ...base.decode!, codec: "h264" } }; + }); + const gates = evaluateCiGates( + profile, + { width: 1920, height: 1080 }, + h264, + reference, + ); + + expect(status(gates, "codec_match")).toBe("fail"); + }); + + test("fails a recording with no frames", () => { + const empty = [1, 2, 3].map((rep) => { + const base = withRss(rep, "candidate", 46 * mb); + return { ...base, decode: { ...base.decode!, frames: [] } }; + }); + const gates = evaluateCiGates( + profile, + { width: 1920, height: 1080 }, + empty, + reference, + ); + + expect(status(gates, "frames_decoded")).toBe("fail"); + }); +}); diff --git a/benchmarks/src/gates.ts b/benchmarks/src/gates.ts index 27c45d1..2835354 100644 --- a/benchmarks/src/gates.ts +++ b/benchmarks/src/gates.ts @@ -1,4 +1,4 @@ -export type SuiteName = "smoke" | "release"; +export type SuiteName = "smoke" | "release" | "ci"; export type VariantName = "candidate" | "reference"; export type Quality = "efficient" | "balanced" | "high"; export type Codec = "hevc" | "h264"; @@ -223,10 +223,22 @@ export const releaseProfiles: ProfileSpec[] = [ }, ]; +// The PR check runs on a CI VM, where capture fps and CPU only measure the VM. +// It keeps the gates a VM can't distort: output correctness and memory. +export const ciProfiles: ProfileSpec[] = releaseProfiles.filter( + (profile) => + profile.name === "balanced-1080p30-hevc" || + profile.name === "balanced-1080p30-h264", +); + const cpuNoiseFloor = 3; const rssNoiseFloor = 20 * 1024 * 1024; const startLatencyNoiseFloor = 500; const finalizeLatencyNoiseFloor = 150; +// Max RSS moved under 0.25% between two copies of one binary on the CI VM, +// so the PR check can flag a memory change far smaller than the release gate. +const ciRssNoiseFloor = 1024 * 1024; +const ciRssPercentTolerance = 2; export const expectedOutputDimensions = ( native: Dimensions, @@ -442,32 +454,7 @@ export const evaluateProfileGates = ( ), }), ); - gates.push({ - name: "pts_monotonic", - threshold: "true", - measured: candidate.observed.ptsMonotonic, - status: candidate.observed.ptsMonotonic ? "pass" : "fail", - }); - gates.push({ - name: "codec_match", - threshold: profile.codec, - measured: candidate.observed.codec, - status: candidateRuns.every((run) => run.decode?.codec === profile.codec) - ? "pass" - : "fail", - }); - gates.push({ - name: "dimensions_match_request", - threshold: `${expectedDimensions.width}x${expectedDimensions.height}`, - measured: candidate.observed.dimensions - ? `${candidate.observed.dimensions.width}x${candidate.observed.dimensions.height}` - : null, - status: candidateRuns.every((run) => - dimensionsEqual(run.decode?.dimensions, expectedDimensions), - ) - ? "pass" - : "fail", - }); + gates.push(...outputGates(profile, expectedDimensions, candidateRuns)); gates.push( numericBudgetGate({ name: "self_report_disagreement", @@ -528,6 +515,68 @@ export const evaluateProfileGates = ( ); }; +export const evaluateCiGates = ( + profile: ProfileSpec, + expectedDimensions: Dimensions, + candidateRuns: RunResult[], + referenceRuns?: RunResult[], +): Gate[] => [ + ...outputGates(profile, expectedDimensions, candidateRuns), + { + name: "frames_decoded", + threshold: "> 0 in every run", + measured: summarizeRuns(candidateRuns).observed.decodedFrames, + status: candidateRuns.every((run) => (run.decode?.frames.length ?? 0) > 0) + ? "pass" + : "fail", + }, + regressionGate( + "regression_max_rss", + "bytes", + candidateRuns, + referenceRuns, + (run) => run.processSummary.maxTotalRssBytes, + ciRssNoiseFloor, + ciRssPercentTolerance, + ), +]; + +const outputGates = ( + profile: ProfileSpec, + expectedDimensions: Dimensions, + candidateRuns: RunResult[], +): Gate[] => { + const candidate = summarizeRuns(candidateRuns); + return [ + { + name: "pts_monotonic", + threshold: "true", + measured: candidate.observed.ptsMonotonic, + status: candidate.observed.ptsMonotonic ? "pass" : "fail", + }, + { + name: "codec_match", + threshold: profile.codec, + measured: candidate.observed.codec, + status: candidateRuns.every((run) => run.decode?.codec === profile.codec) + ? "pass" + : "fail", + }, + { + name: "dimensions_match_request", + threshold: `${expectedDimensions.width}x${expectedDimensions.height}`, + measured: candidate.observed.dimensions + ? `${candidate.observed.dimensions.width}x${candidate.observed.dimensions.height}` + : null, + status: candidateRuns.every((run) => + dimensionsEqual(run.decode?.dimensions, expectedDimensions), + ) + ? "pass" + : "fail", + }, + ]; +}; + export const statusFromGates = ( gates: Gate[], runs: RunResult[], diff --git a/benchmarks/src/run.ts b/benchmarks/src/run.ts index 4b380cf..c6cc18e 100644 --- a/benchmarks/src/run.ts +++ b/benchmarks/src/run.ts @@ -10,7 +10,9 @@ import { import os from "node:os"; import path from "node:path"; import { + ciProfiles, combineStatuses, + evaluateCiGates, evaluateProfileGates, expectedOutputDimensions, releaseProfiles, @@ -294,6 +296,9 @@ const main = async () => { const summaryPath = options.suite === "release" ? await writeSummary(resultsDir, result) : null; + if (result.suite === "ci") { + printGates(result.profiles); + } console.log(`status: ${result.status}`); console.log(`results: ${resultPath}`); if (summaryPath) { @@ -319,7 +324,7 @@ const parseArgs = (args: string[]): CliOptions => { for (let index = 0; index < args.length; index += 1) { const arg = args[index]; - if (arg === "smoke" || arg === "release") { + if (arg === "smoke" || arg === "release" || arg === "ci") { suite = arg; continue; } @@ -384,9 +389,10 @@ Usage (from the repo root): bun run bench # smoke: one ungated 5s recording bun run bench release # gated release profiles bun run bench release --against /path/to/reference/wrec + bun run bench ci --against /path/to/reference/wrec # PR check: output + memory Options: - smoke | release suite to run (default: smoke); --suite also works + smoke | release | ci suite to run (default: smoke); --suite also works --duration