Repository navigation
bench: check recordings and memory on a PR when it gets the bench label #32
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Changes from all commits
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,98 @@ | ||
| name: Bench | ||
|
|
||
| # Runs only when a maintainer adds the `bench` label to a PR. Remove and re-add | ||
| # the label to run it again after new commits. | ||
| on: | ||
| pull_request: | ||
| types: [labeled] | ||
|
|
||
| permissions: | ||
| contents: read | ||
|
|
||
| # The label name is in the group so adding some other label to the PR can't | ||
| # cancel a running bench. | ||
| concurrency: | ||
| group: bench-${{ github.event.pull_request.number }}-${{ github.event.label.name }} | ||
| cancel-in-progress: true | ||
|
|
||
| jobs: | ||
| bench: | ||
| name: Output and memory check | ||
| if: github.event.label.name == 'bench' | ||
| runs-on: blacksmith-6vcpu-macos-latest | ||
| timeout-minutes: 45 | ||
| steps: | ||
| - name: Checkout | ||
| uses: actions/checkout@v4 | ||
| with: | ||
| fetch-depth: 0 | ||
| persist-credentials: false | ||
|
|
||
| - name: Install Rust | ||
| uses: dtolnay/rust-toolchain@stable | ||
|
|
||
| - name: Install Bun | ||
| uses: oven-sh/setup-bun@v2 | ||
|
|
||
| - name: Check out the base branch | ||
| run: | | ||
| echo "BASE_DIR=$RUNNER_TEMP/wrec-base" >> "$GITHUB_ENV" | ||
| git worktree add --detach "$RUNNER_TEMP/wrec-base" "${{ github.event.pull_request.base.sha }}" | ||
|
|
||
| - name: Cache Cargo | ||
| uses: actions/cache@v4 | ||
| with: | ||
| path: | | ||
| ~/.cargo/registry | ||
| ~/.cargo/git | ||
| target | ||
| ${{ env.BASE_DIR }}/target | ||
| key: ${{ runner.os }}-cargo-bench-${{ hashFiles('Cargo.lock') }} | ||
| restore-keys: | | ||
| ${{ runner.os }}-cargo-bench- | ||
|
|
||
| - name: Build base | ||
| working-directory: ${{ env.BASE_DIR }} | ||
| run: cargo build --release -p cli -p daemon | ||
|
|
||
| - name: Build PR | ||
| run: cargo build --release -p cli -p daemon | ||
|
|
||
| # The VM has SIP off, so TCC.db is writable. macOS checks the runner | ||
| # process that launched the recorder, so the runner itself needs the grant. | ||
| - name: Grant Screen Recording | ||
| run: | | ||
| now=$(date +%s) | ||
| for c in \ | ||
| "$PWD/target/release/wrec" "$PWD/target/release/daemon" \ | ||
| "$BASE_DIR/target/release/wrec" "$BASE_DIR/target/release/daemon" \ | ||
| $(find "$PWD/target/release/build" "$BASE_DIR/target/release/build" -name capture-engine -type f) \ | ||
| "$(command -v bun)" "$HOME/bin/Runner.Listener" "$HOME/bin/Runner.Worker"; do | ||
| for db in "$HOME/Library/Application Support/com.apple.TCC/TCC.db" "/Library/Application Support/com.apple.TCC/TCC.db"; do | ||
| sudo sqlite3 "$db" "insert or replace into access (service,client,client_type,auth_value,auth_reason,auth_version,indirect_object_identifier,flags,last_modified) values ('kTCCServiceScreenCapture','$c',1,2,4,1,'UNUSED',0,$now);" | ||
| done | ||
| done | ||
| sudo killall tccd || true | ||
| sleep 2 | ||
|
|
||
| - name: Bench PR against base | ||
| run: | | ||
| set -o pipefail | ||
| bun run bench ci --wrec target/release/wrec --against "$BASE_DIR/target/release/wrec" 2>&1 | tee /tmp/bench.log | ||
|
|
||
| - name: Summary | ||
| if: always() | ||
| run: | | ||
| { | ||
| echo '```' | ||
| sed -n '/^[a-z0-9-]*: \(pass\|fail\|inconclusive\)$/,$p' /tmp/bench.log | ||
| echo '```' | ||
| } >> "$GITHUB_STEP_SUMMARY" | ||
|
|
||
| - name: Upload results | ||
| if: always() | ||
| uses: actions/upload-artifact@v4 | ||
| with: | ||
| name: bench-${{ github.event.pull_request.number }} | ||
| path: benchmarks/runs/ | ||
| if-no-files-found: ignore | ||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,4 +1,4 @@ | ||
| export type SuiteName = "smoke" | "release"; | ||
| export type SuiteName = "smoke" | "release" | "ci"; | ||
| export type VariantName = "candidate" | "reference"; | ||
| export type Quality = "efficient" | "balanced" | "high"; | ||
| export type Codec = "hevc" | "h264"; | ||
|
|
@@ -223,10 +223,22 @@ export const releaseProfiles: ProfileSpec[] = [ | |
| }, | ||
| ]; | ||
|
|
||
| // The PR check runs on a CI VM, where capture fps and CPU only measure the VM. | ||
| // It keeps the gates a VM can't distort: output correctness and memory. | ||
| export const ciProfiles: ProfileSpec[] = releaseProfiles.filter( | ||
| (profile) => | ||
| profile.name === "balanced-1080p30-hevc" || | ||
| profile.name === "balanced-1080p30-h264", | ||
| ); | ||
|
|
||
| const cpuNoiseFloor = 3; | ||
| const rssNoiseFloor = 20 * 1024 * 1024; | ||
| const startLatencyNoiseFloor = 500; | ||
| const finalizeLatencyNoiseFloor = 150; | ||
| // Max RSS moved under 0.25% between two copies of one binary on the CI VM, | ||
| // so the PR check can flag a memory change far smaller than the release gate. | ||
| const ciRssNoiseFloor = 1024 * 1024; | ||
| const ciRssPercentTolerance = 2; | ||
|
|
||
| export const expectedOutputDimensions = ( | ||
| native: Dimensions, | ||
|
|
@@ -442,32 +454,7 @@ export const evaluateProfileGates = ( | |
| ), | ||
| }), | ||
| ); | ||
| gates.push({ | ||
| name: "pts_monotonic", | ||
| threshold: "true", | ||
| measured: candidate.observed.ptsMonotonic, | ||
| status: candidate.observed.ptsMonotonic ? "pass" : "fail", | ||
| }); | ||
| gates.push({ | ||
| name: "codec_match", | ||
| threshold: profile.codec, | ||
| measured: candidate.observed.codec, | ||
| status: candidateRuns.every((run) => run.decode?.codec === profile.codec) | ||
| ? "pass" | ||
| : "fail", | ||
| }); | ||
| gates.push({ | ||
| name: "dimensions_match_request", | ||
| threshold: `${expectedDimensions.width}x${expectedDimensions.height}`, | ||
| measured: candidate.observed.dimensions | ||
| ? `${candidate.observed.dimensions.width}x${candidate.observed.dimensions.height}` | ||
| : null, | ||
| status: candidateRuns.every((run) => | ||
| dimensionsEqual(run.decode?.dimensions, expectedDimensions), | ||
| ) | ||
| ? "pass" | ||
| : "fail", | ||
| }); | ||
| gates.push(...outputGates(profile, expectedDimensions, candidateRuns)); | ||
| gates.push( | ||
| numericBudgetGate({ | ||
| name: "self_report_disagreement", | ||
|
|
@@ -528,6 +515,68 @@ export const evaluateProfileGates = ( | |
| ); | ||
| }; | ||
|
|
||
| export const evaluateCiGates = ( | ||
| profile: ProfileSpec, | ||
| expectedDimensions: Dimensions, | ||
| candidateRuns: RunResult[], | ||
| referenceRuns?: RunResult[], | ||
| ): Gate[] => [ | ||
| ...outputGates(profile, expectedDimensions, candidateRuns), | ||
| { | ||
| name: "frames_decoded", | ||
| threshold: "> 0 in every run", | ||
| measured: summarizeRuns(candidateRuns).observed.decodedFrames, | ||
| status: candidateRuns.every((run) => (run.decode?.frames.length ?? 0) > 0) | ||
| ? "pass" | ||
| : "fail", | ||
| }, | ||
| regressionGate( | ||
|
Comment on lines
+523
to
+533
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more.
Require at least one decoded frame and make the decoder reject a failed read. |
||
| "regression_max_rss", | ||
| "bytes", | ||
| candidateRuns, | ||
| referenceRuns, | ||
| (run) => run.processSummary.maxTotalRssBytes, | ||
| ciRssNoiseFloor, | ||
| ciRssPercentTolerance, | ||
| ), | ||
| ]; | ||
|
|
||
| const outputGates = ( | ||
| profile: ProfileSpec, | ||
| expectedDimensions: Dimensions, | ||
| candidateRuns: RunResult[], | ||
| ): Gate[] => { | ||
| const candidate = summarizeRuns(candidateRuns); | ||
| return [ | ||
| { | ||
| name: "pts_monotonic", | ||
| threshold: "true", | ||
| measured: candidate.observed.ptsMonotonic, | ||
| status: candidate.observed.ptsMonotonic ? "pass" : "fail", | ||
| }, | ||
| { | ||
| name: "codec_match", | ||
| threshold: profile.codec, | ||
| measured: candidate.observed.codec, | ||
| status: candidateRuns.every((run) => run.decode?.codec === profile.codec) | ||
| ? "pass" | ||
| : "fail", | ||
| }, | ||
| { | ||
| name: "dimensions_match_request", | ||
| threshold: `${expectedDimensions.width}x${expectedDimensions.height}`, | ||
| measured: candidate.observed.dimensions | ||
| ? `${candidate.observed.dimensions.width}x${candidate.observed.dimensions.height}` | ||
| : null, | ||
| status: candidateRuns.every((run) => | ||
| dimensionsEqual(run.decode?.dimensions, expectedDimensions), | ||
| ) | ||
| ? "pass" | ||
| : "fail", | ||
| }, | ||
| ]; | ||
| }; | ||
|
|
||
| export const statusFromGates = ( | ||
| gates: Gate[], | ||
| runs: RunResult[], | ||
|
|
||
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Adding an unrelated label while a benchmark runs can cancel it. Every
labeledevent shares this workflow-levelconcurrencygroup. Cancellation happens before the job-levelbenchcheck skips the new run, leaving the PR without benchmark results.Move
concurrencyinto the gated job, or include the label in the group.