diff --git a/.gitattributes b/.gitattributes
index 628472cde462..6d46a68bfcee 100644
--- a/.gitattributes
+++ b/.gitattributes
@@ -1,2 +1,3 @@
*.json linguist-language=JSON-with-Comments
+*.ipynb text eol=lf
**/pnpm-lock.yaml text eol=lf
diff --git a/.github/workflows/typecheck_benchmark_pr.yml b/.github/workflows/typecheck_benchmark_pr.yml
index f455338a8028..61f00a3c4452 100644
--- a/.github/workflows/typecheck_benchmark_pr.yml
+++ b/.github/workflows/typecheck_benchmark_pr.yml
@@ -1,5 +1,5 @@
-name: Type checker benchmark
-run-name: 'Type checker benchmark for PR #${{ inputs.pr_number }}'
+name: Type checker benchmark candidate
+run-name: 'Type checker benchmark candidate for PR #${{ github.event.pull_request.number }}'
env:
BENCHMARK_RUNNER_CLASS: 'github-ubuntu-latest'
@@ -8,41 +8,54 @@ env:
PYTHON_VERSION: '3.14.6'
on:
- workflow_dispatch:
- inputs:
- pr_number:
- description: Pull request number
- required: true
- type: string
- head_sha:
- description: Pull request head commit
- required: true
- type: string
- base_sha:
- description: Pull request base commit
- required: true
- type: string
- merge_sha:
- description: Pull request merge commit
- required: true
- type: string
+ pull_request:
+ types:
+ - opened
+ - reopened
+ - synchronize
+
+permissions:
+ contents: read
+ issues: read
concurrency:
- group: ${{ github.workflow }}-${{ inputs.pr_number }}
+ group: ${{ github.workflow }}-${{ github.event.pull_request.number }}
cancel-in-progress: true
jobs:
- benchmark:
- name: Compare Pyright performance
+ authorize:
+ name: Check benchmark request
+ runs-on: ubuntu-latest
+ outputs:
+ requested: ${{ steps.request.outputs.requested }}
+ steps:
+ - name: Check request label
+ id: request
+ uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0
+ env:
+ RUN_ATTEMPT: ${{ github.run_attempt }}
+ with:
+ script: |
+ const pullRequest = await github.rest.pulls.get({
+ owner: context.repo.owner,
+ repo: context.repo.repo,
+ pull_number: context.issue.number,
+ })
+ const requested = Number(process.env.RUN_ATTEMPT) > 1 && pullRequest.data.labels.some(
+ (label) => label.name === 'benchmark-requested'
+ )
+ core.setOutput('requested', requested)
+
+ candidate-benchmark:
+ name: Benchmark pull request merge
+ needs: authorize
+ if: ${{ needs.authorize.outputs.requested == 'true' }}
runs-on: ubuntu-latest
timeout-minutes: 180
- permissions:
- contents: read
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
with:
- ref: ${{ inputs.merge_sha }}
persist-credentials: false
- uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6
@@ -69,34 +82,11 @@ jobs:
SKIP_LERNA_BOOTSTRAP: 'yes'
run: pnpm install --frozen-lockfile --prefer-offline
- - name: Check out trusted baseline
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
- with:
- ref: ${{ inputs.base_sha }}
- path: benchmark-baseline
- sparse-checkout: build/benchmark/baselines
- persist-credentials: false
-
- - name: Select benchmark baseline
- id: baseline
- run: |
- trusted=benchmark-baseline/build/benchmark/baselines/latest-linux-x64.json
- bootstrap=build/benchmark/baselines/latest-linux-x64.json
- if [[ -f "$trusted" ]]; then
- echo "path=$trusted" >> "$GITHUB_OUTPUT"
- elif [[ -f "$bootstrap" ]]; then
- echo "Using the pull request's initial benchmark baseline"
- echo "path=$bootstrap" >> "$GITHUB_OUTPUT"
- else
- echo "No benchmark baseline is available" >&2
- exit 1
- fi
-
- name: Build Pyright CLI
working-directory: packages/pyright
run: pnpm run build
- - name: Run benchmark
+ - name: Benchmark pull request merge
env:
NODE_OPTIONS: '--max-old-space-size=6656'
PYTHONNOUSERSITE: '1'
@@ -104,176 +94,45 @@ jobs:
python build/benchmark/typecheck_benchmark.py \
-c pyright -r 1 -w 0 -t 1800 --memory-limit-mb 8192 \
--skip-pyright-build --os-name linux-x64 \
- --output build/benchmark/results
-
- - name: Compare with baseline
- id: comparison
- continue-on-error: true
- run: |
- set +e
- python build/benchmark/compare_benchmarks.py \
- "${{ steps.baseline.outputs.path }}" \
- build/benchmark/results/latest-linux-x64.json \
- --fail-on-preparation-error \
- --markdown-output build/benchmark/results/report.md
- comparison_status=$?
- cat build/benchmark/results/report.md >> "$GITHUB_STEP_SUMMARY"
- exit "$comparison_status"
-
- - name: Upload candidate results
- if: always()
- uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
- with:
- name: typecheck-benchmark-linux-x64-pr-${{ inputs.pr_number }}-head-${{ inputs.head_sha }}-base-${{ inputs.base_sha }}-merge-${{ inputs.merge_sha }}
- path: build/benchmark/results/
+ --output build/benchmark/candidate-results
- - name: Fail on benchmark regressions
- if: ${{ steps.comparison.outcome == 'failure' }}
- run: exit 1
-
- comment:
- name: Comment benchmark results
- needs: benchmark
- if: ${{ always() }}
- runs-on: ubuntu-latest
- permissions:
- actions: read
- contents: read
- pull-requests: write
- steps:
- - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
- with:
- ref: ${{ github.event.repository.default_branch }}
- persist-credentials: false
-
- - name: Download benchmark results
- id: download
- uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0
+ - name: Record candidate revisions and profile
env:
- PR_NUMBER: ${{ inputs.pr_number }}
- EXPECTED_HEAD_SHA: ${{ inputs.head_sha }}
- EXPECTED_BASE_SHA: ${{ inputs.base_sha }}
- EXPECTED_MERGE_SHA: ${{ inputs.merge_sha }}
- with:
- script: |
- const fs = require('fs')
- const issueNumber = Number(process.env.PR_NUMBER)
- const expectedHeadSha = process.env.EXPECTED_HEAD_SHA
- const expectedBaseSha = process.env.EXPECTED_BASE_SHA
- const expectedMergeSha = process.env.EXPECTED_MERGE_SHA
- const shaPattern = /^[0-9a-f]{40}$/
- if (
- !Number.isSafeInteger(issueNumber) ||
- issueNumber <= 0 ||
- !shaPattern.test(expectedHeadSha) ||
- !shaPattern.test(expectedBaseSha) ||
- !shaPattern.test(expectedMergeSha)
- ) {
- core.setFailed('The benchmark dispatch inputs are invalid')
- return
- }
- const artifactName =
- `typecheck-benchmark-linux-x64-pr-${issueNumber}` +
- `-head-${expectedHeadSha}-base-${expectedBaseSha}-merge-${expectedMergeSha}`
- const artifacts = await github.paginate(github.rest.actions.listWorkflowRunArtifacts, {
- owner: context.repo.owner,
- repo: context.repo.repo,
- run_id: context.runId,
- per_page: 100,
- })
- const reports = artifacts.filter((artifact) => artifact.name === artifactName)
- if (reports.length !== 1) {
- core.setFailed(`Expected one benchmark report artifact, found ${reports.length}`)
- return
- }
- const pullRequest = await github.rest.pulls.get({
- owner: context.repo.owner,
- repo: context.repo.repo,
- pull_number: issueNumber,
- })
- if (pullRequest.data.head.sha !== expectedHeadSha) {
- core.setFailed('The workflow run head does not match the pull request head')
- return
- }
- if (pullRequest.data.base.sha !== expectedBaseSha) {
- core.setFailed('The workflow run base does not match the pull request base')
- return
- }
- if (pullRequest.data.merge_commit_sha !== expectedMergeSha) {
- core.setFailed('The workflow run merge commit does not match the pull request merge commit')
- return
- }
- const download = await github.rest.actions.downloadArtifact({
- owner: context.repo.owner,
- repo: context.repo.repo,
- artifact_id: reports[0].id,
- archive_format: 'zip',
- })
- fs.writeFileSync('benchmark-report.zip', Buffer.from(download.data))
- core.setOutput('pr-number', issueNumber)
-
- - name: Extract candidate results
- if: ${{ steps.download.outputs.pr-number != '' }}
+ BASE_SHA: ${{ github.event.pull_request.base.sha }}
+ HEAD_SHA: ${{ github.event.pull_request.head.sha }}
+ MERGE_SHA: ${{ github.sha }}
run: |
python - <<'PY'
+ import hashlib
import json
- import zipfile
- from pathlib import PurePosixPath
-
- def reject_constant(value):
- raise ValueError(f'Non-finite JSON number: {value}')
-
- with zipfile.ZipFile('benchmark-report.zip') as archive:
- matches = [
- info
- for info in archive.infolist()
- if PurePosixPath(info.filename).name == 'latest-linux-x64.json'
- and '..' not in PurePosixPath(info.filename).parts
- ]
- if len(matches) != 1:
- raise RuntimeError(f'Expected one candidate result, found {len(matches)}')
- if matches[0].file_size > 5 * 1024 * 1024:
- raise RuntimeError('Candidate result exceeds 5 MB')
- contents = archive.read(matches[0])
- data = json.loads(contents, parse_constant=reject_constant)
- if not isinstance(data, dict):
- raise RuntimeError('Candidate result must be a JSON object')
- with open('candidate.json', 'wb') as output:
- output.write(contents)
+ import os
+ import subprocess
+ from pathlib import Path
+
+ path = Path('build/benchmark/candidate-results/latest-linux-x64.json')
+ data = json.loads(path.read_text(encoding='utf-8'))
+ data['source_revision'] = os.environ['MERGE_SHA']
+ data['source_head_revision'] = os.environ['HEAD_SHA']
+ data['source_base_revision'] = os.environ['BASE_SHA']
+ data['source_commit_subject'] = subprocess.check_output(
+ ['git', 'show', '-s', '--format=%s', os.environ['MERGE_SHA']], text=True
+ ).strip()
+ data['source_commit_timestamp'] = subprocess.check_output(
+ ['git', 'show', '-s', '--format=%cI', os.environ['MERGE_SHA']], text=True
+ ).strip()
+ profile = hashlib.sha256()
+ for profile_path in (
+ Path('build/benchmark/typecheck_benchmark.py'),
+ Path('build/benchmark/install_envs.json'),
+ ):
+ profile.update(profile_path.read_bytes())
+ data['benchmark_profile_hash'] = profile.hexdigest()
+ path.write_text(json.dumps(data, indent=2) + '\n', encoding='utf-8')
PY
- - name: Render benchmark report
- if: ${{ steps.download.outputs.pr-number != '' }}
- run: |
- set +e
- python build/benchmark/compare_benchmarks.py \
- build/benchmark/baselines/latest-linux-x64.json \
- candidate.json \
- --fail-on-preparation-error \
- --markdown-output report.md
- if [[ ! -s report.md ]]; then
- printf '%s\n' \
- '## Type checker benchmark' \
- '' \
- 'š“ **The benchmark results could not be compared. See the workflow run for details.**' \
- > report.md
- fi
-
- - name: Post benchmark comment
- if: ${{ steps.download.outputs.pr-number != '' }}
- uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0
- env:
- PR_NUMBER: ${{ steps.download.outputs.pr-number }}
+ - name: Upload candidate result
+ uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
with:
- script: |
- const fs = require('fs')
- const marker = ''
- const report = fs.readFileSync('report.md', 'utf8')
- const issueNumber = Number(process.env.PR_NUMBER)
- const body = `${marker}\n${report}`
- await github.rest.issues.createComment({
- owner: context.repo.owner,
- repo: context.repo.repo,
- issue_number: issueNumber,
- body,
- })
+ name: typecheck-benchmark-candidate
+ path: build/benchmark/candidate-results/latest-linux-x64.json
+ retention-days: 1
diff --git a/.github/workflows/typecheck_benchmark_report.yml b/.github/workflows/typecheck_benchmark_report.yml
new file mode 100644
index 000000000000..82a6ca87cb97
--- /dev/null
+++ b/.github/workflows/typecheck_benchmark_report.yml
@@ -0,0 +1,502 @@
+name: Type checker benchmark report
+run-name: 'Type checker benchmark report for run #${{ github.event.workflow_run.id }}'
+
+env:
+ BENCHMARK_RUNNER_CLASS: 'github-ubuntu-latest'
+ NODE_VERSION: '24.15.0'
+ PNPM_VERSION: '10.12.2'
+ PYTHON_VERSION: '3.14.6'
+
+on:
+ workflow_run:
+ workflows:
+ - Type checker benchmark candidate
+ types:
+ - completed
+
+permissions: {}
+
+concurrency:
+ group: ${{ github.workflow }}-${{ github.event.workflow_run.id }}
+ cancel-in-progress: false
+
+jobs:
+ metadata:
+ name: Validate pull request revisions
+ if: ${{ github.event.workflow_run.event == 'pull_request' && github.event.workflow_run.run_attempt > 1 }}
+ runs-on: ubuntu-latest
+ permissions:
+ pull-requests: read
+ outputs:
+ pr-number: ${{ steps.pull-request.outputs.pr-number }}
+ head-sha: ${{ steps.pull-request.outputs.head-sha }}
+ head-ref: ${{ steps.pull-request.outputs.head-ref }}
+ head-repository: ${{ steps.pull-request.outputs.head-repository }}
+ merge-sha: ${{ steps.pull-request.outputs.merge-sha }}
+ steps:
+ - name: Validate pull request metadata
+ id: pull-request
+ uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0
+ with:
+ script: |
+ const workflowRun = context.payload.workflow_run
+ let pullRequests = workflowRun.pull_requests
+ if (pullRequests.length === 0) {
+ const associated = await github.rest.repos.listPullRequestsAssociatedWithCommit({
+ owner: context.repo.owner,
+ repo: context.repo.repo,
+ commit_sha: workflowRun.head_sha,
+ })
+ pullRequests = associated.data.filter((pullRequest) => pullRequest.state === 'open')
+ }
+ if (pullRequests.length !== 1) {
+ core.setFailed(`Expected one pull request for the benchmark run, found ${pullRequests.length}`)
+ return
+ }
+ const issueNumber = pullRequests[0].number
+ const pullRequest = await github.rest.pulls.get({
+ owner: context.repo.owner,
+ repo: context.repo.repo,
+ pull_number: issueNumber,
+ })
+ if (
+ pullRequest.data.state !== 'open' ||
+ pullRequest.data.head.sha !== workflowRun.head_sha ||
+ pullRequest.data.base.sha !== context.sha ||
+ !pullRequest.data.merge_commit_sha
+ ) {
+ core.setFailed('The pull request changed after the benchmark was requested')
+ return
+ }
+ core.setOutput('pr-number', issueNumber)
+ core.setOutput('head-sha', pullRequest.data.head.sha)
+ core.setOutput('head-ref', pullRequest.data.head.ref)
+ core.setOutput('head-repository', pullRequest.data.head.repo.full_name)
+ core.setOutput('merge-sha', pullRequest.data.merge_commit_sha)
+
+ base-benchmark:
+ name: Benchmark main commit
+ needs: metadata
+ if: ${{ github.event.workflow_run.conclusion == 'success' }}
+ runs-on: ubuntu-latest
+ timeout-minutes: 180
+ permissions:
+ contents: read
+ outputs:
+ cached: ${{ steps.cached-result.outputs.valid }}
+
+ steps:
+ - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
+ with:
+ ref: ${{ github.sha }}
+ persist-credentials: false
+
+ - uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6
+ with:
+ python-version: ${{ env.PYTHON_VERSION }}
+
+ - name: Determine base cache key
+ id: cache-key
+ shell: bash
+ env:
+ BASE_SHA: ${{ github.sha }}
+ run: |
+ profile="${ImageOS}-${ImageVersion}-$(uname -m)-cpu$(nproc)-python${PYTHON_VERSION}-node${NODE_VERSION}-pnpm${PNPM_VERSION}-r1-w0-m8192-heap6656"
+ profile="$(printf '%s' "$profile" | tr -c 'A-Za-z0-9_.-' '-')"
+ echo "value=typecheck-benchmark-base-v2-${BASE_SHA}-${profile}" >> "$GITHUB_OUTPUT"
+
+ - name: Restore cached base result
+ id: base-cache
+ uses: actions/cache/restore@0057852bfaa89a56745cba8c7296529d2fc39830 # v4
+ with:
+ path: build/benchmark/base-results/latest-linux-x64.json
+ key: ${{ steps.cache-key.outputs.value }}
+
+ - name: Validate cached base result
+ id: cached-result
+ shell: bash
+ env:
+ BASE_SHA: ${{ github.sha }}
+ run: |
+ result=build/benchmark/base-results/latest-linux-x64.json
+ if [[ "${{ steps.base-cache.outputs.cache-hit }}" == "true" ]] && \
+ python -c "import hashlib,json,sys; data=json.load(open(sys.argv[1], encoding='utf-8')); digest=hashlib.sha256(b''.join(open(path, 'rb').read() for path in sys.argv[3:])).hexdigest(); sys.exit(data.get('source_revision') != sys.argv[2] or not data.get('source_commit_subject') or not data.get('source_commit_timestamp') or data.get('benchmark_profile_hash') != digest)" \
+ "$result" "$BASE_SHA" build/benchmark/typecheck_benchmark.py \
+ build/benchmark/install_envs.json && \
+ python build/benchmark/compare_benchmarks.py "$result" "$result" \
+ --fail-on-preparation-error; then
+ echo "valid=true" >> "$GITHUB_OUTPUT"
+ else
+ rm -f "$result"
+ echo "valid=false" >> "$GITHUB_OUTPUT"
+ fi
+
+ - if: ${{ steps.cached-result.outputs.valid != 'true' }}
+ uses: pnpm/action-setup@f520eceda224fe1a4aed5a2a27a194379a409996 # v6
+ with:
+ version: ${{ env.PNPM_VERSION }}
+
+ - if: ${{ steps.cached-result.outputs.valid != 'true' }}
+ uses: actions/setup-node@249970729cb0ef3589644e2896645e5dc5ba9c38 # v6.5.0
+ with:
+ node-version: ${{ env.NODE_VERSION }}
+
+ - name: Verify build prerequisites
+ if: ${{ steps.cached-result.outputs.valid != 'true' }}
+ run: |
+ gcc --version
+ g++ --version
+ make --version
+
+ - name: Install JavaScript dependencies
+ if: ${{ steps.cached-result.outputs.valid != 'true' }}
+ timeout-minutes: 10
+ env:
+ SKIP_LERNA_BOOTSTRAP: 'yes'
+ run: pnpm install --frozen-lockfile --prefer-offline
+
+ - name: Build Pyright CLI
+ if: ${{ steps.cached-result.outputs.valid != 'true' }}
+ working-directory: packages/pyright
+ run: pnpm run build
+
+ - name: Benchmark main commit
+ if: ${{ steps.cached-result.outputs.valid != 'true' }}
+ env:
+ NODE_OPTIONS: '--max-old-space-size=6656'
+ PYTHONNOUSERSITE: '1'
+ run: |
+ python build/benchmark/typecheck_benchmark.py \
+ -c pyright -r 1 -w 0 -t 1800 --memory-limit-mb 8192 \
+ --skip-pyright-build --os-name linux-x64 \
+ --output build/benchmark/base-results
+
+ - name: Record base revision and profile
+ if: ${{ steps.cached-result.outputs.valid != 'true' }}
+ env:
+ BASE_SHA: ${{ github.sha }}
+ run: |
+ python - <<'PY'
+ import hashlib
+ import json
+ import os
+ import subprocess
+ from pathlib import Path
+
+ path = Path('build/benchmark/base-results/latest-linux-x64.json')
+ data = json.loads(path.read_text(encoding='utf-8'))
+ data['source_revision'] = os.environ['BASE_SHA']
+ data['source_commit_subject'] = subprocess.check_output(
+ ['git', 'show', '-s', '--format=%s', os.environ['BASE_SHA']], text=True
+ ).strip()
+ data['source_commit_timestamp'] = subprocess.check_output(
+ ['git', 'show', '-s', '--format=%cI', os.environ['BASE_SHA']], text=True
+ ).strip()
+ profile = hashlib.sha256()
+ for profile_path in (
+ Path('build/benchmark/typecheck_benchmark.py'),
+ Path('build/benchmark/install_envs.json'),
+ ):
+ profile.update(profile_path.read_bytes())
+ data['benchmark_profile_hash'] = profile.hexdigest()
+ path.write_text(json.dumps(data, indent=2) + '\n', encoding='utf-8')
+ PY
+
+ - name: Validate base result
+ env:
+ BASE_SHA: ${{ github.sha }}
+ run: |
+ python build/benchmark/compare_benchmarks.py \
+ build/benchmark/base-results/latest-linux-x64.json \
+ build/benchmark/base-results/latest-linux-x64.json \
+ --fail-on-preparation-error \
+ --baseline-revision "$BASE_SHA" --candidate-revision "$BASE_SHA"
+
+ - name: Save base result cache
+ if: ${{ steps.cached-result.outputs.valid != 'true' }}
+ uses: actions/cache/save@0057852bfaa89a56745cba8c7296529d2fc39830 # v4
+ with:
+ path: build/benchmark/base-results/latest-linux-x64.json
+ key: ${{ steps.cache-key.outputs.value }}
+
+ - name: Upload base result
+ uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
+ with:
+ name: typecheck-benchmark-base
+ path: build/benchmark/base-results/latest-linux-x64.json
+ retention-days: 1
+
+ comparison:
+ name: Compare Pyright performance
+ needs: [metadata, base-benchmark]
+ runs-on: ubuntu-latest
+ permissions:
+ actions: read
+ contents: read
+
+ steps:
+ - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
+ with:
+ ref: ${{ github.sha }}
+ persist-credentials: false
+
+ - name: Download base result
+ uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
+ with:
+ name: typecheck-benchmark-base
+ path: benchmark-report/base
+
+ - name: Download candidate result
+ uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
+ with:
+ name: typecheck-benchmark-candidate
+ path: benchmark-report/candidate
+ run-id: ${{ github.event.workflow_run.id }}
+ github-token: ${{ secrets.GITHUB_TOKEN }}
+
+ - name: Validate candidate provenance
+ env:
+ BASE_SHA: ${{ github.sha }}
+ HEAD_SHA: ${{ needs.metadata.outputs.head-sha }}
+ MERGE_SHA: ${{ needs.metadata.outputs.merge-sha }}
+ run: |
+ python - <<'PY'
+ import json
+ import os
+ from pathlib import Path
+
+ path = Path('benchmark-report/candidate/latest-linux-x64.json')
+ if path.stat().st_size > 5 * 1024 * 1024:
+ raise RuntimeError('Candidate benchmark result exceeds 5 MB')
+ data = json.loads(path.read_text(encoding='utf-8'), parse_constant=lambda value: (_ for _ in ()).throw(ValueError(value)))
+ if not isinstance(data, dict):
+ raise RuntimeError('Candidate benchmark result must be an object')
+ expected = {
+ 'source_revision': os.environ['MERGE_SHA'],
+ 'source_head_revision': os.environ['HEAD_SHA'],
+ 'source_base_revision': os.environ['BASE_SHA'],
+ }
+ for field, value in expected.items():
+ if data.get(field) != value:
+ raise RuntimeError(f'Candidate {field} does not match the pull request')
+ PY
+
+ - name: Compare with main commit
+ id: comparison
+ continue-on-error: true
+ env:
+ BASE_SHA: ${{ github.sha }}
+ MERGE_SHA: ${{ needs.metadata.outputs.merge-sha }}
+ run: |
+ set +e
+ python build/benchmark/compare_benchmarks.py \
+ benchmark-report/base/latest-linux-x64.json \
+ benchmark-report/candidate/latest-linux-x64.json \
+ --fail-on-preparation-error \
+ --allow-incompatible \
+ --baseline-revision "$BASE_SHA" --candidate-revision "$MERGE_SHA" \
+ --markdown-output benchmark-report/report.md
+ comparison_status=$?
+ cat benchmark-report/report.md >> "$GITHUB_STEP_SUMMARY"
+ exit "$comparison_status"
+
+ - name: Upload benchmark report
+ if: always()
+ uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
+ with:
+ name: typecheck-benchmark-report-${{ github.event.workflow_run.id }}
+ path: benchmark-report/
+
+ - name: Fail on benchmark regressions
+ if: ${{ steps.comparison.outcome == 'failure' }}
+ run: exit 1
+
+ comment:
+ name: Comment benchmark results
+ needs: [metadata, comparison]
+ if: ${{ always() && needs.metadata.result == 'success' }}
+ runs-on: ubuntu-latest
+ permissions:
+ actions: read
+ pull-requests: write
+ steps:
+ - name: Download benchmark report
+ if: ${{ needs.comparison.result == 'success' || needs.comparison.result == 'failure' }}
+ uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
+ with:
+ name: typecheck-benchmark-report-${{ github.event.workflow_run.id }}
+ path: benchmark-report
+
+ - name: Post benchmark comment
+ uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0
+ env:
+ PR_NUMBER: ${{ needs.metadata.outputs.pr-number }}
+ CANDIDATE_CONCLUSION: ${{ github.event.workflow_run.conclusion }}
+ COMPARISON_RESULT: ${{ needs.comparison.result }}
+ with:
+ script: |
+ const fs = require('fs')
+ const marker = ''
+ const issueNumber = Number(process.env.PR_NUMBER)
+ let report
+ if (process.env.CANDIDATE_CONCLUSION !== 'success') {
+ report = '## Type checker benchmark\n\nš“ **The candidate benchmark failed. See the workflow run for details.**\n'
+ } else if (process.env.COMPARISON_RESULT === 'success' || process.env.COMPARISON_RESULT === 'failure') {
+ report = fs.readFileSync('benchmark-report/report.md', 'utf8')
+ } else {
+ report = '## Type checker benchmark\n\nš“ **The trusted base benchmark or report failed. See the workflow run for details.**\n'
+ }
+ const body = `${marker}\n${report}`
+ const comments = await github.paginate(github.rest.issues.listComments, {
+ owner: context.repo.owner,
+ repo: context.repo.repo,
+ issue_number: issueNumber,
+ per_page: 100,
+ })
+ const previous = comments.find(
+ (comment) =>
+ comment.user?.login === 'github-actions[bot]' && comment.body?.startsWith(marker)
+ )
+ if (previous) {
+ await github.rest.issues.updateComment({
+ owner: context.repo.owner,
+ repo: context.repo.repo,
+ comment_id: previous.id,
+ body,
+ })
+ } else {
+ await github.rest.issues.createComment({
+ owner: context.repo.owner,
+ repo: context.repo.repo,
+ issue_number: issueNumber,
+ body,
+ })
+ }
+
+ persist-base-result:
+ name: Update pull request baseline
+ needs: [metadata, base-benchmark, comment]
+ if: >-
+ ${{ !cancelled() && needs.base-benchmark.result == 'success' &&
+ needs.base-benchmark.outputs.cached != 'true' &&
+ needs.metadata.outputs.head-repository == github.repository }}
+ runs-on: ubuntu-latest
+ permissions:
+ actions: read
+ contents: write
+ steps:
+ - name: Download base result
+ uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
+ with:
+ name: typecheck-benchmark-base
+ path: benchmark-result
+
+ - name: Commit baseline to pull request branch
+ uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0
+ env:
+ EXPECTED_HEAD_SHA: ${{ needs.metadata.outputs.head-sha }}
+ HEAD_REF: ${{ needs.metadata.outputs.head-ref }}
+ BASE_SHA: ${{ github.sha }}
+ with:
+ script: |
+ const fs = require('fs')
+ const contents = fs.readFileSync('benchmark-result/latest-linux-x64.json', 'utf8')
+ const result = JSON.parse(contents)
+ const expectedHeadSha = process.env.EXPECTED_HEAD_SHA
+ const headRef = process.env.HEAD_REF
+ const baseSha = process.env.BASE_SHA
+ if (
+ result.source_revision !== baseSha ||
+ typeof result.source_commit_subject !== 'string' ||
+ result.source_commit_subject.length === 0 ||
+ Number.isNaN(Date.parse(result.source_commit_timestamp)) ||
+ Number.isNaN(Date.parse(result.timestamp)) ||
+ !/^\d{4}-\d{2}-\d{2}$/.test(result.date)
+ ) {
+ core.setFailed('The base benchmark result metadata is invalid')
+ return
+ }
+ const currentRef = await github.rest.git.getRef({
+ owner: context.repo.owner,
+ repo: context.repo.repo,
+ ref: `heads/${headRef}`,
+ })
+ if (currentRef.data.object.sha !== expectedHeadSha) {
+ core.notice('The pull request head changed; skipping the baseline update')
+ return
+ }
+ const parent = await github.rest.git.getCommit({
+ owner: context.repo.owner,
+ repo: context.repo.repo,
+ commit_sha: expectedHeadSha,
+ })
+ const blob = await github.rest.git.createBlob({
+ owner: context.repo.owner,
+ repo: context.repo.repo,
+ content: Buffer.from(contents).toString('base64'),
+ encoding: 'base64',
+ })
+ const tree = await github.rest.git.createTree({
+ owner: context.repo.owner,
+ repo: context.repo.repo,
+ base_tree: parent.data.tree.sha,
+ tree: [
+ {
+ path: 'build/benchmark/baselines/latest-linux-x64.json',
+ mode: '100644',
+ type: 'blob',
+ sha: blob.data.sha,
+ },
+ {
+ path: `build/benchmark/baselines/benchmark_${result.date}_linux-x64.json`,
+ mode: '100644',
+ type: 'blob',
+ sha: blob.data.sha,
+ },
+ ],
+ })
+ const commit = await github.rest.git.createCommit({
+ owner: context.repo.owner,
+ repo: context.repo.repo,
+ message: `Update benchmark baseline for ${baseSha.slice(0, 12)}`,
+ tree: tree.data.sha,
+ parents: [expectedHeadSha],
+ })
+ await github.rest.git.updateRef({
+ owner: context.repo.owner,
+ repo: context.repo.repo,
+ ref: `heads/${headRef}`,
+ sha: commit.data.sha,
+ force: false,
+ })
+
+ cleanup:
+ name: Clear benchmark request
+ if: >-
+ ${{ always() && github.event.workflow_run.event == 'pull_request' &&
+ github.event.workflow_run.run_attempt > 1 }}
+ needs: [metadata, base-benchmark, comparison, comment, persist-base-result]
+ runs-on: ubuntu-latest
+ permissions:
+ issues: write
+ steps:
+ - name: Remove request label
+ uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0
+ with:
+ script: |
+ const pullRequests = context.payload.workflow_run.pull_requests
+ const issueNumber = Number('${{ needs.metadata.outputs.pr-number }}') || pullRequests[0]?.number
+ if (!issueNumber) {
+ core.notice('No pull request was associated with the benchmark run')
+ return
+ }
+ try {
+ await github.rest.issues.removeLabel({
+ owner: context.repo.owner,
+ repo: context.repo.repo,
+ issue_number: issueNumber,
+ name: 'benchmark-requested',
+ })
+ } catch (error) {
+ if (error.status !== 404) throw error
+ }
diff --git a/.github/workflows/typecheck_benchmark_trigger.yml b/.github/workflows/typecheck_benchmark_trigger.yml
index c8481d9b8392..e39c87c080f4 100644
--- a/.github/workflows/typecheck_benchmark_trigger.yml
+++ b/.github/workflows/typecheck_benchmark_trigger.yml
@@ -7,7 +7,7 @@ on:
permissions:
actions: write
- contents: read
+ issues: write
pull-requests: read
jobs:
@@ -36,24 +36,70 @@ jobs:
return
}
+ const label = 'benchmark-requested'
const pullRequest = await github.rest.pulls.get({
owner: context.repo.owner,
repo: context.repo.repo,
pull_number: context.issue.number,
})
if (!pullRequest.data.merge_commit_sha) {
- core.setFailed('The pull request does not have a merge commit to benchmark')
+ core.setFailed('The pull request must be mergeable before it can be benchmarked')
return
}
- await github.rest.actions.createWorkflowDispatch({
+ let candidateRun
+ for (let attempt = 0; attempt < 12; attempt++) {
+ const runs = await github.rest.actions.listWorkflowRuns({
+ owner: context.repo.owner,
+ repo: context.repo.repo,
+ workflow_id: 'typecheck_benchmark_pr.yml',
+ event: 'pull_request',
+ head_sha: pullRequest.data.head.sha,
+ per_page: 100,
+ })
+ candidateRun = runs.data.workflow_runs.find(
+ (run) => run.pull_requests.some((item) => item.number === context.issue.number)
+ )
+ if (candidateRun?.status === 'completed') break
+ await new Promise((resolve) => setTimeout(resolve, 5000))
+ }
+ if (!candidateRun || candidateRun.status !== 'completed') {
+ core.setFailed('The pull request validation workflow is not ready; retry /benchmark shortly')
+ return
+ }
+ try {
+ await github.rest.issues.getLabel({
+ owner: context.repo.owner,
+ repo: context.repo.repo,
+ name: label,
+ })
+ } catch (error) {
+ if (error.status !== 404) throw error
+ await github.rest.issues.createLabel({
+ owner: context.repo.owner,
+ repo: context.repo.repo,
+ name: label,
+ color: '1d76db',
+ description: 'Run the hosted type checker benchmark',
+ })
+ }
+ try {
+ await github.rest.issues.removeLabel({
+ owner: context.repo.owner,
+ repo: context.repo.repo,
+ issue_number: context.issue.number,
+ name: label,
+ })
+ } catch (error) {
+ if (error.status !== 404) throw error
+ }
+ await github.rest.issues.addLabels({
owner: context.repo.owner,
repo: context.repo.repo,
- workflow_id: 'typecheck_benchmark_pr.yml',
- ref: context.payload.repository.default_branch,
- inputs: {
- pr_number: String(context.issue.number),
- head_sha: pullRequest.data.head.sha,
- base_sha: pullRequest.data.base.sha,
- merge_sha: pullRequest.data.merge_commit_sha,
- },
+ issue_number: context.issue.number,
+ labels: [label],
+ })
+ await github.rest.actions.reRunWorkflow({
+ owner: context.repo.owner,
+ repo: context.repo.repo,
+ run_id: candidateRun.id,
})
diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
index c9039f622791..54af8c11e182 100644
--- a/CONTRIBUTING.md
+++ b/CONTRIBUTING.md
@@ -21,5 +21,9 @@ See the script's `--help` and module docstring for options and methodology.
To compare the local Pyright build's speed and peak memory with the latest PyPI release or with
Pyrefly, ty, mypy, and Zuban on the pinned corpus, use `build/benchmark/typecheck_benchmark.py`.
Maintainers can also request the hosted regression benchmark on a pull request by commenting
-`/benchmark`. See [the benchmark README](build/benchmark/README.md) for the developer and maintainer
-workflows, prerequisites, and methodology.
+`/benchmark`. The hosted workflow compares the pull request's synthetic merge commit with its exact
+base commit and reuses a validated commit-keyed base result when available. Results are attached to
+the workflow and added to the existing pull-request comment. When it measures a previously uncached
+base commit, it also commits the new baseline files to a same-repository pull-request branch. See
+[the benchmark README](build/benchmark/README.md) for the developer and maintainer workflows, cache
+behavior, prerequisites, and methodology.
diff --git a/build/benchmark/README.md b/build/benchmark/README.md
index 16cb357b445f..28ad4b5377b1 100644
--- a/build/benchmark/README.md
+++ b/build/benchmark/README.md
@@ -107,16 +107,19 @@ result contract, and the comparator rejects mismatched environments.
The hosted pull-request benchmark runs only when a maintainer comments exactly `/benchmark` on an
open pull request. The command must be the entire comment. The trusted command workflow checks that
-the commenter has `write`, `maintain`, or `admin` repository permission and then explicitly dispatches
-the benchmark with the pull request's current head, base, and synthetic merge commits. Users without
-one of these permissions cannot start the benchmark.
-
-The dispatched workflow runs the pull request's synthetic merge commit, so all mergeable open pull
-requests use the current pnpm build metadata from the base branch. It uses read-only repository
-permissions. When the run completes, a separate trusted job validates the result's head, base, and
-merge commits, renders it using default-branch code, and creates or updates one benchmark comment.
-The trusted job runs on a separate runner with pull-request write permission. The Actions job summary
-and benchmark artifact contain the same candidate results.
+the commenter has `write`, `maintain`, or `admin` repository permission and then toggles the
+`benchmark-requested` label and reruns the current commit's existing pull-request workflow. Users
+without one of these permissions cannot start the benchmark. The initial workflow run for each pull
+request commit performs only the request check; measured work is restricted to authorized reruns.
+
+The authorized rerun executes an unprivileged `pull_request` workflow that runs the pull request's
+synthetic merge commit with read-only repository and issue permissions. It cannot read or write the
+shared benchmark cache. When it completes, a separate trusted `workflow_run` workflow accepts only
+rerun attempts, uses only default-branch code to validate the candidate artifact and pull request
+revisions, benchmarks or restores the exact base commit, renders the comparison, and creates or
+updates one benchmark comment. Candidate output is treated only as bounded JSON data and is never
+executed by the trusted workflow. The Actions job summary and benchmark artifact contain the same
+candidate results.
Comment `/benchmark` again after pushing a new commit or when rerunning the same head. No benchmark is
started automatically for later commits. The command workflow must already exist on the repository's
@@ -146,57 +149,78 @@ By default, results are written to the ignored `build/benchmark/results/` direct
writes a UTC-dated file such as `benchmark_2026-07-28.json` and updates `latest.json`. With
`--os-name macos`, the names are `benchmark_2026-07-28_macos.json` and `latest-macos.json`.
-Checked-in reference runs live in `build/benchmark/baselines/`. Generate a candidate with the same
-OS label, run count, warmup count, memory limit, Python version, and package commits as its baseline,
-then compare it before submitting a performance-sensitive pull request:
+Checked-in reference runs live in `build/benchmark/baselines/`. To compare two locally generated
+results, use the same OS label, run count, warmup count, memory limit, Python version, and package
+commits for both runs:
```console
-python build/benchmark/typecheck_benchmark.py \
- -c pyright -r 1 -w 0 -t 600 --memory-limit-mb 8192 \
- --os-name linux-x64 --output build/benchmark/results
python build/benchmark/compare_benchmarks.py \
- build/benchmark/baselines/latest-linux-x64.json \
- build/benchmark/results/latest-linux-x64.json
+ path/to/base-result.json path/to/candidate-result.json
```
+`build/benchmark/benchmark_history.ipynb` loads the dated checked-in baselines, graphs package timing
+and peak-memory trends, compares the latest run with an earlier commit, and can export a static
+dashboard under `docs/benchmark-results/`. Install its optional dependencies with
+`python -m pip install -r build/benchmark/requirements-notebook.txt`. Exports and result-file
+enrichment are disabled by default and can be enabled independently in the notebook configuration.
+
The comparator reports per-package timing and memory deltas and exits nonzero when a previously
successful result is missing, the environment contract differs, or a result exceeds the default 20%
regression threshold. Package commits, check paths, and excluded directory names must also match.
For Pyright, result JSON and Markdown reports also include the `--stats` file counts and phase timings
for source discovery, reads, tokenization, parsing, import resolution, binding, checking, and cycle
detection. This does not enable verbose or per-file logging.
-Candidate-only package/checker results are reported as not regression-gated and require a regenerated
-baseline before they are protected. Runner class, hosted runner image, CPU count, and the exact Python
-version are part of the environment contract. The dependency-isolation mode and exact `NODE_OPTIONS`
-value are also recorded and must match so results collected with different dependency or Node heap
+Candidate-only package/checker results are reported as not regression-gated. Runner class, hosted
+runner image, CPU count, and the exact Python version are part of the environment contract. The
+dependency-isolation mode, benchmark profile hash, and exact `NODE_OPTIONS` value are also recorded
+and must match so results collected with different corpus, benchmark code, dependency, or Node heap
settings cannot be compared. Use `--threshold-percent` to select another threshold. Results from
different runner classes are historical data, not a reliable regression gate.
The invocation timeout is recorded but is not an environment compatibility field: raising a kill
-threshold does not alter a checker invocation that completed below either threshold. A timed-out
-candidate cannot replace a successful baseline. If a candidate succeeds where the baseline timed out
-or otherwise failed, it is reported as `No baseline` until a new hosted baseline is checked in.
-The pull-request workflow also fails if any candidate package cannot be prepared or any checker
-times out, crashes, or otherwise fails, even when that package has no successful baseline yet.
-
-On pull requests, the benchmark pins Python 3.14.6, disables shared dependency caches because it
-executes untrusted pull-request code, runs Pyright with a 6.5 GiB V8 old-space limit, and runs each
-package once with no discarded warmup. Each invocation may take up to 30 minutes. A regression must
-exceed both a 20% relative threshold and an absolute variance guard of 1
-second for time or 100 MB for peak memory. These are the comparator defaults, so the gate and trusted
-comment renderer share one configuration source. Reports and artifacts are published before a failed
-comparison marks the job unsuccessful.
+threshold does not alter a checker invocation that completed below either threshold. The pull-request
+workflow fails if either revision cannot prepare a package or if Pyright times out, crashes, or
+otherwise fails.
+
+On pull requests, the workflow compares the synthetic merge commit with the exact default-branch
+commit validated against the pull request base. If the pull request or default branch changes before
+the trusted workflow validates those revisions, the report stops and a new `/benchmark` request is
+required. The base result is cached under an exact key containing its commit and hosted measurement
+profile. A missing, expired, or invalid cache entry causes a fresh base build and benchmark; prefix
+and fallback cache matches are not used. Only the job that checks out trusted default-branch code can
+populate this shared cache. The job that executes pull-request code cannot write it. A newly measured
+base result is also committed to the checked-in dated and `latest` baseline files on same-repository
+pull requests after confirming that the pull request head has not changed.
+
+Both revisions use Python 3.14.6, a 6.5 GiB V8 old-space limit, one measured run, no discarded warmup,
+and a 30-minute invocation timeout. A regression must exceed both a 20% relative threshold and an
+absolute variance guard of 1 second for time or 100 MB for peak memory. If a pull request changes the
+benchmark script, package corpus, package pins, checked paths, exclusions, or measurement environment,
+both revisions must still complete successfully, but the report marks them as not comparable and does
+not apply a performance regression gate. Once that change is merged, its commit becomes the base for
+later pull requests and is cached normally.
+
+The base result, candidate result, and comparison are attached to the workflow run and rendered in the
+Actions job summary and existing benchmark pull-request comment. When the exact base result was not
+already cached, the workflow commits it as both the dated baseline and `latest-linux-x64.json` on a
+same-repository pull-request branch. The write-scoped job uses the GitHub API without checking out or
+executing pull-request code, and skips the update if the pull-request head has changed. Fork pull
+requests retain the result as an artifact because the repository token cannot update their branches.
+The baseline commit changes the pull-request head and can retrigger push-based checks. The report
+comment cannot dispatch another benchmark: the trigger accepts only a newly created comment whose
+entire trimmed body is `/benchmark`, whereas later reports update the marker comment.
The weekly workflow runs Pyright, Pyrefly, ty, mypy, and Zuban in independent hosted-runner jobs.
Each checker performs three measured runs after one warmup over the same pinned corpus. The aggregate
job stores each raw JSON result with a self-contained `index.html` comparison for 90 days; its Actions
job summary links directly to the downloadable report artifact.
-The top-level JSON records the timestamp, platform, checker versions, run settings, aggregate
-statistics, per-package results, configured memory limit, and an `upstream_source` object containing
-the original repository, exact commit, and source-file URL. Each package result records the cloned
-repository commit. Each successful checker result contains the measured wall times and peak-memory
-values, plus min, max, mean, median, and standard deviation.
+The top-level JSON records the benchmark timestamp, benchmarked Pyright commit SHA, commit subject,
+commit timestamp, platform, checker versions, run settings, aggregate statistics, per-package
+results, configured memory limit, and an `upstream_source` object containing the original repository,
+exact commit, and source-file URL. Each package result records the cloned repository commit. Each
+successful checker result contains the measured wall times and peak-memory values, plus min, max,
+mean, median, and standard deviation.
Aggregate data contains package counts and mean, p50, p90, p95, maximum, and total timing or memory
statistics.
@@ -211,8 +235,8 @@ uncounted validation pass labeled `Check`; otherwise exactly the requested numbe
reported and discarded.
Dependency installation failures mark the package as unmeasured and skip its checker runs. The PR
-report displays these preparation failures, but they do not count as performance regressions. If none
-of a package's configured check paths exist, the benchmark warns and checks the full repository.
+benchmark treats these preparation failures as errors. If none of a package's configured check paths
+exist, the benchmark warns and checks the full repository.
## Relationship to `perfCompare.py`
diff --git a/build/benchmark/baselines/benchmark_2026-08-20_linux-x64.json b/build/benchmark/baselines/benchmark_2026-08-20_linux-x64.json
index 60a45469271b..e08a79c7f686 100644
--- a/build/benchmark/baselines/benchmark_2026-08-20_linux-x64.json
+++ b/build/benchmark/baselines/benchmark_2026-08-20_linux-x64.json
@@ -1,6 +1,9 @@
{
"timestamp": "2026-08-20T02:07:10.843078+00:00",
"date": "2026-08-20",
+ "source_revision": "075161cd9160dbefff81e8ec4b28a3be4786bfcb",
+ "source_commit_subject": "Increase benchmark heap headroom",
+ "source_commit_timestamp": "2026-08-19T18:10:07-07:00",
"platform": "linux",
"platform_details": "Linux-6.17.0-1022-azure-x86_64-with-glibc2.39",
"architecture": "x86_64",
diff --git a/build/benchmark/baselines/latest-linux-x64.json b/build/benchmark/baselines/latest-linux-x64.json
index 60a45469271b..e08a79c7f686 100644
--- a/build/benchmark/baselines/latest-linux-x64.json
+++ b/build/benchmark/baselines/latest-linux-x64.json
@@ -1,6 +1,9 @@
{
"timestamp": "2026-08-20T02:07:10.843078+00:00",
"date": "2026-08-20",
+ "source_revision": "075161cd9160dbefff81e8ec4b28a3be4786bfcb",
+ "source_commit_subject": "Increase benchmark heap headroom",
+ "source_commit_timestamp": "2026-08-19T18:10:07-07:00",
"platform": "linux",
"platform_details": "Linux-6.17.0-1022-azure-x86_64-with-glibc2.39",
"architecture": "x86_64",
diff --git a/build/benchmark/benchmark_history.ipynb b/build/benchmark/benchmark_history.ipynb
new file mode 100644
index 000000000000..28bdfcc939f7
--- /dev/null
+++ b/build/benchmark/benchmark_history.ipynb
@@ -0,0 +1,528 @@
+{
+ "cells": [
+ {
+ "cell_type": "markdown",
+ "id": "f974c0d2",
+ "metadata": {},
+ "source": [
+ "# Pyright benchmark history\n",
+ "\n",
+ "Explore checked-in Linux x64 benchmark results across Pyright commits, compare the latest run with a baseline, and optionally export charts and a static GitHub Pages dashboard.\n",
+ "\n",
+ "The workflow adds commit metadata when collecting hosted results. Notebook enrichment is dry-run by default; set `WRITE_ENRICHED_RESULTS = True` only for result files whose source commit is the current checkout."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "a7238571",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# 1. Import analysis and visualization libraries\n",
+ "from __future__ import annotations\n",
+ "\n",
+ "import html\n",
+ "import json\n",
+ "import os\n",
+ "import subprocess\n",
+ "from datetime import datetime\n",
+ "from pathlib import Path\n",
+ "from typing import Any\n",
+ "\n",
+ "import matplotlib.dates as mdates\n",
+ "import matplotlib.pyplot as plt\n",
+ "import pandas as pd\n",
+ "import seaborn as sns\n",
+ "from IPython.display import HTML, Markdown, display\n",
+ "\n",
+ "sns.set_theme(style=\"whitegrid\", context=\"notebook\")\n",
+ "plt.rcParams[\"figure.figsize\"] = (12, 6)\n",
+ "plt.rcParams[\"figure.dpi\"] = 120"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "eeedeada",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# 2. Define benchmark file and metadata schemas\n",
+ "REQUIRED_METADATA = {\n",
+ " \"timestamp\": str,\n",
+ " \"source_revision\": str,\n",
+ " \"source_commit_subject\": str,\n",
+ " \"source_commit_timestamp\": str,\n",
+ " \"results\": list,\n",
+ "}\n",
+ "\n",
+ "METRIC_SPECS = {\n",
+ " \"execution_time_s\": {\"label\": \"Execution time\", \"unit\": \"seconds\", \"lower_is_better\": True},\n",
+ " \"peak_memory_mb\": {\"label\": \"Peak memory\", \"unit\": \"MiB\", \"lower_is_better\": True},\n",
+ "}\n",
+ "\n",
+ "\n",
+ "def find_repo_root(start: Path) -> Path:\n",
+ " for candidate in (start.resolve(), *start.resolve().parents):\n",
+ " if (candidate / \"build\" / \"benchmark\" / \"baselines\").is_dir():\n",
+ " return candidate\n",
+ " raise FileNotFoundError(\"Could not find build/benchmark/baselines from the current directory\")\n",
+ "\n",
+ "\n",
+ "REPO_ROOT = find_repo_root(Path.cwd())\n",
+ "BASELINE_DIR = REPO_ROOT / \"build\" / \"benchmark\" / \"baselines\"\n",
+ "EXPORT_DIR = Path(\n",
+ " os.environ.get(\"PYRIGHT_BENCHMARK_EXPORT_DIR\", REPO_ROOT / \"docs\" / \"benchmark-results\")\n",
+ ").resolve()\n",
+ "WRITE_ENRICHED_RESULTS = os.environ.get(\"PYRIGHT_BENCHMARK_ENRICH\", \"\") == \"1\"\n",
+ "EXPORT_ASSETS = os.environ.get(\"PYRIGHT_BENCHMARK_EXPORT\", \"\") == \"1\"\n",
+ "\n",
+ "print(f\"Repository: {REPO_ROOT}\")\n",
+ "print(f\"Baseline directory: {BASELINE_DIR}\")"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "c5efd90d",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# 3. Capture Git commit name and timestamp\n",
+ "def git_value(*args: str) -> str:\n",
+ " try:\n",
+ " return subprocess.run(\n",
+ " [\"git\", \"-C\", str(REPO_ROOT), *args],\n",
+ " check=True,\n",
+ " capture_output=True,\n",
+ " text=True,\n",
+ " ).stdout.strip()\n",
+ " except (FileNotFoundError, subprocess.CalledProcessError) as error:\n",
+ " raise RuntimeError(\"Git history is unavailable; run this notebook from a Git checkout\") from error\n",
+ "\n",
+ "\n",
+ "def current_commit_metadata() -> dict[str, str]:\n",
+ " revision = git_value(\"rev-parse\", \"HEAD\")\n",
+ " return {\n",
+ " \"source_revision\": revision,\n",
+ " \"source_commit_short\": git_value(\"rev-parse\", \"--short=12\", revision),\n",
+ " \"source_commit_subject\": git_value(\"show\", \"-s\", \"--format=%s\", revision),\n",
+ " \"source_commit_timestamp\": git_value(\"show\", \"-s\", \"--format=%cI\", revision),\n",
+ " }\n",
+ "\n",
+ "\n",
+ "current_commit = current_commit_metadata()\n",
+ "display(current_commit)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "ea4a11e2",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# 4. Attach commit metadata to result files\n",
+ "def enrich_result_file(\n",
+ " path: Path,\n",
+ " commit: dict[str, str],\n",
+ " *,\n",
+ " write: bool = False,\n",
+ ") -> dict[str, Any]:\n",
+ " data = json.loads(path.read_text(encoding=\"utf-8\"))\n",
+ " if not isinstance(data, dict):\n",
+ " raise ValueError(f\"{path} must contain a JSON object\")\n",
+ "\n",
+ " metadata = {key: value for key, value in commit.items() if key != \"source_commit_short\"}\n",
+ " conflicts = {\n",
+ " key: data[key]\n",
+ " for key, value in metadata.items()\n",
+ " if key in data and data[key] != value\n",
+ " }\n",
+ " if conflicts:\n",
+ " raise ValueError(f\"Refusing to replace existing commit metadata in {path}: {conflicts}\")\n",
+ " enriched = {**metadata, **data}\n",
+ " if \"timestamp\" not in enriched:\n",
+ " enriched[\"timestamp\"] = datetime.now().astimezone().isoformat()\n",
+ " if write:\n",
+ " path.write_text(json.dumps(enriched, indent=2) + \"\\n\", encoding=\"utf-8\")\n",
+ " return enriched\n",
+ "\n",
+ "\n",
+ "result_files = sorted(BASELINE_DIR.glob(\"benchmark_*_linux-x64.json\"))\n",
+ "RESULT_FILES_TO_ENRICH: list[Path] = []\n",
+ "if WRITE_ENRICHED_RESULTS:\n",
+ " if not RESULT_FILES_TO_ENRICH:\n",
+ " raise RuntimeError(\"Set RESULT_FILES_TO_ENRICH explicitly before enabling enrichment\")\n",
+ " for result_file in RESULT_FILES_TO_ENRICH:\n",
+ " enrich_result_file(result_file, current_commit, write=True)\n",
+ " print(f\"Enriched {len(RESULT_FILES_TO_ENRICH)} result file(s)\")\n",
+ "else:\n",
+ " print(f\"Dry run: {len(result_files)} result file(s) discovered; no files changed\")"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "10a96e48",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# 5. Load and validate historical benchmark results\n",
+ "def parse_iso8601(value: str) -> datetime:\n",
+ " return datetime.fromisoformat(value.replace(\"Z\", \"+00:00\"))\n",
+ "\n",
+ "\n",
+ "def validate_result(path: Path, data: Any) -> list[str]:\n",
+ " problems: list[str] = []\n",
+ " if not isinstance(data, dict):\n",
+ " return [\"root value is not an object\"]\n",
+ " for field, expected_type in REQUIRED_METADATA.items():\n",
+ " if not isinstance(data.get(field), expected_type):\n",
+ " problems.append(f\"{field} must be {expected_type.__name__}\")\n",
+ " for field in (\"timestamp\", \"source_commit_timestamp\"):\n",
+ " try:\n",
+ " parse_iso8601(str(data.get(field, \"\")))\n",
+ " except ValueError:\n",
+ " problems.append(f\"{field} must be an ISO 8601 timestamp\")\n",
+ " if not isinstance(data.get(\"source_revision\"), str) or len(data.get(\"source_revision\", \"\")) != 40:\n",
+ " problems.append(\"source_revision must be a 40-character SHA\")\n",
+ " return problems\n",
+ "\n",
+ "\n",
+ "def load_history(paths: list[Path]) -> tuple[list[dict[str, Any]], list[dict[str, str]]]:\n",
+ " records: list[dict[str, Any]] = []\n",
+ " failures: list[dict[str, str]] = []\n",
+ " for path in paths:\n",
+ " try:\n",
+ " data = json.loads(path.read_text(encoding=\"utf-8\"))\n",
+ " problems = validate_result(path, data)\n",
+ " if problems:\n",
+ " failures.append({\"file\": path.name, \"error\": \"; \".join(problems)})\n",
+ " continue\n",
+ " for package in data[\"results\"]:\n",
+ " metrics = package.get(\"metrics\", {}).get(\"pyright\", {})\n",
+ " if not metrics.get(\"ok\"):\n",
+ " continue\n",
+ " for metric, spec in METRIC_SPECS.items():\n",
+ " value = metrics.get(metric)\n",
+ " if isinstance(value, (int, float)):\n",
+ " records.append(\n",
+ " {\n",
+ " \"file\": path.name,\n",
+ " \"benchmark\": package[\"package_name\"],\n",
+ " \"metric\": metric,\n",
+ " \"value\": value,\n",
+ " \"unit\": spec[\"unit\"],\n",
+ " \"lower_is_better\": spec[\"lower_is_better\"],\n",
+ " \"commit_sha\": data[\"source_revision\"],\n",
+ " \"commit_title\": data[\"source_commit_subject\"],\n",
+ " \"commit_timestamp\": data[\"source_commit_timestamp\"],\n",
+ " \"collection_timestamp\": data[\"timestamp\"],\n",
+ " }\n",
+ " )\n",
+ " except (OSError, ValueError, KeyError, TypeError) as error:\n",
+ " failures.append({\"file\": path.name, \"error\": str(error)})\n",
+ " return records, failures\n",
+ "\n",
+ "\n",
+ "records, malformed_files = load_history(result_files)\n",
+ "print(f\"Loaded {len(records)} measurements from {len(result_files)} result file(s)\")\n",
+ "if malformed_files:\n",
+ " display(pd.DataFrame(malformed_files))"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "fcf00161",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# 6. Normalize results into a DataFrame\n",
+ "history = pd.DataFrame.from_records(records)\n",
+ "if history.empty:\n",
+ " raise RuntimeError(\"No valid benchmark measurements were found\")\n",
+ "\n",
+ "for column in (\"commit_timestamp\", \"collection_timestamp\"):\n",
+ " history[column] = pd.to_datetime(history[column], utc=True, errors=\"raise\")\n",
+ "history[\"value\"] = pd.to_numeric(history[\"value\"], errors=\"raise\")\n",
+ "history[\"short_sha\"] = history[\"commit_sha\"].str[:12]\n",
+ "history = (\n",
+ " history.sort_values([\"collection_timestamp\", \"benchmark\", \"metric\"])\n",
+ " .drop_duplicates([\"commit_sha\", \"benchmark\", \"metric\"], keep=\"last\")\n",
+ " .reset_index(drop=True)\n",
+ ")\n",
+ "\n",
+ "run_summary = (\n",
+ " history.groupby(\n",
+ " [\"collection_timestamp\", \"commit_sha\", \"short_sha\", \"commit_title\", \"commit_timestamp\"],\n",
+ " as_index=False,\n",
+ " )\n",
+ " .agg(\n",
+ " package_count=(\"benchmark\", \"nunique\"),\n",
+ " total_execution_time_s=(\"value\", lambda values: history.loc[values.index].query(\"metric == 'execution_time_s'\")[\"value\"].sum()),\n",
+ " max_peak_memory_mb=(\"value\", lambda values: history.loc[values.index].query(\"metric == 'peak_memory_mb'\")[\"value\"].max()),\n",
+ " )\n",
+ ")\n",
+ "display(run_summary)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "19e13c7a",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# 7. Graph performance across commits\n",
+ "SELECTED_PACKAGES = sorted(history[\"benchmark\"].unique())\n",
+ "figures: dict[str, plt.Figure] = {}\n",
+ "run_count = history[\"collection_timestamp\"].nunique()\n",
+ "chart_timezone = history[\"collection_timestamp\"].dt.tz\n",
+ "\n",
+ "for metric, spec in METRIC_SPECS.items():\n",
+ " metric_data = history[\n",
+ " (history[\"metric\"] == metric) & history[\"benchmark\"].isin(SELECTED_PACKAGES)\n",
+ " ]\n",
+ " figure, axis = plt.subplots()\n",
+ " sns.lineplot(\n",
+ " data=metric_data,\n",
+ " x=\"collection_timestamp\",\n",
+ " y=\"value\",\n",
+ " hue=\"benchmark\",\n",
+ " marker=\"o\",\n",
+ " errorbar=None,\n",
+ " palette=\"colorblind\",\n",
+ " ax=axis,\n",
+ " )\n",
+ " title = f\"Pyright {spec['label'].lower()} across commits\"\n",
+ " if run_count > 1:\n",
+ " for row in metric_data.itertuples():\n",
+ " axis.annotate(\n",
+ " row.short_sha,\n",
+ " (row.collection_timestamp, row.value),\n",
+ " xytext=(4, 5),\n",
+ " textcoords=\"offset points\",\n",
+ " fontsize=7,\n",
+ " alpha=0.8,\n",
+ " )\n",
+ " else:\n",
+ " collected_at = metric_data[\"collection_timestamp\"].iloc[0]\n",
+ " axis.set_xlim(collected_at - pd.Timedelta(days=2), collected_at + pd.Timedelta(days=2))\n",
+ " axis.xaxis.set_major_locator(mdates.DayLocator(interval=1, tz=chart_timezone))\n",
+ " title += f\"\\nInitial hosted baseline {metric_data['short_sha'].iloc[0]}; trends appear as runs accumulate\"\n",
+ " axis.set_title(title)\n",
+ " axis.set_xlabel(\"Benchmark collection time (UTC)\")\n",
+ " axis.set_ylabel(f\"{spec['label']} ({spec['unit']})\")\n",
+ " axis.xaxis.set_major_formatter(mdates.DateFormatter(\"%Y-%m-%d\", tz=chart_timezone))\n",
+ " axis.legend(title=\"Package\", bbox_to_anchor=(1.02, 1), loc=\"upper left\")\n",
+ " figure.autofmt_xdate()\n",
+ " figure.tight_layout()\n",
+ " figures[metric] = figure\n",
+ " plt.show()"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "d2b310c2",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# 8. Compare the latest commit with the baseline\n",
+ "BASELINE_REVISION: str | None = None\n",
+ "\n",
+ "\n",
+ "def compare_latest(data: pd.DataFrame, baseline_revision: str | None = None) -> pd.DataFrame:\n",
+ " rows: list[dict[str, Any]] = []\n",
+ " for (benchmark, metric), group in data.groupby([\"benchmark\", \"metric\"], sort=True):\n",
+ " ordered = group.sort_values(\"collection_timestamp\")\n",
+ " latest = ordered.iloc[-1]\n",
+ " if baseline_revision:\n",
+ " matches = ordered[ordered[\"commit_sha\"].str.startswith(baseline_revision)]\n",
+ " if matches.empty:\n",
+ " continue\n",
+ " baseline = matches.iloc[-1]\n",
+ " else:\n",
+ " baseline = ordered.iloc[0]\n",
+ " absolute_change = latest[\"value\"] - baseline[\"value\"]\n",
+ " percent_change = absolute_change / baseline[\"value\"] * 100 if baseline[\"value\"] else float(\"nan\")\n",
+ " lower_is_better = bool(latest[\"lower_is_better\"])\n",
+ " if abs(percent_change) < 1:\n",
+ " assessment = \"Stable\"\n",
+ " elif (percent_change < 0) == lower_is_better:\n",
+ " assessment = \"Improvement\"\n",
+ " else:\n",
+ " assessment = \"Regression\"\n",
+ " rows.append(\n",
+ " {\n",
+ " \"benchmark\": benchmark,\n",
+ " \"metric\": metric,\n",
+ " \"unit\": latest[\"unit\"],\n",
+ " \"baseline_sha\": baseline[\"short_sha\"],\n",
+ " \"latest_sha\": latest[\"short_sha\"],\n",
+ " \"baseline\": baseline[\"value\"],\n",
+ " \"latest\": latest[\"value\"],\n",
+ " \"absolute_change\": absolute_change,\n",
+ " \"percent_change\": percent_change,\n",
+ " \"assessment\": assessment,\n",
+ " }\n",
+ " )\n",
+ " return pd.DataFrame(rows)\n",
+ "\n",
+ "\n",
+ "comparison = compare_latest(history, BASELINE_REVISION)\n",
+ "display(\n",
+ " comparison.style.format(\n",
+ " {\"baseline\": \"{:.2f}\", \"latest\": \"{:.2f}\", \"absolute_change\": \"{:+.2f}\", \"percent_change\": \"{:+.1f}%\"}\n",
+ " ).map(\n",
+ " lambda value: \"color: #087f5b; font-weight: bold\" if value == \"Improvement\" else \"color: #c92a2a; font-weight: bold\" if value == \"Regression\" else \"\",\n",
+ " subset=[\"assessment\"],\n",
+ " )\n",
+ ")"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "aec9b3e3",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# 9. Export charts and summary data\n",
+ "def export_assets(\n",
+ " output_dir: Path,\n",
+ " data: pd.DataFrame,\n",
+ " summary: pd.DataFrame,\n",
+ " charts: dict[str, plt.Figure],\n",
+ ") -> dict[str, Path]:\n",
+ " output_dir.mkdir(parents=True, exist_ok=True)\n",
+ " outputs = {\n",
+ " \"history_csv\": output_dir / \"history.csv\",\n",
+ " \"comparison_json\": output_dir / \"comparison.json\",\n",
+ " }\n",
+ " data.to_csv(outputs[\"history_csv\"], index=False)\n",
+ " outputs[\"comparison_json\"].write_text(\n",
+ " json.dumps(summary.to_dict(orient=\"records\"), indent=2, default=str) + \"\\n\",\n",
+ " encoding=\"utf-8\",\n",
+ " )\n",
+ " for metric, figure in charts.items():\n",
+ " chart_path = output_dir / f\"{metric}.svg\"\n",
+ " preview_path = output_dir / f\"{metric}.png\"\n",
+ " figure.savefig(chart_path, format=\"svg\", bbox_inches=\"tight\")\n",
+ " figure.savefig(preview_path, format=\"png\", dpi=144, bbox_inches=\"tight\")\n",
+ " outputs[metric] = chart_path\n",
+ " outputs[f\"{metric}_preview\"] = preview_path\n",
+ " return outputs\n",
+ "\n",
+ "\n",
+ "exported: dict[str, Path] = {}\n",
+ "if EXPORT_ASSETS:\n",
+ " exported = export_assets(EXPORT_DIR, history, comparison, figures)\n",
+ " display(exported)\n",
+ "else:\n",
+ " print(\"Exports disabled. Set EXPORT_ASSETS = True in Section 2 to write dashboard assets.\")"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "47b679bd",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# 10. Generate a GitHub Pages dashboard\n",
+ "def dashboard_html(run_data: pd.DataFrame, summary: pd.DataFrame) -> str:\n",
+ " latest = run_data.sort_values(\"collection_timestamp\").iloc[-1]\n",
+ " status_class = {\"Improvement\": \"good\", \"Regression\": \"bad\", \"Stable\": \"stable\"}\n",
+ " rows = \"\".join(\n",
+ " \"\"\n",
+ " f\"| {html.escape(str(row.benchmark))} | \"\n",
+ " f\"{html.escape(METRIC_SPECS[str(row.metric)]['label'])} | \"\n",
+ " f\"{float(row.baseline):.2f} {html.escape(str(row.unit))} | \"\n",
+ " f\"{float(row.latest):.2f} {html.escape(str(row.unit))} | \"\n",
+ " f\"{float(row.percent_change):+.1f}% | \"\n",
+ " f\"{html.escape(str(row.assessment))} | \"\n",
+ " \"
\"\n",
+ " for row in summary.itertuples()\n",
+ " )\n",
+ " charts = \"\".join(\n",
+ " f\"{html.escape(spec['label'])}
\"\n",
+ " for metric, spec in METRIC_SPECS.items()\n",
+ " )\n",
+ " return f\"\"\"\n",
+ "\n",
+ "Pyright benchmark historyPyright benchmark history
\n",
+ "Latest: {html.escape(str(latest['commit_sha']))} {html.escape(str(latest['commit_title']))}
\n",
+ "Collected {html.escape(str(latest['collection_timestamp']))}
{charts}\n",
+ "Latest compared with baseline
| Package | Metric | Baseline | Latest | Change | Assessment |
|---|
{rows}
\n",
+ "\"\"\"\n",
+ "\n",
+ "\n",
+ "def format_dashboard(path: Path) -> None:\n",
+ " formatter = \"pnpm.cmd\" if os.name == \"nt\" else \"pnpm\"\n",
+ " subprocess.run(\n",
+ " [formatter, \"exec\", \"prettier\", \"--write\", str(path)],\n",
+ " cwd=REPO_ROOT,\n",
+ " check=True,\n",
+ " )\n",
+ "\n",
+ "\n",
+ "dashboard = dashboard_html(history, comparison)\n",
+ "if EXPORT_ASSETS:\n",
+ " dashboard_path = EXPORT_DIR / \"index.html\"\n",
+ " dashboard_path.write_text(dashboard, encoding=\"utf-8\")\n",
+ " format_dashboard(dashboard_path)\n",
+ " print(f\"Dashboard: {dashboard_path}\")\n",
+ "else:\n",
+ " display(HTML(dashboard))"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "d1fb3018",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# 11. Automate notebook and page generation\n",
+ "NOTEBOOK_PATH = REPO_ROOT / \"build\" / \"benchmark\" / \"benchmark_history.ipynb\"\n",
+ "EXECUTED_NOTEBOOK = EXPORT_DIR / \"benchmark_history.executed.ipynb\"\n",
+ "NBCONVERT_COMMAND = [\n",
+ " \"jupyter\",\n",
+ " \"nbconvert\",\n",
+ " \"--to\",\n",
+ " \"notebook\",\n",
+ " \"--execute\",\n",
+ " str(NOTEBOOK_PATH),\n",
+ " \"--output\",\n",
+ " EXECUTED_NOTEBOOK.name,\n",
+ " \"--output-dir\",\n",
+ " str(EXPORT_DIR),\n",
+ " \"--ExecutePreprocessor.timeout=180\",\n",
+ "]\n",
+ "\n",
+ "assert not malformed_files, f\"Malformed benchmark files: {malformed_files}\"\n",
+ "assert not history.empty, \"No normalized benchmark data\"\n",
+ "assert set(METRIC_SPECS).issubset(set(history[\"metric\"])), \"Required metrics are missing\"\n",
+ "\n",
+ "print(\"For CI, execute with PYRIGHT_BENCHMARK_EXPORT=1:\")\n",
+ "print(subprocess.list2cmdline(NBCONVERT_COMMAND))\n",
+ "print(f\"Expected dashboard: {EXPORT_DIR / 'index.html'}\")\n",
+ "print(\"Fail CI if notebook execution fails, malformed_files is non-empty, or index.html is absent.\")"
+ ]
+ }
+ ],
+ "metadata": {
+ "language_info": {
+ "name": "python"
+ }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}
diff --git a/build/benchmark/compare_benchmarks.py b/build/benchmark/compare_benchmarks.py
index d0879a007aad..79575810d202 100644
--- a/build/benchmark/compare_benchmarks.py
+++ b/build/benchmark/compare_benchmarks.py
@@ -13,6 +13,21 @@
DEFAULT_TIME_NOISE_FLOOR_SECONDS = 1.0
DEFAULT_MEMORY_NOISE_FLOOR_MB = 100.0
+ENVIRONMENT_FIELDS = (
+ "platform",
+ "architecture",
+ "runner_class",
+ "runner_image",
+ "cpu_count",
+ "python_version",
+ "memory_limit_mb",
+ "node_options",
+ "runs_per_package",
+ "warmup_runs",
+ "dependency_isolation",
+ "benchmark_profile_hash",
+)
+
class ComparisonRow(TypedDict, total=False):
package: str
@@ -98,10 +113,58 @@ def _packages_by_name(data: dict[str, Any]) -> dict[str, dict[str, Any]]:
return {
package["package_name"]: package
for package in data.get("results", [])
- if package.get("package_name")
+ if isinstance(package, dict) and package.get("package_name")
}
+def _compatibility_failures(
+ baseline: dict[str, Any], candidate: dict[str, Any]
+) -> list[str]:
+ failures: list[str] = []
+ for field in ENVIRONMENT_FIELDS:
+ if baseline.get(field) != candidate.get(field):
+ failures.append(
+ f"environment mismatch for {field}: "
+ f"{baseline.get(field)!r} != {candidate.get(field)!r}"
+ )
+
+ baseline_packages = _packages_by_name(baseline)
+ candidate_packages = _packages_by_name(candidate)
+ if baseline_packages.keys() != candidate_packages.keys():
+ failures.append(
+ "benchmark package set changed from "
+ f"{sorted(baseline_packages)} to {sorted(candidate_packages)}"
+ )
+ for package_name in sorted(baseline_packages.keys() & candidate_packages.keys()):
+ old_package = baseline_packages[package_name]
+ new_package = candidate_packages[package_name]
+ if old_package.get("commit") != new_package.get("commit"):
+ failures.append(
+ f"{package_name}: package commit changed from "
+ f"{old_package.get('commit')} to {new_package.get('commit')}"
+ )
+ for field in ("check_paths", "exclude_directories"):
+ if old_package.get(field, []) != new_package.get(field, []):
+ failures.append(
+ f"{package_name}: package {field} changed from "
+ f"{old_package.get(field, [])} to {new_package.get(field, [])}"
+ )
+ return failures
+
+
+def _measurement_failures(data: dict[str, Any], label: str) -> list[str]:
+ failures: list[str] = []
+ for package in data.get("results", []):
+ package_name = package.get("package_name", "unknown")
+ if package.get("error"):
+ failures.append(f"{label}: {package_name} package preparation failed")
+ continue
+ metrics = package.get("metrics", {}).get("pyright")
+ if not isinstance(metrics, dict) or not metrics.get("ok"):
+ failures.append(f"{label}: {package_name}/pyright result failed or is missing")
+ return failures
+
+
def _percent_change(baseline: float, candidate: float) -> float:
return ((candidate - baseline) / baseline) * 100 if baseline else 0.0
@@ -120,27 +183,32 @@ def _analyze(
time_noise_floor_s: float = 0.0,
memory_noise_floor_mb: float = 0.0,
fail_on_preparation_error: bool = False,
+ baseline_revision: str | None = None,
+ candidate_revision: str | None = None,
+ allow_incompatible: bool = False,
) -> tuple[list[str], list[ComparisonRow]]:
failures = [
*_validate_results(baseline, "baseline"),
*_validate_results(candidate, "candidate"),
]
rows: list[ComparisonRow] = []
+ for data, label, expected_revision in (
+ (baseline, "baseline", baseline_revision),
+ (candidate, "candidate", candidate_revision),
+ ):
+ if expected_revision and data.get("source_revision") != expected_revision:
+ failures.append(
+ f"{label}: source revision {data.get('source_revision')!r} "
+ f"does not match {expected_revision!r}"
+ )
if failures:
return failures, rows
- for field in (
- "platform",
- "architecture",
- "runner_class",
- "runner_image",
- "cpu_count",
- "python_version",
- "memory_limit_mb",
- "node_options",
- "runs_per_package",
- "warmup_runs",
- "dependency_isolation",
- ):
+ compatibility_failures = _compatibility_failures(baseline, candidate)
+ if allow_incompatible and compatibility_failures:
+ failures.extend(_measurement_failures(baseline, "baseline"))
+ failures.extend(_measurement_failures(candidate, "candidate"))
+ return failures, rows
+ for field in ENVIRONMENT_FIELDS:
if baseline.get(field) != candidate.get(field):
failures.append(
f"environment mismatch for {field}: "
@@ -343,6 +411,9 @@ def compare(
time_noise_floor_s: float = 0.0,
memory_noise_floor_mb: float = 0.0,
fail_on_preparation_error: bool = False,
+ baseline_revision: str | None = None,
+ candidate_revision: str | None = None,
+ allow_incompatible: bool = False,
) -> list[str]:
failures, rows = _analyze(
baseline,
@@ -351,6 +422,9 @@ def compare(
time_noise_floor_s,
memory_noise_floor_mb,
fail_on_preparation_error,
+ baseline_revision,
+ candidate_revision,
+ allow_incompatible,
)
print(
f"{'Package':<20} {'Checker':<10} {'Time':>10} {'Delta':>9} "
@@ -382,6 +456,9 @@ def render_markdown(
time_noise_floor_s: float = 0.0,
memory_noise_floor_mb: float = 0.0,
fail_on_preparation_error: bool = False,
+ baseline_revision: str | None = None,
+ candidate_revision: str | None = None,
+ allow_incompatible: bool = False,
) -> str:
failures, rows = _analyze(
baseline,
@@ -390,7 +467,28 @@ def render_markdown(
time_noise_floor_s,
memory_noise_floor_mb,
fail_on_preparation_error,
+ baseline_revision,
+ candidate_revision,
+ allow_incompatible,
)
+ compatibility_failures = _compatibility_failures(baseline, candidate)
+ if allow_incompatible and compatibility_failures and not failures:
+ lines = [
+ "## Type checker benchmark",
+ "",
+ "š” **Performance results are not comparable because the benchmark profile changed.**",
+ ]
+ if baseline_revision and candidate_revision:
+ lines.extend(
+ [
+ "",
+ f"Base commit: `{baseline_revision}`",
+ f"Candidate merge commit: `{candidate_revision}`",
+ ]
+ )
+ lines.extend(["", "### Compatibility changes", ""])
+ lines.extend(f"- {_escape_markdown(failure)}" for failure in compatibility_failures)
+ return "\n".join(lines) + "\n"
if failures:
summary = f"š“ **{len(failures)} regression check(s) failed.**"
else:
@@ -416,6 +514,13 @@ def render_markdown(
"",
f"Regression threshold: `{threshold_percent:.1f}%`",
]
+ if baseline_revision and candidate_revision:
+ lines.extend(
+ [
+ f"Base commit: `{baseline_revision}`",
+ f"Candidate merge commit: `{candidate_revision}`",
+ ]
+ )
if time_noise_floor_s > 0 or memory_noise_floor_mb > 0:
lines.append(
f"Variance guard: `>{time_noise_floor_s:.1f}s` time and "
@@ -525,6 +630,9 @@ def main(argv: list[str] | None = None) -> int:
default=DEFAULT_MEMORY_NOISE_FLOOR_MB,
)
parser.add_argument("--fail-on-preparation-error", action="store_true")
+ parser.add_argument("--baseline-revision")
+ parser.add_argument("--candidate-revision")
+ parser.add_argument("--allow-incompatible", action="store_true")
parser.add_argument("--markdown-output", type=Path)
args = parser.parse_args(argv)
@@ -542,6 +650,9 @@ def main(argv: list[str] | None = None) -> int:
args.time_noise_floor_seconds,
args.memory_noise_floor_mb,
args.fail_on_preparation_error,
+ args.baseline_revision,
+ args.candidate_revision,
+ args.allow_incompatible,
)
if args.markdown_output:
args.markdown_output.write_text(
@@ -552,6 +663,9 @@ def main(argv: list[str] | None = None) -> int:
args.time_noise_floor_seconds,
args.memory_noise_floor_mb,
args.fail_on_preparation_error,
+ args.baseline_revision,
+ args.candidate_revision,
+ args.allow_incompatible,
),
encoding="utf-8",
)
diff --git a/build/benchmark/requirements-notebook.txt b/build/benchmark/requirements-notebook.txt
new file mode 100644
index 000000000000..55a67dcb3d92
--- /dev/null
+++ b/build/benchmark/requirements-notebook.txt
@@ -0,0 +1,5 @@
+ipykernel>=6.30
+matplotlib>=3.10
+nbconvert>=7.16
+pandas>=2.3
+seaborn>=0.13
\ No newline at end of file
diff --git a/build/benchmark/test_compare_benchmarks.py b/build/benchmark/test_compare_benchmarks.py
index a10680716c3a..ae9a45e53608 100644
--- a/build/benchmark/test_compare_benchmarks.py
+++ b/build/benchmark/test_compare_benchmarks.py
@@ -43,6 +43,7 @@ def _result(time: float, memory: float, ok: bool = True) -> dict:
"uncounted_validation_runs_per_checker": 0,
"timeout_s": 600,
"dependency_isolation": "pip-target-per-package",
+ "benchmark_profile_hash": "profile-v1",
"results": [
{
"package_name": "example",
@@ -430,52 +431,110 @@ def test_load_rejects_non_finite_json_numbers(self) -> None:
with self.assertRaisesRegex(ValueError, "non-finite number NaN"):
compare_benchmarks._load_results(result_file)
- def test_workflow_profile_matches_checked_in_baseline(self) -> None:
- workflow = (
- REPO_ROOT / ".github" / "workflows" / "typecheck_benchmark_pr.yml"
- ).read_text(encoding="utf-8")
- timeout_match = re.search(
- r"typecheck_benchmark\.py \\\s+"
- r"-c pyright -r 1 -w 0 -t (\d+)",
- workflow,
+ def test_rejects_unexpected_source_revision(self) -> None:
+ baseline = _result(10.0, 100.0)
+ candidate = _result(10.0, 100.0)
+ baseline["source_revision"] = "a" * 40
+ candidate["source_revision"] = "b" * 40
+
+ failures = compare_benchmarks.compare(
+ baseline,
+ candidate,
+ 10.0,
+ baseline_revision="c" * 40,
+ candidate_revision="b" * 40,
)
- self.assertIsNotNone(timeout_match)
- baseline = json.loads(
- (
- REPO_ROOT
- / "build"
- / "benchmark"
- / "baselines"
- / "latest-linux-x64.json"
- ).read_text(encoding="utf-8")
+ self.assertEqual(
+ failures,
+ [
+ "baseline: source revision "
+ f"{'a' * 40!r} does not match {'c' * 40!r}"
+ ],
)
- config = json.loads(
- (
- REPO_ROOT / "build" / "benchmark" / "install_envs.json"
- ).read_text(encoding="utf-8")
+
+ def test_report_identifies_compared_revisions(self) -> None:
+ baseline = _result(10.0, 100.0)
+ candidate = _result(10.0, 100.0)
+ baseline_revision = "a" * 40
+ candidate_revision = "b" * 40
+ baseline["source_revision"] = baseline_revision
+ candidate["source_revision"] = candidate_revision
+
+ report = compare_benchmarks.render_markdown(
+ baseline,
+ candidate,
+ 10.0,
+ baseline_revision=baseline_revision,
+ candidate_revision=candidate_revision,
)
- self.assertEqual(int(timeout_match.group(1)), 1800)
- baseline_packages = {
- package["package_name"]: package for package in baseline["results"]
- }
- for package in config["packages"]:
- package_name = package.get("name") or package["github_url"].rsplit(
- "/", 1
- )[-1]
- baseline_package = baseline_packages[package_name]
- self.assertEqual(
- package.get("check_paths", []), baseline_package["check_paths"]
- )
- self.assertEqual(
- package.get("exclude_directories", []),
- baseline_package["exclude_directories"],
- )
+ self.assertIn(f"Base commit: `{baseline_revision}`", report)
+ self.assertIn(f"Candidate merge commit: `{candidate_revision}`", report)
+
+ def test_allows_successful_benchmark_profile_change(self) -> None:
+ candidate = _result(10.0, 100.0)
+ candidate["benchmark_profile_hash"] = "profile-v2"
+
+ failures = compare_benchmarks.compare(
+ _result(10.0, 100.0),
+ candidate,
+ 10.0,
+ allow_incompatible=True,
+ )
+ report = compare_benchmarks.render_markdown(
+ _result(10.0, 100.0),
+ candidate,
+ 10.0,
+ allow_incompatible=True,
+ )
+
+ self.assertEqual(failures, [])
+ self.assertIn("Performance results are not comparable", report)
+ self.assertIn(r"benchmark\_profile\_hash", report)
+
+ def test_incompatible_results_still_require_successful_measurements(self) -> None:
+ candidate = _result(0.0, 0.0, ok=False)
+ candidate["benchmark_profile_hash"] = "profile-v2"
+
+ failures = compare_benchmarks.compare(
+ _result(10.0, 100.0),
+ candidate,
+ 10.0,
+ allow_incompatible=True,
+ )
+
+ self.assertEqual(
+ failures, ["candidate: example/pyright result failed or is missing"]
+ )
+
+ def test_pr_workflow_uses_matching_base_and_candidate_profiles(self) -> None:
+ candidate_workflow = (
+ REPO_ROOT / ".github" / "workflows" / "typecheck_benchmark_pr.yml"
+ ).read_text(encoding="utf-8")
+ report_workflow = (
+ REPO_ROOT / ".github" / "workflows" / "typecheck_benchmark_report.yml"
+ ).read_text(encoding="utf-8")
+ timeout_matches = re.findall(
+ r"typecheck_benchmark\.py \\\s+"
+ r"-c pyright -r 1 -w 0 -t (\d+)",
+ candidate_workflow + report_workflow,
+ )
+ self.assertEqual(timeout_matches, ["1800", "1800"])
+ self.assertIn("data['source_revision'] = os.environ['MERGE_SHA']", candidate_workflow)
+ self.assertIn("data['source_head_revision']", candidate_workflow)
+ self.assertIn("data['source_base_revision']", candidate_workflow)
+ self.assertIn("data['source_revision'] = os.environ['BASE_SHA']", report_workflow)
+ for workflow in (candidate_workflow, report_workflow):
+ self.assertIn("data['source_commit_subject']", workflow)
+ self.assertIn("data['source_commit_timestamp']", workflow)
+ self.assertIn("data['benchmark_profile_hash'] = profile.hexdigest()", workflow)
+ self.assertIn("build/benchmark/baselines/latest-linux-x64.json", report_workflow)
def test_workflows_use_current_pnpm_setup(self) -> None:
for workflow_name in (
"typecheck_benchmark_pr.yml",
+ "typecheck_benchmark_report.yml",
"typecheck_benchmark_weekly.yml",
):
workflow = (
@@ -520,7 +579,7 @@ def test_workflows_use_current_pnpm_setup(self) -> None:
REPO_ROOT / ".github" / "workflows" / "typecheck_benchmark_pr.yml"
).read_text(encoding="utf-8")
self.assertIn(
- "run-name: 'Type checker benchmark for PR #${{ inputs.pr_number }}'",
+ "run-name: 'Type checker benchmark candidate for PR #${{ github.event.pull_request.number }}'",
pr_workflow,
)
self.assertIn("PNPM_VERSION: '10.12.2'", pr_workflow)
@@ -538,6 +597,11 @@ def test_pr_benchmark_requires_authorized_comment(self) -> None:
)
benchmark_workflow = benchmark_workflow_path.read_text(encoding="utf-8")
benchmark_workflow_data = _load_yaml(benchmark_workflow_path)
+ report_workflow_path = (
+ REPO_ROOT / ".github" / "workflows" / "typecheck_benchmark_report.yml"
+ )
+ report_workflow = report_workflow_path.read_text(encoding="utf-8")
+ report_workflow_data = _load_yaml(report_workflow_path)
self.assertIn("issue_comment:", trigger_workflow)
self.assertIn(
@@ -549,12 +613,19 @@ def test_pr_benchmark_requires_authorized_comment(self) -> None:
self.assertIn("github.event.issue.state == 'open'", trigger_workflow)
self.assertIn("getCollaboratorPermissionLevel", trigger_workflow)
self.assertIn("['admin', 'maintain', 'write']", trigger_workflow)
- self.assertIn("actions: write", trigger_workflow)
+ self.assertIn("issues: write", trigger_workflow)
self.assertIn("pull-requests: read", trigger_workflow)
- self.assertIn("createWorkflowDispatch", trigger_workflow)
- self.assertIn("workflow_id: 'typecheck_benchmark_pr.yml'", trigger_workflow)
- self.assertIn("base_sha: pullRequest.data.base.sha", trigger_workflow)
- self.assertIn("merge_sha: pullRequest.data.merge_commit_sha", trigger_workflow)
+ self.assertIn("actions: write", trigger_workflow)
+ self.assertIn("actions.listWorkflowRuns", trigger_workflow)
+ self.assertIn("actions.reRunWorkflow", trigger_workflow)
+ self.assertIn("issues.addLabels", trigger_workflow)
+ self.assertIn("issues.removeLabel", trigger_workflow)
+ self.assertIn("issues.createLabel", trigger_workflow)
+ self.assertNotIn("createWorkflowDispatch", trigger_workflow)
+ self.assertNotIn("workflow_dispatch:", trigger_workflow)
+ self.assertNotIn("inputs:", trigger_workflow)
+ self.assertNotIn("base_sha:", trigger_workflow)
+ self.assertNotIn("merge_sha:", trigger_workflow)
self.assertNotIn("actions/checkout", trigger_workflow)
self.assertIn(
"actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0",
@@ -565,71 +636,123 @@ def test_pr_benchmark_requires_authorized_comment(self) -> None:
"actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7",
benchmark_workflow,
)
- self.assertIn(
- "actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0",
- benchmark_workflow,
- )
self.assertNotIn("actions/checkout@v4", benchmark_workflow)
- self.assertNotIn("actions/github-script@v7", benchmark_workflow)
- self.assertIn("workflow_dispatch:", benchmark_workflow)
+ self.assertIn("pull_request:", benchmark_workflow)
+ self.assertIn("- synchronize", benchmark_workflow)
+ self.assertNotIn("- labeled", benchmark_workflow)
+ self.assertIn("Number(process.env.RUN_ATTEMPT) > 1", benchmark_workflow)
+ self.assertIn("label.name === 'benchmark-requested'", benchmark_workflow)
+ self.assertIn("needs.authorize.outputs.requested == 'true'", benchmark_workflow)
+ self.assertNotIn("workflow_dispatch:", benchmark_workflow)
self.assertNotIn("paths:", benchmark_workflow)
- self.assertNotIn("pull_request:", benchmark_workflow)
self.assertNotIn("cache: 'pip'", benchmark_workflow)
self.assertNotIn("cache: 'pnpm'", benchmark_workflow)
self.assertIn("persist-credentials: false", benchmark_workflow)
- self.assertIn("inputs.base_sha", benchmark_workflow)
- self.assertIn("ref: ${{ inputs.merge_sha }}", benchmark_workflow)
- self.assertIn("-merge-${{ inputs.merge_sha }}", benchmark_workflow)
- self.assertIn("if: ${{ always() }}", benchmark_workflow)
- self.assertIn("run_id: context.runId", benchmark_workflow)
- self.assertIn("pullRequest.data.base.sha !== expectedBaseSha", benchmark_workflow)
- self.assertIn(
- "pullRequest.data.merge_commit_sha !== expectedMergeSha",
- benchmark_workflow,
- )
- benchmark_job = benchmark_workflow_data["jobs"]["benchmark"]
- comment_job = benchmark_workflow_data["jobs"]["comment"]
- self.assertEqual(benchmark_job["permissions"], {"contents": "read"})
+ candidate_job = benchmark_workflow_data["jobs"]["candidate-benchmark"]
+ self.assertEqual(
+ benchmark_workflow_data["permissions"],
+ {"contents": "read", "issues": "read"},
+ )
+ self.assertNotIn("permissions", candidate_job)
+ self.assertNotIn("actions/cache", benchmark_workflow)
+ self.assertNotIn("contents: write", benchmark_workflow)
+
+ self.assertIn("workflow_run:", report_workflow)
+ self.assertIn("Type checker benchmark candidate", report_workflow)
+ self.assertIn("github.event.workflow_run.run_attempt > 1", report_workflow)
+ self.assertIn("github.event.workflow_run.id", report_workflow)
+ self.assertIn("ref: ${{ github.sha }}", report_workflow)
+ self.assertNotIn("workflow_dispatch:", report_workflow)
+ metadata_job = report_workflow_data["jobs"]["metadata"]
+ base_job = report_workflow_data["jobs"]["base-benchmark"]
+ comparison_job = report_workflow_data["jobs"]["comparison"]
+ comment_job = report_workflow_data["jobs"]["comment"]
+ persist_job = report_workflow_data["jobs"]["persist-base-result"]
+ self.assertEqual(report_workflow_data["permissions"], {})
+ self.assertEqual(metadata_job["permissions"], {"pull-requests": "read"})
+ self.assertEqual(base_job["permissions"], {"contents": "read"})
+ self.assertEqual(comparison_job["permissions"], {"actions": "read", "contents": "read"})
self.assertEqual(
comment_job["permissions"],
{
"actions": "read",
- "contents": "read",
"pull-requests": "write",
},
)
- self.assertEqual(comment_job["needs"], "benchmark")
+ self.assertEqual(
+ persist_job["permissions"],
+ {"actions": "read", "contents": "write"},
+ )
+ self.assertEqual(comment_job["needs"], ["metadata", "comparison"])
self.assertEqual(
[
job_name
- for job_name, job in benchmark_workflow_data["jobs"].items()
+ for job_name, job in report_workflow_data["jobs"].items()
if job.get("permissions", {}).get("pull-requests") == "write"
],
["comment"],
)
- self.assertFalse(
- (
- REPO_ROOT
- / ".github"
- / "workflows"
- / "typecheck_benchmark_comment.yml"
- ).exists()
+ self.assertEqual(
+ [
+ job_name
+ for job_name, job in report_workflow_data["jobs"].items()
+ if job.get("permissions", {}).get("contents") == "write"
+ ],
+ ["persist-base-result"],
)
+ self.assertIn("github.event.workflow_run.conclusion == 'success'", report_workflow)
- def test_pr_workflow_prefers_trusted_baseline_with_bootstrap_fallback(self) -> None:
- workflow = (
+ def test_pr_workflow_caches_only_the_base_result(self) -> None:
+ candidate_workflow = (
REPO_ROOT / ".github" / "workflows" / "typecheck_benchmark_pr.yml"
).read_text(encoding="utf-8")
-
- trusted = "benchmark-baseline/build/benchmark/baselines/latest-linux-x64.json"
- bootstrap = "build/benchmark/baselines/latest-linux-x64.json"
- self.assertLess(
- workflow.index('if [[ -f "$trusted" ]]'),
- workflow.index('elif [[ -f "$bootstrap" ]]'),
+ report_workflow_path = (
+ REPO_ROOT / ".github" / "workflows" / "typecheck_benchmark_report.yml"
+ )
+ workflow = report_workflow_path.read_text(encoding="utf-8")
+ workflow_data = _load_yaml(report_workflow_path)
+ base_job = json.dumps(workflow_data["jobs"]["base-benchmark"])
+
+ self.assertIn("actions/cache/restore@0057852", base_job)
+ self.assertIn("actions/cache/save@0057852", base_job)
+ self.assertIn("typecheck-benchmark-base-v2-", base_job)
+ self.assertNotIn("restore-keys", base_job)
+ self.assertNotIn("actions/cache/", candidate_workflow)
+ self.assertIn("ref: ${{ github.sha }}", workflow)
+ self.assertNotIn("actions/checkout", workflow_data["jobs"]["persist-base-result"])
+ self.assertIn("--baseline-revision", workflow)
+ self.assertIn("--candidate-revision", workflow)
+ self.assertIn("--allow-incompatible", workflow)
+ self.assertIn("issues.listComments", workflow)
+ self.assertIn("updateComment", workflow)
+ self.assertIn("comment.user?.login === 'github-actions[bot]'", workflow)
+ self.assertNotIn("git push", workflow)
+ persist_job = workflow_data["jobs"]["persist-base-result"]
+ persist_job_text = json.dumps(persist_job)
+ self.assertNotIn("actions/checkout", persist_job_text)
+ self.assertIn("needs.base-benchmark.outputs.cached != 'true'", persist_job["if"])
+ self.assertIn("needs.metadata.outputs.head-repository == github.repository", persist_job["if"])
+ self.assertIn("!cancelled()", persist_job["if"])
+ self.assertIn("github.rest.git.createCommit", workflow)
+ self.assertIn("github.rest.git.updateRef", workflow)
+ self.assertIn("currentRef.data.object.sha !== expectedHeadSha", workflow)
+ self.assertIn("build/benchmark/baselines/latest-linux-x64.json", workflow)
+ self.assertIn("build/benchmark/baselines/benchmark_${result.date}_linux-x64.json", workflow)
+ self.assertTrue(
+ (REPO_ROOT / "build" / "benchmark" / "baselines" / "latest-linux-x64.json").exists()
+ )
+ baseline = json.loads(
+ (
+ REPO_ROOT
+ / "build"
+ / "benchmark"
+ / "baselines"
+ / "latest-linux-x64.json"
+ ).read_text(encoding="utf-8")
)
- self.assertIn('echo "path=$trusted" >> "$GITHUB_OUTPUT"', workflow)
- self.assertIn('echo "path=$bootstrap" >> "$GITHUB_OUTPUT"', workflow)
- self.assertIn('"${{ steps.baseline.outputs.path }}"', workflow)
+ self.assertRegex(baseline["source_revision"], r"^[0-9a-f]{40}$")
+ self.assertTrue(baseline["source_commit_subject"])
+ self.assertTrue(baseline["source_commit_timestamp"])
if __name__ == "__main__":
diff --git a/docs/benchmark-results/comparison.json b/docs/benchmark-results/comparison.json
new file mode 100644
index 000000000000..3d0d2d23e752
--- /dev/null
+++ b/docs/benchmark-results/comparison.json
@@ -0,0 +1,218 @@
+[
+ {
+ "benchmark": "ansible",
+ "metric": "execution_time_s",
+ "unit": "seconds",
+ "baseline_sha": "075161cd9160",
+ "latest_sha": "075161cd9160",
+ "baseline": 21.661,
+ "latest": 21.661,
+ "absolute_change": 0.0,
+ "percent_change": 0.0,
+ "assessment": "Stable"
+ },
+ {
+ "benchmark": "ansible",
+ "metric": "peak_memory_mb",
+ "unit": "MiB",
+ "baseline_sha": "075161cd9160",
+ "latest_sha": "075161cd9160",
+ "baseline": 1110.7,
+ "latest": 1110.7,
+ "absolute_change": 0.0,
+ "percent_change": 0.0,
+ "assessment": "Stable"
+ },
+ {
+ "benchmark": "click",
+ "metric": "execution_time_s",
+ "unit": "seconds",
+ "baseline_sha": "075161cd9160",
+ "latest_sha": "075161cd9160",
+ "baseline": 2.908,
+ "latest": 2.908,
+ "absolute_change": 0.0,
+ "percent_change": 0.0,
+ "assessment": "Stable"
+ },
+ {
+ "benchmark": "click",
+ "metric": "peak_memory_mb",
+ "unit": "MiB",
+ "baseline_sha": "075161cd9160",
+ "latest_sha": "075161cd9160",
+ "baseline": 361.0,
+ "latest": 361.0,
+ "absolute_change": 0.0,
+ "percent_change": 0.0,
+ "assessment": "Stable"
+ },
+ {
+ "benchmark": "homeassistant",
+ "metric": "execution_time_s",
+ "unit": "seconds",
+ "baseline_sha": "075161cd9160",
+ "latest_sha": "075161cd9160",
+ "baseline": 158.341,
+ "latest": 158.341,
+ "absolute_change": 0.0,
+ "percent_change": 0.0,
+ "assessment": "Stable"
+ },
+ {
+ "benchmark": "homeassistant",
+ "metric": "peak_memory_mb",
+ "unit": "MiB",
+ "baseline_sha": "075161cd9160",
+ "latest_sha": "075161cd9160",
+ "baseline": 6192.1,
+ "latest": 6192.1,
+ "absolute_change": 0.0,
+ "percent_change": 0.0,
+ "assessment": "Stable"
+ },
+ {
+ "benchmark": "numpy",
+ "metric": "execution_time_s",
+ "unit": "seconds",
+ "baseline_sha": "075161cd9160",
+ "latest_sha": "075161cd9160",
+ "baseline": 43.435,
+ "latest": 43.435,
+ "absolute_change": 0.0,
+ "percent_change": 0.0,
+ "assessment": "Stable"
+ },
+ {
+ "benchmark": "numpy",
+ "metric": "peak_memory_mb",
+ "unit": "MiB",
+ "baseline_sha": "075161cd9160",
+ "latest_sha": "075161cd9160",
+ "baseline": 1850.0,
+ "latest": 1850.0,
+ "absolute_change": 0.0,
+ "percent_change": 0.0,
+ "assessment": "Stable"
+ },
+ {
+ "benchmark": "pandas",
+ "metric": "execution_time_s",
+ "unit": "seconds",
+ "baseline_sha": "075161cd9160",
+ "latest_sha": "075161cd9160",
+ "baseline": 989.16,
+ "latest": 989.16,
+ "absolute_change": 0.0,
+ "percent_change": 0.0,
+ "assessment": "Stable"
+ },
+ {
+ "benchmark": "pandas",
+ "metric": "peak_memory_mb",
+ "unit": "MiB",
+ "baseline_sha": "075161cd9160",
+ "latest_sha": "075161cd9160",
+ "baseline": 4434.0,
+ "latest": 4434.0,
+ "absolute_change": 0.0,
+ "percent_change": 0.0,
+ "assessment": "Stable"
+ },
+ {
+ "benchmark": "pytest",
+ "metric": "execution_time_s",
+ "unit": "seconds",
+ "baseline_sha": "075161cd9160",
+ "latest_sha": "075161cd9160",
+ "baseline": 12.574,
+ "latest": 12.574,
+ "absolute_change": 0.0,
+ "percent_change": 0.0,
+ "assessment": "Stable"
+ },
+ {
+ "benchmark": "pytest",
+ "metric": "peak_memory_mb",
+ "unit": "MiB",
+ "baseline_sha": "075161cd9160",
+ "latest_sha": "075161cd9160",
+ "baseline": 930.9,
+ "latest": 930.9,
+ "absolute_change": 0.0,
+ "percent_change": 0.0,
+ "assessment": "Stable"
+ },
+ {
+ "benchmark": "requests",
+ "metric": "execution_time_s",
+ "unit": "seconds",
+ "baseline_sha": "075161cd9160",
+ "latest_sha": "075161cd9160",
+ "baseline": 2.299,
+ "latest": 2.299,
+ "absolute_change": 0.0,
+ "percent_change": 0.0,
+ "assessment": "Stable"
+ },
+ {
+ "benchmark": "requests",
+ "metric": "peak_memory_mb",
+ "unit": "MiB",
+ "baseline_sha": "075161cd9160",
+ "latest_sha": "075161cd9160",
+ "baseline": 347.2,
+ "latest": 347.2,
+ "absolute_change": 0.0,
+ "percent_change": 0.0,
+ "assessment": "Stable"
+ },
+ {
+ "benchmark": "torch",
+ "metric": "execution_time_s",
+ "unit": "seconds",
+ "baseline_sha": "075161cd9160",
+ "latest_sha": "075161cd9160",
+ "baseline": 153.779,
+ "latest": 153.779,
+ "absolute_change": 0.0,
+ "percent_change": 0.0,
+ "assessment": "Stable"
+ },
+ {
+ "benchmark": "torch",
+ "metric": "peak_memory_mb",
+ "unit": "MiB",
+ "baseline_sha": "075161cd9160",
+ "latest_sha": "075161cd9160",
+ "baseline": 4891.9,
+ "latest": 4891.9,
+ "absolute_change": 0.0,
+ "percent_change": 0.0,
+ "assessment": "Stable"
+ },
+ {
+ "benchmark": "transformers",
+ "metric": "execution_time_s",
+ "unit": "seconds",
+ "baseline_sha": "075161cd9160",
+ "latest_sha": "075161cd9160",
+ "baseline": 134.544,
+ "latest": 134.544,
+ "absolute_change": 0.0,
+ "percent_change": 0.0,
+ "assessment": "Stable"
+ },
+ {
+ "benchmark": "transformers",
+ "metric": "peak_memory_mb",
+ "unit": "MiB",
+ "baseline_sha": "075161cd9160",
+ "latest_sha": "075161cd9160",
+ "baseline": 5130.0,
+ "latest": 5130.0,
+ "absolute_change": 0.0,
+ "percent_change": 0.0,
+ "assessment": "Stable"
+ }
+]
diff --git a/docs/benchmark-results/execution_time_s.png b/docs/benchmark-results/execution_time_s.png
new file mode 100644
index 000000000000..a812ede997b4
Binary files /dev/null and b/docs/benchmark-results/execution_time_s.png differ
diff --git a/docs/benchmark-results/execution_time_s.svg b/docs/benchmark-results/execution_time_s.svg
new file mode 100644
index 000000000000..d120017cdd5a
--- /dev/null
+++ b/docs/benchmark-results/execution_time_s.svg
@@ -0,0 +1,1792 @@
+
+
+
diff --git a/docs/benchmark-results/history.csv b/docs/benchmark-results/history.csv
new file mode 100644
index 000000000000..068043a911d4
--- /dev/null
+++ b/docs/benchmark-results/history.csv
@@ -0,0 +1,19 @@
+file,benchmark,metric,value,unit,lower_is_better,commit_sha,commit_title,commit_timestamp,collection_timestamp,short_sha
+benchmark_2026-08-20_linux-x64.json,ansible,execution_time_s,21.661,seconds,True,075161cd9160dbefff81e8ec4b28a3be4786bfcb,Increase benchmark heap headroom,2026-08-20 01:10:07+00:00,2026-08-20 02:07:10.843078+00:00,075161cd9160
+benchmark_2026-08-20_linux-x64.json,ansible,peak_memory_mb,1110.7,MiB,True,075161cd9160dbefff81e8ec4b28a3be4786bfcb,Increase benchmark heap headroom,2026-08-20 01:10:07+00:00,2026-08-20 02:07:10.843078+00:00,075161cd9160
+benchmark_2026-08-20_linux-x64.json,click,execution_time_s,2.908,seconds,True,075161cd9160dbefff81e8ec4b28a3be4786bfcb,Increase benchmark heap headroom,2026-08-20 01:10:07+00:00,2026-08-20 02:07:10.843078+00:00,075161cd9160
+benchmark_2026-08-20_linux-x64.json,click,peak_memory_mb,361.0,MiB,True,075161cd9160dbefff81e8ec4b28a3be4786bfcb,Increase benchmark heap headroom,2026-08-20 01:10:07+00:00,2026-08-20 02:07:10.843078+00:00,075161cd9160
+benchmark_2026-08-20_linux-x64.json,homeassistant,execution_time_s,158.341,seconds,True,075161cd9160dbefff81e8ec4b28a3be4786bfcb,Increase benchmark heap headroom,2026-08-20 01:10:07+00:00,2026-08-20 02:07:10.843078+00:00,075161cd9160
+benchmark_2026-08-20_linux-x64.json,homeassistant,peak_memory_mb,6192.1,MiB,True,075161cd9160dbefff81e8ec4b28a3be4786bfcb,Increase benchmark heap headroom,2026-08-20 01:10:07+00:00,2026-08-20 02:07:10.843078+00:00,075161cd9160
+benchmark_2026-08-20_linux-x64.json,numpy,execution_time_s,43.435,seconds,True,075161cd9160dbefff81e8ec4b28a3be4786bfcb,Increase benchmark heap headroom,2026-08-20 01:10:07+00:00,2026-08-20 02:07:10.843078+00:00,075161cd9160
+benchmark_2026-08-20_linux-x64.json,numpy,peak_memory_mb,1850.0,MiB,True,075161cd9160dbefff81e8ec4b28a3be4786bfcb,Increase benchmark heap headroom,2026-08-20 01:10:07+00:00,2026-08-20 02:07:10.843078+00:00,075161cd9160
+benchmark_2026-08-20_linux-x64.json,pandas,execution_time_s,989.16,seconds,True,075161cd9160dbefff81e8ec4b28a3be4786bfcb,Increase benchmark heap headroom,2026-08-20 01:10:07+00:00,2026-08-20 02:07:10.843078+00:00,075161cd9160
+benchmark_2026-08-20_linux-x64.json,pandas,peak_memory_mb,4434.0,MiB,True,075161cd9160dbefff81e8ec4b28a3be4786bfcb,Increase benchmark heap headroom,2026-08-20 01:10:07+00:00,2026-08-20 02:07:10.843078+00:00,075161cd9160
+benchmark_2026-08-20_linux-x64.json,pytest,execution_time_s,12.574,seconds,True,075161cd9160dbefff81e8ec4b28a3be4786bfcb,Increase benchmark heap headroom,2026-08-20 01:10:07+00:00,2026-08-20 02:07:10.843078+00:00,075161cd9160
+benchmark_2026-08-20_linux-x64.json,pytest,peak_memory_mb,930.9,MiB,True,075161cd9160dbefff81e8ec4b28a3be4786bfcb,Increase benchmark heap headroom,2026-08-20 01:10:07+00:00,2026-08-20 02:07:10.843078+00:00,075161cd9160
+benchmark_2026-08-20_linux-x64.json,requests,execution_time_s,2.299,seconds,True,075161cd9160dbefff81e8ec4b28a3be4786bfcb,Increase benchmark heap headroom,2026-08-20 01:10:07+00:00,2026-08-20 02:07:10.843078+00:00,075161cd9160
+benchmark_2026-08-20_linux-x64.json,requests,peak_memory_mb,347.2,MiB,True,075161cd9160dbefff81e8ec4b28a3be4786bfcb,Increase benchmark heap headroom,2026-08-20 01:10:07+00:00,2026-08-20 02:07:10.843078+00:00,075161cd9160
+benchmark_2026-08-20_linux-x64.json,torch,execution_time_s,153.779,seconds,True,075161cd9160dbefff81e8ec4b28a3be4786bfcb,Increase benchmark heap headroom,2026-08-20 01:10:07+00:00,2026-08-20 02:07:10.843078+00:00,075161cd9160
+benchmark_2026-08-20_linux-x64.json,torch,peak_memory_mb,4891.9,MiB,True,075161cd9160dbefff81e8ec4b28a3be4786bfcb,Increase benchmark heap headroom,2026-08-20 01:10:07+00:00,2026-08-20 02:07:10.843078+00:00,075161cd9160
+benchmark_2026-08-20_linux-x64.json,transformers,execution_time_s,134.544,seconds,True,075161cd9160dbefff81e8ec4b28a3be4786bfcb,Increase benchmark heap headroom,2026-08-20 01:10:07+00:00,2026-08-20 02:07:10.843078+00:00,075161cd9160
+benchmark_2026-08-20_linux-x64.json,transformers,peak_memory_mb,5130.0,MiB,True,075161cd9160dbefff81e8ec4b28a3be4786bfcb,Increase benchmark heap headroom,2026-08-20 01:10:07+00:00,2026-08-20 02:07:10.843078+00:00,075161cd9160
diff --git a/docs/benchmark-results/index.html b/docs/benchmark-results/index.html
new file mode 100644
index 000000000000..20279fb052c4
--- /dev/null
+++ b/docs/benchmark-results/index.html
@@ -0,0 +1,251 @@
+
+
+
+
+
+ Pyright benchmark history
+
+
+
+
+ Pyright benchmark history
+ Latest: 075161cd9160dbefff81e8ec4b28a3be4786bfcb Increase benchmark heap headroom
+ Collected 2026-08-20 02:07:10.843078+00:00
+
+
+
+ Execution time
+ 
+
+
+ Peak memory
+ 
+
+
+ Latest compared with baseline
+
+
+
+ | Package |
+ Metric |
+ Baseline |
+ Latest |
+ Change |
+ Assessment |
+
+
+
+
+ | ansible |
+ Execution time |
+ 21.66 seconds |
+ 21.66 seconds |
+ +0.0% |
+ Stable |
+
+
+ | ansible |
+ Peak memory |
+ 1110.70 MiB |
+ 1110.70 MiB |
+ +0.0% |
+ Stable |
+
+
+ | click |
+ Execution time |
+ 2.91 seconds |
+ 2.91 seconds |
+ +0.0% |
+ Stable |
+
+
+ | click |
+ Peak memory |
+ 361.00 MiB |
+ 361.00 MiB |
+ +0.0% |
+ Stable |
+
+
+ | homeassistant |
+ Execution time |
+ 158.34 seconds |
+ 158.34 seconds |
+ +0.0% |
+ Stable |
+
+
+ | homeassistant |
+ Peak memory |
+ 6192.10 MiB |
+ 6192.10 MiB |
+ +0.0% |
+ Stable |
+
+
+ | numpy |
+ Execution time |
+ 43.44 seconds |
+ 43.44 seconds |
+ +0.0% |
+ Stable |
+
+
+ | numpy |
+ Peak memory |
+ 1850.00 MiB |
+ 1850.00 MiB |
+ +0.0% |
+ Stable |
+
+
+ | pandas |
+ Execution time |
+ 989.16 seconds |
+ 989.16 seconds |
+ +0.0% |
+ Stable |
+
+
+ | pandas |
+ Peak memory |
+ 4434.00 MiB |
+ 4434.00 MiB |
+ +0.0% |
+ Stable |
+
+
+ | pytest |
+ Execution time |
+ 12.57 seconds |
+ 12.57 seconds |
+ +0.0% |
+ Stable |
+
+
+ | pytest |
+ Peak memory |
+ 930.90 MiB |
+ 930.90 MiB |
+ +0.0% |
+ Stable |
+
+
+ | requests |
+ Execution time |
+ 2.30 seconds |
+ 2.30 seconds |
+ +0.0% |
+ Stable |
+
+
+ | requests |
+ Peak memory |
+ 347.20 MiB |
+ 347.20 MiB |
+ +0.0% |
+ Stable |
+
+
+ | torch |
+ Execution time |
+ 153.78 seconds |
+ 153.78 seconds |
+ +0.0% |
+ Stable |
+
+
+ | torch |
+ Peak memory |
+ 4891.90 MiB |
+ 4891.90 MiB |
+ +0.0% |
+ Stable |
+
+
+ | transformers |
+ Execution time |
+ 134.54 seconds |
+ 134.54 seconds |
+ +0.0% |
+ Stable |
+
+
+ | transformers |
+ Peak memory |
+ 5130.00 MiB |
+ 5130.00 MiB |
+ +0.0% |
+ Stable |
+
+
+
+
+
+
+
diff --git a/docs/benchmark-results/peak_memory_mb.png b/docs/benchmark-results/peak_memory_mb.png
new file mode 100644
index 000000000000..1a0ec1f1c3d7
Binary files /dev/null and b/docs/benchmark-results/peak_memory_mb.png differ
diff --git a/docs/benchmark-results/peak_memory_mb.svg b/docs/benchmark-results/peak_memory_mb.svg
new file mode 100644
index 000000000000..26638daeeff6
--- /dev/null
+++ b/docs/benchmark-results/peak_memory_mb.svg
@@ -0,0 +1,1811 @@
+
+
+