From 6a5831dd9727a95c392327bf4d77e9d2b32c64d8 Mon Sep 17 00:00:00 2001 From: Bill Schnurr Date: Thu, 27 Aug 2026 16:10:11 -0700 Subject: [PATCH 1/5] Cache benchmark results by base commit --- .github/workflows/typecheck_benchmark_pr.yml | 311 +++++++++++--- CONTRIBUTING.md | 7 +- build/benchmark/README.md | 60 +-- .../benchmark_2026-08-20_linux-x64.json | 405 ------------------ .../benchmark/baselines/latest-linux-x64.json | 405 ------------------ build/benchmark/compare_benchmarks.py | 142 +++++- build/benchmark/test_compare_benchmarks.py | 161 ++++--- 7 files changed, 536 insertions(+), 955 deletions(-) delete mode 100644 build/benchmark/baselines/benchmark_2026-08-20_linux-x64.json delete mode 100644 build/benchmark/baselines/latest-linux-x64.json diff --git a/.github/workflows/typecheck_benchmark_pr.yml b/.github/workflows/typecheck_benchmark_pr.yml index f455338a8028..b35bd022ab41 100644 --- a/.github/workflows/typecheck_benchmark_pr.yml +++ b/.github/workflows/typecheck_benchmark_pr.yml @@ -32,8 +32,8 @@ concurrency: cancel-in-progress: true jobs: - benchmark: - name: Compare Pyright performance + base-benchmark: + name: Benchmark base commit runs-on: ubuntu-latest timeout-minutes: 180 permissions: @@ -42,61 +42,179 @@ jobs: steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 with: - ref: ${{ inputs.merge_sha }} + ref: ${{ inputs.base_sha }} persist-credentials: false - uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6 with: python-version: ${{ env.PYTHON_VERSION }} - - uses: pnpm/action-setup@f520eceda224fe1a4aed5a2a27a194379a409996 # v6 + - name: Determine base cache key + id: cache-key + shell: bash + env: + BASE_SHA: ${{ inputs.base_sha }} + run: | + profile="${ImageOS}-${ImageVersion}-$(uname -m)-cpu$(nproc)-python${PYTHON_VERSION}-node${NODE_VERSION}-pnpm${PNPM_VERSION}-r1-w0-m8192-heap6656" + profile="$(printf '%s' "$profile" | tr -c 'A-Za-z0-9_.-' '-')" + echo "value=typecheck-benchmark-base-v1-${BASE_SHA}-${profile}" >> "$GITHUB_OUTPUT" + + - name: Restore cached base result + id: base-cache + uses: actions/cache/restore@0057852bfaa89a56745cba8c7296529d2fc39830 # v4 + with: + path: build/benchmark/base-results/latest-linux-x64.json + key: ${{ steps.cache-key.outputs.value }} + + - name: Validate cached base result + id: cached-result + shell: bash + env: + BASE_SHA: ${{ inputs.base_sha }} + run: | + result=build/benchmark/base-results/latest-linux-x64.json + if [[ "${{ steps.base-cache.outputs.cache-hit }}" == "true" ]] && \ + python -c "import hashlib,json,sys; data=json.load(open(sys.argv[1], encoding='utf-8')); digest=hashlib.sha256(b''.join(open(path, 'rb').read() for path in sys.argv[3:])).hexdigest(); sys.exit(data.get('source_revision') != sys.argv[2] or data.get('benchmark_profile_hash') != digest)" \ + "$result" "$BASE_SHA" build/benchmark/typecheck_benchmark.py \ + build/benchmark/install_envs.json && \ + python build/benchmark/compare_benchmarks.py "$result" "$result" \ + --fail-on-preparation-error; then + echo "valid=true" >> "$GITHUB_OUTPUT" + else + rm -f "$result" + echo "valid=false" >> "$GITHUB_OUTPUT" + fi + + - if: ${{ steps.cached-result.outputs.valid != 'true' }} + uses: pnpm/action-setup@f520eceda224fe1a4aed5a2a27a194379a409996 # v6 with: version: ${{ env.PNPM_VERSION }} - - uses: actions/setup-node@249970729cb0ef3589644e2896645e5dc5ba9c38 # v6.5.0 + - if: ${{ steps.cached-result.outputs.valid != 'true' }} + uses: actions/setup-node@249970729cb0ef3589644e2896645e5dc5ba9c38 # v6.5.0 with: node-version: ${{ env.NODE_VERSION }} - name: Verify build prerequisites + if: ${{ steps.cached-result.outputs.valid != 'true' }} run: | gcc --version g++ --version make --version - name: Install JavaScript dependencies + if: ${{ steps.cached-result.outputs.valid != 'true' }} timeout-minutes: 10 env: SKIP_LERNA_BOOTSTRAP: 'yes' run: pnpm install --frozen-lockfile --prefer-offline - - name: Check out trusted baseline - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + - name: Build Pyright CLI + if: ${{ steps.cached-result.outputs.valid != 'true' }} + working-directory: packages/pyright + run: pnpm run build + + - name: Benchmark base commit + if: ${{ steps.cached-result.outputs.valid != 'true' }} + env: + NODE_OPTIONS: '--max-old-space-size=6656' + PYTHONNOUSERSITE: '1' + run: | + python build/benchmark/typecheck_benchmark.py \ + -c pyright -r 1 -w 0 -t 1800 --memory-limit-mb 8192 \ + --skip-pyright-build --os-name linux-x64 \ + --output build/benchmark/base-results + + - name: Record base revision + if: ${{ steps.cached-result.outputs.valid != 'true' }} + env: + BASE_SHA: ${{ inputs.base_sha }} + run: | + python - <<'PY' + import hashlib + import json + import os + from pathlib import Path + + path = Path('build/benchmark/base-results/latest-linux-x64.json') + data = json.loads(path.read_text(encoding='utf-8')) + data['source_revision'] = os.environ['BASE_SHA'] + profile = hashlib.sha256() + for profile_path in ( + Path('build/benchmark/typecheck_benchmark.py'), + Path('build/benchmark/install_envs.json'), + ): + profile.update(profile_path.read_bytes()) + data['benchmark_profile_hash'] = profile.hexdigest() + path.write_text(json.dumps(data, indent=2) + '\n', encoding='utf-8') + PY + + - name: Validate base result + env: + BASE_SHA: ${{ inputs.base_sha }} + run: | + python build/benchmark/compare_benchmarks.py \ + build/benchmark/base-results/latest-linux-x64.json \ + build/benchmark/base-results/latest-linux-x64.json \ + --fail-on-preparation-error + python -c "import json,sys; data=json.load(open(sys.argv[1], encoding='utf-8')); sys.exit(data.get('source_revision') != sys.argv[2])" \ + build/benchmark/base-results/latest-linux-x64.json "$BASE_SHA" + + - name: Save base result cache + if: ${{ steps.cached-result.outputs.valid != 'true' }} + uses: actions/cache/save@0057852bfaa89a56745cba8c7296529d2fc39830 # v4 with: - ref: ${{ inputs.base_sha }} - path: benchmark-baseline - sparse-checkout: build/benchmark/baselines + path: build/benchmark/base-results/latest-linux-x64.json + key: ${{ steps.cache-key.outputs.value }} + + - name: Upload base result + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: typecheck-benchmark-base-${{ inputs.base_sha }} + path: build/benchmark/base-results/latest-linux-x64.json + + candidate-benchmark: + name: Benchmark pull request merge + runs-on: ubuntu-latest + timeout-minutes: 180 + permissions: + contents: read + + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + with: + ref: ${{ inputs.merge_sha }} persist-credentials: false - - name: Select benchmark baseline - id: baseline + - uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6 + with: + python-version: ${{ env.PYTHON_VERSION }} + + - uses: pnpm/action-setup@f520eceda224fe1a4aed5a2a27a194379a409996 # v6 + with: + version: ${{ env.PNPM_VERSION }} + + - uses: actions/setup-node@249970729cb0ef3589644e2896645e5dc5ba9c38 # v6.5.0 + with: + node-version: ${{ env.NODE_VERSION }} + + - name: Verify build prerequisites run: | - trusted=benchmark-baseline/build/benchmark/baselines/latest-linux-x64.json - bootstrap=build/benchmark/baselines/latest-linux-x64.json - if [[ -f "$trusted" ]]; then - echo "path=$trusted" >> "$GITHUB_OUTPUT" - elif [[ -f "$bootstrap" ]]; then - echo "Using the pull request's initial benchmark baseline" - echo "path=$bootstrap" >> "$GITHUB_OUTPUT" - else - echo "No benchmark baseline is available" >&2 - exit 1 - fi + gcc --version + g++ --version + make --version + + - name: Install JavaScript dependencies + timeout-minutes: 10 + env: + SKIP_LERNA_BOOTSTRAP: 'yes' + run: pnpm install --frozen-lockfile --prefer-offline - name: Build Pyright CLI working-directory: packages/pyright run: pnpm run build - - name: Run benchmark + - name: Benchmark pull request merge env: NODE_OPTIONS: '--max-old-space-size=6656' PYTHONNOUSERSITE: '1' @@ -104,28 +222,87 @@ jobs: python build/benchmark/typecheck_benchmark.py \ -c pyright -r 1 -w 0 -t 1800 --memory-limit-mb 8192 \ --skip-pyright-build --os-name linux-x64 \ - --output build/benchmark/results + --output build/benchmark/candidate-results - - name: Compare with baseline + - name: Record candidate revision and profile + env: + MERGE_SHA: ${{ inputs.merge_sha }} + run: | + python - <<'PY' + import hashlib + import json + import os + from pathlib import Path + + path = Path('build/benchmark/candidate-results/latest-linux-x64.json') + data = json.loads(path.read_text(encoding='utf-8')) + data['source_revision'] = os.environ['MERGE_SHA'] + profile = hashlib.sha256() + for profile_path in ( + Path('build/benchmark/typecheck_benchmark.py'), + Path('build/benchmark/install_envs.json'), + ): + profile.update(profile_path.read_bytes()) + data['benchmark_profile_hash'] = profile.hexdigest() + path.write_text(json.dumps(data, indent=2) + '\n', encoding='utf-8') + PY + + - name: Upload candidate result + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: typecheck-benchmark-candidate-${{ inputs.merge_sha }} + path: build/benchmark/candidate-results/latest-linux-x64.json + + comparison: + name: Compare Pyright performance + needs: [base-benchmark, candidate-benchmark] + runs-on: ubuntu-latest + permissions: + contents: read + + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + with: + ref: ${{ github.sha }} + persist-credentials: false + + - name: Download base result + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + name: typecheck-benchmark-base-${{ inputs.base_sha }} + path: benchmark-report/base + + - name: Download candidate result + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + name: typecheck-benchmark-candidate-${{ inputs.merge_sha }} + path: benchmark-report/candidate + + - name: Compare with base commit id: comparison continue-on-error: true + env: + BASE_SHA: ${{ inputs.base_sha }} + MERGE_SHA: ${{ inputs.merge_sha }} run: | set +e python build/benchmark/compare_benchmarks.py \ - "${{ steps.baseline.outputs.path }}" \ - build/benchmark/results/latest-linux-x64.json \ + benchmark-report/base/latest-linux-x64.json \ + benchmark-report/candidate/latest-linux-x64.json \ --fail-on-preparation-error \ - --markdown-output build/benchmark/results/report.md + --allow-incompatible \ + --baseline-revision "$BASE_SHA" --candidate-revision "$MERGE_SHA" \ + --markdown-output benchmark-report/report.md comparison_status=$? - cat build/benchmark/results/report.md >> "$GITHUB_STEP_SUMMARY" + cat benchmark-report/report.md >> "$GITHUB_STEP_SUMMARY" exit "$comparison_status" - - name: Upload candidate results + - name: Upload benchmark report if: always() uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: name: typecheck-benchmark-linux-x64-pr-${{ inputs.pr_number }}-head-${{ inputs.head_sha }}-base-${{ inputs.base_sha }}-merge-${{ inputs.merge_sha }} - path: build/benchmark/results/ + path: benchmark-report/ - name: Fail on benchmark regressions if: ${{ steps.comparison.outcome == 'failure' }} @@ -133,7 +310,7 @@ jobs: comment: name: Comment benchmark results - needs: benchmark + needs: comparison if: ${{ always() }} runs-on: ubuntu-latest permissions: @@ -143,7 +320,7 @@ jobs: steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 with: - ref: ${{ github.event.repository.default_branch }} + ref: ${{ github.sha }} persist-credentials: false - name: Download benchmark results @@ -212,7 +389,7 @@ jobs: fs.writeFileSync('benchmark-report.zip', Buffer.from(download.data)) core.setOutput('pr-number', issueNumber) - - name: Extract candidate results + - name: Extract benchmark results if: ${{ steps.download.outputs.pr-number != '' }} run: | python - <<'PY' @@ -224,22 +401,26 @@ jobs: raise ValueError(f'Non-finite JSON number: {value}') with zipfile.ZipFile('benchmark-report.zip') as archive: - matches = [ - info - for info in archive.infolist() - if PurePosixPath(info.filename).name == 'latest-linux-x64.json' - and '..' not in PurePosixPath(info.filename).parts - ] - if len(matches) != 1: - raise RuntimeError(f'Expected one candidate result, found {len(matches)}') - if matches[0].file_size > 5 * 1024 * 1024: - raise RuntimeError('Candidate result exceeds 5 MB') - contents = archive.read(matches[0]) - data = json.loads(contents, parse_constant=reject_constant) - if not isinstance(data, dict): - raise RuntimeError('Candidate result must be a JSON object') - with open('candidate.json', 'wb') as output: - output.write(contents) + entries = { + PurePosixPath(info.filename): info + for info in archive.infolist() + if '..' not in PurePosixPath(info.filename).parts + } + for archive_path, output_path in ( + (PurePosixPath('base/latest-linux-x64.json'), 'base.json'), + (PurePosixPath('candidate/latest-linux-x64.json'), 'candidate.json'), + ): + info = entries.get(archive_path) + if info is None: + raise RuntimeError(f'Missing benchmark result: {archive_path}') + if info.file_size > 5 * 1024 * 1024: + raise RuntimeError(f'Benchmark result exceeds 5 MB: {archive_path}') + contents = archive.read(info) + data = json.loads(contents, parse_constant=reject_constant) + if not isinstance(data, dict): + raise RuntimeError(f'Benchmark result must be an object: {archive_path}') + with open(output_path, 'wb') as output: + output.write(contents) PY - name: Render benchmark report @@ -247,9 +428,12 @@ jobs: run: | set +e python build/benchmark/compare_benchmarks.py \ - build/benchmark/baselines/latest-linux-x64.json \ + base.json \ candidate.json \ --fail-on-preparation-error \ + --allow-incompatible \ + --baseline-revision "${{ inputs.base_sha }}" \ + --candidate-revision "${{ inputs.merge_sha }}" \ --markdown-output report.md if [[ ! -s report.md ]]; then printf '%s\n' \ @@ -271,9 +455,28 @@ jobs: const report = fs.readFileSync('report.md', 'utf8') const issueNumber = Number(process.env.PR_NUMBER) const body = `${marker}\n${report}` - await github.rest.issues.createComment({ + const comments = await github.paginate(github.rest.issues.listComments, { owner: context.repo.owner, repo: context.repo.repo, issue_number: issueNumber, - body, + per_page: 100, }) + const previous = comments.find( + (comment) => + comment.user?.login === 'github-actions[bot]' && comment.body?.startsWith(marker) + ) + if (previous) { + await github.rest.issues.updateComment({ + owner: context.repo.owner, + repo: context.repo.repo, + comment_id: previous.id, + body, + }) + } else { + await github.rest.issues.createComment({ + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: issueNumber, + body, + }) + } diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index c9039f622791..a328320fa37d 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -21,5 +21,8 @@ See the script's `--help` and module docstring for options and methodology. To compare the local Pyright build's speed and peak memory with the latest PyPI release or with Pyrefly, ty, mypy, and Zuban on the pinned corpus, use `build/benchmark/typecheck_benchmark.py`. Maintainers can also request the hosted regression benchmark on a pull request by commenting -`/benchmark`. See [the benchmark README](build/benchmark/README.md) for the developer and maintainer -workflows, prerequisites, and methodology. +`/benchmark`. The hosted workflow compares the pull request's synthetic merge commit with its exact +base commit and reuses a validated commit-keyed base result when available. Results are attached to +the workflow and added to the existing pull-request comment without changing the pull-request branch +or retriggering code tests. See [the benchmark README](build/benchmark/README.md) for the developer +and maintainer workflows, cache behavior, prerequisites, and methodology. diff --git a/build/benchmark/README.md b/build/benchmark/README.md index 16cb357b445f..4b58cc46d205 100644 --- a/build/benchmark/README.md +++ b/build/benchmark/README.md @@ -146,17 +146,12 @@ By default, results are written to the ignored `build/benchmark/results/` direct writes a UTC-dated file such as `benchmark_2026-07-28.json` and updates `latest.json`. With `--os-name macos`, the names are `benchmark_2026-07-28_macos.json` and `latest-macos.json`. -Checked-in reference runs live in `build/benchmark/baselines/`. Generate a candidate with the same -OS label, run count, warmup count, memory limit, Python version, and package commits as its baseline, -then compare it before submitting a performance-sensitive pull request: +To compare two locally generated results, use the same OS label, run count, warmup count, memory +limit, Python version, and package commits for both runs: ```console -python build/benchmark/typecheck_benchmark.py \ - -c pyright -r 1 -w 0 -t 600 --memory-limit-mb 8192 \ - --os-name linux-x64 --output build/benchmark/results python build/benchmark/compare_benchmarks.py \ - build/benchmark/baselines/latest-linux-x64.json \ - build/benchmark/results/latest-linux-x64.json + path/to/base-result.json path/to/candidate-result.json ``` The comparator reports per-package timing and memory deltas and exits nonzero when a previously @@ -165,27 +160,38 @@ regression threshold. Package commits, check paths, and excluded directory names For Pyright, result JSON and Markdown reports also include the `--stats` file counts and phase timings for source discovery, reads, tokenization, parsing, import resolution, binding, checking, and cycle detection. This does not enable verbose or per-file logging. -Candidate-only package/checker results are reported as not regression-gated and require a regenerated -baseline before they are protected. Runner class, hosted runner image, CPU count, and the exact Python -version are part of the environment contract. The dependency-isolation mode and exact `NODE_OPTIONS` -value are also recorded and must match so results collected with different dependency or Node heap +Candidate-only package/checker results are reported as not regression-gated. Runner class, hosted +runner image, CPU count, and the exact Python version are part of the environment contract. The +dependency-isolation mode, benchmark profile hash, and exact `NODE_OPTIONS` value are also recorded +and must match so results collected with different corpus, benchmark code, dependency, or Node heap settings cannot be compared. Use `--threshold-percent` to select another threshold. Results from different runner classes are historical data, not a reliable regression gate. The invocation timeout is recorded but is not an environment compatibility field: raising a kill -threshold does not alter a checker invocation that completed below either threshold. A timed-out -candidate cannot replace a successful baseline. If a candidate succeeds where the baseline timed out -or otherwise failed, it is reported as `No baseline` until a new hosted baseline is checked in. -The pull-request workflow also fails if any candidate package cannot be prepared or any checker -times out, crashes, or otherwise fails, even when that package has no successful baseline yet. - -On pull requests, the benchmark pins Python 3.14.6, disables shared dependency caches because it -executes untrusted pull-request code, runs Pyright with a 6.5 GiB V8 old-space limit, and runs each -package once with no discarded warmup. Each invocation may take up to 30 minutes. A regression must -exceed both a 20% relative threshold and an absolute variance guard of 1 -second for time or 100 MB for peak memory. These are the comparator defaults, so the gate and trusted -comment renderer share one configuration source. Reports and artifacts are published before a failed -comparison marks the job unsuccessful. +threshold does not alter a checker invocation that completed below either threshold. The pull-request +workflow fails if either revision cannot prepare a package or if Pyright times out, crashes, or +otherwise fails. + +On pull requests, the workflow compares the synthetic merge commit with the exact base commit from +which GitHub created that merge. It never compares against a moving `main` reference. The base result +is cached under an exact key containing its commit and hosted measurement profile. A missing, expired, +or invalid cache entry causes a fresh base build and benchmark; prefix and fallback cache matches are +not used. Only the job that checks out trusted base code can populate this shared cache. The job that +executes pull-request code cannot write it. + +Both revisions use Python 3.14.6, a 6.5 GiB V8 old-space limit, one measured run, no discarded warmup, +and a 30-minute invocation timeout. A regression must exceed both a 20% relative threshold and an +absolute variance guard of 1 second for time or 100 MB for peak memory. If a pull request changes the +benchmark script, package corpus, package pins, checked paths, exclusions, or measurement environment, +both revisions must still complete successfully, but the report marks them as not comparable and does +not apply a performance regression gate. Once that change is merged, its commit becomes the base for +later pull requests and is cached normally; no checked-in baseline update is required. + +The base result, candidate result, and comparison are attached to the workflow run and rendered in the +Actions job summary and the existing benchmark pull-request comment. Publishing results does not +commit files, push a branch, or modify the pull request's head SHA, so it does not retrigger code tests. +The report comment also cannot dispatch another benchmark: the trigger accepts only a newly created +comment whose entire trimmed body is `/benchmark`, whereas later reports update the marker comment. The weekly workflow runs Pyright, Pyrefly, ty, mypy, and Zuban in independent hosted-runner jobs. Each checker performs three measured runs after one warmup over the same pinned corpus. The aggregate @@ -211,8 +217,8 @@ uncounted validation pass labeled `Check`; otherwise exactly the requested numbe reported and discarded. Dependency installation failures mark the package as unmeasured and skip its checker runs. The PR -report displays these preparation failures, but they do not count as performance regressions. If none -of a package's configured check paths exist, the benchmark warns and checks the full repository. +benchmark treats these preparation failures as errors. If none of a package's configured check paths +exist, the benchmark warns and checks the full repository. ## Relationship to `perfCompare.py` diff --git a/build/benchmark/baselines/benchmark_2026-08-20_linux-x64.json b/build/benchmark/baselines/benchmark_2026-08-20_linux-x64.json deleted file mode 100644 index 60a45469271b..000000000000 --- a/build/benchmark/baselines/benchmark_2026-08-20_linux-x64.json +++ /dev/null @@ -1,405 +0,0 @@ -{ - "timestamp": "2026-08-20T02:07:10.843078+00:00", - "date": "2026-08-20", - "platform": "linux", - "platform_details": "Linux-6.17.0-1022-azure-x86_64-with-glibc2.39", - "architecture": "x86_64", - "runner_class": "github-ubuntu-latest", - "runner_image": "ubuntu24", - "python_version": "3.14.6", - "cpu_count": 4, - "upstream_source": { - "repository_url": "https://github.com/lolpack/type_coverage_py", - "commit": "85667d6f090ce9648d88cd7a9777b492f3b95f1c", - "source_file_url": "https://github.com/lolpack/type_coverage_py/blob/85667d6f090ce9648d88cd7a9777b492f3b95f1c/typecheck_benchmark/daily_runner.py" - }, - "memory_measurement": "/proc//status", - "memory_limit_mb": 8192, - "node_options": "--max-old-space-size=6656", - "dependency_isolation": "pip-target-per-package", - "type_checkers": [ - "pyright" - ], - "type_checker_versions": { - "pyright": "1.1.413" - }, - "package_count": 9, - "runs_per_package": 1, - "warmup_runs": 0, - "uncounted_validation_runs_per_checker": 1, - "timeout_s": 1800, - "aggregate": { - "pyright": { - "packages_tested": 9, - "packages_failed": 0, - "avg_execution_time_s": 168.745, - "p50_execution_time_s": 43.435, - "p90_execution_time_s": 324.505, - "p95_execution_time_s": 656.832, - "max_execution_time_s": 989.16, - "total_execution_time_s": 1518.701, - "avg_peak_memory_mb": 2805.3, - "p50_peak_memory_mb": 1850.0, - "p90_peak_memory_mb": 5342.4, - "p95_peak_memory_mb": 5767.3, - "max_peak_memory_mb": 6192.1 - } - }, - "results": [ - { - "package_name": "ansible", - "github_url": "https://github.com/ansible/ansible", - "error": null, - "metrics": { - "pyright": { - "ok": true, - "execution_time_s": 21.661, - "peak_memory_mb": 1110.7, - "oom_killed": false, - "runs": 1, - "execution_times_s": [ - 21.661 - ], - "peak_memories_mb": [ - 1110.7 - ], - "execution_time_stats": { - "min": 21.661, - "max": 21.661, - "mean": 21.661, - "median": 21.661, - "stddev": 0.0 - }, - "peak_memory_stats": { - "min": 1110.7, - "max": 1110.7, - "mean": 1110.7, - "median": 1110.7, - "stddev": 0.0 - } - } - }, - "commit": "e8264c418ad2e87f92fa48f75cacfaa451cb38e4", - "check_paths": [ - "lib/ansible" - ], - "exclude_directories": [] - }, - { - "package_name": "click", - "github_url": "https://github.com/pallets/click", - "error": null, - "metrics": { - "pyright": { - "ok": true, - "execution_time_s": 2.908, - "peak_memory_mb": 361.0, - "oom_killed": false, - "runs": 1, - "execution_times_s": [ - 2.908 - ], - "peak_memories_mb": [ - 361.0 - ], - "execution_time_stats": { - "min": 2.908, - "max": 2.908, - "mean": 2.908, - "median": 2.908, - "stddev": 0.0 - }, - "peak_memory_stats": { - "min": 361.0, - "max": 361.0, - "mean": 361.0, - "median": 361.0, - "stddev": 0.0 - } - } - }, - "commit": "00e592cea702e0b2caa0dee42489fdb1c22cd845", - "check_paths": [ - "src/click" - ], - "exclude_directories": [] - }, - { - "package_name": "homeassistant", - "github_url": "https://github.com/home-assistant/core", - "error": null, - "metrics": { - "pyright": { - "ok": true, - "execution_time_s": 158.341, - "peak_memory_mb": 6192.1, - "oom_killed": false, - "runs": 1, - "execution_times_s": [ - 158.341 - ], - "peak_memories_mb": [ - 6192.1 - ], - "execution_time_stats": { - "min": 158.341, - "max": 158.341, - "mean": 158.341, - "median": 158.341, - "stddev": 0.0 - }, - "peak_memory_stats": { - "min": 6192.1, - "max": 6192.1, - "mean": 6192.1, - "median": 6192.1, - "stddev": 0.0 - } - } - }, - "commit": "f002c54d12077d6b906fa624f1c4bba11a71897d", - "check_paths": [ - "homeassistant" - ], - "exclude_directories": [] - }, - { - "package_name": "numpy", - "github_url": "https://github.com/numpy/numpy", - "error": null, - "metrics": { - "pyright": { - "ok": true, - "execution_time_s": 43.435, - "peak_memory_mb": 1850.0, - "oom_killed": false, - "runs": 1, - "execution_times_s": [ - 43.435 - ], - "peak_memories_mb": [ - 1850.0 - ], - "execution_time_stats": { - "min": 43.435, - "max": 43.435, - "mean": 43.435, - "median": 43.435, - "stddev": 0.0 - }, - "peak_memory_stats": { - "min": 1850.0, - "max": 1850.0, - "mean": 1850.0, - "median": 1850.0, - "stddev": 0.0 - } - } - }, - "commit": "db6ccacf630dd90f7c498e0d924bd47b66c83746", - "check_paths": [ - "numpy" - ], - "exclude_directories": [ - "tests" - ] - }, - { - "package_name": "pandas", - "github_url": "https://github.com/pandas-dev/pandas", - "error": null, - "metrics": { - "pyright": { - "ok": true, - "execution_time_s": 989.16, - "peak_memory_mb": 4434.0, - "oom_killed": false, - "runs": 1, - "execution_times_s": [ - 989.16 - ], - "peak_memories_mb": [ - 4434.0 - ], - "execution_time_stats": { - "min": 989.16, - "max": 989.16, - "mean": 989.16, - "median": 989.16, - "stddev": 0.0 - }, - "peak_memory_stats": { - "min": 4434.0, - "max": 4434.0, - "mean": 4434.0, - "median": 4434.0, - "stddev": 0.0 - } - } - }, - "commit": "982854070758cd2015fc9e64395684546b1c5444", - "check_paths": [ - "pandas" - ], - "exclude_directories": [] - }, - { - "package_name": "pytest", - "github_url": "https://github.com/pytest-dev/pytest", - "error": null, - "metrics": { - "pyright": { - "ok": true, - "execution_time_s": 12.574, - "peak_memory_mb": 930.9, - "oom_killed": false, - "runs": 1, - "execution_times_s": [ - 12.574 - ], - "peak_memories_mb": [ - 930.9 - ], - "execution_time_stats": { - "min": 12.574, - "max": 12.574, - "mean": 12.574, - "median": 12.574, - "stddev": 0.0 - }, - "peak_memory_stats": { - "min": 930.9, - "max": 930.9, - "mean": 930.9, - "median": 930.9, - "stddev": 0.0 - } - } - }, - "commit": "56b196e921acec0259d84622a570fde6032e15b5", - "check_paths": [ - "src", - "testing" - ], - "exclude_directories": [] - }, - { - "package_name": "requests", - "github_url": "https://github.com/psf/requests", - "error": null, - "metrics": { - "pyright": { - "ok": true, - "execution_time_s": 2.299, - "peak_memory_mb": 347.2, - "oom_killed": false, - "runs": 1, - "execution_times_s": [ - 2.299 - ], - "peak_memories_mb": [ - 347.2 - ], - "execution_time_stats": { - "min": 2.299, - "max": 2.299, - "mean": 2.299, - "median": 2.299, - "stddev": 0.0 - }, - "peak_memory_stats": { - "min": 347.2, - "max": 347.2, - "mean": 347.2, - "median": 347.2, - "stddev": 0.0 - } - } - }, - "commit": "414f0513c33883adf6f2b46901d4f0b38a455851", - "check_paths": [ - "src/requests" - ], - "exclude_directories": [] - }, - { - "package_name": "torch", - "github_url": "https://github.com/pytorch/pytorch", - "error": null, - "metrics": { - "pyright": { - "ok": true, - "execution_time_s": 153.779, - "peak_memory_mb": 4891.9, - "oom_killed": false, - "runs": 1, - "execution_times_s": [ - 153.779 - ], - "peak_memories_mb": [ - 4891.9 - ], - "execution_time_stats": { - "min": 153.779, - "max": 153.779, - "mean": 153.779, - "median": 153.779, - "stddev": 0.0 - }, - "peak_memory_stats": { - "min": 4891.9, - "max": 4891.9, - "mean": 4891.9, - "median": 4891.9, - "stddev": 0.0 - } - } - }, - "commit": "3ee692b380206d788625e0a4474d758f93571ee9", - "check_paths": [ - "torch" - ], - "exclude_directories": [] - }, - { - "package_name": "transformers", - "github_url": "https://github.com/huggingface/transformers", - "error": null, - "metrics": { - "pyright": { - "ok": true, - "execution_time_s": 134.544, - "peak_memory_mb": 5130.0, - "oom_killed": false, - "runs": 1, - "execution_times_s": [ - 134.544 - ], - "peak_memories_mb": [ - 5130.0 - ], - "execution_time_stats": { - "min": 134.544, - "max": 134.544, - "mean": 134.544, - "median": 134.544, - "stddev": 0.0 - }, - "peak_memory_stats": { - "min": 5130.0, - "max": 5130.0, - "mean": 5130.0, - "median": 5130.0, - "stddev": 0.0 - } - } - }, - "commit": "2ef79f87a02111f8b49a72fb7d0c86b5b0bf10b7", - "check_paths": [ - "src/transformers" - ], - "exclude_directories": [] - } - ], - "os": "linux-x64" -} diff --git a/build/benchmark/baselines/latest-linux-x64.json b/build/benchmark/baselines/latest-linux-x64.json deleted file mode 100644 index 60a45469271b..000000000000 --- a/build/benchmark/baselines/latest-linux-x64.json +++ /dev/null @@ -1,405 +0,0 @@ -{ - "timestamp": "2026-08-20T02:07:10.843078+00:00", - "date": "2026-08-20", - "platform": "linux", - "platform_details": "Linux-6.17.0-1022-azure-x86_64-with-glibc2.39", - "architecture": "x86_64", - "runner_class": "github-ubuntu-latest", - "runner_image": "ubuntu24", - "python_version": "3.14.6", - "cpu_count": 4, - "upstream_source": { - "repository_url": "https://github.com/lolpack/type_coverage_py", - "commit": "85667d6f090ce9648d88cd7a9777b492f3b95f1c", - "source_file_url": "https://github.com/lolpack/type_coverage_py/blob/85667d6f090ce9648d88cd7a9777b492f3b95f1c/typecheck_benchmark/daily_runner.py" - }, - "memory_measurement": "/proc//status", - "memory_limit_mb": 8192, - "node_options": "--max-old-space-size=6656", - "dependency_isolation": "pip-target-per-package", - "type_checkers": [ - "pyright" - ], - "type_checker_versions": { - "pyright": "1.1.413" - }, - "package_count": 9, - "runs_per_package": 1, - "warmup_runs": 0, - "uncounted_validation_runs_per_checker": 1, - "timeout_s": 1800, - "aggregate": { - "pyright": { - "packages_tested": 9, - "packages_failed": 0, - "avg_execution_time_s": 168.745, - "p50_execution_time_s": 43.435, - "p90_execution_time_s": 324.505, - "p95_execution_time_s": 656.832, - "max_execution_time_s": 989.16, - "total_execution_time_s": 1518.701, - "avg_peak_memory_mb": 2805.3, - "p50_peak_memory_mb": 1850.0, - "p90_peak_memory_mb": 5342.4, - "p95_peak_memory_mb": 5767.3, - "max_peak_memory_mb": 6192.1 - } - }, - "results": [ - { - "package_name": "ansible", - "github_url": "https://github.com/ansible/ansible", - "error": null, - "metrics": { - "pyright": { - "ok": true, - "execution_time_s": 21.661, - "peak_memory_mb": 1110.7, - "oom_killed": false, - "runs": 1, - "execution_times_s": [ - 21.661 - ], - "peak_memories_mb": [ - 1110.7 - ], - "execution_time_stats": { - "min": 21.661, - "max": 21.661, - "mean": 21.661, - "median": 21.661, - "stddev": 0.0 - }, - "peak_memory_stats": { - "min": 1110.7, - "max": 1110.7, - "mean": 1110.7, - "median": 1110.7, - "stddev": 0.0 - } - } - }, - "commit": "e8264c418ad2e87f92fa48f75cacfaa451cb38e4", - "check_paths": [ - "lib/ansible" - ], - "exclude_directories": [] - }, - { - "package_name": "click", - "github_url": "https://github.com/pallets/click", - "error": null, - "metrics": { - "pyright": { - "ok": true, - "execution_time_s": 2.908, - "peak_memory_mb": 361.0, - "oom_killed": false, - "runs": 1, - "execution_times_s": [ - 2.908 - ], - "peak_memories_mb": [ - 361.0 - ], - "execution_time_stats": { - "min": 2.908, - "max": 2.908, - "mean": 2.908, - "median": 2.908, - "stddev": 0.0 - }, - "peak_memory_stats": { - "min": 361.0, - "max": 361.0, - "mean": 361.0, - "median": 361.0, - "stddev": 0.0 - } - } - }, - "commit": "00e592cea702e0b2caa0dee42489fdb1c22cd845", - "check_paths": [ - "src/click" - ], - "exclude_directories": [] - }, - { - "package_name": "homeassistant", - "github_url": "https://github.com/home-assistant/core", - "error": null, - "metrics": { - "pyright": { - "ok": true, - "execution_time_s": 158.341, - "peak_memory_mb": 6192.1, - "oom_killed": false, - "runs": 1, - "execution_times_s": [ - 158.341 - ], - "peak_memories_mb": [ - 6192.1 - ], - "execution_time_stats": { - "min": 158.341, - "max": 158.341, - "mean": 158.341, - "median": 158.341, - "stddev": 0.0 - }, - "peak_memory_stats": { - "min": 6192.1, - "max": 6192.1, - "mean": 6192.1, - "median": 6192.1, - "stddev": 0.0 - } - } - }, - "commit": "f002c54d12077d6b906fa624f1c4bba11a71897d", - "check_paths": [ - "homeassistant" - ], - "exclude_directories": [] - }, - { - "package_name": "numpy", - "github_url": "https://github.com/numpy/numpy", - "error": null, - "metrics": { - "pyright": { - "ok": true, - "execution_time_s": 43.435, - "peak_memory_mb": 1850.0, - "oom_killed": false, - "runs": 1, - "execution_times_s": [ - 43.435 - ], - "peak_memories_mb": [ - 1850.0 - ], - "execution_time_stats": { - "min": 43.435, - "max": 43.435, - "mean": 43.435, - "median": 43.435, - "stddev": 0.0 - }, - "peak_memory_stats": { - "min": 1850.0, - "max": 1850.0, - "mean": 1850.0, - "median": 1850.0, - "stddev": 0.0 - } - } - }, - "commit": "db6ccacf630dd90f7c498e0d924bd47b66c83746", - "check_paths": [ - "numpy" - ], - "exclude_directories": [ - "tests" - ] - }, - { - "package_name": "pandas", - "github_url": "https://github.com/pandas-dev/pandas", - "error": null, - "metrics": { - "pyright": { - "ok": true, - "execution_time_s": 989.16, - "peak_memory_mb": 4434.0, - "oom_killed": false, - "runs": 1, - "execution_times_s": [ - 989.16 - ], - "peak_memories_mb": [ - 4434.0 - ], - "execution_time_stats": { - "min": 989.16, - "max": 989.16, - "mean": 989.16, - "median": 989.16, - "stddev": 0.0 - }, - "peak_memory_stats": { - "min": 4434.0, - "max": 4434.0, - "mean": 4434.0, - "median": 4434.0, - "stddev": 0.0 - } - } - }, - "commit": "982854070758cd2015fc9e64395684546b1c5444", - "check_paths": [ - "pandas" - ], - "exclude_directories": [] - }, - { - "package_name": "pytest", - "github_url": "https://github.com/pytest-dev/pytest", - "error": null, - "metrics": { - "pyright": { - "ok": true, - "execution_time_s": 12.574, - "peak_memory_mb": 930.9, - "oom_killed": false, - "runs": 1, - "execution_times_s": [ - 12.574 - ], - "peak_memories_mb": [ - 930.9 - ], - "execution_time_stats": { - "min": 12.574, - "max": 12.574, - "mean": 12.574, - "median": 12.574, - "stddev": 0.0 - }, - "peak_memory_stats": { - "min": 930.9, - "max": 930.9, - "mean": 930.9, - "median": 930.9, - "stddev": 0.0 - } - } - }, - "commit": "56b196e921acec0259d84622a570fde6032e15b5", - "check_paths": [ - "src", - "testing" - ], - "exclude_directories": [] - }, - { - "package_name": "requests", - "github_url": "https://github.com/psf/requests", - "error": null, - "metrics": { - "pyright": { - "ok": true, - "execution_time_s": 2.299, - "peak_memory_mb": 347.2, - "oom_killed": false, - "runs": 1, - "execution_times_s": [ - 2.299 - ], - "peak_memories_mb": [ - 347.2 - ], - "execution_time_stats": { - "min": 2.299, - "max": 2.299, - "mean": 2.299, - "median": 2.299, - "stddev": 0.0 - }, - "peak_memory_stats": { - "min": 347.2, - "max": 347.2, - "mean": 347.2, - "median": 347.2, - "stddev": 0.0 - } - } - }, - "commit": "414f0513c33883adf6f2b46901d4f0b38a455851", - "check_paths": [ - "src/requests" - ], - "exclude_directories": [] - }, - { - "package_name": "torch", - "github_url": "https://github.com/pytorch/pytorch", - "error": null, - "metrics": { - "pyright": { - "ok": true, - "execution_time_s": 153.779, - "peak_memory_mb": 4891.9, - "oom_killed": false, - "runs": 1, - "execution_times_s": [ - 153.779 - ], - "peak_memories_mb": [ - 4891.9 - ], - "execution_time_stats": { - "min": 153.779, - "max": 153.779, - "mean": 153.779, - "median": 153.779, - "stddev": 0.0 - }, - "peak_memory_stats": { - "min": 4891.9, - "max": 4891.9, - "mean": 4891.9, - "median": 4891.9, - "stddev": 0.0 - } - } - }, - "commit": "3ee692b380206d788625e0a4474d758f93571ee9", - "check_paths": [ - "torch" - ], - "exclude_directories": [] - }, - { - "package_name": "transformers", - "github_url": "https://github.com/huggingface/transformers", - "error": null, - "metrics": { - "pyright": { - "ok": true, - "execution_time_s": 134.544, - "peak_memory_mb": 5130.0, - "oom_killed": false, - "runs": 1, - "execution_times_s": [ - 134.544 - ], - "peak_memories_mb": [ - 5130.0 - ], - "execution_time_stats": { - "min": 134.544, - "max": 134.544, - "mean": 134.544, - "median": 134.544, - "stddev": 0.0 - }, - "peak_memory_stats": { - "min": 5130.0, - "max": 5130.0, - "mean": 5130.0, - "median": 5130.0, - "stddev": 0.0 - } - } - }, - "commit": "2ef79f87a02111f8b49a72fb7d0c86b5b0bf10b7", - "check_paths": [ - "src/transformers" - ], - "exclude_directories": [] - } - ], - "os": "linux-x64" -} diff --git a/build/benchmark/compare_benchmarks.py b/build/benchmark/compare_benchmarks.py index d0879a007aad..79575810d202 100644 --- a/build/benchmark/compare_benchmarks.py +++ b/build/benchmark/compare_benchmarks.py @@ -13,6 +13,21 @@ DEFAULT_TIME_NOISE_FLOOR_SECONDS = 1.0 DEFAULT_MEMORY_NOISE_FLOOR_MB = 100.0 +ENVIRONMENT_FIELDS = ( + "platform", + "architecture", + "runner_class", + "runner_image", + "cpu_count", + "python_version", + "memory_limit_mb", + "node_options", + "runs_per_package", + "warmup_runs", + "dependency_isolation", + "benchmark_profile_hash", +) + class ComparisonRow(TypedDict, total=False): package: str @@ -98,10 +113,58 @@ def _packages_by_name(data: dict[str, Any]) -> dict[str, dict[str, Any]]: return { package["package_name"]: package for package in data.get("results", []) - if package.get("package_name") + if isinstance(package, dict) and package.get("package_name") } +def _compatibility_failures( + baseline: dict[str, Any], candidate: dict[str, Any] +) -> list[str]: + failures: list[str] = [] + for field in ENVIRONMENT_FIELDS: + if baseline.get(field) != candidate.get(field): + failures.append( + f"environment mismatch for {field}: " + f"{baseline.get(field)!r} != {candidate.get(field)!r}" + ) + + baseline_packages = _packages_by_name(baseline) + candidate_packages = _packages_by_name(candidate) + if baseline_packages.keys() != candidate_packages.keys(): + failures.append( + "benchmark package set changed from " + f"{sorted(baseline_packages)} to {sorted(candidate_packages)}" + ) + for package_name in sorted(baseline_packages.keys() & candidate_packages.keys()): + old_package = baseline_packages[package_name] + new_package = candidate_packages[package_name] + if old_package.get("commit") != new_package.get("commit"): + failures.append( + f"{package_name}: package commit changed from " + f"{old_package.get('commit')} to {new_package.get('commit')}" + ) + for field in ("check_paths", "exclude_directories"): + if old_package.get(field, []) != new_package.get(field, []): + failures.append( + f"{package_name}: package {field} changed from " + f"{old_package.get(field, [])} to {new_package.get(field, [])}" + ) + return failures + + +def _measurement_failures(data: dict[str, Any], label: str) -> list[str]: + failures: list[str] = [] + for package in data.get("results", []): + package_name = package.get("package_name", "unknown") + if package.get("error"): + failures.append(f"{label}: {package_name} package preparation failed") + continue + metrics = package.get("metrics", {}).get("pyright") + if not isinstance(metrics, dict) or not metrics.get("ok"): + failures.append(f"{label}: {package_name}/pyright result failed or is missing") + return failures + + def _percent_change(baseline: float, candidate: float) -> float: return ((candidate - baseline) / baseline) * 100 if baseline else 0.0 @@ -120,27 +183,32 @@ def _analyze( time_noise_floor_s: float = 0.0, memory_noise_floor_mb: float = 0.0, fail_on_preparation_error: bool = False, + baseline_revision: str | None = None, + candidate_revision: str | None = None, + allow_incompatible: bool = False, ) -> tuple[list[str], list[ComparisonRow]]: failures = [ *_validate_results(baseline, "baseline"), *_validate_results(candidate, "candidate"), ] rows: list[ComparisonRow] = [] + for data, label, expected_revision in ( + (baseline, "baseline", baseline_revision), + (candidate, "candidate", candidate_revision), + ): + if expected_revision and data.get("source_revision") != expected_revision: + failures.append( + f"{label}: source revision {data.get('source_revision')!r} " + f"does not match {expected_revision!r}" + ) if failures: return failures, rows - for field in ( - "platform", - "architecture", - "runner_class", - "runner_image", - "cpu_count", - "python_version", - "memory_limit_mb", - "node_options", - "runs_per_package", - "warmup_runs", - "dependency_isolation", - ): + compatibility_failures = _compatibility_failures(baseline, candidate) + if allow_incompatible and compatibility_failures: + failures.extend(_measurement_failures(baseline, "baseline")) + failures.extend(_measurement_failures(candidate, "candidate")) + return failures, rows + for field in ENVIRONMENT_FIELDS: if baseline.get(field) != candidate.get(field): failures.append( f"environment mismatch for {field}: " @@ -343,6 +411,9 @@ def compare( time_noise_floor_s: float = 0.0, memory_noise_floor_mb: float = 0.0, fail_on_preparation_error: bool = False, + baseline_revision: str | None = None, + candidate_revision: str | None = None, + allow_incompatible: bool = False, ) -> list[str]: failures, rows = _analyze( baseline, @@ -351,6 +422,9 @@ def compare( time_noise_floor_s, memory_noise_floor_mb, fail_on_preparation_error, + baseline_revision, + candidate_revision, + allow_incompatible, ) print( f"{'Package':<20} {'Checker':<10} {'Time':>10} {'Delta':>9} " @@ -382,6 +456,9 @@ def render_markdown( time_noise_floor_s: float = 0.0, memory_noise_floor_mb: float = 0.0, fail_on_preparation_error: bool = False, + baseline_revision: str | None = None, + candidate_revision: str | None = None, + allow_incompatible: bool = False, ) -> str: failures, rows = _analyze( baseline, @@ -390,7 +467,28 @@ def render_markdown( time_noise_floor_s, memory_noise_floor_mb, fail_on_preparation_error, + baseline_revision, + candidate_revision, + allow_incompatible, ) + compatibility_failures = _compatibility_failures(baseline, candidate) + if allow_incompatible and compatibility_failures and not failures: + lines = [ + "## Type checker benchmark", + "", + "🟡 **Performance results are not comparable because the benchmark profile changed.**", + ] + if baseline_revision and candidate_revision: + lines.extend( + [ + "", + f"Base commit: `{baseline_revision}`", + f"Candidate merge commit: `{candidate_revision}`", + ] + ) + lines.extend(["", "### Compatibility changes", ""]) + lines.extend(f"- {_escape_markdown(failure)}" for failure in compatibility_failures) + return "\n".join(lines) + "\n" if failures: summary = f"🔴 **{len(failures)} regression check(s) failed.**" else: @@ -416,6 +514,13 @@ def render_markdown( "", f"Regression threshold: `{threshold_percent:.1f}%`", ] + if baseline_revision and candidate_revision: + lines.extend( + [ + f"Base commit: `{baseline_revision}`", + f"Candidate merge commit: `{candidate_revision}`", + ] + ) if time_noise_floor_s > 0 or memory_noise_floor_mb > 0: lines.append( f"Variance guard: `>{time_noise_floor_s:.1f}s` time and " @@ -525,6 +630,9 @@ def main(argv: list[str] | None = None) -> int: default=DEFAULT_MEMORY_NOISE_FLOOR_MB, ) parser.add_argument("--fail-on-preparation-error", action="store_true") + parser.add_argument("--baseline-revision") + parser.add_argument("--candidate-revision") + parser.add_argument("--allow-incompatible", action="store_true") parser.add_argument("--markdown-output", type=Path) args = parser.parse_args(argv) @@ -542,6 +650,9 @@ def main(argv: list[str] | None = None) -> int: args.time_noise_floor_seconds, args.memory_noise_floor_mb, args.fail_on_preparation_error, + args.baseline_revision, + args.candidate_revision, + args.allow_incompatible, ) if args.markdown_output: args.markdown_output.write_text( @@ -552,6 +663,9 @@ def main(argv: list[str] | None = None) -> int: args.time_noise_floor_seconds, args.memory_noise_floor_mb, args.fail_on_preparation_error, + args.baseline_revision, + args.candidate_revision, + args.allow_incompatible, ), encoding="utf-8", ) diff --git a/build/benchmark/test_compare_benchmarks.py b/build/benchmark/test_compare_benchmarks.py index a10680716c3a..ace8a4a629d3 100644 --- a/build/benchmark/test_compare_benchmarks.py +++ b/build/benchmark/test_compare_benchmarks.py @@ -43,6 +43,7 @@ def _result(time: float, memory: float, ok: bool = True) -> dict: "uncounted_validation_runs_per_checker": 0, "timeout_s": 600, "dependency_isolation": "pip-target-per-package", + "benchmark_profile_hash": "profile-v1", "results": [ { "package_name": "example", @@ -430,48 +431,97 @@ def test_load_rejects_non_finite_json_numbers(self) -> None: with self.assertRaisesRegex(ValueError, "non-finite number NaN"): compare_benchmarks._load_results(result_file) - def test_workflow_profile_matches_checked_in_baseline(self) -> None: + def test_rejects_unexpected_source_revision(self) -> None: + baseline = _result(10.0, 100.0) + candidate = _result(10.0, 100.0) + baseline["source_revision"] = "a" * 40 + candidate["source_revision"] = "b" * 40 + + failures = compare_benchmarks.compare( + baseline, + candidate, + 10.0, + baseline_revision="c" * 40, + candidate_revision="b" * 40, + ) + + self.assertEqual( + failures, + [ + "baseline: source revision " + f"{'a' * 40!r} does not match {'c' * 40!r}" + ], + ) + + def test_report_identifies_compared_revisions(self) -> None: + baseline = _result(10.0, 100.0) + candidate = _result(10.0, 100.0) + baseline_revision = "a" * 40 + candidate_revision = "b" * 40 + baseline["source_revision"] = baseline_revision + candidate["source_revision"] = candidate_revision + + report = compare_benchmarks.render_markdown( + baseline, + candidate, + 10.0, + baseline_revision=baseline_revision, + candidate_revision=candidate_revision, + ) + + self.assertIn(f"Base commit: `{baseline_revision}`", report) + self.assertIn(f"Candidate merge commit: `{candidate_revision}`", report) + + def test_allows_successful_benchmark_profile_change(self) -> None: + candidate = _result(10.0, 100.0) + candidate["benchmark_profile_hash"] = "profile-v2" + + failures = compare_benchmarks.compare( + _result(10.0, 100.0), + candidate, + 10.0, + allow_incompatible=True, + ) + report = compare_benchmarks.render_markdown( + _result(10.0, 100.0), + candidate, + 10.0, + allow_incompatible=True, + ) + + self.assertEqual(failures, []) + self.assertIn("Performance results are not comparable", report) + self.assertIn(r"benchmark\_profile\_hash", report) + + def test_incompatible_results_still_require_successful_measurements(self) -> None: + candidate = _result(0.0, 0.0, ok=False) + candidate["benchmark_profile_hash"] = "profile-v2" + + failures = compare_benchmarks.compare( + _result(10.0, 100.0), + candidate, + 10.0, + allow_incompatible=True, + ) + + self.assertEqual( + failures, ["candidate: example/pyright result failed or is missing"] + ) + + def test_pr_workflow_uses_matching_base_and_candidate_profiles(self) -> None: workflow = ( REPO_ROOT / ".github" / "workflows" / "typecheck_benchmark_pr.yml" ).read_text(encoding="utf-8") - timeout_match = re.search( + timeout_matches = re.findall( r"typecheck_benchmark\.py \\\s+" r"-c pyright -r 1 -w 0 -t (\d+)", workflow, ) - self.assertIsNotNone(timeout_match) - - baseline = json.loads( - ( - REPO_ROOT - / "build" - / "benchmark" - / "baselines" - / "latest-linux-x64.json" - ).read_text(encoding="utf-8") - ) - config = json.loads( - ( - REPO_ROOT / "build" / "benchmark" / "install_envs.json" - ).read_text(encoding="utf-8") - ) - - self.assertEqual(int(timeout_match.group(1)), 1800) - baseline_packages = { - package["package_name"]: package for package in baseline["results"] - } - for package in config["packages"]: - package_name = package.get("name") or package["github_url"].rsplit( - "/", 1 - )[-1] - baseline_package = baseline_packages[package_name] - self.assertEqual( - package.get("check_paths", []), baseline_package["check_paths"] - ) - self.assertEqual( - package.get("exclude_directories", []), - baseline_package["exclude_directories"], - ) + self.assertEqual(timeout_matches, ["1800", "1800"]) + self.assertIn("data['source_revision'] = os.environ['BASE_SHA']", workflow) + self.assertIn("data['source_revision'] = os.environ['MERGE_SHA']", workflow) + self.assertIn("data['benchmark_profile_hash'] = profile.hexdigest()", workflow) + self.assertNotIn("build/benchmark/baselines/", workflow) def test_workflows_use_current_pnpm_setup(self) -> None: for workflow_name in ( @@ -587,9 +637,13 @@ def test_pr_benchmark_requires_authorized_comment(self) -> None: "pullRequest.data.merge_commit_sha !== expectedMergeSha", benchmark_workflow, ) - benchmark_job = benchmark_workflow_data["jobs"]["benchmark"] + base_job = benchmark_workflow_data["jobs"]["base-benchmark"] + candidate_job = benchmark_workflow_data["jobs"]["candidate-benchmark"] + comparison_job = benchmark_workflow_data["jobs"]["comparison"] comment_job = benchmark_workflow_data["jobs"]["comment"] - self.assertEqual(benchmark_job["permissions"], {"contents": "read"}) + self.assertEqual(base_job["permissions"], {"contents": "read"}) + self.assertEqual(candidate_job["permissions"], {"contents": "read"}) + self.assertEqual(comparison_job["permissions"], {"contents": "read"}) self.assertEqual( comment_job["permissions"], { @@ -598,7 +652,7 @@ def test_pr_benchmark_requires_authorized_comment(self) -> None: "pull-requests": "write", }, ) - self.assertEqual(comment_job["needs"], "benchmark") + self.assertEqual(comment_job["needs"], "comparison") self.assertEqual( [ job_name @@ -616,20 +670,31 @@ def test_pr_benchmark_requires_authorized_comment(self) -> None: ).exists() ) - def test_pr_workflow_prefers_trusted_baseline_with_bootstrap_fallback(self) -> None: + def test_pr_workflow_caches_only_the_base_result(self) -> None: workflow = ( REPO_ROOT / ".github" / "workflows" / "typecheck_benchmark_pr.yml" ).read_text(encoding="utf-8") - - trusted = "benchmark-baseline/build/benchmark/baselines/latest-linux-x64.json" - bootstrap = "build/benchmark/baselines/latest-linux-x64.json" - self.assertLess( - workflow.index('if [[ -f "$trusted" ]]'), - workflow.index('elif [[ -f "$bootstrap" ]]'), + workflow_data = _load_yaml( + REPO_ROOT / ".github" / "workflows" / "typecheck_benchmark_pr.yml" ) - self.assertIn('echo "path=$trusted" >> "$GITHUB_OUTPUT"', workflow) - self.assertIn('echo "path=$bootstrap" >> "$GITHUB_OUTPUT"', workflow) - self.assertIn('"${{ steps.baseline.outputs.path }}"', workflow) + base_job = json.dumps(workflow_data["jobs"]["base-benchmark"]) + candidate_job = json.dumps(workflow_data["jobs"]["candidate-benchmark"]) + + self.assertIn("actions/cache/restore@0057852", base_job) + self.assertIn("actions/cache/save@0057852", base_job) + self.assertNotIn("restore-keys", base_job) + self.assertNotIn("actions/cache/", candidate_job) + self.assertIn("ref: ${{ inputs.base_sha }}", workflow) + self.assertIn("ref: ${{ inputs.merge_sha }}", workflow) + self.assertIn("--baseline-revision", workflow) + self.assertIn("--candidate-revision", workflow) + self.assertIn("--allow-incompatible", workflow) + self.assertIn("issues.listComments", workflow) + self.assertIn("updateComment", workflow) + self.assertIn("comment.user?.login === 'github-actions[bot]'", workflow) + self.assertIn("ref: ${{ github.sha }}", workflow) + self.assertNotIn("git push", workflow) + self.assertNotIn("contents: write", workflow) if __name__ == "__main__": From 484a3438e8b5653ecc20521064f387df212189be Mon Sep 17 00:00:00 2001 From: Bill Schnurr Date: Thu, 27 Aug 2026 17:58:27 -0700 Subject: [PATCH 2/5] Address benchmark workflow review feedback Resolve PR revisions from trusted GitHub metadata before executing benchmark code. Retain and update checked-in baseline results with commit provenance, and add a notebook for benchmark history visualization and dashboard export. --- .gitattributes | 1 + .github/workflows/typecheck_benchmark_pr.yml | 217 ++++++-- .../workflows/typecheck_benchmark_trigger.yml | 12 - CONTRIBUTING.md | 7 +- build/benchmark/README.md | 36 +- .../benchmark_2026-08-20_linux-x64.json | 408 ++++++++++++++ .../benchmark/baselines/latest-linux-x64.json | 408 ++++++++++++++ build/benchmark/benchmark_history.ipynb | 506 ++++++++++++++++++ build/benchmark/requirements-notebook.txt | 5 + build/benchmark/test_compare_benchmarks.py | 70 ++- 10 files changed, 1596 insertions(+), 74 deletions(-) create mode 100644 build/benchmark/baselines/benchmark_2026-08-20_linux-x64.json create mode 100644 build/benchmark/baselines/latest-linux-x64.json create mode 100644 build/benchmark/benchmark_history.ipynb create mode 100644 build/benchmark/requirements-notebook.txt diff --git a/.gitattributes b/.gitattributes index 628472cde462..6d46a68bfcee 100644 --- a/.gitattributes +++ b/.gitattributes @@ -1,2 +1,3 @@ *.json linguist-language=JSON-with-Comments +*.ipynb text eol=lf **/pnpm-lock.yaml text eol=lf diff --git a/.github/workflows/typecheck_benchmark_pr.yml b/.github/workflows/typecheck_benchmark_pr.yml index b35bd022ab41..85df25207167 100644 --- a/.github/workflows/typecheck_benchmark_pr.yml +++ b/.github/workflows/typecheck_benchmark_pr.yml @@ -14,35 +14,65 @@ on: description: Pull request number required: true type: string - head_sha: - description: Pull request head commit - required: true - type: string - base_sha: - description: Pull request base commit - required: true - type: string - merge_sha: - description: Pull request merge commit - required: true - type: string concurrency: group: ${{ github.workflow }}-${{ inputs.pr_number }} cancel-in-progress: true jobs: + metadata: + name: Resolve pull request revisions + runs-on: ubuntu-latest + permissions: + pull-requests: read + outputs: + head-sha: ${{ steps.pull-request.outputs.head-sha }} + head-ref: ${{ steps.pull-request.outputs.head-ref }} + head-repository: ${{ steps.pull-request.outputs.head-repository }} + base-sha: ${{ steps.pull-request.outputs.base-sha }} + merge-sha: ${{ steps.pull-request.outputs.merge-sha }} + steps: + - name: Read pull request metadata + id: pull-request + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 + env: + PR_NUMBER: ${{ inputs.pr_number }} + with: + script: | + const issueNumber = Number(process.env.PR_NUMBER) + if (!Number.isSafeInteger(issueNumber) || issueNumber <= 0) { + core.setFailed('The pull request number is invalid') + return + } + const pullRequest = await github.rest.pulls.get({ + owner: context.repo.owner, + repo: context.repo.repo, + pull_number: issueNumber, + }) + if (pullRequest.data.state !== 'open' || !pullRequest.data.merge_commit_sha) { + core.setFailed('The pull request is not open or does not have a merge commit') + return + } + core.setOutput('head-sha', pullRequest.data.head.sha) + core.setOutput('head-ref', pullRequest.data.head.ref) + core.setOutput('head-repository', pullRequest.data.head.repo.full_name) + core.setOutput('base-sha', pullRequest.data.base.sha) + core.setOutput('merge-sha', pullRequest.data.merge_commit_sha) + base-benchmark: name: Benchmark base commit + needs: metadata runs-on: ubuntu-latest timeout-minutes: 180 permissions: contents: read + outputs: + cached: ${{ steps.cached-result.outputs.valid }} steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 with: - ref: ${{ inputs.base_sha }} + ref: ${{ needs.metadata.outputs.base-sha }} persist-credentials: false - uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6 @@ -53,11 +83,11 @@ jobs: id: cache-key shell: bash env: - BASE_SHA: ${{ inputs.base_sha }} + BASE_SHA: ${{ needs.metadata.outputs.base-sha }} run: | profile="${ImageOS}-${ImageVersion}-$(uname -m)-cpu$(nproc)-python${PYTHON_VERSION}-node${NODE_VERSION}-pnpm${PNPM_VERSION}-r1-w0-m8192-heap6656" profile="$(printf '%s' "$profile" | tr -c 'A-Za-z0-9_.-' '-')" - echo "value=typecheck-benchmark-base-v1-${BASE_SHA}-${profile}" >> "$GITHUB_OUTPUT" + echo "value=typecheck-benchmark-base-v2-${BASE_SHA}-${profile}" >> "$GITHUB_OUTPUT" - name: Restore cached base result id: base-cache @@ -70,11 +100,11 @@ jobs: id: cached-result shell: bash env: - BASE_SHA: ${{ inputs.base_sha }} + BASE_SHA: ${{ needs.metadata.outputs.base-sha }} run: | result=build/benchmark/base-results/latest-linux-x64.json if [[ "${{ steps.base-cache.outputs.cache-hit }}" == "true" ]] && \ - python -c "import hashlib,json,sys; data=json.load(open(sys.argv[1], encoding='utf-8')); digest=hashlib.sha256(b''.join(open(path, 'rb').read() for path in sys.argv[3:])).hexdigest(); sys.exit(data.get('source_revision') != sys.argv[2] or data.get('benchmark_profile_hash') != digest)" \ + python -c "import hashlib,json,sys; data=json.load(open(sys.argv[1], encoding='utf-8')); digest=hashlib.sha256(b''.join(open(path, 'rb').read() for path in sys.argv[3:])).hexdigest(); sys.exit(data.get('source_revision') != sys.argv[2] or not data.get('source_commit_subject') or not data.get('source_commit_timestamp') or data.get('benchmark_profile_hash') != digest)" \ "$result" "$BASE_SHA" build/benchmark/typecheck_benchmark.py \ build/benchmark/install_envs.json && \ python build/benchmark/compare_benchmarks.py "$result" "$result" \ @@ -128,17 +158,24 @@ jobs: - name: Record base revision if: ${{ steps.cached-result.outputs.valid != 'true' }} env: - BASE_SHA: ${{ inputs.base_sha }} + BASE_SHA: ${{ needs.metadata.outputs.base-sha }} run: | python - <<'PY' import hashlib import json import os + import subprocess from pathlib import Path path = Path('build/benchmark/base-results/latest-linux-x64.json') data = json.loads(path.read_text(encoding='utf-8')) data['source_revision'] = os.environ['BASE_SHA'] + data['source_commit_subject'] = subprocess.check_output( + ['git', 'show', '-s', '--format=%s', os.environ['BASE_SHA']], text=True + ).strip() + data['source_commit_timestamp'] = subprocess.check_output( + ['git', 'show', '-s', '--format=%cI', os.environ['BASE_SHA']], text=True + ).strip() profile = hashlib.sha256() for profile_path in ( Path('build/benchmark/typecheck_benchmark.py'), @@ -151,7 +188,7 @@ jobs: - name: Validate base result env: - BASE_SHA: ${{ inputs.base_sha }} + BASE_SHA: ${{ needs.metadata.outputs.base-sha }} run: | python build/benchmark/compare_benchmarks.py \ build/benchmark/base-results/latest-linux-x64.json \ @@ -170,11 +207,12 @@ jobs: - name: Upload base result uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: - name: typecheck-benchmark-base-${{ inputs.base_sha }} + name: typecheck-benchmark-base-${{ needs.metadata.outputs.base-sha }} path: build/benchmark/base-results/latest-linux-x64.json candidate-benchmark: name: Benchmark pull request merge + needs: metadata runs-on: ubuntu-latest timeout-minutes: 180 permissions: @@ -183,7 +221,7 @@ jobs: steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 with: - ref: ${{ inputs.merge_sha }} + ref: ${{ needs.metadata.outputs.merge-sha }} persist-credentials: false - uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6 @@ -226,17 +264,24 @@ jobs: - name: Record candidate revision and profile env: - MERGE_SHA: ${{ inputs.merge_sha }} + MERGE_SHA: ${{ needs.metadata.outputs.merge-sha }} run: | python - <<'PY' import hashlib import json import os + import subprocess from pathlib import Path path = Path('build/benchmark/candidate-results/latest-linux-x64.json') data = json.loads(path.read_text(encoding='utf-8')) data['source_revision'] = os.environ['MERGE_SHA'] + data['source_commit_subject'] = subprocess.check_output( + ['git', 'show', '-s', '--format=%s', os.environ['MERGE_SHA']], text=True + ).strip() + data['source_commit_timestamp'] = subprocess.check_output( + ['git', 'show', '-s', '--format=%cI', os.environ['MERGE_SHA']], text=True + ).strip() profile = hashlib.sha256() for profile_path in ( Path('build/benchmark/typecheck_benchmark.py'), @@ -250,12 +295,12 @@ jobs: - name: Upload candidate result uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: - name: typecheck-benchmark-candidate-${{ inputs.merge_sha }} + name: typecheck-benchmark-candidate-${{ needs.metadata.outputs.merge-sha }} path: build/benchmark/candidate-results/latest-linux-x64.json comparison: name: Compare Pyright performance - needs: [base-benchmark, candidate-benchmark] + needs: [metadata, base-benchmark, candidate-benchmark] runs-on: ubuntu-latest permissions: contents: read @@ -269,21 +314,21 @@ jobs: - name: Download base result uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 with: - name: typecheck-benchmark-base-${{ inputs.base_sha }} + name: typecheck-benchmark-base-${{ needs.metadata.outputs.base-sha }} path: benchmark-report/base - name: Download candidate result uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 with: - name: typecheck-benchmark-candidate-${{ inputs.merge_sha }} + name: typecheck-benchmark-candidate-${{ needs.metadata.outputs.merge-sha }} path: benchmark-report/candidate - name: Compare with base commit id: comparison continue-on-error: true env: - BASE_SHA: ${{ inputs.base_sha }} - MERGE_SHA: ${{ inputs.merge_sha }} + BASE_SHA: ${{ needs.metadata.outputs.base-sha }} + MERGE_SHA: ${{ needs.metadata.outputs.merge-sha }} run: | set +e python build/benchmark/compare_benchmarks.py \ @@ -301,7 +346,7 @@ jobs: if: always() uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: - name: typecheck-benchmark-linux-x64-pr-${{ inputs.pr_number }}-head-${{ inputs.head_sha }}-base-${{ inputs.base_sha }}-merge-${{ inputs.merge_sha }} + name: typecheck-benchmark-linux-x64-pr-${{ inputs.pr_number }}-head-${{ needs.metadata.outputs.head-sha }}-base-${{ needs.metadata.outputs.base-sha }}-merge-${{ needs.metadata.outputs.merge-sha }} path: benchmark-report/ - name: Fail on benchmark regressions @@ -310,8 +355,8 @@ jobs: comment: name: Comment benchmark results - needs: comparison - if: ${{ always() }} + needs: [metadata, comparison] + if: ${{ always() && needs.metadata.result == 'success' }} runs-on: ubuntu-latest permissions: actions: read @@ -328,9 +373,9 @@ jobs: uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 env: PR_NUMBER: ${{ inputs.pr_number }} - EXPECTED_HEAD_SHA: ${{ inputs.head_sha }} - EXPECTED_BASE_SHA: ${{ inputs.base_sha }} - EXPECTED_MERGE_SHA: ${{ inputs.merge_sha }} + EXPECTED_HEAD_SHA: ${{ needs.metadata.outputs.head-sha }} + EXPECTED_BASE_SHA: ${{ needs.metadata.outputs.base-sha }} + EXPECTED_MERGE_SHA: ${{ needs.metadata.outputs.merge-sha }} with: script: | const fs = require('fs') @@ -432,8 +477,8 @@ jobs: candidate.json \ --fail-on-preparation-error \ --allow-incompatible \ - --baseline-revision "${{ inputs.base_sha }}" \ - --candidate-revision "${{ inputs.merge_sha }}" \ + --baseline-revision "${{ needs.metadata.outputs.base-sha }}" \ + --candidate-revision "${{ needs.metadata.outputs.merge-sha }}" \ --markdown-output report.md if [[ ! -s report.md ]]; then printf '%s\n' \ @@ -480,3 +525,103 @@ jobs: body, }) } + + persist-base-result: + name: Update pull request baseline + needs: [metadata, base-benchmark, comment] + if: >- + ${{ !cancelled() && needs.base-benchmark.result == 'success' && + needs.base-benchmark.outputs.cached != 'true' && + needs.metadata.outputs.head-repository == github.repository }} + runs-on: ubuntu-latest + permissions: + actions: read + contents: write + steps: + - name: Download base result + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + name: typecheck-benchmark-base-${{ needs.metadata.outputs.base-sha }} + path: benchmark-result + + - name: Commit baseline to pull request branch + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 + env: + EXPECTED_HEAD_SHA: ${{ needs.metadata.outputs.head-sha }} + HEAD_REF: ${{ needs.metadata.outputs.head-ref }} + BASE_SHA: ${{ needs.metadata.outputs.base-sha }} + with: + script: | + const fs = require('fs') + const path = 'benchmark-result/latest-linux-x64.json' + const contents = fs.readFileSync(path, 'utf8') + const result = JSON.parse(contents) + const expectedHeadSha = process.env.EXPECTED_HEAD_SHA + const headRef = process.env.HEAD_REF + const baseSha = process.env.BASE_SHA + if ( + result.source_revision !== baseSha || + typeof result.source_commit_subject !== 'string' || + result.source_commit_subject.length === 0 || + Number.isNaN(Date.parse(result.source_commit_timestamp)) || + Number.isNaN(Date.parse(result.timestamp)) || + !/^\d{4}-\d{2}-\d{2}$/.test(result.date) + ) { + core.setFailed('The base benchmark result metadata is invalid') + return + } + + const currentRef = await github.rest.git.getRef({ + owner: context.repo.owner, + repo: context.repo.repo, + ref: `heads/${headRef}`, + }) + if (currentRef.data.object.sha !== expectedHeadSha) { + core.notice('The pull request head changed; skipping the baseline update') + return + } + + const parent = await github.rest.git.getCommit({ + owner: context.repo.owner, + repo: context.repo.repo, + commit_sha: expectedHeadSha, + }) + const blob = await github.rest.git.createBlob({ + owner: context.repo.owner, + repo: context.repo.repo, + content: Buffer.from(contents).toString('base64'), + encoding: 'base64', + }) + const tree = await github.rest.git.createTree({ + owner: context.repo.owner, + repo: context.repo.repo, + base_tree: parent.data.tree.sha, + tree: [ + { + path: 'build/benchmark/baselines/latest-linux-x64.json', + mode: '100644', + type: 'blob', + sha: blob.data.sha, + }, + { + path: `build/benchmark/baselines/benchmark_${result.date}_linux-x64.json`, + mode: '100644', + type: 'blob', + sha: blob.data.sha, + }, + ], + }) + const commit = await github.rest.git.createCommit({ + owner: context.repo.owner, + repo: context.repo.repo, + message: `Update benchmark baseline for ${baseSha.slice(0, 12)}`, + tree: tree.data.sha, + parents: [expectedHeadSha], + }) + await github.rest.git.updateRef({ + owner: context.repo.owner, + repo: context.repo.repo, + ref: `heads/${headRef}`, + sha: commit.data.sha, + force: false, + }) diff --git a/.github/workflows/typecheck_benchmark_trigger.yml b/.github/workflows/typecheck_benchmark_trigger.yml index c8481d9b8392..686b3b0641b2 100644 --- a/.github/workflows/typecheck_benchmark_trigger.yml +++ b/.github/workflows/typecheck_benchmark_trigger.yml @@ -36,15 +36,6 @@ jobs: return } - const pullRequest = await github.rest.pulls.get({ - owner: context.repo.owner, - repo: context.repo.repo, - pull_number: context.issue.number, - }) - if (!pullRequest.data.merge_commit_sha) { - core.setFailed('The pull request does not have a merge commit to benchmark') - return - } await github.rest.actions.createWorkflowDispatch({ owner: context.repo.owner, repo: context.repo.repo, @@ -52,8 +43,5 @@ jobs: ref: context.payload.repository.default_branch, inputs: { pr_number: String(context.issue.number), - head_sha: pullRequest.data.head.sha, - base_sha: pullRequest.data.base.sha, - merge_sha: pullRequest.data.merge_commit_sha, }, }) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index a328320fa37d..54af8c11e182 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -23,6 +23,7 @@ Pyrefly, ty, mypy, and Zuban on the pinned corpus, use `build/benchmark/typechec Maintainers can also request the hosted regression benchmark on a pull request by commenting `/benchmark`. The hosted workflow compares the pull request's synthetic merge commit with its exact base commit and reuses a validated commit-keyed base result when available. Results are attached to -the workflow and added to the existing pull-request comment without changing the pull-request branch -or retriggering code tests. See [the benchmark README](build/benchmark/README.md) for the developer -and maintainer workflows, cache behavior, prerequisites, and methodology. +the workflow and added to the existing pull-request comment. When it measures a previously uncached +base commit, it also commits the new baseline files to a same-repository pull-request branch. See +[the benchmark README](build/benchmark/README.md) for the developer and maintainer workflows, cache +behavior, prerequisites, and methodology. diff --git a/build/benchmark/README.md b/build/benchmark/README.md index 4b58cc46d205..6c2d0484dba5 100644 --- a/build/benchmark/README.md +++ b/build/benchmark/README.md @@ -146,14 +146,21 @@ By default, results are written to the ignored `build/benchmark/results/` direct writes a UTC-dated file such as `benchmark_2026-07-28.json` and updates `latest.json`. With `--os-name macos`, the names are `benchmark_2026-07-28_macos.json` and `latest-macos.json`. -To compare two locally generated results, use the same OS label, run count, warmup count, memory -limit, Python version, and package commits for both runs: +Checked-in reference runs live in `build/benchmark/baselines/`. To compare two locally generated +results, use the same OS label, run count, warmup count, memory limit, Python version, and package +commits for both runs: ```console python build/benchmark/compare_benchmarks.py \ path/to/base-result.json path/to/candidate-result.json ``` +`build/benchmark/benchmark_history.ipynb` loads the dated checked-in baselines, graphs package timing +and peak-memory trends, compares the latest run with an earlier commit, and can export a static +dashboard under `docs/benchmark-results/`. Install its optional dependencies with +`python -m pip install -r build/benchmark/requirements-notebook.txt`. Exports and result-file +enrichment are disabled by default and can be enabled independently in the notebook configuration. + The comparator reports per-package timing and memory deltas and exits nonzero when a previously successful result is missing, the environment contract differs, or a result exceeds the default 20% regression threshold. Package commits, check paths, and excluded directory names must also match. @@ -185,24 +192,29 @@ absolute variance guard of 1 second for time or 100 MB for peak memory. If a pul benchmark script, package corpus, package pins, checked paths, exclusions, or measurement environment, both revisions must still complete successfully, but the report marks them as not comparable and does not apply a performance regression gate. Once that change is merged, its commit becomes the base for -later pull requests and is cached normally; no checked-in baseline update is required. +later pull requests and is cached normally. The base result, candidate result, and comparison are attached to the workflow run and rendered in the -Actions job summary and the existing benchmark pull-request comment. Publishing results does not -commit files, push a branch, or modify the pull request's head SHA, so it does not retrigger code tests. -The report comment also cannot dispatch another benchmark: the trigger accepts only a newly created -comment whose entire trimmed body is `/benchmark`, whereas later reports update the marker comment. +Actions job summary and existing benchmark pull-request comment. When the exact base result was not +already cached, the workflow commits it as both the dated baseline and `latest-linux-x64.json` on a +same-repository pull-request branch. The write-scoped job uses the GitHub API without checking out or +executing pull-request code, and skips the update if the pull-request head has changed. Fork pull +requests retain the result as an artifact because the repository token cannot update their branches. +The baseline commit changes the pull-request head and can retrigger push-based checks. The report +comment cannot dispatch another benchmark: the trigger accepts only a newly created comment whose +entire trimmed body is `/benchmark`, whereas later reports update the marker comment. The weekly workflow runs Pyright, Pyrefly, ty, mypy, and Zuban in independent hosted-runner jobs. Each checker performs three measured runs after one warmup over the same pinned corpus. The aggregate job stores each raw JSON result with a self-contained `index.html` comparison for 90 days; its Actions job summary links directly to the downloadable report artifact. -The top-level JSON records the timestamp, platform, checker versions, run settings, aggregate -statistics, per-package results, configured memory limit, and an `upstream_source` object containing -the original repository, exact commit, and source-file URL. Each package result records the cloned -repository commit. Each successful checker result contains the measured wall times and peak-memory -values, plus min, max, mean, median, and standard deviation. +The top-level JSON records the benchmark timestamp, benchmarked Pyright commit SHA, commit subject, +commit timestamp, platform, checker versions, run settings, aggregate statistics, per-package +results, configured memory limit, and an `upstream_source` object containing the original repository, +exact commit, and source-file URL. Each package result records the cloned repository commit. Each +successful checker result contains the measured wall times and peak-memory values, plus min, max, +mean, median, and standard deviation. Aggregate data contains package counts and mean, p50, p90, p95, maximum, and total timing or memory statistics. diff --git a/build/benchmark/baselines/benchmark_2026-08-20_linux-x64.json b/build/benchmark/baselines/benchmark_2026-08-20_linux-x64.json new file mode 100644 index 000000000000..e08a79c7f686 --- /dev/null +++ b/build/benchmark/baselines/benchmark_2026-08-20_linux-x64.json @@ -0,0 +1,408 @@ +{ + "timestamp": "2026-08-20T02:07:10.843078+00:00", + "date": "2026-08-20", + "source_revision": "075161cd9160dbefff81e8ec4b28a3be4786bfcb", + "source_commit_subject": "Increase benchmark heap headroom", + "source_commit_timestamp": "2026-08-19T18:10:07-07:00", + "platform": "linux", + "platform_details": "Linux-6.17.0-1022-azure-x86_64-with-glibc2.39", + "architecture": "x86_64", + "runner_class": "github-ubuntu-latest", + "runner_image": "ubuntu24", + "python_version": "3.14.6", + "cpu_count": 4, + "upstream_source": { + "repository_url": "https://github.com/lolpack/type_coverage_py", + "commit": "85667d6f090ce9648d88cd7a9777b492f3b95f1c", + "source_file_url": "https://github.com/lolpack/type_coverage_py/blob/85667d6f090ce9648d88cd7a9777b492f3b95f1c/typecheck_benchmark/daily_runner.py" + }, + "memory_measurement": "/proc//status", + "memory_limit_mb": 8192, + "node_options": "--max-old-space-size=6656", + "dependency_isolation": "pip-target-per-package", + "type_checkers": [ + "pyright" + ], + "type_checker_versions": { + "pyright": "1.1.413" + }, + "package_count": 9, + "runs_per_package": 1, + "warmup_runs": 0, + "uncounted_validation_runs_per_checker": 1, + "timeout_s": 1800, + "aggregate": { + "pyright": { + "packages_tested": 9, + "packages_failed": 0, + "avg_execution_time_s": 168.745, + "p50_execution_time_s": 43.435, + "p90_execution_time_s": 324.505, + "p95_execution_time_s": 656.832, + "max_execution_time_s": 989.16, + "total_execution_time_s": 1518.701, + "avg_peak_memory_mb": 2805.3, + "p50_peak_memory_mb": 1850.0, + "p90_peak_memory_mb": 5342.4, + "p95_peak_memory_mb": 5767.3, + "max_peak_memory_mb": 6192.1 + } + }, + "results": [ + { + "package_name": "ansible", + "github_url": "https://github.com/ansible/ansible", + "error": null, + "metrics": { + "pyright": { + "ok": true, + "execution_time_s": 21.661, + "peak_memory_mb": 1110.7, + "oom_killed": false, + "runs": 1, + "execution_times_s": [ + 21.661 + ], + "peak_memories_mb": [ + 1110.7 + ], + "execution_time_stats": { + "min": 21.661, + "max": 21.661, + "mean": 21.661, + "median": 21.661, + "stddev": 0.0 + }, + "peak_memory_stats": { + "min": 1110.7, + "max": 1110.7, + "mean": 1110.7, + "median": 1110.7, + "stddev": 0.0 + } + } + }, + "commit": "e8264c418ad2e87f92fa48f75cacfaa451cb38e4", + "check_paths": [ + "lib/ansible" + ], + "exclude_directories": [] + }, + { + "package_name": "click", + "github_url": "https://github.com/pallets/click", + "error": null, + "metrics": { + "pyright": { + "ok": true, + "execution_time_s": 2.908, + "peak_memory_mb": 361.0, + "oom_killed": false, + "runs": 1, + "execution_times_s": [ + 2.908 + ], + "peak_memories_mb": [ + 361.0 + ], + "execution_time_stats": { + "min": 2.908, + "max": 2.908, + "mean": 2.908, + "median": 2.908, + "stddev": 0.0 + }, + "peak_memory_stats": { + "min": 361.0, + "max": 361.0, + "mean": 361.0, + "median": 361.0, + "stddev": 0.0 + } + } + }, + "commit": "00e592cea702e0b2caa0dee42489fdb1c22cd845", + "check_paths": [ + "src/click" + ], + "exclude_directories": [] + }, + { + "package_name": "homeassistant", + "github_url": "https://github.com/home-assistant/core", + "error": null, + "metrics": { + "pyright": { + "ok": true, + "execution_time_s": 158.341, + "peak_memory_mb": 6192.1, + "oom_killed": false, + "runs": 1, + "execution_times_s": [ + 158.341 + ], + "peak_memories_mb": [ + 6192.1 + ], + "execution_time_stats": { + "min": 158.341, + "max": 158.341, + "mean": 158.341, + "median": 158.341, + "stddev": 0.0 + }, + "peak_memory_stats": { + "min": 6192.1, + "max": 6192.1, + "mean": 6192.1, + "median": 6192.1, + "stddev": 0.0 + } + } + }, + "commit": "f002c54d12077d6b906fa624f1c4bba11a71897d", + "check_paths": [ + "homeassistant" + ], + "exclude_directories": [] + }, + { + "package_name": "numpy", + "github_url": "https://github.com/numpy/numpy", + "error": null, + "metrics": { + "pyright": { + "ok": true, + "execution_time_s": 43.435, + "peak_memory_mb": 1850.0, + "oom_killed": false, + "runs": 1, + "execution_times_s": [ + 43.435 + ], + "peak_memories_mb": [ + 1850.0 + ], + "execution_time_stats": { + "min": 43.435, + "max": 43.435, + "mean": 43.435, + "median": 43.435, + "stddev": 0.0 + }, + "peak_memory_stats": { + "min": 1850.0, + "max": 1850.0, + "mean": 1850.0, + "median": 1850.0, + "stddev": 0.0 + } + } + }, + "commit": "db6ccacf630dd90f7c498e0d924bd47b66c83746", + "check_paths": [ + "numpy" + ], + "exclude_directories": [ + "tests" + ] + }, + { + "package_name": "pandas", + "github_url": "https://github.com/pandas-dev/pandas", + "error": null, + "metrics": { + "pyright": { + "ok": true, + "execution_time_s": 989.16, + "peak_memory_mb": 4434.0, + "oom_killed": false, + "runs": 1, + "execution_times_s": [ + 989.16 + ], + "peak_memories_mb": [ + 4434.0 + ], + "execution_time_stats": { + "min": 989.16, + "max": 989.16, + "mean": 989.16, + "median": 989.16, + "stddev": 0.0 + }, + "peak_memory_stats": { + "min": 4434.0, + "max": 4434.0, + "mean": 4434.0, + "median": 4434.0, + "stddev": 0.0 + } + } + }, + "commit": "982854070758cd2015fc9e64395684546b1c5444", + "check_paths": [ + "pandas" + ], + "exclude_directories": [] + }, + { + "package_name": "pytest", + "github_url": "https://github.com/pytest-dev/pytest", + "error": null, + "metrics": { + "pyright": { + "ok": true, + "execution_time_s": 12.574, + "peak_memory_mb": 930.9, + "oom_killed": false, + "runs": 1, + "execution_times_s": [ + 12.574 + ], + "peak_memories_mb": [ + 930.9 + ], + "execution_time_stats": { + "min": 12.574, + "max": 12.574, + "mean": 12.574, + "median": 12.574, + "stddev": 0.0 + }, + "peak_memory_stats": { + "min": 930.9, + "max": 930.9, + "mean": 930.9, + "median": 930.9, + "stddev": 0.0 + } + } + }, + "commit": "56b196e921acec0259d84622a570fde6032e15b5", + "check_paths": [ + "src", + "testing" + ], + "exclude_directories": [] + }, + { + "package_name": "requests", + "github_url": "https://github.com/psf/requests", + "error": null, + "metrics": { + "pyright": { + "ok": true, + "execution_time_s": 2.299, + "peak_memory_mb": 347.2, + "oom_killed": false, + "runs": 1, + "execution_times_s": [ + 2.299 + ], + "peak_memories_mb": [ + 347.2 + ], + "execution_time_stats": { + "min": 2.299, + "max": 2.299, + "mean": 2.299, + "median": 2.299, + "stddev": 0.0 + }, + "peak_memory_stats": { + "min": 347.2, + "max": 347.2, + "mean": 347.2, + "median": 347.2, + "stddev": 0.0 + } + } + }, + "commit": "414f0513c33883adf6f2b46901d4f0b38a455851", + "check_paths": [ + "src/requests" + ], + "exclude_directories": [] + }, + { + "package_name": "torch", + "github_url": "https://github.com/pytorch/pytorch", + "error": null, + "metrics": { + "pyright": { + "ok": true, + "execution_time_s": 153.779, + "peak_memory_mb": 4891.9, + "oom_killed": false, + "runs": 1, + "execution_times_s": [ + 153.779 + ], + "peak_memories_mb": [ + 4891.9 + ], + "execution_time_stats": { + "min": 153.779, + "max": 153.779, + "mean": 153.779, + "median": 153.779, + "stddev": 0.0 + }, + "peak_memory_stats": { + "min": 4891.9, + "max": 4891.9, + "mean": 4891.9, + "median": 4891.9, + "stddev": 0.0 + } + } + }, + "commit": "3ee692b380206d788625e0a4474d758f93571ee9", + "check_paths": [ + "torch" + ], + "exclude_directories": [] + }, + { + "package_name": "transformers", + "github_url": "https://github.com/huggingface/transformers", + "error": null, + "metrics": { + "pyright": { + "ok": true, + "execution_time_s": 134.544, + "peak_memory_mb": 5130.0, + "oom_killed": false, + "runs": 1, + "execution_times_s": [ + 134.544 + ], + "peak_memories_mb": [ + 5130.0 + ], + "execution_time_stats": { + "min": 134.544, + "max": 134.544, + "mean": 134.544, + "median": 134.544, + "stddev": 0.0 + }, + "peak_memory_stats": { + "min": 5130.0, + "max": 5130.0, + "mean": 5130.0, + "median": 5130.0, + "stddev": 0.0 + } + } + }, + "commit": "2ef79f87a02111f8b49a72fb7d0c86b5b0bf10b7", + "check_paths": [ + "src/transformers" + ], + "exclude_directories": [] + } + ], + "os": "linux-x64" +} diff --git a/build/benchmark/baselines/latest-linux-x64.json b/build/benchmark/baselines/latest-linux-x64.json new file mode 100644 index 000000000000..e08a79c7f686 --- /dev/null +++ b/build/benchmark/baselines/latest-linux-x64.json @@ -0,0 +1,408 @@ +{ + "timestamp": "2026-08-20T02:07:10.843078+00:00", + "date": "2026-08-20", + "source_revision": "075161cd9160dbefff81e8ec4b28a3be4786bfcb", + "source_commit_subject": "Increase benchmark heap headroom", + "source_commit_timestamp": "2026-08-19T18:10:07-07:00", + "platform": "linux", + "platform_details": "Linux-6.17.0-1022-azure-x86_64-with-glibc2.39", + "architecture": "x86_64", + "runner_class": "github-ubuntu-latest", + "runner_image": "ubuntu24", + "python_version": "3.14.6", + "cpu_count": 4, + "upstream_source": { + "repository_url": "https://github.com/lolpack/type_coverage_py", + "commit": "85667d6f090ce9648d88cd7a9777b492f3b95f1c", + "source_file_url": "https://github.com/lolpack/type_coverage_py/blob/85667d6f090ce9648d88cd7a9777b492f3b95f1c/typecheck_benchmark/daily_runner.py" + }, + "memory_measurement": "/proc//status", + "memory_limit_mb": 8192, + "node_options": "--max-old-space-size=6656", + "dependency_isolation": "pip-target-per-package", + "type_checkers": [ + "pyright" + ], + "type_checker_versions": { + "pyright": "1.1.413" + }, + "package_count": 9, + "runs_per_package": 1, + "warmup_runs": 0, + "uncounted_validation_runs_per_checker": 1, + "timeout_s": 1800, + "aggregate": { + "pyright": { + "packages_tested": 9, + "packages_failed": 0, + "avg_execution_time_s": 168.745, + "p50_execution_time_s": 43.435, + "p90_execution_time_s": 324.505, + "p95_execution_time_s": 656.832, + "max_execution_time_s": 989.16, + "total_execution_time_s": 1518.701, + "avg_peak_memory_mb": 2805.3, + "p50_peak_memory_mb": 1850.0, + "p90_peak_memory_mb": 5342.4, + "p95_peak_memory_mb": 5767.3, + "max_peak_memory_mb": 6192.1 + } + }, + "results": [ + { + "package_name": "ansible", + "github_url": "https://github.com/ansible/ansible", + "error": null, + "metrics": { + "pyright": { + "ok": true, + "execution_time_s": 21.661, + "peak_memory_mb": 1110.7, + "oom_killed": false, + "runs": 1, + "execution_times_s": [ + 21.661 + ], + "peak_memories_mb": [ + 1110.7 + ], + "execution_time_stats": { + "min": 21.661, + "max": 21.661, + "mean": 21.661, + "median": 21.661, + "stddev": 0.0 + }, + "peak_memory_stats": { + "min": 1110.7, + "max": 1110.7, + "mean": 1110.7, + "median": 1110.7, + "stddev": 0.0 + } + } + }, + "commit": "e8264c418ad2e87f92fa48f75cacfaa451cb38e4", + "check_paths": [ + "lib/ansible" + ], + "exclude_directories": [] + }, + { + "package_name": "click", + "github_url": "https://github.com/pallets/click", + "error": null, + "metrics": { + "pyright": { + "ok": true, + "execution_time_s": 2.908, + "peak_memory_mb": 361.0, + "oom_killed": false, + "runs": 1, + "execution_times_s": [ + 2.908 + ], + "peak_memories_mb": [ + 361.0 + ], + "execution_time_stats": { + "min": 2.908, + "max": 2.908, + "mean": 2.908, + "median": 2.908, + "stddev": 0.0 + }, + "peak_memory_stats": { + "min": 361.0, + "max": 361.0, + "mean": 361.0, + "median": 361.0, + "stddev": 0.0 + } + } + }, + "commit": "00e592cea702e0b2caa0dee42489fdb1c22cd845", + "check_paths": [ + "src/click" + ], + "exclude_directories": [] + }, + { + "package_name": "homeassistant", + "github_url": "https://github.com/home-assistant/core", + "error": null, + "metrics": { + "pyright": { + "ok": true, + "execution_time_s": 158.341, + "peak_memory_mb": 6192.1, + "oom_killed": false, + "runs": 1, + "execution_times_s": [ + 158.341 + ], + "peak_memories_mb": [ + 6192.1 + ], + "execution_time_stats": { + "min": 158.341, + "max": 158.341, + "mean": 158.341, + "median": 158.341, + "stddev": 0.0 + }, + "peak_memory_stats": { + "min": 6192.1, + "max": 6192.1, + "mean": 6192.1, + "median": 6192.1, + "stddev": 0.0 + } + } + }, + "commit": "f002c54d12077d6b906fa624f1c4bba11a71897d", + "check_paths": [ + "homeassistant" + ], + "exclude_directories": [] + }, + { + "package_name": "numpy", + "github_url": "https://github.com/numpy/numpy", + "error": null, + "metrics": { + "pyright": { + "ok": true, + "execution_time_s": 43.435, + "peak_memory_mb": 1850.0, + "oom_killed": false, + "runs": 1, + "execution_times_s": [ + 43.435 + ], + "peak_memories_mb": [ + 1850.0 + ], + "execution_time_stats": { + "min": 43.435, + "max": 43.435, + "mean": 43.435, + "median": 43.435, + "stddev": 0.0 + }, + "peak_memory_stats": { + "min": 1850.0, + "max": 1850.0, + "mean": 1850.0, + "median": 1850.0, + "stddev": 0.0 + } + } + }, + "commit": "db6ccacf630dd90f7c498e0d924bd47b66c83746", + "check_paths": [ + "numpy" + ], + "exclude_directories": [ + "tests" + ] + }, + { + "package_name": "pandas", + "github_url": "https://github.com/pandas-dev/pandas", + "error": null, + "metrics": { + "pyright": { + "ok": true, + "execution_time_s": 989.16, + "peak_memory_mb": 4434.0, + "oom_killed": false, + "runs": 1, + "execution_times_s": [ + 989.16 + ], + "peak_memories_mb": [ + 4434.0 + ], + "execution_time_stats": { + "min": 989.16, + "max": 989.16, + "mean": 989.16, + "median": 989.16, + "stddev": 0.0 + }, + "peak_memory_stats": { + "min": 4434.0, + "max": 4434.0, + "mean": 4434.0, + "median": 4434.0, + "stddev": 0.0 + } + } + }, + "commit": "982854070758cd2015fc9e64395684546b1c5444", + "check_paths": [ + "pandas" + ], + "exclude_directories": [] + }, + { + "package_name": "pytest", + "github_url": "https://github.com/pytest-dev/pytest", + "error": null, + "metrics": { + "pyright": { + "ok": true, + "execution_time_s": 12.574, + "peak_memory_mb": 930.9, + "oom_killed": false, + "runs": 1, + "execution_times_s": [ + 12.574 + ], + "peak_memories_mb": [ + 930.9 + ], + "execution_time_stats": { + "min": 12.574, + "max": 12.574, + "mean": 12.574, + "median": 12.574, + "stddev": 0.0 + }, + "peak_memory_stats": { + "min": 930.9, + "max": 930.9, + "mean": 930.9, + "median": 930.9, + "stddev": 0.0 + } + } + }, + "commit": "56b196e921acec0259d84622a570fde6032e15b5", + "check_paths": [ + "src", + "testing" + ], + "exclude_directories": [] + }, + { + "package_name": "requests", + "github_url": "https://github.com/psf/requests", + "error": null, + "metrics": { + "pyright": { + "ok": true, + "execution_time_s": 2.299, + "peak_memory_mb": 347.2, + "oom_killed": false, + "runs": 1, + "execution_times_s": [ + 2.299 + ], + "peak_memories_mb": [ + 347.2 + ], + "execution_time_stats": { + "min": 2.299, + "max": 2.299, + "mean": 2.299, + "median": 2.299, + "stddev": 0.0 + }, + "peak_memory_stats": { + "min": 347.2, + "max": 347.2, + "mean": 347.2, + "median": 347.2, + "stddev": 0.0 + } + } + }, + "commit": "414f0513c33883adf6f2b46901d4f0b38a455851", + "check_paths": [ + "src/requests" + ], + "exclude_directories": [] + }, + { + "package_name": "torch", + "github_url": "https://github.com/pytorch/pytorch", + "error": null, + "metrics": { + "pyright": { + "ok": true, + "execution_time_s": 153.779, + "peak_memory_mb": 4891.9, + "oom_killed": false, + "runs": 1, + "execution_times_s": [ + 153.779 + ], + "peak_memories_mb": [ + 4891.9 + ], + "execution_time_stats": { + "min": 153.779, + "max": 153.779, + "mean": 153.779, + "median": 153.779, + "stddev": 0.0 + }, + "peak_memory_stats": { + "min": 4891.9, + "max": 4891.9, + "mean": 4891.9, + "median": 4891.9, + "stddev": 0.0 + } + } + }, + "commit": "3ee692b380206d788625e0a4474d758f93571ee9", + "check_paths": [ + "torch" + ], + "exclude_directories": [] + }, + { + "package_name": "transformers", + "github_url": "https://github.com/huggingface/transformers", + "error": null, + "metrics": { + "pyright": { + "ok": true, + "execution_time_s": 134.544, + "peak_memory_mb": 5130.0, + "oom_killed": false, + "runs": 1, + "execution_times_s": [ + 134.544 + ], + "peak_memories_mb": [ + 5130.0 + ], + "execution_time_stats": { + "min": 134.544, + "max": 134.544, + "mean": 134.544, + "median": 134.544, + "stddev": 0.0 + }, + "peak_memory_stats": { + "min": 5130.0, + "max": 5130.0, + "mean": 5130.0, + "median": 5130.0, + "stddev": 0.0 + } + } + }, + "commit": "2ef79f87a02111f8b49a72fb7d0c86b5b0bf10b7", + "check_paths": [ + "src/transformers" + ], + "exclude_directories": [] + } + ], + "os": "linux-x64" +} diff --git a/build/benchmark/benchmark_history.ipynb b/build/benchmark/benchmark_history.ipynb new file mode 100644 index 000000000000..431c3564dfbf --- /dev/null +++ b/build/benchmark/benchmark_history.ipynb @@ -0,0 +1,506 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "f974c0d2", + "metadata": {}, + "source": [ + "# Pyright benchmark history\n", + "\n", + "Explore checked-in Linux x64 benchmark results across Pyright commits, compare the latest run with a baseline, and optionally export charts and a static GitHub Pages dashboard.\n", + "\n", + "The workflow adds commit metadata when collecting hosted results. Notebook enrichment is dry-run by default; set `WRITE_ENRICHED_RESULTS = True` only for result files whose source commit is the current checkout." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "a7238571", + "metadata": {}, + "outputs": [], + "source": [ + "# 1. Import analysis and visualization libraries\n", + "from __future__ import annotations\n", + "\n", + "import html\n", + "import json\n", + "import os\n", + "import subprocess\n", + "from datetime import datetime\n", + "from pathlib import Path\n", + "from typing import Any\n", + "\n", + "import matplotlib.dates as mdates\n", + "import matplotlib.pyplot as plt\n", + "import pandas as pd\n", + "import seaborn as sns\n", + "from IPython.display import HTML, Markdown, display\n", + "\n", + "sns.set_theme(style=\"whitegrid\", context=\"notebook\")\n", + "plt.rcParams[\"figure.figsize\"] = (12, 6)\n", + "plt.rcParams[\"figure.dpi\"] = 120" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "eeedeada", + "metadata": {}, + "outputs": [], + "source": [ + "# 2. Define benchmark file and metadata schemas\n", + "REQUIRED_METADATA = {\n", + " \"timestamp\": str,\n", + " \"source_revision\": str,\n", + " \"source_commit_subject\": str,\n", + " \"source_commit_timestamp\": str,\n", + " \"results\": list,\n", + "}\n", + "\n", + "METRIC_SPECS = {\n", + " \"execution_time_s\": {\"label\": \"Execution time\", \"unit\": \"seconds\", \"lower_is_better\": True},\n", + " \"peak_memory_mb\": {\"label\": \"Peak memory\", \"unit\": \"MiB\", \"lower_is_better\": True},\n", + "}\n", + "\n", + "\n", + "def find_repo_root(start: Path) -> Path:\n", + " for candidate in (start.resolve(), *start.resolve().parents):\n", + " if (candidate / \"build\" / \"benchmark\" / \"baselines\").is_dir():\n", + " return candidate\n", + " raise FileNotFoundError(\"Could not find build/benchmark/baselines from the current directory\")\n", + "\n", + "\n", + "REPO_ROOT = find_repo_root(Path.cwd())\n", + "BASELINE_DIR = REPO_ROOT / \"build\" / \"benchmark\" / \"baselines\"\n", + "EXPORT_DIR = Path(\n", + " os.environ.get(\"PYRIGHT_BENCHMARK_EXPORT_DIR\", REPO_ROOT / \"docs\" / \"benchmark-results\")\n", + ").resolve()\n", + "WRITE_ENRICHED_RESULTS = os.environ.get(\"PYRIGHT_BENCHMARK_ENRICH\", \"\") == \"1\"\n", + "EXPORT_ASSETS = os.environ.get(\"PYRIGHT_BENCHMARK_EXPORT\", \"\") == \"1\"\n", + "\n", + "print(f\"Repository: {REPO_ROOT}\")\n", + "print(f\"Baseline directory: {BASELINE_DIR}\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c5efd90d", + "metadata": {}, + "outputs": [], + "source": [ + "# 3. Capture Git commit name and timestamp\n", + "def git_value(*args: str) -> str:\n", + " try:\n", + " return subprocess.run(\n", + " [\"git\", \"-C\", str(REPO_ROOT), *args],\n", + " check=True,\n", + " capture_output=True,\n", + " text=True,\n", + " ).stdout.strip()\n", + " except (FileNotFoundError, subprocess.CalledProcessError) as error:\n", + " raise RuntimeError(\"Git history is unavailable; run this notebook from a Git checkout\") from error\n", + "\n", + "\n", + "def current_commit_metadata() -> dict[str, str]:\n", + " revision = git_value(\"rev-parse\", \"HEAD\")\n", + " return {\n", + " \"source_revision\": revision,\n", + " \"source_commit_short\": git_value(\"rev-parse\", \"--short=12\", revision),\n", + " \"source_commit_subject\": git_value(\"show\", \"-s\", \"--format=%s\", revision),\n", + " \"source_commit_timestamp\": git_value(\"show\", \"-s\", \"--format=%cI\", revision),\n", + " }\n", + "\n", + "\n", + "current_commit = current_commit_metadata()\n", + "display(current_commit)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "ea4a11e2", + "metadata": {}, + "outputs": [], + "source": [ + "# 4. Attach commit metadata to result files\n", + "def enrich_result_file(\n", + " path: Path,\n", + " commit: dict[str, str],\n", + " *,\n", + " write: bool = False,\n", + ") -> dict[str, Any]:\n", + " data = json.loads(path.read_text(encoding=\"utf-8\"))\n", + " if not isinstance(data, dict):\n", + " raise ValueError(f\"{path} must contain a JSON object\")\n", + "\n", + " metadata = {key: value for key, value in commit.items() if key != \"source_commit_short\"}\n", + " conflicts = {\n", + " key: data[key]\n", + " for key, value in metadata.items()\n", + " if key in data and data[key] != value\n", + " }\n", + " if conflicts:\n", + " raise ValueError(f\"Refusing to replace existing commit metadata in {path}: {conflicts}\")\n", + " enriched = {**metadata, **data}\n", + " if \"timestamp\" not in enriched:\n", + " enriched[\"timestamp\"] = datetime.now().astimezone().isoformat()\n", + " if write:\n", + " path.write_text(json.dumps(enriched, indent=2) + \"\\n\", encoding=\"utf-8\")\n", + " return enriched\n", + "\n", + "\n", + "result_files = sorted(BASELINE_DIR.glob(\"benchmark_*_linux-x64.json\"))\n", + "RESULT_FILES_TO_ENRICH: list[Path] = []\n", + "if WRITE_ENRICHED_RESULTS:\n", + " if not RESULT_FILES_TO_ENRICH:\n", + " raise RuntimeError(\"Set RESULT_FILES_TO_ENRICH explicitly before enabling enrichment\")\n", + " for result_file in RESULT_FILES_TO_ENRICH:\n", + " enrich_result_file(result_file, current_commit, write=True)\n", + " print(f\"Enriched {len(RESULT_FILES_TO_ENRICH)} result file(s)\")\n", + "else:\n", + " print(f\"Dry run: {len(result_files)} result file(s) discovered; no files changed\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "10a96e48", + "metadata": {}, + "outputs": [], + "source": [ + "# 5. Load and validate historical benchmark results\n", + "def parse_iso8601(value: str) -> datetime:\n", + " return datetime.fromisoformat(value.replace(\"Z\", \"+00:00\"))\n", + "\n", + "\n", + "def validate_result(path: Path, data: Any) -> list[str]:\n", + " problems: list[str] = []\n", + " if not isinstance(data, dict):\n", + " return [\"root value is not an object\"]\n", + " for field, expected_type in REQUIRED_METADATA.items():\n", + " if not isinstance(data.get(field), expected_type):\n", + " problems.append(f\"{field} must be {expected_type.__name__}\")\n", + " for field in (\"timestamp\", \"source_commit_timestamp\"):\n", + " try:\n", + " parse_iso8601(str(data.get(field, \"\")))\n", + " except ValueError:\n", + " problems.append(f\"{field} must be an ISO 8601 timestamp\")\n", + " if not isinstance(data.get(\"source_revision\"), str) or len(data.get(\"source_revision\", \"\")) != 40:\n", + " problems.append(\"source_revision must be a 40-character SHA\")\n", + " return problems\n", + "\n", + "\n", + "def load_history(paths: list[Path]) -> tuple[list[dict[str, Any]], list[dict[str, str]]]:\n", + " records: list[dict[str, Any]] = []\n", + " failures: list[dict[str, str]] = []\n", + " for path in paths:\n", + " try:\n", + " data = json.loads(path.read_text(encoding=\"utf-8\"))\n", + " problems = validate_result(path, data)\n", + " if problems:\n", + " failures.append({\"file\": path.name, \"error\": \"; \".join(problems)})\n", + " continue\n", + " for package in data[\"results\"]:\n", + " metrics = package.get(\"metrics\", {}).get(\"pyright\", {})\n", + " if not metrics.get(\"ok\"):\n", + " continue\n", + " for metric, spec in METRIC_SPECS.items():\n", + " value = metrics.get(metric)\n", + " if isinstance(value, (int, float)):\n", + " records.append(\n", + " {\n", + " \"file\": path.name,\n", + " \"benchmark\": package[\"package_name\"],\n", + " \"metric\": metric,\n", + " \"value\": value,\n", + " \"unit\": spec[\"unit\"],\n", + " \"lower_is_better\": spec[\"lower_is_better\"],\n", + " \"commit_sha\": data[\"source_revision\"],\n", + " \"commit_title\": data[\"source_commit_subject\"],\n", + " \"commit_timestamp\": data[\"source_commit_timestamp\"],\n", + " \"collection_timestamp\": data[\"timestamp\"],\n", + " }\n", + " )\n", + " except (OSError, ValueError, KeyError, TypeError) as error:\n", + " failures.append({\"file\": path.name, \"error\": str(error)})\n", + " return records, failures\n", + "\n", + "\n", + "records, malformed_files = load_history(result_files)\n", + "print(f\"Loaded {len(records)} measurements from {len(result_files)} result file(s)\")\n", + "if malformed_files:\n", + " display(pd.DataFrame(malformed_files))" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "fcf00161", + "metadata": {}, + "outputs": [], + "source": [ + "# 6. Normalize results into a DataFrame\n", + "history = pd.DataFrame.from_records(records)\n", + "if history.empty:\n", + " raise RuntimeError(\"No valid benchmark measurements were found\")\n", + "\n", + "for column in (\"commit_timestamp\", \"collection_timestamp\"):\n", + " history[column] = pd.to_datetime(history[column], utc=True, errors=\"raise\")\n", + "history[\"value\"] = pd.to_numeric(history[\"value\"], errors=\"raise\")\n", + "history[\"short_sha\"] = history[\"commit_sha\"].str[:12]\n", + "history = (\n", + " history.sort_values([\"collection_timestamp\", \"benchmark\", \"metric\"])\n", + " .drop_duplicates([\"commit_sha\", \"benchmark\", \"metric\"], keep=\"last\")\n", + " .reset_index(drop=True)\n", + ")\n", + "\n", + "run_summary = (\n", + " history.groupby(\n", + " [\"collection_timestamp\", \"commit_sha\", \"short_sha\", \"commit_title\", \"commit_timestamp\"],\n", + " as_index=False,\n", + " )\n", + " .agg(\n", + " package_count=(\"benchmark\", \"nunique\"),\n", + " total_execution_time_s=(\"value\", lambda values: history.loc[values.index].query(\"metric == 'execution_time_s'\")[\"value\"].sum()),\n", + " max_peak_memory_mb=(\"value\", lambda values: history.loc[values.index].query(\"metric == 'peak_memory_mb'\")[\"value\"].max()),\n", + " )\n", + ")\n", + "display(run_summary)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "19e13c7a", + "metadata": {}, + "outputs": [], + "source": [ + "# 7. Graph performance across commits\n", + "SELECTED_PACKAGES = sorted(history[\"benchmark\"].unique())\n", + "figures: dict[str, plt.Figure] = {}\n", + "\n", + "for metric, spec in METRIC_SPECS.items():\n", + " metric_data = history[\n", + " (history[\"metric\"] == metric) & history[\"benchmark\"].isin(SELECTED_PACKAGES)\n", + " ]\n", + " figure, axis = plt.subplots()\n", + " sns.lineplot(\n", + " data=metric_data,\n", + " x=\"collection_timestamp\",\n", + " y=\"value\",\n", + " hue=\"benchmark\",\n", + " marker=\"o\",\n", + " errorbar=None,\n", + " palette=\"colorblind\",\n", + " ax=axis,\n", + " )\n", + " for row in metric_data.itertuples():\n", + " axis.annotate(\n", + " row.short_sha,\n", + " (row.collection_timestamp, row.value),\n", + " xytext=(4, 5),\n", + " textcoords=\"offset points\",\n", + " fontsize=7,\n", + " alpha=0.8,\n", + " )\n", + " axis.set_title(f\"Pyright {spec['label'].lower()} across commits\")\n", + " axis.set_xlabel(\"Benchmark collection time (UTC)\")\n", + " axis.set_ylabel(f\"{spec['label']} ({spec['unit']})\")\n", + " axis.xaxis.set_major_formatter(mdates.DateFormatter(\"%Y-%m-%d\", tz=history[\"collection_timestamp\"].dt.tz))\n", + " axis.legend(title=\"Package\", bbox_to_anchor=(1.02, 1), loc=\"upper left\")\n", + " figure.autofmt_xdate()\n", + " figure.tight_layout()\n", + " figures[metric] = figure\n", + " plt.show()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "d2b310c2", + "metadata": {}, + "outputs": [], + "source": [ + "# 8. Compare the latest commit with the baseline\n", + "BASELINE_REVISION: str | None = None\n", + "\n", + "\n", + "def compare_latest(data: pd.DataFrame, baseline_revision: str | None = None) -> pd.DataFrame:\n", + " rows: list[dict[str, Any]] = []\n", + " for (benchmark, metric), group in data.groupby([\"benchmark\", \"metric\"], sort=True):\n", + " ordered = group.sort_values(\"collection_timestamp\")\n", + " latest = ordered.iloc[-1]\n", + " if baseline_revision:\n", + " matches = ordered[ordered[\"commit_sha\"].str.startswith(baseline_revision)]\n", + " if matches.empty:\n", + " continue\n", + " baseline = matches.iloc[-1]\n", + " else:\n", + " baseline = ordered.iloc[0]\n", + " absolute_change = latest[\"value\"] - baseline[\"value\"]\n", + " percent_change = absolute_change / baseline[\"value\"] * 100 if baseline[\"value\"] else float(\"nan\")\n", + " lower_is_better = bool(latest[\"lower_is_better\"])\n", + " if abs(percent_change) < 1:\n", + " assessment = \"Stable\"\n", + " elif (percent_change < 0) == lower_is_better:\n", + " assessment = \"Improvement\"\n", + " else:\n", + " assessment = \"Regression\"\n", + " rows.append(\n", + " {\n", + " \"benchmark\": benchmark,\n", + " \"metric\": metric,\n", + " \"unit\": latest[\"unit\"],\n", + " \"baseline_sha\": baseline[\"short_sha\"],\n", + " \"latest_sha\": latest[\"short_sha\"],\n", + " \"baseline\": baseline[\"value\"],\n", + " \"latest\": latest[\"value\"],\n", + " \"absolute_change\": absolute_change,\n", + " \"percent_change\": percent_change,\n", + " \"assessment\": assessment,\n", + " }\n", + " )\n", + " return pd.DataFrame(rows)\n", + "\n", + "\n", + "comparison = compare_latest(history, BASELINE_REVISION)\n", + "display(\n", + " comparison.style.format(\n", + " {\"baseline\": \"{:.2f}\", \"latest\": \"{:.2f}\", \"absolute_change\": \"{:+.2f}\", \"percent_change\": \"{:+.1f}%\"}\n", + " ).map(\n", + " lambda value: \"color: #087f5b; font-weight: bold\" if value == \"Improvement\" else \"color: #c92a2a; font-weight: bold\" if value == \"Regression\" else \"\",\n", + " subset=[\"assessment\"],\n", + " )\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "aec9b3e3", + "metadata": {}, + "outputs": [], + "source": [ + "# 9. Export charts and summary data\n", + "def export_assets(\n", + " output_dir: Path,\n", + " data: pd.DataFrame,\n", + " summary: pd.DataFrame,\n", + " charts: dict[str, plt.Figure],\n", + ") -> dict[str, Path]:\n", + " output_dir.mkdir(parents=True, exist_ok=True)\n", + " outputs = {\n", + " \"history_csv\": output_dir / \"history.csv\",\n", + " \"comparison_json\": output_dir / \"comparison.json\",\n", + " }\n", + " data.to_csv(outputs[\"history_csv\"], index=False)\n", + " outputs[\"comparison_json\"].write_text(\n", + " json.dumps(summary.to_dict(orient=\"records\"), indent=2, default=str) + \"\\n\",\n", + " encoding=\"utf-8\",\n", + " )\n", + " for metric, figure in charts.items():\n", + " chart_path = output_dir / f\"{metric}.svg\"\n", + " figure.savefig(chart_path, format=\"svg\", bbox_inches=\"tight\")\n", + " outputs[metric] = chart_path\n", + " return outputs\n", + "\n", + "\n", + "exported: dict[str, Path] = {}\n", + "if EXPORT_ASSETS:\n", + " exported = export_assets(EXPORT_DIR, history, comparison, figures)\n", + " display(exported)\n", + "else:\n", + " print(\"Exports disabled. Set EXPORT_ASSETS = True in Section 2 to write dashboard assets.\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "47b679bd", + "metadata": {}, + "outputs": [], + "source": [ + "# 10. Generate a GitHub Pages dashboard\n", + "def dashboard_html(run_data: pd.DataFrame, summary: pd.DataFrame) -> str:\n", + " latest = run_data.sort_values(\"collection_timestamp\").iloc[-1]\n", + " status_class = {\"Improvement\": \"good\", \"Regression\": \"bad\", \"Stable\": \"stable\"}\n", + " rows = \"\".join(\n", + " \"\"\n", + " f\"{html.escape(str(row.benchmark))}\"\n", + " f\"{html.escape(METRIC_SPECS[str(row.metric)]['label'])}\"\n", + " f\"{float(row.baseline):.2f} {html.escape(str(row.unit))}\"\n", + " f\"{float(row.latest):.2f} {html.escape(str(row.unit))}\"\n", + " f\"{float(row.percent_change):+.1f}%\"\n", + " f\"{html.escape(str(row.assessment))}\"\n", + " \"\"\n", + " for row in summary.itertuples()\n", + " )\n", + " charts = \"\".join(\n", + " f\"

{html.escape(spec['label'])}

{html.escape(spec[
\"\n", + " for metric, spec in METRIC_SPECS.items()\n", + " )\n", + " return f\"\"\"\n", + "\n", + "Pyright benchmark history

Pyright benchmark history

\n", + "

Latest: {html.escape(str(latest['commit_sha']))} {html.escape(str(latest['commit_title']))}

\n", + "

Collected {html.escape(str(latest['collection_timestamp']))}

{charts}\n", + "

Latest compared with baseline

{rows}
PackageMetricBaselineLatestChangeAssessment
\n", + "
\"\"\"\n", + "\n", + "\n", + "dashboard = dashboard_html(history, comparison)\n", + "if EXPORT_ASSETS:\n", + " dashboard_path = EXPORT_DIR / \"index.html\"\n", + " dashboard_path.write_text(dashboard, encoding=\"utf-8\")\n", + " print(f\"Dashboard: {dashboard_path}\")\n", + "else:\n", + " display(HTML(dashboard))" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "d1fb3018", + "metadata": {}, + "outputs": [], + "source": [ + "# 11. Automate notebook and page generation\n", + "NOTEBOOK_PATH = REPO_ROOT / \"build\" / \"benchmark\" / \"benchmark_history.ipynb\"\n", + "EXECUTED_NOTEBOOK = EXPORT_DIR / \"benchmark_history.executed.ipynb\"\n", + "NBCONVERT_COMMAND = [\n", + " \"jupyter\",\n", + " \"nbconvert\",\n", + " \"--to\",\n", + " \"notebook\",\n", + " \"--execute\",\n", + " str(NOTEBOOK_PATH),\n", + " \"--output\",\n", + " EXECUTED_NOTEBOOK.name,\n", + " \"--output-dir\",\n", + " str(EXPORT_DIR),\n", + " \"--ExecutePreprocessor.timeout=180\",\n", + "]\n", + "\n", + "assert not malformed_files, f\"Malformed benchmark files: {malformed_files}\"\n", + "assert not history.empty, \"No normalized benchmark data\"\n", + "assert set(METRIC_SPECS).issubset(set(history[\"metric\"])), \"Required metrics are missing\"\n", + "\n", + "print(\"For CI, execute with PYRIGHT_BENCHMARK_EXPORT=1:\")\n", + "print(subprocess.list2cmdline(NBCONVERT_COMMAND))\n", + "print(f\"Expected dashboard: {EXPORT_DIR / 'index.html'}\")\n", + "print(\"Fail CI if notebook execution fails, malformed_files is non-empty, or index.html is absent.\")" + ] + } + ], + "metadata": { + "language_info": { + "name": "python" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/build/benchmark/requirements-notebook.txt b/build/benchmark/requirements-notebook.txt new file mode 100644 index 000000000000..55a67dcb3d92 --- /dev/null +++ b/build/benchmark/requirements-notebook.txt @@ -0,0 +1,5 @@ +ipykernel>=6.30 +matplotlib>=3.10 +nbconvert>=7.16 +pandas>=2.3 +seaborn>=0.13 \ No newline at end of file diff --git a/build/benchmark/test_compare_benchmarks.py b/build/benchmark/test_compare_benchmarks.py index ace8a4a629d3..b24afceff455 100644 --- a/build/benchmark/test_compare_benchmarks.py +++ b/build/benchmark/test_compare_benchmarks.py @@ -520,8 +520,10 @@ def test_pr_workflow_uses_matching_base_and_candidate_profiles(self) -> None: self.assertEqual(timeout_matches, ["1800", "1800"]) self.assertIn("data['source_revision'] = os.environ['BASE_SHA']", workflow) self.assertIn("data['source_revision'] = os.environ['MERGE_SHA']", workflow) + self.assertEqual(workflow.count("data['source_commit_subject']"), 2) + self.assertEqual(workflow.count("data['source_commit_timestamp']"), 2) self.assertIn("data['benchmark_profile_hash'] = profile.hexdigest()", workflow) - self.assertNotIn("build/benchmark/baselines/", workflow) + self.assertIn("build/benchmark/baselines/latest-linux-x64.json", workflow) def test_workflows_use_current_pnpm_setup(self) -> None: for workflow_name in ( @@ -603,8 +605,9 @@ def test_pr_benchmark_requires_authorized_comment(self) -> None: self.assertIn("pull-requests: read", trigger_workflow) self.assertIn("createWorkflowDispatch", trigger_workflow) self.assertIn("workflow_id: 'typecheck_benchmark_pr.yml'", trigger_workflow) - self.assertIn("base_sha: pullRequest.data.base.sha", trigger_workflow) - self.assertIn("merge_sha: pullRequest.data.merge_commit_sha", trigger_workflow) + self.assertNotIn("head_sha:", trigger_workflow) + self.assertNotIn("base_sha:", trigger_workflow) + self.assertNotIn("merge_sha:", trigger_workflow) self.assertNotIn("actions/checkout", trigger_workflow) self.assertIn( "actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0", @@ -627,20 +630,27 @@ def test_pr_benchmark_requires_authorized_comment(self) -> None: self.assertNotIn("cache: 'pip'", benchmark_workflow) self.assertNotIn("cache: 'pnpm'", benchmark_workflow) self.assertIn("persist-credentials: false", benchmark_workflow) - self.assertIn("inputs.base_sha", benchmark_workflow) - self.assertIn("ref: ${{ inputs.merge_sha }}", benchmark_workflow) - self.assertIn("-merge-${{ inputs.merge_sha }}", benchmark_workflow) - self.assertIn("if: ${{ always() }}", benchmark_workflow) + self.assertNotIn("inputs.head_sha", benchmark_workflow) + self.assertNotIn("inputs.base_sha", benchmark_workflow) + self.assertNotIn("inputs.merge_sha", benchmark_workflow) + self.assertIn("github.rest.pulls.get", benchmark_workflow) + self.assertIn("ref: ${{ needs.metadata.outputs.base-sha }}", benchmark_workflow) + self.assertIn("ref: ${{ needs.metadata.outputs.merge-sha }}", benchmark_workflow) + self.assertIn("-merge-${{ needs.metadata.outputs.merge-sha }}", benchmark_workflow) + self.assertIn("if: ${{ always() && needs.metadata.result == 'success' }}", benchmark_workflow) self.assertIn("run_id: context.runId", benchmark_workflow) self.assertIn("pullRequest.data.base.sha !== expectedBaseSha", benchmark_workflow) self.assertIn( "pullRequest.data.merge_commit_sha !== expectedMergeSha", benchmark_workflow, ) + metadata_job = benchmark_workflow_data["jobs"]["metadata"] base_job = benchmark_workflow_data["jobs"]["base-benchmark"] candidate_job = benchmark_workflow_data["jobs"]["candidate-benchmark"] comparison_job = benchmark_workflow_data["jobs"]["comparison"] comment_job = benchmark_workflow_data["jobs"]["comment"] + persist_job = benchmark_workflow_data["jobs"]["persist-base-result"] + self.assertEqual(metadata_job["permissions"], {"pull-requests": "read"}) self.assertEqual(base_job["permissions"], {"contents": "read"}) self.assertEqual(candidate_job["permissions"], {"contents": "read"}) self.assertEqual(comparison_job["permissions"], {"contents": "read"}) @@ -652,7 +662,11 @@ def test_pr_benchmark_requires_authorized_comment(self) -> None: "pull-requests": "write", }, ) - self.assertEqual(comment_job["needs"], "comparison") + self.assertEqual( + persist_job["permissions"], + {"actions": "read", "contents": "write"}, + ) + self.assertEqual(comment_job["needs"], ["metadata", "comparison"]) self.assertEqual( [ job_name @@ -661,6 +675,14 @@ def test_pr_benchmark_requires_authorized_comment(self) -> None: ], ["comment"], ) + self.assertEqual( + [ + job_name + for job_name, job in benchmark_workflow_data["jobs"].items() + if job.get("permissions", {}).get("contents") == "write" + ], + ["persist-base-result"], + ) self.assertFalse( ( REPO_ROOT @@ -682,10 +704,11 @@ def test_pr_workflow_caches_only_the_base_result(self) -> None: self.assertIn("actions/cache/restore@0057852", base_job) self.assertIn("actions/cache/save@0057852", base_job) + self.assertIn("typecheck-benchmark-base-v2-", base_job) self.assertNotIn("restore-keys", base_job) self.assertNotIn("actions/cache/", candidate_job) - self.assertIn("ref: ${{ inputs.base_sha }}", workflow) - self.assertIn("ref: ${{ inputs.merge_sha }}", workflow) + self.assertIn("ref: ${{ needs.metadata.outputs.base-sha }}", workflow) + self.assertIn("ref: ${{ needs.metadata.outputs.merge-sha }}", workflow) self.assertIn("--baseline-revision", workflow) self.assertIn("--candidate-revision", workflow) self.assertIn("--allow-incompatible", workflow) @@ -694,7 +717,32 @@ def test_pr_workflow_caches_only_the_base_result(self) -> None: self.assertIn("comment.user?.login === 'github-actions[bot]'", workflow) self.assertIn("ref: ${{ github.sha }}", workflow) self.assertNotIn("git push", workflow) - self.assertNotIn("contents: write", workflow) + persist_job = workflow_data["jobs"]["persist-base-result"] + persist_job_text = json.dumps(persist_job) + self.assertNotIn("actions/checkout", persist_job_text) + self.assertIn("needs.base-benchmark.outputs.cached != 'true'", persist_job["if"]) + self.assertIn("needs.metadata.outputs.head-repository == github.repository", persist_job["if"]) + self.assertIn("!cancelled()", persist_job["if"]) + self.assertIn("github.rest.git.createCommit", workflow) + self.assertIn("github.rest.git.updateRef", workflow) + self.assertIn("currentRef.data.object.sha !== expectedHeadSha", workflow) + self.assertIn("build/benchmark/baselines/latest-linux-x64.json", workflow) + self.assertIn("build/benchmark/baselines/benchmark_${result.date}_linux-x64.json", workflow) + self.assertTrue( + (REPO_ROOT / "build" / "benchmark" / "baselines" / "latest-linux-x64.json").exists() + ) + baseline = json.loads( + ( + REPO_ROOT + / "build" + / "benchmark" + / "baselines" + / "latest-linux-x64.json" + ).read_text(encoding="utf-8") + ) + self.assertRegex(baseline["source_revision"], r"^[0-9a-f]{40}$") + self.assertTrue(baseline["source_commit_subject"]) + self.assertTrue(baseline["source_commit_timestamp"]) if __name__ == "__main__": From 9921a9052490f37220030824f1eaa5feafc56352 Mon Sep 17 00:00:00 2001 From: Bill Schnurr Date: Fri, 28 Aug 2026 11:53:36 -0700 Subject: [PATCH 3/5] Isolate benchmark cache from pull request code Run candidate benchmarks in an unprivileged pull_request workflow and move trusted base caching, reporting, and baseline persistence to workflow_run. Authorize measured runs through maintainer-triggered reruns and update tests and documentation. --- .github/workflows/typecheck_benchmark_pr.yml | 565 ++---------------- .../workflows/typecheck_benchmark_report.yml | 502 ++++++++++++++++ .../workflows/typecheck_benchmark_trigger.yml | 72 ++- build/benchmark/README.md | 38 +- build/benchmark/test_compare_benchmarks.py | 130 ++-- 5 files changed, 697 insertions(+), 610 deletions(-) create mode 100644 .github/workflows/typecheck_benchmark_report.yml diff --git a/.github/workflows/typecheck_benchmark_pr.yml b/.github/workflows/typecheck_benchmark_pr.yml index 85df25207167..61f00a3c4452 100644 --- a/.github/workflows/typecheck_benchmark_pr.yml +++ b/.github/workflows/typecheck_benchmark_pr.yml @@ -1,5 +1,5 @@ -name: Type checker benchmark -run-name: 'Type checker benchmark for PR #${{ inputs.pr_number }}' +name: Type checker benchmark candidate +run-name: 'Type checker benchmark candidate for PR #${{ github.event.pull_request.number }}' env: BENCHMARK_RUNNER_CLASS: 'github-ubuntu-latest' @@ -8,220 +8,54 @@ env: PYTHON_VERSION: '3.14.6' on: - workflow_dispatch: - inputs: - pr_number: - description: Pull request number - required: true - type: string + pull_request: + types: + - opened + - reopened + - synchronize + +permissions: + contents: read + issues: read concurrency: - group: ${{ github.workflow }}-${{ inputs.pr_number }} + group: ${{ github.workflow }}-${{ github.event.pull_request.number }} cancel-in-progress: true jobs: - metadata: - name: Resolve pull request revisions + authorize: + name: Check benchmark request runs-on: ubuntu-latest - permissions: - pull-requests: read outputs: - head-sha: ${{ steps.pull-request.outputs.head-sha }} - head-ref: ${{ steps.pull-request.outputs.head-ref }} - head-repository: ${{ steps.pull-request.outputs.head-repository }} - base-sha: ${{ steps.pull-request.outputs.base-sha }} - merge-sha: ${{ steps.pull-request.outputs.merge-sha }} + requested: ${{ steps.request.outputs.requested }} steps: - - name: Read pull request metadata - id: pull-request + - name: Check request label + id: request uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 env: - PR_NUMBER: ${{ inputs.pr_number }} + RUN_ATTEMPT: ${{ github.run_attempt }} with: script: | - const issueNumber = Number(process.env.PR_NUMBER) - if (!Number.isSafeInteger(issueNumber) || issueNumber <= 0) { - core.setFailed('The pull request number is invalid') - return - } const pullRequest = await github.rest.pulls.get({ owner: context.repo.owner, repo: context.repo.repo, - pull_number: issueNumber, + pull_number: context.issue.number, }) - if (pullRequest.data.state !== 'open' || !pullRequest.data.merge_commit_sha) { - core.setFailed('The pull request is not open or does not have a merge commit') - return - } - core.setOutput('head-sha', pullRequest.data.head.sha) - core.setOutput('head-ref', pullRequest.data.head.ref) - core.setOutput('head-repository', pullRequest.data.head.repo.full_name) - core.setOutput('base-sha', pullRequest.data.base.sha) - core.setOutput('merge-sha', pullRequest.data.merge_commit_sha) - - base-benchmark: - name: Benchmark base commit - needs: metadata - runs-on: ubuntu-latest - timeout-minutes: 180 - permissions: - contents: read - outputs: - cached: ${{ steps.cached-result.outputs.valid }} - - steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 - with: - ref: ${{ needs.metadata.outputs.base-sha }} - persist-credentials: false - - - uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6 - with: - python-version: ${{ env.PYTHON_VERSION }} - - - name: Determine base cache key - id: cache-key - shell: bash - env: - BASE_SHA: ${{ needs.metadata.outputs.base-sha }} - run: | - profile="${ImageOS}-${ImageVersion}-$(uname -m)-cpu$(nproc)-python${PYTHON_VERSION}-node${NODE_VERSION}-pnpm${PNPM_VERSION}-r1-w0-m8192-heap6656" - profile="$(printf '%s' "$profile" | tr -c 'A-Za-z0-9_.-' '-')" - echo "value=typecheck-benchmark-base-v2-${BASE_SHA}-${profile}" >> "$GITHUB_OUTPUT" - - - name: Restore cached base result - id: base-cache - uses: actions/cache/restore@0057852bfaa89a56745cba8c7296529d2fc39830 # v4 - with: - path: build/benchmark/base-results/latest-linux-x64.json - key: ${{ steps.cache-key.outputs.value }} - - - name: Validate cached base result - id: cached-result - shell: bash - env: - BASE_SHA: ${{ needs.metadata.outputs.base-sha }} - run: | - result=build/benchmark/base-results/latest-linux-x64.json - if [[ "${{ steps.base-cache.outputs.cache-hit }}" == "true" ]] && \ - python -c "import hashlib,json,sys; data=json.load(open(sys.argv[1], encoding='utf-8')); digest=hashlib.sha256(b''.join(open(path, 'rb').read() for path in sys.argv[3:])).hexdigest(); sys.exit(data.get('source_revision') != sys.argv[2] or not data.get('source_commit_subject') or not data.get('source_commit_timestamp') or data.get('benchmark_profile_hash') != digest)" \ - "$result" "$BASE_SHA" build/benchmark/typecheck_benchmark.py \ - build/benchmark/install_envs.json && \ - python build/benchmark/compare_benchmarks.py "$result" "$result" \ - --fail-on-preparation-error; then - echo "valid=true" >> "$GITHUB_OUTPUT" - else - rm -f "$result" - echo "valid=false" >> "$GITHUB_OUTPUT" - fi - - - if: ${{ steps.cached-result.outputs.valid != 'true' }} - uses: pnpm/action-setup@f520eceda224fe1a4aed5a2a27a194379a409996 # v6 - with: - version: ${{ env.PNPM_VERSION }} - - - if: ${{ steps.cached-result.outputs.valid != 'true' }} - uses: actions/setup-node@249970729cb0ef3589644e2896645e5dc5ba9c38 # v6.5.0 - with: - node-version: ${{ env.NODE_VERSION }} - - - name: Verify build prerequisites - if: ${{ steps.cached-result.outputs.valid != 'true' }} - run: | - gcc --version - g++ --version - make --version - - - name: Install JavaScript dependencies - if: ${{ steps.cached-result.outputs.valid != 'true' }} - timeout-minutes: 10 - env: - SKIP_LERNA_BOOTSTRAP: 'yes' - run: pnpm install --frozen-lockfile --prefer-offline - - - name: Build Pyright CLI - if: ${{ steps.cached-result.outputs.valid != 'true' }} - working-directory: packages/pyright - run: pnpm run build - - - name: Benchmark base commit - if: ${{ steps.cached-result.outputs.valid != 'true' }} - env: - NODE_OPTIONS: '--max-old-space-size=6656' - PYTHONNOUSERSITE: '1' - run: | - python build/benchmark/typecheck_benchmark.py \ - -c pyright -r 1 -w 0 -t 1800 --memory-limit-mb 8192 \ - --skip-pyright-build --os-name linux-x64 \ - --output build/benchmark/base-results - - - name: Record base revision - if: ${{ steps.cached-result.outputs.valid != 'true' }} - env: - BASE_SHA: ${{ needs.metadata.outputs.base-sha }} - run: | - python - <<'PY' - import hashlib - import json - import os - import subprocess - from pathlib import Path - - path = Path('build/benchmark/base-results/latest-linux-x64.json') - data = json.loads(path.read_text(encoding='utf-8')) - data['source_revision'] = os.environ['BASE_SHA'] - data['source_commit_subject'] = subprocess.check_output( - ['git', 'show', '-s', '--format=%s', os.environ['BASE_SHA']], text=True - ).strip() - data['source_commit_timestamp'] = subprocess.check_output( - ['git', 'show', '-s', '--format=%cI', os.environ['BASE_SHA']], text=True - ).strip() - profile = hashlib.sha256() - for profile_path in ( - Path('build/benchmark/typecheck_benchmark.py'), - Path('build/benchmark/install_envs.json'), - ): - profile.update(profile_path.read_bytes()) - data['benchmark_profile_hash'] = profile.hexdigest() - path.write_text(json.dumps(data, indent=2) + '\n', encoding='utf-8') - PY - - - name: Validate base result - env: - BASE_SHA: ${{ needs.metadata.outputs.base-sha }} - run: | - python build/benchmark/compare_benchmarks.py \ - build/benchmark/base-results/latest-linux-x64.json \ - build/benchmark/base-results/latest-linux-x64.json \ - --fail-on-preparation-error - python -c "import json,sys; data=json.load(open(sys.argv[1], encoding='utf-8')); sys.exit(data.get('source_revision') != sys.argv[2])" \ - build/benchmark/base-results/latest-linux-x64.json "$BASE_SHA" - - - name: Save base result cache - if: ${{ steps.cached-result.outputs.valid != 'true' }} - uses: actions/cache/save@0057852bfaa89a56745cba8c7296529d2fc39830 # v4 - with: - path: build/benchmark/base-results/latest-linux-x64.json - key: ${{ steps.cache-key.outputs.value }} - - - name: Upload base result - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 - with: - name: typecheck-benchmark-base-${{ needs.metadata.outputs.base-sha }} - path: build/benchmark/base-results/latest-linux-x64.json + const requested = Number(process.env.RUN_ATTEMPT) > 1 && pullRequest.data.labels.some( + (label) => label.name === 'benchmark-requested' + ) + core.setOutput('requested', requested) candidate-benchmark: name: Benchmark pull request merge - needs: metadata + needs: authorize + if: ${{ needs.authorize.outputs.requested == 'true' }} runs-on: ubuntu-latest timeout-minutes: 180 - permissions: - contents: read steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 with: - ref: ${{ needs.metadata.outputs.merge-sha }} persist-credentials: false - uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6 @@ -262,26 +96,30 @@ jobs: --skip-pyright-build --os-name linux-x64 \ --output build/benchmark/candidate-results - - name: Record candidate revision and profile + - name: Record candidate revisions and profile env: - MERGE_SHA: ${{ needs.metadata.outputs.merge-sha }} + BASE_SHA: ${{ github.event.pull_request.base.sha }} + HEAD_SHA: ${{ github.event.pull_request.head.sha }} + MERGE_SHA: ${{ github.sha }} run: | python - <<'PY' import hashlib import json import os - import subprocess + import subprocess from pathlib import Path path = Path('build/benchmark/candidate-results/latest-linux-x64.json') data = json.loads(path.read_text(encoding='utf-8')) data['source_revision'] = os.environ['MERGE_SHA'] - data['source_commit_subject'] = subprocess.check_output( + data['source_head_revision'] = os.environ['HEAD_SHA'] + data['source_base_revision'] = os.environ['BASE_SHA'] + data['source_commit_subject'] = subprocess.check_output( ['git', 'show', '-s', '--format=%s', os.environ['MERGE_SHA']], text=True - ).strip() - data['source_commit_timestamp'] = subprocess.check_output( + ).strip() + data['source_commit_timestamp'] = subprocess.check_output( ['git', 'show', '-s', '--format=%cI', os.environ['MERGE_SHA']], text=True - ).strip() + ).strip() profile = hashlib.sha256() for profile_path in ( Path('build/benchmark/typecheck_benchmark.py'), @@ -295,333 +133,6 @@ jobs: - name: Upload candidate result uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: - name: typecheck-benchmark-candidate-${{ needs.metadata.outputs.merge-sha }} + name: typecheck-benchmark-candidate path: build/benchmark/candidate-results/latest-linux-x64.json - - comparison: - name: Compare Pyright performance - needs: [metadata, base-benchmark, candidate-benchmark] - runs-on: ubuntu-latest - permissions: - contents: read - - steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 - with: - ref: ${{ github.sha }} - persist-credentials: false - - - name: Download base result - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 - with: - name: typecheck-benchmark-base-${{ needs.metadata.outputs.base-sha }} - path: benchmark-report/base - - - name: Download candidate result - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 - with: - name: typecheck-benchmark-candidate-${{ needs.metadata.outputs.merge-sha }} - path: benchmark-report/candidate - - - name: Compare with base commit - id: comparison - continue-on-error: true - env: - BASE_SHA: ${{ needs.metadata.outputs.base-sha }} - MERGE_SHA: ${{ needs.metadata.outputs.merge-sha }} - run: | - set +e - python build/benchmark/compare_benchmarks.py \ - benchmark-report/base/latest-linux-x64.json \ - benchmark-report/candidate/latest-linux-x64.json \ - --fail-on-preparation-error \ - --allow-incompatible \ - --baseline-revision "$BASE_SHA" --candidate-revision "$MERGE_SHA" \ - --markdown-output benchmark-report/report.md - comparison_status=$? - cat benchmark-report/report.md >> "$GITHUB_STEP_SUMMARY" - exit "$comparison_status" - - - name: Upload benchmark report - if: always() - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 - with: - name: typecheck-benchmark-linux-x64-pr-${{ inputs.pr_number }}-head-${{ needs.metadata.outputs.head-sha }}-base-${{ needs.metadata.outputs.base-sha }}-merge-${{ needs.metadata.outputs.merge-sha }} - path: benchmark-report/ - - - name: Fail on benchmark regressions - if: ${{ steps.comparison.outcome == 'failure' }} - run: exit 1 - - comment: - name: Comment benchmark results - needs: [metadata, comparison] - if: ${{ always() && needs.metadata.result == 'success' }} - runs-on: ubuntu-latest - permissions: - actions: read - contents: read - pull-requests: write - steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 - with: - ref: ${{ github.sha }} - persist-credentials: false - - - name: Download benchmark results - id: download - uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 - env: - PR_NUMBER: ${{ inputs.pr_number }} - EXPECTED_HEAD_SHA: ${{ needs.metadata.outputs.head-sha }} - EXPECTED_BASE_SHA: ${{ needs.metadata.outputs.base-sha }} - EXPECTED_MERGE_SHA: ${{ needs.metadata.outputs.merge-sha }} - with: - script: | - const fs = require('fs') - const issueNumber = Number(process.env.PR_NUMBER) - const expectedHeadSha = process.env.EXPECTED_HEAD_SHA - const expectedBaseSha = process.env.EXPECTED_BASE_SHA - const expectedMergeSha = process.env.EXPECTED_MERGE_SHA - const shaPattern = /^[0-9a-f]{40}$/ - if ( - !Number.isSafeInteger(issueNumber) || - issueNumber <= 0 || - !shaPattern.test(expectedHeadSha) || - !shaPattern.test(expectedBaseSha) || - !shaPattern.test(expectedMergeSha) - ) { - core.setFailed('The benchmark dispatch inputs are invalid') - return - } - const artifactName = - `typecheck-benchmark-linux-x64-pr-${issueNumber}` + - `-head-${expectedHeadSha}-base-${expectedBaseSha}-merge-${expectedMergeSha}` - const artifacts = await github.paginate(github.rest.actions.listWorkflowRunArtifacts, { - owner: context.repo.owner, - repo: context.repo.repo, - run_id: context.runId, - per_page: 100, - }) - const reports = artifacts.filter((artifact) => artifact.name === artifactName) - if (reports.length !== 1) { - core.setFailed(`Expected one benchmark report artifact, found ${reports.length}`) - return - } - const pullRequest = await github.rest.pulls.get({ - owner: context.repo.owner, - repo: context.repo.repo, - pull_number: issueNumber, - }) - if (pullRequest.data.head.sha !== expectedHeadSha) { - core.setFailed('The workflow run head does not match the pull request head') - return - } - if (pullRequest.data.base.sha !== expectedBaseSha) { - core.setFailed('The workflow run base does not match the pull request base') - return - } - if (pullRequest.data.merge_commit_sha !== expectedMergeSha) { - core.setFailed('The workflow run merge commit does not match the pull request merge commit') - return - } - const download = await github.rest.actions.downloadArtifact({ - owner: context.repo.owner, - repo: context.repo.repo, - artifact_id: reports[0].id, - archive_format: 'zip', - }) - fs.writeFileSync('benchmark-report.zip', Buffer.from(download.data)) - core.setOutput('pr-number', issueNumber) - - - name: Extract benchmark results - if: ${{ steps.download.outputs.pr-number != '' }} - run: | - python - <<'PY' - import json - import zipfile - from pathlib import PurePosixPath - - def reject_constant(value): - raise ValueError(f'Non-finite JSON number: {value}') - - with zipfile.ZipFile('benchmark-report.zip') as archive: - entries = { - PurePosixPath(info.filename): info - for info in archive.infolist() - if '..' not in PurePosixPath(info.filename).parts - } - for archive_path, output_path in ( - (PurePosixPath('base/latest-linux-x64.json'), 'base.json'), - (PurePosixPath('candidate/latest-linux-x64.json'), 'candidate.json'), - ): - info = entries.get(archive_path) - if info is None: - raise RuntimeError(f'Missing benchmark result: {archive_path}') - if info.file_size > 5 * 1024 * 1024: - raise RuntimeError(f'Benchmark result exceeds 5 MB: {archive_path}') - contents = archive.read(info) - data = json.loads(contents, parse_constant=reject_constant) - if not isinstance(data, dict): - raise RuntimeError(f'Benchmark result must be an object: {archive_path}') - with open(output_path, 'wb') as output: - output.write(contents) - PY - - - name: Render benchmark report - if: ${{ steps.download.outputs.pr-number != '' }} - run: | - set +e - python build/benchmark/compare_benchmarks.py \ - base.json \ - candidate.json \ - --fail-on-preparation-error \ - --allow-incompatible \ - --baseline-revision "${{ needs.metadata.outputs.base-sha }}" \ - --candidate-revision "${{ needs.metadata.outputs.merge-sha }}" \ - --markdown-output report.md - if [[ ! -s report.md ]]; then - printf '%s\n' \ - '## Type checker benchmark' \ - '' \ - '🔴 **The benchmark results could not be compared. See the workflow run for details.**' \ - > report.md - fi - - - name: Post benchmark comment - if: ${{ steps.download.outputs.pr-number != '' }} - uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 - env: - PR_NUMBER: ${{ steps.download.outputs.pr-number }} - with: - script: | - const fs = require('fs') - const marker = '' - const report = fs.readFileSync('report.md', 'utf8') - const issueNumber = Number(process.env.PR_NUMBER) - const body = `${marker}\n${report}` - const comments = await github.paginate(github.rest.issues.listComments, { - owner: context.repo.owner, - repo: context.repo.repo, - issue_number: issueNumber, - per_page: 100, - }) - const previous = comments.find( - (comment) => - comment.user?.login === 'github-actions[bot]' && comment.body?.startsWith(marker) - ) - if (previous) { - await github.rest.issues.updateComment({ - owner: context.repo.owner, - repo: context.repo.repo, - comment_id: previous.id, - body, - }) - } else { - await github.rest.issues.createComment({ - owner: context.repo.owner, - repo: context.repo.repo, - issue_number: issueNumber, - body, - }) - } - - persist-base-result: - name: Update pull request baseline - needs: [metadata, base-benchmark, comment] - if: >- - ${{ !cancelled() && needs.base-benchmark.result == 'success' && - needs.base-benchmark.outputs.cached != 'true' && - needs.metadata.outputs.head-repository == github.repository }} - runs-on: ubuntu-latest - permissions: - actions: read - contents: write - steps: - - name: Download base result - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 - with: - name: typecheck-benchmark-base-${{ needs.metadata.outputs.base-sha }} - path: benchmark-result - - - name: Commit baseline to pull request branch - uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 - env: - EXPECTED_HEAD_SHA: ${{ needs.metadata.outputs.head-sha }} - HEAD_REF: ${{ needs.metadata.outputs.head-ref }} - BASE_SHA: ${{ needs.metadata.outputs.base-sha }} - with: - script: | - const fs = require('fs') - const path = 'benchmark-result/latest-linux-x64.json' - const contents = fs.readFileSync(path, 'utf8') - const result = JSON.parse(contents) - const expectedHeadSha = process.env.EXPECTED_HEAD_SHA - const headRef = process.env.HEAD_REF - const baseSha = process.env.BASE_SHA - if ( - result.source_revision !== baseSha || - typeof result.source_commit_subject !== 'string' || - result.source_commit_subject.length === 0 || - Number.isNaN(Date.parse(result.source_commit_timestamp)) || - Number.isNaN(Date.parse(result.timestamp)) || - !/^\d{4}-\d{2}-\d{2}$/.test(result.date) - ) { - core.setFailed('The base benchmark result metadata is invalid') - return - } - - const currentRef = await github.rest.git.getRef({ - owner: context.repo.owner, - repo: context.repo.repo, - ref: `heads/${headRef}`, - }) - if (currentRef.data.object.sha !== expectedHeadSha) { - core.notice('The pull request head changed; skipping the baseline update') - return - } - - const parent = await github.rest.git.getCommit({ - owner: context.repo.owner, - repo: context.repo.repo, - commit_sha: expectedHeadSha, - }) - const blob = await github.rest.git.createBlob({ - owner: context.repo.owner, - repo: context.repo.repo, - content: Buffer.from(contents).toString('base64'), - encoding: 'base64', - }) - const tree = await github.rest.git.createTree({ - owner: context.repo.owner, - repo: context.repo.repo, - base_tree: parent.data.tree.sha, - tree: [ - { - path: 'build/benchmark/baselines/latest-linux-x64.json', - mode: '100644', - type: 'blob', - sha: blob.data.sha, - }, - { - path: `build/benchmark/baselines/benchmark_${result.date}_linux-x64.json`, - mode: '100644', - type: 'blob', - sha: blob.data.sha, - }, - ], - }) - const commit = await github.rest.git.createCommit({ - owner: context.repo.owner, - repo: context.repo.repo, - message: `Update benchmark baseline for ${baseSha.slice(0, 12)}`, - tree: tree.data.sha, - parents: [expectedHeadSha], - }) - await github.rest.git.updateRef({ - owner: context.repo.owner, - repo: context.repo.repo, - ref: `heads/${headRef}`, - sha: commit.data.sha, - force: false, - }) + retention-days: 1 diff --git a/.github/workflows/typecheck_benchmark_report.yml b/.github/workflows/typecheck_benchmark_report.yml new file mode 100644 index 000000000000..82a6ca87cb97 --- /dev/null +++ b/.github/workflows/typecheck_benchmark_report.yml @@ -0,0 +1,502 @@ +name: Type checker benchmark report +run-name: 'Type checker benchmark report for run #${{ github.event.workflow_run.id }}' + +env: + BENCHMARK_RUNNER_CLASS: 'github-ubuntu-latest' + NODE_VERSION: '24.15.0' + PNPM_VERSION: '10.12.2' + PYTHON_VERSION: '3.14.6' + +on: + workflow_run: + workflows: + - Type checker benchmark candidate + types: + - completed + +permissions: {} + +concurrency: + group: ${{ github.workflow }}-${{ github.event.workflow_run.id }} + cancel-in-progress: false + +jobs: + metadata: + name: Validate pull request revisions + if: ${{ github.event.workflow_run.event == 'pull_request' && github.event.workflow_run.run_attempt > 1 }} + runs-on: ubuntu-latest + permissions: + pull-requests: read + outputs: + pr-number: ${{ steps.pull-request.outputs.pr-number }} + head-sha: ${{ steps.pull-request.outputs.head-sha }} + head-ref: ${{ steps.pull-request.outputs.head-ref }} + head-repository: ${{ steps.pull-request.outputs.head-repository }} + merge-sha: ${{ steps.pull-request.outputs.merge-sha }} + steps: + - name: Validate pull request metadata + id: pull-request + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 + with: + script: | + const workflowRun = context.payload.workflow_run + let pullRequests = workflowRun.pull_requests + if (pullRequests.length === 0) { + const associated = await github.rest.repos.listPullRequestsAssociatedWithCommit({ + owner: context.repo.owner, + repo: context.repo.repo, + commit_sha: workflowRun.head_sha, + }) + pullRequests = associated.data.filter((pullRequest) => pullRequest.state === 'open') + } + if (pullRequests.length !== 1) { + core.setFailed(`Expected one pull request for the benchmark run, found ${pullRequests.length}`) + return + } + const issueNumber = pullRequests[0].number + const pullRequest = await github.rest.pulls.get({ + owner: context.repo.owner, + repo: context.repo.repo, + pull_number: issueNumber, + }) + if ( + pullRequest.data.state !== 'open' || + pullRequest.data.head.sha !== workflowRun.head_sha || + pullRequest.data.base.sha !== context.sha || + !pullRequest.data.merge_commit_sha + ) { + core.setFailed('The pull request changed after the benchmark was requested') + return + } + core.setOutput('pr-number', issueNumber) + core.setOutput('head-sha', pullRequest.data.head.sha) + core.setOutput('head-ref', pullRequest.data.head.ref) + core.setOutput('head-repository', pullRequest.data.head.repo.full_name) + core.setOutput('merge-sha', pullRequest.data.merge_commit_sha) + + base-benchmark: + name: Benchmark main commit + needs: metadata + if: ${{ github.event.workflow_run.conclusion == 'success' }} + runs-on: ubuntu-latest + timeout-minutes: 180 + permissions: + contents: read + outputs: + cached: ${{ steps.cached-result.outputs.valid }} + + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + with: + ref: ${{ github.sha }} + persist-credentials: false + + - uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6 + with: + python-version: ${{ env.PYTHON_VERSION }} + + - name: Determine base cache key + id: cache-key + shell: bash + env: + BASE_SHA: ${{ github.sha }} + run: | + profile="${ImageOS}-${ImageVersion}-$(uname -m)-cpu$(nproc)-python${PYTHON_VERSION}-node${NODE_VERSION}-pnpm${PNPM_VERSION}-r1-w0-m8192-heap6656" + profile="$(printf '%s' "$profile" | tr -c 'A-Za-z0-9_.-' '-')" + echo "value=typecheck-benchmark-base-v2-${BASE_SHA}-${profile}" >> "$GITHUB_OUTPUT" + + - name: Restore cached base result + id: base-cache + uses: actions/cache/restore@0057852bfaa89a56745cba8c7296529d2fc39830 # v4 + with: + path: build/benchmark/base-results/latest-linux-x64.json + key: ${{ steps.cache-key.outputs.value }} + + - name: Validate cached base result + id: cached-result + shell: bash + env: + BASE_SHA: ${{ github.sha }} + run: | + result=build/benchmark/base-results/latest-linux-x64.json + if [[ "${{ steps.base-cache.outputs.cache-hit }}" == "true" ]] && \ + python -c "import hashlib,json,sys; data=json.load(open(sys.argv[1], encoding='utf-8')); digest=hashlib.sha256(b''.join(open(path, 'rb').read() for path in sys.argv[3:])).hexdigest(); sys.exit(data.get('source_revision') != sys.argv[2] or not data.get('source_commit_subject') or not data.get('source_commit_timestamp') or data.get('benchmark_profile_hash') != digest)" \ + "$result" "$BASE_SHA" build/benchmark/typecheck_benchmark.py \ + build/benchmark/install_envs.json && \ + python build/benchmark/compare_benchmarks.py "$result" "$result" \ + --fail-on-preparation-error; then + echo "valid=true" >> "$GITHUB_OUTPUT" + else + rm -f "$result" + echo "valid=false" >> "$GITHUB_OUTPUT" + fi + + - if: ${{ steps.cached-result.outputs.valid != 'true' }} + uses: pnpm/action-setup@f520eceda224fe1a4aed5a2a27a194379a409996 # v6 + with: + version: ${{ env.PNPM_VERSION }} + + - if: ${{ steps.cached-result.outputs.valid != 'true' }} + uses: actions/setup-node@249970729cb0ef3589644e2896645e5dc5ba9c38 # v6.5.0 + with: + node-version: ${{ env.NODE_VERSION }} + + - name: Verify build prerequisites + if: ${{ steps.cached-result.outputs.valid != 'true' }} + run: | + gcc --version + g++ --version + make --version + + - name: Install JavaScript dependencies + if: ${{ steps.cached-result.outputs.valid != 'true' }} + timeout-minutes: 10 + env: + SKIP_LERNA_BOOTSTRAP: 'yes' + run: pnpm install --frozen-lockfile --prefer-offline + + - name: Build Pyright CLI + if: ${{ steps.cached-result.outputs.valid != 'true' }} + working-directory: packages/pyright + run: pnpm run build + + - name: Benchmark main commit + if: ${{ steps.cached-result.outputs.valid != 'true' }} + env: + NODE_OPTIONS: '--max-old-space-size=6656' + PYTHONNOUSERSITE: '1' + run: | + python build/benchmark/typecheck_benchmark.py \ + -c pyright -r 1 -w 0 -t 1800 --memory-limit-mb 8192 \ + --skip-pyright-build --os-name linux-x64 \ + --output build/benchmark/base-results + + - name: Record base revision and profile + if: ${{ steps.cached-result.outputs.valid != 'true' }} + env: + BASE_SHA: ${{ github.sha }} + run: | + python - <<'PY' + import hashlib + import json + import os + import subprocess + from pathlib import Path + + path = Path('build/benchmark/base-results/latest-linux-x64.json') + data = json.loads(path.read_text(encoding='utf-8')) + data['source_revision'] = os.environ['BASE_SHA'] + data['source_commit_subject'] = subprocess.check_output( + ['git', 'show', '-s', '--format=%s', os.environ['BASE_SHA']], text=True + ).strip() + data['source_commit_timestamp'] = subprocess.check_output( + ['git', 'show', '-s', '--format=%cI', os.environ['BASE_SHA']], text=True + ).strip() + profile = hashlib.sha256() + for profile_path in ( + Path('build/benchmark/typecheck_benchmark.py'), + Path('build/benchmark/install_envs.json'), + ): + profile.update(profile_path.read_bytes()) + data['benchmark_profile_hash'] = profile.hexdigest() + path.write_text(json.dumps(data, indent=2) + '\n', encoding='utf-8') + PY + + - name: Validate base result + env: + BASE_SHA: ${{ github.sha }} + run: | + python build/benchmark/compare_benchmarks.py \ + build/benchmark/base-results/latest-linux-x64.json \ + build/benchmark/base-results/latest-linux-x64.json \ + --fail-on-preparation-error \ + --baseline-revision "$BASE_SHA" --candidate-revision "$BASE_SHA" + + - name: Save base result cache + if: ${{ steps.cached-result.outputs.valid != 'true' }} + uses: actions/cache/save@0057852bfaa89a56745cba8c7296529d2fc39830 # v4 + with: + path: build/benchmark/base-results/latest-linux-x64.json + key: ${{ steps.cache-key.outputs.value }} + + - name: Upload base result + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: typecheck-benchmark-base + path: build/benchmark/base-results/latest-linux-x64.json + retention-days: 1 + + comparison: + name: Compare Pyright performance + needs: [metadata, base-benchmark] + runs-on: ubuntu-latest + permissions: + actions: read + contents: read + + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + with: + ref: ${{ github.sha }} + persist-credentials: false + + - name: Download base result + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + name: typecheck-benchmark-base + path: benchmark-report/base + + - name: Download candidate result + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + name: typecheck-benchmark-candidate + path: benchmark-report/candidate + run-id: ${{ github.event.workflow_run.id }} + github-token: ${{ secrets.GITHUB_TOKEN }} + + - name: Validate candidate provenance + env: + BASE_SHA: ${{ github.sha }} + HEAD_SHA: ${{ needs.metadata.outputs.head-sha }} + MERGE_SHA: ${{ needs.metadata.outputs.merge-sha }} + run: | + python - <<'PY' + import json + import os + from pathlib import Path + + path = Path('benchmark-report/candidate/latest-linux-x64.json') + if path.stat().st_size > 5 * 1024 * 1024: + raise RuntimeError('Candidate benchmark result exceeds 5 MB') + data = json.loads(path.read_text(encoding='utf-8'), parse_constant=lambda value: (_ for _ in ()).throw(ValueError(value))) + if not isinstance(data, dict): + raise RuntimeError('Candidate benchmark result must be an object') + expected = { + 'source_revision': os.environ['MERGE_SHA'], + 'source_head_revision': os.environ['HEAD_SHA'], + 'source_base_revision': os.environ['BASE_SHA'], + } + for field, value in expected.items(): + if data.get(field) != value: + raise RuntimeError(f'Candidate {field} does not match the pull request') + PY + + - name: Compare with main commit + id: comparison + continue-on-error: true + env: + BASE_SHA: ${{ github.sha }} + MERGE_SHA: ${{ needs.metadata.outputs.merge-sha }} + run: | + set +e + python build/benchmark/compare_benchmarks.py \ + benchmark-report/base/latest-linux-x64.json \ + benchmark-report/candidate/latest-linux-x64.json \ + --fail-on-preparation-error \ + --allow-incompatible \ + --baseline-revision "$BASE_SHA" --candidate-revision "$MERGE_SHA" \ + --markdown-output benchmark-report/report.md + comparison_status=$? + cat benchmark-report/report.md >> "$GITHUB_STEP_SUMMARY" + exit "$comparison_status" + + - name: Upload benchmark report + if: always() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: typecheck-benchmark-report-${{ github.event.workflow_run.id }} + path: benchmark-report/ + + - name: Fail on benchmark regressions + if: ${{ steps.comparison.outcome == 'failure' }} + run: exit 1 + + comment: + name: Comment benchmark results + needs: [metadata, comparison] + if: ${{ always() && needs.metadata.result == 'success' }} + runs-on: ubuntu-latest + permissions: + actions: read + pull-requests: write + steps: + - name: Download benchmark report + if: ${{ needs.comparison.result == 'success' || needs.comparison.result == 'failure' }} + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + name: typecheck-benchmark-report-${{ github.event.workflow_run.id }} + path: benchmark-report + + - name: Post benchmark comment + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 + env: + PR_NUMBER: ${{ needs.metadata.outputs.pr-number }} + CANDIDATE_CONCLUSION: ${{ github.event.workflow_run.conclusion }} + COMPARISON_RESULT: ${{ needs.comparison.result }} + with: + script: | + const fs = require('fs') + const marker = '' + const issueNumber = Number(process.env.PR_NUMBER) + let report + if (process.env.CANDIDATE_CONCLUSION !== 'success') { + report = '## Type checker benchmark\n\n🔴 **The candidate benchmark failed. See the workflow run for details.**\n' + } else if (process.env.COMPARISON_RESULT === 'success' || process.env.COMPARISON_RESULT === 'failure') { + report = fs.readFileSync('benchmark-report/report.md', 'utf8') + } else { + report = '## Type checker benchmark\n\n🔴 **The trusted base benchmark or report failed. See the workflow run for details.**\n' + } + const body = `${marker}\n${report}` + const comments = await github.paginate(github.rest.issues.listComments, { + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: issueNumber, + per_page: 100, + }) + const previous = comments.find( + (comment) => + comment.user?.login === 'github-actions[bot]' && comment.body?.startsWith(marker) + ) + if (previous) { + await github.rest.issues.updateComment({ + owner: context.repo.owner, + repo: context.repo.repo, + comment_id: previous.id, + body, + }) + } else { + await github.rest.issues.createComment({ + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: issueNumber, + body, + }) + } + + persist-base-result: + name: Update pull request baseline + needs: [metadata, base-benchmark, comment] + if: >- + ${{ !cancelled() && needs.base-benchmark.result == 'success' && + needs.base-benchmark.outputs.cached != 'true' && + needs.metadata.outputs.head-repository == github.repository }} + runs-on: ubuntu-latest + permissions: + actions: read + contents: write + steps: + - name: Download base result + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + name: typecheck-benchmark-base + path: benchmark-result + + - name: Commit baseline to pull request branch + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 + env: + EXPECTED_HEAD_SHA: ${{ needs.metadata.outputs.head-sha }} + HEAD_REF: ${{ needs.metadata.outputs.head-ref }} + BASE_SHA: ${{ github.sha }} + with: + script: | + const fs = require('fs') + const contents = fs.readFileSync('benchmark-result/latest-linux-x64.json', 'utf8') + const result = JSON.parse(contents) + const expectedHeadSha = process.env.EXPECTED_HEAD_SHA + const headRef = process.env.HEAD_REF + const baseSha = process.env.BASE_SHA + if ( + result.source_revision !== baseSha || + typeof result.source_commit_subject !== 'string' || + result.source_commit_subject.length === 0 || + Number.isNaN(Date.parse(result.source_commit_timestamp)) || + Number.isNaN(Date.parse(result.timestamp)) || + !/^\d{4}-\d{2}-\d{2}$/.test(result.date) + ) { + core.setFailed('The base benchmark result metadata is invalid') + return + } + const currentRef = await github.rest.git.getRef({ + owner: context.repo.owner, + repo: context.repo.repo, + ref: `heads/${headRef}`, + }) + if (currentRef.data.object.sha !== expectedHeadSha) { + core.notice('The pull request head changed; skipping the baseline update') + return + } + const parent = await github.rest.git.getCommit({ + owner: context.repo.owner, + repo: context.repo.repo, + commit_sha: expectedHeadSha, + }) + const blob = await github.rest.git.createBlob({ + owner: context.repo.owner, + repo: context.repo.repo, + content: Buffer.from(contents).toString('base64'), + encoding: 'base64', + }) + const tree = await github.rest.git.createTree({ + owner: context.repo.owner, + repo: context.repo.repo, + base_tree: parent.data.tree.sha, + tree: [ + { + path: 'build/benchmark/baselines/latest-linux-x64.json', + mode: '100644', + type: 'blob', + sha: blob.data.sha, + }, + { + path: `build/benchmark/baselines/benchmark_${result.date}_linux-x64.json`, + mode: '100644', + type: 'blob', + sha: blob.data.sha, + }, + ], + }) + const commit = await github.rest.git.createCommit({ + owner: context.repo.owner, + repo: context.repo.repo, + message: `Update benchmark baseline for ${baseSha.slice(0, 12)}`, + tree: tree.data.sha, + parents: [expectedHeadSha], + }) + await github.rest.git.updateRef({ + owner: context.repo.owner, + repo: context.repo.repo, + ref: `heads/${headRef}`, + sha: commit.data.sha, + force: false, + }) + + cleanup: + name: Clear benchmark request + if: >- + ${{ always() && github.event.workflow_run.event == 'pull_request' && + github.event.workflow_run.run_attempt > 1 }} + needs: [metadata, base-benchmark, comparison, comment, persist-base-result] + runs-on: ubuntu-latest + permissions: + issues: write + steps: + - name: Remove request label + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 + with: + script: | + const pullRequests = context.payload.workflow_run.pull_requests + const issueNumber = Number('${{ needs.metadata.outputs.pr-number }}') || pullRequests[0]?.number + if (!issueNumber) { + core.notice('No pull request was associated with the benchmark run') + return + } + try { + await github.rest.issues.removeLabel({ + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: issueNumber, + name: 'benchmark-requested', + }) + } catch (error) { + if (error.status !== 404) throw error + } diff --git a/.github/workflows/typecheck_benchmark_trigger.yml b/.github/workflows/typecheck_benchmark_trigger.yml index 686b3b0641b2..e39c87c080f4 100644 --- a/.github/workflows/typecheck_benchmark_trigger.yml +++ b/.github/workflows/typecheck_benchmark_trigger.yml @@ -7,7 +7,7 @@ on: permissions: actions: write - contents: read + issues: write pull-requests: read jobs: @@ -36,12 +36,70 @@ jobs: return } - await github.rest.actions.createWorkflowDispatch({ + const label = 'benchmark-requested' + const pullRequest = await github.rest.pulls.get({ owner: context.repo.owner, repo: context.repo.repo, - workflow_id: 'typecheck_benchmark_pr.yml', - ref: context.payload.repository.default_branch, - inputs: { - pr_number: String(context.issue.number), - }, + pull_number: context.issue.number, + }) + if (!pullRequest.data.merge_commit_sha) { + core.setFailed('The pull request must be mergeable before it can be benchmarked') + return + } + let candidateRun + for (let attempt = 0; attempt < 12; attempt++) { + const runs = await github.rest.actions.listWorkflowRuns({ + owner: context.repo.owner, + repo: context.repo.repo, + workflow_id: 'typecheck_benchmark_pr.yml', + event: 'pull_request', + head_sha: pullRequest.data.head.sha, + per_page: 100, + }) + candidateRun = runs.data.workflow_runs.find( + (run) => run.pull_requests.some((item) => item.number === context.issue.number) + ) + if (candidateRun?.status === 'completed') break + await new Promise((resolve) => setTimeout(resolve, 5000)) + } + if (!candidateRun || candidateRun.status !== 'completed') { + core.setFailed('The pull request validation workflow is not ready; retry /benchmark shortly') + return + } + try { + await github.rest.issues.getLabel({ + owner: context.repo.owner, + repo: context.repo.repo, + name: label, + }) + } catch (error) { + if (error.status !== 404) throw error + await github.rest.issues.createLabel({ + owner: context.repo.owner, + repo: context.repo.repo, + name: label, + color: '1d76db', + description: 'Run the hosted type checker benchmark', + }) + } + try { + await github.rest.issues.removeLabel({ + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: context.issue.number, + name: label, + }) + } catch (error) { + if (error.status !== 404) throw error + } + await github.rest.issues.addLabels({ + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: context.issue.number, + labels: [label], + }) + await github.rest.actions.reRunWorkflow({ + owner: context.repo.owner, + repo: context.repo.repo, + run_id: candidateRun.id, }) diff --git a/build/benchmark/README.md b/build/benchmark/README.md index 6c2d0484dba5..28ad4b5377b1 100644 --- a/build/benchmark/README.md +++ b/build/benchmark/README.md @@ -107,16 +107,19 @@ result contract, and the comparator rejects mismatched environments. The hosted pull-request benchmark runs only when a maintainer comments exactly `/benchmark` on an open pull request. The command must be the entire comment. The trusted command workflow checks that -the commenter has `write`, `maintain`, or `admin` repository permission and then explicitly dispatches -the benchmark with the pull request's current head, base, and synthetic merge commits. Users without -one of these permissions cannot start the benchmark. - -The dispatched workflow runs the pull request's synthetic merge commit, so all mergeable open pull -requests use the current pnpm build metadata from the base branch. It uses read-only repository -permissions. When the run completes, a separate trusted job validates the result's head, base, and -merge commits, renders it using default-branch code, and creates or updates one benchmark comment. -The trusted job runs on a separate runner with pull-request write permission. The Actions job summary -and benchmark artifact contain the same candidate results. +the commenter has `write`, `maintain`, or `admin` repository permission and then toggles the +`benchmark-requested` label and reruns the current commit's existing pull-request workflow. Users +without one of these permissions cannot start the benchmark. The initial workflow run for each pull +request commit performs only the request check; measured work is restricted to authorized reruns. + +The authorized rerun executes an unprivileged `pull_request` workflow that runs the pull request's +synthetic merge commit with read-only repository and issue permissions. It cannot read or write the +shared benchmark cache. When it completes, a separate trusted `workflow_run` workflow accepts only +rerun attempts, uses only default-branch code to validate the candidate artifact and pull request +revisions, benchmarks or restores the exact base commit, renders the comparison, and creates or +updates one benchmark comment. Candidate output is treated only as bounded JSON data and is never +executed by the trusted workflow. The Actions job summary and benchmark artifact contain the same +candidate results. Comment `/benchmark` again after pushing a new commit or when rerunning the same head. No benchmark is started automatically for later commits. The command workflow must already exist on the repository's @@ -179,12 +182,15 @@ threshold does not alter a checker invocation that completed below either thresh workflow fails if either revision cannot prepare a package or if Pyright times out, crashes, or otherwise fails. -On pull requests, the workflow compares the synthetic merge commit with the exact base commit from -which GitHub created that merge. It never compares against a moving `main` reference. The base result -is cached under an exact key containing its commit and hosted measurement profile. A missing, expired, -or invalid cache entry causes a fresh base build and benchmark; prefix and fallback cache matches are -not used. Only the job that checks out trusted base code can populate this shared cache. The job that -executes pull-request code cannot write it. +On pull requests, the workflow compares the synthetic merge commit with the exact default-branch +commit validated against the pull request base. If the pull request or default branch changes before +the trusted workflow validates those revisions, the report stops and a new `/benchmark` request is +required. The base result is cached under an exact key containing its commit and hosted measurement +profile. A missing, expired, or invalid cache entry causes a fresh base build and benchmark; prefix +and fallback cache matches are not used. Only the job that checks out trusted default-branch code can +populate this shared cache. The job that executes pull-request code cannot write it. A newly measured +base result is also committed to the checked-in dated and `latest` baseline files on same-repository +pull requests after confirming that the pull request head has not changed. Both revisions use Python 3.14.6, a 6.5 GiB V8 old-space limit, one measured run, no discarded warmup, and a 30-minute invocation timeout. A regression must exceed both a 20% relative threshold and an diff --git a/build/benchmark/test_compare_benchmarks.py b/build/benchmark/test_compare_benchmarks.py index b24afceff455..ae9a45e53608 100644 --- a/build/benchmark/test_compare_benchmarks.py +++ b/build/benchmark/test_compare_benchmarks.py @@ -509,25 +509,32 @@ def test_incompatible_results_still_require_successful_measurements(self) -> Non ) def test_pr_workflow_uses_matching_base_and_candidate_profiles(self) -> None: - workflow = ( + candidate_workflow = ( REPO_ROOT / ".github" / "workflows" / "typecheck_benchmark_pr.yml" ).read_text(encoding="utf-8") + report_workflow = ( + REPO_ROOT / ".github" / "workflows" / "typecheck_benchmark_report.yml" + ).read_text(encoding="utf-8") timeout_matches = re.findall( r"typecheck_benchmark\.py \\\s+" r"-c pyright -r 1 -w 0 -t (\d+)", - workflow, + candidate_workflow + report_workflow, ) self.assertEqual(timeout_matches, ["1800", "1800"]) - self.assertIn("data['source_revision'] = os.environ['BASE_SHA']", workflow) - self.assertIn("data['source_revision'] = os.environ['MERGE_SHA']", workflow) - self.assertEqual(workflow.count("data['source_commit_subject']"), 2) - self.assertEqual(workflow.count("data['source_commit_timestamp']"), 2) - self.assertIn("data['benchmark_profile_hash'] = profile.hexdigest()", workflow) - self.assertIn("build/benchmark/baselines/latest-linux-x64.json", workflow) + self.assertIn("data['source_revision'] = os.environ['MERGE_SHA']", candidate_workflow) + self.assertIn("data['source_head_revision']", candidate_workflow) + self.assertIn("data['source_base_revision']", candidate_workflow) + self.assertIn("data['source_revision'] = os.environ['BASE_SHA']", report_workflow) + for workflow in (candidate_workflow, report_workflow): + self.assertIn("data['source_commit_subject']", workflow) + self.assertIn("data['source_commit_timestamp']", workflow) + self.assertIn("data['benchmark_profile_hash'] = profile.hexdigest()", workflow) + self.assertIn("build/benchmark/baselines/latest-linux-x64.json", report_workflow) def test_workflows_use_current_pnpm_setup(self) -> None: for workflow_name in ( "typecheck_benchmark_pr.yml", + "typecheck_benchmark_report.yml", "typecheck_benchmark_weekly.yml", ): workflow = ( @@ -572,7 +579,7 @@ def test_workflows_use_current_pnpm_setup(self) -> None: REPO_ROOT / ".github" / "workflows" / "typecheck_benchmark_pr.yml" ).read_text(encoding="utf-8") self.assertIn( - "run-name: 'Type checker benchmark for PR #${{ inputs.pr_number }}'", + "run-name: 'Type checker benchmark candidate for PR #${{ github.event.pull_request.number }}'", pr_workflow, ) self.assertIn("PNPM_VERSION: '10.12.2'", pr_workflow) @@ -590,6 +597,11 @@ def test_pr_benchmark_requires_authorized_comment(self) -> None: ) benchmark_workflow = benchmark_workflow_path.read_text(encoding="utf-8") benchmark_workflow_data = _load_yaml(benchmark_workflow_path) + report_workflow_path = ( + REPO_ROOT / ".github" / "workflows" / "typecheck_benchmark_report.yml" + ) + report_workflow = report_workflow_path.read_text(encoding="utf-8") + report_workflow_data = _load_yaml(report_workflow_path) self.assertIn("issue_comment:", trigger_workflow) self.assertIn( @@ -601,11 +613,17 @@ def test_pr_benchmark_requires_authorized_comment(self) -> None: self.assertIn("github.event.issue.state == 'open'", trigger_workflow) self.assertIn("getCollaboratorPermissionLevel", trigger_workflow) self.assertIn("['admin', 'maintain', 'write']", trigger_workflow) - self.assertIn("actions: write", trigger_workflow) + self.assertIn("issues: write", trigger_workflow) self.assertIn("pull-requests: read", trigger_workflow) - self.assertIn("createWorkflowDispatch", trigger_workflow) - self.assertIn("workflow_id: 'typecheck_benchmark_pr.yml'", trigger_workflow) - self.assertNotIn("head_sha:", trigger_workflow) + self.assertIn("actions: write", trigger_workflow) + self.assertIn("actions.listWorkflowRuns", trigger_workflow) + self.assertIn("actions.reRunWorkflow", trigger_workflow) + self.assertIn("issues.addLabels", trigger_workflow) + self.assertIn("issues.removeLabel", trigger_workflow) + self.assertIn("issues.createLabel", trigger_workflow) + self.assertNotIn("createWorkflowDispatch", trigger_workflow) + self.assertNotIn("workflow_dispatch:", trigger_workflow) + self.assertNotIn("inputs:", trigger_workflow) self.assertNotIn("base_sha:", trigger_workflow) self.assertNotIn("merge_sha:", trigger_workflow) self.assertNotIn("actions/checkout", trigger_workflow) @@ -618,47 +636,46 @@ def test_pr_benchmark_requires_authorized_comment(self) -> None: "actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7", benchmark_workflow, ) - self.assertIn( - "actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0", - benchmark_workflow, - ) self.assertNotIn("actions/checkout@v4", benchmark_workflow) - self.assertNotIn("actions/github-script@v7", benchmark_workflow) - self.assertIn("workflow_dispatch:", benchmark_workflow) + self.assertIn("pull_request:", benchmark_workflow) + self.assertIn("- synchronize", benchmark_workflow) + self.assertNotIn("- labeled", benchmark_workflow) + self.assertIn("Number(process.env.RUN_ATTEMPT) > 1", benchmark_workflow) + self.assertIn("label.name === 'benchmark-requested'", benchmark_workflow) + self.assertIn("needs.authorize.outputs.requested == 'true'", benchmark_workflow) + self.assertNotIn("workflow_dispatch:", benchmark_workflow) self.assertNotIn("paths:", benchmark_workflow) - self.assertNotIn("pull_request:", benchmark_workflow) self.assertNotIn("cache: 'pip'", benchmark_workflow) self.assertNotIn("cache: 'pnpm'", benchmark_workflow) self.assertIn("persist-credentials: false", benchmark_workflow) - self.assertNotIn("inputs.head_sha", benchmark_workflow) - self.assertNotIn("inputs.base_sha", benchmark_workflow) - self.assertNotIn("inputs.merge_sha", benchmark_workflow) - self.assertIn("github.rest.pulls.get", benchmark_workflow) - self.assertIn("ref: ${{ needs.metadata.outputs.base-sha }}", benchmark_workflow) - self.assertIn("ref: ${{ needs.metadata.outputs.merge-sha }}", benchmark_workflow) - self.assertIn("-merge-${{ needs.metadata.outputs.merge-sha }}", benchmark_workflow) - self.assertIn("if: ${{ always() && needs.metadata.result == 'success' }}", benchmark_workflow) - self.assertIn("run_id: context.runId", benchmark_workflow) - self.assertIn("pullRequest.data.base.sha !== expectedBaseSha", benchmark_workflow) - self.assertIn( - "pullRequest.data.merge_commit_sha !== expectedMergeSha", - benchmark_workflow, - ) - metadata_job = benchmark_workflow_data["jobs"]["metadata"] - base_job = benchmark_workflow_data["jobs"]["base-benchmark"] candidate_job = benchmark_workflow_data["jobs"]["candidate-benchmark"] - comparison_job = benchmark_workflow_data["jobs"]["comparison"] - comment_job = benchmark_workflow_data["jobs"]["comment"] - persist_job = benchmark_workflow_data["jobs"]["persist-base-result"] + self.assertEqual( + benchmark_workflow_data["permissions"], + {"contents": "read", "issues": "read"}, + ) + self.assertNotIn("permissions", candidate_job) + self.assertNotIn("actions/cache", benchmark_workflow) + self.assertNotIn("contents: write", benchmark_workflow) + + self.assertIn("workflow_run:", report_workflow) + self.assertIn("Type checker benchmark candidate", report_workflow) + self.assertIn("github.event.workflow_run.run_attempt > 1", report_workflow) + self.assertIn("github.event.workflow_run.id", report_workflow) + self.assertIn("ref: ${{ github.sha }}", report_workflow) + self.assertNotIn("workflow_dispatch:", report_workflow) + metadata_job = report_workflow_data["jobs"]["metadata"] + base_job = report_workflow_data["jobs"]["base-benchmark"] + comparison_job = report_workflow_data["jobs"]["comparison"] + comment_job = report_workflow_data["jobs"]["comment"] + persist_job = report_workflow_data["jobs"]["persist-base-result"] + self.assertEqual(report_workflow_data["permissions"], {}) self.assertEqual(metadata_job["permissions"], {"pull-requests": "read"}) self.assertEqual(base_job["permissions"], {"contents": "read"}) - self.assertEqual(candidate_job["permissions"], {"contents": "read"}) - self.assertEqual(comparison_job["permissions"], {"contents": "read"}) + self.assertEqual(comparison_job["permissions"], {"actions": "read", "contents": "read"}) self.assertEqual( comment_job["permissions"], { "actions": "read", - "contents": "read", "pull-requests": "write", }, ) @@ -670,7 +687,7 @@ def test_pr_benchmark_requires_authorized_comment(self) -> None: self.assertEqual( [ job_name - for job_name, job in benchmark_workflow_data["jobs"].items() + for job_name, job in report_workflow_data["jobs"].items() if job.get("permissions", {}).get("pull-requests") == "write" ], ["comment"], @@ -678,44 +695,37 @@ def test_pr_benchmark_requires_authorized_comment(self) -> None: self.assertEqual( [ job_name - for job_name, job in benchmark_workflow_data["jobs"].items() + for job_name, job in report_workflow_data["jobs"].items() if job.get("permissions", {}).get("contents") == "write" ], ["persist-base-result"], ) - self.assertFalse( - ( - REPO_ROOT - / ".github" - / "workflows" - / "typecheck_benchmark_comment.yml" - ).exists() - ) + self.assertIn("github.event.workflow_run.conclusion == 'success'", report_workflow) def test_pr_workflow_caches_only_the_base_result(self) -> None: - workflow = ( + candidate_workflow = ( REPO_ROOT / ".github" / "workflows" / "typecheck_benchmark_pr.yml" ).read_text(encoding="utf-8") - workflow_data = _load_yaml( - REPO_ROOT / ".github" / "workflows" / "typecheck_benchmark_pr.yml" + report_workflow_path = ( + REPO_ROOT / ".github" / "workflows" / "typecheck_benchmark_report.yml" ) + workflow = report_workflow_path.read_text(encoding="utf-8") + workflow_data = _load_yaml(report_workflow_path) base_job = json.dumps(workflow_data["jobs"]["base-benchmark"]) - candidate_job = json.dumps(workflow_data["jobs"]["candidate-benchmark"]) self.assertIn("actions/cache/restore@0057852", base_job) self.assertIn("actions/cache/save@0057852", base_job) self.assertIn("typecheck-benchmark-base-v2-", base_job) self.assertNotIn("restore-keys", base_job) - self.assertNotIn("actions/cache/", candidate_job) - self.assertIn("ref: ${{ needs.metadata.outputs.base-sha }}", workflow) - self.assertIn("ref: ${{ needs.metadata.outputs.merge-sha }}", workflow) + self.assertNotIn("actions/cache/", candidate_workflow) + self.assertIn("ref: ${{ github.sha }}", workflow) + self.assertNotIn("actions/checkout", workflow_data["jobs"]["persist-base-result"]) self.assertIn("--baseline-revision", workflow) self.assertIn("--candidate-revision", workflow) self.assertIn("--allow-incompatible", workflow) self.assertIn("issues.listComments", workflow) self.assertIn("updateComment", workflow) self.assertIn("comment.user?.login === 'github-actions[bot]'", workflow) - self.assertIn("ref: ${{ github.sha }}", workflow) self.assertNotIn("git push", workflow) persist_job = workflow_data["jobs"]["persist-base-result"] persist_job_text = json.dumps(persist_job) From 5a10a8c3ea4f37275fd2a1996312ce57a4081e5b Mon Sep 17 00:00:00 2001 From: Bill Schnurr Date: Fri, 28 Aug 2026 16:04:24 -0700 Subject: [PATCH 4/5] Publish benchmark dashboard preview Export PNG previews alongside SVG dashboard charts and improve the single-baseline layout until additional hosted runs accumulate. --- build/benchmark/benchmark_history.ipynb | 34 +- docs/benchmark-results/comparison.json | 218 +++ docs/benchmark-results/execution_time_s.png | Bin 0 -> 73865 bytes docs/benchmark-results/execution_time_s.svg | 1792 ++++++++++++++++++ docs/benchmark-results/history.csv | 19 + docs/benchmark-results/index.html | 10 + docs/benchmark-results/peak_memory_mb.png | Bin 0 -> 72714 bytes docs/benchmark-results/peak_memory_mb.svg | 1811 +++++++++++++++++++ 8 files changed, 3873 insertions(+), 11 deletions(-) create mode 100644 docs/benchmark-results/comparison.json create mode 100644 docs/benchmark-results/execution_time_s.png create mode 100644 docs/benchmark-results/execution_time_s.svg create mode 100644 docs/benchmark-results/history.csv create mode 100644 docs/benchmark-results/index.html create mode 100644 docs/benchmark-results/peak_memory_mb.png create mode 100644 docs/benchmark-results/peak_memory_mb.svg diff --git a/build/benchmark/benchmark_history.ipynb b/build/benchmark/benchmark_history.ipynb index 431c3564dfbf..350dbc67de11 100644 --- a/build/benchmark/benchmark_history.ipynb +++ b/build/benchmark/benchmark_history.ipynb @@ -279,6 +279,8 @@ "# 7. Graph performance across commits\n", "SELECTED_PACKAGES = sorted(history[\"benchmark\"].unique())\n", "figures: dict[str, plt.Figure] = {}\n", + "run_count = history[\"collection_timestamp\"].nunique()\n", + "chart_timezone = history[\"collection_timestamp\"].dt.tz\n", "\n", "for metric, spec in METRIC_SPECS.items():\n", " metric_data = history[\n", @@ -295,19 +297,26 @@ " palette=\"colorblind\",\n", " ax=axis,\n", " )\n", - " for row in metric_data.itertuples():\n", - " axis.annotate(\n", - " row.short_sha,\n", - " (row.collection_timestamp, row.value),\n", - " xytext=(4, 5),\n", - " textcoords=\"offset points\",\n", - " fontsize=7,\n", - " alpha=0.8,\n", - " )\n", - " axis.set_title(f\"Pyright {spec['label'].lower()} across commits\")\n", + " title = f\"Pyright {spec['label'].lower()} across commits\"\n", + " if run_count > 1:\n", + " for row in metric_data.itertuples():\n", + " axis.annotate(\n", + " row.short_sha,\n", + " (row.collection_timestamp, row.value),\n", + " xytext=(4, 5),\n", + " textcoords=\"offset points\",\n", + " fontsize=7,\n", + " alpha=0.8,\n", + " )\n", + " else:\n", + " collected_at = metric_data[\"collection_timestamp\"].iloc[0]\n", + " axis.set_xlim(collected_at - pd.Timedelta(days=2), collected_at + pd.Timedelta(days=2))\n", + " axis.xaxis.set_major_locator(mdates.DayLocator(interval=1, tz=chart_timezone))\n", + " title += f\"\\nInitial hosted baseline {metric_data['short_sha'].iloc[0]}; trends appear as runs accumulate\"\n", + " axis.set_title(title)\n", " axis.set_xlabel(\"Benchmark collection time (UTC)\")\n", " axis.set_ylabel(f\"{spec['label']} ({spec['unit']})\")\n", - " axis.xaxis.set_major_formatter(mdates.DateFormatter(\"%Y-%m-%d\", tz=history[\"collection_timestamp\"].dt.tz))\n", + " axis.xaxis.set_major_formatter(mdates.DateFormatter(\"%Y-%m-%d\", tz=chart_timezone))\n", " axis.legend(title=\"Package\", bbox_to_anchor=(1.02, 1), loc=\"upper left\")\n", " figure.autofmt_xdate()\n", " figure.tight_layout()\n", @@ -401,8 +410,11 @@ " )\n", " for metric, figure in charts.items():\n", " chart_path = output_dir / f\"{metric}.svg\"\n", + " preview_path = output_dir / f\"{metric}.png\"\n", " figure.savefig(chart_path, format=\"svg\", bbox_inches=\"tight\")\n", + " figure.savefig(preview_path, format=\"png\", dpi=144, bbox_inches=\"tight\")\n", " outputs[metric] = chart_path\n", + " outputs[f\"{metric}_preview\"] = preview_path\n", " return outputs\n", "\n", "\n", diff --git a/docs/benchmark-results/comparison.json b/docs/benchmark-results/comparison.json new file mode 100644 index 000000000000..3d0d2d23e752 --- /dev/null +++ b/docs/benchmark-results/comparison.json @@ -0,0 +1,218 @@ +[ + { + "benchmark": "ansible", + "metric": "execution_time_s", + "unit": "seconds", + "baseline_sha": "075161cd9160", + "latest_sha": "075161cd9160", + "baseline": 21.661, + "latest": 21.661, + "absolute_change": 0.0, + "percent_change": 0.0, + "assessment": "Stable" + }, + { + "benchmark": "ansible", + "metric": "peak_memory_mb", + "unit": "MiB", + "baseline_sha": "075161cd9160", + "latest_sha": "075161cd9160", + "baseline": 1110.7, + "latest": 1110.7, + "absolute_change": 0.0, + "percent_change": 0.0, + "assessment": "Stable" + }, + { + "benchmark": "click", + "metric": "execution_time_s", + "unit": "seconds", + "baseline_sha": "075161cd9160", + "latest_sha": "075161cd9160", + "baseline": 2.908, + "latest": 2.908, + "absolute_change": 0.0, + "percent_change": 0.0, + "assessment": "Stable" + }, + { + "benchmark": "click", + "metric": "peak_memory_mb", + "unit": "MiB", + "baseline_sha": "075161cd9160", + "latest_sha": "075161cd9160", + "baseline": 361.0, + "latest": 361.0, + "absolute_change": 0.0, + "percent_change": 0.0, + "assessment": "Stable" + }, + { + "benchmark": "homeassistant", + "metric": "execution_time_s", + "unit": "seconds", + "baseline_sha": "075161cd9160", + "latest_sha": "075161cd9160", + "baseline": 158.341, + "latest": 158.341, + "absolute_change": 0.0, + "percent_change": 0.0, + "assessment": "Stable" + }, + { + "benchmark": "homeassistant", + "metric": "peak_memory_mb", + "unit": "MiB", + "baseline_sha": "075161cd9160", + "latest_sha": "075161cd9160", + "baseline": 6192.1, + "latest": 6192.1, + "absolute_change": 0.0, + "percent_change": 0.0, + "assessment": "Stable" + }, + { + "benchmark": "numpy", + "metric": "execution_time_s", + "unit": "seconds", + "baseline_sha": "075161cd9160", + "latest_sha": "075161cd9160", + "baseline": 43.435, + "latest": 43.435, + "absolute_change": 0.0, + "percent_change": 0.0, + "assessment": "Stable" + }, + { + "benchmark": "numpy", + "metric": "peak_memory_mb", + "unit": "MiB", + "baseline_sha": "075161cd9160", + "latest_sha": "075161cd9160", + "baseline": 1850.0, + "latest": 1850.0, + "absolute_change": 0.0, + "percent_change": 0.0, + "assessment": "Stable" + }, + { + "benchmark": "pandas", + "metric": "execution_time_s", + "unit": "seconds", + "baseline_sha": "075161cd9160", + "latest_sha": "075161cd9160", + "baseline": 989.16, + "latest": 989.16, + "absolute_change": 0.0, + "percent_change": 0.0, + "assessment": "Stable" + }, + { + "benchmark": "pandas", + "metric": "peak_memory_mb", + "unit": "MiB", + "baseline_sha": "075161cd9160", + "latest_sha": "075161cd9160", + "baseline": 4434.0, + "latest": 4434.0, + "absolute_change": 0.0, + "percent_change": 0.0, + "assessment": "Stable" + }, + { + "benchmark": "pytest", + "metric": "execution_time_s", + "unit": "seconds", + "baseline_sha": "075161cd9160", + "latest_sha": "075161cd9160", + "baseline": 12.574, + "latest": 12.574, + "absolute_change": 0.0, + "percent_change": 0.0, + "assessment": "Stable" + }, + { + "benchmark": "pytest", + "metric": "peak_memory_mb", + "unit": "MiB", + "baseline_sha": "075161cd9160", + "latest_sha": "075161cd9160", + "baseline": 930.9, + "latest": 930.9, + "absolute_change": 0.0, + "percent_change": 0.0, + "assessment": "Stable" + }, + { + "benchmark": "requests", + "metric": "execution_time_s", + "unit": "seconds", + "baseline_sha": "075161cd9160", + "latest_sha": "075161cd9160", + "baseline": 2.299, + "latest": 2.299, + "absolute_change": 0.0, + "percent_change": 0.0, + "assessment": "Stable" + }, + { + "benchmark": "requests", + "metric": "peak_memory_mb", + "unit": "MiB", + "baseline_sha": "075161cd9160", + "latest_sha": "075161cd9160", + "baseline": 347.2, + "latest": 347.2, + "absolute_change": 0.0, + "percent_change": 0.0, + "assessment": "Stable" + }, + { + "benchmark": "torch", + "metric": "execution_time_s", + "unit": "seconds", + "baseline_sha": "075161cd9160", + "latest_sha": "075161cd9160", + "baseline": 153.779, + "latest": 153.779, + "absolute_change": 0.0, + "percent_change": 0.0, + "assessment": "Stable" + }, + { + "benchmark": "torch", + "metric": "peak_memory_mb", + "unit": "MiB", + "baseline_sha": "075161cd9160", + "latest_sha": "075161cd9160", + "baseline": 4891.9, + "latest": 4891.9, + "absolute_change": 0.0, + "percent_change": 0.0, + "assessment": "Stable" + }, + { + "benchmark": "transformers", + "metric": "execution_time_s", + "unit": "seconds", + "baseline_sha": "075161cd9160", + "latest_sha": "075161cd9160", + "baseline": 134.544, + "latest": 134.544, + "absolute_change": 0.0, + "percent_change": 0.0, + "assessment": "Stable" + }, + { + "benchmark": "transformers", + "metric": "peak_memory_mb", + "unit": "MiB", + "baseline_sha": "075161cd9160", + "latest_sha": "075161cd9160", + "baseline": 5130.0, + "latest": 5130.0, + "absolute_change": 0.0, + "percent_change": 0.0, + "assessment": "Stable" + } +] diff --git a/docs/benchmark-results/execution_time_s.png b/docs/benchmark-results/execution_time_s.png new file mode 100644 index 0000000000000000000000000000000000000000..a812ede997b4a1eebcdd44a4dc3b357c08741882 GIT binary patch literal 73865 zcmdSBc|4Tu8$K+hg;Y{_*?kecwOc`F#46%y!?`d7bBR9OrRf&otEJ>1j{XQc+RSD=OU3 zqN3WLK}EG!{@{N2AEkW*lJFPG?WVq)w&Q&_4>K1FDpfN#Cp$+sJL|h=+$~&OtsNZ% z`LA5zzjEn}m7ANBD@s7X{@-WtJGxj3xb*K`hj%&Tq;T7nis~d6@@J2t7Uw+G9x5uu z8`pI_^*lgI7sKZw{|Lb26u2UixfKipv4!}#wHDZvkr+x6E& z^QH{VKmUG^arQ0sKR?Lm{GkEY_t)bL?*G+~`N}dl2=9G^@afqJ`>yw@&!R2AMmAry z#M_CxjupGMNfOB2ndTL)<7!>z;uvyd&1%2x?0D5&EzxE4D`Tg-TD+K`>-rk^*)LVobc7(da=8!Pwg6@Rci9df&+(H9d3Eo*X|| z;cD5v=5t<|9G@^tShSP4e#kx zE3qf`7FxC?o=@ z?#3b?!s6VZzrDRia*)Es3fmf#*!1{Ll{u0el6)7R<(Sm;Z|C~4pxnODpZH#E-KA5v z(-gw$Do4AsLNMj;<~1zMHuha;Tu6L+E0j&ryO>zpYw9yo@~l+Qf6J3tY9ALz$-+$a zUY|RcJkk_e+1ozH{DJ;B&dh(-92HtX(Z`WZ`lOJcrF5%N4jdn$B%YVyu? zNO5I)?F<{0y{dFLTT}CdP?@M_BenOO-wv`nIYW`X{NbBY6wj!{(JGHQ_lMoxncBup zxhR}t?Z()RaYfb&*dWEhjJ%a!83I{wA5e4EOGXzqVK+_L5=(6L2)Cw2ONkE837WgT zhK?0S(3mi91`)}f^^sijl{Hd;hnZUa1M*XL|NQYpA3TlJPWL3r?)zmzLAFy1F^?dg zWe6*;#j4R7E8MdG2+M0LX;h;B_GTX}+MIt|2_ye%F%1{GAZLJ_>JL)r|GAr zpSXM*+F>+`57^n@&sbq+(->DAU;E0N=hn13Q${+EDkYc(3{?ILV8V<{)4-{*zailCyo>-pZSsH6Cl4tY!B--}v8lj&vtyhPbs_q&cqc zZY{OBxA2wee|$ufW7C71Ex>td$rof4T7C)Rh!(ch)yqNG`TG$Hb|K9WAUeZx`At7W3W2Z>2{!TmOn) zPk3y|EM=~agxSPBE{C+hZ>1q4>M$Ut_wcIc{GF=F_i;u(+g_RD1y&trJ>tphz0SCX zjab`U=cvmy-YbQ(gH=zfk8W&E<@x1JLXxxOGbnaq{ZeGv=5B^Xm7p1W-d^8ZrjC~V zqAtB1nepgG!g`F6%dq^d8xyT;G;d)q0yht`Ffmv7Q_7P z?*Pd17Oq&kmD(l^3;#ez3rW0Ar8cBmKO;!FBUpOo=zDQ{4;=Y~r0f#g?hMU=0`sPU z#g!>Sl?587^Nd(vE??cmBJRSO>ps)(467AFEVOv;C}ERF90`?9w32QLWnar7LE1BX zPl04wX#uhFlCbmR<6~#h0jI}vSEhP%mS%{Cz3)}y${|5D`V4$8FdwXZ&R@wrv>W##r35-l^9mYHen1iITOW%p!L4l6hCmKbKn!R(W{Tk-VpJ zoG+-wAGBph9q{%y*ctMfN?3Z^$$GO^uXrbDp)4Pov;^#K zi{?VAcj!w`@{hG*hnTD|^njvMZrkF+HstPR0_FAe+!dq-7^W{p7<^kxn&r7tPOfR^{e3=g?hhQbAXU*9A{O7FX|;^Xf~8x`fhtJ zZOxuLFKac+;;8hNQ_8BhwAG+6!Dz(rq#fngKFU@@jl((T!5iK_4|7+34KcsEwhN0y zr4`Dt1KSmw?>4EWM?b1gsw9Yx3wKBG*j3}W9bad3rYTo_4Q3D6>VO+MhJC!~yVNS` zP|JhXn>1WMWbD5&nu*Gt>`Jd19@VLNC!W4FX}F6s(CAUn5;97_QZJ|5JiGpQe?>5t zy&>^XREQ(oVuuMn^fRth^IYxp5I)1wZ?#*C%6cn5=-*_R2aq$MR!>c3A{`B3;2Itiir*U3F0$Oj$XXtI$`Ye+wf~C*}8NuI@ z#=dIyV5R3`;n^$E-A3}S-Y!kl1=JiBstd*Gxv`*wicyuOl(F_89ZP+@V+Qt%SLi>_YcIx%w zXiJP$M)_YCn{<+3pHEOVa1ana>-D?zrm`WJ`Zv0p8jD)U7|~7W!?kO0J**9!L#u66 zk#SkmZ!8m?Sz`>#956jvxO;Cz9TVyN!*P*cBQB`pLX?S-GHCr2rE{$hm)fLvQ@y2Q zB|e{+b$e%mel&KplTSRh=^{shM8dhi0|1l?nH{UIoTb1Pep*GSP!!N92_LY%$fXn+ zMCdhrJKwxp(tS{#Y>eLAf)N^n7cld z=4igSG|`;~{>0oVuCPDO-TDqGea0t`TS#nez}2|tIo5v(w{&gTi231AS#oXe>C<^o zs%DpD8@iz0d;z#Rj;nR6TKvju|Le;ujxu$n{5I)`V?3xkIMZ;?Wt@a?FNqb5&gP|eg8dy>iI{M+h&2-gIz49v2CC0 zj-QS6vzjQ2#5qZA&CBU+bd)=fcz$eXj<|55q(>uYcro!Q&*ktGv&k)DqJ4dZ33bPS zV_w}(B^}>v)tr>*KFhvB3%|Vd+=3^2(D{U02DQ1kFIqis9V%yqvWFpYl85%j=Gj8g zU%;*%F7-R}Q7Ryp*lsF-WUgVwK|LQpgIDt*GgXRf2<-+xN@e$7|Jb_Z)^U?5ux18@ zA(#8u3m(ISu>rh3%Mm%8Y15rCJj=+>3Kix~TwjqT@yVeRb}Qtbg-0b#Y(C$Puw9$r zIjXbgUGN)W+urZIdDX{PrGz-&aSFF}Kn^qm>|A<)&GSRxpYYnX9Az<~_;q%ljPdOI zt+BI10Vb(3j214#flLMl;t365f!;NoH_^{cFf9Q7=f=(~9%WuZUC7=?yV|etoPh3m zL@Of1kSxy;hpq5k?X&co*#Vdn){kVFB$HgCl>)iukLIPLaII%nelt<6w>Ipp(kYJJ zNB_L~=?J@&9l^xM{suOs^YX*~ovn?f82~P!WwWqbRy&miZceW>$8XgA^m5E7?rvu> zinUe#7>_kjwwmcLSp^8!4;7WoF)$^4$DO0vtnF(G442Q<>(pYw?*WYD=$kj4C>5E+4W! z!bKz6e?Ln%=Tgp-MU=c!20>Y=p~rft>-uLyO!uaOsPN#{N?)2g!0 zAe=_-U~iUQ8{maelWhR?mPdG&QoX_{rn`~fO+?oSsGhE$||Pe)K{(;qn~V9t{KTYQoCEOqUoyKly($` zTY780fqz}AN?D+$yChD?I=nSbv<=g_Hrds*X4GAgJ2UlNsO$xgs*q3|R)SdJn$A4L zJFF4#F}?^WW7YiV#&}X00z3V;7UV_KUz(Ibwf{`mAZ6(Zh^iPH92&)hL~59{MBt#F#MBZbEadslMa+92u zBlD1&e?r5v%AP2KpgkX{FQXVKEQq^kDN|n@!+;OS>A6Xa%_j z%9P$};Yu$dmDEQM3>_*OJGCOZJ2PyXojiH#<>^ssc{DqZeus|` z#2DAd+T0!-x(JASpYa@EMT}?w{b?bmp4TRf_w#48k3+N_{X5#0|B5ygA}9UGevh9A zd}s&()-V)}lVZ@YmWr1VQqctxKG23ctfauYIOL5MU)T0bIdX@7CEbRt)wu@*fHGEr zPnv)@?Yj}t(bXPondL^@izDr${isvs2F6H9Y#6y#XQ zuiaQ^)pV{$U%sa@S)$?~t$1kcwRV3UE}r4XMLQLRJx|%Plp>zUdhzY;xRP}`1NKF& z_>W~N4KWOYU_}y87#HG7UGL)x5v{+F(s-`syIZp?Taf5x4G0yT1-R1}239W~8mswZ zdg=Kx=EUR-*GjG8?0@;8SJ5FTt7mSLr+dPj$OFsR96ADE+gXRr{Q(<`qbG6*qT4-v zd3WSlTakPZRPu3TQxE+80Fj~4>Y2EUTEmgt^ICyA0EP#zp$^ro^cv2ZpplFT4@){8 zL?sNL?^2hp7=A0%-8G&R-~a@|!?i*u&qVU>xQeT=vDau+DeM?gyEoVkZL0TXJMUHA z{KRf`wOg3_E(AvLpo0m2>4x!)zjJVSzRRc*(T~T;0c22eTG%GltFafjfVG7CbiVJ#;_cC|QC+@CzrQb4 z`riNQl|1Jws}R`n`yq2n{=bUlUs57o2w`xn*F4*82)XufasK+m&CwXs=`xak5om_D zVV!>F5FvR-@f)LhW*!&#Zn%i%si!GLWqYrXGh+`}LFv=2_4Ud^XGN!pKE$I9^h@pD zY(pR+`j|fKoVN9)le5AnGN-Qav5Gl;(9JQh0FIr<$fNUx)e_?ScuAa?b4tQBud8UH z9mp7?%@Ix8*=^K*=Q+pg=>*(dEx9a*>jO&zcKxe<2J?p7GchrZ<0$}tM*Mp%!*ZLt zm1g`?7X14vFal8T?X|Q*XaJKF_n#JrK3qtsg99J$(Bo&7lE$G z!XABauV%SZxzxJrybqMD`Xd?l@$rayhiD@`gpYanehxY8_g~Es_8>o9EIry7c~RSL zpxoKLO>!+2XMP6&+v+OO5z%ov(e2>_6-so-ibI_n@?M^Jg*(!>>^*cA(|UvKt-=eq zBDn_OR4GWAq(YE$Zi5<=1N?HKUuxr5g!}aO0v&t3;bJBYz?Su!Yja6<6d;?I`yP;C zUEIe3Sr#^9`AB|i3G^&t?U+v+NY1SRigyvqpE;;*P-nV2J2;j2^pPaZt;FM8O67zx zK!@tiL~AenqY4LnWLxg?yw^dhsxyRA26NVHQC}_7qE7p!*l4s4{EUjD3S&b#!loQ|@ za4awO1Aw>Tfrn!7pskeza*}b{GqxB3Q>iS?cUZ_k3HP z>@w!k5~D1n6O`xjSS8%cFW*AnX|6ev=~p*V5rhp~I-35gdI|5^roJZ%`x&%wE1kC6 zoOyA-QkVNiYo5BPGV&R8qvtM$b@?QFF8t)EF|GGA-B0m4_0>|7Q;M!;Jd>cFq+DrI ztTN+9pX4)Jp_mwVhwa3yrEY%P1!00L=PFY(K_Jas-TKH`$B-hBpY?%u)z^lo6NLZ7 zkK5SvKXqkW*~qAqEB4#APid3hW**-irSiLF%^Vw==QLbDyPYKIvzi~uaeQ1fXZHv+ zB*cu7YQV8D#_SEy-T1)!+J|$yd;yq5EBpyew&!vD@VjAh_a2_uqfM0i0U)3b2Bb#B zF3EE|#3s=S?bMv^Sh~OtplaG=7HRm{%+sM3d%Th zpUKX&+!IC#k7<<{ew=42orqV`l53e9I6#~IcrbKgfcI4Afb7$w?5liwmZVluk{!?u ztLl)tJ3|7S*EBAq>%EHX-ZEKiI4XT(J3i2RP$gchqvk-5xCxXcWksbVW7B}`l9z0f z{l-g(RyOsW5kqUM#)rw>Pw_vr;`itDxEXiSO$h0tcArGhREw*y!bz{C0=7-%687$f zOlq*5ApaW&ZKj;2b&(;GY2rcE;`mDU88V%)Y(C_u#_Pyskx#Em zi&l<3rDqIoQF z`qeywV(50pdAS1eok2ZUis5JNb?sVRrZ?OkH)UPjyhR4GI<3=pX2=mQbb@`Bt8= zE4F|c*tNgC&XFWd<3_&MDV!Wv`$zT>2~4Lo8;j$5xrS-m9lT-2_<26kfmc$A(mU%7 zgBKW2W@t6l8EY-u+t!Hmr`LBssU}~nIZvy$B~g;57;!%LoodhCoo+82;q%m&rMWG2 zg@^A^(>L}6F};}dOA8V8GnX_Q)#0Wrtfu|ANqB*(#jxU(%pk;PC9{qwpgBCFR6o|5 zZb`_T@ZU>%_jP}VZ5dKy;=^0S_jhizDekIeo>e2{NVzid+r)YM)~iiIHu-9M48NpE zHV9hYc>ZgO310x>_td(ZX~akW-2Fr;7TVb5zKq~=Xoks}+%xioYT3)%FH_ni{f&DWos<&tH`Qy? z5s-i4dh+wtVM#~U7}S%M)_Z{(J7`(*1C6Imkf9yU-Fo3fYG=aK2Gug=sND<2>j6_A z5Y0ymC(}NS#7C8;sT~S5CqJ3byRXTex@~K9ruF&=TCN(vb?(wlO0Sct$%Q8SvcWMd zD>NbrvM5K|^x3*ye}d-E-FSw6>xoN}BxurNx4iee4#`T2T5m`N{aTn9U+PDep4n1frxO=yvfT!Z+NjEi|P{E7oYgpL>uGJbEAVI$7;5d>~^_f z5AK}m^?ar~HdfWQi|#jOed_YL-eiz+nj|D9HUWRv`?tfEQ#l}(bI75 z%uPCx{u2{9JN8{m-A3l*du!7BS*t>Zp7P?ucG#rDK=^?pqGPlyrEpIK)#XsWo`^a6=u#%>mKQ79* zgVYm+qh~3?XF!M_@#`8$s@;Pd9;gn9QqYRRqIi~bw03vDJx4JJKY2;$V}3S1#;)eo zrpkIUCB$paFv}#p9kOS9#yJDl4<3Yx2$S9 z3GMup$74=myU>6$Y507ZZtJu-R82U!Tq`ZmM307*{DC04Abj{Oegp9K;7W>Px~*RZ z12(93&=0y6Rb)m?HqJ(#@C-+NQov*y)h{$~M{JoM)8V=}$H0hrV#CcD*le<*oj=D! zpZTk1hgOV%yxz5v_4VK`=^X}HCTOuv@Qgm9->048b%z>5e!Ap;Xv`0iwi%~%L&HXy zDy6;lINM_^?^0c_oSzDJ-gU9I3z@bY;vMT{oWio=tC9O$S(9dMuYPaqxkG#^H#hub zyu;_n(3N$)=t)P>yzp-3Tw%#wN#>Vzh-sjcnh^*_zLDo};GP&J;5vi0o^cA5Kwt*= zj$uNw3KfszYmPOKkD1;+bNcE{3cBf*mTPOqIa$_^ad#&#Ex}8#a9lS?`RX=S@))c)Ys$l+ z6SS5+SKbf5+kg1fvCeBuLtdR>YUDd;`PkjE-W5IIi1*w8Pgd-_CK_%MC$+KKQek!5h$S34+~` z>1T@FgmNT!Z6JWSoc2sc&VBMVX8QD1m9~!EnuPu)t5aY9+8x|cSFN`l6>S0yM^DU> zZl^EB9$s}7{=Pa)tFvd+bS7(fKKwMcCqOEFd40{Uc&8U?ssbIt?eZrK-#pBC+!B{B z!N>ipVm5KxM8suE9xP*Th);kKO>uQB|ZFDbbrdNmb1q7Ae zF+pk<_xSlpwYY8*-d9p=j{0T4Q0-e10(*^<$ltYkUUCYvSlW29zNjqi);wvoxE{IA zp7bc{32U$k;{zT8A%P}sb;(hQrjHt2uSB2f)+bZ%T^K5Y+9Y&15$n*&a{e^6&ci7A z>rvQYElESvyj>kz{NZiOq1{s;qM8F_cOe}%u5`23Ns1G7(^Fy{9a^nc zIz%rjXx+&vdcAB+y4|(E#-|MF(hUk-_zH~n4k9xY5?AWaV^XbZvE6QOy;uvIF}3^1 z*@`cJ;UAabC76<{6FL&q{6BfYhb^LN?A*l)bA_xr%XcefM?h;Z zIP;d8Dqd6Q$?LJN9cA`D%M-&mp|8<=Q&ZQvjnrS!W@xoApS{IeN1iy$(4NsypHUxz)Bc;JDRY&xNs01uKv*W=$Hd zbM+`@v@0&DO_tch?)V@h7v*ds9J02#Upcf(ytDn5BJ1sp761CrszvH=# zIfFY#0Cl+5RHxwWM&)sX{5ub;2vV}FKLxs7PtIaa(06$6VwW~={rtVb!%-p_1D}Q` zT!&pA`>EvTT`%6vQJV@-t{<%ttqJfsK`Y#?u1>g@pOp2nRIgE8sLfpKEavbl-_X>u z-?{YBAA@-{d#S=?E|X$=<}Uz*$#A{NS{p0)VK;7O=-#j*h9#8-GfO@twA%vB$GxQ; zEJC!F-D<22HWZ-z>!^$xS7$_a3<*X$$rd&vbyCOZiN7#vpvNEGL(EW;Yw$rj4;UF6 z&kHmbo8p<~t9VSl|K?=+gBB*JHzrK2kKf`WG+>9fAv>Het#QE)Q?s=u zHQvv9C#${l9*WBAYq2IG-XTN-7;BAV%6~a#a9pzP4$qNrZ>dTXYE8>5ro)sr_Tz^o z)uQ%?cdz7)dm}~e*|?lvUA95VSM(Jt|93JV_O{#`H&twLJ&DJh$kxa@m$I@$Ke|Fc zDtHY*g|>ytnOKs1n<6Xa(0H`+jJd2Aeaf~YMreocP>jbV+Qq#cVZOtt!M|^8Xs?%w zB4-Fk_nHFdZ~mzufbh@sC_VdM^bs*EyN+r5aq;b8B0~Y|+C?=AnosMa^h_rGq48yv zB5vIM$2{=5iljIZdou2#PQqI=eIP%l9cQUR^{jvG(yylCqfnP}E5=T*Eg>?C&)TeN zaSe2rS`jWXmeO(TYQHXm5>Yfq50zye2?2rUiJKII4(GI;R#RWlALjjXI$*ISe2HO&JguW?pA`f>F( z^Y!`RS9w2NvB~*-n;z#;|(MFG(RD{xzbtZ z!^w#xg&o#|(2p{ik2ys3qj<4)Av5Gzn-t|6y=c_D^j0D*BhNavsVktcZexsDbn7|l zP@?~)^%}j|ob+7Bsj)`V!ichO&X z@2KoC*~#^bAQ-uH@@s{nGL*hnfz*(TeiEA@60=Tdaa3r3F8&ohYV|}0t~o2aBc3T% zjz_%GM5wtsS0Yd&R!=c64|-Fl3`{b$xJXmXIM_B-8*i3lN$x~@emBjhNvIv@#r~?@ z&f(~Ddx@1h(VP{oP6)wA@H8y@x(&J2kbTM35MO-1)}Gfg=220gGg-=H;`tHHM2?>l z?iJ~7&M-@)rnisk9lz8)bt{<&z2Vb3X|k7-mJZ#V+Gd`fVg@NltA4vomGLAfo_Q}> zC3+>*nsUeAj;FL4(VQHa(m$ujzuOfvXstl|t;3?v5~pb0QLBUd!p?`Gze!K*rGqs44F&%A77sT zx|%Mo(cO`ZAM1|i#kgg>(`02uiK66p7%o|{>ltIv)fTj`y8pMnpIJ8uRScqA9wWrXa;jotV55x^E@-L7r0+5Py#u=Jt|z}@-uY#? z>P&r&5nx|v+j#PK<^zg-5wMN>#%N3xM?-hPOdew86VyXj{oj{|`$>n7{4PHptd)b; zXk&ywkpWmSuKKI}Bh$RYutl7BW`1yG>(=J~h;&#seB*xA_lfg{%cqa1^y~qRk9J+m zPr`Jn#!D^S;5=EZr)+U$M$>PJKoPNeySf=#f!eE6l8hIoPf9OXTVeMaWCvBip?2f` zn!N|TbINYm)TxreU)alZV-=a_WTt>w#!WC6YU1?m8q{D_8x zr}gycG46OAczJORMl*sHIF6b`XotjSLEjkM!Y2u4x#KAWeb$^;-$P9f)FjpJ?)ZrQ zxeaW7V5tLh8@ek&)uzR2tkJR0l?dfLddn!@d4(Ak18WQ-iT4*AOw>8twPmaEGmEA|k)t9L2_HD+U z?fz%8s+FlYCx?)b{@EEh&2;OL&o7Nny|>vDFBMCi2NP^5*ai9??AhBneFYKn!*pLi zM*4O#!ONGCJU&-#GD`qU-*h4kcXaGK-){N0y(68L+fKn7Rr zFc>TNd!8WZAa))3-2*`t=&a4Wk8>yl4`AA)Ir3g-a(>(Jm5XwIza055szVyr|DO#a z;J$SXvYxeuPM!Pmgy!>?FOg1@vkMR@u2Ne|rm`HR zOnrHEZq|MN$8}y{O$r?Un=kp1or1d5&QRdd+@?`ZFX?BR=?n`ZmPl`Es2s6!wknbXuTOXK`I&8t_4$0 zd#SnQD_{~toNkT{v#5hgVJqnKIq=nV)74m!-tC}s6YDBC^%F{0|9Rgpo$cn~qK+nN zd||hs7d$!(#)a5|y*L<3Ai`Bd*duO^ss55I`~Ko)B<0yc8xWaBfQFb|5&)KN;y+>9 zc|U~Z*~^!s%hc94dv)lMqeEM6LIVOrI&9~N`^@t|t-+ncV%u~wh{ zki8uqw+`rRQ`XecRnX}A4$Gen{i8i3tJ7>6OgoTBr(w^`O7Rh=ULvzv$V0}!0L<4I z2+ckba$PUCh3KGLMAc1+Y;Vd;nLU`K-KTQO`rlrEoKoo+Z(`{t6J-#)tB6dz+>3+F zzR~|4Ls2-OqZAXvXLz#1o?6CWc!`#(lVE89{rLf_eHYXdm$!}Ss3uthw5h3*q7H)e z)A)0n4I1%0**Xw-jkF$ym2R1F1_y$$hyT#X)VadtE$F>wJ;(;%PAq{zf$^f&r(g#D zySIxhR`{gdmtWhH3I?GSK1#Fa*9mQjxDcA>7yw%{ch|(#g~sLQr$c4zGw0f!&v?RxbOvTQ z3xTWW%{|Xl(V$1!UO8kH9{r{8tA z$ntzDGHrG>3>@iSXmkKlJTKdVm=L-D?G$CPMZg~Vx3!w}3}Q%M-Fo!6nD9&-;^o>hQ$KE`Hu%y zVUv}UT0roDx{pcj21(Wus9g?XpIBfka~hWQ8F?vcw4N2F#6b*XGO+@SX=s*++{D!z zGHLS@NMn`>$3F%i{Mm39Mq{d;cVx*bO(B`)!(__W44L6b`IeGemo;xqX%QqI(nEtU z-H1}yxvl};3OguG(ULyyOm{QMR?r8{{b}>_(=+}LZ%`PUYH}JfigN44sbPQJI>h5- z4}G0daH%jIL|irYq8EAgPkt|WH|6LEIfNL!Iou%r9vQO%6>9eVbf#WjB3&8-N^Jf6bChr^+0(6 zT^MkcaV@<7lJC;XDggG1-Tg?~OMmBWv)ezYh}CAxA+3$`w`k{5p^-5|nf-5NtY!x) zW-WK~$7#N5&;Ea99{YstI?~a*v*Ag%gkCQ1vSC z=G!?2CB9|^(E9BJE%7RDEFfSMg2$y?W@<7$sWi@^^5T#`L$S3XGGcX-&)|vks$lM^ zmNO1uyja6(xMV%=diUJO0(;sJF@kB@BAG2dgq)cmy+b15nl&ZHl99;7vmwsyhA}VE z74xhK*^jDEkFXk9gyGVci{xDet@UfA{5C()JxT=xRhzu2!#e5(K{^<8cZOIN%B1#8 zHW55Rw}HX_l>Geeg92KyZhJKjzR?%3j+We5k_>cFfl;iGsySj=ozxSG!CyFGw+%CuerUqBiU zJ*D?|l41>+laBrB-qg9J50^l7BXsP(QV6z1TmPNcn`sHOQXYJ zD!RufnuK<1NRF$ek7|V&v7mG&Dz6CVVjkPrDQP=IJAI@vito>-vo5 z*~V49y*Y-KF#8lr@q+mohkaKx(_T&Ym$aa-D^|FSK0?(B)UMah$QOq~OSsBhS;;0; zaVixI64AVRxeth)16n)KWd1=dsF@ax-B1o*Epi^Y#f0P7dq5|4>k`*#x~s1j&U95G z!@|c}#WQGo_|2+x?Gy`g`ZFaPwvdu(`Sh>M{_!R?E@%@x!$z1_SeoVZ;T5ImOV*HW z7WuTfUIpCLz04ry)i9SCZ1dy8LxEe)UEWVO3f!|ry|6^@)XL#c&(jmU4WMuXtj*Tfz$jmt48 zwX;TsQKK|rKA{YzQiy=w?U0JUy2Ge+;0O!irsL{=KNhB?@!ozjODionj#nu;;(f;( z36ESqpq$UJqjamUUB>YqJf0(fg?0(L1$6fcmdMDBh%h>XA*BRayt#W9aXcC|dKj+EMp*Z!~> z9uQft-V5r1xg7hhq(|pzWtVF;ZY+cBLxXR#J|F ztkR!x*yg3NCpgnO|CE*3^hx0BL86}$9=$!U8e5)pLFIMW#1(L0xXaPoe6I$|%~df6 zz9K^;PS31>UFs=D1%_PbOIGr-7p-!$V4A1N@Vyfmv3*D*{;JyimLyyi3@4A&UqJ_Q zc>YGnGW$mY)xEwl<}W#9+AvD=xU6js1|XsI;Ds{!#!x&Or>j-YNgD(cP0Uk}!X5iw zos;(sMGz;QC(Q4#%sE4KC`9^`fhsr~t^5dvR?`*mcULAMnxSF-o$LBdZg4lN=BHnZ z4&^(KeBP+ff|151);ztmG?7FYhSiV0bbG*8ADJAaYpE7T1_AD?Ut4%WsR22s8SfaF z`qwn~pE~ttg6|*%@4*GcDpCDM>jvTgwe2s??8!AIA|0=KmO-EEq3JO1fOzLC+HXA! z!^P;JF(8l-vu_to!?r8{Lj_j{i_W3RE1=3mQEo3OUG$_~d5ugu!^~AaU7^s^D7N$& zQ}C2JaLeW}32cSK;#Q6)=i5PL$xXpQ>pIs4rAtQ&Y1+pO>nsxP-vTATDx42f80F%xYuA6$=`ogg%$c;r2qgK$KT4wG zFaH1?0dwUTehi~htT_&Dmpjb4ZH{Gz!z^#P`7+zoTtO3?>yljNY4W>JDVHVHKyQ^; zf>P64TAo0y)1`BjQ@KS*iKNj$sRe58z=oS-d*>%`yYLfQ69)0{(DJZFkf||<3)hph zw~|zxAMRKG^rso{x!KnD&qj2P11W?W-ZD^AYe7x*qv$x*dfag;eBy3EdXxWGl0T(& zP%l$!)Qz3Ca}U~d6j(^}3a%aIqd=c5g+zICSCN_wVAx@6I<3EQB;B8>ZOTb`U$=rG1 zKcD{pYKfcdax8v$QAU)v#~rlz)B!{>}JnjFtsg-h^+l zK2(htd!-$*=dW#uJmuJafuPu>@go$c$^FE_-3-RIh*nfLpM8wpC6`Ng(k1W0LN(Gra@3wiVw+?wBWkn!me^d-b84#g-gw`vxpx3COO z@qU=snue$3!?^2uztrNFbF09Z3<13jtTLY8hW;A=k4=Eoc98rp>Z&Lrl)gtBB4bbl z$wACwPH(s2vP7)f-`-=dg4kITqd}&2OCdRzYuGRTYIY|WdsH-$Tw3_d5*45P?+lIy z$FkCevQ5FjFF$3O)8`NqEfDX3%RqK75U>K6Cg_7LMK;`Nmpj@Ab0+to`&0;}-w=kZ zeWzBZ`^;f3KNs-@A{9LvGds|S)_^(l?+ERK=f-Y>3cw#<`72y`8ibJ*NfY#KTrpUx z>x|x_fhsVBYE!_tt?t`nMBBpi=DYb|@#cT+4M=Zr z3ECw~ve%v;Jn*ke{|vYyTMd}STuWHRHc49wA8{sU^Kv}%Gvi!&gr$rsI}CI#-4(|A zn@jIjNvHaa;ZNGCo0v4S*}2yS%0+46Uf>`HRXZs$`~>XVK%)b0O~vK!;9qTGQmZik znm~ksfqQ_M+|0tLdC>e`jC#2CMfPAvfFRddZ9&f(p%>2Jpz&EDTOf`e4zw2dXO|E) z#G*B}hyITV>0LC~GmYCzLo;drVYE=ma)oHATxS4C6! zD52#(tv4BYSo1<81|IN$fVf(>yWO^(y^*Z5uncVSJ9vtXE~k-h%589^CNCb>vf{*j zIZk6ro2&B>0+v_LEX{_8Z|!KXR?3|$=aElx=KzPc589rpPX&?Z)6btjvjK+af_v-6 zREaT-4iFiom4!2f|4Zk8;`gfJk7hlxA-&qIkNXLz8Rxfft#5D}HaDLR?ro>D{3Gg2 z%Ot3IR{U|3n+yh>AS>=`vzeHx^> zLMpv#p~UX`*n;yvx$ca}`3Pf^f*>Tnoy;Lc+>^*5Y-vw6g16IH41!!0%K6%H8p$CR ztzoUGh_Uj;v5HO>G%R4XA4p5coY(tP8ZaG809D2YDz~x@YufMu;TRpQ;FEmhSz@uZ zIi!N%FH6_&ZIiy;`zOVp;l>4T7P{_blBG!*HOO

4i;z%DAEQp>DVS`uw;j!32t z5CeJ9w*BoxzxCm#3(Bjsz1XMg7rPj}A3P&^XBe97ldV{d|)4n0DOR48JcoG#Bok6fJ zgtqrUZ=UHKu5?R|hsG4nKq!Rsa9R<;;8-HZ*@wuO{9liQkC~)|c}>NJH2h6^bi8TP zZ^+GNdXo{W2NcdeWO}eZbFP+xziV!u1EW(SFqQ!awHPW83GELN$8VGl0Le7uyc7@( zr&h}ly@L&DKsL@;g>VPnLw!aXsKYx9PLukG607$rDjveB(kpM-Xn6p^;g51a-01)b z;VAFz!qkCDVm~LpGxpi7ZAQ`Z3C1$6A21*eLjPm!i->W!N+MON(Kg|c$P~`^>zb1li zsA{?WjJnj$y1Cv=s#liKoDL-BX*e1|RVV=cc7zx3)b?D!uIrwj7k>)cA%aN-STP2W z+F3T}9>)l_=tq_(KYn3IHxP+`rJH9m^LZ~EB~#|}Zm*0t%%EKL$aBO|QLSh1quP1D z??KV{E@`qLNh=DK!ovZrhnmkU|Nl7j>Hh@AI1zmmxqT8|i>Z!%dZvD((X0ToQU#f2 zN1Vr6d@KLOz*MQGa@fNKASdesvGFk;%TI@kyzD@x%X1y%63vSo_2X zt9hqwFC29!_-@q5*G4nLC;#T;Bq{^?_>stjODRYR2>C!~YBdHk2_)jgQ;`8C?6$Jl z4bu!_B!GZ^;64K)c>p#@VUNTp|I46&UDVS^n*ET8o)*{Sp+TI z17`y@t7PL#gij_xS$z zeLTMYy6?|bdY|X}IF8rr^<0Mm+UYfHZ0AWI*o2^v;}FxuA=iSWiddY;!xUDBPZ6F} zg345k@9r$-@U%y=|7fA)aeO~|OKD5GjS;FEv48j>TK)vrB&$68EL&`+Z-^E{oo-Q7zs#{p|&B ztclx~k8ZsNNo~wdp`v;)aM4%c>u7WZqdqSIez9YfX|Y@B)6E;sp-F#=E@eTken{q@ zxy|qMx45PT>UK}=zS>I;oA1Xt!jtylhkmw3;izYh3TZ5KoJV`C#{gq7I%|Kn(D+Qj z$=LMYB^AjZXJ6f1)VUb~+@C-0OAk{OJ|E{WR*n^COG&?y?}mcg$d=;JtCXsLE*d>k zUE>sxag~V!6J!0BXs(m-m_==+p_atYr|ZyUw^EERLZ3qXn6p1CJh1wM9v@{YKAv+a zsW(M$NF~?~_HPy;!mCb}$Br_Cqu_izr&DP3@ zBexyRv12YaQ>lnBI+ydxpgHCG|S{1XA?wwNkY#(%fuUERW zs-*WPMeJQ#Cs>_uL%{n>P46F(khE!gk0PFbcC&Z#{68O>?*DR<-~ZuI|G)arNG}{B z84Vj98c$(%0p_!Q5v`ft;oF-_(myx)mCO<3CD2f7wf!x*z zi#kWFUvtwp>CqZ2vP-x%6J{qw*if?{{byic(3mD_+i?n;R3)VaTyZ^0sUeAy>o%>T z8KT=!dd?V$($sH(xAfP@DdIz3r4Je49)X zgvzQ1uTn}fW+r-)OU=;M9kA_C6^dv?_=M$sAxnGuBJA2;;s7Spalu{ES8n0Kw&ANS zw@+!oDOK7GHD>zzWb}sw<0kX{aqL?t9#TvbPaSJey?X$hzWxM>4Kdd*jRp4^#HF#r zVHxRWOv}PONp9uV)T*HwOgLXl4zj-gdK)nd71kZ872QvsAL+GeEh5;9>+>3s;PRI2 zxl`GF|5cgjpIoA(#5Ij@uXWU7qV)#EWPl=OcQ8<9CzZ}=Av628f>9j2^6i2 zuWG8T>mdGP()%DWrlWZ<(n@K9K=&v`#pdnE*326vjzT6&0^hS;>l7YoWN~5T)2ZEY z^zJE4G|{GUU5|;>&eNto|7Ha;G!-u9G5UF!uUFXP>jqM_=h}J;E2+3}v-fxR=5a@z zdKfyB$0}nYChBmJ*H)Qz9`~mqz4mXT_t%uSi_IaE?)$?onp8&#&T@~TxlTT&q0Pg9 zg4qR(K278TqqB&~$jA^D+VB8w_A&-p0!twwD8t#8+bzWhQ&9Hjo>g7Vxz0I^(fHG^ znQ|!%+u8a41HNk`l?Z)u6OW_1I|q$XU7^*uDJwN@je5r5yPcsKhB#*&4yqv8QHScs zoghvtFW+-92?eGlmpjGQrP=zr$HUoN7;D6OF3mFE@>x58m^+8Rq z=B>z~UI2D4w(-vfU-#juejc+(h(DZZ*%Q3~j78IOfuhd1>Fl!25$#i0NSUru-Q>CX zOG#)1BX^mq?d2E_g$A{-Nbb$dO?N6sbN{0uJoHA|>Y!Vzp4Pc8N$<(I8oPY9h&;uw!VGmsaGXo3i` z2W_X<*a-#+pY=laHZlZeK(kr$I`TeuI`T_<xyhq6PzYuH;W ziYJQ&r+-^Omug;u&qQ}2O9SmeePYJ+L9vQ&fNz@m8#DKR8pr8aOGJqw3c;JX)N{;9 zipM}(f+F{S>K%{&ivqYX3MPIhyQoT)?&V5As7-P&TaDj7hkBufOwG>V1ABq@Y8zxoWX;_;M)@`Xix@lfEVqDGw#i z-=I|QoI>8xOKeU*g9+5DswUHnoU5OGXjd;G%hHjmN3t`~>(o;@$iOe8NkM&W%%i4` z;B;KnbinV_=1{KZPD@9BcyjnB>MN=zG|kDb=;&j)O}bhi=q_fx9%PzZ{~(qx@4Eph zUQ99PN*`{teBi;w%)685?&ZduG00>+>WAWvw3mu0(^%I}(W{aI)^};_S|M^S%)`@2 z@gkFg=0Gd+)|k(u0x5b0r~7hDo`H~L`1v$od>sqfi-wrw!=|DFmueowe&8l4Y4InM z-}q#nK&VpoR{(?Zo8H2os_WE5s-oQKi?X}kRbD-J%V=(H6;6(e88m!n!IY5Ma?Vok z%&+F4zb&Nv=_Oi-x^SfZd2Y`&5Y9g#2?N#H4wg0T@m@QayM>4p28>TnxHn%C`cjGT z394Wz30#~;Y>6T2UYsEC#Erf(JiXcni%1Gly^wi*B{NfuNZcmpp1q_BFC2a33%-z?_TovfqN-ttPY1cJ z4HaHIPV$2i`KsGfTT=YE_)hcIoirh})V;jc8%W|q@b9I(`skvRc?`%0kVt{IZMVBWixF<6{~m}01;Cc=a#_b^LDRlK%P^BKatK(v!yyh0%4(WW~Al_^vzksQu{*izYt zK1jntK$t4WKXs|bLDK4y3=JOo;Q}tTBp@I> z6UYS6xlb6tR`!LeY&Xge@|&#+|3(u@RJ|wkS*zgmOawkapB}5) zqcoDVHv987X5UfiRt61_D2ge{xBY&ktE~$uT1;ISedPHnQmJwZ8Gf&4x-Hv^_s~}n z5!0IsAqUKZ39OZ_^Rkb0$nhHMEjK9nT#0?gN?mow@6DUa+F%mad`9TchM4^YHmb%R zd7z+9%uG*_V3=nlbJ?dN=3@-@+bC9e)`9RPQ#q%&xG+-Nj6smZnB@u%Xxn&Wiks!2 zES*CC_n#_b_k1pT_ZheO($`|c2vp??=4KQ#HkpOv5ou%5$K|QWK57XNH7|`ZK~8p* z^R>1EhM%zCYFE0mXV*n}5SZk^#bF}?Vduiibv#COYq|?|Lp3Ja^^z_bOuq2)eV(^l z1b@iCVW>Kmy@=_7dV^#+#lN#z48%BN_JzBHh-kUj6mj43pgDO8855V_OG-BmJLG z%Mk|U{cWa)#M9}?#HO5*2Q7*{fir$QI$+t{)8>uyA%L)d=m%M?oEaa0Ms2^eJni=& zD19hs>pU71G?^2iKtrNx?*Jf|go8i}CwS9S=b`q}bR-*T5m*HQ6nUd4N=YQ>>5cE^{=Sh~(hvZ~GsC8= z#v~|o_EgN@n%||XRjDzmF9>hXrb-gaJ=L2h)yN=9Ogl~ylM>(pEiTcZ;`j*j^rs-> zWjs?GV%WKBe{W5?b!yj^#6;M3mJ{_40zK~joF(;gGC;Noo^oa`u0fPN(YKf;?umn> z?TCfc|NQHWG2)C79_uXlaW>wd#c|ZlRRwH<8zN1=cOWD*Kdl8nH{@TJ6-Y05^+5Dq zPe0wh?|<-bImpxjXxUp;ZD^R7T4Zlfx~4cJKcm&uHo7&zKGV|mViIs*2YtxP9o&GH z5*;i-dG?)G-muZEA*}>!)DTCjN{X_tm~Dq2p~;{w-efgFNNN%ne*pZI*O;Tk@e4TA z{}cWS{thdwZ;S}y%sYiw*CV{kG|u(Sn;nuZ_1MD`yyU#UR^iR~ThT8Q@n2`xT!HwZQYDmK?||^L$HSQ#-}%5Xc!O&|J^5ywqf0WBFjlwyFPlI1s}E) z{1J3WU#F}~y;NIe@7Npe-knOYWro0*U}6&(g;4DT$wqvxk!E)CKE5%gK#C8`GU{>T z30j_dN2K{fai?+ur*%X6$P_}kRh6hOMMl$JOdTnx!3{t~zL>WYl?FnyYB3|518cd^ zpq^R<*t*GH9G#o}7&70e;b+PYkn0bKXz<_$WvQR86S7>kETDOzzdGUx;i}Ys-Mfpa zt>w0?6fS9~!}4;W4~Q% z&x4uEtqJu7)uV|PhlM6LluMpTYER>+kuvm5xcHGn{-7U}rjYu1mLU>H=e4>pePAPE zKJ%;;1;j(y4v~Ixpxz`#ih%;mLpfz5^_bJ=0HbBGXjU3Xa~0}3dXFh4QOF>& zq%rRu&l}a+!Opj$bN=3~KQ|tMBd!yzN^=&Rbr}4iY9!rJ*|i+5#AN>S=d7@&mShLFzl`1Bf07QNB5nhe)Y6dteEXft`+vdl z+kY&-(%w)yvw2=WI+N3u`qX2Ww=a$MfBby+M{AP?##Ew@Ac#mYU*@+}MACu}?O^}u zxf}gALJoISVWs(nF5;LC|Ni;UuDf~d9g{B{Xs!Rd;}!&CBQbXbE0fuMZA{7S&mg}TK)%6+w&bCOlcx+fJv9EoXJMI1 zOS2jq*DX$+KAP~*AG2=e1St~fyZQJ&K(~IHtgoS|e1Mk&qsbRXOb9b~{|XHd#Ep7# zPX`hD)kMkNSY?mwuE-yIIfUQ;c6>>%&H0Z@Gw__nE-7D4LA$7bG|F)WisrKj_b!|z zDNxeYW3(^;>6e7L4mG}_+;)at>nYaiY~bF?$gW*Z&?m-IV%2Zdo8Fc3)IfuhjHz$@ z+Ag{A7d&%IN_xRruN(TS{U2V5<&gIxjt{Vi1AE)Rudz2KnKfj5;7F~4NlAi1mK`7Xf*&F?Hm zl3bR2u@Tjs*1i&>cO$DYW)V#vL76mrJ%|7KQ&y~wJAqVb3ZXhQS{HUovTVQn*MED{ zkms76=x>VA<~VMLAo-KZW2d(+`NJe^(Jj}ZV(MWQ);unQ*2&(+(AX#+YePH#($sEH z-)bVt3;tqNL#%BKIV1LTL4$v^;_Jb0dln%K{nyX`>kEd;;~ALN)WX+oiaBNs--VoN zn}Wrxi;+Cx7JuydbGRag?5%nnmWA&ml>T-f?$7t=0_2%SG8BOpo5Q6_)PU*Y`gtzL zQPJY`aaARrlyFq(c1=yAcB(QOPx+$++L%*-`d* zHn9}ApT93@Wq6?|0tfizl2m($%@YW!g2Wv9wo9~=9dixv`#CE6M!t>|R(0kUS2D`2f zFf4u8M8<6keA_8RTFZ8c82IK<46CqidXq~flku5n>mVUqXI&QL(>}i^^SR>Pww*nt zW16)um?sd-nrh(P%Kf0G$l;0cv%3A!aeiIsLy`!3IXy|7*dw2v`C!|02Fd>fTRhK@ zaB?Kc82fI@(OiDrbI8>UYjkXm<@Clq<%BTcp;OzsbbjAM+UtR03!2vpG~4UBP1zBD z&lmH2Xi?~WrE0(a01X)rr7$1e&>d)toIxuQGhwW#rNU17D%@T4#BFB!fJ2|oojuLq zv=g`$J53Z{juE3EXVmH)N38-i$$^h^hEVFeZQ2CZIj#8hM%}Ngcd+aNL7rKy*{8?f z^uUSA73(_#@OfyYB%g$le*R~(7ACkJP$ns;j^7Kmc`8Hy9rn1>g$&{7jc4zOA~;Sq zm`2?)uk%pP960ryPRL&iXQ9r`FtyqgUUOOBDkIbIsheOvKS<5Eupr4m)~kWNF4k(qix4T}xz`4bw2uJ%OhHj-bq44_&W@EPxog-Zat9&`a8^Jx;~m zYaUI_vuw%BGbF8_>~~(>tjZ>eeB`GNbZmB~XZJVh`pi@Nh>aQOo@BH2B4_b&-9Qb- zFA2yuZLuf88{9E05l#m;d~kQ~g3g-gM6$eNF;1S|c=g&S0Sk3FFvHY7O`B8nN-L&a z8uK-OIFkNA)P7&Y?ZC&57TIB%^GQ1vquV^lU|w}y-rlkJvft>s+k({}jIAHCnz|&& zdvSZO${PA0rYjv;R*IDD#kq$!yH+yD>aE?bMHH{u(n`xVC`{%gWOdQxiUkq_3b-sY za!F3V#PMw@Xv+@U7{snJY0hyP7+wE_3gLZ>& z-99S@iPx!DNh&>jGO&rwgdy;zm^zVsdBj{5U<;Jl&YT;4dR!pjVN}~X>{q5h;1FMF zVVHc{+T6!65E1^0e(X`<4o-9%LTo|X1$|UO&#dBQ8g*=T89L@rF^Cy9!?}s2yy?{(btc_Kf!CYcnF}y27eV1*CV9hJ0G>9rT+pF$id^NA^qwBPc z$R?`pti)bm=_kk-%>lBcoqe~8$POF#+V(d~JZZoDY5UDXF9oc>$eF@!Hv&;vdJm4D zbfEY3=-qn&)!Ew|^VuM>uRMFs1+m3s-|MqUIjoFY3FR{kyN{GzERl*F?RV~ zIaFJk63b0$(>CEVY)G^CF;|A{OHH>iuRIqt^LR+A!_C9(5ZGe#y3O~ zc&$_8zhOWlZ{P>5Ns#-j_OuW;>9m{x9MnVFiOM`!Px>UGr`Wntto(@d2;P781O}D~ zzE9+SoxR*pVSFw67H|lc5)AQWP68I=$oy@F4p*eAp@?X=_KiG|iPk#)^tg4gMqb$5 ze0{NKh@iocdpGX6$ZpV&uPhZPk2dr-IxlXJ+evUt=euF1J4jfbizDHeY+zGW_lQZ%Q8-o(IjjimklZV7ZeCs5?s0U|NI-zo#1l$5W zcqO7xAMGoS3CVvl`;xMNLc~gJ?n1{S*tCp2_8M2l+_H26G2M9e%o2^?b>RbL$(5Ml z4M&}3YZo{YqVaXehajhEq&;qPODOj}*eOnf*&_zblbCAaAn;9D3QO$Hj?! zskb>-?9)dlM!$faDEpkv7g+O+l)0EY(?QAe0LddAc^Y_GF*H0qu;()UaAZGVW`vq- z|8g<#USBVC;SH)@+rca_Q^cC!e{UySFZgJDe zsG{k2P`bfLne?WJGkOS^1WPej-3dT}%F^BfFva%r$%3s@woL7gIjyAe@Gtb-Z@Y{; zaGa||2NrVkrmQO7`>?lKdL++U(@n&cB9Zd*o6?egv+5ZmcjmyG^}U$8?KvH}Zw$5F z%|f(jQ|=Bfa^nSwiJD5V-adDp()m4;6A0^S%uHUNJpgB~m5sXb+kB#t_xapGJ?XUi z9#ny)KpNM-`^}uWQxq(2?)vmEQw+IXlSo|1J(on?`{vScs1A`(5+N#^zKeDEyVAK+ zQQCI9&|wyI1A~>_t=D$B0-MZ~z z#5CH3IQG-MIp7H)JqUT4_WOglFI}Z-oEHlenp5i%!QpU4$b<;g^=L;#iUWD>5QMfm zLD!UQpl+f_h(miST!;|5pFi&ZvZ%5`M4Jcr)Ug=hQRhhmK|mW|mLz&T8QgHoIC}gB z2)25NDHzRU-5FnrHO*zepYzkdt5?7Ckr;Zv-$IWD-JCWU%CDXp;h$&#)@Ciq_|3g& zE^dD7`~B>Gx8~7+V$`AgG{&Hp0!Uq|E{({T*H2r*VsNYeHZio-eQv(m31die)K+kQNrgJaKZ`mZY&&dG2Y9V9#(B;apk*azGKK>qZ?&9pJ z1rcpY=6ay3T5vHx!6{UYjsjv`=Lml!G#(;dsxuV_zurCYRjZQiXpaN+nfJ4^49 z;QhLu6aumi3Fe~Xv*>zAZm=QV0piIeLD3Lr^_WgPpTE2o>|HNefFOr=*2BD>Yp8$e zMuBEFgm!9>P${K;J{8%Cq$S!tirg<!$a4T&93LqV_?ki9)*?gdgYGahtg{H9sG@t1_c&TA94Gb;Ve-% z)6TfCttL+^`(TjX+0to!))QL*t)zj@sV7NEoEn$|n^ax)pL_ol&zqeYDv%R8FCyUD z*lKen0l6@XBO=J#VS1g?L@qO77qT!T5^v%OZe31$THZ@03&68};rI5Rcr}l^EJuR~ zx|H^Ve0-x|!T$5r(Yf1lY8?Hyl}DrM0xSRdMd_1)znoJ(ZJz3almaeimQ}Gbv7qn< zs*Uf0M4*MM3s&^hCC5XYfp)Ev!kYOOQ0Ezp3d(8mZh}GA#x*ztXHNy3W%cCNXc$MI zLO##{ii)LRf8j}1RWA9e*FP}sXyM#xa`Yor6f@IKkN^T3h(RNh}ZC18Qts5Y14PwBEJE#6J!ktHTgcA07nzY3Z ziKk+rq~O|gzwO~T;MI5(G@$29=|`PjlKU4h4kwednp`$_cI*b45_4Q85-eodH{pt- zsV`{5u@KU^C~W*)v(+L$avd;L&i)iMon|@3-7qjQW52+3qK_ZE0phUY> z<>U(jU*dp#M$+=3?bY$DhUcgk?nNKX#gF}zP|9ba%yL;JLFv6G>qsJrVJ`qVA>>ErwIns&WMr#u$OhhcZ!6uyWRoC1yS* z>SUD26ru%w=sJ257Ry|T@s7s_AXFA&yx*2p{?;s5RN)Z?u$N1f7VT5HizYF2O;m}W zATcLmZvbf3A{FNT7s*?fq2B@%O%gbzY%5cuO~7eRI7ZXNt{8O1 zwHHTwjFGZpRm`G8?kVD{RBs^jXDj0~RN(VC&V7U^r_?9p7A}tQUxnnSH_zVVAkgY3 zp$M}-3l35hE232#F5iynQrN1hE>Ls3&BGsR+XZH~5#K`uVE5UL;5oHu<}M{_Vt}ax ztE|JXY3wkgIWh)pB?>(~mjU_p;*L)VT8R79JG~5H(A`)mdF%2#8oViW00+(!=XLg4 z+cG0FMW02n&p5;o4yrgX_ax6dqiHdf%a&vyn98~1(LXvT0ylW-9$`+7{t<`Ie~0GE z{fZmC0GR75@URw9J#++$`uDm-~@>kWl$%| z@;C#SA*HjH#6YQc-uc_D!;Nx8(0$=Hjq9qqaWh&mvR7tvnNTZx$$*H{q%ksL?m@S> z;!0EB`}{@g4ODDo8Q9M)7dEG_f4^DY_NDlq^VE%rNowdE{Z(*@l~( ze*J7=U~fITEj036!VczV>1bPV2l79GxYm1W*;Ss-dmlU^PMM&g?fcRLR@=Xqa*SNq zoe>IwCTr|p1CJ;`fp~0t$SUsBMrn7;#Z|9rC^*npVgKmX6XctPzhj?a(Fy3^%+c}x zuZBIcXjdIBBKI;^2@yUGbe6+-4i}_*>&~%8)i|VxYziOQd+LUjwfDbG*BNqrcf4Q;K<&PM{@7dX#-`+r+vgy8vO%%}-l5P&4Egj*{ z+GxPdkQ_4#aa|p`Bdi|xd%fX9oBF2@Lf#2)nEDDqAUNGE`T0yVj8B~wOWJ8&NEv~U zp2z<;kVGeaXlcxLx9Q2zuDHZ{!lx6XC^lHukEmI!EtV+$0p$NXY%y1~=={{?bE`ll z5RXoGAc$DX3FC1*`*$z-eBG8ENEUc^1{}L_+o%yn3~SkGEW08^r{U=Q>vL`~LghcRAY*@~M;9WY ztUr^mF!SM#vVrQ>4-Xn30W~KL80d>4!px(;HvEHA(fOGWsnxauf=`8{?iUFofMPF6 zF(IyLeAjGGk72qZ#*W`a%o%;K!CFi z+z~|(`2lm|`x~+G5@WpsZA$s6WiJ2Ma5@h5x`_z-dt(He8YUqD`33oeGd<>M(v zYJ`;2dAjB;Y9&EC(?Fq))e_$kSrz!R|0>Mw-{<#^RSA(ME@R{2-pX;=bB9G1iE zYIs2_D!k6MyW%$HieeOvCqNzu?W%cI{mF@EazlIItE@!6NZ0sKsw5|jM#*HN#2ao( zt6ZK~cxRVd0a*+U>H9n&7~d|+ZxUDdJ$?CzW98w*wE8G1X;fzqKtRvL~WW zf{E&I&M5H@5JD`KODE%LHXT^+*(24V*{P6}xu>%^WaN^%0 zgQy<>fu9NQB`(fU%)s?n3r+p#Thg-ovWh)LNq`QTllQDPZ;APcj5?sixj3Pek!->Wo^Seso*DpwY+Y zAymjM<|LzY+I)bk;faWEA z;^Y{kCCdpeGxgI-86F~=4C%Xk^>ZAbrQ&=-*YuT}xOO_mJMdP|YszVaNz((OYS-BU zU9v!SXVm>Cz!CU%E6s(&3- zF*ZjrGH!p#RoeH}-lSn=F=e}U7P{Hbo&{X2w%=#Z&nqM!&KakKYccWUvDE@-?MC+C zos$p&=-wS7kxaB54C;sdxhJQ%)G$*%W-GoYKIpcZHYued{=@&s4T+i(Kl3B zXwcXm@tv9(;Ko_e)t8vr7A)60@<#pBFk1VGdC{kS%QTfkHQajAHR8tZKe*0hKHq7- zc3ZL66xb^6SLVWJ_sS|k=6cLYYLzSlmmoqtnMb7pX_`Kb;ER4n%d9U70hP{j2 z(w)~LC=jZy!C9xMvs%522ikF^KdUynCN_1E%<|~%O7Gpq4y!~aOx@BL*v#g&+PW`e z+d5T3X*KsF&!Ma4I9-6B(7t-Opqp|r>Fja5taITV=V=B!s|wEb#tsWWFx?;aAHSR-Cix!Z22!G{5K6B^cNlCWTyT}+ z8MEBan%pGpN8=uJWQ(EwE5KHXbcJ4gdIy{R@>kwm55UUIwlb=HdfFbqK{gXrc1GgA z=kpJkRPN%@i2Aq5@vD!;Tgi7JBApdq<2B?dPyZZ$c}Mr&+pSU7V^5P_gR*hk;=~s3 z`>?QB;cCf@2?FR}nhXfRA=L-`a^%QOs~|r%QLak$tvB)!$92t4Tu<$vQ8jR14%sLz zI*;B#+0ZQ$4V8ApT0X3#>ju)PDAwI|$m!PggAC%j5!|Wn~m9JRE_!Q zLY4$-9Pv54^pAe}{ft|j{)q=a)&FzuX6?7kSxUDoAx!|mEkzZ zsiobHkUa2`QpFQdrs@L8=HLm84LTX$SW^|kYPQ#!ZxJ~C{!LIVRGPZNk^dce*_LYF zQ0-IIQC3RRBSd(UCkg!UUdSN)5E(-I%En31t8OK6b*&#%k)`)?W4$Zo>en$}2sA|Q zEf4cKU<`s-_iai$iW^g6UhC+*-52=b^gXY$How$oi=!g|NDvBZRtR(DU4D!9wyUH* z5Z4!(lU!_6jI}n8#Ph8G&r6U+nZ&!&|I0{^$z$5K-c)@dGaCwHDFEOTj{0>kSc6R)%5miw4 zg}JrelmEP;aI4yi6r@Pp2No8mll94iNoVr}I;oy(pTK+22vuA|ln@DY-%wD}@UNmJ ztU{Rf>6Z&@`11h8I5yN+Ig#E5PMeHT_8qEDm&ZFSGd?ez0JI@1xrw2b52|8KpBS1HEAI-?wP&gxm{x!NTqw?3zYYCwYTv91~5#V)< zn-cTs(6pk0Yb$NjMdCRlnTjuNSfbe2N780;uex4Lek7UPk>WM z0}v;yck9Nr6)q_3P2+%o>cBm38a`kZ%oCHdPnbahu%CO@r%C?1DN&h`LHzm_iMgY} zm_xXm7hT~q(>--@z9Xx$qI}?}i7x1K*IugU2-00LL#b&UOa4gzql+;bhxS|?2}H+I z)fdEWlGwGgY0?#UqS7~nVq_#ce$j@Ny)YXso^XWe_M1hlbun*xhPH6SbK|8k0c&VW z30r<)$(aj*6vh2i^T?FF!Sq$owKmC_b8pUptaLX(cA{e#J48`zfK5!_IWk4jK+xx6VGbuqbQxjNU`!9gF~kYhKiqH%dLI^dUV z?DF2_RI0MEzaVECxtJM#x@{X1+indCLEtYGd;d);#gCKQp>k+=v;O|dzl{KL^J!vz z885-ZX%k&hs%1-aNqY)7R(%Dc0VN`OQ;hM4<4z_=$=~LVBA(?PLKSDvlM@cfhsAeP zlDLUEn`FE>^spb+IHSLYOMN@X@8etee(av@pF`B8U(Kfx(3TJ=SOj)2zZ;kD@ojVS zhm$#rk%xg2?Tu(FE0@^FXM60F;(kA5hz~3zk`D9B1;YH1=%WQWqt0R?FCv(3Trh!A zl#ak9Z%Ib1|2_OdV$y^6r4U#0hcsLoxPzXmqZ0ydx&c^phc^}AM_2x*?gQgGQw$^sU)w_K zq1?=V2Q{y*G$$>XD9mTCvaJIh@t5cUB;nTS=1UXi4lP&~-Qb8P$i3_&8H1EZ0CCR1 zL%Ka^h%Bxp=rzE-f4^;=gI=&iD5_{s-jjrAY`r9$DVdp3-+UJVF?s30%V%$PyxDg?4HSGC!8WlJ3I5Z7 zYp=g*Z-zC9M1qovgjBax?{w$Zp*w|e#)$cRFyxuMEo?A=SG&4tUMCIbHlm9xgP?4+yazzBu zL{k%V>Tm_`gFX)$xe4sj_s@&QhDde*=d9c9W{uSR@7QBzI2`*j$wiUmE(Q{ml4Cwg zj@qVzMsFbVExyIX^}6x!q5?cw)1spZ_|2w)04NPWz^nYl^Y8^_jFLecz(~E}9o=&z z1tWRr^8VdnqL(H!MsQ8J4^!*}K!|@_RuDSqMJ3c5#ANeZ%Iv=1qhRNVwaNq0DnK3V zmQ4UDfPNSxeB6dPpmRsWIC8#bA&m2^;YORdJr(t&1z}xVzt`aEMv)8Ko4>Mflhh+m zi|Gb+=@Fc)4Ny>ZgU<}!fBq)Z9r&y)375juJ4xgxBvk^dPdld-L1zghWuEWKe`g_$ zCD$`5ez+ zwL^7II~silsfare-00fZ4lR5mHlQI%r6{{yfT05+ z50@nIX$3NNaU9nU96th#iMZA5QSY{MFP{Stx8xpI;R-o2D<6eC;0gNUPVN{*BOqc4 z7Zj`XXyKwr;BOj)R6iq3fvQSCKBoJ_*~lr$A~ShYDM3pS0YOB*&Uplf2k@I|C%Om^ zXv0e3zUV|0y=cnvdKoPu#dQ9krq8>iSxs4Hi8pj(Xeg2NJ?-*oXuOrcIB-^r5+4T#fW6tLhkH~qqfC^@E z+)AN6QD1C_31GZ{%Z<_7b`r)tCFnsfal6ri)b~VpUe2*9x{aV0=eB$28#Bk+Vh4>{ zmc(v3L?6g_%4&467@l&Za*u1UucD=ik9Mpg*D-^rca?I6E(Buh?1YBS|C}Yq+>`ky zLq`Mpb-;^n)ra^1mPB3Q*X$&!N^2y32HyX?%&2*4Mv*Hk1V&21`tqWW&9h@K=l{$i z<-aFnkhntLb$YHU!cNat5#G7xgV{=^n8}?yM5XvUlEiOQ)svI^pD=PCGyS^xVZS9JBTeol!^JA~7YZlTT2)>y{L-^>;7_X4 zu=39ot7SA4UjiYjo5k6iB%Y?P;NiDYrwe3>$Ki1stpPvvvW^MQd%uB*mb%)xkE>8f zl5TR;FJFrsiXMShv-2d*h{oKg{w3eAEp0E~ey9ZaE2#m{qfXB0`Kz^nO%cx)@U0c2 z)?eQT1b7a5(5SMX1WDfw~MD>%Cvv(gMzFr%81maKW z)5|x_td36aiY2;<31ac4?eaW?E%OZ8ylkgT5{?G#xAl-Z!L`s%kl@o4;#HXC(?+2Z zG9xrJ({m881_+amZzu(I){A%T@S+N~7vIXwXf1rb>^;7Ss54)Xb_M{1x_ zuQrb+(RH7m(`$0BQ2^}H3xXmI9eT5Q9Hdc%7>&c1mSbPvj&G-F#g!!_@Swzric$2s zxI@HWiN|N9Nirkq8K=mCFptOpThy=D2e56GthJA9{HHQbv$V;|{z(1+@7!9V`=DsN zO0^ehOtW$ifYR{*C+(r89Iuvu1=3=0|L;F~_UOvZU;058r$T7)(dT3*kPPo^sSDq( z6V4V288cE8?r-_Mw&>O=$?*^l&Kw}S?Pm&#ZBK0U!z2itUYl0H4-c*{fZW#sVkhda zzJriD?66hG`Kx~)22-JpFNGI9R^G_C6M;YQmI#kvhTDd2HU6bFQ=h%s<-U|#9+uPx z2Gw_+NGOA>rWWAw|B{oL01krWeQzB2UN6FdbzhstEN%L~yEI(p(_~qf1Wh#iLMt4~ zd4<2fjxApyzz7s`un-%YNp)aTy`Wk1^|^i{!@LCwR*P9=tQ>e&sEcQ02Yt~BJ7GM4 z|36xog{NwoWZ-hODXt;wPPfn^@9 zOZqen*^Kf4Y54`uYx6#Pwd(c7i$9;Bh*z354Ix+prwgvyKBr$4fvhXLq=s#~fWL>? zV&-ZOO|LhJGm!lC1s#*_Hju0Zn-BtH$h~2J{ZS9>>$yq}G?;awx+?)EEP4@#D?}oY zx7>Qh0gC4opayJ1oj>utUFl0{AetjkXT=wVl2uJYO+p0GICKo-q-gs@L3dTLw{XMl z{GU&_OG>^Zj1hqPvBFKxW_&|qXdk1oM8p?G5j*~xlV@+GfboffDrn7w@vncShK_m= zjC%RzneT<;7k?F>;Fya{s#s)PKI``Hpi(H6PDjyjXn>pS>i)^7yNim$ruyRo_Lrr9 zzZ%7wFaqW1xb8J9lb?8mN|y6T#rxV~Rrx(#&`zj8o?to-)pXR_5Jqt6iX>`qugcG7 zxa$S(ax1e@HKz?It5Xyie;b!rvopbSMm8y~g;-M}4ZHFL-+WAOqrhg+nk2YiRogGJ&} zAmNQLP{cxYWsI>kv!oOhogZ!ooS0l|QzEFT1zAu9c+DYKCVOBOWPOG-UVwbM^`R!Y z%^~>AO%ZiWffZPJ8oci~>laFV|8%8OYUYOuk*L@h99V z#6XGUpp)nBrC|MjE4*KA<27bs(yvy>Fq?eJ|FvXxy|`akaWJ~xLxaJ3>U!w1BTa2N z2L#vPmzyoS>w>CcjA$|?#-GO;27Sy>9QobQ{MojDkXmqHC=YSC#jf^ z+}5FhNykwO_bS+YDOzutbL$~s* z$bH1O*?a`>%=t?n@Kp*np-ueJZRs|lT@S+-PpFMzqR%6?s~R{*`XQ zwz=J~^Ys~(Vv%FT&MfU3_Vl#tqs*Jch4Zb^ZSYjf&Q5sJkKaG^O1fmn5Y6~y)cUc$ zvUxp!%dVstj{Q6LMRQ2AN`!DE$DDJcExt?SYgJk!As33cM1)!Ej;|Ewx=yFMz?DN` z5PeJA?r@K{j$Th5f>Gcv!T0kc>us~w?u5l3dLS?|zRa*j0lG2yWjVWXx{6q7ynW^6 zD0`NN%)6QWlm!kPF?ML#ztL4;*WO+R(P~k<7)q9nl?ndm_b*}!zP_@<0@D=508_!? zz1x3(Le%I**|RS43#+&Jiu+%Y*}5!b!|2s(7y$@Ul-~G7L2oByr zQhfHLWqIAvVsZ5GHT{R`uVdC_xy2puj_9*#@^@A~=8a?H?t~ii1PX=qWa0kdkXZS` z%4ZFdv9dK?^pWE$OsA+<1R>aZOaBO$_DL*~r3D)Xoh&zriw{cvhvT}spfu|;(T=d9 zPn5e@Nab3l@qwFgxp2ryR(jelXU&-M;oG*Bb5Y#F zXqg57grC*l+T97P^WK;4coQh2;+86VN=01P{SK3; zFW*k7H{$e-M$AkBBhM4RP$3W1TR&JL>7#9xUpx8+o4a#!++4X=yzTCpas(m;KX$xT zKs@1EzP-koU&UItl4&jQNtIb?X{~W*jPODSXo(r3<6~R zaA)O-EbObDV$)+SGh+@c5>Zj3Ilo>}bHH`RH~emJk-xXB4k&-e$XqbHTEhZ3n2gWu zzZJfSnyoy)JVR}{%1EvKg9Gmh9YVXSD_JUcI|!j>q~Q0q>nY>pP!{WXDw`0Fu$@qy zJbuM4dHtCBN~&Re6i(|McNqV@boACO;*e^868}952YLm9LHyR}SiKPK-`u`7isjC( z5T%kcUEKXbsfG80We%Op3HnYe254fV8rS8F4OY_QT)BHbWuT3uKpTS-J32Exmd_A6 z`?yBcfSCd(6jLo4gpREpvv^SWy4PAqgM{9qtqRQ6BJSzNNTh|EwqvHRhz6?z#LeiyoP*t|`O=oY@!u!p{S3%Xa;Q zq$2Ly3=SMNooI93x6~Z}VyPu}=p$+M_?f~J)rXBlds8X~OUs5#6od3GcDj%_%$kc6 zC^UCV?d}-{9f?%$&}V(|_^tT;a!s!0$LH3Cd1-m8We=d&=dJ$L(>yK>Hz`Z9={nG! z8e80RMZMwXl*av0p4Q^;A_=|?Uo+lpPr6PE3cDW+^@*51u(gb&eDxQ=@HFaXKt&}+ zK?N5GehBbBb4k7-o7~q4741Uk6B_AYX8YHEG-=%wZct`M%dV62 z7}5!g;GR)*1LXX0Q#>DHedq`t_v9D;@c5OP-j&Zly>CC<^K+SH@+=6gPtniQ;>wq) zrkr6?W-WT<38;P>xm`0pXRs*q4n>!^5gNa=qp>N;jrr0}7)?M-;)@PfBP&{G;U2|~ zlI!qkNW|lO>%4dV;St!#4cYgcl3wS1hjl3Cp5Hl3DpyXV2IE7;`Lher#8=MkKlYYpS^piE9h^@ zsCGu&RxY~ibD|+pDP@!dGum)vVmuJMQ;g(v^JT##A4TK@t1s)6-!kD(?wyjMfbXKf zSMu+CA$PnCZ}5Ji@Am>|d>)jVKk{4?-1T)g?sySd%j%0;UR5H|rU>h|U_ZL_k^8|T z(?Hlop{L==Pg~qEHI_e9QbS^DLEA~?k6B-eyy(mBy@plLX5idM;C&CaMOCbS{+}7MO8;Olu`Pz;gbb>l;;V&7BM`d6TLe?Y~tnS>5m_y z%R658TOb}p#QS{p$@9k`GjZ*!VCf#S8;SL5a09JYi4$W(1mjAR>q{dShqDyXPXp}C zWYL}6P@~CODq`^J`sCVybPWyhX}EW6oFBbCKX7)?QH8s)B>{St zy6u~nh?Vv2D%uWMKDpi%k(CcfhwUu;IQy2WF^9fOg~g5s$5 zoahm8)8O*%`0mMVC#XZIYvjTb6B=Y-OD;D{E;;b>=f9zuLPwnpCl%O0V#Dk@rz@e1 zgoyt9KaHNOqm=!(J2*_f=?(Sx$;iR>PPGjFpZ~4(%(eLx(tp>ZTfA^P0gE*0|M%Yk zw;b{2O*>(D##Tk>5cfWq|rE|j4RWKFWVv|if>wXYXG+vw*~4 ze;*Cljhs$7T$WQyha7?5!)pa1_forb;y!P?i*c(w3NiuweJEkxn{Y%@PA;)-BsFicYRcmH!)y3MO8 z5hwNN6|-WYpk@>xe#*N}5M+1dZ4y#uzjsqM?x1`l6N8c5)dLmOGV8`tn;mqo0kRw9 z%K1D#OHth%+du4De&a+RwqG$V)Kr$G`xyxY5mBaJ8D5&2zrf6(+nfLAzeaDF9;ZSc zn6>~!Oumelb2MaRWMoV~1NfYV%Tif)d1nN<%ybBvCT$pylZMCt&+~YY z`#OCbeumhrDs%al zd%_^QWCtb&&Nz4s(_!j!1Sn6UNm#l<_%@NSD zZpY>S7II&e+zd%OiP*MJK{qhaZ3I1`Hexlb5|4;x_6c4MFnHdBTyrylUptUkxH1E8 z!0`m%nC@M9@ZLR>v#Gc>=LLLu8dzK8Bte>88t@yQ5+zPU2a*c?xe~5KXf2%_p+T-y z%>aBd4M7n&HTQo!_jdgd4#9y7lSz#@8iEkWq?+{Wv&SY|#Azv)a(3w;M9O zPr$w)0c+~3KNWV-^QA%n=nH8;8bl9fyLI+CHl*gmO=0QK())%)D!wRRCLAfz@Tvm< zU+S`VLJR`%vbayVaAuxG)Sv;r%zfSlkc!@ofF7d>d?jWpj8M;~1RAYI7~~DU%z|)R zF#$xOjtDBydzUvsr9qX|L46yIYmnQmJ>3TK6$IeY5Nf+V0U%E57TtXhYCDB~I>PK! z^*Vix5+c&ri@fKtlmfXA7CZalWuz>~oeT#A;%9I@MS=yaG`gJzx>gj4&2U^dZ4!+2 ze8+a=Gttz%-NG#baf1%fjWxm$Fdzr0R5}n5(}0lG{{730(Z_(z^Qe^b zL@Efv(?L7D7d0yd{F88vU1|xmxp&I58$KNvUZ?*Nnv5Cu`>s#!jXpu7(Go`9 z7qWiKBP+2`4EDz(Ams;M>SCGbu!+_-ptT09ixTR3%qIyfHsj!?hRyC=@vGd8uo zW8~zv0h-3s!#yQ|9e$HF;#X|t)fJ3_6V2o~9$tj?W%uKsC(%S8vm^u8eZ8wwl$p$y zFUoQT+z=gb=Mb)l+nTO`UJXHWpa<*k;%sNng7JVGux3>+^6j802Hu$V{!HYAKd8L* zdLUq9_ZIHR{RhjRo-uEFx@~EFQjdxHrlx`$bO^$UH9>}Jic78mi}p_)gb3|P*f%Pk zYTT48U=&{mg53MCqm)Vs|4e}?*?3$3^%9)!N|ro{sj`LF$Kc?uAl+Gwe=w3&Uts@Jc%O=AC4zD{zlz^X zwhIMZWPL=>!Q=tW?T7OLNFTy6gwOzgKc_ zveRH2?OeZ2NiKZbeg!-aI&wkw@63D9*K@-obq#c8rH}kx4hnDUZ^P;~X8qALS14Tw z9k`*ox_b6g$;Z_)GZQ#D#atsa;JLhnk7MnE)BR^@p7Hh0@w<-4padm@0wSRzS)UB&J!-uZ-1AM%)6x$DNg}zPX#%b zPUAR8&pmJzJ7pe$aU$1URT79 z@ThopzM(R%mJC=*DDzJB5NNee{KJVTI(*IFsO``51qjf5naHzXkM;>C`{UoFt8 zrXrJ#a(H$FyCljli|h?#QKe=+ED~ER275Ty$bWxhGwwTYNYisvMzTk5|CfAH?(d)E z&Z@6&Ne6UFXFL{9E3ky8;tBT!fNVOvso%{$0hlUzNGD0hiNj>$@+u+9Hs`(M-#4q% zJcSDnu7hrAB`v=&{Rk!oSTj}dZnz=n({jB1VWYw<`l5l?IV%b_W-)RKRx0w#O+ug1 zCn-OO*m49MyVleYddM8_PI%jic2JSgDnP^L1Q+y><>Ksd>7-vz!o7-G1ad$Ca)F!B zLH4YmQ|bKI_{0mJZhFE=k`5yp4eb7MNKcgd{r733=`Iyfjs&+d9=vm+fFEi|RVo}K z`vWm=@4uWGjnE7Y%5Bg&-2r|&24)!9Qfr~3aIskH6o^j#{jABrW=+^qe%z14&53OQ z^hq55yo_=}GYnDpEkJKG1rB}f@wPz~z`Ai*`|leWRh7rm7%=xEA6|YMpe?q*2s#hL z$*Z#~{@kGjpUmthV*S6?$Oo4E-Tha-0EaRJtgI&ZqBS~T%QnKHpU7!)>e9(dDa}QH ze6p}|XWE|mmvS7J`2CM@ZR1-91&q@Ch~=W@5~Btw4-?195ZQYaMvTA>zb*~Z4+!C< zicKXeK_k*=xiAqk1r!_&>w>)w7<@$~^FJKIeT`I8X@J@CYps+gk~i7T8v|VQFCOqN zOcqcebg(PzIDKQ=(vn!1qN$L(IorwbBklYT{35seVpW&k6!cIB0HZtG>`S_a_zrHr zA^6f-fM?rvKqDS9KehZTDWp}+h)BSGh`&N|epqOSvn45#{~%-A|}6<#=b z@$$(A@bkp3&?mWnP+gsdC=4w95|dwJAhz7ta^il?2cR0}Vi?x?8v-$a`@gT9RUf=# z(w2Dgk`5|V5;h1Z7f&ev`c9G1w&Jn|kn#VUjPg?g8R`e-#`e{L1wkutu2+C|Aqhxb zu^QRB1HTb8LM8oO55jaC0B-K`aB{-8hHg{bO8?r2{6^>#nr%#4pn$!(R4EeT+KzVE z-FIyOW;7P0ahBD;PbQdNK*XsB*YO@hoT>wQ=0@aE@pV>qf-N)_GNeF#UE<$b=Fv|pG*h*6{5T-(vQ~>_YN72~{kalfQ#pv_y zJ9p}z#UY&RB`uHz8*2!pZ@S1(C?R{LSW*$YE=mW23p~=`S_tWEI6h1Les8^k)#TSe z)+arL6RinQ6~5Sz90;(%&SmYd*_~1d=c<^PSOZW}QA^Aja~Ph0B+HNBS-h2$AE`b z5?$nDD_jV-)Wg82DO=!TI{#NIP+C)vQj>~}uY`YKe_%)>!`dING1>qEOjP_IioX?vBacGB; zMGMkS;zpmKH51D@Sf!TGg7|iiWt}0F$MM$YU)L~Er2emgIR9&`$E>(LN|t;c@tT>ft&da`^}#LS@oyJ9!|{<-In4C(uFGeE6o zHUc`}oLk-U6cl8aV|#d+eIXF~)c;pd)BpNY{@(?z{&)LnP6?GCfNrONg)-X6z#yt0 zDglVRazy!6T-qK@!D*b1M;0jFQ{mj;oc=uv85%P&u>K`@HXIv*J|xF`UZzSSS{FF# z{0hNPqvru#&)kNf&Cv!}<02kaFCSu?i~<0_{x2*9*!|P(+is!qDtS6|fMTQ(<5uu6 zPJ?tt8c679Ae@juW$6t1iN&$p-ubSGv@KAmu`Pha@!5n>8zCn?3QG+lwR}wsC*R25 zqXhi$A`l^YoD*X{5~WUl1VY7h>@(rYxBr-*d%bc90gBcVRB^QBcez)kffQv=fy+#Z zw>dPlPFM+RZ+D!5)R4_u64HnNnZ*{>4Juzj{z{@0oY8gf&vXJnJJ|<)qLI%>7zN{{ zQ2tbTiWDs4f35r(fIRm9g%vn?j1W#s30KgB(0me%$E`L`pCW3=IjeCEX%@?ekKU|{ z-c|zQCgc5f8owXaaa%TO2%)L|h=BRP7jRaqy*vprkI6p3t{XuX6_2MhA`kB)#^44+ zC(z8I9;Okx4);L%km5_H>&pSt#6r=ZM~|`G#lvaNZuD7i#LNU##+n#7Y;t7(YzB8! zyc@@B`hUmyJL;>*=7JjG^q00^;udGB7G)Yvs_P+IAAiDQgw+P7!Hc{HE}Sz1XK*6`QKmkq(3|W7LS>|Q z@#@+w6ShEGr2qWA+>Gz?7=fsP+ySg;11Ngp5Xq4PnOYMdNtKwP%w1mT=;KW`p{mL36*eD`?2To5u*?dEE|pbsp!2^#W%}MWE({K(hqK zHd(;3`bUyg$gZXa(rpVM<)?$)M)Bf*mzKR`mFxFcFf86iPUjmCfLPF*dG?sOu#vD1J^UK0%JasXpa!9>_k zY(hB@Odcekl7Cjj$vf&nVluQAh%+8~*@H(th}@$*VS?&^g}PGr9D2oFuue2;3;aSn zA}T>E0eZr|93JUTm&bis8M8{i(4sDi&cp5SZlQwoF@~DUR;B9Zy3p4ovIm&mQ!rvnLBZm$6hF%&nfh5s zP|tS83EE6ibsVmfskYFZbVPJP=9mr$m|Q~9XxY#t+u=r56~DX2asJAtM)1{|SnR#)KicVSBURzcNXIMM;94udM~9&#bX$DG~5BUt%x$$1%E+AfEj+UzAyrGRkO_;@nXT9C3C}>O9*V!(u)$=0z!3dNXxsI zZhgb=$7yAJqr+G9;^E(Ez|cx=>V^MsN@EQSazvgX(AXJ7!gCGc*@3t8*$VnB2`_3csITI;$(yrYLbRTp0pNLc4IXi@$sbuQ~YVZI~LsqOMTQ;FL650u2Bn6}K#_ zbMx=oeQ_Q9^x38&3jsh_9gGCUm(UsLT6MPwhiO6Am0g_uw+9Q&2Bn(%NlOpC8eEe= zCWZ&fhI4--GGh116*x2t6^zJjPiqk;cO&BS>vJ7dFMVeu?PxRmg?6~%s@s_HbJwnn z8>EpZ{-l_8siSn;fu>~j%vJmK;Ge|pJ-ECCJ~*mLzNTM(rhJAg9;QC3^4g_`VXGIu zb>2Mz|DS^qp;#QK%1e*wUT{_hh_3eNlt;|Zt1iFWtxNA#h-1qQ-A=dY^e_A17sRqH zy&es-{9$R=@`Cs$|9aH-f)fov;~AT7-Rae#wzRp1!wFFRDFz!EPqrCrW_Nx|_~hnx zk&g?HG@}2erFCAFjB-w|RrF7@@S*TU%{CNu^Yg%?^mI2is;Z@T#9g@XyYj|Nmcu$MFBg1EJ7Ej%}g+Kq|f|re*qD1-y$C z_Dis9o6>)9^g&ae^cinUIp}K$$}*n(X{35m;As^swtw#fzwfz9 zS3v&yv*J5{Yc%d)Um<+$wt{)d$pcOK+zVY^d9axX8GDzPyzn1C13DtMD!Z1$%6S#G zym%Xme#SR?pM2`LA9t+X)!)ngOy(CQopSFJ{>O1c{_`^qW+;9CxZTUdAx)H@>*;w5 zEB}B0-*%=(llO5;V6fZcD()SOJBoISeJ1$+w93vJ--QO z--+I&#=-F|q#Fv^4MnVVcU#6}FZSU)q;L7_Ecx5if7j)dtv_I8Q>W@)>S z-y=4eZt-!yard{HM+QR;TqW8u#Xo#z1K!21I=Ea2msg8c%}052#YdKLXIpd~+qlNk z^eh9E!v&^KI5p-v$9d&8H_*3AMSVM&d!BddP0V_K27Grp>!r^uoJ?z?Z~b;ma#z@~ zv7P|4+4-M(nIDxsE41j1@vQ0BR`K1|Z9f;~=hvKvL#n=nFP7MQ!KX+!p13yb+IVu0 z={d%BRGIC2y;P<`3FL!3itaWjr|>@rjH33_kYmvee@pdb%EBXix^Z!0{`WD*5JWPj z0l41#KspsJIS!BmMtZU|fENn!8A&rZ1XV7)9Oj_Eb(u`9^^m$P)jM$cfq-%nP9j0G z)YW7*ByLg0dJN$lG&Dfso6yp<-j&w$(AMdo-i@Y< zcO)A}Lv!v0To_n8wi|xav_?&YU?K2@q+fWt*cH-g6j1^Gavq1gB4|)UracLSJs*>4 zMq_uLf*`<4YfIvSw2tq1jM4b?pwW0^n;(x#`Lt{2dA%3MI}ZImo5)cI7Uve7W*{WX zDf*-FGe9;I#akMg2(SKdY)^NM`_UjJI7)1_a$?X(+CE ze`XA37?f){Q$9CJzwf#n4b28=Cmv{9?^pTpR6(zG^50XhR+ z2qvq!lm(p@N3SRo;Z#w9L%Sf^PtkRta&siA6IG(17h^*r}vNiRn3%K_MxoV4*( z^&T1vu&z2ue=r1tC;U%q_WcyrjW-Dcd^m{5@ZS5LGrTIy`HMYnSV;-`0yg>_syl=j zw~mqTzi}}@1-Pin-i5p2j{0tspjEu$;=<%q&Pi`=4$H{I;ati7q1e5r zB!Oj}cHd2qeFLPaTf`k`+}@KD3F$ZRw|A2a-bncxG~ZR{ctJxAW!DY7d8+h+CM@v> z{6>-k4oM9XuIYY%O1MtA;SO6WO(8~~&cfo-1sRtX=-^9jo`Scde8xSulvBKH>HUSS zCubYq%l^8(8UEoHl9(k5vShV_7)Nn@@S6dOM$fwwl@C&_0FhZ8gdI?QE1sh{UES$L z3m?6YanLZ-M=6!_@zbi9aSuE|5`P^a9PfefyAO(wSY++c>tt+!xh&}*%-9XI)j?OV zDwO&v!8bU4?N3r@dq*VMKCsC$?Qng?+aySj@Boe|GjXzfYK-dQRG)E|;W zO%QDlBlwZC5+5><3ATiL6$>Bet)$k=7&CM4Vv73|?~WWNWbx4rj?N!H z?0OXj_kmfx40~Pjx4=fCwf%4Pfm~f(&$VgIAjciV=BtR+E^+g~js`;i2usc>SA*xZ zs59l&lsIc~>Pa~(ROMjMd64uJ$$2Z4T6KlV+7&qwRGs-rlX*nScCs78Y$+wLG` zTlqa1^(nwLYeJ%&)X(p$sei%-(LoNw%=Qks0gPHNqzU8$bAP{`@%RQe&x++>##00E zrb(y{LsgJ(Ns!%Zg`I?Sr|MW~R5;f>fBH%0vJvppmP?=m93U}h*eGV@q{ah~=j@2m zEZmXc!1I{}L2$wb#m~16T<7;4eBdL^6toC{rQSpzK#xg}SqMLpVh@*fNz052ZRbcT zz%2iK5dF{qP27`k?#1Sm0sN`|u#1V<>kZ24C9U!oAVi~euAeO$O82Z3rKJK8 zFS+{B=x652W|W?SsO#|*6ek*(YRIf7!P07>sPBb0@4qdo1eZst?TA@AKO1QwAax%z*j^C*u-h7%Uve%;OmD@zlW8Pq4kco|eC^EB+w zCXKDGl#6sVDF$9jvGiLpo<{L#0Z<*sS#__qu@|9dYRtNO?FT;SM)#1B_IsHnxnC@< z0P)g0q3c2I8giapWIx=*RWUzs zAbY=WBsfI8rVS>z0|Ar<)+iZ!jb^?rfPJ;cOBR5(KOMfe)HaYd`)q*~e((-D-&XBE zA7qFO+wKPf{^_;*dc#n+P}zz04nJH(ExtXVCG4w$e&e~CJ*$4n6qR{><>2s;z6j0fYNR**b1N)r`=C5$kS{x|yc*Z0?NQ8lsC;=`G8v??( z&UTj30cP)|%~a9@=-LFc^Hd*V#m@d&V3^x=c1;_Ad>)Tjsf8Ed)h?M|@j?!#*6)x# zYYuKmMQ(D%zVIx1AaLZU4XJ3woY*0d%xMEueQv`i&?ptDTC5=YqiSh)C|>=dr$9UK zXV8izks5+GSLWE>Mb-_HOZ^cyJ;}UBEaqLmNKm@&Dh@*7XQL^#o2N)D$xb2aS4~b{ z0Eo8{=>7y*y47A62{jiGQ533GVIPSxj(&a;5(K%`xtq=b-P~8&$UT<4V;W=VWHW%8 ziNSlhrYeLYjqZ-Kq9DQ|C|idmU$8M(=u2~QPWOYk^-Y{29rW3lz|+{{>GiV{UY>E17lJ}gMv(c)b^WM^fmosx#j$* z{jJzawAL+z#evS;0|Hyx(@C4R#vk(gy2@cUOo$a}GjD>JR!j&UrsOdkp-L@P_}w2j z@0|Z6&bis@yvyi?_4O*-olYN7;DlEJfqH#a+q~OO=XF|D|51NlOif`3r4@7jHYqLg zoDXx$cxPnt))C^0)+hs7eB2Q_DJ`hTS>7i`|BehIpB`AGWZ?;eJjUA_8|(TO331T? z9;Tr3>Z$VH?pWqwb(2s`55+E# zYyBs2nG8V>217(|Q)oWC1K2QkrL3i?>F8?AsuyYB{Ca-Vi!?z3$NH=kF0#3)(GGwG z;wJZ*a*NZvszVb43|7fkG6KMarLIg0y1u1vYJ^vuW_yPFb!J6_C8#S1t(R}1n8nk z+o`n~qL=jz_`Tq4V?1fwbct@_5N!Ui;EvPHZixpVcANJ7N=W0|1xsQEZewpVE3e#S z`i-<@Yg+OgnPtJcm!0U53oT)jB28jyXF|h1yuh@-iqTGjhjqGJjyI`KC}j~2n4BB3 zt1~QN=&(HlTd->X0;`Tm@6R=R{Gl@+0?AX>v~rZ_IZ)Jm4p?!`+#>hKrq|gfsX#;c zYt8Z!hP5c;q1E-=Tb+?-c{vI#XTgZI+m1YKp%l5elUOqgZOAN-t!_AnWot>~s@c+2 zg3=`_XTj*DyX5}qwJ^8tGt`ol*NJ=zKHbkR$x(k<4nAf6(Bul|CF{GN1!PXT{lIA$_5vfbDqZ%5` zXmRCQ%a{d)XS-eK<}Q%w^yjqYyZNQigpio#h6EU>Q>W%HD&LcVH!R>S`%{0KspP4nU91Y zN_M~tU}#@*X)tXw4-4B=WETAw-O1I*N8p`|p8y5;QIuQl?mQ1`nX9jh5ZAbmC-O1u zn1S^=Q9H?x%RPd_sc7^r=ni0e)dNDNYSBx=b-S#Cgkdke)Ixbv_Z{Bn8JuPg4^BQg z`{%1mveVBX_SfNS(&jW~lu+b@_rv)4LGOjdUV+8f_m5YLgu{8c%N)2n#l!hvGIs<< zNe9{5uLANyf=i*f1KTpePkjitEM8=}#eRc6uXK88{KU;&iATY(QN4dM<(;D|PjaJJ z|Lq4)WatDPx0FU8mGAxBw-E%NtD}*#ooiPV3#|?w+{*DH-ujH$iF0E4(BIeX?VVh} zz6(1&(OMOzxXpmb#XS=7-&~M|bw7Th1Q}Y86z&@{%9b2)^sRK;%o4Ee}x z_|Wt&qOoMIPihEc9Pe!ntk?hr+2GR^Mhvha)&z^Fa0qT?Fie`l$G^im7q6UIK{-VE zPq?Bj3&o0LSS_vss+f~XVwa6&&AP{c_R@r+K4$Y`Hrc)s2hEF#%yKL1FDfbvl-qy# zq1;cYKQE=AT6%%AZ_M6K*Yt6|WwogFVTaQshid!9`LRG6NMudVt#dGj`W%;G;t<{~ zoO^#5Eq`7aOI;__)q$H6NMBzed)*9b#8U?a=Sq<`ueYK$#TSnANe_aYe7%qu=Sh(1 z4EUKsKTu6GIl2!T>+}nYsDhJHdnc$$j#Qp?*iGLYy5*X*eA)YOg$|L(a{LsN`__MG zEB(RAn#5KdSz3p&1s|cAnUN_gMEMCck5(albitJEnum{bSi8Dx{s9ZZL{dw_t zty(pdK8{QmFtJyGJoBEcHd5Z;8Q4g!6-{aKH9T1@yN{*E9--m zgma|Z!{58OTB&Z2P=o}TMldZVXsv4!W;W0DPjG!AQLn{5QSmC|;oM~rC|55DauO$$@BGxwT?+A#OHG!u^>}*6xp+mK$(1;4N38Cf=AOSi?qr~+E&50J!Bu(0m z`eutUo54hNrhD5~rYv-DwpJyA3vk#k(Ih_j*aU>_VxVv<0&q$9TJ9-f-`vNA)5DrTrL7 z(WjBY_*R!6{qau6M10w9?@Eb|;(i4O!3X;5bh_EQ=A@cy=1TRH^B-+#<(MGGWfb7_Fqv%sB1M^m$^@;W`YFVApy(a0CXXG{z6C%YSz#}9(J2c7X-|GekUa!YH%8{Bnn;; z%S%U5oEB3SxRVOcOyiJv#dLfU3b27><2oumWaTB3|Ah1V*2gP-Te|wj5z>quqjePC ztI{ht!_716#kce7-&oU89?LyXTbicsip@@_-@IdT>jhj;!1xl9trPZGIuV0q8Y$P- zeb7(-!*Xi>&1;P&a^YbasF^}o5=^Zb?!}=Wf94+Lea(m!FT1c0B%KE|Gy)q3bGZC7 ztmZ;p7=u-N3tr8kaeE+@w3Wof+psv_v)DIUTk$Zmt{n&Vq{r5}Joj&*>}pPN|HH^v zB+ywY(Za@M@}{hdLy2S?-O_QiGk%qn-znW+)+MW8B1+px5r@jFAN4;1tmJ$P z?F-g_$>$LkX6gd_SGK~A&z1S2GHhD@A4N?a>?jNMG;LPB`{W!f!(uR?#Oape_K(uIV@ubkO&<#<7>sMW(s`5ngE3H2|k@5kbgKVd>@%#+Z7ogI4tbO_B z4Vc9*;Y>JoD!jD$$II<91S@UOU0XID+m&D>xsM}x%8}x7vphoCh){D{m*;g_EQz!K z`nU&6w%`ddnJ=uSq#c^tm`PCSQQ0;vF#m&L*L~zzQ|y&h9-36eU?Pd45>5=*v5&** zt)Yzq+H~fj2d;fR_hw*1zQsgn^%JzaJ-3~5JXCO_0Hy5`k)X;Xjbl9d2c)Nmm3rC^!pl^3%<-o%3RuU`2<17d`A$wz=CtgNOCGU z>{{+y74LVP&?|16kG+Q+s-pDhH3D zbWfHJd_~)A7#-VU+1DMMVT4fqDPbExyycB;zWCYa&{j>B(L}h+NiJ(dT&@CItepS@ zqR>yihwoTy*dYMsf(ZApCjkW zeuEEm6mBy*2J$DJwH8do`N(MLe)Vq-=Iy!$7HPmB4Z)~im3CsCwQaBkN!}d9eT6s3 zQ{$mwD}wr!Xzm>Z7_td7BHC?WbY z976lYDtuuM^GGMNWG`v6L0FmpcEkS98re%-=2;F?mRwc^;mZ1$l~nqDuwPFK4g>bv z0G=_+0cp1CjrE7OtM%MLr!E-UBm)|(7^V`?=-v5 z%_doFIUmI#%|m|})TC9DRovE0aYZ)O&Z6$^zOd&8MgV#%iNE9vDBWj^0Jh?gb-UQ6N&u&w__4K|u*XYM%`w z%?3_=7<9oz>rlIS3DYUJ!9(>?^@0htaIOdJ*mri-t*m$M1BGN2>jpwiDHdala6*l2 zF;)LW3yETG>YQER9vK8cwW7czrs6?wmTo3+3EF-2j*D?N*YC2xjjChtE4&N1oWDXunO+?1Awsk1#{5sd6=|67oHAtWpfDe zvDT?y;fp{FbgM_vyIUI;qP@VqCFQJTvqi>gzH`KdKP*q2bsz}CYt`4Y+Pu|N-@r`F zJH2c-0|o+zIAKQA{L;jXB}A&YrKL-0RufZ2e8#7tkH{%rfM=%zz>D_=!_qU4DjqUB zWQEdlomDFT2A8ZHC1M*Rm*i#k6>p*GHx!4s6dN>FIUU^Xoudr1y3hD&z~bs!)xC!S zK29tjYEyg;O_<2YYB@9Mr=it#&~$H)ap?yAsQBZQ^nN%dtE_FpGTGXsrcThs`oj%Y zDe*tnAC^)ZV?fgCK;a1X%Kd=kCcoSrWOdW$wBMNkxC6{R-M)IpN>>q4*y)#W79PFt zl6P!Fe-gm`imVKUD7Qsq?Q-)U&B!X;EQ_B2oLL*gIZ>nlSZiU_WKH{c@|CLqiEBlNEQ`n)6to{+ZTEm#s zoGYUsngDnrsb)58<`b;dq;BvbA9xy8+8QV5AmC9sNXM|d=1ym(jEzyR+3)Y~0ffq_ z_cJC4R8h*(Yv?y?v~%gAmw&K<`S7vPgUjzxD32~rUmrK(DK~v*BY`S?^V$`(eZmqu;q#X~ zr*6trH$uN$Thez13+$XVhJLDY;i5qt@k{B#GTJbb|KNKR2zfS>@lb1p zgie|DQ`q8XVbYz5m+_t|Pj;G;%6`9ZtsFUOp4~PM_>%dPm2&-eqx#rY=S2QGAenaG zIzDW3dhR_rW$W_zyeH$$CTsOoZ1kDfO)2pqw?!~saTth$zUin1%M+0l228>Uo=~C& zj=57rf(--NMnlY6u_3kylY9ntRd=4dqSNdIhcK%L(^7#mT3?g6{HGW=BthjOZnZRc zBF4ELX8PwRhover%D%R|nNRu4YicS~hCUuYtHq>qj4}~AK{Lo>I61y_yhkb%bgdQ@ zIw491I8j)y;_JLyRre!g32yy0ncmDTejn^6%RkZ#9Z} z;SGS1#8$;`0AM?2Hgv|?uXcBrih@Szz+9W>ER@F|JDJ(*KPM~_V(Ami(8L}?|5yRU zrZs%t_mwMjMZM5Y@8@N;ZJ^hebA_5Y^TcFNa9nHsppwkvWC0>ePV!KIj}4#%iklDcd(A(cnMF{< zr7i~(svi%J9vN1r$UKZ0V|@vknOS+77VK*bR_*c5N6mHNTqWAyv@AI|g)s20>*_Sf zlkJriUM8!0H?>D7wM#!(<*>oJ4+g$F7YS->@4(!@M@j+1jUpqZIW@KnZ@M%qa*OCbD2il+p&bHkrf45_ z5#$tQrLop!@iKYZho@*E-G1N?@LceLrK^Lsh=tK@);30xYYpE22OgEmO6oI!iPA|D zw>0!J+Fa#tH0gb6Ip^CPr zT~;}`Bv%h%2=n86R#_obKbs)V{!7;iE&OyN)@#)d$mM-WS?%d(xd+bsfx><){(DtQ z=h0F(cdxkrdflM87Y@R&$YHOg`OEp(gJvJB=6EF+!n*)c`f8h+H0xx(=Va^Ga__Dx zR!){iWIvN+HZic+E|X~W-FWI=$b=XGf;Ft`*ix4jLhM8m*ZvUYxg1WHy@--k+%11GAB(Vy!xQ`j-+Cn+biW1;?>lBQPa zp|_AMojB$J-j=?sOIJ21iNr`|i*cSy*=s4WM=eP?sQ=b!UXNEmcquf2dgjjgnI4R1 zfg5*g?#Ed%$usLmsq(g#wsC(C-JAhu()sR>mYZJa!DaHXT&Ghu_JCWZhnlmI$}PA; z>@o=>P;1wO*R|)?%jT(L64QYQjeftd@F+6J)_XYqAE1Tl=o@^>XH?t`F`RL`cwjdn z{K#~N2%$-gbG2Xw`-^0&sO^_^qWT3`8PM1G?~ZbZo9_Fqt@HS{d>*{9`T3&tcjM+i zje=e1YffDgxw7&DU!HdFia-i24V`AB1-$LEW+jz;`0e7NiJQX59~atI~0zT3TMPx z<2412P($gQ@#}IOwzih1XS7YRd}vY^op1Y)5AYTZrAwG8;iLQvd4#$?gKjB>AdPPh zHit@AWmI%Uy2%9~^dfw*x?)dw-JmpAc$g7^cO%U7j+^G0td2O3x@SAS3Syd>fPGdK z1Dsv8ZKl;<^4{gOVAm{A?q9Xl<(8E5H#UMp2vVGUY9T9IGQL=nr&xkZuT^Xg4r6S3 zy-8~GeCJ*tE5=hXr?pdGgNu*3UgM=@eHDd?dMksx3x11qLV4VG{-`MQ;zQQDT!%c=g z&REYiE9_K&%|>?K6KI+*=&mzaOMwPNQ|mlHEP;zak~Pi7mK~$U4^S`#67jJG<;ul* zSY}Iq`NKXC3#C%3=v@F(<1L$@QzZLrSHK-L~X>Vk{6M;yIz$u7blgH}7f> z41S93^T~V32>;G+2om1movuNr!$B$G*3_1mTF^y_bP+#C+BN% zV;nbC=B4I_$o!~pEqrA9*ooc%jjep-JqRi=DpUu!`U|HIlZJ2sW%o72lHM zD~Zl6kW%M3r4c?ZTUN8i0xiKieG&-T&mm~=DTA33)pm_Hx~Yc`bSKA#6YrW`R91rSvJA-!wJBk zB=I@mT@p)X#VQ@NGAd>=9(IfDi0Fg^0n@r-%3I@-&sJjMu`&cq&3T;88bxn%ABEH) zeBZ?|Cuid+JB$K@-S4f5GGZk3Fb;d-8PDXaHTeQ=@7V6x>Wt*UsGbT!ii7rQrM*TR z)>hTKddIE)Dj1G;hfj74a|>)mi9uyfD8=gSLa2SD20*&K=}6)niM=o8X2|@*%lX+Y)IZCe7Khb2d}W`xR=22Pi10 zGdq2aW4KSAiG3+6XYWDvvoG3Wjl4E+x0e1W1dbHa5G4u7!rQ1L&n60pOwS(BFxBezDo@Sa0glDD{TaFs}g8*)#cA;WZhNyRTnV_nK zToxStUdnoq9y599N*;|!+Om^|Q&2ttJE-Ai^MiW4Ir+JvOOazO~6#&%hhfeq72QTQjGW|#(I>kYAjN?$Nb@0 zzRTbif`drZR{_>_29-Gqxo^GTOi>IKt7K~Ghl8y$MxM(kWsW?hT?kmf@L$zTn!C#0 z>vx*)9886Xg22z$u3=rY-`Mnoo9*C&pe(h&wz=YE$Zg~LUPRi7el8{qXI-BC-&0B| z0Xmvkl`(XWgb@@NsZ}?x#T1wK#xLhduFNb&Bi-L6D8$0>+Gc+;-b6SAV9Wd0^nEXf zx#ysJ+8-0WyZQu^gA%MqJ4xXuxtS`nTI!9k5&a7$p2W_TZD;}1N@O0^UO~4?99?50 zrH#(uX|LvyUz4%9U@I|Qk6GN^tQ%`>l%B&bJ^ORI4irGqs>f|HcNWlzH za@|Dp{$@=scAVPl1K)45EWP6$_f}ShL84UI3kC`vx}t4CM^tC+#4T&54X)f!_qzdM z^#&2Vu(0dijPvYMK76EQ5Mzn)0XZg;b{8}mI)W_XeI)5bARW$P=^J4`-}QfpwQS+D zWOA6pC-MAG#P9}WM92S9=3J5dmy=vhgpBw~gCeQfjCX293T&@JUW)ICGd#}m`k_cE zWB_M=P7(^{s5_4nZQhF93vNJ9w~mak#Pa06;_Wp)eLi5SGBnTM?IZFKm(({5{H<58JHIf^33-C}PUTae=fM=y1KK z%KNzecZP` zH?(HkA^aQ)hDQ+uEKIr~MfS27e%J8H3*ps4B3hVo*AGwjnxQcu>k&8Kd%|{S^FClN z@k-+^C()XnDH-Mm$;i{WYj{nQb47SROj_UKn9`51geUhjLUzYpE>)pWQ3a9G zWM>;4Ohzs|Z`2GJ?*k1gLqYmTSm7XO5blAy#LUOvX7WbG%O#&ZHm2oW(2VS||HB35 zjzT$k_?VYcFw`QF#8wfhXzSt|LR@lIED3KDHLf>7z9=O}oFbns>U$V9wu(V`-How@ zz+^M>T4!0p>n5Yjc~u_`?%JkOE?f`d-^{Byyg^pE?(5|C=U&((&@Z%)W54bIv{A)B zJ>gW>+LhpGs4VQ8c6IRPX4W+918W~Kbijc<;ctDLPnB``P~qjaV_Wx^Ay<3y@=ovY z(n%&cyC}9zrc`h`XnhUeY;7*VtM*8&tOi6%5(gpjY^Lpyu%k_C1pmRJmh#ZjPLtSR z3C87p3T3rxj1mB*@loBIXJS3zvO|^?`i9av?nXfePwZ<@=^%leG@9UW8kKhHT zBbf@%xv=r5yzp|c^C@r&1Erm9=QyloiPjvG#Ta7fBWnmE5Q#9i zOuZy!N4VHyNx2P8o9gax0l`}w7(l1AH&`RELElT)R3wSlqe|o{@9Tw(*v+b0Kp8w_ z$$_C3+>vtoqh7FENk2$$bL5B~5HTmmZYsaAxM|B^MG!-WCGQQK@8&)q5wToK!7jd{ zpqZmKMhGg~eOWKpIuQ159qU6fMy$Dlu5sokU@STAqhS&ZZXxTeES)J16Ev#7Qen2- z!6{-zbB5SFcgIZbB%xb7pR+rxTlYeZH8hQ^{=P(i@ciapvniUGM6q$rErzQu4-X4j zcGV$?L5)}9W$khJ<=J1;DqPHv+7@jDF9|zSCIg5pHQuBoq1ZscQd?$Yp`Sh5EkX8* z4Uh8o=r}@_O zeiNSFzd_Lu^_V&Nv^Pa-rNah~O730OE@44_4`MW{*9)ZtU?g-eL!qEX)OriCS~Z2; z>A_P`B1vGE^L$$M?iP!RG3c~M5>vAgJFSq|OsL0W*6`?dN!l6ZoV`BoD z`OOKj9m79cTLzVekC$KF>+Q**W_r0Icot5n6VBemDW|n4w(IlCeJt;*r>?$Czh3EQ4p?iUalk?yElh`o08Vv)jng4{PZyaBr|seopyTZkl3UA@~J zy8Q9RJEP}%)jBDkj)g^&e~zgu1XQ19ahiCHb@W6&bif36R8}m_xa9|j=*&WW zpcE#{6rLOgS49I5)W?&LGZ7Cv85o?96c4f!<7J?-M$uYK_>DXQ-9ywX8!ET-wq@=popzC(`L!D=Bw&C0-De+N`#U=!&uXUFW7z7>F@z-77hxhuc)Qva%9t zKEfmseMP^1w8-w9*t=)ktdGIBME)wt!89+T$DtzLx0=!j=d3}>fRStRRWfA;UQ=fS zz#$0(QO;Sp8nsh7l2rt%VnnpTi`0j8-E6neoP@p(kF)eOgGZWg|7b7M51IfOH63Hi&R#n5#gL$VsO@P&_+OWmh;H=$i-O#_R-! z)i<9B&1%3YZ>&usas93cR#%$ifY67pGPYLh- zeea{Je${>z07`g_i0#1*{ss?+F)Y8&%V122*93L*g|A0YatsNp9n^ufAM_J_hv{7# z0T5R>DtThztUx0u&ys3^x;$V}#yty5Y~XePp$|Z)iGXW)UQxwL>)wr;aQPg2-8BBckaK91W3Sb~Zb*eHeIo z9q3bhYu5yKw1W7@vIEcr^hIo)HDcnehTbMt>NuSCa(30Ro={%yBZ?$y!=k+Pb$tmG2 zOBdD>R?5IU`8Pn0#}x`9a^Po3jCC;Dtyfax;a^||BOYd7ASF@pZ9cH&>;0Vw;ZE^> zaMDetjT#0Ol~;ljQpBFZ585q9XddsHuX|+!#Vl{n|7h*n!=cXGzDi}Q^_E4eC{YR% zg_I4IXVAgOX&gg~C5Ki_MGmoAa;PSY92$qHamq0iCR*#1!;lF@Yt|sgvgI%a!*k#B zJkRz1^Iq>?k3ZzPCYNjG_xpYC`~KXY0~(IBLJM&OS9jPq=S~%^eYLt<84P$>AFN=p zVz666NFm+a;QBalX%(u)GwyY;-!zrMDX3MgQJO83p?Od#vr*!tI!24owiUzVh=CLu zv*SVf2CeDw80rmM(ZgaCNOPQ={2H zNs~PMxqjhU;||ZqK!zTkMD2vYP!nJiHGH7W8LOyOL%$#mxJFOfHTwx@_KhTDR`4#k z&;3zkHo@QnY-Efwi$&PXd}U7}N9=*EN3OtR`QBYhWbHIq$I-#T?PgU6SO;8U*jm)@ zKP)5^?TY#YG#%|jF|`yK%Ju*?V9P9oas zyiMZ6MkQhMDg>b$L%LKxyiFcd^M}UZnNCl9a7(GqO}(d7b%3t?5)BrlAW0V4u?+01oCo2!RGWpGO2=_G z6>vJ2>gVmsfd#inqy&QDGTfD?Nc5O8&ELfs6?Aw%)2POeg8 zkw^gALBeoxJ?_t4Qkmq27kN=C zY%Z+7E$w?c08)IB{1lH4g_#UZkY)~|H1lm7u%b?&%AkAzG2_Q*_^n!DpjC2!zp-6k zIFSCJfb~wJfue#QcbG{=d0hu4F`1XBpsMY+WydxD8hz62jvec3nhd4wFa~y=Kl~YO zWV(N*%38Sue1aiRYYF+u?Z4-RfqrZFYXZ{Y(Sn)EGUU&#YH*B>rM;B^{ zlDAOtBinf=G`|q&G4UjBr8aOVE;99*K9~8AoJ#z0+-^cTWo6U3_`n!3dkkeu9Tk&E z&zd%RMEG&`R@9cro`T6VB?HGKz#fjU_m`3~L)u_&vY5Pq>LI{L53xtV=CPT7g;NE# zjRm&3ul#ps678k8^IbRXTkomvJKVfH5Clhl2a8$2+PmUz=D(JTwZG=PVYT*Mav*q}h6b(V(U! zBk>x)1~V6Dp=`>d8gd-kqcceIyBV^X z`E+aAU0+Ky@+zOqQe}dfZg1IO#7;|x?J@n>>Ipob1q@YK_c1~po~c?AR$v&Y>6?pP zzpcZ3X2JjI*itFLG3T%UT#^Ig!`0wY3fg4N%!OYh4l?9kptR;O2skor-OEJ|(Eu3N z+R4G~XfP!&y4N4@GP2zjYWONz?IHkyfRoG+~E()X2w^*GWT-cTjPEYx|Ns)ILk@j(j@ib(PQp+7%Tett__XV_3__LGo>ZTCZsv&Mxge?4 z2)d-v1%T+kvfrWnWF3>NFG&`r3wUI*KX(5Go(7#7_$XH>{QB{bsH?I9cO)`rfxNBM%`2`$yvq5eoLnHv2+0H zd~*xJ4kTPda_tQA|C5&Lh{?~Iuls`lg@t~$u!ygsJuxE-Uqd#Iv{B?PHKYl_7Vcpe zyklBO;iB{)%%lP{2!~;6hKca7T-*qrtq7ypLY~o#GjZNfcu%IGm5Xh?7E(tA?vFuF ztr>tMRnItG({^>aj1iq1+c}GXQY&oSlD9#w4xfNq=B8)8o+MbdbePWhy~R(3Z@;X;nWKX4XAXvB`x1s#J5YZt0fkreNEO3%zREBn04d8uiV zOEvL&AMU_F8l=-eNqN(I{AEagIrz}sa2=z?@Zrj=)=Y}6X+RTiw*`GmD`&AaFPgV* zX6?8d6Z)gQ+_4>N@)n+}oP;{lj^Fgl@h+YnXJvCj*NNu505?J9S~MhxhAQ&h2|@Pf z=xZT?K3wHX_oUS6PCy86e+~tW7pH^tcbYd)UPqK5L&o$*`U{mBB;7@W6$QDv1J28I zZ#$@hN=4{z+JGL`MYXUR*&mpO&a3PSP)d>JrSTpdp=`eZg*u@qcfwQggB!3@i|`mN zAEnMj^46B{t355w4eV(N0j5k7!AQ zwOibuZa{@jc)kiaKO7)08#=&)E9ZN_)f>uE2Dln&;EQ^bR|SgnN6g;uraNGke^?YV zqe$~+)LyecDj#CV3S1dPWVg40&5a-El?r~T{`EN|%21dMD?8I-*{yGB&~Ni}k;GH* z9956J`D5w1QW}>`Mgn#AA!9;UWyOI$46oz|t-{e|^cwEB)ZL!Y02d~KU!BnjupjFz zw$;2Ex~>U05~(L3deOzf93>7mPW+dsOz{6 zPhWCfQJ^=p)eN}v`wLYY7Ivn+4`G)_wdI@i*G5G^NOy4yH}7N8VR=q)454al|2*_e zl7m~;2!%#GhE&*-I#D&6qH zGQf2#RXj0!z@qw~vYe*#~%K)m6rX$b(f>VJY{R8`6L=QMruc)|km7 z>%Kwz!ztZxT~pJK#dJN#LJ>=GX#S<$L6mmw!9`^Ted`Fz$&`^N@&oCNaBXIgb)?!= zQ-do3M$>^iLe@R1&~;pIx>ogG40~=iDYiB%nfEYBQhg~Pd(OMMMtsQI-*q*(S6pWb zhn|)uFE6>A3&%Q~qI$JJ=&gOPhd_nSTH_#)j)dm+2DH9?peW|V4RLX)cUlH+_`_#m zdM?G{i!JrN@x%%dA~eyC-+k`$fdJB_|09^7P`Z5E#P0U z-PAudymmh{QsZJ^pmj;eRFrUbnGa|6X~ejh@1oX!s$2U7eOPPhE`VJ>92`}RfA!Ez zHb+I>H@&AIM*R~5;GwRJQ@4~_MH(yH;7XyYw^d4I`-Ll~Qn{(N#qK}|I0NB5mQ1ne zsRR*|Zi$P8#z0$<9Ha3CJ(EXZ;sL6=X7-!5Ws=71e_(W2AU)2r5n^I(Qv<*T}2SIgyHS=p8@ z(b#mP(uK(JeL;X2u#n}lD$^qopZ}>GdLB}1SuX)h6>~7~xuxxn-ut|p{Kb)0+(fBm zJ)y&)qnr35W*0%-15zp463I)N7e`l7VIQw1n)7&vX@%MYtt!Cnr&cWrvY>SD5H+!$mIi~!E#V)osYW#IYUPte7 zoeAZCc-e|CvHgVkn`{i;+%C`zXL|}q(JhqJRt8X-;W28n1!k~oH4K2laDF?{;5<}r zctvR$)ChTVNp+KVLct3D-`6P=er2CM$ZmN{*uJWL=LCcr=BY$fA-c`J71v1=1yaJ* zu}8N*{$``Dq;@OQ!qUZP8TGnT^Qb&@#6F@s?BuuZUtL zR{I?&suw~gab-L0bLsuf9kqQPl+&*5b)?MYe^Bc<3K;g?-C8 z2yK0{CEX6(E+~>+*a*`k)FBItabG6p9RT&-%)cUaHA+kdtOZJ?HbYylKVyV>OG^=E|%xw?L!}>#T4kS1UHokGLDyl z^kEAXvbJ~3#}6#cM}#giMz^`#d1cAqnl7pitTq=$hn)XXts3u4i15FMq*xpjc-~53 z)Y$b>YjVK5S$^Xrj^k)bKXTFrOmDid&9>dGT)Ou*L|v|mZihdWnJN$W+#z(nn|KWm zM_TQK*yf=U6MM>|BWK(6GxjYW)VHbDbFZ#*5=hSipAj`%+TNjyK9~rzP{Cga&)_n7 ztInm&ji~9ozas{Uw5BSD4NX~GsyD&IFmwp>JOn|TXik!Bqn0IiWdWe^{ahoQJZG#* z*sOJ;$7H!(m9Ufj%G(~Mu}5-{dW;iU2F&61PF@Ebg|le@67eZl{sy+yIRG6gg77nt zE!mE0b&pP_TCxI}4VF_OY@ZdRyam7qGNG-YmMNz`0aQDtjqMlj96BKMCuD#?u64l1 z3*zfq_N^Fejl@fjF#t=3K#VVGUSR3Q^OgdZH@{l+?f zENzM&&6%9H-g>WLvNQpsSgMz!xI80Do+HJ+5#tu5 zyxz9C3z@Qle2S&Lf;>FPKz0Xj9p>~o!sn*T+x}LDe=_|Br8zIt)3^$fujS+92Ob^@ zTKM}d%qAK~Sc_?rQ&55{xAY-|O{G%O;TZAF0BT(@&3Q0rb5TjfB(^&S%oB^D`q#-W z!UoV@SMgb?2klOq+wZH*03WJqWxMOV178?E(&{E zHW(=MVzR-rl9a4E&*MNB2Iv-Ek`tsNUaHFsv6?lCSur!MDXxtAE%!~47g#y8q1)@p z8Ce`EI_nXDald<{=QK?C4(RUUVT{fp&$2T&+Rvtd-&KmhgE!RmFMy)Yqy7OgX>-b9 zaOR@zk<_Vx`R30~G{zD)6B{~KZ8VAk>mk9{|DVJ7U%?&m8OE>w4SzIK + + + + + + + 2026-08-28T16:03:33.944529 + image/svg+xml + + + Matplotlib v3.11.1, https://matplotlib.org/ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/docs/benchmark-results/history.csv b/docs/benchmark-results/history.csv new file mode 100644 index 000000000000..068043a911d4 --- /dev/null +++ b/docs/benchmark-results/history.csv @@ -0,0 +1,19 @@ +file,benchmark,metric,value,unit,lower_is_better,commit_sha,commit_title,commit_timestamp,collection_timestamp,short_sha +benchmark_2026-08-20_linux-x64.json,ansible,execution_time_s,21.661,seconds,True,075161cd9160dbefff81e8ec4b28a3be4786bfcb,Increase benchmark heap headroom,2026-08-20 01:10:07+00:00,2026-08-20 02:07:10.843078+00:00,075161cd9160 +benchmark_2026-08-20_linux-x64.json,ansible,peak_memory_mb,1110.7,MiB,True,075161cd9160dbefff81e8ec4b28a3be4786bfcb,Increase benchmark heap headroom,2026-08-20 01:10:07+00:00,2026-08-20 02:07:10.843078+00:00,075161cd9160 +benchmark_2026-08-20_linux-x64.json,click,execution_time_s,2.908,seconds,True,075161cd9160dbefff81e8ec4b28a3be4786bfcb,Increase benchmark heap headroom,2026-08-20 01:10:07+00:00,2026-08-20 02:07:10.843078+00:00,075161cd9160 +benchmark_2026-08-20_linux-x64.json,click,peak_memory_mb,361.0,MiB,True,075161cd9160dbefff81e8ec4b28a3be4786bfcb,Increase benchmark heap headroom,2026-08-20 01:10:07+00:00,2026-08-20 02:07:10.843078+00:00,075161cd9160 +benchmark_2026-08-20_linux-x64.json,homeassistant,execution_time_s,158.341,seconds,True,075161cd9160dbefff81e8ec4b28a3be4786bfcb,Increase benchmark heap headroom,2026-08-20 01:10:07+00:00,2026-08-20 02:07:10.843078+00:00,075161cd9160 +benchmark_2026-08-20_linux-x64.json,homeassistant,peak_memory_mb,6192.1,MiB,True,075161cd9160dbefff81e8ec4b28a3be4786bfcb,Increase benchmark heap headroom,2026-08-20 01:10:07+00:00,2026-08-20 02:07:10.843078+00:00,075161cd9160 +benchmark_2026-08-20_linux-x64.json,numpy,execution_time_s,43.435,seconds,True,075161cd9160dbefff81e8ec4b28a3be4786bfcb,Increase benchmark heap headroom,2026-08-20 01:10:07+00:00,2026-08-20 02:07:10.843078+00:00,075161cd9160 +benchmark_2026-08-20_linux-x64.json,numpy,peak_memory_mb,1850.0,MiB,True,075161cd9160dbefff81e8ec4b28a3be4786bfcb,Increase benchmark heap headroom,2026-08-20 01:10:07+00:00,2026-08-20 02:07:10.843078+00:00,075161cd9160 +benchmark_2026-08-20_linux-x64.json,pandas,execution_time_s,989.16,seconds,True,075161cd9160dbefff81e8ec4b28a3be4786bfcb,Increase benchmark heap headroom,2026-08-20 01:10:07+00:00,2026-08-20 02:07:10.843078+00:00,075161cd9160 +benchmark_2026-08-20_linux-x64.json,pandas,peak_memory_mb,4434.0,MiB,True,075161cd9160dbefff81e8ec4b28a3be4786bfcb,Increase benchmark heap headroom,2026-08-20 01:10:07+00:00,2026-08-20 02:07:10.843078+00:00,075161cd9160 +benchmark_2026-08-20_linux-x64.json,pytest,execution_time_s,12.574,seconds,True,075161cd9160dbefff81e8ec4b28a3be4786bfcb,Increase benchmark heap headroom,2026-08-20 01:10:07+00:00,2026-08-20 02:07:10.843078+00:00,075161cd9160 +benchmark_2026-08-20_linux-x64.json,pytest,peak_memory_mb,930.9,MiB,True,075161cd9160dbefff81e8ec4b28a3be4786bfcb,Increase benchmark heap headroom,2026-08-20 01:10:07+00:00,2026-08-20 02:07:10.843078+00:00,075161cd9160 +benchmark_2026-08-20_linux-x64.json,requests,execution_time_s,2.299,seconds,True,075161cd9160dbefff81e8ec4b28a3be4786bfcb,Increase benchmark heap headroom,2026-08-20 01:10:07+00:00,2026-08-20 02:07:10.843078+00:00,075161cd9160 +benchmark_2026-08-20_linux-x64.json,requests,peak_memory_mb,347.2,MiB,True,075161cd9160dbefff81e8ec4b28a3be4786bfcb,Increase benchmark heap headroom,2026-08-20 01:10:07+00:00,2026-08-20 02:07:10.843078+00:00,075161cd9160 +benchmark_2026-08-20_linux-x64.json,torch,execution_time_s,153.779,seconds,True,075161cd9160dbefff81e8ec4b28a3be4786bfcb,Increase benchmark heap headroom,2026-08-20 01:10:07+00:00,2026-08-20 02:07:10.843078+00:00,075161cd9160 +benchmark_2026-08-20_linux-x64.json,torch,peak_memory_mb,4891.9,MiB,True,075161cd9160dbefff81e8ec4b28a3be4786bfcb,Increase benchmark heap headroom,2026-08-20 01:10:07+00:00,2026-08-20 02:07:10.843078+00:00,075161cd9160 +benchmark_2026-08-20_linux-x64.json,transformers,execution_time_s,134.544,seconds,True,075161cd9160dbefff81e8ec4b28a3be4786bfcb,Increase benchmark heap headroom,2026-08-20 01:10:07+00:00,2026-08-20 02:07:10.843078+00:00,075161cd9160 +benchmark_2026-08-20_linux-x64.json,transformers,peak_memory_mb,5130.0,MiB,True,075161cd9160dbefff81e8ec4b28a3be4786bfcb,Increase benchmark heap headroom,2026-08-20 01:10:07+00:00,2026-08-20 02:07:10.843078+00:00,075161cd9160 diff --git a/docs/benchmark-results/index.html b/docs/benchmark-results/index.html new file mode 100644 index 000000000000..921deed61f09 --- /dev/null +++ b/docs/benchmark-results/index.html @@ -0,0 +1,10 @@ + + +Pyright benchmark history

Pyright benchmark history

+

Latest: 075161cd9160dbefff81e8ec4b28a3be4786bfcb Increase benchmark heap headroom

+

Collected 2026-08-20 02:07:10.843078+00:00

Execution time

Execution time trend chart

Peak memory

Peak memory trend chart
+

Latest compared with baseline

PackageMetricBaselineLatestChangeAssessment
ansibleExecution time21.66 seconds21.66 seconds+0.0%Stable
ansiblePeak memory1110.70 MiB1110.70 MiB+0.0%Stable
clickExecution time2.91 seconds2.91 seconds+0.0%Stable
clickPeak memory361.00 MiB361.00 MiB+0.0%Stable
homeassistantExecution time158.34 seconds158.34 seconds+0.0%Stable
homeassistantPeak memory6192.10 MiB6192.10 MiB+0.0%Stable
numpyExecution time43.44 seconds43.44 seconds+0.0%Stable
numpyPeak memory1850.00 MiB1850.00 MiB+0.0%Stable
pandasExecution time989.16 seconds989.16 seconds+0.0%Stable
pandasPeak memory4434.00 MiB4434.00 MiB+0.0%Stable
pytestExecution time12.57 seconds12.57 seconds+0.0%Stable
pytestPeak memory930.90 MiB930.90 MiB+0.0%Stable
requestsExecution time2.30 seconds2.30 seconds+0.0%Stable
requestsPeak memory347.20 MiB347.20 MiB+0.0%Stable
torchExecution time153.78 seconds153.78 seconds+0.0%Stable
torchPeak memory4891.90 MiB4891.90 MiB+0.0%Stable
transformersExecution time134.54 seconds134.54 seconds+0.0%Stable
transformersPeak memory5130.00 MiB5130.00 MiB+0.0%Stable
+
\ No newline at end of file diff --git a/docs/benchmark-results/peak_memory_mb.png b/docs/benchmark-results/peak_memory_mb.png new file mode 100644 index 0000000000000000000000000000000000000000..1a0ec1f1c3d756de79b372a29a257010498568ad GIT binary patch literal 72714 zcmdSBXH-*r_bnU|R6s=qMJ#{{ND)w^Lnw;Uq$5bj15xR{1pb-ArS~FI484n@ zNKkt3Qlx{F&_cOu<8#h={`Zdi;T`Xn_uXSC1d`3(zp~a`bIrK})K%pV(VU_|p-_hu z734HgsD0@u)L!`m``{-^lznLUBJOcAv%QlzKfldC@8Gj@ywC60vv&>7LT#_0=Y&F?U`PJ#QPgDnjoO1k zDay%eyGPIT?|0U2UtL*Hx#r&_`_9kTS8$S5Q~L;|oZNd((*znn(tX|B^LKI?bI);~ zuztpB^xF86f)>MM#cD|Tv{s=ThCv$Ty}|@o9H;+VDse# zV^L3ml|lQv+pRdCUq7PmzhLAq)Gx5?e!^&cavA%pA;wBM{L%+;m#I6MI$0Fh83LyM zy>SKBRs2BbI;B0fd`>NGRRm5xU?}A9!s7r%JN*DvpB4NR?@v`d$nq(q}KA-s`HWTC2CfysyVgV z|8Zi2z55Q@>uJRn7mG@`&b-MoC~CO(BN9`!x%h%K(fpZ1HTuDoD_157bShE81<_(o zrE1&qJ=sQO);;+a;nF_a&+Ja4T9$BOyC-E`x*$V^LJ_ihZOeYBSZ?GwPiito_msM1 zzB(<^utA!R;3GG^*Soq(g&M*!!0mlgJ)jP2Z%w1h8&@_OqC8VJUzsQ7^s5+!C%n?h zHnb43Y|9n4>0vXlv}{hiu~4DDd{gr3Loxc(BJmbXem~>=wB@wz-ErsNARm`W%HFzoG((-Z_KIMo+32otEof^DlRNccjgpP(Vpm<>n}69 zr~hc*p}vt2`qQVa;4?+Xd33r4Dm;1BVkLDdy*wg>jw;;irIX$qYDn=5p^-Ylsm`iW z+g)g5kZD{|nx$VrMC%d>nVKojjqIzbPMkeE`*m)xYVck|^dv>()ZLC!7pqk-f9&BF zCzkDvY5pRguJ;I zX5+xYJ=`diuQ_M#1<#ckha!JE7T2vzVmPlAqlwptv2U+0)R`H3E!+^fGNtltdvmFA zr9oo8IR2Ty2iKXdpL4o;6)ZlRSGr>rBOfqJy5~kol2T-DM(`Q+c^kTRV-zX0BZn&H zE6FpkS8mx?`DryuF1?I8azbTDHBJicP0=}BpTewTL9L!Ki061Q=(VI0C4f=#8yk=H z%&u5&QPxSg_JndZ^WyiUAgL)O4vr22M)|aqS1}CyrMkkWhtEzC`8ZOY(wVPz8hd7? z{F=_Gay1iLkaQgVru|5(vR1Fu*}`)=tEf+GDW&fB6|=9!GOc^*B!>bU-3FZx>*qf> z+9IY9gzGA_i9)XpJz8m1lVY$I$$DDvR%B5%)QTva~ z*okv)yzjKr_-7-qbsl?q+KfYKe5tU3foH^qjom^os;LSX?}>n0XrTJb?`&M8OrQw` z`rpBC8yNhA+lDQGqVz`{N1qmQRVu5(A$yI7&;zE&$+aRteDwtO*_2nTZftsEoJG<3Br=WV(X7dk}A+MJ=rgO>; zM#9NTtr;+P%U1_H`KHIi!-?_o_ZW$ys5`nh-};FVH^1R|beqOm z`^fPxvrgLZ!f34Pa@Te!O1sa(uS`jDswX%t&y~%mVkJwrABoA+V5bUhXK1odcju+muV~V- zp>wd6VMW@HXEi;YLqORvAv)X%_=Ey-7NQB`19=VD9^ zXac#OrH(Yo5z1lQY=?E3=!tWSKaRDG=5YiFF=f4cFELmAxjzS%`)WgsMDMes7mitz zpYvqr2A-8DRb1)LyMOU?yr!5)mfHA@hYVRf>XF4AAKW`1x-~HSY&sYe+b0-#jNw<> zwA8-vYpGqqxN%;UAD3;^%iW!TOB^v*(K|GT8~fM_=&*Czwkq^%YEtXwk=Z{@P3X5_ zhxn}h^yd>zdWoi5^?Jn3eLbU=b0X1c-ry5|$q4nYGq|N6B6B(qj|_g7jX7;&vPW_@ zFJj^3#9A07b~C|n@6Xc1tyZ{LWQ%zG*&;#1Jt!1wujald999+lI7btD0?p7wD>?F` z&fB!i>{9OSD-kB%C1vet^T(HekKt{WzOwOoMufvSPpZTYc+OWSa*!lkrp#ZR5>i@O zhbqx7;=CS!;+r%f}d)k92ME%uWM_)xwZgT7g2-RbbeVh=v_wMhFB0( zWukW`2)**=&>7Lob%mriPY<0gonM{p9e3Ml3%wb1`{Q-`0q$2PXB;+E<6jWalEga; zx6{?nMN|({El5^JKeC_uTo-!LDC4@uJ;h;bF${OcOw`0FN$m22{~9+);&X*QJKKr^ z-eJ7yi>^sGUpZn9Xi3DHHonfjvOXT?bL^!qN8g1Y$zeYmZi8`ZXIPqI+v+(F?JG{S zSm7*=H!RV?x@2`Xhy3bmtsK{<7D%kN7}kDxXpmK-HC4r^P8uURNx!|J*YRF&q;lC} zvqq225nx4{MI4fEB5ubsE4oZ~kR)zaJU_v!KkD{Y>Fc@{!CcX8y0@NPo4fm0&Ui4z z1MZ5H78maB6te{ivr{>h0s$Fr;S*Q5Y12MF^gC)SbAF*~&}Rqj9TVxisvE0;`P^o8 zb9$m9%b=-N4{nNW)gy(rLUE<0T*`Pru2?|IPi+6szWp(w*ZmKdjDIb&rEe{1SxRvr zsxMT8hn78kgi~QF+LDQNqEaspiKsiP1+ZvyV1C}+4=s=7<$QE2SIctbS)EXI7`q z<(z;~%Ef5UTDBA=Jk0}%?ZAs&PRkQXK|Qf(PwVZ?b=xWu`xJG`;!`wsS>bo-A!jc~ z2wBUe`zT5dw>rVP$o{h~1Sz4okSTQOwJBPUL8-jr=o4a+#t<2F4=YXo#?G>Jj&Np$ zzO$&K`Az)wgb-ujMCoWrPQPMoDoaj{H?<^BsN`pEhAUNYWq#L6l%tOLnSe?p$TECW z2;4BS@H0ZkAnBOmBKO@B>89DQYy8m2w9bnu+Vl=oavU~IUM@B@h7hq4P2QKKi8Wdq z*_}e}tZ~xGR+f%_tUwy;1zdP~@G4i+LdLATevovV73-PKyVz|0 ztWPexq2ck<-R$Yx7NL}~tdnCrt2E0+D?=YO730{UcmIjARK>@~0=9y0XY9_4g$?lh zOg_lvqaA27KTt8}iT7UO_~Mc&b5p@~3+*o4A&u+7l~`+ig^?7bxbb+!4MZNgP*W&3L)jTV_Y~t zj)~b1$%HHbBzengQYooI9)x$50gNGfTc1x&pgc2`@@g63F%88! zt!27(f;;*P#V){;uE=@#ovmcG$Ubg&9;=gTSYWdq(Jsx_%e%J%Nu+Gi!KNqw!}%M} zACDPYE^2olCdy3>RFsi58yTn`w~4SDGO6pQbtF(5;tMRB;ve$Volvkh{}z0%Bk#VN zRZDUJ&y`f`Lzogy`k7x*c7ZINX=*GH1@VhEG;Qgd1100JRtSE7d(c>Bq>?=6G=iT@ zn8`%2r19ONKF1gJ*EE^30jiX`a49R%Pi^}#1y~5^_co2@QOLOUIr!huj~{16PpoV1 zi-=7^U?5JNx)SQC(iz?u7Ng!B^5D+-?Z^WwVx3pg2l0W5bKLN<{98L1!6dR~-F1?KKEXTy2TR7*5cb}qQvisAq|yJ|c)$KdelGT6>(J9eRRy$Rofex=QN&u=#TvO9);J#qC#q#G)lClaq>6sZ84%xzUii z-AKk-7N4P-x>C2aJ@kmK^irj1Z5mtCh}ygrYo5a77H{-8tQC~ZZ=^2vwZ6f`L{vY9 zzd-wh!!gDF=O?5ptTKxfv76)QgrqXIMD`}sSaEPd)Z-NHv4I_=|%Z;x=RrMU=N zcQvVGKezK?rV?5ltqZlb&u|#|I#-PUFrv&phCqH8#k8p0L&?bz8-tSMnY`?XA5gyi zI6ey7MbrhZ>u_q{d38vz@v{#2ZM{ooT1yl_sMt4aDQ>Fdw)=Zv^S;sB^;FT9NI( zVw@T!aMO9N%}2hmzx#gN+22BECali0S4)l$c;JvS&E@^lJE`mJN@4c3Ih|hdwe%BW znwRBpdD;T1)mJe%E}7}svQB#5u7wQh(SIvi#~kwLzeUwW`RMI3sBL9FGRSB$UBVA7 z{m?(q4)|$~BSL6$ir+Mu8jDA@2^Y*NoECFTu75+b>TqFNIj55nW&t&{Oy5)c6sj`< z7sfT+Fv<`|YQJk#L|i4!_I6fot@N_hhf!a4`gJS9C|+%2Z}(3jq>@94ZmACg`eH@# zS1gC{Totw(kTj_DDw!_QuE3IVW3Q7IH0sJP#(A~i#k9J4^-3I5Xi*QyQ=dn27#Quv zD>;yJpwd`&!{nZ|*`mRjv^a0a!3chn0cOvc54mwwknKxjFV9!3#=c@ki&qPf6enTQ z`n}1scL(``^-SoIGx)aBp&`qoDB{1TKA~mQhw}NoQJJewu4%Qigb$J!ZoD|r4dfNC zuyBV`1fLB=h$7$^?oT)gyUiIIz)o$?(5eO63ZFpsn;}IaCq+d|?NTx~QYf%!n6b(;mDPdNHo!ey*{J#E(|)M-dBU7b_cbM(T^E|Ebmvll0L zm0Y|o!@@E+En*u@n(LQPq2DXL`TJ1?thJ5`&(&MUA1F>5ruYLZ)$=uo#cuEZ!?#)= zTi-+5{Vd(R{Rld#wU5R~POS3r*~C%NFyx-Q{`5`|Gy2#|+;X%0?ApSJg?^4-tFiAC zlu94`nX#N-7IsU|J?|g__Q!O>;H93lUWZ(2J<*h4d%mF)%1^V;{&Z?W6CoY5$*a?t z!AjhmC_|Ul$8TDxx0qJ@oF`k@+g;Me#b`(W@5As{*p()r7b!!@MDDfUpXqpgwl^(K z#l@sJX9sT!ERoir4(|b4V}xLATe*HOVl3t5n?@<}_gJq*MaGkpP^@mQK{D!r>M7J{ z5U5CNh=8uY-rnzFx*vViX9vdQ6w7Y~+YptwBWjacqcG0}^cPi=-6bKK=fNuK$9Nby zH{a+;yg?f=mc&{eDP(0hBsUlV=p%VHxd`sC_m#-Nga9q6v|?i_Gt0959behxNRV{l z&epuooJ$t&$1|rB6E5_Z2Fg&Z&gBslu*Hr-4IxvLZ_)Y{m(uCZipF$IJHXaj`9#Zq zM_W7W71LD?SyUDjQhdb;V`s&kb3@pbOUZalXf(nj!W2wWa1&4R+{rcF6u_s4M%k9u zQI)hPRUe=2<;f`zv5ddLBo8EFE^U6FQzP0H;^J5SR@dXdHJWZ#<7>JNA6j2LHULx^ zV>GvRdN6spoWheiMjD6kWgYKsXR_(tpTsrzmDuiX$^#c6rufFYZ#_LFWI0Myo$W8{ z4CgbN!X{79Z*VUF2^lkVCBSnH7?)0b_A!jpgcjKmm{Ehdz7nn3^(1N?T#rd_hdaqd zV=|?t_Yl4#XZylbl~$p(URCWctHxL~U#naYk703cdzJyRkiv&Y2?A#z0L~~`WgLts z9N{T&8dKJzk9;{$U=j*F!m3IX zIH^|0fn&74`rJ2-f~w08e=3lU4U*dU1|-Hk*pDiU;u=HOy9Ha*)ZAp#q;MO_#o7I3 zZe_=CG|rO1<0%rc*gfx{nAq1Qp*XFXp-y!j5~!{8&ZZM#1V5Iv^`IqL(NE^ZC=p*z zWtw{H?XbY>Yu$nTXcH`Mwy&g>$}Eg3yk#l4qw_`J4eb*RBBJbg?%R&FX(+ssnG_Ib zN746xmfLEE^ms-MVfE4F>Q)+#NjE@v8ic+rq$1;11~-;7*Cu{i)kMy?rah{#< zb3c@RkO0rI`wD_q9pBF;=?21tv+Mh>xXpBZpx&Uh@EalWY|V4dkWs3GHhq=`g zQMO9eCt|b|v-6`HDyn>4;(L$Acv3xWGnfFDdtudzF#j$yn*gwozq-bRHHWc+Urr0!g2r zUfY_zt8LgL9%!U>!pPEg;FoyAgq6UX)pnV${*KJI8?;kyyBs3No;HG-VyB$;$mEue zEINn+lT)#x?`K1mNB2{-YLeu}3i#(H0;&%LY)|e0`00~gga0eZ+T#E$rsk#xtE!^f z`1SMC2WwZL(xvhctWK=+zZmQOsek@-jiUAEr-vZ>xJCR(rqh9qZTc8{JU$p4cG5PtDE$LrHAn;x!zmY)yQ_rWWtg1@2Rj9D(T|`q#w>}+HqOw?DPbI00>AQ!sN zVsbLk=a-Pi64FXIb}QDvw&aVAZi?nIm)6!*78G-+>0? z6zaP;0X15Xo@lQyYiMB< z8K!Q3EA&WZ_azz{3o`eiWM)BQ;Ti+GiZKq8d>cG@kwYvSc7v|#41}sfQ{`NuEffq}x&h?G+M>m>ql9}fQ`G1P1nl@O(&u2aiq&UzeR2AP@%USkPpv)x( zF8;tx(#La;|5ADXIFukLsF9$C-`Kv&AE~ZQecFy7a;Emz>FC8k75Wp20Uq;OnFGm9 z6{`bL=I8b4KN{bB&T*)mz!Rx`LS=iS<#AoV+u$sAPfEf4$5N9#0OD0OEmg=~wP)Rm z=Y<@?i=_h#Lq(%xt5al%sC7;I>v2=6UbhCMycWJNoroHxW)X9bYy-@1V=?kRIz62lUQ9UalwZ3;WAElT+n`rHa zALP{=H`lx)xH`vh@|gbo^u#t%Wo+N|1hw=Rfpu=0+JZ;lX)4fgcy_BjtTfivtn=JH zKBTT*8@=mrI+a;->P-jeKCd$~6X%6( zuJe67|>oo%)(^%2LvJlo4XL|~^227ybam!p3{^?xD9#PEiBPf>rokOwD z^IUZ8uP8lM?zf4vmVaF+F}@o4tO;V_Q@mnIixGEdYKQ>$p{|7M`$cZjA0sBJhhNjT zSp3BEIYrIi8yQr%vM-i(`t^zA|r zW&8Li2{rD?WNICi$6ov6_2-SSd#)MtC&xM>inL*vX zXIZ#?nA9t~E4lq644HHWy;a{X%y%nnWdWTK%#0Ml*Amq*8%flBId|Q#>cR3D%djuW zJ~f@$M$<}0n3@k6HGv{}VSQ;{q?0PcRz9~0U+U%53!hUMayupQztXaMyfWKcR+k*M z;ownd(|cm4f}pRNH0w#vJe7bqtShRCyQi8t%AB^8G18n|zS3PJXTaKcpLLw~jT8O% zWEK4!JdCn#2X6|s;9#AyPiGoOVVKK>I{e5jT01|D&TtOmrcjXDD}?z9alh7FPoaME z#fGEgN|)Kfxabig+amS>JKyn%pw-Aj%cMjBckcdVW)AK>?|kv{54rI_HGeC#Eb7yB ziqY1((A#crEI;1BDo#%ebd8G`&i_oGpkWFe1%C#_xf_D zqu-a9zIi}Jkvv{-Y;LQ0^n(DtZB=gkqe#D+@Z8-|`(cI+Z0nO@`i$7d` za1OmTkWC8RT}W5?x2(aVvLs zJ$@fNKDkxKR@{sX@^OV{MeXD3ThbAVel!7#e!?Sz*D($_sj_LA*lOZ?lm3KpDqbpOUgGXB-lO>q zFz-D`X93W;Z8^50e5pZ#%hu%3hYA}`<@3B-?B!~_R>Mkrgnh=o3 zW!HdKYTCBom8tS3l_xTW=g(>8464Tt+L!zLZDQ7$b<~|wkC%i(y0<%ktUDlQF1>OXC3F6v zm!-~_+Hzz01IHWml~YVZK{L*~mQ(rqScUf%Nmpb~#-b`s*d%#$ zrCXx&;RcX;n*&w$Kd-Hpr_=dysQGzF?M2%`m9reiOdi3YdGF+YoL}@<`c=|<^VKPK zz0{>C^nT-@+))lm6YM0rCv?tjR%W`rYtuVJCe|qGL>1=Toh#Oo>U8)o&-6;O9l=Hm z9AYb(HmSwJTg$#KRaYA8yub64aI$UiTO5dR_`YoOOnNk2E3u_GZtW_UY+O|>Syf%` zp9Uxb^K~t0TS}ZmW#-}>Z$w8e<w7aRkDj@=AIRduFt*kJ4vXNML9UU^AZ$M%)9Jid{p8q>GRU;f)~wzr7C zZo?Qyx_w%}{Mq!4q+QCsn9+`SY)wx0q0_=$%ag7B zt5}=;Wb9i|-AqfQD0b8=pL#235i-5r3NSu6ko^q;6GCI>mnzSbHV*w-KlBSNS>av8 z#rj&*H`QTa$-5g9!Tz>Ob=u6&dGVx4C-1bN>A_4>${cSy=oH*Cq{TCls_zYao`1J*_c>kExq4(5J5Lu1Ny9urQ9(ffh9%Wgfop}(_t(jRs zc1FUP#NdFQHNcbnIr~G zwzdF)cL;#_uW~9U+ZL_jRr#$af>%=16{qx``3;cq=sJk-)6U z)l5p{SSp=kc&ur{j;^91a6U^QXWq^{v9q~vdiVvu>(<-lXcN!b0=|hvYH5|te#d&= zxz<7Mx>PGc21~Y_TT(_6;%dZ$tqT+7li2q4Hjk{^nL155j5u!NJu^Yl+g(rTSmp?{ zE)AqJR%av4wclR!Ny%yCaMHYcc3@Q2a#2IJfChq8SBlcZxM(iXDDFVJww-oQ{ni z@ym9(0-yihY!3g~Y@1KXf6%xuGqTxI6c@sI4NBifA=FBLn&NSMvf?fJipbFlke(R^ zoEi+U-^$DS9{qx^^xIZwc3q%?Cfy#ATk%MsM4!3+aGew0)?<5pe5AhnnZR>FkM|#6 zox+ec=@?X;9}vV~e(BFfK4_EkZ+O-4nxE=K{wrnv3c+d|&3TLSntKEXHBB}i7YN+T z-iC-7r*<3Q(cNbcFm$Wx1*O0n6yDH(jKL=Os3h+mdC68t- zZC3ARDgOAF@K#W7J`5>B+OWrFEPm*!pWFXd%nBO*htC>b zCDp_w>WW>)>~G7|ZJZxjp7i2f-H2v$i+eDv9Y^d;lPA@g%(upGf1PG=QrX98bv@LI zJ1(tT{noj{yBGno+Qn%1>0t2CBjUq47>;|={*Ij(}X66{VKR|sX> zNdC}}LXbS>V=3}x{OX&f6B8v1Za>z_m>+qZr6e1s$jP3=VGX7=IUYCQr4;)?_FF_( z*DR)Nu|waQ)^$-Wc1CJ8Yx;WiARP zwbR)bv1TgyPt7E-%*)sfP(K-6-GF9D=}5DSUXg8oZn<&(E&A}O8!1+XZ+Wn>$A2qW z(Pb@*->)_lfqs`ur%XI9CB$=HX;z5g`!su3Cnmnp?(+PNI_h5)O(MRjbTaPqCE_et zv}H^kTJ50}GV5l z6*N`Frf&#T!47S`zX1*uf~L3^1YIGvL6AdHzRTC>K%Im_Wwcx*-KJ2B`=+aN5_yeE zh)3iEC|~W2gW*b>8DX~v%00x)kl*+Q{}Iv$b3lgsbqhHS>XYBGT@ga?mp16-mAXvd z11*bkXD#wO8EhS*%a5?NLayr1aqQe9P%{4KIC=w-HU2+*fxk+Kx-Ob^=ffw2pl@z& z4!@|ztjhQ2EWXcL&y0cepzq8W$Q0_YqF7Z9sAKM8M}l6xaxuB68vSjefA_j!z|xD9eoc(lwC4tG{o;6ITr;$Dxy0Rn$Cm*{V!cm;a?hdkjsGpejxujpB|;OY%tPsi zE`^N#(RkvgFoDePTvTb$2npZ)9HKrb8{7KxA=8Ebe5gHBx9sCd8wC{QdG7e;zwZ|% z^Xz|gI{#n4aJ=aXwu?W9+}G+y`)zi%w_MkL-<9xKO#brai^J5zBOtQ=3{1EgXzl^7 zU3u}GhL7<%B5m3|hVm0-+MsH*S)J==s^Ix(vA!XWT}~&9Lw6Ms8K$!yMe=!d`uK3DgFgH0--Zp=tLl?ixU|#NZ_d-$ z_i(Gn`<9_WXQ3W+5pe?S9-_0@Q3Ypb=iPS@Utf^->okb}@6k+0-Ol~**NOk*#eWYQ z>#;xmwNw%(w0qmi}U zzG9IcsN9(1*|j+U6ln;?$*p|yiGw%G2q(#eeeYY0AiddoZ6vU+;+7eQi$E0lqFW$t z|Bmm_dMm9F%q<4{BNrlCNvwv&ac~_VA|ED$hsf;P^q02dxO#aE$Ji`3Z*?YKTyW9(!~@QR|D6o5<5 zovl?wg5x3`=ke=pM~;a!(njoe?X>~xKtwmx2=tP+5b0KcQSOBvs$sAax)h8cHjoHh zyYC%f-d*^tr!}aheX6;F<~kQkVr+xfDv^c#P1JZrJyz1=7^PpS+q^B0&imYQkEK#p zom;nF+6R0dR^c_s}|b2wIKfS9$+WeiA%WLzo8un{LvetJ5mRa^RNS|zTD<&S&mx;(~X#@(br zLSzOV?e}EkTJ-|U+r1@DXE zh=7aC$ZtGDh+$CIv`+(R?LyExG-zf4LFL12_3%|JYw8+b{s4NmUf!aZEjaV3WspXFV*D&>XRPGDWNm9epIy9HP4Rn#gm|^1uLch zn6>;v;jB#9V;wNjM;g3WZtNyA*3%d@-0#j|!B_2~uaKN$j|I1dftU0}9M`j?MR%G$ z5RiSw?BnuO8YHjW)6}a30s946R)wj~92S#zYTX*Kh?WuGSoZVj>0^{>aQhPfI_nN++Da+Xl#BN0)mGhX@yApcr(iJ=n>3Aa2~Md`y4oj z#(<$NOcL-*WIgV8eczu&Z;f_jeLn)1hcZ7j^&{*UzFsT$Chn~ytm-r}$U&v*7Dj6O zZLF=|BYmzpPpZUC1UzpI{J*OqF>bIXe#P6%_ZmK$gZ6RJmTU#^HN+2DdgjK zmd>EDZKQ^f8d>2+q}?iz3Bnzb2vW+-CCXhLD=PR^SOX({hXZmAgr^XVFH|?ubdA;LYMW)y8E|=9-1rwzjucO;)#nFC8ZS4iI={4mpC4R;fXXJmb4E zk9c8vkJ^|)i{5rHB(ww9BEV%-ntz;A{o{OTn*p&C8t;Z5Z-r%~2>8@By9(R(P54>< z@dKe)Pm|)OG*S{?kv%QmnF*HM{S)(za3>s zhuz`^$xo1(-Rn>ACQwsF^@K^(B7y0|jDr4rf6^FZ1o~nk8Hus1OH9JnZ*Wk>^7X!h zQ|hFto{eC-ouYI+Q$4{Cz4+~du=nQL+Cn9h(tbMTd{8!B)JwxEaH*%dY`oJ*dOCeP zC0+~cN!B35U4ixkJ6Ap9gcP{Kn);bVgT0uX20up5n}wYv0Xo}2AATH|ji4q7o$u2F zZ0uZQIena~!mZB0&wOXBfc0Gg$xP`y(zQD+WEqB(mae-cU2x}MT~V}ghGxpJ=M}{G zHIJ2ldGao3LR_YoQ)EIZuG0widoBH#t3)(mD|7v)Tcn`s{DHYP+x6iyZawd>FV88Z zGwF0eh0(tXrMM?yTtQZpGU!j@`9VmBfNkX@(QhlAKW!$Woo#1B)Nv`=&|9?7(nwK+0kN|fpvQ|>l8bV|^^`U>XBIs4uG z>1+L!R;qy8=*xheM}>+-XJ>yQpAmbtBU})wJA&}6JoYwGN7G)Bffa-};ebkcwkv@l z`6``lQsvF}5ooJHHupY2#j_CPTxB^hLR`6_XGl3%kTLbIBJ|CV7zq&wx}c3R`aMJH zG%11}b~#rnFl3~V~$(7a1JTPOG{W&l~~DcX79DTtI&Ik(VDS) zGP)3V5iTiMW$YHiv-6h`)bdda3{;HeM@e3wDScgF*-nH2WCu&krTR=i_8~LErPz#q zd(HPC$v2n~5q8@J#;_MeK<{B!t}NE`U!0W1zhQO)BTwmkyROp1G}F}#02i(hGfQQ{ zv3DjN^nJ=)XAQZo@w)9?*xDJ2XnU{Mde+|$V}c{$>(ed!OAG2FPl1UJBtzf8COYb8 za-d-+#s(QYe)GEH_BE?KBzt5PRG0cblC@#9K!G#>Fxl8bGtrb98}`%>s%HH}(y4 zxXG0odbMH$%#M&vPgvSHKZ5hV_QkRKEC_Tf9R_w^L`;z2To$kGFW^A>G})SFfP@*zmxbJan<61>3szFLbU0RiG82M@FqlUqrr+mT zx20=%hZ+Za6K1-lpihmcKEW&G&A6a#s<50F8TsSQmCq3Mua*HB*S`f|xe(kFy+Vds zOM*fIp`b}B{q+Fx`_W#`P_<%+&Je|Vj_Vp$ z_u=+^BQR+LNtaRfjKO!&PfHVujxvNpS|yqaDYHX|0^&g^lJL$ZnDWeL{4N{m5$*y$|AaAXMp z_kY@CLoKX=4dn8Q0+qng%(;T`_~>cG+_ZZH8)52>`6yKweE91Lp4XtG{nahRh5ek7 zpLu5q;GCfycYDx#?F`M3g()L;nCXCT?tg#Qcjl7vzI5%Lc@RY+mR-g(8>69WyPyep zW&;S=?_nlg+i&ZB5fRk#Ne4G)sdy+a&WQ(W_hJgcn}9xR?<*rl zCdg%GApeO(jGbV7{TaSqZ@cJTuPt@?-xY>3zpp1ClM><)3;0k}N|YVkka>tSY!?V_ z*!>==25Yx>5p;#j>atr6QfD~$pM@EZe9khh_5-bXL2eEss%0e903!ZS4Fzf*kNP+m zhO$4kwF8iNYK>b9Wv{mTMAQJ>Xtz#&k{Lb^vUBtK9q3}S9<;1R-czY%1nx;cASB4F5Om>aGSd&;5zd46(23J zzD8+&;qMVgb?b#6ROd3tfK|MJOFyR7ffUPGaskKqJe4AaZKiA?>(C7UI(h z9}{d40%dlOZ`lNMUnSe(pZnY-97h#&r5X`uH{wwyZv*N0vZ@N8Z8qBwK&jYx=6A$J zkY4v-^gX6u*8Fqrrmm1iLF=skZZ!qmFW=nt2>bDdkS7u2#%_rNTiR{UdHhzCxXU{$ zN|5#mAt7yZ4S8GusC4_>sj?BhO6Tb*M0g5NwihH5R-lP~)%9P0<;!v1F3>EQfzt0q z&hw5BX7Z4ILU|zu14s{>vao~2WDqe0Pk*0N{BGMj2{Lb z!`<&tdlm|yO3D#n`Mr4n6jp%jZ9z|G>TJq9i9+Ghzw0KrpKD#t8C~xjLi*9V(juGdD34xUE~}pB$oFK=VnFE6D^{fN~@V=RMs(oo#rJY zY~v0G*X?&K#X(y~TYt7!5GF+g1Tpo%M=*P4pj{P(6P6=g5}r||VCD*C29?p_zd zo{f1W5P@uJuxusL{^C5c49TW!4Dn3)y-b1FR`M~}PX)wxCH3wbfRKQ!E$GypWeXtae=V459?1b7gK@QRg)*ZbB-1fNun-NJ@v=s2B}R&Az@2@F^z{!YoLLDbKe9O7n;*Z%c_d_*-7W z5q_35AzV!|7qS(`l5*ax2ZHbP~Y zE*RP;0x4lkx1q~PSJhJ&Q1ljjy2-qqSa()1ao&4i}zYJJd|WsiE?A5>K(ue zTIW}1#9A;u=;fyCduHiBJ^F`VWbO6Xt}T$=pO&mXu#?(Z#4H+~x?n(e#-(WL#_>zD zIV%!vsM~-NaT}9qbDkuB&9ALcj#vQ@a(c+Mb%}c{;*HW6duse^&I`&6Ynnq!On1&o zxH>cG5K~*WDjz1captxs-e_rg2GGK#i2M4RXmFtD85%~n1Nw!^Hq^YNBNNpQd_qgP z^a1{VR=2P9ctbe;Ix)V-XSr%fW%7-r#VHeUj-0Z7x8=uF%1n^L?bT(bGSXrl(P@%E zTDr(4gK)YF@iS3K2CPq?9>XYb84ziGD;9}7d}PJ3&E<+p$WMPU8acnNH+}~mmv`hl z0A}Dp9ws5=7sq+-LD@h6u8p1C1*qf8{TGK=q$KMW!RcAMe ziPAg**u1yb??<6Mk%-@jd62;{1dekn#NZ=j-F10_#N(cdQ&DDBMZe#yi<9=*EQQ>D zrlOia|IZkqD9?#=`xl_ol95oxd#6gI+d)5jTk9=C{2SR%fwm(Lp8Fu87+*Vm?l?C_ zETb0RIj)3$hgcrT5Eo{l;JPIwQ(!kJg**vn415-fu3<#FcM;l-;Jq2M1vqU9N?_Xh z1=jNIoU8u;odeGZiIrsq0kqfFlgI^zBLcKJn=wl3?1xpk%9OUEM%yxmh%ABd02qiP zf3bM|mh~e;q_zf)pX*?nc5GwPu04sP#0QYwoKCMopq~lxx#q8q0Q&8EZY(?X>;D?< zBiByc{#jmPJuu5V<1eK$`*dS0OQrqToR0LgH*aV0_u-kmM%N-J?FztjQ#0rT;0(Yo z*%}(m)Ek0SN@ElSfw1qT6@|kY4IwfaM|P0l0TCDP_;}f}T2>!uob)0qIgOAKHHK$I>xW!HeuCUpX}8$$AK}?5&!@Yc!8q9q6K@TmDj%tD zAjXfdgz}q6F<#c}F0VlXNBDX;t=B|?|9jXaD^OyU+N2wr0@rK((RYvSY3Nien31|_ zraM+f0kXJut4*YSRHG)A`a=bfZNLm-2_pflu`VjQZbq4|0orSZAUTGd%i{I2w{<@2>m%& z|7Z8$-;D}pwg1n)@HGc#q3*8J9hzgq^HCreDVv8ZEE`jz&Jy&V|NhPSzi$xrt^6j? zN63T$wX5&!oL@Gd5@AdL9;X&i0Ug*t%C?OG zu;JmQC>!nlUw?s(D(s2UPLjZ-{mUw!9nEesO=1)XEd`{-0d1^`bf}`w`czyRlo1C;skGf0yG@{7j@qiRn^tD3n&&a8l_ng3n-u%u>nfafXY_s zMY;u0I#Q(Q zbUe-dDVGZc9YfZLj(TfZ*C>+?>G6Rp_np8!*}%~Dw@HDw-2+IcfBZ=Nj`gpRf1v4n zXyrY{%~2ga;?$v_S12hvOabgZHWrV8{jdo&f0a`R)7g&qkLzd<;s_$5OaQRRjMzM| z{+Dm$I$h!+g0)+`EQTP&n)>qat@48@+pCuP24=Wm-6=%z#1lmj&Z1MsA%O4G$$G_|Y9t#9eM(vs8>#O+YRUwGugOF9C4 z_>)nEsuAH{en3L(~y)?#qNk)~d?Q?~t@{j=qZlrkMTzU)dfEU2@`v}=|0 znPlwjN56fcbQ8-dq2APRDD{1Qb%PzvaGFtFdYz6-3yP1mAFTs;72gqnl~7uu)8C#Q zzSbs5>ZkSh*@K8J^5@TMu2V{{c_+%&KOEjIX9Yq3dE8Qy>Cx`QJvu3u-f9LY!m7)1 zx4)9=3GwR7@Mq6hM(;PKT>iUvD_fb!oTxSgVFo~0Y=cx$Y~~y?4YqLEr>pfZM_UwG zF4DUv2W^t#W@s@dc4nDj8{3v}Agi#eEU0P#e~WdiXVns#8T!XX{VxCSU#I*3->Glg4S>cMEf&ejPs$oco173mXRiNrXC2wmI$%(mkN zYtPgigwya_nx@_TeU8-`CvDOQ9(6g_uIq6!xLRd!sRYqzWh)A|NP_? zHxyG+*tf9x_f-^$p%wT2nB(33pB1}Qo6E2>prG$Y0ee&Jj^*)I`RjYZAf@rz%0#YF zq7?xqjR%|H0fiW=grpGwRT@5OQywgAIt$8hk50a$3Y^)h>k^?}kha(4W$7Y)Ht9wq zWuKT5^E!amCQ>}0BhCPC(1>1Ki#Hd{YepicsQr0x#lFV2I21_ltWn^cr=g9vkYJ*7 z`ny9!tV{aa0Y#dicNxeR)GMhcMBGFsDZbt%5j!fn%T$suX9Os|De1zy*yS~ZQN?dO zlk8<+=LqeF2}p~98REExx#tof&*#`wmpeLs8bRp9zMXcVfLRnWOk@q4emnA?I$ioz zl+&*J>QOPzOkUj|TNwaeE#?rfUF11qhd8!!|09aaVi0!G#~_J&lZqIl20U70cnxcT zGUh;`-Cnpw+_`xmvwbzkXF1nsRAmC^4+0z}*>;*-B}at9InNUA|F{=6t(>8miNOpM zU&SO8%>j%=AA~oS+#PV@8w0Skq4z^G-+1MTlRgTEJ8Ryp+9FcTm}=2k*3F=y?v)NK zXt43y$7d~0p%4`{5_%XRD5;>N)F89~TvhvGHDeV{z2eCsLKkQh-0#K;O5ZGOk!krI zZG{FRnrPlD!}$9PPh94uKiuH+riC(9Ie*Kl8i{xHea8VBk~BaC6_z{o@>LfRX*#vP z+-2t76*t^5rN_8%I06kn5T``Ie146KnhAT0K)A`(2rH_y(oMtI%he^>ohn8_uvU}4}8)HXyMXtiG923TE zX`CTPML9&Gm>Bq8c8ON-nj8_;Emx)GP>@YajmyEXG4fFHvZ!O~VooC)kBB|r08Q;7 ztnT?PbSgOtB(orKa|l43kBqmF*=0!7&l#M0^H;Z=y1)5aN8h?le_W7oUlj#TZPyBW zUgpzE0^bQ9*_s5gRtw+v2s)45{HKre$pR+TO80i>2PSeSXAql>zz4-l^NWFO5qJ5$ zQeP;YlqKp9m$+4X`=4ZN4VS2Z*hP~o5xf(<*VwM=!sV8Z`Q&6k6X3C+_DbSjotRyP zWrRFQ#i0Fo_a7^84ERm!aLww_W9-%Ky{vmLmu5G2*vn9FzSGhi-+Xj1uB&fCbRjT^ z^|rh4Nd2?1#>SAed1lGKZWM&w=2+4A{EAnOa?*~m{R)4?0zZ^~2L)q|-a+1Hr`-1X zCypTgBTBgDeijOpL>zBG@zm;KAn$7Zlyba{&7A%2Hpd`AZo08tHq-0 z{8L|o|6b?LmlhRPhd!vK-Kt>sW9gaLy9ppc-;r((Sdzs=%n`pNiFea1J?-2d+DB?K zueI--h%xgg!J>i;zs^00^pV-&tv6Q7NIz}1Yw5=>Sb6Z%Ce_IWae?)u+!0!$l;0r7 z$teD&_esPmK6zdd7r(U;W4ptsd$#%yoH3o#{vJ#AzFL;Cg;FlQB5r6)%w|Q{_ni0E z&9&cm$4vSthRz@=tg@UWP=2K~hIK?RFhX>9m3CMCUX@x_129nu+NSTrWgNj6_ zJT>Lt??1uZIVs3b+u#MdAqztSx<$eNbbQ||HKiJLN;V5E?P}7tmfe zCzNz>ul}_A%r{hyvfdqs9L9}jU_001Q9u>d?jS{E2c}IWjW`vQUtXSDIXWaf8(b*i zwQoJ~ll$5a5H#Mk>Xu;C>dBQdcBoGkd&paPdXt_MIe->HoYYu+(N>~1VNgP{f^70x zH(vh$UjeP`zpQgm2ii~v;kmkmw&_f&BZw%X*n+OUuDL?nE*F2!vtb7EqI&G8Z3psm z@ls_(RodSvZMJK;1bAi5^>}eNF){RUE5{E#Y-hez&>zV`EAU;7{Y0s$&X5}&Q+J*% zVkq^`*ycvNX}_r{-L&jTgRCo~`*B_Ua+kg738z+PhAQ&)MiG(cK>>7lZNLBt?ojW& z?N7~gFTqDjwPv8P@RAh3_VB(BeE6DP8~^tsAo{ZVf8#t>PntBN#uqV7TNx8xm@q2f ziaXzBfb#ceRH}$2a(cj0?F@e^M#-dM-82y?S887M^O90Yh2M=)Wz^=_nsHoEG~1;o zNEo593WZyWVYE9>op|jpr;=kh5Ck?lOX@>f zGxaCqB(%=i>)gdP{jJ~x`|lnozdcwNVa%U*lJ&F^>Ie72&1gPwf^V%WknUJDmiaY8 zFFv&d-Fe%@Y^1dY!8!EbZwy>Z{oF)Ls~Vz?9wXIpLKJGDmE+s8`ftsoopj}6F1>3u zOXt0rhlP&IGFO$eekys3_^eZGyG7UFh5g*OsQ>#N*eSuNJ71?>TftAxdhbq7B|9c7 zN>x2Jw(K9&Ce7kU)=STEYQBQ7WgX>OaDY+p59a=d6%~>ips<%H_bG=VF-r^XR znkq_4$N4p)*LAM};!$*UuPJ`Jp~2(1SBzb?;?ku{##sGG<4&>K zx>#$`d57pVtm50Z3CN<%VQ|y@=aa)`rU?FoxbhrlS=9dNB(NDuJl3tdDKgm8w7f~R z)Ga24FmrYpKI_c?lNbRX?msqzoy2(Z`$##pw3&Td2vZa|{6uK0gGb3bs3 z_0E6pu3EtRFbEd{LlEBV97qJGL}7{>Ynk^H(i3_0GmLcZaX zLb|2i7^N5==7Ut)TSDFdX#a?agdp5#v}}(@qrg8@Y~erFgbJ07SK$r9`FaLOW@C;P zN@$;(V5IswXO*ouv{{hLa1Vss!ep`Lu~J>^$Gt zsPR#_FB-^~(n99R&_vXam-gA)ckd!650Hy``Q~jXW#|gs!mcPid2k6Cq!qkNMPvMC z`E4Q&0+1S1s?vj_eRdM~XZC&~fOgx?n8aFU4=FbE>n<{9WBnJCr<;w(*wYu^-SVqeUk|6&e2^LcDLEgXU>_k~0?)T*(< zwbH92q><17Z+`yBOr%thx(eBDqaf`;>-#s{UYXQ+C~#u&&2m=l z&GUwJgnfK$iG&cr?pnbAyRtCTZJ5-KaoIgq(Topm5jVR-JOZ^bin~f7NIcF*7qr?% zkwK!a6?#yp%O}==; zVJ!q$hihDNs~l##N8hcXE50a5PbXO6(1Fu<;D}JiridPh<+BHt&@H`Dz)R0%nXjrp zLFYevAU>w`v!9uFY~_xTy$9q!pOWT!R44!|$xPTI(*k%VCW3Ky3yCXi(m3h@gAy@5 zjEw1(_+9I5#U(m@f3@6HMn_yWNboJ{UOmD+;eF0#GMG+oMx8v^Ih{Aw*O;3TQX-H+HD9?{DJ337jKGCRjVu6xeUO$|_sV-oH9e;1v%2aE7*=TF*<-)_WoYjY)4YR-S(c*mG7Whf;OSptLNAS9|a)a9Vpi;SM zJlPXWa?%ixvM7lvru%W+C0mMn3Xn0Jw;;23sZ~H)y=80gI=w!kNi)sqKRg3IQ`40) zK~q-XPqSS80^f6I%O+;Fd3fJn{^ifE%eY9cO+$~?{7utWdn@-pn$`U9iuX?Wgidi5 zHkL?(*cD$2QY zS^_(G&qv+wv&|6p^pmNY48FOr$+o8`|0f;dTTn2Ux9&P03n!o6p($S>A^&xZWwZTJ zwFzMX_68xx>?&Jqflr+W9@&r1(IW4oE?`rEZ#7t_VO~qVoT4!$T#rA92a-J%1ww>~ zO8ddk#y;BPiaDztO&zYj5sSSv-E1IJsgONCENYX^)ScEoD_;e!Gt}7={*@xE4@-g$ zL`j`msB1dY8uxcC2t)$36UHNF*qKi&dgg&035Fvv+x-b0_lMM)&-rzYWDKCssRuCG#8jO9VV0mY8Qdy zznAilAP_G#8u-KoBGx?on0dnwVTt#>Z@%|Xng6Ivk8P%sI>)t7LQVe4Ct=^>fR_TDNU9QVqa`!D|0kauOx&dgtUD z$=0#rE{7wZ+_N<1({{P>G4Y(iJ~Y|V9$tww@To+wh1lw5*eG;SnP#B#jOra%R!_z* zPYbYIOYPfj2g5nZA(eg?SN6nR5m5GXWG4){$nZ%1TuzkDKNanN@qmO0_AcA}kPi(g z{}7TMt+rmibg5ZTS^c{b%m!kJLnOGQCfhAo&d%fbHzANS+&M?YKpDp*>JWv6?AZo6 zb(Ku>FFQ*~Tw6|0Y@8;0VC?begIVcLxDnH$VovL!3bP+@=szI1^9cQvvx4@%n6Cld*VY{S?n%qI>?4ZZ3g7o@2{xj4$f&9r!Il$QV0xMnHAiMNG@`)mgywr7yfApYXI$bw+ zeDcG;zm4wy$|C!(@Qv>O`2%#-iP{CzY=|HfJdFDvB^%29f5{>Kssdnc0?HT8DB(Dz zn031*4j!lZoUjJ*v;ALpDIJsmn*!S^=;qgO&;R&SEK@fIdEs!2dbczO#O3zz* znH#kf(wF!5VlQ+bdjbN9w4rx)g_<`FvqJemG#MbVD9NFO`qxHXzO=UhHnrISm#~=; zFb3z5nyW=urV6mEA0L^9K{W#Bx_!_>*Nndp28E1f(~np!d>B=rRu zgGFjWq7gSyJ&rJi)o;Ck_t=`C1}0z%$cXs&PsCeI%ux0mMs2lohO=mXw@~C~KU22q zN@tW6g^a~ue&J*J?KwErqgoY*e766375*#XoLER}x>PRkD273qZ>SD?hyw(J?Sl@e z)3l`Xi3}8}(^tr}E&jhJfL>}OYG!HbE0e~c9%xDfZNSj%uqW~Az;U^ zH@FF|82&FA*M}}mQ1|dL-q*J4U_)X11W~*12wwTlumNbFiB`6ibaJcPfS0pMXEpJx zpp&?*L!&m9`7;4(Ukof z`wiW;G}!+UZE~{XetN12U&DsI<*DZK`tq0Hm$stk(-VLubDe$3S%nKl%**Z!=%7hr zzFPe1hO}2Nf)dYJJej8O3U&B5p+@2)QsGHFX3>;#@dZby{%qX)m!*npd&Ll*&Lj?K zuYe`Raf#vWJ-=u}h(VfUdqF4Mv8NQ@SyEVeaQ=;3`$|rZRYaDlNgsR6Za8zD-5up5 z?UwNH@ID~zR^t4n-$>jl38n!t-bJCqy~fuZ(XYG0=e%&~vTuAGqMM}$6kGY_U8QsT zujb5O-RiVN$!!3rkaawXbhJG4>Ih>R3CtyCF*S8NMNB__HcQ?t^^b|H+diVjm{ z%&CBq-%p-86=m`6cbR`l??rLQn%K6n()t0N^hi=jY3tD+@I=cvou{LAT0PSB0mE6m zp@9%}HZvx&vJy49`^#EX7AIEw-C?*iA8qv{>B9IaHC3Xdol)5E5~5Pt3=T}88wVwt z?X(_T+UY{KmpgyEx5%EmDs|j0xrlA?IRo1~bMnqpr8O>;_5;4?N?~(t@@Y=;ap3b( zh30?ac}Z_~7XX*@piz8vwPJEZ%}y$XRbQcCo9?#FVo=CuXn6v^UL&D9!J6LL$9=u( z+>r)S(IJ(4K#Vc&ceQ`N^a`KU@Yqq2eT&u9U9|YmcTNzhWni4FKe@v9oEgK(_Ibcfq zN}M%WuCmEQrcb(KW^ZD&(~~Ap9Ag0S!cj);@aWp!dN#rL(g)%pgXHKs#klx0XxN{{ z833`5=|!Q(E0+G~Gagrpm*y|*-ZfA3v=SHAEr^D&&F4FRc3a*D7WxIAeZ!n4oUKNm za zERCWq&h9(3T9Mezx17(})j&M-Pn3jT}WP=%hgjFLadTCT@($_bgeTP^|&&v3SQFX0EN(%r4@daEI-%Q zkh|>uD;(DJyk3Eaq|6E5OXvM*_Yk>lx{7y}$S;i~|BCLx30K;m(f&cL;7uzDCjpK& zj^m;UT^RwD2j*d*)0!_oG>}Pli7m9>L;i2WS?N6_-Uz0$klH56Wy_0Fk;gH;rlO)60_dvNfCX0&9D8bWaw^y%0h#vW zYQ@5_$w)Xn+-FWWhh!I|*YliUvD$gMK`v0sCWybo7CCBaxLsNe`7a=`2`1{p0VBX6 zWL9QmxAeBy&i+<4xcs>Dljzo&xdhs({2i3gPt0=|7HBFDqv|s$Gu^^Ld zT-P(D5;odkBh$4Fp;#`FObRjKL7h|ht0wRyJqIb;@UvRC0UxM=3vDm_1dbE(Fm(rb zQ_0hot{np|*nryd3jUWdGY{_I=$h$yjPJ1Yn&@Yy#u{%dxcJYqQr}%(KrbpxbB9jy z%ZE$)SOXVo!hopdGuThGUf=c6fsOp)mZLT=fS_gVw>LIylI&y|jeB}FND#TCUN<)f z4=X>{KS#>Nf6~PkZ&0Q0uNORsSBRTQ2z7P|0g{gFb3uGgDd2sh(yBBgRRS!v6=E^f z+?xZG>l43AK%ktAyd?TBGJfRkYEUH2sW>+>T@}dWpb8SH0vUSd=X%(;_h2Li;d)3Q z46%L^4cO9NmE;&mHHrP!)59YbOETNFW?_CpytgGM1BiR}HC_{zAk|KLsm1H{zL^>| zp0Kdv$G(%K^X1(Y9>b0UF7u9c?igY*B;ZSV#lvVzl&zETJJV+5zZ-(y89Khrt8pB^ z@hw``b#=?Wfo8OBIA=i=`bJ54`ltqM#Jx5iDxrQlgw5{g-ZligrIiyV>7U;{0jXW~{taEWgIq~-#gmnW9DgjCm zLTyh`Tl^BUjoDVIChFhQ-i8XXS8pg$4R^1}c8(?`@ln+|H@qR-rVnNTn~uQ zq9euPoIpn9Sd&KPDzq3VY-cB*kWt0tfGCZnaSAA0h|Sgr6Rk*q4kPjCu!?I_r@vw* z<>_ljiK-8aFk&TLv#e{oe(>WnxPmes_xpK38loHdAB z9}p+;Wr5SS@Agmsll);HGalb4)bsE`3IGU;E4`#pTLou;q3p4tznD)GyO+TA2tfz< zcr2Xa5AtyL$a9z2D=fskS-n~4_ts6Nz5Lc_2+5dG&EVrYe4^13=TReQNqCAWHV8>j z^>wqk@@`WrXRU^IGs<>blL!VQi*byxYLw#8(j}6^-)c!TCoxU7wEJuQl2`BiiC-$l zUcc9wZZtNlR2UHbsrg<~jnhLtcDs@JK*mm4^aw4%`Svor>#4P(d~YYPb=Tv_Nx&AB znEQl5F~)y_CE<+Hb~&A5qQY~OBeT!aam(wWx$AFRX~-S1ir*=w!f#rESHR79eb`1( zdn-q#Fa~9h=vX&3u#uW36nS=kGmkJsRa2Bs;1fu>Bk~J2kO=VG!B@icM6HU+PpoOR z$=uPO7OH#GCt}V%VUW9XLz2{~fR=gAM+&+y$? zpd9R(iko{M-=iwHltfh4fwdt$JYl{YLAdonZ~6i_>auHJiz-Ep6SuKTrOD=%XN%=Dob)KIjwBnT+^PmZV-DEOKDdbld(;U487 zt>t;s+u~|FO!>9*g4kls6LV@RvUN%i2^_#n?sw+&5Xrnj;1l;mp8N6q>PC_QLpo8_ zi|0lbriJ(Kd)k3kXJ~>r0i8%H(l{9{VpoQA;&{YvjU#BMaM#P-e;Q#Tqfus*Tdn@7vF7NJ?-zjk&a_ask4As%fS4TK;3jvfSZ=m z01ECsConNmm%66Ua^rd8<+P=aSAU9aW!x#_ z|3>Hk$d5bla97JI%Az@iE9dm)+a+bNX9)&qU;o0$ad6#O2nmx)01@Y0cm)2DlE8gt z5X6jXe3JE39>WXP1TjP3{i6fs)M-)|;M}!+l|Uv*{S?Srx7T#_;I3t$V^PC5ZgFDL z%$pr96^N#$XqRqdOu=)}MGS zObJWD<>AivMsSoq4ur{eS|&i1cH? zZ;Jh=6(f~UQ!Z)b5hP=Xe*WlMmUvK%q+en)c{_l+suUN^Tko$KB$~z~f*!UCV>nXy zRN@~-rr?x~Qgby#Nih`T!YL)@tI?UW`Pavg1}&ZJrLNV4@k@PJ{fw#T1D(J7IpTJH zS-SWiN_28ye95$w9rt+krV^bXQLgRueAHH=&tUh!p>cC`(ST0(yU>`Mi1Y0ANA`ze z-h9GVWP)a$z^D@kyf4!8F$SKzzj?#(>cvlXEs8}|Bhl`^U#gb%H7{Sh(c+6rM_0*z zqX!~tct4JnV%#Pn2j;sZu6r%MMCcsWTDelvcxJ|WyL!%*Kit2A_UFHmp*s#R44Oc? zljs~NP7>(Iq)sy8bYpf+qE=I4?l1-3lGC^D(wZT1HASZGtL0)=s^i2>%(upLt5GKb7ybG30UVsRN6Y|v64t2hMqD3nNE6f_UFec#asfeX`vINca-`TX*5%W`5>8yJ{;pfbW7@UC|eK*vG`DbA{_BnIvToq|t59RniA4DV~U=UE$kas=9An*YhRDZ8T8 z?xW2*uQfIpe-wI zx|%pj-m_5WX#=Cy{Jc_gCO^Qn5KrVZVOKi%YK~%{9A-?Mm>;D$xX-6wc*S5#aIo97 zv_GEVh@+ls8>W?~;72x5?|g@kdSemiWKkc1-Up-tqifO^tB;s5LkE}xKfiChS$8FMocNY7`M6D^s@baQoC>C} zsfx*_xI%K2tw-k1l38x>LuKqVxkk&67k74aA6f^BV|S$F#(QIpgS+)V;)u&3Btz3W zBCI0mMH6#4-M-{ot-=dp_o&Y8?D{ZD^AgHjG(yWd1qQH)i=}kgqCNch6Ek0v$&Gb6 z(AfkeHiv$bnn26{M(6H>jf+yeqG?vEVUA#rjIO^z9%WQnr1R+o-AnZ*##TZjB-j)E zDaWXM`Ug^!q3hp=E74bHjxzB91|F(@|RW=EU2cGLyPypes zz=aWKu>TFV91Tq%b(`j1EYID71lUGNF4v;|C*^WqqzRqmc= zr#E2E>ltNKLiR2)(tI+6J77Fo#~?Y)N*>+r|UiUl&RxIFh|J6 zl-@N_2G^T!{ll)zmlP75^4@sC<<4gM(x89BW8a=vs#(v>u`dkC=TgNyc$G5{F2?^# zkUNybSqb#^AjCYC{$*^@-2m?Aaj_SU3I zM}xKZt+}i(dj7uU*!2$tkAf&6?|smLU1Gbb!_L}rxcMs5J<_)x8%px*!qzL`YDJni z$PYWlsj$2VmyZbf4a+(nZjQ`p1&oTC( zL6(V*yEgOMW!oI^YqoK8{?V&K#c`@1KTRJiF+sc#Dyh2C8ZeobdMijv^xyE&uY5Gb zic1YyqC{%IpaWO1@u3U(R!N_C@}m7~RzC%E%7gc(KQj7k)OrId>gr`3=WYExTlFb09k;8aCg%6OEI`;E`Jf zKo+;KBO7;c8f$1vCAou?of{yoYQbFCM8_w5q1|Uh3Bd4`FO;|5&&ND|N%#6Ng^`0b zI>$(4vM};E3HwG4rry0c@t$5#{f&0O0mjW%_+cYC?%!TE%bqb*;A75PEWfRww=N>p zH7Lma5AC3%{&gc!(C#J%2HZcjk{{tFZw~VO4skD&fs2&}C?1m3QtJ8$0%aqq7gs-S zCxCtL-ZAiTJ9$q%JTFx1X2m2I^VbuOV1>VwHz;~|_Y_yEsuLy|&Z+f%zaTyDix!d{ z$(yOe>ewgs9p;T3+EqVlCN3zc$l+YIjSCP(MAg~Y_6m4lZF4YduAnr-xWVVE*uHNO zVJdCu5;#zp4P0^qBpcrWeQnPN!(xJZBAIAQ8AIOP-}2bdoOg4t#8P{?80mVPI!l~U z$9DYd&rjW>*Dra5|6){U|LE2TzDK2dCsG{*kB(Upu@N#D>)y}p5}=R%>JMP=Jjs(r z)O0PVgg_}$=hm?oWpv0QD;1`@9mR*0+)QvmrlR@1sxN!cxc_G0LWp<}?_GV!{bcRb3!Ztz3mlHXF|-^ZLH4 z@x8(@kCX(L@V(;Yh0ukNGo;P0yU`7%{akNOb2=Yq`1Y&jYq+FK|0bLP6f1mDP-8W*yJI! z3KMyaY<9V>1cPfIWVYIJ+Z=&*TIyBZKUWr%cXqF$HggZ}+}+VNrb8~%i{UsiI`Aq+ zW|mROC>cFz+CIHh2d8!a)4w@Awz>ciug4j0M798eoMGK{Fenn86Y7XYXr9)^>vRmm zH>(f2C~H%apc^ud&DEqScc-w;-T1WCznvIIwU#NNgGqqfNg#|`E6I1{JV_iAz2^eC0 z#QM8J?FWJbq0F`I7(G_<&>j(<{cv=ogU`;|vzyVGNF;O+kLp8*kXHXb(k1{;t|KO8 z4&6!-0Q&|w<*nSd`&TieGC8L3p@+;8aZ3f@mi+{4_9{sIYurnm&a5CgD5kAPppJri z-vWD6t=s3Xw?|H7(l=sQxu+6lJr}>0`Y5mE7yIRQbpmAJhYX4C9h>Go#bsQIWVm&X z32fpBC8yI>&s9Z=oiCj?+d?GK>@r!L20Q{2OKRRJPP;e$ZxxGkQAg;{j9klB+wdCU zT=n;2IU_2^v0_CuvaMonp0M7oEUs{9yK>aJ#YasSl@zu_2`J{RGX=ElE(P9R=@?TF zzbOC8HZnui2uZ6X{!0akwHY$M{#tPU2|gd7(Tc=++lM5J6k>HfbU?AC5aD`RJbL&J zX#jX~)E|`i5FaGzoNBzP&mUs(g@l@vphQ$tUVTV-@l}zyP%PlVHaYIcu=A^cb}YX= z>hlK7I=+g)toyy&(#RYl|3{z@>xuG0Rm)o0eVP8^1#$J}ASj}lt04&K>C0+{k(=0>dFgS(&Dbkpd_w1ou3yp5yms8^H~rID{B z-kfO6(Kml-BsNOg&x((v3IooCwmjDVmm5?=3^mrvtf@rD)73iGCt3 zn$pNeM{N^H?%V+0F7;aK84iwQB)y-u@Bh`icE4kmB(M(tT#ual@Bk6=h!}NXrKV0z zp#;hq^uHQ@HWssDs{50!fD{v$(go@^aGr#$K!krTm-rsn8b*Oo<}D5DiH27$;Q}gZQgCTLN-Kl zd~3BG3-w^LD+O_7hIG=%A3I zg$BNUJhc_c^XhcOVUXxf>(_Art`#41MJ$1IJk9s{N_-jiWIF6w4jN}Ag}Yp3vycdoQ)L31b|L46Yd^%2K<;`}Uq zsqm61$}8D_lC;fdIuyT4e^l`j;L?-jJxbtx=^n-GkfNX}Ao5+DGpJc^8U_Bm^K{?I zX_=1wU@7%0!_&~yO!d)SO6pgGDeZ&>Wu~PDG7DtrcdF*A z?fnbDYPA6EPAg6+r28>g8qO(#+FV)+b;cD!+8mwGe!}OK9#W6UhF-zgYJO&1CS$bK z#k-q@hYS2W%M7p&n;(zy+?OvT{af)@22BeHZ2tbw<@KU%T`7X-gk?T&^89_;?~VLa zjO8&X!Q^noF^SiO^@W0hM`|?GNV=r?&_UmnKM%9H?PDRK4g4}B6E zy!J{i{lWOryxre+=DY23DZKXiC4Hb(+fjjQ;T|Q7_6-Vdr-S7?&gd<T#u6Bba)l@uiNOW-*_-qxKWj^LVtQt@rNK$&9995+rP|J7BYV9CZ zAG1*Rd5C`$H3RF)3;8!aBxDp3(x!v!@t)YErdY528;%L9k2+cp7q3ir`6CX(38o(B zLuSRsz@Y1|1Y73}KZf5dbtEb0)8DL5Rz9NB+o1oLmR9QSdB_gR3vglxmo~Sm<^2$+A!toGl{m8 zhFlevW08362%tj}@hV^3L&R}2BX#s4%IaO`F;Zd^hoZzdTRNTp_Gl)Gofv8;rLE66 z;9~qvNRf0*=;f;c4s-8pu;=3nVl!cX%1v7TnT4289xs*=kP$Qwah`GoRy}I=ZazI%$oM)qrXn(44gDF zFbQ9L`C#!sQ@=!s;IO12mQIZ%(isxLu0xO@5~wQuC~UDlr29O|QA<|?S%>MuOzQ&i z%8{^9EY8QwClCZ#h-fY&(1LOI5b;egD$LtNP2=q4XtwyIH+f{cBg)LxU6KD@{gx$4sLa+9g=HVQzZ4>30urrN z5JTj(=W4KLpcy=$dF0po=-Rh2Y*;B)sn}`pQ(VigknGKLEb9O^GI|s{6pei$VwWBw zr7r%~a^mFjXhcEvqsV2)(6GLTh+0x*Ulc~sN#1W9J2v!&WgH2QSBk6hp$8(cdLMQ} z8!VX|K2yM=@5#qt!QY)mZPEvo$2IPG1tuA3JQ8F4&M(csS_VN$&1POGdqf(Z9Poud zKwX69l#qp_yc{~O=OvtlF1=LB z)2p6;rYJPGBzWfJJeSLTd4l7L8$%8&|G*woM)rNlLN~+puI<&fof7YIUhviN;W26C zVVQ9Z-Bq#kz+hSK0co?rX3Gj;gV}F|Rn^4>Q9SQEWfB_4&+hFy|6FEyY)@SDXLj9l zH99ryVIBvw!EYRLajC!fH+73`^L`?W0~BJM{nJ(vmhbVos+#v%*!iM81XH7;aY{3vSDM{L!KM8-vo&nf3cGNva!A{VRd;oYz|OW2sxK zEDhmfl3{x@w!=kKz&dy#KuHym4t&BFhH>j6rP$SW?nOR^0idVy45>eKakd+P0j)S5 zu;Q-dnI~DDkNB5Mk{5Lp2SF6pvcTTF(Mn-~?|2N6Wb6q^g^WS9%U!`jACIlNvk)F9 zui=q*?Na9a3?kT0owL7%ds6Nfz*toY+4JKi7m+YqME~S^A3*xf&*hl8>c8ZOQGCit zu_!>~`<7p6(`-BRm(Y_H(~dAp7~sXJr~ujT5{V!ZE| zJTO9OcjJI8%S(i~?mew7s2QFd6ENn9ZzA4`SH5?D5b$Wu zh!u3puQGky!SAm4;4NFlM?6)>IlRsO6};u*4C>V9iF_haL3UmnwOsisKSFpB+byfiwzPix!CNUNZiw+}UA6A&&sloxQ~3tECAy`|5-Ug-X; zbo;ZMUmw4&sFvmXLsd?B)GgQEtmS?c^F6R&mRtPf?No=G*H1UQc#^WQM+@%uQHD*b$+x8>lDbC2d*uPH{|eMIe=DOhgcTamM#a%I}DJe+K0 z7ANH#bwn<(=zR~b)|q-K3-#ka2kY3}d4ZVrVX;F`Q8Tt2zJW9`?3r2aQQ8uf!(j9+fy4$gTH!K(fRG3NpEBg^2hkA6d>=>D_18T~Mf!Xyk@8=HZE#X+2AJ7D zQrho*HK|Z}eT2tnC0vV?L|kID z;U5v7$r~4lyET6BRY;L0p=syy!`HbeuibE5Uc?-;1Wp5$F_G4xe*5?W=knLGr!d0l zXM3eoQusSc$-ugh>uq-#es`!aA)gHM>)=Lcfru<~wH%w|`gs`-t1e z*gqKu$^d?Qz*qj&Ynrsmsw!ZH6`Ul;cC zvOrXi0^LbU<}fPzkGjP6wENd#4{SdC4PKMr&i&A9_&UNcdq!qrzY3BP9m~MS`YjQr z-dn$ayB5YZPR&3m#m!a^poOfiPL=;)NkD9gu6dEO43YiNgg)LHdnexFUo}HJ_ zMqaiS-ATLNYP*}C6+LU?{CPV#)D)>^NoT^(YG$IHsoZQ?_MleXD53dGtRPArgTyeR zZST}p8nS#bOPtP@=DNP;4j1duJ^SlZ{CNcIR4&_O>rO-3N6>5M0h#JXm{`f+0=tH>yx&Ch@nz9%90j=4&c<6qw@y-ifx;G z&v*AwPO`i}9~K4S0HyGmz>CYt{144o-Ru*_iWXQ??1&}p=tTvA7X|_GY6$^ zDeF1KwvFT2?|2W|QTR05_Uhj^P}WNF`YoA7MPtdEJt>^HZ*M3D&pnvh&u6%a)k_?C zCz^SCQ}6MK3>Mw>a6aGRaLHf1B2%<&EnCp1-h{r{9h~cGzJ{M|!#StgDF#^a94Q*3OMA5;TIg0nYCBvrnqZa!*;$wW_CuT$>f#PEGoZlCNB+ip9)Sk|YHpfuPOv zvs^cwyx-8{2PQ4nDe>+|qbqTV7+~w5puTpVF4dV=uxP^KdjtiY5{vAQa#rFp-;nAb z=AU0$K^GGVrKexFEE_t#K9{`Rk1KP(9X@}QHSd< z+^B{O^&qh5Kw-qfP)2FP*mM%HYIDNhaHecv5MICW-?MZ54$2F{Nq zxpWM>KX|}q_Kx&B$kcv-1L?>)x;~r}r! zo&Uw&n@3aG_iw|S(m(^1xl*J|5g|jNLS+`)lqf@nXfPEK)up?PDKezUw6{4^MT4us zkl98k(UlOt4_EzU%k?@xIUV$78K~t!v$F+urB-JwLX~r*7}=GFfQ<&M-ENn?V%&y0f}|40eZUq}YXXrNZKL!3|s&G9G?OyLCcqOZ=%lB`}()+BtFb zU+ZoXf|Li6Ln;8d-eokB%?|@zDwcYU(%tg9H?`^4o;ri=f;_~#WE8QrFK~2EA}ier z)upN4k&Kr0#z%oW=^_<+k#ogB629>ooz5s}9`}1L%zceGF`tzeUtgi{KA;c@uy)v8 zb`{{_4p#{!Yi0^Psi_GV?QH! zi+%(%VBvJDh@+erZnd8Vh|;T~&oL@CSMKro^;Wc7bJ~PIe1P{E=6yS@xi0QY)u4#x z-&(Yc=NfUIOZrtFs?}m7Pqk7@xUJ!lc+B{CfSCFwkXbtwmy98?ra;)L1bO=AB3Pz4 z$Ye8qFk zq@a2C-uM$n(uCHA2NIH?5lg*oT7!YOq{6#Un)PEhvc%GxfoN3qANZSGSw@Rf`S&+| zO`$^T%P7U8WbAT0yU65KSOE0P%D3IBMEjG1C98NXF`Tz^295HmE1={km2y~skJ#Jo zXc_(90;Ju`30-f_cm)exFYvSvw)l;eGxLZu)rd0jHS~Q&T>L1ePbpfJNWj+;s5M3= z{<#vzp~k+gRW(R8`H-s=S!ZeCTGYZo106RG2&lQ#f6fEw2Pynlh1YPKe>jn|hf30%_LTnLwskArkTJ}Hd1I1IPW z9pt>I!BNNxyglRKo8!8_LC6L(>hS<)oMpZFa_@n$5qNY_CrbL$JV=QS=t0l@TnF)Z z=(t4|=T4QB&h$u44TDiwCp^IltB6QU32x5JV$Nsi=J(IHeNv3#q3@tS#s#lxJ7 z6gjJwl-rL!CH;95sFD#s#~+lYLP;Vv;S=AtnaE~N*y&eeP37$4w)X7#l>`FHQ{FHe8#;dh zh^7q;%3tz{>AK&&2!-!XTFaOkT>C1cBVKJ%_ywG*g)XLJTWln`RHH~ACarsa%~TsG z)I$Lo$huA>G{f6yH?R;|#^~f6KIjw0?_~1CmxH{`z)_^#Ac? zeEI=|bD2^5tK^_0sw}2s=}UzNsu|<{HgM=^qz(q8aKGfvub>l9{wp>T^>+!LOxn2) zjJFv$!#CCgjvMg#O%NG~=*6WcW-p)(XanZbBO&Asz|^V^6^~C1Yyb4YKxVj}Jlsk{ zg-!qXw!Fq5>gtz(wM@2NG@%eqfQ@{w?<`<)cN40euEwbh6if4WiGWdWfQS9ZuOBU_ zmDh&DjH?nbye4oUB1y^~1tiv+dk=(>zGazkay9L1`hDNc78oCe&_WjsXj>%$uwh%@ zwcO)iJt$#=z95t2<4NNs$gMY!=|n?zWFr1f;+po z3%(O+xYFzXK5ltqi9SdZBR%-t7!vbFkvw`fv;fD7Crv96S$p8!Qiy1@`|tD1mwrfjAEr+3h~qV=TYV(Fuf(3 zb|Yw2wQ{~Ux%=4}2cnYA|5zgO zV@Io{dn(wx5C8eb14wo&lnBAW2@+)CRw-1&Q!-UEoa&k_*aW~p8qWOZ1Agz>BK{Px z1UHf<91LIy>jyP5)gL>J7A#t}AN|Uqr(eJ~(IxF1Mos^?ge>$Ga5aM6#VrpISkf-7 zP!h35gZ79t`=d}8(LhX-oJbhJxZtTP=p{M-b@f^3T%a0yAP&<2h^uUU`2M1BD`3s< z#oW||>FBE>e;8+D7~ljq?aY7w+ImN;cPk&F9!-UL_BLedl8+_xClA_Lf(B?2;Efbq zjGKo}#K!H&9sm1whh^q1F~n0eE;jfZ%RVHKkTzYrTf3^ZPO5|&*wdD@7&h3q|_f8YsXF>v1J z_q!V{Y9S{Aj=Q%m8Id^+iC%Hw(^ju7z~xB;u`~r;$}X~WC2B72|Gf{o4*1u24) zq@XE;=phZ~Bf4XK*Tm4<9Ybh-=8H@S4C(*AI?^ovt?rJJn8!l@i3&&d=<`1cai?p( zwV^%x$nZx6=ChvNFdkSb|R&K;T6F+?6cnxNKTlZ2`g7<=H>Z*&2;CesAnSLh!wfGK$zFC2u|ydm@mKe4uu?X+Zt+{>>LzTq zXhNHE6XileGX|%!HZQMW5qrVf$9&k}3ad^XmGOWDEB1?L7asIf`S<6(lz~>XALw)a zGZ;J`qTzmQbYQg5sJkQYMjjv^ECR zFjvedQW<>5vm)@28@>B?v%lPc(_^qr+7-==+}O8YS4Fm$(gJ!X1NFcy1e{xzZ%#&0 zO%9Oa*cmN>=g!}Q@8y*)i|N=e88M1c;4X5EYsScGLsQGs8Uu{f>Oqy?bJpeio7#)X zn&8{y^RA5tQ$@o9#_weAHsXrvQICWlK1gLeZ;Su>457Q!T5w`Wge`o|D*kbmKbT7D zgyv<066hd&G#KDSN05y$1ImYf{SM!mfB~?tYNGKkr{lmo($48}DX3)tvx+GV5BR-{ zs>Un88BXj@Q23@}eCc#m_-0J+ILx89pkL^TE|{J-2B%PD)e6=olK$nX3oc*agL?Sg zPPw5i`OjBi(8-wFrPdTAkOT?&lp9)vf4-pQZ|AaOH~;nl(-8ixfmJh zMw&Zo$Dj;O_%d`C3jTgrOx$@#+%oZ#h$ljjopAbV7()nMvG4v42YCzWFat_hfnjC4 z&oH>d|B=j=NZv;I^?#Gf{!Rt|uPdJaW?>YR;hoV(CdH@`i$@V0=R*XUhPvjz+Ux|_ zAmiI4fkp_{NH)DrO%jlVgspVcsYPyoHzr28$(!r3)|-T160hd76prBv_F8000)^@+ ze|cyd+fA$+Jg`7A4XFK2_foJunsuoAi`Z$-Ekaz*N`!?VE|tRq>o?{!P~P_~fHV zm{JbZA<&`}uW(NELwPt~z9ao4XtQvA{<~Dq=v+i6u7AG8h`M_2ZwWXpr3$F_`sejN zpYHmaKOMAg!{61us8go&yUESVh~7m{GVUb3B`M=j1k#*NkUby6_IA>oVFF;m{Co=D zP}AB{@~l-qmzMMU`|_JeEifim=+*U%N`gO0wVK(6`pyrF(I=*WGN@d1;+;=iHwx4` z%doGw8a4I{;9p>#T=BrM(MG=OS9KLIooTg*p5_1`q#yN!9;s=`Q~&hD%PSmHWCw0k zrxf#k4!fY-IF+YJ(&@R;nsie#R_de^V$R*Q^E{a z>ja3+)Ti;N&{8L)I=C*R-~*Is^u9*tcX%kA&0w|i15~TjlM(@WtNx5%mB{4xGwkIB z`tZUAumXrIMyWsL2IJxB?)mgc3MH8b+*$*=C$ELPC7?SA)X=tBCI14CD6+@Kp1$nD zV}Ix8bZcoENhsjF9f^By?1)lt0K(~cOb)IowBlJ{5!I;^C>O7vGX>)CDq=g5eB&_+2P9#-m%(tL)tb_%d*XcImU=t zHnG0zwde)S8@r1L9R*ES{phUHnJ@8}k>-$+e5WI5-!fYZYGk4oopGN2`&iI%Dj{vT zS$d2L?~Y;9!vqX-e$gy?!{|m@t=KG?zc_0>Sw>dB7UZyB;hX(F;xw80Anm}5%W1vt zbqPv2kNllm{oF5zkypl(C@-7;zBy9*MbiVn2M26R&nDNnzL-1b5NCT;uIc%fW&3sC zdPM3XHL8U=S-7u6^U^FAp-446m9Zfw{B6`U>W9S5OY!&AgsSfFjs3MfY7vd~!?>Zq zu3OxIZ+&8E3e{K^eaXyj5 zNu_Uws+?on5{7pVFrT*z4pkqG9C-NUCspZwx6hBy#IRW{lsI_@Xu<;e@I-CXTJ0>w zcikVxy{b>9&98El( zN*z<;WVb%D);)qh7hvechk{-ubuxt?}nR0q5OYV6#TEd3w-IXjMDTb z9PsVvqP52dBbX=wN}|k$;5;8dvwo;#Xy_}WA)>o!X6R7rtjQq}*G|_v*Wco)PfQ~s zMKz&S&uD*jZ%%VPc~K~7^Pj7*>_;SK;sy{yt|55x-GY=D@hvOzeaWNk~YMJXvf2m+{^7-Z$ zl7&{{b7h|Gc*v(pIIw(D^DkEKJ=v8JR8f@k-JN{M@6LDluAwHGN4e&K9ZZ*UM;{U_ z8;zJAq?o2hV4E)>iB@SHs_+pIE+G zO%9Ef2k^TzG%-6|(ZO;qq60z;C$MUtt=1tl1Ps#N0SYv3Mrcor5o{~;e&EfBW2=wt zob)nFttcD1<#~%n?*n02t(&nyYbI1cRfXj0riGWaf?d)`@)lUVI|b=$D{L^g4Z)kG zA0ljhEX@%k<(dJ%p!mSmy{~I_(}P})oj$jgw6Sujoel=Xln;4VPm8qM9^u5yt%+1i z_W`YrwwC@Z`Z$d_&lb{<_Cwxw09b>Y1uCatQdu^ea8WB>Z)F_!KdidzM|*Dn{RgG` za0jj4MN7)Ew$l?G!rxKT&1~g02Bo{cXGTz=2&|73;hSbj3|mm{e)<-JXQAu}Y7lJ3g*)XkAT>k1!PqqgMV+ymf8d{^2jVy@~uGl|YRWnc}jB>*$MB9+Z* zy3$rm&DnV}^1eiU3kPLIG^Y0*>)Ey`WAYIar8e6t0#MmXN+(_+k6}wju2RV}R`Vc# zktp*)f=|ZL-20mIshQV5ZwV1dI6n)3k<9Mq8@1+0O^&Uik)8ZNmr^U#`=8)DeH=`i z{L|`2h3Nwj3{z3fmb?+*PTcHOj3t{Hon`?R>&0n$vDf2dUE?nM|1g~XdBe2nrL@q& zVEJ{7j_RCSB@&N)Elyov86Ki|hGLlXZmsygmhoQC+{siEqREj+yxNIF?=*&uO(-3W zG7mmde?bPu)v*L@c2JnFzYGouD#&?1j~Q)GHce^VTPorH_Um3Neb+wgot|&MOg3lP z^Y6yPOmWspZs#gSJ)MGbI}I+kN8;WRCZE)~-=l~`dG%w)8qd-%K2$joJ)eDbon@Ia zvLAFRS(D+q*Ac%%04j)ux03zH z{lw)C6hA^g(Z*!^JVA*WkZRtM?>O~h{*;z+nH!e+4?9PN%?>r0+8ZK%7})aZ?oIhL z>jC0Qict7Y8S8S>XU>_~H_5e4T92-#=;((S4yy8r(Qm)QQ0^km#5V{F3_MxaQR5o? zHH0c&7t62>NT1#Wyl5Uh%&(TbMF3k1QdtFMYus9aO2oC|pYMWi>u^@bPDvyENyg2$ z^L1jPSrV<8+_m!p!_&*chaT6IOihZgEy0y-sio=8N6WV6(|`3Cy`ADt_;UL6xqk&l zuZ|~?w)}UzHaC2CU->Cx0~Im2f>c8WMtsI3I3FoDH?Fm~f1rBE>loknp^E+o`qsLu z_pPk|&!>NNBB_Pa8cZ|%TJ7D`G7_)JrG6#oz~41W)#9XiYf}*}5|PQz|EVA5FT?*{ z^+SLr=#;OTewXqieCOX*(tBPet4f&H!Y<(!^yeCfaW`!VyK$BIa|49JRQH(5b$@cAE)HK$N9C*7uo() z>x0woHtlaq1#YEzI7h@~wq+7*nD-CnyJ13|S<>)kAA$Fr!}M>H zw!_Cj&itWnRG4eN0ss68^uPNN{KM@Bo?5ppbmsol+d6!D-YeU{=vQA7tK{kIeBZ0D z>XPwoi;r)+74VM#Ri16hPfEMLKYi-*NSbJAh_+EFKiX{3;y&ZZvEZ8YC3>&!kT3eX zS29|nBG=zBFsJo%c8=BDXnxg9vnbi1%wq_^7b&Vc+Q)$wVn!4a`nO#24?t3W z*foJm?e?mXSM#hT%)Qc`jLtbQRbhFLu@4wu41&9;x)c1v3G`)cOjo`gV_JS0*MlnK zJNb=p3(}YtEJ^|R8Ex4GkvCz>6=JA*Zp#JDukfSPrNz5xO#>g&>I?>{aXUd($kfki z^y^BsXPC0wz5@#HJl_F)rxyMN2362F#i}~2jTSaLXwSjxYM7_ z%opis5(t!oq zGn@Wvt=s+;#`6qD%%(G2Mi$atrPt>a)G0YZ+&|SvrgjAGF7mUW6iG6V6Of9`JzyJJ zeX+8k3)F!T9=?&VOw5S^I9XSB62n=gzy4V5mi`IC{xn$2-Mg>9Jn=Q5 zO-x|*mYbaNZj?eWN7L; zh1s3y*)Hgud#XX4ySLwg&7fd3k$FR~3MG&J4abKDbotTq^~Q~nV?>O@mc>Hq+CwLb z?Ue@ay=+`EH4YgzoD7UAFv@!oV7R}MB1CY%p;Mhy3TuDlkcXrG|s=O*%<7LQ!OHMg32zN?Z0x{B7*l` zm00G-0OQd^S6u=Wj$w#i&$e2+agpN^;#n>W4ss!`b|1?hcxFXBKQ(NWN+a5;sTiC9 zVe1q3+mFpt7a%_OBrdk%zK=!zRRW3OM+RMboVoc--dk0z7|r*hGDADs?QS%~rLYJVCgue9sg5`Rk$!wcTytG1#=-fIf zeC|~kfNUr7iK&=M`P+Be-mm%KzC&#<&{Q?aT5g~5!B}qIq)4hap?(p=W65``gl9j9 zbFH#D<1%=5d86tE=cD`NxzzT41CiomHOBPE{fiLa5x=s8H*PZBZ6IMZLc_a4I2~XyxfS{lp+t+3NBeqWIoEh{DZg* zUC=$FP(+_SxRA2^8KDH*ODu|P1>ZAXj!*cI;6s6=2gVB%lP-9UCS^zji>Y&Qs_k9B zh0WHUTJiW(KexR>a@GLW{F>A{uo|38ia&3Uf9bu<@i<6gR_?W#5G1wUaqmqdlfpWU zDPy4&PBo!373Ju}12nE!_@a>e`{R>op=^{F#RG6=ngKj{xkck8a$O z_!#mq?`9S*v&0&ky$jQ9?r!udHZjf+_9^%SgrwF_#S7NB$NQ4@6CJ$xTJgPh?e+Uz zB7=raIVQtHjVQ~@Ky@J@lZteJ-uWMgGKc-tcJ33D74V<{Da{)fAediYh9=6JxWPg@zmO|))!P9&g z7esy!u4&}quO|BPr-P`wVn5rS-k7f=lq?%(X^onp*p9lez~*1aL$Z+xeGjCo224v- zO%oX_8EDpZTiM(8ppOTFri@7E)|FBDO=|QI6a6cOGmku=*)(laC64AwvlzCjM2jnPFDU((R!k8DNHp zNv(>+y5@#L5@YZv$w_IwtZOHv$**3-PNoUcRBN_e=ER3KXXb!S#KA4{`OafECs19a z5!S4?sY@ht$Z7d}HkBig(SRR$RYxo(iF_ehMsLQweaDGKy2f z`78}%;VlrjKS&XvdK2aYgy=5wgESoegz8K(03t*NC-5x9rxh0%WA9qPqD?Zl#4_hp z3j~!1y5@eKsayXL?y_k>zKqGxRcXgG@o)P=;#Z!97&R?h4M=8;FpI z$~fl7zS^KA3dAzmyTaN2S$YHL^&XWQ|5Z~SyI{?}5GQGxFX~AKreni(Ga{!g3@1HS8WKn-CLtIu4?>&L8wspEso0YWps0n9@-wey-{4jCDK3d;0bGS`$;Kd7d~LC z$IF!37P)4m3NwC*=&+!~maLl-=P;FQ0)=g-Ztl}hvH6f#^xdIR#&(Zdh1FMlH&=cd zu4q5{E?Cz<`i{obkM>xjn^HsE?p1B<7Hw1K|hotyJ+ z%`3+}>GKsQ)?D>mb%PCv`@IYqj$gDVl-ySR(v9DI2qO+mvZLo$1Wa%CQ7^mr8OP$5 z6dNxsVrmWBQpBiN9$)96RrnZq#cY$v5K;XIQ*BNsZA&XxC~h%1)bo83j=VM9vTBF9 zyAMhDYJOAUQq4swCGMGh)jWMfHb}TN{WobrAs0oKAGQc7$~$lb>R6E*shlINB-=YA zJw;a_H~of^QBC-i+jW2x%Y~CE*#H4S$~Z!6$p<0aK|IjN>CMxkoBU7AV$t zikuP;NjF?CS6(;if^=8kcc&56#J}98)h=yPV=>T&$XvT0Pt7aQpN1DxH}Dx&e3aY# zNIEuqzr>Q4((n*0^U;#aR?ATdurvWFe7hj0!Y|GB_vONKd%0)7qFZch=s>XcxE^?5zs-P4t7 zTO-qgzK>#}AvVGjm^TcLNv*w!?Rq}HxepI{q?iVRCkNM>kA0s~?DbD2j@GpZ&%*q7 zwYs*boY@Bj{CM{`VUM3&vsG2EoQFR)D`Lo6L9#cd=|}vE1_Lx8%|_B3EArSp5lr-T zZ~ZJyX(MjR4yv45`kd;#7wHk}l<$2|AvInni@XrAV_p83Ip9M`RK)jDMB5!-KS&X> z(y)z_-Qq~D3Q~&|OHk1?yPK#Dj!I=;)15{w8s)-zr5mD6`_hAnopquKcvT@{WR$tx zEaX84;r0T=;-qD-3dhTENL#sja#p~9#3^4BbwrJ7_e11}%>vwA3!1k`OOlSE_@tGc zVm7)k;Rj`x(@3}Qy`WI!fj=^KpG5Pg&)n{1!?TmD-F6$hWqYrmP8hCWlC{%K8}M0a z;GjK8I1c?tl$8P>v7M#KQaouY@65Q5xf4 z9>ZsQMBCMoUZWgsFww$;OK@?waWgp6U-W z7Y{&wORZz~1M!tXIKzFB)e;3dD=PPieo~OsqMZP5ma~o%;6VHQ)USS0di(^^RBxnO zGG>zoi#57##B4Y-^hQqKh0IcJ89jizpHzHK(QAvThh~zpP|wVY^I;U^!0Vw9OlU814pQ`5LyZ&ZDeX z7Bz3YI4-?eb1r?pk4g=C>T75t%w>nSd@b|HMtoJbO9+k3$fEKMn4GmE{|Y|8CGQkp za9RY%7#^ln0;OOLT&~^Y;CDar_VYcgLINn3kR!q%x8mB+HKw0RKii(3ixKRFeVT$%Hdz4Tfjr0vZRAJWtK2G! z$q4^0$xH>&p`r_SsrW(j=*t*>_xiRYsqrceW~eBerSsI-S2wrWbEb~jJm+W2jbQl} z?mr_W^{^AkSJ-LJuwH5ND?{f2G*PvF2L-op;IiHux$%lpYu*+9+RWrTa%z5DwjaQ% zi@l`v{V%c$y0&%SF=M;kWqbAxwuK+gWWf!0UNMeZZcLH?J=ibR!9^^mT?Ru!59b6< z&r!zA7e*4ob0HLgy@S&+{F)6a%!7?LWVMFhp`{kKb*$m+R)LtzXN9E3_@A$~Bv|Gb zi!vXQ9xw0Mo=+n!v&H#9O2L-$mqeu+wGNnJi%4U-LucDxZ4;P5o5*^S{4wqESlRd#U?_|>tT6a z7Gu_cC|E2l5|a+k8E)RDeu^#esrbTcA71!-pFSK}7?iioYqBD&?IQBJl(puorr{5h zd+t@6EtQSTlacwV?D1(^tt*G4h2L096A0S|o8QW!aoa1_dy7-FMl?B#6HO!kjXig_ zSUYhPhF@>|3!#Vj^lU92LZ+?fPe`)#@Xp(qc#osXZ0R;zAhDi0kxQ?vGbakfKsLHz zlJLOciap0=@2m)P$#6VzID%veYp<`JPc-Y-C7N~HUE!PKy!w+V;xK1BQ{Z)pjXv~0 zK1mx`$rgoG1W(UFczQPUwz8UZMU~lF;xSAs-YUPB*jx$YI0q+=Dwwy&%g8WeGMZ(O z+_%DjyEk&4P~KkPu%6#`f$?tNpJDZbgw2Ue+DRo=zGO6OVHcs{{B4klZxI>+rDIUs z*F+OuXyAi=-cg?nImfc+rt1BBKNDfe+Q-x2bllkTBDJip4Q@U%#B&rw(i0kIStxQy zk1^cs$x-&o8s1T;8q!ejZ;n188WOe9M>mQv=zG?iG*(E-tf25s$ID#&F5O!u_?h%N zGTk3n>+g&pg_}%81rM)n4@i%(UI@s1s{R3SCL;BCUk_rySEt{PBDNXPv%&?m>#1Qp%jq81eF^Nd&YbKFsVhVo3{HFu2J<(_ z{*w;+7%4G(vIT=Tcs(muSDenLG@l^}z)F3cyo(vZoyB%-zBskfXUQL*9x>qBPNWRI zUkNwDGN}gwNF7MyJl`N2YENfUWFr-Oh$&Eky7(&FqYFobu5bz)Mw{7Oz3y@Wp>o&Q z*Dh58B&DD1da5WQhb*|WG(?&!@#!r*T<)5VD67m&BSgj?;0AnLo<#bVr(doJg;ny> z0Uw|H(!>2MlYVN|?m2=;g_oXQM%n(_&N<{BrCGEwQAsO19{EJ^y>WzyG#b<$^TKf{ zx?aq-jnQ-G2)y*Mc!N)NJRTG8XH1EA$qrmN@R)*Au5D#N7x^q&mE-d8ugk^z{v;gt zaI&SH;Y7P_k`}HeQCKeMI_MFaZwj1j@JknDcJ5LcpjRd`vS^?dN=fAt@VvOoDUu~b z@e`37_dX<&?XfTKl()i1Ny7&x%*P7ioBhwKDPz&j-5v>fw~!1Lm%Wsjw+)GBPbv}OZ89hL(=yYme3F=@iy zUbF06w#1T`kGHApP(`v+Y}(%ID@}_h{qfl@!dpepD8G0F9N3z87HUN0!#PNqZ&fg! z_tv*P+?3pN9o0EwE!NM0el2ib@$fV7CJ_h|}Kj;=A-Trx+qed;;GbK9X4pVet}`l;l_| z&<3Zlt5(?SHCEkYBAC1;QRQmQWXxhs6Qh1GRYZYKYHY8W4w;(Ioj+ZYxD@qfCsAJ8XrY9y%6JjO8Sa_-@HNY?9wzmX@~U!#+wzQtiP&*3I}l z`$!Z%{;`N6tLYp=F)o1SH4Sdfa$nyBtzse#WC{4^ZgaH#Q@L4-)uwLeLjZH)KyGBE zq==Nyf|s^b;U-F=)5#7asxhEpJRKu>P zZ5sR_*Z8gGO(Tm7SF{o8H?D4Qas-D#l%oAY8p$m?X(8)o^XPoD+SS|jn8&Dgsv5Dq zz=IT`WkQSX?G!cGRkajMQ$5^6ZWun7mA_H$Fc@QGBKwq2oXPp}GO++>E@4{IrYbb{ z3B(#6a_yFAC^wDF4q~}NG-1)x!7>?nkBUysIi6j&Yc@&8c_;1XK{dZs zsWVA4PLWw+hvRa^+^ft|LJ>AI%O|;KQ^+u)C%?&%Pboi@6}F+bw!_zVFrR3&`22J! zEu!W3Cm&mWx@-C-q`V5qtZf&@GVyhQ>MeUc07liO{m4>*XsYCuro{UzGIz=pBXYayPn?`vqZn!fo z98cb-J<74f@TV-Zg<-G$DOS3^O@Wcbb)wB|8V!x>RN#FwmK5(1xr==7@bYf;@bnm? z^0N;i{pr!U@aYFPZx(FD>V*_oWG^e)hlCnlSA{)E46om~4W6(W!&TdQBc(5EzIH~f zqb=SWdDr-=CgEHc`pXG;j~Ome_F{tONoXUDXCO5pq;}$O^k1g z?S)Z*B|5}q;Ep)ZvJzoi%MgEJxrD|t@@iPEuHm%U^k^V-wn_qrmK-TF9i0uXxwLnM z;!CMzj@xW|(0SdcsETnff5`M<{ zN5_mvidTGZNR;&Xa=q8Y^W>|=*25#JVUGasxNi{SvO!q*V#&A7>*3A%D?y!9!#c6@ z#Q}YXDV{CF7_KieqJ7OsX=PBj_tB}-`5P`Cl%|D*&>4ctVqALN(zyMpO!|sF zkpyyey0cx&tju($=IBtcqWw{qMfBrIHT?)N*9&58vVszQ?E<)ZF9a4_`Rau&*KX{S z%^yl$?BQ#D52n|>#l66tV|J*P&FU%q^~X%?HZ7#N7ov z$f>b>4p^USTrWe~Fzj7M-Vr{V=WBbWY${ATxR|m;q}X}&^UjG+>!8ZQ(0P}Xi2Ahw zVqax`kTQ07@O}93mOzR0i)QRso*Fi;6!S{CF8Vcp2mNsVnVsiff0QlrbE3}KGIu480R9W0@VF_P~FB) zc>q99Lp)YK85X2sOv;-tE7!R<#29I+i9HSfz9!(!7L;#g3?>eppm%9`BJ9uTF>OSB z+iyKLZE9t37DWtED%>V8A2(Z)m!}4q2cQJ(SZrB;_>$=L0;KwYX`G*e5&PU zTh>ThnYhs|rhM%-X8nleMJRCk4*w)2SP9C;k+d{3wNQ$&-L-O0Y5DTmq=oY%gEJG{ z>R+2s{>e!_*A#dPEl8n>wSb!Em1XoLdPJ zj>y$V;$`G_Yg}z&2p$sC>Zus|{*cnV&i;5Z#o(^l+AE0%M}vYiE-sPRi1U}A1Dd{5 ztX@TA+u@Bg$DGTWDC*qAtJ;e8l!UtV70WF%qwPVByV z#HeQVIkb`7dO{6rcvCjY?7SSs^!cWVd@5E>;&AX;@TWI|!Y3|Sv zW6Hn^v?5|xep2$1<-Ce-wjvk#PLbhfHBs|MX9oqgwyk>vIg0UQ3`!gCrxIQJ;1Iy4 zaO%hk!7}FK&EryAeVAW<*kiX;U6Xp{h`+=d^>3+4c*feclme;JT)hF^xuI5WpG^ z&ULm><1TaytO5nw+QZ%*p)5Zdzbr7VWjmj^P>ewPg23|E=qaO0cn?Dphmndroq8)&l|A0T3i5vO4``*lWl8d_Bxb>S|7n zi#K>3iaL30T~eW%8GPNebAki^0JS4C<&Q29bkd5gN7Gm_bPwxdGLU*%*DNCEi9bZc z{&@ZEdY2vd9FBNy#>>mZ0^t9wq&{aX6=krMq#^YUy7MOD(s!o%=6YKu%y(p}u9wHl z_#Ds}H=4?m$;otFV5HG)=W>>4{YEAbE&LC7QJn%A=MgFVIA%w%|NDN&@qruQ*Q6@i z_ZR;~6wqI}cN3TL#ODW_=KvaxJTABwu_mD8uD5FfSlj0STe~+ki`+iQUX+Q1GVJfY{AQ_tj+JnJ12s>|Hi zDplTH#&t+S%91RZC0VP#*+_qkZQ;OU!neN-?QAhzU3Ik^e{Awe`9oyE6I82d_+L(y zdli6QmGHZH*Z6no2p!=JmxgItNA3Xea)RrM^MNVHfD*B)_xr5-o*%1aq-w$LqGRVV zlDx4@*Q%b=mHCRIjh$hcZ{7wi4Qq6&P8fXD`=#0VhN#dQ`Q^&%d3Fe$^#nCzO^+wD zfv0ZT&8`gAhAS^6n`2&P)JZX3*jD*;*C7Ms>Cr*?StY!a3(rFx)X>rwkb=*iO)UlXczw-z?+ zQOwN9?-bb>Hm}SkU0p@%aMH;tJhit~@Hox5AI)+B$x8`xMg{=-k9a)+7DE_P z2i}!pTI3b?=aIW?=hI z!xN#H?h2j}Z;cTF5WDlTOkRtS`;=9iMe^PbsBrmK6Bib(h?)G&pcD)upF%D!TmD4F1Gnk!@83` zT&M+QZk!637F6l@7peDa#u^_gRsAT_Yk^s?$k)N}Wfz8RTAd|Uz1;HxGgFiP5m{I@ zO38|xV{P7H=th$0e()|VV&5B^f#lQ#ba-NIL-@~>(>faMOwxMg9kThxv@Kc|wIYuM z5>;wNZCCKz+Cyee@u7U?(-ADx_G4*eIO+-#7^X=zinf=W>~HBX(GSJF9$!jnzPW!Q zoF!1C$26ktFRWgwkJsSrp@_*~@aJoOez#newMKm;22llsrEu0Jei>(IRO8+Ol;L%3 zY-9>ix1R1OE$B0&aST0YW`rN7aCTA!a6!%slWq5mfxdZt+2P1n~GpalFUTPFA&9 zkY8jSrSn1$&o)|%#pOJOD*^^JA9r}&I}g8uesDOlzWr>}k@HD>sh8bWOSO=Y!R!n! zp%PBV)~)_Pzq{^PcFd-ImpRoAa|LH`h>#VN?u(jP2B|P*w;8rl%sEUTTa~9J9CF&&J)cMyfhIm(uLVY^-+a@F`+RDr~G^ zB1TNY>-oD!PDQr=01ISTI&i9!7<;Q;GX03VQTuFW(q$IN(5=P zM=se^76YvdvH1H^J%NZv<*Ny=xzI*o|A`A4Z7DDjUKS|A#4`~Z7|i1@w08s7Spp)b z!Tg15NNzz*$Pzw{QW`4n9a0xHxr|&4f;3ty<4QoWasNtZxK~x~300$N*o5)8H-bv3 zLeOsE)_wB+R|Ydl_duM;!i$OBTOCgu07^F}=YSFs=6!H|Iklc9cufG5h0!$^=1ptn z;jNva%hDk+6SOuJIc{M!dYQBo?v&4XTd87|-b^$uhdY<>CG-hBlD&AOQD5nNm~IlA z#62|*YdK0wS(RwR`ym()OreKArVL8;`P(fhUGiJuL4kZ9;Ds1lCBPQs7dx z)d@~CM4Q9d!?rbY!EoME(cW``0mf~F=t2MlGA@<5FwOYz6w}f2m!t)ZQ?%b{JvxL^ zsCM{+tINZnJl@2dAn(pDU_p<0MFDkil|Lj@pZ}orMVGJSsAMI-pUSVEtm-7PizX4N z58NyAQ0NhVW1XuTUPCu-@rNm^nl3qo@s(Drc`*i7gM~q_+#_?ubgp#2@V~Ou!%mI5 ziP`)wND8HjPu%hDeHBfXAaw5LPc=ykl%*+xONf|Fq zE^0`Gw28^4X;uj+&K2JaT*zf-Ta(vR1W%XZLTX4oN9Faytj&u9t&#%E7>uNk)*_?I zS5I%9Hsa*6(stNSm8FF&yU5^Ou#w9{=sjn*%M`x6WYBvfJ9)wdERJ+?;!qC?p!Q=B zDHvK`&rwtQ*JXY&-x^}QdGe+hVA259-W%UJcjvTSD*?BQNaQIbL~RF}?`z)3?1)MH z$cN*RJXhXj9wl|9J`ugC8FhBurU;1+~z7b?+$lu~~6vk9A|2w$J&MOIVX6cH60w ztqGjR7OUspBz|SlmfXsRk|&Jq?3kLj;}Uz;LY%5`ZXHEV zv}$r4kO?QauHEUqzh|=(5Qi#DfmsuqPM$}z7(6jXf%GMBisAG-NVrM_eNV$|q*7`n zeFC!Bnx)CD?vY)YE?E+)v=DDwUkHOh%X&l(XIXcrWPSr4VkBbuSjjDg|G@6YLT6%S%C$M65E6Z-W#IB2-Di^Uy|Ua1$z!)<%i z3*BbjlT^V(fu8ga4yiK%<#>2Cnm;L0BSemJ+W-85Tri3l-sk-h*DGQBtaz0S$n@QI z#!~5n1!oqf0PWA8pT>6{|>c(wkvvXj6Zc0co;IG%e`7QIr$uVn9a^_BlcEEJFipjt185O9eM2 z^u~H)wPcP*`Ois%(>qNJT@Q+rq0ZLZylU@6zjE^$H%6ZXze&XXkX_Ulw7mV`cxM!i zETC&6>0K66y{sw!C(#UOJ1jCdQ}tus@*FbLJ87m}WF$v8_QV+S^_wXsgELTC@P(oT z>dCm@W)pn-ptK6$t-JPK^y!qjhfQt9)vq#nE~nQ zUUl1t+$Doi$^OLzSTO9&HMlSKMoATZX=FrMJm?geMk?4Q@Zt^M(pU||llq#m^{(1? zJglb4f2Lc&ds(RWu~DT^HBL(CEnHX;1O=&nf=0?^3$c&EKsb%?7#$X>4}B(@n00Ww zPKUjDkzWR-s27Bj>S*8(dc+T^1c7$g)Gm&+p~Xi%BM-1RW)U*L<2+FrW`f(yPnIId zQ@mpTI&kiYBgz4!shlAr7W!)UD)N*7(v*CMFX}DSO#E8=A_3P-@H0bS=eRa$!uSfs z81wcK$=0?ZGD1o-(o+=1mf@oN`W%UMAyFJJOM%t8^+$;8B;=fDbdjxQVe~&(_w;mt$Aa<}@ZhIVDFU6Fowg?v@1gyT4bWdO6 zjN;vDGPPcu5^8=li--m0%_E6l^L8C5X%4-mEi#~$ze1}mZe1nGLif#j4EWSdvmZW8 zH$dVtXiuf+?8P@&LpW4YeV_o&iy%Np&rT_2rOchoB3IqrI;w4m&(D|=8OC-oNWftYFY^ zAAUhDlP67>G-@JwJ z)K^bD+j@jN2}RYAY(=@^%dRuv{WW2_E#K4!5oYsY-L(iBQYEkU)=-Dmc%Ume^< z$b`fX^mCcADiGUdsNH5}i5DJF)|_T`9o*ZkaDkrsL563wa9B&1{DHWk#q|?A598q{ zO#REVY#8-n8Uevjitbp4=3wlDUv_w(CADI$yUq%kd7=Vg8I_z3{q94&(1>Y;-o#JB zcpx6ABR{@j4_F##_%y1CxqVu_(8V?f@JMA_yD%2$$3@VG{ZRTToepdoU?4kKyM1B# zZpPtwj(8#BA>Gl`2dRCtt_-|%ocAwWeiOVW8S{COJsI*&gR*}R&P^j@W5WAcdWg*| zg4_9wZHKlo3*ODK7i_6;&l&jagXM7yTvt`y00$HJlO=uJCAzgdcZm<#kT`@O3rveQ zDYfk38CYJx6I(xl2C35Ak_8G1OCNakGvF?G zn+)@uyhR1?$6{%2DFd?>_F7 zYxTtKUd9n(o7^Lw3j+Ko#>6xrJt=Wk8zK0-g((}wdAFS|VU`e&^5{ny^aomYmc+`C zIC&J$m`SMYd%atN&$L-Uj4SvI_6GP4l3a4`v3l_Xk;Wp_bEZC$7~M|St*Kh2sTxxm z;MnI!au%F=DP16&1ynEAfm^G|&J!ox4d=Xk$f zujljid_K3!=DGeUrPL2#WHx&dz?s9`*9p_e8eYF>9QzYQY<}5`NyoVmvxJ$V-eMLqy_zvQ(Z)8CY*5o(5=CBvhA*{N1*K%IO{B{`n}?%bxAv2=qyR!d zE+S+R??Y+=ylvMxuztkuau&PWj%{D;zze3m1sLhwJouK4>G}Fv@Kp)q&+hBZPJ4R6B5RvS_9Jpe_BCMmCc3^KKJdh66rGoXe;>e7 zzv5NMTqsrDzXW=}y_?0YH$MZV_$m^@7s!nODTOuID|g&0Tw${Gi#nVCEg>*vfcYP;79Gp+r8HG5&jiAXSCht>UE0bAd+t!$LkO0^D@H7lz? ze0F{Zvf5v3Gs~)_gx-YRjR8b3`~k2>ZqXSJ(Ga<_S6?aG#4GDVV;7C+w*f>&5OmfR zKO=!CBIv!wQVQ%vA>V2os)bOyf((2Ye9e|EMO6wdUHv@Tu{_Azv<1aYK&S7U(Iq4t zk}v*Y5;-t!+Ak||>rx$5Dcjk1mBWMJ!g^^6gc0@&A*ja(EfISV2wxLs z3%ne*>u@SlM{dAB=tzf@w#YSD%hv1hMGG@4F97ZfnSy!N`kOTU4INHW!w~QT_5X&4 zV`qH$oma%{y=lnSHVFYC3XKjs(nAAAG?eugbQCNfi^Gxu%&Hn|kVJ0x_vlWmbu!;- zLH28-1R|y^R?1?+@B`R=D$9>tu*&Lq2?cr~5N&LAhM-k}mOrxmA7sJ3&bP()-uMvW z+7bn(Oqhm0z+6@SP;Q$%S|H73s;m$DdQ>eMGLvO@itZArbE#IJY zJ>x93uL+N^>Cs3D)=qYiKfQ;_?CZ`W7(z$97&6Z_Zsvx#c(W^bfiy~YE?M!k4f-Ot;xIL~s`pFggxJrPWJ+WFLbQXa&T4P^GIEU;n%Bl7 z+3J81F5MvU%3qTyXn?u+M~c?{h4pAv>)kvUk+4VA+p%VywUnF0q|^n~*Iz&^QMMny zH#vx2_2B?Zw(yE)dWR+(a6S#k80}Mva;uAI6YwHj(0%wwmDd+wV(_PyL8BDbaKOW_ z%*jbv%gp;3B?c1&mC$NH+Wll&tTefv(oNj*uzx)k=Lb(SKsE;g2&fN198cE7cS>Ud zSB1L60@u_1FoXPrv653tWJEPPuMb!48=60C^it5GxjoDEaUnsv4g$n~S7$q~==r*| zugSSz@3Wf$QDJ4A0eH(_y1$(Oo)f!Q->2&^shn#=%##Y;0|ESVr;07sOs;iKq;^E zgLCZJ&x@9J#l&RY_%}-nw~-DKo*&cvQVqPD*JE8*QB4V9Hw+_(i$_f5T()VtU84DX zzv|=x<83_R!tW~7x8@43vh-@%7kNM3C9X!wnu#|V>O|8Gd|mj-I95<02FG1;Sqget z?9{=Wo8hnN=!X9n1+RvBiaXuRzTs-#kh&M^)2lm+h8`_OV1;8(MixASsgv4{4#mI4 ze-NalZA+Yo_Pxjbgo?hKM3n#N<#1KcvL@4l^5-Bx%Mibp(Gv8v^^CJKN3m$rj#$>i zz9_2h`Aj(l+&~vKe&*iIp8<#;ZaOZ2i<^NhF8pEs>!XwLpPY<>Mz}z$Qli>05`#`S zxSNz!@jjC>{cROuX(0v%xN-J{Q2rj`e0ce4{W(X{m!O-Wf~ZXS<_3uke^XcOWIsM3 zi6(#-1_)AWHDjnMP$TFDtB2iWW9jbSBcQdNLUwDnJhqOv7Pd1{F-AVCxz*B=?_+-$ zg{3bn;WSM(QA@vz=B;u2aPx6{f86-7o*Y-&^I8JTERfis?l1b~EIP=aLsvhsB~!?c zt@UdB2Xv%l5ul2@1H{djPl=XdB=Vu%~j^ic}BGT4>VK~UH z=K#_hXmu5Q7fYGKKFsQIz-duW)tv?4G_0=Pbv}3%ShdP(mW~mPQ*!{dIJvVm(88BA zeu9404%Kg|tBQcbl%b(=L(^M%jhfP8`GV{;AQ<*&*A1Te3Ou1TQVjK8No=Ka^rLYs zV{J}7#&5{!X{y4Z+VLL);6~tRi=R9C6xS^h< z5L@GnVox)HGjOr8unt2q2%TVgV>jFoN}JKO<_2O=nzRkVG|^CUx#_xxGpr-}u3Q7k z-VVhw_CCajmb3wu@3pXA!D-sZktSA@P2P}U>_!LTdW63)^&wb@+ zb_O{itMdh^lYZ3T66RzDhP1}KA-XQ%ZqfH=rD`nmvS>oFD@LFH5p7fGv)q+HRi7Cd z2DsBx2%oa${&#bBTdo9QrOO**R6KV?tpKdH27{14O5GxV}3N$H|@^5|DkQ> zx8sOLRWKemrxhXLG$I)?yR)6s6l98ORDDO`O65%~BYZ))jOZV8yzU|t!Ihe~_!8de zD)2I1kb>0VRE%yu&tR)AlopOvr-U)5KC{a$4LbU0DP6Zp_ks<%81_nEW z%neCCVYvYz3`B%rU*DVoih~h`NJ4dYe>e!EJ%Gs4G^*MPQ(=G8NBIf{xr zFt6!HqTEB^+;JTkP?vDqgDc_XmvFR(MrSx%3nnom~%=|745iVFnCL`WOZz9l|_5}I17|HqR+zvIvbCkO<{!i)FhC}=L z0@#)laVQ&79qaZI3W9V4v2yXSv6+ z>G1F0=RPCpgh~_ znlV}_V~sHAUOVJ+(&=U(v~$-3^TFoJ{xqs~B2(3qOj+ZBLb6l3?v)FLewR3!o)CL_ z4LPa3-H#d-kwMVnS6d=?&;jJjiM(8P+=E<)@tLy86Y|Xy@Nwc!z!WG_u)z>Iw*kP- zbC5QB&>rk9^#bC}Qz~J;dAAIJpU6;YCujyrTv(gn)4^9O2k>vg3Z<~ Voupg*;92xBtjulAs!V-i{s&!FWt{*3 literal 0 HcmV?d00001 diff --git a/docs/benchmark-results/peak_memory_mb.svg b/docs/benchmark-results/peak_memory_mb.svg new file mode 100644 index 000000000000..26638daeeff6 --- /dev/null +++ b/docs/benchmark-results/peak_memory_mb.svg @@ -0,0 +1,1811 @@ + + + + + + + + 2026-08-28T16:03:34.400749 + image/svg+xml + + + Matplotlib v3.11.1, https://matplotlib.org/ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + From 1babb3829fbbd8920958788f1bc645a802ea1df5 Mon Sep 17 00:00:00 2001 From: Bill Schnurr Date: Fri, 28 Aug 2026 16:11:56 -0700 Subject: [PATCH 5/5] Format generated benchmark dashboard Run the repository's pinned Prettier formatter after notebook dashboard generation and commit the formatted HTML output. --- build/benchmark/benchmark_history.ipynb | 10 + docs/benchmark-results/index.html | 261 +++++++++++++++++++++++- 2 files changed, 261 insertions(+), 10 deletions(-) diff --git a/build/benchmark/benchmark_history.ipynb b/build/benchmark/benchmark_history.ipynb index 350dbc67de11..28bdfcc939f7 100644 --- a/build/benchmark/benchmark_history.ipynb +++ b/build/benchmark/benchmark_history.ipynb @@ -464,10 +464,20 @@ "\"\"\"\n", "\n", "\n", + "def format_dashboard(path: Path) -> None:\n", + " formatter = \"pnpm.cmd\" if os.name == \"nt\" else \"pnpm\"\n", + " subprocess.run(\n", + " [formatter, \"exec\", \"prettier\", \"--write\", str(path)],\n", + " cwd=REPO_ROOT,\n", + " check=True,\n", + " )\n", + "\n", + "\n", "dashboard = dashboard_html(history, comparison)\n", "if EXPORT_ASSETS:\n", " dashboard_path = EXPORT_DIR / \"index.html\"\n", " dashboard_path.write_text(dashboard, encoding=\"utf-8\")\n", + " format_dashboard(dashboard_path)\n", " print(f\"Dashboard: {dashboard_path}\")\n", "else:\n", " display(HTML(dashboard))" diff --git a/docs/benchmark-results/index.html b/docs/benchmark-results/index.html index 921deed61f09..20279fb052c4 100644 --- a/docs/benchmark-results/index.html +++ b/docs/benchmark-results/index.html @@ -1,10 +1,251 @@ - - -Pyright benchmark history

Pyright benchmark history

-

Latest: 075161cd9160dbefff81e8ec4b28a3be4786bfcb Increase benchmark heap headroom

-

Collected 2026-08-20 02:07:10.843078+00:00

Execution time

Execution time trend chart

Peak memory

Peak memory trend chart
-

Latest compared with baseline

PackageMetricBaselineLatestChangeAssessment
ansibleExecution time21.66 seconds21.66 seconds+0.0%Stable
ansiblePeak memory1110.70 MiB1110.70 MiB+0.0%Stable
clickExecution time2.91 seconds2.91 seconds+0.0%Stable
clickPeak memory361.00 MiB361.00 MiB+0.0%Stable
homeassistantExecution time158.34 seconds158.34 seconds+0.0%Stable
homeassistantPeak memory6192.10 MiB6192.10 MiB+0.0%Stable
numpyExecution time43.44 seconds43.44 seconds+0.0%Stable
numpyPeak memory1850.00 MiB1850.00 MiB+0.0%Stable
pandasExecution time989.16 seconds989.16 seconds+0.0%Stable
pandasPeak memory4434.00 MiB4434.00 MiB+0.0%Stable
pytestExecution time12.57 seconds12.57 seconds+0.0%Stable
pytestPeak memory930.90 MiB930.90 MiB+0.0%Stable
requestsExecution time2.30 seconds2.30 seconds+0.0%Stable
requestsPeak memory347.20 MiB347.20 MiB+0.0%Stable
torchExecution time153.78 seconds153.78 seconds+0.0%Stable
torchPeak memory4891.90 MiB4891.90 MiB+0.0%Stable
transformersExecution time134.54 seconds134.54 seconds+0.0%Stable
transformersPeak memory5130.00 MiB5130.00 MiB+0.0%Stable
-
\ No newline at end of file + + + + + + Pyright benchmark history + + + +
+

Pyright benchmark history

+

Latest: 075161cd9160dbefff81e8ec4b28a3be4786bfcb Increase benchmark heap headroom

+

Collected 2026-08-20 02:07:10.843078+00:00

+
+
+
+

Execution time

+ Execution time trend chart +
+
+

Peak memory

+ Peak memory trend chart +
+
+

Latest compared with baseline

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
PackageMetricBaselineLatestChangeAssessment
ansibleExecution time21.66 seconds21.66 seconds+0.0%Stable
ansiblePeak memory1110.70 MiB1110.70 MiB+0.0%Stable
clickExecution time2.91 seconds2.91 seconds+0.0%Stable
clickPeak memory361.00 MiB361.00 MiB+0.0%Stable
homeassistantExecution time158.34 seconds158.34 seconds+0.0%Stable
homeassistantPeak memory6192.10 MiB6192.10 MiB+0.0%Stable
numpyExecution time43.44 seconds43.44 seconds+0.0%Stable
numpyPeak memory1850.00 MiB1850.00 MiB+0.0%Stable
pandasExecution time989.16 seconds989.16 seconds+0.0%Stable
pandasPeak memory4434.00 MiB4434.00 MiB+0.0%Stable
pytestExecution time12.57 seconds12.57 seconds+0.0%Stable
pytestPeak memory930.90 MiB930.90 MiB+0.0%Stable
requestsExecution time2.30 seconds2.30 seconds+0.0%Stable
requestsPeak memory347.20 MiB347.20 MiB+0.0%Stable
torchExecution time153.78 seconds153.78 seconds+0.0%Stable
torchPeak memory4891.90 MiB4891.90 MiB+0.0%Stable
transformersExecution time134.54 seconds134.54 seconds+0.0%Stable
transformersPeak memory5130.00 MiB5130.00 MiB+0.0%Stable
+
+
+ +