diff --git a/.github/workflows/cppreportcard.yml b/.github/workflows/cppreportcard.yml index a7cea97d..86f744b3 100644 --- a/.github/workflows/cppreportcard.yml +++ b/.github/workflows/cppreportcard.yml @@ -57,12 +57,18 @@ jobs: build/report/compile_commands.json build/report-consumer/compile_commands.json \ build/report-l6/compile_commands.json python3 -m unittest discover -s validation -p 'test_quality_tools.py' + - name: Prepare analyzer timing + id: timing + run: | + python3 validation/report_timing.py prepare --config .cppreportcard.json --directory build/report-timing - name: Generate complete quality report - if: ${{ !cancelled() && steps.inputs.outcome == 'success' }} + if: ${{ !cancelled() && steps.inputs.outcome == 'success' && steps.timing.outcome == 'success' }} uses: soulteary/cppreportcard-action@8de72292324308064276a8feca167d543828d218 # v1.0.4 + env: + REPORT_TIMING_DIR: ${{ github.workspace }}/build/report-timing with: mode: full - config: .cppreportcard.json + config: build/report-timing/config.json compile_commands: build/report-compile-commands.json output_dir: .github/cppreportcard install_tools: 'false' @@ -71,6 +77,11 @@ jobs: commit: 'false' - name: Require complete A+ analysis with zero diagnostics run: python3 validation/check_report_complete.py .github/cppreportcard/report.json --require-aplus + - name: Summarize analyzer timings + if: ${{ always() && steps.timing.outcome == 'success' }} + run: | + python3 validation/report_timing.py summarize --directory build/report-timing \ + --report .github/cppreportcard/report.json --output .github/cppreportcard/timing - name: Upload report and compilation scope if: always() uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 @@ -83,3 +94,6 @@ jobs: .github/cppreportcard/report.md .github/cppreportcard/badge.svg build/report-compile-commands.json + .github/cppreportcard/timing.json + .github/cppreportcard/timing.md + build/report-timing/records/ diff --git a/.github/workflows/release-packages.yml b/.github/workflows/release-packages.yml index 3f131552..73fee399 100644 --- a/.github/workflows/release-packages.yml +++ b/.github/workflows/release-packages.yml @@ -70,7 +70,6 @@ jobs: for row in targets['windows']: arm = row['arch'] == 'ARM64' row.update(runner='windows-11-vs2026-arm' if arm else 'windows-2022', - generator='Visual Studio 18 2026' if arm else 'Visual Studio 17 2022', cmake='4.2.3' if arm else '3.31.8', cmake_arch='arm64' if arm else 'x86_64') with open(os.environ['GITHUB_OUTPUT'], 'a') as output: @@ -103,39 +102,76 @@ jobs: ref: ${{ env.COMM_SHA }} path: linkerbot-transport-cpp token: ${{ secrets.LINKERBOT_COMM_TOKEN || github.token }} - - name: Pull native distribution image + - name: Prepare native distribution and build tools + id: native env: UBUNTU: ${{ matrix.ubuntu }} + EXPECTED_ARCH: ${{ matrix.arch }} run: | docker pull "ubuntu:$UBUNTU" docker image inspect "ubuntu:$UBUNTU" --format '{{index .RepoDigests 0}}' > "$RUNNER_TEMP/build-image.txt" - mkdir -p "$RUNNER_TEMP/release" + mkdir -p "$RUNNER_TEMP/release" "$RUNNER_TEMP/ccache" "$RUNNER_TEMP/catch2-install" + docker run -d --name linkerbot-release \ + -v "$GITHUB_WORKSPACE:/src" -v "$RUNNER_TEMP/release:/release" \ + -v "$RUNNER_TEMP/ccache:/cache/ccache" -v "$RUNNER_TEMP/catch2-install:/cache/catch2-install" \ + -w /src -e HAND_SHA -e COMM_SHA -e RELEASE_VERSION -e EXPECTED_ARCH \ + -e BUILD_IMAGE="$(cat "$RUNNER_TEMP/build-image.txt")" \ + -e PATH=/opt/release-tools/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin \ + -e CCACHE_DIR=/cache/ccache -e CCACHE_BASEDIR=/src -e CCACHE_COMPILERCHECK=content \ + -e CCACHE_MAXSIZE=1G -e CMAKE_BUILD_PARALLEL_LEVEL=4 \ + "$(cat "$RUNNER_TEMP/build-image.txt")" sleep infinity + docker exec linkerbot-release bash hand-cpp/validation/setup_linux_build.sh + echo "image=$(sha256sum "$RUNNER_TEMP/build-image.txt" | cut -d ' ' -f 1)" >> "$GITHUB_OUTPUT" + echo "toolchain=$(cat "$RUNNER_TEMP/release/toolchain-id")" >> "$GITHUB_OUTPUT" + - name: Restore compiler outputs + uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4 + with: + path: ${{ runner.temp }}/ccache + key: linux-ccache-v1-${{ matrix.distribution }}-${{ matrix.arch }}-Release-${{ steps.native.outputs.image }}-${{ steps.native.outputs.toolchain }}-${{ hashFiles('hand-cpp/validation/build_support.py') }}-${{ env.HAND_SHA }} + restore-keys: | + linux-ccache-v1-${{ matrix.distribution }}-${{ matrix.arch }}-Release-${{ steps.native.outputs.image }}-${{ steps.native.outputs.toolchain }}-${{ hashFiles('hand-cpp/validation/build_support.py') }}- + - name: Restore shared Catch2 installation + uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4 + with: + path: ${{ runner.temp }}/catch2-install + key: linux-catch2-v1-${{ matrix.distribution }}-${{ matrix.arch }}-Release-${{ steps.native.outputs.image }}-${{ steps.native.outputs.toolchain }}-${{ hashFiles('hand-cpp/validation/prepare_catch2.py', 'hand-cpp/validation/build_support.py', 'hand-cpp/validation/setup_linux_build.sh') }} + - name: Prepare one Catch2 for transport and hand tests + run: | + docker exec linkerbot-release ccache --zero-stats + docker exec linkerbot-release bash -euc ' + export LINKERBOT_CI_TOOLCHAIN="$(cat /release/toolchain-id)" + python hand-cpp/validation/prepare_catch2.py \ + --prefix /cache/catch2-install --work-dir /release/catch2-build \ + --config Release --jobs 4 --generator Ninja --compiler-cache ccache + ' - name: Build, test, package and relocate inside target userspace env: - EXPECTED_ARCH: ${{ matrix.arch }} EXPECTED_DISTRIBUTION: ${{ matrix.distribution }} run: | - docker run --rm -v "$GITHUB_WORKSPACE:/src" -v "$RUNNER_TEMP/release:/release" -w /src \ - -e HAND_SHA -e COMM_SHA -e RELEASE_VERSION -e EXPECTED_ARCH -e EXPECTED_DISTRIBUTION \ - -e BUILD_IMAGE="$(cat "$RUNNER_TEMP/build-image.txt")" \ - "$(cat "$RUNNER_TEMP/build-image.txt")" bash -euc ' - export DEBIAN_FRONTEND=noninteractive - apt-get update - apt-get install -y --no-install-recommends build-essential git ca-certificates python3 python3-venv ninja-build - test "$(uname -m)" = "$EXPECTED_ARCH" - python3 -m venv /opt/release-tools - /opt/release-tools/bin/pip install cmake==3.31.10 - export PATH=/opt/release-tools/bin:$PATH - git config --global --add safe.directory /src/hand-cpp - git config --global --add safe.directory /src/linkerbot-transport-cpp - python hand-cpp/validation/run_release_validation.py \ - --comm-source linkerbot-transport-cpp --work-dir /release \ - --hand-sha "$HAND_SHA" --comm-sha "$COMM_SHA" \ - --expected-version "$RELEASE_VERSION" --package-format TGZ \ - --expected-arch "$EXPECTED_ARCH" --expected-distribution "$EXPECTED_DISTRIBUTION" \ - --build-image "$BUILD_IMAGE" \ - --config Release --suite all --require-slcan --allow-downloads --jobs 4 - ' + docker exec -e EXPECTED_DISTRIBUTION linkerbot-release bash -euc ' + python hand-cpp/validation/run_release_validation.py \ + --comm-source linkerbot-transport-cpp --work-dir /release \ + --hand-sha "$HAND_SHA" --comm-sha "$COMM_SHA" \ + --expected-version "$RELEASE_VERSION" --package-format TGZ \ + --expected-arch "$EXPECTED_ARCH" --expected-distribution "$EXPECTED_DISTRIBUTION" \ + --build-image "$BUILD_IMAGE" \ + --config Release --suite all --require-slcan --allow-downloads --jobs 4 \ + --catch2-dir /cache/catch2-install/lib/cmake/Catch2 --generator Ninja --compiler-cache ccache + ' + - name: Record compiler cache statistics + if: always() + run: | + mkdir -p "$RUNNER_TEMP/release/evidence" + if docker exec linkerbot-release sh -c 'command -v ccache' >/dev/null 2>&1; then + docker exec linkerbot-release sh -c 'ccache --show-stats > /release/evidence/ccache-stats.txt' + docker exec linkerbot-release chmod -R a+rX /cache/ccache /cache/catch2-install /release/evidence + fi + - name: Remove build container + if: always() + run: | + if docker container inspect linkerbot-release >/dev/null 2>&1; then + docker rm -f linkerbot-release + fi - name: Upload diagnostic evidence if: always() uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 @@ -168,6 +204,12 @@ jobs: HAND_SHA: ${{ needs.targets.outputs.hand_sha }} COMM_SHA: ${{ needs.targets.outputs.comm_sha }} RELEASE_VERSION: ${{ needs.targets.outputs.version }} + # Ninja parallelizes files within the large tests target. VS generators + # ignore compiler launchers, so use the native MSVC environment below. + CMAKE_GENERATOR: Ninja + CMAKE_BUILD_PARALLEL_LEVEL: '4' + SCCACHE_CACHE_SIZE: 1G + SCCACHE_IGNORE_SERVER_IO_ERROR: '1' steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: @@ -181,10 +223,31 @@ jobs: token: ${{ secrets.LINKERBOT_COMM_TOKEN || github.token }} - name: Install native CMake run: ./hand-cpp/validation/setup_windows.ps1 -Version '${{ matrix.cmake }}' -Architecture '${{ matrix.cmake_arch }}' - - name: Require native runner architecture + - name: Initialize native MSVC and Ninja + id: native + run: ./hand-cpp/validation/setup_windows_build.ps1 -Architecture '${{ matrix.arch }}' + - name: Install compiler cache + uses: mozilla-actions/sccache-action@7d986dd989559c6ecdb630a3fd2557667be217ad # v0.0.9 + with: + version: v0.19.0 + - name: Restore compiler outputs + uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4 + with: + path: ${{ runner.temp }}/sccache + key: windows-sccache-v1-${{ matrix.arch }}-${{ matrix.configuration }}-${{ steps.native.outputs.toolchain }}-${{ hashFiles('hand-cpp/validation/build_support.py') }}-${{ env.HAND_SHA }} + restore-keys: | + windows-sccache-v1-${{ matrix.arch }}-${{ matrix.configuration }}-${{ steps.native.outputs.toolchain }}-${{ hashFiles('hand-cpp/validation/build_support.py') }}- + - name: Restore shared Catch2 installation + uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4 + with: + path: ${{ runner.temp }}/catch2-install + key: windows-catch2-v1-${{ matrix.arch }}-${{ matrix.configuration }}-${{ steps.native.outputs.toolchain }}-cmake${{ matrix.cmake }}-${{ hashFiles('hand-cpp/validation/prepare_catch2.py', 'hand-cpp/validation/build_support.py', 'hand-cpp/validation/setup_windows_build.ps1') }} + - name: Prepare one Catch2 for transport and hand tests run: | - $Expected = if ('${{ matrix.arch }}' -eq 'ARM64') { 'Arm64' } else { 'X64' } - if ([System.Runtime.InteropServices.RuntimeInformation]::OSArchitecture.ToString() -ne $Expected) { throw 'Native runner architecture mismatch' } + python hand-cpp/validation/prepare_catch2.py ` + --prefix "$env:RUNNER_TEMP/catch2-install" --work-dir "$env:RUNNER_TEMP/catch2-build" ` + --config '${{ matrix.configuration }}' --jobs 4 --generator Ninja --compiler-cache sccache + if ($LASTEXITCODE -ne 0) { throw 'Catch2 preparation failed' } - name: Build, test, package and relocate Windows SDK run: | python hand-cpp/validation/run_release_validation.py ` @@ -193,8 +256,15 @@ jobs: --expected-version $env:RELEASE_VERSION --package-format ZIP ` --expected-arch '${{ matrix.arch }}' --expected-distribution Windows ` --config '${{ matrix.configuration }}' --suite all --require-slcan --allow-downloads --jobs 4 ` - --generator '${{ matrix.generator }}' --architecture '${{ matrix.arch }}' --toolset 'v143,version=14.44' + --catch2-dir "$env:RUNNER_TEMP/catch2-install/lib/cmake/Catch2" --generator Ninja --compiler-cache sccache if ($LASTEXITCODE -ne 0) { throw 'Windows release acceptance failed' } + - name: Record compiler cache statistics + if: always() + run: | + New-Item -ItemType Directory -Force "$env:RUNNER_TEMP/release/evidence" | Out-Null + if (Get-Command sccache -ErrorAction SilentlyContinue) { + sccache --show-stats | Tee-Object -FilePath "$env:RUNNER_TEMP/release/evidence/sccache-stats.txt" + } - name: Upload diagnostic evidence if: always() uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 @@ -211,7 +281,7 @@ jobs: ${{ runner.temp }}/release/packages/hand/*.sha256 collect: - name: Verify complete twelve-package release set + name: Verify complete ten-package release set needs: [targets, linux, windows] runs-on: ubuntu-24.04 steps: @@ -231,9 +301,9 @@ jobs: run: | python3 validation/release_matrix.py release-assets \ --version "$VERSION" --hand-sha "$HAND_SHA" --comm-sha "$COMM_SHA" - cd release-assets - sha256sum --check SHA256SUMS.txt - test "$(find . -maxdepth 1 -type f | wc -l)" -eq 26 + (cd release-assets && sha256sum --check SHA256SUMS.txt) + expected=$(python3 -c 'import sys; sys.path.insert(0,"validation"); from release_matrix import load_targets; t=load_targets("ci/package-targets.json"); print(2*(len(t["linux"])+len(t["windows"]))+2)') + test "$(find release-assets -maxdepth 1 -type f | wc -l)" -eq "$expected" - uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 with: name: release-assets diff --git a/README.en.md b/README.en.md index fa986c98..c6891b6c 100644 --- a/README.en.md +++ b/README.en.md @@ -67,7 +67,7 @@ To reuse a prebuilt transport v0.3.4 SDK, configure with `cmake --preset prebuilt -DLINKERBOT_COMM_PREBUILT_PREFIX=/absolute/path/to/comm-sdk`. See the [build guide](docs/en/guides/validation.md) for source/prebuilt modes, archive verification and downstream paths. -Release target types follow transport: eight Ubuntu Release SDKs and four Windows x64/ARM64 Debug/Release SDKs. See [release artifacts](docs/en/guides/releases.md). +The release workflow builds eight Ubuntu Release SDKs and two Windows x64/ARM64 Release SDKs. See [release artifacts](docs/en/guides/releases.md). ### 2. Connect the CAN bus diff --git a/README.md b/README.md index fa986c98..c6891b6c 100644 --- a/README.md +++ b/README.md @@ -67,7 +67,7 @@ To reuse a prebuilt transport v0.3.4 SDK, configure with `cmake --preset prebuilt -DLINKERBOT_COMM_PREBUILT_PREFIX=/absolute/path/to/comm-sdk`. See the [build guide](docs/en/guides/validation.md) for source/prebuilt modes, archive verification and downstream paths. -Release target types follow transport: eight Ubuntu Release SDKs and four Windows x64/ARM64 Debug/Release SDKs. See [release artifacts](docs/en/guides/releases.md). +The release workflow builds eight Ubuntu Release SDKs and two Windows x64/ARM64 Release SDKs. See [release artifacts](docs/en/guides/releases.md). ### 2. Connect the CAN bus diff --git a/README.zh-CN.md b/README.zh-CN.md index 3970976c..06404c9a 100644 --- a/README.zh-CN.md +++ b/README.zh-CN.md @@ -62,7 +62,7 @@ hand SDK 0.3.12 的源码、预构建及安装消费者均要求 transport 0.3.x 使用 `cmake --preset prebuilt -DLINKERBOT_COMM_PREBUILT_PREFIX=/absolute/path/to/comm-sdk`。 源码与预构建两种方式、下载校验及下游路径见[构建指南](docs/zh/guides/validation.md)。 -发布目标类型与 transport 对齐:八份 Ubuntu Release SDK 和四份 Windows x64/ARM64 Debug/Release SDK,见[发布产物与校验](docs/zh/guides/releases.md)。 +发布工作流构建八份 Ubuntu Release SDK 和两份 Windows x64/ARM64 Release SDK,见[发布产物与校验](docs/zh/guides/releases.md)。 ### 2. 连接 CAN 总线 diff --git a/ci/package-targets.json b/ci/package-targets.json index ce90d200..9a97edda 100644 --- a/ci/package-targets.json +++ b/ci/package-targets.json @@ -51,24 +51,12 @@ } ], "windows": [ - { - "arch": "x64", - "configuration": "Debug", - "runtime": "MDd", - "toolset": "143" - }, { "arch": "x64", "configuration": "Release", "runtime": "MD", "toolset": "143" }, - { - "arch": "ARM64", - "configuration": "Debug", - "runtime": "MDd", - "toolset": "143" - }, { "arch": "ARM64", "configuration": "Release", diff --git a/docs/en/guides/quality.md b/docs/en/guides/quality.md index 05b7bb39..3bcbdab7 100644 --- a/docs/en/guides/quality.md +++ b/docs/en/guides/quality.md @@ -26,6 +26,14 @@ The numeric score does not measure test coverage or hardware compatibility. Every run uploads `report.json`, `report.md`, `badge.svg` and the merged compilation database as `cppreportcard-ubuntu-24.04-arm`. Reports are not automatically committed. +The Actions Summary and the same artifact also contain `timing.md`, `timing.json` +and raw per-invocation records: elapsed spans for each analyzer and the slowest 20 +clang-tidy invocations. Different configurations of one file remain separate. +Summed call time includes overlapping calls and is neither CPU time nor job duration. +Version probes are excluded; Cppcheck is measured as one database invocation. +Incomplete calls or missing report coverage are labeled explicitly. Instrumentation +preserves analyzer arguments, output, exit status, four-way parallelism and timeouts; +analysis scope and the quality gate are unchanged. The [report index](https://github.com/linker-bot/linkerbot-hand-cpp-sdk/blob/main/.github/cppreportcard/README.md) links to the workflow. ## Reproduce diff --git a/docs/en/guides/releases.md b/docs/en/guides/releases.md index 623c20ef..17f2db9a 100644 --- a/docs/en/guides/releases.md +++ b/docs/en/guides/releases.md @@ -9,18 +9,36 @@ explicit Python runtime policy while retaining Native defaults and existing C++ The [release notes](../../../RELEASE_NOTES.md#0312) describe compatibility, recorded validation, O6/L20/O30 hardware results and the untested shared-Session/fault timing scope. -## Targets aligned with transport +## Current workflow targets -The Hand release workflow uses the transport v0.3.4 target matrix: +The current Hand release workflow builds the following targets. Previously published assets remain unchanged: - Ubuntu 20.04, 22.04, 24.04 and 26.04, each with x86_64 and aarch64 (ARM64) Release TGZ SDKs: eight archives. -- Windows x64 and ARM64, each with Debug `/MDd` and Release `/MD` MSVC v143 ZIP SDKs: four archives. +- Windows x64 and ARM64, each with Release `/MD` MSVC v143 ZIP SDKs: two archives. Linux builds run in native architecture containers for each distribution, recording the image digest, using its default GCC, libstdc++ ABI1 and C++17. Windows builds use native runners and v143 14.44. Container checks do not certify native kernels or hardware; ARM64 SDK validation does not establish availability of proprietary ARM64 drivers. +Windows CI uses Ninja with the native MSVC v143 14.44 environment to compile up to +four source files concurrently. CI opts into sccache. Debug ZIP packages are no longer +built by the release workflow; local Debug builds remain supported. +GitHub Actions caches compiler outputs and a shared, pinned Catch2 3.8.1 installation; +transport and hand tests use the same installation. Caches are separated by target +architecture, configuration and compiler/Windows SDK identity. Catch2 additionally +keys on CMake and preparation scripts. Cold caches build normally; cache reuse does +not skip tests, package audits or source/architecture checks. Diagnostic artifacts +include compiler cache statistics and per-step validation durations. No additional +repository secrets or external cache service are required. + +Linux CI also uses Ninja, with ccache supplied by each target distribution. +The native container stays alive across preparation, cache restoration and validation. +Compiler outputs and the shared C++17 PIC Catch2 installation are cached separately, +partitioned by distribution, architecture, image digest and installed toolchain identity. +The default distribution GCC and libstdc++ ABI1 remain unchanged. All tests, package audits, +relocation checks and header/example compilation still run. Evidence includes ccache statistics. + Hand SDK packages contain both hand and paired transport static libraries, headers, CMake packages, documentation and read-only command-line examples. Test executables and proprietary drivers/libraries are excluded. Versions remain independent. Hand 0.3.12 requires transport 0.3.4 or newer in @@ -34,18 +52,40 @@ acceptance; the 0.3.12 scheduling evidence has the limited scope linked above. Transport archives now use the `linkerbot-transport-` prefix. Hand continues to consume the supported `linkerbot-comm` CMake package and CAN target aliases. +## Verify cache benefits + +Re-run a completed Release job on the same source and PR to compare a cold cache +with a restored cache. Check the cache-restoration logs, Catch2 preparation time, +compiler hit/miss counts and per-step durations in the diagnostic artifacts. +Keep the full tests and package checks enabled. Compare job execution time +separately from runner queue time; a new PR may have a different cache scope. +Changes to the toolchain or cache preparation scripts can intentionally cause a cold run. + +For source commit `c7ba44a`, [Release packages run 43827841694](https://github.com/YasenSS/linkerbot-hand-cpp-sdk/actions/runs/43827841694) +recorded the following native Windows Release comparisons: + +| Architecture | Cold job | Warm job | Compiler cache hits | Warm cache errors | +| --- | --- | --- | --- | --- | +| x64 | 19 min 34 s | 5 min 49 s | 771 / 793 (97.23%) | 0 | +| ARM64 | 18 min 50 s | 7 min 29 s | 771 / 793 (97.23%) | 0 | + +Both warm jobs reused the installed Catch2 and passed the complete acceptance. +These are same-source, same-PR measurements on hosted runners, not guaranteed timings +for changed sources or other runner/toolchain versions. Linux workflow timings must be +measured independently; the Windows comparison does not establish Linux speedups. + ## Names and checksums Examples use hand version `0.3.12`; compiler versions reflect the actual build: ```text linkerbot-hand-cpp-0.3.12-Linux-aarch64-ubuntu24.04-GNU13.3.0-libstdcxx-abi1-Release.tar.gz -linkerbot-hand-cpp-0.3.12-Windows-ARM64-msvc143-19.44.35229.0-Debug-MDd.zip +linkerbot-hand-cpp-0.3.12-Windows-ARM64-msvc143-19.44.35229.0-Release-MD.zip ``` Each archive has an adjacent `.sha256` file. The complete set also contains `release-manifest.json` -and `SHA256SUMS.txt`: 26 files in total. The manifest records both source SHAs, build identities, -archive sizes and verified object counts. Collection requires all twelve distinct targets and checks +and `SHA256SUMS.txt`: 22 files in total. The manifest records both source SHAs, build identities, +archive sizes and verified object counts. Collection requires all ten distinct targets and checks filenames, metadata and every static-library object's actual machine type. Missing archives, checksum mismatches, wrong architectures/CRT settings and source identity mismatches fail collection. @@ -55,8 +95,8 @@ and upload Actions artifacts. PR artifacts describe the tested merge commit. To attach the complete set to a GitHub Release, manually run **Release packages** on an existing matching `v` tag, supply the full transport SHA and version, and enable `create_draft`. The exact tagged hand commit must already have a successful main-push or manually dispatched -**C++ Report Card** run under the A+ gate. All twelve builds and collection must pass first. -The optional job creates a new **draft** with all 26 files and downloads them to verify their bytes. +**C++ Report Card** run under the A+ gate. All ten builds and collection must pass first. +The optional job creates a new **draft** with all 22 files and downloads them to verify their bytes. It rejects branch dispatches, moved/mismatched tags and existing releases; it never replaces assets. Default runs only upload Actions artifacts. Publishing the draft remains a separate maintainer action. Refer to actual attachments on [Releases](https://github.com/linker-bot/linkerbot-hand-cpp-sdk/releases) diff --git a/docs/zh/guides/quality.md b/docs/zh/guides/quality.md index 5f08faba..c14ca989 100644 --- a/docs/zh/guides/quality.md +++ b/docs/zh/guides/quality.md @@ -21,6 +21,12 @@ Action 使用 `fail_on: findings` 与 `threshold: 100`,另用独立校验器 每次上传 `cppreportcard-ubuntu-24.04-arm` 附件,包含 `report.json`、`report.md`、 `badge.svg` 和合并编译数据库。报告不自动提交到仓库。 +Actions Summary 和同一附件还包含 `timing.md`、`timing.json` 及每次调用的原始记录: +三个分析工具各自的耗时,以及最慢的 20 次 clang-tidy 调用。同一文件的不同配置分别保留。 +累计调用耗时包含并行重叠部分,不等于 CPU 时间或整个任务耗时。 +版本查询不计入分析耗时;Cppcheck 按一次完整数据库调用计时。 +未完成调用和缺失报告覆盖核对会明确标出。计时保留分析参数、输出、退出状态、四路并行 +和超时设置;分析范围与质量门禁不变。 [报告索引](https://github.com/linker-bot/linkerbot-hand-cpp-sdk/blob/main/.github/cppreportcard/README.md)链接到工作流。 ## 本地复现 diff --git a/docs/zh/guides/releases.md b/docs/zh/guides/releases.md index e403e0d8..d07b8ac4 100644 --- a/docs/zh/guides/releases.md +++ b/docs/zh/guides/releases.md @@ -7,17 +7,30 @@ 0.3.12 在显式 Python 运行时策略下增加触觉安全阶段边界的角度调度,保留 Native 默认行为及现有 C++ API。 [发行说明](../../../RELEASE_NOTES.md#0312)记录兼容性、验证结果、O6/L20/O30 实机证据及共享 Session、故障时序等未测范围。 -## 与 transport 对齐的目标 +## 当前工作流目标 -Hand 发布工作流使用与 transport v0.3.4 相同的目标矩阵: +当前 Hand 发布工作流构建以下目标;已发布的历史附件保持不变: - Ubuntu 20.04、22.04、24.04、26.04,各有 x86_64 和 aarch64(ARM64)的 Release TGZ,共八份。 -- Windows x64、ARM64,各有 Debug `/MDd` 和 Release `/MD` 的 MSVC v143 ZIP,共四份。 +- Windows x64、ARM64,各有 Release `/MD` 的 MSVC v143 ZIP,共两份。 Linux 在对应发行版的原生架构容器内构建,并记录镜像 digest;使用发行版默认 GCC、 libstdc++ ABI1 和 C++17。Windows 使用原生架构 runner 与 v143 14.44 工具集。 容器验证不代表原生内核或硬件验收;ARM64 SDK 通过不代表厂商提供了 ARM64 驱动。 +Windows CI 使用 Ninja 和原生 MSVC v143 14.44 环境,最多并行编译四个源文件。 +CI 显式启用 sccache;发布工作流不再构建 Debug ZIP,本地仍支持 Debug 构建。 +GitHub Actions 缓存编译结果,以及固定为 Catch2 3.8.1 的 +公共安装目录,供 transport 和 hand 测试共同使用。缓存按目标架构、构建配置和 +编译器/Windows SDK 身份隔离;Catch2 还区分 CMake 版本和准备脚本。 +缓存未命中时正常构建;缓存复用不跳过测试、打包审计或源码/架构检查。 +诊断附件包含编译缓存统计和验证各阶段耗时,无需新增仓库密钥或外部缓存服务。 + +Linux CI 同样使用 Ninja,ccache 由目标发行版提供;原生容器在工具准备、缓存恢复和验证期间持续运行。 +编译结果与公共 C++17 PIC Catch2 安装分别缓存,按发行版、架构、镜像 digest 和已安装工具链身份隔离。 +保留发行版默认 GCC 和 libstdc++ ABI1;所有测试、包审计、重定位检查及头文件/示例编译仍执行, +诊断附件增加 ccache 统计。 + 产物是开发 SDK,Hand 包包含 hand 与配套 transport 静态库、头文件、CMake 包和文档, 保留只读命令行示例;不包含测试程序、厂商驱动或厂商库。 两个项目版本独立。hand 0.3.12 的源码、预构建及安装消费者均要求 transport 0.3.x 且不低于 0.3.4。 @@ -28,18 +41,36 @@ libstdc++ ABI1 和 C++17。Windows 使用原生架构 runner 与 v143 14.44 工 transport 归档名现以 `linkerbot-transport-` 开头;Hand 继续使用受支持的 `linkerbot-comm` CMake 包及 CAN 目标别名。 +## 验证缓存收益 + +在相同源码、相同 PR 上复跑已完成的 Release 任务,对比首次构建和恢复缓存后的运行。 +检查缓存恢复日志、Catch2 准备时间、编译命中/未命中数,以及诊断附件中各阶段耗时。 +保留完整测试和包校验,单独比较任务执行时间与 runner 排队时间;新 PR 的缓存可见范围可能不同。 +工具链或缓存准备脚本变化时,首次未命中属于预期情况。 + +源码提交 `c7ba44a` 的 [Release packages 运行 43827841694](https://github.com/YasenSS/linkerbot-hand-cpp-sdk/actions/runs/43827841694) +得到以下 Windows 原生 Release 对比: + +| 架构 | 首次任务耗时 | 恢复缓存后耗时 | 编译缓存命中 | 恢复后缓存错误 | +| --- | --- | --- | --- | --- | +| x64 | 19 分 34 秒 | 5 分 49 秒 | 771 / 793(97.23%) | 0 | +| ARM64 | 18 分 50 秒 | 7 分 29 秒 | 771 / 793(97.23%) | 0 | + +两组复跑均复用了 Catch2 安装并通过完整验收。这是托管 runner 上同源码、同 PR 的测量, +不保证源码、runner 或工具链变化后仍有相同耗时。Linux 工作流需独立测量,不能由 Windows 结果推断提速。 + ## 命名与校验 以下示例中的 hand 版本为 `0.3.12`,编译器版本以实际构建结果为准: ```text linkerbot-hand-cpp-0.3.12-Linux-aarch64-ubuntu24.04-GNU13.3.0-libstdcxx-abi1-Release.tar.gz -linkerbot-hand-cpp-0.3.12-Windows-ARM64-msvc143-19.44.35229.0-Debug-MDd.zip +linkerbot-hand-cpp-0.3.12-Windows-ARM64-msvc143-19.44.35229.0-Release-MD.zip ``` 每份归档旁有 `.sha256`;完整集合另有 `release-manifest.json` 和 `SHA256SUMS.txt`, -共 26 个文件。清单记录 hand/transport 源码 SHA、构建身份、文件大小与对象检查数量。 -收集步骤要求十二个目标完整且不重复,核对文件名、元数据和每个静态库对象的实际架构, +共 22 个文件。清单记录 hand/transport 源码 SHA、构建身份、文件大小与对象检查数量。 +收集步骤要求十个目标完整且不重复,核对文件名、元数据和每个静态库对象的实际架构, 拒绝缺包、校验不符、跨架构对象、错误 CRT 和不同源码来源。 构建/发布文件的 PR 变更、版本标签或手动 dispatch 会触发验证并上传 Actions 附件。 @@ -48,7 +79,7 @@ PR 产物记录测试用合并提交。 如需将完整集合附加到 GitHub Release,在已有且匹配的 `v` 标签上手动运行 **Release packages**,填写完整 transport SHA 和版本,并启用 `create_draft`。 标签对应的 hand 提交须已有通过 A+ 门禁的主分支推送或手动触发的 **C++ Report Card** 运行。 -十二个平台构建及集合校验全部通过后,可选任务才会创建含 26 个文件的 **Release 草稿**, +十个平台构建及集合校验全部通过后,可选任务才会创建含 22 个文件的 **Release 草稿**, 并下载核对上传字节。分支触发、移动或不匹配的标签、已有 Release 均被拒绝,不覆盖附件。 默认运行只上传 Actions 附件;正式发布草稿由维护者另行操作。 下载可用性以 [Releases](https://github.com/linker-bot/linkerbot-hand-cpp-sdk/releases) 的实际附件为准。 diff --git a/validation/build_support.py b/validation/build_support.py index c00368c9..e05a19e4 100644 --- a/validation/build_support.py +++ b/validation/build_support.py @@ -1,4 +1,5 @@ """CMake configuration shared by offline validation entry points.""" +import os def add_build_arguments(parser): @@ -6,6 +7,8 @@ def add_build_arguments(parser): parser.add_argument('--generator') parser.add_argument('--architecture') parser.add_argument('--toolset') + parser.add_argument('--compiler-cache', choices=['sccache', 'ccache'], + help='Optional compiler launcher for Ninja/Makefile builds') def cmake_options(args): @@ -14,13 +17,21 @@ def cmake_options(args): value = getattr(args, name, None) if value: options.extend([flag, value]) + if args.compiler_cache: + generator = args.generator or os.environ.get('CMAKE_GENERATOR', '') + if not (generator.startswith('Ninja') or 'Makefiles' in generator): + raise ValueError('Compiler caching requires an explicit Ninja or Makefile generator') + options.extend([f'-DCMAKE_C_COMPILER_LAUNCHER={args.compiler_cache}', + f'-DCMAKE_CXX_COMPILER_LAUNCHER={args.compiler_cache}', + '-DCMAKE_POLICY_DEFAULT_CMP0141=NEW', + '-DCMAKE_MSVC_DEBUG_INFORMATION_FORMAT=$<$:Embedded>']) return options def validation_options(args): options = ['--config', args.config] - for name in ['generator', 'architecture', 'toolset']: + for name in ['generator', 'architecture', 'toolset', 'compiler_cache']: value = getattr(args, name, None) if value: - options.extend(['--' + name, value]) + options.extend(['--' + name.replace('_', '-'), value]) return options diff --git a/validation/cppcheck_report.py b/validation/cppcheck_report.py index 5acae870..5f949d41 100644 --- a/validation/cppcheck_report.py +++ b/validation/cppcheck_report.py @@ -1,10 +1,11 @@ """Run Cppcheck for the Linux report without changing Clang's compile arguments.""" import json from pathlib import Path -import subprocess import sys import tempfile +from report_timing import run_timed + def command_database(entries): result = [] @@ -32,7 +33,7 @@ def main(): command = ["cppcheck", "--platform=unix64", "-D__unix__=1", "-D__linux__=1", "-U_WIN32", "-U__APPLE__", "--library=std", "--library=" + str(library), "--suppress=missingIncludeSystem", *args] - return subprocess.run(command, check=False).returncode + return run_timed('cppcheck', command) if __name__ == "__main__": diff --git a/validation/prepare_catch2.py b/validation/prepare_catch2.py new file mode 100644 index 00000000..5ab2e156 --- /dev/null +++ b/validation/prepare_catch2.py @@ -0,0 +1,79 @@ +"""Build one pinned Catch2 installation for both transport and hand CI tests.""" +import argparse +import json +import os +from pathlib import Path +import platform +import subprocess +import sys +from types import SimpleNamespace + +from build_support import add_build_arguments, cmake_options + + +VERSION = '3.8.1' +REVISION = '2b60af89e23d28eefc081bc930831ee9d45ea58b' + + +def prepare(prefix, work, configuration, jobs, generator=None, compiler_cache=None): + prefix, work = Path(prefix).resolve(), Path(work).resolve() + options = cmake_options(SimpleNamespace(config=configuration, generator=generator, + architecture=None, toolset=None, compiler_cache=compiler_cache)) + identity = { + 'revision': REVISION, 'configuration': configuration, + 'cxx_standard': 17, + 'pic': True, + 'platform': sys.platform, 'generator': generator or os.environ.get('CMAKE_GENERATOR', ''), + 'compiler': os.environ.get('VCToolsVersion', ''), + 'sdk': os.environ.get('WindowsSDKVersion', ''), + 'architecture': os.environ.get('VSCMD_ARG_TGT_ARCH', ''), + 'host_architecture': platform.machine(), + 'toolchain_identity': os.environ.get('LINKERBOT_CI_TOOLCHAIN', ''), + 'compiler_cache': compiler_cache, + } + marker = prefix / 'ci-identity.json' + config = prefix / 'lib/cmake/Catch2/Catch2Config.cmake' + if marker.is_file() and json.loads(marker.read_text(encoding='utf-8')) == identity and config.is_file(): + print(f'Reusing Catch2 {VERSION} {configuration}: {prefix}', flush=True) + return config.parent + if prefix.exists() and any(prefix.iterdir()): + raise RuntimeError('Catch2 prefix is incomplete or incompatible; use a fresh cache key/prefix') + source, build = work / 'source', work / 'build' + work.mkdir(parents=True, exist_ok=True) + subprocess.run(['git', 'clone', '--depth', '1', '--branch', 'v' + VERSION, + 'https://github.com/catchorg/Catch2.git', str(source)], check=True) + actual = subprocess.check_output(['git', '-C', str(source), 'rev-parse', 'HEAD'], text=True).strip() + if actual != REVISION: + raise RuntimeError('Catch2 tag does not match the pinned source revision') + subprocess.run(['cmake', '-S', str(source), '-B', str(build), + *options, f'-DCMAKE_INSTALL_PREFIX={prefix}', + '-DCMAKE_INSTALL_LIBDIR=lib', '-DBUILD_SHARED_LIBS=OFF', '-DCATCH_BUILD_TESTING=OFF', + '-DCMAKE_POSITION_INDEPENDENT_CODE=ON', + '-DCMAKE_CXX_STANDARD=17', '-DCMAKE_CXX_STANDARD_REQUIRED=ON', '-DCMAKE_CXX_EXTENSIONS=OFF', + '-DCATCH_INSTALL_DOCS=OFF', '-DCATCH_INSTALL_EXTRAS=ON', + '-DCMAKE_MSVC_RUNTIME_LIBRARY=MultiThreaded$<$:Debug>DLL'], check=True) + subprocess.run(['cmake', '--build', str(build), '--config', configuration, + '--parallel', str(jobs)], check=True) + subprocess.run(['cmake', '--install', str(build), '--config', configuration], check=True) + if not config.is_file(): + raise RuntimeError('Catch2 installation is missing its CMake package') + marker.write_text(json.dumps(identity, sort_keys=True) + '\n', encoding='utf-8') + return config.parent + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('--prefix', type=Path, required=True) + parser.add_argument('--work-dir', type=Path, required=True) + add_build_arguments(parser) + parser.add_argument('--jobs', type=int, default=4) + args = parser.parse_args() + if args.jobs < 1: + parser.error('--jobs must be positive') + if args.architecture or args.toolset: + parser.error('Use a native compiler environment rather than generator architecture/toolset overrides') + prepare(args.prefix, args.work_dir, args.config, args.jobs, args.generator, args.compiler_cache) + + +if __name__ == '__main__': + main() diff --git a/validation/publish_release.py b/validation/publish_release.py index 8091d8e2..23ca9c45 100644 --- a/validation/publish_release.py +++ b/validation/publish_release.py @@ -46,10 +46,10 @@ def verify_assets(assets, targets, version, hand_sha, comm_sha): with tempfile.TemporaryDirectory() as temporary: copied = Path(temporary) / 'assets' shutil.copytree(assets, copied) - collect(copied, targets, version, hand_sha, comm_sha) + manifest = collect(copied, targets, version, hand_sha, comm_sha) checked = {p.name: hashlib.sha256(p.read_bytes()).hexdigest() for p in copied.iterdir() if p.is_file()} - if len(original) != 26 or checked != original: - raise ValueError('Release assets differ from the validated twelve-package set') + if len(original) != 2 * len(manifest['packages']) + 2 or checked != original: + raise ValueError('Release assets differ from the validated package set') def main(): @@ -85,8 +85,8 @@ def get(path, allow_missing=False): with tempfile.TemporaryDirectory() as temporary: notes = Path(temporary) / 'notes.md' notes.write_text( - 'Eight native Ubuntu 20.04/22.04/24.04/26.04 x86_64/ARM64 Release SDKs and four ' - 'Windows x64/ARM64 Debug/Release SDKs.\n\n' + 'Eight native Ubuntu 20.04/22.04/24.04/26.04 x86_64/ARM64 Release SDKs and two ' + 'Windows x64/ARM64 Release SDKs.\n\n' f'Hand source: `{args.hand_sha}`; transport source: `{args.comm_sha}`.\n\n' f'A+ quality gate: {quality_url}\n\n' 'See release-manifest.json and SHA256SUMS.txt for build identity and checksums.\n', encoding='utf-8') @@ -100,7 +100,7 @@ def get(path, allow_missing=False): actual = {p.name: hashlib.sha256(p.read_bytes()).hexdigest() for p in downloaded.iterdir()} if actual != expected: raise RuntimeError('Uploaded draft bytes do not match validated assets') - print('Draft created; all 26 uploaded assets match the validated bytes') + print(f'Draft created; all {len(expected)} uploaded assets match the validated bytes') if __name__ == '__main__': diff --git a/validation/release_matrix.py b/validation/release_matrix.py index e0defb6c..a3993ae5 100644 --- a/validation/release_matrix.py +++ b/validation/release_matrix.py @@ -19,9 +19,9 @@ def load_targets(path): raise ValueError('Expected exactly eight Ubuntu targets') if any(r['distribution'] != 'ubuntu' + r['ubuntu'] for r in linux): raise ValueError('Ubuntu distribution identity mismatch') - if len(windows) != 4 or {(r['arch'], r['configuration']) for r in windows} != { - (a, c) for a in ('x64', 'ARM64') for c in ('Debug', 'Release')}: - raise ValueError('Expected exactly four Windows targets') + if len(windows) != 2 or {(r['arch'], r['configuration']) for r in windows} != { + (a, 'Release') for a in ('x64', 'ARM64')}: + raise ValueError('Expected exactly two Windows Release targets') if any(r['runtime'] != {'Debug': 'MDd', 'Release': 'MD'}[r['configuration']] or r['toolset'] != '143' for r in windows): raise ValueError('Windows runtime/toolset identity mismatch') @@ -98,8 +98,8 @@ def collect(directory, targets_path, version, hand_sha, comm_sha): expected |= {('Windows', 'Windows', r['arch'], r['configuration']) for r in targets['windows']} seen, packages = set(), [] archives = sorted([*directory.glob('*.tar.gz'), *directory.glob('*.zip')]) - if len(archives) != 12: - raise ValueError('Release requires exactly twelve hand SDK archives') + if len(archives) != len(expected): + raise ValueError(f'Release requires exactly {len(expected)} hand SDK archives') for archive in archives: with tempfile.TemporaryDirectory() as temporary: prefix, sha = unpack_archive(archive, Path(temporary) / 'sdk') diff --git a/validation/report_timing.py b/validation/report_timing.py new file mode 100644 index 00000000..e11af6bc --- /dev/null +++ b/validation/report_timing.py @@ -0,0 +1,173 @@ +"""Measure Report Card tools without mixing timing output into diagnostics.""" +import argparse +import ctypes +import json +import os +from pathlib import Path +import shlex +import signal +import subprocess +import sys +import time +import uuid + + +TOOLS = ('clang-format', 'cppcheck', 'clang-tidy') + + +def child_setup(): + # The action kills a timed-out wrapper. Kill its analyzer too, including + # the race where the wrapper exited just before prctl was installed. + parent = os.getppid() + libc = ctypes.CDLL(None, use_errno=True) + if libc.prctl(1, signal.SIGKILL, 0, 0, 0) != 0: # PR_SET_PDEATHSIG + raise OSError(ctypes.get_errno(), 'prctl(PR_SET_PDEATHSIG) failed') + if os.getppid() != parent or parent == 1: + os.kill(os.getpid(), signal.SIGKILL) + + +def write_record(path, record): + temporary = path.with_suffix('.tmp') + temporary.write_text(json.dumps(record, ensure_ascii=False) + '\n', encoding='utf-8') + temporary.replace(path) + + +def run_timed(tool, command): + directory = os.environ.get('REPORT_TIMING_DIR') + if not directory: + return subprocess.run(command, check=False).returncode + records = Path(directory) / 'records' + records.mkdir(parents=True, exist_ok=True) + path = records / (uuid.uuid4().hex + '.json') + arguments = command[1:] + phase = 'version' if '--version' in arguments else 'analysis' + source = arguments[-1] if phase == 'analysis' and tool != 'cppcheck' else None + if source: + source = os.path.relpath(source, Path.cwd()) + record = {'tool': tool, 'phase': phase, 'file': source, 'command': command, + 'started_at': time.time(), 'start': time.monotonic(), 'end': None, 'exit': None} + write_record(path, record) + try: + process = subprocess.run(command, check=False, + preexec_fn=child_setup if sys.platform == 'linux' else None) + record['exit'] = process.returncode + return process.returncode + finally: + record['end'] = time.monotonic() + write_record(path, record) + + +def prepare(config, directory): + directory = directory.resolve() + directory.mkdir(parents=True, exist_ok=True) + data = json.loads(config.read_text(encoding='utf-8')) + # Cppcheck already has an adapter; time the native invocation inside it. + for tool in ('clang-format', 'clang-tidy'): + wrapper = directory / tool + command = [sys.executable, str(Path(__file__).resolve()), 'run', tool, data['tools'][tool]] + wrapper.write_text('#!/bin/sh\nexec ' + shlex.join(command) + ' "$@"\n', encoding='utf-8') + wrapper.chmod(0o755) + data['tools'][tool] = str(wrapper) + (directory / 'config.json').write_text(json.dumps(data, indent=2) + '\n', encoding='utf-8') + + +def summarize(records, report=None): + analysis = [record for record in records if record['phase'] == 'analysis'] + tools = [] + for tool in TOOLS: + rows = [r for r in analysis if r['tool'] == tool] + finished = [r for r in rows if r['end'] is not None] + tools.append({'tool': tool, 'calls': len(rows), 'finished': len(finished), + 'failures': sum(r['exit'] != 0 for r in finished), + 'wall_seconds': round(max(r['end'] for r in finished) - min(r['start'] for r in rows), 3) + if finished else None, + 'summed_seconds': round(sum(r['end'] - r['start'] for r in finished), 3)}) + coverage_errors = [] + if report is not None: + checks = {check['tool']: check for check in report.get('checks', [])} + for row in tools: + check = checks.get(row['tool'], {}) + expected = 1 if row['tool'] == 'cppcheck' else check.get('total') + if not check or row['calls'] != expected or row['finished'] != row['calls']: + coverage_errors.append(row['tool'] + ': timing does not cover the reported analysis') + slowest = sorted((dict(file=r['file'], seconds=round(r['end'] - r['start'], 3), + exit=r['exit'], command=r['command']) + for r in analysis if r['tool'] == 'clang-tidy' and r['end'] is not None), + key=lambda row: row['seconds'], reverse=True)[:20] + return {'tools': tools, 'slowest_clang_tidy': slowest, 'coverage_errors': coverage_errors, + 'report_available': report is not None, + 'unfinished_calls': sum(r['end'] is None for r in analysis), + 'version_calls': sum(r['phase'] == 'version' for r in records)} + + +def markdown(summary): + lines = ['## Report Card analyzer timings', '', + 'Wall span is elapsed time from the first recorded call to the last completion of each tool.', + 'Summed call time includes overlapping work; it is not CPU time or job duration.', + 'Version probes are excluded. Cppcheck is timed as one complete database invocation.', '', + '| Tool | Calls / finished | Failures | Wall span (s) | Summed call time (s) |', + '| --- | ---: | ---: | ---: | ---: |'] + for row in summary['tools']: + wall = '-' if row['wall_seconds'] is None else f"{row['wall_seconds']:.3f}" + lines.append(f"| {row['tool']} | {row['calls']} / {row['finished']} | {row['failures']} | " + f"{wall} | {row['summed_seconds']:.3f} |") + lines += ['', '### Slowest clang-tidy invocations', '', + 'Each invocation is retained separately, including different configurations of the same file.', '', + '| File | Seconds | Exit |', '| --- | ---: | ---: |'] + for row in summary['slowest_clang_tidy']: + name = row['file'].replace('|', '|').replace('\n', ' ') + lines.append(f"| {name} | {row['seconds']:.3f} | {row['exit']} |") + if summary['unfinished_calls']: + lines += ['', f"Unfinished invocations: {summary['unfinished_calls']}. Timings are partial."] + if not summary['report_available']: + lines += ['', 'No report is available; timing coverage has not been verified.'] + if summary['coverage_errors']: + lines += ['', *summary['coverage_errors']] + return '\n'.join(lines) + '\n' + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + commands = parser.add_subparsers(dest='operation', required=True) + setup = commands.add_parser('prepare') + setup.add_argument('--config', type=Path, required=True) + setup.add_argument('--directory', type=Path, required=True) + run = commands.add_parser('run') + run.add_argument('tool', choices=TOOLS) + run.add_argument('command', nargs=argparse.REMAINDER) + summary = commands.add_parser('summarize') + summary.add_argument('--directory', type=Path, required=True) + summary.add_argument('--output', type=Path, required=True) + summary.add_argument('--report', type=Path) + args = parser.parse_args() + if args.operation == 'prepare': + prepare(args.config, args.directory) + elif args.operation == 'run': + if not args.command: + parser.error('run requires an analyzer command') + code = run_timed(args.tool, args.command) + if code < 0: + # Preserve signal termination as well as ordinary exit codes. + if -code not in (signal.SIGKILL, signal.SIGSTOP): + signal.signal(-code, signal.SIG_DFL) + os.kill(os.getpid(), -code) + return code + else: + rows = [json.loads(path.read_text(encoding='utf-8')) + for path in sorted((args.directory / 'records').glob('*.json'))] + report = json.loads(args.report.read_text(encoding='utf-8')) if args.report and args.report.is_file() else None + result = summarize(rows, report) + args.output.parent.mkdir(parents=True, exist_ok=True) + args.output.with_suffix('.json').write_text(json.dumps(result, indent=2) + '\n', encoding='utf-8') + content = markdown(result) + args.output.with_suffix('.md').write_text(content, encoding='utf-8') + print(content, end='') + if os.environ.get('GITHUB_STEP_SUMMARY'): + with Path(os.environ['GITHUB_STEP_SUMMARY']).open('a', encoding='utf-8') as output: + output.write(content) + return int(bool(result['coverage_errors'])) + return 0 + + +if __name__ == '__main__': + sys.exit(main()) diff --git a/validation/run_release_validation.py b/validation/run_release_validation.py index 6ab1690f..2eda59da 100644 --- a/validation/run_release_validation.py +++ b/validation/run_release_validation.py @@ -11,6 +11,7 @@ import subprocess import sys import tempfile +import time from pathlib import Path from build_support import add_build_arguments, cmake_options, validation_options @@ -62,12 +63,16 @@ def save(): def run(name, command, cwd=None): print(f'[{args.config}/{args.suite}] {name}', flush=True) + started = time.monotonic() logfile = evidence / (name + '.log') with logfile.open('w', encoding='utf-8') as output: output.write(json.dumps([str(x) for x in command], ensure_ascii=False) + '\n') output.flush() process = subprocess.run([str(x) for x in command], cwd=cwd, stdout=output, stderr=subprocess.STDOUT) - report['steps'].append({'step': name, 'exit': process.returncode, 'log': logfile.name}) + elapsed = round(time.monotonic() - started, 3) + report['steps'].append({'step': name, 'exit': process.returncode, 'log': logfile.name, + 'seconds': elapsed}) + print(f'[{args.config}/{args.suite}] {name}: {elapsed:.3f}s', flush=True) save() if process.returncode: print(logfile.read_text(encoding='utf-8', errors='replace')[-12000:]) diff --git a/validation/setup_linux_build.sh b/validation/setup_linux_build.sh new file mode 100644 index 00000000..399c8175 --- /dev/null +++ b/validation/setup_linux_build.sh @@ -0,0 +1,22 @@ +#!/usr/bin/env bash +# Run inside the persistent native release container, before restoring caches. +set -euo pipefail +export DEBIAN_FRONTEND=noninteractive +apt-get update +apt-get install -y --no-install-recommends \ + build-essential git ca-certificates python3 python3-venv ninja-build ccache +python3 -m venv /opt/release-tools +/opt/release-tools/bin/pip install cmake==3.31.10 +test "$(uname -m)" = "$EXPECTED_ARCH" +git config --global --add safe.directory /src/hand-cpp +git config --global --add safe.directory /src/linkerbot-transport-cpp + +# Include the image digest in the Actions key, and fingerprint the packages +# installed on top of that image as well as the actual compiler binary. +{ + dpkg-query -W -f='${Package}=${Version}\n' | LC_ALL=C sort + sha256sum "$(readlink -f /usr/bin/g++)" + /opt/release-tools/bin/cmake --version + ninja --version + ccache --version +} | sha256sum | cut -d ' ' -f 1 > /release/toolchain-id diff --git a/validation/setup_windows_build.ps1 b/validation/setup_windows_build.ps1 new file mode 100644 index 00000000..7355cc8c --- /dev/null +++ b/validation/setup_windows_build.ps1 @@ -0,0 +1,66 @@ +param( + [Parameter(Mandatory)][ValidateSet('x64', 'ARM64')][string]$Architecture +) +$ErrorActionPreference = 'Stop' +$NativeArch = if ($Architecture -eq 'ARM64') { 'arm64' } else { 'x64' } +$Expected = if ($Architecture -eq 'ARM64') { 'Arm64' } else { 'X64' } +if ([System.Runtime.InteropServices.RuntimeInformation]::OSArchitecture.ToString() -ne $Expected) { + throw 'Native runner architecture mismatch' +} + +# Ninja does not select a VS toolset with -A/-T. Import the same native v143 +# 14.44 environment explicitly, including the SDK include and library paths. +$VsWhere = "${env:ProgramFiles(x86)}/Microsoft Visual Studio/Installer/vswhere.exe" +$Installation = & $VsWhere -latest -products '*' -property installationPath +if ($LASTEXITCODE -ne 0 -or -not $Installation) { throw 'Visual Studio discovery failed' } +$DevCmd = Join-Path $Installation 'Common7/Tools/VsDevCmd.bat' +# Pass cmd.exe its raw command line. PowerShell's native argument conversion +# otherwise changes the nested quotes around a batch path containing spaces. +$StartInfo = [System.Diagnostics.ProcessStartInfo]::new() +$StartInfo.FileName = $env:COMSPEC +$StartInfo.Arguments = "/d /s /c `"`"$DevCmd`" -no_logo -arch=$NativeArch -host_arch=$NativeArch -vcvars_ver=14.44 && set`"" +$StartInfo.UseShellExecute = $false +$StartInfo.RedirectStandardOutput = $true +$StartInfo.RedirectStandardError = $true +$Process = [System.Diagnostics.Process]::Start($StartInfo) +$OutputTask = $Process.StandardOutput.ReadToEndAsync() +$ErrorTask = $Process.StandardError.ReadToEndAsync() +$Process.WaitForExit() +if ($Process.ExitCode -ne 0) { + throw "MSVC v143 14.44 environment initialization failed: $($ErrorTask.GetAwaiter().GetResult())" +} +$Environment = $OutputTask.GetAwaiter().GetResult() -split '\r?\n' +$Names = @('PATH', 'INCLUDE', 'LIB', 'LIBPATH', 'VCToolsVersion', 'VCToolsInstallDir', + 'WindowsSdkDir', 'WindowsSDKVersion', 'WindowsSdkBinPath', 'UniversalCRTSdkDir', + 'UCRTVersion', 'VSINSTALLDIR', 'VCINSTALLDIR', 'VSCMD_ARG_TGT_ARCH', 'VSCMD_ARG_HOST_ARCH') +foreach ($Line in $Environment) { + $Pair = $Line -split '=', 2 + if ($Pair.Count -eq 2 -and $Names -contains $Pair[0]) { + [Environment]::SetEnvironmentVariable($Pair[0], $Pair[1], 'Process') + "$($Pair[0])=$($Pair[1])" >> $env:GITHUB_ENV + } +} +if ($env:VCToolsVersion -notlike '14.44.*' -or $env:VSCMD_ARG_TGT_ARCH -ne $NativeArch -or + $env:VSCMD_ARG_HOST_ARCH -ne $NativeArch) { throw 'Unexpected MSVC toolset or architecture' } +$CompilerHash = (Get-FileHash (Get-Command cl.exe).Source -Algorithm SHA256).Hash.ToLowerInvariant() +"toolchain=$($env:VCToolsVersion)-$($env:WindowsSDKVersion.TrimEnd('\'))-$CompilerHash" >> $env:GITHUB_OUTPUT +"SCCACHE_DIR=$env:RUNNER_TEMP/sccache" >> $env:GITHUB_ENV + +# Use a pinned native Ninja binary on both runner architectures. +$NinjaVersion = '1.13.2' +if ($Architecture -eq 'ARM64') { + $Archive = 'ninja-winarm64.zip' + $Checksum = 'e52f0bdef9dfb1003229dbd6508a508c4073fd017247002adc66e5e806cb0391' +} else { + $Archive = 'ninja-win.zip' + $Checksum = '07fc8261b42b20e71d1720b39068c2e14ffcee6396b76fb7a795fb460b78dc65' +} +$Download = Join-Path $env:RUNNER_TEMP $Archive +Invoke-WebRequest "https://github.com/ninja-build/ninja/releases/download/v$NinjaVersion/$Archive" ` + -OutFile $Download -MaximumRetryCount 5 -RetryIntervalSec 5 -TimeoutSec 120 +if ((Get-FileHash $Download -Algorithm SHA256).Hash -ne $Checksum) { throw 'Ninja SHA256 mismatch' } +$Destination = Join-Path $env:RUNNER_TEMP 'linkerbot-ninja' +Expand-Archive $Download -DestinationPath $Destination -Force +"$Destination" >> $env:GITHUB_PATH +& "$Destination/ninja.exe" --version +if ($LASTEXITCODE -ne 0) { throw 'Ninja execution failed' } diff --git a/validation/test_quality_tools.py b/validation/test_quality_tools.py index 67f3be1f..50c0ff72 100644 --- a/validation/test_quality_tools.py +++ b/validation/test_quality_tools.py @@ -1,11 +1,19 @@ import copy +from concurrent.futures import ThreadPoolExecutor import json +import os from pathlib import Path +import selectors +import signal +import subprocess +import sys import tempfile +import time import unittest from check_report_complete import validate from prepare_report_db import prepare from cppcheck_report import command_database +from report_timing import prepare as prepare_timing, summarize, markdown class QualityToolsTests(unittest.TestCase): @@ -86,6 +94,123 @@ def test_database_keeps_sdk_units_and_excludes_dependencies(self): with self.assertRaises(ValueError): prepare(root, [db]) +class ReportTimingTests(unittest.TestCase): + script = Path(__file__).with_name('report_timing.py').resolve() + + def invoke(self, directory, *command): + return subprocess.run([sys.executable, str(self.script), 'run', 'clang-tidy', *command], + env={**os.environ, 'REPORT_TIMING_DIR': str(directory)}, + capture_output=True, timeout=10) + + def records(self, directory): + return [json.loads(path.read_text()) for path in (Path(directory) / 'records').glob('*.json')] + + def test_binary_output_arguments_and_nonzero_exit_are_preserved(self): + with tempfile.TemporaryDirectory() as tmp: + code = 'import os,sys; os.write(1,b"out\\xff"); os.write(2,b"err\\xfe"); sys.exit(7)' + result = self.invoke(tmp, sys.executable, '-c', code, 'space and 中文.cpp') + self.assertEqual((result.stdout, result.stderr, result.returncode), (b'out\xff', b'err\xfe', 7)) + row, = self.records(tmp) + self.assertEqual(row['command'], [sys.executable, '-c', code, 'space and 中文.cpp']) + self.assertEqual(row['file'], 'space and 中文.cpp') + self.assertEqual(row['exit'], 7) + self.assertGreater(row['end'], row['start']) + + def test_generated_wrapper_only_changes_tool_paths(self): + with tempfile.TemporaryDirectory(prefix='timing space ') as tmp: + root = Path(tmp) + config = root / 'original.json' + original = {'tools': {'clang-format': sys.executable, 'clang-tidy': sys.executable, + 'cppcheck': 'validation/cppcheck-report.sh'}, + 'jobs': 4, 'timeout_seconds': 300, 'exclude': ['build']} + config.write_text(json.dumps(original)) + prepare_timing(config, root / 'wrappers with spaces') + prepared = json.loads((root / 'wrappers with spaces/config.json').read_text()) + for tool in ('clang-format', 'clang-tidy'): + result = subprocess.run([prepared['tools'][tool], '-c', + 'import sys; print(repr(sys.argv[1:]))', 'a b', '中文'], + env={**os.environ, 'REPORT_TIMING_DIR': tmp}, + capture_output=True, timeout=10) + self.assertEqual(result.returncode, 0, result.stderr) + self.assertEqual(result.stdout.decode().strip(), repr(['a b', '中文'])) + prepared['tools'][tool] = original['tools'][tool] + self.assertEqual(prepared, original) + self.assertEqual(json.loads(config.read_text()), original) + + def test_parallel_calls_do_not_overwrite_records_and_versions_are_excluded(self): + with tempfile.TemporaryDirectory() as tmp: + with ThreadPoolExecutor(max_workers=4) as pool: + results = list(pool.map(lambda i: self.invoke(tmp, sys.executable, '-c', 'pass', str(i)), range(8))) + self.assertTrue(all(result.returncode == 0 for result in results)) + self.assertEqual(self.invoke(tmp, sys.executable, '--version').returncode, 0) + rows = self.records(tmp) + self.assertEqual(len(rows), 9) + summary = summarize(rows) + self.assertEqual(summary['version_calls'], 1) + self.assertEqual(next(r for r in summary['tools'] if r['tool'] == 'clang-tidy')['finished'], 8) + + def test_summary_handles_overlap_duplicate_files_failure_and_partial_records(self): + def row(tool, start, end, code=0): + return dict(tool=tool, phase='analysis', file='same.cpp', start=start, end=end, + exit=code, command=[tool, str(start), 'same.cpp']) + rows = [row('clang-tidy', 0, 5), row('clang-tidy', 1, 4, 2), + row('clang-format', 0, 1), row('cppcheck', 1, 2)] + report = {'checks': [{'tool': tool, 'total': 2 if tool == 'clang-tidy' else 1} + for tool in ('clang-format', 'cppcheck', 'clang-tidy')]} + result = summarize(rows, report) + self.assertEqual(result['coverage_errors'], []) + tidy = result['tools'][2] + self.assertEqual((tidy['wall_seconds'], tidy['summed_seconds'], tidy['failures']), (5, 8, 1)) + self.assertEqual([r['seconds'] for r in result['slowest_clang_tidy']], [5, 3]) + rows.append(row('clang-tidy', 6, None, None)) + result = summarize(rows, report) + self.assertEqual(result['unfinished_calls'], 1) + self.assertTrue(result['coverage_errors']) + self.assertIn('Timings are partial', markdown(result)) + self.assertIn('coverage has not been verified', markdown(summarize([]))) + + @unittest.skipUnless(os.name == 'posix', 'requires POSIX signal exit status') + def test_child_signal_exit_is_preserved(self): + with tempfile.TemporaryDirectory() as tmp: + result = self.invoke(tmp, sys.executable, '-c', 'import os,signal; os.kill(os.getpid(), signal.SIGKILL)') + self.assertEqual(result.returncode, -signal.SIGKILL) + self.assertEqual(self.records(tmp)[0]['exit'], -signal.SIGKILL) + + @unittest.skipUnless(sys.platform == 'linux', 'CI timeout cleanup uses Linux parent-death signals') + def test_killed_wrapper_does_not_leave_analyzer_running(self): + with tempfile.TemporaryDirectory() as tmp: + code = 'import os,sys; print(os.getpid(), flush=True); sys.stdin.buffer.read()' + process = subprocess.Popen([sys.executable, str(self.script), 'run', 'clang-tidy', + sys.executable, '-c', code], stdin=subprocess.PIPE, stdout=subprocess.PIPE, + env={**os.environ, 'REPORT_TIMING_DIR': tmp}) + child = None + try: + with selectors.DefaultSelector() as ready: + ready.register(process.stdout, selectors.EVENT_READ) + self.assertTrue(ready.select(timeout=5), 'analyzer did not start') + child = int(process.stdout.readline()) + process.kill() + process.wait(timeout=5) + deadline = time.monotonic() + 5 + while time.monotonic() < deadline: + status = Path(f'/proc/{child}/stat') + if not status.exists() or status.read_text().split()[2] == 'Z': + break + time.sleep(0.02) + else: + self.fail('analyzer survived the wrapper timeout') + self.assertIsNone(self.records(tmp)[0]['end']) + finally: + if child is not None: + try: + os.kill(child, signal.SIGKILL) + except ProcessLookupError: + pass + if process.poll() is None: + process.kill() + process.communicate(timeout=5) + + class DocumentationAnchorTests(unittest.TestCase): def test_unicode_duplicate_and_explicit_anchors(self): from check_docs import anchors diff --git a/validation/test_release_build_cache.py b/validation/test_release_build_cache.py new file mode 100644 index 00000000..d97a1e44 --- /dev/null +++ b/validation/test_release_build_cache.py @@ -0,0 +1,126 @@ +"""Check shared Catch2 cache reuse and rejection of incompatible installations.""" +import argparse +import contextlib +import io +import os +from pathlib import Path +import subprocess +import tempfile +import unittest +from unittest.mock import patch + +from prepare_catch2 import prepare, REVISION +from build_support import add_build_arguments, cmake_options, validation_options + + +class CompilerCacheOptionsTests(unittest.TestCase): + def parse(self, arguments): + parser = argparse.ArgumentParser() + add_build_arguments(parser) + return parser.parse_args(arguments) + + def test_cache_and_debug_policy_are_forwarded_to_child_validators(self): + args = self.parse(['--config', 'Debug', '--generator', 'Ninja', '--compiler-cache', 'sccache']) + options = cmake_options(args) + self.assertIn('-DCMAKE_CXX_COMPILER_LAUNCHER=sccache', options) + self.assertIn('-DCMAKE_POLICY_DEFAULT_CMP0141=NEW', options) + self.assertIn('-DCMAKE_MSVC_DEBUG_INFORMATION_FORMAT=$<$:Embedded>', options) + self.assertEqual(cmake_options(self.parse(validation_options(args))), options) + + def test_visual_studio_cannot_silently_ignore_the_launcher(self): + args = self.parse(['--generator', 'Visual Studio 17 2022', '--compiler-cache', 'sccache']) + with self.assertRaisesRegex(ValueError, 'Ninja or Makefile'): + cmake_options(args) + + def test_existing_builds_do_not_gain_cache_or_debug_flags(self): + options = cmake_options(self.parse(['--generator', 'Visual Studio 17 2022'])) + self.assertFalse(any('LAUNCHER' in option or 'MSVC_DEBUG' in option or 'CMP0141' in option + for option in options)) + + def test_linux_ccache_is_forwarded_to_child_validators(self): + args = self.parse(['--generator', 'Ninja', '--compiler-cache', 'ccache']) + options = cmake_options(args) + self.assertIn('-DCMAKE_CXX_COMPILER_LAUNCHER=ccache', options) + self.assertEqual(cmake_options(self.parse(validation_options(args))), options) + + +class Catch2CacheTests(unittest.TestCase): + def setUp(self): + self.temporary = tempfile.TemporaryDirectory() + self.addCleanup(self.temporary.cleanup) + self.root = Path(self.temporary.name).resolve() + self.prefix = self.root / 'prefix' + self.config = self.prefix / 'lib/cmake/Catch2/Catch2Config.cmake' + self.environment = patch.dict(os.environ, { + 'CMAKE_GENERATOR': 'Ninja', 'VCToolsVersion': '14.44.35207', + 'WindowsSDKVersion': '10.0.26100.0', 'VSCMD_ARG_TGT_ARCH': 'arm64', + }) + self.environment.start() + self.addCleanup(self.environment.stop) + + def cold_install(self): + def run(command, **kwargs): + if command[:2] == ['cmake', '--install']: + self.config.parent.mkdir(parents=True) + self.config.write_text('# installed package\n', encoding='utf-8') + return subprocess.CompletedProcess(command, 0) + + with patch('prepare_catch2.subprocess.run', side_effect=run) as execute, \ + patch('prepare_catch2.subprocess.check_output', return_value=REVISION + '\n'): + result = prepare(self.prefix, self.root / 'work', 'Release', 4) + return result, [call.args[0] for call in execute.call_args_list] + + def test_one_install_is_reused_without_git_or_compiler_invocations(self): + result, commands = self.cold_install() + self.assertEqual(result, self.config.parent) + self.assertEqual(sum(c[:2] == ['cmake', '--build'] for c in commands), 1) + configure = next(c for c in commands if c[:2] == ['cmake', '-S']) + self.assertIn('-DCMAKE_CXX_STANDARD=17', configure) + self.assertIn('-DCMAKE_POSITION_INDEPENDENT_CODE=ON', configure) + self.assertIn('-DCMAKE_MSVC_RUNTIME_LIBRARY=MultiThreaded$<$:Debug>DLL', configure) + with patch('prepare_catch2.subprocess.run') as execute, \ + patch('prepare_catch2.subprocess.check_output') as git, \ + contextlib.redirect_stdout(io.StringIO()): + self.assertEqual(prepare(self.prefix, self.root / 'other-work', 'Release', 4), result) + execute.assert_not_called() + git.assert_not_called() + + def test_configuration_architecture_compiler_and_sdk_cannot_share_a_prefix(self): + self.cold_install() + changes = [{'VSCMD_ARG_TGT_ARCH': 'x64'}, {'VCToolsVersion': '14.45.1'}, + {'WindowsSDKVersion': '10.0.99999.0'}, {'CMAKE_GENERATOR': 'Visual Studio 17 2022'}] + for environment in changes: + with self.subTest(environment=environment), patch.dict(os.environ, environment), \ + self.assertRaisesRegex(RuntimeError, 'incompatible'): + prepare(self.prefix, self.root / 'work', 'Release', 4) + with self.assertRaisesRegex(RuntimeError, 'incompatible'): + prepare(self.prefix, self.root / 'work', 'Debug', 4) + + def test_missing_package_does_not_count_as_a_cache_hit(self): + self.cold_install() + self.config.unlink() + with self.assertRaisesRegex(RuntimeError, 'incomplete'): + prepare(self.prefix, self.root / 'work', 'Release', 4) + + def test_changed_compiler_cache_options_invalidate_the_installation(self): + self.cold_install() + with self.assertRaisesRegex(RuntimeError, 'incompatible'): + prepare(self.prefix, self.root / 'work', 'Release', 4, compiler_cache='sccache') + + def test_linux_toolchain_fingerprint_cannot_share_an_installation(self): + self.cold_install() + with patch.dict(os.environ, {'LINKERBOT_CI_TOOLCHAIN': 'different-compiler-and-sysroot'}), \ + self.assertRaisesRegex(RuntimeError, 'incompatible'): + prepare(self.prefix, self.root / 'work', 'Release', 4) + + def test_changed_upstream_tag_fails_before_building(self): + with patch('prepare_catch2.subprocess.run') as execute, \ + patch('prepare_catch2.subprocess.check_output', return_value='a' * 40), \ + self.assertRaisesRegex(RuntimeError, 'pinned source'): + prepare(self.prefix, self.root / 'work', 'Release', 4) + self.assertEqual(execute.call_count, 1) # Clone only; no configure/build/install. + self.assertFalse((self.prefix / 'ci-identity.json').exists()) + + +if __name__ == '__main__': + unittest.main() diff --git a/validation/test_release_matrix.py b/validation/test_release_matrix.py index 0b0d1de7..3a7245d8 100644 --- a/validation/test_release_matrix.py +++ b/validation/test_release_matrix.py @@ -83,19 +83,19 @@ def complete_set(self, transport_package='linkerbot-comm'): def collect(self): return collect(self.assets, self.targets_path, '0.3.0', 'a' * 40, 'b' * 40) - def test_exact_twelve_packages_produce_twenty_six_assets(self): + def test_exact_ten_packages_produce_twenty_two_assets(self): self.complete_set() result = self.collect() - self.assertEqual(len(result['packages']), 12) - self.assertEqual(len(list(self.assets.iterdir())), 26) - self.assertEqual(len((self.assets / 'SHA256SUMS.txt').read_text().splitlines()), 25) + self.assertEqual(len(result['packages']), 10) + self.assertEqual(len(list(self.assets.iterdir())), 22) + self.assertEqual(len((self.assets / 'SHA256SUMS.txt').read_text().splitlines()), 21) for row in result['packages']: self.assertEqual(row['verified_objects'], {'hand': 1, 'comm': 1}) def test_transport_033_metadata_in_legacy_share_is_accepted(self): self.complete_set(transport_package='linkerbot-transport') result = self.collect() - self.assertEqual(len(result['packages']), 12) + self.assertEqual(len(result['packages']), 10) for row in result['packages']: self.assertEqual(row['transport']['package'], 'linkerbot-transport') self.assertEqual(row['transport']['version'], '0.3.4') @@ -109,7 +109,7 @@ def test_unknown_transport_package_is_rejected(self): def test_missing_archive_is_rejected(self): self.complete_set() next(self.assets.glob('*.zip')).unlink() - with self.assertRaisesRegex(ValueError, 'exactly twelve'): + with self.assertRaisesRegex(ValueError, 'exactly 10'): self.collect() def test_renamed_archive_cannot_claim_another_target(self): @@ -135,6 +135,13 @@ def test_duplicate_matrix_target_is_rejected(self): with self.assertRaisesRegex(ValueError, 'eight'): load_targets(path) + def test_debug_windows_target_is_not_in_the_release_matrix(self): + self.targets['windows'].append(dict(self.targets['windows'][0], configuration='Debug', runtime='MDd')) + path = self.root / 'targets.json' + path.write_text(json.dumps(self.targets)) + with self.assertRaisesRegex(ValueError, 'two Windows Release'): + load_targets(path) + def test_wrong_member_machine_is_rejected(self): info = metadata('Windows', 'ARM64', 'Windows') lib = self.root / 'lib' diff --git a/validation/test_release_validation.py b/validation/test_release_validation.py index 1d3426d1..f79e41d8 100644 --- a/validation/test_release_validation.py +++ b/validation/test_release_validation.py @@ -14,7 +14,7 @@ class ValidationRoutingTests(unittest.TestCase): - def run_validation(self, suite='all', quick=False): + def run_validation(self, suite='all', quick=False, extra_args=()): with tempfile.TemporaryDirectory() as directory: root = Path(directory) comm = root / 'comm' @@ -51,6 +51,7 @@ def execute(command, **options): return subprocess.CompletedProcess(command, 23 if '--install' in command else 0) argv = ['run_release_validation.py', '--comm-source', str(comm), '--work-dir', str(work), '--suite', suite] + argv.extend(extra_args) if quick: argv.append('--quick') with patch.object(sys, 'argv', argv), patch('subprocess.check_output', side_effect=git), \ @@ -80,6 +81,14 @@ def test_full_is_still_the_default(self): self.assertEqual(report['steps'][-1]['step'], 'comm-install') self.assertIn('example', next(command for command in commands if '--target' in command)) + def test_both_test_builds_use_the_same_external_catch2(self): + code, _, commands = self.run_validation(quick=True, extra_args=['--catch2-dir', '/shared-catch2']) + self.assertEqual(code, 0) + configurations = [c for c in commands if c[:2] == ['cmake', '-S']] + self.assertEqual(len(configurations), 2) + for command in configurations: + self.assertIn('-DCatch2_DIR=' + str(Path('/shared-catch2').resolve()), command) + def test_quick_suite_filters_remain_independent(self): for suite, expected in [('no-drivers', {'comm-no-drivers', 'hand-no-drivers'}), ('native-loader', {'comm-native-loader', 'hand-native-loader'})]: