From b440018c4fb77b5adde60a217c4c1c14fd78c976 Mon Sep 17 00:00:00 2001 From: Dimitrios Vlachos Date: Thu, 29 Oct 2026 14:33:08 +0000 Subject: [PATCH 1/3] Cache ccache and the fetched dependency sources in CI --- .github/workflows/build_test.yml | 49 +++++++++++++++++++++++++++++++- 1 file changed, 48 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build_test.yml b/.github/workflows/build_test.yml index 562bb9c..53aa6e3 100644 --- a/.github/workflows/build_test.yml +++ b/.github/workflows/build_test.yml @@ -9,6 +9,12 @@ jobs: run: shell: bash -l {0} runs-on: ${{ matrix.os }}-latest + env: + CCACHE_DIR: ${{ runner.temp }}/ccache + # The default compiler check compares the compiler's mtime, which a + # fresh conda install may not preserve, so compare contents instead. + CCACHE_COMPILERCHECK: content + DEPS_DIR: ${{ runner.temp }}/deps steps: - uses: actions/checkout@v7 - uses: conda-incubator/setup-miniconda@v4 @@ -18,8 +24,49 @@ jobs: - name: Install dependencies run: | conda env update --file requirements.txt --name test + conda install --yes --name test ccache conda list + # Each run restores the newest cache under the prefix and saves its + # own under the commit, so the cache rolls forward rather than + # freezing at the first save. + - name: Restore ccache + uses: actions/cache@v6 + with: + path: ${{ env.CCACHE_DIR }} + key: ccache-${{ runner.os }}-${{ github.sha }} + restore-keys: | + ccache-${{ runner.os }}- + # Persist the fetched dependency sources so a configure does not + # depend on reaching every git host. The key covers the files that + # choose the dependencies and the toolchain, so a changed pin or + # environment starts from an empty directory. + - name: Restore dependency sources + uses: actions/cache@v6 + with: + path: ${{ env.DEPS_DIR }} + key: deps-${{ runner.os }}-${{ hashFiles('CMakeLists.txt', 'requirements.txt') }}-${{ github.sha }} + restore-keys: | + deps-${{ runner.os }}-${{ hashFiles('CMakeLists.txt', 'requirements.txt') }}- + # The cache action does not report which key matched, so list each + # restored dependency with the commit it sits at. + - name: Report restored dependency sources + run: | + if ! compgen -G "$DEPS_DIR/*-src" > /dev/null; then + echo "No dependency sources restored, cold start" + exit 0 + fi + for src in "$DEPS_DIR"/*-src; do + [ -d "$src" ] || continue + rev=$(git -C "$src" rev-parse --short HEAD 2>/dev/null || echo "?") + echo "$(basename "$src" -src) ${rev} $(du -sh "$src" | cut -f1)" + done - name: Build - run: mkdir build && cd build && cmake .. && make + run: | + ccache --zero-stats + cmake -B build \ + -DCMAKE_CXX_COMPILER_LAUNCHER=ccache \ + -DFETCHCONTENT_BASE_DIR="$DEPS_DIR" + cmake --build build + ccache --show-stats - name: Run tests run: cd build && ctest -L dx2tests --output-on-failure From 15bf370ea213604da111560569e0694d21696b91 Mon Sep 17 00:00:00 2001 From: Dimitrios Vlachos Date: Thu, 29 Oct 2026 17:06:34 +0000 Subject: [PATCH 2/3] Build from the cached dependency sources in CI FetchContent keeps its record of completed downloads in the build tree, which every run starts without, so it deleted each restored source and cloned it again. The cache directories also move out of the job env, where the runner context is not available. --- .github/workflows/build_test.yml | 37 +++++++++++++++++++++----------- 1 file changed, 25 insertions(+), 12 deletions(-) diff --git a/.github/workflows/build_test.yml b/.github/workflows/build_test.yml index 53aa6e3..f648064 100644 --- a/.github/workflows/build_test.yml +++ b/.github/workflows/build_test.yml @@ -10,13 +10,17 @@ jobs: shell: bash -l {0} runs-on: ${{ matrix.os }}-latest env: - CCACHE_DIR: ${{ runner.temp }}/ccache # The default compiler check compares the compiler's mtime, which a # fresh conda install may not preserve, so compare contents instead. CCACHE_COMPILERCHECK: content - DEPS_DIR: ${{ runner.temp }}/deps steps: - uses: actions/checkout@v7 + # The runner context is not available in the job env, so the cache + # directories are set here for every later step. + - name: Set cache directories + run: | + echo "CCACHE_DIR=${RUNNER_TEMP}/ccache" >> "$GITHUB_ENV" + echo "DEPS_DIR=${RUNNER_TEMP}/deps" >> "$GITHUB_ENV" - uses: conda-incubator/setup-miniconda@v4 with: channels: conda-forge @@ -37,22 +41,21 @@ jobs: restore-keys: | ccache-${{ runner.os }}- # Persist the fetched dependency sources so a configure does not - # depend on reaching every git host. The key covers the files that - # choose the dependencies and the toolchain, so a changed pin or - # environment starts from an empty directory. + # depend on reaching every git host. The build uses them as they are, + # so the key covers the file that pins them and the one that chooses + # the toolchain. - name: Restore dependency sources uses: actions/cache@v6 with: path: ${{ env.DEPS_DIR }} - key: deps-${{ runner.os }}-${{ hashFiles('CMakeLists.txt', 'requirements.txt') }}-${{ github.sha }} - restore-keys: | - deps-${{ runner.os }}-${{ hashFiles('CMakeLists.txt', 'requirements.txt') }}- - # The cache action does not report which key matched, so list each - # restored dependency with the commit it sits at. + key: deps-${{ runner.os }}-${{ hashFiles('CMakeLists.txt', 'requirements.txt') }} + # List each restored dependency with the commit it sits at. The + # directory starts empty on every job, so anything in it came from + # the cache. - name: Report restored dependency sources run: | if ! compgen -G "$DEPS_DIR/*-src" > /dev/null; then - echo "No dependency sources restored, cold start" + echo "No dependency sources restored, every dependency will be fetched" exit 0 fi for src in "$DEPS_DIR"/*-src; do @@ -60,12 +63,22 @@ jobs: rev=$(git -C "$src" rev-parse --short HEAD 2>/dev/null || echo "?") echo "$(basename "$src" -src) ${rev} $(du -sh "$src" | cut -f1)" done + # Each cached source is passed to cmake as an override, since + # FetchContent keeps its download record in the build tree and would + # re-clone it. - name: Build run: | + deps_args=() + for src in "$DEPS_DIR"/*-src; do + [ -d "$src" ] || continue + name=$(basename "$src" -src | tr a-z A-Z) + deps_args+=("-DFETCHCONTENT_SOURCE_DIR_${name}=$src") + done ccache --zero-stats cmake -B build \ -DCMAKE_CXX_COMPILER_LAUNCHER=ccache \ - -DFETCHCONTENT_BASE_DIR="$DEPS_DIR" + -DFETCHCONTENT_BASE_DIR="$DEPS_DIR" \ + "${deps_args[@]}" cmake --build build ccache --show-stats - name: Run tests From ae2e1fdb2e70fa09d7030d55e61180da7db653b0 Mon Sep 17 00:00:00 2001 From: Dimitrios Vlachos Date: Thu, 29 Oct 2026 17:10:48 +0000 Subject: [PATCH 3/3] Run the dependency source report outside the login shell --- .github/workflows/build_test.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build_test.yml b/.github/workflows/build_test.yml index f648064..8765d03 100644 --- a/.github/workflows/build_test.yml +++ b/.github/workflows/build_test.yml @@ -51,8 +51,10 @@ jobs: key: deps-${{ runner.os }}-${{ hashFiles('CMakeLists.txt', 'requirements.txt') }} # List each restored dependency with the commit it sits at. The # directory starts empty on every job, so anything in it came from - # the cache. + # the cache. A login shell runs ~/.bash_logout on exit, which can fail + # the step, and this step does not need the conda environment. - name: Report restored dependency sources + shell: bash run: | if ! compgen -G "$DEPS_DIR/*-src" > /dev/null; then echo "No dependency sources restored, every dependency will be fetched"