From 623d0fc2182cd02dde42c97472a87bdb6acef70c Mon Sep 17 00:00:00 2001 From: dayfine Date: Thu, 29 Oct 2026 10:55:36 -0700 Subject: [PATCH 1/2] Build: add an opt-in macOS ccache preset New worktrees and branch switches on Mac rebuild everything from scratch. The macos-ccache preset is RelWithDebInfo plus a shared 10 GB ccache with checkout-relative paths, so worktrees reuse each other's compiles. A short guide is linked from the Mac build instructions. Other presets and CI are unchanged. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01AKYzhrMnPK61XHWEch4a6E --- .../Build-Mac-CompilerCache.md | 65 +++++++++++++++++++ .../BuildInstructions/CompilingForMac.md | 5 ++ SerialPrograms/CMakePresets.json | 21 ++++++ 3 files changed, 91 insertions(+) create mode 100644 SerialPrograms/BuildInstructions/Build-Mac-CompilerCache.md diff --git a/SerialPrograms/BuildInstructions/Build-Mac-CompilerCache.md b/SerialPrograms/BuildInstructions/Build-Mac-CompilerCache.md new file mode 100644 index 0000000000..f066038f9d --- /dev/null +++ b/SerialPrograms/BuildInstructions/Build-Mac-CompilerCache.md @@ -0,0 +1,65 @@ +# Faster rebuilds on macOS with ccache + +Optional, for developers who build often or keep several worktrees. Get a +normal build working first with the [Mac build guide](CompilingForMac.md). + +The `macos-ccache` preset is the `RelWithDebInfo` preset plus +[ccache](https://ccache.dev): compilations are cached in one place for all your +checkouts, so switching branches or setting up another worktree reuses earlier +compiles instead of starting over. It needs ccache 4.8 or newer and Apple Clang. +Other presets and CI are unchanged. + +## Set up + +```sh +brew install ninja ccache +``` + +From the repository root, configure with the same options your normal build +uses (for example `CMAKE_PREFIX_PATH` for Qt, or `PKG_CONFIG_PATH` if OpenCV is +installed keg-only), then build: + +```sh +cmake -S SerialPrograms --preset macos-ccache \ + -DCMAKE_PREFIX_PATH=/path/to/Qt/6.8.3/macos +cmake --build build/macos-ccache --parallel 4 +``` + +The first build fills the cache and takes as long as a normal build. + +## What the preset sets + +- **Cache location and size:** `~/Library/Caches/pokemonautomation-ccache`, + capped at 5 GB; ccache evicts old entries itself. A full build adds about + 250 MB (compressed), so this holds around 20 revisions' worth of compiles. +- **Path rewriting:** `base_dir` is the checkout, so paths inside it are made + relative and two worktrees produce the same compiler commands. +- **Debug paths:** `-fdebug-prefix-map==.` and + `-fdebug-compilation-dir=.` keep the checkout's path out of the objects, so + they can be shared. Debug info then has relative paths: start the debugger + from the build directory or set its source path. + +These are passed on the compiler command line, so no environment variables are +needed. The preset sets `CMAKE_C_FLAGS` and `CMAKE_CXX_FLAGS`; if you need other +flags, add them to these rather than replacing them. + +To look at the cache: + +```sh +ccache --dir ~/Library/Caches/pokemonautomation-ccache --show-stats +``` + +## Getting cache hits + +Hits need the same compiler, SDK, Qt, dependencies and flags in each worktree. +A change to a widely included header still recompiles everything that includes +it, cached or not. + +Measured on Apple Silicon (Apple Clang 21, Qt 6.8.3, ccache 4.14): a second +worktree at the same revision built in 41 s with 1,601 of 1,607 compiles from +the cache, against 13 to 14 minutes uncached. + +## Turning it off + +Use a normal preset in a separate build directory. Switching presets in the +same directory keeps the launcher and flags stored in `CMakeCache.txt`. diff --git a/SerialPrograms/BuildInstructions/CompilingForMac.md b/SerialPrograms/BuildInstructions/CompilingForMac.md index 8569cd0f76..3a3a80de1c 100644 --- a/SerialPrograms/BuildInstructions/CompilingForMac.md +++ b/SerialPrograms/BuildInstructions/CompilingForMac.md @@ -12,6 +12,11 @@ If you have any questions regarding the documentation or find any errors in the Warning: if you don't have software development experience and deviate from the guide steps, you are at your own risk. +## Faster rebuilds for developers + +After the normal build works, the optional [ccache guide](Build-Mac-CompilerCache.md) +makes rebuilds and new worktrees reuse earlier compiles. + ## Part 1: Install Pre-requisites ### Install XCode App diff --git a/SerialPrograms/CMakePresets.json b/SerialPrograms/CMakePresets.json index 1660f3f00b..d51487f19b 100644 --- a/SerialPrograms/CMakePresets.json +++ b/SerialPrograms/CMakePresets.json @@ -72,6 +72,23 @@ "CMAKE_BUILD_TYPE": "RelWithDebInfo", "PACKAGE_BUILD": "ON" } + }, + { + "name": "macos-ccache", + "inherits": "RelWithDebInfo", + "displayName": "macOS RelWithDebInfo with shared ccache", + "description": "Apple Clang builds with a 5 GB local compiler cache shared across worktrees.", + "condition": { + "type": "equals", + "lhs": "${hostSystemName}", + "rhs": "Darwin" + }, + "cacheVariables": { + "CMAKE_C_COMPILER_LAUNCHER": "ccache;cache_dir=$penv{HOME}/Library/Caches/pokemonautomation-ccache;base_dir=${sourceParentDir};max_size=5G", + "CMAKE_CXX_COMPILER_LAUNCHER": "ccache;cache_dir=$penv{HOME}/Library/Caches/pokemonautomation-ccache;base_dir=${sourceParentDir};max_size=5G", + "CMAKE_C_FLAGS": "-fdebug-prefix-map=${sourceParentDir}=. -fdebug-compilation-dir=.", + "CMAKE_CXX_FLAGS": "-fdebug-prefix-map=${sourceParentDir}=. -fdebug-compilation-dir=." + } } ], "buildPresets": [ @@ -99,6 +116,10 @@ { "name": "Publish", "configurePreset": "Publish" + }, + { + "name": "macos-ccache", + "configurePreset": "macos-ccache" } ] } From d3c8f52f9f670728acd2893990dd430d8335f25f Mon Sep 17 00:00:00 2001 From: Gin <> Date: Wed, 4 Nov 2026 19:49:11 -0800 Subject: [PATCH 2/2] Set up ccache in CMake and add it to the Mac build guide Address review on #1767. - Add cmake/Ccache.cmake, included from CMakeLists.txt. With -DPA_USE_CCACHE=ON (default OFF) and ccache 4.8+ installed, every compile runs through ccache: - PA_CCACHE_CONFIG: ccache settings passed on every compile's command line. The default keeps this project's cache in a "PA-Arduino-Source" folder inside ccache's own cache folder, and sets base_dir to the checkout's root folder, so all worktrees share one cache without changing ccache's global config. - PA_CCACHE_DEBUG_PATHS (default ON): adds -fdebug-prefix-map and -fdebug-compilation-dir to Debug and RelWithDebInfo builds, so builds with debug info share the cache across worktrees too. - Stops with an error if a compiler launcher is also given, or if ccache is older than 4.8. - Mac build guide: install ccache as a normal step, turn it on in the CMake command, and add "Part 7: Tuning ccache". - Remove the macos-ccache preset and Build-Mac-CompilerCache.md: the preset's settings are now in cmake/Ccache.cmake, and the guide covers the rest. Co-Authored-By: Claude Opus 5.5 --- .../Build-Mac-CompilerCache.md | 65 ------------- .../BuildInstructions/CompilingForMac.md | 38 ++++++-- SerialPrograms/CMakeLists.txt | 3 + SerialPrograms/CMakePresets.json | 21 ----- SerialPrograms/cmake/Ccache.cmake | 93 +++++++++++++++++++ 5 files changed, 125 insertions(+), 95 deletions(-) delete mode 100644 SerialPrograms/BuildInstructions/Build-Mac-CompilerCache.md create mode 100644 SerialPrograms/cmake/Ccache.cmake diff --git a/SerialPrograms/BuildInstructions/Build-Mac-CompilerCache.md b/SerialPrograms/BuildInstructions/Build-Mac-CompilerCache.md deleted file mode 100644 index f066038f9d..0000000000 --- a/SerialPrograms/BuildInstructions/Build-Mac-CompilerCache.md +++ /dev/null @@ -1,65 +0,0 @@ -# Faster rebuilds on macOS with ccache - -Optional, for developers who build often or keep several worktrees. Get a -normal build working first with the [Mac build guide](CompilingForMac.md). - -The `macos-ccache` preset is the `RelWithDebInfo` preset plus -[ccache](https://ccache.dev): compilations are cached in one place for all your -checkouts, so switching branches or setting up another worktree reuses earlier -compiles instead of starting over. It needs ccache 4.8 or newer and Apple Clang. -Other presets and CI are unchanged. - -## Set up - -```sh -brew install ninja ccache -``` - -From the repository root, configure with the same options your normal build -uses (for example `CMAKE_PREFIX_PATH` for Qt, or `PKG_CONFIG_PATH` if OpenCV is -installed keg-only), then build: - -```sh -cmake -S SerialPrograms --preset macos-ccache \ - -DCMAKE_PREFIX_PATH=/path/to/Qt/6.8.3/macos -cmake --build build/macos-ccache --parallel 4 -``` - -The first build fills the cache and takes as long as a normal build. - -## What the preset sets - -- **Cache location and size:** `~/Library/Caches/pokemonautomation-ccache`, - capped at 5 GB; ccache evicts old entries itself. A full build adds about - 250 MB (compressed), so this holds around 20 revisions' worth of compiles. -- **Path rewriting:** `base_dir` is the checkout, so paths inside it are made - relative and two worktrees produce the same compiler commands. -- **Debug paths:** `-fdebug-prefix-map==.` and - `-fdebug-compilation-dir=.` keep the checkout's path out of the objects, so - they can be shared. Debug info then has relative paths: start the debugger - from the build directory or set its source path. - -These are passed on the compiler command line, so no environment variables are -needed. The preset sets `CMAKE_C_FLAGS` and `CMAKE_CXX_FLAGS`; if you need other -flags, add them to these rather than replacing them. - -To look at the cache: - -```sh -ccache --dir ~/Library/Caches/pokemonautomation-ccache --show-stats -``` - -## Getting cache hits - -Hits need the same compiler, SDK, Qt, dependencies and flags in each worktree. -A change to a widely included header still recompiles everything that includes -it, cached or not. - -Measured on Apple Silicon (Apple Clang 21, Qt 6.8.3, ccache 4.14): a second -worktree at the same revision built in 41 s with 1,601 of 1,607 compiles from -the cache, against 13 to 14 minutes uncached. - -## Turning it off - -Use a normal preset in a separate build directory. Switching presets in the -same directory keeps the launcher and flags stored in `CMakeCache.txt`. diff --git a/SerialPrograms/BuildInstructions/CompilingForMac.md b/SerialPrograms/BuildInstructions/CompilingForMac.md index 3a3a80de1c..af12896b66 100644 --- a/SerialPrograms/BuildInstructions/CompilingForMac.md +++ b/SerialPrograms/BuildInstructions/CompilingForMac.md @@ -12,11 +12,6 @@ If you have any questions regarding the documentation or find any errors in the Warning: if you don't have software development experience and deviate from the guide steps, you are at your own risk. -## Faster rebuilds for developers - -After the normal build works, the optional [ccache guide](Build-Mac-CompilerCache.md) -makes rebuilds and new worktrees reuse earlier compiles. - ## Part 1: Install Pre-requisites ### Install XCode App @@ -99,11 +94,18 @@ brew install cmake ### Install Ninja -Ninja is a build tool that CMake can use to find out quickly what files need rebuilding. Install it using Homebrew by running in Terminal +Ninja is a build tool that CMake can use to find out quickly what files need rebuilding by file modification time. Install it using Homebrew by running in Terminal ``` brew install ninja ``` +### Install ccache + +ccache is another build tool that saves build time. It uses a cache to skip building a source code file if that file's content has been compiled before. Install it using Homebrew by running in Terminal +``` +brew install ccache +``` + ### Install Qt6 We use Qt6 software library to build UI (windows, panels, buttons...) for **SerialPrograms**. It also plays and decodes audio. @@ -273,11 +275,13 @@ This makes a new folder called "build_mac" in "Arduino-Source" and moves the Ter Execute the command in Terminal: ``` -cmake ../SerialPrograms/ -G Ninja -DCMAKE_BUILD_TYPE:STRING=Release -DCMAKE_PREFIX_PATH=$HOME/Qt/6.10.2/macos +cmake ../SerialPrograms/ -G Ninja -DCMAKE_BUILD_TYPE:STRING=Release -DCMAKE_PREFIX_PATH=$HOME/Qt/6.10.2/macos -DPA_USE_CCACHE=ON ``` -This command uses CMake to configure our code to prepare for compilation. `-G Ninja` tells CMake to compile with Ninja, and `-DCMAKE_PREFIX_PATH` tells CMake where the Qt you installed is. If successful, it should print texts like +This command uses CMake to configure our code to prepare for compilation. `-G Ninja` tells CMake to compile with Ninja, `-DCMAKE_PREFIX_PATH` tells CMake where the Qt you installed is, and `-DPA_USE_CCACHE=ON` tells CMake to compile with ccache. If successful, it should print texts like ``` ... +-- Using ccache: /opt/homebrew/bin/ccache cache_dir=/Users//Library/Caches/ccache/PA-Arduino-Source base_dir= +... -- Configuring done -- Generating done -- Build files have been written to: /build_mac @@ -291,6 +295,7 @@ cmake --build . -j 10 ``` to compile and build the program in "build_mac" folder. It will take a while to build depending on your computer's speed. +Later builds should be faster thanks to Ninja and ccache. Each line starts with a counter like `[25/1613]`. If successful, the counter reaches the total, and the last lines in Terminal look like ``` @@ -356,11 +361,26 @@ rm -rf build_mac to delete the old "build_mac" folder. Then follow [Part 4: Compile the Program](#part-4-compile-the-program) again, with the new Qt version in the `-DCMAKE_PREFIX_PATH` part of the CMake command. +## Part 7: Tuning ccache (optional) + +The default ccache settings work for most people. This part is for developers who build often or keep several copies of the code (a.k.a worktrees) in the filesystem. + +To check cache statistics on the codebase (how many cache hits and misses so far and storage space uses), run in Terminal +``` +ccache --dir $(ccache --get-config cache_dir)/PA-Arduino-Source -s +``` + +By default ccache has a max cache size of 5 GB. When the storage is full, ccache deletes the files it used least recently. +If you often switch between many branches and worktrees of the codebase, you can increase its size to say 20 GB by +``` +ccache --dir $(ccache --get-config cache_dir)/PA-Arduino-Source --set-config max_size=20G +``` +For more info on how we set up ccache when configuring the program, see `SerialPrograms/cmake/Ccache.cmake`.
-Guide Written by: jw098 +Guide Written by: Gin, jw098 **Discord Server:** diff --git a/SerialPrograms/CMakeLists.txt b/SerialPrograms/CMakeLists.txt index 35bb597e32..892a04d94b 100644 --- a/SerialPrograms/CMakeLists.txt +++ b/SerialPrograms/CMakeLists.txt @@ -26,6 +26,9 @@ set(CMAKE_CXX_STANDARD_REQUIRED ON) set(CMAKE_EXPORT_COMPILE_COMMANDS ON) #set(CMAKE_VERBOSE_MAKEFILE ON) +# Compile through ccache when it is installed. See cmake/Ccache.cmake for the options. +include(cmake/Ccache.cmake) + # Set these three on globally so all intermediate Qt targets can get processed automatically set(CMAKE_AUTOMOC ON) set(CMAKE_AUTORCC ON) diff --git a/SerialPrograms/CMakePresets.json b/SerialPrograms/CMakePresets.json index d51487f19b..1660f3f00b 100644 --- a/SerialPrograms/CMakePresets.json +++ b/SerialPrograms/CMakePresets.json @@ -72,23 +72,6 @@ "CMAKE_BUILD_TYPE": "RelWithDebInfo", "PACKAGE_BUILD": "ON" } - }, - { - "name": "macos-ccache", - "inherits": "RelWithDebInfo", - "displayName": "macOS RelWithDebInfo with shared ccache", - "description": "Apple Clang builds with a 5 GB local compiler cache shared across worktrees.", - "condition": { - "type": "equals", - "lhs": "${hostSystemName}", - "rhs": "Darwin" - }, - "cacheVariables": { - "CMAKE_C_COMPILER_LAUNCHER": "ccache;cache_dir=$penv{HOME}/Library/Caches/pokemonautomation-ccache;base_dir=${sourceParentDir};max_size=5G", - "CMAKE_CXX_COMPILER_LAUNCHER": "ccache;cache_dir=$penv{HOME}/Library/Caches/pokemonautomation-ccache;base_dir=${sourceParentDir};max_size=5G", - "CMAKE_C_FLAGS": "-fdebug-prefix-map=${sourceParentDir}=. -fdebug-compilation-dir=.", - "CMAKE_CXX_FLAGS": "-fdebug-prefix-map=${sourceParentDir}=. -fdebug-compilation-dir=." - } } ], "buildPresets": [ @@ -116,10 +99,6 @@ { "name": "Publish", "configurePreset": "Publish" - }, - { - "name": "macos-ccache", - "configurePreset": "macos-ccache" } ] } diff --git a/SerialPrograms/cmake/Ccache.cmake b/SerialPrograms/cmake/Ccache.cmake new file mode 100644 index 0000000000..6ebad144a1 --- /dev/null +++ b/SerialPrograms/cmake/Ccache.cmake @@ -0,0 +1,93 @@ +# Ccache.cmake: set up ccache to save compile time on mac/Linux. +# +# Included from CMakeLists.txt right after project() and before any target is created. +# +# Options, passed to CMake as -D=: +# PA_USE_CCACHE Use ccache on mac/Linux if it is installed (default OFF). Needs +# ccache 4.8+. Stops with an error if a compiler launcher is also +# given, e.g. -DCMAKE_CXX_COMPILER_LAUNCHER=ccache: use one or the other. +# PA_CCACHE_CONFIG ccache settings for every compile (default: Arduino-Source has +# its own ccache folder, and base_dir set to the checkout's root +# folder). base_dir allows builds to share the cache across worktrees. +# PA_CCACHE_DEBUG_PATHS Change to using relative paths in debug (Debug and RelWithDebInfo) +# builds (default ON) when ccache is on. This allows debug builds +# to also share the cache across worktrees. + +option(PA_USE_CCACHE "Compile through ccache on mac/Linux if it is installed" OFF) + +# set _pa_checkout_dir to be the repo root path +cmake_path(GET CMAKE_CURRENT_SOURCE_DIR PARENT_PATH _pa_checkout_dir) + +# By default, builds with debug info (Debug and RelWithDebInfo) store the build folder's +# full paths in it, so ccache can't share cache across different worktrees, as worktrees +# have their own full paths. +# When both PA_USE_CCACHE and PA_CCACHE_DEBUG_PATHS are on, we added option +# -fdebug-prefix-map +# -fdebug-compilation-dir=. +# as compiler flags. This lets the debug builds use relative paths, enabling cache sharing +# across worktrees. +option(PA_CCACHE_DEBUG_PATHS "With ccache, use relative paths in debug info so Debug and RelWithDebInfo builds share the cache across worktrees" ON) + +if(PA_USE_CCACHE AND NOT MSVC) + find_program(CCACHE_PROGRAM ccache) + if(CCACHE_PROGRAM) + if(CMAKE_C_COMPILER_LAUNCHER OR CMAKE_CXX_COMPILER_LAUNCHER) + message(FATAL_ERROR + "PA_USE_CCACHE=ON also sets up CMAKE_C_COMPILER_LAUNCHER and CMAKE_CXX_COMPILER_LAUNCHER.\n" + "Either remove CMAKE_C_COMPILER_LAUNCHER and CMAKE_CXX_COMPILER_LAUNCHER or disable PA_USE_CCACHE." + ) + endif() + + execute_process( + COMMAND "${CCACHE_PROGRAM}" --version + OUTPUT_VARIABLE _pa_ccache_version_output + ERROR_QUIET + ) + string(REGEX MATCH "version ([0-9]+\\.[0-9]+(\\.[0-9]+)?)" _ "${_pa_ccache_version_output}") + set(_pa_ccache_version "${CMAKE_MATCH_1}") + + # ccache 4.8 is the first version that takes settings on the command line, which is how + # PA_CCACHE_CONFIG reaches every compile. + if(_pa_ccache_version VERSION_LESS 4.8) + message(FATAL_ERROR + "PA_USE_CCACHE=ON needs ccache 4.8 or newer, but ${CCACHE_PROGRAM} is version ${_pa_ccache_version}.\n" + "Either upgrade ccache or disable PA_USE_CCACHE." + ) + endif() + + execute_process( + COMMAND "${CCACHE_PROGRAM}" --get-config cache_dir + OUTPUT_VARIABLE _pa_ccache_main_dir + OUTPUT_STRIP_TRAILING_WHITESPACE + ERROR_QUIET + ) + if(_pa_ccache_main_dir) + set(_pa_ccache_default_config "cache_dir=${_pa_ccache_main_dir}/PA-Arduino-Source;base_dir=${_pa_checkout_dir}") + else() + set(_pa_ccache_default_config "base_dir=${_pa_checkout_dir}") + endif() + + # set PA_CCACHE_CONFIG in CMake cache + set(PA_CCACHE_CONFIG "${_pa_ccache_default_config}" + CACHE STRING "ccache settings passed on every compile's command line, as key=value pairs") + + set(_pa_ccache_launcher "${CCACHE_PROGRAM}" ${PA_CCACHE_CONFIG}) + list(JOIN _pa_ccache_launcher " " _pa_ccache_launcher_text) + message(STATUS "Using ccache: ${_pa_ccache_launcher_text}") + set(CMAKE_C_COMPILER_LAUNCHER "${_pa_ccache_launcher}") + set(CMAKE_CXX_COMPILER_LAUNCHER "${_pa_ccache_launcher}") + + if(PA_CCACHE_DEBUG_PATHS) + # Only builds with debug info get these options. + set(_pa_has_debug_info "$") + add_compile_options("$<${_pa_has_debug_info}:-fdebug-prefix-map=${_pa_checkout_dir}=.>") + # Clang has this option, but not every compiler does. Without it, the prefix map + # above still covers the compile folder when the build folder is inside the checkout. + include(CheckCompilerFlag) + check_compiler_flag(CXX "-fdebug-compilation-dir=." PA_HAS_DEBUG_COMPILATION_DIR) + if(PA_HAS_DEBUG_COMPILATION_DIR) + add_compile_options("$<${_pa_has_debug_info}:-fdebug-compilation-dir=.>") + endif() + endif() + endif() +endif()