cpp-ci-serial-programs-base.yml

name: C++ CI Serial Programs base
on:
  workflow_call:
    inputs:
      os:
        required: true
        type: string

      compiler:
        required: true
        type: string

      upload-build:
        type: boolean
        default: false

      run-tests:
        type: boolean
        default: false

      run-clang-query:
        type: boolean
        default: false

jobs:
  build:
    runs-on: ${{inputs.os}}

    steps:
      - name: Set variables
        shell: bash
        run: |
          BUILD_PARALLEL=10
          # Build with Ninja on every OS. The Ubuntu and macOS runner images already have it, so
          # it needs no install step. It also finds what to rebuild much faster than Make, which
          # walks CMake's nested makefiles first.
          CMAKE_GENERATOR="Ninja"
          if [[ "${{inputs.os}}" == windows* ]]; then
            # On Windows, Ninja also lets ccache work: ccache can't wrap the compiler under the
            # Visual Studio generator.
            # "Ninja Multi-Config" keeps Visual Studio's output layout (bin/RelWithDebInfo/).
            # ccache also can't cache /FAs listings, so turn them off.
            # CI needs no debug info, so keep RelWithDebInfo's optimization but leave out its
            # debug info: with policy CMP0141, an empty CMAKE_MSVC_DEBUG_INFORMATION_FORMAT
            # drops /Zi, and linking without /debug writes no PDB. Release builds made outside
            # CI keep their debug info.
            CMAKE_GENERATOR="Ninja Multi-Config"
            CMAKE_ADDITIONAL_FLAGS="-DCMAKE_POLICY_DEFAULT_CMP0141=NEW -DCMAKE_MSVC_DEBUG_INFORMATION_FORMAT= -DCMAKE_EXE_LINKER_FLAGS_RELWITHDEBINFO=/INCREMENTAL:NO -DPA_ASSEMBLY_LISTINGS=OFF"
            if [[ "${{inputs.compiler}}" == "clang" ]]; then
              CMAKE_ADDITIONAL_FLAGS="$CMAKE_ADDITIONAL_FLAGS -DCMAKE_C_COMPILER=clang-cl -DCMAKE_CXX_COMPILER=clang-cl"
            else
              CMAKE_ADDITIONAL_FLAGS="$CMAKE_ADDITIONAL_FLAGS -DCMAKE_C_COMPILER=cl -DCMAKE_CXX_COMPILER=cl"
            fi
            # Visual Studio ran about one compile per core (/MP). Keep Ninja near that instead of
            # 10 at once, so memory use stays close to what the runner handled before.
            BUILD_PARALLEL=$(( $(nproc) + 2 ))
          elif [[ "${{inputs.compiler}}" == "clang" && "${{inputs.os}}" == ubuntu* ]]; then
            # RelWithDebInfo's -O2 without its -g: CI doesn't use the debug info.
            CMAKE_ADDITIONAL_FLAGS="-DCMAKE_C_COMPILER=clang -DCMAKE_CXX_COMPILER=clang++ -DCMAKE_BUILD_TYPE=RelWithDebInfo '-DCMAKE_C_FLAGS_RELWITHDEBINFO=-O2 -DNDEBUG' '-DCMAKE_CXX_FLAGS_RELWITHDEBINFO=-O2 -DNDEBUG'"
          fi
          CMAKE_ADDITIONAL_FLAGS="$CMAKE_ADDITIONAL_FLAGS -DCMAKE_C_COMPILER_LAUNCHER=ccache -DCMAKE_CXX_COMPILER_LAUNCHER=ccache"
          echo "CMAKE_GENERATOR=$CMAKE_GENERATOR" >> $GITHUB_ENV
          echo "CMAKE_ADDITIONAL_FLAGS=$CMAKE_ADDITIONAL_FLAGS" >> $GITHUB_ENV
          echo "BUILD_PARALLEL=$BUILD_PARALLEL" >> $GITHUB_ENV

          if [[ "${{inputs.os}}" == windows* ]]; then
            UPLOAD_FOLDER="Arduino-Source/SerialPrograms/bin/RelWithDebInfo"
          else
            UPLOAD_FOLDER="Arduino-Source/SerialPrograms/bin"
          fi
          echo "UPLOAD_FOLDER=$UPLOAD_FOLDER" >> $GITHUB_ENV

      - name: Checkout Arduino-Source
        uses: actions/checkout@v7
        with:
          path: 'Arduino-Source'
          submodules: 'recursive'

      # Check out the Packages repo (resources, firmware, scripts) before CMake configures.
      # Otherwise CMake does a full `git clone` of it (see SerialPrograms/CMakeLists.txt), which
      # downloads the entire history: every old version of every model and zip ever committed,
      # which only grows. actions/checkout fetches only the latest commit (fetch-depth: 1), and
      # CMake skips its own clone when Arduino-Source/Packages already exists.
      - name: Checkout Packages
        uses: actions/checkout@v7
        with:
          repository: 'PokemonAutomation/Packages'
          ref: 'master'
          path: 'Arduino-Source/Packages'

      - name: Install Qt
        uses: jurplel/install-qt-action@v4
        with:
          # Delete aqtsource once https://github.com/miurahr/aqtinstall/issues/1007 is fixed and released
          aqtsource: 'git+https://github.com/miurahr/aqtinstall'
          version: '6.10.2'
          modules: 'qtmultimedia qtserialport'
          cache: true

      - name: Install dependencies (Ubuntu)
        if: startsWith(inputs.os, 'ubuntu')
        run: |
          cd Arduino-Source
          sudo apt update
          sudo apt install ccache clang-tools libopencv-dev libsystemd-dev libsdl3-dev

          sudo apt install ./3rdPartyBinaries/libdpp-10.1.6-linux-x64.deb

      - name: Install dependencies (Mac)
        if: startsWith(inputs.os, 'mac')
        run: |
          cd Arduino-Source
          # libsodium, openssl@3, opus and pkgconf are libdpp's dependencies, installed here so
          # their versions can go into the libdpp cache key below.
          brew install opencv@4 onnxruntime ccache libsodium openssl@3 opus pkgconf sdl3

          brew tap-new --no-git PA/libdpp
          brew trust PA/libdpp
          FORMULA_DIR="$(brew --repo PA/libdpp)/Formula"
          mkdir -p "$FORMULA_DIR"
          cp 3rdPartyBinaries/libdpp@10.1.6.rb "$FORMULA_DIR/"

          echo "LIBDPP_KEG=$(brew --cellar)/libdpp@10.1.6" >> $GITHUB_ENV
          echo "LIBDPP_DEPS_HASH=$(brew list --versions libsodium openssl@3 opus | shasum | cut -c1-16)" >> $GITHUB_ENV

      # Homebrew has no prebuilt bottle for our pinned libdpp formula, so `brew install` compiles
      # DPP from source: about 4 minutes on Apple Silicon and 13 on Intel. Cache the installed keg
      # instead. The key includes the versions of the libraries libdpp links against, so when
      # Homebrew updates one of them, libdpp is rebuilt against it rather than restored.
      - name: Cache libdpp (Mac)
        id: libdpp-cache
        if: startsWith(inputs.os, 'mac')
        uses: actions/cache@v6
        with:
          path: ${{env.LIBDPP_KEG}}
          key: libdpp-${{inputs.os}}-${{hashFiles('Arduino-Source/3rdPartyBinaries/libdpp@10.1.6.rb')}}-${{env.LIBDPP_DEPS_HASH}}

      - name: Install libdpp (Mac)
        if: startsWith(inputs.os, 'mac')
        run: |
          if [[ "${{steps.libdpp-cache.outputs.cache-hit}}" == "true" ]]; then
            brew link libdpp@10.1.6
          else
            brew install libdpp@10.1.6
          fi
          # CMake quietly builds without Discord integration when it can't find DPP, so check here.
          pkg-config --exact-version=10.1.6 dpp

      # Ninja needs the MSVC build environment (cl and link on PATH, INCLUDE, LIB, ...), which the
      # Visual Studio generator set up by itself. Run vcvars64.bat from the newest Visual Studio,
      # the one that generator used, and pass every variable it adds or changes to later steps.
      - name: Set up MSVC environment (Windows)
        if: startsWith(inputs.os, 'windows')
        shell: pwsh
        run: |
          $vs = & "${env:ProgramFiles(x86)}\Microsoft Visual Studio\Installer\vswhere.exe" -latest -products * -requires Microsoft.VisualStudio.Component.VC.Tools.x86.x64 -property installationPath
          $vcvars = cmd /c "`"$vs\VC\Auxiliary\Build\vcvars64.bat`" >nul && set"
          if ($LASTEXITCODE -ne 0) {
            throw "vcvars64.bat failed with exit code $LASTEXITCODE"
          }
          foreach ($line in $vcvars) {
            # Skip anything that isn't a NAME=value line.
            if ($line -notmatch '^([^=]+)=(.*)$') {
              continue
            }
            $name = $Matches[1]
            $value = $Matches[2]
            if ($name -eq 'Path' -and '${{inputs.compiler}}' -eq 'clang') {
              # Find Visual Studio's own clang-cl, the one its ClangCL toolset used, before any
              # other LLVM install on the runner.
              $value = "$vs\VC\Tools\Llvm\x64\bin;$value"
            }
            if ([Environment]::GetEnvironmentVariable($name) -ne $value) {
              "$name=$value" | Out-File -FilePath $env:GITHUB_ENV -Append -Encoding utf8
              [Environment]::SetEnvironmentVariable($name, $value)
            }
          }
          if (-not (Get-Command cl -ErrorAction SilentlyContinue)) {
            throw "cl.exe isn't on PATH after vcvars64.bat"
          }
          Get-Command cl, link, clang-cl, ninja, cmake -ErrorAction SilentlyContinue | Format-Table Name, Version, Source -AutoSize

      # Keep compiler outputs between runs, so a build only recompiles what a change affects.
      # Only pushes to main save the cache: every PR restores main's latest one, and PR runs don't
      # fill the repo's 10 GB cache quota with entries that no other run can read.
      # max-size is only a cap: evict-old-files drops everything this build didn't use, so each
      # saved cache is about one build's worth.
      # On Windows, install the action's own ccache: the runner's Strawberry Perl puts an older
      # ccache (4.9.1) on PATH, which the default install: detect would use instead.
      - name: Set up ccache
        uses: hendrikmuhs/ccache-action@v1.2.24
        with:
          key: ${{inputs.os}}-${{inputs.compiler}}
          max-size: 5G
          evict-old-files: job
          save: ${{github.ref == 'refs/heads/main'}}
          install: ${{startsWith(inputs.os, 'windows') && 'binary' || 'detect'}}

      - name: Generate binaries
        env:
          # Depend mode: on a cache miss, ccache hashes the include files the compiler reports
          # (-MD or /showIncludes) instead of running the preprocessor a second time, which the
          # ccache manual says makes a miss much cheaper. In the first runs with ccache, where
          # every file missed, compiling took 10-40% longer than before on Linux and macOS, and
          # about twice as long on Windows.
          CCACHE_DEPEND: 1
        run: |
          cd Arduino-Source/SerialPrograms
          mkdir bin
          cd bin
          cmake .. -DQT_MAJOR:STRING=6 -DPA_SDL=ON ${{env.CMAKE_ADDITIONAL_FLAGS}}
          # By default Ninja reads each compile's .d dependency file into its own database, then deletes it.
          # To keep them for the downstream clang-query cache (see .github/scripts/clang-query-cached.py), add
          # -d keepdepfile
          cmake --build . --config RelWithDebInfo --parallel ${{env.BUILD_PARALLEL}} -- -d keepdepfile

      - name: Prepare upload build
        if: inputs.upload-build
        shell: bash
        run: |
          {
            echo "https://github.com/${{github.repository_owner}}/Arduino-Source/commit/$(git -C Arduino-Source rev-parse HEAD)"
            echo "https://github.com/${{github.repository_owner}}/Packages/commit/$(git -C Arduino-Source/Packages rev-parse HEAD)"
          } > ${{env.UPLOAD_FOLDER}}/version.txt

      # Leave out the static libraries, which only the build itself uses. With debug info,
      # SerialProgramsLib.lib alone was over 1 GB uncompressed.
      - name: Upload Build
        uses: actions/upload-artifact@v7
        if: inputs.upload-build
        with:
          name: Serial Programs (os=${{inputs.os}} - compiler=${{inputs.compiler}})
          path: |
            ${{env.UPLOAD_FOLDER}}
            !${{env.UPLOAD_FOLDER}}/*.lib

      - name: Upload Executable Only
        uses: actions/upload-artifact@v7
        if: inputs.upload-build && startsWith(inputs.os, 'windows')
        with:
          name: Standalone Executable (os=${{inputs.os}})
          path: ${{env.UPLOAD_FOLDER}}/SerialPrograms.exe

      - name: Checkout CommandLineTests
        uses: actions/checkout@v7
        if: inputs.run-tests
        with:
          repository: 'PokemonAutomation/CommandLineTests'
          path: 'CommandLineTests'

      # The unit tests assume the downloadable resources they need are already downloaded
      # (see Packages/Resources/ResourceDownloadList.json). Unzip them from the Packages
      # checkout into where the program looks for them, DOWNLOADED_RESOURCE_PATH():
      # the working directory on Windows/Linux, and ~/Library/Application Support/SerialPrograms
      # on macOS.
      - name: Install downloadable resources for tests
        if: inputs.run-tests
        shell: bash
        run: |
          if [[ "${{inputs.os}}" == mac* ]]; then
            DOWNLOADED_RESOURCES="$HOME/Library/Application Support/SerialPrograms/DownloadedResources"
          else
            DOWNLOADED_RESOURCES="Arduino-Source/SerialPrograms/bin/DownloadedResources"
          fi
          for RESOURCE in PaddleOCRDetection; do
            mkdir -p "$DOWNLOADED_RESOURCES/$RESOURCE"
            (cd "$DOWNLOADED_RESOURCES/$RESOURCE" && cmake -E tar xf "${{github.workspace}}/Arduino-Source/Packages/DownloadedResources/$RESOURCE.zip")
            ls -la "$DOWNLOADED_RESOURCES/$RESOURCE"
          done

      - name: Run tests (Windows)
        if: startsWith(inputs.os, 'windows') && inputs.run-tests
        run: |
          cd Arduino-Source/SerialPrograms/bin
          $process = Start-Process -FilePath "./RelWithDebInfo/SerialPrograms.exe" `
                                   -ArgumentList "--command-line-test-mode" `
                                   -NoNewWindow -Wait -PassThru
          if ($process.ExitCode -ne 0) {
              Write-Error "Tests failed with exit code $($process.ExitCode)"
              exit 1
          }

      - name: Run tests (Ubuntu)
        if: startsWith(inputs.os, 'ubuntu') && inputs.run-tests
        env:
          QT_QPA_PLATFORM: offscreen
          TESSDATA_PREFIX: ${{ github.workspace }}/Arduino-Source/SerialPrograms/bin/Resources/Tesseract
        run: |
          cd Arduino-Source/SerialPrograms/bin
          ./SerialPrograms --command-line-test-mode

      - name: Run tests (Mac)
        if: startsWith(inputs.os, 'mac') && inputs.run-tests
        run: |
          cd Arduino-Source/SerialPrograms/bin
          ./SerialPrograms.app/Contents/MacOS/SerialPrograms --command-line-test-mode

      # The list of files that passed clang-query, from the latest run on this branch or on main.
      # See .github/scripts/clang-query-cached.py. Unlike ccache, every run saves it: it is only
      # about 100 KB, and a PR's next push then checks only the files that changed since.
      - name: Restore clang query cache
        if: inputs.run-clang-query
        uses: actions/cache/restore@v6
        with:
          path: Arduino-Source/clang-query-passed.txt
          key: clang-query-${{github.run_id}}-${{github.run_attempt}}
          restore-keys: clang-query-

      - name: Run clang query
        if: inputs.run-clang-query
        shell: bash
        run: |
          cd Arduino-Source

          # Both rules share one matcher and one traversal of each translation unit.
          cat << 'EOF' > query.txt
          let convertedView expr(
            ignoringParenImpCasts(ignoringImplicit(equalsBoundNode("snapshot_view")))
          )
          let storedView expr(ignoringParenImpCasts(ignoringImplicit(
            expr(anyOf(
              convertedView,
              initListExpr(has(convertedView)),
              explicitCastExpr(hasSourceExpression(convertedView))
            ))
          )))
          match expr(
            anyOf(
              cxxConstructExpr(
                hasDeclaration(cxxConstructorDecl(ofClass(hasName("std::filesystem::path")))),
                hasArgument(0, hasType(asString("std::string")))
              ).bind("path_from_string"),
              cxxMemberCallExpr(
                callee(cxxConversionDecl(
                  ofClass(hasName("PokemonAutomation::VideoSnapshot")),
                  returns(hasDeclaration(cxxRecordDecl(hasName("PokemonAutomation::ImageViewRGB32"))))
                )),
                expr().bind("snapshot_view"),
                anyOf(
                  hasAncestor(varDecl(hasInitializer(storedView))),
                  hasAncestor(fieldDecl(hasInClassInitializer(storedView))),
                  hasAncestor(returnStmt(hasReturnValue(storedView))),
                  hasAncestor(cxxConstructorDecl(forEachConstructorInitializer(withInitializer(storedView)))),
                  hasAncestor(cxxOperatorCallExpr(
                    hasOverloadedOperatorName("="),
                    hasArgument(1, storedView)
                  ))
                )
              )
            ),
            isExpansionInFileMatching("SerialPrograms[/\\]")
          )
          EOF

          # Check each source file in batches, one batch per core, and skip the files that passed
          # before with the same file, headers, compile command, query and tools.
          status=0
          python3 .github/scripts/clang-query-cached.py --build-dir SerialPrograms/bin \
            --query query.txt --cache clang-query-passed.txt --output clang-query-output.txt ||
            status=$?
          echo "Clang query output (begin)"
          cat clang-query-output.txt
          echo "Clang query output (end)"
          if [[ $status -ne 0 ]]; then
            echo "::error::clang-query failed on some files (exit code $status). See the output above."
            exit 1
          fi
          if grep --silent "Match #" clang-query-output.txt; then
            echo "::error Forbidden code detected! It can be one of the following:"
            echo "::error - std::filesystem::path created from std::string. More information https://discord.com/channels/695809740428673034/1462210406616531259/1462567541825339635"
            echo "::error - ImageViewRGB32 created from VideoSnapshot&& that is stored. It is a dangling pointer as nothing hold the data anymore"
            echo "::error You can read more about the exact error with the artifact below."
            exit 1
          fi

      - name: Save clang query cache
        if: always() && inputs.run-clang-query && hashFiles('Arduino-Source/clang-query-passed.txt') != ''
        uses: actions/cache/save@v6
        with:
          path: Arduino-Source/clang-query-passed.txt
          key: clang-query-${{github.run_id}}-${{github.run_attempt}}

      - name: Upload Clang query output
        uses: actions/upload-artifact@v7
        if: inputs.run-clang-query && always()
        with:
          name: Clang query output (compiler=${{inputs.compiler}})
          path: |
            Arduino-Source/SerialPrograms/bin/compile_commands.json
            Arduino-Source/clang-query-output.txt