From d93d7bbe6c042c8c60d22e5c0e23928f302c8ba4 Mon Sep 17 00:00:00 2001 From: Jeff Daily Date: Tue, 16 Jun 2026 18:16:49 +0000 Subject: [PATCH 01/15] [ROCm] Add HIP/ROCm support to the Caspar CUDA backend Adds AMD GPU support to Caspar so its generated kernels and runtime also build and run on AMD GPUs via HIP/ROCm, while leaving the default NVIDIA CUDA build unchanged (enabled with use_hip=True / -DUSE_HIP=ON; off by default). Review in this order: 1. symforce/caspar/source/runtime/cuda_to_hip.h (new): a compatibility header that maps the CUDA spellings Caspar emits and uses (cudaMalloc, __syncthreads, cooperative-group reductions, CUB primitives, the runtime API) onto their HIP equivalents, and supplies device-side fallbacks where HIP lacks a cg:: primitive (reduce, labeled_partition). 2. code_generation/library.py and source/templates/*.jinja: the codegen gains a use_hip/hip_arch path; the generated build file and kernel templates emit the compat include and HIP-correct spellings, so the symbolic kernel definitions are unchanged. 3. source/runtime/*.cu, memops.cuh, pybind_array_tools.{cc,h}: the runtime HIP-compat (shared-memory atomics, the cooperative-group reduction fallback, and a host-side pointer-attribute lookup). 4. A small Windows build fix (C++17, git path separator) for the all-clang ROCm toolchain. 5. README: how to build a generated library for AMD GPUs. Authored with assistance from Claude. Test Plan: The full code-generation pipeline (generate -> HIP-compile -> execute on GPU -> verify numerical output) was exercised on real AMD hardware, Linux CDNA2 (gfx90a) and RDNA3 (gfx1100) and Windows RDNA4 (gfx1201): compile_caspar_library(caslib, out_dir, use_hip=True, hip_arch="gfx90a") # generated kernel executes on GPU; output matches the CUDA path. The default NVIDIA CUDA build (use_hip=False) is unchanged. --- cmake/rerun_if_needed.py | 2 +- symforce/caspar/README.md | 11 ++ symforce/caspar/code_generation/library.py | 11 +- symforce/caspar/source/runtime/cuda_to_hip.h | 134 ++++++++++++++++++ symforce/caspar/source/runtime/memops.cuh | 96 ++++++++++++- .../source/runtime/pybind_array_tools.cc | 13 ++ .../source/runtime/pybind_array_tools.h | 4 + .../caspar/source/runtime/shared_indices.cu | 22 ++- .../caspar/source/runtime/shared_indices.h | 4 + .../caspar/source/runtime/solver_tools.cu | 4 +- .../caspar/source/runtime/sort_indices.cu | 3 +- symforce/caspar/source/runtime/sort_indices.h | 4 + .../templates/buildfiles/CMakeLists.txt.jinja | 70 +++++++-- .../source/templates/caspar_mappings.cu.jinja | 5 +- .../source/templates/caspar_mappings.h.jinja | 2 +- .../caspar/source/templates/kernel.cu.jinja | 8 +- .../caspar/source/templates/kernel.h.jinja | 2 +- .../caspar/source/templates/solver.h.jinja | 2 +- 18 files changed, 353 insertions(+), 44 deletions(-) create mode 100644 symforce/caspar/source/runtime/cuda_to_hip.h diff --git a/cmake/rerun_if_needed.py b/cmake/rerun_if_needed.py index 925bb1f2..b8f47e57 100755 --- a/cmake/rerun_if_needed.py +++ b/cmake/rerun_if_needed.py @@ -84,7 +84,7 @@ def get_path_git_hash(path_to_check: str, repo_root: T.Optional[str] = None) -> Changes to untracked files won't affect the hash. """ # Get relative path from possibly absolute path - relative_path = os.path.relpath(path_to_check, repo_root or os.getcwd()) + relative_path = os.path.relpath(path_to_check, repo_root or os.getcwd()).replace("\\", "/") tree_hash, diff_index = _get_hashes(relative_path, cwd=repo_root) diff --git a/symforce/caspar/README.md b/symforce/caspar/README.md index bdc3dfd0..58c50f2d 100644 --- a/symforce/caspar/README.md +++ b/symforce/caspar/README.md @@ -110,6 +110,17 @@ The following access patterns are currently supported: +### AMD GPUs (ROCm/HIP) + +Caspar also runs on AMD GPUs through ROCm/HIP. The same generated kernels and runtime compile against the HIP toolchain via a small `cuda_to_hip.h` compatibility header that maps the CUDA spellings Caspar emits (`cudaMalloc`, `__syncthreads`, cooperative groups, CUB primitives, and friends) onto their HIP equivalents, so the symbolic kernel definitions are unchanged. + +To build a generated library for AMD GPUs, enable the HIP path when compiling: + +```python +compile_caspar_library(caslib, output_dir, use_hip=True, hip_arch="gfx90a") +``` + +Set the target AMD GPU via the `hip_arch` argument, for example `gfx90a` for CDNA2 (MI200) or `gfx1100` for RDNA3. The ROCm build needs a HIP-enabled compiler (`hipcc`/`amdclang++`) and the `hip` and `hipcub` packages from a ROCm installation. When `use_hip` is off the build is unchanged and continues to use CUDA. ### Etymology Caspar, an acronym for **C**UDA **A**ccelerator for **S**ymbolic **P**rogramming with **A**daptive **R**eordering, is named after the Danish–Norwegian mathematician [Caspar Wessel](https://en.wikipedia.org/wiki/Caspar_Wessel). Wessel was the first to describe the geometrical interpretation of complex numbers as points in the complex plane and as vectors. However, since his thesis was written in Danish, it initially received little recognition. When his work was rediscovered later, the mathematician Sophus Lie, known for his discovery of Lie algebra, wrote the following in the newspaper: diff --git a/symforce/caspar/code_generation/library.py b/symforce/caspar/code_generation/library.py index a1e452ff..a6da1bd4 100644 --- a/symforce/caspar/code_generation/library.py +++ b/symforce/caspar/code_generation/library.py @@ -180,6 +180,8 @@ def compile( debug: bool = False, cuda_arch: str | None = None, jobs: int | None = None, + use_hip: bool = False, + hip_arch: str | None = None, ) -> None: build_dir = out_dir / "build" config_cmd: list[str | Path] = [ @@ -189,8 +191,15 @@ def compile( "-B", build_dir, f"-DCMAKE_BUILD_TYPE={'Debug' if debug else 'Release'}", - f"-DCMAKE_CUDA_ARCHITECTURES={cuda_arch if cuda_arch is not None else _real_cuda_arch_string()}", ] + if use_hip: + config_cmd.append("-DUSE_HIP=ON") + if hip_arch: + config_cmd.append(f"-DCMAKE_HIP_ARCHITECTURES={hip_arch}") + else: + config_cmd.append( + f"-DCMAKE_CUDA_ARCHITECTURES={cuda_arch if cuda_arch is not None else _real_cuda_arch_string()}" + ) subprocess.run(config_cmd, check=True) build_cmd: list[str | Path] = ["cmake", "--build", build_dir, "--parallel"] if jobs: diff --git a/symforce/caspar/source/runtime/cuda_to_hip.h b/symforce/caspar/source/runtime/cuda_to_hip.h new file mode 100644 index 00000000..5fa7148f --- /dev/null +++ b/symforce/caspar/source/runtime/cuda_to_hip.h @@ -0,0 +1,134 @@ +/* ---------------------------------------------------------------------------- + * SymForce - Copyright 2025, Skydio, Inc. + * Copyright (c) 2026 Advanced Micro Devices, Inc. + * This source code is under the Apache 2.0 license found in the LICENSE file. + * + * Author: Jeff Daily + * ---------------------------------------------------------------------------- */ + +#pragma once + +// CUDA-to-HIP compatibility header for ROCm/HIP builds. +// Provides the CUDA API spellings the project uses via aliases to HIP equivalents. + +#if defined(USE_HIP) || defined(__HIP_PLATFORM_AMD__) + +#include +#include +#include + +// CUDA runtime API -> HIP runtime API +#define cudaMalloc hipMalloc +#define cudaFree hipFree +#define cudaMemset hipMemset +#define cudaMemcpy hipMemcpy +#define cudaMemcpyDeviceToDevice hipMemcpyDeviceToDevice +#define cudaMemcpyDeviceToHost hipMemcpyDeviceToHost +#define cudaMemcpyHostToDevice hipMemcpyHostToDevice +#define cudaError_t hipError_t +#define cudaSuccess hipSuccess +#define cudaStream_t hipStream_t +#define cudaDeviceSynchronize hipDeviceSynchronize +#define cudaGetLastError hipGetLastError +#define cudaGetErrorString hipGetErrorString +#define cudaSetDevice hipSetDevice +#define cudaGetDevice hipGetDevice +#define cudaPointerGetAttributes hipPointerGetAttributes +#define cudaPointerAttributes hipPointerAttribute_t + +// HIP shared-memory atomics are block-scoped by definition (no inter-block visibility), +// so atomicAdd_block is equivalent to atomicAdd on shared memory. +#define atomicAdd_block atomicAdd + +// CUB -> hipCUB namespace +namespace cub = hipcub; + +// Cooperative groups: HIP has cg basics but lacks cg::reduce, cg::labeled_partition, memcpy_async. +// Provide manual implementations where needed. + +namespace caspar_hip { + +// Butterfly reduction within a cooperative group (thread_block_tile or coalesced_group). +// Replaces cg::reduce(group, val, cg::plus()). +template +__device__ __forceinline__ T reduce_sum(GroupT group, T val) { + for (unsigned int offset = group.size() / 2; offset > 0; offset >>= 1) { + val += group.shfl_xor(val, offset); + } + return val; +} + +// Butterfly reduction for max (cg::greater). +template +__device__ __forceinline__ T reduce_max(GroupT group, T val) { + for (unsigned int offset = group.size() / 2; offset > 0; offset >>= 1) { + T other = group.shfl_xor(val, offset); + val = (val > other) ? val : other; + } + return val; +} + +// Match_any: return a mask of lanes in the tile that have the same label. +// HIP CG has match_any() for coalesced groups. +template +__device__ __forceinline__ unsigned long long match_any_mask(GroupT group, LabelT label) { + // HIP cooperative_groups::coalesced_group has match_any + return group.match_any(label); +} + +// Labeled partition emulation: reduce values within lanes sharing the same label, +// and have exactly one lane per unique label perform the atomic. +// Returns the reduced value and sets is_leader=true for exactly one lane per label. +template +__device__ __forceinline__ T labeled_reduce_sum(GroupT group, T val, LabelT label, bool& is_leader) { + // Get mask of lanes with same label + unsigned long long same_label_mask = group.match_any(label); + + // Find my position within the matching lanes + unsigned int my_lane = group.thread_rank(); + unsigned long long lower_mask = (1ULL << my_lane) - 1; + unsigned int rank_in_label = __popcll(same_label_mask & lower_mask); + + // Leader is the lowest-numbered lane in the group + is_leader = (rank_in_label == 0); + + // Count total lanes with this label + unsigned int label_size = __popcll(same_label_mask); + + // Butterfly reduction over the masked lanes + // For each reduction step, exchange with lane at offset if both are in same_label_mask + T result = val; + for (unsigned int offset = 1; offset < group.size(); offset <<= 1) { + unsigned int partner_lane = my_lane ^ offset; + bool partner_has_same_label = (same_label_mask >> partner_lane) & 1ULL; + T partner_val = group.shfl_xor(result, offset); + if (partner_has_same_label && partner_lane < group.size()) { + result += partner_val; + } + } + + return result; +} + +} // namespace caspar_hip + +// Macros to replace cg::reduce calls (used in device code) +#define CG_REDUCE_SUM(group, val) caspar_hip::reduce_sum(group, val) +#define CG_LABELED_REDUCE_SUM(group, val, label, is_leader) \ + caspar_hip::labeled_reduce_sum(group, val, label, is_leader) + +#else // CUDA path + +#include +#include +#include +#include +#include +#include + +namespace cg = cooperative_groups; + +// On CUDA, cg::reduce is available +#define CG_REDUCE_SUM(group, val) cg::reduce(group, val, cg::plus()) + +#endif diff --git a/symforce/caspar/source/runtime/memops.cuh b/symforce/caspar/source/runtime/memops.cuh index 49dc5299..dcdd31f6 100644 --- a/symforce/caspar/source/runtime/memops.cuh +++ b/symforce/caspar/source/runtime/memops.cuh @@ -7,11 +7,7 @@ #include -#include -#include -#include -#include - +#include "cuda_to_hip.h" #include "shared_indices.h" namespace cg = cooperative_groups; @@ -260,6 +256,32 @@ __forceinline__ __device__ void FlushSumShared(StorageT* const output, const uin if (idx.argsort != 0xffff) { // 0xffff indicates the thread is not used. unique = indices[indices[idx.argsort].target].unique; } + +#if defined(USE_HIP) || defined(__HIP_PLATFORM_AMD__) + // HIP lacks cg::labeled_partition; use per-lane atomicAdd fallback. + // The butterfly approach doesn't work for non-contiguous label groups (lanes at + // arbitrary positions share a label, but XOR only pairs specific distances). + // Per-lane atomicAdd is simpler and correct (shared-memory atomics are fast). + +#pragma unroll + for (int i = 0; i < dim_target; i++) { + const SharedIndex idx_inner = indices[threadIdx.x]; + StorageT val = StorageT(0); + if (idx_inner.argsort != 0xffff) { + // Read value BEFORE zeroing (order matters!) + val = inout_shared[idx_inner.argsort * dim_target + i]; + } + __syncthreads(); + inout_shared[threadIdx.x * dim_target + i] = 0.0f; + __syncthreads(); + + if (idx_inner.argsort != 0xffff) { + // Each lane with valid data does its own atomicAdd to the target location + atomicAdd_block(&inout_shared[indices[idx_inner.argsort].target * dim_target + i], val); + } + __syncthreads(); + } +#else const cg::coalesced_group group = cg::labeled_partition(cg::coalesced_threads(), unique); #pragma unroll @@ -279,6 +301,7 @@ __forceinline__ __device__ void FlushSumShared(StorageT* const output, const uin } __syncthreads(); } +#endif constexpr uint dim_aligned = dim_target == 3 ? 4 : dim_target; for (int i = 0; i < dim_target; i++) { @@ -303,6 +326,29 @@ template __forceinline__ __device__ void FlushSumBlock(StorageT* const output, StorageT* const inout_shared, const bool valid) { __syncthreads(); +#if defined(USE_HIP) || defined(__HIP_PLATFORM_AMD__) + // HIP coalesced_group lacks shfl_xor; use shared-memory atomics directly. + constexpr uint dim_aligned = dim_target == 3 ? 4 : dim_target; + +#pragma unroll + for (int i = 0; i < dim_target; i++) { + StorageT val = StorageT(0); + if (valid) { + val = inout_shared[threadIdx.x * dim_target + i]; + } + __syncthreads(); + inout_shared[threadIdx.x * dim_target + i] = 0.0f; + __syncthreads(); + + if (valid) { + atomicAdd_block(&inout_shared[i], val); + } + __syncthreads(); + } + for (int i = threadIdx.x; i < dim_target; i += blockDim.x) { + output[blockIdx.x * dim_aligned + i] = inout_shared[i]; + } +#else const cg::coalesced_group group = cg::binary_partition(cg::coalesced_threads(), valid); constexpr uint dim_aligned = dim_target == 3 ? 4 : dim_target; @@ -326,6 +372,7 @@ __forceinline__ __device__ void FlushSumBlock(StorageT* const output, StorageT* for (int i = threadIdx.x; i < dim_target; i += blockDim.x) { output[blockIdx.x * dim_aligned + i] = inout_shared[i]; } +#endif } template __forceinline__ __device__ void SumStore(StorageT* const shared_tmp, StorageT* const inout_shared, @@ -333,13 +380,28 @@ __forceinline__ __device__ void SumStore(StorageT* const shared_tmp, StorageT* c auto group = cg::tiled_partition<32>(cg::this_thread_block()); __syncthreads(); +#if defined(USE_HIP) || defined(__HIP_PLATFORM_AMD__) + // HIP lacks cg::reduce; use butterfly reduction within the tile + StorageT tot = valid ? data : StorageT(0); + for (unsigned int o = group.size() / 2; o > 0; o >>= 1) { + tot += group.shfl_xor(tot, o); + } +#else StorageT tot = cg::reduce(group, valid ? data : 0.0f, cg::plus()); +#endif if (group.thread_rank() == 0) { inout_shared[group.meta_group_rank()] = tot; } __syncthreads(); if (group.meta_group_rank() == 0) { +#if defined(USE_HIP) || defined(__HIP_PLATFORM_AMD__) + tot = inout_shared[group.thread_rank()]; + for (unsigned int o = group.size() / 2; o > 0; o >>= 1) { + tot += group.shfl_xor(tot, o); + } +#else tot = cg::reduce(group, inout_shared[group.thread_rank()], cg::plus()); +#endif if (group.thread_rank() == 0) { shared_tmp[offset] = tot; } @@ -364,6 +426,29 @@ template __forceinline__ __device__ void FlushSumBlockAdd(StorageT* const output, StorageT* const inout_shared, const bool valid) { __syncthreads(); +#if defined(USE_HIP) || defined(__HIP_PLATFORM_AMD__) + // HIP coalesced_group lacks shfl_xor; use shared-memory atomics directly. + constexpr uint dim_aligned = dim_target == 3 ? 4 : dim_target; + +#pragma unroll + for (int i = 0; i < dim_target; i++) { + StorageT val = StorageT(0); + if (valid) { + val = inout_shared[threadIdx.x * dim_target + i]; + } + __syncthreads(); + inout_shared[threadIdx.x * dim_target + i] = 0.0f; + __syncthreads(); + + if (valid) { + atomicAdd_block(&inout_shared[i], val); + } + __syncthreads(); + } + for (int i = threadIdx.x; i < dim_target; i += blockDim.x) { + output[blockIdx.x * dim_aligned + i] += inout_shared[i]; + } +#else const cg::coalesced_group group = cg::binary_partition(cg::coalesced_threads(), valid); constexpr uint dim_aligned = dim_target == 3 ? 4 : dim_target; @@ -387,6 +472,7 @@ __forceinline__ __device__ void FlushSumBlockAdd(StorageT* const output, for (int i = threadIdx.x; i < dim_target; i += blockDim.x) { output[blockIdx.x * dim_aligned + i] += inout_shared[i]; } +#endif } // READ SHARED diff --git a/symforce/caspar/source/runtime/pybind_array_tools.cc b/symforce/caspar/source/runtime/pybind_array_tools.cc index 9e43f198..7f057905 100644 --- a/symforce/caspar/source/runtime/pybind_array_tools.cc +++ b/symforce/caspar/source/runtime/pybind_array_tools.cc @@ -174,6 +174,18 @@ int GetDeviceId(const py::object& obj) { auto interface = obj.attr("__cuda_array_interface__").cast(); auto data = interface["data"].cast(); void* ptr = reinterpret_cast(data[0].cast()); + // This is a host translation unit, so it uses local cuda*->hip* aliases here + // rather than the device-side cuda_to_hip.h (which pulls in hipcub and + // __device__ helpers intended for the .cu sources). +#if defined(USE_HIP) || defined(__HIP_PLATFORM_AMD__) + hipPointerAttribute_t attrs; + hipError_t err = hipPointerGetAttributes(&attrs, ptr); + if (err != hipSuccess) { + hipGetLastError(); + return -1; + } + return attrs.device; +#else cudaPointerAttributes attrs; cudaError_t err = cudaPointerGetAttributes(&attrs, ptr); if (err != cudaSuccess) { @@ -181,6 +193,7 @@ int GetDeviceId(const py::object& obj) { return -1; } return attrs.device; +#endif } catch (...) { return -1; // Fallback if interface or attributes aren't available } diff --git a/symforce/caspar/source/runtime/pybind_array_tools.h b/symforce/caspar/source/runtime/pybind_array_tools.h index 96445288..1e938ed6 100644 --- a/symforce/caspar/source/runtime/pybind_array_tools.h +++ b/symforce/caspar/source/runtime/pybind_array_tools.h @@ -14,7 +14,11 @@ #pragma once +#if defined(USE_HIP) || defined(__HIP_PLATFORM_AMD__) +#include +#else #include +#endif #include namespace py = pybind11; diff --git a/symforce/caspar/source/runtime/shared_indices.cu b/symforce/caspar/source/runtime/shared_indices.cu index f3d20c61..ecc4baab 100644 --- a/symforce/caspar/source/runtime/shared_indices.cu +++ b/symforce/caspar/source/runtime/shared_indices.cu @@ -3,10 +3,7 @@ * This source code is under the Apache 2.0 license found in the LICENSE file. * ---------------------------------------------------------------------------- */ -#include -#include -#include - +#include "cuda_to_hip.h" #include "shared_indices.h" namespace cg = cooperative_groups; @@ -111,16 +108,31 @@ __forceinline__ __device__ uint GetOrd(const uint* values, const uint length, co __global__ void SharedIndicesKernel(const uint* const __restrict__ indices, SharedIndex* const __restrict__ shared_indices_out, const uint size) { - const auto block = cg::this_thread_block(); + auto block = cg::this_thread_block(); const auto gtrank = cg::this_grid().thread_rank(); const auto btrank = block.thread_rank(); +#if defined(USE_HIP) || defined(__HIP_PLATFORM_AMD__) + // HIP's group_dim() is non-const; use blockDim directly + const int start = blockIdx.x * blockDim.x; +#else const int start = block.group_index().x * block.group_dim().x; +#endif const int num = min(1024, size - start) * sizeof(uint); __shared__ uint indices_loc[1024]; +#if defined(USE_HIP) || defined(__HIP_PLATFORM_AMD__) + // HIP lacks cg::memcpy_async; use synchronous block-strided copy + const uint* src = indices + start; + const uint num_elements = num / sizeof(uint); + for (uint i = btrank; i < num_elements; i += block.size()) { + indices_loc[i] = src[i]; + } + __syncthreads(); +#else cg::memcpy_async(block, indices_loc, indices + start, num); cg::wait(block); +#endif const uint val = gtrank < size ? indices_loc[btrank] : 0xFFFFFFFF; diff --git a/symforce/caspar/source/runtime/shared_indices.h b/symforce/caspar/source/runtime/shared_indices.h index 8b2db65e..e40fed15 100644 --- a/symforce/caspar/source/runtime/shared_indices.h +++ b/symforce/caspar/source/runtime/shared_indices.h @@ -7,7 +7,11 @@ #include +#if defined(USE_HIP) || defined(__HIP_PLATFORM_AMD__) +#include +#else #include +#endif namespace caspar { diff --git a/symforce/caspar/source/runtime/solver_tools.cu b/symforce/caspar/source/runtime/solver_tools.cu index 5954fb04..ca19dc0f 100644 --- a/symforce/caspar/source/runtime/solver_tools.cu +++ b/symforce/caspar/source/runtime/solver_tools.cu @@ -3,9 +3,7 @@ * This source code is under the Apache 2.0 license found in the LICENSE file. * ---------------------------------------------------------------------------- */ -#include -#include - +#include "cuda_to_hip.h" #include "solver_tools.h" namespace caspar { diff --git a/symforce/caspar/source/runtime/sort_indices.cu b/symforce/caspar/source/runtime/sort_indices.cu index 0e1356ec..ad14e711 100644 --- a/symforce/caspar/source/runtime/sort_indices.cu +++ b/symforce/caspar/source/runtime/sort_indices.cu @@ -3,8 +3,7 @@ * This source code is under the Apache 2.0 license found in the LICENSE file. * ---------------------------------------------------------------------------- */ -#include - +#include "cuda_to_hip.h" #include "sort_indices.h" namespace { diff --git a/symforce/caspar/source/runtime/sort_indices.h b/symforce/caspar/source/runtime/sort_indices.h index 37800303..311790d8 100644 --- a/symforce/caspar/source/runtime/sort_indices.h +++ b/symforce/caspar/source/runtime/sort_indices.h @@ -5,7 +5,11 @@ #pragma once +#if defined(USE_HIP) || defined(__HIP_PLATFORM_AMD__) +#include +#else #include +#endif namespace caspar { diff --git a/symforce/caspar/source/templates/buildfiles/CMakeLists.txt.jinja b/symforce/caspar/source/templates/buildfiles/CMakeLists.txt.jinja index c7b1c6eb..208f86ca 100644 --- a/symforce/caspar/source/templates/buildfiles/CMakeLists.txt.jinja +++ b/symforce/caspar/source/templates/buildfiles/CMakeLists.txt.jinja @@ -4,18 +4,35 @@ # ---------------------------------------------------------------------------- #} cmake_minimum_required(VERSION 3.18) -if(NOT DEFINED CMAKE_CUDA_COMPILER AND DEFINED ENV{CUDACXX}) - set(CMAKE_CUDA_COMPILER "$ENV{CUDACXX}" CACHE FILEPATH "CUDA compiler") -endif() +set(CMAKE_CXX_STANDARD 17) +set(CMAKE_CXX_STANDARD_REQUIRED ON) -project(caspar_library LANGUAGES CXX CUDA) +option(USE_HIP "Build with HIP for AMD GPUs" OFF) -# SASS for Turing/Ampere/Ada (sm_75–89) + PTX fallback for forward JIT-compatibility. sm_75 minimum required by cooperative_groups::reduce. -if(NOT DEFINED CMAKE_CUDA_ARCHITECTURES) - set(CMAKE_CUDA_ARCHITECTURES 75 80 86 89 75-virtual) -endif() +if(USE_HIP) + project(caspar_library LANGUAGES CXX HIP) + + # Default to gfx90a if not specified + if(NOT DEFINED CMAKE_HIP_ARCHITECTURES OR CMAKE_HIP_ARCHITECTURES STREQUAL "") + set(CMAKE_HIP_ARCHITECTURES "gfx90a") + endif() + + find_package(hip REQUIRED) + find_package(hipcub REQUIRED) +else() + if(NOT DEFINED CMAKE_CUDA_COMPILER AND DEFINED ENV{CUDACXX}) + set(CMAKE_CUDA_COMPILER "$ENV{CUDACXX}" CACHE FILEPATH "CUDA compiler") + endif() + + project(caspar_library LANGUAGES CXX CUDA) -find_package(CUDAToolkit REQUIRED) + # SASS for Turing/Ampere/Ada (sm_75-89) + PTX fallback for forward JIT-compatibility. sm_75 minimum required by cooperative_groups::reduce. + if(NOT DEFINED CMAKE_CUDA_ARCHITECTURES) + set(CMAKE_CUDA_ARCHITECTURES 75 80 86 89 75-virtual) + endif() + + find_package(CUDAToolkit REQUIRED) +endif() {% if python_bindings %} option(CASPAR_BUILD_PYTHON_BINDINGS "Build Python bindings via pybind11" ON) @@ -45,10 +62,31 @@ list(FILTER CUDA_HEADERS EXCLUDE REGEX "pybind.*") add_library({{caslib.name}}_core STATIC ${CUDA_SOURCES} ${CUDA_HEADERS}) target_include_directories({{caslib.name}}_core PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}) -target_link_libraries({{caslib.name}}_core PUBLIC CUDA::cudart) -target_compile_options({{caslib.name}}_core PRIVATE - $<$:--use_fast_math> -) + +if(USE_HIP) + # Mark .cu files as HIP language + set_source_files_properties(${CUDA_SOURCES} PROPERTIES LANGUAGE HIP) + target_compile_definitions({{caslib.name}}_core PUBLIC USE_HIP) + # Link hip targets PRIVATE to avoid propagating HIP compile options to pure-CXX consumers. + # The static library has the HIP code linked in; pybind consumers only need the includes. + target_link_libraries({{caslib.name}}_core PRIVATE hip::host hip::hipcub) + # Expose ROCm include directory for consumers that need hipError_t etc + target_include_directories({{caslib.name}}_core PUBLIC ${hip_INCLUDE_DIRS}) + # Use -ffp-contract=fast -fno-math-errno instead of -ffast-math. + # clang's -ffast-math enables -fassociative-math which can NaN backward passes. + target_compile_options({{caslib.name}}_core PRIVATE + $<$:-ffp-contract=fast -fno-math-errno> + ) + set_target_properties({{caslib.name}}_core PROPERTIES + HIP_ARCHITECTURES "${CMAKE_HIP_ARCHITECTURES}" + ) +else() + target_link_libraries({{caslib.name}}_core PUBLIC CUDA::cudart) + target_compile_options({{caslib.name}}_core PRIVATE + $<$:--use_fast_math> + ) +endif() + set_target_properties({{caslib.name}}_core PROPERTIES POSITION_INDEPENDENT_CODE ON ) @@ -60,6 +98,12 @@ if(CASPAR_BUILD_PYTHON_BINDINGS) pybind11_add_module({{caslib.name}} ${CPP_SOURCES} ${CPP_HEADERS}) target_link_libraries({{caslib.name}} PRIVATE {{caslib.name}}_core) + if(USE_HIP) + # Compile pybind sources as HIP (so HIP headers work) but they have no device code. + # The HIP compiler handles them as host-only code. + set_source_files_properties(${CPP_SOURCES} PROPERTIES LANGUAGE HIP) + endif() + set_target_properties({{caslib.name}} PROPERTIES LIBRARY_OUTPUT_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} RUNTIME_OUTPUT_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} diff --git a/symforce/caspar/source/templates/caspar_mappings.cu.jinja b/symforce/caspar/source/templates/caspar_mappings.cu.jinja index eb398676..ac314bf0 100644 --- a/symforce/caspar/source/templates/caspar_mappings.cu.jinja +++ b/symforce/caspar/source/templates/caspar_mappings.cu.jinja @@ -4,13 +4,10 @@ # ---------------------------------------------------------------------------- #} #include -#include -#include +#include "cuda_to_hip.h" #include "caspar_mappings.h" -namespace cg = cooperative_groups; - // We use shared memory to improve the memory access. // A smaller block size of 32 allows for larger nodetypes. constexpr int block_size = 32; diff --git a/symforce/caspar/source/templates/caspar_mappings.h.jinja b/symforce/caspar/source/templates/caspar_mappings.h.jinja index af3795ad..53684205 100644 --- a/symforce/caspar/source/templates/caspar_mappings.h.jinja +++ b/symforce/caspar/source/templates/caspar_mappings.h.jinja @@ -4,7 +4,7 @@ # ---------------------------------------------------------------------------- #} #pragma once -#include +#include "cuda_to_hip.h" namespace caspar { {% for nodetype in caslib.exposed_types%} diff --git a/symforce/caspar/source/templates/kernel.cu.jinja b/symforce/caspar/source/templates/kernel.cu.jinja index fa30d92d..e18addd0 100644 --- a/symforce/caspar/source/templates/kernel.cu.jinja +++ b/symforce/caspar/source/templates/kernel.cu.jinja @@ -2,17 +2,11 @@ # SymForce - Copyright 2025, Skydio, Inc. # This source code is under the Apache 2.0 license found in the LICENSE file. # ---------------------------------------------------------------------------- #} -#include -#include -#include -#include -#include +#include "cuda_to_hip.h" #include "kernel_{{kernel.name}}.h" #include "memops.cuh" -namespace cg = cooperative_groups; - namespace caspar { __global__ void diff --git a/symforce/caspar/source/templates/kernel.h.jinja b/symforce/caspar/source/templates/kernel.h.jinja index 476a50e4..8c189f4c 100644 --- a/symforce/caspar/source/templates/kernel.h.jinja +++ b/symforce/caspar/source/templates/kernel.h.jinja @@ -4,7 +4,7 @@ # ---------------------------------------------------------------------------- #} #pragma once -#include +#include "cuda_to_hip.h" #include "shared_indices.h" diff --git a/symforce/caspar/source/templates/solver.h.jinja b/symforce/caspar/source/templates/solver.h.jinja index 855a9789..c9a35c7a 100644 --- a/symforce/caspar/source/templates/solver.h.jinja +++ b/symforce/caspar/source/templates/solver.h.jinja @@ -6,7 +6,7 @@ #include -#include +#include "cuda_to_hip.h" #include From e994ef0b441d5502b07cd58e21a0fc2489eccec3 Mon Sep 17 00:00:00 2001 From: Jeff Daily Date: Tue, 23 Jun 2026 22:23:00 +0000 Subject: [PATCH 02/15] [ROCm] Rely on HIP arch auto-detect instead of pinning gfx90a The generated CMake template pinned CMAKE_HIP_ARCHITECTURES to gfx90a when unset, but the pin ran after project(... LANGUAGES ... HIP), which already enables the HIP language and resolves the architecture. The block was dead code that could only mislead, or become a live footgun if file order ever changed. project(... LANGUAGES ... HIP) already honors an explicit -DCMAKE_HIP_ARCHITECTURES, otherwise auto-detects the host GPU, and errors on a no-GPU build host. Removing the pin lets that single mechanism decide the arch, so a user on a non-gfx90a card no longer risks a silently mistargeted default. The hip_arch argument still threads through as -DCMAKE_HIP_ARCHITECTURES. This work was authored with the assistance of the Claude AI assistant. --- .../source/templates/buildfiles/CMakeLists.txt.jinja | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/symforce/caspar/source/templates/buildfiles/CMakeLists.txt.jinja b/symforce/caspar/source/templates/buildfiles/CMakeLists.txt.jinja index 208f86ca..bec1bc7c 100644 --- a/symforce/caspar/source/templates/buildfiles/CMakeLists.txt.jinja +++ b/symforce/caspar/source/templates/buildfiles/CMakeLists.txt.jinja @@ -10,13 +10,10 @@ set(CMAKE_CXX_STANDARD_REQUIRED ON) option(USE_HIP "Build with HIP for AMD GPUs" OFF) if(USE_HIP) + # project(... LANGUAGES ... HIP) honors -DCMAKE_HIP_ARCHITECTURES, otherwise + # auto-detects the host GPU and errors on a no-GPU build host. project(caspar_library LANGUAGES CXX HIP) - # Default to gfx90a if not specified - if(NOT DEFINED CMAKE_HIP_ARCHITECTURES OR CMAKE_HIP_ARCHITECTURES STREQUAL "") - set(CMAKE_HIP_ARCHITECTURES "gfx90a") - endif() - find_package(hip REQUIRED) find_package(hipcub REQUIRED) else() From 1a9e9770c375840a277a5fcd04416eb2c5a7f009 Mon Sep 17 00:00:00 2001 From: Jeff Daily Date: Thu, 20 Aug 2026 17:48:02 +0000 Subject: [PATCH 03/15] [ROCm] Harden SumStore's shared scratch buffer reuse contract SumStore's second stage has the first warp tile read inout_shared and reduce it into shared_tmp, and the function returns without a trailing __syncthreads(). The buffer is therefore not safe to reuse until the next block-wide barrier, and today that safety rests on non-local invariants of the call sites: SumStore and SumFlushFinal each open with a block-wide barrier, and every other store into the buffer is indexed by threadIdx.x scaled by the storing accessor's element stride, so nothing below element 32 is written by a thread outside the reading tile, whose own reads the shuffle collectives order ahead of any lane leaving the function. The trailing barrier replaces those implicit guarantees with a local one, at the cost of one barrier per call. Suggested by a user running this branch on gfx1151 / ROCm 7.2, where solver runs showed intermittent NaNs and run-to-run nondeterminism on the same binary, and adding this barrier made their determinism protocol pass 22/22 across fresh seeds and a larger synthetic problem. The precise interleaving behind their failure was not pinned down (their own revert-and-retest was inconclusive), so this change is offered on the strength of that empirical evidence and of the contract argument above, not a root-caused race. The barrier is not gated to HIP: the reuse contract is not platform-specific. Authored with assistance from Claude (Anthropic). Test Plan: Linux gfx1100 (Radeon Pro W7800, ROCm 7.2.1). A standalone harness compiles memops.cuh with hipcc and launches a 1024-thread kernel issuing four back-to-back SumStore calls against the same inout_shared buffer followed by SumFlushFinal, over a problem size that exercises the valid mask: ``` hipcc -x hip sumstore_test.cpp -I symforce/caspar/source/runtime --offload-arch=gfx1100 -o sumstore_test ./sumstore_test # all 4 sums exact, 20 reruns bit-identical ``` The pre-change code passes the same harness on this card (that call pattern is protected by SumStore's own leading barrier, and the gfx1151 failure does not reproduce here), so this is verified as a no-regression hardening. The changed header is not part of the prebuilt runtime library; it is compiled by the generated kernels, so the CUDA side is covered by compiling a representative translation unit with nvcc 12.8: ``` nvcc -arch=sm_75 -c nvcc_memops.cu -I symforce/caspar/source/runtime -o nvcc_memops.o # nvcc_memops.cu: includes memops.cuh and instantiates SumStore, SumFlushFinal, FlushSumBlock ``` Co-authored-by: bjoernellens1 <64093272+bjoernellens1@users.noreply.github.com> --- symforce/caspar/source/runtime/memops.cuh | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/symforce/caspar/source/runtime/memops.cuh b/symforce/caspar/source/runtime/memops.cuh index dcdd31f6..63d89de0 100644 --- a/symforce/caspar/source/runtime/memops.cuh +++ b/symforce/caspar/source/runtime/memops.cuh @@ -406,6 +406,10 @@ __forceinline__ __device__ void SumStore(StorageT* const shared_tmp, StorageT* c shared_tmp[offset] = tot; } } + // The second-stage tile reads inout_shared above; without a trailing barrier the + // buffer is not safe to reuse until the next block-wide sync, which callers were + // implicitly relied on to provide. + __syncthreads(); } template From 38f16ea39b5e30daca70bb9da4d05cb3d01ce453 Mon Sep 17 00:00:00 2001 From: Jeff Daily Date: Wed, 4 Nov 2026 19:47:51 +0000 Subject: [PATCH 04/15] [ROCm] Map the CUDA calls used by the generated kernel launches Upstream now launches generated kernels with dynamic shared memory, opting in with cudaFuncSetAttribute(..., cudaFuncAttributeMaxDynamicSharedMemorySize, ...) when a kernel needs more than 48 KB, and the generated solver checks cudaGetDeviceCount. The HIP compatibility header did not map these, so generated code no longer compiled for ROCm. hipFuncSetAttribute only takes the kernel as const void*, while the CUDA runtime also accepts the kernel function itself, so the header adds a small template overload that casts it. Authored with Claude. Test Plan: Build the Caspar GPU tests (generated kernels and solvers checked into test/) for gfx90a and run them on an MI250X: ``` cmake -B build -DCMAKE_HIP_ARCHITECTURES=gfx90a -DCMAKE_PREFIX_PATH= cmake --build build --target bin.tests.caspar_padding_lane_cuda_test bin.tests.caspar_solver_cuda_test bin.tests.caspar_backend_coverage_cuda_test ./build/bin/tests/caspar_padding_lane_cuda_test ./build/bin/tests/caspar_solver_cuda_test ./build/bin/tests/caspar_backend_coverage_cuda_test ``` --- symforce/caspar/source/runtime/cuda_to_hip.h | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/symforce/caspar/source/runtime/cuda_to_hip.h b/symforce/caspar/source/runtime/cuda_to_hip.h index 5fa7148f..e9141b2c 100644 --- a/symforce/caspar/source/runtime/cuda_to_hip.h +++ b/symforce/caspar/source/runtime/cuda_to_hip.h @@ -35,6 +35,14 @@ #define cudaGetDevice hipGetDevice #define cudaPointerGetAttributes hipPointerGetAttributes #define cudaPointerAttributes hipPointerAttribute_t +#define cudaGetDeviceCount hipGetDeviceCount +#define cudaFuncAttributeMaxDynamicSharedMemorySize hipFuncAttributeMaxDynamicSharedMemorySize + +// hipFuncSetAttribute takes the kernel as const void*; CUDA also accepts the kernel itself. +template +inline hipError_t cudaFuncSetAttribute(T* func, hipFuncAttribute attr, int value) { + return hipFuncSetAttribute(reinterpret_cast(func), attr, value); +} // HIP shared-memory atomics are block-scoped by definition (no inter-block visibility), // so atomicAdd_block is equivalent to atomicAdd on shared memory. From 1a984fc098e5992b5f6b05e047f04b2fbf97793e Mon Sep 17 00:00:00 2001 From: Jeff Daily Date: Wed, 4 Nov 2026 19:47:51 +0000 Subject: [PATCH 05/15] [ROCm] Drop the trailing SumStore barrier This reverts the trailing __syncthreads() added to SumStore in "[ROCm] Harden SumStore's shared scratch buffer reuse contract". That barrier was added for a gfx1151 report of NaN scores and run-to-run nondeterminism in solves, without a root-caused race. Upstream has since fixed the mechanism behind that symptom family in "[Caspar] Zero initialize caspar kernel registers" (#469): lanes past problem_size passed uninitialized registers to SumStore, and LLVM can drop the valid-lane mask. Uninitialized registers explain both the NaN and the run-to-run variation. With register zeroing in place, the barrier is not needed: - SumStore and SumFlushFinal open with a block-wide barrier, and every other store into the scratch buffer below element 32 comes from a lane of the tile that reads it. - On an MI250X (gfx90a), the padding-lane score kernel and the solver's alpha numerator/denominator kernel (two SumStore calls back to back into one buffer) give bit-identical per-block partial sums over 200 runs at sizes from 1000 to 1000003, with and without the barrier. With integer inputs, the alpha kernel's per-block sums equal an exact host reference in every run. - Without register zeroing, the padding-lane score is NaN at every size that leaves padding lanes, whether or not the barrier is present. So the barrier does not fix that defect either. Authored with Claude. Test Plan: Build upstream's checked-in kernels against memops.cuh with and without the barrier, with SumFlushFinal's atomicAdd replaced by a per-block store so cross-block float atomic ordering does not mask anything, and run each 200 times: ``` hipcc -x hip -std=c++17 -O3 -DUSE_HIP --offload-arch=gfx90a -I. -Iruntime main.cc kernel_caspar_padding_lane_score.cu runtime/shared_indices.cu -o padding_lane hipcc -x hip -std=c++17 -O3 -DUSE_HIP --offload-arch=gfx90a -I. -Iruntime main_alpha.cc kernel_Matrix21_alpha_numerator_denominator.cu -o alpha ./padding_lane 200 ./alpha 200 ``` --- symforce/caspar/source/runtime/memops.cuh | 4 ---- 1 file changed, 4 deletions(-) diff --git a/symforce/caspar/source/runtime/memops.cuh b/symforce/caspar/source/runtime/memops.cuh index 63d89de0..dcdd31f6 100644 --- a/symforce/caspar/source/runtime/memops.cuh +++ b/symforce/caspar/source/runtime/memops.cuh @@ -406,10 +406,6 @@ __forceinline__ __device__ void SumStore(StorageT* const shared_tmp, StorageT* c shared_tmp[offset] = tot; } } - // The second-stage tile reads inout_shared above; without a trailing barrier the - // buffer is not safe to reuse until the next block-wide sync, which callers were - // implicitly relied on to provide. - __syncthreads(); } template From 5256d2ada46e5244cd6b3fd0b2b7f43a435ef550 Mon Sep 17 00:00:00 2001 From: Jeff Daily Date: Wed, 4 Nov 2026 19:48:02 +0000 Subject: [PATCH 06/15] [ROCm] Regenerate the Caspar codegen test data for the HIP header The generated kernels and solvers include cuda_to_hip.h instead of the CUDA runtime and cooperative groups headers, so the code symforce_caspar_codegen_test checks in no longer matched what the templates generate. This regenerates it with the test's own --update; nothing was edited by hand. With USE_HIP unset, cuda_to_hip.h includes the same CUDA headers as before, so the CUDA build of this code is unchanged. Authored with Claude. Test Plan: ``` python test/symforce_caspar_codegen_test.py --update python test/symforce_caspar_codegen_test.py ``` --- .../kernel_backend_test_function.cu | 9 +-------- .../kernel_backend_test_function.h | 3 +-- .../kernel_caspar_padding_lane_score.cu | 9 +-------- .../kernel_caspar_padding_lane_score.h | 3 +-- .../symforce_caspar_solver_test/caspar_mappings.cu | 6 +----- .../symforce_caspar_solver_test/caspar_mappings.h | 2 +- ...ernel_Matrix21_alpha_denominator_or_beta_numerator.cu | 9 +-------- ...kernel_Matrix21_alpha_denominator_or_beta_numerator.h | 3 +-- .../kernel_Matrix21_alpha_numerator_denominator.cu | 9 +-------- .../kernel_Matrix21_alpha_numerator_denominator.h | 3 +-- .../kernel_Matrix21_normalize.cu | 9 +-------- .../kernel_Matrix21_normalize.h | 3 +-- .../kernel_Matrix21_pred_decrease_times_two.cu | 9 +-------- .../kernel_Matrix21_pred_decrease_times_two.h | 3 +-- .../kernel_Matrix21_retract.cu | 9 +-------- .../kernel_Matrix21_retract.h | 3 +-- .../kernel_Matrix21_start_w.cu | 9 +-------- .../kernel_Matrix21_start_w.h | 3 +-- .../kernel_Matrix21_start_w_contribute.cu | 9 +-------- .../kernel_Matrix21_start_w_contribute.h | 3 +-- .../kernel_Matrix21_update_Mp.cu | 9 +-------- .../kernel_Matrix21_update_Mp.h | 3 +-- .../kernel_Matrix21_update_p.cu | 9 +-------- .../kernel_Matrix21_update_p.h | 3 +-- .../kernel_Matrix21_update_r.cu | 9 +-------- .../kernel_Matrix21_update_r.h | 3 +-- .../kernel_Matrix21_update_r_first.cu | 9 +-------- .../kernel_Matrix21_update_r_first.h | 3 +-- .../kernel_Matrix21_update_step.cu | 9 +-------- .../kernel_Matrix21_update_step.h | 3 +-- .../kernel_Matrix21_update_step_first.cu | 9 +-------- .../kernel_Matrix21_update_step_first.h | 3 +-- .../kernel_rosenbrock_jtjnjtr_direct.cu | 9 +-------- .../kernel_rosenbrock_jtjnjtr_direct.h | 3 +-- .../kernel_rosenbrock_res_jac.cu | 9 +-------- .../kernel_rosenbrock_res_jac.h | 3 +-- .../kernel_rosenbrock_res_jac_first.cu | 9 +-------- .../kernel_rosenbrock_res_jac_first.h | 3 +-- .../kernel_rosenbrock_score.cu | 9 +-------- .../kernel_rosenbrock_score.h | 3 +-- .../symengine/symforce_caspar_solver_test/solver.h | 3 +-- .../symforce_caspar_tridiagonal_test/caspar_mappings.cu | 6 +----- .../symforce_caspar_tridiagonal_test/caspar_mappings.h | 2 +- .../kernel_ChainA_alpha_denominator_or_beta_numerator.cu | 9 +-------- .../kernel_ChainA_alpha_denominator_or_beta_numerator.h | 3 +-- .../kernel_ChainA_alpha_numerator_denominator.cu | 9 +-------- .../kernel_ChainA_alpha_numerator_denominator.h | 3 +-- .../kernel_ChainA_cyclic_reduction_down.cu | 9 +-------- .../kernel_ChainA_cyclic_reduction_down.h | 3 +-- .../kernel_ChainA_cyclic_reduction_solve_two.cu | 9 +-------- .../kernel_ChainA_cyclic_reduction_solve_two.h | 3 +-- .../kernel_ChainA_cyclic_reduction_up.cu | 9 +-------- .../kernel_ChainA_cyclic_reduction_up.h | 3 +-- .../kernel_ChainA_normalize.cu | 9 +-------- .../kernel_ChainA_normalize.h | 3 +-- .../kernel_ChainA_pred_decrease_times_two.cu | 9 +-------- .../kernel_ChainA_pred_decrease_times_two.h | 3 +-- .../kernel_ChainA_retract.cu | 9 +-------- .../kernel_ChainA_retract.h | 3 +-- .../kernel_ChainA_start_w.cu | 9 +-------- .../kernel_ChainA_start_w.h | 3 +-- .../kernel_ChainA_start_w_contribute.cu | 9 +-------- .../kernel_ChainA_start_w_contribute.h | 3 +-- .../kernel_ChainA_tridig_make_eq.cu | 9 +-------- .../kernel_ChainA_tridig_make_eq.h | 3 +-- .../kernel_ChainA_update_Mp.cu | 9 +-------- .../kernel_ChainA_update_Mp.h | 3 +-- .../kernel_ChainA_update_p.cu | 9 +-------- .../kernel_ChainA_update_p.h | 3 +-- .../kernel_ChainA_update_r.cu | 9 +-------- .../kernel_ChainA_update_r.h | 3 +-- .../kernel_ChainA_update_r_first.cu | 9 +-------- .../kernel_ChainA_update_r_first.h | 3 +-- .../kernel_ChainA_update_step.cu | 9 +-------- .../kernel_ChainA_update_step.h | 3 +-- .../kernel_ChainA_update_step_first.cu | 9 +-------- .../kernel_ChainA_update_step_first.h | 3 +-- .../kernel_ChainB_alpha_denominator_or_beta_numerator.cu | 9 +-------- .../kernel_ChainB_alpha_denominator_or_beta_numerator.h | 3 +-- .../kernel_ChainB_alpha_numerator_denominator.cu | 9 +-------- .../kernel_ChainB_alpha_numerator_denominator.h | 3 +-- .../kernel_ChainB_cyclic_reduction_down.cu | 9 +-------- .../kernel_ChainB_cyclic_reduction_down.h | 3 +-- .../kernel_ChainB_cyclic_reduction_solve_two.cu | 9 +-------- .../kernel_ChainB_cyclic_reduction_solve_two.h | 3 +-- .../kernel_ChainB_cyclic_reduction_up.cu | 9 +-------- .../kernel_ChainB_cyclic_reduction_up.h | 3 +-- .../kernel_ChainB_normalize.cu | 9 +-------- .../kernel_ChainB_normalize.h | 3 +-- .../kernel_ChainB_pred_decrease_times_two.cu | 9 +-------- .../kernel_ChainB_pred_decrease_times_two.h | 3 +-- .../kernel_ChainB_retract.cu | 9 +-------- .../kernel_ChainB_retract.h | 3 +-- .../kernel_ChainB_start_w.cu | 9 +-------- .../kernel_ChainB_start_w.h | 3 +-- .../kernel_ChainB_start_w_contribute.cu | 9 +-------- .../kernel_ChainB_start_w_contribute.h | 3 +-- .../kernel_ChainB_tridig_make_eq.cu | 9 +-------- .../kernel_ChainB_tridig_make_eq.h | 3 +-- .../kernel_ChainB_update_Mp.cu | 9 +-------- .../kernel_ChainB_update_Mp.h | 3 +-- .../kernel_ChainB_update_p.cu | 9 +-------- .../kernel_ChainB_update_p.h | 3 +-- .../kernel_ChainB_update_r.cu | 9 +-------- .../kernel_ChainB_update_r.h | 3 +-- .../kernel_ChainB_update_r_first.cu | 9 +-------- .../kernel_ChainB_update_r_first.h | 3 +-- .../kernel_ChainB_update_step.cu | 9 +-------- .../kernel_ChainB_update_step.h | 3 +-- .../kernel_ChainB_update_step_first.cu | 9 +-------- .../kernel_ChainB_update_step_first.h | 3 +-- .../kernel_chain_a_link_jtjnjtr_direct.cu | 9 +-------- .../kernel_chain_a_link_jtjnjtr_direct.h | 3 +-- .../kernel_chain_a_link_res_jac.cu | 9 +-------- .../kernel_chain_a_link_res_jac.h | 3 +-- .../kernel_chain_a_link_res_jac_first.cu | 9 +-------- .../kernel_chain_a_link_res_jac_first.h | 3 +-- .../kernel_chain_a_link_score.cu | 9 +-------- .../kernel_chain_a_link_score.h | 3 +-- .../kernel_chain_a_prior_jtjnjtr_direct.cu | 9 +-------- .../kernel_chain_a_prior_jtjnjtr_direct.h | 3 +-- .../kernel_chain_a_prior_res_jac.cu | 9 +-------- .../kernel_chain_a_prior_res_jac.h | 3 +-- .../kernel_chain_a_prior_res_jac_first.cu | 9 +-------- .../kernel_chain_a_prior_res_jac_first.h | 3 +-- .../kernel_chain_a_prior_score.cu | 9 +-------- .../kernel_chain_a_prior_score.h | 3 +-- .../kernel_chain_b_link_jtjnjtr_direct.cu | 9 +-------- .../kernel_chain_b_link_jtjnjtr_direct.h | 3 +-- .../kernel_chain_b_link_res_jac.cu | 9 +-------- .../kernel_chain_b_link_res_jac.h | 3 +-- .../kernel_chain_b_link_res_jac_first.cu | 9 +-------- .../kernel_chain_b_link_res_jac_first.h | 3 +-- .../kernel_chain_b_link_score.cu | 9 +-------- .../kernel_chain_b_link_score.h | 3 +-- .../kernel_chain_b_prior_jtjnjtr_direct.cu | 9 +-------- .../kernel_chain_b_prior_jtjnjtr_direct.h | 3 +-- .../kernel_chain_b_prior_res_jac.cu | 9 +-------- .../kernel_chain_b_prior_res_jac.h | 3 +-- .../kernel_chain_b_prior_res_jac_first.cu | 9 +-------- .../kernel_chain_b_prior_res_jac_first.h | 3 +-- .../kernel_chain_b_prior_score.cu | 9 +-------- .../kernel_chain_b_prior_score.h | 3 +-- .../kernel_chain_coupling_jtjnjtr_direct.cu | 9 +-------- .../kernel_chain_coupling_jtjnjtr_direct.h | 3 +-- .../kernel_chain_coupling_res_jac.cu | 9 +-------- .../kernel_chain_coupling_res_jac.h | 3 +-- .../kernel_chain_coupling_res_jac_first.cu | 9 +-------- .../kernel_chain_coupling_res_jac_first.h | 3 +-- .../kernel_chain_coupling_score.cu | 9 +-------- .../kernel_chain_coupling_score.h | 3 +-- .../symengine/symforce_caspar_tridiagonal_test/solver.h | 3 +-- 152 files changed, 152 insertions(+), 746 deletions(-) diff --git a/test/symforce_function_codegen_test_data/symengine/symforce_caspar_backend_coverage_test/kernel_backend_test_function.cu b/test/symforce_function_codegen_test_data/symengine/symforce_caspar_backend_coverage_test/kernel_backend_test_function.cu index 7827509c..6224331e 100644 --- a/test/symforce_function_codegen_test_data/symengine/symforce_caspar_backend_coverage_test/kernel_backend_test_function.cu +++ b/test/symforce_function_codegen_test_data/symengine/symforce_caspar_backend_coverage_test/kernel_backend_test_function.cu @@ -1,17 +1,10 @@ #include #include -#include -#include -#include -#include -#include - +#include "cuda_to_hip.h" #include "kernel_backend_test_function.h" #include "memops.cuh" -namespace cg = cooperative_groups; - namespace caspar { __global__ void __launch_bounds__(1024, 1) diff --git a/test/symforce_function_codegen_test_data/symengine/symforce_caspar_backend_coverage_test/kernel_backend_test_function.h b/test/symforce_function_codegen_test_data/symengine/symforce_caspar_backend_coverage_test/kernel_backend_test_function.h index 7f37a3b3..82ce6477 100644 --- a/test/symforce_function_codegen_test_data/symengine/symforce_caspar_backend_coverage_test/kernel_backend_test_function.h +++ b/test/symforce_function_codegen_test_data/symengine/symforce_caspar_backend_coverage_test/kernel_backend_test_function.h @@ -1,7 +1,6 @@ #pragma once -#include - +#include "cuda_to_hip.h" #include "shared_indices.h" namespace caspar { diff --git a/test/symforce_function_codegen_test_data/symengine/symforce_caspar_codegen_test/kernel_caspar_padding_lane_score.cu b/test/symforce_function_codegen_test_data/symengine/symforce_caspar_codegen_test/kernel_caspar_padding_lane_score.cu index 82e4d136..4ee6d7d5 100644 --- a/test/symforce_function_codegen_test_data/symengine/symforce_caspar_codegen_test/kernel_caspar_padding_lane_score.cu +++ b/test/symforce_function_codegen_test_data/symengine/symforce_caspar_codegen_test/kernel_caspar_padding_lane_score.cu @@ -1,17 +1,10 @@ #include #include -#include -#include -#include -#include -#include - +#include "cuda_to_hip.h" #include "kernel_caspar_padding_lane_score.h" #include "memops.cuh" -namespace cg = cooperative_groups; - namespace caspar { __global__ void __launch_bounds__(1024, 1) CasparPaddingLaneScoreKernel( diff --git a/test/symforce_function_codegen_test_data/symengine/symforce_caspar_codegen_test/kernel_caspar_padding_lane_score.h b/test/symforce_function_codegen_test_data/symengine/symforce_caspar_codegen_test/kernel_caspar_padding_lane_score.h index a78f744a..1c779f24 100644 --- a/test/symforce_function_codegen_test_data/symengine/symforce_caspar_codegen_test/kernel_caspar_padding_lane_score.h +++ b/test/symforce_function_codegen_test_data/symengine/symforce_caspar_codegen_test/kernel_caspar_padding_lane_score.h @@ -1,7 +1,6 @@ #pragma once -#include - +#include "cuda_to_hip.h" #include "shared_indices.h" namespace caspar { diff --git a/test/symforce_function_codegen_test_data/symengine/symforce_caspar_solver_test/caspar_mappings.cu b/test/symforce_function_codegen_test_data/symengine/symforce_caspar_solver_test/caspar_mappings.cu index 7edd0f2b..9c68da60 100644 --- a/test/symforce_function_codegen_test_data/symengine/symforce_caspar_solver_test/caspar_mappings.cu +++ b/test/symforce_function_codegen_test_data/symengine/symforce_caspar_solver_test/caspar_mappings.cu @@ -1,11 +1,7 @@ #include -#include -#include - #include "caspar_mappings.h" - -namespace cg = cooperative_groups; +#include "cuda_to_hip.h" // We use shared memory to improve the memory access. // A smaller block size of 32 allows for larger nodetypes. diff --git a/test/symforce_function_codegen_test_data/symengine/symforce_caspar_solver_test/caspar_mappings.h b/test/symforce_function_codegen_test_data/symengine/symforce_caspar_solver_test/caspar_mappings.h index 4b4c2719..ebe55ef9 100644 --- a/test/symforce_function_codegen_test_data/symengine/symforce_caspar_solver_test/caspar_mappings.h +++ b/test/symforce_function_codegen_test_data/symengine/symforce_caspar_solver_test/caspar_mappings.h @@ -1,6 +1,6 @@ #pragma once -#include +#include "cuda_to_hip.h" namespace caspar { diff --git a/test/symforce_function_codegen_test_data/symengine/symforce_caspar_solver_test/kernel_Matrix21_alpha_denominator_or_beta_numerator.cu b/test/symforce_function_codegen_test_data/symengine/symforce_caspar_solver_test/kernel_Matrix21_alpha_denominator_or_beta_numerator.cu index 1331046a..95c8344a 100644 --- a/test/symforce_function_codegen_test_data/symengine/symforce_caspar_solver_test/kernel_Matrix21_alpha_denominator_or_beta_numerator.cu +++ b/test/symforce_function_codegen_test_data/symengine/symforce_caspar_solver_test/kernel_Matrix21_alpha_denominator_or_beta_numerator.cu @@ -1,17 +1,10 @@ #include #include -#include -#include -#include -#include -#include - +#include "cuda_to_hip.h" #include "kernel_Matrix21_alpha_denominator_or_beta_numerator.h" #include "memops.cuh" -namespace cg = cooperative_groups; - namespace caspar { __global__ void __launch_bounds__(1024, 1) Matrix21AlphaDenominatorOrBetaNumeratorKernel( diff --git a/test/symforce_function_codegen_test_data/symengine/symforce_caspar_solver_test/kernel_Matrix21_alpha_denominator_or_beta_numerator.h b/test/symforce_function_codegen_test_data/symengine/symforce_caspar_solver_test/kernel_Matrix21_alpha_denominator_or_beta_numerator.h index fe955dba..7a3bbf55 100644 --- a/test/symforce_function_codegen_test_data/symengine/symforce_caspar_solver_test/kernel_Matrix21_alpha_denominator_or_beta_numerator.h +++ b/test/symforce_function_codegen_test_data/symengine/symforce_caspar_solver_test/kernel_Matrix21_alpha_denominator_or_beta_numerator.h @@ -1,7 +1,6 @@ #pragma once -#include - +#include "cuda_to_hip.h" #include "shared_indices.h" namespace caspar { diff --git a/test/symforce_function_codegen_test_data/symengine/symforce_caspar_solver_test/kernel_Matrix21_alpha_numerator_denominator.cu b/test/symforce_function_codegen_test_data/symengine/symforce_caspar_solver_test/kernel_Matrix21_alpha_numerator_denominator.cu index f90df48a..ea6c5e3c 100644 --- a/test/symforce_function_codegen_test_data/symengine/symforce_caspar_solver_test/kernel_Matrix21_alpha_numerator_denominator.cu +++ b/test/symforce_function_codegen_test_data/symengine/symforce_caspar_solver_test/kernel_Matrix21_alpha_numerator_denominator.cu @@ -1,17 +1,10 @@ #include #include -#include -#include -#include -#include -#include - +#include "cuda_to_hip.h" #include "kernel_Matrix21_alpha_numerator_denominator.h" #include "memops.cuh" -namespace cg = cooperative_groups; - namespace caspar { __global__ void __launch_bounds__(1024, 1) Matrix21AlphaNumeratorDenominatorKernel( diff --git a/test/symforce_function_codegen_test_data/symengine/symforce_caspar_solver_test/kernel_Matrix21_alpha_numerator_denominator.h b/test/symforce_function_codegen_test_data/symengine/symforce_caspar_solver_test/kernel_Matrix21_alpha_numerator_denominator.h index 863a162c..7e9a4fb2 100644 --- a/test/symforce_function_codegen_test_data/symengine/symforce_caspar_solver_test/kernel_Matrix21_alpha_numerator_denominator.h +++ b/test/symforce_function_codegen_test_data/symengine/symforce_caspar_solver_test/kernel_Matrix21_alpha_numerator_denominator.h @@ -1,7 +1,6 @@ #pragma once -#include - +#include "cuda_to_hip.h" #include "shared_indices.h" namespace caspar { diff --git a/test/symforce_function_codegen_test_data/symengine/symforce_caspar_solver_test/kernel_Matrix21_normalize.cu b/test/symforce_function_codegen_test_data/symengine/symforce_caspar_solver_test/kernel_Matrix21_normalize.cu index 65302139..7ff9c296 100644 --- a/test/symforce_function_codegen_test_data/symengine/symforce_caspar_solver_test/kernel_Matrix21_normalize.cu +++ b/test/symforce_function_codegen_test_data/symengine/symforce_caspar_solver_test/kernel_Matrix21_normalize.cu @@ -1,17 +1,10 @@ #include #include -#include -#include -#include -#include -#include - +#include "cuda_to_hip.h" #include "kernel_Matrix21_normalize.h" #include "memops.cuh" -namespace cg = cooperative_groups; - namespace caspar { __global__ void __launch_bounds__(1024, 1) diff --git a/test/symforce_function_codegen_test_data/symengine/symforce_caspar_solver_test/kernel_Matrix21_normalize.h b/test/symforce_function_codegen_test_data/symengine/symforce_caspar_solver_test/kernel_Matrix21_normalize.h index c6db2e2f..7b39f71c 100644 --- a/test/symforce_function_codegen_test_data/symengine/symforce_caspar_solver_test/kernel_Matrix21_normalize.h +++ b/test/symforce_function_codegen_test_data/symengine/symforce_caspar_solver_test/kernel_Matrix21_normalize.h @@ -1,7 +1,6 @@ #pragma once -#include - +#include "cuda_to_hip.h" #include "shared_indices.h" namespace caspar { diff --git a/test/symforce_function_codegen_test_data/symengine/symforce_caspar_solver_test/kernel_Matrix21_pred_decrease_times_two.cu b/test/symforce_function_codegen_test_data/symengine/symforce_caspar_solver_test/kernel_Matrix21_pred_decrease_times_two.cu index 44488925..fc96efd7 100644 --- a/test/symforce_function_codegen_test_data/symengine/symforce_caspar_solver_test/kernel_Matrix21_pred_decrease_times_two.cu +++ b/test/symforce_function_codegen_test_data/symengine/symforce_caspar_solver_test/kernel_Matrix21_pred_decrease_times_two.cu @@ -1,17 +1,10 @@ #include #include -#include -#include -#include -#include -#include - +#include "cuda_to_hip.h" #include "kernel_Matrix21_pred_decrease_times_two.h" #include "memops.cuh" -namespace cg = cooperative_groups; - namespace caspar { __global__ void __launch_bounds__(1024, 1) diff --git a/test/symforce_function_codegen_test_data/symengine/symforce_caspar_solver_test/kernel_Matrix21_pred_decrease_times_two.h b/test/symforce_function_codegen_test_data/symengine/symforce_caspar_solver_test/kernel_Matrix21_pred_decrease_times_two.h index 4e8ad4e1..1ee881bc 100644 --- a/test/symforce_function_codegen_test_data/symengine/symforce_caspar_solver_test/kernel_Matrix21_pred_decrease_times_two.h +++ b/test/symforce_function_codegen_test_data/symengine/symforce_caspar_solver_test/kernel_Matrix21_pred_decrease_times_two.h @@ -1,7 +1,6 @@ #pragma once -#include - +#include "cuda_to_hip.h" #include "shared_indices.h" namespace caspar { diff --git a/test/symforce_function_codegen_test_data/symengine/symforce_caspar_solver_test/kernel_Matrix21_retract.cu b/test/symforce_function_codegen_test_data/symengine/symforce_caspar_solver_test/kernel_Matrix21_retract.cu index 14e7d572..318990d6 100644 --- a/test/symforce_function_codegen_test_data/symengine/symforce_caspar_solver_test/kernel_Matrix21_retract.cu +++ b/test/symforce_function_codegen_test_data/symengine/symforce_caspar_solver_test/kernel_Matrix21_retract.cu @@ -1,17 +1,10 @@ #include #include -#include -#include -#include -#include -#include - +#include "cuda_to_hip.h" #include "kernel_Matrix21_retract.h" #include "memops.cuh" -namespace cg = cooperative_groups; - namespace caspar { __global__ void __launch_bounds__(1024, 1) diff --git a/test/symforce_function_codegen_test_data/symengine/symforce_caspar_solver_test/kernel_Matrix21_retract.h b/test/symforce_function_codegen_test_data/symengine/symforce_caspar_solver_test/kernel_Matrix21_retract.h index 1219c59d..5c0eb51a 100644 --- a/test/symforce_function_codegen_test_data/symengine/symforce_caspar_solver_test/kernel_Matrix21_retract.h +++ b/test/symforce_function_codegen_test_data/symengine/symforce_caspar_solver_test/kernel_Matrix21_retract.h @@ -1,7 +1,6 @@ #pragma once -#include - +#include "cuda_to_hip.h" #include "shared_indices.h" namespace caspar { diff --git a/test/symforce_function_codegen_test_data/symengine/symforce_caspar_solver_test/kernel_Matrix21_start_w.cu b/test/symforce_function_codegen_test_data/symengine/symforce_caspar_solver_test/kernel_Matrix21_start_w.cu index 1a370cb0..f0cfdd0a 100644 --- a/test/symforce_function_codegen_test_data/symengine/symforce_caspar_solver_test/kernel_Matrix21_start_w.cu +++ b/test/symforce_function_codegen_test_data/symengine/symforce_caspar_solver_test/kernel_Matrix21_start_w.cu @@ -1,17 +1,10 @@ #include #include -#include -#include -#include -#include -#include - +#include "cuda_to_hip.h" #include "kernel_Matrix21_start_w.h" #include "memops.cuh" -namespace cg = cooperative_groups; - namespace caspar { __global__ void __launch_bounds__(1024, 1) diff --git a/test/symforce_function_codegen_test_data/symengine/symforce_caspar_solver_test/kernel_Matrix21_start_w.h b/test/symforce_function_codegen_test_data/symengine/symforce_caspar_solver_test/kernel_Matrix21_start_w.h index fa1e7963..f34294fc 100644 --- a/test/symforce_function_codegen_test_data/symengine/symforce_caspar_solver_test/kernel_Matrix21_start_w.h +++ b/test/symforce_function_codegen_test_data/symengine/symforce_caspar_solver_test/kernel_Matrix21_start_w.h @@ -1,7 +1,6 @@ #pragma once -#include - +#include "cuda_to_hip.h" #include "shared_indices.h" namespace caspar { diff --git a/test/symforce_function_codegen_test_data/symengine/symforce_caspar_solver_test/kernel_Matrix21_start_w_contribute.cu b/test/symforce_function_codegen_test_data/symengine/symforce_caspar_solver_test/kernel_Matrix21_start_w_contribute.cu index 95c95c4c..ce0d8a0d 100644 --- a/test/symforce_function_codegen_test_data/symengine/symforce_caspar_solver_test/kernel_Matrix21_start_w_contribute.cu +++ b/test/symforce_function_codegen_test_data/symengine/symforce_caspar_solver_test/kernel_Matrix21_start_w_contribute.cu @@ -1,17 +1,10 @@ #include #include -#include -#include -#include -#include -#include - +#include "cuda_to_hip.h" #include "kernel_Matrix21_start_w_contribute.h" #include "memops.cuh" -namespace cg = cooperative_groups; - namespace caspar { __global__ void __launch_bounds__(1024, 1) diff --git a/test/symforce_function_codegen_test_data/symengine/symforce_caspar_solver_test/kernel_Matrix21_start_w_contribute.h b/test/symforce_function_codegen_test_data/symengine/symforce_caspar_solver_test/kernel_Matrix21_start_w_contribute.h index 67a41b95..f5de02be 100644 --- a/test/symforce_function_codegen_test_data/symengine/symforce_caspar_solver_test/kernel_Matrix21_start_w_contribute.h +++ b/test/symforce_function_codegen_test_data/symengine/symforce_caspar_solver_test/kernel_Matrix21_start_w_contribute.h @@ -1,7 +1,6 @@ #pragma once -#include - +#include "cuda_to_hip.h" #include "shared_indices.h" namespace caspar { diff --git a/test/symforce_function_codegen_test_data/symengine/symforce_caspar_solver_test/kernel_Matrix21_update_Mp.cu b/test/symforce_function_codegen_test_data/symengine/symforce_caspar_solver_test/kernel_Matrix21_update_Mp.cu index 49611002..dcb0c2ca 100644 --- a/test/symforce_function_codegen_test_data/symengine/symforce_caspar_solver_test/kernel_Matrix21_update_Mp.cu +++ b/test/symforce_function_codegen_test_data/symengine/symforce_caspar_solver_test/kernel_Matrix21_update_Mp.cu @@ -1,17 +1,10 @@ #include #include -#include -#include -#include -#include -#include - +#include "cuda_to_hip.h" #include "kernel_Matrix21_update_Mp.h" #include "memops.cuh" -namespace cg = cooperative_groups; - namespace caspar { __global__ void __launch_bounds__(1024, 1) diff --git a/test/symforce_function_codegen_test_data/symengine/symforce_caspar_solver_test/kernel_Matrix21_update_Mp.h b/test/symforce_function_codegen_test_data/symengine/symforce_caspar_solver_test/kernel_Matrix21_update_Mp.h index 4e3352d5..28a45178 100644 --- a/test/symforce_function_codegen_test_data/symengine/symforce_caspar_solver_test/kernel_Matrix21_update_Mp.h +++ b/test/symforce_function_codegen_test_data/symengine/symforce_caspar_solver_test/kernel_Matrix21_update_Mp.h @@ -1,7 +1,6 @@ #pragma once -#include - +#include "cuda_to_hip.h" #include "shared_indices.h" namespace caspar { diff --git a/test/symforce_function_codegen_test_data/symengine/symforce_caspar_solver_test/kernel_Matrix21_update_p.cu b/test/symforce_function_codegen_test_data/symengine/symforce_caspar_solver_test/kernel_Matrix21_update_p.cu index eee35a76..4b7ca93b 100644 --- a/test/symforce_function_codegen_test_data/symengine/symforce_caspar_solver_test/kernel_Matrix21_update_p.cu +++ b/test/symforce_function_codegen_test_data/symengine/symforce_caspar_solver_test/kernel_Matrix21_update_p.cu @@ -1,17 +1,10 @@ #include #include -#include -#include -#include -#include -#include - +#include "cuda_to_hip.h" #include "kernel_Matrix21_update_p.h" #include "memops.cuh" -namespace cg = cooperative_groups; - namespace caspar { __global__ void __launch_bounds__(1024, 1) diff --git a/test/symforce_function_codegen_test_data/symengine/symforce_caspar_solver_test/kernel_Matrix21_update_p.h b/test/symforce_function_codegen_test_data/symengine/symforce_caspar_solver_test/kernel_Matrix21_update_p.h index a6ea4824..847f2b40 100644 --- a/test/symforce_function_codegen_test_data/symengine/symforce_caspar_solver_test/kernel_Matrix21_update_p.h +++ b/test/symforce_function_codegen_test_data/symengine/symforce_caspar_solver_test/kernel_Matrix21_update_p.h @@ -1,7 +1,6 @@ #pragma once -#include - +#include "cuda_to_hip.h" #include "shared_indices.h" namespace caspar { diff --git a/test/symforce_function_codegen_test_data/symengine/symforce_caspar_solver_test/kernel_Matrix21_update_r.cu b/test/symforce_function_codegen_test_data/symengine/symforce_caspar_solver_test/kernel_Matrix21_update_r.cu index 2dca6a4e..83b4dd6c 100644 --- a/test/symforce_function_codegen_test_data/symengine/symforce_caspar_solver_test/kernel_Matrix21_update_r.cu +++ b/test/symforce_function_codegen_test_data/symengine/symforce_caspar_solver_test/kernel_Matrix21_update_r.cu @@ -1,17 +1,10 @@ #include #include -#include -#include -#include -#include -#include - +#include "cuda_to_hip.h" #include "kernel_Matrix21_update_r.h" #include "memops.cuh" -namespace cg = cooperative_groups; - namespace caspar { __global__ void __launch_bounds__(1024, 1) diff --git a/test/symforce_function_codegen_test_data/symengine/symforce_caspar_solver_test/kernel_Matrix21_update_r.h b/test/symforce_function_codegen_test_data/symengine/symforce_caspar_solver_test/kernel_Matrix21_update_r.h index 33ce7fea..2ae46944 100644 --- a/test/symforce_function_codegen_test_data/symengine/symforce_caspar_solver_test/kernel_Matrix21_update_r.h +++ b/test/symforce_function_codegen_test_data/symengine/symforce_caspar_solver_test/kernel_Matrix21_update_r.h @@ -1,7 +1,6 @@ #pragma once -#include - +#include "cuda_to_hip.h" #include "shared_indices.h" namespace caspar { diff --git a/test/symforce_function_codegen_test_data/symengine/symforce_caspar_solver_test/kernel_Matrix21_update_r_first.cu b/test/symforce_function_codegen_test_data/symengine/symforce_caspar_solver_test/kernel_Matrix21_update_r_first.cu index 9519aa50..2908c211 100644 --- a/test/symforce_function_codegen_test_data/symengine/symforce_caspar_solver_test/kernel_Matrix21_update_r_first.cu +++ b/test/symforce_function_codegen_test_data/symengine/symforce_caspar_solver_test/kernel_Matrix21_update_r_first.cu @@ -1,17 +1,10 @@ #include #include -#include -#include -#include -#include -#include - +#include "cuda_to_hip.h" #include "kernel_Matrix21_update_r_first.h" #include "memops.cuh" -namespace cg = cooperative_groups; - namespace caspar { __global__ void __launch_bounds__(1024, 1) diff --git a/test/symforce_function_codegen_test_data/symengine/symforce_caspar_solver_test/kernel_Matrix21_update_r_first.h b/test/symforce_function_codegen_test_data/symengine/symforce_caspar_solver_test/kernel_Matrix21_update_r_first.h index 24f801d0..1e869640 100644 --- a/test/symforce_function_codegen_test_data/symengine/symforce_caspar_solver_test/kernel_Matrix21_update_r_first.h +++ b/test/symforce_function_codegen_test_data/symengine/symforce_caspar_solver_test/kernel_Matrix21_update_r_first.h @@ -1,7 +1,6 @@ #pragma once -#include - +#include "cuda_to_hip.h" #include "shared_indices.h" namespace caspar { diff --git a/test/symforce_function_codegen_test_data/symengine/symforce_caspar_solver_test/kernel_Matrix21_update_step.cu b/test/symforce_function_codegen_test_data/symengine/symforce_caspar_solver_test/kernel_Matrix21_update_step.cu index fdcbc605..b0387b24 100644 --- a/test/symforce_function_codegen_test_data/symengine/symforce_caspar_solver_test/kernel_Matrix21_update_step.cu +++ b/test/symforce_function_codegen_test_data/symengine/symforce_caspar_solver_test/kernel_Matrix21_update_step.cu @@ -1,17 +1,10 @@ #include #include -#include -#include -#include -#include -#include - +#include "cuda_to_hip.h" #include "kernel_Matrix21_update_step.h" #include "memops.cuh" -namespace cg = cooperative_groups; - namespace caspar { __global__ void __launch_bounds__(1024, 1) diff --git a/test/symforce_function_codegen_test_data/symengine/symforce_caspar_solver_test/kernel_Matrix21_update_step.h b/test/symforce_function_codegen_test_data/symengine/symforce_caspar_solver_test/kernel_Matrix21_update_step.h index e1709af9..b0a2ceb5 100644 --- a/test/symforce_function_codegen_test_data/symengine/symforce_caspar_solver_test/kernel_Matrix21_update_step.h +++ b/test/symforce_function_codegen_test_data/symengine/symforce_caspar_solver_test/kernel_Matrix21_update_step.h @@ -1,7 +1,6 @@ #pragma once -#include - +#include "cuda_to_hip.h" #include "shared_indices.h" namespace caspar { diff --git a/test/symforce_function_codegen_test_data/symengine/symforce_caspar_solver_test/kernel_Matrix21_update_step_first.cu b/test/symforce_function_codegen_test_data/symengine/symforce_caspar_solver_test/kernel_Matrix21_update_step_first.cu index 7bf44110..498da5b0 100644 --- a/test/symforce_function_codegen_test_data/symengine/symforce_caspar_solver_test/kernel_Matrix21_update_step_first.cu +++ b/test/symforce_function_codegen_test_data/symengine/symforce_caspar_solver_test/kernel_Matrix21_update_step_first.cu @@ -1,17 +1,10 @@ #include #include -#include -#include -#include -#include -#include - +#include "cuda_to_hip.h" #include "kernel_Matrix21_update_step_first.h" #include "memops.cuh" -namespace cg = cooperative_groups; - namespace caspar { __global__ void __launch_bounds__(1024, 1) diff --git a/test/symforce_function_codegen_test_data/symengine/symforce_caspar_solver_test/kernel_Matrix21_update_step_first.h b/test/symforce_function_codegen_test_data/symengine/symforce_caspar_solver_test/kernel_Matrix21_update_step_first.h index bbf51fa9..4f1ae077 100644 --- a/test/symforce_function_codegen_test_data/symengine/symforce_caspar_solver_test/kernel_Matrix21_update_step_first.h +++ b/test/symforce_function_codegen_test_data/symengine/symforce_caspar_solver_test/kernel_Matrix21_update_step_first.h @@ -1,7 +1,6 @@ #pragma once -#include - +#include "cuda_to_hip.h" #include "shared_indices.h" namespace caspar { diff --git a/test/symforce_function_codegen_test_data/symengine/symforce_caspar_solver_test/kernel_rosenbrock_jtjnjtr_direct.cu b/test/symforce_function_codegen_test_data/symengine/symforce_caspar_solver_test/kernel_rosenbrock_jtjnjtr_direct.cu index 956e7b88..df820641 100644 --- a/test/symforce_function_codegen_test_data/symengine/symforce_caspar_solver_test/kernel_rosenbrock_jtjnjtr_direct.cu +++ b/test/symforce_function_codegen_test_data/symengine/symforce_caspar_solver_test/kernel_rosenbrock_jtjnjtr_direct.cu @@ -1,17 +1,10 @@ #include #include -#include -#include -#include -#include -#include - +#include "cuda_to_hip.h" #include "kernel_rosenbrock_jtjnjtr_direct.h" #include "memops.cuh" -namespace cg = cooperative_groups; - namespace caspar { __global__ void __launch_bounds__(1024, 1) diff --git a/test/symforce_function_codegen_test_data/symengine/symforce_caspar_solver_test/kernel_rosenbrock_jtjnjtr_direct.h b/test/symforce_function_codegen_test_data/symengine/symforce_caspar_solver_test/kernel_rosenbrock_jtjnjtr_direct.h index f0556d5f..1af63135 100644 --- a/test/symforce_function_codegen_test_data/symengine/symforce_caspar_solver_test/kernel_rosenbrock_jtjnjtr_direct.h +++ b/test/symforce_function_codegen_test_data/symengine/symforce_caspar_solver_test/kernel_rosenbrock_jtjnjtr_direct.h @@ -1,7 +1,6 @@ #pragma once -#include - +#include "cuda_to_hip.h" #include "shared_indices.h" namespace caspar { diff --git a/test/symforce_function_codegen_test_data/symengine/symforce_caspar_solver_test/kernel_rosenbrock_res_jac.cu b/test/symforce_function_codegen_test_data/symengine/symforce_caspar_solver_test/kernel_rosenbrock_res_jac.cu index 8c17c53f..e237c533 100644 --- a/test/symforce_function_codegen_test_data/symengine/symforce_caspar_solver_test/kernel_rosenbrock_res_jac.cu +++ b/test/symforce_function_codegen_test_data/symengine/symforce_caspar_solver_test/kernel_rosenbrock_res_jac.cu @@ -1,17 +1,10 @@ #include #include -#include -#include -#include -#include -#include - +#include "cuda_to_hip.h" #include "kernel_rosenbrock_res_jac.h" #include "memops.cuh" -namespace cg = cooperative_groups; - namespace caspar { __global__ void __launch_bounds__(1024, 1) diff --git a/test/symforce_function_codegen_test_data/symengine/symforce_caspar_solver_test/kernel_rosenbrock_res_jac.h b/test/symforce_function_codegen_test_data/symengine/symforce_caspar_solver_test/kernel_rosenbrock_res_jac.h index 84617734..ad7a2e6c 100644 --- a/test/symforce_function_codegen_test_data/symengine/symforce_caspar_solver_test/kernel_rosenbrock_res_jac.h +++ b/test/symforce_function_codegen_test_data/symengine/symforce_caspar_solver_test/kernel_rosenbrock_res_jac.h @@ -1,7 +1,6 @@ #pragma once -#include - +#include "cuda_to_hip.h" #include "shared_indices.h" namespace caspar { diff --git a/test/symforce_function_codegen_test_data/symengine/symforce_caspar_solver_test/kernel_rosenbrock_res_jac_first.cu b/test/symforce_function_codegen_test_data/symengine/symforce_caspar_solver_test/kernel_rosenbrock_res_jac_first.cu index 94cdbac1..3c67e39d 100644 --- a/test/symforce_function_codegen_test_data/symengine/symforce_caspar_solver_test/kernel_rosenbrock_res_jac_first.cu +++ b/test/symforce_function_codegen_test_data/symengine/symforce_caspar_solver_test/kernel_rosenbrock_res_jac_first.cu @@ -1,17 +1,10 @@ #include #include -#include -#include -#include -#include -#include - +#include "cuda_to_hip.h" #include "kernel_rosenbrock_res_jac_first.h" #include "memops.cuh" -namespace cg = cooperative_groups; - namespace caspar { __global__ void __launch_bounds__(1024, 1) diff --git a/test/symforce_function_codegen_test_data/symengine/symforce_caspar_solver_test/kernel_rosenbrock_res_jac_first.h b/test/symforce_function_codegen_test_data/symengine/symforce_caspar_solver_test/kernel_rosenbrock_res_jac_first.h index c2186b6e..44c70fe1 100644 --- a/test/symforce_function_codegen_test_data/symengine/symforce_caspar_solver_test/kernel_rosenbrock_res_jac_first.h +++ b/test/symforce_function_codegen_test_data/symengine/symforce_caspar_solver_test/kernel_rosenbrock_res_jac_first.h @@ -1,7 +1,6 @@ #pragma once -#include - +#include "cuda_to_hip.h" #include "shared_indices.h" namespace caspar { diff --git a/test/symforce_function_codegen_test_data/symengine/symforce_caspar_solver_test/kernel_rosenbrock_score.cu b/test/symforce_function_codegen_test_data/symengine/symforce_caspar_solver_test/kernel_rosenbrock_score.cu index 0932d18e..dd4afec3 100644 --- a/test/symforce_function_codegen_test_data/symengine/symforce_caspar_solver_test/kernel_rosenbrock_score.cu +++ b/test/symforce_function_codegen_test_data/symengine/symforce_caspar_solver_test/kernel_rosenbrock_score.cu @@ -1,17 +1,10 @@ #include #include -#include -#include -#include -#include -#include - +#include "cuda_to_hip.h" #include "kernel_rosenbrock_score.h" #include "memops.cuh" -namespace cg = cooperative_groups; - namespace caspar { __global__ void __launch_bounds__(1024, 1) diff --git a/test/symforce_function_codegen_test_data/symengine/symforce_caspar_solver_test/kernel_rosenbrock_score.h b/test/symforce_function_codegen_test_data/symengine/symforce_caspar_solver_test/kernel_rosenbrock_score.h index aee00b4e..43f4b867 100644 --- a/test/symforce_function_codegen_test_data/symengine/symforce_caspar_solver_test/kernel_rosenbrock_score.h +++ b/test/symforce_function_codegen_test_data/symengine/symforce_caspar_solver_test/kernel_rosenbrock_score.h @@ -1,7 +1,6 @@ #pragma once -#include - +#include "cuda_to_hip.h" #include "shared_indices.h" namespace caspar { diff --git a/test/symforce_function_codegen_test_data/symengine/symforce_caspar_solver_test/solver.h b/test/symforce_function_codegen_test_data/symengine/symforce_caspar_solver_test/solver.h index 247055a1..6b3f0b1d 100644 --- a/test/symforce_function_codegen_test_data/symengine/symforce_caspar_solver_test/solver.h +++ b/test/symforce_function_codegen_test_data/symengine/symforce_caspar_solver_test/solver.h @@ -3,8 +3,7 @@ #include #include -#include - +#include "cuda_to_hip.h" #include "shared_indices.h" #include "solver_params.h" diff --git a/test/symforce_function_codegen_test_data/symengine/symforce_caspar_tridiagonal_test/caspar_mappings.cu b/test/symforce_function_codegen_test_data/symengine/symforce_caspar_tridiagonal_test/caspar_mappings.cu index 0b2a5761..3a525387 100644 --- a/test/symforce_function_codegen_test_data/symengine/symforce_caspar_tridiagonal_test/caspar_mappings.cu +++ b/test/symforce_function_codegen_test_data/symengine/symforce_caspar_tridiagonal_test/caspar_mappings.cu @@ -1,11 +1,7 @@ #include -#include -#include - #include "caspar_mappings.h" - -namespace cg = cooperative_groups; +#include "cuda_to_hip.h" // We use shared memory to improve the memory access. // A smaller block size of 32 allows for larger nodetypes. diff --git a/test/symforce_function_codegen_test_data/symengine/symforce_caspar_tridiagonal_test/caspar_mappings.h b/test/symforce_function_codegen_test_data/symengine/symforce_caspar_tridiagonal_test/caspar_mappings.h index 6d656e85..08b042c4 100644 --- a/test/symforce_function_codegen_test_data/symengine/symforce_caspar_tridiagonal_test/caspar_mappings.h +++ b/test/symforce_function_codegen_test_data/symengine/symforce_caspar_tridiagonal_test/caspar_mappings.h @@ -1,6 +1,6 @@ #pragma once -#include +#include "cuda_to_hip.h" namespace caspar { diff --git a/test/symforce_function_codegen_test_data/symengine/symforce_caspar_tridiagonal_test/kernel_ChainA_alpha_denominator_or_beta_numerator.cu b/test/symforce_function_codegen_test_data/symengine/symforce_caspar_tridiagonal_test/kernel_ChainA_alpha_denominator_or_beta_numerator.cu index 495c7be2..c33a2eab 100644 --- a/test/symforce_function_codegen_test_data/symengine/symforce_caspar_tridiagonal_test/kernel_ChainA_alpha_denominator_or_beta_numerator.cu +++ b/test/symforce_function_codegen_test_data/symengine/symforce_caspar_tridiagonal_test/kernel_ChainA_alpha_denominator_or_beta_numerator.cu @@ -1,17 +1,10 @@ #include #include -#include -#include -#include -#include -#include - +#include "cuda_to_hip.h" #include "kernel_ChainA_alpha_denominator_or_beta_numerator.h" #include "memops.cuh" -namespace cg = cooperative_groups; - namespace caspar { __global__ void __launch_bounds__(1024, 1) diff --git a/test/symforce_function_codegen_test_data/symengine/symforce_caspar_tridiagonal_test/kernel_ChainA_alpha_denominator_or_beta_numerator.h b/test/symforce_function_codegen_test_data/symengine/symforce_caspar_tridiagonal_test/kernel_ChainA_alpha_denominator_or_beta_numerator.h index b02b452a..4247df5d 100644 --- a/test/symforce_function_codegen_test_data/symengine/symforce_caspar_tridiagonal_test/kernel_ChainA_alpha_denominator_or_beta_numerator.h +++ b/test/symforce_function_codegen_test_data/symengine/symforce_caspar_tridiagonal_test/kernel_ChainA_alpha_denominator_or_beta_numerator.h @@ -1,7 +1,6 @@ #pragma once -#include - +#include "cuda_to_hip.h" #include "shared_indices.h" namespace caspar { diff --git a/test/symforce_function_codegen_test_data/symengine/symforce_caspar_tridiagonal_test/kernel_ChainA_alpha_numerator_denominator.cu b/test/symforce_function_codegen_test_data/symengine/symforce_caspar_tridiagonal_test/kernel_ChainA_alpha_numerator_denominator.cu index 20c3096e..83f2f2dd 100644 --- a/test/symforce_function_codegen_test_data/symengine/symforce_caspar_tridiagonal_test/kernel_ChainA_alpha_numerator_denominator.cu +++ b/test/symforce_function_codegen_test_data/symengine/symforce_caspar_tridiagonal_test/kernel_ChainA_alpha_numerator_denominator.cu @@ -1,17 +1,10 @@ #include #include -#include -#include -#include -#include -#include - +#include "cuda_to_hip.h" #include "kernel_ChainA_alpha_numerator_denominator.h" #include "memops.cuh" -namespace cg = cooperative_groups; - namespace caspar { __global__ void __launch_bounds__(1024, 1) diff --git a/test/symforce_function_codegen_test_data/symengine/symforce_caspar_tridiagonal_test/kernel_ChainA_alpha_numerator_denominator.h b/test/symforce_function_codegen_test_data/symengine/symforce_caspar_tridiagonal_test/kernel_ChainA_alpha_numerator_denominator.h index 94a7d660..b6a13e5b 100644 --- a/test/symforce_function_codegen_test_data/symengine/symforce_caspar_tridiagonal_test/kernel_ChainA_alpha_numerator_denominator.h +++ b/test/symforce_function_codegen_test_data/symengine/symforce_caspar_tridiagonal_test/kernel_ChainA_alpha_numerator_denominator.h @@ -1,7 +1,6 @@ #pragma once -#include - +#include "cuda_to_hip.h" #include "shared_indices.h" namespace caspar { diff --git a/test/symforce_function_codegen_test_data/symengine/symforce_caspar_tridiagonal_test/kernel_ChainA_cyclic_reduction_down.cu b/test/symforce_function_codegen_test_data/symengine/symforce_caspar_tridiagonal_test/kernel_ChainA_cyclic_reduction_down.cu index 5a945d07..a2efd490 100644 --- a/test/symforce_function_codegen_test_data/symengine/symforce_caspar_tridiagonal_test/kernel_ChainA_cyclic_reduction_down.cu +++ b/test/symforce_function_codegen_test_data/symengine/symforce_caspar_tridiagonal_test/kernel_ChainA_cyclic_reduction_down.cu @@ -1,17 +1,10 @@ #include #include -#include -#include -#include -#include -#include - +#include "cuda_to_hip.h" #include "kernel_ChainA_cyclic_reduction_down.h" #include "memops.cuh" -namespace cg = cooperative_groups; - namespace caspar { __global__ void __launch_bounds__(64, 1) diff --git a/test/symforce_function_codegen_test_data/symengine/symforce_caspar_tridiagonal_test/kernel_ChainA_cyclic_reduction_down.h b/test/symforce_function_codegen_test_data/symengine/symforce_caspar_tridiagonal_test/kernel_ChainA_cyclic_reduction_down.h index ce406fc6..2cb52fd8 100644 --- a/test/symforce_function_codegen_test_data/symengine/symforce_caspar_tridiagonal_test/kernel_ChainA_cyclic_reduction_down.h +++ b/test/symforce_function_codegen_test_data/symengine/symforce_caspar_tridiagonal_test/kernel_ChainA_cyclic_reduction_down.h @@ -1,7 +1,6 @@ #pragma once -#include - +#include "cuda_to_hip.h" #include "shared_indices.h" namespace caspar { diff --git a/test/symforce_function_codegen_test_data/symengine/symforce_caspar_tridiagonal_test/kernel_ChainA_cyclic_reduction_solve_two.cu b/test/symforce_function_codegen_test_data/symengine/symforce_caspar_tridiagonal_test/kernel_ChainA_cyclic_reduction_solve_two.cu index b3cca154..2bcd3719 100644 --- a/test/symforce_function_codegen_test_data/symengine/symforce_caspar_tridiagonal_test/kernel_ChainA_cyclic_reduction_solve_two.cu +++ b/test/symforce_function_codegen_test_data/symengine/symforce_caspar_tridiagonal_test/kernel_ChainA_cyclic_reduction_solve_two.cu @@ -1,17 +1,10 @@ #include #include -#include -#include -#include -#include -#include - +#include "cuda_to_hip.h" #include "kernel_ChainA_cyclic_reduction_solve_two.h" #include "memops.cuh" -namespace cg = cooperative_groups; - namespace caspar { __global__ void __launch_bounds__(32, 1) ChainACyclicReductionSolveTwoKernel( diff --git a/test/symforce_function_codegen_test_data/symengine/symforce_caspar_tridiagonal_test/kernel_ChainA_cyclic_reduction_solve_two.h b/test/symforce_function_codegen_test_data/symengine/symforce_caspar_tridiagonal_test/kernel_ChainA_cyclic_reduction_solve_two.h index 06a5d1c6..7ec47c94 100644 --- a/test/symforce_function_codegen_test_data/symengine/symforce_caspar_tridiagonal_test/kernel_ChainA_cyclic_reduction_solve_two.h +++ b/test/symforce_function_codegen_test_data/symengine/symforce_caspar_tridiagonal_test/kernel_ChainA_cyclic_reduction_solve_two.h @@ -1,7 +1,6 @@ #pragma once -#include - +#include "cuda_to_hip.h" #include "shared_indices.h" namespace caspar { diff --git a/test/symforce_function_codegen_test_data/symengine/symforce_caspar_tridiagonal_test/kernel_ChainA_cyclic_reduction_up.cu b/test/symforce_function_codegen_test_data/symengine/symforce_caspar_tridiagonal_test/kernel_ChainA_cyclic_reduction_up.cu index 4528c2a3..dd839d7b 100644 --- a/test/symforce_function_codegen_test_data/symengine/symforce_caspar_tridiagonal_test/kernel_ChainA_cyclic_reduction_up.cu +++ b/test/symforce_function_codegen_test_data/symengine/symforce_caspar_tridiagonal_test/kernel_ChainA_cyclic_reduction_up.cu @@ -1,17 +1,10 @@ #include #include -#include -#include -#include -#include -#include - +#include "cuda_to_hip.h" #include "kernel_ChainA_cyclic_reduction_up.h" #include "memops.cuh" -namespace cg = cooperative_groups; - namespace caspar { __global__ void __launch_bounds__(32, 1) diff --git a/test/symforce_function_codegen_test_data/symengine/symforce_caspar_tridiagonal_test/kernel_ChainA_cyclic_reduction_up.h b/test/symforce_function_codegen_test_data/symengine/symforce_caspar_tridiagonal_test/kernel_ChainA_cyclic_reduction_up.h index 39ef29d7..3147af1e 100644 --- a/test/symforce_function_codegen_test_data/symengine/symforce_caspar_tridiagonal_test/kernel_ChainA_cyclic_reduction_up.h +++ b/test/symforce_function_codegen_test_data/symengine/symforce_caspar_tridiagonal_test/kernel_ChainA_cyclic_reduction_up.h @@ -1,7 +1,6 @@ #pragma once -#include - +#include "cuda_to_hip.h" #include "shared_indices.h" namespace caspar { diff --git a/test/symforce_function_codegen_test_data/symengine/symforce_caspar_tridiagonal_test/kernel_ChainA_normalize.cu b/test/symforce_function_codegen_test_data/symengine/symforce_caspar_tridiagonal_test/kernel_ChainA_normalize.cu index 79c73bb1..f10a72e5 100644 --- a/test/symforce_function_codegen_test_data/symengine/symforce_caspar_tridiagonal_test/kernel_ChainA_normalize.cu +++ b/test/symforce_function_codegen_test_data/symengine/symforce_caspar_tridiagonal_test/kernel_ChainA_normalize.cu @@ -1,17 +1,10 @@ #include #include -#include -#include -#include -#include -#include - +#include "cuda_to_hip.h" #include "kernel_ChainA_normalize.h" #include "memops.cuh" -namespace cg = cooperative_groups; - namespace caspar { __global__ void __launch_bounds__(1024, 1) diff --git a/test/symforce_function_codegen_test_data/symengine/symforce_caspar_tridiagonal_test/kernel_ChainA_normalize.h b/test/symforce_function_codegen_test_data/symengine/symforce_caspar_tridiagonal_test/kernel_ChainA_normalize.h index 89f4215b..3c133ee6 100644 --- a/test/symforce_function_codegen_test_data/symengine/symforce_caspar_tridiagonal_test/kernel_ChainA_normalize.h +++ b/test/symforce_function_codegen_test_data/symengine/symforce_caspar_tridiagonal_test/kernel_ChainA_normalize.h @@ -1,7 +1,6 @@ #pragma once -#include - +#include "cuda_to_hip.h" #include "shared_indices.h" namespace caspar { diff --git a/test/symforce_function_codegen_test_data/symengine/symforce_caspar_tridiagonal_test/kernel_ChainA_pred_decrease_times_two.cu b/test/symforce_function_codegen_test_data/symengine/symforce_caspar_tridiagonal_test/kernel_ChainA_pred_decrease_times_two.cu index f65466e3..9e043786 100644 --- a/test/symforce_function_codegen_test_data/symengine/symforce_caspar_tridiagonal_test/kernel_ChainA_pred_decrease_times_two.cu +++ b/test/symforce_function_codegen_test_data/symengine/symforce_caspar_tridiagonal_test/kernel_ChainA_pred_decrease_times_two.cu @@ -1,17 +1,10 @@ #include #include -#include -#include -#include -#include -#include - +#include "cuda_to_hip.h" #include "kernel_ChainA_pred_decrease_times_two.h" #include "memops.cuh" -namespace cg = cooperative_groups; - namespace caspar { __global__ void __launch_bounds__(1024, 1) ChainAPredDecreaseTimesTwoKernel( diff --git a/test/symforce_function_codegen_test_data/symengine/symforce_caspar_tridiagonal_test/kernel_ChainA_pred_decrease_times_two.h b/test/symforce_function_codegen_test_data/symengine/symforce_caspar_tridiagonal_test/kernel_ChainA_pred_decrease_times_two.h index e4dce58f..3aa94f50 100644 --- a/test/symforce_function_codegen_test_data/symengine/symforce_caspar_tridiagonal_test/kernel_ChainA_pred_decrease_times_two.h +++ b/test/symforce_function_codegen_test_data/symengine/symforce_caspar_tridiagonal_test/kernel_ChainA_pred_decrease_times_two.h @@ -1,7 +1,6 @@ #pragma once -#include - +#include "cuda_to_hip.h" #include "shared_indices.h" namespace caspar { diff --git a/test/symforce_function_codegen_test_data/symengine/symforce_caspar_tridiagonal_test/kernel_ChainA_retract.cu b/test/symforce_function_codegen_test_data/symengine/symforce_caspar_tridiagonal_test/kernel_ChainA_retract.cu index 840f1547..23af59fd 100644 --- a/test/symforce_function_codegen_test_data/symengine/symforce_caspar_tridiagonal_test/kernel_ChainA_retract.cu +++ b/test/symforce_function_codegen_test_data/symengine/symforce_caspar_tridiagonal_test/kernel_ChainA_retract.cu @@ -1,17 +1,10 @@ #include #include -#include -#include -#include -#include -#include - +#include "cuda_to_hip.h" #include "kernel_ChainA_retract.h" #include "memops.cuh" -namespace cg = cooperative_groups; - namespace caspar { __global__ void __launch_bounds__(1024, 1) diff --git a/test/symforce_function_codegen_test_data/symengine/symforce_caspar_tridiagonal_test/kernel_ChainA_retract.h b/test/symforce_function_codegen_test_data/symengine/symforce_caspar_tridiagonal_test/kernel_ChainA_retract.h index ed36ee7e..69f2cbdf 100644 --- a/test/symforce_function_codegen_test_data/symengine/symforce_caspar_tridiagonal_test/kernel_ChainA_retract.h +++ b/test/symforce_function_codegen_test_data/symengine/symforce_caspar_tridiagonal_test/kernel_ChainA_retract.h @@ -1,7 +1,6 @@ #pragma once -#include - +#include "cuda_to_hip.h" #include "shared_indices.h" namespace caspar { diff --git a/test/symforce_function_codegen_test_data/symengine/symforce_caspar_tridiagonal_test/kernel_ChainA_start_w.cu b/test/symforce_function_codegen_test_data/symengine/symforce_caspar_tridiagonal_test/kernel_ChainA_start_w.cu index 8e5a69c0..9c2acfc4 100644 --- a/test/symforce_function_codegen_test_data/symengine/symforce_caspar_tridiagonal_test/kernel_ChainA_start_w.cu +++ b/test/symforce_function_codegen_test_data/symengine/symforce_caspar_tridiagonal_test/kernel_ChainA_start_w.cu @@ -1,17 +1,10 @@ #include #include -#include -#include -#include -#include -#include - +#include "cuda_to_hip.h" #include "kernel_ChainA_start_w.h" #include "memops.cuh" -namespace cg = cooperative_groups; - namespace caspar { __global__ void __launch_bounds__(1024, 1) diff --git a/test/symforce_function_codegen_test_data/symengine/symforce_caspar_tridiagonal_test/kernel_ChainA_start_w.h b/test/symforce_function_codegen_test_data/symengine/symforce_caspar_tridiagonal_test/kernel_ChainA_start_w.h index e64ad145..e2b2b853 100644 --- a/test/symforce_function_codegen_test_data/symengine/symforce_caspar_tridiagonal_test/kernel_ChainA_start_w.h +++ b/test/symforce_function_codegen_test_data/symengine/symforce_caspar_tridiagonal_test/kernel_ChainA_start_w.h @@ -1,7 +1,6 @@ #pragma once -#include - +#include "cuda_to_hip.h" #include "shared_indices.h" namespace caspar { diff --git a/test/symforce_function_codegen_test_data/symengine/symforce_caspar_tridiagonal_test/kernel_ChainA_start_w_contribute.cu b/test/symforce_function_codegen_test_data/symengine/symforce_caspar_tridiagonal_test/kernel_ChainA_start_w_contribute.cu index a6b6817b..c9a00429 100644 --- a/test/symforce_function_codegen_test_data/symengine/symforce_caspar_tridiagonal_test/kernel_ChainA_start_w_contribute.cu +++ b/test/symforce_function_codegen_test_data/symengine/symforce_caspar_tridiagonal_test/kernel_ChainA_start_w_contribute.cu @@ -1,17 +1,10 @@ #include #include -#include -#include -#include -#include -#include - +#include "cuda_to_hip.h" #include "kernel_ChainA_start_w_contribute.h" #include "memops.cuh" -namespace cg = cooperative_groups; - namespace caspar { __global__ void __launch_bounds__(1024, 1) diff --git a/test/symforce_function_codegen_test_data/symengine/symforce_caspar_tridiagonal_test/kernel_ChainA_start_w_contribute.h b/test/symforce_function_codegen_test_data/symengine/symforce_caspar_tridiagonal_test/kernel_ChainA_start_w_contribute.h index 31df0106..74a7b12d 100644 --- a/test/symforce_function_codegen_test_data/symengine/symforce_caspar_tridiagonal_test/kernel_ChainA_start_w_contribute.h +++ b/test/symforce_function_codegen_test_data/symengine/symforce_caspar_tridiagonal_test/kernel_ChainA_start_w_contribute.h @@ -1,7 +1,6 @@ #pragma once -#include - +#include "cuda_to_hip.h" #include "shared_indices.h" namespace caspar { diff --git a/test/symforce_function_codegen_test_data/symengine/symforce_caspar_tridiagonal_test/kernel_ChainA_tridig_make_eq.cu b/test/symforce_function_codegen_test_data/symengine/symforce_caspar_tridiagonal_test/kernel_ChainA_tridig_make_eq.cu index a0781406..436e8edc 100644 --- a/test/symforce_function_codegen_test_data/symengine/symforce_caspar_tridiagonal_test/kernel_ChainA_tridig_make_eq.cu +++ b/test/symforce_function_codegen_test_data/symengine/symforce_caspar_tridiagonal_test/kernel_ChainA_tridig_make_eq.cu @@ -1,17 +1,10 @@ #include #include -#include -#include -#include -#include -#include - +#include "cuda_to_hip.h" #include "kernel_ChainA_tridig_make_eq.h" #include "memops.cuh" -namespace cg = cooperative_groups; - namespace caspar { __global__ void __launch_bounds__(1024, 1) diff --git a/test/symforce_function_codegen_test_data/symengine/symforce_caspar_tridiagonal_test/kernel_ChainA_tridig_make_eq.h b/test/symforce_function_codegen_test_data/symengine/symforce_caspar_tridiagonal_test/kernel_ChainA_tridig_make_eq.h index 9052869a..449052fa 100644 --- a/test/symforce_function_codegen_test_data/symengine/symforce_caspar_tridiagonal_test/kernel_ChainA_tridig_make_eq.h +++ b/test/symforce_function_codegen_test_data/symengine/symforce_caspar_tridiagonal_test/kernel_ChainA_tridig_make_eq.h @@ -1,7 +1,6 @@ #pragma once -#include - +#include "cuda_to_hip.h" #include "shared_indices.h" namespace caspar { diff --git a/test/symforce_function_codegen_test_data/symengine/symforce_caspar_tridiagonal_test/kernel_ChainA_update_Mp.cu b/test/symforce_function_codegen_test_data/symengine/symforce_caspar_tridiagonal_test/kernel_ChainA_update_Mp.cu index 48d5d026..b7f41bef 100644 --- a/test/symforce_function_codegen_test_data/symengine/symforce_caspar_tridiagonal_test/kernel_ChainA_update_Mp.cu +++ b/test/symforce_function_codegen_test_data/symengine/symforce_caspar_tridiagonal_test/kernel_ChainA_update_Mp.cu @@ -1,17 +1,10 @@ #include #include -#include -#include -#include -#include -#include - +#include "cuda_to_hip.h" #include "kernel_ChainA_update_Mp.h" #include "memops.cuh" -namespace cg = cooperative_groups; - namespace caspar { __global__ void __launch_bounds__(1024, 1) diff --git a/test/symforce_function_codegen_test_data/symengine/symforce_caspar_tridiagonal_test/kernel_ChainA_update_Mp.h b/test/symforce_function_codegen_test_data/symengine/symforce_caspar_tridiagonal_test/kernel_ChainA_update_Mp.h index 12f1567d..f1d9753a 100644 --- a/test/symforce_function_codegen_test_data/symengine/symforce_caspar_tridiagonal_test/kernel_ChainA_update_Mp.h +++ b/test/symforce_function_codegen_test_data/symengine/symforce_caspar_tridiagonal_test/kernel_ChainA_update_Mp.h @@ -1,7 +1,6 @@ #pragma once -#include - +#include "cuda_to_hip.h" #include "shared_indices.h" namespace caspar { diff --git a/test/symforce_function_codegen_test_data/symengine/symforce_caspar_tridiagonal_test/kernel_ChainA_update_p.cu b/test/symforce_function_codegen_test_data/symengine/symforce_caspar_tridiagonal_test/kernel_ChainA_update_p.cu index c2116156..dc900ffb 100644 --- a/test/symforce_function_codegen_test_data/symengine/symforce_caspar_tridiagonal_test/kernel_ChainA_update_p.cu +++ b/test/symforce_function_codegen_test_data/symengine/symforce_caspar_tridiagonal_test/kernel_ChainA_update_p.cu @@ -1,17 +1,10 @@ #include #include -#include -#include -#include -#include -#include - +#include "cuda_to_hip.h" #include "kernel_ChainA_update_p.h" #include "memops.cuh" -namespace cg = cooperative_groups; - namespace caspar { __global__ void __launch_bounds__(1024, 1) diff --git a/test/symforce_function_codegen_test_data/symengine/symforce_caspar_tridiagonal_test/kernel_ChainA_update_p.h b/test/symforce_function_codegen_test_data/symengine/symforce_caspar_tridiagonal_test/kernel_ChainA_update_p.h index ffce655a..0a420698 100644 --- a/test/symforce_function_codegen_test_data/symengine/symforce_caspar_tridiagonal_test/kernel_ChainA_update_p.h +++ b/test/symforce_function_codegen_test_data/symengine/symforce_caspar_tridiagonal_test/kernel_ChainA_update_p.h @@ -1,7 +1,6 @@ #pragma once -#include - +#include "cuda_to_hip.h" #include "shared_indices.h" namespace caspar { diff --git a/test/symforce_function_codegen_test_data/symengine/symforce_caspar_tridiagonal_test/kernel_ChainA_update_r.cu b/test/symforce_function_codegen_test_data/symengine/symforce_caspar_tridiagonal_test/kernel_ChainA_update_r.cu index ff725db8..ed16ba53 100644 --- a/test/symforce_function_codegen_test_data/symengine/symforce_caspar_tridiagonal_test/kernel_ChainA_update_r.cu +++ b/test/symforce_function_codegen_test_data/symengine/symforce_caspar_tridiagonal_test/kernel_ChainA_update_r.cu @@ -1,17 +1,10 @@ #include #include -#include -#include -#include -#include -#include - +#include "cuda_to_hip.h" #include "kernel_ChainA_update_r.h" #include "memops.cuh" -namespace cg = cooperative_groups; - namespace caspar { __global__ void __launch_bounds__(1024, 1) diff --git a/test/symforce_function_codegen_test_data/symengine/symforce_caspar_tridiagonal_test/kernel_ChainA_update_r.h b/test/symforce_function_codegen_test_data/symengine/symforce_caspar_tridiagonal_test/kernel_ChainA_update_r.h index c5caa08e..9fe69710 100644 --- a/test/symforce_function_codegen_test_data/symengine/symforce_caspar_tridiagonal_test/kernel_ChainA_update_r.h +++ b/test/symforce_function_codegen_test_data/symengine/symforce_caspar_tridiagonal_test/kernel_ChainA_update_r.h @@ -1,7 +1,6 @@ #pragma once -#include - +#include "cuda_to_hip.h" #include "shared_indices.h" namespace caspar { diff --git a/test/symforce_function_codegen_test_data/symengine/symforce_caspar_tridiagonal_test/kernel_ChainA_update_r_first.cu b/test/symforce_function_codegen_test_data/symengine/symforce_caspar_tridiagonal_test/kernel_ChainA_update_r_first.cu index e35670ca..c1ede4bf 100644 --- a/test/symforce_function_codegen_test_data/symengine/symforce_caspar_tridiagonal_test/kernel_ChainA_update_r_first.cu +++ b/test/symforce_function_codegen_test_data/symengine/symforce_caspar_tridiagonal_test/kernel_ChainA_update_r_first.cu @@ -1,17 +1,10 @@ #include #include -#include -#include -#include -#include -#include - +#include "cuda_to_hip.h" #include "kernel_ChainA_update_r_first.h" #include "memops.cuh" -namespace cg = cooperative_groups; - namespace caspar { __global__ void __launch_bounds__(1024, 1) diff --git a/test/symforce_function_codegen_test_data/symengine/symforce_caspar_tridiagonal_test/kernel_ChainA_update_r_first.h b/test/symforce_function_codegen_test_data/symengine/symforce_caspar_tridiagonal_test/kernel_ChainA_update_r_first.h index 270fdda1..b17dd0d5 100644 --- a/test/symforce_function_codegen_test_data/symengine/symforce_caspar_tridiagonal_test/kernel_ChainA_update_r_first.h +++ b/test/symforce_function_codegen_test_data/symengine/symforce_caspar_tridiagonal_test/kernel_ChainA_update_r_first.h @@ -1,7 +1,6 @@ #pragma once -#include - +#include "cuda_to_hip.h" #include "shared_indices.h" namespace caspar { diff --git a/test/symforce_function_codegen_test_data/symengine/symforce_caspar_tridiagonal_test/kernel_ChainA_update_step.cu b/test/symforce_function_codegen_test_data/symengine/symforce_caspar_tridiagonal_test/kernel_ChainA_update_step.cu index 187263ef..3fcb94cc 100644 --- a/test/symforce_function_codegen_test_data/symengine/symforce_caspar_tridiagonal_test/kernel_ChainA_update_step.cu +++ b/test/symforce_function_codegen_test_data/symengine/symforce_caspar_tridiagonal_test/kernel_ChainA_update_step.cu @@ -1,17 +1,10 @@ #include #include -#include -#include -#include -#include -#include - +#include "cuda_to_hip.h" #include "kernel_ChainA_update_step.h" #include "memops.cuh" -namespace cg = cooperative_groups; - namespace caspar { __global__ void __launch_bounds__(1024, 1) diff --git a/test/symforce_function_codegen_test_data/symengine/symforce_caspar_tridiagonal_test/kernel_ChainA_update_step.h b/test/symforce_function_codegen_test_data/symengine/symforce_caspar_tridiagonal_test/kernel_ChainA_update_step.h index 1e4a1968..b109de03 100644 --- a/test/symforce_function_codegen_test_data/symengine/symforce_caspar_tridiagonal_test/kernel_ChainA_update_step.h +++ b/test/symforce_function_codegen_test_data/symengine/symforce_caspar_tridiagonal_test/kernel_ChainA_update_step.h @@ -1,7 +1,6 @@ #pragma once -#include - +#include "cuda_to_hip.h" #include "shared_indices.h" namespace caspar { diff --git a/test/symforce_function_codegen_test_data/symengine/symforce_caspar_tridiagonal_test/kernel_ChainA_update_step_first.cu b/test/symforce_function_codegen_test_data/symengine/symforce_caspar_tridiagonal_test/kernel_ChainA_update_step_first.cu index 6ce1c156..ad116a77 100644 --- a/test/symforce_function_codegen_test_data/symengine/symforce_caspar_tridiagonal_test/kernel_ChainA_update_step_first.cu +++ b/test/symforce_function_codegen_test_data/symengine/symforce_caspar_tridiagonal_test/kernel_ChainA_update_step_first.cu @@ -1,17 +1,10 @@ #include #include -#include -#include -#include -#include -#include - +#include "cuda_to_hip.h" #include "kernel_ChainA_update_step_first.h" #include "memops.cuh" -namespace cg = cooperative_groups; - namespace caspar { __global__ void __launch_bounds__(1024, 1) diff --git a/test/symforce_function_codegen_test_data/symengine/symforce_caspar_tridiagonal_test/kernel_ChainA_update_step_first.h b/test/symforce_function_codegen_test_data/symengine/symforce_caspar_tridiagonal_test/kernel_ChainA_update_step_first.h index 461de55f..0385d1ff 100644 --- a/test/symforce_function_codegen_test_data/symengine/symforce_caspar_tridiagonal_test/kernel_ChainA_update_step_first.h +++ b/test/symforce_function_codegen_test_data/symengine/symforce_caspar_tridiagonal_test/kernel_ChainA_update_step_first.h @@ -1,7 +1,6 @@ #pragma once -#include - +#include "cuda_to_hip.h" #include "shared_indices.h" namespace caspar { diff --git a/test/symforce_function_codegen_test_data/symengine/symforce_caspar_tridiagonal_test/kernel_ChainB_alpha_denominator_or_beta_numerator.cu b/test/symforce_function_codegen_test_data/symengine/symforce_caspar_tridiagonal_test/kernel_ChainB_alpha_denominator_or_beta_numerator.cu index c5e4aa53..cd4445c4 100644 --- a/test/symforce_function_codegen_test_data/symengine/symforce_caspar_tridiagonal_test/kernel_ChainB_alpha_denominator_or_beta_numerator.cu +++ b/test/symforce_function_codegen_test_data/symengine/symforce_caspar_tridiagonal_test/kernel_ChainB_alpha_denominator_or_beta_numerator.cu @@ -1,17 +1,10 @@ #include #include -#include -#include -#include -#include -#include - +#include "cuda_to_hip.h" #include "kernel_ChainB_alpha_denominator_or_beta_numerator.h" #include "memops.cuh" -namespace cg = cooperative_groups; - namespace caspar { __global__ void __launch_bounds__(1024, 1) diff --git a/test/symforce_function_codegen_test_data/symengine/symforce_caspar_tridiagonal_test/kernel_ChainB_alpha_denominator_or_beta_numerator.h b/test/symforce_function_codegen_test_data/symengine/symforce_caspar_tridiagonal_test/kernel_ChainB_alpha_denominator_or_beta_numerator.h index 5089b1c7..0ac4db49 100644 --- a/test/symforce_function_codegen_test_data/symengine/symforce_caspar_tridiagonal_test/kernel_ChainB_alpha_denominator_or_beta_numerator.h +++ b/test/symforce_function_codegen_test_data/symengine/symforce_caspar_tridiagonal_test/kernel_ChainB_alpha_denominator_or_beta_numerator.h @@ -1,7 +1,6 @@ #pragma once -#include - +#include "cuda_to_hip.h" #include "shared_indices.h" namespace caspar { diff --git a/test/symforce_function_codegen_test_data/symengine/symforce_caspar_tridiagonal_test/kernel_ChainB_alpha_numerator_denominator.cu b/test/symforce_function_codegen_test_data/symengine/symforce_caspar_tridiagonal_test/kernel_ChainB_alpha_numerator_denominator.cu index 4c8b49e2..b9502b30 100644 --- a/test/symforce_function_codegen_test_data/symengine/symforce_caspar_tridiagonal_test/kernel_ChainB_alpha_numerator_denominator.cu +++ b/test/symforce_function_codegen_test_data/symengine/symforce_caspar_tridiagonal_test/kernel_ChainB_alpha_numerator_denominator.cu @@ -1,17 +1,10 @@ #include #include -#include -#include -#include -#include -#include - +#include "cuda_to_hip.h" #include "kernel_ChainB_alpha_numerator_denominator.h" #include "memops.cuh" -namespace cg = cooperative_groups; - namespace caspar { __global__ void __launch_bounds__(1024, 1) diff --git a/test/symforce_function_codegen_test_data/symengine/symforce_caspar_tridiagonal_test/kernel_ChainB_alpha_numerator_denominator.h b/test/symforce_function_codegen_test_data/symengine/symforce_caspar_tridiagonal_test/kernel_ChainB_alpha_numerator_denominator.h index 613341f5..19dff844 100644 --- a/test/symforce_function_codegen_test_data/symengine/symforce_caspar_tridiagonal_test/kernel_ChainB_alpha_numerator_denominator.h +++ b/test/symforce_function_codegen_test_data/symengine/symforce_caspar_tridiagonal_test/kernel_ChainB_alpha_numerator_denominator.h @@ -1,7 +1,6 @@ #pragma once -#include - +#include "cuda_to_hip.h" #include "shared_indices.h" namespace caspar { diff --git a/test/symforce_function_codegen_test_data/symengine/symforce_caspar_tridiagonal_test/kernel_ChainB_cyclic_reduction_down.cu b/test/symforce_function_codegen_test_data/symengine/symforce_caspar_tridiagonal_test/kernel_ChainB_cyclic_reduction_down.cu index d1485291..8947c3fe 100644 --- a/test/symforce_function_codegen_test_data/symengine/symforce_caspar_tridiagonal_test/kernel_ChainB_cyclic_reduction_down.cu +++ b/test/symforce_function_codegen_test_data/symengine/symforce_caspar_tridiagonal_test/kernel_ChainB_cyclic_reduction_down.cu @@ -1,17 +1,10 @@ #include #include -#include -#include -#include -#include -#include - +#include "cuda_to_hip.h" #include "kernel_ChainB_cyclic_reduction_down.h" #include "memops.cuh" -namespace cg = cooperative_groups; - namespace caspar { __global__ void __launch_bounds__(64, 1) diff --git a/test/symforce_function_codegen_test_data/symengine/symforce_caspar_tridiagonal_test/kernel_ChainB_cyclic_reduction_down.h b/test/symforce_function_codegen_test_data/symengine/symforce_caspar_tridiagonal_test/kernel_ChainB_cyclic_reduction_down.h index 8ceb4040..977263fc 100644 --- a/test/symforce_function_codegen_test_data/symengine/symforce_caspar_tridiagonal_test/kernel_ChainB_cyclic_reduction_down.h +++ b/test/symforce_function_codegen_test_data/symengine/symforce_caspar_tridiagonal_test/kernel_ChainB_cyclic_reduction_down.h @@ -1,7 +1,6 @@ #pragma once -#include - +#include "cuda_to_hip.h" #include "shared_indices.h" namespace caspar { diff --git a/test/symforce_function_codegen_test_data/symengine/symforce_caspar_tridiagonal_test/kernel_ChainB_cyclic_reduction_solve_two.cu b/test/symforce_function_codegen_test_data/symengine/symforce_caspar_tridiagonal_test/kernel_ChainB_cyclic_reduction_solve_two.cu index aee9c96d..5cd6fe7e 100644 --- a/test/symforce_function_codegen_test_data/symengine/symforce_caspar_tridiagonal_test/kernel_ChainB_cyclic_reduction_solve_two.cu +++ b/test/symforce_function_codegen_test_data/symengine/symforce_caspar_tridiagonal_test/kernel_ChainB_cyclic_reduction_solve_two.cu @@ -1,17 +1,10 @@ #include #include -#include -#include -#include -#include -#include - +#include "cuda_to_hip.h" #include "kernel_ChainB_cyclic_reduction_solve_two.h" #include "memops.cuh" -namespace cg = cooperative_groups; - namespace caspar { __global__ void __launch_bounds__(32, 1) ChainBCyclicReductionSolveTwoKernel( diff --git a/test/symforce_function_codegen_test_data/symengine/symforce_caspar_tridiagonal_test/kernel_ChainB_cyclic_reduction_solve_two.h b/test/symforce_function_codegen_test_data/symengine/symforce_caspar_tridiagonal_test/kernel_ChainB_cyclic_reduction_solve_two.h index 9fae819f..45685b8d 100644 --- a/test/symforce_function_codegen_test_data/symengine/symforce_caspar_tridiagonal_test/kernel_ChainB_cyclic_reduction_solve_two.h +++ b/test/symforce_function_codegen_test_data/symengine/symforce_caspar_tridiagonal_test/kernel_ChainB_cyclic_reduction_solve_two.h @@ -1,7 +1,6 @@ #pragma once -#include - +#include "cuda_to_hip.h" #include "shared_indices.h" namespace caspar { diff --git a/test/symforce_function_codegen_test_data/symengine/symforce_caspar_tridiagonal_test/kernel_ChainB_cyclic_reduction_up.cu b/test/symforce_function_codegen_test_data/symengine/symforce_caspar_tridiagonal_test/kernel_ChainB_cyclic_reduction_up.cu index 66441e66..d5c75540 100644 --- a/test/symforce_function_codegen_test_data/symengine/symforce_caspar_tridiagonal_test/kernel_ChainB_cyclic_reduction_up.cu +++ b/test/symforce_function_codegen_test_data/symengine/symforce_caspar_tridiagonal_test/kernel_ChainB_cyclic_reduction_up.cu @@ -1,17 +1,10 @@ #include #include -#include -#include -#include -#include -#include - +#include "cuda_to_hip.h" #include "kernel_ChainB_cyclic_reduction_up.h" #include "memops.cuh" -namespace cg = cooperative_groups; - namespace caspar { __global__ void __launch_bounds__(32, 1) diff --git a/test/symforce_function_codegen_test_data/symengine/symforce_caspar_tridiagonal_test/kernel_ChainB_cyclic_reduction_up.h b/test/symforce_function_codegen_test_data/symengine/symforce_caspar_tridiagonal_test/kernel_ChainB_cyclic_reduction_up.h index 01c1a58a..7d294b91 100644 --- a/test/symforce_function_codegen_test_data/symengine/symforce_caspar_tridiagonal_test/kernel_ChainB_cyclic_reduction_up.h +++ b/test/symforce_function_codegen_test_data/symengine/symforce_caspar_tridiagonal_test/kernel_ChainB_cyclic_reduction_up.h @@ -1,7 +1,6 @@ #pragma once -#include - +#include "cuda_to_hip.h" #include "shared_indices.h" namespace caspar { diff --git a/test/symforce_function_codegen_test_data/symengine/symforce_caspar_tridiagonal_test/kernel_ChainB_normalize.cu b/test/symforce_function_codegen_test_data/symengine/symforce_caspar_tridiagonal_test/kernel_ChainB_normalize.cu index 15e36382..e330dc79 100644 --- a/test/symforce_function_codegen_test_data/symengine/symforce_caspar_tridiagonal_test/kernel_ChainB_normalize.cu +++ b/test/symforce_function_codegen_test_data/symengine/symforce_caspar_tridiagonal_test/kernel_ChainB_normalize.cu @@ -1,17 +1,10 @@ #include #include -#include -#include -#include -#include -#include - +#include "cuda_to_hip.h" #include "kernel_ChainB_normalize.h" #include "memops.cuh" -namespace cg = cooperative_groups; - namespace caspar { __global__ void __launch_bounds__(1024, 1) diff --git a/test/symforce_function_codegen_test_data/symengine/symforce_caspar_tridiagonal_test/kernel_ChainB_normalize.h b/test/symforce_function_codegen_test_data/symengine/symforce_caspar_tridiagonal_test/kernel_ChainB_normalize.h index de57b746..4f55d267 100644 --- a/test/symforce_function_codegen_test_data/symengine/symforce_caspar_tridiagonal_test/kernel_ChainB_normalize.h +++ b/test/symforce_function_codegen_test_data/symengine/symforce_caspar_tridiagonal_test/kernel_ChainB_normalize.h @@ -1,7 +1,6 @@ #pragma once -#include - +#include "cuda_to_hip.h" #include "shared_indices.h" namespace caspar { diff --git a/test/symforce_function_codegen_test_data/symengine/symforce_caspar_tridiagonal_test/kernel_ChainB_pred_decrease_times_two.cu b/test/symforce_function_codegen_test_data/symengine/symforce_caspar_tridiagonal_test/kernel_ChainB_pred_decrease_times_two.cu index f2e5ad81..9c905759 100644 --- a/test/symforce_function_codegen_test_data/symengine/symforce_caspar_tridiagonal_test/kernel_ChainB_pred_decrease_times_two.cu +++ b/test/symforce_function_codegen_test_data/symengine/symforce_caspar_tridiagonal_test/kernel_ChainB_pred_decrease_times_two.cu @@ -1,17 +1,10 @@ #include #include -#include -#include -#include -#include -#include - +#include "cuda_to_hip.h" #include "kernel_ChainB_pred_decrease_times_two.h" #include "memops.cuh" -namespace cg = cooperative_groups; - namespace caspar { __global__ void __launch_bounds__(1024, 1) ChainBPredDecreaseTimesTwoKernel( diff --git a/test/symforce_function_codegen_test_data/symengine/symforce_caspar_tridiagonal_test/kernel_ChainB_pred_decrease_times_two.h b/test/symforce_function_codegen_test_data/symengine/symforce_caspar_tridiagonal_test/kernel_ChainB_pred_decrease_times_two.h index 88b292b1..67d42837 100644 --- a/test/symforce_function_codegen_test_data/symengine/symforce_caspar_tridiagonal_test/kernel_ChainB_pred_decrease_times_two.h +++ b/test/symforce_function_codegen_test_data/symengine/symforce_caspar_tridiagonal_test/kernel_ChainB_pred_decrease_times_two.h @@ -1,7 +1,6 @@ #pragma once -#include - +#include "cuda_to_hip.h" #include "shared_indices.h" namespace caspar { diff --git a/test/symforce_function_codegen_test_data/symengine/symforce_caspar_tridiagonal_test/kernel_ChainB_retract.cu b/test/symforce_function_codegen_test_data/symengine/symforce_caspar_tridiagonal_test/kernel_ChainB_retract.cu index a70477ad..9aed0485 100644 --- a/test/symforce_function_codegen_test_data/symengine/symforce_caspar_tridiagonal_test/kernel_ChainB_retract.cu +++ b/test/symforce_function_codegen_test_data/symengine/symforce_caspar_tridiagonal_test/kernel_ChainB_retract.cu @@ -1,17 +1,10 @@ #include #include -#include -#include -#include -#include -#include - +#include "cuda_to_hip.h" #include "kernel_ChainB_retract.h" #include "memops.cuh" -namespace cg = cooperative_groups; - namespace caspar { __global__ void __launch_bounds__(1024, 1) diff --git a/test/symforce_function_codegen_test_data/symengine/symforce_caspar_tridiagonal_test/kernel_ChainB_retract.h b/test/symforce_function_codegen_test_data/symengine/symforce_caspar_tridiagonal_test/kernel_ChainB_retract.h index 4b7699ee..16aded79 100644 --- a/test/symforce_function_codegen_test_data/symengine/symforce_caspar_tridiagonal_test/kernel_ChainB_retract.h +++ b/test/symforce_function_codegen_test_data/symengine/symforce_caspar_tridiagonal_test/kernel_ChainB_retract.h @@ -1,7 +1,6 @@ #pragma once -#include - +#include "cuda_to_hip.h" #include "shared_indices.h" namespace caspar { diff --git a/test/symforce_function_codegen_test_data/symengine/symforce_caspar_tridiagonal_test/kernel_ChainB_start_w.cu b/test/symforce_function_codegen_test_data/symengine/symforce_caspar_tridiagonal_test/kernel_ChainB_start_w.cu index b704e6e7..e398255b 100644 --- a/test/symforce_function_codegen_test_data/symengine/symforce_caspar_tridiagonal_test/kernel_ChainB_start_w.cu +++ b/test/symforce_function_codegen_test_data/symengine/symforce_caspar_tridiagonal_test/kernel_ChainB_start_w.cu @@ -1,17 +1,10 @@ #include #include -#include -#include -#include -#include -#include - +#include "cuda_to_hip.h" #include "kernel_ChainB_start_w.h" #include "memops.cuh" -namespace cg = cooperative_groups; - namespace caspar { __global__ void __launch_bounds__(1024, 1) diff --git a/test/symforce_function_codegen_test_data/symengine/symforce_caspar_tridiagonal_test/kernel_ChainB_start_w.h b/test/symforce_function_codegen_test_data/symengine/symforce_caspar_tridiagonal_test/kernel_ChainB_start_w.h index 537d1853..6163eb68 100644 --- a/test/symforce_function_codegen_test_data/symengine/symforce_caspar_tridiagonal_test/kernel_ChainB_start_w.h +++ b/test/symforce_function_codegen_test_data/symengine/symforce_caspar_tridiagonal_test/kernel_ChainB_start_w.h @@ -1,7 +1,6 @@ #pragma once -#include - +#include "cuda_to_hip.h" #include "shared_indices.h" namespace caspar { diff --git a/test/symforce_function_codegen_test_data/symengine/symforce_caspar_tridiagonal_test/kernel_ChainB_start_w_contribute.cu b/test/symforce_function_codegen_test_data/symengine/symforce_caspar_tridiagonal_test/kernel_ChainB_start_w_contribute.cu index e1dbea27..5b527170 100644 --- a/test/symforce_function_codegen_test_data/symengine/symforce_caspar_tridiagonal_test/kernel_ChainB_start_w_contribute.cu +++ b/test/symforce_function_codegen_test_data/symengine/symforce_caspar_tridiagonal_test/kernel_ChainB_start_w_contribute.cu @@ -1,17 +1,10 @@ #include #include -#include -#include -#include -#include -#include - +#include "cuda_to_hip.h" #include "kernel_ChainB_start_w_contribute.h" #include "memops.cuh" -namespace cg = cooperative_groups; - namespace caspar { __global__ void __launch_bounds__(1024, 1) diff --git a/test/symforce_function_codegen_test_data/symengine/symforce_caspar_tridiagonal_test/kernel_ChainB_start_w_contribute.h b/test/symforce_function_codegen_test_data/symengine/symforce_caspar_tridiagonal_test/kernel_ChainB_start_w_contribute.h index 5ccd0fd1..a7fd62c0 100644 --- a/test/symforce_function_codegen_test_data/symengine/symforce_caspar_tridiagonal_test/kernel_ChainB_start_w_contribute.h +++ b/test/symforce_function_codegen_test_data/symengine/symforce_caspar_tridiagonal_test/kernel_ChainB_start_w_contribute.h @@ -1,7 +1,6 @@ #pragma once -#include - +#include "cuda_to_hip.h" #include "shared_indices.h" namespace caspar { diff --git a/test/symforce_function_codegen_test_data/symengine/symforce_caspar_tridiagonal_test/kernel_ChainB_tridig_make_eq.cu b/test/symforce_function_codegen_test_data/symengine/symforce_caspar_tridiagonal_test/kernel_ChainB_tridig_make_eq.cu index d6d1fb4b..8a566df8 100644 --- a/test/symforce_function_codegen_test_data/symengine/symforce_caspar_tridiagonal_test/kernel_ChainB_tridig_make_eq.cu +++ b/test/symforce_function_codegen_test_data/symengine/symforce_caspar_tridiagonal_test/kernel_ChainB_tridig_make_eq.cu @@ -1,17 +1,10 @@ #include #include -#include -#include -#include -#include -#include - +#include "cuda_to_hip.h" #include "kernel_ChainB_tridig_make_eq.h" #include "memops.cuh" -namespace cg = cooperative_groups; - namespace caspar { __global__ void __launch_bounds__(1024, 1) diff --git a/test/symforce_function_codegen_test_data/symengine/symforce_caspar_tridiagonal_test/kernel_ChainB_tridig_make_eq.h b/test/symforce_function_codegen_test_data/symengine/symforce_caspar_tridiagonal_test/kernel_ChainB_tridig_make_eq.h index 8872219e..56daa017 100644 --- a/test/symforce_function_codegen_test_data/symengine/symforce_caspar_tridiagonal_test/kernel_ChainB_tridig_make_eq.h +++ b/test/symforce_function_codegen_test_data/symengine/symforce_caspar_tridiagonal_test/kernel_ChainB_tridig_make_eq.h @@ -1,7 +1,6 @@ #pragma once -#include - +#include "cuda_to_hip.h" #include "shared_indices.h" namespace caspar { diff --git a/test/symforce_function_codegen_test_data/symengine/symforce_caspar_tridiagonal_test/kernel_ChainB_update_Mp.cu b/test/symforce_function_codegen_test_data/symengine/symforce_caspar_tridiagonal_test/kernel_ChainB_update_Mp.cu index 479a3026..033982da 100644 --- a/test/symforce_function_codegen_test_data/symengine/symforce_caspar_tridiagonal_test/kernel_ChainB_update_Mp.cu +++ b/test/symforce_function_codegen_test_data/symengine/symforce_caspar_tridiagonal_test/kernel_ChainB_update_Mp.cu @@ -1,17 +1,10 @@ #include #include -#include -#include -#include -#include -#include - +#include "cuda_to_hip.h" #include "kernel_ChainB_update_Mp.h" #include "memops.cuh" -namespace cg = cooperative_groups; - namespace caspar { __global__ void __launch_bounds__(1024, 1) diff --git a/test/symforce_function_codegen_test_data/symengine/symforce_caspar_tridiagonal_test/kernel_ChainB_update_Mp.h b/test/symforce_function_codegen_test_data/symengine/symforce_caspar_tridiagonal_test/kernel_ChainB_update_Mp.h index 13c7935f..21c02603 100644 --- a/test/symforce_function_codegen_test_data/symengine/symforce_caspar_tridiagonal_test/kernel_ChainB_update_Mp.h +++ b/test/symforce_function_codegen_test_data/symengine/symforce_caspar_tridiagonal_test/kernel_ChainB_update_Mp.h @@ -1,7 +1,6 @@ #pragma once -#include - +#include "cuda_to_hip.h" #include "shared_indices.h" namespace caspar { diff --git a/test/symforce_function_codegen_test_data/symengine/symforce_caspar_tridiagonal_test/kernel_ChainB_update_p.cu b/test/symforce_function_codegen_test_data/symengine/symforce_caspar_tridiagonal_test/kernel_ChainB_update_p.cu index e2a6843d..cdaec3d8 100644 --- a/test/symforce_function_codegen_test_data/symengine/symforce_caspar_tridiagonal_test/kernel_ChainB_update_p.cu +++ b/test/symforce_function_codegen_test_data/symengine/symforce_caspar_tridiagonal_test/kernel_ChainB_update_p.cu @@ -1,17 +1,10 @@ #include #include -#include -#include -#include -#include -#include - +#include "cuda_to_hip.h" #include "kernel_ChainB_update_p.h" #include "memops.cuh" -namespace cg = cooperative_groups; - namespace caspar { __global__ void __launch_bounds__(1024, 1) diff --git a/test/symforce_function_codegen_test_data/symengine/symforce_caspar_tridiagonal_test/kernel_ChainB_update_p.h b/test/symforce_function_codegen_test_data/symengine/symforce_caspar_tridiagonal_test/kernel_ChainB_update_p.h index 3c6125d5..4dd63968 100644 --- a/test/symforce_function_codegen_test_data/symengine/symforce_caspar_tridiagonal_test/kernel_ChainB_update_p.h +++ b/test/symforce_function_codegen_test_data/symengine/symforce_caspar_tridiagonal_test/kernel_ChainB_update_p.h @@ -1,7 +1,6 @@ #pragma once -#include - +#include "cuda_to_hip.h" #include "shared_indices.h" namespace caspar { diff --git a/test/symforce_function_codegen_test_data/symengine/symforce_caspar_tridiagonal_test/kernel_ChainB_update_r.cu b/test/symforce_function_codegen_test_data/symengine/symforce_caspar_tridiagonal_test/kernel_ChainB_update_r.cu index 355656ca..fb60a073 100644 --- a/test/symforce_function_codegen_test_data/symengine/symforce_caspar_tridiagonal_test/kernel_ChainB_update_r.cu +++ b/test/symforce_function_codegen_test_data/symengine/symforce_caspar_tridiagonal_test/kernel_ChainB_update_r.cu @@ -1,17 +1,10 @@ #include #include -#include -#include -#include -#include -#include - +#include "cuda_to_hip.h" #include "kernel_ChainB_update_r.h" #include "memops.cuh" -namespace cg = cooperative_groups; - namespace caspar { __global__ void __launch_bounds__(1024, 1) diff --git a/test/symforce_function_codegen_test_data/symengine/symforce_caspar_tridiagonal_test/kernel_ChainB_update_r.h b/test/symforce_function_codegen_test_data/symengine/symforce_caspar_tridiagonal_test/kernel_ChainB_update_r.h index c8eb9f28..2268a65b 100644 --- a/test/symforce_function_codegen_test_data/symengine/symforce_caspar_tridiagonal_test/kernel_ChainB_update_r.h +++ b/test/symforce_function_codegen_test_data/symengine/symforce_caspar_tridiagonal_test/kernel_ChainB_update_r.h @@ -1,7 +1,6 @@ #pragma once -#include - +#include "cuda_to_hip.h" #include "shared_indices.h" namespace caspar { diff --git a/test/symforce_function_codegen_test_data/symengine/symforce_caspar_tridiagonal_test/kernel_ChainB_update_r_first.cu b/test/symforce_function_codegen_test_data/symengine/symforce_caspar_tridiagonal_test/kernel_ChainB_update_r_first.cu index 9f7c746f..2426e9a5 100644 --- a/test/symforce_function_codegen_test_data/symengine/symforce_caspar_tridiagonal_test/kernel_ChainB_update_r_first.cu +++ b/test/symforce_function_codegen_test_data/symengine/symforce_caspar_tridiagonal_test/kernel_ChainB_update_r_first.cu @@ -1,17 +1,10 @@ #include #include -#include -#include -#include -#include -#include - +#include "cuda_to_hip.h" #include "kernel_ChainB_update_r_first.h" #include "memops.cuh" -namespace cg = cooperative_groups; - namespace caspar { __global__ void __launch_bounds__(1024, 1) diff --git a/test/symforce_function_codegen_test_data/symengine/symforce_caspar_tridiagonal_test/kernel_ChainB_update_r_first.h b/test/symforce_function_codegen_test_data/symengine/symforce_caspar_tridiagonal_test/kernel_ChainB_update_r_first.h index 1ff22f75..27bc58a9 100644 --- a/test/symforce_function_codegen_test_data/symengine/symforce_caspar_tridiagonal_test/kernel_ChainB_update_r_first.h +++ b/test/symforce_function_codegen_test_data/symengine/symforce_caspar_tridiagonal_test/kernel_ChainB_update_r_first.h @@ -1,7 +1,6 @@ #pragma once -#include - +#include "cuda_to_hip.h" #include "shared_indices.h" namespace caspar { diff --git a/test/symforce_function_codegen_test_data/symengine/symforce_caspar_tridiagonal_test/kernel_ChainB_update_step.cu b/test/symforce_function_codegen_test_data/symengine/symforce_caspar_tridiagonal_test/kernel_ChainB_update_step.cu index 9cd0cc24..bd0cac85 100644 --- a/test/symforce_function_codegen_test_data/symengine/symforce_caspar_tridiagonal_test/kernel_ChainB_update_step.cu +++ b/test/symforce_function_codegen_test_data/symengine/symforce_caspar_tridiagonal_test/kernel_ChainB_update_step.cu @@ -1,17 +1,10 @@ #include #include -#include -#include -#include -#include -#include - +#include "cuda_to_hip.h" #include "kernel_ChainB_update_step.h" #include "memops.cuh" -namespace cg = cooperative_groups; - namespace caspar { __global__ void __launch_bounds__(1024, 1) diff --git a/test/symforce_function_codegen_test_data/symengine/symforce_caspar_tridiagonal_test/kernel_ChainB_update_step.h b/test/symforce_function_codegen_test_data/symengine/symforce_caspar_tridiagonal_test/kernel_ChainB_update_step.h index 006779e7..eeb880b1 100644 --- a/test/symforce_function_codegen_test_data/symengine/symforce_caspar_tridiagonal_test/kernel_ChainB_update_step.h +++ b/test/symforce_function_codegen_test_data/symengine/symforce_caspar_tridiagonal_test/kernel_ChainB_update_step.h @@ -1,7 +1,6 @@ #pragma once -#include - +#include "cuda_to_hip.h" #include "shared_indices.h" namespace caspar { diff --git a/test/symforce_function_codegen_test_data/symengine/symforce_caspar_tridiagonal_test/kernel_ChainB_update_step_first.cu b/test/symforce_function_codegen_test_data/symengine/symforce_caspar_tridiagonal_test/kernel_ChainB_update_step_first.cu index e3f1301d..c332c87b 100644 --- a/test/symforce_function_codegen_test_data/symengine/symforce_caspar_tridiagonal_test/kernel_ChainB_update_step_first.cu +++ b/test/symforce_function_codegen_test_data/symengine/symforce_caspar_tridiagonal_test/kernel_ChainB_update_step_first.cu @@ -1,17 +1,10 @@ #include #include -#include -#include -#include -#include -#include - +#include "cuda_to_hip.h" #include "kernel_ChainB_update_step_first.h" #include "memops.cuh" -namespace cg = cooperative_groups; - namespace caspar { __global__ void __launch_bounds__(1024, 1) diff --git a/test/symforce_function_codegen_test_data/symengine/symforce_caspar_tridiagonal_test/kernel_ChainB_update_step_first.h b/test/symforce_function_codegen_test_data/symengine/symforce_caspar_tridiagonal_test/kernel_ChainB_update_step_first.h index 1036592e..caf242c5 100644 --- a/test/symforce_function_codegen_test_data/symengine/symforce_caspar_tridiagonal_test/kernel_ChainB_update_step_first.h +++ b/test/symforce_function_codegen_test_data/symengine/symforce_caspar_tridiagonal_test/kernel_ChainB_update_step_first.h @@ -1,7 +1,6 @@ #pragma once -#include - +#include "cuda_to_hip.h" #include "shared_indices.h" namespace caspar { diff --git a/test/symforce_function_codegen_test_data/symengine/symforce_caspar_tridiagonal_test/kernel_chain_a_link_jtjnjtr_direct.cu b/test/symforce_function_codegen_test_data/symengine/symforce_caspar_tridiagonal_test/kernel_chain_a_link_jtjnjtr_direct.cu index 369a87ed..f7d98995 100644 --- a/test/symforce_function_codegen_test_data/symengine/symforce_caspar_tridiagonal_test/kernel_chain_a_link_jtjnjtr_direct.cu +++ b/test/symforce_function_codegen_test_data/symengine/symforce_caspar_tridiagonal_test/kernel_chain_a_link_jtjnjtr_direct.cu @@ -1,17 +1,10 @@ #include #include -#include -#include -#include -#include -#include - +#include "cuda_to_hip.h" #include "kernel_chain_a_link_jtjnjtr_direct.h" #include "memops.cuh" -namespace cg = cooperative_groups; - namespace caspar { __global__ void __launch_bounds__(1024, 1) diff --git a/test/symforce_function_codegen_test_data/symengine/symforce_caspar_tridiagonal_test/kernel_chain_a_link_jtjnjtr_direct.h b/test/symforce_function_codegen_test_data/symengine/symforce_caspar_tridiagonal_test/kernel_chain_a_link_jtjnjtr_direct.h index ed6ae6d4..02d71b44 100644 --- a/test/symforce_function_codegen_test_data/symengine/symforce_caspar_tridiagonal_test/kernel_chain_a_link_jtjnjtr_direct.h +++ b/test/symforce_function_codegen_test_data/symengine/symforce_caspar_tridiagonal_test/kernel_chain_a_link_jtjnjtr_direct.h @@ -1,7 +1,6 @@ #pragma once -#include - +#include "cuda_to_hip.h" #include "shared_indices.h" namespace caspar { diff --git a/test/symforce_function_codegen_test_data/symengine/symforce_caspar_tridiagonal_test/kernel_chain_a_link_res_jac.cu b/test/symforce_function_codegen_test_data/symengine/symforce_caspar_tridiagonal_test/kernel_chain_a_link_res_jac.cu index 7495787b..9545daf0 100644 --- a/test/symforce_function_codegen_test_data/symengine/symforce_caspar_tridiagonal_test/kernel_chain_a_link_res_jac.cu +++ b/test/symforce_function_codegen_test_data/symengine/symforce_caspar_tridiagonal_test/kernel_chain_a_link_res_jac.cu @@ -1,17 +1,10 @@ #include #include -#include -#include -#include -#include -#include - +#include "cuda_to_hip.h" #include "kernel_chain_a_link_res_jac.h" #include "memops.cuh" -namespace cg = cooperative_groups; - namespace caspar { __global__ void __launch_bounds__(1024, 1) ChainALinkResJacKernel( diff --git a/test/symforce_function_codegen_test_data/symengine/symforce_caspar_tridiagonal_test/kernel_chain_a_link_res_jac.h b/test/symforce_function_codegen_test_data/symengine/symforce_caspar_tridiagonal_test/kernel_chain_a_link_res_jac.h index 28cff54b..b91e80a3 100644 --- a/test/symforce_function_codegen_test_data/symengine/symforce_caspar_tridiagonal_test/kernel_chain_a_link_res_jac.h +++ b/test/symforce_function_codegen_test_data/symengine/symforce_caspar_tridiagonal_test/kernel_chain_a_link_res_jac.h @@ -1,7 +1,6 @@ #pragma once -#include - +#include "cuda_to_hip.h" #include "shared_indices.h" namespace caspar { diff --git a/test/symforce_function_codegen_test_data/symengine/symforce_caspar_tridiagonal_test/kernel_chain_a_link_res_jac_first.cu b/test/symforce_function_codegen_test_data/symengine/symforce_caspar_tridiagonal_test/kernel_chain_a_link_res_jac_first.cu index df421ac3..cc6e6e35 100644 --- a/test/symforce_function_codegen_test_data/symengine/symforce_caspar_tridiagonal_test/kernel_chain_a_link_res_jac_first.cu +++ b/test/symforce_function_codegen_test_data/symengine/symforce_caspar_tridiagonal_test/kernel_chain_a_link_res_jac_first.cu @@ -1,17 +1,10 @@ #include #include -#include -#include -#include -#include -#include - +#include "cuda_to_hip.h" #include "kernel_chain_a_link_res_jac_first.h" #include "memops.cuh" -namespace cg = cooperative_groups; - namespace caspar { __global__ void __launch_bounds__(1024, 1) ChainALinkResJacFirstKernel( diff --git a/test/symforce_function_codegen_test_data/symengine/symforce_caspar_tridiagonal_test/kernel_chain_a_link_res_jac_first.h b/test/symforce_function_codegen_test_data/symengine/symforce_caspar_tridiagonal_test/kernel_chain_a_link_res_jac_first.h index b757536c..64422a63 100644 --- a/test/symforce_function_codegen_test_data/symengine/symforce_caspar_tridiagonal_test/kernel_chain_a_link_res_jac_first.h +++ b/test/symforce_function_codegen_test_data/symengine/symforce_caspar_tridiagonal_test/kernel_chain_a_link_res_jac_first.h @@ -1,7 +1,6 @@ #pragma once -#include - +#include "cuda_to_hip.h" #include "shared_indices.h" namespace caspar { diff --git a/test/symforce_function_codegen_test_data/symengine/symforce_caspar_tridiagonal_test/kernel_chain_a_link_score.cu b/test/symforce_function_codegen_test_data/symengine/symforce_caspar_tridiagonal_test/kernel_chain_a_link_score.cu index 50df9650..969b30f8 100644 --- a/test/symforce_function_codegen_test_data/symengine/symforce_caspar_tridiagonal_test/kernel_chain_a_link_score.cu +++ b/test/symforce_function_codegen_test_data/symengine/symforce_caspar_tridiagonal_test/kernel_chain_a_link_score.cu @@ -1,17 +1,10 @@ #include #include -#include -#include -#include -#include -#include - +#include "cuda_to_hip.h" #include "kernel_chain_a_link_score.h" #include "memops.cuh" -namespace cg = cooperative_groups; - namespace caspar { __global__ void __launch_bounds__(1024, 1) diff --git a/test/symforce_function_codegen_test_data/symengine/symforce_caspar_tridiagonal_test/kernel_chain_a_link_score.h b/test/symforce_function_codegen_test_data/symengine/symforce_caspar_tridiagonal_test/kernel_chain_a_link_score.h index 633032eb..35749c52 100644 --- a/test/symforce_function_codegen_test_data/symengine/symforce_caspar_tridiagonal_test/kernel_chain_a_link_score.h +++ b/test/symforce_function_codegen_test_data/symengine/symforce_caspar_tridiagonal_test/kernel_chain_a_link_score.h @@ -1,7 +1,6 @@ #pragma once -#include - +#include "cuda_to_hip.h" #include "shared_indices.h" namespace caspar { diff --git a/test/symforce_function_codegen_test_data/symengine/symforce_caspar_tridiagonal_test/kernel_chain_a_prior_jtjnjtr_direct.cu b/test/symforce_function_codegen_test_data/symengine/symforce_caspar_tridiagonal_test/kernel_chain_a_prior_jtjnjtr_direct.cu index 4f00ae2b..372cb17d 100644 --- a/test/symforce_function_codegen_test_data/symengine/symforce_caspar_tridiagonal_test/kernel_chain_a_prior_jtjnjtr_direct.cu +++ b/test/symforce_function_codegen_test_data/symengine/symforce_caspar_tridiagonal_test/kernel_chain_a_prior_jtjnjtr_direct.cu @@ -1,17 +1,10 @@ #include #include -#include -#include -#include -#include -#include - +#include "cuda_to_hip.h" #include "kernel_chain_a_prior_jtjnjtr_direct.h" #include "memops.cuh" -namespace cg = cooperative_groups; - namespace caspar { __global__ void __launch_bounds__(1024, 1) diff --git a/test/symforce_function_codegen_test_data/symengine/symforce_caspar_tridiagonal_test/kernel_chain_a_prior_jtjnjtr_direct.h b/test/symforce_function_codegen_test_data/symengine/symforce_caspar_tridiagonal_test/kernel_chain_a_prior_jtjnjtr_direct.h index 1f456876..f2c87bf4 100644 --- a/test/symforce_function_codegen_test_data/symengine/symforce_caspar_tridiagonal_test/kernel_chain_a_prior_jtjnjtr_direct.h +++ b/test/symforce_function_codegen_test_data/symengine/symforce_caspar_tridiagonal_test/kernel_chain_a_prior_jtjnjtr_direct.h @@ -1,7 +1,6 @@ #pragma once -#include - +#include "cuda_to_hip.h" #include "shared_indices.h" namespace caspar { diff --git a/test/symforce_function_codegen_test_data/symengine/symforce_caspar_tridiagonal_test/kernel_chain_a_prior_res_jac.cu b/test/symforce_function_codegen_test_data/symengine/symforce_caspar_tridiagonal_test/kernel_chain_a_prior_res_jac.cu index 7da8cd8d..c38f5fe9 100644 --- a/test/symforce_function_codegen_test_data/symengine/symforce_caspar_tridiagonal_test/kernel_chain_a_prior_res_jac.cu +++ b/test/symforce_function_codegen_test_data/symengine/symforce_caspar_tridiagonal_test/kernel_chain_a_prior_res_jac.cu @@ -1,17 +1,10 @@ #include #include -#include -#include -#include -#include -#include - +#include "cuda_to_hip.h" #include "kernel_chain_a_prior_res_jac.h" #include "memops.cuh" -namespace cg = cooperative_groups; - namespace caspar { __global__ void __launch_bounds__(1024, 1) diff --git a/test/symforce_function_codegen_test_data/symengine/symforce_caspar_tridiagonal_test/kernel_chain_a_prior_res_jac.h b/test/symforce_function_codegen_test_data/symengine/symforce_caspar_tridiagonal_test/kernel_chain_a_prior_res_jac.h index 5ca63a90..c23d38b9 100644 --- a/test/symforce_function_codegen_test_data/symengine/symforce_caspar_tridiagonal_test/kernel_chain_a_prior_res_jac.h +++ b/test/symforce_function_codegen_test_data/symengine/symforce_caspar_tridiagonal_test/kernel_chain_a_prior_res_jac.h @@ -1,7 +1,6 @@ #pragma once -#include - +#include "cuda_to_hip.h" #include "shared_indices.h" namespace caspar { diff --git a/test/symforce_function_codegen_test_data/symengine/symforce_caspar_tridiagonal_test/kernel_chain_a_prior_res_jac_first.cu b/test/symforce_function_codegen_test_data/symengine/symforce_caspar_tridiagonal_test/kernel_chain_a_prior_res_jac_first.cu index ddbab613..b2e6a226 100644 --- a/test/symforce_function_codegen_test_data/symengine/symforce_caspar_tridiagonal_test/kernel_chain_a_prior_res_jac_first.cu +++ b/test/symforce_function_codegen_test_data/symengine/symforce_caspar_tridiagonal_test/kernel_chain_a_prior_res_jac_first.cu @@ -1,17 +1,10 @@ #include #include -#include -#include -#include -#include -#include - +#include "cuda_to_hip.h" #include "kernel_chain_a_prior_res_jac_first.h" #include "memops.cuh" -namespace cg = cooperative_groups; - namespace caspar { __global__ void __launch_bounds__(1024, 1) diff --git a/test/symforce_function_codegen_test_data/symengine/symforce_caspar_tridiagonal_test/kernel_chain_a_prior_res_jac_first.h b/test/symforce_function_codegen_test_data/symengine/symforce_caspar_tridiagonal_test/kernel_chain_a_prior_res_jac_first.h index 3cdceb88..f678cda1 100644 --- a/test/symforce_function_codegen_test_data/symengine/symforce_caspar_tridiagonal_test/kernel_chain_a_prior_res_jac_first.h +++ b/test/symforce_function_codegen_test_data/symengine/symforce_caspar_tridiagonal_test/kernel_chain_a_prior_res_jac_first.h @@ -1,7 +1,6 @@ #pragma once -#include - +#include "cuda_to_hip.h" #include "shared_indices.h" namespace caspar { diff --git a/test/symforce_function_codegen_test_data/symengine/symforce_caspar_tridiagonal_test/kernel_chain_a_prior_score.cu b/test/symforce_function_codegen_test_data/symengine/symforce_caspar_tridiagonal_test/kernel_chain_a_prior_score.cu index 6335d83c..90275926 100644 --- a/test/symforce_function_codegen_test_data/symengine/symforce_caspar_tridiagonal_test/kernel_chain_a_prior_score.cu +++ b/test/symforce_function_codegen_test_data/symengine/symforce_caspar_tridiagonal_test/kernel_chain_a_prior_score.cu @@ -1,17 +1,10 @@ #include #include -#include -#include -#include -#include -#include - +#include "cuda_to_hip.h" #include "kernel_chain_a_prior_score.h" #include "memops.cuh" -namespace cg = cooperative_groups; - namespace caspar { __global__ void __launch_bounds__(1024, 1) diff --git a/test/symforce_function_codegen_test_data/symengine/symforce_caspar_tridiagonal_test/kernel_chain_a_prior_score.h b/test/symforce_function_codegen_test_data/symengine/symforce_caspar_tridiagonal_test/kernel_chain_a_prior_score.h index f380ace1..10b0b6b9 100644 --- a/test/symforce_function_codegen_test_data/symengine/symforce_caspar_tridiagonal_test/kernel_chain_a_prior_score.h +++ b/test/symforce_function_codegen_test_data/symengine/symforce_caspar_tridiagonal_test/kernel_chain_a_prior_score.h @@ -1,7 +1,6 @@ #pragma once -#include - +#include "cuda_to_hip.h" #include "shared_indices.h" namespace caspar { diff --git a/test/symforce_function_codegen_test_data/symengine/symforce_caspar_tridiagonal_test/kernel_chain_b_link_jtjnjtr_direct.cu b/test/symforce_function_codegen_test_data/symengine/symforce_caspar_tridiagonal_test/kernel_chain_b_link_jtjnjtr_direct.cu index 3b336af2..61df87a9 100644 --- a/test/symforce_function_codegen_test_data/symengine/symforce_caspar_tridiagonal_test/kernel_chain_b_link_jtjnjtr_direct.cu +++ b/test/symforce_function_codegen_test_data/symengine/symforce_caspar_tridiagonal_test/kernel_chain_b_link_jtjnjtr_direct.cu @@ -1,17 +1,10 @@ #include #include -#include -#include -#include -#include -#include - +#include "cuda_to_hip.h" #include "kernel_chain_b_link_jtjnjtr_direct.h" #include "memops.cuh" -namespace cg = cooperative_groups; - namespace caspar { __global__ void __launch_bounds__(1024, 1) diff --git a/test/symforce_function_codegen_test_data/symengine/symforce_caspar_tridiagonal_test/kernel_chain_b_link_jtjnjtr_direct.h b/test/symforce_function_codegen_test_data/symengine/symforce_caspar_tridiagonal_test/kernel_chain_b_link_jtjnjtr_direct.h index de1461c2..5721bcce 100644 --- a/test/symforce_function_codegen_test_data/symengine/symforce_caspar_tridiagonal_test/kernel_chain_b_link_jtjnjtr_direct.h +++ b/test/symforce_function_codegen_test_data/symengine/symforce_caspar_tridiagonal_test/kernel_chain_b_link_jtjnjtr_direct.h @@ -1,7 +1,6 @@ #pragma once -#include - +#include "cuda_to_hip.h" #include "shared_indices.h" namespace caspar { diff --git a/test/symforce_function_codegen_test_data/symengine/symforce_caspar_tridiagonal_test/kernel_chain_b_link_res_jac.cu b/test/symforce_function_codegen_test_data/symengine/symforce_caspar_tridiagonal_test/kernel_chain_b_link_res_jac.cu index 4642b781..30281b27 100644 --- a/test/symforce_function_codegen_test_data/symengine/symforce_caspar_tridiagonal_test/kernel_chain_b_link_res_jac.cu +++ b/test/symforce_function_codegen_test_data/symengine/symforce_caspar_tridiagonal_test/kernel_chain_b_link_res_jac.cu @@ -1,17 +1,10 @@ #include #include -#include -#include -#include -#include -#include - +#include "cuda_to_hip.h" #include "kernel_chain_b_link_res_jac.h" #include "memops.cuh" -namespace cg = cooperative_groups; - namespace caspar { __global__ void __launch_bounds__(1024, 1) ChainBLinkResJacKernel( diff --git a/test/symforce_function_codegen_test_data/symengine/symforce_caspar_tridiagonal_test/kernel_chain_b_link_res_jac.h b/test/symforce_function_codegen_test_data/symengine/symforce_caspar_tridiagonal_test/kernel_chain_b_link_res_jac.h index d3ca19d4..6355e2b6 100644 --- a/test/symforce_function_codegen_test_data/symengine/symforce_caspar_tridiagonal_test/kernel_chain_b_link_res_jac.h +++ b/test/symforce_function_codegen_test_data/symengine/symforce_caspar_tridiagonal_test/kernel_chain_b_link_res_jac.h @@ -1,7 +1,6 @@ #pragma once -#include - +#include "cuda_to_hip.h" #include "shared_indices.h" namespace caspar { diff --git a/test/symforce_function_codegen_test_data/symengine/symforce_caspar_tridiagonal_test/kernel_chain_b_link_res_jac_first.cu b/test/symforce_function_codegen_test_data/symengine/symforce_caspar_tridiagonal_test/kernel_chain_b_link_res_jac_first.cu index 22ff48f3..ac1d68cf 100644 --- a/test/symforce_function_codegen_test_data/symengine/symforce_caspar_tridiagonal_test/kernel_chain_b_link_res_jac_first.cu +++ b/test/symforce_function_codegen_test_data/symengine/symforce_caspar_tridiagonal_test/kernel_chain_b_link_res_jac_first.cu @@ -1,17 +1,10 @@ #include #include -#include -#include -#include -#include -#include - +#include "cuda_to_hip.h" #include "kernel_chain_b_link_res_jac_first.h" #include "memops.cuh" -namespace cg = cooperative_groups; - namespace caspar { __global__ void __launch_bounds__(1024, 1) ChainBLinkResJacFirstKernel( diff --git a/test/symforce_function_codegen_test_data/symengine/symforce_caspar_tridiagonal_test/kernel_chain_b_link_res_jac_first.h b/test/symforce_function_codegen_test_data/symengine/symforce_caspar_tridiagonal_test/kernel_chain_b_link_res_jac_first.h index 748c5f8e..3278e5ab 100644 --- a/test/symforce_function_codegen_test_data/symengine/symforce_caspar_tridiagonal_test/kernel_chain_b_link_res_jac_first.h +++ b/test/symforce_function_codegen_test_data/symengine/symforce_caspar_tridiagonal_test/kernel_chain_b_link_res_jac_first.h @@ -1,7 +1,6 @@ #pragma once -#include - +#include "cuda_to_hip.h" #include "shared_indices.h" namespace caspar { diff --git a/test/symforce_function_codegen_test_data/symengine/symforce_caspar_tridiagonal_test/kernel_chain_b_link_score.cu b/test/symforce_function_codegen_test_data/symengine/symforce_caspar_tridiagonal_test/kernel_chain_b_link_score.cu index 5fb16e3b..24aba18a 100644 --- a/test/symforce_function_codegen_test_data/symengine/symforce_caspar_tridiagonal_test/kernel_chain_b_link_score.cu +++ b/test/symforce_function_codegen_test_data/symengine/symforce_caspar_tridiagonal_test/kernel_chain_b_link_score.cu @@ -1,17 +1,10 @@ #include #include -#include -#include -#include -#include -#include - +#include "cuda_to_hip.h" #include "kernel_chain_b_link_score.h" #include "memops.cuh" -namespace cg = cooperative_groups; - namespace caspar { __global__ void __launch_bounds__(1024, 1) diff --git a/test/symforce_function_codegen_test_data/symengine/symforce_caspar_tridiagonal_test/kernel_chain_b_link_score.h b/test/symforce_function_codegen_test_data/symengine/symforce_caspar_tridiagonal_test/kernel_chain_b_link_score.h index c7986d12..2ba36574 100644 --- a/test/symforce_function_codegen_test_data/symengine/symforce_caspar_tridiagonal_test/kernel_chain_b_link_score.h +++ b/test/symforce_function_codegen_test_data/symengine/symforce_caspar_tridiagonal_test/kernel_chain_b_link_score.h @@ -1,7 +1,6 @@ #pragma once -#include - +#include "cuda_to_hip.h" #include "shared_indices.h" namespace caspar { diff --git a/test/symforce_function_codegen_test_data/symengine/symforce_caspar_tridiagonal_test/kernel_chain_b_prior_jtjnjtr_direct.cu b/test/symforce_function_codegen_test_data/symengine/symforce_caspar_tridiagonal_test/kernel_chain_b_prior_jtjnjtr_direct.cu index fad4fa17..6b67dae7 100644 --- a/test/symforce_function_codegen_test_data/symengine/symforce_caspar_tridiagonal_test/kernel_chain_b_prior_jtjnjtr_direct.cu +++ b/test/symforce_function_codegen_test_data/symengine/symforce_caspar_tridiagonal_test/kernel_chain_b_prior_jtjnjtr_direct.cu @@ -1,17 +1,10 @@ #include #include -#include -#include -#include -#include -#include - +#include "cuda_to_hip.h" #include "kernel_chain_b_prior_jtjnjtr_direct.h" #include "memops.cuh" -namespace cg = cooperative_groups; - namespace caspar { __global__ void __launch_bounds__(1024, 1) diff --git a/test/symforce_function_codegen_test_data/symengine/symforce_caspar_tridiagonal_test/kernel_chain_b_prior_jtjnjtr_direct.h b/test/symforce_function_codegen_test_data/symengine/symforce_caspar_tridiagonal_test/kernel_chain_b_prior_jtjnjtr_direct.h index 34f3b745..16c36b08 100644 --- a/test/symforce_function_codegen_test_data/symengine/symforce_caspar_tridiagonal_test/kernel_chain_b_prior_jtjnjtr_direct.h +++ b/test/symforce_function_codegen_test_data/symengine/symforce_caspar_tridiagonal_test/kernel_chain_b_prior_jtjnjtr_direct.h @@ -1,7 +1,6 @@ #pragma once -#include - +#include "cuda_to_hip.h" #include "shared_indices.h" namespace caspar { diff --git a/test/symforce_function_codegen_test_data/symengine/symforce_caspar_tridiagonal_test/kernel_chain_b_prior_res_jac.cu b/test/symforce_function_codegen_test_data/symengine/symforce_caspar_tridiagonal_test/kernel_chain_b_prior_res_jac.cu index 1d01a9ea..d66f18b7 100644 --- a/test/symforce_function_codegen_test_data/symengine/symforce_caspar_tridiagonal_test/kernel_chain_b_prior_res_jac.cu +++ b/test/symforce_function_codegen_test_data/symengine/symforce_caspar_tridiagonal_test/kernel_chain_b_prior_res_jac.cu @@ -1,17 +1,10 @@ #include #include -#include -#include -#include -#include -#include - +#include "cuda_to_hip.h" #include "kernel_chain_b_prior_res_jac.h" #include "memops.cuh" -namespace cg = cooperative_groups; - namespace caspar { __global__ void __launch_bounds__(1024, 1) diff --git a/test/symforce_function_codegen_test_data/symengine/symforce_caspar_tridiagonal_test/kernel_chain_b_prior_res_jac.h b/test/symforce_function_codegen_test_data/symengine/symforce_caspar_tridiagonal_test/kernel_chain_b_prior_res_jac.h index 434b6f81..8f7ae36d 100644 --- a/test/symforce_function_codegen_test_data/symengine/symforce_caspar_tridiagonal_test/kernel_chain_b_prior_res_jac.h +++ b/test/symforce_function_codegen_test_data/symengine/symforce_caspar_tridiagonal_test/kernel_chain_b_prior_res_jac.h @@ -1,7 +1,6 @@ #pragma once -#include - +#include "cuda_to_hip.h" #include "shared_indices.h" namespace caspar { diff --git a/test/symforce_function_codegen_test_data/symengine/symforce_caspar_tridiagonal_test/kernel_chain_b_prior_res_jac_first.cu b/test/symforce_function_codegen_test_data/symengine/symforce_caspar_tridiagonal_test/kernel_chain_b_prior_res_jac_first.cu index a7df9eb8..244f3fce 100644 --- a/test/symforce_function_codegen_test_data/symengine/symforce_caspar_tridiagonal_test/kernel_chain_b_prior_res_jac_first.cu +++ b/test/symforce_function_codegen_test_data/symengine/symforce_caspar_tridiagonal_test/kernel_chain_b_prior_res_jac_first.cu @@ -1,17 +1,10 @@ #include #include -#include -#include -#include -#include -#include - +#include "cuda_to_hip.h" #include "kernel_chain_b_prior_res_jac_first.h" #include "memops.cuh" -namespace cg = cooperative_groups; - namespace caspar { __global__ void __launch_bounds__(1024, 1) diff --git a/test/symforce_function_codegen_test_data/symengine/symforce_caspar_tridiagonal_test/kernel_chain_b_prior_res_jac_first.h b/test/symforce_function_codegen_test_data/symengine/symforce_caspar_tridiagonal_test/kernel_chain_b_prior_res_jac_first.h index 47af0a26..84f2cef8 100644 --- a/test/symforce_function_codegen_test_data/symengine/symforce_caspar_tridiagonal_test/kernel_chain_b_prior_res_jac_first.h +++ b/test/symforce_function_codegen_test_data/symengine/symforce_caspar_tridiagonal_test/kernel_chain_b_prior_res_jac_first.h @@ -1,7 +1,6 @@ #pragma once -#include - +#include "cuda_to_hip.h" #include "shared_indices.h" namespace caspar { diff --git a/test/symforce_function_codegen_test_data/symengine/symforce_caspar_tridiagonal_test/kernel_chain_b_prior_score.cu b/test/symforce_function_codegen_test_data/symengine/symforce_caspar_tridiagonal_test/kernel_chain_b_prior_score.cu index cb476e7f..db6534c9 100644 --- a/test/symforce_function_codegen_test_data/symengine/symforce_caspar_tridiagonal_test/kernel_chain_b_prior_score.cu +++ b/test/symforce_function_codegen_test_data/symengine/symforce_caspar_tridiagonal_test/kernel_chain_b_prior_score.cu @@ -1,17 +1,10 @@ #include #include -#include -#include -#include -#include -#include - +#include "cuda_to_hip.h" #include "kernel_chain_b_prior_score.h" #include "memops.cuh" -namespace cg = cooperative_groups; - namespace caspar { __global__ void __launch_bounds__(1024, 1) diff --git a/test/symforce_function_codegen_test_data/symengine/symforce_caspar_tridiagonal_test/kernel_chain_b_prior_score.h b/test/symforce_function_codegen_test_data/symengine/symforce_caspar_tridiagonal_test/kernel_chain_b_prior_score.h index 239ad096..2d39b16d 100644 --- a/test/symforce_function_codegen_test_data/symengine/symforce_caspar_tridiagonal_test/kernel_chain_b_prior_score.h +++ b/test/symforce_function_codegen_test_data/symengine/symforce_caspar_tridiagonal_test/kernel_chain_b_prior_score.h @@ -1,7 +1,6 @@ #pragma once -#include - +#include "cuda_to_hip.h" #include "shared_indices.h" namespace caspar { diff --git a/test/symforce_function_codegen_test_data/symengine/symforce_caspar_tridiagonal_test/kernel_chain_coupling_jtjnjtr_direct.cu b/test/symforce_function_codegen_test_data/symengine/symforce_caspar_tridiagonal_test/kernel_chain_coupling_jtjnjtr_direct.cu index 5836bbfc..e315f870 100644 --- a/test/symforce_function_codegen_test_data/symengine/symforce_caspar_tridiagonal_test/kernel_chain_coupling_jtjnjtr_direct.cu +++ b/test/symforce_function_codegen_test_data/symengine/symforce_caspar_tridiagonal_test/kernel_chain_coupling_jtjnjtr_direct.cu @@ -1,17 +1,10 @@ #include #include -#include -#include -#include -#include -#include - +#include "cuda_to_hip.h" #include "kernel_chain_coupling_jtjnjtr_direct.h" #include "memops.cuh" -namespace cg = cooperative_groups; - namespace caspar { __global__ void __launch_bounds__(1024, 1) ChainCouplingJtjnjtrDirectKernel( diff --git a/test/symforce_function_codegen_test_data/symengine/symforce_caspar_tridiagonal_test/kernel_chain_coupling_jtjnjtr_direct.h b/test/symforce_function_codegen_test_data/symengine/symforce_caspar_tridiagonal_test/kernel_chain_coupling_jtjnjtr_direct.h index 0f76b031..2369c40f 100644 --- a/test/symforce_function_codegen_test_data/symengine/symforce_caspar_tridiagonal_test/kernel_chain_coupling_jtjnjtr_direct.h +++ b/test/symforce_function_codegen_test_data/symengine/symforce_caspar_tridiagonal_test/kernel_chain_coupling_jtjnjtr_direct.h @@ -1,7 +1,6 @@ #pragma once -#include - +#include "cuda_to_hip.h" #include "shared_indices.h" namespace caspar { diff --git a/test/symforce_function_codegen_test_data/symengine/symforce_caspar_tridiagonal_test/kernel_chain_coupling_res_jac.cu b/test/symforce_function_codegen_test_data/symengine/symforce_caspar_tridiagonal_test/kernel_chain_coupling_res_jac.cu index 44138dfc..a8770378 100644 --- a/test/symforce_function_codegen_test_data/symengine/symforce_caspar_tridiagonal_test/kernel_chain_coupling_res_jac.cu +++ b/test/symforce_function_codegen_test_data/symengine/symforce_caspar_tridiagonal_test/kernel_chain_coupling_res_jac.cu @@ -1,17 +1,10 @@ #include #include -#include -#include -#include -#include -#include - +#include "cuda_to_hip.h" #include "kernel_chain_coupling_res_jac.h" #include "memops.cuh" -namespace cg = cooperative_groups; - namespace caspar { __global__ void __launch_bounds__(1024, 1) ChainCouplingResJacKernel( diff --git a/test/symforce_function_codegen_test_data/symengine/symforce_caspar_tridiagonal_test/kernel_chain_coupling_res_jac.h b/test/symforce_function_codegen_test_data/symengine/symforce_caspar_tridiagonal_test/kernel_chain_coupling_res_jac.h index 234789c6..857df690 100644 --- a/test/symforce_function_codegen_test_data/symengine/symforce_caspar_tridiagonal_test/kernel_chain_coupling_res_jac.h +++ b/test/symforce_function_codegen_test_data/symengine/symforce_caspar_tridiagonal_test/kernel_chain_coupling_res_jac.h @@ -1,7 +1,6 @@ #pragma once -#include - +#include "cuda_to_hip.h" #include "shared_indices.h" namespace caspar { diff --git a/test/symforce_function_codegen_test_data/symengine/symforce_caspar_tridiagonal_test/kernel_chain_coupling_res_jac_first.cu b/test/symforce_function_codegen_test_data/symengine/symforce_caspar_tridiagonal_test/kernel_chain_coupling_res_jac_first.cu index 38f5df97..30a89d9c 100644 --- a/test/symforce_function_codegen_test_data/symengine/symforce_caspar_tridiagonal_test/kernel_chain_coupling_res_jac_first.cu +++ b/test/symforce_function_codegen_test_data/symengine/symforce_caspar_tridiagonal_test/kernel_chain_coupling_res_jac_first.cu @@ -1,17 +1,10 @@ #include #include -#include -#include -#include -#include -#include - +#include "cuda_to_hip.h" #include "kernel_chain_coupling_res_jac_first.h" #include "memops.cuh" -namespace cg = cooperative_groups; - namespace caspar { __global__ void __launch_bounds__(1024, 1) ChainCouplingResJacFirstKernel( diff --git a/test/symforce_function_codegen_test_data/symengine/symforce_caspar_tridiagonal_test/kernel_chain_coupling_res_jac_first.h b/test/symforce_function_codegen_test_data/symengine/symforce_caspar_tridiagonal_test/kernel_chain_coupling_res_jac_first.h index 6ab04b67..b72d524d 100644 --- a/test/symforce_function_codegen_test_data/symengine/symforce_caspar_tridiagonal_test/kernel_chain_coupling_res_jac_first.h +++ b/test/symforce_function_codegen_test_data/symengine/symforce_caspar_tridiagonal_test/kernel_chain_coupling_res_jac_first.h @@ -1,7 +1,6 @@ #pragma once -#include - +#include "cuda_to_hip.h" #include "shared_indices.h" namespace caspar { diff --git a/test/symforce_function_codegen_test_data/symengine/symforce_caspar_tridiagonal_test/kernel_chain_coupling_score.cu b/test/symforce_function_codegen_test_data/symengine/symforce_caspar_tridiagonal_test/kernel_chain_coupling_score.cu index ef2a7945..c953cb96 100644 --- a/test/symforce_function_codegen_test_data/symengine/symforce_caspar_tridiagonal_test/kernel_chain_coupling_score.cu +++ b/test/symforce_function_codegen_test_data/symengine/symforce_caspar_tridiagonal_test/kernel_chain_coupling_score.cu @@ -1,17 +1,10 @@ #include #include -#include -#include -#include -#include -#include - +#include "cuda_to_hip.h" #include "kernel_chain_coupling_score.h" #include "memops.cuh" -namespace cg = cooperative_groups; - namespace caspar { __global__ void __launch_bounds__(1024, 1) diff --git a/test/symforce_function_codegen_test_data/symengine/symforce_caspar_tridiagonal_test/kernel_chain_coupling_score.h b/test/symforce_function_codegen_test_data/symengine/symforce_caspar_tridiagonal_test/kernel_chain_coupling_score.h index a073d176..138c5066 100644 --- a/test/symforce_function_codegen_test_data/symengine/symforce_caspar_tridiagonal_test/kernel_chain_coupling_score.h +++ b/test/symforce_function_codegen_test_data/symengine/symforce_caspar_tridiagonal_test/kernel_chain_coupling_score.h @@ -1,7 +1,6 @@ #pragma once -#include - +#include "cuda_to_hip.h" #include "shared_indices.h" namespace caspar { diff --git a/test/symforce_function_codegen_test_data/symengine/symforce_caspar_tridiagonal_test/solver.h b/test/symforce_function_codegen_test_data/symengine/symforce_caspar_tridiagonal_test/solver.h index 5a2242d3..e1757944 100644 --- a/test/symforce_function_codegen_test_data/symengine/symforce_caspar_tridiagonal_test/solver.h +++ b/test/symforce_function_codegen_test_data/symengine/symforce_caspar_tridiagonal_test/solver.h @@ -3,8 +3,7 @@ #include #include -#include - +#include "cuda_to_hip.h" #include "shared_indices.h" #include "solver_params.h" From daa1a7bf632d81c76e88814428aa534d2570e9e7 Mon Sep 17 00:00:00 2001 From: Jeff Daily Date: Wed, 4 Nov 2026 19:48:25 +0000 Subject: [PATCH 07/15] [ROCm] Zero r_kp1_norm2 before the first PCG residual update DoUpdateRFirst() zeroes r_0_norm2_tot and then accumulates into both r_0_norm2_tot and r_kp1_norm2_tot, but only DoUpdateR() zeroes r_kp1_norm2_tot. The first PCG iteration therefore adds the new residual norm onto whatever the solver's allocation held. The bad value feeds the PCG exit test and beta, so the result depends on uninitialized device memory. Freshly allocated device memory is usually zero, which hides this. On ROCm, a second solver in the same process gets memory freed by the first, and caspar_tridiagonal_cuda_test failed a varying 5 to 7 of its 20 uncoupled cases on an MI250X: step rejected, 199 PCG iterations. Filling the allocation with 0xFF makes the failures deterministic, and zero-filling it makes them disappear. That narrowed the read to the solver's scalar accumulators. The same defect is reachable on CUDA, so the fix is not gated on HIP. Authored with Claude. Test Plan: ``` python test/symforce_caspar_codegen_test.py --update python test/symforce_caspar_codegen_test.py cmake --build build --target bin.tests.caspar_tridiagonal_cuda_test bin.tests.caspar_solver_cuda_test ./build/bin/tests/caspar_tridiagonal_cuda_test # 5/5 runs pass, also with the allocation filled with 0xFF ./build/bin/tests/caspar_solver_cuda_test ``` --- symforce/caspar/source/templates/solver.cc.jinja | 1 + .../symengine/symforce_caspar_solver_test/solver.cc | 1 + .../symengine/symforce_caspar_tridiagonal_test/solver.cc | 1 + 3 files changed, 3 insertions(+) diff --git a/symforce/caspar/source/templates/solver.cc.jinja b/symforce/caspar/source/templates/solver.cc.jinja index 9d15f5c5..866a31ef 100644 --- a/symforce/caspar/source/templates/solver.cc.jinja +++ b/symforce/caspar/source/templates/solver.cc.jinja @@ -535,6 +535,7 @@ void {{ solver.struct_name }}::DoUpdateStep() { void {{ solver.struct_name }}::DoUpdateRFirst() { Zero({{solver_key("r_0_norm2_tot")}}, {{solver_key("r_0_norm2_tot")}}+1); + Zero({{solver_key("r_kp1_norm2_tot")}}, {{solver_key("r_kp1_norm2_tot")}}+1); {% for nodetype in solver.node_types %} {{nodetype.__name__}}UpdateRFirst( diff --git a/test/symforce_function_codegen_test_data/symengine/symforce_caspar_solver_test/solver.cc b/test/symforce_function_codegen_test_data/symengine/symforce_caspar_solver_test/solver.cc index 1bfb4c03..e910f2c6 100644 --- a/test/symforce_function_codegen_test_data/symengine/symforce_caspar_solver_test/solver.cc +++ b/test/symforce_function_codegen_test_data/symengine/symforce_caspar_solver_test/solver.cc @@ -369,6 +369,7 @@ void GraphSolver::DoUpdateStep() { void GraphSolver::DoUpdateRFirst() { Zero(solver__r_0_norm2_tot_, solver__r_0_norm2_tot_ + 1); + Zero(solver__r_kp1_norm2_tot_, solver__r_kp1_norm2_tot_ + 1); Matrix21UpdateRFirst(nodes__Matrix21__r_k_, Matrix21_num_, nodes__Matrix21__w_, Matrix21_num_, solver__neg_alpha_, nodes__Matrix21__r_k_, Matrix21_num_, diff --git a/test/symforce_function_codegen_test_data/symengine/symforce_caspar_tridiagonal_test/solver.cc b/test/symforce_function_codegen_test_data/symengine/symforce_caspar_tridiagonal_test/solver.cc index 098befc3..e82e6cd0 100644 --- a/test/symforce_function_codegen_test_data/symengine/symforce_caspar_tridiagonal_test/solver.cc +++ b/test/symforce_function_codegen_test_data/symengine/symforce_caspar_tridiagonal_test/solver.cc @@ -683,6 +683,7 @@ void GraphSolver::DoUpdateStep() { void GraphSolver::DoUpdateRFirst() { Zero(solver__r_0_norm2_tot_, solver__r_0_norm2_tot_ + 1); + Zero(solver__r_kp1_norm2_tot_, solver__r_kp1_norm2_tot_ + 1); ChainAUpdateRFirst(nodes__ChainA__r_k_, ChainA_num_, nodes__ChainA__w_, ChainA_num_, solver__neg_alpha_, nodes__ChainA__r_k_, ChainA_num_, solver__r_0_norm2_tot_, From 9ef7a6a3142f2d186ab75517a6ce38dc187844b2 Mon Sep 17 00:00:00 2001 From: Jeff Daily Date: Wed, 4 Nov 2026 19:48:25 +0000 Subject: [PATCH 08/15] [ROCm] Build the Caspar GPU tests with HIP when CUDA is absent The *_cuda_test.cc tests run code checked in by symforce_caspar_codegen_test and are built only when CMake finds a CUDA compiler. When it finds no CUDA compiler but finds a HIP compiler, this builds the same tests for AMD GPUs instead. The runtime, the generated .cu/.cc files and the test sources are compiled as HIP, with USE_HIP defined and hip::host and hip::hipcub linked. The tests include cuda_to_hip.h, which includes the CUDA runtime when USE_HIP is unset, so the CUDA build is unchanged. The build itself needs no AMD GPU: with -DCMAKE_HIP_ARCHITECTURES set, CMake does not query the device, so a ROCm-only machine or container compiles every checked-in kernel and solver with the HIP toolchain. That catches HIP compile breaks from template or runtime changes. Running the tests needs an AMD GPU, as the CUDA tests need an NVIDIA one. HIP language support needs CMake 3.21, so the HIP check is skipped on older CMake. Authored with Claude. Test Plan: On an MI250X (gfx90a), ROCm 7.x: ``` cmake -S . -B build -DCMAKE_HIP_ARCHITECTURES=gfx90a -DCMAKE_PREFIX_PATH= cmake --build build --target bin.tests.caspar_padding_lane_cuda_test bin.tests.caspar_solver_cuda_test bin.tests.caspar_tridiagonal_cuda_test bin.tests.caspar_backend_coverage_cuda_test cd build && ctest -R caspar_ --output-on-failure ``` --- test/CMakeLists.txt | 39 ++++++++++++++++++----- test/caspar_backend_coverage_cuda_test.cc | 2 +- test/caspar_padding_lane_cuda_test.cc | 2 +- test/caspar_solver_cuda_test.cc | 2 +- test/caspar_tridiagonal_cuda_test.cc | 2 +- 5 files changed, 35 insertions(+), 12 deletions(-) diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index 8a34770d..ce003cdd 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -47,7 +47,7 @@ target_compile_definitions( file(GLOB SYMFORCE_TEST_CC_SOURCES CONFIGURE_DEPENDS *_test.cc) -# *_cuda_test.cc tests are built below, only when a CUDA compiler is available. +# *_cuda_test.cc tests are built below, only when a CUDA or HIP compiler is available. file(GLOB SYMFORCE_CUDA_TEST_CC_SOURCES CONFIGURE_DEPENDS *_cuda_test.cc) list(REMOVE_ITEM SYMFORCE_TEST_CC_SOURCES ${SYMFORCE_CUDA_TEST_CC_SOURCES}) @@ -80,9 +80,21 @@ endforeach() include(CheckLanguage) check_language(CUDA) -if(CMAKE_CUDA_COMPILER) - enable_language(CUDA) - find_package(CUDAToolkit REQUIRED) +if(NOT CMAKE_CUDA_COMPILER AND CMAKE_VERSION VERSION_GREATER_EQUAL 3.21) + # Without CUDA, build the same tests for AMD GPUs with HIP. With no AMD GPU on the build host, + # pass -DCMAKE_HIP_ARCHITECTURES (e.g. gfx90a) to build them anyway. + check_language(HIP) +endif() + +if(CMAKE_CUDA_COMPILER OR CMAKE_HIP_COMPILER) + if(CMAKE_CUDA_COMPILER) + enable_language(CUDA) + find_package(CUDAToolkit REQUIRED) + else() + enable_language(HIP) + find_package(hip REQUIRED) + find_package(hipcub REQUIRED) + endif() set(CASPAR_RUNTIME_DIR ${CMAKE_CURRENT_SOURCE_DIR}/../symforce/caspar/source/runtime) set(CASPAR_TEST_DATA_DIR ${CMAKE_CURRENT_SOURCE_DIR}/symforce_function_codegen_test_data/symengine) @@ -92,11 +104,18 @@ if(CMAKE_CUDA_COMPILER) ${CASPAR_RUNTIME_DIR}/sort_indices.cu ${CASPAR_RUNTIME_DIR}/solver_tools.cu ) - set_target_properties(caspar_test_runtime PROPERTIES - CUDA_ARCHITECTURES 75 # Minimum for cooperative_groups::reduce - ) target_include_directories(caspar_test_runtime PUBLIC ${CASPAR_RUNTIME_DIR}) - target_link_libraries(caspar_test_runtime PUBLIC CUDA::cudart) + if(CMAKE_CUDA_COMPILER) + set_target_properties(caspar_test_runtime PROPERTIES + CUDA_ARCHITECTURES 75 # Minimum for cooperative_groups::reduce + ) + target_link_libraries(caspar_test_runtime PUBLIC CUDA::cudart) + else() + get_target_property(CASPAR_RUNTIME_SOURCES caspar_test_runtime SOURCES) + set_source_files_properties(${CASPAR_RUNTIME_SOURCES} PROPERTIES LANGUAGE HIP) + target_compile_definitions(caspar_test_runtime PUBLIC USE_HIP) + target_link_libraries(caspar_test_runtime PUBLIC hip::host hip::hipcub) + endif() # Each caspar__cuda_test.cc runs the code generated into symforce_caspar__test, # except caspar_padding_lane_cuda_test, whose kernel symforce_caspar_codegen_test generates @@ -117,6 +136,10 @@ if(CMAKE_CUDA_COMPILER) file(GLOB DATA_SOURCES CONFIGURE_DEPENDS ${DATA_DIR}/*.cu ${DATA_DIR}/*.cc) add_executable(${TARGET_NAME} ${SOURCE_FILE} ${DATA_SOURCES}) + if(CMAKE_HIP_COMPILER) + # The test and generated .cc files include the HIP runtime too. + set_source_files_properties(${SOURCE_FILE} ${DATA_SOURCES} PROPERTIES LANGUAGE HIP) + endif() set_target_properties(${TARGET_NAME} PROPERTIES CUDA_ARCHITECTURES 75 OUTPUT_NAME ${BINARY_NAME} diff --git a/test/caspar_backend_coverage_cuda_test.cc b/test/caspar_backend_coverage_cuda_test.cc index 71dfabe8..47c2fc7a 100644 --- a/test/caspar_backend_coverage_cuda_test.cc +++ b/test/caspar_backend_coverage_cuda_test.cc @@ -9,9 +9,9 @@ #include #include -#include #include "backend_test_function_expected.h" +#include "cuda_to_hip.h" #include "kernel_backend_test_function.h" namespace { diff --git a/test/caspar_padding_lane_cuda_test.cc b/test/caspar_padding_lane_cuda_test.cc index a7e2ea9a..4848b63a 100644 --- a/test/caspar_padding_lane_cuda_test.cc +++ b/test/caspar_padding_lane_cuda_test.cc @@ -8,8 +8,8 @@ #include #include -#include +#include "cuda_to_hip.h" #include "kernel_caspar_padding_lane_score.h" #include "shared_indices.h" diff --git a/test/caspar_solver_cuda_test.cc b/test/caspar_solver_cuda_test.cc index bce016fe..259b6d9c 100644 --- a/test/caspar_solver_cuda_test.cc +++ b/test/caspar_solver_cuda_test.cc @@ -9,8 +9,8 @@ #include #include -#include +#include "cuda_to_hip.h" #include "solver.h" namespace { diff --git a/test/caspar_tridiagonal_cuda_test.cc b/test/caspar_tridiagonal_cuda_test.cc index 0f303918..0a04c2df 100644 --- a/test/caspar_tridiagonal_cuda_test.cc +++ b/test/caspar_tridiagonal_cuda_test.cc @@ -10,8 +10,8 @@ #include #include #include -#include +#include "cuda_to_hip.h" #include "symforce_caspar_tridiagonal_test/solver.h" namespace { From 92ec5462e1bcf17f42a682a8f9f9bbbdf67b0c5a Mon Sep 17 00:00:00 2001 From: Jeff Daily Date: Wed, 4 Nov 2026 19:50:07 +0000 Subject: [PATCH 09/15] [ROCm] Check the generated Caspar build files in the codegen test symforce_caspar_codegen_test checks in the kernels and solvers caspar generates, but not the CMakeLists.txt it generates next to them. That file holds the only generated code that differs between the CUDA and HIP builds: the USE_HIP branch, which selects the HIP language, finds hip and hipcub, compiles the .cu sources (and, with Python bindings, the pybind sources) as HIP, and sets the HIP math flags. A change to that branch, or to the CUDA one, went unnoticed until someone built a generated library on that backend. test_generate_build_files generates the build files for the padding-lane library with and without Python bindings and compares them with checked-in copies, like the other codegen tests. The copies were written with the test's own --update. Changing the HIP compile flags in CMakeLists.txt.jinja makes the test fail. Authored with Claude. Test Plan: ``` python test/symforce_caspar_codegen_test.py --update python test/symforce_caspar_codegen_test.py SYMFORCE_SYMBOLIC_API=sympy python test/symforce_caspar_codegen_test.py ``` --- test/symforce_caspar_codegen_test.py | 29 +++++ .../CMakeLists.txt | 67 ++++++++++++ .../CMakeLists_python_bindings.txt | 103 ++++++++++++++++++ 3 files changed, 199 insertions(+) create mode 100644 test/symforce_function_codegen_test_data/symengine/symforce_caspar_build_files_test/CMakeLists.txt create mode 100644 test/symforce_function_codegen_test_data/symengine/symforce_caspar_build_files_test/CMakeLists_python_bindings.txt diff --git a/test/symforce_caspar_codegen_test.py b/test/symforce_caspar_codegen_test.py index e7824349..33843535 100644 --- a/test/symforce_caspar_codegen_test.py +++ b/test/symforce_caspar_codegen_test.py @@ -55,6 +55,14 @@ / "symforce_caspar_backend_coverage_test" ) +BUILD_FILES_TEST_DATA_DIR = ( + path_util.symforce_data_root(__file__) + / "test" + / "symforce_function_codegen_test_data" + / symforce.get_symbolic_api() + / "symforce_caspar_build_files_test" +) + KERNEL_FILENAMES = ( "kernel_caspar_padding_lane_score.cu", "kernel_caspar_padding_lane_score.h", @@ -326,6 +334,27 @@ def test_generate_backend_coverage_kernel(self) -> None: self.compare_or_update_directory(kernel_dir, BACKEND_COVERAGE_TEST_DATA_DIR) + @symengine_only + def test_generate_build_files(self) -> None: + """ + Keep the generated CMakeLists.txt up to date, with and without Python bindings. It holds + the only generated code that differs between the CUDA and HIP builds. + """ + output_dir = self.make_output_dir("sf_caspar_build_files_") + caslib = _make_padding_lane_library() + + build_files_dir = output_dir / "build_files" + build_files_dir.mkdir() + for python_bindings in (False, True): + generated_dir = output_dir / f"python_bindings_{python_bindings}" + generated_dir.mkdir() + caslib.generate_buildfiles(generated_dir, python_bindings=python_bindings) + for path in generated_dir.iterdir(): + suffix = "_python_bindings" if python_bindings else "" + (build_files_dir / f"{path.stem}{suffix}{path.suffix}").write_text(path.read_text()) + + self.compare_or_update_directory(build_files_dir, BUILD_FILES_TEST_DATA_DIR) + @symengine_only def test_generate_solver(self) -> None: """ diff --git a/test/symforce_function_codegen_test_data/symengine/symforce_caspar_build_files_test/CMakeLists.txt b/test/symforce_function_codegen_test_data/symengine/symforce_caspar_build_files_test/CMakeLists.txt new file mode 100644 index 00000000..c4d72201 --- /dev/null +++ b/test/symforce_function_codegen_test_data/symengine/symforce_caspar_build_files_test/CMakeLists.txt @@ -0,0 +1,67 @@ +cmake_minimum_required(VERSION 3.18) + +set(CMAKE_CXX_STANDARD 17) +set(CMAKE_CXX_STANDARD_REQUIRED ON) + +option(USE_HIP "Build with HIP for AMD GPUs" OFF) + +if(USE_HIP) + # project(... LANGUAGES ... HIP) honors -DCMAKE_HIP_ARCHITECTURES, otherwise + # auto-detects the host GPU and errors on a no-GPU build host. + project(caspar_library LANGUAGES CXX HIP) + + find_package(hip REQUIRED) + find_package(hipcub REQUIRED) +else() + if(NOT DEFINED CMAKE_CUDA_COMPILER AND DEFINED ENV{CUDACXX}) + set(CMAKE_CUDA_COMPILER "$ENV{CUDACXX}" CACHE FILEPATH "CUDA compiler") + endif() + + project(caspar_library LANGUAGES CXX CUDA) + + set(CASPAR_MIN_ARCH 75 CACHE INTERNAL "Minimum CUDA arch required by Caspar") + + # SASS for Turing/Ampere/Ada (sm_75–89) + PTX fallback for forward JIT-compatibility. sm_75 minimum required by cooperative_groups::reduce. + if(NOT DEFINED CMAKE_CUDA_ARCHITECTURES) + set(CMAKE_CUDA_ARCHITECTURES ${CASPAR_MIN_ARCH} 80-real 86-real 89-real) + endif() + + find_package(CUDAToolkit REQUIRED) +endif() + +file(GLOB CUDA_SOURCES RELATIVE ${CMAKE_CURRENT_SOURCE_DIR} "*.cu" "*.cc") +list(FILTER CUDA_SOURCES EXCLUDE REGEX "pybind.*") +file(GLOB CUDA_HEADERS RELATIVE ${CMAKE_CURRENT_SOURCE_DIR} "*.cuh" "*.h") +list(FILTER CUDA_HEADERS EXCLUDE REGEX "pybind.*") + +add_library(caspar_lib_core STATIC ${CUDA_SOURCES} ${CUDA_HEADERS}) +target_include_directories(caspar_lib_core PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}) + +if(USE_HIP) + # Mark .cu files as HIP language + set_source_files_properties(${CUDA_SOURCES} PROPERTIES LANGUAGE HIP) + target_compile_definitions(caspar_lib_core PUBLIC USE_HIP) + # Link hip targets PRIVATE to avoid propagating HIP compile options to pure-CXX consumers. + # The static library has the HIP code linked in; pybind consumers only need the includes. + target_link_libraries(caspar_lib_core PRIVATE hip::host hip::hipcub) + # Expose ROCm include directory for consumers that need hipError_t etc + target_include_directories(caspar_lib_core PUBLIC ${hip_INCLUDE_DIRS}) + # Use -ffp-contract=fast -fno-math-errno instead of -ffast-math. + # clang's -ffast-math enables -fassociative-math which can NaN backward passes. + target_compile_options(caspar_lib_core PRIVATE + $<$:-ffp-contract=fast -fno-math-errno> + ) + set_target_properties(caspar_lib_core PROPERTIES + HIP_ARCHITECTURES "${CMAKE_HIP_ARCHITECTURES}" + ) +else() + target_link_libraries(caspar_lib_core PUBLIC CUDA::cudart) + target_compile_options(caspar_lib_core PRIVATE + $<$:--use_fast_math> + ) +endif() + +set_target_properties(caspar_lib_core PROPERTIES + POSITION_INDEPENDENT_CODE ON +) + diff --git a/test/symforce_function_codegen_test_data/symengine/symforce_caspar_build_files_test/CMakeLists_python_bindings.txt b/test/symforce_function_codegen_test_data/symengine/symforce_caspar_build_files_test/CMakeLists_python_bindings.txt new file mode 100644 index 00000000..6bb71efd --- /dev/null +++ b/test/symforce_function_codegen_test_data/symengine/symforce_caspar_build_files_test/CMakeLists_python_bindings.txt @@ -0,0 +1,103 @@ +cmake_minimum_required(VERSION 3.18) + +set(CMAKE_CXX_STANDARD 17) +set(CMAKE_CXX_STANDARD_REQUIRED ON) + +option(USE_HIP "Build with HIP for AMD GPUs" OFF) + +if(USE_HIP) + # project(... LANGUAGES ... HIP) honors -DCMAKE_HIP_ARCHITECTURES, otherwise + # auto-detects the host GPU and errors on a no-GPU build host. + project(caspar_library LANGUAGES CXX HIP) + + find_package(hip REQUIRED) + find_package(hipcub REQUIRED) +else() + if(NOT DEFINED CMAKE_CUDA_COMPILER AND DEFINED ENV{CUDACXX}) + set(CMAKE_CUDA_COMPILER "$ENV{CUDACXX}" CACHE FILEPATH "CUDA compiler") + endif() + + project(caspar_library LANGUAGES CXX CUDA) + + set(CASPAR_MIN_ARCH 75 CACHE INTERNAL "Minimum CUDA arch required by Caspar") + + # SASS for Turing/Ampere/Ada (sm_75–89) + PTX fallback for forward JIT-compatibility. sm_75 minimum required by cooperative_groups::reduce. + if(NOT DEFINED CMAKE_CUDA_ARCHITECTURES) + set(CMAKE_CUDA_ARCHITECTURES ${CASPAR_MIN_ARCH} 80-real 86-real 89-real) + endif() + + find_package(CUDAToolkit REQUIRED) +endif() + +option(CASPAR_BUILD_PYTHON_BINDINGS "Build Python bindings via pybind11" ON) + +if(CASPAR_BUILD_PYTHON_BINDINGS) + include(FetchContent) + find_package(pybind11 2.13.6 QUIET) + if(NOT pybind11_FOUND) + message(STATUS "pybind11 not found, adding with FetchContent") + FetchContent_Declare( + pybind11 + URL https://github.com/pybind/pybind11/archive/v2.13.6.zip + URL_HASH SHA256=d0a116e91f64a4a2d8fb7590c34242df92258a61ec644b79127951e821b47be6 + DOWNLOAD_EXTRACT_TIMESTAMP TRUE + ) + FetchContent_MakeAvailable(pybind11) + else() + message(STATUS "pybind11 found") + endif() +endif() + +file(GLOB CUDA_SOURCES RELATIVE ${CMAKE_CURRENT_SOURCE_DIR} "*.cu" "*.cc") +list(FILTER CUDA_SOURCES EXCLUDE REGEX "pybind.*") +file(GLOB CUDA_HEADERS RELATIVE ${CMAKE_CURRENT_SOURCE_DIR} "*.cuh" "*.h") +list(FILTER CUDA_HEADERS EXCLUDE REGEX "pybind.*") + +add_library(caspar_lib_core STATIC ${CUDA_SOURCES} ${CUDA_HEADERS}) +target_include_directories(caspar_lib_core PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}) + +if(USE_HIP) + # Mark .cu files as HIP language + set_source_files_properties(${CUDA_SOURCES} PROPERTIES LANGUAGE HIP) + target_compile_definitions(caspar_lib_core PUBLIC USE_HIP) + # Link hip targets PRIVATE to avoid propagating HIP compile options to pure-CXX consumers. + # The static library has the HIP code linked in; pybind consumers only need the includes. + target_link_libraries(caspar_lib_core PRIVATE hip::host hip::hipcub) + # Expose ROCm include directory for consumers that need hipError_t etc + target_include_directories(caspar_lib_core PUBLIC ${hip_INCLUDE_DIRS}) + # Use -ffp-contract=fast -fno-math-errno instead of -ffast-math. + # clang's -ffast-math enables -fassociative-math which can NaN backward passes. + target_compile_options(caspar_lib_core PRIVATE + $<$:-ffp-contract=fast -fno-math-errno> + ) + set_target_properties(caspar_lib_core PROPERTIES + HIP_ARCHITECTURES "${CMAKE_HIP_ARCHITECTURES}" + ) +else() + target_link_libraries(caspar_lib_core PUBLIC CUDA::cudart) + target_compile_options(caspar_lib_core PRIVATE + $<$:--use_fast_math> + ) +endif() + +set_target_properties(caspar_lib_core PROPERTIES + POSITION_INDEPENDENT_CODE ON +) + +if(CASPAR_BUILD_PYTHON_BINDINGS) + file(GLOB CPP_SOURCES "pybind*.cc") + + pybind11_add_module(caspar_lib ${CPP_SOURCES} ${CPP_HEADERS}) + target_link_libraries(caspar_lib PRIVATE caspar_lib_core) + + if(USE_HIP) + # Compile pybind sources as HIP (so HIP headers work) but they have no device code. + # The HIP compiler handles them as host-only code. + set_source_files_properties(${CPP_SOURCES} PROPERTIES LANGUAGE HIP) + endif() + + set_target_properties(caspar_lib PROPERTIES + LIBRARY_OUTPUT_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} + RUNTIME_OUTPUT_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} + ) +endif() From 1ee93b2ccc1145a5d1076db8edff0061f97c4aa5 Mon Sep 17 00:00:00 2001 From: Jeff Daily Date: Wed, 4 Nov 2026 19:55:05 +0000 Subject: [PATCH 10/15] [ROCm] Keep device-only CUDA headers out of host compiles The generated kernel and solver headers include cuda_to_hip.h, and so does host C++ that uses them: the generated solver.cc, the Python bindings, and the Caspar GPU tests, all compiled by the host compiler in a CUDA build. On the CUDA path the header also included cooperative_groups and CUB, which only compile under nvcc. A g++ build of caspar_padding_lane_cuda_test failed in cooperative_groups/details/sync.h ('threadIdx' was not declared). Those includes, the cg alias and CG_REDUCE_SUM are only used by device code, so they are now included only when __CUDACC__ is defined. Host-only translation units get just the CUDA runtime API, which is what they included before the HIP port. The HIP path is unchanged: every file that includes the header is compiled as HIP there. Authored with Claude. Test Plan: CUDA 12.8 (no NVIDIA GPU on this machine, so compile and link only): ``` cmake -S . -B build_cuda -DCMAKE_CUDA_COMPILER=/path/to/cuda-12.8/bin/nvcc cmake --build build_cuda --target bin.tests.caspar_padding_lane_cuda_test bin.tests.caspar_solver_cuda_test bin.tests.caspar_tridiagonal_cuda_test bin.tests.caspar_backend_coverage_cuda_test ``` HIP, on an MI250X (gfx90a): ``` cmake --build build --target bin.tests.caspar_padding_lane_cuda_test bin.tests.caspar_solver_cuda_test bin.tests.caspar_tridiagonal_cuda_test bin.tests.caspar_backend_coverage_cuda_test ctest --test-dir build -R caspar_ ``` --- symforce/caspar/source/runtime/cuda_to_hip.h | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/symforce/caspar/source/runtime/cuda_to_hip.h b/symforce/caspar/source/runtime/cuda_to_hip.h index e9141b2c..2058cfd3 100644 --- a/symforce/caspar/source/runtime/cuda_to_hip.h +++ b/symforce/caspar/source/runtime/cuda_to_hip.h @@ -128,6 +128,9 @@ __device__ __forceinline__ T labeled_reduce_sum(GroupT group, T val, LabelT labe #else // CUDA path #include + +// Host-only translation units (e.g. code using the generated headers) get just the runtime API. +#if defined(__CUDACC__) #include #include #include @@ -138,5 +141,6 @@ namespace cg = cooperative_groups; // On CUDA, cg::reduce is available #define CG_REDUCE_SUM(group, val) cg::reduce(group, val, cg::plus()) +#endif #endif From d5eb335dbe09b2f83de75fd0aa5e437d8acbacc0 Mon Sep 17 00:00:00 2001 From: Jeff Daily Date: Wed, 4 Nov 2026 20:02:17 +0000 Subject: [PATCH 11/15] [ROCm] Apply clang-format to cuda_to_hip.h make lint runs clang-format --dry-run -Werror over the tree, and the HIP compatibility header was not formatted with the repository's .clang-format. Formatting only; no code change. Authored with Claude. Test Plan: ``` make lint ``` --- symforce/caspar/source/runtime/cuda_to_hip.h | 41 ++++++++++---------- 1 file changed, 21 insertions(+), 20 deletions(-) diff --git a/symforce/caspar/source/runtime/cuda_to_hip.h b/symforce/caspar/source/runtime/cuda_to_hip.h index 2058cfd3..c9c763bf 100644 --- a/symforce/caspar/source/runtime/cuda_to_hip.h +++ b/symforce/caspar/source/runtime/cuda_to_hip.h @@ -13,29 +13,29 @@ #if defined(USE_HIP) || defined(__HIP_PLATFORM_AMD__) -#include #include +#include #include // CUDA runtime API -> HIP runtime API -#define cudaMalloc hipMalloc -#define cudaFree hipFree -#define cudaMemset hipMemset -#define cudaMemcpy hipMemcpy +#define cudaMalloc hipMalloc +#define cudaFree hipFree +#define cudaMemset hipMemset +#define cudaMemcpy hipMemcpy #define cudaMemcpyDeviceToDevice hipMemcpyDeviceToDevice -#define cudaMemcpyDeviceToHost hipMemcpyDeviceToHost -#define cudaMemcpyHostToDevice hipMemcpyHostToDevice -#define cudaError_t hipError_t -#define cudaSuccess hipSuccess -#define cudaStream_t hipStream_t -#define cudaDeviceSynchronize hipDeviceSynchronize -#define cudaGetLastError hipGetLastError -#define cudaGetErrorString hipGetErrorString -#define cudaSetDevice hipSetDevice -#define cudaGetDevice hipGetDevice +#define cudaMemcpyDeviceToHost hipMemcpyDeviceToHost +#define cudaMemcpyHostToDevice hipMemcpyHostToDevice +#define cudaError_t hipError_t +#define cudaSuccess hipSuccess +#define cudaStream_t hipStream_t +#define cudaDeviceSynchronize hipDeviceSynchronize +#define cudaGetLastError hipGetLastError +#define cudaGetErrorString hipGetErrorString +#define cudaSetDevice hipSetDevice +#define cudaGetDevice hipGetDevice #define cudaPointerGetAttributes hipPointerGetAttributes -#define cudaPointerAttributes hipPointerAttribute_t -#define cudaGetDeviceCount hipGetDeviceCount +#define cudaPointerAttributes hipPointerAttribute_t +#define cudaGetDeviceCount hipGetDeviceCount #define cudaFuncAttributeMaxDynamicSharedMemorySize hipFuncAttributeMaxDynamicSharedMemorySize // hipFuncSetAttribute takes the kernel as const void*; CUDA also accepts the kernel itself. @@ -88,7 +88,8 @@ __device__ __forceinline__ unsigned long long match_any_mask(GroupT group, Label // and have exactly one lane per unique label perform the atomic. // Returns the reduced value and sets is_leader=true for exactly one lane per label. template -__device__ __forceinline__ T labeled_reduce_sum(GroupT group, T val, LabelT label, bool& is_leader) { +__device__ __forceinline__ T labeled_reduce_sum(GroupT group, T val, LabelT label, + bool& is_leader) { // Get mask of lanes with same label unsigned long long same_label_mask = group.match_any(label); @@ -123,7 +124,7 @@ __device__ __forceinline__ T labeled_reduce_sum(GroupT group, T val, LabelT labe // Macros to replace cg::reduce calls (used in device code) #define CG_REDUCE_SUM(group, val) caspar_hip::reduce_sum(group, val) #define CG_LABELED_REDUCE_SUM(group, val, label, is_leader) \ - caspar_hip::labeled_reduce_sum(group, val, label, is_leader) + caspar_hip::labeled_reduce_sum(group, val, label, is_leader) #else // CUDA path @@ -133,8 +134,8 @@ __device__ __forceinline__ T labeled_reduce_sum(GroupT group, T val, LabelT labe #if defined(__CUDACC__) #include #include -#include #include +#include #include namespace cg = cooperative_groups; From f82e4abfac7f1081254dfa781b0a692d47e2a0cf Mon Sep 17 00:00:00 2001 From: Jeff Daily Date: Wed, 4 Nov 2026 20:39:59 +0000 Subject: [PATCH 12/15] [ROCm] Remove unused reduction helpers from cuda_to_hip.h The ROCm branch of cuda_to_hip.h defined caspar_hip::reduce_sum, reduce_max, match_any_mask and labeled_reduce_sum, plus the CG_REDUCE_SUM and CG_LABELED_REDUCE_SUM macros, and the CUDA branch defined its own CG_REDUCE_SUM. Nothing in the tree calls any of them: memops.cuh and shared_indices.cu do their HIP reductions and copies inline. labeled_reduce_sum was also wrong when lanes with the same label are not contiguous, because its xor butterfly only adds a partner with the same label, so lanes 0 and 3 sharing a label never combine. Remove them so the header carries only the runtime API mapping. The cg namespace alias stays on the CUDA path. Templates and macros that are never instantiated emit no code, so the device code objects of the Caspar test binaries are unchanged. Authored with Claude. Test Plan: ``` grep -rn 'caspar_hip\|CG_REDUCE_SUM\|CG_LABELED_REDUCE_SUM' symforce test cmake -S . -B build -DCMAKE_BUILD_TYPE=Release -DCMAKE_HIP_ARCHITECTURES=gfx90a cmake --build build ctest --test-dir build -R caspar cmake -S . -B build-cuda -DCMAKE_CUDA_COMPILER=nvcc cmake --build build-cuda --target bin.tests.caspar_padding_lane_cuda_test bin.tests.caspar_solver_cuda_test bin.tests.caspar_tridiagonal_cuda_test bin.tests.caspar_backend_coverage_cuda_test ``` --- symforce/caspar/source/runtime/cuda_to_hip.h | 78 -------------------- 1 file changed, 78 deletions(-) diff --git a/symforce/caspar/source/runtime/cuda_to_hip.h b/symforce/caspar/source/runtime/cuda_to_hip.h index c9c763bf..a8d959f8 100644 --- a/symforce/caspar/source/runtime/cuda_to_hip.h +++ b/symforce/caspar/source/runtime/cuda_to_hip.h @@ -51,81 +51,6 @@ inline hipError_t cudaFuncSetAttribute(T* func, hipFuncAttribute attr, int value // CUB -> hipCUB namespace namespace cub = hipcub; -// Cooperative groups: HIP has cg basics but lacks cg::reduce, cg::labeled_partition, memcpy_async. -// Provide manual implementations where needed. - -namespace caspar_hip { - -// Butterfly reduction within a cooperative group (thread_block_tile or coalesced_group). -// Replaces cg::reduce(group, val, cg::plus()). -template -__device__ __forceinline__ T reduce_sum(GroupT group, T val) { - for (unsigned int offset = group.size() / 2; offset > 0; offset >>= 1) { - val += group.shfl_xor(val, offset); - } - return val; -} - -// Butterfly reduction for max (cg::greater). -template -__device__ __forceinline__ T reduce_max(GroupT group, T val) { - for (unsigned int offset = group.size() / 2; offset > 0; offset >>= 1) { - T other = group.shfl_xor(val, offset); - val = (val > other) ? val : other; - } - return val; -} - -// Match_any: return a mask of lanes in the tile that have the same label. -// HIP CG has match_any() for coalesced groups. -template -__device__ __forceinline__ unsigned long long match_any_mask(GroupT group, LabelT label) { - // HIP cooperative_groups::coalesced_group has match_any - return group.match_any(label); -} - -// Labeled partition emulation: reduce values within lanes sharing the same label, -// and have exactly one lane per unique label perform the atomic. -// Returns the reduced value and sets is_leader=true for exactly one lane per label. -template -__device__ __forceinline__ T labeled_reduce_sum(GroupT group, T val, LabelT label, - bool& is_leader) { - // Get mask of lanes with same label - unsigned long long same_label_mask = group.match_any(label); - - // Find my position within the matching lanes - unsigned int my_lane = group.thread_rank(); - unsigned long long lower_mask = (1ULL << my_lane) - 1; - unsigned int rank_in_label = __popcll(same_label_mask & lower_mask); - - // Leader is the lowest-numbered lane in the group - is_leader = (rank_in_label == 0); - - // Count total lanes with this label - unsigned int label_size = __popcll(same_label_mask); - - // Butterfly reduction over the masked lanes - // For each reduction step, exchange with lane at offset if both are in same_label_mask - T result = val; - for (unsigned int offset = 1; offset < group.size(); offset <<= 1) { - unsigned int partner_lane = my_lane ^ offset; - bool partner_has_same_label = (same_label_mask >> partner_lane) & 1ULL; - T partner_val = group.shfl_xor(result, offset); - if (partner_has_same_label && partner_lane < group.size()) { - result += partner_val; - } - } - - return result; -} - -} // namespace caspar_hip - -// Macros to replace cg::reduce calls (used in device code) -#define CG_REDUCE_SUM(group, val) caspar_hip::reduce_sum(group, val) -#define CG_LABELED_REDUCE_SUM(group, val, label, is_leader) \ - caspar_hip::labeled_reduce_sum(group, val, label, is_leader) - #else // CUDA path #include @@ -139,9 +64,6 @@ __device__ __forceinline__ T labeled_reduce_sum(GroupT group, T val, LabelT labe #include namespace cg = cooperative_groups; - -// On CUDA, cg::reduce is available -#define CG_REDUCE_SUM(group, val) cg::reduce(group, val, cg::plus()) #endif #endif From e7e0b00c5e343e57724226aca38b92438868cc6b Mon Sep 17 00:00:00 2001 From: Jeff Daily Date: Wed, 4 Nov 2026 21:04:07 +0000 Subject: [PATCH 13/15] [ROCm] Fix the Caspar README HIP example and a stale comment The README's AMD GPU section called a compile_caspar_library() function that does not exist. The API is the CasparLibrary.compile() static method, so the example now calls caslib.compile(out_dir, use_hip=True, hip_arch=...) after caslib.generate(out_dir), as the examples do. The same section also described the compatibility header wrongly. It does not map __syncthreads or any cooperative groups spelling; it maps the CUDA runtime calls Caspar emits and the cub namespace. It also said hipcc works as the compiler, but the generated build uses CMake's HIP language support, which rejects the hipcc wrapper and needs the ROCm Clang compiler. The text now says so, and notes that without hip_arch CMake builds for the GPU it detects. The comment above the HIP branch of GetDeviceId() in pybind_array_tools.cc still mentioned __device__ helpers in cuda_to_hip.h, which were removed, and local aliases the code never defines. It now says what the code does. No code change outside the comment. Authored with Claude. Test Plan: ``` cmake -S . -B build -DCMAKE_BUILD_TYPE=Release -DCMAKE_HIP_ARCHITECTURES=gfx90a cmake --build build ctest --test-dir build -R caspar --output-on-failure ctest --test-dir build python symforce/caspar/examples/kernel_example/gen_and_run.py # with caslib.compile(out_dir, use_hip=True, hip_arch="gfx90a") ``` --- symforce/caspar/README.md | 7 ++++--- symforce/caspar/source/runtime/pybind_array_tools.cc | 5 ++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/symforce/caspar/README.md b/symforce/caspar/README.md index 58c50f2d..e96c850c 100644 --- a/symforce/caspar/README.md +++ b/symforce/caspar/README.md @@ -112,15 +112,16 @@ The following access patterns are currently supported: ### AMD GPUs (ROCm/HIP) -Caspar also runs on AMD GPUs through ROCm/HIP. The same generated kernels and runtime compile against the HIP toolchain via a small `cuda_to_hip.h` compatibility header that maps the CUDA spellings Caspar emits (`cudaMalloc`, `__syncthreads`, cooperative groups, CUB primitives, and friends) onto their HIP equivalents, so the symbolic kernel definitions are unchanged. +Caspar also runs on AMD GPUs through ROCm/HIP. The generated kernels and runtime compile with HIP through a small `cuda_to_hip.h` compatibility header that maps the CUDA runtime calls Caspar emits (`cudaMalloc`, `cudaMemcpy`, `cudaFuncSetAttribute`, and friends) and the `cub` namespace onto their HIP equivalents, so the symbolic kernel definitions are unchanged. To build a generated library for AMD GPUs, enable the HIP path when compiling: ```python -compile_caspar_library(caslib, output_dir, use_hip=True, hip_arch="gfx90a") +caslib.generate(out_dir) +caslib.compile(out_dir, use_hip=True, hip_arch="gfx90a") ``` -Set the target AMD GPU via the `hip_arch` argument, for example `gfx90a` for CDNA2 (MI200) or `gfx1100` for RDNA3. The ROCm build needs a HIP-enabled compiler (`hipcc`/`amdclang++`) and the `hip` and `hipcub` packages from a ROCm installation. When `use_hip` is off the build is unchanged and continues to use CUDA. +Set the target AMD GPU via the `hip_arch` argument, for example `gfx90a` for CDNA2 (MI200) or `gfx1100` for RDNA3. Without it, CMake builds for the GPU it detects on the build machine. The build uses CMake's HIP language support, so it needs the ROCm Clang compiler (`amdclang++`; CMake does not accept the `hipcc` wrapper) and the `hip` and `hipcub` packages from a ROCm installation. When `use_hip` is off the build is unchanged and continues to use CUDA. ### Etymology Caspar, an acronym for **C**UDA **A**ccelerator for **S**ymbolic **P**rogramming with **A**daptive **R**eordering, is named after the Danish–Norwegian mathematician [Caspar Wessel](https://en.wikipedia.org/wiki/Caspar_Wessel). Wessel was the first to describe the geometrical interpretation of complex numbers as points in the complex plane and as vectors. However, since his thesis was written in Danish, it initially received little recognition. When his work was rediscovered later, the mathematician Sophus Lie, known for his discovery of Lie algebra, wrote the following in the newspaper: diff --git a/symforce/caspar/source/runtime/pybind_array_tools.cc b/symforce/caspar/source/runtime/pybind_array_tools.cc index 7f057905..db221163 100644 --- a/symforce/caspar/source/runtime/pybind_array_tools.cc +++ b/symforce/caspar/source/runtime/pybind_array_tools.cc @@ -174,9 +174,8 @@ int GetDeviceId(const py::object& obj) { auto interface = obj.attr("__cuda_array_interface__").cast(); auto data = interface["data"].cast(); void* ptr = reinterpret_cast(data[0].cast()); - // This is a host translation unit, so it uses local cuda*->hip* aliases here - // rather than the device-side cuda_to_hip.h (which pulls in hipcub and - // __device__ helpers intended for the .cu sources). + // This host translation unit calls the HIP runtime directly rather than + // including cuda_to_hip.h, which also pulls in hipcub. #if defined(USE_HIP) || defined(__HIP_PLATFORM_AMD__) hipPointerAttribute_t attrs; hipError_t err = hipPointerGetAttributes(&attrs, ptr); From aa31e636e72ba79fc676a04657fdd8bb6bf3c91f Mon Sep 17 00:00:00 2001 From: Jeff Daily Date: Wed, 18 Nov 2026 03:58:56 +0000 Subject: [PATCH 14/15] [ROCm] Keep device-only HIP headers out of host compiles The HIP branch of cuda_to_hip.h included hip_cooperative_groups.h and hipcub/hipcub.hpp, and declared namespace cub = hipcub, unconditionally. Every generated kernel.h, caspar_mappings.h and solver.h includes cuda_to_hip.h, so a host translation unit built with the host C++ compiler that includes a generated header fails inside rocPRIM, for example with g++: "'__builtin_amdgcn_wavefrontsize' was not declared in this scope". A project that links a generated Caspar library and includes solver.h from its own .cc files cannot build. The CUDA branch already keeps cooperative groups and CUB behind __CUDACC__; the HIP branch now does the same behind __HIPCC__, which clang defines when it compiles HIP and the host compiler does not. Host code still gets the HIP runtime API and the cuda* spellings. The CUDA branch is unchanged. The HIP build of the *_cuda_test.cc tests compiled every test and generated .cc file as HIP, which is why it never saw this. They now stay with the host C++ compiler, as in the CUDA build, so caspar_solver_cuda_test.cc, caspar_tridiagonal_cuda_test.cc and both generated solver.cc files check that solver.h is includable from host code. Linking hip::hipcub would pass its hip::device options (-x hip, --offload-arch) to those host sources too, so the test runtime takes only the hipCUB and rocPRIM headers, which is all it needs. Without the header change, the six host objects of the four test targets fail to compile with the error above; with it, all four build and pass on an MI250X. Authored with Claude. Test Plan: ``` cmake -S . -B build -DCMAKE_BUILD_TYPE=Release -DCMAKE_HIP_ARCHITECTURES=gfx90a cmake --build build ctest --test-dir build -R caspar --output-on-failure ctest --test-dir build ``` --- symforce/caspar/source/runtime/cuda_to_hip.h | 8 +++++++- test/CMakeLists.txt | 15 ++++++++++++--- 2 files changed, 19 insertions(+), 4 deletions(-) diff --git a/symforce/caspar/source/runtime/cuda_to_hip.h b/symforce/caspar/source/runtime/cuda_to_hip.h index a8d959f8..122b886b 100644 --- a/symforce/caspar/source/runtime/cuda_to_hip.h +++ b/symforce/caspar/source/runtime/cuda_to_hip.h @@ -13,9 +13,13 @@ #if defined(USE_HIP) || defined(__HIP_PLATFORM_AMD__) -#include #include + +// Host-only translation units (e.g. code using the generated headers) get just the runtime API. +#if defined(__HIPCC__) +#include #include +#endif // CUDA runtime API -> HIP runtime API #define cudaMalloc hipMalloc @@ -48,8 +52,10 @@ inline hipError_t cudaFuncSetAttribute(T* func, hipFuncAttribute attr, int value // so atomicAdd_block is equivalent to atomicAdd on shared memory. #define atomicAdd_block atomicAdd +#if defined(__HIPCC__) // CUB -> hipCUB namespace namespace cub = hipcub; +#endif #else // CUDA path diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index ce003cdd..3d1892f4 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -114,7 +114,12 @@ if(CMAKE_CUDA_COMPILER OR CMAKE_HIP_COMPILER) get_target_property(CASPAR_RUNTIME_SOURCES caspar_test_runtime SOURCES) set_source_files_properties(${CASPAR_RUNTIME_SOURCES} PROPERTIES LANGUAGE HIP) target_compile_definitions(caspar_test_runtime PUBLIC USE_HIP) - target_link_libraries(caspar_test_runtime PUBLIC hip::host hip::hipcub) + # hip::hipcub also carries hip::device, whose -x hip would reach the host-compiled .cc test + # sources below. hipCUB and rocPRIM are header-only, so take just their headers. + target_link_libraries(caspar_test_runtime PUBLIC hip::host roc::rocprim) + target_include_directories(caspar_test_runtime PUBLIC + $ + ) endif() # Each caspar__cuda_test.cc runs the code generated into symforce_caspar__test, @@ -137,8 +142,12 @@ if(CMAKE_CUDA_COMPILER OR CMAKE_HIP_COMPILER) add_executable(${TARGET_NAME} ${SOURCE_FILE} ${DATA_SOURCES}) if(CMAKE_HIP_COMPILER) - # The test and generated .cc files include the HIP runtime too. - set_source_files_properties(${SOURCE_FILE} ${DATA_SOURCES} PROPERTIES LANGUAGE HIP) + # Only the .cu files are HIP. The test and generated .cc files stay with the host C++ + # compiler, as in the CUDA build, so the generated headers are checked to be includable + # from host code. + set(HIP_SOURCES ${DATA_SOURCES}) + list(FILTER HIP_SOURCES INCLUDE REGEX "\\.cu$") + set_source_files_properties(${HIP_SOURCES} PROPERTIES LANGUAGE HIP) endif() set_target_properties(${TARGET_NAME} PROPERTIES CUDA_ARCHITECTURES 75 From c665e5cc723ccefd02e3505b0adeabc99cdb4838 Mon Sep 17 00:00:00 2001 From: Jeff Daily Date: Wed, 18 Nov 2026 04:49:22 +0000 Subject: [PATCH 15/15] [ROCm] Fix host compile of generated solver.cc on Windows The HIP test build compiles the generated solver.cc files with the host C++ compiler, as the CUDA build does. On Windows (clang++ over the MSVC STL and UCRT) both failed on names that only glibc and libstdc++ supply: sort_indices.h used uint, a glibc typedef it took from , and solver.cc called std::to_string without including . Compiled as HIP they had built, because the HIP headers supply both. sort_indices.h now spells the type unsigned int and includes for size_t. The solver.cc template includes , and also for printf and for std::swap, which it used without including either; the checked-in generated solvers are regenerated from it. The changes are type spellings and includes only: uint is unsigned int wherever it exists, so the declarations, their mangled names and the CUDA build are unchanged. To check on Linux, compile the generated solver.cc with uint, ushort and ulong defined to undeclared names after (which hides the glibc typedefs as Windows lacks them): both files fail before this change and build after it. Authored with Claude. Test Plan: ``` cmake -S . -B build -DCMAKE_BUILD_TYPE=Release -DCMAKE_HIP_ARCHITECTURES=gfx90a cmake --build build ctest --test-dir build -R caspar --output-on-failure ctest --test-dir build ``` --- symforce/caspar/source/runtime/sort_indices.h | 15 ++++++++------- symforce/caspar/source/templates/solver.cc.jinja | 3 +++ .../symforce_caspar_solver_test/solver.cc | 3 +++ .../symforce_caspar_tridiagonal_test/solver.cc | 3 +++ 4 files changed, 17 insertions(+), 7 deletions(-) diff --git a/symforce/caspar/source/runtime/sort_indices.h b/symforce/caspar/source/runtime/sort_indices.h index 180be9b7..700622d8 100644 --- a/symforce/caspar/source/runtime/sort_indices.h +++ b/symforce/caspar/source/runtime/sort_indices.h @@ -5,7 +5,7 @@ #pragma once -#include +#include #if defined(USE_HIP) || defined(__HIP_PLATFORM_AMD__) #include @@ -17,15 +17,16 @@ namespace caspar { size_t SortIndicesGetTmpNbytes(size_t problem_size); -void SortIndices(void* tmp_storage, size_t tmp_storage_bytes, const uint* keys_in, uint* sorted_out, - uint* target_out, uint* argsort_out, size_t problem_size); +void SortIndices(void* tmp_storage, size_t tmp_storage_bytes, const unsigned int* keys_in, + unsigned int* sorted_out, unsigned int* target_out, unsigned int* argsort_out, + size_t problem_size); size_t SortKeysGetTmpNbytes(size_t problem_size); -void SortKeys(void* tmp_storage, size_t tmp_storage_bytes, const uint* keys_in, uint* sorted_out, - size_t problem_size); +void SortKeys(void* tmp_storage, size_t tmp_storage_bytes, const unsigned int* keys_in, + unsigned int* sorted_out, size_t problem_size); -void SelectIndex(const uint* const input, const uint* const selections, uint* const output, - size_t problem_size); +void SelectIndex(const unsigned int* const input, const unsigned int* const selections, + unsigned int* const output, size_t problem_size); } // namespace caspar diff --git a/symforce/caspar/source/templates/solver.cc.jinja b/symforce/caspar/source/templates/solver.cc.jinja index 866a31ef..1119f1a5 100644 --- a/symforce/caspar/source/templates/solver.cc.jinja +++ b/symforce/caspar/source/templates/solver.cc.jinja @@ -7,6 +7,9 @@ #include #include #include +#include +#include +#include #include "caspar_mappings.h" #include "shared_indices.h" diff --git a/test/symforce_function_codegen_test_data/symengine/symforce_caspar_solver_test/solver.cc b/test/symforce_function_codegen_test_data/symengine/symforce_caspar_solver_test/solver.cc index e910f2c6..266df822 100644 --- a/test/symforce_function_codegen_test_data/symengine/symforce_caspar_solver_test/solver.cc +++ b/test/symforce_function_codegen_test_data/symengine/symforce_caspar_solver_test/solver.cc @@ -2,7 +2,10 @@ #include #include +#include #include +#include +#include #include "caspar_mappings.h" #include "kernel_Matrix21_alpha_denominator_or_beta_numerator.h" diff --git a/test/symforce_function_codegen_test_data/symengine/symforce_caspar_tridiagonal_test/solver.cc b/test/symforce_function_codegen_test_data/symengine/symforce_caspar_tridiagonal_test/solver.cc index e82e6cd0..69b3b22b 100644 --- a/test/symforce_function_codegen_test_data/symengine/symforce_caspar_tridiagonal_test/solver.cc +++ b/test/symforce_function_codegen_test_data/symengine/symforce_caspar_tridiagonal_test/solver.cc @@ -2,7 +2,10 @@ #include #include +#include #include +#include +#include #include "caspar_mappings.h" #include "kernel_ChainA_alpha_denominator_or_beta_numerator.h"