From ab2089b9f246310a91ee9544ba92f1742e7597b8 Mon Sep 17 00:00:00 2001 From: Paul-Edouard Sarlin <15985472+sarlinpe@users.noreply.github.com> Date: Fri, 2 Oct 2026 10:20:08 +0000 Subject: [PATCH 1/3] [Caspar] Fix FlushSumShared warp reduction and zero-initialize reduction accumulators In FlushSumShared, cg::labeled_partition(cg::coalesced_threads(), unique) followed by cg::reduce inside the divergent (idx.argsort != 0xffff) branch can produce incorrect warp reductions when compiled with Clang CUDA (where floating-point coalesced_group reductions fall back to inclusive_scan_non_contiguous with per-group variable shuffle loop trip counts, and tot is uninitialized across __syncthreads() in inactive lanes). Replace the labeled_partition + cg::reduce call in FlushSumShared with an explicit uniform warp-shuffle reduction over __activemask(), and zero-initialize tot in FlushSumShared, FlushSumBlock, and FlushSumBlockAdd (following up on a768308e). Also add a test case in caspar_solver_cuda_test covering shared nodes and partial blocks. Signed-off-by: Paul-Edouard Sarlin <15985472+sarlinpe@users.noreply.github.com> --- symforce/caspar/source/runtime/memops.cuh | 29 ++++++++++++---- test/caspar_solver_cuda_test.cc | 40 +++++++++++++++++++++++ 2 files changed, 63 insertions(+), 6 deletions(-) diff --git a/symforce/caspar/source/runtime/memops.cuh b/symforce/caspar/source/runtime/memops.cuh index 49dc5299..4be187d0 100644 --- a/symforce/caspar/source/runtime/memops.cuh +++ b/symforce/caspar/source/runtime/memops.cuh @@ -260,21 +260,38 @@ __forceinline__ __device__ void FlushSumShared(StorageT* const output, const uin if (idx.argsort != 0xffff) { // 0xffff indicates the thread is not used. unique = indices[indices[idx.argsort].target].unique; } - const cg::coalesced_group group = cg::labeled_partition(cg::coalesced_threads(), unique); + const unsigned int lane_id = threadIdx.x & 31; #pragma unroll for (int i = 0; i < dim_target; i++) { const SharedIndex idx = indices[threadIdx.x]; - StorageT tot; + StorageT tot = 0; + int group_leader = -1; if (idx.argsort != 0xffff) { // 0xffff indicates the thread is not used. - tot = cg::reduce(group, inout_shared[idx.argsort * dim_target + i], cg::plus()); + const StorageT my_val = inout_shared[idx.argsort * dim_target + i]; + const unsigned int active_mask = __activemask(); + for (int lane = 0; lane < 32; ++lane) { + if ((active_mask & (1u << lane)) == 0) { + continue; + } + const uint other_unique = __shfl_sync(active_mask, unique, lane); + const StorageT other_tot = __shfl_sync(active_mask, my_val, lane); + if (other_unique == unique) { + if (group_leader == -1) { + group_leader = lane; + } + if (lane_id == static_cast(group_leader)) { + tot += other_tot; + } + } + } } __syncthreads(); inout_shared[threadIdx.x * dim_target + i] = 0.0f; __syncthreads(); // 0xffff indicates the thread is not used. - if (idx.argsort != 0xffff && group.thread_rank() == 0) { + if (idx.argsort != 0xffff && lane_id == static_cast(group_leader)) { atomicAdd_block(&inout_shared[indices[idx.argsort].target * dim_target + i], tot); } __syncthreads(); @@ -308,7 +325,7 @@ __forceinline__ __device__ void FlushSumBlock(StorageT* const output, StorageT* #pragma unroll for (int i = 0; i < dim_target; i++) { - StorageT tot; + StorageT tot = 0; if (valid) { tot = cg::reduce(group, inout_shared[threadIdx.x * dim_target + i], cg::plus()); @@ -369,7 +386,7 @@ __forceinline__ __device__ void FlushSumBlockAdd(StorageT* const output, #pragma unroll for (int i = 0; i < dim_target; i++) { - StorageT tot; + StorageT tot = 0; if (valid) { tot = cg::reduce(group, inout_shared[threadIdx.x * dim_target + i], cg::plus()); diff --git a/test/caspar_solver_cuda_test.cc b/test/caspar_solver_cuda_test.cc index bce016fe..38ed2844 100644 --- a/test/caspar_solver_cuda_test.cc +++ b/test/caspar_solver_cuda_test.cc @@ -123,6 +123,46 @@ TEST_CASE("A solve that accepts non-monotonically still returns what it reports" CHECK(ScoreOf(solve.x) == Catch::Approx(solve.result.final_score).epsilon(1e-9)); } +TEST_CASE("Solver handles shared nodes and partial blocks", "[caspar_solver_cuda_test]") { + REQUIRE(HaveDevice()); + + constexpr size_t kNumSharedNodes = 64; + constexpr size_t kNumFactors = 1500; + caspar::SolverParams params = ParamsWithIterations(50); + caspar::GraphSolver solver(params, kNumSharedNodes, kNumFactors); + + std::vector indices(kNumFactors); + for (size_t i = 0; i < kNumFactors; i++) { + indices[i] = static_cast(i % kNumSharedNodes); + } + std::vector x(2 * kNumSharedNodes); + for (size_t i = 0; i < kNumSharedNodes; i++) { + x[2 * i + 0] = -1.2; + x[2 * i + 1] = 1.0; + } + + solver.SetRosenbrockXIndicesFromHost(indices.data(), kNumFactors); + solver.SetMatrix21NodesFromStackedHost(x.data(), 0, kNumSharedNodes); + solver.finish_indices(); + + const caspar::SolveResult result = + solver.solve(/* print_progress */ false, /* verbose_logging */ true); + solver.GetMatrix21NodesToStackedHost(x.data(), 0, kNumSharedNodes); + + double expected_score = 0.0; + for (size_t i = 0; i < kNumFactors; i++) { + const size_t node = indices[i]; + const double r0 = 10.0 * (x[2 * node + 1] - x[2 * node + 0] * x[2 * node + 0]); + const double r1 = 1.0 - x[2 * node + 0]; + expected_score += 0.5 * (r0 * r0 + r1 * r1); + } + + CHECK(std::isfinite(result.final_score)); + CHECK(result.final_score < result.initial_score); + CHECK(x[0] > -1.2); + CHECK(expected_score == Catch::Approx(result.final_score).epsilon(1e-9)); +} + TEST_CASE("A solver with no inner iterations is rejected", "[caspar_solver_cuda_test]") { REQUIRE(HaveDevice()); From 0c32747fef78f35fba61c7a126c452cb38c538f4 Mon Sep 17 00:00:00 2001 From: Paul-Edouard Sarlin <15985472+sarlinpe@users.noreply.github.com> Date: Fri, 23 Oct 2026 07:19:00 +0000 Subject: [PATCH 2/3] [Caspar] Only use explicit warp-shuffle FlushSumShared reduction on Clang The explicit warp-shuffle reduction in FlushSumShared works around incorrect cg::reduce results on labeled_partition groups under Clang CUDA, but regresses performance under nvcc. Guard it with __clang__ && !__NVCC__ and restore the original cg::labeled_partition + cg::reduce path for nvcc. Zero-initialization of the reduction accumulators is kept unconditionally. Signed-off-by: Paul-Edouard Sarlin <15985472+sarlinpe@users.noreply.github.com> --- symforce/caspar/source/runtime/memops.cuh | 26 +++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/symforce/caspar/source/runtime/memops.cuh b/symforce/caspar/source/runtime/memops.cuh index 4be187d0..9e521356 100644 --- a/symforce/caspar/source/runtime/memops.cuh +++ b/symforce/caspar/source/runtime/memops.cuh @@ -260,6 +260,11 @@ __forceinline__ __device__ void FlushSumShared(StorageT* const output, const uin if (idx.argsort != 0xffff) { // 0xffff indicates the thread is not used. unique = indices[indices[idx.argsort].target].unique; } +#if defined(__clang__) && !defined(__NVCC__) + // Clang CUDA: cg::reduce on a floating-point coalesced_group from labeled_partition falls back + // to a non-contiguous inclusive scan whose shuffle loop trip counts vary per group, which can + // produce incorrect reductions inside this divergent branch. Use an explicit uniform warp + // shuffle over __activemask() instead. This is slower under nvcc, so it is only used on clang. const unsigned int lane_id = threadIdx.x & 31; #pragma unroll @@ -296,6 +301,27 @@ __forceinline__ __device__ void FlushSumShared(StorageT* const output, const uin } __syncthreads(); } +#else + const cg::coalesced_group group = cg::labeled_partition(cg::coalesced_threads(), unique); + +#pragma unroll + for (int i = 0; i < dim_target; i++) { + const SharedIndex idx = indices[threadIdx.x]; + StorageT tot = 0; + if (idx.argsort != 0xffff) { // 0xffff indicates the thread is not used. + tot = cg::reduce(group, inout_shared[idx.argsort * dim_target + i], cg::plus()); + } + __syncthreads(); + inout_shared[threadIdx.x * dim_target + i] = 0.0f; + __syncthreads(); + + // 0xffff indicates the thread is not used. + if (idx.argsort != 0xffff && group.thread_rank() == 0) { + atomicAdd_block(&inout_shared[indices[idx.argsort].target * dim_target + i], tot); + } + __syncthreads(); + } +#endif constexpr uint dim_aligned = dim_target == 3 ? 4 : dim_target; for (int i = 0; i < dim_target; i++) { From 25454b315848d1c33f7577e48b0598e84efe5278 Mon Sep 17 00:00:00 2001 From: Paul-Edouard Sarlin <15985472+sarlinpe@users.noreply.github.com> Date: Fri, 23 Oct 2026 17:05:27 +0000 Subject: [PATCH 3/3] [Caspar] Factor Clang FlushSumShared reduction into LabeledWarpSum helper Move the Clang-only warp-shuffle reduction into a LabeledWarpSum helper so that FlushSumShared keeps a single loop and only the reduction call differs between Clang and nvcc, reducing duplication. Signed-off-by: Paul-Edouard Sarlin <15985472+sarlinpe@users.noreply.github.com> --- symforce/caspar/source/runtime/memops.cuh | 90 ++++++++++++----------- 1 file changed, 46 insertions(+), 44 deletions(-) diff --git a/symforce/caspar/source/runtime/memops.cuh b/symforce/caspar/source/runtime/memops.cuh index 9e521356..d8608622 100644 --- a/symforce/caspar/source/runtime/memops.cuh +++ b/symforce/caspar/source/runtime/memops.cuh @@ -246,6 +246,43 @@ __forceinline__ __device__ void WriteSum4(StorageT* const inout_shared, const Ke inout_shared[threadIdx.x * 4 + 3] = w; } +#if defined(__clang__) && !defined(__NVCC__) +/** + * Sums val over the active lanes of the warp that share the same label. Returns the sum on the + * lowest such lane, which is flagged with is_leader. + * + * Replaces cg::reduce over cg::labeled_partition on Clang CUDA, where floating-point + * coalesced_group reductions fall back to a non-contiguous inclusive scan whose shuffle loop trip + * counts vary per group, which can produce incorrect results in divergent code. This is slower + * under nvcc, so it is only used on Clang. + */ +template +__forceinline__ __device__ StorageT LabeledWarpSum(const uint label, const StorageT val, + bool* const is_leader) { + const unsigned int lane_id = threadIdx.x & 31; + const unsigned int active_mask = __activemask(); + StorageT tot = 0; + int group_leader = -1; + for (int lane = 0; lane < 32; ++lane) { + if ((active_mask & (1u << lane)) == 0) { + continue; + } + const uint other_label = __shfl_sync(active_mask, label, lane); + const StorageT other_val = __shfl_sync(active_mask, val, lane); + if (other_label == label) { + if (group_leader == -1) { + group_leader = lane; + } + if (lane_id == static_cast(group_leader)) { + tot += other_val; + } + } + } + *is_leader = lane_id == static_cast(group_leader); + return tot; +} +#endif + /** * Function used to perform collaborative reductions. Read more on caspar.argtypes.accessors.Sum. */ @@ -260,68 +297,33 @@ __forceinline__ __device__ void FlushSumShared(StorageT* const output, const uin if (idx.argsort != 0xffff) { // 0xffff indicates the thread is not used. unique = indices[indices[idx.argsort].target].unique; } -#if defined(__clang__) && !defined(__NVCC__) - // Clang CUDA: cg::reduce on a floating-point coalesced_group from labeled_partition falls back - // to a non-contiguous inclusive scan whose shuffle loop trip counts vary per group, which can - // produce incorrect reductions inside this divergent branch. Use an explicit uniform warp - // shuffle over __activemask() instead. This is slower under nvcc, so it is only used on clang. - const unsigned int lane_id = threadIdx.x & 31; - -#pragma unroll - for (int i = 0; i < dim_target; i++) { - const SharedIndex idx = indices[threadIdx.x]; - StorageT tot = 0; - int group_leader = -1; - if (idx.argsort != 0xffff) { // 0xffff indicates the thread is not used. - const StorageT my_val = inout_shared[idx.argsort * dim_target + i]; - const unsigned int active_mask = __activemask(); - for (int lane = 0; lane < 32; ++lane) { - if ((active_mask & (1u << lane)) == 0) { - continue; - } - const uint other_unique = __shfl_sync(active_mask, unique, lane); - const StorageT other_tot = __shfl_sync(active_mask, my_val, lane); - if (other_unique == unique) { - if (group_leader == -1) { - group_leader = lane; - } - if (lane_id == static_cast(group_leader)) { - tot += other_tot; - } - } - } - } - __syncthreads(); - inout_shared[threadIdx.x * dim_target + i] = 0.0f; - __syncthreads(); - - // 0xffff indicates the thread is not used. - if (idx.argsort != 0xffff && lane_id == static_cast(group_leader)) { - atomicAdd_block(&inout_shared[indices[idx.argsort].target * dim_target + i], tot); - } - __syncthreads(); - } -#else +#if !defined(__clang__) || defined(__NVCC__) // Clang uses LabeledWarpSum instead. const cg::coalesced_group group = cg::labeled_partition(cg::coalesced_threads(), unique); +#endif #pragma unroll for (int i = 0; i < dim_target; i++) { const SharedIndex idx = indices[threadIdx.x]; StorageT tot = 0; + bool is_leader = false; if (idx.argsort != 0xffff) { // 0xffff indicates the thread is not used. +#if defined(__clang__) && !defined(__NVCC__) + tot = LabeledWarpSum(unique, inout_shared[idx.argsort * dim_target + i], &is_leader); +#else tot = cg::reduce(group, inout_shared[idx.argsort * dim_target + i], cg::plus()); + is_leader = group.thread_rank() == 0; +#endif } __syncthreads(); inout_shared[threadIdx.x * dim_target + i] = 0.0f; __syncthreads(); // 0xffff indicates the thread is not used. - if (idx.argsort != 0xffff && group.thread_rank() == 0) { + if (idx.argsort != 0xffff && is_leader) { atomicAdd_block(&inout_shared[indices[idx.argsort].target * dim_target + i], tot); } __syncthreads(); } -#endif constexpr uint dim_aligned = dim_target == 3 ? 4 : dim_target; for (int i = 0; i < dim_target; i++) {