diff --git a/symforce/caspar/source/runtime/memops.cuh b/symforce/caspar/source/runtime/memops.cuh index 49dc5299..d8608622 100644 --- a/symforce/caspar/source/runtime/memops.cuh +++ b/symforce/caspar/source/runtime/memops.cuh @@ -246,6 +246,43 @@ __forceinline__ __device__ void WriteSum4(StorageT* const inout_shared, const Ke inout_shared[threadIdx.x * 4 + 3] = w; } +#if defined(__clang__) && !defined(__NVCC__) +/** + * Sums val over the active lanes of the warp that share the same label. Returns the sum on the + * lowest such lane, which is flagged with is_leader. + * + * Replaces cg::reduce over cg::labeled_partition on Clang CUDA, where floating-point + * coalesced_group reductions fall back to a non-contiguous inclusive scan whose shuffle loop trip + * counts vary per group, which can produce incorrect results in divergent code. This is slower + * under nvcc, so it is only used on Clang. + */ +template +__forceinline__ __device__ StorageT LabeledWarpSum(const uint label, const StorageT val, + bool* const is_leader) { + const unsigned int lane_id = threadIdx.x & 31; + const unsigned int active_mask = __activemask(); + StorageT tot = 0; + int group_leader = -1; + for (int lane = 0; lane < 32; ++lane) { + if ((active_mask & (1u << lane)) == 0) { + continue; + } + const uint other_label = __shfl_sync(active_mask, label, lane); + const StorageT other_val = __shfl_sync(active_mask, val, lane); + if (other_label == label) { + if (group_leader == -1) { + group_leader = lane; + } + if (lane_id == static_cast(group_leader)) { + tot += other_val; + } + } + } + *is_leader = lane_id == static_cast(group_leader); + return tot; +} +#endif + /** * Function used to perform collaborative reductions. Read more on caspar.argtypes.accessors.Sum. */ @@ -260,21 +297,29 @@ __forceinline__ __device__ void FlushSumShared(StorageT* const output, const uin if (idx.argsort != 0xffff) { // 0xffff indicates the thread is not used. unique = indices[indices[idx.argsort].target].unique; } +#if !defined(__clang__) || defined(__NVCC__) // Clang uses LabeledWarpSum instead. const cg::coalesced_group group = cg::labeled_partition(cg::coalesced_threads(), unique); +#endif #pragma unroll for (int i = 0; i < dim_target; i++) { const SharedIndex idx = indices[threadIdx.x]; - StorageT tot; + StorageT tot = 0; + bool is_leader = false; if (idx.argsort != 0xffff) { // 0xffff indicates the thread is not used. +#if defined(__clang__) && !defined(__NVCC__) + tot = LabeledWarpSum(unique, inout_shared[idx.argsort * dim_target + i], &is_leader); +#else tot = cg::reduce(group, inout_shared[idx.argsort * dim_target + i], cg::plus()); + is_leader = group.thread_rank() == 0; +#endif } __syncthreads(); inout_shared[threadIdx.x * dim_target + i] = 0.0f; __syncthreads(); // 0xffff indicates the thread is not used. - if (idx.argsort != 0xffff && group.thread_rank() == 0) { + if (idx.argsort != 0xffff && is_leader) { atomicAdd_block(&inout_shared[indices[idx.argsort].target * dim_target + i], tot); } __syncthreads(); @@ -308,7 +353,7 @@ __forceinline__ __device__ void FlushSumBlock(StorageT* const output, StorageT* #pragma unroll for (int i = 0; i < dim_target; i++) { - StorageT tot; + StorageT tot = 0; if (valid) { tot = cg::reduce(group, inout_shared[threadIdx.x * dim_target + i], cg::plus()); @@ -369,7 +414,7 @@ __forceinline__ __device__ void FlushSumBlockAdd(StorageT* const output, #pragma unroll for (int i = 0; i < dim_target; i++) { - StorageT tot; + StorageT tot = 0; if (valid) { tot = cg::reduce(group, inout_shared[threadIdx.x * dim_target + i], cg::plus()); diff --git a/test/caspar_solver_cuda_test.cc b/test/caspar_solver_cuda_test.cc index bce016fe..38ed2844 100644 --- a/test/caspar_solver_cuda_test.cc +++ b/test/caspar_solver_cuda_test.cc @@ -123,6 +123,46 @@ TEST_CASE("A solve that accepts non-monotonically still returns what it reports" CHECK(ScoreOf(solve.x) == Catch::Approx(solve.result.final_score).epsilon(1e-9)); } +TEST_CASE("Solver handles shared nodes and partial blocks", "[caspar_solver_cuda_test]") { + REQUIRE(HaveDevice()); + + constexpr size_t kNumSharedNodes = 64; + constexpr size_t kNumFactors = 1500; + caspar::SolverParams params = ParamsWithIterations(50); + caspar::GraphSolver solver(params, kNumSharedNodes, kNumFactors); + + std::vector indices(kNumFactors); + for (size_t i = 0; i < kNumFactors; i++) { + indices[i] = static_cast(i % kNumSharedNodes); + } + std::vector x(2 * kNumSharedNodes); + for (size_t i = 0; i < kNumSharedNodes; i++) { + x[2 * i + 0] = -1.2; + x[2 * i + 1] = 1.0; + } + + solver.SetRosenbrockXIndicesFromHost(indices.data(), kNumFactors); + solver.SetMatrix21NodesFromStackedHost(x.data(), 0, kNumSharedNodes); + solver.finish_indices(); + + const caspar::SolveResult result = + solver.solve(/* print_progress */ false, /* verbose_logging */ true); + solver.GetMatrix21NodesToStackedHost(x.data(), 0, kNumSharedNodes); + + double expected_score = 0.0; + for (size_t i = 0; i < kNumFactors; i++) { + const size_t node = indices[i]; + const double r0 = 10.0 * (x[2 * node + 1] - x[2 * node + 0] * x[2 * node + 0]); + const double r1 = 1.0 - x[2 * node + 0]; + expected_score += 0.5 * (r0 * r0 + r1 * r1); + } + + CHECK(std::isfinite(result.final_score)); + CHECK(result.final_score < result.initial_score); + CHECK(x[0] > -1.2); + CHECK(expected_score == Catch::Approx(result.final_score).epsilon(1e-9)); +} + TEST_CASE("A solver with no inner iterations is rejected", "[caspar_solver_cuda_test]") { REQUIRE(HaveDevice());