Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
53 changes: 49 additions & 4 deletions symforce/caspar/source/runtime/memops.cuh
Original file line number Diff line number Diff line change
Expand Up @@ -246,6 +246,43 @@ __forceinline__ __device__ void WriteSum4(StorageT* const inout_shared, const Ke
inout_shared[threadIdx.x * 4 + 3] = w;
}

#if defined(__clang__) && !defined(__NVCC__)
/**
* Sums val over the active lanes of the warp that share the same label. Returns the sum on the
* lowest such lane, which is flagged with is_leader.
*
* Replaces cg::reduce over cg::labeled_partition on Clang CUDA, where floating-point
* coalesced_group reductions fall back to a non-contiguous inclusive scan whose shuffle loop trip
* counts vary per group, which can produce incorrect results in divergent code. This is slower
* under nvcc, so it is only used on Clang.
*/
template <typename StorageT>
__forceinline__ __device__ StorageT LabeledWarpSum(const uint label, const StorageT val,
bool* const is_leader) {
const unsigned int lane_id = threadIdx.x & 31;
const unsigned int active_mask = __activemask();
StorageT tot = 0;
int group_leader = -1;
for (int lane = 0; lane < 32; ++lane) {
if ((active_mask & (1u << lane)) == 0) {
continue;
}
const uint other_label = __shfl_sync(active_mask, label, lane);
const StorageT other_val = __shfl_sync(active_mask, val, lane);
if (other_label == label) {
if (group_leader == -1) {
group_leader = lane;
}
if (lane_id == static_cast<unsigned int>(group_leader)) {
tot += other_val;
}
}
}
*is_leader = lane_id == static_cast<unsigned int>(group_leader);
return tot;
}
#endif

/**
* Function used to perform collaborative reductions. Read more on caspar.argtypes.accessors.Sum.
*/
Expand All @@ -260,21 +297,29 @@ __forceinline__ __device__ void FlushSumShared(StorageT* const output, const uin
if (idx.argsort != 0xffff) { // 0xffff indicates the thread is not used.
unique = indices[indices[idx.argsort].target].unique;
}
#if !defined(__clang__) || defined(__NVCC__) // Clang uses LabeledWarpSum instead.
const cg::coalesced_group group = cg::labeled_partition(cg::coalesced_threads(), unique);
#endif

#pragma unroll
for (int i = 0; i < dim_target; i++) {
const SharedIndex idx = indices[threadIdx.x];
StorageT tot;
StorageT tot = 0;
bool is_leader = false;
if (idx.argsort != 0xffff) { // 0xffff indicates the thread is not used.
#if defined(__clang__) && !defined(__NVCC__)
tot = LabeledWarpSum(unique, inout_shared[idx.argsort * dim_target + i], &is_leader);
#else
tot = cg::reduce(group, inout_shared[idx.argsort * dim_target + i], cg::plus<StorageT>());
is_leader = group.thread_rank() == 0;
#endif
}
__syncthreads();
inout_shared[threadIdx.x * dim_target + i] = 0.0f;
__syncthreads();

// 0xffff indicates the thread is not used.
if (idx.argsort != 0xffff && group.thread_rank() == 0) {
if (idx.argsort != 0xffff && is_leader) {
atomicAdd_block(&inout_shared[indices[idx.argsort].target * dim_target + i], tot);
}
__syncthreads();
Expand Down Expand Up @@ -308,7 +353,7 @@ __forceinline__ __device__ void FlushSumBlock(StorageT* const output, StorageT*

#pragma unroll
for (int i = 0; i < dim_target; i++) {
StorageT tot;
StorageT tot = 0;

if (valid) {
tot = cg::reduce(group, inout_shared[threadIdx.x * dim_target + i], cg::plus<StorageT>());
Expand Down Expand Up @@ -369,7 +414,7 @@ __forceinline__ __device__ void FlushSumBlockAdd(StorageT* const output,

#pragma unroll
for (int i = 0; i < dim_target; i++) {
StorageT tot;
StorageT tot = 0;

if (valid) {
tot = cg::reduce(group, inout_shared[threadIdx.x * dim_target + i], cg::plus<StorageT>());
Expand Down
40 changes: 40 additions & 0 deletions test/caspar_solver_cuda_test.cc
Original file line number Diff line number Diff line change
Expand Up @@ -123,6 +123,46 @@ TEST_CASE("A solve that accepts non-monotonically still returns what it reports"
CHECK(ScoreOf(solve.x) == Catch::Approx(solve.result.final_score).epsilon(1e-9));
}

TEST_CASE("Solver handles shared nodes and partial blocks", "[caspar_solver_cuda_test]") {
REQUIRE(HaveDevice());

constexpr size_t kNumSharedNodes = 64;
constexpr size_t kNumFactors = 1500;
caspar::SolverParams<double> params = ParamsWithIterations(50);
caspar::GraphSolver solver(params, kNumSharedNodes, kNumFactors);

std::vector<unsigned int> indices(kNumFactors);
for (size_t i = 0; i < kNumFactors; i++) {
indices[i] = static_cast<unsigned int>(i % kNumSharedNodes);
}
std::vector<double> x(2 * kNumSharedNodes);
for (size_t i = 0; i < kNumSharedNodes; i++) {
x[2 * i + 0] = -1.2;
x[2 * i + 1] = 1.0;
}

solver.SetRosenbrockXIndicesFromHost(indices.data(), kNumFactors);
solver.SetMatrix21NodesFromStackedHost(x.data(), 0, kNumSharedNodes);
solver.finish_indices();

const caspar::SolveResult result =
solver.solve(/* print_progress */ false, /* verbose_logging */ true);
solver.GetMatrix21NodesToStackedHost(x.data(), 0, kNumSharedNodes);

double expected_score = 0.0;
for (size_t i = 0; i < kNumFactors; i++) {
const size_t node = indices[i];
const double r0 = 10.0 * (x[2 * node + 1] - x[2 * node + 0] * x[2 * node + 0]);
const double r1 = 1.0 - x[2 * node + 0];
expected_score += 0.5 * (r0 * r0 + r1 * r1);
}

CHECK(std::isfinite(result.final_score));
CHECK(result.final_score < result.initial_score);
CHECK(x[0] > -1.2);
CHECK(expected_score == Catch::Approx(result.final_score).epsilon(1e-9));
}

TEST_CASE("A solver with no inner iterations is rejected", "[caspar_solver_cuda_test]") {
REQUIRE(HaveDevice());

Expand Down