From 193241537a9de551f7afc90402c23eee7029a8f4 Mon Sep 17 00:00:00 2001 From: shsahiti Date: Tue, 22 Sep 2026 11:52:29 -0400 Subject: [PATCH 1/7] Add accumulate, average, and geometric_mean reductions --- xpu/reduce.hpp | 62 ++++++++++++++++++++++++++++++++++++++++++++++++++ xpu/xpu.hpp | 1 + 2 files changed, 63 insertions(+) create mode 100644 xpu/reduce.hpp diff --git a/xpu/reduce.hpp b/xpu/reduce.hpp new file mode 100644 index 0000000..a3e6f3c --- /dev/null +++ b/xpu/reduce.hpp @@ -0,0 +1,62 @@ +#pragma once + +#include +#include +#include +#include +#include + +#include + +namespace xpu { + +template +inline auto accumulate(const T* ptr, std::size_t count) -> T { + const auto range = xpu::range<1uz>{ + .begin = {0uz}, .end = {count}, .step = {1uz} + }; + + const auto contribution = [ptr] CUDA_CALLABLE (const xpu::array& index) -> T { + return ptr[index[0]]; + }; + + const auto scratch_bytes{xpu::parallel_reduce_sum_bytes(range, contribution)}; + const auto scratch{xpu::make_unique(scratch_bytes)}; + const auto sum_device{xpu::make_unique()}; + + xpu::parallel_reduce_sum(range, sum_device.get(), contribution, scratch.get(), scratch_bytes); + + auto sum{T{0}}; + xpu::copy_n(&sum, sum_device.get(), 1uz); + + return sum; +} + +template +inline auto average(const T* ptr, std::size_t count) -> T { + return accumulate(ptr, count) / static_cast(count); +} + +template +inline auto geometric_mean(const T* ptr, std::size_t count) -> T { + const auto range = xpu::range<1uz>{ + .begin = {0uz}, .end = {count}, .step = {1uz} + }; + + const auto contribution = [ptr] CUDA_CALLABLE (const xpu::array& index) -> T { + return xpu::log(ptr[index[0]]); + }; + + const auto scratch_bytes{xpu::parallel_reduce_sum_bytes(range, contribution)}; + const auto scratch{xpu::make_unique(scratch_bytes)}; + const auto log_sum_device{xpu::make_unique()}; + + xpu::parallel_reduce_sum(range, log_sum_device.get(), contribution, scratch.get(), scratch_bytes); + + auto log_sum{T{0}}; + xpu::copy_n(&log_sum, log_sum_device.get(), 1uz); + + return xpu::exp(log_sum / static_cast(count)); +} + +} // namespace xpu diff --git a/xpu/xpu.hpp b/xpu/xpu.hpp index c59b316..6146ce8 100644 --- a/xpu/xpu.hpp +++ b/xpu/xpu.hpp @@ -7,4 +7,5 @@ #include #include #include +#include #include From 5777f44ce823c507718cc737cae507b9b2ca3261 Mon Sep 17 00:00:00 2001 From: shsahiti Date: Sat, 17 Oct 2026 13:09:00 -0400 Subject: [PATCH 2/7] Adapt reductions to scratch-free parallel_reduce_sum and XPU_ macros --- xpu/reduce.hpp | 16 ++++++---------- 1 file changed, 6 insertions(+), 10 deletions(-) diff --git a/xpu/reduce.hpp b/xpu/reduce.hpp index a3e6f3c..18e358f 100644 --- a/xpu/reduce.hpp +++ b/xpu/reduce.hpp @@ -16,15 +16,13 @@ inline auto accumulate(const T* ptr, std::size_t count) -> T { .begin = {0uz}, .end = {count}, .step = {1uz} }; - const auto contribution = [ptr] CUDA_CALLABLE (const xpu::array& index) -> T { + const auto contribution = [ptr] XPU_CUDA_CALLABLE (const xpu::array& index) -> T { return ptr[index[0]]; }; - const auto scratch_bytes{xpu::parallel_reduce_sum_bytes(range, contribution)}; - const auto scratch{xpu::make_unique(scratch_bytes)}; - const auto sum_device{xpu::make_unique()}; + const auto sum_device{xpu::make_unique(1uz)}; - xpu::parallel_reduce_sum(range, sum_device.get(), contribution, scratch.get(), scratch_bytes); + xpu::parallel_reduce_sum(range, sum_device.get(), contribution); auto sum{T{0}}; xpu::copy_n(&sum, sum_device.get(), 1uz); @@ -43,15 +41,13 @@ inline auto geometric_mean(const T* ptr, std::size_t count) -> T { .begin = {0uz}, .end = {count}, .step = {1uz} }; - const auto contribution = [ptr] CUDA_CALLABLE (const xpu::array& index) -> T { + const auto contribution = [ptr] XPU_CUDA_CALLABLE (const xpu::array& index) -> T { return xpu::log(ptr[index[0]]); }; - const auto scratch_bytes{xpu::parallel_reduce_sum_bytes(range, contribution)}; - const auto scratch{xpu::make_unique(scratch_bytes)}; - const auto log_sum_device{xpu::make_unique()}; + const auto log_sum_device{xpu::make_unique(1uz)}; - xpu::parallel_reduce_sum(range, log_sum_device.get(), contribution, scratch.get(), scratch_bytes); + xpu::parallel_reduce_sum(range, log_sum_device.get(), contribution); auto log_sum{T{0}}; xpu::copy_n(&log_sum, log_sum_device.get(), 1uz); From beb0603ddde7d206a0025ebf4ceeefc50f77498b Mon Sep 17 00:00:00 2001 From: shsahiti Date: Sat, 17 Oct 2026 13:12:06 -0400 Subject: [PATCH 3/7] Use functors instead of extended lambdas in reductions --- xpu/reduce.hpp | 32 ++++++++++++++++++++++++++------ 1 file changed, 26 insertions(+), 6 deletions(-) diff --git a/xpu/reduce.hpp b/xpu/reduce.hpp index 18e358f..aae51a0 100644 --- a/xpu/reduce.hpp +++ b/xpu/reduce.hpp @@ -10,15 +10,37 @@ namespace xpu { +namespace detail { + +template +struct load_element { + const T* ptr; + + XPU_CUDA_CALLABLE + auto operator()(const xpu::array& index) const -> T { + return ptr[index[0]]; + } +}; + +template +struct log_element { + const T* ptr; + + XPU_CUDA_CALLABLE + auto operator()(const xpu::array& index) const -> T { + return xpu::log(ptr[index[0]]); + } +}; + +} // namespace detail + template inline auto accumulate(const T* ptr, std::size_t count) -> T { const auto range = xpu::range<1uz>{ .begin = {0uz}, .end = {count}, .step = {1uz} }; - const auto contribution = [ptr] XPU_CUDA_CALLABLE (const xpu::array& index) -> T { - return ptr[index[0]]; - }; + const auto contribution{xpu::detail::load_element{ptr}}; const auto sum_device{xpu::make_unique(1uz)}; @@ -41,9 +63,7 @@ inline auto geometric_mean(const T* ptr, std::size_t count) -> T { .begin = {0uz}, .end = {count}, .step = {1uz} }; - const auto contribution = [ptr] XPU_CUDA_CALLABLE (const xpu::array& index) -> T { - return xpu::log(ptr[index[0]]); - }; + const auto contribution{xpu::detail::log_element{ptr}}; const auto log_sum_device{xpu::make_unique(1uz)}; From dd5e1160ea6c176b10cf35fac018d4d345341051 Mon Sep 17 00:00:00 2001 From: shsahiti Date: Sat, 17 Oct 2026 13:22:24 -0400 Subject: [PATCH 4/7] Share the reduce-to-host-scalar step between reductions --- xpu/reduce.hpp | 28 ++++++++++------------------ 1 file changed, 10 insertions(+), 18 deletions(-) diff --git a/xpu/reduce.hpp b/xpu/reduce.hpp index aae51a0..6249430 100644 --- a/xpu/reduce.hpp +++ b/xpu/reduce.hpp @@ -32,16 +32,12 @@ struct log_element { } }; -} // namespace detail - -template -inline auto accumulate(const T* ptr, std::size_t count) -> T { +template +inline auto reduce_sum(std::size_t count, const F& contribution) -> T { const auto range = xpu::range<1uz>{ .begin = {0uz}, .end = {count}, .step = {1uz} }; - const auto contribution{xpu::detail::load_element{ptr}}; - const auto sum_device{xpu::make_unique(1uz)}; xpu::parallel_reduce_sum(range, sum_device.get(), contribution); @@ -52,6 +48,13 @@ inline auto accumulate(const T* ptr, std::size_t count) -> T { return sum; } +} // namespace detail + +template +inline auto accumulate(const T* ptr, std::size_t count) -> T { + return xpu::detail::reduce_sum(count, xpu::detail::load_element{ptr}); +} + template inline auto average(const T* ptr, std::size_t count) -> T { return accumulate(ptr, count) / static_cast(count); @@ -59,18 +62,7 @@ inline auto average(const T* ptr, std::size_t count) -> T { template inline auto geometric_mean(const T* ptr, std::size_t count) -> T { - const auto range = xpu::range<1uz>{ - .begin = {0uz}, .end = {count}, .step = {1uz} - }; - - const auto contribution{xpu::detail::log_element{ptr}}; - - const auto log_sum_device{xpu::make_unique(1uz)}; - - xpu::parallel_reduce_sum(range, log_sum_device.get(), contribution); - - auto log_sum{T{0}}; - xpu::copy_n(&log_sum, log_sum_device.get(), 1uz); + const auto log_sum{xpu::detail::reduce_sum(count, xpu::detail::log_element{ptr})}; return xpu::exp(log_sum / static_cast(count)); } From be1607b66fe2230429b8ccfb98e16a6472e47608 Mon Sep 17 00:00:00 2001 From: shsahiti Date: Sat, 17 Oct 2026 13:25:12 -0400 Subject: [PATCH 5/7] Revert "Share the reduce-to-host-scalar step between reductions" This reverts commit dd5e1160ea6c176b10cf35fac018d4d345341051. --- xpu/reduce.hpp | 28 ++++++++++++++++++---------- 1 file changed, 18 insertions(+), 10 deletions(-) diff --git a/xpu/reduce.hpp b/xpu/reduce.hpp index 6249430..aae51a0 100644 --- a/xpu/reduce.hpp +++ b/xpu/reduce.hpp @@ -32,12 +32,16 @@ struct log_element { } }; -template -inline auto reduce_sum(std::size_t count, const F& contribution) -> T { +} // namespace detail + +template +inline auto accumulate(const T* ptr, std::size_t count) -> T { const auto range = xpu::range<1uz>{ .begin = {0uz}, .end = {count}, .step = {1uz} }; + const auto contribution{xpu::detail::load_element{ptr}}; + const auto sum_device{xpu::make_unique(1uz)}; xpu::parallel_reduce_sum(range, sum_device.get(), contribution); @@ -48,13 +52,6 @@ inline auto reduce_sum(std::size_t count, const F& contribution) -> T { return sum; } -} // namespace detail - -template -inline auto accumulate(const T* ptr, std::size_t count) -> T { - return xpu::detail::reduce_sum(count, xpu::detail::load_element{ptr}); -} - template inline auto average(const T* ptr, std::size_t count) -> T { return accumulate(ptr, count) / static_cast(count); @@ -62,7 +59,18 @@ inline auto average(const T* ptr, std::size_t count) -> T { template inline auto geometric_mean(const T* ptr, std::size_t count) -> T { - const auto log_sum{xpu::detail::reduce_sum(count, xpu::detail::log_element{ptr})}; + const auto range = xpu::range<1uz>{ + .begin = {0uz}, .end = {count}, .step = {1uz} + }; + + const auto contribution{xpu::detail::log_element{ptr}}; + + const auto log_sum_device{xpu::make_unique(1uz)}; + + xpu::parallel_reduce_sum(range, log_sum_device.get(), contribution); + + auto log_sum{T{0}}; + xpu::copy_n(&log_sum, log_sum_device.get(), 1uz); return xpu::exp(log_sum / static_cast(count)); } From 648e55c402a1f616e54a00d3e2e8913d46934809 Mon Sep 17 00:00:00 2001 From: shsahiti Date: Sat, 17 Oct 2026 13:25:12 -0400 Subject: [PATCH 6/7] Revert "Use functors instead of extended lambdas in reductions" This reverts commit beb0603ddde7d206a0025ebf4ceeefc50f77498b. --- xpu/reduce.hpp | 32 ++++++-------------------------- 1 file changed, 6 insertions(+), 26 deletions(-) diff --git a/xpu/reduce.hpp b/xpu/reduce.hpp index aae51a0..18e358f 100644 --- a/xpu/reduce.hpp +++ b/xpu/reduce.hpp @@ -10,37 +10,15 @@ namespace xpu { -namespace detail { - -template -struct load_element { - const T* ptr; - - XPU_CUDA_CALLABLE - auto operator()(const xpu::array& index) const -> T { - return ptr[index[0]]; - } -}; - -template -struct log_element { - const T* ptr; - - XPU_CUDA_CALLABLE - auto operator()(const xpu::array& index) const -> T { - return xpu::log(ptr[index[0]]); - } -}; - -} // namespace detail - template inline auto accumulate(const T* ptr, std::size_t count) -> T { const auto range = xpu::range<1uz>{ .begin = {0uz}, .end = {count}, .step = {1uz} }; - const auto contribution{xpu::detail::load_element{ptr}}; + const auto contribution = [ptr] XPU_CUDA_CALLABLE (const xpu::array& index) -> T { + return ptr[index[0]]; + }; const auto sum_device{xpu::make_unique(1uz)}; @@ -63,7 +41,9 @@ inline auto geometric_mean(const T* ptr, std::size_t count) -> T { .begin = {0uz}, .end = {count}, .step = {1uz} }; - const auto contribution{xpu::detail::log_element{ptr}}; + const auto contribution = [ptr] XPU_CUDA_CALLABLE (const xpu::array& index) -> T { + return xpu::log(ptr[index[0]]); + }; const auto log_sum_device{xpu::make_unique(1uz)}; From 74bd6a4ff3be330fcb5044c6577daf443c81a315 Mon Sep 17 00:00:00 2001 From: shsahiti Date: Sat, 17 Oct 2026 13:27:29 -0400 Subject: [PATCH 7/7] Reapply "Use functors instead of extended lambdas in reductions" This reverts commit 648e55c402a1f616e54a00d3e2e8913d46934809. --- xpu/reduce.hpp | 32 ++++++++++++++++++++++++++------ 1 file changed, 26 insertions(+), 6 deletions(-) diff --git a/xpu/reduce.hpp b/xpu/reduce.hpp index 18e358f..aae51a0 100644 --- a/xpu/reduce.hpp +++ b/xpu/reduce.hpp @@ -10,15 +10,37 @@ namespace xpu { +namespace detail { + +template +struct load_element { + const T* ptr; + + XPU_CUDA_CALLABLE + auto operator()(const xpu::array& index) const -> T { + return ptr[index[0]]; + } +}; + +template +struct log_element { + const T* ptr; + + XPU_CUDA_CALLABLE + auto operator()(const xpu::array& index) const -> T { + return xpu::log(ptr[index[0]]); + } +}; + +} // namespace detail + template inline auto accumulate(const T* ptr, std::size_t count) -> T { const auto range = xpu::range<1uz>{ .begin = {0uz}, .end = {count}, .step = {1uz} }; - const auto contribution = [ptr] XPU_CUDA_CALLABLE (const xpu::array& index) -> T { - return ptr[index[0]]; - }; + const auto contribution{xpu::detail::load_element{ptr}}; const auto sum_device{xpu::make_unique(1uz)}; @@ -41,9 +63,7 @@ inline auto geometric_mean(const T* ptr, std::size_t count) -> T { .begin = {0uz}, .end = {count}, .step = {1uz} }; - const auto contribution = [ptr] XPU_CUDA_CALLABLE (const xpu::array& index) -> T { - return xpu::log(ptr[index[0]]); - }; + const auto contribution{xpu::detail::log_element{ptr}}; const auto log_sum_device{xpu::make_unique(1uz)};