diff --git a/tests/reduce/cases.hpp b/tests/reduce/cases.hpp index 17b5eca..5ca60e2 100644 --- a/tests/reduce/cases.hpp +++ b/tests/reduce/cases.hpp @@ -34,7 +34,6 @@ struct nonconst_sum { template concept supports_sum = requires(const xpu::range<2uz>& range, T* output, F contribution) { xpu::parallel_reduce_sum(range, output, contribution); - xpu::parallel_reduce_sum_bytes(range, contribution); }; static_assert(supports_sum>); @@ -54,16 +53,6 @@ inline auto run_sum_type() -> int { const auto range = xpu::range<2uz>{ {2uz, 1uz}, {6uz, 1uz + 3uz * count}, {2uz, 3uz} }; - const auto required_bytes{xpu::parallel_reduce_sum_bytes(range, contribution)}; - const auto empty{count == 0uz}; - const auto invalid_empty_query{empty && required_bytes != 0uz}; - - if (invalid_empty_query) { - const auto failure{test::fail("empty reduction requested scratch storage")}; - - return failure; - } - auto expected{T{}}; for (auto row{2uz}; row < 6uz; row += 2uz) { @@ -72,16 +61,11 @@ inline auto run_sum_type() -> int { } } - const auto capacity{required_bytes + 64uz}; - auto scratch{xpu::buffer{capacity}}; const auto initial{T{91}}; xpu::copy_n(output.data(), &initial, 1uz); for (auto repeat{0uz}; repeat < 2uz; ++repeat) { - xpu::parallel_reduce_sum( - range, output.data(), contribution, - empty ? nullptr : scratch.data(), empty ? 0uz : capacity - ); + xpu::parallel_reduce_sum(range, output.data(), contribution); auto actual{T{}}; xpu::copy_n(&actual, output.data(), 1uz); @@ -116,13 +100,9 @@ struct sum_coordinates { template inline auto check_coordinate_sum(const xpu::range& range, int expected) -> int { const auto contribution{sum_coordinates{}}; - const auto scratch_bytes{xpu::parallel_reduce_sum_bytes(range, contribution)}; - auto scratch{xpu::buffer{scratch_bytes}}; auto output{xpu::buffer{1uz}}; - xpu::parallel_reduce_sum( - range, output.data(), contribution, scratch.data(), scratch_bytes - ); + xpu::parallel_reduce_sum(range, output.data(), contribution); auto actual{0}; xpu::copy_n(&actual, output.data(), 1uz); diff --git a/xpu/launch.hpp b/xpu/launch.hpp index f04272a..2e283af 100644 --- a/xpu/launch.hpp +++ b/xpu/launch.hpp @@ -1,6 +1,8 @@ #pragma once #include +#include +#include #include #include #include @@ -40,6 +42,24 @@ inline auto device_SMs() -> unsigned int { return cached; } +inline auto retain_default_pool() -> void { + static const auto once{[] { + auto device{0}; + auto pool{cudaMemPool_t{}}; + auto threshold{std::numeric_limits::max()}; + xpu::cu_check(cudaGetDevice(&device)); + xpu::cu_check(cudaDeviceGetDefaultMemPool(&pool, device)); + xpu::cu_check(cudaMemPoolSetAttribute( + pool, cudaMemPoolAttrReleaseThreshold, &threshold + )); + + return true; + }() + }; + + static_cast(once); +} + template [[nodiscard]] inline auto wave_blocks(Kernel kernel, dim3 threads, std::size_t smem = 0uz) -> unsigned int { const auto thread_budget{xpu::detail::checked_mul( @@ -249,17 +269,6 @@ inline auto reduction_input( return input; } -template [[nodiscard]] -inline auto reduction_bytes(Input input, std::ptrdiff_t count) -> std::size_t { - auto required_bytes{0uz}; - - xpu::cu_check(cub::DeviceReduce::Sum( - nullptr, required_bytes, input, static_cast(nullptr), count - )); - - return required_bytes; -} - template struct parallel_for_function { xpu::range range; @@ -476,34 +485,14 @@ inline auto parallel_reduce_sum_impl( const xpu::range& range, std::size_t total, T* output_ptr, - F&& contribution, - [[maybe_unused]] void* scratch = nullptr, - [[maybe_unused]] std::size_t scratch_bytes = 0uz + F&& contribution ) -> void { #if defined(XPU_CUDA) - using function_t = std::decay_t; - - const auto transform{reduction_contribution{ - range, function_t{std::forward(contribution)} - }}; - - const auto indices{thrust::counting_iterator{0uz}}; - const auto input{thrust::make_transform_iterator(indices, transform)}; - + const auto input{reduction_input(range, std::forward(contribution))}; const auto count{xpu::detail::checked_cast(total)}; - const auto required_bytes{reduction_bytes(input, count)}; - - const auto insufficient_scratch{ - scratch == nullptr || scratch_bytes < required_bytes - }; - if (insufficient_scratch) { - xpu::detail::checked_error("insufficient reduction scratch storage"); - } - - xpu::cu_check(cub::DeviceReduce::Sum( - scratch, scratch_bytes, input, output_ptr, count - )); + xpu::detail::retain_default_pool(); + xpu::cu_check(cub::DeviceReduce::Sum(input, output_ptr, count)); #else if (range.step[dims - 1uz] == 1uz) { *output_ptr = parallel_reduce_cpu(range, total, contribution); @@ -525,43 +514,12 @@ inline auto parallel_for( detail::parallel_for_impl(range, std::forward(fcn)); } -template - requires detail::sum_contribution -[[nodiscard]] -inline auto parallel_reduce_sum_bytes( - [[maybe_unused]] const xpu::range& range, - [[maybe_unused]] F&& contribution -) -> std::size_t { - static_assert(dims > 0uz, "ERROR: Dimension must be greater than 0."); - - auto required_bytes{0uz}; - -#if defined(XPU_CUDA) - const auto total{xpu::detail::num_itrs(range)}; - const auto empty{total == 0uz}; - - if (empty) { - - return required_bytes; - } - - const auto input{detail::reduction_input(range, std::forward(contribution))}; - const auto count{xpu::detail::checked_cast(total)}; - - required_bytes = detail::reduction_bytes(input, count); -#endif - - return required_bytes; -} - template requires detail::sum_contribution inline auto parallel_reduce_sum( const xpu::range& range, T* output_ptr, - F&& contribution, - void* scratch = nullptr, - std::size_t scratch_bytes = 0uz + F&& contribution ) -> void { static_assert(dims > 0uz, "ERROR: Dimension must be greater than 0."); @@ -579,7 +537,7 @@ inline auto parallel_reduce_sum( } detail::parallel_reduce_sum_impl( - range, total, output_ptr, std::forward(contribution), scratch, scratch_bytes + range, total, output_ptr, std::forward(contribution) ); }