diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt index 3bdf14500..e6e66caaf 100755 --- a/cpp/CMakeLists.txt +++ b/cpp/CMakeLists.txt @@ -486,6 +486,13 @@ if (BUILD_BENCHMARK) ${PROJECT_SRC_DIR}) target_link_libraries(alp_vs_gorilla_benchmark PRIVATE tsfile Threads::Threads) + + add_executable(sprintz_batch_decode_benchmark + bench_mark/sprintz_batch_decode_benchmark.cc) + target_include_directories(sprintz_batch_decode_benchmark PRIVATE + ${PROJECT_SRC_DIR}) + target_link_libraries(sprintz_batch_decode_benchmark PRIVATE + tsfile Threads::Threads) endif () if (BUILD_TOOLS) add_subdirectory(tools) diff --git a/cpp/bench_mark/sprintz_batch_decode_benchmark.cc b/cpp/bench_mark/sprintz_batch_decode_benchmark.cc new file mode 100644 index 000000000..e8feb1866 --- /dev/null +++ b/cpp/bench_mark/sprintz_batch_decode_benchmark.cc @@ -0,0 +1,404 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +// SPRINTZ decode throughput: the batch overrides against the per-value +// loop they replace, for all four value types. +// +// ./sprintz_batch_decode_benchmark [values] [reps] [batch] +// +// The per-value arm calls storage::Decoder::read_batch_* qualified, which is +// exactly the loop that ran before the SPRINTZ overrides existed (it reads one +// value per virtual call). Built from a revision without the overrides, this +// file measures the old path unchanged, which isolates the per-block +// allocation removal from the batch loop itself. + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "common/allocator/byte_stream.h" +#include "common/db_common.h" +#include "encoding/double_sprintz_decoder.h" +#include "encoding/double_sprintz_encoder.h" +#include "encoding/float_sprintz_decoder.h" +#include "encoding/float_sprintz_encoder.h" +#include "encoding/int32_sprintz_decoder.h" +#include "encoding/int32_sprintz_encoder.h" +#include "encoding/int64_sprintz_decoder.h" +#include "encoding/int64_sprintz_encoder.h" + +namespace { + +using Clock = std::chrono::high_resolution_clock; + +template +struct BatchApi; + +template <> +struct BatchApi { + typedef storage::Int32SprintzEncoder Encoder; + typedef storage::Int32SprintzDecoder Decoder; + + static int Fast(Decoder& decoder, int32_t* out, int capacity, int& actual, + common::ByteStream& in) { + return decoder.read_batch_int32(out, capacity, actual, in); + } + + static int PerValue(Decoder& decoder, int32_t* out, int capacity, + int& actual, common::ByteStream& in) { + return decoder.Decoder::read_batch_int32(out, capacity, actual, in); + } +}; + +template <> +struct BatchApi { + typedef storage::Int64SprintzEncoder Encoder; + typedef storage::Int64SprintzDecoder Decoder; + + static int Fast(Decoder& decoder, int64_t* out, int capacity, int& actual, + common::ByteStream& in) { + return decoder.read_batch_int64(out, capacity, actual, in); + } + + static int PerValue(Decoder& decoder, int64_t* out, int capacity, + int& actual, common::ByteStream& in) { + return decoder.Decoder::read_batch_int64(out, capacity, actual, in); + } +}; + +template <> +struct BatchApi { + typedef storage::FloatSprintzEncoder Encoder; + typedef storage::FloatSprintzDecoder Decoder; + + static int Fast(Decoder& decoder, float* out, int capacity, int& actual, + common::ByteStream& in) { + return decoder.read_batch_float(out, capacity, actual, in); + } + + static int PerValue(Decoder& decoder, float* out, int capacity, int& actual, + common::ByteStream& in) { + return decoder.Decoder::read_batch_float(out, capacity, actual, in); + } +}; + +template <> +struct BatchApi { + typedef storage::DoubleSprintzEncoder Encoder; + typedef storage::DoubleSprintzDecoder Decoder; + + static int Fast(Decoder& decoder, double* out, int capacity, int& actual, + common::ByteStream& in) { + return decoder.read_batch_double(out, capacity, actual, in); + } + + static int PerValue(Decoder& decoder, double* out, int capacity, + int& actual, common::ByteStream& in) { + return decoder.Decoder::read_batch_double(out, capacity, actual, in); + } +}; + +template +std::vector EncodeAll(const std::vector& values, + uint64_t& encoded_bytes) { + typedef typename BatchApi::Encoder Encoder; + common::ByteStream stream(1 << 16, common::MOD_ENCODER_OBJ); + Encoder encoder; + for (size_t i = 0; i < values.size(); ++i) { + encoder.encode(values[i], stream); + } + if (encoder.flush(stream) != common::E_OK) { + std::cerr << "encode failed" << std::endl; + std::exit(1); + } + std::vector bytes; + bytes.reserve(static_cast(stream.total_size())); + common::ByteStream::BufferIterator iter = stream.init_buffer_iterator(); + while (true) { + common::ByteStream::Buffer buffer = iter.get_next_buf(); + if (buffer.buf_ == nullptr || buffer.len_ == 0) { + break; + } + const uint8_t* begin = reinterpret_cast(buffer.buf_); + bytes.insert(bytes.end(), begin, begin + buffer.len_); + } + encoded_bytes = bytes.size(); + return bytes; +} + +// One timed decode of the whole stream on the given arm. +template +double DecodeOnce(const std::vector& bytes, int batch, + int (*read)(typename BatchApi::Decoder&, T*, int, int&, + common::ByteStream&), + uint64_t& decoded) { + typedef typename BatchApi::Decoder Decoder; + common::ByteStream stream(1 << 16, common::MOD_DECODER_OBJ); + stream.write_buf(reinterpret_cast(bytes.data()), + static_cast(bytes.size())); + Decoder decoder; + std::vector buffer(static_cast(batch)); + decoded = 0; + const Clock::time_point start = Clock::now(); + while (decoder.has_remaining(stream)) { + int actual = 0; + if (read(decoder, buffer.data(), batch, actual, stream) != + common::E_OK) { + std::cerr << "decode failed" << std::endl; + std::exit(1); + } + if (actual <= 0) { + break; + } + decoded += static_cast(actual); + } + const Clock::time_point end = Clock::now(); + return std::chrono::duration_cast>(end - + start) + .count(); +} + +template +bool DecodeEqual(const std::vector& bytes, int batch, + std::vector& fast_out, std::vector& per_value_out) { + typedef typename BatchApi::Decoder Decoder; + // Decode with each arm into scratch streams and compare value by value. + common::ByteStream fast_stream(1 << 16, common::MOD_DECODER_OBJ); + common::ByteStream slow_stream(1 << 16, common::MOD_DECODER_OBJ); + fast_stream.write_buf(reinterpret_cast(bytes.data()), + static_cast(bytes.size())); + slow_stream.write_buf(reinterpret_cast(bytes.data()), + static_cast(bytes.size())); + Decoder fast_decoder; + Decoder slow_decoder; + fast_out.clear(); + per_value_out.clear(); + std::vector a(static_cast(batch)); + std::vector b(static_cast(batch)); + while (fast_decoder.has_remaining(fast_stream)) { + int fast_actual = 0; + int slow_actual = 0; + if (BatchApi::Fast(fast_decoder, a.data(), batch, fast_actual, + fast_stream) != common::E_OK || + BatchApi::PerValue(slow_decoder, b.data(), batch, slow_actual, + slow_stream) != common::E_OK) { + return false; + } + if (fast_actual != slow_actual) { + return false; + } + if (fast_actual <= 0) { + break; + } + fast_out.insert(fast_out.end(), a.begin(), a.begin() + fast_actual); + per_value_out.insert(per_value_out.end(), b.begin(), + b.begin() + slow_actual); + } + if (fast_out.size() != per_value_out.size()) { + return false; + } + return std::memcmp(fast_out.data(), per_value_out.data(), + fast_out.size() * sizeof(T)) == 0; +} + +template +void RunPattern(const std::string& type_name, const std::string& pattern, + const std::vector& values, int batch, int reps) { + typedef typename BatchApi::Decoder Decoder; + uint64_t encoded_bytes = 0; + std::vector bytes = EncodeAll(values, encoded_bytes); + + std::vector fast_out; + std::vector slow_out; + if (!DecodeEqual(bytes, batch, fast_out, slow_out)) { + std::cout << type_name << " " << pattern + << ": VALUE MISMATCH between arms" << std::endl; + } + + double best_fast = 1e9; + double best_slow = 1e9; + uint64_t decoded_fast = 0; + uint64_t decoded_slow = 0; + for (int rep = 0; rep < reps; ++rep) { + best_fast = std::min( + best_fast, + DecodeOnce(bytes, batch, &BatchApi::Fast, decoded_fast)); + best_slow = std::min( + best_slow, + DecodeOnce(bytes, batch, &BatchApi::PerValue, decoded_slow)); + } + if (decoded_fast != decoded_slow) { + std::cout << type_name << " " << pattern + << ": COUNT MISMATCH between arms" << std::endl; + } + + const double fast_rates = decoded_fast / best_fast; + const double slow_rates = decoded_slow / best_slow; + std::cout << std::left << std::setw(8) << type_name << std::setw(12) + << pattern << std::right << std::setw(10) << decoded_fast + << std::setw(12) << encoded_bytes << std::setw(16) << std::fixed + << std::setprecision(1) << slow_rates << std::setw(16) + << fast_rates << std::setw(10) << std::setprecision(2) + << (fast_rates / slow_rates) << "x" << std::endl; +} + +template +std::vector SmoothValues(int count, uint32_t seed); + +template <> +std::vector SmoothValues(int count, uint32_t seed) { + std::mt19937 rng(seed); + std::vector out; + out.reserve(static_cast(count)); + int32_t value = 20000; + for (int i = 0; i < count; ++i) { + value += static_cast(rng() % 11) - 5; + out.push_back(value); + } + return out; +} + +template <> +std::vector SmoothValues(int count, uint32_t seed) { + std::mt19937_64 rng(seed); + std::vector out; + out.reserve(static_cast(count)); + int64_t value = 1732000000000LL; // epoch milliseconds + for (int i = 0; i < count; ++i) { + value += 1000 + static_cast(rng() % 7) - 3; + out.push_back(value); + } + return out; +} + +template <> +std::vector SmoothValues(int count, uint32_t seed) { + std::mt19937 rng(seed); + std::vector out; + out.reserve(static_cast(count)); + float value = 20.0f; + for (int i = 0; i < count; ++i) { + value += 0.01f * static_cast(static_cast(rng() % 21) - 10); + out.push_back(value); + } + return out; +} + +template <> +std::vector SmoothValues(int count, uint32_t seed) { + std::mt19937_64 rng(seed); + std::vector out; + out.reserve(static_cast(count)); + double value = 1000000.0; + for (int i = 0; i < count; ++i) { + value += 0.001 * static_cast(static_cast(rng() % 21) - 10); + out.push_back(value); + } + return out; +} + +template +std::vector HighEntropyValues(int count, uint32_t seed); + +template <> +std::vector HighEntropyValues(int count, uint32_t seed) { + std::mt19937 rng(seed); + std::uniform_int_distribution dist(-250000000, 250000000); + std::vector out; + out.reserve(static_cast(count)); + for (int i = 0; i < count; ++i) { + out.push_back(dist(rng)); + } + return out; +} + +template <> +std::vector HighEntropyValues(int count, uint32_t seed) { + std::mt19937_64 rng(seed); + std::uniform_int_distribution dist(-1000000000000000LL, + 1000000000000000LL); + std::vector out; + out.reserve(static_cast(count)); + for (int i = 0; i < count; ++i) { + out.push_back(dist(rng)); + } + return out; +} + +template <> +std::vector HighEntropyValues(int count, uint32_t seed) { + std::mt19937 rng(seed); + std::uniform_real_distribution dist(1.0f, 8.0f); + std::vector out; + out.reserve(static_cast(count)); + for (int i = 0; i < count; ++i) { + out.push_back(dist(rng)); + } + return out; +} + +template <> +std::vector HighEntropyValues(int count, uint32_t seed) { + std::mt19937_64 rng(seed); + std::uniform_real_distribution dist(1.0, 8.0); + std::vector out; + out.reserve(static_cast(count)); + for (int i = 0; i < count; ++i) { + out.push_back(dist(rng)); + } + return out; +} + +template +void RunType(const std::string& type_name, int count, int batch, int reps) { + RunPattern(type_name, "smooth", SmoothValues(count, 1), batch, reps); + RunPattern(type_name, "entropy", HighEntropyValues(count, 2), batch, + reps); + RunPattern(type_name, "constant", std::vector(count, T(0)), batch, + reps); +} + +} // namespace + +int main(int argc, char** argv) { + int count = argc > 1 ? std::atoi(argv[1]) : 4000000; + int reps = argc > 2 ? std::atoi(argv[2]) : 5; + int batch = argc > 3 ? std::atoi(argv[3]) : 129; + + std::cout << "SPRINTZ decode, " << count << " values, best of " << reps + << ", batch capacity " << batch << std::endl; + std::cout << std::left << std::setw(8) << "type" << std::setw(12) + << "pattern" << std::right << std::setw(10) << "values" + << std::setw(12) << "bytes" << std::setw(16) << "per-value/s" + << std::setw(16) << "batch/s" << std::setw(10) << "speedup" + << std::endl; + + RunType("INT32", count, batch, reps); + RunType("INT64", count, batch, reps); + RunType("FLOAT", count, batch, reps); + RunType("DOUBLE", count, batch, reps); + return 0; +} diff --git a/cpp/src/encoding/double_sprintz_decoder.h b/cpp/src/encoding/double_sprintz_decoder.h index 69bf5ba87..bee5ecece 100644 --- a/cpp/src/encoding/double_sprintz_decoder.h +++ b/cpp/src/encoding/double_sprintz_decoder.h @@ -36,15 +36,16 @@ namespace storage { class DoubleSprintzDecoder : public SprintzDecoder { public: - DoubleSprintzDecoder() : fire_pred_(3), predict_scheme_("fire") { + DoubleSprintzDecoder() + : pre_value_(0), + current_value_(0.0), + current_buffer_(block_size_ + 1), + convert_buffer_(block_size_), + packer_(0), + pack_buf_(block_size_ * sizeof(int64_t) + 8, 0), + fire_pred_(3), + predict_scheme_("fire") { SprintzDecoder::reset(); - current_buffer_.resize(block_size_ + 1); - convert_buffer_.resize(block_size_); - pre_value_ = 0; - current_value_ = 0.0; - current_count_ = 0; - decode_size_ = 0; - is_block_read_ = false; std::fill(current_buffer_.begin(), current_buffer_.end(), 0.0); std::fill(convert_buffer_.begin(), convert_buffer_.end(), 0); fire_pred_.reset(); @@ -73,12 +74,20 @@ class DoubleSprintzDecoder : public SprintzDecoder { } } ret_value = current_buffer_[current_count_++]; - if (current_count_ == (size_t)decode_size_) { + if (current_count_ == decode_size_) { is_block_read_ = false; current_count_ = 0; } return ret; } + + int read_batch_double(double* out, int capacity, int& actual, + common::ByteStream& in) override { + // has_remaining() compares current_count_ against decode_size_, so a + // pending block is readable to its end and needs no data gate. + return batch_read(out, capacity, actual, current_buffer_.data(), in, + false, sizeof(uint32_t) + 1); + } int read_float(float& ret_value, common::ByteStream& in) override { return common::E_TYPE_NOT_MATCH; } @@ -121,7 +130,7 @@ class DoubleSprintzDecoder : public SprintzDecoder { if ((bit_width_ & (1 << 7)) != 0) { decode_size_ = bit_width_ & ~(1 << 7); DoubleGorillaDecoder decoder; - for (size_t i = 0; i < decode_size_; ++i) { + for (int i = 0; i < decode_size_; ++i) { if (RET_FAIL(decoder.read_double(current_buffer_[i], input))) { return ret; } @@ -135,18 +144,19 @@ class DoubleSprintzDecoder : public SprintzDecoder { decode_size_ = block_size_ + 1; common::SerializationUtil::read_double(pre_value_, input); current_buffer_[0] = pre_value_; - // +8 zero bytes: the SIMD unpack gathers 8 bytes per value and - // may read up to 7 bytes past the packed group. - std::vector pack_buf(bit_width_ + 8, 0); uint32_t read_len = 0; - input.read_buf(reinterpret_cast(pack_buf.data()), bit_width_, - read_len); - packer_ = std::make_shared(bit_width_); - std::vector tmp_buffer(block_size_); - packer_->unpack_8values(pack_buf.data(), 0, tmp_buffer.data()); - for (int i = 0; i < block_size_; ++i) { - convert_buffer_[i] = tmp_buffer[i]; + input.read_buf(reinterpret_cast(pack_buf_.data()), + bit_width_, read_len); + // The SIMD unpack gathers 8 bytes per value and may read up to 7 + // bytes past the packed group. Zero everything the read did not + // fill so a reused buffer never carries a previous block's bytes + // into the padding or into a short read's tail. + if (read_len < pack_buf_.size()) { + std::memset(pack_buf_.data() + read_len, 0, + pack_buf_.size() - read_len); } + packer_.set_width(bit_width_); + packer_.unpack_8values(pack_buf_.data(), 0, convert_buffer_.data()); ret = recalculate(); } is_block_read_ = true; @@ -214,13 +224,11 @@ class DoubleSprintzDecoder : public SprintzDecoder { private: double pre_value_; double current_value_; - size_t current_count_; - size_t decode_size_; - bool is_block_read_ = false; std::vector current_buffer_; std::vector convert_buffer_; - std::shared_ptr packer_; + Int64Packer packer_; + std::vector pack_buf_; LongFire fire_pred_; std::string predict_scheme_; }; diff --git a/cpp/src/encoding/float_sprintz_decoder.h b/cpp/src/encoding/float_sprintz_decoder.h index 3dd42f0f7..7cabb18ca 100644 --- a/cpp/src/encoding/float_sprintz_decoder.h +++ b/cpp/src/encoding/float_sprintz_decoder.h @@ -36,15 +36,16 @@ namespace storage { class FloatSprintzDecoder : public SprintzDecoder { public: - FloatSprintzDecoder() : fire_pred_(2), predict_scheme_("fire") { + FloatSprintzDecoder() + : pre_value_(0), + current_value_(0.0f), + current_buffer_(block_size_ + 1), + convert_buffer_(block_size_), + packer_(0), + pack_buf_(block_size_ * sizeof(int32_t) + 8, 0), + fire_pred_(2), + predict_scheme_("fire") { SprintzDecoder::reset(); - current_buffer_.resize(block_size_ + 1); - convert_buffer_.resize(block_size_); - pre_value_ = 0; - current_value_ = 0.0f; - current_count_ = 0; - decode_size_ = 0; - is_block_read_ = false; std::fill(current_buffer_.begin(), current_buffer_.end(), 0.0f); std::fill(convert_buffer_.begin(), convert_buffer_.end(), 0); fire_pred_.reset(); @@ -106,6 +107,14 @@ class FloatSprintzDecoder : public SprintzDecoder { return ret; } + int read_batch_float(float* out, int capacity, int& actual, + common::ByteStream& in) override { + // has_remaining() compares current_count_ against decode_size_, so a + // pending block is readable to its end and needs no data gate. + return batch_read(out, capacity, actual, current_buffer_.data(), in, + false, sizeof(uint32_t) + 1); + } + protected: int decode_block(common::ByteStream& input) override { // read header bitWidth @@ -122,7 +131,7 @@ class FloatSprintzDecoder : public SprintzDecoder { if ((bit_width_ & (1 << 7)) != 0) { decode_size_ = bit_width_ & ~(1 << 7); FloatGorillaDecoder decoder; - for (size_t i = 0; i < decode_size_; ++i) { + for (int i = 0; i < decode_size_; ++i) { if (RET_FAIL(decoder.read_float(current_buffer_[i], input))) { return ret; } @@ -138,19 +147,19 @@ class FloatSprintzDecoder : public SprintzDecoder { common::SerializationUtil::read_float(pre_value_, input); current_buffer_[0] = pre_value_; // read packed data - // +8 zero bytes: the SIMD unpack gathers 8 bytes per value and - // may read up to 7 bytes past the packed group. - std::vector pack_buf(bit_width_ + 8, 0); uint32_t read_len = 0; - input.read_buf(reinterpret_cast(pack_buf.data()), bit_width_, - read_len); - packer_ = std::make_shared(bit_width_); - std::vector tmp_buffer(block_size_); - packer_->unpack_8values(pack_buf.data(), 0, tmp_buffer.data()); - // move into convert_buffer_ - for (int i = 0; i < block_size_; ++i) { - convert_buffer_[i] = tmp_buffer[i]; + input.read_buf(reinterpret_cast(pack_buf_.data()), + bit_width_, read_len); + // The SIMD unpack gathers 8 bytes per value and may read up to 7 + // bytes past the packed group. Zero everything the read did not + // fill so a reused buffer never carries a previous block's bytes + // into the padding or into a short read's tail. + if (read_len < pack_buf_.size()) { + std::memset(pack_buf_.data() + read_len, 0, + pack_buf_.size() - read_len); } + packer_.set_width(bit_width_); + packer_.unpack_8values(pack_buf_.data(), 0, convert_buffer_.data()); ret = recalculate(); } is_block_read_ = true; @@ -227,13 +236,11 @@ class FloatSprintzDecoder : public SprintzDecoder { private: float pre_value_; float current_value_; - size_t current_count_; - size_t decode_size_; - bool is_block_read_ = false; std::vector current_buffer_; std::vector convert_buffer_; - std::shared_ptr packer_; + Int32Packer packer_; + std::vector pack_buf_; IntFire fire_pred_; std::string predict_scheme_; }; diff --git a/cpp/src/encoding/int32_sprintz_decoder.h b/cpp/src/encoding/int32_sprintz_decoder.h index 7f4016fc3..d361acecd 100644 --- a/cpp/src/encoding/int32_sprintz_decoder.h +++ b/cpp/src/encoding/int32_sprintz_decoder.h @@ -20,6 +20,7 @@ #ifndef INT32_SPRINTZ_DECODER_H #define INT32_SPRINTZ_DECODER_H +#include #include #include #include @@ -37,10 +38,12 @@ namespace storage { class Int32SprintzDecoder : public SprintzDecoder { public: Int32SprintzDecoder() - : fire_pred_(2), + : packer_(0), + fire_pred_(2), pre_value_(0), current_value_(0), current_buffer_(block_size_ + 1), + pack_buf_(block_size_ * sizeof(int32_t) + 8, 0), predict_scheme_("fire") { SprintzDecoder::reset(); current_value_ = 0; @@ -94,6 +97,12 @@ class Int32SprintzDecoder : public SprintzDecoder { return ret; } + int read_batch_int32(int32_t* out, int capacity, int& actual, + common::ByteStream& in) override { + return batch_read(out, capacity, actual, current_buffer_.data(), in, + true, sizeof(int32_t) + 1); + } + void reset() override { SprintzDecoder::reset(); current_value_ = 0; @@ -141,20 +150,22 @@ class Int32SprintzDecoder : public SprintzDecoder { pre_value_ = tmp_prev_value; current_buffer_[0] = pre_value_; - // +8 zero bytes: the SIMD unpack gathers 8 bytes per value and - // may read up to 7 bytes past the packed group. - std::vector pack_buf(bit_width_ + 8, 0); uint32_t read_len = 0; - input.read_buf(reinterpret_cast(pack_buf.data()), bit_width_, - read_len); + input.read_buf(reinterpret_cast(pack_buf_.data()), + bit_width_, read_len); + // The SIMD unpack gathers 8 bytes per value and may read up to 7 + // bytes past the packed group. Zero everything the read did not + // fill so a reused buffer never carries a previous block's bytes + // into the padding or into a short read's tail. + if (read_len < pack_buf_.size()) { + std::memset(pack_buf_.data() + read_len, 0, + pack_buf_.size() - read_len); + } - std::vector tmp_buffer(8); - packer_ = std::make_shared(bit_width_); - packer_->unpack_8values(pack_buf.data(), 0, tmp_buffer.data()); + packer_.set_width(bit_width_); + packer_.unpack_8values(pack_buf_.data(), 0, + current_buffer_.data() + 1); - for (int i = 0; i < 8; ++i) { - current_buffer_[i + 1] = tmp_buffer[i]; - } ret = recalculate(); } is_block_read_ = true; @@ -191,11 +202,12 @@ class Int32SprintzDecoder : public SprintzDecoder { } private: - std::shared_ptr packer_; + Int32Packer packer_; IntFire fire_pred_; int32_t pre_value_; int32_t current_value_; std::vector current_buffer_; + std::vector pack_buf_; std::string predict_scheme_; }; diff --git a/cpp/src/encoding/int64_sprintz_decoder.h b/cpp/src/encoding/int64_sprintz_decoder.h index 71363f756..e7870fd73 100644 --- a/cpp/src/encoding/int64_sprintz_decoder.h +++ b/cpp/src/encoding/int64_sprintz_decoder.h @@ -22,6 +22,7 @@ #include #include +#include #include #include #include @@ -37,10 +38,12 @@ namespace storage { class Int64SprintzDecoder : public SprintzDecoder { public: Int64SprintzDecoder() - : fire_pred_(3), + : packer_(0), + fire_pred_(3), pre_value_(0), current_value_(0), current_buffer_(block_size_ + 1), + pack_buf_(block_size_ * sizeof(int64_t) + 8, 0), predict_scheme_("fire") { SprintzDecoder::reset(); current_count_ = 0; @@ -107,6 +110,13 @@ class Int64SprintzDecoder : public SprintzDecoder { return ret; } + int read_batch_int64(int64_t* out, int capacity, int& actual, + common::ByteStream& in) override { + // has_remaining() here only asks for any unread byte. + return batch_read(out, capacity, actual, current_buffer_.data(), in, + true, 1); + } + protected: int decode_block(common::ByteStream& input) override { // read header bitWidth @@ -140,21 +150,22 @@ class Int64SprintzDecoder : public SprintzDecoder { current_buffer_[0] = pre_value_; // Read packed buffer - // +8 zero bytes: the SIMD unpack gathers 8 bytes per value and - // may read up to 7 bytes past the packed group. - std::vector pack_buf(bit_width_ + 8, 0); uint32_t read_len = 0; - input.read_buf(reinterpret_cast(pack_buf.data()), bit_width_, - read_len); - - std::vector tmp_buffer(8); - packer_ = std::make_shared(bit_width_); - packer_->unpack_8values(pack_buf.data(), 0, tmp_buffer.data()); - - for (int i = 0; i < 8; ++i) { - current_buffer_[i + 1] = tmp_buffer[i]; + input.read_buf(reinterpret_cast(pack_buf_.data()), + bit_width_, read_len); + // The SIMD unpack gathers 8 bytes per value and may read up to 7 + // bytes past the packed group. Zero everything the read did not + // fill so a reused buffer never carries a previous block's bytes + // into the padding or into a short read's tail. + if (read_len < pack_buf_.size()) { + std::memset(pack_buf_.data() + read_len, 0, + pack_buf_.size() - read_len); } + packer_.set_width(bit_width_); + packer_.unpack_8values(pack_buf_.data(), 0, + current_buffer_.data() + 1); + ret = recalculate(); } @@ -192,11 +203,12 @@ class Int64SprintzDecoder : public SprintzDecoder { } private: - std::shared_ptr packer_; + Int64Packer packer_; LongFire fire_pred_; int64_t pre_value_; int64_t current_value_; std::vector current_buffer_; + std::vector pack_buf_; std::string predict_scheme_; }; diff --git a/cpp/src/encoding/sprintz_decoder.h b/cpp/src/encoding/sprintz_decoder.h index 3ad30d4ee..fca18278d 100644 --- a/cpp/src/encoding/sprintz_decoder.h +++ b/cpp/src/encoding/sprintz_decoder.h @@ -21,6 +21,7 @@ #define SPRINTZ_DECODER_H #include +#include #include #include #include @@ -57,6 +58,72 @@ class SprintzDecoder : public Decoder { current_count_(0), decode_size_(0) {} + protected: + // Copy already-reconstructed values into `out`, decoding further blocks as + // needed. The state machine is the per-value one, so a batch emits exactly + // the values (and leaves exactly the residual block state) that the same + // number of read_*() calls would: no value is emitted that the per-value + // has_remaining() gate would have stopped at, and a partially consumed + // block is left pending. + // + // `buffer` is the subclass's decoded-block buffer. A full block holds + // block_size_ + 1 values: buffer[0] is the block's first value, stored + // verbatim, and buffer[1 .. block_size_] are the reconstructed rest. A + // short block holds decode_size_ values. + // + // `min_block_bytes` is how much unread data the subclass's has_remaining() + // needs before a new block may be started. `gate_tail` mirrors the other + // half of that check: INT32/INT64 has_remaining() reports a pending block + // as readable only while current_count_ < block_size_, so a full block's + // last value is returned only when the stream still has data; FLOAT/DOUBLE + // compare against decode_size_ and return it unconditionally. + template + int batch_read(T* out, int capacity, int& actual, T* buffer, + common::ByteStream& in, bool gate_tail, + uint64_t min_block_bytes) { + actual = 0; + if (capacity <= 0) { + return common::E_OK; + } + if (out == nullptr) { + return common::E_INVALID_ARG; + } + int ret = common::E_OK; + while (actual < capacity) { + if (!is_block_read_) { + if (in.remaining_size() < min_block_bytes) { + break; + } + if (RET_FAIL(decode_block(in))) { + return ret; + } + } + int usable = decode_size_ - current_count_; + if (usable > 0 && current_count_ + usable > block_size_) { + // The full block's last value (index block_size_) is the one + // the per-value gate only returns while more data follows. + usable = block_size_ - current_count_; + if (!gate_tail || in.remaining_size() >= min_block_bytes) { + ++usable; + } + } + if (usable <= 0) { + break; + } + const int n = + usable < capacity - actual ? usable : capacity - actual; + std::memcpy(out + actual, buffer + current_count_, + static_cast(n) * sizeof(T)); + actual += n; + current_count_ += n; + if (current_count_ == decode_size_) { + is_block_read_ = false; + current_count_ = 0; + } + } + return common::E_OK; + } + protected: int bit_width_; // Current bit width being used int block_size_; // Default is 8 diff --git a/cpp/test/encoding/sprintz_codec_test.cc b/cpp/test/encoding/sprintz_codec_test.cc index ec43ff406..f1dbcc81e 100644 --- a/cpp/test/encoding/sprintz_codec_test.cc +++ b/cpp/test/encoding/sprintz_codec_test.cc @@ -18,9 +18,13 @@ */ #include +#include #include #include #include +#include +#include +#include #include "common/allocator/byte_stream.h" #include "encoding/double_sprintz_decoder.h" @@ -384,4 +388,929 @@ TEST_F(SprintzCodecTest, DoubleExtremeValues) { ASSERT_FALSE(decoder.has_remaining(stream)); } +// ── Batch decode vs per-value decode ───────────────────────────────────── +// +// A SPRINTZ decoder reconstructs one block at a time into a buffer and serves +// values out of it. The batch overrides copy whole runs out of that buffer, +// so they must reproduce the per-value state machine exactly: same values, +// same stop points, same residual block state. These tests pin the two +// implementations to each other on the same byte stream. + +int ReadOne(Int32SprintzDecoder& decoder, int32_t& value, ByteStream& in) { + return decoder.read_int32(value, in); +} + +int ReadOne(Int64SprintzDecoder& decoder, int64_t& value, ByteStream& in) { + return decoder.read_int64(value, in); +} + +int ReadOne(FloatSprintzDecoder& decoder, float& value, ByteStream& in) { + return decoder.read_float(value, in); +} + +int ReadOne(DoubleSprintzDecoder& decoder, double& value, ByteStream& in) { + return decoder.read_double(value, in); +} + +int ReadBatch(Int32SprintzDecoder& decoder, int32_t* out, int capacity, + int& actual, ByteStream& in) { + return decoder.read_batch_int32(out, capacity, actual, in); +} + +int ReadBatch(Int64SprintzDecoder& decoder, int64_t* out, int capacity, + int& actual, ByteStream& in) { + return decoder.read_batch_int64(out, capacity, actual, in); +} + +int ReadBatch(FloatSprintzDecoder& decoder, float* out, int capacity, + int& actual, ByteStream& in) { + return decoder.read_batch_float(out, capacity, actual, in); +} + +int ReadBatch(DoubleSprintzDecoder& decoder, double* out, int capacity, + int& actual, ByteStream& in) { + return decoder.read_batch_double(out, capacity, actual, in); +} + +// The loop Decoder::read_batch_* ran before the SPRINTZ overrides existed. +// This is the reference behaviour the overrides must reproduce. +template +int ReferenceBatch(DecoderT& decoder, T* out, int capacity, int& actual, + ByteStream& in) { + actual = 0; + T value; + while (actual < capacity && decoder.has_remaining(in)) { + int ret = ReadOne(decoder, value, in); + if (ret != E_OK) { + return ret; + } + out[actual++] = value; + } + return E_OK; +} + +template +struct SprintzTypes; + +template <> +struct SprintzTypes { + typedef Int32SprintzEncoder Encoder; + typedef Int32SprintzDecoder Decoder; +}; + +template <> +struct SprintzTypes { + typedef Int64SprintzEncoder Encoder; + typedef Int64SprintzDecoder Decoder; +}; + +template <> +struct SprintzTypes { + typedef FloatSprintzEncoder Encoder; + typedef FloatSprintzDecoder Decoder; +}; + +template <> +struct SprintzTypes { + typedef DoubleSprintzEncoder Encoder; + typedef DoubleSprintzDecoder Decoder; +}; + +template +void EncodeAll(const std::vector& values, ByteStream& out) { + typename SprintzTypes::Encoder encoder; + for (size_t i = 0; i < values.size(); ++i) { + ASSERT_EQ(E_OK, encoder.encode(values[i], out)); + } + ASSERT_EQ(E_OK, encoder.flush(out)); +} + +// Values are float/double with possible NaNs, so compare bit patterns. +template +void ExpectSameBits(const T& expected, const T& actual, const char* what) { + ASSERT_EQ(0, std::memcmp(&expected, &actual, sizeof(T))) << what; +} + +int ReadExact(Int32SprintzDecoder& decoder, int32_t* out, int count, + ByteStream& in) { + return decoder.read_exact_int32(out, count, in); +} + +int ReadExact(Int64SprintzDecoder& decoder, int64_t* out, int count, + ByteStream& in) { + return decoder.read_exact_int64(out, count, in); +} + +int ReadExact(FloatSprintzDecoder& decoder, float* out, int count, + ByteStream& in) { + return decoder.read_exact_float(out, count, in); +} + +int ReadExact(DoubleSprintzDecoder& decoder, double* out, int count, + ByteStream& in) { + return decoder.read_exact_double(out, count, in); +} + +std::vector RepeatScript(const std::vector& base, int rounds) { + std::vector script; + script.reserve(static_cast(rounds)); + for (int i = 0; i < rounds; ++i) { + script.push_back(base[static_cast(i) % base.size()]); + } + return script; +} + +// Compare batch reads against the per-value loop, round by round on identical +// streams, including a per-value read between rounds so a batch that stopped +// at a gated block tail is resumed on both paths. +template +void CheckBatchParity(const std::vector& values, + const std::vector& capacities, + uint32_t page_size = 1024) { + typedef typename SprintzTypes::Decoder Decoder; + + ByteStream ref_in(page_size, MOD_ENCODER_OBJ); + ByteStream bat_in(page_size, MOD_ENCODER_OBJ); + EncodeAll(values, ref_in); + EncodeAll(values, bat_in); + ASSERT_EQ(ref_in.total_size(), bat_in.total_size()); + + Decoder ref_dec; + Decoder bat_dec; + std::vector emitted; + for (size_t round = 0; round < capacities.size(); ++round) { + const int capacity = capacities[round]; + const size_t buf_len = static_cast(capacity > 0 ? capacity : 1); + std::vector ref_buf(buf_len); + std::vector bat_buf(buf_len); + int ref_actual = -1; + int bat_actual = -1; + int ref_ret = ReferenceBatch(ref_dec, ref_buf.data(), capacity, + ref_actual, ref_in); + int bat_ret = + ReadBatch(bat_dec, bat_buf.data(), capacity, bat_actual, bat_in); + + SCOPED_TRACE(::testing::Message() + << "round " << round << " capacity " << capacity); + ASSERT_EQ(ref_ret, bat_ret); + ASSERT_EQ(ref_actual, bat_actual); + for (int i = 0; i < bat_actual; ++i) { + ExpectSameBits(ref_buf[i], bat_buf[i], "batch value"); + emitted.push_back(bat_buf[i]); + } + ASSERT_EQ(ref_in.remaining_size(), bat_in.remaining_size()); + ASSERT_EQ(ref_dec.has_remaining(ref_in), bat_dec.has_remaining(bat_in)); + + if (ref_dec.has_remaining(ref_in)) { + T ref_value = T(); + T bat_value = T(); + int ref_one = ReadOne(ref_dec, ref_value, ref_in); + int bat_one = ReadOne(bat_dec, bat_value, bat_in); + ASSERT_EQ(ref_one, bat_one); + ASSERT_EQ(E_OK, ref_one); + ExpectSameBits(ref_value, bat_value, "handoff value"); + emitted.push_back(bat_value); + } + } + + // Drain the rest per value; the residuals must still agree. + while (ref_dec.has_remaining(ref_in)) { + ASSERT_TRUE(bat_dec.has_remaining(bat_in)); + T ref_value = T(); + T bat_value = T(); + ASSERT_EQ(ReadOne(ref_dec, ref_value, ref_in), + ReadOne(bat_dec, bat_value, bat_in)); + ExpectSameBits(ref_value, bat_value, "drained value"); + emitted.push_back(bat_value); + } + ASSERT_FALSE(bat_dec.has_remaining(bat_in)); + ASSERT_EQ(ref_in.remaining_size(), bat_in.remaining_size()); + + // What both paths produced is a prefix of the encoded sequence (a stream + // that ends on a full block leaves its last value unreadable per value). + ASSERT_LE(emitted.size(), values.size()); + for (size_t i = 0; i < emitted.size(); ++i) { + ExpectSameBits(values[i], emitted[i], "emitted value"); + } +} + +template +struct ValueSets { + std::vector smooth; + std::vector random; + std::vector constant; + std::vector jumps; +}; + +template +ValueSets MakeValueSets(int count, uint32_t seed); + +template <> +ValueSets MakeValueSets(int count, uint32_t seed) { + std::mt19937 rng(seed); + // Delta magnitudes stay below 2^31 so the encoder's zigzag step (2 * d) + // does not overflow. + std::uniform_int_distribution wide(-250000000, 250000000); + ValueSets sets; + int32_t smooth = 1000; + for (int i = 0; i < count; ++i) { + smooth += 1 + static_cast(rng() % 7); + sets.smooth.push_back(smooth); + sets.random.push_back(wide(rng)); + sets.constant.push_back(-7); + sets.jumps.push_back((i % 2 == 0) ? 500000000 : -500000000); + } + return sets; +} + +template <> +ValueSets MakeValueSets(int count, uint32_t seed) { + std::mt19937_64 rng(seed); + std::uniform_int_distribution wide(-1000000000000000LL, + 1000000000000000LL); + ValueSets sets; + int64_t smooth = 1600000000000LL; + for (int i = 0; i < count; ++i) { + smooth += 1 + static_cast(rng() % 97); + sets.smooth.push_back(smooth); + sets.random.push_back(wide(rng)); + sets.constant.push_back(INT64_MIN); + sets.jumps.push_back((i % 2 == 0) ? 1000000000000000LL + : -1000000000000000LL); + } + return sets; +} + +template <> +ValueSets MakeValueSets(int count, uint32_t seed) { + std::mt19937 rng(seed); + std::uniform_real_distribution wide(1.0f, 8.0f); + ValueSets sets; + float smooth = 100.0f; + for (int i = 0; i < count; ++i) { + smooth += 0.25f * static_cast(1 + rng() % 4) + + 0.001f * static_cast(rng() % 1000); + sets.smooth.push_back(smooth); + sets.random.push_back(wide(rng)); + sets.constant.push_back(12.5f); + sets.jumps.push_back((i % 2 == 0) ? 1000.0f : 1.0f); + } + return sets; +} + +template <> +ValueSets MakeValueSets(int count, uint32_t seed) { + std::mt19937_64 rng(seed); + std::uniform_real_distribution wide(1.0, 8.0); + ValueSets sets; + double smooth = 100.0; + for (int i = 0; i < count; ++i) { + smooth += 0.25 * static_cast(1 + rng() % 4) + + 0.000001 * static_cast(rng() % 1000000); + sets.smooth.push_back(smooth); + sets.random.push_back(wide(rng)); + sets.constant.push_back(-0.0); + sets.jumps.push_back((i % 2 == 0) ? 1000000.0 : 1.0); + } + return sets; +} + +std::vector BoundaryCapacities() { + const int caps[] = {1, 2, 3, 5, 7, 8, 9, 10, 17, 129}; + return std::vector(caps, caps + sizeof(caps) / sizeof(caps[0])); +} + +// A batch script that also splits blocks at arbitrary points. +std::vector RandomCapacities(int rounds, uint32_t seed) { + std::mt19937 rng(seed); + std::vector script; + script.reserve(static_cast(rounds)); + for (int i = 0; i < rounds; ++i) { + script.push_back(static_cast(rng() % 23)); + } + return script; +} + +// The reference loop reads a huge capacity in one call; the batch override +// must agree with it for every count, including ones that end mid-block, +// exactly on a block boundary and inside the final short block. +template +void CheckBlockBoundaryCounts() { + typedef typename SprintzTypes::Decoder Decoder; + ValueSets sets = MakeValueSets(64, 7); + for (int count = 0; count <= 40; ++count) { + std::vector values(sets.smooth.begin(), sets.smooth.begin() + count); + ByteStream ref_in(1024, MOD_ENCODER_OBJ); + ByteStream bat_in(1024, MOD_ENCODER_OBJ); + EncodeAll(values, ref_in); + EncodeAll(values, bat_in); + + Decoder ref_dec; + Decoder bat_dec; + const int capacity = count + 16; + std::vector ref_buf(static_cast(capacity)); + std::vector bat_buf(static_cast(capacity)); + int ref_actual = -1; + int bat_actual = -1; + SCOPED_TRACE(::testing::Message() << "count " << count); + ASSERT_EQ( + ReferenceBatch(ref_dec, ref_buf.data(), capacity, ref_actual, + ref_in), + ReadBatch(bat_dec, bat_buf.data(), capacity, bat_actual, bat_in)); + ASSERT_EQ(ref_actual, bat_actual); + for (int i = 0; i < bat_actual; ++i) { + ExpectSameBits(ref_buf[i], bat_buf[i], "batch value"); + } + ASSERT_EQ(ref_in.remaining_size(), bat_in.remaining_size()); + ASSERT_EQ(ref_dec.has_remaining(ref_in), bat_dec.has_remaining(bat_in)); + + // read_exact recovers the full input, so it also exercises the + // batch/scalar handoff at a block tail. + ByteStream exact_in(1024, MOD_ENCODER_OBJ); + EncodeAll(values, exact_in); + Decoder exact_dec; + std::vector exact(static_cast(count > 0 ? count : 1)); + ASSERT_EQ(E_OK, ReadExact(exact_dec, exact.data(), count, exact_in)); + for (int i = 0; i < count; ++i) { + ExpectSameBits(values[static_cast(i)], + exact[static_cast(i)], "exact value"); + } + // FLOAT/DOUBLE short blocks are Gorilla-encoded and can leave the + // final bytes of the end marker buffered (the per-value path does the + // same), so only bound the residue here. + ASSERT_LE(exact_in.remaining_size(), 8u); + } +} + +std::vector CopyStreamBytes(ByteStream& stream) { + std::vector bytes; + bytes.reserve(static_cast(stream.total_size())); + ByteStream::BufferIterator iter = stream.init_buffer_iterator(); + while (true) { + ByteStream::Buffer buffer = iter.get_next_buf(); + if (buffer.buf_ == nullptr || buffer.len_ == 0) { + break; + } + const uint8_t* begin = reinterpret_cast(buffer.buf_); + bytes.insert(bytes.end(), begin, begin + buffer.len_); + } + return bytes; +} + +template +void CheckTruncatedStreams() { + typedef typename SprintzTypes::Decoder Decoder; + ValueSets sets = MakeValueSets(40, 11); + ByteStream full(1024, MOD_ENCODER_OBJ); + EncodeAll(sets.smooth, full); + + std::vector bytes = CopyStreamBytes(full); + ASSERT_EQ(bytes.size(), full.total_size()); + + for (size_t len = 0; len <= bytes.size(); ++len) { + ByteStream ref_in(1024, MOD_ENCODER_OBJ); + ByteStream bat_in(1024, MOD_ENCODER_OBJ); + if (len > 0) { + ref_in.write_buf(reinterpret_cast(bytes.data()), + static_cast(len)); + bat_in.write_buf(reinterpret_cast(bytes.data()), + static_cast(len)); + } + Decoder ref_dec; + Decoder bat_dec; + const int capacity = 1000; + std::vector ref_buf(static_cast(capacity)); + std::vector bat_buf(static_cast(capacity)); + int ref_actual = -1; + int bat_actual = -1; + int ref_ret = ReferenceBatch(ref_dec, ref_buf.data(), capacity, + ref_actual, ref_in); + int bat_ret = + ReadBatch(bat_dec, bat_buf.data(), capacity, bat_actual, bat_in); + SCOPED_TRACE(::testing::Message() + << "prefix " << len << " of " << bytes.size()); + ASSERT_EQ(ref_ret, bat_ret); + ASSERT_EQ(ref_actual, bat_actual); + for (int i = 0; i < bat_actual; ++i) { + ExpectSameBits(ref_buf[i], bat_buf[i], "truncated value"); + } + ASSERT_EQ(ref_in.remaining_size(), bat_in.remaining_size()); + ASSERT_EQ(ref_dec.has_remaining(ref_in), bat_dec.has_remaining(bat_in)); + } +} + +template +void CheckZeroCapacity() { + typedef typename SprintzTypes::Decoder Decoder; + ValueSets sets = MakeValueSets(32, 5); + ByteStream in(1024, MOD_ENCODER_OBJ); + EncodeAll(sets.smooth, in); + const uint64_t remaining = in.remaining_size(); + + Decoder decoder; + T value = T(); + int actual = -1; + ASSERT_EQ(E_OK, ReadBatch(decoder, &value, 0, actual, in)); + ASSERT_EQ(0, actual); + ASSERT_EQ(E_OK, ReadBatch(decoder, &value, -3, actual, in)); + ASSERT_EQ(0, actual); + ASSERT_EQ(remaining, in.remaining_size()); + + // No state was consumed: the first batch starts at the first value. + std::vector out(4); + ASSERT_EQ(E_OK, ReadBatch(decoder, out.data(), 4, actual, in)); + ASSERT_EQ(4, actual); + for (int i = 0; i < actual; ++i) { + ExpectSameBits(sets.smooth[static_cast(i)], out[i], + "value after zero capacity"); + } +} + +template +void CheckEndOfStream() { + typedef typename SprintzTypes::Decoder Decoder; + Decoder empty_decoder; + ByteStream empty_in(64, MOD_ENCODER_OBJ); + T value = T(); + int actual = -1; + ASSERT_EQ(E_OK, ReadBatch(empty_decoder, &value, 8, actual, empty_in)); + ASSERT_EQ(0, actual); + ASSERT_FALSE(empty_decoder.has_remaining(empty_in)); + + ValueSets sets = MakeValueSets(50, 9); + ByteStream in(1024, MOD_ENCODER_OBJ); + EncodeAll(sets.smooth, in); + Decoder decoder; + std::vector out(1024); + uint64_t guard = 0; + while (decoder.has_remaining(in)) { + ASSERT_EQ(E_OK, ReadBatch(decoder, out.data(), 1024, actual, in)); + if (actual == 0) { + break; + } + guard += static_cast(actual); + ASSERT_LE(guard, sets.smooth.size()); + } + ASSERT_FALSE(decoder.has_remaining(in)); + ASSERT_EQ(0u, in.remaining_size()); + // Once drained, further batch calls report no values and no error. + ASSERT_EQ(E_OK, ReadBatch(decoder, out.data(), 1024, actual, in)); + ASSERT_EQ(0, actual); +} + +// read_exact must hand whole pages through the batch path, including the +// per-page tail the batch deliberately leaves for a scalar read. +template +void CheckReadExactChunks(int total, int chunk) { + typedef typename SprintzTypes::Decoder Decoder; + ValueSets sets = MakeValueSets(total, 21); + ByteStream in(1024, MOD_ENCODER_OBJ); + EncodeAll(sets.smooth, in); + Decoder decoder; + std::vector out(static_cast(total)); + for (int done = 0; done < total; done += chunk) { + const int count = std::min(chunk, total - done); + ASSERT_EQ(E_OK, ReadExact(decoder, out.data() + done, count, in)); + } + for (int i = 0; i < total; ++i) { + ExpectSameBits(sets.smooth[static_cast(i)], + out[static_cast(i)], "chunked exact value"); + } + // FLOAT/DOUBLE short blocks are Gorilla-encoded and can leave the final + // bytes of the end marker buffered, exactly as the per-value path does. + ASSERT_LE(in.remaining_size(), 8u); +} + +TEST_F(SprintzCodecTest, Int32BatchParitySmooth) { + CheckBatchParity(MakeValueSets(3000, 1).smooth, + RepeatScript(BoundaryCapacities(), 300)); +} + +TEST_F(SprintzCodecTest, Int64BatchParitySmooth) { + CheckBatchParity(MakeValueSets(3000, 2).smooth, + RepeatScript(BoundaryCapacities(), 300)); +} + +TEST_F(SprintzCodecTest, FloatBatchParitySmooth) { + CheckBatchParity(MakeValueSets(3000, 3).smooth, + RepeatScript(BoundaryCapacities(), 300)); +} + +TEST_F(SprintzCodecTest, DoubleBatchParitySmooth) { + CheckBatchParity(MakeValueSets(3000, 4).smooth, + RepeatScript(BoundaryCapacities(), 300)); +} + +TEST_F(SprintzCodecTest, Int32BatchParityRandom) { + CheckBatchParity(MakeValueSets(3000, 5).random, + RandomCapacities(400, 6)); +} + +TEST_F(SprintzCodecTest, Int64BatchParityRandom) { + CheckBatchParity(MakeValueSets(3000, 7).random, + RandomCapacities(400, 8)); +} + +TEST_F(SprintzCodecTest, FloatBatchParityRandom) { + CheckBatchParity(MakeValueSets(3000, 9).random, + RandomCapacities(400, 10)); +} + +TEST_F(SprintzCodecTest, DoubleBatchParityRandom) { + CheckBatchParity(MakeValueSets(3000, 11).random, + RandomCapacities(400, 12)); +} + +TEST_F(SprintzCodecTest, Int32BatchParityConstant) { + CheckBatchParity(MakeValueSets(1000, 13).constant, + RepeatScript(BoundaryCapacities(), 200)); +} + +TEST_F(SprintzCodecTest, Int64BatchParityConstant) { + CheckBatchParity(MakeValueSets(1000, 14).constant, + RepeatScript(BoundaryCapacities(), 200)); +} + +TEST_F(SprintzCodecTest, FloatBatchParityConstant) { + CheckBatchParity(MakeValueSets(1000, 15).constant, + RepeatScript(BoundaryCapacities(), 200)); +} + +TEST_F(SprintzCodecTest, DoubleBatchParityConstant) { + CheckBatchParity(MakeValueSets(1000, 16).constant, + RepeatScript(BoundaryCapacities(), 200)); +} + +TEST_F(SprintzCodecTest, Int32BatchParityJumps) { + CheckBatchParity(MakeValueSets(1000, 17).jumps, + RepeatScript(BoundaryCapacities(), 200)); +} + +TEST_F(SprintzCodecTest, Int64BatchParityJumps) { + CheckBatchParity(MakeValueSets(1000, 18).jumps, + RepeatScript(BoundaryCapacities(), 200)); +} + +TEST_F(SprintzCodecTest, FloatBatchParityJumps) { + CheckBatchParity(MakeValueSets(1000, 19).jumps, + RepeatScript(BoundaryCapacities(), 200)); +} + +TEST_F(SprintzCodecTest, DoubleBatchParityJumps) { + CheckBatchParity(MakeValueSets(1000, 20).jumps, + RepeatScript(BoundaryCapacities(), 200)); +} + +// A small ByteStream page size splits block payload reads across pages. +TEST_F(SprintzCodecTest, Int32BatchParityAcrossStreamPages) { + CheckBatchParity(MakeValueSets(2000, 22).smooth, + RepeatScript(BoundaryCapacities(), 300), 16); +} + +TEST_F(SprintzCodecTest, Int64BatchParityAcrossStreamPages) { + CheckBatchParity(MakeValueSets(2000, 23).smooth, + RepeatScript(BoundaryCapacities(), 300), 16); +} + +TEST_F(SprintzCodecTest, FloatBatchParityAcrossStreamPages) { + CheckBatchParity(MakeValueSets(2000, 24).smooth, + RepeatScript(BoundaryCapacities(), 300), 16); +} + +TEST_F(SprintzCodecTest, DoubleBatchParityAcrossStreamPages) { + CheckBatchParity(MakeValueSets(2000, 25).smooth, + RepeatScript(BoundaryCapacities(), 300), 16); +} + +TEST_F(SprintzCodecTest, Int32BatchBlockBoundaryCounts) { + CheckBlockBoundaryCounts(); +} + +TEST_F(SprintzCodecTest, Int64BatchBlockBoundaryCounts) { + CheckBlockBoundaryCounts(); +} + +TEST_F(SprintzCodecTest, FloatBatchBlockBoundaryCounts) { + CheckBlockBoundaryCounts(); +} + +TEST_F(SprintzCodecTest, DoubleBatchBlockBoundaryCounts) { + CheckBlockBoundaryCounts(); +} + +TEST_F(SprintzCodecTest, Int32BatchZeroCapacity) { + CheckZeroCapacity(); +} + +TEST_F(SprintzCodecTest, Int64BatchZeroCapacity) { + CheckZeroCapacity(); +} + +TEST_F(SprintzCodecTest, FloatBatchZeroCapacity) { CheckZeroCapacity(); } + +TEST_F(SprintzCodecTest, DoubleBatchZeroCapacity) { + CheckZeroCapacity(); +} + +TEST_F(SprintzCodecTest, Int32BatchEndOfStream) { CheckEndOfStream(); } + +TEST_F(SprintzCodecTest, Int64BatchEndOfStream) { CheckEndOfStream(); } + +TEST_F(SprintzCodecTest, FloatBatchEndOfStream) { CheckEndOfStream(); } + +TEST_F(SprintzCodecTest, DoubleBatchEndOfStream) { CheckEndOfStream(); } + +TEST_F(SprintzCodecTest, Int32BatchTruncatedStream) { + CheckTruncatedStreams(); +} + +TEST_F(SprintzCodecTest, Int64BatchTruncatedStream) { + CheckTruncatedStreams(); +} + +TEST_F(SprintzCodecTest, FloatBatchTruncatedStream) { + CheckTruncatedStreams(); +} + +TEST_F(SprintzCodecTest, DoubleBatchTruncatedStream) { + CheckTruncatedStreams(); +} + +TEST_F(SprintzCodecTest, Int32BatchReadExactChunks) { + CheckReadExactChunks(1000, 37); +} + +TEST_F(SprintzCodecTest, Int64BatchReadExactChunks) { + CheckReadExactChunks(1000, 129); +} + +TEST_F(SprintzCodecTest, FloatBatchReadExactChunks) { + CheckReadExactChunks(1000, 8); +} + +TEST_F(SprintzCodecTest, DoubleBatchReadExactChunks) { + CheckReadExactChunks(1000, 1); +} + +// ── Java-generated golden streams ──────────────────────────────────────── +// +// These byte streams were produced by the Java implementation +// (java/tsfile/src/main/java/org/apache/tsfile/encoding/encoder/ +// {Int,Long,Float,Double}SprintzEncoder with its default "fire" predictor), +// so the decoders below are exercised on bytes they did not produce. The C++ +// encoder is asserted to emit the same bytes for the same values, which pins +// each fixture to the shared on-disk format, and the decoder is then driven +// through read_exact, the per-value loop and the batch loop. + +// JavaInt32Smooth13: first block plus a short tail block (Java +// IntSprintzEncoder, 13 values) +const uint8_t kJavaInt32Smooth13Bytes[] = { + 0x03, 0xe8, 0x07, 0xb6, 0xdb, 0x6d, 0x84, 0x0e, 0x0b, 0x03, 0x04, + 0x80, 0x70, 0x1a, 0x04, 0xc0, 0xc0, 0x00, 0x00, 0x00, 0x00, 0x00, +}; +const int32_t kJavaInt32Smooth13Values[] = { + 1000, 1003, 1006, 1009, 1012, 1015, 1018, + 1021, 1024, 1027, 1030, 1033, 1036, +}; +// JavaInt32Constant12: constant run (Java IntSprintzEncoder, 12 values) +const uint8_t kJavaInt32Constant12Bytes[] = { + 0x01, 0x07, 0x00, 0x83, 0x06, 0x03, 0x03, 0x03, 0xff, 0x80, 0x00, +}; +const int32_t kJavaInt32Constant12Values[] = { + 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, +}; +// JavaInt32GroupFlush140: crosses the encoder's 16-group flush and ends in a +// short block (Java IntSprintzEncoder, 140 values) +const uint8_t kJavaInt32GroupFlush140Bytes[] = { + 0x04, 0xf4, 0x03, 0x11, 0x11, 0x11, 0xc3, 0x04, 0xf6, 0x03, 0x11, 0x11, + 0xc1, 0x33, 0x04, 0xf8, 0x03, 0x11, 0xc1, 0x33, 0x33, 0x04, 0xfa, 0x03, + 0xc1, 0x33, 0x33, 0x3a, 0x04, 0xf5, 0x03, 0x11, 0x11, 0x1c, 0x13, 0x04, + 0xf7, 0x03, 0x11, 0x1c, 0x13, 0x33, 0x04, 0xf9, 0x03, 0x1c, 0x13, 0x33, + 0x33, 0x04, 0xf4, 0x03, 0x11, 0x11, 0x11, 0xc3, 0x04, 0xf6, 0x03, 0x11, + 0x11, 0xc1, 0x33, 0x04, 0xf8, 0x03, 0x11, 0xc1, 0x33, 0x33, 0x04, 0xfa, + 0x03, 0xc1, 0x33, 0x33, 0x3a, 0x04, 0xf5, 0x03, 0x11, 0x11, 0x1c, 0x13, + 0x04, 0xf7, 0x03, 0x11, 0x1c, 0x13, 0x33, 0x04, 0xf9, 0x03, 0x1c, 0x13, + 0x33, 0x33, 0x04, 0xf4, 0x03, 0x11, 0x11, 0x11, 0xc3, 0x85, 0x0c, 0x09, + 0x03, 0x05, 0xfb, 0x7d, 0xff, 0x1f, 0x9f, 0xd0, 0x00, 0x00, 0x00, +}; +const int32_t kJavaInt32GroupFlush140Values[] = { + 500, 501, 502, 503, 504, 505, 506, 500, 501, 502, 503, 504, 505, 506, + 500, 501, 502, 503, 504, 505, 506, 500, 501, 502, 503, 504, 505, 506, + 500, 501, 502, 503, 504, 505, 506, 500, 501, 502, 503, 504, 505, 506, + 500, 501, 502, 503, 504, 505, 506, 500, 501, 502, 503, 504, 505, 506, + 500, 501, 502, 503, 504, 505, 506, 500, 501, 502, 503, 504, 505, 506, + 500, 501, 502, 503, 504, 505, 506, 500, 501, 502, 503, 504, 505, 506, + 500, 501, 502, 503, 504, 505, 506, 500, 501, 502, 503, 504, 505, 506, + 500, 501, 502, 503, 504, 505, 506, 500, 501, 502, 503, 504, 505, 506, + 500, 501, 502, 503, 504, 505, 506, 500, 501, 502, 503, 504, 505, 506, + 500, 501, 502, 503, 504, 505, 506, 500, 501, 502, 503, 504, 505, 506, +}; +// JavaInt64Timestamps17: epoch-millisecond timestamps (Java LongSprintzEncoder, +// 17 values) +const uint8_t kJavaInt64Timestamps17Bytes[] = { + 0x0b, 0x00, 0x00, 0x01, 0x93, 0x43, 0x3e, 0xa8, 0x00, 0xf9, 0xff, + 0x3f, 0xe6, 0xfc, 0x9f, 0x8b, 0xf0, 0x7d, 0xef, 0xb9, 0x88, 0x2c, + 0x29, 0x03, 0x08, 0xc9, 0xa1, 0x9f, 0x65, 0x94, 0x64, 0xd0, 0xcf, + 0xb3, 0xc4, 0x32, 0x68, 0x67, 0xda, 0x5f, 0x19, 0x34, 0x33, 0xed, + 0x6e, 0x0c, 0x9a, 0x19, 0xf6, 0xd6, 0x46, 0x4d, 0x0c, 0xfb, 0x7a, + 0xc3, 0x26, 0x86, 0x7d, 0xc5, 0x31, 0x93, 0x43, 0x3e, 0xe6, 0x80, +}; +const int64_t kJavaInt64Timestamps17Values[] = { + 1732000000000LL, 1732000001000LL, 1732000002000LL, 1732000003000LL, + 1732000004000LL, 1732000005000LL, 1732000006000LL, 1732000007000LL, + 1732000008000LL, 1732000009000LL, 1732000010000LL, 1732000011000LL, + 1732000012000LL, 1732000013000LL, 1732000014000LL, 1732000015000LL, + 1732000016000LL, +}; +// JavaFloatSmooth12: smooth float series (Java FloatSprintzEncoder, 12 values) +const uint8_t kJavaFloatSmooth12Bytes[] = { + 0x12, 0x41, 0xa0, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x83, + 0x41, 0xb2, 0x00, 0x00, 0xda, 0x1e, 0x71, 0x33, 0xe7, 0x60, +}; +const uint32_t kJavaFloatSmooth12Bits[] = { + 0x41a00000U, 0x41a20000U, 0x41a40000U, 0x41a60000U, + 0x41a80000U, 0x41aa0000U, 0x41ac0000U, 0x41ae0000U, + 0x41b00000U, 0x41b20000U, 0x41b40000U, 0x41b60000U, +}; +// JavaDoubleSmooth25: smooth double series (Java DoubleSprintzEncoder, 25 +// values) +const uint8_t kJavaDoubleSmooth25Bytes[] = { + 0x1c, 0x41, 0x2e, 0x84, 0x80, 0x00, 0x00, 0x00, 0x00, 0x10, 0x62, 0x4d, + 0xd1, 0x06, 0x24, 0xdb, 0xfc, 0x0d, 0x1d, 0x61, 0x06, 0x24, 0xdb, 0xfc, + 0x0d, 0x2d, 0xc1, 0x06, 0x24, 0xdd, 0xfc, 0x0d, 0x30, 0x01, 0x06, 0x24, + 0xdd, 0x1c, 0x41, 0x2e, 0x84, 0x80, 0x04, 0x9b, 0xa5, 0xe3, 0x10, 0x62, + 0x4d, 0xd1, 0x06, 0x24, 0xdd, 0xfc, 0x0d, 0x1f, 0xa1, 0x06, 0x24, 0xdd, + 0xfc, 0x0d, 0x1f, 0xa1, 0x06, 0x25, 0xe5, 0xfc, 0x0d, 0x1f, 0xa1, 0x06, + 0x25, 0xe5, 0x87, 0x41, 0x2e, 0x84, 0x80, 0x09, 0x37, 0x4b, 0xc7, 0xe8, + 0x5a, 0x34, 0x57, 0xcf, 0x33, 0x3c, 0x39, 0x74, 0x8c, 0x7e, 0xf9, 0xdb, + 0x4c, 0x18, 0xb4, 0x9c, 0x42, 0x99, 0x38, 0xf2, 0xb3, 0xe3, 0xc7, 0x6b, + 0xe1, 0x79, 0xf6, 0xb4, 0x24, 0x00, 0x62, 0x4d, 0xd2, 0xf0, +}; +const uint64_t kJavaDoubleSmooth25Bits[] = { + 0x412e848000000000ULL, 0x412e84800083126fULL, 0x412e8480010624ddULL, + 0x412e84800189374cULL, 0x412e8480020c49baULL, 0x412e8480028f5c29ULL, + 0x412e848003126e98ULL, 0x412e848003958106ULL, 0x412e848004189375ULL, + 0x412e8480049ba5e3ULL, 0x412e8480051eb852ULL, 0x412e848005a1cac1ULL, + 0x412e84800624dd2fULL, 0x412e848006a7ef9eULL, 0x412e8480072b020cULL, + 0x412e848007ae147bULL, 0x412e8480083126e9ULL, 0x412e848008b43958ULL, + 0x412e848009374bc7ULL, 0x412e848009ba5e35ULL, 0x412e84800a3d70a4ULL, + 0x412e84800ac08312ULL, 0x412e84800b439581ULL, 0x412e84800bc6a7f0ULL, + 0x412e84800c49ba5eULL, +}; + +float FloatFromBits(uint32_t bits) { + float value; + std::memcpy(&value, &bits, sizeof(value)); + return value; +} + +double DoubleFromBits(uint64_t bits) { + double value; + std::memcpy(&value, &bits, sizeof(value)); + return value; +} + +template +std::vector ToVector(const T (&values)[N]) { + return std::vector(values, values + N); +} + +// The fixture must be what the C++ encoder emits for the same values; +// otherwise the bytes and the expected values describe different streams. +template +void ExpectEncoderMatchesJava(const std::vector& values, + const uint8_t* expected, size_t expected_len) { + ByteStream stream(1024, MOD_ENCODER_OBJ); + EncoderT encoder; + for (size_t i = 0; i < values.size(); ++i) { + ASSERT_EQ(E_OK, encoder.encode(values[i], stream)); + } + ASSERT_EQ(E_OK, encoder.flush(stream)); + std::vector bytes = CopyStreamBytes(stream); + ASSERT_EQ(expected_len, bytes.size()); + ASSERT_EQ(0, std::memcmp(expected, bytes.data(), expected_len)); +} + +// Decode Java bytes three ways: read_exact (batch plus scalar handoff), a +// pure per-value drain, and a batch drain with block-splitting capacities. +template +void CheckJavaGolden(const uint8_t* bytes, size_t len, const T* expected, + int count) { + { + ByteStream in(1024, MOD_ENCODER_OBJ); + in.write_buf(reinterpret_cast(bytes), + static_cast(len)); + DecoderT decoder; + std::vector out(static_cast(count > 0 ? count : 1)); + ASSERT_EQ(E_OK, ReadExact(decoder, out.data(), count, in)); + for (int i = 0; i < count; ++i) { + ExpectSameBits(expected[i], out[static_cast(i)], + "java golden, read_exact"); + } + ASSERT_LE(in.remaining_size(), 8u); + } + + ByteStream ref_in(1024, MOD_ENCODER_OBJ); + ByteStream bat_in(1024, MOD_ENCODER_OBJ); + ref_in.write_buf(reinterpret_cast(bytes), + static_cast(len)); + bat_in.write_buf(reinterpret_cast(bytes), + static_cast(len)); + + std::vector ref_values; + DecoderT ref_decoder; + T one = T(); + while (ref_decoder.has_remaining(ref_in)) { + ASSERT_EQ(E_OK, ReadOne(ref_decoder, one, ref_in)); + ref_values.push_back(one); + } + + std::vector batch_values; + DecoderT bat_decoder; + const int capacities[] = {1, 3, 7, 9, 5, 129}; + std::vector buffer(129); + int round = 0; + while (bat_decoder.has_remaining(bat_in)) { + int actual = 0; + const int capacity = capacities[round++ % 6]; + ASSERT_EQ(E_OK, ReadBatch(bat_decoder, buffer.data(), capacity, actual, + bat_in)); + if (actual <= 0) { + break; + } + batch_values.insert(batch_values.end(), buffer.begin(), + buffer.begin() + actual); + } + + ASSERT_EQ(ref_in.remaining_size(), bat_in.remaining_size()); + ASSERT_EQ(ref_values.size(), batch_values.size()); + // Every fixture ends in a short block, so a drain returns every value. + ASSERT_EQ(static_cast(count), ref_values.size()); + for (size_t i = 0; i < ref_values.size(); ++i) { + ExpectSameBits(expected[i], ref_values[i], "java golden, per-value"); + ExpectSameBits(expected[i], batch_values[i], "java golden, batch"); + } +} + +TEST_F(SprintzCodecTest, JavaGoldenInt32) { + std::vector smooth = ToVector(kJavaInt32Smooth13Values); + ExpectEncoderMatchesJava( + smooth, kJavaInt32Smooth13Bytes, sizeof(kJavaInt32Smooth13Bytes)); + CheckJavaGolden( + kJavaInt32Smooth13Bytes, sizeof(kJavaInt32Smooth13Bytes), + kJavaInt32Smooth13Values, static_cast(smooth.size())); + + std::vector constant = ToVector(kJavaInt32Constant12Values); + ExpectEncoderMatchesJava( + constant, kJavaInt32Constant12Bytes, sizeof(kJavaInt32Constant12Bytes)); + CheckJavaGolden( + kJavaInt32Constant12Bytes, sizeof(kJavaInt32Constant12Bytes), + kJavaInt32Constant12Values, static_cast(constant.size())); + + std::vector group = ToVector(kJavaInt32GroupFlush140Values); + ExpectEncoderMatchesJava( + group, kJavaInt32GroupFlush140Bytes, + sizeof(kJavaInt32GroupFlush140Bytes)); + CheckJavaGolden( + kJavaInt32GroupFlush140Bytes, sizeof(kJavaInt32GroupFlush140Bytes), + kJavaInt32GroupFlush140Values, static_cast(group.size())); +} + +TEST_F(SprintzCodecTest, JavaGoldenInt64) { + std::vector values = ToVector(kJavaInt64Timestamps17Values); + ExpectEncoderMatchesJava( + values, kJavaInt64Timestamps17Bytes, + sizeof(kJavaInt64Timestamps17Bytes)); + CheckJavaGolden( + kJavaInt64Timestamps17Bytes, sizeof(kJavaInt64Timestamps17Bytes), + kJavaInt64Timestamps17Values, static_cast(values.size())); +} + +TEST_F(SprintzCodecTest, JavaGoldenFloat) { + std::vector values; + for (size_t i = 0; i < sizeof(kJavaFloatSmooth12Bits) / sizeof(uint32_t); + ++i) { + values.push_back(FloatFromBits(kJavaFloatSmooth12Bits[i])); + } + ExpectEncoderMatchesJava( + values, kJavaFloatSmooth12Bytes, sizeof(kJavaFloatSmooth12Bytes)); + CheckJavaGolden( + kJavaFloatSmooth12Bytes, sizeof(kJavaFloatSmooth12Bytes), values.data(), + static_cast(values.size())); +} + +TEST_F(SprintzCodecTest, JavaGoldenDouble) { + std::vector values; + for (size_t i = 0; i < sizeof(kJavaDoubleSmooth25Bits) / sizeof(uint64_t); + ++i) { + values.push_back(DoubleFromBits(kJavaDoubleSmooth25Bits[i])); + } + ExpectEncoderMatchesJava( + values, kJavaDoubleSmooth25Bytes, sizeof(kJavaDoubleSmooth25Bytes)); + CheckJavaGolden( + kJavaDoubleSmooth25Bytes, sizeof(kJavaDoubleSmooth25Bytes), + values.data(), static_cast(values.size())); +} + } // namespace