From 1badf95d925e1f7ad940ef2c15927a1aa04d2348 Mon Sep 17 00:00:00 2001 From: ColinLee Date: Mon, 9 Nov 2026 08:26:05 +0800 Subject: [PATCH 1/2] perf(cpp): batch and SIMD decode for the RLE bit-packing codecs RLE was the last hot value codec without a batch path: Plain, Gorilla and TS_2DIFF got read_batch_* in #823/#898, while RLE still decoded value by value through read_int and unpacked each bit-packing group with the portable scalar packer. On a wide table the RLE decoder dominates the read profile. - add read_batch_int32/int64 and read_batch_float/double: an RLE run becomes a std::fill, a bit-packing group becomes one memcpy out of the unpacked buffer. - unpack four values per step with a SIMD gather+shuffle kernel and dispatch to native AVX2 intrinsics at runtime (__builtin_cpu_supports) instead of relying on the translation unit being compiled with -mavx2; the simde fallback stays for other targets, and the widths that can cross a ninth byte (59/61/62/63) use a 128-bit buffer where the compiler has one. - read each bit-packing group with one bulk read instead of per-byte read_ui8, and wrap the caller's stream for the pattern length instead of copying it. - the SIMD gather reads up to 7 bytes past the last group, so the RLE and Sprintz decoders pad their buffers by 8 zero bytes; the packer tests do the same and pass the exact packed length. - guard the 128-bit scalar path with __SIZEOF_INT128__ and keep the portable bit loop on MSVC. --- cpp/src/common/simd_dispatch.h | 58 ++++++++++ cpp/src/encoding/double_sprintz_decoder.h | 4 +- cpp/src/encoding/float_rle_decoder.h | 49 ++++++++ cpp/src/encoding/float_sprintz_decoder.h | 4 +- cpp/src/encoding/int32_packer.h | 99 +++++++++++++++- cpp/src/encoding/int32_rle_decoder.h | 87 +++++++++++--- cpp/src/encoding/int32_sprintz_decoder.h | 4 +- cpp/src/encoding/int64_packer.h | 132 +++++++++++++++++++++- cpp/src/encoding/int64_rle_decoder.h | 87 ++++++++++++-- cpp/src/encoding/int64_sprintz_decoder.h | 4 +- cpp/test/encoding/int32_packer_test.cc | 26 +++-- cpp/test/encoding/int64_packer_test.cc | 27 +++-- 12 files changed, 525 insertions(+), 56 deletions(-) create mode 100644 cpp/src/common/simd_dispatch.h diff --git a/cpp/src/common/simd_dispatch.h b/cpp/src/common/simd_dispatch.h new file mode 100644 index 0000000000..8663498588 --- /dev/null +++ b/cpp/src/common/simd_dispatch.h @@ -0,0 +1,58 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +#ifndef COMMON_SIMD_DISPATCH_H +#define COMMON_SIMD_DISPATCH_H + +#include "common/global.h" + +// Runtime SIMD dispatch. +// +// The library is built for the portable x86-64 baseline (SSE2 only), so a +// kernel written with simde's AVX2/SSSE3 API is compiled into simde's portable +// fallback — measured ~6x slower than real AVX2 for the RLE bit-unpack. To run +// the fast instructions without raising the binary's ISA floor (an -mavx2 +// binary faults on older CPUs), such kernels get a second copy annotated with +// __attribute__((target("avx2"))) that uses raw intrinsics, and call sites pick +// it at runtime via cpu_has_avx2(). +// +// Only GCC/Clang on x86 provide both pieces; elsewhere callers keep the +// portable path. + +#if defined(__x86_64__) || defined(__i386__) +#if defined(__GNUC__) || defined(__clang__) +#define TSFILE_X86_AVX2_DISPATCH 1 +#define TSFILE_TARGET_AVX2 __attribute__((target("avx2"))) +#endif +#endif + +namespace common { + +#ifdef TSFILE_X86_AVX2_DISPATCH +// True when the running CPU implements AVX2. Cached after the first call; +// glibc/libgcc initialize the CPU feature bits before any regular function +// runs, so no explicit __builtin_cpu_init() is required here. +inline bool cpu_has_avx2() { + static const bool has_avx2 = __builtin_cpu_supports("avx2") != 0; + return has_avx2; +} +#endif + +} // namespace common + +#endif // COMMON_SIMD_DISPATCH_H diff --git a/cpp/src/encoding/double_sprintz_decoder.h b/cpp/src/encoding/double_sprintz_decoder.h index 57f4b0b237..c189192e18 100644 --- a/cpp/src/encoding/double_sprintz_decoder.h +++ b/cpp/src/encoding/double_sprintz_decoder.h @@ -130,7 +130,9 @@ class DoubleSprintzDecoder : public SprintzDecoder { decode_size_ = block_size_ + 1; common::SerializationUtil::read_double(pre_value_, input); current_buffer_[0] = pre_value_; - std::vector pack_buf(bit_width_); + // +8 zero bytes: the SIMD unpack gathers 8 bytes per value and + // may read up to 7 bytes past the packed group. + std::vector pack_buf(bit_width_ + 8, 0); uint32_t read_len = 0; input.read_buf(reinterpret_cast(pack_buf.data()), bit_width_, read_len); diff --git a/cpp/src/encoding/float_rle_decoder.h b/cpp/src/encoding/float_rle_decoder.h index 6cdea98259..8084fe69c2 100644 --- a/cpp/src/encoding/float_rle_decoder.h +++ b/cpp/src/encoding/float_rle_decoder.h @@ -20,6 +20,7 @@ #ifndef ENCODING_FLOAT_RLE_DECODER_H #define ENCODING_FLOAT_RLE_DECODER_H +#include #include #include #include @@ -75,6 +76,30 @@ class FloatRLEDecoder : public Decoder { page_pos_++; return ret; } + int read_batch_float(float* out, int capacity, int& actual, + common::ByteStream& in) override { + actual = 0; + int ret = ensure_page_meta(in); + if (RET_FAIL(ret)) { + return ret; + } + int32_t tmp[64]; + while (actual < capacity && int_decoder_.has_remaining(in)) { + const int chunk = std::min(capacity - actual, 64); + int got = 0; + if (RET_FAIL(int_decoder_.read_batch_int32(tmp, chunk, got, in))) { + return ret; + } + if (got == 0) { + break; + } + for (int i = 0; i < got; i++) { + out[actual++] = value_at(tmp[i]); + page_pos_++; + } + } + return ret; + } int read_double(double& /*ret_value*/, common::ByteStream& /*in*/) override { return common::E_TYPE_NOT_MATCH; @@ -170,6 +195,30 @@ class DoubleRLEDecoder : public Decoder { page_pos_++; return ret; } + int read_batch_double(double* out, int capacity, int& actual, + common::ByteStream& in) override { + actual = 0; + int ret = ensure_page_meta(in); + if (RET_FAIL(ret)) { + return ret; + } + int64_t tmp[64]; + while (actual < capacity && int_decoder_.has_remaining(in)) { + const int chunk = std::min(capacity - actual, 64); + int got = 0; + if (RET_FAIL(int_decoder_.read_batch_int64(tmp, chunk, got, in))) { + return ret; + } + if (got == 0) { + break; + } + for (int i = 0; i < got; i++) { + out[actual++] = value_at(tmp[i]); + page_pos_++; + } + } + return ret; + } int read_String(common::String& /*ret_value*/, common::PageArena& /*pa*/, common::ByteStream& /*in*/) override { return common::E_TYPE_NOT_MATCH; diff --git a/cpp/src/encoding/float_sprintz_decoder.h b/cpp/src/encoding/float_sprintz_decoder.h index e3842eaafb..c97cc1551f 100644 --- a/cpp/src/encoding/float_sprintz_decoder.h +++ b/cpp/src/encoding/float_sprintz_decoder.h @@ -133,7 +133,9 @@ class FloatSprintzDecoder : public SprintzDecoder { common::SerializationUtil::read_float(pre_value_, input); current_buffer_[0] = pre_value_; // read packed data - std::vector pack_buf(bit_width_); + // +8 zero bytes: the SIMD unpack gathers 8 bytes per value and + // may read up to 7 bytes past the packed group. + std::vector pack_buf(bit_width_ + 8, 0); uint32_t read_len = 0; input.read_buf(reinterpret_cast(pack_buf.data()), bit_width_, read_len); diff --git a/cpp/src/encoding/int32_packer.h b/cpp/src/encoding/int32_packer.h index ecb42b248f..3ff0b81b7a 100644 --- a/cpp/src/encoding/int32_packer.h +++ b/cpp/src/encoding/int32_packer.h @@ -24,14 +24,92 @@ #include "encoder.h" +#ifdef ENABLE_SIMD +#include "common/simd_dispatch.h" +#include "simde/x86/avx2.h" +#ifdef TSFILE_X86_AVX2_DISPATCH +#include +#endif +#endif + namespace storage { +#ifdef ENABLE_SIMD +// SIMD decode of 4 big-endian bit-packed INT32 values. Uses the 64-bit gather +// so all widths 1..32 fit without a width guard; the trailing permute packs +// the low 32 bits of each 64-bit lane into four contiguous INT32 lanes. +static inline void rle_simd_unpack_4_i32(const uint8_t* in, int32_t width, + int32_t index, int32_t out[4]) { + static const simde__m256i SHUF_REV8 = simde_mm256_setr_epi8( + 7, 6, 5, 4, 3, 2, 1, 0, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, + 1, 0, 15, 14, 13, 12, 11, 10, 9, 8); + const int32_t pos0 = index * width; + const int32_t pos[4] = {pos0, pos0 + width, pos0 + 2 * width, + pos0 + 3 * width}; + const int32_t bidx[4] = {pos[0] >> 3, pos[1] >> 3, pos[2] >> 3, + pos[3] >> 3}; + const int32_t off[4] = {pos[0] & 7, pos[1] & 7, pos[2] & 7, pos[3] & 7}; + const simde__m128i IDX = + simde_mm_setr_epi32(bidx[0], bidx[1], bidx[2], bidx[3]); + const int32_t rshift = 64 - width; + simde__m256i w64_le = + simde_mm256_i32gather_epi64((const int64_t*)in, IDX, 1); + simde__m256i w64_be = simde_mm256_shuffle_epi8(w64_le, SHUF_REV8); + simde__m256i OFF64 = simde_mm256_cvtepu32_epi64( + simde_mm_setr_epi32(off[0], off[1], off[2], off[3])); + simde__m256i U64 = simde_mm256_sllv_epi64(w64_be, OFF64); + simde__m256i V64 = + simde_mm256_srl_epi64(U64, simde_mm_cvtsi32_si128(rshift)); + const simde__m256i perm = simde_mm256_setr_epi32(0, 2, 4, 6, 0, 0, 0, 0); + simde__m256i comp = simde_mm256_permutevar8x32_epi32(V64, perm); + simde_mm_storeu_si128((simde__m128i*)out, + simde_mm256_castsi256_si128(comp)); +} + +#ifdef TSFILE_X86_AVX2_DISPATCH +// Native-AVX2 twin of rle_simd_unpack_4_i32; see int64_packer.h for why the +// target attribute is required (simde otherwise emits its portable fallback). +TSFILE_TARGET_AVX2 +static inline void rle_avx2_unpack_4_i32(const uint8_t* in, int32_t width, + int32_t index, int32_t out[4]) { + const __m256i SHUF_REV8 = + _mm256_setr_epi8(7, 6, 5, 4, 3, 2, 1, 0, 15, 14, 13, 12, 11, 10, 9, 8, + 7, 6, 5, 4, 3, 2, 1, 0, 15, 14, 13, 12, 11, 10, 9, 8); + const int32_t pos0 = index * width; + const int32_t pos[4] = {pos0, pos0 + width, pos0 + 2 * width, + pos0 + 3 * width}; + const __m128i IDX = + _mm_setr_epi32(pos[0] >> 3, pos[1] >> 3, pos[2] >> 3, pos[3] >> 3); + const int32_t rshift = 64 - width; + __m256i w64_le = _mm256_i32gather_epi64((const long long*)in, IDX, 1); + __m256i w64_be = _mm256_shuffle_epi8(w64_le, SHUF_REV8); + __m256i OFF64 = _mm256_cvtepu32_epi64( + _mm_setr_epi32(pos[0] & 7, pos[1] & 7, pos[2] & 7, pos[3] & 7)); + __m256i U64 = _mm256_sllv_epi64(w64_be, OFF64); + __m256i V64 = _mm256_srl_epi64(U64, _mm_cvtsi32_si128(rshift)); + const __m256i perm = _mm256_setr_epi32(0, 2, 4, 6, 0, 0, 0, 0); + __m256i comp = _mm256_permutevar8x32_epi32(V64, perm); + _mm_storeu_si128((__m128i*)out, _mm256_castsi256_si128(comp)); +} +#endif // TSFILE_X86_AVX2_DISPATCH +#endif + class Int32Packer { private: int width_; +#ifdef TSFILE_X86_AVX2_DISPATCH + // Resolved once per packer (one per bit-width pattern) so the hot + // unpack loop pays no thread-safe-static guard on every group. + bool use_avx2_; +#endif public: - Int32Packer(int width_) { this->width_ = width_; } +#ifdef TSFILE_X86_AVX2_DISPATCH + Int32Packer(int width_) + : width_(width_), use_avx2_(common::cpu_has_avx2()) {} +#else + Int32Packer(int width_) : width_(width_) {} +#endif ~Int32Packer() { destroy(); } void destroy() { /* do nothing for IntPacker */ @@ -98,9 +176,28 @@ class Int32Packer { * @param offset - offset of first byte to be decoded in buf * @param values - decoded result , the length of 'values' should be @{link * IntPacker#NUM_OF_INTS} + * + * The SIMD paths gather 8 bytes per value, so the buffer must hold at + * least 8 readable bytes past the packed group; callers allocate that + * padding (the RLE decoders zero it, the Sprintz decoders size their + * block buffer with +8). */ void unpack_8values(const unsigned char buf[], int offset, int32_t values[]) { +#ifdef ENABLE_SIMD + // Every width 1..32 fits within the 8-byte gather, so no width guard + // is needed here (unlike the 64-bit packer). +#ifdef TSFILE_X86_AVX2_DISPATCH + if (use_avx2_) { + rle_avx2_unpack_4_i32(buf + offset, width_, 0, values); + rle_avx2_unpack_4_i32(buf + offset, width_, 4, values + 4); + return; + } +#endif + rle_simd_unpack_4_i32(buf + offset, width_, 0, values); + rle_simd_unpack_4_i32(buf + offset, width_, 4, values + 4); + return; +#endif int byte_idx = offset; uint64_t buffer = 0; // total bits which have reader from 'buf' to 'buffer'. i.e., diff --git a/cpp/src/encoding/int32_rle_decoder.h b/cpp/src/encoding/int32_rle_decoder.h index ef7b6f0954..09e6318783 100644 --- a/cpp/src/encoding/int32_rle_decoder.h +++ b/cpp/src/encoding/int32_rle_decoder.h @@ -20,6 +20,8 @@ #ifndef ENCODING_INT32RLE_DECODER_H #define ENCODING_INT32RLE_DECODER_H +#include +#include #include #include "common/allocator/alloc_base.h" @@ -73,6 +75,49 @@ class Int32RleDecoder : public Decoder { int read_int32(int32_t& ret_value, common::ByteStream& in) override { return read_int(ret_value, in); } + int read_batch_int32(int32_t* out, int capacity, int& actual, + common::ByteStream& in) override { + actual = 0; + int ret = common::E_OK; + while (actual < capacity && has_remaining(in)) { + if (!is_length_and_bitwidth_readed_) { + if (RET_FAIL(read_length_and_bitwidth(in))) { + return ret; + } + } + if (current_count_ == 0) { + uint32_t header_value = 0; + if (RET_FAIL(common::SerializationUtil::read_var_uint( + header_value, byte_cache_))) { + return ret; + } + if (header_value & 1) { + if (RET_FAIL(call_read_bit_packing_buffer(header_value))) { + return ret; + } + } else { + if (RET_FAIL(call_read_rle_run(header_value))) { + return ret; + } + } + } + const int n = std::min(current_count_, capacity - actual); + if (is_rle_run_) { + std::fill(out + actual, out + actual + n, rle_value_); + } else { + const int32_t* src = + current_buffer_ + (bitpacking_num_ - current_count_); + std::memcpy(out + actual, src, + static_cast(n) * sizeof(int32_t)); + } + actual += n; + current_count_ -= n; + if (!has_next_package()) { + is_length_and_bitwidth_readed_ = false; + } + } + return ret; + } int read_int64(int64_t& ret_value, common::ByteStream& in) override { return common::E_TYPE_NOT_MATCH; } @@ -210,13 +255,17 @@ class Int32RleDecoder : public Decoder { if (bytes_to_read > (int)byte_cache_.remaining_size()) { bytes_to_read = byte_cache_.remaining_size(); } - std::vector bytes(bytes_to_read); + // +8 zero padding: the SIMD unpack gathers 8 bytes per value and may + // read up to 7 bytes past the last group's end for small bit widths. + std::vector bytes(bytes_to_read + 8, 0); - for (int i = 0; i < bytes_to_read; i++) { - if (RET_FAIL(common::SerializationUtil::read_ui8(bytes[i], - byte_cache_))) { - return ret; - } + uint32_t got = 0; + if (RET_FAIL(byte_cache_.read_buf( + bytes.data(), static_cast(bytes_to_read), got))) { + return ret; + } + if (got != static_cast(bytes_to_read)) { + return common::E_PARTIAL_READ; } // save all int values in currentBuffer @@ -231,6 +280,16 @@ class Int32RleDecoder : public Decoder { if (RET_FAIL( common::SerializationUtil::read_var_uint(length_, buffer))) { return common::E_PARTIAL_READ; + } + // Prefer a read-only view over the caller's buffer instead of copying + // the whole pattern into tmp_buf_. `buffer` is the current value + // page's ByteStream and outlives the pattern decode; byte_cache_ only + // reads from it. Fall back to the copy when the stream is paged or + // shorter than the pattern. + if (buffer.is_wrapped() && buffer.remaining_size() >= length_) { + byte_cache_.wrap_from(buffer.get_wrapped_buf() + buffer.read_pos(), + static_cast(length_)); + buffer.wrapped_buf_advance_read_pos(length_); } else { if (tmp_buf_) { common::mem_free(tmp_buf_); @@ -248,15 +307,15 @@ class Int32RleDecoder : public Decoder { ret = common::E_PARTIAL_READ; } byte_cache_.wrap_from((char*)tmp_buf_, length_); - is_length_and_bitwidth_readed_ = true; - uint8_t tmp_bit_width; - common::SerializationUtil::read_ui8(tmp_bit_width, byte_cache_); - bit_width_ = tmp_bit_width; - if (packer_ != nullptr) { - delete packer_; - } - init_packer(); } + is_length_and_bitwidth_readed_ = true; + uint8_t tmp_bit_width; + common::SerializationUtil::read_ui8(tmp_bit_width, byte_cache_); + bit_width_ = tmp_bit_width; + if (packer_ != nullptr) { + delete packer_; + } + init_packer(); return ret; } diff --git a/cpp/src/encoding/int32_sprintz_decoder.h b/cpp/src/encoding/int32_sprintz_decoder.h index a7c92eede3..81c5127f0c 100644 --- a/cpp/src/encoding/int32_sprintz_decoder.h +++ b/cpp/src/encoding/int32_sprintz_decoder.h @@ -136,7 +136,9 @@ class Int32SprintzDecoder : public SprintzDecoder { pre_value_ = tmp_prev_value; current_buffer_[0] = pre_value_; - std::vector pack_buf(bit_width_); + // +8 zero bytes: the SIMD unpack gathers 8 bytes per value and + // may read up to 7 bytes past the packed group. + std::vector pack_buf(bit_width_ + 8, 0); uint32_t read_len = 0; input.read_buf(reinterpret_cast(pack_buf.data()), bit_width_, read_len); diff --git a/cpp/src/encoding/int64_packer.h b/cpp/src/encoding/int64_packer.h index 23267206f6..d9c0ef5894 100644 --- a/cpp/src/encoding/int64_packer.h +++ b/cpp/src/encoding/int64_packer.h @@ -24,14 +24,92 @@ #include "encoder.h" +#ifdef ENABLE_SIMD +#include "common/simd_dispatch.h" +#include "simde/x86/avx2.h" +#ifdef TSFILE_X86_AVX2_DISPATCH +#include +#endif +#endif + namespace storage { +#ifdef ENABLE_SIMD +// SIMD decode of 4 big-endian bit-packed INT64 values. Mirrors the TS_2DIFF +// helper but skips delta reconstruction: RLE packs raw values, so the gathered +// and shifted bits are already the final value (negative values are always +// stored with the full 64-bit width, so the logical shift is lossless). +// `in` points at the start of the packed group; `index` is the first value's +// position within the group (0 or 4 for an 8-value group). +static inline void rle_simd_unpack_4_i64(const uint8_t* in, int32_t width, + int32_t index, int64_t out[4]) { + static const simde__m256i SHUF_REV8 = simde_mm256_setr_epi8( + 7, 6, 5, 4, 3, 2, 1, 0, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, + 1, 0, 15, 14, 13, 12, 11, 10, 9, 8); + const int32_t pos0 = index * width; + const int32_t pos[4] = {pos0, pos0 + width, pos0 + 2 * width, + pos0 + 3 * width}; + const int32_t bidx[4] = {pos[0] >> 3, pos[1] >> 3, pos[2] >> 3, + pos[3] >> 3}; + const int32_t off[4] = {pos[0] & 7, pos[1] & 7, pos[2] & 7, pos[3] & 7}; + const simde__m128i IDX = + simde_mm_setr_epi32(bidx[0], bidx[1], bidx[2], bidx[3]); + const int32_t rshift = 64 - width; + simde__m256i w64_le = + simde_mm256_i32gather_epi64((const int64_t*)in, IDX, 1); + simde__m256i w64_be = simde_mm256_shuffle_epi8(w64_le, SHUF_REV8); + simde__m256i OFF64 = simde_mm256_cvtepu32_epi64( + simde_mm_setr_epi32(off[0], off[1], off[2], off[3])); + simde__m256i U64 = simde_mm256_sllv_epi64(w64_be, OFF64); + simde__m256i V64 = + simde_mm256_srl_epi64(U64, simde_mm_cvtsi32_si128(rshift)); + simde_mm256_storeu_si256((simde__m256i*)out, V64); +} + +#ifdef TSFILE_X86_AVX2_DISPATCH +// Same kernel with real AVX2 intrinsics. simde picks its native path only when +// the whole translation unit is compiled with -mavx2, which the library is not +// (portable baseline), so a target-attributed copy is needed to actually emit +// vpgatherqq/vpshufb. Selected once per packer via common::cpu_has_avx2(). +TSFILE_TARGET_AVX2 +static inline void rle_avx2_unpack_4_i64(const uint8_t* in, int32_t width, + int32_t index, int64_t out[4]) { + const __m256i SHUF_REV8 = + _mm256_setr_epi8(7, 6, 5, 4, 3, 2, 1, 0, 15, 14, 13, 12, 11, 10, 9, 8, + 7, 6, 5, 4, 3, 2, 1, 0, 15, 14, 13, 12, 11, 10, 9, 8); + const int32_t pos0 = index * width; + const int32_t pos[4] = {pos0, pos0 + width, pos0 + 2 * width, + pos0 + 3 * width}; + const __m128i IDX = + _mm_setr_epi32(pos[0] >> 3, pos[1] >> 3, pos[2] >> 3, pos[3] >> 3); + const int32_t rshift = 64 - width; + __m256i w64_le = _mm256_i32gather_epi64((const long long*)in, IDX, 1); + __m256i w64_be = _mm256_shuffle_epi8(w64_le, SHUF_REV8); + __m256i OFF64 = _mm256_cvtepu32_epi64( + _mm_setr_epi32(pos[0] & 7, pos[1] & 7, pos[2] & 7, pos[3] & 7)); + __m256i U64 = _mm256_sllv_epi64(w64_be, OFF64); + __m256i V64 = _mm256_srl_epi64(U64, _mm_cvtsi32_si128(rshift)); + _mm256_storeu_si256((__m256i*)out, V64); +} +#endif // TSFILE_X86_AVX2_DISPATCH +#endif + class Int64Packer { private: int width_; +#ifdef TSFILE_X86_AVX2_DISPATCH + // Resolved once per packer (one per bit-width pattern) so the hot + // unpack loop pays no thread-safe-static guard on every group. + bool use_avx2_; +#endif public: - Int64Packer(int width_) { this->width_ = width_; } +#ifdef TSFILE_X86_AVX2_DISPATCH + Int64Packer(int width_) + : width_(width_), use_avx2_(common::cpu_has_avx2()) {} +#else + Int64Packer(int width_) : width_(width_) {} +#endif ~Int64Packer() { destroy(); } void destroy() { /* do nothing for IntPacker */ @@ -99,18 +177,64 @@ class Int64Packer { * @param offset - offset of first byte to be decoded in buf * @param values - decoded result , the length of 'values' should be @{link * IntPacker#NUM_OF_INTS} + * + * The SIMD paths gather 8 bytes per value, so the buffer must hold at + * least 8 readable bytes past the packed group; callers allocate that + * padding (the RLE decoders zero it, the Sprintz decoders size their + * block buffer with +8). */ void unpack_8values(const unsigned char buf[], int offset, int64_t values[]) { +#ifdef ENABLE_SIMD + // Widths 59/61/62/63 can start at a bit offset that needs a ninth + // byte for the gather; decode those with the scalar path below. + if (width_ <= 58 || width_ == 60 || width_ == 64) { +#ifdef TSFILE_X86_AVX2_DISPATCH + if (use_avx2_) { + rle_avx2_unpack_4_i64(buf + offset, width_, 0, values); + rle_avx2_unpack_4_i64(buf + offset, width_, 4, values + 4); + return; + } +#endif + rle_simd_unpack_4_i64(buf + offset, width_, 0, values); + rle_simd_unpack_4_i64(buf + offset, width_, 4, values + 4); + return; + } +#endif + // Scalar fallback for widths that can cross a ninth byte (59/61/62/63). +#if defined(__SIZEOF_INT128__) + // Decode whole bytes into a 128-bit buffer — ~8x fewer iterations than + // the bit-by-bit loop — then shift out one width-wide value at a time. + // Widths up to 63 need up to 63 + 7 = 70 buffered bits, which a 64-bit + // buffer cannot hold; hence 128 bits. + int byte_idx = offset; + unsigned __int128 buffer = 0; + int total_bits = 0; + int value_idx = 0; + while (value_idx < 8) { + while (total_bits < width_) { + buffer = (buffer << 8) | (buf[byte_idx] & 0xFF); + byte_idx++; + total_bits += 8; + } + while (total_bits >= width_ && value_idx < 8) { + values[value_idx] = + static_cast(buffer >> (total_bits - width_)); + value_idx++; + total_bits -= width_; + buffer &= (((unsigned __int128)1 << total_bits) - 1); + } + } +#else + // MSVC has no 128-bit integer; fall back to the portable bit-by-bit + // loop used before the wide-buffer variant. int byte_idx = offset; int value_idx = 0; int left_bits = 8; int total_bits = 0; - while (value_idx < 8) { values[value_idx] = 0; total_bits = 0; - while (total_bits < width_) { if (width_ - total_bits >= left_bits) { values[value_idx] <<= left_bits; @@ -129,9 +253,9 @@ class Int64Packer { total_bits += t; } } - value_idx++; } +#endif } /** diff --git a/cpp/src/encoding/int64_rle_decoder.h b/cpp/src/encoding/int64_rle_decoder.h index bef72bbed1..0d8fa94f43 100644 --- a/cpp/src/encoding/int64_rle_decoder.h +++ b/cpp/src/encoding/int64_rle_decoder.h @@ -20,6 +20,8 @@ #ifndef ENCODING_INT64RLE_DECODER_H #define ENCODING_INT64RLE_DECODER_H +#include +#include #include #include "common/allocator/alloc_base.h" @@ -71,6 +73,55 @@ class Int64RleDecoder : public Decoder { int read_int64(int64_t& ret_value, common::ByteStream& in) override { return read_int(ret_value, in); } + int read_batch_int64(int64_t* out, int capacity, int& actual, + common::ByteStream& in) override { + actual = 0; + int ret = common::E_OK; + while (actual < capacity && has_remaining(in)) { + if (!is_length_and_bitwidth_readed_) { + if (RET_FAIL(read_length_and_bitwidth(in))) { + return ret; + } + } + if (current_count_ == 0) { + // Low bit = 0 => RLE run, = 1 => bit-packing group (same + // header dispatch as read_int). + uint32_t header_value = 0; + if (RET_FAIL(common::SerializationUtil::read_var_uint( + header_value, byte_cache_))) { + return ret; + } + if (header_value & 1) { + if (RET_FAIL(call_read_bit_packing_buffer(header_value))) { + return ret; + } + } else { + if (RET_FAIL(call_read_rle_run(header_value))) { + return ret; + } + } + } + const int n = std::min(current_count_, capacity - actual); + if (is_rle_run_) { + // The whole run holds the same value; a broadcast fill is + // auto-vectorized by the compiler. + std::fill(out + actual, out + actual + n, rle_value_); + } else { + // Bit-packing groups are already unpacked into current_buffer_ + // by the packer, so the remaining values are a straight copy. + const int64_t* src = + current_buffer_ + (bitpacking_num_ - current_count_); + std::memcpy(out + actual, src, + static_cast(n) * sizeof(int64_t)); + } + actual += n; + current_count_ -= n; + if (!has_next_package()) { + is_length_and_bitwidth_readed_ = false; + } + } + return ret; + } int read_float(float& ret_value, common::ByteStream& in) override { return common::E_TYPE_NOT_MATCH; } @@ -209,13 +260,17 @@ class Int64RleDecoder : public Decoder { if (bytes_to_read > (int)byte_cache_.remaining_size()) { bytes_to_read = byte_cache_.remaining_size(); } - std::vector bytes(bytes_to_read); + // +8 zero padding: the SIMD unpack gathers 8 bytes per value and may + // read up to 7 bytes past the last group's end for small bit widths. + std::vector bytes(bytes_to_read + 8, 0); - for (int i = 0; i < bytes_to_read; i++) { - if (RET_FAIL(common::SerializationUtil::read_ui8(bytes[i], - byte_cache_))) { - return ret; - } + uint32_t got = 0; + if (RET_FAIL(byte_cache_.read_buf( + bytes.data(), static_cast(bytes_to_read), got))) { + return ret; + } + if (got != static_cast(bytes_to_read)) { + return common::E_PARTIAL_READ; } // save all int values in currentBuffer @@ -230,6 +285,16 @@ class Int64RleDecoder : public Decoder { if (RET_FAIL( common::SerializationUtil::read_var_uint(length_, buffer))) { return common::E_PARTIAL_READ; + } + // Prefer a read-only view over the caller's buffer instead of copying + // the whole pattern into tmp_buf_. `buffer` is the current value + // page's ByteStream and outlives the pattern decode; byte_cache_ only + // reads from it. Fall back to the copy when the stream is paged or + // shorter than the pattern. + if (buffer.is_wrapped() && buffer.remaining_size() >= length_) { + byte_cache_.wrap_from(buffer.get_wrapped_buf() + buffer.read_pos(), + static_cast(length_)); + buffer.wrapped_buf_advance_read_pos(length_); } else { if (tmp_buf_) { common::mem_free(tmp_buf_); @@ -247,12 +312,12 @@ class Int64RleDecoder : public Decoder { ret = common::E_PARTIAL_READ; } byte_cache_.wrap_from((char*)tmp_buf_, length_); - is_length_and_bitwidth_readed_ = true; - uint8_t tmp_bit_width; - common::SerializationUtil::read_ui8(tmp_bit_width, byte_cache_); - bit_width_ = tmp_bit_width; - init_packer(); } + is_length_and_bitwidth_readed_ = true; + uint8_t tmp_bit_width; + common::SerializationUtil::read_ui8(tmp_bit_width, byte_cache_); + bit_width_ = tmp_bit_width; + init_packer(); return ret; } diff --git a/cpp/src/encoding/int64_sprintz_decoder.h b/cpp/src/encoding/int64_sprintz_decoder.h index 7b0827688d..245984670c 100644 --- a/cpp/src/encoding/int64_sprintz_decoder.h +++ b/cpp/src/encoding/int64_sprintz_decoder.h @@ -135,7 +135,9 @@ class Int64SprintzDecoder : public SprintzDecoder { current_buffer_[0] = pre_value_; // Read packed buffer - std::vector pack_buf(bit_width_); + // +8 zero bytes: the SIMD unpack gathers 8 bytes per value and + // may read up to 7 bytes past the packed group. + std::vector pack_buf(bit_width_ + 8, 0); uint32_t read_len = 0; input.read_buf(reinterpret_cast(pack_buf.data()), bit_width_, read_len); diff --git a/cpp/test/encoding/int32_packer_test.cc b/cpp/test/encoding/int32_packer_test.cc index 1b072383fe..3f4e151a3d 100644 --- a/cpp/test/encoding/int32_packer_test.cc +++ b/cpp/test/encoding/int32_packer_test.cc @@ -31,7 +31,8 @@ TEST(IntPackerTest, SequentialValues) { for (int i = 0; i < 8; ++i) arr[i] = i; Int32Packer packer(width); const int bufSize = NUM_OF_INTS * width / 8; - std::vector buf(bufSize, 0); + // +8: the SIMD unpack may read up to 7 bytes past the last group. + std::vector buf(bufSize + 8, 0); packer.pack_8values(arr, 0, buf.data()); int32_t res[8] = {0}; packer.unpack_8values(buf.data(), 0, res); @@ -50,7 +51,8 @@ TEST(IntPackerStressTest, PackUnpackRandomPositiveValues) { std::vector pre_values; std::vector buffer; pre_values.reserve(total_values); - buffer.resize(count * width); + // +8: the SIMD unpack may read up to 7 bytes past the last group. + buffer.resize(count * width + 8, 0); int idx = 0; std::srand(12345); // Optional: deterministic seed for (int i = 0; i < count; ++i) { @@ -68,8 +70,7 @@ TEST(IntPackerStressTest, PackUnpackRandomPositiveValues) { } std::vector res(total_values); - packer.unpack_all_values(buffer.data(), static_cast(buffer.size()), - res.data()); + packer.unpack_all_values(buffer.data(), count * width, res.data()); std::string diff_msg; for (int i = 0; i < total_values; ++i) { if (res[i] != pre_values[i]) { @@ -87,7 +88,8 @@ TEST(Int32PackerTest, AllZeroValues) { int32_t arr[NUM_OF_INTS] = {0}; Int32Packer packer(width); const int bufSize = NUM_OF_INTS * width / 8; - std::vector buf(bufSize, 0); + // +8: the SIMD unpack may read up to 7 bytes past the last group. + std::vector buf(bufSize + 8, 0); packer.pack_8values(arr, 0, buf.data()); int32_t res[NUM_OF_INTS] = { 1}; // initialize non-zero to catch failures @@ -104,7 +106,8 @@ TEST(Int32PackerTest, BoundaryWidthOneAlternating) { int32_t arr[NUM_OF_INTS] = {0, 1, 0, 1, 0, 1, 0, 1}; Int32Packer packer(width); const int bufSize = NUM_OF_INTS * width / 8; - std::vector buf(bufSize, 0); + // +8: the SIMD unpack may read up to 7 bytes past the last group. + std::vector buf(bufSize + 8, 0); packer.pack_8values(arr, 0, buf.data()); int32_t res[NUM_OF_INTS] = {0}; packer.unpack_8values(buf.data(), 0, res); @@ -127,7 +130,8 @@ TEST(Int32PackerTest, MaxWidth32Random) { } Int32Packer packer(width); const int bufSize = NUM_OF_INTS * width / 8; - std::vector buf(bufSize, 0); + // +8: the SIMD unpack may read up to 7 bytes past the last group. + std::vector buf(bufSize + 8, 0); packer.pack_8values(arr, 0, buf.data()); int32_t res[NUM_OF_INTS] = {0}; packer.unpack_8values(buf.data(), 0, res); @@ -150,7 +154,8 @@ TEST(Int32PackerTest, AllNegative32Random) { } Int32Packer packer(width); const int bufSize = NUM_OF_INTS * width / 8; - std::vector buf(bufSize, 0); + // +8: the SIMD unpack may read up to 7 bytes past the last group. + std::vector buf(bufSize + 8, 0); packer.pack_8values(arr, 0, buf.data()); int32_t res[NUM_OF_INTS] = {0}; packer.unpack_8values(buf.data(), 0, res); @@ -167,7 +172,7 @@ TEST(Int32PackerTest, UnpackAllValuesMultipleBlocks) { const int blocks = 10; Int32Packer packer(width); std::vector orig(blocks * NUM_OF_INTS); - std::vector buf(blocks * width); + std::vector buf(blocks * width + 8, 0); // Fill orig with pattern: block * 16 + index // Example: block 0 = [0,1,...,7], block 1 = [16,17,...,23], etc. @@ -182,8 +187,7 @@ TEST(Int32PackerTest, UnpackAllValuesMultipleBlocks) { std::vector res(blocks * NUM_OF_INTS, 0); // Unpack all blocks at once - packer.unpack_all_values(buf.data(), static_cast(buf.size()), - res.data()); + packer.unpack_all_values(buf.data(), blocks * width, res.data()); // Verify each unpacked value matches the original sequence for (size_t i = 0; i < orig.size(); ++i) { diff --git a/cpp/test/encoding/int64_packer_test.cc b/cpp/test/encoding/int64_packer_test.cc index 846f9a0c9f..2e463f7f35 100644 --- a/cpp/test/encoding/int64_packer_test.cc +++ b/cpp/test/encoding/int64_packer_test.cc @@ -32,7 +32,8 @@ TEST(Int64PackerTest, SequentialValues) { for (int i = 0; i < 8; ++i) arr[i] = i; Int64Packer packer(width); const int bufSize = NUM_OF_INTS * width / 8; - std::vector buf(bufSize, 0); + // +8: the SIMD unpack may read up to 7 bytes past the last group. + std::vector buf(bufSize + 8, 0); packer.pack_8values(arr, 0, buf.data()); int64_t res[8] = {0}; packer.unpack_8values(buf.data(), 0, res); @@ -49,7 +50,7 @@ TEST(Int64PackerTest, PackUnpackSingleBatchRandomPositiveLongs) { Int64Packer packer(byte_count); std::vector pre_values; - std::vector buffer(count * byte_count); + std::vector buffer(count * byte_count + 8, 0); pre_values.reserve(total_values); int idx = 0; @@ -72,8 +73,7 @@ TEST(Int64PackerTest, PackUnpackSingleBatchRandomPositiveLongs) { } std::vector result(total_values); - packer.unpack_all_values(buffer.data(), static_cast(buffer.size()), - result.data()); + packer.unpack_all_values(buffer.data(), count * byte_count, result.data()); for (int i = 0; i < total_values; ++i) { ASSERT_EQ(result[i], pre_values[i]) << "Mismatch at index " << i; @@ -124,6 +124,8 @@ TEST(Int64PackerTest, PackAllManualBitWidth) { std::string packed_data = oss.str(); ASSERT_EQ(static_cast(packed_data.size()), 2 * bp_bit_width); + // +8: the SIMD unpack may read up to 7 bytes past the last group. + packed_data.append(8, '\0'); // Decode int64_t read_array[16] = {0}; @@ -143,7 +145,8 @@ TEST(Int64PackerTest, AllZeroValues) { int64_t arr[NUM_OF_INTS] = {0}; Int64Packer packer(width); const int bufSize = NUM_OF_INTS * width / 8; - std::vector buf(bufSize, 0); + // +8: the SIMD unpack may read up to 7 bytes past the last group. + std::vector buf(bufSize + 8, 0); packer.pack_8values(arr, 0, buf.data()); int64_t res[NUM_OF_INTS] = { 1}; // initialize non-zero to catch failures @@ -160,7 +163,8 @@ TEST(Int64PackerTest, BoundaryWidthOneAlternating) { int64_t arr[NUM_OF_INTS] = {0, 1, 0, 1, 0, 1, 0, 1}; Int64Packer packer(width); const int bufSize = NUM_OF_INTS * width / 8; - std::vector buf(bufSize, 0); + // +8: the SIMD unpack may read up to 7 bytes past the last group. + std::vector buf(bufSize + 8, 0); packer.pack_8values(arr, 0, buf.data()); int64_t res[NUM_OF_INTS] = {0}; packer.unpack_8values(buf.data(), 0, res); @@ -183,7 +187,8 @@ TEST(Int64PackerTest, MaxWidth64Random) { } Int64Packer packer(width); const int bufSize = NUM_OF_INTS * width / 8; - std::vector buf(bufSize, 0); + // +8: the SIMD unpack may read up to 7 bytes past the last group. + std::vector buf(bufSize + 8, 0); packer.pack_8values(arr, 0, buf.data()); int64_t res[NUM_OF_INTS] = {0}; packer.unpack_8values(buf.data(), 0, res); @@ -206,7 +211,8 @@ TEST(Int64PackerTest, AllNegative64Random) { } Int64Packer packer(width); const int bufSize = NUM_OF_INTS * width / 8; - std::vector buf(bufSize, 0); + // +8: the SIMD unpack may read up to 7 bytes past the last group. + std::vector buf(bufSize + 8, 0); packer.pack_8values(arr, 0, buf.data()); int64_t res[NUM_OF_INTS] = {0}; packer.unpack_8values(buf.data(), 0, res); @@ -223,7 +229,7 @@ TEST(Int64PackerTest, UnpackAllValuesMultipleBlocks) { const int blocks = 10; Int64Packer packer(width); std::vector orig(blocks * NUM_OF_INTS); - std::vector buf(blocks * width); + std::vector buf(blocks * width + 8, 0); // Fill orig with pattern: block * 16 + index // Example: block 0 = [0,1,...,7], block 1 = [16,17,...,23], etc. @@ -238,8 +244,7 @@ TEST(Int64PackerTest, UnpackAllValuesMultipleBlocks) { std::vector res(blocks * NUM_OF_INTS, 0); // Unpack all blocks at once - packer.unpack_all_values(buf.data(), static_cast(buf.size()), - res.data()); + packer.unpack_all_values(buf.data(), blocks * width, res.data()); // Verify each unpacked value matches the original sequence for (size_t i = 0; i < orig.size(); ++i) { From 1fb68ff539860c63267c10e83356a71ad42bdd07 Mon Sep 17 00:00:00 2001 From: ColinLee Date: Mon, 9 Nov 2026 11:39:26 +0800 Subject: [PATCH 2/2] fix(cpp): harden RLE batch decode against malformed headers A corrupt bit-packing header could size the new bulk read with a wrapped uint32 product, turn bytes_to_read negative, and write the rest of the pattern past a 7-byte scratch vector. Validate the bit width, group byte count and last_bit_packed_num before sizing anything, and reject patterns whose declared groups do not fit instead of clamping them. - read_bit_packing_buffer: compute group_bytes in 64-bit, bound it by the remaining pattern bytes and mem_alloc's uint32 argument; validate bit_width_ (1..32 / 1..64). - call_read_bit_packing_buffer: require last_bit_packed_num in [1,8], bound the value count by INT32_MAX, and clear the count when the group load fails; read_batch_* now refuses a non-positive count instead of passing it to memcpy. - read_length_and_bitwidth: reject oversized length prefixes, check the bit-width read, and drop the borrowed byte_cache_ view on reset()/destroy(); the int64 decoder deletes the old packer per pattern. - unpack_all_values now decodes the final group from a zero-padded copy so callers no longer need the +8 tail; guard zero-width non-advancing loops. - hoist the simde shuffle mask to a constant byte array, removing the thread-safe-static guard from every kernel call. - Sprintz decoders reject out-of-range block widths. Add differential batch/scalar tests (wrapped and owning streams, capacities 1..1024, mixed run and bit-packed segments, cross-pattern batches, reset and reuse), malformed-header rejection tests, Float/Double RLE batch tests, and exact-size packer tests. --- cpp/src/encoding/double_sprintz_decoder.h | 5 + cpp/src/encoding/float_sprintz_decoder.h | 5 + cpp/src/encoding/int32_packer.h | 59 ++++- cpp/src/encoding/int32_rle_decoder.h | 113 ++++++--- cpp/src/encoding/int32_sprintz_decoder.h | 5 + cpp/src/encoding/int64_packer.h | 59 ++++- cpp/src/encoding/int64_rle_decoder.h | 116 ++++++--- cpp/src/encoding/int64_sprintz_decoder.h | 5 + cpp/test/encoding/float_rle_codec_test.cc | 218 ++++++++++++++++ cpp/test/encoding/int32_packer_test.cc | 33 +++ cpp/test/encoding/int32_rle_codec_test.cc | 292 ++++++++++++++++++++++ cpp/test/encoding/int64_packer_test.cc | 42 ++++ cpp/test/encoding/int64_rle_codec_test.cc | 255 +++++++++++++++++++ 13 files changed, 1113 insertions(+), 94 deletions(-) create mode 100644 cpp/test/encoding/float_rle_codec_test.cc diff --git a/cpp/src/encoding/double_sprintz_decoder.h b/cpp/src/encoding/double_sprintz_decoder.h index c189192e18..69bf5ba873 100644 --- a/cpp/src/encoding/double_sprintz_decoder.h +++ b/cpp/src/encoding/double_sprintz_decoder.h @@ -127,6 +127,11 @@ class DoubleSprintzDecoder : public SprintzDecoder { } } } else { + if (bit_width_ > 64) { + // Int64Packer supports at most 64 bits per value; the header + // byte is not otherwise validated. + return common::E_DECODE_ERR; + } decode_size_ = block_size_ + 1; common::SerializationUtil::read_double(pre_value_, input); current_buffer_[0] = pre_value_; diff --git a/cpp/src/encoding/float_sprintz_decoder.h b/cpp/src/encoding/float_sprintz_decoder.h index c97cc1551f..3dd42f0f70 100644 --- a/cpp/src/encoding/float_sprintz_decoder.h +++ b/cpp/src/encoding/float_sprintz_decoder.h @@ -128,6 +128,11 @@ class FloatSprintzDecoder : public SprintzDecoder { } } } else { + if (bit_width_ > 32) { + // Int32Packer supports at most 32 bits per value; the header + // byte is not otherwise validated. + return common::E_DECODE_ERR; + } // packed block decode_size_ = block_size_ + 1; common::SerializationUtil::read_float(pre_value_, input); diff --git a/cpp/src/encoding/int32_packer.h b/cpp/src/encoding/int32_packer.h index 3ff0b81b7a..a37efc34c2 100644 --- a/cpp/src/encoding/int32_packer.h +++ b/cpp/src/encoding/int32_packer.h @@ -22,6 +22,9 @@ #define NUM_OF_INTS 8 +#include +#include + #include "encoder.h" #ifdef ENABLE_SIMD @@ -35,14 +38,20 @@ namespace storage { #ifdef ENABLE_SIMD +namespace rle_simd_detail { +// A byte array (constant-initialized, no thread-safe-static guard) that the +// hot kernel loads into a vector. A function-local static simde__m256i would +// instead be dynamically initialized and pay an __cxa_guard_acquire per call. +static const unsigned char SHUF_REV8_I32_BYTES[32] = { + 7, 6, 5, 4, 3, 2, 1, 0, 15, 14, 13, 12, 11, 10, 9, 8, + 7, 6, 5, 4, 3, 2, 1, 0, 15, 14, 13, 12, 11, 10, 9, 8}; +} // namespace rle_simd_detail + // SIMD decode of 4 big-endian bit-packed INT32 values. Uses the 64-bit gather // so all widths 1..32 fit without a width guard; the trailing permute packs // the low 32 bits of each 64-bit lane into four contiguous INT32 lanes. static inline void rle_simd_unpack_4_i32(const uint8_t* in, int32_t width, int32_t index, int32_t out[4]) { - static const simde__m256i SHUF_REV8 = simde_mm256_setr_epi8( - 7, 6, 5, 4, 3, 2, 1, 0, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, - 1, 0, 15, 14, 13, 12, 11, 10, 9, 8); const int32_t pos0 = index * width; const int32_t pos[4] = {pos0, pos0 + width, pos0 + 2 * width, pos0 + 3 * width}; @@ -54,7 +63,9 @@ static inline void rle_simd_unpack_4_i32(const uint8_t* in, int32_t width, const int32_t rshift = 64 - width; simde__m256i w64_le = simde_mm256_i32gather_epi64((const int64_t*)in, IDX, 1); - simde__m256i w64_be = simde_mm256_shuffle_epi8(w64_le, SHUF_REV8); + const simde__m256i shuf_mask = simde_mm256_loadu_si256( + (const simde__m256i*)rle_simd_detail::SHUF_REV8_I32_BYTES); + simde__m256i w64_be = simde_mm256_shuffle_epi8(w64_le, shuf_mask); simde__m256i OFF64 = simde_mm256_cvtepu32_epi64( simde_mm_setr_epi32(off[0], off[1], off[2], off[3])); simde__m256i U64 = simde_mm256_sllv_epi64(w64_be, OFF64); @@ -177,10 +188,10 @@ class Int32Packer { * @param values - decoded result , the length of 'values' should be @{link * IntPacker#NUM_OF_INTS} * - * The SIMD paths gather 8 bytes per value, so the buffer must hold at - * least 8 readable bytes past the packed group; callers allocate that - * padding (the RLE decoders zero it, the Sprintz decoders size their - * block buffer with +8). + * The SIMD paths gather 8 bytes per value, so a direct caller must hold + * at least 8 readable bytes past the packed group (the Sprintz decoders + * size their block buffer with +8). unpack_all_values() removes that + * requirement by decoding the final group from a zero-padded copy. */ void unpack_8values(const unsigned char buf[], int offset, int32_t values[]) { @@ -236,16 +247,44 @@ class Int32Packer { */ void unpack_all_values(const unsigned char buf[], int length, int32_t values[]) { + if (width_ <= 0) { + return; // invalid width; avoid the non-advancing loop below + } int idx = 0; int k = 0; while (idx < length) { int32_t tv[8]; // decode 8 values one time, current result will be saved in the // array named 'tv' - unpack_8values(buf, idx, tv); + if (static_cast(idx) + width_ + 8 > length) { + // unpack_8values may read up to 7 bytes past the group. + // Decode the final (possibly partial) group from a + // zero-padded copy so callers can pass an exact-size buffer. + const int avail = length - idx; + const int copy_len = avail < width_ ? avail : width_; + if (width_ <= 64) { + unsigned char tail[64 + 8] = {0}; + std::memcpy(tail, buf + idx, static_cast(copy_len)); + unpack_8values(tail, 0, tv); + } else { + // Width is outside this packer's valid range; stay + // memory-safe without a fixed-size stack buffer. + std::vector tail( + static_cast(width_) + 8, 0); + std::memcpy(tail.data(), buf + idx, + static_cast(copy_len)); + unpack_8values(tail.data(), 0, tv); + } + } else { + unpack_8values(buf, idx, tv); + } // System.arraycopy(tv, 0, values, k, 8); std::memmove(values + k, tv, 8 * sizeof(int32_t)); - idx += width_; + if (width_ > length - idx) { + idx = length; + } else { + idx += width_; + } k += 8; } } diff --git a/cpp/src/encoding/int32_rle_decoder.h b/cpp/src/encoding/int32_rle_decoder.h index 09e6318783..72dba87189 100644 --- a/cpp/src/encoding/int32_rle_decoder.h +++ b/cpp/src/encoding/int32_rle_decoder.h @@ -21,6 +21,7 @@ #define ENCODING_INT32RLE_DECODER_H #include +#include #include #include @@ -101,6 +102,11 @@ class Int32RleDecoder : public Decoder { } } } + if (current_count_ <= 0) { + // A validated header always yields at least one value; a + // non-positive count here means the pattern was corrupted. + return common::E_TSFILE_CORRUPTED; + } const int n = std::min(current_count_, capacity - actual); if (is_rle_run_) { std::fill(out + actual, out + actual + n, rle_value_); @@ -132,16 +138,7 @@ class Int32RleDecoder : public Decoder { return common::E_TYPE_NOT_MATCH; } - void init() { - packer_ = nullptr; - is_length_and_bitwidth_readed_ = false; - length_ = 0; - bit_width_ = 0; - bitpacking_num_ = 0; - current_count_ = 0; - is_rle_run_ = false; - rle_value_ = 0; - } + void init() { reset(); } bool has_next(common::ByteStream& buffer) { if (current_count_ > 0 || buffer.remaining_size() > 0 || @@ -226,47 +223,77 @@ class Int32RleDecoder : public Decoder { byte_cache_))) { return ret; } - if (bit_packed_group_count > 0) { - current_count_ = - (bit_packed_group_count - 1) * 8 + last_bit_packed_num; - bitpacking_num_ = current_count_; - is_rle_run_ = false; - } else { + if (bit_packed_group_count <= 0) { return common::E_DECODE_ERR; } + if (last_bit_packed_num == 0 || last_bit_packed_num > NUM_OF_INTS) { + // Encoders write 1..8. Anything else makes current_count_ disagree + // with the number of values actually packed, which the batch + // memcpy would then use as an unbounded length. + return common::E_TSFILE_CORRUPTED; + } + const int64_t value_count = + (static_cast(bit_packed_group_count) - 1) * NUM_OF_INTS + + last_bit_packed_num; + if (value_count > INT32_MAX) { + return common::E_TSFILE_CORRUPTED; + } + current_count_ = static_cast(value_count); + bitpacking_num_ = current_count_; + is_rle_run_ = false; ret = read_bit_packing_buffer(bit_packed_group_count, last_bit_packed_num); + if (ret != common::E_OK) { + // Do not leave a value count pointing at an unfilled buffer. + current_count_ = 0; + bitpacking_num_ = 0; + } return ret; } int read_bit_packing_buffer(int bit_packed_group_count, int last_bit_packed_num) { + (void)last_bit_packed_num; int ret = common::E_OK; + // bit_width_ comes from the file. Validate it before using it to size + // the bulk read: a 32-bit product of a corrupt group count and width + // can wrap to a negative byte count and make the read below write past + // the bytes vector. + if (bit_width_ == 0 || bit_width_ > 32) { + return common::E_TSFILE_CORRUPTED; + } + const uint64_t group_bytes = + static_cast(bit_packed_group_count) * bit_width_; + if (group_bytes > byte_cache_.remaining_size() || + group_bytes > static_cast(INT32_MAX)) { + // The declared groups do not fit in the pattern. + return common::E_PARTIAL_READ; + } + const uint64_t value_count = + static_cast(bit_packed_group_count) * NUM_OF_INTS; + if (value_count > UINT32_MAX / sizeof(int32_t)) { + // The value buffer size must fit mem_alloc's uint32_t argument. + return common::E_TSFILE_CORRUPTED; + } + if (current_buffer_ != nullptr) { common::mem_free(current_buffer_); + current_buffer_ = nullptr; } - current_buffer_ = static_cast( - common::mem_alloc(sizeof(int32_t) * bit_packed_group_count * 8, - common::MOD_DECODER_OBJ)); + current_buffer_ = static_cast(common::mem_alloc( + static_cast(value_count * sizeof(int32_t)), + common::MOD_DECODER_OBJ)); if (IS_NULL(current_buffer_)) { return common::E_OOM; } - int bytes_to_read = bit_packed_group_count * bit_width_; - if (bytes_to_read > (int)byte_cache_.remaining_size()) { - bytes_to_read = byte_cache_.remaining_size(); - } - // +8 zero padding: the SIMD unpack gathers 8 bytes per value and may - // read up to 7 bytes past the last group's end for small bit widths. - std::vector bytes(bytes_to_read + 8, 0); + const int bytes_to_read = static_cast(group_bytes); + std::vector bytes(bytes_to_read, 0); uint32_t got = 0; if (RET_FAIL(byte_cache_.read_buf( bytes.data(), static_cast(bytes_to_read), got))) { return ret; } - if (got != static_cast(bytes_to_read)) { - return common::E_PARTIAL_READ; - } // save all int values in currentBuffer packer_->unpack_all_values( @@ -281,18 +308,22 @@ class Int32RleDecoder : public Decoder { common::SerializationUtil::read_var_uint(length_, buffer))) { return common::E_PARTIAL_READ; } + if (length_ > static_cast(INT32_MAX)) { + return common::E_TSFILE_CORRUPTED; + } // Prefer a read-only view over the caller's buffer instead of copying // the whole pattern into tmp_buf_. `buffer` is the current value // page's ByteStream and outlives the pattern decode; byte_cache_ only - // reads from it. Fall back to the copy when the stream is paged or - // shorter than the pattern. + // reads from it and is cleared by reset()/destroy(). Fall back to the + // copy when the stream is paged or shorter than the pattern. if (buffer.is_wrapped() && buffer.remaining_size() >= length_) { byte_cache_.wrap_from(buffer.get_wrapped_buf() + buffer.read_pos(), static_cast(length_)); buffer.wrapped_buf_advance_read_pos(length_); } else { - if (tmp_buf_) { + if (tmp_buf_ != nullptr) { common::mem_free(tmp_buf_); + tmp_buf_ = nullptr; } tmp_buf_ = (uint8_t*)common::mem_alloc(length_, common::MOD_DECODER_OBJ); @@ -303,25 +334,32 @@ class Int32RleDecoder : public Decoder { if (RET_FAIL(buffer.read_buf((uint8_t*)tmp_buf_, length_, ret_read_len))) { return ret; - } else if (length_ != ret_read_len) { - ret = common::E_PARTIAL_READ; } byte_cache_.wrap_from((char*)tmp_buf_, length_); } - is_length_and_bitwidth_readed_ = true; - uint8_t tmp_bit_width; - common::SerializationUtil::read_ui8(tmp_bit_width, byte_cache_); + uint8_t tmp_bit_width = 0; + if (RET_FAIL(common::SerializationUtil::read_ui8(tmp_bit_width, + byte_cache_))) { + return ret; + } + if (tmp_bit_width == 0 || tmp_bit_width > 32) { + return common::E_TSFILE_CORRUPTED; + } bit_width_ = tmp_bit_width; if (packer_ != nullptr) { delete packer_; } init_packer(); + is_length_and_bitwidth_readed_ = true; return ret; } void init_packer() { packer_ = new Int32Packer(bit_width_); } void destroy() { /* do nothing for BitpackEncoder */ + // Drop the borrowed pattern view before freeing tmp_buf_ so no stale + // pointer into caller memory (or into tmp_buf_) survives. + byte_cache_.reset(); if (packer_) { delete (packer_); packer_ = nullptr; @@ -344,6 +382,7 @@ class Int32RleDecoder : public Decoder { current_count_ = 0; is_rle_run_ = false; rle_value_ = 0; + byte_cache_.reset(); if (current_buffer_) { common::mem_free(current_buffer_); current_buffer_ = nullptr; diff --git a/cpp/src/encoding/int32_sprintz_decoder.h b/cpp/src/encoding/int32_sprintz_decoder.h index 81c5127f0c..7f4016fc3c 100644 --- a/cpp/src/encoding/int32_sprintz_decoder.h +++ b/cpp/src/encoding/int32_sprintz_decoder.h @@ -130,6 +130,11 @@ class Int32SprintzDecoder : public SprintzDecoder { } } } else { + if (bit_width_ > 32) { + // Int32Packer supports at most 32 bits per value; the header + // byte is not otherwise validated. + return common::E_DECODE_ERR; + } decode_size_ = block_size_ + 1; uint32_t tmp_prev_value; common::SerializationUtil::read_var_uint(tmp_prev_value, input); diff --git a/cpp/src/encoding/int64_packer.h b/cpp/src/encoding/int64_packer.h index d9c0ef5894..9c68954bf9 100644 --- a/cpp/src/encoding/int64_packer.h +++ b/cpp/src/encoding/int64_packer.h @@ -22,6 +22,9 @@ #define NUM_OF_INTS 8 +#include +#include + #include "encoder.h" #ifdef ENABLE_SIMD @@ -35,6 +38,15 @@ namespace storage { #ifdef ENABLE_SIMD +namespace rle_simd_detail { +// A byte array (constant-initialized, no thread-safe-static guard) that the +// hot kernel loads into a vector. A function-local static simde__m256i would +// instead be dynamically initialized and pay an __cxa_guard_acquire per call. +static const unsigned char SHUF_REV8_I64_BYTES[32] = { + 7, 6, 5, 4, 3, 2, 1, 0, 15, 14, 13, 12, 11, 10, 9, 8, + 7, 6, 5, 4, 3, 2, 1, 0, 15, 14, 13, 12, 11, 10, 9, 8}; +} // namespace rle_simd_detail + // SIMD decode of 4 big-endian bit-packed INT64 values. Mirrors the TS_2DIFF // helper but skips delta reconstruction: RLE packs raw values, so the gathered // and shifted bits are already the final value (negative values are always @@ -43,9 +55,6 @@ namespace storage { // position within the group (0 or 4 for an 8-value group). static inline void rle_simd_unpack_4_i64(const uint8_t* in, int32_t width, int32_t index, int64_t out[4]) { - static const simde__m256i SHUF_REV8 = simde_mm256_setr_epi8( - 7, 6, 5, 4, 3, 2, 1, 0, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, - 1, 0, 15, 14, 13, 12, 11, 10, 9, 8); const int32_t pos0 = index * width; const int32_t pos[4] = {pos0, pos0 + width, pos0 + 2 * width, pos0 + 3 * width}; @@ -57,7 +66,9 @@ static inline void rle_simd_unpack_4_i64(const uint8_t* in, int32_t width, const int32_t rshift = 64 - width; simde__m256i w64_le = simde_mm256_i32gather_epi64((const int64_t*)in, IDX, 1); - simde__m256i w64_be = simde_mm256_shuffle_epi8(w64_le, SHUF_REV8); + const simde__m256i shuf_mask = simde_mm256_loadu_si256( + (const simde__m256i*)rle_simd_detail::SHUF_REV8_I64_BYTES); + simde__m256i w64_be = simde_mm256_shuffle_epi8(w64_le, shuf_mask); simde__m256i OFF64 = simde_mm256_cvtepu32_epi64( simde_mm_setr_epi32(off[0], off[1], off[2], off[3])); simde__m256i U64 = simde_mm256_sllv_epi64(w64_be, OFF64); @@ -178,10 +189,10 @@ class Int64Packer { * @param values - decoded result , the length of 'values' should be @{link * IntPacker#NUM_OF_INTS} * - * The SIMD paths gather 8 bytes per value, so the buffer must hold at - * least 8 readable bytes past the packed group; callers allocate that - * padding (the RLE decoders zero it, the Sprintz decoders size their - * block buffer with +8). + * The SIMD paths gather 8 bytes per value, so a direct caller must hold + * at least 8 readable bytes past the packed group (the Sprintz decoders + * size their block buffer with +8). unpack_all_values() removes that + * requirement by decoding the final group from a zero-padded copy. */ void unpack_8values(const unsigned char buf[], int offset, int64_t values[]) { @@ -268,16 +279,44 @@ class Int64Packer { */ void unpack_all_values(const unsigned char buf[], int length, int64_t values[]) { + if (width_ <= 0) { + return; // invalid width; avoid the non-advancing loop below + } int idx = 0; int k = 0; while (idx < length) { int64_t tv[8]; // decode 8 values one time, current result will be saved in the // array named 'tv' - unpack_8values(buf, idx, tv); + if (static_cast(idx) + width_ + 8 > length) { + // unpack_8values may read up to 7 bytes past the group. + // Decode the final (possibly partial) group from a + // zero-padded copy so callers can pass an exact-size buffer. + const int avail = length - idx; + const int copy_len = avail < width_ ? avail : width_; + if (width_ <= 64) { + unsigned char tail[64 + 8] = {0}; + std::memcpy(tail, buf + idx, static_cast(copy_len)); + unpack_8values(tail, 0, tv); + } else { + // Width is outside this packer's valid range; stay + // memory-safe without a fixed-size stack buffer. + std::vector tail( + static_cast(width_) + 8, 0); + std::memcpy(tail.data(), buf + idx, + static_cast(copy_len)); + unpack_8values(tail.data(), 0, tv); + } + } else { + unpack_8values(buf, idx, tv); + } // System.arraycopy(tv, 0, values, k, 8); std::memmove(values + k, tv, 8 * sizeof(int64_t)); - idx += width_; + if (width_ > length - idx) { + idx = length; + } else { + idx += width_; + } k += 8; } } diff --git a/cpp/src/encoding/int64_rle_decoder.h b/cpp/src/encoding/int64_rle_decoder.h index 0d8fa94f43..4d37c1c044 100644 --- a/cpp/src/encoding/int64_rle_decoder.h +++ b/cpp/src/encoding/int64_rle_decoder.h @@ -21,6 +21,7 @@ #define ENCODING_INT64RLE_DECODER_H #include +#include #include #include @@ -101,6 +102,11 @@ class Int64RleDecoder : public Decoder { } } } + if (current_count_ <= 0) { + // A validated header always yields at least one value; a + // non-positive count here means the pattern was corrupted. + return common::E_TSFILE_CORRUPTED; + } const int n = std::min(current_count_, capacity - actual); if (is_rle_run_) { // The whole run holds the same value; a broadcast fill is @@ -133,16 +139,7 @@ class Int64RleDecoder : public Decoder { return common::E_TYPE_NOT_MATCH; } - void init() { - packer_ = nullptr; - is_length_and_bitwidth_readed_ = false; - length_ = 0; - bit_width_ = 0; - bitpacking_num_ = 0; - current_count_ = 0; - is_rle_run_ = false; - rle_value_ = 0; - } + void init() { reset(); } bool has_next(common::ByteStream& buffer) { if (current_count_ > 0 || buffer.remaining_size() > 0 || @@ -231,47 +228,77 @@ class Int64RleDecoder : public Decoder { byte_cache_))) { return ret; } - if (bit_packed_group_count > 0) { - current_count_ = - (bit_packed_group_count - 1) * 8 + last_bit_packed_num; - bitpacking_num_ = current_count_; - is_rle_run_ = false; - } else { + if (bit_packed_group_count <= 0) { return common::E_DECODE_ERR; } + if (last_bit_packed_num == 0 || last_bit_packed_num > NUM_OF_INTS) { + // Encoders write 1..8. Anything else makes current_count_ disagree + // with the number of values actually packed, which the batch + // memcpy would then use as an unbounded length. + return common::E_TSFILE_CORRUPTED; + } + const int64_t value_count = + (static_cast(bit_packed_group_count) - 1) * NUM_OF_INTS + + last_bit_packed_num; + if (value_count > INT32_MAX) { + return common::E_TSFILE_CORRUPTED; + } + current_count_ = static_cast(value_count); + bitpacking_num_ = current_count_; + is_rle_run_ = false; ret = read_bit_packing_buffer(bit_packed_group_count, last_bit_packed_num); + if (ret != common::E_OK) { + // Do not leave a value count pointing at an unfilled buffer. + current_count_ = 0; + bitpacking_num_ = 0; + } return ret; } int read_bit_packing_buffer(int bit_packed_group_count, int last_bit_packed_num) { + (void)last_bit_packed_num; int ret = common::E_OK; + // bit_width_ comes from the file. Validate it before using it to size + // the bulk read: a 32-bit product of a corrupt group count and width + // can wrap to a negative byte count and make the read below write past + // the bytes vector. + if (bit_width_ == 0 || bit_width_ > 64) { + return common::E_TSFILE_CORRUPTED; + } + const uint64_t group_bytes = + static_cast(bit_packed_group_count) * bit_width_; + if (group_bytes > byte_cache_.remaining_size() || + group_bytes > static_cast(INT32_MAX)) { + // The declared groups do not fit in the pattern. + return common::E_PARTIAL_READ; + } + const uint64_t value_count = + static_cast(bit_packed_group_count) * NUM_OF_INTS; + if (value_count > UINT32_MAX / sizeof(int64_t)) { + // The value buffer size must fit mem_alloc's uint32_t argument. + return common::E_TSFILE_CORRUPTED; + } + if (current_buffer_ != nullptr) { common::mem_free(current_buffer_); + current_buffer_ = nullptr; } - current_buffer_ = static_cast( - common::mem_alloc(sizeof(int64_t) * bit_packed_group_count * 8, - common::MOD_DECODER_OBJ)); + current_buffer_ = static_cast(common::mem_alloc( + static_cast(value_count * sizeof(int64_t)), + common::MOD_DECODER_OBJ)); if (IS_NULL(current_buffer_)) { return common::E_OOM; } - int bytes_to_read = bit_packed_group_count * bit_width_; - if (bytes_to_read > (int)byte_cache_.remaining_size()) { - bytes_to_read = byte_cache_.remaining_size(); - } - // +8 zero padding: the SIMD unpack gathers 8 bytes per value and may - // read up to 7 bytes past the last group's end for small bit widths. - std::vector bytes(bytes_to_read + 8, 0); + const int bytes_to_read = static_cast(group_bytes); + std::vector bytes(bytes_to_read, 0); uint32_t got = 0; if (RET_FAIL(byte_cache_.read_buf( bytes.data(), static_cast(bytes_to_read), got))) { return ret; } - if (got != static_cast(bytes_to_read)) { - return common::E_PARTIAL_READ; - } // save all int values in currentBuffer packer_->unpack_all_values( @@ -286,18 +313,22 @@ class Int64RleDecoder : public Decoder { common::SerializationUtil::read_var_uint(length_, buffer))) { return common::E_PARTIAL_READ; } + if (length_ > static_cast(INT32_MAX)) { + return common::E_TSFILE_CORRUPTED; + } // Prefer a read-only view over the caller's buffer instead of copying // the whole pattern into tmp_buf_. `buffer` is the current value // page's ByteStream and outlives the pattern decode; byte_cache_ only - // reads from it. Fall back to the copy when the stream is paged or - // shorter than the pattern. + // reads from it and is cleared by reset()/destroy(). Fall back to the + // copy when the stream is paged or shorter than the pattern. if (buffer.is_wrapped() && buffer.remaining_size() >= length_) { byte_cache_.wrap_from(buffer.get_wrapped_buf() + buffer.read_pos(), static_cast(length_)); buffer.wrapped_buf_advance_read_pos(length_); } else { - if (tmp_buf_) { + if (tmp_buf_ != nullptr) { common::mem_free(tmp_buf_); + tmp_buf_ = nullptr; } tmp_buf_ = (uint8_t*)common::mem_alloc(length_, common::MOD_DECODER_OBJ); @@ -308,22 +339,32 @@ class Int64RleDecoder : public Decoder { if (RET_FAIL(buffer.read_buf((uint8_t*)tmp_buf_, length_, ret_read_len))) { return ret; - } else if (length_ != ret_read_len) { - ret = common::E_PARTIAL_READ; } byte_cache_.wrap_from((char*)tmp_buf_, length_); } - is_length_and_bitwidth_readed_ = true; - uint8_t tmp_bit_width; - common::SerializationUtil::read_ui8(tmp_bit_width, byte_cache_); + uint8_t tmp_bit_width = 0; + if (RET_FAIL(common::SerializationUtil::read_ui8(tmp_bit_width, + byte_cache_))) { + return ret; + } + if (tmp_bit_width == 0 || tmp_bit_width > 64) { + return common::E_TSFILE_CORRUPTED; + } bit_width_ = tmp_bit_width; + if (packer_ != nullptr) { + delete packer_; + } init_packer(); + is_length_and_bitwidth_readed_ = true; return ret; } void init_packer() { packer_ = new Int64Packer(bit_width_); } void destroy() { + // Drop the borrowed pattern view before freeing tmp_buf_ so no stale + // pointer into caller memory (or into tmp_buf_) survives. + byte_cache_.reset(); if (packer_) { delete (packer_); packer_ = nullptr; @@ -346,6 +387,7 @@ class Int64RleDecoder : public Decoder { current_count_ = 0; is_rle_run_ = false; rle_value_ = 0; + byte_cache_.reset(); if (current_buffer_) { common::mem_free(current_buffer_); current_buffer_ = nullptr; diff --git a/cpp/src/encoding/int64_sprintz_decoder.h b/cpp/src/encoding/int64_sprintz_decoder.h index 245984670c..71363f7564 100644 --- a/cpp/src/encoding/int64_sprintz_decoder.h +++ b/cpp/src/encoding/int64_sprintz_decoder.h @@ -129,6 +129,11 @@ class Int64SprintzDecoder : public SprintzDecoder { } } } else { + if (bit_width_ > 64) { + // Int64Packer supports at most 64 bits per value; the header + // byte is not otherwise validated. + return common::E_DECODE_ERR; + } decode_size_ = block_size_ + 1; common::SerializationUtil::read_i64(pre_value_, input); diff --git a/cpp/test/encoding/float_rle_codec_test.cc b/cpp/test/encoding/float_rle_codec_test.cc new file mode 100644 index 0000000000..31f1877836 --- /dev/null +++ b/cpp/test/encoding/float_rle_codec_test.cc @@ -0,0 +1,218 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +#include + +#include +#include + +#include "common/allocator/byte_stream.h" +#include "encoding/float_rle_decoder.h" +#include "encoding/float_rle_encoder.h" + +namespace storage { + +namespace { + +template +std::vector encode_rle_stream(const std::vector& values) { + common::ByteStream stream(1024, common::MOD_ENCODER_OBJ); + EncoderT encoder; + for (T value : values) { + encoder.encode(value, stream); + } + encoder.flush(stream); + std::vector out; + if (stream.total_size() == 0) { + return out; + } + char* bytes = common::get_bytes_from_bytestream(stream); + out.assign(bytes, bytes + stream.total_size()); + common::mem_free(bytes); + return out; +} + +int read_one(FloatRLEDecoder& decoder, float& value, common::ByteStream& in) { + return decoder.read_float(value, in); +} + +int read_one(DoubleRLEDecoder& decoder, double& value, common::ByteStream& in) { + return decoder.read_double(value, in); +} + +int read_batch(FloatRLEDecoder& decoder, float* out, int capacity, int& actual, + common::ByteStream& in) { + return decoder.read_batch_float(out, capacity, actual, in); +} + +int read_batch(DoubleRLEDecoder& decoder, double* out, int capacity, + int& actual, common::ByteStream& in) { + return decoder.read_batch_double(out, capacity, actual, in); +} + +// Decodes with either the scalar read (capacity == 0) or the batch read. +template +std::vector decode_rle_stream(const std::vector& bytes, bool wrapped, + int capacity) { + std::vector out; + common::ByteStream owning(1024, common::MOD_DECODER_OBJ); + common::ByteStream wrapped_stream; + common::ByteStream* in = &owning; + if (wrapped) { + wrapped_stream.wrap_from(const_cast(bytes.data()), + static_cast(bytes.size())); + in = &wrapped_stream; + } else { + owning.write_buf(reinterpret_cast(bytes.data()), + static_cast(bytes.size())); + } + DecoderT decoder; + decoder.reset(); + if (capacity == 0) { + while (decoder.has_remaining(*in)) { + T value = T(); + const int ret = read_one(decoder, value, *in); + if (ret != common::E_OK) { + break; + } + out.push_back(value); + } + } else { + std::vector buf(static_cast(capacity)); + while (decoder.has_remaining(*in)) { + int actual = 0; + const int ret = + read_batch(decoder, buf.data(), capacity, actual, *in); + if (ret != common::E_OK) { + break; + } + if (actual == 0) { + break; + } + out.insert(out.end(), buf.begin(), buf.begin() + actual); + } + } + return out; +} + +std::vector float_samples() { + std::mt19937 gen(20261109); + std::vector values; + for (int i = 0; i < 1000; ++i) { + const int32_t cents = static_cast(gen() % 200000) - 100000; + values.push_back(static_cast(cents) / 100.0f); + } + values.insert(values.end(), 300, 0.0f); + values.insert(values.end(), 300, -1.25f); + return values; +} + +std::vector double_samples() { + std::mt19937_64 gen(20261110); + std::vector values; + for (int i = 0; i < 1000; ++i) { + const int64_t cents = static_cast(gen() % 2000000) - 1000000; + values.push_back(static_cast(cents) / 100.0); + } + values.insert(values.end(), 300, 0.0); + values.insert(values.end(), 300, -3.14); + return values; +} + +void check_float_case(const std::vector& input) { + const std::vector bytes = + encode_rle_stream(input); + for (bool wrapped : {false, true}) { + const std::vector scalar = + decode_rle_stream(bytes, wrapped, 0); + ASSERT_EQ(scalar, input) << "wrapped=" << wrapped; + for (int capacity : {1, 3, 7, 64, 129, 1024}) { + const std::vector batch = + decode_rle_stream(bytes, wrapped, + capacity); + ASSERT_EQ(batch, scalar) + << "wrapped=" << wrapped << " capacity=" << capacity; + } + } +} + +void check_double_case(const std::vector& input) { + const std::vector bytes = + encode_rle_stream(input); + for (bool wrapped : {false, true}) { + const std::vector scalar = + decode_rle_stream(bytes, wrapped, 0); + ASSERT_EQ(scalar, input) << "wrapped=" << wrapped; + for (int capacity : {1, 3, 7, 64, 129, 1024}) { + const std::vector batch = + decode_rle_stream(bytes, wrapped, + capacity); + ASSERT_EQ(batch, scalar) + << "wrapped=" << wrapped << " capacity=" << capacity; + } + } +} + +} // namespace + +TEST(FloatRleCodecTest, BatchDecodeMatchesScalar) { + check_float_case(float_samples()); +} + +TEST(DoubleRleCodecTest, BatchDecodeMatchesScalar) { + check_double_case(double_samples()); +} + +TEST(FloatRleCodecTest, BatchDecodeSmallPageResetBetweenPages) { + // Two independent pages decoded with the same decoder object. + FloatRLEDecoder decoder; + const std::vector first = float_samples(); + const std::vector first_bytes = + encode_rle_stream(first); + common::ByteStream first_in; + first_in.wrap_from(const_cast(first_bytes.data()), + static_cast(first_bytes.size())); + std::vector got(first.size()); + int actual = 0; + decoder.reset(); + ASSERT_EQ(decoder.read_batch_float(got.data(), static_cast(got.size()), + actual, first_in), + common::E_OK); + ASSERT_EQ(actual, static_cast(first.size())); + ASSERT_EQ(got, first); + + std::vector second; + for (int i = 0; i < 37; ++i) { + second.push_back(static_cast(i) / 4.0f); + } + const std::vector second_bytes = + encode_rle_stream(second); + common::ByteStream second_in; + second_in.wrap_from(const_cast(second_bytes.data()), + static_cast(second_bytes.size())); + got.assign(second.size(), 0.0f); + actual = 0; + decoder.reset(); + ASSERT_EQ(decoder.read_batch_float(got.data(), static_cast(got.size()), + actual, second_in), + common::E_OK); + ASSERT_EQ(actual, static_cast(second.size())); + ASSERT_EQ(got, second); +} + +} // namespace storage diff --git a/cpp/test/encoding/int32_packer_test.cc b/cpp/test/encoding/int32_packer_test.cc index 3f4e151a3d..38cec6773e 100644 --- a/cpp/test/encoding/int32_packer_test.cc +++ b/cpp/test/encoding/int32_packer_test.cc @@ -195,4 +195,37 @@ TEST(Int32PackerTest, UnpackAllValuesMultipleBlocks) { } } +// unpack_all_values must not require padding past the final group: it decodes +// the last group from an internal zero-padded copy. +TEST(Int32PackerTest, UnpackAllValuesWithoutTrailingPadding) { + const int width = 16; + const int blocks = 10; + Int32Packer packer(width); + std::vector orig(blocks * NUM_OF_INTS); + std::vector buf(blocks * width); // exact size, no +8 + for (int b = 0; b < blocks; ++b) { + for (int i = 0; i < NUM_OF_INTS; ++i) { + orig[b * NUM_OF_INTS + i] = (b << 4) | i; + } + packer.pack_8values(orig.data() + b * NUM_OF_INTS, 0, + buf.data() + b * width); + } + + std::vector res(blocks * NUM_OF_INTS, 0); + packer.unpack_all_values(buf.data(), blocks * width, res.data()); + EXPECT_EQ(res, orig); +} + +// A zero width is invalid; unpack_all_values must return instead of looping +// forever without advancing. +TEST(Int32PackerTest, ZeroWidthUnpackAllValuesDoesNotHang) { + Int32Packer packer(0); + unsigned char buf[8] = {0}; + int32_t out[8] = {7, 7, 7, 7, 7, 7, 7, 7}; + packer.unpack_all_values(buf, 8, out); + for (int i = 0; i < NUM_OF_INTS; ++i) { + EXPECT_EQ(out[i], 7); + } +} + } // namespace storage \ No newline at end of file diff --git a/cpp/test/encoding/int32_rle_codec_test.cc b/cpp/test/encoding/int32_rle_codec_test.cc index dfc737c8b2..2d352404e1 100644 --- a/cpp/test/encoding/int32_rle_codec_test.cc +++ b/cpp/test/encoding/int32_rle_codec_test.cc @@ -293,4 +293,296 @@ TEST_F(Int32RleEncoderTest, ResetAfterDecodeNoCrash) { ASSERT_EQ(decoded, input); } +namespace { + +void append_var_uint(std::vector& out, uint32_t value) { + while (value & 0xFFFFFF80u) { + out.push_back(static_cast((value & 0x7F) | 0x80)); + value >>= 7; + } + out.push_back(static_cast(value)); +} + +// Build one length-prefixed RLE pattern: [length][bit width][body...]. +std::vector make_rle_pattern(uint8_t bit_width, + const std::vector& body) { + std::vector pattern; + append_var_uint(pattern, static_cast(body.size() + 1)); + pattern.push_back(bit_width); + pattern.insert(pattern.end(), body.begin(), body.end()); + return std::vector(pattern.begin(), pattern.end()); +} + +std::vector encode_int32_stream(const std::vector& input, + int split_at = -1) { + common::ByteStream stream(1024, common::MOD_ENCODER_OBJ); + Int32RleEncoder encoder; + for (size_t i = 0; i < input.size(); ++i) { + encoder.encode(input[i], stream); + if (split_at >= 0 && static_cast(i) + 1 == split_at) { + encoder.flush(stream); + } + } + encoder.flush(stream); + std::vector out; + if (stream.total_size() == 0) { + return out; + } + char* bytes = common::get_bytes_from_bytestream(stream); + out.assign(bytes, bytes + stream.total_size()); + common::mem_free(bytes); + return out; +} + +enum DecodeMode { kScalar = 0, kBatch = 1, kMixed = 2 }; + +std::vector decode_int32(const std::vector& bytes, bool wrapped, + DecodeMode mode, int capacity) { + std::vector out; + common::ByteStream owning(1024, common::MOD_DECODER_OBJ); + common::ByteStream wrapped_stream; + common::ByteStream* in = &owning; + if (wrapped) { + wrapped_stream.wrap_from(const_cast(bytes.data()), + static_cast(bytes.size())); + in = &wrapped_stream; + } else { + owning.write_buf(reinterpret_cast(bytes.data()), + static_cast(bytes.size())); + } + Int32RleDecoder decoder; + decoder.reset(); + if (mode == kScalar) { + while (decoder.has_remaining(*in)) { + int32_t v = 0; + if (decoder.read_int32(v, *in) != common::E_OK) { + break; + } + out.push_back(v); + } + } else if (mode == kBatch) { + std::vector buf(static_cast(capacity)); + while (decoder.has_remaining(*in)) { + int actual = 0; + if (decoder.read_batch_int32(buf.data(), capacity, actual, *in) != + common::E_OK) { + break; + } + if (actual == 0) { + break; + } + out.insert(out.end(), buf.begin(), buf.begin() + actual); + } + } else { + size_t turn = 0; + while (decoder.has_remaining(*in)) { + if (turn++ % 2 == 0) { + int32_t v = 0; + if (decoder.read_int32(v, *in) != common::E_OK) { + break; + } + out.push_back(v); + } else { + int32_t buf[5]; + int actual = 0; + if (decoder.read_batch_int32(buf, 5, actual, *in) != + common::E_OK) { + break; + } + if (actual == 0) { + break; + } + out.insert(out.end(), buf, buf + actual); + } + } + } + return out; +} + +int decode_int32_expect_error(const std::vector& pattern) { + common::ByteStream in; + in.wrap_from(const_cast(pattern.data()), + static_cast(pattern.size())); + Int32RleDecoder decoder; + decoder.reset(); + int32_t value = 0; + return decoder.read_int32(value, in); +} + +int decode_int32_batch_expect_error(const std::vector& pattern) { + common::ByteStream in; + in.wrap_from(const_cast(pattern.data()), + static_cast(pattern.size())); + Int32RleDecoder decoder; + decoder.reset(); + int32_t values[8]; + int actual = 0; + return decoder.read_batch_int32(values, 8, actual, in); +} + +std::vector> malformed_patterns() { + std::vector> patterns; + // last_bit_packed_num = 0 (encoders write 1..8). + { + std::vector body; + append_var_uint(body, (1u << 1) | 1u); + body.push_back(0); + body.insert(body.end(), 8, 0); + patterns.push_back(make_rle_pattern(8, body)); + } + // last_bit_packed_num > 8. + { + std::vector body; + append_var_uint(body, (1u << 1) | 1u); + body.push_back(9); + body.insert(body.end(), 8, 0); + patterns.push_back(make_rle_pattern(8, body)); + } + // Group count claims more bytes than the pattern holds. + { + std::vector body; + append_var_uint(body, (100u << 1) | 1u); + body.push_back(8); + body.insert(body.end(), 8, 0); + patterns.push_back(make_rle_pattern(8, body)); + } + // The pre-fix overflow: width 255 * group count 0x01010101 wraps uint32. + { + std::vector body; + append_var_uint(body, (0x01010101u << 1) | 1u); + body.push_back(8); + body.insert(body.end(), 16, 0); + patterns.push_back(make_rle_pattern(255, body)); + } + // Invalid bit width 0. + { + std::vector body; + append_var_uint(body, (1u << 1) | 1u); + body.push_back(8); + body.insert(body.end(), 8, 0); + patterns.push_back(make_rle_pattern(0, body)); + } + // Length prefix larger than the available bytes. + { + std::vector truncated; + truncated.push_back(32); // claims 32 bytes + truncated.push_back(8); + truncated.push_back(3); + truncated.push_back(8); + truncated.push_back(0); + patterns.push_back(truncated); + } + return patterns; +} + +} // namespace + +TEST_F(Int32RleEncoderTest, BatchDecodeMatchesScalar) { + std::vector> cases; + cases.push_back(std::vector(64, 0)); + cases.push_back(std::vector(300, -7)); + { + std::vector ascending; + for (int i = 0; i < 257; ++i) { + ascending.push_back(i); + } + cases.push_back(ascending); + } + { + std::vector random_values; + for (int i = 0; i < 513; ++i) { + random_values.push_back(static_cast(std::rand())); + } + cases.push_back(random_values); + } + { + std::vector negative_values; + for (int i = 0; i < 257; ++i) { + negative_values.push_back( + -1 - static_cast(std::rand() & 0x7FFFFFFF)); + } + cases.push_back(negative_values); + } + cases.push_back( + std::vector({std::numeric_limits::min(), + std::numeric_limits::max(), 0, -1, 1})); + + for (size_t c = 0; c < cases.size(); ++c) { + const std::vector& input = cases[c]; + const std::vector bytes = encode_int32_stream(input); + ASSERT_EQ(decode_int32(bytes, false, kScalar, 0), input) + << "case=" << c; + ASSERT_EQ(decode_int32(bytes, true, kScalar, 0), input) << "case=" << c; + for (int capacity : {1, 3, 7, 64, 129, 1024}) { + ASSERT_EQ(decode_int32(bytes, false, kBatch, capacity), input) + << "case=" << c << " capacity=" << capacity; + ASSERT_EQ(decode_int32(bytes, true, kBatch, capacity), input) + << "case=" << c << " capacity=" << capacity; + } + ASSERT_EQ(decode_int32(bytes, true, kMixed, 0), input) << "case=" << c; + } +} + +TEST_F(Int32RleEncoderTest, BatchDecodeSpansMultiplePatterns) { + std::vector input; + for (int i = 0; i < 200; ++i) { + input.push_back(i % 17); + } + // Two length-prefixed patterns in one stream; a single large batch call + // has to cross the pattern boundary. + const std::vector bytes = encode_int32_stream(input, 37); + for (int capacity : {1, 37, 129, 1024}) { + ASSERT_EQ(decode_int32(bytes, true, kBatch, capacity), input) + << "capacity=" << capacity; + } +} + +TEST_F(Int32RleEncoderTest, MalformedBitPackedInputIsRejected) { + const std::vector> patterns = malformed_patterns(); + for (size_t i = 0; i < patterns.size(); ++i) { + EXPECT_NE(decode_int32_expect_error(patterns[i]), common::E_OK) + << "scalar malformed case " << i; + EXPECT_NE(decode_int32_batch_expect_error(patterns[i]), common::E_OK) + << "batch malformed case " << i; + } +} + +TEST_F(Int32RleEncoderTest, ResetDropsBorrowedPatternView) { + std::vector first; + for (int i = 0; i < 100; ++i) { + first.push_back(i); + } + std::vector second; + for (int i = 0; i < 50; ++i) { + second.push_back(1000 + i); + } + std::vector first_bytes = encode_int32_stream(first); + std::vector second_bytes = encode_int32_stream(second); + + common::ByteStream in; + in.wrap_from(first_bytes.data(), static_cast(first_bytes.size())); + Int32RleDecoder decoder; + decoder.reset(); + int32_t value = 0; + ASSERT_EQ(decoder.read_int32(value, in), common::E_OK); + EXPECT_EQ(value, first[0]); + + // Abandon the first pattern and switch to a second wrapped buffer. + decoder.reset(); + // reset() must drop the borrowed view: an empty stream must not look + // like a partially consumed pattern. + common::ByteStream empty; + EXPECT_FALSE(decoder.has_remaining(empty)); + in.wrap_from(second_bytes.data(), + static_cast(second_bytes.size())); + std::vector decoded; + while (decoder.has_remaining(in)) { + if (decoder.read_int32(value, in) != common::E_OK) { + break; + } + decoded.push_back(value); + } + EXPECT_EQ(decoded, second); +} + } // namespace storage diff --git a/cpp/test/encoding/int64_packer_test.cc b/cpp/test/encoding/int64_packer_test.cc index 2e463f7f35..1cbd1dde30 100644 --- a/cpp/test/encoding/int64_packer_test.cc +++ b/cpp/test/encoding/int64_packer_test.cc @@ -252,4 +252,46 @@ TEST(Int64PackerTest, UnpackAllValuesMultipleBlocks) { } } +// unpack_all_values must not require padding past the final group: it decodes +// the last group from an internal zero-padded copy. 59/61/62/63 take the +// scalar fallback, 64 the SIMD path. +TEST(Int64PackerTest, UnpackAllValuesWithoutTrailingPadding) { + for (int width : {16, 59, 63, 64}) { + const int blocks = 3; + Int64Packer packer(width); + std::vector orig(blocks * NUM_OF_INTS); + std::vector buf(blocks * width); // exact, no +8 + const uint64_t mask = width == 64 + ? ~static_cast(0) + : ((static_cast(1) << width) - 1); + for (int b = 0; b < blocks; ++b) { + for (int i = 0; i < NUM_OF_INTS; ++i) { + const uint64_t raw = + (static_cast(b * NUM_OF_INTS + i) * + 0x9E3779B97F4A7C15ULL) & + mask; + orig[b * NUM_OF_INTS + i] = static_cast(raw); + } + packer.pack_8values(orig.data() + b * NUM_OF_INTS, 0, + buf.data() + b * width); + } + + std::vector res(blocks * NUM_OF_INTS, 0); + packer.unpack_all_values(buf.data(), blocks * width, res.data()); + EXPECT_EQ(res, orig) << "width=" << width; + } +} + +// A zero width is invalid; unpack_all_values must return instead of looping +// forever without advancing. +TEST(Int64PackerTest, ZeroWidthUnpackAllValuesDoesNotHang) { + Int64Packer packer(0); + unsigned char buf[8] = {0}; + int64_t out[8] = {7, 7, 7, 7, 7, 7, 7, 7}; + packer.unpack_all_values(buf, 8, out); + for (int i = 0; i < NUM_OF_INTS; ++i) { + EXPECT_EQ(out[i], 7); + } +} + } // namespace storage \ No newline at end of file diff --git a/cpp/test/encoding/int64_rle_codec_test.cc b/cpp/test/encoding/int64_rle_codec_test.cc index 7583faadd0..334183ee1b 100644 --- a/cpp/test/encoding/int64_rle_codec_test.cc +++ b/cpp/test/encoding/int64_rle_codec_test.cc @@ -130,4 +130,259 @@ TEST_F(Int64RleCodecTest, EncodeFlushWithoutData) { EXPECT_EQ(stream.total_size(), 0u); } +namespace { + +void append_var_uint(std::vector& out, uint32_t value) { + while (value & 0xFFFFFF80u) { + out.push_back(static_cast((value & 0x7F) | 0x80)); + value >>= 7; + } + out.push_back(static_cast(value)); +} + +// Build one length-prefixed RLE pattern: [length][bit width][body...]. +std::vector make_rle_pattern(uint8_t bit_width, + const std::vector& body) { + std::vector pattern; + append_var_uint(pattern, static_cast(body.size() + 1)); + pattern.push_back(bit_width); + pattern.insert(pattern.end(), body.begin(), body.end()); + return std::vector(pattern.begin(), pattern.end()); +} + +std::vector encode_int64_stream(const std::vector& input, + int split_at = -1) { + common::ByteStream stream(1024, common::MOD_ENCODER_OBJ); + Int64RleEncoder encoder; + for (size_t i = 0; i < input.size(); ++i) { + encoder.encode(input[i], stream); + if (split_at >= 0 && static_cast(i) + 1 == split_at) { + encoder.flush(stream); + } + } + encoder.flush(stream); + std::vector out; + if (stream.total_size() == 0) { + return out; + } + char* bytes = common::get_bytes_from_bytestream(stream); + out.assign(bytes, bytes + stream.total_size()); + common::mem_free(bytes); + return out; +} + +enum DecodeMode { kScalar = 0, kBatch = 1, kMixed = 2 }; + +std::vector decode_int64(const std::vector& bytes, bool wrapped, + DecodeMode mode, int capacity) { + std::vector out; + common::ByteStream owning(1024, common::MOD_DECODER_OBJ); + common::ByteStream wrapped_stream; + common::ByteStream* in = &owning; + if (wrapped) { + wrapped_stream.wrap_from(const_cast(bytes.data()), + static_cast(bytes.size())); + in = &wrapped_stream; + } else { + owning.write_buf(reinterpret_cast(bytes.data()), + static_cast(bytes.size())); + } + Int64RleDecoder decoder; + decoder.reset(); + if (mode == kScalar) { + while (decoder.has_remaining(*in)) { + int64_t v = 0; + if (decoder.read_int64(v, *in) != common::E_OK) { + break; + } + out.push_back(v); + } + } else if (mode == kBatch) { + std::vector buf(static_cast(capacity)); + while (decoder.has_remaining(*in)) { + int actual = 0; + if (decoder.read_batch_int64(buf.data(), capacity, actual, *in) != + common::E_OK) { + break; + } + if (actual == 0) { + break; + } + out.insert(out.end(), buf.begin(), buf.begin() + actual); + } + } else { + size_t turn = 0; + while (decoder.has_remaining(*in)) { + if (turn++ % 2 == 0) { + int64_t v = 0; + if (decoder.read_int64(v, *in) != common::E_OK) { + break; + } + out.push_back(v); + } else { + int64_t buf[5]; + int actual = 0; + if (decoder.read_batch_int64(buf, 5, actual, *in) != + common::E_OK) { + break; + } + if (actual == 0) { + break; + } + out.insert(out.end(), buf, buf + actual); + } + } + } + return out; +} + +int decode_int64_expect_error(const std::vector& pattern) { + common::ByteStream in; + in.wrap_from(const_cast(pattern.data()), + static_cast(pattern.size())); + Int64RleDecoder decoder; + decoder.reset(); + int64_t value = 0; + return decoder.read_int64(value, in); +} + +int decode_int64_batch_expect_error(const std::vector& pattern) { + common::ByteStream in; + in.wrap_from(const_cast(pattern.data()), + static_cast(pattern.size())); + Int64RleDecoder decoder; + decoder.reset(); + int64_t values[8]; + int actual = 0; + return decoder.read_batch_int64(values, 8, actual, in); +} + +std::vector> malformed_patterns() { + std::vector> patterns; + // last_bit_packed_num = 0 (encoders write 1..8). + { + std::vector body; + append_var_uint(body, (1u << 1) | 1u); + body.push_back(0); + body.insert(body.end(), 8, 0); + patterns.push_back(make_rle_pattern(8, body)); + } + // last_bit_packed_num > 8. + { + std::vector body; + append_var_uint(body, (1u << 1) | 1u); + body.push_back(9); + body.insert(body.end(), 8, 0); + patterns.push_back(make_rle_pattern(8, body)); + } + // Group count claims more bytes than the pattern holds. + { + std::vector body; + append_var_uint(body, (100u << 1) | 1u); + body.push_back(8); + body.insert(body.end(), 8, 0); + patterns.push_back(make_rle_pattern(8, body)); + } + // The pre-fix overflow: width 255 * group count 0x01010101 wraps uint32. + { + std::vector body; + append_var_uint(body, (0x01010101u << 1) | 1u); + body.push_back(8); + body.insert(body.end(), 16, 0); + patterns.push_back(make_rle_pattern(255, body)); + } + // Width above Int64Packer's 64-bit limit. + { + std::vector body; + append_var_uint(body, (1u << 1) | 1u); + body.push_back(8); + body.insert(body.end(), 65, 0); + patterns.push_back(make_rle_pattern(65, body)); + } + // Length prefix larger than the available bytes. + { + std::vector truncated; + truncated.push_back(32); // claims 32 bytes + truncated.push_back(8); + truncated.push_back(3); + truncated.push_back(8); + truncated.push_back(0); + patterns.push_back(truncated); + } + return patterns; +} + +} // namespace + +TEST_F(Int64RleCodecTest, BatchDecodeMatchesScalar) { + std::vector> cases; + cases.push_back(std::vector(64, 0)); + cases.push_back(std::vector(300, -7)); + { + std::vector ascending; + for (int64_t i = 0; i < 257; ++i) { + ascending.push_back(i); + } + cases.push_back(ascending); + } + { + std::vector random_values; + for (int i = 0; i < 513; ++i) { + random_values.push_back((static_cast(std::rand()) << 32) | + std::rand()); + } + cases.push_back(random_values); + } + { + // Exercise the 59/61/62/63-bit scalar fallback and the 64-bit SIMD + // path. + std::vector wide; + for (int shift : {0, 6, 7, 31, 58, 60, 62, 63}) { + wide.push_back((static_cast(1) << shift) | 1); + } + cases.push_back(wide); + } + cases.push_back( + std::vector({std::numeric_limits::min(), + std::numeric_limits::max(), 0, -1, 1})); + + for (size_t c = 0; c < cases.size(); ++c) { + const std::vector& input = cases[c]; + const std::vector bytes = encode_int64_stream(input); + ASSERT_EQ(decode_int64(bytes, false, kScalar, 0), input) + << "case=" << c; + ASSERT_EQ(decode_int64(bytes, true, kScalar, 0), input) << "case=" << c; + for (int capacity : {1, 3, 7, 64, 129, 1024}) { + ASSERT_EQ(decode_int64(bytes, false, kBatch, capacity), input) + << "case=" << c << " capacity=" << capacity; + ASSERT_EQ(decode_int64(bytes, true, kBatch, capacity), input) + << "case=" << c << " capacity=" << capacity; + } + ASSERT_EQ(decode_int64(bytes, true, kMixed, 0), input) << "case=" << c; + } +} + +TEST_F(Int64RleCodecTest, BatchDecodeSpansMultiplePatterns) { + std::vector input; + for (int i = 0; i < 200; ++i) { + input.push_back((static_cast(i) << 40) | + static_cast(i % 17)); + } + const std::vector bytes = encode_int64_stream(input, 37); + for (int capacity : {1, 37, 129, 1024}) { + ASSERT_EQ(decode_int64(bytes, true, kBatch, capacity), input) + << "capacity=" << capacity; + } +} + +TEST_F(Int64RleCodecTest, MalformedBitPackedInputIsRejected) { + const std::vector> patterns = malformed_patterns(); + for (size_t i = 0; i < patterns.size(); ++i) { + EXPECT_NE(decode_int64_expect_error(patterns[i]), common::E_OK) + << "scalar malformed case " << i; + EXPECT_NE(decode_int64_batch_expect_error(patterns[i]), common::E_OK) + << "batch malformed case " << i; + } +} + } // namespace storage