From 91d72833701c211bad2c4342bb139c6e909040b9 Mon Sep 17 00:00:00 2001 From: ColinLee Date: Mon, 16 Nov 2026 11:00:15 +0800 Subject: [PATCH] fix(cpp): bound ZigZag page length and continuation bytes The ZigZag decoder trusted the page header's var-int byte count: it read that many bytes into a list one at a time, wrote one 7-bit group per continuation byte into a fixed-size buffer without a bound, and indexed the byte list after it ran out. Header var-ints were also parsed with an unbounded loop, and values wider than the target integer were silently truncated. Parse both header var-ints with a five-byte bound and at most four payload bits in the fifth byte; reject a declared length larger than the remaining input; cap groups at five for int32 and ten for int64; reject payload bits that do not fit into the target type; and treat an exhausted byte list as corruption. decode() now reports errors through read_int32/read_int64. Valid pages decode identically. --- cpp/src/encoding/zigzag_decoder.h | 159 +++++++++++------- cpp/test/encoding/encoding_coverage_test.cc | 7 +- cpp/test/encoding/zigzag_codec_test.cc | 171 +++++++++++++++++++- 3 files changed, 274 insertions(+), 63 deletions(-) diff --git a/cpp/src/encoding/zigzag_decoder.h b/cpp/src/encoding/zigzag_decoder.h index 69ef3b209b..5c6f6f506d 100644 --- a/cpp/src/encoding/zigzag_decoder.h +++ b/cpp/src/encoding/zigzag_decoder.h @@ -32,6 +32,10 @@ namespace storage { template class ZigzagDecoder : public Decoder { public: + // A zigzag-encoded int32 value is at most 5 groups of 7 bits, and an + // int64 value at most 10. + static const int DECODE_ARR_CAPACITY = sizeof(T) == 8 ? 10 : 5; + ZigzagDecoder() { init(); } ~ZigzagDecoder() override { destroy(); } @@ -53,11 +57,21 @@ class ZigzagDecoder : public Decoder { return common::E_TYPE_NOT_MATCH; } inline int read_int32(int32_t& ret_value, common::ByteStream& in) override { - ret_value = decode(in); + uint64_t stored_value = 0; + int ret = decode(in, stored_value); + if (ret != common::E_OK) { + return ret; + } + ret_value = static_cast(zigzag_decoder(stored_value)); return common::E_OK; } inline int read_int64(int64_t& ret_value, common::ByteStream& in) override { - ret_value = decode(in); + uint64_t stored_value = 0; + int ret = decode(in, stored_value); + if (ret != common::E_OK) { + return ret; + } + ret_value = static_cast(zigzag_decoder(stored_value)); return common::E_OK; } int read_float(float& ret_value, common::ByteStream& in) override { @@ -97,17 +111,35 @@ class ZigzagDecoder : public Decoder { } } - void read_header(common::ByteStream& in) { - common::SerializationUtil::read_var_uint(zigzag_length_, in); - common::SerializationUtil::read_var_uint(int_length_, in); + // Reads an unsigned 32-bit var-int with at most five bytes and rejects a + // continuation past the fifth byte and payload bits above bit 31. + int read_bounded_var_uint(common::ByteStream& in, uint32_t& value) { + value = 0; + for (int i = 0; i < 5; ++i) { + uint8_t byte = 0; + uint32_t read_len = 0; + int ret = in.read_buf(&byte, 1, read_len); + if (UNLIKELY(ret != common::E_OK || read_len != 1)) { + return common::E_PARTIAL_READ; + } + // The fifth byte may carry only the four bits that still fit. + if (UNLIKELY(i == 4 && (byte & 0xF0) != 0)) { + return common::E_TSFILE_CORRUPTED; + } + value |= static_cast(byte & 0x7F) << (7 * i); + if ((byte & 0x80) == 0) { + return common::E_OK; + } + } + return common::E_TSFILE_CORRUPTED; } - void flush_byte_if_empty(common::ByteStream& in) { - if (bits_left_ == 0) { - uint32_t read_len = 0; - in.read_buf(&buffer_, 1, read_len); - bits_left_ = 8; + int read_header(common::ByteStream& in) { + int ret = read_bounded_var_uint(in, zigzag_length_); + if (ret != common::E_OK) { + return ret; } + return read_bounded_var_uint(in, int_length_); } void read_byte_from_list() { @@ -115,13 +147,19 @@ class ZigzagDecoder : public Decoder { list_transit_in_zd_.erase(list_transit_in_zd_.begin()); } - void fill_in_arr() { + bool fill_in_arr() { + if (UNLIKELY(num_of_sorts_of_zigzag_ >= DECODE_ARR_CAPACITY)) { + // More continuation bytes than any valid value can carry; writing + // one more group would run past zigzag_decode_arr_. + return false; + } buffer_ &= ~(1 << 7); zigzag_decode_arr_[num_of_sorts_of_zigzag_] = buffer_; num_of_sorts_of_zigzag_ += 1; buffer_ = 0; bits_left_ = 0; + return true; } void read_the_first_bit_of_byte_() { @@ -142,12 +180,11 @@ class ZigzagDecoder : public Decoder { } } - int64_t zigzag_decoder(int64_t stored_value_) { - stored_value_ = ((uint64_t)stored_value_ >> 1) ^ -(stored_value_ & 1); - return stored_value_; + static uint64_t zigzag_decoder(uint64_t stored_value) { + return (stored_value >> 1) ^ (0 - (stored_value & 1)); } - inline T decode(common::ByteStream& in); + int decode(common::ByteStream& in, uint64_t& ret_stored_value); public: common::TSEncoding type_; @@ -163,70 +200,78 @@ class ZigzagDecoder : public Decoder { std::vector list_transit_in_zd_; }; -template <> -inline int32_t ZigzagDecoder::decode(common::ByteStream& in) { +template +inline int ZigzagDecoder::decode(common::ByteStream& in, + uint64_t& ret_stored_value) { + int ret = common::E_OK; if (UNLIKELY(first_read_ == true)) { - read_header(in); - zigzag_decode_arr_ = - (uint8_t*)common::mem_alloc(10, common::MOD_ZIGZAG_OBJ); + // zigzag_length_ is the byte count of every encoded value in the page, + // so it cannot exceed the remaining input. Bounding it here also bounds + // list_transit_in_zd_: a crafted var-int used to make the decoder push + // that many bytes one at a time into memory. + if (RET_FAIL(read_header(in))) { + return common::E_TSFILE_CORRUPTED; + } + if (static_cast(zigzag_length_) > in.remaining_size()) { + return common::E_TSFILE_CORRUPTED; + } + if (zigzag_decode_arr_ == nullptr) { + zigzag_decode_arr_ = (uint8_t*)common::mem_alloc( + DECODE_ARR_CAPACITY, common::MOD_ZIGZAG_OBJ); + if (UNLIKELY(zigzag_decode_arr_ == nullptr)) { + return common::E_OOM; + } + } buffer_ = 0; first_read_ = false; list_transit_in_zd_.clear(); for (uint32_t i = 0; i < zigzag_length_; i++) { - flush_byte_if_empty(in); + uint32_t read_len = 0; + ret = in.read_buf(&buffer_, 1, read_len); + if (UNLIKELY(ret != common::E_OK || read_len != 1)) { + return common::E_PARTIAL_READ; + } list_transit_in_zd_.push_back(buffer_); buffer_ = 0; bits_left_ = 0; } } + if (UNLIKELY(list_transit_in_zd_.empty())) { + // A value whose continuation bit asks for another byte, after the + // page's declared bytes are exhausted. + return common::E_TSFILE_CORRUPTED; + } read_byte_from_list(); read_the_first_bit_of_byte_(); while (first_bit_of_byte_ == 1) { - fill_in_arr(); + if (UNLIKELY(!fill_in_arr())) { + return common::E_TSFILE_CORRUPTED; + } + if (UNLIKELY(list_transit_in_zd_.empty())) { + return common::E_TSFILE_CORRUPTED; + } read_byte_from_list(); read_the_first_bit_of_byte_(); } - fill_in_arr(); - splice_bytes_in_arr(); - - int32_t ret_value = (int32_t)(stored_value_); - ret_value = (int32_t)(zigzag_decoder(stored_value_)); - each_encode_reset(); - return ret_value; -} - -template <> -inline int64_t ZigzagDecoder::decode(common::ByteStream& in) { - if (UNLIKELY(first_read_ == true)) { - read_header(in); - zigzag_decode_arr_ = - (uint8_t*)common::mem_alloc(10, common::MOD_ZIGZAG_OBJ); - buffer_ = 0; - first_read_ = false; - list_transit_in_zd_.clear(); - for (uint32_t i = 0; i < zigzag_length_; i++) { - flush_byte_if_empty(in); - list_transit_in_zd_.push_back(buffer_); - buffer_ = 0; - bits_left_ = 0; - } + if (UNLIKELY(!fill_in_arr())) { + return common::E_TSFILE_CORRUPTED; } - - read_byte_from_list(); - read_the_first_bit_of_byte_(); - while (first_bit_of_byte_ == 1) { - fill_in_arr(); - read_byte_from_list(); - read_the_first_bit_of_byte_(); + if (UNLIKELY(num_of_sorts_of_zigzag_ == DECODE_ARR_CAPACITY)) { + // The last group may carry only the bits that still fit into T: four + // bits for the fifth int32 group, one bit for the tenth int64 group. + // Anything above would be silently truncated by the final cast. + const uint8_t top = zigzag_decode_arr_[DECODE_ARR_CAPACITY - 1]; + const uint8_t excess_mask = sizeof(T) == 4 ? 0x70 : 0x7E; + if (UNLIKELY((top & excess_mask) != 0)) { + return common::E_TSFILE_CORRUPTED; + } } - fill_in_arr(); splice_bytes_in_arr(); - int64_t ret_value = (int64_t)(stored_value_); - ret_value = (int64_t)(zigzag_decoder(stored_value_)); + ret_stored_value = stored_value_; each_encode_reset(); - return ret_value; + return common::E_OK; } typedef ZigzagDecoder IntZigzagDecoder; diff --git a/cpp/test/encoding/encoding_coverage_test.cc b/cpp/test/encoding/encoding_coverage_test.cc index 4c715d8b85..9334868e51 100644 --- a/cpp/test/encoding/encoding_coverage_test.cc +++ b/cpp/test/encoding/encoding_coverage_test.cc @@ -113,12 +113,15 @@ TEST(EncodingCoverage, ResetClearsState) { common::ByteStream s(64, common::MOD_DEFAULT); EXPECT_EQ(enc.encode(123, s), common::E_OK); enc.flush(s); - EXPECT_EQ(dec.decode(s), 123); + int32_t decoded = 0; + EXPECT_EQ(dec.read_int32(decoded, s), common::E_OK); + EXPECT_EQ(decoded, 123); dec.reset(); common::ByteStream s2(64, common::MOD_DEFAULT); EXPECT_EQ(enc.encode(-456, s2), common::E_OK); enc.flush(s2); - EXPECT_EQ(dec.decode(s2), -456); + EXPECT_EQ(dec.read_int32(decoded, s2), common::E_OK); + EXPECT_EQ(decoded, -456); } { IntGorillaEncoder enc; diff --git a/cpp/test/encoding/zigzag_codec_test.cc b/cpp/test/encoding/zigzag_codec_test.cc index 8606e5160f..2b5287ac5e 100644 --- a/cpp/test/encoding/zigzag_codec_test.cc +++ b/cpp/test/encoding/zigzag_codec_test.cc @@ -86,7 +86,9 @@ TEST_F(ZigzagDecoderTest, DecodeInt32) { IntZigzagDecoder decoder; for (int i = 0; i < (int)(sizeof(data) / sizeof(int32_t)); i++) { - EXPECT_EQ(data[i], decoder.decode(stream)); + int32_t actual = 0; + EXPECT_EQ(decoder.read_int32(actual, stream), common::E_OK); + EXPECT_EQ(data[i], actual); } } @@ -101,7 +103,9 @@ TEST_F(ZigzagDecoderTest, DecodeInt64) { LongZigzagDecoder decoder; for (int i = 0; i < (int)(sizeof(data) / sizeof(int64_t)); i++) { - EXPECT_EQ(data[i], decoder.decode(stream)); + int64_t actual = 0; + EXPECT_EQ(decoder.read_int64(actual, stream), common::E_OK); + EXPECT_EQ(data[i], actual); } } @@ -115,7 +119,9 @@ TEST_F(ZigzagDecoderTest, DecodeInt32LargeQuantities) { IntZigzagDecoder decoder; for (int32_t value = 0; value < 20000; value++) { - EXPECT_EQ(value, decoder.decode(stream)); + int32_t actual = 0; + EXPECT_EQ(decoder.read_int32(actual, stream), common::E_OK); + EXPECT_EQ(value, actual); } } @@ -129,8 +135,165 @@ TEST_F(ZigzagDecoderTest, DecodeInt64LargeQuantities) { LongZigzagDecoder decoder; for (int64_t value = 0; value < 50000; value++) { - EXPECT_EQ(value, decoder.decode(stream)); + int64_t actual = 0; + EXPECT_EQ(decoder.read_int64(actual, stream), common::E_OK); + EXPECT_EQ(value, actual); } } +namespace { + +void WriteZigzagHeader(common::ByteStream& stream, uint32_t zigzag_length, + uint32_t int_length) { + common::SerializationUtil::write_var_uint(zigzag_length, stream); + common::SerializationUtil::write_var_uint(int_length, stream); +} + +} // namespace + +// The page header carries the byte count of every encoded +// value as a var-int. A crafted header used to make the decoder push that many +// bytes into memory one at a time, regardless of what the page holds. +TEST_F(ZigzagDecoderTest, RejectsDeclaredLengthBeyondInput) { + LongZigzagDecoder decoder; + common::ByteStream stream(64, common::MOD_ZIGZAG_OBJ); + WriteZigzagHeader(stream, 0xFFFFFFFFu, 1); + uint8_t zero = 0; + stream.write_buf(&zero, 1); + int64_t value = 0; + EXPECT_EQ(decoder.read_int64(value, stream), common::E_TSFILE_CORRUPTED); +} + +TEST_F(ZigzagDecoderTest, RejectsValueRunningPastDeclaredLength) { + // Both declared bytes set the continuation bit, so the value asks for a + // third byte that the page does not contain. + for (int wide : {0, 1}) { + common::ByteStream stream(64, common::MOD_ZIGZAG_OBJ); + WriteZigzagHeader(stream, 2, 1); + uint8_t continuation = 0x80; + stream.write_buf(&continuation, 1); + stream.write_buf(&continuation, 1); + if (wide) { + LongZigzagDecoder decoder; + int64_t value = 0; + EXPECT_EQ(decoder.read_int64(value, stream), + common::E_TSFILE_CORRUPTED); + } else { + IntZigzagDecoder decoder; + int32_t value = 0; + EXPECT_EQ(decoder.read_int32(value, stream), + common::E_TSFILE_CORRUPTED); + } + } +} + +TEST_F(ZigzagDecoderTest, RejectsValueWithTooManyContinuationBytes) { + // Ten 7-bit groups are the longest int64 value; a value with more groups + // used to be written past the ten-byte decode buffer. + for (int wide : {0, 1}) { + common::ByteStream stream(64, common::MOD_ZIGZAG_OBJ); + WriteZigzagHeader(stream, 65, 1); + for (int i = 0; i < 64; ++i) { + uint8_t continuation = 0xFF; + stream.write_buf(&continuation, 1); + } + uint8_t last = 0x01; + stream.write_buf(&last, 1); + if (wide) { + LongZigzagDecoder decoder; + int64_t value = 0; + EXPECT_EQ(decoder.read_int64(value, stream), + common::E_TSFILE_CORRUPTED); + } else { + IntZigzagDecoder decoder; + int32_t value = 0; + EXPECT_EQ(decoder.read_int32(value, stream), + common::E_TSFILE_CORRUPTED); + } + } +} + +TEST_F(ZigzagDecoderTest, RejectsEmptyAndHeaderlessStreams) { + { + IntZigzagDecoder decoder; + common::ByteStream empty(64, common::MOD_ZIGZAG_OBJ); + int32_t value = 0; + EXPECT_EQ(decoder.read_int32(value, empty), common::E_TSFILE_CORRUPTED); + } + { + // A header declaring no encoded bytes cannot describe a value. + LongZigzagDecoder decoder; + common::ByteStream stream(64, common::MOD_ZIGZAG_OBJ); + WriteZigzagHeader(stream, 0, 0); + int64_t value = 0; + EXPECT_EQ(decoder.read_int64(value, stream), + common::E_TSFILE_CORRUPTED); + } +} + +// The page header fields are unsigned 32-bit var-ints: at most five bytes, +// with four payload bits in the fifth and no continuation after it. +TEST_F(ZigzagDecoderTest, RejectsMalformedHeaderVarint) { + { + // A continuation in the fifth byte would need a sixth byte. + common::ByteStream stream(64, common::MOD_ZIGZAG_OBJ); + for (int i = 0; i < 5; ++i) { + uint8_t byte = 0x80; + stream.write_buf(&byte, 1); + } + uint8_t zero = 0; + stream.write_buf(&zero, 1); + LongZigzagDecoder decoder; + int64_t value = 0; + EXPECT_EQ(decoder.read_int64(value, stream), + common::E_TSFILE_CORRUPTED); + } + { + // A fifth byte carrying bits above bit 31 is out of range. + common::ByteStream stream(64, common::MOD_ZIGZAG_OBJ); + for (int i = 0; i < 4; ++i) { + uint8_t byte = 0x80; + stream.write_buf(&byte, 1); + } + uint8_t too_big = 0x10; + stream.write_buf(&too_big, 1); + LongZigzagDecoder decoder; + int64_t value = 0; + EXPECT_EQ(decoder.read_int64(value, stream), + common::E_TSFILE_CORRUPTED); + } +} + +TEST_F(ZigzagDecoderTest, RejectsValueWiderThanInt32) { + // Four continuation bytes and a fifth group of 0x20 put payload bits + // above bit 31; the value must be rejected, not truncated. + common::ByteStream stream(64, common::MOD_ZIGZAG_OBJ); + WriteZigzagHeader(stream, 5, 1); + for (int i = 0; i < 4; ++i) { + uint8_t byte = 0x80; + stream.write_buf(&byte, 1); + } + uint8_t tail = 0x20; + stream.write_buf(&tail, 1); + IntZigzagDecoder decoder; + int32_t value = 0; + EXPECT_EQ(decoder.read_int32(value, stream), common::E_TSFILE_CORRUPTED); +} + +TEST_F(ZigzagDecoderTest, RejectsValueWiderThanInt64) { + // Nine continuation bytes and a tenth group of 0x02 put payload bits + // above bit 63; the value must be rejected, not truncated. + common::ByteStream stream(64, common::MOD_ZIGZAG_OBJ); + WriteZigzagHeader(stream, 10, 1); + for (int i = 0; i < 9; ++i) { + uint8_t byte = 0x80; + stream.write_buf(&byte, 1); + } + uint8_t tail = 0x02; + stream.write_buf(&tail, 1); + LongZigzagDecoder decoder; + int64_t value = 0; + EXPECT_EQ(decoder.read_int64(value, stream), common::E_TSFILE_CORRUPTED); +} + } // namespace storage