Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
159 changes: 102 additions & 57 deletions cpp/src/encoding/zigzag_decoder.h
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,10 @@ namespace storage {
template <typename T>
class ZigzagDecoder : public Decoder {
public:
// A zigzag-encoded int32 value is at most 5 groups of 7 bits, and an
// int64 value at most 10.
static const int DECODE_ARR_CAPACITY = sizeof(T) == 8 ? 10 : 5;

ZigzagDecoder() { init(); }
~ZigzagDecoder() override { destroy(); }

Expand All @@ -53,11 +57,21 @@ class ZigzagDecoder : public Decoder {
return common::E_TYPE_NOT_MATCH;
}
inline int read_int32(int32_t& ret_value, common::ByteStream& in) override {
ret_value = decode(in);
uint64_t stored_value = 0;
int ret = decode(in, stored_value);
if (ret != common::E_OK) {
return ret;
}
ret_value = static_cast<int32_t>(zigzag_decoder(stored_value));
return common::E_OK;
}
inline int read_int64(int64_t& ret_value, common::ByteStream& in) override {
ret_value = decode(in);
uint64_t stored_value = 0;
int ret = decode(in, stored_value);
if (ret != common::E_OK) {
return ret;
}
ret_value = static_cast<int64_t>(zigzag_decoder(stored_value));
return common::E_OK;
}
int read_float(float& ret_value, common::ByteStream& in) override {
Expand Down Expand Up @@ -97,31 +111,55 @@ class ZigzagDecoder : public Decoder {
}
}

void read_header(common::ByteStream& in) {
common::SerializationUtil::read_var_uint(zigzag_length_, in);
common::SerializationUtil::read_var_uint(int_length_, in);
// Reads an unsigned 32-bit var-int with at most five bytes and rejects a
// continuation past the fifth byte and payload bits above bit 31.
int read_bounded_var_uint(common::ByteStream& in, uint32_t& value) {
value = 0;
for (int i = 0; i < 5; ++i) {
uint8_t byte = 0;
uint32_t read_len = 0;
int ret = in.read_buf(&byte, 1, read_len);
if (UNLIKELY(ret != common::E_OK || read_len != 1)) {
return common::E_PARTIAL_READ;
}
// The fifth byte may carry only the four bits that still fit.
if (UNLIKELY(i == 4 && (byte & 0xF0) != 0)) {
return common::E_TSFILE_CORRUPTED;
}
value |= static_cast<uint32_t>(byte & 0x7F) << (7 * i);
if ((byte & 0x80) == 0) {
return common::E_OK;
}
}
return common::E_TSFILE_CORRUPTED;
}

void flush_byte_if_empty(common::ByteStream& in) {
if (bits_left_ == 0) {
uint32_t read_len = 0;
in.read_buf(&buffer_, 1, read_len);
bits_left_ = 8;
int read_header(common::ByteStream& in) {
int ret = read_bounded_var_uint(in, zigzag_length_);
if (ret != common::E_OK) {
return ret;
}
return read_bounded_var_uint(in, int_length_);
}

void read_byte_from_list() {
buffer_ = (uint8_t)(list_transit_in_zd_[0]);
list_transit_in_zd_.erase(list_transit_in_zd_.begin());
}

void fill_in_arr() {
bool fill_in_arr() {
if (UNLIKELY(num_of_sorts_of_zigzag_ >= DECODE_ARR_CAPACITY)) {
// More continuation bytes than any valid value can carry; writing
// one more group would run past zigzag_decode_arr_.
return false;
}
buffer_ &= ~(1 << 7);
zigzag_decode_arr_[num_of_sorts_of_zigzag_] = buffer_;
num_of_sorts_of_zigzag_ += 1;

buffer_ = 0;
bits_left_ = 0;
return true;
}

void read_the_first_bit_of_byte_() {
Expand All @@ -142,12 +180,11 @@ class ZigzagDecoder : public Decoder {
}
}

int64_t zigzag_decoder(int64_t stored_value_) {
stored_value_ = ((uint64_t)stored_value_ >> 1) ^ -(stored_value_ & 1);
return stored_value_;
static uint64_t zigzag_decoder(uint64_t stored_value) {
return (stored_value >> 1) ^ (0 - (stored_value & 1));
}

inline T decode(common::ByteStream& in);
int decode(common::ByteStream& in, uint64_t& ret_stored_value);

public:
common::TSEncoding type_;
Expand All @@ -163,70 +200,78 @@ class ZigzagDecoder : public Decoder {
std::vector<uint8_t> list_transit_in_zd_;
};

template <>
inline int32_t ZigzagDecoder<int32_t>::decode(common::ByteStream& in) {
template <typename T>
inline int ZigzagDecoder<T>::decode(common::ByteStream& in,
uint64_t& ret_stored_value) {
int ret = common::E_OK;
if (UNLIKELY(first_read_ == true)) {
read_header(in);
zigzag_decode_arr_ =
(uint8_t*)common::mem_alloc(10, common::MOD_ZIGZAG_OBJ);
// zigzag_length_ is the byte count of every encoded value in the page,
// so it cannot exceed the remaining input. Bounding it here also bounds
// list_transit_in_zd_: a crafted var-int used to make the decoder push
// that many bytes one at a time into memory.
if (RET_FAIL(read_header(in))) {
return common::E_TSFILE_CORRUPTED;
}
if (static_cast<uint64_t>(zigzag_length_) > in.remaining_size()) {
return common::E_TSFILE_CORRUPTED;
}
if (zigzag_decode_arr_ == nullptr) {
zigzag_decode_arr_ = (uint8_t*)common::mem_alloc(
DECODE_ARR_CAPACITY, common::MOD_ZIGZAG_OBJ);
if (UNLIKELY(zigzag_decode_arr_ == nullptr)) {
return common::E_OOM;
}
}
buffer_ = 0;
first_read_ = false;
list_transit_in_zd_.clear();
for (uint32_t i = 0; i < zigzag_length_; i++) {
flush_byte_if_empty(in);
uint32_t read_len = 0;
ret = in.read_buf(&buffer_, 1, read_len);
if (UNLIKELY(ret != common::E_OK || read_len != 1)) {
return common::E_PARTIAL_READ;
}
list_transit_in_zd_.push_back(buffer_);
buffer_ = 0;
bits_left_ = 0;
}
}

if (UNLIKELY(list_transit_in_zd_.empty())) {
// A value whose continuation bit asks for another byte, after the
// page's declared bytes are exhausted.
return common::E_TSFILE_CORRUPTED;
}
read_byte_from_list();
read_the_first_bit_of_byte_();
while (first_bit_of_byte_ == 1) {
fill_in_arr();
if (UNLIKELY(!fill_in_arr())) {
return common::E_TSFILE_CORRUPTED;
}
if (UNLIKELY(list_transit_in_zd_.empty())) {
return common::E_TSFILE_CORRUPTED;
}
read_byte_from_list();
read_the_first_bit_of_byte_();
}
fill_in_arr();
splice_bytes_in_arr();

int32_t ret_value = (int32_t)(stored_value_);
ret_value = (int32_t)(zigzag_decoder(stored_value_));
each_encode_reset();
return ret_value;
}

template <>
inline int64_t ZigzagDecoder<int64_t>::decode(common::ByteStream& in) {
if (UNLIKELY(first_read_ == true)) {
read_header(in);
zigzag_decode_arr_ =
(uint8_t*)common::mem_alloc(10, common::MOD_ZIGZAG_OBJ);
buffer_ = 0;
first_read_ = false;
list_transit_in_zd_.clear();
for (uint32_t i = 0; i < zigzag_length_; i++) {
flush_byte_if_empty(in);
list_transit_in_zd_.push_back(buffer_);
buffer_ = 0;
bits_left_ = 0;
}
if (UNLIKELY(!fill_in_arr())) {
return common::E_TSFILE_CORRUPTED;
}

read_byte_from_list();
read_the_first_bit_of_byte_();
while (first_bit_of_byte_ == 1) {
fill_in_arr();
read_byte_from_list();
read_the_first_bit_of_byte_();
if (UNLIKELY(num_of_sorts_of_zigzag_ == DECODE_ARR_CAPACITY)) {
// The last group may carry only the bits that still fit into T: four
// bits for the fifth int32 group, one bit for the tenth int64 group.
// Anything above would be silently truncated by the final cast.
const uint8_t top = zigzag_decode_arr_[DECODE_ARR_CAPACITY - 1];
const uint8_t excess_mask = sizeof(T) == 4 ? 0x70 : 0x7E;
if (UNLIKELY((top & excess_mask) != 0)) {
return common::E_TSFILE_CORRUPTED;
}
}
fill_in_arr();
splice_bytes_in_arr();

int64_t ret_value = (int64_t)(stored_value_);
ret_value = (int64_t)(zigzag_decoder(stored_value_));
ret_stored_value = stored_value_;
each_encode_reset();
return ret_value;
return common::E_OK;
}

typedef ZigzagDecoder<int32_t> IntZigzagDecoder;
Expand Down
7 changes: 5 additions & 2 deletions cpp/test/encoding/encoding_coverage_test.cc
Original file line number Diff line number Diff line change
Expand Up @@ -113,12 +113,15 @@ TEST(EncodingCoverage, ResetClearsState) {
common::ByteStream s(64, common::MOD_DEFAULT);
EXPECT_EQ(enc.encode(123, s), common::E_OK);
enc.flush(s);
EXPECT_EQ(dec.decode(s), 123);
int32_t decoded = 0;
EXPECT_EQ(dec.read_int32(decoded, s), common::E_OK);
EXPECT_EQ(decoded, 123);
dec.reset();
common::ByteStream s2(64, common::MOD_DEFAULT);
EXPECT_EQ(enc.encode(-456, s2), common::E_OK);
enc.flush(s2);
EXPECT_EQ(dec.decode(s2), -456);
EXPECT_EQ(dec.read_int32(decoded, s2), common::E_OK);
EXPECT_EQ(decoded, -456);
}
{
IntGorillaEncoder enc;
Expand Down
Loading
Loading