Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
84 changes: 84 additions & 0 deletions cpp/src/common/container/byte_buffer.h
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,9 @@
#include <stdlib.h>
#include <string.h>

#include <algorithm>
#include <cstring>

#include "common/allocator/alloc_base.h"
Comment thread
ColinLeeo marked this conversation as resolved.
Resolved
#include "common/global.h"
#include "utils/util_define.h"
Expand Down Expand Up @@ -133,6 +136,87 @@ class ByteBuffer {

FORCE_INLINE uint32_t get_data_size() const { return real_data_size_; }

// Append @p count copies of one variable-length value. A block's TAG column
// repeats the same value on every row; writing the pattern once and then
// doubling the filled region beats @p count individual append calls.
FORCE_INLINE int fill_repeated_variable(const char* value, uint32_t len,
uint32_t count) {
if (count == 0) {
return E_OK;
}
// The stored length prefix is copied from a uint32_t, so the width
// must not exceed it (the constructor fixes it to sizeof(uint32_t)).
if (variable_type_len_ > sizeof(uint32_t)) {
return E_INVALID_ARG;
}
const uint64_t elem_size =
static_cast<uint64_t>(variable_type_len_) + len;
Comment thread
ColinLeeo marked this conversation as resolved.
Resolved
if (elem_size == 0) {
return E_INVALID_ARG;
}
// elem_size * count can overflow uint64_t (both factors are 32-bit),
// which would wrap `need` to a small value and bypass the capacity
// checks before a huge payload copy. Reject anything that cannot fit
// in the 32-bit buffer size first.
const uint64_t available =
static_cast<uint64_t>(0xFFFFFFFFULL) - real_data_size_;
if (count > available / elem_size) {
Comment thread
ColinLeeo marked this conversation as resolved.
Resolved
return E_OOM;
}
const uint64_t need =
static_cast<uint64_t>(real_data_size_) + elem_size * count;
if (need > reserved_size_) {
if (extend_memory(static_cast<uint32_t>(need)) != E_OK) {
return E_OOM;
}
}
char* dst = data_ + real_data_size_;
std::memcpy(dst, &len, variable_type_len_);
if (len > 0) {
std::memcpy(dst + variable_type_len_, value, len);
}
Comment thread
ColinLeeo marked this conversation as resolved.
Resolved
uint32_t filled = 1;
while (filled < count) {
const uint32_t copy = std::min(filled, count - filled);
std::memcpy(dst + static_cast<size_t>(filled) * elem_size, dst,
static_cast<size_t>(copy) * elem_size);
filled += copy;
}
real_data_size_ = static_cast<uint32_t>(need);
return E_OK;
}

// Fixed-width counterpart of fill_repeated_variable().
FORCE_INLINE int fill_repeated_fixed(const char* value, uint32_t elem_size,
uint32_t count) {
if (count == 0) {
return E_OK;
}
const uint64_t available =
static_cast<uint64_t>(0xFFFFFFFFULL) - real_data_size_;
if (elem_size != 0 && count > available / elem_size) {
return E_OOM;
}
const uint64_t need = static_cast<uint64_t>(real_data_size_) +
static_cast<uint64_t>(elem_size) * count;
if (need > reserved_size_) {
if (extend_memory(static_cast<uint32_t>(need)) != E_OK) {
return E_OOM;
}
}
char* dst = data_ + real_data_size_;
std::memcpy(dst, value, elem_size);
uint32_t filled = 1;
while (filled < count) {
const uint32_t copy = std::min(filled, count - filled);
std::memcpy(dst + static_cast<size_t>(filled) * elem_size, dst,
static_cast<size_t>(copy) * elem_size);
filled += copy;
}
real_data_size_ = static_cast<uint32_t>(need);
return E_OK;
}

// Declare bytes that were written directly through get_data() without going
// through append_*(). Used when a caller decodes straight into this buffer
// at a known offset.
Expand Down
17 changes: 12 additions & 5 deletions cpp/src/common/tsblock/tsblock.h
Original file line number Diff line number Diff line change
Expand Up @@ -244,12 +244,19 @@ class ColAppender {
return E_OK;
}
FORCE_INLINE int fill(const char* value, uint32_t len, uint32_t end_index) {
while (column_row_count_ < end_index) {
if (!add_row()) {
return E_INVALID_ARG;
}
append(value, len);
if (column_row_count_ >= end_index) {
return E_OK;
}
const uint32_t count = end_index - column_row_count_;
if (column_row_count_ + count > tsblock_->max_row_count_) {
return E_INVALID_ARG;
}
const int ret = vec_->fill_repeated(value, len, count);
if (ret != E_OK) {
// E_OOM from a failed growth is not an argument error.
return ret;
}
column_row_count_ += count;
return E_OK;
}
FORCE_INLINE void reset() { column_row_count_ = 0; }
Expand Down
19 changes: 19 additions & 0 deletions cpp/src/common/tsblock/vector/fixed_length_vector.h
Original file line number Diff line number Diff line change
Expand Up @@ -54,6 +54,25 @@ class FixedLengthVector : public Vector {
add_row_nums(count);
}

int fill_repeated(const char* value, uint32_t len,
uint32_t count) OVERRIDE {
if (count == 0) {
return common::E_OK;
}
// Readers always advance by type_len_, so the fill must use the same
// stride. A mismatch is a caller error and, in a release build, would
// read past a shorter value buffer.
if (len != type_len_) {
return common::E_INVALID_ARG;
}
if (values_.fill_repeated_fixed(value, type_len_, count) !=
common::E_OK) {
return common::E_OOM;
}
Comment thread
ColinLeeo marked this conversation as resolved.
Resolved
add_row_nums(count);
return common::E_OK;
}

FORCE_INLINE uint32_t get_type_len() const { return type_len_; }

// cppcheck-suppress missingOverride
Expand Down
13 changes: 13 additions & 0 deletions cpp/src/common/tsblock/vector/variable_length_vector.h
Original file line number Diff line number Diff line change
Expand Up @@ -61,6 +61,19 @@ class VariableLengthVector : public Vector {
values_.append_variable_value(value, len);
}

int fill_repeated(const char* value, uint32_t len,
uint32_t count) OVERRIDE {
if (count == 0) {
return common::E_OK;
}
const int ret = values_.fill_repeated_variable(value, len, count);
if (ret != common::E_OK) {
return ret;
}
add_row_nums(count);
return common::E_OK;
}
Comment thread
ColinLeeo marked this conversation as resolved.
Resolved

// cppcheck-suppress missingOverride
FORCE_INLINE char* read(uint32_t* __restrict len, bool* __restrict null,
uint32_t rowid) OVERRIDE {
Expand Down
11 changes: 11 additions & 0 deletions cpp/src/common/tsblock/vector/vector.h
Original file line number Diff line number Diff line change
Expand Up @@ -47,6 +47,17 @@ class Vector {
// len == 0 is '' or ""
virtual void append(const char* value, uint32_t len) = 0;

// Append @p count copies of the same value and bump the row count. The
// default is a plain loop; vectors whose layout allows it override this
// with a bulk fill (a block's TAG column repeats one value on every row).
virtual int fill_repeated(const char* value, uint32_t len, uint32_t count) {
for (uint32_t i = 0; i < count; i++) {
append(value, len);
}
add_row_nums(count);
return E_OK;
}

virtual char* read(uint32_t* __restrict len, bool* __restrict null,
uint32_t rowid) = 0;

Expand Down
142 changes: 142 additions & 0 deletions cpp/test/common/tsblock/tslock_test.cc
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,9 @@
*/
#include <gtest/gtest.h>

#include <cstring>

Comment thread
ColinLeeo marked this conversation as resolved.
Resolved
#include "common/container/byte_buffer.h"
#include "common/tsblock/tsblock.h"

namespace common {
Expand Down Expand Up @@ -182,4 +185,143 @@ TEST(TsBlockTest, ColIterator_ReadAndNext) {
EXPECT_TRUE(col_iterator.end());
}

TEST(TsBlockTest, ColAppenderFillRepeatedFixed) {
TupleDesc tuple_desc;
tuple_desc.push_back(ColumnSchema("value", INT32, UNCOMPRESSED, PLAIN));
TsBlock ts_block(&tuple_desc, 10);
ASSERT_EQ(ts_block.init(), E_OK);
ColAppender col_appender(0, &ts_block);

int32_t value = 42;
ASSERT_EQ(col_appender.fill(reinterpret_cast<const char*>(&value),
sizeof(value), 5),
E_OK);
EXPECT_EQ(col_appender.get_col_row_count(), 5u);
EXPECT_EQ(ts_block.get_vector(0)->get_row_num(), 5u);

ColIterator iter(0, &ts_block);
for (int i = 0; i < 5; ++i) {
uint32_t len = 0;
char* read = iter.read(&len);
ASSERT_NE(read, nullptr);
EXPECT_EQ(len, sizeof(int32_t));
int32_t read_value = 0;
std::memcpy(&read_value, read, sizeof(read_value));
EXPECT_EQ(read_value, value);
iter.next();
}
}

TEST(TsBlockTest, ColAppenderFillRepeatedVariable) {
TupleDesc tuple_desc;
tuple_desc.push_back(ColumnSchema("tag", STRING, UNCOMPRESSED, PLAIN));
TsBlock ts_block(&tuple_desc, 10);
ASSERT_EQ(ts_block.init(), E_OK);
ColAppender col_appender(0, &ts_block);

const std::string tag = "device-a";
ASSERT_EQ(col_appender.fill(tag.data(), tag.size(), 4), E_OK);
// A second fill continues after the rows already present.
ASSERT_EQ(col_appender.fill(tag.data(), tag.size(), 7), E_OK);
EXPECT_EQ(col_appender.get_col_row_count(), 7u);
EXPECT_EQ(ts_block.get_vector(0)->get_row_num(), 7u);

ColIterator iter(0, &ts_block);
for (int i = 0; i < 7; ++i) {
uint32_t len = 0;
char* read = iter.read(&len);
ASSERT_NE(read, nullptr);
EXPECT_EQ(std::string(read, len), tag);
iter.next();
}
}

TEST(TsBlockTest, ColAppenderFillRepeatedRejectsOverflow) {
TupleDesc tuple_desc;
tuple_desc.push_back(ColumnSchema("value", INT32, UNCOMPRESSED, PLAIN));
TsBlock ts_block(&tuple_desc, 3);
ASSERT_EQ(ts_block.init(), E_OK);
ColAppender col_appender(0, &ts_block);

int32_t value = 1;
EXPECT_EQ(col_appender.fill(reinterpret_cast<const char*>(&value),
sizeof(value), 4),
E_INVALID_ARG);
EXPECT_EQ(col_appender.get_col_row_count(), 0u);
EXPECT_EQ(ts_block.get_vector(0)->get_row_num(), 0u);
}

// ByteBuffer-level zero-count guard: the helpers used to write the pattern
// before checking count, which writes past the end of an exactly-full buffer.
TEST(TsBlockTest, ByteBufferFillRepeatedZeroCountWritesNothing) {
ByteBuffer buffer;
buffer.init(64);
char payload[16] = {'a', 'b', 'c'};
Comment thread
ColinLeeo marked this conversation as resolved.
Resolved
ASSERT_EQ(buffer.append_variable_value(payload, 16), E_OK);
const uint32_t size_before = buffer.get_data_size();
char* data = buffer.get_data();
data[size_before] = '\x5a'; // sentinel in the slack after the data

EXPECT_EQ(buffer.fill_repeated_variable("wxyz", 4, 0), E_OK);
EXPECT_EQ(buffer.fill_repeated_fixed("ABCD", 4, 0), E_OK);
EXPECT_EQ(buffer.get_data_size(), size_before);
EXPECT_EQ(data[size_before], '\x5a');
}

// Fill a tag longer than the per-row reservation so extend_memory() has to
// grow (and copy) the buffer mid-fill.
TEST(TsBlockTest, ColAppenderFillRepeatedVariableGrows) {
TupleDesc tuple_desc;
tuple_desc.push_back(ColumnSchema("tag", STRING, UNCOMPRESSED, PLAIN));
TsBlock ts_block(&tuple_desc, 100);
ASSERT_EQ(ts_block.init(), E_OK);
ColAppender col_appender(0, &ts_block);

const std::string tag(64, 'g');
ASSERT_EQ(col_appender.fill(tag.data(), tag.size(), 100), E_OK);
EXPECT_EQ(ts_block.get_vector(0)->get_row_num(), 100u);

ColIterator iter(0, &ts_block);
for (int i = 0; i < 100; ++i) {
uint32_t len = 0;
char* read = iter.read(&len);
ASSERT_NE(read, nullptr);
EXPECT_EQ(std::string(read, len), tag);
iter.next();
}
EXPECT_TRUE(iter.end());
}

// The size computation multiplies two 32-bit values; without a pre-check it
// wraps uint64_t and bypasses both the capacity and the 4 GiB limit, after
// which a huge memcpy runs into a small buffer.
TEST(TsBlockTest, ByteBufferFillRepeatedRejectsSizeOverflow) {
ByteBuffer buffer;
buffer.init(64);
char payload[4] = {'a', 'b', 'c', 'd'};
ASSERT_EQ(buffer.append_variable_value(payload, 4), E_OK);
const uint32_t size_before = buffer.get_data_size();

EXPECT_EQ(buffer.fill_repeated_variable(payload, 0xFFFFFFFDu, 0xFFFFFFFFu),
E_OOM);
EXPECT_EQ(buffer.get_data_size(), size_before);
EXPECT_EQ(buffer.fill_repeated_fixed(payload, 4, 0xFFFFFFFFu), E_OOM);
EXPECT_EQ(buffer.get_data_size(), size_before);
}

TEST(TsBlockTest, ColAppenderFillRepeatedZeroCountIsNoOp) {
TupleDesc tuple_desc;
tuple_desc.push_back(ColumnSchema("value", INT32, UNCOMPRESSED, PLAIN));
TsBlock ts_block(&tuple_desc, 3);
ASSERT_EQ(ts_block.init(), E_OK);
ColAppender col_appender(0, &ts_block);

int32_t value = 1;
EXPECT_EQ(col_appender.fill(reinterpret_cast<const char*>(&value),
sizeof(value), 0),
E_OK);
EXPECT_EQ(col_appender.get_col_row_count(), 0u);
EXPECT_EQ(ts_block.get_vector(0)->get_row_num(), 0u);
}

} // namespace common
Loading