diff --git a/cpp/src/common/container/byte_buffer.h b/cpp/src/common/container/byte_buffer.h index a3b2c525f..fc5e9f99e 100644 --- a/cpp/src/common/container/byte_buffer.h +++ b/cpp/src/common/container/byte_buffer.h @@ -23,6 +23,9 @@ #include #include +#include +#include + #include "common/allocator/alloc_base.h" #include "common/global.h" #include "utils/util_define.h" @@ -133,6 +136,87 @@ class ByteBuffer { FORCE_INLINE uint32_t get_data_size() const { return real_data_size_; } + // Append @p count copies of one variable-length value. A block's TAG column + // repeats the same value on every row; writing the pattern once and then + // doubling the filled region beats @p count individual append calls. + FORCE_INLINE int fill_repeated_variable(const char* value, uint32_t len, + uint32_t count) { + if (count == 0) { + return E_OK; + } + // The stored length prefix is copied from a uint32_t, so the width + // must not exceed it (the constructor fixes it to sizeof(uint32_t)). + if (variable_type_len_ > sizeof(uint32_t)) { + return E_INVALID_ARG; + } + const uint64_t elem_size = + static_cast(variable_type_len_) + len; + if (elem_size == 0) { + return E_INVALID_ARG; + } + // elem_size * count can overflow uint64_t (both factors are 32-bit), + // which would wrap `need` to a small value and bypass the capacity + // checks before a huge payload copy. Reject anything that cannot fit + // in the 32-bit buffer size first. + const uint64_t available = + static_cast(0xFFFFFFFFULL) - real_data_size_; + if (count > available / elem_size) { + return E_OOM; + } + const uint64_t need = + static_cast(real_data_size_) + elem_size * count; + if (need > reserved_size_) { + if (extend_memory(static_cast(need)) != E_OK) { + return E_OOM; + } + } + char* dst = data_ + real_data_size_; + std::memcpy(dst, &len, variable_type_len_); + if (len > 0) { + std::memcpy(dst + variable_type_len_, value, len); + } + uint32_t filled = 1; + while (filled < count) { + const uint32_t copy = std::min(filled, count - filled); + std::memcpy(dst + static_cast(filled) * elem_size, dst, + static_cast(copy) * elem_size); + filled += copy; + } + real_data_size_ = static_cast(need); + return E_OK; + } + + // Fixed-width counterpart of fill_repeated_variable(). + FORCE_INLINE int fill_repeated_fixed(const char* value, uint32_t elem_size, + uint32_t count) { + if (count == 0) { + return E_OK; + } + const uint64_t available = + static_cast(0xFFFFFFFFULL) - real_data_size_; + if (elem_size != 0 && count > available / elem_size) { + return E_OOM; + } + const uint64_t need = static_cast(real_data_size_) + + static_cast(elem_size) * count; + if (need > reserved_size_) { + if (extend_memory(static_cast(need)) != E_OK) { + return E_OOM; + } + } + char* dst = data_ + real_data_size_; + std::memcpy(dst, value, elem_size); + uint32_t filled = 1; + while (filled < count) { + const uint32_t copy = std::min(filled, count - filled); + std::memcpy(dst + static_cast(filled) * elem_size, dst, + static_cast(copy) * elem_size); + filled += copy; + } + real_data_size_ = static_cast(need); + return E_OK; + } + // Declare bytes that were written directly through get_data() without going // through append_*(). Used when a caller decodes straight into this buffer // at a known offset. diff --git a/cpp/src/common/tsblock/tsblock.h b/cpp/src/common/tsblock/tsblock.h index 056186211..f0ed388c0 100644 --- a/cpp/src/common/tsblock/tsblock.h +++ b/cpp/src/common/tsblock/tsblock.h @@ -244,12 +244,19 @@ class ColAppender { return E_OK; } FORCE_INLINE int fill(const char* value, uint32_t len, uint32_t end_index) { - while (column_row_count_ < end_index) { - if (!add_row()) { - return E_INVALID_ARG; - } - append(value, len); + if (column_row_count_ >= end_index) { + return E_OK; } + const uint32_t count = end_index - column_row_count_; + if (column_row_count_ + count > tsblock_->max_row_count_) { + return E_INVALID_ARG; + } + const int ret = vec_->fill_repeated(value, len, count); + if (ret != E_OK) { + // E_OOM from a failed growth is not an argument error. + return ret; + } + column_row_count_ += count; return E_OK; } FORCE_INLINE void reset() { column_row_count_ = 0; } diff --git a/cpp/src/common/tsblock/vector/fixed_length_vector.h b/cpp/src/common/tsblock/vector/fixed_length_vector.h index eb5596343..72b5abe7d 100644 --- a/cpp/src/common/tsblock/vector/fixed_length_vector.h +++ b/cpp/src/common/tsblock/vector/fixed_length_vector.h @@ -54,6 +54,25 @@ class FixedLengthVector : public Vector { add_row_nums(count); } + int fill_repeated(const char* value, uint32_t len, + uint32_t count) OVERRIDE { + if (count == 0) { + return common::E_OK; + } + // Readers always advance by type_len_, so the fill must use the same + // stride. A mismatch is a caller error and, in a release build, would + // read past a shorter value buffer. + if (len != type_len_) { + return common::E_INVALID_ARG; + } + if (values_.fill_repeated_fixed(value, type_len_, count) != + common::E_OK) { + return common::E_OOM; + } + add_row_nums(count); + return common::E_OK; + } + FORCE_INLINE uint32_t get_type_len() const { return type_len_; } // cppcheck-suppress missingOverride diff --git a/cpp/src/common/tsblock/vector/variable_length_vector.h b/cpp/src/common/tsblock/vector/variable_length_vector.h index 84e541e5c..95800ba0c 100644 --- a/cpp/src/common/tsblock/vector/variable_length_vector.h +++ b/cpp/src/common/tsblock/vector/variable_length_vector.h @@ -61,6 +61,19 @@ class VariableLengthVector : public Vector { values_.append_variable_value(value, len); } + int fill_repeated(const char* value, uint32_t len, + uint32_t count) OVERRIDE { + if (count == 0) { + return common::E_OK; + } + const int ret = values_.fill_repeated_variable(value, len, count); + if (ret != common::E_OK) { + return ret; + } + add_row_nums(count); + return common::E_OK; + } + // cppcheck-suppress missingOverride FORCE_INLINE char* read(uint32_t* __restrict len, bool* __restrict null, uint32_t rowid) OVERRIDE { diff --git a/cpp/src/common/tsblock/vector/vector.h b/cpp/src/common/tsblock/vector/vector.h index dde3e76cc..93b886f02 100644 --- a/cpp/src/common/tsblock/vector/vector.h +++ b/cpp/src/common/tsblock/vector/vector.h @@ -47,6 +47,17 @@ class Vector { // len == 0 is '' or "" virtual void append(const char* value, uint32_t len) = 0; + // Append @p count copies of the same value and bump the row count. The + // default is a plain loop; vectors whose layout allows it override this + // with a bulk fill (a block's TAG column repeats one value on every row). + virtual int fill_repeated(const char* value, uint32_t len, uint32_t count) { + for (uint32_t i = 0; i < count; i++) { + append(value, len); + } + add_row_nums(count); + return E_OK; + } + virtual char* read(uint32_t* __restrict len, bool* __restrict null, uint32_t rowid) = 0; diff --git a/cpp/test/common/tsblock/tslock_test.cc b/cpp/test/common/tsblock/tslock_test.cc index c58b368f5..753b96ad0 100644 --- a/cpp/test/common/tsblock/tslock_test.cc +++ b/cpp/test/common/tsblock/tslock_test.cc @@ -18,6 +18,9 @@ */ #include +#include + +#include "common/container/byte_buffer.h" #include "common/tsblock/tsblock.h" namespace common { @@ -182,4 +185,143 @@ TEST(TsBlockTest, ColIterator_ReadAndNext) { EXPECT_TRUE(col_iterator.end()); } +TEST(TsBlockTest, ColAppenderFillRepeatedFixed) { + TupleDesc tuple_desc; + tuple_desc.push_back(ColumnSchema("value", INT32, UNCOMPRESSED, PLAIN)); + TsBlock ts_block(&tuple_desc, 10); + ASSERT_EQ(ts_block.init(), E_OK); + ColAppender col_appender(0, &ts_block); + + int32_t value = 42; + ASSERT_EQ(col_appender.fill(reinterpret_cast(&value), + sizeof(value), 5), + E_OK); + EXPECT_EQ(col_appender.get_col_row_count(), 5u); + EXPECT_EQ(ts_block.get_vector(0)->get_row_num(), 5u); + + ColIterator iter(0, &ts_block); + for (int i = 0; i < 5; ++i) { + uint32_t len = 0; + char* read = iter.read(&len); + ASSERT_NE(read, nullptr); + EXPECT_EQ(len, sizeof(int32_t)); + int32_t read_value = 0; + std::memcpy(&read_value, read, sizeof(read_value)); + EXPECT_EQ(read_value, value); + iter.next(); + } +} + +TEST(TsBlockTest, ColAppenderFillRepeatedVariable) { + TupleDesc tuple_desc; + tuple_desc.push_back(ColumnSchema("tag", STRING, UNCOMPRESSED, PLAIN)); + TsBlock ts_block(&tuple_desc, 10); + ASSERT_EQ(ts_block.init(), E_OK); + ColAppender col_appender(0, &ts_block); + + const std::string tag = "device-a"; + ASSERT_EQ(col_appender.fill(tag.data(), tag.size(), 4), E_OK); + // A second fill continues after the rows already present. + ASSERT_EQ(col_appender.fill(tag.data(), tag.size(), 7), E_OK); + EXPECT_EQ(col_appender.get_col_row_count(), 7u); + EXPECT_EQ(ts_block.get_vector(0)->get_row_num(), 7u); + + ColIterator iter(0, &ts_block); + for (int i = 0; i < 7; ++i) { + uint32_t len = 0; + char* read = iter.read(&len); + ASSERT_NE(read, nullptr); + EXPECT_EQ(std::string(read, len), tag); + iter.next(); + } +} + +TEST(TsBlockTest, ColAppenderFillRepeatedRejectsOverflow) { + TupleDesc tuple_desc; + tuple_desc.push_back(ColumnSchema("value", INT32, UNCOMPRESSED, PLAIN)); + TsBlock ts_block(&tuple_desc, 3); + ASSERT_EQ(ts_block.init(), E_OK); + ColAppender col_appender(0, &ts_block); + + int32_t value = 1; + EXPECT_EQ(col_appender.fill(reinterpret_cast(&value), + sizeof(value), 4), + E_INVALID_ARG); + EXPECT_EQ(col_appender.get_col_row_count(), 0u); + EXPECT_EQ(ts_block.get_vector(0)->get_row_num(), 0u); +} + +// ByteBuffer-level zero-count guard: the helpers used to write the pattern +// before checking count, which writes past the end of an exactly-full buffer. +TEST(TsBlockTest, ByteBufferFillRepeatedZeroCountWritesNothing) { + ByteBuffer buffer; + buffer.init(64); + char payload[16] = {'a', 'b', 'c'}; + ASSERT_EQ(buffer.append_variable_value(payload, 16), E_OK); + const uint32_t size_before = buffer.get_data_size(); + char* data = buffer.get_data(); + data[size_before] = '\x5a'; // sentinel in the slack after the data + + EXPECT_EQ(buffer.fill_repeated_variable("wxyz", 4, 0), E_OK); + EXPECT_EQ(buffer.fill_repeated_fixed("ABCD", 4, 0), E_OK); + EXPECT_EQ(buffer.get_data_size(), size_before); + EXPECT_EQ(data[size_before], '\x5a'); +} + +// Fill a tag longer than the per-row reservation so extend_memory() has to +// grow (and copy) the buffer mid-fill. +TEST(TsBlockTest, ColAppenderFillRepeatedVariableGrows) { + TupleDesc tuple_desc; + tuple_desc.push_back(ColumnSchema("tag", STRING, UNCOMPRESSED, PLAIN)); + TsBlock ts_block(&tuple_desc, 100); + ASSERT_EQ(ts_block.init(), E_OK); + ColAppender col_appender(0, &ts_block); + + const std::string tag(64, 'g'); + ASSERT_EQ(col_appender.fill(tag.data(), tag.size(), 100), E_OK); + EXPECT_EQ(ts_block.get_vector(0)->get_row_num(), 100u); + + ColIterator iter(0, &ts_block); + for (int i = 0; i < 100; ++i) { + uint32_t len = 0; + char* read = iter.read(&len); + ASSERT_NE(read, nullptr); + EXPECT_EQ(std::string(read, len), tag); + iter.next(); + } + EXPECT_TRUE(iter.end()); +} + +// The size computation multiplies two 32-bit values; without a pre-check it +// wraps uint64_t and bypasses both the capacity and the 4 GiB limit, after +// which a huge memcpy runs into a small buffer. +TEST(TsBlockTest, ByteBufferFillRepeatedRejectsSizeOverflow) { + ByteBuffer buffer; + buffer.init(64); + char payload[4] = {'a', 'b', 'c', 'd'}; + ASSERT_EQ(buffer.append_variable_value(payload, 4), E_OK); + const uint32_t size_before = buffer.get_data_size(); + + EXPECT_EQ(buffer.fill_repeated_variable(payload, 0xFFFFFFFDu, 0xFFFFFFFFu), + E_OOM); + EXPECT_EQ(buffer.get_data_size(), size_before); + EXPECT_EQ(buffer.fill_repeated_fixed(payload, 4, 0xFFFFFFFFu), E_OOM); + EXPECT_EQ(buffer.get_data_size(), size_before); +} + +TEST(TsBlockTest, ColAppenderFillRepeatedZeroCountIsNoOp) { + TupleDesc tuple_desc; + tuple_desc.push_back(ColumnSchema("value", INT32, UNCOMPRESSED, PLAIN)); + TsBlock ts_block(&tuple_desc, 3); + ASSERT_EQ(ts_block.init(), E_OK); + ColAppender col_appender(0, &ts_block); + + int32_t value = 1; + EXPECT_EQ(col_appender.fill(reinterpret_cast(&value), + sizeof(value), 0), + E_OK); + EXPECT_EQ(col_appender.get_col_row_count(), 0u); + EXPECT_EQ(ts_block.get_vector(0)->get_row_num(), 0u); +} + } // namespace common