From 19a10ec32d8bbfe805148b38972af476016c79c7 Mon Sep 17 00:00:00 2001 From: ColinLee Date: Thu, 12 Nov 2026 16:02:57 +0800 Subject: [PATCH 01/12] perf(cpp): fill repeated TAG values in bulk A block's TAG (id) column repeats the same value on every row, but ColAppender::fill() appended it row by row: one add_row() plus one append() per row. On a 3.2M-row table scan reading the TAG column this showed up as ~10% of read time in a sampled profile. Add Vector::fill_repeated() (defaulting to a per-row loop) with overrides for the fixed-length and variable-length vectors. Both build the value pattern once and then double the filled region, so the per-row work becomes a single memcpy copy step instead of a full append. 3.2M-row full scan: 0.298s -> 0.269s (~10%) with unchanged output. --- cpp/src/common/container/byte_buffer.h | 58 +++++++++++++++++++ cpp/src/common/tsblock/tsblock.h | 15 +++-- .../tsblock/vector/fixed_length_vector.h | 12 ++++ .../tsblock/vector/variable_length_vector.h | 12 ++++ cpp/src/common/tsblock/vector/vector.h | 11 ++++ 5 files changed, 103 insertions(+), 5 deletions(-) diff --git a/cpp/src/common/container/byte_buffer.h b/cpp/src/common/container/byte_buffer.h index 3f49540932..2b3341f8d3 100644 --- a/cpp/src/common/container/byte_buffer.h +++ b/cpp/src/common/container/byte_buffer.h @@ -23,6 +23,8 @@ #include #include +#include + #include "common/allocator/alloc_base.h" #include "common/global.h" #include "utils/util_define.h" @@ -133,6 +135,62 @@ class ByteBuffer { FORCE_INLINE uint32_t get_data_size() const { return real_data_size_; } + + // Append @p count copies of one variable-length value. A block's TAG column + // repeats the same value on every row; writing the pattern once and then + // doubling the filled region beats @p count individual append calls. + FORCE_INLINE int fill_repeated_variable(const char* value, uint32_t len, + uint32_t count) { + const uint64_t elem_size = + static_cast(variable_type_len_) + len; + const uint64_t need = + static_cast(real_data_size_) + elem_size * count; + if (need > reserved_size_) { + if (need > 0xFFFFFFFFULL || + extend_memory(static_cast(need)) != E_OK) { + return E_OOM; + } + } + char* dst = data_ + real_data_size_; + std::memcpy(dst, &len, variable_type_len_); + if (len > 0) { + std::memcpy(dst + variable_type_len_, value, len); + } + uint32_t filled = 1; + while (filled < count) { + const uint32_t copy = std::min(filled, count - filled); + std::memcpy(dst + static_cast(filled) * elem_size, dst, + static_cast(copy) * elem_size); + filled += copy; + } + real_data_size_ = static_cast(need); + return E_OK; + } + + // Fixed-width counterpart of fill_repeated_variable(). + FORCE_INLINE int fill_repeated_fixed(const char* value, uint32_t elem_size, + uint32_t count) { + const uint64_t need = static_cast(real_data_size_) + + static_cast(elem_size) * count; + if (need > reserved_size_) { + if (need > 0xFFFFFFFFULL || + extend_memory(static_cast(need)) != E_OK) { + return E_OOM; + } + } + char* dst = data_ + real_data_size_; + std::memcpy(dst, value, elem_size); + uint32_t filled = 1; + while (filled < count) { + const uint32_t copy = std::min(filled, count - filled); + std::memcpy(dst + static_cast(filled) * elem_size, dst, + static_cast(copy) * elem_size); + filled += copy; + } + real_data_size_ = static_cast(need); + return E_OK; + } + private: char* data_; uint8_t variable_type_len_; diff --git a/cpp/src/common/tsblock/tsblock.h b/cpp/src/common/tsblock/tsblock.h index 0561862115..dcd155a956 100644 --- a/cpp/src/common/tsblock/tsblock.h +++ b/cpp/src/common/tsblock/tsblock.h @@ -244,12 +244,17 @@ class ColAppender { return E_OK; } FORCE_INLINE int fill(const char* value, uint32_t len, uint32_t end_index) { - while (column_row_count_ < end_index) { - if (!add_row()) { - return E_INVALID_ARG; - } - append(value, len); + if (column_row_count_ >= end_index) { + return E_OK; } + const uint32_t count = end_index - column_row_count_; + if (column_row_count_ + count > tsblock_->max_row_count_) { + return E_INVALID_ARG; + } + if (vec_->fill_repeated(value, len, count) != E_OK) { + return E_INVALID_ARG; + } + column_row_count_ += count; return E_OK; } FORCE_INLINE void reset() { column_row_count_ = 0; } diff --git a/cpp/src/common/tsblock/vector/fixed_length_vector.h b/cpp/src/common/tsblock/vector/fixed_length_vector.h index eb55963438..3ccc46fa02 100644 --- a/cpp/src/common/tsblock/vector/fixed_length_vector.h +++ b/cpp/src/common/tsblock/vector/fixed_length_vector.h @@ -54,6 +54,18 @@ class FixedLengthVector : public Vector { add_row_nums(count); } + int fill_repeated(const char* value, uint32_t len, + uint32_t count) OVERRIDE { + if (count == 0) { + return common::E_OK; + } + if (values_.fill_repeated_fixed(value, len, count) != common::E_OK) { + return common::E_OOM; + } + add_row_nums(count); + return common::E_OK; + } + FORCE_INLINE uint32_t get_type_len() const { return type_len_; } // cppcheck-suppress missingOverride diff --git a/cpp/src/common/tsblock/vector/variable_length_vector.h b/cpp/src/common/tsblock/vector/variable_length_vector.h index 84e541e5c8..0e6d661c3a 100644 --- a/cpp/src/common/tsblock/vector/variable_length_vector.h +++ b/cpp/src/common/tsblock/vector/variable_length_vector.h @@ -61,6 +61,18 @@ class VariableLengthVector : public Vector { values_.append_variable_value(value, len); } + int fill_repeated(const char* value, uint32_t len, + uint32_t count) OVERRIDE { + if (count == 0) { + return common::E_OK; + } + if (values_.fill_repeated_variable(value, len, count) != common::E_OK) { + return common::E_OOM; + } + add_row_nums(count); + return common::E_OK; + } + // cppcheck-suppress missingOverride FORCE_INLINE char* read(uint32_t* __restrict len, bool* __restrict null, uint32_t rowid) OVERRIDE { diff --git a/cpp/src/common/tsblock/vector/vector.h b/cpp/src/common/tsblock/vector/vector.h index dde3e76cce..93b886f024 100644 --- a/cpp/src/common/tsblock/vector/vector.h +++ b/cpp/src/common/tsblock/vector/vector.h @@ -47,6 +47,17 @@ class Vector { // len == 0 is '' or "" virtual void append(const char* value, uint32_t len) = 0; + // Append @p count copies of the same value and bump the row count. The + // default is a plain loop; vectors whose layout allows it override this + // with a bulk fill (a block's TAG column repeats one value on every row). + virtual int fill_repeated(const char* value, uint32_t len, uint32_t count) { + for (uint32_t i = 0; i < count; i++) { + append(value, len); + } + add_row_nums(count); + return E_OK; + } + virtual char* read(uint32_t* __restrict len, bool* __restrict null, uint32_t rowid) = 0; From a0ec911981d441127cc4efb4b55c03f0b4776d98 Mon Sep 17 00:00:00 2001 From: ColinLee Date: Thu, 12 Nov 2026 16:29:51 +0800 Subject: [PATCH 02/12] style: apply spotless formatting --- cpp/src/common/container/byte_buffer.h | 1 - 1 file changed, 1 deletion(-) diff --git a/cpp/src/common/container/byte_buffer.h b/cpp/src/common/container/byte_buffer.h index 2b3341f8d3..22b2f62bcf 100644 --- a/cpp/src/common/container/byte_buffer.h +++ b/cpp/src/common/container/byte_buffer.h @@ -135,7 +135,6 @@ class ByteBuffer { FORCE_INLINE uint32_t get_data_size() const { return real_data_size_; } - // Append @p count copies of one variable-length value. A block's TAG column // repeats the same value on every row; writing the pattern once and then // doubling the filled region beats @p count individual append calls. From 1f2916be34af0000e4556e3fd6e2a3b93d5beaa4 Mon Sep 17 00:00:00 2001 From: ColinLee Date: Thu, 12 Nov 2026 16:35:36 +0800 Subject: [PATCH 03/12] test(cpp): cover Vector::fill_repeated bulk TAG filling Exercise the fixed-length and variable-length overrides directly: value and length correctness after multiple fills, row counters, capacity overflow rejection, and the zero-count no-op. --- cpp/test/common/tsblock/tslock_test.cc | 79 ++++++++++++++++++++++++++ 1 file changed, 79 insertions(+) diff --git a/cpp/test/common/tsblock/tslock_test.cc b/cpp/test/common/tsblock/tslock_test.cc index c58b368f5e..c133cbc6f4 100644 --- a/cpp/test/common/tsblock/tslock_test.cc +++ b/cpp/test/common/tsblock/tslock_test.cc @@ -182,4 +182,83 @@ TEST(TsBlockTest, ColIterator_ReadAndNext) { EXPECT_TRUE(col_iterator.end()); } +TEST(TsBlockTest, ColAppenderFillRepeatedFixed) { + TupleDesc tuple_desc; + tuple_desc.push_back(ColumnSchema("value", INT32, UNCOMPRESSED, PLAIN)); + TsBlock ts_block(&tuple_desc, 10); + ASSERT_EQ(ts_block.init(), E_OK); + ColAppender col_appender(0, &ts_block); + + int32_t value = 42; + ASSERT_EQ(col_appender.fill(reinterpret_cast(&value), + sizeof(value), 5), + E_OK); + EXPECT_EQ(col_appender.get_col_row_count(), 5u); + EXPECT_EQ(ts_block.get_vector(0)->get_row_num(), 5u); + + ColIterator iter(0, &ts_block); + for (int i = 0; i < 5; ++i) { + uint32_t len = 0; + char* read = iter.read(&len); + ASSERT_NE(read, nullptr); + EXPECT_EQ(len, sizeof(int32_t)); + EXPECT_EQ(*reinterpret_cast(read), value); + iter.next(); + } +} + +TEST(TsBlockTest, ColAppenderFillRepeatedVariable) { + TupleDesc tuple_desc; + tuple_desc.push_back(ColumnSchema("tag", STRING, UNCOMPRESSED, PLAIN)); + TsBlock ts_block(&tuple_desc, 10); + ASSERT_EQ(ts_block.init(), E_OK); + ColAppender col_appender(0, &ts_block); + + const std::string tag = "device-a"; + ASSERT_EQ(col_appender.fill(tag.data(), tag.size(), 4), E_OK); + // A second fill continues after the rows already present. + ASSERT_EQ(col_appender.fill(tag.data(), tag.size(), 7), E_OK); + EXPECT_EQ(col_appender.get_col_row_count(), 7u); + EXPECT_EQ(ts_block.get_vector(0)->get_row_num(), 7u); + + ColIterator iter(0, &ts_block); + for (int i = 0; i < 7; ++i) { + uint32_t len = 0; + char* read = iter.read(&len); + ASSERT_NE(read, nullptr); + EXPECT_EQ(std::string(read, len), tag); + iter.next(); + } +} + +TEST(TsBlockTest, ColAppenderFillRepeatedRejectsOverflow) { + TupleDesc tuple_desc; + tuple_desc.push_back(ColumnSchema("value", INT32, UNCOMPRESSED, PLAIN)); + TsBlock ts_block(&tuple_desc, 3); + ASSERT_EQ(ts_block.init(), E_OK); + ColAppender col_appender(0, &ts_block); + + int32_t value = 1; + EXPECT_EQ(col_appender.fill(reinterpret_cast(&value), + sizeof(value), 4), + E_INVALID_ARG); + EXPECT_EQ(col_appender.get_col_row_count(), 0u); + EXPECT_EQ(ts_block.get_vector(0)->get_row_num(), 0u); +} + +TEST(TsBlockTest, ColAppenderFillRepeatedZeroCountIsNoOp) { + TupleDesc tuple_desc; + tuple_desc.push_back(ColumnSchema("value", INT32, UNCOMPRESSED, PLAIN)); + TsBlock ts_block(&tuple_desc, 3); + ASSERT_EQ(ts_block.init(), E_OK); + ColAppender col_appender(0, &ts_block); + + int32_t value = 1; + EXPECT_EQ(col_appender.fill(reinterpret_cast(&value), + sizeof(value), 0), + E_OK); + EXPECT_EQ(col_appender.get_col_row_count(), 0u); + EXPECT_EQ(ts_block.get_vector(0)->get_row_num(), 0u); +} + } // namespace common From 4973961342560c31781295632b2b927c20623559 Mon Sep 17 00:00:00 2001 From: ColinLee Date: Thu, 12 Nov 2026 17:42:19 +0800 Subject: [PATCH 04/12] fix(cpp): harden the bulk TAG-column fill helpers MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Review follow-ups on the new fill path: * Both ByteBuffer fill helpers wrote the first element before looking at count, so a count == 0 call wrote into the slack after the data — or past the end of an exactly-full buffer. Return early instead. * FixedLengthVector::fill_repeated used the caller's len as the element stride while every read path advances by type_len_. Use type_len_ and assert the caller agrees. * ColAppender::fill reported every failure (including E_OOM from a failed growth) as E_INVALID_ARG; propagate the actual code. Tests: zero-count at the ByteBuffer level (fails without the guard) and a variable-length fill larger than the per-row reservation, which exercises the grow-and-copy path. --- cpp/src/common/container/byte_buffer.h | 6 +++ cpp/src/common/tsblock/tsblock.h | 6 ++- .../tsblock/vector/fixed_length_vector.h | 7 +++- cpp/test/common/tsblock/tslock_test.cc | 42 +++++++++++++++++++ 4 files changed, 58 insertions(+), 3 deletions(-) diff --git a/cpp/src/common/container/byte_buffer.h b/cpp/src/common/container/byte_buffer.h index 22b2f62bcf..06e8c76550 100644 --- a/cpp/src/common/container/byte_buffer.h +++ b/cpp/src/common/container/byte_buffer.h @@ -140,6 +140,9 @@ class ByteBuffer { // doubling the filled region beats @p count individual append calls. FORCE_INLINE int fill_repeated_variable(const char* value, uint32_t len, uint32_t count) { + if (count == 0) { + return E_OK; + } const uint64_t elem_size = static_cast(variable_type_len_) + len; const uint64_t need = @@ -169,6 +172,9 @@ class ByteBuffer { // Fixed-width counterpart of fill_repeated_variable(). FORCE_INLINE int fill_repeated_fixed(const char* value, uint32_t elem_size, uint32_t count) { + if (count == 0) { + return E_OK; + } const uint64_t need = static_cast(real_data_size_) + static_cast(elem_size) * count; if (need > reserved_size_) { diff --git a/cpp/src/common/tsblock/tsblock.h b/cpp/src/common/tsblock/tsblock.h index dcd155a956..f0ed388c0f 100644 --- a/cpp/src/common/tsblock/tsblock.h +++ b/cpp/src/common/tsblock/tsblock.h @@ -251,8 +251,10 @@ class ColAppender { if (column_row_count_ + count > tsblock_->max_row_count_) { return E_INVALID_ARG; } - if (vec_->fill_repeated(value, len, count) != E_OK) { - return E_INVALID_ARG; + const int ret = vec_->fill_repeated(value, len, count); + if (ret != E_OK) { + // E_OOM from a failed growth is not an argument error. + return ret; } column_row_count_ += count; return E_OK; diff --git a/cpp/src/common/tsblock/vector/fixed_length_vector.h b/cpp/src/common/tsblock/vector/fixed_length_vector.h index 3ccc46fa02..49db3bba2e 100644 --- a/cpp/src/common/tsblock/vector/fixed_length_vector.h +++ b/cpp/src/common/tsblock/vector/fixed_length_vector.h @@ -59,7 +59,12 @@ class FixedLengthVector : public Vector { if (count == 0) { return common::E_OK; } - if (values_.fill_repeated_fixed(value, len, count) != common::E_OK) { + // Readers always advance by type_len_, so the fill must use the same + // stride; a mismatched len would produce a buffer that is written with + // one stride and read with another. + ASSERT(len == type_len_); + if (values_.fill_repeated_fixed(value, type_len_, count) != + common::E_OK) { return common::E_OOM; } add_row_nums(count); diff --git a/cpp/test/common/tsblock/tslock_test.cc b/cpp/test/common/tsblock/tslock_test.cc index c133cbc6f4..bc802d7020 100644 --- a/cpp/test/common/tsblock/tslock_test.cc +++ b/cpp/test/common/tsblock/tslock_test.cc @@ -18,6 +18,7 @@ */ #include +#include "common/container/byte_buffer.h" #include "common/tsblock/tsblock.h" namespace common { @@ -246,6 +247,47 @@ TEST(TsBlockTest, ColAppenderFillRepeatedRejectsOverflow) { EXPECT_EQ(ts_block.get_vector(0)->get_row_num(), 0u); } +// ByteBuffer-level zero-count guard: the helpers used to write the pattern +// before checking count, which writes past the end of an exactly-full buffer. +TEST(TsBlockTest, ByteBufferFillRepeatedZeroCountWritesNothing) { + ByteBuffer buffer; + buffer.init(64); + char payload[16] = {'a', 'b', 'c'}; + ASSERT_EQ(buffer.append_variable_value(payload, 16), E_OK); + const uint32_t size_before = buffer.get_data_size(); + char* data = buffer.get_data(); + data[size_before] = '\x5a'; // sentinel in the slack after the data + + EXPECT_EQ(buffer.fill_repeated_variable("wxyz", 4, 0), E_OK); + EXPECT_EQ(buffer.fill_repeated_fixed("ABCD", 4, 0), E_OK); + EXPECT_EQ(buffer.get_data_size(), size_before); + EXPECT_EQ(data[size_before], '\x5a'); +} + +// Fill a tag longer than the per-row reservation so extend_memory() has to +// grow (and copy) the buffer mid-fill. +TEST(TsBlockTest, ColAppenderFillRepeatedVariableGrows) { + TupleDesc tuple_desc; + tuple_desc.push_back(ColumnSchema("tag", STRING, UNCOMPRESSED, PLAIN)); + TsBlock ts_block(&tuple_desc, 100); + ASSERT_EQ(ts_block.init(), E_OK); + ColAppender col_appender(0, &ts_block); + + const std::string tag(64, 'g'); + ASSERT_EQ(col_appender.fill(tag.data(), tag.size(), 100), E_OK); + EXPECT_EQ(ts_block.get_vector(0)->get_row_num(), 100u); + + ColIterator iter(0, &ts_block); + for (int i = 0; i < 100; ++i) { + uint32_t len = 0; + char* read = iter.read(&len); + ASSERT_NE(read, nullptr); + EXPECT_EQ(std::string(read, len), tag); + iter.next(); + } + EXPECT_TRUE(iter.end()); +} + TEST(TsBlockTest, ColAppenderFillRepeatedZeroCountIsNoOp) { TupleDesc tuple_desc; tuple_desc.push_back(ColumnSchema("value", INT32, UNCOMPRESSED, PLAIN)); From 8f7f83114989c718f4bf6eb3b693c7790e2140fb Mon Sep 17 00:00:00 2001 From: ColinLee Date: Thu, 12 Nov 2026 23:33:52 +0800 Subject: [PATCH 05/12] fix(cpp): make the fixed-length fill stride a runtime check Return E_INVALID_ARG when the caller's len differs from type_len_ instead of relying on ASSERT, which is compiled out in release and would make the fill read past a shorter value buffer. Also include for std::memcpy and copy test values into aligned locals instead of reinterpret_cast'ing the row buffer. --- cpp/src/common/container/byte_buffer.h | 1 + cpp/src/common/tsblock/vector/fixed_length_vector.h | 8 +++++--- cpp/test/common/tsblock/tslock_test.cc | 2 ++ 3 files changed, 8 insertions(+), 3 deletions(-) diff --git a/cpp/src/common/container/byte_buffer.h b/cpp/src/common/container/byte_buffer.h index 06e8c76550..92de248cbe 100644 --- a/cpp/src/common/container/byte_buffer.h +++ b/cpp/src/common/container/byte_buffer.h @@ -24,6 +24,7 @@ #include #include +#include #include "common/allocator/alloc_base.h" #include "common/global.h" diff --git a/cpp/src/common/tsblock/vector/fixed_length_vector.h b/cpp/src/common/tsblock/vector/fixed_length_vector.h index 49db3bba2e..72b5abe7db 100644 --- a/cpp/src/common/tsblock/vector/fixed_length_vector.h +++ b/cpp/src/common/tsblock/vector/fixed_length_vector.h @@ -60,9 +60,11 @@ class FixedLengthVector : public Vector { return common::E_OK; } // Readers always advance by type_len_, so the fill must use the same - // stride; a mismatched len would produce a buffer that is written with - // one stride and read with another. - ASSERT(len == type_len_); + // stride. A mismatch is a caller error and, in a release build, would + // read past a shorter value buffer. + if (len != type_len_) { + return common::E_INVALID_ARG; + } if (values_.fill_repeated_fixed(value, type_len_, count) != common::E_OK) { return common::E_OOM; diff --git a/cpp/test/common/tsblock/tslock_test.cc b/cpp/test/common/tsblock/tslock_test.cc index bc802d7020..820275a7b8 100644 --- a/cpp/test/common/tsblock/tslock_test.cc +++ b/cpp/test/common/tsblock/tslock_test.cc @@ -18,6 +18,8 @@ */ #include +#include + #include "common/container/byte_buffer.h" #include "common/tsblock/tsblock.h" From 2bfd148b4d6f578555f476ceaef942f4deecf0fc Mon Sep 17 00:00:00 2001 From: ColinLee Date: Fri, 13 Nov 2026 01:04:10 +0800 Subject: [PATCH 06/12] fix(cpp): guard the stored-length width; copy test value out Return E_INVALID_ARG if variable_type_len_ ever exceeds sizeof(uint32_t) before the length prefix is copied from a uint32_t (it is fixed to sizeof(uint32_t) today, but make the invariant explicit). Also copy the int32_t test value out with memcpy instead of reinterpret_cast'ing the row buffer, which is what was added for. --- cpp/src/common/container/byte_buffer.h | 5 +++++ cpp/test/common/tsblock/tslock_test.cc | 4 +++- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/cpp/src/common/container/byte_buffer.h b/cpp/src/common/container/byte_buffer.h index 92de248cbe..f755ce3376 100644 --- a/cpp/src/common/container/byte_buffer.h +++ b/cpp/src/common/container/byte_buffer.h @@ -144,6 +144,11 @@ class ByteBuffer { if (count == 0) { return E_OK; } + // The stored length prefix is copied from a uint32_t, so the width + // must not exceed it (the constructor fixes it to sizeof(uint32_t)). + if (variable_type_len_ > sizeof(uint32_t)) { + return E_INVALID_ARG; + } const uint64_t elem_size = static_cast(variable_type_len_) + len; const uint64_t need = diff --git a/cpp/test/common/tsblock/tslock_test.cc b/cpp/test/common/tsblock/tslock_test.cc index 820275a7b8..e3c5e2d5ba 100644 --- a/cpp/test/common/tsblock/tslock_test.cc +++ b/cpp/test/common/tsblock/tslock_test.cc @@ -205,7 +205,9 @@ TEST(TsBlockTest, ColAppenderFillRepeatedFixed) { char* read = iter.read(&len); ASSERT_NE(read, nullptr); EXPECT_EQ(len, sizeof(int32_t)); - EXPECT_EQ(*reinterpret_cast(read), value); + int32_t read_value = 0; + std::memcpy(&read_value, read, sizeof(read_value)); + EXPECT_EQ(read_value, value); iter.next(); } } From e9e9a23bc0177c974816955ee486fd1e0d40b87f Mon Sep 17 00:00:00 2001 From: ColinLee Date: Fri, 13 Nov 2026 01:32:56 +0800 Subject: [PATCH 07/12] test(cpp): give reader-test files a per-process unique name Same isolation fix as the writer tests: parallel CTest processes can seed the name generator in the same clock tick, and one then removes the file another is opening (E_ALREADY_EXIST at the fixture's open, observed on the macOS 3.11 and py-msvc 3.11 jobs). Append pid and a per-process counter. --- cpp/test/reader/tsfile_reader_test.cc | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/cpp/test/reader/tsfile_reader_test.cc b/cpp/test/reader/tsfile_reader_test.cc index 060c7a5411..1fe277ddef 100644 --- a/cpp/test/reader/tsfile_reader_test.cc +++ b/cpp/test/reader/tsfile_reader_test.cc @@ -21,6 +21,7 @@ #include #include +#include #include #include #include @@ -43,6 +44,12 @@ #include "reader/block/single_device_tsblock_reader.h" #include "reader/filter/time_operator.h" #include "reader/qds_without_timegenerator.h" + +#ifdef _WIN32 +#include +#else +#include +#endif #include "reader/tsfile_series_scan_iterator.h" #include "writer/chunk_writer.h" #include "writer/tsfile_writer.h" @@ -118,6 +125,18 @@ class TsFileReaderTest : public ::testing::Test { random_string += chars[dis(gen)]; } + // CTest runs test cases in parallel processes, and two of them can + // seed the generator in the same clock tick. Without a per-process + // unique suffix one process can remove the file another is using, + // which surfaced as E_ALREADY_EXIST from TsFileWriter::open. +#ifdef _WIN32 + const auto process_id = static_cast(_getpid()); +#else + const auto process_id = static_cast(getpid()); +#endif + static std::atomic counter{0}; + random_string += "_" + std::to_string(process_id) + "_" + + std::to_string(counter.fetch_add(1)); return random_string; } From 450805f288163a9f41e3b6f687591e1adc1dfd54 Mon Sep 17 00:00:00 2001 From: ColinLee Date: Fri, 13 Nov 2026 14:26:03 +0800 Subject: [PATCH 08/12] fix(cpp): reject repeated-fill sizes that overflow before multiplying elem_size * count multiplies two 32-bit values, so with len near UINT32_MAX and a large count the product wraps uint64_t, need lands below the capacity check and a ~4 GiB copy runs into the small buffer. Bound the count by the remaining 32-bit capacity first and return E_OOM; the fixed-width helper gets the same check. Test reproduces the wrapping case (crashes without the guard). --- cpp/src/common/container/byte_buffer.h | 20 ++++++++++++++++---- cpp/test/common/tsblock/tslock_test.cc | 17 +++++++++++++++++ 2 files changed, 33 insertions(+), 4 deletions(-) diff --git a/cpp/src/common/container/byte_buffer.h b/cpp/src/common/container/byte_buffer.h index c2defea248..90e7900963 100644 --- a/cpp/src/common/container/byte_buffer.h +++ b/cpp/src/common/container/byte_buffer.h @@ -151,11 +151,19 @@ class ByteBuffer { } const uint64_t elem_size = static_cast(variable_type_len_) + len; + // elem_size * count can overflow uint64_t (both factors are 32-bit), + // which would wrap `need` to a small value and bypass the capacity + // checks before a huge payload copy. Reject anything that cannot fit + // in the 32-bit buffer size first. + const uint64_t available = + static_cast(0xFFFFFFFFULL) - real_data_size_; + if (count > available / elem_size) { + return E_OOM; + } const uint64_t need = static_cast(real_data_size_) + elem_size * count; if (need > reserved_size_) { - if (need > 0xFFFFFFFFULL || - extend_memory(static_cast(need)) != E_OK) { + if (extend_memory(static_cast(need)) != E_OK) { return E_OOM; } } @@ -181,11 +189,15 @@ class ByteBuffer { if (count == 0) { return E_OK; } + const uint64_t available = + static_cast(0xFFFFFFFFULL) - real_data_size_; + if (elem_size != 0 && count > available / elem_size) { + return E_OOM; + } const uint64_t need = static_cast(real_data_size_) + static_cast(elem_size) * count; if (need > reserved_size_) { - if (need > 0xFFFFFFFFULL || - extend_memory(static_cast(need)) != E_OK) { + if (extend_memory(static_cast(need)) != E_OK) { return E_OOM; } } diff --git a/cpp/test/common/tsblock/tslock_test.cc b/cpp/test/common/tsblock/tslock_test.cc index e3c5e2d5ba..753b96ad00 100644 --- a/cpp/test/common/tsblock/tslock_test.cc +++ b/cpp/test/common/tsblock/tslock_test.cc @@ -292,6 +292,23 @@ TEST(TsBlockTest, ColAppenderFillRepeatedVariableGrows) { EXPECT_TRUE(iter.end()); } +// The size computation multiplies two 32-bit values; without a pre-check it +// wraps uint64_t and bypasses both the capacity and the 4 GiB limit, after +// which a huge memcpy runs into a small buffer. +TEST(TsBlockTest, ByteBufferFillRepeatedRejectsSizeOverflow) { + ByteBuffer buffer; + buffer.init(64); + char payload[4] = {'a', 'b', 'c', 'd'}; + ASSERT_EQ(buffer.append_variable_value(payload, 4), E_OK); + const uint32_t size_before = buffer.get_data_size(); + + EXPECT_EQ(buffer.fill_repeated_variable(payload, 0xFFFFFFFDu, 0xFFFFFFFFu), + E_OOM); + EXPECT_EQ(buffer.get_data_size(), size_before); + EXPECT_EQ(buffer.fill_repeated_fixed(payload, 4, 0xFFFFFFFFu), E_OOM); + EXPECT_EQ(buffer.get_data_size(), size_before); +} + TEST(TsBlockTest, ColAppenderFillRepeatedZeroCountIsNoOp) { TupleDesc tuple_desc; tuple_desc.push_back(ColumnSchema("value", INT32, UNCOMPRESSED, PLAIN)); From 3a92f528ab314fe3df6323fa87ccac3ecc3257e2 Mon Sep 17 00:00:00 2001 From: ColinLee Date: Fri, 13 Nov 2026 14:45:10 +0800 Subject: [PATCH 09/12] test(cpp): give the field-filter test files a per-process unique name Same parallel-CTest collision as the reader tests: two processes can seed from the same clock tick and one removes the file the other opens (E_FILE_OPEN_ERR), observed on the macOS Release job. --- cpp/test/reader/table_field_filter_test.cc | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/cpp/test/reader/table_field_filter_test.cc b/cpp/test/reader/table_field_filter_test.cc index 62b495a832..f2378b0fd8 100644 --- a/cpp/test/reader/table_field_filter_test.cc +++ b/cpp/test/reader/table_field_filter_test.cc @@ -20,7 +20,14 @@ #include +#include #include + +#ifdef _WIN32 +#include +#else +#include +#endif #include #include #include @@ -164,11 +171,20 @@ class TableFieldFilterTest saved_ = g_config_value_; g_config_value_.page_writer_max_point_num_ = 3; g_config_value_.parallel_read_enabled_ = std::get<0>(GetParam()); + // CTest runs cases in parallel processes; two can seed from the same + // clock tick and one then removes the file the other is opening. +#ifdef _WIN32 + const auto process_id = static_cast(_getpid()); +#else + const auto process_id = static_cast(getpid()); +#endif + static std::atomic filename_counter{0}; filename_ = "table_field_filter_" + std::to_string( std::chrono::steady_clock::now().time_since_epoch().count()) + - ".tsfile"; + "_" + std::to_string(process_id) + "_" + + std::to_string(filename_counter.fetch_add(1)) + ".tsfile"; WriteFile file; int flags = O_WRONLY | O_CREAT | O_TRUNC; #ifdef _WIN32 From efc71e26d3c5e514ecb2c4970571f76f2e0887e4 Mon Sep 17 00:00:00 2001 From: ColinLee Date: Fri, 13 Nov 2026 14:53:17 +0800 Subject: [PATCH 10/12] test(cpp): give clock-seeded test files unique names Several fixtures build their .tsfile name from a clock count alone. CTest runs cases in parallel processes, so two can read the same tick and then one removes the file the other is opening or reopening (E_ALREADY_EXIST / E_FILE_OPEN_ERR, seen as SEGFAULT on macOS). Append the process id and a per-process counter, the pattern the writer tests already use. Covers table_read_failure, table_field_filter, chunk_reader_resource, aligned_chunk_reader_bulk, aligned_field_filter_regression, masked_bulk_copy and timeseries_schema_codec. --- .../reader/aligned_chunk_reader_bulk_test.cc | 18 ++++++++++++++++++ .../aligned_field_filter_regression_test.cc | 17 ++++++++++++++++- cpp/test/reader/block/masked_bulk_copy_test.cc | 17 ++++++++++++++++- cpp/test/reader/chunk_reader_resource_test.cc | 17 +++++++++++++++++ cpp/test/reader/table_field_filter_test.cc | 17 ++++++++++++++++- cpp/test/reader/table_read_failure_test.cc | 17 ++++++++++++++++- .../reader/timeseries_schema_codec_test.cc | 17 +++++++++++++++++ 7 files changed, 116 insertions(+), 4 deletions(-) diff --git a/cpp/test/reader/aligned_chunk_reader_bulk_test.cc b/cpp/test/reader/aligned_chunk_reader_bulk_test.cc index 0be62def83..d3efd87967 100644 --- a/cpp/test/reader/aligned_chunk_reader_bulk_test.cc +++ b/cpp/test/reader/aligned_chunk_reader_bulk_test.cc @@ -20,6 +20,7 @@ #include #include +#include #include #include #include @@ -41,6 +42,12 @@ #include "writer/tsfile_table_writer.h" #ifdef ENABLE_THREADS #include "common/thread_pool.h" + +#ifdef _WIN32 +#include +#else +#include +#endif #endif using namespace common; @@ -93,6 +100,17 @@ class AlignedBulkCopyTest : public ::testing::Test { "_" + std::to_string( std::chrono::steady_clock::now().time_since_epoch().count()) + + "_" + std::to_string([] { +#ifdef _WIN32 + return static_cast(_getpid()); +#else + return static_cast(getpid()); +#endif + }()) + + "_" + std::to_string([] { + static std::atomic counter{0}; + return counter.fetch_add(1); + }()) + ".tsfile"; std::remove(path_.c_str()); { diff --git a/cpp/test/reader/aligned_field_filter_regression_test.cc b/cpp/test/reader/aligned_field_filter_regression_test.cc index c2b7506d1c..265a075aa4 100644 --- a/cpp/test/reader/aligned_field_filter_regression_test.cc +++ b/cpp/test/reader/aligned_field_filter_regression_test.cc @@ -19,6 +19,7 @@ #include #include +#include #include #include #include @@ -37,6 +38,12 @@ #include "reader/table_query_executor.h" #include "writer/tsfile_table_writer.h" +#ifdef _WIN32 +#include +#else +#include +#endif + namespace { using namespace common; using namespace storage; @@ -70,11 +77,19 @@ class AlignedFieldFilterRegressionTest saved_ = g_config_value_; g_config_value_.page_writer_max_point_num_ = 3; g_config_value_.parallel_read_enabled_ = std::get<0>(GetParam()); + // Parallel CTest processes can seed from the same clock tick. +#ifdef _WIN32 + const auto process_id = static_cast(_getpid()); +#else + const auto process_id = static_cast(getpid()); +#endif + static std::atomic filename_counter{0}; filename_ = "aligned_field_filter_regression_" + std::to_string( std::chrono::steady_clock::now().time_since_epoch().count()) + - ".tsfile"; + "_" + std::to_string(process_id) + "_" + + std::to_string(filename_counter.fetch_add(1)) + ".tsfile"; } void TearDown() override { std::remove(filename_.c_str()); diff --git a/cpp/test/reader/block/masked_bulk_copy_test.cc b/cpp/test/reader/block/masked_bulk_copy_test.cc index 21a9e1b50c..d2a467a314 100644 --- a/cpp/test/reader/block/masked_bulk_copy_test.cc +++ b/cpp/test/reader/block/masked_bulk_copy_test.cc @@ -19,6 +19,7 @@ #include #include +#include #include #include #include @@ -33,6 +34,12 @@ #include "reader/block/single_device_tsblock_reader.h" #include "writer/tsfile_table_writer.h" +#ifdef _WIN32 +#include +#else +#include +#endif + namespace { using namespace common; using namespace storage; @@ -47,11 +54,19 @@ class MaskedBulkCopyTest : public ::testing::TestWithParam { // Source tuple: time + v + n + s + hidden + empty = 72 bytes/row. // Each source TsBlock has 137 rows, crossing two bitmap words. g_config_value_.tsblock_max_memory_ = 72 * 137; + // Parallel CTest processes can seed from the same clock tick. +#ifdef _WIN32 + const auto process_id = static_cast(_getpid()); +#else + const auto process_id = static_cast(getpid()); +#endif + static std::atomic filename_counter{0}; filename_ = "masked_bulk_copy_" + std::to_string( std::chrono::steady_clock::now().time_since_epoch().count()) + - ".tsfile"; + "_" + std::to_string(process_id) + "_" + + std::to_string(filename_counter.fetch_add(1)) + ".tsfile"; const std::vector columns{ {"v", INT64, UNCOMPRESSED, PLAIN, ColumnCategory::FIELD}, {"n", INT64, UNCOMPRESSED, PLAIN, ColumnCategory::FIELD}, diff --git a/cpp/test/reader/chunk_reader_resource_test.cc b/cpp/test/reader/chunk_reader_resource_test.cc index 7e0a0af734..159db638c5 100644 --- a/cpp/test/reader/chunk_reader_resource_test.cc +++ b/cpp/test/reader/chunk_reader_resource_test.cc @@ -41,6 +41,12 @@ #include "writer/tsfile_table_writer.h" #ifdef ENABLE_THREADS #include "common/thread_pool.h" + +#ifdef _WIN32 +#include +#else +#include +#endif #endif namespace storage { @@ -180,6 +186,17 @@ class AlignedReaderFailureTest : public ::testing::Test { "_" + std::to_string( std::chrono::steady_clock::now().time_since_epoch().count()) + + "_" + std::to_string([] { +#ifdef _WIN32 + return static_cast(_getpid()); +#else + return static_cast(getpid()); +#endif + }()) + + "_" + std::to_string([] { + static std::atomic counter{0}; + return counter.fetch_add(1); + }()) + ".tsfile"; WriteFile file; int flags = O_WRONLY | O_CREAT | O_TRUNC; diff --git a/cpp/test/reader/table_field_filter_test.cc b/cpp/test/reader/table_field_filter_test.cc index 62b495a832..a7c9aebc07 100644 --- a/cpp/test/reader/table_field_filter_test.cc +++ b/cpp/test/reader/table_field_filter_test.cc @@ -20,6 +20,7 @@ #include +#include #include #include #include @@ -36,6 +37,12 @@ #include "reader/table_query_executor.h" #include "writer/tsfile_table_writer.h" +#ifdef _WIN32 +#include +#else +#include +#endif + namespace { using namespace common; using namespace storage; @@ -164,11 +171,19 @@ class TableFieldFilterTest saved_ = g_config_value_; g_config_value_.page_writer_max_point_num_ = 3; g_config_value_.parallel_read_enabled_ = std::get<0>(GetParam()); + // Parallel CTest processes can seed from the same clock tick. +#ifdef _WIN32 + const auto process_id = static_cast(_getpid()); +#else + const auto process_id = static_cast(getpid()); +#endif + static std::atomic filename_counter{0}; filename_ = "table_field_filter_" + std::to_string( std::chrono::steady_clock::now().time_since_epoch().count()) + - ".tsfile"; + "_" + std::to_string(process_id) + "_" + + std::to_string(filename_counter.fetch_add(1)) + ".tsfile"; WriteFile file; int flags = O_WRONLY | O_CREAT | O_TRUNC; #ifdef _WIN32 diff --git a/cpp/test/reader/table_read_failure_test.cc b/cpp/test/reader/table_read_failure_test.cc index 9e8076612f..99d02669be 100644 --- a/cpp/test/reader/table_read_failure_test.cc +++ b/cpp/test/reader/table_read_failure_test.cc @@ -19,6 +19,7 @@ #include #include +#include #include #include #include @@ -42,6 +43,12 @@ #include "utils/injection.h" #include "writer/tsfile_table_writer.h" +#ifdef _WIN32 +#include +#else +#include +#endif + namespace { class InjectionGuard { @@ -106,11 +113,19 @@ class TableReadFailureTest storage::libtsfile_init(); saved_index_degree_ = common::g_config_value_.max_degree_of_index_node_; ASSERT_EQ(storage::set_max_degree_of_index_node(2), common::E_OK); + // Parallel CTest processes can seed from the same clock tick. +#ifdef _WIN32 + const auto process_id = static_cast(_getpid()); +#else + const auto process_id = static_cast(getpid()); +#endif + static std::atomic filename_counter{0}; filename_ = "table_read_failure_" + std::to_string( std::chrono::steady_clock::now().time_since_epoch().count()) + - ".tsfile"; + "_" + std::to_string(process_id) + "_" + + std::to_string(filename_counter.fetch_add(1)) + ".tsfile"; storage::WriteFile file; int flags = O_WRONLY | O_CREAT | O_TRUNC; #ifdef _WIN32 diff --git a/cpp/test/reader/timeseries_schema_codec_test.cc b/cpp/test/reader/timeseries_schema_codec_test.cc index cebb515411..613dda28ab 100644 --- a/cpp/test/reader/timeseries_schema_codec_test.cc +++ b/cpp/test/reader/timeseries_schema_codec_test.cc @@ -19,6 +19,7 @@ #include +#include #include #include #include @@ -31,6 +32,12 @@ #include "reader/tsfile_reader.h" #include "writer/tsfile_writer.h" +#ifdef _WIN32 +#include +#else +#include +#endif + namespace storage { // get_timeseries_schema() reports the encoding and the compressor of every @@ -77,6 +84,16 @@ class TimeseriesSchemaCodecTest : public ::testing::Test { for (int i = 0; i < length; ++i) { s += chars[static_cast(dis(gen))]; } + // CTest runs cases in parallel processes; two can seed from the same + // clock tick and then collide on the file name. +#ifdef _WIN32 + const auto process_id = static_cast(_getpid()); +#else + const auto process_id = static_cast(getpid()); +#endif + static std::atomic filename_counter{0}; + s += "_" + std::to_string(process_id) + "_" + + std::to_string(filename_counter.fetch_add(1)); return s; } From 3f8078b253436e44e7b89b5c7ec1fcbce99ef829 Mon Sep 17 00:00:00 2001 From: ColinLee Date: Sat, 14 Nov 2026 15:35:53 +0800 Subject: [PATCH 11/12] test(cpp): keep the pid includes outside ENABLE_THREADS getpid()/_getpid() are used in the fixtures unconditionally, but the process.h/unistd.h block landed inside the ENABLE_THREADS guard in aligned_chunk_reader_bulk_test and chunk_reader_resource_test, which would break a build with threads disabled. --- cpp/test/reader/aligned_chunk_reader_bulk_test.cc | 12 ++++++------ cpp/test/reader/chunk_reader_resource_test.cc | 12 ++++++------ 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/cpp/test/reader/aligned_chunk_reader_bulk_test.cc b/cpp/test/reader/aligned_chunk_reader_bulk_test.cc index d3efd87967..c5c5d22917 100644 --- a/cpp/test/reader/aligned_chunk_reader_bulk_test.cc +++ b/cpp/test/reader/aligned_chunk_reader_bulk_test.cc @@ -29,6 +29,12 @@ #include #include +#ifdef _WIN32 +#include +#else +#include +#endif + #include "common/allocator/alloc_base.h" #include "common/config/config.h" #include "common/tablet.h" @@ -42,12 +48,6 @@ #include "writer/tsfile_table_writer.h" #ifdef ENABLE_THREADS #include "common/thread_pool.h" - -#ifdef _WIN32 -#include -#else -#include -#endif #endif using namespace common; diff --git a/cpp/test/reader/chunk_reader_resource_test.cc b/cpp/test/reader/chunk_reader_resource_test.cc index 159db638c5..3da4d840da 100644 --- a/cpp/test/reader/chunk_reader_resource_test.cc +++ b/cpp/test/reader/chunk_reader_resource_test.cc @@ -29,6 +29,12 @@ #include #include +#ifdef _WIN32 +#include +#else +#include +#endif + #include "common/allocator/alloc_base.h" #include "common/config/config.h" #include "common/tablet.h" @@ -41,12 +47,6 @@ #include "writer/tsfile_table_writer.h" #ifdef ENABLE_THREADS #include "common/thread_pool.h" - -#ifdef _WIN32 -#include -#else -#include -#endif #endif namespace storage { From e158376684c0de292c92fc4cd555c4505e5b9ab6 Mon Sep 17 00:00:00 2001 From: ColinLee Date: Sat, 14 Nov 2026 15:40:06 +0800 Subject: [PATCH 12/12] fix(cpp): guard a zero element size and propagate fill errors Reject elem_size == 0 before dividing, and let the variable-length vector return the underlying error code (E_INVALID_ARG vs E_OOM) instead of always reporting E_OOM. --- cpp/src/common/container/byte_buffer.h | 3 +++ cpp/src/common/tsblock/vector/variable_length_vector.h | 5 +++-- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/cpp/src/common/container/byte_buffer.h b/cpp/src/common/container/byte_buffer.h index 90e7900963..fc5e9f99e8 100644 --- a/cpp/src/common/container/byte_buffer.h +++ b/cpp/src/common/container/byte_buffer.h @@ -151,6 +151,9 @@ class ByteBuffer { } const uint64_t elem_size = static_cast(variable_type_len_) + len; + if (elem_size == 0) { + return E_INVALID_ARG; + } // elem_size * count can overflow uint64_t (both factors are 32-bit), // which would wrap `need` to a small value and bypass the capacity // checks before a huge payload copy. Reject anything that cannot fit diff --git a/cpp/src/common/tsblock/vector/variable_length_vector.h b/cpp/src/common/tsblock/vector/variable_length_vector.h index 0e6d661c3a..95800ba0c3 100644 --- a/cpp/src/common/tsblock/vector/variable_length_vector.h +++ b/cpp/src/common/tsblock/vector/variable_length_vector.h @@ -66,8 +66,9 @@ class VariableLengthVector : public Vector { if (count == 0) { return common::E_OK; } - if (values_.fill_repeated_variable(value, len, count) != common::E_OK) { - return common::E_OOM; + const int ret = values_.fill_repeated_variable(value, len, count); + if (ret != common::E_OK) { + return ret; } add_row_nums(count); return common::E_OK;