From 2c7a2b6434ce7cbc0975ca5ad5dbb8c2e4e2d6b2 Mon Sep 17 00:00:00 2001 From: ColinLee Date: Thu, 12 Nov 2026 16:15:10 +0800 Subject: [PATCH 1/7] perf(cpp): decode each time page once and skip unbounded time filters Two related fixes on the aligned chunk reader: * build_page_plan() decodes a page's timestamps to classify it, and decode_page_lazy() then decoded the same page again. For single-page chunks the times kept by the planner are exactly what the lazy path needs, so hand them over instead of reading and decoding the page twice. * When the time predicate accepts every timestamp (TimeBetween over the full int64 range, which is what a query without a time bound uses), the planner no longer evaluates the predicate row by row; it knows the whole page matches and only has to apply OFFSET. Filter gains is_unbounded_time_filter(), false by default and overridden by TimeBetween. 3.2M-row scan: 0.069s -> 0.064s (~6%). --- cpp/src/reader/aligned_chunk_reader.cc | 68 +++++++++++++++++++------- cpp/src/reader/filter/filter.h | 4 ++ cpp/src/reader/filter/time_operator.h | 4 ++ 3 files changed, 59 insertions(+), 17 deletions(-) diff --git a/cpp/src/reader/aligned_chunk_reader.cc b/cpp/src/reader/aligned_chunk_reader.cc index a0a0b23069..a101b5e924 100644 --- a/cpp/src/reader/aligned_chunk_reader.cc +++ b/cpp/src/reader/aligned_chunk_reader.cc @@ -1531,6 +1531,10 @@ int AlignedChunkReader::build_page_plan(Filter* filter, int& row_offset) { } const uint32_t num_cols = value_columns_.size(); + // For single-page chunks the times decoded while classifying the page are + // exactly the ones the lazy decode path would decode again; keep them so + // each page is read and decoded once. + std::vector single_page_times; while (IS_SUCC(ret)) { if (time_chunk_visit_offset_ - time_chunk_header_.serialized_size_ >= time_chunk_header_.data_size_) { @@ -1548,6 +1552,9 @@ int AlignedChunkReader::build_page_plan(Filter* filter, int& row_offset) { } ChunkPageInfo page_info; + // Times decoded below for classification are handed to the lazy + // decode path for single-page chunks, so the page is decoded once. + std::vector decoded_page_times; page_info.time_file_offset = time_chunk_meta_->offset_of_chunk_header_ + time_chunk_visit_offset_; page_info.time_compressed_size = cur_time_page_header_.compressed_size_; @@ -1619,24 +1626,40 @@ int AlignedChunkReader::build_page_plan(Filter* filter, int& row_offset) { page_info.row_begin = 0; page_info.row_end = stat != nullptr ? stat->count_ : 0; } else { - std::vector times; - if (RET_FAIL(decode_time_page_direct(page_info, times))) { + if (RET_FAIL( + decode_time_page_direct(page_info, decoded_page_times))) { break; } int32_t first = -1; int32_t last = -1; - for (int32_t i = 0; i < static_cast(times.size()); - i++) { - if (filter->satisfy_start_end_time(times[i], times[i])) { - // Offset is defined on the filtered row stream. Consume - // matching rows only; holes inside a boundary page do - // not count toward it. - if (field_filter_ == nullptr && row_offset > 0) { - row_offset--; - continue; + const int32_t n = + static_cast(decoded_page_times.size()); + if (filter->is_unbounded_time_filter()) { + // Every timestamp matches; skip the per-row predicate. + // OFFSET still consumes rows from this page. + int32_t begin = 0; + if (field_filter_ == nullptr && row_offset > 0) { + begin = std::min(n, row_offset); + row_offset -= begin; + } + if (begin < n) { + first = begin; + last = n - 1; + } + } else { + for (int32_t i = 0; i < n; i++) { + if (filter->satisfy_start_end_time( + decoded_page_times[i], decoded_page_times[i])) { + // Offset is defined on the filtered row stream. + // Consume matching rows only; holes inside a + // boundary page do not count toward it. + if (field_filter_ == nullptr && row_offset > 0) { + row_offset--; + continue; + } + if (first < 0) first = i; + last = i; } - if (first < 0) first = i; - last = i; } } if (first >= 0) { @@ -1650,11 +1673,12 @@ int AlignedChunkReader::build_page_plan(Filter* filter, int& row_offset) { if (page_info.pass_type != PagePassType::SKIP) { if (page_info.row_end == 0) { - std::vector times; - if (RET_FAIL(decode_time_page_direct(page_info, times))) { + if (RET_FAIL( + decode_time_page_direct(page_info, decoded_page_times))) { break; } - page_info.row_end = static_cast(times.size()); + page_info.row_end = + static_cast(decoded_page_times.size()); } if (page_info.pass_type == PagePassType::FULL_PASS && row_offset > 0) { @@ -1670,6 +1694,9 @@ int AlignedChunkReader::build_page_plan(Filter* filter, int& row_offset) { } if (page_info.row_begin < page_info.row_end) { chunk_pages_.push_back(std::move(page_info)); + if (chunk_pages_.size() == 1 && !decoded_page_times.empty()) { + single_page_times = std::move(decoded_page_times); + } } } @@ -1688,6 +1715,9 @@ int AlignedChunkReader::build_page_plan(Filter* filter, int& row_offset) { if (page_plan_built_) { per_page_times_.assign(chunk_pages_.size(), std::vector{}); + if (chunk_pages_.size() == 1 && !single_page_times.empty()) { + per_page_times_[0] = std::move(single_page_times); + } for (auto* col : value_columns_) { col->per_page_state.clear(); col->per_page_state.resize(chunk_pages_.size()); @@ -2010,7 +2040,11 @@ int AlignedChunkReader::decode_all_planned_pages() { // matching row. int AlignedChunkReader::decode_page_lazy(size_t page_idx) { int ret = E_OK; - if (RET_FAIL(decode_time_page_direct(chunk_pages_[page_idx], + // build_page_plan() already decoded (and kept) this page's times for + // single-page chunks; decoding again would re-read and re-decode the same + // page. + if (per_page_times_[page_idx].empty() && + RET_FAIL(decode_time_page_direct(chunk_pages_[page_idx], per_page_times_[page_idx]))) { return ret; } diff --git a/cpp/src/reader/filter/filter.h b/cpp/src/reader/filter/filter.h index e539923086..0e2f8c3807 100644 --- a/cpp/src/reader/filter/filter.h +++ b/cpp/src/reader/filter/filter.h @@ -64,6 +64,10 @@ class Filter { return nullptr; } + // True when the predicate accepts every timestamp (e.g. TimeBetween over + // the full int64 range). Callers may then skip per-row evaluation. + virtual bool is_unbounded_time_filter() const { return false; } + // Batch time filter: evaluate time filter on an array of timestamps. // Writes true/false into @mask for each element. // Returns the number of elements that passed (mask[i] == true). diff --git a/cpp/src/reader/filter/time_operator.h b/cpp/src/reader/filter/time_operator.h index f972a42598..5e51f5131c 100644 --- a/cpp/src/reader/filter/time_operator.h +++ b/cpp/src/reader/filter/time_operator.h @@ -48,6 +48,10 @@ class TimeBetween : public Filter { std::vector* get_time_ranges(); + bool is_unbounded_time_filter() const override { + return !not_ && value1_ == INT64_MIN && value2_ == INT64_MAX; + } + int satisfy_batch_time(const int64_t* times, int count, bool* mask); FilterType get_filter_type() { return type_; } From da8a637cf5f9250c96c1616524328d7f0f8b0e49 Mon Sep 17 00:00:00 2001 From: ColinLee Date: Thu, 12 Nov 2026 16:29:56 +0800 Subject: [PATCH 2/7] style: apply spotless formatting --- cpp/src/reader/aligned_chunk_reader.cc | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/cpp/src/reader/aligned_chunk_reader.cc b/cpp/src/reader/aligned_chunk_reader.cc index a101b5e924..eb65933216 100644 --- a/cpp/src/reader/aligned_chunk_reader.cc +++ b/cpp/src/reader/aligned_chunk_reader.cc @@ -1626,8 +1626,8 @@ int AlignedChunkReader::build_page_plan(Filter* filter, int& row_offset) { page_info.row_begin = 0; page_info.row_end = stat != nullptr ? stat->count_ : 0; } else { - if (RET_FAIL( - decode_time_page_direct(page_info, decoded_page_times))) { + if (RET_FAIL(decode_time_page_direct(page_info, + decoded_page_times))) { break; } int32_t first = -1; @@ -1673,8 +1673,8 @@ int AlignedChunkReader::build_page_plan(Filter* filter, int& row_offset) { if (page_info.pass_type != PagePassType::SKIP) { if (page_info.row_end == 0) { - if (RET_FAIL( - decode_time_page_direct(page_info, decoded_page_times))) { + if (RET_FAIL(decode_time_page_direct(page_info, + decoded_page_times))) { break; } page_info.row_end = From a6dabc21b649855af652e4ca995fdf759cd3f7cc Mon Sep 17 00:00:00 2001 From: ColinLee Date: Thu, 12 Nov 2026 16:53:08 +0800 Subject: [PATCH 3/7] test(cpp): cover unbounded time filters and offset handling Add regression tests for the chunk-reader changes: * UnboundedTimeFilterReturnsAllRows drives the table-reader path with a TimeBetween over the full int64 range, checking that every row comes back and that OFFSET/LIMIT still apply, in serial and parallel mode. * MultiPreparedUnboundedTimeFilterWithOffset uses the prepared multi-column path, which pushes OFFSET into the page plan; the written chunks have a single statistic-less page, so the planner reaches the BOUNDARY branch where the unbounded shortcut replaces the per-row time predicate. It also covers a bounded window and an out-of-range query. --- cpp/test/reader/prepared_series_test.cc | 161 ++++++++++++++++++ .../table_view/tsfile_reader_table_test.cc | 101 +++++++++++ 2 files changed, 262 insertions(+) diff --git a/cpp/test/reader/prepared_series_test.cc b/cpp/test/reader/prepared_series_test.cc index d90101fd9a..04ad57e137 100644 --- a/cpp/test/reader/prepared_series_test.cc +++ b/cpp/test/reader/prepared_series_test.cc @@ -24,6 +24,7 @@ #include #include +#include #include #include "common/global.h" @@ -560,5 +561,165 @@ TEST_F(PreparedSeriesBatchTest, EXPECT_EQ(common::E_OK, reader.close()); } +// A prepared multi-column query without a time bound uses a TimeBetween +// spanning the full int64 range. The chunk reader recognises it while +// building the page plan (the prepared path pushes OFFSET down), returns every +// row of a boundary page without evaluating the predicate row by row, and +// still consumes OFFSET from the retained rows. +TEST_F(PreparedSeriesBatchTest, MultiPreparedUnboundedTimeFilterWithOffset) { + constexpr int kRows = 30; + constexpr int kRowsPerChunk = 10; + // The page limit keeps every chunk to a single page, which carries no + // page statistics; such pages are BOUNDARY pages, the case where the + // unbounded shortcut has to skip the per-row predicate itself. + PagePointGuard guard(kRowsPerChunk); + { + WriteFile write_file; + int flags = O_WRONLY | O_CREAT | O_TRUNC; +#ifdef _WIN32 + flags |= O_BINARY; +#endif + ASSERT_EQ(common::E_OK, write_file.create(file_name_, flags, 0666)); + std::vector columns = { + common::ColumnSchema("device", common::STRING, + common::ColumnCategory::TAG), + common::ColumnSchema("value", common::DOUBLE, + common::ColumnCategory::FIELD), + common::ColumnSchema("value2", common::DOUBLE, + common::ColumnCategory::FIELD), + }; + TableSchema schema("weather", columns); + TsFileTableWriter writer(&write_file, &schema); + for (int chunk = 0; chunk < kRows / kRowsPerChunk; ++chunk) { + Tablet tablet( + "weather", {"device", "value", "value2"}, + {common::STRING, common::DOUBLE, common::DOUBLE}, + {common::ColumnCategory::TAG, common::ColumnCategory::FIELD, + common::ColumnCategory::FIELD}, + kRowsPerChunk); + for (int row = 0; row < kRowsPerChunk; ++row) { + const int time = chunk * kRowsPerChunk + row; + ASSERT_EQ(common::E_OK, tablet.add_timestamp(row, time)); + ASSERT_EQ(common::E_OK, tablet.add_value(row, "device", "d0")); + ASSERT_EQ( + common::E_OK, + tablet.add_value(row, "value", static_cast(time))); + ASSERT_EQ(common::E_OK, + tablet.add_value(row, "value2", + static_cast(time * 10))); + } + ASSERT_EQ(common::E_OK, writer.write_table(tablet)); + ASSERT_EQ(common::E_OK, writer.flush()); + } + ASSERT_EQ(common::E_OK, writer.close()); + } + + TsFileReader reader; + ASSERT_EQ(common::E_OK, reader.open(file_name_)); + std::shared_ptr prepared_value; + std::shared_ptr prepared_value2; + auto metadata = reader.get_timeseries_metadata(); + FileGeneration generation; + generation.mapped_index_identity = 1; + generation.file_id = 0; + struct stat file_stat {}; + ASSERT_EQ(0, stat(file_name_.c_str(), &file_stat)); + generation.file_size = static_cast(file_stat.st_size); + for (const auto& device_entry : metadata) { + for (const auto& index : device_entry.second) { + auto* aligned = dynamic_cast(index.get()); + if (aligned == nullptr || aligned->value_ts_idx_ == nullptr) { + continue; + } + const std::string name = + aligned->value_ts_idx_->get_measurement_name().to_std_string(); + PreparedLocator locator; + locator.locator_id = name == "value" ? 0 : 1; + locator.layout = 1; + locator.flags = 1; + locator.value_metadata_offset = + aligned->value_ts_idx_->get_metadata_offset(); + locator.value_metadata_length = + aligned->value_ts_idx_->get_metadata_length(); + locator.time_metadata_offset = + aligned->time_ts_idx_->get_metadata_offset(); + locator.time_metadata_length = + aligned->time_ts_idx_->get_metadata_length(); + if (name == "value") { + ASSERT_EQ( + common::E_OK, + reader.prepare_series(generation, locator, prepared_value)); + } else if (name == "value2") { + ASSERT_EQ(common::E_OK, + reader.prepare_series(generation, locator, + prepared_value2)); + } + } + } + ASSERT_NE(nullptr, prepared_value); + ASSERT_NE(nullptr, prepared_value2); + + auto collect = [&](int64_t start_time, int64_t end_time, int offset, + int limit) { + ResultSet* result = nullptr; + EXPECT_EQ(common::E_OK, + reader.query_prepared_multi({prepared_value2, prepared_value}, + start_time, end_time, offset, + limit, result)); + auto* table_result = dynamic_cast(result); + EXPECT_NE(nullptr, table_result); + std::vector times; + common::TsBlock* block = nullptr; + while (table_result->get_next_tsblock(block) == common::E_OK) { + common::RowIterator rows(block); + while (rows.has_next()) { + uint32_t len = 0; + bool is_null = false; + const char* timestamp = rows.read(0, &len, &is_null); + EXPECT_FALSE(is_null); + times.push_back(*reinterpret_cast(timestamp)); + const char* value2 = rows.read(1, &len, &is_null); + EXPECT_FALSE(is_null); + EXPECT_DOUBLE_EQ(times.back() * 10.0, + *reinterpret_cast(value2)); + const char* value = rows.read(2, &len, &is_null); + EXPECT_FALSE(is_null); + EXPECT_DOUBLE_EQ(static_cast(times.back()), + *reinterpret_cast(value)); + rows.next(); + } + block = nullptr; + } + reader.destroy_query_data_set(result); + return times; + }; + + std::vector all_rows; + for (int row = 0; row < kRows; ++row) all_rows.push_back(row); + + const int64_t min_time = std::numeric_limits::min(); + const int64_t max_time = std::numeric_limits::max(); + EXPECT_EQ(collect(min_time, max_time, 0, -1), all_rows); + for (int offset : {1, 3, 4, 5, 17, 28, 29, 30}) { + SCOPED_TRACE(::testing::Message() << "offset=" << offset); + const std::vector expected(all_rows.begin() + offset, + all_rows.end()); + EXPECT_EQ(collect(min_time, max_time, offset, -1), expected); + } + + std::vector in_window; + for (int row = 5; row <= 22; ++row) in_window.push_back(row); + EXPECT_EQ(collect(5, 22, 0, -1), in_window); + const std::vector offset_in_window(in_window.begin() + 6, + in_window.begin() + 10); + EXPECT_EQ(collect(5, 22, 6, 4), offset_in_window); + const std::vector tail_in_window(in_window.begin() + 10, + in_window.end()); + EXPECT_EQ(collect(5, 22, 10, -1), tail_in_window); + EXPECT_TRUE(collect(100, 200, 0, -1).empty()); + + EXPECT_EQ(common::E_OK, reader.close()); +} + } // namespace } // namespace storage diff --git a/cpp/test/reader/table_view/tsfile_reader_table_test.cc b/cpp/test/reader/table_view/tsfile_reader_table_test.cc index 07be3cd919..6a3db128a6 100644 --- a/cpp/test/reader/table_view/tsfile_reader_table_test.cc +++ b/cpp/test/reader/table_view/tsfile_reader_table_test.cc @@ -18,6 +18,7 @@ */ #include +#include #include #include #include @@ -352,6 +353,106 @@ TEST_P(TsFileTableReaderTimeFilterTest, QueryContinuesAfterEmptyPageInChunk) { } } +// A query without a time bound is a TimeBetween spanning the full int64 +// range. The chunk reader recognises it, returns every row of a boundary +// page without evaluating the predicate row by row, and still applies OFFSET +// to the filtered, device-ordered row stream. +TEST_P(TsFileTableReaderTimeFilterTest, UnboundedTimeFilterReturnsAllRows) { + std::unique_ptr table_schema(gen_table_schema(0)); + TsFileTableWriter writer(&write_file_, table_schema.get()); + constexpr int kDevices = 2; + constexpr int kRowsPerDevice = 10; + constexpr int kChunks = 3; + for (int chunk = 0; chunk < kChunks; ++chunk) { + Tablet tablet(table_schema->get_table_name(), + table_schema->get_measurement_names(), + table_schema->get_data_types(), + table_schema->get_column_categories(), + kDevices * kRowsPerDevice); + int index = 0; + for (int device = 0; device < kDevices; ++device) { + const std::string tag = "d" + std::to_string(device); + for (int row = 0; row < kRowsPerDevice; ++row) { + const int64_t time = chunk * kRowsPerDevice + row; + ASSERT_EQ(tablet.add_timestamp(index, time), common::E_OK); + for (int c = 0; c < 5; ++c) { + const std::string name = "id" + std::to_string(c); + ASSERT_EQ(tablet.add_value(index, name, tag.c_str()), + common::E_OK); + } + for (int c = 0; c < 5; ++c) { + const std::string name = "s" + std::to_string(c); + ASSERT_EQ(tablet.add_value(index, name, int64_t(device)), + common::E_OK); + } + ++index; + } + } + ASSERT_EQ(writer.write_table(tablet), common::E_OK); + // Each flush ends a chunk; with the page limit above every chunk + // holds a single page, so the planner's decoded times are reused by + // the lazy decode instead of decoding the page twice. + ASSERT_EQ(writer.flush(), common::E_OK); + } + ASSERT_EQ(writer.close(), common::E_OK); + + TsFileReader reader; + ASSERT_EQ(reader.open(file_name_), common::E_OK); + + std::vector> all_rows; + for (int device = 0; device < kDevices; ++device) { + for (int chunk = 0; chunk < kChunks; ++chunk) { + for (int row = 0; row < kRowsPerDevice; ++row) { + all_rows.emplace_back(chunk * kRowsPerDevice + row, device); + } + } + } + + // Selecting several value columns uses the aligned multi-value reader, + // which is where the page planner and the unbounded shortcut live. + const std::vector query_columns{"s0", "s1", "s2", "s3", "s4"}; + auto collect = [&](int64_t start_time, int64_t end_time, int offset, + int limit) { + ResultSet* result = nullptr; + EXPECT_EQ(reader.query(table_schema->get_table_name(), query_columns, + start_time, end_time, offset, limit, result), + common::E_OK); + std::vector> rows; + bool has_next = false; + int ret = common::E_OK; + while ((ret = result->next(has_next)) == common::E_OK && has_next) { + const int64_t device = result->get_value("s0"); + for (const auto& name : query_columns) { + EXPECT_EQ(result->get_value(name), device); + } + rows.emplace_back(result->get_value(1), device); + } + EXPECT_EQ(ret, common::E_OK); + reader.destroy_query_data_set(result); + return rows; + }; + + const int64_t min_time = std::numeric_limits::min(); + const int64_t max_time = std::numeric_limits::max(); + EXPECT_EQ(collect(min_time, max_time, 0, -1), all_rows); + const std::vector> offset_window( + all_rows.begin() + 7, all_rows.begin() + 15); + EXPECT_EQ(collect(min_time, max_time, 7, 8), offset_window); + + std::vector> in_window; + for (const auto& row : all_rows) { + if (row.first >= 5 && row.first <= 14) in_window.push_back(row); + } + EXPECT_EQ(collect(5, 14, 0, -1), in_window); + const std::vector> offset_in_window( + in_window.begin() + 3, in_window.begin() + 7); + EXPECT_EQ(collect(5, 14, 3, 4), offset_in_window); + + EXPECT_TRUE(collect(1000, 2000, 0, -1).empty()); + + ASSERT_EQ(reader.close(), common::E_OK); +} + // Regression for #1038: a page whose rows are all rejected by the time filter // must not end the series. Requesting the never-written measurement "ghost" // (and omitting "v" on one device) forces the per-column aligned read path, From 369171d3fcb0b5d9e9740a6c7886663ec8c9b655 Mon Sep 17 00:00:00 2001 From: ColinLee Date: Thu, 12 Nov 2026 17:48:43 +0800 Subject: [PATCH 4/7] perf(cpp): finish the unbounded-time-filter fast path Review follow-ups on the page-plan work: * An unbounded time predicate made a page a BOUNDARY page, so the read loop still called satisfy_start_end_time() once per row and lost the bulk-copy path. Without a FIELD filter such a page is now marked FULL_PASS and OFFSET is consumed by the shared FULL_PASS handling; the duplicated OFFSET logic in the unbounded branch is gone. With a FIELD filter the page stays a BOUNDARY page (the rows depend on the mask, and OFFSET is applied after the mask by the block reader). * The parallel page-plan workers re-decoded every time page, undoing the planner's reuse exactly in the default (parallel) configuration. Skip slots the planner already filled. Tests: * TimeBetween::is_unbounded_time_filter() only for a non-negated full-range TimeBetween (negated/inverted/half-open/narrow/TimeIn are false). * NOT BETWEEN over the full range returns no rows. * The prepared multi-column test now runs its whole matrix with the parallel path on and off; with the pool off it reaches decode_page_lazy(), where the planner's times are handed over (probe: every lazy decode used prefilled times), and with the pool on every worker slot is served from the planner's times. * Guard the new collect lambdas against a null result set. --- cpp/src/reader/aligned_chunk_reader.cc | 27 ++++++---- cpp/src/reader/filter/time_operator.h | 3 +- cpp/test/reader/filter/time_in_filter_test.cc | 29 ++++++++++ cpp/test/reader/prepared_series_test.cc | 54 +++++++++++++------ .../table_view/tsfile_reader_table_test.cc | 39 ++++++++++++++ 5 files changed, 125 insertions(+), 27 deletions(-) diff --git a/cpp/src/reader/aligned_chunk_reader.cc b/cpp/src/reader/aligned_chunk_reader.cc index eb65933216..4952dfbdba 100644 --- a/cpp/src/reader/aligned_chunk_reader.cc +++ b/cpp/src/reader/aligned_chunk_reader.cc @@ -1635,16 +1635,19 @@ int AlignedChunkReader::build_page_plan(Filter* filter, int& row_offset) { const int32_t n = static_cast(decoded_page_times.size()); if (filter->is_unbounded_time_filter()) { - // Every timestamp matches; skip the per-row predicate. - // OFFSET still consumes rows from this page. - int32_t begin = 0; - if (field_filter_ == nullptr && row_offset > 0) { - begin = std::min(n, row_offset); - row_offset -= begin; - } - if (begin < n) { - first = begin; + // Every timestamp matches, so the per-row predicate can + // be skipped. Without a FIELD filter the whole page is + // passed and takes the bulk-copy path; the shared + // FULL_PASS handling below consumes OFFSET. With a FIELD + // filter the exact rows depend on statistics that are not + // available here, so the page stays a boundary page and + // the mask decides (OFFSET is applied after the mask). + if (n > 0) { + first = 0; last = n - 1; + if (field_filter_ == nullptr) { + page_info.pass_type = PagePassType::FULL_PASS; + } } } else { for (int32_t i = 0; i < n; i++) { @@ -2009,6 +2012,12 @@ int AlignedChunkReader::decode_all_planned_pages() { Compressor* comp = time_compressor_pool_[k]; for (size_t p = k; p < chunk_pages_.size(); p += time_task_count) { + // The planner may already have decoded this page to + // classify it (single-page plans prefill slot 0); + // decoding it again would undo that reuse. + if (!per_page_times_[p].empty()) { + continue; + } int r = decode_time_page_with( chunk_pages_[p], per_page_times_[p], dec, comp); if (IS_FAIL(r)) { diff --git a/cpp/src/reader/filter/time_operator.h b/cpp/src/reader/filter/time_operator.h index 5e51f5131c..6bff1122f8 100644 --- a/cpp/src/reader/filter/time_operator.h +++ b/cpp/src/reader/filter/time_operator.h @@ -49,7 +49,8 @@ class TimeBetween : public Filter { std::vector* get_time_ranges(); bool is_unbounded_time_filter() const override { - return !not_ && value1_ == INT64_MIN && value2_ == INT64_MAX; + return !not_ && value1_ == std::numeric_limits::min() && + value2_ == std::numeric_limits::max(); } int satisfy_batch_time(const int64_t* times, int count, bool* mask); diff --git a/cpp/test/reader/filter/time_in_filter_test.cc b/cpp/test/reader/filter/time_in_filter_test.cc index 9eceaaaa54..fede7827c6 100644 --- a/cpp/test/reader/filter/time_in_filter_test.cc +++ b/cpp/test/reader/filter/time_in_filter_test.cc @@ -82,3 +82,32 @@ TEST(TimeInFilterTest, BatchTimeFallbackUsesScalarSemantics) { EXPECT_TRUE(mask[4]); EXPECT_FALSE(mask[5]); } + +// The aligned chunk reader uses this predicate to skip the per-row time check +// and to take the bulk path; it must only be true for a plain TimeBetween that +// accepts every int64 value. +TEST(TimeBetweenFilterTest, IsUnboundedTimeFilterOnlyForFullRangeBetween) { + const int64_t min_time = std::numeric_limits::min(); + const int64_t max_time = std::numeric_limits::max(); + + TimeBetween full(min_time, max_time, /*not_between=*/false); + EXPECT_TRUE(full.is_unbounded_time_filter()); + + // Negated full range accepts nothing, so it must not take the shortcut. + TimeBetween negated(min_time, max_time, /*not_between=*/true); + EXPECT_FALSE(negated.is_unbounded_time_filter()); + + // Inverted bounds are not "everything". + TimeBetween inverted(max_time, min_time, /*not_between=*/false); + EXPECT_FALSE(inverted.is_unbounded_time_filter()); + + // One missing bound is a real restriction. + TimeBetween half_open(min_time + 1, max_time, /*not_between=*/false); + EXPECT_FALSE(half_open.is_unbounded_time_filter()); + TimeBetween narrow(0, 100, /*not_between=*/false); + EXPECT_FALSE(narrow.is_unbounded_time_filter()); + + // Other filter types never claim to be unbounded. + TimeIn time_in({1, 2}, /*not_in=*/false); + EXPECT_FALSE(time_in.is_unbounded_time_filter()); +} diff --git a/cpp/test/reader/prepared_series_test.cc b/cpp/test/reader/prepared_series_test.cc index 04ad57e137..b9ebead068 100644 --- a/cpp/test/reader/prepared_series_test.cc +++ b/cpp/test/reader/prepared_series_test.cc @@ -666,8 +666,16 @@ TEST_F(PreparedSeriesBatchTest, MultiPreparedUnboundedTimeFilterWithOffset) { reader.query_prepared_multi({prepared_value2, prepared_value}, start_time, end_time, offset, limit, result)); + if (result == nullptr) { + ADD_FAILURE() << "query_prepared_multi returned no result set"; + return std::vector(); + } auto* table_result = dynamic_cast(result); - EXPECT_NE(nullptr, table_result); + if (table_result == nullptr) { + ADD_FAILURE() << "query_prepared_multi returned a non-table result"; + reader.destroy_query_data_set(result); + return std::vector(); + } std::vector times; common::TsBlock* block = nullptr; while (table_result->get_next_tsblock(block) == common::E_OK) { @@ -699,24 +707,36 @@ TEST_F(PreparedSeriesBatchTest, MultiPreparedUnboundedTimeFilterWithOffset) { const int64_t min_time = std::numeric_limits::min(); const int64_t max_time = std::numeric_limits::max(); - EXPECT_EQ(collect(min_time, max_time, 0, -1), all_rows); - for (int offset : {1, 3, 4, 5, 17, 28, 29, 30}) { - SCOPED_TRACE(::testing::Message() << "offset=" << offset); - const std::vector expected(all_rows.begin() + offset, - all_rows.end()); - EXPECT_EQ(collect(min_time, max_time, offset, -1), expected); - } - std::vector in_window; for (int row = 5; row <= 22; ++row) in_window.push_back(row); - EXPECT_EQ(collect(5, 22, 0, -1), in_window); - const std::vector offset_in_window(in_window.begin() + 6, - in_window.begin() + 10); - EXPECT_EQ(collect(5, 22, 6, 4), offset_in_window); - const std::vector tail_in_window(in_window.begin() + 10, - in_window.end()); - EXPECT_EQ(collect(5, 22, 10, -1), tail_in_window); - EXPECT_TRUE(collect(100, 200, 0, -1).empty()); + + // Exercise both decode paths: with the worker pool the page plan is + // built and decoded in parallel (the planner's prefilled times must be + // reused by the workers); without it the plan is decoded lazily, which is + // where the planner's times are handed to decode_page_lazy(). + const bool saved_parallel = common::g_config_value_.parallel_read_enabled_; + for (bool parallel : {true, false}) { + SCOPED_TRACE(::testing::Message() << "parallel=" << parallel); + common::g_config_value_.parallel_read_enabled_ = parallel; + + EXPECT_EQ(collect(min_time, max_time, 0, -1), all_rows); + for (int offset : {1, 3, 4, 5, 17, 28, 29, 30}) { + SCOPED_TRACE(::testing::Message() << "offset=" << offset); + const std::vector expected(all_rows.begin() + offset, + all_rows.end()); + EXPECT_EQ(collect(min_time, max_time, offset, -1), expected); + } + + EXPECT_EQ(collect(5, 22, 0, -1), in_window); + const std::vector offset_in_window(in_window.begin() + 6, + in_window.begin() + 10); + EXPECT_EQ(collect(5, 22, 6, 4), offset_in_window); + const std::vector tail_in_window(in_window.begin() + 10, + in_window.end()); + EXPECT_EQ(collect(5, 22, 10, -1), tail_in_window); + EXPECT_TRUE(collect(100, 200, 0, -1).empty()); + } + common::g_config_value_.parallel_read_enabled_ = saved_parallel; EXPECT_EQ(common::E_OK, reader.close()); } diff --git a/cpp/test/reader/table_view/tsfile_reader_table_test.cc b/cpp/test/reader/table_view/tsfile_reader_table_test.cc index 6a3db128a6..0a09248762 100644 --- a/cpp/test/reader/table_view/tsfile_reader_table_test.cc +++ b/cpp/test/reader/table_view/tsfile_reader_table_test.cc @@ -417,6 +417,10 @@ TEST_P(TsFileTableReaderTimeFilterTest, UnboundedTimeFilterReturnsAllRows) { EXPECT_EQ(reader.query(table_schema->get_table_name(), query_columns, start_time, end_time, offset, limit, result), common::E_OK); + if (result == nullptr) { + ADD_FAILURE() << "query returned no result set"; + return std::vector>(); + } std::vector> rows; bool has_next = false; int ret = common::E_OK; @@ -453,6 +457,41 @@ TEST_P(TsFileTableReaderTimeFilterTest, UnboundedTimeFilterReturnsAllRows) { ASSERT_EQ(reader.close(), common::E_OK); } +// The unbounded shortcut must only fire for a plain TimeBetween; its negation +// accepts no rows and must stay empty even though the bounds are full-range. +TEST_P(TsFileTableReaderTimeFilterTest, NotBetweenFullRangeReturnsNoRows) { + std::unique_ptr table_schema(gen_table_schema(0)); + TsFileTableWriter writer(&write_file_, table_schema.get()); + auto tablet = gen_tablet(table_schema.get(), 0, 1, 10); + ASSERT_EQ(writer.write_table(tablet), common::E_OK); + ASSERT_EQ(writer.flush(), common::E_OK); + ASSERT_EQ(writer.close(), common::E_OK); + + LocalRandomAccessReadFile read_file; + ASSERT_EQ(read_file.open(file_name_), common::E_OK); + TableQueryExecutor executor(&read_file); + + const int64_t min_time = std::numeric_limits::min(); + const int64_t max_time = std::numeric_limits::max(); + ResultSet* result = nullptr; + ASSERT_EQ(executor.query( + table_schema->get_table_name(), + std::vector{"s0", "s1"}, + new TimeBetween(min_time, max_time, /*not_between=*/true), + nullptr, nullptr, 0, -1, result), + common::E_OK); + ASSERT_NE(nullptr, result); + bool has_next = false; + int ret = common::E_OK; + int rows = 0; + while ((ret = result->next(has_next)) == common::E_OK && has_next) { + ++rows; + } + EXPECT_EQ(ret, common::E_OK); + EXPECT_EQ(rows, 0); + executor.destroy_query_data_set(result); +} + // Regression for #1038: a page whose rows are all rejected by the time filter // must not end the series. Requesting the never-written measurement "ghost" // (and omitting "v" on one device) forces the per-column aligned read path, From 244fbb2bb768048508bf2920655697df09a9d5ad Mon Sep 17 00:00:00 2001 From: ColinLee Date: Thu, 12 Nov 2026 23:11:55 +0800 Subject: [PATCH 5/7] test(cpp): include explicitly in the time filter test --- cpp/test/reader/filter/time_in_filter_test.cc | 2 ++ 1 file changed, 2 insertions(+) diff --git a/cpp/test/reader/filter/time_in_filter_test.cc b/cpp/test/reader/filter/time_in_filter_test.cc index fede7827c6..f8fc4ec6df 100644 --- a/cpp/test/reader/filter/time_in_filter_test.cc +++ b/cpp/test/reader/filter/time_in_filter_test.cc @@ -18,6 +18,8 @@ */ #include +#include + #include "reader/filter/time_operator.h" using namespace storage; From d024450426d73cf306785d3b0d93662729a11f55 Mon Sep 17 00:00:00 2001 From: ColinLee Date: Thu, 12 Nov 2026 23:17:02 +0800 Subject: [PATCH 6/7] test(cpp): copy row values out instead of casting the buffer Avoid reinterpret_cast to int64_t/double on the row buffer, which has no alignment guarantee; memcpy into aligned locals. --- cpp/test/reader/prepared_series_test.cc | 21 ++++++++++++++------- 1 file changed, 14 insertions(+), 7 deletions(-) diff --git a/cpp/test/reader/prepared_series_test.cc b/cpp/test/reader/prepared_series_test.cc index b9ebead068..fff9e75e6c 100644 --- a/cpp/test/reader/prepared_series_test.cc +++ b/cpp/test/reader/prepared_series_test.cc @@ -24,6 +24,7 @@ #include #include +#include #include #include @@ -683,17 +684,23 @@ TEST_F(PreparedSeriesBatchTest, MultiPreparedUnboundedTimeFilterWithOffset) { while (rows.has_next()) { uint32_t len = 0; bool is_null = false; + // Values are copied out instead of reinterpret_cast'ing the + // returned buffer: the row data has no alignment guarantee. const char* timestamp = rows.read(0, &len, &is_null); EXPECT_FALSE(is_null); - times.push_back(*reinterpret_cast(timestamp)); - const char* value2 = rows.read(1, &len, &is_null); + int64_t time = 0; + std::memcpy(&time, timestamp, sizeof(time)); + times.push_back(time); + const char* value2_buf = rows.read(1, &len, &is_null); EXPECT_FALSE(is_null); - EXPECT_DOUBLE_EQ(times.back() * 10.0, - *reinterpret_cast(value2)); - const char* value = rows.read(2, &len, &is_null); + double value2 = 0.0; + std::memcpy(&value2, value2_buf, sizeof(value2)); + EXPECT_DOUBLE_EQ(time * 10.0, value2); + const char* value_buf = rows.read(2, &len, &is_null); EXPECT_FALSE(is_null); - EXPECT_DOUBLE_EQ(static_cast(times.back()), - *reinterpret_cast(value)); + double value = 0.0; + std::memcpy(&value, value_buf, sizeof(value)); + EXPECT_DOUBLE_EQ(static_cast(time), value); rows.next(); } block = nullptr; From 0278ac0905f3b1e8d006aacd1c5efa6fb394da7b Mon Sep 17 00:00:00 2001 From: ColinLee Date: Fri, 13 Nov 2026 01:37:37 +0800 Subject: [PATCH 7/7] test(cpp): give reader-test files a per-process unique name Parallel CTest processes can seed the name generator in the same clock tick; one then removes the file another is opening (E_ALREADY_EXIST at the fixture's open, observed as SEGFAULT / test failure on CI). Append pid and a per-process counter, as the writer tests already do. --- cpp/test/reader/tsfile_reader_test.cc | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/cpp/test/reader/tsfile_reader_test.cc b/cpp/test/reader/tsfile_reader_test.cc index 060c7a5411..1fe277ddef 100644 --- a/cpp/test/reader/tsfile_reader_test.cc +++ b/cpp/test/reader/tsfile_reader_test.cc @@ -21,6 +21,7 @@ #include #include +#include #include #include #include @@ -43,6 +44,12 @@ #include "reader/block/single_device_tsblock_reader.h" #include "reader/filter/time_operator.h" #include "reader/qds_without_timegenerator.h" + +#ifdef _WIN32 +#include +#else +#include +#endif #include "reader/tsfile_series_scan_iterator.h" #include "writer/chunk_writer.h" #include "writer/tsfile_writer.h" @@ -118,6 +125,18 @@ class TsFileReaderTest : public ::testing::Test { random_string += chars[dis(gen)]; } + // CTest runs test cases in parallel processes, and two of them can + // seed the generator in the same clock tick. Without a per-process + // unique suffix one process can remove the file another is using, + // which surfaced as E_ALREADY_EXIST from TsFileWriter::open. +#ifdef _WIN32 + const auto process_id = static_cast(_getpid()); +#else + const auto process_id = static_cast(getpid()); +#endif + static std::atomic counter{0}; + random_string += "_" + std::to_string(process_id) + "_" + + std::to_string(counter.fetch_add(1)); return random_string; }