From 8feaa471b0d82dbd1703268a26810b62e52f43fe Mon Sep 17 00:00:00 2001 From: ColinLee Date: Fri, 10 Jul 2026 09:36:03 +0800 Subject: [PATCH 1/5] perf(cpp): push down multi-value aligned offsets --- cpp/src/reader/aligned_chunk_reader.cc | 70 +++++-- cpp/src/reader/aligned_chunk_reader.h | 7 +- .../block/single_device_tsblock_reader.cc | 41 +++- .../block/single_device_tsblock_reader.h | 3 +- cpp/src/reader/tsfile_series_scan_iterator.cc | 29 +++ cpp/src/reader/tsfile_series_scan_iterator.h | 2 + cpp/test/reader/tsfile_reader_test.cc | 176 +++++++++++++++++- 7 files changed, 296 insertions(+), 32 deletions(-) diff --git a/cpp/src/reader/aligned_chunk_reader.cc b/cpp/src/reader/aligned_chunk_reader.cc index f02497363f..fcf82e1648 100644 --- a/cpp/src/reader/aligned_chunk_reader.cc +++ b/cpp/src/reader/aligned_chunk_reader.cc @@ -77,6 +77,7 @@ void AlignedChunkReader::reset() { page_plan_built_ = false; current_page_loaded_ = false; current_page_plan_index_ = 0; + multi_serial_decode_current_chunk_ = false; time_predecoded_ = false; page_all_times_.clear(); page_time_count_ = 0; @@ -333,7 +334,9 @@ int AlignedChunkReader::alloc_compressor_and_decoder( int AlignedChunkReader::get_next_page(TsBlock* ret_tsblock, Filter* oneshoot_filter, PageArena& pa) { if (multi_value_mode_) { - return get_next_page_multi(ret_tsblock, oneshoot_filter, pa); + int row_offset = 0; + return get_next_page_multi(ret_tsblock, oneshoot_filter, pa, + row_offset); } int ret = E_OK; Filter* filter = @@ -1082,17 +1085,19 @@ int AlignedChunkReader::get_next_page(TsBlock* ret_tsblock, int64_t min_time_hint, int& row_offset, int& row_limit) { if (multi_value_mode_) { - // Multi-value aligned path doesn't yet honour row_offset / row_limit - // / min_time_hint — they get dropped on the floor, which silently - // returns full chunk data when the caller asked for a sub-range. - // Refuse the combination so the caller sees an actual error instead - // of garbage results. set_row_range(0, -1) keeps the all-rows - // contract intact for normal queries. - if (row_offset > 0 || row_limit >= 0 || + if (row_limit == 0) { + return E_NO_MORE_DATA; + } + // The multi-value path can consume offset through chunk/page count + // statistics. Limit and min-time pushdown still need separate state + // handling, so keep those combinations explicit instead of silently + // returning a wider range. + if (row_limit >= 0 || min_time_hint != std::numeric_limits::min()) { return common::E_NOT_SUPPORT; } - return get_next_page_multi(ret_tsblock, oneshoot_filter, pa); + return get_next_page_multi(ret_tsblock, oneshoot_filter, pa, + row_offset); } int ret = E_OK; Filter* filter = @@ -1157,6 +1162,7 @@ int AlignedChunkReader::load_by_aligned_meta_multi( page_plan_built_ = false; current_page_loaded_ = false; current_page_plan_index_ = 0; + multi_serial_decode_current_chunk_ = false; time_predecoded_ = false; page_all_times_.clear(); page_time_count_ = 0; @@ -1833,11 +1839,15 @@ void AlignedChunkReader::release_page_slot(size_t page_idx) { int AlignedChunkReader::get_next_page_multi(TsBlock* ret_tsblock, Filter* oneshoot_filter, - PageArena& pa) { + PageArena& pa, int& row_offset) { int ret = E_OK; Filter* filter = (oneshoot_filter != nullptr ? oneshoot_filter : time_filter_); + if (row_offset > 0) { + multi_serial_decode_current_chunk_ = true; + } + // Dispatch: // - Multi-column with a thread pool → chunk-level pre-decode: one task // per value column decodes that column's whole chunk up front, then the @@ -1850,13 +1860,14 @@ int AlignedChunkReader::get_next_page_multi(TsBlock* ret_tsblock, // page's columns inline (multi_DECODE_TV_BATCH), no thread-pool // fan-out. #ifdef ENABLE_THREADS - const bool use_chunk_level = - decode_pool_ != nullptr && value_columns_.size() > 1; + const bool use_chunk_level = decode_pool_ != nullptr && + value_columns_.size() > 1 && + !multi_serial_decode_current_chunk_; #else const bool use_chunk_level = false; #endif if (!use_chunk_level) { - return get_next_page_multi_serial(ret_tsblock, filter, pa); + return get_next_page_multi_serial(ret_tsblock, filter, pa, row_offset); } if (!page_plan_built_) { @@ -2044,7 +2055,8 @@ int AlignedChunkReader::get_next_page_multi(TsBlock* ret_tsblock, int AlignedChunkReader::get_next_page_multi_serial(TsBlock* ret_tsblock, Filter* filter, - PageArena& pa) { + PageArena& pa, + int& row_offset) { int ret = E_OK; bool pt = prev_time_page_not_finish(); bool pv = prev_any_value_page_not_finish_multi(); @@ -2069,8 +2081,13 @@ int AlignedChunkReader::get_next_page_multi_serial(TsBlock* ret_tsblock, } } if (IS_FAIL(ret)) break; - if (cur_page_statisify_filter_multi(filter)) break; - if (RET_FAIL(skip_cur_page_multi())) break; + if (!cur_page_statisify_filter_multi(filter)) { + if (RET_FAIL(skip_cur_page_multi())) break; + } else if (should_skip_page_by_offset_multi(row_offset, filter)) { + if (RET_FAIL(skip_cur_page_multi())) break; + } else { + break; + } if (!has_more_data()) { ret = E_NO_MORE_DATA; break; @@ -2095,6 +2112,27 @@ bool AlignedChunkReader::cur_page_statisify_filter_multi(Filter* filter) { return time_satisfy; } +bool AlignedChunkReader::should_skip_page_by_offset_multi(int& row_offset, + Filter* filter) { + if (row_offset <= 0) { + return false; + } + Statistic* stat = cur_time_page_header_.statistic_; + if (stat == nullptr || stat->count_ == 0) { + return false; + } + if (filter != nullptr && + !filter->contain_start_end_time(stat->start_time_, stat->end_time_)) { + return false; + } + int32_t count = stat->count_; + if (row_offset >= count) { + row_offset -= count; + return true; + } + return false; +} + int AlignedChunkReader::skip_cur_page_multi() { time_chunk_visit_offset_ += cur_time_page_header_.compressed_size_; time_in_stream_.wrapped_buf_advance_read_pos( diff --git a/cpp/src/reader/aligned_chunk_reader.h b/cpp/src/reader/aligned_chunk_reader.h index b92c4d7b5e..9117eecc4a 100644 --- a/cpp/src/reader/aligned_chunk_reader.h +++ b/cpp/src/reader/aligned_chunk_reader.h @@ -238,11 +238,13 @@ class AlignedChunkReader : public IChunkReader { bool has_more_data_multi() const; bool prev_any_value_page_not_finish_multi() const; int get_next_page_multi(common::TsBlock* ret_tsblock, - Filter* oneshoot_filter, common::PageArena& pa); + Filter* oneshoot_filter, common::PageArena& pa, + int& row_offset); int get_next_page_multi_serial(common::TsBlock* ret_tsblock, Filter* filter, - common::PageArena& pa); + common::PageArena& pa, int& row_offset); int skip_cur_page_multi(); bool cur_page_statisify_filter_multi(Filter* filter); + bool should_skip_page_by_offset_multi(int& row_offset, Filter* filter); int decode_cur_value_pages_multi(); int decode_cur_value_page_data_for(ValueColumnState& col); int ensure_value_page_loaded(ValueColumnState& col); @@ -318,6 +320,7 @@ class AlignedChunkReader : public IChunkReader { bool page_plan_built_ = false; bool current_page_loaded_ = false; size_t current_page_plan_index_ = 0; + bool multi_serial_decode_current_chunk_ = false; #ifdef ENABLE_THREADS common::ThreadPool* decode_pool_ = nullptr; // borrowed, not owned diff --git a/cpp/src/reader/block/single_device_tsblock_reader.cc b/cpp/src/reader/block/single_device_tsblock_reader.cc index d842a35d9e..0b547efac9 100644 --- a/cpp/src/reader/block/single_device_tsblock_reader.cc +++ b/cpp/src/reader/block/single_device_tsblock_reader.cc @@ -47,6 +47,7 @@ int SingleDeviceTsBlockReader::init(DeviceQueryTask* device_query_task, remaining_offset_ = 0; remaining_limit_ = -1; dense_row_count_ = -1; + row_offset_pushed_to_ssi_ = false; return init_internal(device_query_task, block_size, time_filter, field_filter); } @@ -58,6 +59,7 @@ int SingleDeviceTsBlockReader::init(DeviceQueryTask* device_query_task, remaining_offset_ = row_offset; remaining_limit_ = row_limit; dense_row_count_ = -1; + row_offset_pushed_to_ssi_ = false; return init_internal(device_query_task, block_size, time_filter, field_filter); } @@ -182,6 +184,13 @@ int SingleDeviceTsBlockReader::init_internal(DeviceQueryTask* device_query_task, return ret; } dense_row_count_ = compute_dense_row_count(time_series_indexs); + if (time_filter == nullptr && field_filter == nullptr && + dense_row_count_ >= 0 && remaining_offset_ >= dense_row_count_) { + remaining_offset_ -= dense_row_count_; + delete current_block_; + current_block_ = nullptr; + return common::E_OK; + } // Early device-level time skip: if time_filter is set and ALL chunks of // this device have statistics that fall outside the filter range, skip the // entire device. Chunks without statistics are assumed to satisfy. @@ -278,8 +287,18 @@ int SingleDeviceTsBlockReader::init_internal(DeviceQueryTask* device_query_task, } auto* ctx = new VectorMeasurementColumnContext(tsfile_io_reader_); + const int ssi_offset = + (time_filter == nullptr && field_filter == nullptr && + dense_row_count_ >= 0) + ? remaining_offset_ + : 0; if (common::E_OK == ctx->init(device_query_task_, meas_names, - time_filter, pos_list, pa_)) { + time_filter, pos_list, pa_, + ssi_offset, -1)) { + if (ssi_offset > 0) { + row_offset_pushed_to_ssi_ = true; + remaining_offset_ = ctx->get_ssi_row_offset(); + } // The shared ctx is referenced from N map entries; close() // and the merge loop dedupe by pointer (already in place). for (const auto& name : meas_names) { @@ -540,7 +559,11 @@ int SingleDeviceTsBlockReader::has_next_aligned(bool& result_has_next) { int sr = ctx->skip_rows(skip); if (sr != common::E_OK) return sr; } - remaining_offset_ -= skip; + if (row_offset_pushed_to_ssi_ && !aligned_vec_.empty()) { + remaining_offset_ = aligned_vec_[0]->get_ssi_row_offset(); + } else { + remaining_offset_ -= skip; + } continue; } @@ -973,6 +996,11 @@ int SingleMeasurementColumnContext::skip_rows(uint32_t count) { const uint32_t val_elem_size = common::get_data_type_size(dt); value_iter_->advance(to_skip, val_elem_size); } + int ssi_offset = get_ssi_row_offset(); + if (ssi_offset > 0) { + set_ssi_row_range(std::max(0, ssi_offset - static_cast(to_skip)), + get_ssi_row_limit()); + } if (time_iter_->end()) { // Propagate hard errors from the next-tsblock load; E_NO_MORE_DATA // is the legitimate end-of-stream signal and gets squashed back to @@ -1004,7 +1032,8 @@ VectorMeasurementColumnContext::~VectorMeasurementColumnContext() { int VectorMeasurementColumnContext::init( DeviceQueryTask* device_query_task, const std::vector& measurement_names, Filter* time_filter, - std::vector>& pos_in_result, common::PageArena& pa) { + std::vector>& pos_in_result, common::PageArena& pa, + int ssi_offset, int ssi_limit) { int ret = common::E_OK; pos_in_result_ = pos_in_result; column_names_ = measurement_names; @@ -1013,6 +1042,7 @@ int VectorMeasurementColumnContext::init( time_filter))) { return ret; } + ssi_->set_row_range(ssi_offset, ssi_limit); if (RET_FAIL(get_next_tsblock(true))) { return ret; } @@ -1224,6 +1254,11 @@ int VectorMeasurementColumnContext::skip_rows(uint32_t count) { } } } + int ssi_offset = get_ssi_row_offset(); + if (ssi_offset > 0) { + set_ssi_row_range(std::max(0, ssi_offset - static_cast(to_skip)), + get_ssi_row_limit()); + } if (time_iter_->end()) { int r = get_next_tsblock(false); if (r != common::E_OK && r != common::E_NO_MORE_DATA) return r; diff --git a/cpp/src/reader/block/single_device_tsblock_reader.h b/cpp/src/reader/block/single_device_tsblock_reader.h index e74304bafe..cccc7fd3fd 100644 --- a/cpp/src/reader/block/single_device_tsblock_reader.h +++ b/cpp/src/reader/block/single_device_tsblock_reader.h @@ -86,6 +86,7 @@ class SingleDeviceTsBlockReader : public TsBlockReader { int remaining_offset_ = 0; int remaining_limit_ = -1; int32_t dense_row_count_ = -1; + bool row_offset_pushed_to_ssi_ = false; // Populated in init() when every field column comes from an aligned chunk. // Provides cache-friendly vector iteration for has_next_aligned(). bool all_aligned_ = false; @@ -195,7 +196,7 @@ class VectorMeasurementColumnContext final : public MeasurementColumnContext { const std::vector& measurement_names, Filter* time_filter, std::vector>& pos_in_result, - common::PageArena& pa); + common::PageArena& pa, int ssi_offset = 0, int ssi_limit = -1); int get_next_tsblock(bool alloc_mem) override; int get_current_time(int64_t& time) override; int get_current_value(char*& value, uint32_t& len) override; diff --git a/cpp/src/reader/tsfile_series_scan_iterator.cc b/cpp/src/reader/tsfile_series_scan_iterator.cc index 6e318b38e7..b73d5b0020 100644 --- a/cpp/src/reader/tsfile_series_scan_iterator.cc +++ b/cpp/src/reader/tsfile_series_scan_iterator.cc @@ -122,6 +122,31 @@ bool TsFileSeriesScanIterator::should_skip_aligned_chunk_by_offset( return false; } +bool TsFileSeriesScanIterator::should_skip_multi_aligned_chunk_by_offset( + ChunkMeta* time_cm, const std::vector& value_cms) { + if (row_offset_ <= 0) { + return false; + } + if (time_cm == nullptr || time_cm->statistic_ == nullptr) { + return false; + } + int32_t time_count = time_cm->statistic_->count_; + if (time_count <= 0) { + return false; + } + for (const auto* value_cm : value_cms) { + if (value_cm == nullptr || value_cm->statistic_ == nullptr || + value_cm->statistic_->count_ != time_count) { + return false; + } + } + if (row_offset_ >= time_count) { + row_offset_ -= time_count; + return true; + } + return false; +} + int TsFileSeriesScanIterator::get_next(TsBlock*& ret_tsblock, bool alloc, Filter* oneshoot_filter, int64_t min_time_hint) { @@ -158,6 +183,10 @@ int TsFileSeriesScanIterator::get_next(TsBlock*& ret_tsblock, bool alloc, if (should_skip_chunk_by_time(time_cm, min_time_hint)) { continue; } + if (should_skip_multi_aligned_chunk_by_offset(time_cm, + value_cms)) { + continue; + } chunk_reader_->reset(); auto* acr = static_cast(chunk_reader_); if (RET_FAIL(acr->load_by_aligned_meta_multi(time_cm, diff --git a/cpp/src/reader/tsfile_series_scan_iterator.h b/cpp/src/reader/tsfile_series_scan_iterator.h index 77037d8e17..187e9d9965 100644 --- a/cpp/src/reader/tsfile_series_scan_iterator.h +++ b/cpp/src/reader/tsfile_series_scan_iterator.h @@ -167,6 +167,8 @@ class TsFileSeriesScanIterator { bool should_skip_chunk_by_offset(ChunkMeta* cm); bool should_skip_aligned_chunk_by_offset(ChunkMeta* time_cm, ChunkMeta* value_cm); + bool should_skip_multi_aligned_chunk_by_offset( + ChunkMeta* time_cm, const std::vector& value_cms); common::TsBlock* alloc_tsblock(); common::TsBlock* alloc_tsblock_multi(); diff --git a/cpp/test/reader/tsfile_reader_test.cc b/cpp/test/reader/tsfile_reader_test.cc index 2e74e870f0..5d5a29322e 100644 --- a/cpp/test/reader/tsfile_reader_test.cc +++ b/cpp/test/reader/tsfile_reader_test.cc @@ -463,12 +463,10 @@ TEST_F(TsFileReaderTest, reader.close(); } -// Multi-value aligned chunk reader doesn't honour row_offset / row_limit / -// min_time_hint pushdown — silently dropping those args would hand the caller -// full-chunk data when it asked for a sub-range. The guard at the top of -// AlignedChunkReader::get_next_page must turn the unsupported combination -// into an explicit E_NOT_SUPPORT. -TEST_F(TsFileReaderTest, MultiValueAlignedRowOffsetReturnsNotSupport) { +// Multi-value aligned chunk reader consumes row_offset at chunk/page +// granularity. If the remaining offset lands inside the current page, the SSI +// leaves that residual for the row reader to consume from the returned TsBlock. +TEST_F(TsFileReaderTest, MultiValueAlignedRowOffsetKeepsPartialPageResidual) { const std::string device = "root.dev_multi_offset"; std::vector schema_vec; schema_vec.emplace_back("v0", INT64, PLAIN, UNCOMPRESSED); @@ -505,12 +503,12 @@ TEST_F(TsFileReaderTest, MultiValueAlignedRowOffsetReturnsNotSupport) { E_OK); ASSERT_NE(ssi, nullptr); - // row_offset > 0 hits the multi-value guard at the top of - // AlignedChunkReader::get_next_page; the SSI propagates the error code. ssi->set_row_range(/*offset=*/5, /*limit=*/-1); common::TsBlock* block = nullptr; - EXPECT_EQ(ssi->get_next(block, /*alloc_tsblock=*/true), - common::E_NOT_SUPPORT); + EXPECT_EQ(ssi->get_next(block, /*alloc_tsblock=*/true), common::E_OK); + ASSERT_NE(block, nullptr); + EXPECT_EQ(block->get_row_count(), static_cast(N)); + EXPECT_EQ(ssi->get_row_offset(), 5); if (block != nullptr) { ssi->revert_tsblock(); @@ -521,6 +519,164 @@ TEST_F(TsFileReaderTest, MultiValueAlignedRowOffsetReturnsNotSupport) { // it, then ~TsFileMeta would call self_deleter on freed memory. } +TEST_F(TsFileReaderTest, MultiValueAlignedRowOffsetSkipsWholePage) { + struct PagePointGuard { + explicit PagePointGuard(uint32_t page_points) + : saved_(common::g_config_value_.page_writer_max_point_num_) { + common::g_config_value_.page_writer_max_point_num_ = page_points; + } + ~PagePointGuard() { + common::g_config_value_.page_writer_max_point_num_ = saved_; + } + uint32_t saved_; + } page_point_guard(16); + + const std::string device = "root.dev_multi_page_offset"; + std::vector schema_vec; + schema_vec.emplace_back("v0", INT64, PLAIN, UNCOMPRESSED); + schema_vec.emplace_back("v1", INT64, PLAIN, UNCOMPRESSED); + { + std::vector reg; + for (auto& s : schema_vec) reg.push_back(new MeasurementSchema(s)); + ASSERT_EQ(tsfile_writer_->register_aligned_timeseries(device, reg), + E_OK); + } + + const int rows_per_page = 16; + const int N = rows_per_page * 4; + Tablet tablet(device, + std::make_shared>(schema_vec), + N); + for (int i = 0; i < N; ++i) { + ASSERT_EQ(tablet.add_timestamp(i, static_cast(i)), E_OK); + ASSERT_EQ(tablet.add_value(i, 0u, static_cast(i)), E_OK); + ASSERT_EQ(tablet.add_value(i, 1u, static_cast(i * 2)), E_OK); + } + ASSERT_EQ(tsfile_writer_->write_tablet_aligned(tablet), E_OK); + ASSERT_EQ(tsfile_writer_->flush(), E_OK); + ASSERT_EQ(tsfile_writer_->close(), E_OK); + + storage::TsFileIOReader io_reader; + ASSERT_EQ(io_reader.init(file_name_), E_OK); + + auto device_id = std::make_shared(device); + std::vector measurements = {"v0", "v1"}; + storage::TsFileSeriesScanIterator* ssi = nullptr; + common::PageArena pa; + pa.init(512, common::MOD_TSFILE_READER); + ASSERT_EQ(io_reader.alloc_multi_ssi(device_id, measurements, ssi, pa, + /*time_filter=*/nullptr), + E_OK); + ASSERT_NE(ssi, nullptr); + + // The first chunk is preloaded before set_row_range(). This offset must be + // consumed at page granularity: skip page 0 and leave 4 rows for the row + // reader to skip inside page 1. + ssi->set_row_range(/*offset=*/20, /*limit=*/-1); + common::TsBlock* block = nullptr; + ASSERT_EQ(ssi->get_next(block, /*alloc_tsblock=*/true), common::E_OK); + ASSERT_NE(block, nullptr); + ASSERT_EQ(block->get_row_count(), static_cast(rows_per_page)); + EXPECT_EQ(ssi->get_row_offset(), 4); + + { + common::ColIterator time_iter(0, block); + common::ColIterator v0_iter(1, block); + common::ColIterator v1_iter(2, block); + uint32_t len = 0; + EXPECT_EQ(*reinterpret_cast(time_iter.read(&len)), 16); + EXPECT_EQ(*reinterpret_cast(v0_iter.read(&len)), 16); + EXPECT_EQ(*reinterpret_cast(v1_iter.read(&len)), 32); + } + + ssi->revert_tsblock(); + io_reader.revert_ssi(ssi); +} + +TEST_F(TsFileReaderTest, MultiValueAlignedRowOffsetSkipsWholeChunk) { + const std::string device = "root.dev_multi_chunk_offset"; + std::vector schema_vec; + schema_vec.emplace_back("v0", INT64, PLAIN, UNCOMPRESSED); + schema_vec.emplace_back("v1", INT64, PLAIN, UNCOMPRESSED); + { + std::vector reg; + for (auto& s : schema_vec) reg.push_back(new MeasurementSchema(s)); + ASSERT_EQ(tsfile_writer_->register_aligned_timeseries(device, reg), + E_OK); + } + + const int rows_per_chunk = 64; + const int chunk_count = 4; + for (int chunk = 0; chunk < chunk_count; ++chunk) { + Tablet tablet( + device, + std::make_shared>(schema_vec), + rows_per_chunk); + const int base = chunk * rows_per_chunk; + for (int i = 0; i < rows_per_chunk; ++i) { + const int row = base + i; + ASSERT_EQ(tablet.add_timestamp(i, static_cast(row)), E_OK); + ASSERT_EQ(tablet.add_value(i, 0u, static_cast(row)), E_OK); + ASSERT_EQ(tablet.add_value(i, 1u, static_cast(row * 2)), + E_OK); + } + ASSERT_EQ(tsfile_writer_->write_tablet_aligned(tablet), E_OK); + ASSERT_EQ(tsfile_writer_->flush(), E_OK); + } + ASSERT_EQ(tsfile_writer_->close(), E_OK); + + storage::TsFileIOReader io_reader; + ASSERT_EQ(io_reader.init(file_name_), E_OK); + + auto device_id = std::make_shared(device); + std::vector measurements = {"v0", "v1"}; + storage::TsFileSeriesScanIterator* ssi = nullptr; + common::PageArena pa; + pa.init(512, common::MOD_TSFILE_READER); + ASSERT_EQ(io_reader.alloc_multi_ssi(device_id, measurements, ssi, pa, + /*time_filter=*/nullptr), + E_OK); + ASSERT_NE(ssi, nullptr); + + // alloc_multi_ssi() preloads chunk 0 before row range is set. Read it + // normally first, then set an offset that skips the next whole chunk by + // chunk metadata count and leaves a 32-row residual inside chunk 2. + common::TsBlock* block = nullptr; + ASSERT_EQ(ssi->get_next(block, /*alloc_tsblock=*/true), common::E_OK); + ASSERT_NE(block, nullptr); + ASSERT_EQ(block->get_row_count(), static_cast(rows_per_chunk)); + { + common::ColIterator time_iter(0, block); + common::ColIterator v0_iter(1, block); + common::ColIterator v1_iter(2, block); + uint32_t len = 0; + EXPECT_EQ(*reinterpret_cast(time_iter.read(&len)), 0); + EXPECT_EQ(*reinterpret_cast(v0_iter.read(&len)), 0); + EXPECT_EQ(*reinterpret_cast(v1_iter.read(&len)), 0); + } + ssi->revert_tsblock(); + + ssi->set_row_range(/*offset=*/96, /*limit=*/-1); + block = nullptr; + ASSERT_EQ(ssi->get_next(block, /*alloc_tsblock=*/true), common::E_OK); + ASSERT_NE(block, nullptr); + ASSERT_EQ(block->get_row_count(), static_cast(rows_per_chunk)); + EXPECT_EQ(ssi->get_row_offset(), 32); + + { + common::ColIterator time_iter(0, block); + common::ColIterator v0_iter(1, block); + common::ColIterator v1_iter(2, block); + uint32_t len = 0; + EXPECT_EQ(*reinterpret_cast(time_iter.read(&len)), 128); + EXPECT_EQ(*reinterpret_cast(v0_iter.read(&len)), 128); + EXPECT_EQ(*reinterpret_cast(v1_iter.read(&len)), 256); + } + + ssi->revert_tsblock(); + io_reader.revert_ssi(ssi); +} + namespace storage { // Subclass that lets the test (a) inject an error from the next-tsblock load // and (b) wire a manually constructed TsBlock into the inherited iterator From 7efa8aee53a3c779bad72918d4aa755dd64570fb Mon Sep 17 00:00:00 2001 From: ColinLee Date: Mon, 20 Jul 2026 12:09:43 +0800 Subject: [PATCH 2/5] fix(cpp): apply offsets after time filters --- cpp/src/reader/aligned_chunk_reader.cc | 9 +- cpp/src/reader/aligned_chunk_reader.h | 2 +- cpp/src/reader/chunk_reader.cc | 11 +- cpp/src/reader/chunk_reader.h | 2 +- cpp/src/reader/tsfile_series_scan_iterator.cc | 33 +++- cpp/src/reader/tsfile_series_scan_iterator.h | 8 +- cpp/test/reader/tsfile_reader_test.cc | 186 ++++++++++++++++++ 7 files changed, 233 insertions(+), 18 deletions(-) diff --git a/cpp/src/reader/aligned_chunk_reader.cc b/cpp/src/reader/aligned_chunk_reader.cc index fcf82e1648..02008081a5 100644 --- a/cpp/src/reader/aligned_chunk_reader.cc +++ b/cpp/src/reader/aligned_chunk_reader.cc @@ -1060,7 +1060,8 @@ bool AlignedChunkReader::should_skip_page_by_time(int64_t min_time_hint) { return false; } -bool AlignedChunkReader::should_skip_page_by_offset(int& row_offset) { +bool AlignedChunkReader::should_skip_page_by_offset(int& row_offset, + Filter* filter) { if (row_offset <= 0) { return false; } @@ -1072,6 +1073,10 @@ bool AlignedChunkReader::should_skip_page_by_offset(int& row_offset) { if (stat == nullptr || stat->count_ == 0) { return false; } + if (filter != nullptr && + !filter->contain_start_end_time(stat->start_time_, stat->end_time_)) { + return false; + } int32_t count = stat->count_; if (row_offset >= count) { row_offset -= count; @@ -1129,7 +1134,7 @@ int AlignedChunkReader::get_next_page(TsBlock* ret_tsblock, } else if (should_skip_page_by_time(min_time_hint)) { if (RET_FAIL(skip_cur_page())) { } - } else if (should_skip_page_by_offset(row_offset)) { + } else if (should_skip_page_by_offset(row_offset, filter)) { if (RET_FAIL(skip_cur_page())) { } } else { diff --git a/cpp/src/reader/aligned_chunk_reader.h b/cpp/src/reader/aligned_chunk_reader.h index 9117eecc4a..ef0e65af2d 100644 --- a/cpp/src/reader/aligned_chunk_reader.h +++ b/cpp/src/reader/aligned_chunk_reader.h @@ -172,7 +172,7 @@ class AlignedChunkReader : public IChunkReader { private: bool should_skip_page_by_time(int64_t min_time_hint); - bool should_skip_page_by_offset(int& row_offset); + bool should_skip_page_by_offset(int& row_offset, Filter* filter); FORCE_INLINE bool chunk_has_only_one_page( const ChunkHeader& chunk_header) const { return (chunk_header.chunk_type_ & ONLY_ONE_PAGE_CHUNK_HEADER_MARKER) == diff --git a/cpp/src/reader/chunk_reader.cc b/cpp/src/reader/chunk_reader.cc index 7c36ea07fa..cfc1a0c68e 100644 --- a/cpp/src/reader/chunk_reader.cc +++ b/cpp/src/reader/chunk_reader.cc @@ -856,7 +856,7 @@ bool ChunkReader::should_skip_page_by_time(int64_t min_time_hint) { return cur_page_header_.statistic_->end_time_ < min_time_hint; } -bool ChunkReader::should_skip_page_by_offset(int& row_offset) { +bool ChunkReader::should_skip_page_by_offset(int& row_offset, Filter* filter) { if (row_offset <= 0) { return false; } @@ -864,6 +864,11 @@ bool ChunkReader::should_skip_page_by_offset(int& row_offset) { cur_page_header_.statistic_->count_ == 0) { return false; } + if (filter != nullptr && !filter->contain_start_end_time( + cur_page_header_.statistic_->start_time_, + cur_page_header_.statistic_->end_time_)) { + return false; + } int32_t count = cur_page_header_.statistic_->count_; if (row_offset >= count) { row_offset -= count; @@ -910,7 +915,7 @@ int ChunkReader::get_next_page(TsBlock* ret_tsblock, Filter* oneshoot_filter, } else if (should_skip_page_by_time(min_time_hint)) { if (RET_FAIL(skip_cur_page())) { } - } else if (should_skip_page_by_offset(row_offset)) { + } else if (should_skip_page_by_offset(row_offset, filter)) { if (RET_FAIL(skip_cur_page())) { } } else { @@ -924,4 +929,4 @@ int ChunkReader::get_next_page(TsBlock* ret_tsblock, Filter* oneshoot_filter, return ret; } -} // end namespace storage \ No newline at end of file +} // end namespace storage diff --git a/cpp/src/reader/chunk_reader.h b/cpp/src/reader/chunk_reader.h index a1196c3309..6b872ad7db 100644 --- a/cpp/src/reader/chunk_reader.h +++ b/cpp/src/reader/chunk_reader.h @@ -76,7 +76,7 @@ class ChunkReader : public IChunkReader { private: bool should_skip_page_by_time(int64_t min_time_hint); - bool should_skip_page_by_offset(int& row_offset); + bool should_skip_page_by_offset(int& row_offset, Filter* filter); FORCE_INLINE bool chunk_has_only_one_page() const { return (chunk_header_.chunk_type_ & ONLY_ONE_PAGE_CHUNK_HEADER_MARKER) == diff --git a/cpp/src/reader/tsfile_series_scan_iterator.cc b/cpp/src/reader/tsfile_series_scan_iterator.cc index b73d5b0020..dfc51c9f94 100644 --- a/cpp/src/reader/tsfile_series_scan_iterator.cc +++ b/cpp/src/reader/tsfile_series_scan_iterator.cc @@ -79,13 +79,19 @@ bool TsFileSeriesScanIterator::should_skip_chunk_by_time( return cm->statistic_->end_time_ < min_time_hint; } -bool TsFileSeriesScanIterator::should_skip_chunk_by_offset(ChunkMeta* cm) { +bool TsFileSeriesScanIterator::should_skip_chunk_by_offset(ChunkMeta* cm, + Filter* filter) { if (row_offset_ <= 0) { return false; } if (cm->statistic_ == nullptr || cm->statistic_->count_ == 0) { return false; } + if (filter != nullptr && + !filter->contain_start_end_time(cm->statistic_->start_time_, + cm->statistic_->end_time_)) { + return false; + } int32_t count = cm->statistic_->count_; if (row_offset_ >= count) { row_offset_ -= count; @@ -95,7 +101,7 @@ bool TsFileSeriesScanIterator::should_skip_chunk_by_offset(ChunkMeta* cm) { } bool TsFileSeriesScanIterator::should_skip_aligned_chunk_by_offset( - ChunkMeta* time_cm, ChunkMeta* value_cm) { + ChunkMeta* time_cm, ChunkMeta* value_cm, Filter* filter) { if (row_offset_ <= 0) { return false; } @@ -110,6 +116,11 @@ bool TsFileSeriesScanIterator::should_skip_aligned_chunk_by_offset( time_cm->statistic_ == nullptr || value_cm->statistic_ == nullptr) { return false; } + if (filter != nullptr && + !filter->contain_start_end_time(time_cm->statistic_->start_time_, + time_cm->statistic_->end_time_)) { + return false; + } int32_t tc = time_cm->statistic_->count_; int32_t vc = value_cm->statistic_->count_; if (tc <= 0 || vc <= 0 || tc != vc) { @@ -123,7 +134,8 @@ bool TsFileSeriesScanIterator::should_skip_aligned_chunk_by_offset( } bool TsFileSeriesScanIterator::should_skip_multi_aligned_chunk_by_offset( - ChunkMeta* time_cm, const std::vector& value_cms) { + ChunkMeta* time_cm, const std::vector& value_cms, + Filter* filter) { if (row_offset_ <= 0) { return false; } @@ -134,6 +146,11 @@ bool TsFileSeriesScanIterator::should_skip_multi_aligned_chunk_by_offset( if (time_count <= 0) { return false; } + if (filter != nullptr && + !filter->contain_start_end_time(time_cm->statistic_->start_time_, + time_cm->statistic_->end_time_)) { + return false; + } for (const auto* value_cm : value_cms) { if (value_cm == nullptr || value_cm->statistic_ == nullptr || value_cm->statistic_->count_ != time_count) { @@ -183,8 +200,8 @@ int TsFileSeriesScanIterator::get_next(TsBlock*& ret_tsblock, bool alloc, if (should_skip_chunk_by_time(time_cm, min_time_hint)) { continue; } - if (should_skip_multi_aligned_chunk_by_offset(time_cm, - value_cms)) { + if (should_skip_multi_aligned_chunk_by_offset( + time_cm, value_cms, filter)) { continue; } chunk_reader_->reset(); @@ -205,7 +222,7 @@ int TsFileSeriesScanIterator::get_next(TsBlock*& ret_tsblock, bool alloc, continue; } // Single-path: skip entire chunk by offset using count. - if (should_skip_chunk_by_offset(cm)) { + if (should_skip_chunk_by_offset(cm, filter)) { continue; } chunk_reader_->reset(); @@ -226,8 +243,8 @@ int TsFileSeriesScanIterator::get_next(TsBlock*& ret_tsblock, bool alloc, if (should_skip_chunk_by_time(filter_cm, min_time_hint)) { continue; } - if (should_skip_aligned_chunk_by_offset(time_cm, - value_cm)) { + if (should_skip_aligned_chunk_by_offset(time_cm, value_cm, + filter)) { continue; } chunk_reader_->reset(); diff --git a/cpp/src/reader/tsfile_series_scan_iterator.h b/cpp/src/reader/tsfile_series_scan_iterator.h index 187e9d9965..d0e2dbfee1 100644 --- a/cpp/src/reader/tsfile_series_scan_iterator.h +++ b/cpp/src/reader/tsfile_series_scan_iterator.h @@ -164,11 +164,13 @@ class TsFileSeriesScanIterator { return chunk_meta_cursor_.get(); } bool should_skip_chunk_by_time(ChunkMeta* cm, int64_t min_time_hint); - bool should_skip_chunk_by_offset(ChunkMeta* cm); + bool should_skip_chunk_by_offset(ChunkMeta* cm, Filter* filter); bool should_skip_aligned_chunk_by_offset(ChunkMeta* time_cm, - ChunkMeta* value_cm); + ChunkMeta* value_cm, + Filter* filter); bool should_skip_multi_aligned_chunk_by_offset( - ChunkMeta* time_cm, const std::vector& value_cms); + ChunkMeta* time_cm, const std::vector& value_cms, + Filter* filter); common::TsBlock* alloc_tsblock(); common::TsBlock* alloc_tsblock_multi(); diff --git a/cpp/test/reader/tsfile_reader_test.cc b/cpp/test/reader/tsfile_reader_test.cc index 5d5a29322e..80c43e208d 100644 --- a/cpp/test/reader/tsfile_reader_test.cc +++ b/cpp/test/reader/tsfile_reader_test.cc @@ -677,6 +677,192 @@ TEST_F(TsFileReaderTest, MultiValueAlignedRowOffsetSkipsWholeChunk) { io_reader.revert_ssi(ssi); } +// Offset is defined over the rows left after filtering. A partially matching +// chunk therefore cannot consume its full time-row count from the offset. +TEST_F(TsFileReaderTest, MultiValueAlignedRowOffsetCountsOnlyFilteredRows) { + const std::string device = "root.dev_multi_filtered_offset"; + std::vector schema_vec; + schema_vec.emplace_back("v0", INT64, PLAIN, UNCOMPRESSED); + schema_vec.emplace_back("v1", INT64, PLAIN, UNCOMPRESSED); + { + std::vector reg; + for (auto& s : schema_vec) reg.push_back(new MeasurementSchema(s)); + ASSERT_EQ(tsfile_writer_->register_aligned_timeseries(device, reg), + E_OK); + } + + const int rows_per_chunk = 64; + const int chunk_count = 4; + for (int chunk = 0; chunk < chunk_count; ++chunk) { + Tablet tablet( + device, + std::make_shared>(schema_vec), + rows_per_chunk); + const int base = chunk * rows_per_chunk; + for (int i = 0; i < rows_per_chunk; ++i) { + const int row = base + i; + ASSERT_EQ(tablet.add_timestamp(i, static_cast(row)), E_OK); + ASSERT_EQ(tablet.add_value(i, 0u, static_cast(row)), E_OK); + ASSERT_EQ(tablet.add_value(i, 1u, static_cast(row * 2)), + E_OK); + } + ASSERT_EQ(tsfile_writer_->write_tablet_aligned(tablet), E_OK); + ASSERT_EQ(tsfile_writer_->flush(), E_OK); + } + ASSERT_EQ(tsfile_writer_->close(), E_OK); + + storage::TsFileIOReader io_reader; + ASSERT_EQ(io_reader.init(file_name_), E_OK); + + auto device_id = std::make_shared(device); + std::vector measurements = {"v0", "v1"}; + storage::TsFileSeriesScanIterator* ssi = nullptr; + common::PageArena pa; + pa.init(512, common::MOD_TSFILE_READER); + + // Keep all 64 rows from chunk 0, only 10 rows from chunk 1, and all 128 + // rows from chunks 2 and 3. The filtered stream has 202 rows, so offset + // 200 must leave only timestamps 254 and 255. + std::vector selected_times; + for (int64_t t = 0; t < 64; ++t) selected_times.push_back(t); + for (int64_t t = 64; t < 74; ++t) selected_times.push_back(t); + for (int64_t t = 128; t < 256; ++t) selected_times.push_back(t); + storage::TimeIn time_filter(selected_times, /*not_in=*/false); + + ASSERT_EQ(io_reader.alloc_multi_ssi(device_id, measurements, ssi, pa, + &time_filter), + E_OK); + ASSERT_NE(ssi, nullptr); + ssi->set_row_range(/*offset=*/200, /*limit=*/-1); + + std::vector actual_times; + while (true) { + common::TsBlock* block = nullptr; + int ret = ssi->get_next(block, /*alloc_tsblock=*/true, &time_filter); + if (ret == common::E_NO_MORE_DATA) break; + ASSERT_EQ(ret, common::E_OK); + ASSERT_NE(block, nullptr); + + const uint32_t rows = block->get_row_count(); + const int remaining_offset = ssi->get_row_offset(); + const uint32_t rows_to_skip = std::min( + rows, static_cast(std::max(remaining_offset, 0))); + { + common::ColIterator time_iter(0, block); + for (uint32_t row = 0; row < rows; ++row) { + uint32_t len = 0; + int64_t time = + *reinterpret_cast(time_iter.read(&len)); + if (row >= rows_to_skip) actual_times.push_back(time); + time_iter.next(); + } + } + ssi->set_row_range(remaining_offset - static_cast(rows_to_skip), + /*limit=*/-1); + ssi->revert_tsblock(); + } + + ASSERT_EQ(actual_times.size(), 2u); + EXPECT_EQ(actual_times[0], 254); + EXPECT_EQ(actual_times[1], 255); + + io_reader.revert_ssi(ssi); +} + +// The single-value SSI paths must apply offset to the filtered stream too. +// Cover both a non-aligned series (ChunkReader) and a single-value aligned +// series (AlignedChunkReader), including their chunk- and page-level skips. +TEST_F(TsFileReaderTest, SingleValueRowOffsetCountsOnlyFilteredRows) { + const std::string non_aligned_device = "root.dev_filtered_offset"; + const std::string aligned_device = "root.dev_aligned_filtered_offset"; + MeasurementSchema schema("v0", INT64, PLAIN, UNCOMPRESSED); + ASSERT_EQ(tsfile_writer_->register_timeseries(non_aligned_device, schema), + E_OK); + ASSERT_EQ( + tsfile_writer_->register_aligned_timeseries(aligned_device, schema), + E_OK); + + const int rows_per_chunk = 64; + const int chunk_count = 4; + auto schemas = std::make_shared>(1, schema); + for (int chunk = 0; chunk < chunk_count; ++chunk) { + const int base = chunk * rows_per_chunk; + Tablet non_aligned_tablet(non_aligned_device, schemas, rows_per_chunk); + Tablet aligned_tablet(aligned_device, schemas, rows_per_chunk); + for (int i = 0; i < rows_per_chunk; ++i) { + const int row = base + i; + ASSERT_EQ(non_aligned_tablet.add_timestamp(i, row), E_OK); + ASSERT_EQ( + non_aligned_tablet.add_value(i, 0u, static_cast(row)), + E_OK); + ASSERT_EQ(aligned_tablet.add_timestamp(i, row), E_OK); + ASSERT_EQ( + aligned_tablet.add_value(i, 0u, static_cast(row)), + E_OK); + } + ASSERT_EQ(tsfile_writer_->write_tablet(non_aligned_tablet), E_OK); + ASSERT_EQ(tsfile_writer_->write_tablet_aligned(aligned_tablet), E_OK); + ASSERT_EQ(tsfile_writer_->flush(), E_OK); + } + ASSERT_EQ(tsfile_writer_->close(), E_OK); + + std::vector selected_times; + for (int64_t t = 0; t < 64; ++t) selected_times.push_back(t); + for (int64_t t = 64; t < 74; ++t) selected_times.push_back(t); + for (int64_t t = 128; t < 256; ++t) selected_times.push_back(t); + storage::TimeIn time_filter(selected_times, /*not_in=*/false); + + storage::TsFileIOReader io_reader; + ASSERT_EQ(io_reader.init(file_name_), E_OK); + for (const std::string& device : {non_aligned_device, aligned_device}) { + SCOPED_TRACE(device); + auto device_id = std::make_shared(device); + storage::TsFileSeriesScanIterator* ssi = nullptr; + common::PageArena pa; + pa.init(512, common::MOD_TSFILE_READER); + ASSERT_EQ(io_reader.alloc_ssi(device_id, "v0", ssi, pa, &time_filter), + E_OK); + ASSERT_NE(ssi, nullptr); + ssi->set_row_range(/*offset=*/200, /*limit=*/-1); + + std::vector actual_times; + while (true) { + common::TsBlock* block = nullptr; + int ret = + ssi->get_next(block, /*alloc_tsblock=*/true, &time_filter); + if (ret == common::E_NO_MORE_DATA) break; + ASSERT_EQ(ret, common::E_OK); + ASSERT_NE(block, nullptr); + + const uint32_t rows = block->get_row_count(); + const int remaining_offset = ssi->get_row_offset(); + const uint32_t rows_to_skip = std::min( + rows, static_cast(std::max(remaining_offset, 0))); + { + common::ColIterator time_iter(0, block); + for (uint32_t row = 0; row < rows; ++row) { + uint32_t len = 0; + int64_t time = + *reinterpret_cast(time_iter.read(&len)); + if (row >= rows_to_skip) actual_times.push_back(time); + time_iter.next(); + } + } + ssi->set_row_range( + remaining_offset - static_cast(rows_to_skip), + /*limit=*/-1); + ssi->revert_tsblock(); + } + + EXPECT_EQ(actual_times.size(), 2u); + if (actual_times.size() == 2u) { + EXPECT_EQ(actual_times[0], 254); + EXPECT_EQ(actual_times[1], 255); + } + io_reader.revert_ssi(ssi); + } +} + namespace storage { // Subclass that lets the test (a) inject an error from the next-tsblock load // and (b) wire a manually constructed TsBlock into the inherited iterator From f488f713a1b8faea259c542d6ef19bfbd080887a Mon Sep 17 00:00:00 2001 From: ColinLee Date: Mon, 20 Jul 2026 12:20:05 +0800 Subject: [PATCH 3/5] refactor(cpp): separate filter and offset skip checks --- cpp/src/reader/aligned_chunk_reader.cc | 47 +++++++++------ cpp/src/reader/aligned_chunk_reader.h | 10 ++-- cpp/src/reader/chunk_reader.cc | 23 +++---- cpp/src/reader/chunk_reader.h | 5 +- cpp/src/reader/tsfile_series_scan_iterator.cc | 60 +++++++++---------- cpp/src/reader/tsfile_series_scan_iterator.h | 8 +-- 6 files changed, 82 insertions(+), 71 deletions(-) diff --git a/cpp/src/reader/aligned_chunk_reader.cc b/cpp/src/reader/aligned_chunk_reader.cc index 02008081a5..94d4915cee 100644 --- a/cpp/src/reader/aligned_chunk_reader.cc +++ b/cpp/src/reader/aligned_chunk_reader.cc @@ -356,7 +356,7 @@ int AlignedChunkReader::get_next_page(TsBlock* ret_tsblock, value_chunk_meta_, value_in_stream_, cur_value_page_header_, value_chunk_visit_offset_, value_chunk_header_))) { - } else if (cur_page_statisify_filter(filter)) { + } else if (cur_page_may_satisfy_filter(filter)) { break; } else if (RET_FAIL(skip_cur_page())) { } @@ -470,7 +470,7 @@ int AlignedChunkReader::read_from_file_and_rewrap( return ret; } -bool AlignedChunkReader::cur_page_statisify_filter(Filter* filter) { +bool AlignedChunkReader::cur_page_may_satisfy_filter(Filter* filter) { bool value_satisfy = filter == nullptr || cur_value_page_header_.statistic_ == nullptr || filter->satisfy(cur_value_page_header_.statistic_); @@ -480,6 +480,16 @@ bool AlignedChunkReader::cur_page_statisify_filter(Filter* filter) { return time_satisfy && value_satisfy; } +bool AlignedChunkReader::cur_page_fully_satisfies_filter(Filter* filter) { + Statistic* stat = cur_time_page_header_.statistic_; + if (stat == nullptr) { + stat = cur_value_page_header_.statistic_; + } + return filter == nullptr || + (stat != nullptr && + filter->contain_start_end_time(stat->start_time_, stat->end_time_)); +} + int AlignedChunkReader::skip_cur_page() { int ret = E_OK; // visit a page tv data @@ -1060,8 +1070,7 @@ bool AlignedChunkReader::should_skip_page_by_time(int64_t min_time_hint) { return false; } -bool AlignedChunkReader::should_skip_page_by_offset(int& row_offset, - Filter* filter) { +bool AlignedChunkReader::should_skip_page_by_offset(int& row_offset) { if (row_offset <= 0) { return false; } @@ -1073,10 +1082,6 @@ bool AlignedChunkReader::should_skip_page_by_offset(int& row_offset, if (stat == nullptr || stat->count_ == 0) { return false; } - if (filter != nullptr && - !filter->contain_start_end_time(stat->start_time_, stat->end_time_)) { - return false; - } int32_t count = stat->count_; if (row_offset >= count) { row_offset -= count; @@ -1128,13 +1133,14 @@ int AlignedChunkReader::get_next_page(TsBlock* ret_tsblock, value_chunk_meta_, value_in_stream_, cur_value_page_header_, value_chunk_visit_offset_, value_chunk_header_))) { - } else if (!cur_page_statisify_filter(filter)) { + } else if (!cur_page_may_satisfy_filter(filter)) { if (RET_FAIL(skip_cur_page())) { } } else if (should_skip_page_by_time(min_time_hint)) { if (RET_FAIL(skip_cur_page())) { } - } else if (should_skip_page_by_offset(row_offset, filter)) { + } else if (cur_page_fully_satisfies_filter(filter) && + should_skip_page_by_offset(row_offset)) { if (RET_FAIL(skip_cur_page())) { } } else { @@ -2086,9 +2092,10 @@ int AlignedChunkReader::get_next_page_multi_serial(TsBlock* ret_tsblock, } } if (IS_FAIL(ret)) break; - if (!cur_page_statisify_filter_multi(filter)) { + if (!cur_page_may_satisfy_filter_multi(filter)) { if (RET_FAIL(skip_cur_page_multi())) break; - } else if (should_skip_page_by_offset_multi(row_offset, filter)) { + } else if (cur_page_fully_satisfies_filter_multi(filter) && + should_skip_page_by_offset_multi(row_offset)) { if (RET_FAIL(skip_cur_page_multi())) break; } else { break; @@ -2110,15 +2117,21 @@ int AlignedChunkReader::get_next_page_multi_serial(TsBlock* ret_tsblock, return ret; } -bool AlignedChunkReader::cur_page_statisify_filter_multi(Filter* filter) { +bool AlignedChunkReader::cur_page_may_satisfy_filter_multi(Filter* filter) { bool time_satisfy = filter == nullptr || cur_time_page_header_.statistic_ == nullptr || filter->satisfy(cur_time_page_header_.statistic_); return time_satisfy; } -bool AlignedChunkReader::should_skip_page_by_offset_multi(int& row_offset, - Filter* filter) { +bool AlignedChunkReader::cur_page_fully_satisfies_filter_multi(Filter* filter) { + Statistic* stat = cur_time_page_header_.statistic_; + return filter == nullptr || + (stat != nullptr && + filter->contain_start_end_time(stat->start_time_, stat->end_time_)); +} + +bool AlignedChunkReader::should_skip_page_by_offset_multi(int& row_offset) { if (row_offset <= 0) { return false; } @@ -2126,10 +2139,6 @@ bool AlignedChunkReader::should_skip_page_by_offset_multi(int& row_offset, if (stat == nullptr || stat->count_ == 0) { return false; } - if (filter != nullptr && - !filter->contain_start_end_time(stat->start_time_, stat->end_time_)) { - return false; - } int32_t count = stat->count_; if (row_offset >= count) { row_offset -= count; diff --git a/cpp/src/reader/aligned_chunk_reader.h b/cpp/src/reader/aligned_chunk_reader.h index ef0e65af2d..ee321cf0d3 100644 --- a/cpp/src/reader/aligned_chunk_reader.h +++ b/cpp/src/reader/aligned_chunk_reader.h @@ -172,7 +172,7 @@ class AlignedChunkReader : public IChunkReader { private: bool should_skip_page_by_time(int64_t min_time_hint); - bool should_skip_page_by_offset(int& row_offset, Filter* filter); + bool should_skip_page_by_offset(int& row_offset); FORCE_INLINE bool chunk_has_only_one_page( const ChunkHeader& chunk_header) const { return (chunk_header.chunk_type_ & ONLY_ONE_PAGE_CHUNK_HEADER_MARKER) == @@ -194,7 +194,8 @@ class AlignedChunkReader : public IChunkReader { uint32_t& chunk_visit_offset, int32_t& file_data_buf_size, int want_size = 0, bool may_shrink = true); - bool cur_page_statisify_filter(Filter* filter); + bool cur_page_may_satisfy_filter(Filter* filter); + bool cur_page_fully_satisfies_filter(Filter* filter); int skip_cur_page(); int decode_cur_time_page_data(); int decode_cur_value_page_data(); @@ -243,8 +244,9 @@ class AlignedChunkReader : public IChunkReader { int get_next_page_multi_serial(common::TsBlock* ret_tsblock, Filter* filter, common::PageArena& pa, int& row_offset); int skip_cur_page_multi(); - bool cur_page_statisify_filter_multi(Filter* filter); - bool should_skip_page_by_offset_multi(int& row_offset, Filter* filter); + bool cur_page_may_satisfy_filter_multi(Filter* filter); + bool cur_page_fully_satisfies_filter_multi(Filter* filter); + bool should_skip_page_by_offset_multi(int& row_offset); int decode_cur_value_pages_multi(); int decode_cur_value_page_data_for(ValueColumnState& col); int ensure_value_page_loaded(ValueColumnState& col); diff --git a/cpp/src/reader/chunk_reader.cc b/cpp/src/reader/chunk_reader.cc index cfc1a0c68e..83ba69493f 100644 --- a/cpp/src/reader/chunk_reader.cc +++ b/cpp/src/reader/chunk_reader.cc @@ -192,7 +192,7 @@ int ChunkReader::get_next_page(TsBlock* ret_tsblock, Filter* oneshoot_filter, return E_NO_MORE_DATA; } if (RET_FAIL(get_cur_page_header())) { - } else if (cur_page_statisify_filter(filter)) { + } else if (cur_page_may_satisfy_filter(filter)) { break; } else if (RET_FAIL(skip_cur_page())) { } @@ -265,11 +265,18 @@ int ChunkReader::read_from_file_and_rewrap(int want_size) { return ret; } -bool ChunkReader::cur_page_statisify_filter(Filter* filter) { +bool ChunkReader::cur_page_may_satisfy_filter(Filter* filter) { return filter == nullptr || cur_page_header_.statistic_ == nullptr || filter->satisfy(cur_page_header_.statistic_); } +bool ChunkReader::cur_page_fully_satisfies_filter(Filter* filter) { + return filter == nullptr || (cur_page_header_.statistic_ != nullptr && + filter->contain_start_end_time( + cur_page_header_.statistic_->start_time_, + cur_page_header_.statistic_->end_time_)); +} + int ChunkReader::skip_cur_page() { int ret = E_OK; // visit a page tv data @@ -856,7 +863,7 @@ bool ChunkReader::should_skip_page_by_time(int64_t min_time_hint) { return cur_page_header_.statistic_->end_time_ < min_time_hint; } -bool ChunkReader::should_skip_page_by_offset(int& row_offset, Filter* filter) { +bool ChunkReader::should_skip_page_by_offset(int& row_offset) { if (row_offset <= 0) { return false; } @@ -864,11 +871,6 @@ bool ChunkReader::should_skip_page_by_offset(int& row_offset, Filter* filter) { cur_page_header_.statistic_->count_ == 0) { return false; } - if (filter != nullptr && !filter->contain_start_end_time( - cur_page_header_.statistic_->start_time_, - cur_page_header_.statistic_->end_time_)) { - return false; - } int32_t count = cur_page_header_.statistic_->count_; if (row_offset >= count) { row_offset -= count; @@ -909,13 +911,14 @@ int ChunkReader::get_next_page(TsBlock* ret_tsblock, Filter* oneshoot_filter, return E_NO_MORE_DATA; } if (RET_FAIL(get_cur_page_header())) { - } else if (!cur_page_statisify_filter(filter)) { + } else if (!cur_page_may_satisfy_filter(filter)) { if (RET_FAIL(skip_cur_page())) { } } else if (should_skip_page_by_time(min_time_hint)) { if (RET_FAIL(skip_cur_page())) { } - } else if (should_skip_page_by_offset(row_offset, filter)) { + } else if (cur_page_fully_satisfies_filter(filter) && + should_skip_page_by_offset(row_offset)) { if (RET_FAIL(skip_cur_page())) { } } else { diff --git a/cpp/src/reader/chunk_reader.h b/cpp/src/reader/chunk_reader.h index 6b872ad7db..b92be81699 100644 --- a/cpp/src/reader/chunk_reader.h +++ b/cpp/src/reader/chunk_reader.h @@ -76,7 +76,7 @@ class ChunkReader : public IChunkReader { private: bool should_skip_page_by_time(int64_t min_time_hint); - bool should_skip_page_by_offset(int& row_offset, Filter* filter); + bool should_skip_page_by_offset(int& row_offset); FORCE_INLINE bool chunk_has_only_one_page() const { return (chunk_header_.chunk_type_ & ONLY_ONE_PAGE_CHUNK_HEADER_MARKER) == @@ -87,7 +87,8 @@ class ChunkReader : public IChunkReader { common::CompressionType compression_type); int get_cur_page_header(); int read_from_file_and_rewrap(int want_size = 0); - bool cur_page_statisify_filter(Filter* filter); + bool cur_page_may_satisfy_filter(Filter* filter); + bool cur_page_fully_satisfies_filter(Filter* filter); int skip_cur_page(); int decode_cur_page_data(common::TsBlock*& ret_tsblock, Filter* filter, common::PageArena& pa); diff --git a/cpp/src/reader/tsfile_series_scan_iterator.cc b/cpp/src/reader/tsfile_series_scan_iterator.cc index dfc51c9f94..79ac8d308b 100644 --- a/cpp/src/reader/tsfile_series_scan_iterator.cc +++ b/cpp/src/reader/tsfile_series_scan_iterator.cc @@ -30,6 +30,21 @@ using namespace common; namespace storage { +namespace { +bool chunk_may_satisfy_filter(ChunkMeta* chunk_meta, Filter* filter) { + return filter == nullptr || chunk_meta == nullptr || + chunk_meta->statistic_ == nullptr || + filter->satisfy(chunk_meta->statistic_); +} + +bool chunk_fully_satisfies_filter(ChunkMeta* chunk_meta, Filter* filter) { + return filter == nullptr || + (chunk_meta != nullptr && chunk_meta->statistic_ != nullptr && + filter->contain_start_end_time(chunk_meta->statistic_->start_time_, + chunk_meta->statistic_->end_time_)); +} +} // namespace + void TsFileSeriesScanIterator::destroy() { // MultiAlignedTimeseriesIndex is placement-new'd inside // timeseries_index_pa_ (see TsFileIOReader::alloc_multi_ssi). The arena's @@ -79,19 +94,13 @@ bool TsFileSeriesScanIterator::should_skip_chunk_by_time( return cm->statistic_->end_time_ < min_time_hint; } -bool TsFileSeriesScanIterator::should_skip_chunk_by_offset(ChunkMeta* cm, - Filter* filter) { +bool TsFileSeriesScanIterator::should_skip_chunk_by_offset(ChunkMeta* cm) { if (row_offset_ <= 0) { return false; } if (cm->statistic_ == nullptr || cm->statistic_->count_ == 0) { return false; } - if (filter != nullptr && - !filter->contain_start_end_time(cm->statistic_->start_time_, - cm->statistic_->end_time_)) { - return false; - } int32_t count = cm->statistic_->count_; if (row_offset_ >= count) { row_offset_ -= count; @@ -101,7 +110,7 @@ bool TsFileSeriesScanIterator::should_skip_chunk_by_offset(ChunkMeta* cm, } bool TsFileSeriesScanIterator::should_skip_aligned_chunk_by_offset( - ChunkMeta* time_cm, ChunkMeta* value_cm, Filter* filter) { + ChunkMeta* time_cm, ChunkMeta* value_cm) { if (row_offset_ <= 0) { return false; } @@ -116,11 +125,6 @@ bool TsFileSeriesScanIterator::should_skip_aligned_chunk_by_offset( time_cm->statistic_ == nullptr || value_cm->statistic_ == nullptr) { return false; } - if (filter != nullptr && - !filter->contain_start_end_time(time_cm->statistic_->start_time_, - time_cm->statistic_->end_time_)) { - return false; - } int32_t tc = time_cm->statistic_->count_; int32_t vc = value_cm->statistic_->count_; if (tc <= 0 || vc <= 0 || tc != vc) { @@ -134,8 +138,7 @@ bool TsFileSeriesScanIterator::should_skip_aligned_chunk_by_offset( } bool TsFileSeriesScanIterator::should_skip_multi_aligned_chunk_by_offset( - ChunkMeta* time_cm, const std::vector& value_cms, - Filter* filter) { + ChunkMeta* time_cm, const std::vector& value_cms) { if (row_offset_ <= 0) { return false; } @@ -146,11 +149,6 @@ bool TsFileSeriesScanIterator::should_skip_multi_aligned_chunk_by_offset( if (time_count <= 0) { return false; } - if (filter != nullptr && - !filter->contain_start_end_time(time_cm->statistic_->start_time_, - time_cm->statistic_->end_time_)) { - return false; - } for (const auto* value_cm : value_cms) { if (value_cm == nullptr || value_cm->statistic_ == nullptr || value_cm->statistic_->count_ != time_count) { @@ -193,15 +191,15 @@ int TsFileSeriesScanIterator::get_next(TsBlock*& ret_tsblock, bool alloc, } advance_to_next_chunk(); // Skip chunk by time filter using time chunk statistics. - if (filter != nullptr && time_cm->statistic_ != nullptr && - !filter->satisfy(time_cm->statistic_)) { + if (!chunk_may_satisfy_filter(time_cm, filter)) { continue; } if (should_skip_chunk_by_time(time_cm, min_time_hint)) { continue; } - if (should_skip_multi_aligned_chunk_by_offset( - time_cm, value_cms, filter)) { + if (chunk_fully_satisfies_filter(time_cm, filter) && + should_skip_multi_aligned_chunk_by_offset(time_cm, + value_cms)) { continue; } chunk_reader_->reset(); @@ -213,8 +211,7 @@ int TsFileSeriesScanIterator::get_next(TsBlock*& ret_tsblock, bool alloc, } else if (!is_aligned_) { ChunkMeta* cm = get_current_chunk_meta(); advance_to_next_chunk(); - if (filter != nullptr && cm->statistic_ != nullptr && - !filter->satisfy(cm->statistic_)) { + if (!chunk_may_satisfy_filter(cm, filter)) { continue; } // Skip by min_time_hint (merge cursor). @@ -222,7 +219,8 @@ int TsFileSeriesScanIterator::get_next(TsBlock*& ret_tsblock, bool alloc, continue; } // Single-path: skip entire chunk by offset using count. - if (should_skip_chunk_by_offset(cm, filter)) { + if (chunk_fully_satisfies_filter(cm, filter) && + should_skip_chunk_by_offset(cm)) { continue; } chunk_reader_->reset(); @@ -236,15 +234,15 @@ int TsFileSeriesScanIterator::get_next(TsBlock*& ret_tsblock, bool alloc, // Use time chunk statistics for time-based filtering. ChunkMeta* filter_cm = (time_cm->statistic_ != nullptr) ? time_cm : value_cm; - if (filter != nullptr && filter_cm->statistic_ != nullptr && - !filter->satisfy(filter_cm->statistic_)) { + if (!chunk_may_satisfy_filter(filter_cm, filter)) { continue; } if (should_skip_chunk_by_time(filter_cm, min_time_hint)) { continue; } - if (should_skip_aligned_chunk_by_offset(time_cm, value_cm, - filter)) { + if (chunk_fully_satisfies_filter(time_cm, filter) && + should_skip_aligned_chunk_by_offset(time_cm, + value_cm)) { continue; } chunk_reader_->reset(); diff --git a/cpp/src/reader/tsfile_series_scan_iterator.h b/cpp/src/reader/tsfile_series_scan_iterator.h index d0e2dbfee1..187e9d9965 100644 --- a/cpp/src/reader/tsfile_series_scan_iterator.h +++ b/cpp/src/reader/tsfile_series_scan_iterator.h @@ -164,13 +164,11 @@ class TsFileSeriesScanIterator { return chunk_meta_cursor_.get(); } bool should_skip_chunk_by_time(ChunkMeta* cm, int64_t min_time_hint); - bool should_skip_chunk_by_offset(ChunkMeta* cm, Filter* filter); + bool should_skip_chunk_by_offset(ChunkMeta* cm); bool should_skip_aligned_chunk_by_offset(ChunkMeta* time_cm, - ChunkMeta* value_cm, - Filter* filter); + ChunkMeta* value_cm); bool should_skip_multi_aligned_chunk_by_offset( - ChunkMeta* time_cm, const std::vector& value_cms, - Filter* filter); + ChunkMeta* time_cm, const std::vector& value_cms); common::TsBlock* alloc_tsblock(); common::TsBlock* alloc_tsblock_multi(); From 6a3ce310f67ab62aa302a0ff6501bf6bf43d8d90 Mon Sep 17 00:00:00 2001 From: ColinLee Date: Mon, 20 Jul 2026 15:11:54 +0800 Subject: [PATCH 4/5] perf(cpp): handle aligned offsets in page plans --- cpp/src/reader/aligned_chunk_reader.cc | 66 ++++++---- cpp/src/reader/aligned_chunk_reader.h | 10 +- .../block/single_device_tsblock_reader.cc | 14 +- .../block/single_device_tsblock_reader.h | 7 +- cpp/src/reader/chunk_reader.cc | 3 + cpp/src/reader/tsfile_series_scan_iterator.h | 4 + cpp/test/reader/tsfile_reader_test.cc | 122 +++++++++++++++--- 7 files changed, 163 insertions(+), 63 deletions(-) diff --git a/cpp/src/reader/aligned_chunk_reader.cc b/cpp/src/reader/aligned_chunk_reader.cc index 94d4915cee..4b89e659bb 100644 --- a/cpp/src/reader/aligned_chunk_reader.cc +++ b/cpp/src/reader/aligned_chunk_reader.cc @@ -77,7 +77,6 @@ void AlignedChunkReader::reset() { page_plan_built_ = false; current_page_loaded_ = false; current_page_plan_index_ = 0; - multi_serial_decode_current_chunk_ = false; time_predecoded_ = false; page_all_times_.clear(); page_time_count_ = 0; @@ -470,6 +469,9 @@ int AlignedChunkReader::read_from_file_and_rewrap( return ret; } +// Page statistics provide two levels of certainty: "may satisfy" means the +// page cannot be ruled out, while "fully satisfies" means every row is known +// to match and the page count can safely be used for offset pushdown. bool AlignedChunkReader::cur_page_may_satisfy_filter(Filter* filter) { bool value_satisfy = filter == nullptr || cur_value_page_header_.statistic_ == nullptr || @@ -1173,7 +1175,6 @@ int AlignedChunkReader::load_by_aligned_meta_multi( page_plan_built_ = false; current_page_loaded_ = false; current_page_plan_index_ = 0; - multi_serial_decode_current_chunk_ = false; time_predecoded_ = false; page_all_times_.clear(); page_time_count_ = 0; @@ -1400,7 +1401,7 @@ int AlignedChunkReader::decode_time_page_with(const ChunkPageInfo& page_info, return ret; } -int AlignedChunkReader::build_page_plan(Filter* filter) { +int AlignedChunkReader::build_page_plan(Filter* filter, int& row_offset) { int ret = E_OK; chunk_pages_.clear(); current_page_plan_index_ = 0; @@ -1476,6 +1477,13 @@ int AlignedChunkReader::build_page_plan(Filter* filter) { int32_t last = -1; for (int32_t i = 0; i < static_cast(times.size()); i++) { if (filter->satisfy_start_end_time(times[i], times[i])) { + // Offset is defined on the filtered row stream. Consume + // matching rows only; holes inside a boundary page do not + // count toward it. + if (row_offset > 0) { + row_offset--; + continue; + } if (first < 0) first = i; last = i; } @@ -1496,6 +1504,18 @@ int AlignedChunkReader::build_page_plan(Filter* filter) { } page_info.row_end = static_cast(times.size()); } + if (page_info.pass_type == PagePassType::FULL_PASS && + row_offset > 0) { + const int32_t page_row_count = + page_info.row_end - page_info.row_begin; + const int32_t rows_to_skip = + std::min(page_row_count, row_offset); + page_info.row_begin += rows_to_skip; + row_offset -= rows_to_skip; + if (page_info.row_begin == page_info.row_end) { + page_info.pass_type = PagePassType::SKIP; + } + } if (page_info.row_begin < page_info.row_end) { chunk_pages_.push_back(std::move(page_info)); } @@ -1855,34 +1875,31 @@ int AlignedChunkReader::get_next_page_multi(TsBlock* ret_tsblock, Filter* filter = (oneshoot_filter != nullptr ? oneshoot_filter : time_filter_); - if (row_offset > 0) { - multi_serial_decode_current_chunk_ = true; - } - // Dispatch: - // - Multi-column with a thread pool → chunk-level pre-decode: one task - // per value column decodes that column's whole chunk up front, then the - // scatter loop bulk-memcpys. decode_all_planned_pages() works for any - // column count. (An earlier cutoff sent >6 columns down the serial - // path because per_page_state — the upfront predecode buffer — grows - // with column count and was feared to thrash cache; it still grows, so - // very wide aligned chunks are the case to watch if reads regress.) - // - Single column, or no thread pool → serial path: decode the current - // page's columns inline (multi_DECODE_TV_BATCH), no thread-pool - // fan-out. + // - Offset pushdown always uses the page plan so fully covered pages and + // the prefix of the first retained page can be removed before value + // decoding, even when no worker pool is available. + // - Multi-column with a thread pool also uses the page plan for + // chunk-level parallel pre-decode. + // - Otherwise decode the current page's columns inline. #ifdef ENABLE_THREADS - const bool use_chunk_level = decode_pool_ != nullptr && - value_columns_.size() > 1 && - !multi_serial_decode_current_chunk_; + const bool use_parallel_page_plan = + decode_pool_ != nullptr && value_columns_.size() > 1; #else - const bool use_chunk_level = false; + const bool use_parallel_page_plan = false; #endif - if (!use_chunk_level) { + const bool serial_page_in_progress = + !page_plan_built_ && + (prev_time_page_not_finish() || prev_any_value_page_not_finish_multi()); + const bool use_page_plan = + page_plan_built_ || (!serial_page_in_progress && + (row_offset > 0 || use_parallel_page_plan)); + if (!use_page_plan) { return get_next_page_multi_serial(ret_tsblock, filter, pa, row_offset); } if (!page_plan_built_) { - if (RET_FAIL(build_page_plan(filter))) { + if (RET_FAIL(build_page_plan(filter, row_offset))) { return ret; } if (RET_FAIL(decode_all_planned_pages())) { @@ -2117,6 +2134,9 @@ int AlignedChunkReader::get_next_page_multi_serial(TsBlock* ret_tsblock, return ret; } +// Keep the same conservative distinction as the single-value path: a page +// that may satisfy the filter still needs row-level filtering unless its full +// time range is covered by the filter. bool AlignedChunkReader::cur_page_may_satisfy_filter_multi(Filter* filter) { bool time_satisfy = filter == nullptr || cur_time_page_header_.statistic_ == nullptr || diff --git a/cpp/src/reader/aligned_chunk_reader.h b/cpp/src/reader/aligned_chunk_reader.h index ee321cf0d3..45dd2ef999 100644 --- a/cpp/src/reader/aligned_chunk_reader.h +++ b/cpp/src/reader/aligned_chunk_reader.h @@ -36,7 +36,7 @@ class ThreadPool; namespace storage { -// Page classification for chunk-level parallel decode. +// Page classification for chunk-level planned decode. enum class PagePassType { SKIP, FULL_PASS, BOUNDARY }; // Metadata collected per page during the chunk scan phase. @@ -53,8 +53,9 @@ struct ChunkPageInfo { std::vector value_uncompressed_sizes; }; -// Decoded state for one (column, page) slot. Populated by chunk-level -// parallel decode; consumed by the scatter loop. +// Decoded state for one (column, page) slot. Populated eagerly by parallel +// decode or lazily by the single-thread page-plan path, then consumed by the +// scatter loop. struct PageDecodedState { std::vector notnull_bitmap; std::vector predecoded_values; @@ -259,7 +260,7 @@ class AlignedChunkReader : public IChunkReader { int multi_DECODE_TV_BATCH(common::TsBlock* ret_tsblock, common::RowAppender& row_appender, Filter* filter, common::PageArena* pa); - int build_page_plan(Filter* filter); + int build_page_plan(Filter* filter, int& row_offset); int decode_time_page_direct(const ChunkPageInfo& page_info, std::vector& out_times); int decode_time_page_with(const ChunkPageInfo& page_info, @@ -322,7 +323,6 @@ class AlignedChunkReader : public IChunkReader { bool page_plan_built_ = false; bool current_page_loaded_ = false; size_t current_page_plan_index_ = 0; - bool multi_serial_decode_current_chunk_ = false; #ifdef ENABLE_THREADS common::ThreadPool* decode_pool_ = nullptr; // borrowed, not owned diff --git a/cpp/src/reader/block/single_device_tsblock_reader.cc b/cpp/src/reader/block/single_device_tsblock_reader.cc index 0b547efac9..79874d79ea 100644 --- a/cpp/src/reader/block/single_device_tsblock_reader.cc +++ b/cpp/src/reader/block/single_device_tsblock_reader.cc @@ -297,6 +297,8 @@ int SingleDeviceTsBlockReader::init_internal(DeviceQueryTask* device_query_task, ssi_offset, -1)) { if (ssi_offset > 0) { row_offset_pushed_to_ssi_ = true; + // init() prefetches the first TsBlock and may consume part + // of the offset by skipping whole chunks/pages. remaining_offset_ = ctx->get_ssi_row_offset(); } // The shared ctx is referenced from N map entries; close() @@ -996,11 +998,7 @@ int SingleMeasurementColumnContext::skip_rows(uint32_t count) { const uint32_t val_elem_size = common::get_data_type_size(dt); value_iter_->advance(to_skip, val_elem_size); } - int ssi_offset = get_ssi_row_offset(); - if (ssi_offset > 0) { - set_ssi_row_range(std::max(0, ssi_offset - static_cast(to_skip)), - get_ssi_row_limit()); - } + consume_ssi_row_offset(static_cast(to_skip)); if (time_iter_->end()) { // Propagate hard errors from the next-tsblock load; E_NO_MORE_DATA // is the legitimate end-of-stream signal and gets squashed back to @@ -1254,11 +1252,7 @@ int VectorMeasurementColumnContext::skip_rows(uint32_t count) { } } } - int ssi_offset = get_ssi_row_offset(); - if (ssi_offset > 0) { - set_ssi_row_range(std::max(0, ssi_offset - static_cast(to_skip)), - get_ssi_row_limit()); - } + consume_ssi_row_offset(static_cast(to_skip)); if (time_iter_->end()) { int r = get_next_tsblock(false); if (r != common::E_OK && r != common::E_NO_MORE_DATA) return r; diff --git a/cpp/src/reader/block/single_device_tsblock_reader.h b/cpp/src/reader/block/single_device_tsblock_reader.h index cccc7fd3fd..eb6e81a5de 100644 --- a/cpp/src/reader/block/single_device_tsblock_reader.h +++ b/cpp/src/reader/block/single_device_tsblock_reader.h @@ -115,14 +115,11 @@ class MeasurementColumnContext { virtual int move_iter() = 0; - virtual void set_ssi_row_range(int offset, int limit) { - if (ssi_) ssi_->set_row_range(offset, limit); - } virtual int get_ssi_row_offset() const { return ssi_ ? ssi_->get_row_offset() : 0; } - virtual int get_ssi_row_limit() const { - return ssi_ ? ssi_->get_row_limit() : -1; + virtual void consume_ssi_row_offset(int count) { + if (ssi_) ssi_->consume_row_offset(count); } virtual uint32_t available_rows() const = 0; diff --git a/cpp/src/reader/chunk_reader.cc b/cpp/src/reader/chunk_reader.cc index 83ba69493f..6b3d853d94 100644 --- a/cpp/src/reader/chunk_reader.cc +++ b/cpp/src/reader/chunk_reader.cc @@ -265,6 +265,9 @@ int ChunkReader::read_from_file_and_rewrap(int want_size) { return ret; } +// Page statistics provide two levels of certainty: "may satisfy" means the +// page cannot be ruled out, while "fully satisfies" means every row is known +// to match and the page count can safely be used for offset pushdown. bool ChunkReader::cur_page_may_satisfy_filter(Filter* filter) { return filter == nullptr || cur_page_header_.statistic_ == nullptr || filter->satisfy(cur_page_header_.statistic_); diff --git a/cpp/src/reader/tsfile_series_scan_iterator.h b/cpp/src/reader/tsfile_series_scan_iterator.h index 187e9d9965..f8b88b4927 100644 --- a/cpp/src/reader/tsfile_series_scan_iterator.h +++ b/cpp/src/reader/tsfile_series_scan_iterator.h @@ -20,6 +20,7 @@ #ifndef READER_TSFILE_SERIES_SCAN_ITERATOR_H #define READER_TSFILE_SERIES_SCAN_ITERATOR_H +#include #include #include @@ -81,6 +82,9 @@ class TsFileSeriesScanIterator { * for single-path. */ int get_row_offset() const { return row_offset_; } int get_row_limit() const { return row_limit_; } + void consume_row_offset(int count) { + row_offset_ = std::max(0, row_offset_ - count); + } /* * If oneshoot filter specified, use it instead of this->time_filter_. diff --git a/cpp/test/reader/tsfile_reader_test.cc b/cpp/test/reader/tsfile_reader_test.cc index 80c43e208d..4e5aa420b4 100644 --- a/cpp/test/reader/tsfile_reader_test.cc +++ b/cpp/test/reader/tsfile_reader_test.cc @@ -463,10 +463,10 @@ TEST_F(TsFileReaderTest, reader.close(); } -// Multi-value aligned chunk reader consumes row_offset at chunk/page -// granularity. If the remaining offset lands inside the current page, the SSI -// leaves that residual for the row reader to consume from the returned TsBlock. -TEST_F(TsFileReaderTest, MultiValueAlignedRowOffsetKeepsPartialPageResidual) { +// The multi-value page plan consumes row_offset before value decoding. When +// the offset lands inside a page, the returned TsBlock starts at the first row +// after the offset instead of exposing a residual to the row reader. +TEST_F(TsFileReaderTest, MultiValueAlignedRowOffsetTrimsPartialPage) { const std::string device = "root.dev_multi_offset"; std::vector schema_vec; schema_vec.emplace_back("v0", INT64, PLAIN, UNCOMPRESSED); @@ -507,8 +507,17 @@ TEST_F(TsFileReaderTest, MultiValueAlignedRowOffsetKeepsPartialPageResidual) { common::TsBlock* block = nullptr; EXPECT_EQ(ssi->get_next(block, /*alloc_tsblock=*/true), common::E_OK); ASSERT_NE(block, nullptr); - EXPECT_EQ(block->get_row_count(), static_cast(N)); - EXPECT_EQ(ssi->get_row_offset(), 5); + EXPECT_EQ(block->get_row_count(), static_cast(N - 5)); + EXPECT_EQ(ssi->get_row_offset(), 0); + { + common::ColIterator time_iter(0, block); + common::ColIterator v0_iter(1, block); + common::ColIterator v1_iter(2, block); + uint32_t len = 0; + EXPECT_EQ(*reinterpret_cast(time_iter.read(&len)), 1005); + EXPECT_EQ(*reinterpret_cast(v0_iter.read(&len)), 5); + EXPECT_EQ(*reinterpret_cast(v1_iter.read(&len)), 10); + } if (block != nullptr) { ssi->revert_tsblock(); @@ -569,24 +578,23 @@ TEST_F(TsFileReaderTest, MultiValueAlignedRowOffsetSkipsWholePage) { E_OK); ASSERT_NE(ssi, nullptr); - // The first chunk is preloaded before set_row_range(). This offset must be - // consumed at page granularity: skip page 0 and leave 4 rows for the row - // reader to skip inside page 1. + // The first chunk is preloaded before set_row_range(). The page plan skips + // page 0 and trims the first four rows from page 1 before value decoding. ssi->set_row_range(/*offset=*/20, /*limit=*/-1); common::TsBlock* block = nullptr; ASSERT_EQ(ssi->get_next(block, /*alloc_tsblock=*/true), common::E_OK); ASSERT_NE(block, nullptr); - ASSERT_EQ(block->get_row_count(), static_cast(rows_per_page)); - EXPECT_EQ(ssi->get_row_offset(), 4); + ASSERT_EQ(block->get_row_count(), static_cast(N - 20)); + EXPECT_EQ(ssi->get_row_offset(), 0); { common::ColIterator time_iter(0, block); common::ColIterator v0_iter(1, block); common::ColIterator v1_iter(2, block); uint32_t len = 0; - EXPECT_EQ(*reinterpret_cast(time_iter.read(&len)), 16); - EXPECT_EQ(*reinterpret_cast(v0_iter.read(&len)), 16); - EXPECT_EQ(*reinterpret_cast(v1_iter.read(&len)), 32); + EXPECT_EQ(*reinterpret_cast(time_iter.read(&len)), 20); + EXPECT_EQ(*reinterpret_cast(v0_iter.read(&len)), 20); + EXPECT_EQ(*reinterpret_cast(v1_iter.read(&len)), 40); } ssi->revert_tsblock(); @@ -640,7 +648,7 @@ TEST_F(TsFileReaderTest, MultiValueAlignedRowOffsetSkipsWholeChunk) { // alloc_multi_ssi() preloads chunk 0 before row range is set. Read it // normally first, then set an offset that skips the next whole chunk by - // chunk metadata count and leaves a 32-row residual inside chunk 2. + // chunk metadata count and trims the first 32 rows from chunk 2's plan. common::TsBlock* block = nullptr; ASSERT_EQ(ssi->get_next(block, /*alloc_tsblock=*/true), common::E_OK); ASSERT_NE(block, nullptr); @@ -660,17 +668,18 @@ TEST_F(TsFileReaderTest, MultiValueAlignedRowOffsetSkipsWholeChunk) { block = nullptr; ASSERT_EQ(ssi->get_next(block, /*alloc_tsblock=*/true), common::E_OK); ASSERT_NE(block, nullptr); - ASSERT_EQ(block->get_row_count(), static_cast(rows_per_chunk)); - EXPECT_EQ(ssi->get_row_offset(), 32); + ASSERT_EQ(block->get_row_count(), + static_cast(rows_per_chunk / 2)); + EXPECT_EQ(ssi->get_row_offset(), 0); { common::ColIterator time_iter(0, block); common::ColIterator v0_iter(1, block); common::ColIterator v1_iter(2, block); uint32_t len = 0; - EXPECT_EQ(*reinterpret_cast(time_iter.read(&len)), 128); - EXPECT_EQ(*reinterpret_cast(v0_iter.read(&len)), 128); - EXPECT_EQ(*reinterpret_cast(v1_iter.read(&len)), 256); + EXPECT_EQ(*reinterpret_cast(time_iter.read(&len)), 160); + EXPECT_EQ(*reinterpret_cast(v0_iter.read(&len)), 160); + EXPECT_EQ(*reinterpret_cast(v1_iter.read(&len)), 320); } ssi->revert_tsblock(); @@ -769,6 +778,79 @@ TEST_F(TsFileReaderTest, MultiValueAlignedRowOffsetCountsOnlyFilteredRows) { io_reader.revert_ssi(ssi); } +// A boundary page may contain holes between matching timestamps. Offset must +// consume only matching rows and keep every value column aligned with the +// first retained match. +TEST_F(TsFileReaderTest, MultiValueAlignedPagePlanOffsetSkipsBoundaryMatches) { + const std::string device = "root.dev_multi_boundary_offset"; + std::vector schema_vec; + schema_vec.emplace_back("v0", INT64, PLAIN, UNCOMPRESSED); + schema_vec.emplace_back("v1", INT64, PLAIN, UNCOMPRESSED); + { + std::vector reg; + for (auto& s : schema_vec) reg.push_back(new MeasurementSchema(s)); + ASSERT_EQ(tsfile_writer_->register_aligned_timeseries(device, reg), + E_OK); + } + + constexpr int kRowCount = 16; + Tablet tablet(device, + std::make_shared>(schema_vec), + kRowCount); + for (int i = 0; i < kRowCount; ++i) { + ASSERT_EQ(tablet.add_timestamp(i, i), E_OK); + ASSERT_EQ(tablet.add_value(i, 0u, static_cast(i)), E_OK); + ASSERT_EQ(tablet.add_value(i, 1u, static_cast(i * 10)), E_OK); + } + ASSERT_EQ(tsfile_writer_->write_tablet_aligned(tablet), E_OK); + ASSERT_EQ(tsfile_writer_->flush(), E_OK); + ASSERT_EQ(tsfile_writer_->close(), E_OK); + + storage::TsFileIOReader io_reader; + ASSERT_EQ(io_reader.init(file_name_), E_OK); + + auto device_id = std::make_shared(device); + std::vector measurements = {"v0", "v1"}; + storage::TsFileSeriesScanIterator* ssi = nullptr; + common::PageArena pa; + pa.init(512, common::MOD_TSFILE_READER); + const std::vector selected_times = {1, 3, 5, 7, 9}; + storage::TimeIn time_filter(selected_times, /*not_in=*/false); + ASSERT_EQ(io_reader.alloc_multi_ssi(device_id, measurements, ssi, pa, + &time_filter), + E_OK); + ASSERT_NE(ssi, nullptr); + + ssi->set_row_range(/*offset=*/2, /*limit=*/-1); + common::TsBlock* block = nullptr; + ASSERT_EQ(ssi->get_next(block, /*alloc_tsblock=*/true, &time_filter), E_OK); + ASSERT_NE(block, nullptr); + EXPECT_EQ(ssi->get_row_offset(), 0); + ASSERT_EQ(block->get_row_count(), 3u); + + const std::vector expected_times = {5, 7, 9}; + { + common::ColIterator time_iter(0, block); + common::ColIterator v0_iter(1, block); + common::ColIterator v1_iter(2, block); + for (int64_t expected : expected_times) { + uint32_t len = 0; + EXPECT_EQ(*reinterpret_cast(time_iter.read(&len)), + expected); + EXPECT_EQ(*reinterpret_cast(v0_iter.read(&len)), + expected); + EXPECT_EQ(*reinterpret_cast(v1_iter.read(&len)), + expected * 10); + time_iter.next(); + v0_iter.next(); + v1_iter.next(); + } + } + + ssi->revert_tsblock(); + io_reader.revert_ssi(ssi); +} + // The single-value SSI paths must apply offset to the filtered stream too. // Cover both a non-aligned series (ChunkReader) and a single-value aligned // series (AlignedChunkReader), including their chunk- and page-level skips. From 473941b3391d6b11a2fe1305539e8d9157921b61 Mon Sep 17 00:00:00 2001 From: ColinLee Date: Mon, 20 Jul 2026 16:49:10 +0800 Subject: [PATCH 5/5] fix(cpp): harden offset skip handling --- .../block/single_device_tsblock_reader.cc | 33 ++++++++----------- .../block/single_device_tsblock_reader.h | 2 +- cpp/src/reader/tsfile_series_scan_iterator.h | 6 ++-- cpp/test/reader/tsfile_reader_test.cc | 17 ++++++++++ 4 files changed, 36 insertions(+), 22 deletions(-) diff --git a/cpp/src/reader/block/single_device_tsblock_reader.cc b/cpp/src/reader/block/single_device_tsblock_reader.cc index 79874d79ea..5d66bb7e9e 100644 --- a/cpp/src/reader/block/single_device_tsblock_reader.cc +++ b/cpp/src/reader/block/single_device_tsblock_reader.cc @@ -164,15 +164,6 @@ int SingleDeviceTsBlockReader::init_internal(DeviceQueryTask* device_query_task, } } time_column_index_ = 0; - if (RET_FAIL(common::TsBlock::create_tsblock(&tuple_desc_, current_block_, - block_size))) { - return ret; - } - col_appenders_.resize(tuple_desc_.get_column_count()); - for (uint32_t i = 0; i < tuple_desc_.get_column_count(); i++) { - col_appenders_[i] = new common::ColAppender(i, current_block_); - } - row_appender_ = new common::RowAppender(current_block_); std::vector time_series_indexs( device_query_task_->get_column_mapping() ->get_measurement_columns() @@ -187,8 +178,6 @@ int SingleDeviceTsBlockReader::init_internal(DeviceQueryTask* device_query_task, if (time_filter == nullptr && field_filter == nullptr && dense_row_count_ >= 0 && remaining_offset_ >= dense_row_count_) { remaining_offset_ -= dense_row_count_; - delete current_block_; - current_block_ = nullptr; return common::E_OK; } // Early device-level time skip: if time_filter is set and ALL chunks of @@ -226,8 +215,6 @@ int SingleDeviceTsBlockReader::init_internal(DeviceQueryTask* device_query_task, } if (examined_any && all_outside) { // No data in this device matches the time filter. - delete current_block_; - current_block_ = nullptr; return common::E_OK; } } @@ -347,8 +334,6 @@ int SingleDeviceTsBlockReader::init_internal(DeviceQueryTask* device_query_task, } } if (any_value_column_requested) { - delete current_block_; - current_block_ = nullptr; return common::E_OK; } @@ -369,8 +354,6 @@ int SingleDeviceTsBlockReader::init_internal(DeviceQueryTask* device_query_task, // propagate so the caller sees the actual failure instead of // an empty resultset wearing E_OK. if (time_only_ret != common::E_NO_MORE_DATA) { - delete current_block_; - current_block_ = nullptr; return time_only_ret; } } @@ -394,11 +377,23 @@ int SingleDeviceTsBlockReader::init_internal(DeviceQueryTask* device_query_task, } if (field_column_contexts_.empty()) { - delete current_block_; - current_block_ = nullptr; return common::E_OK; } + // Metadata and context initialization above can prove that a device has no + // rows to return. Delay result-buffer allocation until those early exits + // have been ruled out, so skipped devices never create appenders that + // briefly point at a deleted TsBlock. + if (RET_FAIL(common::TsBlock::create_tsblock(&tuple_desc_, current_block_, + block_size))) { + return ret; + } + col_appenders_.resize(tuple_desc_.get_column_count()); + for (uint32_t i = 0; i < tuple_desc_.get_column_count(); i++) { + col_appenders_[i] = new common::ColAppender(i, current_block_); + } + row_appender_ = new common::RowAppender(current_block_); + for (const auto& id_column : device_query_task->get_column_mapping()->get_id_columns()) { const auto& column_pos_in_result = diff --git a/cpp/src/reader/block/single_device_tsblock_reader.h b/cpp/src/reader/block/single_device_tsblock_reader.h index eb6e81a5de..4bf0cca0fb 100644 --- a/cpp/src/reader/block/single_device_tsblock_reader.h +++ b/cpp/src/reader/block/single_device_tsblock_reader.h @@ -74,7 +74,7 @@ class SingleDeviceTsBlockReader : public TsBlockReader { uint32_t block_size_; common::TsBlock* current_block_ = nullptr; std::vector col_appenders_; - common::RowAppender* row_appender_; + common::RowAppender* row_appender_ = nullptr; common::TupleDesc tuple_desc_; bool last_block_returned_ = true; std::map field_column_contexts_; diff --git a/cpp/src/reader/tsfile_series_scan_iterator.h b/cpp/src/reader/tsfile_series_scan_iterator.h index f8b88b4927..636ccac8c0 100644 --- a/cpp/src/reader/tsfile_series_scan_iterator.h +++ b/cpp/src/reader/tsfile_series_scan_iterator.h @@ -20,7 +20,6 @@ #ifndef READER_TSFILE_SERIES_SCAN_ITERATOR_H #define READER_TSFILE_SERIES_SCAN_ITERATOR_H -#include #include #include @@ -83,7 +82,10 @@ class TsFileSeriesScanIterator { int get_row_offset() const { return row_offset_; } int get_row_limit() const { return row_limit_; } void consume_row_offset(int count) { - row_offset_ = std::max(0, row_offset_ - count); + if (count <= 0 || row_offset_ <= 0) { + return; + } + row_offset_ = count >= row_offset_ ? 0 : row_offset_ - count; } /* diff --git a/cpp/test/reader/tsfile_reader_test.cc b/cpp/test/reader/tsfile_reader_test.cc index 4e5aa420b4..f7df9c8c92 100644 --- a/cpp/test/reader/tsfile_reader_test.cc +++ b/cpp/test/reader/tsfile_reader_test.cc @@ -42,6 +42,23 @@ using namespace storage; using namespace common; +TEST(TsFileSeriesScanIteratorTest, ConsumeRowOffsetSaturates) { + storage::TsFileSeriesScanIterator ssi; + ssi.set_row_range(/*offset=*/10, /*limit=*/-1); + + ssi.consume_row_offset(-1); + EXPECT_EQ(ssi.get_row_offset(), 10); + ssi.consume_row_offset(std::numeric_limits::min()); + EXPECT_EQ(ssi.get_row_offset(), 10); + + ssi.consume_row_offset(4); + EXPECT_EQ(ssi.get_row_offset(), 6); + ssi.consume_row_offset(6); + EXPECT_EQ(ssi.get_row_offset(), 0); + ssi.consume_row_offset(1); + EXPECT_EQ(ssi.get_row_offset(), 0); +} + class TsFileReaderTest : public ::testing::Test { protected: void SetUp() override {