diff --git a/cpp/src/common/container/byte_buffer.h b/cpp/src/common/container/byte_buffer.h index 3f4954093..a3b2c525f 100644 --- a/cpp/src/common/container/byte_buffer.h +++ b/cpp/src/common/container/byte_buffer.h @@ -133,6 +133,14 @@ class ByteBuffer { FORCE_INLINE uint32_t get_data_size() const { return real_data_size_; } + // Declare bytes that were written directly through get_data() without going + // through append_*(). Used when a caller decodes straight into this buffer + // at a known offset. + FORCE_INLINE void set_data_size(uint32_t size) { + ASSERT(size <= reserved_size_); + real_data_size_ = size; + } + private: char* data_; uint8_t variable_type_len_; diff --git a/cpp/src/reader/aligned_chunk_reader.cc b/cpp/src/reader/aligned_chunk_reader.cc index a0a0b2306..f104e1d7a 100644 --- a/cpp/src/reader/aligned_chunk_reader.cc +++ b/cpp/src/reader/aligned_chunk_reader.cc @@ -117,6 +117,7 @@ void AlignedChunkReader::reset() { col->chunk_meta = nullptr; for (auto& pps : col->per_page_state) { pps.predecode_pa.destroy(); + pps.direct_values = nullptr; } col->per_page_state.clear(); col->pending_decoded_values.clear(); @@ -129,6 +130,9 @@ void AlignedChunkReader::reset() { release_current_page_state(); chunk_pages_.clear(); per_page_times_.clear(); +#ifdef ENABLE_TEST + bulk_page_count_ = 0; +#endif } void AlignedChunkReader::destroy() { @@ -138,6 +142,7 @@ void AlignedChunkReader::destroy() { // LeakSanitizer doesn't flag the (rather large) ChunkPageInfo buffers. std::vector{}.swap(chunk_pages_); std::vector{}.swap(page_all_times_); + std::vector{}.swap(direct_col_off_); if (time_uncompressed_buf_ != nullptr && time_compressor_ != nullptr) { time_compressor_->after_uncompress(time_uncompressed_buf_); time_uncompressed_buf_ = nullptr; @@ -202,6 +207,7 @@ void AlignedChunkReader::destroy() { } for (auto& pps : col->per_page_state) { pps.predecode_pa.destroy(); + pps.direct_values = nullptr; } col->per_page_state.clear(); col->pending_decoded_values.clear(); @@ -1687,6 +1693,11 @@ int AlignedChunkReader::build_page_plan(Filter* filter, int& row_offset) { page_plan_built_ = IS_SUCC(ret); if (page_plan_built_) { + int32_t out_row = 0; + for (auto& page : chunk_pages_) { + page.out_row_begin = out_row; + out_row += page.row_end - page.row_begin; + } per_page_times_.assign(chunk_pages_.size(), std::vector{}); for (auto* col : value_columns_) { col->per_page_state.clear(); @@ -1701,6 +1712,10 @@ void AlignedChunkReader::release_current_page_state() { page_all_times_.clear(); page_time_count_ = 0; page_time_cursor_ = 0; + direct_page_end_ = 0; + direct_tsblock_ = nullptr; + direct_base_row_ = 0; + direct_col_off_.clear(); for (auto* col : value_columns_) { if (col->uncompressed_buf != nullptr && col->compressor != nullptr) { col->compressor->after_uncompress(col->uncompressed_buf); @@ -1711,6 +1726,7 @@ void AlignedChunkReader::release_current_page_state() { col->in.reset(); for (auto& pps : col->per_page_state) { pps.predecode_pa.destroy(); + pps.direct_values = nullptr; } col->per_page_state.clear(); col->pending_decoded_values.clear(); @@ -1734,6 +1750,7 @@ int AlignedChunkReader::decode_value_page_for_slot(uint32_t col_idx, pps.predecoded_strings.clear(); pps.predecoded_read_pos = 0; pps.predecoded_count = 0; + pps.direct_values = nullptr; pps.predecode_pa.destroy(); if (page_info.value_compressed_sizes[col_idx] == 0) { @@ -1848,11 +1865,28 @@ int AlignedChunkReader::decode_value_page_for_slot(uint32_t col_idx, } uint32_t elem_size = common::get_data_type_size(dt); - pps.predecoded_values.resize(static_cast(nonnull_total) * - elem_size); + char* values_dst = nullptr; + // Direct decode: prepare_direct_decode() reserved this page's output rows + // in the destination Vector before decoding started, and a page without + // nulls maps one decoded value to one row, so the value stream can be + // written in place. Pages that carry nulls, or whose retained range is only + // part of the page, keep their own staging buffer (the row/scatter code + // copies those). + if (page_idx < direct_page_end_ && page_info.row_begin == 0 && + page_info.row_end == static_cast(data_num) && + nonnull_total == static_cast(data_num)) { + common::Vector* vec = direct_tsblock_->get_vector(col_idx + 1); + values_dst = + vec->get_value_data().get_data() + direct_col_off_[col_idx]; + pps.direct_values = values_dst; + } else { + pps.predecoded_values.resize(static_cast(nonnull_total) * + elem_size); + values_dst = pps.predecoded_values.data(); + } switch (dt) { case common::BOOLEAN: { - bool* out = reinterpret_cast(pps.predecoded_values.data()); + bool* out = reinterpret_cast(values_dst); for (int i = 0; i < nonnull_total; i++) { if (RET_FAIL(col->decoder->read_boolean(out[i], in))) { cleanup(); @@ -1864,8 +1898,8 @@ int AlignedChunkReader::decode_value_page_for_slot(uint32_t col_idx, case common::INT32: case common::DATE: if (RET_FAIL(col->decoder->read_exact_int32( - reinterpret_cast(pps.predecoded_values.data()), - nonnull_total, in))) { + reinterpret_cast(values_dst), nonnull_total, + in))) { cleanup(); return ret; } @@ -1873,24 +1907,23 @@ int AlignedChunkReader::decode_value_page_for_slot(uint32_t col_idx, case common::INT64: case common::TIMESTAMP: if (RET_FAIL(col->decoder->read_exact_int64( - reinterpret_cast(pps.predecoded_values.data()), - nonnull_total, in))) { + reinterpret_cast(values_dst), nonnull_total, + in))) { cleanup(); return ret; } break; case common::FLOAT: if (RET_FAIL(col->decoder->read_exact_float( - reinterpret_cast(pps.predecoded_values.data()), - nonnull_total, in))) { + reinterpret_cast(values_dst), nonnull_total, in))) { cleanup(); return ret; } break; case common::DOUBLE: if (RET_FAIL(col->decoder->read_exact_double( - reinterpret_cast(pps.predecoded_values.data()), - nonnull_total, in))) { + reinterpret_cast(values_dst), nonnull_total, + in))) { cleanup(); return ret; } @@ -1900,10 +1933,121 @@ int AlignedChunkReader::decode_value_page_for_slot(uint32_t col_idx, return E_NOT_SUPPORT; } pps.predecoded_count = nonnull_total; + if (page_idx < direct_page_end_) { + // Reserve in the destination column exactly the values the scatter + // will emit for this page: the non-null ones in [row_begin, row_end). + // A page that could not be written in place (nulls, or a retained + // range shorter than the page) is appended from its staging buffer + // instead, so advancing by the whole page's non-null count would push + // the offsets of the following pages too far ahead. + int values_in_range = nonnull_total; + if (pps.direct_values == nullptr) { + const int up_to_end = + count_non_null_prefix(pps.notnull_bitmap, page_info.row_end); + const int up_to_begin = + count_non_null_prefix(pps.notnull_bitmap, page_info.row_begin); + if (up_to_end < 0 || up_to_begin < 0) { + cleanup(); + return E_TSFILE_CORRUPTED; + } + values_in_range = up_to_end - up_to_begin; + } + direct_col_off_[col_idx] += + static_cast(values_in_range) * elem_size; + } cleanup(); return E_OK; } +// Decide how many leading pages of the plan can be decoded straight into the +// destination TsBlock's column buffers. The destination address of a row must +// be fixed before decode starts, because parallel decode runs ahead of the +// scatter loop, so the pages written in place must all be scattered within the +// same call — that is exactly the prefix of pages whose output rows still fit +// in the rows available in the current TsBlock (the scatter consumes pages in +// order and only swaps blocks once the budget is exhausted). +// - no FIELD filter: a page-level mask can drop rows and shift the output; +// - a contiguous time filter: a page's retained rows stay one contiguous run; +// - FULL_PASS pages starting at row 0. A page whose decoded stream starts +// before its output slice (partial range / offset pushdown) would write +// before the destination base, so the direct prefix stops there; +// - fixed-length columns whose type matches the destination Vector. +// In lazy (single-thread) mode pages are decoded during the scatter itself, so +// a partial prefix cannot be used: the block may be swapped between pages +// before they are decoded. There, the whole plan must fit. +bool AlignedChunkReader::prepare_direct_decode(TsBlock* tsblock, + Filter* filter) { + direct_page_end_ = 0; + direct_tsblock_ = nullptr; + direct_base_row_ = 0; + direct_col_off_.clear(); + if (field_filter_ != nullptr || tsblock == nullptr || + chunk_pages_.empty()) { + return false; + } + if (filter != nullptr && !filter->is_contiguous_time_filter()) { + return false; + } + if (has_variable_length_value_column()) { + return false; + } + const uint32_t num_cols = value_columns_.size(); + if (num_cols == 0 || tsblock->get_column_count() < num_cols + 1) { + return false; + } + for (uint32_t c = 0; c < num_cols; c++) { + if (value_columns_[c]->chunk_header.data_type_ != + tsblock->get_vector(c + 1)->get_vector_type()) { + return false; + } + } + const int64_t base_row = static_cast(tsblock->get_row_count()); + const int64_t available = + static_cast(tsblock->get_max_row_count()) - base_row; + size_t end = 0; + for (size_t i = 0; i < chunk_pages_.size(); i++) { + const auto& page = chunk_pages_[i]; + // A page whose retained range starts at row 0 and covers the whole page + // (checked again after decode, once the row count is known) writes its + // values in order. That includes conservatively classified BOUNDARY + // pages: when the time predicate is contiguous, [row_begin, row_end) + // holds exactly the matching rows, so an all-covering range is + // equivalent to FULL_PASS even without page statistics. + if (page.row_begin != 0) { + break; + } + const int64_t page_end = + static_cast(page.out_row_begin) + page.row_end; + if (page_end > available) { + break; + } + end = i + 1; + } + if (end == 0) { + return false; + } + // decode_all_planned_pages() keeps a partial prefix only when it decodes + // every page up front; mirror that decision here. +#ifdef ENABLE_THREADS + const bool eager_decode = + decode_pool_ != nullptr && value_columns_.size() > 1; +#else + const bool eager_decode = false; +#endif + if (!eager_decode && end != chunk_pages_.size()) { + return false; + } + direct_page_end_ = end; + direct_base_row_ = static_cast(base_row); + direct_tsblock_ = tsblock; + direct_col_off_.assign(num_cols, 0); + for (uint32_t c = 0; c < num_cols; c++) { + direct_col_off_[c] = + tsblock->get_vector(c + 1)->get_value_data().get_data_size(); + } + return true; +} + // Multi-thread path: one task per value column, each decoding all non-SKIP // pages of that column serially. Time pages dispatched as worker-bucketed // strided tasks using a separate decoder/compressor for each task (filled @@ -2061,6 +2205,7 @@ void AlignedChunkReader::release_page_slot(size_t page_idx) { pps.predecode_pa.destroy(); pps.predecoded_count = 0; pps.predecoded_read_pos = 0; + pps.direct_values = nullptr; } } @@ -2133,6 +2278,7 @@ int AlignedChunkReader::get_next_page_multi_impl(TsBlock* ret_tsblock, if (RET_FAIL(build_page_plan(filter, row_offset))) { return ret; } + prepare_direct_decode(ret_tsblock, filter); if (RET_FAIL(decode_all_planned_pages())) { return ret; } @@ -2194,12 +2340,25 @@ int AlignedChunkReader::get_next_page_multi_impl(TsBlock* ret_tsblock, int32_t remaining_in_page = page_time_count_ - page_time_cursor_; uint32_t budget = row_appender.remaining(); - // Fast path: FULL_PASS page, no nulls in any value column, types - // match destination, budget > 0. Bulk-memcpys up to + // Fast path: FULL_PASS page, or a BOUNDARY page under a contiguous + // time filter (e.g. TimeBetween) whose whole [row_begin, row_end] + // range matches — no nulls in any value column, types match + // destination, budget > 0. Bulk-memcpys up to // min(budget, remaining_in_page) rows from page_time_cursor_; tail // pages of an SSI tsblock still take the memcpy path instead of - // falling into the row-by-row scatter loop. - bool can_bulk = page_info.pass_type == PagePassType::FULL_PASS && + // falling into the row-by-row scatter loop. Discrete filters (TimeIn, + // NOT BETWEEN) keep the slow path because the compressed range can + // still contain unmatched timestamps. + // A BOUNDARY page may be bounded by the time filter or by a FIELD + // predicate mask. Only the former keeps every row of + // [row_begin, row_end): a field mask can still reject rows inside that + // range, so those pages must stay on the row-by-row path. + const bool boundary_contiguous = + page_info.pass_type == PagePassType::BOUNDARY && + field_filter_ == nullptr && + (filter == nullptr || filter->is_contiguous_time_filter()); + bool can_bulk = (page_info.pass_type == PagePassType::FULL_PASS || + boundary_contiguous) && remaining_in_page > 0 && budget > 0; if (can_bulk) { for (uint32_t c = 0; c < num_cols; c++) { @@ -2209,7 +2368,14 @@ int AlignedChunkReader::get_next_page_multi_impl(TsBlock* ret_tsblock, if (dt == common::STRING || dt == common::TEXT || dt == common::BLOB || ret_tsblock->get_vector(c + 1)->get_vector_type() != dt || - pps.predecoded_count != page_time_count_) { + pps.predecoded_count != pps.row_count || + // The decoded buffer must cover every row about to be + // copied. A column without a value page for this page (a + // missing aligned measurement) decodes to an empty slot + // with row_count == 0 while the plan still expects rows; + // those rows are all null and the row-by-row path marks + // them as such. + pps.row_count < static_cast(remaining_in_page)) { can_bulk = false; break; } @@ -2238,13 +2404,30 @@ int AlignedChunkReader::get_next_page_multi_impl(TsBlock* ret_tsblock, uint32_t elem_size = common::get_data_type_size(col->chunk_header.data_type_); common::Vector* vec = ret_tsblock->get_vector(c + 1); - vec->get_value_data().append_fixed_value( - pps.predecoded_values.data() + - static_cast(page_time_cursor_) * elem_size, - bulk_count * elem_size); + // A directly decoded page already sits in the destination + // buffer; only declare its bytes. Everything else is appended, + // which also keeps the column's compact layout (null rows take + // no space) in sync. + if (pps.direct_values == nullptr) { + vec->get_value_data().append_fixed_value( + pps.values() + + static_cast(page_time_cursor_) * elem_size, + static_cast(bulk_count) * elem_size); + } else { + vec->get_value_data().set_data_size( + vec->get_value_data().get_data_size() + + static_cast(bulk_count) * elem_size); + } vec->add_row_nums(bulk_count); } row_appender.add_rows(bulk_count); +#ifdef ENABLE_TEST + // One count per page, not per bulk operation: a page is copied in + // several operations when the destination block fills up first. + if (page_time_cursor_ == page_info.row_begin) { + ++bulk_page_count_; + } +#endif page_time_cursor_ += bulk_count; if (page_time_cursor_ >= page_time_count_) { if (single_thread_lazy) { @@ -2324,12 +2507,22 @@ int AlignedChunkReader::get_next_page_multi_impl(TsBlock* ret_tsblock, } else { uint32_t elem_size = common::get_data_type_size( col->chunk_header.data_type_); - row_appender.append( - c + 1, - pps.predecoded_values.data() + - static_cast(pps.predecoded_read_pos++) * - elem_size, - elem_size); + if (pps.direct_values != nullptr) { + // Decode already placed this value in the destination + // buffer; only declare it, do not copy bytes onto + // themselves. + common::Vector* vec = ret_tsblock->get_vector(c + 1); + vec->get_value_data().set_data_size( + vec->get_value_data().get_data_size() + elem_size); + pps.predecoded_read_pos++; + } else { + row_appender.append( + c + 1, + pps.values() + + static_cast(pps.predecoded_read_pos++) * + elem_size, + elem_size); + } } } page_time_cursor_++; diff --git a/cpp/src/reader/aligned_chunk_reader.h b/cpp/src/reader/aligned_chunk_reader.h index 54c82799f..176c0b020 100644 --- a/cpp/src/reader/aligned_chunk_reader.h +++ b/cpp/src/reader/aligned_chunk_reader.h @@ -49,6 +49,9 @@ struct ChunkPageInfo { uint32_t time_uncompressed_size = 0; int32_t row_begin = 0; // inclusive int32_t row_end = 0; // exclusive + // Row index of this page's first retained row in the chunk reader's output + // stream. Filled by build_page_plan() once the whole plan is known. + int32_t out_row_begin = 0; std::vector value_file_offsets; std::vector value_compressed_sizes; std::vector value_uncompressed_sizes; @@ -65,6 +68,17 @@ struct PageDecodedState { common::PageArena predecode_pa; int32_t predecoded_count = 0; int32_t predecoded_read_pos = 0; + // Non-null when the page was decoded straight into the destination TsBlock + // column buffer (see AlignedChunkReader::prepare_direct_decode()). Aliases + // memory owned by the Vector; predecoded_values stays empty in that case. + char* direct_values = nullptr; + + // Where this slot's fixed-length values live: the direct view when present, + // otherwise the slot's own buffer. + char* values() { + return direct_values != nullptr ? direct_values + : predecoded_values.data(); + } }; // Per-value-column state for multi-value AlignedChunkReader. @@ -177,6 +191,15 @@ class AlignedChunkReader : public IChunkReader { bool is_multi_value_mode() const { return multi_value_mode_; } +#ifdef ENABLE_TEST + // Pages scattered by the bulk-copy path so far (FULL_PASS pages and + // BOUNDARY pages a contiguous time filter proves fully matched). + size_t TEST_bulk_page_count() const { return bulk_page_count_; } + // Leading pages of the current plan that decode straight into the + // destination block's column buffers (see prepare_direct_decode()). + size_t TEST_direct_page_end() const { return direct_page_end_; } +#endif + #ifdef ENABLE_THREADS // Set external thread pool for parallel decode (not owned). void set_decode_pool(common::ThreadPool* pool) { decode_pool_ = pool; } @@ -274,6 +297,7 @@ class AlignedChunkReader : public IChunkReader { common::RowAppender& row_appender, Filter* filter, common::PageArena* pa); int build_page_plan(Filter* filter, int& row_offset); + bool prepare_direct_decode(common::TsBlock* tsblock, Filter* filter); void build_filter_column_mask(); int evaluate_page_field_mask(size_t page_idx); int decode_output_columns_for_page(size_t page_idx); @@ -346,6 +370,22 @@ class AlignedChunkReader : public IChunkReader { SelectionMask current_page_field_mask_; bool current_page_field_mask_valid_ = false; + // Direct decode: pages [0, direct_page_end_) are decoded straight into the + // destination Vector memory instead of a per-page staging buffer. See + // prepare_direct_decode(). + size_t direct_page_end_ = 0; + int32_t direct_base_row_ = 0; + common::TsBlock* direct_tsblock_ = nullptr; + // Per value column: byte offset in the Vector's buffer where the next + // decoded value goes. TsBlock columns store non-null values compactly + // (RowIterator advances offsets only for non-null rows), so this advances + // by the non-null count of each page, not by its row index. + std::vector direct_col_off_; + +#ifdef ENABLE_TEST + size_t bulk_page_count_ = 0; +#endif + #ifdef ENABLE_THREADS common::ThreadPool* decode_pool_ = nullptr; // borrowed, not owned // Per-task time decoder + compressor pool for parallel time-page decode. diff --git a/cpp/src/reader/filter/filter.h b/cpp/src/reader/filter/filter.h index e53992308..b230854d5 100644 --- a/cpp/src/reader/filter/filter.h +++ b/cpp/src/reader/filter/filter.h @@ -64,6 +64,14 @@ class Filter { return nullptr; } + // True when this time filter selects one contiguous interval: inside a + // BOUNDARY page's [row_begin, row_end) range every timestamp matches (e.g. + // TimeBetween, one-sided Gt/Lt), so the range can be bulk-copied. Discrete + // sets (TimeIn) or negated forms (NOT BETWEEN / NotEq) may contain holes + // and must still be checked per value. Defaults to false; only explicitly + // contiguous time filters override it to true. + virtual bool is_contiguous_time_filter() const { return false; } + // Batch time filter: evaluate time filter on an array of timestamps. // Writes true/false into @mask for each element. // Returns the number of elements that passed (mask[i] == true). diff --git a/cpp/src/reader/filter/time_operator.h b/cpp/src/reader/filter/time_operator.h index f972a4259..b48ca7fba 100644 --- a/cpp/src/reader/filter/time_operator.h +++ b/cpp/src/reader/filter/time_operator.h @@ -52,6 +52,8 @@ class TimeBetween : public Filter { FilterType get_filter_type() { return type_; } + bool is_contiguous_time_filter() const override { return !not_; } + private: int64_t value1_; int64_t value2_; diff --git a/cpp/test/reader/aligned_chunk_reader_bulk_test.cc b/cpp/test/reader/aligned_chunk_reader_bulk_test.cc new file mode 100644 index 000000000..0be62def8 --- /dev/null +++ b/cpp/test/reader/aligned_chunk_reader_bulk_test.cc @@ -0,0 +1,409 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "common/allocator/alloc_base.h" +#include "common/config/config.h" +#include "common/tablet.h" +#include "common/tsblock/tsblock.h" +#include "file/local_random_access_read_file.h" +#include "file/tsfile_io_reader.h" +#include "file/write_file.h" +#include "reader/aligned_chunk_reader.h" +#include "reader/filter/aligned_field_filter.h" +#include "reader/filter/time_operator.h" +#include "writer/tsfile_table_writer.h" +#ifdef ENABLE_THREADS +#include "common/thread_pool.h" +#endif + +using namespace common; + +// A multi-column reader uses the page-plan path when a decode pool is set +// (what a parallel read does) or when an offset / FIELD filter is present. +// The no-filter bulk-copy and direct-decode tests need the pool, so they are +// skipped in a build without threads; the FIELD filter test reaches the plan +// either way. +#ifdef ENABLE_THREADS +#define SKIP_WITHOUT_THREADS() ((void)0) +#else +#define SKIP_WITHOUT_THREADS() \ + GTEST_SKIP() << "page-plan decode needs a decode pool (ENABLE_THREADS)" +#endif + +namespace storage { +namespace { + +// 8 rows per page and 40 rows per chunk: five pages, so a time window can start +// and end inside a page and still leave whole pages in between. +constexpr int kPagePoints = 8; +constexpr int kRowCount = 40; +constexpr int64_t kBaseTime = 1000; + +struct Row { + int64_t time = 0; + int64_t v = 0; + int64_t w = 0; + std::string s; +}; + +class AlignedBulkCopyTest : public ::testing::Test { + protected: + void SetUp() override { + libtsfile_init(); + saved_config_ = common::g_config_value_; + common::g_config_value_.page_writer_max_point_num_ = kPagePoints; + schema_ = std::make_shared( + "t", + std::vector{ + {"v", INT64, UNCOMPRESSED, PLAIN, ColumnCategory::FIELD}, + {"w", INT64, UNCOMPRESSED, PLAIN, ColumnCategory::FIELD}, + {"s", STRING, UNCOMPRESSED, PLAIN, ColumnCategory::FIELD}}); + // Cases of parallel runs (ctest -j, repeated runs) share a working + // directory, so the name carries the case and a wall-clock suffix. + path_ = + std::string("aligned_bulk_copy_") + + ::testing::UnitTest::GetInstance()->current_test_info()->name() + + "_" + + std::to_string( + std::chrono::steady_clock::now().time_since_epoch().count()) + + ".tsfile"; + std::remove(path_.c_str()); + { + WriteFile file; + ASSERT_EQ(file.create(path_, O_WRONLY | O_CREAT | O_TRUNC, 0666), + E_OK); + TsFileTableWriter writer(&file, schema_.get()); + Tablet tablet("t", {"v", "w", "s"}, {INT64, INT64, STRING}, + {ColumnCategory::FIELD, ColumnCategory::FIELD, + ColumnCategory::FIELD}, + kRowCount); + for (int r = 0; r < kRowCount; r++) { + ASSERT_EQ(tablet.add_timestamp(r, kBaseTime + r), E_OK); + ASSERT_EQ(tablet.add_value(r, "v", static_cast(r)), + E_OK); + ASSERT_EQ(tablet.add_value(r, "w", static_cast(r) * 2), + E_OK); + ASSERT_EQ(tablet.add_value(r, "s", "s" + std::to_string(r)), + E_OK); + } + ASSERT_EQ(writer.write_table(tablet), E_OK); + ASSERT_EQ(writer.close(), E_OK); + } + + ASSERT_EQ(read_file_.open(path_), E_OK); + ASSERT_EQ(io_.init(&read_file_), E_OK); + TsFileMeta* tsfile_meta = nullptr; + ASSERT_EQ(io_.get_tsfile_meta(tsfile_meta), E_OK); + arena_.init(512, MOD_TSFILE_READER); + auto device = std::make_shared( + std::vector{"t"}); + std::vector indexes(3, nullptr); + ASSERT_EQ(io_.get_timeseries_indexes(device, {"v", "w", "s"}, indexes, + arena_), + E_OK); + for (ITimeseriesIndex* index : indexes) { + ASSERT_NE(index, nullptr); + // The aligned time column is shared by every measurement. + ChunkMeta* time_meta = + index->get_time_chunk_meta_list()->begin().get(); + if (time_meta_ == nullptr) { + time_meta_ = time_meta; + } else { + EXPECT_EQ(time_meta_, time_meta) + << "measurements disagree on the shared time chunk"; + } + value_metas_[index->get_measurement_name().to_std_string()] = + index->get_value_chunk_meta_list()->begin().get(); + } + ASSERT_EQ(value_metas_.size(), 3u); + + reader_ = allocate_reader(); + ASSERT_NE(reader_, nullptr); +#ifdef ENABLE_THREADS + pool_.reset(new common::ThreadPool(2)); + reader_->set_decode_pool(pool_.get()); +#endif + } + + void TearDown() override { + if (reader_ != nullptr) { + reader_->destroy(); + common::mem_free(reader_); + reader_ = nullptr; + } + std::remove(path_.c_str()); + common::g_config_value_ = saved_config_; + libtsfile_destroy(); + } + + AlignedChunkReader* allocate_reader() { + void* memory = common::mem_alloc(sizeof(AlignedChunkReader), + common::MOD_CHUNK_READER); + if (memory == nullptr) { + return nullptr; + } + // The production reader is placement-new'ed and destroyed explicitly. + auto* reader = new (memory) AlignedChunkReader(); + if (reader->init(&read_file_, common::String("v", 1), common::INT64, + nullptr) != E_OK) { + reader->destroy(); + common::mem_free(reader); + return nullptr; + } + return reader; + } + + // Reads the chunk through AlignedChunkReader and returns every row in + // output order. `max_direct_pages` reports the largest direct-decode prefix + // the reader decided on, `bulk_pages` the number of pages copied by the + // bulk path. + // Reads the whole chunk once; the optional out parameters are written, + // not accumulated into. + std::vector read_all(const std::vector& columns, + Filter* time_filter, uint32_t block_rows, + size_t* max_direct_pages = nullptr, + size_t* bulk_pages = nullptr) { + if (max_direct_pages != nullptr) *max_direct_pages = 0; + if (bulk_pages != nullptr) *bulk_pages = 0; + std::vector value_metas; + for (const auto& name : columns) { + value_metas.push_back(value_metas_[name]); + } + if (reader_->load_by_aligned_meta_multi(time_meta_, value_metas) != + E_OK) { + ADD_FAILURE() << "load_by_aligned_meta_multi failed"; + return {}; + } + + TupleDesc tuple; + tuple.push_back(common::g_time_column_schema); + for (const auto& name : columns) { + tuple.push_back(schema_->get_column_schema(name)); + } + common::TsBlock block(&tuple, block_rows); + if (block.init() != common::E_OK) { + ADD_FAILURE() << "TsBlock init failed"; + return {}; + } + + std::vector rows; + size_t direct_pages = 0; + size_t last_bulk_seen = 0; + for (int call = 0; call < 64; call++) { + block.reset(); + const int ret = reader_->get_next_page(&block, time_filter, arena_); + EXPECT_TRUE(ret == E_OK || ret == E_NO_MORE_DATA) + << "get_next_page returned " << ret; + direct_pages = + std::max(direct_pages, + static_cast(reader_->TEST_direct_page_end())); + if (bulk_pages != nullptr) { + // The reader's counter is cumulative. + const size_t seen = reader_->TEST_bulk_page_count(); + *bulk_pages += seen - last_bulk_seen; + last_bulk_seen = seen; + } + common::RowIterator iter(&block); + while (iter.has_next()) { + Row row; + uint32_t len = 0; + bool is_null = false; + std::memcpy(&row.time, iter.read(0, &len, &is_null), + sizeof(row.time)); + std::memcpy(&row.v, iter.read(1, &len, &is_null), + sizeof(row.v)); + if (columns.size() > 1) { + std::memcpy(&row.w, iter.read(2, &len, &is_null), + sizeof(row.w)); + } + if (columns.size() > 2) { + const char* s = iter.read(3, &len, &is_null); + if (!is_null) { + row.s.assign(s, len); + } + } + rows.push_back(std::move(row)); + iter.next(); + } + if (ret == E_NO_MORE_DATA) { + break; + } + } + if (max_direct_pages != nullptr) { + *max_direct_pages = direct_pages; + } + return rows; + } + + // Rows of the source table whose timestamp satisfies @range. + std::vector expected(TimeBetween& range) { + std::vector rows; + for (int r = 0; r < kRowCount; r++) { + if (!range.satisfy_start_end_time(kBaseTime + r, kBaseTime + r)) { + continue; + } + Row row; + row.time = kBaseTime + r; + row.v = r; + row.w = static_cast(r) * 2; + row.s = "s" + std::to_string(r); + rows.push_back(row); + } + return rows; + } + + static void expect_rows_equal(const std::vector& got, + const std::vector& want) { + ASSERT_EQ(got.size(), want.size()); + for (size_t i = 0; i < got.size(); i++) { + EXPECT_EQ(got[i].time, want[i].time) << i; + EXPECT_EQ(got[i].v, want[i].v) << i; + EXPECT_EQ(got[i].w, want[i].w) << i; + } + } + + common::ConfigValue saved_config_; +#ifdef ENABLE_THREADS + std::unique_ptr pool_; +#endif + std::shared_ptr schema_; + std::string path_; + LocalRandomAccessReadFile read_file_; + TsFileIOReader io_; + PageArena arena_; + ChunkMeta* time_meta_ = nullptr; + std::map value_metas_; + AlignedChunkReader* reader_ = nullptr; +}; + +// A window whose endpoints fall inside pages leaves whole pages in the middle. +// Those are BOUNDARY pages restricted to [0, data_num), and a contiguous time +// filter proves every row in that range matches, so they are bulk-copied. +TEST_F(AlignedBulkCopyTest, ContiguousWindowBulkCopiesInnerPages) { + SKIP_WITHOUT_THREADS(); + TimeBetween range(kBaseTime + 10, kBaseTime + 25, /*not_between=*/false); + size_t bulk_pages = 0; + const std::vector rows = + read_all({"v", "w"}, &range, kRowCount, nullptr, &bulk_pages); + expect_rows_equal(rows, expected(range)); + EXPECT_GT(bulk_pages, 0u); +} + +// A window that starts and ends inside one page still leaves a contiguous run +// of matching rows, so the retained slice is bulk-copied even though the page +// is not fully covered. +TEST_F(AlignedBulkCopyTest, WindowInsideOnePageIsBulkCopied) { + SKIP_WITHOUT_THREADS(); + TimeBetween range(kBaseTime + 9, kBaseTime + 11, /*not_between=*/false); + size_t bulk_pages = 0; + const std::vector rows = + read_all({"v", "w"}, &range, kRowCount, nullptr, &bulk_pages); + expect_rows_equal(rows, expected(range)); + EXPECT_EQ(bulk_pages, 1u); +} + +// A discrete time predicate may leave holes inside a page's retained range, so +// boundary pages must keep re-checking every timestamp. +TEST_F(AlignedBulkCopyTest, DiscreteTimeFilterStaysOnSlowPath) { + SKIP_WITHOUT_THREADS(); + TimeIn in({kBaseTime + 1, kBaseTime + 2, kBaseTime + 20}, /*not_in=*/false); + size_t bulk_pages = 0; + const std::vector rows = + read_all({"v", "w"}, &in, kRowCount, nullptr, &bulk_pages); + ASSERT_EQ(rows.size(), 3u); + EXPECT_EQ(rows[0].time, kBaseTime + 1); + EXPECT_EQ(rows[1].time, kBaseTime + 2); + EXPECT_EQ(rows[2].time, kBaseTime + 20); + EXPECT_EQ(bulk_pages, 0u); +} + +// Without a filter every page is fully retained and the columns are +// fixed-length, so the whole plan can be decoded straight into the destination +// block's column buffers. +TEST_F(AlignedBulkCopyTest, NoFilterDecodesDirectlyIntoBlock) { + SKIP_WITHOUT_THREADS(); + size_t direct_pages = 0; + const std::vector rows = + read_all({"v", "w"}, nullptr, kRowCount, &direct_pages); + ASSERT_EQ(rows.size(), static_cast(kRowCount)); + for (int r = 0; r < kRowCount; r++) { + EXPECT_EQ(rows[r].time, kBaseTime + r); + EXPECT_EQ(rows[r].v, r); + EXPECT_EQ(rows[r].w, static_cast(r) * 2); + } + EXPECT_GT(direct_pages, 0u); +} + +// Variable-length columns keep their per-page staging buffers: the destination +// rows of a page are not fixed-size, so the decoder cannot write in place. +TEST_F(AlignedBulkCopyTest, VariableLengthColumnSkipsDirectDecode) { + SKIP_WITHOUT_THREADS(); + size_t direct_pages = 0; + const std::vector rows = + read_all({"v", "w", "s"}, nullptr, kRowCount, &direct_pages); + ASSERT_EQ(rows.size(), static_cast(kRowCount)); + for (int r = 0; r < kRowCount; r++) { + EXPECT_EQ(rows[r].v, r); + EXPECT_EQ(rows[r].s, "s" + std::to_string(r)); + } + EXPECT_EQ(direct_pages, 0u); +} + +// Rows resume across block swaps: only the pages that fit in the first block +// can be decoded in place, the rest are staged page by page. +TEST_F(AlignedBulkCopyTest, RowsSpanMultipleBlocks) { + SKIP_WITHOUT_THREADS(); + const std::vector rows = read_all({"v", "w"}, nullptr, 16); + ASSERT_EQ(rows.size(), static_cast(kRowCount)); + for (int r = 0; r < kRowCount; r++) { + EXPECT_EQ(rows[r].time, kBaseTime + r); + EXPECT_EQ(rows[r].v, r); + } +} + +// A field predicate can still reject rows inside a page's retained range, so a +// partially matching page must stay on the row-by-row path. Page 1 (rows 8..15) +// is only half retained; if it were bulk-copied, rows 8..11 would leak out. +TEST_F(AlignedBulkCopyTest, FieldFilterKeepsPartialPageOnSlowPath) { + auto field_filter = + AlignedFieldFilter::compare("v", AlignedFieldFilter::CompareOp::GE, + AlignedFieldFilter::Literal::int64(12)); + ASSERT_NE(field_filter, nullptr); + reader_->set_field_filter(field_filter.get()); + const std::vector rows = read_all({"v", "w"}, nullptr, kRowCount); + ASSERT_EQ(rows.size(), static_cast(kRowCount - 12)); + for (int r = 12; r < kRowCount; r++) { + EXPECT_EQ(rows[r - 12].time, kBaseTime + r); + EXPECT_EQ(rows[r - 12].v, r); + } +} + +} // namespace +} // namespace storage diff --git a/cpp/test/reader/tsfile_reader_test.cc b/cpp/test/reader/tsfile_reader_test.cc index 060c7a541..8b4818442 100644 --- a/cpp/test/reader/tsfile_reader_test.cc +++ b/cpp/test/reader/tsfile_reader_test.cc @@ -89,7 +89,8 @@ class TsFileReaderTest : public ::testing::Test { flags |= O_BINARY; #endif mode_t mode = 0666; - EXPECT_EQ(tsfile_writer_->open(file_name_, flags, mode), common::E_OK); + // Without an open writer the body below crashes instead of reporting. + ASSERT_EQ(tsfile_writer_->open(file_name_, flags, mode), common::E_OK); } void TearDown() override { @@ -103,8 +104,12 @@ class TsFileReaderTest : public ::testing::Test { public: static std::string generate_random_string(int length) { - std::mt19937 gen(static_cast( - std::chrono::system_clock::now().time_since_epoch().count())); + // Seed from random_device, not the clock: cases of one suite run as + // separate processes in a shared directory (ctest --parallel), and a + // time-seeded generator can hand two of them the same name, leaving + // one open() to fail with E_ALREADY_EXIST. + std::random_device rd; + std::mt19937 gen(rd()); std::uniform_int_distribution<> dis(0, 61); const std::string chars =