From a2d3ccafe3a6d49c408a7ed2addb9adc47410209 Mon Sep 17 00:00:00 2001 From: ColinLee Date: Thu, 12 Nov 2026 18:13:37 +0800 Subject: [PATCH 1/3] perf(cpp): batch-load a device's timeseries indexes get_timeseries_indexes() resolved each requested measurement on its own: binary search into the measurement index, one pread, a deserialize into a scratch arena and a clone into the caller's arena. A wide table pays that path once per column (4500 times for a 4500-column device), and do_load_all_timeseries_index() issued one pread per measurement too. For projections of 16 or more measurements, walk the measurement index once and read the device's whole TimeseriesIndex range with a single pread: the leaf entries tile one contiguous file range, and each index is deserialized straight into the caller's arena, so the clone per column disappears. Narrow projections keep the per-measurement path, where a full walk would decode every column to find a handful. do_load_all_timeseries_index() now also uses the single range read (with a per-entry fallback when the entries do not tile), which speeds up the prepared-series and device-metadata paths as well. Loading all indexes of one device: 0.53 ms instead of 42 ms for 4500 columns, and 0.006 ms instead of 0.074 ms for 40 (10 devices x 20..200 iterations, 2 rows per series, local macOS release build). An 8-column request is unchanged: it keeps the per-measurement path. --- cpp/src/file/tsfile_io_reader.cc | 160 +++++++--- cpp/src/file/tsfile_io_reader.h | 6 + .../file/timeseries_index_batch_load_test.cc | 288 ++++++++++++++++++ 3 files changed, 418 insertions(+), 36 deletions(-) create mode 100644 cpp/test/file/timeseries_index_batch_load_test.cc diff --git a/cpp/src/file/tsfile_io_reader.cc b/cpp/src/file/tsfile_io_reader.cc index 7c2b1897b..1bdbd322a 100644 --- a/cpp/src/file/tsfile_io_reader.cc +++ b/cpp/src/file/tsfile_io_reader.cc @@ -936,12 +936,56 @@ int TsFileIOReader::get_timeseries_indexes( } bool is_aligned = is_aligned_device(top_node); - TimeseriesIndex* timeseries_index = nullptr; + TimeseriesIndex* time_timeseries_index = nullptr; if (is_aligned) { - if (RET_FAIL( - get_time_column_metadata(top_node, timeseries_index, pa))) { + if (RET_FAIL(get_time_column_metadata(top_node, time_timeseries_index, + pa))) { + return ret; + } + } + + // A wide projection is served better by one pass over the measurement index + // tree and one pread of the whole TimeseriesIndex range than by a binary + // search, a pread, a deserialize into a scratch arena and a clone per + // measurement. A narrow projection keeps the per-measurement path: walking + // the tree would deserialize every column's index to find a handful. + static constexpr size_t kBatchLoadMinMeasurements = 16; + if (measurement_names.size() >= kBatchLoadMinMeasurements) { + std::vector, int64_t>> + leaf_entries; + if (RET_FAIL(get_all_leaf(top_node, leaf_entries, pa))) { return ret; } + std::vector loaded_indexes; + if (RET_FAIL(do_load_all_timeseries_index(leaf_entries, pa, + loaded_indexes))) { + return ret; + } + std::unordered_map name_to_index; + for (ITimeseriesIndex* loaded : loaded_indexes) { + if (is_aligned) { + void* aligned_buf = pa.alloc(sizeof(AlignedTimeseriesIndex)); + if (IS_NULL(aligned_buf)) { + return E_OOM; + } + auto* aligned_ts_idx = new (aligned_buf) AlignedTimeseriesIndex; + aligned_ts_idx->value_ts_idx_ = + static_cast(loaded); + aligned_ts_idx->time_ts_idx_ = time_timeseries_index; + name_to_index[aligned_ts_idx->get_measurement_name() + .to_std_string()] = aligned_ts_idx; + } else { + name_to_index[loaded->get_measurement_name().to_std_string()] = + loaded; + } + } + int64_t idx = 0; + for (const auto& measurement_name : measurement_names) { + const auto it = name_to_index.find(measurement_name); + timeseries_indexs[idx++] = + it != name_to_index.end() ? it->second : nullptr; + } + return ret; } int64_t idx = 0; @@ -974,7 +1018,7 @@ int TsFileIOReader::get_timeseries_indexes( AlignedTimeseriesIndex* aligned_timeseries_index = dynamic_cast(timeseries_indexs[idx]); if (aligned_timeseries_index) { - aligned_timeseries_index->time_ts_idx_ = timeseries_index; + aligned_timeseries_index->time_ts_idx_ = time_timeseries_index; } } @@ -1233,46 +1277,90 @@ int TsFileIOReader::do_load_timeseries_index( return ret; } +namespace { +// True when consecutive leaf entries of one device's measurement index end +// exactly where the next one starts, i.e. they tile one file range. +bool leaf_entries_tile_range( + const std::vector, int64_t>>& + entries) { + for (size_t i = 1; i < entries.size(); i++) { + if (entries[i].first->get_offset() != entries[i - 1].second) { + return false; + } + } + return true; +} +} // namespace + +int TsFileIOReader::do_load_timeseries_index_range( + int64_t start_offset, int64_t end_offset, common::PageArena& pa, + std::vector& ts_indexs) { + int ret = E_OK; + const int64_t read_size_i64 = end_offset - start_offset; + if (read_size_i64 <= 0 || read_size_i64 > INT32_MAX) { + return E_TSFILE_CORRUPTED; + } + const int32_t read_size = static_cast(read_size_i64); + int32_t ret_read_len = 0; + char* ti_buf = static_cast(pa.alloc(read_size)); + if (IS_NULL(ti_buf)) { + return E_OOM; + } + if (RET_FAIL( + read_file_->read(start_offset, ti_buf, read_size, ret_read_len))) { + return ret; + } else if (ret_read_len != read_size) { + return E_FILE_READ_ERR; + } + ByteStream bs; + bs.wrap_from(ti_buf, read_size); + while (bs.has_remaining()) { + const uint64_t relative_start = bs.read_pos(); + void* buf = pa.alloc(sizeof(TimeseriesIndex)); + if (IS_NULL(buf)) { + return E_OOM; + } + auto ts_idx = new (buf) TimeseriesIndex; + if (RET_FAIL(ts_idx->deserialize_from(bs, &pa))) { + return ret; + } + if (bs.read_pos() < relative_start || + bs.read_pos() - relative_start > UINT32_MAX) { + return E_OVERFLOW; + } + ts_idx->set_metadata_range( + start_offset + relative_start, + static_cast(bs.read_pos() - relative_start)); + if (ts_idx->get_measurement_name().len_ == 0) continue; + ts_indexs.push_back(ts_idx); + } + return E_OK; +} + int TsFileIOReader::do_load_all_timeseries_index( std::vector, int64_t>>& index_node_entry_list, common::PageArena& in_timeseries_index_pa, std::vector& ts_indexs) { int ret = E_OK; + if (index_node_entry_list.empty()) { + return ret; + } + // The leaf entries usually tile one contiguous file range, so the whole + // device's TimeseriesIndex blocks can be read back-to-back with a single + // pread instead of one syscall per measurement (4500 of them on a wide + // table). Fall back to per-entry reads if the entries do not tile. + if (leaf_entries_tile_range(index_node_entry_list)) { + return do_load_timeseries_index_range( + index_node_entry_list.front().first->get_offset(), + index_node_entry_list.back().second, in_timeseries_index_pa, + ts_indexs); + } for (const auto& index_node_entry : index_node_entry_list) { - int64_t start_offset = index_node_entry.first->get_offset(), - end_offset = index_node_entry.second; - int32_t read_size = (int32_t)(end_offset - start_offset); - int32_t ret_read_len = 0; - char* ti_buf = in_timeseries_index_pa.alloc(read_size); - if (IS_NULL(ti_buf)) { - return E_OOM; - } - if (RET_FAIL(read_file_->read(start_offset, ti_buf, read_size, - ret_read_len))) { + if (RET_FAIL(do_load_timeseries_index_range( + index_node_entry.first->get_offset(), index_node_entry.second, + in_timeseries_index_pa, ts_indexs))) { return ret; - } else if (ret_read_len != read_size) { - return E_FILE_READ_ERR; - } - ByteStream bs; - bs.wrap_from(ti_buf, read_size); - while (bs.has_remaining()) { - const uint64_t relative_start = bs.read_pos(); - void* buf = in_timeseries_index_pa.alloc(sizeof(TimeseriesIndex)); - auto ts_idx = new (buf) TimeseriesIndex; - if (RET_FAIL( - ts_idx->deserialize_from(bs, &in_timeseries_index_pa))) { - return ret; - } - if (bs.read_pos() < relative_start || - bs.read_pos() - relative_start > UINT32_MAX) { - return E_OVERFLOW; - } - ts_idx->set_metadata_range( - start_offset + relative_start, - static_cast(bs.read_pos() - relative_start)); - if (ts_idx->get_measurement_name().len_ == 0) continue; - ts_indexs.push_back(ts_idx); } } return ret; diff --git a/cpp/src/file/tsfile_io_reader.h b/cpp/src/file/tsfile_io_reader.h index 82ee60f99..8e2125b73 100644 --- a/cpp/src/file/tsfile_io_reader.h +++ b/cpp/src/file/tsfile_io_reader.h @@ -191,6 +191,12 @@ class TsFileIOReader { common::PageArena& in_timeseries_index_pa, std::vector& ts_indexs); + // Reads [start_offset, end_offset) once and deserializes every + // TimeseriesIndex stored in it into @pa, appending them to @ts_indexs. + int do_load_timeseries_index_range( + int64_t start_offset, int64_t end_offset, common::PageArena& pa, + std::vector& ts_indexs); + int load_timeseries_index_for_ssi(std::shared_ptr device_id, const std::string& measurement_name, TsFileSeriesScanIterator*& ssi); diff --git a/cpp/test/file/timeseries_index_batch_load_test.cc b/cpp/test/file/timeseries_index_batch_load_test.cc new file mode 100644 index 000000000..862343849 --- /dev/null +++ b/cpp/test/file/timeseries_index_batch_load_test.cc @@ -0,0 +1,288 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include + +#include +#include +#include +#include +#include +#include + +#include "common/allocator/alloc_base.h" +#include "common/record.h" +#include "common/schema.h" +#include "common/tablet.h" +#include "common/tsfile_common.h" +#include "file/tsfile_io_reader.h" +#include "writer/tsfile_writer.h" + +using namespace common; + +namespace storage { +namespace { + +// Above the batch-load threshold in TsFileIOReader::get_timeseries_indexes(), +// so a request for every column takes the bulk path. +constexpr int kMeasurementCount = 40; +// Below the threshold, so the per-measurement path handles it. +constexpr int kNarrowMeasurementCount = 5; + +const char* kNonAlignedDevice = "root.sg.batch_plain"; +const char* kAlignedDevice = "root.sg.batch_aligned"; + +std::string MeasurementName(int i) { return "m" + std::to_string(i); } + +// Every gtest case runs as its own process in a shared working directory, so +// each case writes to its own file (named after the case). +std::string TestFileName() { + const ::testing::TestInfo* info = + ::testing::UnitTest::GetInstance()->current_test_info(); + return std::string("timeseries_index_batch_load_") + + (info != nullptr ? info->name() : "unknown") + ".tsfile"; +} + +class TimeseriesIndexBatchLoadTest : public ::testing::Test { + protected: + void SetUp() override { + libtsfile_init(); + file_name_ = TestFileName(); + std::remove(file_name_.c_str()); + TsFileWriter writer; + ASSERT_EQ(writer.open(file_name_, O_WRONLY | O_CREAT | O_TRUNC, 0666), + E_OK); + + for (int i = 0; i < kMeasurementCount; i++) { + ASSERT_EQ(writer.register_timeseries( + kNonAlignedDevice, + MeasurementSchema(MeasurementName(i), INT32, PLAIN, + UNCOMPRESSED)), + E_OK); + } + for (int64_t row = 0; row < 2; row++) { + TsRecord record(1000 + row, kNonAlignedDevice); + for (int i = 0; i < kMeasurementCount; i++) { + record.add_point(MeasurementName(i), + static_cast(i * 10 + row)); + } + ASSERT_EQ(writer.write_record(record), E_OK); + } + + std::vector aligned_schemas; + for (int i = 0; i < kMeasurementCount; i++) { + aligned_schemas.emplace_back(MeasurementName(i), INT64, PLAIN, + UNCOMPRESSED); + } + { + std::vector to_register; + for (const auto& schema : aligned_schemas) { + to_register.push_back(new MeasurementSchema(schema)); + } + ASSERT_EQ( + writer.register_aligned_timeseries(kAlignedDevice, to_register), + E_OK); + } + Tablet tablet( + kAlignedDevice, + std::make_shared>(aligned_schemas), + 4); + for (int64_t row = 0; row < 4; row++) { + ASSERT_EQ(tablet.add_timestamp(row, 2000 + row), E_OK); + for (int i = 0; i < kMeasurementCount; i++) { + ASSERT_EQ(tablet.add_value(row, static_cast(i), + static_cast(i * 100 + row)), + E_OK); + } + } + ASSERT_EQ(writer.write_tablet_aligned(tablet), E_OK); + ASSERT_EQ(writer.flush(), E_OK); + ASSERT_EQ(writer.close(), E_OK); + + reader_ = std::make_shared(); + ASSERT_EQ(reader_->init(file_name_), E_OK); + // The metadata must be loaded before the index lookups below; a + // TsFileReader does this during its own init(). + TsFileMeta* tsfile_meta = nullptr; + ASSERT_EQ(reader_->get_tsfile_meta(tsfile_meta), E_OK); + pa_.init(512, MOD_TSFILE_READER); + } + + void TearDown() override { + reader_.reset(); + pa_.destroy(); + std::remove(file_name_.c_str()); + libtsfile_destroy(); + } + + // Names of every measurement of @device, all in one request set. + static std::unordered_set AllMeasurementNames() { + std::unordered_set names; + for (int i = 0; i < kMeasurementCount; i++) { + names.insert(MeasurementName(i)); + } + return names; + } + + std::string file_name_; + std::shared_ptr reader_; + PageArena pa_; +}; + +TEST_F(TimeseriesIndexBatchLoadTest, LoadsEveryMeasurementOfWideDevice) { + const std::unordered_set names = AllMeasurementNames(); + std::vector indexes(names.size(), nullptr); + ASSERT_EQ(reader_->get_timeseries_indexes( + std::make_shared(kNonAlignedDevice), + names, indexes, pa_), + E_OK); + + std::map by_name; + for (ITimeseriesIndex* index : indexes) { + ASSERT_NE(index, nullptr); + EXPECT_FALSE(index->is_aligned()); + EXPECT_EQ(index->get_data_type(), INT32); + SimpleList* chunks = index->get_chunk_meta_list(); + ASSERT_NE(chunks, nullptr); + EXPECT_EQ(chunks->size(), 1u); + by_name[index->get_measurement_name().to_std_string()] = index; + } + EXPECT_EQ(by_name.size(), names.size()); + for (int i = 0; i < kMeasurementCount; i++) { + EXPECT_NE(by_name.find(MeasurementName(i)), by_name.end()); + } +} + +TEST_F(TimeseriesIndexBatchLoadTest, MissingMeasurementBecomesNull) { + std::unordered_set names = AllMeasurementNames(); + names.erase(MeasurementName(0)); + names.insert("does_not_exist"); + std::vector indexes(names.size(), nullptr); + ASSERT_EQ(reader_->get_timeseries_indexes( + std::make_shared(kNonAlignedDevice), + names, indexes, pa_), + E_OK); + + int null_count = 0; + for (ITimeseriesIndex* index : indexes) { + if (index == nullptr) { + null_count++; + } else { + EXPECT_NE(index->get_measurement_name().to_std_string(), + MeasurementName(0)); + } + } + EXPECT_EQ(null_count, 1); +} + +TEST_F(TimeseriesIndexBatchLoadTest, AlignedDeviceSharesTheTimeIndex) { + const std::unordered_set names = AllMeasurementNames(); + std::vector indexes(names.size(), nullptr); + ASSERT_EQ(reader_->get_timeseries_indexes( + std::make_shared(kAlignedDevice), names, + indexes, pa_), + E_OK); + + std::map by_name; + for (ITimeseriesIndex* index : indexes) { + ASSERT_NE(index, nullptr); + // An aligned measurement is wrapped with the device's shared time + // index and reports the value column's type, not the time column's. + EXPECT_TRUE(index->is_aligned()); + EXPECT_EQ(index->get_data_type(), INT64); + ASSERT_NE(index->get_time_chunk_meta_list(), nullptr); + ASSERT_NE(index->get_value_chunk_meta_list(), nullptr); + EXPECT_EQ(index->get_value_chunk_meta_list()->size(), 1u); + by_name[index->get_measurement_name().to_std_string()] = index; + } + EXPECT_EQ(by_name.size(), names.size()); +} + +TEST_F(TimeseriesIndexBatchLoadTest, NarrowRequestUsesPerMeasurementPath) { + std::unordered_set names; + for (int i = 0; i < kNarrowMeasurementCount; i++) { + names.insert(MeasurementName(i)); + } + names.insert("absent_measurement"); + std::vector indexes(names.size(), nullptr); + ASSERT_EQ(reader_->get_timeseries_indexes( + std::make_shared(kNonAlignedDevice), + names, indexes, pa_), + E_OK); + + int found = 0; + for (ITimeseriesIndex* index : indexes) { + if (index == nullptr) continue; + found++; + EXPECT_EQ(index->get_measurement_name().to_std_string().substr(0, 1), + "m"); + EXPECT_EQ(index->get_data_type(), INT32); + } + EXPECT_EQ(found, kNarrowMeasurementCount); +} + +// The bulk path must return the same chunk metadata as the per-measurement +// path (a different loader: binary search + deserialize + clone per column). +TEST_F(TimeseriesIndexBatchLoadTest, MatchesPerMeasurementPath) { + const std::unordered_set names = AllMeasurementNames(); + std::vector bulk(names.size(), nullptr); + ASSERT_EQ(reader_->get_timeseries_indexes( + std::make_shared(kNonAlignedDevice), + names, bulk, pa_), + E_OK); + + std::unordered_set narrow_names; + for (int i = 0; i < kNarrowMeasurementCount; i++) { + narrow_names.insert(MeasurementName(i)); + } + std::vector narrow(narrow_names.size(), nullptr); + ASSERT_EQ(reader_->get_timeseries_indexes( + std::make_shared(kNonAlignedDevice), + narrow_names, narrow, pa_), + E_OK); + + std::map expected; + for (ITimeseriesIndex* index : narrow) { + ASSERT_NE(index, nullptr); + expected[index->get_measurement_name().to_std_string()] = index; + } + EXPECT_EQ(expected.size(), narrow_names.size()); + + for (ITimeseriesIndex* index : bulk) { + ASSERT_NE(index, nullptr); + const std::string name = index->get_measurement_name().to_std_string(); + auto it = expected.find(name); + if (it == expected.end()) continue; + SimpleList* got = index->get_chunk_meta_list(); + SimpleList* want = it->second->get_chunk_meta_list(); + ASSERT_EQ(got->size(), want->size()) << name; + SimpleList::Iterator got_it = got->begin(); + SimpleList::Iterator want_it = want->begin(); + while (got_it != got->end()) { + EXPECT_EQ(got_it.get()->offset_of_chunk_header_, + want_it.get()->offset_of_chunk_header_) + << name; + got_it++; + want_it++; + } + } +} + +} // namespace +} // namespace storage From 73d8373d82155cccc3e306b84e40c98430e05b57 Mon Sep 17 00:00:00 2001 From: ColinLee Date: Thu, 12 Nov 2026 22:42:34 +0800 Subject: [PATCH 2/3] perf(cpp): address review feedback on batch index loading Release the raw index-range buffer right after the indexes are parsed: it comes from the heap now, so a wide device's whole serialized range no longer stays resident in the caller's arena after deserialization. Test cleanups: own the registered measurement schemas in unique_ptrs and release them only after the registration that takes ownership succeeds; accept one or more chunks instead of exactly one; and describe the per-case file name in terms of how ctest and a direct binary run actually share a working directory. --- cpp/src/file/tsfile_io_reader.cc | 14 ++++++++++++-- cpp/src/file/tsfile_io_reader.h | 6 ++++-- .../file/timeseries_index_batch_load_test.cc | 16 ++++++++++++---- 3 files changed, 28 insertions(+), 8 deletions(-) diff --git a/cpp/src/file/tsfile_io_reader.cc b/cpp/src/file/tsfile_io_reader.cc index 1bdbd322a..c08536ea4 100644 --- a/cpp/src/file/tsfile_io_reader.cc +++ b/cpp/src/file/tsfile_io_reader.cc @@ -1301,15 +1301,21 @@ int TsFileIOReader::do_load_timeseries_index_range( return E_TSFILE_CORRUPTED; } const int32_t read_size = static_cast(read_size_i64); - int32_t ret_read_len = 0; - char* ti_buf = static_cast(pa.alloc(read_size)); + // The raw bytes are needed only while the indexes are parsed, so they are + // heap-allocated and released below instead of coming from the caller's + // arena: a wide device's whole range would otherwise stay resident for as + // long as the arena lives. + char* ti_buf = static_cast(mem_alloc(read_size, MOD_TSFILE_READER)); if (IS_NULL(ti_buf)) { return E_OOM; } + int32_t ret_read_len = 0; if (RET_FAIL( read_file_->read(start_offset, ti_buf, read_size, ret_read_len))) { + mem_free(ti_buf); return ret; } else if (ret_read_len != read_size) { + mem_free(ti_buf); return E_FILE_READ_ERR; } ByteStream bs; @@ -1318,14 +1324,17 @@ int TsFileIOReader::do_load_timeseries_index_range( const uint64_t relative_start = bs.read_pos(); void* buf = pa.alloc(sizeof(TimeseriesIndex)); if (IS_NULL(buf)) { + mem_free(ti_buf); return E_OOM; } auto ts_idx = new (buf) TimeseriesIndex; if (RET_FAIL(ts_idx->deserialize_from(bs, &pa))) { + mem_free(ti_buf); return ret; } if (bs.read_pos() < relative_start || bs.read_pos() - relative_start > UINT32_MAX) { + mem_free(ti_buf); return E_OVERFLOW; } ts_idx->set_metadata_range( @@ -1334,6 +1343,7 @@ int TsFileIOReader::do_load_timeseries_index_range( if (ts_idx->get_measurement_name().len_ == 0) continue; ts_indexs.push_back(ts_idx); } + mem_free(ti_buf); return E_OK; } diff --git a/cpp/src/file/tsfile_io_reader.h b/cpp/src/file/tsfile_io_reader.h index 8e2125b73..598d6092d 100644 --- a/cpp/src/file/tsfile_io_reader.h +++ b/cpp/src/file/tsfile_io_reader.h @@ -191,8 +191,10 @@ class TsFileIOReader { common::PageArena& in_timeseries_index_pa, std::vector& ts_indexs); - // Reads [start_offset, end_offset) once and deserializes every - // TimeseriesIndex stored in it into @pa, appending them to @ts_indexs. + // Reads [start_offset, end_offset) once and deserializes the + // TimeseriesIndex stored there into @pa, appending them to @ts_indexs. + // Entries with an empty measurement name (the aligned time column) are + // skipped, matching do_load_all_timeseries_index(). int do_load_timeseries_index_range( int64_t start_offset, int64_t end_offset, common::PageArena& pa, std::vector& ts_indexs); diff --git a/cpp/test/file/timeseries_index_batch_load_test.cc b/cpp/test/file/timeseries_index_batch_load_test.cc index 862343849..0ac4cd3db 100644 --- a/cpp/test/file/timeseries_index_batch_load_test.cc +++ b/cpp/test/file/timeseries_index_batch_load_test.cc @@ -50,8 +50,9 @@ const char* kAlignedDevice = "root.sg.batch_aligned"; std::string MeasurementName(int i) { return "m" + std::to_string(i); } -// Every gtest case runs as its own process in a shared working directory, so -// each case writes to its own file (named after the case). +// CTest runs each case as its own process in a shared working directory (and +// running the binary directly runs them all in one process), so each case +// writes to a file named after itself. std::string TestFileName() { const ::testing::TestInfo* info = ::testing::UnitTest::GetInstance()->current_test_info(); @@ -91,13 +92,20 @@ class TimeseriesIndexBatchLoadTest : public ::testing::Test { UNCOMPRESSED); } { + // The writer takes ownership of the schemas of a successful + // registration, so the raw pointers are released on success only. + std::vector> owned; std::vector to_register; for (const auto& schema : aligned_schemas) { - to_register.push_back(new MeasurementSchema(schema)); + owned.emplace_back(new MeasurementSchema(schema)); + to_register.push_back(owned.back().get()); } ASSERT_EQ( writer.register_aligned_timeseries(kAlignedDevice, to_register), E_OK); + for (auto& schema : owned) { + schema.release(); + } } Tablet tablet( kAlignedDevice, @@ -160,7 +168,7 @@ TEST_F(TimeseriesIndexBatchLoadTest, LoadsEveryMeasurementOfWideDevice) { EXPECT_EQ(index->get_data_type(), INT32); SimpleList* chunks = index->get_chunk_meta_list(); ASSERT_NE(chunks, nullptr); - EXPECT_EQ(chunks->size(), 1u); + EXPECT_GE(chunks->size(), 1u); by_name[index->get_measurement_name().to_std_string()] = index; } EXPECT_EQ(by_name.size(), names.size()); From 2afbd45f4da5e03fcea5be536f20c87305ae979c Mon Sep 17 00:00:00 2001 From: ColinLee Date: Thu, 12 Nov 2026 23:16:35 +0800 Subject: [PATCH 3/3] perf(cpp): address further review feedback on batch index loading Fall back to per-entry reads when a device's combined index range exceeds INT32_MAX bytes instead of rejecting it, and report E_OVERFLOW (not E_TSFILE_CORRUPTED) for a single entry that large. Reserve the name map for the wide-device case, include the headers the test uses directly, and make each test's file name unique across suites. --- cpp/src/file/tsfile_io_reader.cc | 14 +++++++++++--- cpp/test/file/timeseries_index_batch_load_test.cc | 5 +++++ 2 files changed, 16 insertions(+), 3 deletions(-) diff --git a/cpp/src/file/tsfile_io_reader.cc b/cpp/src/file/tsfile_io_reader.cc index c08536ea4..3abe2eeb4 100644 --- a/cpp/src/file/tsfile_io_reader.cc +++ b/cpp/src/file/tsfile_io_reader.cc @@ -962,6 +962,7 @@ int TsFileIOReader::get_timeseries_indexes( return ret; } std::unordered_map name_to_index; + name_to_index.reserve(loaded_indexes.size()); for (ITimeseriesIndex* loaded : loaded_indexes) { if (is_aligned) { void* aligned_buf = pa.alloc(sizeof(AlignedTimeseriesIndex)); @@ -1297,9 +1298,12 @@ int TsFileIOReader::do_load_timeseries_index_range( std::vector& ts_indexs) { int ret = E_OK; const int64_t read_size_i64 = end_offset - start_offset; - if (read_size_i64 <= 0 || read_size_i64 > INT32_MAX) { + if (read_size_i64 <= 0) { return E_TSFILE_CORRUPTED; } + if (read_size_i64 > INT32_MAX) { + return E_OVERFLOW; + } const int32_t read_size = static_cast(read_size_i64); // The raw bytes are needed only while the indexes are parsed, so they are // heap-allocated and released below instead of coming from the caller's @@ -1359,8 +1363,12 @@ int TsFileIOReader::do_load_all_timeseries_index( // The leaf entries usually tile one contiguous file range, so the whole // device's TimeseriesIndex blocks can be read back-to-back with a single // pread instead of one syscall per measurement (4500 of them on a wide - // table). Fall back to per-entry reads if the entries do not tile. - if (leaf_entries_tile_range(index_node_entry_list)) { + // table). Fall back to per-entry reads if the entries do not tile, or if + // the combined range does not fit one read buffer. + if (leaf_entries_tile_range(index_node_entry_list) && + index_node_entry_list.back().second - + index_node_entry_list.front().first->get_offset() <= + INT32_MAX) { return do_load_timeseries_index_range( index_node_entry_list.front().first->get_offset(), index_node_entry_list.back().second, in_timeseries_index_pa, diff --git a/cpp/test/file/timeseries_index_batch_load_test.cc b/cpp/test/file/timeseries_index_batch_load_test.cc index 0ac4cd3db..76302b75e 100644 --- a/cpp/test/file/timeseries_index_batch_load_test.cc +++ b/cpp/test/file/timeseries_index_batch_load_test.cc @@ -17,8 +17,10 @@ * under the License. */ +#include #include +#include #include #include #include @@ -56,7 +58,10 @@ std::string MeasurementName(int i) { return "m" + std::to_string(i); } std::string TestFileName() { const ::testing::TestInfo* info = ::testing::UnitTest::GetInstance()->current_test_info(); + // Include the suite name too: only the suite/name pair is unique within + // the binary, and cases of different suites may share a name. return std::string("timeseries_index_batch_load_") + + (info != nullptr ? info->test_suite_name() : "unknown") + "_" + (info != nullptr ? info->name() : "unknown") + ".tsfile"; }