diff --git a/.github/workflows/unit-test-cpp.yml b/.github/workflows/unit-test-cpp.yml index 7a0605c8b..3e46d6920 100644 --- a/.github/workflows/unit-test-cpp.yml +++ b/.github/workflows/unit-test-cpp.yml @@ -80,6 +80,12 @@ jobs: - name: Checkout repository uses: actions/checkout@v7 + - name: Set up Python for clang-format + if: runner.os == 'Linux' + uses: actions/setup-python@v7 + with: + python-version: "3.11" + - name: Set up JDK 17 uses: actions/setup-java@v6.0.1 with: @@ -121,10 +127,10 @@ jobs: elif command -v yum >/dev/null 2>&1; then sudo yum install -y libuuid-devel fi - sudo update-alternatives --install /usr/bin/clang-format clang-format /usr/bin/clang-format-17 100 - sudo update-alternatives --set clang-format /usr/bin/clang-format-17 - sudo apt-get update - sudo apt-get install -y uuid-dev + python -m venv "$RUNNER_TEMP/clang-format" + "$RUNNER_TEMP/clang-format/bin/python" -m pip install --only-binary=:all: clang-format==17.0.6 + "$RUNNER_TEMP/clang-format/bin/clang-format" --version + echo "$RUNNER_TEMP/clang-format/bin" >> "$GITHUB_PATH" elif [[ "$RUNNER_OS" == "Windows" ]]; then choco install llvm --version 17.0.6 --force else diff --git a/.github/workflows/unit-test-python.yml b/.github/workflows/unit-test-python.yml index b01c09341..854774034 100644 --- a/.github/workflows/unit-test-python.yml +++ b/.github/workflows/unit-test-python.yml @@ -45,6 +45,13 @@ jobs: matrix: os: [ ubuntu-latest, macos-latest, windows-latest ] python-version: ["3.9", "3.11", "3.14"] + exclude: + - os: ubuntu-latest + python-version: "3.9" + include: + # Python 3.9 has no setup-python build for Ubuntu 26.04. + - os: ubuntu-22.04 + python-version: "3.9" runs-on: ${{ matrix.os }} steps: @@ -92,8 +99,10 @@ jobs: shell: bash run: | if [[ "$RUNNER_OS" == "Linux" ]]; then - sudo update-alternatives --install /usr/bin/clang-format clang-format /usr/bin/clang-format-17 100 - sudo update-alternatives --set clang-format /usr/bin/clang-format-17 + python -m venv "$RUNNER_TEMP/clang-format" + "$RUNNER_TEMP/clang-format/bin/python" -m pip install --only-binary=:all: clang-format==17.0.6 + "$RUNNER_TEMP/clang-format/bin/clang-format" --version + echo "$RUNNER_TEMP/clang-format/bin" >> "$GITHUB_PATH" sudo apt-get update sudo apt-get install -y uuid-dev elif [[ "$RUNNER_OS" == "Windows" ]]; then diff --git a/cpp/src/common/tsfile_common.cc b/cpp/src/common/tsfile_common.cc index 3cac258ff..02a0b1216 100644 --- a/cpp/src/common/tsfile_common.cc +++ b/cpp/src/common/tsfile_common.cc @@ -386,6 +386,14 @@ int MetaIndexNode::binary_search_children( std::shared_ptr key, bool exact_search, std::shared_ptr& ret_index_entry, int64_t& ret_end_offset) { + return binary_search_children(key, exact_search, ret_index_entry, + ret_end_offset, *pa_); +} + +int MetaIndexNode::binary_search_children( + std::shared_ptr key, bool exact_search, + std::shared_ptr& ret_index_entry, int64_t& ret_end_offset, + common::PageArena& result_pa) { #if DEBUG_SE std::cout << "MetaIndexNode::binary_search_children start, name=" << key << ", exact_search=" << exact_search @@ -436,7 +444,7 @@ int MetaIndexNode::binary_search_children( return E_NOT_EXIST; } } - ret_index_entry = children_[l]->clone(pa_); + ret_index_entry = children_[l]->clone(&result_pa); if (l == (int)children_.size() - 1) { ret_end_offset = this->end_offset_; } else { diff --git a/cpp/src/common/tsfile_common.h b/cpp/src/common/tsfile_common.h index bfe597d8a..1a5a07aed 100644 --- a/cpp/src/common/tsfile_common.h +++ b/cpp/src/common/tsfile_common.h @@ -1008,6 +1008,13 @@ struct MetaIndexNode { std::shared_ptr& ret_index_entry, int64_t& ret_end_offset); + // Lookup copies can outlive the source node but must not accumulate in a + // cached node's arena. The caller keeps result_pa alive with the result. + int binary_search_children( + std::shared_ptr key, bool exact_search, + std::shared_ptr& ret_index_entry, + int64_t& ret_end_offset, common::PageArena& result_pa); + int serialize_to(common::ByteStream& out) { int ret = common::E_OK; #if DEBUG_SE diff --git a/cpp/src/file/tsfile_io_reader.cc b/cpp/src/file/tsfile_io_reader.cc index 37225677c..5d95665fb 100644 --- a/cpp/src/file/tsfile_io_reader.cc +++ b/cpp/src/file/tsfile_io_reader.cc @@ -733,7 +733,7 @@ int TsFileIOReader::load_timeseries_index_for_ssi( int64_t measurement_ie_end_offset = 0; if (RET_FAIL(load_measurement_index_entry(measurement_name, top_node, measurement_index_entry, - measurement_ie_end_offset))) { + measurement_ie_end_offset, pa))) { return ret; } else if (RET_FAIL(do_load_timeseries_index( measurement_name, measurement_index_entry->get_offset(), @@ -800,7 +800,7 @@ int TsFileIOReader::load_measurement_index_entry( const std::string& measurement_name_str, std::shared_ptr top_node, std::shared_ptr& ret_measurement_index_entry, - int64_t& ret_end_offset) { + int64_t& ret_end_offset, common::PageArena& result_pa) { int ret = E_OK; // search from top_node in top-down way auto measurement_name = @@ -808,11 +808,11 @@ int TsFileIOReader::load_measurement_index_entry( if (top_node->node_type_ == LEAF_MEASUREMENT) { ret = top_node->binary_search_children( measurement_name, /*exact*/ false, ret_measurement_index_entry, - ret_end_offset); + ret_end_offset, result_pa); } else { ret = search_from_internal_node(measurement_name, false, top_node, ret_measurement_index_entry, - ret_end_offset); + ret_end_offset, &result_pa); } if (ret == E_NOT_EXIST) { ret = E_MEASUREMENT_NOT_EXIST; @@ -945,7 +945,7 @@ int TsFileIOReader::get_timeseries_indexes( timeseries_indexs[idx] = nullptr; ret = load_measurement_index_entry(measurement_name, top_node, measurement_index_entry, - measurement_ie_end_offset); + measurement_ie_end_offset, pa); if (ret == E_MEASUREMENT_NOT_EXIST || ret == E_NOT_EXIST) { ret = E_OK; idx++; @@ -997,16 +997,18 @@ int TsFileIOReader::search_from_leaf_node( int TsFileIOReader::search_from_internal_node( std::shared_ptr target_name, bool is_device, std::shared_ptr index_node, - std::shared_ptr& ret_index_entry, - int64_t& ret_end_offset) { + std::shared_ptr& ret_index_entry, int64_t& ret_end_offset, + common::PageArena* result_pa) { int ret = E_OK; + PageArena& output_pa = result_pa == nullptr ? *index_node->pa_ : *result_pa; std::shared_ptr index_entry; int64_t end_offset = 0; ASSERT(index_node->node_type_ == INTERNAL_MEASUREMENT || index_node->node_type_ == INTERNAL_DEVICE); if (RET_FAIL(index_node->binary_search_children( - target_name, /*exact=*/false, index_entry, end_offset))) { + target_name, /*exact=*/false, index_entry, end_offset, + output_pa))) { return ret; } @@ -1045,17 +1047,20 @@ int TsFileIOReader::search_from_internal_node( } if (cur_level_index_node->node_type_ == LEAF_DEVICE) { ret = cur_level_index_node->binary_search_children( - target_name, /*exact=*/true, ret_index_entry, ret_end_offset); + target_name, /*exact=*/true, ret_index_entry, ret_end_offset, + output_pa); cur_level_index_node->destroy(); return ret; //// FIXME } else if (cur_level_index_node->node_type_ == LEAF_MEASUREMENT) { ret = cur_level_index_node->binary_search_children( - target_name, /*exact=*/false, ret_index_entry, ret_end_offset); + target_name, /*exact=*/false, ret_index_entry, ret_end_offset, + output_pa); cur_level_index_node->destroy(); return ret; //// FIXME } else { ret = cur_level_index_node->binary_search_children( - target_name, /*exact=*/false, index_entry, end_offset); + target_name, /*exact=*/false, index_entry, end_offset, + output_pa); cur_level_index_node->destroy(); } } diff --git a/cpp/src/file/tsfile_io_reader.h b/cpp/src/file/tsfile_io_reader.h index f0c18f60f..51c89dd7e 100644 --- a/cpp/src/file/tsfile_io_reader.h +++ b/cpp/src/file/tsfile_io_reader.h @@ -165,7 +165,7 @@ class TsFileIOReader { const std::string& measurement_name, std::shared_ptr top_node, std::shared_ptr& ret_measurement_index_entry, - int64_t& ret_end_offset); + int64_t& ret_end_offset, common::PageArena& result_pa); int load_all_measurement_index_entry( int64_t start_offset, int64_t end_offset, common::PageArena& pa, @@ -203,7 +203,7 @@ class TsFileIOReader { std::shared_ptr target_name, bool is_device, std::shared_ptr index_node, std::shared_ptr& ret_index_entry, - int64_t& ret_end_offset); + int64_t& ret_end_offset, common::PageArena* result_pa = nullptr); bool filter_stasify(ITimeseriesIndex* ts_index, Filter* time_filter); diff --git a/cpp/test/common/tsfile_common_test.cc b/cpp/test/common/tsfile_common_test.cc index 2f28a0b74..b34470aff 100644 --- a/cpp/test/common/tsfile_common_test.cc +++ b/cpp/test/common/tsfile_common_test.cc @@ -456,6 +456,24 @@ TEST_F(MetaIndexNodeSearchTest, NonExactSearchNotFound) { ASSERT_EQ(result, common::E_NOT_EXIST); } +TEST_F(MetaIndexNodeSearchTest, ResultCopyOutlivesSourceArena) { + common::PageArena result_arena; + std::shared_ptr result; + int64_t end_offset = 0; + ASSERT_EQ(node_.binary_search_children( + std::make_shared("banana"), true, result, + end_offset, result_arena), + common::E_OK); + node_.children_.clear(); + entry1_.reset(); + entry2_.reset(); + entry3_.reset(); + arena_.reset(); + EXPECT_EQ(result->get_name().to_std_string(), "banana"); + EXPECT_EQ(result->get_offset(), 20); + EXPECT_EQ(end_offset, 30); +} + class TsFileMetaTest : public ::testing::Test { protected: common::PageArena pa_; diff --git a/cpp/test/reader/tsfile_reader_test.cc b/cpp/test/reader/tsfile_reader_test.cc index 56c63adaa..060c7a541 100644 --- a/cpp/test/reader/tsfile_reader_test.cc +++ b/cpp/test/reader/tsfile_reader_test.cc @@ -557,6 +557,112 @@ TEST_F(TsFileReaderTest, DeviceNodeCacheIsBoundedAndReloadsEvictedDevices) { #endif } +class HotDeviceMetadataTest : public TsFileReaderTest, + public ::testing::WithParamInterface { + protected: + void SetUp() override { + TsFileReaderTest::SetUp(); + saved_degree_ = g_config_value_.max_degree_of_index_node_; + ASSERT_EQ(set_max_degree_of_index_node(2), E_OK); + } + void TearDown() override { + set_max_degree_of_index_node(saved_degree_); + TsFileReaderTest::TearDown(); + } + uint32_t saved_degree_ = 0; +}; + +TEST_P(HotDeviceMetadataTest, RepeatedLookupsDoNotGrowRetainedArenas) { + const std::string device = "root.followup.d0"; + const int measurements = GetParam() ? 12 : 1; + for (int i = 0; i < measurements; ++i) { + const std::string measurement = "value" + std::to_string(i); + ASSERT_EQ(tsfile_writer_->register_timeseries( + device, MeasurementSchema(measurement, INT32, PLAIN, + UNCOMPRESSED)), + E_OK); + TsRecord record(1, device); + record.add_point(measurement, int32_t(42 + i)); + ASSERT_EQ(tsfile_writer_->write_record(record), E_OK); + } + ASSERT_EQ(tsfile_writer_->close(), E_OK); + + TsFileIOReader reader; + ASSERT_EQ(reader.init(file_name_), E_OK); + auto id = std::make_shared(device); + PageArena query_pa; + query_pa.init(512, MOD_TSFILE_READER); + auto load = [&]() { + TsFileSeriesScanIterator* ssi = nullptr; + int ret = reader.alloc_ssi(id, "value0", ssi, query_pa); + if (ssi != nullptr) reader.revert_ssi(ssi); + query_pa.reset(); + return ret; + }; + ASSERT_EQ(load(), E_OK); +#ifdef ENABLE_MEM_STAT + const int64_t before = reader.TEST_reader_memory_bytes(); +#endif + for (int i = 0; i < 10000; ++i) ASSERT_EQ(load(), E_OK); + EXPECT_EQ(reader.TEST_device_node_cache_size(), 1u); +#ifdef ENABLE_MEM_STAT + EXPECT_EQ(reader.TEST_reader_memory_bytes(), before); +#endif +} + +INSTANTIATE_TEST_SUITE_P(IndexDepth, HotDeviceMetadataTest, + ::testing::Values(false, true)); + +TEST_F(TsFileReaderTest, ActiveIteratorSurvivesDeviceCacheEviction) { + const size_t capacity = TsFileIOReader::TEST_device_node_cache_capacity(); + for (size_t i = 0; i <= capacity; ++i) { + const std::string device = "root.eviction.d" + std::to_string(i); + ASSERT_EQ( + tsfile_writer_->register_timeseries( + device, MeasurementSchema("value", INT32, PLAIN, UNCOMPRESSED)), + E_OK); + TsRecord record(1, device); + record.add_point("value", int32_t(42)); + ASSERT_EQ(tsfile_writer_->write_record(record), E_OK); + } + ASSERT_EQ(tsfile_writer_->close(), E_OK); + TsFileIOReader reader; + ASSERT_EQ(reader.init(file_name_), E_OK); + PageArena active_pa; + active_pa.init(512, MOD_TSFILE_READER); + TsFileSeriesScanIterator* active = nullptr; + ASSERT_EQ(reader.alloc_ssi( + std::make_shared("root.eviction.d0"), + "value", active, active_pa), + E_OK); + ASSERT_NE(active, nullptr); + PageArena transient_pa; + transient_pa.init(512, MOD_TSFILE_READER); + for (size_t i = 1; i <= capacity; ++i) { + TsFileSeriesScanIterator* transient = nullptr; + ASSERT_EQ(reader.alloc_ssi(std::make_shared( + "root.eviction.d" + std::to_string(i)), + "value", transient, transient_pa), + E_OK); + reader.revert_ssi(transient); + transient_pa.reset(); + } + EXPECT_EQ(reader.TEST_device_node_cache_size(), capacity); + TsBlock* block = nullptr; + ASSERT_EQ(active->get_next(block, true), E_OK); + ASSERT_NE(block, nullptr); + ASSERT_EQ(block->get_row_count(), 1u); + { + ColIterator time(0, block); + ColIterator value(1, block); + uint32_t len = 0; + EXPECT_EQ(*reinterpret_cast(time.read(&len)), 1); + EXPECT_EQ(*reinterpret_cast(value.read(&len)), 42); + } + active->revert_tsblock(); + reader.revert_ssi(active); +} + TEST_F(TsFileReaderTest, ResultSetMetadata) { std::string device_path = "device1"; std::string measurement_name = "temperature";