From 8ca59ac01e0f5c94b7c0c3e9cdfef26876792d8b Mon Sep 17 00:00:00 2001 From: 761417898 <761417898@qq.com> Date: Fri, 17 Apr 2026 16:03:21 +0800 Subject: [PATCH 1/4] fix cpp restore statistic --- cpp/src/file/restorable_tsfile_io_writer.cc | 37 ++++- .../file/restorable_tsfile_io_writer_test.cc | 153 ++++++++++++++++++ 2 files changed, 186 insertions(+), 4 deletions(-) diff --git a/cpp/src/file/restorable_tsfile_io_writer.cc b/cpp/src/file/restorable_tsfile_io_writer.cc index 05e5b5112..720ac9502 100644 --- a/cpp/src/file/restorable_tsfile_io_writer.cc +++ b/cpp/src/file/restorable_tsfile_io_writer.cc @@ -328,8 +328,12 @@ static int recover_chunk_statistic( uint32_t value_buf_size = 0; std::vector time_decode_buf; const std::vector* times = nullptr; + std::vector aligned_value_notnull_bitmap; + uint32_t aligned_num_values = 0; + const bool is_aligned_value_chunk = + (time_batch != nullptr && !time_batch->empty()); - if (time_batch != nullptr && !time_batch->empty()) { + if (is_aligned_value_chunk) { // Aligned value page: uncompressed layout = uint32(num_values) + bitmap // + value_buf if (uncompressed_size < 4) { @@ -337,7 +341,7 @@ static int recover_chunk_statistic( CompressorFactory::free(compressor); return E_OK; } - uint32_t num_values = + aligned_num_values = (static_cast( static_cast(uncompressed_buf[0])) << 24) | @@ -349,12 +353,17 @@ static int recover_chunk_statistic( << 8) | (static_cast( static_cast(uncompressed_buf[3]))); - uint32_t bitmap_size = (num_values + 7) / 8; + uint32_t bitmap_size = (aligned_num_values + 7) / 8; if (uncompressed_size < 4 + bitmap_size) { compressor->after_uncompress(uncompressed_buf); CompressorFactory::free(compressor); return E_OK; } + aligned_value_notnull_bitmap.resize(bitmap_size); + if (bitmap_size > 0) { + std::memcpy(aligned_value_notnull_bitmap.data(), + uncompressed_buf + 4, bitmap_size); + } value_buf = uncompressed_buf + 4 + bitmap_size; value_buf_size = uncompressed_size - 4 - bitmap_size; times = time_batch; @@ -410,8 +419,28 @@ static int recover_chunk_statistic( value_decoder->reset(); size_t idx = 0; const size_t num_times = times->size(); - while (idx < num_times && value_decoder->has_remaining(value_in)) { + while (idx < num_times) { int64_t t = (*times)[idx]; + bool has_value = true; + if (is_aligned_value_chunk) { + has_value = false; + if (idx < aligned_num_values) { + const uint32_t byte_idx = static_cast(idx / 8); + const uint32_t bit_shift = static_cast(idx % 8); + if (byte_idx < aligned_value_notnull_bitmap.size()) { + has_value = + ((aligned_value_notnull_bitmap[byte_idx] & 0xFF) & + (0x80 >> bit_shift)) != 0; + } + } + } + if (!has_value) { + idx++; + continue; + } + if (!value_decoder->has_remaining(value_in)) { + break; + } switch (chdr.data_type_) { case common::BOOLEAN: { bool v; diff --git a/cpp/test/file/restorable_tsfile_io_writer_test.cc b/cpp/test/file/restorable_tsfile_io_writer_test.cc index 3c6687e78..934d11067 100644 --- a/cpp/test/file/restorable_tsfile_io_writer_test.cc +++ b/cpp/test/file/restorable_tsfile_io_writer_test.cc @@ -696,4 +696,157 @@ TEST_F(RestorableTsFileIOWriterTest, TableWriterRecoverAndWrite1) { table_result_set->close(); table_reader.destroy_query_data_set(temp_ret); table_reader.close(); +} + +TEST_F(RestorableTsFileIOWriterTest, + TableWriterRecoverAndWriteNullTagFloatDoubleStatistics) { + using namespace std; + const string table_name = "test_table"; + vector column_names = {"t1", "t2", "t3", "f1", "f2", "f3", "f4", + "f5", "f6", "f7", "f8", "f9", "f10"}; + vector data_types = {STRING, STRING, STRING, BOOLEAN, INT32, + INT64, FLOAT, DOUBLE, TEXT, STRING, + BLOB, DATE, TIMESTAMP}; + std::vector column_schemas; + for (size_t i = 0; i < column_names.size(); i++) { + column_schemas.push_back( + new MeasurementSchema(column_names[i], data_types[i])); + } + std::vector column_categories = { + ColumnCategory::TAG, ColumnCategory::TAG, ColumnCategory::TAG, + ColumnCategory::FIELD, ColumnCategory::FIELD, ColumnCategory::FIELD, + ColumnCategory::FIELD, ColumnCategory::FIELD, ColumnCategory::FIELD, + ColumnCategory::FIELD, ColumnCategory::FIELD, ColumnCategory::FIELD, + ColumnCategory::FIELD}; + TableSchema table_schema(table_name, column_schemas, column_categories); + + // 首轮写入:奇数行仅写时间,制造 __level1/2/3 全空值的 TAG 组。 + WriteFile write_file; + ASSERT_EQ(write_file.create(file_name_, GetWriteCreateFlags(), 0666), E_OK); + TsFileTableWriter table_writer(&write_file, &table_schema); + constexpr uint32_t max_rows = 10; + Tablet tablet(table_schema.get_measurement_names(), + table_schema.get_data_types(), max_rows); + tablet.set_table_name(table_name); + for (int row = 0; row < static_cast(max_rows); row++) { + ASSERT_EQ(tablet.add_timestamp(row, static_cast(row)), E_OK); + if (row % 2 == 0) { + ASSERT_EQ(tablet.add_value(row, "t1", "device1"), E_OK); + ASSERT_EQ(tablet.add_value(row, "t2", "device2"), E_OK); + ASSERT_EQ(tablet.add_value(row, "t3", "device3"), E_OK); + ASSERT_EQ(tablet.add_value(row, "f1", row % 2 == 0), E_OK); + ASSERT_EQ(tablet.add_value(row, "f2", static_cast(row)), + E_OK); + ASSERT_EQ(tablet.add_value(row, "f3", static_cast(row)), + E_OK); + ASSERT_EQ( + tablet.add_value(row, "f4", static_cast(row * 1.1)), + E_OK); + ASSERT_EQ( + tablet.add_value(row, "f5", static_cast(row * 1.1)), + E_OK); + ASSERT_EQ( + tablet.add_value(row, "f6", ("text" + to_string(row)).c_str()), + E_OK); + ASSERT_EQ(tablet.add_value(row, "f7", + ("string" + to_string(row)).c_str()), + E_OK); + ASSERT_EQ( + tablet.add_value(row, "f8", ("blob" + to_string(row)).c_str()), + E_OK); + ASSERT_EQ(tablet.add_value(row, "f9", static_cast(row)), + E_OK); + ASSERT_EQ(tablet.add_value(row, "f10", static_cast(row)), + E_OK); + } + } + ASSERT_EQ(table_writer.write_table(tablet), E_OK); + ASSERT_EQ(table_writer.flush(), E_OK); + ASSERT_EQ(table_writer.close(), E_OK); + ASSERT_EQ(write_file.close(), E_OK); + + CorruptCurrentFileTail(10); + + RestorableTsFileIOWriter rw; + ASSERT_EQ(rw.open(file_name_, true), E_OK); + ASSERT_TRUE(rw.can_write()); + + TsFileTableWriter table_writer2(&rw); + vector column_names2 = { + "__level1", "__level2", "__level3", "f1", "f2", "f3", "f4", + "f5", "f6", "f7", "f8", "f9", "f10"}; + Tablet tablet2(column_names2, data_types, max_rows); + tablet2.set_table_name(table_name); + for (int row = 0; row < static_cast(max_rows); row++) { + ASSERT_EQ( + tablet2.add_timestamp(row, static_cast(row + max_rows)), + E_OK); + ASSERT_EQ(tablet2.add_value(row, "__level1", "device1"), E_OK); + ASSERT_EQ(tablet2.add_value(row, "__level2", "device2"), E_OK); + ASSERT_EQ(tablet2.add_value(row, "__level3", "device3"), E_OK); + ASSERT_EQ(tablet2.add_value(row, "f1", row % 2 == 0), E_OK); + ASSERT_EQ(tablet2.add_value(row, "f2", static_cast(row)), + E_OK); + ASSERT_EQ(tablet2.add_value(row, "f3", static_cast(row)), + E_OK); + ASSERT_EQ(tablet2.add_value(row, "f4", static_cast(row * 1.1)), + E_OK); + ASSERT_EQ(tablet2.add_value(row, "f5", static_cast(row * 1.1)), + E_OK); + ASSERT_EQ( + tablet2.add_value(row, "f6", ("text" + to_string(row)).c_str()), + E_OK); + ASSERT_EQ( + tablet2.add_value(row, "f7", ("string" + to_string(row)).c_str()), + E_OK); + ASSERT_EQ( + tablet2.add_value(row, "f8", ("blob" + to_string(row)).c_str()), + E_OK); + ASSERT_EQ(tablet2.add_value(row, "f9", static_cast(row)), + E_OK); + ASSERT_EQ(tablet2.add_value(row, "f10", static_cast(row)), + E_OK); + } + ASSERT_EQ(table_writer2.write_table(tablet2), E_OK); + ASSERT_EQ(table_writer2.flush(), E_OK); + ASSERT_EQ(table_writer2.close(), E_OK); + + TsFileReader table_reader; + ASSERT_EQ(table_reader.open(file_name_), E_OK); + DeviceTimeseriesMetadataMap metadata = + table_reader.get_timeseries_metadata(); + + bool checked_null_tag_group = false; + for (const auto& entry : metadata) { + const auto& device_id = entry.first; + if (device_id == nullptr) { + continue; + } + const std::string device_name = device_id->get_device_name(); + if (device_name.find("null.null.null") == std::string::npos) { + continue; + } + bool checked_f4 = false; + bool checked_f5 = false; + for (const auto& field : entry.second) { + const auto field_name = + field->get_measurement_name().to_std_string(); + if (field_name == "f4" || field_name == "f5") { + ASSERT_NE(field->get_statistic(), nullptr); + EXPECT_EQ(field->get_statistic()->count_, 0); + EXPECT_EQ(field->get_statistic()->start_time_, 0); + EXPECT_EQ(field->get_statistic()->end_time_, 0); + if (field_name == "f4") { + checked_f4 = true; + } else { + checked_f5 = true; + } + } + } + EXPECT_TRUE(checked_f4); + EXPECT_TRUE(checked_f5); + checked_null_tag_group = true; + } + EXPECT_TRUE(checked_null_tag_group); + table_reader.close(); } \ No newline at end of file From e45c5d502da1828086e42e2b4a21b8e78e8e72a7 Mon Sep 17 00:00:00 2001 From: 761417898 <761417898@qq.com> Date: Fri, 17 Apr 2026 16:25:00 +0800 Subject: [PATCH 2/4] fix cpp restore device last_time --- cpp/src/common/schema.h | 2 + cpp/src/writer/tsfile_writer.cc | 100 ++++++++++-- .../file/restorable_tsfile_io_writer_test.cc | 149 ++++++++++++++++++ 3 files changed, 236 insertions(+), 15 deletions(-) diff --git a/cpp/src/common/schema.h b/cpp/src/common/schema.h index a2c989af2..81008b715 100644 --- a/cpp/src/common/schema.h +++ b/cpp/src/common/schema.h @@ -23,6 +23,7 @@ #include #include +#include #include // use unordered_map instead #include #include @@ -165,6 +166,7 @@ struct MeasurementSchemaGroup { MeasurementSchemaMap measurement_schema_map_; bool is_aligned_ = false; TimeChunkWriter* time_chunk_writer_ = nullptr; + int64_t last_time_ = INT64_MIN; ~MeasurementSchemaGroup() { if (time_chunk_writer_ != nullptr) { diff --git a/cpp/src/writer/tsfile_writer.cc b/cpp/src/writer/tsfile_writer.cc index 35520c88d..7ae265ac7 100644 --- a/cpp/src/writer/tsfile_writer.cc +++ b/cpp/src/writer/tsfile_writer.cc @@ -182,6 +182,10 @@ int TsFileWriter::init(RestorableTsFileIOWriter* rw) { if (cm == nullptr) { continue; } + if (cm->statistic_ != nullptr && cm->statistic_->count_ > 0) { + group->last_time_ = + std::max(group->last_time_, cm->statistic_->end_time_); + } std::string mname = cm->measurement_name_.to_std_string(); if (mname.empty()) { continue; @@ -692,13 +696,21 @@ int TsFileWriter::check_memory_size_and_may_flush_chunks() { int TsFileWriter::write_record(const TsRecord& record) { int ret = E_OK; + auto device_id = std::make_shared(record.device_id_); + auto schema_it = schemas_.find(device_id); + if (schema_it == schemas_.end() || schema_it->second == nullptr) { + return E_DEVICE_NOT_EXIST; + } + MeasurementSchemaGroup* device_schema = schema_it->second; + if (record.timestamp_ <= device_schema->last_time_) { + return E_OUT_OF_ORDER; + } // std::vector chunk_writers; SimpleVector chunk_writers; SimpleVector data_types; MeasurementNamesFromRecord mnames_getter(record); - if (RET_FAIL(do_check_schema( - std::make_shared(record.device_id_), - mnames_getter, chunk_writers, data_types))) { + if (RET_FAIL(do_check_schema(device_id, mnames_getter, chunk_writers, + data_types))) { return ret; } @@ -713,6 +725,8 @@ int TsFileWriter::write_record(const TsRecord& record) { record.points_[c]); } + device_schema->last_time_ = + std::max(device_schema->last_time_, record.timestamp_); record_count_since_last_flush_++; ret = check_memory_size_and_may_flush_chunks(); return ret; @@ -720,14 +734,22 @@ int TsFileWriter::write_record(const TsRecord& record) { int TsFileWriter::write_record_aligned(const TsRecord& record) { int ret = E_OK; + auto device_id = std::make_shared(record.device_id_); + auto schema_it = schemas_.find(device_id); + if (schema_it == schemas_.end() || schema_it->second == nullptr) { + return E_DEVICE_NOT_EXIST; + } + MeasurementSchemaGroup* device_schema = schema_it->second; + if (record.timestamp_ <= device_schema->last_time_) { + return E_OUT_OF_ORDER; + } SimpleVector value_chunk_writers; SimpleVector data_types; TimeChunkWriter* time_chunk_writer; MeasurementNamesFromRecord mnames_getter(record); - if (RET_FAIL(do_check_schema_aligned( - std::make_shared(record.device_id_), - mnames_getter, time_chunk_writer, value_chunk_writers, - data_types))) { + if (RET_FAIL(do_check_schema_aligned(device_id, mnames_getter, + time_chunk_writer, value_chunk_writers, + data_types))) { return ret; } if (value_chunk_writers.size() != record.points_.size()) { @@ -755,6 +777,8 @@ int TsFileWriter::write_record_aligned(const TsRecord& record) { value_pages_before))) { return ret; } + device_schema->last_time_ = + std::max(device_schema->last_time_, record.timestamp_); return ret; } @@ -853,17 +877,26 @@ int TsFileWriter::maybe_seal_aligned_pages_together( int TsFileWriter::write_tablet_aligned(const Tablet& tablet) { int ret = E_OK; + auto device_id = + std::make_shared(tablet.insert_target_name_); + auto schema_it = schemas_.find(device_id); + if (schema_it == schemas_.end() || schema_it->second == nullptr) { + return E_DEVICE_NOT_EXIST; + } + MeasurementSchemaGroup* device_schema = schema_it->second; + const uint32_t total_rows = tablet.get_cur_row_size(); + if (total_rows > 0 && tablet.timestamps_[0] <= device_schema->last_time_) { + return E_OUT_OF_ORDER; + } SimpleVector value_chunk_writers; TimeChunkWriter* time_chunk_writer = nullptr; SimpleVector data_types; MeasurementNamesFromTablet mnames_getter(tablet); - if (RET_FAIL(do_check_schema_aligned( - std::make_shared(tablet.insert_target_name_), - mnames_getter, time_chunk_writer, value_chunk_writers, - data_types))) { + if (RET_FAIL(do_check_schema_aligned(device_id, mnames_getter, + time_chunk_writer, value_chunk_writers, + data_types))) { return ret; } - const uint32_t total_rows = tablet.get_cur_row_size(); const bool strict_page_size = common::g_config_value_.strict_page_size_; // Decide whether we have string/blob/text columns. @@ -919,6 +952,10 @@ int TsFileWriter::write_tablet_aligned(const Tablet& tablet) { return ret; } } + if (total_rows > 0) { + device_schema->last_time_ = std::max( + device_schema->last_time_, tablet.timestamps_[total_rows - 1]); + } return ret; } @@ -1010,6 +1047,10 @@ int TsFileWriter::write_tablet_aligned(const Tablet& tablet) { seg_len = points_per_page; } } + if (total_rows > 0) { + device_schema->last_time_ = std::max( + device_schema->last_time_, tablet.timestamps_[total_rows - 1]); + } return ret; } @@ -1077,17 +1118,31 @@ int TsFileWriter::write_tablet_aligned(const Tablet& tablet) { } } } + if (total_rows > 0) { + device_schema->last_time_ = std::max( + device_schema->last_time_, tablet.timestamps_[total_rows - 1]); + } return ret; } int TsFileWriter::write_tablet(const Tablet& tablet) { int ret = E_OK; + auto device_id = + std::make_shared(tablet.insert_target_name_); + auto schema_it = schemas_.find(device_id); + if (schema_it == schemas_.end() || schema_it->second == nullptr) { + return E_DEVICE_NOT_EXIST; + } + MeasurementSchemaGroup* device_schema = schema_it->second; + const uint32_t total_rows = tablet.get_cur_row_size(); + if (total_rows > 0 && tablet.timestamps_[0] <= device_schema->last_time_) { + return E_OUT_OF_ORDER; + } SimpleVector chunk_writers; SimpleVector data_types; MeasurementNamesFromTablet mnames_getter(tablet); - if (RET_FAIL(do_check_schema( - std::make_shared(tablet.insert_target_name_), - mnames_getter, chunk_writers, data_types))) { + if (RET_FAIL(do_check_schema(device_id, mnames_getter, chunk_writers, + data_types))) { return ret; } ASSERT(chunk_writers.size() == tablet.get_column_count()); @@ -1101,6 +1156,10 @@ int TsFileWriter::write_tablet(const Tablet& tablet) { } } + if (total_rows > 0) { + device_schema->last_time_ = std::max( + device_schema->last_time_, tablet.timestamps_[total_rows - 1]); + } record_count_since_last_flush_ += tablet.max_row_num_; ret = check_memory_size_and_may_flush_chunks(); return ret; @@ -1156,6 +1215,9 @@ int TsFileWriter::write_table(Tablet& tablet) { value_chunk_writers))) { return ret; } + auto schema_it = schemas_.find(device_id); + MeasurementSchemaGroup* device_schema = + (schema_it == schemas_.end()) ? nullptr : schema_it->second; std::vector field_columns; field_columns.reserve(tablet.get_column_count()); @@ -1174,6 +1236,10 @@ int TsFileWriter::write_table(Tablet& tablet) { 1, common::g_config_value_.page_writer_max_point_num_); const uint32_t si = static_cast(start_idx); const uint32_t ei = static_cast(end_idx); + if (device_schema != nullptr && si < ei && + tablet.timestamps_[si] <= device_schema->last_time_) { + return E_OUT_OF_ORDER; + } // If the current unsealed page is already at or past capacity (from // a previous write_table call), seal it before starting new segments. @@ -1301,6 +1367,10 @@ int TsFileWriter::write_table(Tablet& tablet) { } } } + if (device_schema != nullptr && si < ei) { + device_schema->last_time_ = + std::max(device_schema->last_time_, tablet.timestamps_[ei - 1]); + } start_idx = end_idx; } record_count_since_last_flush_ += tablet.cur_row_size_; diff --git a/cpp/test/file/restorable_tsfile_io_writer_test.cc b/cpp/test/file/restorable_tsfile_io_writer_test.cc index 934d11067..4490534c3 100644 --- a/cpp/test/file/restorable_tsfile_io_writer_test.cc +++ b/cpp/test/file/restorable_tsfile_io_writer_test.cc @@ -44,6 +44,7 @@ namespace storage { class ResultSet; } + using namespace storage; using namespace common; @@ -404,6 +405,41 @@ TEST_F(RestorableTsFileIOWriterTest, reader.close(); } +TEST_F(RestorableTsFileIOWriterTest, + TreeWriterRepeatedWriteAfterRecoveryShouldRejectDuplicateTimestamps) { + TsFileWriter tw; + ASSERT_EQ(tw.open(file_name_, GetWriteCreateFlags(), 0666), E_OK); + tw.register_timeseries( + "root.d1", + MeasurementSchema("s1", FLOAT, GORILLA, CompressionType::UNCOMPRESSED)); + TsRecord record(1, "root.d1"); + record.add_point("s1", 1.0f); + ASSERT_EQ(tw.write_record(record), E_OK); + record.timestamp_ = 2; + ASSERT_EQ(tw.write_record(record), E_OK); + tw.flush(); + tw.close(); + + for (int round = 0; round < 2; ++round) { + CorruptCurrentFileTail(3); + + RestorableTsFileIOWriter rw; + ASSERT_EQ(rw.open(file_name_, true), E_OK); + ASSERT_TRUE(rw.can_write()); + + TsFileTreeWriter tree_writer(&rw); + TsRecord record2(3, "root.d1"); + record2.add_point("s1", 3.0f); + if (round == 0) { + ASSERT_EQ(tree_writer.write(record2), E_OK); + ASSERT_EQ(tree_writer.flush(), E_OK); + } else { + ASSERT_EQ(tree_writer.write(record2), E_OUT_OF_ORDER); + } + ASSERT_EQ(tree_writer.close(), E_OK); + } +} + // ----------------------------------------------------------------------------- // Tree model + Recovery + continued write with aligned timeseries, then // read-back verify @@ -849,4 +885,117 @@ TEST_F(RestorableTsFileIOWriterTest, } EXPECT_TRUE(checked_null_tag_group); table_reader.close(); +} + +TEST_F(RestorableTsFileIOWriterTest, + TableWriterRepeatedWriteAfterRecoveryShouldRejectDuplicateTimestamps) { + using namespace std; + const string table_name = "test_table"; + vector column_names = {"t1", "t2", "t3", "f1", "f2", "f3", "f4", + "f5", "f6", "f7", "f8", "f9", "f10"}; + vector data_types = {STRING, STRING, STRING, BOOLEAN, INT32, + INT64, FLOAT, DOUBLE, TEXT, STRING, + BLOB, DATE, TIMESTAMP}; + std::vector column_schemas; + for (size_t i = 0; i < column_names.size(); i++) { + column_schemas.push_back( + new MeasurementSchema(column_names[i], data_types[i])); + } + std::vector column_categories = { + ColumnCategory::TAG, ColumnCategory::TAG, ColumnCategory::TAG, + ColumnCategory::FIELD, ColumnCategory::FIELD, ColumnCategory::FIELD, + ColumnCategory::FIELD, ColumnCategory::FIELD, ColumnCategory::FIELD, + ColumnCategory::FIELD, ColumnCategory::FIELD, ColumnCategory::FIELD, + ColumnCategory::FIELD}; + TableSchema table_schema(table_name, column_schemas, column_categories); + + WriteFile write_file; + ASSERT_EQ(write_file.create(file_name_, GetWriteCreateFlags(), 0666), E_OK); + TsFileTableWriter table_writer(&write_file, &table_schema); + constexpr uint32_t max_rows = 10; + Tablet tablet(table_schema.get_measurement_names(), + table_schema.get_data_types(), max_rows); + tablet.set_table_name(table_name); + for (int row = 0; row < static_cast(max_rows); row++) { + ASSERT_EQ(tablet.add_timestamp(row, static_cast(row)), E_OK); + ASSERT_EQ(tablet.add_value(row, "t1", "device1"), E_OK); + ASSERT_EQ(tablet.add_value(row, "t2", "device2"), E_OK); + ASSERT_EQ(tablet.add_value(row, "t3", "device3"), E_OK); + ASSERT_EQ(tablet.add_value(row, "f1", row % 2 == 0), E_OK); + ASSERT_EQ(tablet.add_value(row, "f2", static_cast(row)), E_OK); + ASSERT_EQ(tablet.add_value(row, "f3", static_cast(row)), E_OK); + ASSERT_EQ(tablet.add_value(row, "f4", static_cast(row * 1.1)), + E_OK); + ASSERT_EQ(tablet.add_value(row, "f5", static_cast(row * 1.1)), + E_OK); + ASSERT_EQ( + tablet.add_value(row, "f6", ("text" + to_string(row)).c_str()), + E_OK); + ASSERT_EQ( + tablet.add_value(row, "f7", ("string" + to_string(row)).c_str()), + E_OK); + ASSERT_EQ( + tablet.add_value(row, "f8", ("blob" + to_string(row)).c_str()), + E_OK); + ASSERT_EQ(tablet.add_value(row, "f9", static_cast(row)), E_OK); + ASSERT_EQ(tablet.add_value(row, "f10", static_cast(row)), + E_OK); + } + ASSERT_EQ(table_writer.write_table(tablet), E_OK); + ASSERT_EQ(table_writer.flush(), E_OK); + ASSERT_EQ(table_writer.close(), E_OK); + ASSERT_EQ(write_file.close(), E_OK); + + vector recovered_column_names = { + "__level1", "__level2", "__level3", "f1", "f2", "f3", "f4", + "f5", "f6", "f7", "f8", "f9", "f10"}; + for (int round = 0; round < 2; ++round) { + CorruptCurrentFileTail(10); + RestorableTsFileIOWriter rw; + ASSERT_EQ(rw.open(file_name_, true), E_OK); + ASSERT_TRUE(rw.can_write()); + + TsFileTableWriter table_writer2(&rw); + Tablet tablet2(recovered_column_names, data_types, max_rows); + tablet2.set_table_name(table_name); + for (int row = 0; row < static_cast(max_rows); row++) { + ASSERT_EQ( + tablet2.add_timestamp(row, static_cast(row + 10)), + E_OK); + ASSERT_EQ(tablet2.add_value(row, "__level1", "device1"), E_OK); + ASSERT_EQ(tablet2.add_value(row, "__level2", "device2"), E_OK); + ASSERT_EQ(tablet2.add_value(row, "__level3", "device3"), E_OK); + ASSERT_EQ(tablet2.add_value(row, "f1", row % 2 == 0), E_OK); + ASSERT_EQ(tablet2.add_value(row, "f2", static_cast(row)), + E_OK); + ASSERT_EQ(tablet2.add_value(row, "f3", static_cast(row)), + E_OK); + ASSERT_EQ( + tablet2.add_value(row, "f4", static_cast(row * 1.1)), + E_OK); + ASSERT_EQ( + tablet2.add_value(row, "f5", static_cast(row * 1.1)), + E_OK); + ASSERT_EQ( + tablet2.add_value(row, "f6", ("text" + to_string(row)).c_str()), + E_OK); + ASSERT_EQ(tablet2.add_value(row, "f7", + ("string" + to_string(row)).c_str()), + E_OK); + ASSERT_EQ( + tablet2.add_value(row, "f8", ("blob" + to_string(row)).c_str()), + E_OK); + ASSERT_EQ(tablet2.add_value(row, "f9", static_cast(row)), + E_OK); + ASSERT_EQ(tablet2.add_value(row, "f10", static_cast(row)), + E_OK); + } + if (round == 0) { + ASSERT_EQ(table_writer2.write_table(tablet2), E_OK); + ASSERT_EQ(table_writer2.flush(), E_OK); + } else { + ASSERT_EQ(table_writer2.write_table(tablet2), E_OUT_OF_ORDER); + } + ASSERT_EQ(table_writer2.close(), E_OK); + } } \ No newline at end of file From 577f78ff2f26a3be4438b77922e20882524bcfbc Mon Sep 17 00:00:00 2001 From: 761417898 <761417898@qq.com> Date: Fri, 17 Apr 2026 16:38:58 +0800 Subject: [PATCH 3/4] fix ci --- cpp/src/writer/tsfile_writer.cc | 21 ++++++++++++++------- cpp/src/writer/tsfile_writer.h | 1 + 2 files changed, 15 insertions(+), 7 deletions(-) diff --git a/cpp/src/writer/tsfile_writer.cc b/cpp/src/writer/tsfile_writer.cc index 7ae265ac7..9bdbaa08d 100644 --- a/cpp/src/writer/tsfile_writer.cc +++ b/cpp/src/writer/tsfile_writer.cc @@ -81,7 +81,8 @@ TsFileWriter::TsFileWriter() record_count_for_next_mem_check_( g_config_value_.record_count_for_next_mem_check_), write_file_created_(false), - io_writer_owned_(true) {} + io_writer_owned_(true), + enforce_recovered_last_time_order_(false) {} TsFileWriter::~TsFileWriter() { destroy(); } @@ -127,6 +128,7 @@ int TsFileWriter::init(WriteFile* write_file) { write_file_ = write_file; write_file_created_ = false; io_writer_owned_ = true; + enforce_recovered_last_time_order_ = false; io_writer_ = new TsFileIOWriter(); io_writer_->init(write_file_); return E_OK; @@ -146,6 +148,7 @@ int TsFileWriter::init(RestorableTsFileIOWriter* rw) { write_file_ = rw->get_write_file(); write_file_created_ = false; io_writer_owned_ = false; + enforce_recovered_last_time_order_ = true; io_writer_ = rw; const std::vector& recovered = @@ -702,7 +705,8 @@ int TsFileWriter::write_record(const TsRecord& record) { return E_DEVICE_NOT_EXIST; } MeasurementSchemaGroup* device_schema = schema_it->second; - if (record.timestamp_ <= device_schema->last_time_) { + if (enforce_recovered_last_time_order_ && + record.timestamp_ <= device_schema->last_time_) { return E_OUT_OF_ORDER; } // std::vector chunk_writers; @@ -740,7 +744,8 @@ int TsFileWriter::write_record_aligned(const TsRecord& record) { return E_DEVICE_NOT_EXIST; } MeasurementSchemaGroup* device_schema = schema_it->second; - if (record.timestamp_ <= device_schema->last_time_) { + if (enforce_recovered_last_time_order_ && + record.timestamp_ <= device_schema->last_time_) { return E_OUT_OF_ORDER; } SimpleVector value_chunk_writers; @@ -885,7 +890,8 @@ int TsFileWriter::write_tablet_aligned(const Tablet& tablet) { } MeasurementSchemaGroup* device_schema = schema_it->second; const uint32_t total_rows = tablet.get_cur_row_size(); - if (total_rows > 0 && tablet.timestamps_[0] <= device_schema->last_time_) { + if (enforce_recovered_last_time_order_ && total_rows > 0 && + tablet.timestamps_[0] <= device_schema->last_time_) { return E_OUT_OF_ORDER; } SimpleVector value_chunk_writers; @@ -1135,7 +1141,8 @@ int TsFileWriter::write_tablet(const Tablet& tablet) { } MeasurementSchemaGroup* device_schema = schema_it->second; const uint32_t total_rows = tablet.get_cur_row_size(); - if (total_rows > 0 && tablet.timestamps_[0] <= device_schema->last_time_) { + if (enforce_recovered_last_time_order_ && total_rows > 0 && + tablet.timestamps_[0] <= device_schema->last_time_) { return E_OUT_OF_ORDER; } SimpleVector chunk_writers; @@ -1236,8 +1243,8 @@ int TsFileWriter::write_table(Tablet& tablet) { 1, common::g_config_value_.page_writer_max_point_num_); const uint32_t si = static_cast(start_idx); const uint32_t ei = static_cast(end_idx); - if (device_schema != nullptr && si < ei && - tablet.timestamps_[si] <= device_schema->last_time_) { + if (enforce_recovered_last_time_order_ && device_schema != nullptr && + si < ei && tablet.timestamps_[si] <= device_schema->last_time_) { return E_OUT_OF_ORDER; } diff --git a/cpp/src/writer/tsfile_writer.h b/cpp/src/writer/tsfile_writer.h index cb1b952e8..a931345af 100644 --- a/cpp/src/writer/tsfile_writer.h +++ b/cpp/src/writer/tsfile_writer.h @@ -195,6 +195,7 @@ class TsFileWriter { int64_t record_count_for_next_mem_check_; bool write_file_created_; bool io_writer_owned_; // false when init(RestorableTsFileIOWriter*) + bool enforce_recovered_last_time_order_; int write_typed_column(ValueChunkWriter* value_chunk_writer, int64_t* timestamps, bool* col_values, From c3e78943295f8a8b01d277b31a34e80cc7bc19f0 Mon Sep 17 00:00:00 2001 From: 761417898 <761417898@qq.com> Date: Mon, 20 Apr 2026 09:27:52 +0800 Subject: [PATCH 4/4] removed unused aligned value column count --- cpp/src/file/restorable_tsfile_io_writer.cc | 13 +++++-------- cpp/test/file/restorable_tsfile_io_writer_test.cc | 4 ---- 2 files changed, 5 insertions(+), 12 deletions(-) diff --git a/cpp/src/file/restorable_tsfile_io_writer.cc b/cpp/src/file/restorable_tsfile_io_writer.cc index 720ac9502..22a3fb500 100644 --- a/cpp/src/file/restorable_tsfile_io_writer.cc +++ b/cpp/src/file/restorable_tsfile_io_writer.cc @@ -424,14 +424,11 @@ static int recover_chunk_statistic( bool has_value = true; if (is_aligned_value_chunk) { has_value = false; - if (idx < aligned_num_values) { - const uint32_t byte_idx = static_cast(idx / 8); - const uint32_t bit_shift = static_cast(idx % 8); - if (byte_idx < aligned_value_notnull_bitmap.size()) { - has_value = - ((aligned_value_notnull_bitmap[byte_idx] & 0xFF) & - (0x80 >> bit_shift)) != 0; - } + const uint32_t byte_idx = static_cast(idx / 8); + const uint32_t bit_shift = static_cast(idx % 8); + if (byte_idx < aligned_value_notnull_bitmap.size()) { + has_value = ((aligned_value_notnull_bitmap[byte_idx] & 0xFF) & + (0x80 >> bit_shift)) != 0; } } if (!has_value) { diff --git a/cpp/test/file/restorable_tsfile_io_writer_test.cc b/cpp/test/file/restorable_tsfile_io_writer_test.cc index 4490534c3..8f723e056 100644 --- a/cpp/test/file/restorable_tsfile_io_writer_test.cc +++ b/cpp/test/file/restorable_tsfile_io_writer_test.cc @@ -603,7 +603,6 @@ TEST_F(RestorableTsFileIOWriterTest, TableWriterRecoverAndWrite1) { ColumnCategory::FIELD, ColumnCategory::FIELD}; TableSchema table_schema(table_name, column_schemas, column_categories); - // 2. 写入数据 WriteFile write_file; write_file.create(file_name_, GetWriteCreateFlags(), 0666); TsFileTableWriter table_writer(&write_file, &table_schema); @@ -651,7 +650,6 @@ TEST_F(RestorableTsFileIOWriterTest, TableWriterRecoverAndWrite1) { ASSERT_EQ(table_writer.close(), E_OK); ASSERT_EQ(write_file.close(), E_OK); - // 3. 损坏文件并继续写入数据 CorruptCurrentFileTail(10); RestorableTsFileIOWriter rw; ASSERT_EQ(rw.open(file_name_, true), E_OK); @@ -708,7 +706,6 @@ TEST_F(RestorableTsFileIOWriterTest, TableWriterRecoverAndWrite1) { ASSERT_EQ(table_writer2.flush(), E_OK); ASSERT_EQ(table_writer2.close(), E_OK); - // 4. 查询元数据与行数(与 TableWriterRecoverAndWrite 一致:统计行,不打印) TsFileReader table_reader; ASSERT_EQ(table_reader.open(file_name_), E_OK); DeviceTimeseriesMetadataMap metadata = @@ -756,7 +753,6 @@ TEST_F(RestorableTsFileIOWriterTest, ColumnCategory::FIELD}; TableSchema table_schema(table_name, column_schemas, column_categories); - // 首轮写入:奇数行仅写时间,制造 __level1/2/3 全空值的 TAG 组。 WriteFile write_file; ASSERT_EQ(write_file.create(file_name_, GetWriteCreateFlags(), 0666), E_OK); TsFileTableWriter table_writer(&write_file, &table_schema);