diff --git a/cpp/src/common/schema.h b/cpp/src/common/schema.h index a2c989af2..81008b715 100644 --- a/cpp/src/common/schema.h +++ b/cpp/src/common/schema.h @@ -23,6 +23,7 @@ #include #include +#include #include // use unordered_map instead #include #include @@ -165,6 +166,7 @@ struct MeasurementSchemaGroup { MeasurementSchemaMap measurement_schema_map_; bool is_aligned_ = false; TimeChunkWriter* time_chunk_writer_ = nullptr; + int64_t last_time_ = INT64_MIN; ~MeasurementSchemaGroup() { if (time_chunk_writer_ != nullptr) { diff --git a/cpp/src/file/restorable_tsfile_io_writer.cc b/cpp/src/file/restorable_tsfile_io_writer.cc index 05e5b5112..22a3fb500 100644 --- a/cpp/src/file/restorable_tsfile_io_writer.cc +++ b/cpp/src/file/restorable_tsfile_io_writer.cc @@ -328,8 +328,12 @@ static int recover_chunk_statistic( uint32_t value_buf_size = 0; std::vector time_decode_buf; const std::vector* times = nullptr; + std::vector aligned_value_notnull_bitmap; + uint32_t aligned_num_values = 0; + const bool is_aligned_value_chunk = + (time_batch != nullptr && !time_batch->empty()); - if (time_batch != nullptr && !time_batch->empty()) { + if (is_aligned_value_chunk) { // Aligned value page: uncompressed layout = uint32(num_values) + bitmap // + value_buf if (uncompressed_size < 4) { @@ -337,7 +341,7 @@ static int recover_chunk_statistic( CompressorFactory::free(compressor); return E_OK; } - uint32_t num_values = + aligned_num_values = (static_cast( static_cast(uncompressed_buf[0])) << 24) | @@ -349,12 +353,17 @@ static int recover_chunk_statistic( << 8) | (static_cast( static_cast(uncompressed_buf[3]))); - uint32_t bitmap_size = (num_values + 7) / 8; + uint32_t bitmap_size = (aligned_num_values + 7) / 8; if (uncompressed_size < 4 + bitmap_size) { compressor->after_uncompress(uncompressed_buf); CompressorFactory::free(compressor); return E_OK; } + aligned_value_notnull_bitmap.resize(bitmap_size); + if (bitmap_size > 0) { + std::memcpy(aligned_value_notnull_bitmap.data(), + uncompressed_buf + 4, bitmap_size); + } value_buf = uncompressed_buf + 4 + bitmap_size; value_buf_size = uncompressed_size - 4 - bitmap_size; times = time_batch; @@ -410,8 +419,25 @@ static int recover_chunk_statistic( value_decoder->reset(); size_t idx = 0; const size_t num_times = times->size(); - while (idx < num_times && value_decoder->has_remaining(value_in)) { + while (idx < num_times) { int64_t t = (*times)[idx]; + bool has_value = true; + if (is_aligned_value_chunk) { + has_value = false; + const uint32_t byte_idx = static_cast(idx / 8); + const uint32_t bit_shift = static_cast(idx % 8); + if (byte_idx < aligned_value_notnull_bitmap.size()) { + has_value = ((aligned_value_notnull_bitmap[byte_idx] & 0xFF) & + (0x80 >> bit_shift)) != 0; + } + } + if (!has_value) { + idx++; + continue; + } + if (!value_decoder->has_remaining(value_in)) { + break; + } switch (chdr.data_type_) { case common::BOOLEAN: { bool v; diff --git a/cpp/src/writer/tsfile_writer.cc b/cpp/src/writer/tsfile_writer.cc index 35520c88d..9bdbaa08d 100644 --- a/cpp/src/writer/tsfile_writer.cc +++ b/cpp/src/writer/tsfile_writer.cc @@ -81,7 +81,8 @@ TsFileWriter::TsFileWriter() record_count_for_next_mem_check_( g_config_value_.record_count_for_next_mem_check_), write_file_created_(false), - io_writer_owned_(true) {} + io_writer_owned_(true), + enforce_recovered_last_time_order_(false) {} TsFileWriter::~TsFileWriter() { destroy(); } @@ -127,6 +128,7 @@ int TsFileWriter::init(WriteFile* write_file) { write_file_ = write_file; write_file_created_ = false; io_writer_owned_ = true; + enforce_recovered_last_time_order_ = false; io_writer_ = new TsFileIOWriter(); io_writer_->init(write_file_); return E_OK; @@ -146,6 +148,7 @@ int TsFileWriter::init(RestorableTsFileIOWriter* rw) { write_file_ = rw->get_write_file(); write_file_created_ = false; io_writer_owned_ = false; + enforce_recovered_last_time_order_ = true; io_writer_ = rw; const std::vector& recovered = @@ -182,6 +185,10 @@ int TsFileWriter::init(RestorableTsFileIOWriter* rw) { if (cm == nullptr) { continue; } + if (cm->statistic_ != nullptr && cm->statistic_->count_ > 0) { + group->last_time_ = + std::max(group->last_time_, cm->statistic_->end_time_); + } std::string mname = cm->measurement_name_.to_std_string(); if (mname.empty()) { continue; @@ -692,13 +699,22 @@ int TsFileWriter::check_memory_size_and_may_flush_chunks() { int TsFileWriter::write_record(const TsRecord& record) { int ret = E_OK; + auto device_id = std::make_shared(record.device_id_); + auto schema_it = schemas_.find(device_id); + if (schema_it == schemas_.end() || schema_it->second == nullptr) { + return E_DEVICE_NOT_EXIST; + } + MeasurementSchemaGroup* device_schema = schema_it->second; + if (enforce_recovered_last_time_order_ && + record.timestamp_ <= device_schema->last_time_) { + return E_OUT_OF_ORDER; + } // std::vector chunk_writers; SimpleVector chunk_writers; SimpleVector data_types; MeasurementNamesFromRecord mnames_getter(record); - if (RET_FAIL(do_check_schema( - std::make_shared(record.device_id_), - mnames_getter, chunk_writers, data_types))) { + if (RET_FAIL(do_check_schema(device_id, mnames_getter, chunk_writers, + data_types))) { return ret; } @@ -713,6 +729,8 @@ int TsFileWriter::write_record(const TsRecord& record) { record.points_[c]); } + device_schema->last_time_ = + std::max(device_schema->last_time_, record.timestamp_); record_count_since_last_flush_++; ret = check_memory_size_and_may_flush_chunks(); return ret; @@ -720,14 +738,23 @@ int TsFileWriter::write_record(const TsRecord& record) { int TsFileWriter::write_record_aligned(const TsRecord& record) { int ret = E_OK; + auto device_id = std::make_shared(record.device_id_); + auto schema_it = schemas_.find(device_id); + if (schema_it == schemas_.end() || schema_it->second == nullptr) { + return E_DEVICE_NOT_EXIST; + } + MeasurementSchemaGroup* device_schema = schema_it->second; + if (enforce_recovered_last_time_order_ && + record.timestamp_ <= device_schema->last_time_) { + return E_OUT_OF_ORDER; + } SimpleVector value_chunk_writers; SimpleVector data_types; TimeChunkWriter* time_chunk_writer; MeasurementNamesFromRecord mnames_getter(record); - if (RET_FAIL(do_check_schema_aligned( - std::make_shared(record.device_id_), - mnames_getter, time_chunk_writer, value_chunk_writers, - data_types))) { + if (RET_FAIL(do_check_schema_aligned(device_id, mnames_getter, + time_chunk_writer, value_chunk_writers, + data_types))) { return ret; } if (value_chunk_writers.size() != record.points_.size()) { @@ -755,6 +782,8 @@ int TsFileWriter::write_record_aligned(const TsRecord& record) { value_pages_before))) { return ret; } + device_schema->last_time_ = + std::max(device_schema->last_time_, record.timestamp_); return ret; } @@ -853,17 +882,27 @@ int TsFileWriter::maybe_seal_aligned_pages_together( int TsFileWriter::write_tablet_aligned(const Tablet& tablet) { int ret = E_OK; + auto device_id = + std::make_shared(tablet.insert_target_name_); + auto schema_it = schemas_.find(device_id); + if (schema_it == schemas_.end() || schema_it->second == nullptr) { + return E_DEVICE_NOT_EXIST; + } + MeasurementSchemaGroup* device_schema = schema_it->second; + const uint32_t total_rows = tablet.get_cur_row_size(); + if (enforce_recovered_last_time_order_ && total_rows > 0 && + tablet.timestamps_[0] <= device_schema->last_time_) { + return E_OUT_OF_ORDER; + } SimpleVector value_chunk_writers; TimeChunkWriter* time_chunk_writer = nullptr; SimpleVector data_types; MeasurementNamesFromTablet mnames_getter(tablet); - if (RET_FAIL(do_check_schema_aligned( - std::make_shared(tablet.insert_target_name_), - mnames_getter, time_chunk_writer, value_chunk_writers, - data_types))) { + if (RET_FAIL(do_check_schema_aligned(device_id, mnames_getter, + time_chunk_writer, value_chunk_writers, + data_types))) { return ret; } - const uint32_t total_rows = tablet.get_cur_row_size(); const bool strict_page_size = common::g_config_value_.strict_page_size_; // Decide whether we have string/blob/text columns. @@ -919,6 +958,10 @@ int TsFileWriter::write_tablet_aligned(const Tablet& tablet) { return ret; } } + if (total_rows > 0) { + device_schema->last_time_ = std::max( + device_schema->last_time_, tablet.timestamps_[total_rows - 1]); + } return ret; } @@ -1010,6 +1053,10 @@ int TsFileWriter::write_tablet_aligned(const Tablet& tablet) { seg_len = points_per_page; } } + if (total_rows > 0) { + device_schema->last_time_ = std::max( + device_schema->last_time_, tablet.timestamps_[total_rows - 1]); + } return ret; } @@ -1077,17 +1124,32 @@ int TsFileWriter::write_tablet_aligned(const Tablet& tablet) { } } } + if (total_rows > 0) { + device_schema->last_time_ = std::max( + device_schema->last_time_, tablet.timestamps_[total_rows - 1]); + } return ret; } int TsFileWriter::write_tablet(const Tablet& tablet) { int ret = E_OK; + auto device_id = + std::make_shared(tablet.insert_target_name_); + auto schema_it = schemas_.find(device_id); + if (schema_it == schemas_.end() || schema_it->second == nullptr) { + return E_DEVICE_NOT_EXIST; + } + MeasurementSchemaGroup* device_schema = schema_it->second; + const uint32_t total_rows = tablet.get_cur_row_size(); + if (enforce_recovered_last_time_order_ && total_rows > 0 && + tablet.timestamps_[0] <= device_schema->last_time_) { + return E_OUT_OF_ORDER; + } SimpleVector chunk_writers; SimpleVector data_types; MeasurementNamesFromTablet mnames_getter(tablet); - if (RET_FAIL(do_check_schema( - std::make_shared(tablet.insert_target_name_), - mnames_getter, chunk_writers, data_types))) { + if (RET_FAIL(do_check_schema(device_id, mnames_getter, chunk_writers, + data_types))) { return ret; } ASSERT(chunk_writers.size() == tablet.get_column_count()); @@ -1101,6 +1163,10 @@ int TsFileWriter::write_tablet(const Tablet& tablet) { } } + if (total_rows > 0) { + device_schema->last_time_ = std::max( + device_schema->last_time_, tablet.timestamps_[total_rows - 1]); + } record_count_since_last_flush_ += tablet.max_row_num_; ret = check_memory_size_and_may_flush_chunks(); return ret; @@ -1156,6 +1222,9 @@ int TsFileWriter::write_table(Tablet& tablet) { value_chunk_writers))) { return ret; } + auto schema_it = schemas_.find(device_id); + MeasurementSchemaGroup* device_schema = + (schema_it == schemas_.end()) ? nullptr : schema_it->second; std::vector field_columns; field_columns.reserve(tablet.get_column_count()); @@ -1174,6 +1243,10 @@ int TsFileWriter::write_table(Tablet& tablet) { 1, common::g_config_value_.page_writer_max_point_num_); const uint32_t si = static_cast(start_idx); const uint32_t ei = static_cast(end_idx); + if (enforce_recovered_last_time_order_ && device_schema != nullptr && + si < ei && tablet.timestamps_[si] <= device_schema->last_time_) { + return E_OUT_OF_ORDER; + } // If the current unsealed page is already at or past capacity (from // a previous write_table call), seal it before starting new segments. @@ -1301,6 +1374,10 @@ int TsFileWriter::write_table(Tablet& tablet) { } } } + if (device_schema != nullptr && si < ei) { + device_schema->last_time_ = + std::max(device_schema->last_time_, tablet.timestamps_[ei - 1]); + } start_idx = end_idx; } record_count_since_last_flush_ += tablet.cur_row_size_; diff --git a/cpp/src/writer/tsfile_writer.h b/cpp/src/writer/tsfile_writer.h index cb1b952e8..a931345af 100644 --- a/cpp/src/writer/tsfile_writer.h +++ b/cpp/src/writer/tsfile_writer.h @@ -195,6 +195,7 @@ class TsFileWriter { int64_t record_count_for_next_mem_check_; bool write_file_created_; bool io_writer_owned_; // false when init(RestorableTsFileIOWriter*) + bool enforce_recovered_last_time_order_; int write_typed_column(ValueChunkWriter* value_chunk_writer, int64_t* timestamps, bool* col_values, diff --git a/cpp/test/file/restorable_tsfile_io_writer_test.cc b/cpp/test/file/restorable_tsfile_io_writer_test.cc index 3c6687e78..8f723e056 100644 --- a/cpp/test/file/restorable_tsfile_io_writer_test.cc +++ b/cpp/test/file/restorable_tsfile_io_writer_test.cc @@ -44,6 +44,7 @@ namespace storage { class ResultSet; } + using namespace storage; using namespace common; @@ -404,6 +405,41 @@ TEST_F(RestorableTsFileIOWriterTest, reader.close(); } +TEST_F(RestorableTsFileIOWriterTest, + TreeWriterRepeatedWriteAfterRecoveryShouldRejectDuplicateTimestamps) { + TsFileWriter tw; + ASSERT_EQ(tw.open(file_name_, GetWriteCreateFlags(), 0666), E_OK); + tw.register_timeseries( + "root.d1", + MeasurementSchema("s1", FLOAT, GORILLA, CompressionType::UNCOMPRESSED)); + TsRecord record(1, "root.d1"); + record.add_point("s1", 1.0f); + ASSERT_EQ(tw.write_record(record), E_OK); + record.timestamp_ = 2; + ASSERT_EQ(tw.write_record(record), E_OK); + tw.flush(); + tw.close(); + + for (int round = 0; round < 2; ++round) { + CorruptCurrentFileTail(3); + + RestorableTsFileIOWriter rw; + ASSERT_EQ(rw.open(file_name_, true), E_OK); + ASSERT_TRUE(rw.can_write()); + + TsFileTreeWriter tree_writer(&rw); + TsRecord record2(3, "root.d1"); + record2.add_point("s1", 3.0f); + if (round == 0) { + ASSERT_EQ(tree_writer.write(record2), E_OK); + ASSERT_EQ(tree_writer.flush(), E_OK); + } else { + ASSERT_EQ(tree_writer.write(record2), E_OUT_OF_ORDER); + } + ASSERT_EQ(tree_writer.close(), E_OK); + } +} + // ----------------------------------------------------------------------------- // Tree model + Recovery + continued write with aligned timeseries, then // read-back verify @@ -567,7 +603,6 @@ TEST_F(RestorableTsFileIOWriterTest, TableWriterRecoverAndWrite1) { ColumnCategory::FIELD, ColumnCategory::FIELD}; TableSchema table_schema(table_name, column_schemas, column_categories); - // 2. 写入数据 WriteFile write_file; write_file.create(file_name_, GetWriteCreateFlags(), 0666); TsFileTableWriter table_writer(&write_file, &table_schema); @@ -615,7 +650,6 @@ TEST_F(RestorableTsFileIOWriterTest, TableWriterRecoverAndWrite1) { ASSERT_EQ(table_writer.close(), E_OK); ASSERT_EQ(write_file.close(), E_OK); - // 3. 损坏文件并继续写入数据 CorruptCurrentFileTail(10); RestorableTsFileIOWriter rw; ASSERT_EQ(rw.open(file_name_, true), E_OK); @@ -672,7 +706,6 @@ TEST_F(RestorableTsFileIOWriterTest, TableWriterRecoverAndWrite1) { ASSERT_EQ(table_writer2.flush(), E_OK); ASSERT_EQ(table_writer2.close(), E_OK); - // 4. 查询元数据与行数(与 TableWriterRecoverAndWrite 一致:统计行,不打印) TsFileReader table_reader; ASSERT_EQ(table_reader.open(file_name_), E_OK); DeviceTimeseriesMetadataMap metadata = @@ -696,4 +729,269 @@ TEST_F(RestorableTsFileIOWriterTest, TableWriterRecoverAndWrite1) { table_result_set->close(); table_reader.destroy_query_data_set(temp_ret); table_reader.close(); +} + +TEST_F(RestorableTsFileIOWriterTest, + TableWriterRecoverAndWriteNullTagFloatDoubleStatistics) { + using namespace std; + const string table_name = "test_table"; + vector column_names = {"t1", "t2", "t3", "f1", "f2", "f3", "f4", + "f5", "f6", "f7", "f8", "f9", "f10"}; + vector data_types = {STRING, STRING, STRING, BOOLEAN, INT32, + INT64, FLOAT, DOUBLE, TEXT, STRING, + BLOB, DATE, TIMESTAMP}; + std::vector column_schemas; + for (size_t i = 0; i < column_names.size(); i++) { + column_schemas.push_back( + new MeasurementSchema(column_names[i], data_types[i])); + } + std::vector column_categories = { + ColumnCategory::TAG, ColumnCategory::TAG, ColumnCategory::TAG, + ColumnCategory::FIELD, ColumnCategory::FIELD, ColumnCategory::FIELD, + ColumnCategory::FIELD, ColumnCategory::FIELD, ColumnCategory::FIELD, + ColumnCategory::FIELD, ColumnCategory::FIELD, ColumnCategory::FIELD, + ColumnCategory::FIELD}; + TableSchema table_schema(table_name, column_schemas, column_categories); + + WriteFile write_file; + ASSERT_EQ(write_file.create(file_name_, GetWriteCreateFlags(), 0666), E_OK); + TsFileTableWriter table_writer(&write_file, &table_schema); + constexpr uint32_t max_rows = 10; + Tablet tablet(table_schema.get_measurement_names(), + table_schema.get_data_types(), max_rows); + tablet.set_table_name(table_name); + for (int row = 0; row < static_cast(max_rows); row++) { + ASSERT_EQ(tablet.add_timestamp(row, static_cast(row)), E_OK); + if (row % 2 == 0) { + ASSERT_EQ(tablet.add_value(row, "t1", "device1"), E_OK); + ASSERT_EQ(tablet.add_value(row, "t2", "device2"), E_OK); + ASSERT_EQ(tablet.add_value(row, "t3", "device3"), E_OK); + ASSERT_EQ(tablet.add_value(row, "f1", row % 2 == 0), E_OK); + ASSERT_EQ(tablet.add_value(row, "f2", static_cast(row)), + E_OK); + ASSERT_EQ(tablet.add_value(row, "f3", static_cast(row)), + E_OK); + ASSERT_EQ( + tablet.add_value(row, "f4", static_cast(row * 1.1)), + E_OK); + ASSERT_EQ( + tablet.add_value(row, "f5", static_cast(row * 1.1)), + E_OK); + ASSERT_EQ( + tablet.add_value(row, "f6", ("text" + to_string(row)).c_str()), + E_OK); + ASSERT_EQ(tablet.add_value(row, "f7", + ("string" + to_string(row)).c_str()), + E_OK); + ASSERT_EQ( + tablet.add_value(row, "f8", ("blob" + to_string(row)).c_str()), + E_OK); + ASSERT_EQ(tablet.add_value(row, "f9", static_cast(row)), + E_OK); + ASSERT_EQ(tablet.add_value(row, "f10", static_cast(row)), + E_OK); + } + } + ASSERT_EQ(table_writer.write_table(tablet), E_OK); + ASSERT_EQ(table_writer.flush(), E_OK); + ASSERT_EQ(table_writer.close(), E_OK); + ASSERT_EQ(write_file.close(), E_OK); + + CorruptCurrentFileTail(10); + + RestorableTsFileIOWriter rw; + ASSERT_EQ(rw.open(file_name_, true), E_OK); + ASSERT_TRUE(rw.can_write()); + + TsFileTableWriter table_writer2(&rw); + vector column_names2 = { + "__level1", "__level2", "__level3", "f1", "f2", "f3", "f4", + "f5", "f6", "f7", "f8", "f9", "f10"}; + Tablet tablet2(column_names2, data_types, max_rows); + tablet2.set_table_name(table_name); + for (int row = 0; row < static_cast(max_rows); row++) { + ASSERT_EQ( + tablet2.add_timestamp(row, static_cast(row + max_rows)), + E_OK); + ASSERT_EQ(tablet2.add_value(row, "__level1", "device1"), E_OK); + ASSERT_EQ(tablet2.add_value(row, "__level2", "device2"), E_OK); + ASSERT_EQ(tablet2.add_value(row, "__level3", "device3"), E_OK); + ASSERT_EQ(tablet2.add_value(row, "f1", row % 2 == 0), E_OK); + ASSERT_EQ(tablet2.add_value(row, "f2", static_cast(row)), + E_OK); + ASSERT_EQ(tablet2.add_value(row, "f3", static_cast(row)), + E_OK); + ASSERT_EQ(tablet2.add_value(row, "f4", static_cast(row * 1.1)), + E_OK); + ASSERT_EQ(tablet2.add_value(row, "f5", static_cast(row * 1.1)), + E_OK); + ASSERT_EQ( + tablet2.add_value(row, "f6", ("text" + to_string(row)).c_str()), + E_OK); + ASSERT_EQ( + tablet2.add_value(row, "f7", ("string" + to_string(row)).c_str()), + E_OK); + ASSERT_EQ( + tablet2.add_value(row, "f8", ("blob" + to_string(row)).c_str()), + E_OK); + ASSERT_EQ(tablet2.add_value(row, "f9", static_cast(row)), + E_OK); + ASSERT_EQ(tablet2.add_value(row, "f10", static_cast(row)), + E_OK); + } + ASSERT_EQ(table_writer2.write_table(tablet2), E_OK); + ASSERT_EQ(table_writer2.flush(), E_OK); + ASSERT_EQ(table_writer2.close(), E_OK); + + TsFileReader table_reader; + ASSERT_EQ(table_reader.open(file_name_), E_OK); + DeviceTimeseriesMetadataMap metadata = + table_reader.get_timeseries_metadata(); + + bool checked_null_tag_group = false; + for (const auto& entry : metadata) { + const auto& device_id = entry.first; + if (device_id == nullptr) { + continue; + } + const std::string device_name = device_id->get_device_name(); + if (device_name.find("null.null.null") == std::string::npos) { + continue; + } + bool checked_f4 = false; + bool checked_f5 = false; + for (const auto& field : entry.second) { + const auto field_name = + field->get_measurement_name().to_std_string(); + if (field_name == "f4" || field_name == "f5") { + ASSERT_NE(field->get_statistic(), nullptr); + EXPECT_EQ(field->get_statistic()->count_, 0); + EXPECT_EQ(field->get_statistic()->start_time_, 0); + EXPECT_EQ(field->get_statistic()->end_time_, 0); + if (field_name == "f4") { + checked_f4 = true; + } else { + checked_f5 = true; + } + } + } + EXPECT_TRUE(checked_f4); + EXPECT_TRUE(checked_f5); + checked_null_tag_group = true; + } + EXPECT_TRUE(checked_null_tag_group); + table_reader.close(); +} + +TEST_F(RestorableTsFileIOWriterTest, + TableWriterRepeatedWriteAfterRecoveryShouldRejectDuplicateTimestamps) { + using namespace std; + const string table_name = "test_table"; + vector column_names = {"t1", "t2", "t3", "f1", "f2", "f3", "f4", + "f5", "f6", "f7", "f8", "f9", "f10"}; + vector data_types = {STRING, STRING, STRING, BOOLEAN, INT32, + INT64, FLOAT, DOUBLE, TEXT, STRING, + BLOB, DATE, TIMESTAMP}; + std::vector column_schemas; + for (size_t i = 0; i < column_names.size(); i++) { + column_schemas.push_back( + new MeasurementSchema(column_names[i], data_types[i])); + } + std::vector column_categories = { + ColumnCategory::TAG, ColumnCategory::TAG, ColumnCategory::TAG, + ColumnCategory::FIELD, ColumnCategory::FIELD, ColumnCategory::FIELD, + ColumnCategory::FIELD, ColumnCategory::FIELD, ColumnCategory::FIELD, + ColumnCategory::FIELD, ColumnCategory::FIELD, ColumnCategory::FIELD, + ColumnCategory::FIELD}; + TableSchema table_schema(table_name, column_schemas, column_categories); + + WriteFile write_file; + ASSERT_EQ(write_file.create(file_name_, GetWriteCreateFlags(), 0666), E_OK); + TsFileTableWriter table_writer(&write_file, &table_schema); + constexpr uint32_t max_rows = 10; + Tablet tablet(table_schema.get_measurement_names(), + table_schema.get_data_types(), max_rows); + tablet.set_table_name(table_name); + for (int row = 0; row < static_cast(max_rows); row++) { + ASSERT_EQ(tablet.add_timestamp(row, static_cast(row)), E_OK); + ASSERT_EQ(tablet.add_value(row, "t1", "device1"), E_OK); + ASSERT_EQ(tablet.add_value(row, "t2", "device2"), E_OK); + ASSERT_EQ(tablet.add_value(row, "t3", "device3"), E_OK); + ASSERT_EQ(tablet.add_value(row, "f1", row % 2 == 0), E_OK); + ASSERT_EQ(tablet.add_value(row, "f2", static_cast(row)), E_OK); + ASSERT_EQ(tablet.add_value(row, "f3", static_cast(row)), E_OK); + ASSERT_EQ(tablet.add_value(row, "f4", static_cast(row * 1.1)), + E_OK); + ASSERT_EQ(tablet.add_value(row, "f5", static_cast(row * 1.1)), + E_OK); + ASSERT_EQ( + tablet.add_value(row, "f6", ("text" + to_string(row)).c_str()), + E_OK); + ASSERT_EQ( + tablet.add_value(row, "f7", ("string" + to_string(row)).c_str()), + E_OK); + ASSERT_EQ( + tablet.add_value(row, "f8", ("blob" + to_string(row)).c_str()), + E_OK); + ASSERT_EQ(tablet.add_value(row, "f9", static_cast(row)), E_OK); + ASSERT_EQ(tablet.add_value(row, "f10", static_cast(row)), + E_OK); + } + ASSERT_EQ(table_writer.write_table(tablet), E_OK); + ASSERT_EQ(table_writer.flush(), E_OK); + ASSERT_EQ(table_writer.close(), E_OK); + ASSERT_EQ(write_file.close(), E_OK); + + vector recovered_column_names = { + "__level1", "__level2", "__level3", "f1", "f2", "f3", "f4", + "f5", "f6", "f7", "f8", "f9", "f10"}; + for (int round = 0; round < 2; ++round) { + CorruptCurrentFileTail(10); + RestorableTsFileIOWriter rw; + ASSERT_EQ(rw.open(file_name_, true), E_OK); + ASSERT_TRUE(rw.can_write()); + + TsFileTableWriter table_writer2(&rw); + Tablet tablet2(recovered_column_names, data_types, max_rows); + tablet2.set_table_name(table_name); + for (int row = 0; row < static_cast(max_rows); row++) { + ASSERT_EQ( + tablet2.add_timestamp(row, static_cast(row + 10)), + E_OK); + ASSERT_EQ(tablet2.add_value(row, "__level1", "device1"), E_OK); + ASSERT_EQ(tablet2.add_value(row, "__level2", "device2"), E_OK); + ASSERT_EQ(tablet2.add_value(row, "__level3", "device3"), E_OK); + ASSERT_EQ(tablet2.add_value(row, "f1", row % 2 == 0), E_OK); + ASSERT_EQ(tablet2.add_value(row, "f2", static_cast(row)), + E_OK); + ASSERT_EQ(tablet2.add_value(row, "f3", static_cast(row)), + E_OK); + ASSERT_EQ( + tablet2.add_value(row, "f4", static_cast(row * 1.1)), + E_OK); + ASSERT_EQ( + tablet2.add_value(row, "f5", static_cast(row * 1.1)), + E_OK); + ASSERT_EQ( + tablet2.add_value(row, "f6", ("text" + to_string(row)).c_str()), + E_OK); + ASSERT_EQ(tablet2.add_value(row, "f7", + ("string" + to_string(row)).c_str()), + E_OK); + ASSERT_EQ( + tablet2.add_value(row, "f8", ("blob" + to_string(row)).c_str()), + E_OK); + ASSERT_EQ(tablet2.add_value(row, "f9", static_cast(row)), + E_OK); + ASSERT_EQ(tablet2.add_value(row, "f10", static_cast(row)), + E_OK); + } + if (round == 0) { + ASSERT_EQ(table_writer2.write_table(tablet2), E_OK); + ASSERT_EQ(table_writer2.flush(), E_OK); + } else { + ASSERT_EQ(table_writer2.write_table(tablet2), E_OUT_OF_ORDER); + } + ASSERT_EQ(table_writer2.close(), E_OK); + } } \ No newline at end of file