Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions cpp/src/common/schema.h
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@
#include <writer/chunk_writer.h>

#include <algorithm>
#include <climits>
#include <map> // use unordered_map instead
#include <memory>
#include <string>
Expand Down Expand Up @@ -165,6 +166,7 @@ struct MeasurementSchemaGroup {
MeasurementSchemaMap measurement_schema_map_;
bool is_aligned_ = false;
TimeChunkWriter* time_chunk_writer_ = nullptr;
int64_t last_time_ = INT64_MIN;

~MeasurementSchemaGroup() {
if (time_chunk_writer_ != nullptr) {
Expand Down
34 changes: 30 additions & 4 deletions cpp/src/file/restorable_tsfile_io_writer.cc
Original file line number Diff line number Diff line change
Expand Up @@ -328,16 +328,20 @@ static int recover_chunk_statistic(
uint32_t value_buf_size = 0;
std::vector<int64_t> time_decode_buf;
const std::vector<int64_t>* times = nullptr;
std::vector<uint8_t> aligned_value_notnull_bitmap;
uint32_t aligned_num_values = 0;
const bool is_aligned_value_chunk =
(time_batch != nullptr && !time_batch->empty());

if (time_batch != nullptr && !time_batch->empty()) {
if (is_aligned_value_chunk) {
// Aligned value page: uncompressed layout = uint32(num_values) + bitmap
// + value_buf
if (uncompressed_size < 4) {
compressor->after_uncompress(uncompressed_buf);
CompressorFactory::free(compressor);
return E_OK;
}
uint32_t num_values =
aligned_num_values =
(static_cast<uint32_t>(
static_cast<unsigned char>(uncompressed_buf[0]))
<< 24) |
Expand All @@ -349,12 +353,17 @@ static int recover_chunk_statistic(
<< 8) |
(static_cast<uint32_t>(
static_cast<unsigned char>(uncompressed_buf[3])));
uint32_t bitmap_size = (num_values + 7) / 8;
uint32_t bitmap_size = (aligned_num_values + 7) / 8;
if (uncompressed_size < 4 + bitmap_size) {
compressor->after_uncompress(uncompressed_buf);
CompressorFactory::free(compressor);
return E_OK;
}
aligned_value_notnull_bitmap.resize(bitmap_size);
if (bitmap_size > 0) {
std::memcpy(aligned_value_notnull_bitmap.data(),
uncompressed_buf + 4, bitmap_size);
}
value_buf = uncompressed_buf + 4 + bitmap_size;
value_buf_size = uncompressed_size - 4 - bitmap_size;
times = time_batch;
Expand Down Expand Up @@ -410,8 +419,25 @@ static int recover_chunk_statistic(
value_decoder->reset();
size_t idx = 0;
const size_t num_times = times->size();
while (idx < num_times && value_decoder->has_remaining(value_in)) {
while (idx < num_times) {
int64_t t = (*times)[idx];
bool has_value = true;
if (is_aligned_value_chunk) {
has_value = false;
const uint32_t byte_idx = static_cast<uint32_t>(idx / 8);
const uint32_t bit_shift = static_cast<uint32_t>(idx % 8);
if (byte_idx < aligned_value_notnull_bitmap.size()) {
has_value = ((aligned_value_notnull_bitmap[byte_idx] & 0xFF) &
(0x80 >> bit_shift)) != 0;
}
}
if (!has_value) {
idx++;
continue;
}
if (!value_decoder->has_remaining(value_in)) {
break;
}
switch (chdr.data_type_) {
case common::BOOLEAN: {
bool v;
Expand Down
109 changes: 93 additions & 16 deletions cpp/src/writer/tsfile_writer.cc
Original file line number Diff line number Diff line change
Expand Up @@ -81,7 +81,8 @@ TsFileWriter::TsFileWriter()
record_count_for_next_mem_check_(
g_config_value_.record_count_for_next_mem_check_),
write_file_created_(false),
io_writer_owned_(true) {}
io_writer_owned_(true),
enforce_recovered_last_time_order_(false) {}

TsFileWriter::~TsFileWriter() { destroy(); }

Expand Down Expand Up @@ -127,6 +128,7 @@ int TsFileWriter::init(WriteFile* write_file) {
write_file_ = write_file;
write_file_created_ = false;
io_writer_owned_ = true;
enforce_recovered_last_time_order_ = false;
io_writer_ = new TsFileIOWriter();
io_writer_->init(write_file_);
return E_OK;
Expand All @@ -146,6 +148,7 @@ int TsFileWriter::init(RestorableTsFileIOWriter* rw) {
write_file_ = rw->get_write_file();
write_file_created_ = false;
io_writer_owned_ = false;
enforce_recovered_last_time_order_ = true;
io_writer_ = rw;

const std::vector<ChunkGroupMeta*>& recovered =
Expand Down Expand Up @@ -182,6 +185,10 @@ int TsFileWriter::init(RestorableTsFileIOWriter* rw) {
if (cm == nullptr) {
continue;
}
if (cm->statistic_ != nullptr && cm->statistic_->count_ > 0) {
group->last_time_ =
std::max(group->last_time_, cm->statistic_->end_time_);
}
std::string mname = cm->measurement_name_.to_std_string();
if (mname.empty()) {
continue;
Expand Down Expand Up @@ -692,13 +699,22 @@ int TsFileWriter::check_memory_size_and_may_flush_chunks() {

int TsFileWriter::write_record(const TsRecord& record) {
int ret = E_OK;
auto device_id = std::make_shared<StringArrayDeviceID>(record.device_id_);
auto schema_it = schemas_.find(device_id);
if (schema_it == schemas_.end() || schema_it->second == nullptr) {
return E_DEVICE_NOT_EXIST;
}
MeasurementSchemaGroup* device_schema = schema_it->second;
if (enforce_recovered_last_time_order_ &&
record.timestamp_ <= device_schema->last_time_) {
return E_OUT_OF_ORDER;
}
// std::vector<ChunkWriter*> chunk_writers;
SimpleVector<ChunkWriter*> chunk_writers;
SimpleVector<common::TSDataType> data_types;
MeasurementNamesFromRecord mnames_getter(record);
if (RET_FAIL(do_check_schema(
std::make_shared<StringArrayDeviceID>(record.device_id_),
mnames_getter, chunk_writers, data_types))) {
if (RET_FAIL(do_check_schema(device_id, mnames_getter, chunk_writers,
data_types))) {
return ret;
}

Expand All @@ -713,21 +729,32 @@ int TsFileWriter::write_record(const TsRecord& record) {
record.points_[c]);
}

device_schema->last_time_ =
std::max(device_schema->last_time_, record.timestamp_);
record_count_since_last_flush_++;
ret = check_memory_size_and_may_flush_chunks();
return ret;
}

int TsFileWriter::write_record_aligned(const TsRecord& record) {
int ret = E_OK;
auto device_id = std::make_shared<StringArrayDeviceID>(record.device_id_);
auto schema_it = schemas_.find(device_id);
if (schema_it == schemas_.end() || schema_it->second == nullptr) {
return E_DEVICE_NOT_EXIST;
}
MeasurementSchemaGroup* device_schema = schema_it->second;
if (enforce_recovered_last_time_order_ &&
record.timestamp_ <= device_schema->last_time_) {
return E_OUT_OF_ORDER;
}
SimpleVector<ValueChunkWriter*> value_chunk_writers;
SimpleVector<common::TSDataType> data_types;
TimeChunkWriter* time_chunk_writer;
MeasurementNamesFromRecord mnames_getter(record);
if (RET_FAIL(do_check_schema_aligned(
std::make_shared<StringArrayDeviceID>(record.device_id_),
mnames_getter, time_chunk_writer, value_chunk_writers,
data_types))) {
if (RET_FAIL(do_check_schema_aligned(device_id, mnames_getter,
time_chunk_writer, value_chunk_writers,
data_types))) {
return ret;
}
if (value_chunk_writers.size() != record.points_.size()) {
Expand Down Expand Up @@ -755,6 +782,8 @@ int TsFileWriter::write_record_aligned(const TsRecord& record) {
value_pages_before))) {
return ret;
}
device_schema->last_time_ =
std::max(device_schema->last_time_, record.timestamp_);
return ret;
}

Expand Down Expand Up @@ -853,17 +882,27 @@ int TsFileWriter::maybe_seal_aligned_pages_together(

int TsFileWriter::write_tablet_aligned(const Tablet& tablet) {
int ret = E_OK;
auto device_id =
std::make_shared<StringArrayDeviceID>(tablet.insert_target_name_);
auto schema_it = schemas_.find(device_id);
if (schema_it == schemas_.end() || schema_it->second == nullptr) {
return E_DEVICE_NOT_EXIST;
}
MeasurementSchemaGroup* device_schema = schema_it->second;
const uint32_t total_rows = tablet.get_cur_row_size();
if (enforce_recovered_last_time_order_ && total_rows > 0 &&
tablet.timestamps_[0] <= device_schema->last_time_) {
return E_OUT_OF_ORDER;
}
SimpleVector<ValueChunkWriter*> value_chunk_writers;
TimeChunkWriter* time_chunk_writer = nullptr;
SimpleVector<common::TSDataType> data_types;
MeasurementNamesFromTablet mnames_getter(tablet);
if (RET_FAIL(do_check_schema_aligned(
std::make_shared<StringArrayDeviceID>(tablet.insert_target_name_),
mnames_getter, time_chunk_writer, value_chunk_writers,
data_types))) {
if (RET_FAIL(do_check_schema_aligned(device_id, mnames_getter,
time_chunk_writer, value_chunk_writers,
data_types))) {
return ret;
}
const uint32_t total_rows = tablet.get_cur_row_size();
const bool strict_page_size = common::g_config_value_.strict_page_size_;

// Decide whether we have string/blob/text columns.
Expand Down Expand Up @@ -919,6 +958,10 @@ int TsFileWriter::write_tablet_aligned(const Tablet& tablet) {
return ret;
}
}
if (total_rows > 0) {
device_schema->last_time_ = std::max(
device_schema->last_time_, tablet.timestamps_[total_rows - 1]);
}
return ret;
}

Expand Down Expand Up @@ -1010,6 +1053,10 @@ int TsFileWriter::write_tablet_aligned(const Tablet& tablet) {
seg_len = points_per_page;
}
}
if (total_rows > 0) {
device_schema->last_time_ = std::max(
device_schema->last_time_, tablet.timestamps_[total_rows - 1]);
}
return ret;
}

Expand Down Expand Up @@ -1077,17 +1124,32 @@ int TsFileWriter::write_tablet_aligned(const Tablet& tablet) {
}
}
}
if (total_rows > 0) {
device_schema->last_time_ = std::max(
device_schema->last_time_, tablet.timestamps_[total_rows - 1]);
}
return ret;
}

int TsFileWriter::write_tablet(const Tablet& tablet) {
int ret = E_OK;
auto device_id =
std::make_shared<StringArrayDeviceID>(tablet.insert_target_name_);
auto schema_it = schemas_.find(device_id);
if (schema_it == schemas_.end() || schema_it->second == nullptr) {
return E_DEVICE_NOT_EXIST;
}
MeasurementSchemaGroup* device_schema = schema_it->second;
const uint32_t total_rows = tablet.get_cur_row_size();
if (enforce_recovered_last_time_order_ && total_rows > 0 &&
tablet.timestamps_[0] <= device_schema->last_time_) {
return E_OUT_OF_ORDER;
}
SimpleVector<ChunkWriter*> chunk_writers;
SimpleVector<common::TSDataType> data_types;
MeasurementNamesFromTablet mnames_getter(tablet);
if (RET_FAIL(do_check_schema(
std::make_shared<StringArrayDeviceID>(tablet.insert_target_name_),
mnames_getter, chunk_writers, data_types))) {
if (RET_FAIL(do_check_schema(device_id, mnames_getter, chunk_writers,
data_types))) {
return ret;
}
ASSERT(chunk_writers.size() == tablet.get_column_count());
Expand All @@ -1101,6 +1163,10 @@ int TsFileWriter::write_tablet(const Tablet& tablet) {
}
}

if (total_rows > 0) {
device_schema->last_time_ = std::max(
device_schema->last_time_, tablet.timestamps_[total_rows - 1]);
}
record_count_since_last_flush_ += tablet.max_row_num_;
ret = check_memory_size_and_may_flush_chunks();
return ret;
Expand Down Expand Up @@ -1156,6 +1222,9 @@ int TsFileWriter::write_table(Tablet& tablet) {
value_chunk_writers))) {
return ret;
}
auto schema_it = schemas_.find(device_id);
MeasurementSchemaGroup* device_schema =
(schema_it == schemas_.end()) ? nullptr : schema_it->second;

std::vector<uint32_t> field_columns;
field_columns.reserve(tablet.get_column_count());
Expand All @@ -1174,6 +1243,10 @@ int TsFileWriter::write_table(Tablet& tablet) {
1, common::g_config_value_.page_writer_max_point_num_);
const uint32_t si = static_cast<uint32_t>(start_idx);
const uint32_t ei = static_cast<uint32_t>(end_idx);
if (enforce_recovered_last_time_order_ && device_schema != nullptr &&
si < ei && tablet.timestamps_[si] <= device_schema->last_time_) {
return E_OUT_OF_ORDER;
}

// If the current unsealed page is already at or past capacity (from
// a previous write_table call), seal it before starting new segments.
Expand Down Expand Up @@ -1301,6 +1374,10 @@ int TsFileWriter::write_table(Tablet& tablet) {
}
}
}
if (device_schema != nullptr && si < ei) {
device_schema->last_time_ =
std::max(device_schema->last_time_, tablet.timestamps_[ei - 1]);
}
start_idx = end_idx;
}
record_count_since_last_flush_ += tablet.cur_row_size_;
Expand Down
1 change: 1 addition & 0 deletions cpp/src/writer/tsfile_writer.h
Original file line number Diff line number Diff line change
Expand Up @@ -195,6 +195,7 @@ class TsFileWriter {
int64_t record_count_for_next_mem_check_;
bool write_file_created_;
bool io_writer_owned_; // false when init(RestorableTsFileIOWriter*)
bool enforce_recovered_last_time_order_;

int write_typed_column(ValueChunkWriter* value_chunk_writer,
int64_t* timestamps, bool* col_values,
Expand Down
Loading
Loading