Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
178 changes: 142 additions & 36 deletions cpp/src/file/tsfile_io_reader.cc
Original file line number Diff line number Diff line change
Expand Up @@ -936,14 +936,59 @@ int TsFileIOReader::get_timeseries_indexes(
}

bool is_aligned = is_aligned_device(top_node);
TimeseriesIndex* timeseries_index = nullptr;
TimeseriesIndex* time_timeseries_index = nullptr;
if (is_aligned) {
if (RET_FAIL(
get_time_column_metadata(top_node, timeseries_index, pa))) {
if (RET_FAIL(get_time_column_metadata(top_node, time_timeseries_index,
pa))) {
return ret;
}
}

// A wide projection is served better by one pass over the measurement index
// tree and one pread of the whole TimeseriesIndex range than by a binary
// search, a pread, a deserialize into a scratch arena and a clone per
// measurement. A narrow projection keeps the per-measurement path: walking
// the tree would deserialize every column's index to find a handful.
static constexpr size_t kBatchLoadMinMeasurements = 16;
if (measurement_names.size() >= kBatchLoadMinMeasurements) {
std::vector<std::pair<std::shared_ptr<IMetaIndexEntry>, int64_t>>
leaf_entries;
if (RET_FAIL(get_all_leaf(top_node, leaf_entries, pa))) {
return ret;
}
std::vector<ITimeseriesIndex*> loaded_indexes;
if (RET_FAIL(do_load_all_timeseries_index(leaf_entries, pa,
loaded_indexes))) {
return ret;
}
std::unordered_map<std::string, ITimeseriesIndex*> name_to_index;
name_to_index.reserve(loaded_indexes.size());
for (ITimeseriesIndex* loaded : loaded_indexes) {
if (is_aligned) {
void* aligned_buf = pa.alloc(sizeof(AlignedTimeseriesIndex));
if (IS_NULL(aligned_buf)) {
return E_OOM;
}
auto* aligned_ts_idx = new (aligned_buf) AlignedTimeseriesIndex;
aligned_ts_idx->value_ts_idx_ =
static_cast<TimeseriesIndex*>(loaded);
aligned_ts_idx->time_ts_idx_ = time_timeseries_index;
name_to_index[aligned_ts_idx->get_measurement_name()
.to_std_string()] = aligned_ts_idx;
} else {
name_to_index[loaded->get_measurement_name().to_std_string()] =
loaded;
}
}
Comment thread
ColinLeeo marked this conversation as resolved.
Resolved
int64_t idx = 0;
for (const auto& measurement_name : measurement_names) {
const auto it = name_to_index.find(measurement_name);
timeseries_indexs[idx++] =
it != name_to_index.end() ? it->second : nullptr;
}
return ret;
}

int64_t idx = 0;
for (const auto& measurement_name : measurement_names) {
timeseries_indexs[idx] = nullptr;
Expand Down Expand Up @@ -974,7 +1019,7 @@ int TsFileIOReader::get_timeseries_indexes(
AlignedTimeseriesIndex* aligned_timeseries_index =
dynamic_cast<AlignedTimeseriesIndex*>(timeseries_indexs[idx]);
if (aligned_timeseries_index) {
aligned_timeseries_index->time_ts_idx_ = timeseries_index;
aligned_timeseries_index->time_ts_idx_ = time_timeseries_index;
}
}

Expand Down Expand Up @@ -1233,46 +1278,107 @@ int TsFileIOReader::do_load_timeseries_index(
return ret;
}

namespace {
// True when consecutive leaf entries of one device's measurement index end
// exactly where the next one starts, i.e. they tile one file range.
bool leaf_entries_tile_range(
const std::vector<std::pair<std::shared_ptr<IMetaIndexEntry>, int64_t>>&
entries) {
for (size_t i = 1; i < entries.size(); i++) {
if (entries[i].first->get_offset() != entries[i - 1].second) {
return false;
}
}
return true;
}
} // namespace

int TsFileIOReader::do_load_timeseries_index_range(
int64_t start_offset, int64_t end_offset, common::PageArena& pa,
std::vector<ITimeseriesIndex*>& ts_indexs) {
int ret = E_OK;
const int64_t read_size_i64 = end_offset - start_offset;
if (read_size_i64 <= 0) {
return E_TSFILE_CORRUPTED;
}
if (read_size_i64 > INT32_MAX) {
return E_OVERFLOW;
}
const int32_t read_size = static_cast<int32_t>(read_size_i64);
// The raw bytes are needed only while the indexes are parsed, so they are
// heap-allocated and released below instead of coming from the caller's
// arena: a wide device's whole range would otherwise stay resident for as
// long as the arena lives.
char* ti_buf = static_cast<char*>(mem_alloc(read_size, MOD_TSFILE_READER));
if (IS_NULL(ti_buf)) {
return E_OOM;
}
int32_t ret_read_len = 0;
if (RET_FAIL(
read_file_->read(start_offset, ti_buf, read_size, ret_read_len))) {
mem_free(ti_buf);
return ret;
} else if (ret_read_len != read_size) {
mem_free(ti_buf);
return E_FILE_READ_ERR;
}
ByteStream bs;
bs.wrap_from(ti_buf, read_size);
while (bs.has_remaining()) {
const uint64_t relative_start = bs.read_pos();
void* buf = pa.alloc(sizeof(TimeseriesIndex));
if (IS_NULL(buf)) {
mem_free(ti_buf);
return E_OOM;
}
auto ts_idx = new (buf) TimeseriesIndex;
if (RET_FAIL(ts_idx->deserialize_from(bs, &pa))) {
mem_free(ti_buf);
return ret;
}
if (bs.read_pos() < relative_start ||
bs.read_pos() - relative_start > UINT32_MAX) {
mem_free(ti_buf);
return E_OVERFLOW;
}
ts_idx->set_metadata_range(
start_offset + relative_start,
static_cast<uint32_t>(bs.read_pos() - relative_start));
if (ts_idx->get_measurement_name().len_ == 0) continue;
ts_indexs.push_back(ts_idx);
}
mem_free(ti_buf);
return E_OK;
}

int TsFileIOReader::do_load_all_timeseries_index(
std::vector<std::pair<std::shared_ptr<IMetaIndexEntry>, int64_t>>&
index_node_entry_list,
common::PageArena& in_timeseries_index_pa,
std::vector<ITimeseriesIndex*>& ts_indexs) {
int ret = E_OK;
if (index_node_entry_list.empty()) {
return ret;
}
// The leaf entries usually tile one contiguous file range, so the whole
// device's TimeseriesIndex blocks can be read back-to-back with a single
// pread instead of one syscall per measurement (4500 of them on a wide
// table). Fall back to per-entry reads if the entries do not tile, or if
// the combined range does not fit one read buffer.
if (leaf_entries_tile_range(index_node_entry_list) &&
index_node_entry_list.back().second -
index_node_entry_list.front().first->get_offset() <=
INT32_MAX) {
return do_load_timeseries_index_range(
index_node_entry_list.front().first->get_offset(),
index_node_entry_list.back().second, in_timeseries_index_pa,
ts_indexs);
}
for (const auto& index_node_entry : index_node_entry_list) {
int64_t start_offset = index_node_entry.first->get_offset(),
end_offset = index_node_entry.second;
int32_t read_size = (int32_t)(end_offset - start_offset);
int32_t ret_read_len = 0;
char* ti_buf = in_timeseries_index_pa.alloc(read_size);
if (IS_NULL(ti_buf)) {
return E_OOM;
}
if (RET_FAIL(read_file_->read(start_offset, ti_buf, read_size,
ret_read_len))) {
if (RET_FAIL(do_load_timeseries_index_range(
index_node_entry.first->get_offset(), index_node_entry.second,
in_timeseries_index_pa, ts_indexs))) {
return ret;
} else if (ret_read_len != read_size) {
return E_FILE_READ_ERR;
}
ByteStream bs;
bs.wrap_from(ti_buf, read_size);
while (bs.has_remaining()) {
const uint64_t relative_start = bs.read_pos();
void* buf = in_timeseries_index_pa.alloc(sizeof(TimeseriesIndex));
auto ts_idx = new (buf) TimeseriesIndex;
if (RET_FAIL(
ts_idx->deserialize_from(bs, &in_timeseries_index_pa))) {
return ret;
}
if (bs.read_pos() < relative_start ||
bs.read_pos() - relative_start > UINT32_MAX) {
return E_OVERFLOW;
}
ts_idx->set_metadata_range(
start_offset + relative_start,
static_cast<uint32_t>(bs.read_pos() - relative_start));
if (ts_idx->get_measurement_name().len_ == 0) continue;
ts_indexs.push_back(ts_idx);
}
}
return ret;
Expand Down
8 changes: 8 additions & 0 deletions cpp/src/file/tsfile_io_reader.h
Original file line number Diff line number Diff line change
Expand Up @@ -191,6 +191,14 @@ class TsFileIOReader {
common::PageArena& in_timeseries_index_pa,
std::vector<ITimeseriesIndex*>& ts_indexs);

// Reads [start_offset, end_offset) once and deserializes the
// TimeseriesIndex stored there into @pa, appending them to @ts_indexs.
// Entries with an empty measurement name (the aligned time column) are
// skipped, matching do_load_all_timeseries_index().
int do_load_timeseries_index_range(
int64_t start_offset, int64_t end_offset, common::PageArena& pa,
std::vector<ITimeseriesIndex*>& ts_indexs);

int load_timeseries_index_for_ssi(std::shared_ptr<IDeviceID> device_id,
const std::string& measurement_name,
TsFileSeriesScanIterator*& ssi);
Expand Down
Loading
Loading