Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
136 changes: 103 additions & 33 deletions cpp/src/file/tsfile_io_reader.cc
Original file line number Diff line number Diff line change
Expand Up @@ -61,11 +61,27 @@ void TsFileIOReader::reset() {
delete read_file_;
}
read_file_ = nullptr;
// Drop everything that lives inside tsfile_meta_page_arena_ before
// the arena is released: the index graph and the bloom filter are
// both allocated there, and touching them afterwards is a
// use-after-free (traps under MSVC debug heap and ASan). Destroying
// the bloom filter first releases its separately allocated bit set;
// tsfile_meta_'s destructor would otherwise dereference the freed
// BloomFilter object.
if (tsfile_meta_.bloom_filter_ != nullptr) {
tsfile_meta_.bloom_filter_->destroy();
tsfile_meta_.bloom_filter_ = nullptr;
}
tsfile_meta_.table_metadata_index_node_map_.clear();
tsfile_meta_page_arena_.destroy();
{
std::lock_guard<std::mutex> lk(device_node_cache_mu_);
device_node_cache_.clear();
}
{
std::lock_guard<std::mutex> lk(index_node_cache_mu_);
index_node_cache_.clear();
}
tsfile_meta_ready_ = false;
}
}
Expand Down Expand Up @@ -1043,6 +1059,78 @@ int TsFileIOReader::search_from_leaf_node(
return ret;
}

int TsFileIOReader::get_cached_index_node(int64_t start_offset,
int64_t end_offset, bool is_device,
CachedIndexNode& out) {
{
std::lock_guard<std::mutex> lk(index_node_cache_mu_);
if (index_node_cache_.tryGetCopy(start_offset, out)) {
return E_OK;
}
}
Comment thread
ColinLeeo marked this conversation as resolved.
Resolved

if (start_offset >= end_offset) {
return E_TSFILE_CORRUPTED;
}
const int64_t read_size_i64 = end_offset - start_offset;
if (read_size_i64 > INT32_MAX) {
return E_OVERFLOW;
}
const int32_t read_size = static_cast<int32_t>(read_size_i64);
auto arena = std::make_shared<common::PageArena>();
Comment thread
ColinLeeo marked this conversation as resolved.
Resolved
arena->init(512, common::MOD_TSFILE_READER);
void* node_buf = arena->alloc(sizeof(MetaIndexNode));
if (IS_NULL(node_buf)) {
return E_OOM;
}
// The read buffer is only needed while deserializing; keeping it in the
// cached arena would pin read_size bytes per cached node until eviction.
std::unique_ptr<char[]> data_buf(new (std::nothrow) char[read_size]);
if (data_buf == nullptr) {
return E_OOM;
}
auto* node_ptr = new (node_buf) MetaIndexNode(arena.get());
std::shared_ptr<MetaIndexNode> node(node_ptr, MetaIndexNode::self_deleter);
int32_t ret_read_len = 0;
int ret =
read_file_->read(start_offset, data_buf.get(), read_size, ret_read_len);
if (RET_FAIL(ret)) {
return ret;
} else if (ret_read_len != read_size) {
return E_FILE_READ_ERR;
}
if (is_device) {
ret = node->device_deserialize_from(data_buf.get(), read_size);
} else {
ret = node->deserialize_from(data_buf.get(), read_size);
}
if (ret != E_OK) {
return ret;
}

CachedIndexNode entry;
entry.arena = arena;
entry.node = node;
{
std::lock_guard<std::mutex> lk(index_node_cache_mu_);
#ifdef ENABLE_TEST
++index_node_cache_misses_;
#endif
// Re-check under the lock: another query may have loaded the same
// node while we were reading it outside the lock. Replacing an
// existing entry would move-assign over it, and the old entry must
// be released through its own assignment (node before arena).
CachedIndexNode existing;
if (index_node_cache_.tryGetCopy(start_offset, existing)) {
out = std::move(existing);
return E_OK;
}
index_node_cache_.insert(start_offset, entry);
}
out = std::move(entry);
return E_OK;
}

int TsFileIOReader::search_from_internal_node(
std::shared_ptr<IComparable> target_name, bool is_device,
std::shared_ptr<MetaIndexNode> index_node,
Expand All @@ -1063,54 +1151,36 @@ int TsFileIOReader::search_from_internal_node(

while (IS_SUCC(ret)) {
// reader next level index node
const int read_size = end_offset - index_entry->get_offset();
const int64_t read_size = end_offset - index_entry->get_offset();
#if DEBUG_SE
std::cout << "search_from_internal_node, end_offset=" << end_offset
<< ", index_entry.offset_=" << index_entry.get_offset()
<< std::endl;
#endif
ASSERT(read_size > 0 && read_size < (1 << 30));
PageArena cur_level_index_node_pa;
void* buf = cur_level_index_node_pa.alloc(sizeof(MetaIndexNode));
char* data_buf = (char*)cur_level_index_node_pa.alloc(read_size);
if (IS_NULL(buf) || IS_NULL(data_buf)) {
return E_OOM;
}
MetaIndexNode* cur_level_index_node =
new (buf) MetaIndexNode(&cur_level_index_node_pa);
int32_t ret_read_len = 0;
if (RET_FAIL(read_file_->read(index_entry->get_offset(), data_buf,
read_size, ret_read_len))) {
return ret;
} else if (read_size != ret_read_len) {
return E_FILE_READ_ERR;
}
if (!is_device) {
ret = cur_level_index_node->deserialize_from(data_buf, read_size);
} else {
ret = cur_level_index_node->device_deserialize_from(data_buf,
read_size);
}
if (ret != E_OK) {
// Served from the index-node cache when it is the same node as a
// previous lookup: one leaf covers up to max_degree_of_index_node
// devices, so an uncached walk would re-read and re-deserialize the
// same node for every device in it.
// Keep the arena alive for as long as the node is used: the cache
// may evict this entry (and release its arena) concurrently.
CachedIndexNode cur_level;
if (RET_FAIL(get_cached_index_node(index_entry->get_offset(),
end_offset, is_device, cur_level))) {
return ret;
}
if (cur_level_index_node->node_type_ == LEAF_DEVICE) {
ret = cur_level_index_node->binary_search_children(
if (cur_level.node->node_type_ == LEAF_DEVICE) {
return cur_level.node->binary_search_children(
target_name, /*exact=*/true, ret_index_entry, ret_end_offset,
output_pa);
cur_level_index_node->destroy();
return ret; //// FIXME
} else if (cur_level_index_node->node_type_ == LEAF_MEASUREMENT) {
ret = cur_level_index_node->binary_search_children(
} else if (cur_level.node->node_type_ == LEAF_MEASUREMENT) {
return cur_level.node->binary_search_children(
target_name, /*exact=*/false, ret_index_entry, ret_end_offset,
output_pa);
cur_level_index_node->destroy();
return ret; //// FIXME
} else {
ret = cur_level_index_node->binary_search_children(
ret = cur_level.node->binary_search_children(
target_name, /*exact=*/false, index_entry, end_offset,
output_pa);
cur_level_index_node->destroy();
}
}
return ret;
Expand Down
99 changes: 96 additions & 3 deletions cpp/src/file/tsfile_io_reader.h
Original file line number Diff line number Diff line change
Expand Up @@ -52,7 +52,8 @@ class TsFileIOReader {
tsfile_meta_(&tsfile_meta_page_arena_),
tsfile_meta_ready_(false),
read_file_created_(false),
device_node_cache_(DEVICE_NODE_CACHE_CAPACITY, 0) {
device_node_cache_(DEVICE_NODE_CACHE_CAPACITY, 0),
index_node_cache_(INDEX_NODE_CACHE_CAPACITY, 0) {
tsfile_meta_page_arena_.init(512, common::MOD_TSFILE_READER);
}

Expand Down Expand Up @@ -232,15 +233,88 @@ class TsFileIOReader {
common::PageArena& pa);

struct CachedDeviceNode {
// Declared before top_node so top_node is destroyed first.
// MetaIndexNode and its children live in this arena.
// The node and its children live in this arena, so the node must be
// released before the arena. Destruction already does that (reverse
// declaration order); assignment is explicit because the implicit
// member-wise form would release the arena first.
std::shared_ptr<common::PageArena> arena;
std::shared_ptr<MetaIndexNode> top_node;
bool is_aligned = false;

CachedDeviceNode() = default;
CachedDeviceNode(const CachedDeviceNode&) = default;
CachedDeviceNode& operator=(const CachedDeviceNode& other) {
if (this != &other) {
top_node = other.top_node;
arena = other.arena;
is_aligned = other.is_aligned;
}
return *this;
}
CachedDeviceNode(CachedDeviceNode&& other) noexcept
: arena(std::move(other.arena)),
top_node(std::move(other.top_node)),
is_aligned(other.is_aligned) {}
CachedDeviceNode& operator=(CachedDeviceNode&& other) noexcept {
if (this != &other) {
top_node = std::move(other.top_node);
arena = std::move(other.arena);
is_aligned = other.is_aligned;
}
return *this;
}
};

static constexpr size_t DEVICE_NODE_CACHE_CAPACITY = 64;

// A deserialized index-tree node (device- or measurement-level) together
// with the arena backing it. Declared before `node` so the node is
// destroyed first.
struct CachedIndexNode {
std::shared_ptr<common::PageArena> arena;
std::shared_ptr<MetaIndexNode> node;

CachedIndexNode() = default;
CachedIndexNode(const CachedIndexNode&) = default;
// node must be released before the arena it was allocated in; the
// implicit member-wise assignment would release the arena first.
CachedIndexNode& operator=(const CachedIndexNode& other) {
if (this != &other) {
node = other.node;
arena = other.arena;
}
return *this;
}
CachedIndexNode(CachedIndexNode&& other) noexcept
: arena(std::move(other.arena)), node(std::move(other.node)) {}
CachedIndexNode& operator=(CachedIndexNode&& other) noexcept {
if (this != &other) {
node = std::move(other.node);
arena = std::move(other.arena);
}
return *this;
}
};

// One leaf of the device index holds up to max_degree_of_index_node
// devices, so an uncached tree walk re-deserializes the same node (and
// allocates every device-id string in it) once per device. A small LRU
// keyed by the node's file offset turns that into once per node.
static constexpr size_t INDEX_NODE_CACHE_CAPACITY = 16;

/**
* Returns the index node serialized at [start_offset, end_offset),
* reading and deserializing it on a cache miss.
*
* The out parameter carries the node *and* the arena it was allocated in:
* the node, its children and their names are placement-new'ed into that
* arena, and the cache may evict the entry (and release the arena) at any
* time. Callers must keep the returned CachedIndexNode alive for as long
* as they touch the node.
*/
int get_cached_index_node(int64_t start_offset, int64_t end_offset,
bool is_device, CachedIndexNode& out);

// Returns E_OK on hit (out is filled), or an error code on miss / load
// failure (E_DEVICE_NOT_EXIST when the device is absent, the propagated
// error otherwise). Copying into out keeps the caller safe from rehash /
Expand All @@ -261,6 +335,19 @@ class TsFileIOReader {
static size_t TEST_device_node_cache_capacity() {
return DEVICE_NODE_CACHE_CAPACITY;
}
size_t TEST_index_node_cache_size() const {
std::lock_guard<std::mutex> lk(index_node_cache_mu_);
return index_node_cache_.size();
}
static size_t TEST_index_node_cache_capacity() {
return INDEX_NODE_CACHE_CAPACITY;
}
// Nodes actually read from the file (i.e. cache misses). Lets a test
// prove that a second lookup round is served from the cache.
size_t TEST_index_node_cache_misses() const {
std::lock_guard<std::mutex> lk(index_node_cache_mu_);
return index_node_cache_misses_;
}
#endif

private:
Expand All @@ -285,6 +372,12 @@ class TsFileIOReader {
// shared ownership keeps an entry alive after a concurrent eviction.
common::Cache<std::string, CachedDeviceNode> device_node_cache_;
mutable std::mutex device_node_cache_mu_;
// Keyed by the node's start offset in the file.
common::Cache<int64_t, CachedIndexNode> index_node_cache_;
mutable std::mutex index_node_cache_mu_;
#ifdef ENABLE_TEST
size_t index_node_cache_misses_ = 0;
#endif
};

} // end namespace storage
Expand Down
Loading
Loading