diff --git a/.gitignore b/.gitignore index fcd5f2b6b..6b5d1d9b2 100644 --- a/.gitignore +++ b/.gitignore @@ -32,6 +32,8 @@ python/tsfile/*dll* python/tsfile/*dylib* python/tsfile/*.h python/tsfile/*.cpp +python/tsfile/**/*.cpp +python/tsfile/**/*so* python/data python/venv/* python/tests/__pycache__/* diff --git a/cpp/src/CMakeLists.txt b/cpp/src/CMakeLists.txt index 41d2cde12..1c3dd90c1 100644 --- a/cpp/src/CMakeLists.txt +++ b/cpp/src/CMakeLists.txt @@ -118,6 +118,7 @@ endif() add_subdirectory(common) add_subdirectory(compress) add_subdirectory(cwrapper) +add_subdirectory(dataset) add_subdirectory(encoding) add_subdirectory(file) add_subdirectory(reader) @@ -173,16 +174,16 @@ if (${COV_ENABLED}) set(COV_LINK_LIB -lgcov) endif() if (ENABLE_ANTLR4) - target_link_libraries(tsfile common_obj compress_obj cwrapper_obj file_obj read_obj write_obj parser_obj ${COV_LINK_LIB}) + target_link_libraries(tsfile common_obj compress_obj cwrapper_obj dataset_obj file_obj read_obj write_obj parser_obj ${COV_LINK_LIB}) else() - target_link_libraries(tsfile common_obj compress_obj cwrapper_obj file_obj read_obj write_obj ${COV_LINK_LIB}) + target_link_libraries(tsfile common_obj compress_obj cwrapper_obj dataset_obj file_obj read_obj write_obj ${COV_LINK_LIB}) endif() else() message("Disable code cov...") if (ENABLE_ANTLR4) - target_link_libraries(tsfile common_obj compress_obj cwrapper_obj file_obj read_obj write_obj parser_obj) + target_link_libraries(tsfile common_obj compress_obj cwrapper_obj dataset_obj file_obj read_obj write_obj parser_obj) else() - target_link_libraries(tsfile common_obj compress_obj cwrapper_obj file_obj read_obj write_obj) + target_link_libraries(tsfile common_obj compress_obj cwrapper_obj dataset_obj file_obj read_obj write_obj) endif() endif() diff --git a/cpp/src/common/tsfile_common.h b/cpp/src/common/tsfile_common.h index d763acbbd..be046a9b0 100644 --- a/cpp/src/common/tsfile_common.h +++ b/cpp/src/common/tsfile_common.h @@ -351,6 +351,8 @@ class TimeseriesIndex : public ITimeseriesIndex { TimeseriesIndex() : timeseries_meta_type_((char)255), chunk_meta_list_data_size_(0), + metadata_offset_(-1), + metadata_length_(0), measurement_name_(), data_type_(common::INVALID_DATATYPE), statistic_(nullptr), @@ -369,6 +371,8 @@ class TimeseriesIndex : public ITimeseriesIndex { { timeseries_meta_type_ = 0; chunk_meta_list_data_size_ = 0; + metadata_offset_ = -1; + metadata_length_ = 0; measurement_name_.reset(); data_type_ = common::VECTOR; chunk_meta_list_serialized_buf_.reset(); @@ -401,6 +405,16 @@ class TimeseriesIndex : public ITimeseriesIndex { FORCE_INLINE virtual common::TSDataType get_data_type() const { return data_type_; } + FORCE_INLINE void set_metadata_range(int64_t offset, uint32_t length) { + metadata_offset_ = offset; + metadata_length_ = length; + } + FORCE_INLINE int64_t get_metadata_offset() const { + return metadata_offset_; + } + FORCE_INLINE uint32_t get_metadata_length() const { + return metadata_length_; + } int init_statistic(common::TSDataType data_type) { if (statistic_ != nullptr && !statistic_from_pa_) { // clear old statistic @@ -488,6 +502,8 @@ class TimeseriesIndex : public ITimeseriesIndex { int ret = common::E_OK; timeseries_meta_type_ = that.timeseries_meta_type_; chunk_meta_list_data_size_ = that.chunk_meta_list_data_size_; + metadata_offset_ = that.metadata_offset_; + metadata_length_ = that.metadata_length_; data_type_ = that.data_type_; statistic_ = StatisticFactory::alloc_statistic_with_pa(data_type_, pa); @@ -560,6 +576,12 @@ class TimeseriesIndex : public ITimeseriesIndex { // Sum of chunk meta serialized size in List of this timeseries. uint32_t chunk_meta_list_data_size_; + // Exact byte range of this TimeseriesMetadata in the source TsFile. + // It is assigned by TsFileIOReader after deserialization and is not part + // of the on-wire TimeseriesMetadata encoding. + int64_t metadata_offset_; + uint32_t metadata_length_; + // std::string measurement_name_; common::String measurement_name_; common::TSDataType data_type_; diff --git a/cpp/src/cwrapper/tsfile_cwrapper.cc b/cpp/src/cwrapper/tsfile_cwrapper.cc index 3e27f60d0..825345f03 100644 --- a/cpp/src/cwrapper/tsfile_cwrapper.cc +++ b/cpp/src/cwrapper/tsfile_cwrapper.cc @@ -419,6 +419,67 @@ ERRNO tsfile_writer_write(TsFileWriter writer, Tablet tablet) { // Query +PreparedSeriesHandle tsfile_reader_prepare_series( + TsFileReader reader, const TsFilePreparedLocator* locator, + ERRNO* err_code) { + if (err_code == nullptr) { + return nullptr; + } + *err_code = common::E_INVALID_ARG; + if (reader == nullptr || locator == nullptr) { + return nullptr; + } + storage::FileGeneration generation; + generation.mapped_index_identity = locator->mapped_index_identity; + generation.file_id = locator->file_id; + generation.file_size = locator->file_size; + generation.file_fingerprint = locator->file_fingerprint; + storage::PreparedLocator native_locator; + native_locator.locator_id = locator->locator_id; + native_locator.layout = locator->layout; + native_locator.flags = locator->flags; + native_locator.value_metadata_offset = locator->value_metadata_offset; + native_locator.value_metadata_length = locator->value_metadata_length; + native_locator.time_metadata_offset = locator->time_metadata_offset; + native_locator.time_metadata_length = locator->time_metadata_length; + native_locator.chunk_count_hint = locator->chunk_count_hint; + std::shared_ptr prepared; + *err_code = static_cast(reader)->prepare_series( + generation, native_locator, prepared); + if (*err_code != common::E_OK) { + return nullptr; + } + auto* handle = new (std::nothrow) + std::shared_ptr(std::move(prepared)); + if (handle == nullptr) { + *err_code = common::E_OOM; + } + return handle; +} + +void tsfile_prepared_series_free(PreparedSeriesHandle prepared) { + delete static_cast*>(prepared); +} + +ResultSet tsfile_reader_query_prepared(TsFileReader reader, + PreparedSeriesHandle prepared, + Timestamp start_time, Timestamp end_time, + int offset, int limit, ERRNO* err_code) { + if (err_code == nullptr) { + return nullptr; + } + *err_code = common::E_INVALID_ARG; + if (reader == nullptr || prepared == nullptr) { + return nullptr; + } + auto* handle = + static_cast*>(prepared); + storage::ResultSet* result = nullptr; + *err_code = static_cast(reader)->query_prepared( + *handle, start_time, end_time, offset, limit, result); + return result; +} + ResultSet tsfile_query_table(TsFileReader reader, const char* table_name, char** columns, uint32_t column_num, Timestamp start_time, Timestamp end_time, @@ -1305,8 +1366,32 @@ ERRNO populate_c_metadata_map_from_cpp( aligned_idx->value_ts_idx_ != nullptr) { m.data_type = static_cast( aligned_idx->value_ts_idx_->get_data_type()); + const storage::TimeseriesIndex* value_idx = + aligned_idx->value_ts_idx_; + const storage::TimeseriesIndex* time_idx = + aligned_idx->time_ts_idx_; + if (value_idx->get_metadata_offset() >= 0) { + m.value_metadata_offset = + static_cast(value_idx->get_metadata_offset()); + m.value_metadata_length = value_idx->get_metadata_length(); + } + if (time_idx != nullptr && + time_idx->get_metadata_offset() >= 0) { + m.time_metadata_offset = + static_cast(time_idx->get_metadata_offset()); + m.time_metadata_length = time_idx->get_metadata_length(); + } + m.layout = 1; } else { m.data_type = static_cast(idx->get_data_type()); + const storage::TimeseriesIndex* value_idx = + dynamic_cast(idx.get()); + if (value_idx != nullptr && + value_idx->get_metadata_offset() >= 0) { + m.value_metadata_offset = + static_cast(value_idx->get_metadata_offset()); + m.value_metadata_length = value_idx->get_metadata_length(); + } } storage::Statistic* st = idx->get_statistic(); int32_t chunk_cnt = 0; @@ -1316,6 +1401,21 @@ ERRNO populate_c_metadata_map_from_cpp( chunk_cnt = static_cast(cl->size()); } m.chunk_meta_count = chunk_cnt; + if (chunk_cnt >= 0 && m.value_metadata_length > 0) { + m.locator_flags |= 1; + } + if (aligned_idx != nullptr) { + auto* time_chunks = idx->get_time_chunk_meta_list(); + if (time_chunks != nullptr) { + m.time_chunk_meta_count = + static_cast(time_chunks->size()); + } + if (m.time_metadata_length == 0 || + m.time_chunk_meta_count != + static_cast(chunk_cnt)) { + m.locator_flags &= ~static_cast(1); + } + } const int st_rc = fill_timeseries_statistic(st, &m.statistic); if (st_rc != common::E_OK) { for (uint32_t u = 0; u < slot; u++) { diff --git a/cpp/src/cwrapper/tsfile_cwrapper.h b/cpp/src/cwrapper/tsfile_cwrapper.h index 16d392bae..2b97c811d 100644 --- a/cpp/src/cwrapper/tsfile_cwrapper.h +++ b/cpp/src/cwrapper/tsfile_cwrapper.h @@ -194,6 +194,13 @@ typedef struct TimeseriesMetadata { int32_t chunk_meta_count; TimeseriesStatistic statistic; TimeseriesStatistic timeline_statistic; + uint64_t value_metadata_offset; + uint32_t value_metadata_length; + uint64_t time_metadata_offset; + uint32_t time_metadata_length; + uint32_t time_chunk_meta_count; + uint16_t layout; + uint16_t locator_flags; } TimeseriesMetadata; /** @@ -269,6 +276,22 @@ typedef void* TsRecord; typedef void* ResultSet; typedef void* TagFilterHandle; +typedef void* PreparedSeriesHandle; + +typedef struct TsFilePreparedLocator { + uint64_t mapped_index_identity; + uint32_t file_id; + uint64_t file_size; + uint64_t file_fingerprint; + uint32_t locator_id; + uint16_t layout; + uint16_t flags; + uint64_t value_metadata_offset; + uint32_t value_metadata_length; + uint64_t time_metadata_offset; + uint32_t time_metadata_length; + uint32_t chunk_count_hint; +} TsFilePreparedLocator; typedef struct arrow_schema { // Array type description @@ -656,6 +679,20 @@ ERRNO tsfile_writer_write(TsFileWriter writer, Tablet tablet); /*-------------------TsFile reader query data------------------ */ +/** Deserialize one exact Dataset Index locator into a reusable series. */ +PreparedSeriesHandle tsfile_reader_prepare_series( + TsFileReader reader, const TsFilePreparedLocator* locator, ERRNO* err_code); + +/** Release a prepared handle. Existing result sets remain independently owned. + */ +void tsfile_prepared_series_free(PreparedSeriesHandle prepared); + +/** Query a prepared series without traversing the TsFile footer index. */ +ResultSet tsfile_reader_query_prepared(TsFileReader reader, + PreparedSeriesHandle prepared, + Timestamp start_time, Timestamp end_time, + int offset, int limit, ERRNO* err_code); + /** * @brief Queries time series data from a specific table within time range. * diff --git a/cpp/src/dataset/CMakeLists.txt b/cpp/src/dataset/CMakeLists.txt new file mode 100644 index 000000000..7adbdcc8f --- /dev/null +++ b/cpp/src/dataset/CMakeLists.txt @@ -0,0 +1,25 @@ +#[[ +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + https://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, +software distributed under the License is distributed on an +"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +KIND, either express or implied. See the License for the +specific language governing permissions and limitations +under the License. +]] +message("Running in src/dataset directory") +set(CMAKE_POSITION_INDEPENDENT_CODE ON) +aux_source_directory(${CMAKE_CURRENT_SOURCE_DIR} dataset_SRC_LIST) +add_library(dataset_obj OBJECT ${dataset_SRC_LIST}) + +file(GLOB HEADERS "${CMAKE_CURRENT_SOURCE_DIR}/*.h") +copy_to_dir(${HEADERS} "dataset_obj") diff --git a/cpp/src/dataset/dataset_index.cc b/cpp/src/dataset/dataset_index.cc new file mode 100644 index 000000000..2591415a4 --- /dev/null +++ b/cpp/src/dataset/dataset_index.cc @@ -0,0 +1,1023 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include "dataset/dataset_index.h" + +#include +#include +#include +#include +#include + +#ifdef _WIN32 +#include +#include +#include +#include +#include +#else +#include +#include +#include +#include +#include +#endif + +namespace storage { +namespace dataset { + +namespace { + +const char DATASET_INDEX_MAGIC[8] = {'T', 'S', 'I', 'D', 'X', 0, 0, 0}; + +bool is_little_endian() { + const uint16_t value = 1; + return *reinterpret_cast(&value) == 1; +} + +uint64_t align64(uint64_t value) { + if (value > std::numeric_limits::max() - 63) { + return 0; + } + return (value + 63) & ~static_cast(63); +} + +bool add_overflows(uint64_t left, uint64_t right) { + return left > std::numeric_limits::max() - right; +} + +bool multiply_overflows(uint64_t left, uint64_t right) { + return right != 0 && left > std::numeric_limits::max() / right; +} + +bool range_valid(uint32_t first, uint32_t count, uint32_t total) { + return first <= total && count <= total - first; +} + +uint32_t expected_record_size(DatasetIndexSectionType type) { + switch (type) { + case DatasetIndexSectionType::STRING_OFFSETS: + return sizeof(uint32_t); + case DatasetIndexSectionType::STRING_BYTES: + return 0; + case DatasetIndexSectionType::TABLE_NAME_INDEX: + return sizeof(TableNameIndexRecord); + case DatasetIndexSectionType::TABLE_RECORD: + return sizeof(TableRecord); + case DatasetIndexSectionType::DEVICE_NAME_INDEX: + return sizeof(DeviceNameIndexRecord); + case DatasetIndexSectionType::DEVICE_RECORD: + return sizeof(DeviceRecord); + case DatasetIndexSectionType::COLUMN_NAME_INDEX: + return sizeof(ColumnNameIndexRecord); + case DatasetIndexSectionType::COLUMN_SCHEMA: + return sizeof(ColumnSchemaRecord); + case DatasetIndexSectionType::LOGICAL_SERIES: + return sizeof(LogicalSeriesRecord); + case DatasetIndexSectionType::TSFILE_RECORD: + return sizeof(TsFileRecord); + case DatasetIndexSectionType::TABLE_FILE_SPAN: + return sizeof(TableFileSpanRecord); + case DatasetIndexSectionType::DEVICE_FILE_SPAN: + return sizeof(DeviceFileSpanRecord); + case DatasetIndexSectionType::SERIES_FILE_SPAN: + return sizeof(SeriesFileSpanRecord); + case DatasetIndexSectionType::SERIES_LOCATOR: + return sizeof(SeriesLocatorRecord); + } + return std::numeric_limits::max(); +} + +std::string parent_directory(const std::string& path) { + const std::string::size_type pos = path.find_last_of("/\\"); + if (pos == std::string::npos) { + return "."; + } + if (pos == 0) { + return path.substr(0, 1); + } + return path.substr(0, pos); +} + +std::string system_error_message(const char* operation) { + std::ostringstream stream; + stream << operation << " failed: " << std::strerror(errno); + return stream.str(); +} + +#ifdef _WIN32 +typedef int NativeFile; +const NativeFile INVALID_NATIVE_FILE = -1; + +NativeFile open_exclusive(const std::string& path) { + return _open(path.c_str(), _O_BINARY | _O_CREAT | _O_EXCL | _O_WRONLY, + _S_IREAD | _S_IWRITE); +} + +int native_write(NativeFile file, const void* data, size_t size) { + const char* cursor = static_cast(data); + while (size > 0) { + const unsigned int chunk = static_cast( + std::min(size, std::numeric_limits::max())); + const int written = _write(file, cursor, chunk); + if (written <= 0) { + return -1; + } + cursor += written; + size -= static_cast(written); + } + return 0; +} + +int native_sync(NativeFile file) { return _commit(file); } +void native_close(NativeFile file) { _close(file); } +int process_id() { return _getpid(); } +#else +typedef int NativeFile; +const NativeFile INVALID_NATIVE_FILE = -1; + +NativeFile open_exclusive(const std::string& path) { + return ::open(path.c_str(), O_CREAT | O_EXCL | O_WRONLY, 0644); +} + +int native_write(NativeFile file, const void* data, size_t size) { + const uint8_t* cursor = static_cast(data); + while (size > 0) { + const ssize_t written = ::write(file, cursor, size); + if (written < 0) { + if (errno == EINTR) { + continue; + } + return -1; + } + if (written == 0) { + errno = EIO; + return -1; + } + cursor += written; + size -= static_cast(written); + } + return 0; +} + +int native_sync(NativeFile file) { return ::fsync(file); } +void native_close(NativeFile file) { ::close(file); } +int process_id() { return static_cast(::getpid()); } +#endif + +int write_zero_padding(NativeFile file, uint64_t length) { + static const uint8_t zeros[64] = {0}; + while (length > 0) { + const size_t chunk = + static_cast(std::min(length, sizeof(zeros))); + if (native_write(file, zeros, chunk) != 0) { + return -1; + } + length -= chunk; + } + return 0; +} + +bool string_equals(const MappedDatasetIndex& index, uint32_t sid, + const std::string& expected) { + DatasetIndexStringView value; + return index.string(sid, value) == DatasetIndexStatus::OK && + value.length == expected.size() && + (value.length == 0 || + std::memcmp(value.data, expected.data(), value.length) == 0); +} + +} // namespace + +const char* dataset_index_status_name(DatasetIndexStatus status) { + switch (status) { + case DatasetIndexStatus::OK: + return "OK"; + case DatasetIndexStatus::INVALID_ARGUMENT: + return "INVALID_ARGUMENT"; + case DatasetIndexStatus::IO_ERROR: + return "IO_ERROR"; + case DatasetIndexStatus::OUT_OF_MEMORY: + return "OUT_OF_MEMORY"; + case DatasetIndexStatus::BAD_MAGIC: + return "BAD_MAGIC"; + case DatasetIndexStatus::UNSUPPORTED_VERSION: + return "UNSUPPORTED_VERSION"; + case DatasetIndexStatus::BAD_HEADER: + return "BAD_HEADER"; + case DatasetIndexStatus::BAD_DIRECTORY: + return "BAD_DIRECTORY"; + case DatasetIndexStatus::BAD_SECTION: + return "BAD_SECTION"; + case DatasetIndexStatus::BAD_CHECKSUM: + return "BAD_CHECKSUM"; + case DatasetIndexStatus::BAD_REFERENCE: + return "BAD_REFERENCE"; + case DatasetIndexStatus::NOT_FOUND: + return "NOT_FOUND"; + } + return "UNKNOWN"; +} + +uint32_t dataset_index_crc32c(const void* data, size_t length) { + const uint8_t* bytes = static_cast(data); + uint32_t crc = ~static_cast(0); + for (size_t i = 0; i < length; ++i) { + crc ^= bytes[i]; + for (int bit = 0; bit < 8; ++bit) { + const uint32_t mask = + static_cast(-(static_cast(crc & 1))); + crc = (crc >> 1) ^ (0x82F63B78U & mask); + } + } + return ~crc; +} + +uint64_t dataset_index_name_hash(const char* data, size_t length) { + uint64_t hash = 1469598103934665603ULL; + for (size_t i = 0; i < length; ++i) { + hash ^= static_cast(data[i]); + hash *= 1099511628211ULL; + } + return hash; +} + +DatasetIndexStatus DatasetIndexWriter::write_atomic( + const std::string& output_path, + const std::vector& input_sections, + std::string& error_message) { + error_message.clear(); + if (output_path.empty() || !is_little_endian()) { + error_message = "empty output path or unsupported host byte order"; + return DatasetIndexStatus::INVALID_ARGUMENT; + } + if (input_sections.size() != DATASET_INDEX_SECTION_COUNT) { + error_message = "current v1 writer requires exactly 14 sections"; + return DatasetIndexStatus::INVALID_ARGUMENT; + } + + std::vector sections(input_sections); + std::sort(sections.begin(), sections.end(), + [](const DatasetIndexSectionData& left, + const DatasetIndexSectionData& right) { + return static_cast(left.type) < + static_cast(right.type); + }); + + DatasetIndexHeader header; + std::memset(&header, 0, sizeof(header)); + std::memcpy(header.magic, DATASET_INDEX_MAGIC, sizeof(header.magic)); + header.version_major = DATASET_INDEX_VERSION_MAJOR; + header.version_minor = DATASET_INDEX_VERSION_MINOR; + header.header_size = DATASET_INDEX_HEADER_SIZE; + header.directory_offset = DATASET_INDEX_HEADER_SIZE; + header.section_count = DATASET_INDEX_SECTION_COUNT; + header.directory_entry_size = DATASET_INDEX_DIRECTORY_ENTRY_SIZE; + + std::vector directory( + DATASET_INDEX_SECTION_COUNT); + uint64_t next_offset = align64(header.directory_offset + + static_cast(directory.size()) * + sizeof(DatasetIndexDirectoryEntry)); + if (next_offset == 0) { + error_message = "directory size overflows"; + return DatasetIndexStatus::INVALID_ARGUMENT; + } + + for (uint32_t i = 0; i < DATASET_INDEX_SECTION_COUNT; ++i) { + const uint32_t expected_type = i + 1; + const uint32_t actual_type = static_cast(sections[i].type); + const uint32_t expected_size = expected_record_size(sections[i].type); + if (actual_type != expected_type || + sections[i].record_size != expected_size) { + error_message = "section type or record size does not match v1"; + return DatasetIndexStatus::INVALID_ARGUMENT; + } + if (expected_size != 0 && + (multiply_overflows(sections[i].count, expected_size) || + static_cast(sections[i].count) * expected_size != + sections[i].bytes.size())) { + error_message = "fixed-size section count does not match bytes"; + return DatasetIndexStatus::INVALID_ARGUMENT; + } + if (expected_size == 0 && + sections[i].count != sections[i].bytes.size()) { + error_message = "blob section count must equal byte length"; + return DatasetIndexStatus::INVALID_ARGUMENT; + } + DatasetIndexDirectoryEntry& entry = directory[i]; + std::memset(&entry, 0, sizeof(entry)); + entry.section_type = actual_type; + entry.record_size = expected_size; + entry.offset = next_offset; + entry.length = sections[i].bytes.size(); + entry.count = sections[i].count; + entry.crc32c = dataset_index_crc32c( + sections[i].bytes.empty() ? nullptr : sections[i].bytes.data(), + sections[i].bytes.size()); + if (add_overflows(entry.offset, entry.length)) { + error_message = "section file range overflows"; + return DatasetIndexStatus::INVALID_ARGUMENT; + } + next_offset = align64(entry.offset + entry.length); + if (next_offset == 0 && i + 1 < DATASET_INDEX_SECTION_COUNT) { + error_message = "section alignment overflows"; + return DatasetIndexStatus::INVALID_ARGUMENT; + } + } + + const DatasetIndexDirectoryEntry& last = directory.back(); + header.file_length = last.offset + last.length; + header.header_crc32c = 0; + header.header_crc32c = dataset_index_crc32c(&header, sizeof(header)); + + std::ostringstream temp_name; + temp_name << output_path << ".tmp." << process_id(); + const std::string temp_path = temp_name.str(); + NativeFile file = open_exclusive(temp_path); + if (file == INVALID_NATIVE_FILE) { + error_message = system_error_message("create temporary index"); + return DatasetIndexStatus::IO_ERROR; + } + + DatasetIndexStatus status = DatasetIndexStatus::OK; + uint64_t cursor = 0; + if (native_write(file, &header, sizeof(header)) != 0 || + native_write(file, directory.data(), + directory.size() * sizeof(directory[0])) != 0) { + error_message = system_error_message("write index header"); + status = DatasetIndexStatus::IO_ERROR; + } else { + cursor = header.directory_offset + + directory.size() * sizeof(DatasetIndexDirectoryEntry); + } + + for (uint32_t i = 0; + status == DatasetIndexStatus::OK && i < directory.size(); ++i) { + if (directory[i].offset < cursor || + write_zero_padding(file, directory[i].offset - cursor) != 0 || + native_write( + file, + sections[i].bytes.empty() ? nullptr : sections[i].bytes.data(), + sections[i].bytes.size()) != 0) { + error_message = system_error_message("write index section"); + status = DatasetIndexStatus::IO_ERROR; + break; + } + cursor = directory[i].offset + directory[i].length; + } + if (status == DatasetIndexStatus::OK && native_sync(file) != 0) { + error_message = system_error_message("fsync index"); + status = DatasetIndexStatus::IO_ERROR; + } + native_close(file); + + if (status == DatasetIndexStatus::OK) { +#ifdef _WIN32 + if (!MoveFileExA(temp_path.c_str(), output_path.c_str(), + MOVEFILE_REPLACE_EXISTING | MOVEFILE_WRITE_THROUGH)) { + error_message = "replace index failed"; + status = DatasetIndexStatus::IO_ERROR; + } +#else + if (::rename(temp_path.c_str(), output_path.c_str()) != 0) { + error_message = system_error_message("replace index"); + status = DatasetIndexStatus::IO_ERROR; + } else { + const std::string directory_path = parent_directory(output_path); + const int directory_fd = ::open(directory_path.c_str(), O_RDONLY); + if (directory_fd >= 0) { + if (::fsync(directory_fd) != 0) { + error_message = + system_error_message("fsync index directory"); + status = DatasetIndexStatus::IO_ERROR; + } + ::close(directory_fd); + } + } +#endif + } + if (status != DatasetIndexStatus::OK) { +#ifdef _WIN32 + _unlink(temp_path.c_str()); +#else + ::unlink(temp_path.c_str()); +#endif + } + return status; +} + +MappedDatasetIndex::MappedDatasetIndex() + : mapping_(nullptr), + mapping_size_(0), + header_(nullptr), + directory_(nullptr) +#ifdef _WIN32 + , + file_handle_(INVALID_HANDLE_VALUE), + mapping_handle_(nullptr) +#else + , + fd_(-1) +#endif +{ +} + +MappedDatasetIndex::~MappedDatasetIndex() { close(); } + +DatasetIndexStatus MappedDatasetIndex::fail(DatasetIndexStatus status, + const std::string& message) { + error_message_ = message; + return status; +} + +DatasetIndexStatus MappedDatasetIndex::open(const std::string& path) { + close(); + path_ = path; + if (path.empty() || !is_little_endian()) { + return fail(DatasetIndexStatus::INVALID_ARGUMENT, + "empty path or unsupported host byte order"); + } +#ifdef _WIN32 + HANDLE file = + CreateFileA(path.c_str(), GENERIC_READ, FILE_SHARE_READ, nullptr, + OPEN_EXISTING, FILE_ATTRIBUTE_NORMAL, nullptr); + if (file == INVALID_HANDLE_VALUE) { + return fail(DatasetIndexStatus::IO_ERROR, "open index failed"); + } + LARGE_INTEGER size; + if (!GetFileSizeEx(file, &size) || size.QuadPart <= 0) { + CloseHandle(file); + return fail(DatasetIndexStatus::IO_ERROR, "stat index failed"); + } + HANDLE mapping = + CreateFileMappingA(file, nullptr, PAGE_READONLY, 0, 0, nullptr); + if (mapping == nullptr) { + CloseHandle(file); + return fail(DatasetIndexStatus::IO_ERROR, "map index failed"); + } + const void* address = MapViewOfFile(mapping, FILE_MAP_READ, 0, 0, 0); + if (address == nullptr) { + CloseHandle(mapping); + CloseHandle(file); + return fail(DatasetIndexStatus::IO_ERROR, "map view failed"); + } + file_handle_ = file; + mapping_handle_ = mapping; + mapping_ = static_cast(address); + mapping_size_ = static_cast(size.QuadPart); +#else + fd_ = ::open(path.c_str(), O_RDONLY); + if (fd_ < 0) { + return fail(DatasetIndexStatus::IO_ERROR, + system_error_message("open index")); + } + struct stat stat_buffer; + if (::fstat(fd_, &stat_buffer) != 0 || stat_buffer.st_size <= 0) { + close(); + return fail(DatasetIndexStatus::IO_ERROR, + system_error_message("stat index")); + } + mapping_size_ = static_cast(stat_buffer.st_size); + void* address = + ::mmap(nullptr, mapping_size_, PROT_READ, MAP_SHARED, fd_, 0); + if (address == MAP_FAILED) { + mapping_ = nullptr; + close(); + return fail(DatasetIndexStatus::IO_ERROR, + system_error_message("mmap index")); + } + mapping_ = static_cast(address); +#endif + DatasetIndexStatus status = validate(); + if (status != DatasetIndexStatus::OK) { + const std::string message = error_message_; + close(); + error_message_ = message; + } + return status; +} + +void MappedDatasetIndex::close() { +#ifdef _WIN32 + if (mapping_ != nullptr) { + UnmapViewOfFile(mapping_); + } + if (mapping_handle_ != nullptr) { + CloseHandle(static_cast(mapping_handle_)); + } + if (file_handle_ != INVALID_HANDLE_VALUE) { + CloseHandle(static_cast(file_handle_)); + } + file_handle_ = INVALID_HANDLE_VALUE; + mapping_handle_ = nullptr; +#else + if (mapping_ != nullptr) { + ::munmap(const_cast(mapping_), mapping_size_); + } + if (fd_ >= 0) { + ::close(fd_); + } + fd_ = -1; +#endif + mapping_ = nullptr; + mapping_size_ = 0; + header_ = nullptr; + directory_ = nullptr; + path_.clear(); +} + +DatasetIndexStatus MappedDatasetIndex::section( + DatasetIndexSectionType type, DatasetIndexSectionView& result) const { + result = DatasetIndexSectionView(); + if (directory_ == nullptr) { + return DatasetIndexStatus::BAD_HEADER; + } + const uint32_t raw_type = static_cast(type); + if (raw_type == 0 || raw_type > header_->section_count) { + return DatasetIndexStatus::NOT_FOUND; + } + const DatasetIndexDirectoryEntry& entry = directory_[raw_type - 1]; + if (entry.section_type != raw_type) { + return DatasetIndexStatus::NOT_FOUND; + } + result.data = mapping_ + entry.offset; + result.length = entry.length; + result.record_size = entry.record_size; + result.count = entry.count; + return DatasetIndexStatus::OK; +} + +DatasetIndexStatus MappedDatasetIndex::string( + uint32_t sid, DatasetIndexStringView& result) const { + result = DatasetIndexStringView(); + DatasetIndexSectionView offsets; + DatasetIndexSectionView bytes; + if (section(DatasetIndexSectionType::STRING_OFFSETS, offsets) != + DatasetIndexStatus::OK || + section(DatasetIndexSectionType::STRING_BYTES, bytes) != + DatasetIndexStatus::OK || + offsets.count == 0 || sid + 1 >= offsets.count) { + return DatasetIndexStatus::NOT_FOUND; + } + const uint32_t* values = reinterpret_cast(offsets.data); + result.data = reinterpret_cast(bytes.data + values[sid]); + result.length = values[sid + 1] - values[sid]; + return DatasetIndexStatus::OK; +} + +DatasetIndexStatus MappedDatasetIndex::validate() { + if (mapping_size_ < sizeof(DatasetIndexHeader)) { + return fail(DatasetIndexStatus::BAD_HEADER, + "index is shorter than header"); + } + header_ = reinterpret_cast(mapping_); + if (std::memcmp(header_->magic, DATASET_INDEX_MAGIC, + sizeof(header_->magic)) != 0) { + return fail(DatasetIndexStatus::BAD_MAGIC, "bad dataset index magic"); + } + if (header_->version_major != DATASET_INDEX_VERSION_MAJOR || + header_->version_minor != DATASET_INDEX_VERSION_MINOR) { + return fail(DatasetIndexStatus::UNSUPPORTED_VERSION, + "unsupported dataset index version"); + } + if (header_->header_size != sizeof(DatasetIndexHeader) || + header_->directory_entry_size != sizeof(DatasetIndexDirectoryEntry) || + header_->section_count != DATASET_INDEX_SECTION_COUNT || + header_->directory_offset < sizeof(DatasetIndexHeader) || + header_->file_length != mapping_size_) { + return fail(DatasetIndexStatus::BAD_HEADER, + "header size, directory shape, or file length is invalid"); + } + for (size_t i = 0; i < sizeof(header_->reserved); ++i) { + if (header_->reserved[i] != 0) { + return fail(DatasetIndexStatus::BAD_HEADER, + "header reserved bytes are not zero"); + } + } + DatasetIndexHeader header_copy = *header_; + const uint32_t expected_header_crc = header_copy.header_crc32c; + header_copy.header_crc32c = 0; + if (dataset_index_crc32c(&header_copy, sizeof(header_copy)) != + expected_header_crc) { + return fail(DatasetIndexStatus::BAD_CHECKSUM, + "header checksum does not match"); + } + const uint64_t directory_length = + static_cast(header_->section_count) * + header_->directory_entry_size; + if (add_overflows(header_->directory_offset, directory_length) || + header_->directory_offset + directory_length > mapping_size_) { + return fail(DatasetIndexStatus::BAD_DIRECTORY, + "section directory is outside the file"); + } + directory_ = reinterpret_cast( + mapping_ + header_->directory_offset); + uint64_t minimum_section_offset = + align64(header_->directory_offset + directory_length); + uint64_t previous_end = minimum_section_offset; + for (uint32_t i = 0; i < header_->section_count; ++i) { + const DatasetIndexDirectoryEntry& entry = directory_[i]; + const DatasetIndexSectionType type = + static_cast(i + 1); + if (entry.section_type != i + 1 || + entry.record_size != expected_record_size(type)) { + return fail(DatasetIndexStatus::BAD_DIRECTORY, + "section type order or record size is invalid"); + } + if (entry.offset % DATASET_INDEX_ALIGNMENT != 0 || + entry.offset < minimum_section_offset || + entry.offset < previous_end || + add_overflows(entry.offset, entry.length) || + entry.offset + entry.length > mapping_size_) { + return fail( + DatasetIndexStatus::BAD_SECTION, + "section range is unaligned, overlapping, or out of bounds"); + } + if (entry.record_size == 0) { + if (entry.count != entry.length) { + return fail(DatasetIndexStatus::BAD_SECTION, + "blob count does not equal byte length"); + } + } else if (multiply_overflows(entry.count, entry.record_size) || + static_cast(entry.count) * entry.record_size > + entry.length) { + return fail(DatasetIndexStatus::BAD_SECTION, + "section record count exceeds section length"); + } + const uint32_t actual_crc = dataset_index_crc32c( + entry.length == 0 ? nullptr : mapping_ + entry.offset, + static_cast(entry.length)); + if (actual_crc != entry.crc32c) { + return fail(DatasetIndexStatus::BAD_CHECKSUM, + "section checksum does not match"); + } + previous_end = entry.offset + entry.length; + } + + DatasetIndexSectionView string_offsets_view; + DatasetIndexSectionView string_bytes_view; + section(DatasetIndexSectionType::STRING_OFFSETS, string_offsets_view); + section(DatasetIndexSectionType::STRING_BYTES, string_bytes_view); + if (string_offsets_view.count == 0) { + return fail(DatasetIndexStatus::BAD_REFERENCE, + "StringOffsets must contain the terminal offset"); + } + const uint32_t* string_offsets = + reinterpret_cast(string_offsets_view.data); + if (string_offsets[0] != 0) { + return fail(DatasetIndexStatus::BAD_REFERENCE, + "StringOffsets must start at zero"); + } + for (uint32_t i = 1; i < string_offsets_view.count; ++i) { + if (string_offsets[i] < string_offsets[i - 1] || + string_offsets[i] > string_bytes_view.length) { + return fail(DatasetIndexStatus::BAD_REFERENCE, + "StringOffsets is not monotonic or is out of bounds"); + } + } + if (string_offsets[string_offsets_view.count - 1] != + string_bytes_view.length) { + return fail(DatasetIndexStatus::BAD_REFERENCE, + "terminal string offset does not equal StringBytes length"); + } + const uint32_t string_count = string_offsets_view.count - 1; + + DatasetIndexSectionView table_names; + DatasetIndexSectionView tables; + DatasetIndexSectionView device_names; + DatasetIndexSectionView devices; + DatasetIndexSectionView column_names; + DatasetIndexSectionView columns; + DatasetIndexSectionView series; + DatasetIndexSectionView files; + DatasetIndexSectionView table_spans; + DatasetIndexSectionView device_spans; + DatasetIndexSectionView series_spans; + DatasetIndexSectionView locators; + section(DatasetIndexSectionType::TABLE_NAME_INDEX, table_names); + section(DatasetIndexSectionType::TABLE_RECORD, tables); + section(DatasetIndexSectionType::DEVICE_NAME_INDEX, device_names); + section(DatasetIndexSectionType::DEVICE_RECORD, devices); + section(DatasetIndexSectionType::COLUMN_NAME_INDEX, column_names); + section(DatasetIndexSectionType::COLUMN_SCHEMA, columns); + section(DatasetIndexSectionType::LOGICAL_SERIES, series); + section(DatasetIndexSectionType::TSFILE_RECORD, files); + section(DatasetIndexSectionType::TABLE_FILE_SPAN, table_spans); + section(DatasetIndexSectionType::DEVICE_FILE_SPAN, device_spans); + section(DatasetIndexSectionType::SERIES_FILE_SPAN, series_spans); + section(DatasetIndexSectionType::SERIES_LOCATOR, locators); + +#define DATASET_RECORDS(view, type) reinterpret_cast((view).data) + const TableNameIndexRecord* table_name_records = + DATASET_RECORDS(table_names, TableNameIndexRecord); + const TableRecord* table_records = DATASET_RECORDS(tables, TableRecord); + const DeviceNameIndexRecord* device_name_records = + DATASET_RECORDS(device_names, DeviceNameIndexRecord); + const DeviceRecord* device_records = DATASET_RECORDS(devices, DeviceRecord); + const ColumnNameIndexRecord* column_name_records = + DATASET_RECORDS(column_names, ColumnNameIndexRecord); + const ColumnSchemaRecord* column_records = + DATASET_RECORDS(columns, ColumnSchemaRecord); + const LogicalSeriesRecord* series_records = + DATASET_RECORDS(series, LogicalSeriesRecord); + const TsFileRecord* file_records = DATASET_RECORDS(files, TsFileRecord); + const TableFileSpanRecord* table_span_records = + DATASET_RECORDS(table_spans, TableFileSpanRecord); + const DeviceFileSpanRecord* device_span_records = + DATASET_RECORDS(device_spans, DeviceFileSpanRecord); + const SeriesFileSpanRecord* series_span_records = + DATASET_RECORDS(series_spans, SeriesFileSpanRecord); + const SeriesLocatorRecord* locator_records = + DATASET_RECORDS(locators, SeriesLocatorRecord); +#undef DATASET_RECORDS + + for (uint32_t i = 0; i < table_names.count; ++i) { + if (table_name_records[i].name_sid >= string_count || + table_name_records[i].table_id >= tables.count) { + return fail(DatasetIndexStatus::BAD_REFERENCE, + "TableNameIndex contains an invalid reference"); + } + } + for (uint32_t i = 0; i < tables.count; ++i) { + const TableRecord& value = table_records[i]; + if (value.name_sid >= string_count || value.reserved0 != 0 || + value.reserved1 != 0 || + !range_valid(value.first_device_name_index, value.device_count, + device_names.count) || + !range_valid(value.first_column_name_index, value.column_count, + column_names.count) || + !range_valid(value.first_file_span, value.file_span_count, + table_spans.count)) { + return fail( + DatasetIndexStatus::BAD_REFERENCE, + "TableRecord contains an invalid range or reserved value"); + } + } + for (uint32_t i = 0; i < device_names.count; ++i) { + const DeviceNameIndexRecord& value = device_name_records[i]; + if (value.table_id >= tables.count || + value.device_id >= devices.count || + value.name_sid >= string_count || value.reserved != 0) { + return fail(DatasetIndexStatus::BAD_REFERENCE, + "DeviceNameIndex contains an invalid reference"); + } + } + for (uint32_t i = 0; i < devices.count; ++i) { + const DeviceRecord& value = device_records[i]; + if (value.table_id >= tables.count || value.name_sid >= string_count || + value.reserved0 != 0 || value.reserved1 != 0 || + !range_valid(value.first_series_id, value.series_count, + series.count) || + !range_valid(value.first_file_span, value.file_span_count, + device_spans.count) || + (value.series_count != 0 && value.min_time > value.max_time)) { + return fail(DatasetIndexStatus::BAD_REFERENCE, + "DeviceRecord contains an invalid reference or range"); + } + } + for (uint32_t i = 0; i < column_names.count; ++i) { + const ColumnNameIndexRecord& value = column_name_records[i]; + if (value.table_id >= tables.count || + value.column_id >= columns.count || + value.name_sid >= string_count || value.reserved != 0) { + return fail(DatasetIndexStatus::BAD_REFERENCE, + "ColumnNameIndex contains an invalid reference"); + } + } + for (uint32_t i = 0; i < columns.count; ++i) { + const ColumnSchemaRecord& value = column_records[i]; + if (value.table_id >= tables.count || value.name_sid >= string_count || + value.nullable > 1 || value.reserved != 0) { + return fail(DatasetIndexStatus::BAD_REFERENCE, + "ColumnSchema contains an invalid reference or enum"); + } + } + for (uint32_t i = 0; i < series.count; ++i) { + const LogicalSeriesRecord& value = series_records[i]; + if (value.device_id >= devices.count || + value.column_id >= columns.count || + !range_valid(value.first_file_span, value.file_span_count, + series_spans.count) || + (value.file_span_count != 0 && value.min_time > value.max_time)) { + return fail(DatasetIndexStatus::BAD_REFERENCE, + "LogicalSeries contains an invalid reference or range"); + } + } + for (uint32_t i = 0; i < files.count; ++i) { + const TsFileRecord& value = file_records[i]; + if (value.path_sid >= string_count || value.reserved0 != 0 || + value.reserved1 != 0 || value.min_time > value.max_time) { + return fail(DatasetIndexStatus::BAD_REFERENCE, + "TsFileRecord contains an invalid reference or range"); + } + } + for (uint32_t i = 0; i < table_spans.count; ++i) { + const TableFileSpanRecord& value = table_span_records[i]; + if (value.table_id >= tables.count || value.file_id >= files.count || + value.min_time > value.max_time) { + return fail(DatasetIndexStatus::BAD_REFERENCE, + "TableFileSpan contains an invalid reference or range"); + } + } + for (uint32_t i = 0; i < device_spans.count; ++i) { + const DeviceFileSpanRecord& value = device_span_records[i]; + if (value.device_id >= devices.count || value.file_id >= files.count || + value.layout > 1 || + (value.flags & ~static_cast(1)) != 0 || + value.min_time > value.max_time || + (value.layout == 0 && + (value.time_meta_offset != 0 || value.time_meta_length != 0)) || + (value.layout == 1 && value.time_meta_length == 0)) { + return fail( + DatasetIndexStatus::BAD_REFERENCE, + "DeviceFileSpan contains an invalid reference or layout"); + } + if (value.layout == 1 && + (add_overflows(value.time_meta_offset, value.time_meta_length) || + value.time_meta_offset + value.time_meta_length > + file_records[value.file_id].file_size)) { + return fail(DatasetIndexStatus::BAD_REFERENCE, + "aligned time metadata range is outside its TsFile"); + } + } + for (uint32_t i = 0; i < series_spans.count; ++i) { + const SeriesFileSpanRecord& value = series_span_records[i]; + if (value.series_id >= series.count || value.file_id >= files.count || + value.locator_id >= locators.count || value.reserved != 0 || + value.min_time > value.max_time || + value.prefix_max_time < value.max_time) { + return fail( + DatasetIndexStatus::BAD_REFERENCE, + "SeriesFileSpan contains an invalid reference or range"); + } + } + for (uint32_t i = 0; i < locators.count; ++i) { + const SeriesLocatorRecord& value = locator_records[i]; + if (value.device_file_span_id >= device_spans.count || + value.locator_kind > 1 || + (value.flags & ~static_cast(1)) != 0 || + value.timeseries_meta_length == 0) { + return fail(DatasetIndexStatus::BAD_REFERENCE, + "SeriesLocator contains an invalid reference or enum"); + } + const DeviceFileSpanRecord& device_span = + device_span_records[value.device_file_span_id]; + if (value.locator_kind != device_span.layout || + add_overflows(value.timeseries_meta_offset, + value.timeseries_meta_length) || + value.timeseries_meta_offset + value.timeseries_meta_length > + file_records[device_span.file_id].file_size) { + return fail(DatasetIndexStatus::BAD_REFERENCE, + "SeriesLocator metadata range is outside its TsFile"); + } + } + error_message_.clear(); + return DatasetIndexStatus::OK; +} + +DatasetIndexStatus MappedDatasetIndex::find_table_ids( + const std::string& name, std::vector& table_ids) const { + table_ids.clear(); + DatasetIndexSectionView view; + if (section(DatasetIndexSectionType::TABLE_NAME_INDEX, view) != + DatasetIndexStatus::OK) { + return DatasetIndexStatus::BAD_SECTION; + } + const TableNameIndexRecord* records = + reinterpret_cast(view.data); + const uint64_t hash = dataset_index_name_hash(name.data(), name.size()); + uint32_t low = 0; + uint32_t high = view.count; + while (low < high) { + const uint32_t middle = low + (high - low) / 2; + if (records[middle].name_hash < hash) { + low = middle + 1; + } else { + high = middle; + } + } + for (uint32_t i = low; i < view.count && records[i].name_hash == hash; + ++i) { + if (string_equals(*this, records[i].name_sid, name)) { + table_ids.push_back(records[i].table_id); + } + } + return table_ids.empty() ? DatasetIndexStatus::NOT_FOUND + : DatasetIndexStatus::OK; +} + +DatasetIndexStatus MappedDatasetIndex::find_device_id( + uint32_t table_id, const std::string& name, uint32_t& device_id) const { + const TableRecord* table = nullptr; + if (record(DatasetIndexSectionType::TABLE_RECORD, table_id, table) != + DatasetIndexStatus::OK) { + return DatasetIndexStatus::NOT_FOUND; + } + DatasetIndexSectionView view; + section(DatasetIndexSectionType::DEVICE_NAME_INDEX, view); + const DeviceNameIndexRecord* records = + reinterpret_cast(view.data); + const uint64_t hash = dataset_index_name_hash(name.data(), name.size()); + uint32_t low = table->first_device_name_index; + uint32_t high = low + table->device_count; + while (low < high) { + const uint32_t middle = low + (high - low) / 2; + if (records[middle].name_hash < hash) { + low = middle + 1; + } else { + high = middle; + } + } + const uint32_t end = table->first_device_name_index + table->device_count; + for (uint32_t i = low; i < end && records[i].name_hash == hash; ++i) { + if (records[i].table_id == table_id && + string_equals(*this, records[i].name_sid, name)) { + device_id = records[i].device_id; + return DatasetIndexStatus::OK; + } + } + return DatasetIndexStatus::NOT_FOUND; +} + +DatasetIndexStatus MappedDatasetIndex::find_column_id( + uint32_t table_id, const std::string& name, uint32_t& column_id) const { + const TableRecord* table = nullptr; + if (record(DatasetIndexSectionType::TABLE_RECORD, table_id, table) != + DatasetIndexStatus::OK) { + return DatasetIndexStatus::NOT_FOUND; + } + DatasetIndexSectionView view; + section(DatasetIndexSectionType::COLUMN_NAME_INDEX, view); + const ColumnNameIndexRecord* records = + reinterpret_cast(view.data); + const uint64_t hash = dataset_index_name_hash(name.data(), name.size()); + uint32_t low = table->first_column_name_index; + uint32_t high = low + table->column_count; + while (low < high) { + const uint32_t middle = low + (high - low) / 2; + if (records[middle].name_hash < hash) { + low = middle + 1; + } else { + high = middle; + } + } + const uint32_t end = table->first_column_name_index + table->column_count; + for (uint32_t i = low; i < end && records[i].name_hash == hash; ++i) { + if (records[i].table_id == table_id && + string_equals(*this, records[i].name_sid, name)) { + column_id = records[i].column_id; + return DatasetIndexStatus::OK; + } + } + return DatasetIndexStatus::NOT_FOUND; +} + +DatasetIndexStatus MappedDatasetIndex::find_series_id( + uint32_t device_id, uint32_t column_id, uint32_t& series_id) const { + const DeviceRecord* device = nullptr; + if (record(DatasetIndexSectionType::DEVICE_RECORD, device_id, device) != + DatasetIndexStatus::OK) { + return DatasetIndexStatus::NOT_FOUND; + } + DatasetIndexSectionView view; + section(DatasetIndexSectionType::LOGICAL_SERIES, view); + const LogicalSeriesRecord* records = + reinterpret_cast(view.data); + uint32_t low = device->first_series_id; + uint32_t high = low + device->series_count; + while (low < high) { + const uint32_t middle = low + (high - low) / 2; + if (records[middle].column_id < column_id) { + low = middle + 1; + } else { + high = middle; + } + } + if (low < device->first_series_id + device->series_count && + records[low].device_id == device_id && + records[low].column_id == column_id) { + series_id = low; + return DatasetIndexStatus::OK; + } + return DatasetIndexStatus::NOT_FOUND; +} + +} // namespace dataset +} // namespace storage diff --git a/cpp/src/dataset/dataset_index.h b/cpp/src/dataset/dataset_index.h new file mode 100644 index 000000000..0ca119f38 --- /dev/null +++ b/cpp/src/dataset/dataset_index.h @@ -0,0 +1,350 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#ifndef DATASET_DATASET_INDEX_H +#define DATASET_DATASET_INDEX_H + +#include +#include + +#include +#include + +namespace storage { +namespace dataset { + +static const uint16_t DATASET_INDEX_VERSION_MAJOR = 1; +static const uint16_t DATASET_INDEX_VERSION_MINOR = 0; +static const uint32_t DATASET_INDEX_HEADER_SIZE = 64; +static const uint32_t DATASET_INDEX_DIRECTORY_ENTRY_SIZE = 32; +static const uint32_t DATASET_INDEX_SECTION_COUNT = 14; +static const uint64_t DATASET_INDEX_ALIGNMENT = 64; + +enum class DatasetIndexStatus { + OK = 0, + INVALID_ARGUMENT, + IO_ERROR, + OUT_OF_MEMORY, + BAD_MAGIC, + UNSUPPORTED_VERSION, + BAD_HEADER, + BAD_DIRECTORY, + BAD_SECTION, + BAD_CHECKSUM, + BAD_REFERENCE, + NOT_FOUND, +}; + +const char* dataset_index_status_name(DatasetIndexStatus status); + +enum class DatasetIndexSectionType : uint32_t { + STRING_OFFSETS = 1, + STRING_BYTES = 2, + TABLE_NAME_INDEX = 3, + TABLE_RECORD = 4, + DEVICE_NAME_INDEX = 5, + DEVICE_RECORD = 6, + COLUMN_NAME_INDEX = 7, + COLUMN_SCHEMA = 8, + LOGICAL_SERIES = 9, + TSFILE_RECORD = 10, + TABLE_FILE_SPAN = 11, + DEVICE_FILE_SPAN = 12, + SERIES_FILE_SPAN = 13, + SERIES_LOCATOR = 14, +}; + +#pragma pack(push, 1) +struct DatasetIndexHeader { + char magic[8]; + uint16_t version_major; + uint16_t version_minor; + uint32_t header_size; + uint64_t directory_offset; + uint32_t section_count; + uint32_t directory_entry_size; + uint64_t file_length; + uint32_t header_crc32c; + uint8_t reserved[20]; +}; + +struct DatasetIndexDirectoryEntry { + uint32_t section_type; + uint32_t record_size; + uint64_t offset; + uint64_t length; + uint32_t count; + uint32_t crc32c; +}; + +struct TableNameIndexRecord { + uint64_t name_hash; + uint32_t name_sid; + uint32_t table_id; +}; + +struct TableRecord { + uint32_t name_sid; + uint32_t reserved0; + uint64_t schema_fingerprint; + uint32_t first_device_name_index; + uint32_t device_count; + uint32_t first_column_name_index; + uint32_t column_count; + uint32_t first_file_span; + uint32_t file_span_count; + uint64_t reserved1; +}; + +struct DeviceNameIndexRecord { + uint32_t table_id; + uint32_t device_id; + uint64_t name_hash; + uint32_t name_sid; + uint32_t reserved; +}; + +struct DeviceRecord { + uint32_t table_id; + uint32_t name_sid; + uint32_t reserved0; + uint32_t reserved1; + uint32_t first_series_id; + uint32_t series_count; + uint32_t first_file_span; + uint32_t file_span_count; + int64_t min_time; + int64_t max_time; +}; + +struct ColumnNameIndexRecord { + uint32_t table_id; + uint32_t column_id; + uint64_t name_hash; + uint32_t name_sid; + uint32_t reserved; +}; + +struct ColumnSchemaRecord { + uint32_t table_id; + uint32_t name_sid; + uint32_t column_ordinal; + uint16_t logical_type; + uint16_t physical_type; + uint16_t encoding; + uint16_t compression; + uint16_t role; + uint16_t nullable; + uint64_t reserved; +}; + +struct LogicalSeriesRecord { + uint32_t device_id; + uint32_t column_id; + uint32_t first_file_span; + uint32_t file_span_count; + int64_t min_time; + int64_t max_time; +}; + +struct TsFileRecord { + uint32_t path_sid; + uint32_t reserved0; + uint64_t file_size; + int64_t min_time; + int64_t max_time; + uint64_t file_fingerprint; + uint64_t reserved1; +}; + +struct TableFileSpanRecord { + uint32_t table_id; + uint32_t file_id; + int64_t min_time; + int64_t max_time; + uint64_t row_count; +}; + +struct DeviceFileSpanRecord { + uint32_t device_id; + uint32_t file_id; + uint64_t time_meta_offset; + uint32_t time_meta_length; + uint16_t layout; + uint16_t flags; + int64_t min_time; + int64_t max_time; + uint64_t row_count; +}; + +struct SeriesFileSpanRecord { + uint32_t series_id; + uint32_t file_id; + uint32_t locator_id; + uint32_t reserved; + int64_t min_time; + int64_t max_time; + int64_t prefix_max_time; + uint64_t point_count; +}; + +struct SeriesLocatorRecord { + uint32_t device_file_span_id; + uint16_t locator_kind; + uint16_t flags; + uint64_t timeseries_meta_offset; + uint32_t timeseries_meta_length; + uint32_t chunk_count; +}; +#pragma pack(pop) + +static_assert(sizeof(DatasetIndexHeader) == 64, + "DatasetIndexHeader must be 64 bytes"); +static_assert(sizeof(DatasetIndexDirectoryEntry) == 32, + "DatasetIndexDirectoryEntry must be 32 bytes"); +static_assert(sizeof(TableNameIndexRecord) == 16, + "TableNameIndexRecord must be 16 bytes"); +static_assert(sizeof(TableRecord) == 48, "TableRecord must be 48 bytes"); +static_assert(sizeof(DeviceNameIndexRecord) == 24, + "DeviceNameIndexRecord must be 24 bytes"); +static_assert(sizeof(DeviceRecord) == 48, "DeviceRecord must be 48 bytes"); +static_assert(sizeof(ColumnNameIndexRecord) == 24, + "ColumnNameIndexRecord must be 24 bytes"); +static_assert(sizeof(ColumnSchemaRecord) == 32, + "ColumnSchemaRecord must be 32 bytes"); +static_assert(sizeof(LogicalSeriesRecord) == 32, + "LogicalSeriesRecord must be 32 bytes"); +static_assert(sizeof(TsFileRecord) == 48, "TsFileRecord must be 48 bytes"); +static_assert(sizeof(TableFileSpanRecord) == 32, + "TableFileSpanRecord must be 32 bytes"); +static_assert(sizeof(DeviceFileSpanRecord) == 48, + "DeviceFileSpanRecord must be 48 bytes"); +static_assert(sizeof(SeriesFileSpanRecord) == 48, + "SeriesFileSpanRecord must be 48 bytes"); +static_assert(sizeof(SeriesLocatorRecord) == 24, + "SeriesLocatorRecord must be 24 bytes"); + +struct DatasetIndexSectionData { + DatasetIndexSectionType type; + uint32_t record_size; + uint32_t count; + std::vector bytes; +}; + +struct DatasetIndexSectionView { + const uint8_t* data; + uint64_t length; + uint32_t record_size; + uint32_t count; + + DatasetIndexSectionView() + : data(nullptr), length(0), record_size(0), count(0) {} +}; + +struct DatasetIndexStringView { + const char* data; + uint32_t length; + + DatasetIndexStringView() : data(nullptr), length(0) {} + std::string to_string() const { + return data == nullptr ? std::string() : std::string(data, length); + } +}; + +uint32_t dataset_index_crc32c(const void* data, size_t length); +uint64_t dataset_index_name_hash(const char* data, size_t length); + +class DatasetIndexWriter { + public: + static DatasetIndexStatus write_atomic( + const std::string& output_path, + const std::vector& sections, + std::string& error_message); +}; + +class MappedDatasetIndex { + public: + MappedDatasetIndex(); + ~MappedDatasetIndex(); + + DatasetIndexStatus open(const std::string& path); + void close(); + bool is_open() const { return mapping_ != nullptr; } + const std::string& error_message() const { return error_message_; } + const std::string& path() const { return path_; } + uint64_t file_length() const { return mapping_size_; } + const DatasetIndexHeader* header() const { return header_; } + + DatasetIndexStatus section(DatasetIndexSectionType type, + DatasetIndexSectionView& result) const; + DatasetIndexStatus string(uint32_t sid, + DatasetIndexStringView& result) const; + + template + DatasetIndexStatus record(DatasetIndexSectionType type, uint32_t id, + const T*& result) const { + DatasetIndexSectionView view; + DatasetIndexStatus status = section(type, view); + if (status != DatasetIndexStatus::OK) { + result = nullptr; + return status; + } + if (view.record_size != sizeof(T) || id >= view.count) { + result = nullptr; + return DatasetIndexStatus::NOT_FOUND; + } + result = reinterpret_cast( + view.data + static_cast(id) * view.record_size); + return DatasetIndexStatus::OK; + } + + DatasetIndexStatus find_table_ids(const std::string& name, + std::vector& table_ids) const; + DatasetIndexStatus find_device_id(uint32_t table_id, + const std::string& name, + uint32_t& device_id) const; + DatasetIndexStatus find_column_id(uint32_t table_id, + const std::string& name, + uint32_t& column_id) const; + DatasetIndexStatus find_series_id(uint32_t device_id, uint32_t column_id, + uint32_t& series_id) const; + + private: + DatasetIndexStatus validate(); + DatasetIndexStatus fail(DatasetIndexStatus status, + const std::string& message); + + std::string path_; + std::string error_message_; + const uint8_t* mapping_; + uint64_t mapping_size_; + const DatasetIndexHeader* header_; + const DatasetIndexDirectoryEntry* directory_; +#ifdef _WIN32 + void* file_handle_; + void* mapping_handle_; +#else + int fd_; +#endif +}; + +} // namespace dataset +} // namespace storage + +#endif // DATASET_DATASET_INDEX_H diff --git a/cpp/src/file/read_file.cc b/cpp/src/file/read_file.cc index 7d41d7095..ce1f67197 100644 --- a/cpp/src/file/read_file.cc +++ b/cpp/src/file/read_file.cc @@ -39,6 +39,55 @@ ssize_t pread(int fd, void* buf, size_t count, uint64_t offset); using namespace common; namespace storage { +namespace { +uint64_t generation_hash(uint64_t size, int64_t mtime_ns) { + uint64_t hash = 1469598103934665603ULL; + const uint64_t values[2] = {size, static_cast(mtime_ns)}; + for (size_t word = 0; word < 2; ++word) { + for (size_t byte = 0; byte < 8; ++byte) { + hash ^= static_cast(values[word] >> (byte * 8)); + hash *= 1099511628211ULL; + } + } + return hash; +} +} // namespace + +int ReadFile::generation(uint64_t& size, uint64_t& fingerprint) const { + if (fd_ < 0) { + return E_FILE_READ_ERR; + } + int64_t mtime_ns = 0; +#ifdef _WIN32 + intptr_t handle_value = _get_osfhandle(fd_); + if (handle_value == -1) { + return E_FILE_READ_ERR; + } + FILE_BASIC_INFO info; + if (!GetFileInformationByHandleEx(reinterpret_cast(handle_value), + FileBasicInfo, &info, sizeof(info))) { + return E_FILE_READ_ERR; + } + static const int64_t WINDOWS_TO_UNIX_100NS = 116444736000000000LL; + mtime_ns = (info.LastWriteTime.QuadPart - WINDOWS_TO_UNIX_100NS) * 100; +#else + struct stat info; + if (::fstat(fd_, &info) != 0) { + return E_FILE_READ_ERR; + } +#ifdef __APPLE__ + mtime_ns = static_cast(info.st_mtimespec.tv_sec) * 1000000000LL + + info.st_mtimespec.tv_nsec; +#else + mtime_ns = static_cast(info.st_mtim.tv_sec) * 1000000000LL + + info.st_mtim.tv_nsec; +#endif +#endif + size = static_cast(file_size_); + fingerprint = generation_hash(size, mtime_ns); + return E_OK; +} + void ReadFile::close() { if (fd_ >= 0) { ::close(fd_); diff --git a/cpp/src/file/read_file.h b/cpp/src/file/read_file.h index c38940098..fbfe50a8b 100644 --- a/cpp/src/file/read_file.h +++ b/cpp/src/file/read_file.h @@ -40,6 +40,10 @@ class ReadFile { FORCE_INLINE int64_t file_size() const { return file_size_; } FORCE_INLINE const std::string& file_path() const { return file_path_; } + /** Return size and the Dataset Index v1 FNV fingerprint of size+mtime_ns. + */ + int generation(uint64_t& size, uint64_t& fingerprint) const; + /* * try to reader @buf_size bytes from @offset of this file * into @buf. @read_len return the actual len reader. diff --git a/cpp/src/file/tsfile_io_reader.cc b/cpp/src/file/tsfile_io_reader.cc index e028cf902..0f425ae5d 100644 --- a/cpp/src/file/tsfile_io_reader.cc +++ b/cpp/src/file/tsfile_io_reader.cc @@ -19,7 +19,10 @@ #include "file/tsfile_io_reader.h" +#include + #include "common/allocator/alloc_base.h" +#include "reader/prepared_series.h" using namespace common; @@ -88,6 +91,131 @@ int TsFileIOReader::alloc_ssi(std::shared_ptr device_id, return ret; } +namespace { +int load_exact_timeseries_index(ReadFile* read_file, uint64_t offset, + uint32_t length, PageArena& arena, + TimeseriesIndex*& index) { + if (read_file == nullptr || length == 0 || + length > static_cast(std::numeric_limits::max()) || + offset > static_cast(std::numeric_limits::max()) || + offset > static_cast(read_file->file_size()) || + length > static_cast(read_file->file_size()) - offset) { + return E_TSFILE_CORRUPTED; + } + char* bytes = static_cast(arena.alloc(length)); + void* index_memory = arena.alloc(sizeof(TimeseriesIndex)); + if (bytes == nullptr || index_memory == nullptr) { + return E_OOM; + } + int32_t read_length = 0; + int ret = read_file->read(static_cast(offset), bytes, + static_cast(length), read_length); + if (ret != E_OK || read_length != static_cast(length)) { + return ret == E_OK ? E_TSFILE_CORRUPTED : ret; + } + ByteStream stream; + stream.wrap_from(bytes, length); + index = new (index_memory) TimeseriesIndex; + ret = index->deserialize_from(stream, &arena); + if (ret != E_OK || stream.read_pos() != length) { + index = nullptr; + return ret == E_OK ? E_TSFILE_CORRUPTED : ret; + } + index->set_metadata_range(static_cast(offset), length); + return E_OK; +} +} // namespace + +int TsFileIOReader::prepare_series(const FileGeneration& generation, + const PreparedLocator& locator, + std::shared_ptr& prepared) { + prepared.reset(); + uint64_t actual_size = 0; + uint64_t actual_fingerprint = 0; + if (read_file_ == nullptr || generation.file_size == 0 || + read_file_->generation(actual_size, actual_fingerprint) != E_OK || + actual_size != generation.file_size || + (generation.file_fingerprint != 0 && + actual_fingerprint != generation.file_fingerprint) || + locator.value_metadata_length == 0 || locator.layout > 1) { + return E_INVALID_ARG; + } + std::shared_ptr candidate = + std::make_shared(generation, locator); + TimeseriesIndex* value_index = nullptr; + int ret = load_exact_timeseries_index( + read_file_, locator.value_metadata_offset, + locator.value_metadata_length, candidate->arena(), value_index); + if (ret != E_OK) { + return ret; + } + if (locator.chunk_count_hint != 0 && + value_index->get_chunk_meta_list()->size() != + locator.chunk_count_hint) { + return E_TSFILE_CORRUPTED; + } + if (locator.layout == 0) { + candidate->set_index(value_index); + } else { + if ((locator.flags & 1U) == 0 || locator.time_metadata_length == 0) { + return E_NOT_SUPPORT; + } + TimeseriesIndex* time_index = nullptr; + ret = load_exact_timeseries_index( + read_file_, locator.time_metadata_offset, + locator.time_metadata_length, candidate->arena(), time_index); + if (ret != E_OK) { + return ret; + } + if (time_index->get_chunk_meta_list()->size() != + value_index->get_chunk_meta_list()->size()) { + return E_NOT_SUPPORT; + } + void* aligned_memory = + candidate->arena().alloc(sizeof(AlignedTimeseriesIndex)); + if (aligned_memory == nullptr) { + return E_OOM; + } + AlignedTimeseriesIndex* aligned = + new (aligned_memory) AlignedTimeseriesIndex; + aligned->time_ts_idx_ = time_index; + aligned->value_ts_idx_ = value_index; + candidate->set_index(aligned); + } + prepared = candidate; + return E_OK; +} + +int TsFileIOReader::alloc_prepared_ssi( + const std::shared_ptr& prepared, + TsFileSeriesScanIterator*& ssi, common::PageArena& pa, + Filter* time_filter) { + ssi = nullptr; + if (prepared == nullptr || prepared->index() == nullptr) { + return E_INVALID_ARG; + } + if (time_filter != nullptr && + !filter_stasify(prepared->index(), time_filter)) { + return E_NO_MORE_DATA; + } + void* memory = + mem_alloc(sizeof(TsFileSeriesScanIterator), MOD_TSFILE_READER); + if (memory == nullptr) { + return E_OOM; + } + ssi = new (memory) TsFileSeriesScanIterator; + int ret = ssi->init_prepared(prepared, read_file_, time_filter, pa); + if (ret == E_OK) { + ret = ssi->init_chunk_reader(); + } + if (ret != E_OK) { + ssi->destroy(); + mem_free(ssi); + ssi = nullptr; + } + return ret; +} + int TsFileIOReader::alloc_multi_ssi( std::shared_ptr device_id, const std::vector& measurement_names, @@ -900,7 +1028,14 @@ int TsFileIOReader::get_time_column_metadata( return E_OOM; } ret_timeseries_index = new (buf) TimeseriesIndex; - ret_timeseries_index->deserialize_from(buffer, &pa); + if (RET_FAIL(ret_timeseries_index->deserialize_from(buffer, &pa))) { + return ret; + } + if (buffer.read_pos() > UINT32_MAX) { + return E_OVERFLOW; + } + ret_timeseries_index->set_metadata_range( + start_idx, static_cast(buffer.read_pos())); } else if (measurement_node->node_type_ == INTERNAL_MEASUREMENT) { start_idx = measurement_node->children_[0]->get_offset(); end_idx = measurement_node->children_[1]->get_offset(); @@ -947,8 +1082,18 @@ int TsFileIOReader::do_load_timeseries_index( TimeseriesIndex cur_timeseries_index; PageArena cur_timeseries_index_pa; cur_timeseries_index_pa.init(512, MOD_TSFILE_READER); // TODO 512 + const uint64_t relative_start = bs.read_pos(); if (RET_FAIL(cur_timeseries_index.deserialize_from( bs, &cur_timeseries_index_pa))) { + } else if (bs.read_pos() < relative_start || + bs.read_pos() - relative_start > UINT32_MAX) { + ret = E_OVERFLOW; + } else { + cur_timeseries_index.set_metadata_range( + start_offset + relative_start, + static_cast(bs.read_pos() - relative_start)); + } + if (RET_FAIL(ret)) { } else if (is_aligned && cur_timeseries_index.get_measurement_name().equal_to( target_measurement_name)) { @@ -1012,12 +1157,20 @@ int TsFileIOReader::do_load_all_timeseries_index( ByteStream bs; bs.wrap_from(ti_buf, read_size); while (bs.has_remaining()) { + const uint64_t relative_start = bs.read_pos(); void* buf = in_timeseries_index_pa.alloc(sizeof(TimeseriesIndex)); auto ts_idx = new (buf) TimeseriesIndex; if (RET_FAIL( ts_idx->deserialize_from(bs, &in_timeseries_index_pa))) { return ret; } + if (bs.read_pos() < relative_start || + bs.read_pos() - relative_start > UINT32_MAX) { + return E_OVERFLOW; + } + ts_idx->set_metadata_range( + start_offset + relative_start, + static_cast(bs.read_pos() - relative_start)); if (ts_idx->get_measurement_name().len_ == 0) continue; ts_indexs.push_back(ts_idx); } diff --git a/cpp/src/file/tsfile_io_reader.h b/cpp/src/file/tsfile_io_reader.h index dc9069b5c..9fbec51d4 100644 --- a/cpp/src/file/tsfile_io_reader.h +++ b/cpp/src/file/tsfile_io_reader.h @@ -20,6 +20,7 @@ #ifndef FILE_TSFILE_IO_REAER_H #define FILE_TSFILE_IO_REAER_H +#include #include #include #include @@ -33,6 +34,9 @@ #include "utils/storage_utils.h" namespace storage { class TsFileSeriesScanIterator; +class PreparedSeries; +struct FileGeneration; +struct PreparedLocator; /* * TODO: @@ -86,6 +90,15 @@ class TsFileIOReader { TsFileSeriesScanIterator*& ssi, common::PageArena& pa, Filter* time_filter = nullptr); + int prepare_series(const FileGeneration& generation, + const PreparedLocator& locator, + std::shared_ptr& prepared); + + int alloc_prepared_ssi(const std::shared_ptr& prepared, + TsFileSeriesScanIterator*& ssi, + common::PageArena& pa, + Filter* time_filter = nullptr); + void revert_ssi(TsFileSeriesScanIterator* ssi); std::string get_file_path() const { return read_file_->file_path(); } diff --git a/cpp/src/reader/prepared_series.cc b/cpp/src/reader/prepared_series.cc new file mode 100644 index 000000000..15261e91f --- /dev/null +++ b/cpp/src/reader/prepared_series.cc @@ -0,0 +1,79 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include "reader/prepared_series.h" + +#include + +namespace storage { + +bool PagePositionIndex::append_complete(const PageLocator& locator) { + std::lock_guard guard(mutex_); + if (locator.row_begin != covered_rows_ || + locator.row_end <= locator.row_begin) { + return false; + } + pages_.push_back(locator); + covered_rows_ = locator.row_end; + return true; +} + +bool PagePositionIndex::find(uint64_t row, PageLocator& result) const { + std::lock_guard guard(mutex_); + if (row >= covered_rows_) { + return false; + } + std::vector::const_iterator it = + std::upper_bound(pages_.begin(), pages_.end(), row, + [](uint64_t value, const PageLocator& locator) { + return value < locator.row_begin; + }); + if (it == pages_.begin()) { + return false; + } + --it; + if (row < it->row_begin || row >= it->row_end) { + return false; + } + result = *it; + return true; +} + +uint64_t PagePositionIndex::covered_rows() const { + std::lock_guard guard(mutex_); + return covered_rows_; +} + +size_t PagePositionIndex::size() const { + std::lock_guard guard(mutex_); + return pages_.size(); +} + +PreparedSeries::PreparedSeries(const FileGeneration& generation, + const PreparedLocator& locator) + : generation_(generation), locator_(locator), arena_(), index_(nullptr) { + arena_.init(512, common::MOD_TSFILE_READER); +} + +PreparedSeries::~PreparedSeries() { + index_ = nullptr; + arena_.destroy(); +} + +} // namespace storage diff --git a/cpp/src/reader/prepared_series.h b/cpp/src/reader/prepared_series.h new file mode 100644 index 000000000..4cb50b985 --- /dev/null +++ b/cpp/src/reader/prepared_series.h @@ -0,0 +1,130 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#ifndef READER_PREPARED_SERIES_H +#define READER_PREPARED_SERIES_H + +#include + +#include +#include + +#include "common/allocator/page_arena.h" +#include "common/tsfile_common.h" + +namespace storage { + +struct FileGeneration { + uint64_t mapped_index_identity; + uint32_t file_id; + uint64_t file_size; + uint64_t file_fingerprint; + + FileGeneration() + : mapped_index_identity(0), + file_id(0), + file_size(0), + file_fingerprint(0) {} +}; + +struct PreparedLocator { + uint32_t locator_id; + uint16_t layout; + uint16_t flags; + uint64_t value_metadata_offset; + uint32_t value_metadata_length; + uint64_t time_metadata_offset; + uint32_t time_metadata_length; + uint32_t chunk_count_hint; + + PreparedLocator() + : locator_id(0), + layout(0), + flags(0), + value_metadata_offset(0), + value_metadata_length(0), + time_metadata_offset(0), + time_metadata_length(0), + chunk_count_hint(0) {} +}; + +struct PageLocator { + uint64_t row_begin; + uint64_t row_end; + uint32_t chunk_ordinal; + uint64_t chunk_header_offset; + uint64_t page_header_offset; + uint64_t page_data_offset; + uint64_t value_chunk_header_offset; + uint64_t value_page_header_offset; + uint64_t value_page_data_offset; + + PageLocator() + : row_begin(0), + row_end(0), + chunk_ordinal(0), + chunk_header_offset(0), + page_header_offset(0), + page_data_offset(0), + value_chunk_header_offset(0), + value_page_header_offset(0), + value_page_data_offset(0) {} +}; + +class PagePositionIndex { + public: + PagePositionIndex() : covered_rows_(0) {} + + // Publishes only a complete, gap-free prefix. A failed append leaves the + // visible prefix unchanged. + bool append_complete(const PageLocator& locator); + bool find(uint64_t row, PageLocator& result) const; + uint64_t covered_rows() const; + size_t size() const; + + private: + mutable std::mutex mutex_; + std::vector pages_; + uint64_t covered_rows_; +}; + +class PreparedSeries { + public: + PreparedSeries(const FileGeneration& generation, + const PreparedLocator& locator); + ~PreparedSeries(); + + const FileGeneration& generation() const { return generation_; } + const PreparedLocator& locator() const { return locator_; } + ITimeseriesIndex* index() const { return index_; } + common::PageArena& arena() { return arena_; } + void set_index(ITimeseriesIndex* index) { index_ = index; } + PagePositionIndex& page_positions() { return page_positions_; } + + private: + FileGeneration generation_; + PreparedLocator locator_; + common::PageArena arena_; + ITimeseriesIndex* index_; + PagePositionIndex page_positions_; +}; + +} // namespace storage + +#endif // READER_PREPARED_SERIES_H diff --git a/cpp/src/reader/qds_without_timegenerator.cc b/cpp/src/reader/qds_without_timegenerator.cc index 7f520f339..dcdd79183 100644 --- a/cpp/src/reader/qds_without_timegenerator.cc +++ b/cpp/src/reader/qds_without_timegenerator.cc @@ -34,6 +34,50 @@ int QDSWithoutTimeGenerator::init(TsFileIOReader* io_reader, return init_internal(io_reader, qe); } +int QDSWithoutTimeGenerator::init_prepared( + TsFileIOReader* io_reader, const std::shared_ptr& prepared, + Filter* owned_time_filter, int offset, int limit, + const std::string& column_name) { + pa_.reset(); + pa_.init(512, common::MOD_TSFILE_READER); + io_reader_ = io_reader; + qe_ = nullptr; + owned_time_filter_ = owned_time_filter; + remaining_offset_ = offset; + remaining_limit_ = limit; + is_single_path_ = true; + index_lookup_.insert({"time", 0}); + + TsFileSeriesScanIterator* ssi = nullptr; + int ret = + io_reader_->alloc_prepared_ssi(prepared, ssi, pa_, owned_time_filter_); + if (ret == E_NO_MORE_DATA) { + // Preserve the normal empty-result contract even when the global + // statistic rejects the range before an SSI is allocated. + row_record_ = new RowRecord(1); + result_set_metadata_ = std::make_shared( + std::vector(), std::vector()); + return E_OK; + } + if (ret != E_OK) { + return ret; + } + ssi->set_row_range(offset, limit); + ssi_vec_.push_back(ssi); + tsblocks_.resize(1); + time_iters_.resize(1); + value_iters_.resize(1); + row_record_ = new RowRecord(2); + index_lookup_.insert({column_name, 1}); + get_next_tsblock(0, true); + remaining_offset_ = ssi->get_row_offset(); + remaining_limit_ = ssi->get_row_limit(); + result_set_metadata_ = std::make_shared( + std::vector(1, column_name), + std::vector(1, ssi->get_data_type())); + return E_OK; +} + int QDSWithoutTimeGenerator::init(TsFileIOReader* io_reader, QueryExpression* qe, int offset, int limit) { remaining_offset_ = offset; @@ -167,6 +211,10 @@ void QDSWithoutTimeGenerator::close() { delete qe_; qe_ = nullptr; } + if (owned_time_filter_ != nullptr) { + delete owned_time_filter_; + owned_time_filter_ = nullptr; + } pa_.destroy(); } diff --git a/cpp/src/reader/qds_without_timegenerator.h b/cpp/src/reader/qds_without_timegenerator.h index 1d929e575..cc9789e19 100644 --- a/cpp/src/reader/qds_without_timegenerator.h +++ b/cpp/src/reader/qds_without_timegenerator.h @@ -21,6 +21,7 @@ #define READER_QDS_WITHOUT_TIMEGENERATOR_H #include +#include #include #include "expression.h" @@ -42,11 +43,16 @@ class QDSWithoutTimeGenerator : public ResultSet { heap_time_(), remaining_offset_(0), remaining_limit_(-1), + owned_time_filter_(nullptr), is_single_path_(false) {} ~QDSWithoutTimeGenerator() { close(); } int init(TsFileIOReader* io_reader, QueryExpression* qe); int init(TsFileIOReader* io_reader, QueryExpression* qe, int offset, int limit); + int init_prepared(TsFileIOReader* io_reader, + const std::shared_ptr& prepared, + Filter* owned_time_filter, int offset, int limit, + const std::string& column_name); void close(); int next(bool& has_next); bool is_null(const std::string& column_name); @@ -72,6 +78,7 @@ class QDSWithoutTimeGenerator : public ResultSet { heap_time_; // key-->time, value-->path_index int remaining_offset_; int remaining_limit_; + Filter* owned_time_filter_; bool is_single_path_; }; diff --git a/cpp/src/reader/tsfile_executor.cc b/cpp/src/reader/tsfile_executor.cc index 15aaa161b..2f5ff3572 100644 --- a/cpp/src/reader/tsfile_executor.cc +++ b/cpp/src/reader/tsfile_executor.cc @@ -22,6 +22,8 @@ #include "expression.h" #include "qds_with_timegenerator.h" #include "qds_without_timegenerator.h" +#include "reader/filter/time_operator.h" +#include "reader/prepared_series.h" using namespace common; @@ -102,6 +104,34 @@ int TsFileExecutor::execute(QueryExpression* query_expr, ResultSet*& ret_qds, return ret; } +int TsFileExecutor::prepare_series(const FileGeneration& generation, + const PreparedLocator& locator, + std::shared_ptr& prepared) { + ASSERT(is_inited_); + return io_reader_.prepare_series(generation, locator, prepared); +} + +int TsFileExecutor::execute_prepared( + const std::shared_ptr& prepared, int64_t start_time, + int64_t end_time, int offset, int limit, const std::string& column_name, + ResultSet*& ret_qds) { + ASSERT(is_inited_); + ret_qds = nullptr; + if (prepared == nullptr || start_time > end_time || offset < 0) { + return E_INVALID_ARG; + } + QDSWithoutTimeGenerator* qds = new QDSWithoutTimeGenerator; + Filter* time_filter = new TimeBetween(start_time, end_time, false); + int ret = qds->init_prepared(&io_reader_, prepared, time_filter, offset, + limit, column_name); + if (ret != E_OK) { + delete qds; + return ret; + } + ret_qds = qds; + return E_OK; +} + int TsFileExecutor::execute_may_with_global_timefilter(QueryExpression* qe, ResultSet*& ret_qds) { int ret = E_OK; diff --git a/cpp/src/reader/tsfile_executor.h b/cpp/src/reader/tsfile_executor.h index 335134c89..dd857de2c 100644 --- a/cpp/src/reader/tsfile_executor.h +++ b/cpp/src/reader/tsfile_executor.h @@ -20,6 +20,7 @@ #define READER_TSFILE_EXECUTOR_H #include +#include #include "file/read_file.h" #include "query_executor.h" @@ -37,6 +38,13 @@ class TsFileExecutor // : public QueryExecutor int execute(QueryExpression* query_expr, ResultSet*& ret_qds); int execute(QueryExpression* query_expr, ResultSet*& ret_qds, int offset, int limit); + int prepare_series(const FileGeneration& generation, + const PreparedLocator& locator, + std::shared_ptr& prepared); + int execute_prepared(const std::shared_ptr& prepared, + int64_t start_time, int64_t end_time, int offset, + int limit, const std::string& column_name, + ResultSet*& ret_qds); void destroy_query_data_set(ResultSet* qds); TsFileMeta* get_tsfile_meta() { return io_reader_.get_tsfile_meta(); } TsFileIOReader* get_tsfile_io_reader() { return &io_reader_; } diff --git a/cpp/src/reader/tsfile_reader.cc b/cpp/src/reader/tsfile_reader.cc index fb5f8fd92..bd80f64fc 100644 --- a/cpp/src/reader/tsfile_reader.cc +++ b/cpp/src/reader/tsfile_reader.cc @@ -228,6 +228,23 @@ int TsFileReader::queryByRow(std::vector& path_list, int offset, return ret; } +int TsFileReader::prepare_series(const FileGeneration& generation, + const PreparedLocator& locator, + std::shared_ptr& prepared) { + return tsfile_executor_->prepare_series(generation, locator, prepared); +} + +int TsFileReader::query_prepared( + const std::shared_ptr& prepared, int64_t start_time, + int64_t end_time, int offset, int limit, ResultSet*& result_set) { + if (prepared == nullptr || prepared->index() == nullptr) { + return E_INVALID_ARG; + } + return tsfile_executor_->execute_prepared( + prepared, start_time, end_time, offset, limit, + prepared->index()->get_measurement_name().to_std_string(), result_set); +} + int TsFileReader::queryByRow(const std::string& table_name, const std::vector& column_names, int offset, int limit, ResultSet*& result_set, diff --git a/cpp/src/reader/tsfile_reader.h b/cpp/src/reader/tsfile_reader.h index 3ba490045..b166fa143 100644 --- a/cpp/src/reader/tsfile_reader.h +++ b/cpp/src/reader/tsfile_reader.h @@ -24,6 +24,7 @@ #include "common/tsfile_common.h" #include "expression.h" #include "file/read_file.h" +#include "reader/prepared_series.h" #include "reader/table_query_executor.h" namespace storage { class TsFileExecutor; @@ -128,6 +129,13 @@ class TsFileReader { int queryByRow(std::vector& path_list, int offset, int limit, ResultSet*& result_set); + int prepare_series(const FileGeneration& generation, + const PreparedLocator& locator, + std::shared_ptr& prepared); + int query_prepared(const std::shared_ptr& prepared, + int64_t start_time, int64_t end_time, int offset, + int limit, ResultSet*& result_set); + /** * @brief Query table-model data by row with offset/limit pushdown. * diff --git a/cpp/src/reader/tsfile_series_scan_iterator.cc b/cpp/src/reader/tsfile_series_scan_iterator.cc index 79ac8d308..2bd6a0187 100644 --- a/cpp/src/reader/tsfile_series_scan_iterator.cc +++ b/cpp/src/reader/tsfile_series_scan_iterator.cc @@ -22,6 +22,7 @@ #include #include "common/global.h" +#include "reader/prepared_series.h" #ifdef ENABLE_THREADS #include "common/thread_pool.h" #endif @@ -30,6 +31,23 @@ using namespace common; namespace storage { +int TsFileSeriesScanIterator::init_prepared( + const std::shared_ptr& prepared, ReadFile* read_file, + Filter* time_filter, common::PageArena& data_pa) { + if (prepared == nullptr || prepared->index() == nullptr || + read_file == nullptr) { + return E_INVALID_ARG; + } + prepared_ = prepared; + itimeseries_index_ = prepared->index(); + measurement_name_ = + itimeseries_index_->get_measurement_name().to_std_string(); + read_file_ = read_file; + time_filter_ = time_filter; + data_pa_ = &data_pa; + return E_OK; +} + namespace { bool chunk_may_satisfy_filter(ChunkMeta* chunk_meta, Filter* filter) { return filter == nullptr || chunk_meta == nullptr || @@ -57,8 +75,6 @@ void TsFileSeriesScanIterator::destroy() { dynamic_cast(itimeseries_index_)) { std::vector().swap(multi->value_ts_idxs_); } - itimeseries_index_ = nullptr; - timeseries_index_pa_.destroy(); if (chunk_reader_ != nullptr) { // destroy() already runs manual destructors on internal members // (chunk_header_, decoders, compressor, ...), so calling @@ -69,6 +85,10 @@ void TsFileSeriesScanIterator::destroy() { common::mem_free(chunk_reader_); chunk_reader_ = nullptr; } + itimeseries_index_ = nullptr; + if (prepared_ == nullptr) { + timeseries_index_pa_.destroy(); + } if (tsblock_ != nullptr) { tsblock_->~TsBlock(); tsblock_ = nullptr; @@ -82,6 +102,7 @@ void TsFileSeriesScanIterator::destroy() { std::vector::Iterator>().swap( value_chunk_meta_cursors_); device_id_.reset(); + prepared_.reset(); std::string().swap(measurement_name_); } diff --git a/cpp/src/reader/tsfile_series_scan_iterator.h b/cpp/src/reader/tsfile_series_scan_iterator.h index cb3832787..70cfe14f6 100644 --- a/cpp/src/reader/tsfile_series_scan_iterator.h +++ b/cpp/src/reader/tsfile_series_scan_iterator.h @@ -21,6 +21,7 @@ #define READER_TSFILE_SERIES_SCAN_ITERATOR_H #include +#include #include #include "aligned_chunk_reader.h" @@ -34,6 +35,7 @@ namespace storage { class TsFileIOReader; +class PreparedSeries; class TsFileSeriesScanIterator { public: @@ -49,6 +51,7 @@ class TsFileSeriesScanIterator { tuple_desc_(), tsblock_(nullptr), time_filter_(nullptr), + prepared_(), is_aligned_(false), is_multi_value_(false), row_offset_(0), @@ -65,6 +68,9 @@ class TsFileSeriesScanIterator { data_pa_ = &data_pa; return common::E_OK; } + int init_prepared(const std::shared_ptr& prepared, + ReadFile* read_file, Filter* time_filter, + common::PageArena& data_pa); void destroy(); /** @@ -210,6 +216,9 @@ class TsFileSeriesScanIterator { common::TupleDesc tuple_desc_; common::TsBlock* tsblock_; Filter* time_filter_; + // Keeps the arena-backed index alive until the chunk reader has released + // every pointer into it. Empty for the legacy path-owned metadata arena. + std::shared_ptr prepared_; bool is_aligned_ = false; bool is_multi_value_ = false; int row_offset_; diff --git a/cpp/test/CMakeLists.txt b/cpp/test/CMakeLists.txt index 7fc9041b1..2e88b02ca 100644 --- a/cpp/test/CMakeLists.txt +++ b/cpp/test/CMakeLists.txt @@ -187,6 +187,7 @@ file(GLOB_RECURSE TEST_SRCS "writer/*_test.cc" "cwrapper/*_test.cc" "compress/*_test.cc" + "dataset/*_test.cc" ) # Parser tests depend on the ANTLR4 runtime; only build them when it is enabled. diff --git a/cpp/test/dataset/dataset_index_test.cc b/cpp/test/dataset/dataset_index_test.cc new file mode 100644 index 000000000..57d8a4bf1 --- /dev/null +++ b/cpp/test/dataset/dataset_index_test.cc @@ -0,0 +1,273 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include "dataset/dataset_index.h" + +#include + +#include +#include +#include +#include +#include +#include + +#ifdef _WIN32 +#include +#else +#include +#endif + +namespace storage { +namespace dataset { +namespace { + +template +DatasetIndexSectionData fixed_section(DatasetIndexSectionType type, + const std::vector& records) { + DatasetIndexSectionData section; + section.type = type; + section.record_size = sizeof(T); + section.count = static_cast(records.size()); + section.bytes.resize(records.size() * sizeof(T)); + if (!records.empty()) { + std::memcpy(section.bytes.data(), records.data(), section.bytes.size()); + } + return section; +} + +std::vector make_minimal_sections() { + const std::string strings[] = {"table", "device", "value", "/tmp/a.tsfile"}; + std::vector offsets(1, 0); + std::vector bytes; + for (const std::string& value : strings) { + bytes.insert(bytes.end(), value.begin(), value.end()); + offsets.push_back(static_cast(bytes.size())); + } + + DatasetIndexSectionData string_bytes; + string_bytes.type = DatasetIndexSectionType::STRING_BYTES; + string_bytes.record_size = 0; + string_bytes.count = static_cast(bytes.size()); + string_bytes.bytes = bytes; + + TableNameIndexRecord table_name = { + dataset_index_name_hash(strings[0].data(), strings[0].size()), 0, 0}; + TableRecord table = {0, 0, 0x1234, 0, 1, 0, 1, 0, 1, 0}; + DeviceNameIndexRecord device_name = { + 0, 0, dataset_index_name_hash(strings[1].data(), strings[1].size()), 1, + 0}; + DeviceRecord device = {0, 1, 0, 0, 0, 1, 0, 1, 0, 9}; + ColumnNameIndexRecord column_name = { + 0, 0, dataset_index_name_hash(strings[2].data(), strings[2].size()), 2, + 0}; + ColumnSchemaRecord column = {0, 2, 0, 2, 2, 0, 0, 0, 1, 0}; + LogicalSeriesRecord series = {0, 0, 0, 1, 0, 9}; + TsFileRecord file = {3, 0, 4096, 0, 9, 0x5678, 0}; + TableFileSpanRecord table_span = {0, 0, 0, 9, 10}; + DeviceFileSpanRecord device_span = {0, 0, 0, 0, 0, 0, 0, 9, 10}; + SeriesFileSpanRecord series_span = {0, 0, 0, 0, 0, 9, 9, 10}; + SeriesLocatorRecord locator = {0, 0, 1, 128, 16, 1}; + + std::vector sections; + sections.push_back( + fixed_section(DatasetIndexSectionType::STRING_OFFSETS, offsets)); + sections.push_back(string_bytes); + sections.push_back( + fixed_section(DatasetIndexSectionType::TABLE_NAME_INDEX, + std::vector(1, table_name))); + sections.push_back(fixed_section(DatasetIndexSectionType::TABLE_RECORD, + std::vector(1, table))); + sections.push_back( + fixed_section(DatasetIndexSectionType::DEVICE_NAME_INDEX, + std::vector(1, device_name))); + sections.push_back(fixed_section(DatasetIndexSectionType::DEVICE_RECORD, + std::vector(1, device))); + sections.push_back( + fixed_section(DatasetIndexSectionType::COLUMN_NAME_INDEX, + std::vector(1, column_name))); + sections.push_back( + fixed_section(DatasetIndexSectionType::COLUMN_SCHEMA, + std::vector(1, column))); + sections.push_back( + fixed_section(DatasetIndexSectionType::LOGICAL_SERIES, + std::vector(1, series))); + sections.push_back(fixed_section(DatasetIndexSectionType::TSFILE_RECORD, + std::vector(1, file))); + sections.push_back( + fixed_section(DatasetIndexSectionType::TABLE_FILE_SPAN, + std::vector(1, table_span))); + sections.push_back( + fixed_section(DatasetIndexSectionType::DEVICE_FILE_SPAN, + std::vector(1, device_span))); + sections.push_back( + fixed_section(DatasetIndexSectionType::SERIES_FILE_SPAN, + std::vector(1, series_span))); + sections.push_back( + fixed_section(DatasetIndexSectionType::SERIES_LOCATOR, + std::vector(1, locator))); + return sections; +} + +int current_process_id() { +#ifdef _WIN32 + return _getpid(); +#else + return static_cast(getpid()); +#endif +} + +class DatasetIndexTest : public ::testing::Test { + protected: + void SetUp() override { + std::ostringstream stream; + stream << "dataset_index_test_" << current_process_id() << ".tsidx"; + path_ = stream.str(); + std::remove(path_.c_str()); + std::ostringstream temp; + temp << path_ << ".tmp." << current_process_id(); + std::remove(temp.str().c_str()); + } + + void TearDown() override { std::remove(path_.c_str()); } + + void write_valid() { + std::string error; + ASSERT_EQ(DatasetIndexStatus::OK, + DatasetIndexWriter::write_atomic( + path_, make_minimal_sections(), error)) + << error; + } + + template + void overwrite(uint64_t offset, const T& value) { + std::fstream file(path_.c_str(), + std::ios::binary | std::ios::in | std::ios::out); + ASSERT_TRUE(file.good()); + file.seekp(static_cast(offset)); + file.write(reinterpret_cast(&value), sizeof(value)); + ASSERT_TRUE(file.good()); + } + + std::string path_; +}; + +TEST_F(DatasetIndexTest, WritesMapsAndLooksUpMinimalIndex) { + write_valid(); + MappedDatasetIndex index; + ASSERT_EQ(DatasetIndexStatus::OK, index.open(path_)) + << index.error_message(); + EXPECT_EQ(DATASET_INDEX_SECTION_COUNT, index.header()->section_count); + + std::vector tables; + ASSERT_EQ(DatasetIndexStatus::OK, index.find_table_ids("table", tables)); + ASSERT_EQ(1U, tables.size()); + EXPECT_EQ(0U, tables[0]); + + uint32_t device_id = 99; + uint32_t column_id = 99; + uint32_t series_id = 99; + EXPECT_EQ(DatasetIndexStatus::OK, + index.find_device_id(0, "device", device_id)); + EXPECT_EQ(DatasetIndexStatus::OK, + index.find_column_id(0, "value", column_id)); + EXPECT_EQ(DatasetIndexStatus::OK, + index.find_series_id(device_id, column_id, series_id)); + EXPECT_EQ(0U, device_id); + EXPECT_EQ(0U, column_id); + EXPECT_EQ(0U, series_id); + + DatasetIndexStringView path; + ASSERT_EQ(DatasetIndexStatus::OK, index.string(3, path)); + EXPECT_EQ("/tmp/a.tsfile", path.to_string()); + EXPECT_EQ(DatasetIndexStatus::NOT_FOUND, + index.find_device_id(0, "missing", device_id)); +} + +TEST_F(DatasetIndexTest, RejectsUnsupportedVersionBeforePublishingViews) { + write_valid(); + uint16_t version = 2; + overwrite(offsetof(DatasetIndexHeader, version_major), version); + MappedDatasetIndex index; + EXPECT_EQ(DatasetIndexStatus::UNSUPPORTED_VERSION, index.open(path_)); + EXPECT_FALSE(index.is_open()); +} + +TEST_F(DatasetIndexTest, RejectsHeaderChecksumMismatch) { + write_valid(); + uint64_t wrong_length = 1; + overwrite(offsetof(DatasetIndexHeader, file_length), wrong_length); + MappedDatasetIndex index; + EXPECT_EQ(DatasetIndexStatus::BAD_HEADER, index.open(path_)); + EXPECT_FALSE(index.is_open()); +} + +TEST_F(DatasetIndexTest, RejectsSectionChecksumMismatch) { + write_valid(); + MappedDatasetIndex valid; + ASSERT_EQ(DatasetIndexStatus::OK, valid.open(path_)); + DatasetIndexSectionView strings; + ASSERT_EQ(DatasetIndexStatus::OK, + valid.section(DatasetIndexSectionType::STRING_BYTES, strings)); + const uint64_t string_offset = static_cast( + strings.data - reinterpret_cast(valid.header())); + valid.close(); + const uint8_t changed = 'X'; + overwrite(string_offset, changed); + + MappedDatasetIndex index; + EXPECT_EQ(DatasetIndexStatus::BAD_CHECKSUM, index.open(path_)); +} + +TEST_F(DatasetIndexTest, WriterRejectsMalformedSectionCount) { + std::vector sections = make_minimal_sections(); + sections.pop_back(); + std::string error; + EXPECT_EQ(DatasetIndexStatus::INVALID_ARGUMENT, + DatasetIndexWriter::write_atomic(path_, sections, error)); + EXPECT_FALSE(error.empty()); +} + +TEST(DatasetIndexChecksumTest, MatchesKnownCrc32cVector) { + const char* value = "123456789"; + EXPECT_EQ(0xE3069283U, dataset_index_crc32c(value, 9)); +} + +TEST(DatasetIndexCrossLanguageTest, OpensExternalIndexWhenConfigured) { + const char* path = std::getenv("TSFILE_DATASET_INDEX_TEST_PATH"); + if (path == nullptr || path[0] == '\0') { + GTEST_SKIP() << "TSFILE_DATASET_INDEX_TEST_PATH is not set"; + } + + MappedDatasetIndex index; + ASSERT_EQ(DatasetIndexStatus::OK, index.open(path)) + << index.error_message(); + DatasetIndexSectionView files; + DatasetIndexSectionView series; + ASSERT_EQ(DatasetIndexStatus::OK, + index.section(DatasetIndexSectionType::TSFILE_RECORD, files)); + ASSERT_EQ(DatasetIndexStatus::OK, + index.section(DatasetIndexSectionType::LOGICAL_SERIES, series)); + EXPECT_GT(files.count, 0U); + EXPECT_GT(series.count, 0U); +} + +} // namespace +} // namespace dataset +} // namespace storage diff --git a/cpp/test/reader/prepared_series_test.cc b/cpp/test/reader/prepared_series_test.cc new file mode 100644 index 000000000..aebc433a9 --- /dev/null +++ b/cpp/test/reader/prepared_series_test.cc @@ -0,0 +1,85 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include "reader/prepared_series.h" + +#include + +#include +#include + +namespace storage { +namespace { + +PageLocator page(uint64_t begin, uint64_t end, uint64_t offset) { + PageLocator result; + result.row_begin = begin; + result.row_end = end; + result.page_data_offset = offset; + return result; +} + +TEST(PagePositionIndexTest, PublishesOnlyCompleteGapFreePrefixes) { + PagePositionIndex index; + EXPECT_FALSE(index.append_complete(page(1, 5, 100))); + EXPECT_EQ(0U, index.covered_rows()); + EXPECT_TRUE(index.append_complete(page(0, 5, 100))); + EXPECT_TRUE(index.append_complete(page(5, 9, 200))); + EXPECT_FALSE(index.append_complete(page(8, 12, 300))); + EXPECT_EQ(9U, index.covered_rows()); + EXPECT_EQ(2U, index.size()); + + PageLocator found; + EXPECT_TRUE(index.find(0, found)); + EXPECT_EQ(100U, found.page_data_offset); + EXPECT_TRUE(index.find(8, found)); + EXPECT_EQ(200U, found.page_data_offset); + EXPECT_FALSE(index.find(9, found)); +} + +TEST(PagePositionIndexTest, ConcurrentWritersPublishOneGapFreePrefix) { + PagePositionIndex index; + constexpr uint64_t page_count = 64; + std::vector writers; + writers.reserve(page_count); + for (uint64_t row = 0; row < page_count; ++row) { + writers.emplace_back([row, &index]() { + PageLocator locator = page(row, row + 1, 1000 + row); + while (index.covered_rows() <= row && + !index.append_complete(locator)) { + std::this_thread::yield(); + } + }); + } + for (std::thread& writer : writers) { + writer.join(); + } + + EXPECT_EQ(page_count, index.covered_rows()); + EXPECT_EQ(page_count, index.size()); + for (uint64_t row = 0; row < page_count; ++row) { + PageLocator found; + ASSERT_TRUE(index.find(row, found)); + EXPECT_EQ(row, found.row_begin); + EXPECT_EQ(1000 + row, found.page_data_offset); + } +} + +} // namespace +} // namespace storage diff --git "a/design/Dataset Index \346\226\207\344\273\266\346\240\274\345\274\217\344\270\216\346\237\245\350\257\242\350\256\276\350\256\241.md" "b/design/Dataset Index \346\226\207\344\273\266\346\240\274\345\274\217\344\270\216\346\237\245\350\257\242\350\256\276\350\256\241.md" new file mode 100644 index 000000000..bf5ea274f --- /dev/null +++ "b/design/Dataset Index \346\226\207\344\273\266\346\240\274\345\274\217\344\270\216\346\237\245\350\257\242\350\256\276\350\256\241.md" @@ -0,0 +1,297 @@ +# Dataset Index 文件格式与查询设计 + +**核心查询链路。** `table name + optional schema_fingerprint → table_id → device name → device_id → measurement name → column_id → series_id → SeriesFileSpan → SeriesLocator → DeviceFileSpan → TsFileReader`。Runtime 不硬编码任何业务 section 的绝对 offset;所有业务地址都来自 SectionDirectory。 + +## Header 与 SectionDirectory + +|Header 字段|offset|大小|类型 / 本例值|语义| +|---|---|---|---|---| +|`magic`|`0x00–0x08`|8 B|`byte[8]`|`TSIDX\0\0\0`,逐字节比较,不按整数解释| +|`version_major`|`0x08–0x0A`|2 B|`u16 = 1`|不兼容的主版本直接拒绝| +|`version_minor`|`0x0A–0x0C`|2 B|`u16 = 0`|同一主版本内的兼容扩展| +|`header_size`|`0x0C–0x10`|4 B|`u32 = 64`|当前 v1 Header 编码长度;当前 parser 要求为 64 B| +|`directory_offset`|`0x10–0x18`|8 B|`u64 = 0x40`|SectionDirectory 的实际位置| +|`section_count`|`0x18–0x1C`|4 B|`u32 = 14`|当前文件实际包含的 DirectoryEntry 数量,不作为永久上限| +|`directory_entry_size`|`0x1C–0x20`|4 B|`u32 = 32`|当前 v1 DirectoryEntry 编码长度;当前 parser 要求为 32 B| +|`file_length`|`0x20–0x28`|8 B|`u64`|必须等于实际索引文件长度| +|`header_crc32c`|`0x28–0x2C`|4 B|`u32`|发现关键 offset 被随机损坏;计算时该字段按 0| +|`reserved`|`0x2C–0x40`|20 B|`byte[20] = 0`|一个连续保留区,不再拆分| + +端序固定为 little-endian,排序约束也是 v1 格式强制条件,因此两者不占 flags。当前 parser 支持 major=1、minor=0、header_size=64、directory_entry_size=32 和当前文件的 section_count=14;这些是当前编码值和兼容条件,不是业务容量或架构永久上限。未来新增可选 section、扩大 Header 或 DirectoryEntry 时,通过 version、size 字段和 SectionDirectory 定义兼容规则;不兼容变化升级 major version。UUID、snapshot_epoch 与 build_time 不进入核心格式;索引通过“构建临时文件 → fsync → 原子替换”发布,Runtime 用索引文件 identity / fingerprint 区分快照。 + +|DirectoryEntry 字段|相对 offset|大小|类型|用途| +|---|---|---|---|---| +|`section_type`|`+0`|4 B|`u32`|Section 枚举;Runtime 按 type 建立 SectionView| +|`record_size`|`+4`|4 B|`u32`|定长记录大小;StringBytes 等 blob 为 0| +|`offset`|`+8`|8 B|`u64`|Section 相对索引文件起点的实际 offset| +|`length`|`+16`|8 B|`u64`|Section 字节长度| +|`count`|`+24`|4 B|`u32`|记录数;必须满足 `count × record_size ≤ length`| +|`crc32c`|`+28`|4 B|`u32`
|该 Section 的 CRC32C;不再需要独立 SectionChecksum| + +当前 14 条 DirectoryEntry 位于 `0x40–0x200`;首个业务 Section 从 `align64(0x200) = 0x200` 开始。后续位置统一由 `next.offset = align64(current.offset + current.length)` 生成。这里的数值只描述当前 v1 编码;`TableRecord` 等业务 section 的 offset 必须读取对应 DirectoryEntry,不能写死为 `0x02C0`。 + +## 当前 v1 查询所需的 14 个 Section + +|\#|Section|record\_size|保存内容|排序 / 访问约束| +|---|---|---|---|---| +|1|`StringOffsets`|4 B|offset\[0\.\.N\]:u32;每项 4 B,共 4 × \(N\+1\) B,N 是去重字符串数;offset\[N\] 是 StringBytes 的有效长度|sid 的字符串为 `StringBytes[offset[sid]..offset[sid+1])`| +|2|`StringBytes`
|0|byte\[length\];变长 Section,每个对象是一段不含结尾 NUL 的 canonical UTF\-8 bytes,由相邻两个 StringOffsets 界定|单池小于 4 GiB;超出时分池或升级格式
| +|3|`TableNameIndex`
|16 B
|`name_hash:u64, name_sid:u32, table_id:u32;每行 16 B,表示一个 canonical table name 到一个逻辑 table_id 的映射`|按 \(name\_hash, name\_bytes\) 排序并允许同名多条;二分得到同名 table\_id 等值区间,再读取各 TableRecord 的 schema\_fingerprint 区分 schema variant| +|4|`TableRecord`
|48 B
|name\_sid:u32, reserved0:u32, schema\_fingerprint:u64, first\_device\_name\_index:u32, device\_count:u32, first\_column\_name\_index:u32, column\_count:u32, first\_file\_span:u32, file\_span\_count:u32, reserved1:u64;每行 48 B,描述一个 schema variant 及其 device\-name、column\-name、TableFileSpan 三个连续子区间|record ID 就是 table\_id
| +|5|`DeviceNameIndex`
|24 B
|`table_id:u32, device_id:u32, name_hash:u64, name_sid:u32, reserved:u32;每行 24 B,表示某一 table 内 canonical device name 到 device_id 的映射`|先按 table\_id 聚簇,再按 name\_hash/name\_bytes 排序| +|6|`DeviceRecord`
|48 B|table\_id:u32, name\_sid:u32, reserved0:u32, reserved1:u32, first\_series\_id:u32, series\_count:u32, first\_file\_span:u32, file\_span\_count:u32, min\_time:i64, max\_time:i64;每行 48 B,描述一个逻辑设备、它实际存在的连续 series 区间、跨文件区间及全局时间边界|record ID 就是 device\_id
| +|7|`ColumnNameIndex`|24 B|`table_id:u32, column_id:u32, name_hash:u64, name_sid:u32, reserved:u32;每行 24 B,表示某一 table 内 measurement name 到 column_id 的映射`|在 TableRecord 给出的 measurement 列索引区间内二分;时间列为隐式列| +|8|`ColumnSchema`|32 B|table\_id:u32, name\_sid:u32, column\_ordinal:u32, logical\_type:u16, physical\_type:u16, encoding:u16, compression:u16, role:u16, nullable:u16, reserved:u64;每行 32 B,描述 canonical 列定义|record ID 就是 column\_id;同一 table schema variant 只保存一份 canonical schema,实际 chunk 解码参数以 SeriesLocator/DeviceFileSpan 指向的 value/time TimeseriesMetadata 与 ChunkHeader 为准;aligned time 的 VECTOR 仅是物理时间列标记,不写入 value 列的 physical\_type| +|9|`LogicalSeries`|32 B
|`device_id:u32, column_id:u32, first_file_span:u32, file_span_count:u32, min_time:i64, max_time:i64;每行 32 B,表示一个真实存在的逻辑 series 及其连续物理 fragment 区间`|按 `(device_id, column_id)` 聚簇排序;在设备连续区间内二分,稀疏列不生成空记录
| +|10|`TsFileRecord`
|48 B|path\_sid:u32, reserved0:u32, file\_size:u64, min\_time:i64, max\_time:i64, file\_fingerprint:u64, reserved1:u64;每行 48 B,描述一个 TsFile 身份与文件级裁剪信息;TsFile 自身格式版本由 Reader 打开文件时校验,不在 Dataset Index 重复保存|record ID 就是 file\_id;文件身份由 path、file\_size 与 fingerprint 联合校验,不在本版本定义覆盖优先级| +|11|`TableFileSpan`
|32 B|table\_id:u32, file\_id:u32, min\_time:i64, max\_time:i64, row\_count:u64;每行 32 B,表示一张逻辑表在一个 TsFile 中的最小/最大时间及该表所有设备时间轴行数之和,用于表级文件裁剪与规模估算|按 `(table_id, min_time, file_id)` 聚簇排序
| +|12|`DeviceFileSpan`|48 B|device\_id:u32, file\_id:u32, time\_meta\_offset:u64, time\_meta\_length:u32, layout:u16, flags:u16, min\_time:i64, max\_time:i64, row\_count:u64;每行 48 B,表示一个逻辑设备在一个 TsFile 中的时间轴与共享 time metadata。layout 为 0=non\-aligned、1=aligned;non\-aligned 的 time\_meta\_offset/time\_meta\_length 必须为 0;aligned 时二者给出共享 time TimeseriesMetadata 的精确字节范围。flags 的 bit 0 表示构建期已验证 time/value chunk 可按 ordinal 一一配对,其余位必须为 0|按 \(device\_id, min\_time, file\_id\) 聚簇排序;同一个 device/file 的 aligned value locator 共享本记录中的 time metadata range| +|13|`SeriesFileSpan`|48 B|`series_id:u32, file_id:u32, locator_id:u32, reserved:u32, min_time:i64, max_time:i64, prefix_max_time:i64, point_count:u64;每行 48 B,表示一个逻辑 series 在一个 TsFile 中的物理 fragment、时间边界、累计重叠裁剪值和实际 value 数`|按 `(series_id, min_time, file_id)` 聚簇;prefix\_max\_time 支持重叠区间二分
| +|14|`SeriesLocator`|24 B|`device_file_span_id:u32, locator_kind:u16, flags:u16, timeseries_meta_offset:u64, timeseries_meta_length:u32, chunk_count:u32;每行 24 B。locator_kind 为 0=non-aligned TimeseriesMetadata、1=aligned value TimeseriesMetadata;flags 的 bit 0 表示 chunk_count 有效,其余位必须为 0。offset/length 始终指向一条完整、精确的 TimeseriesMetadata,而不是 measurement index block 或内部 chunk metadata 子区间`|record ID 就是 locator\_id;由 device\_file\_span\_id 取得 file\_id、layout 和共享 time metadata,再把精确 TimeseriesMetadata range 交给 TsFileReader| + +**同名表与 schema variant。** `TableNameIndex` 不要求 table name 唯一:每一种规范化 schema 生成独立 `TableRecord`,同名记录形成连续等值区间。`schema_fingerprint` 由有序的 TAG/FIELD 名称、角色、逻辑类型和 nullable 属性计算;不同 TsFile 仅改变 encoding/compression 时仍属于同一逻辑 schema,实际物理解码参数以 SeriesLocator/DeviceFileSpan 指向的 value/time TimeseriesMetadata 与 ChunkHeader 为准。查询只给 table name 且等值区间中存在多个 fingerprint 时,Runtime 必须返回“schema variant 不明确”,由调用方显式选择 fingerprint;不得把 TAG/FIELD 集合或类型不同的表静默合并。以后如需 schema union,应在 Dataset 逻辑层显式定义列对齐与缺失值规则,不能改变本索引的精确映射。 + +## 核心记录布局 + +|Record|固定大小|字段|查询作用| +|---|---|---|---| +|`TableRecord`|48 B|`name_sid:u32, reserved0:u32, schema_fingerprint:u64, first_device_name_index:u32, device_count:u32, first_column_name_index:u32, column_count:u32, first_file_span:u32, file_span_count:u32, reserved1:u64`|限定设备名和 measurement 名的二分范围,并支持表级文件裁剪| +|`DeviceRecord`|48 B|`table_id:u32, name_sid:u32, reserved0:u32, reserved1:u32, first_series_id:u32, series_count:u32, first_file_span:u32, file_span_count:u32, min_time:i64, max_time:i64`|把设备和它的连续 LogicalSeries / DeviceFileSpan 关联起来| +|`ColumnSchema`|32 B|`table_id:u32, name_sid:u32, column_ordinal:u32, logical_type:u16, physical_type:u16, encoding:u16, compression:u16, role:u16, nullable:u16, reserved:u64`|提供 measurement 的逻辑类型与实际 value 类型、角色、nullable 属性和表内 ordinal;aligned time 的 VECTOR 物理标记不作为 value 列类型| +|`LogicalSeries`|32 B|`device_id:u32, column_id:u32, first_file_span:u32, file_span_count:u32, min_time:i64, max_time:i64`|直接取得该序列的物理文件区间| +|`SeriesFileSpan`|48 B|`series_id:u32, file_id:u32, locator_id:u32, reserved:u32, min_time:i64, max_time:i64, prefix_max_time:i64, point_count:u64`|按查询时间过滤 TsFile;prefix\_max\_time 处理文件时间范围重叠| +|`SeriesLocator`|24 B|`device_file_span_id:u32, locator_kind:u16, flags:u16, timeseries_meta_offset:u64, timeseries_meta_length:u32, chunk_count:u32`|locator\_kind 区分 non\-aligned metadata 与 aligned value metadata;通过 device\_file\_span\_id 复用共享 time metadata,不遍历 TsFile 文件级 metadata index 即可构造目标 TimeseriesIndex| + +## A / B / C 的完整示例 + +本例完整定义如下:`shared_table` 的 schema 为 `s1:INT64`、`s2:DOUBLE`;A 保存 d0、d1 的 `[0,99]`,B 保存 d0、d2 的 `[100,199]`。A 另含 `table_A/a0/mA:INT64`,B 另含 `table_B/b0/mB:DOUBLE`,C 含 `table_C1/c0/mC1:INT64` 与 `table_C2/c1/mC2:DOUBLE`,这四条序列均取 `[0,99]`。每个设备在每个文件中有 100 行,每条 measurement 有 100 个 value。本节是 non\-aligned 基线示例;aligned 的共享时间轴与稀疏 value 示例在下一节单独给出。下表沿用上方六列表格样式,每个索引对象独占一行,列出 14 个 Section 的全部逻辑对象和物理 fragment。本例首个业务 Section 从 0x0200 开始,各 Section 按目录顺序以 64 B 对齐;“索引内直接地址”均由 section\_offset \+ record\_id × record\_size 计算。TsFile 内部 offset 仅为本例数值。 + +|Section / record|对象|索引内直接地址|实际保存值|关联记录|含义| +|---|---|---|---|---|---| +|**StringOffsets \+ StringBytes:每个 sid 一行,同时给出 offset 项与 UTF\-8 字节范围**|||||| +|`StringPool[0]`|/dataset/A\.tsfile
|`offset[0]@0x0200;bytes[0x0280..0x0291)`|`offset=0,next_offset=17,UTF-8=/dataset/A.tsfile`|sid=0;StringOffsets\[0\.\.1\]|相邻两个 u32 offset 直接切出字符串;不保存结尾 NUL| +|`StringPool[1]`|/dataset/B\.tsfile
|`offset[1]@0x0204;bytes[0x0291..0x02A2)`|`offset=17,next_offset=34,UTF-8=/dataset/B.tsfile`|sid=1;StringOffsets\[1\.\.2\]|相邻两个 u32 offset 直接切出字符串;不保存结尾 NUL| +|`StringPool[2]`|/dataset/C\.tsfile
|`offset[2]@0x0208;bytes[0x02A2..0x02B3)`|`offset=34,next_offset=51,UTF-8=/dataset/C.tsfile`|sid=2;StringOffsets\[2\.\.3\]|相邻两个 u32 offset 直接切出字符串;不保存结尾 NUL| +|`StringPool[3]`|shared\_table
|`offset[3]@0x020C;bytes[0x02B3..0x02BF)`|`offset=51,next_offset=63,UTF-8=shared_table`|sid=3;StringOffsets\[3\.\.4\]|相邻两个 u32 offset 直接切出字符串;不保存结尾 NUL| +|`StringPool[4]`|table\_A
|`offset[4]@0x0210;bytes[0x02BF..0x02C6)`|`offset=63,next_offset=70,UTF-8=table_A`|sid=4;StringOffsets\[4\.\.5\]|相邻两个 u32 offset 直接切出字符串;不保存结尾 NUL| +|`StringPool[5]`|table\_B|`offset[5]@0x0214;bytes[0x02C6..0x02CD)`|`offset=70,next_offset=77,UTF-8=table_B`|sid=5;StringOffsets\[5\.\.6\]|相邻两个 u32 offset 直接切出字符串;不保存结尾 NUL| +|`StringPool[6]`|table\_C1|`offset[6]@0x0218;bytes[0x02CD..0x02D5)`|`offset=77,next_offset=85,UTF-8=table_C1`|sid=6;StringOffsets\[6\.\.7\]|相邻两个 u32 offset 直接切出字符串;不保存结尾 NUL| +|`StringPool[7]`|table\_C2|`offset[7]@0x021C;bytes[0x02D5..0x02DD)`|`offset=85,next_offset=93,UTF-8=table_C2`|sid=7;StringOffsets\[7\.\.8\]|相邻两个 u32 offset 直接切出字符串;不保存结尾 NUL| +|`StringPool[8]`|shared\_table/d0|`offset[8]@0x0220;bytes[0x02DD..0x02EC)`|`offset=93,next_offset=108,UTF-8=shared_table/d0`|sid=8;StringOffsets\[8\.\.9\]|相邻两个 u32 offset 直接切出字符串;不保存结尾 NUL| +|`StringPool[9]`|shared\_table/d1|`offset[9]@0x0224;bytes[0x02EC..0x02FB)`|`offset=108,next_offset=123,UTF-8=shared_table/d1`|sid=9;StringOffsets\[9\.\.10\]|相邻两个 u32 offset 直接切出字符串;不保存结尾 NUL| +|`StringPool[10]`|shared\_table/d2|`offset[10]@0x0228;bytes[0x02FB..0x030A)`|`offset=123,next_offset=138,UTF-8=shared_table/d2`|sid=10;StringOffsets\[10\.\.11\]|相邻两个 u32 offset 直接切出字符串;不保存结尾 NUL| +|`StringPool[11]`|table\_A/a0|`offset[11]@0x022C;bytes[0x030A..0x0314)`|`offset=138,next_offset=148,UTF-8=table_A/a0`|sid=11;StringOffsets\[11\.\.12\]|相邻两个 u32 offset 直接切出字符串;不保存结尾 NUL| +|`StringPool[12]`|table\_B/b0|`offset[12]@0x0230;bytes[0x0314..0x031E)`|`offset=148,next_offset=158,UTF-8=table_B/b0`|sid=12;StringOffsets\[12\.\.13\]|相邻两个 u32 offset 直接切出字符串;不保存结尾 NUL| +|`StringPool[13]`|table\_C1/c0|`offset[13]@0x0234;bytes[0x031E..0x0329)`|`offset=158,next_offset=169,UTF-8=table_C1/c0`|sid=13;StringOffsets\[13\.\.14\]|相邻两个 u32 offset 直接切出字符串;不保存结尾 NUL| +|`StringPool[14]`|table\_C2/c1|`offset[14]@0x0238;bytes[0x0329..0x0334)`|`offset=169,next_offset=180,UTF-8=table_C2/c1`|sid=14;StringOffsets\[14\.\.15\]|相邻两个 u32 offset 直接切出字符串;不保存结尾 NUL| +|`StringPool[15]`|s1
|`offset[15]@0x023C;bytes[0x0334..0x0336)`|`offset=180,next_offset=182,UTF-8=s1`|sid=15;StringOffsets\[15\.\.16\]|相邻两个 u32 offset 直接切出字符串;不保存结尾 NUL| +|`StringPool[16]`|s2|`offset[16]@0x0240;bytes[0x0336..0x0338)`|`offset=182,next_offset=184,UTF-8=s2`|sid=16;StringOffsets\[16\.\.17\]|相邻两个 u32 offset 直接切出字符串;不保存结尾 NUL| +|`StringPool[17]`|mA|`offset[17]@0x0244;bytes[0x0338..0x033A)`|`offset=184,next_offset=186,UTF-8=mA`|sid=17;StringOffsets\[17\.\.18\]|相邻两个 u32 offset 直接切出字符串;不保存结尾 NUL| +|`StringPool[18]`|mB|`offset[18]@0x0248;bytes[0x033A..0x033C)`|`offset=186,next_offset=188,UTF-8=mB`|sid=18;StringOffsets\[18\.\.19\]|相邻两个 u32 offset 直接切出字符串;不保存结尾 NUL| +|`StringPool[19]`|mC1|`offset[19]@0x024C;bytes[0x033C..0x033F)`|`offset=188,next_offset=191,UTF-8=mC1`|sid=19;StringOffsets\[19\.\.20\]|相邻两个 u32 offset 直接切出字符串;不保存结尾 NUL| +|`StringPool[20]`|mC2|`offset[20]@0x0250;bytes[0x033F..0x0342)`|`offset=191,next_offset=194,UTF-8=mC2`|sid=20;StringOffsets\[20\.\.21\]|相邻两个 u32 offset 直接切出字符串;不保存结尾 NUL| +|**TableNameIndex:每个 canonical table name / schema variant 一行**|||||| +|`TableNameIndex[0]`|shared\_table
|`0x0380 + 0×16 = 0x0380`|`name_hash=H(shared_table),name_sid=3,table_id=0`
|TableRecord\[0\]|示例行按名称展示;文件内实际按 \(name\_hash,name\_bytes\) 排序| +|`TableNameIndex[1]`|table\_A
|`0x0380 + 1×16 = 0x0390`|`name_hash=H(table_A),name_sid=4,table_id=1`
|TableRecord\[1\]|示例行按名称展示;文件内实际按 \(name\_hash,name\_bytes\) 排序| +|`TableNameIndex[2]`|table\_B
|`0x0380 + 2×16 = 0x03A0`|`name_hash=H(table_B),name_sid=5,table_id=2`
|TableRecord\[2\]|示例行按名称展示;文件内实际按 \(name\_hash,name\_bytes\) 排序| +|`TableNameIndex[3]`|table\_C1|`0x0380 + 3×16 = 0x03B0`|`name_hash=H(table_C1),name_sid=6,table_id=3`
|TableRecord\[3\]|示例行按名称展示;文件内实际按 \(name\_hash,name\_bytes\) 排序| +|`TableNameIndex[4]`|table\_C2
|`0x0380 + 4×16 = 0x03C0`|`name_hash=H(table_C2),name_sid=7,table_id=4`|TableRecord\[4\]
|示例行按名称展示;文件内实际按 \(name\_hash,name\_bytes\) 排序| +|**TableRecord:5 个 schema variant;shared\_table 跨 A、B 仍只保存一条逻辑记录**|||||| +|`TableRecord[0]`
|shared\_table
|`0x0400 + 0×48 = 0x0400`
|`name_sid=3, reserved0=0, schema_fingerprint=fp_shared, first_device_name_index=0, device_count=3, first_column_name_index=0, column_count=2, first_file_span=0, file_span_count=2, reserved1=0`|DeviceNameIndex\[0\.\.2\];ColumnNameIndex\[0\.\.1\];TableFileSpan\[0\.\.1\]|record ID 即 table\_id| +|`TableRecord[1]`|table\_A
|`0x0400 + 1×48 = 0x0430`|`name_sid=4, reserved0=0, schema_fingerprint=fp_table_A, first_device_name_index=3, device_count=1, first_column_name_index=2, column_count=1, first_file_span=2, file_span_count=1, reserved1=0`|DeviceNameIndex\[3\.\.3\];ColumnNameIndex\[2\.\.2\];TableFileSpan\[2\.\.2\]|record ID 即 table\_id| +|`TableRecord[2]`|table\_B|`0x0400 + 2×48 = 0x0460`|`name_sid=5, reserved0=0, schema_fingerprint=fp_table_B, first_device_name_index=4, device_count=1, first_column_name_index=3, column_count=1, first_file_span=3, file_span_count=1, reserved1=0`|DeviceNameIndex\[4\.\.4\];ColumnNameIndex\[3\.\.3\];TableFileSpan\[3\.\.3\]|record ID 即 table\_id| +|`TableRecord[3]`|table\_C1|`0x0400 + 3×48 = 0x0490`|`name_sid=6, reserved0=0, schema_fingerprint=fp_table_C1, first_device_name_index=5, device_count=1, first_column_name_index=4, column_count=1, first_file_span=4, file_span_count=1, reserved1=0`|DeviceNameIndex\[5\.\.5\];ColumnNameIndex\[4\.\.4\];TableFileSpan\[4\.\.4\]|record ID 即 table\_id| +|`TableRecord[4]`|table\_C2|`0x0400 + 4×48 = 0x04C0`|`name_sid=7, reserved0=0, schema_fingerprint=fp_table_C2, first_device_name_index=6, device_count=1, first_column_name_index=5, column_count=1, first_file_span=5, file_span_count=1, reserved1=0`|DeviceNameIndex\[6\.\.6\];ColumnNameIndex\[5\.\.5\];TableFileSpan\[5\.\.5\]|record ID 即 table\_id| +|**DeviceNameIndex:每个 table 内的 canonical device name 一行**|||||| +|`DeviceNameIndex[0]`|shared\_table/d0|`0x0500 + 0×24 = 0x0500`|`table_id=0, device_id=0, name_hash=H(shared_table/d0), name_sid=8, reserved=0`|DeviceRecord\[0\]|在 TableRecord 限定的连续区间内按 hash/name 二分| +|`DeviceNameIndex[1]`|shared\_table/d1|`0x0500 + 1×24 = 0x0518`|`table_id=0, device_id=1, name_hash=H(shared_table/d1), name_sid=9, reserved=0`|DeviceRecord\[1\]|在 TableRecord 限定的连续区间内按 hash/name 二分| +|`DeviceNameIndex[2]`|shared\_table/d2|`0x0500 + 2×24 = 0x0530`|`table_id=0, device_id=2, name_hash=H(shared_table/d2), name_sid=10, reserved=0`|DeviceRecord\[2\]|在 TableRecord 限定的连续区间内按 hash/name 二分| +|`DeviceNameIndex[3]`|table\_A/a0|`0x0500 + 3×24 = 0x0548`|`table_id=1, device_id=3, name_hash=H(table_A/a0), name_sid=11, reserved=0`|DeviceRecord\[3\]|在 TableRecord 限定的连续区间内按 hash/name 二分| +|`DeviceNameIndex[4]`|table\_B/b0|`0x0500 + 4×24 = 0x0560`|`table_id=2, device_id=4, name_hash=H(table_B/b0), name_sid=12, reserved=0`|DeviceRecord\[4\]|在 TableRecord 限定的连续区间内按 hash/name 二分| +|`DeviceNameIndex[5]`|table\_C1/c0|`0x0500 + 5×24 = 0x0578`|`table_id=3, device_id=5, name_hash=H(table_C1/c0), name_sid=13, reserved=0`|DeviceRecord\[5\]|在 TableRecord 限定的连续区间内按 hash/name 二分| +|`DeviceNameIndex[6]`|table\_C2/c1|`0x0500 + 6×24 = 0x0590`|`table_id=4, device_id=6, name_hash=H(table_C2/c1), name_sid=14, reserved=0`|DeviceRecord\[6\]|在 TableRecord 限定的连续区间内按 hash/name 二分| +|**DeviceRecord:7 个逻辑设备;d0 横跨 A、B 仍只保存一条逻辑记录**|||||| +|`DeviceRecord[0]`|shared\_table/d0
|`0x05C0 + 0×48 = 0x05C0`
|`table_id=0, name_sid=8, reserved0=0, reserved1=0, first_series_id=0, series_count=2, first_file_span=0, file_span_count=2, min_time=0, max_time=199`|LogicalSeries\[0\.\.1\];DeviceFileSpan\[0\.\.1\]
|record ID 即 device\_id| +|`DeviceRecord[1]`|shared\_table/d1|`0x05C0 + 1×48 = 0x05F0`|`table_id=0, name_sid=9, reserved0=0, reserved1=0, first_series_id=2, series_count=2, first_file_span=2, file_span_count=1, min_time=0, max_time=99`|LogicalSeries\[2\.\.3\];DeviceFileSpan\[2\.\.2\]|record ID 即 device\_id| +|`DeviceRecord[2]`|shared\_table/d2|`0x05C0 + 2×48 = 0x0620`|`table_id=0, name_sid=10, reserved0=0, reserved1=0, first_series_id=4, series_count=2, first_file_span=3, file_span_count=1, min_time=100, max_time=199`|LogicalSeries\[4\.\.5\];DeviceFileSpan\[3\.\.3\]|record ID 即 device\_id| +|`DeviceRecord[3]`|table\_A/a0|`0x05C0 + 3×48 = 0x0650`
|`table_id=1, name_sid=11, reserved0=0, reserved1=0, first_series_id=6, series_count=1, first_file_span=4, file_span_count=1, min_time=0, max_time=99`|LogicalSeries\[6\.\.6\];DeviceFileSpan\[4\.\.4\]|record ID 即 device\_id| +|`DeviceRecord[4]`|table\_B/b0|`0x05C0 + 4×48 = 0x0680`|`table_id=2, name_sid=12, reserved0=0, reserved1=0, first_series_id=7, series_count=1, first_file_span=5, file_span_count=1, min_time=0, max_time=99`|LogicalSeries\[7\.\.7\];DeviceFileSpan\[5\.\.5\]|record ID 即 device\_id| +|`DeviceRecord[5]`|table\_C1/c0|`0x05C0 + 5×48 = 0x06B0`|`table_id=3, name_sid=13, reserved0=0, reserved1=0, first_series_id=8, series_count=1, first_file_span=6, file_span_count=1, min_time=0, max_time=99`|LogicalSeries\[8\.\.8\];DeviceFileSpan\[6\.\.6\]|record ID 即 device\_id| +|`DeviceRecord[6]`|table\_C2/c1|`0x05C0 + 6×48 = 0x06E0`|`table_id=4, name_sid=14, reserved0=0, reserved1=0, first_series_id=9, series_count=1, first_file_span=7, file_span_count=1, min_time=0, max_time=99`|LogicalSeries\[9\.\.9\];DeviceFileSpan\[7\.\.7\]|record ID 即 device\_id| +|**ColumnNameIndex:每个 schema variant 内的 measurement name 一行**|||||| +|`ColumnNameIndex[0]`|shared\_table\.s1|`0x0740 + 0×24 = 0x0740`
|`table_id=0, column_id=0, name_hash=H(s1), name_sid=15, reserved=0`|ColumnSchema\[0\]|在 TableRecord 限定的连续区间内按 hash/name 二分| +|`ColumnNameIndex[1]`|shared\_table\.s2|`0x0740 + 1×24 = 0x0758`|`table_id=0, column_id=1, name_hash=H(s2), name_sid=16, reserved=0`|ColumnSchema\[1\]|在 TableRecord 限定的连续区间内按 hash/name 二分| +|`ColumnNameIndex[2]`|table\_A\.mA|`0x0740 + 2×24 = 0x0770`|`table_id=1, column_id=2, name_hash=H(mA), name_sid=17, reserved=0`|ColumnSchema\[2\]|在 TableRecord 限定的连续区间内按 hash/name 二分| +|`ColumnNameIndex[3]`|table\_B\.mB|`0x0740 + 3×24 = 0x0788`|`table_id=2, column_id=3, name_hash=H(mB), name_sid=18, reserved=0`|ColumnSchema\[3\]|在 TableRecord 限定的连续区间内按 hash/name 二分| +|`ColumnNameIndex[4]`|table\_C1\.mC1|`0x0740 + 4×24 = 0x07A0`|`table_id=3, column_id=4, name_hash=H(mC1), name_sid=19, reserved=0`|ColumnSchema\[4\]|在 TableRecord 限定的连续区间内按 hash/name 二分| +|`ColumnNameIndex[5]`|table\_C2\.mC2|`0x0740 + 5×24 = 0x07B8`|`table_id=4, column_id=5, name_hash=H(mC2), name_sid=20, reserved=0`|ColumnSchema\[5\]|在 TableRecord 限定的连续区间内按 hash/name 二分| +|**ColumnSchema:每个 canonical measurement 定义一行**|||||| +|`ColumnSchema[0]`|shared\_table\.s1|`0x0800 + 0×32 = 0x0800`|`table_id=0, name_sid=15, column_ordinal=0, logical_type=INT64, physical_type=INT64, encoding=INHERIT_TSFILE(0), compression=INHERIT_TSFILE(0), role=FIELD, nullable=0, reserved=0`|SeriesLocator 指向的 TsFile chunk metadata 提供实际 encoding/compression|record ID 即 column\_id;时间列为隐式列| +|`ColumnSchema[1]`|shared\_table\.s2|`0x0800 + 1×32 = 0x0820`|`table_id=0, name_sid=16, column_ordinal=1, logical_type=DOUBLE, physical_type=DOUBLE, encoding=INHERIT_TSFILE(0), compression=INHERIT_TSFILE(0), role=FIELD, nullable=0, reserved=0`|SeriesLocator 指向的 TsFile chunk metadata 提供实际 encoding/compression|record ID 即 column\_id;时间列为隐式列| +|`ColumnSchema[2]`|table\_A\.mA|`0x0800 + 2×32 = 0x0840`|`table_id=1, name_sid=17, column_ordinal=0, logical_type=INT64, physical_type=INT64, encoding=INHERIT_TSFILE(0), compression=INHERIT_TSFILE(0), role=FIELD, nullable=0, reserved=0`|SeriesLocator 指向的 TsFile chunk metadata 提供实际 encoding/compression|record ID 即 column\_id;时间列为隐式列| +|`ColumnSchema[3]`|table\_B\.mB|`0x0800 + 3×32 = 0x0860`|`table_id=2, name_sid=18, column_ordinal=0, logical_type=DOUBLE, physical_type=DOUBLE, encoding=INHERIT_TSFILE(0), compression=INHERIT_TSFILE(0), role=FIELD, nullable=0, reserved=0`|SeriesLocator 指向的 TsFile chunk metadata 提供实际 encoding/compression|record ID 即 column\_id;时间列为隐式列| +|`ColumnSchema[4]`|table\_C1\.mC1|`0x0800 + 4×32 = 0x0880`|`table_id=3, name_sid=19, column_ordinal=0, logical_type=INT64, physical_type=INT64, encoding=INHERIT_TSFILE(0), compression=INHERIT_TSFILE(0), role=FIELD, nullable=0, reserved=0`|SeriesLocator 指向的 TsFile chunk metadata 提供实际 encoding/compression|record ID 即 column\_id;时间列为隐式列| +|`ColumnSchema[5]`|table\_C2\.mC2|`0x0800 + 5×32 = 0x08A0`|`table_id=4, name_sid=20, column_ordinal=0, logical_type=DOUBLE, physical_type=DOUBLE, encoding=INHERIT_TSFILE(0), compression=INHERIT_TSFILE(0), role=FIELD, nullable=0, reserved=0`|SeriesLocator 指向的 TsFile chunk metadata 提供实际 encoding/compression|record ID 即 column\_id;时间列为隐式列| +|**LogicalSeries:每条真实存在的 device × measurement 一行**|||||| +|`LogicalSeries[0]`|d0/s1|`0x08C0 + 0×32 = 0x08C0`|`device_id=0, column_id=0, first_file_span=0, file_span_count=2, min_time=0, max_time=199`|SeriesFileSpan\[0\.\.1\]|按 \(device\_id,column\_id\) 聚簇;不存在的稀疏列不生成记录| +|`LogicalSeries[1]`|d0/s2|`0x08C0 + 1×32 = 0x08E0`|`device_id=0, column_id=1, first_file_span=2, file_span_count=2, min_time=0, max_time=199`|SeriesFileSpan\[2\.\.3\]|按 \(device\_id,column\_id\) 聚簇;不存在的稀疏列不生成记录| +|`LogicalSeries[2]`|d1/s1|`0x08C0 + 2×32 = 0x0900`|`device_id=1, column_id=0, first_file_span=4, file_span_count=1, min_time=0, max_time=99`|SeriesFileSpan\[4\.\.4\]|按 \(device\_id,column\_id\) 聚簇;不存在的稀疏列不生成记录| +|`LogicalSeries[3]`|d1/s2|`0x08C0 + 3×32 = 0x0920`|`device_id=1, column_id=1, first_file_span=5, file_span_count=1, min_time=0, max_time=99`|SeriesFileSpan\[5\.\.5\]|按 \(device\_id,column\_id\) 聚簇;不存在的稀疏列不生成记录| +|`LogicalSeries[4]`|d2/s1|`0x08C0 + 4×32 = 0x0940`|`device_id=2, column_id=0, first_file_span=6, file_span_count=1, min_time=100, max_time=199`|SeriesFileSpan\[6\.\.6\]|按 \(device\_id,column\_id\) 聚簇;不存在的稀疏列不生成记录| +|`LogicalSeries[5]`|d2/s2|`0x08C0 + 5×32 = 0x0960`|`device_id=2, column_id=1, first_file_span=7, file_span_count=1, min_time=100, max_time=199`|SeriesFileSpan\[7\.\.7\]|按 \(device\_id,column\_id\) 聚簇;不存在的稀疏列不生成记录| +|`LogicalSeries[6]`|a0/mA|`0x08C0 + 6×32 = 0x0980`|`device_id=3, column_id=2, first_file_span=8, file_span_count=1, min_time=0, max_time=99`|SeriesFileSpan\[8\.\.8\]|按 \(device\_id,column\_id\) 聚簇;不存在的稀疏列不生成记录| +|`LogicalSeries[7]`|b0/mB|`0x08C0 + 7×32 = 0x09A0`|`device_id=4, column_id=3, first_file_span=9, file_span_count=1, min_time=0, max_time=99`|SeriesFileSpan\[9\.\.9\]|按 \(device\_id,column\_id\) 聚簇;不存在的稀疏列不生成记录| +|`LogicalSeries[8]`|c0/mC1|`0x08C0 + 8×32 = 0x09C0`|`device_id=5, column_id=4, first_file_span=10, file_span_count=1, min_time=0, max_time=99`|SeriesFileSpan\[10\.\.10\]|按 \(device\_id,column\_id\) 聚簇;不存在的稀疏列不生成记录| +|`LogicalSeries[9]`|c1/mC2|`0x08C0 + 9×32 = 0x09E0`|`device_id=6, column_id=5, first_file_span=11, file_span_count=1, min_time=0, max_time=99`|SeriesFileSpan\[11\.\.11\]|按 \(device\_id,column\_id\) 聚簇;不存在的稀疏列不生成记录| +|**TsFileRecord:3 个物理文件;文件版本由 Reader 打开文件时校验**|||||| +|`TsFileRecord[0]`|A\.tsfile|`0x0A00 + 0×48 = 0x0A00`|`path_sid=0, reserved0=0, file_size=65536, min_time=0, max_time=99, file_fingerprint=0xA001A001A001A001, reserved1=0`|path=StringPool\[0\]|file\_id 由其 record ID 给出;size/fingerprint 为本例具体示例值| +|`TsFileRecord[1]`|B\.tsfile|`0x0A00 + 1×48 = 0x0A30`|`path_sid=1, reserved0=0, file_size=69632, min_time=100, max_time=199, file_fingerprint=0xB002B002B002B002, reserved1=0`|path=StringPool\[1\]|file\_id 由其 record ID 给出;size/fingerprint 为本例具体示例值| +|`TsFileRecord[2]`|C\.tsfile|`0x0A00 + 2×48 = 0x0A60`|`path_sid=2, reserved0=0, file_size=49152, min_time=0, max_time=99, file_fingerprint=0xC003C003C003C003, reserved1=0`|path=StringPool\[2\]|file\_id 由其 record ID 给出;size/fingerprint 为本例具体示例值| +|**TableFileSpan:每个 table × TsFile 物理出现一行**|||||| +|`TableFileSpan[0]`|shared\_table in A|`0x0AC0 + 0×32 = 0x0AC0`|`table_id=0, file_id=0, min_time=0, max_time=99, row_count=200`|TableRecord\[0\];TsFileRecord\[0\]|表级文件裁剪;row\_count 为该表各设备时间轴行数之和| +|`TableFileSpan[1]`|shared\_table in B|`0x0AC0 + 1×32 = 0x0AE0`|`table_id=0, file_id=1, min_time=100, max_time=199, row_count=200`|TableRecord\[0\];TsFileRecord\[1\]|表级文件裁剪;row\_count 为该表各设备时间轴行数之和| +|`TableFileSpan[2]`|table\_A in A|`0x0AC0 + 2×32 = 0x0B00`|`table_id=1, file_id=0, min_time=0, max_time=99, row_count=100`|TableRecord\[1\];TsFileRecord\[0\]|表级文件裁剪;row\_count 为该表各设备时间轴行数之和| +|`TableFileSpan[3]`|table\_B in B|`0x0AC0 + 3×32 = 0x0B20`|`table_id=2, file_id=1, min_time=0, max_time=99, row_count=100`|TableRecord\[2\];TsFileRecord\[1\]|表级文件裁剪;row\_count 为该表各设备时间轴行数之和| +|`TableFileSpan[4]`|table\_C1 in C|`0x0AC0 + 4×32 = 0x0B40`|`table_id=3, file_id=2, min_time=0, max_time=99, row_count=100`|TableRecord\[3\];TsFileRecord\[2\]|表级文件裁剪;row\_count 为该表各设备时间轴行数之和| +|`TableFileSpan[5]`|table\_C2 in C|`0x0AC0 + 5×32 = 0x0B60`|`table_id=4, file_id=2, min_time=0, max_time=99, row_count=100`|TableRecord\[4\];TsFileRecord\[2\]|表级文件裁剪;row\_count 为该表各设备时间轴行数之和| +|**DeviceFileSpan:每个 device × TsFile 物理出现一行**|||||| +|`DeviceFileSpan[0]`|d0 in A|`0x0B80 + 0×48 = 0x0B80`|`device_id=0, file_id=0, time_meta_offset=0, time_meta_length=0, layout=0(non-aligned), flags=0, min_time=0, max_time=99, row_count=100`|DeviceRecord\[0\];TsFileRecord\[0\]|设备级文件裁剪;row\_count 是时间轴行数,不是 measurement 非空点数| +|`DeviceFileSpan[1]`|d0 in B|`0x0B80 + 1×48 = 0x0BB0`|`device_id=0, file_id=1, time_meta_offset=0, time_meta_length=0, layout=0(non-aligned), flags=0, min_time=100, max_time=199, row_count=100`|DeviceRecord\[0\];TsFileRecord\[1\]|设备级文件裁剪;row\_count 是时间轴行数,不是 measurement 非空点数| +|`DeviceFileSpan[2]`|d1 in A|`0x0B80 + 2×48 = 0x0BE0`|`device_id=1, file_id=0, time_meta_offset=0, time_meta_length=0, layout=0(non-aligned), flags=0, min_time=0, max_time=99, row_count=100`|DeviceRecord\[1\];TsFileRecord\[0\]|设备级文件裁剪;row\_count 是时间轴行数,不是 measurement 非空点数| +|`DeviceFileSpan[3]`|d2 in B|`0x0B80 + 3×48 = 0x0C10`|`device_id=2, file_id=1, time_meta_offset=0, time_meta_length=0, layout=0(non-aligned), flags=0, min_time=100, max_time=199, row_count=100`|DeviceRecord\[2\];TsFileRecord\[1\]|设备级文件裁剪;row\_count 是时间轴行数,不是 measurement 非空点数| +|`DeviceFileSpan[4]`|a0 in A|`0x0B80 + 4×48 = 0x0C40`|`device_id=3, file_id=0, time_meta_offset=0, time_meta_length=0, layout=0(non-aligned), flags=0, min_time=0, max_time=99, row_count=100`|DeviceRecord\[3\];TsFileRecord\[0\]|设备级文件裁剪;row\_count 是时间轴行数,不是 measurement 非空点数| +|`DeviceFileSpan[5]`|b0 in B|`0x0B80 + 5×48 = 0x0C70`|`device_id=4, file_id=1, time_meta_offset=0, time_meta_length=0, layout=0(non-aligned), flags=0, min_time=0, max_time=99, row_count=100`|DeviceRecord\[4\];TsFileRecord\[1\]|设备级文件裁剪;row\_count 是时间轴行数,不是 measurement 非空点数| +|`DeviceFileSpan[6]`|c0 in C|`0x0B80 + 6×48 = 0x0CA0`|`device_id=5, file_id=2, time_meta_offset=0, time_meta_length=0, layout=0(non-aligned), flags=0, min_time=0, max_time=99, row_count=100`|DeviceRecord\[5\];TsFileRecord\[2\]|设备级文件裁剪;row\_count 是时间轴行数,不是 measurement 非空点数| +|`DeviceFileSpan[7]`|c1 in C|`0x0B80 + 7×48 = 0x0CD0`|`device_id=6, file_id=2, time_meta_offset=0, time_meta_length=0, layout=0(non-aligned), flags=0, min_time=0, max_time=99, row_count=100`|DeviceRecord\[6\];TsFileRecord\[2\]|设备级文件裁剪;row\_count 是时间轴行数,不是 measurement 非空点数| +|**SeriesFileSpan:每个 logical series × TsFile fragment 一行**|||||| +|`SeriesFileSpan[0]`|d0/s1 in A|`0x0D00 + 0×48 = 0x0D00`|`series_id=0, file_id=0, locator_id=0, reserved=0, min_time=0, max_time=99, prefix_max_time=99, point_count=100`|LogicalSeries\[0\];SeriesLocator\[0\]|按 \(series\_id,min\_time,file\_id\) 聚簇;prefix\_max\_time 支持重叠时间范围二分| +|`SeriesFileSpan[1]`|d0/s1 in B|`0x0D00 + 1×48 = 0x0D30`|`series_id=0, file_id=1, locator_id=1, reserved=0, min_time=100, max_time=199, prefix_max_time=199, point_count=100`|LogicalSeries\[0\];SeriesLocator\[1\]|按 \(series\_id,min\_time,file\_id\) 聚簇;prefix\_max\_time 支持重叠时间范围二分| +|`SeriesFileSpan[2]`|d0/s2 in A|`0x0D00 + 2×48 = 0x0D60`|`series_id=1, file_id=0, locator_id=2, reserved=0, min_time=0, max_time=99, prefix_max_time=99, point_count=100`|LogicalSeries\[1\];SeriesLocator\[2\]|按 \(series\_id,min\_time,file\_id\) 聚簇;prefix\_max\_time 支持重叠时间范围二分| +|`SeriesFileSpan[3]`|d0/s2 in B|`0x0D00 + 3×48 = 0x0D90`|`series_id=1, file_id=1, locator_id=3, reserved=0, min_time=100, max_time=199, prefix_max_time=199, point_count=100`|LogicalSeries\[1\];SeriesLocator\[3\]|按 \(series\_id,min\_time,file\_id\) 聚簇;prefix\_max\_time 支持重叠时间范围二分| +|`SeriesFileSpan[4]`|d1/s1 in A|`0x0D00 + 4×48 = 0x0DC0`|`series_id=2, file_id=0, locator_id=4, reserved=0, min_time=0, max_time=99, prefix_max_time=99, point_count=100`|LogicalSeries\[2\];SeriesLocator\[4\]|按 \(series\_id,min\_time,file\_id\) 聚簇;prefix\_max\_time 支持重叠时间范围二分| +|`SeriesFileSpan[5]`|d1/s2 in A|`0x0D00 + 5×48 = 0x0DF0`|`series_id=3, file_id=0, locator_id=5, reserved=0, min_time=0, max_time=99, prefix_max_time=99, point_count=100`|LogicalSeries\[3\];SeriesLocator\[5\]|按 \(series\_id,min\_time,file\_id\) 聚簇;prefix\_max\_time 支持重叠时间范围二分| +|`SeriesFileSpan[6]`|d2/s1 in B|`0x0D00 + 6×48 = 0x0E20`|`series_id=4, file_id=1, locator_id=6, reserved=0, min_time=100, max_time=199, prefix_max_time=199, point_count=100`|LogicalSeries\[4\];SeriesLocator\[6\]|按 \(series\_id,min\_time,file\_id\) 聚簇;prefix\_max\_time 支持重叠时间范围二分| +|`SeriesFileSpan[7]`|d2/s2 in B|`0x0D00 + 7×48 = 0x0E50`|`series_id=5, file_id=1, locator_id=7, reserved=0, min_time=100, max_time=199, prefix_max_time=199, point_count=100`|LogicalSeries\[5\];SeriesLocator\[7\]|按 \(series\_id,min\_time,file\_id\) 聚簇;prefix\_max\_time 支持重叠时间范围二分| +|`SeriesFileSpan[8]`|a0/mA in A|`0x0D00 + 8×48 = 0x0E80`|`series_id=6, file_id=0, locator_id=8, reserved=0, min_time=0, max_time=99, prefix_max_time=99, point_count=100`|LogicalSeries\[6\];SeriesLocator\[8\]|按 \(series\_id,min\_time,file\_id\) 聚簇;prefix\_max\_time 支持重叠时间范围二分| +|`SeriesFileSpan[9]`|b0/mB in B|`0x0D00 + 9×48 = 0x0EB0`|`series_id=7, file_id=1, locator_id=9, reserved=0, min_time=0, max_time=99, prefix_max_time=99, point_count=100`|LogicalSeries\[7\];SeriesLocator\[9\]|按 \(series\_id,min\_time,file\_id\) 聚簇;prefix\_max\_time 支持重叠时间范围二分| +|`SeriesFileSpan[10]`|c0/mC1 in C|`0x0D00 + 10×48 = 0x0EE0`|`series_id=8, file_id=2, locator_id=10, reserved=0, min_time=0, max_time=99, prefix_max_time=99, point_count=100`|LogicalSeries\[8\];SeriesLocator\[10\]|按 \(series\_id,min\_time,file\_id\) 聚簇;prefix\_max\_time 支持重叠时间范围二分| +|`SeriesFileSpan[11]`|c1/mC2 in C|`0x0D00 + 11×48 = 0x0F10`|`series_id=9, file_id=2, locator_id=11, reserved=0, min_time=0, max_time=99, prefix_max_time=99, point_count=100`|LogicalSeries\[9\];SeriesLocator\[11\]|按 \(series\_id,min\_time,file\_id\) 聚簇;prefix\_max\_time 支持重叠时间范围二分| +|**SeriesLocator:每个 physical fragment 的精确 TimeseriesMetadata range 一行**|||||| +|`SeriesLocator[0]`|d0/s1 in A|`0x0F40 + 0×24 = 0x0F40`
|`device_file_span_id=0, locator_kind=0(non-aligned), flags=1(CHUNK_COUNT_VALID), timeseries_meta_offset=0x1200, timeseries_meta_length=0x80, chunk_count=1`|SeriesFileSpan\[0\];TsFileRecord\[0\]|本例 offset 指向完整 TimeseriesMetadata;Runtime 从其中解析 ChunkMetadata,再按 offsetOfChunkHeader 读取实际 chunk
| +|`SeriesLocator[1]`|d0/s1 in B|`0x0F40 + 1×24 = 0x0F58`|`device_file_span_id=1, locator_kind=0(non-aligned), flags=1(CHUNK_COUNT_VALID), timeseries_meta_offset=0x1600, timeseries_meta_length=0x80, chunk_count=1`|SeriesFileSpan\[1\];TsFileRecord\[1\]|本例 offset 指向完整 TimeseriesMetadata;Runtime 从其中解析 ChunkMetadata,再按 offsetOfChunkHeader 读取实际 chunk| +|`SeriesLocator[2]`|d0/s2 in A
|`0x0F40 + 2×24 = 0x0F70`|`device_file_span_id=0, locator_kind=0(non-aligned), flags=1(CHUNK_COUNT_VALID), timeseries_meta_offset=0x1280, timeseries_meta_length=0x80, chunk_count=1`|SeriesFileSpan\[2\];TsFileRecord\[0\]|本例 offset 指向完整 TimeseriesMetadata;Runtime 从其中解析 ChunkMetadata,再按 offsetOfChunkHeader 读取实际 chunk| +|`SeriesLocator[3]`|d0/s2 in B|`0x0F40 + 3×24 = 0x0F88`|`device_file_span_id=1, locator_kind=0(non-aligned), flags=1(CHUNK_COUNT_VALID), timeseries_meta_offset=0x1680, timeseries_meta_length=0x80, chunk_count=1`|SeriesFileSpan\[3\];TsFileRecord\[1\]|本例 offset 指向完整 TimeseriesMetadata;Runtime 从其中解析 ChunkMetadata,再按 offsetOfChunkHeader 读取实际 chunk| +|`SeriesLocator[4]`|d1/s1 in A|`0x0F40 + 4×24 = 0x0FA0`|`device_file_span_id=2, locator_kind=0(non-aligned), flags=1(CHUNK_COUNT_VALID), timeseries_meta_offset=0x1300, timeseries_meta_length=0x80, chunk_count=1`|SeriesFileSpan\[4\];TsFileRecord\[0\]
|本例 offset 指向完整 TimeseriesMetadata;Runtime 从其中解析 ChunkMetadata,再按 offsetOfChunkHeader 读取实际 chunk| +|`SeriesLocator[5]`|d1/s2 in A|`0x0F40 + 5×24 = 0x0FB8`|`device_file_span_id=2, locator_kind=0(non-aligned), flags=1(CHUNK_COUNT_VALID), timeseries_meta_offset=0x1380, timeseries_meta_length=0x80, chunk_count=1`|SeriesFileSpan\[5\];TsFileRecord\[0\]|本例 offset 指向完整 TimeseriesMetadata;Runtime 从其中解析 ChunkMetadata,再按 offsetOfChunkHeader 读取实际 chunk| +|`SeriesLocator[6]`|d2/s1 in B|`0x0F40 + 6×24 = 0x0FD0`|`device_file_span_id=3, locator_kind=0(non-aligned), flags=1(CHUNK_COUNT_VALID), timeseries_meta_offset=0x1700, timeseries_meta_length=0x80, chunk_count=1`|SeriesFileSpan\[6\];TsFileRecord\[1\]|本例 offset 指向完整 TimeseriesMetadata;Runtime 从其中解析 ChunkMetadata,再按 offsetOfChunkHeader 读取实际 chunk| +|`SeriesLocator[7]`|d2/s2 in B|`0x0F40 + 7×24 = 0x0FE8`|`device_file_span_id=3, locator_kind=0(non-aligned), flags=1(CHUNK_COUNT_VALID), timeseries_meta_offset=0x1780, timeseries_meta_length=0x80, chunk_count=1`|SeriesFileSpan\[7\];TsFileRecord\[1\]|本例 offset 指向完整 TimeseriesMetadata;Runtime 从其中解析 ChunkMetadata,再按 offsetOfChunkHeader 读取实际 chunk| +|`SeriesLocator[8]`|a0/mA in A|`0x0F40 + 8×24 = 0x1000`|`device_file_span_id=4, locator_kind=0(non-aligned), flags=1(CHUNK_COUNT_VALID), timeseries_meta_offset=0x1400, timeseries_meta_length=0x80, chunk_count=1`|SeriesFileSpan\[8\];TsFileRecord\[0\]|本例 offset 指向完整 TimeseriesMetadata;Runtime 从其中解析 ChunkMetadata,再按 offsetOfChunkHeader 读取实际 chunk| +|`SeriesLocator[9]`|b0/mB in B|`0x0F40 + 9×24 = 0x1018`|`device_file_span_id=5, locator_kind=0(non-aligned), flags=1(CHUNK_COUNT_VALID), timeseries_meta_offset=0x1800, timeseries_meta_length=0x80, chunk_count=1`|SeriesFileSpan\[9\];TsFileRecord\[1\]|本例 offset 指向完整 TimeseriesMetadata;Runtime 从其中解析 ChunkMetadata,再按 offsetOfChunkHeader 读取实际 chunk| +|`SeriesLocator[10]`|c0/mC1 in C|`0x0F40 + 10×24 = 0x1030`|`device_file_span_id=6, locator_kind=0(non-aligned), flags=1(CHUNK_COUNT_VALID), timeseries_meta_offset=0x1000, timeseries_meta_length=0x80, chunk_count=1`|SeriesFileSpan\[10\];TsFileRecord\[2\]|本例 offset 指向完整 TimeseriesMetadata;Runtime 从其中解析 ChunkMetadata,再按 offsetOfChunkHeader 读取实际 chunk| +|`SeriesLocator[11]`|c1/mC2 in C|`0x0F40 + 11×24 = 0x1048`|`device_file_span_id=7, locator_kind=0(non-aligned), flags=1(CHUNK_COUNT_VALID), timeseries_meta_offset=0x1080, timeseries_meta_length=0x80, chunk_count=1`|SeriesFileSpan\[11\];TsFileRecord\[2\]|本例 offset 指向完整 TimeseriesMetadata;Runtime 从其中解析 ChunkMetadata,再按 offsetOfChunkHeader 读取实际 chunk| + +## Aligned 序列的物理定位示例 + +本节是独立的小型示例,只展示与 aligned 有关的记录。A\.tsfile 中的 shared\_table/d0 采用 aligned 布局,共享 100 行时间轴;s1:INT64 有 95 个非空 value,s2:DOUBLE 有 90 个非空 value。时间列在逻辑 ColumnSchema 中保持隐式,但在物理定位中必须由 DeviceFileSpan 保存一份共享 time TimeseriesMetadata range。以下 record ID 为本节局部示意,不改变上一节 non\-aligned 示例的 ID。 + +|索引记录|对象|实际保存值|作用| +|---|---|---|---| +|`DeviceFileSpan[0]`|d0 in A|`device_id=0, file_id=0, time_meta_offset=0x1100, time_meta_length=0x60, layout=1(aligned), flags=1(ALIGNED_PAIRING_VALIDATED), min_time=0, max_time=99, row_count=100`|共享时间轴只保存一次;s1、s2 的 locator 都引用本记录| +|`SeriesFileSpan[0]`|d0/s1 in A|`series_id=0, file_id=0, locator_id=0, min_time=0, max_time=99, prefix_max_time=99, point_count=95`|point\_count 是非空 value 数,不是 aligned 行数| +|`SeriesLocator[0]`|d0/s1 value metadata|`device_file_span_id=0, locator_kind=1(aligned-value), flags=1(CHUNK_COUNT_VALID), timeseries_meta_offset=0x1160, timeseries_meta_length=0x60, chunk_count=2`|与 DeviceFileSpan\[0\] 的 time metadata 组合| +|`SeriesFileSpan[1]`|d0/s2 in A|`series_id=1, file_id=0, locator_id=1, min_time=0, max_time=99, prefix_max_time=99, point_count=90`|同一时间轴可以对应不同非空 value 数| +|`SeriesLocator[1]`|d0/s2 value metadata|`device_file_span_id=0, locator_kind=1(aligned-value), flags=1(CHUNK_COUNT_VALID), timeseries_meta_offset=0x11C0, timeseries_meta_length=0x60, chunk_count=2`|与 s1 共享 DeviceFileSpan\[0\],不重复保存 time offset| + +|TsFile metadata 对象|精确字节范围|ChunkMetadata\[0\]|ChunkMetadata\[1\]|统计语义| +|---|---|---|---|---| +|time TimeseriesMetadata,measurement ID 为空,物理类型标记为 VECTOR|`[0x1100,0x1160)`|`offsetOfChunkHeader=0x2000, min=0, max=49, count=50`|`offsetOfChunkHeader=0x3000, min=50, max=99, count=50`|count 表示时间轴行数,共 100 行| +|s1 value TimeseriesMetadata,实际类型 INT64|`[0x1160,0x11C0)`|`offsetOfChunkHeader=0x2400, min=0, max=49, count=45`|`offsetOfChunkHeader=0x3400, min=50, max=99, count=50`|共 95 个非空 value;第一个 aligned chunk 有 5 个 null| +|s2 value TimeseriesMetadata,实际类型 DOUBLE|`[0x11C0,0x1220)`|`offsetOfChunkHeader=0x2800, min=0, max=49, count=50`|`offsetOfChunkHeader=0x3800, min=50, max=99, count=40`|共 90 个非空 value;第二个 aligned chunk 有 10 个 null| + +配对规则是 `timeChunk[i] + valueChunk[i]`。构建器必须验证 aligned value 的 `chunk_count` 与共享 time metadata 中的 chunk 数相等,且每个 ordinal 对应同一 aligned chunk group;验证成功才设置 `ALIGNED_PAIRING_VALIDATED`。当前版本遇到无法验证的旧文件或 schema 演进文件时应拒绝生成 fast\-path locator,而不能按 ordinal 猜测。`timeseries_meta_length` 是单条 TimeseriesMetadata 的精确长度,使用 u32;若单条记录达到 4 GiB,构建器必须报格式不支持。 + +|步骤|读取对象|动作|结果| +|---|---|---|---| +|1|`SeriesFileSpan[0] → SeriesLocator[0]`|定点得到 s1 value TimeseriesMetadata 的 `[0x1160,0x11C0)`|不遍历 TsFile measurement metadata index| +|2|`DeviceFileSpan[0]`|由 device\_file\_span\_id 得到共享 time TimeseriesMetadata 的 `[0x1100,0x1160)`|同一查询选择 s1、s2 时只读取或缓存一次 time metadata| +|3|time/value TimeseriesMetadata|分别反序列化两个 ChunkMetadata list,并校验数量均为 2|生成 `(0x2000,0x2400)`、`(0x3000,0x3400)` 两个 time/value chunk pair| +|4|AlignedChunkReader|先按 time statistics 做时间裁剪,再分别读取配对的 time/value chunk header 与 page;null bitmap 决定每行是否存在 s1 value|输出 100 行时间轴、95 个非空 s1 value;不会把 point\_count 当作 row\_count| + +ColumnSchema 仍只描述逻辑 measurement。s1 的 physical\_type 是 INT64,s2 是 DOUBLE;time chunk 的 VECTOR 只是 TsFile on\-wire 标记。encoding/compression 继续从 time/value ChunkHeader 分别读取。schema nullable 表示逻辑约束,某个 physical fragment 是否实际含 null 则由 time row\_count、value point\_count 和 value bitmap 共同确定。 + +**空间开销。** 相比旧布局,DeviceFileSpan 从 32 B 增加到 48 B,SeriesLocator 从 48 B 减少到 24 B;在不改变 SeriesFileSpan 的前提下,净变化为 `+16 × device-file 数量 − 24 × series-file 数量`。一个 device/file 包含多条 aligned measurement 时,共享 time metadata 只增加一次,因此 measurement 越多,节省越明显。 + +## 查询 shared\_table / d0 / s1,时间范围 \[80,120\] + +|步骤|读取对象|计算|本例结果| +|---|---|---|---| +|0|Header \+ SectionDirectory
|校验 magic/version/size/file\_length/CRC、每个 section 的边界、对齐、record\_size 和乘法溢出;按当前 section_count 建立 SectionView|Runtime 只常驻 mmap\_base 与当前目录对应的小型 view,不复制整份索引| +|1|TableNameIndex
|计算 hash\("shared\_table"\),在 TableNameIndex 中二分得到同名等值区间;比较 StringBytes 消除 hash 碰撞,再按可选 schema\_fingerprint 选择 variant|`table_id = 0`
| +|2|TableRecord\[0\] \+ DeviceNameIndex|在 `[first_device_name_index, +device_count)` 中二分 `d0`|`device_id = 0`| +|3|TableRecord\[0\] \+ ColumnNameIndex|在该表的列索引区间中二分 `s1`,读取 ColumnSchema
|`column_id = 0`,`column_ordinal = 0`,type=INT64| +|4|DeviceRecord\[0\] \+ LogicalSeries|在设备的 `[first_series_id, +series_count)` 连续区间内按 column\_id 二分;不为不存在的稀疏列生成 LogicalSeries|`series_id = 0`| +|5|LogicalSeries\[0\] \+ SeriesFileSpan|先用 series 全局 min/max 快速拒绝;在其连续 span 区间中二分首个 `prefix_max_time ≥ 80`,随后扫描到 `min_time > 120`|命中 span 0:A \[0,99\];span 1:B \[100,199\]
| +|6|SeriesLocator\[0\.\.1\] \+ DeviceFileSpan \+ TsFileRecord|由 locator\_id 取得精确 TimeseriesMetadata offset/length、locator\_kind 与 device\_file\_span\_id;由 DeviceFileSpan 取得 file\_id、layout 和可选共享 time metadata,再由 TsFileRecord 取得路径、file\_size 与 fingerprint|生成 A/B 两个 value metadata range;本例 locator\_kind=non\-aligned,因此不读取 time metadata range。Reader 打开文件时自行校验 TsFile format version| +|7|ReaderSessionPool \+ TsFileReader|复用受 FD 预算约束的 ReaderSession,先反序列化目标 TimeseriesMetadata 中的 ChunkMetadata,再按 offsetOfChunkHeader 读取相关 chunk/page,分别裁剪为 \[80,99\] 和 \[100,120\]|只触碰 A/B 中 d0/s1 对应的数据页| +|8|Merge operator
|按 timestamp 归并;当前格式不定义 file\_order,跨 TsFile 出现相同 timestamp 时返回重复时间戳错误,后续如需覆盖语义再以新策略字段扩展|输出有序、去重的 s1 列,再构造 DataFrame
| + +```text +dir = validate_header_and_directory(mmap_base) +table_ids = lookup_table_variants(dir.TABLE_NAME_INDEX, "shared_table") +table_id = select_schema_variant(table_ids, requested_schema_fingerprint) +table = record(dir.TABLE_RECORD, table_id) + +device_id = lookup_child_name(dir.DEVICE_NAME_INDEX, + table.first_device_name_index, + table.device_count, "d0") +column_id = lookup_child_name(dir.COLUMN_NAME_INDEX, + table.first_column_name_index, + table.column_count, "s1") +device = record(dir.DEVICE_RECORD, device_id) +series_id = lookup_series_by_column(dir.LOGICAL_SERIES, + device.first_series_id, + device.series_count, column_id) +series = record(dir.LOGICAL_SERIES, series_id) + +spans = slice(dir.SERIES_FILE_SPAN, + series.first_file_span, series.file_span_count) +i = lower_bound(spans.prefix_max_time, query_start) +while i < len(spans) and spans[i].min_time <= query_end: + if spans[i].max_time >= query_start: + locator = record(dir.SERIES_LOCATOR, spans[i].locator_id) + device_span = record(dir.DEVICE_FILE_SPAN, + locator.device_file_span_id) + require spans[i].file_id == device_span.file_id + reader = reader_pool.acquire(device_span.file_id) + value_meta = range(locator.timeseries_meta_offset, + locator.timeseries_meta_length) + + if locator.locator_kind == NON_ALIGNED: + yield reader.read_non_aligned(value_meta, + query_start, query_end) + else: + require device_span.layout == ALIGNED + require device_span.flags & ALIGNED_PAIRING_VALIDATED + time_meta = range(device_span.time_meta_offset, + device_span.time_meta_length) + require locator.flags & CHUNK_COUNT_VALID + require chunk_count(time_meta) == locator.chunk_count + yield reader.read_aligned(time_meta, value_meta, + query_start, query_end) + i += 1 +``` + +**内存与 mmap。** Runtime 可以 mmap 整个索引的虚拟地址空间,但启动阶段仅主动读取 64 B Header 和 448 B Directory。SectionView 只保存 `base pointer + count + record_size`;名称索引、逻辑记录和 span 均在查询时按需触页。名称 lookup 的结果可以做有界缓存,但不要求把字符串、schema 或所有 record 反序列化成堆对象。 + +## 实施与验收记录(2026-08-14) + +Format v1 已在 C++ 与 Python 两侧落地。C++ `dataset_index` 模块实现 14 个 section 的定长布局、CRC32C、边界/宽度/字符串引用/外键校验、只读 mmap、名称二分查找,以及临时文件写入、文件与目录 fsync、原子 rename。Python builder 使用同一布局,冷构建完成后由 `MappedDatasetIndex` 直接提供惰性 typed view;热启动不再恢复完整 Python metadata catalog。 + +构建器已从 Reader 导出的精确 TimeseriesMetadata byte range 生成 `DeviceFileSpan` 与 `SeriesLocator`。aligned locator 只有在 time/value chunk 数量一致时才设置配对有效标志;无法证明配对关系时拒绝发布 fast-path locator。文件 generation 使用 file size 与 mtime-ns 的稳定 fingerprint,映射和 Reader prepare 均会重新校验。 + +验收覆盖如下: + +- C++ format 聚焦测试覆盖原子写入/映射/查找、版本拒绝、Header 与 section checksum、非法 section 数和 CRC32C 标准向量。 +- Python format/runtime 测试覆盖布局、损坏 Header、真实 aligned locator、过期 fingerprint、range 越界、热启动不调用旧 metadata reader,以及 Reader/Prepared cache 行为。 +- Python 生成的真实大索引由 C++ parser 完整打开并校验通过:61 个 TsFile、1,792,328 条 LogicalSeries、索引长度 506,136,064 bytes;该跨语言校验耗时 4.66 秒。 +- `/data/lotsa/buildings_900k` 的冷构建耗时 205.13 秒、峰值 RSS 11,250,788 KiB。该峰值属于一次性 builder;稳定索引发布后的 DataFrame 热启动使用只读 mmap,不复制这套对象图。 diff --git "a/design/TsFile Reader \344\270\216 Prepared Query Context \350\256\276\350\256\241.md" "b/design/TsFile Reader \344\270\216 Prepared Query Context \350\256\276\350\256\241.md" new file mode 100644 index 000000000..fe3a8d346 --- /dev/null +++ "b/design/TsFile Reader \344\270\216 Prepared Query Context \350\256\276\350\256\241.md" @@ -0,0 +1,223 @@ + + +# TsFile Reader 与 Prepared Query Context 设计 + +## 1\. 文档边界 + +本文是 C++ Reader 与 Prepared Query Context 的权威设计。它定义单个 TsFile 内的定点 metadata 解析、PreparedSeries 所有权、prepared 查询接口、PagePositionIndex、查询局部对象和 ResultSet 生命周期。 + +相关文档: + +- [Dataset Index 文件格式与查询设计](<./Dataset Index 文件格式与查询设计.md>)定义 SeriesLocator、DeviceFileSpan、TsFileRecord 和物理字节范围。 +- [TsFileDataFrame Runtime 设计](<./TsFileDataFrame Runtime 设计.md>)定义 FileGeneration、ReaderSessionPool、PreparedSeriesCache、RuntimeLease/QueryLease 和跨文件执行边界。 +- [TsFileDataFrame 持久化 mmap 索引与进程本地 Runtime 技术设计](<./TsFileDataFrame 持久化 mmap 索引与进程本地 Runtime 技术设计.md>)定义总体目标和公开 API。 + +本文不定义 Dataset 路由、跨文件 merge、Runtime cache 淘汰策略、Python DataFrame 选择语义或持久化索引格式。 + +## 2\. 现有 Reader 边界 + +|层级|现有对象|职责与所有权| +|---|---|---| +|Cython|TsFileReaderPy|持有原生 TsFileReader;把 Python 参数转换为 C wrapper 调用;用 WeakSet 登记活动 ResultSetPy。| +|Cython|ResultSetPy|持有原生 ResultSet、metadata、TagFilterHandle 和 Reader 弱引用;close 后清空原生指针并标记失效。| +|C\+\+|TsFileReader|持有 ReadFile、TsFileExecutor、TableQueryExecutor 和 Reader 自有 PageArena。| +|C\+\+|TableResultSet|持有 TsBlockReader、RowIterator、当前 TsBlock、结果元数据和列读取器。| + +普通单文件查询继续沿用现有公开接口。prepared 接口只服务 DatasetRuntime,不改变公开的 TsFileReaderPy 查询签名和 ResultSetPy 返回类型。 + +## 3\. FileGeneration 与 locator 输入契约 + +Runtime 为每次 prepared 操作传入已经验证的 FileGeneration: + +```text +FileGeneration = ( + mapped_index_identity, + file_id, + file_size, + file_fingerprint, +) +``` + +ReaderSession 打开规范化 path 后复核 file\_size 与 file\_fingerprint。活动 Runtime 只访问静态 Dataset,不接受 TsFile 原地更新或 generation 热切换。generation 不匹配时必须在读取 locator 指向的字节前失败。 + +prepared locator 是从 Dataset Index 复制出的稳定值,不保存 mmap 指针: + +```text +PreparedLocator { + file_id + locator_id + layout + value_metadata_offset + value_metadata_length + optional time_metadata_offset + optional time_metadata_length + chunk_count_hint +} +``` + +- non\-aligned 序列只使用 value TimeseriesMetadata range。 +- aligned 序列同时使用 DeviceFileSpan 的共享 time range 和 SeriesLocator 的 value range。 +- Reader 在反序列化前校验 offset/length 不溢出且不超过当前 TsFile 长度。 +- aligned time/value metadata 必须属于同一 file、device 和已验证的 aligned group,并按 chunk ordinal 配对。 + +## 4\. PreparedSeries 所有权 + +Prepared Query Context 的具体实现类型是 PreparedSeries。它绑定 FileGeneration 和 locator,不绑定某个 Reader 实例。 + +```text +PreparedSeries { + generation: FileGeneration + locator: PreparedLocator + arena: owned PageArena + value_index: ITimeseriesIndex* allocated in arena + aligned_time_index: optional ITimeseriesIndex* allocated in arena + page_positions: optional PagePositionIndex + page_index_state: synchronization state +} +``` + +不变量: + +- PreparedSeries 独占其 PageArena;ITimeseriesIndex、ChunkMetadata 和相关字符串的生命周期不超过该 arena。 +- PreparedSeries 不保存指向 Reader 自有 arena、ReadFile、TsFileExecutor、TableQueryExecutor、decoder 或查询局部 SSI 的指针。 +- PreparedSeries 不缓存压缩块、解压页、TsBlock、value page 或查询结果。 +- ReaderSession 关闭不影响同一 FileGeneration 的 PreparedSeries;重新打开并验证 generation 后可以继续使用。 +- PreparedSeriesCache 是长期 owner;一次查询通过 shared ownership/lease 固定对象。本文不定义 cache 的 entry/byte 上限。 + +## 5\. 私有 prepared 接口 + +接口可以按现有 C wrapper/Cython 风格实现,下面的签名只冻结语义,不冻结最终命名: + +```cpp +int prepareSeriesFromLocator( + const FileGeneration& generation, + const PreparedLocator& locator, + std::shared_ptr& prepared); + +int queryPrepared( + const std::shared_ptr& prepared, + int64_t start_time, + int64_t end_time, + ResultSet*& result); + +int queryPreparedByRow( + const std::shared_ptr& prepared, + int64_t offset, + int64_t limit, + ResultSet*& result); +``` + +`prepareSeriesFromLocator` 使用当前 ReaderSession 的 ReadFile 读取精确 TimeseriesMetadata range,但把反序列化对象写入 PreparedSeries 自有 arena。`queryPrepared` 和 `queryPreparedByRow` 使用当前 ReaderSession 的 ReadFile/executor/decoder 创建本次查询的局部读取器和现有 ResultSet。 + +Runtime 可以保留按 device/measurement 的 `prepareSeries` 作为兼容 fallback;Dataset Index 路径以 locator 定点构建为主,不重新遍历文件级 metadata index。 + +## 6\. 查询局部状态与 ResultSet + +每次 prepared 查询独立创建可变状态: + +- SSI 或等价查询游标; +- 时间/行过滤条件; +- ChunkReader、PageReader 与 decoder; +- TsBlockReader、RowIterator 和当前 batch; +- ResultSet/TableResultSet。 + +PreparedSeries 只提供不可变索引和可选的增量 page locator,不共享扫描位置、decoder buffer 或 ResultSet 状态。 + +Dataset prepared 查询在 Runtime 层同时持有 QueryLease、ReaderSession lease 和 PreparedSeries lease。ResultSetPy 必须在释放 ReaderSession lease 前完整消费并关闭;随后依次释放 PreparedSeries lease、ReaderSession lease 和 QueryLease。ReaderSessionPool 不得回收仍有 active use 的 Reader。 + +## 7\. PagePositionIndex + +### 7\.1 用途 + +PagePositionIndex 只优化重复的 `queryPreparedByRow`。SeriesLocator/TimeseriesIndex 能定位序列和 chunk,但不能直接回答“全序列第 N 行位于哪个 page”;没有该索引时,每次位置查询都可能重复扫描相同 page header 前缀。 + +它不用于初始化 ReaderSession,不参与纯时间范围查询,也不是 page value cache。没有重复位置查询时可以一直为空。 + +### 7\.2 最小记录 + +```text +PageLocator { + row_begin + row_end // half-open [row_begin, row_end) + chunk_ordinal + chunk_header_offset + page_header_offset + page_data_offset + optional value_chunk_header_offset + optional value_page_header_offset + optional value_page_data_offset +} +``` + +- non\-aligned 序列的行数来自该 page 的时间值数量。 +- aligned 序列以 time page 的 row count 为准,并保存配对 value page 的位置;value point count 不能替代时间轴行数。 +- page offset 不能脱离 FileGeneration、行区间、chunk identity 和 aligned 配对信息单独使用。 + +### 7\.3 构建与查询 + +1. 首次位置查询从已有连续前缀末尾开始顺序读取 page header。 +2. 每发现一个完整 page,构造完整 PageLocator;只有校验通过后才追加。 +3. 目标行已被覆盖时,按 row\_begin 二分找到 PageLocator。 +4. 当前 ReaderSession 从记录的 chunk/page 位置创建查询局部 cursor,并只跳过 page 内的剩余行。 +5. 目标行尚未覆盖时继续扩展索引;失败时回退为普通扫描,不影响查询语义。 + +同一 PreparedSeries 的扩展由 entry 级互斥或 single writer 串行化。Reader 只向其他查询发布无空洞、不可变的完整前缀;失败时不得发布半条记录或推进 covered row 边界。 + +## 8\. Cython 与 C\+\+ 边界 + +- PreparedSeries、PageArena、ITimeseriesIndex 和 PagePositionIndex 的核心所有权在 C\+\+。 +- C wrapper 暴露不透明 PreparedSeries handle,以及 prepare/query/release 所需的窄接口。 +- Cython 负责 handle 的 Python 生命周期、错误码到现有异常的映射,并把 ResultSet 包装为现有 ResultSetPy。 +- Cython 不把 mmap 指针、Reader 内部指针或 C\+\+ arena 对象暴露给 Python。 +- 跨文件 DatasetMergeKernel 属于 Runtime/Cython 子文档边界,不放入单文件 Reader。 + +## 9\. 错误与关闭安全 + +|场景|行为| +|---|---| +|FileGeneration 与当前文件不匹配|在解释 locator 前返回 metadata/file identity 错误。| +|metadata range 越界、溢出或反序列化失败|不发布 PreparedSeries;返回现有 MetadataError/FileReadError 映射。| +|aligned time/value 配对不一致|拒绝 fast path,不按 ordinal 猜测。| +|PagePositionIndex 扩展失败|不发布不完整前缀;当前查询回退扫描或返回底层读取错误。| +|ReaderSession 被请求关闭但仍有活动 ResultSet|pool 不回收该 session;先等待 active use 归零。| +|PreparedSeriesCache/Runtime 最终关闭|新查询被 Runtime 拒绝;已有 lease 延迟对象实际释放。| + +## 10\. 验收 + +- locator prepare 不遍历文件级 measurement metadata index。 +- 同一 FileGeneration 的 ReaderSession 淘汰并重新打开后,已有 PreparedSeries 仍可查询。 +- PreparedSeries 销毁后,其独立 arena、TimeseriesIndex、ChunkMetadata 和 PagePositionIndex 全部释放。 +- 并发首次 prepare 由 Runtime cache single\-flight 去重;并发 PagePositionIndex 扩展不产生空洞或重复发布。 +- aligned row 查询以 time page 行数为准,并正确处理 value null bitmap。 +- ResultSet 消费期间 ReaderSession 与 PreparedSeries 均保持存活。 +- 普通单文件 TsFileReaderPy API 与异常类型保持兼容。 +- 基准分别记录 prepare 延迟、重复 row window 延迟、PagePositionIndex 容量和 Reader reopen 后的复用收益;v1 不以这些指标声明内存硬上限。 + +## 11\. 实施与验收记录(2026-08-14) + +C++ Reader 已记录每条 `TimeseriesIndex` 的精确 source offset/length,并在 C wrapper、Cython metadata 与 Dataset Index builder 之间完整传递。`prepare_series` 会先校验当前 `ReadFile` 的 FileGeneration 和 locator range,再只读取该段 bytes,在 `PreparedSeries` 自有 `PageArena` 中完整反序列化并要求无尾随数据。aligned 路径分别读取共享 time 与 value metadata,并校验 chunk cardinality 后构造 `AlignedTimeseriesIndex`;对象不引用 Reader arena、executor 或 decoder。 + +C wrapper 已提供不透明 PreparedSeries handle 的 prepare/query/free 窄接口,Cython `PreparedSeriesPy` 负责确定性释放。当前实现以同一个 prepared query 入口的 `start_time/end_time + offset/limit` 参数承载 `queryPrepared` 与 `queryPreparedByRow` 两种语义;每次调用仍创建独立 SSI、filter、decoder、TsBlock 和 ResultSet。Runtime 在 ResultSet 完整消费期间同时持有 QueryLease、ReaderSession active use 和 PreparedSeries strong reference。 + +`PagePositionIndex` 已实现线程安全的无空洞前缀发布、按 `row_begin` 二分和失败不推进边界,并通过顺序与 64 writer 并发测试。当前 v1 查询仍使用 SSI 已有的 chunk/page count 跳过逻辑,尚未把扫描得到的 page 物理 offset 回填到 `PagePositionIndex`;因此该可选索引在实际 benchmark 中保持为空,重复查询收益来自 PreparedSeries、Reader 与 OS page cache 复用,不能归因于 page locator。后续接入时不改变公开 API 或现有正确性路径。 + +验证结果:Prepared/PagePosition 聚焦 C++ 测试 2/2 通过;Dataset Index 聚焦及真实跨语言索引 C++ 测试 7/7 通过;Python 完整回归 181/181 通过。`buildings_900k` 首次 256 行 prepared 查询在 1/2/4 进程组中的中位数分别为 1.26/1.26/1.20 ms,同一 PreparedSeries 的 20 次重复读取中位数约 0.30 ms,最大组内 p95 为 0.345 ms。 diff --git "a/design/TsFileDataFrame 2880 \347\252\227\345\217\243\345\206\267\347\274\223\345\255\230\346\200\247\350\203\275\346\212\245\345\221\212.md" "b/design/TsFileDataFrame 2880 \347\252\227\345\217\243\345\206\267\347\274\223\345\255\230\346\200\247\350\203\275\346\212\245\345\221\212.md" new file mode 100644 index 000000000..3e216ad9f --- /dev/null +++ "b/design/TsFileDataFrame 2880 \347\252\227\345\217\243\345\206\267\347\274\223\345\255\230\346\200\247\350\203\275\346\212\245\345\221\212.md" @@ -0,0 +1,248 @@ + + +# TsFileDataFrame 2880 窗口冷缓存性能报告 + +日期:2026-08-14 + +## 1. 结论 + +这轮结果不能概括成单一的“新实现更快”。在每组测试前都清理 Linux OS page cache、随机读取 2880 行窗口时,新 Runtime 同时呈现出两个方向相反且都很明确的结果: + +- DataFrame 冷构造从 46.24–49.11 秒降至 0.54–0.55 秒,快 83.9–90.4 倍。 +- 随机查询后的总 PSS 从 5.50/10.92/21.77 GiB 降至 153.96/245.31/413.54 MiB,低 36.5–53.9 倍;4 进程 PSS 只占机器物理内存的 0.643%,旧实现为 34.666%。 +- 首次随机查询延迟回退 2.74–4.10 倍。 +- 后续随机查询 p50 回退 4.74–6.06 倍,聚合 QPS 下降 81.1%–82.8%。 + +因此,持久化 mmap 索引和进程本地 Runtime 已经实现预期的启动时间与多进程内存目标,但严格冷缓存下的按需查询路径存在稳定且显著的性能回退。当前结果不支持宣称“查询性能没有下降”。在完成冷路径定位和优化前,不建议把本实现按“查询性能无回退”验收。 + +## 2. 被测环境与数据规模 + +### 2.1 机器 + +|项目|值| +|---|---:| +|主机|`vm-127`(`192.168.99.110`)| +|CPU|32 vCPU,AMD EPYC 9J14,2 sockets × 16 cores| +|物理内存|67,417,718,784 bytes(约 62.79 GiB)| +|swap|4 GiB;所有正式测试中均为 0 使用| +|系统|Ubuntu/Linux 6.8.0-136-generic,x86_64| +|Python|3.13.14| + +### 2.2 数据 + +`/data/lotsa` 是多个独立 schema 数据集的集合,不能整体作为一个 DataFrame。本次继续选择其中最大的完整数据集 `/data/lotsa/buildings_900k`。 + +|项目|`/data/lotsa` 全部数据|本次数据集 `buildings_900k`| +|---|---:|---:| +|独立数据集目录|116|1| +|TsFile 数|647|61| +|TsFile bytes|555,096,347,047|65,856,302,628| +|TsFile GiB|516.97|61.33| +|逻辑序列数|不跨 schema 聚合|1,792,328| + +索引文件规模如下: + +|索引|bytes|MiB|占 TsFile 数据比例| +|---|---:|---:|---:| +|PyPI 2.4.0 `.tsfidx`|248,139,282|236.64|0.377%| +|新 Runtime `.tsidx`|506,136,064|482.69|0.769%| + +### 2.3 实现 + +|标识|实现| +|---|---| +|旧实现|隔离 venv 中通过 `pip install tsfile` 安装的 PyPI `tsfile==2.4.0`| +|新实现|`feature/tsfile-dataframe-runtime`,本地验收提交 `b4e61644e`,包版本 `2.3.2.dev0`| + +## 3. 测试方法 + +### 3.1 固定随机 workload + +随机种子为 `20260814`。为了让 2880 行窗口在两个实现上都可执行,先用 PyPI 2.4.0 从固定随机候选中筛选公共 workload: + +- 共检查 517 条不同的候选序列。 +- 接受 516 条能够完整返回 2880 行的序列。 +- 剔除 1 条;旧实现读取它时报告跨 shard 重复时间戳。 +- 516 条序列分给 4 个 worker,每个 worker 129 条:1 条用于首次查询,另外 128 条用于随机查询。 +- worker 之间没有重复序列。1 进程使用 worker 0;2 进程使用 worker 0–1;4 进程使用 worker 0–3,因此 1/2/4 进程 workload 是嵌套可比的。 +- 新旧实现及全部三轮测试的 worker workload SHA-256 完全一致。 + +4 个 worker 的 workload SHA-256 为: + +```text +worker 0: 7d86fb92d6f83c96d03886df8a7cbd159de1f2d833b2796564d0fe030c5b5d5b +worker 1: 553043633e37501a546ff4c07dd76ee667a59102dde12965bee9545f2b6ca190 +worker 2: 83755d3fb5daa66be368ac55deef0692ca6e8510585c3438842ba2355d18eacb +worker 3: d5667a7c4942469ad0e426ada42e2c0e60b39e2c0b029d87a9e0d627d00d89d9 +``` + +### 3.2 冷缓存与执行顺序 + +每一个“实现 × 进程数 × 轮次”都是独立测试组。每组启动 worker 之前,由 root 执行: + +```text +os.sync() +write 3 to /proc/sys/vm/drop_caches +``` + +共执行 18 个正式测试组:2 个实现 × 1/2/4 进程 × 3 轮。每次清理后 `/proc/meminfo` 中 `Cached` 为 97,536–98,536 KiB(95.25–96.23 MiB),所有组的 swap 使用量均为 0。 + +每组内部顺序为: + +1. 清理 OS cache。 +2. spawn 1/2/4 个 worker;每个 worker 在清理后才 import 被测包并构造 DataFrame。 +3. 所有 worker 构造完成后,采集查询前 RSS/PSS/USS/FD。 +4. barrier 同步后,每个 worker 查询自己的第一条随机序列。 +5. 再次 barrier 同步后,每个 worker依次查询另外 128 条不同的随机序列。 +6. 清理临时结果对象并执行 Python GC,在 DataFrame 和 Runtime 仍存活时采集查询后内存。 + +单次查询计时包含 `dataframe[position]`、读取 `series[:2880]` 和结果物化,不提前调用 `len(series)`,也不提前构造 PreparedSeries。全部首次查询和随机查询都实际返回 2880 行。 + +报告数值取三轮中位数。首次查询的 p50 是组内 worker 首次查询 p50,再取三轮中位数;随机查询分位数在每组的全部 `128 × 进程数` 次查询上计算,再取三轮中位数。聚合 QPS 使用“总查询数 / 最慢 worker 的纯查询 wall time”,不包含查询完成后的 Python GC。 + +## 4. 冷构造结果 + +|进程数|PyPI 2.4.0 构造 p50|新 Runtime 构造 p50|加速| +|---:|---:|---:|---:| +|1|46,238.57 ms|551.19 ms|83.89×| +|2|47,627.85 ms|548.70 ms|86.80×| +|4|49,111.47 ms|543.32 ms|90.39×| + +这里的“构造 p50”是每个并发组内 worker 构造延迟的中位数。旧实现把完整 metadata catalog 保存在每个进程的 Python heap;新实现打开并映射已有 `.tsidx`。两者构造语义不同,但都代表应用调用 `TsFileDataFrame(dataset)` 后可开始查询的实际成本。 + +## 5. 内存结果 + +下面是每个 worker 完成 1 次首次查询和 128 次随机查询后,DataFrame/Runtime 仍保持存活时的进程组总量。PSS 最接近该进程组对机器物理内存的实际占用,因此“物理内存占比”以 PSS 计算;RSS 会在不同进程中重复统计共享 file-backed 页,USS 只统计私有页。 + +|进程数|实现|总 RSS|总 PSS|总 USS|PSS/物理内存|总 FD| +|---:|---|---:|---:|---:|---:|---:| +|1|PyPI 2.4.0|5,634.98 MiB|5,626.21 MiB|5,621.88 MiB|8.751%|70| +|1|新 Runtime|162.67 MiB|153.96 MiB|149.63 MiB|0.239%|27| +|2|PyPI 2.4.0|11,254.58 MiB|11,190.89 MiB|11,141.01 MiB|17.406%|140| +|2|新 Runtime|322.94 MiB|245.31 MiB|181.30 MiB|0.382%|54| +|4|PyPI 2.4.0|22,463.25 MiB|22,288.24 MiB|22,235.87 MiB|34.666%|280| +|4|新 Runtime|642.50 MiB|413.54 MiB|326.59 MiB|0.643%|108| + +按 PSS 比较,新 Runtime 分别减少 36.54×、45.62×、53.90×;按 USS 比较,分别减少 37.57×、61.45×、68.08×。 + +新 Runtime 的查询前 PSS 为 106.31/157.82/258.57 MiB,随机查询后增至 153.96/245.31/413.54 MiB。这部分增长来自本次 129 条/worker 随机序列访问后仍存活的 Reader、PreparedSeries 和 file-backed working set。即便采用查询后的保守数字,4 进程仍只占 0.643% 物理内存。 + +## 6. 首次随机查询 + +首次查询发生在冷 cache 下构造 DataFrame 之后,且每个 worker 选择不同的随机序列。 + +|进程数|PyPI 2.4.0 p50|新 Runtime p50|新/旧|延迟变化| +|---:|---:|---:|---:|---:| +|1|4.170 ms|11.424 ms|2.74×|+174.0%| +|2|4.310 ms|13.724 ms|3.18×|+218.4%| +|4|4.259 ms|17.447 ms|4.10×|+309.6%| + +首次查询随新 Runtime 的并发进程数增加而变慢,1→4 进程从 11.424 ms 增至 17.447 ms;这符合多个进程同时从冷文件系统读取按需 metadata/data 页时的 I/O 竞争特征。 + +## 7. 随机查询 + +|进程数|实现|p50|p95|p99|max|聚合 QPS| +|---:|---|---:|---:|---:|---:|---:| +|1|PyPI 2.4.0|1.245 ms|1.462 ms|1.557 ms|1.584 ms|803.72| +|1|新 Runtime|7.540 ms|10.329 ms|11.675 ms|14.164 ms|138.10| +|2|PyPI 2.4.0|1.295 ms|1.567 ms|1.683 ms|1.878 ms|1,523.80| +|2|新 Runtime|7.829 ms|12.372 ms|14.142 ms|14.886 ms|263.77| +|4|PyPI 2.4.0|1.475 ms|2.193 ms|2.744 ms|3.218 ms|2,563.97| +|4|新 Runtime|6.993 ms|16.554 ms|18.576 ms|21.036 ms|483.72| + +随机查询 p50 的新/旧比值为 6.06×、6.04×、4.74×;聚合 QPS 分别下降 82.8%、82.7%、81.1%。 + +两个实现的并发扩展都正常:从 1 到 4 进程,旧实现 QPS 提升 3.19×,新 Runtime 提升 3.50×。问题不是进程并发失效,而是新 Runtime 每次冷随机访问的绝对延迟较高。 + +## 8. 结果解释 + +### 8.1 已验证的收益 + +- mmap Dataset Index 避免了每个进程各自反序列化并常驻完整 catalog。 +- 1/2/4 进程构造延迟没有随进程数放大,且保持在约 0.55 秒。 +- 4 进程完成 516 条不同序列的随机访问后,总 PSS 仍只有 413.54 MiB,而旧实现为 21.77 GiB。 +- 新 Runtime 的 1→4 进程吞吐扩展比例不低于旧实现。 + +### 8.2 冷查询回退的判断 + +结合当前实现结构,最可能的解释是:旧实现用约 5.5 GiB/进程的代价在 DataFrame 构造阶段把 metadata catalog 和 Reader 状态放入内存;新 Runtime 只 mmap 紧凑 Dataset Index,首次访问某条随机序列时仍需按需读取/解析 TsFile 中的 TimeseriesMetadata,并在没有真实 page locator 的情况下走按行查询 fallback。清理 OS page cache 后,这些按需读取直接暴露为随机 I/O 延迟。 + +这是基于架构与结果的推断,不是 I/O trace 的最终归因。已有 256 行、暖 OS cache 测试中,新 Runtime 的查询 p50/p95/p99 曾比 PyPI 快约 10%–14%,但由于窗口长度和 cache 条件都不同,不能用它抵消本报告的冷缓存结果。它只说明当前回退更可能集中在冷启动的 metadata/page 定位路径,而不是所有稳定态查询路径。 + +### 8.3 对使用场景的含义 + +- 对短生命周期、只做少量查询的任务,新 Runtime 仍可能更快完成:旧实现约 46–49 秒的构造时间远高于查询本身,新 Runtime 约 0.55 秒即可进入查询。 +- 对长生命周期、低内存要求的多进程服务,新 Runtime 的内存优势成立,但冷随机查询的延迟目标尚未满足。 +- 对 OS cache 稳定命中的服务,需要补一组同为 2880 行的暖缓存对照,不能直接沿用此前 256 行结果。 + +## 9. 建议的下一步 + +1. 在 PreparedSeries 首次查询路径增加分段计时或 I/O trace,区分 Dataset Index lookup、TimeseriesMetadata range read/parse、SSI/page 定位、page read/decompress 和结果 merge。 +2. 将实际 SSI 扫描结果发布到 `PagePositionIndex`,或把足够的 page locator 持久化进索引,避免 2880 行首次窗口走完整 fallback。 +3. 评估对 TimeseriesMetadata 和 page locator 的有界预取/缓存,继续保留 ReaderSessionPool 与 PreparedSeriesCache 的硬上限,不能用恢复 5.5 GiB/进程 catalog 的方式换回查询速度。 +4. 优化后严格复用本报告的 workload plan、2880 行窗口、三轮 cache drop 和 1/2/4 进程口径复测。 +5. 建议验收条件同时约束:4 进程 PSS 不超过 1 GiB,且冷随机 p50/p95 不劣于 PyPI 2.4.0;若二者不能同时达到,应在设计中明确可接受的内存—延迟 Pareto 点。 + +## 10. 复现与原始结果 + +基准脚本:`python/tests/benchmark_tsfile_dataframe_cold_cache.py` + +生成固定 workload plan: + +```bash +/root/tsfile_pypi_baseline_20260814/bin/python \ + /root/tsfile_dataframe_cold_cache_benchmark.py \ + /data/lotsa/buildings_900k \ + --sample-count 128 \ + --query-rows 2880 \ + --seed 20260814 \ + --plan-worker-count 4 \ + --generate-workload-plan /tmp/tsfile-dataframe-2880-workload-plan.json +``` + +正式运行参数的核心部分为: + +```bash +python /root/tsfile_dataframe_cold_cache_benchmark.py \ + /data/lotsa/buildings_900k \ + --processes 1,2,4 \ + --sample-count 128 \ + --query-rows 2880 \ + --seed 20260814 \ + --workload-plan /tmp/tsfile-dataframe-2880-workload-plan.json \ + --drop-os-cache \ + --output RESULT.json +``` + +远端原始文件: + +```text +/tmp/tsfile-dataframe-2880-workload-plan.json +/tmp/tsfile-cold-cache-pypi-2.4.0-2880.json +/tmp/tsfile-cold-cache-pypi-2.4.0-2880-trial2.json +/tmp/tsfile-cold-cache-pypi-2.4.0-2880-trial3.json +/tmp/tsfile-cold-cache-feature-2880.json +/tmp/tsfile-cold-cache-feature-2880-trial2.json +/tmp/tsfile-cold-cache-feature-2880-trial3.json +``` + +第一轮 JSON 由修正 QPS 字段之前的脚本生成,其逐查询延迟、worker `random_wall_ms`、内存和 cache 数据均有效。本报告对全部三轮统一使用 `query_count / max(worker.random_wall_ms)` 重新计算 QPS;第二、三轮脚本已直接输出该修正值。 diff --git "a/design/TsFileDataFrame Runtime \350\256\276\350\256\241.md" "b/design/TsFileDataFrame Runtime \350\256\276\350\256\241.md" new file mode 100644 index 000000000..9a65f8c4e --- /dev/null +++ "b/design/TsFileDataFrame Runtime \350\256\276\350\256\241.md" @@ -0,0 +1,260 @@ + + +# TsFileDataFrame Runtime 设计 + +## 1. 文档边界 + +本文是 DatasetRuntime、对象/查询 lease、ReaderSessionPool、PreparedSeriesCache 和跨文件 merge 的权威设计。 + +- [主技术设计](<./TsFileDataFrame 持久化 mmap 索引与进程本地 Runtime 技术设计.md>)只保留背景、目标、总体架构、跨文档契约、端到端流程与公开 API。 +- [Dataset Index 文件格式与查询设计](<./Dataset Index 文件格式与查询设计.md>)定义索引的字节布局、校验规则与版本演进。 +- [TsFile Reader 与 Prepared Query Context 设计](<./TsFile Reader 与 Prepared Query Context 设计.md>)定义 C++ Reader、PreparedSeries 表示、prepared 查询接口、PagePositionIndex、查询局部对象和 ResultSet 生命周期。 + +每次独立构造 TsFileDataFrame 都创建独立 Runtime;只有由该 DataFrame 派生的 view 与惰性对象共享它。Runtime 不提供进程级注册或跨独立 DataFrame 的资源仲裁,也不增加通用查询规划对象。 + +## 2. 现有实现边界 + +|层级|现有类型|接入后的职责| +|---|---|---| +|Python Dataset|TsFileDataFrame、Timeseries、AlignedTimeseries|保留现有查询语义与返回类型;对象只通过 lease 持有 Runtime。| +|Python 单文件访问|TsFileSeriesReader|保留单文件模型识别和普通查询能力;Dataset 快速路径由 Runtime 提供 locator。| +|Cython|TsFileReaderPy、ResultSetPy|持有原生 Reader/ResultSet,管理 Python 生命周期和异常映射。| +|C++|TsFileReader、TableResultSet|执行单文件读取;prepared 接口只增加定点准备与查询能力。| + +Runtime 是内部实现边界。除 Timeseries 增加显式生命周期方法外,TsFileDataFrame 构造参数、索引/loc 语义和结果对象类型保持不变。 + +## 3. Runtime 对象模型 + +### 3.1 FileGeneration + +Runtime 使用以下值标识索引映射中某个 TsFile 的不可变 generation: + +```text +FileGeneration = ( + mapped_index_identity, + file_id, + file_size, + file_fingerprint, +) +``` + +规范化 path 来自对应 TsFileRecord,但不单独充当 generation。当前数据集模型是静态的:活动 Runtime 不接受索引热切换、文件原地更新或 generation 切换。重新生成文件或索引后,调用者需要重新构造 TsFileDataFrame。 + +### 3.2 DatasetRuntime + +```text +DatasetRuntime { + mapped_index: MappedDatasetIndex + readers: ReaderSessionPool(max_open_files) + prepared: PreparedSeriesCache<(FileGeneration, locator_id), PreparedSeries> + merge_kernel: Cython DatasetMergeKernel + object_leases: count + query_leases: count + accepting_leases: bool +} +``` + +|成员|职责| +|---|---| +|MappedDatasetIndex|只读映射并校验 Dataset Index;按 id、span 和 locator 路由。| +|ReaderSessionPool|按 FileGeneration 管理已经打开的 ReaderSession,并严格执行 max_open_files。| +|PreparedSeriesCache|以 (FileGeneration, locator_id) 复用独立所有权的 PreparedSeries。| +|DatasetMergeKernel|在 Cython 中完成跨文件有序拼接、k-way merge、重复时间戳检查和多列对齐。| +|lease 状态|线性化对象关闭、查询开始和 Runtime 最终 teardown。| + +MappedDatasetIndex 不持有 TsFileReader。ReaderSessionPool 和 PreparedSeriesCache 也彼此独立:ReaderSession 被回收不会使 PreparedSeries 失效。 + +## 4. 对象 lease 与关闭语义 + +### 4.1 RuntimeLease + +以下仍可访问 Runtime 的对象各持有一个独立、幂等释放的 RuntimeLease: + +- 独立构造的 TsFileDataFrame; +- 由列表、切片或其他索引操作产生的 DataFrame view; +- 仍可发起惰性读取的 Timeseries。 + +完全物化的 AlignedTimeseries 不再访问 Runtime,因此不持有 RuntimeLease。未来若增加惰性 iterator,iterator 自身必须持有 RuntimeLease,或者确保每个已产出的惰性 Timeseries 各自持有 lease。 + +DataFrame.close() 只标记当前 handle 已关闭并释放它自己的 lease;它不能使其他 view 或 Timeseries 失效。Timeseries 提供公开、幂等的 close() 和上下文管理器: + +```python +with dataframe["root.sg.d.s"] as series: + values = series[:] +``` + +finalizer 只作为遗漏 close() 时的兜底,不能作为 Runtime 可确定关闭的主要机制。 + +### 4.2 QueryLease + +每次读取必须先从发起查询的 handle 获取 QueryLease。获取 QueryLease 与该 handle 的 close() 在同一状态锁下原子化: + +- close 先成功:后续查询被拒绝; +- 查询先取得 QueryLease:该查询允许完整结束,close 不提前释放其 Runtime 资源。 + +查询持有 QueryLease、ReaderSession lease 和 PreparedSeries lease,直到 ResultSet 被完整消费并关闭。释放顺序为 ResultSet、PreparedSeries lease、ReaderSession lease、QueryLease。 + +### 4.3 Runtime 最终 teardown + +最后一个 RuntimeLease 释放后,Runtime 停止接受新的对象 lease 和查询。已经取得的 QueryLease 继续完成;当活动 QueryLease 归零后,Runtime 按以下顺序关闭: + +1. 释放 PreparedSeriesCache 及其独立 arena; +2. 关闭 ReaderSessionPool 中的所有 Reader; +3. 解除 Dataset Index 映射。 + +每个 lease 和 close() 都必须幂等,异常路径不得重复减计数或提前 teardown。 + +## 5. ReaderSessionPool + +### 5.1 ReaderSession 的范围 + +ReaderSession 绑定一个 FileGeneration,并持有一次打开后的 FD、ReadFile、executor/decoder、Reader arena 和本次查询所需的执行状态。它不是只包装 FD,也不拥有 PreparedSeries 的长期内存。 + +### 5.2 acquire/release + +1. acquire(FileGeneration) 查找 generation 完全一致的现有 session;命中后增加 active use。 +2. 未命中时,pool 从 Dataset Index 解析 path,打开 TsFileReaderPy,并复核 file_size 与 file_fingerprint。 +3. 已打开 session 数达到 max_open_files 时,优先关闭 active use 为 0 的最旧空闲 session。 +4. 如果所有 session 都在使用,acquire 等待 release;任何时候都不得临时突破 max_open_files。 +5. release 把 active use 减一;归零的 session 进入空闲集合,供复用或回收。 + +ReaderSession 的创建和复用都必须校验 FileGeneration。校验失败直接拒绝当前查询,不尝试在活动 Runtime 内重建索引或切换文件 generation。 + +### 5.3 唯一的 v1 资源硬限制 + +v1 只确认 max_open_files 是 Runtime 的资源硬限制。它直接约束同时打开的 ReaderSession 数量。 + +除 max_open_files 外,本阶段不定义其他资源预算、进程级总 FD 配额或 DataFrame 内存上限。不同的独立 Runtime 不共享 Reader,也不做全局资源仲裁。 + +## 6. PreparedSeriesCache + +PreparedSeries 的内存表示和 Reader 接口由 [Reader 子文档](<./TsFile Reader 与 Prepared Query Context 设计.md>)定义。Runtime 只冻结以下缓存契约: + +- key 固定为 (FileGeneration, locator_id),不得以 Reader identity、path 或逻辑序列名替代; +- 同 key 的首次 prepare 使用 single-flight,避免并发重复构建; +- cache 是 PreparedSeries 的长期强 owner,查询通过 lease 临时固定 entry; +- PreparedSeries 自带 PageArena 和索引对象,不指向 Reader arena、ReadFile、executor 或 decoder; +- ReaderSession 淘汰或关闭不使 cache entry 失效; +- FileGeneration 不匹配时绝不复用。 + +v1 不定义 entry 数量上限、字节预算、LRU 淘汰或“超预算则不入 cache”的分支。已成功插入的 PreparedSeries 保留到 Runtime 最终 teardown。是否增加淘汰策略,必须以实现后的占用数据和 benchmark 为依据,另行设计。 + +## 7. Cython DatasetMergeKernel + +第一版跨文件 merge 实现在 typed Cython DatasetMergeKernel 中,不先增加 C++ 类或 C ABI。 + +实现约束: + +- 输入是单文件 ResultSet 已消费得到的 typed timestamp/value array parts; +- 使用 typed memoryview、NumPy C API、C scalar 和预分配输出; +- 可释放 GIL 的纯计算段使用 nogil; +- 已按时间排序且互不重叠的片段走连续复制快速路径; +- 存在重叠时执行有序 k-way merge,并检查同一逻辑序列的重复时间戳; +- loc 多字段结果按时间戳并集对齐,缺失值按现有 Dataset 语义填充; +- Python 层只负责 span 路由、公开结果对象构造和异常映射,不执行逐行 heap merge。 + +该内核不保存长期查询状态,也不承担跨文件 Planner 职责。只有 profiling 证明 Cython 仍是瓶颈时,才考虑把同一窄接口下沉到 C++。 + +## 8. 端到端流程 + +### 8.1 初始化 + +1. 按现有规则展开、排序并规范化输入 .tsfile 路径。 +2. 校验或构建 Dataset Index;构建流程完成临时文件校验、fsync 和原子发布。 +3. 只读映射索引并建立 typed view。 +4. 创建独立 DatasetRuntime、ReaderSessionPool、空 PreparedSeriesCache 和初始 RuntimeLease。 +5. 构造完成后开放查询;初始化期间不允许数据访问。 + +初始化不预先打开所有 Reader,也不预先构建 PreparedSeries。 + +### 8.2 按时间读取单个逻辑序列 + +1. handle 原子获取 QueryLease。 +2. MappedDatasetIndex 根据逻辑序列和时间范围定位 SeriesFileSpan。 +3. 对每个 span 获取并校验 ReaderSession。 +4. 以 (FileGeneration, locator_id) 获取或构建 PreparedSeries。 +5. ReaderSession 执行 queryPrepared,完整消费并关闭 ResultSet。 +6. DatasetMergeKernel 拼接或归并各文件片段。 +7. 构造现有 Timeseries/NumPy/pandas 结果并释放各级 lease。 + +### 8.3 按行读取 + +流程与按时间读取相同,但单文件执行使用 queryPreparedByRow。可选 PagePositionIndex 由 PreparedSeries 持有,并在重复按行查询时增量扩展;具体记录和并发规则见 Reader 子文档。 + +### 8.4 loc 多字段读取 + +Runtime 为各字段分别定位 spans 和 PreparedSeries,读出 typed parts 后交给 DatasetMergeKernel 做时间戳并集对齐,最后构造现有 AlignedTimeseries 或 pandas.DataFrame。完全物化的返回对象不持有 RuntimeLease。 + +## 9. 公开 API + +TsFileDataFrame 构造函数继续只接收现有 paths 和 show_progress: + +```python +df = TsFileDataFrame(paths, show_progress=False) +``` + +不增加 memory budget、prepared cache budget、schema fingerprint、runtime reuse 或 index view 参数。索引、loc、list_timeseries 和 list_timeseries_metadata 的公开语义保持不变。 + +生命周期 API 的唯一新增项是惰性 Timeseries 的幂等 close() 与上下文管理器。DataFrame/view 原有 close 语义改为“只关闭当前 handle”;其他持有独立 RuntimeLease 的对象继续可用。 + +## 10. 错误与安全约束 + +|场景|行为| +|---|---| +|索引映射或 section 校验失败|仅在初始化阶段进入重建;重建失败则构造失败。| +|打开 Reader 时 generation 不匹配|拒绝查询;不热切换、不原地重建。| +|达到 max_open_files 且全部 Reader 活跃|等待 release;不突破上限。| +|同 key 并发 prepare|single-flight;等待同一次构建结果。| +|prepare 失败|不发布半初始化 entry;错误返回所有等待者。| +|ResultSet 尚未消费完|ReaderSession 与 PreparedSeries lease 继续存活。| +|handle 已 close|拒绝新查询;其他 handle 不受影响。| +|Runtime 正在 teardown|拒绝新 lease;已开始查询继续完成。| +|重复时间戳或非法对齐关系|终止当前查询并映射为现有 Python 异常。| + +## 11. 验收要点 + +- 两次独立构造的 TsFileDataFrame 拥有两个互不共享的 Runtime。 +- 关闭源 DataFrame 后,仍持 lease 的 view 或 Timeseries 可以继续读取。 +- Timeseries.close() 和上下文管理器能够确定性释放最后一个对象 lease。 +- ReaderSessionPool 在压力下从不超过 max_open_files;无空闲 Reader 时等待。 +- ReaderSession 被回收后,同 FileGeneration 的 PreparedSeries 仍可由新 session 复用。 +- PreparedSeriesCache v1 没有虚构的 entry/bytes 预算或 LRU 语义。 +- 跨文件 merge 首版位于 typed Cython,Python 不做逐行归并。 +- 最后一个对象 lease 和所有 QueryLease 结束后,Reader、PreparedSeries 与 mmap 均被释放。 + +## 12. 实施与验收记录(2026-08-14) + +Runtime 主链路已经实现:每次独立 `TsFileDataFrame` 构造创建独立 `DatasetRuntime`;Runtime 持有只读 `MappedDatasetIndex`、严格 `max_open_files` 的 LRU `ReaderSessionPool`、single-flight `PreparedSeriesCache` 和对象/查询 lease 状态。DataFrame view 与惰性 Timeseries 各自克隆 `RuntimeLease`,任一 handle 的 `close()` 只释放自身;最后一个对象 lease 结束后拒绝新查询,等待活动 `QueryLease` 归零,再按 Prepared cache、Reader pool、mmap 的顺序 teardown。 + +跨文件不重叠片段走 typed Cython 连续复制,重叠片段走 typed k-way merge 并检查重复时间戳。Reader 在命中 span 后才打开,prepared entry 由 `(mapped index identity, FileGeneration, locator_id)` 唯一确定;Reader 被 LRU 回收不会使其独立 arena 中的 PreparedSeries 失效。公开构造和选择 API 保持不变。 + +本地回归结果为 Python `181 passed`;其中包含 DataFrame/view/Timeseries 生命周期、Reader 数量硬上限、prepared cache 复用、过期 generation、非法 locator、aligned exact range、跨文件 merge 和原有 Dataset API 回归。正式 C++ Release shared library 在 Snappy、LZ4、LZO、Zlib 全启用配置下构建通过。 + +远端 `buildings_900k` 热启动验收如下。机器为 32 CPU、62 GiB RAM、Linux 6.8,数据集为 61 个文件、65,856,302,628 bytes,索引为 506,136,064 bytes。每个 worker 构造独立 Runtime,读取同一只读 mmap,随后读取首条序列的 256 行并保留一个 PreparedSeries;内存来自 `/proc//smaps_rollup`。 + +|进程数|构造中位数|首次查询中位数|缓存后 20 次查询中位数|总 RSS|总 PSS|总 USS|总 open FD| +|---:|---:|---:|---:|---:|---:|---:|---:| +|1|529.26 ms|1.26 ms|0.300 ms|116.64 MiB|84.95 MiB|57.98 MiB|12| +|2|527.26 ms|1.26 ms|0.302 ms|233.01 MiB|144.19 MiB|99.45 MiB|24| +|4|523.28 ms|1.20 ms|0.300 ms|465.93 MiB|251.13 MiB|198.09 MiB|48| + +这些数据表明 483 MiB 的索引映射没有按进程复制为等量私有内存;4 进程总 USS 约 198 MiB。RSS 会把共享 file-backed 页在每个进程中重复计数,因此跨进程容量判断应优先看 PSS 与 USS。上述数值是观测结果,不改变“只有 `max_open_files` 是 v1 硬约束”的设计结论。 diff --git "a/design/TsFileDataFrame \345\256\236\347\216\260\343\200\201\346\265\213\350\257\225\344\270\216\346\200\247\350\203\275\351\252\214\346\224\266\346\212\245\345\221\212.md" "b/design/TsFileDataFrame \345\256\236\347\216\260\343\200\201\346\265\213\350\257\225\344\270\216\346\200\247\350\203\275\351\252\214\346\224\266\346\212\245\345\221\212.md" new file mode 100644 index 000000000..4b0960f63 --- /dev/null +++ "b/design/TsFileDataFrame \345\256\236\347\216\260\343\200\201\346\265\213\350\257\225\344\270\216\346\200\247\350\203\275\351\252\214\346\224\266\346\212\245\345\221\212.md" @@ -0,0 +1,158 @@ + + +# TsFileDataFrame 实现、测试与性能验收报告 + +## 1. 范围与版本 + +本次实现基于 Apache `origin/develop` 的 `5681978b84a0d9ee26e1eab94f78afaccf93607a`,独立 worktree 为 `/Users/colin/dev/tsfile/worktrees/tsfile-dataframe-runtime`,分支为 `feature/tsfile-dataframe-runtime`。 + +|提交|内容| +|---|---| +|`506f2a8bb`|加入并冻结 Format、Runtime、Reader/Prepared 与主设计四份文档| +|`a03a7f1d3`|实现 Dataset Index、C++ prepared path、进程本地 Runtime、Cython merge、Python 生命周期与测试| + +权威设计仍分别位于以下四份文档: + +- [Dataset Index 文件格式与查询设计](<./Dataset Index 文件格式与查询设计.md>) +- [TsFileDataFrame Runtime 设计](<./TsFileDataFrame Runtime 设计.md>) +- [TsFile Reader 与 Prepared Query Context 设计](<./TsFile Reader 与 Prepared Query Context 设计.md>) +- [TsFileDataFrame 持久化 mmap 索引与进程本地 Runtime 技术设计](<./TsFileDataFrame 持久化 mmap 索引与进程本地 Runtime 技术设计.md>) + +## 2. 实现清单 + +### 2.1 Dataset Index + +- C++ 与 Python 共用 Format v1 的 64-byte Header、14 个 SectionDirectory entry 和 14 个业务 section。 +- 实现 CRC32C、record width/count、offset/length 溢出、字符串引用、外键、连续 span 与名称索引校验。 +- Builder 写入同目录临时文件,完成校验和 fsync 后原子 rename;Reader 使用只读 mmap。 +- Runtime 通过 typed view 懒读取表、设备、列、逻辑序列、文件 span 和精确 metadata locator,不恢复完整 Python metadata tree。 + +### 2.2 C++ Reader 与 Prepared Query Context + +- `TimeseriesIndex` 记录其在 TsFile 中的精确 source offset/length。 +- `prepare_series` 校验 FileGeneration 和 range,在 PreparedSeries 独立 `PageArena` 中定点反序列化 metadata。 +- aligned time/value metadata 独立读取并校验 chunk cardinality;普通与 aligned 查询均复用现有 SSI/ChunkReader/ResultSet。 +- C wrapper 与 Cython 提供不透明 PreparedSeries handle 的 prepare/query/free 生命周期。 +- `PagePositionIndex` 已实现并发无空洞发布与二分查找,但实际 SSI page offset 尚未回填,按行查询仍走正确性 fallback;2880 行冷缓存结果表明该缺口需要作为性能验收项继续处理。 + +### 2.3 Runtime 与 Python + +- 每次独立 DataFrame 构造创建独立 DatasetRuntime;不同进程和不同 DataFrame 不共享 Python/C++ 对象。 +- ReaderSessionPool 严格执行 `TSFILE_DATAFRAME_MAX_OPEN_FILES`,只淘汰无 active use 的 Reader。 +- PreparedSeriesCache 以映射 identity、FileGeneration 和 locator 唯一确定 entry,并对首次 prepare 做 single-flight。 +- DataFrame view 与惰性 Timeseries 各自持有 RuntimeLease;close 只关闭当前 handle,最后一个对象 lease 才触发 teardown。 +- 重叠跨文件 merge 使用 typed Cython kernel;重复 timestamp 保持抛错语义。 + +## 3. 测试结果 + +|层次|配置与覆盖|结果| +|---|---|---| +|C++ 正式构建与聚合回归|Release shared library;Snappy、LZ4、LZO、Zlib enabled;ANTLR4 disabled|127 suites、758 tests:757 passed、1 个外部数据可选测试 skipped| +|Dataset Index C++|写入/mmap/lookup、版本与 checksum 拒绝、CRC32C、真实 Python 大索引跨语言打开|远端 7/7 通过;真实索引全校验 4.66 秒| +|Prepared C++|PagePositionIndex 顺序发布、失败不推进、64 writer 并发无空洞发布|2/2 通过| +|Python 完整回归|原有 Reader/Writer/Dataset API,加 format/runtime/lifecycle/stale locator/cache/merge 测试|181/181 通过| +|远端 smoke|4 个 TsFile、1.50 GB、145,063 条逻辑序列,冷构建及 1/2 进程查询|通过;构建 14.43 秒| + +聚合回归中唯一 skipped 的用例是未设置 `TSFILE_DATASET_INDEX_TEST_PATH` 的外部索引测试;随后以 `buildings_900k/.tsfile_dataframe_index.tsidx` 设置该变量单独执行,7/7 全部通过。 + +## 4. 大数据集基准 + +### 4.1 数据与环境 + +`/data/lotsa` 是 116 个独立数据集目录的集合,共 647 个 TsFile、约 519 GiB。不同目录可能属于不同 schema,因此本次不把整个目录误当成一个 DataFrame,而选择其中最大的完整数据集: + +|项目|值| +|---|---:| +|数据集|`/data/lotsa/buildings_900k`| +|TsFile 数|61| +|TsFile 总 bytes|65,856,302,628| +|逻辑序列数|1,792,328| +|机器|32 CPU,62 GiB RAM,4 GiB swap| +|系统|Ubuntu/Linux 6.8.0-136-generic,x86_64| +|Python|3.13.14| + +基准脚本为 `python/tests/benchmark_tsfile_dataframe_runtime.py`。内存取自 `/proc//smaps_rollup`;USS 定义为 `Private_Clean + Private_Dirty`。热测没有 drop 全机 page cache,结果明确代表稳定索引和暖 OS cache 场景。 + +### 4.2 冷构建 + +|指标|结果| +|---|---:| +|冷构建 wall time|205.13 秒| +|索引长度|506,136,064 bytes(约 482.69 MiB)| +|索引/原数据比例|约 0.77%| +|峰值 RSS|11,250,788 KiB(约 10.73 GiB)| +|swap|0| + +冷构建峰值包含 61 个文件的旧 metadata catalog bridge、全局排序、section bytes 与最终校验。它是一次性构建成本,不代表热启动进程的常驻内存。 + +### 4.3 1/2/4 进程热启动与查询 + +> 本节保留的是早期 256 行、首条序列、暖 OS cache 微基准,只能说明稳定热路径和 mmap 内存共享,不能代表随机序列或冷缓存查询性能。严格冷缓存的 2880 行结果见下一节;涉及“查询是否回退”的判断以后者为准。 + +每个 worker 独立构造 DataFrame,映射同一索引,选择首条逻辑序列,执行一次 256 行查询,再在相同 PreparedSeries 上重复 20 次。采样发生在 DataFrame、Reader 和一个 PreparedSeries 保持存活时。 + +|进程数|组 ready|DataFrame 构造中位数|首次查询中位数|重复查询中位数|重复查询最大 p95|总 RSS|总 PSS|总 USS|总 FD| +|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:| +|1|815.01 ms|529.26 ms|1.261 ms|0.300 ms|0.337 ms|116.64 MiB|84.95 MiB|57.98 MiB|12| +|2|867.98 ms|527.26 ms|1.261 ms|0.302 ms|0.331 ms|233.01 MiB|144.19 MiB|99.45 MiB|24| +|4|923.49 ms|523.28 ms|1.204 ms|0.300 ms|0.345 ms|465.93 MiB|251.13 MiB|198.09 MiB|48| + +结论: + +- DataFrame 热构造保持在约 0.53 秒,2/4 进程并发没有明显延迟放大。 +- 483 MiB 索引没有按进程复制成同等私有内存;4 进程总 USS 约 198 MiB,约 49.5 MiB/进程。 +- RSS 会在每个进程中重复计入共享 file-backed 页;评估机器实际容量应使用 PSS/USS,而不是简单相加 RSS。 +- 首次查询包含 Reader open、generation 校验和 PreparedSeries 构建;随后 256 行窗口约 0.30 ms。 +- 总 FD 包含 Python multiprocessing pipe、stdio、mmap file 和 Reader FD,不等同于 ReaderSession 数;Reader 硬上限由独立压力测试验证。 + +### 4.4 2880 行随机序列冷缓存对照 + +2026-08-14 补充了 PyPI 2.4.0 与新 Runtime 的严格冷缓存对照。每个实现的 1/2/4 进程分别运行三轮;每组启动前执行 `sync` 和 `drop_caches=3`,并复用同一份固定随机 workload。全部查询均返回 2880 行。 + +三轮中位数显示: + +- 新 Runtime 冷构造快 83.9–90.4 倍。 +- 查询后 PSS 低 36.5–53.9 倍;4 进程 PSS 为 413.54 MiB,对比旧实现 21.77 GiB。 +- 首次随机查询慢 2.74–4.10 倍。 +- 后续随机查询 p50 慢 4.74–6.06 倍,聚合 QPS 低 81.1%–82.8%。 + +完整方法、数据表、workload hash、结果解释和原始 JSON 路径见 [TsFileDataFrame 2880 窗口冷缓存性能报告]()。这一结果表明内存与构造目标已经达到,但冷随机查询性能尚未通过无回退验收。 + +## 5. 复现方式 + +在已构建的 Python 包目录运行: + +```bash +python tests/benchmark_tsfile_dataframe_runtime.py \ + /data/lotsa/buildings_900k \ + --mode multiprocess \ + --processes 1,2,4 \ + --query-rows 256 \ + --query-repeats 20 \ + --output /tmp/tsdf-buildings-hot.json +``` + +若索引不存在,先将 `--mode` 改为 `build`。构建会创建 `.tsfile_dataframe_index.tsidx` 和零长度 lock 文件;已有旧格式 `.tsfile_dataframe_index.tsfidx` 不会被覆盖。 + +## 6. 剩余项与发布判断 + +当前 v1 的 Format、Runtime、prepared Reader、Python API、生命周期、跨文件 merge 和多进程 mmap 功能目标均已实现,并通过正确性与多进程内存验收。但 2880 行随机序列冷缓存基准发现 4.74–6.06 倍的 p50 查询回退,因此性能验收尚未完成。 + +把 SSI 扫描得到的真实 page offset 发布到 `PagePositionIndex`,或提供等价的持久化 page locator/有界预取,已经不是单纯的可选优化。若发布标准包含“冷随机查询不回退”,则该项及其 I/O profiling 应作为发布前工作;优化后必须复用固定 workload 和 cache-drop 口径重新验收。 diff --git "a/design/TsFileDataFrame \346\214\201\344\271\205\345\214\226 mmap \347\264\242\345\274\225\344\270\216\350\277\233\347\250\213\346\234\254\345\234\260 Runtime \346\212\200\346\234\257\350\256\276\350\256\241.md" "b/design/TsFileDataFrame \346\214\201\344\271\205\345\214\226 mmap \347\264\242\345\274\225\344\270\216\350\277\233\347\250\213\346\234\254\345\234\260 Runtime \346\212\200\346\234\257\350\256\276\350\256\241.md" new file mode 100644 index 000000000..c7323fa8e --- /dev/null +++ "b/design/TsFileDataFrame \346\214\201\344\271\205\345\214\226 mmap \347\264\242\345\274\225\344\270\216\350\277\233\347\250\213\346\234\254\345\234\260 Runtime \346\212\200\346\234\257\350\256\276\350\256\241.md" @@ -0,0 +1,434 @@ + + +# TsFileDataFrame 持久化 mmap 索引与进程本地 Runtime 技术设计 + +# 0\. 文档信息 + +|设计日期|状态|设计结论| +|---|---|---| +|2026\.08\.14|Final|Dataset 是静态 sealed TsFile 文件集合;TsFileDataFrame 通过只读 MappedDatasetIndex 按需访问持久化统计与路由,并由每次独立构造产生的 DatasetRuntime 管理 Reader、PreparedSeries、对象/查询 lease 和跨文件执行。v1 只对同时打开的 Reader 数量设置硬限制。| + + + +# 1\. 设计背景 + +`Dataset` 是磁盘上的静态 TsFile 文件集合。`TsFileDataFrame` 是用户可见的跨文件逻辑视图,提供选择、切片与查询,并产出训练使用的 dataframe 或 batch。海量多文件数据集如果为每个 TsFile 和训练进程恢复完整 schema、device、series、物理 series 段与统计对象,会产生与 series 数量线性相关的初始化时间和私有内存。新设计把不可变统计与路由持久化为 mmap-friendly Dataset Index,并由进程本地 Runtime 负责按需 Reader、PreparedSeries 和跨文件执行。 + +这些工作会在每个训练 worker、分布式训练进程和重复创建的 DataFrame 中独立发生。操作系统 Page Cache 可以缓存原始文件页,却不能共享当前 Python 统计与路由对象图,也不能消除逐文件元数据解析和跨文件 merge。 + +## 1\.1 元数据对象化问题 + +```text +TsFile metadata bytes + → per-file Python MetadataCatalog / tuple / dict / list + → root DataFrame catalog 与跨文件 routing/statistics +``` + +当前实现把不可变统计与跨文件路由展开为 tuple、dict、list、string 等 Python 对象图。单条记录包含对象头、指针和容器容量,布局不紧凑;这些对象也不是 file-backed,独立训练进程不能像 mmap 干净页一样共享。新设计把统计与路由写入紧凑的 Format v1 文件,DataFrame 通过只读 mmap typed view 按需访问,只在构造公开返回值时创建必要的 Python 对象。 + +## 1\.2 训练场景中的放大 + +若 Dataset 包含 N 个 TsFile 文件、D 个 device、S 个物理 series 段,并由 P 个训练进程独立加载,初始化总工作量接近 `P × O(N + D + S)`。物理 series 段表示一个逻辑 series 在一个 TsFile 文件中的记录;同一逻辑 series 可以跨多个 TsFile 文件。实测 `buildings_900k` 包含 61 个 TsFile 文件、1,792,328 条 series,单进程加载后 PSS 约 6\.31 GiB;按当前实现外推,8 个训练进程约为 50\.5 GiB。每个进程还会持有自己的 Reader、文件描述符、native metadata 和 Python catalog,因此,当 PyTorch DataLoader 使用 spawn 方式创建多个子进程时,每个子进程都会重新加载 DataFrame;它与其他训练进程一样会继续放大私有内存。 + +## 1\.3 当前 TsFileDataFrame 的源码与 Python 对象占用 + +当前实现的常驻 Python 对象直接来自 `python/tsfile/dataset/metadata.py` 与 `python/tsfile/dataset/dataframe.py`。下面保留实际字段定义,省略与内存布局无关的方法。 + +```python +class SeriesStats(NamedTuple): + length: int + min_time: int + max_time: int + timeline_length: int + timeline_min_time: int + timeline_max_time: int + +@dataclass(**_DATACLASS_SLOTS) +class TableEntry: + table_name: str + tag_columns: Tuple[str, ...] + tag_types: Tuple[TSDataType, ...] + field_columns: Tuple[str, ...] + _field_index_by_name: Dict[str, int] = field(init=False, repr=False) + +@dataclass(**_DATACLASS_SLOTS) +class DeviceEntry: + table_id: int + tag_values: Tuple[Any, ...] + min_time: int + max_time: int + +@dataclass(**_DATACLASS_SLOTS) +class MetadataCatalog: + table_entries: List[TableEntry] = field(default_factory=list) + device_entries: List[DeviceEntry] = field(default_factory=list) + table_id_by_name: Dict[str, int] = field(default_factory=dict) + device_id_by_key: Dict[Tuple[int, tuple], int] = field(default_factory=dict) + series_stats_by_ref: Dict[Tuple[int, int], SeriesStats] = field(default_factory=dict) +``` + +```python +# DataFrame-wide logical key: (device_idx, global_field_idx). +SeriesRefKey = Tuple[int, int] +# One physical occurrence in a TsFile shard: (reader, local_device_id, local_field_idx). +SeriesRef = Tuple[object, int, int] + +@dataclass(**_DATACLASS_SLOTS) +class _DataFrameCatalog: + model: Optional[str] = None + table_entries: Dict[str, TableEntry] = field(default_factory=dict) + devices: List[DeviceKey] = field(default_factory=list) + device_index: Dict[DeviceKey, int] = field(default_factory=dict) + device_time_bounds: List[Tuple[Optional[int], Optional[int]]] = field(default_factory=list) + series: List[SeriesRefKey] = field(default_factory=list) + series_shards: Dict[SeriesRefKey, List[SeriesRef]] = field(default_factory=dict) + +class TsFileDataFrame: + def __init__(self, paths, show_progress=True): + self._paths = _expand_paths(paths) + self._show_progress = show_progress + self._readers: Dict[str, object] = {} + self._index = _DataFrameCatalog() + self._is_view = False + self._root = None + self._closed = False + self._load_metadata() +``` + +`TsFileSeriesReader` 为每个 TsFile 建立一个 `MetadataCatalog`;`_register_reader()` 把各文件目录再次合并到 `_DataFrameCatalog`。因此同一个物理 series 至少对应单文件目录中的 `SeriesRefKey + SeriesStats`,以及跨文件目录中的 `series` 项、`series_shards` 键、list 和 `SeriesRef` tuple。 + +```python +import sys +from tsfile.dataset.metadata import SeriesStats + +N = 100_000 +reader = object() +series_stats_by_ref = {} +series = [] +series_shards = {} + +for i in range(N): + device_id = 10_000 + i + ref = (device_id, 0) + stats = SeriesStats( + 1_000_000 + i, + 1_700_000_000 + i, + 1_800_000_000 + i, + 2_000_000 + i, + 1_600_000_000 + i, + 1_900_000_000 + i, + ) + series_stats_by_ref[ref] = stats + series.append(ref) + series_shards[ref] = [(reader, device_id, 0)] + +def retained_bytes(root): + seen = set() + stack = [root] + total = 0 + while stack: + obj = stack.pop() + if id(obj) in seen: + continue + seen.add(id(obj)) + total += sys.getsizeof(obj) + if isinstance(obj, dict): + stack.extend(obj.keys()) + stack.extend(obj.values()) + elif isinstance(obj, (list, tuple)): + stack.extend(obj) + return total + +sample_ref = series[-1] +sample_stats = series_stats_by_ref[sample_ref] +sample_route_list = series_shards[sample_ref] +sample_route = sample_route_list[0] + +print(sys.getsizeof(sample_ref)) +print(sys.getsizeof(sample_stats)) +print(sys.getsizeof(sample_route_list)) +print(sys.getsizeof(sample_route)) +print(sys.getsizeof(series_stats_by_ref) / N) +print(sys.getsizeof(series) / N) +print(sys.getsizeof(series_shards) / N) +print(retained_bytes(series_stats_by_ref) / N) +print(retained_bytes({"series": series, "series_shards": series_shards}) / N) +print(retained_bytes({ + "stats": series_stats_by_ref, + "series": series, + "shards": series_shards, +}) / N) +``` + +```text +SeriesRefKey tuple 56 B +SeriesStats tuple 88 B +单元素 SeriesRef list 64 B +SeriesRef tuple 64 B +series_stats_by_ref dict 浅层摊销 52.42960 B/entry +series list 浅层摊销 8.00984 B/entry +series_shards dict 浅层摊销 52.42960 B/entry +单文件 series_stats_by_ref 保留对象图 408.42988 B/series +跨文件 series + series_shards 保留对象图 272.44273 B/series +两个目录去重后的合计 596.87272 B/series +N = 100,000 时合计 59,687,272 B + 56.922 MiB +``` + +这些数值来自上面明确给定的真实 Python 类型、字段值和容器容量。它们只计算 Python 可达对象;`TableEntry`、`DeviceEntry`、tag/path 字符串会按实际数据内容另行计入,Cython/C\+\+ Reader 与分配器占用由下一节的进程 PSS 实测覆盖。这样可以把“Python 对象逐项计算”和“整个进程实际占用”分开,避免把未测量部分分摊成推测比例。 + +## 1\.4 远程数据规模与内存实测 + +在 `192.168.99.20:/data/tsfile_datasets/TimeBench_TsFile/lotsa` 上进行了只读盘点和 DataFrame 加载实测。目录包含 130 个数据集、1,145 个 TsFile,原始数据约 1\.018 TiB;已有索引覆盖 18,330,657 个 series fragment,磁盘大小约 1\.955 GiB。 + +|数据集|TsFile|series|初始化|加载后 PSS|增量 PSS/series| +|---|---|---|---|---|---| +|`cmip6_1935`|14|434,176|32\.45 秒|1\.61 GiB|约 3\.72 KiB| +|`buildings_900k`|61|1,792,328|140\.77 秒|6\.31 GiB|约 3\.65 KiB| + +按 3\.65~3\.72 KiB/series 外推,完整 `lotsa` 对象化约需 64~66 GiB/进程,8 个进程约需 512~528 GiB。该结果说明本设计不仅要缩短启动时间,还必须避免恢复完整 Python/native catalog 对象图。 + +# 2\. 设计目标 + +1. 为每个 Dataset 建立一个持久化、可重建的外部索引文件,统一描述 schema、device/tag、logical series、fragment、聚合统计和 metadata locator。 + +2. 索引采用紧凑、只读、mmap\-friendly 的二进制格式;多个训练进程映射同一 inode,由操作系统共享干净文件页。 + +3. DataFrame 直接在 mmap view 上完成 len、lookup、prefix range、stats 和 fragment routing,不恢复全量 Python catalog。 + +4. 通过进程本地 ReaderSessionPool 快速切换文件,并通过 FileGeneration-bound PreparedSeries 复用活跃 series 的执行元数据。 + +5. 保持 C++ Reader 的单文件职责;跨文件 merge 第一版由 typed Cython 窄内核完成,不把 dataset root 或 Python 对象引入 libtsfile 核心。 + +6. 通过 FileGeneration、fingerprint、原子发布、构建锁和打开时校验保证静态 Dataset 的一致性与崩溃安全。 + +7. v1 严格限制同时打开的 Reader 数量,并观测索引、PreparedSeries、查询临时输出和结果内存;本阶段不承诺 DataFrame 总内存硬上限。 + +# 3\. 技术设计 + +TsFileDataFrame 面向静态 TsFile 文件集合提供跨文件逻辑视图。整体设计分成一个主文档和三个权威子文档: + +- [Dataset Index 文件格式与查询设计](<./Dataset Index 文件格式与查询设计.md>):二进制格式、记录布局、排序、校验和物理定位。 +- [TsFileDataFrame Runtime 设计](<./TsFileDataFrame Runtime 设计.md>):DatasetRuntime、FileGeneration、RuntimeLease/QueryLease、ReaderSessionPool、PreparedSeriesCache、Cython DatasetMergeKernel 和关闭安全。 +- [TsFile Reader 与 Prepared Query Context 设计](<./TsFile Reader 与 Prepared Query Context 设计.md>):C++ Reader、PreparedSeries、prepared 接口、PagePositionIndex、查询局部状态和 ResultSet。 + +端到端链路是:MappedDatasetIndex 定位逻辑序列、文件 span 和 metadata locator;DatasetRuntime 获取匹配 FileGeneration 的 ReaderSession 与 PreparedSeries;Reader 创建查询局部执行状态和 ResultSet;Cython DatasetMergeKernel 归并单文件数组片段;Python 构造现有公开结果对象。 + +## 3\.2 架构设计 + +### 3\.2\.1 独立 Runtime 与 lease 所有权 + + +每次独立构造 TsFileDataFrame 都创建独立 DatasetRuntime。初始 DataFrame、派生 view 与惰性 Timeseries 分别持有自己的 RuntimeLease;任一 DataFrame 的 close() 只关闭当前 handle。最后一个对象 lease 释放且活动 QueryLease 归零后,Runtime 才完成最终 teardown。不同独立 DataFrame 之间不复用 Runtime、ReaderSession 或 PreparedSeries,也不做进程级 registry、跨 Runtime 复用或全局资源仲裁。 + + + +MappedDatasetIndex 直接在当前版本的 SectionDirectory 上完成名称查找、定长记录读取与 span 遍历,不恢复完整元数据树。DatasetRuntime 负责 mapping、ReaderSessionPool、PreparedSeriesCache、DatasetMergeKernel 和 lease 状态。对象关系和关闭算法以 Runtime 子文档为准。 + +### 3\.2\.2 职责边界 + +|对象/组件|职责| +|---|---| +|TsFileDataFrame|保持现有 Python Dataset 查询 API、选择语义和返回类型;管理当前 handle 的 RuntimeLease。| +|MappedDatasetIndex|只读映射并校验 Dataset Index,返回 Table、Device、Column、LogicalSeries、TsFile 与 locator/span 的 typed view;不持有 Reader。| +|DatasetRuntime|绑定一次独立 DataFrame 构造,管理 FileGeneration、对象/查询 lease、ReaderSessionPool、PreparedSeriesCache 和 DatasetMergeKernel。| +|ReaderSessionPool / ReaderSession|按 FileGeneration 获取 ReaderSession;session 持有 FD、ReadFile、executor/decoder、Reader arena 和执行状态;max_open_files 是 v1 唯一 Runtime 资源硬限制。| +|PreparedSeriesCache|以 (FileGeneration, locator_id) 为 key,single-flight 构建并长期持有独立所有权的 PreparedSeries;v1 不定义 entry/byte 上限或 LRU。| +|TsFileReader / ResultSet|负责单文件定点 prepare、prepared 查询、解码和现有 ResultSet;查询期间由 Runtime 同时固定 QueryLease、ReaderSession lease 和 PreparedSeries lease。| +|DatasetMergeKernel|第一版在 typed Cython 中完成连续拼接、k-way merge、重复时间戳检查和多列时间戳对齐。| + +### 3\.2\.3 逻辑序列与物理定位 + +Format v1 的查找与路由链路为 TableNameIndex/TableRecord → DeviceNameIndex/DeviceRecord → ColumnNameIndex/ColumnSchema → LogicalSeries → SeriesFileSpan → SeriesLocator;TsFileRecord 提供 file\_id 对应的源文件记录。设计中不存在 MetaTree、SeriesSegment 或 local\_series\_id。 + +- 时间范围查询使用 SeriesFileSpan 的时间边界和 prefix\_max\_time 跳过不相交文件,再展开命中的 SeriesLocator。 + +- 非对齐序列由 SeriesLocator 指向 value TimeseriesMetadata;对齐序列由 DeviceFileSpan 提供共享 time metadata,SeriesLocator 提供 value metadata,并校验二者属于同一 file、device 与 aligned group。 + +- 位置路由必须区分 row\_count 与 point\_count:aligned row 位置以共享时间轴为准,value point\_count 只表示非空值数量,不能代替行数。 + +- 不重叠文件可利用累计 count 直接定位;存在时间重叠时由 Cython DatasetMergeKernel 执行有序归并,并保持重复时间戳错误语义。 + +## 3\.3 Dataset Index 文件与数据结构 + +[Dataset Index 文件格式与查询设计](<./Dataset Index 文件格式与查询设计.md>) + + + +## 3\.4 进程本地 Native Runtime + +[TsFileDataFrame Runtime 设计](<./TsFileDataFrame Runtime 设计.md>) + +## 3\.5 Reader 与 Prepared Query Context + +[TsFile Reader 与 Prepared Query Context 设计](<./TsFile Reader 与 Prepared Query Context 设计.md>) + +跨层契约只有以下几点:Runtime 向 ReaderSession 提供已验证的 FileGeneration 和 locator;Reader 在独立 arena 中构建 FileGeneration-bound PreparedSeries,并为每次查询创建局部 SSI/decoder/ResultSet;PreparedSeries 不持有 Reader 内部指针,可跨同 generation 的 ReaderSession 复用;可选 PagePositionIndex 只优化重复位置查询。Dataset 查询在 ResultSet 完整消费并关闭前同时固定 QueryLease、ReaderSession lease 和 PreparedSeries lease。C++ 表示、prepared 接口和 PagePositionIndex 细节只由 Reader 子文档定义。 + +## 3\.6 Dataset Index 与运行时加载策略 + +### 3\.6\.1 Format v1 映射与校验 + +Dataset Index 的唯一格式定义见 Format 子文档。当前 v1 编码的 `header_size=64`、`directory_entry_size=32` 和 `section_count=14` 是当前 parser 的兼容条件,不是永久架构上限。解析器按 version、size 和 SectionDirectory 校验当前支持的布局;新增可选 section 或扩大结构通过版本化兼容规则演进,不兼容变化升级 major version。 + +MappedDatasetIndex 只有在 Header、DirectoryEntry、section 边界、记录宽度、字符串引用和外键等全部校验通过后才暴露 typed view。业务 section 的实际位置始终从 SectionDirectory 读取;主文档不维护 offset、record width 或字段表。 + +### 3\.6\.2 加载后的内存分层 + +|区域|内容与生命周期| +|---|---| +|Dataset Index mapping|完整索引以只读 file-backed mmap 建立虚拟地址映射,section 不复制到完整 Python catalog。驻留页按访问触发并由 OS 回收;同机进程可共享底层干净页。| +|Runtime 基础对象|每次独立 DataFrame 构造对应的 mapping、ReaderSessionPool、PreparedSeriesCache、DatasetMergeKernel 和 lease 状态。| +|ReaderSessionPool|保存当前打开的 ReaderSession;每个 session 包含 FD、ReadFile、executor/decoder、Reader arena 和执行状态;严格执行内部 max_open_files。| +|PreparedSeriesCache|保存 FileGeneration-bound 的独立序列索引、arena 和可选 PagePositionIndex;v1 不设置 entry/byte 硬限制。| +|查询与结果|QueryLease、ReaderSession/PreparedSeries lease、ResultSet、decoder、typed merge 输入/输出和公开返回对象;随查询或对象生命周期释放。| + +### 3\.6\.3 按需访问策略 + +1. DataFrame 初始化只完成路径展开、索引校验或构建、只读映射、Runtime 和初始 RuntimeLease 创建;ReaderSessionPool 与 PreparedSeriesCache 均为空。 + +2. 精确名称查找通过 TableNameIndex、DeviceNameIndex 与 ColumnNameIndex;列表与结构化 prefix 查询按 Format v1 的记录顺序扫描需要的 LogicalSeries,并只在构造 Python 返回值时解码 StringOffsets/StringBytes。 + +3. 查询从 LogicalSeries 进入 SeriesFileSpan,使用时间统计与 prefix\_max\_time 剪枝,再读取命中的 DeviceFileSpan/SeriesLocator;不会在 heap 上恢复完整 MetaTree。 + +4. 命中具体 FileGeneration 后才获取 ReaderSession;命中 locator 后以 (FileGeneration, locator_id) 获取或构建 PreparedSeries。ReaderSession 淘汰不使 PreparedSeries 失效。 + +### 3\.6\.4 容量与验收口径 + +索引精确长度 I 由 Format builder 根据当前版本的 SectionDirectory、记录数量、记录宽度、字符串字节数和对齐规则计算;不能把旧布局的固定小计当作当前版本容量。 + +```text +mapped_bytes = I +0 ≤ resident_index_pages(t) ≤ I +observed_private(t) = runtime_base + open_readers + + prepared_resident + active_query/merge/results +``` + +验收时分别记录 mapped_bytes、索引页 RSS/PSS、进程 USS/private dirty、open Reader 数、PreparedSeries 数量及其 PageArena/TimeseriesIndex/ChunkMeta/PagePositionIndex 占用、Cython merge 临时输出和公开结果。只有 max_open_files 是 v1 硬约束;这些内存数据是观测指标,不构成 DataFrame 总内存上限。 + +## 3\.7 完整执行流程、一致性与跨进程并发 + +### 3\.7\.1 独立 DataFrame 初始化 + +1. TsFileDataFrame\.\_expand\_paths 沿用现有语义展开文件或目录,递归收集 \.tsfile,排序并转为绝对路径;公开构造参数仍只有 paths 与 show\_progress。 + +2. 初始化取得目标 Dataset Index 的构建锁,按 Format 子文档校验当前版本布局及其与输入文件记录的关系。有效时直接映射;缺失或无效时由 DatasetIndexBuilder 构建。 + +3. Builder 可以并行扫描单文件元数据,但字符串驻留、schema/series 合并、file\_id 分配、section 排序与最终写出必须全局确定。完整临时文件写入并校验后通过 fsync 与原子 rename 发布。 + +4. MappedDatasetIndex 对稳定索引建立只读 mmap,完成结构校验并创建 typed view;Runtime 不把全部 section 复制为 Python catalog。 + +5. 当前 DataFrame 创建独立 DatasetRuntime、空 ReaderSessionPool、空 PreparedSeriesCache 和自己的 RuntimeLease。构造函数返回后才开放查询;此时不预先打开 Reader,也不预构建 PreparedSeries。 + +活动 Runtime 不接收索引重建通知,也不切换到新的 mapped index 或文件 generation;调用者必须重新构造 DataFrame 才能读取新 generation。 + +### 3\.7\.2 按时间范围读取 + +1. 当前 handle 原子获取 QueryLease;close 已先完成时拒绝查询,反之查询允许完整结束。 + +2. MappedDatasetIndex 通过名称索引解析 LogicalSeries,读取 SeriesFileSpan,并使用时间边界与 prefix\_max\_time 跳过不相交文件;命中的 span 展开为 DeviceFileSpan/SeriesLocator。 + +3. Runtime 按 FileGeneration 获取并校验 ReaderSession,再以 (FileGeneration, locator_id) 查询 PreparedSeriesCache。miss 时调用 prepareSeriesFromLocator;同 key 并发 prepare 由 single-flight 去重。 + +4. Reader 调用 queryPrepared。整个 ResultSet 消费期间同时持有 ReaderSession lease 和 PreparedSeries lease。 + +5. 单文件 ResultSet 在当前 Dataset 读取方法中被完整消费并关闭;typed timestamp/value parts 交给 Cython DatasetMergeKernel,执行连续拼接或有序 k-way merge,并完成重复时间戳检查和多字段对齐。 + +6. ResultSet 关闭后依次释放 PreparedSeries lease、ReaderSession lease 和 QueryLease,最终返回现有 NumPy、Timeseries 或 AlignedTimeseries 结果。 + +### 3\.7\.3 按位置读取 + +不重叠文件按 SeriesFileSpan 的 count 与时间统计直接路由;存在重叠时由 Cython DatasetMergeKernel 归并,并保持重复时间戳抛出 ValueError 的语义。aligned 数据的位置语义使用共享时间轴的 row_count,不以 value point_count 代替。 + +命中单文件后调用 queryPreparedByRow。重复位置查询可使用 PreparedSeries 的增量 PagePositionIndex;没有重复查询时该索引可以保持为空。其定位与并发发布规则见 Reader 子文档。 + +### 3\.7\.4 一致性、并发与关闭 + +- 构建锁只串行化指向同一索引文件的初始化。拿到锁后重新校验;若其他进程已发布有效 v1 索引则直接映射。查询阶段不持有构建锁。 + +- 稳定索引发布后按不可变文件使用。不同 Runtime 和不同进程只通过只读 mmap 与 OS Page Cache 共享文件页;不共享 Reader、PreparedSeries、FD、ResultSet 或 C++ 指针。 + +- ReaderSessionPool 只回收 active_uses=0 的 idle session。达到 max_open_files 且全部 session 活跃时,acquire 等待 release,不临时超限;session 关闭不影响同 FileGeneration 的 PreparedSeries。 + +- 任一 DataFrame close 只关闭当前 handle 并释放其 RuntimeLease;其他 view 和 Timeseries 继续可用。最后一个对象 lease 释放后停止接受新查询,等活动 QueryLease 归零,再清空 PreparedSeriesCache、关闭 ReaderSessionPool 并解除 mmap。 + +## 3\.8 Python API 与兼容性 + +### 3\.8\.1 公开入口保持不变 + +```python +df = TsFileDataFrame(paths, show_progress=True) +series = df[0] +window = df.loc[start:end, series_names] +df.close() +``` + +公开构造签名保持 TsFileDataFrame\(paths: Union\[str, List\[str\]\], show\_progress: bool = True\)。索引路径、校验/构建、mmap 和 max_open_files 都是内部实现;不增加 index_path、RuntimeConfig、memory budget、prepared cache budget 或查询配置对象。 + +### 3\.8\.2 现有选择与返回类型 + +|入口|保持的语义与返回| +|---|---| +|df\[int\] / df\[str\]|按位置或完整逻辑序列名选择,返回现有 Timeseries;字符串未命中序列名时沿用元数据列选择语义。| +|df\[slice\] / df\[list\[int\]\] / df\[pandas\.Series\]|创建轻量 TsFileDataFrame 子视图,不复制索引、Reader 或 PreparedSeries cache。| +|df\.loc\[start:end, series\]|沿用 int/slice 时间范围和 str/int 序列选择,按时间戳并集对齐并返回 AlignedTimeseries。| +|list\_timeseries / list\_timeseries\_metadata|分别返回现有 List\[SeriesPath\] 与 pandas\.DataFrame;数据来自 mapped Format v1 记录,仅在构造返回值时创建 Python 字符串和对象。| +|Timeseries|保持惰性读取与数值返回语义;持有 RuntimeLease,并增加幂等 close() 和上下文管理器。| +|AlignedTimeseries|是已经完成读取和对齐的物化结果,不持有 RuntimeLease。| + +### 3\.8\.3 子视图、关闭与进程边界 + +初始 DataFrame、派生 view 与惰性 Timeseries 各自持有 RuntimeLease。关闭初始 DataFrame 不影响其他未关闭 handle;Timeseries 的显式 close()/context manager 是确定释放 lease 的主路径,finalizer 只作兜底。未来若增加惰性 iterator,iterator 自身或每个产出的 Timeseries 也必须持有独立 lease。 + +每次独立构造 DataFrame 始终创建独立 Runtime;当前阶段不定义新的 pickle/spawn 协议。跨进程可共享的只有 Dataset Index 和 TsFile 的 file-backed OS pages。 + +### 3\.8\.4 内部类型与格式约束 + +Python 包不公开 DatasetRuntime、ReaderSessionPool、PreparedSeriesCache、PreparedSeries 或 DatasetMergeKernel。用户仍通过 TsFileDataFrame、Timeseries 与 AlignedTimeseries 完成 Dataset 操作。 + +Dataset Index 不保存 JSON、pickle、C++ 指针、Python 对象或执行状态。具体 magic、Header、DirectoryEntry、section、字段和兼容规则只由 Format 子文档定义;主文档不复制第二套布局。 + +## 3\.9 实施结论与发布验收(2026-08-14) + +本设计已在 `feature/tsfile-dataframe-runtime` 分支实现。实现从 Apache `origin/develop` 的 `5681978b84a0d9ee26e1eab94f78afaccf93607a` 建立独立 worktree;四份冻结设计位于提交 `506f2a8bb`,主体代码位于提交 `a03a7f1d3`。Format、Runtime、Reader/Prepared 与 Python DataFrame 的实现边界分别落在对应子文档中,本节只汇总发布判断。 + +|范围|已实现内容|验收| +|---|---|---| +|Dataset Index Format v1|C++/Python 14-section codec、CRC32C、结构与引用校验、原子发布、只读 mmap、名称/series/span 查找、generation 校验|合成 C++ 测试与 506,136,064-byte Python 实际索引的 C++ 完整打开均通过| +|C++ Reader / Prepared|精确 TimeseriesMetadata range、独立 arena、aligned time/value 配对、prepared query C/Cython handle、查询局部 ResultSet|Release shared library 构建通过;C++ 聚合回归 757 passed、1 个外部数据可选测试 skipped;真实索引测试另行 7/7 通过| +|进程本地 Runtime|独立 Runtime、对象/查询 lease、Reader LRU 硬上限、Prepared single-flight、lazy mapped catalog、确定性 close|生命周期、并发、stale generation、Reader cap 与 cache 复用测试通过| +|Python / Cython|公开 DataFrame API 兼容、热启动 mmap 路径、typed overlap merge、view/Timeseries lease|Python 181/181 通过;Snappy/LZ4/LZO/Zlib 数据均由正式构建回归| + +`/data/lotsa` 实际是 116 个独立数据集目录,共 647 个 TsFile、约 519 GiB,不应作为一个混合 schema DataFrame 直接加载。本次选择其中最大的完整数据集 `buildings_900k`:61 个 TsFile、65,856,302,628 bytes、1,792,328 条逻辑序列。冷构建耗时 205.13 秒,索引 506,136,064 bytes,峰值 RSS 11,250,788 KiB,无 swap;这是一次性的 metadata object graph 与 section 排序/写出成本。 + +稳定索引上的 1/2/4 进程热启动结果如下。每个进程构造独立 Runtime,读取同一 file-backed mmap,并执行一次及重复 256 行 prepared 查询。 + +|进程数|构造中位数|首次查询中位数|重复查询中位数|总 PSS|总 USS| +|---:|---:|---:|---:|---:|---:| +|1|529.26 ms|1.26 ms|0.300 ms|84.95 MiB|57.98 MiB| +|2|527.26 ms|1.26 ms|0.302 ms|144.19 MiB|99.45 MiB| +|4|523.28 ms|1.20 ms|0.300 ms|251.13 MiB|198.09 MiB| + +发布结论:核心 Format、Runtime、C++ prepared path、Python API 与 1/2/4 多进程目标已经达到当前 v1 验收条件。`PagePositionIndex` 的并发、无空洞发布结构已经实现,但实际 SSI 扫描尚未向其中回填 page offset;当前按行查询继续走正确的 fallback scan。这是可选性能优化的明确剩余项,不影响索引格式、查询正确性或本次多进程内存结论。 diff --git a/python/setup.py b/python/setup.py index d8e9a502f..a69db1558 100644 --- a/python/setup.py +++ b/python/setup.py @@ -264,6 +264,7 @@ def finalize_options(self): ) exts = [ + Extension("tsfile.dataset._merge", ["tsfile/dataset/_merge.pyx"], **common), Extension("tsfile.tsfile_py_cpp", ["tsfile/tsfile_py_cpp.pyx"], **common), Extension("tsfile.tsfile_reader", ["tsfile/tsfile_reader.pyx"], **common), Extension("tsfile.tsfile_writer", ["tsfile/tsfile_writer.pyx"], **common), diff --git a/python/tests/benchmark_tsfile_dataframe_cold_cache.py b/python/tests/benchmark_tsfile_dataframe_cold_cache.py new file mode 100644 index 000000000..f1befbe60 --- /dev/null +++ b/python/tests/benchmark_tsfile_dataframe_cold_cache.py @@ -0,0 +1,747 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""Cold/hot-cache, random-series TsFileDataFrame comparison benchmark. + +The benchmark intentionally imports ``tsfile`` only in spawned workers. This +lets the coordinator execute ``sync`` and Linux ``drop_caches`` before every +process-count group without importing the implementation under test first. + +Each worker uses a deterministic, worker-specific random seed. Worker 0 +therefore uses the same positions in the 1, 2, and 4 process groups, while the +additional workers exercise different series. After the cold random pass, +each worker repeats the exact same series in the same order without +dropping caches, producing a directly comparable hot-cache pass. The output +includes every logical series name so implementations with different catalog +lengths or integer ordering can still be checked against an identical +workload. +""" + +from __future__ import annotations + +import argparse +import gc +import hashlib +import importlib.metadata +import json +import multiprocessing as mp +import os +import platform +import random +import statistics +import sys +import time +from typing import Any, Dict + + +def _percentile(values: list[float], fraction: float) -> float: + ordered = sorted(values) + rank = max(1, int(len(ordered) * fraction + 0.999999999)) + return ordered[min(len(ordered), rank) - 1] + + +def _summary(values: list[float]) -> dict[str, float | int]: + return { + "count": len(values), + "mean_ms": statistics.fmean(values), + "p50_ms": statistics.median(values), + "p95_ms": _percentile(values, 0.95), + "p99_ms": _percentile(values, 0.99), + "max_ms": max(values), + } + + +def _meminfo_kib() -> Dict[str, int]: + values: Dict[str, int] = {} + accepted = { + "MemTotal", + "MemFree", + "MemAvailable", + "Buffers", + "Cached", + "SReclaimable", + "Shmem", + "SwapTotal", + "SwapFree", + } + with open("/proc/meminfo", encoding="ascii") as stream: + for line in stream: + name, separator, remainder = line.partition(":") + if not separator or name not in accepted: + continue + token = remainder.strip().split(maxsplit=1)[0] + if token.isdigit(): + values[name] = int(token) + return values + + +def _drop_os_caches() -> dict[str, Any]: + before_sync = _meminfo_kib() + os.sync() + with open("/proc/sys/vm/drop_caches", "w", encoding="ascii") as stream: + stream.write("3\n") + after_drop = _meminfo_kib() + return { + "method": "os.sync(); write 3 to /proc/sys/vm/drop_caches", + "before_sync_kib": before_sync, + "after_drop_kib": after_drop, + } + + +def _process_memory_kib(pid: int) -> Dict[str, int]: + values: Dict[str, int] = {} + accepted = { + "Rss", + "Pss", + "Pss_Dirty", + "Pss_Anon", + "Pss_File", + "Pss_Shmem", + "Shared_Clean", + "Shared_Dirty", + "Private_Clean", + "Private_Dirty", + "Anonymous", + "Swap", + "SwapPss", + } + with open(f"/proc/{pid}/smaps_rollup", encoding="ascii") as stream: + for line in stream: + name, separator, remainder = line.partition(":") + if not separator or name not in accepted: + continue + token = remainder.strip().split(maxsplit=1)[0] + if token.isdigit(): + values[name] = int(token) + values["Uss"] = values.get("Private_Clean", 0) + values.get("Private_Dirty", 0) + values["Shared"] = values.get("Shared_Clean", 0) + values.get("Shared_Dirty", 0) + values["OpenFDs"] = len(os.listdir(f"/proc/{pid}/fd")) + return values + + +def _memory_snapshot(worker_results: list[dict[str, Any]]) -> dict[str, Any]: + by_pid = { + str(result["pid"]): _process_memory_kib(result["pid"]) + for result in worker_results + } + totals = { + key: sum(process_memory.get(key, 0) for process_memory in by_pid.values()) + for key in ("Rss", "Pss", "Uss", "Shared", "OpenFDs") + } + return {"by_pid": by_pid, "total": totals} + + +def _read_window(dataframe, selector: Any, query_rows: int) -> tuple[float, int]: + series = None + started = time.perf_counter_ns() + try: + series = dataframe[selector] + values = series[:query_rows] + elapsed_ms = (time.perf_counter_ns() - started) / 1_000_000 + rows = len(values) + del values + return elapsed_ms, rows + finally: + if series is not None: + close = getattr(series, "close", None) + if close is not None: + close() + + +def _positions_hash(positions: list[int]) -> str: + rendered = ",".join(str(position) for position in positions).encode("ascii") + return hashlib.sha256(rendered).hexdigest() + + +def _selectors_hash(selectors: list[Any]) -> str: + rendered = json.dumps( + selectors, ensure_ascii=False, separators=(",", ":") + ).encode("utf-8") + return hashlib.sha256(rendered).hexdigest() + + +def _worker( + dataset: str, + worker_index: int, + planned_selectors: list[Any] | None, + sample_count: int, + query_rows: int, + seed: int, + first_start, + random_start, + hot_start, + release, + sender, +) -> None: + dataframe = None + try: + from tsfile import TsFileDataFrame + + construct_started = time.perf_counter_ns() + dataframe = TsFileDataFrame(dataset, show_progress=False) + construct_ms = (time.perf_counter_ns() - construct_started) / 1_000_000 + worker_seed = None if planned_selectors is not None else seed + worker_index + selectors = planned_selectors + if selectors is None: + selectors = random.Random(worker_seed).sample( + range(len(dataframe)), sample_count + 1 + ) + if len(selectors) != sample_count + 1: + raise ValueError( + f"worker {worker_index} needs {sample_count + 1} selectors, " + f"got {len(selectors)}" + ) + if all(isinstance(selector, int) for selector in selectors): + if min(selectors) < 0 or max(selectors) >= len(dataframe): + raise ValueError( + f"worker {worker_index} workload position is out of range" + ) + positions = selectors + elif all(isinstance(selector, str) for selector in selectors): + positions = None + else: + raise TypeError("workload selectors must be all integers or all strings") + common = { + "pid": os.getpid(), + "worker_index": worker_index, + "worker_seed": worker_seed, + "construct_ms": construct_ms, + "series_count": len(dataframe), + "file_count": len(dataframe._paths), + "first_selector": selectors[0], + "random_selectors": selectors[1:], + "selectors_sha256": _selectors_hash(selectors), + "first_position": None if positions is None else positions[0], + "random_positions": None if positions is None else positions[1:], + "positions_sha256": ( + None if positions is None else _positions_hash(positions) + ), + } + sender.send({"phase": "constructed", **common}) + + first_start.wait() + first_ms, first_rows = _read_window(dataframe, selectors[0], query_rows) + sender.send( + { + "phase": "first_complete", + **common, + "first_query_ms": first_ms, + "first_query_rows": first_rows, + } + ) + + random_start.wait() + random_started = time.perf_counter_ns() + random_ms = [] + random_rows = [] + for selector in selectors[1:]: + elapsed_ms, rows = _read_window(dataframe, selector, query_rows) + random_ms.append(elapsed_ms) + random_rows.append(rows) + random_wall_ms = (time.perf_counter_ns() - random_started) / 1_000_000 + gc.collect() + sender.send( + { + "phase": "random_complete", + **common, + "first_query_ms": first_ms, + "first_query_rows": first_rows, + "random_query_ms": random_ms, + "random_query_summary": _summary(random_ms), + "random_query_rows_min": min(random_rows), + "random_query_rows_max": max(random_rows), + "random_query_rows_total": sum(random_rows), + "random_wall_ms": random_wall_ms, + } + ) + + hot_start.wait() + hot_started = time.perf_counter_ns() + hot_ms = [] + hot_rows = [] + for selector in selectors[1:]: + elapsed_ms, rows = _read_window(dataframe, selector, query_rows) + hot_ms.append(elapsed_ms) + hot_rows.append(rows) + hot_wall_ms = (time.perf_counter_ns() - hot_started) / 1_000_000 + gc.collect() + sender.send( + { + "phase": "hot_complete", + **common, + "hot_query_ms": hot_ms, + "hot_query_summary": _summary(hot_ms), + "hot_query_rows_min": min(hot_rows), + "hot_query_rows_max": max(hot_rows), + "hot_query_rows_total": sum(hot_rows), + "hot_wall_ms": hot_wall_ms, + } + ) + release.wait() + except BaseException as exc: + try: + sender.send( + { + "phase": "error", + "pid": os.getpid(), + "worker_index": worker_index, + "error": f"{type(exc).__name__}: {exc}", + } + ) + finally: + release.wait() + raise + finally: + if dataframe is not None: + dataframe.close() + + +def _receive_phase(receivers, phase: str) -> list[dict[str, Any]]: + results = [receiver.recv() for receiver in receivers] + errors = [result for result in results if result.get("phase") == "error"] + if errors: + raise RuntimeError(f"workers failed before {phase}: {errors}") + unexpected = [result for result in results if result.get("phase") != phase] + if unexpected: + raise RuntimeError(f"expected phase {phase}, received {unexpected}") + return results + + +def _run_group( + dataset: str, + process_count: int, + sample_count: int, + query_rows: int, + seed: int, + drop_os_cache: bool, + workload_plan: dict[int, list[Any]] | None, +) -> dict[str, Any]: + cache_drop = _drop_os_caches() if drop_os_cache else None + print( + f"starting cold-cache group: processes={process_count}", + file=sys.stderr, + flush=True, + ) + context = mp.get_context("spawn") + first_start = context.Event() + random_start = context.Event() + hot_start = context.Event() + release = context.Event() + processes = [] + receivers = [] + group_started = time.perf_counter_ns() + for worker_index in range(process_count): + receiver, sender = context.Pipe(duplex=False) + process = context.Process( + target=_worker, + args=( + dataset, + worker_index, + None if workload_plan is None else workload_plan[worker_index], + sample_count, + query_rows, + seed, + first_start, + random_start, + hot_start, + release, + sender, + ), + ) + process.start() + sender.close() + processes.append(process) + receivers.append(receiver) + + try: + constructed = _receive_phase(receivers, "constructed") + constructed_wall_ms = (time.perf_counter_ns() - group_started) / 1_000_000 + memory_before_query = _memory_snapshot(constructed) + + first_phase_started = time.perf_counter_ns() + first_start.set() + first_complete = _receive_phase(receivers, "first_complete") + first_phase_wall_ms = (time.perf_counter_ns() - first_phase_started) / 1_000_000 + + random_phase_started = time.perf_counter_ns() + random_start.set() + random_complete = _receive_phase(receivers, "random_complete") + random_phase_wall_ms = ( + time.perf_counter_ns() - random_phase_started + ) / 1_000_000 + memory_after_random_query = _memory_snapshot(random_complete) + + hot_phase_started = time.perf_counter_ns() + hot_start.set() + hot_complete = _receive_phase(receivers, "hot_complete") + hot_phase_wall_ms = ( + time.perf_counter_ns() - hot_phase_started + ) / 1_000_000 + memory_after_hot_query = _memory_snapshot(hot_complete) + + first_ms = [result["first_query_ms"] for result in first_complete] + random_ms = [ + elapsed + for result in random_complete + for elapsed in result["random_query_ms"] + ] + hot_ms = [ + elapsed + for result in hot_complete + for elapsed in result["hot_query_ms"] + ] + query_wall_ms = max(result["random_wall_ms"] for result in random_complete) + hot_query_wall_ms = max(result["hot_wall_ms"] for result in hot_complete) + if any(result["first_query_rows"] != query_rows for result in first_complete): + raise RuntimeError(f"first query did not return {query_rows} rows") + if any( + result["random_query_rows_min"] != query_rows + or result["random_query_rows_max"] != query_rows + for result in random_complete + ): + raise RuntimeError(f"random query did not return {query_rows} rows") + if any( + result["hot_query_rows_min"] != query_rows + or result["hot_query_rows_max"] != query_rows + for result in hot_complete + ): + raise RuntimeError(f"hot query did not return {query_rows} rows") + if len({result["series_count"] for result in constructed}) != 1: + raise RuntimeError("workers observed different series counts") + + result = { + "processes": process_count, + "cache_drop": cache_drop, + "constructed_wall_ms": constructed_wall_ms, + "construct_ms": _summary( + [result["construct_ms"] for result in constructed] + ), + "series_count": constructed[0]["series_count"], + "file_count": constructed[0]["file_count"], + "workload": [ + { + "worker_index": item["worker_index"], + "worker_seed": item["worker_seed"], + "first_selector": item["first_selector"], + "random_selectors": item["random_selectors"], + "selectors_sha256": item["selectors_sha256"], + "first_position": item["first_position"], + "random_positions": item["random_positions"], + "positions_sha256": item["positions_sha256"], + } + for item in constructed + ], + "first_query": { + "phase_wall_ms": first_phase_wall_ms, + "latency": _summary(first_ms), + "rows_per_worker": query_rows, + "workers": first_complete, + }, + "random_query": { + "cache_state": "cold first pass over random_selectors", + "query_wall_ms": query_wall_ms, + "coordinator_phase_wall_ms_including_worker_gc": random_phase_wall_ms, + "aggregate_qps": len(random_ms) / (query_wall_ms / 1000), + "latency": _summary(random_ms), + "query_count": len(random_ms), + "rows_per_query": query_rows, + "rows_total": len(random_ms) * query_rows, + "workers": random_complete, + }, + "hot_query": { + "cache_state": ( + "immediate second pass over the same random_selectors " + "in the same order without dropping caches" + ), + "query_wall_ms": hot_query_wall_ms, + "coordinator_phase_wall_ms_including_worker_gc": hot_phase_wall_ms, + "aggregate_qps": len(hot_ms) / (hot_query_wall_ms / 1000), + "latency": _summary(hot_ms), + "query_count": len(hot_ms), + "rows_per_query": query_rows, + "rows_total": len(hot_ms) * query_rows, + "workers": hot_complete, + }, + "memory_before_query_kib": memory_before_query, + "memory_after_random_query_kib": memory_after_random_query, + "memory_after_hot_query_kib": memory_after_hot_query, + } + print( + "completed cold-cache group: " + f"processes={process_count}, " + f"first_p50={result['first_query']['latency']['p50_ms']:.3f} ms, " + f"cold_random_p50={result['random_query']['latency']['p50_ms']:.3f} ms, " + f"hot_p50={result['hot_query']['latency']['p50_ms']:.3f} ms", + file=sys.stderr, + flush=True, + ) + return result + finally: + release.set() + for process in processes: + process.join() + bad = [process for process in processes if process.exitcode != 0] + if bad and sys.exc_info()[0] is None: + raise RuntimeError( + "benchmark workers exited abnormally: " + + ", ".join(f"{process.pid}={process.exitcode}" for process in bad) + ) + + +def _dataset_stats(dataset: str) -> dict[str, int]: + paths = [ + os.path.join(root, name) + for root, _, names in os.walk(dataset) + for name in names + if name.endswith(".tsfile") + ] + return { + "tsfile_count": len(paths), + "tsfile_bytes": sum(os.path.getsize(path) for path in paths), + } + + +def _rejection_key(exc: Exception) -> str: + message = str(exc).split(" for series ", maxsplit=1)[0] + return f"{type(exc).__name__}: {message}" + + +def _generate_workload_plan( + dataset: str, + worker_count: int, + sample_count: int, + query_rows: int, + seed: int, +) -> dict[str, Any]: + from tsfile import TsFileDataFrame + + required = worker_count * (sample_count + 1) + accepted: list[tuple[int, str]] = [] + accepted_names: set[str] = set() + attempted: set[int] = set() + rejected: dict[str, int] = {} + dataframe = TsFileDataFrame(dataset, show_progress=False) + try: + random_source = random.Random(seed) + max_attempts = min(len(dataframe), required * 100) + while len(accepted) < required and len(attempted) < max_attempts: + position = random_source.randrange(len(dataframe)) + if position in attempted: + continue + attempted.add(position) + try: + series = dataframe[position] + try: + series_name = str(series.name) + finally: + close = getattr(series, "close", None) + if close is not None: + close() + if series_name in accepted_names: + key = "duplicate logical series name" + rejected[key] = rejected.get(key, 0) + 1 + continue + _, rows = _read_window(dataframe, position, query_rows) + if rows != query_rows: + key = f"short result: {rows} rows" + rejected[key] = rejected.get(key, 0) + 1 + continue + except Exception as exc: + key = _rejection_key(exc) + rejected[key] = rejected.get(key, 0) + 1 + continue + accepted.append((position, series_name)) + accepted_names.add(series_name) + if len(accepted) % 100 == 0 or len(accepted) == required: + print( + f"workload plan: accepted {len(accepted)}/{required}, " + f"attempted {len(attempted)}", + file=sys.stderr, + flush=True, + ) + if len(accepted) != required: + raise RuntimeError( + f"found only {len(accepted)} valid positions after " + f"{len(attempted)} attempts; need {required}" + ) + workers = [] + width = sample_count + 1 + for worker_index in range(worker_count): + entries = accepted[worker_index * width : (worker_index + 1) * width] + positions = [position for position, _ in entries] + series_names = [series_name for _, series_name in entries] + workers.append( + { + "worker_index": worker_index, + "positions": positions, + "positions_sha256": _positions_hash(positions), + "series_names": series_names, + "series_names_sha256": _selectors_hash(series_names), + } + ) + return { + "dataset": dataset, + "generated_with_package_version": importlib.metadata.version("tsfile"), + "query_rows": query_rows, + "sample_count_per_worker": sample_count, + "worker_count": worker_count, + "seed": seed, + "series_count": len(dataframe), + "file_count": len(dataframe._paths), + "attempted_candidate_count": len(attempted), + "accepted_position_count": len(accepted), + "rejected_candidate_count": len(attempted) - len(accepted), + "rejected_by_reason": rejected, + "workers": workers, + } + finally: + dataframe.close() + + +def _load_workload_plan( + path: str, sample_count: int, query_rows: int +) -> tuple[dict[str, Any], dict[int, list[Any]]]: + with open(path, encoding="utf-8") as stream: + raw = json.load(stream) + if raw.get("query_rows") != query_rows: + raise ValueError( + f"workload plan query_rows={raw.get('query_rows')}, expected {query_rows}" + ) + if raw.get("sample_count_per_worker") != sample_count: + raise ValueError( + "workload plan sample_count_per_worker=" + f"{raw.get('sample_count_per_worker')}, expected {sample_count}" + ) + workers = {} + for worker in raw["workers"]: + if "series_names" in worker: + selectors = [str(value) for value in worker["series_names"]] + else: + selectors = [int(value) for value in worker["positions"]] + workers[int(worker["worker_index"])] = selectors + if any(len(set(selectors)) != len(selectors) for selectors in workers.values()): + raise ValueError("workload plan contains duplicate selectors within a worker") + flattened = [selector for selectors in workers.values() for selector in selectors] + if len(set(flattened)) != len(flattened): + raise ValueError("workload plan contains selectors shared by workers") + return raw, workers + + +def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument("dataset") + parser.add_argument("--processes", default="1,2,4") + parser.add_argument("--sample-count", type=int, default=128) + parser.add_argument("--query-rows", type=int, default=2880) + parser.add_argument("--seed", type=int, default=20260814) + parser.add_argument("--label") + parser.add_argument("--output") + parser.add_argument("--workload-plan") + parser.add_argument("--generate-workload-plan") + parser.add_argument("--plan-worker-count", type=int, default=4) + parser.add_argument( + "--drop-os-cache", + action="store_true", + help="run sync and write 3 to /proc/sys/vm/drop_caches before every group", + ) + args = parser.parse_args() + if args.sample_count < 1 or args.query_rows < 1: + parser.error("sample count and query rows must be positive") + if args.drop_os_cache and os.geteuid() != 0: + parser.error("--drop-os-cache requires root on Linux") + + dataset = os.path.abspath(args.dataset) + if args.generate_workload_plan: + plan = _generate_workload_plan( + dataset, + args.plan_worker_count, + args.sample_count, + args.query_rows, + args.seed, + ) + rendered = json.dumps(plan, ensure_ascii=False, indent=2) + print(rendered) + with open(args.generate_workload_plan, "w", encoding="utf-8") as stream: + stream.write(rendered) + stream.write("\n") + return + + raw_workload_plan = None + workload_plan = None + if args.workload_plan: + raw_workload_plan, workload_plan = _load_workload_plan( + args.workload_plan, args.sample_count, args.query_rows + ) + requested_processes = [int(value) for value in args.processes.split(",")] + missing_workers = [ + worker_index + for worker_index in range(max(requested_processes)) + if worker_index not in workload_plan + ] + if missing_workers: + parser.error(f"workload plan is missing workers {missing_workers}") + else: + requested_processes = [int(value) for value in args.processes.split(",")] + report = { + "label": args.label, + "package_version": importlib.metadata.version("tsfile"), + "python": sys.version, + "machine": { + "hostname": platform.node(), + "platform": platform.platform(), + "cpu_count": os.cpu_count(), + "meminfo_kib": _meminfo_kib(), + }, + "dataset": dataset, + "dataset_stats": _dataset_stats(dataset), + "sample_count_per_worker": args.sample_count, + "query_rows": args.query_rows, + "seed": args.seed, + "workload_plan_path": args.workload_plan, + "workload_plan_metadata": ( + None + if raw_workload_plan is None + else { + key: value + for key, value in raw_workload_plan.items() + if key != "workers" + } + ), + "drop_os_cache_before_every_group": args.drop_os_cache, + "groups": [ + _run_group( + dataset, + process_count, + args.sample_count, + args.query_rows, + args.seed, + args.drop_os_cache, + workload_plan, + ) + for process_count in requested_processes + ], + } + rendered = json.dumps(report, ensure_ascii=False, indent=2) + print(rendered) + if args.output: + with open(args.output, "w", encoding="utf-8") as stream: + stream.write(rendered) + stream.write("\n") + + +if __name__ == "__main__": + main() diff --git a/python/tests/benchmark_tsfile_dataframe_runtime.py b/python/tests/benchmark_tsfile_dataframe_runtime.py new file mode 100644 index 000000000..ceb393ce7 --- /dev/null +++ b/python/tests/benchmark_tsfile_dataframe_runtime.py @@ -0,0 +1,281 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""Reproducible cold-build and multi-process TsFileDataFrame benchmark. + +This is intentionally a standalone benchmark rather than a pytest test. It +uses only the Python standard library for process coordination and Linux +``/proc`` accounting, so it can run in the project's wheel-building virtual +environment without installing psutil. +""" + +from __future__ import annotations + +import argparse +import gc +import json +import multiprocessing as mp +import os +import platform +import resource +import statistics +import sys +import time +from typing import Dict + +from tsfile import TsFileDataFrame +from tsfile.dataset.index import INDEX_FILE_NAME + + +def _index_path(dataset: str) -> str: + return os.path.join(os.path.abspath(dataset), INDEX_FILE_NAME) + + +def _process_memory_kib(pid: int) -> Dict[str, int]: + values: Dict[str, int] = {} + accepted = { + "Rss", + "Pss", + "Pss_Dirty", + "Pss_Anon", + "Pss_File", + "Pss_Shmem", + "Shared_Clean", + "Shared_Dirty", + "Private_Clean", + "Private_Dirty", + "Anonymous", + "Swap", + "SwapPss", + } + with open(f"/proc/{pid}/smaps_rollup", encoding="ascii") as stream: + for line in stream: + name, separator, remainder = line.partition(":") + if not separator or name not in accepted: + continue + token = remainder.strip().split(maxsplit=1)[0] + if token.isdigit(): + values[name] = int(token) + values["Uss"] = values.get("Private_Clean", 0) + values.get("Private_Dirty", 0) + values["Shared"] = values.get("Shared_Clean", 0) + values.get("Shared_Dirty", 0) + values["OpenFDs"] = len(os.listdir(f"/proc/{pid}/fd")) + return values + + +def _percentile(values, fraction: float) -> float: + ordered = sorted(values) + position = max(0, min(len(ordered) - 1, int(len(ordered) * fraction) - 1)) + return ordered[position] + + +def _worker(dataset: str, query_rows: int, query_repeats: int, ready, release) -> None: + started = time.perf_counter_ns() + dataframe = None + series = None + try: + dataframe = TsFileDataFrame(dataset, show_progress=False) + constructed = time.perf_counter_ns() + series = dataframe[0] + query_started = time.perf_counter_ns() + values = series[: min(query_rows, len(series))] + query_finished = time.perf_counter_ns() + repeat_query_ms = [] + for _ in range(query_repeats): + repeat_started = time.perf_counter_ns() + values = series[: min(query_rows, len(series))] + repeat_query_ms.append( + (time.perf_counter_ns() - repeat_started) / 1_000_000 + ) + gc.collect() + ready.send( + { + "pid": os.getpid(), + "construct_ms": (constructed - started) / 1_000_000, + "first_query_ms": (query_finished - query_started) / 1_000_000, + "repeat_query_ms_median": statistics.median(repeat_query_ms), + "repeat_query_ms_p95": _percentile(repeat_query_ms, 0.95), + "query_repeats": query_repeats, + "query_rows": int(len(values)), + "series_count": len(dataframe), + "first_series_count": len(series), + "model": dataframe.model, + } + ) + release.wait() + except BaseException as exc: + ready.send( + { + "pid": os.getpid(), + "error": f"{type(exc).__name__}: {exc}", + } + ) + release.wait() + raise + finally: + if series is not None: + series.close() + if dataframe is not None: + dataframe.close() + + +def _run_group( + dataset: str, process_count: int, query_rows: int, query_repeats: int +) -> dict: + context = mp.get_context("spawn") + release = context.Event() + processes = [] + receivers = [] + group_started = time.perf_counter_ns() + for _ in range(process_count): + receiver, sender = context.Pipe(duplex=False) + process = context.Process( + target=_worker, + args=(dataset, query_rows, query_repeats, sender, release), + ) + process.start() + sender.close() + processes.append(process) + receivers.append(receiver) + + worker_results = [receiver.recv() for receiver in receivers] + ready_ms = (time.perf_counter_ns() - group_started) / 1_000_000 + if any("error" in result for result in worker_results): + release.set() + for process in processes: + process.join() + raise RuntimeError(f"worker failed: {worker_results}") + + memory = { + str(result["pid"]): _process_memory_kib(result["pid"]) + for result in worker_results + } + totals = { + key: sum(process_memory.get(key, 0) for process_memory in memory.values()) + for key in ("Rss", "Pss", "Uss", "Shared", "OpenFDs") + } + + release.set() + for process in processes: + process.join() + if process.exitcode != 0: + raise RuntimeError( + f"benchmark worker {process.pid} exited with {process.exitcode}" + ) + + return { + "processes": process_count, + "group_ready_ms": ready_ms, + "construct_ms": [result["construct_ms"] for result in worker_results], + "construct_ms_median": statistics.median( + result["construct_ms"] for result in worker_results + ), + "first_query_ms": [result["first_query_ms"] for result in worker_results], + "first_query_ms_median": statistics.median( + result["first_query_ms"] for result in worker_results + ), + "repeat_query_ms_median": statistics.median( + result["repeat_query_ms_median"] for result in worker_results + ), + "repeat_query_ms_p95_max": max( + result["repeat_query_ms_p95"] for result in worker_results + ), + "worker_results": worker_results, + "memory_kib_by_pid": memory, + "memory_kib_total": totals, + } + + +def _build_index(dataset: str) -> dict: + path = _index_path(dataset) + existed = os.path.exists(path) + started = time.perf_counter_ns() + with TsFileDataFrame(dataset, show_progress=True) as dataframe: + result = { + "model": dataframe.model, + "series_count": len(dataframe), + "file_count": len(dataframe._paths), + } + finished = time.perf_counter_ns() + result.update( + { + "index_path": path, + "index_existed_before": existed, + "index_size_bytes": os.path.getsize(path), + "elapsed_ms": (finished - started) / 1_000_000, + "max_rss_kib": resource.getrusage(resource.RUSAGE_SELF).ru_maxrss, + } + ) + return result + + +def _machine() -> dict: + return { + "hostname": platform.node(), + "platform": platform.platform(), + "python": sys.version, + "cpu_count": os.cpu_count(), + } + + +def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument("dataset") + parser.add_argument( + "--mode", choices=("build", "multiprocess", "all"), default="all" + ) + parser.add_argument("--processes", default="1,2,4") + parser.add_argument("--query-rows", type=int, default=256) + parser.add_argument("--query-repeats", type=int, default=20) + parser.add_argument("--output") + args = parser.parse_args() + if args.query_rows < 0 or args.query_repeats < 1: + parser.error("--query-rows must be non-negative and --query-repeats positive") + + dataset = os.path.abspath(args.dataset) + report = { + "machine": _machine(), + "dataset": dataset, + "dataset_bytes": sum( + os.path.getsize(os.path.join(root, name)) + for root, _, names in os.walk(dataset) + for name in names + if name.endswith(".tsfile") + ), + "index_path": _index_path(dataset), + } + if args.mode in ("build", "all"): + report["build"] = _build_index(dataset) + if args.mode in ("multiprocess", "all"): + if not os.path.exists(report["index_path"]): + raise FileNotFoundError( + f"hot-start benchmark requires {report['index_path']}" + ) + report["groups"] = [ + _run_group(dataset, count, args.query_rows, args.query_repeats) + for count in (int(value) for value in args.processes.split(",")) + ] + + rendered = json.dumps(report, ensure_ascii=False, indent=2) + print(rendered) + if args.output: + with open(args.output, "w", encoding="utf-8") as stream: + stream.write(rendered) + stream.write("\n") + + +if __name__ == "__main__": + main() diff --git a/python/tests/test_dataset_index.py b/python/tests/test_dataset_index.py new file mode 100644 index 000000000..242aad91e --- /dev/null +++ b/python/tests/test_dataset_index.py @@ -0,0 +1,242 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +from types import SimpleNamespace +import os + +import pandas as pd +import pytest + +from tsfile import ColumnCategory, ColumnSchema, TableSchema, TsFileTableWriter +from tsfile import TsFileDataFrame +from tsfile.constants import TSDataType +from tsfile.dataset.index import ( + COLUMN_SCHEMA, + DEVICE_FILE_SPAN, + DIRECTORY, + HEADER, + LOGICAL_SERIES, + MappedDatasetIndex, + RECORDS, + SERIES_FILE_SPAN, + SERIES_LOCATOR, + TSFILE_RECORD, + build_sections_from_dataframe, + crc32c, + write_index_atomic, +) +from tsfile.dataset.metadata import MetadataCatalog, SeriesStats +from tsfile.tsfile_reader import TsFileReaderPy + + +def _synthetic_dataframe(source_path): + catalog = MetadataCatalog() + table_id = catalog.add_table("root", (), (), ("s1",)) + device_id = catalog.add_device(table_id, (), 1, 10) + catalog.series_stats_by_ref[(device_id, 0)] = SeriesStats( + 10, + 1, + 10, + 10, + 1, + 10, + int(TSDataType.INT64), + 128, + 16, + 64, + 16, + 1, + 1, + 1, + 1, + ) + reader = SimpleNamespace(file_path=source_path, catalog=catalog) + index = SimpleNamespace( + table_entries={"root": catalog.table_entries[0]}, + devices=[("root", ())], + series=[(0, 0)], + series_shards={(0, 0): [(reader, 0, 0)]}, + ) + return SimpleNamespace( + _index=index, + _readers={source_path: reader}, + _paths=[source_path], + ) + + +def test_binary_layout_matches_cpp_v1(): + assert HEADER.size == 64 + assert DIRECTORY.size == 32 + assert RECORDS[COLUMN_SCHEMA].size == 32 + assert RECORDS[LOGICAL_SERIES].size == 32 + assert RECORDS[TSFILE_RECORD].size == 48 + assert RECORDS[DEVICE_FILE_SPAN].size == 48 + assert RECORDS[SERIES_FILE_SPAN].size == 48 + assert RECORDS[SERIES_LOCATOR].size == 24 + + +def test_crc32c_known_vector(): + assert crc32c(b"123456789") == 0xE3069283 + + +def test_build_publish_map_and_lookup(tmp_path): + source = tmp_path / "source.tsfile" + source.write_bytes(b"T" * 4096) + output = tmp_path / "dataset.tsidx" + dataframe = _synthetic_dataframe(str(source)) + write_index_atomic(str(output), build_sections_from_dataframe(dataframe)) + + with MappedDatasetIndex(str(output), verify_sections=True) as index: + table_id = index.find_table_ids("root")[0] + device_id = index.find_device_id(table_id, "root.") + column_id = index.find_column_id(table_id, "s1") + assert index.find_series_id(device_id, column_id) == 0 + assert index.count(LOGICAL_SERIES) == 1 + assert index.count(SERIES_LOCATOR) == 1 + file_record = index.record(TSFILE_RECORD, 0) + assert index.string(file_record[0]) == str(source) + + +def test_rejects_damaged_header_checksum(tmp_path): + source = tmp_path / "source.tsfile" + source.write_bytes(b"T" * 4096) + output = tmp_path / "dataset.tsidx" + write_index_atomic( + str(output), build_sections_from_dataframe(_synthetic_dataframe(str(source))) + ) + with output.open("r+b") as stream: + stream.seek(32) + stream.write((1).to_bytes(8, "little")) + with pytest.raises(ValueError, match="header shape|checksum"): + MappedDatasetIndex(str(output)) + + +def test_reader_exposes_exact_aligned_locator_ranges(tmp_path): + source = tmp_path / "aligned.tsfile" + schema = TableSchema( + "weather", + [ + ColumnSchema("device", TSDataType.STRING, ColumnCategory.TAG), + ColumnSchema("value", TSDataType.DOUBLE, ColumnCategory.FIELD), + ], + ) + with TsFileTableWriter(str(source), schema) as writer: + writer.write_dataframe( + pd.DataFrame( + { + "time": [0, 1, 2], + "device": ["d0", "d0", "d0"], + "value": [1.0, 2.0, 3.0], + } + ) + ) + reader = TsFileReaderPy(str(source)) + try: + groups = reader.get_timeseries_metadata() + metadata = [item for group in groups.values() for item in group.timeseries] + assert metadata + assert all(item.value_metadata_length > 0 for item in metadata) + assert all( + item.value_metadata_offset + item.value_metadata_length + <= os.path.getsize(source) + for item in metadata + ) + aligned = [item for item in metadata if item.layout == 1] + assert aligned + assert all(item.time_metadata_length > 0 for item in aligned) + assert all( + item.time_chunk_meta_count == item.chunk_meta_count for item in aligned + ) + finally: + reader.close() + + +def _write_runtime_file(path, start): + schema = TableSchema( + "weather", + [ + ColumnSchema("device", TSDataType.STRING, ColumnCategory.TAG), + ColumnSchema("value", TSDataType.DOUBLE, ColumnCategory.FIELD), + ], + ) + with TsFileTableWriter(str(path), schema) as writer: + writer.write_dataframe( + pd.DataFrame( + { + "time": [start, start + 1], + "device": ["d0", "d0"], + "value": [float(start), float(start + 1)], + } + ) + ) + + +def test_hot_construction_maps_index_without_opening_readers(tmp_path, monkeypatch): + source = tmp_path / "part.tsfile" + _write_runtime_file(source, 0) + with TsFileDataFrame(str(source), show_progress=False) as first: + assert len(first) == 1 + + def fail_legacy_scan(*_args, **_kwargs): + raise AssertionError("hot construction must not build a legacy catalog") + + monkeypatch.setattr("tsfile.dataset.reader.TsFileSeriesReader", fail_legacy_scan) + with TsFileDataFrame(str(source), show_progress=False) as second: + assert len(second) == 1 + assert second._runtime.readers.open_count == 0 + series = second[0] + assert second._runtime.readers.open_count == 0 + assert series[0] == 0.0 + assert second._runtime.readers.open_count == 1 + assert second._runtime.prepared.size == 1 + assert series[1] == 1.0 + assert second._runtime.prepared.size == 1 + series.close() + + +def test_reader_pool_enforces_open_file_cap(tmp_path, monkeypatch): + first = tmp_path / "part1.tsfile" + second = tmp_path / "part2.tsfile" + _write_runtime_file(first, 0) + _write_runtime_file(second, 2) + monkeypatch.setenv("TSFILE_DATAFRAME_MAX_OPEN_FILES", "1") + with TsFileDataFrame([str(first), str(second)], show_progress=False) as dataframe: + series = dataframe[0] + assert list(series[:]) == [0.0, 1.0, 2.0, 3.0] + assert dataframe._runtime.readers.open_count == 1 + assert dataframe._runtime.prepared.size == 2 + series.close() + + +def test_prepared_locator_rejects_stale_generation_and_bad_range(tmp_path): + source = tmp_path / "part.tsfile" + _write_runtime_file(source, 0) + with TsFileDataFrame(str(source), show_progress=False) as dataframe: + runtime = dataframe._runtime + series = runtime.index.record(LOGICAL_SERIES, 0) + span = runtime.index.record(SERIES_FILE_SPAN, series[2]) + locator = list(runtime.prepared._locator_tuple(0, span[2])) + with runtime.readers.acquire(0) as reader: + stale = list(locator) + stale[3] ^= 1 + with pytest.raises(Exception, match="prepare Dataset Index locator"): + reader.prepare_series(stale) + + out_of_range = list(locator) + out_of_range[7] = os.path.getsize(source) + 1 + with pytest.raises(Exception, match="prepare Dataset Index locator"): + reader.prepare_series(out_of_range) diff --git a/python/tests/test_tsfile_dataset.py b/python/tests/test_tsfile_dataset.py index ee016d087..125425eab 100644 --- a/python/tests/test_tsfile_dataset.py +++ b/python/tests/test_tsfile_dataset.py @@ -833,7 +833,7 @@ def fail_merge(*_args, **_kwargs): np.testing.assert_array_equal(series[1:5], np.array([20.0, 30.0, 40.0, 50.0])) -def test_dataset_rejects_data_access_after_close(tmp_path): +def test_dataset_close_only_releases_current_handle(tmp_path): path = tmp_path / "weather.tsfile" _write_weather_file(path, 0) @@ -844,18 +844,22 @@ def test_dataset_rejects_data_access_after_close(tmp_path): with pytest.raises(RuntimeError, match="TsFileDataFrame is closed"): _ = tsdf[0] - with pytest.raises(RuntimeError, match="TsFileDataFrame is closed"): + assert series[0] == 20.0 + series.close() + with pytest.raises(RuntimeError, match="Timeseries is closed"): _ = series[0] -def test_subset_close_warns_and_does_not_close_root(tmp_path): +def test_subset_close_releases_only_subset_lease(tmp_path): path = tmp_path / "weather.tsfile" _write_weather_file(path, 0) with TsFileDataFrame(str(path), show_progress=False) as tsdf: subset = tsdf[:1] - with pytest.warns(RuntimeWarning, match="no-op"): - subset.close() + subset.close() + + with pytest.raises(RuntimeError, match="TsFileDataFrame is closed"): + _ = subset[0] series = tsdf[0] assert series[0] == 20.0 diff --git a/python/tsfile/dataset/_merge.pyx b/python/tsfile/dataset/_merge.pyx new file mode 100644 index 000000000..7282dda40 --- /dev/null +++ b/python/tsfile/dataset/_merge.pyx @@ -0,0 +1,172 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# cython: boundscheck=False, wraparound=False, cdivision=True, language_level=3 + +"""Narrow typed kernels for overlapping cross-shard merges.""" + +import numpy as np +cimport numpy as cnp +from libc.stdint cimport int64_t + +cnp.import_array() + + +cdef inline bint _less( + int left, int right, int64_t[:] times, cnp.intp_t[:] starts, + cnp.intp_t[:] cursors) noexcept: + cdef int64_t left_time = times[starts[left] + cursors[left]] + cdef int64_t right_time = times[starts[right] + cursors[right]] + return left_time < right_time or (left_time == right_time and left < right) + + +cdef void _push(int[:] heap, int* heap_size, int part, + int64_t[:] times, cnp.intp_t[:] starts, + cnp.intp_t[:] cursors) noexcept: + cdef int pos = heap_size[0] + cdef int parent + heap_size[0] += 1 + while pos > 0: + parent = (pos - 1) // 2 + if not _less(part, heap[parent], times, starts, cursors): + break + heap[pos] = heap[parent] + pos = parent + heap[pos] = part + + +cdef int _pop(int[:] heap, int* heap_size, int64_t[:] times, + cnp.intp_t[:] starts, cnp.intp_t[:] cursors) noexcept: + cdef int result = heap[0] + cdef int last + cdef int pos = 0 + cdef int child + heap_size[0] -= 1 + if heap_size[0] == 0: + return result + last = heap[heap_size[0]] + while True: + child = pos * 2 + 1 + if child >= heap_size[0]: + break + if child + 1 < heap_size[0] and _less( + heap[child + 1], heap[child], times, starts, cursors): + child += 1 + if not _less(heap[child], last, times, starts, cursors): + break + heap[pos] = heap[child] + pos = child + heap[pos] = last + return result + + +cdef tuple _flatten(list time_parts): + cdef Py_ssize_t count = len(time_parts) + cdef cnp.ndarray[cnp.intp_t, ndim=1] starts = np.empty(count, dtype=np.intp) + cdef cnp.ndarray[cnp.intp_t, ndim=1] lengths = np.empty(count, dtype=np.intp) + cdef Py_ssize_t index + cdef Py_ssize_t total = 0 + for index in range(count): + starts[index] = total + lengths[index] = len(time_parts[index]) + total += lengths[index] + return np.concatenate(time_parts).astype(np.int64, copy=False), starts, lengths, total + + +cpdef merge_timestamp_parts_overlap(list time_parts, bint deduplicate, + bint validate_unique): + cdef object flat_object + cdef cnp.ndarray[cnp.intp_t, ndim=1] starts_array + cdef cnp.ndarray[cnp.intp_t, ndim=1] lengths_array + cdef Py_ssize_t total + flat_object, starts_array, lengths_array, total = _flatten(time_parts) + cdef cnp.ndarray[cnp.int64_t, ndim=1] flat_array = flat_object + cdef int64_t[:] flat = flat_array + cdef cnp.intp_t[:] starts = starts_array + cdef cnp.intp_t[:] lengths = lengths_array + cdef cnp.ndarray[cnp.intp_t, ndim=1] cursors_array = np.zeros(len(time_parts), dtype=np.intp) + cdef cnp.intp_t[:] cursors = cursors_array + cdef cnp.ndarray[cnp.int32_t, ndim=1] heap_array = np.empty(len(time_parts), dtype=np.int32) + cdef int[:] heap = heap_array + cdef cnp.ndarray[cnp.int64_t, ndim=1] output_array = np.empty(total, dtype=np.int64) + cdef int64_t[:] output = output_array + cdef int heap_size = 0 + cdef int part + cdef Py_ssize_t out_size = 0 + cdef int64_t timestamp + cdef int64_t last_timestamp = 0 + cdef bint has_last = False + for part in range(len(time_parts)): + if lengths[part] > 0: + _push(heap, &heap_size, part, flat, starts, cursors) + while heap_size > 0: + part = _pop(heap, &heap_size, flat, starts, cursors) + timestamp = flat[starts[part] + cursors[part]] + if has_last and timestamp == last_timestamp: + if validate_unique: + raise ValueError(f"Duplicate timestamp {timestamp} found across shards.") + if not deduplicate: + output[out_size] = timestamp + out_size += 1 + else: + output[out_size] = timestamp + out_size += 1 + last_timestamp = timestamp + has_last = True + cursors[part] += 1 + if cursors[part] < lengths[part]: + _push(heap, &heap_size, part, flat, starts, cursors) + return output_array[:out_size] + + +cpdef merge_time_value_parts_overlap(list time_parts, list value_parts): + cdef object flat_object + cdef cnp.ndarray[cnp.intp_t, ndim=1] starts_array + cdef cnp.ndarray[cnp.intp_t, ndim=1] lengths_array + cdef Py_ssize_t total + flat_object, starts_array, lengths_array, total = _flatten(time_parts) + cdef cnp.ndarray[cnp.int64_t, ndim=1] flat_times_array = flat_object + cdef cnp.ndarray[cnp.float64_t, ndim=1] flat_values_array = np.concatenate(value_parts).astype(np.float64, copy=False) + cdef int64_t[:] flat_times = flat_times_array + cdef double[:] flat_values = flat_values_array + cdef cnp.intp_t[:] starts = starts_array + cdef cnp.intp_t[:] lengths = lengths_array + cdef cnp.ndarray[cnp.intp_t, ndim=1] cursors_array = np.zeros(len(time_parts), dtype=np.intp) + cdef cnp.intp_t[:] cursors = cursors_array + cdef cnp.ndarray[cnp.int32_t, ndim=1] heap_array = np.empty(len(time_parts), dtype=np.int32) + cdef int[:] heap = heap_array + cdef cnp.ndarray[cnp.int64_t, ndim=1] output_times_array = np.empty(total, dtype=np.int64) + cdef cnp.ndarray[cnp.float64_t, ndim=1] output_values_array = np.empty(total, dtype=np.float64) + cdef int64_t[:] output_times = output_times_array + cdef double[:] output_values = output_values_array + cdef int heap_size = 0 + cdef int part + cdef Py_ssize_t source + cdef Py_ssize_t out_index = 0 + for part in range(len(time_parts)): + if lengths[part] > 0: + _push(heap, &heap_size, part, flat_times, starts, cursors) + while heap_size > 0: + part = _pop(heap, &heap_size, flat_times, starts, cursors) + source = starts[part] + cursors[part] + output_times[out_index] = flat_times[source] + output_values[out_index] = flat_values[source] + out_index += 1 + cursors[part] += 1 + if cursors[part] < lengths[part]: + _push(heap, &heap_size, part, flat_times, starts, cursors) + return output_times_array, output_values_array diff --git a/python/tsfile/dataset/dataframe.py b/python/tsfile/dataset/dataframe.py index a65b231bc..8d2ef0d35 100644 --- a/python/tsfile/dataset/dataframe.py +++ b/python/tsfile/dataset/dataframe.py @@ -23,8 +23,8 @@ import heapq import os import sys +from types import SimpleNamespace from typing import Dict, List, Optional, Tuple, Union -import warnings import numpy as np @@ -628,6 +628,8 @@ def __init__(self, paths: Union[str, List[str]], show_progress: bool = True): self._is_view = False self._root = None self._closed = False + self._runtime = None + self._runtime_lease = None self._load_metadata() @classmethod @@ -641,40 +643,74 @@ def _from_subset( obj._paths = parent._paths obj._show_progress = parent._show_progress obj._readers = parent._readers - # Reuse the parent's full mapping but restrict the membership scope to - # the requested subset. subset_refs = list(series_refs) - parent_shards = parent._index.series_shards - subset_shards = {ref: parent_shards[ref] for ref in subset_refs} - obj._index = _DataFrameCatalog( + obj._index = SimpleNamespace( model=parent._index.model, table_entries=parent._index.table_entries, devices=parent._index.devices, device_index=parent._index.device_index, device_time_bounds=parent._index.device_time_bounds, series=subset_refs, - series_shards=subset_shards, + series_shards=parent._index.series_shards, ) + obj._runtime = parent._runtime + obj._runtime_lease = ( + parent._runtime_lease.clone() if parent._runtime_lease is not None else None + ) + obj._readers = parent._readers obj._closed = False return obj def _owner(self) -> "TsFileDataFrame": - return self._root if self._is_view else self + return self def _assert_open(self): - if self._owner()._closed: + if self._closed: raise RuntimeError("Current TsFileDataFrame is closed.") def _load_metadata(self): - """Build the logical cross-file index and the derived per-series caches.""" + """Map a valid persistent index, or build it once under a file lock.""" from .reader import TsFileSeriesReader - - if len(self._paths) >= 2: - self._load_metadata_parallel(TsFileSeriesReader) - else: - self._load_metadata_serial(TsFileSeriesReader) - - if not self._index.series: + from .index import ( + build_index_from_dataframe, + index_matches_paths, + index_path_for, + ) + from .runtime import DatasetRuntime + + index_path = index_path_for(self._paths) + if not index_matches_paths(index_path, self._paths): + lock_path = index_path + ".lock" + os.makedirs(os.path.dirname(lock_path) or ".", exist_ok=True) + with open(lock_path, "a+b") as lock_file: + try: + import fcntl + + fcntl.flock(lock_file.fileno(), fcntl.LOCK_EX) + except ImportError: + pass + if not index_matches_paths(index_path, self._paths): + if len(self._paths) >= 2: + self._load_metadata_parallel(TsFileSeriesReader) + else: + self._load_metadata_serial(TsFileSeriesReader) + + if not self._index.series: + raise ValueError( + "No valid time series found in the provided TsFile files" + ) + try: + build_index_from_dataframe(self, index_path) + finally: + for reader in self._readers.values(): + reader.close() + self._readers.clear() + + self._runtime = DatasetRuntime(index_path) + self._runtime_lease = self._runtime.lease() + self._index = self._runtime.catalog + if len(self._index.series) == 0: + self._runtime_lease.close() raise ValueError("No valid time series found in the provided TsFile files") def _show_loading_progress(self, done: int, total: int, total_series: int = None): @@ -913,17 +949,21 @@ def list_timeseries_metadata(self, path_prefix: str = ""): def _get_timeseries(self, series_ref: SeriesRefKey) -> Timeseries: self._assert_open() series_name = self._build_series_name(series_ref) + runtime_lease = ( + self._runtime_lease.clone() if self._runtime_lease is not None else None + ) return Timeseries( series_name, self._index.series_shards[series_ref], _build_runtime_series_stats(self._index.series_shards[series_ref]), - self._assert_open, + self._assert_open if runtime_lease is None else None, lambda: _merge_field_timestamps( series_name, self._index.series_shards[series_ref] ), lambda offset, limit: _read_field_by_position( series_name, self._index.series_shards[series_ref], offset, limit ), + runtime_lease=runtime_lease, ) def __getitem__(self, key): @@ -1087,24 +1127,19 @@ def show(self, max_rows: int = 20): print(self._repr_header() + self._format_table(max_rows=max_rows)) def close(self): - if self._is_view: - warnings.warn( - "close() on a subset TsFileDataFrame is a no-op; only the root dataframe owns the readers.", - RuntimeWarning, - stacklevel=2, - ) - return if self._closed: return - for reader in self._readers.values(): - reader.close() - self._readers.clear() self._closed = True + if self._runtime_lease is not None: + self._runtime_lease.close() + else: + for reader in self._readers.values(): + reader.close() + self._readers.clear() def __del__(self): try: - if not getattr(self, "_is_view", False): - self.close() + self.close() except Exception: pass diff --git a/python/tsfile/dataset/index.py b/python/tsfile/dataset/index.py new file mode 100644 index 000000000..427143135 --- /dev/null +++ b/python/tsfile/dataset/index.py @@ -0,0 +1,838 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +"""Persistent mmap-backed Dataset Index v1. + +The binary layout in this module is intentionally identical to +``cpp/src/dataset/dataset_index.h``. Python owns cold-build orchestration and +public object construction; hot-path catalog lookup reads packed records +directly from a read-only mmap without recreating the old dictionary graph. +""" + +from __future__ import annotations + +from collections import defaultdict +import contextlib +import mmap +import os +import struct +from typing import Dict, Iterable, Iterator, List, Mapping, Optional, Sequence, Tuple + +from .metadata import MODEL_TREE, _join_series_path + +MAGIC = b"TSIDX\0\0\0" +VERSION_MAJOR = 1 +VERSION_MINOR = 0 +HEADER_SIZE = 64 +DIRECTORY_ENTRY_SIZE = 32 +SECTION_COUNT = 14 +ALIGNMENT = 64 +INDEX_FILE_NAME = ".tsfile_dataframe_index.tsidx" + +STRING_OFFSETS = 1 +STRING_BYTES = 2 +TABLE_NAME_INDEX = 3 +TABLE_RECORD = 4 +DEVICE_NAME_INDEX = 5 +DEVICE_RECORD = 6 +COLUMN_NAME_INDEX = 7 +COLUMN_SCHEMA = 8 +LOGICAL_SERIES = 9 +TSFILE_RECORD = 10 +TABLE_FILE_SPAN = 11 +DEVICE_FILE_SPAN = 12 +SERIES_FILE_SPAN = 13 +SERIES_LOCATOR = 14 + +HEADER = struct.Struct("<8sHHIQIIQI20s") +DIRECTORY = struct.Struct(" int: + return (value + ALIGNMENT - 1) & ~(ALIGNMENT - 1) + + +def name_hash(value: bytes) -> int: + """Return the format-v1 FNV-1a hash (matching the C++ implementation).""" + result = 1469598103934665603 + for byte in value: + result ^= byte + result = (result * 1099511628211) & 0xFFFFFFFFFFFFFFFF + return result + + +_CRC32C_TABLE = [] +for _value in range(256): + _crc = _value + for _ in range(8): + _crc = (_crc >> 1) ^ (0x82F63B78 if _crc & 1 else 0) + _CRC32C_TABLE.append(_crc) + + +def crc32c(data) -> int: + crc = 0xFFFFFFFF + for byte in data: + crc = _CRC32C_TABLE[(crc ^ byte) & 0xFF] ^ (crc >> 8) + return crc ^ 0xFFFFFFFF + + +def file_fingerprint(path: str, stat_result=None) -> int: + """Cheap sealed-dataset generation fingerprint. + + v1 combines size and nanosecond mtime. ReaderSession rechecks the same + tuple before interpreting a locator; content replacement therefore cannot + silently reuse an active generation under the static Dataset contract. + """ + st = os.stat(path) if stat_result is None else stat_result + return name_hash(struct.pack(" int: + result = self._ids.get(value) + if result is not None: + return result + encoded = value.encode("utf-8") + result = len(self._values) + self._ids[value] = result + self._values.append(encoded) + return result + + def sections(self) -> Tuple[bytes, bytes]: + offsets = bytearray(RECORDS[STRING_OFFSETS].size * (len(self._values) + 1)) + contents = bytearray() + for index, value in enumerate(self._values): + RECORDS[STRING_OFFSETS].pack_into(offsets, index * 4, len(contents)) + contents.extend(value) + RECORDS[STRING_OFFSETS].pack_into(offsets, len(self._values) * 4, len(contents)) + if len(contents) >= 1 << 32: + raise OverflowError("Dataset Index v1 string pool exceeds 4 GiB") + return bytes(offsets), bytes(contents) + + +def _pack_records(section_type: int, rows: Iterable[tuple]) -> bytes: + record = RECORDS[section_type] + rows = list(rows) + result = bytearray(record.size * len(rows)) + for index, row in enumerate(rows): + record.pack_into(result, index * record.size, *row) + return bytes(result) + + +def write_index_atomic(path: str, section_payloads: Mapping[int, bytes]) -> None: + """Write, fsync, validate, and atomically publish one format-v1 index.""" + if set(section_payloads) != set(range(1, SECTION_COUNT + 1)): + raise ValueError("Dataset Index v1 requires exactly 14 sections") + directory_offset = HEADER_SIZE + cursor = _align64(HEADER_SIZE + SECTION_COUNT * DIRECTORY_ENTRY_SIZE) + entries = [] + for section_type in range(1, SECTION_COUNT + 1): + payload = section_payloads[section_type] + record_size = 0 if section_type == STRING_BYTES else RECORDS[section_type].size + if record_size: + if len(payload) % record_size: + raise ValueError(f"section {section_type} has a partial record") + count = len(payload) // record_size + else: + count = len(payload) + if count >= 1 << 32: + raise OverflowError(f"section {section_type} count exceeds uint32") + entries.append( + (section_type, record_size, cursor, len(payload), count, crc32c(payload)) + ) + cursor = _align64(cursor + len(payload)) + + file_length = entries[-1][2] + entries[-1][3] + header_without_crc = HEADER.pack( + MAGIC, + VERSION_MAJOR, + VERSION_MINOR, + HEADER_SIZE, + directory_offset, + SECTION_COUNT, + DIRECTORY_ENTRY_SIZE, + file_length, + 0, + b"\0" * 20, + ) + header = bytearray(header_without_crc) + struct.pack_into(" len(self._view): + raise ValueError("Dataset Index directory is out of range") + + entries = {} + previous_end = _align64(directory_end) + for index in range(section_count): + entry = DIRECTORY.unpack_from( + self._view, directory_offset + index * DIRECTORY_ENTRY_SIZE + ) + section_type, record_size, offset, length, count, checksum = entry + if section_type != STRING_BYTES and section_type not in RECORDS: + raise ValueError("invalid Dataset Index section type") + expected_size = ( + 0 if section_type == STRING_BYTES else RECORDS[section_type].size + ) + if section_type != index + 1 or record_size != expected_size: + raise ValueError("invalid Dataset Index section directory") + if ( + offset % ALIGNMENT + or offset < previous_end + or offset + length > len(self._view) + or (record_size and count * record_size > length) + or (not record_size and count != length) + ): + raise ValueError("invalid Dataset Index section range") + if ( + verify_sections + and crc32c(self._view[offset : offset + length]) != checksum + ): + raise ValueError( + f"Dataset Index section {section_type} checksum mismatch" + ) + entries[section_type] = entry + previous_end = offset + length + + offsets = entries[STRING_OFFSETS] + strings = entries[STRING_BYTES] + if offsets[4] == 0: + raise ValueError("StringOffsets lacks terminal offset") + previous = 0 + for index in range(offsets[4]): + current = RECORDS[STRING_OFFSETS].unpack_from( + self._view, offsets[2] + index * 4 + )[0] + if current < previous or current > strings[3]: + raise ValueError("invalid Dataset Index string offset") + previous = current + if previous != strings[3]: + raise ValueError("Dataset Index terminal string offset mismatch") + return entries + + def close(self): + if getattr(self, "_view", None) is not None: + self._view.release() + self._view = None + if getattr(self, "_mmap", None) is not None: + self._mmap.close() + self._mmap = None + if getattr(self, "_file", None) is not None: + self._file.close() + self._file = None + + def __enter__(self): + return self + + def __exit__(self, *_): + self.close() + + def count(self, section_type: int) -> int: + return self._entries[section_type][4] + + def record(self, section_type: int, record_id: int) -> tuple: + entry = self._entries[section_type] + if record_id < 0 or record_id >= entry[4]: + raise IndexError(record_id) + return RECORDS[section_type].unpack_from( + self._view, entry[2] + record_id * entry[1] + ) + + def records(self, section_type: int, first: int = 0, count: Optional[int] = None): + total = self.count(section_type) + end = total if count is None else first + count + if first < 0 or end < first or end > total: + raise IndexError((first, count)) + for record_id in range(first, end): + yield self.record(section_type, record_id) + + def string_bytes(self, sid: int) -> bytes: + offsets = self._entries[STRING_OFFSETS] + strings = self._entries[STRING_BYTES] + if sid < 0 or sid + 1 >= offsets[4]: + raise IndexError(sid) + start = RECORDS[STRING_OFFSETS].unpack_from(self._view, offsets[2] + sid * 4)[0] + end = RECORDS[STRING_OFFSETS].unpack_from( + self._view, offsets[2] + (sid + 1) * 4 + )[0] + return bytes(self._view[strings[2] + start : strings[2] + end]) + + def string(self, sid: int) -> str: + return self.string_bytes(sid).decode("utf-8") + + def _equal_hash_range(self, section_type: int, hash_index: int, value_hash: int): + low = 0 + high = self.count(section_type) + while low < high: + middle = low + (high - low) // 2 + if self.record(section_type, middle)[hash_index] < value_hash: + low = middle + 1 + else: + high = middle + first = low + while ( + low < self.count(section_type) + and self.record(section_type, low)[hash_index] == value_hash + ): + low += 1 + return first, low + + def find_table_ids(self, name: str) -> List[int]: + encoded = name.encode("utf-8") + first, end = self._equal_hash_range(TABLE_NAME_INDEX, 0, name_hash(encoded)) + return [ + table_id + for _, sid, table_id in self.records(TABLE_NAME_INDEX, first, end - first) + if self.string_bytes(sid) == encoded + ] + + def _find_child( + self, section_type: int, table_id: int, name: str, first: int, count: int + ): + encoded = name.encode("utf-8") + target_hash = name_hash(encoded) + low, high = first, first + count + while low < high: + middle = low + (high - low) // 2 + row = self.record(section_type, middle) + key = (row[0], row[2], self.string_bytes(row[3])) + target = (table_id, target_hash, encoded) + if key < target: + low = middle + 1 + else: + high = middle + if low < first + count: + row = self.record(section_type, low) + if ( + row[0] == table_id + and row[2] == target_hash + and self.string_bytes(row[3]) == encoded + ): + return row[1] + raise KeyError(name) + + def find_device_id(self, table_id: int, name: str) -> int: + table = self.record(TABLE_RECORD, table_id) + return self._find_child(DEVICE_NAME_INDEX, table_id, name, table[3], table[4]) + + def find_column_id(self, table_id: int, name: str) -> int: + table = self.record(TABLE_RECORD, table_id) + return self._find_child(COLUMN_NAME_INDEX, table_id, name, table[5], table[6]) + + def find_series_id(self, device_id: int, column_id: int) -> int: + device = self.record(DEVICE_RECORD, device_id) + low, high = device[4], device[4] + device[5] + while low < high: + middle = low + (high - low) // 2 + if self.record(LOGICAL_SERIES, middle)[1] < column_id: + low = middle + 1 + else: + high = middle + if ( + low < device[4] + device[5] + and self.record(LOGICAL_SERIES, low)[1] == column_id + ): + return low + raise KeyError(column_id) + + +def index_path_for(paths: Sequence[str]) -> str: + common = os.path.commonpath([os.path.abspath(path) for path in paths]) + if not os.path.isdir(common): + common = os.path.dirname(common) + return os.path.join(common, INDEX_FILE_NAME) + + +def index_matches_paths(path: str, paths: Sequence[str]) -> bool: + """Return whether an index describes exactly the current sealed files.""" + expected = sorted(os.path.abspath(item) for item in paths) + try: + with MappedDatasetIndex(path) as index: + if index.count(TSFILE_RECORD) != len(expected): + return False + actual = [] + for file_id in range(index.count(TSFILE_RECORD)): + record = index.record(TSFILE_RECORD, file_id) + file_path = index.string(record[0]) + actual.append(file_path) + st = os.stat(file_path) + if ( + st.st_size != record[2] + or file_fingerprint(file_path, st) != record[5] + ): + return False + return actual == expected + except (OSError, ValueError, UnicodeError, IndexError): + return False + + +def _schema_fingerprint(table_entry, field_types: Mapping[str, int]) -> int: + parts = [] + for name, data_type in zip(table_entry.tag_columns, table_entry.tag_types): + parts.append(f"T\0{name}\0{int(data_type)}\01") + for name in table_entry.field_columns: + parts.append(f"F\0{name}\0{field_types.get(name, -1)}\01") + return name_hash("\0".join(parts).encode("utf-8")) + + +def build_sections_from_dataframe(dataframe) -> Mapping[int, bytes]: + """Convert one fully scanned legacy DataFrame into deterministic v1 sections. + + This is the cold-build bridge. Once published, subsequent constructions + use :class:`MappedDatasetIndex` and do not recreate this object graph. + """ + index = dataframe._index + pool = _StringPool() + file_paths = sorted(dataframe._readers) + file_ids = {path: file_id for file_id, path in enumerate(file_paths)} + + table_names = sorted(index.table_entries) + table_ids = {name: table_id for table_id, name in enumerate(table_names)} + field_types: Dict[Tuple[str, str], int] = {} + for series_ref, shards in index.series_shards.items(): + device_idx, field_idx = series_ref + table_name, _ = index.devices[device_idx] + field_name = index.table_entries[table_name].field_columns[field_idx] + for reader, local_device, local_field in shards: + stats = reader.catalog.series_stats_by_ref[(local_device, local_field)] + current = field_types.setdefault((table_name, field_name), stats.data_type) + if current != stats.data_type: + raise ValueError( + f"incompatible physical type for {table_name}.{field_name}" + ) + + columns = [] + columns_by_table = defaultdict(list) + for table_name in table_names: + table_id = table_ids[table_name] + table = index.table_entries[table_name] + definitions = [ + (name, int(data_type), 0) + for name, data_type in zip(table.tag_columns, table.tag_types) + ] + [ + (name, field_types.get((table_name, name), -1), 1) + for name in table.field_columns + ] + for ordinal, (name, data_type, role) in enumerate(definitions): + column_id = len(columns) + columns.append( + ( + table_id, + pool.intern(name), + ordinal, + data_type, + data_type, + 0, + 0, + role, + 1, + 0, + ) + ) + columns_by_table[table_id].append((name, column_id)) + + device_specs = [] + old_to_new_device = {} + for old_device_id, (table_name, tags) in enumerate(index.devices): + canonical_name = _join_series_path(table_name, tags, "") + device_specs.append( + (table_ids[table_name], canonical_name, old_device_id, tags) + ) + device_specs.sort(key=lambda item: (item[0], item[1].encode("utf-8"))) + for new_device_id, (_, _, old_device_id, _) in enumerate(device_specs): + old_to_new_device[old_device_id] = new_device_id + + series_specs = [] + for old_device_id, field_idx in index.series: + table_name, _ = index.devices[old_device_id] + field_name = index.table_entries[table_name].field_columns[field_idx] + column_id = dict(columns_by_table[table_ids[table_name]])[field_name] + series_specs.append( + (old_to_new_device[old_device_id], column_id, (old_device_id, field_idx)) + ) + series_specs.sort(key=lambda item: (item[0], item[1])) + old_series_to_new = {old: new for new, (_, _, old) in enumerate(series_specs)} + + locators = [] + series_span_rows = [] + device_span_specs = {} + file_bounds = {file_id: [None, None] for file_id in range(len(file_paths))} + series_spans_by_id = defaultdict(list) + device_spans_by_device = defaultdict(list) + table_spans = defaultdict(lambda: [None, None, 0]) + + for _, _, old_series_ref in series_specs: + series_id = old_series_to_new[old_series_ref] + old_device_id, _ = old_series_ref + device_id = old_to_new_device[old_device_id] + table_name, _ = index.devices[old_device_id] + table_id = table_ids[table_name] + for reader, local_device, local_field in index.series_shards[old_series_ref]: + file_id = file_ids[reader.file_path] + stats = reader.catalog.series_stats_by_ref[(local_device, local_field)] + if stats.value_metadata_length <= 0: + raise ValueError( + f"missing exact TimeseriesMetadata locator in {reader.file_path}" + ) + if stats.layout and not (stats.locator_flags & 1): + raise ValueError( + f"aligned time/value chunk metadata mismatch in {reader.file_path}" + ) + span_key = (device_id, file_id) + candidate = ( + stats.time_metadata_offset, + stats.time_metadata_length, + stats.layout, + 1 if stats.layout and stats.locator_flags & 1 else 0, + stats.timeline_min_time, + stats.timeline_max_time, + stats.timeline_length, + ) + previous = device_span_specs.get(span_key) + if previous is None: + device_span_specs[span_key] = candidate + else: + if previous[:4] != candidate[:4]: + raise ValueError( + "inconsistent aligned time locator for one device/file" + ) + device_span_specs[span_key] = ( + *previous[:4], + min(previous[4], candidate[4]), + max(previous[5], candidate[5]), + max(previous[6], candidate[6]), + ) + locator_id = len(locators) + locators.append( + ( + span_key, + stats.layout, + stats.locator_flags & 1, + stats.value_metadata_offset, + stats.value_metadata_length, + stats.chunk_meta_count, + ) + ) + series_spans_by_id[series_id].append( + [ + series_id, + file_id, + locator_id, + 0, + stats.timeline_min_time, + stats.timeline_max_time, + 0, + stats.length, + ] + ) + for bounds in (file_bounds[file_id], table_spans[(table_id, file_id)]): + bounds[0] = ( + stats.timeline_min_time + if bounds[0] is None + else min(bounds[0], stats.timeline_min_time) + ) + bounds[1] = ( + stats.timeline_max_time + if bounds[1] is None + else max(bounds[1], stats.timeline_max_time) + ) + + device_span_rows = [] + device_span_ids = {} + for key, values in sorted(device_span_specs.items()): + device_id, file_id = key + device_span_ids[key] = len(device_span_rows) + device_span_rows.append((device_id, file_id, *values)) + device_spans_by_device[device_id].append(device_span_ids[key]) + table_id = device_specs[device_id][0] + table_spans[(table_id, file_id)][2] += values[6] + locator_rows = [ + (device_span_ids[key], kind, flags, offset, length, count) + for key, kind, flags, offset, length, count in locators + ] + + logical_series_rows = [] + for series_id, (device_id, column_id, _) in enumerate(series_specs): + spans = sorted(series_spans_by_id[series_id], key=lambda row: (row[4], row[1])) + first = len(series_span_rows) + prefix_max = None + for row in spans: + prefix_max = row[5] if prefix_max is None else max(prefix_max, row[5]) + row[6] = prefix_max + series_span_rows.append(tuple(row)) + logical_series_rows.append( + ( + device_id, + column_id, + first, + len(spans), + min(row[4] for row in spans), + max(row[5] for row in spans), + ) + ) + + device_rows = [] + device_names_by_table = defaultdict(list) + series_by_device = defaultdict(list) + for series_id, row in enumerate(logical_series_rows): + series_by_device[row[0]].append(series_id) + device_span_flat = [] + # DeviceFileSpan is already globally sorted by device, so each device range + # is contiguous and its first id can be recorded directly. + for device_id, (table_id, canonical_name, _, _) in enumerate(device_specs): + name_sid = pool.intern(canonical_name) + device_names_by_table[table_id].append((canonical_name, device_id, name_sid)) + series_ids = series_by_device[device_id] + span_ids = device_spans_by_device[device_id] + first_series = series_ids[0] if series_ids else 0 + first_span = span_ids[0] if span_ids else 0 + if series_ids: + minimum = min(logical_series_rows[sid][4] for sid in series_ids) + maximum = max(logical_series_rows[sid][5] for sid in series_ids) + else: + minimum = maximum = 0 + device_rows.append( + ( + table_id, + name_sid, + 0, + 0, + first_series, + len(series_ids), + first_span, + len(span_ids), + minimum, + maximum, + ) + ) + + table_span_rows = [] + table_spans_by_table = defaultdict(list) + for (table_id, file_id), (minimum, maximum, rows) in sorted( + table_spans.items(), key=lambda item: (item[0][0], item[1][0], item[0][1]) + ): + span_id = len(table_span_rows) + table_span_rows.append((table_id, file_id, minimum, maximum, rows)) + table_spans_by_table[table_id].append(span_id) + + device_name_rows = [] + column_name_rows = [] + table_rows = [] + table_name_rows = [] + for table_name in table_names: + table_id = table_ids[table_name] + table = index.table_entries[table_name] + name_sid = pool.intern(table_name) + table_name_rows.append( + (name_hash(table_name.encode("utf-8")), name_sid, table_id) + ) + first_device = len(device_name_rows) + names = sorted( + device_names_by_table[table_id], + key=lambda item: ( + name_hash(item[0].encode("utf-8")), + item[0].encode("utf-8"), + ), + ) + for name, device_id, sid in names: + device_name_rows.append( + (table_id, device_id, name_hash(name.encode("utf-8")), sid, 0) + ) + first_column = len(column_name_rows) + column_names = sorted( + columns_by_table[table_id], + key=lambda item: ( + name_hash(item[0].encode("utf-8")), + item[0].encode("utf-8"), + ), + ) + for name, column_id in column_names: + column_name_rows.append( + ( + table_id, + column_id, + name_hash(name.encode("utf-8")), + pool.intern(name), + 0, + ) + ) + spans = table_spans_by_table[table_id] + first_span = spans[0] if spans else 0 + field_type_map = { + name: field_types.get((table_name, name), -1) + for name in table.field_columns + } + table_rows.append( + ( + name_sid, + 0, + _schema_fingerprint(table, field_type_map), + first_device, + len(names), + first_column, + len(column_names), + first_span, + len(spans), + 0, + ) + ) + table_name_rows.sort(key=lambda row: (row[0], pool._values[row[1]], row[2])) + + file_rows = [] + for file_id, path in enumerate(file_paths): + st = os.stat(path) + minimum, maximum = file_bounds[file_id] + if minimum is None: + minimum = maximum = 0 + file_rows.append( + ( + pool.intern(path), + 0, + st.st_size, + minimum, + maximum, + file_fingerprint(path, st), + 0, + ) + ) + + string_offsets, string_bytes = pool.sections() + return { + STRING_OFFSETS: string_offsets, + STRING_BYTES: string_bytes, + TABLE_NAME_INDEX: _pack_records(TABLE_NAME_INDEX, table_name_rows), + TABLE_RECORD: _pack_records(TABLE_RECORD, table_rows), + DEVICE_NAME_INDEX: _pack_records(DEVICE_NAME_INDEX, device_name_rows), + DEVICE_RECORD: _pack_records(DEVICE_RECORD, device_rows), + COLUMN_NAME_INDEX: _pack_records(COLUMN_NAME_INDEX, column_name_rows), + COLUMN_SCHEMA: _pack_records(COLUMN_SCHEMA, columns), + LOGICAL_SERIES: _pack_records(LOGICAL_SERIES, logical_series_rows), + TSFILE_RECORD: _pack_records(TSFILE_RECORD, file_rows), + TABLE_FILE_SPAN: _pack_records(TABLE_FILE_SPAN, table_span_rows), + DEVICE_FILE_SPAN: _pack_records(DEVICE_FILE_SPAN, device_span_rows), + SERIES_FILE_SPAN: _pack_records(SERIES_FILE_SPAN, series_span_rows), + SERIES_LOCATOR: _pack_records(SERIES_LOCATOR, locator_rows), + } + + +def build_index_from_dataframe(dataframe, path: Optional[str] = None) -> str: + path = index_path_for(dataframe._paths) if path is None else path + write_index_atomic(path, build_sections_from_dataframe(dataframe)) + return path diff --git a/python/tsfile/dataset/merge.py b/python/tsfile/dataset/merge.py index 8d70dc552..8762728d9 100644 --- a/python/tsfile/dataset/merge.py +++ b/python/tsfile/dataset/merge.py @@ -24,11 +24,12 @@ - duplicate timestamps for the same logical series across shards are rejected. """ -import heapq from typing import Dict, List, Tuple import numpy as np +from ._merge import merge_time_value_parts_overlap, merge_timestamp_parts_overlap + def merge_timestamp_parts( time_parts: List[np.ndarray], @@ -49,39 +50,7 @@ def merge_timestamp_parts( ): return np.concatenate(parts) - total_length = sum(len(ts_part) for ts_part in parts) - merged = np.empty(total_length, dtype=np.int64) - - heap = [(int(ts_part[0]), part_idx, 0) for part_idx, ts_part in enumerate(parts)] - heapq.heapify(heap) - - out_idx = 0 - last_ts = None - while heap: - ts, part_idx, offset = heapq.heappop(heap) - - if last_ts is not None and ts == last_ts: - if validate_unique: - raise ValueError(f"Duplicate timestamp {ts} found across shards.") - if not deduplicate: - merged[out_idx] = ts - out_idx += 1 - else: - merged[out_idx] = ts - out_idx += 1 - last_ts = ts - - next_offset = offset + 1 - if next_offset < len(parts[part_idx]): - heapq.heappush( - heap, (int(parts[part_idx][next_offset]), part_idx, next_offset) - ) - - if validate_unique: - return merged[:out_idx] - if deduplicate: - return merged[:out_idx] - return merged[:out_idx] + return merge_timestamp_parts_overlap(parts, deduplicate, validate_unique) def merge_time_value_parts( @@ -117,29 +86,7 @@ def merge_time_value_parts( ): return np.concatenate(time_parts), np.concatenate(value_parts) - total_length = sum(len(ts_part) for ts_part in time_parts) - merged_ts = np.empty(total_length, dtype=np.int64) - merged_vals = np.empty(total_length, dtype=np.float64) - - heap = [ - (int(ts_part[0]), part_idx, 0) for part_idx, ts_part in enumerate(time_parts) - ] - heapq.heapify(heap) - - out_idx = 0 - while heap: - _, part_idx, offset = heapq.heappop(heap) - merged_ts[out_idx] = time_parts[part_idx][offset] - merged_vals[out_idx] = value_parts[part_idx][offset] - out_idx += 1 - - next_offset = offset + 1 - if next_offset < len(time_parts[part_idx]): - heapq.heappush( - heap, (int(time_parts[part_idx][next_offset]), part_idx, next_offset) - ) - - return merged_ts, merged_vals + return merge_time_value_parts_overlap(time_parts, value_parts) def build_aligned_matrix( diff --git a/python/tsfile/dataset/metadata.py b/python/tsfile/dataset/metadata.py index 5eb367273..9f1598fee 100644 --- a/python/tsfile/dataset/metadata.py +++ b/python/tsfile/dataset/metadata.py @@ -41,6 +41,15 @@ class SeriesStats(NamedTuple): timeline_length: int timeline_min_time: int timeline_max_time: int + data_type: int = -1 + value_metadata_offset: int = 0 + value_metadata_length: int = 0 + time_metadata_offset: int = 0 + time_metadata_length: int = 0 + chunk_meta_count: int = 0 + time_chunk_meta_count: int = 0 + layout: int = 0 + locator_flags: int = 0 @dataclass(**_DATACLASS_SLOTS) diff --git a/python/tsfile/dataset/reader.py b/python/tsfile/dataset/reader.py index 9b77190e1..5ec95f241 100644 --- a/python/tsfile/dataset/reader.py +++ b/python/tsfile/dataset/reader.py @@ -419,6 +419,15 @@ def _metadata_field_stats(group) -> Dict[str, SeriesStats]: timeline_length=int(timeline_statistic.row_count), timeline_min_time=int(timeline_statistic.start_time), timeline_max_time=int(timeline_statistic.end_time), + data_type=int(timeseries.data_type), + value_metadata_offset=int(timeseries.value_metadata_offset), + value_metadata_length=int(timeseries.value_metadata_length), + time_metadata_offset=int(timeseries.time_metadata_offset), + time_metadata_length=int(timeseries.time_metadata_length), + chunk_meta_count=int(timeseries.chunk_meta_count), + time_chunk_meta_count=int(timeseries.time_chunk_meta_count), + layout=int(timeseries.layout), + locator_flags=int(timeseries.locator_flags), ) return stats diff --git a/python/tsfile/dataset/runtime.py b/python/tsfile/dataset/runtime.py new file mode 100644 index 000000000..31fc65d04 --- /dev/null +++ b/python/tsfile/dataset/runtime.py @@ -0,0 +1,654 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""Process-local Runtime over the persistent Dataset Index.""" + +from __future__ import annotations + +from collections import OrderedDict +from collections.abc import Mapping, Sequence +import contextlib +import os +import threading +from typing import Dict, Optional + +import numpy as np + +from ..constants import TSDataType +from ..tag_filter import tag_eq, tag_is_null +from ..tsfile_reader import TsFileReaderPy +from .index import ( + COLUMN_NAME_INDEX, + COLUMN_SCHEMA, + DEVICE_FILE_SPAN, + DEVICE_RECORD, + LOGICAL_SERIES, + SERIES_FILE_SPAN, + SERIES_LOCATOR, + TABLE_RECORD, + TSFILE_RECORD, + MappedDatasetIndex, + file_fingerprint, +) +from .metadata import MODEL_TABLE, MODEL_TREE, TableEntry, split_logical_series_path + + +def _exact_tag_filter(tag_columns, tag_values): + result = None + for index, name in enumerate(tag_columns): + value = tag_values[index] if index < len(tag_values) else None + current = tag_is_null(name) if value is None else tag_eq(name, str(value)) + result = current if result is None else result & current + return result + + +class RuntimeLease: + def __init__(self, runtime: "DatasetRuntime"): + self._runtime = runtime + self._closed = False + runtime._acquire_object() + + def clone(self): + if self._closed: + raise RuntimeError("Runtime lease is closed") + return RuntimeLease(self._runtime) + + def close(self): + if not self._closed: + self._closed = True + self._runtime._release_object() + + +class _QueryLease: + def __init__(self, runtime: "DatasetRuntime"): + self._runtime = runtime + + def __enter__(self): + self._runtime._acquire_query() + return self + + def __exit__(self, *_): + self._runtime._release_query() + + +class _ReaderSession: + def __init__(self, file_id: int, path: str, expected_size: int, fingerprint: int): + st = os.stat(path) + if st.st_size != expected_size or file_fingerprint(path, st) != fingerprint: + raise RuntimeError( + f"TsFile generation changed after Dataset Index publication: {path}" + ) + self.file_id = file_id + self.path = path + self.reader = TsFileReaderPy(path) + self.active_uses = 0 + + def close(self): + self.reader.close() + + +class ReaderSessionPool: + """Per-Runtime LRU pool with a hard cap on simultaneously open Readers.""" + + def __init__(self, index: MappedDatasetIndex, max_open_files: int): + self._index = index + self.max_open_files = max(1, int(max_open_files)) + self._sessions: "OrderedDict[int, _ReaderSession]" = OrderedDict() + self._condition = threading.Condition() + self._closed = False + + def _new_session(self, file_id: int): + record = self._index.record(TSFILE_RECORD, file_id) + return _ReaderSession( + file_id, + self._index.string(record[0]), + record[2], + record[5], + ) + + @contextlib.contextmanager + def acquire(self, file_id: int): + with self._condition: + while True: + if self._closed: + raise RuntimeError("ReaderSessionPool is closed") + session = self._sessions.get(file_id) + if session is not None: + self._sessions.move_to_end(file_id) + session.active_uses += 1 + break + if len(self._sessions) < self.max_open_files: + session = self._new_session(file_id) + session.active_uses = 1 + self._sessions[file_id] = session + break + idle_id = next( + ( + key + for key, value in self._sessions.items() + if value.active_uses == 0 + ), + None, + ) + if idle_id is None: + self._condition.wait() + continue + idle = self._sessions.pop(idle_id) + idle.close() + session = self._new_session(file_id) + session.active_uses = 1 + self._sessions[file_id] = session + break + try: + yield session.reader + finally: + with self._condition: + session.active_uses -= 1 + self._condition.notify_all() + + def close(self): + with self._condition: + self._closed = True + while any(session.active_uses for session in self._sessions.values()): + self._condition.wait() + sessions = list(self._sessions.values()) + self._sessions.clear() + for session in sessions: + session.close() + + @property + def open_count(self): + with self._condition: + return len(self._sessions) + + +class PreparedSeriesCache: + """Runtime-wide single-flight cache of native exact-locator metadata.""" + + def __init__(self, index: MappedDatasetIndex): + self._index = index + self._condition = threading.Condition() + self._entries = {} + self._loading = set() + self._closed = False + + def _locator_tuple(self, file_id, locator_id): + locator = self._index.record(SERIES_LOCATOR, locator_id) + device_span = self._index.record(DEVICE_FILE_SPAN, locator[0]) + file_record = self._index.record(TSFILE_RECORD, file_id) + if device_span[1] != file_id: + raise ValueError("series locator points at another TsFile") + return ( + id(self._index), + file_id, + file_record[2], + file_record[5], + locator_id, + locator[1], + locator[2], + locator[3], + locator[4], + device_span[2], + device_span[3], + locator[5], + ) + + def get(self, file_id, locator_id, reader): + key = (id(self._index), file_id, locator_id) + with self._condition: + while True: + if self._closed: + raise RuntimeError("PreparedSeriesCache is closed") + result = self._entries.get(key) + if result is not None: + return result + if key not in self._loading: + self._loading.add(key) + break + self._condition.wait() + try: + result = reader.prepare_series(self._locator_tuple(file_id, locator_id)) + except Exception: + with self._condition: + self._loading.remove(key) + self._condition.notify_all() + raise + with self._condition: + if self._closed: + result.close() + self._loading.remove(key) + self._condition.notify_all() + raise RuntimeError("PreparedSeriesCache is closed") + self._entries[key] = result + self._loading.remove(key) + self._condition.notify_all() + return result + + def close(self): + with self._condition: + self._closed = True + while self._loading: + self._condition.wait() + entries = list(self._entries.values()) + self._entries.clear() + for prepared in entries: + prepared.close() + + @property + def size(self): + with self._condition: + return len(self._entries) + + +class DatasetRuntime: + def __init__(self, path: str, max_open_files: Optional[int] = None): + self.index = MappedDatasetIndex(path) + maximum = ( + int(os.environ.get("TSFILE_DATAFRAME_MAX_OPEN_FILES", "16")) + if max_open_files is None + else max_open_files + ) + self.readers = ReaderSessionPool(self.index, maximum) + self.prepared = PreparedSeriesCache(self.index) + self._condition = threading.Condition() + self._object_leases = 0 + self._query_leases = 0 + self._accepting = True + self._torn_down = False + self.catalog = MappedDataFrameCatalog(self) + + def lease(self): + return RuntimeLease(self) + + def query_lease(self): + return _QueryLease(self) + + def _acquire_object(self): + with self._condition: + if not self._accepting: + raise RuntimeError("Dataset Runtime is closing") + self._object_leases += 1 + + def _release_object(self): + teardown = False + with self._condition: + self._object_leases -= 1 + if self._object_leases == 0: + self._accepting = False + while self._query_leases: + self._condition.wait() + teardown = not self._torn_down + self._torn_down = True + if teardown: + self.prepared.close() + self.readers.close() + self.index.close() + + def _acquire_query(self): + with self._condition: + if not self._accepting: + raise RuntimeError("Dataset Runtime is closing") + self._query_leases += 1 + + def _release_query(self): + with self._condition: + self._query_leases -= 1 + self._condition.notify_all() + + +class _TableMapping(Mapping): + def __init__(self, catalog: "MappedDataFrameCatalog"): + self._catalog = catalog + self._names = {} + self._cache = {} + for table_id in range(catalog.index.count(TABLE_RECORD)): + record = catalog.index.record(TABLE_RECORD, table_id) + name = catalog.index.string(record[0]) + if name in self._names: + raise ValueError( + f"Schema variant for table '{name}' is ambiguous without a fingerprint" + ) + self._names[name] = table_id + + def __getitem__(self, name): + result = self._cache.get(name) + if result is not None: + return result + table_id = self._names[name] + table = self._catalog.index.record(TABLE_RECORD, table_id) + tags = [] + tag_types = [] + fields = [] + for name_index in self._catalog.index.records( + COLUMN_NAME_INDEX, table[5], table[6] + ): + column = self._catalog.index.record(COLUMN_SCHEMA, name_index[1]) + column_name = self._catalog.index.string(column[1]) + if column[7] == 0: + tags.append((column[2], column_name, TSDataType(column[3]))) + elif column[7] == 1: + fields.append((column[2], column_name)) + tags.sort() + fields.sort() + result = TableEntry( + name, + tuple(item[1] for item in tags), + tuple(item[2] for item in tags), + tuple(item[1] for item in fields), + ) + self._cache[name] = result + return result + + def __iter__(self): + return iter(self._names) + + def __len__(self): + return len(self._names) + + def table_id(self, name): + return self._names[name] + + +class _DeviceSequence(Sequence): + def __init__(self, catalog): + self._catalog = catalog + + def __len__(self): + return self._catalog.index.count(DEVICE_RECORD) + + def __getitem__(self, device_id): + if isinstance(device_id, slice): + return [self[index] for index in range(*device_id.indices(len(self)))] + record = self._catalog.index.record(DEVICE_RECORD, device_id) + table = self._catalog.index.record(TABLE_RECORD, record[0]) + table_name = self._catalog.index.string(table[0]) + components = split_logical_series_path(self._catalog.index.string(record[1])) + return table_name, tuple(components[1:-1]) + + +class _DeviceIndexMapping(Mapping): + def __init__(self, catalog): + self._catalog = catalog + + def get(self, key, default=None): + table_name, tags = key + table_ids = self._catalog.index.find_table_ids(table_name) + if len(table_ids) != 1: + return default + from .metadata import _join_series_path + + try: + return self._catalog.index.find_device_id( + table_ids[0], _join_series_path(table_name, tags, "") + ) + except KeyError: + return default + + def __getitem__(self, key): + result = self.get(key) + if result is None: + raise KeyError(key) + return result + + def __iter__(self): + return iter(self._catalog.devices) + + def __len__(self): + return len(self._catalog.devices) + + +class _SeriesSequence(Sequence): + def __init__(self, catalog, ids=None): + self._catalog = catalog + self._ids = ids + + def __len__(self): + return ( + self._catalog.index.count(LOGICAL_SERIES) + if self._ids is None + else len(self._ids) + ) + + def series_id(self, position): + return position if self._ids is None else self._ids[position] + + def __getitem__(self, position): + if isinstance(position, slice): + return [self[index] for index in range(*position.indices(len(self)))] + series_id = self.series_id(position) + series = self._catalog.index.record(LOGICAL_SERIES, series_id) + device = self._catalog.index.record(DEVICE_RECORD, series[0]) + table_name = self._catalog.index.string( + self._catalog.index.record(TABLE_RECORD, device[0])[0] + ) + table = self._catalog.table_entries[table_name] + column_name = self._catalog.index.string( + self._catalog.index.record(COLUMN_SCHEMA, series[1])[1] + ) + return series[0], table.get_field_index(column_name) + + def __iter__(self): + for index in range(len(self)): + yield self[index] + + +class _RouteMapping(Mapping): + def __init__(self, catalog): + self._catalog = catalog + + def _series_id(self, ref): + device_id, field_idx = ref + device = self._catalog.index.record(DEVICE_RECORD, device_id) + table_name = self._catalog.index.string( + self._catalog.index.record(TABLE_RECORD, device[0])[0] + ) + field_name = self._catalog.table_entries[table_name].field_columns[field_idx] + column_id = self._catalog.index.find_column_id(device[0], field_name) + return self._catalog.index.find_series_id(device_id, column_id) + + def __contains__(self, ref): + try: + self._series_id(ref) + return True + except KeyError: + return False + + def __getitem__(self, ref): + series_id = self._series_id(ref) + series = self._catalog.index.record(LOGICAL_SERIES, series_id) + result = [] + for span_id in range(series[2], series[2] + series[3]): + span = self._catalog.index.record(SERIES_FILE_SPAN, span_id) + result.append((self._catalog.reader_for(span[1]), series[0], series[1])) + return result + + def __iter__(self): + return iter(self._catalog.series) + + def __len__(self): + return len(self._catalog.series) + + +class MappedDataFrameCatalog: + def __init__(self, runtime: DatasetRuntime): + self.runtime = runtime + self.index = runtime.index + self.table_entries = _TableMapping(self) + self.devices = _DeviceSequence(self) + self.device_index = _DeviceIndexMapping(self) + self.device_time_bounds = _DeviceTimeBounds(self) + self.series = _SeriesSequence(self) + self.series_shards = _RouteMapping(self) + self._readers = {} + self.model = self._infer_model() + + def _infer_model(self): + if len(self.table_entries) != 1: + return MODEL_TABLE + table = next(iter(self.table_entries.values())) + if table.tag_columns == tuple( + f"_col_{index + 1}" for index in range(len(table.tag_columns)) + ): + return MODEL_TREE + return MODEL_TABLE + + def reader_for(self, file_id): + reader = self._readers.get(file_id) + if reader is None: + reader = RuntimeSeriesReader(self.runtime, file_id) + self._readers[file_id] = reader + return reader + + +class _DeviceTimeBounds(Sequence): + def __init__(self, catalog): + self._catalog = catalog + + def __len__(self): + return len(self._catalog.devices) + + def __getitem__(self, device_id): + record = self._catalog.index.record(DEVICE_RECORD, device_id) + return record[8], record[9] + + +class RuntimeSeriesReader: + """File-specific facade whose metadata comes from mmap, not a Python catalog.""" + + def __init__(self, runtime: DatasetRuntime, file_id: int): + self.runtime = runtime + self.file_id = file_id + + def _series(self, device_id, column_id): + return self.runtime.index.find_series_id(device_id, column_id) + + def _span(self, device_id, column_id): + series_id = self._series(device_id, column_id) + series = self.runtime.index.record(LOGICAL_SERIES, series_id) + for span_id in range(series[2], series[2] + series[3]): + span = self.runtime.index.record(SERIES_FILE_SPAN, span_id) + if span[1] == self.file_id: + return span + raise KeyError((device_id, column_id, self.file_id)) + + def _identity(self, device_id, column_id): + index = self.runtime.index + device = index.record(DEVICE_RECORD, device_id) + table = index.record(TABLE_RECORD, device[0]) + table_name = index.string(table[0]) + components = split_logical_series_path(index.string(device[1])) + tags = tuple(components[1:-1]) + table_entry = self.runtime.catalog.table_entries[table_name] + column_name = index.string(index.record(COLUMN_SCHEMA, column_id)[1]) + return table_name, tags, table_entry, column_name + + def get_device_info(self, device_id): + record = self.runtime.index.record(DEVICE_RECORD, device_id) + table_name, tags = self.runtime.catalog.devices[device_id] + table = self.runtime.catalog.table_entries[table_name] + return { + "table_name": table_name, + "tag_columns": table.tag_columns, + "tag_values": dict(zip(table.tag_columns, tags)), + "min_time": record[8], + "max_time": record[9], + } + + def get_series_info_by_ref(self, device_id, column_id): + span = self._span(device_id, column_id) + locator = self.runtime.index.record(SERIES_LOCATOR, span[2]) + device_span = self.runtime.index.record(DEVICE_FILE_SPAN, locator[0]) + table_name, tags, table, column_name = self._identity(device_id, column_id) + timeline_length = device_span[8] if device_span[4] == 1 else span[7] + return { + "length": timeline_length, + "min_time": span[4], + "max_time": span[5], + "timeline_length": timeline_length, + "timeline_min_time": span[4], + "timeline_max_time": span[5], + "table_name": table_name, + "column_name": column_name, + "device_id": device_id, + "field_idx": column_id, + "tag_columns": table.tag_columns, + "tag_values": dict(zip(table.tag_columns, tags)), + } + + @staticmethod + def _consume(result): + timestamps = [] + values = [] + with result: + while result.next(): + timestamp = result.get_value_by_index(1) + value = result.get_value_by_index(2) + timestamps.append(int(timestamp)) + values.append(np.nan if value is None else float(value)) + return np.asarray(timestamps, dtype=np.int64), np.asarray( + values, dtype=np.float64 + ) + + def _query( + self, + device_id, + column_id, + start_time=None, + end_time=None, + offset=None, + limit=None, + ): + span = self._span(device_id, column_id) + with self.runtime.query_lease(), self.runtime.readers.acquire( + self.file_id + ) as reader: + prepared = self.runtime.prepared.get(self.file_id, span[2], reader) + if offset is None: + result = reader.query_prepared( + prepared, start_time=start_time, end_time=end_time + ) + else: + result = reader.query_prepared(prepared, offset=offset, limit=limit) + return self._consume(result) + + def read_series_by_ref(self, device_id, column_id, start_time, end_time): + return self._query(device_id, column_id, start_time, end_time) + + def read_series_by_row(self, device_id, column_id, offset, limit): + if limit <= 0: + return np.array([], dtype=np.int64), np.array([], dtype=np.float64) + return self._query(device_id, column_id, offset=offset, limit=limit) + + def read_device_fields_by_time_range( + self, device_id, column_ids, start_time, end_time + ): + parts = [ + self.read_series_by_ref(device_id, column_id, start_time, end_time) + for column_id in column_ids + ] + if not parts: + return np.array([], dtype=np.int64), {} + # Existing dataframe merge aligns separate field parts across files; + # this method is only a compatibility surface for a single file. + timestamps = parts[0][0] + values = {} + for column_id, (current_timestamps, current_values) in zip(column_ids, parts): + if not np.array_equal(current_timestamps, timestamps): + raise ValueError("single-file fields do not share one timeline") + _, _, _, name = self._identity(device_id, column_id) + values[name] = current_values + return timestamps, values diff --git a/python/tsfile/dataset/timeseries.py b/python/tsfile/dataset/timeseries.py index 39d43b442..43acd44c0 100644 --- a/python/tsfile/dataset/timeseries.py +++ b/python/tsfile/dataset/timeseries.py @@ -76,9 +76,10 @@ def __init__( name: str, series_refs: list, stats: dict, - ensure_open: Callable[[], None], + ensure_open: Optional[Callable[[], None]], load_timestamps: Callable[[], np.ndarray], read_by_position: Callable[[int, int], Tuple[np.ndarray, np.ndarray]], + runtime_lease=None, ): self._name = name self._series_refs = series_refs @@ -87,6 +88,14 @@ def __init__( self._load_timestamps = load_timestamps self._read_by_position = read_by_position self._timestamps = None + self._runtime_lease = runtime_lease + self._closed = False + + def _assert_open(self): + if self._closed: + raise RuntimeError("Current Timeseries is closed.") + if self._ensure_open is not None: + self._ensure_open() @property def name(self) -> str: @@ -94,7 +103,7 @@ def name(self) -> str: @property def timestamps(self) -> np.ndarray: - self._ensure_open() + self._assert_open() if self._timestamps is None: self._timestamps = self._load_timestamps() return self._timestamps @@ -111,7 +120,7 @@ def __len__(self) -> int: return self._stats["count"] def __getitem__(self, key): - self._ensure_open() + self._assert_open() length = len(self) if isinstance(key, int): @@ -148,7 +157,7 @@ def __getitem__(self, key): def _query_time_range( self, start_time: int, end_time: int ) -> Tuple[np.ndarray, np.ndarray]: - self._ensure_open() + self._assert_open() time_parts = [] value_parts = [] for reader, device_id, field_idx in self._series_refs: @@ -166,6 +175,26 @@ def _query_time_range( value_parts.append(val_arr) return merge_time_value_parts(time_parts, value_parts) + def close(self): + if self._closed: + return + self._closed = True + if self._runtime_lease is not None: + self._runtime_lease.close() + + def __enter__(self): + self._assert_open() + return self + + def __exit__(self, *_): + self.close() + + def __del__(self): + try: + self.close() + except Exception: + pass + def __repr__(self): stats = self.stats if stats["count"] == 0: diff --git a/python/tsfile/schema.py b/python/tsfile/schema.py index fcfee5e99..6a8bdaa9b 100644 --- a/python/tsfile/schema.py +++ b/python/tsfile/schema.py @@ -100,6 +100,13 @@ class TimeseriesMetadata: chunk_meta_count: int statistic: TimeseriesStatisticType timeline_statistic: TimeseriesStatisticType + value_metadata_offset: int = 0 + value_metadata_length: int = 0 + time_metadata_offset: int = 0 + time_metadata_length: int = 0 + time_chunk_meta_count: int = 0 + layout: int = 0 + locator_flags: int = 0 @dataclass(frozen=True) diff --git a/python/tsfile/tsfile_cpp.pxd b/python/tsfile/tsfile_cpp.pxd index 4daf0b153..b0ebba31e 100644 --- a/python/tsfile/tsfile_cpp.pxd +++ b/python/tsfile/tsfile_cpp.pxd @@ -17,7 +17,7 @@ # #cython: language_level=3 -from libc.stdint cimport uint32_t, int32_t, int64_t, uint64_t, uint8_t +from libc.stdint cimport uint16_t, uint32_t, int32_t, int64_t, uint64_t, uint8_t ctypedef int32_t ErrorCode @@ -38,6 +38,7 @@ cdef extern from "cwrapper/tsfile_cwrapper.h": ctypedef void * Tablet ctypedef void * TsRecord ctypedef void * ResultSet + ctypedef void * PreparedSeriesHandle # enum types ctypedef enum TSDataType: @@ -166,6 +167,13 @@ cdef extern from "cwrapper/tsfile_cwrapper.h": int32_t chunk_meta_count TimeseriesStatistic statistic TimeseriesStatistic timeline_statistic + uint64_t value_metadata_offset + uint32_t value_metadata_length + uint64_t time_metadata_offset + uint32_t time_metadata_length + uint32_t time_chunk_meta_count + uint16_t layout + uint16_t locator_flags ctypedef struct DeviceID: char * path @@ -194,6 +202,20 @@ cdef extern from "cwrapper/tsfile_cwrapper.h": TSDataType * data_types int column_num + ctypedef struct TsFilePreparedLocator: + uint64_t mapped_index_identity + uint32_t file_id + uint64_t file_size + uint64_t file_fingerprint + uint32_t locator_id + uint16_t layout + uint16_t flags + uint64_t value_metadata_offset + uint32_t value_metadata_length + uint64_t time_metadata_offset + uint32_t time_metadata_length + uint32_t chunk_count_hint + # Function Declarations ctypedef void * TagFilterHandle @@ -270,6 +292,15 @@ cdef extern from "cwrapper/tsfile_cwrapper.h": const char** columns, uint32_t column_num, int64_t start_time, int64_t end_time, ErrorCode *err_code) + PreparedSeriesHandle tsfile_reader_prepare_series( + TsFileReader reader, const TsFilePreparedLocator * locator, + ErrorCode * err_code) + void tsfile_prepared_series_free(PreparedSeriesHandle prepared) + ResultSet tsfile_reader_query_prepared( + TsFileReader reader, PreparedSeriesHandle prepared, + int64_t start_time, int64_t end_time, int offset, int limit, + ErrorCode * err_code) + ResultSet tsfile_query_table_on_tree(TsFileReader reader, char** columns, uint32_t column_num, int64_t start_time, int64_t end_time, diff --git a/python/tsfile/tsfile_py_cpp.pxd b/python/tsfile/tsfile_py_cpp.pxd index 0e2f91cbc..bda07eb28 100644 --- a/python/tsfile/tsfile_py_cpp.pxd +++ b/python/tsfile/tsfile_py_cpp.pxd @@ -43,6 +43,11 @@ cdef public api void free_c_tablet(Tablet tablet) cdef public api void free_c_row_record(TsRecord record) cdef public api TsFileWriter tsfile_writer_new_c(object pathname, uint64_t memory_threshold) except NULL cdef public api TsFileReader tsfile_reader_new_c(object pathname) except NULL +cdef public api PreparedSeriesHandle tsfile_reader_prepare_series_c( + TsFileReader reader, object locator) except NULL +cdef public api ResultSet tsfile_reader_query_prepared_c( + TsFileReader reader, PreparedSeriesHandle prepared, int64_t start_time, + int64_t end_time, int offset, int limit) cdef public api ErrorCode tsfile_writer_register_device_py_cpp(TsFileWriter writer, DeviceSchema *schema) cdef public api ErrorCode tsfile_writer_register_timeseries_py_cpp(TsFileWriter writer, object device_name, TimeseriesSchema *schema) diff --git a/python/tsfile/tsfile_py_cpp.pyx b/python/tsfile/tsfile_py_cpp.pyx index 936c74af1..c5182c0ce 100644 --- a/python/tsfile/tsfile_py_cpp.pyx +++ b/python/tsfile/tsfile_py_cpp.pyx @@ -783,6 +783,36 @@ cdef TsFileReader tsfile_reader_new_c(object pathname) except NULL: check_error(errno) return reader +cdef PreparedSeriesHandle tsfile_reader_prepare_series_c( + TsFileReader reader, object locator) except NULL: + cdef TsFilePreparedLocator native + cdef ErrorCode code = 0 + native.mapped_index_identity = locator[0] + native.file_id = locator[1] + native.file_size = locator[2] + native.file_fingerprint = locator[3] + native.locator_id = locator[4] + native.layout = locator[5] + native.flags = locator[6] + native.value_metadata_offset = locator[7] + native.value_metadata_length = locator[8] + native.time_metadata_offset = locator[9] + native.time_metadata_length = locator[10] + native.chunk_count_hint = locator[11] + cdef PreparedSeriesHandle prepared = tsfile_reader_prepare_series( + reader, &native, &code) + check_error(code, b"Failed to prepare Dataset Index locator") + return prepared + +cdef ResultSet tsfile_reader_query_prepared_c( + TsFileReader reader, PreparedSeriesHandle prepared, + int64_t start_time, int64_t end_time, int offset, int limit): + cdef ErrorCode code = 0 + cdef ResultSet result = tsfile_reader_query_prepared( + reader, prepared, start_time, end_time, offset, limit, &code) + check_error(code, b"Failed to query prepared series") + return result + cpdef object get_tsfile_config(): return { "tsblock_mem_inc_step_size_": g_config_value_.tsblock_mem_inc_step_size_, @@ -1257,6 +1287,13 @@ cdef object timeseries_metadata_c_to_py(TimeseriesMetadata* m): int(m.chunk_meta_count), stat, timeline_stat, + int(m.value_metadata_offset), + int(m.value_metadata_length), + int(m.time_metadata_offset), + int(m.time_metadata_length), + int(m.time_chunk_meta_count), + int(m.layout), + int(m.locator_flags), ) cdef tuple c_device_segments_to_tuple(char** segs, uint32_t n): diff --git a/python/tsfile/tsfile_reader.pyx b/python/tsfile/tsfile_reader.pyx index a2e8fe263..e5e9ecc58 100644 --- a/python/tsfile/tsfile_reader.pyx +++ b/python/tsfile/tsfile_reader.pyx @@ -309,6 +309,30 @@ cdef class ResultSetPy: def __exit__(self, exc_type, exc_val, exc_tb): self.close() +cdef class PreparedSeriesPy: + """Reusable native metadata parsed from one exact Dataset Index locator.""" + cdef PreparedSeriesHandle prepared + + def __cinit__(self): + self.prepared = NULL + + cdef init_c(self, PreparedSeriesHandle prepared): + self.prepared = prepared + + def close(self): + if self.prepared != NULL: + tsfile_prepared_series_free(self.prepared) + self.prepared = NULL + + def __dealloc__(self): + self.close() + + def __enter__(self): + return self + + def __exit__(self, exc_type, exc_val, exc_tb): + self.close() + cdef class TsFileReaderPy: """ Cython wrapper class for interacting with TsFileReader C implementation. @@ -470,6 +494,29 @@ cdef class TsFileReaderPy: self.activate_result_set_list.add(pyresult) return pyresult + def prepare_series(self, locator) -> PreparedSeriesPy: + """Prepare a 12-field native locator tuple for repeated queries.""" + if len(locator) != 12: + raise ValueError("prepared locator must contain exactly 12 fields") + cdef PreparedSeriesHandle prepared = tsfile_reader_prepare_series_c( + self.reader, locator) + py_prepared = PreparedSeriesPy() + py_prepared.init_c(prepared) + return py_prepared + + def query_prepared(self, PreparedSeriesPy prepared, + start_time : int = INT64_MIN, + end_time : int = INT64_MAX, + offset : int = 0, limit : int = -1) -> ResultSetPy: + if prepared.prepared == NULL: + raise RuntimeError("PreparedSeries is closed") + cdef ResultSet result = tsfile_reader_query_prepared_c( + self.reader, prepared.prepared, start_time, end_time, offset, limit) + pyresult = ResultSetPy(self, True) + pyresult.init_c(result, "prepared") + self.activate_result_set_list.add(pyresult) + return pyresult + def notify_result_set_discard(self, result_set: ResultSetPy): """ Remove activate result set from activate_result_set_list, called when a result set close.