Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,8 @@ python/tsfile/*dll*
python/tsfile/*dylib*
python/tsfile/*.h
python/tsfile/*.cpp
python/tsfile/**/*.cpp
python/tsfile/**/*so*
python/data
python/venv/*
python/tests/__pycache__/*
Expand Down
9 changes: 5 additions & 4 deletions cpp/src/CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -118,6 +118,7 @@ endif()
add_subdirectory(common)
add_subdirectory(compress)
add_subdirectory(cwrapper)
add_subdirectory(dataset)
add_subdirectory(encoding)
add_subdirectory(file)
add_subdirectory(reader)
Expand Down Expand Up @@ -173,16 +174,16 @@ if (${COV_ENABLED})
set(COV_LINK_LIB -lgcov)
endif()
if (ENABLE_ANTLR4)
target_link_libraries(tsfile common_obj compress_obj cwrapper_obj file_obj read_obj write_obj parser_obj ${COV_LINK_LIB})
target_link_libraries(tsfile common_obj compress_obj cwrapper_obj dataset_obj file_obj read_obj write_obj parser_obj ${COV_LINK_LIB})
else()
target_link_libraries(tsfile common_obj compress_obj cwrapper_obj file_obj read_obj write_obj ${COV_LINK_LIB})
target_link_libraries(tsfile common_obj compress_obj cwrapper_obj dataset_obj file_obj read_obj write_obj ${COV_LINK_LIB})
endif()
else()
message("Disable code cov...")
if (ENABLE_ANTLR4)
target_link_libraries(tsfile common_obj compress_obj cwrapper_obj file_obj read_obj write_obj parser_obj)
target_link_libraries(tsfile common_obj compress_obj cwrapper_obj dataset_obj file_obj read_obj write_obj parser_obj)
else()
target_link_libraries(tsfile common_obj compress_obj cwrapper_obj file_obj read_obj write_obj)
target_link_libraries(tsfile common_obj compress_obj cwrapper_obj dataset_obj file_obj read_obj write_obj)
endif()
endif()

Expand Down
22 changes: 22 additions & 0 deletions cpp/src/common/tsfile_common.h
Original file line number Diff line number Diff line change
Expand Up @@ -351,6 +351,8 @@ class TimeseriesIndex : public ITimeseriesIndex {
TimeseriesIndex()
: timeseries_meta_type_((char)255),
chunk_meta_list_data_size_(0),
metadata_offset_(-1),
metadata_length_(0),
measurement_name_(),
data_type_(common::INVALID_DATATYPE),
statistic_(nullptr),
Expand All @@ -369,6 +371,8 @@ class TimeseriesIndex : public ITimeseriesIndex {
{
timeseries_meta_type_ = 0;
chunk_meta_list_data_size_ = 0;
metadata_offset_ = -1;
metadata_length_ = 0;
measurement_name_.reset();
data_type_ = common::VECTOR;
chunk_meta_list_serialized_buf_.reset();
Expand Down Expand Up @@ -401,6 +405,16 @@ class TimeseriesIndex : public ITimeseriesIndex {
FORCE_INLINE virtual common::TSDataType get_data_type() const {
return data_type_;
}
FORCE_INLINE void set_metadata_range(int64_t offset, uint32_t length) {
metadata_offset_ = offset;
metadata_length_ = length;
}
FORCE_INLINE int64_t get_metadata_offset() const {
return metadata_offset_;
}
FORCE_INLINE uint32_t get_metadata_length() const {
return metadata_length_;
}
int init_statistic(common::TSDataType data_type) {
if (statistic_ != nullptr &&
!statistic_from_pa_) { // clear old statistic
Expand Down Expand Up @@ -488,6 +502,8 @@ class TimeseriesIndex : public ITimeseriesIndex {
int ret = common::E_OK;
timeseries_meta_type_ = that.timeseries_meta_type_;
chunk_meta_list_data_size_ = that.chunk_meta_list_data_size_;
metadata_offset_ = that.metadata_offset_;
metadata_length_ = that.metadata_length_;
data_type_ = that.data_type_;

statistic_ = StatisticFactory::alloc_statistic_with_pa(data_type_, pa);
Expand Down Expand Up @@ -560,6 +576,12 @@ class TimeseriesIndex : public ITimeseriesIndex {
// Sum of chunk meta serialized size in List<ChunkMeta> of this timeseries.
uint32_t chunk_meta_list_data_size_;

// Exact byte range of this TimeseriesMetadata in the source TsFile.
// It is assigned by TsFileIOReader after deserialization and is not part
// of the on-wire TimeseriesMetadata encoding.
int64_t metadata_offset_;
uint32_t metadata_length_;

// std::string measurement_name_;
common::String measurement_name_;
common::TSDataType data_type_;
Expand Down
100 changes: 100 additions & 0 deletions cpp/src/cwrapper/tsfile_cwrapper.cc
Original file line number Diff line number Diff line change
Expand Up @@ -419,6 +419,67 @@ ERRNO tsfile_writer_write(TsFileWriter writer, Tablet tablet) {

// Query

PreparedSeriesHandle tsfile_reader_prepare_series(
TsFileReader reader, const TsFilePreparedLocator* locator,
ERRNO* err_code) {
if (err_code == nullptr) {
return nullptr;
}
*err_code = common::E_INVALID_ARG;
if (reader == nullptr || locator == nullptr) {
return nullptr;
}
storage::FileGeneration generation;
generation.mapped_index_identity = locator->mapped_index_identity;
generation.file_id = locator->file_id;
generation.file_size = locator->file_size;
generation.file_fingerprint = locator->file_fingerprint;
storage::PreparedLocator native_locator;
native_locator.locator_id = locator->locator_id;
native_locator.layout = locator->layout;
native_locator.flags = locator->flags;
native_locator.value_metadata_offset = locator->value_metadata_offset;
native_locator.value_metadata_length = locator->value_metadata_length;
native_locator.time_metadata_offset = locator->time_metadata_offset;
native_locator.time_metadata_length = locator->time_metadata_length;
native_locator.chunk_count_hint = locator->chunk_count_hint;
std::shared_ptr<storage::PreparedSeries> prepared;
*err_code = static_cast<storage::TsFileReader*>(reader)->prepare_series(
generation, native_locator, prepared);
if (*err_code != common::E_OK) {
return nullptr;
}
auto* handle = new (std::nothrow)
std::shared_ptr<storage::PreparedSeries>(std::move(prepared));
if (handle == nullptr) {
*err_code = common::E_OOM;
}
return handle;
}

void tsfile_prepared_series_free(PreparedSeriesHandle prepared) {
delete static_cast<std::shared_ptr<storage::PreparedSeries>*>(prepared);
}

ResultSet tsfile_reader_query_prepared(TsFileReader reader,
PreparedSeriesHandle prepared,
Timestamp start_time, Timestamp end_time,
int offset, int limit, ERRNO* err_code) {
if (err_code == nullptr) {
return nullptr;
}
*err_code = common::E_INVALID_ARG;
if (reader == nullptr || prepared == nullptr) {
return nullptr;
}
auto* handle =
static_cast<std::shared_ptr<storage::PreparedSeries>*>(prepared);
storage::ResultSet* result = nullptr;
*err_code = static_cast<storage::TsFileReader*>(reader)->query_prepared(
*handle, start_time, end_time, offset, limit, result);
return result;
}

ResultSet tsfile_query_table(TsFileReader reader, const char* table_name,
char** columns, uint32_t column_num,
Timestamp start_time, Timestamp end_time,
Expand Down Expand Up @@ -1305,8 +1366,32 @@ ERRNO populate_c_metadata_map_from_cpp(
aligned_idx->value_ts_idx_ != nullptr) {
m.data_type = static_cast<TSDataType>(
aligned_idx->value_ts_idx_->get_data_type());
const storage::TimeseriesIndex* value_idx =
aligned_idx->value_ts_idx_;
const storage::TimeseriesIndex* time_idx =
aligned_idx->time_ts_idx_;
if (value_idx->get_metadata_offset() >= 0) {
m.value_metadata_offset =
static_cast<uint64_t>(value_idx->get_metadata_offset());
m.value_metadata_length = value_idx->get_metadata_length();
}
if (time_idx != nullptr &&
time_idx->get_metadata_offset() >= 0) {
m.time_metadata_offset =
static_cast<uint64_t>(time_idx->get_metadata_offset());
m.time_metadata_length = time_idx->get_metadata_length();
}
m.layout = 1;
} else {
m.data_type = static_cast<TSDataType>(idx->get_data_type());
const storage::TimeseriesIndex* value_idx =
dynamic_cast<const storage::TimeseriesIndex*>(idx.get());
if (value_idx != nullptr &&
value_idx->get_metadata_offset() >= 0) {
m.value_metadata_offset =
static_cast<uint64_t>(value_idx->get_metadata_offset());
m.value_metadata_length = value_idx->get_metadata_length();
}
}
storage::Statistic* st = idx->get_statistic();
int32_t chunk_cnt = 0;
Expand All @@ -1316,6 +1401,21 @@ ERRNO populate_c_metadata_map_from_cpp(
chunk_cnt = static_cast<int32_t>(cl->size());
}
m.chunk_meta_count = chunk_cnt;
if (chunk_cnt >= 0 && m.value_metadata_length > 0) {
m.locator_flags |= 1;
}
if (aligned_idx != nullptr) {
auto* time_chunks = idx->get_time_chunk_meta_list();
if (time_chunks != nullptr) {
m.time_chunk_meta_count =
static_cast<uint32_t>(time_chunks->size());
}
if (m.time_metadata_length == 0 ||
m.time_chunk_meta_count !=
static_cast<uint32_t>(chunk_cnt)) {
m.locator_flags &= ~static_cast<uint16_t>(1);
}
}
const int st_rc = fill_timeseries_statistic(st, &m.statistic);
if (st_rc != common::E_OK) {
for (uint32_t u = 0; u < slot; u++) {
Expand Down
37 changes: 37 additions & 0 deletions cpp/src/cwrapper/tsfile_cwrapper.h
Original file line number Diff line number Diff line change
Expand Up @@ -194,6 +194,13 @@ typedef struct TimeseriesMetadata {
int32_t chunk_meta_count;
TimeseriesStatistic statistic;
TimeseriesStatistic timeline_statistic;
uint64_t value_metadata_offset;
uint32_t value_metadata_length;
uint64_t time_metadata_offset;
uint32_t time_metadata_length;
uint32_t time_chunk_meta_count;
uint16_t layout;
uint16_t locator_flags;
} TimeseriesMetadata;

/**
Expand Down Expand Up @@ -269,6 +276,22 @@ typedef void* TsRecord;

typedef void* ResultSet;
typedef void* TagFilterHandle;
typedef void* PreparedSeriesHandle;

typedef struct TsFilePreparedLocator {
uint64_t mapped_index_identity;
uint32_t file_id;
uint64_t file_size;
uint64_t file_fingerprint;
uint32_t locator_id;
uint16_t layout;
uint16_t flags;
uint64_t value_metadata_offset;
uint32_t value_metadata_length;
uint64_t time_metadata_offset;
uint32_t time_metadata_length;
uint32_t chunk_count_hint;
} TsFilePreparedLocator;

typedef struct arrow_schema {
// Array type description
Expand Down Expand Up @@ -656,6 +679,20 @@ ERRNO tsfile_writer_write(TsFileWriter writer, Tablet tablet);

/*-------------------TsFile reader query data------------------ */

/** Deserialize one exact Dataset Index locator into a reusable series. */
PreparedSeriesHandle tsfile_reader_prepare_series(
TsFileReader reader, const TsFilePreparedLocator* locator, ERRNO* err_code);

/** Release a prepared handle. Existing result sets remain independently owned.
*/
void tsfile_prepared_series_free(PreparedSeriesHandle prepared);

/** Query a prepared series without traversing the TsFile footer index. */
ResultSet tsfile_reader_query_prepared(TsFileReader reader,
PreparedSeriesHandle prepared,
Timestamp start_time, Timestamp end_time,
int offset, int limit, ERRNO* err_code);

/**
* @brief Queries time series data from a specific table within time range.
*
Expand Down
25 changes: 25 additions & 0 deletions cpp/src/dataset/CMakeLists.txt
Original file line number Diff line number Diff line change
@@ -0,0 +1,25 @@
#[[
Licensed to the Apache Software Foundation (ASF) under one
or more contributor license agreements. See the NOTICE file
distributed with this work for additional information
regarding copyright ownership. The ASF licenses this file
to you under the Apache License, Version 2.0 (the
"License"); you may not use this file except in compliance
with the License. You may obtain a copy of the License at

https://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing,
software distributed under the License is distributed on an
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
KIND, either express or implied. See the License for the
specific language governing permissions and limitations
under the License.
]]
message("Running in src/dataset directory")
set(CMAKE_POSITION_INDEPENDENT_CODE ON)
aux_source_directory(${CMAKE_CURRENT_SOURCE_DIR} dataset_SRC_LIST)
add_library(dataset_obj OBJECT ${dataset_SRC_LIST})

file(GLOB HEADERS "${CMAKE_CURRENT_SOURCE_DIR}/*.h")
copy_to_dir(${HEADERS} "dataset_obj")
Loading
Loading