From 0f9b46ebb9839f97f29633249fdc8ad8e8913030 Mon Sep 17 00:00:00 2001 From: 761417898 <761417898@qq.com> Date: Wed, 8 Apr 2026 14:53:25 +0800 Subject: [PATCH 1/4] test: repro query_tree_by_row failure on three-segment device paths Add C++ cwrapper tests: two-segment root.device1 succeeds; three-segment root.db.device1 returns RET_DEVICRET_NOT_EXIST (44) for tsfile_reader_query_tree_by_row. Add Python pytest module for the same scenario (run after building the Python extension, e.g. mvn -P with-cpp,with-python). Made-with: Cursor --- cpp/test/cwrapper/cwrapper_test.cc | 84 ++++++++++++++ ...est_query_tree_by_row_three_level_repro.py | 105 ++++++++++++++++++ 2 files changed, 189 insertions(+) create mode 100644 python/tests/test_query_tree_by_row_three_level_repro.py diff --git a/cpp/test/cwrapper/cwrapper_test.cc b/cpp/test/cwrapper/cwrapper_test.cc index 5998939af..6a0949c4e 100644 --- a/cpp/test/cwrapper/cwrapper_test.cc +++ b/cpp/test/cwrapper/cwrapper_test.cc @@ -310,4 +310,88 @@ TEST_F(CWrapperTest, WriterFlushTabletAndReadData) { free(data_types); free_write_file(&file); } + +// Repro: query_tree_by_row fails with RET_DEVICRET_NOT_EXIST (44) for a +// three-segment device path (e.g. root.db.device1), while two-segment paths work. +// Root cause: Path(full_path) vs StringArrayDeviceID(device_string) split mismatch. +TEST_F(CWrapperTest, QueryTreeByRow_TwoSegmentDevice_Succeeds) { + ERRNO code = 0; + const char* filename = "cwrapper_query_tree_by_row_two_seg.tsfile"; + remove(filename); + + auto* writer = (storage::TsFileWriter*)_tsfile_writer_new( + filename, 128 * 1024 * 1024, &code); + ASSERT_OK(code); + + const char* device = "root.device1"; + timeseries_schema ts_schema{}; + ts_schema.timeseries_name = const_cast("s1"); + ts_schema.data_type = TS_DATATYPE_INT64; + ts_schema.encoding = TS_ENCODING_PLAIN; + ts_schema.compression = TS_COMPRESSION_UNCOMPRESSED; + + ASSERT_OK(_tsfile_writer_register_timeseries(writer, device, &ts_schema)); + + auto* record = (storage::TsRecord*)_ts_record_new(device, 0, 1); + ASSERT_OK(_insert_data_into_ts_record_by_name_int64_t(record, "s1", 42)); + ASSERT_OK(_tsfile_writer_write_ts_record(writer, record)); + _free_tsfile_ts_record(reinterpret_cast(&record)); + ASSERT_OK(_tsfile_writer_flush(writer)); + ASSERT_OK(_tsfile_writer_close(writer)); + + auto* reader = (storage::TsFileReader*)tsfile_reader_new(filename, &code); + ASSERT_OK(code); + + char* devs[] = {const_cast(device)}; + char* meas[] = {const_cast("s1")}; + ResultSet rs = tsfile_reader_query_tree_by_row(reader, devs, 1, meas, 1, 0, + 10, &code); + ASSERT_OK(code); + ASSERT_NE(rs, nullptr); + free_tsfile_result_set(&rs); + tsfile_reader_close(reader); + remove(filename); +} + +TEST_F(CWrapperTest, QueryTreeByRow_ThreeSegmentDevice_ReproducesDeviceNotExist) { + ERRNO code = 0; + const char* filename = "cwrapper_query_tree_by_row_three_seg.tsfile"; + remove(filename); + + auto* writer = (storage::TsFileWriter*)_tsfile_writer_new( + filename, 128 * 1024 * 1024, &code); + ASSERT_OK(code); + + const char* device = "root.db.device1"; + timeseries_schema ts_schema{}; + ts_schema.timeseries_name = const_cast("s1"); + ts_schema.data_type = TS_DATATYPE_INT64; + ts_schema.encoding = TS_ENCODING_PLAIN; + ts_schema.compression = TS_COMPRESSION_UNCOMPRESSED; + + ASSERT_OK(_tsfile_writer_register_timeseries(writer, device, &ts_schema)); + + auto* record = (storage::TsRecord*)_ts_record_new(device, 0, 1); + ASSERT_OK(_insert_data_into_ts_record_by_name_int64_t(record, "s1", 1)); + ASSERT_OK(_tsfile_writer_write_ts_record(writer, record)); + _free_tsfile_ts_record(reinterpret_cast(&record)); + ASSERT_OK(_tsfile_writer_flush(writer)); + ASSERT_OK(_tsfile_writer_close(writer)); + + auto* reader = (storage::TsFileReader*)tsfile_reader_new(filename, &code); + ASSERT_OK(code); + + char* devs[] = {const_cast(device)}; + char* meas[] = {const_cast("s1")}; + ResultSet rs = tsfile_reader_query_tree_by_row(reader, devs, 1, meas, 1, 0, + 10, &code); + EXPECT_EQ(rs, nullptr); + EXPECT_EQ(code, RET_DEVICRET_NOT_EXIST) + << "repro: three-segment device path should currently fail query_tree_by_row"; + if (rs != nullptr) { + free_tsfile_result_set(&rs); + } + tsfile_reader_close(reader); + remove(filename); +} } // namespace cwrapper \ No newline at end of file diff --git a/python/tests/test_query_tree_by_row_three_level_repro.py b/python/tests/test_query_tree_by_row_three_level_repro.py new file mode 100644 index 000000000..f6a3dbe24 --- /dev/null +++ b/python/tests/test_query_tree_by_row_three_level_repro.py @@ -0,0 +1,105 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +"""Reproduce DeviceNotExistError (44) for query_tree_by_row on 3-segment device paths.""" + +import os + +import pytest + +from tsfile import Field, RowRecord, TimeseriesSchema, TsFileReader, TsFileWriter +from tsfile import TSDataType +from tsfile.exceptions import DeviceNotExistError + + +def _remove_if_exists(path: str) -> None: + if os.path.exists(path): + os.remove(path) + + +def test_query_tree_by_row_two_level_device_ok(): + """Two-segment device path: query_tree_by_row should succeed.""" + path = "repro_query_tree_by_row_two_level.tsfile" + device = "root.device1" + measurements = ["s1"] + try: + _remove_if_exists(path) + writer = TsFileWriter(path) + writer.register_timeseries( + device, TimeseriesSchema("s1", TSDataType.INT64) + ) + writer.write_row_record( + RowRecord( + device, + 0, + [Field("s1", 42, TSDataType.INT64)], + ) + ) + writer.close() + + with TsFileReader(path) as reader: + schemas = reader.get_all_timeseries_schemas() + assert device.lower() in schemas + rs = reader.query_tree_by_row([device], measurements, 0, 10) + assert rs.next() + assert rs.get_value_by_index(2) == 42 + rs.close() + finally: + _remove_if_exists(path) + + +def test_repro_query_tree_by_row_three_level_device_not_exist(): + """ + Three-segment device path (e.g. root.db.device1): metadata lists the device, + but query_tree_by_row currently raises DeviceNotExistError (upstream bug repro). + + When fixed, replace this with assertions that rows are returned (like the + two-level test above). + """ + path = "repro_query_tree_by_row_three_level.tsfile" + device = "root.db.device1" + measurements = ["s1", "s2"] + try: + _remove_if_exists(path) + writer = TsFileWriter(path) + writer.register_timeseries( + device, TimeseriesSchema("s1", TSDataType.INT64) + ) + writer.register_timeseries( + device, TimeseriesSchema("s2", TSDataType.INT64) + ) + writer.write_row_record( + RowRecord( + device, + 0, + [ + Field("s1", 1, TSDataType.INT64), + Field("s2", 2, TSDataType.INT64), + ], + ) + ) + writer.close() + + with TsFileReader(path) as reader: + schemas = reader.get_all_timeseries_schemas() + assert device.lower() in schemas + assert len(schemas[device.lower()].get_timeseries_list()) == 2 + + with pytest.raises(DeviceNotExistError): + reader.query_tree_by_row([device], measurements, 0, 10) + finally: + _remove_if_exists(path) From 8b6b21601ea8fe28a6339a79b08617e03a9235cc Mon Sep 17 00:00:00 2001 From: 761417898 <761417898@qq.com> Date: Sat, 11 Apr 2026 20:52:33 +0800 Subject: [PATCH 2/4] fix: queryByRow table TAG-only scan, tree missing columns, multi-segment device paths - Table: drive TAG-only row iteration with internal_row_scan_fields (single VECTOR, all VECTORs, or all FIELD columns) instead of only the first FIELD - Tree: treat missing measurements as null columns; align Path split with StringArrayDeviceID string form for dotted device paths - Tests: extend C++/Python queryByRow coverage; cwrapper three-segment device query --- cpp/src/common/path.cc | 18 +- .../block/single_device_tsblock_reader.cc | 22 ++- cpp/src/reader/qds_without_timegenerator.cc | 77 +++++--- cpp/src/reader/qds_without_timegenerator.h | 4 + cpp/src/reader/table_query_executor.cc | 59 +++++- cpp/src/reader/task/device_query_task.cc | 6 +- cpp/src/reader/task/device_query_task.h | 24 ++- cpp/src/reader/task/device_task_iterator.cc | 3 +- cpp/src/reader/task/device_task_iterator.h | 33 ++-- cpp/test/cwrapper/cwrapper_test.cc | 25 ++- .../tsfile_table_query_by_row_test.cc | 180 ++++++++++++++++++ .../tsfile_tree_query_by_row_test.cc | 66 +++++++ python/tests/test_query_by_row.py | 37 ++++ 13 files changed, 481 insertions(+), 73 deletions(-) diff --git a/cpp/src/common/path.cc b/cpp/src/common/path.cc index fae3bff26..6e3302c36 100644 --- a/cpp/src/common/path.cc +++ b/cpp/src/common/path.cc @@ -19,6 +19,8 @@ #include "common/path.h" +#include "constant/tsfile_constant.h" + #ifdef ENABLE_ANTLR4 #include "parser/path_nodes_generator.h" #endif @@ -47,10 +49,20 @@ Path::Path(const std::string& path_sc, bool if_split) { IDeviceID::split_string(path_sc, '.'); #endif if (nodes.size() > 1) { - device_id_ = std::make_shared( - std::vector(nodes.begin(), nodes.end() - 1)); + // Build device_id from the same string form as writers use so + // StringArrayDeviceID::split_device_id_string canonicalizes + // multi-segment paths (e.g. root.sg1.FeederA) consistently. + std::string device_str; + for (size_t j = 0; j + 1 < nodes.size(); ++j) { + if (j > 0) { + device_str += PATH_SEPARATOR; + } + device_str += nodes[j]; + } + device_id_ = std::make_shared(device_str); measurement_ = nodes[nodes.size() - 1]; - full_path_ = device_id_->get_device_name() + "." + measurement_; + full_path_ = device_id_->get_device_name() + PATH_SEPARATOR + + measurement_; } else { full_path_ = path_sc; device_id_ = std::make_shared(); diff --git a/cpp/src/reader/block/single_device_tsblock_reader.cc b/cpp/src/reader/block/single_device_tsblock_reader.cc index 9b21e41b1..4610a66d9 100644 --- a/cpp/src/reader/block/single_device_tsblock_reader.cc +++ b/cpp/src/reader/block/single_device_tsblock_reader.cc @@ -19,6 +19,8 @@ #include "single_device_tsblock_reader.h" +#include + namespace storage { SingleDeviceTsBlockReader::SingleDeviceTsBlockReader( @@ -139,14 +141,20 @@ int SingleDeviceTsBlockReader::init_internal(DeviceQueryTask* device_query_task, col_appenders_[i] = new common::ColAppender(i, current_block_); } row_appender_ = new common::RowAppender(current_block_); - std::vector time_series_indexs( - device_query_task_->get_column_mapping() - ->get_measurement_columns() - .size()); + // ColumnMapping already uses unordered_set; iteration order matches + // TsFileIOReader::get_timeseries_indexes (unchanged from prior behavior). + std::unordered_set meas_names( + device_query_task_->get_column_mapping()->get_measurement_columns()); + if (meas_names.empty()) { + for (const auto& f : + device_query_task_->get_internal_row_scan_fields()) { + meas_names.insert(f); + } + } + std::vector time_series_indexs(meas_names.size()); if (RET_FAIL(tsfile_io_reader_->get_timeseries_indexes( - device_query_task->get_device_id(), - device_query_task->get_column_mapping()->get_measurement_columns(), - time_series_indexs, pa_))) { + device_query_task->get_device_id(), meas_names, time_series_indexs, + pa_))) { return ret; } diff --git a/cpp/src/reader/qds_without_timegenerator.cc b/cpp/src/reader/qds_without_timegenerator.cc index 0710b5873..a80911826 100644 --- a/cpp/src/reader/qds_without_timegenerator.cc +++ b/cpp/src/reader/qds_without_timegenerator.cc @@ -19,6 +19,7 @@ #include "qds_without_timegenerator.h" +#include "utils/errno_define.h" #include "utils/util_define.h" using namespace common; @@ -49,9 +50,8 @@ int QDSWithoutTimeGenerator::init_internal(TsFileIOReader* io_reader, io_reader_ = io_reader; qe_ = qe; - std::vector paths = qe_->selected_series_; - size_t origin_path_count = paths.size(); - std::vector valid_paths; + const std::vector paths = qe_->selected_series_; + const size_t origin_path_count = paths.size(); std::vector column_names; std::vector data_types; column_names.reserve(origin_path_count); @@ -61,52 +61,71 @@ int QDSWithoutTimeGenerator::init_internal(TsFileIOReader* io_reader, if (global_time_expression != nullptr) { global_time_filter = global_time_expression->filter_; } + index_lookup_.clear(); index_lookup_.insert({"time", 0}); + ssi_vec_.clear(); + path_has_ssi_.clear(); + path_has_ssi_.reserve(origin_path_count); + + auto register_path_columns = [&](size_t i, const Path& p) { + column_names.push_back(p.full_path_); + const uint32_t col_idx = static_cast(i) + 1; + index_lookup_.insert({p.measurement_, col_idx}); + if (p.full_path_ != p.measurement_) { + index_lookup_.insert({p.full_path_, col_idx}); + } + }; + for (size_t i = 0; i < origin_path_count; i++) { TsFileSeriesScanIterator* ssi = nullptr; ret = io_reader_->alloc_ssi(paths[i].device_id_, paths[i].measurement_, ssi, pa_, global_time_filter); + if (ret == E_MEASUREMENT_NOT_EXIST) { + ssi_vec_.push_back(nullptr); + path_has_ssi_.push_back(0); + register_path_columns(i, paths[i]); + continue; + } if (ret != 0) { return ret; - } else { - index_lookup_.insert({paths[i].measurement_, i + 1}); - if (paths[i].full_path_ != paths[i].measurement_) { - index_lookup_.insert({paths[i].full_path_, i + 1}); - } - ssi_vec_.push_back(ssi); - valid_paths.push_back(paths[i]); - column_names.push_back(paths[i].full_path_); } + ssi_vec_.push_back(ssi); + path_has_ssi_.push_back(1); + register_path_columns(i, paths[i]); } - size_t path_count = valid_paths.size(); + const size_t path_count = origin_path_count; is_single_path_ = (path_count == 1); - // Only push offset/limit to SSI for single-path; multi-path applies at - // merge. - for (size_t i = 0; i < path_count; i++) { - ssi_vec_[i]->set_row_range(is_single_path_ ? remaining_offset_ : 0, - is_single_path_ ? remaining_limit_ : -1); - } - row_record_ = new RowRecord(path_count + 1); + tsblocks_.resize(path_count); time_iters_.resize(path_count); value_iters_.resize(path_count); for (size_t i = 0; i < path_count; i++) { + if (!path_has_ssi_[i]) { + tsblocks_[i] = nullptr; + time_iters_[i] = nullptr; + value_iters_[i] = nullptr; + data_types.push_back(TSDataType::NULL_TYPE); + continue; + } + ssi_vec_[i]->set_row_range(is_single_path_ ? remaining_offset_ : 0, + is_single_path_ ? remaining_limit_ : -1); get_next_tsblock(i, true); data_types.push_back(value_iters_[i] != nullptr ? value_iters_[i]->get_data_type() : TSDataType::NULL_TYPE); } + row_record_ = new RowRecord(path_count + 1); // Single-path: SSI may have consumed offset/limit by skipping chunks/pages // during first get_next_tsblock(); sync so QDS does not double-apply. - if (is_single_path_) { + if (is_single_path_ && path_count > 0 && path_has_ssi_[0]) { remaining_offset_ = ssi_vec_[0]->get_row_offset(); remaining_limit_ = ssi_vec_[0]->get_row_limit(); } result_set_metadata_ = std::make_shared(column_names, data_types); - return E_OK; // ignore invalid timeseries + return E_OK; } void QDSWithoutTimeGenerator::close() { @@ -128,13 +147,17 @@ void QDSWithoutTimeGenerator::close() { ASSERT(ssi_vec_.size() == tsblocks_.size()); for (size_t i = 0; i < ssi_vec_.size(); i++) { - ssi_vec_[i]->revert_tsblock(); + if (path_has_ssi_[i]) { + ssi_vec_[i]->revert_tsblock(); + } } for (size_t i = 0; i < ssi_vec_.size(); i++) { - TsFileSeriesScanIterator* ssi = ssi_vec_[i]; - io_reader_->revert_ssi(ssi); + if (path_has_ssi_[i]) { + io_reader_->revert_ssi(ssi_vec_[i]); + } } ssi_vec_.clear(); + path_has_ssi_.clear(); if (qe_ != nullptr) { delete qe_; qe_ = nullptr; @@ -283,6 +306,9 @@ std::shared_ptr QDSWithoutTimeGenerator::get_metadata() { } int QDSWithoutTimeGenerator::get_next_tsblock(uint32_t index, bool alloc_mem) { + if (index >= ssi_vec_.size() || ssi_vec_[index] == nullptr) { + return E_OK; + } if (tsblocks_[index] != nullptr) { delete time_iters_[index]; time_iters_[index] = nullptr; @@ -320,6 +346,9 @@ int QDSWithoutTimeGenerator::get_next_tsblock(uint32_t index, bool alloc_mem) { int QDSWithoutTimeGenerator::get_next_tsblock_with_hint(uint32_t index, bool alloc_mem, int64_t min_time_hint) { + if (index >= ssi_vec_.size() || ssi_vec_[index] == nullptr) { + return E_OK; + } if (tsblocks_[index] != nullptr) { delete time_iters_[index]; time_iters_[index] = nullptr; diff --git a/cpp/src/reader/qds_without_timegenerator.h b/cpp/src/reader/qds_without_timegenerator.h index 1d929e575..0988dd955 100644 --- a/cpp/src/reader/qds_without_timegenerator.h +++ b/cpp/src/reader/qds_without_timegenerator.h @@ -40,6 +40,7 @@ class QDSWithoutTimeGenerator : public ResultSet { time_iters_(), value_iters_(), heap_time_(), + path_has_ssi_(), remaining_offset_(0), remaining_limit_(-1), is_single_path_(false) {} @@ -70,6 +71,9 @@ class QDSWithoutTimeGenerator : public ResultSet { std::vector value_iters_; std::multimap heap_time_; // key-->time, value-->path_index + /** Same length as ssi_vec_; false if measurement missing (Java + * EmptyFileSeriesReader). */ + std::vector path_has_ssi_; int remaining_offset_; int remaining_limit_; bool is_single_path_; diff --git a/cpp/src/reader/table_query_executor.cc b/cpp/src/reader/table_query_executor.cc index c23ffc0f4..d4c8f65d1 100644 --- a/cpp/src/reader/table_query_executor.cc +++ b/cpp/src/reader/table_query_executor.cc @@ -19,9 +19,44 @@ #include "reader/table_query_executor.h" +#include + +#include "common/db_common.h" #include "utils/db_utils.h" namespace storage { +namespace { + +/** When the query selects no FIELD columns (TAG-only), drive row scan from the + * table's VECTOR time axis if unambiguous, else all FIELD series (aligned + * dense row count), instead of only the first FIELD column. */ +std::vector collect_tag_only_scan_fields( + const std::shared_ptr& table_schema) { + std::vector all_fields; + std::vector vector_fields; + const auto categories = table_schema->get_column_categories(); + const auto& schemas = table_schema->get_measurement_schemas(); + for (size_t i = 0; i < schemas.size() && i < categories.size(); ++i) { + if (categories[i] != common::ColumnCategory::FIELD) { + continue; + } + std::string name = schemas[i]->measurement_name_; + to_lowercase_inplace(name); + all_fields.push_back(name); + if (schemas[i]->data_type_ == common::TSDataType::VECTOR) { + vector_fields.push_back(name); + } + } + if (vector_fields.size() == 1U) { + return vector_fields; + } + if (!vector_fields.empty()) { + return vector_fields; + } + return all_fields; +} + +} // namespace int TableQueryExecutor::query(const std::string& table_name, const std::vector& columns, Filter* time_filter, Filter* tag_filter, @@ -136,10 +171,26 @@ int TableQueryExecutor::query(const std::string& table_name, data_types.push_back(table_schema->get_data_types()[ind]); } - auto device_task_iterator = - std::unique_ptr(new DeviceTaskIterator( - lower_case_column_names, table_root, column_mapping, - meta_data_querier_, tag_filter, table_schema)); + std::vector internal_row_scan_fields; + const auto column_categories = table_schema->get_column_categories(); + bool selection_has_field = false; + for (const auto& col : lower_case_column_names) { + int ind = table_schema->find_column_index(col); + if (ind >= 0 && static_cast(ind) < column_categories.size() && + column_categories[static_cast(ind)] == + common::ColumnCategory::FIELD) { + selection_has_field = true; + break; + } + } + if (!selection_has_field) { + internal_row_scan_fields = collect_tag_only_scan_fields(table_schema); + } + + auto device_task_iterator = std::unique_ptr( + new DeviceTaskIterator(lower_case_column_names, table_root, + column_mapping, meta_data_querier_, tag_filter, + table_schema, internal_row_scan_fields)); std::unique_ptr tsblock_reader; switch (table_query_ordering_) { diff --git a/cpp/src/reader/task/device_query_task.cc b/cpp/src/reader/task/device_query_task.cc index c7e7091ff..6bf40a299 100644 --- a/cpp/src/reader/task/device_query_task.cc +++ b/cpp/src/reader/task/device_query_task.cc @@ -23,13 +23,15 @@ namespace storage { DeviceQueryTask* DeviceQueryTask::create_device_query_task( std::shared_ptr device_id, std::vector column_names, std::shared_ptr column_mapping, MetaIndexNode* index_root, - std::shared_ptr table_schema, common::PageArena& pa) { + std::shared_ptr table_schema, common::PageArena& pa, + const std::vector& internal_row_scan_fields) { void* buf = pa.alloc(sizeof(DeviceQueryTask)); if (UNLIKELY(buf == nullptr)) { return nullptr; } DeviceQueryTask* task = new (buf) DeviceQueryTask( - device_id, column_names, column_mapping, index_root, table_schema); + device_id, std::move(column_names), std::move(column_mapping), + index_root, table_schema, internal_row_scan_fields); return task; } diff --git a/cpp/src/reader/task/device_query_task.h b/cpp/src/reader/task/device_query_task.h index 75043e44b..8201666cc 100644 --- a/cpp/src/reader/task/device_query_task.h +++ b/cpp/src/reader/task/device_query_task.h @@ -19,6 +19,9 @@ #ifndef READER_TASK_DEVICE_QUERY_TASK_H #define READER_TASK_DEVICE_QUERY_TASK_H +#include +#include + #include "common/device_id.h" #include "reader/column_mapping.h" @@ -29,12 +32,14 @@ class DeviceQueryTask { std::vector column_names, std::shared_ptr column_mapping, MetaIndexNode* index_root, - std::shared_ptr table_schema) + std::shared_ptr table_schema, + std::vector internal_row_scan_fields = {}) : device_id_(device_id), - column_names_(column_names), - column_mapping_(column_mapping), + column_names_(std::move(column_names)), + column_mapping_(std::move(column_mapping)), index_root_(index_root), - table_schema_(table_schema) {} + table_schema_(std::move(table_schema)), + internal_row_scan_fields_(std::move(internal_row_scan_fields)) {} ~DeviceQueryTask(); static DeviceQueryTask* create_device_query_task( @@ -42,7 +47,8 @@ class DeviceQueryTask { std::vector column_names, std::shared_ptr column_mapping, MetaIndexNode* index_root, std::shared_ptr table_schema, - common::PageArena& pa); + common::PageArena& pa, + const std::vector& internal_row_scan_fields = {}); const std::vector& get_column_names() const { return column_names_; @@ -60,12 +66,20 @@ class DeviceQueryTask { std::shared_ptr get_device_id() const { return device_id_; } + /** When the query selects no FIELD columns, scan uses these (aligned + * VECTOR and/or all FIELD series) for row iteration only; not in + * column_names_. */ + const std::vector& get_internal_row_scan_fields() const { + return internal_row_scan_fields_; + } + private: std::shared_ptr device_id_; std::vector column_names_; std::shared_ptr column_mapping_; MetaIndexNode* index_root_; std::shared_ptr table_schema_; + std::vector internal_row_scan_fields_; }; } // namespace storage diff --git a/cpp/src/reader/task/device_task_iterator.cc b/cpp/src/reader/task/device_task_iterator.cc index dbe763303..be445afb5 100644 --- a/cpp/src/reader/task/device_task_iterator.cc +++ b/cpp/src/reader/task/device_task_iterator.cc @@ -36,7 +36,8 @@ int DeviceTaskIterator::next(DeviceQueryTask*& task) { } else { task = DeviceQueryTask::create_device_query_task( device_meta_pair.first, column_names_, column_mapping_, - device_meta_pair.second, table_schema_, pa_); + device_meta_pair.second, table_schema_, pa_, + internal_row_scan_fields_); } return ret; } diff --git a/cpp/src/reader/task/device_task_iterator.h b/cpp/src/reader/task/device_task_iterator.h index 061711c17..e050d43ab 100644 --- a/cpp/src/reader/task/device_task_iterator.h +++ b/cpp/src/reader/task/device_task_iterator.h @@ -19,6 +19,9 @@ #ifndef READER_TASK_DEVICE_TASK_ITERATOR_H #define READER_TASK_DEVICE_TASK_ITERATOR_H +#include +#include + #include "common/device_id.h" #include "reader/imeta_data_querier.h" #include "reader/task/device_query_task.h" @@ -30,17 +33,18 @@ class DeviceQueryTask; class DeviceTaskIterator { public: - explicit DeviceTaskIterator(std::vector column_names, - MetaIndexNode* index_root, - std::shared_ptr column_mapping, - IMetadataQuerier* metadata_querier, - const Filter* id_filter, - std::shared_ptr table_schema) - : column_names_(column_names), - column_mapping_(column_mapping), + explicit DeviceTaskIterator( + std::vector column_names, MetaIndexNode* index_root, + std::shared_ptr column_mapping, + IMetadataQuerier* metadata_querier, const Filter* id_filter, + std::shared_ptr table_schema, + std::vector internal_row_scan_fields = {}) + : column_names_(std::move(column_names)), + column_mapping_(std::move(column_mapping)), device_meta_iterator_( metadata_querier->device_iterator(index_root, id_filter)), - table_schema_(table_schema) { + table_schema_(std::move(table_schema)), + internal_row_scan_fields_(std::move(internal_row_scan_fields)) { pa_.init(512, common::MOD_DEVICE_TASK_ITER); } @@ -49,12 +53,14 @@ class DeviceTaskIterator { std::shared_ptr column_mapping, IMetadataQuerier* metadata_querier, const Filter* id_filter, - std::shared_ptr table_schema) - : column_names_(column_names), - column_mapping_(column_mapping), + std::shared_ptr table_schema, + std::vector internal_row_scan_fields = {}) + : column_names_(std::move(column_names)), + column_mapping_(std::move(column_mapping)), device_meta_iterator_( metadata_querier->device_iterator(index_roots, id_filter)), - table_schema_(table_schema) { + table_schema_(std::move(table_schema)), + internal_row_scan_fields_(std::move(internal_row_scan_fields)) { pa_.init(512, common::MOD_DEVICE_TASK_ITER); } @@ -71,6 +77,7 @@ class DeviceTaskIterator { std::shared_ptr column_mapping_; std::unique_ptr device_meta_iterator_; std::shared_ptr table_schema_; + std::vector internal_row_scan_fields_; common::PageArena pa_; }; diff --git a/cpp/test/cwrapper/cwrapper_test.cc b/cpp/test/cwrapper/cwrapper_test.cc index 6a0949c4e..863a2056b 100644 --- a/cpp/test/cwrapper/cwrapper_test.cc +++ b/cpp/test/cwrapper/cwrapper_test.cc @@ -311,9 +311,9 @@ TEST_F(CWrapperTest, WriterFlushTabletAndReadData) { free_write_file(&file); } -// Repro: query_tree_by_row fails with RET_DEVICRET_NOT_EXIST (44) for a -// three-segment device path (e.g. root.db.device1), while two-segment paths work. -// Root cause: Path(full_path) vs StringArrayDeviceID(device_string) split mismatch. +// Three-segment device paths (e.g. root.db.device1) must resolve like +// writer-side StringArrayDeviceID(device_string) canonicalization (see Path +// constructor). TEST_F(CWrapperTest, QueryTreeByRow_TwoSegmentDevice_Succeeds) { ERRNO code = 0; const char* filename = "cwrapper_query_tree_by_row_two_seg.tsfile"; @@ -344,8 +344,8 @@ TEST_F(CWrapperTest, QueryTreeByRow_TwoSegmentDevice_Succeeds) { char* devs[] = {const_cast(device)}; char* meas[] = {const_cast("s1")}; - ResultSet rs = tsfile_reader_query_tree_by_row(reader, devs, 1, meas, 1, 0, - 10, &code); + ResultSet rs = + tsfile_reader_query_tree_by_row(reader, devs, 1, meas, 1, 0, 10, &code); ASSERT_OK(code); ASSERT_NE(rs, nullptr); free_tsfile_result_set(&rs); @@ -353,7 +353,7 @@ TEST_F(CWrapperTest, QueryTreeByRow_TwoSegmentDevice_Succeeds) { remove(filename); } -TEST_F(CWrapperTest, QueryTreeByRow_ThreeSegmentDevice_ReproducesDeviceNotExist) { +TEST_F(CWrapperTest, QueryTreeByRow_ThreeSegmentDevice_Succeeds) { ERRNO code = 0; const char* filename = "cwrapper_query_tree_by_row_three_seg.tsfile"; remove(filename); @@ -383,14 +383,11 @@ TEST_F(CWrapperTest, QueryTreeByRow_ThreeSegmentDevice_ReproducesDeviceNotExist) char* devs[] = {const_cast(device)}; char* meas[] = {const_cast("s1")}; - ResultSet rs = tsfile_reader_query_tree_by_row(reader, devs, 1, meas, 1, 0, - 10, &code); - EXPECT_EQ(rs, nullptr); - EXPECT_EQ(code, RET_DEVICRET_NOT_EXIST) - << "repro: three-segment device path should currently fail query_tree_by_row"; - if (rs != nullptr) { - free_tsfile_result_set(&rs); - } + ResultSet rs = + tsfile_reader_query_tree_by_row(reader, devs, 1, meas, 1, 0, 10, &code); + ASSERT_OK(code); + ASSERT_NE(rs, nullptr); + free_tsfile_result_set(&rs); tsfile_reader_close(reader); remove(filename); } diff --git a/cpp/test/reader/table_view/tsfile_table_query_by_row_test.cc b/cpp/test/reader/table_view/tsfile_table_query_by_row_test.cc index 71440208f..873c5c609 100644 --- a/cpp/test/reader/table_view/tsfile_table_query_by_row_test.cc +++ b/cpp/test/reader/table_view/tsfile_table_query_by_row_test.cc @@ -247,6 +247,118 @@ class TableQueryByRowTest : public ::testing::Test { delete schema; } + /** id1 + s1 + s2; s1 (first FIELD) is never written — only s2 has points. + * TAG-only queryByRow must still see all rows by scanning every FIELD + * (not only the first), aligned with collect_tag_only_scan_fields(). */ + void write_tag_two_int_fields_first_never_written(int num_rows) { + std::vector col_schemas = { + ColumnSchema("id1", TSDataType::STRING, + CompressionType::UNCOMPRESSED, TSEncoding::PLAIN, + ColumnCategory::TAG), + ColumnSchema("s1", TSDataType::INT64, CompressionType::UNCOMPRESSED, + TSEncoding::PLAIN, ColumnCategory::FIELD), + ColumnSchema("s2", TSDataType::INT64, CompressionType::UNCOMPRESSED, + TSEncoding::PLAIN, ColumnCategory::FIELD), + }; + auto* schema = new TableSchema("t1", col_schemas); + auto* writer = new TsFileTableWriter(&write_file_, schema); + + Tablet tablet( + "t1", {"id1", "s1", "s2"}, + {TSDataType::STRING, TSDataType::INT64, TSDataType::INT64}, + {ColumnCategory::TAG, ColumnCategory::FIELD, ColumnCategory::FIELD}, + num_rows); + + for (int i = 0; i < num_rows; i++) { + tablet.add_timestamp(i, static_cast(i)); + tablet.add_value(i, "id1", "device_a"); + tablet.add_value(i, "s2", static_cast(i * 100)); + } + + ASSERT_EQ(writer->write_table(tablet), E_OK); + ASSERT_EQ(writer->flush(), E_OK); + ASSERT_EQ(writer->close(), E_OK); + delete writer; + delete schema; + } + + /** s1 is sparse; s2 is dense on every row. TAG-only row count must follow + * the full aligned row set, not only the first FIELD series. */ + void write_tag_two_int_fields_first_sparse_second_dense(int num_rows) { + std::vector col_schemas = { + ColumnSchema("id1", TSDataType::STRING, + CompressionType::UNCOMPRESSED, TSEncoding::PLAIN, + ColumnCategory::TAG), + ColumnSchema("s1", TSDataType::INT64, CompressionType::UNCOMPRESSED, + TSEncoding::PLAIN, ColumnCategory::FIELD), + ColumnSchema("s2", TSDataType::INT64, CompressionType::UNCOMPRESSED, + TSEncoding::PLAIN, ColumnCategory::FIELD), + }; + auto* schema = new TableSchema("t1", col_schemas); + auto* writer = new TsFileTableWriter(&write_file_, schema); + + Tablet tablet( + "t1", {"id1", "s1", "s2"}, + {TSDataType::STRING, TSDataType::INT64, TSDataType::INT64}, + {ColumnCategory::TAG, ColumnCategory::FIELD, ColumnCategory::FIELD}, + num_rows); + + for (int i = 0; i < num_rows; i++) { + tablet.add_timestamp(i, static_cast(i)); + tablet.add_value(i, "id1", "device_a"); + if ((i % 3) != 0) { + tablet.add_value(i, "s1", static_cast(i * 10)); + } + tablet.add_value(i, "s2", static_cast(i * 100)); + } + + ASSERT_EQ(writer->write_table(tablet), E_OK); + ASSERT_EQ(writer->flush(), E_OK); + ASSERT_EQ(writer->close(), E_OK); + delete writer; + delete schema; + } + + int count_rows_query(const std::vector& cols) { + TsFileReader reader; + if (reader.open(file_name_) != E_OK) { + return -1; + } + ResultSet* rs = nullptr; + if (reader.query("t1", cols, INT64_MIN, INT64_MAX, rs) != E_OK) { + reader.close(); + return -1; + } + int n = 0; + bool has_next = false; + while (IS_SUCC(rs->next(has_next)) && has_next) { + n++; + } + reader.destroy_query_data_set(rs); + reader.close(); + return n; + } + + int count_rows_query_by_row(const std::vector& cols) { + TsFileReader reader; + if (reader.open(file_name_) != E_OK) { + return -1; + } + ResultSet* rs = nullptr; + if (reader.queryByRow("t1", cols, 0, -1, rs) != E_OK) { + reader.close(); + return -1; + } + int n = 0; + bool has_next = false; + while (IS_SUCC(rs->next(has_next)) && has_next) { + n++; + } + reader.destroy_query_data_set(rs); + reader.close(); + return n; + } + std::vector query_all_s1(const std::string& table_name, const std::vector& columns) { TsFileReader reader; @@ -355,6 +467,74 @@ TEST_F(TableQueryByRowTest, NoOffsetNoLimit) { ASSERT_EQ(result, all); } +// TAG-only column list still returns all rows (internal scan uses VECTOR or +// all FIELD columns; see collect_tag_only_scan_fields). +TEST_F(TableQueryByRowTest, TagOnlyColumnsReturnRows) { + int num_rows = 10; + write_single_device_file(num_rows); + + TsFileReader reader; + ASSERT_EQ(reader.open(file_name_), E_OK); + ResultSet* rs = nullptr; + ASSERT_EQ(reader.queryByRow("t1", {"id1"}, 0, -1, rs), E_OK); + ASSERT_NE(rs, nullptr); + int count = 0; + bool has_next = false; + while (IS_SUCC(rs->next(has_next)) && has_next) { + count++; + } + ASSERT_EQ(count, num_rows); + reader.destroy_query_data_set(rs); + reader.close(); +} + +// First FIELD (s1) has no points; second FIELD (s2) is dense. TAG-only must +// still return the same row count as a full time-range query (all FIELD +// series participate in internal scan, not only the first column). +TEST_F(TableQueryByRowTest, TagOnly_FirstFieldNeverWritten_RowCount) { + const int num_rows = 42; + write_tag_two_int_fields_first_never_written(num_rows); + + ASSERT_EQ(count_rows_query_by_row({"id1"}), num_rows); + ASSERT_EQ(count_rows_query({"id1", "s1", "s2"}), num_rows); +} + +// Same layout as above: queryByRow(offset, limit) on selected columns must +// match manual skip/limit on the full scan (uses s1 nulls). +TEST_F(TableQueryByRowTest, TagOnly_FirstFieldNeverWritten_OffsetLimit) { + const int num_rows = 80; + write_tag_two_int_fields_first_never_written(num_rows); + const int offset = 17; + const int limit = 23; + const std::vector cols = {"id1", "s1", "s2"}; + auto by_row = query_by_row_time_and_s1("t1", cols, offset, limit); + auto manual = query_manual_time_and_s1("t1", cols, offset, limit); + ASSERT_EQ(by_row.size(), manual.size()); + ASSERT_EQ(by_row, manual); +} + +// First FIELD sparse, second dense: TAG-only row count still matches full +// logical row count. +TEST_F(TableQueryByRowTest, TagOnly_FirstFieldSparse_RowCount) { + const int num_rows = 60; + write_tag_two_int_fields_first_sparse_second_dense(num_rows); + + ASSERT_EQ(count_rows_query_by_row({"id1"}), num_rows); + ASSERT_EQ(count_rows_query({"id1", "s1", "s2"}), num_rows); +} + +TEST_F(TableQueryByRowTest, TagOnly_FirstFieldSparse_OffsetLimit) { + const int num_rows = 100; + write_tag_two_int_fields_first_sparse_second_dense(num_rows); + const int offset = 31; + const int limit = 29; + const std::vector cols = {"id1", "s1", "s2"}; + auto by_row = query_by_row_time_and_s1("t1", cols, offset, limit); + auto manual = query_manual_time_and_s1("t1", cols, offset, limit); + ASSERT_EQ(by_row.size(), manual.size()); + ASSERT_EQ(by_row, manual); +} + // Offset only: skip first N rows, return the rest; limit=-1 means no cap. TEST_F(TableQueryByRowTest, OffsetOnly) { int num_rows = 50; diff --git a/cpp/test/reader/tree_view/tsfile_tree_query_by_row_test.cc b/cpp/test/reader/tree_view/tsfile_tree_query_by_row_test.cc index 74845e441..d6c6b839e 100644 --- a/cpp/test/reader/tree_view/tsfile_tree_query_by_row_test.cc +++ b/cpp/test/reader/tree_view/tsfile_tree_query_by_row_test.cc @@ -25,6 +25,7 @@ #include "common/record.h" #include "common/schema.h" #include "file/write_file.h" +#include "reader/result_set.h" #include "reader/tsfile_reader.h" #include "reader/tsfile_tree_reader.h" #include "writer/tsfile_tree_writer.h" @@ -133,6 +134,71 @@ TEST_F(TreeQueryByRowTest, NoOffsetNoLimit) { reader.close(); } +// Missing measurements are not an error; columns are kept with null values +// (aligned with Java EmptyFileSeriesReader + DataSetWithoutTimeGenerator). +TEST_F(TreeQueryByRowTest, MissingMeasurementsYieldNullColumns) { + std::vector devices = {"d1"}; + std::vector measurements = {"s1"}; + int num_rows = 10; + write_test_file(devices, measurements, num_rows); + + TsFileTreeReader reader; + ASSERT_EQ(E_OK, reader.open(file_name_)); + + ResultSet* result = nullptr; + ASSERT_EQ(E_OK, reader.queryByRow(devices, {"s1", "_not_exist_col"}, 0, -1, + result)); + ASSERT_NE(result, nullptr); + auto meta = result->get_metadata(); + ASSERT_NE(meta, nullptr); + EXPECT_EQ(meta->get_column_count(), 3U); + + int row = 0; + bool has_next = false; + while (IS_SUCC(result->next(has_next)) && has_next) { + auto* rr = result->get_row_record(); + ASSERT_NE(rr, nullptr); + EXPECT_EQ(rr->get_timestamp(), static_cast(row)); + EXPECT_TRUE(rr->get_field(1)->is_type(INT64)); + EXPECT_EQ(rr->get_field(1)->get_value(), + static_cast(row * 100)); + EXPECT_TRUE(rr->get_field(2)->is_type(NULL_TYPE)); + row++; + } + EXPECT_EQ(row, num_rows); + + reader.destroy_query_data_set(result); + + ResultSet* empty_rs = nullptr; + ASSERT_EQ(E_OK, + reader.queryByRow(devices, {"_only_missing"}, 0, -1, empty_rs)); + ASSERT_NE(empty_rs, nullptr); + auto empty_ts = collect_timestamps(empty_rs); + EXPECT_EQ(empty_ts.size(), 0U); + reader.destroy_query_data_set(empty_rs); + + reader.close(); +} + +// Device path with more than two dot-separated segments matches storage layout. +TEST_F(TreeQueryByRowTest, MultiSegmentDevicePath) { + std::vector devices = {"root.sg1.FeederA"}; + std::vector measurements = {"s1"}; + int num_rows = 10; + write_test_file(devices, measurements, num_rows); + + TsFileTreeReader reader; + ASSERT_EQ(E_OK, reader.open(file_name_)); + + ResultSet* result = nullptr; + ASSERT_EQ(E_OK, reader.queryByRow(devices, measurements, 0, -1, result)); + ASSERT_NE(result, nullptr); + auto timestamps = collect_timestamps(result); + EXPECT_EQ(timestamps.size(), static_cast(num_rows)); + reader.destroy_query_data_set(result); + reader.close(); +} + // Test: offset skips leading rows. TEST_F(TreeQueryByRowTest, OffsetOnly) { std::vector devices = {"d1"}; diff --git a/python/tests/test_query_by_row.py b/python/tests/test_query_by_row.py index e45cd1b20..865cface3 100644 --- a/python/tests/test_query_by_row.py +++ b/python/tests/test_query_by_row.py @@ -110,3 +110,40 @@ def test_query_table_by_row_offset_limit(): if os.path.exists(file_path): os.remove(file_path) + +def test_query_table_by_row_tag_only_columns(): + """TAG-only column list must return the same row count as when a FIELD is included.""" + file_path = "python_table_query_by_row_tag_only.tsfile" + if os.path.exists(file_path): + os.remove(file_path) + + try: + table_name = "t1" + schema = TableSchema( + table_name, + [ + ColumnSchema("tag1", TSDataType.STRING, ColumnCategory.TAG), + ColumnSchema("s1", TSDataType.INT64, ColumnCategory.FIELD), + ], + ) + + num_rows = 10 + with TsFileTableWriter(file_path, schema) as writer: + tablet = Tablet(["tag1", "s1"], [TSDataType.STRING, TSDataType.INT64], num_rows) + for t in range(num_rows): + tablet.add_timestamp(t, t) + tablet.add_value_by_name("tag1", t, f"v{t}") + tablet.add_value_by_name("s1", t, t * 10) + writer.write_table(tablet) + + reader = TsFileReader(file_path) + with reader.query_table_by_row(table_name, ["tag1"]) as result: + count = 0 + while result.next(): + count += 1 + assert count == num_rows, f"Expected {num_rows} rows for TAG-only query, got {count}" + reader.close() + finally: + if os.path.exists(file_path): + os.remove(file_path) + From c1b0b8b8469855776a5d59b8e2fe6e4768e8b2a9 Mon Sep 17 00:00:00 2001 From: 761417898 <761417898@qq.com> Date: Sat, 11 Apr 2026 21:11:43 +0800 Subject: [PATCH 3/4] fixed ci --- cpp/src/file/tsfile_io_reader.cc | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/cpp/src/file/tsfile_io_reader.cc b/cpp/src/file/tsfile_io_reader.cc index 31d4a303a..c71406fda 100644 --- a/cpp/src/file/tsfile_io_reader.cc +++ b/cpp/src/file/tsfile_io_reader.cc @@ -19,6 +19,9 @@ #include "file/tsfile_io_reader.h" +#include +#include + #include "common/allocator/alloc_base.h" using namespace common; @@ -457,8 +460,14 @@ int TsFileIOReader::get_timeseries_indexes( get_time_column_metadata(top_node, timeseries_index, pa); } + // Sort names so slot order is identical on all platforms (unordered_set + // iteration order differs between libc++ and libstdc++). + std::vector sorted_names(measurement_names.begin(), + measurement_names.end()); + std::sort(sorted_names.begin(), sorted_names.end()); + int64_t idx = 0; - for (const auto& measurement_name : measurement_names) { + for (const auto& measurement_name : sorted_names) { if (RET_FAIL(load_measurement_index_entry(measurement_name, top_node, measurement_index_entry, measurement_ie_end_offset))) { From 782048c7e07fdafd117b1ecc7b4984ad35c490f8 Mon Sep 17 00:00:00 2001 From: 761417898 <761417898@qq.com> Date: Sat, 11 Apr 2026 21:47:06 +0800 Subject: [PATCH 4/4] fixed ci --- ...est_query_tree_by_row_three_level_repro.py | 23 ++++++++----------- python/tsfile/tsfile_reader.pyx | 7 ++++++ python/tsfile/tsfile_table_writer.py | 8 +++++++ 3 files changed, 24 insertions(+), 14 deletions(-) diff --git a/python/tests/test_query_tree_by_row_three_level_repro.py b/python/tests/test_query_tree_by_row_three_level_repro.py index f6a3dbe24..0dcea8539 100644 --- a/python/tests/test_query_tree_by_row_three_level_repro.py +++ b/python/tests/test_query_tree_by_row_three_level_repro.py @@ -15,15 +15,12 @@ # specific language governing permissions and limitations # under the License. # -"""Reproduce DeviceNotExistError (44) for query_tree_by_row on 3-segment device paths.""" +"""query_tree_by_row on 3-segment device paths (Path / StringArrayDeviceID alignment).""" import os -import pytest - from tsfile import Field, RowRecord, TimeseriesSchema, TsFileReader, TsFileWriter from tsfile import TSDataType -from tsfile.exceptions import DeviceNotExistError def _remove_if_exists(path: str) -> None: @@ -62,14 +59,8 @@ def test_query_tree_by_row_two_level_device_ok(): _remove_if_exists(path) -def test_repro_query_tree_by_row_three_level_device_not_exist(): - """ - Three-segment device path (e.g. root.db.device1): metadata lists the device, - but query_tree_by_row currently raises DeviceNotExistError (upstream bug repro). - - When fixed, replace this with assertions that rows are returned (like the - two-level test above). - """ +def test_query_tree_by_row_three_level_device_ok(): + """Three-segment device path: query_tree_by_row should succeed (same as two-level).""" path = "repro_query_tree_by_row_three_level.tsfile" device = "root.db.device1" measurements = ["s1", "s2"] @@ -99,7 +90,11 @@ def test_repro_query_tree_by_row_three_level_device_not_exist(): assert device.lower() in schemas assert len(schemas[device.lower()].get_timeseries_list()) == 2 - with pytest.raises(DeviceNotExistError): - reader.query_tree_by_row([device], measurements, 0, 10) + rs = reader.query_tree_by_row([device], measurements, 0, 10) + assert rs.next() + assert rs.get_value_by_index(1) == 0 + assert rs.get_value_by_index(2) == 1 + assert rs.get_value_by_index(3) == 2 + rs.close() finally: _remove_if_exists(path) diff --git a/python/tsfile/tsfile_reader.pyx b/python/tsfile/tsfile_reader.pyx index 2259f7704..92ac91a6a 100644 --- a/python/tsfile/tsfile_reader.pyx +++ b/python/tsfile/tsfile_reader.pyx @@ -145,6 +145,13 @@ cdef class ResultSetPy: df = pd.DataFrame(data_container) data_type_dict = {col: dtype for col, dtype in zip(column_names, data_type)} df = df.astype(data_type_dict) + # Pandas 3 defaults string columns to str dtype; tests compare with DataFrame literals + # using that dtype. Series.equals is False if dtypes differ (object vs str). + for i in range(column_num): + col = column_names[i] + pydt = self.metadata.get_data_type(i + 1) + if pydt in (TSDataTypePy.STRING, TSDataTypePy.TEXT): + df[col] = df[col].astype(str) return df def read_arrow_batch(self): diff --git a/python/tsfile/tsfile_table_writer.py b/python/tsfile/tsfile_table_writer.py index cfd817fec..4860a4979 100644 --- a/python/tsfile/tsfile_table_writer.py +++ b/python/tsfile/tsfile_table_writer.py @@ -36,6 +36,14 @@ def validate_dataframe_for_tsfile(df: pd.DataFrame) -> None: for c in columns: if c is None or (isinstance(c, str) and len(c) == 0): raise ValueError("Column name cannot be None or empty") + # pandas may promote None in columns=[None, ...] to float nan in the Index + if not isinstance(c, str): + try: + if pd.isna(c): + raise ValueError("Column name cannot be None or empty") + except TypeError: + pass + raise ValueError("Column name cannot be None or empty") lower = c.lower() if lower in seen: duplicates.append(c)