Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 8 additions & 0 deletions cpp/src/common/device_id.cc
Original file line number Diff line number Diff line change
Expand Up @@ -103,6 +103,13 @@ std::string StringArrayDeviceID::get_device_name() const {
}

void StringArrayDeviceID::init_prefix_segments() {
// Idempotent: device IDs are cached and reused across queries, so clear
// previous prefixes before rebuilding to avoid accumulation and leaks.
for (const auto& prefix_segment : prefix_segments_) {
delete prefix_segment;
}
prefix_segments_.clear();

Comment on lines +106 to +112

@ColinLeeo ColinLeeo Jul 15, 2026 •

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

It's not a good idea but useful...

#ifdef ENABLE_ANTLR4
auto splits = storage::PathNodesGenerator::invokeParser(*segments_[0]);
#else
Expand Down Expand Up @@ -130,6 +137,7 @@ int StringArrayDeviceID::serialize(common::ByteStream& write_stream) {

int StringArrayDeviceID::deserialize(common::ByteStream& read_stream) {
int ret = common::E_OK;

uint32_t num_segments;
if (RET_FAIL(common::SerializationUtil::read_var_uint(num_segments,
read_stream))) {
Expand Down
35 changes: 23 additions & 12 deletions cpp/src/common/schema.h
Original file line number Diff line number Diff line change
Expand Up @@ -198,9 +198,16 @@ class TableSchema {
* in the table.
*/
TableSchema(const std::string& table_name,
const std::vector<common::ColumnSchema>& column_schemas)
: table_name_(table_name), updatable_(false) {
to_lowercase_inplace(table_name_);
const std::vector<common::ColumnSchema>& column_schemas,
bool virtual_table = false)
: table_name_(table_name),
is_virtual_table_(virtual_table),
updatable_(false) {
// Virtual (tree-derived) schemas are case-sensitive; only real tables
// are normalized to lower case.
if (!is_virtual_table_) {
to_lowercase_inplace(table_name_);
}
for (const common::ColumnSchema& column_schema : column_schemas) {
column_schemas_.emplace_back(std::make_shared<MeasurementSchema>(
column_schema.get_column_name(),
Expand All @@ -210,7 +217,9 @@ class TableSchema {
}
int idx = 0;
for (const auto& measurement_schema : column_schemas_) {
to_lowercase_inplace(measurement_schema->measurement_name_);
if (!is_virtual_table_) {
to_lowercase_inplace(measurement_schema->measurement_name_);
}
column_pos_index_.insert(
std::make_pair(measurement_schema->measurement_name_, idx++));
}
Expand Down Expand Up @@ -325,21 +334,23 @@ class TableSchema {
int32_t get_columns_num() const { return column_schemas_.size(); }

int find_column_index(const std::string& column_name) {
std::string lower_case_column_name = to_lower(column_name);
auto it = column_pos_index_.find(lower_case_column_name);
// Virtual tables match case-sensitively; real tables are lower-cased.
std::string lookup_name =

is_virtual_table_ ? column_name : to_lower(column_name);
auto it = column_pos_index_.find(lookup_name);
if (it != column_pos_index_.end()) {
return it->second;
} else {
int index = -1;
for (size_t i = 0; i < column_schemas_.size(); ++i) {
if (column_schemas_[i]->measurement_name_ ==
lower_case_column_name) {
if (column_schemas_[i]->measurement_name_ == lookup_name) {
index = static_cast<int>(i);
break;
}
}
if (index != -1) {
column_pos_index_[lower_case_column_name] = index;
column_pos_index_[lookup_name] = index;
}
return index;
}
Expand Down Expand Up @@ -440,12 +451,12 @@ class TableSchema {
}

int32_t find_id_column_order(const std::string& column_name) {
std::string lower_case_column_name = to_lower(column_name);
std::string lookup_name =
is_virtual_table_ ? column_name : to_lower(column_name);

int column_order = 0;
for (size_t i = 0; i < column_schemas_.size(); ++i) {
if (column_schemas_[i]->measurement_name_ ==
lower_case_column_name &&
if (column_schemas_[i]->measurement_name_ == lookup_name &&
column_categories_[i] == common::ColumnCategory::TAG) {
return column_order;
} else if (column_categories_[i] == common::ColumnCategory::TAG) {
Expand Down
8 changes: 6 additions & 2 deletions cpp/src/reader/table_query_executor.cc
Original file line number Diff line number Diff line change
Expand Up @@ -233,8 +233,12 @@ int TableQueryExecutor::query_on_tree(
}
}

auto schema = std::make_shared<TableSchema>("default", col_schema);
schema->set_virtual_table();
// Tree-derived (virtual) tables are case-sensitive: keep the original
// casing of device path segments and measurement names so that
// case-distinct series (e.g. "temperature" vs "Temperature") map to
// separate columns instead of colliding.
auto schema = std::make_shared<TableSchema>("default", col_schema,
/*virtual_table=*/true);
std::shared_ptr<ColumnMapping> column_mapping =
std::make_shared<ColumnMapping>();
for (size_t i = 0; i < col_schema.size(); ++i) {
Expand Down
20 changes: 20 additions & 0 deletions cpp/test/common/device_id_test.cc
Original file line number Diff line number Diff line change
Expand Up @@ -99,4 +99,24 @@ TEST(DeviceIdTest, NullTagVsLiteralNullAreDistinct) {
ASSERT_FALSE(null_first == literal_null);
ASSERT_TRUE(null_first != literal_null);
}

// Regression: cached device IDs are reused across queries, so
// split_table_name() must be idempotent and not accumulate prefix segments.
TEST(DeviceIdTest, SplitTableNameIsIdempotent) {
StringArrayDeviceID device_id("root.ln.wf01.wt01");

const std::vector<std::string> expected = {"root", "ln", "wf01", "wt01"};

for (int round = 0; round < 3; ++round) {
device_id.split_table_name();

ASSERT_EQ(static_cast<int>(expected.size()),
device_id.get_split_seg_num());
for (int i = 0; i < device_id.get_split_seg_num(); ++i) {
std::string* seg = device_id.get_split_segname_at(i);
ASSERT_NE(nullptr, seg);
ASSERT_EQ(expected[static_cast<size_t>(i)], *seg);
}
}
}
} // namespace storage
164 changes: 164 additions & 0 deletions python/tests/test_tsfile_dataset.py
Original file line number Diff line number Diff line change
Expand Up @@ -1507,12 +1507,176 @@ def test_dataset_tree_model_series_access(tmp_path):
np.testing.assert_array_equal(aligned.timestamps, np.arange(5, dtype=np.int64))


def test_dataset_tree_model_reads_uppercase_measurement_names(tmp_path):
"""Tree-model series with uppercase measurement names must read data.

Regression: a measurement like ``Temperature``/``STATUS`` must return its
values, not an empty array, when read back through TsFileDataFrame.
"""
from tsfile import Field, RowRecord, TimeseriesSchema, TsFileWriter

path = tmp_path / "tree_upper.tsfile"
writer = TsFileWriter(str(path))
writer.register_timeseries(
"root.ln.wf01.wt01", TimeseriesSchema("Temperature", TSDataType.DOUBLE)
)
writer.register_timeseries(
"root.ln.wf01.wt01", TimeseriesSchema("STATUS", TSDataType.INT32)
)
for t in range(5):
writer.write_row_record(
RowRecord(
"root.ln.wf01.wt01",
t,
[
Field("Temperature", float(t) + 0.5, TSDataType.DOUBLE),
Field("STATUS", t * 2, TSDataType.INT32),
],
)
)
writer.close()

with TsFileDataFrame(str(path), show_progress=False) as tsdf:
assert sorted(tsdf.list_timeseries()) == [
"root.ln.wf01.wt01.STATUS",
"root.ln.wf01.wt01.Temperature",
]
np.testing.assert_array_equal(
tsdf["root.ln.wf01.wt01.Temperature"][:],
np.array([0.5, 1.5, 2.5, 3.5, 4.5]),
)
np.testing.assert_array_equal(
tsdf["root.ln.wf01.wt01.STATUS"][:],
np.array([0.0, 2.0, 4.0, 6.0, 8.0]),
)


def test_dataset_tree_model_case_distinct_measurements_do_not_collide(tmp_path, capsys):
"""Case-distinct measurements on one device must not be conflated.

``temperature`` and ``Temperature`` are two independent series; each must
return its own values instead of an empty array or the other's data.
"""
from tsfile import Field, RowRecord, TimeseriesSchema, TsFileWriter

path = tmp_path / "tree_case.tsfile"

writer = TsFileWriter(str(path))
writer.register_timeseries(
"root.case.d1", TimeseriesSchema("temperature", TSDataType.DOUBLE)
)
writer.register_timeseries(
"root.case.d1", TimeseriesSchema("Temperature", TSDataType.DOUBLE)
)
for t in range(3):
writer.write_row_record(
RowRecord(
"root.case.d1",
t,
[
Field("temperature", 30.0 + t, TSDataType.DOUBLE),
Field("Temperature", 40.0 + t, TSDataType.DOUBLE),
],
)
)
writer.close()

with TsFileDataFrame(str(path), show_progress=False) as tsdf:
# Two independent tree-model series are discovered (case preserved).
assert len(tsdf) == 2
assert sorted(tsdf.list_timeseries()) == [
"root.case.d1.Temperature",
"root.case.d1.temperature",
]

# Metadata layer keeps both rows distinct with their original casing.
meta = tsdf.list_timeseries_metadata()
assert sorted(meta.index.tolist()) == [
"root.case.d1.Temperature",
"root.case.d1.temperature",
]
for name in ("root.case.d1.temperature", "root.case.d1.Temperature"):
assert meta.loc[name, "_col_1"] == "case"
assert meta.loc[name, "_col_2"] == "d1"
assert meta.loc[name, "count"] == 3
assert set(meta["field"]) == {"temperature", "Temperature"}

# print(tsdf) renders both case-distinct series verbatim. The timestamp
# text is derived from format_timestamp so the whole-string match stays
# stable across timezones.
t0, t2 = format_timestamp(0), format_timestamp(2)
expected_repr = "\n".join(
[
"TsFileDataFrame(tree model, 2 time series, 1 files)",
" _col_1 _col_2 field start_time end_time count",
f"0 case d1 Temperature {t0} {t2} 3",
f"1 case d1 temperature {t0} {t2} 3",
]
)
assert repr(tsdf) == expected_repr
print(tsdf)
assert capsys.readouterr().out == expected_repr + "\n"

# Each series must read back its OWN values -- not empty, not the
# other's data. Lower-casing anywhere in the read path breaks this.
np.testing.assert_array_equal(
tsdf["root.case.d1.temperature"][:],
np.array([30.0, 31.0, 32.0]),
)
np.testing.assert_array_equal(
tsdf["root.case.d1.Temperature"][:],
np.array([40.0, 41.0, 42.0]),
)

# Aligned read of both case-distinct series keeps them separated.
aligned = tsdf.loc[
0:2,
["root.case.d1.temperature", "root.case.d1.Temperature"],
]
assert aligned.series_names == [
"root.case.d1.temperature",
"root.case.d1.Temperature",
]
np.testing.assert_array_equal(
aligned.values,
np.array([[30.0, 40.0], [31.0, 41.0], [32.0, 42.0]]),
)


def test_tree_reader_handles_stale_path_columns_after_reused_queries(tmp_path):
"""Reusing a reader must not leak prefix path state across queries.

Reading one device series then another reuses the cached device id; stale
prefix segments used to mismatch the device and return empty data.
"""
path = tmp_path / "tree.tsfile"
_write_tree_file(path)

with TsFileDataFrame(str(path), show_progress=False) as tsdf:
# First read establishes query state on the reader.
np.testing.assert_array_equal(
tsdf["root.ln.wf01.wt01.temperature"][:],
np.array([0.5, 1.5, 2.5, 3.5, 4.5]),
)
# Second read reuses the same reader for another device.
np.testing.assert_array_equal(
tsdf["root.ln.wf02.wt02.status"][:],
np.array([0.0, 2.0, 4.0, 6.0, 8.0]),
)
# Read back the first series to confirm alternating reads stay stable.
np.testing.assert_array_equal(
tsdf["root.ln.wf01.wt01.temperature"][:],
np.array([0.5, 1.5, 2.5, 3.5, 4.5]),
)


def test_dataset_tree_model_list_timeseries_metadata(tmp_path):
path = tmp_path / "tree.tsfile"
_write_tree_file(path)

with TsFileDataFrame(str(path), show_progress=False) as tsdf:
meta = tsdf.list_timeseries_metadata()

assert isinstance(meta, pd.DataFrame)
assert list(meta.columns) == [
"field",
Expand Down
Loading