Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
17 changes: 15 additions & 2 deletions python/tests/test_load_tsfile_from_iotdb.py
Original file line number Diff line number Diff line change
Expand Up @@ -51,6 +51,7 @@ def test_load_tsfile_from_iotdb():
(1760106080000 + 1760106109000) * 30 // 2
)
assert df["s0"].isna().sum() == 0
df_s0 = df["s0"]
assert df["s1"].isna().sum() == 0
assert df["s2"].isna().sum() == 8
assert df["s3"].isna().sum() == 0
Expand All @@ -73,6 +74,12 @@ def test_load_tsfile_from_iotdb():
assert df["s8"].isna().sum() == 0
assert df["s8"].nunique() == 60
assert df["s9"].isna().sum() == 8

df = ts.to_dataframe(simple_tabl1_path, table_name="test", column_names=["s0"])
assert len(df) == 60
assert len(df.columns) == 2
assert df["s0"].equals(df_s0)

## ---------

simple_tabl2_path = os.path.join(dir_path, 'simple_table_t2.tsfile')
Expand Down Expand Up @@ -118,17 +125,23 @@ def test_load_tsfile_from_iotdb():
assert math.isclose(df["temperature"].sum(), 2.5, rel_tol=1e-9)
assert math.isclose(df["humidity"].sum(), 2.5, rel_tol=1e-9)
assert (df["region_id"] == "loc").sum() == 25
df_id = df["id"]

df = ts.to_dataframe(table_with_time_column_path, table_name="table2", column_names=["region_id", "temperature", "humidity"])
df = ts.to_dataframe(table_with_time_column_path, table_name="table2",
column_names=["region_id", "temperature", "humidity"])
assert list(df.columns)[0] == "id"
assert len(df) == 25
assert math.isclose(df["temperature"].sum(), 2.5, rel_tol=1e-9)
assert (df["region_id"] == "loc").sum() == 25

df = ts.to_dataframe(table_with_time_column_path, table_name="table2", column_names=["id", "temperature", "humidity"])
df = ts.to_dataframe(table_with_time_column_path, table_name="table2",
column_names=["id", "temperature", "humidity"])
assert list(df.columns)[0] == "time"
assert df["id"].equals(df["time"])
assert len(df) == 25
assert math.isclose(df["temperature"].sum(), 2.5, rel_tol=1e-9)
assert math.isclose(df["humidity"].sum(), 2.5, rel_tol=1e-9)

df = ts.to_dataframe(table_with_time_column_path, table_name="table2", column_names=["id"])
assert len(df.columns) == 2
assert df_id.equals(df["id"])
34 changes: 26 additions & 8 deletions python/tsfile/utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@
import pandas as pd
from pandas.core.dtypes.common import is_integer_dtype, is_object_dtype

from tsfile import ColumnSchema, TableSchema, ColumnCategory, TSDataType
from tsfile import ColumnSchema, TableSchema, ColumnCategory, TSDataType, TIME_COLUMN
from tsfile.exceptions import TableNotExistError, ColumnNotExistError
from tsfile.tsfile_reader import TsFileReaderPy
from tsfile.tsfile_table_writer import TsFileTableWriter, infer_object_column_type, validate_dataframe_for_tsfile
Expand Down Expand Up @@ -116,10 +116,16 @@ def _gen(is_iterator: bool) -> Iterator[pd.DataFrame]:

is_tree_model = len(table_schema) == 0
time_column = None
column_name_to_query = []
no_field_query = True
if is_tree_model:
if _column_names is None:
print("columns name is None, return all columns")
# When querying tables in the tree, only measurements are allowed currently.
no_field_query = False
else:
_table_name = _table_name.lower() if _table_name else None
_column_names = [column.lower() for column in _column_names] if _column_names else None
if _table_name is None:
_table_name, table_schema = next(iter(table_schema.items()))
else:
Expand All @@ -137,17 +143,26 @@ def _gen(is_iterator: bool) -> Iterator[pd.DataFrame]:

if _column_names is not None:
for column in _column_names:
if column.lower() not in column_names_in_file and column.lower() != time_column :
if column not in column_names_in_file and column != time_column:
raise ColumnNotExistError(column)
if table_schema.get_column(column).get_category() == ColumnCategory.FIELD:
no_field_query = False
if no_field_query:
if time_column is not None:
column_name_to_query.append(time_column)
column_name_to_query.extend(column_names_in_file)
else:
column_name_to_query = _column_names
else:
_column_names = column_names_in_file
no_field_query = False
column_name_to_query = column_names_in_file

if is_tree_model:
if _column_names is None:
_column_names = []
query_result = reader.query_table_on_tree(_column_names, _start_time, _end_time)
if _column_names is not None:
column_name_to_query = _column_names
query_result = reader.query_table_on_tree(column_name_to_query, _start_time, _end_time)
else:
query_result = reader.query_table(_table_name, _column_names, _start_time, _end_time)
query_result = reader.query_table(_table_name, column_name_to_query, _start_time, _end_time)

with query_result as result:
while result.next():
Expand All @@ -164,8 +179,11 @@ def _gen(is_iterator: bool) -> Iterator[pd.DataFrame]:
continue
total_rows += len(dataframe)
if time_column is not None:
if _column_names is None or time_column.lower() not in [c.lower() for c in _column_names]:
if _column_names is None or time_column not in _column_names:
dataframe = dataframe.rename(columns={dataframe.columns[0]: time_column})
if no_field_query and _column_names is not None:
_column_names.insert(0, TIME_COLUMN)
dataframe = dataframe[_column_names]
Comment on lines +185 to +186

Copilot AI Feb 25, 2026

Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This code modifies the caller's original list by calling insert on _column_names. Since _column_names is a reference to the column_names parameter (after lowercasing at line 128), this mutates the caller's list. This mutation will cause issues if the same list is reused across multiple calls or if the caller expects their list to remain unchanged. Additionally, this mutation happens inside the iterator loop, so it will occur for every batch of data, repeatedly inserting TIME_COLUMN at position 0, which will cause incorrect behavior. The insertion should be done on a copy of the list, not on the original reference.

Suggested change
_column_names.insert(0, TIME_COLUMN)
dataframe = dataframe[_column_names]
columns_with_time = [TIME_COLUMN] + list(_column_names)
dataframe = dataframe[columns_with_time]

Copilot uses AI. Check for mistakes.
yield dataframe
if (not is_iterator) and max_row_num is not None and total_rows >= max_row_num:
break
Expand Down
Loading