From f531700b1b58bc3ef5775ebf2a3f1d2ab5ee3991 Mon Sep 17 00:00:00 2001 From: ColinLee Date: Tue, 17 Nov 2026 12:15:16 +0800 Subject: [PATCH 1/7] feat(python): union table-model columns across TsFiles in TsFileDataFrame A physical TsFile registers only the columns it actually wrote, so one table can span files whose per-file column sets differ (compaction outputs, files written at different schema snapshots). TsFileDataFrame rejected such load sets outright. Merge table entries instead: FIELD columns are unioned in first-seen order and a device/field missing from a file reads back as NaN; TAG columns are prefix-compatible (extra trailing tags are appended and shorter devices read with a null tag), and only a differing shared tag prefix, a tag-vs-field category change, or a same-named FIELD column with a different data type is rejected, with an error naming the table, file, column and types. The merged entry rebuilds schema_columns so the persistent dataset index keeps defining columns and validating physical types. Shard registration already keys logical series by the global field index resolved by name, so table-model reader-local field indices no longer need to coincide with the merged global ones. --- python/tests/test_tsfile_dataset.py | 369 ++++++++++++++++++-- python/tsfile/dataset/dataframe.py | 167 +++++++-- skills/tsfile/references/dataframe-arrow.md | 9 + 3 files changed, 485 insertions(+), 60 deletions(-) diff --git a/python/tests/test_tsfile_dataset.py b/python/tests/test_tsfile_dataset.py index 964dde6f8..489d73d53 100644 --- a/python/tests/test_tsfile_dataset.py +++ b/python/tests/test_tsfile_dataset.py @@ -101,6 +101,12 @@ def _write_empty_weather_file(path): pass +def _write_table_rows_file(path, columns, rows): + schema = TableSchema("weather", columns) + with TsFileTableWriter(str(path), schema) as writer: + writer.write_dataframe(pd.DataFrame(rows)) + + def _write_numeric_and_text_file(path): schema = TableSchema( "weather", @@ -143,29 +149,6 @@ def _write_partial_numeric_rows_file(path): writer.write_dataframe(df) -def _write_weather_with_extra_field_file(path, start): - schema = TableSchema( - "weather", - [ - ColumnSchema("device", TSDataType.STRING, ColumnCategory.TAG), - ColumnSchema("temperature", TSDataType.DOUBLE, ColumnCategory.FIELD), - ColumnSchema("humidity", TSDataType.DOUBLE, ColumnCategory.FIELD), - ColumnSchema("pressure", TSDataType.DOUBLE, ColumnCategory.FIELD), - ], - ) - df = pd.DataFrame( - { - "time": [start, start + 1], - "device": ["device_a", "device_a"], - "temperature": [20.0, 21.0], - "humidity": [50.0, 51.0], - "pressure": [1000.0, 1001.0], - } - ) - with TsFileTableWriter(str(path), schema) as writer: - writer.write_dataframe(df) - - def _write_weather_int_temperature_file(path, start): schema = TableSchema( "weather", @@ -1272,22 +1255,268 @@ def test_no_index_subset_rejects_reads_after_root_close(subset_lifecycle_dataset assert result.returncode == 0, result.stdout + result.stderr -def test_dataset_rejects_incompatible_table_schemas_across_shards( +_WEATHER_DEVICE_TAG = ColumnSchema("device", TSDataType.STRING, ColumnCategory.TAG) +_WEATHER_TEMPERATURE = ColumnSchema( + "temperature", TSDataType.DOUBLE, ColumnCategory.FIELD +) +_WEATHER_HUMIDITY = ColumnSchema("humidity", TSDataType.DOUBLE, ColumnCategory.FIELD) +_WEATHER_PRESSURE = ColumnSchema("pressure", TSDataType.DOUBLE, ColumnCategory.FIELD) + + +def test_dataset_unions_table_field_columns_across_shards( tmp_path, dataframe_use_index ): + # A table may span TsFiles that each register only the columns they wrote. + # The dataset unions the field columns and reads a missing column as NaN. path1 = tmp_path / "part1.tsfile" path2 = tmp_path / "part2.tsfile" - _write_weather_file(path1, 0) - _write_weather_with_extra_field_file(path2, 2) + _write_table_rows_file( + path1, + [_WEATHER_DEVICE_TAG, _WEATHER_TEMPERATURE, _WEATHER_HUMIDITY], + { + "time": [0, 1, 2], + "device": ["device_a"] * 3, + "temperature": [20.0, 21.5, 23.0], + "humidity": [50.0, 52.0, 55.0], + }, + ) + _write_table_rows_file( + path2, + [ + _WEATHER_DEVICE_TAG, + _WEATHER_TEMPERATURE, + _WEATHER_HUMIDITY, + _WEATHER_PRESSURE, + ], + { + "time": [3, 4], + "device": ["device_a"] * 2, + "temperature": [24.0, 25.0], + "humidity": [56.0, 57.0], + "pressure": [1000.0, 1001.0], + }, + ) - with pytest.raises(ValueError, match="Incompatible schema for table 'weather'"): - TsFileDataFrame( - [str(path1), str(path2)], - show_progress=False, - use_index=dataframe_use_index, + with TsFileDataFrame( + [str(path1), str(path2)], + show_progress=False, + use_index=dataframe_use_index, + ) as tsdf: + assert tsdf.list_timeseries() == [ + "weather.device_a.temperature", + "weather.device_a.humidity", + "weather.device_a.pressure", + ] + aligned = tsdf.loc[ + :, + [ + "weather.device_a.temperature", + "weather.device_a.humidity", + "weather.device_a.pressure", + ], + ] + np.testing.assert_array_equal(aligned.timestamps, np.array([0, 1, 2, 3, 4])) + np.testing.assert_equal( + np.asarray(aligned.values), + np.array( + [ + [20.0, 50.0, np.nan], + [21.5, 52.0, np.nan], + [23.0, 55.0, np.nan], + [24.0, 56.0, 1000.0], + [25.0, 57.0, 1001.0], + ] + ), ) +def test_dataset_unions_disjoint_field_sets_across_shards( + tmp_path, dataframe_use_index +): + paths = [] + for name, column, value in ( + ("temperature.tsfile", _WEATHER_TEMPERATURE, 1.0), + ("humidity.tsfile", _WEATHER_HUMIDITY, 2.0), + ("pressure.tsfile", _WEATHER_PRESSURE, 3.0), + ): + path = tmp_path / name + _write_table_rows_file( + path, + [_WEATHER_DEVICE_TAG, column], + { + "time": [0], + "device": ["device_a"], + column.get_column_name(): [value], + }, + ) + paths.append(str(path)) + + with TsFileDataFrame( + paths, show_progress=False, use_index=dataframe_use_index + ) as tsdf: + assert tsdf.list_timeseries() == [ + "weather.device_a.temperature", + "weather.device_a.humidity", + "weather.device_a.pressure", + ] + aligned = tsdf.loc[:, tsdf.list_timeseries()] + np.testing.assert_array_equal(aligned.timestamps, np.array([0])) + np.testing.assert_equal(np.asarray(aligned.values), np.array([[1.0, 2.0, 3.0]])) + + +def test_dataset_unions_overlapping_field_subsets_for_different_devices( + tmp_path, dataframe_use_index +): + path1 = tmp_path / "temperature.tsfile" + path2 = tmp_path / "humidity.tsfile" + path3 = tmp_path / "both.tsfile" + _write_table_rows_file( + path1, + [_WEATHER_DEVICE_TAG, _WEATHER_TEMPERATURE], + {"time": [0], "device": ["device_a"], "temperature": [1.0]}, + ) + _write_table_rows_file( + path2, + [_WEATHER_DEVICE_TAG, _WEATHER_HUMIDITY], + {"time": [1], "device": ["device_b"], "humidity": [9.0]}, + ) + _write_table_rows_file( + path3, + [_WEATHER_DEVICE_TAG, _WEATHER_TEMPERATURE, _WEATHER_HUMIDITY], + {"time": [2], "device": ["device_a"], "temperature": [5.0], "humidity": [6.0]}, + ) + + with TsFileDataFrame( + [str(path1), str(path2), str(path3)], + show_progress=False, + use_index=dataframe_use_index, + ) as tsdf: + assert set(tsdf.list_timeseries()) == { + "weather.device_a.temperature", + "weather.device_a.humidity", + "weather.device_b.humidity", + } + aligned = tsdf.loc[ + :, + [ + "weather.device_a.temperature", + "weather.device_a.humidity", + "weather.device_b.humidity", + ], + ] + np.testing.assert_array_equal(aligned.timestamps, np.array([0, 1, 2])) + np.testing.assert_equal( + np.asarray(aligned.values), + np.array( + [ + [1.0, np.nan, np.nan], + [np.nan, np.nan, 9.0], + [5.0, 6.0, np.nan], + ] + ), + ) + + +def test_dataset_unions_tag_columns_with_null_padding(tmp_path, dataframe_use_index): + # TAG columns define device identity: a file written before a trailing tag + # existed keeps its shorter device ids, and the dataset pads the missing + # tag values with null. + region_only = tmp_path / "region.tsfile" + region_and_site = tmp_path / "region_site.tsfile" + _write_table_rows_file( + region_only, + [ + ColumnSchema("region", TSDataType.STRING, ColumnCategory.TAG), + _WEATHER_TEMPERATURE, + ], + {"time": [0], "region": ["east"], "temperature": [1.0]}, + ) + _write_table_rows_file( + region_and_site, + [ + ColumnSchema("region", TSDataType.STRING, ColumnCategory.TAG), + ColumnSchema("site", TSDataType.STRING, ColumnCategory.TAG), + _WEATHER_TEMPERATURE, + ], + { + "time": [1, 2], + "region": ["east", "west"], + "site": ["s1", "s2"], + "temperature": [2.0, 3.0], + }, + ) + + expected = { + "weather.east.temperature": [1.0], + "weather.east.s1.temperature": [2.0], + "weather.west.s2.temperature": [3.0], + } + # The union must not depend on which file is loaded first. + for paths in ( + [str(region_only), str(region_and_site)], + [str(region_and_site), str(region_only)], + ): + with TsFileDataFrame( + paths, show_progress=False, use_index=dataframe_use_index + ) as tsdf: + assert set(tsdf.list_timeseries()) == set(expected) + for name, values in expected.items(): + np.testing.assert_equal(tsdf[name][:], np.array(values)) + metadata = tsdf.list_timeseries_metadata() + assert list(metadata.columns)[-2:] == ["region", "site"] + assert np.isnan(metadata.loc["weather.east.temperature", "site"]) + assert metadata.loc["weather.east.s1.temperature", "site"] == "s1" + + +def test_dataset_unions_nonnumeric_declared_columns_without_exposing_them( + tmp_path, dataframe_use_index +): + path1 = tmp_path / "status.tsfile" + path2 = tmp_path / "note.tsfile" + _write_table_rows_file( + path1, + [ + _WEATHER_DEVICE_TAG, + _WEATHER_TEMPERATURE, + ColumnSchema("status", TSDataType.STRING, ColumnCategory.FIELD), + ], + { + "time": [0], + "device": ["device_a"], + "temperature": [1.0], + "status": ["ok"], + }, + ) + _write_table_rows_file( + path2, + [ + _WEATHER_DEVICE_TAG, + _WEATHER_TEMPERATURE, + ColumnSchema("note", TSDataType.STRING, ColumnCategory.FIELD), + ], + { + "time": [1], + "device": ["device_a"], + "temperature": [2.0], + "note": ["calm"], + }, + ) + + with TsFileDataFrame( + [str(path1), str(path2)], + show_progress=False, + use_index=dataframe_use_index, + ) as tsdf: + assert tsdf.list_timeseries() == ["weather.device_a.temperature"] + schema_columns = tsdf._index.table_entries["weather"].schema_columns + assert [name for name, _, _ in schema_columns] == [ + "device", + "temperature", + "status", + "note", + ] + + def test_dataset_rejects_same_field_name_with_different_type( tmp_path, dataframe_use_index ): @@ -1296,7 +1525,10 @@ def test_dataset_rejects_same_field_name_with_different_type( _write_weather_file(path1, 0) _write_weather_int_temperature_file(path2, 3) - with pytest.raises(ValueError, match="Incompatible schema for table 'weather'"): + with pytest.raises( + ValueError, + match="Incompatible field column 'temperature' for table 'weather'.*INT64.*DOUBLE", + ): TsFileDataFrame( [str(path1), str(path2)], show_progress=False, @@ -1304,7 +1536,7 @@ def test_dataset_rejects_same_field_name_with_different_type( ) -def test_dataset_rejects_nonnumeric_declared_schema_difference( +def test_dataset_rejects_same_nonnumeric_field_name_with_different_type( tmp_path, dataframe_use_index ): path1 = tmp_path / "text-status.tsfile" @@ -1312,7 +1544,76 @@ def test_dataset_rejects_nonnumeric_declared_schema_difference( _write_numeric_and_text_file(path1) _write_weather_boolean_status_file(path2, 3) - with pytest.raises(ValueError, match="Incompatible schema for table 'weather'"): + with pytest.raises( + ValueError, + match="Incompatible field column 'status' for table 'weather'.*BOOLEAN.*STRING", + ): + TsFileDataFrame( + [str(path1), str(path2)], + show_progress=False, + use_index=dataframe_use_index, + ) + + +def test_dataset_rejects_incompatible_tag_prefix(tmp_path, dataframe_use_index): + # A differing shared tag prefix means the table was dropped and recreated + # with a new schema, so the shards cannot be stitched into one device id. + path1 = tmp_path / "region.tsfile" + path2 = tmp_path / "zone.tsfile" + _write_table_rows_file( + path1, + [ + ColumnSchema("region", TSDataType.STRING, ColumnCategory.TAG), + _WEATHER_TEMPERATURE, + ], + {"time": [0], "region": ["east"], "temperature": [1.0]}, + ) + _write_table_rows_file( + path2, + [ + ColumnSchema("zone", TSDataType.STRING, ColumnCategory.TAG), + _WEATHER_TEMPERATURE, + ], + {"time": [1], "zone": ["east"], "temperature": [2.0]}, + ) + + with pytest.raises( + ValueError, + match="Incompatible tag column for table 'weather'.*'zone'.*'region'", + ): + TsFileDataFrame( + [str(path1), str(path2)], + show_progress=False, + use_index=dataframe_use_index, + ) + + +def test_dataset_rejects_same_tag_name_with_different_type( + tmp_path, dataframe_use_index +): + path1 = tmp_path / "string-region.tsfile" + path2 = tmp_path / "text-region.tsfile" + _write_table_rows_file( + path1, + [ + ColumnSchema("region", TSDataType.STRING, ColumnCategory.TAG), + _WEATHER_TEMPERATURE, + ], + {"time": [0], "region": ["east"], "temperature": [1.0]}, + ) + _write_table_rows_file( + path2, + [ + ColumnSchema("region", TSDataType.TEXT, ColumnCategory.TAG), + _WEATHER_TEMPERATURE, + ], + {"time": [1], "region": ["east"], "temperature": [2.0]}, + ) + + with pytest.raises( + ValueError, + match="Incompatible tag column for table 'weather'.*'region'.*TEXT.*STRING", + ): TsFileDataFrame( [str(path1), str(path2)], show_progress=False, diff --git a/python/tsfile/dataset/dataframe.py b/python/tsfile/dataset/dataframe.py index 893419d96..0ee5d0c6f 100644 --- a/python/tsfile/dataset/dataframe.py +++ b/python/tsfile/dataset/dataframe.py @@ -29,6 +29,7 @@ import numpy as np +from ..constants import NUMERIC_DATASET_FIELD_TYPES, ColumnCategory, TSDataType from .formatting import format_dataframe_table from ._config import resolve_read_options from .metadata import ( @@ -51,6 +52,7 @@ _QUERY_START = np.iinfo(np.int64).min _QUERY_END = np.iinfo(np.int64).max _DATACLASS_SLOTS = {"slots": True} if sys.version_info >= (3, 10) else {} +_NUMERIC_FIELD_TYPE_IDS = frozenset(int(item) for item in NUMERIC_DATASET_FIELD_TYPES) # Overlap position reads use chunked k-way merge. Keep the default chunk small # enough to avoid large read amplification for `series[i]` / short slices, but # large enough to avoid excessive query_by_row round-trips when overlap spans @@ -122,30 +124,138 @@ def _series_lookup_hint(name: str) -> str: return f"Series not found: '{name}'. Use df.list_timeseries() to inspect available series." -def _validate_table_schema( +def _type_name(data_type) -> str: + """Render a data type for user-facing errors (``DOUBLE`` rather than ``4``).""" + try: + return TSDataType(int(data_type)).name + except (TypeError, ValueError): + return str(data_type) + + +def _column_description(name, data_type) -> str: + return f"'{name}' ({_type_name(data_type)})" + + +def _declared_schema_columns(entry: TableEntry) -> List[Tuple[str, int, int]]: + """Ordered ``(name, data_type, category)`` columns of one table entry. + + Table-model entries built by the reader always carry ``schema_columns``; + the fallback keeps hand-built entries (and older catalogs) usable. + """ + if entry.schema_columns: + return [ + (name, int(data_type), int(category)) + for name, data_type, category in entry.schema_columns + ] + columns = [ + (name, int(data_type), int(ColumnCategory.TAG)) + for name, data_type in zip(entry.tag_columns, entry.tag_types) + ] + columns.extend( + (name, int(data_type), int(ColumnCategory.FIELD)) + for name, data_type in zip(entry.field_columns, entry.field_types) + ) + return columns + + +def _merge_table_model_entries( existing: TableEntry, incoming: TableEntry, file_path: str -) -> None: - """Reject same-name tables whose complete ordered schema differs.""" - if ( - existing.schema_columns - and incoming.schema_columns - and existing.schema_columns == incoming.schema_columns - ): - return - if ( - not existing.schema_columns - and not incoming.schema_columns - and existing.tag_columns == incoming.tag_columns - and existing.tag_types == incoming.tag_types - and existing.field_columns == incoming.field_columns - and existing.field_types == incoming.field_types - ): - return +) -> TableEntry: + """Union two per-file table-model schemas into one global table. + + One physical TsFile registers only the columns it actually wrote, so the + same table may legitimately span files with different column sets. TAG + columns define device identity and must be prefix-compatible: the shared + prefix has to match by name and type (a mismatch means the table was + dropped and recreated with a different schema), while extra trailing tags + are appended and shorter devices read as null. FIELD columns are unioned + in first-seen order -- a device/field absent from a file simply has no + shard and reads back as NaN. + + A same-named FIELD column must keep one data type across every file: a + standalone load set has no external "current schema" to reconcile a type + change against, so a differing type is rejected. Non-numeric fields stay + in ``schema_columns`` (for the persistent index) but are not part of the + numeric ``field_columns`` surface, exactly as for a single file. + """ + table_name = existing.table_name + existing_tags = list(zip(existing.tag_columns, existing.tag_types)) + incoming_tags = list(zip(incoming.tag_columns, incoming.tag_types)) + for position in range(min(len(existing_tags), len(incoming_tags))): + existing_name, existing_type = existing_tags[position] + incoming_name, incoming_type = incoming_tags[position] + if existing_name != incoming_name or existing_type != incoming_type: + raise ValueError( + f"Incompatible tag column for table '{table_name}' in " + f"'{file_path}': position {position} is " + f"{_column_description(incoming_name, incoming_type)} but earlier " + f"TsFiles declare {_column_description(existing_name, existing_type)}. " + "Tag columns must share the same name and type across TsFiles; a " + "differing shared prefix means the table was recreated with a new " + "schema." + ) + if len(incoming_tags) > len(existing_tags): + tag_columns, tag_types = incoming.tag_columns, incoming.tag_types + else: + tag_columns, tag_types = existing.tag_columns, existing.tag_types + + declared_fields: List[Tuple[str, int]] = [] + field_type_by_name: Dict[str, int] = {} + time_columns: List[Tuple[str, int, int]] = [] + for entry in (existing, incoming): + for name, data_type, category in _declared_schema_columns(entry): + if category == int(ColumnCategory.TIME): + if entry is existing: + time_columns.append((name, data_type, category)) + continue + if category != int(ColumnCategory.FIELD): + continue + previous_type = field_type_by_name.get(name) + if previous_type is None: + field_type_by_name[name] = data_type + declared_fields.append((name, data_type)) + elif previous_type != data_type: + raise ValueError( + f"Incompatible field column '{name}' for table " + f"'{table_name}' in '{file_path}': declared " + f"{_type_name(data_type)} here but " + f"{_type_name(previous_type)} in earlier TsFiles. A field " + "column must keep the same data type across TsFiles." + ) + + tag_column_set = set(tag_columns) + for name, _ in declared_fields: + if name in tag_column_set: + raise ValueError( + f"Incompatible column '{name}' for table '{table_name}' in " + f"'{file_path}': it is a TAG column in some TsFiles and a FIELD " + "column in others. A column cannot change category across TsFiles." + ) - raise ValueError( - f"Incompatible schema for table '{incoming.table_name}' in '{file_path}'. " - f"Expected columns={list(existing.schema_columns)} but found " - f"columns={list(incoming.schema_columns)}." + numeric_fields = [ + (name, data_type) + for name, data_type in declared_fields + if data_type in _NUMERIC_FIELD_TYPE_IDS + ] + schema_columns = ( + tuple(time_columns) + + tuple( + (name, int(data_type), int(ColumnCategory.TAG)) + for name, data_type in zip(tag_columns, tag_types) + ) + + tuple( + (name, data_type, int(ColumnCategory.FIELD)) + for name, data_type in declared_fields + ) + ) + + return TableEntry( + table_name=table_name, + tag_columns=tag_columns, + tag_types=tag_types, + field_columns=tuple(name for name, _ in numeric_fields), + field_types=tuple(TSDataType(data_type) for _, data_type in numeric_fields), + schema_columns=schema_columns, ) @@ -216,7 +326,11 @@ def _register_reader( existing_entry, table_entry ) else: - _validate_table_schema(existing_entry, table_entry, file_path) + # Table-model files may register only the columns they physically + # wrote, so union this file's table into the cross-file view. + index.table_entries[table_entry.table_name] = _merge_table_model_entries( + existing_entry, table_entry, file_path + ) for device_id, device_entry in enumerate(catalog.device_entries): table_entry = catalog.table_entries[device_entry.table_id] @@ -246,9 +360,10 @@ def _register_reader( # Register every (device, field) pair the reader physically holds. The # logical series is keyed by the GLOBAL field index (resolved by measurement # name against the merged table), while the per-shard tuple keeps the - # reader-local field index for the read path. This lets tree shards with - # different field subsets/orders merge without mis-mapping measurements - # (for table model the global and local indices always coincide). + # reader-local field index for the read path. This lets shards with + # different field subsets/orders merge without mis-mapping measurements: + # tree files synthesize their own measurement order, and table-model files + # may register only the columns they wrote. for device_id, field_idx in catalog.series_stats_by_ref: device_entry = catalog.device_entries[device_id] reader_table_entry = catalog.table_entries[device_entry.table_id] diff --git a/skills/tsfile/references/dataframe-arrow.md b/skills/tsfile/references/dataframe-arrow.md index d82734242..f95ef2dfe 100644 --- a/skills/tsfile/references/dataframe-arrow.md +++ b/skills/tsfile/references/dataframe-arrow.md @@ -70,6 +70,15 @@ subset view to close shared readers. Boolean dataset selection is positional: build the mask from `ds[...]`, as above; use `list_timeseries_metadata()` for named inspection rather than as the mask source. +Table-model files may register only the columns they actually wrote, so the +same table can span files with different column sets. The dataset unions the +FIELD columns in first-seen order and reads a device/field missing from a file +as NaN; a same-named FIELD column must keep one data type across files. TAG +columns are prefix-compatible: extra trailing tags may be added by later files, +and devices from files without them read back with a null tag, while a differing +shared tag prefix (a recreated table) is rejected. Duplicate timestamps for one +logical series across files remain rejected. + `use_index=True` builds/reuses an index (default `False`). Indexed mode assumes immutable sources (`trust_index=True`); use `False` for generation checks and stale-index rebuilds. It also accepts `max_prepared_series`, `descriptor_cache_size`, From 3522e9b0825d04e1cb7e2b6b9496fd017da81f59 Mon Sep 17 00:00:00 2001 From: ColinLee Date: Tue, 17 Nov 2026 12:38:56 +0800 Subject: [PATCH 2/7] feat(python): merge same-position tag renames across TsFiles Device identity in the dataset is the ordered tuple of tag values, not the tag column names, and IoTDB keeps tag ordinals across RENAME COLUMN, so a tag renamed at the same position is the same tag: keep the first-seen name for display and let both files' devices line up position by position. Only a data-type change at a shared tag position is still rejected, since it would make the shared device ids uninterpretable. --- python/tests/test_tsfile_dataset.py | 89 +++++++++++++++++---- python/tsfile/dataset/dataframe.py | 40 +++++---- skills/tsfile/references/dataframe-arrow.md | 8 +- 3 files changed, 100 insertions(+), 37 deletions(-) diff --git a/python/tests/test_tsfile_dataset.py b/python/tests/test_tsfile_dataset.py index 489d73d53..05176c89e 100644 --- a/python/tests/test_tsfile_dataset.py +++ b/python/tests/test_tsfile_dataset.py @@ -1464,7 +1464,7 @@ def test_dataset_unions_tag_columns_with_null_padding(tmp_path, dataframe_use_in np.testing.assert_equal(tsdf[name][:], np.array(values)) metadata = tsdf.list_timeseries_metadata() assert list(metadata.columns)[-2:] == ["region", "site"] - assert np.isnan(metadata.loc["weather.east.temperature", "site"]) + assert pd.isna(metadata.loc["weather.east.temperature", "site"]) assert metadata.loc["weather.east.s1.temperature", "site"] == "s1" @@ -1555,11 +1555,13 @@ def test_dataset_rejects_same_nonnumeric_field_name_with_different_type( ) -def test_dataset_rejects_incompatible_tag_prefix(tmp_path, dataframe_use_index): - # A differing shared tag prefix means the table was dropped and recreated - # with a new schema, so the shards cannot be stitched into one device id. +def test_dataset_unions_renamed_tag_columns(tmp_path, dataframe_use_index): + # IoTDB keeps tag ordinals across RENAME COLUMN, and device identity in the + # dataset is the ordered tag *values*, not the tag column names, so a + # same-position rename keeps one tag column under the first-seen name and + # the devices still line up. path1 = tmp_path / "region.tsfile" - path2 = tmp_path / "zone.tsfile" + path2 = tmp_path / "site.tsfile" _write_table_rows_file( path1, [ @@ -1571,24 +1573,74 @@ def test_dataset_rejects_incompatible_tag_prefix(tmp_path, dataframe_use_index): _write_table_rows_file( path2, [ + ColumnSchema("site", TSDataType.STRING, ColumnCategory.TAG), + _WEATHER_TEMPERATURE, + ], + {"time": [1], "site": ["east"], "temperature": [2.0]}, + ) + + # Device identity carries no tag column name, so both files' rows land on + # the same device in either load order. Exactly one of the two names + # survives: the first-declaring file's. (An index fixes that choice per + # file set, so the indexed run may reuse the name from the first order.) + for paths in ([str(path1), str(path2)], [str(path2), str(path1)]): + with TsFileDataFrame( + paths, show_progress=False, use_index=dataframe_use_index + ) as tsdf: + assert tsdf.list_timeseries() == ["weather.east.temperature"] + np.testing.assert_equal( + tsdf["weather.east.temperature"][:], np.array([1.0, 2.0]) + ) + metadata = tsdf.list_timeseries_metadata() + surviving_names = set(metadata.columns) & {"region", "site"} + assert len(surviving_names) == 1 + tag_name = surviving_names.pop() + assert metadata.loc["weather.east.temperature", tag_name] == "east" + + +def test_dataset_unions_renamed_and_appended_tag_columns(tmp_path, dataframe_use_index): + # A rename of the shared prefix combines with a trailing tag added later. + path1 = tmp_path / "region.tsfile" + path2 = tmp_path / "site_zone.tsfile" + _write_table_rows_file( + path1, + [ + ColumnSchema("region", TSDataType.STRING, ColumnCategory.TAG), + _WEATHER_TEMPERATURE, + ], + {"time": [0], "region": ["east"], "temperature": [1.0]}, + ) + _write_table_rows_file( + path2, + [ + ColumnSchema("site", TSDataType.STRING, ColumnCategory.TAG), ColumnSchema("zone", TSDataType.STRING, ColumnCategory.TAG), _WEATHER_TEMPERATURE, ], - {"time": [1], "zone": ["east"], "temperature": [2.0]}, + { + "time": [1, 2], + "site": ["east", "west"], + "zone": ["z1", "z2"], + "temperature": [2.0, 3.0], + }, ) - with pytest.raises( - ValueError, - match="Incompatible tag column for table 'weather'.*'zone'.*'region'", - ): - TsFileDataFrame( - [str(path1), str(path2)], - show_progress=False, - use_index=dataframe_use_index, - ) + with TsFileDataFrame( + [str(path1), str(path2)], + show_progress=False, + use_index=dataframe_use_index, + ) as tsdf: + assert set(tsdf.list_timeseries()) == { + "weather.east.temperature", + "weather.east.z1.temperature", + "weather.west.z2.temperature", + } + metadata = tsdf.list_timeseries_metadata() + assert list(metadata.columns)[-2:] == ["region", "zone"] + assert pd.isna(metadata.loc["weather.east.temperature", "zone"]) -def test_dataset_rejects_same_tag_name_with_different_type( +def test_dataset_rejects_same_tag_position_with_different_type( tmp_path, dataframe_use_index ): path1 = tmp_path / "string-region.tsfile" @@ -1612,7 +1664,10 @@ def test_dataset_rejects_same_tag_name_with_different_type( with pytest.raises( ValueError, - match="Incompatible tag column for table 'weather'.*'region'.*TEXT.*STRING", + match=( + "Incompatible tag column at position 0 for table 'weather'.*" + "'region' \\(TEXT\\).*'region' \\(STRING\\)" + ), ): TsFileDataFrame( [str(path1), str(path2)], diff --git a/python/tsfile/dataset/dataframe.py b/python/tsfile/dataset/dataframe.py index 0ee5d0c6f..647df048f 100644 --- a/python/tsfile/dataset/dataframe.py +++ b/python/tsfile/dataset/dataframe.py @@ -165,12 +165,16 @@ def _merge_table_model_entries( One physical TsFile registers only the columns it actually wrote, so the same table may legitimately span files with different column sets. TAG - columns define device identity and must be prefix-compatible: the shared - prefix has to match by name and type (a mismatch means the table was - dropped and recreated with a different schema), while extra trailing tags - are appended and shorter devices read as null. FIELD columns are unioned - in first-seen order -- a device/field absent from a file simply has no - shard and reads back as NaN. + columns define device identity positionally -- a device is keyed by its + ordered tag *values*, never by tag column names -- so tag columns merge as + a prefix union: the shared prefix must keep the same data type (a type + change makes the tag values uninterpretable), while a trailing-tag + addition is appended and shorter devices read as null. A same-position + tag whose name differs is a renamed column (IoTDB keeps tag ordinals across + RENAME COLUMN), so the merged schema keeps the first-seen name and the + values still line up position by position. FIELD columns are unioned in + first-seen order -- a device/field absent from a file simply has no shard + and reads back as NaN. A same-named FIELD column must keep one data type across every file: a standalone load set has no external "current schema" to reconcile a type @@ -181,23 +185,25 @@ def _merge_table_model_entries( table_name = existing.table_name existing_tags = list(zip(existing.tag_columns, existing.tag_types)) incoming_tags = list(zip(incoming.tag_columns, incoming.tag_types)) - for position in range(min(len(existing_tags), len(incoming_tags))): + shared_tag_count = min(len(existing_tags), len(incoming_tags)) + for position in range(shared_tag_count): existing_name, existing_type = existing_tags[position] incoming_name, incoming_type = incoming_tags[position] - if existing_name != incoming_name or existing_type != incoming_type: + if existing_type != incoming_type: raise ValueError( - f"Incompatible tag column for table '{table_name}' in " - f"'{file_path}': position {position} is " + f"Incompatible tag column at position {position} for table " + f"'{table_name}' in '{file_path}': " f"{_column_description(incoming_name, incoming_type)} but earlier " f"TsFiles declare {_column_description(existing_name, existing_type)}. " - "Tag columns must share the same name and type across TsFiles; a " - "differing shared prefix means the table was recreated with a new " - "schema." + "Tag columns must keep the same data type across TsFiles; a " + "differing type makes the shared device ids uninterpretable." ) - if len(incoming_tags) > len(existing_tags): - tag_columns, tag_types = incoming.tag_columns, incoming.tag_types - else: - tag_columns, tag_types = existing.tag_columns, existing.tag_types + tag_columns = tuple(name for name, _ in existing_tags) + tuple( + name for name, _ in incoming_tags[shared_tag_count:] + ) + tag_types = tuple(data_type for _, data_type in existing_tags) + tuple( + data_type for _, data_type in incoming_tags[shared_tag_count:] + ) declared_fields: List[Tuple[str, int]] = [] field_type_by_name: Dict[str, int] = {} diff --git a/skills/tsfile/references/dataframe-arrow.md b/skills/tsfile/references/dataframe-arrow.md index f95ef2dfe..f3cf73135 100644 --- a/skills/tsfile/references/dataframe-arrow.md +++ b/skills/tsfile/references/dataframe-arrow.md @@ -75,9 +75,11 @@ same table can span files with different column sets. The dataset unions the FIELD columns in first-seen order and reads a device/field missing from a file as NaN; a same-named FIELD column must keep one data type across files. TAG columns are prefix-compatible: extra trailing tags may be added by later files, -and devices from files without them read back with a null tag, while a differing -shared tag prefix (a recreated table) is rejected. Duplicate timestamps for one -logical series across files remain rejected. +and devices from files without them read back with a null tag. Device identity +is the ordered tag *values*, not the tag column names, so a tag renamed at the +same position merges under the first-seen name instead of failing; a tag at the +same position with a different data type is rejected. Duplicate timestamps for +one logical series across files remain rejected. `use_index=True` builds/reuses an index (default `False`). Indexed mode assumes immutable sources (`trust_index=True`); use `False` for generation checks and From 0c9a47ecac66059fc34d53f0e1656648e0f7f2e9 Mon Sep 17 00:00:00 2001 From: ColinLee Date: Tue, 17 Nov 2026 15:19:53 +0800 Subject: [PATCH 3/7] refactor(python): make the table-model merge a pure union The dataset only consumes the table name, the positional tag values that identify a device, the readable field names and where each shard lives; reads go through every file's own catalog and never use an authored declaration. Drop the remaining schema comparisons: tag types, field declared types and tag/field categories are no longer validated, and a tag renamed at the same position simply keeps the first-seen name. The merged entry carries no field_types or schema_columns, so the persistent index derives column types from the physical data (its cross-shard physical-type check is unchanged). One rejection remains: a name that is a TAG in one file and a FIELD in another, because the index keys columns by (table, name) and a duplicate makes lookups ambiguous. --- python/tests/test_tsfile_dataset.py | 79 ++++------ python/tsfile/dataset/dataframe.py | 166 ++++++-------------- skills/tsfile/references/dataframe-arrow.md | 14 +- 3 files changed, 82 insertions(+), 177 deletions(-) diff --git a/python/tests/test_tsfile_dataset.py b/python/tests/test_tsfile_dataset.py index 05176c89e..be45e36ba 100644 --- a/python/tests/test_tsfile_dataset.py +++ b/python/tests/test_tsfile_dataset.py @@ -1508,50 +1508,35 @@ def test_dataset_unions_nonnumeric_declared_columns_without_exposing_them( use_index=dataframe_use_index, ) as tsdf: assert tsdf.list_timeseries() == ["weather.device_a.temperature"] - schema_columns = tsdf._index.table_entries["weather"].schema_columns - assert [name for name, _, _ in schema_columns] == [ - "device", - "temperature", - "status", - "note", - ] + entry = tsdf._index.table_entries["weather"] + assert entry.field_columns == ("temperature",) -def test_dataset_rejects_same_field_name_with_different_type( - tmp_path, dataframe_use_index -): +def test_dataset_merges_same_field_with_different_types(tmp_path, dataframe_use_index): + # The dataset does not compare authored declarations: each shard is read + # through its own catalog and converted to float64, so a same-named field + # with different declared types merges. The persistent index still refuses + # shards whose *physical* types disagree, exactly as it did before. path1 = tmp_path / "double.tsfile" path2 = tmp_path / "int64.tsfile" _write_weather_file(path1, 0) _write_weather_int_temperature_file(path2, 3) - with pytest.raises( - ValueError, - match="Incompatible field column 'temperature' for table 'weather'.*INT64.*DOUBLE", - ): - TsFileDataFrame( - [str(path1), str(path2)], - show_progress=False, - use_index=dataframe_use_index, - ) - - -def test_dataset_rejects_same_nonnumeric_field_name_with_different_type( - tmp_path, dataframe_use_index -): - path1 = tmp_path / "text-status.tsfile" - path2 = tmp_path / "boolean-status.tsfile" - _write_numeric_and_text_file(path1) - _write_weather_boolean_status_file(path2, 3) + if dataframe_use_index: + with pytest.raises(ValueError, match="incompatible physical type"): + TsFileDataFrame( + [str(path1), str(path2)], show_progress=False, use_index=True + ) + return - with pytest.raises( - ValueError, - match="Incompatible field column 'status' for table 'weather'.*BOOLEAN.*STRING", - ): - TsFileDataFrame( - [str(path1), str(path2)], - show_progress=False, - use_index=dataframe_use_index, + with TsFileDataFrame([str(path1), str(path2)], show_progress=False) as tsdf: + assert tsdf.list_timeseries() == [ + "weather.device_a.temperature", + "weather.device_a.humidity", + ] + np.testing.assert_equal( + tsdf["weather.device_a.temperature"][:], + np.array([20.0, 21.5, 23.0, 20.0, 21.0]), ) @@ -1640,11 +1625,12 @@ def test_dataset_unions_renamed_and_appended_tag_columns(tmp_path, dataframe_use assert pd.isna(metadata.loc["weather.east.temperature", "zone"]) -def test_dataset_rejects_same_tag_position_with_different_type( - tmp_path, dataframe_use_index -): - path1 = tmp_path / "string-region.tsfile" - path2 = tmp_path / "text-region.tsfile" +def test_dataset_rejects_tag_and_field_with_same_name(tmp_path, dataframe_use_index): + # The only definition-shape conflict left: one name cannot be both a tag + # and a field of the same table, because the persistent index keys columns + # by (table, name) and a duplicate would make series lookups ambiguous. + path1 = tmp_path / "tag-region.tsfile" + path2 = tmp_path / "field-region.tsfile" _write_table_rows_file( path1, [ @@ -1656,18 +1642,15 @@ def test_dataset_rejects_same_tag_position_with_different_type( _write_table_rows_file( path2, [ - ColumnSchema("region", TSDataType.TEXT, ColumnCategory.TAG), - _WEATHER_TEMPERATURE, + _WEATHER_DEVICE_TAG, + ColumnSchema("region", TSDataType.DOUBLE, ColumnCategory.FIELD), ], - {"time": [1], "region": ["east"], "temperature": [2.0]}, + {"time": [1], "device": ["device_a"], "region": [2.0]}, ) with pytest.raises( ValueError, - match=( - "Incompatible tag column at position 0 for table 'weather'.*" - "'region' \\(TEXT\\).*'region' \\(STRING\\)" - ), + match="Column 'region' of table 'weather'.*TAG in some TsFiles and a FIELD", ): TsFileDataFrame( [str(path1), str(path2)], diff --git a/python/tsfile/dataset/dataframe.py b/python/tsfile/dataset/dataframe.py index 647df048f..b5d5067cd 100644 --- a/python/tsfile/dataset/dataframe.py +++ b/python/tsfile/dataset/dataframe.py @@ -29,7 +29,6 @@ import numpy as np -from ..constants import NUMERIC_DATASET_FIELD_TYPES, ColumnCategory, TSDataType from .formatting import format_dataframe_table from ._config import resolve_read_options from .metadata import ( @@ -52,7 +51,6 @@ _QUERY_START = np.iinfo(np.int64).min _QUERY_END = np.iinfo(np.int64).max _DATACLASS_SLOTS = {"slots": True} if sys.version_info >= (3, 10) else {} -_NUMERIC_FIELD_TYPE_IDS = frozenset(int(item) for item in NUMERIC_DATASET_FIELD_TYPES) # Overlap position reads use chunked k-way merge. Keep the default chunk small # enough to avoid large read amplification for `series[i]` / short slices, but # large enough to avoid excessive query_by_row round-trips when overlap spans @@ -124,144 +122,68 @@ def _series_lookup_hint(name: str) -> str: return f"Series not found: '{name}'. Use df.list_timeseries() to inspect available series." -def _type_name(data_type) -> str: - """Render a data type for user-facing errors (``DOUBLE`` rather than ``4``).""" - try: - return TSDataType(int(data_type)).name - except (TypeError, ValueError): - return str(data_type) - - -def _column_description(name, data_type) -> str: - return f"'{name}' ({_type_name(data_type)})" - - -def _declared_schema_columns(entry: TableEntry) -> List[Tuple[str, int, int]]: - """Ordered ``(name, data_type, category)`` columns of one table entry. - - Table-model entries built by the reader always carry ``schema_columns``; - the fallback keeps hand-built entries (and older catalogs) usable. - """ - if entry.schema_columns: - return [ - (name, int(data_type), int(category)) - for name, data_type, category in entry.schema_columns - ] - columns = [ - (name, int(data_type), int(ColumnCategory.TAG)) - for name, data_type in zip(entry.tag_columns, entry.tag_types) - ] - columns.extend( - (name, int(data_type), int(ColumnCategory.FIELD)) - for name, data_type in zip(entry.field_columns, entry.field_types) - ) - return columns - - def _merge_table_model_entries( existing: TableEntry, incoming: TableEntry, file_path: str ) -> TableEntry: """Union two per-file table-model schemas into one global table. - One physical TsFile registers only the columns it actually wrote, so the - same table may legitimately span files with different column sets. TAG - columns define device identity positionally -- a device is keyed by its - ordered tag *values*, never by tag column names -- so tag columns merge as - a prefix union: the shared prefix must keep the same data type (a type - change makes the tag values uninterpretable), while a trailing-tag - addition is appended and shorter devices read as null. A same-position - tag whose name differs is a renamed column (IoTDB keeps tag ordinals across - RENAME COLUMN), so the merged schema keeps the first-seen name and the - values still line up position by position. FIELD columns are unioned in - first-seen order -- a device/field absent from a file simply has no shard - and reads back as NaN. - - A same-named FIELD column must keep one data type across every file: a - standalone load set has no external "current schema" to reconcile a type - change against, so a differing type is rejected. Non-numeric fields stay - in ``schema_columns`` (for the persistent index) but are not part of the - numeric ``field_columns`` surface, exactly as for a single file. + A TsFile registers only the columns it actually wrote, and independently + written files may come from different schema snapshots, so the same table + legitimately spans files with different column sets. The dataset never + interprets an authored schema: reads go through each file's own catalog, + and the global view only needs the table name, the positional tag values + that identify a device, the field names it can expose, and where every + (device, field) shard lives. The merge is therefore a pure union -- no + declaration is compared, validated, or carried forward: + + - TAG columns are positional; a device is keyed by its ordered tag + *values*, never by column names. The longer list wins (devices from + files without a trailing tag read it as null), and a tag renamed at the + same position keeps the first-seen name, which is display-only. + - FIELD columns are the union of the readable fields in first-seen order. + A device/field absent from a file simply has no shard and reads back as + NaN. + + ``field_types`` and ``schema_columns`` are left empty, so persistent-index + construction derives column types from the physical data instead of a + declaration (``dataset/index.py`` has that fallback, plus its own + cross-shard physical-type check). """ table_name = existing.table_name - existing_tags = list(zip(existing.tag_columns, existing.tag_types)) - incoming_tags = list(zip(incoming.tag_columns, incoming.tag_types)) - shared_tag_count = min(len(existing_tags), len(incoming_tags)) - for position in range(shared_tag_count): - existing_name, existing_type = existing_tags[position] - incoming_name, incoming_type = incoming_tags[position] - if existing_type != incoming_type: - raise ValueError( - f"Incompatible tag column at position {position} for table " - f"'{table_name}' in '{file_path}': " - f"{_column_description(incoming_name, incoming_type)} but earlier " - f"TsFiles declare {_column_description(existing_name, existing_type)}. " - "Tag columns must keep the same data type across TsFiles; a " - "differing type makes the shared device ids uninterpretable." - ) - tag_columns = tuple(name for name, _ in existing_tags) + tuple( - name for name, _ in incoming_tags[shared_tag_count:] + tag_columns = tuple(existing.tag_columns) + tuple( + incoming.tag_columns[len(existing.tag_columns) :] ) - tag_types = tuple(data_type for _, data_type in existing_tags) + tuple( - data_type for _, data_type in incoming_tags[shared_tag_count:] + tag_types = tuple(existing.tag_types) + tuple( + incoming.tag_types[len(existing.tag_columns) :] ) - declared_fields: List[Tuple[str, int]] = [] - field_type_by_name: Dict[str, int] = {} - time_columns: List[Tuple[str, int, int]] = [] - for entry in (existing, incoming): - for name, data_type, category in _declared_schema_columns(entry): - if category == int(ColumnCategory.TIME): - if entry is existing: - time_columns.append((name, data_type, category)) - continue - if category != int(ColumnCategory.FIELD): - continue - previous_type = field_type_by_name.get(name) - if previous_type is None: - field_type_by_name[name] = data_type - declared_fields.append((name, data_type)) - elif previous_type != data_type: - raise ValueError( - f"Incompatible field column '{name}' for table " - f"'{table_name}' in '{file_path}': declared " - f"{_type_name(data_type)} here but " - f"{_type_name(previous_type)} in earlier TsFiles. A field " - "column must keep the same data type across TsFiles." - ) + field_columns = list(existing.field_columns) + seen = set(field_columns) + for name in incoming.field_columns: + if name not in seen: + seen.add(name) + field_columns.append(name) - tag_column_set = set(tag_columns) - for name, _ in declared_fields: - if name in tag_column_set: + # A name cannot be both a tag and a field of the same table: the + # persistent index keys columns by (table, name), so a duplicate would make + # series lookups ambiguous. This is a namespace conflict within one table, + # not a schema-version difference, so it is the one case still rejected. + tag_names = set(tag_columns) + for name in field_columns: + if name in tag_names: raise ValueError( - f"Incompatible column '{name}' for table '{table_name}' in " - f"'{file_path}': it is a TAG column in some TsFiles and a FIELD " - "column in others. A column cannot change category across TsFiles." + f"Column '{name}' of table '{table_name}' in '{file_path}' is a " + "TAG in some TsFiles and a FIELD in others. A column name cannot " + "be both a tag and a field of the same table." ) - numeric_fields = [ - (name, data_type) - for name, data_type in declared_fields - if data_type in _NUMERIC_FIELD_TYPE_IDS - ] - schema_columns = ( - tuple(time_columns) - + tuple( - (name, int(data_type), int(ColumnCategory.TAG)) - for name, data_type in zip(tag_columns, tag_types) - ) - + tuple( - (name, data_type, int(ColumnCategory.FIELD)) - for name, data_type in declared_fields - ) - ) - return TableEntry( table_name=table_name, tag_columns=tag_columns, tag_types=tag_types, - field_columns=tuple(name for name, _ in numeric_fields), - field_types=tuple(TSDataType(data_type) for _, data_type in numeric_fields), - schema_columns=schema_columns, + field_columns=tuple(field_columns), + field_types=(), + schema_columns=(), ) diff --git a/skills/tsfile/references/dataframe-arrow.md b/skills/tsfile/references/dataframe-arrow.md index f3cf73135..6c215671b 100644 --- a/skills/tsfile/references/dataframe-arrow.md +++ b/skills/tsfile/references/dataframe-arrow.md @@ -73,13 +73,13 @@ named inspection rather than as the mask source. Table-model files may register only the columns they actually wrote, so the same table can span files with different column sets. The dataset unions the FIELD columns in first-seen order and reads a device/field missing from a file -as NaN; a same-named FIELD column must keep one data type across files. TAG -columns are prefix-compatible: extra trailing tags may be added by later files, -and devices from files without them read back with a null tag. Device identity -is the ordered tag *values*, not the tag column names, so a tag renamed at the -same position merges under the first-seen name instead of failing; a tag at the -same position with a different data type is rejected. Duplicate timestamps for -one logical series across files remain rejected. +as NaN. TAG columns are positional: extra trailing tags may be added by later +files, and devices from files without them read back with a null tag. Device +identity is the ordered tag *values*, not the tag column names, so a tag +renamed at the same position merges under the first-seen name. The merge never +compares authored column definitions; the only rejected shape is a name that +is a TAG in one file and a FIELD in another. Duplicate timestamps for one +logical series across files remain rejected. `use_index=True` builds/reuses an index (default `False`). Indexed mode assumes immutable sources (`trust_index=True`); use `False` for generation checks and From 17face7967e8a4ef998cb139b8637b8110d4c8f0 Mon Sep 17 00:00:00 2001 From: ColinLee Date: Tue, 17 Nov 2026 15:28:42 +0800 Subject: [PATCH 4/7] fix(python): skip data-less fields when building the dataset index With the pure-union table merge, a numeric field that one file declares but no device ever writes enters the merged field_columns without any physical type. The index column fallback packed a -1 placeholder into COLUMN_SCHEMA (unsigned short) and the build failed with struct.error. Emit columns only for fields that actually have a physical type. Such a field has no series and is unreadable through the dataset anyway, so leaving it out matches the non-indexed view. This path was previously unreachable: tree fields always carry data, and table entries took the declared schema_columns branch. --- python/tests/test_tsfile_dataset.py | 35 +++++++++++++++++++++++++++++ python/tsfile/dataset/index.py | 24 ++++++++++++-------- 2 files changed, 50 insertions(+), 9 deletions(-) diff --git a/python/tests/test_tsfile_dataset.py b/python/tests/test_tsfile_dataset.py index be45e36ba..60f12f8a3 100644 --- a/python/tests/test_tsfile_dataset.py +++ b/python/tests/test_tsfile_dataset.py @@ -1512,6 +1512,41 @@ def test_dataset_unions_nonnumeric_declared_columns_without_exposing_them( assert entry.field_columns == ("temperature",) +def test_dataset_skips_declared_field_without_values(tmp_path, dataframe_use_index): + # A numeric field one file declares but never writes (all-NaN column) has + # no series, hence no physical type. It must stay unreadable and must not + # corrupt persistent-index construction (its column packing needs a type). + path1 = tmp_path / "declared.tsfile" + path2 = tmp_path / "written.tsfile" + _write_table_rows_file( + path1, + [_WEATHER_DEVICE_TAG, _WEATHER_TEMPERATURE, _WEATHER_PRESSURE], + { + "time": [0], + "device": ["device_a"], + "temperature": [1.0], + "pressure": [np.nan], + }, + ) + _write_table_rows_file( + path2, + [_WEATHER_DEVICE_TAG, _WEATHER_TEMPERATURE], + {"time": [1], "device": ["device_a"], "temperature": [2.0]}, + ) + + with TsFileDataFrame( + [str(path1), str(path2)], + show_progress=False, + use_index=dataframe_use_index, + ) as tsdf: + assert tsdf.list_timeseries() == ["weather.device_a.temperature"] + np.testing.assert_equal( + tsdf["weather.device_a.temperature"][:], np.array([1.0, 2.0]) + ) + with pytest.raises(KeyError, match="Series not found"): + tsdf["weather.device_a.pressure"][:] + + def test_dataset_merges_same_field_with_different_types(tmp_path, dataframe_use_index): # The dataset does not compare authored declarations: each shard is read # through its own catalog and converted to float64, so a same-named field diff --git a/python/tsfile/dataset/index.py b/python/tsfile/dataset/index.py index 7c1246ba0..a60484853 100644 --- a/python/tsfile/dataset/index.py +++ b/python/tsfile/dataset/index.py @@ -583,16 +583,22 @@ def build_sections_from_dataframe(dataframe) -> Mapping[int, bytes]: ) ] first_field_ordinal = len(definitions) + # A declared field that no device ever wrote has no series, so it + # has no physical type. It cannot be served by the index either -- + # packing a placeholder type is invalid -- so leave it out. + readable_fields = [ + name + for name in table.field_columns + if (table_name, name) in field_types + ] definitions.extend( - [ - ( - first_field_ordinal + ordinal, - name, - field_types.get((table_name, name), -1), - int(ColumnCategory.FIELD), - ) - for ordinal, name in enumerate(table.field_columns) - ] + ( + first_field_ordinal + ordinal, + name, + field_types[(table_name, name)], + int(ColumnCategory.FIELD), + ) + for ordinal, name in enumerate(readable_fields) ) for ordinal, name, data_type, role in definitions: column_id = len(columns) From 6cbe09601ea121483fabab4f63533033f9c46025 Mon Sep 17 00:00:00 2001 From: ColinLee Date: Tue, 17 Nov 2026 15:40:49 +0800 Subject: [PATCH 5/7] fix(python): reject a differing shared tag prefix across TsFiles Positional device identity only works while files agree on the tag columns they share. IoTDB never reorders tags, so a same table name with a different tag name at the same position means the table was recreated with a different tag layout; merging by position would conflate unrelated devices. Require the shared tag prefix to match by name and keep rejecting a name that is a TAG in one file and a FIELD in another. Appending trailing tags in either load order still unions correctly (devices from files without the new tag read it as null), and FIELD declarations remain unvalidated. --- python/tests/test_tsfile_dataset.py | 88 +++++---------------- python/tsfile/dataset/dataframe.py | 30 +++++-- skills/tsfile/references/dataframe-arrow.md | 14 ++-- 3 files changed, 50 insertions(+), 82 deletions(-) diff --git a/python/tests/test_tsfile_dataset.py b/python/tests/test_tsfile_dataset.py index 60f12f8a3..0eeeace1c 100644 --- a/python/tests/test_tsfile_dataset.py +++ b/python/tests/test_tsfile_dataset.py @@ -1575,11 +1575,11 @@ def test_dataset_merges_same_field_with_different_types(tmp_path, dataframe_use_ ) -def test_dataset_unions_renamed_tag_columns(tmp_path, dataframe_use_index): - # IoTDB keeps tag ordinals across RENAME COLUMN, and device identity in the - # dataset is the ordered tag *values*, not the tag column names, so a - # same-position rename keeps one tag column under the first-seen name and - # the devices still line up. +def test_dataset_rejects_different_tag_prefix(tmp_path, dataframe_use_index): + # Positional device identity only works while files agree on the tag + # columns they share. A different name at the same position means the table + # name points at a different tag layout, so merging would conflate + # unrelated devices; this must be rejected in either load order. path1 = tmp_path / "region.tsfile" path2 = tmp_path / "site.tsfile" _write_table_rows_file( @@ -1599,80 +1599,34 @@ def test_dataset_unions_renamed_tag_columns(tmp_path, dataframe_use_index): {"time": [1], "site": ["east"], "temperature": [2.0]}, ) - # Device identity carries no tag column name, so both files' rows land on - # the same device in either load order. Exactly one of the two names - # survives: the first-declaring file's. (An index fixes that choice per - # file set, so the indexed run may reuse the name from the first order.) for paths in ([str(path1), str(path2)], [str(path2), str(path1)]): - with TsFileDataFrame( - paths, show_progress=False, use_index=dataframe_use_index - ) as tsdf: - assert tsdf.list_timeseries() == ["weather.east.temperature"] - np.testing.assert_equal( - tsdf["weather.east.temperature"][:], np.array([1.0, 2.0]) - ) - metadata = tsdf.list_timeseries_metadata() - surviving_names = set(metadata.columns) & {"region", "site"} - assert len(surviving_names) == 1 - tag_name = surviving_names.pop() - assert metadata.loc["weather.east.temperature", tag_name] == "east" - - -def test_dataset_unions_renamed_and_appended_tag_columns(tmp_path, dataframe_use_index): - # A rename of the shared prefix combines with a trailing tag added later. - path1 = tmp_path / "region.tsfile" - path2 = tmp_path / "site_zone.tsfile" - _write_table_rows_file( - path1, - [ - ColumnSchema("region", TSDataType.STRING, ColumnCategory.TAG), - _WEATHER_TEMPERATURE, - ], - {"time": [0], "region": ["east"], "temperature": [1.0]}, - ) - _write_table_rows_file( - path2, - [ - ColumnSchema("site", TSDataType.STRING, ColumnCategory.TAG), - ColumnSchema("zone", TSDataType.STRING, ColumnCategory.TAG), - _WEATHER_TEMPERATURE, - ], - { - "time": [1, 2], - "site": ["east", "west"], - "zone": ["z1", "z2"], - "temperature": [2.0, 3.0], - }, - ) - - with TsFileDataFrame( - [str(path1), str(path2)], - show_progress=False, - use_index=dataframe_use_index, - ) as tsdf: - assert set(tsdf.list_timeseries()) == { - "weather.east.temperature", - "weather.east.z1.temperature", - "weather.west.z2.temperature", - } - metadata = tsdf.list_timeseries_metadata() - assert list(metadata.columns)[-2:] == ["region", "zone"] - assert pd.isna(metadata.loc["weather.east.temperature", "zone"]) + with pytest.raises( + ValueError, + match="must agree on the tag columns they share", + ): + TsFileDataFrame(paths, show_progress=False, use_index=dataframe_use_index) def test_dataset_rejects_tag_and_field_with_same_name(tmp_path, dataframe_use_index): - # The only definition-shape conflict left: one name cannot be both a tag - # and a field of the same table, because the persistent index keys columns - # by (table, name) and a duplicate would make series lookups ambiguous. + # One name cannot be both a tag and a field of the same table, because the + # persistent index keys columns by (table, name) and a duplicate would make + # series lookups ambiguous. Here file A's second tag is "region" while + # file B exposes "region" as a numeric field under a compatible tag prefix. path1 = tmp_path / "tag-region.tsfile" path2 = tmp_path / "field-region.tsfile" _write_table_rows_file( path1, [ + _WEATHER_DEVICE_TAG, ColumnSchema("region", TSDataType.STRING, ColumnCategory.TAG), _WEATHER_TEMPERATURE, ], - {"time": [0], "region": ["east"], "temperature": [1.0]}, + { + "time": [0], + "device": ["device_a"], + "region": ["east"], + "temperature": [1.0], + }, ) _write_table_rows_file( path2, diff --git a/python/tsfile/dataset/dataframe.py b/python/tsfile/dataset/dataframe.py index b5d5067cd..1620da611 100644 --- a/python/tsfile/dataset/dataframe.py +++ b/python/tsfile/dataset/dataframe.py @@ -133,13 +133,16 @@ def _merge_table_model_entries( interprets an authored schema: reads go through each file's own catalog, and the global view only needs the table name, the positional tag values that identify a device, the field names it can expose, and where every - (device, field) shard lives. The merge is therefore a pure union -- no + (device, field) shard lives. The merge is therefore a union -- no FIELD declaration is compared, validated, or carried forward: - TAG columns are positional; a device is keyed by its ordered tag - *values*, never by column names. The longer list wins (devices from - files without a trailing tag read it as null), and a tag renamed at the - same position keeps the first-seen name, which is display-only. + *values*, never by column names. Files may only *append* trailing tags + (devices from files without them read the new tag as null), so the tag + list a file shares with earlier ones must name the same columns. A + different name at the same position means the same table name refers to + a different tag layout -- IoTDB never reorders tags -- and merging by + position would conflate unrelated devices, so it is rejected. - FIELD columns are the union of the readable fields in first-seen order. A device/field absent from a file simply has no shard and reads back as NaN. @@ -150,12 +153,23 @@ def _merge_table_model_entries( cross-shard physical-type check). """ table_name = existing.table_name + shared_tag_count = min(len(existing.tag_columns), len(incoming.tag_columns)) + for position in range(shared_tag_count): + existing_name = existing.tag_columns[position] + incoming_name = incoming.tag_columns[position] + if existing_name != incoming_name: + raise ValueError( + f"Table '{table_name}' in '{file_path}' declares tag column " + f"'{incoming_name}' at position {position} but earlier TsFiles " + f"declare '{existing_name}'. TsFiles of one table must agree on " + "the tag columns they share; a different name means the table " + "was recreated with a different tag layout, and merging by " + "position would conflate unrelated devices." + ) tag_columns = tuple(existing.tag_columns) + tuple( - incoming.tag_columns[len(existing.tag_columns) :] - ) - tag_types = tuple(existing.tag_types) + tuple( - incoming.tag_types[len(existing.tag_columns) :] + incoming.tag_columns[shared_tag_count:] ) + tag_types = tuple(existing.tag_types) + tuple(incoming.tag_types[shared_tag_count:]) field_columns = list(existing.field_columns) seen = set(field_columns) diff --git a/skills/tsfile/references/dataframe-arrow.md b/skills/tsfile/references/dataframe-arrow.md index 6c215671b..d9afe4d44 100644 --- a/skills/tsfile/references/dataframe-arrow.md +++ b/skills/tsfile/references/dataframe-arrow.md @@ -73,13 +73,13 @@ named inspection rather than as the mask source. Table-model files may register only the columns they actually wrote, so the same table can span files with different column sets. The dataset unions the FIELD columns in first-seen order and reads a device/field missing from a file -as NaN. TAG columns are positional: extra trailing tags may be added by later -files, and devices from files without them read back with a null tag. Device -identity is the ordered tag *values*, not the tag column names, so a tag -renamed at the same position merges under the first-seen name. The merge never -compares authored column definitions; the only rejected shape is a name that -is a TAG in one file and a FIELD in another. Duplicate timestamps for one -logical series across files remain rejected. +as NaN. TAG columns are positional: files may append trailing tags, and devices +from files without them read back with a null tag, but files must agree on the +tag columns they share — a different name at the same position means the table +name points at a different tag layout and is rejected. A name that is a TAG in +one file and a FIELD in another is rejected as well. FIELD declarations are +never compared. Duplicate timestamps for one logical series across files remain +rejected. `use_index=True` builds/reuses an index (default `False`). Indexed mode assumes immutable sources (`trust_index=True`); use `False` for generation checks and From 31db22e861a527a179fa820e7739bfc63d96d800 Mon Sep 17 00:00:00 2001 From: ColinLee Date: Tue, 17 Nov 2026 15:50:09 +0800 Subject: [PATCH 6/7] test(python): cover incremental tag appends and merged index columns Address the review follow-ups on the table-model union: - exercise a three-file tag-append chain, where the second merge runs against an entry that already unions two files; - assert the persisted index defines a merged table as tags plus readable fields only (and that the in-memory merged entry carries no declaration); - share the first-seen field union between the table and tree merges; - correct a test comment that still described tag column names as part of device identity. --- python/tests/test_tsfile_dataset.py | 86 ++++++++++++++++++++++++++++- python/tsfile/dataset/dataframe.py | 28 +++++----- 2 files changed, 98 insertions(+), 16 deletions(-) diff --git a/python/tests/test_tsfile_dataset.py b/python/tests/test_tsfile_dataset.py index 0eeeace1c..0c93eb09e 100644 --- a/python/tests/test_tsfile_dataset.py +++ b/python/tests/test_tsfile_dataset.py @@ -1418,9 +1418,9 @@ def test_dataset_unions_overlapping_field_subsets_for_different_devices( def test_dataset_unions_tag_columns_with_null_padding(tmp_path, dataframe_use_index): - # TAG columns define device identity: a file written before a trailing tag - # existed keeps its shorter device ids, and the dataset pads the missing - # tag values with null. + # Device identity is the ordered tuple of tag *values*, not the tag column + # names: a file written before a trailing tag existed keeps its shorter + # device ids, and the dataset pads the missing tag values with null. region_only = tmp_path / "region.tsfile" region_and_site = tmp_path / "region_site.tsfile" _write_table_rows_file( @@ -1468,6 +1468,75 @@ def test_dataset_unions_tag_columns_with_null_padding(tmp_path, dataframe_use_in assert metadata.loc["weather.east.s1.temperature", "site"] == "s1" +def test_dataset_unions_incremental_tag_appends_across_three_files( + tmp_path, dataframe_use_index +): + # The merge is incremental: the third file is merged against an entry that + # already unions two files, so an appended tag must survive the second + # merge instead of the later file's shorter tag list winning. + first = tmp_path / "one_tag.tsfile" + second = tmp_path / "two_tags.tsfile" + third = tmp_path / "three_tags.tsfile" + _write_table_rows_file( + first, + [ + ColumnSchema("region", TSDataType.STRING, ColumnCategory.TAG), + _WEATHER_TEMPERATURE, + ], + {"time": [0], "region": ["east"], "temperature": [1.0]}, + ) + _write_table_rows_file( + second, + [ + ColumnSchema("region", TSDataType.STRING, ColumnCategory.TAG), + ColumnSchema("site", TSDataType.STRING, ColumnCategory.TAG), + _WEATHER_TEMPERATURE, + ], + {"time": [1], "region": ["east"], "site": ["s1"], "temperature": [2.0]}, + ) + _write_table_rows_file( + third, + [ + ColumnSchema("region", TSDataType.STRING, ColumnCategory.TAG), + ColumnSchema("site", TSDataType.STRING, ColumnCategory.TAG), + ColumnSchema("zone", TSDataType.STRING, ColumnCategory.TAG), + _WEATHER_HUMIDITY, + ], + { + "time": [2], + "region": ["east"], + "site": ["s1"], + "zone": ["z1"], + "humidity": [3.0], + }, + ) + + expected = { + "weather.east.temperature": (0, 1.0), + "weather.east.s1.temperature": (1, 2.0), + "weather.east.s1.z1.humidity": (2, 3.0), + } + for paths in ( + [str(first), str(second), str(third)], + [str(third), str(second), str(first)], + ): + with TsFileDataFrame( + paths, show_progress=False, use_index=dataframe_use_index + ) as tsdf: + assert set(tsdf.list_timeseries()) == set(expected) + for name, (timestamp, value) in expected.items(): + series = tsdf[name] + np.testing.assert_equal(series.timestamps, np.array([timestamp])) + np.testing.assert_equal(series[:], np.array([value])) + metadata = tsdf.list_timeseries_metadata() + assert list(metadata.columns)[-3:] == ["region", "site", "zone"] + assert pd.isna(metadata.loc["weather.east.temperature", "site"]) + assert pd.isna(metadata.loc["weather.east.temperature", "zone"]) + assert pd.isna(metadata.loc["weather.east.s1.temperature", "zone"]) + assert metadata.loc["weather.east.s1.z1.humidity", "site"] == "s1" + assert metadata.loc["weather.east.s1.z1.humidity", "zone"] == "z1" + + def test_dataset_unions_nonnumeric_declared_columns_without_exposing_them( tmp_path, dataframe_use_index ): @@ -1510,6 +1579,17 @@ def test_dataset_unions_nonnumeric_declared_columns_without_exposing_them( assert tsdf.list_timeseries() == ["weather.device_a.temperature"] entry = tsdf._index.table_entries["weather"] assert entry.field_columns == ("temperature",) + if dataframe_use_index: + # The persisted index defines a merged table's columns as its tags + # plus the readable fields only; declared non-numeric columns are + # not carried over. + assert [(name, role) for name, _, role in entry.schema_columns] == [ + ("device", int(ColumnCategory.TAG)), + ("temperature", int(ColumnCategory.FIELD)), + ] + else: + # The in-memory merged entry carries no declaration at all. + assert entry.schema_columns == () def test_dataset_skips_declared_field_without_values(tmp_path, dataframe_use_index): diff --git a/python/tsfile/dataset/dataframe.py b/python/tsfile/dataset/dataframe.py index 1620da611..f89b096ab 100644 --- a/python/tsfile/dataset/dataframe.py +++ b/python/tsfile/dataset/dataframe.py @@ -25,7 +25,7 @@ import os import sys from types import SimpleNamespace -from typing import Dict, List, Optional, Tuple, Union +from typing import Dict, Iterable, List, Optional, Tuple, Union import numpy as np @@ -122,6 +122,18 @@ def _series_lookup_hint(name: str) -> str: return f"Series not found: '{name}'. Use df.list_timeseries() to inspect available series." +def _union_field_columns(first: Iterable[str], second: Iterable[str]) -> List[str]: + """First-seen union of two field-column name sequences.""" + merged: List[str] = [] + seen = set() + for columns in (first, second): + for name in columns: + if name not in seen: + seen.add(name) + merged.append(name) + return merged + + def _merge_table_model_entries( existing: TableEntry, incoming: TableEntry, file_path: str ) -> TableEntry: @@ -171,12 +183,7 @@ def _merge_table_model_entries( ) tag_types = tuple(existing.tag_types) + tuple(incoming.tag_types[shared_tag_count:]) - field_columns = list(existing.field_columns) - seen = set(field_columns) - for name in incoming.field_columns: - if name not in seen: - seen.add(name) - field_columns.append(name) + field_columns = _union_field_columns(existing.field_columns, incoming.field_columns) # A name cannot be both a tag and a field of the same table: the # persistent index keys columns by (table, name), so a duplicate would make @@ -218,12 +225,7 @@ def _merge_tree_table_entries(existing: TableEntry, incoming: TableEntry) -> Tab else: tag_columns, tag_types = existing.tag_columns, existing.tag_types - field_columns = list(existing.field_columns) - seen = set(field_columns) - for name in incoming.field_columns: - if name not in seen: - seen.add(name) - field_columns.append(name) + field_columns = _union_field_columns(existing.field_columns, incoming.field_columns) return TableEntry( table_name=existing.table_name, From 54e047487c354f9530748b8f7d33b47583d064bc Mon Sep 17 00:00:00 2001 From: ColinLee Date: Tue, 17 Nov 2026 16:07:42 +0800 Subject: [PATCH 7/7] fix(python): key the dataset index by load order The persistent index identity hashed the sorted source paths, so loading the same files in reverse order reused an index whose merged field order came from the other order; positional selection then depended on cache history instead of the documented first-seen order. Hash the caller's order instead. index_matches_paths keeps validating the canonical sorted file list, and sorted loads keep their existing cache identity. Also scope the TAG/FIELD namespace guard to readable fields explicitly: non-numeric fields are never exposed or indexed, so they cannot collide in the index, and the docs no longer promise they are rejected. --- python/tests/test_tsfile_dataset.py | 84 ++++++++++++++++++++- python/tsfile/dataset/dataframe.py | 11 +-- python/tsfile/dataset/index.py | 12 ++- skills/tsfile/references/dataframe-arrow.md | 7 +- 4 files changed, 104 insertions(+), 10 deletions(-) diff --git a/python/tests/test_tsfile_dataset.py b/python/tests/test_tsfile_dataset.py index 0c93eb09e..5a9478773 100644 --- a/python/tests/test_tsfile_dataset.py +++ b/python/tests/test_tsfile_dataset.py @@ -1364,6 +1364,37 @@ def test_dataset_unions_disjoint_field_sets_across_shards( np.testing.assert_equal(np.asarray(aligned.values), np.array([[1.0, 2.0, 3.0]])) +def test_dataset_index_cache_follows_load_order(tmp_path): + # The persistent index is cached per load order, because the union's field + # order is first-seen. Reversing the order must build its own index rather + # than reuse one whose columns follow the other order, and repeating an + # order must still reuse that order's index. + temperature = tmp_path / "temperature.tsfile" + humidity = tmp_path / "humidity.tsfile" + _write_table_rows_file( + temperature, + [_WEATHER_DEVICE_TAG, _WEATHER_TEMPERATURE], + {"time": [0], "device": ["device_a"], "temperature": [1.0]}, + ) + _write_table_rows_file( + humidity, + [_WEATHER_DEVICE_TAG, _WEATHER_HUMIDITY], + {"time": [1], "device": ["device_a"], "humidity": [2.0]}, + ) + + for paths, fields in ( + ([str(temperature), str(humidity)], ["temperature", "humidity"]), + ([str(humidity), str(temperature)], ["humidity", "temperature"]), + ([str(temperature), str(humidity)], ["temperature", "humidity"]), + ): + with TsFileDataFrame(paths, show_progress=False, use_index=True) as tsdf: + entry = tsdf._index.table_entries["weather"] + assert entry.field_columns == tuple(fields) + assert tsdf.list_timeseries() == [ + f"weather.device_a.{field}" for field in fields + ] + + def test_dataset_unions_overlapping_field_subsets_for_different_devices( tmp_path, dataframe_use_index ): @@ -1687,6 +1718,57 @@ def test_dataset_rejects_different_tag_prefix(tmp_path, dataframe_use_index): TsFileDataFrame(paths, show_progress=False, use_index=dataframe_use_index) +def test_dataset_ignores_nonnumeric_field_sharing_a_tag_name( + tmp_path, dataframe_use_index +): + # Only readable fields join the column namespace: a non-numeric field with + # a tag's name is never exposed or indexed, so it does not conflict with + # the tag. The merge keeps the tag and ignores the field's values. + path1 = tmp_path / "tag-region.tsfile" + path2 = tmp_path / "string-field-region.tsfile" + _write_table_rows_file( + path1, + [ + _WEATHER_DEVICE_TAG, + ColumnSchema("region", TSDataType.STRING, ColumnCategory.TAG), + _WEATHER_TEMPERATURE, + ], + { + "time": [0], + "device": ["device_a"], + "region": ["east"], + "temperature": [1.0], + }, + ) + _write_table_rows_file( + path2, + [ + _WEATHER_DEVICE_TAG, + ColumnSchema("region", TSDataType.STRING, ColumnCategory.FIELD), + _WEATHER_TEMPERATURE, + ], + { + "time": [1], + "device": ["device_b"], + "region": ["west"], + "temperature": [2.0], + }, + ) + + with TsFileDataFrame( + [str(path1), str(path2)], + show_progress=False, + use_index=dataframe_use_index, + ) as tsdf: + assert tsdf.list_timeseries() == [ + "weather.device_a.east.temperature", + "weather.device_b.temperature", + ] + metadata = tsdf.list_timeseries_metadata() + assert metadata.loc["weather.device_a.east.temperature", "region"] == "east" + assert pd.isna(metadata.loc["weather.device_b.temperature", "region"]) + + def test_dataset_rejects_tag_and_field_with_same_name(tmp_path, dataframe_use_index): # One name cannot be both a tag and a field of the same table, because the # persistent index keys columns by (table, name) and a duplicate would make @@ -1719,7 +1801,7 @@ def test_dataset_rejects_tag_and_field_with_same_name(tmp_path, dataframe_use_in with pytest.raises( ValueError, - match="Column 'region' of table 'weather'.*TAG in some TsFiles and a FIELD", + match="Column 'region' of table 'weather'.*TAG in some TsFiles and a readable FIELD", ): TsFileDataFrame( [str(path1), str(path2)], diff --git a/python/tsfile/dataset/dataframe.py b/python/tsfile/dataset/dataframe.py index f89b096ab..58ce31661 100644 --- a/python/tsfile/dataset/dataframe.py +++ b/python/tsfile/dataset/dataframe.py @@ -185,17 +185,18 @@ def _merge_table_model_entries( field_columns = _union_field_columns(existing.field_columns, incoming.field_columns) - # A name cannot be both a tag and a field of the same table: the + # A readable field name cannot also be a tag of the same table: the # persistent index keys columns by (table, name), so a duplicate would make - # series lookups ambiguous. This is a namespace conflict within one table, - # not a schema-version difference, so it is the one case still rejected. + # series lookups ambiguous. Non-numeric fields never become columns here + # (they are neither exposed nor indexed), so they are not part of that + # namespace and are ignored instead of rejected. tag_names = set(tag_columns) for name in field_columns: if name in tag_names: raise ValueError( f"Column '{name}' of table '{table_name}' in '{file_path}' is a " - "TAG in some TsFiles and a FIELD in others. A column name cannot " - "be both a tag and a field of the same table." + "TAG in some TsFiles and a readable FIELD in others. A column " + "name cannot be both a tag and a field of the same table." ) return TableEntry( diff --git a/python/tsfile/dataset/index.py b/python/tsfile/dataset/index.py index a60484853..c958b5214 100644 --- a/python/tsfile/dataset/index.py +++ b/python/tsfile/dataset/index.py @@ -485,7 +485,14 @@ def device_time_bounds(self, device_id: int): def index_path_for(paths: Sequence[str]) -> str: - canonical_paths = sorted(os.path.abspath(path) for path in paths) + """Cache identity of one load set. + + The caller's source order is part of the identity: a merged table's + first-seen field order -- and every positional selection derived from it -- + follows load order, so a reversed load set must get its own index instead + of reusing one built in a different order. + """ + canonical_paths = [os.path.abspath(path) for path in paths] common = os.path.commonpath(canonical_paths) if not os.path.isdir(common): common = os.path.dirname(common) @@ -503,6 +510,9 @@ def index_matches_paths( path: str, paths: Sequence[str], trust_index: bool = True ) -> bool: """Match the file set, checking source generations only when untrusted.""" + # The stored file records are canonical (sorted); only the cache identity + # above encodes the caller's load order, so this comparison stays + # order-independent. expected = sorted(os.path.abspath(item) for item in paths) try: with MappedDatasetIndex(path) as index: diff --git a/skills/tsfile/references/dataframe-arrow.md b/skills/tsfile/references/dataframe-arrow.md index d9afe4d44..89521de99 100644 --- a/skills/tsfile/references/dataframe-arrow.md +++ b/skills/tsfile/references/dataframe-arrow.md @@ -77,9 +77,10 @@ as NaN. TAG columns are positional: files may append trailing tags, and devices from files without them read back with a null tag, but files must agree on the tag columns they share — a different name at the same position means the table name points at a different tag layout and is rejected. A name that is a TAG in -one file and a FIELD in another is rejected as well. FIELD declarations are -never compared. Duplicate timestamps for one logical series across files remain -rejected. +one file and a readable (numeric) FIELD in another is rejected as well; +non-numeric fields are never exposed or indexed, so they do not take part in +that check. FIELD declarations are never compared otherwise. Duplicate +timestamps for one logical series across files remain rejected. `use_index=True` builds/reuses an index (default `False`). Indexed mode assumes immutable sources (`trust_index=True`); use `False` for generation checks and