diff --git a/src/datasets/features/features.py b/src/datasets/features/features.py index 84e3c06b4cc..ba29003ba4c 100644 --- a/src/datasets/features/features.py +++ b/src/datasets/features/features.py @@ -22,7 +22,7 @@ from collections.abc import Iterable, Mapping from collections.abc import Sequence as SequenceABC from collections.abc import Sequence as Sequence_ -from dataclasses import InitVar, dataclass, field, fields +from dataclasses import InitVar, dataclass, field, fields, replace from functools import reduce, wraps from operator import mul from typing import Any, Callable, ClassVar, Literal, Optional, Union @@ -2505,6 +2505,13 @@ def _is_null_feature(feature) -> bool: return False +def _is_same_list_type(feature: FeatureType, other: FeatureType) -> bool: + """Check if two features are lists of the same kind (`List` with the same length, or `LargeList`).""" + if isinstance(feature, LargeList): + return isinstance(other, LargeList) + return isinstance(feature, List) and isinstance(other, List) and feature.length == other.length + + def _align_features(features_list: list[Features]) -> list[Features]: """Align dictionaries of features so that the keys that are found in multiple dictionaries share the same feature.""" name2feature = {} @@ -2515,6 +2522,10 @@ def _align_features(features_list: list[Features]) -> list[Features]: name2feature[k] = _align_features([name2feature[k], v])[0] elif k not in name2feature or _is_null_feature(name2feature[k]): name2feature[k] = v + elif _is_same_list_type(name2feature[k], v): + # Recursively align the items, e.g. a struct field that is null in one list of structs. + item = _align_features([{k: name2feature[k].feature}, {k: v.feature}])[0][k] + name2feature[k] = replace(name2feature[k], feature=item) return [Features({k: name2feature[k] for k in features.keys()}) for features in features_list] @@ -2536,6 +2547,9 @@ def _check_if_features_can_be_aligned(features_list: list[Features]): if isinstance(v, dict) and isinstance(name2feature[k], dict): # Deep checks for structure. _check_if_features_can_be_aligned([name2feature[k], v]) + elif not _is_null_feature(v) and _is_same_list_type(name2feature[k], v): + # Deep checks for the items of the lists. + _check_if_features_can_be_aligned([{k: name2feature[k].feature}, {k: v.feature}]) elif not _is_null_feature(v) and name2feature[k] != v: raise ValueError( f'The features can\'t be aligned because the key {k} of features {features} has unexpected type - {v} (expected either {name2feature[k]} or Value("null").' diff --git a/tests/features/test_features.py b/tests/features/test_features.py index f3aaeae5b59..d84e37318f7 100644 --- a/tests/features/test_features.py +++ b/tests/features/test_features.py @@ -943,6 +943,50 @@ def test_features_to_arrow_schema(features: Features): Features({"label": List({"type": Value("string"), "score": Value("float64")})}), ], ], + # A struct field that is null in one list of structs should be aligned with the other list's field + [ + [ + Features({"a": List({"x": Value("int64"), "y": Value("string")})}), + Features({"a": List({"x": Value("int64"), "y": Value("null")})}), + ], + [ + Features({"a": List({"x": Value("int64"), "y": Value("string")})}), + Features({"a": List({"x": Value("int64"), "y": Value("string")})}), + ], + ], + # Same with the null field first + [ + [ + Features({"a": List({"x": Value("int64"), "y": Value("null")})}), + Features({"a": List({"x": Value("int64"), "y": Value("string")})}), + ], + [ + Features({"a": List({"x": Value("int64"), "y": Value("string")})}), + Features({"a": List({"x": Value("int64"), "y": Value("string")})}), + ], + ], + # Same in a LargeList + [ + [ + Features({"a": LargeList({"y": Value("null")})}), + Features({"a": LargeList({"y": Value("string")})}), + ], + [ + Features({"a": LargeList({"y": Value("string")})}), + Features({"a": LargeList({"y": Value("string")})}), + ], + ], + # Same in a list of lists + [ + [ + Features({"a": List(List({"y": Value("null")}))}), + Features({"a": List(List({"y": Value("string")}))}), + ], + [ + Features({"a": List(List({"y": Value("string")}))}), + Features({"a": List(List({"y": Value("string")}))}), + ], + ], ] @@ -953,6 +997,19 @@ def test_features_alignment(features: tuple[list[Features], list[Features]]): assert _align_features(inputs) == expected +@pytest.mark.parametrize( + "inputs", + [ + [Features({"a": List({"y": Value("int64")})}), Features({"a": List({"y": Value("string")})})], + [Features({"a": List(Value("int64"))}), Features({"a": LargeList(Value("int64"))})], + [Features({"a": List(Value("int64"), length=2)}), Features({"a": List(Value("int64"))})], + ], +) +def test_features_alignment_of_different_lists_fails(inputs: list[Features]): + with pytest.raises(ValueError, match="can't be aligned"): + _check_if_features_can_be_aligned(inputs) + + @pytest.mark.parametrize("dtype", [pa.int32, pa.string]) def test_features_from_arrow_schema_primitive_data_type(dtype): schema = pa.schema([("column_name", dtype())]) diff --git a/tests/test_arrow_dataset.py b/tests/test_arrow_dataset.py index 1ebc9cd0990..355188cc8df 100644 --- a/tests/test_arrow_dataset.py +++ b/tests/test_arrow_dataset.py @@ -3807,6 +3807,28 @@ def test_concatenate_datasets_new_columns(): } +def test_concatenate_datasets_list_of_structs_with_null_field(): + dataset1 = Dataset.from_dict({"a": [[{"x": 1, "y": "s"}]]}) + dataset2 = Dataset.from_dict({"a": [[{"x": 2, "y": None}]]}) + expected_features = Features({"a": List({"x": Value("int64"), "y": Value("string")})}) + dataset = concatenate_datasets([dataset1, dataset2]) + assert dataset.features == expected_features + assert dataset["a"] == [[{"x": 1, "y": "s"}], [{"x": 2, "y": None}]] + dataset = concatenate_datasets([dataset2, dataset1]) + assert dataset.features == expected_features + assert dataset["a"] == [[{"x": 2, "y": None}], [{"x": 1, "y": "s"}]] + + +def test_map_multiprocessing_list_of_structs_with_null_field(): + # the first shard only has None for "y", so its features have Value("null") for it + def add_list_of_structs(example): + return {"a": [{"x": example["i"], "y": None if example["i"] < 2 else "s"}]} + + dataset = Dataset.from_dict({"i": list(range(4))}).map(add_list_of_structs, num_proc=2) + assert dataset.features == Features({"i": Value("int64"), "a": List({"x": Value("int64"), "y": Value("string")})}) + assert dataset["a"] == [[{"x": 0, "y": None}], [{"x": 1, "y": None}], [{"x": 2, "y": "s"}], [{"x": 3, "y": "s"}]] + + @pytest.mark.parametrize("axis", [0, 1]) def test_concatenate_datasets_complex_features(axis): n = 5