Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
16 changes: 15 additions & 1 deletion src/datasets/features/features.py
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@
from collections.abc import Iterable, Mapping
from collections.abc import Sequence as SequenceABC
from collections.abc import Sequence as Sequence_
from dataclasses import InitVar, dataclass, field, fields
from dataclasses import InitVar, dataclass, field, fields, replace
from functools import reduce, wraps
from operator import mul
from typing import Any, Callable, ClassVar, Literal, Optional, Union
Expand Down Expand Up @@ -2505,6 +2505,13 @@ def _is_null_feature(feature) -> bool:
return False


def _is_same_list_type(feature: FeatureType, other: FeatureType) -> bool:
"""Check if two features are lists of the same kind (`List` with the same length, or `LargeList`)."""
if isinstance(feature, LargeList):
return isinstance(other, LargeList)
return isinstance(feature, List) and isinstance(other, List) and feature.length == other.length


def _align_features(features_list: list[Features]) -> list[Features]:
"""Align dictionaries of features so that the keys that are found in multiple dictionaries share the same feature."""
name2feature = {}
Expand All @@ -2515,6 +2522,10 @@ def _align_features(features_list: list[Features]) -> list[Features]:
name2feature[k] = _align_features([name2feature[k], v])[0]
elif k not in name2feature or _is_null_feature(name2feature[k]):
name2feature[k] = v
elif _is_same_list_type(name2feature[k], v):
# Recursively align the items, e.g. a struct field that is null in one list of structs.
item = _align_features([{k: name2feature[k].feature}, {k: v.feature}])[0][k]
name2feature[k] = replace(name2feature[k], feature=item)

return [Features({k: name2feature[k] for k in features.keys()}) for features in features_list]

Expand All @@ -2536,6 +2547,9 @@ def _check_if_features_can_be_aligned(features_list: list[Features]):
if isinstance(v, dict) and isinstance(name2feature[k], dict):
# Deep checks for structure.
_check_if_features_can_be_aligned([name2feature[k], v])
elif not _is_null_feature(v) and _is_same_list_type(name2feature[k], v):
# Deep checks for the items of the lists.
_check_if_features_can_be_aligned([{k: name2feature[k].feature}, {k: v.feature}])
elif not _is_null_feature(v) and name2feature[k] != v:
raise ValueError(
f'The features can\'t be aligned because the key {k} of features {features} has unexpected type - {v} (expected either {name2feature[k]} or Value("null").'
Expand Down
57 changes: 57 additions & 0 deletions tests/features/test_features.py
Original file line number Diff line number Diff line change
Expand Up @@ -943,6 +943,50 @@ def test_features_to_arrow_schema(features: Features):
Features({"label": List({"type": Value("string"), "score": Value("float64")})}),
],
],
# A struct field that is null in one list of structs should be aligned with the other list's field
[
[
Features({"a": List({"x": Value("int64"), "y": Value("string")})}),
Features({"a": List({"x": Value("int64"), "y": Value("null")})}),
],
[
Features({"a": List({"x": Value("int64"), "y": Value("string")})}),
Features({"a": List({"x": Value("int64"), "y": Value("string")})}),
],
],
# Same with the null field first
[
[
Features({"a": List({"x": Value("int64"), "y": Value("null")})}),
Features({"a": List({"x": Value("int64"), "y": Value("string")})}),
],
[
Features({"a": List({"x": Value("int64"), "y": Value("string")})}),
Features({"a": List({"x": Value("int64"), "y": Value("string")})}),
],
],
# Same in a LargeList
[
[
Features({"a": LargeList({"y": Value("null")})}),
Features({"a": LargeList({"y": Value("string")})}),
],
[
Features({"a": LargeList({"y": Value("string")})}),
Features({"a": LargeList({"y": Value("string")})}),
],
],
# Same in a list of lists
[
[
Features({"a": List(List({"y": Value("null")}))}),
Features({"a": List(List({"y": Value("string")}))}),
],
[
Features({"a": List(List({"y": Value("string")}))}),
Features({"a": List(List({"y": Value("string")}))}),
],
],
]


Expand All @@ -953,6 +997,19 @@ def test_features_alignment(features: tuple[list[Features], list[Features]]):
assert _align_features(inputs) == expected


@pytest.mark.parametrize(
"inputs",
[
[Features({"a": List({"y": Value("int64")})}), Features({"a": List({"y": Value("string")})})],
[Features({"a": List(Value("int64"))}), Features({"a": LargeList(Value("int64"))})],
[Features({"a": List(Value("int64"), length=2)}), Features({"a": List(Value("int64"))})],
],
)
def test_features_alignment_of_different_lists_fails(inputs: list[Features]):
with pytest.raises(ValueError, match="can't be aligned"):
_check_if_features_can_be_aligned(inputs)


@pytest.mark.parametrize("dtype", [pa.int32, pa.string])
def test_features_from_arrow_schema_primitive_data_type(dtype):
schema = pa.schema([("column_name", dtype())])
Expand Down
22 changes: 22 additions & 0 deletions tests/test_arrow_dataset.py
Original file line number Diff line number Diff line change
Expand Up @@ -3807,6 +3807,28 @@ def test_concatenate_datasets_new_columns():
}


def test_concatenate_datasets_list_of_structs_with_null_field():
dataset1 = Dataset.from_dict({"a": [[{"x": 1, "y": "s"}]]})
dataset2 = Dataset.from_dict({"a": [[{"x": 2, "y": None}]]})
expected_features = Features({"a": List({"x": Value("int64"), "y": Value("string")})})
dataset = concatenate_datasets([dataset1, dataset2])
assert dataset.features == expected_features
assert dataset["a"] == [[{"x": 1, "y": "s"}], [{"x": 2, "y": None}]]
dataset = concatenate_datasets([dataset2, dataset1])
assert dataset.features == expected_features
assert dataset["a"] == [[{"x": 2, "y": None}], [{"x": 1, "y": "s"}]]


def test_map_multiprocessing_list_of_structs_with_null_field():
# the first shard only has None for "y", so its features have Value("null") for it
def add_list_of_structs(example):
return {"a": [{"x": example["i"], "y": None if example["i"] < 2 else "s"}]}

dataset = Dataset.from_dict({"i": list(range(4))}).map(add_list_of_structs, num_proc=2)
assert dataset.features == Features({"i": Value("int64"), "a": List({"x": Value("int64"), "y": Value("string")})})
assert dataset["a"] == [[{"x": 0, "y": None}], [{"x": 1, "y": None}], [{"x": 2, "y": "s"}], [{"x": 3, "y": "s"}]]


@pytest.mark.parametrize("axis", [0, 1])
def test_concatenate_datasets_complex_features(axis):
n = 5
Expand Down