Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 3 additions & 2 deletions src/datasets/utils/file_utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -1087,7 +1087,8 @@ class xPath(type(Path())):
"""Extension of `pathlib.Path` to support both local paths and remote URLs."""

def __init__(self, *args, download_config: Optional[DownloadConfig] = None):
super().__init__(*args)
if sys.version_info >= (3, 12):
super().__init__(*args)
self.download_config = download_config

def __str__(self):
Expand Down Expand Up @@ -1160,7 +1161,7 @@ def glob(self, pattern, download_config: Optional[DownloadConfig] = None):
globbed_paths = fs.glob(xjoin(main_hop, pattern))
for globbed_path in globbed_paths:
yield type(self)(
"::".join([f"{fs.protocol}://{globbed_path}"] + rest_hops, download_config=self.download_config)
"::".join([f"{fs.protocol}://{globbed_path}"] + rest_hops), download_config=self.download_config
)

def rglob(self, pattern, **kwargs):
Expand Down
23 changes: 23 additions & 0 deletions tests/test_file_utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -731,6 +731,18 @@ def test_xrelpath(input_path, start_path, expected_path):


class TestxPath:
@pytest.mark.parametrize("args", [(), ("file.txt",), ("dir", "file.txt"), (Path("file.txt"),)])
@pytest.mark.parametrize("download_config", [None, DownloadConfig(token="token")])
def test_xpath_init(self, args, download_config):
path = xPath(*args, download_config=download_config)

assert str(path) == str(Path(*args))
assert path.download_config is download_config

def test_xpath_init_invalid_path(self):
with pytest.raises(TypeError):
xPath(None)

@pytest.mark.parametrize(
"input_path",
[
Expand Down Expand Up @@ -808,6 +820,17 @@ def test_xpath_glob(self, input_path, pattern, expected_paths, tmp_path, mock_fs
output_paths = sorted([str(path) for path in xPath(input_path).glob(pattern)])
assert output_paths == expected_paths

@pytest.mark.parametrize("method", ["glob", "rglob"])
def test_xpath_glob_preserves_download_config(self, method, mock_fsspec2):
download_config = DownloadConfig(token="token")
path = xPath("mock://top_level", download_config=download_config)

output_paths = list(getattr(path, method)("*"))

assert output_paths
assert all(isinstance(output_path, xPath) for output_path in output_paths)
assert all(output_path.download_config is download_config for output_path in output_paths)

@pytest.mark.parametrize(
"input_path, pattern, expected_paths",
[
Expand Down