diff --git a/src/datasets/utils/file_utils.py b/src/datasets/utils/file_utils.py index 880af6f33d4..62d2cd083ef 100644 --- a/src/datasets/utils/file_utils.py +++ b/src/datasets/utils/file_utils.py @@ -1087,7 +1087,8 @@ class xPath(type(Path())): """Extension of `pathlib.Path` to support both local paths and remote URLs.""" def __init__(self, *args, download_config: Optional[DownloadConfig] = None): - super().__init__(*args) + if sys.version_info >= (3, 12): + super().__init__(*args) self.download_config = download_config def __str__(self): @@ -1160,7 +1161,7 @@ def glob(self, pattern, download_config: Optional[DownloadConfig] = None): globbed_paths = fs.glob(xjoin(main_hop, pattern)) for globbed_path in globbed_paths: yield type(self)( - "::".join([f"{fs.protocol}://{globbed_path}"] + rest_hops, download_config=self.download_config) + "::".join([f"{fs.protocol}://{globbed_path}"] + rest_hops), download_config=self.download_config ) def rglob(self, pattern, **kwargs): diff --git a/tests/test_file_utils.py b/tests/test_file_utils.py index 350e58f9381..6f0af244acd 100644 --- a/tests/test_file_utils.py +++ b/tests/test_file_utils.py @@ -731,6 +731,18 @@ def test_xrelpath(input_path, start_path, expected_path): class TestxPath: + @pytest.mark.parametrize("args", [(), ("file.txt",), ("dir", "file.txt"), (Path("file.txt"),)]) + @pytest.mark.parametrize("download_config", [None, DownloadConfig(token="token")]) + def test_xpath_init(self, args, download_config): + path = xPath(*args, download_config=download_config) + + assert str(path) == str(Path(*args)) + assert path.download_config is download_config + + def test_xpath_init_invalid_path(self): + with pytest.raises(TypeError): + xPath(None) + @pytest.mark.parametrize( "input_path", [ @@ -808,6 +820,17 @@ def test_xpath_glob(self, input_path, pattern, expected_paths, tmp_path, mock_fs output_paths = sorted([str(path) for path in xPath(input_path).glob(pattern)]) assert output_paths == expected_paths + @pytest.mark.parametrize("method", ["glob", "rglob"]) + def test_xpath_glob_preserves_download_config(self, method, mock_fsspec2): + download_config = DownloadConfig(token="token") + path = xPath("mock://top_level", download_config=download_config) + + output_paths = list(getattr(path, method)("*")) + + assert output_paths + assert all(isinstance(output_path, xPath) for output_path in output_paths) + assert all(output_path.download_config is download_config for output_path in output_paths) + @pytest.mark.parametrize( "input_path, pattern, expected_paths", [