From 787130bd04dc3ce7f0f5d9b503b627c1789d4e31 Mon Sep 17 00:00:00 2001 From: krishnakaanchan-png Date: Tue, 1 Sep 2026 00:32:06 +0530 Subject: [PATCH 1/2] PyArrow: Keep the storage account out of ADLS paths in parse_location For Azure the netloc is @., so building the path as netloc + path put the account inside the path and PyArrow then read the whole first segment as the container name. Return only the container instead, which matches what PyArrow's own from_uri produces for the same location. --- pyiceberg/io/pyarrow.py | 8 +++++++- tests/io/test_pyarrow.py | 24 ++++++++++++++++++++++++ 2 files changed, 31 insertions(+), 1 deletion(-) diff --git a/pyiceberg/io/pyarrow.py b/pyiceberg/io/pyarrow.py index c36f1639d9..1e59107da3 100644 --- a/pyiceberg/io/pyarrow.py +++ b/pyiceberg/io/pyarrow.py @@ -209,6 +209,7 @@ MAP_VALUE_NAME = "value" DOC = "doc" UTC_ALIASES = {"UTC", "+00:00", "Etc/UTC", "Z"} +ADLS_SCHEMES = frozenset({"abfs", "abfss", "wasb", "wasbs"}) T = TypeVar("T") @@ -421,6 +422,11 @@ def parse_location(location: str, properties: Properties = EMPTY_DICT) -> tuple[ return default_scheme, default_netloc, os.path.abspath(location) elif uri.scheme in ("hdfs", "viewfs"): return uri.scheme, uri.netloc, uri.path + elif uri.scheme in ADLS_SCHEMES and uri.username: + # Azure locations are @./. The account is + # configured on the AzureFileSystem itself, which expects paths of the form + # /, so only the container belongs in the path here. + return uri.scheme, uri.netloc, f"{uri.username}{uri.path}" else: return uri.scheme, uri.netloc, f"{uri.netloc}{uri.path}" @@ -438,7 +444,7 @@ def _initialize_fs(self, scheme: str, netloc: str | None = None) -> FileSystem: elif scheme in {"gs", "gcs"}: return self._initialize_gcs_fs() - elif scheme in {"abfs", "abfss", "wasb", "wasbs"}: + elif scheme in ADLS_SCHEMES: return self._initialize_azure_fs() elif scheme in {"file"}: diff --git a/tests/io/test_pyarrow.py b/tests/io/test_pyarrow.py index b31c18949b..892d8e54eb 100644 --- a/tests/io/test_pyarrow.py +++ b/tests/io/test_pyarrow.py @@ -2326,6 +2326,30 @@ def check_results(location: str, expected_schema: str, expected_netloc: str, exp check_results("/root/foo.txt", "file", "", os.path.abspath("/root/foo.txt")) check_results("/root/tmp/foo.txt", "file", "", os.path.abspath("/root/tmp/foo.txt")) + check_results("s3://bucket/root/foo.txt", "s3", "bucket", "bucket/root/foo.txt") + + +@pytest.mark.parametrize("scheme", ["abfs", "abfss", "wasb", "wasbs"]) +def test_parse_location_adls_account_qualified(scheme: str) -> None: + """The account must not leak into the path, PyArrow takes it on the filesystem instead.""" + scheme_, netloc, path = PyArrowFileIO.parse_location( + f"{scheme}://mycontainer@myaccount.dfs.core.windows.net/wh/db/tbl/data.parquet" + ) + + assert scheme_ == scheme + assert netloc == "mycontainer@myaccount.dfs.core.windows.net" + assert path == "mycontainer/wh/db/tbl/data.parquet" + + +@pytest.mark.parametrize("scheme", ["abfs", "abfss", "wasb", "wasbs"]) +def test_parse_location_adls_container_only(scheme: str) -> None: + """Locations without an account keep the netloc as the container.""" + scheme_, netloc, path = PyArrowFileIO.parse_location(f"{scheme}://mycontainer/wh/db/tbl/data.parquet") + + assert scheme_ == scheme + assert netloc == "mycontainer" + assert path == "mycontainer/wh/db/tbl/data.parquet" + @pytest.mark.skipif(sys.platform != "win32", reason="Windows-only behavior") def test_parse_location_windows_drive_letter() -> None: From 8fc649e3f1f45ad842af5c23d28c9294c2b9e51c Mon Sep 17 00:00:00 2001 From: krishnakaanchan-png Date: Sun, 6 Sep 2026 00:24:29 +0530 Subject: [PATCH 2/2] PyArrow: Derive the ADLS account from the location when it is not configured _initialize_azure_fs took no netloc, unlike the S3 and HDFS initialisers next to it, so the account could only come from adls.account-name and the account in an abfss location was dropped. Pass the netloc through and fall back to the account in its host part. An explicit adls.account-name still wins, same precedence as FsspecFileIO. The fallback is gated on the userinfo part being present, since a container only netloc carries no account. --- pyiceberg/io/pyarrow.py | 14 +++++++++++--- tests/io/test_pyarrow.py | 42 +++++++++++++++++++++++++++++++++++++++- 2 files changed, 52 insertions(+), 4 deletions(-) diff --git a/pyiceberg/io/pyarrow.py b/pyiceberg/io/pyarrow.py index 1e59107da3..4b78dad5ce 100644 --- a/pyiceberg/io/pyarrow.py +++ b/pyiceberg/io/pyarrow.py @@ -445,7 +445,7 @@ def _initialize_fs(self, scheme: str, netloc: str | None = None) -> FileSystem: return self._initialize_gcs_fs() elif scheme in ADLS_SCHEMES: - return self._initialize_azure_fs() + return self._initialize_azure_fs(netloc) elif scheme in {"file"}: return self._initialize_local_fs() @@ -539,7 +539,7 @@ def _initialize_s3_fs(self, netloc: str | None) -> FileSystem: return S3FileSystem(**client_kwargs) - def _initialize_azure_fs(self) -> FileSystem: + def _initialize_azure_fs(self, netloc: str | None = None) -> FileSystem: # https://arrow.apache.org/docs/python/generated/pyarrow.fs.AzureFileSystem.html from packaging import version @@ -554,7 +554,15 @@ def _initialize_azure_fs(self) -> FileSystem: client_kwargs: dict[str, str] = {} - if account_name := self.properties.get(ADLS_ACCOUNT_NAME): + account_name = self.properties.get(ADLS_ACCOUNT_NAME) + if account_name is None and netloc and "@" in netloc: + # An account qualified location carries the account in its host part, + # abfss://@./. Only that form has a userinfo + # part, without it the netloc is just the container and there is no account + # to read. An explicit adls.account-name always wins, same as FsspecFileIO. + account_name = netloc.rpartition("@")[2].split(".")[0] or None + + if account_name: client_kwargs["account_name"] = account_name if account_key := self.properties.get(ADLS_ACCOUNT_KEY): diff --git a/tests/io/test_pyarrow.py b/tests/io/test_pyarrow.py index 892d8e54eb..a1b8c32f92 100644 --- a/tests/io/test_pyarrow.py +++ b/tests/io/test_pyarrow.py @@ -63,7 +63,7 @@ Or, ) from pyiceberg.expressions.literals import literal -from pyiceberg.io import S3_RETRY_STRATEGY_IMPL, InputStream, OutputStream, load_file_io +from pyiceberg.io import ADLS_ACCOUNT_NAME, S3_RETRY_STRATEGY_IMPL, InputStream, OutputStream, load_file_io from pyiceberg.io.pyarrow import ( ICEBERG_SCHEMA, PYARROW_PARQUET_FIELD_ID_KEY, @@ -3234,6 +3234,46 @@ def test__to_requested_schema_float_promotion( assert result.column(0).to_pylist() == [1.5, 2.25, 3.0, None] +@skip_if_pyarrow_too_old +@pytest.mark.parametrize("scheme", ["abfs", "abfss", "wasb", "wasbs"]) +def test_adls_account_name_from_location(scheme: str) -> None: + """The account in an account qualified location is used when the property is not set.""" + with patch("pyarrow.fs.AzureFileSystem") as mock_azure_fs: + PyArrowFileIO().fs_by_scheme(scheme, "mycontainer@myaccount.dfs.core.windows.net") + + assert mock_azure_fs.call_args.kwargs["account_name"] == "myaccount" + + +@skip_if_pyarrow_too_old +def test_adls_account_name_property_wins_over_location() -> None: + """An explicit adls.account-name is not overridden by the account in the location.""" + with patch("pyarrow.fs.AzureFileSystem") as mock_azure_fs: + PyArrowFileIO({ADLS_ACCOUNT_NAME: "configured"}).fs_by_scheme("abfss", "mycontainer@myaccount.dfs.core.windows.net") + + assert mock_azure_fs.call_args.kwargs["account_name"] == "configured" + + +@skip_if_pyarrow_too_old +def test_adls_no_account_name_from_container_only_location() -> None: + """A container only netloc carries no account, so nothing should be inferred from it.""" + with patch("pyarrow.fs.AzureFileSystem") as mock_azure_fs: + PyArrowFileIO().fs_by_scheme("abfss", "warehouse") + + assert "account_name" not in mock_azure_fs.call_args.kwargs + + +@skip_if_pyarrow_too_old +def test_adls_account_name_per_location() -> None: + """Two accounts served by one FileIO must each get their own filesystem.""" + file_io = PyArrowFileIO() + + with patch("pyarrow.fs.AzureFileSystem") as mock_azure_fs: + file_io.fs_by_scheme("abfss", "data@accountone.dfs.core.windows.net") + file_io.fs_by_scheme("abfss", "data@accounttwo.dfs.core.windows.net") + + assert [call.kwargs["account_name"] for call in mock_azure_fs.call_args_list] == ["accountone", "accounttwo"] + + def test_pyarrow_file_io_fs_by_scheme_cache() -> None: # It's better to set up multi-region minio servers for an integration test once `endpoint_url` argument # becomes available for `resolve_s3_region`