Skip to content

Commit 27cea27

Browse files
FsspecFileIO: Resolve the ADLS account per location instead of caching it in properties
_adls wrote the account name it inferred back into the properties dict it was given, and the caller passes the FileIO's own properties. The first location therefore pinned the account for every later one, so a second storage account was served a filesystem built for the first. Resolve the account and the SAS token into locals and leave the caller's dict alone. Precedence is unchanged: an explicit adls.account-name still wins, then a per account SAS token key, then the hostname.
1 parent db5bd04 commit 27cea27

2 files changed

Lines changed: 41 additions & 9 deletions

File tree

‎pyiceberg/io/fsspec.py‎

Lines changed: 15 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -265,17 +265,23 @@ def _adls(properties: Properties, hostname: str | None = None) -> AbstractFileSy
265265
from azure.core.credentials import AccessToken
266266
from azure.core.credentials_async import AsyncTokenCredential
267267

268-
for key, sas_token in {
268+
# These are resolved into locals rather than written back into `properties`. The same
269+
# properties dict is shared by every location this FileIO serves, so storing an account
270+
# name derived from one location would apply it to all the later ones as well.
271+
account_name = properties.get(ADLS_ACCOUNT_NAME)
272+
sas_token = properties.get(ADLS_SAS_TOKEN)
273+
274+
for key, per_account_sas_token in {
269275
key.replace(f"{ADLS_SAS_TOKEN}.", ""): value for key, value in properties.items() if key.startswith(f"{ADLS_SAS_TOKEN}.")
270276
}.items():
271-
if ADLS_ACCOUNT_NAME not in properties:
272-
properties[ADLS_ACCOUNT_NAME] = key.split(".")[0]
273-
if ADLS_SAS_TOKEN not in properties:
274-
properties[ADLS_SAS_TOKEN] = sas_token
277+
if account_name is None:
278+
account_name = key.split(".")[0]
279+
if sas_token is None:
280+
sas_token = per_account_sas_token
275281

276282
# Fallback: extract account_name from URI hostname (e.g. "account.dfs.core.windows.net" -> "account")
277-
if hostname and ADLS_ACCOUNT_NAME not in properties:
278-
properties[ADLS_ACCOUNT_NAME] = hostname.split(".")[0]
283+
if hostname and account_name is None:
284+
account_name = hostname.split(".")[0]
279285

280286
class StaticTokenCredential(AsyncTokenCredential):
281287
_DEFAULT_EXPIRY_SECONDS = 3600
@@ -298,9 +304,9 @@ async def get_token(self, *scopes: str, **kwargs: Any) -> AccessToken:
298304
return AzureBlobFileSystem(
299305
connection_string=properties.get(ADLS_CONNECTION_STRING),
300306
credential=credential,
301-
account_name=properties.get(ADLS_ACCOUNT_NAME),
307+
account_name=account_name,
302308
account_key=properties.get(ADLS_ACCOUNT_KEY),
303-
sas_token=properties.get(ADLS_SAS_TOKEN),
309+
sas_token=sas_token,
304310
tenant_id=properties.get(ADLS_TENANT_ID),
305311
client_id=properties.get(ADLS_CLIENT_ID),
306312
client_secret=properties.get(ADLS_CLIENT_SECRET),

‎tests/io/test_fsspec.py‎

Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -697,6 +697,32 @@ def test_adls_account_name_extracted_from_uri_hostname() -> None:
697697
)
698698

699699

700+
def test_adls_account_name_resolved_per_location() -> None:
701+
"""Two locations in different accounts must each get a filesystem for their own account."""
702+
session_properties: Properties = {"adls.tenant-id": "test-tenant-id"}
703+
704+
with mock.patch("adlfs.AzureBlobFileSystem") as mock_adlfs:
705+
adls_fileio = FsspecFileIO(properties=session_properties)
706+
707+
adls_fileio.new_input(location="abfss://data@accountone.dfs.core.windows.net/wh/t/a.parquet")
708+
adls_fileio.new_input(location="abfss://data@accounttwo.dfs.core.windows.net/wh/t/b.parquet")
709+
710+
account_names = [call.kwargs["account_name"] for call in mock_adlfs.call_args_list]
711+
assert account_names == ["accountone", "accounttwo"]
712+
713+
714+
def test_adls_does_not_mutate_properties() -> None:
715+
"""Resolving an account from a location must not write it back into the shared properties."""
716+
session_properties: Properties = {"adls.tenant-id": "test-tenant-id"}
717+
718+
with mock.patch("adlfs.AzureBlobFileSystem"):
719+
adls_fileio = FsspecFileIO(properties=session_properties)
720+
adls_fileio.new_input(location="abfss://data@accountone.dfs.core.windows.net/wh/t/a.parquet")
721+
722+
assert "adls.account-name" not in adls_fileio.properties
723+
assert session_properties == {"adls.tenant-id": "test-tenant-id"}
724+
725+
700726
def test_adls_account_name_not_overridden_when_in_properties() -> None:
701727
"""Test that explicit adls.account-name in properties is not overridden by URI hostname."""
702728
session_properties: Properties = {

0 commit comments

Comments
 (0)