diff --git a/pyiceberg/io/fsspec.py b/pyiceberg/io/fsspec.py index 55368e1948..522e660b15 100644 --- a/pyiceberg/io/fsspec.py +++ b/pyiceberg/io/fsspec.py @@ -70,6 +70,7 @@ S3_ANONYMOUS, S3_CONNECT_TIMEOUT, S3_ENDPOINT, + S3_FORCE_VIRTUAL_ADDRESSING, S3_PROXY_URI, S3_REGION, S3_REQUEST_TIMEOUT, @@ -168,6 +169,9 @@ def _s3(properties: Properties) -> AbstractFileSystem: if request_timeout := properties.get(S3_REQUEST_TIMEOUT): config_kwargs["read_timeout"] = float(request_timeout) + if _force_virtual_addressing := properties.get(S3_FORCE_VIRTUAL_ADDRESSING): + config_kwargs["s3"] = {"addressing_style": "virtual"} + if s3_anonymous := properties.get(S3_ANONYMOUS): anon = strtobool(s3_anonymous) else: diff --git a/tests/io/test_fsspec.py b/tests/io/test_fsspec.py index 6924d6b1c3..fb22eefbfe 100644 --- a/tests/io/test_fsspec.py +++ b/tests/io/test_fsspec.py @@ -306,6 +306,36 @@ def test_fsspec_s3_session_properties() -> None: ) +def test_fsspec_s3_session_properties_force_virtual_addressing() -> None: + session_properties: Properties = { + "s3.force-virtual-addressing": True, + "s3.endpoint": "http://localhost:9000", + "s3.access-key-id": "admin", + "s3.secret-access-key": "password", + "s3.region": "us-east-1", + "s3.session-token": "s3.session-token", + **UNIFIED_AWS_SESSION_PROPERTIES, + } + + with mock.patch("s3fs.S3FileSystem") as mock_s3fs: + s3_fileio = FsspecFileIO(properties=session_properties) + filename = str(uuid.uuid4()) + + s3_fileio.new_input(location=f"s3://warehouse/{filename}") + + mock_s3fs.assert_called_with( + anon=False, + client_kwargs={ + "endpoint_url": "http://localhost:9000", + "aws_access_key_id": "admin", + "aws_secret_access_key": "password", + "region_name": "us-east-1", + "aws_session_token": "s3.session-token", + }, + config_kwargs={"s3": {"addressing_style": "virtual"}}, + ) + + def test_fsspec_s3_session_properties_with_anonymous() -> None: session_properties: Properties = { "s3.anonymous": "true",