Skip to content

[python] optimize file io retry #6919

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
JingsongLi merged 4 commits into from
Dec 30, 2025
+31 −0
Merged

[python] optimize file io retry #6919

Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
59f872a
[python] optimize io retry
Dec 28, 2025
7bc391b
fix code format
Dec 29, 2025
0dc9f3d
make file_io retry compitable with python 3.6
Dec 29, 2025
2de94ba
do not provide request_timeout and connect_timeout when pyarrow versi…
Dec 29, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
31 changes: 31 additions & 0 deletions paimon-python/pypaimon/common/file_io.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -63,6 +63,31 @@ def parse_location(location: str):
else:
return uri.scheme, uri.netloc, f"{uri.netloc}{uri.path}"

@staticmethod
def _create_s3_retry_config(
max_attempts: int = 10,
request_timeout: int = 60,
connect_timeout: int = 60
) -> Dict[str, Any]:
"""
AwsStandardS3RetryStrategy and timeout parameters are only available
in PyArrow >= 8.0.0.
"""
if parse(pyarrow.__version__) >= parse("8.0.0"):
config = {
'request_timeout': request_timeout,
'connect_timeout': connect_timeout
}
try:
from pyarrow.fs import AwsStandardS3RetryStrategy
retry_strategy = AwsStandardS3RetryStrategy(max_attempts=max_attempts)
config['retry_strategy'] = retry_strategy
except ImportError:
pass
return config
else:
return {}

def _extract_oss_bucket(self, location) -> str:
uri = urlparse(location)
if uri.scheme and uri.scheme != "oss":
Expand Down Expand Up @@ -104,6 +129,9 @@ def _initialize_oss_fs(self, path) -> FileSystem:
client_kwargs['endpoint_override'] = (oss_bucket + "." +
self.properties.get(OssOptions.OSS_ENDPOINT))

retry_config = self._create_s3_retry_config()
client_kwargs.update(retry_config)

return S3FileSystem(**client_kwargs)

def _initialize_s3_fs(self) -> FileSystem:
Expand All @@ -118,6 +146,9 @@ def _initialize_s3_fs(self) -> FileSystem:
"force_virtual_addressing": True,
}

retry_config = self._create_s3_retry_config()
client_kwargs.update(retry_config)

return S3FileSystem(**client_kwargs)

def _initialize_hdfs_fs(self, scheme: str, netloc: Optional[str]) -> FileSystem:
Expand Down