Skip to content

fsspec: Remove botocore as a module import #1609

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
kevinjqliu merged 1 commit into from
Feb 8, 2025
Merged

fsspec: Remove botocore as a module import #1609

Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 8 additions & 4 deletions pyiceberg/io/fsspec.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@
from copy import copy
from functools import lru_cache, partial
from typing import (
TYPE_CHECKING,
Any,
Callable,
Dict,
Expand All @@ -31,8 +32,6 @@
from urllib.parse import urlparse

import requests
from botocore import UNSIGNED
from botocore.awsrequest import AWSRequest
from fsspec import AbstractFileSystem
from fsspec.implementations.local import LocalFileSystem
from requests import HTTPError
Expand Down Expand Up @@ -83,8 +82,11 @@

logger = logging.getLogger(__name__)

if TYPE_CHECKING:
from botocore.awsrequest import AWSRequest

def s3v4_rest_signer(properties: Properties, request: AWSRequest, **_: Any) -> AWSRequest:

def s3v4_rest_signer(properties: Properties, request: "AWSRequest", **_: Any) -> "AWSRequest":
signer_url = properties.get(S3_SIGNER_URI, properties["uri"]).rstrip("/")
signer_endpoint = properties.get(S3_SIGNER_ENDPOINT, S3_SIGNER_ENDPOINT_DEFAULT)

Expand Down Expand Up @@ -114,7 +116,7 @@ def s3v4_rest_signer(properties: Properties, request: AWSRequest, **_: Any) -> A
return request


SIGNERS: Dict[str, Callable[[Properties, AWSRequest], AWSRequest]] = {"S3V4RestSigner": s3v4_rest_signer}
SIGNERS: Dict[str, Callable[[Properties, "AWSRequest"], "AWSRequest"]] = {"S3V4RestSigner": s3v4_rest_signer}


def _file(_: Properties) -> LocalFileSystem:
Expand All @@ -141,6 +143,8 @@ def _s3(properties: Properties) -> AbstractFileSystem:
register_events["before-sign.s3"] = signer_func_with_properties

# Disable the AWS Signer
from botocore import UNSIGNED

config_kwargs["signature_version"] = UNSIGNED
else:
raise ValueError(f"Signer not available: {signer}")
Expand Down