From f36d24239f0dd33aeb9bf3855a3b7348089ddca8 Mon Sep 17 00:00:00 2001 From: Sung Yun <107272191+sungwy@users.noreply.github.com> Date: Sun, 13 Apr 2025 15:17:01 +0000 Subject: [PATCH 1/5] introduce auth manager --- dev/Dockerfile | 2 +- .../catalog/{rest.py => rest/__init__.py} | 0 pyiceberg/catalog/rest/auth.py | 81 +++++++++++++++++++ tests/catalog/test_rest_auth.py | 63 +++++++++++++++ 4 files changed, 145 insertions(+), 1 deletion(-) rename pyiceberg/catalog/{rest.py => rest/__init__.py} (100%) create mode 100644 pyiceberg/catalog/rest/auth.py create mode 100644 tests/catalog/test_rest_auth.py diff --git a/dev/Dockerfile b/dev/Dockerfile index 97f6ac642f..3b7e593a1a 100644 --- a/dev/Dockerfile +++ b/dev/Dockerfile @@ -52,7 +52,7 @@ RUN curl --retry 5 -s https://repository.apache.org/content/groups/snapshots/org # Download AWS bundle -RUN curl --retry 5 -s https://repository.apache.org/content/groups/snapshots/org/apache/iceberg/iceberg-aws-bundle/1.9.0-SNAPSHOT/iceberg-aws-bundle-1.9.0-20250408.002722-86.jar \ +RUN curl --retry 5 -s https://repository.apache.org/content/groups/snapshots/org/apache/iceberg/iceberg-aws-bundle/1.9.0-SNAPSHOT/iceberg-aws-bundle-1.9.0-20250409.002731-88.jar \ -Lo /opt/spark/jars/iceberg-aws-bundle-${ICEBERG_VERSION}.jar COPY spark-defaults.conf /opt/spark/conf diff --git a/pyiceberg/catalog/rest.py b/pyiceberg/catalog/rest/__init__.py similarity index 100% rename from pyiceberg/catalog/rest.py rename to pyiceberg/catalog/rest/__init__.py diff --git a/pyiceberg/catalog/rest/auth.py b/pyiceberg/catalog/rest/auth.py new file mode 100644 index 0000000000..ea9e89ad81 --- /dev/null +++ b/pyiceberg/catalog/rest/auth.py @@ -0,0 +1,81 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +from abc import ABC, abstractmethod +from typing import Optional +from requests import PreparedRequest +from requests.auth import AuthBase + +import base64 + +class AuthManager(ABC): + """ + Abstract base class for Authentication Managers used to supply authorization headers + to HTTP clients (e.g. requests.Session). + + Subclasses must implement the `auth_header` method to return an Authorization header value. + """ + @abstractmethod + def auth_header(self) -> Optional[str]: + """Return the Authorization header value, or None if not applicable.""" + pass + + +class NoopAuthManager(AuthManager): + def auth_header(self) -> Optional[str]: + return None + + +class BasicAuthManager(AuthManager): + def __init__(self, username: str, password: str): + credentials = f"{username}:{password}" + self._token = base64.b64encode(credentials.encode()).decode() + + def auth_header(self) -> str: + return f"Basic {self._token}" + + +class AuthManagerAdapter(AuthBase): + """ + A `requests.auth.AuthBase` adapter that integrates an `AuthManager` + into a `requests.Session` to automatically attach the appropriate + Authorization header to every request. + + This adapter is useful when working with `requests.Session.auth` + and allows reuse of authentication strategies defined by `AuthManager`. + """ + def __init__(self, auth_manager: AuthManager): + """ + Args: + auth_manager (AuthManager): An instance of an AuthManager subclass. + """ + self.auth_manager = auth_manager + + def __call__(self, r: PreparedRequest) -> PreparedRequest: + """ + Modifies the outgoing request to include the Authorization header. + + Args: + r (requests.PreparedRequest): The HTTP request being prepared. + + Returns: + requests.PreparedRequest: The modified request with Authorization header. + """ + auth_header = self.auth_manager.auth_header() + if auth_header: + r.headers['Authorization'] = auth_header + return r diff --git a/tests/catalog/test_rest_auth.py b/tests/catalog/test_rest_auth.py new file mode 100644 index 0000000000..776e12aa0f --- /dev/null +++ b/tests/catalog/test_rest_auth.py @@ -0,0 +1,63 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +from pyiceberg.catalog.rest.auth import AuthManagerAdapter, NoopAuthManager, BasicAuthManager + +import base64 +import pytest +import requests +from requests_mock import Mocker + +TEST_URI = "https://iceberg-test-catalog/" + +@pytest.fixture +def rest_mock(requests_mock: Mocker) -> Mocker: + requests_mock.get( + TEST_URI, + json={}, + status_code=200, + ) + return requests_mock + + +def test_noop_auth_header(rest_mock: Mocker): + auth_manager = NoopAuthManager() + session = requests.Session() + session.auth = AuthManagerAdapter(auth_manager) + + response = session.get(TEST_URI) + history = rest_mock.request_history + assert len(history) == 1 + actual_headers = history[0].headers + assert "Authorization" not in actual_headers + + +def test_basic_auth_header(rest_mock: Mocker): + username = "testuser" + password = "testpassword" + expected_token = base64.b64encode(f"{username}:{password}".encode()).decode() + expected_header = f"Basic {expected_token}" + + auth_manager = BasicAuthManager(username=username, password=password) + session = requests.Session() + session.auth = AuthManagerAdapter(auth_manager) + + response = session.get(TEST_URI) + history = rest_mock.request_history + assert len(history) == 1 + actual_headers = history[0].headers + assert actual_headers["Authorization"] == expected_header From 9bac693c845f5b29e7df4f8d84c83426234ec471 Mon Sep 17 00:00:00 2001 From: Sung Yun <107272191+sungwy@users.noreply.github.com> Date: Sun, 13 Apr 2025 15:26:23 +0000 Subject: [PATCH 2/5] lint --- pyiceberg/catalog/rest/auth.py | 28 ++++++++++++++-------------- tests/catalog/test_rest_auth.py | 14 ++++++++------ 2 files changed, 22 insertions(+), 20 deletions(-) diff --git a/pyiceberg/catalog/rest/auth.py b/pyiceberg/catalog/rest/auth.py index ea9e89ad81..a45f6b790f 100644 --- a/pyiceberg/catalog/rest/auth.py +++ b/pyiceberg/catalog/rest/auth.py @@ -15,24 +15,24 @@ # specific language governing permissions and limitations # under the License. +import base64 from abc import ABC, abstractmethod from typing import Optional + from requests import PreparedRequest from requests.auth import AuthBase -import base64 class AuthManager(ABC): """ - Abstract base class for Authentication Managers used to supply authorization headers - to HTTP clients (e.g. requests.Session). - + Abstract base class for Authentication Managers used to supply authorization headers to HTTP clients (e.g. requests.Session). + Subclasses must implement the `auth_header` method to return an Authorization header value. """ + @abstractmethod def auth_header(self) -> Optional[str]: """Return the Authorization header value, or None if not applicable.""" - pass class NoopAuthManager(AuthManager): @@ -41,7 +41,7 @@ def auth_header(self) -> Optional[str]: class BasicAuthManager(AuthManager): - def __init__(self, username: str, password: str): + def __init__(self, username: str, password: str): credentials = f"{username}:{password}" self._token = base64.b64encode(credentials.encode()).decode() @@ -50,24 +50,24 @@ def auth_header(self) -> str: class AuthManagerAdapter(AuthBase): - """ - A `requests.auth.AuthBase` adapter that integrates an `AuthManager` - into a `requests.Session` to automatically attach the appropriate - Authorization header to every request. + """A `requests.auth.AuthBase` adapter that integrates an `AuthManager` into a `requests.Session` to automatically attach the appropriate Authorization header to every request. This adapter is useful when working with `requests.Session.auth` and allows reuse of authentication strategies defined by `AuthManager`. """ + def __init__(self, auth_manager: AuthManager): """ - Args: - auth_manager (AuthManager): An instance of an AuthManager subclass. + Initialize AuthManagerAdapter. + + Args: + auth_manager (AuthManager): An instance of an AuthManager subclass. """ self.auth_manager = auth_manager def __call__(self, r: PreparedRequest) -> PreparedRequest: """ - Modifies the outgoing request to include the Authorization header. + Modify the outgoing request to include the Authorization header. Args: r (requests.PreparedRequest): The HTTP request being prepared. @@ -77,5 +77,5 @@ def __call__(self, r: PreparedRequest) -> PreparedRequest: """ auth_header = self.auth_manager.auth_header() if auth_header: - r.headers['Authorization'] = auth_header + r.headers["Authorization"] = auth_header return r diff --git a/tests/catalog/test_rest_auth.py b/tests/catalog/test_rest_auth.py index 776e12aa0f..3d3d4a807d 100644 --- a/tests/catalog/test_rest_auth.py +++ b/tests/catalog/test_rest_auth.py @@ -15,15 +15,17 @@ # specific language governing permissions and limitations # under the License. -from pyiceberg.catalog.rest.auth import AuthManagerAdapter, NoopAuthManager, BasicAuthManager - import base64 + import pytest import requests from requests_mock import Mocker +from pyiceberg.catalog.rest.auth import AuthManagerAdapter, BasicAuthManager, NoopAuthManager + TEST_URI = "https://iceberg-test-catalog/" + @pytest.fixture def rest_mock(requests_mock: Mocker) -> Mocker: requests_mock.get( @@ -34,19 +36,19 @@ def rest_mock(requests_mock: Mocker) -> Mocker: return requests_mock -def test_noop_auth_header(rest_mock: Mocker): +def test_noop_auth_header(rest_mock: Mocker) -> None: auth_manager = NoopAuthManager() session = requests.Session() session.auth = AuthManagerAdapter(auth_manager) - response = session.get(TEST_URI) + session.get(TEST_URI) history = rest_mock.request_history assert len(history) == 1 actual_headers = history[0].headers assert "Authorization" not in actual_headers -def test_basic_auth_header(rest_mock: Mocker): +def test_basic_auth_header(rest_mock: Mocker) -> None: username = "testuser" password = "testpassword" expected_token = base64.b64encode(f"{username}:{password}".encode()).decode() @@ -56,7 +58,7 @@ def test_basic_auth_header(rest_mock: Mocker): session = requests.Session() session.auth = AuthManagerAdapter(auth_manager) - response = session.get(TEST_URI) + session.get(TEST_URI) history = rest_mock.request_history assert len(history) == 1 actual_headers = history[0].headers From 81d41656b983c0679f8edd0460d11cd3853f0f83 Mon Sep 17 00:00:00 2001 From: Sung Yun <107272191+sungwy@users.noreply.github.com> Date: Tue, 15 Apr 2025 00:41:29 +0000 Subject: [PATCH 3/5] adopt feedback --- pyiceberg/catalog/rest/auth.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/pyiceberg/catalog/rest/auth.py b/pyiceberg/catalog/rest/auth.py index a45f6b790f..92c184a708 100644 --- a/pyiceberg/catalog/rest/auth.py +++ b/pyiceberg/catalog/rest/auth.py @@ -54,6 +54,8 @@ class AuthManagerAdapter(AuthBase): This adapter is useful when working with `requests.Session.auth` and allows reuse of authentication strategies defined by `AuthManager`. + This AuthManagerAdapter is only intended to be used against the REST Catalog + Server that expects the Authorization Header. """ def __init__(self, auth_manager: AuthManager): @@ -65,7 +67,7 @@ def __init__(self, auth_manager: AuthManager): """ self.auth_manager = auth_manager - def __call__(self, r: PreparedRequest) -> PreparedRequest: + def __call__(self, request: PreparedRequest) -> PreparedRequest: """ Modify the outgoing request to include the Authorization header. @@ -75,7 +77,6 @@ def __call__(self, r: PreparedRequest) -> PreparedRequest: Returns: requests.PreparedRequest: The modified request with Authorization header. """ - auth_header = self.auth_manager.auth_header() - if auth_header: - r.headers["Authorization"] = auth_header - return r + if auth_header := self.auth_manager.auth_header(): + request.headers["Authorization"] = auth_header + return request From 984d79f0475a6b5134ad4f538b70fc0c5b5e596a Mon Sep 17 00:00:00 2001 From: Sung Yun <107272191+sungwy@users.noreply.github.com> Date: Tue, 15 Apr 2025 00:45:07 +0000 Subject: [PATCH 4/5] lint --- pyiceberg/catalog/rest/auth.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyiceberg/catalog/rest/auth.py b/pyiceberg/catalog/rest/auth.py index 92c184a708..07cd2bac4f 100644 --- a/pyiceberg/catalog/rest/auth.py +++ b/pyiceberg/catalog/rest/auth.py @@ -72,7 +72,7 @@ def __call__(self, request: PreparedRequest) -> PreparedRequest: Modify the outgoing request to include the Authorization header. Args: - r (requests.PreparedRequest): The HTTP request being prepared. + requests (requests.PreparedRequest): The HTTP request being prepared. Returns: requests.PreparedRequest: The modified request with Authorization header. From 713b17c20de7fda9bde6c267ca042dec94e8368c Mon Sep 17 00:00:00 2001 From: Sung Yun <107272191+sungwy@users.noreply.github.com> Date: Tue, 15 Apr 2025 01:03:49 +0000 Subject: [PATCH 5/5] lint --- pyiceberg/catalog/rest/auth.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyiceberg/catalog/rest/auth.py b/pyiceberg/catalog/rest/auth.py index 07cd2bac4f..041a8a4cd1 100644 --- a/pyiceberg/catalog/rest/auth.py +++ b/pyiceberg/catalog/rest/auth.py @@ -72,7 +72,7 @@ def __call__(self, request: PreparedRequest) -> PreparedRequest: Modify the outgoing request to include the Authorization header. Args: - requests (requests.PreparedRequest): The HTTP request being prepared. + request (requests.PreparedRequest): The HTTP request being prepared. Returns: requests.PreparedRequest: The modified request with Authorization header.