From 6cc1735fa692f02e880af45081c6cdef1e2fe154 Mon Sep 17 00:00:00 2001 From: Fokko Date: Tue, 13 May 2025 18:21:28 +0200 Subject: [PATCH 1/3] Add `snapshot-loading-mode` to RESTCatalog This will allow to set the snapshots to be send back. In case of refs, only the snapshots referenced by branches or tags will be returned: https://github.com/apache/iceberg/blob/5d2230ead79da64a8c871a02eb1304a94aaece5c/open-api/rest-catalog-open-api.yaml#L954-L956 --- pyiceberg/catalog/rest/__init__.py | 14 ++++++++-- tests/catalog/test_rest.py | 43 +++++++++++++++++++++++++++++- 2 files changed, 54 insertions(+), 3 deletions(-) diff --git a/pyiceberg/catalog/rest/__init__.py b/pyiceberg/catalog/rest/__init__.py index 8ee9e5fdc9..633ef6498b 100644 --- a/pyiceberg/catalog/rest/__init__.py +++ b/pyiceberg/catalog/rest/__init__.py @@ -138,6 +138,7 @@ class IdentifierKind(Enum): SIGV4_REGION = "rest.signing-region" SIGV4_SERVICE = "rest.signing-name" OAUTH2_SERVER_URI = "oauth2-server-uri" +SNAPSHOT_LOADING_MODE = "snapshot-loading-mode" NAMESPACE_SEPARATOR = b"\x1f".decode(UTF8) @@ -678,7 +679,16 @@ def list_tables(self, namespace: Union[str, Identifier]) -> List[Identifier]: @retry(**_RETRY_ARGS) def load_table(self, identifier: Union[str, Identifier]) -> Table: - response = self._session.get(self.url(Endpoints.load_table, prefixed=True, **self._split_identifier_for_path(identifier))) + params = {} + if mode := self.properties.get(SNAPSHOT_LOADING_MODE): + if mode in {"all", "refs"}: + params["snapshots"] = mode + else: + raise ValueError("Invalid snapshot-loading-mode: {}") + + response = self._session.get( + self.url(Endpoints.load_table, prefixed=True, **self._split_identifier_for_path(identifier)), params=params + ) try: response.raise_for_status() except HTTPError as exc: @@ -816,7 +826,7 @@ def list_namespaces(self, namespace: Union[str, Identifier] = ()) -> List[Identi try: response.raise_for_status() except HTTPError as exc: - self._handle_non_200_response(exc, {}) + self._handle_non_200_response(exc, {404: NoSuchNamespaceError}) return ListNamespaceResponse.model_validate_json(response.text).namespaces diff --git a/tests/catalog/test_rest.py b/tests/catalog/test_rest.py index f2fc6ceb6b..dfc6a0e2fb 100644 --- a/tests/catalog/test_rest.py +++ b/tests/catalog/test_rest.py @@ -24,7 +24,7 @@ import pyiceberg from pyiceberg.catalog import PropertiesUpdateSummary, load_catalog -from pyiceberg.catalog.rest import OAUTH2_SERVER_URI, RestCatalog +from pyiceberg.catalog.rest import OAUTH2_SERVER_URI, SNAPSHOT_LOADING_MODE, RestCatalog from pyiceberg.exceptions import ( AuthorizationExpiredError, NamespaceAlreadyExistsError, @@ -555,6 +555,24 @@ def test_list_namespace_with_parent_200(rest_mock: Mocker) -> None: ] +def test_list_namespace_with_parent_404(rest_mock: Mocker) -> None: + rest_mock.get( + f"{TEST_URI}v1/namespaces?parent=some_namespace", + json={ + "error": { + "message": "Namespace provided in the `parent` query parameter is not found", + "type": "NoSuchNamespaceException", + "code": 404, + } + }, + status_code=404, + request_headers=TEST_HEADERS, + ) + + with pytest.raises(NoSuchNamespaceError): + RestCatalog("rest", uri=TEST_URI, token=TEST_TOKEN).list_namespaces(("some_namespace",)) + + @pytest.mark.filterwarnings( "ignore:Deprecated in 0.8.0, will be removed in 1.0.0. Iceberg REST client is missing the OAuth2 server URI:DeprecationWarning" ) @@ -835,6 +853,29 @@ def test_load_table_200(rest_mock: Mocker, example_table_metadata_with_snapshot_ assert actual == expected +def test_load_table_200_loading_mode( + rest_mock: Mocker, example_table_metadata_with_snapshot_v1_rest_json: Dict[str, Any] +) -> None: + rest_mock.get( + f"{TEST_URI}v1/namespaces/fokko/tables/table?snapshots=refs", + json=example_table_metadata_with_snapshot_v1_rest_json, + status_code=200, + request_headers=TEST_HEADERS, + ) + catalog = RestCatalog("rest", uri=TEST_URI, token=TEST_TOKEN, **{SNAPSHOT_LOADING_MODE: "refs"}) + actual = catalog.load_table(("fokko", "table")) + expected = Table( + identifier=("fokko", "table"), + metadata_location=example_table_metadata_with_snapshot_v1_rest_json["metadata-location"], + metadata=TableMetadataV1(**example_table_metadata_with_snapshot_v1_rest_json["metadata"]), + io=load_file_io(), + catalog=catalog, + ) + # First compare the dicts + assert actual.metadata.model_dump() == expected.metadata.model_dump() + assert actual == expected + + def test_load_table_honor_access_delegation( rest_mock: Mocker, example_table_metadata_with_snapshot_v1_rest_json: Dict[str, Any] ) -> None: From e3c1c1b1497f9cd27be2d4e54d1e5aec1e69c198 Mon Sep 17 00:00:00 2001 From: Fokko Date: Tue, 13 May 2025 18:26:57 +0200 Subject: [PATCH 2/3] Add docs --- mkdocs/docs/configuration.md | 1 + 1 file changed, 1 insertion(+) diff --git a/mkdocs/docs/configuration.md b/mkdocs/docs/configuration.md index 1e364a11fe..dc51f81434 100644 --- a/mkdocs/docs/configuration.md +++ b/mkdocs/docs/configuration.md @@ -332,6 +332,7 @@ catalog: | rest.signing-region | us-east-1 | The region to use when SigV4 signing a request | | rest.signing-name | execute-api | The service signing name to use when SigV4 signing a request | | oauth2-server-uri | | Authentication URL to use for client credentials authentication (default: uri + 'v1/oauth/tokens') | +| snapshot-loading-mode | refs | The snapshots to return in the body of the metadata. Setting the value to `all` would return the full set of snapshots currently valid for the table. Setting the value to `refs` would load all snapshots referenced by branches or tags. | From 0e527c1b43ba5239550bb64278554d8b4891ada5 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 2 Jun 2025 02:05:55 +0000 Subject: [PATCH 3/3] Build: Bump pytest-mock from 3.14.0 to 3.14.1 Bumps [pytest-mock](https://github.com/pytest-dev/pytest-mock) from 3.14.0 to 3.14.1. - [Release notes](https://github.com/pytest-dev/pytest-mock/releases) - [Changelog](https://github.com/pytest-dev/pytest-mock/blob/main/CHANGELOG.rst) - [Commits](https://github.com/pytest-dev/pytest-mock/compare/v3.14.0...v3.14.1) --- updated-dependencies: - dependency-name: pytest-mock dependency-version: 3.14.1 dependency-type: direct:development update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- poetry.lock | 8 ++++---- pyproject.toml | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/poetry.lock b/poetry.lock index 849485ffbb..c8e3d6b4dd 100644 --- a/poetry.lock +++ b/poetry.lock @@ -4352,14 +4352,14 @@ pytest = ">=3.2.5" [[package]] name = "pytest-mock" -version = "3.14.0" +version = "3.14.1" description = "Thin-wrapper around the mock package for easier use with pytest" optional = false python-versions = ">=3.8" groups = ["dev"] files = [ - {file = "pytest-mock-3.14.0.tar.gz", hash = "sha256:2719255a1efeceadbc056d6bf3df3d1c5015530fb40cf347c0f9afac88410bd0"}, - {file = "pytest_mock-3.14.0-py3-none-any.whl", hash = "sha256:0b72c38033392a5f4621342fe11e9219ac11ec9d375f8e2a0c164539e0d70f6f"}, + {file = "pytest_mock-3.14.1-py3-none-any.whl", hash = "sha256:178aefcd11307d874b4cd3100344e7e2d888d9791a6a1d9bfe90fbc1b74fd1d0"}, + {file = "pytest_mock-3.14.1.tar.gz", hash = "sha256:159e9edac4c451ce77a5cdb9fc5d1100708d2dd4ba3c3df572f14097351af80e"}, ] [package.dependencies] @@ -5992,4 +5992,4 @@ zstandard = ["zstandard"] [metadata] lock-version = "2.1" python-versions = "^3.9.2, !=3.9.7" -content-hash = "f3d2267ce4f380399dc767a6483a1f198127856883e394d8befbd8a871bbeab9" +content-hash = "b4caa24bc825d9894fa897e91358700a1f7d7769555f638ce92e770af9a60c5c" diff --git a/pyproject.toml b/pyproject.toml index 3116214f71..71420aca5d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -95,7 +95,7 @@ coverage = { version = "^7.4.2", extras = ["toml"] } requests-mock = "1.12.1" moto = { version = "^5.0.2", extras = ["server"] } typing-extensions = "4.13.2" -pytest-mock = "3.14.0" +pytest-mock = "3.14.1" pyspark = "3.5.5" cython = "3.0.12" deptry = ">=0.14,<0.24"