Merge branch 'main' into auth-manager-oauth

Author: sungwy

.github/workflows/python-ci.yml

@@ -65,4 +65,22 @@ jobs:
     - name: Linters
       run: make lint
     - name: Tests
-      run: make test-coverage
+      run: make test-coverage-unit
+
+  integration-test:
+    runs-on: ubuntu-22.04
+    strategy:
+      matrix:
+        python: ['3.9', '3.10', '3.11', '3.12']
+
+    steps:
+    - uses: actions/checkout@v4
+    - name: Install system dependencies
+      run: sudo apt-get update && sudo apt-get install -y libkrb5-dev # for kerberos
+    - name: Install
+      run: make install
+    - name: Run integration tests
+      run: make test-coverage-integration
+    - name: Show debug logs
+      if: ${{ failure() }}
+      run: docker compose -f dev/docker-compose.yml logs

.github/workflows/python-integration.yml

@@ -36,6 +36,7 @@
 from hive_metastore.ttypes import (
     AlreadyExistsException,
     CheckLockRequest,
+    EnvironmentContext,
     FieldSchema,
     InvalidOperationException,
     LockComponent,
@@ -135,6 +136,8 @@
 DEFAULT_LOCK_CHECK_MIN_WAIT_TIME = 0.1  # 100 milliseconds
 DEFAULT_LOCK_CHECK_MAX_WAIT_TIME = 60  # 1 min
 DEFAULT_LOCK_CHECK_RETRIES = 4
+DO_NOT_UPDATE_STATS = "DO_NOT_UPDATE_STATS"
+DO_NOT_UPDATE_STATS_DEFAULT = "true"
 
 logger = logging.getLogger(__name__)
 
@@ -539,7 +542,12 @@ def commit_table(
                         metadata_location=updated_staged_table.metadata_location,
                         previous_metadata_location=current_table.metadata_location,
                     )
-                    open_client.alter_table(dbname=database_name, tbl_name=table_name, new_tbl=hive_table)
+                    open_client.alter_table_with_environment_context(
+                        dbname=database_name,
+                        tbl_name=table_name,
+                        new_tbl=hive_table,
+                        environment_context=EnvironmentContext(properties={DO_NOT_UPDATE_STATS: DO_NOT_UPDATE_STATS_DEFAULT}),
+                    )
                 else:
                     # Table does not exist, create it.
                     hive_table = self._convert_iceberg_into_hive(
@@ -626,7 +634,12 @@ def rename_table(self, from_identifier: Union[str, Identifier], to_identifier: U
                 tbl = open_client.get_table(dbname=from_database_name, tbl_name=from_table_name)
                 tbl.dbName = to_database_name
                 tbl.tableName = to_table_name
-                open_client.alter_table(dbname=from_database_name, tbl_name=from_table_name, new_tbl=tbl)
+                open_client.alter_table_with_environment_context(
+                    dbname=from_database_name,
+                    tbl_name=from_table_name,
+                    new_tbl=tbl,
+                    environment_context=EnvironmentContext(properties={DO_NOT_UPDATE_STATS: DO_NOT_UPDATE_STATS_DEFAULT}),
+                )
         except NoSuchObjectException as e:
             raise NoSuchTableError(f"Table does not exist: {from_table_name}") from e
         except InvalidOperationException as e:

poetry.lock

@@ -725,7 +725,7 @@ def list_namespaces(self, namespace: Union[str, Identifier] = ()) -> List[Identi
         try:
             response.raise_for_status()
         except HTTPError as exc:
-            _handle_non_200_response(exc, {})
+            _handle_non_200_response(exc, {404: NoSuchNamespaceError})
 
         return ListNamespaceResponse.model_validate_json(response.text).namespaces
 

pyiceberg/catalog/hive.py

@@ -549,13 +549,6 @@ def construct_refs(cls, table_metadata: TableMetadata) -> TableMetadata:
     """The table’s highest assigned sequence number, a monotonically
     increasing long that tracks the order of snapshots in a table."""
 
-    row_lineage: bool = Field(alias="row-lineage", default=False)
-    """Indicates that row-lineage is enabled on the table
-
-    For more information:
-    https://iceberg.apache.org/spec/?column-projection#row-lineage
-    """
-
     next_row_id: Optional[int] = Field(alias="next-row-id", default=None)
     """A long higher than all assigned row IDs; the next snapshot's `first-row-id`."""
 

pyiceberg/catalog/rest/__init__.py

@@ -438,7 +438,7 @@ def ancestors_of(current_snapshot: Optional[Snapshot], table_metadata: TableMeta
 
 
 def ancestors_between(
-    to_snapshot: Snapshot, from_snapshot: Optional[Snapshot], table_metadata: TableMetadata
+    from_snapshot: Optional[Snapshot], to_snapshot: Snapshot, table_metadata: TableMetadata
 ) -> Iterable[Snapshot]:
     """Get the ancestors of and including the given snapshot between the to and from snapshots."""
     if from_snapshot is not None:

pyiceberg/table/metadata.py

@@ -23,17 +23,17 @@
 
 def validation_history(
     table: Table,
-    to_snapshot: Snapshot,
     from_snapshot: Snapshot,
+    to_snapshot: Snapshot,
     matching_operations: set[Operation],
     manifest_content_filter: ManifestContent,
 ) -> tuple[list[ManifestFile], set[int]]:
     """Return newly added manifests and snapshot IDs between the starting snapshot and parent snapshot.
 
     Args:
         table: Table to get the history from
-        to_snapshot: Starting snapshot
         from_snapshot: Parent snapshot to get the history from
+        to_snapshot: Starting snapshot
         matching_operations: Operations to match on
         manifest_content_filter: Manifest content type to filter
 
@@ -47,7 +47,7 @@ def validation_history(
     snapshots: set[int] = set()
 
     last_snapshot = None
-    for snapshot in ancestors_between(to_snapshot, from_snapshot, table.metadata):
+    for snapshot in ancestors_between(from_snapshot, to_snapshot, table.metadata):
         last_snapshot = snapshot
         summary = snapshot.summary
         if summary is None:

pyiceberg/table/snapshots.py

@@ -29,6 +29,7 @@
 import thrift.transport.TSocket
 from hive_metastore.ttypes import (
     AlreadyExistsException,
+    EnvironmentContext,
     FieldSchema,
     InvalidOperationException,
     LockResponse,
@@ -44,6 +45,8 @@
 
 from pyiceberg.catalog import PropertiesUpdateSummary
 from pyiceberg.catalog.hive import (
+    DO_NOT_UPDATE_STATS,
+    DO_NOT_UPDATE_STATS_DEFAULT,
     HIVE_KERBEROS_AUTH,
     LOCK_CHECK_MAX_WAIT_TIME,
     LOCK_CHECK_MIN_WAIT_TIME,
@@ -874,7 +877,7 @@ def test_rename_table(hive_table: HiveTable) -> None:
 
     catalog._client = MagicMock()
     catalog._client.__enter__().get_table.side_effect = [hive_table, renamed_table]
-    catalog._client.__enter__().alter_table.return_value = None
+    catalog._client.__enter__().alter_table_with_environment_context.return_value = None
 
     from_identifier = ("default", "new_tabl2e")
     to_identifier = ("default", "new_tabl3e")
@@ -884,7 +887,12 @@ def test_rename_table(hive_table: HiveTable) -> None:
 
     calls = [call(dbname="default", tbl_name="new_tabl2e"), call(dbname="default", tbl_name="new_tabl3e")]
     catalog._client.__enter__().get_table.assert_has_calls(calls)
-    catalog._client.__enter__().alter_table.assert_called_with(dbname="default", tbl_name="new_tabl2e", new_tbl=renamed_table)
+    catalog._client.__enter__().alter_table_with_environment_context.assert_called_with(
+        dbname="default",
+        tbl_name="new_tabl2e",
+        new_tbl=renamed_table,
+        environment_context=EnvironmentContext(properties={DO_NOT_UPDATE_STATS: DO_NOT_UPDATE_STATS_DEFAULT}),
+    )
 
 
 def test_rename_table_from_self_identifier(hive_table: HiveTable) -> None:
@@ -902,22 +910,27 @@ def test_rename_table_from_self_identifier(hive_table: HiveTable) -> None:
     renamed_table.tableName = "new_tabl3e"
 
     catalog._client.__enter__().get_table.side_effect = [hive_table, renamed_table]
-    catalog._client.__enter__().alter_table.return_value = None
+    catalog._client.__enter__().alter_table_with_environment_context.return_value = None
     to_identifier = ("default", "new_tabl3e")
     table = catalog.rename_table(from_table.name(), to_identifier)
 
     assert table.name() == to_identifier
 
     calls = [call(dbname="default", tbl_name="new_tabl2e"), call(dbname="default", tbl_name="new_tabl3e")]
     catalog._client.__enter__().get_table.assert_has_calls(calls)
-    catalog._client.__enter__().alter_table.assert_called_with(dbname="default", tbl_name="new_tabl2e", new_tbl=renamed_table)
+    catalog._client.__enter__().alter_table_with_environment_context.assert_called_with(
+        dbname="default",
+        tbl_name="new_tabl2e",
+        new_tbl=renamed_table,
+        environment_context=EnvironmentContext(properties={DO_NOT_UPDATE_STATS: DO_NOT_UPDATE_STATS_DEFAULT}),
+    )
 
 
 def test_rename_table_from_does_not_exists() -> None:
     catalog = HiveCatalog(HIVE_CATALOG_NAME, uri=HIVE_METASTORE_FAKE_URL)
 
     catalog._client = MagicMock()
-    catalog._client.__enter__().alter_table.side_effect = NoSuchObjectException(
+    catalog._client.__enter__().alter_table_with_environment_context.side_effect = NoSuchObjectException(
         message="hive.default.does_not_exists table not found"
     )
 
@@ -931,7 +944,7 @@ def test_rename_table_to_namespace_does_not_exists() -> None:
     catalog = HiveCatalog(HIVE_CATALOG_NAME, uri=HIVE_METASTORE_FAKE_URL)
 
     catalog._client = MagicMock()
-    catalog._client.__enter__().alter_table.side_effect = InvalidOperationException(
+    catalog._client.__enter__().alter_table_with_environment_context.side_effect = InvalidOperationException(
         message="Unable to change partition or table. Database default does not exist Check metastore logs for detailed stack.does_not_exists"
     )
 

pyiceberg/table/update/validate.py

@@ -555,6 +555,24 @@ def test_list_namespace_with_parent_200(rest_mock: Mocker) -> None:
     ]
 
 
+def test_list_namespace_with_parent_404(rest_mock: Mocker) -> None:
+    rest_mock.get(
+        f"{TEST_URI}v1/namespaces?parent=some_namespace",
+        json={
+            "error": {
+                "message": "Namespace provided in the `parent` query parameter is not found",
+                "type": "NoSuchNamespaceException",
+                "code": 404,
+            }
+        },
+        status_code=404,
+        request_headers=TEST_HEADERS,
+    )
+
+    with pytest.raises(NoSuchNamespaceError):
+        RestCatalog("rest", uri=TEST_URI, token=TEST_TOKEN).list_namespaces(("some_namespace",))
+
+
 @pytest.mark.filterwarnings(
     "ignore:Deprecated in 0.8.0, will be removed in 1.0.0. Iceberg REST client is missing the OAuth2 server URI:DeprecationWarning"
 )