Merge branch 'main' of github.com:apache/iceberg-python into fd-rust

Author: Fokko

.github/workflows/license_check.yml

@@ -22,7 +22,7 @@ on: pull_request
 
 jobs:
   rat:
-    runs-on: ubuntu-22.04
+    runs-on: ubuntu-latest
     steps:
     - uses: actions/checkout@v5
     - run: dev/check-license

.github/workflows/pypi-build-artifacts.yml

@@ -32,7 +32,7 @@ jobs:
     runs-on: ${{ matrix.os }}
     strategy:
       matrix:
-        os: [ ubuntu-22.04, windows-2022, macos-13, macos-14 ]
+        os: [ ubuntu-latest, windows-latest, macos-latest ]
 
     steps:
       - uses: actions/checkout@v5

.github/workflows/python-ci-docs.yml

@@ -32,7 +32,7 @@ concurrency:
 
 jobs:
   docs:
-    runs-on: ubuntu-22.04
+    runs-on: ubuntu-latest
 
     steps:
       - uses: actions/checkout@v5

.github/workflows/python-ci.yml

@@ -44,7 +44,7 @@ concurrency:
 
 jobs:
   lint-and-test:
-    runs-on: ubuntu-22.04
+    runs-on: ubuntu-latest
     strategy:
       matrix:
         python: ['3.9', '3.10', '3.11', '3.12']
@@ -68,7 +68,7 @@ jobs:
       run: COVERAGE_FAIL_UNDER=85 make coverage-report
 
   integration-test:
-    runs-on: ubuntu-22.04
+    runs-on: ubuntu-latest
     strategy:
       matrix:
         python: ['3.9', '3.10', '3.11', '3.12']

.github/workflows/python-release-docs.yml

@@ -27,7 +27,7 @@ concurrency:
 
 jobs:
   docs:
-    runs-on: ubuntu-22.04
+    runs-on: ubuntu-latest
 
     steps:
       - uses: actions/checkout@v5

.github/workflows/stale.yml

@@ -29,7 +29,7 @@ permissions:
 jobs:
   stale:
     if: github.repository_owner == 'apache'
-    runs-on: ubuntu-22.04
+    runs-on: ubuntu-latest
     steps:
       - uses: actions/stale@v10.0.0
         with:

.github/workflows/svn-build-artifacts.yml

@@ -32,7 +32,7 @@ jobs:
     runs-on: ${{ matrix.os }}
     strategy:
       matrix:
-        os: [ ubuntu-22.04, windows-2022, macos-13, macos-14 ]
+        os: [ ubuntu-latest, windows-latest, macos-latest ]
 
     steps:
       - uses: actions/checkout@v5

Makefile

@@ -58,12 +58,12 @@ help: ## Display this help message
 install-poetry: ## Ensure Poetry is installed at the specified version
 	@if ! command -v ${POETRY} &> /dev/null; then \
 		echo "Poetry not found. Installing..."; \
-		${PIP} install --user poetry==$(POETRY_VERSION); \
+		${PIP} install poetry==$(POETRY_VERSION); \
 	else \
 		INSTALLED_VERSION=$$(${PIP} show poetry | grep Version | awk '{print $$2}'); \
 		if [ "$$INSTALLED_VERSION" != "$(POETRY_VERSION)" ]; then \
 			echo "Updating Poetry to version $(POETRY_VERSION)..."; \
-			${PIP} install --user --upgrade poetry==$(POETRY_VERSION); \
+			${PIP} install --upgrade poetry==$(POETRY_VERSION); \
 		else \
 			echo "Poetry version $(POETRY_VERSION) already installed."; \
 		fi; \

dev/Dockerfile

@@ -42,9 +42,22 @@ ENV ICEBERG_SPARK_RUNTIME_VERSION=3.5_2.12
 ENV ICEBERG_VERSION=1.9.1
 ENV PYICEBERG_VERSION=0.10.0
 
-RUN curl --retry 5 -s -C - https://archive.apache.org/dist/spark/spark-${SPARK_VERSION}/spark-${SPARK_VERSION}-bin-hadoop3.tgz -o spark-${SPARK_VERSION}-bin-hadoop3.tgz \
- && tar xzf spark-${SPARK_VERSION}-bin-hadoop3.tgz --directory /opt/spark --strip-components 1 \
- && rm -rf spark-${SPARK_VERSION}-bin-hadoop3.tgz
+# Try the primary Apache mirror (downloads.apache.org) first, then fall back to the archive
+RUN set -eux; \
+  FILE=spark-${SPARK_VERSION}-bin-hadoop3.tgz; \
+  URLS="https://downloads.apache.org/spark/spark-${SPARK_VERSION}/${FILE} https://archive.apache.org/dist/spark/spark-${SPARK_VERSION}/${FILE}"; \
+  for url in $URLS; do \
+    echo "Attempting download: $url"; \
+    if curl --retry 3 --retry-delay 5 -f -s -C - "$url" -o "$FILE"; then \
+      echo "Downloaded from: $url"; \
+      break; \
+    else \
+      echo "Failed to download from: $url"; \
+    fi; \
+  done; \
+  if [ ! -f "$FILE" ]; then echo "Failed to download Spark from all mirrors" >&2; exit 1; fi; \
+  tar xzf "$FILE" --directory /opt/spark --strip-components 1; \
+  rm -rf "$FILE"
 
 # Download iceberg spark runtime
 RUN curl --retry 5 -s https://repo1.maven.org/maven2/org/apache/iceberg/iceberg-spark-runtime-${ICEBERG_SPARK_RUNTIME_VERSION}/${ICEBERG_VERSION}/iceberg-spark-runtime-${ICEBERG_SPARK_RUNTIME_VERSION}-${ICEBERG_VERSION}.jar \

pyiceberg/table/update/spec.py

@@ -32,6 +32,7 @@
 from pyiceberg.schema import Schema
 from pyiceberg.table.update import (
     AddPartitionSpecUpdate,
+    AssertDefaultSpecId,
     AssertLastAssignedPartitionId,
     SetDefaultSpecUpdate,
     TableRequirement,
@@ -169,7 +170,11 @@ def _commit(self) -> UpdatesAndRequirements:
                 updates = (SetDefaultSpecUpdate(spec_id=new_spec.spec_id),)
 
             required_last_assigned_partitioned_id = self._transaction.table_metadata.last_partition_id
-            requirements = (AssertLastAssignedPartitionId(last_assigned_partition_id=required_last_assigned_partitioned_id),)
+            default_spec_id = self._transaction.table_metadata.default_spec_id
+            requirements = (
+                AssertLastAssignedPartitionId(last_assigned_partition_id=required_last_assigned_partitioned_id),
+                AssertDefaultSpecId(default_spec_id=default_spec_id),
+            )
 
         return updates, requirements