From 8d60727c0874a6e2b7f42d9c2fcfe00a66bfd9b9 Mon Sep 17 00:00:00 2001 From: Fokko Date: Mon, 28 Apr 2025 09:52:00 +0200 Subject: [PATCH 1/3] Revert "Bump Snapshot versions (#1907)" This reverts commit bb45d1ea90a49cf3e9c883a95b9aa6bb2419b55b. --- dev/Dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dev/Dockerfile b/dev/Dockerfile index a3f8795d26..97f6ac642f 100644 --- a/dev/Dockerfile +++ b/dev/Dockerfile @@ -47,12 +47,12 @@ RUN curl --retry 5 -s -C - https://archive.apache.org/dist/spark/spark-${SPARK_V && rm -rf spark-${SPARK_VERSION}-bin-hadoop3.tgz # Download iceberg spark runtime -RUN curl --retry 5 -s https://repository.apache.org/content/groups/snapshots/org/apache/iceberg/iceberg-spark-runtime-3.5_2.12/1.9.0-SNAPSHOT/iceberg-spark-runtime-3.5_2.12-1.9.0-20250409.001855-44.jar \ +RUN curl --retry 5 -s https://repository.apache.org/content/groups/snapshots/org/apache/iceberg/iceberg-spark-runtime-3.5_2.12/1.9.0-SNAPSHOT/iceberg-spark-runtime-3.5_2.12-1.9.0-20250408.001846-43.jar \ -Lo /opt/spark/jars/iceberg-spark-runtime-${ICEBERG_SPARK_RUNTIME_VERSION}-${ICEBERG_VERSION}.jar # Download AWS bundle -RUN curl --retry 5 -s https://repository.apache.org/content/groups/snapshots/org/apache/iceberg/iceberg-aws-bundle/1.9.0-SNAPSHOT/iceberg-aws-bundle-1.9.0-20250409.002731-88.jar \ +RUN curl --retry 5 -s https://repository.apache.org/content/groups/snapshots/org/apache/iceberg/iceberg-aws-bundle/1.9.0-SNAPSHOT/iceberg-aws-bundle-1.9.0-20250408.002722-86.jar \ -Lo /opt/spark/jars/iceberg-aws-bundle-${ICEBERG_VERSION}.jar COPY spark-defaults.conf /opt/spark/conf From 54ae850ed5461cdecf85d468e2e2763e66a08ed1 Mon Sep 17 00:00:00 2001 From: Fokko Date: Mon, 28 Apr 2025 09:52:05 +0200 Subject: [PATCH 2/3] Revert "CI: Use Java 1.9.0-SNAPSHOT for testing (#1899)" This reverts commit aeb4493b1652341bece67f42e892d66563175115. --- dev/Dockerfile | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/dev/Dockerfile b/dev/Dockerfile index 97f6ac642f..bb43ea1df8 100644 --- a/dev/Dockerfile +++ b/dev/Dockerfile @@ -39,7 +39,7 @@ WORKDIR ${SPARK_HOME} # Remember to also update `tests/conftest`'s spark setting ENV SPARK_VERSION=3.5.4 ENV ICEBERG_SPARK_RUNTIME_VERSION=3.5_2.12 -ENV ICEBERG_VERSION=1.9.0-SNAPSHOT +ENV ICEBERG_VERSION=1.8.0 ENV PYICEBERG_VERSION=0.9.0 RUN curl --retry 5 -s -C - https://archive.apache.org/dist/spark/spark-${SPARK_VERSION}/spark-${SPARK_VERSION}-bin-hadoop3.tgz -o spark-${SPARK_VERSION}-bin-hadoop3.tgz \ @@ -47,12 +47,12 @@ RUN curl --retry 5 -s -C - https://archive.apache.org/dist/spark/spark-${SPARK_V && rm -rf spark-${SPARK_VERSION}-bin-hadoop3.tgz # Download iceberg spark runtime -RUN curl --retry 5 -s https://repository.apache.org/content/groups/snapshots/org/apache/iceberg/iceberg-spark-runtime-3.5_2.12/1.9.0-SNAPSHOT/iceberg-spark-runtime-3.5_2.12-1.9.0-20250408.001846-43.jar \ +RUN curl --retry 5 -s https://repo1.maven.org/maven2/org/apache/iceberg/iceberg-spark-runtime-${ICEBERG_SPARK_RUNTIME_VERSION}/${ICEBERG_VERSION}/iceberg-spark-runtime-${ICEBERG_SPARK_RUNTIME_VERSION}-${ICEBERG_VERSION}.jar \ -Lo /opt/spark/jars/iceberg-spark-runtime-${ICEBERG_SPARK_RUNTIME_VERSION}-${ICEBERG_VERSION}.jar # Download AWS bundle -RUN curl --retry 5 -s https://repository.apache.org/content/groups/snapshots/org/apache/iceberg/iceberg-aws-bundle/1.9.0-SNAPSHOT/iceberg-aws-bundle-1.9.0-20250408.002722-86.jar \ +RUN curl --retry 5 -s https://repo1.maven.org/maven2/org/apache/iceberg/iceberg-aws-bundle/${ICEBERG_VERSION}/iceberg-aws-bundle-${ICEBERG_VERSION}.jar \ -Lo /opt/spark/jars/iceberg-aws-bundle-${ICEBERG_VERSION}.jar COPY spark-defaults.conf /opt/spark/conf From c1b935ddb1e648d808089daa8173cad1756cf136 Mon Sep 17 00:00:00 2001 From: Fokko Date: Mon, 28 Apr 2025 09:52:48 +0200 Subject: [PATCH 3/3] Bump to Iceberg 1.9.0 --- dev/Dockerfile | 2 +- tests/conftest.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/dev/Dockerfile b/dev/Dockerfile index bb43ea1df8..b5b72e5617 100644 --- a/dev/Dockerfile +++ b/dev/Dockerfile @@ -39,7 +39,7 @@ WORKDIR ${SPARK_HOME} # Remember to also update `tests/conftest`'s spark setting ENV SPARK_VERSION=3.5.4 ENV ICEBERG_SPARK_RUNTIME_VERSION=3.5_2.12 -ENV ICEBERG_VERSION=1.8.0 +ENV ICEBERG_VERSION=1.9.0 ENV PYICEBERG_VERSION=0.9.0 RUN curl --retry 5 -s -C - https://archive.apache.org/dist/spark/spark-${SPARK_VERSION}/spark-${SPARK_VERSION}-bin-hadoop3.tgz -o spark-${SPARK_VERSION}-bin-hadoop3.tgz \ diff --git a/tests/conftest.py b/tests/conftest.py index 09f3a15d56..21fc6c7400 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -2434,7 +2434,7 @@ def spark() -> "SparkSession": # Remember to also update `dev/Dockerfile` spark_version = ".".join(importlib.metadata.version("pyspark").split(".")[:2]) scala_version = "2.12" - iceberg_version = "1.8.0" + iceberg_version = "1.9.0" os.environ["PYSPARK_SUBMIT_ARGS"] = ( f"--packages org.apache.iceberg:iceberg-spark-runtime-{spark_version}_{scala_version}:{iceberg_version},"