From 2ef3bef5bf19c38feae3ed7bdf13aae39a4e107d Mon Sep 17 00:00:00 2001 From: Kevin Liu Date: Tue, 18 Mar 2025 12:28:32 -0700 Subject: [PATCH] 1.7.2rc3 --- dev/Dockerfile | 7 ++++--- tests/conftest.py | 3 ++- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/dev/Dockerfile b/dev/Dockerfile index bb43ea1df8..75a17e1a13 100644 --- a/dev/Dockerfile +++ b/dev/Dockerfile @@ -39,20 +39,21 @@ WORKDIR ${SPARK_HOME} # Remember to also update `tests/conftest`'s spark setting ENV SPARK_VERSION=3.5.4 ENV ICEBERG_SPARK_RUNTIME_VERSION=3.5_2.12 -ENV ICEBERG_VERSION=1.8.0 +ENV ICEBERG_VERSION=1.7.2 ENV PYICEBERG_VERSION=0.9.0 +ENV BASE_ARTIFACT_URL=https://repository.apache.org/content/repositories/orgapacheiceberg-1187 RUN curl --retry 5 -s -C - https://archive.apache.org/dist/spark/spark-${SPARK_VERSION}/spark-${SPARK_VERSION}-bin-hadoop3.tgz -o spark-${SPARK_VERSION}-bin-hadoop3.tgz \ && tar xzf spark-${SPARK_VERSION}-bin-hadoop3.tgz --directory /opt/spark --strip-components 1 \ && rm -rf spark-${SPARK_VERSION}-bin-hadoop3.tgz # Download iceberg spark runtime -RUN curl --retry 5 -s https://repo1.maven.org/maven2/org/apache/iceberg/iceberg-spark-runtime-${ICEBERG_SPARK_RUNTIME_VERSION}/${ICEBERG_VERSION}/iceberg-spark-runtime-${ICEBERG_SPARK_RUNTIME_VERSION}-${ICEBERG_VERSION}.jar \ +RUN curl --retry 5 -s ${BASE_ARTIFACT_URL}/org/apache/iceberg/iceberg-spark-runtime-${ICEBERG_SPARK_RUNTIME_VERSION}/${ICEBERG_VERSION}/iceberg-spark-runtime-${ICEBERG_SPARK_RUNTIME_VERSION}-${ICEBERG_VERSION}.jar \ -Lo /opt/spark/jars/iceberg-spark-runtime-${ICEBERG_SPARK_RUNTIME_VERSION}-${ICEBERG_VERSION}.jar # Download AWS bundle -RUN curl --retry 5 -s https://repo1.maven.org/maven2/org/apache/iceberg/iceberg-aws-bundle/${ICEBERG_VERSION}/iceberg-aws-bundle-${ICEBERG_VERSION}.jar \ +RUN curl --retry 5 -s ${BASE_ARTIFACT_URL}/org/apache/iceberg/iceberg-aws-bundle/${ICEBERG_VERSION}/iceberg-aws-bundle-${ICEBERG_VERSION}.jar \ -Lo /opt/spark/jars/iceberg-aws-bundle-${ICEBERG_VERSION}.jar COPY spark-defaults.conf /opt/spark/conf diff --git a/tests/conftest.py b/tests/conftest.py index 1cbb3cfa4d..b0aa2968b6 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -2420,7 +2420,7 @@ def spark() -> "SparkSession": # Remember to also update `dev/Dockerfile` spark_version = ".".join(importlib.metadata.version("pyspark").split(".")[:2]) scala_version = "2.12" - iceberg_version = "1.8.0" + iceberg_version = "1.7.2" os.environ["PYSPARK_SUBMIT_ARGS"] = ( f"--packages org.apache.iceberg:iceberg-spark-runtime-{spark_version}_{scala_version}:{iceberg_version}," @@ -2432,6 +2432,7 @@ def spark() -> "SparkSession": spark = ( SparkSession.builder.appName("PyIceberg integration test") + .config("spark.jars.repositories", "https://repository.apache.org/content/repositories/orgapacheiceberg-1187/") .config("spark.sql.session.timeZone", "UTC") .config("spark.sql.shuffle.partitions", "1") .config("spark.default.parallelism", "1")