Run Iceberg Spark test w/ comet-enabled Spark

Author: hsiang-c

.github/actions/setup-spark-local-jar/action.yaml

@@ -0,0 +1,50 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+name: Setup Spark Local Jar
+description: 'Build comet-patched Apache Spark for Iceberg Spark tests'
+inputs:
+  spark-short-version:
+    description: 'The Apache Spark short version (e.g., 3.5) to build'
+    required: true
+  spark-version:
+    description: 'The Apache Spark version (e.g., 3.5.6) to build'
+    required: true
+  scala-version:
+    description: 'The Scala short version (e.g., 2.13) to build'
+    required: true
+runs:
+  using: "composite"
+  steps:
+    - name: Clone Spark repo
+      uses: actions/checkout@v4
+      with:
+        repository: apache/spark
+        path: apache-spark
+        ref: v${{inputs.spark-version}}
+        fetch-depth: 1
+
+    - name: Publish local Spark snapshot w/ Comet
+      shell: bash
+      run: |
+        cd apache-spark
+        git apply ../dev/diffs/${{inputs.spark-version}}.diff
+# https://spark.apache.org/docs/3.5.0/building-spark.html#change-scala-version
+        ./dev/change-scala-version.sh ${{inputs.scala-version}
+        ./build/mvn versions:set -DnewVersion=${{inputs.spark-version}}-SNAPSHOT
+# Might need to skip enforcer b/c comet is snapshot
+        ./build/mvn -Pscala-${{inputs.scala-version}} -Phive -Phive-thriftserver -DskipTests -Denforcer.skip=true clean install

.github/workflows/iceberg_spark_test.yml

@@ -46,8 +46,8 @@ jobs:
         os: [ubuntu-24.04]
         java-version: [11, 17]
         iceberg-version: [{short: '1.8', full: '1.8.1'}]
-        spark-version: [{short: '3.5', full: '3.5.4'}]
-        scala-version: [ '2.12', '2.13']
+        spark-version: [{short: '3.5', full: '3.5.6'}]
+        scala-version: ['2.13']
       fail-fast: false
     name: iceberg-spark-sql/${{ matrix.os }}/iceberg-${{ matrix.iceberg-version.full }}/spark-${{ matrix.spark-version.full }}/scala-${{ matrix.scala-version }}/java-${{ matrix.java-version }}
     runs-on: ${{ matrix.os }}
@@ -62,6 +62,12 @@ jobs:
         with:
           rust-version: ${{env.RUST_VERSION}}
           jdk-version: ${{ matrix.java-version }}
+      - name: Build local Spark jar with comet patch
+        uses: ./.github/actions/setup-spark-local-jar
+        with:
+          spark-version: ${{ matrix.spark-version.full }}
+          spark-short-version: ${{ matrix.spark-version.short }}
+          scala-version: ${{ matrix.scala-version }}
       - name: Setup Iceberg
         uses: ./.github/actions/setup-iceberg-builder
         with:
@@ -73,7 +79,7 @@ jobs:
         run: |
           cd apache-iceberg
           rm -rf /root/.m2/repository/org/apache/parquet # somehow parquet cache requires cleanups
-          ./gradlew -DsparkVersions=${{ matrix.spark-version.short }} -DscalaVersion=${{ matrix.scala-version }} -DflinkVersions= -DkafkaVersions= \
+          ENABLE_COMET=true ./gradlew -DsparkVersions=${{ matrix.spark-version.short }} -DscalaVersion=${{ matrix.scala-version }} -DflinkVersions= -DkafkaVersions= \
             :iceberg-spark:iceberg-spark-${{ matrix.spark-version.short }}_${{ matrix.scala-version }}:check \
             :iceberg-spark:iceberg-spark-extensions-${{ matrix.spark-version.short }}_${{ matrix.scala-version }}:check \
             :iceberg-spark:iceberg-spark-runtime-${{ matrix.spark-version.short }}_${{ matrix.scala-version }}:check \

dev/diffs/iceberg/1.8.1.diff

@@ -1,3 +1,16 @@
+diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml
+index 04ffa8f..d4107be 100644
+--- a/gradle/libs.versions.toml
++++ b/gradle/libs.versions.toml
+@@ -81,7 +81,7 @@ slf4j = "2.0.16"
+ snowflake-jdbc = "3.22.0"
+ spark-hive33 = "3.3.4"
+ spark-hive34 = "3.4.4"
+-spark-hive35 = "3.5.4"
++spark-hive35 = "3.5.6-SNAPSHOT"
+ sqlite-jdbc = "3.48.0.0"
+ testcontainers = "1.20.4"
+ tez010 = "0.10.4"
 diff --git a/spark/v3.4/build.gradle b/spark/v3.4/build.gradle
 index 6eb26e8..c288e72 100644
 --- a/spark/v3.4/build.gradle