chore: Comet + Iceberg (1.8.1) CI

.github/actions/setup-iceberg-builder/action.yaml

@@ -0,0 +1,50 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+name: Setup Iceberg Builder
+description: 'Setup Apache Iceberg to run Spark SQL tests'
+inputs:
+  iceberg-version:
+    description: 'The Apache Iceberg version (e.g., 1.8.1) to build'
+    required: true
+  scala-version:
+    description: 'The Scala short version (e.g., 2.13) to build'
+    required: true
+  spark-short-version:
+    description: 'The Apache Spark short version (e.g., 3.5) to build'
+    required: true
+runs:
+  using: "composite"
+  steps:
+    - name: Clone Iceberg repo
+      uses: actions/checkout@v4
+      with:
+        repository: apache/iceberg
+        path: apache-iceberg
+        ref: apache-iceberg-${{inputs.iceberg-version}}
+        fetch-depth: 1
+
+    - name: Setup Iceberg for Comet
+      shell: bash
+      run: |
+        cd apache-iceberg
+        git apply ../dev/diffs/iceberg/${{inputs.iceberg-version}}.diff
+
+    - name: Build Comet
+      shell: bash
+      run: |
+        PROFILES="-Pspark-${{inputs.spark-short-version}} -Pscala-${{inputs.scala-version}}" make release

.github/actions/setup-spark-local-jar/action.yaml

@@ -0,0 +1,48 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+name: Setup Spark Local Jar
+description: 'Build comet-patched Apache Spark for Iceberg Spark tests'
+inputs:
+  spark-short-version:
+    description: 'The Apache Spark short version (e.g., 3.5) to build'
+    required: true
+  spark-version:
+    description: 'The Apache Spark version (e.g., 3.5.6) to build'
+    required: true
+  scala-version:
+    description: 'The Scala short version (e.g., 2.13) to build'
+    required: true
+runs:
+  using: "composite"
+  steps:
+    - name: Clone Spark repo
+      uses: actions/checkout@v4
+      with:
+        repository: apache/spark
+        path: apache-spark
+        ref: v${{inputs.spark-version}}
+        fetch-depth: 1
+
+    - name: Publish local Spark snapshot w/ Comet
+      shell: bash
+      run: |
+        cd apache-spark
+        git apply ../dev/diffs/${{inputs.spark-version}}.diff
+        ./dev/change-scala-version.sh ${{inputs.scala-version}}
+        ./build/mvn versions:set -DnewVersion=${{inputs.spark-version}}-SNAPSHOT
+        ./build/mvn -Pscala-${{inputs.scala-version}} -Phive -Phive-thriftserver -DskipTests -Denforcer.skip=true clean install

.github/workflows/iceberg_spark_test.yml

@@ -0,0 +1,85 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+name: Iceberg Spark SQL Tests
+
+concurrency:
+  group: ${{ github.repository }}-${{ github.head_ref || github.sha }}-${{ github.workflow }}
+  cancel-in-progress: true
+
+on:
+  push:
+    paths-ignore:
+      - "doc/**"
+      - "docs/**"
+      - "**.md"
+  pull_request:
+    paths-ignore:
+      - "doc/**"
+      - "docs/**"
+      - "**.md"
+  # manual trigger
+  # https://docs.github.com/en/actions/managing-workflow-runs/manually-running-a-workflow
+  workflow_dispatch:
+
+env:
+  RUST_VERSION: stable
+
+jobs:
+  iceberg-spark-sql:
+    strategy:
+      matrix:
+        os: [ubuntu-24.04]
+        java-version: [11, 17]
+        iceberg-version: [{short: '1.8', full: '1.8.1'}]
+        spark-version: [{short: '3.5', full: '3.5.6'}]
+        scala-version: ['2.13']
+      fail-fast: false
+    name: iceberg-spark-sql/${{ matrix.os }}/iceberg-${{ matrix.iceberg-version.full }}/spark-${{ matrix.spark-version.full }}/scala-${{ matrix.scala-version }}/java-${{ matrix.java-version }}
+    runs-on: ${{ matrix.os }}
+    container:
+      image: amd64/rust
+    env:
+      SPARK_LOCAL_IP: localhost
+    steps:
+      - uses: actions/checkout@v4
+      - name: Setup Rust & Java toolchain
+        uses: ./.github/actions/setup-builder
+        with:
+          rust-version: ${{env.RUST_VERSION}}
+          jdk-version: ${{ matrix.java-version }}
+      - name: Setup Iceberg
+        uses: ./.github/actions/setup-iceberg-builder
+        with:
+          iceberg-version: ${{ matrix.iceberg-version.full }}
+          scala-version: ${{ matrix.scala-version }}
+          spark-short-version: ${{ matrix.spark-version.short }}
+      - name: Build local Spark jar with comet patch
+        uses: ./.github/actions/setup-spark-local-jar
+        with:
+          spark-short-version: ${{ matrix.spark-version.short }}
+          spark-version: ${{ matrix.spark-version.full }}
+          scala-version: ${{ matrix.scala-version }}
+      - name: Run Iceberg Spark tests
+        run: |
+          cd apache-iceberg
+          rm -rf /root/.m2/repository/org/apache/parquet # somehow parquet cache requires cleanups
+          ENABLE_COMET=true ./gradlew -DsparkVersions=${{ matrix.spark-version.short }} -DscalaVersion=${{ matrix.scala-version }} -DflinkVersions= -DkafkaVersions= \
+            :iceberg-spark:iceberg-spark-${{ matrix.spark-version.short }}_${{ matrix.scala-version }}:check \
+            :iceberg-spark:iceberg-spark-extensions-${{ matrix.spark-version.short }}_${{ matrix.scala-version }}:check \
+            :iceberg-spark:iceberg-spark-runtime-${{ matrix.spark-version.short }}_${{ matrix.scala-version }}:check \
+            -Pquick=true -x javadoc