From 4fe0dc8de2ef0bad099007048baa3b457930d0c4 Mon Sep 17 00:00:00 2001 From: "carter.fendley" Date: Mon, 14 Apr 2025 13:27:35 -0400 Subject: [PATCH 1/3] [SPARK-51795][BUILD] Bump Parquet 1.15.1 --- dev/deps/spark-deps-hadoop-2-hive-2.3 | 12 ++++++------ dev/deps/spark-deps-hadoop-3-hive-2.3 | 12 ++++++------ pom.xml | 2 +- 3 files changed, 13 insertions(+), 13 deletions(-) diff --git a/dev/deps/spark-deps-hadoop-2-hive-2.3 b/dev/deps/spark-deps-hadoop-2-hive-2.3 index 1df44c045336..003a4b0b9edb 100644 --- a/dev/deps/spark-deps-hadoop-2-hive-2.3 +++ b/dev/deps/spark-deps-hadoop-2-hive-2.3 @@ -228,12 +228,12 @@ orc-shims/1.8.7//orc-shims-1.8.7.jar oro/2.0.8//oro-2.0.8.jar osgi-resource-locator/1.0.3//osgi-resource-locator-1.0.3.jar paranamer/2.8//paranamer-2.8.jar -parquet-column/1.12.3//parquet-column-1.12.3.jar -parquet-common/1.12.3//parquet-common-1.12.3.jar -parquet-encoding/1.12.3//parquet-encoding-1.12.3.jar -parquet-format-structures/1.12.3//parquet-format-structures-1.12.3.jar -parquet-hadoop/1.12.3//parquet-hadoop-1.12.3.jar -parquet-jackson/1.12.3//parquet-jackson-1.12.3.jar +parquet-column/1.15.1//parquet-column-1.15.1.jar +parquet-common/1.15.1//parquet-common-1.15.1.jar +parquet-encoding/1.15.1//parquet-encoding-1.15.1.jar +parquet-format-structures/1.15.1//parquet-format-structures-1.15.1.jar +parquet-hadoop/1.15.1//parquet-hadoop-1.15.1.jar +parquet-jackson/1.15.1//parquet-jackson-1.15.1.jar pickle/1.3//pickle-1.3.jar protobuf-java/2.5.0//protobuf-java-2.5.0.jar py4j/0.10.9.7//py4j-0.10.9.7.jar diff --git a/dev/deps/spark-deps-hadoop-3-hive-2.3 b/dev/deps/spark-deps-hadoop-3-hive-2.3 index f0785806d564..1dbc03f6d9e3 100644 --- a/dev/deps/spark-deps-hadoop-3-hive-2.3 +++ b/dev/deps/spark-deps-hadoop-3-hive-2.3 @@ -215,12 +215,12 @@ orc-shims/1.8.7//orc-shims-1.8.7.jar oro/2.0.8//oro-2.0.8.jar osgi-resource-locator/1.0.3//osgi-resource-locator-1.0.3.jar paranamer/2.8//paranamer-2.8.jar -parquet-column/1.12.3//parquet-column-1.12.3.jar -parquet-common/1.12.3//parquet-common-1.12.3.jar -parquet-encoding/1.12.3//parquet-encoding-1.12.3.jar -parquet-format-structures/1.12.3//parquet-format-structures-1.12.3.jar -parquet-hadoop/1.12.3//parquet-hadoop-1.12.3.jar -parquet-jackson/1.12.3//parquet-jackson-1.12.3.jar +parquet-column/1.15.1//parquet-column-1.15.1.jar +parquet-common/1.15.1//parquet-common-1.15.1.jar +parquet-encoding/1.15.1//parquet-encoding-1.15.1.jar +parquet-format-structures/1.15.1//parquet-format-structures-1.15.1.jar +parquet-hadoop/1.15.1//parquet-hadoop-1.15.1.jar +parquet-jackson/1.15.1//parquet-jackson-1.15.1.jar pickle/1.3//pickle-1.3.jar protobuf-java/2.5.0//protobuf-java-2.5.0.jar py4j/0.10.9.7//py4j-0.10.9.7.jar diff --git a/pom.xml b/pom.xml index 5ade8019b1e7..657fc18c999b 100644 --- a/pom.xml +++ b/pom.xml @@ -140,7 +140,7 @@ 3.3.2 10.14.2.0 - 1.12.3 + 1.15.1 1.8.7 shaded-protobuf 9.4.54.v20240208 From b4ffce9dc3631ccc76ab2d88dd70e2ac15272822 Mon Sep 17 00:00:00 2001 From: "carter.fendley" Date: Mon, 14 Apr 2025 16:37:02 -0400 Subject: [PATCH 2/3] Bump upload-artifact to v4 version GHA failure due to deprecation --- .github/workflows/benchmark.yml | 2 +- .github/workflows/build_and_test.yml | 20 ++++++++++---------- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml index 8671cff054bb..814f243a50dc 100644 --- a/.github/workflows/benchmark.yml +++ b/.github/workflows/benchmark.yml @@ -188,7 +188,7 @@ jobs: echo "Preparing the benchmark results:" tar -cvf benchmark-results-${{ github.event.inputs.jdk }}-${{ github.event.inputs.scala }}.tar `git diff --name-only` `git ls-files --others --exclude=tpcds-sf-1 --exclude-standard` - name: Upload benchmark results - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: name: benchmark-results-${{ github.event.inputs.jdk }}-${{ github.event.inputs.scala }}-${{ matrix.split }} path: benchmark-results-${{ github.event.inputs.jdk }}-${{ github.event.inputs.scala }}.tar diff --git a/.github/workflows/build_and_test.yml b/.github/workflows/build_and_test.yml index 3e44d6cfd179..b00645919376 100644 --- a/.github/workflows/build_and_test.yml +++ b/.github/workflows/build_and_test.yml @@ -274,13 +274,13 @@ jobs: ./dev/run-tests --parallelism 1 --modules "$MODULES_TO_TEST" --included-tags "$INCLUDED_TAGS" --excluded-tags "$EXCLUDED_TAGS" - name: Upload test results to report if: always() - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: name: test-results-${{ matrix.modules }}-${{ matrix.comment }}-${{ matrix.java }}-${{ matrix.hadoop }}-${{ matrix.hive }} path: "**/target/test-reports/*.xml" - name: Upload unit tests log files if: failure() - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: name: unit-tests-log-${{ matrix.modules }}-${{ matrix.comment }}-${{ matrix.java }}-${{ matrix.hadoop }}-${{ matrix.hive }} path: "**/target/unit-tests.log" @@ -452,13 +452,13 @@ jobs: name: PySpark - name: Upload test results to report if: always() - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: name: test-results-${{ matrix.modules }}--8-${{ inputs.hadoop }}-hive2.3 path: "**/target/test-reports/*.xml" - name: Upload unit tests log files if: failure() - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: name: unit-tests-log-${{ matrix.modules }}--8-${{ inputs.hadoop }}-hive2.3 path: "**/target/unit-tests.log" @@ -536,7 +536,7 @@ jobs: ./dev/run-tests --parallelism 1 --modules sparkr - name: Upload test results to report if: always() - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: name: test-results-sparkr--8-${{ inputs.hadoop }}-hive2.3 path: "**/target/test-reports/*.xml" @@ -873,13 +873,13 @@ jobs: spark.sql.join.forceApplyShuffledHashJoin=true - name: Upload test results to report if: always() - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: name: test-results-tpcds--8-${{ inputs.hadoop }}-hive2.3 path: "**/target/test-reports/*.xml" - name: Upload unit tests log files if: failure() - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: name: unit-tests-log-tpcds--8-${{ inputs.hadoop }}-hive2.3 path: "**/target/unit-tests.log" @@ -939,13 +939,13 @@ jobs: ./dev/run-tests --parallelism 1 --modules docker-integration-tests --included-tags org.apache.spark.tags.DockerTest - name: Upload test results to report if: always() - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: name: test-results-docker-integration--8-${{ inputs.hadoop }}-hive2.3 path: "**/target/test-reports/*.xml" - name: Upload unit tests log files if: failure() - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: name: unit-tests-log-docker-integration--8-${{ inputs.hadoop }}-hive2.3 path: "**/target/unit-tests.log" @@ -1016,7 +1016,7 @@ jobs: build/sbt -Psparkr -Pkubernetes -Pvolcano -Pkubernetes-integration-tests -Dspark.kubernetes.test.volcanoMaxConcurrencyJobNum=1 -Dtest.exclude.tags=local "kubernetes-integration-tests/test" - name: Upload Spark on K8S integration tests log files if: failure() - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: name: spark-on-kubernetes-it-log path: "**/target/integration-tests.log" From 2f592ab2aa8c49aa86e8df04fa1d6af2df74253a Mon Sep 17 00:00:00 2001 From: "carter.fendley" Date: Mon, 14 Apr 2025 19:41:45 -0400 Subject: [PATCH 3/3] Add exclusion for h2 to prevent error --- pom.xml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/pom.xml b/pom.xml index 657fc18c999b..70de57e6493b 100644 --- a/pom.xml +++ b/pom.xml @@ -2638,6 +2638,12 @@ ${parquet.version} ${parquet.test.deps.scope} tests + + + com.h2database + h2 + + org.apache.parquet