GH-527: [Release] Add support for .jar for Windows (#528)

Fixes GH-527.

It seems that arrow-testing with ARROW_BUILD_STATIC=ON and
GTest_SOURCE=BUNDLED is broken. We'll enable JNI test after we fix it in
apache/arrow.

Author: kou

.github/workflows/rc.yml

@@ -76,7 +76,7 @@ jobs:
           name: release-source
           path: |
             apache-arrow-java-*
-  jni-ubuntu:
+  jni-linux:
     name: JNI ${{ matrix.platform.runs_on }} ${{ matrix.platform.arch }}
     runs-on: ${{ matrix.platform.runs_on }}
     needs:
@@ -100,7 +100,7 @@ jobs:
         uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8
         with:
           name: release-source
-      - name: Extract Download the latest Apache Arrow C++
+      - name: Extract source archive
         run: |
           tar -xf apache-arrow-java-*.tar.gz --strip-components=1
       - name: Download the latest Apache Arrow C++
@@ -127,7 +127,7 @@ jobs:
           path: .docker
           key: jni-linux-${{ matrix.platform.arch }}-${{ hashFiles('arrow/cpp/**') }}
           restore-keys: jni-linux-${{ matrix.platform.arch }}-
-      - name: Build C++ libraries
+      - name: Build
         run: |
           docker compose run vcpkg-jni
       - name: Push Docker image
@@ -159,7 +159,7 @@ jobs:
         uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8
         with:
           name: release-source
-      - name: Extract Download the latest Apache Arrow C++
+      - name: Extract source archive
         run: |
           tar -xf apache-arrow-java-*.tar.gz --strip-components=1
       - name: Download the latest Apache Arrow C++
@@ -176,7 +176,7 @@ jobs:
           repository: apache/parquet-testing
           path: arrow/cpp/submodules/parquet-testing
       - name: Set up Python
-        uses: actions/setup-python@v5
+        uses: actions/setup-python@0b93645e9fea7318ecaed2b359559ac225c90a2b # v5.3.0
         with:
           cache: 'pip'
           python-version: 3.12
@@ -246,7 +246,7 @@ jobs:
           path: ccache
           key: jni-macos-${{ matrix.platform.arch }}-${{ hashFiles('arrow/cpp/**') }}
           restore-keys: jni-macos-${{ matrix.platform.arch }}-
-      - name: Build C++ libraries
+      - name: Build
         run: |
           set -e
           # make brew Java available to CMake
@@ -259,12 +259,83 @@ jobs:
         with:
           name: jni-macos-${{ matrix.platform.arch }}
           path: jni-macos-${{ matrix.platform.arch }}.tar.gz
+  jni-windows:
+    name: JNI ${{ matrix.platform.runs_on }} ${{ matrix.platform.arch }}
+    runs-on: ${{ matrix.platform.runs_on }}
+    needs:
+      - source
+    strategy:
+      fail-fast: false
+      matrix:
+        platform:
+          - runs_on: windows-2019
+            arch: "x86_64"
+    steps:
+      - name: Download source archive
+        uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8
+        with:
+          name: release-source
+      - name: Extract source archive
+        shell: bash
+        run: |
+          tar -xf apache-arrow-java-*.tar.gz --strip-components=1
+      - name: Download the latest Apache Arrow C++
+        shell: bash
+        run: |
+          ci/scripts/download_cpp.sh
+      - name: Set up Java
+        uses: actions/setup-java@7a6d8a8234af8eb26422e24e3006232cccaa061b # v4.6.0
+        with:
+          java-version: '11'
+          distribution: 'temurin'
+      - name: Download Timezone Database
+        shell: bash
+        run: |
+          arrow/ci/scripts/download_tz_database.sh
+      - name: Install ccache
+        shell: bash
+        run: |
+          env | sort
+          version=4.10.2
+          base_name="ccache-${version}-windows-x86_64"
+          url="https://github.com/ccache/ccache/releases/download/v${version}/${base_name}.zip"
+          curl --fail --location --remote-name "${url}"
+          unzip "${base_name}.zip"
+          chmod +x "${base_name}/ccache.exe"
+          mv "${base_name}/ccache.exe" /usr/bin/
+          rm -rf "${base_name}"{,.zip}
+      - name: Prepare ccache
+        shell: bash
+        run: |
+          echo "CCACHE_DIR=${PWD}/ccache" >> ${GITHUB_ENV}
+      - name: Cache ccache
+        uses: actions/cache@1bd1e32a3bdc45362d1e726936510720a7c30a57 # v4.2.0
+        with:
+          path: ccache
+          key: jni-windows-${{ matrix.platform.arch }}-${{ hashFiles('arrow/cpp/**') }}
+          restore-keys: jni-windows-${{ matrix.platform.arch }}-
+      - name: Build
+        shell: cmd
+        run: |
+          call "C:\Program Files (x86)\Microsoft Visual Studio\2019\Enterprise\VC\Auxiliary\Build\vcvarsall.bat" x64
+          REM For ORC
+          set TZDIR=/c/msys64/usr/share/zoneinfo
+          bash -c "ci/scripts/jni_windows_build.sh . arrow build jni"
+      - name: Compress into single artifact to keep directory structure
+        shell: bash
+        run: tar -cvzf jni-windows-${{ matrix.platform.arch }}.tar.gz jni/
+      - name: Upload artifacts
+        uses: actions/upload-artifact@65c4c4a1ddee5b72f698fdd19549f0f0fb45cf08 # v4.6.0
+        with:
+          name: jni-windows-${{ matrix.platform.arch }}
+          path: jni-windows-${{ matrix.platform.arch }}.tar.gz
   binaries:
     name: Binaries
     runs-on: ubuntu-latest
     needs:
-      - jni-ubuntu
+      - jni-linux
       - jni-macos
+      - jni-windows
     steps:
       - name: Download artifacts
         uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8
@@ -278,7 +349,7 @@ jobs:
           # tar -xvzf jni-linux-aarch_64.tar.gz
           tar -xvzf jni-macos-x86_64.tar.gz
           tar -xvzf jni-macos-aarch_64.tar.gz
-          # tar -xvzf jni-windows.tar.gz
+          tar -xvzf jni-windows-x86_64.tar.gz
       - name: Test that shared libraries exist
         run: |
           set -x
@@ -303,9 +374,9 @@ jobs:
           test -f jni/arrow_orc_jni/aarch_64/libarrow_orc_jni.dylib
           test -f jni/gandiva_jni/aarch_64/libgandiva_jni.dylib
 
-          # test -f jni/arrow_cdata_jni/x86_64/arrow_cdata_jni.dll
-          # test -f jni/arrow_dataset_jni/x86_64/arrow_dataset_jni.dll
-          # test -f jni/arrow_orc_jni/x86_64/arrow_orc_jni.dll
+          test -f jni/arrow_cdata_jni/x86_64/arrow_cdata_jni.dll
+          test -f jni/arrow_dataset_jni/x86_64/arrow_dataset_jni.dll
+          test -f jni/arrow_orc_jni/x86_64/arrow_orc_jni.dll
       - name: Checkout apache/arrow-testing
         uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
         with:

.pre-commit-config.yaml

@@ -47,6 +47,8 @@ repos:
     rev: v0.10.0
     hooks:
       - id: shellcheck
+        args:
+          - "--external-sources"
   - repo: https://github.com/scop/pre-commit-shfmt
     rev: v3.9.0-1
     hooks:

ci/scripts/download_cpp.sh

@@ -47,6 +47,6 @@ fi
 if [ -z "${url}" ]; then
   url="https://www.apache.org/dyn/closer.lua?action=download&filename=arrow/arrow-${version}/apache-arrow-${version}.tar.gz"
 fi
-wget --output-document "apache-arrow-${version}.tar.gz" "${url}"
+curl --location --output "apache-arrow-${version}.tar.gz" "${url}"
 tar xf "apache-arrow-${version}.tar.gz"
 mv "apache-arrow-${version}" arrow

ci/scripts/jni_build.sh

@@ -17,20 +17,25 @@
 # specific language governing permissions and limitations
 # under the License.
 
-set -euxo pipefail
+set -euo pipefail
 
+# shellcheck source=ci/scripts/util_log.sh
+. "$(dirname "${0}")/util_log.sh"
+
+github_actions_group_begin "Prepare arguments"
 source_dir=${1}
 arrow_install_dir=${2}
 build_dir=${3}/java_jni
 # The directory where the final binaries will be stored when scripts finish
 dist_dir=${4}
 prefix_dir="${build_dir}/java-jni"
+github_actions_group_end
 
-echo "=== Clear output directories and leftovers ==="
-# Clear output directories and leftovers
+github_actions_group_begin "Clear output directories and leftovers"
 rm -rf "${build_dir}"
+github_actions_group_end
 
-echo "=== Building Arrow Java C Data Interface native library ==="
+github_actions_group_begin "Building Arrow Java C Data Interface native library"
 
 case "$(uname)" in
 Linux)
@@ -71,10 +76,14 @@ if [ "${ARROW_JAVA_BUILD_TESTS}" = "ON" ]; then
 fi
 cmake --build "${build_dir}" --target install
 
+github_actions_group_end
+
+github_actions_group_begin "Copying artifacts"
 mkdir -p "${dist_dir}"
 # For Windows. *.dll are installed into bin/ on Windows.
 if [ -d "${prefix_dir}/bin" ]; then
   mv "${prefix_dir}"/bin/* "${dist_dir}"/
 else
   mv "${prefix_dir}"/lib/* "${dist_dir}"/
 fi
+github_actions_group_end

ci/scripts/jni_macos_build.sh

@@ -20,8 +20,12 @@
 # This script is like java_jni_build.sh, but is meant for release artifacts
 # and hardcodes assumptions about the environment it is being run in.
 
-set -ex
+set -euo pipefail
 
+# shellcheck source=ci/scripts/util_log.sh
+. "$(dirname "${0}")/util_log.sh"
+
+github_actions_group_begin "Prepare arguments"
 source_dir="$(cd "${1}" && pwd)"
 arrow_dir="$(cd "${2}" && pwd)"
 build_dir="${3}"
@@ -36,20 +40,29 @@ i386)
 esac
 # The directory where the final binaries will be stored when scripts finish
 dist_dir="${4}"
+github_actions_group_end
 
-echo "=== Clear output directories and leftovers ==="
-# Clear output directories and leftovers
+github_actions_group_begin "Clear output directories and leftovers"
 rm -rf "${build_dir}"
 rm -rf "${dist_dir}"
 
 mkdir -p "${build_dir}"
 build_dir="$(cd "${build_dir}" && pwd)"
+github_actions_group_end
+
+: "${ARROW_USE_CCACHE:=ON}"
+if [ "${ARROW_USE_CCACHE}" == "ON" ]; then
+  github_actions_group_begin "ccache statistics before build"
+  ccache -sv 2>/dev/null || ccache -s
+  github_actions_group_end
+fi
 
-echo "=== Building Arrow C++ libraries ==="
+github_actions_group_begin "Building Arrow C++ libraries"
 install_dir="${build_dir}/cpp-install"
 : "${ARROW_ACERO:=ON}"
 export ARROW_ACERO
 : "${ARROW_BUILD_TESTS:=ON}"
+export ARROW_BUILD_TESTS
 : "${ARROW_DATASET:=ON}"
 export ARROW_DATASET
 : "${ARROW_GANDIVA:=ON}"
@@ -58,15 +71,9 @@ export ARROW_GANDIVA
 export ARROW_ORC
 : "${ARROW_PARQUET:=ON}"
 : "${ARROW_S3:=ON}"
-: "${ARROW_USE_CCACHE:=ON}"
 : "${CMAKE_BUILD_TYPE:=Release}"
 : "${CMAKE_UNITY_BUILD:=ON}"
 
-if [ "${ARROW_USE_CCACHE}" == "ON" ]; then
-  echo "=== ccache statistics before build ==="
-  ccache -sv 2>/dev/null || ccache -s
-fi
-
 export ARROW_TEST_DATA="${arrow_dir}/testing/data"
 export PARQUET_TEST_DATA="${arrow_dir}/cpp/submodules/parquet-testing/data"
 export AWS_EC2_METADATA_DISABLED=TRUE
@@ -99,8 +106,10 @@ cmake \
   -Dre2_SOURCE=BUNDLED \
   -GNinja
 cmake --build "${build_dir}/cpp" --target install
+github_actions_group_end
 
 if [ "${ARROW_RUN_TESTS:-}" == "ON" ]; then
+  github_actions_group_begin "Running Arrow C++ libraries tests"
   # MinIO is required
   exclude_tests="arrow-s3fs-test"
   # unstable
@@ -113,6 +122,7 @@ if [ "${ARROW_RUN_TESTS:-}" == "ON" ]; then
     --parallel "$(sysctl -n hw.ncpu)" \
     --test-dir "${build_dir}/cpp" \
     --timeout 300
+  github_actions_group_end
 fi
 
 export JAVA_JNI_CMAKE_ARGS="-DProtobuf_ROOT=${build_dir}/cpp/protobuf_ep-install"
@@ -123,11 +133,12 @@ export JAVA_JNI_CMAKE_ARGS="-DProtobuf_ROOT=${build_dir}/cpp/protobuf_ep-install
   "${dist_dir}"
 
 if [ "${ARROW_USE_CCACHE}" == "ON" ]; then
-  echo "=== ccache statistics after build ==="
+  github_actions_group_begin "ccache statistics after build"
   ccache -sv 2>/dev/null || ccache -s
+  github_actions_group_end
 fi
 
-echo "=== Checking shared dependencies for libraries ==="
+github_actions_group_begin "Checking shared dependencies for libraries"
 pushd "${dist_dir}"
 archery linking check-dependencies \
   --allow CoreFoundation \
@@ -147,3 +158,4 @@ archery linking check-dependencies \
   "arrow_orc_jni/${normalized_arch}/libarrow_orc_jni.dylib" \
   "gandiva_jni/${normalized_arch}/libgandiva_jni.dylib"
 popd
+github_actions_group_end