From 8fd09cd8ddb5b370cbb3113e754e24ded7f0db2a Mon Sep 17 00:00:00 2001 From: David Li Date: Mon, 2 Dec 2024 03:17:00 -0500 Subject: [PATCH 1/6] GH-14: Add JNI CI test Fixes #14. --- .github/workflows/test.yml | 2 +- ci/docker/conda-java-jni.dockerfile | 30 +++++++++++ ci/scripts/java_jni_build.sh | 80 +++++++++++++++++++++++++++++ docker-compose.yml | 34 +++++++++++- 4 files changed, 143 insertions(+), 3 deletions(-) create mode 100644 ci/docker/conda-java-jni.dockerfile create mode 100755 ci/scripts/java_jni_build.sh diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 473ce84ca7..13a3b4a63a 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -47,7 +47,7 @@ jobs: matrix: jdk: [11, 17, 21, 22] maven: [3.9.6] - image: [java] + image: [java, conda-java-jni-cdata] env: JDK: ${{ matrix.jdk }} MAVEN: ${{ matrix.maven }} diff --git a/ci/docker/conda-java-jni.dockerfile b/ci/docker/conda-java-jni.dockerfile new file mode 100644 index 0000000000..0cb997c14a --- /dev/null +++ b/ci/docker/conda-java-jni.dockerfile @@ -0,0 +1,30 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +FROM ghcr.io/mamba-org/micromamba:ubuntu24.04 + +ARG jdk=11 +ARG maven=3.8.7 + +RUN micromamba install -y \ + -c conda-forge \ + cmake \ + compilers \ + maven=${maven} \ + ninja \ + openjdk=${jdk} && \ + micromamba clean --all diff --git a/ci/scripts/java_jni_build.sh b/ci/scripts/java_jni_build.sh new file mode 100755 index 0000000000..99d8cb06e3 --- /dev/null +++ b/ci/scripts/java_jni_build.sh @@ -0,0 +1,80 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +set -eo pipefail + +arrow_dir=${1} +arrow_install_dir=${2} +build_dir=${3}/java_jni +# The directory where the final binaries will be stored when scripts finish +dist_dir=${4} +prefix_dir="${build_dir}/java-jni" + +echo "=== Clear output directories and leftovers ===" +# Clear output directories and leftovers +rm -rf ${build_dir} + +echo "=== Building Arrow Java C Data Interface native library ===" +mkdir -p "${build_dir}" +pushd "${build_dir}" + +case "$(uname)" in + Linux) + n_jobs=$(nproc) + ;; + Darwin) + n_jobs=$(sysctl -n hw.ncpu) + ;; + *) + n_jobs=${NPROC:-1} + ;; +esac + +: ${ARROW_JAVA_BUILD_TESTS:=${ARROW_BUILD_TESTS:-OFF}} +: ${CMAKE_BUILD_TYPE:=release} +cmake \ + -DARROW_JAVA_JNI_ENABLE_DATASET=${ARROW_DATASET:-OFF} \ + -DARROW_JAVA_JNI_ENABLE_GANDIVA=${ARROW_GANDIVA:-OFF} \ + -DARROW_JAVA_JNI_ENABLE_ORC=${ARROW_ORC:-OFF} \ + -DBUILD_TESTING=${ARROW_JAVA_BUILD_TESTS} \ + -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE} \ + -DCMAKE_PREFIX_PATH=${arrow_install_dir} \ + -DCMAKE_INSTALL_PREFIX=${prefix_dir} \ + -DCMAKE_UNITY_BUILD=${CMAKE_UNITY_BUILD:-OFF} \ + -DProtobuf_USE_STATIC_LIBS=ON \ + -GNinja \ + ${JAVA_JNI_CMAKE_ARGS:-} \ + ${arrow_dir} +export CMAKE_BUILD_PARALLEL_LEVEL=${n_jobs} +cmake --build . --config ${CMAKE_BUILD_TYPE} +if [ "${ARROW_JAVA_BUILD_TESTS}" = "ON" ]; then + ctest \ + --output-on-failure \ + --parallel ${n_jobs} \ + --timeout 300 +fi +cmake --build . --config ${CMAKE_BUILD_TYPE} --target install +popd + +mkdir -p ${dist_dir} +# For Windows. *.dll are installed into bin/ on Windows. +if [ -d "${prefix_dir}/bin" ]; then + mv ${prefix_dir}/bin/* ${dist_dir}/ +else + mv ${prefix_dir}/lib/* ${dist_dir}/ +fi diff --git a/docker-compose.yml b/docker-compose.yml index 103f2f3ad0..871c0d1387 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -38,10 +38,40 @@ services: # MAVEN: 3.9.6 # JDK: 11, 17, 21 image: ${ARCH}/maven:${MAVEN}-eclipse-temurin-${JDK} - volumes: &java-volumes + volumes: - .:/arrow-java:delegated - ${DOCKER_VOLUME_PREFIX}maven-cache:/root/.m2:delegated - command: &java-command > + command: /bin/bash -c " /arrow-java/ci/scripts/java_build.sh /arrow-java /build && /arrow-java/ci/scripts/java_test.sh /arrow-java /build" + + conda-java-jni-cdata: + # Usage: + # docker compose build java-jni-cdata + # docker compose run java-jni-cdata + # Parameters: + # MAVEN: 3.9.6 + # JDK: 11, 17, 21 + image: ${REPO}:${ARCH}-conda-java-${JDK}-maven-${MAVEN}-jni-integration + build: + context: . + dockerfile: ci/docker/conda-java-jni.dockerfile + cache_from: + - ${REPO}:${ARCH}-conda-java-${JDK}-maven-${MAVEN}-jni-integration + args: + jdk: ${JDK} + maven: ${MAVEN} + # required to use micromamba with rootless docker + # https://github.com/mamba-org/micromamba-docker/issues/407#issuecomment-2088523507 + user: root + volumes: + - .:/arrow-java:delegated + - ${DOCKER_VOLUME_PREFIX}maven-cache:/root/.m2:delegated + environment: + ARROW_JAVA_CDATA: "ON" + command: + /bin/bash -c " + /arrow-java/ci/scripts/java_jni_build.sh /arrow-java /build/jni /build /jni && + /arrow-java/ci/scripts/java_build.sh /arrow-java /build /jni && + /arrow-java/ci/scripts/java_test.sh /arrow-java /build /jni" From cba8272396e09f58d270238952eac1b7e995e816 Mon Sep 17 00:00:00 2001 From: David Li Date: Mon, 2 Dec 2024 04:20:19 -0500 Subject: [PATCH 2/6] use 3.9.9 --- .env | 2 +- .github/workflows/test.yml | 2 +- docker-compose.yml | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.env b/.env index bb9f63ab30..b50a16eb01 100644 --- a/.env +++ b/.env @@ -47,4 +47,4 @@ ULIMIT_CORE=-1 # Default versions for various dependencies JDK=11 -MAVEN=3.9.6 +MAVEN=3.9.9 diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 13a3b4a63a..7b6d410043 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -46,7 +46,7 @@ jobs: fail-fast: false matrix: jdk: [11, 17, 21, 22] - maven: [3.9.6] + maven: [3.9.9] image: [java, conda-java-jni-cdata] env: JDK: ${{ matrix.jdk }} diff --git a/docker-compose.yml b/docker-compose.yml index 871c0d1387..06226bad80 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -35,7 +35,7 @@ services: # docker compose build java # docker compose run java # Parameters: - # MAVEN: 3.9.6 + # MAVEN: 3.9.9 # JDK: 11, 17, 21 image: ${ARCH}/maven:${MAVEN}-eclipse-temurin-${JDK} volumes: @@ -51,7 +51,7 @@ services: # docker compose build java-jni-cdata # docker compose run java-jni-cdata # Parameters: - # MAVEN: 3.9.6 + # MAVEN: 3.9.9 # JDK: 11, 17, 21 image: ${REPO}:${ARCH}-conda-java-${JDK}-maven-${MAVEN}-jni-integration build: From 5fcfca477f9d7c9a8def263956477201c68591ef Mon Sep 17 00:00:00 2001 From: David Li Date: Tue, 3 Dec 2024 03:37:08 -0500 Subject: [PATCH 3/6] Update ci/docker/conda-java-jni.dockerfile --- ci/docker/conda-java-jni.dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/docker/conda-java-jni.dockerfile b/ci/docker/conda-java-jni.dockerfile index 0cb997c14a..e14db73688 100644 --- a/ci/docker/conda-java-jni.dockerfile +++ b/ci/docker/conda-java-jni.dockerfile @@ -18,7 +18,7 @@ FROM ghcr.io/mamba-org/micromamba:ubuntu24.04 ARG jdk=11 -ARG maven=3.8.7 +ARG maven=3.9.9 RUN micromamba install -y \ -c conda-forge \ From 99014d88df6e2d3347e95a4e13bc571eb4450e49 Mon Sep 17 00:00:00 2001 From: David Li Date: Tue, 3 Dec 2024 03:40:42 -0500 Subject: [PATCH 4/6] name? --- .github/workflows/test.yml | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 7b6d410043..2db39b22ed 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -38,7 +38,7 @@ env: jobs: ubuntu: - name: AMD64 Ubuntu 22.04 JDK ${{ matrix.jdk }} Maven ${{ matrix.maven }} + name: ${{ matrix.name }} AMD64 Ubuntu 22.04 JDK ${{ matrix.jdk }} Maven ${{ matrix.maven }} runs-on: ubuntu-latest if: ${{ !contains(github.event.pull_request.title, 'WIP') }} timeout-minutes: 30 @@ -48,6 +48,11 @@ jobs: jdk: [11, 17, 21, 22] maven: [3.9.9] image: [java, conda-java-jni-cdata] + include: + - image: java + name: "" + - image: conda-java-jni-cdata + name: JNI env: JDK: ${{ matrix.jdk }} MAVEN: ${{ matrix.maven }} From 4b6e284bdc78339c27e2441d7f06f980acf8aaf1 Mon Sep 17 00:00:00 2001 From: David Li Date: Tue, 3 Dec 2024 19:42:05 -0500 Subject: [PATCH 5/6] feedback --- .github/workflows/test.yml | 6 +++--- .../{conda-java-jni.dockerfile => conda-jni.dockerfile} | 0 ci/scripts/java_jni_build.sh | 2 +- docker-compose.yml | 8 ++++---- 4 files changed, 8 insertions(+), 8 deletions(-) rename ci/docker/{conda-java-jni.dockerfile => conda-jni.dockerfile} (100%) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 2db39b22ed..cb47c21d11 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -38,7 +38,7 @@ env: jobs: ubuntu: - name: ${{ matrix.name }} AMD64 Ubuntu 22.04 JDK ${{ matrix.jdk }} Maven ${{ matrix.maven }} + name: ${{ matrix.name }} AMD64 Ubuntu JDK ${{ matrix.jdk }} Maven ${{ matrix.maven }} runs-on: ubuntu-latest if: ${{ !contains(github.event.pull_request.title, 'WIP') }} timeout-minutes: 30 @@ -47,11 +47,11 @@ jobs: matrix: jdk: [11, 17, 21, 22] maven: [3.9.9] - image: [java, conda-java-jni-cdata] + image: [java, conda-jni-cdata] include: - image: java name: "" - - image: conda-java-jni-cdata + - image: conda-jni-cdata name: JNI env: JDK: ${{ matrix.jdk }} diff --git a/ci/docker/conda-java-jni.dockerfile b/ci/docker/conda-jni.dockerfile similarity index 100% rename from ci/docker/conda-java-jni.dockerfile rename to ci/docker/conda-jni.dockerfile diff --git a/ci/scripts/java_jni_build.sh b/ci/scripts/java_jni_build.sh index 99d8cb06e3..7b63dbeeaf 100755 --- a/ci/scripts/java_jni_build.sh +++ b/ci/scripts/java_jni_build.sh @@ -38,7 +38,7 @@ case "$(uname)" in n_jobs=$(nproc) ;; Darwin) - n_jobs=$(sysctl -n hw.ncpu) + n_jobs=$(sysctl -n hw.logicalcpu) ;; *) n_jobs=${NPROC:-1} diff --git a/docker-compose.yml b/docker-compose.yml index 06226bad80..9b84d32d57 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -46,17 +46,17 @@ services: /arrow-java/ci/scripts/java_build.sh /arrow-java /build && /arrow-java/ci/scripts/java_test.sh /arrow-java /build" - conda-java-jni-cdata: + conda-jni-cdata: # Usage: - # docker compose build java-jni-cdata - # docker compose run java-jni-cdata + # docker compose build conda-jni-cdata + # docker compose run conda-jni-cdata # Parameters: # MAVEN: 3.9.9 # JDK: 11, 17, 21 image: ${REPO}:${ARCH}-conda-java-${JDK}-maven-${MAVEN}-jni-integration build: context: . - dockerfile: ci/docker/conda-java-jni.dockerfile + dockerfile: ci/docker/conda-jni.dockerfile cache_from: - ${REPO}:${ARCH}-conda-java-${JDK}-maven-${MAVEN}-jni-integration args: From 49237d25910747869458eb60d0c25bf2372daeb6 Mon Sep 17 00:00:00 2001 From: David Li Date: Wed, 4 Dec 2024 01:08:47 -0500 Subject: [PATCH 6/6] update --- .github/workflows/test.yml | 10 +++++----- docker-compose.yml | 6 +++--- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index cb47c21d11..69c8db0f15 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -38,7 +38,7 @@ env: jobs: ubuntu: - name: ${{ matrix.name }} AMD64 Ubuntu JDK ${{ matrix.jdk }} Maven ${{ matrix.maven }} + name: AMD64 ${{ matrix.name }} JDK ${{ matrix.jdk }} Maven ${{ matrix.maven }} runs-on: ubuntu-latest if: ${{ !contains(github.event.pull_request.title, 'WIP') }} timeout-minutes: 30 @@ -47,12 +47,12 @@ jobs: matrix: jdk: [11, 17, 21, 22] maven: [3.9.9] - image: [java, conda-jni-cdata] + image: [ubuntu, conda-jni-cdata] include: - - image: java - name: "" + - image: ubuntu + name: "Ubuntu" - image: conda-jni-cdata - name: JNI + name: "Conda JNI" env: JDK: ${{ matrix.jdk }} MAVEN: ${{ matrix.maven }} diff --git a/docker-compose.yml b/docker-compose.yml index 9b84d32d57..ae378865b3 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -30,10 +30,10 @@ volumes: name: maven-cache services: - java: + ubuntu: # Usage: - # docker compose build java - # docker compose run java + # docker compose build ubuntu + # docker compose run ubuntu # Parameters: # MAVEN: 3.9.9 # JDK: 11, 17, 21