Skip to content

Commit 32c6e1a

Browse files
lidavidmlidavidm
committed
GH-13: Set up JNI build (dataset, etc.)
Fixes #13.
1 parent 480e1be commit 32c6e1a

File tree

8 files changed

+755
-2
lines changed

8 files changed

+755
-2
lines changed

‎.env‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -40,6 +40,7 @@ ARCH_SHORT=amd64
4040

4141
# Default repository to pull and push images from
4242
REPO=ghcr.io/apache/arrow-java-dev
43+
ARROW_REPO=apache/arrow-dev
4344

4445
# The setup attempts to generate coredumps by default, in order to disable the
4546
# coredump generation set it to 0
@@ -48,3 +49,9 @@ ULIMIT_CORE=-1
4849
# Default versions for various dependencies
4950
JDK=11
5051
MAVEN=3.9.9
52+
53+
# Versions for various dependencies used to build artifacts
54+
# Keep in sync with apache/arrow
55+
ARROW_REPO_ROOT=./arrow
56+
PYTHON=3.9
57+
VCPKG="943c5ef1c8f6b5e6ced092b242c8299caae2ff01" # 2024.04.26 Release

‎.github/workflows/test_jni.yml‎

Lines changed: 273 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,273 @@
1+
# Licensed to the Apache Software Foundation (ASF) under one
2+
# or more contributor license agreements. See the NOTICE file
3+
# distributed with this work for additional information
4+
# regarding copyright ownership. The ASF licenses this file
5+
# to you under the Apache License, Version 2.0 (the
6+
# "License"); you may not use this file except in compliance
7+
# with the License. You may obtain a copy of the License at
8+
#
9+
# http://www.apache.org/licenses/LICENSE-2.0
10+
#
11+
# Unless required by applicable law or agreed to in writing,
12+
# software distributed under the License is distributed on an
13+
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14+
# KIND, either express or implied. See the License for the
15+
# specific language governing permissions and limitations
16+
# under the License.
17+
18+
name: Test (JNI)
19+
20+
on:
21+
push:
22+
branches:
23+
- '**'
24+
- '!dependabot/**'
25+
tags:
26+
- '**'
27+
pull_request:
28+
29+
concurrency:
30+
group: ${{ github.repository }}-${{ github.head_ref || github.sha }}-${{ github.workflow }}
31+
cancel-in-progress: true
32+
33+
permissions:
34+
contents: read
35+
36+
env:
37+
DOCKER_VOLUME_PREFIX: ".docker/"
38+
39+
jobs:
40+
cpp-ubuntu:
41+
name: Build C++ libraries ${{ matrix.platform.runs_on }} ${{ matrix.platform.arch }}
42+
runs-on: ${{ matrix.platform.runs_on }}
43+
strategy:
44+
fail-fast: false
45+
matrix:
46+
platform:
47+
- runs_on: ubuntu-latest
48+
arch: "x86_64"
49+
archery_arch: "amd64"
50+
archery_arch_alias: "x86_64"
51+
archery_arch_short: "amd64"
52+
env:
53+
# architecture name used for archery build
54+
ARCH: ${{ matrix.platform.archery_arch }}
55+
ARCH_ALIAS: ${{ matrix.platform.archery_arch_alias }}
56+
ARCH_SHORT: ${{ matrix.platform.archery_arch_short }}
57+
permissions:
58+
contents: read
59+
packages: write
60+
steps:
61+
- name: Checkout apache/arrow-java
62+
uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
63+
with:
64+
fetch-depth: 0
65+
submodules: recursive
66+
- name: Checkout apache/arrow
67+
uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
68+
with:
69+
repository: apache/arrow
70+
fetch-depth: 0
71+
path: arrow
72+
submodules: recursive
73+
- uses: docker/login-action@9780b0c442fbb1117ed29e0efdff1e18412f7567 # v3.3.0
74+
with:
75+
registry: ghcr.io
76+
username: ${{ github.actor }}
77+
password: ${{ secrets.GITHUB_TOKEN }}
78+
- name: Build C++ libraries
79+
env:
80+
VCPKG_BINARY_SOURCES: "clear;nuget,GitHub,readwrite"
81+
run: |
82+
docker compose run vcpkg-jni
83+
- name: Push Docker image
84+
if: success() && github.event_name == 'push' && github.repository == 'apache/arrow-java' && github.ref_name == 'main'
85+
run: |
86+
docker push ghcr.io/apache/arrow-java-dev:amd64-vcpkg-jni
87+
- name: Compress into single artifact to keep directory structure
88+
run: tar -cvzf arrow-shared-libs-linux-${{ matrix.platform.arch }}.tar.gz dist/
89+
- name: Upload artifacts
90+
uses: actions/upload-artifact@v4
91+
with:
92+
name: ubuntu-shared-lib-${{ matrix.platform.arch }}
93+
path: arrow-shared-libs-linux-${{ matrix.platform.arch }}.tar.gz
94+
95+
cpp-macos:
96+
name: Build C++ libraries macOS ${{ matrix.platform.runs_on }} ${{ matrix.platform.arch }}
97+
runs-on: ${{ matrix.platform.runs_on }}
98+
strategy:
99+
fail-fast: false
100+
matrix:
101+
platform:
102+
- { runs_on: macos-13, arch: "x86_64"}
103+
- { runs_on: macos-14, arch: "aarch_64" }
104+
env:
105+
MACOSX_DEPLOYMENT_TARGET: "14.0"
106+
steps:
107+
- name: Checkout apache/arrow-java
108+
uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
109+
with:
110+
fetch-depth: 0
111+
submodules: recursive
112+
- name: Checkout apache/arrow
113+
uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
114+
with:
115+
repository: apache/arrow
116+
fetch-depth: 0
117+
path: arrow
118+
submodules: recursive
119+
- name: Set up Python
120+
uses: actions/setup-python@v4
121+
with:
122+
cache: 'pip'
123+
python-version: 3.12
124+
- name: Install Archery
125+
run: pip install -e arrow/dev/archery[all]
126+
- name: Install dependencies
127+
run: |
128+
# We want to use llvm@14 to avoid shared z3
129+
# dependency. llvm@14 doesn't depend on z3 and llvm depends
130+
# on z3. And Homebrew's z3 provides only shared library. It
131+
# doesn't provides static z3 because z3's CMake doesn't accept
132+
# building both shared and static libraries at once.
133+
# See also: Z3_BUILD_LIBZ3_SHARED in
134+
# https://github.com/Z3Prover/z3/blob/master/README-CMake.md
135+
#
136+
# If llvm is installed, Apache Arrow C++ uses llvm rather than
137+
# llvm@14 because llvm is newer than llvm@14.
138+
brew uninstall llvm || :
139+
140+
# Ensure updating python@XXX with the "--overwrite" option.
141+
# If python@XXX is updated without "--overwrite", it causes
142+
# a conflict error. Because Python 3 installed not by
143+
# Homebrew exists in /usr/local on GitHub Actions. If
144+
# Homebrew's python@XXX is updated without "--overwrite", it
145+
# tries to replace /usr/local/bin/2to3 and so on and causes
146+
# a conflict error.
147+
brew update
148+
for python_package in $(brew list | grep python@); do
149+
brew install --overwrite ${python_package}
150+
done
151+
brew install --overwrite python
152+
153+
if [ "$(uname -m)" = "arm64" ]; then
154+
# pkg-config formula is deprecated but it's still installed
155+
# in GitHub Actions runner now. We can remove this once
156+
# pkg-config formula is removed from GitHub Actions runner.
157+
brew uninstall pkg-config || :
158+
brew uninstall pkg-config@0.29.2 || :
159+
fi
160+
161+
brew bundle --file=arrow/cpp/Brewfile
162+
# We want to link aws-sdk-cpp statically but Homebrew's
163+
# aws-sdk-cpp provides only shared library. If we have
164+
# Homebrew's aws-sdk-cpp, our build mix Homebrew's
165+
# aws-sdk-cpp and bundled aws-sdk-cpp. We uninstall Homebrew's
166+
# aws-sdk-cpp to ensure using only bundled aws-sdk-cpp.
167+
brew uninstall aws-sdk-cpp
168+
# We want to use bundled RE2 for static linking. If
169+
# Homebrew's RE2 is installed, its header file may be used.
170+
# We uninstall Homebrew's RE2 to ensure using bundled RE2.
171+
brew uninstall grpc || : # gRPC depends on RE2
172+
brew uninstall grpc@1.54 || : # gRPC 1.54 may be installed too
173+
brew uninstall re2
174+
# We want to use bundled Protobuf for static linking. If
175+
# Homebrew's Protobuf is installed, its library file may be
176+
# used on test We uninstall Homebrew's Protobuf to ensure using
177+
# bundled Protobuf.
178+
brew uninstall protobuf
179+
180+
brew bundle --file=Brewfile
181+
- name: Build C++ libraries
182+
run: |
183+
set -e
184+
# make brew Java available to CMake
185+
export JAVA_HOME=$(brew --prefix openjdk@11)/libexec/openjdk.jdk/Contents/Home
186+
./ci/scripts/java_jni_macos_build.sh \
187+
$GITHUB_WORKSPACE \
188+
$GITHUB_WORKSPACE/arrow \
189+
$GITHUB_WORKSPACE/arrow-java/cpp-build \
190+
$GITHUB_WORKSPACE/dist
191+
- name: Compress into single artifact to keep directory structure
192+
run: tar -cvzf arrow-shared-libs-macos-${{ matrix.platform.arch }}.tar.gz dist/
193+
- name: Upload artifacts
194+
uses: actions/upload-artifact@v4
195+
with:
196+
name: macos-shared-lib-${{ matrix.platform.arch }}
197+
path: arrow-shared-libs-macos-${{ matrix.platform.arch }}.tar.gz
198+
199+
java-jars:
200+
name: Build JAR files
201+
runs-on: ubuntu-latest
202+
needs:
203+
- cpp-ubuntu
204+
- cpp-macos
205+
steps:
206+
- name: Checkout apache/arrow-java
207+
uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
208+
with:
209+
fetch-depth: 0
210+
submodules: recursive
211+
- name: Checkout apache/arrow
212+
uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
213+
with:
214+
repository: apache/arrow
215+
fetch-depth: 0
216+
path: arrow
217+
submodules: recursive
218+
- name: Download Libraries
219+
uses: actions/download-artifact@v4
220+
with:
221+
path: artifacts
222+
- name: Decompress artifacts
223+
run: |
224+
mv artifacts/*/*.tar.gz .
225+
tar -xvzf arrow-shared-libs-linux-x86_64.tar.gz
226+
# tar -xvzf arrow-shared-libs-linux-aarch_64.tar.gz
227+
tar -xvzf arrow-shared-libs-macos-x86_64.tar.gz
228+
tar -xvzf arrow-shared-libs-macos-aarch_64.tar.gz
229+
# tar -xvzf arrow-shared-libs-windows.tar.gz
230+
- name: Test that shared libraries exist
231+
run: |
232+
set -x
233+
234+
test -f dist/arrow_cdata_jni/x86_64/libarrow_cdata_jni.so
235+
test -f dist/arrow_dataset_jni/x86_64/libarrow_dataset_jni.so
236+
test -f dist/arrow_orc_jni/x86_64/libarrow_orc_jni.so
237+
test -f dist/gandiva_jni/x86_64/libgandiva_jni.so
238+
239+
# test -f dist/arrow_cdata_jni/aarch_64/libarrow_cdata_jni.so
240+
# test -f dist/arrow_dataset_jni/aarch_64/libarrow_dataset_jni.so
241+
# test -f dist/arrow_orc_jni/aarch_64/libarrow_orc_jni.so
242+
# test -f dist/gandiva_jni/aarch_64/libgandiva_jni.so
243+
244+
test -f dist/arrow_cdata_jni/x86_64/libarrow_cdata_jni.dylib
245+
test -f dist/arrow_dataset_jni/x86_64/libarrow_dataset_jni.dylib
246+
test -f dist/arrow_orc_jni/x86_64/libarrow_orc_jni.dylib
247+
test -f dist/gandiva_jni/x86_64/libgandiva_jni.dylib
248+
249+
test -f dist/arrow_cdata_jni/aarch_64/libarrow_cdata_jni.dylib
250+
test -f dist/arrow_dataset_jni/aarch_64/libarrow_dataset_jni.dylib
251+
test -f dist/arrow_orc_jni/aarch_64/libarrow_orc_jni.dylib
252+
test -f dist/gandiva_jni/aarch_64/libgandiva_jni.dylib
253+
254+
# test -f dist/arrow_cdata_jni/x86_64/arrow_cdata_jni.dll
255+
# test -f dist/arrow_dataset_jni/x86_64/arrow_dataset_jni.dll
256+
# test -f dist/arrow_orc_jni/x86_64/arrow_orc_jni.dll
257+
- name: Build bundled jar
258+
env:
259+
MAVEN_ARGS: >-
260+
--no-transfer-progress
261+
run: |
262+
set -e
263+
# mvn versions:set -DnewVersion={{ arrow.no_rc_snapshot_version }}
264+
# mvn versions:set -DnewVersion={{ arrow.no_rc_snapshot_version }} -f bom
265+
./ci/scripts/java_full_build.sh \
266+
$GITHUB_WORKSPACE \
267+
$GITHUB_WORKSPACE/arrow \
268+
$GITHUB_WORKSPACE/dist
269+
- name: Upload artifacts
270+
uses: actions/upload-artifact@v4
271+
with:
272+
name: java-jars
273+
path: ${{ github.workspace }}/arrow-java/java-dist

‎ci/docker/vcpkg-jni.dockerfile‎

Lines changed: 46 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,46 @@
1+
# Licensed to the Apache Software Foundation (ASF) under one
2+
# or more contributor license agreements. See the NOTICE file
3+
# distributed with this work for additional information
4+
# regarding copyright ownership. The ASF licenses this file
5+
# to you under the Apache License, Version 2.0 (the
6+
# "License"); you may not use this file except in compliance
7+
# with the License. You may obtain a copy of the License at
8+
#
9+
# http://www.apache.org/licenses/LICENSE-2.0
10+
#
11+
# Unless required by applicable law or agreed to in writing,
12+
# software distributed under the License is distributed on an
13+
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14+
# KIND, either express or implied. See the License for the
15+
# specific language governing permissions and limitations
16+
# under the License.
17+
18+
ARG base
19+
FROM ${base}
20+
21+
# Install the libraries required by Gandiva to run
22+
# Use enable llvm[enable-rtti] in the vcpkg.json to avoid link problems in Gandiva
23+
RUN vcpkg install \
24+
--clean-after-build \
25+
--x-install-root=${VCPKG_ROOT}/installed \
26+
--x-manifest-root=/arrow/ci/vcpkg \
27+
--x-feature=dev \
28+
--x-feature=flight \
29+
--x-feature=gcs \
30+
--x-feature=json \
31+
--x-feature=parquet \
32+
--x-feature=gandiva \
33+
--x-feature=s3
34+
35+
# Install Java
36+
# We need Java for JNI headers, but we don't invoke Maven in this build.
37+
ARG java=11
38+
RUN yum install -y java-$java-openjdk-devel && yum clean all
39+
40+
# For ci/scripts/{cpp,java}_*.sh
41+
ENV ARROW_HOME=/tmp/local \
42+
ARROW_JAVA_CDATA=ON \
43+
ARROW_JAVA_JNI=ON \
44+
ARROW_USE_CCACHE=ON
45+
46+
LABEL org.opencontainers.image.source https://github.com/apache/arrow-java

0 commit comments

Comments
 (0)