Skip to content

Commit c8eb06b

Browse files
lidavidmlidavidm
committed
GH-13: Set up JNI build (dataset, etc.)
Fixes #13.
1 parent d650aa0 commit c8eb06b

File tree

6 files changed

+602
-0
lines changed

6 files changed

+602
-0
lines changed

‎.env‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -40,6 +40,7 @@ ARCH_SHORT=amd64
4040

4141
# Default repository to pull and push images from
4242
REPO=ghcr.io/apache/arrow-java-dev
43+
ARROW_REPO=apache/arrow-dev
4344

4445
# The setup attempts to generate coredumps by default, in order to disable the
4546
# coredump generation set it to 0
@@ -48,3 +49,9 @@ ULIMIT_CORE=-1
4849
# Default versions for various dependencies
4950
JDK=11
5051
MAVEN=3.9.9
52+
53+
# Versions for various dependencies used to build artifacts
54+
# Keep in sync with apache/arrow
55+
ARROW_REPO_ROOT=./arrow
56+
PYTHON=3.9
57+
VCPKG="943c5ef1c8f6b5e6ced092b242c8299caae2ff01" # 2024.04.26 Release

‎.github/workflows/test_jni.yml‎

Lines changed: 195 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,195 @@
1+
# Licensed to the Apache Software Foundation (ASF) under one
2+
# or more contributor license agreements. See the NOTICE file
3+
# distributed with this work for additional information
4+
# regarding copyright ownership. The ASF licenses this file
5+
# to you under the Apache License, Version 2.0 (the
6+
# "License"); you may not use this file except in compliance
7+
# with the License. You may obtain a copy of the License at
8+
#
9+
# http://www.apache.org/licenses/LICENSE-2.0
10+
#
11+
# Unless required by applicable law or agreed to in writing,
12+
# software distributed under the License is distributed on an
13+
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14+
# KIND, either express or implied. See the License for the
15+
# specific language governing permissions and limitations
16+
# under the License.
17+
18+
name: Test (JNI)
19+
20+
on:
21+
push:
22+
branches:
23+
- '**'
24+
- '!dependabot/**'
25+
tags:
26+
- '**'
27+
pull_request:
28+
29+
concurrency:
30+
group: ${{ github.repository }}-${{ github.head_ref || github.sha }}-${{ github.workflow }}
31+
cancel-in-progress: true
32+
33+
permissions:
34+
contents: read
35+
36+
env:
37+
DOCKER_VOLUME_PREFIX: ".docker/"
38+
39+
jobs:
40+
cpp-ubuntu:
41+
name: Build C++ libraries ${{ matrix.platform.runs_on }} ${{ matrix.platform.arch }}
42+
runs-on: ${{ matrix.platform.runs_on }}
43+
strategy:
44+
fail-fast: false
45+
matrix:
46+
platform:
47+
- runs_on: ubuntu-latest
48+
arch: "x86_64"
49+
archery_arch: "amd64"
50+
archery_arch_alias: "x86_64"
51+
archery_arch_short: "amd64"
52+
env:
53+
# architecture name used for archery build
54+
ARCH: ${{ matrix.platform.archery_arch }}
55+
ARCH_ALIAS: ${{ matrix.platform.archery_arch_alias }}
56+
ARCH_SHORT: ${{ matrix.platform.archery_arch_short }}
57+
permissions:
58+
contents: read
59+
packages: write
60+
steps:
61+
- name: Checkout apache/arrow-java
62+
uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
63+
with:
64+
fetch-depth: 0
65+
submodules: recursive
66+
- name: Checkout apache/arrow
67+
uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
68+
with:
69+
repository: apache/arrow
70+
fetch-depth: 0
71+
path: arrow
72+
submodules: recursive
73+
- uses: docker/login-action@9780b0c442fbb1117ed29e0efdff1e18412f7567 # v3.3.0
74+
with:
75+
registry: ghcr.io
76+
username: ${{ github.actor }}
77+
password: ${{ secrets.GITHUB_TOKEN }}
78+
- name: Build C++ libraries
79+
env:
80+
VCPKG_BINARY_SOURCES: "clear;nuget,GitHub,readwrite"
81+
run: |
82+
docker compose run vcpkg-jni
83+
- name: Push Docker image
84+
run: |
85+
docker push ghcr.io/apache/arrow-java-dev:amd64-vcpkg-jni
86+
- name: Compress into single artifact to keep directory structure
87+
run: tar -cvzf arrow-shared-libs-linux-${{ matrix.platform.arch }}.tar.gz dist/
88+
- name: Upload artifacts
89+
uses: actions/upload-artifact@v4
90+
with:
91+
name: ubuntu-shared-lib-${{ matrix.platform.arch }}
92+
path: arrow-shared-libs-linux-${{ matrix.platform.arch }}.tar.gz
93+
94+
cpp-macos:
95+
name: Build C++ libraries macOS ${{ matrix.platform.runs_on }} ${{ matrix.platform.arch }}
96+
runs-on: ${{ matrix.platform.runs_on }}
97+
strategy:
98+
fail-fast: false
99+
matrix:
100+
platform:
101+
- { runs_on: macos-13, arch: "x86_64"}
102+
- { runs_on: macos-14, arch: "aarch_64" }
103+
env:
104+
MACOSX_DEPLOYMENT_TARGET: "14.0"
105+
steps:
106+
- name: Checkout apache/arrow-java
107+
uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
108+
with:
109+
fetch-depth: 0
110+
submodules: recursive
111+
- name: Checkout apache/arrow
112+
uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
113+
with:
114+
repository: apache/arrow
115+
fetch-depth: 0
116+
path: arrow
117+
submodules: recursive
118+
- name: Set up Python
119+
uses: actions/setup-python@v4
120+
with:
121+
cache: 'pip'
122+
python-version: 3.12
123+
- name: Install Archery
124+
run: pip install -e arrow/dev/archery[all]
125+
- name: Install dependencies
126+
run: |
127+
# We want to use llvm@14 to avoid shared z3
128+
# dependency. llvm@14 doesn't depend on z3 and llvm depends
129+
# on z3. And Homebrew's z3 provides only shared library. It
130+
# doesn't provides static z3 because z3's CMake doesn't accept
131+
# building both shared and static libraries at once.
132+
# See also: Z3_BUILD_LIBZ3_SHARED in
133+
# https://github.com/Z3Prover/z3/blob/master/README-CMake.md
134+
#
135+
# If llvm is installed, Apache Arrow C++ uses llvm rather than
136+
# llvm@14 because llvm is newer than llvm@14.
137+
brew uninstall llvm || :
138+
139+
# Ensure updating python@XXX with the "--overwrite" option.
140+
# If python@XXX is updated without "--overwrite", it causes
141+
# a conflict error. Because Python 3 installed not by
142+
# Homebrew exists in /usr/local on GitHub Actions. If
143+
# Homebrew's python@XXX is updated without "--overwrite", it
144+
# tries to replace /usr/local/bin/2to3 and so on and causes
145+
# a conflict error.
146+
brew update
147+
for python_package in $(brew list | grep python@); do
148+
brew install --overwrite ${python_package}
149+
done
150+
brew install --overwrite python
151+
152+
if [ "$(uname -m)" = "arm64" ]; then
153+
# pkg-config formula is deprecated but it's still installed
154+
# in GitHub Actions runner now. We can remove this once
155+
# pkg-config formula is removed from GitHub Actions runner.
156+
brew uninstall pkg-config || :
157+
brew uninstall pkg-config@0.29.2 || :
158+
fi
159+
160+
brew bundle --file=arrow/cpp/Brewfile
161+
# We want to link aws-sdk-cpp statically but Homebrew's
162+
# aws-sdk-cpp provides only shared library. If we have
163+
# Homebrew's aws-sdk-cpp, our build mix Homebrew's
164+
# aws-sdk-cpp and bundled aws-sdk-cpp. We uninstall Homebrew's
165+
# aws-sdk-cpp to ensure using only bundled aws-sdk-cpp.
166+
brew uninstall aws-sdk-cpp
167+
# We want to use bundled RE2 for static linking. If
168+
# Homebrew's RE2 is installed, its header file may be used.
169+
# We uninstall Homebrew's RE2 to ensure using bundled RE2.
170+
brew uninstall grpc || : # gRPC depends on RE2
171+
brew uninstall grpc@1.54 || : # gRPC 1.54 may be installed too
172+
brew uninstall re2
173+
# We want to use bundled Protobuf for static linking. If
174+
# Homebrew's Protobuf is installed, its library file may be
175+
# used on test We uninstall Homebrew's Protobuf to ensure using
176+
# bundled Protobuf.
177+
brew uninstall protobuf
178+
179+
brew bundle --file=Brewfile
180+
- name: Build C++ libraries
181+
run: |
182+
set -e
183+
# make brew Java available to CMake
184+
export JAVA_HOME=$(brew --prefix openjdk@11)/libexec/openjdk.jdk/Contents/Home
185+
./ci/scripts/java_jni_macos_build.sh \
186+
$GITHUB_WORKSPACE/arrow \
187+
$GITHUB_WORKSPACE/arrow/cpp-build \
188+
$GITHUB_WORKSPACE/dist
189+
- name: Compress into single artifact to keep directory structure
190+
run: tar -cvzf arrow-shared-libs-macos-${{ matrix.platform.arch }}.tar.gz dist/
191+
- name: Upload artifacts
192+
uses: actions/upload-artifact@v4
193+
with:
194+
name: macos-shared-lib-${{ matrix.platform.arch }}
195+
path: arrow-shared-libs-macos-${{ matrix.platform.arch }}.tar.gz

‎ci/docker/vcpkg-jni.dockerfile‎

Lines changed: 46 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,46 @@
1+
# Licensed to the Apache Software Foundation (ASF) under one
2+
# or more contributor license agreements. See the NOTICE file
3+
# distributed with this work for additional information
4+
# regarding copyright ownership. The ASF licenses this file
5+
# to you under the Apache License, Version 2.0 (the
6+
# "License"); you may not use this file except in compliance
7+
# with the License. You may obtain a copy of the License at
8+
#
9+
# http://www.apache.org/licenses/LICENSE-2.0
10+
#
11+
# Unless required by applicable law or agreed to in writing,
12+
# software distributed under the License is distributed on an
13+
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14+
# KIND, either express or implied. See the License for the
15+
# specific language governing permissions and limitations
16+
# under the License.
17+
18+
ARG base
19+
FROM ${base}
20+
21+
# Install the libraries required by Gandiva to run
22+
# Use enable llvm[enable-rtti] in the vcpkg.json to avoid link problems in Gandiva
23+
RUN vcpkg install \
24+
--clean-after-build \
25+
--x-install-root=${VCPKG_ROOT}/installed \
26+
--x-manifest-root=/arrow/ci/vcpkg \
27+
--x-feature=dev \
28+
--x-feature=flight \
29+
--x-feature=gcs \
30+
--x-feature=json \
31+
--x-feature=parquet \
32+
--x-feature=gandiva \
33+
--x-feature=s3
34+
35+
# Install Java
36+
# We need Java for JNI headers, but we don't invoke Maven in this build.
37+
ARG java=11
38+
RUN yum install -y java-$java-openjdk-devel && yum clean all
39+
40+
# For ci/scripts/{cpp,java}_*.sh
41+
ENV ARROW_HOME=/tmp/local \
42+
ARROW_JAVA_CDATA=ON \
43+
ARROW_JAVA_JNI=ON \
44+
ARROW_USE_CCACHE=ON
45+
46+
LABEL org.opencontainers.image.source https://github.com/apache/arrow-java

0 commit comments

Comments
 (0)