Skip to content

Commit 2ede415

Browse files
lidavidmlidavidm
committed
GH-13: Set up JNI build (dataset, etc.)
Fixes #13.
1 parent d650aa0 commit 2ede415

File tree

6 files changed

+601
-0
lines changed

6 files changed

+601
-0
lines changed

‎.env‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -40,6 +40,7 @@ ARCH_SHORT=amd64
4040

4141
# Default repository to pull and push images from
4242
REPO=ghcr.io/apache/arrow-java-dev
43+
ARROW_REPO=apache/arrow-dev
4344

4445
# The setup attempts to generate coredumps by default, in order to disable the
4546
# coredump generation set it to 0
@@ -48,3 +49,9 @@ ULIMIT_CORE=-1
4849
# Default versions for various dependencies
4950
JDK=11
5051
MAVEN=3.9.9
52+
53+
# Versions for various dependencies used to build artifacts
54+
# Keep in sync with apache/arrow
55+
ARROW_REPO_ROOT=./arrow
56+
PYTHON=3.9
57+
VCPKG="943c5ef1c8f6b5e6ced092b242c8299caae2ff01" # 2024.04.26 Release

‎.github/workflows/test_jni.yml‎

Lines changed: 194 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,194 @@
1+
# Licensed to the Apache Software Foundation (ASF) under one
2+
# or more contributor license agreements. See the NOTICE file
3+
# distributed with this work for additional information
4+
# regarding copyright ownership. The ASF licenses this file
5+
# to you under the Apache License, Version 2.0 (the
6+
# "License"); you may not use this file except in compliance
7+
# with the License. You may obtain a copy of the License at
8+
#
9+
# http://www.apache.org/licenses/LICENSE-2.0
10+
#
11+
# Unless required by applicable law or agreed to in writing,
12+
# software distributed under the License is distributed on an
13+
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14+
# KIND, either express or implied. See the License for the
15+
# specific language governing permissions and limitations
16+
# under the License.
17+
18+
name: Test (JNI)
19+
20+
on:
21+
push:
22+
branches:
23+
- '**'
24+
- '!dependabot/**'
25+
tags:
26+
- '**'
27+
pull_request:
28+
29+
concurrency:
30+
group: ${{ github.repository }}-${{ github.head_ref || github.sha }}-${{ github.workflow }}
31+
cancel-in-progress: true
32+
33+
permissions:
34+
contents: read
35+
36+
env:
37+
DOCKER_VOLUME_PREFIX: ".docker/"
38+
39+
jobs:
40+
cpp-ubuntu:
41+
name: Build C++ libraries ${{ matrix.platform.runs_on }} ${{ matrix.platform.arch }}
42+
runs-on: ${{ matrix.platform.runs_on }}
43+
strategy:
44+
fail-fast: false
45+
matrix:
46+
platform:
47+
- runs_on: ubuntu-latest
48+
arch: "x86_64"
49+
archery_arch: "amd64"
50+
archery_arch_alias: "x86_64"
51+
archery_arch_short: "amd64"
52+
env:
53+
# architecture name used for archery build
54+
ARCH: ${{ matrix.platform.archery_arch }}
55+
ARCH_ALIAS: ${{ matrix.platform.archery_arch_alias }}
56+
ARCH_SHORT: ${{ matrix.platform.archery_arch_short }}
57+
permissions:
58+
packages: write
59+
steps:
60+
- name: Checkout apache/arrow-java
61+
uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
62+
with:
63+
fetch-depth: 0
64+
submodules: recursive
65+
- name: Checkout apache/arrow
66+
uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
67+
with:
68+
repository: apache/arrow
69+
fetch-depth: 0
70+
path: arrow
71+
submodules: recursive
72+
- uses: docker/login-action@9780b0c442fbb1117ed29e0efdff1e18412f7567 # v3.3.0
73+
with:
74+
registry: ghcr.io
75+
username: ${{ github.actor }}
76+
password: ${{ secrets.GITHUB_TOKEN }}
77+
- name: Build C++ libraries
78+
env:
79+
VCPKG_BINARY_SOURCES: "clear;nuget,GitHub,readwrite"
80+
run: |
81+
docker compose run vcpkg-jni
82+
- name: Push Docker image
83+
run: |
84+
docker push ghcr.io/apache/arrow-java-dev:amd64-vcpkg-jni
85+
- name: Compress into single artifact to keep directory structure
86+
run: tar -cvzf arrow-shared-libs-linux-${{ matrix.platform.arch }}.tar.gz dist/
87+
- name: Upload artifacts
88+
uses: actions/upload-artifact@v4
89+
with:
90+
name: ubuntu-shared-lib-${{ matrix.platform.arch }}
91+
path: arrow-shared-libs-linux-${{ matrix.platform.arch }}.tar.gz
92+
93+
cpp-macos:
94+
name: Build C++ libraries macOS ${{ matrix.platform.runs_on }} ${{ matrix.platform.arch }}
95+
runs-on: ${{ matrix.platform.runs_on }}
96+
strategy:
97+
fail-fast: false
98+
matrix:
99+
platform:
100+
- { runs_on: macos-13, arch: "x86_64"}
101+
- { runs_on: macos-14, arch: "aarch_64" }
102+
env:
103+
MACOSX_DEPLOYMENT_TARGET: "14.0"
104+
steps:
105+
- name: Checkout apache/arrow-java
106+
uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
107+
with:
108+
fetch-depth: 0
109+
submodules: recursive
110+
- name: Checkout apache/arrow
111+
uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
112+
with:
113+
repository: apache/arrow
114+
fetch-depth: 0
115+
path: arrow
116+
submodules: recursive
117+
- name: Set up Python
118+
uses: actions/setup-python@v4
119+
with:
120+
cache: 'pip'
121+
python-version: 3.12
122+
- name: Install Archery
123+
run: pip install -e arrow/dev/archery[all]
124+
- name: Install dependencies
125+
run: |
126+
# We want to use llvm@14 to avoid shared z3
127+
# dependency. llvm@14 doesn't depend on z3 and llvm depends
128+
# on z3. And Homebrew's z3 provides only shared library. It
129+
# doesn't provides static z3 because z3's CMake doesn't accept
130+
# building both shared and static libraries at once.
131+
# See also: Z3_BUILD_LIBZ3_SHARED in
132+
# https://github.com/Z3Prover/z3/blob/master/README-CMake.md
133+
#
134+
# If llvm is installed, Apache Arrow C++ uses llvm rather than
135+
# llvm@14 because llvm is newer than llvm@14.
136+
brew uninstall llvm || :
137+
138+
# Ensure updating python@XXX with the "--overwrite" option.
139+
# If python@XXX is updated without "--overwrite", it causes
140+
# a conflict error. Because Python 3 installed not by
141+
# Homebrew exists in /usr/local on GitHub Actions. If
142+
# Homebrew's python@XXX is updated without "--overwrite", it
143+
# tries to replace /usr/local/bin/2to3 and so on and causes
144+
# a conflict error.
145+
brew update
146+
for python_package in $(brew list | grep python@); do
147+
brew install --overwrite ${python_package}
148+
done
149+
brew install --overwrite python
150+
151+
if [ "$(uname -m)" = "arm64" ]; then
152+
# pkg-config formula is deprecated but it's still installed
153+
# in GitHub Actions runner now. We can remove this once
154+
# pkg-config formula is removed from GitHub Actions runner.
155+
brew uninstall pkg-config || :
156+
brew uninstall pkg-config@0.29.2 || :
157+
fi
158+
159+
brew bundle --file=arrow/cpp/Brewfile
160+
# We want to link aws-sdk-cpp statically but Homebrew's
161+
# aws-sdk-cpp provides only shared library. If we have
162+
# Homebrew's aws-sdk-cpp, our build mix Homebrew's
163+
# aws-sdk-cpp and bundled aws-sdk-cpp. We uninstall Homebrew's
164+
# aws-sdk-cpp to ensure using only bundled aws-sdk-cpp.
165+
brew uninstall aws-sdk-cpp
166+
# We want to use bundled RE2 for static linking. If
167+
# Homebrew's RE2 is installed, its header file may be used.
168+
# We uninstall Homebrew's RE2 to ensure using bundled RE2.
169+
brew uninstall grpc || : # gRPC depends on RE2
170+
brew uninstall grpc@1.54 || : # gRPC 1.54 may be installed too
171+
brew uninstall re2
172+
# We want to use bundled Protobuf for static linking. If
173+
# Homebrew's Protobuf is installed, its library file may be
174+
# used on test We uninstall Homebrew's Protobuf to ensure using
175+
# bundled Protobuf.
176+
brew uninstall protobuf
177+
178+
brew bundle --file=Brewfile
179+
- name: Build C++ libraries
180+
run: |
181+
set -e
182+
# make brew Java available to CMake
183+
export JAVA_HOME=$(brew --prefix openjdk@11)/libexec/openjdk.jdk/Contents/Home
184+
arrow-java/ci/scripts/java_jni_macos_build.sh \
185+
$GITHUB_WORKSPACE/arrow \
186+
$GITHUB_WORKSPACE/arrow/cpp-build \
187+
$GITHUB_WORKSPACE/dist
188+
- name: Compress into single artifact to keep directory structure
189+
run: tar -cvzf arrow-shared-libs-macos-${{ matrix.platform.arch }}.tar.gz dist/
190+
- name: Upload artifacts
191+
uses: actions/upload-artifact@v4
192+
with:
193+
name: macos-shared-lib-${{ matrix.platform.arch }}
194+
path: arrow-shared-libs-macos-${{ matrix.platform.arch }}.tar.gz

‎ci/docker/vcpkg-jni.dockerfile‎

Lines changed: 46 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,46 @@
1+
# Licensed to the Apache Software Foundation (ASF) under one
2+
# or more contributor license agreements. See the NOTICE file
3+
# distributed with this work for additional information
4+
# regarding copyright ownership. The ASF licenses this file
5+
# to you under the Apache License, Version 2.0 (the
6+
# "License"); you may not use this file except in compliance
7+
# with the License. You may obtain a copy of the License at
8+
#
9+
# http://www.apache.org/licenses/LICENSE-2.0
10+
#
11+
# Unless required by applicable law or agreed to in writing,
12+
# software distributed under the License is distributed on an
13+
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14+
# KIND, either express or implied. See the License for the
15+
# specific language governing permissions and limitations
16+
# under the License.
17+
18+
ARG base
19+
FROM ${base}
20+
21+
# Install the libraries required by Gandiva to run
22+
# Use enable llvm[enable-rtti] in the vcpkg.json to avoid link problems in Gandiva
23+
RUN vcpkg install \
24+
--clean-after-build \
25+
--x-install-root=${VCPKG_ROOT}/installed \
26+
--x-manifest-root=/arrow/ci/vcpkg \
27+
--x-feature=dev \
28+
--x-feature=flight \
29+
--x-feature=gcs \
30+
--x-feature=json \
31+
--x-feature=parquet \
32+
--x-feature=gandiva \
33+
--x-feature=s3
34+
35+
# Install Java
36+
# We need Java for JNI headers, but we don't invoke Maven in this build.
37+
ARG java=11
38+
RUN yum install -y java-$java-openjdk-devel && yum clean all
39+
40+
# For ci/scripts/{cpp,java}_*.sh
41+
ENV ARROW_HOME=/tmp/local \
42+
ARROW_JAVA_CDATA=ON \
43+
ARROW_JAVA_JNI=ON \
44+
ARROW_USE_CCACHE=ON
45+
46+
LABEL org.opencontainers.image.source https://github.com/apache/arrow-java

0 commit comments

Comments
 (0)