GH-537: [Release][Docs] Generate API reference

The followings are out-of-scope:

* Generate Sphinx based documents
* Publish this to https://arrow.apache.org/java/reference/ in release
  process
* Publish this to https://arrow.apache.org/dev/reference/ nightly

Author: kou

.github/workflows/rc.yml

@@ -76,6 +76,49 @@ jobs:
           name: release-source
           path: |
             apache-arrow-java-*
+  docs:
+    name: Docs
+    runs-on: ubuntu-latest
+    timeout-minutes: 30
+    needs:
+      - source
+    env:
+      DOCKER_VOLUME_PREFIX: .docker/
+    permissions:
+      contents: read
+      packages: write
+    steps:
+      - name: Download source archive
+        uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8
+        with:
+          name: release-source
+      - name: Extract source archive
+        run: |
+          tar -xf apache-arrow-java-*.tar.gz --strip-components=1
+      - uses: docker/login-action@9780b0c442fbb1117ed29e0efdff1e18412f7567 # v3.3.0
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+      - name: Cache
+        uses: actions/cache@1bd1e32a3bdc45362d1e726936510720a7c30a57 # v4.2.0
+        with:
+          path: .docker
+          key: docs-${{ hashFiles('compose.yaml', '**/pom.xml', '**/*.java') }}
+          restore-keys: docs-
+      - name: Build
+        run: |
+          docker compose run \
+            --volume "${PWD}/build/:/build/" \
+            docs
+      - name: Compress into single artifact to keep directory structure
+        run: |
+          tar -cvzf docs.tar.gz -C build/java docs
+      - name: Upload artifacts
+        uses: actions/upload-artifact@65c4c4a1ddee5b72f698fdd19549f0f0fb45cf08 # v4.6.0
+        with:
+          name: release-docs-${{ matrix.platform.arch }}
+          path: docs.tar.gz
   jni-linux:
     name: JNI ${{ matrix.platform.runs_on }} ${{ matrix.platform.arch }}
     runs-on: ${{ matrix.platform.runs_on }}

.github/workflows/test.yml

@@ -66,7 +66,7 @@ jobs:
         uses: actions/cache@6849a6489940f00c2f30c0fb92c6274307ccb58a # v4.1.2
         with:
           path: .docker
-          key: maven-${{ matrix.jdk }}-${{ matrix.maven }}-${{ hashFiles('**/compose.yaml', '**/pom.xml') }}
+          key: maven-${{ matrix.jdk }}-${{ matrix.maven }}-${{ hashFiles('compose.yaml', '**/pom.xml', '**/*.java') }}
           restore-keys: maven-${{ matrix.jdk }}-${{ matrix.maven }}-
       - name: Execute Docker Build
         env:

ci/scripts/build.sh

@@ -17,21 +17,19 @@
 # specific language governing permissions and limitations
 # under the License.
 
-set -eo pipefail
+set -euo pipefail
 
 if [[ "${ARROW_JAVA_BUILD:-ON}" != "ON" ]]; then
   exit
 fi
 
 source_dir=${1}
-build_dir=${2}
+build_dir=${2}/java
 java_jni_dist_dir=${3}
 
-: "${BUILD_DOCS_JAVA:=OFF}"
-
 mvn="mvn -B -DskipTests -Drat.skip=true -Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=warn"
 
-if [ "$ARROW_JAVA_SKIP_GIT_PLUGIN" ]; then
+if [ "${ARROW_JAVA_SKIP_GIT_PLUGIN:-OFF}" = "ON" ]; then
   mvn="${mvn} -Dmaven.gitcommitid.skip=true"
 fi
 
@@ -61,27 +59,38 @@ done
 
 pushd "${build_dir}"
 
-if [ "${ARROW_JAVA_SHADE_FLATBUFFERS}" == "ON" ]; then
+# TODO: ARROW_JAVA_SHADE_FLATBUFFERS isn't used for our artifacts. Do
+# we need this?
+# See also: https://github.com/apache/arrow/issues/22021
+if [ "${ARROW_JAVA_SHADE_FLATBUFFERS:-OFF}" == "ON" ]; then
   mvn="${mvn} -Pshade-flatbuffers"
 fi
 
-if [ "${ARROW_JAVA_CDATA}" = "ON" ]; then
+if [ "${ARROW_JAVA_CDATA:-OFF}" = "ON" ]; then
   mvn="${mvn} -Darrow.c.jni.dist.dir=${java_jni_dist_dir} -Parrow-c-data"
 fi
 
-if [ "${ARROW_JAVA_JNI}" = "ON" ]; then
+if [ "${ARROW_JAVA_JNI:-OFF}" = "ON" ]; then
   mvn="${mvn} -Darrow.cpp.build.dir=${java_jni_dist_dir} -Parrow-jni"
 fi
 
 # Use `2 * ncores` threads
 ${mvn} -T 2C clean install
 
-if [ "${BUILD_DOCS_JAVA}" == "ON" ]; then
-  # HTTP pooling is turned of to avoid download issues https://issues.apache.org/jira/browse/ARROW-11633
-  # GH-43378: Maven site plugins not compatible with multithreading
-  mkdir -p "${build_dir}"/docs/java/reference
-  ${mvn} -Dcheckstyle.skip=true -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false clean install site
-  rsync -a target/site/apidocs/ "${build_dir}"/docs/java/reference
+if [ "${ARROW_JAVA_BUILD_DOCS:-OFF}" == "ON" ]; then
+  # HTTP pooling is turned off to avoid download issues:
+  # https://github.com/apache/arrow/issues/27496
+  #
+  # Maven site plugins not compatible with multithreading:
+  # https://github.com/apache/arrow/issues/43378
+  ${mvn} \
+    -Dcheckstyle.skip=true \
+    -Dhttp.keepAlive=false \
+    -Dmaven.wagon.http.pool=false \
+    site
+  rm -rf docs/reference
+  mkdir -p docs
+  cp -a target/site/apidocs/ docs/reference
 fi
 
 popd

ci/scripts/test.sh

@@ -24,7 +24,7 @@ if [[ "${ARROW_JAVA_TEST:-ON}" != "ON" ]]; then
 fi
 
 source_dir=${1}
-build_dir=${2}
+build_dir=${2}/java
 java_jni_dist_dir=${3}
 
 mvn="mvn -B -Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=warn"

compose.yaml

@@ -47,8 +47,30 @@ services:
       - ${DOCKER_VOLUME_PREFIX}maven-cache:/root/.m2:delegated
     command:
       /bin/bash -c "
-        /arrow-java/ci/scripts/build.sh /arrow-java /build &&
-        /arrow-java/ci/scripts/test.sh /arrow-java /build"
+        /arrow-java/ci/scripts/build.sh /arrow-java /build /jni &&
+        /arrow-java/ci/scripts/test.sh /arrow-java /build /jni"
+
+  docs:
+    # Build docs.
+    #
+    # Usage:
+    #   docker compose build docs
+    #   docker compose run docs
+    # Parameters:
+    #   MAVEN: 3.9.9
+    #   JDK: 11, 17, 21
+    image: ${ARCH}/maven:${MAVEN}-eclipse-temurin-${JDK}
+    volumes:
+      - .:/arrow-java:delegated
+      - ${DOCKER_VOLUME_PREFIX}maven-cache:/root/.m2:delegated
+    environment:
+      ARROW_JAVA_BUILD_DOCS: "ON"
+      ARROW_JAVA_SKIP_GIT_PLUGIN: "ON"
+    command:
+      - /arrow-java/ci/scripts/build.sh
+      - /arrow-java
+      - /build
+      - /jni
 
   conda-jni-cdata:
     # Builds and tests just the C Data Interface JNI library and JARs.
@@ -103,7 +125,7 @@ services:
     volumes:
       - .:/arrow-java:delegated
       - ${ARROW_REPO_ROOT}:/arrow:delegated
-      - ${DOCKER_VOLUME_PREFIX}ccache:/ccache:delegated
+      - ${DOCKER_VOLUME_PREFIX}ccache-cache:/ccache:delegated
       - ${DOCKER_VOLUME_PREFIX}maven-cache:/root/.m2:delegated
     environment:
       ARROW_JAVA_CDATA: "ON"