diff --git a/.gitignore b/.gitignore
index 8171952dad..8c7bd135e1 100644
--- a/.gitignore
+++ b/.gitignore
@@ -13,6 +13,7 @@
 .settings/
 /*-build/
 /.mvn/.develocity/
+/apache-arrow-java-*
 /apache-arrow-java.tar.gz
 /build/
 /dev/release/apache-rat-0.16.1.jar
diff --git a/ci/scripts/jni_full_build.sh b/ci/scripts/jni_full_build.sh
index 15cf72f21b..49cbdc138f 100755
--- a/ci/scripts/jni_full_build.sh
+++ b/ci/scripts/jni_full_build.sh
@@ -17,11 +17,12 @@
 # specific language governing permissions and limitations
 # under the License.
 
-set -e
+set -eu
 
 source_dir="$(cd "${1}" && pwd)"
 jni_build_dir="$(cd "${2}" && pwd)"
 dist_dir="${3}"
+rm -rf "${dist_dir}"
 mkdir -p "${dist_dir}"
 dist_dir="$(cd "${dist_dir}" && pwd)"
 
@@ -69,3 +70,8 @@ find ~/.m2/repository/org/apache/arrow \
   ")" \
   -exec echo "{}" ";" \
   -exec cp "{}" "${dist_dir}" ";"
+
+for artifact in "${dist_dir}"/*; do
+  sha256sum "${artifact}" >"${artifact}.sha256"
+  sha512sum "${artifact}" >"${artifact}.sha512"
+done
diff --git a/dev/release/release_rc.sh b/dev/release/release_rc.sh
index c8f1024a3f..de053578e0 100755
--- a/dev/release/release_rc.sh
+++ b/dev/release/release_rc.sh
@@ -80,8 +80,8 @@ fi
 
 rc_hash="$(git rev-list --max-count=1 "${rc_tag}")"
 
-id="apache-arrow-java-${version}"
-tar_gz="${id}.tar.gz"
+artifacts_dir="apache-arrow-java-${version}-rc${rc}"
+signed_artifacts_dir="${artifacts_dir}-signed"
 
 if [ "${RELEASE_SIGN}" -gt 0 ]; then
   git_origin_url="$(git remote get-url origin)"
@@ -105,15 +105,26 @@ if [ "${RELEASE_SIGN}" -gt 0 ]; then
   echo "Found GitHub Actions workflow with ID: ${run_id}"
   gh run watch --repo "${repository}" --exit-status "${run_id}"
 
-  echo "Downloading .tar.gz from GitHub Releases"
+  echo "Downloading artifacts from GitHub Releases"
   gh release download "${rc_tag}" \
-    --dir . \
-    --pattern "${tar_gz}" \
+    --dir "${artifacts_dir}" \
     --repo "${repository}" \
     --skip-existing
 
-  echo "Signing tar.gz and creating checksums"
-  gpg --armor --output "${tar_gz}.asc" --detach-sig "${tar_gz}"
+  echo "Signing artifacts"
+  rm -rf "${signed_artifacts_dir}"
+  mkdir -p "${signed_artifacts_dir}"
+  for artifact in "${artifacts_dir}"/*; do
+    case "${artifact}" in
+    *.asc | *.sha256 | *.sha512)
+      continue
+      ;;
+    esac
+    gpg --armor \
+      --detach-sig \
+      --output "${signed_artifacts_dir}/$(basename "${artifact}").asc" \
+      "${artifact}"
+  done
 fi
 
 if [ "${RELEASE_UPLOAD}" -gt 0 ]; then
@@ -121,7 +132,7 @@ if [ "${RELEASE_UPLOAD}" -gt 0 ]; then
   gh release upload "${rc_tag}" \
     --clobber \
     --repo "${repository}" \
-    "${tar_gz}.asc"
+    "${signed_artifacts_dir}"/*.asc
 fi
 
 echo "Draft email for dev@arrow.apache.org mailing list"