Merge branch 'optionalConversionTraits' into combined2

Author: Rafał Hibner

.pre-commit-config.yaml

@@ -192,16 +192,35 @@ repos:
         # TODO: Remove this when we fix all lint failures
         files: >-
           (
+          ?^c_glib/test/run-test\.sh$|
           ?^ci/scripts/c_glib_build\.sh$|
           ?^ci/scripts/c_glib_test\.sh$|
-          ?^c_glib/test/run-test\.sh$|
+          ?^ci/scripts/conan_setup\.sh$|
+          ?^ci/scripts/download_tz_database\.sh$|
+          ?^ci/scripts/install_azurite\.sh$|
+          ?^ci/scripts/install_ccache\.sh$|
+          ?^ci/scripts/install_ceph\.sh$|
+          ?^ci/scripts/install_spark\.sh$|
+          ?^ci/scripts/integration_dask\.sh$|
+          ?^ci/scripts/msys2_system_upgrade\.sh$|
+          ?^ci/scripts/util_free_space\.sh$|
+          ?^cpp/build-support/build-lz4-lib\.sh$|
+          ?^cpp/build-support/build-zstd-lib\.sh$|
+          ?^cpp/build-support/get-upstream-commit\.sh$|
+          ?^cpp/build-support/update-thrift\.sh$|
+          ?^cpp/examples/minimal_build/run\.sh$|
+          ?^cpp/examples/tutorial_examples/run\.sh$|
           ?^dev/release/05-binary-upload\.sh$|
           ?^dev/release/07-binary-verify\.sh$|
           ?^dev/release/binary-recover\.sh$|
           ?^dev/release/post-03-binary\.sh$|
           ?^dev/release/post-10-docs\.sh$|
           ?^dev/release/post-11-python\.sh$|
+          ?^dev/release/setup-rhel-rebuilds\.sh$|
           ?^dev/release/utils-generate-checksum\.sh$|
+          ?^python/asv-install\.sh$|
+          ?^python/asv-uninstall\.sh$|
+          ?^swift/gen-protobuffers\.sh$|
           )
   - repo: https://github.com/scop/pre-commit-shfmt
     rev: v3.11.0-1

c_glib/arrow-glib/basic-data-type.cpp

@@ -2300,6 +2300,8 @@ garrow_fixed_shape_tensor_data_type_class_init(GArrowFixedShapeTensorDataTypeCla
  * @error: (nullable): Return location for a #GError or %NULL.
  *
  * Returns: The newly created fixed shape tensor data type.
+ *
+ * Since: 21.0.0
  */
 GArrowFixedShapeTensorDataType *
 garrow_fixed_shape_tensor_data_type_new(GArrowDataType *value_type,
@@ -2353,6 +2355,8 @@ garrow_fixed_shape_tensor_data_type_new(GArrowDataType *value_type,
  * @length: (out): Return location for the number of dimensions of the tensor.
  *
  * Returns: (array length=length): Shape of the tensor.
+ *
+ * Since: 21.0.0
  */
 const gint64 *
 garrow_fixed_shape_tensor_data_type_get_shape(GArrowFixedShapeTensorDataType *data_type,
@@ -2372,6 +2376,8 @@ garrow_fixed_shape_tensor_data_type_get_shape(GArrowFixedShapeTensorDataType *da
  * @length: (out): Return location for the number of elements of permutation.
  *
  * Returns: (array length=length): Permutation of the tensor.
+ *
+ * Since: 21.0.0
  */
 const gint64 *
 garrow_fixed_shape_tensor_data_type_get_permutation(
@@ -2394,6 +2400,8 @@ garrow_fixed_shape_tensor_data_type_get_permutation(
  *
  *   It's a %NULL-terminated string array. It must be freed with
  *   g_strfreev() when no longer needed.
+ *
+ * Since: 21.0.0
  */
 gchar **
 garrow_fixed_shape_tensor_data_type_get_dim_names(

ci/scripts/install_ccache.sh

@@ -32,23 +32,23 @@ case $(uname) in
   MINGW64*)
     url="https://github.com/ccache/ccache/releases/download/v${version}/ccache-${version}-windows-x86_64.zip"
     pushd /tmp/ccache
-    curl --fail --location --remote-name ${url}
-    unzip -j ccache-${version}-windows-x86_64.zip
+    curl --fail --location --remote-name "${url}"
+    unzip -j "ccache-${version}-windows-x86_64.zip"
     chmod +x ccache.exe
-    mv ccache.exe ${prefix}/bin/
+    mv ccache.exe "${prefix}/bin/"
     popd
     ;;
   *)
     url="https://github.com/ccache/ccache/archive/v${version}.tar.gz"
 
-    wget -q ${url} -O - | tar -xzf - --directory /tmp/ccache --strip-components=1
+    wget -q "${url}" -O - | tar -xzf - --directory /tmp/ccache --strip-components=1
 
     mkdir /tmp/ccache/build
     pushd /tmp/ccache/build
     cmake \
       -GNinja \
       -DCMAKE_BUILD_TYPE=Release \
-      -DCMAKE_INSTALL_PREFIX=${prefix} \
+      -DCMAKE_INSTALL_PREFIX="${prefix}" \
       -DZSTD_FROM_INTERNET=ON \
       ..
     ninja install

cpp/cmake_modules/FindBrotliAlt.cmake

@@ -21,7 +21,7 @@ if(BrotliAlt_FOUND)
   return()
 endif()
 
-if(ARROW_PACKAGE_KIND STREQUAL "vcpkg" OR ARROW_PACKAGE_KIND STREQUAL "conan")
+if(ARROW_VCPKG OR ARROW_PACKAGE_KIND STREQUAL "conan")
   set(find_package_args "")
   if(BrotliAlt_FIND_VERSION)
     list(APPEND find_package_args ${BrotliAlt_FIND_VERSION})
@@ -32,14 +32,14 @@ if(ARROW_PACKAGE_KIND STREQUAL "vcpkg" OR ARROW_PACKAGE_KIND STREQUAL "conan")
   if(BrotliAlt_FIND_REQUIRED)
     list(APPEND find_package_args REQUIRED)
   endif()
-  if(ARROW_PACKAGE_KIND STREQUAL "vcpkg")
+  if(ARROW_VCPKG)
     find_package(BrotliAlt NAMES unofficial-brotli ${find_package_args})
   else()
     find_package(BrotliAlt NAMES brotli ${find_package_args})
   endif()
   set(Brotli_FOUND ${BrotliAlt_FOUND})
   if(BrotliAlt_FOUND)
-    if(ARROW_PACKAGE_KIND STREQUAL "vcpkg")
+    if(ARROW_VCPKG)
       add_library(Brotli::brotlicommon ALIAS unofficial::brotli::brotlicommon)
       add_library(Brotli::brotlienc ALIAS unofficial::brotli::brotlienc)
       add_library(Brotli::brotlidec ALIAS unofficial::brotli::brotlidec)

cpp/cmake_modules/Findutf8proc.cmake

@@ -19,7 +19,7 @@ if(utf8proc_FOUND)
   return()
 endif()
 
-if(ARROW_PACKAGE_KIND STREQUAL "vcpkg" OR VCPKG_TOOLCHAIN)
+if(ARROW_VCPKG)
   set(find_package_args "")
   if(utf8proc_FIND_VERSION)
     list(APPEND find_package_args ${utf8proc_FIND_VERSION})

cpp/cmake_modules/ThirdpartyToolchain.cmake

@@ -607,8 +607,6 @@ else()
            # our currently used packages and doesn't fall out of sync with
            # ${ARROW_BOOST_BUILD_VERSION_UNDERSCORES}
            "${THIRDPARTY_MIRROR_URL}/boost_${ARROW_BOOST_BUILD_VERSION_UNDERSCORES}.tar.gz"
-           "https://boostorg.jfrog.io/artifactory/main/release/${ARROW_BOOST_BUILD_VERSION}/source/boost_${ARROW_BOOST_BUILD_VERSION_UNDERSCORES}.tar.gz"
-           "https://sourceforge.net/projects/boost/files/boost/${ARROW_BOOST_BUILD_VERSION}/boost_${ARROW_BOOST_BUILD_VERSION_UNDERSCORES}.tar.gz"
   )
 endif()
 
@@ -2652,33 +2650,40 @@ if(ARROW_WITH_ZLIB)
 endif()
 
 macro(build_lz4)
-  message(STATUS "Building LZ4 from source")
+  message(STATUS "Building LZ4 from source using FetchContent")
 
-  set(LZ4_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/lz4_ep-install")
+  # Set LZ4 as vendored
+  set(LZ4_VENDORED TRUE)
 
-  set(LZ4_STATIC_LIB
-      "${LZ4_PREFIX}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}lz4${CMAKE_STATIC_LIBRARY_SUFFIX}")
+  # Declare the content
+  fetchcontent_declare(lz4
+                       URL ${LZ4_SOURCE_URL}
+                       URL_HASH "SHA256=${ARROW_LZ4_BUILD_SHA256_CHECKSUM}"
+                       SOURCE_SUBDIR "build/cmake")
 
-  set(LZ4_CMAKE_ARGS ${EP_COMMON_CMAKE_ARGS} -DCMAKE_INSTALL_PREFIX=<INSTALL_DIR>
-                     -DLZ4_BUILD_CLI=OFF -DLZ4_BUILD_LEGACY_LZ4C=OFF)
+  # Prepare fetch content environment
+  prepare_fetchcontent()
 
-  # We need to copy the header in lib to directory outside of the build
-  externalproject_add(lz4_ep
-                      ${EP_COMMON_OPTIONS}
-                      CMAKE_ARGS ${LZ4_CMAKE_ARGS}
-                      SOURCE_SUBDIR "build/cmake"
-                      INSTALL_DIR ${LZ4_PREFIX}
-                      URL ${LZ4_SOURCE_URL}
-                      URL_HASH "SHA256=${ARROW_LZ4_BUILD_SHA256_CHECKSUM}"
-                      BUILD_BYPRODUCTS ${LZ4_STATIC_LIB})
-
-  file(MAKE_DIRECTORY "${LZ4_PREFIX}/include")
-  add_library(LZ4::lz4 STATIC IMPORTED)
-  set_target_properties(LZ4::lz4 PROPERTIES IMPORTED_LOCATION "${LZ4_STATIC_LIB}")
-  target_include_directories(LZ4::lz4 BEFORE INTERFACE "${LZ4_PREFIX}/include")
-  add_dependencies(LZ4::lz4 lz4_ep)
-
-  list(APPEND ARROW_BUNDLED_STATIC_LIBS LZ4::lz4)
+  # Set LZ4-specific build options as cache variables
+  set(LZ4_BUILD_CLI
+      OFF
+      CACHE BOOL "Don't build LZ4 CLI" FORCE)
+  set(LZ4_BUILD_LEGACY_LZ4C
+      OFF
+      CACHE BOOL "Don't build legacy LZ4 tools" FORCE)
+
+  # Make the dependency available - this will actually perform the download and configure
+  fetchcontent_makeavailable(lz4)
+
+  # Use LZ4::lz4 as an imported library not an alias of lz4_static so other targets such as orc
+  # can depend on it as an external library. External libraries are ignored in
+  # install(TARGETS orc EXPORT orc_targets) and install(EXPORT orc_targets).
+  add_library(LZ4::lz4 INTERFACE IMPORTED)
+  target_link_libraries(LZ4::lz4 INTERFACE lz4_static)
+
+  # Add to bundled static libs.
+  # We must use lz4_static (not imported target) not LZ4::lz4 (imported target).
+  list(APPEND ARROW_BUNDLED_STATIC_LIBS lz4_static)
 endmacro()
 
 if(ARROW_WITH_LZ4)
@@ -2908,7 +2913,7 @@ endmacro()
 
 if(ARROW_WITH_UTF8PROC)
   set(utf8proc_resolve_dependency_args utf8proc PC_PACKAGE_NAMES libutf8proc)
-  if(NOT VCPKG_TOOLCHAIN)
+  if(NOT ARROW_VCPKG)
     # utf8proc in vcpkg doesn't provide version information:
     # https://github.com/microsoft/vcpkg/issues/39176
     list(APPEND utf8proc_resolve_dependency_args REQUIRED_VERSION "2.2.0")
@@ -4589,6 +4594,17 @@ target_include_directories(arrow::hadoop INTERFACE "${HADOOP_HOME}/include")
 function(build_orc)
   message(STATUS "Building Apache ORC from source")
 
+  if(LZ4_VENDORED)
+    set(ORC_LZ4_TARGET lz4_static)
+    set(ORC_LZ4_ROOT "${lz4_SOURCE_DIR}")
+    set(ORC_LZ4_INCLUDE_DIR "${lz4_SOURCE_DIR}/lib")
+  else()
+    set(ORC_LZ4_TARGET LZ4::lz4)
+    get_target_property(ORC_LZ4_INCLUDE_DIR ${ORC_LZ4_TARGET}
+                        INTERFACE_INCLUDE_DIRECTORIES)
+    get_filename_component(ORC_LZ4_ROOT "${ORC_LZ4_INCLUDE_DIR}" DIRECTORY)
+  endif()
+
   if(CMAKE_VERSION VERSION_GREATER_EQUAL 3.29)
     fetchcontent_declare(orc
                          ${FC_DECLARE_COMMON_OPTIONS}
@@ -4601,16 +4617,14 @@ function(build_orc)
     set(ORC_PREFER_STATIC_LZ4
         OFF
         CACHE BOOL "" FORCE)
-    get_target_property(LZ4_INCLUDE_DIR LZ4::lz4 INTERFACE_INCLUDE_DIRECTORIES)
-    if(NOT LZ4_INCLUDE_DIR)
-      find_path(LZ4_INCLUDE_DIR NAMES lz4.h)
-    endif()
-    get_filename_component(LZ4_ROOT "${LZ4_INCLUDE_DIR}" DIRECTORY)
     set(LZ4_HOME
-        "${LZ4_ROOT}"
+        "${ORC_LZ4_ROOT}"
+        CACHE STRING "" FORCE)
+    set(LZ4_INCLUDE_DIR
+        "${ORC_LZ4_INCLUDE_DIR}"
         CACHE STRING "" FORCE)
     set(LZ4_LIBRARY
-        LZ4::lz4
+        ${ORC_LZ4_TARGET}
         CACHE STRING "" FORCE)
 
     set(ORC_PREFER_STATIC_PROTOBUF
@@ -4709,9 +4723,6 @@ function(build_orc)
                         INTERFACE_INCLUDE_DIRECTORIES)
     get_filename_component(ORC_SNAPPY_ROOT "${ORC_SNAPPY_INCLUDE_DIR}" DIRECTORY)
 
-    get_target_property(ORC_LZ4_ROOT LZ4::lz4 INTERFACE_INCLUDE_DIRECTORIES)
-    get_filename_component(ORC_LZ4_ROOT "${ORC_LZ4_ROOT}" DIRECTORY)
-
     get_target_property(ORC_ZSTD_ROOT ${ARROW_ZSTD_LIBZSTD} INTERFACE_INCLUDE_DIRECTORIES)
     get_filename_component(ORC_ZSTD_ROOT "${ORC_ZSTD_ROOT}" DIRECTORY)
 
@@ -4735,9 +4746,9 @@ function(build_orc)
         "-DSNAPPY_HOME=${ORC_SNAPPY_ROOT}"
         "-DSNAPPY_LIBRARY=$<TARGET_FILE:${Snappy_TARGET}>"
         "-DLZ4_HOME=${ORC_LZ4_ROOT}"
-        "-DLZ4_LIBRARY=$<TARGET_FILE:LZ4::lz4>"
-        "-DLZ4_STATIC_LIB=$<TARGET_FILE:LZ4::lz4>"
-        "-DLZ4_INCLUDE_DIR=${ORC_LZ4_ROOT}/include"
+        "-DLZ4_LIBRARY=$<TARGET_FILE:${ORC_LZ4_TARGET}>"
+        "-DLZ4_STATIC_LIB=$<TARGET_FILE:${ORC_LZ4_TARGET}>"
+        "-DLZ4_INCLUDE_DIR=${ORC_LZ4_INCLUDE_DIR}"
         "-DSNAPPY_INCLUDE_DIR=${ORC_SNAPPY_INCLUDE_DIR}"
         "-DZSTD_HOME=${ORC_ZSTD_ROOT}"
         "-DZSTD_INCLUDE_DIR=$<TARGET_PROPERTY:${ARROW_ZSTD_LIBZSTD},INTERFACE_INCLUDE_DIRECTORIES>"