Merge branch 'pivotLongerScalarFeatures' into combined2

Author: Rafał Hibner

.pre-commit-config.yaml

@@ -331,6 +331,7 @@ repos:
           ?^ci/scripts/integration_arrow_build\.sh$|
           ?^ci/scripts/integration_arrow\.sh$|
           ?^ci/scripts/integration_dask\.sh$|
+          ?^ci/scripts/integration_hdfs\.sh$|
           ?^ci/scripts/integration_spark\.sh$|
           ?^ci/scripts/matlab_build\.sh$|
           ?^ci/scripts/msys2_system_clean\.sh$|

c_glib/meson.build

@@ -34,7 +34,7 @@ project(
     #   * 22.04: 0.61.2
     #   * 24.04: 1.3.2
     meson_version: '>=0.61.2',
-    version: '21.0.0-SNAPSHOT',
+    version: '22.0.0-SNAPSHOT',
 )
 
 version = meson.project_version()

c_glib/tool/generate-version-header.py

@@ -140,6 +140,7 @@ def generate_availability_macros(library: str) -> str:
 
 
 ALL_VERSIONS = [
+    (22, 0),
     (21, 0),
     (20, 0),
     (19, 0),

c_glib/vcpkg.json

@@ -1,6 +1,6 @@
 {
   "name": "arrow-glib",
-  "version-string": "21.0.0-SNAPSHOT",
+  "version-string": "22.0.0-SNAPSHOT",
   "$comment:dependencies": "We can enable gobject-introspection again once it's updated",
   "dependencies": [
     "glib",

ci/scripts/PKGBUILD

@@ -18,7 +18,7 @@
 _realname=arrow
 pkgbase=mingw-w64-${_realname}
 pkgname="${MINGW_PACKAGE_PREFIX}-${_realname}"
-pkgver=20.0.0.9000
+pkgver=21.0.0.9000
 pkgrel=8000
 pkgdesc="Apache Arrow is a cross-language development platform for in-memory data (mingw-w64)"
 arch=("any")

ci/scripts/integration_hdfs.sh

@@ -19,10 +19,12 @@
 
 set -e
 
+# shellcheck disable=SC2034
 source_dir=${1}/cpp
 build_dir=${2}/cpp
 
-export CLASSPATH=$($HADOOP_HOME/bin/hadoop classpath --glob)
+HADOOP_CLASSPATH=$("$HADOOP_HOME/bin/hadoop" classpath --glob)
+export CLASSPATH="${HADOOP_CLASSPATH}"
 export HADOOP_CONF_DIR=$HADOOP_HOME/etc/hadoop
 export LIBHDFS3_CONF=$HADOOP_CONF_DIR/hdfs-site.xml
 export ARROW_LIBHDFS3_DIR=$CONDA_PREFIX/lib
@@ -42,7 +44,7 @@ function use_libhdfs_dir() {
 
 # execute cpp tests
 export ARROW_HDFS_TEST_LIBHDFS_REQUIRE=ON
-pushd ${build_dir}
+pushd "${build_dir}"
 
 debug/arrow-io-hdfs-test
 debug/arrow-hdfs-test

cpp/CMakeLists.txt

@@ -96,7 +96,7 @@ if(POLICY CMP0170)
   cmake_policy(SET CMP0170 NEW)
 endif()
 
-set(ARROW_VERSION "21.0.0-SNAPSHOT")
+set(ARROW_VERSION "22.0.0-SNAPSHOT")
 
 string(REGEX MATCH "^[0-9]+\\.[0-9]+\\.[0-9]+" ARROW_BASE_VERSION "${ARROW_VERSION}")
 

cpp/cmake_modules/BuildUtils.cmake

@@ -265,6 +265,7 @@ function(ADD_ARROW_LIB LIB_NAME)
     if(ARG_DEFINITIONS)
       target_compile_definitions(${LIB_NAME}_objlib PRIVATE ${ARG_DEFINITIONS})
     endif()
+    target_compile_options(${LIB_NAME}_objlib PRIVATE ${ARROW_LIBRARIES_ONLY_CXX_FLAGS})
     set(LIB_DEPS $<TARGET_OBJECTS:${LIB_NAME}_objlib>)
     set(EXTRA_DEPS)
 
@@ -326,6 +327,7 @@ function(ADD_ARROW_LIB LIB_NAME)
     if(ARG_DEFINITIONS)
       target_compile_definitions(${LIB_NAME}_shared PRIVATE ${ARG_DEFINITIONS})
     endif()
+    target_compile_options(${LIB_NAME}_shared PRIVATE ${ARROW_LIBRARIES_ONLY_CXX_FLAGS})
 
     if(ARG_OUTPUTS)
       list(APPEND ${ARG_OUTPUTS} ${LIB_NAME}_shared)
@@ -416,6 +418,7 @@ function(ADD_ARROW_LIB LIB_NAME)
     if(ARG_DEFINITIONS)
       target_compile_definitions(${LIB_NAME}_static PRIVATE ${ARG_DEFINITIONS})
     endif()
+    target_compile_options(${LIB_NAME}_static PRIVATE ${ARROW_LIBRARIES_ONLY_CXX_FLAGS})
 
     if(ARG_OUTPUTS)
       list(APPEND ${ARG_OUTPUTS} ${LIB_NAME}_static)

cpp/cmake_modules/SetupCxxFlags.cmake

@@ -159,6 +159,9 @@ set(CMAKE_POSITION_INDEPENDENT_CODE ${ARROW_POSITION_INDEPENDENT_CODE})
 set(UNKNOWN_COMPILER_MESSAGE
     "Unknown compiler: ${CMAKE_CXX_COMPILER_ID} ${CMAKE_CXX_COMPILER_VERSION}")
 
+# Compiler flags used when building Arrow libraries (but not tests, utilities, etc.)
+set(ARROW_LIBRARIES_ONLY_CXX_FLAGS)
+
 # compiler flags that are common across debug/release builds
 if(WIN32)
   # TODO(wesm): Change usages of C runtime functions that MSVC says are
@@ -322,6 +325,9 @@ if("${BUILD_WARNING_LEVEL}" STREQUAL "CHECKIN")
     set(CXX_COMMON_FLAGS "${CXX_COMMON_FLAGS} -Wimplicit-fallthrough")
     string(APPEND CXX_ONLY_FLAGS " -Wredundant-move")
     set(CXX_COMMON_FLAGS "${CXX_COMMON_FLAGS} -Wunused-result")
+    # Flag non-static functions that don't have corresponding declaration in a .h file.
+    # Only for Arrow libraries, since this is not a problem in tests or utilities.
+    list(APPEND ARROW_LIBRARIES_ONLY_CXX_FLAGS "-Wmissing-declarations")
   elseif(CMAKE_CXX_COMPILER_ID STREQUAL "Intel" OR CMAKE_CXX_COMPILER_ID STREQUAL
                                                    "IntelLLVM")
     if(WIN32)

cpp/examples/arrow/parquet_column_encryption.cc

@@ -19,6 +19,7 @@
 #include "arrow/dataset/file_parquet.h"
 #include "arrow/dataset/parquet_encryption_config.h"
 #include "arrow/filesystem/localfs.h"
+#include "arrow/util/secure_string.h"
 #include "parquet/encryption/crypto_factory.h"
 #include "parquet/encryption/test_in_memory_kms.h"
 
@@ -106,9 +107,9 @@ arrow::Result<std::shared_ptr<arrow::Table>> GetTable() {
 
 std::shared_ptr<parquet::encryption::CryptoFactory> GetCryptoFactory() {
   // Configure KMS.
-  std::unordered_map<std::string, std::string> key_map;
-  key_map.emplace("footerKeyId", "0123456789012345");
-  key_map.emplace("columnKeyId", "1234567890123456");
+  std::unordered_map<std::string, arrow::util::SecureString> key_map;
+  key_map.emplace("footerKeyId", arrow::util::SecureString("0123456789012345"));
+  key_map.emplace("columnKeyId", arrow::util::SecureString("1234567890123456"));
 
   auto crypto_factory = std::make_shared<parquet::encryption::CryptoFactory>();
   auto kms_client_factory =