GH-36411: [Python] Use scikit-build-core as build backend for PyArrow and get rid of setup.py

.env

@@ -99,8 +99,8 @@ VCPKG="66c0373dc7fca549e5803087b9487edfe3aca0a1"    # 2026.01.16 Release
 # ci/docker/python-*-windows-*.dockerfile or the vcpkg config.
 # This is a workaround for our CI problem that "archery docker build" doesn't
 # use pulled built images in dev/tasks/python-wheels/github.windows.yml.
-PYTHON_WHEEL_WINDOWS_IMAGE_REVISION=2026-02-07
-PYTHON_WHEEL_WINDOWS_TEST_IMAGE_REVISION=2026-02-07
+PYTHON_WHEEL_WINDOWS_IMAGE_REVISION=2026-02-13
+PYTHON_WHEEL_WINDOWS_TEST_IMAGE_REVISION=2026-02-13
 
 # Use conanio/${CONAN_BASE}:{CONAN_VERSION} for "docker compose run --rm conan".
 # See https://github.com/conan-io/conan-docker-tools#readme and

.github/workflows/dev.yml

@@ -103,7 +103,7 @@ jobs:
         shell: bash
         run: |
           gem install test-unit openssl
-          pip install "cython>=3.1" setuptools pytest requests setuptools-scm
+          pip install build "cython>=3.1" pytest requests scikit-build-core setuptools-scm
       - name: Run Release Test
         shell: bash
         run: |

ci/conda_env_python.txt

@@ -29,5 +29,5 @@ numpy>=1.16.6
 pytest
 pytest-faulthandler
 s3fs>=2023.10.0
-setuptools>=77
+scikit-build-core
 setuptools_scm>=8

ci/scripts/python_sdist_build.sh

@@ -23,5 +23,5 @@ source_dir=${1}/python
 
 pushd "${source_dir}"
 export SETUPTOOLS_SCM_PRETEND_VERSION=${PYARROW_VERSION:-}
-${PYTHON:-python} setup.py sdist
+${PYTHON:-python} -m build --sdist
 popd

dev/release/02-source-test.rb

@@ -60,7 +60,7 @@ def test_symbolic_links
   def test_python_version
     source
     Dir.chdir("#{@tag_name_no_rc}/python") do
-      sh("python3", "setup.py", "sdist")
+      sh("python", "-m", "build", "--sdist")
       if on_release_branch?
         pyarrow_source_archive = "dist/pyarrow-#{@release_version}.tar.gz"
       else

python/.gitignore

@@ -44,7 +44,3 @@ manylinux1/arrow
 nm_arrow.log
 visible_symbols.log
 
-# the purpose of the custom SDist class in setup.py is to include these files
-# in the sdist tarball, but we don't want to track duplicates
-LICENSE.txt
-NOTICE.txt

python/CMakeLists.txt

@@ -352,7 +352,21 @@ set(PYARROW_CPP_ROOT_DIR pyarrow/src)
 set(PYARROW_CPP_SOURCE_DIR ${PYARROW_CPP_ROOT_DIR}/arrow/python)
 
 # Write out compile-time configuration constants
-string(TOUPPER ${CMAKE_BUILD_TYPE} UPPERCASE_PYBUILD_TYPE)
+if(CMAKE_BUILD_TYPE)
+  string(TOUPPER "${CMAKE_BUILD_TYPE}" UPPERCASE_PYBUILD_TYPE)
+else()
+  # For multi-config generators (XCode and Visual Studio),
+  # CMAKE_BUILD_TYPE is not set at configure time.
+  # scikit-build-core does the right thing with cmake.build-type and
+  # adds the corresponding --config but does not populate CMAKE_BUILD_TYPE
+  # for those. On this specific case, we set the default to "RELEASE"
+  # as it's the most common build type for users building from source.
+  # This is mainly relevant for our Windows wheels, which are built with
+  # Visual Studio and thus use a multi-config generator with Release.
+  # As a note this is only to populate config_internal.h.cmake.
+  set(UPPERCASE_PYBUILD_TYPE "RELEASE")
+endif()
+
 configure_file("${PYARROW_CPP_SOURCE_DIR}/config_internal.h.cmake"
                "${PYARROW_CPP_SOURCE_DIR}/config_internal.h" ESCAPE_QUOTES)
 

python/LICENSE.txt

@@ -0,0 +1 @@
+../LICENSE.txt

python/NOTICE.txt

@@ -0,0 +1 @@
+../NOTICE.txt

python/_build_backend/__init__.py

@@ -0,0 +1,79 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+"""
+Build backend wrapper that resolves license symlinks before delegating
+to scikit-build-core.
+
+Arrow's LICENSE.txt and NOTICE.txt live at the repository root, one level
+above python/. They are symlinked into python/ so that license-files in
+pyproject.toml can reference them otherwise project metadata fails validation.
+This is done before any build backend is invoked that's why symlinks are necessary.
+But when building sdist tarballs symlinks are not copied and we end up with
+broken LICENSE.txt and NOTICE.txt.
+
+This custom build backend only replace the symlinks with hardlinks
+before scikit_build_core.build.build_sdist so
+that sdist contains the actual file content. The symlinks are restored
+afterwards so the git working tree stays clean.
+"""
+
+from contextlib import contextmanager
+import os
+from pathlib import Path
+import shutil
+import sys
+
+from scikit_build_core.build import *  # noqa: F401,F403
+from scikit_build_core.build import build_sdist as scikit_build_sdist
+
+LICENSE_FILES = ("LICENSE.txt", "NOTICE.txt")
+PYTHON_DIR = Path(__file__).resolve().parent.parent
+
+
+@contextmanager
+def prepare_licenses():
+    # Temporarily replace symlinks with hardlinks so sdist gets real content.
+    # On Windows we just copy the files since hardlinks might not be supported.
+    for name in LICENSE_FILES:
+        parent_license = PYTHON_DIR.parent / name
+        pyarrow_license = PYTHON_DIR / name
+        if sys.platform == "win32":
+            # For Windows copy the files.
+            pyarrow_license.unlink(missing_ok=True)
+            shutil.copy2(parent_license, pyarrow_license)
+        else:
+            # For Unix-like systems we replace the symlink with
+            # a hardlink to avoid copying the file content.
+            if pyarrow_license.is_symlink():
+                target = pyarrow_license.resolve()
+                pyarrow_license.unlink()
+                os.link(target, pyarrow_license)
+    try:
+        yield
+    finally:
+        if sys.platform != "win32":
+            # Copy back the original symlinks so git status is clean.
+            for name in LICENSE_FILES:
+                filepath = PYTHON_DIR / name
+                filepath.unlink()
+                os.symlink(f"../{name}", filepath)
+
+
+def build_sdist(sdist_directory, config_settings=None):
+    with prepare_licenses():
+        return scikit_build_sdist(sdist_directory, config_settings)

python/pyproject.toml

@@ -17,16 +17,17 @@
 
 [build-system]
 requires = [
+    "scikit-build-core",
     "cython >= 3.1",
     # Needed for build-time stub docstring extraction
     "libcst>=1.8.6",
     "numpy>=1.25",
-    # configuring setuptools_scm in pyproject.toml requires
-    # versions released after 2022
     "setuptools_scm[toml]>=8",
-    "setuptools>=77",
 ]
-build-backend = "setuptools.build_meta"
+# We use a really simple build backend wrapper over scikit-build-core
+# to solve licenses to work around links not being included in sdists.
+build-backend = "_build_backend"
+backend-path = ["."]
 
 [project]
 name = "pyarrow"
@@ -81,16 +82,18 @@ exclude = [
     '\._.*$',
 ]
 
-[tool.setuptools]
-zip-safe=false
-include-package-data=true
+[tool.scikit-build]
+cmake.build-type = "Release"
+metadata.version.provider = "scikit_build_core.metadata.setuptools_scm"
+sdist.include = ["pyarrow/_generated_version.py", "cmake_modules/"]
+wheel.packages = ["pyarrow"]
+wheel.install-dir = "pyarrow"
 
-[tool.setuptools.packages.find]
-include = ["pyarrow"]
-namespaces = false
-
-[tool.setuptools.package-data]
-pyarrow = ["*.pxd", "*.pyi", "*.pyx", "includes/*.pxd", "py.typed"]
+[tool.scikit-build.cmake.define]
+PYARROW_BUNDLE_ARROW_CPP = {env = "PYARROW_BUNDLE_ARROW_CPP", default = "OFF"}
+PYARROW_BUNDLE_CYTHON_CPP = {env = "PYARROW_BUNDLE_CYTHON_CPP", default = "OFF"}
+PYARROW_GENERATE_COVERAGE = {env = "PYARROW_GENERATE_COVERAGE", default = "OFF"}
+PYARROW_CXXFLAGS = {env = "PYARROW_CXXFLAGS", default = ""}
 
 [tool.setuptools_scm]
 root = '..'

python/requirements-build.txt

@@ -1,5 +1,6 @@
+build
 cython>=3.1
 libcst>=1.8.6
 numpy>=1.25
+scikit-build-core
 setuptools_scm>=8
-setuptools>=77

python/requirements-wheel-build.txt

@@ -3,6 +3,6 @@ cython>=3.1
 # Needed for build-time stub docstring extraction
 libcst>=1.8.6
 numpy>=2.0.0
+scikit-build-core
 setuptools_scm
-setuptools>=77
 wheel