Merge branch 'main' into arm-runners

Author: diegorusso

.github/workflows/jit.yml

@@ -54,9 +54,7 @@ jobs:
           - x86_64-apple-darwin/clang
           - aarch64-apple-darwin/clang
           - x86_64-unknown-linux-gnu/gcc
-          - x86_64-unknown-linux-gnu/clang
           - aarch64-unknown-linux-gnu/gcc
-          - aarch64-unknown-linux-gnu/clang
         debug:
           - true
           - false
@@ -68,46 +66,27 @@ jobs:
           - target: i686-pc-windows-msvc/msvc
             architecture: Win32
             runner: windows-latest
-            compiler: msvc
           - target: x86_64-pc-windows-msvc/msvc
             architecture: x64
             runner: windows-latest
-            compiler: msvc
           - target: aarch64-pc-windows-msvc/msvc
             architecture: ARM64
             runner: windows-latest
-            compiler: msvc
           - target: x86_64-apple-darwin/clang
             architecture: x86_64
             runner: macos-13
-            compiler: clang
           - target: aarch64-apple-darwin/clang
             architecture: aarch64
             runner: macos-14
-            compiler: clang
           - target: x86_64-unknown-linux-gnu/gcc
             architecture: x86_64
             runner: ubuntu-24.04
-            compiler: gcc
-          - target: x86_64-unknown-linux-gnu/clang
-            architecture: x86_64
-            runner: ubuntu-24.04
-            compiler: clang
           - target: aarch64-unknown-linux-gnu/gcc
             architecture: aarch64
             runner: ubuntu-24.04-aarch64
-            compiler: gcc
-          - target: aarch64-unknown-linux-gnu/clang
-            architecture: aarch64
-            runner: ubuntu-24.04-aarch64
-            compiler: clang
         exclude:
         - target: aarch64-unknown-linux-gnu/gcc
           is-fork: true
-        - target: aarch64-unknown-linux-gnu/clang
-          is-fork: true
-    env:
-      CC: ${{ matrix.compiler }}
     steps:
       - uses: actions/checkout@v4
       - uses: actions/setup-python@v5
@@ -118,10 +97,10 @@ jobs:
         if: runner.os == 'Windows' && matrix.architecture != 'ARM64'
         run: |
           choco install llvm --allow-downgrade --no-progress --version ${{ matrix.llvm }}.1.0
-          ./PCbuild/build.bat --experimental-jit ${{ matrix.debug && '-d' || '--pgo' }} -p ${{ matrix.architecture }}
+          ./PCbuild/build.bat --experimental-jit ${{ matrix.debug && '-d' || '' }} -p ${{ matrix.architecture }}
           ./PCbuild/rt.bat ${{ matrix.debug && '-d' || '' }} -p ${{ matrix.architecture }} -q --multiprocess 0 --timeout 4500 --verbose2 --verbose3
 
-      # No PGO or tests (yet):
+      # No tests (yet):
       - name: Emulated Windows
         if: runner.os == 'Windows' && matrix.architecture == 'ARM64'
         run: |
@@ -139,7 +118,7 @@ jobs:
           find /usr/local/bin -lname '*/Library/Frameworks/Python.framework/*' -delete
           brew install llvm@${{ matrix.llvm }}
           export SDKROOT="$(xcrun --show-sdk-path)"
-          ./configure --enable-experimental-jit ${{ matrix.debug && '--with-pydebug' || '--enable-optimizations --with-lto' }}
+          ./configure --enable-experimental-jit ${{ matrix.debug && '--with-pydebug' || '' }}
           make all --jobs 4
           ./python.exe -m test --multiprocess 0 --timeout 4500 --verbose2 --verbose3
 
@@ -148,7 +127,7 @@ jobs:
         run: |
           sudo bash -c "$(wget -O - https://apt.llvm.org/llvm.sh)" ./llvm.sh ${{ matrix.llvm }}
           export PATH="$(llvm-config-${{ matrix.llvm }} --bindir):$PATH"
-          ./configure --enable-experimental-jit ${{ matrix.debug && '--with-pydebug' || '--enable-optimizations --with-lto' }}
+          ./configure --enable-experimental-jit ${{ matrix.debug && '--with-pydebug' || '' }}
           make all --jobs 4
           ./python -m test --multiprocess 0 --timeout 4500 --verbose2 --verbose3
 

Lib/sysconfig/__main__.py

@@ -1,5 +1,7 @@
+import json
 import os
 import sys
+import types
 from sysconfig import (
     _ALWAYS_STR,
     _PYTHON_BUILD,
@@ -157,6 +159,19 @@ def _print_config_dict(d, stream):
     print ("}", file=stream)
 
 
+def _get_pybuilddir():
+    pybuilddir = f'build/lib.{get_platform()}-{get_python_version()}'
+    if hasattr(sys, "gettotalrefcount"):
+        pybuilddir += '-pydebug'
+    return pybuilddir
+
+
+def _get_json_data_name():
+    name = _get_sysconfigdata_name()
+    assert name.startswith('_sysconfigdata')
+    return name.replace('_sysconfigdata', '_sysconfig_vars') + '.json'
+
+
 def _generate_posix_vars():
     """Generate the Python module containing build-time variables."""
     vars = {}
@@ -185,6 +200,8 @@ def _generate_posix_vars():
     if _PYTHON_BUILD:
         vars['BLDSHARED'] = vars['LDSHARED']
 
+    name = _get_sysconfigdata_name()
+
     # There's a chicken-and-egg situation on OS X with regards to the
     # _sysconfigdata module after the changes introduced by #15298:
     # get_config_vars() is called by get_platform() as part of the
@@ -196,16 +213,13 @@ def _generate_posix_vars():
     # _sysconfigdata module manually and populate it with the build vars.
     # This is more than sufficient for ensuring the subsequent call to
     # get_platform() succeeds.
-    name = _get_sysconfigdata_name()
-    if 'darwin' in sys.platform:
-        import types
-        module = types.ModuleType(name)
-        module.build_time_vars = vars
-        sys.modules[name] = module
+    # GH-127178: Since we started generating a .json file, we also need this to
+    #            be able to run sysconfig.get_config_vars().
+    module = types.ModuleType(name)
+    module.build_time_vars = vars
+    sys.modules[name] = module
 
-    pybuilddir = f'build/lib.{get_platform()}-{get_python_version()}'
-    if hasattr(sys, "gettotalrefcount"):
-        pybuilddir += '-pydebug'
+    pybuilddir = _get_pybuilddir()
     os.makedirs(pybuilddir, exist_ok=True)
     destfile = os.path.join(pybuilddir, name + '.py')
 
@@ -215,6 +229,11 @@ def _generate_posix_vars():
         f.write('build_time_vars = ')
         _print_config_dict(vars, stream=f)
 
+    # Write a JSON file with the output of sysconfig.get_config_vars
+    jsonfile = os.path.join(pybuilddir, _get_json_data_name())
+    with open(jsonfile, 'w') as f:
+        json.dump(get_config_vars(), f, indent=2)
+
     # Create file used for sys.path fixup -- see Modules/getpath.c
     with open('pybuilddir.txt', 'w', encoding='utf8') as f:
         f.write(pybuilddir)

Lib/test/test_sysconfig.py

@@ -11,6 +11,7 @@
 
 from test.support import (
     captured_stdout,
+    is_android,
     is_apple_mobile,
     is_wasi,
     PythonSymlink,
@@ -25,8 +26,9 @@
 from sysconfig import (get_paths, get_platform, get_config_vars,
                        get_path, get_path_names, _INSTALL_SCHEMES,
                        get_default_scheme, get_scheme_names, get_config_var,
-                       _expand_vars, _get_preferred_schemes)
-from sysconfig.__main__ import _main, _parse_makefile
+                       _expand_vars, _get_preferred_schemes,
+                       is_python_build, _PROJECT_BASE)
+from sysconfig.__main__ import _main, _parse_makefile, _get_pybuilddir, _get_json_data_name
 import _imp
 import _osx_support
 import _sysconfig
@@ -39,6 +41,7 @@ class TestSysConfig(unittest.TestCase):
 
     def setUp(self):
         super(TestSysConfig, self).setUp()
+        self.maxDiff = None
         self.sys_path = sys.path[:]
         # patching os.uname
         if hasattr(os, 'uname'):
@@ -625,6 +628,32 @@ def test_makefile_overwrites_config_vars(self):
         self.assertNotEqual(data['prefix'], data['base_prefix'])
         self.assertNotEqual(data['exec_prefix'], data['base_exec_prefix'])
 
+    @unittest.skipIf(os.name != 'posix', '_sysconfig-vars JSON file is only available on POSIX')
+    @unittest.skipIf(is_wasi, "_sysconfig-vars JSON file currently isn't available on WASI")
+    @unittest.skipIf(is_android or is_apple_mobile, 'Android and iOS change the prefix')
+    def test_sysconfigdata_json(self):
+        if '_PYTHON_SYSCONFIGDATA_PATH' in os.environ:
+            data_dir = os.environ['_PYTHON_SYSCONFIGDATA_PATH']
+        elif is_python_build():
+            data_dir = os.path.join(_PROJECT_BASE, _get_pybuilddir())
+        else:
+            data_dir = sys._stdlib_dir
+
+        json_data_path = os.path.join(data_dir, _get_json_data_name())
+
+        with open(json_data_path) as f:
+            json_config_vars = json.load(f)
+
+        system_config_vars = get_config_vars()
+
+        # Ignore keys in the check
+        for key in ('projectbase', 'srcdir'):
+            json_config_vars.pop(key)
+            system_config_vars.pop(key)
+
+        self.assertEqual(system_config_vars, json_config_vars)
+
+
 class MakefileTests(unittest.TestCase):
 
     @unittest.skipIf(sys.platform.startswith('win'),

Makefile.pre.in

@@ -2645,8 +2645,8 @@ libinstall:	all $(srcdir)/Modules/xxmodule.c
 			esac; \
 		done; \
 	done
-	$(INSTALL_DATA) `cat pybuilddir.txt`/_sysconfigdata_$(ABIFLAGS)_$(MACHDEP)_$(MULTIARCH).py \
-		$(DESTDIR)$(LIBDEST); \
+	$(INSTALL_DATA) `cat pybuilddir.txt`/_sysconfigdata_$(ABIFLAGS)_$(MACHDEP)_$(MULTIARCH).py $(DESTDIR)$(LIBDEST); \
+	$(INSTALL_DATA) `cat pybuilddir.txt`/_sysconfig_vars_$(ABIFLAGS)_$(MACHDEP)_$(MULTIARCH).json $(DESTDIR)$(LIBDEST); \
 	$(INSTALL_DATA) $(srcdir)/LICENSE $(DESTDIR)$(LIBDEST)/LICENSE.txt
 	@ # If app store compliance has been configured, apply the patch to the
 	@ # installed library code. The patch has been previously validated against

Misc/NEWS.d/next/Library/2024-11-26-17-42-00.gh-issue-127178.U8hxjc.rst

@@ -0,0 +1,4 @@
+A ``_sysconfig_vars_(...).json`` file is now shipped in the standard library
+directory. It contains the output of :func:`sysconfig.get_config_vars` on
+the default environment encoded as JSON data. This is an implementation
+detail, and may change at any time.