gh-144278: Enable overriding sys.implementation's name and cache_tag when building sysmodule.c (GH-144293)

Changing the values requires forking and patching, which is intentional. Simply rebuilding from source does not change the implementation enough to justify changing these values - they would still be `cpython` and compatible with existing `.pyc` files. But people who maintain forks are better served by being able to easily override these values in a place that can be forward-ported reliably.

Author: zooba

Lib/compileall.py

@@ -165,6 +165,14 @@ def compile_file(fullname, ddir=None, force=False, rx=None, quiet=0,
     stripdir = os.fspath(stripdir) if stripdir is not None else None
     name = os.path.basename(fullname)
 
+    # Without a cache_tag, we can only create legacy .pyc files. None of our
+    # callers seem to expect this, so the best we can do is fail without raising
+    if not legacy and sys.implementation.cache_tag is None:
+        if not quiet:
+            print("No cache tag is available to generate .pyc path for",
+                  repr(fullname))
+        return False
+
     dfile = None
 
     if ddir is not None:

Lib/ensurepip/__init__.py

@@ -177,6 +177,8 @@ def _bootstrap(*, root=None, upgrade=False, user=False,
             args += ["--user"]
         if verbosity:
             args += ["-" + "v" * verbosity]
+        if sys.implementation.cache_tag is None:
+            args += ["--no-compile"]
 
         return _run_pip([*args, "pip"], [os.fsdecode(tmp_wheel_path)])
 

Lib/py_compile.py

@@ -194,8 +194,10 @@ def main():
     else:
         filenames = args.filenames
     for filename in filenames:
+        cfilename = (None if sys.implementation.cache_tag
+                     else f"{filename.rpartition('.')[0]}.pyc")
         try:
-            compile(filename, doraise=True)
+            compile(filename, cfilename, doraise=True)
         except PyCompileError as error:
             if args.quiet:
                 parser.exit(1)

Lib/test/support/import_helper.py

@@ -4,6 +4,7 @@
 import importlib.machinery
 import importlib.util
 import os
+import py_compile
 import shutil
 import sys
 import textwrap
@@ -49,20 +50,31 @@ def forget(modname):
         # combinations of PEP 3147/488 and legacy pyc files.
         unlink(source + 'c')
         for opt in ('', 1, 2):
-            unlink(importlib.util.cache_from_source(source, optimization=opt))
+            try:
+                unlink(importlib.util.cache_from_source(source, optimization=opt))
+            except NotImplementedError:
+                pass
 
 
-def make_legacy_pyc(source):
+def make_legacy_pyc(source, allow_compile=False):
     """Move a PEP 3147/488 pyc file to its legacy pyc location.
 
     :param source: The file system path to the source file.  The source file
-        does not need to exist, however the PEP 3147/488 pyc file must exist.
+        does not need to exist, however the PEP 3147/488 pyc file must exist or
+        allow_compile must be set.
+    :param allow_compile: If True, uses py_compile to create a .pyc if it does
+        not exist. This should be passed as True if cache_tag may be None.
     :return: The file system path to the legacy pyc file.
     """
-    pyc_file = importlib.util.cache_from_source(source)
     assert source.endswith('.py')
     legacy_pyc = source + 'c'
-    shutil.move(pyc_file, legacy_pyc)
+    try:
+        pyc_file = importlib.util.cache_from_source(source)
+        shutil.move(pyc_file, legacy_pyc)
+    except (FileNotFoundError, NotImplementedError):
+        if not allow_compile:
+            raise
+        py_compile.compile(source, legacy_pyc, doraise=True)
     return legacy_pyc
 
 

Lib/test/test_argparse.py

@@ -7,7 +7,6 @@
 import io
 import operator
 import os
-import py_compile
 import shutil
 import stat
 import sys
@@ -7162,9 +7161,8 @@ def make_script(self, dirname, basename, *, compiled=False):
         script_name = script_helper.make_script(dirname, basename, self.source)
         if not compiled:
             return script_name
-        py_compile.compile(script_name, doraise=True)
+        pyc_file = import_helper.make_legacy_pyc(script_name, allow_compile=True)
         os.remove(script_name)
-        pyc_file = import_helper.make_legacy_pyc(script_name)
         return pyc_file
 
     def make_zip_script(self, script_name, name_in_zip=None):

Lib/test/test_capi/test_import.py

@@ -289,7 +289,10 @@ def check_executecode_pathnames(self, execute_code_func, object=False):
             self.check_executecodemodule(execute_code_func, NULL, pathname)
 
         # Test NULL pathname and non-NULL cpathname
-        pyc_filename = importlib.util.cache_from_source(__file__)
+        try:
+            pyc_filename = importlib.util.cache_from_source(__file__)
+        except NotImplementedError:
+            return
         py_filename = importlib.util.source_from_cache(pyc_filename)
         origin = self.check_executecodemodule(execute_code_func, NULL, pyc_filename)
         if not object:

Lib/test/test_cmd_line_script.py

@@ -240,9 +240,8 @@ def test_script_abspath(self):
     def test_script_compiled(self):
         with os_helper.temp_dir() as script_dir:
             script_name = _make_test_script(script_dir, 'script')
-            py_compile.compile(script_name, doraise=True)
+            pyc_file = import_helper.make_legacy_pyc(script_name, allow_compile=True)
             os.remove(script_name)
-            pyc_file = import_helper.make_legacy_pyc(script_name)
             self._check_script(pyc_file, pyc_file,
                                pyc_file, script_dir, None,
                                importlib.machinery.SourcelessFileLoader)
@@ -257,9 +256,8 @@ def test_directory(self):
     def test_directory_compiled(self):
         with os_helper.temp_dir() as script_dir:
             script_name = _make_test_script(script_dir, '__main__')
-            py_compile.compile(script_name, doraise=True)
+            pyc_file = import_helper.make_legacy_pyc(script_name, allow_compile=True)
             os.remove(script_name)
-            pyc_file = import_helper.make_legacy_pyc(script_name)
             self._check_script(script_dir, pyc_file, script_dir,
                                script_dir, '',
                                importlib.machinery.SourcelessFileLoader)
@@ -279,8 +277,8 @@ def test_zipfile(self):
     def test_zipfile_compiled_timestamp(self):
         with os_helper.temp_dir() as script_dir:
             script_name = _make_test_script(script_dir, '__main__')
-            compiled_name = py_compile.compile(
-                script_name, doraise=True,
+            compiled_name = script_name + 'c'
+            py_compile.compile(script_name, compiled_name, doraise=True,
                 invalidation_mode=py_compile.PycInvalidationMode.TIMESTAMP)
             zip_name, run_name = make_zip_script(script_dir, 'test_zip', compiled_name)
             self._check_script(zip_name, run_name, zip_name, zip_name, '',
@@ -289,8 +287,8 @@ def test_zipfile_compiled_timestamp(self):
     def test_zipfile_compiled_checked_hash(self):
         with os_helper.temp_dir() as script_dir:
             script_name = _make_test_script(script_dir, '__main__')
-            compiled_name = py_compile.compile(
-                script_name, doraise=True,
+            compiled_name = script_name + 'c'
+            py_compile.compile(script_name, compiled_name, doraise=True,
                 invalidation_mode=py_compile.PycInvalidationMode.CHECKED_HASH)
             zip_name, run_name = make_zip_script(script_dir, 'test_zip', compiled_name)
             self._check_script(zip_name, run_name, zip_name, zip_name, '',
@@ -299,8 +297,8 @@ def test_zipfile_compiled_checked_hash(self):
     def test_zipfile_compiled_unchecked_hash(self):
         with os_helper.temp_dir() as script_dir:
             script_name = _make_test_script(script_dir, '__main__')
-            compiled_name = py_compile.compile(
-                script_name, doraise=True,
+            compiled_name = script_name + 'c'
+            py_compile.compile(script_name, compiled_name, doraise=True,
                 invalidation_mode=py_compile.PycInvalidationMode.UNCHECKED_HASH)
             zip_name, run_name = make_zip_script(script_dir, 'test_zip', compiled_name)
             self._check_script(zip_name, run_name, zip_name, zip_name, '',
@@ -353,9 +351,8 @@ def test_package_compiled(self):
             pkg_dir = os.path.join(script_dir, 'test_pkg')
             make_pkg(pkg_dir)
             script_name = _make_test_script(pkg_dir, '__main__')
-            compiled_name = py_compile.compile(script_name, doraise=True)
+            pyc_file = import_helper.make_legacy_pyc(script_name, allow_compile=True)
             os.remove(script_name)
-            pyc_file = import_helper.make_legacy_pyc(script_name)
             self._check_script(["-m", "test_pkg"], pyc_file,
                                pyc_file, script_dir, 'test_pkg',
                                importlib.machinery.SourcelessFileLoader,

Lib/test/test_compileall.py

@@ -33,6 +33,10 @@
 from test.support.os_helper import FakePath
 
 
+if sys.implementation.cache_tag is None:
+    raise unittest.SkipTest('requires sys.implementation.cache_tag is not None')
+
+
 def get_pyc(script, opt):
     if not opt:
         # Replace None and 0 with ''

Lib/test/test_ensurepip.py

@@ -12,6 +12,12 @@
 import ensurepip._uninstall
 
 
+if sys.implementation.cache_tag is None:
+    COMPILE_OPT = ["--no-compile"]
+else:
+    COMPILE_OPT = []
+
+
 class TestPackages(unittest.TestCase):
     def touch(self, directory, filename):
         fullname = os.path.join(directory, filename)
@@ -85,7 +91,7 @@ def test_basic_bootstrapping(self):
         self.run_pip.assert_called_once_with(
             [
                 "install", "--no-cache-dir", "--no-index", "--find-links",
-                unittest.mock.ANY, "pip",
+                unittest.mock.ANY, *COMPILE_OPT, "pip",
             ],
             unittest.mock.ANY,
         )
@@ -99,7 +105,7 @@ def test_bootstrapping_with_root(self):
         self.run_pip.assert_called_once_with(
             [
                 "install", "--no-cache-dir", "--no-index", "--find-links",
-                unittest.mock.ANY, "--root", "/foo/bar/",
+                unittest.mock.ANY, "--root", "/foo/bar/", *COMPILE_OPT,
                 "pip",
             ],
             unittest.mock.ANY,
@@ -111,7 +117,7 @@ def test_bootstrapping_with_user(self):
         self.run_pip.assert_called_once_with(
             [
                 "install", "--no-cache-dir", "--no-index", "--find-links",
-                unittest.mock.ANY, "--user", "pip",
+                unittest.mock.ANY, "--user", *COMPILE_OPT, "pip",
             ],
             unittest.mock.ANY,
         )
@@ -122,7 +128,7 @@ def test_bootstrapping_with_upgrade(self):
         self.run_pip.assert_called_once_with(
             [
                 "install", "--no-cache-dir", "--no-index", "--find-links",
-                unittest.mock.ANY, "--upgrade", "pip",
+                unittest.mock.ANY, "--upgrade", *COMPILE_OPT, "pip",
             ],
             unittest.mock.ANY,
         )
@@ -133,7 +139,7 @@ def test_bootstrapping_with_verbosity_1(self):
         self.run_pip.assert_called_once_with(
             [
                 "install", "--no-cache-dir", "--no-index", "--find-links",
-                unittest.mock.ANY, "-v", "pip",
+                unittest.mock.ANY, "-v", *COMPILE_OPT, "pip",
             ],
             unittest.mock.ANY,
         )
@@ -144,7 +150,7 @@ def test_bootstrapping_with_verbosity_2(self):
         self.run_pip.assert_called_once_with(
             [
                 "install", "--no-cache-dir", "--no-index", "--find-links",
-                unittest.mock.ANY, "-vv", "pip",
+                unittest.mock.ANY, "-vv", *COMPILE_OPT, "pip",
             ],
             unittest.mock.ANY,
         )
@@ -155,7 +161,7 @@ def test_bootstrapping_with_verbosity_3(self):
         self.run_pip.assert_called_once_with(
             [
                 "install", "--no-cache-dir", "--no-index", "--find-links",
-                unittest.mock.ANY, "-vvv", "pip",
+                unittest.mock.ANY, "-vvv", *COMPILE_OPT, "pip",
             ],
             unittest.mock.ANY,
         )
@@ -312,7 +318,7 @@ def test_basic_bootstrapping(self):
         self.run_pip.assert_called_once_with(
             [
                 "install", "--no-cache-dir", "--no-index", "--find-links",
-                unittest.mock.ANY, "pip",
+                unittest.mock.ANY, *COMPILE_OPT, "pip",
             ],
             unittest.mock.ANY,
         )

Lib/test/test_import/__init__.py

@@ -77,7 +77,7 @@
 
 
 skip_if_dont_write_bytecode = unittest.skipIf(
-        sys.dont_write_bytecode,
+        sys.dont_write_bytecode or sys.implementation.cache_tag is None,
         "test meaningful only when writing bytecode")
 
 
@@ -505,7 +505,7 @@ def test_module_with_large_stack(self, module='longlist'):
         try:
             # Compile & remove .py file; we only need .pyc.
             # Bytecode must be relocated from the PEP 3147 bytecode-only location.
-            py_compile.compile(filename)
+            make_legacy_pyc(filename, allow_compile=True)
         finally:
             unlink(filename)
 
@@ -515,7 +515,6 @@ def test_module_with_large_stack(self, module='longlist'):
 
         namespace = {}
         try:
-            make_legacy_pyc(filename)
             # This used to crash.
             exec('import ' + module, None, namespace)
         finally:
@@ -1400,7 +1399,10 @@ def func():
 """
     dir_name = os.path.abspath(TESTFN)
     file_name = os.path.join(dir_name, module_name) + os.extsep + "py"
-    compiled_name = importlib.util.cache_from_source(file_name)
+    try:
+        compiled_name = importlib.util.cache_from_source(file_name)
+    except NotImplementedError:
+        compiled_name = None
 
     def setUp(self):
         self.sys_path = sys.path[:]
@@ -1418,7 +1420,8 @@ def tearDown(self):
         else:
             unload(self.module_name)
         unlink(self.file_name)
-        unlink(self.compiled_name)
+        if self.compiled_name:
+            unlink(self.compiled_name)
         rmtree(self.dir_name)
 
     def import_module(self):
@@ -1437,6 +1440,8 @@ def test_basics(self):
         self.assertEqual(mod.code_filename, self.file_name)
         self.assertEqual(mod.func_filename, self.file_name)
 
+    @unittest.skipIf(sys.implementation.cache_tag is None,
+                     'requires sys.implementation.cache_tag is not None')
     def test_incorrect_code_name(self):
         py_compile.compile(self.file_name, dfile="another_module.py")
         mod = self.import_module()
@@ -1446,28 +1451,31 @@ def test_incorrect_code_name(self):
 
     def test_module_without_source(self):
         target = "another_module.py"
-        py_compile.compile(self.file_name, dfile=target)
+        pyc_file = self.file_name + 'c'
+        py_compile.compile(self.file_name, pyc_file, dfile=target)
         os.remove(self.file_name)
-        pyc_file = make_legacy_pyc(self.file_name)
         importlib.invalidate_caches()
         mod = self.import_module()
         self.assertEqual(mod.module_filename, pyc_file)
         self.assertEqual(mod.code_filename, target)
         self.assertEqual(mod.func_filename, target)
 
     def test_foreign_code(self):
-        py_compile.compile(self.file_name)
-        with open(self.compiled_name, "rb") as f:
+        compiled_name = self.compiled_name or (self.file_name + 'c')
+        py_compile.compile(self.file_name, compiled_name)
+        with open(compiled_name, "rb") as f:
             header = f.read(16)
             code = marshal.load(f)
         constants = list(code.co_consts)
         foreign_code = importlib.import_module.__code__
         pos = constants.index(1000)
         constants[pos] = foreign_code
         code = code.replace(co_consts=tuple(constants))
-        with open(self.compiled_name, "wb") as f:
+        with open(compiled_name, "wb") as f:
             f.write(header)
             marshal.dump(code, f)
+        if not self.compiled_name:
+            os.remove(self.file_name)
         mod = self.import_module()
         self.assertEqual(mod.constant.co_filename, foreign_code.co_filename)