First draft of supporting fat builds on macOS with the experimental JIT

Author: ronaldoussoren

Tools/jit/_targets.py

@@ -25,6 +25,8 @@
 PYTHON_EXECUTOR_CASES_C_H = CPYTHON / "Python" / "executor_cases.c.h"
 TOOLS_JIT_TEMPLATE_C = TOOLS_JIT / "template.c"
 
+ASYNCIO_RUNNER=asyncio.Runner()
+
 
 _S = typing.TypeVar("_S", _schema.COFFSection, _schema.ELFSection, _schema.MachOSection)
 _R = typing.TypeVar(
@@ -106,6 +108,7 @@ async def _compile(
         o = tempdir / f"{opname}.o"
         args = [
             f"--target={self.triple}",
+            "-isysroot", "/Users/ronald/Applications/Xcode-beta.app/Contents/Developer/Platforms/MacOSX.platform/Developer/SDKs/MacOSX.sdk",
             "-DPy_BUILD_CORE",
             "-D_DEBUG" if self.debug else "-DNDEBUG",
             f"-D_JIT_OPCODE={opname}",
@@ -152,17 +155,17 @@ async def _build_stencils(self) -> dict[str, _stencils.StencilGroup]:
                     tasks.append(group.create_task(coro, name=opname))
         return {task.get_name(): task.result() for task in tasks}
 
-    def build(self, out: pathlib.Path, *, comment: str = "") -> None:
+    def build(self, out: pathlib.Path, *, comment: str = "", stencils_h: str = "jit_stencils.h") -> None:
         """Build jit_stencils.h in the given directory."""
         digest = f"// {self._compute_digest(out)}\n"
-        jit_stencils = out / "jit_stencils.h"
+        jit_stencils = out / stencils_h
         if (
             not self.force
             and jit_stencils.exists()
             and jit_stencils.read_text().startswith(digest)
         ):
             return
-        stencil_groups = asyncio.run(self._build_stencils())
+        stencil_groups = ASYNCIO_RUNNER.run(self._build_stencils())
         with jit_stencils.open("w") as file:
             file.write(digest)
             if comment:

Tools/jit/build.py

@@ -10,7 +10,7 @@
     comment = f"$ {shlex.join([sys.executable] + sys.argv)}"
     parser = argparse.ArgumentParser(description=__doc__)
     parser.add_argument(
-        "target", type=_targets.get_target, help="a PEP 11 target triple to compile for"
+        "target", nargs="+", type=_targets.get_target, help="a PEP 11 target triple to compile for"
     )
     parser.add_argument(
         "-d", "--debug", action="store_true", help="compile for a debug build of Python"
@@ -22,7 +22,30 @@
         "-v", "--verbose", action="store_true", help="echo commands as they are run"
     )
     args = parser.parse_args()
-    args.target.debug = args.debug
-    args.target.force = args.force
-    args.target.verbose = args.verbose
-    args.target.build(pathlib.Path.cwd(), comment=comment)
+
+    if len(args.target) == -1:
+        args.target.debug = args.debug
+        args.target.force = args.force
+        args.target.verbose = args.verbose
+        args.target.build(pathlib.Path.cwd(), comment=comment)
+
+    else:
+        # Multiple triples specified, assume this is a macOS multiarchitecture build
+        # - Generate multiple stencil headers
+        # - Generate a helper header that include sthe stencils for the current
+        #   architecture.
+        for target in args.target:
+            target.debug = args.debug
+            target.force = args.force
+            target.verbose = args.verbose
+            target.build(pathlib.Path.cwd(), comment=comment, stencils_h=f"jit_stencils-{target.triple}.h")
+
+        with open("jit_stencils.h", "w") as fp:
+            for idx, target in enumerate(args.target):
+                cpu, _, _ = target.triple.partition("-")
+                fp.write(f"#{'if' if idx == 0 else 'elif'} defined(__{cpu}__)\n")
+                fp.write(f'#   include "jit_stencils-{target.triple}.h"\n')
+
+            fp.write("#else\n")
+            fp.write('#  error "unexpected cpu type"\n')
+            fp.write("#endif\n")

configure

@@ -1621,27 +1621,6 @@ else
   AC_MSG_RESULT([no])
 fi
 
-# Check for --enable-experimental-jit:
-AC_MSG_CHECKING([for --enable-experimental-jit])
-AC_ARG_ENABLE([experimental-jit],
-              [AS_HELP_STRING([--enable-experimental-jit],
-                              [build the experimental just-in-time compiler (default is no)])],
-              [],
-              [enable_experimental_jit=no])
-AS_VAR_IF([enable_experimental_jit],
-          [no],
-          [],
-          [AS_VAR_APPEND([CFLAGS_NODIST], [" -D_Py_JIT"])
-           AS_VAR_SET([REGEN_JIT_COMMAND],
-                      ["\$(PYTHON_FOR_REGEN) \$(srcdir)/Tools/jit/build.py $host"])
-           AS_VAR_SET([JIT_STENCILS_H], ["jit_stencils.h"])
-           AS_VAR_IF([Py_DEBUG],
-                     [true],
-                     [AS_VAR_APPEND([REGEN_JIT_COMMAND], [" --debug"])],
-                     [])])
-AC_SUBST([REGEN_JIT_COMMAND])
-AC_SUBST([JIT_STENCILS_H])
-AC_MSG_RESULT([$enable_experimental_jit])
 
 # Enable optimization flags
 AC_SUBST([DEF_MAKE_ALL_RULE])
@@ -2436,42 +2415,50 @@ yes)
                UNIVERSAL_ARCH_FLAGS="-arch ppc -arch i386"
                LIPO_32BIT_FLAGS=""
                ARCH_RUN_32BIT=""
+	       ARCH_TRIPPLES=`echo {ppc,i386}-apple-darwin`
                ;;
             64-bit)
                UNIVERSAL_ARCH_FLAGS="-arch ppc64 -arch x86_64"
                LIPO_32BIT_FLAGS=""
                ARCH_RUN_32BIT="true"
+	       ARCH_TRIPPLES=`echo {ppc64,x86_64}-apple-darwin`
                ;;
             all)
                UNIVERSAL_ARCH_FLAGS="-arch i386 -arch ppc -arch ppc64 -arch x86_64"
                LIPO_32BIT_FLAGS="-extract ppc7400 -extract i386"
                ARCH_RUN_32BIT="/usr/bin/arch -i386 -ppc"
+	       ARCH_TRIPPLES=`echo {i386,ppc,ppc64,x86_64}-apple-darwin`
                ;;
             universal2)
                UNIVERSAL_ARCH_FLAGS="-arch arm64 -arch x86_64"
                LIPO_32BIT_FLAGS=""
                LIPO_INTEL64_FLAGS="-extract x86_64"
                ARCH_RUN_32BIT="true"
+	       ARCH_TRIPPLES=`echo {aarch64,x86_64}-apple-darwin`
                ;;
             intel)
                UNIVERSAL_ARCH_FLAGS="-arch i386 -arch x86_64"
                LIPO_32BIT_FLAGS="-extract i386"
                ARCH_RUN_32BIT="/usr/bin/arch -i386"
+	       ARCH_TRIPPLES=`echo {i386,x86_64}-apple-darwin`
                ;;
             intel-32)
                UNIVERSAL_ARCH_FLAGS="-arch i386"
                LIPO_32BIT_FLAGS=""
                ARCH_RUN_32BIT=""
+	       ARCH_TRIPPLES=i386-apple-darwin
                ;;
             intel-64)
                UNIVERSAL_ARCH_FLAGS="-arch x86_64"
                LIPO_32BIT_FLAGS=""
                ARCH_RUN_32BIT="true"
+	       ARCH_TRIPPLES=x86_64-apple-darwin
                ;;
             3-way)
                UNIVERSAL_ARCH_FLAGS="-arch i386 -arch ppc -arch x86_64"
                LIPO_32BIT_FLAGS="-extract ppc7400 -extract i386"
                ARCH_RUN_32BIT="/usr/bin/arch -i386 -ppc"
+	       ARCH_TRIPPLES=`echo {i386,ppc,x86_64}-apple-darwin`
                ;;
             *)
                AC_MSG_ERROR([proper usage is --with-universal-arch=universal2|32-bit|64-bit|all|intel|3-way])
@@ -2565,6 +2552,28 @@ yes)
     ;;
 esac
 
+# Check for --enable-experimental-jit:
+AC_MSG_CHECKING([for --enable-experimental-jit])
+AC_ARG_ENABLE([experimental-jit],
+              [AS_HELP_STRING([--enable-experimental-jit],
+                              [build the experimental just-in-time compiler (default is no)])],
+              [],
+              [enable_experimental_jit=no])
+AS_VAR_IF([enable_experimental_jit],
+          [no],
+          [],
+          [AS_VAR_APPEND([CFLAGS_NODIST], [" -D_Py_JIT"])
+           AS_VAR_SET([REGEN_JIT_COMMAND],
+                      ["\$(PYTHON_FOR_REGEN) \$(srcdir)/Tools/jit/build.py ${ARCH_TRIPPLES:-$host}"])
+           AS_VAR_SET([JIT_STENCILS_H], ["jit_stencils.h"])
+           AS_VAR_IF([Py_DEBUG],
+                     [true],
+                     [AS_VAR_APPEND([REGEN_JIT_COMMAND], [" --debug"])],
+                     [])])
+AC_SUBST([REGEN_JIT_COMMAND])
+AC_SUBST([JIT_STENCILS_H])
+AC_MSG_RESULT([$enable_experimental_jit])
+
 case "$CC_BASENAME" in
 *mpicc*)
     CFLAGS_NODIST="$CFLAGS_NODIST"