Hosted stencil changes

Author: savannahostrowski

.gitattributes

@@ -20,6 +20,9 @@
 # Specific binary files
 PC/classicAppCompat.* binary
 
+# JIT stencils are generated by the build and should not be modified
+Tools/jit/stencils/* text eol=lf
+
 # Text files that should not be subject to eol conversion
 [attr]noeol -text
 

.github/workflows/jit.yml

@@ -132,6 +132,71 @@ jobs:
           make all --jobs 4
           ./python -m test --multiprocess 0 --timeout 4500 --verbose2 --verbose3
 
+      - name: Check if stencils need regeneration
+        id: check-stencils
+        if: ${{!matrix.debug }}
+        shell: bash
+        run: |
+          git add Tools/jit/stencils
+
+          if ! git diff --staged --exit-code --quiet; then
+            git diff --staged > jit_stencils.patch
+            exit 1
+          fi
+
+      - name: Format target name
+        if: ${{ failure() && steps.check-stencils.conclusion == 'failure' && !matrix.debug }}
+        id: strip-target
+        shell: bash
+        run: |
+          target=${{ matrix.target }}
+          target="${target%%/*}"
+          echo "target=$target" >> $GITHUB_OUTPUT
+
+      - name: Upload stencil patch
+        if: ${{ failure() && steps.check-stencils.conclusion == 'failure' && !matrix.debug }}
+        uses: actions/upload-artifact@v4
+        with:
+          name: ${{ steps.strip-target.outputs.target }}-jit-stencils
+          path: jit_stencils.patch
+
+  aggregate-stencil-patches:
+    name: Aggregate stencil patches
+    needs: jit
+    runs-on: ubuntu-24.04
+    if: ${{ failure() }}
+    steps:
+      - name: Download stencil artifacts
+        run: |
+          mkdir -p artifacts
+          gh run download ${{ github.run_id }} --pattern '*jit-stencils*' --dir artifacts --repo ${{ github.repository }}
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Aggregate stencil patches
+        run: |
+          temp_file="$(mktemp)"
+          : > "$temp_file"
+
+          find artifacts -mindepth 1 -maxdepth 1 -type d | while read -r dir; do
+              find "$dir" -type f -name "*.patch" -exec cat {} + >> "$temp_file"
+          done
+
+          sed_command="s/[[:space:]]*$//"
+          if [[ "$(uname)" == "Darwin" ]]; then
+              sed -i '' "$sed_command" "$temp_file"
+          else
+              sed -i "$sed_command" "$temp_file"
+          fi
+
+          mv "$temp_file" aggregated_jit_stencils.patch
+
+      - name: Upload aggregated stencil patch
+        uses: actions/upload-artifact@v4
+        with:
+          name: aggregated-jit-stencils
+          path: aggregated_jit_stencils.patch
+
   jit-with-disabled-gil:
     name: Free-Threaded (Debug)
     needs: interpreter

.pre-commit-config.yaml

@@ -33,7 +33,7 @@ repos:
         args: [--line-length=79]
       - id: black
         name: Run Black on Tools/jit/
-        files: ^Tools/jit/
+        files: ^Tools/jit/(?!stencils/).*
         language_version: python3.12
 
   - repo: https://github.com/pre-commit/pre-commit-hooks

PCbuild/regen.targets

@@ -30,7 +30,7 @@
     <_KeywordOutputs Include="$(PySourcePath)Lib\keyword.py" />
     <!-- Taken from _Target._compute_digest in Tools\jit\_targets.py: -->
     <_JITSources Include="$(PySourcePath)Python\executor_cases.c.h;$(GeneratedPyConfigDir)pyconfig.h;$(PySourcePath)Tools\jit\**"/>
-    <_JITOutputs Include="$(GeneratedPyConfigDir)jit_stencils.h"/>
+    <_JITOutputs Include="$(GeneratedPyConfigDir)jit_stencils.h;$(PySourcePath)Tools\jit\stencils\**"/>
     <_CasesSources Include="$(PySourcePath)Python\bytecodes.c;$(PySourcePath)Python\optimizer_bytecodes.c;"/>
     <_CasesOutputs Include="$(PySourcePath)Python\generated_cases.c.h;$(PySourcePath)Include\opcode_ids.h;$(PySourcePath)Include\internal\pycore_uop_ids.h;$(PySourcePath)Python\opcode_targets.h;$(PySourcePath)Include\internal\pycore_opcode_metadata.h;$(PySourcePath)Include\internal\pycore_uop_metadata.h;$(PySourcePath)Python\optimizer_cases.c.h;$(PySourcePath)Lib\_opcode_metadata.py"/>
     <_SbomSources Include="$(PySourcePath)PCbuild\get_externals.bat" />

Tools/jit/_targets.py

@@ -2,11 +2,10 @@
 
 import asyncio
 import dataclasses
-import hashlib
 import json
-import os
 import pathlib
 import re
+import shutil
 import sys
 import tempfile
 import typing
@@ -44,18 +43,6 @@ class _Target(typing.Generic[_S, _R]):
     verbose: bool = False
     known_symbols: dict[str, int] = dataclasses.field(default_factory=dict)
 
-    def _compute_digest(self, out: pathlib.Path) -> str:
-        hasher = hashlib.sha256()
-        hasher.update(self.triple.encode())
-        hasher.update(self.debug.to_bytes())
-        # These dependencies are also reflected in _JITSources in regen.targets:
-        hasher.update(PYTHON_EXECUTOR_CASES_C_H.read_bytes())
-        hasher.update((out / "pyconfig.h").read_bytes())
-        for dirpath, _, filenames in sorted(os.walk(TOOLS_JIT)):
-            for filename in filenames:
-                hasher.update(pathlib.Path(dirpath, filename).read_bytes())
-        return hasher.hexdigest()
-
     async def _parse(self, path: pathlib.Path) -> _stencils.StencilGroup:
         group = _stencils.StencilGroup()
         args = ["--disassemble", "--reloc", f"{path}"]
@@ -176,35 +163,28 @@ async def _build_stencils(self) -> dict[str, _stencils.StencilGroup]:
             )
         return stencil_groups
 
-    def build(
-        self, out: pathlib.Path, *, comment: str = "", force: bool = False
-    ) -> None:
+    def build(self, out: pathlib.Path, *, force: bool = False) -> None:
         """Build jit_stencils.h in the given directory."""
         if not self.stable:
             warning = f"JIT support for {self.triple} is still experimental!"
             request = "Please report any issues you encounter.".center(len(warning))
             outline = "=" * len(warning)
             print("\n".join(["", outline, warning, request, outline, ""]))
-        digest = f"// {self._compute_digest(out)}\n"
-        jit_stencils = out / "jit_stencils.h"
-        if (
-            not force
-            and jit_stencils.exists()
-            and jit_stencils.read_text().startswith(digest)
-        ):
-            return
+        target = self.triple
+        darwin_index = target.find("-darwin")
+        if darwin_index != -1:
+            target = self.triple[: darwin_index + len("-darwin")]
+
+        jit_stencils = CPYTHON / "Tools" / "jit" / "stencils" / f"{target}.h"
         stencil_groups = asyncio.run(self._build_stencils())
-        jit_stencils_new = out / "jit_stencils.h.new"
+        jit_stencils_new = CPYTHON / "Tools" / "jit" / "stencils" / f"{target}.h.new"
         try:
-            with jit_stencils_new.open("w") as file:
-                file.write(digest)
-                if comment:
-                    file.write(f"// {comment}\n")
-                file.write("\n")
+            with jit_stencils_new.open("w", newline="\n") as file:
                 for line in _writer.dump(stencil_groups, self.known_symbols):
                     file.write(f"{line}\n")
             try:
                 jit_stencils_new.replace(jit_stencils)
+                shutil.copy(jit_stencils, out / "jit_stencils.h")
             except FileNotFoundError:
                 # another process probably already moved the file
                 if not jit_stencils.is_file():

Tools/jit/_writer.py

@@ -48,7 +48,7 @@ def _dump_stencil(opname: str, group: _stencils.StencilGroup) -> typing.Iterator
     yield "{"
     for part, stencil in [("code", group.code), ("data", group.data)]:
         for line in stencil.disassembly:
-            yield f"    // {line}"
+            yield f"    // {line}".rstrip()
         if stencil.body:
             yield f"    const unsigned char {part}_body[{len(stencil.body)}] = {{"
             for i in range(0, len(stencil.body), 8):

Tools/jit/build.py

@@ -2,13 +2,10 @@
 
 import argparse
 import pathlib
-import shlex
-import sys
 
 import _targets
 
 if __name__ == "__main__":
-    comment = f"$ {shlex.join([pathlib.Path(sys.executable).name] + sys.argv)}"
     parser = argparse.ArgumentParser(description=__doc__)
     parser.add_argument(
         "target", type=_targets.get_target, help="a PEP 11 target triple to compile for"
@@ -25,4 +22,4 @@
     args = parser.parse_args()
     args.target.debug = args.debug
     args.target.verbose = args.verbose
-    args.target.build(pathlib.Path.cwd(), comment=comment, force=args.force)
+    args.target.build(pathlib.Path.cwd(), force=args.force)