Basic trace recording

Author: Fidget-Spinner

Include/internal/pycore_interp_structs.h

@@ -943,6 +943,9 @@ struct _is {
     struct types_state types;
     struct callable_cache callable_cache;
     PyObject *common_consts[NUM_COMMON_CONSTANTS];
+    int jit_tracer_code_curr_size;
+    _Py_CODEUNIT *jit_tracer_code_buffer;
+    _Py_CODEUNIT *jit_tracer_initial_instr;
     bool jit;
     bool compiling;
     struct _PyUOpInstruction *jit_uop_buffer;

Include/internal/pycore_uop.h

@@ -35,10 +35,15 @@ typedef struct _PyUOpInstruction{
 #endif
 } _PyUOpInstruction;
 
-// This is the length of the trace we project initially.
+// This is the length of the trace we translate initially.
 #define UOP_MAX_TRACE_LENGTH 1200
 #define UOP_BUFFER_SIZE (UOP_MAX_TRACE_LENGTH * sizeof(_PyUOpInstruction))
 
+// This is the length of the trace we record.
+// This includes the inline caches.
+#define TRACE_MAX_TRACE_LENGTH 1000
+#define TRACER_BUFFER_SIZE ((int)(TRACE_MAX_TRACE_LENGTH * sizeof(_Py_CODEUNIT)))
+
 #ifdef __cplusplus
 }
 #endif

Python/bytecodes.c

@@ -2960,19 +2960,26 @@ dummy_func(
                     oparg >>= 8;
                     start--;
                 }
-                _PyExecutorObject *executor;
-                int optimized = _PyOptimizer_Optimize(frame, start, &executor, 0);
-                if (optimized <= 0) {
-                    this_instr[1].counter = restart_backoff_counter(counter);
-                    ERROR_IF(optimized < 0);
+                if (tstate->interp->jit_tracer_code_buffer == NULL) {
+                    tstate->interp->jit_tracer_code_buffer = (_Py_CODEUNIT *)_PyObject_VirtualAlloc(TRACER_BUFFER_SIZE);
+                    tstate->interp->jit_tracer_code_curr_size = 0;
+                    if (tstate->interp->jit_tracer_code_buffer == NULL) {
+                        // Don't error, just go to next instruction.
+                        DISPATCH();
+                    }
                 }
-                else {
-                    this_instr[1].counter = initial_jump_backoff_counter();
-                    assert(tstate->current_executor == NULL);
-                    assert(executor != tstate->interp->cold_executor);
-                    tstate->jit_exit = NULL;
-                    TIER1_TO_TIER2(executor);
+                if (this_instr == tstate->interp->jit_tracer_initial_instr) {
+                    // Looped back to initial instr. End tracing.
+                    LEAVE_TRACING();
+                    DISPATCH();
+                }
+                ENTER_TRACING();
+                // Nothing in the buffer, begin tracing!
+                if (tstate->interp->jit_tracer_code_curr_size == 0) {
+                    tstate->interp->jit_tracer_initial_instr = this_instr;
                 }
+                // Not tracing dispatch, normal dispatch because we don't record the current instruction.
+                DISPATCH();
             }
             else {
                 ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter);

Python/ceval.c

@@ -973,6 +973,21 @@ _PyObjectArray_Free(PyObject **array, PyObject **scratch)
     }
 }
 
+// 1 for trace full, 0 for successful write.
+static inline int
+add_to_code_trace(PyThreadState *tstate, _Py_CODEUNIT *this_instr)
+{
+    assert(tstate->interp->jit_tracer_code_curr_size < TRACE_MAX_TRACE_LENGTH);
+    int curr_size = tstate->interp->jit_tracer_code_curr_size;
+    int nsize = _PyOpcode_Caches[this_instr->op.code] + 1;
+    if (curr_size + nsize > TRACE_MAX_TRACE_LENGTH) {
+        return 1;
+    }
+    for (int i = 0; i < nsize; i++) {
+        tstate->interp->jit_tracer_code_buffer[curr_size + i] = *(this_instr + i);
+    }
+    return 0;
+}
 
 /* _PyEval_EvalFrameDefault is too large to optimize for speed with PGO on MSVC.
  */
@@ -1102,9 +1117,9 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int
         stack_pointer = _PyFrame_GetStackPointer(frame);
 #if _Py_TAIL_CALL_INTERP
 #   if Py_STATS
-        return _TAIL_CALL_error(frame, stack_pointer, tstate, next_instr, instruction_funcptr_table, 0, lastopcode);
+        return _TAIL_CALL_error(frame, stack_pointer, tstate, next_instr, instruction_funcptr_handler_table, 0, lastopcode);
 #   else
-        return _TAIL_CALL_error(frame, stack_pointer, tstate, next_instr, instruction_funcptr_table, 0);
+        return _TAIL_CALL_error(frame, stack_pointer, tstate, next_instr, instruction_funcptr_handler_table, 0);
 #   endif
 #else
         goto error;
@@ -1113,9 +1128,9 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int
 
 #if _Py_TAIL_CALL_INTERP
 #   if Py_STATS
-        return _TAIL_CALL_start_frame(frame, NULL, tstate, NULL, instruction_funcptr_table, 0, lastopcode);
+        return _TAIL_CALL_start_frame(frame, NULL, tstate, NULL, instruction_funcptr_handler_table, 0, lastopcode);
 #   else
-        return _TAIL_CALL_start_frame(frame, NULL, tstate, NULL, instruction_funcptr_table, 0);
+        return _TAIL_CALL_start_frame(frame, NULL, tstate, NULL, instruction_funcptr_handler_table, 0);
 #   endif
 #else
     goto start_frame;

Python/ceval_macros.h

@@ -78,13 +78,23 @@
 #   define TAIL_CALL_ARGS frame, stack_pointer, tstate, next_instr, instruction_funcptr_table, oparg
 #endif
 
+#  define TRACING_DISPATCH_GOTO() do { \
+    if (add_to_code_trace(tstate, this_instr)) { \
+        LEAVE_TRACING(); \
+    } \
+        DISPATCH_GOTO(); \
+    } while (0);
+
 #if _Py_TAIL_CALL_INTERP
     // Note: [[clang::musttail]] works for GCC 15, but not __attribute__((musttail)) at the moment.
 #   define Py_MUSTTAIL [[clang::musttail]]
 #   define Py_PRESERVE_NONE_CC __attribute__((preserve_none))
     Py_PRESERVE_NONE_CC typedef PyObject* (*py_tail_call_funcptr)(TAIL_CALL_PARAMS);
 
 #   define TARGET(op) Py_PRESERVE_NONE_CC PyObject *_TAIL_CALL_##op(TAIL_CALL_PARAMS)
+#   define TRACING_TARGET(op) Py_PRESERVE_NONE_CC PyObject *_TAIL_CALL_TRACING_##op(TAIL_CALL_PARAMS)
+#   define LEAVE_TRACING() instruction_funcptr_table = instruction_funcptr_handler_table;
+#   define ENTER_TRACING() instruction_funcptr_table = instruction_funcptr_tracing_table
 #   define DISPATCH_GOTO() \
         do { \
             Py_MUSTTAIL return (((py_tail_call_funcptr *)instruction_funcptr_table)[opcode])(TAIL_CALL_ARGS); \
@@ -107,7 +117,10 @@
 #    define LABEL(name) TARGET(name)
 #elif USE_COMPUTED_GOTOS
 #  define TARGET(op) TARGET_##op:
+#  define TRACING_TARGET(op) TARGET_TRACING_##op:
 #  define DISPATCH_GOTO() goto *opcode_targets[opcode]
+#  define LEAVE_TRACING() opcode_targets = opcode_targets_table;
+#  define ENTER_TRACING() opcode_targets = opcode_tracing_targets_table;
 #  define JUMP_TO_LABEL(name) goto name;
 #  define JUMP_TO_PREDICTED(name) goto PREDICTED_##name;
 #  define LABEL(name) name:
@@ -172,6 +185,14 @@ do { \
         JUMP_TO_LABEL(start_frame);                      \
     } while (0)
 
+#define TRACING_DISPATCH() \
+    { \
+        assert(frame->stackpointer == NULL); \
+        NEXTOPARG(); \
+        PRE_DISPATCH_GOTO(); \
+        TRACING_DISPATCH_GOTO(); \
+    }
+
 /* Tuple access macros */
 
 #ifndef Py_DEBUG