Skip to content

Commit 8e0fb21

Browse files
Fidget-SpinnerFidget-Spinner
committed
Fix recursive tracing and dynamic exits
1 parent 299a068 commit 8e0fb21

File tree

5 files changed

+84
-25
lines changed

5 files changed

+84
-25
lines changed

‎Python/bytecodes.c‎

Lines changed: 28 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -5429,8 +5429,7 @@ dummy_func(
54295429
tier2 op(_COLD_EXIT, ( -- )) {
54305430
_PyExitData *exit = tstate->jit_exit;
54315431
assert(exit != NULL);
5432-
bool is_dynamic = exit->is_dynamic;
5433-
_Py_CODEUNIT *target = is_dynamic ? frame->instr_ptr : (_PyFrame_GetBytecode(frame) + exit->target);
5432+
_Py_CODEUNIT *target = (_PyFrame_GetBytecode(frame) + exit->target);
54345433
_Py_BackoffCounter temperature = exit->temperature;
54355434
if (target->op.code == ENTER_EXECUTOR) {
54365435
PyCodeObject *code = _PyFrame_GetCode(frame);
@@ -5451,7 +5450,7 @@ dummy_func(
54515450
exit->temperature = initial_temperature_backoff_counter();
54525451
_PyExecutorObject *previous_executor = _PyExecutor_FromExit(exit);
54535452
assert(tstate->current_executor == (PyObject *)previous_executor);
5454-
int chain_depth = is_dynamic ? 0 : current_executor->vm_data.chain_depth + 1;
5453+
int chain_depth = previous_executor->vm_data.chain_depth + 1;
54555454
_PyJIT_InitializeTracing(tstate, frame, target, STACK_LEVEL(), chain_depth, exit);
54565455
GOTO_TIER_ONE(target, 1);
54575456
}
@@ -5461,9 +5460,13 @@ dummy_func(
54615460
EXIT_IF(frame->instr_ptr != (_Py_CODEUNIT *)ip);
54625461
}
54635462

5463+
// Note: this is different than _COLD_EXIT/_EXIT_TRACE, as it may lead to multiple executors
5464+
// from a single exit!
54645465
tier2 op(_DYNAMIC_EXIT, (exit_p/4 --)) {
54655466
_Py_CODEUNIT *target = frame->instr_ptr;
54665467
_PyExitData *exit = (_PyExitData *)exit_p;
5468+
_Py_BackoffCounter temperature = exit->temperature;
5469+
tstate->jit_exit = exit;
54675470
#if defined(Py_DEBUG) && !defined(_Py_JIT)
54685471
OPT_HIST(trace_uop_execution_counter, trace_run_length_hist);
54695472
if (frame->lltrace >= 2) {
@@ -5475,9 +5478,28 @@ dummy_func(
54755478
_PyOpcode_OpName[target->op.code]);
54765479
}
54775480
#endif
5478-
assert(exit->is_dynamic);
5479-
tstate->jit_exit = exit;
5480-
TIER2_TO_TIER2(exit->executor);
5481+
if (target->op.code == ENTER_EXECUTOR) {
5482+
PyCodeObject *code = _PyFrame_GetCode(frame);
5483+
_PyExecutorObject *executor = code->co_executors->executors[target->op.arg];
5484+
Py_INCREF(executor);
5485+
assert(tstate->jit_exit == exit);
5486+
exit->executor = executor;
5487+
TIER2_TO_TIER2(exit->executor);
5488+
}
5489+
else {
5490+
if (frame->owner >= FRAME_OWNED_BY_INTERPRETER) {
5491+
GOTO_TIER_ONE(target, 0);
5492+
}
5493+
if (!backoff_counter_triggers(temperature)) {
5494+
exit->temperature = advance_backoff_counter(temperature);
5495+
GOTO_TIER_ONE(target, 0);
5496+
}
5497+
exit->temperature = initial_temperature_backoff_counter();
5498+
_PyExecutorObject *previous_executor = _PyExecutor_FromExit(exit);
5499+
assert(tstate->current_executor == (PyObject *)previous_executor);
5500+
_PyJIT_InitializeTracing(tstate, frame, target, STACK_LEVEL(), 0, exit);
5501+
GOTO_TIER_ONE(target, 1);
5502+
}
54815503
}
54825504

54835505
label(pop_2_error) {

‎Python/ceval.c‎

Lines changed: 15 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -1049,6 +1049,12 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int
10491049
return NULL;
10501050
}
10511051

1052+
// We came in already tracing, this means a recursive trace in the form of
1053+
// Python -> C -> Python happened. We want to abandon the outer trace then.
1054+
if (IS_JIT_TRACING()) {
1055+
_PyJIT_FinalizeTracing(tstate);
1056+
tstate->interp->jit_is_tracing = false;
1057+
}
10521058
/* Local "register" variables.
10531059
* These are cached values from the frame and code object. */
10541060
_Py_CODEUNIT *next_instr;
@@ -1194,15 +1200,15 @@ _PyTier2Interpreter(
11941200
for (;;) {
11951201
uopcode = next_uop->opcode;
11961202
#ifdef Py_DEBUG
1197-
if (frame->lltrace >= 4) {
1198-
if (next_uop->opcode != _YIELD_VALUE &&
1199-
next_uop->opcode != _FOR_ITER_GEN_FRAME &&
1200-
next_uop->opcode != _PUSH_FRAME &&
1201-
next_uop->opcode != _PY_FRAME_KW &&
1202-
next_uop->opcode != _SAVE_RETURN_OFFSET &&
1203-
next_uop->opcode != _SAVE_RETURN_OFFSET) {
1204-
dump_stack(frame, stack_pointer);
1205-
}
1203+
if (frame->lltrace >= 2) {
1204+
// if (next_uop->opcode != _YIELD_VALUE &&
1205+
// next_uop->opcode != _FOR_ITER_GEN_FRAME &&
1206+
// next_uop->opcode != _PUSH_FRAME &&
1207+
// next_uop->opcode != _PY_FRAME_KW &&
1208+
// next_uop->opcode != _SAVE_RETURN_OFFSET &&
1209+
// next_uop->opcode != _SAVE_RETURN_OFFSET) {
1210+
// dump_stack(frame, stack_pointer);
1211+
// }
12061212
if (next_uop->opcode == _START_EXECUTOR) {
12071213
printf("%4d uop: ", 0);
12081214
}

‎Python/ceval_macros.h‎

Lines changed: 8 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -137,12 +137,16 @@
137137

138138
#if _Py_TAIL_CALL_INTERP || USE_COMPUTED_GOTOS
139139
# define IS_JIT_TRACING() (tstate->interp->jit_is_tracing)
140+
# define SET_TRACING() \
141+
DISPATCH_TABLE_VAR = TRACING_DISPATCH_TABLE;
140142
# define ENTER_TRACING() \
141-
DISPATCH_TABLE_VAR = TRACING_DISPATCH_TABLE; \
143+
SET_TRACING(); \
142144
tstate->interp->jit_is_tracing = true;
145+
# define UNSET_TRACING() \
146+
DISPATCH_TABLE_VAR = DISPATCH_TABLE;
143147
# define LEAVE_TRACING() \
144148
assert(IS_JIT_TRACING()); \
145-
DISPATCH_TABLE_VAR = DISPATCH_TABLE; \
149+
UNSET_TRACING(); \
146150
tstate->interp->jit_is_tracing = false;
147151
// This handles recursive tracing over C calls.
148152
# define RELOAD_TRACING() \
@@ -157,7 +161,7 @@
157161
JUMP_TO_LABEL(error); \
158162
}
159163
# define RECORD_TRACE_NO_DISPATCH() do { \
160-
if (tstate->interp->jit_is_tracing && add_to_code_trace(tstate, frame, old_code, old_func, this_instr, next_instr, opcode, oparg, _jump_taken)) { \
164+
if (DISPATCH_TABLE_VAR == TRACING_DISPATCH_TABLE && add_to_code_trace(tstate, frame, old_code, old_func, this_instr, next_instr, opcode, oparg, _jump_taken)) { \
161165
BAIL_TRACING_NO_DISPATCH(); \
162166
} \
163167
} while (0);
@@ -218,6 +222,7 @@ do { \
218222
} while (0)
219223

220224
#define TRACING_DISPATCH_INLINED(NEW_FRAME) \
225+
RELOAD_TRACING(); \
221226
RECORD_TRACE_NO_DISPATCH(); \
222227
DISPATCH_INLINED(NEW_FRAME);
223228

‎Python/executor_cases.c.h‎

Lines changed: 26 additions & 6 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎Python/optimizer.c‎

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -131,6 +131,11 @@ _PyOptimizer_Optimize(
131131
bool progress_needed = (chain_depth % MAX_CHAIN_DEPTH) == 0;
132132
PyCodeObject *code = (PyCodeObject *)tstate->interp->jit_tracer_initial_code;
133133
_Py_CODEUNIT *start = tstate->interp->jit_tracer_initial_instr;
134+
// A recursive trace might've cleared the values. In that case, bail.
135+
if (code == NULL) {
136+
interp->compiling = false;
137+
return 0;
138+
}
134139
if (progress_needed && !has_space_for_executor(code, start)) {
135140
interp->compiling = false;
136141
return 0;
@@ -868,6 +873,7 @@ _PyJIT_FinalizeTracing(PyThreadState *tstate)
868873
Py_CLEAR(tstate->interp->jit_tracer_initial_code);
869874
Py_CLEAR(tstate->interp->jit_tracer_initial_func);
870875
tstate->interp->jit_tracer_code_curr_size = 2;
876+
tstate->interp->jit_tracer_code_max_size = UOP_MAX_TRACE_LENGTH - 1;
871877
}
872878

873879

@@ -1197,7 +1203,7 @@ uop_optimize(
11971203
int curr_stackentries = tstate->interp->jit_tracer_initial_stack_depth;
11981204
int length = interp->jit_tracer_code_curr_size;
11991205
// Trace too short, don't bother.
1200-
if (length <= 4) {
1206+
if (length <= 5) {
12011207
return 0;
12021208
}
12031209
assert(length > 0);

0 commit comments

Comments
 (0)