Fix recursive tracing and dynamic exits

diff --git a/Python/bytecodes.c b/Python/bytecodes.c
index 0f9ec19d38f9fed3d2020e9d80c094c2f5157a88..15dbab0bef70f9e649920bfbfb3355881231fea2 100644 (file)
--- a/Python/bytecodes.c
+++ b/Python/bytecodes.c
@@ -5429,8 +5429,7 @@ dummy_func(
         tier2 op(_COLD_EXIT, ( -- )) {
             _PyExitData *exit = tstate->jit_exit;
             assert(exit != NULL);
-            bool is_dynamic = exit->is_dynamic;
-            _Py_CODEUNIT *target = is_dynamic ? frame->instr_ptr : (_PyFrame_GetBytecode(frame) + exit->target);
+            _Py_CODEUNIT *target = (_PyFrame_GetBytecode(frame) + exit->target);
             _Py_BackoffCounter temperature = exit->temperature;
             if (target->op.code == ENTER_EXECUTOR) {
                 PyCodeObject *code = _PyFrame_GetCode(frame);
@@ -5451,7 +5450,7 @@ dummy_func(
                 exit->temperature = initial_temperature_backoff_counter();
                 _PyExecutorObject *previous_executor = _PyExecutor_FromExit(exit);
                 assert(tstate->current_executor == (PyObject *)previous_executor);
-                int chain_depth = is_dynamic ? 0 : current_executor->vm_data.chain_depth + 1;
+                int chain_depth = previous_executor->vm_data.chain_depth + 1;
                 _PyJIT_InitializeTracing(tstate, frame, target, STACK_LEVEL(), chain_depth, exit);
                 GOTO_TIER_ONE(target, 1);
             }
@@ -5461,9 +5460,13 @@ dummy_func(
             EXIT_IF(frame->instr_ptr != (_Py_CODEUNIT *)ip);
         }
 
+        // Note: this is different than _COLD_EXIT/_EXIT_TRACE, as it may lead to multiple executors
+        // from a single exit!
         tier2 op(_DYNAMIC_EXIT, (exit_p/4 --)) {
             _Py_CODEUNIT *target = frame->instr_ptr;
             _PyExitData *exit = (_PyExitData *)exit_p;
+            _Py_BackoffCounter temperature = exit->temperature;
+            tstate->jit_exit = exit;
 #if defined(Py_DEBUG) && !defined(_Py_JIT)
             OPT_HIST(trace_uop_execution_counter, trace_run_length_hist);
             if (frame->lltrace >= 2) {
@@ -5475,9 +5478,28 @@ dummy_func(
                     _PyOpcode_OpName[target->op.code]);
             }
 #endif
-            assert(exit->is_dynamic);
-            tstate->jit_exit = exit;
-            TIER2_TO_TIER2(exit->executor);
+            if (target->op.code == ENTER_EXECUTOR) {
+                PyCodeObject *code = _PyFrame_GetCode(frame);
+                _PyExecutorObject *executor = code->co_executors->executors[target->op.arg];
+                Py_INCREF(executor);
+                assert(tstate->jit_exit == exit);
+                exit->executor = executor;
+                TIER2_TO_TIER2(exit->executor);
+            }
+            else {
+                if (frame->owner >= FRAME_OWNED_BY_INTERPRETER) {
+                    GOTO_TIER_ONE(target, 0);
+                }
+                if (!backoff_counter_triggers(temperature)) {
+                    exit->temperature = advance_backoff_counter(temperature);
+                    GOTO_TIER_ONE(target, 0);
+                }
+                exit->temperature = initial_temperature_backoff_counter();
+                _PyExecutorObject *previous_executor = _PyExecutor_FromExit(exit);
+                assert(tstate->current_executor == (PyObject *)previous_executor);
+                _PyJIT_InitializeTracing(tstate, frame, target, STACK_LEVEL(), 0, exit);
+                GOTO_TIER_ONE(target, 1);
+            }
         }
 
         label(pop_2_error) {

diff --git a/Python/ceval.c b/Python/ceval.c
index a6219e5c26d6707e94ea6b3094dcbd45ce3c555e..bf1616b04bf50b584a30cc2942d9a640e4573046 100644 (file)
--- a/Python/ceval.c
+++ b/Python/ceval.c
@@ -1049,6 +1049,12 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int
         return NULL;
     }
 
+    // We came in already tracing, this means a recursive trace in the form of
+    // Python -> C -> Python happened. We want to abandon the outer trace then.
+    if (IS_JIT_TRACING()) {
+        _PyJIT_FinalizeTracing(tstate);
+        tstate->interp->jit_is_tracing = false;
+    }
     /* Local "register" variables.
      * These are cached values from the frame and code object.  */
     _Py_CODEUNIT *next_instr;
@@ -1194,15 +1200,15 @@ tier2_dispatch:
     for (;;) {
         uopcode = next_uop->opcode;
 #ifdef Py_DEBUG
-        if (frame->lltrace >= 4) {
-            if (next_uop->opcode != _YIELD_VALUE &&
-            next_uop->opcode != _FOR_ITER_GEN_FRAME &&
-            next_uop->opcode != _PUSH_FRAME &&
-            next_uop->opcode != _PY_FRAME_KW &&
-            next_uop->opcode != _SAVE_RETURN_OFFSET &&
-            next_uop->opcode != _SAVE_RETURN_OFFSET) {
-                dump_stack(frame, stack_pointer);
-            }
+        if (frame->lltrace >= 2) {
+            // if (next_uop->opcode != _YIELD_VALUE &&
+            // next_uop->opcode != _FOR_ITER_GEN_FRAME &&
+            // next_uop->opcode != _PUSH_FRAME &&
+            // next_uop->opcode != _PY_FRAME_KW &&
+            // next_uop->opcode != _SAVE_RETURN_OFFSET &&
+            // next_uop->opcode != _SAVE_RETURN_OFFSET) {
+            //     dump_stack(frame, stack_pointer);
+            // }
             if (next_uop->opcode == _START_EXECUTOR) {
                 printf("%4d uop: ", 0);
             }

diff --git a/Python/ceval_macros.h b/Python/ceval_macros.h
index 7167b8f4f63b232e69f3189602a31b33f09cab51..edacf4418b97fdfaa29b52c4fb18035ae86927de 100644 (file)
--- a/Python/ceval_macros.h
+++ b/Python/ceval_macros.h
@@ -137,12 +137,16 @@
 
 #if _Py_TAIL_CALL_INTERP || USE_COMPUTED_GOTOS
 #  define IS_JIT_TRACING() (tstate->interp->jit_is_tracing)
+#  define SET_TRACING() \
+    DISPATCH_TABLE_VAR = TRACING_DISPATCH_TABLE;
 #  define ENTER_TRACING() \
-    DISPATCH_TABLE_VAR = TRACING_DISPATCH_TABLE; \
+    SET_TRACING(); \
     tstate->interp->jit_is_tracing = true;
+#  define UNSET_TRACING() \
+    DISPATCH_TABLE_VAR = DISPATCH_TABLE;
 #  define LEAVE_TRACING() \
     assert(IS_JIT_TRACING()); \
-    DISPATCH_TABLE_VAR = DISPATCH_TABLE; \
+    UNSET_TRACING(); \
     tstate->interp->jit_is_tracing = false;
 // This handles recursive tracing over C calls.
 #  define RELOAD_TRACING() \
@@ -157,7 +161,7 @@
         JUMP_TO_LABEL(error); \
     }
 #  define RECORD_TRACE_NO_DISPATCH() do { \
-        if (tstate->interp->jit_is_tracing && add_to_code_trace(tstate, frame, old_code, old_func, this_instr, next_instr, opcode, oparg, _jump_taken)) { \
+        if (DISPATCH_TABLE_VAR == TRACING_DISPATCH_TABLE && add_to_code_trace(tstate, frame, old_code, old_func, this_instr, next_instr, opcode, oparg, _jump_taken)) { \
             BAIL_TRACING_NO_DISPATCH(); \
         } \
     } while (0);
@@ -218,6 +222,7 @@ do { \
     } while (0)
 
 #define TRACING_DISPATCH_INLINED(NEW_FRAME) \
+    RELOAD_TRACING(); \
     RECORD_TRACE_NO_DISPATCH(); \
     DISPATCH_INLINED(NEW_FRAME);
 

diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h
index 73eb2692d99933e2ea277afa04898b2a101e493a..86c47b5601048e551c2f77e645bcf39406f9d89a 100644 (file)
--- a/Python/executor_cases.c.h
+++ b/Python/executor_cases.c.h
@@ -7486,8 +7486,7 @@
         case _COLD_EXIT: {
             _PyExitData *exit = tstate->jit_exit;
             assert(exit != NULL);
-            bool is_dynamic = exit->is_dynamic;
-            _Py_CODEUNIT *target = is_dynamic ? frame->instr_ptr : (_PyFrame_GetBytecode(frame) + exit->target);
+            _Py_CODEUNIT *target = (_PyFrame_GetBytecode(frame) + exit->target);
             _Py_BackoffCounter temperature = exit->temperature;
             if (target->op.code == ENTER_EXECUTOR) {
                 PyCodeObject *code = _PyFrame_GetCode(frame);
@@ -7508,7 +7507,7 @@
                 exit->temperature = initial_temperature_backoff_counter();
                 _PyExecutorObject *previous_executor = _PyExecutor_FromExit(exit);
                 assert(tstate->current_executor == (PyObject *)previous_executor);
-                int chain_depth = is_dynamic ? 0 : current_executor->vm_data.chain_depth + 1;
+                int chain_depth = previous_executor->vm_data.chain_depth + 1;
                 _PyJIT_InitializeTracing(tstate, frame, target, STACK_LEVEL(), chain_depth, exit);
                 GOTO_TIER_ONE(target, 1);
             }
@@ -7528,6 +7527,8 @@
             PyObject *exit_p = (PyObject *)CURRENT_OPERAND0();
             _Py_CODEUNIT *target = frame->instr_ptr;
             _PyExitData *exit = (_PyExitData *)exit_p;
+            _Py_BackoffCounter temperature = exit->temperature;
+            tstate->jit_exit = exit;
             #if defined(Py_DEBUG) && !defined(_Py_JIT)
             OPT_HIST(trace_uop_execution_counter, trace_run_length_hist);
             if (frame->lltrace >= 2) {
@@ -7541,9 +7542,28 @@
                 stack_pointer = _PyFrame_GetStackPointer(frame);
             }
             #endif
-            assert(exit->is_dynamic);
-            tstate->jit_exit = exit;
-            TIER2_TO_TIER2(exit->executor);
+            if (target->op.code == ENTER_EXECUTOR) {
+                PyCodeObject *code = _PyFrame_GetCode(frame);
+                _PyExecutorObject *executor = code->co_executors->executors[target->op.arg];
+                Py_INCREF(executor);
+                assert(tstate->jit_exit == exit);
+                exit->executor = executor;
+                TIER2_TO_TIER2(exit->executor);
+            }
+            else {
+                if (frame->owner >= FRAME_OWNED_BY_INTERPRETER) {
+                    GOTO_TIER_ONE(target, 0);
+                }
+                if (!backoff_counter_triggers(temperature)) {
+                    exit->temperature = advance_backoff_counter(temperature);
+                    GOTO_TIER_ONE(target, 0);
+                }
+                exit->temperature = initial_temperature_backoff_counter();
+                _PyExecutorObject *previous_executor = _PyExecutor_FromExit(exit);
+                assert(tstate->current_executor == (PyObject *)previous_executor);
+                _PyJIT_InitializeTracing(tstate, frame, target, STACK_LEVEL(), 0, exit);
+                GOTO_TIER_ONE(target, 1);
+            }
             break;
         }
 

diff --git a/Python/optimizer.c b/Python/optimizer.c
index 9e5ec472caa43a16b54843ec67fb9f5b394bbd79..03cd070c163e4929c181c80ea595f7208649613d 100644 (file)
--- a/Python/optimizer.c
+++ b/Python/optimizer.c
@@ -131,6 +131,11 @@ _PyOptimizer_Optimize(
     bool progress_needed = (chain_depth % MAX_CHAIN_DEPTH) == 0;
     PyCodeObject *code = (PyCodeObject *)tstate->interp->jit_tracer_initial_code;
     _Py_CODEUNIT *start = tstate->interp->jit_tracer_initial_instr;
+    // A recursive trace might've cleared the values. In that case, bail.
+    if (code == NULL) {
+        interp->compiling = false;
+        return 0;
+    }
     if (progress_needed && !has_space_for_executor(code, start)) {
         interp->compiling = false;
         return 0;
@@ -868,6 +873,7 @@ _PyJIT_FinalizeTracing(PyThreadState *tstate)
     Py_CLEAR(tstate->interp->jit_tracer_initial_code);
     Py_CLEAR(tstate->interp->jit_tracer_initial_func);
     tstate->interp->jit_tracer_code_curr_size = 2;
+    tstate->interp->jit_tracer_code_max_size = UOP_MAX_TRACE_LENGTH - 1;
 }
 
 
@@ -1197,7 +1203,7 @@ uop_optimize(
     int curr_stackentries = tstate->interp->jit_tracer_initial_stack_depth;
     int length = interp->jit_tracer_code_curr_size;
     // Trace too short, don't bother.
-    if (length <= 4) {
+    if (length <= 5) {
         return 0;
     }
     assert(length > 0);