gh-143604: Hold strong reference to executor during JIT tracing (GH-143646)

Co-authored-by: Ken Jin <kenjin4096@gmail.com>

diff --git a/Include/internal/pycore_optimizer.h b/Include/internal/pycore_optimizer.h
index ced7e0d8af26a9fd4b8b7051accc326f1f8c9ccc..a2d9d2d4dfc86f3d75a6c8ab210f45994058c306 100644 (file)
--- a/Include/internal/pycore_optimizer.h
+++ b/Include/internal/pycore_optimizer.h
@@ -222,7 +222,7 @@ extern void _PyExecutor_Free(_PyExecutorObject *self);
 
 PyAPI_FUNC(int) _PyDumpExecutors(FILE *out);
 #ifdef _Py_TIER2
-extern void _Py_ClearExecutorDeletionList(PyInterpreterState *interp);
+PyAPI_FUNC(void) _Py_ClearExecutorDeletionList(PyInterpreterState *interp);
 #endif
 
 int _PyJit_translate_single_bytecode_to_trace(PyThreadState *tstate, _PyInterpreterFrame *frame, _Py_CODEUNIT *next_instr, int stop_tracing_opcode);
@@ -231,7 +231,7 @@ PyAPI_FUNC(int)
 _PyJit_TryInitializeTracing(PyThreadState *tstate, _PyInterpreterFrame *frame,
     _Py_CODEUNIT *curr_instr, _Py_CODEUNIT *start_instr,
     _Py_CODEUNIT *close_loop_instr, int curr_stackdepth, int chain_depth, _PyExitData *exit,
-    int oparg);
+    int oparg, _PyExecutorObject *current_executor);
 
 void _PyJit_FinalizeTracing(PyThreadState *tstate);
 void _PyJit_TracerFree(_PyThreadStateImpl *_tstate);

diff --git a/Include/internal/pycore_tstate.h b/Include/internal/pycore_tstate.h
index ff9327ff57833a9b2b0919772b784a3b1b2345ee..518fd94a31ae5ebaf15653e02e4a0f6d38bdd652 100644 (file)
--- a/Include/internal/pycore_tstate.h
+++ b/Include/internal/pycore_tstate.h
@@ -31,6 +31,7 @@ typedef struct _PyJitTracerInitialState {
     struct _PyExitData *exit;
     PyCodeObject *code; // Strong
     PyFunctionObject *func; // Strong
+    struct _PyExecutorObject *executor; // Strong
     _Py_CODEUNIT *start_instr;
     _Py_CODEUNIT *close_loop_instr;
     _Py_CODEUNIT *jump_backward_instr;

diff --git a/Lib/test/test_capi/test_opt.py b/Lib/test/test_capi/test_opt.py
index 834a3d4b0a4408a70c11898c77921e6fba66ee4f..7dcf8761d9f837809f99414287cc540c475e58ff 100644 (file)
--- a/Lib/test/test_capi/test_opt.py
+++ b/Lib/test/test_capi/test_opt.py
@@ -139,6 +139,19 @@ class TestExecutorInvalidation(unittest.TestCase):
         exe = get_first_executor(f)
         self.assertIsNone(exe)
 
+    def test_prev_executor_freed_while_tracing(self):
+        def f(start, end, way):
+            for x in range(start, end):
+                # For the first trace, create a bad branch on purpose to trace into.
+                # A side exit will form from here on the second trace.
+                y = way + way
+                if x >= TIER2_THRESHOLD:
+                    # Invalidate the first trace while tracing the second.
+                    _testinternalcapi.invalidate_executors(f.__code__)
+                    _testinternalcapi.clear_executor_deletion_list()
+        f(0, TIER2_THRESHOLD, 1)
+        f(1, TIER2_THRESHOLD + 1, 1.0)
+
 
 @requires_specialization
 @unittest.skipIf(Py_GIL_DISABLED, "optimizer not yet supported in free-threaded builds")

diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-01-10-17-13-04.gh-issue-143604.BygbVT.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-01-10-17-13-04.gh-issue-143604.BygbVT.rst
new file mode 100644 (file)
index 0000000..391186b
--- /dev/null
+++ b/Misc/NEWS.d/next/Core_and_Builtins/2026-01-10-17-13-04.gh-issue-143604.BygbVT.rst
@@ -0,0 +1,2 @@
+Fix a reference counting issue in the JIT tracer where the current executor
+could be prematurely freed during tracing.

diff --git a/Modules/_testinternalcapi.c b/Modules/_testinternalcapi.c
index ea09f2d5ef836f1862537ba46f02a5531c5a952a..ad665a2cd78685954169c13b9e3a63d9872ca7dd 100644 (file)
--- a/Modules/_testinternalcapi.c
+++ b/Modules/_testinternalcapi.c
@@ -1245,6 +1245,14 @@ invalidate_executors(PyObject *self, PyObject *obj)
     Py_RETURN_NONE;
 }
 
+static PyObject *
+clear_executor_deletion_list(PyObject *self, PyObject *Py_UNUSED(ignored))
+{
+    PyInterpreterState *interp = PyInterpreterState_Get();
+    _Py_ClearExecutorDeletionList(interp);
+    Py_RETURN_NONE;
+}
+
 static PyObject *
 get_exit_executor(PyObject *self, PyObject *arg)
 {
@@ -2562,6 +2570,7 @@ static PyMethodDef module_functions[] = {
 #ifdef _Py_TIER2
     {"add_executor_dependency", add_executor_dependency, METH_VARARGS, NULL},
     {"invalidate_executors", invalidate_executors, METH_O, NULL},
+    {"clear_executor_deletion_list", clear_executor_deletion_list, METH_NOARGS, NULL},
     {"get_exit_executor", get_exit_executor, METH_O, NULL},
 #endif
     {"pending_threadfunc", _PyCFunction_CAST(pending_threadfunc),

diff --git a/Python/bytecodes.c b/Python/bytecodes.c
index 77dc82aa853ca40e5ca709b628d3ca669ba618d7..25bfbe555f8596c6ef40eb642ade8077bfa6ce53 100644 (file)
--- a/Python/bytecodes.c
+++ b/Python/bytecodes.c
@@ -2990,7 +2990,7 @@ dummy_func(
                     oparg >>= 8;
                     insert_exec_at--;
                 }
-                int succ = _PyJit_TryInitializeTracing(tstate, frame, this_instr, insert_exec_at, next_instr, STACK_LEVEL(), 0, NULL, oparg);
+                int succ = _PyJit_TryInitializeTracing(tstate, frame, this_instr, insert_exec_at, next_instr, STACK_LEVEL(), 0, NULL, oparg, NULL);
                 if (succ) {
                     ENTER_TRACING();
                 }
@@ -5525,7 +5525,7 @@ dummy_func(
                 // Note: it's safe to use target->op.arg here instead of the oparg given by EXTENDED_ARG.
                 // The invariant in the optimizer is the deopt target always points back to the first EXTENDED_ARG.
                 // So setting it to anything else is wrong.
-                int succ = _PyJit_TryInitializeTracing(tstate, frame, target, target, target, STACK_LEVEL(), chain_depth, exit, target->op.arg);
+                int succ = _PyJit_TryInitializeTracing(tstate, frame, target, target, target, STACK_LEVEL(), chain_depth, exit, target->op.arg, previous_executor);
                 exit->temperature = restart_backoff_counter(exit->temperature);
                 if (succ) {
                     GOTO_TIER_ONE_CONTINUE_TRACING(target);

diff --git a/Python/ceval.c b/Python/ceval.c
index dfd014e90b0e17065800df995dfbe3de38c43bf9..e67ff082ef9fac8dd434e886dc9a13b239b504de 100644 (file)
--- a/Python/ceval.c
+++ b/Python/ceval.c
@@ -1475,7 +1475,7 @@ stop_tracing_and_jit(PyThreadState *tstate, _PyInterpreterFrame *frame)
             tracer->initial_state.jump_backward_instr[1].counter = initial_jump_backoff_counter(&_tstate->policy);
         }
     }
-    else {
+    else if (tracer->initial_state.executor->vm_data.valid) {
         // Likewise, we hold a strong reference to the executor containing this exit, so the exit is guaranteed
         // to be valid to access.
         if (err <= 0) {

diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h
index 1053c288bc4313b9e394bad2dd46e50da37dbaed..1a49ffb10d423d47835d3c0fa9f8af0e51666013 100644 (file)
--- a/Python/executor_cases.c.h
+++ b/Python/executor_cases.c.h
@@ -18996,7 +18996,7 @@
                 _PyExecutorObject *previous_executor = _PyExecutor_FromExit(exit);
                 assert(tstate->current_executor == (PyObject *)previous_executor);
                 int chain_depth = previous_executor->vm_data.chain_depth + !exit->is_control_flow;
-                int succ = _PyJit_TryInitializeTracing(tstate, frame, target, target, target, STACK_LEVEL(), chain_depth, exit, target->op.arg);
+                int succ = _PyJit_TryInitializeTracing(tstate, frame, target, target, target, STACK_LEVEL(), chain_depth, exit, target->op.arg, previous_executor);
                 exit->temperature = restart_backoff_counter(exit->temperature);
                 if (succ) {
                     GOTO_TIER_ONE_CONTINUE_TRACING(target);

diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h
index 959b3a37e5b6fafd9e7fbbe467641008ebdb4807..acf977eb8d9a4324d014f7df45fa88c042dd5f60 100644 (file)
--- a/Python/generated_cases.c.h
+++ b/Python/generated_cases.c.h
@@ -7662,7 +7662,7 @@
                         oparg >>= 8;
                         insert_exec_at--;
                     }
-                    int succ = _PyJit_TryInitializeTracing(tstate, frame, this_instr, insert_exec_at, next_instr, STACK_LEVEL(), 0, NULL, oparg);
+                    int succ = _PyJit_TryInitializeTracing(tstate, frame, this_instr, insert_exec_at, next_instr, STACK_LEVEL(), 0, NULL, oparg, NULL);
                     if (succ) {
                         ENTER_TRACING();
                     }

diff --git a/Python/optimizer.c b/Python/optimizer.c
index 3c561a8a7fd0e8de92d4713434316309f09c7f4d..79ac179d0b710ab32d696dcb7a486b7bf346e6fd 100644 (file)
--- a/Python/optimizer.c
+++ b/Python/optimizer.c
@@ -138,6 +138,12 @@ _PyOptimizer_Optimize(
         // return immediately without optimization.
         return 0;
     }
+    _PyExecutorObject *prev_executor = _tstate->jit_tracer_state->initial_state.executor;
+    if (prev_executor != NULL && !prev_executor->vm_data.valid) {
+        // gh-143604: If we are a side exit executor and the original executor is no
+        // longer valid, don't compile to prevent a reference leak.
+        return 0;
+    }
     assert(!interp->compiling);
     assert(_tstate->jit_tracer_state->initial_state.stack_depth >= 0);
 #ifndef Py_GIL_DISABLED
@@ -1015,7 +1021,7 @@ Py_NO_INLINE int
 _PyJit_TryInitializeTracing(
     PyThreadState *tstate, _PyInterpreterFrame *frame, _Py_CODEUNIT *curr_instr,
     _Py_CODEUNIT *start_instr, _Py_CODEUNIT *close_loop_instr, int curr_stackdepth, int chain_depth,
-    _PyExitData *exit, int oparg)
+    _PyExitData *exit, int oparg, _PyExecutorObject *current_executor)
 {
     _PyThreadStateImpl *_tstate = (_PyThreadStateImpl *)tstate;
     if (_tstate->jit_tracer_state == NULL) {
@@ -1062,6 +1068,7 @@ _PyJit_TryInitializeTracing(
     tracer->initial_state.close_loop_instr = close_loop_instr;
     tracer->initial_state.code = (PyCodeObject *)Py_NewRef(code);
     tracer->initial_state.func = (PyFunctionObject *)Py_NewRef(func);
+    tracer->initial_state.executor = (_PyExecutorObject *)Py_XNewRef(current_executor);
     tracer->initial_state.exit = exit;
     tracer->initial_state.stack_depth = curr_stackdepth;
     tracer->initial_state.chain_depth = chain_depth;
@@ -1089,6 +1096,7 @@ _PyJit_FinalizeTracing(PyThreadState *tstate)
     _PyJitTracerState *tracer = _tstate->jit_tracer_state;
     Py_CLEAR(tracer->initial_state.code);
     Py_CLEAR(tracer->initial_state.func);
+    Py_CLEAR(tracer->initial_state.executor);
     Py_CLEAR(tracer->prev_state.instr_code);
     tracer->prev_state.code_curr_size = CODE_SIZE_EMPTY;
     tracer->prev_state.code_max_size = UOP_MAX_TRACE_LENGTH/2 - 1;