properly restore exponential backoffs

diff --git a/Include/internal/pycore_interp_structs.h b/Include/internal/pycore_interp_structs.h
index 5c9f25eff6c9518e767d4ec04f5b2be2dee97da6..546cc98d166c61da9428de6c7f70bc467d2b26c5 100644 (file)
--- a/Include/internal/pycore_interp_structs.h
+++ b/Include/internal/pycore_interp_structs.h
@@ -775,6 +775,8 @@ typedef struct _PyJitTracerState {
     _Py_CODEUNIT *prev_instr;
     PyCodeObject *prev_instr_code; // Strong
     struct _PyExitData *prev_exit;
+    struct _PyExecutorObject *prev_executor; // Strong
+    _Py_CODEUNIT *jump_backward_instr;
     _PyInterpreterFrame *prev_instr_frame;
     _PyBloomFilter dependencies;
 } _PyJitTracerState;

diff --git a/Include/internal/pycore_optimizer.h b/Include/internal/pycore_optimizer.h
index 5208fa9aab1db7f3c6fc90e8443244213134b4e3..2f51485e93fcb5f8351fc29dce0b1fe89ee4e5a1 100644 (file)
--- a/Include/internal/pycore_optimizer.h
+++ b/Include/internal/pycore_optimizer.h
@@ -365,7 +365,8 @@ int _PyJit_translate_single_bytecode_to_trace(PyThreadState *tstate, _PyInterpre
 int
 _PyJit_TryInitializeTracing(PyThreadState *tstate, _PyInterpreterFrame *frame,
     _Py_CODEUNIT *curr_instr, _Py_CODEUNIT *insert_exec_instr,
-    _Py_CODEUNIT *close_loop_instr, int curr_stackdepth, int chain_depth, _PyExitData *exit, int oparg);
+    _Py_CODEUNIT *close_loop_instr, int curr_stackdepth, int chain_depth, _PyExitData *exit,
+    _PyExecutorObject *prev_exec, int oparg);
 
 void _PyJit_FinalizeTracing(PyThreadState *tstate);
 

diff --git a/Python/bytecodes.c b/Python/bytecodes.c
index 96772a772f242f9f07e9fe5e642b357f30e179f6..beed5e091b6f3036f17b2cfa22b3dd815cd8ae4d 100644 (file)
--- a/Python/bytecodes.c
+++ b/Python/bytecodes.c
@@ -2956,7 +2956,6 @@ dummy_func(
             if (!IS_JIT_TRACING() && backoff_counter_triggers(counter) &&
                 this_instr->op.code == JUMP_BACKWARD_JIT &&
                 next_instr->op.code != ENTER_EXECUTOR) {
-                this_instr[1].counter = restart_backoff_counter(counter);
                 if (tstate->interp->jit_state.code_buffer == NULL) {
                     tstate->interp->jit_state.code_buffer = (_PyUOpInstruction *)_PyObject_VirtualAlloc(UOP_BUFFER_SIZE);
                     if (tstate->interp->jit_state.code_buffer == NULL) {
@@ -2972,7 +2971,7 @@ dummy_func(
                         oparg >>= 8;
                         insert_exec_at--;
                     }
-                    int succ = _PyJit_TryInitializeTracing(tstate, frame, this_instr, insert_exec_at, next_instr, STACK_LEVEL(), 0, NULL, oparg);
+                    int succ = _PyJit_TryInitializeTracing(tstate, frame, this_instr, insert_exec_at, next_instr, STACK_LEVEL(), 0, NULL, NULL, oparg);
                     if (succ) {
                         ENTER_TRACING();
                     }
@@ -5432,8 +5431,7 @@ dummy_func(
                 // Note: it's safe to use target->op.arg here instead of the oparg given by EXTENDED_ARG.
                 // The invariant in the optimizer is the deopt target always points back to the first EXTENDED_ARG.
                 // So setting it to anything else is wrong.
-                int succ = _PyJit_TryInitializeTracing(tstate, frame, target, target, target, STACK_LEVEL(), chain_depth, exit, target->op.arg);
-                exit->temperature = initial_temperature_backoff_counter();
+                int succ = _PyJit_TryInitializeTracing(tstate, frame, target, target, target, STACK_LEVEL(), chain_depth, exit, previous_executor, target->op.arg);
                 if (succ) {
                     GOTO_TIER_ONE_CONTINUE_TRACING(target);
                 }

diff --git a/Python/ceval.c b/Python/ceval.c
index 18fe74bcc82920df6c74d036945797974ce05af0..29a110ce28da6cb657d883f6634b87fd167b4ad3 100644 (file)
--- a/Python/ceval.c
+++ b/Python/ceval.c
@@ -998,6 +998,29 @@ bail_tracing_and_jit(PyThreadState *tstate, _PyInterpreterFrame *frame)
     if (!_PyErr_Occurred(tstate) && !_is_sys_tracing) {
         err = _PyOptimizer_Optimize(frame, tstate);
     }
+    // Deal with backoffs
+    _PyExitData *exit = tstate->interp->jit_state.prev_exit;
+    if (exit == NULL) {
+        // We hold a strong reference to the code object, so the instruction won't be freed.
+        if (err <= 0) {
+            assert(tstate->interp->jit_state.jump_backward_instr->op.code == JUMP_BACKWARD_JIT);
+            _Py_BackoffCounter counter = tstate->interp->jit_state.jump_backward_instr[1].counter;
+            tstate->interp->jit_state.jump_backward_instr[1].counter = restart_backoff_counter(counter);
+        }
+        else {
+            tstate->interp->jit_state.jump_backward_instr[1].counter = initial_jump_backoff_counter();
+        }
+    }
+    else {
+        // Likewise, we hold a strong reference to the executor containing this exit, so the exit is guaranteed
+        // to be valid to access.
+        if (err <= 0) {
+            exit->temperature = restart_backoff_counter(exit->temperature);
+        }
+        else {
+            exit->temperature = initial_temperature_backoff_counter();
+        }
+    }
     _PyJit_FinalizeTracing(tstate);
     return err;
 }

diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h
index 7f915e31b12108e6e0ed9b484c12c5c3293450de..62ddb22a4b0b52b65ad920244bc040faa06f19f3 100644 (file)
--- a/Python/executor_cases.c.h
+++ b/Python/executor_cases.c.h
@@ -7480,8 +7480,7 @@
                 _PyExecutorObject *previous_executor = _PyExecutor_FromExit(exit);
                 assert(tstate->current_executor == (PyObject *)previous_executor);
                 int chain_depth = previous_executor->vm_data.chain_depth + 1;
-                int succ = _PyJit_TryInitializeTracing(tstate, frame, target, target, target, STACK_LEVEL(), chain_depth, exit, target->op.arg);
-                exit->temperature = initial_temperature_backoff_counter();
+                int succ = _PyJit_TryInitializeTracing(tstate, frame, target, target, target, STACK_LEVEL(), chain_depth, exit, previous_executor, target->op.arg);
                 if (succ) {
                     GOTO_TIER_ONE_CONTINUE_TRACING(target);
                 }

diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h
index 7daab9556e74bdd0d9d81c61a3b67f498cb23fd4..c0dbf3091f45df1b39ef3bf07743a7e55ccf7b72 100644 (file)
--- a/Python/generated_cases.c.h
+++ b/Python/generated_cases.c.h
@@ -7682,7 +7682,6 @@
                 if (!IS_JIT_TRACING() && backoff_counter_triggers(counter) &&
                     this_instr->op.code == JUMP_BACKWARD_JIT &&
                     next_instr->op.code != ENTER_EXECUTOR) {
-                    this_instr[1].counter = restart_backoff_counter(counter);
                     if (tstate->interp->jit_state.code_buffer == NULL) {
                         _PyFrame_SetStackPointer(frame, stack_pointer);
                         tstate->interp->jit_state.code_buffer = (_PyUOpInstruction *)_PyObject_VirtualAlloc(UOP_BUFFER_SIZE);
@@ -7698,7 +7697,7 @@
                             oparg >>= 8;
                             insert_exec_at--;
                         }
-                        int succ = _PyJit_TryInitializeTracing(tstate, frame, this_instr, insert_exec_at, next_instr, STACK_LEVEL(), 0, NULL, oparg);
+                        int succ = _PyJit_TryInitializeTracing(tstate, frame, this_instr, insert_exec_at, next_instr, STACK_LEVEL(), 0, NULL, NULL, oparg);
                         if (succ) {
                             ENTER_TRACING();
                         }

diff --git a/Python/optimizer.c b/Python/optimizer.c
index e6955edbc194c31a0bc75cbafe38fe1eae9e3482..166d73d4409ca8545f32e368352731ab2710a8da 100644 (file)
--- a/Python/optimizer.c
+++ b/Python/optimizer.c
@@ -949,7 +949,7 @@ full:
 
 // Returns 0 for do not enter tracing, 1 on enter tracing.
 int
-_PyJit_TryInitializeTracing(PyThreadState *tstate, _PyInterpreterFrame *frame, _Py_CODEUNIT *curr_instr, _Py_CODEUNIT *insert_exec_instr, _Py_CODEUNIT *close_loop_instr, int curr_stackdepth, int chain_depth, _PyExitData *exit, int oparg)
+_PyJit_TryInitializeTracing(PyThreadState *tstate, _PyInterpreterFrame *frame, _Py_CODEUNIT *curr_instr, _Py_CODEUNIT *insert_exec_instr, _Py_CODEUNIT *close_loop_instr, int curr_stackdepth, int chain_depth, _PyExitData *exit, _PyExecutorObject *prev_exec, int oparg)
 {
     // A recursive trace.
     // Don't trace into the inner call because it will stomp on the previous trace, causing endless retraces.
@@ -991,6 +991,9 @@ _PyJit_TryInitializeTracing(PyThreadState *tstate, _PyInterpreterFrame *frame, _
     tstate->interp->jit_state.prev_instr_oparg = oparg;
     tstate->interp->jit_state.prev_instr_stacklevel = curr_stackdepth;
     tstate->interp->jit_state.prev_instr_is_super = false;
+    assert(curr_instr->op.code == JUMP_BACKWARD_JIT || (prev_exec != NULL && exit != NULL));
+    tstate->interp->jit_state.jump_backward_instr = curr_instr;
+    tstate->interp->jit_state.prev_executor = (_PyExecutorObject *)Py_XNewRef(prev_exec);
     _Py_BloomFilter_Init(&tstate->interp->jit_state.dependencies);
     return 1;
 }
@@ -1001,6 +1004,7 @@ _PyJit_FinalizeTracing(PyThreadState *tstate)
     Py_CLEAR(tstate->interp->jit_state.initial_code);
     Py_CLEAR(tstate->interp->jit_state.initial_func);
     Py_CLEAR(tstate->interp->jit_state.prev_instr_code);
+    Py_CLEAR(tstate->interp->jit_state.prev_executor);
     tstate->interp->jit_state.code_curr_size = 2;
     tstate->interp->jit_state.code_max_size = UOP_MAX_TRACE_LENGTH - 1;
 }