Specialize on deopt when tracing

diff --git a/Include/internal/pycore_interp_structs.h b/Include/internal/pycore_interp_structs.h
index fdbba611c2dec2ac3e1e1867b28574d83b2127c2..142bcbece6220b362b87ec394720cd4e170ca35f 100644 (file)
--- a/Include/internal/pycore_interp_structs.h
+++ b/Include/internal/pycore_interp_structs.h
@@ -765,6 +765,7 @@ typedef struct _PyJitState {
     _PyUOpInstruction *jit_tracer_code_buffer;
     _Py_CODEUNIT *jit_tracer_insert_exec_instr;
     _Py_CODEUNIT *jit_tracer_close_loop_instr;
+    _Py_CODEUNIT *last_specialized_instr;
     int jit_tracer_initial_stack_depth;
     int jit_tracer_initial_chain_depth;
     PyCodeObject *jit_tracer_initial_code; // Strong

diff --git a/Python/ceval_macros.h b/Python/ceval_macros.h
index 18ccc3dc7bf88cbd91d920ac16814b37de10fe27..256175b476dc559a2f80670f100e34212931b236 100644 (file)
--- a/Python/ceval_macros.h
+++ b/Python/ceval_macros.h
@@ -137,6 +137,9 @@
 
 #if _Py_TAIL_CALL_INTERP || USE_COMPUTED_GOTOS
 #  define IS_JIT_TRACING() (DISPATCH_TABLE_VAR == TRACING_DISPATCH_TABLE)
+// tstate->interp->jit_state.last_specialized_instr != this_instr is required to not get stuck in infinite
+// specialization loops due to specialization failure.
+#  define IS_JIT_TRACING_MAKING_PROGRESS() (IS_JIT_TRACING() && tstate->interp->jit_state.last_specialized_instr != this_instr)
 #  define ENTER_TRACING() \
     DISPATCH_TABLE_VAR = TRACING_DISPATCH_TABLE;
 #  define LEAVE_TRACING() \
@@ -214,6 +217,14 @@ do { \
         DISPATCH_GOTO(); \
     }
 
+#define TRACING_SPECIALIZE_DISPATCH_SAME_OPARG() \
+{ \
+    tstate->interp->jit_state.last_specialized_instr = this_instr; \
+    opcode = next_instr->op.code; \
+    PRE_DISPATCH_GOTO(); \
+    DISPATCH_GOTO(); \
+}
+
 #define DISPATCH_INLINED(NEW_FRAME)                     \
     do {                                                \
         assert(tstate->interp->eval_frame == NULL);     \
@@ -225,11 +236,13 @@ do { \
     } while (0)
 
 #define TRACING_DISPATCH_INLINED(NEW_FRAME) \
+    tstate->interp->jit_state.last_specialized_instr = this_instr; \
     RECORD_TRACE_NO_DISPATCH(); \
     DISPATCH_INLINED(NEW_FRAME);
 
 #define TRACING_DISPATCH() \
     { \
+        tstate->interp->jit_state.last_specialized_instr = this_instr; \
         assert(frame->stackpointer == NULL); \
         RECORD_TRACE_NO_DISPATCH(); \
         NEXTOPARG(); \
@@ -337,9 +350,14 @@ GETITEM(PyObject *v, Py_ssize_t i) {
 /* This takes a uint16_t instead of a _Py_BackoffCounter,
  * because it is used directly on the cache entry in generated code,
  * which is always an integral type. */
+#if _Py_TIER2
+// Force re-specialization when tracing a side exit to get good side exits.
+#define ADAPTIVE_COUNTER_TRIGGERS(COUNTER) \
+    backoff_counter_triggers(forge_backoff_counter((COUNTER))) || IS_JIT_TRACING_MAKING_PROGRESS()
+#else
 #define ADAPTIVE_COUNTER_TRIGGERS(COUNTER) \
     backoff_counter_triggers(forge_backoff_counter((COUNTER)))
-
+#endif
 #define ADVANCE_ADAPTIVE_COUNTER(COUNTER) \
     do { \
         (COUNTER) = advance_backoff_counter((COUNTER)); \

diff --git a/Python/generated_tracer_cases.c.h b/Python/generated_tracer_cases.c.h
index cdd7246fbb4a0628070a5b948075db4da460e07e..9d739dc00db983b6ccc761936025e44ffaa3dd49 100644 (file)
--- a/Python/generated_tracer_cases.c.h
+++ b/Python/generated_tracer_cases.c.h
@@ -43,7 +43,7 @@
                     _PyFrame_SetStackPointer(frame, stack_pointer);
                     _Py_Specialize_BinaryOp(lhs, rhs, next_instr, oparg, LOCALS_ARRAY);
                     stack_pointer = _PyFrame_GetStackPointer(frame);
-                    DISPATCH_SAME_OPARG();
+                    TRACING_SPECIALIZE_DISPATCH_SAME_OPARG()
                 }
                 OPCODE_DEFERRED_INC(BINARY_OP);
                 ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter);
@@ -1814,7 +1814,7 @@
                     _PyFrame_SetStackPointer(frame, stack_pointer);
                     _Py_Specialize_Call(callable, next_instr, oparg + !PyStackRef_IsNull(self_or_null));
                     stack_pointer = _PyFrame_GetStackPointer(frame);
-                    DISPATCH_SAME_OPARG();
+                    TRACING_SPECIALIZE_DISPATCH_SAME_OPARG()
                 }
                 OPCODE_DEFERRED_INC(CALL);
                 ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter);
@@ -3228,7 +3228,7 @@
                     _PyFrame_SetStackPointer(frame, stack_pointer);
                     _Py_Specialize_CallKw(callable, next_instr, oparg + !PyStackRef_IsNull(self_or_null));
                     stack_pointer = _PyFrame_GetStackPointer(frame);
-                    DISPATCH_SAME_OPARG();
+                    TRACING_SPECIALIZE_DISPATCH_SAME_OPARG()
                 }
                 OPCODE_DEFERRED_INC(CALL_KW);
                 ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter);
@@ -5265,7 +5265,7 @@
                     _PyFrame_SetStackPointer(frame, stack_pointer);
                     _Py_Specialize_CompareOp(left, right, next_instr, oparg);
                     stack_pointer = _PyFrame_GetStackPointer(frame);
-                    DISPATCH_SAME_OPARG();
+                    TRACING_SPECIALIZE_DISPATCH_SAME_OPARG()
                 }
                 OPCODE_DEFERRED_INC(COMPARE_OP);
                 ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter);
@@ -5550,7 +5550,7 @@
                     _PyFrame_SetStackPointer(frame, stack_pointer);
                     _Py_Specialize_ContainsOp(right, next_instr);
                     stack_pointer = _PyFrame_GetStackPointer(frame);
-                    DISPATCH_SAME_OPARG();
+                    TRACING_SPECIALIZE_DISPATCH_SAME_OPARG()
                 }
                 OPCODE_DEFERRED_INC(CONTAINS_OP);
                 ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter);
@@ -6531,7 +6531,7 @@
                     _PyFrame_SetStackPointer(frame, stack_pointer);
                     _Py_Specialize_ForIter(iter, null_or_index, next_instr, oparg);
                     stack_pointer = _PyFrame_GetStackPointer(frame);
-                    DISPATCH_SAME_OPARG();
+                    TRACING_SPECIALIZE_DISPATCH_SAME_OPARG()
                 }
                 OPCODE_DEFERRED_INC(FOR_ITER);
                 ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter);
@@ -8882,7 +8882,7 @@
                     uint8_t desired = tstate->interp->jit ? JUMP_BACKWARD_JIT : JUMP_BACKWARD_NO_JIT;
                     FT_ATOMIC_STORE_UINT8_RELAXED(this_instr->op.code, desired);
                     next_instr = this_instr;
-                    DISPATCH_SAME_OPARG();
+                    DISPATCH_SAME_OPARG()
                 }
                 #endif
             }
@@ -9190,7 +9190,7 @@
                     _PyFrame_SetStackPointer(frame, stack_pointer);
                     _Py_Specialize_LoadAttr(owner, next_instr, name);
                     stack_pointer = _PyFrame_GetStackPointer(frame);
-                    DISPATCH_SAME_OPARG();
+                    TRACING_SPECIALIZE_DISPATCH_SAME_OPARG()
                 }
                 OPCODE_DEFERRED_INC(LOAD_ATTR);
                 ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter);
@@ -10779,7 +10779,7 @@
                     _PyFrame_SetStackPointer(frame, stack_pointer);
                     _Py_Specialize_LoadGlobal(GLOBALS(), BUILTINS(), next_instr, name);
                     stack_pointer = _PyFrame_GetStackPointer(frame);
-                    DISPATCH_SAME_OPARG();
+                    TRACING_SPECIALIZE_DISPATCH_SAME_OPARG()
                 }
                 OPCODE_DEFERRED_INC(LOAD_GLOBAL);
                 ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter);
@@ -11177,7 +11177,7 @@
                     _PyFrame_SetStackPointer(frame, stack_pointer);
                     _Py_Specialize_LoadSuperAttr(global_super_st, class_st, next_instr, load_method);
                     stack_pointer = _PyFrame_GetStackPointer(frame);
-                    DISPATCH_SAME_OPARG();
+                    TRACING_SPECIALIZE_DISPATCH_SAME_OPARG()
                 }
                 OPCODE_DEFERRED_INC(LOAD_SUPER_ATTR);
                 ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter);
@@ -12486,7 +12486,7 @@
                     _PyFrame_SetStackPointer(frame, stack_pointer);
                     _Py_Specialize_Send(receiver, next_instr);
                     stack_pointer = _PyFrame_GetStackPointer(frame);
-                    DISPATCH_SAME_OPARG();
+                    TRACING_SPECIALIZE_DISPATCH_SAME_OPARG()
                 }
                 OPCODE_DEFERRED_INC(SEND);
                 ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter);
@@ -12854,7 +12854,7 @@
                     _PyFrame_SetStackPointer(frame, stack_pointer);
                     _Py_Specialize_StoreAttr(owner, next_instr, name);
                     stack_pointer = _PyFrame_GetStackPointer(frame);
-                    DISPATCH_SAME_OPARG();
+                    TRACING_SPECIALIZE_DISPATCH_SAME_OPARG()
                 }
                 OPCODE_DEFERRED_INC(STORE_ATTR);
                 ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter);
@@ -13475,7 +13475,7 @@
                     _PyFrame_SetStackPointer(frame, stack_pointer);
                     _Py_Specialize_StoreSubscr(container, sub, next_instr);
                     stack_pointer = _PyFrame_GetStackPointer(frame);
-                    DISPATCH_SAME_OPARG();
+                    TRACING_SPECIALIZE_DISPATCH_SAME_OPARG()
                 }
                 OPCODE_DEFERRED_INC(STORE_SUBSCR);
                 ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter);
@@ -13726,7 +13726,7 @@
                     _PyFrame_SetStackPointer(frame, stack_pointer);
                     _Py_Specialize_ToBool(value, next_instr);
                     stack_pointer = _PyFrame_GetStackPointer(frame);
-                    DISPATCH_SAME_OPARG();
+                    TRACING_SPECIALIZE_DISPATCH_SAME_OPARG()
                 }
                 OPCODE_DEFERRED_INC(TO_BOOL);
                 ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter);
@@ -14224,7 +14224,7 @@
                     _PyFrame_SetStackPointer(frame, stack_pointer);
                     _Py_Specialize_UnpackSequence(seq, next_instr, oparg);
                     stack_pointer = _PyFrame_GetStackPointer(frame);
-                    DISPATCH_SAME_OPARG();
+                    TRACING_SPECIALIZE_DISPATCH_SAME_OPARG()
                 }
                 OPCODE_DEFERRED_INC(UNPACK_SEQUENCE);
                 ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter);

diff --git a/Python/optimizer.c b/Python/optimizer.c
index 2fcf6b763691acb14618a014221e3dec90629dd7..ca9967e4272d429fa46c38b27f5539717032f1f2 100644 (file)
--- a/Python/optimizer.c
+++ b/Python/optimizer.c
@@ -904,9 +904,10 @@ _PyJIT_InitializeTracing(PyThreadState *tstate, _PyInterpreterFrame *frame, _Py_
     tstate->interp->jit_state.jit_tracer_previous_exit = exit;
     _Py_BloomFilter_Init(&tstate->interp->jit_state.jit_tracer_dependencies);
     tstate->interp->jit_state.jit_tracer_initial_stack_depth = curr_stackdepth;
-    tstate->interp->jit_state.jit_tracer_initial_chain_depth = chain_depth;
+    tstate->interp->jit_state.jit_tracer_initial_chain_depth = chain_depth % MAX_CHAIN_DEPTH;
     tstate->interp->jit_state.jit_tracer_current_frame = frame;
     tstate->interp->jit_state.jit_tracer_dependencies_still_valid = true;
+    tstate->interp->jit_state.last_specialized_instr = NULL;
 }
 
 void

diff --git a/Tools/cases_generator/tracer_generator.py b/Tools/cases_generator/tracer_generator.py
index 30a34a91d65b2e9094d74d19d9315cec5e511623..7039079dd895f30e4a7e112aa3857ba374534709 100644 (file)
--- a/Tools/cases_generator/tracer_generator.py
+++ b/Tools/cases_generator/tracer_generator.py
@@ -44,6 +44,7 @@ class TracerEmitter(Emitter):
             **self._replacers,
             "DISPATCH": self.dispatch,
             "DISPATCH_INLINED": self.dispatch_inlined,
+            "DISPATCH_SAME_OPARG": self.dispatch_same_oparg,
         }
 
     def dispatch(
@@ -75,6 +76,26 @@ class TracerEmitter(Emitter):
         self.out.start_line()
         self.emit("TRACING_DISPATCH_INLINED")
         return False
+
+    def dispatch_same_oparg(
+        self,
+        tkn: Token,
+        tkn_iter: TokenIterator,
+        uop: CodeSection,
+        storage: Storage,
+        inst: Instruction | None,
+    ) -> bool:
+        if storage.spilled:
+            raise analysis_error("stack_pointer needs reloading before dispatch", tkn)
+        storage.stack.flush(self.out)
+        self.out.start_line()
+        if "specializing" in uop.annotations:
+            self.emit("TRACING_SPECIALIZE_DISPATCH_SAME_OPARG")
+        else:
+            self.emit(tkn)
+        emit_to(self.out, tkn_iter, "SEMI")
+        return False
+
     def record_jump_taken(
         self,
         tkn: Token,