From: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Mon, 20 Oct 2025 16:16:25 +0000 (+0100) Subject: Specialize on deopt when tracing X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=6045a677ac736652e43bacb313f2fa2ab56c7cda;p=thirdparty%2FPython%2Fcpython.git Specialize on deopt when tracing --- diff --git a/Include/internal/pycore_interp_structs.h b/Include/internal/pycore_interp_structs.h index fdbba611c2de..142bcbece622 100644 --- a/Include/internal/pycore_interp_structs.h +++ b/Include/internal/pycore_interp_structs.h @@ -765,6 +765,7 @@ typedef struct _PyJitState { _PyUOpInstruction *jit_tracer_code_buffer; _Py_CODEUNIT *jit_tracer_insert_exec_instr; _Py_CODEUNIT *jit_tracer_close_loop_instr; + _Py_CODEUNIT *last_specialized_instr; int jit_tracer_initial_stack_depth; int jit_tracer_initial_chain_depth; PyCodeObject *jit_tracer_initial_code; // Strong diff --git a/Python/ceval_macros.h b/Python/ceval_macros.h index 18ccc3dc7bf8..256175b476dc 100644 --- a/Python/ceval_macros.h +++ b/Python/ceval_macros.h @@ -137,6 +137,9 @@ #if _Py_TAIL_CALL_INTERP || USE_COMPUTED_GOTOS # define IS_JIT_TRACING() (DISPATCH_TABLE_VAR == TRACING_DISPATCH_TABLE) +// tstate->interp->jit_state.last_specialized_instr != this_instr is required to not get stuck in infinite +// specialization loops due to specialization failure. +# define IS_JIT_TRACING_MAKING_PROGRESS() (IS_JIT_TRACING() && tstate->interp->jit_state.last_specialized_instr != this_instr) # define ENTER_TRACING() \ DISPATCH_TABLE_VAR = TRACING_DISPATCH_TABLE; # define LEAVE_TRACING() \ @@ -214,6 +217,14 @@ do { \ DISPATCH_GOTO(); \ } +#define TRACING_SPECIALIZE_DISPATCH_SAME_OPARG() \ +{ \ + tstate->interp->jit_state.last_specialized_instr = this_instr; \ + opcode = next_instr->op.code; \ + PRE_DISPATCH_GOTO(); \ + DISPATCH_GOTO(); \ +} + #define DISPATCH_INLINED(NEW_FRAME) \ do { \ assert(tstate->interp->eval_frame == NULL); \ @@ -225,11 +236,13 @@ do { \ } while (0) #define TRACING_DISPATCH_INLINED(NEW_FRAME) \ + tstate->interp->jit_state.last_specialized_instr = this_instr; \ RECORD_TRACE_NO_DISPATCH(); \ DISPATCH_INLINED(NEW_FRAME); #define TRACING_DISPATCH() \ { \ + tstate->interp->jit_state.last_specialized_instr = this_instr; \ assert(frame->stackpointer == NULL); \ RECORD_TRACE_NO_DISPATCH(); \ NEXTOPARG(); \ @@ -337,9 +350,14 @@ GETITEM(PyObject *v, Py_ssize_t i) { /* This takes a uint16_t instead of a _Py_BackoffCounter, * because it is used directly on the cache entry in generated code, * which is always an integral type. */ +#if _Py_TIER2 +// Force re-specialization when tracing a side exit to get good side exits. +#define ADAPTIVE_COUNTER_TRIGGERS(COUNTER) \ + backoff_counter_triggers(forge_backoff_counter((COUNTER))) || IS_JIT_TRACING_MAKING_PROGRESS() +#else #define ADAPTIVE_COUNTER_TRIGGERS(COUNTER) \ backoff_counter_triggers(forge_backoff_counter((COUNTER))) - +#endif #define ADVANCE_ADAPTIVE_COUNTER(COUNTER) \ do { \ (COUNTER) = advance_backoff_counter((COUNTER)); \ diff --git a/Python/generated_tracer_cases.c.h b/Python/generated_tracer_cases.c.h index cdd7246fbb4a..9d739dc00db9 100644 --- a/Python/generated_tracer_cases.c.h +++ b/Python/generated_tracer_cases.c.h @@ -43,7 +43,7 @@ _PyFrame_SetStackPointer(frame, stack_pointer); _Py_Specialize_BinaryOp(lhs, rhs, next_instr, oparg, LOCALS_ARRAY); stack_pointer = _PyFrame_GetStackPointer(frame); - DISPATCH_SAME_OPARG(); + TRACING_SPECIALIZE_DISPATCH_SAME_OPARG() } OPCODE_DEFERRED_INC(BINARY_OP); ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); @@ -1814,7 +1814,7 @@ _PyFrame_SetStackPointer(frame, stack_pointer); _Py_Specialize_Call(callable, next_instr, oparg + !PyStackRef_IsNull(self_or_null)); stack_pointer = _PyFrame_GetStackPointer(frame); - DISPATCH_SAME_OPARG(); + TRACING_SPECIALIZE_DISPATCH_SAME_OPARG() } OPCODE_DEFERRED_INC(CALL); ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); @@ -3228,7 +3228,7 @@ _PyFrame_SetStackPointer(frame, stack_pointer); _Py_Specialize_CallKw(callable, next_instr, oparg + !PyStackRef_IsNull(self_or_null)); stack_pointer = _PyFrame_GetStackPointer(frame); - DISPATCH_SAME_OPARG(); + TRACING_SPECIALIZE_DISPATCH_SAME_OPARG() } OPCODE_DEFERRED_INC(CALL_KW); ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); @@ -5265,7 +5265,7 @@ _PyFrame_SetStackPointer(frame, stack_pointer); _Py_Specialize_CompareOp(left, right, next_instr, oparg); stack_pointer = _PyFrame_GetStackPointer(frame); - DISPATCH_SAME_OPARG(); + TRACING_SPECIALIZE_DISPATCH_SAME_OPARG() } OPCODE_DEFERRED_INC(COMPARE_OP); ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); @@ -5550,7 +5550,7 @@ _PyFrame_SetStackPointer(frame, stack_pointer); _Py_Specialize_ContainsOp(right, next_instr); stack_pointer = _PyFrame_GetStackPointer(frame); - DISPATCH_SAME_OPARG(); + TRACING_SPECIALIZE_DISPATCH_SAME_OPARG() } OPCODE_DEFERRED_INC(CONTAINS_OP); ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); @@ -6531,7 +6531,7 @@ _PyFrame_SetStackPointer(frame, stack_pointer); _Py_Specialize_ForIter(iter, null_or_index, next_instr, oparg); stack_pointer = _PyFrame_GetStackPointer(frame); - DISPATCH_SAME_OPARG(); + TRACING_SPECIALIZE_DISPATCH_SAME_OPARG() } OPCODE_DEFERRED_INC(FOR_ITER); ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); @@ -8882,7 +8882,7 @@ uint8_t desired = tstate->interp->jit ? JUMP_BACKWARD_JIT : JUMP_BACKWARD_NO_JIT; FT_ATOMIC_STORE_UINT8_RELAXED(this_instr->op.code, desired); next_instr = this_instr; - DISPATCH_SAME_OPARG(); + DISPATCH_SAME_OPARG() } #endif } @@ -9190,7 +9190,7 @@ _PyFrame_SetStackPointer(frame, stack_pointer); _Py_Specialize_LoadAttr(owner, next_instr, name); stack_pointer = _PyFrame_GetStackPointer(frame); - DISPATCH_SAME_OPARG(); + TRACING_SPECIALIZE_DISPATCH_SAME_OPARG() } OPCODE_DEFERRED_INC(LOAD_ATTR); ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); @@ -10779,7 +10779,7 @@ _PyFrame_SetStackPointer(frame, stack_pointer); _Py_Specialize_LoadGlobal(GLOBALS(), BUILTINS(), next_instr, name); stack_pointer = _PyFrame_GetStackPointer(frame); - DISPATCH_SAME_OPARG(); + TRACING_SPECIALIZE_DISPATCH_SAME_OPARG() } OPCODE_DEFERRED_INC(LOAD_GLOBAL); ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); @@ -11177,7 +11177,7 @@ _PyFrame_SetStackPointer(frame, stack_pointer); _Py_Specialize_LoadSuperAttr(global_super_st, class_st, next_instr, load_method); stack_pointer = _PyFrame_GetStackPointer(frame); - DISPATCH_SAME_OPARG(); + TRACING_SPECIALIZE_DISPATCH_SAME_OPARG() } OPCODE_DEFERRED_INC(LOAD_SUPER_ATTR); ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); @@ -12486,7 +12486,7 @@ _PyFrame_SetStackPointer(frame, stack_pointer); _Py_Specialize_Send(receiver, next_instr); stack_pointer = _PyFrame_GetStackPointer(frame); - DISPATCH_SAME_OPARG(); + TRACING_SPECIALIZE_DISPATCH_SAME_OPARG() } OPCODE_DEFERRED_INC(SEND); ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); @@ -12854,7 +12854,7 @@ _PyFrame_SetStackPointer(frame, stack_pointer); _Py_Specialize_StoreAttr(owner, next_instr, name); stack_pointer = _PyFrame_GetStackPointer(frame); - DISPATCH_SAME_OPARG(); + TRACING_SPECIALIZE_DISPATCH_SAME_OPARG() } OPCODE_DEFERRED_INC(STORE_ATTR); ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); @@ -13475,7 +13475,7 @@ _PyFrame_SetStackPointer(frame, stack_pointer); _Py_Specialize_StoreSubscr(container, sub, next_instr); stack_pointer = _PyFrame_GetStackPointer(frame); - DISPATCH_SAME_OPARG(); + TRACING_SPECIALIZE_DISPATCH_SAME_OPARG() } OPCODE_DEFERRED_INC(STORE_SUBSCR); ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); @@ -13726,7 +13726,7 @@ _PyFrame_SetStackPointer(frame, stack_pointer); _Py_Specialize_ToBool(value, next_instr); stack_pointer = _PyFrame_GetStackPointer(frame); - DISPATCH_SAME_OPARG(); + TRACING_SPECIALIZE_DISPATCH_SAME_OPARG() } OPCODE_DEFERRED_INC(TO_BOOL); ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); @@ -14224,7 +14224,7 @@ _PyFrame_SetStackPointer(frame, stack_pointer); _Py_Specialize_UnpackSequence(seq, next_instr, oparg); stack_pointer = _PyFrame_GetStackPointer(frame); - DISPATCH_SAME_OPARG(); + TRACING_SPECIALIZE_DISPATCH_SAME_OPARG() } OPCODE_DEFERRED_INC(UNPACK_SEQUENCE); ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); diff --git a/Python/optimizer.c b/Python/optimizer.c index 2fcf6b763691..ca9967e4272d 100644 --- a/Python/optimizer.c +++ b/Python/optimizer.c @@ -904,9 +904,10 @@ _PyJIT_InitializeTracing(PyThreadState *tstate, _PyInterpreterFrame *frame, _Py_ tstate->interp->jit_state.jit_tracer_previous_exit = exit; _Py_BloomFilter_Init(&tstate->interp->jit_state.jit_tracer_dependencies); tstate->interp->jit_state.jit_tracer_initial_stack_depth = curr_stackdepth; - tstate->interp->jit_state.jit_tracer_initial_chain_depth = chain_depth; + tstate->interp->jit_state.jit_tracer_initial_chain_depth = chain_depth % MAX_CHAIN_DEPTH; tstate->interp->jit_state.jit_tracer_current_frame = frame; tstate->interp->jit_state.jit_tracer_dependencies_still_valid = true; + tstate->interp->jit_state.last_specialized_instr = NULL; } void diff --git a/Tools/cases_generator/tracer_generator.py b/Tools/cases_generator/tracer_generator.py index 30a34a91d65b..7039079dd895 100644 --- a/Tools/cases_generator/tracer_generator.py +++ b/Tools/cases_generator/tracer_generator.py @@ -44,6 +44,7 @@ class TracerEmitter(Emitter): **self._replacers, "DISPATCH": self.dispatch, "DISPATCH_INLINED": self.dispatch_inlined, + "DISPATCH_SAME_OPARG": self.dispatch_same_oparg, } def dispatch( @@ -75,6 +76,26 @@ class TracerEmitter(Emitter): self.out.start_line() self.emit("TRACING_DISPATCH_INLINED") return False + + def dispatch_same_oparg( + self, + tkn: Token, + tkn_iter: TokenIterator, + uop: CodeSection, + storage: Storage, + inst: Instruction | None, + ) -> bool: + if storage.spilled: + raise analysis_error("stack_pointer needs reloading before dispatch", tkn) + storage.stack.flush(self.out) + self.out.start_line() + if "specializing" in uop.annotations: + self.emit("TRACING_SPECIALIZE_DISPATCH_SAME_OPARG") + else: + self.emit(tkn) + emit_to(self.out, tkn_iter, "SEMI") + return False + def record_jump_taken( self, tkn: Token,