From fa3e285d064c2e6748766e04ec6bcb71d0b68c65 Mon Sep 17 00:00:00 2001 From: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Mon, 10 Nov 2025 17:02:27 +0000 Subject: [PATCH] Clean up labels --- Include/internal/pycore_opcode_metadata.h | 4 +- Include/internal/pycore_optimizer.h | 2 +- Python/bytecodes.c | 62 +++++++-------- Python/ceval.c | 8 +- Python/generated_cases.c.h | 91 +++++++++++------------ Python/opcode_targets.h | 1 + Python/optimizer.c | 27 ++++--- 7 files changed, 91 insertions(+), 104 deletions(-) diff --git a/Include/internal/pycore_opcode_metadata.h b/Include/internal/pycore_opcode_metadata.h index 825321739c40..548627dc7982 100644 --- a/Include/internal/pycore_opcode_metadata.h +++ b/Include/internal/pycore_opcode_metadata.h @@ -1150,7 +1150,7 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[267] = { [END_ASYNC_FOR] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG | HAS_UNPREDICTABLE_JUMP_FLAG | HAS_NEEDS_GUARD_IP_FLAG }, [END_FOR] = { true, INSTR_FMT_IX, HAS_ESCAPES_FLAG | HAS_NO_SAVE_IP_FLAG }, [END_SEND] = { true, INSTR_FMT_IX, HAS_ESCAPES_FLAG | HAS_PURE_FLAG }, - [ENTER_EXECUTOR] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [ENTER_EXECUTOR] = { true, INSTR_FMT_IB, HAS_ARG_FLAG }, [EXIT_INIT_CHECK] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, [EXTENDED_ARG] = { true, INSTR_FMT_IB, HAS_ARG_FLAG }, [FORMAT_SIMPLE] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, @@ -1189,7 +1189,7 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[267] = { [INSTRUMENTED_RESUME] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, [INSTRUMENTED_RETURN_VALUE] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG | HAS_NEEDS_GUARD_IP_FLAG }, [INSTRUMENTED_YIELD_VALUE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG | HAS_NEEDS_GUARD_IP_FLAG }, - [INTERPRETER_EXIT] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [INTERPRETER_EXIT] = { true, INSTR_FMT_IX, HAS_ESCAPES_FLAG }, [IS_OP] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ESCAPES_FLAG }, [JUMP_BACKWARD] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_EVAL_BREAK_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [JUMP_BACKWARD_JIT] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_EVAL_BREAK_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, diff --git a/Include/internal/pycore_optimizer.h b/Include/internal/pycore_optimizer.h index e4219ebc86ae..653285a2c6b7 100644 --- a/Include/internal/pycore_optimizer.h +++ b/Include/internal/pycore_optimizer.h @@ -362,7 +362,7 @@ PyAPI_FUNC(int) _PyDumpExecutors(FILE *out); extern void _Py_ClearExecutorDeletionList(PyInterpreterState *interp); #endif -int _PyJit_translate_single_bytecode_to_trace(PyThreadState *tstate, _PyInterpreterFrame *frame, _Py_CODEUNIT *next_instr); +int _PyJit_translate_single_bytecode_to_trace(PyThreadState *tstate, _PyInterpreterFrame *frame, _Py_CODEUNIT *next_instr, bool stop_tracing); int _PyJit_TryInitializeTracing(PyThreadState *tstate, _PyInterpreterFrame *frame, diff --git a/Python/bytecodes.c b/Python/bytecodes.c index 283a18424715..5bfd9ea8a5a0 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -1218,33 +1218,19 @@ dummy_func( tstate->current_frame = frame->previous; assert(!_PyErr_Occurred(tstate)); PyObject *result = PyStackRef_AsPyObjectSteal(retval); - if (IS_JIT_TRACING()) { -#if _Py_TIER2 - _PyJit_translate_single_bytecode_to_trace(tstate, frame, NULL); - LEAVE_TRACING(); - int err = bail_tracing_and_jit(tstate, frame); - if (err < 0) { - Py_DECREF(result); - ERROR_IF(true); - } - return result; -#endif - } - else { #if !_Py_TAIL_CALL_INTERP - assert(frame == &entry.frame); + assert(frame == &entry.frame); #endif #ifdef _Py_TIER2 - _PyStackRef executor = frame->localsplus[0]; - assert(tstate->current_executor == NULL); - if (!PyStackRef_IsNull(executor)) { - tstate->current_executor = PyStackRef_AsPyObjectBorrow(executor); - PyStackRef_CLOSE(executor); - } -#endif - LLTRACE_RESUME_FRAME(); - return result; + _PyStackRef executor = frame->localsplus[0]; + assert(tstate->current_executor == NULL); + if (!PyStackRef_IsNull(executor)) { + tstate->current_executor = PyStackRef_AsPyObjectBorrow(executor); + PyStackRef_CLOSE(executor); } +#endif + LLTRACE_RESUME_FRAME(); + return result; } // The stack effect here is a bit misleading. @@ -3028,10 +3014,8 @@ dummy_func( tier1 inst(ENTER_EXECUTOR, (--)) { #ifdef _Py_TIER2 if (IS_JIT_TRACING()) { - _PyJit_translate_single_bytecode_to_trace(tstate, frame, next_instr); - LEAVE_TRACING(); - int err = bail_tracing_and_jit(tstate, frame); - ERROR_IF(err < 0); + next_instr = this_instr; + goto stop_tracing; } PyCodeObject *code = _PyFrame_GetCode(frame); _PyExecutorObject *executor = code->co_executors->executors[oparg & 255]; @@ -5657,7 +5641,10 @@ dummy_func( #if _Py_TIER2 assert(IS_JIT_TRACING()); int opcode = next_instr->op.code; - int full = !_PyJit_translate_single_bytecode_to_trace(tstate, frame, next_instr); + bool stop_tracing = (opcode == WITH_EXCEPT_START || + opcode == RERAISE || opcode == CLEANUP_THROW || + opcode == PUSH_EXC_INFO || opcode == INTERPRETER_EXIT); + int full = !_PyJit_translate_single_bytecode_to_trace(tstate, frame, next_instr, stop_tracing); if (full) { LEAVE_TRACING(); int err = bail_tracing_and_jit(tstate, frame); @@ -5677,9 +5664,9 @@ dummy_func( _tstate->jit_state.prev_state.instr = next_instr; } _tstate->jit_state.prev_state.specialize_counter = 0; - PyCodeObject *prev_code = (PyCodeObject *)Py_NewRef(PyStackRef_AsPyObjectBorrow(frame->f_executable)); - if (_tstate->jit_state.prev_state.instr_code != prev_code) { - Py_SETREF(_tstate->jit_state.prev_state.instr_code, prev_code); + PyObject *prev_code = PyStackRef_AsPyObjectBorrow(frame->f_executable); + if (_tstate->jit_state.prev_state.instr_code != (PyCodeObject *)prev_code) { + Py_SETREF(_tstate->jit_state.prev_state.instr_code, (PyCodeObject*)Py_NewRef((prev_code))); } _tstate->jit_state.prev_state.instr_frame = frame; @@ -5691,6 +5678,19 @@ dummy_func( #endif } + label(stop_tracing) { +#if _Py_TIER2 + assert(IS_JIT_TRACING()); + _PyJit_translate_single_bytecode_to_trace(tstate, frame, NULL, true); + LEAVE_TRACING(); + int err = bail_tracing_and_jit(tstate, frame); + ERROR_IF(err < 0); + DISPATCH_GOTO_NON_TRACING(); +#else + Py_FatalError("JIT label executed in non-jit build."); +#endif + } + // END BYTECODES // diff --git a/Python/ceval.c b/Python/ceval.c index e6424503e889..aea353603b54 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -1017,13 +1017,7 @@ bail_tracing_and_jit(PyThreadState *tstate, _PyInterpreterFrame *frame) // Likewise, we hold a strong reference to the executor containing this exit, so the exit is guaranteed // to be valid to access. if (err <= 0) { - // Some opcodes will forever be unchanged. Don't ever bother specializing for them ever again. - if (_tstate->jit_state.prev_state.instr->op.code == INTERPRETER_EXIT) { - exit->temperature = initial_unreachable_backoff_counter(); - } - else { - exit->temperature = restart_backoff_counter(exit->temperature); - } + exit->temperature = restart_backoff_counter(exit->temperature); } else { exit->temperature = initial_temperature_backoff_counter(); diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index 284d90869aea..29b294f0f6e9 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -5492,16 +5492,8 @@ opcode = ENTER_EXECUTOR; #ifdef _Py_TIER2 if (IS_JIT_TRACING()) { - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyJit_translate_single_bytecode_to_trace(tstate, frame, next_instr); - stack_pointer = _PyFrame_GetStackPointer(frame); - LEAVE_TRACING(); - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = bail_tracing_and_jit(tstate, frame); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err < 0) { - JUMP_TO_LABEL(error); - } + next_instr = this_instr; + JUMP_TO_LABEL(stop_tracing); } PyCodeObject *code = _PyFrame_GetCode(frame); _PyExecutorObject *executor = code->co_executors->executors[oparg & 255]; @@ -7552,46 +7544,24 @@ tstate->current_frame = frame->previous; assert(!_PyErr_Occurred(tstate)); PyObject *result = PyStackRef_AsPyObjectSteal(retval); - if (IS_JIT_TRACING()) { - #if _Py_TIER2 + #if !_Py_TAIL_CALL_INTERP + assert(frame == &entry.frame); + #endif + #ifdef _Py_TIER2 + _PyStackRef executor = frame->localsplus[0]; + assert(tstate->current_executor == NULL); + if (!PyStackRef_IsNull(executor)) { + tstate->current_executor = PyStackRef_AsPyObjectBorrow(executor); stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); _PyFrame_SetStackPointer(frame, stack_pointer); - _PyJit_translate_single_bytecode_to_trace(tstate, frame, NULL); - stack_pointer = _PyFrame_GetStackPointer(frame); - LEAVE_TRACING(); - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = bail_tracing_and_jit(tstate, frame); + PyStackRef_CLOSE(executor); stack_pointer = _PyFrame_GetStackPointer(frame); - if (err < 0) { - _PyFrame_SetStackPointer(frame, stack_pointer); - Py_DECREF(result); - stack_pointer = _PyFrame_GetStackPointer(frame); - JUMP_TO_LABEL(error); - } - return result; - #endif - } - else { - #if !_Py_TAIL_CALL_INTERP - assert(frame == &entry.frame); - #endif - #ifdef _Py_TIER2 - _PyStackRef executor = frame->localsplus[0]; - assert(tstate->current_executor == NULL); - if (!PyStackRef_IsNull(executor)) { - tstate->current_executor = PyStackRef_AsPyObjectBorrow(executor); - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(executor); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += 1; - } - #endif - LLTRACE_RESUME_FRAME(); - return result; + stack_pointer += 1; } + #endif + LLTRACE_RESUME_FRAME(); + return result; } TARGET(IS_OP) { @@ -12339,8 +12309,11 @@ JUMP_TO_LABEL(error); #if _Py_TIER2 assert(IS_JIT_TRACING()); int opcode = next_instr->op.code; + bool stop_tracing = (opcode == WITH_EXCEPT_START || + opcode == RERAISE || opcode == CLEANUP_THROW || + opcode == PUSH_EXC_INFO || opcode == INTERPRETER_EXIT); _PyFrame_SetStackPointer(frame, stack_pointer); - int full = !_PyJit_translate_single_bytecode_to_trace(tstate, frame, next_instr); + int full = !_PyJit_translate_single_bytecode_to_trace(tstate, frame, next_instr, stop_tracing); stack_pointer = _PyFrame_GetStackPointer(frame); if (full) { LEAVE_TRACING(); @@ -12363,10 +12336,10 @@ JUMP_TO_LABEL(error); _tstate->jit_state.prev_state.instr = next_instr; } _tstate->jit_state.prev_state.specialize_counter = 0; - PyCodeObject *prev_code = (PyCodeObject *)Py_NewRef(PyStackRef_AsPyObjectBorrow(frame->f_executable)); - if (_tstate->jit_state.prev_state.instr_code != prev_code) { + PyObject *prev_code = PyStackRef_AsPyObjectBorrow(frame->f_executable); + if (_tstate->jit_state.prev_state.instr_code != (PyCodeObject *)prev_code) { _PyFrame_SetStackPointer(frame, stack_pointer); - Py_SETREF(_tstate->jit_state.prev_state.instr_code, prev_code); + Py_SETREF(_tstate->jit_state.prev_state.instr_code, (PyCodeObject*)Py_NewRef((prev_code))); stack_pointer = _PyFrame_GetStackPointer(frame); } _tstate->jit_state.prev_state.instr_frame = frame; @@ -12378,5 +12351,25 @@ JUMP_TO_LABEL(error); #endif } + LABEL(stop_tracing) + { + #if _Py_TIER2 + assert(IS_JIT_TRACING()); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyJit_translate_single_bytecode_to_trace(tstate, frame, NULL, true); + stack_pointer = _PyFrame_GetStackPointer(frame); + LEAVE_TRACING(); + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = bail_tracing_and_jit(tstate, frame); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err < 0) { + JUMP_TO_LABEL(error); + } + DISPATCH_GOTO_NON_TRACING(); + #else + Py_FatalError("JIT label executed in non-jit build."); + #endif + } + /* END LABELS */ #undef TIER_ONE diff --git a/Python/opcode_targets.h b/Python/opcode_targets.h index 288e45ce8bc8..1b9196503b57 100644 --- a/Python/opcode_targets.h +++ b/Python/opcode_targets.h @@ -529,6 +529,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_exception_unwind(TAIL_CALL_PARAM Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_exit_unwind(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_start_frame(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_record_previous_inst(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_stop_tracing(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_OP(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_OP_ADD_FLOAT(TAIL_CALL_PARAMS); diff --git a/Python/optimizer.c b/Python/optimizer.c index 5a7de32f4607..64e57f26094d 100644 --- a/Python/optimizer.c +++ b/Python/optimizer.c @@ -552,7 +552,8 @@ int _PyJit_translate_single_bytecode_to_trace( PyThreadState *tstate, _PyInterpreterFrame *frame, - _Py_CODEUNIT *next_instr) + _Py_CODEUNIT *next_instr, + bool stop_tracing) { #ifdef Py_DEBUG @@ -571,11 +572,6 @@ _PyJit_translate_single_bytecode_to_trace( _PyUOpInstruction *trace = _tstate->jit_state.code_buffer; int max_length = _tstate->jit_state.prev_state.code_max_size; - int is_sys_tracing = (tstate->c_tracefunc != NULL) || (tstate->c_profilefunc != NULL); - if (is_sys_tracing) { - goto full; - } - _Py_CODEUNIT *this_instr = _tstate->jit_state.prev_state.instr; _Py_CODEUNIT *target_instr = this_instr; uint32_t target = 0; @@ -619,6 +615,17 @@ _PyJit_translate_single_bytecode_to_trace( DPRINTF(2, "Unsupported: dynamic jump taken %s\n", _PyOpcode_OpName[opcode]); goto unsupported; } + + int is_sys_tracing = (tstate->c_tracefunc != NULL) || (tstate->c_profilefunc != NULL); + if (is_sys_tracing) { + goto full; + } + + if (stop_tracing) { + ADD_TO_TRACE(_DEOPT, 0, 0, target); + goto done; + } + DPRINTF(2, "%p %d: %s(%d) %d %d\n", old_code, target, _PyOpcode_OpName[opcode], oparg, needs_guard_ip, old_stack_level); #ifdef Py_DEBUG @@ -655,11 +662,6 @@ _PyJit_translate_single_bytecode_to_trace( // TODO (gh-140277): The constituent use one extra stack slot. So we need to check for headroom. if (opcode == BINARY_OP_SUBSCR_GETITEM && old_stack_level + 1 > old_code->co_stacksize) { - goto unsupported; - } - - if (opcode == WITH_EXCEPT_START || opcode == RERAISE || opcode == CLEANUP_THROW || opcode == PUSH_EXC_INFO) { - DPRINTF(2, "Unsupported: strange control-flow\n"); unsupported: { // Rewind to previous instruction and replace with _EXIT_TRACE. @@ -773,9 +775,6 @@ _PyJit_translate_single_bytecode_to_trace( * start with RESUME_CHECK */ ADD_TO_TRACE(_TIER2_RESUME_CHECK, 0, 0, target); break; - case INTERPRETER_EXIT: - ADD_TO_TRACE(_DEOPT, 0, 0, target); - goto done;; default: { const struct opcode_macro_expansion *expansion = &_PyOpcode_macro_expansion[opcode]; -- 2.47.3