From: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Fri, 24 Oct 2025 10:27:20 +0000 (+0100) Subject: specialization and deopt fixes X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=eb970e06fdfd2bc28e2d79fa29e110803090b48d;p=thirdparty%2FPython%2Fcpython.git specialization and deopt fixes --- diff --git a/Include/internal/pycore_interp_structs.h b/Include/internal/pycore_interp_structs.h index c2cddfe47d3f..bc0ae6ea97d6 100644 --- a/Include/internal/pycore_interp_structs.h +++ b/Include/internal/pycore_interp_structs.h @@ -759,14 +759,15 @@ typedef _Py_CODEUNIT *(*_PyJitEntryFuncPtr)(struct _PyExecutorObject *exec, _PyI typedef struct _PyJitTracerState { bool dependencies_still_valid; - bool do_not_specialize; bool dynamic_jump_taken; + bool prev_instr_is_super; int code_max_size; int code_curr_size; int initial_stack_depth; int initial_chain_depth; int prev_instr_oparg; int prev_instr_stacklevel; + int specialize_counter; _PyUOpInstruction *code_buffer; _Py_CODEUNIT *insert_exec_instr; _Py_CODEUNIT *close_loop_instr; diff --git a/Python/bytecodes.c b/Python/bytecodes.c index be98f34ce819..e872dabda874 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -5229,10 +5229,21 @@ dummy_func( ERROR_IF(err < 0); DISPATCH_GOTO_NON_TRACING(); } - tstate->interp->jit_state.do_not_specialize = false; + // Super instructions. Instruction deopted, There's a mismatch in what the stack expects + // in the optimizer. So we have to reflect in the trace correctly. + if ((tstate->interp->jit_state.prev_instr->op.code == CALL_LIST_APPEND && + opcode == POP_TOP) || + (tstate->interp->jit_state.prev_instr->op.code == BINARY_OP_INPLACE_ADD_UNICODE && + opcode == STORE_FAST)) { + tstate->interp->jit_state.prev_instr_is_super = true; + } + else { + tstate->interp->jit_state.prev_instr = next_instr; + } + tstate->interp->jit_state.specialize_counter = 0; PyCodeObject *prev_code = (PyCodeObject *)Py_NewRef(_PyFrame_GetCode(frame)); Py_SETREF(tstate->interp->jit_state.prev_instr_code, prev_code); - tstate->interp->jit_state.prev_instr = next_instr; + tstate->interp->jit_state.prev_instr_frame = frame; tstate->interp->jit_state.prev_instr_oparg = oparg; tstate->interp->jit_state.prev_instr_stacklevel = STACK_LEVEL(); diff --git a/Python/ceval_macros.h b/Python/ceval_macros.h index b741707377ae..abef480b902a 100644 --- a/Python/ceval_macros.h +++ b/Python/ceval_macros.h @@ -135,8 +135,8 @@ #if (_Py_TAIL_CALL_INTERP || USE_COMPUTED_GOTOS) && _Py_TIER2 # define IS_JIT_TRACING() (DISPATCH_TABLE_VAR == TRACING_DISPATCH_TABLE) -// Required to not get stuck in infinite pecialization loops due to specialization failure. -# define IS_JIT_TRACING_MAKING_PROGRESS() (IS_JIT_TRACING() && !tstate->interp->jit_state.do_not_specialize) +// Required to not get stuck in infinite specialization loops due to specialization failure. +# define IS_JIT_TRACING_MAKING_PROGRESS() (IS_JIT_TRACING() && tstate->interp->jit_state.specialize_counter <= 2) # define ENTER_TRACING() \ DISPATCH_TABLE_VAR = TRACING_DISPATCH_TABLE; # define LEAVE_TRACING() \ @@ -195,7 +195,7 @@ do { \ #if _Py_TIER2 #define DISPATCH_SAME_OPARG() \ { \ - tstate->interp->jit_state.do_not_specialize = true; \ + tstate->interp->jit_state.specialize_counter++; \ opcode = next_instr->op.code; \ PRE_DISPATCH_GOTO(); \ DISPATCH_GOTO_NON_TRACING(); \ diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index 0665f985ff08..7b759811b392 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -103,6 +103,9 @@ if (!PyFloat_CheckExact(value_o)) { UPDATE_MISS_STATS(BINARY_OP); assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); + #if _Py_TIER2 + tstate->interp->jit_state.specialize_counter++; + #endif JUMP_TO_PREDICTED(BINARY_OP); } } @@ -113,6 +116,9 @@ if (!PyFloat_CheckExact(left_o)) { UPDATE_MISS_STATS(BINARY_OP); assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); + #if _Py_TIER2 + tstate->interp->jit_state.specialize_counter++; + #endif JUMP_TO_PREDICTED(BINARY_OP); } } @@ -161,6 +167,9 @@ if (!_PyLong_CheckExactAndCompact(value_o)) { UPDATE_MISS_STATS(BINARY_OP); assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); + #if _Py_TIER2 + tstate->interp->jit_state.specialize_counter++; + #endif JUMP_TO_PREDICTED(BINARY_OP); } } @@ -171,6 +180,9 @@ if (!_PyLong_CheckExactAndCompact(left_o)) { UPDATE_MISS_STATS(BINARY_OP); assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); + #if _Py_TIER2 + tstate->interp->jit_state.specialize_counter++; + #endif JUMP_TO_PREDICTED(BINARY_OP); } } @@ -188,6 +200,9 @@ if (PyStackRef_IsNull(res)) { UPDATE_MISS_STATS(BINARY_OP); assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); + #if _Py_TIER2 + tstate->interp->jit_state.specialize_counter++; + #endif JUMP_TO_PREDICTED(BINARY_OP); } PyStackRef_CLOSE_SPECIALIZED(right, _PyLong_ExactDealloc); @@ -222,6 +237,9 @@ if (!PyUnicode_CheckExact(value_o)) { UPDATE_MISS_STATS(BINARY_OP); assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); + #if _Py_TIER2 + tstate->interp->jit_state.specialize_counter++; + #endif JUMP_TO_PREDICTED(BINARY_OP); } } @@ -232,6 +250,9 @@ if (!PyUnicode_CheckExact(o)) { UPDATE_MISS_STATS(BINARY_OP); assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); + #if _Py_TIER2 + tstate->interp->jit_state.specialize_counter++; + #endif JUMP_TO_PREDICTED(BINARY_OP); } } @@ -290,6 +311,9 @@ if (!res) { UPDATE_MISS_STATS(BINARY_OP); assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); + #if _Py_TIER2 + tstate->interp->jit_state.specialize_counter++; + #endif JUMP_TO_PREDICTED(BINARY_OP); } } @@ -345,6 +369,9 @@ if (!PyUnicode_CheckExact(value_o)) { UPDATE_MISS_STATS(BINARY_OP); assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); + #if _Py_TIER2 + tstate->interp->jit_state.specialize_counter++; + #endif JUMP_TO_PREDICTED(BINARY_OP); } } @@ -355,6 +382,9 @@ if (!PyUnicode_CheckExact(o)) { UPDATE_MISS_STATS(BINARY_OP); assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); + #if _Py_TIER2 + tstate->interp->jit_state.specialize_counter++; + #endif JUMP_TO_PREDICTED(BINARY_OP); } } @@ -378,6 +408,9 @@ if (PyStackRef_AsPyObjectBorrow(*target_local) != left_o) { UPDATE_MISS_STATS(BINARY_OP); assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); + #if _Py_TIER2 + tstate->interp->jit_state.specialize_counter++; + #endif JUMP_TO_PREDICTED(BINARY_OP); } STAT_INC(BINARY_OP, hit); @@ -428,6 +461,9 @@ if (!PyFloat_CheckExact(value_o)) { UPDATE_MISS_STATS(BINARY_OP); assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); + #if _Py_TIER2 + tstate->interp->jit_state.specialize_counter++; + #endif JUMP_TO_PREDICTED(BINARY_OP); } } @@ -438,6 +474,9 @@ if (!PyFloat_CheckExact(left_o)) { UPDATE_MISS_STATS(BINARY_OP); assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); + #if _Py_TIER2 + tstate->interp->jit_state.specialize_counter++; + #endif JUMP_TO_PREDICTED(BINARY_OP); } } @@ -486,6 +525,9 @@ if (!_PyLong_CheckExactAndCompact(value_o)) { UPDATE_MISS_STATS(BINARY_OP); assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); + #if _Py_TIER2 + tstate->interp->jit_state.specialize_counter++; + #endif JUMP_TO_PREDICTED(BINARY_OP); } } @@ -496,6 +538,9 @@ if (!_PyLong_CheckExactAndCompact(left_o)) { UPDATE_MISS_STATS(BINARY_OP); assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); + #if _Py_TIER2 + tstate->interp->jit_state.specialize_counter++; + #endif JUMP_TO_PREDICTED(BINARY_OP); } } @@ -513,6 +558,9 @@ if (PyStackRef_IsNull(res)) { UPDATE_MISS_STATS(BINARY_OP); assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); + #if _Py_TIER2 + tstate->interp->jit_state.specialize_counter++; + #endif JUMP_TO_PREDICTED(BINARY_OP); } PyStackRef_CLOSE_SPECIALIZED(right, _PyLong_ExactDealloc); @@ -546,6 +594,9 @@ if (!PyDict_CheckExact(o)) { UPDATE_MISS_STATS(BINARY_OP); assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); + #if _Py_TIER2 + tstate->interp->jit_state.specialize_counter++; + #endif JUMP_TO_PREDICTED(BINARY_OP); } } @@ -611,6 +662,9 @@ if (tstate->interp->eval_frame) { UPDATE_MISS_STATS(BINARY_OP); assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); + #if _Py_TIER2 + tstate->interp->jit_state.specialize_counter++; + #endif JUMP_TO_PREDICTED(BINARY_OP); } } @@ -621,6 +675,9 @@ if (!PyType_HasFeature(tp, Py_TPFLAGS_HEAPTYPE)) { UPDATE_MISS_STATS(BINARY_OP); assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); + #if _Py_TIER2 + tstate->interp->jit_state.specialize_counter++; + #endif JUMP_TO_PREDICTED(BINARY_OP); } PyHeapTypeObject *ht = (PyHeapTypeObject *)tp; @@ -628,6 +685,9 @@ if (getitem_o == NULL) { UPDATE_MISS_STATS(BINARY_OP); assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); + #if _Py_TIER2 + tstate->interp->jit_state.specialize_counter++; + #endif JUMP_TO_PREDICTED(BINARY_OP); } assert(PyFunction_Check(getitem_o)); @@ -635,6 +695,9 @@ if (((PyFunctionObject *)getitem_o)->func_version != cached_version) { UPDATE_MISS_STATS(BINARY_OP); assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); + #if _Py_TIER2 + tstate->interp->jit_state.specialize_counter++; + #endif JUMP_TO_PREDICTED(BINARY_OP); } PyCodeObject *code = (PyCodeObject *)PyFunction_GET_CODE(getitem_o); @@ -642,6 +705,9 @@ if (!_PyThreadState_HasStackSpace(tstate, code->co_framesize)) { UPDATE_MISS_STATS(BINARY_OP); assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); + #if _Py_TIER2 + tstate->interp->jit_state.specialize_counter++; + #endif JUMP_TO_PREDICTED(BINARY_OP); } getitem = PyStackRef_FromPyObjectNew(getitem_o); @@ -697,6 +763,9 @@ if (!_PyLong_CheckExactAndCompact(value_o)) { UPDATE_MISS_STATS(BINARY_OP); assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); + #if _Py_TIER2 + tstate->interp->jit_state.specialize_counter++; + #endif JUMP_TO_PREDICTED(BINARY_OP); } } @@ -707,6 +776,9 @@ if (!PyList_CheckExact(o)) { UPDATE_MISS_STATS(BINARY_OP); assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); + #if _Py_TIER2 + tstate->interp->jit_state.specialize_counter++; + #endif JUMP_TO_PREDICTED(BINARY_OP); } } @@ -722,6 +794,9 @@ if (!_PyLong_IsNonNegativeCompact((PyLongObject *)sub)) { UPDATE_MISS_STATS(BINARY_OP); assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); + #if _Py_TIER2 + tstate->interp->jit_state.specialize_counter++; + #endif JUMP_TO_PREDICTED(BINARY_OP); } Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0]; @@ -732,6 +807,9 @@ if (res_o == NULL) { UPDATE_MISS_STATS(BINARY_OP); assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); + #if _Py_TIER2 + tstate->interp->jit_state.specialize_counter++; + #endif JUMP_TO_PREDICTED(BINARY_OP); } STAT_INC(BINARY_OP, hit); @@ -740,6 +818,9 @@ if (index >= PyList_GET_SIZE(list)) { UPDATE_MISS_STATS(BINARY_OP); assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); + #if _Py_TIER2 + tstate->interp->jit_state.specialize_counter++; + #endif JUMP_TO_PREDICTED(BINARY_OP); } STAT_INC(BINARY_OP, hit); @@ -787,6 +868,9 @@ if (!PySlice_Check(o)) { UPDATE_MISS_STATS(BINARY_OP); assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); + #if _Py_TIER2 + tstate->interp->jit_state.specialize_counter++; + #endif JUMP_TO_PREDICTED(BINARY_OP); } } @@ -797,6 +881,9 @@ if (!PyList_CheckExact(o)) { UPDATE_MISS_STATS(BINARY_OP); assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); + #if _Py_TIER2 + tstate->interp->jit_state.specialize_counter++; + #endif JUMP_TO_PREDICTED(BINARY_OP); } } @@ -859,6 +946,9 @@ if (!_PyLong_CheckExactAndCompact(value_o)) { UPDATE_MISS_STATS(BINARY_OP); assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); + #if _Py_TIER2 + tstate->interp->jit_state.specialize_counter++; + #endif JUMP_TO_PREDICTED(BINARY_OP); } } @@ -869,6 +959,9 @@ if (!PyUnicode_CheckExact(o)) { UPDATE_MISS_STATS(BINARY_OP); assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); + #if _Py_TIER2 + tstate->interp->jit_state.specialize_counter++; + #endif JUMP_TO_PREDICTED(BINARY_OP); } } @@ -884,18 +977,27 @@ if (!_PyLong_IsNonNegativeCompact((PyLongObject *)sub)) { UPDATE_MISS_STATS(BINARY_OP); assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); + #if _Py_TIER2 + tstate->interp->jit_state.specialize_counter++; + #endif JUMP_TO_PREDICTED(BINARY_OP); } Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0]; if (PyUnicode_GET_LENGTH(str) <= index) { UPDATE_MISS_STATS(BINARY_OP); assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); + #if _Py_TIER2 + tstate->interp->jit_state.specialize_counter++; + #endif JUMP_TO_PREDICTED(BINARY_OP); } Py_UCS4 c = PyUnicode_READ_CHAR(str, index); if (Py_ARRAY_LENGTH(_Py_SINGLETON(strings).ascii) <= c) { UPDATE_MISS_STATS(BINARY_OP); assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); + #if _Py_TIER2 + tstate->interp->jit_state.specialize_counter++; + #endif JUMP_TO_PREDICTED(BINARY_OP); } STAT_INC(BINARY_OP, hit); @@ -937,6 +1039,9 @@ if (!_PyLong_CheckExactAndCompact(value_o)) { UPDATE_MISS_STATS(BINARY_OP); assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); + #if _Py_TIER2 + tstate->interp->jit_state.specialize_counter++; + #endif JUMP_TO_PREDICTED(BINARY_OP); } } @@ -947,6 +1052,9 @@ if (!PyTuple_CheckExact(o)) { UPDATE_MISS_STATS(BINARY_OP); assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); + #if _Py_TIER2 + tstate->interp->jit_state.specialize_counter++; + #endif JUMP_TO_PREDICTED(BINARY_OP); } } @@ -962,12 +1070,18 @@ if (!_PyLong_IsNonNegativeCompact((PyLongObject *)sub)) { UPDATE_MISS_STATS(BINARY_OP); assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); + #if _Py_TIER2 + tstate->interp->jit_state.specialize_counter++; + #endif JUMP_TO_PREDICTED(BINARY_OP); } Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0]; if (index >= PyTuple_GET_SIZE(tuple)) { UPDATE_MISS_STATS(BINARY_OP); assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); + #if _Py_TIER2 + tstate->interp->jit_state.specialize_counter++; + #endif JUMP_TO_PREDICTED(BINARY_OP); } STAT_INC(BINARY_OP, hit); @@ -1009,6 +1123,9 @@ if (!PyFloat_CheckExact(value_o)) { UPDATE_MISS_STATS(BINARY_OP); assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); + #if _Py_TIER2 + tstate->interp->jit_state.specialize_counter++; + #endif JUMP_TO_PREDICTED(BINARY_OP); } } @@ -1019,6 +1136,9 @@ if (!PyFloat_CheckExact(left_o)) { UPDATE_MISS_STATS(BINARY_OP); assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); + #if _Py_TIER2 + tstate->interp->jit_state.specialize_counter++; + #endif JUMP_TO_PREDICTED(BINARY_OP); } } @@ -1067,6 +1187,9 @@ if (!_PyLong_CheckExactAndCompact(value_o)) { UPDATE_MISS_STATS(BINARY_OP); assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); + #if _Py_TIER2 + tstate->interp->jit_state.specialize_counter++; + #endif JUMP_TO_PREDICTED(BINARY_OP); } } @@ -1077,6 +1200,9 @@ if (!_PyLong_CheckExactAndCompact(left_o)) { UPDATE_MISS_STATS(BINARY_OP); assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); + #if _Py_TIER2 + tstate->interp->jit_state.specialize_counter++; + #endif JUMP_TO_PREDICTED(BINARY_OP); } } @@ -1094,6 +1220,9 @@ if (PyStackRef_IsNull(res)) { UPDATE_MISS_STATS(BINARY_OP); assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); + #if _Py_TIER2 + tstate->interp->jit_state.specialize_counter++; + #endif JUMP_TO_PREDICTED(BINARY_OP); } PyStackRef_CLOSE_SPECIALIZED(right, _PyLong_ExactDealloc); @@ -1709,6 +1838,9 @@ if (tstate->interp->eval_frame) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); + #if _Py_TIER2 + tstate->interp->jit_state.specialize_counter++; + #endif JUMP_TO_PREDICTED(CALL); } } @@ -1721,17 +1853,26 @@ if (!PyStackRef_IsNull(self_or_null)) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); + #if _Py_TIER2 + tstate->interp->jit_state.specialize_counter++; + #endif JUMP_TO_PREDICTED(CALL); } if (!PyType_Check(callable_o)) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); + #if _Py_TIER2 + tstate->interp->jit_state.specialize_counter++; + #endif JUMP_TO_PREDICTED(CALL); } PyTypeObject *tp = (PyTypeObject *)callable_o; if (FT_ATOMIC_LOAD_UINT32_RELAXED(tp->tp_version_tag) != type_version) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); + #if _Py_TIER2 + tstate->interp->jit_state.specialize_counter++; + #endif JUMP_TO_PREDICTED(CALL); } assert(tp->tp_new == PyBaseObject_Type.tp_new); @@ -1743,6 +1884,9 @@ if (!_PyThreadState_HasStackSpace(tstate, code->co_framesize + _Py_InitCleanup.co_framesize)) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); + #if _Py_TIER2 + tstate->interp->jit_state.specialize_counter++; + #endif JUMP_TO_PREDICTED(CALL); } STAT_INC(CALL, hit); @@ -1828,6 +1972,9 @@ if (tstate->interp->eval_frame) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); + #if _Py_TIER2 + tstate->interp->jit_state.specialize_counter++; + #endif JUMP_TO_PREDICTED(CALL); } } @@ -1838,11 +1985,17 @@ if (!PyStackRef_IsNull(null)) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); + #if _Py_TIER2 + tstate->interp->jit_state.specialize_counter++; + #endif JUMP_TO_PREDICTED(CALL); } if (Py_TYPE(PyStackRef_AsPyObjectBorrow(callable)) != &PyMethod_Type) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); + #if _Py_TIER2 + tstate->interp->jit_state.specialize_counter++; + #endif JUMP_TO_PREDICTED(CALL); } } @@ -1869,12 +2022,18 @@ if (!PyFunction_Check(callable_o)) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); + #if _Py_TIER2 + tstate->interp->jit_state.specialize_counter++; + #endif JUMP_TO_PREDICTED(CALL); } PyFunctionObject *func = (PyFunctionObject *)callable_o; if (func->func_version != func_version) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); + #if _Py_TIER2 + tstate->interp->jit_state.specialize_counter++; + #endif JUMP_TO_PREDICTED(CALL); } } @@ -1887,6 +2046,9 @@ if (code->co_argcount != oparg + (!PyStackRef_IsNull(self_or_null))) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); + #if _Py_TIER2 + tstate->interp->jit_state.specialize_counter++; + #endif JUMP_TO_PREDICTED(CALL); } } @@ -1898,6 +2060,9 @@ if (!_PyThreadState_HasStackSpace(tstate, code->co_framesize)) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); + #if _Py_TIER2 + tstate->interp->jit_state.specialize_counter++; + #endif JUMP_TO_PREDICTED(CALL); } } @@ -1906,6 +2071,9 @@ if (tstate->py_recursion_remaining <= 1) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); + #if _Py_TIER2 + tstate->interp->jit_state.specialize_counter++; + #endif JUMP_TO_PREDICTED(CALL); } } @@ -1971,6 +2139,9 @@ if (tstate->interp->eval_frame) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); + #if _Py_TIER2 + tstate->interp->jit_state.specialize_counter++; + #endif JUMP_TO_PREDICTED(CALL); } } @@ -1983,22 +2154,34 @@ if (Py_TYPE(callable_o) != &PyMethod_Type) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); + #if _Py_TIER2 + tstate->interp->jit_state.specialize_counter++; + #endif JUMP_TO_PREDICTED(CALL); } PyObject *func = ((PyMethodObject *)callable_o)->im_func; if (!PyFunction_Check(func)) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); + #if _Py_TIER2 + tstate->interp->jit_state.specialize_counter++; + #endif JUMP_TO_PREDICTED(CALL); } if (((PyFunctionObject *)func)->func_version != func_version) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); + #if _Py_TIER2 + tstate->interp->jit_state.specialize_counter++; + #endif JUMP_TO_PREDICTED(CALL); } if (!PyStackRef_IsNull(null)) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); + #if _Py_TIER2 + tstate->interp->jit_state.specialize_counter++; + #endif JUMP_TO_PREDICTED(CALL); } } @@ -2024,6 +2207,9 @@ if (tstate->py_recursion_remaining <= 1) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); + #if _Py_TIER2 + tstate->interp->jit_state.specialize_counter++; + #endif JUMP_TO_PREDICTED(CALL); } } @@ -2103,6 +2289,9 @@ if (!PyType_Check(callable_o)) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); + #if _Py_TIER2 + tstate->interp->jit_state.specialize_counter++; + #endif JUMP_TO_PREDICTED(CALL); } PyTypeObject *tp = (PyTypeObject *)callable_o; @@ -2115,6 +2304,9 @@ if (tp->tp_vectorcall == NULL) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); + #if _Py_TIER2 + tstate->interp->jit_state.specialize_counter++; + #endif JUMP_TO_PREDICTED(CALL); } STAT_INC(CALL, hit); @@ -2214,11 +2406,17 @@ if (!PyCFunction_CheckExact(callable_o)) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); + #if _Py_TIER2 + tstate->interp->jit_state.specialize_counter++; + #endif JUMP_TO_PREDICTED(CALL); } if (PyCFunction_GET_FLAGS(callable_o) != METH_FASTCALL) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); + #if _Py_TIER2 + tstate->interp->jit_state.specialize_counter++; + #endif JUMP_TO_PREDICTED(CALL); } STAT_INC(CALL, hit); @@ -2323,11 +2521,17 @@ if (!PyCFunction_CheckExact(callable_o)) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); + #if _Py_TIER2 + tstate->interp->jit_state.specialize_counter++; + #endif JUMP_TO_PREDICTED(CALL); } if (PyCFunction_GET_FLAGS(callable_o) != (METH_FASTCALL | METH_KEYWORDS)) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); + #if _Py_TIER2 + tstate->interp->jit_state.specialize_counter++; + #endif JUMP_TO_PREDICTED(CALL); } STAT_INC(CALL, hit); @@ -2431,21 +2635,33 @@ if (total_args != 1) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); + #if _Py_TIER2 + tstate->interp->jit_state.specialize_counter++; + #endif JUMP_TO_PREDICTED(CALL); } if (!PyCFunction_CheckExact(callable_o)) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); + #if _Py_TIER2 + tstate->interp->jit_state.specialize_counter++; + #endif JUMP_TO_PREDICTED(CALL); } if (PyCFunction_GET_FLAGS(callable_o) != METH_O) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); + #if _Py_TIER2 + tstate->interp->jit_state.specialize_counter++; + #endif JUMP_TO_PREDICTED(CALL); } if (_Py_ReachedRecursionLimit(tstate)) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); + #if _Py_TIER2 + tstate->interp->jit_state.specialize_counter++; + #endif JUMP_TO_PREDICTED(CALL); } STAT_INC(CALL, hit); @@ -2744,6 +2960,9 @@ if (!PyStackRef_IsNull(null)) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); + #if _Py_TIER2 + tstate->interp->jit_state.specialize_counter++; + #endif JUMP_TO_PREDICTED(CALL); } } @@ -2755,6 +2974,9 @@ if (callable_o != interp->callable_cache.isinstance) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); + #if _Py_TIER2 + tstate->interp->jit_state.specialize_counter++; + #endif JUMP_TO_PREDICTED(CALL); } } @@ -3001,6 +3223,9 @@ if (tstate->interp->eval_frame) { UPDATE_MISS_STATS(CALL_KW); assert(_PyOpcode_Deopt[opcode] == (CALL_KW)); + #if _Py_TIER2 + tstate->interp->jit_state.specialize_counter++; + #endif JUMP_TO_PREDICTED(CALL_KW); } } @@ -3013,22 +3238,34 @@ if (Py_TYPE(callable_o) != &PyMethod_Type) { UPDATE_MISS_STATS(CALL_KW); assert(_PyOpcode_Deopt[opcode] == (CALL_KW)); + #if _Py_TIER2 + tstate->interp->jit_state.specialize_counter++; + #endif JUMP_TO_PREDICTED(CALL_KW); } PyObject *func = ((PyMethodObject *)callable_o)->im_func; if (!PyFunction_Check(func)) { UPDATE_MISS_STATS(CALL_KW); assert(_PyOpcode_Deopt[opcode] == (CALL_KW)); + #if _Py_TIER2 + tstate->interp->jit_state.specialize_counter++; + #endif JUMP_TO_PREDICTED(CALL_KW); } if (((PyFunctionObject *)func)->func_version != func_version) { UPDATE_MISS_STATS(CALL_KW); assert(_PyOpcode_Deopt[opcode] == (CALL_KW)); + #if _Py_TIER2 + tstate->interp->jit_state.specialize_counter++; + #endif JUMP_TO_PREDICTED(CALL_KW); } if (!PyStackRef_IsNull(null)) { UPDATE_MISS_STATS(CALL_KW); assert(_PyOpcode_Deopt[opcode] == (CALL_KW)); + #if _Py_TIER2 + tstate->interp->jit_state.specialize_counter++; + #endif JUMP_TO_PREDICTED(CALL_KW); } } @@ -3134,11 +3371,17 @@ if (PyFunction_Check(callable_o)) { UPDATE_MISS_STATS(CALL_KW); assert(_PyOpcode_Deopt[opcode] == (CALL_KW)); + #if _Py_TIER2 + tstate->interp->jit_state.specialize_counter++; + #endif JUMP_TO_PREDICTED(CALL_KW); } if (Py_TYPE(callable_o) == &PyMethod_Type) { UPDATE_MISS_STATS(CALL_KW); assert(_PyOpcode_Deopt[opcode] == (CALL_KW)); + #if _Py_TIER2 + tstate->interp->jit_state.specialize_counter++; + #endif JUMP_TO_PREDICTED(CALL_KW); } } @@ -3257,6 +3500,9 @@ if (tstate->interp->eval_frame) { UPDATE_MISS_STATS(CALL_KW); assert(_PyOpcode_Deopt[opcode] == (CALL_KW)); + #if _Py_TIER2 + tstate->interp->jit_state.specialize_counter++; + #endif JUMP_TO_PREDICTED(CALL_KW); } } @@ -3268,12 +3514,18 @@ if (!PyFunction_Check(callable_o)) { UPDATE_MISS_STATS(CALL_KW); assert(_PyOpcode_Deopt[opcode] == (CALL_KW)); + #if _Py_TIER2 + tstate->interp->jit_state.specialize_counter++; + #endif JUMP_TO_PREDICTED(CALL_KW); } PyFunctionObject *func = (PyFunctionObject *)callable_o; if (func->func_version != func_version) { UPDATE_MISS_STATS(CALL_KW); assert(_PyOpcode_Deopt[opcode] == (CALL_KW)); + #if _Py_TIER2 + tstate->interp->jit_state.specialize_counter++; + #endif JUMP_TO_PREDICTED(CALL_KW); } } @@ -3282,6 +3534,9 @@ if (tstate->py_recursion_remaining <= 1) { UPDATE_MISS_STATS(CALL_KW); assert(_PyOpcode_Deopt[opcode] == (CALL_KW)); + #if _Py_TIER2 + tstate->interp->jit_state.specialize_counter++; + #endif JUMP_TO_PREDICTED(CALL_KW); } } @@ -3368,6 +3623,9 @@ if (!PyStackRef_IsNull(null)) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); + #if _Py_TIER2 + tstate->interp->jit_state.specialize_counter++; + #endif JUMP_TO_PREDICTED(CALL); } } @@ -3379,6 +3637,9 @@ if (callable_o != interp->callable_cache.len) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); + #if _Py_TIER2 + tstate->interp->jit_state.specialize_counter++; + #endif JUMP_TO_PREDICTED(CALL); } } @@ -3442,6 +3703,9 @@ if (callable_o != interp->callable_cache.list_append) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); + #if _Py_TIER2 + tstate->interp->jit_state.specialize_counter++; + #endif JUMP_TO_PREDICTED(CALL); } } @@ -3452,6 +3716,9 @@ if (o == NULL) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); + #if _Py_TIER2 + tstate->interp->jit_state.specialize_counter++; + #endif JUMP_TO_PREDICTED(CALL); } } @@ -3461,6 +3728,9 @@ if (!PyList_CheckExact(o)) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); + #if _Py_TIER2 + tstate->interp->jit_state.specialize_counter++; + #endif JUMP_TO_PREDICTED(CALL); } } @@ -3473,11 +3743,17 @@ if (!PyList_CheckExact(self_o)) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); + #if _Py_TIER2 + tstate->interp->jit_state.specialize_counter++; + #endif JUMP_TO_PREDICTED(CALL); } if (!LOCK_OBJECT(self_o)) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); + #if _Py_TIER2 + tstate->interp->jit_state.specialize_counter++; + #endif JUMP_TO_PREDICTED(CALL); } STAT_INC(CALL, hit); @@ -3537,18 +3813,27 @@ if (total_args == 0) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); + #if _Py_TIER2 + tstate->interp->jit_state.specialize_counter++; + #endif JUMP_TO_PREDICTED(CALL); } PyMethodDescrObject *method = (PyMethodDescrObject *)callable_o; if (!Py_IS_TYPE(method, &PyMethodDescr_Type)) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); + #if _Py_TIER2 + tstate->interp->jit_state.specialize_counter++; + #endif JUMP_TO_PREDICTED(CALL); } PyMethodDef *meth = method->d_method; if (meth->ml_flags != METH_FASTCALL) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); + #if _Py_TIER2 + tstate->interp->jit_state.specialize_counter++; + #endif JUMP_TO_PREDICTED(CALL); } PyObject *self = PyStackRef_AsPyObjectBorrow(arguments[0]); @@ -3556,6 +3841,9 @@ if (!Py_IS_TYPE(self, method->d_common.d_type)) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); + #if _Py_TIER2 + tstate->interp->jit_state.specialize_counter++; + #endif JUMP_TO_PREDICTED(CALL); } STAT_INC(CALL, hit); @@ -3658,18 +3946,27 @@ if (total_args == 0) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); + #if _Py_TIER2 + tstate->interp->jit_state.specialize_counter++; + #endif JUMP_TO_PREDICTED(CALL); } PyMethodDescrObject *method = (PyMethodDescrObject *)callable_o; if (!Py_IS_TYPE(method, &PyMethodDescr_Type)) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); + #if _Py_TIER2 + tstate->interp->jit_state.specialize_counter++; + #endif JUMP_TO_PREDICTED(CALL); } PyMethodDef *meth = method->d_method; if (meth->ml_flags != (METH_FASTCALL|METH_KEYWORDS)) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); + #if _Py_TIER2 + tstate->interp->jit_state.specialize_counter++; + #endif JUMP_TO_PREDICTED(CALL); } PyTypeObject *d_type = method->d_common.d_type; @@ -3678,6 +3975,9 @@ if (!Py_IS_TYPE(self, d_type)) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); + #if _Py_TIER2 + tstate->interp->jit_state.specialize_counter++; + #endif JUMP_TO_PREDICTED(CALL); } STAT_INC(CALL, hit); @@ -3781,12 +4081,18 @@ if (total_args != 1) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); + #if _Py_TIER2 + tstate->interp->jit_state.specialize_counter++; + #endif JUMP_TO_PREDICTED(CALL); } PyMethodDescrObject *method = (PyMethodDescrObject *)callable_o; if (!Py_IS_TYPE(method, &PyMethodDescr_Type)) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); + #if _Py_TIER2 + tstate->interp->jit_state.specialize_counter++; + #endif JUMP_TO_PREDICTED(CALL); } PyMethodDef *meth = method->d_method; @@ -3795,16 +4101,25 @@ if (!Py_IS_TYPE(self, method->d_common.d_type)) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); + #if _Py_TIER2 + tstate->interp->jit_state.specialize_counter++; + #endif JUMP_TO_PREDICTED(CALL); } if (meth->ml_flags != METH_NOARGS) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); + #if _Py_TIER2 + tstate->interp->jit_state.specialize_counter++; + #endif JUMP_TO_PREDICTED(CALL); } if (_Py_ReachedRecursionLimit(tstate)) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); + #if _Py_TIER2 + tstate->interp->jit_state.specialize_counter++; + #endif JUMP_TO_PREDICTED(CALL); } STAT_INC(CALL, hit); @@ -3875,22 +4190,34 @@ if (total_args != 2) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); + #if _Py_TIER2 + tstate->interp->jit_state.specialize_counter++; + #endif JUMP_TO_PREDICTED(CALL); } if (!Py_IS_TYPE(method, &PyMethodDescr_Type)) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); + #if _Py_TIER2 + tstate->interp->jit_state.specialize_counter++; + #endif JUMP_TO_PREDICTED(CALL); } PyMethodDef *meth = method->d_method; if (meth->ml_flags != METH_O) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); + #if _Py_TIER2 + tstate->interp->jit_state.specialize_counter++; + #endif JUMP_TO_PREDICTED(CALL); } if (_Py_ReachedRecursionLimit(tstate)) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); + #if _Py_TIER2 + tstate->interp->jit_state.specialize_counter++; + #endif JUMP_TO_PREDICTED(CALL); } _PyStackRef arg_stackref = arguments[1]; @@ -3899,6 +4226,9 @@ method->d_common.d_type)) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); + #if _Py_TIER2 + tstate->interp->jit_state.specialize_counter++; + #endif JUMP_TO_PREDICTED(CALL); } STAT_INC(CALL, hit); @@ -3973,11 +4303,17 @@ if (PyFunction_Check(callable_o)) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); + #if _Py_TIER2 + tstate->interp->jit_state.specialize_counter++; + #endif JUMP_TO_PREDICTED(CALL); } if (Py_TYPE(callable_o) == &PyMethod_Type) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); + #if _Py_TIER2 + tstate->interp->jit_state.specialize_counter++; + #endif JUMP_TO_PREDICTED(CALL); } } @@ -4084,6 +4420,9 @@ if (tstate->interp->eval_frame) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); + #if _Py_TIER2 + tstate->interp->jit_state.specialize_counter++; + #endif JUMP_TO_PREDICTED(CALL); } } @@ -4095,12 +4434,18 @@ if (!PyFunction_Check(callable_o)) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); + #if _Py_TIER2 + tstate->interp->jit_state.specialize_counter++; + #endif JUMP_TO_PREDICTED(CALL); } PyFunctionObject *func = (PyFunctionObject *)callable_o; if (func->func_version != func_version) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); + #if _Py_TIER2 + tstate->interp->jit_state.specialize_counter++; + #endif JUMP_TO_PREDICTED(CALL); } } @@ -4114,6 +4459,9 @@ if (code->co_argcount != oparg + (!PyStackRef_IsNull(self_or_null))) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); + #if _Py_TIER2 + tstate->interp->jit_state.specialize_counter++; + #endif JUMP_TO_PREDICTED(CALL); } } @@ -4125,6 +4473,9 @@ if (!_PyThreadState_HasStackSpace(tstate, code->co_framesize)) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); + #if _Py_TIER2 + tstate->interp->jit_state.specialize_counter++; + #endif JUMP_TO_PREDICTED(CALL); } } @@ -4133,6 +4484,9 @@ if (tstate->py_recursion_remaining <= 1) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); + #if _Py_TIER2 + tstate->interp->jit_state.specialize_counter++; + #endif JUMP_TO_PREDICTED(CALL); } } @@ -4197,6 +4551,9 @@ if (tstate->interp->eval_frame) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); + #if _Py_TIER2 + tstate->interp->jit_state.specialize_counter++; + #endif JUMP_TO_PREDICTED(CALL); } } @@ -4208,12 +4565,18 @@ if (!PyFunction_Check(callable_o)) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); + #if _Py_TIER2 + tstate->interp->jit_state.specialize_counter++; + #endif JUMP_TO_PREDICTED(CALL); } PyFunctionObject *func = (PyFunctionObject *)callable_o; if (func->func_version != func_version) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); + #if _Py_TIER2 + tstate->interp->jit_state.specialize_counter++; + #endif JUMP_TO_PREDICTED(CALL); } } @@ -4222,6 +4585,9 @@ if (tstate->py_recursion_remaining <= 1) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); + #if _Py_TIER2 + tstate->interp->jit_state.specialize_counter++; + #endif JUMP_TO_PREDICTED(CALL); } } @@ -4299,6 +4665,9 @@ if (!PyStackRef_IsNull(null)) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); + #if _Py_TIER2 + tstate->interp->jit_state.specialize_counter++; + #endif JUMP_TO_PREDICTED(CALL); } } @@ -4309,6 +4678,9 @@ if (callable_o != (PyObject *)&PyUnicode_Type) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); + #if _Py_TIER2 + tstate->interp->jit_state.specialize_counter++; + #endif JUMP_TO_PREDICTED(CALL); } } @@ -4371,6 +4743,9 @@ if (!PyStackRef_IsNull(null)) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); + #if _Py_TIER2 + tstate->interp->jit_state.specialize_counter++; + #endif JUMP_TO_PREDICTED(CALL); } } @@ -4381,6 +4756,9 @@ if (callable_o != (PyObject *)&PyTuple_Type) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); + #if _Py_TIER2 + tstate->interp->jit_state.specialize_counter++; + #endif JUMP_TO_PREDICTED(CALL); } } @@ -4443,6 +4821,9 @@ if (!PyStackRef_IsNull(null)) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); + #if _Py_TIER2 + tstate->interp->jit_state.specialize_counter++; + #endif JUMP_TO_PREDICTED(CALL); } } @@ -4453,6 +4834,9 @@ if (callable_o != (PyObject *)&PyType_Type) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); + #if _Py_TIER2 + tstate->interp->jit_state.specialize_counter++; + #endif JUMP_TO_PREDICTED(CALL); } } @@ -4737,6 +5121,9 @@ if (!PyFloat_CheckExact(value_o)) { UPDATE_MISS_STATS(COMPARE_OP); assert(_PyOpcode_Deopt[opcode] == (COMPARE_OP)); + #if _Py_TIER2 + tstate->interp->jit_state.specialize_counter++; + #endif JUMP_TO_PREDICTED(COMPARE_OP); } } @@ -4747,6 +5134,9 @@ if (!PyFloat_CheckExact(left_o)) { UPDATE_MISS_STATS(COMPARE_OP); assert(_PyOpcode_Deopt[opcode] == (COMPARE_OP)); + #if _Py_TIER2 + tstate->interp->jit_state.specialize_counter++; + #endif JUMP_TO_PREDICTED(COMPARE_OP); } } @@ -4792,6 +5182,9 @@ if (!_PyLong_CheckExactAndCompact(value_o)) { UPDATE_MISS_STATS(COMPARE_OP); assert(_PyOpcode_Deopt[opcode] == (COMPARE_OP)); + #if _Py_TIER2 + tstate->interp->jit_state.specialize_counter++; + #endif JUMP_TO_PREDICTED(COMPARE_OP); } } @@ -4802,6 +5195,9 @@ if (!_PyLong_CheckExactAndCompact(left_o)) { UPDATE_MISS_STATS(COMPARE_OP); assert(_PyOpcode_Deopt[opcode] == (COMPARE_OP)); + #if _Py_TIER2 + tstate->interp->jit_state.specialize_counter++; + #endif JUMP_TO_PREDICTED(COMPARE_OP); } } @@ -4852,6 +5248,9 @@ if (!PyUnicode_CheckExact(value_o)) { UPDATE_MISS_STATS(COMPARE_OP); assert(_PyOpcode_Deopt[opcode] == (COMPARE_OP)); + #if _Py_TIER2 + tstate->interp->jit_state.specialize_counter++; + #endif JUMP_TO_PREDICTED(COMPARE_OP); } } @@ -4862,6 +5261,9 @@ if (!PyUnicode_CheckExact(o)) { UPDATE_MISS_STATS(COMPARE_OP); assert(_PyOpcode_Deopt[opcode] == (COMPARE_OP)); + #if _Py_TIER2 + tstate->interp->jit_state.specialize_counter++; + #endif JUMP_TO_PREDICTED(COMPARE_OP); } } @@ -4970,6 +5372,9 @@ if (!PyDict_CheckExact(o)) { UPDATE_MISS_STATS(CONTAINS_OP); assert(_PyOpcode_Deopt[opcode] == (CONTAINS_OP)); + #if _Py_TIER2 + tstate->interp->jit_state.specialize_counter++; + #endif JUMP_TO_PREDICTED(CONTAINS_OP); } } @@ -5028,6 +5433,9 @@ if (!PyAnySet_CheckExact(o)) { UPDATE_MISS_STATS(CONTAINS_OP); assert(_PyOpcode_Deopt[opcode] == (CONTAINS_OP)); + #if _Py_TIER2 + tstate->interp->jit_state.specialize_counter++; + #endif JUMP_TO_PREDICTED(CONTAINS_OP); } } @@ -5686,6 +6094,9 @@ if (tstate->interp->eval_frame) { UPDATE_MISS_STATS(FOR_ITER); assert(_PyOpcode_Deopt[opcode] == (FOR_ITER)); + #if _Py_TIER2 + tstate->interp->jit_state.specialize_counter++; + #endif JUMP_TO_PREDICTED(FOR_ITER); } } @@ -5696,18 +6107,27 @@ if (Py_TYPE(gen) != &PyGen_Type) { UPDATE_MISS_STATS(FOR_ITER); assert(_PyOpcode_Deopt[opcode] == (FOR_ITER)); + #if _Py_TIER2 + tstate->interp->jit_state.specialize_counter++; + #endif JUMP_TO_PREDICTED(FOR_ITER); } #ifdef Py_GIL_DISABLED if (!_PyObject_IsUniquelyReferenced((PyObject *)gen)) { UPDATE_MISS_STATS(FOR_ITER); assert(_PyOpcode_Deopt[opcode] == (FOR_ITER)); + #if _Py_TIER2 + tstate->interp->jit_state.specialize_counter++; + #endif JUMP_TO_PREDICTED(FOR_ITER); } #endif if (gen->gi_frame_state >= FRAME_EXECUTING) { UPDATE_MISS_STATS(FOR_ITER); assert(_PyOpcode_Deopt[opcode] == (FOR_ITER)); + #if _Py_TIER2 + tstate->interp->jit_state.specialize_counter++; + #endif JUMP_TO_PREDICTED(FOR_ITER); } STAT_INC(FOR_ITER, hit); @@ -5760,6 +6180,9 @@ if (Py_TYPE(iter_o) != &PyList_Type) { UPDATE_MISS_STATS(FOR_ITER); assert(_PyOpcode_Deopt[opcode] == (FOR_ITER)); + #if _Py_TIER2 + tstate->interp->jit_state.specialize_counter++; + #endif JUMP_TO_PREDICTED(FOR_ITER); } assert(PyStackRef_IsTaggedInt(null_or_index)); @@ -5767,6 +6190,9 @@ if (!_Py_IsOwnedByCurrentThread(iter_o) && !_PyObject_GC_IS_SHARED(iter_o)) { UPDATE_MISS_STATS(FOR_ITER); assert(_PyOpcode_Deopt[opcode] == (FOR_ITER)); + #if _Py_TIER2 + tstate->interp->jit_state.specialize_counter++; + #endif JUMP_TO_PREDICTED(FOR_ITER); } #endif @@ -5802,6 +6228,9 @@ if (result < 0) { UPDATE_MISS_STATS(FOR_ITER); assert(_PyOpcode_Deopt[opcode] == (FOR_ITER)); + #if _Py_TIER2 + tstate->interp->jit_state.specialize_counter++; + #endif JUMP_TO_PREDICTED(FOR_ITER); } if (result == 0) { @@ -5843,12 +6272,18 @@ if (Py_TYPE(r) != &PyRangeIter_Type) { UPDATE_MISS_STATS(FOR_ITER); assert(_PyOpcode_Deopt[opcode] == (FOR_ITER)); + #if _Py_TIER2 + tstate->interp->jit_state.specialize_counter++; + #endif JUMP_TO_PREDICTED(FOR_ITER); } #ifdef Py_GIL_DISABLED if (!_PyObject_IsUniquelyReferenced((PyObject *)r)) { UPDATE_MISS_STATS(FOR_ITER); assert(_PyOpcode_Deopt[opcode] == (FOR_ITER)); + #if _Py_TIER2 + tstate->interp->jit_state.specialize_counter++; + #endif JUMP_TO_PREDICTED(FOR_ITER); } #endif @@ -5913,6 +6348,9 @@ if (Py_TYPE(iter_o) != &PyTuple_Type) { UPDATE_MISS_STATS(FOR_ITER); assert(_PyOpcode_Deopt[opcode] == (FOR_ITER)); + #if _Py_TIER2 + tstate->interp->jit_state.specialize_counter++; + #endif JUMP_TO_PREDICTED(FOR_ITER); } assert(PyStackRef_IsTaggedInt(null_or_index)); @@ -7914,12 +8352,18 @@ if (!PyType_Check(owner_o)) { UPDATE_MISS_STATS(LOAD_ATTR); assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + #if _Py_TIER2 + tstate->interp->jit_state.specialize_counter++; + #endif JUMP_TO_PREDICTED(LOAD_ATTR); } assert(type_version != 0); if (FT_ATOMIC_LOAD_UINT_RELAXED(((PyTypeObject *)owner_o)->tp_version_tag) != type_version) { UPDATE_MISS_STATS(LOAD_ATTR); assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + #if _Py_TIER2 + tstate->interp->jit_state.specialize_counter++; + #endif JUMP_TO_PREDICTED(LOAD_ATTR); } } @@ -7972,12 +8416,18 @@ if (!PyType_Check(owner_o)) { UPDATE_MISS_STATS(LOAD_ATTR); assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + #if _Py_TIER2 + tstate->interp->jit_state.specialize_counter++; + #endif JUMP_TO_PREDICTED(LOAD_ATTR); } assert(type_version != 0); if (FT_ATOMIC_LOAD_UINT_RELAXED(((PyTypeObject *)owner_o)->tp_version_tag) != type_version) { UPDATE_MISS_STATS(LOAD_ATTR); assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + #if _Py_TIER2 + tstate->interp->jit_state.specialize_counter++; + #endif JUMP_TO_PREDICTED(LOAD_ATTR); } } @@ -7989,6 +8439,9 @@ if (FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version) { UPDATE_MISS_STATS(LOAD_ATTR); assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + #if _Py_TIER2 + tstate->interp->jit_state.specialize_counter++; + #endif JUMP_TO_PREDICTED(LOAD_ATTR); } } @@ -8039,6 +8492,9 @@ if (tstate->interp->eval_frame) { UPDATE_MISS_STATS(LOAD_ATTR); assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + #if _Py_TIER2 + tstate->interp->jit_state.specialize_counter++; + #endif JUMP_TO_PREDICTED(LOAD_ATTR); } PyTypeObject *cls = Py_TYPE(owner_o); @@ -8046,6 +8502,9 @@ if (FT_ATOMIC_LOAD_UINT_RELAXED(cls->tp_version_tag) != type_version) { UPDATE_MISS_STATS(LOAD_ATTR); assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + #if _Py_TIER2 + tstate->interp->jit_state.specialize_counter++; + #endif JUMP_TO_PREDICTED(LOAD_ATTR); } assert(Py_IS_TYPE(getattribute, &PyFunction_Type)); @@ -8054,6 +8513,9 @@ if (f->func_version != func_version) { UPDATE_MISS_STATS(LOAD_ATTR); assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + #if _Py_TIER2 + tstate->interp->jit_state.specialize_counter++; + #endif JUMP_TO_PREDICTED(LOAD_ATTR); } PyCodeObject *code = (PyCodeObject *)f->func_code; @@ -8061,6 +8523,9 @@ if (!_PyThreadState_HasStackSpace(tstate, code->co_framesize)) { UPDATE_MISS_STATS(LOAD_ATTR); assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + #if _Py_TIER2 + tstate->interp->jit_state.specialize_counter++; + #endif JUMP_TO_PREDICTED(LOAD_ATTR); } STAT_INC(LOAD_ATTR, hit); @@ -8099,6 +8564,9 @@ if (FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version) { UPDATE_MISS_STATS(LOAD_ATTR); assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + #if _Py_TIER2 + tstate->interp->jit_state.specialize_counter++; + #endif JUMP_TO_PREDICTED(LOAD_ATTR); } } @@ -8110,6 +8578,9 @@ if (!FT_ATOMIC_LOAD_UINT8(_PyObject_InlineValues(owner_o)->valid)) { UPDATE_MISS_STATS(LOAD_ATTR); assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + #if _Py_TIER2 + tstate->interp->jit_state.specialize_counter++; + #endif JUMP_TO_PREDICTED(LOAD_ATTR); } } @@ -8122,6 +8593,9 @@ if (attr_o == NULL) { UPDATE_MISS_STATS(LOAD_ATTR); assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + #if _Py_TIER2 + tstate->interp->jit_state.specialize_counter++; + #endif JUMP_TO_PREDICTED(LOAD_ATTR); } #ifdef Py_GIL_DISABLED @@ -8130,6 +8604,9 @@ if (true) { UPDATE_MISS_STATS(LOAD_ATTR); assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + #if _Py_TIER2 + tstate->interp->jit_state.specialize_counter++; + #endif JUMP_TO_PREDICTED(LOAD_ATTR); } } @@ -8179,6 +8656,9 @@ if (FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version) { UPDATE_MISS_STATS(LOAD_ATTR); assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + #if _Py_TIER2 + tstate->interp->jit_state.specialize_counter++; + #endif JUMP_TO_PREDICTED(LOAD_ATTR); } } @@ -8190,6 +8670,9 @@ if (dict != NULL) { UPDATE_MISS_STATS(LOAD_ATTR); assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + #if _Py_TIER2 + tstate->interp->jit_state.specialize_counter++; + #endif JUMP_TO_PREDICTED(LOAD_ATTR); } } @@ -8235,6 +8718,9 @@ if (FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version) { UPDATE_MISS_STATS(LOAD_ATTR); assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + #if _Py_TIER2 + tstate->interp->jit_state.specialize_counter++; + #endif JUMP_TO_PREDICTED(LOAD_ATTR); } } @@ -8281,6 +8767,9 @@ if (FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version) { UPDATE_MISS_STATS(LOAD_ATTR); assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + #if _Py_TIER2 + tstate->interp->jit_state.specialize_counter++; + #endif JUMP_TO_PREDICTED(LOAD_ATTR); } } @@ -8292,6 +8781,9 @@ if (!FT_ATOMIC_LOAD_UINT8(ivs->valid)) { UPDATE_MISS_STATS(LOAD_ATTR); assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + #if _Py_TIER2 + tstate->interp->jit_state.specialize_counter++; + #endif JUMP_TO_PREDICTED(LOAD_ATTR); } } @@ -8304,6 +8796,9 @@ if (FT_ATOMIC_LOAD_UINT32_RELAXED(keys->dk_version) != keys_version) { UPDATE_MISS_STATS(LOAD_ATTR); assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + #if _Py_TIER2 + tstate->interp->jit_state.specialize_counter++; + #endif JUMP_TO_PREDICTED(LOAD_ATTR); } } @@ -8348,6 +8843,9 @@ if (Py_TYPE(owner_o)->tp_getattro != PyModule_Type.tp_getattro) { UPDATE_MISS_STATS(LOAD_ATTR); assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + #if _Py_TIER2 + tstate->interp->jit_state.specialize_counter++; + #endif JUMP_TO_PREDICTED(LOAD_ATTR); } PyDictObject *dict = (PyDictObject *)((PyModuleObject *)owner_o)->md_dict; @@ -8356,6 +8854,9 @@ if (FT_ATOMIC_LOAD_UINT32_RELAXED(keys->dk_version) != dict_version) { UPDATE_MISS_STATS(LOAD_ATTR); assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + #if _Py_TIER2 + tstate->interp->jit_state.specialize_counter++; + #endif JUMP_TO_PREDICTED(LOAD_ATTR); } assert(keys->dk_kind == DICT_KEYS_UNICODE); @@ -8365,6 +8866,9 @@ if (attr_o == NULL) { UPDATE_MISS_STATS(LOAD_ATTR); assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + #if _Py_TIER2 + tstate->interp->jit_state.specialize_counter++; + #endif JUMP_TO_PREDICTED(LOAD_ATTR); } #ifdef Py_GIL_DISABLED @@ -8373,6 +8877,9 @@ if (true) { UPDATE_MISS_STATS(LOAD_ATTR); assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + #if _Py_TIER2 + tstate->interp->jit_state.specialize_counter++; + #endif JUMP_TO_PREDICTED(LOAD_ATTR); } } @@ -8421,6 +8928,9 @@ if (FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version) { UPDATE_MISS_STATS(LOAD_ATTR); assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + #if _Py_TIER2 + tstate->interp->jit_state.specialize_counter++; + #endif JUMP_TO_PREDICTED(LOAD_ATTR); } } @@ -8468,6 +8978,9 @@ if (FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version) { UPDATE_MISS_STATS(LOAD_ATTR); assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + #if _Py_TIER2 + tstate->interp->jit_state.specialize_counter++; + #endif JUMP_TO_PREDICTED(LOAD_ATTR); } } @@ -8479,6 +8992,9 @@ if (!FT_ATOMIC_LOAD_UINT8(ivs->valid)) { UPDATE_MISS_STATS(LOAD_ATTR); assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + #if _Py_TIER2 + tstate->interp->jit_state.specialize_counter++; + #endif JUMP_TO_PREDICTED(LOAD_ATTR); } } @@ -8491,6 +9007,9 @@ if (FT_ATOMIC_LOAD_UINT32_RELAXED(keys->dk_version) != keys_version) { UPDATE_MISS_STATS(LOAD_ATTR); assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + #if _Py_TIER2 + tstate->interp->jit_state.specialize_counter++; + #endif JUMP_TO_PREDICTED(LOAD_ATTR); } } @@ -8532,6 +9051,9 @@ if (tstate->interp->eval_frame) { UPDATE_MISS_STATS(LOAD_ATTR); assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + #if _Py_TIER2 + tstate->interp->jit_state.specialize_counter++; + #endif JUMP_TO_PREDICTED(LOAD_ATTR); } } @@ -8544,6 +9066,9 @@ if (FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version) { UPDATE_MISS_STATS(LOAD_ATTR); assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + #if _Py_TIER2 + tstate->interp->jit_state.specialize_counter++; + #endif JUMP_TO_PREDICTED(LOAD_ATTR); } } @@ -8558,21 +9083,33 @@ if ((code->co_flags & (CO_VARKEYWORDS | CO_VARARGS | CO_OPTIMIZED)) != CO_OPTIMIZED) { UPDATE_MISS_STATS(LOAD_ATTR); assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + #if _Py_TIER2 + tstate->interp->jit_state.specialize_counter++; + #endif JUMP_TO_PREDICTED(LOAD_ATTR); } if (code->co_kwonlyargcount) { UPDATE_MISS_STATS(LOAD_ATTR); assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + #if _Py_TIER2 + tstate->interp->jit_state.specialize_counter++; + #endif JUMP_TO_PREDICTED(LOAD_ATTR); } if (code->co_argcount != 1) { UPDATE_MISS_STATS(LOAD_ATTR); assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + #if _Py_TIER2 + tstate->interp->jit_state.specialize_counter++; + #endif JUMP_TO_PREDICTED(LOAD_ATTR); } if (!_PyThreadState_HasStackSpace(tstate, code->co_framesize)) { UPDATE_MISS_STATS(LOAD_ATTR); assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + #if _Py_TIER2 + tstate->interp->jit_state.specialize_counter++; + #endif JUMP_TO_PREDICTED(LOAD_ATTR); } STAT_INC(LOAD_ATTR, hit); @@ -8631,6 +9168,9 @@ if (FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version) { UPDATE_MISS_STATS(LOAD_ATTR); assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + #if _Py_TIER2 + tstate->interp->jit_state.specialize_counter++; + #endif JUMP_TO_PREDICTED(LOAD_ATTR); } } @@ -8643,6 +9183,9 @@ if (attr_o == NULL) { UPDATE_MISS_STATS(LOAD_ATTR); assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + #if _Py_TIER2 + tstate->interp->jit_state.specialize_counter++; + #endif JUMP_TO_PREDICTED(LOAD_ATTR); } #ifdef Py_GIL_DISABLED @@ -8650,6 +9193,9 @@ if (!increfed) { UPDATE_MISS_STATS(LOAD_ATTR); assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + #if _Py_TIER2 + tstate->interp->jit_state.specialize_counter++; + #endif JUMP_TO_PREDICTED(LOAD_ATTR); } #else @@ -8700,6 +9246,9 @@ if (FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version) { UPDATE_MISS_STATS(LOAD_ATTR); assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + #if _Py_TIER2 + tstate->interp->jit_state.specialize_counter++; + #endif JUMP_TO_PREDICTED(LOAD_ATTR); } } @@ -8712,6 +9261,9 @@ if (dict == NULL) { UPDATE_MISS_STATS(LOAD_ATTR); assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + #if _Py_TIER2 + tstate->interp->jit_state.specialize_counter++; + #endif JUMP_TO_PREDICTED(LOAD_ATTR); } PyDictKeysObject *dk = FT_ATOMIC_LOAD_PTR(dict->ma_keys); @@ -8720,6 +9272,9 @@ if (!_Py_IsOwnedByCurrentThread((PyObject *)dict) && !_PyObject_GC_IS_SHARED(dict)) { UPDATE_MISS_STATS(LOAD_ATTR); assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + #if _Py_TIER2 + tstate->interp->jit_state.specialize_counter++; + #endif JUMP_TO_PREDICTED(LOAD_ATTR); } #endif @@ -8728,6 +9283,9 @@ if (true) { UPDATE_MISS_STATS(LOAD_ATTR); assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + #if _Py_TIER2 + tstate->interp->jit_state.specialize_counter++; + #endif JUMP_TO_PREDICTED(LOAD_ATTR); } } @@ -8736,6 +9294,9 @@ if (true) { UPDATE_MISS_STATS(LOAD_ATTR); assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + #if _Py_TIER2 + tstate->interp->jit_state.specialize_counter++; + #endif JUMP_TO_PREDICTED(LOAD_ATTR); } } @@ -8744,6 +9305,9 @@ if (true) { UPDATE_MISS_STATS(LOAD_ATTR); assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + #if _Py_TIER2 + tstate->interp->jit_state.specialize_counter++; + #endif JUMP_TO_PREDICTED(LOAD_ATTR); } } @@ -8752,6 +9316,9 @@ if (true) { UPDATE_MISS_STATS(LOAD_ATTR); assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + #if _Py_TIER2 + tstate->interp->jit_state.specialize_counter++; + #endif JUMP_TO_PREDICTED(LOAD_ATTR); } } @@ -8762,6 +9329,9 @@ if (true) { UPDATE_MISS_STATS(LOAD_ATTR); assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + #if _Py_TIER2 + tstate->interp->jit_state.specialize_counter++; + #endif JUMP_TO_PREDICTED(LOAD_ATTR); } } @@ -9194,12 +9764,18 @@ if (!PyDict_CheckExact(dict)) { UPDATE_MISS_STATS(LOAD_GLOBAL); assert(_PyOpcode_Deopt[opcode] == (LOAD_GLOBAL)); + #if _Py_TIER2 + tstate->interp->jit_state.specialize_counter++; + #endif JUMP_TO_PREDICTED(LOAD_GLOBAL); } PyDictKeysObject *keys = FT_ATOMIC_LOAD_PTR_ACQUIRE(dict->ma_keys); if (FT_ATOMIC_LOAD_UINT32_RELAXED(keys->dk_version) != version) { UPDATE_MISS_STATS(LOAD_GLOBAL); assert(_PyOpcode_Deopt[opcode] == (LOAD_GLOBAL)); + #if _Py_TIER2 + tstate->interp->jit_state.specialize_counter++; + #endif JUMP_TO_PREDICTED(LOAD_GLOBAL); } assert(DK_IS_UNICODE(keys)); @@ -9212,12 +9788,18 @@ if (!PyDict_CheckExact(dict)) { UPDATE_MISS_STATS(LOAD_GLOBAL); assert(_PyOpcode_Deopt[opcode] == (LOAD_GLOBAL)); + #if _Py_TIER2 + tstate->interp->jit_state.specialize_counter++; + #endif JUMP_TO_PREDICTED(LOAD_GLOBAL); } PyDictKeysObject *keys = FT_ATOMIC_LOAD_PTR_ACQUIRE(dict->ma_keys); if (FT_ATOMIC_LOAD_UINT32_RELAXED(keys->dk_version) != version) { UPDATE_MISS_STATS(LOAD_GLOBAL); assert(_PyOpcode_Deopt[opcode] == (LOAD_GLOBAL)); + #if _Py_TIER2 + tstate->interp->jit_state.specialize_counter++; + #endif JUMP_TO_PREDICTED(LOAD_GLOBAL); } assert(DK_IS_UNICODE(keys)); @@ -9226,6 +9808,9 @@ if (res_o == NULL) { UPDATE_MISS_STATS(LOAD_GLOBAL); assert(_PyOpcode_Deopt[opcode] == (LOAD_GLOBAL)); + #if _Py_TIER2 + tstate->interp->jit_state.specialize_counter++; + #endif JUMP_TO_PREDICTED(LOAD_GLOBAL); } #if Py_GIL_DISABLED @@ -9233,6 +9818,9 @@ if (!increfed) { UPDATE_MISS_STATS(LOAD_GLOBAL); assert(_PyOpcode_Deopt[opcode] == (LOAD_GLOBAL)); + #if _Py_TIER2 + tstate->interp->jit_state.specialize_counter++; + #endif JUMP_TO_PREDICTED(LOAD_GLOBAL); } #else @@ -9278,12 +9866,18 @@ if (!PyDict_CheckExact(dict)) { UPDATE_MISS_STATS(LOAD_GLOBAL); assert(_PyOpcode_Deopt[opcode] == (LOAD_GLOBAL)); + #if _Py_TIER2 + tstate->interp->jit_state.specialize_counter++; + #endif JUMP_TO_PREDICTED(LOAD_GLOBAL); } PyDictKeysObject *keys = FT_ATOMIC_LOAD_PTR_ACQUIRE(dict->ma_keys); if (FT_ATOMIC_LOAD_UINT32_RELAXED(keys->dk_version) != version) { UPDATE_MISS_STATS(LOAD_GLOBAL); assert(_PyOpcode_Deopt[opcode] == (LOAD_GLOBAL)); + #if _Py_TIER2 + tstate->interp->jit_state.specialize_counter++; + #endif JUMP_TO_PREDICTED(LOAD_GLOBAL); } assert(DK_IS_UNICODE(keys)); @@ -9293,6 +9887,9 @@ if (res_o == NULL) { UPDATE_MISS_STATS(LOAD_GLOBAL); assert(_PyOpcode_Deopt[opcode] == (LOAD_GLOBAL)); + #if _Py_TIER2 + tstate->interp->jit_state.specialize_counter++; + #endif JUMP_TO_PREDICTED(LOAD_GLOBAL); } #if Py_GIL_DISABLED @@ -9300,6 +9897,9 @@ if (!increfed) { UPDATE_MISS_STATS(LOAD_GLOBAL); assert(_PyOpcode_Deopt[opcode] == (LOAD_GLOBAL)); + #if _Py_TIER2 + tstate->interp->jit_state.specialize_counter++; + #endif JUMP_TO_PREDICTED(LOAD_GLOBAL); } #else @@ -9594,11 +10194,17 @@ if (global_super != (PyObject *)&PySuper_Type) { UPDATE_MISS_STATS(LOAD_SUPER_ATTR); assert(_PyOpcode_Deopt[opcode] == (LOAD_SUPER_ATTR)); + #if _Py_TIER2 + tstate->interp->jit_state.specialize_counter++; + #endif JUMP_TO_PREDICTED(LOAD_SUPER_ATTR); } if (!PyType_Check(class)) { UPDATE_MISS_STATS(LOAD_SUPER_ATTR); assert(_PyOpcode_Deopt[opcode] == (LOAD_SUPER_ATTR)); + #if _Py_TIER2 + tstate->interp->jit_state.specialize_counter++; + #endif JUMP_TO_PREDICTED(LOAD_SUPER_ATTR); } STAT_INC(LOAD_SUPER_ATTR, hit); @@ -9657,11 +10263,17 @@ if (global_super != (PyObject *)&PySuper_Type) { UPDATE_MISS_STATS(LOAD_SUPER_ATTR); assert(_PyOpcode_Deopt[opcode] == (LOAD_SUPER_ATTR)); + #if _Py_TIER2 + tstate->interp->jit_state.specialize_counter++; + #endif JUMP_TO_PREDICTED(LOAD_SUPER_ATTR); } if (!PyType_Check(class)) { UPDATE_MISS_STATS(LOAD_SUPER_ATTR); assert(_PyOpcode_Deopt[opcode] == (LOAD_SUPER_ATTR)); + #if _Py_TIER2 + tstate->interp->jit_state.specialize_counter++; + #endif JUMP_TO_PREDICTED(LOAD_SUPER_ATTR); } STAT_INC(LOAD_SUPER_ATTR, hit); @@ -10215,12 +10827,20 @@ } DISPATCH_GOTO_NON_TRACING(); } - tstate->interp->jit_state.do_not_specialize = false; + if ((tstate->interp->jit_state.prev_instr->op.code == CALL_LIST_APPEND && + opcode == POP_TOP) || + (tstate->interp->jit_state.prev_instr->op.code == BINARY_OP_INPLACE_ADD_UNICODE && + opcode == STORE_FAST)) { + tstate->interp->jit_state.prev_instr_is_super = true; + } + else { + tstate->interp->jit_state.prev_instr = next_instr; + } + tstate->interp->jit_state.specialize_counter = 0; PyCodeObject *prev_code = (PyCodeObject *)Py_NewRef(_PyFrame_GetCode(frame)); _PyFrame_SetStackPointer(frame, stack_pointer); Py_SETREF(tstate->interp->jit_state.prev_instr_code, prev_code); stack_pointer = _PyFrame_GetStackPointer(frame); - tstate->interp->jit_state.prev_instr = next_instr; tstate->interp->jit_state.prev_instr_frame = frame; tstate->interp->jit_state.prev_instr_oparg = oparg; tstate->interp->jit_state.prev_instr_stacklevel = STACK_LEVEL(); @@ -10362,6 +10982,9 @@ if (_Py_emscripten_signal_clock == 0) { UPDATE_MISS_STATS(RESUME); assert(_PyOpcode_Deopt[opcode] == (RESUME)); + #if _Py_TIER2 + tstate->interp->jit_state.specialize_counter++; + #endif JUMP_TO_PREDICTED(RESUME); } _Py_emscripten_signal_clock -= Py_EMSCRIPTEN_SIGNAL_HANDLING; @@ -10372,6 +10995,9 @@ if (eval_breaker != version) { UPDATE_MISS_STATS(RESUME); assert(_PyOpcode_Deopt[opcode] == (RESUME)); + #if _Py_TIER2 + tstate->interp->jit_state.specialize_counter++; + #endif JUMP_TO_PREDICTED(RESUME); } #ifdef Py_GIL_DISABLED @@ -10379,6 +11005,9 @@ ((_PyThreadStateImpl *)tstate)->tlbc_index) { UPDATE_MISS_STATS(RESUME); assert(_PyOpcode_Deopt[opcode] == (RESUME)); + #if _Py_TIER2 + tstate->interp->jit_state.specialize_counter++; + #endif JUMP_TO_PREDICTED(RESUME); } #endif @@ -10592,6 +11221,9 @@ if (tstate->interp->eval_frame) { UPDATE_MISS_STATS(SEND); assert(_PyOpcode_Deopt[opcode] == (SEND)); + #if _Py_TIER2 + tstate->interp->jit_state.specialize_counter++; + #endif JUMP_TO_PREDICTED(SEND); } } @@ -10603,11 +11235,17 @@ if (Py_TYPE(gen) != &PyGen_Type && Py_TYPE(gen) != &PyCoro_Type) { UPDATE_MISS_STATS(SEND); assert(_PyOpcode_Deopt[opcode] == (SEND)); + #if _Py_TIER2 + tstate->interp->jit_state.specialize_counter++; + #endif JUMP_TO_PREDICTED(SEND); } if (gen->gi_frame_state >= FRAME_EXECUTING) { UPDATE_MISS_STATS(SEND); assert(_PyOpcode_Deopt[opcode] == (SEND)); + #if _Py_TIER2 + tstate->interp->jit_state.specialize_counter++; + #endif JUMP_TO_PREDICTED(SEND); } STAT_INC(SEND, hit); @@ -10845,6 +11483,9 @@ if (!LOCK_OBJECT(owner_o)) { UPDATE_MISS_STATS(STORE_ATTR); assert(_PyOpcode_Deopt[opcode] == (STORE_ATTR)); + #if _Py_TIER2 + tstate->interp->jit_state.specialize_counter++; + #endif JUMP_TO_PREDICTED(STORE_ATTR); } PyTypeObject *tp = Py_TYPE(owner_o); @@ -10853,6 +11494,9 @@ if (true) { UPDATE_MISS_STATS(STORE_ATTR); assert(_PyOpcode_Deopt[opcode] == (STORE_ATTR)); + #if _Py_TIER2 + tstate->interp->jit_state.specialize_counter++; + #endif JUMP_TO_PREDICTED(STORE_ATTR); } } @@ -10868,6 +11512,9 @@ if (true) { UPDATE_MISS_STATS(STORE_ATTR); assert(_PyOpcode_Deopt[opcode] == (STORE_ATTR)); + #if _Py_TIER2 + tstate->interp->jit_state.specialize_counter++; + #endif JUMP_TO_PREDICTED(STORE_ATTR); } } @@ -10921,6 +11568,9 @@ if (FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version) { UPDATE_MISS_STATS(STORE_ATTR); assert(_PyOpcode_Deopt[opcode] == (STORE_ATTR)); + #if _Py_TIER2 + tstate->interp->jit_state.specialize_counter++; + #endif JUMP_TO_PREDICTED(STORE_ATTR); } } @@ -10932,6 +11582,9 @@ if (!LOCK_OBJECT(owner_o)) { UPDATE_MISS_STATS(STORE_ATTR); assert(_PyOpcode_Deopt[opcode] == (STORE_ATTR)); + #if _Py_TIER2 + tstate->interp->jit_state.specialize_counter++; + #endif JUMP_TO_PREDICTED(STORE_ATTR); } char *addr = (char *)owner_o + index; @@ -10972,6 +11625,9 @@ if (FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version) { UPDATE_MISS_STATS(STORE_ATTR); assert(_PyOpcode_Deopt[opcode] == (STORE_ATTR)); + #if _Py_TIER2 + tstate->interp->jit_state.specialize_counter++; + #endif JUMP_TO_PREDICTED(STORE_ATTR); } } @@ -10985,11 +11641,17 @@ if (dict == NULL) { UPDATE_MISS_STATS(STORE_ATTR); assert(_PyOpcode_Deopt[opcode] == (STORE_ATTR)); + #if _Py_TIER2 + tstate->interp->jit_state.specialize_counter++; + #endif JUMP_TO_PREDICTED(STORE_ATTR); } if (!LOCK_OBJECT(dict)) { UPDATE_MISS_STATS(STORE_ATTR); assert(_PyOpcode_Deopt[opcode] == (STORE_ATTR)); + #if _Py_TIER2 + tstate->interp->jit_state.specialize_counter++; + #endif JUMP_TO_PREDICTED(STORE_ATTR); } assert(PyDict_CheckExact((PyObject *)dict)); @@ -11000,6 +11662,9 @@ if (true) { UPDATE_MISS_STATS(STORE_ATTR); assert(_PyOpcode_Deopt[opcode] == (STORE_ATTR)); + #if _Py_TIER2 + tstate->interp->jit_state.specialize_counter++; + #endif JUMP_TO_PREDICTED(STORE_ATTR); } } @@ -11009,6 +11674,9 @@ if (true) { UPDATE_MISS_STATS(STORE_ATTR); assert(_PyOpcode_Deopt[opcode] == (STORE_ATTR)); + #if _Py_TIER2 + tstate->interp->jit_state.specialize_counter++; + #endif JUMP_TO_PREDICTED(STORE_ATTR); } } @@ -11018,6 +11686,9 @@ if (true) { UPDATE_MISS_STATS(STORE_ATTR); assert(_PyOpcode_Deopt[opcode] == (STORE_ATTR)); + #if _Py_TIER2 + tstate->interp->jit_state.specialize_counter++; + #endif JUMP_TO_PREDICTED(STORE_ATTR); } } @@ -11342,6 +12013,9 @@ if (!PyDict_CheckExact(o)) { UPDATE_MISS_STATS(STORE_SUBSCR); assert(_PyOpcode_Deopt[opcode] == (STORE_SUBSCR)); + #if _Py_TIER2 + tstate->interp->jit_state.specialize_counter++; + #endif JUMP_TO_PREDICTED(STORE_SUBSCR); } } @@ -11393,6 +12067,9 @@ if (!_PyLong_CheckExactAndCompact(value_o)) { UPDATE_MISS_STATS(STORE_SUBSCR); assert(_PyOpcode_Deopt[opcode] == (STORE_SUBSCR)); + #if _Py_TIER2 + tstate->interp->jit_state.specialize_counter++; + #endif JUMP_TO_PREDICTED(STORE_SUBSCR); } } @@ -11403,6 +12080,9 @@ if (!PyList_CheckExact(o)) { UPDATE_MISS_STATS(STORE_SUBSCR); assert(_PyOpcode_Deopt[opcode] == (STORE_SUBSCR)); + #if _Py_TIER2 + tstate->interp->jit_state.specialize_counter++; + #endif JUMP_TO_PREDICTED(STORE_SUBSCR); } } @@ -11419,12 +12099,18 @@ if (!_PyLong_IsNonNegativeCompact((PyLongObject *)sub)) { UPDATE_MISS_STATS(STORE_SUBSCR); assert(_PyOpcode_Deopt[opcode] == (STORE_SUBSCR)); + #if _Py_TIER2 + tstate->interp->jit_state.specialize_counter++; + #endif JUMP_TO_PREDICTED(STORE_SUBSCR); } Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0]; if (!LOCK_OBJECT(list)) { UPDATE_MISS_STATS(STORE_SUBSCR); assert(_PyOpcode_Deopt[opcode] == (STORE_SUBSCR)); + #if _Py_TIER2 + tstate->interp->jit_state.specialize_counter++; + #endif JUMP_TO_PREDICTED(STORE_SUBSCR); } if (index >= PyList_GET_SIZE(list)) { @@ -11432,6 +12118,9 @@ if (true) { UPDATE_MISS_STATS(STORE_SUBSCR); assert(_PyOpcode_Deopt[opcode] == (STORE_SUBSCR)); + #if _Py_TIER2 + tstate->interp->jit_state.specialize_counter++; + #endif JUMP_TO_PREDICTED(STORE_SUBSCR); } } @@ -11548,6 +12237,9 @@ if (FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version) { UPDATE_MISS_STATS(TO_BOOL); assert(_PyOpcode_Deopt[opcode] == (TO_BOOL)); + #if _Py_TIER2 + tstate->interp->jit_state.specialize_counter++; + #endif JUMP_TO_PREDICTED(TO_BOOL); } } @@ -11585,6 +12277,9 @@ if (!PyStackRef_BoolCheck(value)) { UPDATE_MISS_STATS(TO_BOOL); assert(_PyOpcode_Deopt[opcode] == (TO_BOOL)); + #if _Py_TIER2 + tstate->interp->jit_state.specialize_counter++; + #endif JUMP_TO_PREDICTED(TO_BOOL); } STAT_INC(TO_BOOL, hit); @@ -11611,6 +12306,9 @@ if (!PyLong_CheckExact(value_o)) { UPDATE_MISS_STATS(TO_BOOL); assert(_PyOpcode_Deopt[opcode] == (TO_BOOL)); + #if _Py_TIER2 + tstate->interp->jit_state.specialize_counter++; + #endif JUMP_TO_PREDICTED(TO_BOOL); } STAT_INC(TO_BOOL, hit); @@ -11652,6 +12350,9 @@ if (!PyList_CheckExact(o)) { UPDATE_MISS_STATS(TO_BOOL); assert(_PyOpcode_Deopt[opcode] == (TO_BOOL)); + #if _Py_TIER2 + tstate->interp->jit_state.specialize_counter++; + #endif JUMP_TO_PREDICTED(TO_BOOL); } } @@ -11693,6 +12394,9 @@ if (!PyStackRef_IsNone(value)) { UPDATE_MISS_STATS(TO_BOOL); assert(_PyOpcode_Deopt[opcode] == (TO_BOOL)); + #if _Py_TIER2 + tstate->interp->jit_state.specialize_counter++; + #endif JUMP_TO_PREDICTED(TO_BOOL); } STAT_INC(TO_BOOL, hit); @@ -11721,6 +12425,9 @@ if (!PyUnicode_CheckExact(value_o)) { UPDATE_MISS_STATS(TO_BOOL); assert(_PyOpcode_Deopt[opcode] == (TO_BOOL)); + #if _Py_TIER2 + tstate->interp->jit_state.specialize_counter++; + #endif JUMP_TO_PREDICTED(TO_BOOL); } } @@ -11924,6 +12631,9 @@ if (!PyList_CheckExact(o)) { UPDATE_MISS_STATS(UNPACK_SEQUENCE); assert(_PyOpcode_Deopt[opcode] == (UNPACK_SEQUENCE)); + #if _Py_TIER2 + tstate->interp->jit_state.specialize_counter++; + #endif JUMP_TO_PREDICTED(UNPACK_SEQUENCE); } } @@ -11937,6 +12647,9 @@ if (!LOCK_OBJECT(seq_o)) { UPDATE_MISS_STATS(UNPACK_SEQUENCE); assert(_PyOpcode_Deopt[opcode] == (UNPACK_SEQUENCE)); + #if _Py_TIER2 + tstate->interp->jit_state.specialize_counter++; + #endif JUMP_TO_PREDICTED(UNPACK_SEQUENCE); } if (PyList_GET_SIZE(seq_o) != oparg) { @@ -11944,6 +12657,9 @@ if (true) { UPDATE_MISS_STATS(UNPACK_SEQUENCE); assert(_PyOpcode_Deopt[opcode] == (UNPACK_SEQUENCE)); + #if _Py_TIER2 + tstate->interp->jit_state.specialize_counter++; + #endif JUMP_TO_PREDICTED(UNPACK_SEQUENCE); } } @@ -11983,6 +12699,9 @@ if (!PyTuple_CheckExact(o)) { UPDATE_MISS_STATS(UNPACK_SEQUENCE); assert(_PyOpcode_Deopt[opcode] == (UNPACK_SEQUENCE)); + #if _Py_TIER2 + tstate->interp->jit_state.specialize_counter++; + #endif JUMP_TO_PREDICTED(UNPACK_SEQUENCE); } } @@ -11996,6 +12715,9 @@ if (PyTuple_GET_SIZE(seq_o) != oparg) { UPDATE_MISS_STATS(UNPACK_SEQUENCE); assert(_PyOpcode_Deopt[opcode] == (UNPACK_SEQUENCE)); + #if _Py_TIER2 + tstate->interp->jit_state.specialize_counter++; + #endif JUMP_TO_PREDICTED(UNPACK_SEQUENCE); } STAT_INC(UNPACK_SEQUENCE, hit); @@ -12034,6 +12756,9 @@ if (!PyTuple_CheckExact(o)) { UPDATE_MISS_STATS(UNPACK_SEQUENCE); assert(_PyOpcode_Deopt[opcode] == (UNPACK_SEQUENCE)); + #if _Py_TIER2 + tstate->interp->jit_state.specialize_counter++; + #endif JUMP_TO_PREDICTED(UNPACK_SEQUENCE); } } @@ -12047,6 +12772,9 @@ if (PyTuple_GET_SIZE(seq_o) != 2) { UPDATE_MISS_STATS(UNPACK_SEQUENCE); assert(_PyOpcode_Deopt[opcode] == (UNPACK_SEQUENCE)); + #if _Py_TIER2 + tstate->interp->jit_state.specialize_counter++; + #endif JUMP_TO_PREDICTED(UNPACK_SEQUENCE); } STAT_INC(UNPACK_SEQUENCE, hit); diff --git a/Python/optimizer.c b/Python/optimizer.c index c11cd22cf9df..31c9995c4ed6 100644 --- a/Python/optimizer.c +++ b/Python/optimizer.c @@ -593,14 +593,20 @@ _PyJit_translate_single_bytecode_to_trace( // We must point to the first EXTENDED_ARG when deopting. int oparg = tstate->interp->jit_state.prev_instr_oparg; int opcode = this_instr->op.code; + // Failed specialization twice in a row. Deopt! + if (tstate->interp->jit_state.specialize_counter >= 3) { + opcode = _PyOpcode_Deopt[opcode]; + } int rewind_oparg = oparg; while (rewind_oparg > 255) { rewind_oparg >>= 8; target--; } + int old_stack_level = tstate->interp->jit_state.prev_instr_stacklevel; + bool needs_guard_ip = _PyOpcode_NeedsGuardIp[opcode]; - DPRINTF(2, "%p %d: %s(%d) %d\n", old_code, target, _PyOpcode_OpName[opcode], oparg, needs_guard_ip); + DPRINTF(2, "%p %d: %s(%d) %d %d\n", old_code, target, _PyOpcode_OpName[opcode], oparg, needs_guard_ip, old_stack_level); #ifdef Py_DEBUG if (oparg > 255) { @@ -608,11 +614,16 @@ _PyJit_translate_single_bytecode_to_trace( } #endif + // Skip over super instructions. + if (tstate->interp->jit_state.prev_instr_is_super) { + tstate->interp->jit_state.prev_instr_is_super = false; + return 1; + } + if (!tstate->interp->jit_state.dependencies_still_valid) { goto done; } - int old_stack_level = tstate->interp->jit_state.prev_instr_oparg; // Strange control-flow, unsupported opcode, etc. if (tstate->interp->jit_state.dynamic_jump_taken) { goto unsupported; @@ -939,13 +950,14 @@ _PyJit_InitializeTracing(PyThreadState *tstate, _PyInterpreterFrame *frame, _Py_ tstate->interp->jit_state.initial_chain_depth = chain_depth % MAX_CHAIN_DEPTH; tstate->interp->jit_state.prev_instr_frame = frame; tstate->interp->jit_state.dependencies_still_valid = true; - tstate->interp->jit_state.do_not_specialize = false; + tstate->interp->jit_state.specialize_counter = 0; tstate->interp->jit_state.prev_instr_code = (PyCodeObject *)Py_NewRef(_PyFrame_GetCode(frame)); tstate->interp->jit_state.prev_instr = curr_instr; tstate->interp->jit_state.prev_instr_frame = frame; tstate->interp->jit_state.prev_instr_oparg = oparg; tstate->interp->jit_state.prev_instr_stacklevel = curr_stackdepth; tstate->interp->jit_state.dynamic_jump_taken = false; + tstate->interp->jit_state.prev_instr_is_super = false; _Py_BloomFilter_Init(&tstate->interp->jit_state.dependencies); } @@ -1740,7 +1752,7 @@ executor_to_gv(_PyExecutorObject *executor, FILE *out) #ifdef Py_STATS fprintf(out, " %s -- %" PRIu64 "\n", i, opname, inst->execution_count); #else - fprintf(out, " %s\n", i, opname); + fprintf(out, " %s op0=%" PRIu64 "\n", i, opname, inst->operand0); #endif if (inst->opcode == _EXIT_TRACE || inst->opcode == _JUMP_TO_TOP) { break; diff --git a/Tools/cases_generator/generators_common.py b/Tools/cases_generator/generators_common.py index 4a66b7d51180..804bd9773b16 100644 --- a/Tools/cases_generator/generators_common.py +++ b/Tools/cases_generator/generators_common.py @@ -169,6 +169,9 @@ class Emitter: family_name = inst.family.name self.emit(f"UPDATE_MISS_STATS({family_name});\n") self.emit(f"assert(_PyOpcode_Deopt[opcode] == ({family_name}));\n") + self.emit(f"#if _Py_TIER2\n") + self.emit(f"tstate->interp->jit_state.specialize_counter++;\n") + self.emit(f"#endif\n") self.emit(f"JUMP_TO_PREDICTED({self.jump_prefix}{family_name});\n") self.emit("}\n") return not always_true(first_tkn)