#endif
PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable);
(void)old_code;
+ _Py_CODEUNIT* const this_instr = next_instr;
+ (void)this_instr;
frame->instr_ptr = next_instr;
next_instr += 2;
INSTRUCTION_STATS(POP_JUMP_IF_FALSE);
cond = stack_pointer[-1];
assert(PyStackRef_BoolCheck(cond));
int flag = PyStackRef_IsFalse(cond);
- JUMPBY(flag ? oparg : 0);
+ RECORD_BRANCH_TAKEN(this_instr[1].cache, flag);
+ JUMPBY(flag ? oparg : next_instr->op.code == NOT_TAKEN);
stack_pointer += -1;
assert(WITHIN_STACK_BOUNDS());
DISPATCH();
#endif
PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable);
(void)old_code;
+ _Py_CODEUNIT* const this_instr = next_instr;
+ (void)this_instr;
frame->instr_ptr = next_instr;
next_instr += 2;
INSTRUCTION_STATS(POP_JUMP_IF_NONE);
cond = b;
assert(PyStackRef_BoolCheck(cond));
int flag = PyStackRef_IsTrue(cond);
- JUMPBY(flag ? oparg : 0);
+ RECORD_BRANCH_TAKEN(this_instr[1].cache, flag);
+ JUMPBY(flag ? oparg : next_instr->op.code == NOT_TAKEN);
}
stack_pointer += -1;
assert(WITHIN_STACK_BOUNDS());
#endif
PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable);
(void)old_code;
+ _Py_CODEUNIT* const this_instr = next_instr;
+ (void)this_instr;
frame->instr_ptr = next_instr;
next_instr += 2;
INSTRUCTION_STATS(POP_JUMP_IF_NOT_NONE);
cond = b;
assert(PyStackRef_BoolCheck(cond));
int flag = PyStackRef_IsFalse(cond);
- JUMPBY(flag ? oparg : 0);
+ RECORD_BRANCH_TAKEN(this_instr[1].cache, flag);
+ JUMPBY(flag ? oparg : next_instr->op.code == NOT_TAKEN);
}
stack_pointer += -1;
assert(WITHIN_STACK_BOUNDS());
#endif
PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable);
(void)old_code;
+ _Py_CODEUNIT* const this_instr = next_instr;
+ (void)this_instr;
frame->instr_ptr = next_instr;
next_instr += 2;
INSTRUCTION_STATS(POP_JUMP_IF_TRUE);
cond = stack_pointer[-1];
assert(PyStackRef_BoolCheck(cond));
int flag = PyStackRef_IsTrue(cond);
- JUMPBY(flag ? oparg : 0);
+ RECORD_BRANCH_TAKEN(this_instr[1].cache, flag);
+ JUMPBY(flag ? oparg : next_instr->op.code == NOT_TAKEN);
stack_pointer += -1;
assert(WITHIN_STACK_BOUNDS());
DISPATCH();
TRACING_TARGET(BINARY_OP) {
+ assert(IS_JIT_TRACING());
#if _Py_TAIL_CALL_INTERP
int opcode = BINARY_OP;
(void)(opcode);
}
TRACING_TARGET(BINARY_OP_ADD_FLOAT) {
+ assert(IS_JIT_TRACING());
#if _Py_TAIL_CALL_INTERP
int opcode = BINARY_OP_ADD_FLOAT;
(void)(opcode);
if (!PyFloat_CheckExact(value_o)) {
UPDATE_MISS_STATS(BINARY_OP);
assert(_PyOpcode_Deopt[opcode] == (BINARY_OP));
- JUMP_TO_PREDICTED(BINARY_OP);
+ JUMP_TO_PREDICTED(TRACING_BINARY_OP);
}
}
// _GUARD_NOS_FLOAT
if (!PyFloat_CheckExact(left_o)) {
UPDATE_MISS_STATS(BINARY_OP);
assert(_PyOpcode_Deopt[opcode] == (BINARY_OP));
- JUMP_TO_PREDICTED(BINARY_OP);
+ JUMP_TO_PREDICTED(TRACING_BINARY_OP);
}
}
/* Skip 5 cache entries */
}
TRACING_TARGET(BINARY_OP_ADD_INT) {
+ assert(IS_JIT_TRACING());
#if _Py_TAIL_CALL_INTERP
int opcode = BINARY_OP_ADD_INT;
(void)(opcode);
if (!_PyLong_CheckExactAndCompact(value_o)) {
UPDATE_MISS_STATS(BINARY_OP);
assert(_PyOpcode_Deopt[opcode] == (BINARY_OP));
- JUMP_TO_PREDICTED(BINARY_OP);
+ JUMP_TO_PREDICTED(TRACING_BINARY_OP);
}
}
// _GUARD_NOS_INT
if (!_PyLong_CheckExactAndCompact(left_o)) {
UPDATE_MISS_STATS(BINARY_OP);
assert(_PyOpcode_Deopt[opcode] == (BINARY_OP));
- JUMP_TO_PREDICTED(BINARY_OP);
+ JUMP_TO_PREDICTED(TRACING_BINARY_OP);
}
}
/* Skip 5 cache entries */
if (PyStackRef_IsNull(res)) {
UPDATE_MISS_STATS(BINARY_OP);
assert(_PyOpcode_Deopt[opcode] == (BINARY_OP));
- JUMP_TO_PREDICTED(BINARY_OP);
+ JUMP_TO_PREDICTED(TRACING_BINARY_OP);
}
PyStackRef_CLOSE_SPECIALIZED(right, _PyLong_ExactDealloc);
PyStackRef_CLOSE_SPECIALIZED(left, _PyLong_ExactDealloc);
}
TRACING_TARGET(BINARY_OP_ADD_UNICODE) {
+ assert(IS_JIT_TRACING());
#if _Py_TAIL_CALL_INTERP
int opcode = BINARY_OP_ADD_UNICODE;
(void)(opcode);
if (!PyUnicode_CheckExact(value_o)) {
UPDATE_MISS_STATS(BINARY_OP);
assert(_PyOpcode_Deopt[opcode] == (BINARY_OP));
- JUMP_TO_PREDICTED(BINARY_OP);
+ JUMP_TO_PREDICTED(TRACING_BINARY_OP);
}
}
// _GUARD_NOS_UNICODE
if (!PyUnicode_CheckExact(o)) {
UPDATE_MISS_STATS(BINARY_OP);
assert(_PyOpcode_Deopt[opcode] == (BINARY_OP));
- JUMP_TO_PREDICTED(BINARY_OP);
+ JUMP_TO_PREDICTED(TRACING_BINARY_OP);
}
}
/* Skip 5 cache entries */
}
TRACING_TARGET(BINARY_OP_EXTEND) {
+ assert(IS_JIT_TRACING());
#if _Py_TAIL_CALL_INTERP
int opcode = BINARY_OP_EXTEND;
(void)(opcode);
if (!res) {
UPDATE_MISS_STATS(BINARY_OP);
assert(_PyOpcode_Deopt[opcode] == (BINARY_OP));
- JUMP_TO_PREDICTED(BINARY_OP);
+ JUMP_TO_PREDICTED(TRACING_BINARY_OP);
}
}
/* Skip -4 cache entry */
}
TRACING_TARGET(BINARY_OP_INPLACE_ADD_UNICODE) {
+ assert(IS_JIT_TRACING());
#if _Py_TAIL_CALL_INTERP
int opcode = BINARY_OP_INPLACE_ADD_UNICODE;
(void)(opcode);
if (!PyUnicode_CheckExact(value_o)) {
UPDATE_MISS_STATS(BINARY_OP);
assert(_PyOpcode_Deopt[opcode] == (BINARY_OP));
- JUMP_TO_PREDICTED(BINARY_OP);
+ JUMP_TO_PREDICTED(TRACING_BINARY_OP);
}
}
// _GUARD_NOS_UNICODE
if (!PyUnicode_CheckExact(o)) {
UPDATE_MISS_STATS(BINARY_OP);
assert(_PyOpcode_Deopt[opcode] == (BINARY_OP));
- JUMP_TO_PREDICTED(BINARY_OP);
+ JUMP_TO_PREDICTED(TRACING_BINARY_OP);
}
}
/* Skip 5 cache entries */
if (PyStackRef_AsPyObjectBorrow(*target_local) != left_o) {
UPDATE_MISS_STATS(BINARY_OP);
assert(_PyOpcode_Deopt[opcode] == (BINARY_OP));
- JUMP_TO_PREDICTED(BINARY_OP);
+ JUMP_TO_PREDICTED(TRACING_BINARY_OP);
}
STAT_INC(BINARY_OP, hit);
assert(Py_REFCNT(left_o) >= 2 || !PyStackRef_IsHeapSafe(left));
}
TRACING_TARGET(BINARY_OP_MULTIPLY_FLOAT) {
+ assert(IS_JIT_TRACING());
#if _Py_TAIL_CALL_INTERP
int opcode = BINARY_OP_MULTIPLY_FLOAT;
(void)(opcode);
if (!PyFloat_CheckExact(value_o)) {
UPDATE_MISS_STATS(BINARY_OP);
assert(_PyOpcode_Deopt[opcode] == (BINARY_OP));
- JUMP_TO_PREDICTED(BINARY_OP);
+ JUMP_TO_PREDICTED(TRACING_BINARY_OP);
}
}
// _GUARD_NOS_FLOAT
if (!PyFloat_CheckExact(left_o)) {
UPDATE_MISS_STATS(BINARY_OP);
assert(_PyOpcode_Deopt[opcode] == (BINARY_OP));
- JUMP_TO_PREDICTED(BINARY_OP);
+ JUMP_TO_PREDICTED(TRACING_BINARY_OP);
}
}
/* Skip 5 cache entries */
}
TRACING_TARGET(BINARY_OP_MULTIPLY_INT) {
+ assert(IS_JIT_TRACING());
#if _Py_TAIL_CALL_INTERP
int opcode = BINARY_OP_MULTIPLY_INT;
(void)(opcode);
if (!_PyLong_CheckExactAndCompact(value_o)) {
UPDATE_MISS_STATS(BINARY_OP);
assert(_PyOpcode_Deopt[opcode] == (BINARY_OP));
- JUMP_TO_PREDICTED(BINARY_OP);
+ JUMP_TO_PREDICTED(TRACING_BINARY_OP);
}
}
// _GUARD_NOS_INT
if (!_PyLong_CheckExactAndCompact(left_o)) {
UPDATE_MISS_STATS(BINARY_OP);
assert(_PyOpcode_Deopt[opcode] == (BINARY_OP));
- JUMP_TO_PREDICTED(BINARY_OP);
+ JUMP_TO_PREDICTED(TRACING_BINARY_OP);
}
}
/* Skip 5 cache entries */
if (PyStackRef_IsNull(res)) {
UPDATE_MISS_STATS(BINARY_OP);
assert(_PyOpcode_Deopt[opcode] == (BINARY_OP));
- JUMP_TO_PREDICTED(BINARY_OP);
+ JUMP_TO_PREDICTED(TRACING_BINARY_OP);
}
PyStackRef_CLOSE_SPECIALIZED(right, _PyLong_ExactDealloc);
PyStackRef_CLOSE_SPECIALIZED(left, _PyLong_ExactDealloc);
}
TRACING_TARGET(BINARY_OP_SUBSCR_DICT) {
+ assert(IS_JIT_TRACING());
#if _Py_TAIL_CALL_INTERP
int opcode = BINARY_OP_SUBSCR_DICT;
(void)(opcode);
if (!PyDict_CheckExact(o)) {
UPDATE_MISS_STATS(BINARY_OP);
assert(_PyOpcode_Deopt[opcode] == (BINARY_OP));
- JUMP_TO_PREDICTED(BINARY_OP);
+ JUMP_TO_PREDICTED(TRACING_BINARY_OP);
}
}
/* Skip 5 cache entries */
}
TRACING_TARGET(BINARY_OP_SUBSCR_GETITEM) {
+ assert(IS_JIT_TRACING());
#if _Py_TAIL_CALL_INTERP
int opcode = BINARY_OP_SUBSCR_GETITEM;
(void)(opcode);
if (tstate->interp->eval_frame) {
UPDATE_MISS_STATS(BINARY_OP);
assert(_PyOpcode_Deopt[opcode] == (BINARY_OP));
- JUMP_TO_PREDICTED(BINARY_OP);
+ JUMP_TO_PREDICTED(TRACING_BINARY_OP);
}
}
// _BINARY_OP_SUBSCR_CHECK_FUNC
if (!PyType_HasFeature(tp, Py_TPFLAGS_HEAPTYPE)) {
UPDATE_MISS_STATS(BINARY_OP);
assert(_PyOpcode_Deopt[opcode] == (BINARY_OP));
- JUMP_TO_PREDICTED(BINARY_OP);
+ JUMP_TO_PREDICTED(TRACING_BINARY_OP);
}
PyHeapTypeObject *ht = (PyHeapTypeObject *)tp;
PyObject *getitem_o = FT_ATOMIC_LOAD_PTR_ACQUIRE(ht->_spec_cache.getitem);
if (getitem_o == NULL) {
UPDATE_MISS_STATS(BINARY_OP);
assert(_PyOpcode_Deopt[opcode] == (BINARY_OP));
- JUMP_TO_PREDICTED(BINARY_OP);
+ JUMP_TO_PREDICTED(TRACING_BINARY_OP);
}
assert(PyFunction_Check(getitem_o));
uint32_t cached_version = FT_ATOMIC_LOAD_UINT32_RELAXED(ht->_spec_cache.getitem_version);
if (((PyFunctionObject *)getitem_o)->func_version != cached_version) {
UPDATE_MISS_STATS(BINARY_OP);
assert(_PyOpcode_Deopt[opcode] == (BINARY_OP));
- JUMP_TO_PREDICTED(BINARY_OP);
+ JUMP_TO_PREDICTED(TRACING_BINARY_OP);
}
PyCodeObject *code = (PyCodeObject *)PyFunction_GET_CODE(getitem_o);
assert(code->co_argcount == 2);
if (!_PyThreadState_HasStackSpace(tstate, code->co_framesize)) {
UPDATE_MISS_STATS(BINARY_OP);
assert(_PyOpcode_Deopt[opcode] == (BINARY_OP));
- JUMP_TO_PREDICTED(BINARY_OP);
+ JUMP_TO_PREDICTED(TRACING_BINARY_OP);
}
getitem = PyStackRef_FromPyObjectNew(getitem_o);
STAT_INC(BINARY_OP, hit);
}
TRACING_TARGET(BINARY_OP_SUBSCR_LIST_INT) {
+ assert(IS_JIT_TRACING());
#if _Py_TAIL_CALL_INTERP
int opcode = BINARY_OP_SUBSCR_LIST_INT;
(void)(opcode);
if (!_PyLong_CheckExactAndCompact(value_o)) {
UPDATE_MISS_STATS(BINARY_OP);
assert(_PyOpcode_Deopt[opcode] == (BINARY_OP));
- JUMP_TO_PREDICTED(BINARY_OP);
+ JUMP_TO_PREDICTED(TRACING_BINARY_OP);
}
}
// _GUARD_NOS_LIST
if (!PyList_CheckExact(o)) {
UPDATE_MISS_STATS(BINARY_OP);
assert(_PyOpcode_Deopt[opcode] == (BINARY_OP));
- JUMP_TO_PREDICTED(BINARY_OP);
+ JUMP_TO_PREDICTED(TRACING_BINARY_OP);
}
}
/* Skip 5 cache entries */
if (!_PyLong_IsNonNegativeCompact((PyLongObject *)sub)) {
UPDATE_MISS_STATS(BINARY_OP);
assert(_PyOpcode_Deopt[opcode] == (BINARY_OP));
- JUMP_TO_PREDICTED(BINARY_OP);
+ JUMP_TO_PREDICTED(TRACING_BINARY_OP);
}
Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0];
#ifdef Py_GIL_DISABLED
if (res_o == NULL) {
UPDATE_MISS_STATS(BINARY_OP);
assert(_PyOpcode_Deopt[opcode] == (BINARY_OP));
- JUMP_TO_PREDICTED(BINARY_OP);
+ JUMP_TO_PREDICTED(TRACING_BINARY_OP);
}
STAT_INC(BINARY_OP, hit);
res = PyStackRef_FromPyObjectSteal(res_o);
if (index >= PyList_GET_SIZE(list)) {
UPDATE_MISS_STATS(BINARY_OP);
assert(_PyOpcode_Deopt[opcode] == (BINARY_OP));
- JUMP_TO_PREDICTED(BINARY_OP);
+ JUMP_TO_PREDICTED(TRACING_BINARY_OP);
}
STAT_INC(BINARY_OP, hit);
PyObject *res_o = PyList_GET_ITEM(list, index);
}
TRACING_TARGET(BINARY_OP_SUBSCR_LIST_SLICE) {
+ assert(IS_JIT_TRACING());
#if _Py_TAIL_CALL_INTERP
int opcode = BINARY_OP_SUBSCR_LIST_SLICE;
(void)(opcode);
if (!PySlice_Check(o)) {
UPDATE_MISS_STATS(BINARY_OP);
assert(_PyOpcode_Deopt[opcode] == (BINARY_OP));
- JUMP_TO_PREDICTED(BINARY_OP);
+ JUMP_TO_PREDICTED(TRACING_BINARY_OP);
}
}
// _GUARD_NOS_LIST
if (!PyList_CheckExact(o)) {
UPDATE_MISS_STATS(BINARY_OP);
assert(_PyOpcode_Deopt[opcode] == (BINARY_OP));
- JUMP_TO_PREDICTED(BINARY_OP);
+ JUMP_TO_PREDICTED(TRACING_BINARY_OP);
}
}
/* Skip 5 cache entries */
}
TRACING_TARGET(BINARY_OP_SUBSCR_STR_INT) {
+ assert(IS_JIT_TRACING());
#if _Py_TAIL_CALL_INTERP
int opcode = BINARY_OP_SUBSCR_STR_INT;
(void)(opcode);
if (!_PyLong_CheckExactAndCompact(value_o)) {
UPDATE_MISS_STATS(BINARY_OP);
assert(_PyOpcode_Deopt[opcode] == (BINARY_OP));
- JUMP_TO_PREDICTED(BINARY_OP);
+ JUMP_TO_PREDICTED(TRACING_BINARY_OP);
}
}
// _GUARD_NOS_UNICODE
if (!PyUnicode_CheckExact(o)) {
UPDATE_MISS_STATS(BINARY_OP);
assert(_PyOpcode_Deopt[opcode] == (BINARY_OP));
- JUMP_TO_PREDICTED(BINARY_OP);
+ JUMP_TO_PREDICTED(TRACING_BINARY_OP);
}
}
/* Skip 5 cache entries */
if (!_PyLong_IsNonNegativeCompact((PyLongObject *)sub)) {
UPDATE_MISS_STATS(BINARY_OP);
assert(_PyOpcode_Deopt[opcode] == (BINARY_OP));
- JUMP_TO_PREDICTED(BINARY_OP);
+ JUMP_TO_PREDICTED(TRACING_BINARY_OP);
}
Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0];
if (PyUnicode_GET_LENGTH(str) <= index) {
UPDATE_MISS_STATS(BINARY_OP);
assert(_PyOpcode_Deopt[opcode] == (BINARY_OP));
- JUMP_TO_PREDICTED(BINARY_OP);
+ JUMP_TO_PREDICTED(TRACING_BINARY_OP);
}
Py_UCS4 c = PyUnicode_READ_CHAR(str, index);
if (Py_ARRAY_LENGTH(_Py_SINGLETON(strings).ascii) <= c) {
UPDATE_MISS_STATS(BINARY_OP);
assert(_PyOpcode_Deopt[opcode] == (BINARY_OP));
- JUMP_TO_PREDICTED(BINARY_OP);
+ JUMP_TO_PREDICTED(TRACING_BINARY_OP);
}
STAT_INC(BINARY_OP, hit);
PyObject *res_o = (PyObject*)&_Py_SINGLETON(strings).ascii[c];
}
TRACING_TARGET(BINARY_OP_SUBSCR_TUPLE_INT) {
+ assert(IS_JIT_TRACING());
#if _Py_TAIL_CALL_INTERP
int opcode = BINARY_OP_SUBSCR_TUPLE_INT;
(void)(opcode);
if (!_PyLong_CheckExactAndCompact(value_o)) {
UPDATE_MISS_STATS(BINARY_OP);
assert(_PyOpcode_Deopt[opcode] == (BINARY_OP));
- JUMP_TO_PREDICTED(BINARY_OP);
+ JUMP_TO_PREDICTED(TRACING_BINARY_OP);
}
}
// _GUARD_NOS_TUPLE
if (!PyTuple_CheckExact(o)) {
UPDATE_MISS_STATS(BINARY_OP);
assert(_PyOpcode_Deopt[opcode] == (BINARY_OP));
- JUMP_TO_PREDICTED(BINARY_OP);
+ JUMP_TO_PREDICTED(TRACING_BINARY_OP);
}
}
/* Skip 5 cache entries */
if (!_PyLong_IsNonNegativeCompact((PyLongObject *)sub)) {
UPDATE_MISS_STATS(BINARY_OP);
assert(_PyOpcode_Deopt[opcode] == (BINARY_OP));
- JUMP_TO_PREDICTED(BINARY_OP);
+ JUMP_TO_PREDICTED(TRACING_BINARY_OP);
}
Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0];
if (index >= PyTuple_GET_SIZE(tuple)) {
UPDATE_MISS_STATS(BINARY_OP);
assert(_PyOpcode_Deopt[opcode] == (BINARY_OP));
- JUMP_TO_PREDICTED(BINARY_OP);
+ JUMP_TO_PREDICTED(TRACING_BINARY_OP);
}
STAT_INC(BINARY_OP, hit);
PyObject *res_o = PyTuple_GET_ITEM(tuple, index);
}
TRACING_TARGET(BINARY_OP_SUBTRACT_FLOAT) {
+ assert(IS_JIT_TRACING());
#if _Py_TAIL_CALL_INTERP
int opcode = BINARY_OP_SUBTRACT_FLOAT;
(void)(opcode);
if (!PyFloat_CheckExact(value_o)) {
UPDATE_MISS_STATS(BINARY_OP);
assert(_PyOpcode_Deopt[opcode] == (BINARY_OP));
- JUMP_TO_PREDICTED(BINARY_OP);
+ JUMP_TO_PREDICTED(TRACING_BINARY_OP);
}
}
// _GUARD_NOS_FLOAT
if (!PyFloat_CheckExact(left_o)) {
UPDATE_MISS_STATS(BINARY_OP);
assert(_PyOpcode_Deopt[opcode] == (BINARY_OP));
- JUMP_TO_PREDICTED(BINARY_OP);
+ JUMP_TO_PREDICTED(TRACING_BINARY_OP);
}
}
/* Skip 5 cache entries */
}
TRACING_TARGET(BINARY_OP_SUBTRACT_INT) {
+ assert(IS_JIT_TRACING());
#if _Py_TAIL_CALL_INTERP
int opcode = BINARY_OP_SUBTRACT_INT;
(void)(opcode);
if (!_PyLong_CheckExactAndCompact(value_o)) {
UPDATE_MISS_STATS(BINARY_OP);
assert(_PyOpcode_Deopt[opcode] == (BINARY_OP));
- JUMP_TO_PREDICTED(BINARY_OP);
+ JUMP_TO_PREDICTED(TRACING_BINARY_OP);
}
}
// _GUARD_NOS_INT
if (!_PyLong_CheckExactAndCompact(left_o)) {
UPDATE_MISS_STATS(BINARY_OP);
assert(_PyOpcode_Deopt[opcode] == (BINARY_OP));
- JUMP_TO_PREDICTED(BINARY_OP);
+ JUMP_TO_PREDICTED(TRACING_BINARY_OP);
}
}
/* Skip 5 cache entries */
if (PyStackRef_IsNull(res)) {
UPDATE_MISS_STATS(BINARY_OP);
assert(_PyOpcode_Deopt[opcode] == (BINARY_OP));
- JUMP_TO_PREDICTED(BINARY_OP);
+ JUMP_TO_PREDICTED(TRACING_BINARY_OP);
}
PyStackRef_CLOSE_SPECIALIZED(right, _PyLong_ExactDealloc);
PyStackRef_CLOSE_SPECIALIZED(left, _PyLong_ExactDealloc);
}
TRACING_TARGET(BINARY_SLICE) {
+ assert(IS_JIT_TRACING());
#if _Py_TAIL_CALL_INTERP
int opcode = BINARY_SLICE;
(void)(opcode);
}
TRACING_TARGET(BUILD_INTERPOLATION) {
+ assert(IS_JIT_TRACING());
#if _Py_TAIL_CALL_INTERP
int opcode = BUILD_INTERPOLATION;
(void)(opcode);
}
TRACING_TARGET(BUILD_LIST) {
+ assert(IS_JIT_TRACING());
#if _Py_TAIL_CALL_INTERP
int opcode = BUILD_LIST;
(void)(opcode);
}
TRACING_TARGET(BUILD_MAP) {
+ assert(IS_JIT_TRACING());
#if _Py_TAIL_CALL_INTERP
int opcode = BUILD_MAP;
(void)(opcode);
}
TRACING_TARGET(BUILD_SET) {
+ assert(IS_JIT_TRACING());
#if _Py_TAIL_CALL_INTERP
int opcode = BUILD_SET;
(void)(opcode);
}
TRACING_TARGET(BUILD_SLICE) {
+ assert(IS_JIT_TRACING());
#if _Py_TAIL_CALL_INTERP
int opcode = BUILD_SLICE;
(void)(opcode);
}
TRACING_TARGET(BUILD_STRING) {
+ assert(IS_JIT_TRACING());
#if _Py_TAIL_CALL_INTERP
int opcode = BUILD_STRING;
(void)(opcode);
}
TRACING_TARGET(BUILD_TEMPLATE) {
+ assert(IS_JIT_TRACING());
#if _Py_TAIL_CALL_INTERP
int opcode = BUILD_TEMPLATE;
(void)(opcode);
}
TRACING_TARGET(BUILD_TUPLE) {
+ assert(IS_JIT_TRACING());
#if _Py_TAIL_CALL_INTERP
int opcode = BUILD_TUPLE;
(void)(opcode);
}
TRACING_TARGET(CACHE) {
+ assert(IS_JIT_TRACING());
#if _Py_TAIL_CALL_INTERP
int opcode = CACHE;
(void)(opcode);
}
TRACING_TARGET(CALL) {
+ assert(IS_JIT_TRACING());
#if _Py_TAIL_CALL_INTERP
int opcode = CALL;
(void)(opcode);
}
TRACING_TARGET(CALL_ALLOC_AND_ENTER_INIT) {
+ assert(IS_JIT_TRACING());
#if _Py_TAIL_CALL_INTERP
int opcode = CALL_ALLOC_AND_ENTER_INIT;
(void)(opcode);
if (tstate->interp->eval_frame) {
UPDATE_MISS_STATS(CALL);
assert(_PyOpcode_Deopt[opcode] == (CALL));
- JUMP_TO_PREDICTED(CALL);
+ JUMP_TO_PREDICTED(TRACING_CALL);
}
}
// _CHECK_AND_ALLOCATE_OBJECT
if (!PyStackRef_IsNull(self_or_null)) {
UPDATE_MISS_STATS(CALL);
assert(_PyOpcode_Deopt[opcode] == (CALL));
- JUMP_TO_PREDICTED(CALL);
+ JUMP_TO_PREDICTED(TRACING_CALL);
}
if (!PyType_Check(callable_o)) {
UPDATE_MISS_STATS(CALL);
assert(_PyOpcode_Deopt[opcode] == (CALL));
- JUMP_TO_PREDICTED(CALL);
+ JUMP_TO_PREDICTED(TRACING_CALL);
}
PyTypeObject *tp = (PyTypeObject *)callable_o;
if (FT_ATOMIC_LOAD_UINT32_RELAXED(tp->tp_version_tag) != type_version) {
UPDATE_MISS_STATS(CALL);
assert(_PyOpcode_Deopt[opcode] == (CALL));
- JUMP_TO_PREDICTED(CALL);
+ JUMP_TO_PREDICTED(TRACING_CALL);
}
assert(tp->tp_new == PyBaseObject_Type.tp_new);
assert(tp->tp_flags & Py_TPFLAGS_HEAPTYPE);
if (!_PyThreadState_HasStackSpace(tstate, code->co_framesize + _Py_InitCleanup.co_framesize)) {
UPDATE_MISS_STATS(CALL);
assert(_PyOpcode_Deopt[opcode] == (CALL));
- JUMP_TO_PREDICTED(CALL);
+ JUMP_TO_PREDICTED(TRACING_CALL);
}
STAT_INC(CALL, hit);
_PyFrame_SetStackPointer(frame, stack_pointer);
}
TRACING_TARGET(CALL_BOUND_METHOD_EXACT_ARGS) {
+ assert(IS_JIT_TRACING());
#if _Py_TAIL_CALL_INTERP
int opcode = CALL_BOUND_METHOD_EXACT_ARGS;
(void)(opcode);
if (tstate->interp->eval_frame) {
UPDATE_MISS_STATS(CALL);
assert(_PyOpcode_Deopt[opcode] == (CALL));
- JUMP_TO_PREDICTED(CALL);
+ JUMP_TO_PREDICTED(TRACING_CALL);
}
}
// _CHECK_CALL_BOUND_METHOD_EXACT_ARGS
if (!PyStackRef_IsNull(null)) {
UPDATE_MISS_STATS(CALL);
assert(_PyOpcode_Deopt[opcode] == (CALL));
- JUMP_TO_PREDICTED(CALL);
+ JUMP_TO_PREDICTED(TRACING_CALL);
}
if (Py_TYPE(PyStackRef_AsPyObjectBorrow(callable)) != &PyMethod_Type) {
UPDATE_MISS_STATS(CALL);
assert(_PyOpcode_Deopt[opcode] == (CALL));
- JUMP_TO_PREDICTED(CALL);
+ JUMP_TO_PREDICTED(TRACING_CALL);
}
}
// _INIT_CALL_BOUND_METHOD_EXACT_ARGS
if (!PyFunction_Check(callable_o)) {
UPDATE_MISS_STATS(CALL);
assert(_PyOpcode_Deopt[opcode] == (CALL));
- JUMP_TO_PREDICTED(CALL);
+ JUMP_TO_PREDICTED(TRACING_CALL);
}
PyFunctionObject *func = (PyFunctionObject *)callable_o;
if (func->func_version != func_version) {
UPDATE_MISS_STATS(CALL);
assert(_PyOpcode_Deopt[opcode] == (CALL));
- JUMP_TO_PREDICTED(CALL);
+ JUMP_TO_PREDICTED(TRACING_CALL);
}
}
// _CHECK_FUNCTION_EXACT_ARGS
if (code->co_argcount != oparg + (!PyStackRef_IsNull(self_or_null))) {
UPDATE_MISS_STATS(CALL);
assert(_PyOpcode_Deopt[opcode] == (CALL));
- JUMP_TO_PREDICTED(CALL);
+ JUMP_TO_PREDICTED(TRACING_CALL);
}
}
// _CHECK_STACK_SPACE
if (!_PyThreadState_HasStackSpace(tstate, code->co_framesize)) {
UPDATE_MISS_STATS(CALL);
assert(_PyOpcode_Deopt[opcode] == (CALL));
- JUMP_TO_PREDICTED(CALL);
+ JUMP_TO_PREDICTED(TRACING_CALL);
}
}
// _CHECK_RECURSION_REMAINING
if (tstate->py_recursion_remaining <= 1) {
UPDATE_MISS_STATS(CALL);
assert(_PyOpcode_Deopt[opcode] == (CALL));
- JUMP_TO_PREDICTED(CALL);
+ JUMP_TO_PREDICTED(TRACING_CALL);
}
}
// _INIT_CALL_PY_EXACT_ARGS
}
TRACING_TARGET(CALL_BOUND_METHOD_GENERAL) {
+ assert(IS_JIT_TRACING());
#if _Py_TAIL_CALL_INTERP
int opcode = CALL_BOUND_METHOD_GENERAL;
(void)(opcode);
if (tstate->interp->eval_frame) {
UPDATE_MISS_STATS(CALL);
assert(_PyOpcode_Deopt[opcode] == (CALL));
- JUMP_TO_PREDICTED(CALL);
+ JUMP_TO_PREDICTED(TRACING_CALL);
}
}
// _CHECK_METHOD_VERSION
if (Py_TYPE(callable_o) != &PyMethod_Type) {
UPDATE_MISS_STATS(CALL);
assert(_PyOpcode_Deopt[opcode] == (CALL));
- JUMP_TO_PREDICTED(CALL);
+ JUMP_TO_PREDICTED(TRACING_CALL);
}
PyObject *func = ((PyMethodObject *)callable_o)->im_func;
if (!PyFunction_Check(func)) {
UPDATE_MISS_STATS(CALL);
assert(_PyOpcode_Deopt[opcode] == (CALL));
- JUMP_TO_PREDICTED(CALL);
+ JUMP_TO_PREDICTED(TRACING_CALL);
}
if (((PyFunctionObject *)func)->func_version != func_version) {
UPDATE_MISS_STATS(CALL);
assert(_PyOpcode_Deopt[opcode] == (CALL));
- JUMP_TO_PREDICTED(CALL);
+ JUMP_TO_PREDICTED(TRACING_CALL);
}
if (!PyStackRef_IsNull(null)) {
UPDATE_MISS_STATS(CALL);
assert(_PyOpcode_Deopt[opcode] == (CALL));
- JUMP_TO_PREDICTED(CALL);
+ JUMP_TO_PREDICTED(TRACING_CALL);
}
}
// _EXPAND_METHOD
if (tstate->py_recursion_remaining <= 1) {
UPDATE_MISS_STATS(CALL);
assert(_PyOpcode_Deopt[opcode] == (CALL));
- JUMP_TO_PREDICTED(CALL);
+ JUMP_TO_PREDICTED(TRACING_CALL);
}
}
// _PY_FRAME_GENERAL
}
TRACING_TARGET(CALL_BUILTIN_CLASS) {
+ assert(IS_JIT_TRACING());
#if _Py_TAIL_CALL_INTERP
int opcode = CALL_BUILTIN_CLASS;
(void)(opcode);
if (!PyType_Check(callable_o)) {
UPDATE_MISS_STATS(CALL);
assert(_PyOpcode_Deopt[opcode] == (CALL));
- JUMP_TO_PREDICTED(CALL);
+ JUMP_TO_PREDICTED(TRACING_CALL);
}
PyTypeObject *tp = (PyTypeObject *)callable_o;
int total_args = oparg;
if (tp->tp_vectorcall == NULL) {
UPDATE_MISS_STATS(CALL);
assert(_PyOpcode_Deopt[opcode] == (CALL));
- JUMP_TO_PREDICTED(CALL);
+ JUMP_TO_PREDICTED(TRACING_CALL);
}
STAT_INC(CALL, hit);
STACKREFS_TO_PYOBJECTS(arguments, total_args, args_o);
}
TRACING_TARGET(CALL_BUILTIN_FAST) {
+ assert(IS_JIT_TRACING());
#if _Py_TAIL_CALL_INTERP
int opcode = CALL_BUILTIN_FAST;
(void)(opcode);
if (!PyCFunction_CheckExact(callable_o)) {
UPDATE_MISS_STATS(CALL);
assert(_PyOpcode_Deopt[opcode] == (CALL));
- JUMP_TO_PREDICTED(CALL);
+ JUMP_TO_PREDICTED(TRACING_CALL);
}
if (PyCFunction_GET_FLAGS(callable_o) != METH_FASTCALL) {
UPDATE_MISS_STATS(CALL);
assert(_PyOpcode_Deopt[opcode] == (CALL));
- JUMP_TO_PREDICTED(CALL);
+ JUMP_TO_PREDICTED(TRACING_CALL);
}
STAT_INC(CALL, hit);
PyCFunction cfunc = PyCFunction_GET_FUNCTION(callable_o);
}
TRACING_TARGET(CALL_BUILTIN_FAST_WITH_KEYWORDS) {
+ assert(IS_JIT_TRACING());
#if _Py_TAIL_CALL_INTERP
int opcode = CALL_BUILTIN_FAST_WITH_KEYWORDS;
(void)(opcode);
if (!PyCFunction_CheckExact(callable_o)) {
UPDATE_MISS_STATS(CALL);
assert(_PyOpcode_Deopt[opcode] == (CALL));
- JUMP_TO_PREDICTED(CALL);
+ JUMP_TO_PREDICTED(TRACING_CALL);
}
if (PyCFunction_GET_FLAGS(callable_o) != (METH_FASTCALL | METH_KEYWORDS)) {
UPDATE_MISS_STATS(CALL);
assert(_PyOpcode_Deopt[opcode] == (CALL));
- JUMP_TO_PREDICTED(CALL);
+ JUMP_TO_PREDICTED(TRACING_CALL);
}
STAT_INC(CALL, hit);
_PyFrame_SetStackPointer(frame, stack_pointer);
}
TRACING_TARGET(CALL_BUILTIN_O) {
+ assert(IS_JIT_TRACING());
#if _Py_TAIL_CALL_INTERP
int opcode = CALL_BUILTIN_O;
(void)(opcode);
if (total_args != 1) {
UPDATE_MISS_STATS(CALL);
assert(_PyOpcode_Deopt[opcode] == (CALL));
- JUMP_TO_PREDICTED(CALL);
+ JUMP_TO_PREDICTED(TRACING_CALL);
}
if (!PyCFunction_CheckExact(callable_o)) {
UPDATE_MISS_STATS(CALL);
assert(_PyOpcode_Deopt[opcode] == (CALL));
- JUMP_TO_PREDICTED(CALL);
+ JUMP_TO_PREDICTED(TRACING_CALL);
}
if (PyCFunction_GET_FLAGS(callable_o) != METH_O) {
UPDATE_MISS_STATS(CALL);
assert(_PyOpcode_Deopt[opcode] == (CALL));
- JUMP_TO_PREDICTED(CALL);
+ JUMP_TO_PREDICTED(TRACING_CALL);
}
if (_Py_ReachedRecursionLimit(tstate)) {
UPDATE_MISS_STATS(CALL);
assert(_PyOpcode_Deopt[opcode] == (CALL));
- JUMP_TO_PREDICTED(CALL);
+ JUMP_TO_PREDICTED(TRACING_CALL);
}
STAT_INC(CALL, hit);
PyCFunction cfunc = PyCFunction_GET_FUNCTION(callable_o);
}
TRACING_TARGET(CALL_FUNCTION_EX) {
+ assert(IS_JIT_TRACING());
#if _Py_TAIL_CALL_INTERP
int opcode = CALL_FUNCTION_EX;
(void)(opcode);
}
TRACING_TARGET(CALL_INTRINSIC_1) {
+ assert(IS_JIT_TRACING());
#if _Py_TAIL_CALL_INTERP
int opcode = CALL_INTRINSIC_1;
(void)(opcode);
}
TRACING_TARGET(CALL_INTRINSIC_2) {
+ assert(IS_JIT_TRACING());
#if _Py_TAIL_CALL_INTERP
int opcode = CALL_INTRINSIC_2;
(void)(opcode);
}
TRACING_TARGET(CALL_ISINSTANCE) {
+ assert(IS_JIT_TRACING());
#if _Py_TAIL_CALL_INTERP
int opcode = CALL_ISINSTANCE;
(void)(opcode);
if (!PyStackRef_IsNull(null)) {
UPDATE_MISS_STATS(CALL);
assert(_PyOpcode_Deopt[opcode] == (CALL));
- JUMP_TO_PREDICTED(CALL);
+ JUMP_TO_PREDICTED(TRACING_CALL);
}
}
// _GUARD_CALLABLE_ISINSTANCE
if (callable_o != interp->callable_cache.isinstance) {
UPDATE_MISS_STATS(CALL);
assert(_PyOpcode_Deopt[opcode] == (CALL));
- JUMP_TO_PREDICTED(CALL);
+ JUMP_TO_PREDICTED(TRACING_CALL);
}
}
// _CALL_ISINSTANCE
}
TRACING_TARGET(CALL_KW) {
+ assert(IS_JIT_TRACING());
#if _Py_TAIL_CALL_INTERP
int opcode = CALL_KW;
(void)(opcode);
}
TRACING_TARGET(CALL_KW_BOUND_METHOD) {
+ assert(IS_JIT_TRACING());
#if _Py_TAIL_CALL_INTERP
int opcode = CALL_KW_BOUND_METHOD;
(void)(opcode);
if (tstate->interp->eval_frame) {
UPDATE_MISS_STATS(CALL_KW);
assert(_PyOpcode_Deopt[opcode] == (CALL_KW));
- JUMP_TO_PREDICTED(CALL_KW);
+ JUMP_TO_PREDICTED(TRACING_CALL_KW);
}
}
// _CHECK_METHOD_VERSION_KW
if (Py_TYPE(callable_o) != &PyMethod_Type) {
UPDATE_MISS_STATS(CALL_KW);
assert(_PyOpcode_Deopt[opcode] == (CALL_KW));
- JUMP_TO_PREDICTED(CALL_KW);
+ JUMP_TO_PREDICTED(TRACING_CALL_KW);
}
PyObject *func = ((PyMethodObject *)callable_o)->im_func;
if (!PyFunction_Check(func)) {
UPDATE_MISS_STATS(CALL_KW);
assert(_PyOpcode_Deopt[opcode] == (CALL_KW));
- JUMP_TO_PREDICTED(CALL_KW);
+ JUMP_TO_PREDICTED(TRACING_CALL_KW);
}
if (((PyFunctionObject *)func)->func_version != func_version) {
UPDATE_MISS_STATS(CALL_KW);
assert(_PyOpcode_Deopt[opcode] == (CALL_KW));
- JUMP_TO_PREDICTED(CALL_KW);
+ JUMP_TO_PREDICTED(TRACING_CALL_KW);
}
if (!PyStackRef_IsNull(null)) {
UPDATE_MISS_STATS(CALL_KW);
assert(_PyOpcode_Deopt[opcode] == (CALL_KW));
- JUMP_TO_PREDICTED(CALL_KW);
+ JUMP_TO_PREDICTED(TRACING_CALL_KW);
}
}
// _EXPAND_METHOD_KW
}
TRACING_TARGET(CALL_KW_NON_PY) {
+ assert(IS_JIT_TRACING());
#if _Py_TAIL_CALL_INTERP
int opcode = CALL_KW_NON_PY;
(void)(opcode);
if (PyFunction_Check(callable_o)) {
UPDATE_MISS_STATS(CALL_KW);
assert(_PyOpcode_Deopt[opcode] == (CALL_KW));
- JUMP_TO_PREDICTED(CALL_KW);
+ JUMP_TO_PREDICTED(TRACING_CALL_KW);
}
if (Py_TYPE(callable_o) == &PyMethod_Type) {
UPDATE_MISS_STATS(CALL_KW);
assert(_PyOpcode_Deopt[opcode] == (CALL_KW));
- JUMP_TO_PREDICTED(CALL_KW);
+ JUMP_TO_PREDICTED(TRACING_CALL_KW);
}
}
// _CALL_KW_NON_PY
}
TRACING_TARGET(CALL_KW_PY) {
+ assert(IS_JIT_TRACING());
#if _Py_TAIL_CALL_INTERP
int opcode = CALL_KW_PY;
(void)(opcode);
if (tstate->interp->eval_frame) {
UPDATE_MISS_STATS(CALL_KW);
assert(_PyOpcode_Deopt[opcode] == (CALL_KW));
- JUMP_TO_PREDICTED(CALL_KW);
+ JUMP_TO_PREDICTED(TRACING_CALL_KW);
}
}
// _CHECK_FUNCTION_VERSION_KW
if (!PyFunction_Check(callable_o)) {
UPDATE_MISS_STATS(CALL_KW);
assert(_PyOpcode_Deopt[opcode] == (CALL_KW));
- JUMP_TO_PREDICTED(CALL_KW);
+ JUMP_TO_PREDICTED(TRACING_CALL_KW);
}
PyFunctionObject *func = (PyFunctionObject *)callable_o;
if (func->func_version != func_version) {
UPDATE_MISS_STATS(CALL_KW);
assert(_PyOpcode_Deopt[opcode] == (CALL_KW));
- JUMP_TO_PREDICTED(CALL_KW);
+ JUMP_TO_PREDICTED(TRACING_CALL_KW);
}
}
// _CHECK_RECURSION_REMAINING
if (tstate->py_recursion_remaining <= 1) {
UPDATE_MISS_STATS(CALL_KW);
assert(_PyOpcode_Deopt[opcode] == (CALL_KW));
- JUMP_TO_PREDICTED(CALL_KW);
+ JUMP_TO_PREDICTED(TRACING_CALL_KW);
}
}
// _PY_FRAME_KW
}
TRACING_TARGET(CALL_LEN) {
+ assert(IS_JIT_TRACING());
#if _Py_TAIL_CALL_INTERP
int opcode = CALL_LEN;
(void)(opcode);
if (!PyStackRef_IsNull(null)) {
UPDATE_MISS_STATS(CALL);
assert(_PyOpcode_Deopt[opcode] == (CALL));
- JUMP_TO_PREDICTED(CALL);
+ JUMP_TO_PREDICTED(TRACING_CALL);
}
}
// _GUARD_CALLABLE_LEN
if (callable_o != interp->callable_cache.len) {
UPDATE_MISS_STATS(CALL);
assert(_PyOpcode_Deopt[opcode] == (CALL));
- JUMP_TO_PREDICTED(CALL);
+ JUMP_TO_PREDICTED(TRACING_CALL);
}
}
// _CALL_LEN
}
TRACING_TARGET(CALL_LIST_APPEND) {
+ assert(IS_JIT_TRACING());
#if _Py_TAIL_CALL_INTERP
int opcode = CALL_LIST_APPEND;
(void)(opcode);
if (callable_o != interp->callable_cache.list_append) {
UPDATE_MISS_STATS(CALL);
assert(_PyOpcode_Deopt[opcode] == (CALL));
- JUMP_TO_PREDICTED(CALL);
+ JUMP_TO_PREDICTED(TRACING_CALL);
}
}
// _GUARD_NOS_NOT_NULL
if (o == NULL) {
UPDATE_MISS_STATS(CALL);
assert(_PyOpcode_Deopt[opcode] == (CALL));
- JUMP_TO_PREDICTED(CALL);
+ JUMP_TO_PREDICTED(TRACING_CALL);
}
}
// _GUARD_NOS_LIST
if (!PyList_CheckExact(o)) {
UPDATE_MISS_STATS(CALL);
assert(_PyOpcode_Deopt[opcode] == (CALL));
- JUMP_TO_PREDICTED(CALL);
+ JUMP_TO_PREDICTED(TRACING_CALL);
}
}
// _CALL_LIST_APPEND
if (!PyList_CheckExact(self_o)) {
UPDATE_MISS_STATS(CALL);
assert(_PyOpcode_Deopt[opcode] == (CALL));
- JUMP_TO_PREDICTED(CALL);
+ JUMP_TO_PREDICTED(TRACING_CALL);
}
if (!LOCK_OBJECT(self_o)) {
UPDATE_MISS_STATS(CALL);
assert(_PyOpcode_Deopt[opcode] == (CALL));
- JUMP_TO_PREDICTED(CALL);
+ JUMP_TO_PREDICTED(TRACING_CALL);
}
STAT_INC(CALL, hit);
int err = _PyList_AppendTakeRef((PyListObject *)self_o, PyStackRef_AsPyObjectSteal(arg));
}
TRACING_TARGET(CALL_METHOD_DESCRIPTOR_FAST) {
+ assert(IS_JIT_TRACING());
#if _Py_TAIL_CALL_INTERP
int opcode = CALL_METHOD_DESCRIPTOR_FAST;
(void)(opcode);
if (total_args == 0) {
UPDATE_MISS_STATS(CALL);
assert(_PyOpcode_Deopt[opcode] == (CALL));
- JUMP_TO_PREDICTED(CALL);
+ JUMP_TO_PREDICTED(TRACING_CALL);
}
PyMethodDescrObject *method = (PyMethodDescrObject *)callable_o;
if (!Py_IS_TYPE(method, &PyMethodDescr_Type)) {
UPDATE_MISS_STATS(CALL);
assert(_PyOpcode_Deopt[opcode] == (CALL));
- JUMP_TO_PREDICTED(CALL);
+ JUMP_TO_PREDICTED(TRACING_CALL);
}
PyMethodDef *meth = method->d_method;
if (meth->ml_flags != METH_FASTCALL) {
UPDATE_MISS_STATS(CALL);
assert(_PyOpcode_Deopt[opcode] == (CALL));
- JUMP_TO_PREDICTED(CALL);
+ JUMP_TO_PREDICTED(TRACING_CALL);
}
PyObject *self = PyStackRef_AsPyObjectBorrow(arguments[0]);
assert(self != NULL);
if (!Py_IS_TYPE(self, method->d_common.d_type)) {
UPDATE_MISS_STATS(CALL);
assert(_PyOpcode_Deopt[opcode] == (CALL));
- JUMP_TO_PREDICTED(CALL);
+ JUMP_TO_PREDICTED(TRACING_CALL);
}
STAT_INC(CALL, hit);
int nargs = total_args - 1;
}
TRACING_TARGET(CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS) {
+ assert(IS_JIT_TRACING());
#if _Py_TAIL_CALL_INTERP
int opcode = CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS;
(void)(opcode);
if (total_args == 0) {
UPDATE_MISS_STATS(CALL);
assert(_PyOpcode_Deopt[opcode] == (CALL));
- JUMP_TO_PREDICTED(CALL);
+ JUMP_TO_PREDICTED(TRACING_CALL);
}
PyMethodDescrObject *method = (PyMethodDescrObject *)callable_o;
if (!Py_IS_TYPE(method, &PyMethodDescr_Type)) {
UPDATE_MISS_STATS(CALL);
assert(_PyOpcode_Deopt[opcode] == (CALL));
- JUMP_TO_PREDICTED(CALL);
+ JUMP_TO_PREDICTED(TRACING_CALL);
}
PyMethodDef *meth = method->d_method;
if (meth->ml_flags != (METH_FASTCALL|METH_KEYWORDS)) {
UPDATE_MISS_STATS(CALL);
assert(_PyOpcode_Deopt[opcode] == (CALL));
- JUMP_TO_PREDICTED(CALL);
+ JUMP_TO_PREDICTED(TRACING_CALL);
}
PyTypeObject *d_type = method->d_common.d_type;
PyObject *self = PyStackRef_AsPyObjectBorrow(arguments[0]);
if (!Py_IS_TYPE(self, d_type)) {
UPDATE_MISS_STATS(CALL);
assert(_PyOpcode_Deopt[opcode] == (CALL));
- JUMP_TO_PREDICTED(CALL);
+ JUMP_TO_PREDICTED(TRACING_CALL);
}
STAT_INC(CALL, hit);
int nargs = total_args - 1;
}
TRACING_TARGET(CALL_METHOD_DESCRIPTOR_NOARGS) {
+ assert(IS_JIT_TRACING());
#if _Py_TAIL_CALL_INTERP
int opcode = CALL_METHOD_DESCRIPTOR_NOARGS;
(void)(opcode);
if (total_args != 1) {
UPDATE_MISS_STATS(CALL);
assert(_PyOpcode_Deopt[opcode] == (CALL));
- JUMP_TO_PREDICTED(CALL);
+ JUMP_TO_PREDICTED(TRACING_CALL);
}
PyMethodDescrObject *method = (PyMethodDescrObject *)callable_o;
if (!Py_IS_TYPE(method, &PyMethodDescr_Type)) {
UPDATE_MISS_STATS(CALL);
assert(_PyOpcode_Deopt[opcode] == (CALL));
- JUMP_TO_PREDICTED(CALL);
+ JUMP_TO_PREDICTED(TRACING_CALL);
}
PyMethodDef *meth = method->d_method;
_PyStackRef self_stackref = args[0];
if (!Py_IS_TYPE(self, method->d_common.d_type)) {
UPDATE_MISS_STATS(CALL);
assert(_PyOpcode_Deopt[opcode] == (CALL));
- JUMP_TO_PREDICTED(CALL);
+ JUMP_TO_PREDICTED(TRACING_CALL);
}
if (meth->ml_flags != METH_NOARGS) {
UPDATE_MISS_STATS(CALL);
assert(_PyOpcode_Deopt[opcode] == (CALL));
- JUMP_TO_PREDICTED(CALL);
+ JUMP_TO_PREDICTED(TRACING_CALL);
}
if (_Py_ReachedRecursionLimit(tstate)) {
UPDATE_MISS_STATS(CALL);
assert(_PyOpcode_Deopt[opcode] == (CALL));
- JUMP_TO_PREDICTED(CALL);
+ JUMP_TO_PREDICTED(TRACING_CALL);
}
STAT_INC(CALL, hit);
PyCFunction cfunc = meth->ml_meth;
}
TRACING_TARGET(CALL_METHOD_DESCRIPTOR_O) {
+ assert(IS_JIT_TRACING());
#if _Py_TAIL_CALL_INTERP
int opcode = CALL_METHOD_DESCRIPTOR_O;
(void)(opcode);
if (total_args != 2) {
UPDATE_MISS_STATS(CALL);
assert(_PyOpcode_Deopt[opcode] == (CALL));
- JUMP_TO_PREDICTED(CALL);
+ JUMP_TO_PREDICTED(TRACING_CALL);
}
if (!Py_IS_TYPE(method, &PyMethodDescr_Type)) {
UPDATE_MISS_STATS(CALL);
assert(_PyOpcode_Deopt[opcode] == (CALL));
- JUMP_TO_PREDICTED(CALL);
+ JUMP_TO_PREDICTED(TRACING_CALL);
}
PyMethodDef *meth = method->d_method;
if (meth->ml_flags != METH_O) {
UPDATE_MISS_STATS(CALL);
assert(_PyOpcode_Deopt[opcode] == (CALL));
- JUMP_TO_PREDICTED(CALL);
+ JUMP_TO_PREDICTED(TRACING_CALL);
}
if (_Py_ReachedRecursionLimit(tstate)) {
UPDATE_MISS_STATS(CALL);
assert(_PyOpcode_Deopt[opcode] == (CALL));
- JUMP_TO_PREDICTED(CALL);
+ JUMP_TO_PREDICTED(TRACING_CALL);
}
_PyStackRef arg_stackref = arguments[1];
_PyStackRef self_stackref = arguments[0];
method->d_common.d_type)) {
UPDATE_MISS_STATS(CALL);
assert(_PyOpcode_Deopt[opcode] == (CALL));
- JUMP_TO_PREDICTED(CALL);
+ JUMP_TO_PREDICTED(TRACING_CALL);
}
STAT_INC(CALL, hit);
PyCFunction cfunc = meth->ml_meth;
}
TRACING_TARGET(CALL_NON_PY_GENERAL) {
+ assert(IS_JIT_TRACING());
#if _Py_TAIL_CALL_INTERP
int opcode = CALL_NON_PY_GENERAL;
(void)(opcode);
if (PyFunction_Check(callable_o)) {
UPDATE_MISS_STATS(CALL);
assert(_PyOpcode_Deopt[opcode] == (CALL));
- JUMP_TO_PREDICTED(CALL);
+ JUMP_TO_PREDICTED(TRACING_CALL);
}
if (Py_TYPE(callable_o) == &PyMethod_Type) {
UPDATE_MISS_STATS(CALL);
assert(_PyOpcode_Deopt[opcode] == (CALL));
- JUMP_TO_PREDICTED(CALL);
+ JUMP_TO_PREDICTED(TRACING_CALL);
}
}
// _CALL_NON_PY_GENERAL
}
TRACING_TARGET(CALL_PY_EXACT_ARGS) {
+ assert(IS_JIT_TRACING());
#if _Py_TAIL_CALL_INTERP
int opcode = CALL_PY_EXACT_ARGS;
(void)(opcode);
if (tstate->interp->eval_frame) {
UPDATE_MISS_STATS(CALL);
assert(_PyOpcode_Deopt[opcode] == (CALL));
- JUMP_TO_PREDICTED(CALL);
+ JUMP_TO_PREDICTED(TRACING_CALL);
}
}
// _CHECK_FUNCTION_VERSION
if (!PyFunction_Check(callable_o)) {
UPDATE_MISS_STATS(CALL);
assert(_PyOpcode_Deopt[opcode] == (CALL));
- JUMP_TO_PREDICTED(CALL);
+ JUMP_TO_PREDICTED(TRACING_CALL);
}
PyFunctionObject *func = (PyFunctionObject *)callable_o;
if (func->func_version != func_version) {
UPDATE_MISS_STATS(CALL);
assert(_PyOpcode_Deopt[opcode] == (CALL));
- JUMP_TO_PREDICTED(CALL);
+ JUMP_TO_PREDICTED(TRACING_CALL);
}
}
// _CHECK_FUNCTION_EXACT_ARGS
if (code->co_argcount != oparg + (!PyStackRef_IsNull(self_or_null))) {
UPDATE_MISS_STATS(CALL);
assert(_PyOpcode_Deopt[opcode] == (CALL));
- JUMP_TO_PREDICTED(CALL);
+ JUMP_TO_PREDICTED(TRACING_CALL);
}
}
// _CHECK_STACK_SPACE
if (!_PyThreadState_HasStackSpace(tstate, code->co_framesize)) {
UPDATE_MISS_STATS(CALL);
assert(_PyOpcode_Deopt[opcode] == (CALL));
- JUMP_TO_PREDICTED(CALL);
+ JUMP_TO_PREDICTED(TRACING_CALL);
}
}
// _CHECK_RECURSION_REMAINING
if (tstate->py_recursion_remaining <= 1) {
UPDATE_MISS_STATS(CALL);
assert(_PyOpcode_Deopt[opcode] == (CALL));
- JUMP_TO_PREDICTED(CALL);
+ JUMP_TO_PREDICTED(TRACING_CALL);
}
}
// _INIT_CALL_PY_EXACT_ARGS
}
TRACING_TARGET(CALL_PY_GENERAL) {
+ assert(IS_JIT_TRACING());
#if _Py_TAIL_CALL_INTERP
int opcode = CALL_PY_GENERAL;
(void)(opcode);
if (tstate->interp->eval_frame) {
UPDATE_MISS_STATS(CALL);
assert(_PyOpcode_Deopt[opcode] == (CALL));
- JUMP_TO_PREDICTED(CALL);
+ JUMP_TO_PREDICTED(TRACING_CALL);
}
}
// _CHECK_FUNCTION_VERSION
if (!PyFunction_Check(callable_o)) {
UPDATE_MISS_STATS(CALL);
assert(_PyOpcode_Deopt[opcode] == (CALL));
- JUMP_TO_PREDICTED(CALL);
+ JUMP_TO_PREDICTED(TRACING_CALL);
}
PyFunctionObject *func = (PyFunctionObject *)callable_o;
if (func->func_version != func_version) {
UPDATE_MISS_STATS(CALL);
assert(_PyOpcode_Deopt[opcode] == (CALL));
- JUMP_TO_PREDICTED(CALL);
+ JUMP_TO_PREDICTED(TRACING_CALL);
}
}
// _CHECK_RECURSION_REMAINING
if (tstate->py_recursion_remaining <= 1) {
UPDATE_MISS_STATS(CALL);
assert(_PyOpcode_Deopt[opcode] == (CALL));
- JUMP_TO_PREDICTED(CALL);
+ JUMP_TO_PREDICTED(TRACING_CALL);
}
}
// _PY_FRAME_GENERAL
}
TRACING_TARGET(CALL_STR_1) {
+ assert(IS_JIT_TRACING());
#if _Py_TAIL_CALL_INTERP
int opcode = CALL_STR_1;
(void)(opcode);
if (!PyStackRef_IsNull(null)) {
UPDATE_MISS_STATS(CALL);
assert(_PyOpcode_Deopt[opcode] == (CALL));
- JUMP_TO_PREDICTED(CALL);
+ JUMP_TO_PREDICTED(TRACING_CALL);
}
}
// _GUARD_CALLABLE_STR_1
if (callable_o != (PyObject *)&PyUnicode_Type) {
UPDATE_MISS_STATS(CALL);
assert(_PyOpcode_Deopt[opcode] == (CALL));
- JUMP_TO_PREDICTED(CALL);
+ JUMP_TO_PREDICTED(TRACING_CALL);
}
}
// _CALL_STR_1
}
TRACING_TARGET(CALL_TUPLE_1) {
+ assert(IS_JIT_TRACING());
#if _Py_TAIL_CALL_INTERP
int opcode = CALL_TUPLE_1;
(void)(opcode);
if (!PyStackRef_IsNull(null)) {
UPDATE_MISS_STATS(CALL);
assert(_PyOpcode_Deopt[opcode] == (CALL));
- JUMP_TO_PREDICTED(CALL);
+ JUMP_TO_PREDICTED(TRACING_CALL);
}
}
// _GUARD_CALLABLE_TUPLE_1
if (callable_o != (PyObject *)&PyTuple_Type) {
UPDATE_MISS_STATS(CALL);
assert(_PyOpcode_Deopt[opcode] == (CALL));
- JUMP_TO_PREDICTED(CALL);
+ JUMP_TO_PREDICTED(TRACING_CALL);
}
}
// _CALL_TUPLE_1
}
TRACING_TARGET(CALL_TYPE_1) {
+ assert(IS_JIT_TRACING());
#if _Py_TAIL_CALL_INTERP
int opcode = CALL_TYPE_1;
(void)(opcode);
if (!PyStackRef_IsNull(null)) {
UPDATE_MISS_STATS(CALL);
assert(_PyOpcode_Deopt[opcode] == (CALL));
- JUMP_TO_PREDICTED(CALL);
+ JUMP_TO_PREDICTED(TRACING_CALL);
}
}
// _GUARD_CALLABLE_TYPE_1
if (callable_o != (PyObject *)&PyType_Type) {
UPDATE_MISS_STATS(CALL);
assert(_PyOpcode_Deopt[opcode] == (CALL));
- JUMP_TO_PREDICTED(CALL);
+ JUMP_TO_PREDICTED(TRACING_CALL);
}
}
// _CALL_TYPE_1
}
TRACING_TARGET(CHECK_EG_MATCH) {
+ assert(IS_JIT_TRACING());
#if _Py_TAIL_CALL_INTERP
int opcode = CHECK_EG_MATCH;
(void)(opcode);
}
TRACING_TARGET(CHECK_EXC_MATCH) {
+ assert(IS_JIT_TRACING());
#if _Py_TAIL_CALL_INTERP
int opcode = CHECK_EXC_MATCH;
(void)(opcode);
}
TRACING_TARGET(CLEANUP_THROW) {
+ assert(IS_JIT_TRACING());
#if _Py_TAIL_CALL_INTERP
int opcode = CLEANUP_THROW;
(void)(opcode);
}
TRACING_TARGET(COMPARE_OP) {
+ assert(IS_JIT_TRACING());
#if _Py_TAIL_CALL_INTERP
int opcode = COMPARE_OP;
(void)(opcode);
}
TRACING_TARGET(COMPARE_OP_FLOAT) {
+ assert(IS_JIT_TRACING());
#if _Py_TAIL_CALL_INTERP
int opcode = COMPARE_OP_FLOAT;
(void)(opcode);
if (!PyFloat_CheckExact(value_o)) {
UPDATE_MISS_STATS(COMPARE_OP);
assert(_PyOpcode_Deopt[opcode] == (COMPARE_OP));
- JUMP_TO_PREDICTED(COMPARE_OP);
+ JUMP_TO_PREDICTED(TRACING_COMPARE_OP);
}
}
// _GUARD_NOS_FLOAT
if (!PyFloat_CheckExact(left_o)) {
UPDATE_MISS_STATS(COMPARE_OP);
assert(_PyOpcode_Deopt[opcode] == (COMPARE_OP));
- JUMP_TO_PREDICTED(COMPARE_OP);
+ JUMP_TO_PREDICTED(TRACING_COMPARE_OP);
}
}
/* Skip 1 cache entry */
}
TRACING_TARGET(COMPARE_OP_INT) {
+ assert(IS_JIT_TRACING());
#if _Py_TAIL_CALL_INTERP
int opcode = COMPARE_OP_INT;
(void)(opcode);
if (!_PyLong_CheckExactAndCompact(value_o)) {
UPDATE_MISS_STATS(COMPARE_OP);
assert(_PyOpcode_Deopt[opcode] == (COMPARE_OP));
- JUMP_TO_PREDICTED(COMPARE_OP);
+ JUMP_TO_PREDICTED(TRACING_COMPARE_OP);
}
}
// _GUARD_NOS_INT
if (!_PyLong_CheckExactAndCompact(left_o)) {
UPDATE_MISS_STATS(COMPARE_OP);
assert(_PyOpcode_Deopt[opcode] == (COMPARE_OP));
- JUMP_TO_PREDICTED(COMPARE_OP);
+ JUMP_TO_PREDICTED(TRACING_COMPARE_OP);
}
}
/* Skip 1 cache entry */
}
TRACING_TARGET(COMPARE_OP_STR) {
+ assert(IS_JIT_TRACING());
#if _Py_TAIL_CALL_INTERP
int opcode = COMPARE_OP_STR;
(void)(opcode);
if (!PyUnicode_CheckExact(value_o)) {
UPDATE_MISS_STATS(COMPARE_OP);
assert(_PyOpcode_Deopt[opcode] == (COMPARE_OP));
- JUMP_TO_PREDICTED(COMPARE_OP);
+ JUMP_TO_PREDICTED(TRACING_COMPARE_OP);
}
}
// _GUARD_NOS_UNICODE
if (!PyUnicode_CheckExact(o)) {
UPDATE_MISS_STATS(COMPARE_OP);
assert(_PyOpcode_Deopt[opcode] == (COMPARE_OP));
- JUMP_TO_PREDICTED(COMPARE_OP);
+ JUMP_TO_PREDICTED(TRACING_COMPARE_OP);
}
}
/* Skip 1 cache entry */
}
TRACING_TARGET(CONTAINS_OP) {
+ assert(IS_JIT_TRACING());
#if _Py_TAIL_CALL_INTERP
int opcode = CONTAINS_OP;
(void)(opcode);
}
TRACING_TARGET(CONTAINS_OP_DICT) {
+ assert(IS_JIT_TRACING());
#if _Py_TAIL_CALL_INTERP
int opcode = CONTAINS_OP_DICT;
(void)(opcode);
if (!PyDict_CheckExact(o)) {
UPDATE_MISS_STATS(CONTAINS_OP);
assert(_PyOpcode_Deopt[opcode] == (CONTAINS_OP));
- JUMP_TO_PREDICTED(CONTAINS_OP);
+ JUMP_TO_PREDICTED(TRACING_CONTAINS_OP);
}
}
/* Skip 1 cache entry */
}
TRACING_TARGET(CONTAINS_OP_SET) {
+ assert(IS_JIT_TRACING());
#if _Py_TAIL_CALL_INTERP
int opcode = CONTAINS_OP_SET;
(void)(opcode);
if (!PyAnySet_CheckExact(o)) {
UPDATE_MISS_STATS(CONTAINS_OP);
assert(_PyOpcode_Deopt[opcode] == (CONTAINS_OP));
- JUMP_TO_PREDICTED(CONTAINS_OP);
+ JUMP_TO_PREDICTED(TRACING_CONTAINS_OP);
}
}
/* Skip 1 cache entry */
}
TRACING_TARGET(CONVERT_VALUE) {
+ assert(IS_JIT_TRACING());
#if _Py_TAIL_CALL_INTERP
int opcode = CONVERT_VALUE;
(void)(opcode);
}
TRACING_TARGET(COPY) {
+ assert(IS_JIT_TRACING());
#if _Py_TAIL_CALL_INTERP
int opcode = COPY;
(void)(opcode);
}
TRACING_TARGET(COPY_FREE_VARS) {
+ assert(IS_JIT_TRACING());
#if _Py_TAIL_CALL_INTERP
int opcode = COPY_FREE_VARS;
(void)(opcode);
}
TRACING_TARGET(DELETE_ATTR) {
+ assert(IS_JIT_TRACING());
#if _Py_TAIL_CALL_INTERP
int opcode = DELETE_ATTR;
(void)(opcode);
}
TRACING_TARGET(DELETE_DEREF) {
+ assert(IS_JIT_TRACING());
#if _Py_TAIL_CALL_INTERP
int opcode = DELETE_DEREF;
(void)(opcode);
}
TRACING_TARGET(DELETE_FAST) {
+ assert(IS_JIT_TRACING());
#if _Py_TAIL_CALL_INTERP
int opcode = DELETE_FAST;
(void)(opcode);
}
TRACING_TARGET(DELETE_GLOBAL) {
+ assert(IS_JIT_TRACING());
#if _Py_TAIL_CALL_INTERP
int opcode = DELETE_GLOBAL;
(void)(opcode);
}
TRACING_TARGET(DELETE_NAME) {
+ assert(IS_JIT_TRACING());
#if _Py_TAIL_CALL_INTERP
int opcode = DELETE_NAME;
(void)(opcode);
}
TRACING_TARGET(DELETE_SUBSCR) {
+ assert(IS_JIT_TRACING());
#if _Py_TAIL_CALL_INTERP
int opcode = DELETE_SUBSCR;
(void)(opcode);
}
TRACING_TARGET(DICT_MERGE) {
+ assert(IS_JIT_TRACING());
#if _Py_TAIL_CALL_INTERP
int opcode = DICT_MERGE;
(void)(opcode);
}
TRACING_TARGET(DICT_UPDATE) {
+ assert(IS_JIT_TRACING());
#if _Py_TAIL_CALL_INTERP
int opcode = DICT_UPDATE;
(void)(opcode);
}
TRACING_TARGET(END_ASYNC_FOR) {
+ assert(IS_JIT_TRACING());
#if _Py_TAIL_CALL_INTERP
int opcode = END_ASYNC_FOR;
(void)(opcode);
}
TRACING_TARGET(END_FOR) {
+ assert(IS_JIT_TRACING());
#if _Py_TAIL_CALL_INTERP
int opcode = END_FOR;
(void)(opcode);
}
TRACING_TARGET(END_SEND) {
+ assert(IS_JIT_TRACING());
#if _Py_TAIL_CALL_INTERP
int opcode = END_SEND;
(void)(opcode);
}
TRACING_TARGET(ENTER_EXECUTOR) {
+ assert(IS_JIT_TRACING());
#if _Py_TAIL_CALL_INTERP
int opcode = ENTER_EXECUTOR;
(void)(opcode);
}
TRACING_TARGET(EXIT_INIT_CHECK) {
+ assert(IS_JIT_TRACING());
#if _Py_TAIL_CALL_INTERP
int opcode = EXIT_INIT_CHECK;
(void)(opcode);
}
TRACING_TARGET(EXTENDED_ARG) {
+ assert(IS_JIT_TRACING());
#if _Py_TAIL_CALL_INTERP
int opcode = EXTENDED_ARG;
(void)(opcode);
}
TRACING_TARGET(FORMAT_SIMPLE) {
+ assert(IS_JIT_TRACING());
#if _Py_TAIL_CALL_INTERP
int opcode = FORMAT_SIMPLE;
(void)(opcode);
}
TRACING_TARGET(FORMAT_WITH_SPEC) {
+ assert(IS_JIT_TRACING());
#if _Py_TAIL_CALL_INTERP
int opcode = FORMAT_WITH_SPEC;
(void)(opcode);
}
TRACING_TARGET(FOR_ITER) {
+ assert(IS_JIT_TRACING());
#if _Py_TAIL_CALL_INTERP
int opcode = FOR_ITER;
(void)(opcode);
}
TRACING_TARGET(FOR_ITER_GEN) {
+ assert(IS_JIT_TRACING());
#if _Py_TAIL_CALL_INTERP
int opcode = FOR_ITER_GEN;
(void)(opcode);
if (tstate->interp->eval_frame) {
UPDATE_MISS_STATS(FOR_ITER);
assert(_PyOpcode_Deopt[opcode] == (FOR_ITER));
- JUMP_TO_PREDICTED(FOR_ITER);
+ JUMP_TO_PREDICTED(TRACING_FOR_ITER);
}
}
// _FOR_ITER_GEN_FRAME
if (Py_TYPE(gen) != &PyGen_Type) {
UPDATE_MISS_STATS(FOR_ITER);
assert(_PyOpcode_Deopt[opcode] == (FOR_ITER));
- JUMP_TO_PREDICTED(FOR_ITER);
+ JUMP_TO_PREDICTED(TRACING_FOR_ITER);
}
#ifdef Py_GIL_DISABLED
if (!_PyObject_IsUniquelyReferenced((PyObject *)gen)) {
UPDATE_MISS_STATS(FOR_ITER);
assert(_PyOpcode_Deopt[opcode] == (FOR_ITER));
- JUMP_TO_PREDICTED(FOR_ITER);
+ JUMP_TO_PREDICTED(TRACING_FOR_ITER);
}
#endif
if (gen->gi_frame_state >= FRAME_EXECUTING) {
UPDATE_MISS_STATS(FOR_ITER);
assert(_PyOpcode_Deopt[opcode] == (FOR_ITER));
- JUMP_TO_PREDICTED(FOR_ITER);
+ JUMP_TO_PREDICTED(TRACING_FOR_ITER);
}
STAT_INC(FOR_ITER, hit);
_PyInterpreterFrame *pushed_frame = &gen->gi_iframe;
}
TRACING_TARGET(FOR_ITER_LIST) {
+ assert(IS_JIT_TRACING());
#if _Py_TAIL_CALL_INTERP
int opcode = FOR_ITER_LIST;
(void)(opcode);
if (Py_TYPE(iter_o) != &PyList_Type) {
UPDATE_MISS_STATS(FOR_ITER);
assert(_PyOpcode_Deopt[opcode] == (FOR_ITER));
- JUMP_TO_PREDICTED(FOR_ITER);
+ JUMP_TO_PREDICTED(TRACING_FOR_ITER);
}
assert(PyStackRef_IsTaggedInt(null_or_index));
#ifdef Py_GIL_DISABLED
if (!_Py_IsOwnedByCurrentThread(iter_o) && !_PyObject_GC_IS_SHARED(iter_o)) {
UPDATE_MISS_STATS(FOR_ITER);
assert(_PyOpcode_Deopt[opcode] == (FOR_ITER));
- JUMP_TO_PREDICTED(FOR_ITER);
+ JUMP_TO_PREDICTED(TRACING_FOR_ITER);
}
#endif
}
if (result < 0) {
UPDATE_MISS_STATS(FOR_ITER);
assert(_PyOpcode_Deopt[opcode] == (FOR_ITER));
- JUMP_TO_PREDICTED(FOR_ITER);
+ JUMP_TO_PREDICTED(TRACING_FOR_ITER);
}
if (result == 0) {
null_or_index = PyStackRef_TagInt(-1);
}
TRACING_TARGET(FOR_ITER_RANGE) {
+ assert(IS_JIT_TRACING());
#if _Py_TAIL_CALL_INTERP
int opcode = FOR_ITER_RANGE;
(void)(opcode);
if (Py_TYPE(r) != &PyRangeIter_Type) {
UPDATE_MISS_STATS(FOR_ITER);
assert(_PyOpcode_Deopt[opcode] == (FOR_ITER));
- JUMP_TO_PREDICTED(FOR_ITER);
+ JUMP_TO_PREDICTED(TRACING_FOR_ITER);
}
#ifdef Py_GIL_DISABLED
if (!_PyObject_IsUniquelyReferenced((PyObject *)r)) {
UPDATE_MISS_STATS(FOR_ITER);
assert(_PyOpcode_Deopt[opcode] == (FOR_ITER));
- JUMP_TO_PREDICTED(FOR_ITER);
+ JUMP_TO_PREDICTED(TRACING_FOR_ITER);
}
#endif
}
}
TRACING_TARGET(FOR_ITER_TUPLE) {
+ assert(IS_JIT_TRACING());
#if _Py_TAIL_CALL_INTERP
int opcode = FOR_ITER_TUPLE;
(void)(opcode);
if (Py_TYPE(iter_o) != &PyTuple_Type) {
UPDATE_MISS_STATS(FOR_ITER);
assert(_PyOpcode_Deopt[opcode] == (FOR_ITER));
- JUMP_TO_PREDICTED(FOR_ITER);
+ JUMP_TO_PREDICTED(TRACING_FOR_ITER);
}
assert(PyStackRef_IsTaggedInt(null_or_index));
}
}
TRACING_TARGET(GET_AITER) {
+ assert(IS_JIT_TRACING());
#if _Py_TAIL_CALL_INTERP
int opcode = GET_AITER;
(void)(opcode);
}
TRACING_TARGET(GET_ANEXT) {
+ assert(IS_JIT_TRACING());
#if _Py_TAIL_CALL_INTERP
int opcode = GET_ANEXT;
(void)(opcode);
}
TRACING_TARGET(GET_AWAITABLE) {
+ assert(IS_JIT_TRACING());
#if _Py_TAIL_CALL_INTERP
int opcode = GET_AWAITABLE;
(void)(opcode);
}
TRACING_TARGET(GET_ITER) {
+ assert(IS_JIT_TRACING());
#if _Py_TAIL_CALL_INTERP
int opcode = GET_ITER;
(void)(opcode);
}
TRACING_TARGET(GET_LEN) {
+ assert(IS_JIT_TRACING());
#if _Py_TAIL_CALL_INTERP
int opcode = GET_LEN;
(void)(opcode);
}
TRACING_TARGET(GET_YIELD_FROM_ITER) {
+ assert(IS_JIT_TRACING());
#if _Py_TAIL_CALL_INTERP
int opcode = GET_YIELD_FROM_ITER;
(void)(opcode);
}
TRACING_TARGET(IMPORT_FROM) {
+ assert(IS_JIT_TRACING());
#if _Py_TAIL_CALL_INTERP
int opcode = IMPORT_FROM;
(void)(opcode);
}
TRACING_TARGET(IMPORT_NAME) {
+ assert(IS_JIT_TRACING());
#if _Py_TAIL_CALL_INTERP
int opcode = IMPORT_NAME;
(void)(opcode);
}
TRACING_TARGET(INSTRUMENTED_CALL) {
+ assert(IS_JIT_TRACING());
#if _Py_TAIL_CALL_INTERP
int opcode = INSTRUMENTED_CALL;
(void)(opcode);
}
TRACING_TARGET(INSTRUMENTED_CALL_FUNCTION_EX) {
+ assert(IS_JIT_TRACING());
#if _Py_TAIL_CALL_INTERP
int opcode = INSTRUMENTED_CALL_FUNCTION_EX;
(void)(opcode);
}
TRACING_TARGET(INSTRUMENTED_CALL_KW) {
+ assert(IS_JIT_TRACING());
#if _Py_TAIL_CALL_INTERP
int opcode = INSTRUMENTED_CALL_KW;
(void)(opcode);
}
TRACING_TARGET(INSTRUMENTED_END_ASYNC_FOR) {
+ assert(IS_JIT_TRACING());
#if _Py_TAIL_CALL_INTERP
int opcode = INSTRUMENTED_END_ASYNC_FOR;
(void)(opcode);
}
TRACING_TARGET(INSTRUMENTED_END_FOR) {
+ assert(IS_JIT_TRACING());
#if _Py_TAIL_CALL_INTERP
int opcode = INSTRUMENTED_END_FOR;
(void)(opcode);
}
TRACING_TARGET(INSTRUMENTED_END_SEND) {
+ assert(IS_JIT_TRACING());
#if _Py_TAIL_CALL_INTERP
int opcode = INSTRUMENTED_END_SEND;
(void)(opcode);
}
TRACING_TARGET(INSTRUMENTED_FOR_ITER) {
+ assert(IS_JIT_TRACING());
#if _Py_TAIL_CALL_INTERP
int opcode = INSTRUMENTED_FOR_ITER;
(void)(opcode);
}
TRACING_TARGET(INSTRUMENTED_INSTRUCTION) {
+ assert(IS_JIT_TRACING());
#if _Py_TAIL_CALL_INTERP
int opcode = INSTRUMENTED_INSTRUCTION;
(void)(opcode);
}
TRACING_TARGET(INSTRUMENTED_JUMP_BACKWARD) {
+ assert(IS_JIT_TRACING());
#if _Py_TAIL_CALL_INTERP
int opcode = INSTRUMENTED_JUMP_BACKWARD;
(void)(opcode);
}
TRACING_TARGET(INSTRUMENTED_JUMP_FORWARD) {
+ assert(IS_JIT_TRACING());
#if _Py_TAIL_CALL_INTERP
int opcode = INSTRUMENTED_JUMP_FORWARD;
(void)(opcode);
}
TRACING_TARGET(INSTRUMENTED_LINE) {
+ assert(IS_JIT_TRACING());
#if _Py_TAIL_CALL_INTERP
int opcode = INSTRUMENTED_LINE;
(void)(opcode);
}
TRACING_TARGET(INSTRUMENTED_LOAD_SUPER_ATTR) {
+ assert(IS_JIT_TRACING());
#if _Py_TAIL_CALL_INTERP
int opcode = INSTRUMENTED_LOAD_SUPER_ATTR;
(void)(opcode);
}
TRACING_TARGET(INSTRUMENTED_NOT_TAKEN) {
+ assert(IS_JIT_TRACING());
#if _Py_TAIL_CALL_INTERP
int opcode = INSTRUMENTED_NOT_TAKEN;
(void)(opcode);
}
TRACING_TARGET(INSTRUMENTED_POP_ITER) {
+ assert(IS_JIT_TRACING());
#if _Py_TAIL_CALL_INTERP
int opcode = INSTRUMENTED_POP_ITER;
(void)(opcode);
}
TRACING_TARGET(INSTRUMENTED_POP_JUMP_IF_FALSE) {
+ assert(IS_JIT_TRACING());
#if _Py_TAIL_CALL_INTERP
int opcode = INSTRUMENTED_POP_JUMP_IF_FALSE;
(void)(opcode);
}
TRACING_TARGET(INSTRUMENTED_POP_JUMP_IF_NONE) {
+ assert(IS_JIT_TRACING());
#if _Py_TAIL_CALL_INTERP
int opcode = INSTRUMENTED_POP_JUMP_IF_NONE;
(void)(opcode);
}
TRACING_TARGET(INSTRUMENTED_POP_JUMP_IF_NOT_NONE) {
+ assert(IS_JIT_TRACING());
#if _Py_TAIL_CALL_INTERP
int opcode = INSTRUMENTED_POP_JUMP_IF_NOT_NONE;
(void)(opcode);
}
TRACING_TARGET(INSTRUMENTED_POP_JUMP_IF_TRUE) {
+ assert(IS_JIT_TRACING());
#if _Py_TAIL_CALL_INTERP
int opcode = INSTRUMENTED_POP_JUMP_IF_TRUE;
(void)(opcode);
}
TRACING_TARGET(INSTRUMENTED_RESUME) {
+ assert(IS_JIT_TRACING());
#if _Py_TAIL_CALL_INTERP
int opcode = INSTRUMENTED_RESUME;
(void)(opcode);
}
TRACING_TARGET(INSTRUMENTED_RETURN_VALUE) {
+ assert(IS_JIT_TRACING());
#if _Py_TAIL_CALL_INTERP
int opcode = INSTRUMENTED_RETURN_VALUE;
(void)(opcode);
}
TRACING_TARGET(INSTRUMENTED_YIELD_VALUE) {
+ assert(IS_JIT_TRACING());
#if _Py_TAIL_CALL_INTERP
int opcode = INSTRUMENTED_YIELD_VALUE;
(void)(opcode);
}
TRACING_TARGET(INTERPRETER_EXIT) {
+ assert(IS_JIT_TRACING());
#if _Py_TAIL_CALL_INTERP
int opcode = INTERPRETER_EXIT;
(void)(opcode);
}
TRACING_TARGET(IS_OP) {
+ assert(IS_JIT_TRACING());
#if _Py_TAIL_CALL_INTERP
int opcode = IS_OP;
(void)(opcode);
}
TRACING_TARGET(JUMP_BACKWARD) {
+ assert(IS_JIT_TRACING());
#if _Py_TAIL_CALL_INTERP
int opcode = JUMP_BACKWARD;
(void)(opcode);
}
TRACING_TARGET(JUMP_BACKWARD_JIT) {
+ assert(IS_JIT_TRACING());
#if _Py_TAIL_CALL_INTERP
int opcode = JUMP_BACKWARD_JIT;
(void)(opcode);
}
TRACING_TARGET(JUMP_BACKWARD_NO_INTERRUPT) {
+ assert(IS_JIT_TRACING());
#if _Py_TAIL_CALL_INTERP
int opcode = JUMP_BACKWARD_NO_INTERRUPT;
(void)(opcode);
}
TRACING_TARGET(JUMP_BACKWARD_NO_JIT) {
+ assert(IS_JIT_TRACING());
#if _Py_TAIL_CALL_INTERP
int opcode = JUMP_BACKWARD_NO_JIT;
(void)(opcode);
}
TRACING_TARGET(JUMP_FORWARD) {
+ assert(IS_JIT_TRACING());
#if _Py_TAIL_CALL_INTERP
int opcode = JUMP_FORWARD;
(void)(opcode);
}
TRACING_TARGET(LIST_APPEND) {
+ assert(IS_JIT_TRACING());
#if _Py_TAIL_CALL_INTERP
int opcode = LIST_APPEND;
(void)(opcode);
}
TRACING_TARGET(LIST_EXTEND) {
+ assert(IS_JIT_TRACING());
#if _Py_TAIL_CALL_INTERP
int opcode = LIST_EXTEND;
(void)(opcode);
}
TRACING_TARGET(LOAD_ATTR) {
+ assert(IS_JIT_TRACING());
#if _Py_TAIL_CALL_INTERP
int opcode = LOAD_ATTR;
(void)(opcode);
}
TRACING_TARGET(LOAD_ATTR_CLASS) {
+ assert(IS_JIT_TRACING());
#if _Py_TAIL_CALL_INTERP
int opcode = LOAD_ATTR_CLASS;
(void)(opcode);
if (!PyType_Check(owner_o)) {
UPDATE_MISS_STATS(LOAD_ATTR);
assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR));
- JUMP_TO_PREDICTED(LOAD_ATTR);
+ JUMP_TO_PREDICTED(TRACING_LOAD_ATTR);
}
assert(type_version != 0);
if (FT_ATOMIC_LOAD_UINT_RELAXED(((PyTypeObject *)owner_o)->tp_version_tag) != type_version) {
UPDATE_MISS_STATS(LOAD_ATTR);
assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR));
- JUMP_TO_PREDICTED(LOAD_ATTR);
+ JUMP_TO_PREDICTED(TRACING_LOAD_ATTR);
}
}
/* Skip 2 cache entries */
}
TRACING_TARGET(LOAD_ATTR_CLASS_WITH_METACLASS_CHECK) {
+ assert(IS_JIT_TRACING());
#if _Py_TAIL_CALL_INTERP
int opcode = LOAD_ATTR_CLASS_WITH_METACLASS_CHECK;
(void)(opcode);
if (!PyType_Check(owner_o)) {
UPDATE_MISS_STATS(LOAD_ATTR);
assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR));
- JUMP_TO_PREDICTED(LOAD_ATTR);
+ JUMP_TO_PREDICTED(TRACING_LOAD_ATTR);
}
assert(type_version != 0);
if (FT_ATOMIC_LOAD_UINT_RELAXED(((PyTypeObject *)owner_o)->tp_version_tag) != type_version) {
UPDATE_MISS_STATS(LOAD_ATTR);
assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR));
- JUMP_TO_PREDICTED(LOAD_ATTR);
+ JUMP_TO_PREDICTED(TRACING_LOAD_ATTR);
}
}
// _GUARD_TYPE_VERSION
if (FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version) {
UPDATE_MISS_STATS(LOAD_ATTR);
assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR));
- JUMP_TO_PREDICTED(LOAD_ATTR);
+ JUMP_TO_PREDICTED(TRACING_LOAD_ATTR);
}
}
// _LOAD_ATTR_CLASS
}
TRACING_TARGET(LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN) {
+ assert(IS_JIT_TRACING());
#if _Py_TAIL_CALL_INTERP
int opcode = LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN;
(void)(opcode);
if (tstate->interp->eval_frame) {
UPDATE_MISS_STATS(LOAD_ATTR);
assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR));
- JUMP_TO_PREDICTED(LOAD_ATTR);
+ JUMP_TO_PREDICTED(TRACING_LOAD_ATTR);
}
PyTypeObject *cls = Py_TYPE(owner_o);
assert(type_version != 0);
if (FT_ATOMIC_LOAD_UINT_RELAXED(cls->tp_version_tag) != type_version) {
UPDATE_MISS_STATS(LOAD_ATTR);
assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR));
- JUMP_TO_PREDICTED(LOAD_ATTR);
+ JUMP_TO_PREDICTED(TRACING_LOAD_ATTR);
}
assert(Py_IS_TYPE(getattribute, &PyFunction_Type));
PyFunctionObject *f = (PyFunctionObject *)getattribute;
if (f->func_version != func_version) {
UPDATE_MISS_STATS(LOAD_ATTR);
assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR));
- JUMP_TO_PREDICTED(LOAD_ATTR);
+ JUMP_TO_PREDICTED(TRACING_LOAD_ATTR);
}
PyCodeObject *code = (PyCodeObject *)f->func_code;
assert(code->co_argcount == 2);
if (!_PyThreadState_HasStackSpace(tstate, code->co_framesize)) {
UPDATE_MISS_STATS(LOAD_ATTR);
assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR));
- JUMP_TO_PREDICTED(LOAD_ATTR);
+ JUMP_TO_PREDICTED(TRACING_LOAD_ATTR);
}
STAT_INC(LOAD_ATTR, hit);
PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 1);
}
TRACING_TARGET(LOAD_ATTR_INSTANCE_VALUE) {
+ assert(IS_JIT_TRACING());
#if _Py_TAIL_CALL_INTERP
int opcode = LOAD_ATTR_INSTANCE_VALUE;
(void)(opcode);
if (FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version) {
UPDATE_MISS_STATS(LOAD_ATTR);
assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR));
- JUMP_TO_PREDICTED(LOAD_ATTR);
+ JUMP_TO_PREDICTED(TRACING_LOAD_ATTR);
}
}
// _CHECK_MANAGED_OBJECT_HAS_VALUES
if (!FT_ATOMIC_LOAD_UINT8(_PyObject_InlineValues(owner_o)->valid)) {
UPDATE_MISS_STATS(LOAD_ATTR);
assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR));
- JUMP_TO_PREDICTED(LOAD_ATTR);
+ JUMP_TO_PREDICTED(TRACING_LOAD_ATTR);
}
}
// _LOAD_ATTR_INSTANCE_VALUE
if (attr_o == NULL) {
UPDATE_MISS_STATS(LOAD_ATTR);
assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR));
- JUMP_TO_PREDICTED(LOAD_ATTR);
+ JUMP_TO_PREDICTED(TRACING_LOAD_ATTR);
}
#ifdef Py_GIL_DISABLED
int increfed = _Py_TryIncrefCompareStackRef(value_ptr, attr_o, &attr);
if (true) {
UPDATE_MISS_STATS(LOAD_ATTR);
assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR));
- JUMP_TO_PREDICTED(LOAD_ATTR);
+ JUMP_TO_PREDICTED(TRACING_LOAD_ATTR);
}
}
#else
}
TRACING_TARGET(LOAD_ATTR_METHOD_LAZY_DICT) {
+ assert(IS_JIT_TRACING());
#if _Py_TAIL_CALL_INTERP
int opcode = LOAD_ATTR_METHOD_LAZY_DICT;
(void)(opcode);
if (FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version) {
UPDATE_MISS_STATS(LOAD_ATTR);
assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR));
- JUMP_TO_PREDICTED(LOAD_ATTR);
+ JUMP_TO_PREDICTED(TRACING_LOAD_ATTR);
}
}
// _CHECK_ATTR_METHOD_LAZY_DICT
if (dict != NULL) {
UPDATE_MISS_STATS(LOAD_ATTR);
assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR));
- JUMP_TO_PREDICTED(LOAD_ATTR);
+ JUMP_TO_PREDICTED(TRACING_LOAD_ATTR);
}
}
/* Skip 1 cache entry */
}
TRACING_TARGET(LOAD_ATTR_METHOD_NO_DICT) {
+ assert(IS_JIT_TRACING());
#if _Py_TAIL_CALL_INTERP
int opcode = LOAD_ATTR_METHOD_NO_DICT;
(void)(opcode);
if (FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version) {
UPDATE_MISS_STATS(LOAD_ATTR);
assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR));
- JUMP_TO_PREDICTED(LOAD_ATTR);
+ JUMP_TO_PREDICTED(TRACING_LOAD_ATTR);
}
}
/* Skip 2 cache entries */
}
TRACING_TARGET(LOAD_ATTR_METHOD_WITH_VALUES) {
+ assert(IS_JIT_TRACING());
#if _Py_TAIL_CALL_INTERP
int opcode = LOAD_ATTR_METHOD_WITH_VALUES;
(void)(opcode);
if (FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version) {
UPDATE_MISS_STATS(LOAD_ATTR);
assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR));
- JUMP_TO_PREDICTED(LOAD_ATTR);
+ JUMP_TO_PREDICTED(TRACING_LOAD_ATTR);
}
}
// _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT
if (!FT_ATOMIC_LOAD_UINT8(ivs->valid)) {
UPDATE_MISS_STATS(LOAD_ATTR);
assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR));
- JUMP_TO_PREDICTED(LOAD_ATTR);
+ JUMP_TO_PREDICTED(TRACING_LOAD_ATTR);
}
}
// _GUARD_KEYS_VERSION
if (FT_ATOMIC_LOAD_UINT32_RELAXED(keys->dk_version) != keys_version) {
UPDATE_MISS_STATS(LOAD_ATTR);
assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR));
- JUMP_TO_PREDICTED(LOAD_ATTR);
+ JUMP_TO_PREDICTED(TRACING_LOAD_ATTR);
}
}
// _LOAD_ATTR_METHOD_WITH_VALUES
}
TRACING_TARGET(LOAD_ATTR_MODULE) {
+ assert(IS_JIT_TRACING());
#if _Py_TAIL_CALL_INTERP
int opcode = LOAD_ATTR_MODULE;
(void)(opcode);
if (Py_TYPE(owner_o)->tp_getattro != PyModule_Type.tp_getattro) {
UPDATE_MISS_STATS(LOAD_ATTR);
assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR));
- JUMP_TO_PREDICTED(LOAD_ATTR);
+ JUMP_TO_PREDICTED(TRACING_LOAD_ATTR);
}
PyDictObject *dict = (PyDictObject *)((PyModuleObject *)owner_o)->md_dict;
assert(dict != NULL);
if (FT_ATOMIC_LOAD_UINT32_RELAXED(keys->dk_version) != dict_version) {
UPDATE_MISS_STATS(LOAD_ATTR);
assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR));
- JUMP_TO_PREDICTED(LOAD_ATTR);
+ JUMP_TO_PREDICTED(TRACING_LOAD_ATTR);
}
assert(keys->dk_kind == DICT_KEYS_UNICODE);
assert(index < FT_ATOMIC_LOAD_SSIZE_RELAXED(keys->dk_nentries));
if (attr_o == NULL) {
UPDATE_MISS_STATS(LOAD_ATTR);
assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR));
- JUMP_TO_PREDICTED(LOAD_ATTR);
+ JUMP_TO_PREDICTED(TRACING_LOAD_ATTR);
}
#ifdef Py_GIL_DISABLED
int increfed = _Py_TryIncrefCompareStackRef(&ep->me_value, attr_o, &attr);
if (true) {
UPDATE_MISS_STATS(LOAD_ATTR);
assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR));
- JUMP_TO_PREDICTED(LOAD_ATTR);
+ JUMP_TO_PREDICTED(TRACING_LOAD_ATTR);
}
}
#else
}
TRACING_TARGET(LOAD_ATTR_NONDESCRIPTOR_NO_DICT) {
+ assert(IS_JIT_TRACING());
#if _Py_TAIL_CALL_INTERP
int opcode = LOAD_ATTR_NONDESCRIPTOR_NO_DICT;
(void)(opcode);
if (FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version) {
UPDATE_MISS_STATS(LOAD_ATTR);
assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR));
- JUMP_TO_PREDICTED(LOAD_ATTR);
+ JUMP_TO_PREDICTED(TRACING_LOAD_ATTR);
}
}
/* Skip 2 cache entries */
}
TRACING_TARGET(LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES) {
+ assert(IS_JIT_TRACING());
#if _Py_TAIL_CALL_INTERP
int opcode = LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES;
(void)(opcode);
if (FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version) {
UPDATE_MISS_STATS(LOAD_ATTR);
assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR));
- JUMP_TO_PREDICTED(LOAD_ATTR);
+ JUMP_TO_PREDICTED(TRACING_LOAD_ATTR);
}
}
// _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT
if (!FT_ATOMIC_LOAD_UINT8(ivs->valid)) {
UPDATE_MISS_STATS(LOAD_ATTR);
assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR));
- JUMP_TO_PREDICTED(LOAD_ATTR);
+ JUMP_TO_PREDICTED(TRACING_LOAD_ATTR);
}
}
// _GUARD_KEYS_VERSION
if (FT_ATOMIC_LOAD_UINT32_RELAXED(keys->dk_version) != keys_version) {
UPDATE_MISS_STATS(LOAD_ATTR);
assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR));
- JUMP_TO_PREDICTED(LOAD_ATTR);
+ JUMP_TO_PREDICTED(TRACING_LOAD_ATTR);
}
}
// _LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES
}
TRACING_TARGET(LOAD_ATTR_PROPERTY) {
+ assert(IS_JIT_TRACING());
#if _Py_TAIL_CALL_INTERP
int opcode = LOAD_ATTR_PROPERTY;
(void)(opcode);
if (tstate->interp->eval_frame) {
UPDATE_MISS_STATS(LOAD_ATTR);
assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR));
- JUMP_TO_PREDICTED(LOAD_ATTR);
+ JUMP_TO_PREDICTED(TRACING_LOAD_ATTR);
}
}
// _GUARD_TYPE_VERSION
if (FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version) {
UPDATE_MISS_STATS(LOAD_ATTR);
assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR));
- JUMP_TO_PREDICTED(LOAD_ATTR);
+ JUMP_TO_PREDICTED(TRACING_LOAD_ATTR);
}
}
/* Skip 2 cache entries */
if ((code->co_flags & (CO_VARKEYWORDS | CO_VARARGS | CO_OPTIMIZED)) != CO_OPTIMIZED) {
UPDATE_MISS_STATS(LOAD_ATTR);
assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR));
- JUMP_TO_PREDICTED(LOAD_ATTR);
+ JUMP_TO_PREDICTED(TRACING_LOAD_ATTR);
}
if (code->co_kwonlyargcount) {
UPDATE_MISS_STATS(LOAD_ATTR);
assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR));
- JUMP_TO_PREDICTED(LOAD_ATTR);
+ JUMP_TO_PREDICTED(TRACING_LOAD_ATTR);
}
if (code->co_argcount != 1) {
UPDATE_MISS_STATS(LOAD_ATTR);
assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR));
- JUMP_TO_PREDICTED(LOAD_ATTR);
+ JUMP_TO_PREDICTED(TRACING_LOAD_ATTR);
}
if (!_PyThreadState_HasStackSpace(tstate, code->co_framesize)) {
UPDATE_MISS_STATS(LOAD_ATTR);
assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR));
- JUMP_TO_PREDICTED(LOAD_ATTR);
+ JUMP_TO_PREDICTED(TRACING_LOAD_ATTR);
}
STAT_INC(LOAD_ATTR, hit);
_PyInterpreterFrame *pushed_frame = _PyFrame_PushUnchecked(tstate, PyStackRef_FromPyObjectNew(fget), 1, frame);
}
TRACING_TARGET(LOAD_ATTR_SLOT) {
+ assert(IS_JIT_TRACING());
#if _Py_TAIL_CALL_INTERP
int opcode = LOAD_ATTR_SLOT;
(void)(opcode);
if (FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version) {
UPDATE_MISS_STATS(LOAD_ATTR);
assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR));
- JUMP_TO_PREDICTED(LOAD_ATTR);
+ JUMP_TO_PREDICTED(TRACING_LOAD_ATTR);
}
}
// _LOAD_ATTR_SLOT
if (attr_o == NULL) {
UPDATE_MISS_STATS(LOAD_ATTR);
assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR));
- JUMP_TO_PREDICTED(LOAD_ATTR);
+ JUMP_TO_PREDICTED(TRACING_LOAD_ATTR);
}
#ifdef Py_GIL_DISABLED
int increfed = _Py_TryIncrefCompareStackRef(addr, attr_o, &attr);
if (!increfed) {
UPDATE_MISS_STATS(LOAD_ATTR);
assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR));
- JUMP_TO_PREDICTED(LOAD_ATTR);
+ JUMP_TO_PREDICTED(TRACING_LOAD_ATTR);
}
#else
attr = PyStackRef_FromPyObjectNew(attr_o);
}
TRACING_TARGET(LOAD_ATTR_WITH_HINT) {
+ assert(IS_JIT_TRACING());
#if _Py_TAIL_CALL_INTERP
int opcode = LOAD_ATTR_WITH_HINT;
(void)(opcode);
if (FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version) {
UPDATE_MISS_STATS(LOAD_ATTR);
assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR));
- JUMP_TO_PREDICTED(LOAD_ATTR);
+ JUMP_TO_PREDICTED(TRACING_LOAD_ATTR);
}
}
// _LOAD_ATTR_WITH_HINT
if (dict == NULL) {
UPDATE_MISS_STATS(LOAD_ATTR);
assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR));
- JUMP_TO_PREDICTED(LOAD_ATTR);
+ JUMP_TO_PREDICTED(TRACING_LOAD_ATTR);
}
PyDictKeysObject *dk = FT_ATOMIC_LOAD_PTR(dict->ma_keys);
assert(PyDict_CheckExact((PyObject *)dict));
if (!_Py_IsOwnedByCurrentThread((PyObject *)dict) && !_PyObject_GC_IS_SHARED(dict)) {
UPDATE_MISS_STATS(LOAD_ATTR);
assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR));
- JUMP_TO_PREDICTED(LOAD_ATTR);
+ JUMP_TO_PREDICTED(TRACING_LOAD_ATTR);
}
#endif
PyObject *attr_o;
if (true) {
UPDATE_MISS_STATS(LOAD_ATTR);
assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR));
- JUMP_TO_PREDICTED(LOAD_ATTR);
+ JUMP_TO_PREDICTED(TRACING_LOAD_ATTR);
}
}
PyObject *name = GETITEM(FRAME_CO_NAMES, oparg>>1);
if (true) {
UPDATE_MISS_STATS(LOAD_ATTR);
assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR));
- JUMP_TO_PREDICTED(LOAD_ATTR);
+ JUMP_TO_PREDICTED(TRACING_LOAD_ATTR);
}
}
PyDictUnicodeEntry *ep = DK_UNICODE_ENTRIES(dk) + hint;
if (true) {
UPDATE_MISS_STATS(LOAD_ATTR);
assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR));
- JUMP_TO_PREDICTED(LOAD_ATTR);
+ JUMP_TO_PREDICTED(TRACING_LOAD_ATTR);
}
}
attr_o = FT_ATOMIC_LOAD_PTR(ep->me_value);
if (true) {
UPDATE_MISS_STATS(LOAD_ATTR);
assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR));
- JUMP_TO_PREDICTED(LOAD_ATTR);
+ JUMP_TO_PREDICTED(TRACING_LOAD_ATTR);
}
}
STAT_INC(LOAD_ATTR, hit);
if (true) {
UPDATE_MISS_STATS(LOAD_ATTR);
assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR));
- JUMP_TO_PREDICTED(LOAD_ATTR);
+ JUMP_TO_PREDICTED(TRACING_LOAD_ATTR);
}
}
#else
}
TRACING_TARGET(LOAD_BUILD_CLASS) {
+ assert(IS_JIT_TRACING());
#if _Py_TAIL_CALL_INTERP
int opcode = LOAD_BUILD_CLASS;
(void)(opcode);
}
TRACING_TARGET(LOAD_COMMON_CONSTANT) {
+ assert(IS_JIT_TRACING());
#if _Py_TAIL_CALL_INTERP
int opcode = LOAD_COMMON_CONSTANT;
(void)(opcode);
}
TRACING_TARGET(LOAD_CONST) {
+ assert(IS_JIT_TRACING());
#if _Py_TAIL_CALL_INTERP
int opcode = LOAD_CONST;
(void)(opcode);
}
TRACING_TARGET(LOAD_DEREF) {
+ assert(IS_JIT_TRACING());
#if _Py_TAIL_CALL_INTERP
int opcode = LOAD_DEREF;
(void)(opcode);
}
TRACING_TARGET(LOAD_FAST) {
+ assert(IS_JIT_TRACING());
#if _Py_TAIL_CALL_INTERP
int opcode = LOAD_FAST;
(void)(opcode);
}
TRACING_TARGET(LOAD_FAST_AND_CLEAR) {
+ assert(IS_JIT_TRACING());
#if _Py_TAIL_CALL_INTERP
int opcode = LOAD_FAST_AND_CLEAR;
(void)(opcode);
}
TRACING_TARGET(LOAD_FAST_BORROW) {
+ assert(IS_JIT_TRACING());
#if _Py_TAIL_CALL_INTERP
int opcode = LOAD_FAST_BORROW;
(void)(opcode);
}
TRACING_TARGET(LOAD_FAST_BORROW_LOAD_FAST_BORROW) {
+ assert(IS_JIT_TRACING());
#if _Py_TAIL_CALL_INTERP
int opcode = LOAD_FAST_BORROW_LOAD_FAST_BORROW;
(void)(opcode);
}
TRACING_TARGET(LOAD_FAST_CHECK) {
+ assert(IS_JIT_TRACING());
#if _Py_TAIL_CALL_INTERP
int opcode = LOAD_FAST_CHECK;
(void)(opcode);
}
TRACING_TARGET(LOAD_FAST_LOAD_FAST) {
+ assert(IS_JIT_TRACING());
#if _Py_TAIL_CALL_INTERP
int opcode = LOAD_FAST_LOAD_FAST;
(void)(opcode);
}
TRACING_TARGET(LOAD_FROM_DICT_OR_DEREF) {
+ assert(IS_JIT_TRACING());
#if _Py_TAIL_CALL_INTERP
int opcode = LOAD_FROM_DICT_OR_DEREF;
(void)(opcode);
}
TRACING_TARGET(LOAD_FROM_DICT_OR_GLOBALS) {
+ assert(IS_JIT_TRACING());
#if _Py_TAIL_CALL_INTERP
int opcode = LOAD_FROM_DICT_OR_GLOBALS;
(void)(opcode);
}
TRACING_TARGET(LOAD_GLOBAL) {
+ assert(IS_JIT_TRACING());
#if _Py_TAIL_CALL_INTERP
int opcode = LOAD_GLOBAL;
(void)(opcode);
}
TRACING_TARGET(LOAD_GLOBAL_BUILTIN) {
+ assert(IS_JIT_TRACING());
#if _Py_TAIL_CALL_INTERP
int opcode = LOAD_GLOBAL_BUILTIN;
(void)(opcode);
if (!PyDict_CheckExact(dict)) {
UPDATE_MISS_STATS(LOAD_GLOBAL);
assert(_PyOpcode_Deopt[opcode] == (LOAD_GLOBAL));
- JUMP_TO_PREDICTED(LOAD_GLOBAL);
+ JUMP_TO_PREDICTED(TRACING_LOAD_GLOBAL);
}
PyDictKeysObject *keys = FT_ATOMIC_LOAD_PTR_ACQUIRE(dict->ma_keys);
if (FT_ATOMIC_LOAD_UINT32_RELAXED(keys->dk_version) != version) {
UPDATE_MISS_STATS(LOAD_GLOBAL);
assert(_PyOpcode_Deopt[opcode] == (LOAD_GLOBAL));
- JUMP_TO_PREDICTED(LOAD_GLOBAL);
+ JUMP_TO_PREDICTED(TRACING_LOAD_GLOBAL);
}
assert(DK_IS_UNICODE(keys));
}
if (!PyDict_CheckExact(dict)) {
UPDATE_MISS_STATS(LOAD_GLOBAL);
assert(_PyOpcode_Deopt[opcode] == (LOAD_GLOBAL));
- JUMP_TO_PREDICTED(LOAD_GLOBAL);
+ JUMP_TO_PREDICTED(TRACING_LOAD_GLOBAL);
}
PyDictKeysObject *keys = FT_ATOMIC_LOAD_PTR_ACQUIRE(dict->ma_keys);
if (FT_ATOMIC_LOAD_UINT32_RELAXED(keys->dk_version) != version) {
UPDATE_MISS_STATS(LOAD_GLOBAL);
assert(_PyOpcode_Deopt[opcode] == (LOAD_GLOBAL));
- JUMP_TO_PREDICTED(LOAD_GLOBAL);
+ JUMP_TO_PREDICTED(TRACING_LOAD_GLOBAL);
}
assert(DK_IS_UNICODE(keys));
PyDictUnicodeEntry *entries = DK_UNICODE_ENTRIES(keys);
if (res_o == NULL) {
UPDATE_MISS_STATS(LOAD_GLOBAL);
assert(_PyOpcode_Deopt[opcode] == (LOAD_GLOBAL));
- JUMP_TO_PREDICTED(LOAD_GLOBAL);
+ JUMP_TO_PREDICTED(TRACING_LOAD_GLOBAL);
}
#if Py_GIL_DISABLED
int increfed = _Py_TryIncrefCompareStackRef(&entries[index].me_value, res_o, &res);
if (!increfed) {
UPDATE_MISS_STATS(LOAD_GLOBAL);
assert(_PyOpcode_Deopt[opcode] == (LOAD_GLOBAL));
- JUMP_TO_PREDICTED(LOAD_GLOBAL);
+ JUMP_TO_PREDICTED(TRACING_LOAD_GLOBAL);
}
#else
res = PyStackRef_FromPyObjectNew(res_o);
}
TRACING_TARGET(LOAD_GLOBAL_MODULE) {
+ assert(IS_JIT_TRACING());
#if _Py_TAIL_CALL_INTERP
int opcode = LOAD_GLOBAL_MODULE;
(void)(opcode);
if (!PyDict_CheckExact(dict)) {
UPDATE_MISS_STATS(LOAD_GLOBAL);
assert(_PyOpcode_Deopt[opcode] == (LOAD_GLOBAL));
- JUMP_TO_PREDICTED(LOAD_GLOBAL);
+ JUMP_TO_PREDICTED(TRACING_LOAD_GLOBAL);
}
PyDictKeysObject *keys = FT_ATOMIC_LOAD_PTR_ACQUIRE(dict->ma_keys);
if (FT_ATOMIC_LOAD_UINT32_RELAXED(keys->dk_version) != version) {
UPDATE_MISS_STATS(LOAD_GLOBAL);
assert(_PyOpcode_Deopt[opcode] == (LOAD_GLOBAL));
- JUMP_TO_PREDICTED(LOAD_GLOBAL);
+ JUMP_TO_PREDICTED(TRACING_LOAD_GLOBAL);
}
assert(DK_IS_UNICODE(keys));
PyDictUnicodeEntry *entries = DK_UNICODE_ENTRIES(keys);
if (res_o == NULL) {
UPDATE_MISS_STATS(LOAD_GLOBAL);
assert(_PyOpcode_Deopt[opcode] == (LOAD_GLOBAL));
- JUMP_TO_PREDICTED(LOAD_GLOBAL);
+ JUMP_TO_PREDICTED(TRACING_LOAD_GLOBAL);
}
#if Py_GIL_DISABLED
int increfed = _Py_TryIncrefCompareStackRef(&entries[index].me_value, res_o, &res);
if (!increfed) {
UPDATE_MISS_STATS(LOAD_GLOBAL);
assert(_PyOpcode_Deopt[opcode] == (LOAD_GLOBAL));
- JUMP_TO_PREDICTED(LOAD_GLOBAL);
+ JUMP_TO_PREDICTED(TRACING_LOAD_GLOBAL);
}
#else
res = PyStackRef_FromPyObjectNew(res_o);
}
TRACING_TARGET(LOAD_LOCALS) {
+ assert(IS_JIT_TRACING());
#if _Py_TAIL_CALL_INTERP
int opcode = LOAD_LOCALS;
(void)(opcode);
}
TRACING_TARGET(LOAD_NAME) {
+ assert(IS_JIT_TRACING());
#if _Py_TAIL_CALL_INTERP
int opcode = LOAD_NAME;
(void)(opcode);
}
TRACING_TARGET(LOAD_SMALL_INT) {
+ assert(IS_JIT_TRACING());
#if _Py_TAIL_CALL_INTERP
int opcode = LOAD_SMALL_INT;
(void)(opcode);
}
TRACING_TARGET(LOAD_SPECIAL) {
+ assert(IS_JIT_TRACING());
#if _Py_TAIL_CALL_INTERP
int opcode = LOAD_SPECIAL;
(void)(opcode);
}
TRACING_TARGET(LOAD_SUPER_ATTR) {
+ assert(IS_JIT_TRACING());
#if _Py_TAIL_CALL_INTERP
int opcode = LOAD_SUPER_ATTR;
(void)(opcode);
}
TRACING_TARGET(LOAD_SUPER_ATTR_ATTR) {
+ assert(IS_JIT_TRACING());
#if _Py_TAIL_CALL_INTERP
int opcode = LOAD_SUPER_ATTR_ATTR;
(void)(opcode);
if (global_super != (PyObject *)&PySuper_Type) {
UPDATE_MISS_STATS(LOAD_SUPER_ATTR);
assert(_PyOpcode_Deopt[opcode] == (LOAD_SUPER_ATTR));
- JUMP_TO_PREDICTED(LOAD_SUPER_ATTR);
+ JUMP_TO_PREDICTED(TRACING_LOAD_SUPER_ATTR);
}
if (!PyType_Check(class)) {
UPDATE_MISS_STATS(LOAD_SUPER_ATTR);
assert(_PyOpcode_Deopt[opcode] == (LOAD_SUPER_ATTR));
- JUMP_TO_PREDICTED(LOAD_SUPER_ATTR);
+ JUMP_TO_PREDICTED(TRACING_LOAD_SUPER_ATTR);
}
STAT_INC(LOAD_SUPER_ATTR, hit);
PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 2);
}
TRACING_TARGET(LOAD_SUPER_ATTR_METHOD) {
+ assert(IS_JIT_TRACING());
#if _Py_TAIL_CALL_INTERP
int opcode = LOAD_SUPER_ATTR_METHOD;
(void)(opcode);
if (global_super != (PyObject *)&PySuper_Type) {
UPDATE_MISS_STATS(LOAD_SUPER_ATTR);
assert(_PyOpcode_Deopt[opcode] == (LOAD_SUPER_ATTR));
- JUMP_TO_PREDICTED(LOAD_SUPER_ATTR);
+ JUMP_TO_PREDICTED(TRACING_LOAD_SUPER_ATTR);
}
if (!PyType_Check(class)) {
UPDATE_MISS_STATS(LOAD_SUPER_ATTR);
assert(_PyOpcode_Deopt[opcode] == (LOAD_SUPER_ATTR));
- JUMP_TO_PREDICTED(LOAD_SUPER_ATTR);
+ JUMP_TO_PREDICTED(TRACING_LOAD_SUPER_ATTR);
}
STAT_INC(LOAD_SUPER_ATTR, hit);
PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 2);
}
TRACING_TARGET(MAKE_CELL) {
+ assert(IS_JIT_TRACING());
#if _Py_TAIL_CALL_INTERP
int opcode = MAKE_CELL;
(void)(opcode);
}
TRACING_TARGET(MAKE_FUNCTION) {
+ assert(IS_JIT_TRACING());
#if _Py_TAIL_CALL_INTERP
int opcode = MAKE_FUNCTION;
(void)(opcode);
}
TRACING_TARGET(MAP_ADD) {
+ assert(IS_JIT_TRACING());
#if _Py_TAIL_CALL_INTERP
int opcode = MAP_ADD;
(void)(opcode);
}
TRACING_TARGET(MATCH_CLASS) {
+ assert(IS_JIT_TRACING());
#if _Py_TAIL_CALL_INTERP
int opcode = MATCH_CLASS;
(void)(opcode);
}
TRACING_TARGET(MATCH_KEYS) {
+ assert(IS_JIT_TRACING());
#if _Py_TAIL_CALL_INTERP
int opcode = MATCH_KEYS;
(void)(opcode);
}
TRACING_TARGET(MATCH_MAPPING) {
+ assert(IS_JIT_TRACING());
#if _Py_TAIL_CALL_INTERP
int opcode = MATCH_MAPPING;
(void)(opcode);
}
TRACING_TARGET(MATCH_SEQUENCE) {
+ assert(IS_JIT_TRACING());
#if _Py_TAIL_CALL_INTERP
int opcode = MATCH_SEQUENCE;
(void)(opcode);
}
TRACING_TARGET(NOP) {
+ assert(IS_JIT_TRACING());
#if _Py_TAIL_CALL_INTERP
int opcode = NOP;
(void)(opcode);
}
TRACING_TARGET(NOT_TAKEN) {
+ assert(IS_JIT_TRACING());
#if _Py_TAIL_CALL_INTERP
int opcode = NOT_TAKEN;
(void)(opcode);
}
TRACING_TARGET(POP_EXCEPT) {
+ assert(IS_JIT_TRACING());
#if _Py_TAIL_CALL_INTERP
int opcode = POP_EXCEPT;
(void)(opcode);
}
TRACING_TARGET(POP_ITER) {
+ assert(IS_JIT_TRACING());
#if _Py_TAIL_CALL_INTERP
int opcode = POP_ITER;
(void)(opcode);
}
TRACING_TARGET(POP_JUMP_IF_FALSE) {
+ assert(IS_JIT_TRACING());
#if _Py_TAIL_CALL_INTERP
int opcode = POP_JUMP_IF_FALSE;
(void)(opcode);
cond = stack_pointer[-1];
assert(PyStackRef_BoolCheck(cond));
int flag = PyStackRef_IsFalse(cond);
- JUMPBY(flag ? oparg : 0);
+ RECORD_BRANCH_TAKEN(this_instr[1].cache, flag);
+ JUMPBY(flag ? oparg : next_instr->op.code == NOT_TAKEN);
stack_pointer += -1;
assert(WITHIN_STACK_BOUNDS());
TRACING_DISPATCH();
}
TRACING_TARGET(POP_JUMP_IF_NONE) {
+ assert(IS_JIT_TRACING());
#if _Py_TAIL_CALL_INTERP
int opcode = POP_JUMP_IF_NONE;
(void)(opcode);
cond = b;
assert(PyStackRef_BoolCheck(cond));
int flag = PyStackRef_IsTrue(cond);
- JUMPBY(flag ? oparg : 0);
+ RECORD_BRANCH_TAKEN(this_instr[1].cache, flag);
+ JUMPBY(flag ? oparg : next_instr->op.code == NOT_TAKEN);
}
stack_pointer += -1;
assert(WITHIN_STACK_BOUNDS());
}
TRACING_TARGET(POP_JUMP_IF_NOT_NONE) {
+ assert(IS_JIT_TRACING());
#if _Py_TAIL_CALL_INTERP
int opcode = POP_JUMP_IF_NOT_NONE;
(void)(opcode);
cond = b;
assert(PyStackRef_BoolCheck(cond));
int flag = PyStackRef_IsFalse(cond);
- JUMPBY(flag ? oparg : 0);
+ RECORD_BRANCH_TAKEN(this_instr[1].cache, flag);
+ JUMPBY(flag ? oparg : next_instr->op.code == NOT_TAKEN);
}
stack_pointer += -1;
assert(WITHIN_STACK_BOUNDS());
}
TRACING_TARGET(POP_JUMP_IF_TRUE) {
+ assert(IS_JIT_TRACING());
#if _Py_TAIL_CALL_INTERP
int opcode = POP_JUMP_IF_TRUE;
(void)(opcode);
cond = stack_pointer[-1];
assert(PyStackRef_BoolCheck(cond));
int flag = PyStackRef_IsTrue(cond);
- JUMPBY(flag ? oparg : 0);
+ RECORD_BRANCH_TAKEN(this_instr[1].cache, flag);
+ JUMPBY(flag ? oparg : next_instr->op.code == NOT_TAKEN);
stack_pointer += -1;
assert(WITHIN_STACK_BOUNDS());
TRACING_DISPATCH();
}
TRACING_TARGET(POP_TOP) {
+ assert(IS_JIT_TRACING());
#if _Py_TAIL_CALL_INTERP
int opcode = POP_TOP;
(void)(opcode);
}
TRACING_TARGET(PUSH_EXC_INFO) {
+ assert(IS_JIT_TRACING());
#if _Py_TAIL_CALL_INTERP
int opcode = PUSH_EXC_INFO;
(void)(opcode);
}
TRACING_TARGET(PUSH_NULL) {
+ assert(IS_JIT_TRACING());
#if _Py_TAIL_CALL_INTERP
int opcode = PUSH_NULL;
(void)(opcode);
}
TRACING_TARGET(RAISE_VARARGS) {
+ assert(IS_JIT_TRACING());
#if _Py_TAIL_CALL_INTERP
int opcode = RAISE_VARARGS;
(void)(opcode);
}
TRACING_TARGET(RERAISE) {
+ assert(IS_JIT_TRACING());
#if _Py_TAIL_CALL_INTERP
int opcode = RERAISE;
(void)(opcode);
}
TRACING_TARGET(RESERVED) {
+ assert(IS_JIT_TRACING());
#if _Py_TAIL_CALL_INTERP
int opcode = RESERVED;
(void)(opcode);
}
TRACING_TARGET(RESUME) {
+ assert(IS_JIT_TRACING());
#if _Py_TAIL_CALL_INTERP
int opcode = RESUME;
(void)(opcode);
}
TRACING_TARGET(RESUME_CHECK) {
+ assert(IS_JIT_TRACING());
#if _Py_TAIL_CALL_INTERP
int opcode = RESUME_CHECK;
(void)(opcode);
if (_Py_emscripten_signal_clock == 0) {
UPDATE_MISS_STATS(RESUME);
assert(_PyOpcode_Deopt[opcode] == (RESUME));
- JUMP_TO_PREDICTED(RESUME);
+ JUMP_TO_PREDICTED(TRACING_RESUME);
}
_Py_emscripten_signal_clock -= Py_EMSCRIPTEN_SIGNAL_HANDLING;
#endif
if (eval_breaker != version) {
UPDATE_MISS_STATS(RESUME);
assert(_PyOpcode_Deopt[opcode] == (RESUME));
- JUMP_TO_PREDICTED(RESUME);
+ JUMP_TO_PREDICTED(TRACING_RESUME);
}
#ifdef Py_GIL_DISABLED
if (frame->tlbc_index !=
((_PyThreadStateImpl *)tstate)->tlbc_index) {
UPDATE_MISS_STATS(RESUME);
assert(_PyOpcode_Deopt[opcode] == (RESUME));
- JUMP_TO_PREDICTED(RESUME);
+ JUMP_TO_PREDICTED(TRACING_RESUME);
}
#endif
TRACING_DISPATCH();
}
TRACING_TARGET(RETURN_GENERATOR) {
+ assert(IS_JIT_TRACING());
#if _Py_TAIL_CALL_INTERP
int opcode = RETURN_GENERATOR;
(void)(opcode);
}
TRACING_TARGET(RETURN_VALUE) {
+ assert(IS_JIT_TRACING());
#if _Py_TAIL_CALL_INTERP
int opcode = RETURN_VALUE;
(void)(opcode);
}
TRACING_TARGET(SEND) {
+ assert(IS_JIT_TRACING());
#if _Py_TAIL_CALL_INTERP
int opcode = SEND;
(void)(opcode);
}
TRACING_TARGET(SEND_GEN) {
+ assert(IS_JIT_TRACING());
#if _Py_TAIL_CALL_INTERP
int opcode = SEND_GEN;
(void)(opcode);
if (tstate->interp->eval_frame) {
UPDATE_MISS_STATS(SEND);
assert(_PyOpcode_Deopt[opcode] == (SEND));
- JUMP_TO_PREDICTED(SEND);
+ JUMP_TO_PREDICTED(TRACING_SEND);
}
}
// _SEND_GEN_FRAME
if (Py_TYPE(gen) != &PyGen_Type && Py_TYPE(gen) != &PyCoro_Type) {
UPDATE_MISS_STATS(SEND);
assert(_PyOpcode_Deopt[opcode] == (SEND));
- JUMP_TO_PREDICTED(SEND);
+ JUMP_TO_PREDICTED(TRACING_SEND);
}
if (gen->gi_frame_state >= FRAME_EXECUTING) {
UPDATE_MISS_STATS(SEND);
assert(_PyOpcode_Deopt[opcode] == (SEND));
- JUMP_TO_PREDICTED(SEND);
+ JUMP_TO_PREDICTED(TRACING_SEND);
}
STAT_INC(SEND, hit);
_PyInterpreterFrame *pushed_frame = &gen->gi_iframe;
}
TRACING_TARGET(SETUP_ANNOTATIONS) {
+ assert(IS_JIT_TRACING());
#if _Py_TAIL_CALL_INTERP
int opcode = SETUP_ANNOTATIONS;
(void)(opcode);
}
TRACING_TARGET(SET_ADD) {
+ assert(IS_JIT_TRACING());
#if _Py_TAIL_CALL_INTERP
int opcode = SET_ADD;
(void)(opcode);
}
TRACING_TARGET(SET_FUNCTION_ATTRIBUTE) {
+ assert(IS_JIT_TRACING());
#if _Py_TAIL_CALL_INTERP
int opcode = SET_FUNCTION_ATTRIBUTE;
(void)(opcode);
}
TRACING_TARGET(SET_UPDATE) {
+ assert(IS_JIT_TRACING());
#if _Py_TAIL_CALL_INTERP
int opcode = SET_UPDATE;
(void)(opcode);
}
TRACING_TARGET(STORE_ATTR) {
+ assert(IS_JIT_TRACING());
#if _Py_TAIL_CALL_INTERP
int opcode = STORE_ATTR;
(void)(opcode);
}
TRACING_TARGET(STORE_ATTR_INSTANCE_VALUE) {
+ assert(IS_JIT_TRACING());
#if _Py_TAIL_CALL_INTERP
int opcode = STORE_ATTR_INSTANCE_VALUE;
(void)(opcode);
if (!LOCK_OBJECT(owner_o)) {
UPDATE_MISS_STATS(STORE_ATTR);
assert(_PyOpcode_Deopt[opcode] == (STORE_ATTR));
- JUMP_TO_PREDICTED(STORE_ATTR);
+ JUMP_TO_PREDICTED(TRACING_STORE_ATTR);
}
PyTypeObject *tp = Py_TYPE(owner_o);
if (FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version) {
if (true) {
UPDATE_MISS_STATS(STORE_ATTR);
assert(_PyOpcode_Deopt[opcode] == (STORE_ATTR));
- JUMP_TO_PREDICTED(STORE_ATTR);
+ JUMP_TO_PREDICTED(TRACING_STORE_ATTR);
}
}
}
if (true) {
UPDATE_MISS_STATS(STORE_ATTR);
assert(_PyOpcode_Deopt[opcode] == (STORE_ATTR));
- JUMP_TO_PREDICTED(STORE_ATTR);
+ JUMP_TO_PREDICTED(TRACING_STORE_ATTR);
}
}
}
}
TRACING_TARGET(STORE_ATTR_SLOT) {
+ assert(IS_JIT_TRACING());
#if _Py_TAIL_CALL_INTERP
int opcode = STORE_ATTR_SLOT;
(void)(opcode);
if (FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version) {
UPDATE_MISS_STATS(STORE_ATTR);
assert(_PyOpcode_Deopt[opcode] == (STORE_ATTR));
- JUMP_TO_PREDICTED(STORE_ATTR);
+ JUMP_TO_PREDICTED(TRACING_STORE_ATTR);
}
}
// _STORE_ATTR_SLOT
if (!LOCK_OBJECT(owner_o)) {
UPDATE_MISS_STATS(STORE_ATTR);
assert(_PyOpcode_Deopt[opcode] == (STORE_ATTR));
- JUMP_TO_PREDICTED(STORE_ATTR);
+ JUMP_TO_PREDICTED(TRACING_STORE_ATTR);
}
char *addr = (char *)owner_o + index;
STAT_INC(STORE_ATTR, hit);
}
TRACING_TARGET(STORE_ATTR_WITH_HINT) {
+ assert(IS_JIT_TRACING());
#if _Py_TAIL_CALL_INTERP
int opcode = STORE_ATTR_WITH_HINT;
(void)(opcode);
if (FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version) {
UPDATE_MISS_STATS(STORE_ATTR);
assert(_PyOpcode_Deopt[opcode] == (STORE_ATTR));
- JUMP_TO_PREDICTED(STORE_ATTR);
+ JUMP_TO_PREDICTED(TRACING_STORE_ATTR);
}
}
// _STORE_ATTR_WITH_HINT
if (dict == NULL) {
UPDATE_MISS_STATS(STORE_ATTR);
assert(_PyOpcode_Deopt[opcode] == (STORE_ATTR));
- JUMP_TO_PREDICTED(STORE_ATTR);
+ JUMP_TO_PREDICTED(TRACING_STORE_ATTR);
}
if (!LOCK_OBJECT(dict)) {
UPDATE_MISS_STATS(STORE_ATTR);
assert(_PyOpcode_Deopt[opcode] == (STORE_ATTR));
- JUMP_TO_PREDICTED(STORE_ATTR);
+ JUMP_TO_PREDICTED(TRACING_STORE_ATTR);
}
assert(PyDict_CheckExact((PyObject *)dict));
PyObject *name = GETITEM(FRAME_CO_NAMES, oparg);
if (true) {
UPDATE_MISS_STATS(STORE_ATTR);
assert(_PyOpcode_Deopt[opcode] == (STORE_ATTR));
- JUMP_TO_PREDICTED(STORE_ATTR);
+ JUMP_TO_PREDICTED(TRACING_STORE_ATTR);
}
}
PyDictUnicodeEntry *ep = DK_UNICODE_ENTRIES(dict->ma_keys) + hint;
if (true) {
UPDATE_MISS_STATS(STORE_ATTR);
assert(_PyOpcode_Deopt[opcode] == (STORE_ATTR));
- JUMP_TO_PREDICTED(STORE_ATTR);
+ JUMP_TO_PREDICTED(TRACING_STORE_ATTR);
}
}
PyObject *old_value = ep->me_value;
if (true) {
UPDATE_MISS_STATS(STORE_ATTR);
assert(_PyOpcode_Deopt[opcode] == (STORE_ATTR));
- JUMP_TO_PREDICTED(STORE_ATTR);
+ JUMP_TO_PREDICTED(TRACING_STORE_ATTR);
}
}
_PyFrame_SetStackPointer(frame, stack_pointer);
}
TRACING_TARGET(STORE_DEREF) {
+ assert(IS_JIT_TRACING());
#if _Py_TAIL_CALL_INTERP
int opcode = STORE_DEREF;
(void)(opcode);
}
TRACING_TARGET(STORE_FAST) {
+ assert(IS_JIT_TRACING());
#if _Py_TAIL_CALL_INTERP
int opcode = STORE_FAST;
(void)(opcode);
}
TRACING_TARGET(STORE_FAST_LOAD_FAST) {
+ assert(IS_JIT_TRACING());
#if _Py_TAIL_CALL_INTERP
int opcode = STORE_FAST_LOAD_FAST;
(void)(opcode);
}
TRACING_TARGET(STORE_FAST_STORE_FAST) {
+ assert(IS_JIT_TRACING());
#if _Py_TAIL_CALL_INTERP
int opcode = STORE_FAST_STORE_FAST;
(void)(opcode);
}
TRACING_TARGET(STORE_GLOBAL) {
+ assert(IS_JIT_TRACING());
#if _Py_TAIL_CALL_INTERP
int opcode = STORE_GLOBAL;
(void)(opcode);
}
TRACING_TARGET(STORE_NAME) {
+ assert(IS_JIT_TRACING());
#if _Py_TAIL_CALL_INTERP
int opcode = STORE_NAME;
(void)(opcode);
}
TRACING_TARGET(STORE_SLICE) {
+ assert(IS_JIT_TRACING());
#if _Py_TAIL_CALL_INTERP
int opcode = STORE_SLICE;
(void)(opcode);
}
TRACING_TARGET(STORE_SUBSCR) {
+ assert(IS_JIT_TRACING());
#if _Py_TAIL_CALL_INTERP
int opcode = STORE_SUBSCR;
(void)(opcode);
}
TRACING_TARGET(STORE_SUBSCR_DICT) {
+ assert(IS_JIT_TRACING());
#if _Py_TAIL_CALL_INTERP
int opcode = STORE_SUBSCR_DICT;
(void)(opcode);
if (!PyDict_CheckExact(o)) {
UPDATE_MISS_STATS(STORE_SUBSCR);
assert(_PyOpcode_Deopt[opcode] == (STORE_SUBSCR));
- JUMP_TO_PREDICTED(STORE_SUBSCR);
+ JUMP_TO_PREDICTED(TRACING_STORE_SUBSCR);
}
}
/* Skip 1 cache entry */
}
TRACING_TARGET(STORE_SUBSCR_LIST_INT) {
+ assert(IS_JIT_TRACING());
#if _Py_TAIL_CALL_INTERP
int opcode = STORE_SUBSCR_LIST_INT;
(void)(opcode);
if (!_PyLong_CheckExactAndCompact(value_o)) {
UPDATE_MISS_STATS(STORE_SUBSCR);
assert(_PyOpcode_Deopt[opcode] == (STORE_SUBSCR));
- JUMP_TO_PREDICTED(STORE_SUBSCR);
+ JUMP_TO_PREDICTED(TRACING_STORE_SUBSCR);
}
}
// _GUARD_NOS_LIST
if (!PyList_CheckExact(o)) {
UPDATE_MISS_STATS(STORE_SUBSCR);
assert(_PyOpcode_Deopt[opcode] == (STORE_SUBSCR));
- JUMP_TO_PREDICTED(STORE_SUBSCR);
+ JUMP_TO_PREDICTED(TRACING_STORE_SUBSCR);
}
}
/* Skip 1 cache entry */
if (!_PyLong_IsNonNegativeCompact((PyLongObject *)sub)) {
UPDATE_MISS_STATS(STORE_SUBSCR);
assert(_PyOpcode_Deopt[opcode] == (STORE_SUBSCR));
- JUMP_TO_PREDICTED(STORE_SUBSCR);
+ JUMP_TO_PREDICTED(TRACING_STORE_SUBSCR);
}
Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0];
if (!LOCK_OBJECT(list)) {
UPDATE_MISS_STATS(STORE_SUBSCR);
assert(_PyOpcode_Deopt[opcode] == (STORE_SUBSCR));
- JUMP_TO_PREDICTED(STORE_SUBSCR);
+ JUMP_TO_PREDICTED(TRACING_STORE_SUBSCR);
}
if (index >= PyList_GET_SIZE(list)) {
UNLOCK_OBJECT(list);
if (true) {
UPDATE_MISS_STATS(STORE_SUBSCR);
assert(_PyOpcode_Deopt[opcode] == (STORE_SUBSCR));
- JUMP_TO_PREDICTED(STORE_SUBSCR);
+ JUMP_TO_PREDICTED(TRACING_STORE_SUBSCR);
}
}
STAT_INC(STORE_SUBSCR, hit);
}
TRACING_TARGET(SWAP) {
+ assert(IS_JIT_TRACING());
#if _Py_TAIL_CALL_INTERP
int opcode = SWAP;
(void)(opcode);
}
TRACING_TARGET(TO_BOOL) {
+ assert(IS_JIT_TRACING());
#if _Py_TAIL_CALL_INTERP
int opcode = TO_BOOL;
(void)(opcode);
}
TRACING_TARGET(TO_BOOL_ALWAYS_TRUE) {
+ assert(IS_JIT_TRACING());
#if _Py_TAIL_CALL_INTERP
int opcode = TO_BOOL_ALWAYS_TRUE;
(void)(opcode);
if (FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version) {
UPDATE_MISS_STATS(TO_BOOL);
assert(_PyOpcode_Deopt[opcode] == (TO_BOOL));
- JUMP_TO_PREDICTED(TO_BOOL);
+ JUMP_TO_PREDICTED(TRACING_TO_BOOL);
}
}
// _REPLACE_WITH_TRUE
}
TRACING_TARGET(TO_BOOL_BOOL) {
+ assert(IS_JIT_TRACING());
#if _Py_TAIL_CALL_INTERP
int opcode = TO_BOOL_BOOL;
(void)(opcode);
if (!PyStackRef_BoolCheck(value)) {
UPDATE_MISS_STATS(TO_BOOL);
assert(_PyOpcode_Deopt[opcode] == (TO_BOOL));
- JUMP_TO_PREDICTED(TO_BOOL);
+ JUMP_TO_PREDICTED(TRACING_TO_BOOL);
}
STAT_INC(TO_BOOL, hit);
TRACING_DISPATCH();
}
TRACING_TARGET(TO_BOOL_INT) {
+ assert(IS_JIT_TRACING());
#if _Py_TAIL_CALL_INTERP
int opcode = TO_BOOL_INT;
(void)(opcode);
if (!PyLong_CheckExact(value_o)) {
UPDATE_MISS_STATS(TO_BOOL);
assert(_PyOpcode_Deopt[opcode] == (TO_BOOL));
- JUMP_TO_PREDICTED(TO_BOOL);
+ JUMP_TO_PREDICTED(TRACING_TO_BOOL);
}
STAT_INC(TO_BOOL, hit);
if (_PyLong_IsZero((PyLongObject *)value_o)) {
}
TRACING_TARGET(TO_BOOL_LIST) {
+ assert(IS_JIT_TRACING());
#if _Py_TAIL_CALL_INTERP
int opcode = TO_BOOL_LIST;
(void)(opcode);
if (!PyList_CheckExact(o)) {
UPDATE_MISS_STATS(TO_BOOL);
assert(_PyOpcode_Deopt[opcode] == (TO_BOOL));
- JUMP_TO_PREDICTED(TO_BOOL);
+ JUMP_TO_PREDICTED(TRACING_TO_BOOL);
}
}
/* Skip 1 cache entry */
}
TRACING_TARGET(TO_BOOL_NONE) {
+ assert(IS_JIT_TRACING());
#if _Py_TAIL_CALL_INTERP
int opcode = TO_BOOL_NONE;
(void)(opcode);
if (!PyStackRef_IsNone(value)) {
UPDATE_MISS_STATS(TO_BOOL);
assert(_PyOpcode_Deopt[opcode] == (TO_BOOL));
- JUMP_TO_PREDICTED(TO_BOOL);
+ JUMP_TO_PREDICTED(TRACING_TO_BOOL);
}
STAT_INC(TO_BOOL, hit);
res = PyStackRef_False;
}
TRACING_TARGET(TO_BOOL_STR) {
+ assert(IS_JIT_TRACING());
#if _Py_TAIL_CALL_INTERP
int opcode = TO_BOOL_STR;
(void)(opcode);
if (!PyUnicode_CheckExact(value_o)) {
UPDATE_MISS_STATS(TO_BOOL);
assert(_PyOpcode_Deopt[opcode] == (TO_BOOL));
- JUMP_TO_PREDICTED(TO_BOOL);
+ JUMP_TO_PREDICTED(TRACING_TO_BOOL);
}
}
/* Skip 1 cache entry */
}
TRACING_TARGET(UNARY_INVERT) {
+ assert(IS_JIT_TRACING());
#if _Py_TAIL_CALL_INTERP
int opcode = UNARY_INVERT;
(void)(opcode);
}
TRACING_TARGET(UNARY_NEGATIVE) {
+ assert(IS_JIT_TRACING());
#if _Py_TAIL_CALL_INTERP
int opcode = UNARY_NEGATIVE;
(void)(opcode);
}
TRACING_TARGET(UNARY_NOT) {
+ assert(IS_JIT_TRACING());
#if _Py_TAIL_CALL_INTERP
int opcode = UNARY_NOT;
(void)(opcode);
}
TRACING_TARGET(UNPACK_EX) {
+ assert(IS_JIT_TRACING());
#if _Py_TAIL_CALL_INTERP
int opcode = UNPACK_EX;
(void)(opcode);
}
TRACING_TARGET(UNPACK_SEQUENCE) {
+ assert(IS_JIT_TRACING());
#if _Py_TAIL_CALL_INTERP
int opcode = UNPACK_SEQUENCE;
(void)(opcode);
}
TRACING_TARGET(UNPACK_SEQUENCE_LIST) {
+ assert(IS_JIT_TRACING());
#if _Py_TAIL_CALL_INTERP
int opcode = UNPACK_SEQUENCE_LIST;
(void)(opcode);
if (!PyList_CheckExact(o)) {
UPDATE_MISS_STATS(UNPACK_SEQUENCE);
assert(_PyOpcode_Deopt[opcode] == (UNPACK_SEQUENCE));
- JUMP_TO_PREDICTED(UNPACK_SEQUENCE);
+ JUMP_TO_PREDICTED(TRACING_UNPACK_SEQUENCE);
}
}
/* Skip 1 cache entry */
if (!LOCK_OBJECT(seq_o)) {
UPDATE_MISS_STATS(UNPACK_SEQUENCE);
assert(_PyOpcode_Deopt[opcode] == (UNPACK_SEQUENCE));
- JUMP_TO_PREDICTED(UNPACK_SEQUENCE);
+ JUMP_TO_PREDICTED(TRACING_UNPACK_SEQUENCE);
}
if (PyList_GET_SIZE(seq_o) != oparg) {
UNLOCK_OBJECT(seq_o);
if (true) {
UPDATE_MISS_STATS(UNPACK_SEQUENCE);
assert(_PyOpcode_Deopt[opcode] == (UNPACK_SEQUENCE));
- JUMP_TO_PREDICTED(UNPACK_SEQUENCE);
+ JUMP_TO_PREDICTED(TRACING_UNPACK_SEQUENCE);
}
}
STAT_INC(UNPACK_SEQUENCE, hit);
}
TRACING_TARGET(UNPACK_SEQUENCE_TUPLE) {
+ assert(IS_JIT_TRACING());
#if _Py_TAIL_CALL_INTERP
int opcode = UNPACK_SEQUENCE_TUPLE;
(void)(opcode);
if (!PyTuple_CheckExact(o)) {
UPDATE_MISS_STATS(UNPACK_SEQUENCE);
assert(_PyOpcode_Deopt[opcode] == (UNPACK_SEQUENCE));
- JUMP_TO_PREDICTED(UNPACK_SEQUENCE);
+ JUMP_TO_PREDICTED(TRACING_UNPACK_SEQUENCE);
}
}
/* Skip 1 cache entry */
if (PyTuple_GET_SIZE(seq_o) != oparg) {
UPDATE_MISS_STATS(UNPACK_SEQUENCE);
assert(_PyOpcode_Deopt[opcode] == (UNPACK_SEQUENCE));
- JUMP_TO_PREDICTED(UNPACK_SEQUENCE);
+ JUMP_TO_PREDICTED(TRACING_UNPACK_SEQUENCE);
}
STAT_INC(UNPACK_SEQUENCE, hit);
PyObject **items = _PyTuple_ITEMS(seq_o);
}
TRACING_TARGET(UNPACK_SEQUENCE_TWO_TUPLE) {
+ assert(IS_JIT_TRACING());
#if _Py_TAIL_CALL_INTERP
int opcode = UNPACK_SEQUENCE_TWO_TUPLE;
(void)(opcode);
if (!PyTuple_CheckExact(o)) {
UPDATE_MISS_STATS(UNPACK_SEQUENCE);
assert(_PyOpcode_Deopt[opcode] == (UNPACK_SEQUENCE));
- JUMP_TO_PREDICTED(UNPACK_SEQUENCE);
+ JUMP_TO_PREDICTED(TRACING_UNPACK_SEQUENCE);
}
}
/* Skip 1 cache entry */
if (PyTuple_GET_SIZE(seq_o) != 2) {
UPDATE_MISS_STATS(UNPACK_SEQUENCE);
assert(_PyOpcode_Deopt[opcode] == (UNPACK_SEQUENCE));
- JUMP_TO_PREDICTED(UNPACK_SEQUENCE);
+ JUMP_TO_PREDICTED(TRACING_UNPACK_SEQUENCE);
}
STAT_INC(UNPACK_SEQUENCE, hit);
val0 = PyStackRef_FromPyObjectNew(PyTuple_GET_ITEM(seq_o, 0));
}
TRACING_TARGET(WITH_EXCEPT_START) {
+ assert(IS_JIT_TRACING());
#if _Py_TAIL_CALL_INTERP
int opcode = WITH_EXCEPT_START;
(void)(opcode);
}
TRACING_TARGET(YIELD_VALUE) {
+ assert(IS_JIT_TRACING());
#if _Py_TAIL_CALL_INTERP
int opcode = YIELD_VALUE;
(void)(opcode);
projects / thirdparty / Python / cpython.git / commitdiff
