#endif
#define TIER_ONE 1
+#ifndef Py_TAIL_CALL_INTERP
#if !USE_COMPUTED_GOTOS
dispatch_opcode:
switch (opcode)
#endif
{
+#endif /* Py_TAIL_CALL_INTERP */
/* BEGIN INSTRUCTIONS */
TARGET(BINARY_OP) {
+ #if defined(Py_TAIL_CALL_INTERP)
+ int opcode;
+ #endif
+ opcode = BINARY_OP;
+ (void)(opcode);
frame->instr_ptr = next_instr;
next_instr += 6;
INSTRUCTION_STATS(BINARY_OP);
PyStackRef_CLOSE(lhs);
PyStackRef_CLOSE(rhs);
if (res_o == NULL) {
- goto pop_2_error;
+ JUMP_TO_LABEL(pop_2_error);
}
res = PyStackRef_FromPyObjectSteal(res_o);
}
}
TARGET(BINARY_OP_ADD_FLOAT) {
+ #if defined(Py_TAIL_CALL_INTERP)
+ int opcode;
+ #endif
+ opcode = BINARY_OP_ADD_FLOAT;
+ (void)(opcode);
+ _Py_CODEUNIT* const this_instr = next_instr;
+ (void)this_instr;
frame->instr_ptr = next_instr;
next_instr += 6;
INSTRUCTION_STATS(BINARY_OP_ADD_FLOAT);
left = stack_pointer[-2];
PyObject *left_o = PyStackRef_AsPyObjectBorrow(left);
PyObject *right_o = PyStackRef_AsPyObjectBorrow(right);
- DEOPT_IF(!PyFloat_CheckExact(left_o), BINARY_OP);
- DEOPT_IF(!PyFloat_CheckExact(right_o), BINARY_OP);
+ if (!PyFloat_CheckExact(left_o)) {
+ UPDATE_MISS_STATS(BINARY_OP);
+ assert(_PyOpcode_Deopt[opcode] == (BINARY_OP));
+ JUMP_TO_PREDICTED(BINARY_OP);
+ }
+ if (!PyFloat_CheckExact(right_o)) {
+ UPDATE_MISS_STATS(BINARY_OP);
+ assert(_PyOpcode_Deopt[opcode] == (BINARY_OP));
+ JUMP_TO_PREDICTED(BINARY_OP);
+ }
}
/* Skip 5 cache entries */
// _BINARY_OP_ADD_FLOAT
((PyFloatObject *)right_o)->ob_fval;
PyObject *res_o = _PyFloat_FromDouble_ConsumeInputs(left, right, dres);
if (res_o == NULL) {
- goto pop_2_error;
+ JUMP_TO_LABEL(pop_2_error);
}
res = PyStackRef_FromPyObjectSteal(res_o);
}
}
TARGET(BINARY_OP_ADD_INT) {
+ #if defined(Py_TAIL_CALL_INTERP)
+ int opcode;
+ #endif
+ opcode = BINARY_OP_ADD_INT;
+ (void)(opcode);
+ _Py_CODEUNIT* const this_instr = next_instr;
+ (void)this_instr;
frame->instr_ptr = next_instr;
next_instr += 6;
INSTRUCTION_STATS(BINARY_OP_ADD_INT);
left = stack_pointer[-2];
PyObject *left_o = PyStackRef_AsPyObjectBorrow(left);
PyObject *right_o = PyStackRef_AsPyObjectBorrow(right);
- DEOPT_IF(!PyLong_CheckExact(left_o), BINARY_OP);
- DEOPT_IF(!PyLong_CheckExact(right_o), BINARY_OP);
+ if (!PyLong_CheckExact(left_o)) {
+ UPDATE_MISS_STATS(BINARY_OP);
+ assert(_PyOpcode_Deopt[opcode] == (BINARY_OP));
+ JUMP_TO_PREDICTED(BINARY_OP);
+ }
+ if (!PyLong_CheckExact(right_o)) {
+ UPDATE_MISS_STATS(BINARY_OP);
+ assert(_PyOpcode_Deopt[opcode] == (BINARY_OP));
+ JUMP_TO_PREDICTED(BINARY_OP);
+ }
}
/* Skip 5 cache entries */
// _BINARY_OP_ADD_INT
PyStackRef_CLOSE_SPECIALIZED(right, _PyLong_ExactDealloc);
PyStackRef_CLOSE_SPECIALIZED(left, _PyLong_ExactDealloc);
if (res_o == NULL) {
- goto pop_2_error;
+ JUMP_TO_LABEL(pop_2_error);
}
res = PyStackRef_FromPyObjectSteal(res_o);
}
}
TARGET(BINARY_OP_ADD_UNICODE) {
+ #if defined(Py_TAIL_CALL_INTERP)
+ int opcode;
+ #endif
+ opcode = BINARY_OP_ADD_UNICODE;
+ (void)(opcode);
+ _Py_CODEUNIT* const this_instr = next_instr;
+ (void)this_instr;
frame->instr_ptr = next_instr;
next_instr += 6;
INSTRUCTION_STATS(BINARY_OP_ADD_UNICODE);
left = stack_pointer[-2];
PyObject *left_o = PyStackRef_AsPyObjectBorrow(left);
PyObject *right_o = PyStackRef_AsPyObjectBorrow(right);
- DEOPT_IF(!PyUnicode_CheckExact(left_o), BINARY_OP);
- DEOPT_IF(!PyUnicode_CheckExact(right_o), BINARY_OP);
+ if (!PyUnicode_CheckExact(left_o)) {
+ UPDATE_MISS_STATS(BINARY_OP);
+ assert(_PyOpcode_Deopt[opcode] == (BINARY_OP));
+ JUMP_TO_PREDICTED(BINARY_OP);
+ }
+ if (!PyUnicode_CheckExact(right_o)) {
+ UPDATE_MISS_STATS(BINARY_OP);
+ assert(_PyOpcode_Deopt[opcode] == (BINARY_OP));
+ JUMP_TO_PREDICTED(BINARY_OP);
+ }
}
/* Skip 5 cache entries */
// _BINARY_OP_ADD_UNICODE
PyStackRef_CLOSE_SPECIALIZED(right, _PyUnicode_ExactDealloc);
PyStackRef_CLOSE_SPECIALIZED(left, _PyUnicode_ExactDealloc);
if (res_o == NULL) {
- goto pop_2_error;
+ JUMP_TO_LABEL(pop_2_error);
}
res = PyStackRef_FromPyObjectSteal(res_o);
}
}
TARGET(BINARY_OP_EXTEND) {
- _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr;
+ #if defined(Py_TAIL_CALL_INTERP)
+ int opcode;
+ #endif
+ opcode = BINARY_OP_EXTEND;
+ (void)(opcode);
+ _Py_CODEUNIT* const this_instr = next_instr;
(void)this_instr;
+ frame->instr_ptr = next_instr;
next_instr += 6;
INSTRUCTION_STATS(BINARY_OP_EXTEND);
static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5, "incorrect cache size");
_PyFrame_SetStackPointer(frame, stack_pointer);
int res = d->guard(left_o, right_o);
stack_pointer = _PyFrame_GetStackPointer(frame);
- DEOPT_IF(!res, BINARY_OP);
+ if (!res) {
+ UPDATE_MISS_STATS(BINARY_OP);
+ assert(_PyOpcode_Deopt[opcode] == (BINARY_OP));
+ JUMP_TO_PREDICTED(BINARY_OP);
+ }
}
/* Skip -4 cache entry */
// _BINARY_OP_EXTEND
}
TARGET(BINARY_OP_INPLACE_ADD_UNICODE) {
+ #if defined(Py_TAIL_CALL_INTERP)
+ int opcode;
+ #endif
+ opcode = BINARY_OP_INPLACE_ADD_UNICODE;
+ (void)(opcode);
+ _Py_CODEUNIT* const this_instr = next_instr;
+ (void)this_instr;
frame->instr_ptr = next_instr;
next_instr += 6;
INSTRUCTION_STATS(BINARY_OP_INPLACE_ADD_UNICODE);
left = stack_pointer[-2];
PyObject *left_o = PyStackRef_AsPyObjectBorrow(left);
PyObject *right_o = PyStackRef_AsPyObjectBorrow(right);
- DEOPT_IF(!PyUnicode_CheckExact(left_o), BINARY_OP);
- DEOPT_IF(!PyUnicode_CheckExact(right_o), BINARY_OP);
+ if (!PyUnicode_CheckExact(left_o)) {
+ UPDATE_MISS_STATS(BINARY_OP);
+ assert(_PyOpcode_Deopt[opcode] == (BINARY_OP));
+ JUMP_TO_PREDICTED(BINARY_OP);
+ }
+ if (!PyUnicode_CheckExact(right_o)) {
+ UPDATE_MISS_STATS(BINARY_OP);
+ assert(_PyOpcode_Deopt[opcode] == (BINARY_OP));
+ JUMP_TO_PREDICTED(BINARY_OP);
+ }
}
/* Skip 5 cache entries */
// _BINARY_OP_INPLACE_ADD_UNICODE
next_oparg = CURRENT_OPERAND0();
#endif
_PyStackRef *target_local = &GETLOCAL(next_oparg);
- DEOPT_IF(PyStackRef_AsPyObjectBorrow(*target_local) != left_o, BINARY_OP);
+ if (PyStackRef_AsPyObjectBorrow(*target_local) != left_o) {
+ UPDATE_MISS_STATS(BINARY_OP);
+ assert(_PyOpcode_Deopt[opcode] == (BINARY_OP));
+ JUMP_TO_PREDICTED(BINARY_OP);
+ }
STAT_INC(BINARY_OP, hit);
/* Handle `left = left + right` or `left += right` for str.
*
*target_local = PyStackRef_FromPyObjectSteal(temp);
PyStackRef_CLOSE_SPECIALIZED(right, _PyUnicode_ExactDealloc);
if (PyStackRef_IsNull(*target_local)) {
- goto pop_2_error;
+ JUMP_TO_LABEL(pop_2_error);
}
#if TIER_ONE
// The STORE_FAST is already done. This is done here in tier one,
}
TARGET(BINARY_OP_MULTIPLY_FLOAT) {
+ #if defined(Py_TAIL_CALL_INTERP)
+ int opcode;
+ #endif
+ opcode = BINARY_OP_MULTIPLY_FLOAT;
+ (void)(opcode);
+ _Py_CODEUNIT* const this_instr = next_instr;
+ (void)this_instr;
frame->instr_ptr = next_instr;
next_instr += 6;
INSTRUCTION_STATS(BINARY_OP_MULTIPLY_FLOAT);
left = stack_pointer[-2];
PyObject *left_o = PyStackRef_AsPyObjectBorrow(left);
PyObject *right_o = PyStackRef_AsPyObjectBorrow(right);
- DEOPT_IF(!PyFloat_CheckExact(left_o), BINARY_OP);
- DEOPT_IF(!PyFloat_CheckExact(right_o), BINARY_OP);
+ if (!PyFloat_CheckExact(left_o)) {
+ UPDATE_MISS_STATS(BINARY_OP);
+ assert(_PyOpcode_Deopt[opcode] == (BINARY_OP));
+ JUMP_TO_PREDICTED(BINARY_OP);
+ }
+ if (!PyFloat_CheckExact(right_o)) {
+ UPDATE_MISS_STATS(BINARY_OP);
+ assert(_PyOpcode_Deopt[opcode] == (BINARY_OP));
+ JUMP_TO_PREDICTED(BINARY_OP);
+ }
}
/* Skip 5 cache entries */
// _BINARY_OP_MULTIPLY_FLOAT
((PyFloatObject *)right_o)->ob_fval;
PyObject *res_o = _PyFloat_FromDouble_ConsumeInputs(left, right, dres);
if (res_o == NULL) {
- goto pop_2_error;
+ JUMP_TO_LABEL(pop_2_error);
}
res = PyStackRef_FromPyObjectSteal(res_o);
}
}
TARGET(BINARY_OP_MULTIPLY_INT) {
+ #if defined(Py_TAIL_CALL_INTERP)
+ int opcode;
+ #endif
+ opcode = BINARY_OP_MULTIPLY_INT;
+ (void)(opcode);
+ _Py_CODEUNIT* const this_instr = next_instr;
+ (void)this_instr;
frame->instr_ptr = next_instr;
next_instr += 6;
INSTRUCTION_STATS(BINARY_OP_MULTIPLY_INT);
left = stack_pointer[-2];
PyObject *left_o = PyStackRef_AsPyObjectBorrow(left);
PyObject *right_o = PyStackRef_AsPyObjectBorrow(right);
- DEOPT_IF(!PyLong_CheckExact(left_o), BINARY_OP);
- DEOPT_IF(!PyLong_CheckExact(right_o), BINARY_OP);
+ if (!PyLong_CheckExact(left_o)) {
+ UPDATE_MISS_STATS(BINARY_OP);
+ assert(_PyOpcode_Deopt[opcode] == (BINARY_OP));
+ JUMP_TO_PREDICTED(BINARY_OP);
+ }
+ if (!PyLong_CheckExact(right_o)) {
+ UPDATE_MISS_STATS(BINARY_OP);
+ assert(_PyOpcode_Deopt[opcode] == (BINARY_OP));
+ JUMP_TO_PREDICTED(BINARY_OP);
+ }
}
/* Skip 5 cache entries */
// _BINARY_OP_MULTIPLY_INT
PyStackRef_CLOSE_SPECIALIZED(right, _PyLong_ExactDealloc);
PyStackRef_CLOSE_SPECIALIZED(left, _PyLong_ExactDealloc);
if (res_o == NULL) {
- goto pop_2_error;
+ JUMP_TO_LABEL(pop_2_error);
}
res = PyStackRef_FromPyObjectSteal(res_o);
}
}
TARGET(BINARY_OP_SUBTRACT_FLOAT) {
+ #if defined(Py_TAIL_CALL_INTERP)
+ int opcode;
+ #endif
+ opcode = BINARY_OP_SUBTRACT_FLOAT;
+ (void)(opcode);
+ _Py_CODEUNIT* const this_instr = next_instr;
+ (void)this_instr;
frame->instr_ptr = next_instr;
next_instr += 6;
INSTRUCTION_STATS(BINARY_OP_SUBTRACT_FLOAT);
left = stack_pointer[-2];
PyObject *left_o = PyStackRef_AsPyObjectBorrow(left);
PyObject *right_o = PyStackRef_AsPyObjectBorrow(right);
- DEOPT_IF(!PyFloat_CheckExact(left_o), BINARY_OP);
- DEOPT_IF(!PyFloat_CheckExact(right_o), BINARY_OP);
+ if (!PyFloat_CheckExact(left_o)) {
+ UPDATE_MISS_STATS(BINARY_OP);
+ assert(_PyOpcode_Deopt[opcode] == (BINARY_OP));
+ JUMP_TO_PREDICTED(BINARY_OP);
+ }
+ if (!PyFloat_CheckExact(right_o)) {
+ UPDATE_MISS_STATS(BINARY_OP);
+ assert(_PyOpcode_Deopt[opcode] == (BINARY_OP));
+ JUMP_TO_PREDICTED(BINARY_OP);
+ }
}
/* Skip 5 cache entries */
// _BINARY_OP_SUBTRACT_FLOAT
((PyFloatObject *)right_o)->ob_fval;
PyObject *res_o = _PyFloat_FromDouble_ConsumeInputs(left, right, dres);
if (res_o == NULL) {
- goto pop_2_error;
+ JUMP_TO_LABEL(pop_2_error);
}
res = PyStackRef_FromPyObjectSteal(res_o);
}
}
TARGET(BINARY_OP_SUBTRACT_INT) {
+ #if defined(Py_TAIL_CALL_INTERP)
+ int opcode;
+ #endif
+ opcode = BINARY_OP_SUBTRACT_INT;
+ (void)(opcode);
+ _Py_CODEUNIT* const this_instr = next_instr;
+ (void)this_instr;
frame->instr_ptr = next_instr;
next_instr += 6;
INSTRUCTION_STATS(BINARY_OP_SUBTRACT_INT);
left = stack_pointer[-2];
PyObject *left_o = PyStackRef_AsPyObjectBorrow(left);
PyObject *right_o = PyStackRef_AsPyObjectBorrow(right);
- DEOPT_IF(!PyLong_CheckExact(left_o), BINARY_OP);
- DEOPT_IF(!PyLong_CheckExact(right_o), BINARY_OP);
+ if (!PyLong_CheckExact(left_o)) {
+ UPDATE_MISS_STATS(BINARY_OP);
+ assert(_PyOpcode_Deopt[opcode] == (BINARY_OP));
+ JUMP_TO_PREDICTED(BINARY_OP);
+ }
+ if (!PyLong_CheckExact(right_o)) {
+ UPDATE_MISS_STATS(BINARY_OP);
+ assert(_PyOpcode_Deopt[opcode] == (BINARY_OP));
+ JUMP_TO_PREDICTED(BINARY_OP);
+ }
}
/* Skip 5 cache entries */
// _BINARY_OP_SUBTRACT_INT
PyStackRef_CLOSE_SPECIALIZED(right, _PyLong_ExactDealloc);
PyStackRef_CLOSE_SPECIALIZED(left, _PyLong_ExactDealloc);
if (res_o == NULL) {
- goto pop_2_error;
+ JUMP_TO_LABEL(pop_2_error);
}
res = PyStackRef_FromPyObjectSteal(res_o);
}
}
TARGET(BINARY_SLICE) {
+ #if defined(Py_TAIL_CALL_INTERP)
+ int opcode;
+ #endif
+ opcode = BINARY_SLICE;
+ (void)(opcode);
frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(BINARY_SLICE);
PyStackRef_CLOSE(container);
stack_pointer = _PyFrame_GetStackPointer(frame);
if (res_o == NULL) {
- goto error;
+ JUMP_TO_LABEL(error);
}
res = PyStackRef_FromPyObjectSteal(res_o);
}
}
TARGET(BINARY_SUBSCR) {
+ #if defined(Py_TAIL_CALL_INTERP)
+ int opcode;
+ #endif
+ opcode = BINARY_SUBSCR;
+ (void)(opcode);
frame->instr_ptr = next_instr;
next_instr += 2;
INSTRUCTION_STATS(BINARY_SUBSCR);
PyStackRef_CLOSE(container);
PyStackRef_CLOSE(sub);
if (res_o == NULL) {
- goto pop_2_error;
+ JUMP_TO_LABEL(pop_2_error);
}
res = PyStackRef_FromPyObjectSteal(res_o);
}
}
TARGET(BINARY_SUBSCR_DICT) {
+ #if defined(Py_TAIL_CALL_INTERP)
+ int opcode;
+ #endif
+ opcode = BINARY_SUBSCR_DICT;
+ (void)(opcode);
+ _Py_CODEUNIT* const this_instr = next_instr;
+ (void)this_instr;
frame->instr_ptr = next_instr;
next_instr += 2;
INSTRUCTION_STATS(BINARY_SUBSCR_DICT);
dict_st = stack_pointer[-2];
PyObject *sub = PyStackRef_AsPyObjectBorrow(sub_st);
PyObject *dict = PyStackRef_AsPyObjectBorrow(dict_st);
- DEOPT_IF(!PyDict_CheckExact(dict), BINARY_SUBSCR);
+ if (!PyDict_CheckExact(dict)) {
+ UPDATE_MISS_STATS(BINARY_SUBSCR);
+ assert(_PyOpcode_Deopt[opcode] == (BINARY_SUBSCR));
+ JUMP_TO_PREDICTED(BINARY_SUBSCR);
+ }
STAT_INC(BINARY_SUBSCR, hit);
PyObject *res_o;
_PyFrame_SetStackPointer(frame, stack_pointer);
PyStackRef_CLOSE(dict_st);
PyStackRef_CLOSE(sub_st);
if (rc <= 0) {
- goto pop_2_error;
+ JUMP_TO_LABEL(pop_2_error);
}
// not found or error
res = PyStackRef_FromPyObjectSteal(res_o);
}
TARGET(BINARY_SUBSCR_GETITEM) {
+ #if defined(Py_TAIL_CALL_INTERP)
+ int opcode;
+ #endif
+ opcode = BINARY_SUBSCR_GETITEM;
+ (void)(opcode);
+ _Py_CODEUNIT* const this_instr = next_instr;
+ (void)this_instr;
frame->instr_ptr = next_instr;
next_instr += 2;
INSTRUCTION_STATS(BINARY_SUBSCR_GETITEM);
/* Skip 1 cache entry */
// _CHECK_PEP_523
{
- DEOPT_IF(tstate->interp->eval_frame, BINARY_SUBSCR);
+ if (tstate->interp->eval_frame) {
+ UPDATE_MISS_STATS(BINARY_SUBSCR);
+ assert(_PyOpcode_Deopt[opcode] == (BINARY_SUBSCR));
+ JUMP_TO_PREDICTED(BINARY_SUBSCR);
+ }
}
// _BINARY_SUBSCR_CHECK_FUNC
{
container = stack_pointer[-2];
PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(container));
- DEOPT_IF(!PyType_HasFeature(tp, Py_TPFLAGS_HEAPTYPE), BINARY_SUBSCR);
+ if (!PyType_HasFeature(tp, Py_TPFLAGS_HEAPTYPE)) {
+ UPDATE_MISS_STATS(BINARY_SUBSCR);
+ assert(_PyOpcode_Deopt[opcode] == (BINARY_SUBSCR));
+ JUMP_TO_PREDICTED(BINARY_SUBSCR);
+ }
PyHeapTypeObject *ht = (PyHeapTypeObject *)tp;
PyObject *getitem_o = FT_ATOMIC_LOAD_PTR_ACQUIRE(ht->_spec_cache.getitem);
- DEOPT_IF(getitem_o == NULL, BINARY_SUBSCR);
+ if (getitem_o == NULL) {
+ UPDATE_MISS_STATS(BINARY_SUBSCR);
+ assert(_PyOpcode_Deopt[opcode] == (BINARY_SUBSCR));
+ JUMP_TO_PREDICTED(BINARY_SUBSCR);
+ }
assert(PyFunction_Check(getitem_o));
uint32_t cached_version = FT_ATOMIC_LOAD_UINT32_RELAXED(ht->_spec_cache.getitem_version);
- DEOPT_IF(((PyFunctionObject *)getitem_o)->func_version != cached_version, BINARY_SUBSCR);
+ if (((PyFunctionObject *)getitem_o)->func_version != cached_version) {
+ UPDATE_MISS_STATS(BINARY_SUBSCR);
+ assert(_PyOpcode_Deopt[opcode] == (BINARY_SUBSCR));
+ JUMP_TO_PREDICTED(BINARY_SUBSCR);
+ }
PyCodeObject *code = (PyCodeObject *)PyFunction_GET_CODE(getitem_o);
assert(code->co_argcount == 2);
- DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize), BINARY_SUBSCR);
+ if (!_PyThreadState_HasStackSpace(tstate, code->co_framesize)) {
+ UPDATE_MISS_STATS(BINARY_SUBSCR);
+ assert(_PyOpcode_Deopt[opcode] == (BINARY_SUBSCR));
+ JUMP_TO_PREDICTED(BINARY_SUBSCR);
+ }
getitem = PyStackRef_FromPyObjectNew(getitem_o);
STAT_INC(BINARY_SUBSCR, hit);
}
}
TARGET(BINARY_SUBSCR_LIST_INT) {
+ #if defined(Py_TAIL_CALL_INTERP)
+ int opcode;
+ #endif
+ opcode = BINARY_SUBSCR_LIST_INT;
+ (void)(opcode);
+ _Py_CODEUNIT* const this_instr = next_instr;
+ (void)this_instr;
frame->instr_ptr = next_instr;
next_instr += 2;
INSTRUCTION_STATS(BINARY_SUBSCR_LIST_INT);
list_st = stack_pointer[-2];
PyObject *sub = PyStackRef_AsPyObjectBorrow(sub_st);
PyObject *list = PyStackRef_AsPyObjectBorrow(list_st);
- DEOPT_IF(!PyLong_CheckExact(sub), BINARY_SUBSCR);
- DEOPT_IF(!PyList_CheckExact(list), BINARY_SUBSCR);
+ if (!PyLong_CheckExact(sub)) {
+ UPDATE_MISS_STATS(BINARY_SUBSCR);
+ assert(_PyOpcode_Deopt[opcode] == (BINARY_SUBSCR));
+ JUMP_TO_PREDICTED(BINARY_SUBSCR);
+ }
+ if (!PyList_CheckExact(list)) {
+ UPDATE_MISS_STATS(BINARY_SUBSCR);
+ assert(_PyOpcode_Deopt[opcode] == (BINARY_SUBSCR));
+ JUMP_TO_PREDICTED(BINARY_SUBSCR);
+ }
// Deopt unless 0 <= sub < PyList_Size(list)
- DEOPT_IF(!_PyLong_IsNonNegativeCompact((PyLongObject *)sub), BINARY_SUBSCR);
+ if (!_PyLong_IsNonNegativeCompact((PyLongObject *)sub)) {
+ UPDATE_MISS_STATS(BINARY_SUBSCR);
+ assert(_PyOpcode_Deopt[opcode] == (BINARY_SUBSCR));
+ JUMP_TO_PREDICTED(BINARY_SUBSCR);
+ }
Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0];
#ifdef Py_GIL_DISABLED
_PyFrame_SetStackPointer(frame, stack_pointer);
PyObject *res_o = _PyList_GetItemRef((PyListObject*)list, index);
stack_pointer = _PyFrame_GetStackPointer(frame);
- DEOPT_IF(res_o == NULL, BINARY_SUBSCR);
+ if (res_o == NULL) {
+ UPDATE_MISS_STATS(BINARY_SUBSCR);
+ assert(_PyOpcode_Deopt[opcode] == (BINARY_SUBSCR));
+ JUMP_TO_PREDICTED(BINARY_SUBSCR);
+ }
STAT_INC(BINARY_SUBSCR, hit);
#else
- DEOPT_IF(index >= PyList_GET_SIZE(list), BINARY_SUBSCR);
+ if (index >= PyList_GET_SIZE(list)) {
+ UPDATE_MISS_STATS(BINARY_SUBSCR);
+ assert(_PyOpcode_Deopt[opcode] == (BINARY_SUBSCR));
+ JUMP_TO_PREDICTED(BINARY_SUBSCR);
+ }
STAT_INC(BINARY_SUBSCR, hit);
PyObject *res_o = PyList_GET_ITEM(list, index);
assert(res_o != NULL);
}
TARGET(BINARY_SUBSCR_STR_INT) {
+ #if defined(Py_TAIL_CALL_INTERP)
+ int opcode;
+ #endif
+ opcode = BINARY_SUBSCR_STR_INT;
+ (void)(opcode);
+ _Py_CODEUNIT* const this_instr = next_instr;
+ (void)this_instr;
frame->instr_ptr = next_instr;
next_instr += 2;
INSTRUCTION_STATS(BINARY_SUBSCR_STR_INT);
str_st = stack_pointer[-2];
PyObject *sub = PyStackRef_AsPyObjectBorrow(sub_st);
PyObject *str = PyStackRef_AsPyObjectBorrow(str_st);
- DEOPT_IF(!PyLong_CheckExact(sub), BINARY_SUBSCR);
- DEOPT_IF(!PyUnicode_CheckExact(str), BINARY_SUBSCR);
- DEOPT_IF(!_PyLong_IsNonNegativeCompact((PyLongObject *)sub), BINARY_SUBSCR);
+ if (!PyLong_CheckExact(sub)) {
+ UPDATE_MISS_STATS(BINARY_SUBSCR);
+ assert(_PyOpcode_Deopt[opcode] == (BINARY_SUBSCR));
+ JUMP_TO_PREDICTED(BINARY_SUBSCR);
+ }
+ if (!PyUnicode_CheckExact(str)) {
+ UPDATE_MISS_STATS(BINARY_SUBSCR);
+ assert(_PyOpcode_Deopt[opcode] == (BINARY_SUBSCR));
+ JUMP_TO_PREDICTED(BINARY_SUBSCR);
+ }
+ if (!_PyLong_IsNonNegativeCompact((PyLongObject *)sub)) {
+ UPDATE_MISS_STATS(BINARY_SUBSCR);
+ assert(_PyOpcode_Deopt[opcode] == (BINARY_SUBSCR));
+ JUMP_TO_PREDICTED(BINARY_SUBSCR);
+ }
Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0];
- DEOPT_IF(PyUnicode_GET_LENGTH(str) <= index, BINARY_SUBSCR);
+ if (PyUnicode_GET_LENGTH(str) <= index) {
+ UPDATE_MISS_STATS(BINARY_SUBSCR);
+ assert(_PyOpcode_Deopt[opcode] == (BINARY_SUBSCR));
+ JUMP_TO_PREDICTED(BINARY_SUBSCR);
+ }
// Specialize for reading an ASCII character from any string:
Py_UCS4 c = PyUnicode_READ_CHAR(str, index);
- DEOPT_IF(Py_ARRAY_LENGTH(_Py_SINGLETON(strings).ascii) <= c, BINARY_SUBSCR);
+ if (Py_ARRAY_LENGTH(_Py_SINGLETON(strings).ascii) <= c) {
+ UPDATE_MISS_STATS(BINARY_SUBSCR);
+ assert(_PyOpcode_Deopt[opcode] == (BINARY_SUBSCR));
+ JUMP_TO_PREDICTED(BINARY_SUBSCR);
+ }
STAT_INC(BINARY_SUBSCR, hit);
PyObject *res_o = (PyObject*)&_Py_SINGLETON(strings).ascii[c];
PyStackRef_CLOSE_SPECIALIZED(sub_st, _PyLong_ExactDealloc);
}
TARGET(BINARY_SUBSCR_TUPLE_INT) {
+ #if defined(Py_TAIL_CALL_INTERP)
+ int opcode;
+ #endif
+ opcode = BINARY_SUBSCR_TUPLE_INT;
+ (void)(opcode);
+ _Py_CODEUNIT* const this_instr = next_instr;
+ (void)this_instr;
frame->instr_ptr = next_instr;
next_instr += 2;
INSTRUCTION_STATS(BINARY_SUBSCR_TUPLE_INT);
tuple_st = stack_pointer[-2];
PyObject *sub = PyStackRef_AsPyObjectBorrow(sub_st);
PyObject *tuple = PyStackRef_AsPyObjectBorrow(tuple_st);
- DEOPT_IF(!PyLong_CheckExact(sub), BINARY_SUBSCR);
- DEOPT_IF(!PyTuple_CheckExact(tuple), BINARY_SUBSCR);
+ if (!PyLong_CheckExact(sub)) {
+ UPDATE_MISS_STATS(BINARY_SUBSCR);
+ assert(_PyOpcode_Deopt[opcode] == (BINARY_SUBSCR));
+ JUMP_TO_PREDICTED(BINARY_SUBSCR);
+ }
+ if (!PyTuple_CheckExact(tuple)) {
+ UPDATE_MISS_STATS(BINARY_SUBSCR);
+ assert(_PyOpcode_Deopt[opcode] == (BINARY_SUBSCR));
+ JUMP_TO_PREDICTED(BINARY_SUBSCR);
+ }
// Deopt unless 0 <= sub < PyTuple_Size(list)
- DEOPT_IF(!_PyLong_IsNonNegativeCompact((PyLongObject *)sub), BINARY_SUBSCR);
+ if (!_PyLong_IsNonNegativeCompact((PyLongObject *)sub)) {
+ UPDATE_MISS_STATS(BINARY_SUBSCR);
+ assert(_PyOpcode_Deopt[opcode] == (BINARY_SUBSCR));
+ JUMP_TO_PREDICTED(BINARY_SUBSCR);
+ }
Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0];
- DEOPT_IF(index >= PyTuple_GET_SIZE(tuple), BINARY_SUBSCR);
+ if (index >= PyTuple_GET_SIZE(tuple)) {
+ UPDATE_MISS_STATS(BINARY_SUBSCR);
+ assert(_PyOpcode_Deopt[opcode] == (BINARY_SUBSCR));
+ JUMP_TO_PREDICTED(BINARY_SUBSCR);
+ }
STAT_INC(BINARY_SUBSCR, hit);
PyObject *res_o = PyTuple_GET_ITEM(tuple, index);
assert(res_o != NULL);
}
TARGET(BUILD_LIST) {
+ #if defined(Py_TAIL_CALL_INTERP)
+ int opcode;
+ #endif
+ opcode = BUILD_LIST;
+ (void)(opcode);
frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(BUILD_LIST);
values = &stack_pointer[-oparg];
PyObject *list_o = _PyList_FromStackRefStealOnSuccess(values, oparg);
if (list_o == NULL) {
- goto error;
+ JUMP_TO_LABEL(error);
}
list = PyStackRef_FromPyObjectSteal(list_o);
stack_pointer[-oparg] = list;
}
TARGET(BUILD_MAP) {
+ #if defined(Py_TAIL_CALL_INTERP)
+ int opcode;
+ #endif
+ opcode = BUILD_MAP;
+ (void)(opcode);
frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(BUILD_MAP);
}
stack_pointer += -oparg*2;
assert(WITHIN_STACK_BOUNDS());
- goto error;
+ JUMP_TO_LABEL(error);
}
_PyFrame_SetStackPointer(frame, stack_pointer);
PyObject *map_o = _PyDict_FromItems(
if (map_o == NULL) {
stack_pointer += -oparg*2;
assert(WITHIN_STACK_BOUNDS());
- goto error;
+ JUMP_TO_LABEL(error);
}
map = PyStackRef_FromPyObjectSteal(map_o);
stack_pointer[-oparg*2] = map;
}
TARGET(BUILD_SET) {
+ #if defined(Py_TAIL_CALL_INTERP)
+ int opcode;
+ #endif
+ opcode = BUILD_SET;
+ (void)(opcode);
frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(BUILD_SET);
}
stack_pointer += -oparg;
assert(WITHIN_STACK_BOUNDS());
- goto error;
+ JUMP_TO_LABEL(error);
}
int err = 0;
for (int i = 0; i < oparg; i++) {
_PyFrame_SetStackPointer(frame, stack_pointer);
Py_DECREF(set_o);
stack_pointer = _PyFrame_GetStackPointer(frame);
- goto error;
+ JUMP_TO_LABEL(error);
}
set = PyStackRef_FromPyObjectSteal(set_o);
stack_pointer[-oparg] = set;
}
TARGET(BUILD_SLICE) {
+ #if defined(Py_TAIL_CALL_INTERP)
+ int opcode;
+ #endif
+ opcode = BUILD_SLICE;
+ (void)(opcode);
frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(BUILD_SLICE);
if (slice_o == NULL) {
stack_pointer += -oparg;
assert(WITHIN_STACK_BOUNDS());
- goto error;
+ JUMP_TO_LABEL(error);
}
slice = PyStackRef_FromPyObjectSteal(slice_o);
stack_pointer[-oparg] = slice;
}
TARGET(BUILD_STRING) {
+ #if defined(Py_TAIL_CALL_INTERP)
+ int opcode;
+ #endif
+ opcode = BUILD_STRING;
+ (void)(opcode);
frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(BUILD_STRING);
}
stack_pointer += -oparg;
assert(WITHIN_STACK_BOUNDS());
- goto error;
+ JUMP_TO_LABEL(error);
}
PyObject *str_o = _PyUnicode_JoinArray(&_Py_STR(empty), pieces_o, oparg);
STACKREFS_TO_PYOBJECTS_CLEANUP(pieces_o);
if (str_o == NULL) {
stack_pointer += -oparg;
assert(WITHIN_STACK_BOUNDS());
- goto error;
+ JUMP_TO_LABEL(error);
}
str = PyStackRef_FromPyObjectSteal(str_o);
stack_pointer[-oparg] = str;
}
TARGET(BUILD_TUPLE) {
+ #if defined(Py_TAIL_CALL_INTERP)
+ int opcode;
+ #endif
+ opcode = BUILD_TUPLE;
+ (void)(opcode);
frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(BUILD_TUPLE);
values = &stack_pointer[-oparg];
PyObject *tup_o = _PyTuple_FromStackRefStealOnSuccess(values, oparg);
if (tup_o == NULL) {
- goto error;
+ JUMP_TO_LABEL(error);
}
tup = PyStackRef_FromPyObjectSteal(tup_o);
stack_pointer[-oparg] = tup;
}
TARGET(CACHE) {
+ #if defined(Py_TAIL_CALL_INTERP)
+ int opcode;
+ #endif
+ opcode = CACHE;
+ (void)(opcode);
frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(CACHE);
}
TARGET(CALL) {
+ #if defined(Py_TAIL_CALL_INTERP)
+ int opcode;
+ #endif
+ opcode = CALL;
+ (void)(opcode);
frame->instr_ptr = next_instr;
next_instr += 4;
INSTRUCTION_STATS(CALL);
PREDICTED_CALL:;
_Py_CODEUNIT* const this_instr = next_instr - 4;
(void)this_instr;
- opcode = CALL;
_PyStackRef *callable;
_PyStackRef *self_or_null;
_PyStackRef *args;
// The frame has stolen all the arguments from the stack,
// so there is no need to clean them up.
if (new_frame == NULL) {
- goto error;
+ JUMP_TO_LABEL(error);
}
frame->return_offset = 4 ;
DISPATCH_INLINED(new_frame);
}
stack_pointer += -2 - oparg;
assert(WITHIN_STACK_BOUNDS());
- goto error;
+ JUMP_TO_LABEL(error);
}
_PyFrame_SetStackPointer(frame, stack_pointer);
PyObject *res_o = PyObject_Vectorcall(
if (res_o == NULL) {
stack_pointer += -2 - oparg;
assert(WITHIN_STACK_BOUNDS());
- goto error;
+ JUMP_TO_LABEL(error);
}
res = PyStackRef_FromPyObjectSteal(res_o);
}
int err = _Py_HandlePending(tstate);
stack_pointer = _PyFrame_GetStackPointer(frame);
if (err != 0) {
- goto error;
+ JUMP_TO_LABEL(error);
}
stack_pointer += 1 + oparg;
assert(WITHIN_STACK_BOUNDS());
}
TARGET(CALL_ALLOC_AND_ENTER_INIT) {
- _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr;
+ #if defined(Py_TAIL_CALL_INTERP)
+ int opcode;
+ #endif
+ opcode = CALL_ALLOC_AND_ENTER_INIT;
+ (void)(opcode);
+ _Py_CODEUNIT* const this_instr = next_instr;
(void)this_instr;
+ frame->instr_ptr = next_instr;
next_instr += 4;
INSTRUCTION_STATS(CALL_ALLOC_AND_ENTER_INIT);
static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size");
/* Skip 1 cache entry */
// _CHECK_PEP_523
{
- DEOPT_IF(tstate->interp->eval_frame, CALL);
+ if (tstate->interp->eval_frame) {
+ UPDATE_MISS_STATS(CALL);
+ assert(_PyOpcode_Deopt[opcode] == (CALL));
+ JUMP_TO_PREDICTED(CALL);
+ }
}
// _CHECK_AND_ALLOCATE_OBJECT
{
self = &stack_pointer[-1 - oparg];
uint32_t type_version = read_u32(&this_instr[2].cache);
PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]);
- DEOPT_IF(!PyStackRef_IsNull(null[0]), CALL);
- DEOPT_IF(!PyType_Check(callable_o), CALL);
+ if (!PyStackRef_IsNull(null[0])) {
+ UPDATE_MISS_STATS(CALL);
+ assert(_PyOpcode_Deopt[opcode] == (CALL));
+ JUMP_TO_PREDICTED(CALL);
+ }
+ if (!PyType_Check(callable_o)) {
+ UPDATE_MISS_STATS(CALL);
+ assert(_PyOpcode_Deopt[opcode] == (CALL));
+ JUMP_TO_PREDICTED(CALL);
+ }
PyTypeObject *tp = (PyTypeObject *)callable_o;
- DEOPT_IF(FT_ATOMIC_LOAD_UINT32_RELAXED(tp->tp_version_tag) != type_version, CALL);
+ if (FT_ATOMIC_LOAD_UINT32_RELAXED(tp->tp_version_tag) != type_version) {
+ UPDATE_MISS_STATS(CALL);
+ assert(_PyOpcode_Deopt[opcode] == (CALL));
+ JUMP_TO_PREDICTED(CALL);
+ }
assert(tp->tp_new == PyBaseObject_Type.tp_new);
assert(tp->tp_flags & Py_TPFLAGS_HEAPTYPE);
assert(tp->tp_alloc == PyType_GenericAlloc);
PyHeapTypeObject *cls = (PyHeapTypeObject *)callable_o;
PyFunctionObject *init_func = (PyFunctionObject *)FT_ATOMIC_LOAD_PTR_ACQUIRE(cls->_spec_cache.init);
PyCodeObject *code = (PyCodeObject *)init_func->func_code;
- DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize + _Py_InitCleanup.co_framesize), CALL);
+ if (!_PyThreadState_HasStackSpace(tstate, code->co_framesize + _Py_InitCleanup.co_framesize)) {
+ UPDATE_MISS_STATS(CALL);
+ assert(_PyOpcode_Deopt[opcode] == (CALL));
+ JUMP_TO_PREDICTED(CALL);
+ }
STAT_INC(CALL, hit);
_PyFrame_SetStackPointer(frame, stack_pointer);
PyObject *self_o = PyType_GenericAlloc(tp, 0);
stack_pointer = _PyFrame_GetStackPointer(frame);
if (self_o == NULL) {
- goto error;
+ JUMP_TO_LABEL(error);
}
self[0] = PyStackRef_FromPyObjectSteal(self_o);
_PyStackRef temp = callable[0];
assert(WITHIN_STACK_BOUNDS());
if (temp == NULL) {
_PyEval_FrameClearAndPop(tstate, shim);
- goto error;
+ JUMP_TO_LABEL(error);
}
init_frame = temp;
frame->return_offset = 1 + INLINE_CACHE_ENTRIES_CALL;
}
TARGET(CALL_BOUND_METHOD_EXACT_ARGS) {
- _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr;
+ #if defined(Py_TAIL_CALL_INTERP)
+ int opcode;
+ #endif
+ opcode = CALL_BOUND_METHOD_EXACT_ARGS;
+ (void)(opcode);
+ _Py_CODEUNIT* const this_instr = next_instr;
(void)this_instr;
+ frame->instr_ptr = next_instr;
next_instr += 4;
INSTRUCTION_STATS(CALL_BOUND_METHOD_EXACT_ARGS);
static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size");
/* Skip 1 cache entry */
// _CHECK_PEP_523
{
- DEOPT_IF(tstate->interp->eval_frame, CALL);
+ if (tstate->interp->eval_frame) {
+ UPDATE_MISS_STATS(CALL);
+ assert(_PyOpcode_Deopt[opcode] == (CALL));
+ JUMP_TO_PREDICTED(CALL);
+ }
}
// _CHECK_CALL_BOUND_METHOD_EXACT_ARGS
{
null = &stack_pointer[-1 - oparg];
callable = &stack_pointer[-2 - oparg];
- DEOPT_IF(!PyStackRef_IsNull(null[0]), CALL);
- DEOPT_IF(Py_TYPE(PyStackRef_AsPyObjectBorrow(callable[0])) != &PyMethod_Type, CALL);
+ if (!PyStackRef_IsNull(null[0])) {
+ UPDATE_MISS_STATS(CALL);
+ assert(_PyOpcode_Deopt[opcode] == (CALL));
+ JUMP_TO_PREDICTED(CALL);
+ }
+ if (Py_TYPE(PyStackRef_AsPyObjectBorrow(callable[0])) != &PyMethod_Type) {
+ UPDATE_MISS_STATS(CALL);
+ assert(_PyOpcode_Deopt[opcode] == (CALL));
+ JUMP_TO_PREDICTED(CALL);
+ }
}
// _INIT_CALL_BOUND_METHOD_EXACT_ARGS
{
{
uint32_t func_version = read_u32(&this_instr[2].cache);
PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]);
- DEOPT_IF(!PyFunction_Check(callable_o), CALL);
+ if (!PyFunction_Check(callable_o)) {
+ UPDATE_MISS_STATS(CALL);
+ assert(_PyOpcode_Deopt[opcode] == (CALL));
+ JUMP_TO_PREDICTED(CALL);
+ }
PyFunctionObject *func = (PyFunctionObject *)callable_o;
- DEOPT_IF(func->func_version != func_version, CALL);
+ if (func->func_version != func_version) {
+ UPDATE_MISS_STATS(CALL);
+ assert(_PyOpcode_Deopt[opcode] == (CALL));
+ JUMP_TO_PREDICTED(CALL);
+ }
}
// _CHECK_FUNCTION_EXACT_ARGS
{
assert(PyFunction_Check(callable_o));
PyFunctionObject *func = (PyFunctionObject *)callable_o;
PyCodeObject *code = (PyCodeObject *)func->func_code;
- DEOPT_IF(code->co_argcount != oparg + (!PyStackRef_IsNull(self_or_null[0])), CALL);
+ if (code->co_argcount != oparg + (!PyStackRef_IsNull(self_or_null[0]))) {
+ UPDATE_MISS_STATS(CALL);
+ assert(_PyOpcode_Deopt[opcode] == (CALL));
+ JUMP_TO_PREDICTED(CALL);
+ }
}
// _CHECK_STACK_SPACE
{
PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]);
PyFunctionObject *func = (PyFunctionObject *)callable_o;
PyCodeObject *code = (PyCodeObject *)func->func_code;
- DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize), CALL);
- DEOPT_IF(tstate->py_recursion_remaining <= 1, CALL);
+ if (!_PyThreadState_HasStackSpace(tstate, code->co_framesize)) {
+ UPDATE_MISS_STATS(CALL);
+ assert(_PyOpcode_Deopt[opcode] == (CALL));
+ JUMP_TO_PREDICTED(CALL);
+ }
+ if (tstate->py_recursion_remaining <= 1) {
+ UPDATE_MISS_STATS(CALL);
+ assert(_PyOpcode_Deopt[opcode] == (CALL));
+ JUMP_TO_PREDICTED(CALL);
+ }
}
// _INIT_CALL_PY_EXACT_ARGS
{
}
TARGET(CALL_BOUND_METHOD_GENERAL) {
- _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr;
+ #if defined(Py_TAIL_CALL_INTERP)
+ int opcode;
+ #endif
+ opcode = CALL_BOUND_METHOD_GENERAL;
+ (void)(opcode);
+ _Py_CODEUNIT* const this_instr = next_instr;
(void)this_instr;
+ frame->instr_ptr = next_instr;
next_instr += 4;
INSTRUCTION_STATS(CALL_BOUND_METHOD_GENERAL);
static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size");
/* Skip 1 cache entry */
// _CHECK_PEP_523
{
- DEOPT_IF(tstate->interp->eval_frame, CALL);
+ if (tstate->interp->eval_frame) {
+ UPDATE_MISS_STATS(CALL);
+ assert(_PyOpcode_Deopt[opcode] == (CALL));
+ JUMP_TO_PREDICTED(CALL);
+ }
}
// _CHECK_METHOD_VERSION
{
callable = &stack_pointer[-2 - oparg];
uint32_t func_version = read_u32(&this_instr[2].cache);
PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]);
- DEOPT_IF(Py_TYPE(callable_o) != &PyMethod_Type, CALL);
+ if (Py_TYPE(callable_o) != &PyMethod_Type) {
+ UPDATE_MISS_STATS(CALL);
+ assert(_PyOpcode_Deopt[opcode] == (CALL));
+ JUMP_TO_PREDICTED(CALL);
+ }
PyObject *func = ((PyMethodObject *)callable_o)->im_func;
- DEOPT_IF(!PyFunction_Check(func), CALL);
- DEOPT_IF(((PyFunctionObject *)func)->func_version != func_version, CALL);
- DEOPT_IF(!PyStackRef_IsNull(null[0]), CALL);
+ if (!PyFunction_Check(func)) {
+ UPDATE_MISS_STATS(CALL);
+ assert(_PyOpcode_Deopt[opcode] == (CALL));
+ JUMP_TO_PREDICTED(CALL);
+ }
+ if (((PyFunctionObject *)func)->func_version != func_version) {
+ UPDATE_MISS_STATS(CALL);
+ assert(_PyOpcode_Deopt[opcode] == (CALL));
+ JUMP_TO_PREDICTED(CALL);
+ }
+ if (!PyStackRef_IsNull(null[0])) {
+ UPDATE_MISS_STATS(CALL);
+ assert(_PyOpcode_Deopt[opcode] == (CALL));
+ JUMP_TO_PREDICTED(CALL);
+ }
}
// _EXPAND_METHOD
{
stack_pointer += -2 - oparg;
assert(WITHIN_STACK_BOUNDS());
if (temp == NULL) {
- goto error;
+ JUMP_TO_LABEL(error);
}
new_frame = temp;
}
}
TARGET(CALL_BUILTIN_CLASS) {
+ #if defined(Py_TAIL_CALL_INTERP)
+ int opcode;
+ #endif
+ opcode = CALL_BUILTIN_CLASS;
+ (void)(opcode);
+ _Py_CODEUNIT* const this_instr = next_instr;
+ (void)this_instr;
frame->instr_ptr = next_instr;
next_instr += 4;
INSTRUCTION_STATS(CALL_BUILTIN_CLASS);
self_or_null = &stack_pointer[-1 - oparg];
callable = &stack_pointer[-2 - oparg];
PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]);
- DEOPT_IF(!PyType_Check(callable_o), CALL);
+ if (!PyType_Check(callable_o)) {
+ UPDATE_MISS_STATS(CALL);
+ assert(_PyOpcode_Deopt[opcode] == (CALL));
+ JUMP_TO_PREDICTED(CALL);
+ }
PyTypeObject *tp = (PyTypeObject *)callable_o;
int total_args = oparg;
_PyStackRef *arguments = args;
arguments--;
total_args++;
}
- DEOPT_IF(tp->tp_vectorcall == NULL, CALL);
+ if (tp->tp_vectorcall == NULL) {
+ UPDATE_MISS_STATS(CALL);
+ assert(_PyOpcode_Deopt[opcode] == (CALL));
+ JUMP_TO_PREDICTED(CALL);
+ }
STAT_INC(CALL, hit);
STACKREFS_TO_PYOBJECTS(arguments, total_args, args_o);
if (CONVERSION_FAILED(args_o)) {
}
stack_pointer += -2 - oparg;
assert(WITHIN_STACK_BOUNDS());
- goto error;
+ JUMP_TO_LABEL(error);
}
_PyFrame_SetStackPointer(frame, stack_pointer);
PyObject *res_o = tp->tp_vectorcall((PyObject *)tp, args_o, total_args, NULL);
if (res_o == NULL) {
stack_pointer += -2 - oparg;
assert(WITHIN_STACK_BOUNDS());
- goto error;
+ JUMP_TO_LABEL(error);
}
res = PyStackRef_FromPyObjectSteal(res_o);
}
int err = _Py_HandlePending(tstate);
stack_pointer = _PyFrame_GetStackPointer(frame);
if (err != 0) {
- goto error;
+ JUMP_TO_LABEL(error);
}
stack_pointer += 1 + oparg;
assert(WITHIN_STACK_BOUNDS());
}
TARGET(CALL_BUILTIN_FAST) {
+ #if defined(Py_TAIL_CALL_INTERP)
+ int opcode;
+ #endif
+ opcode = CALL_BUILTIN_FAST;
+ (void)(opcode);
+ _Py_CODEUNIT* const this_instr = next_instr;
+ (void)this_instr;
frame->instr_ptr = next_instr;
next_instr += 4;
INSTRUCTION_STATS(CALL_BUILTIN_FAST);
arguments--;
total_args++;
}
- DEOPT_IF(!PyCFunction_CheckExact(callable_o), CALL);
- DEOPT_IF(PyCFunction_GET_FLAGS(callable_o) != METH_FASTCALL, CALL);
+ if (!PyCFunction_CheckExact(callable_o)) {
+ UPDATE_MISS_STATS(CALL);
+ assert(_PyOpcode_Deopt[opcode] == (CALL));
+ JUMP_TO_PREDICTED(CALL);
+ }
+ if (PyCFunction_GET_FLAGS(callable_o) != METH_FASTCALL) {
+ UPDATE_MISS_STATS(CALL);
+ assert(_PyOpcode_Deopt[opcode] == (CALL));
+ JUMP_TO_PREDICTED(CALL);
+ }
STAT_INC(CALL, hit);
PyCFunction cfunc = PyCFunction_GET_FUNCTION(callable_o);
/* res = func(self, args, nargs) */
}
stack_pointer += -2 - oparg;
assert(WITHIN_STACK_BOUNDS());
- goto error;
+ JUMP_TO_LABEL(error);
}
_PyFrame_SetStackPointer(frame, stack_pointer);
PyObject *res_o = ((PyCFunctionFast)(void(*)(void))cfunc)(
if (res_o == NULL) {
stack_pointer += -2 - oparg;
assert(WITHIN_STACK_BOUNDS());
- goto error;
+ JUMP_TO_LABEL(error);
}
res = PyStackRef_FromPyObjectSteal(res_o);
}
int err = _Py_HandlePending(tstate);
stack_pointer = _PyFrame_GetStackPointer(frame);
if (err != 0) {
- goto error;
+ JUMP_TO_LABEL(error);
}
stack_pointer += 1 + oparg;
assert(WITHIN_STACK_BOUNDS());
}
TARGET(CALL_BUILTIN_FAST_WITH_KEYWORDS) {
+ #if defined(Py_TAIL_CALL_INTERP)
+ int opcode;
+ #endif
+ opcode = CALL_BUILTIN_FAST_WITH_KEYWORDS;
+ (void)(opcode);
+ _Py_CODEUNIT* const this_instr = next_instr;
+ (void)this_instr;
frame->instr_ptr = next_instr;
next_instr += 4;
INSTRUCTION_STATS(CALL_BUILTIN_FAST_WITH_KEYWORDS);
arguments--;
total_args++;
}
- DEOPT_IF(!PyCFunction_CheckExact(callable_o), CALL);
- DEOPT_IF(PyCFunction_GET_FLAGS(callable_o) != (METH_FASTCALL | METH_KEYWORDS), CALL);
+ if (!PyCFunction_CheckExact(callable_o)) {
+ UPDATE_MISS_STATS(CALL);
+ assert(_PyOpcode_Deopt[opcode] == (CALL));
+ JUMP_TO_PREDICTED(CALL);
+ }
+ if (PyCFunction_GET_FLAGS(callable_o) != (METH_FASTCALL | METH_KEYWORDS)) {
+ UPDATE_MISS_STATS(CALL);
+ assert(_PyOpcode_Deopt[opcode] == (CALL));
+ JUMP_TO_PREDICTED(CALL);
+ }
STAT_INC(CALL, hit);
/* res = func(self, arguments, nargs, kwnames) */
_PyFrame_SetStackPointer(frame, stack_pointer);
}
stack_pointer += -2 - oparg;
assert(WITHIN_STACK_BOUNDS());
- goto error;
+ JUMP_TO_LABEL(error);
}
_PyFrame_SetStackPointer(frame, stack_pointer);
PyObject *res_o = cfunc(PyCFunction_GET_SELF(callable_o), args_o, total_args, NULL);
if (res_o == NULL) {
stack_pointer += -2 - oparg;
assert(WITHIN_STACK_BOUNDS());
- goto error;
+ JUMP_TO_LABEL(error);
}
res = PyStackRef_FromPyObjectSteal(res_o);
}
int err = _Py_HandlePending(tstate);
stack_pointer = _PyFrame_GetStackPointer(frame);
if (err != 0) {
- goto error;
+ JUMP_TO_LABEL(error);
}
stack_pointer += 1 + oparg;
assert(WITHIN_STACK_BOUNDS());
}
TARGET(CALL_BUILTIN_O) {
+ #if defined(Py_TAIL_CALL_INTERP)
+ int opcode;
+ #endif
+ opcode = CALL_BUILTIN_O;
+ (void)(opcode);
+ _Py_CODEUNIT* const this_instr = next_instr;
+ (void)this_instr;
frame->instr_ptr = next_instr;
next_instr += 4;
INSTRUCTION_STATS(CALL_BUILTIN_O);
args--;
total_args++;
}
- DEOPT_IF(total_args != 1, CALL);
- DEOPT_IF(!PyCFunction_CheckExact(callable_o), CALL);
- DEOPT_IF(PyCFunction_GET_FLAGS(callable_o) != METH_O, CALL);
+ if (total_args != 1) {
+ UPDATE_MISS_STATS(CALL);
+ assert(_PyOpcode_Deopt[opcode] == (CALL));
+ JUMP_TO_PREDICTED(CALL);
+ }
+ if (!PyCFunction_CheckExact(callable_o)) {
+ UPDATE_MISS_STATS(CALL);
+ assert(_PyOpcode_Deopt[opcode] == (CALL));
+ JUMP_TO_PREDICTED(CALL);
+ }
+ if (PyCFunction_GET_FLAGS(callable_o) != METH_O) {
+ UPDATE_MISS_STATS(CALL);
+ assert(_PyOpcode_Deopt[opcode] == (CALL));
+ JUMP_TO_PREDICTED(CALL);
+ }
// CPython promises to check all non-vectorcall function calls.
- DEOPT_IF(tstate->c_recursion_remaining <= 0, CALL);
+ if (tstate->c_recursion_remaining <= 0) {
+ UPDATE_MISS_STATS(CALL);
+ assert(_PyOpcode_Deopt[opcode] == (CALL));
+ JUMP_TO_PREDICTED(CALL);
+ }
STAT_INC(CALL, hit);
PyCFunction cfunc = PyCFunction_GET_FUNCTION(callable_o);
_PyStackRef arg = args[0];
PyStackRef_CLOSE(callable[0]);
stack_pointer = _PyFrame_GetStackPointer(frame);
if (res_o == NULL) {
- goto error;
+ JUMP_TO_LABEL(error);
}
res = PyStackRef_FromPyObjectSteal(res_o);
}
int err = _Py_HandlePending(tstate);
stack_pointer = _PyFrame_GetStackPointer(frame);
if (err != 0) {
- goto error;
+ JUMP_TO_LABEL(error);
}
stack_pointer += -1;
assert(WITHIN_STACK_BOUNDS());
}
TARGET(CALL_FUNCTION_EX) {
- _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr;
+ #if defined(Py_TAIL_CALL_INTERP)
+ int opcode;
+ #endif
+ opcode = CALL_FUNCTION_EX;
+ (void)(opcode);
+ _Py_CODEUNIT* const this_instr = next_instr;
(void)this_instr;
+ frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(CALL_FUNCTION_EX);
- opcode = CALL_FUNCTION_EX;
_PyStackRef func;
_PyStackRef callargs;
_PyStackRef kwargs_in;
int err = _Py_Check_ArgsIterable(tstate, PyStackRef_AsPyObjectBorrow(func), callargs_o);
stack_pointer = _PyFrame_GetStackPointer(frame);
if (err < 0) {
- goto error;
+ JUMP_TO_LABEL(error);
}
_PyFrame_SetStackPointer(frame, stack_pointer);
PyObject *tuple_o = PySequence_Tuple(callargs_o);
stack_pointer = _PyFrame_GetStackPointer(frame);
if (tuple_o == NULL) {
- goto error;
+ JUMP_TO_LABEL(error);
}
kwargs_out = kwargs_in;
stack_pointer += -2;
frame, this_instr, func, arg);
stack_pointer = _PyFrame_GetStackPointer(frame);
if (err) {
- goto error;
+ JUMP_TO_LABEL(error);
}
_PyFrame_SetStackPointer(frame, stack_pointer);
result_o = PyObject_Call(func, callargs, kwargs);
stack_pointer += -2;
assert(WITHIN_STACK_BOUNDS());
if (new_frame == NULL) {
- goto error;
+ JUMP_TO_LABEL(error);
}
assert( 1 == 1);
frame->return_offset = 1;
PyStackRef_CLOSE(func_st);
stack_pointer = _PyFrame_GetStackPointer(frame);
if (result_o == NULL) {
- goto error;
+ JUMP_TO_LABEL(error);
}
result = PyStackRef_FromPyObjectSteal(result_o);
}
int err = _Py_HandlePending(tstate);
stack_pointer = _PyFrame_GetStackPointer(frame);
if (err != 0) {
- goto error;
+ JUMP_TO_LABEL(error);
}
stack_pointer += -1;
assert(WITHIN_STACK_BOUNDS());
}
TARGET(CALL_INTRINSIC_1) {
+ #if defined(Py_TAIL_CALL_INTERP)
+ int opcode;
+ #endif
+ opcode = CALL_INTRINSIC_1;
+ (void)(opcode);
frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(CALL_INTRINSIC_1);
stack_pointer = _PyFrame_GetStackPointer(frame);
PyStackRef_CLOSE(value);
if (res_o == NULL) {
- goto pop_1_error;
+ JUMP_TO_LABEL(pop_1_error);
}
res = PyStackRef_FromPyObjectSteal(res_o);
stack_pointer[-1] = res;
}
TARGET(CALL_INTRINSIC_2) {
+ #if defined(Py_TAIL_CALL_INTERP)
+ int opcode;
+ #endif
+ opcode = CALL_INTRINSIC_2;
+ (void)(opcode);
frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(CALL_INTRINSIC_2);
PyStackRef_CLOSE(value2_st);
PyStackRef_CLOSE(value1_st);
if (res_o == NULL) {
- goto pop_2_error;
+ JUMP_TO_LABEL(pop_2_error);
}
res = PyStackRef_FromPyObjectSteal(res_o);
stack_pointer[-2] = res;
}
TARGET(CALL_ISINSTANCE) {
+ #if defined(Py_TAIL_CALL_INTERP)
+ int opcode;
+ #endif
+ opcode = CALL_ISINSTANCE;
+ (void)(opcode);
+ _Py_CODEUNIT* const this_instr = next_instr;
+ (void)this_instr;
frame->instr_ptr = next_instr;
next_instr += 4;
INSTRUCTION_STATS(CALL_ISINSTANCE);
arguments--;
total_args++;
}
- DEOPT_IF(total_args != 2, CALL);
+ if (total_args != 2) {
+ UPDATE_MISS_STATS(CALL);
+ assert(_PyOpcode_Deopt[opcode] == (CALL));
+ JUMP_TO_PREDICTED(CALL);
+ }
PyInterpreterState *interp = tstate->interp;
- DEOPT_IF(callable_o != interp->callable_cache.isinstance, CALL);
+ if (callable_o != interp->callable_cache.isinstance) {
+ UPDATE_MISS_STATS(CALL);
+ assert(_PyOpcode_Deopt[opcode] == (CALL));
+ JUMP_TO_PREDICTED(CALL);
+ }
STAT_INC(CALL, hit);
_PyStackRef cls_stackref = arguments[1];
_PyStackRef inst_stackref = arguments[0];
int retval = PyObject_IsInstance(PyStackRef_AsPyObjectBorrow(inst_stackref), PyStackRef_AsPyObjectBorrow(cls_stackref));
stack_pointer = _PyFrame_GetStackPointer(frame);
if (retval < 0) {
- goto error;
+ JUMP_TO_LABEL(error);
}
res = retval ? PyStackRef_True : PyStackRef_False;
assert((!PyStackRef_IsNull(res)) ^ (_PyErr_Occurred(tstate) != NULL));
}
TARGET(CALL_KW) {
+ #if defined(Py_TAIL_CALL_INTERP)
+ int opcode;
+ #endif
+ opcode = CALL_KW;
+ (void)(opcode);
frame->instr_ptr = next_instr;
next_instr += 4;
INSTRUCTION_STATS(CALL_KW);
PREDICTED_CALL_KW:;
_Py_CODEUNIT* const this_instr = next_instr - 4;
(void)this_instr;
- opcode = CALL_KW;
_PyStackRef *callable;
_PyStackRef *self_or_null;
_PyStackRef *args;
// The frame has stolen all the arguments from the stack,
// so there is no need to clean them up.
if (new_frame == NULL) {
- goto error;
+ JUMP_TO_LABEL(error);
}
assert( 4 == 1 + INLINE_CACHE_ENTRIES_CALL_KW);
frame->return_offset = 4 ;
PyStackRef_CLOSE(kwnames);
stack_pointer += -3 - oparg;
assert(WITHIN_STACK_BOUNDS());
- goto error;
+ JUMP_TO_LABEL(error);
}
stack_pointer[-1] = kwnames;
_PyFrame_SetStackPointer(frame, stack_pointer);
if (res_o == NULL) {
stack_pointer += -3 - oparg;
assert(WITHIN_STACK_BOUNDS());
- goto error;
+ JUMP_TO_LABEL(error);
}
res = PyStackRef_FromPyObjectSteal(res_o);
}
}
TARGET(CALL_KW_BOUND_METHOD) {
- _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr;
+ #if defined(Py_TAIL_CALL_INTERP)
+ int opcode;
+ #endif
+ opcode = CALL_KW_BOUND_METHOD;
+ (void)(opcode);
+ _Py_CODEUNIT* const this_instr = next_instr;
(void)this_instr;
+ frame->instr_ptr = next_instr;
next_instr += 4;
INSTRUCTION_STATS(CALL_KW_BOUND_METHOD);
static_assert(INLINE_CACHE_ENTRIES_CALL_KW == 3, "incorrect cache size");
/* Skip 1 cache entry */
// _CHECK_PEP_523
{
- DEOPT_IF(tstate->interp->eval_frame, CALL_KW);
+ if (tstate->interp->eval_frame) {
+ UPDATE_MISS_STATS(CALL_KW);
+ assert(_PyOpcode_Deopt[opcode] == (CALL_KW));
+ JUMP_TO_PREDICTED(CALL_KW);
+ }
}
// _CHECK_METHOD_VERSION_KW
{
callable = &stack_pointer[-3 - oparg];
uint32_t func_version = read_u32(&this_instr[2].cache);
PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]);
- DEOPT_IF(Py_TYPE(callable_o) != &PyMethod_Type, CALL_KW);
+ if (Py_TYPE(callable_o) != &PyMethod_Type) {
+ UPDATE_MISS_STATS(CALL_KW);
+ assert(_PyOpcode_Deopt[opcode] == (CALL_KW));
+ JUMP_TO_PREDICTED(CALL_KW);
+ }
PyObject *func = ((PyMethodObject *)callable_o)->im_func;
- DEOPT_IF(!PyFunction_Check(func), CALL_KW);
- DEOPT_IF(((PyFunctionObject *)func)->func_version != func_version, CALL_KW);
- DEOPT_IF(!PyStackRef_IsNull(null[0]), CALL_KW);
+ if (!PyFunction_Check(func)) {
+ UPDATE_MISS_STATS(CALL_KW);
+ assert(_PyOpcode_Deopt[opcode] == (CALL_KW));
+ JUMP_TO_PREDICTED(CALL_KW);
+ }
+ if (((PyFunctionObject *)func)->func_version != func_version) {
+ UPDATE_MISS_STATS(CALL_KW);
+ assert(_PyOpcode_Deopt[opcode] == (CALL_KW));
+ JUMP_TO_PREDICTED(CALL_KW);
+ }
+ if (!PyStackRef_IsNull(null[0])) {
+ UPDATE_MISS_STATS(CALL_KW);
+ assert(_PyOpcode_Deopt[opcode] == (CALL_KW));
+ JUMP_TO_PREDICTED(CALL_KW);
+ }
}
// _EXPAND_METHOD_KW
{
stack_pointer += -2 - oparg;
assert(WITHIN_STACK_BOUNDS());
if (temp == NULL) {
- goto error;
+ JUMP_TO_LABEL(error);
}
new_frame = temp;
}
}
TARGET(CALL_KW_NON_PY) {
+ #if defined(Py_TAIL_CALL_INTERP)
+ int opcode;
+ #endif
+ opcode = CALL_KW_NON_PY;
+ (void)(opcode);
+ _Py_CODEUNIT* const this_instr = next_instr;
+ (void)this_instr;
frame->instr_ptr = next_instr;
next_instr += 4;
INSTRUCTION_STATS(CALL_KW_NON_PY);
- opcode = CALL_KW_NON_PY;
static_assert(INLINE_CACHE_ENTRIES_CALL_KW == 3, "incorrect cache size");
_PyStackRef *callable;
_PyStackRef kwnames;
{
callable = &stack_pointer[-3 - oparg];
PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]);
- DEOPT_IF(PyFunction_Check(callable_o), CALL_KW);
- DEOPT_IF(Py_TYPE(callable_o) == &PyMethod_Type, CALL_KW);
+ if (PyFunction_Check(callable_o)) {
+ UPDATE_MISS_STATS(CALL_KW);
+ assert(_PyOpcode_Deopt[opcode] == (CALL_KW));
+ JUMP_TO_PREDICTED(CALL_KW);
+ }
+ if (Py_TYPE(callable_o) == &PyMethod_Type) {
+ UPDATE_MISS_STATS(CALL_KW);
+ assert(_PyOpcode_Deopt[opcode] == (CALL_KW));
+ JUMP_TO_PREDICTED(CALL_KW);
+ }
}
// _CALL_KW_NON_PY
{
PyStackRef_CLOSE(kwnames);
stack_pointer += -3 - oparg;
assert(WITHIN_STACK_BOUNDS());
- goto error;
+ JUMP_TO_LABEL(error);
}
PyObject *kwnames_o = PyStackRef_AsPyObjectBorrow(kwnames);
int positional_args = total_args - (int)PyTuple_GET_SIZE(kwnames_o);
if (res_o == NULL) {
stack_pointer += -2 - oparg;
assert(WITHIN_STACK_BOUNDS());
- goto error;
+ JUMP_TO_LABEL(error);
}
res = PyStackRef_FromPyObjectSteal(res_o);
}
int err = _Py_HandlePending(tstate);
stack_pointer = _PyFrame_GetStackPointer(frame);
if (err != 0) {
- goto error;
+ JUMP_TO_LABEL(error);
}
stack_pointer += 1 + oparg;
assert(WITHIN_STACK_BOUNDS());
}
TARGET(CALL_KW_PY) {
- _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr;
+ #if defined(Py_TAIL_CALL_INTERP)
+ int opcode;
+ #endif
+ opcode = CALL_KW_PY;
+ (void)(opcode);
+ _Py_CODEUNIT* const this_instr = next_instr;
(void)this_instr;
+ frame->instr_ptr = next_instr;
next_instr += 4;
INSTRUCTION_STATS(CALL_KW_PY);
static_assert(INLINE_CACHE_ENTRIES_CALL_KW == 3, "incorrect cache size");
/* Skip 1 cache entry */
// _CHECK_PEP_523
{
- DEOPT_IF(tstate->interp->eval_frame, CALL_KW);
+ if (tstate->interp->eval_frame) {
+ UPDATE_MISS_STATS(CALL_KW);
+ assert(_PyOpcode_Deopt[opcode] == (CALL_KW));
+ JUMP_TO_PREDICTED(CALL_KW);
+ }
}
// _CHECK_FUNCTION_VERSION_KW
{
callable = &stack_pointer[-3 - oparg];
uint32_t func_version = read_u32(&this_instr[2].cache);
PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]);
- DEOPT_IF(!PyFunction_Check(callable_o), CALL_KW);
+ if (!PyFunction_Check(callable_o)) {
+ UPDATE_MISS_STATS(CALL_KW);
+ assert(_PyOpcode_Deopt[opcode] == (CALL_KW));
+ JUMP_TO_PREDICTED(CALL_KW);
+ }
PyFunctionObject *func = (PyFunctionObject *)callable_o;
- DEOPT_IF(func->func_version != func_version, CALL_KW);
+ if (func->func_version != func_version) {
+ UPDATE_MISS_STATS(CALL_KW);
+ assert(_PyOpcode_Deopt[opcode] == (CALL_KW));
+ JUMP_TO_PREDICTED(CALL_KW);
+ }
}
// _PY_FRAME_KW
{
stack_pointer += -2 - oparg;
assert(WITHIN_STACK_BOUNDS());
if (temp == NULL) {
- goto error;
+ JUMP_TO_LABEL(error);
}
new_frame = temp;
}
}
TARGET(CALL_LEN) {
+ #if defined(Py_TAIL_CALL_INTERP)
+ int opcode;
+ #endif
+ opcode = CALL_LEN;
+ (void)(opcode);
+ _Py_CODEUNIT* const this_instr = next_instr;
+ (void)this_instr;
frame->instr_ptr = next_instr;
next_instr += 4;
INSTRUCTION_STATS(CALL_LEN);
args--;
total_args++;
}
- DEOPT_IF(total_args != 1, CALL);
+ if (total_args != 1) {
+ UPDATE_MISS_STATS(CALL);
+ assert(_PyOpcode_Deopt[opcode] == (CALL));
+ JUMP_TO_PREDICTED(CALL);
+ }
PyInterpreterState *interp = tstate->interp;
- DEOPT_IF(callable_o != interp->callable_cache.len, CALL);
+ if (callable_o != interp->callable_cache.len) {
+ UPDATE_MISS_STATS(CALL);
+ assert(_PyOpcode_Deopt[opcode] == (CALL));
+ JUMP_TO_PREDICTED(CALL);
+ }
STAT_INC(CALL, hit);
_PyStackRef arg_stackref = args[0];
PyObject *arg = PyStackRef_AsPyObjectBorrow(arg_stackref);
Py_ssize_t len_i = PyObject_Length(arg);
stack_pointer = _PyFrame_GetStackPointer(frame);
if (len_i < 0) {
- goto error;
+ JUMP_TO_LABEL(error);
}
PyObject *res_o = PyLong_FromSsize_t(len_i);
assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL));
}
TARGET(CALL_LIST_APPEND) {
+ #if defined(Py_TAIL_CALL_INTERP)
+ int opcode;
+ #endif
+ opcode = CALL_LIST_APPEND;
+ (void)(opcode);
+ _Py_CODEUNIT* const this_instr = next_instr;
+ (void)this_instr;
frame->instr_ptr = next_instr;
next_instr += 4;
INSTRUCTION_STATS(CALL_LIST_APPEND);
PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable);
PyObject *self_o = PyStackRef_AsPyObjectBorrow(self);
PyInterpreterState *interp = tstate->interp;
- DEOPT_IF(callable_o != interp->callable_cache.list_append, CALL);
+ if (callable_o != interp->callable_cache.list_append) {
+ UPDATE_MISS_STATS(CALL);
+ assert(_PyOpcode_Deopt[opcode] == (CALL));
+ JUMP_TO_PREDICTED(CALL);
+ }
assert(self_o != NULL);
- DEOPT_IF(!PyList_Check(self_o), CALL);
- DEOPT_IF(!LOCK_OBJECT(self_o), CALL);
+ if (!PyList_Check(self_o)) {
+ UPDATE_MISS_STATS(CALL);
+ assert(_PyOpcode_Deopt[opcode] == (CALL));
+ JUMP_TO_PREDICTED(CALL);
+ }
+ if (!LOCK_OBJECT(self_o)) {
+ UPDATE_MISS_STATS(CALL);
+ assert(_PyOpcode_Deopt[opcode] == (CALL));
+ JUMP_TO_PREDICTED(CALL);
+ }
STAT_INC(CALL, hit);
int err = _PyList_AppendTakeRef((PyListObject *)self_o, PyStackRef_AsPyObjectSteal(arg));
UNLOCK_OBJECT(self_o);
PyStackRef_CLOSE(callable);
stack_pointer = _PyFrame_GetStackPointer(frame);
if (err) {
- goto error;
+ JUMP_TO_LABEL(error);
}
#if TIER_ONE
// Skip the following POP_TOP. This is done here in tier one, and
}
TARGET(CALL_METHOD_DESCRIPTOR_FAST) {
+ #if defined(Py_TAIL_CALL_INTERP)
+ int opcode;
+ #endif
+ opcode = CALL_METHOD_DESCRIPTOR_FAST;
+ (void)(opcode);
+ _Py_CODEUNIT* const this_instr = next_instr;
+ (void)this_instr;
frame->instr_ptr = next_instr;
next_instr += 4;
INSTRUCTION_STATS(CALL_METHOD_DESCRIPTOR_FAST);
}
PyMethodDescrObject *method = (PyMethodDescrObject *)callable_o;
/* Builtin METH_FASTCALL methods, without keywords */
- DEOPT_IF(!Py_IS_TYPE(method, &PyMethodDescr_Type), CALL);
+ if (!Py_IS_TYPE(method, &PyMethodDescr_Type)) {
+ UPDATE_MISS_STATS(CALL);
+ assert(_PyOpcode_Deopt[opcode] == (CALL));
+ JUMP_TO_PREDICTED(CALL);
+ }
PyMethodDef *meth = method->d_method;
- DEOPT_IF(meth->ml_flags != METH_FASTCALL, CALL);
+ if (meth->ml_flags != METH_FASTCALL) {
+ UPDATE_MISS_STATS(CALL);
+ assert(_PyOpcode_Deopt[opcode] == (CALL));
+ JUMP_TO_PREDICTED(CALL);
+ }
PyObject *self = PyStackRef_AsPyObjectBorrow(arguments[0]);
- DEOPT_IF(!Py_IS_TYPE(self, method->d_common.d_type), CALL);
+ if (!Py_IS_TYPE(self, method->d_common.d_type)) {
+ UPDATE_MISS_STATS(CALL);
+ assert(_PyOpcode_Deopt[opcode] == (CALL));
+ JUMP_TO_PREDICTED(CALL);
+ }
STAT_INC(CALL, hit);
int nargs = total_args - 1;
STACKREFS_TO_PYOBJECTS(arguments, total_args, args_o);
}
stack_pointer += -2 - oparg;
assert(WITHIN_STACK_BOUNDS());
- goto error;
+ JUMP_TO_LABEL(error);
}
_PyFrame_SetStackPointer(frame, stack_pointer);
PyCFunctionFast cfunc =
if (res_o == NULL) {
stack_pointer += -2 - oparg;
assert(WITHIN_STACK_BOUNDS());
- goto error;
+ JUMP_TO_LABEL(error);
}
res = PyStackRef_FromPyObjectSteal(res_o);
}
int err = _Py_HandlePending(tstate);
stack_pointer = _PyFrame_GetStackPointer(frame);
if (err != 0) {
- goto error;
+ JUMP_TO_LABEL(error);
}
stack_pointer += 1 + oparg;
assert(WITHIN_STACK_BOUNDS());
}
TARGET(CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS) {
+ #if defined(Py_TAIL_CALL_INTERP)
+ int opcode;
+ #endif
+ opcode = CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS;
+ (void)(opcode);
+ _Py_CODEUNIT* const this_instr = next_instr;
+ (void)this_instr;
frame->instr_ptr = next_instr;
next_instr += 4;
INSTRUCTION_STATS(CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS);
total_args++;
}
PyMethodDescrObject *method = (PyMethodDescrObject *)callable_o;
- DEOPT_IF(!Py_IS_TYPE(method, &PyMethodDescr_Type), CALL);
+ if (!Py_IS_TYPE(method, &PyMethodDescr_Type)) {
+ UPDATE_MISS_STATS(CALL);
+ assert(_PyOpcode_Deopt[opcode] == (CALL));
+ JUMP_TO_PREDICTED(CALL);
+ }
PyMethodDef *meth = method->d_method;
- DEOPT_IF(meth->ml_flags != (METH_FASTCALL|METH_KEYWORDS), CALL);
+ if (meth->ml_flags != (METH_FASTCALL|METH_KEYWORDS)) {
+ UPDATE_MISS_STATS(CALL);
+ assert(_PyOpcode_Deopt[opcode] == (CALL));
+ JUMP_TO_PREDICTED(CALL);
+ }
PyTypeObject *d_type = method->d_common.d_type;
PyObject *self = PyStackRef_AsPyObjectBorrow(arguments[0]);
- DEOPT_IF(!Py_IS_TYPE(self, d_type), CALL);
+ if (!Py_IS_TYPE(self, d_type)) {
+ UPDATE_MISS_STATS(CALL);
+ assert(_PyOpcode_Deopt[opcode] == (CALL));
+ JUMP_TO_PREDICTED(CALL);
+ }
STAT_INC(CALL, hit);
int nargs = total_args - 1;
STACKREFS_TO_PYOBJECTS(arguments, total_args, args_o);
}
stack_pointer += -2 - oparg;
assert(WITHIN_STACK_BOUNDS());
- goto error;
+ JUMP_TO_LABEL(error);
}
_PyFrame_SetStackPointer(frame, stack_pointer);
PyCFunctionFastWithKeywords cfunc =
if (res_o == NULL) {
stack_pointer += -2 - oparg;
assert(WITHIN_STACK_BOUNDS());
- goto error;
+ JUMP_TO_LABEL(error);
}
res = PyStackRef_FromPyObjectSteal(res_o);
}
int err = _Py_HandlePending(tstate);
stack_pointer = _PyFrame_GetStackPointer(frame);
if (err != 0) {
- goto error;
+ JUMP_TO_LABEL(error);
}
stack_pointer += 1 + oparg;
assert(WITHIN_STACK_BOUNDS());
}
TARGET(CALL_METHOD_DESCRIPTOR_NOARGS) {
+ #if defined(Py_TAIL_CALL_INTERP)
+ int opcode;
+ #endif
+ opcode = CALL_METHOD_DESCRIPTOR_NOARGS;
+ (void)(opcode);
+ _Py_CODEUNIT* const this_instr = next_instr;
+ (void)this_instr;
frame->instr_ptr = next_instr;
next_instr += 4;
INSTRUCTION_STATS(CALL_METHOD_DESCRIPTOR_NOARGS);
args--;
total_args++;
}
- DEOPT_IF(total_args != 1, CALL);
+ if (total_args != 1) {
+ UPDATE_MISS_STATS(CALL);
+ assert(_PyOpcode_Deopt[opcode] == (CALL));
+ JUMP_TO_PREDICTED(CALL);
+ }
PyMethodDescrObject *method = (PyMethodDescrObject *)callable_o;
- DEOPT_IF(!Py_IS_TYPE(method, &PyMethodDescr_Type), CALL);
+ if (!Py_IS_TYPE(method, &PyMethodDescr_Type)) {
+ UPDATE_MISS_STATS(CALL);
+ assert(_PyOpcode_Deopt[opcode] == (CALL));
+ JUMP_TO_PREDICTED(CALL);
+ }
PyMethodDef *meth = method->d_method;
_PyStackRef self_stackref = args[0];
PyObject *self = PyStackRef_AsPyObjectBorrow(self_stackref);
- DEOPT_IF(!Py_IS_TYPE(self, method->d_common.d_type), CALL);
- DEOPT_IF(meth->ml_flags != METH_NOARGS, CALL);
+ if (!Py_IS_TYPE(self, method->d_common.d_type)) {
+ UPDATE_MISS_STATS(CALL);
+ assert(_PyOpcode_Deopt[opcode] == (CALL));
+ JUMP_TO_PREDICTED(CALL);
+ }
+ if (meth->ml_flags != METH_NOARGS) {
+ UPDATE_MISS_STATS(CALL);
+ assert(_PyOpcode_Deopt[opcode] == (CALL));
+ JUMP_TO_PREDICTED(CALL);
+ }
// CPython promises to check all non-vectorcall function calls.
- DEOPT_IF(tstate->c_recursion_remaining <= 0, CALL);
+ if (tstate->c_recursion_remaining <= 0) {
+ UPDATE_MISS_STATS(CALL);
+ assert(_PyOpcode_Deopt[opcode] == (CALL));
+ JUMP_TO_PREDICTED(CALL);
+ }
STAT_INC(CALL, hit);
PyCFunction cfunc = meth->ml_meth;
_Py_EnterRecursiveCallTstateUnchecked(tstate);
PyStackRef_CLOSE(callable[0]);
stack_pointer = _PyFrame_GetStackPointer(frame);
if (res_o == NULL) {
- goto error;
+ JUMP_TO_LABEL(error);
}
res = PyStackRef_FromPyObjectSteal(res_o);
}
int err = _Py_HandlePending(tstate);
stack_pointer = _PyFrame_GetStackPointer(frame);
if (err != 0) {
- goto error;
+ JUMP_TO_LABEL(error);
}
stack_pointer += -1;
assert(WITHIN_STACK_BOUNDS());
}
TARGET(CALL_METHOD_DESCRIPTOR_O) {
+ #if defined(Py_TAIL_CALL_INTERP)
+ int opcode;
+ #endif
+ opcode = CALL_METHOD_DESCRIPTOR_O;
+ (void)(opcode);
+ _Py_CODEUNIT* const this_instr = next_instr;
+ (void)this_instr;
frame->instr_ptr = next_instr;
next_instr += 4;
INSTRUCTION_STATS(CALL_METHOD_DESCRIPTOR_O);
total_args++;
}
PyMethodDescrObject *method = (PyMethodDescrObject *)callable_o;
- DEOPT_IF(total_args != 2, CALL);
- DEOPT_IF(!Py_IS_TYPE(method, &PyMethodDescr_Type), CALL);
+ if (total_args != 2) {
+ UPDATE_MISS_STATS(CALL);
+ assert(_PyOpcode_Deopt[opcode] == (CALL));
+ JUMP_TO_PREDICTED(CALL);
+ }
+ if (!Py_IS_TYPE(method, &PyMethodDescr_Type)) {
+ UPDATE_MISS_STATS(CALL);
+ assert(_PyOpcode_Deopt[opcode] == (CALL));
+ JUMP_TO_PREDICTED(CALL);
+ }
PyMethodDef *meth = method->d_method;
- DEOPT_IF(meth->ml_flags != METH_O, CALL);
+ if (meth->ml_flags != METH_O) {
+ UPDATE_MISS_STATS(CALL);
+ assert(_PyOpcode_Deopt[opcode] == (CALL));
+ JUMP_TO_PREDICTED(CALL);
+ }
// CPython promises to check all non-vectorcall function calls.
- DEOPT_IF(tstate->c_recursion_remaining <= 0, CALL);
+ if (tstate->c_recursion_remaining <= 0) {
+ UPDATE_MISS_STATS(CALL);
+ assert(_PyOpcode_Deopt[opcode] == (CALL));
+ JUMP_TO_PREDICTED(CALL);
+ }
_PyStackRef arg_stackref = arguments[1];
_PyStackRef self_stackref = arguments[0];
- DEOPT_IF(!Py_IS_TYPE(PyStackRef_AsPyObjectBorrow(self_stackref),
- method->d_common.d_type), CALL);
+ if (!Py_IS_TYPE(PyStackRef_AsPyObjectBorrow(self_stackref),
+ method->d_common.d_type)) {
+ UPDATE_MISS_STATS(CALL);
+ assert(_PyOpcode_Deopt[opcode] == (CALL));
+ JUMP_TO_PREDICTED(CALL);
+ }
STAT_INC(CALL, hit);
PyCFunction cfunc = meth->ml_meth;
_Py_EnterRecursiveCallTstateUnchecked(tstate);
if (res_o == NULL) {
stack_pointer += -2 - oparg;
assert(WITHIN_STACK_BOUNDS());
- goto error;
+ JUMP_TO_LABEL(error);
}
res = PyStackRef_FromPyObjectSteal(res_o);
}
int err = _Py_HandlePending(tstate);
stack_pointer = _PyFrame_GetStackPointer(frame);
if (err != 0) {
- goto error;
+ JUMP_TO_LABEL(error);
}
stack_pointer += 1 + oparg;
assert(WITHIN_STACK_BOUNDS());
}
TARGET(CALL_NON_PY_GENERAL) {
+ #if defined(Py_TAIL_CALL_INTERP)
+ int opcode;
+ #endif
+ opcode = CALL_NON_PY_GENERAL;
+ (void)(opcode);
+ _Py_CODEUNIT* const this_instr = next_instr;
+ (void)this_instr;
frame->instr_ptr = next_instr;
next_instr += 4;
INSTRUCTION_STATS(CALL_NON_PY_GENERAL);
- opcode = CALL_NON_PY_GENERAL;
static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size");
_PyStackRef *callable;
_PyStackRef *self_or_null;
{
callable = &stack_pointer[-2 - oparg];
PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]);
- DEOPT_IF(PyFunction_Check(callable_o), CALL);
- DEOPT_IF(Py_TYPE(callable_o) == &PyMethod_Type, CALL);
+ if (PyFunction_Check(callable_o)) {
+ UPDATE_MISS_STATS(CALL);
+ assert(_PyOpcode_Deopt[opcode] == (CALL));
+ JUMP_TO_PREDICTED(CALL);
+ }
+ if (Py_TYPE(callable_o) == &PyMethod_Type) {
+ UPDATE_MISS_STATS(CALL);
+ assert(_PyOpcode_Deopt[opcode] == (CALL));
+ JUMP_TO_PREDICTED(CALL);
+ }
}
// _CALL_NON_PY_GENERAL
{
}
stack_pointer += -2 - oparg;
assert(WITHIN_STACK_BOUNDS());
- goto error;
+ JUMP_TO_LABEL(error);
}
_PyFrame_SetStackPointer(frame, stack_pointer);
PyObject *res_o = PyObject_Vectorcall(
if (res_o == NULL) {
stack_pointer += -2 - oparg;
assert(WITHIN_STACK_BOUNDS());
- goto error;
+ JUMP_TO_LABEL(error);
}
res = PyStackRef_FromPyObjectSteal(res_o);
}
int err = _Py_HandlePending(tstate);
stack_pointer = _PyFrame_GetStackPointer(frame);
if (err != 0) {
- goto error;
+ JUMP_TO_LABEL(error);
}
stack_pointer += 1 + oparg;
assert(WITHIN_STACK_BOUNDS());
}
TARGET(CALL_PY_EXACT_ARGS) {
- _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr;
+ #if defined(Py_TAIL_CALL_INTERP)
+ int opcode;
+ #endif
+ opcode = CALL_PY_EXACT_ARGS;
+ (void)(opcode);
+ _Py_CODEUNIT* const this_instr = next_instr;
(void)this_instr;
+ frame->instr_ptr = next_instr;
next_instr += 4;
INSTRUCTION_STATS(CALL_PY_EXACT_ARGS);
static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size");
/* Skip 1 cache entry */
// _CHECK_PEP_523
{
- DEOPT_IF(tstate->interp->eval_frame, CALL);
+ if (tstate->interp->eval_frame) {
+ UPDATE_MISS_STATS(CALL);
+ assert(_PyOpcode_Deopt[opcode] == (CALL));
+ JUMP_TO_PREDICTED(CALL);
+ }
}
// _CHECK_FUNCTION_VERSION
{
callable = &stack_pointer[-2 - oparg];
uint32_t func_version = read_u32(&this_instr[2].cache);
PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]);
- DEOPT_IF(!PyFunction_Check(callable_o), CALL);
+ if (!PyFunction_Check(callable_o)) {
+ UPDATE_MISS_STATS(CALL);
+ assert(_PyOpcode_Deopt[opcode] == (CALL));
+ JUMP_TO_PREDICTED(CALL);
+ }
PyFunctionObject *func = (PyFunctionObject *)callable_o;
- DEOPT_IF(func->func_version != func_version, CALL);
+ if (func->func_version != func_version) {
+ UPDATE_MISS_STATS(CALL);
+ assert(_PyOpcode_Deopt[opcode] == (CALL));
+ JUMP_TO_PREDICTED(CALL);
+ }
}
// _CHECK_FUNCTION_EXACT_ARGS
{
assert(PyFunction_Check(callable_o));
PyFunctionObject *func = (PyFunctionObject *)callable_o;
PyCodeObject *code = (PyCodeObject *)func->func_code;
- DEOPT_IF(code->co_argcount != oparg + (!PyStackRef_IsNull(self_or_null[0])), CALL);
+ if (code->co_argcount != oparg + (!PyStackRef_IsNull(self_or_null[0]))) {
+ UPDATE_MISS_STATS(CALL);
+ assert(_PyOpcode_Deopt[opcode] == (CALL));
+ JUMP_TO_PREDICTED(CALL);
+ }
}
// _CHECK_STACK_SPACE
{
PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]);
PyFunctionObject *func = (PyFunctionObject *)callable_o;
PyCodeObject *code = (PyCodeObject *)func->func_code;
- DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize), CALL);
- DEOPT_IF(tstate->py_recursion_remaining <= 1, CALL);
+ if (!_PyThreadState_HasStackSpace(tstate, code->co_framesize)) {
+ UPDATE_MISS_STATS(CALL);
+ assert(_PyOpcode_Deopt[opcode] == (CALL));
+ JUMP_TO_PREDICTED(CALL);
+ }
+ if (tstate->py_recursion_remaining <= 1) {
+ UPDATE_MISS_STATS(CALL);
+ assert(_PyOpcode_Deopt[opcode] == (CALL));
+ JUMP_TO_PREDICTED(CALL);
+ }
}
// _INIT_CALL_PY_EXACT_ARGS
{
}
TARGET(CALL_PY_GENERAL) {
- _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr;
+ #if defined(Py_TAIL_CALL_INTERP)
+ int opcode;
+ #endif
+ opcode = CALL_PY_GENERAL;
+ (void)(opcode);
+ _Py_CODEUNIT* const this_instr = next_instr;
(void)this_instr;
+ frame->instr_ptr = next_instr;
next_instr += 4;
INSTRUCTION_STATS(CALL_PY_GENERAL);
static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size");
/* Skip 1 cache entry */
// _CHECK_PEP_523
{
- DEOPT_IF(tstate->interp->eval_frame, CALL);
+ if (tstate->interp->eval_frame) {
+ UPDATE_MISS_STATS(CALL);
+ assert(_PyOpcode_Deopt[opcode] == (CALL));
+ JUMP_TO_PREDICTED(CALL);
+ }
}
// _CHECK_FUNCTION_VERSION
{
callable = &stack_pointer[-2 - oparg];
uint32_t func_version = read_u32(&this_instr[2].cache);
PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]);
- DEOPT_IF(!PyFunction_Check(callable_o), CALL);
+ if (!PyFunction_Check(callable_o)) {
+ UPDATE_MISS_STATS(CALL);
+ assert(_PyOpcode_Deopt[opcode] == (CALL));
+ JUMP_TO_PREDICTED(CALL);
+ }
PyFunctionObject *func = (PyFunctionObject *)callable_o;
- DEOPT_IF(func->func_version != func_version, CALL);
+ if (func->func_version != func_version) {
+ UPDATE_MISS_STATS(CALL);
+ assert(_PyOpcode_Deopt[opcode] == (CALL));
+ JUMP_TO_PREDICTED(CALL);
+ }
}
// _PY_FRAME_GENERAL
{
stack_pointer += -2 - oparg;
assert(WITHIN_STACK_BOUNDS());
if (temp == NULL) {
- goto error;
+ JUMP_TO_LABEL(error);
}
new_frame = temp;
}
}
TARGET(CALL_STR_1) {
+ #if defined(Py_TAIL_CALL_INTERP)
+ int opcode;
+ #endif
+ opcode = CALL_STR_1;
+ (void)(opcode);
+ _Py_CODEUNIT* const this_instr = next_instr;
+ (void)this_instr;
frame->instr_ptr = next_instr;
next_instr += 4;
INSTRUCTION_STATS(CALL_STR_1);
PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable);
PyObject *arg_o = PyStackRef_AsPyObjectBorrow(arg);
assert(oparg == 1);
- DEOPT_IF(!PyStackRef_IsNull(null), CALL);
- DEOPT_IF(callable_o != (PyObject *)&PyUnicode_Type, CALL);
+ if (!PyStackRef_IsNull(null)) {
+ UPDATE_MISS_STATS(CALL);
+ assert(_PyOpcode_Deopt[opcode] == (CALL));
+ JUMP_TO_PREDICTED(CALL);
+ }
+ if (callable_o != (PyObject *)&PyUnicode_Type) {
+ UPDATE_MISS_STATS(CALL);
+ assert(_PyOpcode_Deopt[opcode] == (CALL));
+ JUMP_TO_PREDICTED(CALL);
+ }
STAT_INC(CALL, hit);
_PyFrame_SetStackPointer(frame, stack_pointer);
PyObject *res_o = PyObject_Str(arg_o);
PyStackRef_CLOSE(arg);
stack_pointer = _PyFrame_GetStackPointer(frame);
if (res_o == NULL) {
- goto error;
+ JUMP_TO_LABEL(error);
}
res = PyStackRef_FromPyObjectSteal(res_o);
}
int err = _Py_HandlePending(tstate);
stack_pointer = _PyFrame_GetStackPointer(frame);
if (err != 0) {
- goto error;
+ JUMP_TO_LABEL(error);
}
stack_pointer += -1;
assert(WITHIN_STACK_BOUNDS());
}
TARGET(CALL_TUPLE_1) {
+ #if defined(Py_TAIL_CALL_INTERP)
+ int opcode;
+ #endif
+ opcode = CALL_TUPLE_1;
+ (void)(opcode);
+ _Py_CODEUNIT* const this_instr = next_instr;
+ (void)this_instr;
frame->instr_ptr = next_instr;
next_instr += 4;
INSTRUCTION_STATS(CALL_TUPLE_1);
PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable);
PyObject *arg_o = PyStackRef_AsPyObjectBorrow(arg);
assert(oparg == 1);
- DEOPT_IF(!PyStackRef_IsNull(null), CALL);
- DEOPT_IF(callable_o != (PyObject *)&PyTuple_Type, CALL);
+ if (!PyStackRef_IsNull(null)) {
+ UPDATE_MISS_STATS(CALL);
+ assert(_PyOpcode_Deopt[opcode] == (CALL));
+ JUMP_TO_PREDICTED(CALL);
+ }
+ if (callable_o != (PyObject *)&PyTuple_Type) {
+ UPDATE_MISS_STATS(CALL);
+ assert(_PyOpcode_Deopt[opcode] == (CALL));
+ JUMP_TO_PREDICTED(CALL);
+ }
STAT_INC(CALL, hit);
_PyFrame_SetStackPointer(frame, stack_pointer);
PyObject *res_o = PySequence_Tuple(arg_o);
PyStackRef_CLOSE(arg);
stack_pointer = _PyFrame_GetStackPointer(frame);
if (res_o == NULL) {
- goto error;
+ JUMP_TO_LABEL(error);
}
res = PyStackRef_FromPyObjectSteal(res_o);
}
int err = _Py_HandlePending(tstate);
stack_pointer = _PyFrame_GetStackPointer(frame);
if (err != 0) {
- goto error;
+ JUMP_TO_LABEL(error);
}
stack_pointer += -1;
assert(WITHIN_STACK_BOUNDS());
}
TARGET(CALL_TYPE_1) {
+ #if defined(Py_TAIL_CALL_INTERP)
+ int opcode;
+ #endif
+ opcode = CALL_TYPE_1;
+ (void)(opcode);
+ _Py_CODEUNIT* const this_instr = next_instr;
+ (void)this_instr;
frame->instr_ptr = next_instr;
next_instr += 4;
INSTRUCTION_STATS(CALL_TYPE_1);
PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable);
PyObject *arg_o = PyStackRef_AsPyObjectBorrow(arg);
assert(oparg == 1);
- DEOPT_IF(!PyStackRef_IsNull(null), CALL);
- DEOPT_IF(callable_o != (PyObject *)&PyType_Type, CALL);
+ if (!PyStackRef_IsNull(null)) {
+ UPDATE_MISS_STATS(CALL);
+ assert(_PyOpcode_Deopt[opcode] == (CALL));
+ JUMP_TO_PREDICTED(CALL);
+ }
+ if (callable_o != (PyObject *)&PyType_Type) {
+ UPDATE_MISS_STATS(CALL);
+ assert(_PyOpcode_Deopt[opcode] == (CALL));
+ JUMP_TO_PREDICTED(CALL);
+ }
STAT_INC(CALL, hit);
res = PyStackRef_FromPyObjectSteal(Py_NewRef(Py_TYPE(arg_o)));
stack_pointer[-3] = res;
}
TARGET(CHECK_EG_MATCH) {
+ #if defined(Py_TAIL_CALL_INTERP)
+ int opcode;
+ #endif
+ opcode = CHECK_EG_MATCH;
+ (void)(opcode);
frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(CHECK_EG_MATCH);
if (err < 0) {
PyStackRef_CLOSE(exc_value_st);
PyStackRef_CLOSE(match_type_st);
- goto pop_2_error;
+ JUMP_TO_LABEL(pop_2_error);
}
PyObject *match_o = NULL;
PyObject *rest_o = NULL;
PyStackRef_CLOSE(exc_value_st);
PyStackRef_CLOSE(match_type_st);
if (res < 0) {
- goto pop_2_error;
+ JUMP_TO_LABEL(pop_2_error);
}
assert((match_o == NULL) == (rest_o == NULL));
if (match_o == NULL) {
- goto pop_2_error;
+ JUMP_TO_LABEL(pop_2_error);
}
if (!Py_IsNone(match_o)) {
stack_pointer += -2;
}
TARGET(CHECK_EXC_MATCH) {
+ #if defined(Py_TAIL_CALL_INTERP)
+ int opcode;
+ #endif
+ opcode = CHECK_EXC_MATCH;
+ (void)(opcode);
frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(CHECK_EXC_MATCH);
stack_pointer = _PyFrame_GetStackPointer(frame);
if (err < 0) {
PyStackRef_CLOSE(right);
- goto pop_1_error;
+ JUMP_TO_LABEL(pop_1_error);
}
_PyFrame_SetStackPointer(frame, stack_pointer);
int res = PyErr_GivenExceptionMatches(left_o, right_o);
}
TARGET(CLEANUP_THROW) {
- _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr;
+ #if defined(Py_TAIL_CALL_INTERP)
+ int opcode;
+ #endif
+ opcode = CLEANUP_THROW;
+ (void)(opcode);
+ _Py_CODEUNIT* const this_instr = next_instr;
(void)this_instr;
+ frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(CLEANUP_THROW);
_PyStackRef sub_iter_st;
last_sent_val_st = stack_pointer[-2];
sub_iter_st = stack_pointer[-3];
PyObject *exc_value = PyStackRef_AsPyObjectBorrow(exc_value_st);
+ #ifndef Py_TAIL_CALL_INTERP
assert(throwflag);
+ #endif
assert(exc_value && PyExceptionInstance_Check(exc_value));
_PyFrame_SetStackPointer(frame, stack_pointer);
int matches = PyErr_GivenExceptionMatches(exc_value, PyExc_StopIteration);
monitor_reraise(tstate, frame, this_instr);
stack_pointer = _PyFrame_GetStackPointer(frame);
_PyFrame_SetStackPointer(frame, stack_pointer);
- goto exception_unwind;
+ JUMP_TO_LABEL(exception_unwind);
}
stack_pointer[-3] = none;
stack_pointer[-2] = value;
}
TARGET(COMPARE_OP) {
+ #if defined(Py_TAIL_CALL_INTERP)
+ int opcode;
+ #endif
+ opcode = COMPARE_OP;
+ (void)(opcode);
frame->instr_ptr = next_instr;
next_instr += 2;
INSTRUCTION_STATS(COMPARE_OP);
PyStackRef_CLOSE(left);
PyStackRef_CLOSE(right);
if (res_o == NULL) {
- goto pop_2_error;
+ JUMP_TO_LABEL(pop_2_error);
}
if (oparg & 16) {
stack_pointer += -2;
Py_DECREF(res_o);
stack_pointer = _PyFrame_GetStackPointer(frame);
if (res_bool < 0) {
- goto error;
+ JUMP_TO_LABEL(error);
}
res = res_bool ? PyStackRef_True : PyStackRef_False;
}
}
TARGET(COMPARE_OP_FLOAT) {
+ #if defined(Py_TAIL_CALL_INTERP)
+ int opcode;
+ #endif
+ opcode = COMPARE_OP_FLOAT;
+ (void)(opcode);
+ _Py_CODEUNIT* const this_instr = next_instr;
+ (void)this_instr;
frame->instr_ptr = next_instr;
next_instr += 2;
INSTRUCTION_STATS(COMPARE_OP_FLOAT);
left = stack_pointer[-2];
PyObject *left_o = PyStackRef_AsPyObjectBorrow(left);
PyObject *right_o = PyStackRef_AsPyObjectBorrow(right);
- DEOPT_IF(!PyFloat_CheckExact(left_o), COMPARE_OP);
- DEOPT_IF(!PyFloat_CheckExact(right_o), COMPARE_OP);
+ if (!PyFloat_CheckExact(left_o)) {
+ UPDATE_MISS_STATS(COMPARE_OP);
+ assert(_PyOpcode_Deopt[opcode] == (COMPARE_OP));
+ JUMP_TO_PREDICTED(COMPARE_OP);
+ }
+ if (!PyFloat_CheckExact(right_o)) {
+ UPDATE_MISS_STATS(COMPARE_OP);
+ assert(_PyOpcode_Deopt[opcode] == (COMPARE_OP));
+ JUMP_TO_PREDICTED(COMPARE_OP);
+ }
}
/* Skip 1 cache entry */
// _COMPARE_OP_FLOAT
}
TARGET(COMPARE_OP_INT) {
+ #if defined(Py_TAIL_CALL_INTERP)
+ int opcode;
+ #endif
+ opcode = COMPARE_OP_INT;
+ (void)(opcode);
+ _Py_CODEUNIT* const this_instr = next_instr;
+ (void)this_instr;
frame->instr_ptr = next_instr;
next_instr += 2;
INSTRUCTION_STATS(COMPARE_OP_INT);
left = stack_pointer[-2];
PyObject *left_o = PyStackRef_AsPyObjectBorrow(left);
PyObject *right_o = PyStackRef_AsPyObjectBorrow(right);
- DEOPT_IF(!PyLong_CheckExact(left_o), COMPARE_OP);
- DEOPT_IF(!PyLong_CheckExact(right_o), COMPARE_OP);
+ if (!PyLong_CheckExact(left_o)) {
+ UPDATE_MISS_STATS(COMPARE_OP);
+ assert(_PyOpcode_Deopt[opcode] == (COMPARE_OP));
+ JUMP_TO_PREDICTED(COMPARE_OP);
+ }
+ if (!PyLong_CheckExact(right_o)) {
+ UPDATE_MISS_STATS(COMPARE_OP);
+ assert(_PyOpcode_Deopt[opcode] == (COMPARE_OP));
+ JUMP_TO_PREDICTED(COMPARE_OP);
+ }
}
/* Skip 1 cache entry */
// _COMPARE_OP_INT
{
PyObject *left_o = PyStackRef_AsPyObjectBorrow(left);
PyObject *right_o = PyStackRef_AsPyObjectBorrow(right);
- DEOPT_IF(!_PyLong_IsCompact((PyLongObject *)left_o), COMPARE_OP);
- DEOPT_IF(!_PyLong_IsCompact((PyLongObject *)right_o), COMPARE_OP);
+ if (!_PyLong_IsCompact((PyLongObject *)left_o)) {
+ UPDATE_MISS_STATS(COMPARE_OP);
+ assert(_PyOpcode_Deopt[opcode] == (COMPARE_OP));
+ JUMP_TO_PREDICTED(COMPARE_OP);
+ }
+ if (!_PyLong_IsCompact((PyLongObject *)right_o)) {
+ UPDATE_MISS_STATS(COMPARE_OP);
+ assert(_PyOpcode_Deopt[opcode] == (COMPARE_OP));
+ JUMP_TO_PREDICTED(COMPARE_OP);
+ }
STAT_INC(COMPARE_OP, hit);
assert(_PyLong_DigitCount((PyLongObject *)left_o) <= 1 &&
_PyLong_DigitCount((PyLongObject *)right_o) <= 1);
}
TARGET(COMPARE_OP_STR) {
+ #if defined(Py_TAIL_CALL_INTERP)
+ int opcode;
+ #endif
+ opcode = COMPARE_OP_STR;
+ (void)(opcode);
+ _Py_CODEUNIT* const this_instr = next_instr;
+ (void)this_instr;
frame->instr_ptr = next_instr;
next_instr += 2;
INSTRUCTION_STATS(COMPARE_OP_STR);
left = stack_pointer[-2];
PyObject *left_o = PyStackRef_AsPyObjectBorrow(left);
PyObject *right_o = PyStackRef_AsPyObjectBorrow(right);
- DEOPT_IF(!PyUnicode_CheckExact(left_o), COMPARE_OP);
- DEOPT_IF(!PyUnicode_CheckExact(right_o), COMPARE_OP);
+ if (!PyUnicode_CheckExact(left_o)) {
+ UPDATE_MISS_STATS(COMPARE_OP);
+ assert(_PyOpcode_Deopt[opcode] == (COMPARE_OP));
+ JUMP_TO_PREDICTED(COMPARE_OP);
+ }
+ if (!PyUnicode_CheckExact(right_o)) {
+ UPDATE_MISS_STATS(COMPARE_OP);
+ assert(_PyOpcode_Deopt[opcode] == (COMPARE_OP));
+ JUMP_TO_PREDICTED(COMPARE_OP);
+ }
}
/* Skip 1 cache entry */
// _COMPARE_OP_STR
}
TARGET(CONTAINS_OP) {
+ #if defined(Py_TAIL_CALL_INTERP)
+ int opcode;
+ #endif
+ opcode = CONTAINS_OP;
+ (void)(opcode);
frame->instr_ptr = next_instr;
next_instr += 2;
INSTRUCTION_STATS(CONTAINS_OP);
PyStackRef_CLOSE(left);
PyStackRef_CLOSE(right);
if (res < 0) {
- goto pop_2_error;
+ JUMP_TO_LABEL(pop_2_error);
}
b = (res ^ oparg) ? PyStackRef_True : PyStackRef_False;
}
}
TARGET(CONTAINS_OP_DICT) {
+ #if defined(Py_TAIL_CALL_INTERP)
+ int opcode;
+ #endif
+ opcode = CONTAINS_OP_DICT;
+ (void)(opcode);
+ _Py_CODEUNIT* const this_instr = next_instr;
+ (void)this_instr;
frame->instr_ptr = next_instr;
next_instr += 2;
INSTRUCTION_STATS(CONTAINS_OP_DICT);
left = stack_pointer[-2];
PyObject *left_o = PyStackRef_AsPyObjectBorrow(left);
PyObject *right_o = PyStackRef_AsPyObjectBorrow(right);
- DEOPT_IF(!PyDict_CheckExact(right_o), CONTAINS_OP);
+ if (!PyDict_CheckExact(right_o)) {
+ UPDATE_MISS_STATS(CONTAINS_OP);
+ assert(_PyOpcode_Deopt[opcode] == (CONTAINS_OP));
+ JUMP_TO_PREDICTED(CONTAINS_OP);
+ }
STAT_INC(CONTAINS_OP, hit);
_PyFrame_SetStackPointer(frame, stack_pointer);
int res = PyDict_Contains(right_o, left_o);
PyStackRef_CLOSE(left);
PyStackRef_CLOSE(right);
if (res < 0) {
- goto pop_2_error;
+ JUMP_TO_LABEL(pop_2_error);
}
b = (res ^ oparg) ? PyStackRef_True : PyStackRef_False;
stack_pointer[-2] = b;
}
TARGET(CONTAINS_OP_SET) {
+ #if defined(Py_TAIL_CALL_INTERP)
+ int opcode;
+ #endif
+ opcode = CONTAINS_OP_SET;
+ (void)(opcode);
+ _Py_CODEUNIT* const this_instr = next_instr;
+ (void)this_instr;
frame->instr_ptr = next_instr;
next_instr += 2;
INSTRUCTION_STATS(CONTAINS_OP_SET);
left = stack_pointer[-2];
PyObject *left_o = PyStackRef_AsPyObjectBorrow(left);
PyObject *right_o = PyStackRef_AsPyObjectBorrow(right);
- DEOPT_IF(!(PySet_CheckExact(right_o) || PyFrozenSet_CheckExact(right_o)), CONTAINS_OP);
+ if (!(PySet_CheckExact(right_o) || PyFrozenSet_CheckExact(right_o))) {
+ UPDATE_MISS_STATS(CONTAINS_OP);
+ assert(_PyOpcode_Deopt[opcode] == (CONTAINS_OP));
+ JUMP_TO_PREDICTED(CONTAINS_OP);
+ }
STAT_INC(CONTAINS_OP, hit);
// Note: both set and frozenset use the same seq_contains method!
_PyFrame_SetStackPointer(frame, stack_pointer);
PyStackRef_CLOSE(left);
PyStackRef_CLOSE(right);
if (res < 0) {
- goto pop_2_error;
+ JUMP_TO_LABEL(pop_2_error);
}
b = (res ^ oparg) ? PyStackRef_True : PyStackRef_False;
stack_pointer[-2] = b;
}
TARGET(CONVERT_VALUE) {
+ #if defined(Py_TAIL_CALL_INTERP)
+ int opcode;
+ #endif
+ opcode = CONVERT_VALUE;
+ (void)(opcode);
frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(CONVERT_VALUE);
PyStackRef_CLOSE(value);
stack_pointer = _PyFrame_GetStackPointer(frame);
if (result_o == NULL) {
- goto error;
+ JUMP_TO_LABEL(error);
}
result = PyStackRef_FromPyObjectSteal(result_o);
stack_pointer[0] = result;
}
TARGET(COPY) {
+ #if defined(Py_TAIL_CALL_INTERP)
+ int opcode;
+ #endif
+ opcode = COPY;
+ (void)(opcode);
frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(COPY);
}
TARGET(COPY_FREE_VARS) {
+ #if defined(Py_TAIL_CALL_INTERP)
+ int opcode;
+ #endif
+ opcode = COPY_FREE_VARS;
+ (void)(opcode);
frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(COPY_FREE_VARS);
}
TARGET(DELETE_ATTR) {
+ #if defined(Py_TAIL_CALL_INTERP)
+ int opcode;
+ #endif
+ opcode = DELETE_ATTR;
+ (void)(opcode);
frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(DELETE_ATTR);
stack_pointer = _PyFrame_GetStackPointer(frame);
PyStackRef_CLOSE(owner);
if (err) {
- goto pop_1_error;
+ JUMP_TO_LABEL(pop_1_error);
}
stack_pointer += -1;
assert(WITHIN_STACK_BOUNDS());
}
TARGET(DELETE_DEREF) {
+ #if defined(Py_TAIL_CALL_INTERP)
+ int opcode;
+ #endif
+ opcode = DELETE_DEREF;
+ (void)(opcode);
frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(DELETE_DEREF);
_PyFrame_SetStackPointer(frame, stack_pointer);
_PyEval_FormatExcUnbound(tstate, _PyFrame_GetCode(frame), oparg);
stack_pointer = _PyFrame_GetStackPointer(frame);
- goto error;
+ JUMP_TO_LABEL(error);
}
_PyFrame_SetStackPointer(frame, stack_pointer);
Py_DECREF(oldobj);
}
TARGET(DELETE_FAST) {
+ #if defined(Py_TAIL_CALL_INTERP)
+ int opcode;
+ #endif
+ opcode = DELETE_FAST;
+ (void)(opcode);
frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(DELETE_FAST);
PyTuple_GetItem(_PyFrame_GetCode(frame)->co_localsplusnames, oparg)
);
stack_pointer = _PyFrame_GetStackPointer(frame);
- goto error;
+ JUMP_TO_LABEL(error);
}
_PyStackRef tmp = GETLOCAL(oparg);
GETLOCAL(oparg) = PyStackRef_NULL;
}
TARGET(DELETE_GLOBAL) {
+ #if defined(Py_TAIL_CALL_INTERP)
+ int opcode;
+ #endif
+ opcode = DELETE_GLOBAL;
+ (void)(opcode);
frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(DELETE_GLOBAL);
stack_pointer = _PyFrame_GetStackPointer(frame);
// Can't use ERROR_IF here.
if (err < 0) {
- goto error;
+ JUMP_TO_LABEL(error);
}
if (err == 0) {
_PyFrame_SetStackPointer(frame, stack_pointer);
_PyEval_FormatExcCheckArg(tstate, PyExc_NameError,
NAME_ERROR_MSG, name);
stack_pointer = _PyFrame_GetStackPointer(frame);
- goto error;
+ JUMP_TO_LABEL(error);
}
DISPATCH();
}
TARGET(DELETE_NAME) {
+ #if defined(Py_TAIL_CALL_INTERP)
+ int opcode;
+ #endif
+ opcode = DELETE_NAME;
+ (void)(opcode);
frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(DELETE_NAME);
_PyErr_Format(tstate, PyExc_SystemError,
"no locals when deleting %R", name);
stack_pointer = _PyFrame_GetStackPointer(frame);
- goto error;
+ JUMP_TO_LABEL(error);
}
_PyFrame_SetStackPointer(frame, stack_pointer);
err = PyObject_DelItem(ns, name);
NAME_ERROR_MSG,
name);
stack_pointer = _PyFrame_GetStackPointer(frame);
- goto error;
+ JUMP_TO_LABEL(error);
}
DISPATCH();
}
TARGET(DELETE_SUBSCR) {
+ #if defined(Py_TAIL_CALL_INTERP)
+ int opcode;
+ #endif
+ opcode = DELETE_SUBSCR;
+ (void)(opcode);
frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(DELETE_SUBSCR);
PyStackRef_CLOSE(container);
PyStackRef_CLOSE(sub);
if (err) {
- goto pop_2_error;
+ JUMP_TO_LABEL(pop_2_error);
}
stack_pointer += -2;
assert(WITHIN_STACK_BOUNDS());
}
TARGET(DICT_MERGE) {
+ #if defined(Py_TAIL_CALL_INTERP)
+ int opcode;
+ #endif
+ opcode = DICT_MERGE;
+ (void)(opcode);
frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(DICT_MERGE);
_PyEval_FormatKwargsError(tstate, callable_o, update_o);
stack_pointer = _PyFrame_GetStackPointer(frame);
PyStackRef_CLOSE(update);
- goto pop_1_error;
+ JUMP_TO_LABEL(pop_1_error);
}
PyStackRef_CLOSE(update);
stack_pointer += -1;
}
TARGET(DICT_UPDATE) {
+ #if defined(Py_TAIL_CALL_INTERP)
+ int opcode;
+ #endif
+ opcode = DICT_UPDATE;
+ (void)(opcode);
frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(DICT_UPDATE);
stack_pointer = _PyFrame_GetStackPointer(frame);
}
PyStackRef_CLOSE(update);
- goto pop_1_error;
+ JUMP_TO_LABEL(pop_1_error);
}
PyStackRef_CLOSE(update);
stack_pointer += -1;
}
TARGET(END_ASYNC_FOR) {
- _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr;
+ #if defined(Py_TAIL_CALL_INTERP)
+ int opcode;
+ #endif
+ opcode = END_ASYNC_FOR;
+ (void)(opcode);
+ _Py_CODEUNIT* const this_instr = next_instr;
(void)this_instr;
+ frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(END_ASYNC_FOR);
_PyStackRef awaitable_st;
monitor_reraise(tstate, frame, this_instr);
stack_pointer = _PyFrame_GetStackPointer(frame);
_PyFrame_SetStackPointer(frame, stack_pointer);
- goto exception_unwind;
+ JUMP_TO_LABEL(exception_unwind);
}
stack_pointer += -2;
assert(WITHIN_STACK_BOUNDS());
}
TARGET(END_FOR) {
+ #if defined(Py_TAIL_CALL_INTERP)
+ int opcode;
+ #endif
+ opcode = END_FOR;
+ (void)(opcode);
next_instr += 1;
INSTRUCTION_STATS(END_FOR);
_PyStackRef value;
}
TARGET(END_SEND) {
+ #if defined(Py_TAIL_CALL_INTERP)
+ int opcode;
+ #endif
+ opcode = END_SEND;
+ (void)(opcode);
frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(END_SEND);
}
TARGET(ENTER_EXECUTOR) {
- _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr;
+ #if defined(Py_TAIL_CALL_INTERP)
+ int opcode;
+ #endif
+ opcode = ENTER_EXECUTOR;
+ (void)(opcode);
+ _Py_CODEUNIT* const this_instr = next_instr;
(void)this_instr;
+ frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(ENTER_EXECUTOR);
- opcode = ENTER_EXECUTOR;
#ifdef _Py_TIER2
PyCodeObject *code = _PyFrame_GetCode(frame);
_PyExecutorObject *executor = code->co_executors->executors[oparg & 255];
}
TARGET(EXIT_INIT_CHECK) {
+ #if defined(Py_TAIL_CALL_INTERP)
+ int opcode;
+ #endif
+ opcode = EXIT_INIT_CHECK;
+ (void)(opcode);
frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(EXIT_INIT_CHECK);
"__init__() should return None, not '%.200s'",
Py_TYPE(PyStackRef_AsPyObjectBorrow(should_be_none))->tp_name);
stack_pointer = _PyFrame_GetStackPointer(frame);
- goto error;
+ JUMP_TO_LABEL(error);
}
stack_pointer += -1;
assert(WITHIN_STACK_BOUNDS());
}
TARGET(EXTENDED_ARG) {
+ #if defined(Py_TAIL_CALL_INTERP)
+ int opcode;
+ #endif
+ opcode = EXTENDED_ARG;
+ (void)(opcode);
frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(EXTENDED_ARG);
- opcode = EXTENDED_ARG;
assert(oparg);
opcode = next_instr->op.code;
oparg = oparg << 8 | next_instr->op.arg;
}
TARGET(FORMAT_SIMPLE) {
+ #if defined(Py_TAIL_CALL_INTERP)
+ int opcode;
+ #endif
+ opcode = FORMAT_SIMPLE;
+ (void)(opcode);
frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(FORMAT_SIMPLE);
PyStackRef_CLOSE(value);
stack_pointer = _PyFrame_GetStackPointer(frame);
if (res_o == NULL) {
- goto error;
+ JUMP_TO_LABEL(error);
}
res = PyStackRef_FromPyObjectSteal(res_o);
}
}
TARGET(FORMAT_WITH_SPEC) {
+ #if defined(Py_TAIL_CALL_INTERP)
+ int opcode;
+ #endif
+ opcode = FORMAT_WITH_SPEC;
+ (void)(opcode);
frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(FORMAT_WITH_SPEC);
PyStackRef_CLOSE(value);
PyStackRef_CLOSE(fmt_spec);
if (res_o == NULL) {
- goto pop_2_error;
+ JUMP_TO_LABEL(pop_2_error);
}
res = PyStackRef_FromPyObjectSteal(res_o);
stack_pointer[-2] = res;
}
TARGET(FOR_ITER) {
+ #if defined(Py_TAIL_CALL_INTERP)
+ int opcode;
+ #endif
+ opcode = FOR_ITER;
+ (void)(opcode);
frame->instr_ptr = next_instr;
next_instr += 2;
INSTRUCTION_STATS(FOR_ITER);
int matches = _PyErr_ExceptionMatches(tstate, PyExc_StopIteration);
stack_pointer = _PyFrame_GetStackPointer(frame);
if (!matches) {
- goto error;
+ JUMP_TO_LABEL(error);
}
_PyFrame_SetStackPointer(frame, stack_pointer);
_PyEval_MonitorRaise(tstate, frame, this_instr);
}
TARGET(FOR_ITER_GEN) {
+ #if defined(Py_TAIL_CALL_INTERP)
+ int opcode;
+ #endif
+ opcode = FOR_ITER_GEN;
+ (void)(opcode);
+ _Py_CODEUNIT* const this_instr = next_instr;
+ (void)this_instr;
frame->instr_ptr = next_instr;
next_instr += 2;
INSTRUCTION_STATS(FOR_ITER_GEN);
/* Skip 1 cache entry */
// _CHECK_PEP_523
{
- DEOPT_IF(tstate->interp->eval_frame, FOR_ITER);
+ if (tstate->interp->eval_frame) {
+ UPDATE_MISS_STATS(FOR_ITER);
+ assert(_PyOpcode_Deopt[opcode] == (FOR_ITER));
+ JUMP_TO_PREDICTED(FOR_ITER);
+ }
}
// _FOR_ITER_GEN_FRAME
{
iter = stack_pointer[-1];
PyGenObject *gen = (PyGenObject *)PyStackRef_AsPyObjectBorrow(iter);
- DEOPT_IF(Py_TYPE(gen) != &PyGen_Type, FOR_ITER);
- DEOPT_IF(gen->gi_frame_state >= FRAME_EXECUTING, FOR_ITER);
+ if (Py_TYPE(gen) != &PyGen_Type) {
+ UPDATE_MISS_STATS(FOR_ITER);
+ assert(_PyOpcode_Deopt[opcode] == (FOR_ITER));
+ JUMP_TO_PREDICTED(FOR_ITER);
+ }
+ if (gen->gi_frame_state >= FRAME_EXECUTING) {
+ UPDATE_MISS_STATS(FOR_ITER);
+ assert(_PyOpcode_Deopt[opcode] == (FOR_ITER));
+ JUMP_TO_PREDICTED(FOR_ITER);
+ }
STAT_INC(FOR_ITER, hit);
gen_frame = &gen->gi_iframe;
_PyFrame_StackPush(gen_frame, PyStackRef_None);
}
TARGET(FOR_ITER_LIST) {
+ #if defined(Py_TAIL_CALL_INTERP)
+ int opcode;
+ #endif
+ opcode = FOR_ITER_LIST;
+ (void)(opcode);
+ _Py_CODEUNIT* const this_instr = next_instr;
+ (void)this_instr;
frame->instr_ptr = next_instr;
next_instr += 2;
INSTRUCTION_STATS(FOR_ITER_LIST);
// _ITER_CHECK_LIST
{
iter = stack_pointer[-1];
- DEOPT_IF(Py_TYPE(PyStackRef_AsPyObjectBorrow(iter)) != &PyListIter_Type, FOR_ITER);
+ if (Py_TYPE(PyStackRef_AsPyObjectBorrow(iter)) != &PyListIter_Type) {
+ UPDATE_MISS_STATS(FOR_ITER);
+ assert(_PyOpcode_Deopt[opcode] == (FOR_ITER));
+ JUMP_TO_PREDICTED(FOR_ITER);
+ }
}
// _ITER_JUMP_LIST
{
}
TARGET(FOR_ITER_RANGE) {
+ #if defined(Py_TAIL_CALL_INTERP)
+ int opcode;
+ #endif
+ opcode = FOR_ITER_RANGE;
+ (void)(opcode);
+ _Py_CODEUNIT* const this_instr = next_instr;
+ (void)this_instr;
frame->instr_ptr = next_instr;
next_instr += 2;
INSTRUCTION_STATS(FOR_ITER_RANGE);
{
iter = stack_pointer[-1];
_PyRangeIterObject *r = (_PyRangeIterObject *)PyStackRef_AsPyObjectBorrow(iter);
- DEOPT_IF(Py_TYPE(r) != &PyRangeIter_Type, FOR_ITER);
+ if (Py_TYPE(r) != &PyRangeIter_Type) {
+ UPDATE_MISS_STATS(FOR_ITER);
+ assert(_PyOpcode_Deopt[opcode] == (FOR_ITER));
+ JUMP_TO_PREDICTED(FOR_ITER);
+ }
}
// _ITER_JUMP_RANGE
{
r->len--;
PyObject *res = PyLong_FromLong(value);
if (res == NULL) {
- goto error;
+ JUMP_TO_LABEL(error);
}
next = PyStackRef_FromPyObjectSteal(res);
}
}
TARGET(FOR_ITER_TUPLE) {
+ #if defined(Py_TAIL_CALL_INTERP)
+ int opcode;
+ #endif
+ opcode = FOR_ITER_TUPLE;
+ (void)(opcode);
+ _Py_CODEUNIT* const this_instr = next_instr;
+ (void)this_instr;
frame->instr_ptr = next_instr;
next_instr += 2;
INSTRUCTION_STATS(FOR_ITER_TUPLE);
// _ITER_CHECK_TUPLE
{
iter = stack_pointer[-1];
- DEOPT_IF(Py_TYPE(PyStackRef_AsPyObjectBorrow(iter)) != &PyTupleIter_Type, FOR_ITER);
+ if (Py_TYPE(PyStackRef_AsPyObjectBorrow(iter)) != &PyTupleIter_Type) {
+ UPDATE_MISS_STATS(FOR_ITER);
+ assert(_PyOpcode_Deopt[opcode] == (FOR_ITER));
+ JUMP_TO_PREDICTED(FOR_ITER);
+ }
}
// _ITER_JUMP_TUPLE
{
}
TARGET(GET_AITER) {
+ #if defined(Py_TAIL_CALL_INTERP)
+ int opcode;
+ #endif
+ opcode = GET_AITER;
+ (void)(opcode);
frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(GET_AITER);
type->tp_name);
stack_pointer = _PyFrame_GetStackPointer(frame);
PyStackRef_CLOSE(obj);
- goto pop_1_error;
+ JUMP_TO_LABEL(pop_1_error);
}
_PyFrame_SetStackPointer(frame, stack_pointer);
iter_o = (*getter)(obj_o);
stack_pointer = _PyFrame_GetStackPointer(frame);
PyStackRef_CLOSE(obj);
if (iter_o == NULL) {
- goto pop_1_error;
+ JUMP_TO_LABEL(pop_1_error);
}
if (Py_TYPE(iter_o)->tp_as_async == NULL ||
Py_TYPE(iter_o)->tp_as_async->am_anext == NULL) {
Py_TYPE(iter_o)->tp_name);
Py_DECREF(iter_o);
stack_pointer = _PyFrame_GetStackPointer(frame);
- goto error;
+ JUMP_TO_LABEL(error);
}
iter = PyStackRef_FromPyObjectSteal(iter_o);
stack_pointer[-1] = iter;
}
TARGET(GET_ANEXT) {
+ #if defined(Py_TAIL_CALL_INTERP)
+ int opcode;
+ #endif
+ opcode = GET_ANEXT;
+ (void)(opcode);
frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(GET_ANEXT);
PyObject *awaitable_o = _PyEval_GetANext(PyStackRef_AsPyObjectBorrow(aiter));
stack_pointer = _PyFrame_GetStackPointer(frame);
if (awaitable_o == NULL) {
- goto error;
+ JUMP_TO_LABEL(error);
}
awaitable = PyStackRef_FromPyObjectSteal(awaitable_o);
stack_pointer[0] = awaitable;
}
TARGET(GET_AWAITABLE) {
+ #if defined(Py_TAIL_CALL_INTERP)
+ int opcode;
+ #endif
+ opcode = GET_AWAITABLE;
+ (void)(opcode);
frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(GET_AWAITABLE);
stack_pointer = _PyFrame_GetStackPointer(frame);
PyStackRef_CLOSE(iterable);
if (iter_o == NULL) {
- goto pop_1_error;
+ JUMP_TO_LABEL(pop_1_error);
}
iter = PyStackRef_FromPyObjectSteal(iter_o);
stack_pointer[-1] = iter;
}
TARGET(GET_ITER) {
+ #if defined(Py_TAIL_CALL_INTERP)
+ int opcode;
+ #endif
+ opcode = GET_ITER;
+ (void)(opcode);
frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(GET_ITER);
stack_pointer = _PyFrame_GetStackPointer(frame);
PyStackRef_CLOSE(iterable);
if (iter_o == NULL) {
- goto pop_1_error;
+ JUMP_TO_LABEL(pop_1_error);
}
iter = PyStackRef_FromPyObjectSteal(iter_o);
stack_pointer[-1] = iter;
}
TARGET(GET_LEN) {
+ #if defined(Py_TAIL_CALL_INTERP)
+ int opcode;
+ #endif
+ opcode = GET_LEN;
+ (void)(opcode);
frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(GET_LEN);
Py_ssize_t len_i = PyObject_Length(PyStackRef_AsPyObjectBorrow(obj));
stack_pointer = _PyFrame_GetStackPointer(frame);
if (len_i < 0) {
- goto error;
+ JUMP_TO_LABEL(error);
}
PyObject *len_o = PyLong_FromSsize_t(len_i);
if (len_o == NULL) {
- goto error;
+ JUMP_TO_LABEL(error);
}
len = PyStackRef_FromPyObjectSteal(len_o);
stack_pointer[0] = len;
}
TARGET(GET_YIELD_FROM_ITER) {
+ #if defined(Py_TAIL_CALL_INTERP)
+ int opcode;
+ #endif
+ opcode = GET_YIELD_FROM_ITER;
+ (void)(opcode);
frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(GET_YIELD_FROM_ITER);
"cannot 'yield from' a coroutine object "
"in a non-coroutine generator");
stack_pointer = _PyFrame_GetStackPointer(frame);
- goto error;
+ JUMP_TO_LABEL(error);
}
iter = iterable;
}
PyObject *iter_o = PyObject_GetIter(iterable_o);
stack_pointer = _PyFrame_GetStackPointer(frame);
if (iter_o == NULL) {
- goto error;
+ JUMP_TO_LABEL(error);
}
iter = PyStackRef_FromPyObjectSteal(iter_o);
PyStackRef_CLOSE(iterable);
}
TARGET(IMPORT_FROM) {
+ #if defined(Py_TAIL_CALL_INTERP)
+ int opcode;
+ #endif
+ opcode = IMPORT_FROM;
+ (void)(opcode);
frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(IMPORT_FROM);
PyObject *res_o = _PyEval_ImportFrom(tstate, PyStackRef_AsPyObjectBorrow(from), name);
stack_pointer = _PyFrame_GetStackPointer(frame);
if (res_o == NULL) {
- goto error;
+ JUMP_TO_LABEL(error);
}
res = PyStackRef_FromPyObjectSteal(res_o);
stack_pointer[0] = res;
}
TARGET(IMPORT_NAME) {
+ #if defined(Py_TAIL_CALL_INTERP)
+ int opcode;
+ #endif
+ opcode = IMPORT_NAME;
+ (void)(opcode);
frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(IMPORT_NAME);
PyStackRef_CLOSE(level);
PyStackRef_CLOSE(fromlist);
if (res_o == NULL) {
- goto pop_2_error;
+ JUMP_TO_LABEL(pop_2_error);
}
res = PyStackRef_FromPyObjectSteal(res_o);
stack_pointer[-2] = res;
}
TARGET(INSTRUMENTED_CALL) {
- _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr;
+ #if defined(Py_TAIL_CALL_INTERP)
+ int opcode;
+ #endif
+ opcode = INSTRUMENTED_CALL;
+ (void)(opcode);
+ _Py_CODEUNIT* const this_instr = next_instr;
(void)this_instr;
+ frame->instr_ptr = next_instr;
next_instr += 4;
INSTRUCTION_STATS(INSTRUMENTED_CALL);
- opcode = INSTRUMENTED_CALL;
_PyStackRef *callable;
_PyStackRef *self_or_null;
_PyStackRef *args;
);
stack_pointer = _PyFrame_GetStackPointer(frame);
if (err) {
- goto error;
+ JUMP_TO_LABEL(error);
}
}
// _DO_CALL
// The frame has stolen all the arguments from the stack,
// so there is no need to clean them up.
if (new_frame == NULL) {
- goto error;
+ JUMP_TO_LABEL(error);
}
frame->return_offset = 4 ;
DISPATCH_INLINED(new_frame);
}
stack_pointer += -2 - oparg;
assert(WITHIN_STACK_BOUNDS());
- goto error;
+ JUMP_TO_LABEL(error);
}
_PyFrame_SetStackPointer(frame, stack_pointer);
PyObject *res_o = PyObject_Vectorcall(
if (res_o == NULL) {
stack_pointer += -2 - oparg;
assert(WITHIN_STACK_BOUNDS());
- goto error;
+ JUMP_TO_LABEL(error);
}
res = PyStackRef_FromPyObjectSteal(res_o);
}
int err = _Py_HandlePending(tstate);
stack_pointer = _PyFrame_GetStackPointer(frame);
if (err != 0) {
- goto error;
+ JUMP_TO_LABEL(error);
}
stack_pointer += 1 + oparg;
assert(WITHIN_STACK_BOUNDS());
}
TARGET(INSTRUMENTED_CALL_FUNCTION_EX) {
- _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr;
+ #if defined(Py_TAIL_CALL_INTERP)
+ int opcode;
+ #endif
+ opcode = INSTRUMENTED_CALL_FUNCTION_EX;
+ (void)(opcode);
+ _Py_CODEUNIT* const this_instr = next_instr;
(void)this_instr;
+ frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(INSTRUMENTED_CALL_FUNCTION_EX);
- opcode = INSTRUMENTED_CALL_FUNCTION_EX;
_PyStackRef func;
_PyStackRef callargs;
_PyStackRef kwargs_in;
int err = _Py_Check_ArgsIterable(tstate, PyStackRef_AsPyObjectBorrow(func), callargs_o);
stack_pointer = _PyFrame_GetStackPointer(frame);
if (err < 0) {
- goto error;
+ JUMP_TO_LABEL(error);
}
_PyFrame_SetStackPointer(frame, stack_pointer);
PyObject *tuple_o = PySequence_Tuple(callargs_o);
stack_pointer = _PyFrame_GetStackPointer(frame);
if (tuple_o == NULL) {
- goto error;
+ JUMP_TO_LABEL(error);
}
kwargs_out = kwargs_in;
stack_pointer += -2;
frame, this_instr, func, arg);
stack_pointer = _PyFrame_GetStackPointer(frame);
if (err) {
- goto error;
+ JUMP_TO_LABEL(error);
}
_PyFrame_SetStackPointer(frame, stack_pointer);
result_o = PyObject_Call(func, callargs, kwargs);
stack_pointer += -2;
assert(WITHIN_STACK_BOUNDS());
if (new_frame == NULL) {
- goto error;
+ JUMP_TO_LABEL(error);
}
assert( 1 == 1);
frame->return_offset = 1;
PyStackRef_CLOSE(func_st);
stack_pointer = _PyFrame_GetStackPointer(frame);
if (result_o == NULL) {
- goto error;
+ JUMP_TO_LABEL(error);
}
result = PyStackRef_FromPyObjectSteal(result_o);
}
int err = _Py_HandlePending(tstate);
stack_pointer = _PyFrame_GetStackPointer(frame);
if (err != 0) {
- goto error;
+ JUMP_TO_LABEL(error);
}
stack_pointer += -1;
assert(WITHIN_STACK_BOUNDS());
}
TARGET(INSTRUMENTED_CALL_KW) {
- _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr;
+ #if defined(Py_TAIL_CALL_INTERP)
+ int opcode;
+ #endif
+ opcode = INSTRUMENTED_CALL_KW;
+ (void)(opcode);
+ _Py_CODEUNIT* const this_instr = next_instr;
(void)this_instr;
+ frame->instr_ptr = next_instr;
next_instr += 4;
INSTRUCTION_STATS(INSTRUMENTED_CALL_KW);
- opcode = INSTRUMENTED_CALL_KW;
_PyStackRef *callable;
_PyStackRef *self_or_null;
_PyStackRef *args;
frame, this_instr, function, arg);
stack_pointer = _PyFrame_GetStackPointer(frame);
if (err) {
- goto error;
+ JUMP_TO_LABEL(error);
}
}
// _MAYBE_EXPAND_METHOD_KW
// The frame has stolen all the arguments from the stack,
// so there is no need to clean them up.
if (new_frame == NULL) {
- goto error;
+ JUMP_TO_LABEL(error);
}
assert( 4 == 1 + INLINE_CACHE_ENTRIES_CALL_KW);
frame->return_offset = 4 ;
PyStackRef_CLOSE(kwnames);
stack_pointer += -3 - oparg;
assert(WITHIN_STACK_BOUNDS());
- goto error;
+ JUMP_TO_LABEL(error);
}
stack_pointer[-1] = kwnames;
_PyFrame_SetStackPointer(frame, stack_pointer);
if (res_o == NULL) {
stack_pointer += -3 - oparg;
assert(WITHIN_STACK_BOUNDS());
- goto error;
+ JUMP_TO_LABEL(error);
}
res = PyStackRef_FromPyObjectSteal(res_o);
}
}
TARGET(INSTRUMENTED_END_FOR) {
+ #if defined(Py_TAIL_CALL_INTERP)
+ int opcode;
+ #endif
+ opcode = INSTRUMENTED_END_FOR;
+ (void)(opcode);
_Py_CODEUNIT* const this_instr = next_instr;
(void)this_instr;
next_instr += 1;
int err = monitor_stop_iteration(tstate, frame, this_instr, PyStackRef_AsPyObjectBorrow(value));
stack_pointer = _PyFrame_GetStackPointer(frame);
if (err) {
- goto error;
+ JUMP_TO_LABEL(error);
}
}
PyStackRef_CLOSE(value);
}
TARGET(INSTRUMENTED_END_SEND) {
- _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr;
+ #if defined(Py_TAIL_CALL_INTERP)
+ int opcode;
+ #endif
+ opcode = INSTRUMENTED_END_SEND;
+ (void)(opcode);
+ _Py_CODEUNIT* const this_instr = next_instr;
(void)this_instr;
+ frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(INSTRUMENTED_END_SEND);
_PyStackRef receiver;
int err = monitor_stop_iteration(tstate, frame, this_instr, PyStackRef_AsPyObjectBorrow(value));
stack_pointer = _PyFrame_GetStackPointer(frame);
if (err) {
- goto error;
+ JUMP_TO_LABEL(error);
}
}
val = value;
}
TARGET(INSTRUMENTED_FOR_ITER) {
- _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr;
+ #if defined(Py_TAIL_CALL_INTERP)
+ int opcode;
+ #endif
+ opcode = INSTRUMENTED_FOR_ITER;
+ (void)(opcode);
+ _Py_CODEUNIT* const this_instr = next_instr;
(void)this_instr;
+ frame->instr_ptr = next_instr;
next_instr += 2;
INSTRUCTION_STATS(INSTRUMENTED_FOR_ITER);
/* Skip 1 cache entry */
int matches = _PyErr_ExceptionMatches(tstate, PyExc_StopIteration);
stack_pointer = _PyFrame_GetStackPointer(frame);
if (!matches) {
- goto error;
+ JUMP_TO_LABEL(error);
}
_PyFrame_SetStackPointer(frame, stack_pointer);
_PyEval_MonitorRaise(tstate, frame, this_instr);
}
TARGET(INSTRUMENTED_INSTRUCTION) {
- _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr;
+ #if defined(Py_TAIL_CALL_INTERP)
+ int opcode;
+ #endif
+ opcode = INSTRUMENTED_INSTRUCTION;
+ (void)(opcode);
+ _Py_CODEUNIT* const this_instr = next_instr;
(void)this_instr;
+ frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(INSTRUMENTED_INSTRUCTION);
- opcode = INSTRUMENTED_INSTRUCTION;
_PyFrame_SetStackPointer(frame, stack_pointer);
int next_opcode = _Py_call_instrumentation_instruction(
tstate, frame, this_instr);
stack_pointer = _PyFrame_GetStackPointer(frame);
if (next_opcode < 0) {
- goto error;
+ JUMP_TO_LABEL(error);
}
next_instr = this_instr;
if (_PyOpcode_Caches[next_opcode]) {
}
TARGET(INSTRUMENTED_JUMP_BACKWARD) {
- _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr;
+ #if defined(Py_TAIL_CALL_INTERP)
+ int opcode;
+ #endif
+ opcode = INSTRUMENTED_JUMP_BACKWARD;
+ (void)(opcode);
+ _Py_CODEUNIT* const this_instr = next_instr;
(void)this_instr;
+ frame->instr_ptr = next_instr;
next_instr += 2;
INSTRUCTION_STATS(INSTRUMENTED_JUMP_BACKWARD);
/* Skip 1 cache entry */
int err = _Py_HandlePending(tstate);
stack_pointer = _PyFrame_GetStackPointer(frame);
if (err != 0) {
- goto error;
+ JUMP_TO_LABEL(error);
}
}
}
}
TARGET(INSTRUMENTED_JUMP_FORWARD) {
- _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr;
+ #if defined(Py_TAIL_CALL_INTERP)
+ int opcode;
+ #endif
+ opcode = INSTRUMENTED_JUMP_FORWARD;
+ (void)(opcode);
+ _Py_CODEUNIT* const this_instr = next_instr;
(void)this_instr;
+ frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(INSTRUMENTED_JUMP_FORWARD);
INSTRUMENTED_JUMP(this_instr, next_instr + oparg, PY_MONITORING_EVENT_JUMP);
}
TARGET(INSTRUMENTED_LINE) {
+ #if defined(Py_TAIL_CALL_INTERP)
+ int opcode;
+ #endif
+ opcode = INSTRUMENTED_LINE;
+ (void)(opcode);
_Py_CODEUNIT* const prev_instr = frame->instr_ptr;
- _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr;
+ _Py_CODEUNIT* const this_instr = next_instr;
(void)this_instr;
+ frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(INSTRUMENTED_LINE);
- opcode = INSTRUMENTED_LINE;
int original_opcode = 0;
if (tstate->tracing) {
PyCodeObject *code = _PyFrame_GetCode(frame);
stack_pointer = _PyFrame_GetStackPointer(frame);
if (original_opcode < 0) {
next_instr = this_instr+1;
- goto error;
+ JUMP_TO_LABEL(error);
}
next_instr = frame->instr_ptr;
if (next_instr != this_instr) {
}
TARGET(INSTRUMENTED_LOAD_SUPER_ATTR) {
- _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr;
+ #if defined(Py_TAIL_CALL_INTERP)
+ int opcode;
+ #endif
+ opcode = INSTRUMENTED_LOAD_SUPER_ATTR;
+ (void)(opcode);
+ _Py_CODEUNIT* const this_instr = next_instr;
(void)this_instr;
+ frame->instr_ptr = next_instr;
next_instr += 2;
INSTRUCTION_STATS(INSTRUMENTED_LOAD_SUPER_ATTR);
- opcode = INSTRUMENTED_LOAD_SUPER_ATTR;
_PyStackRef global_super_st;
_PyStackRef class_st;
_PyStackRef self_st;
PyStackRef_CLOSE(global_super_st);
PyStackRef_CLOSE(class_st);
PyStackRef_CLOSE(self_st);
- goto pop_3_error;
+ JUMP_TO_LABEL(pop_3_error);
}
}
// we make no attempt to optimize here; specializations should
PyStackRef_CLOSE(class_st);
PyStackRef_CLOSE(self_st);
if (super == NULL) {
- goto pop_3_error;
+ JUMP_TO_LABEL(pop_3_error);
}
PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 2);
stack_pointer += -3;
Py_DECREF(super);
stack_pointer = _PyFrame_GetStackPointer(frame);
if (attr_o == NULL) {
- goto error;
+ JUMP_TO_LABEL(error);
}
attr = PyStackRef_FromPyObjectSteal(attr_o);
}
}
TARGET(INSTRUMENTED_NOT_TAKEN) {
+ #if defined(Py_TAIL_CALL_INTERP)
+ int opcode;
+ #endif
+ opcode = INSTRUMENTED_NOT_TAKEN;
+ (void)(opcode);
_Py_CODEUNIT* const prev_instr = frame->instr_ptr;
- _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr;
+ _Py_CODEUNIT* const this_instr = next_instr;
(void)this_instr;
+ frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(INSTRUMENTED_NOT_TAKEN);
(void)this_instr; // INSTRUMENTED_JUMP requires this_instr
}
TARGET(INSTRUMENTED_POP_ITER) {
+ #if defined(Py_TAIL_CALL_INTERP)
+ int opcode;
+ #endif
+ opcode = INSTRUMENTED_POP_ITER;
+ (void)(opcode);
_Py_CODEUNIT* const prev_instr = frame->instr_ptr;
- _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr;
+ _Py_CODEUNIT* const this_instr = next_instr;
(void)this_instr;
+ frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(INSTRUMENTED_POP_ITER);
_PyStackRef iter;
}
TARGET(INSTRUMENTED_POP_JUMP_IF_FALSE) {
- _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr;
+ #if defined(Py_TAIL_CALL_INTERP)
+ int opcode;
+ #endif
+ opcode = INSTRUMENTED_POP_JUMP_IF_FALSE;
+ (void)(opcode);
+ _Py_CODEUNIT* const this_instr = next_instr;
(void)this_instr;
+ frame->instr_ptr = next_instr;
next_instr += 2;
INSTRUCTION_STATS(INSTRUMENTED_POP_JUMP_IF_FALSE);
/* Skip 1 cache entry */
}
TARGET(INSTRUMENTED_POP_JUMP_IF_NONE) {
- _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr;
+ #if defined(Py_TAIL_CALL_INTERP)
+ int opcode;
+ #endif
+ opcode = INSTRUMENTED_POP_JUMP_IF_NONE;
+ (void)(opcode);
+ _Py_CODEUNIT* const this_instr = next_instr;
(void)this_instr;
+ frame->instr_ptr = next_instr;
next_instr += 2;
INSTRUCTION_STATS(INSTRUMENTED_POP_JUMP_IF_NONE);
/* Skip 1 cache entry */
}
TARGET(INSTRUMENTED_POP_JUMP_IF_NOT_NONE) {
- _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr;
+ #if defined(Py_TAIL_CALL_INTERP)
+ int opcode;
+ #endif
+ opcode = INSTRUMENTED_POP_JUMP_IF_NOT_NONE;
+ (void)(opcode);
+ _Py_CODEUNIT* const this_instr = next_instr;
(void)this_instr;
+ frame->instr_ptr = next_instr;
next_instr += 2;
INSTRUCTION_STATS(INSTRUMENTED_POP_JUMP_IF_NOT_NONE);
/* Skip 1 cache entry */
}
TARGET(INSTRUMENTED_POP_JUMP_IF_TRUE) {
- _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr;
+ #if defined(Py_TAIL_CALL_INTERP)
+ int opcode;
+ #endif
+ opcode = INSTRUMENTED_POP_JUMP_IF_TRUE;
+ (void)(opcode);
+ _Py_CODEUNIT* const this_instr = next_instr;
(void)this_instr;
+ frame->instr_ptr = next_instr;
next_instr += 2;
INSTRUCTION_STATS(INSTRUMENTED_POP_JUMP_IF_TRUE);
/* Skip 1 cache entry */
}
TARGET(INSTRUMENTED_RESUME) {
- _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr;
+ #if defined(Py_TAIL_CALL_INTERP)
+ int opcode;
+ #endif
+ opcode = INSTRUMENTED_RESUME;
+ (void)(opcode);
+ _Py_CODEUNIT* const this_instr = next_instr;
(void)this_instr;
+ frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(INSTRUMENTED_RESUME);
// _LOAD_BYTECODE
_PyEval_GetExecutableCode(tstate, _PyFrame_GetCode(frame));
stack_pointer = _PyFrame_GetStackPointer(frame);
if (bytecode == NULL) {
- goto error;
+ JUMP_TO_LABEL(error);
}
_PyFrame_SetStackPointer(frame, stack_pointer);
ptrdiff_t off = this_instr - _PyFrame_GetBytecode(frame);
int err = _Py_Instrument(_PyFrame_GetCode(frame), tstate->interp);
stack_pointer = _PyFrame_GetStackPointer(frame);
if (err) {
- goto error;
+ JUMP_TO_LABEL(error);
}
next_instr = this_instr;
DISPATCH();
int err = _Py_HandlePending(tstate);
stack_pointer = _PyFrame_GetStackPointer(frame);
if (err != 0) {
- goto error;
+ JUMP_TO_LABEL(error);
}
}
}
tstate, oparg > 0, frame, this_instr);
stack_pointer = _PyFrame_GetStackPointer(frame);
if (err) {
- goto error;
+ JUMP_TO_LABEL(error);
}
if (frame->instr_ptr != this_instr) {
/* Instrumentation has jumped */
}
TARGET(INSTRUMENTED_RETURN_VALUE) {
- _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr;
+ #if defined(Py_TAIL_CALL_INTERP)
+ int opcode;
+ #endif
+ opcode = INSTRUMENTED_RETURN_VALUE;
+ (void)(opcode);
+ _Py_CODEUNIT* const this_instr = next_instr;
(void)this_instr;
+ frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(INSTRUMENTED_RETURN_VALUE);
_PyStackRef val;
frame, this_instr, PyStackRef_AsPyObjectBorrow(val));
stack_pointer = _PyFrame_GetStackPointer(frame);
if (err) {
- goto error;
+ JUMP_TO_LABEL(error);
}
}
// _RETURN_VALUE
}
TARGET(INSTRUMENTED_YIELD_VALUE) {
- _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr;
+ #if defined(Py_TAIL_CALL_INTERP)
+ int opcode;
+ #endif
+ opcode = INSTRUMENTED_YIELD_VALUE;
+ (void)(opcode);
+ _Py_CODEUNIT* const this_instr = next_instr;
(void)this_instr;
+ frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(INSTRUMENTED_YIELD_VALUE);
_PyStackRef val;
frame, this_instr, PyStackRef_AsPyObjectBorrow(val));
stack_pointer = _PyFrame_GetStackPointer(frame);
if (err) {
- goto error;
+ JUMP_TO_LABEL(error);
}
if (frame->instr_ptr != this_instr) {
next_instr = frame->instr_ptr;
}
TARGET(INTERPRETER_EXIT) {
+ #if defined(Py_TAIL_CALL_INTERP)
+ int opcode;
+ #endif
+ opcode = INTERPRETER_EXIT;
+ (void)(opcode);
frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(INTERPRETER_EXIT);
}
TARGET(IS_OP) {
+ #if defined(Py_TAIL_CALL_INTERP)
+ int opcode;
+ #endif
+ opcode = IS_OP;
+ (void)(opcode);
frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(IS_OP);
}
TARGET(JUMP_BACKWARD) {
+ #if defined(Py_TAIL_CALL_INTERP)
+ int opcode;
+ #endif
+ opcode = JUMP_BACKWARD;
+ (void)(opcode);
frame->instr_ptr = next_instr;
next_instr += 2;
INSTRUCTION_STATS(JUMP_BACKWARD);
int err = _Py_HandlePending(tstate);
stack_pointer = _PyFrame_GetStackPointer(frame);
if (err != 0) {
- goto error;
+ JUMP_TO_LABEL(error);
}
}
}
}
TARGET(JUMP_BACKWARD_JIT) {
- _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr;
+ #if defined(Py_TAIL_CALL_INTERP)
+ int opcode;
+ #endif
+ opcode = JUMP_BACKWARD_JIT;
+ (void)(opcode);
+ _Py_CODEUNIT* const this_instr = next_instr;
(void)this_instr;
+ frame->instr_ptr = next_instr;
next_instr += 2;
INSTRUCTION_STATS(JUMP_BACKWARD_JIT);
static_assert(1 == 1, "incorrect cache size");
int err = _Py_HandlePending(tstate);
stack_pointer = _PyFrame_GetStackPointer(frame);
if (err != 0) {
- goto error;
+ JUMP_TO_LABEL(error);
}
}
}
if (optimized <= 0) {
this_instr[1].counter = restart_backoff_counter(counter);
if (optimized < 0) {
- goto error;
+ JUMP_TO_LABEL(error);
}
}
else {
}
TARGET(JUMP_BACKWARD_NO_INTERRUPT) {
+ #if defined(Py_TAIL_CALL_INTERP)
+ int opcode;
+ #endif
+ opcode = JUMP_BACKWARD_NO_INTERRUPT;
+ (void)(opcode);
frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(JUMP_BACKWARD_NO_INTERRUPT);
}
TARGET(JUMP_BACKWARD_NO_JIT) {
+ #if defined(Py_TAIL_CALL_INTERP)
+ int opcode;
+ #endif
+ opcode = JUMP_BACKWARD_NO_JIT;
+ (void)(opcode);
frame->instr_ptr = next_instr;
next_instr += 2;
INSTRUCTION_STATS(JUMP_BACKWARD_NO_JIT);
int err = _Py_HandlePending(tstate);
stack_pointer = _PyFrame_GetStackPointer(frame);
if (err != 0) {
- goto error;
+ JUMP_TO_LABEL(error);
}
}
}
}
TARGET(JUMP_FORWARD) {
+ #if defined(Py_TAIL_CALL_INTERP)
+ int opcode;
+ #endif
+ opcode = JUMP_FORWARD;
+ (void)(opcode);
frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(JUMP_FORWARD);
}
TARGET(LIST_APPEND) {
+ #if defined(Py_TAIL_CALL_INTERP)
+ int opcode;
+ #endif
+ opcode = LIST_APPEND;
+ (void)(opcode);
frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(LIST_APPEND);
int err = _PyList_AppendTakeRef((PyListObject *)PyStackRef_AsPyObjectBorrow(list),
PyStackRef_AsPyObjectSteal(v));
if (err < 0) {
- goto pop_1_error;
+ JUMP_TO_LABEL(pop_1_error);
}
stack_pointer += -1;
assert(WITHIN_STACK_BOUNDS());
}
TARGET(LIST_EXTEND) {
+ #if defined(Py_TAIL_CALL_INTERP)
+ int opcode;
+ #endif
+ opcode = LIST_EXTEND;
+ (void)(opcode);
frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(LIST_EXTEND);
stack_pointer = _PyFrame_GetStackPointer(frame);
}
PyStackRef_CLOSE(iterable_st);
- goto pop_1_error;
+ JUMP_TO_LABEL(pop_1_error);
}
assert(Py_IsNone(none_val));
PyStackRef_CLOSE(iterable_st);
}
TARGET(LOAD_ATTR) {
+ #if defined(Py_TAIL_CALL_INTERP)
+ int opcode;
+ #endif
+ opcode = LOAD_ATTR;
+ (void)(opcode);
frame->instr_ptr = next_instr;
next_instr += 10;
INSTRUCTION_STATS(LOAD_ATTR);
*/
PyStackRef_CLOSE(owner);
if (attr_o == NULL) {
- goto pop_1_error;
+ JUMP_TO_LABEL(pop_1_error);
}
self_or_null[0] = PyStackRef_NULL;
}
stack_pointer = _PyFrame_GetStackPointer(frame);
PyStackRef_CLOSE(owner);
if (attr_o == NULL) {
- goto pop_1_error;
+ JUMP_TO_LABEL(pop_1_error);
}
}
attr = PyStackRef_FromPyObjectSteal(attr_o);
}
TARGET(LOAD_ATTR_CLASS) {
- _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr;
+ #if defined(Py_TAIL_CALL_INTERP)
+ int opcode;
+ #endif
+ opcode = LOAD_ATTR_CLASS;
+ (void)(opcode);
+ _Py_CODEUNIT* const this_instr = next_instr;
(void)this_instr;
+ frame->instr_ptr = next_instr;
next_instr += 10;
INSTRUCTION_STATS(LOAD_ATTR_CLASS);
static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size");
owner = stack_pointer[-1];
uint32_t type_version = read_u32(&this_instr[2].cache);
PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner);
- DEOPT_IF(!PyType_Check(owner_o), LOAD_ATTR);
+ if (!PyType_Check(owner_o)) {
+ UPDATE_MISS_STATS(LOAD_ATTR);
+ assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR));
+ JUMP_TO_PREDICTED(LOAD_ATTR);
+ }
assert(type_version != 0);
- DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(((PyTypeObject *)owner_o)->tp_version_tag) != type_version, LOAD_ATTR);
+ if (FT_ATOMIC_LOAD_UINT_RELAXED(((PyTypeObject *)owner_o)->tp_version_tag) != type_version) {
+ UPDATE_MISS_STATS(LOAD_ATTR);
+ assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR));
+ JUMP_TO_PREDICTED(LOAD_ATTR);
+ }
}
/* Skip 2 cache entries */
// _LOAD_ATTR_CLASS
}
TARGET(LOAD_ATTR_CLASS_WITH_METACLASS_CHECK) {
- _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr;
+ #if defined(Py_TAIL_CALL_INTERP)
+ int opcode;
+ #endif
+ opcode = LOAD_ATTR_CLASS_WITH_METACLASS_CHECK;
+ (void)(opcode);
+ _Py_CODEUNIT* const this_instr = next_instr;
(void)this_instr;
+ frame->instr_ptr = next_instr;
next_instr += 10;
INSTRUCTION_STATS(LOAD_ATTR_CLASS_WITH_METACLASS_CHECK);
static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size");
owner = stack_pointer[-1];
uint32_t type_version = read_u32(&this_instr[2].cache);
PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner);
- DEOPT_IF(!PyType_Check(owner_o), LOAD_ATTR);
+ if (!PyType_Check(owner_o)) {
+ UPDATE_MISS_STATS(LOAD_ATTR);
+ assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR));
+ JUMP_TO_PREDICTED(LOAD_ATTR);
+ }
assert(type_version != 0);
- DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(((PyTypeObject *)owner_o)->tp_version_tag) != type_version, LOAD_ATTR);
+ if (FT_ATOMIC_LOAD_UINT_RELAXED(((PyTypeObject *)owner_o)->tp_version_tag) != type_version) {
+ UPDATE_MISS_STATS(LOAD_ATTR);
+ assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR));
+ JUMP_TO_PREDICTED(LOAD_ATTR);
+ }
}
// _GUARD_TYPE_VERSION
{
uint32_t type_version = read_u32(&this_instr[4].cache);
PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner));
assert(type_version != 0);
- DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, LOAD_ATTR);
+ if (FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version) {
+ UPDATE_MISS_STATS(LOAD_ATTR);
+ assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR));
+ JUMP_TO_PREDICTED(LOAD_ATTR);
+ }
}
// _LOAD_ATTR_CLASS
{
}
TARGET(LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN) {
- _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr;
+ #if defined(Py_TAIL_CALL_INTERP)
+ int opcode;
+ #endif
+ opcode = LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN;
+ (void)(opcode);
+ _Py_CODEUNIT* const this_instr = next_instr;
(void)this_instr;
+ frame->instr_ptr = next_instr;
next_instr += 10;
INSTRUCTION_STATS(LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN);
static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size");
PyObject *getattribute = read_obj(&this_instr[6].cache);
PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner);
assert((oparg & 1) == 0);
- DEOPT_IF(tstate->interp->eval_frame, LOAD_ATTR);
+ if (tstate->interp->eval_frame) {
+ UPDATE_MISS_STATS(LOAD_ATTR);
+ assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR));
+ JUMP_TO_PREDICTED(LOAD_ATTR);
+ }
PyTypeObject *cls = Py_TYPE(owner_o);
assert(type_version != 0);
- DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(cls->tp_version_tag) != type_version, LOAD_ATTR);
+ if (FT_ATOMIC_LOAD_UINT_RELAXED(cls->tp_version_tag) != type_version) {
+ UPDATE_MISS_STATS(LOAD_ATTR);
+ assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR));
+ JUMP_TO_PREDICTED(LOAD_ATTR);
+ }
assert(Py_IS_TYPE(getattribute, &PyFunction_Type));
PyFunctionObject *f = (PyFunctionObject *)getattribute;
assert(func_version != 0);
- DEOPT_IF(f->func_version != func_version, LOAD_ATTR);
+ if (f->func_version != func_version) {
+ UPDATE_MISS_STATS(LOAD_ATTR);
+ assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR));
+ JUMP_TO_PREDICTED(LOAD_ATTR);
+ }
PyCodeObject *code = (PyCodeObject *)f->func_code;
assert(code->co_argcount == 2);
- DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize), LOAD_ATTR);
+ if (!_PyThreadState_HasStackSpace(tstate, code->co_framesize)) {
+ UPDATE_MISS_STATS(LOAD_ATTR);
+ assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR));
+ JUMP_TO_PREDICTED(LOAD_ATTR);
+ }
STAT_INC(LOAD_ATTR, hit);
PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 1);
_PyInterpreterFrame *new_frame = _PyFrame_PushUnchecked(
}
TARGET(LOAD_ATTR_INSTANCE_VALUE) {
- _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr;
+ #if defined(Py_TAIL_CALL_INTERP)
+ int opcode;
+ #endif
+ opcode = LOAD_ATTR_INSTANCE_VALUE;
+ (void)(opcode);
+ _Py_CODEUNIT* const this_instr = next_instr;
(void)this_instr;
+ frame->instr_ptr = next_instr;
next_instr += 10;
INSTRUCTION_STATS(LOAD_ATTR_INSTANCE_VALUE);
static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size");
uint32_t type_version = read_u32(&this_instr[2].cache);
PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner));
assert(type_version != 0);
- DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, LOAD_ATTR);
+ if (FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version) {
+ UPDATE_MISS_STATS(LOAD_ATTR);
+ assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR));
+ JUMP_TO_PREDICTED(LOAD_ATTR);
+ }
}
// _CHECK_MANAGED_OBJECT_HAS_VALUES
{
PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner);
assert(Py_TYPE(owner_o)->tp_dictoffset < 0);
assert(Py_TYPE(owner_o)->tp_flags & Py_TPFLAGS_INLINE_VALUES);
- DEOPT_IF(!FT_ATOMIC_LOAD_UINT8(_PyObject_InlineValues(owner_o)->valid), LOAD_ATTR);
+ if (!FT_ATOMIC_LOAD_UINT8(_PyObject_InlineValues(owner_o)->valid)) {
+ UPDATE_MISS_STATS(LOAD_ATTR);
+ assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR));
+ JUMP_TO_PREDICTED(LOAD_ATTR);
+ }
}
// _LOAD_ATTR_INSTANCE_VALUE
{
PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner);
PyObject **value_ptr = (PyObject**)(((char *)owner_o) + offset);
PyObject *attr_o = FT_ATOMIC_LOAD_PTR_ACQUIRE(*value_ptr);
- DEOPT_IF(attr_o == NULL, LOAD_ATTR);
+ if (attr_o == NULL) {
+ UPDATE_MISS_STATS(LOAD_ATTR);
+ assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR));
+ JUMP_TO_PREDICTED(LOAD_ATTR);
+ }
#ifdef Py_GIL_DISABLED
if (!_Py_TryIncrefCompareStackRef(value_ptr, attr_o, &attr)) {
- DEOPT_IF(true, LOAD_ATTR);
+ if (true) {
+ UPDATE_MISS_STATS(LOAD_ATTR);
+ assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR));
+ JUMP_TO_PREDICTED(LOAD_ATTR);
+ }
}
#else
attr = PyStackRef_FromPyObjectNew(attr_o);
}
TARGET(LOAD_ATTR_METHOD_LAZY_DICT) {
- _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr;
+ #if defined(Py_TAIL_CALL_INTERP)
+ int opcode;
+ #endif
+ opcode = LOAD_ATTR_METHOD_LAZY_DICT;
+ (void)(opcode);
+ _Py_CODEUNIT* const this_instr = next_instr;
(void)this_instr;
+ frame->instr_ptr = next_instr;
next_instr += 10;
INSTRUCTION_STATS(LOAD_ATTR_METHOD_LAZY_DICT);
static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size");
uint32_t type_version = read_u32(&this_instr[2].cache);
PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner));
assert(type_version != 0);
- DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, LOAD_ATTR);
+ if (FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version) {
+ UPDATE_MISS_STATS(LOAD_ATTR);
+ assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR));
+ JUMP_TO_PREDICTED(LOAD_ATTR);
+ }
}
// _CHECK_ATTR_METHOD_LAZY_DICT
{
char *ptr = ((char *)PyStackRef_AsPyObjectBorrow(owner)) + MANAGED_DICT_OFFSET + dictoffset;
PyObject *dict = FT_ATOMIC_LOAD_PTR_ACQUIRE(*(PyObject **)ptr);
/* This object has a __dict__, just not yet created */
- DEOPT_IF(dict != NULL, LOAD_ATTR);
+ if (dict != NULL) {
+ UPDATE_MISS_STATS(LOAD_ATTR);
+ assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR));
+ JUMP_TO_PREDICTED(LOAD_ATTR);
+ }
}
/* Skip 1 cache entry */
// _LOAD_ATTR_METHOD_LAZY_DICT
}
TARGET(LOAD_ATTR_METHOD_NO_DICT) {
- _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr;
+ #if defined(Py_TAIL_CALL_INTERP)
+ int opcode;
+ #endif
+ opcode = LOAD_ATTR_METHOD_NO_DICT;
+ (void)(opcode);
+ _Py_CODEUNIT* const this_instr = next_instr;
(void)this_instr;
+ frame->instr_ptr = next_instr;
next_instr += 10;
INSTRUCTION_STATS(LOAD_ATTR_METHOD_NO_DICT);
static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size");
uint32_t type_version = read_u32(&this_instr[2].cache);
PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner));
assert(type_version != 0);
- DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, LOAD_ATTR);
+ if (FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version) {
+ UPDATE_MISS_STATS(LOAD_ATTR);
+ assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR));
+ JUMP_TO_PREDICTED(LOAD_ATTR);
+ }
}
/* Skip 2 cache entries */
// _LOAD_ATTR_METHOD_NO_DICT
}
TARGET(LOAD_ATTR_METHOD_WITH_VALUES) {
- _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr;
+ #if defined(Py_TAIL_CALL_INTERP)
+ int opcode;
+ #endif
+ opcode = LOAD_ATTR_METHOD_WITH_VALUES;
+ (void)(opcode);
+ _Py_CODEUNIT* const this_instr = next_instr;
(void)this_instr;
+ frame->instr_ptr = next_instr;
next_instr += 10;
INSTRUCTION_STATS(LOAD_ATTR_METHOD_WITH_VALUES);
static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size");
uint32_t type_version = read_u32(&this_instr[2].cache);
PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner));
assert(type_version != 0);
- DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, LOAD_ATTR);
+ if (FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version) {
+ UPDATE_MISS_STATS(LOAD_ATTR);
+ assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR));
+ JUMP_TO_PREDICTED(LOAD_ATTR);
+ }
}
// _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT
{
PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner);
assert(Py_TYPE(owner_o)->tp_flags & Py_TPFLAGS_INLINE_VALUES);
PyDictValues *ivs = _PyObject_InlineValues(owner_o);
- DEOPT_IF(!FT_ATOMIC_LOAD_UINT8(ivs->valid), LOAD_ATTR);
+ if (!FT_ATOMIC_LOAD_UINT8(ivs->valid)) {
+ UPDATE_MISS_STATS(LOAD_ATTR);
+ assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR));
+ JUMP_TO_PREDICTED(LOAD_ATTR);
+ }
}
// _GUARD_KEYS_VERSION
{
PyTypeObject *owner_cls = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner));
PyHeapTypeObject *owner_heap_type = (PyHeapTypeObject *)owner_cls;
PyDictKeysObject *keys = owner_heap_type->ht_cached_keys;
- DEOPT_IF(FT_ATOMIC_LOAD_UINT32_RELAXED(keys->dk_version) != keys_version, LOAD_ATTR);
+ if (FT_ATOMIC_LOAD_UINT32_RELAXED(keys->dk_version) != keys_version) {
+ UPDATE_MISS_STATS(LOAD_ATTR);
+ assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR));
+ JUMP_TO_PREDICTED(LOAD_ATTR);
+ }
}
// _LOAD_ATTR_METHOD_WITH_VALUES
{
}
TARGET(LOAD_ATTR_MODULE) {
- _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr;
+ #if defined(Py_TAIL_CALL_INTERP)
+ int opcode;
+ #endif
+ opcode = LOAD_ATTR_MODULE;
+ (void)(opcode);
+ _Py_CODEUNIT* const this_instr = next_instr;
(void)this_instr;
+ frame->instr_ptr = next_instr;
next_instr += 10;
INSTRUCTION_STATS(LOAD_ATTR_MODULE);
static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size");
owner = stack_pointer[-1];
uint32_t dict_version = read_u32(&this_instr[2].cache);
PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner);
- DEOPT_IF(Py_TYPE(owner_o)->tp_getattro != PyModule_Type.tp_getattro, LOAD_ATTR);
+ if (Py_TYPE(owner_o)->tp_getattro != PyModule_Type.tp_getattro) {
+ UPDATE_MISS_STATS(LOAD_ATTR);
+ assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR));
+ JUMP_TO_PREDICTED(LOAD_ATTR);
+ }
PyDictObject *dict = (PyDictObject *)((PyModuleObject *)owner_o)->md_dict;
assert(dict != NULL);
PyDictKeysObject *keys = FT_ATOMIC_LOAD_PTR_ACQUIRE(dict->ma_keys);
- DEOPT_IF(FT_ATOMIC_LOAD_UINT32_RELAXED(keys->dk_version) != dict_version, LOAD_ATTR);
+ if (FT_ATOMIC_LOAD_UINT32_RELAXED(keys->dk_version) != dict_version) {
+ UPDATE_MISS_STATS(LOAD_ATTR);
+ assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR));
+ JUMP_TO_PREDICTED(LOAD_ATTR);
+ }
mod_keys = keys;
}
// _LOAD_ATTR_MODULE_FROM_KEYS
PyDictUnicodeEntry *ep = DK_UNICODE_ENTRIES(mod_keys) + index;
PyObject *attr_o = FT_ATOMIC_LOAD_PTR_RELAXED(ep->me_value);
// Clear mod_keys from stack in case we need to deopt
- DEOPT_IF(attr_o == NULL, LOAD_ATTR);
+ if (attr_o == NULL) {
+ UPDATE_MISS_STATS(LOAD_ATTR);
+ assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR));
+ JUMP_TO_PREDICTED(LOAD_ATTR);
+ }
#ifdef Py_GIL_DISABLED
int increfed = _Py_TryIncrefCompareStackRef(&ep->me_value, attr_o, &attr);
if (!increfed) {
- DEOPT_IF(true, LOAD_ATTR);
+ if (true) {
+ UPDATE_MISS_STATS(LOAD_ATTR);
+ assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR));
+ JUMP_TO_PREDICTED(LOAD_ATTR);
+ }
}
#else
Py_INCREF(attr_o);
}
TARGET(LOAD_ATTR_NONDESCRIPTOR_NO_DICT) {
- _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr;
+ #if defined(Py_TAIL_CALL_INTERP)
+ int opcode;
+ #endif
+ opcode = LOAD_ATTR_NONDESCRIPTOR_NO_DICT;
+ (void)(opcode);
+ _Py_CODEUNIT* const this_instr = next_instr;
(void)this_instr;
+ frame->instr_ptr = next_instr;
next_instr += 10;
INSTRUCTION_STATS(LOAD_ATTR_NONDESCRIPTOR_NO_DICT);
static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size");
uint32_t type_version = read_u32(&this_instr[2].cache);
PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner));
assert(type_version != 0);
- DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, LOAD_ATTR);
+ if (FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version) {
+ UPDATE_MISS_STATS(LOAD_ATTR);
+ assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR));
+ JUMP_TO_PREDICTED(LOAD_ATTR);
+ }
}
/* Skip 2 cache entries */
// _LOAD_ATTR_NONDESCRIPTOR_NO_DICT
}
TARGET(LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES) {
- _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr;
+ #if defined(Py_TAIL_CALL_INTERP)
+ int opcode;
+ #endif
+ opcode = LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES;
+ (void)(opcode);
+ _Py_CODEUNIT* const this_instr = next_instr;
(void)this_instr;
+ frame->instr_ptr = next_instr;
next_instr += 10;
INSTRUCTION_STATS(LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES);
static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size");
uint32_t type_version = read_u32(&this_instr[2].cache);
PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner));
assert(type_version != 0);
- DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, LOAD_ATTR);
+ if (FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version) {
+ UPDATE_MISS_STATS(LOAD_ATTR);
+ assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR));
+ JUMP_TO_PREDICTED(LOAD_ATTR);
+ }
}
// _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT
{
PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner);
assert(Py_TYPE(owner_o)->tp_flags & Py_TPFLAGS_INLINE_VALUES);
PyDictValues *ivs = _PyObject_InlineValues(owner_o);
- DEOPT_IF(!FT_ATOMIC_LOAD_UINT8(ivs->valid), LOAD_ATTR);
+ if (!FT_ATOMIC_LOAD_UINT8(ivs->valid)) {
+ UPDATE_MISS_STATS(LOAD_ATTR);
+ assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR));
+ JUMP_TO_PREDICTED(LOAD_ATTR);
+ }
}
// _GUARD_KEYS_VERSION
{
PyTypeObject *owner_cls = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner));
PyHeapTypeObject *owner_heap_type = (PyHeapTypeObject *)owner_cls;
PyDictKeysObject *keys = owner_heap_type->ht_cached_keys;
- DEOPT_IF(FT_ATOMIC_LOAD_UINT32_RELAXED(keys->dk_version) != keys_version, LOAD_ATTR);
+ if (FT_ATOMIC_LOAD_UINT32_RELAXED(keys->dk_version) != keys_version) {
+ UPDATE_MISS_STATS(LOAD_ATTR);
+ assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR));
+ JUMP_TO_PREDICTED(LOAD_ATTR);
+ }
}
// _LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES
{
}
TARGET(LOAD_ATTR_PROPERTY) {
- _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr;
+ #if defined(Py_TAIL_CALL_INTERP)
+ int opcode;
+ #endif
+ opcode = LOAD_ATTR_PROPERTY;
+ (void)(opcode);
+ _Py_CODEUNIT* const this_instr = next_instr;
(void)this_instr;
+ frame->instr_ptr = next_instr;
next_instr += 10;
INSTRUCTION_STATS(LOAD_ATTR_PROPERTY);
static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size");
/* Skip 1 cache entry */
// _CHECK_PEP_523
{
- DEOPT_IF(tstate->interp->eval_frame, LOAD_ATTR);
+ if (tstate->interp->eval_frame) {
+ UPDATE_MISS_STATS(LOAD_ATTR);
+ assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR));
+ JUMP_TO_PREDICTED(LOAD_ATTR);
+ }
}
// _GUARD_TYPE_VERSION
{
uint32_t type_version = read_u32(&this_instr[2].cache);
PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner));
assert(type_version != 0);
- DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, LOAD_ATTR);
+ if (FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version) {
+ UPDATE_MISS_STATS(LOAD_ATTR);
+ assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR));
+ JUMP_TO_PREDICTED(LOAD_ATTR);
+ }
}
/* Skip 2 cache entries */
// _LOAD_ATTR_PROPERTY_FRAME
assert(Py_IS_TYPE(fget, &PyFunction_Type));
PyFunctionObject *f = (PyFunctionObject *)fget;
PyCodeObject *code = (PyCodeObject *)f->func_code;
- DEOPT_IF((code->co_flags & (CO_VARKEYWORDS | CO_VARARGS | CO_OPTIMIZED)) != CO_OPTIMIZED, LOAD_ATTR);
- DEOPT_IF(code->co_kwonlyargcount, LOAD_ATTR);
- DEOPT_IF(code->co_argcount != 1, LOAD_ATTR);
- DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize), LOAD_ATTR);
+ if ((code->co_flags & (CO_VARKEYWORDS | CO_VARARGS | CO_OPTIMIZED)) != CO_OPTIMIZED) {
+ UPDATE_MISS_STATS(LOAD_ATTR);
+ assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR));
+ JUMP_TO_PREDICTED(LOAD_ATTR);
+ }
+ if (code->co_kwonlyargcount) {
+ UPDATE_MISS_STATS(LOAD_ATTR);
+ assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR));
+ JUMP_TO_PREDICTED(LOAD_ATTR);
+ }
+ if (code->co_argcount != 1) {
+ UPDATE_MISS_STATS(LOAD_ATTR);
+ assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR));
+ JUMP_TO_PREDICTED(LOAD_ATTR);
+ }
+ if (!_PyThreadState_HasStackSpace(tstate, code->co_framesize)) {
+ UPDATE_MISS_STATS(LOAD_ATTR);
+ assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR));
+ JUMP_TO_PREDICTED(LOAD_ATTR);
+ }
STAT_INC(LOAD_ATTR, hit);
new_frame = _PyFrame_PushUnchecked(tstate, PyStackRef_FromPyObjectNew(fget), 1, frame);
new_frame->localsplus[0] = owner;
}
TARGET(LOAD_ATTR_SLOT) {
- _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr;
+ #if defined(Py_TAIL_CALL_INTERP)
+ int opcode;
+ #endif
+ opcode = LOAD_ATTR_SLOT;
+ (void)(opcode);
+ _Py_CODEUNIT* const this_instr = next_instr;
(void)this_instr;
+ frame->instr_ptr = next_instr;
next_instr += 10;
INSTRUCTION_STATS(LOAD_ATTR_SLOT);
static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size");
uint32_t type_version = read_u32(&this_instr[2].cache);
PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner));
assert(type_version != 0);
- DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, LOAD_ATTR);
+ if (FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version) {
+ UPDATE_MISS_STATS(LOAD_ATTR);
+ assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR));
+ JUMP_TO_PREDICTED(LOAD_ATTR);
+ }
}
// _LOAD_ATTR_SLOT
{
PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner);
PyObject **addr = (PyObject **)((char *)owner_o + index);
PyObject *attr_o = FT_ATOMIC_LOAD_PTR(*addr);
- DEOPT_IF(attr_o == NULL, LOAD_ATTR);
+ if (attr_o == NULL) {
+ UPDATE_MISS_STATS(LOAD_ATTR);
+ assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR));
+ JUMP_TO_PREDICTED(LOAD_ATTR);
+ }
#ifdef Py_GIL_DISABLED
int increfed = _Py_TryIncrefCompareStackRef(addr, attr_o, &attr);
- DEOPT_IF(!increfed, LOAD_ATTR);
+ if (!increfed) {
+ UPDATE_MISS_STATS(LOAD_ATTR);
+ assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR));
+ JUMP_TO_PREDICTED(LOAD_ATTR);
+ }
#else
attr = PyStackRef_FromPyObjectNew(attr_o);
#endif
}
TARGET(LOAD_ATTR_WITH_HINT) {
- _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr;
+ #if defined(Py_TAIL_CALL_INTERP)
+ int opcode;
+ #endif
+ opcode = LOAD_ATTR_WITH_HINT;
+ (void)(opcode);
+ _Py_CODEUNIT* const this_instr = next_instr;
(void)this_instr;
+ frame->instr_ptr = next_instr;
next_instr += 10;
INSTRUCTION_STATS(LOAD_ATTR_WITH_HINT);
static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size");
uint32_t type_version = read_u32(&this_instr[2].cache);
PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner));
assert(type_version != 0);
- DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, LOAD_ATTR);
+ if (FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version) {
+ UPDATE_MISS_STATS(LOAD_ATTR);
+ assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR));
+ JUMP_TO_PREDICTED(LOAD_ATTR);
+ }
}
// _CHECK_ATTR_WITH_HINT
{
PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner);
assert(Py_TYPE(owner_o)->tp_flags & Py_TPFLAGS_MANAGED_DICT);
PyDictObject *dict_o = _PyObject_GetManagedDict(owner_o);
- DEOPT_IF(dict_o == NULL, LOAD_ATTR);
+ if (dict_o == NULL) {
+ UPDATE_MISS_STATS(LOAD_ATTR);
+ assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR));
+ JUMP_TO_PREDICTED(LOAD_ATTR);
+ }
assert(PyDict_CheckExact((PyObject *)dict_o));
dict = dict_o;
}
uint16_t hint = read_u16(&this_instr[4].cache);
PyObject *attr_o;
if (!LOCK_OBJECT(dict)) {
- DEOPT_IF(true, LOAD_ATTR);
+ if (true) {
+ UPDATE_MISS_STATS(LOAD_ATTR);
+ assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR));
+ JUMP_TO_PREDICTED(LOAD_ATTR);
+ }
}
if (hint >= (size_t)dict->ma_keys->dk_nentries) {
UNLOCK_OBJECT(dict);
- DEOPT_IF(true, LOAD_ATTR);
+ if (true) {
+ UPDATE_MISS_STATS(LOAD_ATTR);
+ assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR));
+ JUMP_TO_PREDICTED(LOAD_ATTR);
+ }
}
PyObject *name = GETITEM(FRAME_CO_NAMES, oparg>>1);
if (dict->ma_keys->dk_kind != DICT_KEYS_UNICODE) {
UNLOCK_OBJECT(dict);
- DEOPT_IF(true, LOAD_ATTR);
+ if (true) {
+ UPDATE_MISS_STATS(LOAD_ATTR);
+ assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR));
+ JUMP_TO_PREDICTED(LOAD_ATTR);
+ }
}
PyDictUnicodeEntry *ep = DK_UNICODE_ENTRIES(dict->ma_keys) + hint;
if (ep->me_key != name) {
UNLOCK_OBJECT(dict);
- DEOPT_IF(true, LOAD_ATTR);
+ if (true) {
+ UPDATE_MISS_STATS(LOAD_ATTR);
+ assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR));
+ JUMP_TO_PREDICTED(LOAD_ATTR);
+ }
}
attr_o = ep->me_value;
if (attr_o == NULL) {
UNLOCK_OBJECT(dict);
- DEOPT_IF(true, LOAD_ATTR);
+ if (true) {
+ UPDATE_MISS_STATS(LOAD_ATTR);
+ assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR));
+ JUMP_TO_PREDICTED(LOAD_ATTR);
+ }
}
STAT_INC(LOAD_ATTR, hit);
attr = PyStackRef_FromPyObjectNew(attr_o);
}
TARGET(LOAD_BUILD_CLASS) {
+ #if defined(Py_TAIL_CALL_INTERP)
+ int opcode;
+ #endif
+ opcode = LOAD_BUILD_CLASS;
+ (void)(opcode);
frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(LOAD_BUILD_CLASS);
int err = PyMapping_GetOptionalItem(BUILTINS(), &_Py_ID(__build_class__), &bc_o);
stack_pointer = _PyFrame_GetStackPointer(frame);
if (err < 0) {
- goto error;
+ JUMP_TO_LABEL(error);
}
if (bc_o == NULL) {
_PyFrame_SetStackPointer(frame, stack_pointer);
_PyErr_SetString(tstate, PyExc_NameError,
"__build_class__ not found");
stack_pointer = _PyFrame_GetStackPointer(frame);
- goto error;
+ JUMP_TO_LABEL(error);
}
bc = PyStackRef_FromPyObjectSteal(bc_o);
stack_pointer[0] = bc;
}
TARGET(LOAD_COMMON_CONSTANT) {
+ #if defined(Py_TAIL_CALL_INTERP)
+ int opcode;
+ #endif
+ opcode = LOAD_COMMON_CONSTANT;
+ (void)(opcode);
frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(LOAD_COMMON_CONSTANT);
}
TARGET(LOAD_CONST) {
+ #if defined(Py_TAIL_CALL_INTERP)
+ int opcode;
+ #endif
+ opcode = LOAD_CONST;
+ (void)(opcode);
frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(LOAD_CONST);
}
TARGET(LOAD_CONST_IMMORTAL) {
+ #if defined(Py_TAIL_CALL_INTERP)
+ int opcode;
+ #endif
+ opcode = LOAD_CONST_IMMORTAL;
+ (void)(opcode);
frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(LOAD_CONST_IMMORTAL);
}
TARGET(LOAD_CONST_MORTAL) {
+ #if defined(Py_TAIL_CALL_INTERP)
+ int opcode;
+ #endif
+ opcode = LOAD_CONST_MORTAL;
+ (void)(opcode);
frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(LOAD_CONST_MORTAL);
}
TARGET(LOAD_DEREF) {
+ #if defined(Py_TAIL_CALL_INTERP)
+ int opcode;
+ #endif
+ opcode = LOAD_DEREF;
+ (void)(opcode);
frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(LOAD_DEREF);
_PyFrame_SetStackPointer(frame, stack_pointer);
_PyEval_FormatExcUnbound(tstate, _PyFrame_GetCode(frame), oparg);
stack_pointer = _PyFrame_GetStackPointer(frame);
- goto error;
+ JUMP_TO_LABEL(error);
}
value = PyStackRef_FromPyObjectSteal(value_o);
stack_pointer[0] = value;
}
TARGET(LOAD_FAST) {
+ #if defined(Py_TAIL_CALL_INTERP)
+ int opcode;
+ #endif
+ opcode = LOAD_FAST;
+ (void)(opcode);
frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(LOAD_FAST);
}
TARGET(LOAD_FAST_AND_CLEAR) {
+ #if defined(Py_TAIL_CALL_INTERP)
+ int opcode;
+ #endif
+ opcode = LOAD_FAST_AND_CLEAR;
+ (void)(opcode);
frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(LOAD_FAST_AND_CLEAR);
}
TARGET(LOAD_FAST_CHECK) {
+ #if defined(Py_TAIL_CALL_INTERP)
+ int opcode;
+ #endif
+ opcode = LOAD_FAST_CHECK;
+ (void)(opcode);
frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(LOAD_FAST_CHECK);
PyTuple_GetItem(_PyFrame_GetCode(frame)->co_localsplusnames, oparg)
);
stack_pointer = _PyFrame_GetStackPointer(frame);
- goto error;
+ JUMP_TO_LABEL(error);
}
value = PyStackRef_DUP(value_s);
stack_pointer[0] = value;
}
TARGET(LOAD_FAST_LOAD_FAST) {
+ #if defined(Py_TAIL_CALL_INTERP)
+ int opcode;
+ #endif
+ opcode = LOAD_FAST_LOAD_FAST;
+ (void)(opcode);
frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(LOAD_FAST_LOAD_FAST);
}
TARGET(LOAD_FROM_DICT_OR_DEREF) {
+ #if defined(Py_TAIL_CALL_INTERP)
+ int opcode;
+ #endif
+ opcode = LOAD_FROM_DICT_OR_DEREF;
+ (void)(opcode);
frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(LOAD_FROM_DICT_OR_DEREF);
int err = PyMapping_GetOptionalItem(class_dict, name, &value_o);
stack_pointer = _PyFrame_GetStackPointer(frame);
if (err < 0) {
- goto error;
+ JUMP_TO_LABEL(error);
}
if (!value_o) {
PyCellObject *cell = (PyCellObject *)PyStackRef_AsPyObjectBorrow(GETLOCAL(oparg));
_PyFrame_SetStackPointer(frame, stack_pointer);
_PyEval_FormatExcUnbound(tstate, _PyFrame_GetCode(frame), oparg);
stack_pointer = _PyFrame_GetStackPointer(frame);
- goto error;
+ JUMP_TO_LABEL(error);
}
}
stack_pointer += -1;
}
TARGET(LOAD_FROM_DICT_OR_GLOBALS) {
+ #if defined(Py_TAIL_CALL_INTERP)
+ int opcode;
+ #endif
+ opcode = LOAD_FROM_DICT_OR_GLOBALS;
+ (void)(opcode);
frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(LOAD_FROM_DICT_OR_GLOBALS);
stack_pointer = _PyFrame_GetStackPointer(frame);
PyStackRef_CLOSE(mod_or_class_dict);
if (err < 0) {
- goto pop_1_error;
+ JUMP_TO_LABEL(pop_1_error);
}
if (v_o == NULL) {
if (PyDict_CheckExact(GLOBALS())
NAME_ERROR_MSG, name);
stack_pointer = _PyFrame_GetStackPointer(frame);
}
- goto error;
+ JUMP_TO_LABEL(error);
}
}
else {
int err = PyMapping_GetOptionalItem(GLOBALS(), name, &v_o);
stack_pointer = _PyFrame_GetStackPointer(frame);
if (err < 0) {
- goto error;
+ JUMP_TO_LABEL(error);
}
if (v_o == NULL) {
/* namespace 2: builtins */
int err = PyMapping_GetOptionalItem(BUILTINS(), name, &v_o);
stack_pointer = _PyFrame_GetStackPointer(frame);
if (err < 0) {
- goto error;
+ JUMP_TO_LABEL(error);
}
if (v_o == NULL) {
_PyFrame_SetStackPointer(frame, stack_pointer);
tstate, PyExc_NameError,
NAME_ERROR_MSG, name);
stack_pointer = _PyFrame_GetStackPointer(frame);
- goto error;
+ JUMP_TO_LABEL(error);
}
}
}
}
TARGET(LOAD_GLOBAL) {
+ #if defined(Py_TAIL_CALL_INTERP)
+ int opcode;
+ #endif
+ opcode = LOAD_GLOBAL;
+ (void)(opcode);
frame->instr_ptr = next_instr;
next_instr += 5;
INSTRUCTION_STATS(LOAD_GLOBAL);
_PyEval_LoadGlobalStackRef(GLOBALS(), BUILTINS(), name, res);
stack_pointer = _PyFrame_GetStackPointer(frame);
if (PyStackRef_IsNull(*res)) {
- goto error;
+ JUMP_TO_LABEL(error);
}
}
// _PUSH_NULL_CONDITIONAL
}
TARGET(LOAD_GLOBAL_BUILTIN) {
- _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr;
+ #if defined(Py_TAIL_CALL_INTERP)
+ int opcode;
+ #endif
+ opcode = LOAD_GLOBAL_BUILTIN;
+ (void)(opcode);
+ _Py_CODEUNIT* const this_instr = next_instr;
(void)this_instr;
+ frame->instr_ptr = next_instr;
next_instr += 5;
INSTRUCTION_STATS(LOAD_GLOBAL_BUILTIN);
static_assert(INLINE_CACHE_ENTRIES_LOAD_GLOBAL == 4, "incorrect cache size");
{
uint16_t version = read_u16(&this_instr[2].cache);
PyDictObject *dict = (PyDictObject *)GLOBALS();
- DEOPT_IF(!PyDict_CheckExact(dict), LOAD_GLOBAL);
+ if (!PyDict_CheckExact(dict)) {
+ UPDATE_MISS_STATS(LOAD_GLOBAL);
+ assert(_PyOpcode_Deopt[opcode] == (LOAD_GLOBAL));
+ JUMP_TO_PREDICTED(LOAD_GLOBAL);
+ }
PyDictKeysObject *keys = FT_ATOMIC_LOAD_PTR_ACQUIRE(dict->ma_keys);
- DEOPT_IF(FT_ATOMIC_LOAD_UINT32_RELAXED(keys->dk_version) != version, LOAD_GLOBAL);
+ if (FT_ATOMIC_LOAD_UINT32_RELAXED(keys->dk_version) != version) {
+ UPDATE_MISS_STATS(LOAD_GLOBAL);
+ assert(_PyOpcode_Deopt[opcode] == (LOAD_GLOBAL));
+ JUMP_TO_PREDICTED(LOAD_GLOBAL);
+ }
assert(DK_IS_UNICODE(keys));
}
// _GUARD_BUILTINS_VERSION_PUSH_KEYS
{
uint16_t version = read_u16(&this_instr[3].cache);
PyDictObject *dict = (PyDictObject *)BUILTINS();
- DEOPT_IF(!PyDict_CheckExact(dict), LOAD_GLOBAL);
+ if (!PyDict_CheckExact(dict)) {
+ UPDATE_MISS_STATS(LOAD_GLOBAL);
+ assert(_PyOpcode_Deopt[opcode] == (LOAD_GLOBAL));
+ JUMP_TO_PREDICTED(LOAD_GLOBAL);
+ }
PyDictKeysObject *keys = FT_ATOMIC_LOAD_PTR_ACQUIRE(dict->ma_keys);
- DEOPT_IF(FT_ATOMIC_LOAD_UINT32_RELAXED(keys->dk_version) != version, LOAD_GLOBAL);
+ if (FT_ATOMIC_LOAD_UINT32_RELAXED(keys->dk_version) != version) {
+ UPDATE_MISS_STATS(LOAD_GLOBAL);
+ assert(_PyOpcode_Deopt[opcode] == (LOAD_GLOBAL));
+ JUMP_TO_PREDICTED(LOAD_GLOBAL);
+ }
builtins_keys = keys;
assert(DK_IS_UNICODE(builtins_keys));
}
uint16_t index = read_u16(&this_instr[4].cache);
PyDictUnicodeEntry *entries = DK_UNICODE_ENTRIES(builtins_keys);
PyObject *res_o = FT_ATOMIC_LOAD_PTR_RELAXED(entries[index].me_value);
- DEOPT_IF(res_o == NULL, LOAD_GLOBAL);
+ if (res_o == NULL) {
+ UPDATE_MISS_STATS(LOAD_GLOBAL);
+ assert(_PyOpcode_Deopt[opcode] == (LOAD_GLOBAL));
+ JUMP_TO_PREDICTED(LOAD_GLOBAL);
+ }
#if Py_GIL_DISABLED
int increfed = _Py_TryIncrefCompareStackRef(&entries[index].me_value, res_o, &res);
- DEOPT_IF(!increfed, LOAD_GLOBAL);
+ if (!increfed) {
+ UPDATE_MISS_STATS(LOAD_GLOBAL);
+ assert(_PyOpcode_Deopt[opcode] == (LOAD_GLOBAL));
+ JUMP_TO_PREDICTED(LOAD_GLOBAL);
+ }
#else
Py_INCREF(res_o);
res = PyStackRef_FromPyObjectSteal(res_o);
}
TARGET(LOAD_GLOBAL_MODULE) {
- _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr;
+ #if defined(Py_TAIL_CALL_INTERP)
+ int opcode;
+ #endif
+ opcode = LOAD_GLOBAL_MODULE;
+ (void)(opcode);
+ _Py_CODEUNIT* const this_instr = next_instr;
(void)this_instr;
+ frame->instr_ptr = next_instr;
next_instr += 5;
INSTRUCTION_STATS(LOAD_GLOBAL_MODULE);
static_assert(INLINE_CACHE_ENTRIES_LOAD_GLOBAL == 4, "incorrect cache size");
{
uint16_t version = read_u16(&this_instr[2].cache);
PyDictObject *dict = (PyDictObject *)GLOBALS();
- DEOPT_IF(!PyDict_CheckExact(dict), LOAD_GLOBAL);
+ if (!PyDict_CheckExact(dict)) {
+ UPDATE_MISS_STATS(LOAD_GLOBAL);
+ assert(_PyOpcode_Deopt[opcode] == (LOAD_GLOBAL));
+ JUMP_TO_PREDICTED(LOAD_GLOBAL);
+ }
PyDictKeysObject *keys = FT_ATOMIC_LOAD_PTR_ACQUIRE(dict->ma_keys);
- DEOPT_IF(FT_ATOMIC_LOAD_UINT32_RELAXED(keys->dk_version) != version, LOAD_GLOBAL);
+ if (FT_ATOMIC_LOAD_UINT32_RELAXED(keys->dk_version) != version) {
+ UPDATE_MISS_STATS(LOAD_GLOBAL);
+ assert(_PyOpcode_Deopt[opcode] == (LOAD_GLOBAL));
+ JUMP_TO_PREDICTED(LOAD_GLOBAL);
+ }
globals_keys = keys;
assert(DK_IS_UNICODE(globals_keys));
}
uint16_t index = read_u16(&this_instr[4].cache);
PyDictUnicodeEntry *entries = DK_UNICODE_ENTRIES(globals_keys);
PyObject *res_o = FT_ATOMIC_LOAD_PTR_RELAXED(entries[index].me_value);
- DEOPT_IF(res_o == NULL, LOAD_GLOBAL);
+ if (res_o == NULL) {
+ UPDATE_MISS_STATS(LOAD_GLOBAL);
+ assert(_PyOpcode_Deopt[opcode] == (LOAD_GLOBAL));
+ JUMP_TO_PREDICTED(LOAD_GLOBAL);
+ }
#if Py_GIL_DISABLED
int increfed = _Py_TryIncrefCompareStackRef(&entries[index].me_value, res_o, &res);
- DEOPT_IF(!increfed, LOAD_GLOBAL);
+ if (!increfed) {
+ UPDATE_MISS_STATS(LOAD_GLOBAL);
+ assert(_PyOpcode_Deopt[opcode] == (LOAD_GLOBAL));
+ JUMP_TO_PREDICTED(LOAD_GLOBAL);
+ }
#else
Py_INCREF(res_o);
res = PyStackRef_FromPyObjectSteal(res_o);
}
TARGET(LOAD_LOCALS) {
+ #if defined(Py_TAIL_CALL_INTERP)
+ int opcode;
+ #endif
+ opcode = LOAD_LOCALS;
+ (void)(opcode);
frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(LOAD_LOCALS);
_PyErr_SetString(tstate, PyExc_SystemError,
"no locals found");
stack_pointer = _PyFrame_GetStackPointer(frame);
- goto error;
+ JUMP_TO_LABEL(error);
}
locals = PyStackRef_FromPyObjectNew(l);
stack_pointer[0] = locals;
}
TARGET(LOAD_NAME) {
+ #if defined(Py_TAIL_CALL_INTERP)
+ int opcode;
+ #endif
+ opcode = LOAD_NAME;
+ (void)(opcode);
frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(LOAD_NAME);
PyObject *v_o = _PyEval_LoadName(tstate, frame, name);
stack_pointer = _PyFrame_GetStackPointer(frame);
if (v_o == NULL) {
- goto error;
+ JUMP_TO_LABEL(error);
}
v = PyStackRef_FromPyObjectSteal(v_o);
stack_pointer[0] = v;
}
TARGET(LOAD_SMALL_INT) {
+ #if defined(Py_TAIL_CALL_INTERP)
+ int opcode;
+ #endif
+ opcode = LOAD_SMALL_INT;
+ (void)(opcode);
frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(LOAD_SMALL_INT);
}
TARGET(LOAD_SPECIAL) {
+ #if defined(Py_TAIL_CALL_INTERP)
+ int opcode;
+ #endif
+ opcode = LOAD_SPECIAL;
+ (void)(opcode);
frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(LOAD_SPECIAL);
Py_TYPE(owner_o)->tp_name);
stack_pointer = _PyFrame_GetStackPointer(frame);
}
- goto error;
+ JUMP_TO_LABEL(error);
}
attr = PyStackRef_FromPyObjectSteal(attr_o);
self_or_null = self_or_null_o == NULL ?
}
TARGET(LOAD_SUPER_ATTR) {
+ #if defined(Py_TAIL_CALL_INTERP)
+ int opcode;
+ #endif
+ opcode = LOAD_SUPER_ATTR;
+ (void)(opcode);
frame->instr_ptr = next_instr;
next_instr += 2;
INSTRUCTION_STATS(LOAD_SUPER_ATTR);
PREDICTED_LOAD_SUPER_ATTR:;
_Py_CODEUNIT* const this_instr = next_instr - 2;
(void)this_instr;
- opcode = LOAD_SUPER_ATTR;
_PyStackRef global_super_st;
_PyStackRef class_st;
_PyStackRef self_st;
PyStackRef_CLOSE(global_super_st);
PyStackRef_CLOSE(class_st);
PyStackRef_CLOSE(self_st);
- goto pop_3_error;
+ JUMP_TO_LABEL(pop_3_error);
}
}
// we make no attempt to optimize here; specializations should
PyStackRef_CLOSE(class_st);
PyStackRef_CLOSE(self_st);
if (super == NULL) {
- goto pop_3_error;
+ JUMP_TO_LABEL(pop_3_error);
}
PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 2);
stack_pointer += -3;
Py_DECREF(super);
stack_pointer = _PyFrame_GetStackPointer(frame);
if (attr_o == NULL) {
- goto error;
+ JUMP_TO_LABEL(error);
}
attr = PyStackRef_FromPyObjectSteal(attr_o);
}
}
TARGET(LOAD_SUPER_ATTR_ATTR) {
+ #if defined(Py_TAIL_CALL_INTERP)
+ int opcode;
+ #endif
+ opcode = LOAD_SUPER_ATTR_ATTR;
+ (void)(opcode);
+ _Py_CODEUNIT* const this_instr = next_instr;
+ (void)this_instr;
frame->instr_ptr = next_instr;
next_instr += 2;
INSTRUCTION_STATS(LOAD_SUPER_ATTR_ATTR);
PyObject *class = PyStackRef_AsPyObjectBorrow(class_st);
PyObject *self = PyStackRef_AsPyObjectBorrow(self_st);
assert(!(oparg & 1));
- DEOPT_IF(global_super != (PyObject *)&PySuper_Type, LOAD_SUPER_ATTR);
- DEOPT_IF(!PyType_Check(class), LOAD_SUPER_ATTR);
+ if (global_super != (PyObject *)&PySuper_Type) {
+ UPDATE_MISS_STATS(LOAD_SUPER_ATTR);
+ assert(_PyOpcode_Deopt[opcode] == (LOAD_SUPER_ATTR));
+ JUMP_TO_PREDICTED(LOAD_SUPER_ATTR);
+ }
+ if (!PyType_Check(class)) {
+ UPDATE_MISS_STATS(LOAD_SUPER_ATTR);
+ assert(_PyOpcode_Deopt[opcode] == (LOAD_SUPER_ATTR));
+ JUMP_TO_PREDICTED(LOAD_SUPER_ATTR);
+ }
STAT_INC(LOAD_SUPER_ATTR, hit);
PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 2);
_PyFrame_SetStackPointer(frame, stack_pointer);
PyStackRef_CLOSE(class_st);
PyStackRef_CLOSE(self_st);
if (attr == NULL) {
- goto pop_3_error;
+ JUMP_TO_LABEL(pop_3_error);
}
attr_st = PyStackRef_FromPyObjectSteal(attr);
stack_pointer[-3] = attr_st;
}
TARGET(LOAD_SUPER_ATTR_METHOD) {
+ #if defined(Py_TAIL_CALL_INTERP)
+ int opcode;
+ #endif
+ opcode = LOAD_SUPER_ATTR_METHOD;
+ (void)(opcode);
+ _Py_CODEUNIT* const this_instr = next_instr;
+ (void)this_instr;
frame->instr_ptr = next_instr;
next_instr += 2;
INSTRUCTION_STATS(LOAD_SUPER_ATTR_METHOD);
PyObject *class = PyStackRef_AsPyObjectBorrow(class_st);
PyObject *self = PyStackRef_AsPyObjectBorrow(self_st);
assert(oparg & 1);
- DEOPT_IF(global_super != (PyObject *)&PySuper_Type, LOAD_SUPER_ATTR);
- DEOPT_IF(!PyType_Check(class), LOAD_SUPER_ATTR);
+ if (global_super != (PyObject *)&PySuper_Type) {
+ UPDATE_MISS_STATS(LOAD_SUPER_ATTR);
+ assert(_PyOpcode_Deopt[opcode] == (LOAD_SUPER_ATTR));
+ JUMP_TO_PREDICTED(LOAD_SUPER_ATTR);
+ }
+ if (!PyType_Check(class)) {
+ UPDATE_MISS_STATS(LOAD_SUPER_ATTR);
+ assert(_PyOpcode_Deopt[opcode] == (LOAD_SUPER_ATTR));
+ JUMP_TO_PREDICTED(LOAD_SUPER_ATTR);
+ }
STAT_INC(LOAD_SUPER_ATTR, hit);
PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 2);
PyTypeObject *cls = (PyTypeObject *)class;
Py_TYPE(self)->tp_getattro == PyObject_GenericGetAttr ? &method_found : NULL);
stack_pointer = _PyFrame_GetStackPointer(frame);
if (attr_o == NULL) {
- goto error;
+ JUMP_TO_LABEL(error);
}
if (method_found) {
self_or_null = self_st; // transfer ownership
}
TARGET(MAKE_CELL) {
+ #if defined(Py_TAIL_CALL_INTERP)
+ int opcode;
+ #endif
+ opcode = MAKE_CELL;
+ (void)(opcode);
frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(MAKE_CELL);
PyObject *initial = PyStackRef_AsPyObjectBorrow(GETLOCAL(oparg));
PyObject *cell = PyCell_New(initial);
if (cell == NULL) {
- goto error;
+ JUMP_TO_LABEL(error);
}
_PyStackRef tmp = GETLOCAL(oparg);
GETLOCAL(oparg) = PyStackRef_FromPyObjectSteal(cell);
}
TARGET(MAKE_FUNCTION) {
+ #if defined(Py_TAIL_CALL_INTERP)
+ int opcode;
+ #endif
+ opcode = MAKE_FUNCTION;
+ (void)(opcode);
frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(MAKE_FUNCTION);
PyStackRef_CLOSE(codeobj_st);
stack_pointer = _PyFrame_GetStackPointer(frame);
if (func_obj == NULL) {
- goto error;
+ JUMP_TO_LABEL(error);
}
_PyFunction_SetVersion(
func_obj, ((PyCodeObject *)codeobj)->co_version);
}
TARGET(MAP_ADD) {
+ #if defined(Py_TAIL_CALL_INTERP)
+ int opcode;
+ #endif
+ opcode = MAP_ADD;
+ (void)(opcode);
frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(MAP_ADD);
);
stack_pointer = _PyFrame_GetStackPointer(frame);
if (err != 0) {
- goto pop_2_error;
+ JUMP_TO_LABEL(pop_2_error);
}
stack_pointer += -2;
assert(WITHIN_STACK_BOUNDS());
}
TARGET(MATCH_CLASS) {
+ #if defined(Py_TAIL_CALL_INTERP)
+ int opcode;
+ #endif
+ opcode = MATCH_CLASS;
+ (void)(opcode);
frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(MATCH_CLASS);
}
else {
if (_PyErr_Occurred(tstate)) {
- goto pop_3_error;
+ JUMP_TO_LABEL(pop_3_error);
}
// Error!
attrs = PyStackRef_None; // Failure!
}
TARGET(MATCH_KEYS) {
+ #if defined(Py_TAIL_CALL_INTERP)
+ int opcode;
+ #endif
+ opcode = MATCH_KEYS;
+ (void)(opcode);
frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(MATCH_KEYS);
PyStackRef_AsPyObjectBorrow(subject), PyStackRef_AsPyObjectBorrow(keys));
stack_pointer = _PyFrame_GetStackPointer(frame);
if (values_or_none_o == NULL) {
- goto error;
+ JUMP_TO_LABEL(error);
}
values_or_none = PyStackRef_FromPyObjectSteal(values_or_none_o);
stack_pointer[0] = values_or_none;
}
TARGET(MATCH_MAPPING) {
+ #if defined(Py_TAIL_CALL_INTERP)
+ int opcode;
+ #endif
+ opcode = MATCH_MAPPING;
+ (void)(opcode);
frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(MATCH_MAPPING);
}
TARGET(MATCH_SEQUENCE) {
+ #if defined(Py_TAIL_CALL_INTERP)
+ int opcode;
+ #endif
+ opcode = MATCH_SEQUENCE;
+ (void)(opcode);
frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(MATCH_SEQUENCE);
}
TARGET(NOP) {
+ #if defined(Py_TAIL_CALL_INTERP)
+ int opcode;
+ #endif
+ opcode = NOP;
+ (void)(opcode);
frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(NOP);
}
TARGET(NOT_TAKEN) {
+ #if defined(Py_TAIL_CALL_INTERP)
+ int opcode;
+ #endif
+ opcode = NOT_TAKEN;
+ (void)(opcode);
frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(NOT_TAKEN);
}
TARGET(POP_EXCEPT) {
+ #if defined(Py_TAIL_CALL_INTERP)
+ int opcode;
+ #endif
+ opcode = POP_EXCEPT;
+ (void)(opcode);
frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(POP_EXCEPT);
}
TARGET(POP_ITER) {
+ #if defined(Py_TAIL_CALL_INTERP)
+ int opcode;
+ #endif
+ opcode = POP_ITER;
+ (void)(opcode);
frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(POP_ITER);
}
TARGET(POP_JUMP_IF_FALSE) {
- _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr;
+ #if defined(Py_TAIL_CALL_INTERP)
+ int opcode;
+ #endif
+ opcode = POP_JUMP_IF_FALSE;
+ (void)(opcode);
+ _Py_CODEUNIT* const this_instr = next_instr;
(void)this_instr;
+ frame->instr_ptr = next_instr;
next_instr += 2;
INSTRUCTION_STATS(POP_JUMP_IF_FALSE);
_PyStackRef cond;
}
TARGET(POP_JUMP_IF_NONE) {
- _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr;
+ #if defined(Py_TAIL_CALL_INTERP)
+ int opcode;
+ #endif
+ opcode = POP_JUMP_IF_NONE;
+ (void)(opcode);
+ _Py_CODEUNIT* const this_instr = next_instr;
(void)this_instr;
+ frame->instr_ptr = next_instr;
next_instr += 2;
INSTRUCTION_STATS(POP_JUMP_IF_NONE);
_PyStackRef value;
}
TARGET(POP_JUMP_IF_NOT_NONE) {
- _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr;
+ #if defined(Py_TAIL_CALL_INTERP)
+ int opcode;
+ #endif
+ opcode = POP_JUMP_IF_NOT_NONE;
+ (void)(opcode);
+ _Py_CODEUNIT* const this_instr = next_instr;
(void)this_instr;
+ frame->instr_ptr = next_instr;
next_instr += 2;
INSTRUCTION_STATS(POP_JUMP_IF_NOT_NONE);
_PyStackRef value;
}
TARGET(POP_JUMP_IF_TRUE) {
- _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr;
+ #if defined(Py_TAIL_CALL_INTERP)
+ int opcode;
+ #endif
+ opcode = POP_JUMP_IF_TRUE;
+ (void)(opcode);
+ _Py_CODEUNIT* const this_instr = next_instr;
(void)this_instr;
+ frame->instr_ptr = next_instr;
next_instr += 2;
INSTRUCTION_STATS(POP_JUMP_IF_TRUE);
_PyStackRef cond;
}
TARGET(POP_TOP) {
+ #if defined(Py_TAIL_CALL_INTERP)
+ int opcode;
+ #endif
+ opcode = POP_TOP;
+ (void)(opcode);
frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(POP_TOP);
}
TARGET(PUSH_EXC_INFO) {
+ #if defined(Py_TAIL_CALL_INTERP)
+ int opcode;
+ #endif
+ opcode = PUSH_EXC_INFO;
+ (void)(opcode);
frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(PUSH_EXC_INFO);
}
TARGET(PUSH_NULL) {
+ #if defined(Py_TAIL_CALL_INTERP)
+ int opcode;
+ #endif
+ opcode = PUSH_NULL;
+ (void)(opcode);
frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(PUSH_NULL);
}
TARGET(RAISE_VARARGS) {
- _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr;
+ #if defined(Py_TAIL_CALL_INTERP)
+ int opcode;
+ #endif
+ opcode = RAISE_VARARGS;
+ (void)(opcode);
+ _Py_CODEUNIT* const this_instr = next_instr;
(void)this_instr;
+ frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(RAISE_VARARGS);
_PyStackRef *args;
monitor_reraise(tstate, frame, this_instr);
stack_pointer = _PyFrame_GetStackPointer(frame);
_PyFrame_SetStackPointer(frame, stack_pointer);
- goto exception_unwind;
+ JUMP_TO_LABEL(exception_unwind);
}
- goto error;
+ JUMP_TO_LABEL(error);
}
TARGET(RERAISE) {
- _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr;
+ #if defined(Py_TAIL_CALL_INTERP)
+ int opcode;
+ #endif
+ opcode = RERAISE;
+ (void)(opcode);
+ _Py_CODEUNIT* const this_instr = next_instr;
(void)this_instr;
+ frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(RERAISE);
_PyStackRef *values;
_PyErr_SetString(tstate, PyExc_SystemError, "lasti is not an int");
Py_DECREF(exc);
stack_pointer = _PyFrame_GetStackPointer(frame);
- goto error;
+ JUMP_TO_LABEL(error);
}
stack_pointer += 1;
assert(WITHIN_STACK_BOUNDS());
monitor_reraise(tstate, frame, this_instr);
stack_pointer = _PyFrame_GetStackPointer(frame);
_PyFrame_SetStackPointer(frame, stack_pointer);
- goto exception_unwind;
+ JUMP_TO_LABEL(exception_unwind);
}
TARGET(RESERVED) {
+ #if defined(Py_TAIL_CALL_INTERP)
+ int opcode;
+ #endif
+ opcode = RESERVED;
+ (void)(opcode);
frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(RESERVED);
}
TARGET(RESUME) {
+ #if defined(Py_TAIL_CALL_INTERP)
+ int opcode;
+ #endif
+ opcode = RESUME;
+ (void)(opcode);
frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(RESUME);
_PyEval_GetExecutableCode(tstate, _PyFrame_GetCode(frame));
stack_pointer = _PyFrame_GetStackPointer(frame);
if (bytecode == NULL) {
- goto error;
+ JUMP_TO_LABEL(error);
}
_PyFrame_SetStackPointer(frame, stack_pointer);
ptrdiff_t off = this_instr - _PyFrame_GetBytecode(frame);
int err = _Py_Instrument(_PyFrame_GetCode(frame), tstate->interp);
stack_pointer = _PyFrame_GetStackPointer(frame);
if (err) {
- goto error;
+ JUMP_TO_LABEL(error);
}
next_instr = this_instr;
DISPATCH();
int err = _Py_HandlePending(tstate);
stack_pointer = _PyFrame_GetStackPointer(frame);
if (err != 0) {
- goto error;
+ JUMP_TO_LABEL(error);
}
}
}
}
TARGET(RESUME_CHECK) {
+ #if defined(Py_TAIL_CALL_INTERP)
+ int opcode;
+ #endif
+ opcode = RESUME_CHECK;
+ (void)(opcode);
+ _Py_CODEUNIT* const this_instr = next_instr;
+ (void)this_instr;
frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(RESUME_CHECK);
static_assert(0 == 0, "incorrect cache size");
#if defined(__EMSCRIPTEN__)
- DEOPT_IF(_Py_emscripten_signal_clock == 0, RESUME);
+ if (_Py_emscripten_signal_clock == 0) {
+ UPDATE_MISS_STATS(RESUME);
+ assert(_PyOpcode_Deopt[opcode] == (RESUME));
+ JUMP_TO_PREDICTED(RESUME);
+ }
_Py_emscripten_signal_clock -= Py_EMSCRIPTEN_SIGNAL_HANDLING;
#endif
uintptr_t eval_breaker = _Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker);
uintptr_t version = FT_ATOMIC_LOAD_UINTPTR_ACQUIRE(_PyFrame_GetCode(frame)->_co_instrumentation_version);
assert((version & _PY_EVAL_EVENTS_MASK) == 0);
- DEOPT_IF(eval_breaker != version, RESUME);
+ if (eval_breaker != version) {
+ UPDATE_MISS_STATS(RESUME);
+ assert(_PyOpcode_Deopt[opcode] == (RESUME));
+ JUMP_TO_PREDICTED(RESUME);
+ }
#ifdef Py_GIL_DISABLED
- DEOPT_IF(frame->tlbc_index !=
- ((_PyThreadStateImpl *)tstate)->tlbc_index, RESUME);
+ if (frame->tlbc_index !=
+ ((_PyThreadStateImpl *)tstate)->tlbc_index) {
+ UPDATE_MISS_STATS(RESUME);
+ assert(_PyOpcode_Deopt[opcode] == (RESUME));
+ JUMP_TO_PREDICTED(RESUME);
+ }
#endif
DISPATCH();
}
TARGET(RETURN_GENERATOR) {
+ #if defined(Py_TAIL_CALL_INTERP)
+ int opcode;
+ #endif
+ opcode = RETURN_GENERATOR;
+ (void)(opcode);
frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(RETURN_GENERATOR);
PyGenObject *gen = (PyGenObject *)_Py_MakeCoro(func);
stack_pointer = _PyFrame_GetStackPointer(frame);
if (gen == NULL) {
- goto error;
+ JUMP_TO_LABEL(error);
}
assert(EMPTY());
_PyFrame_SetStackPointer(frame, stack_pointer);
}
TARGET(RETURN_VALUE) {
+ #if defined(Py_TAIL_CALL_INTERP)
+ int opcode;
+ #endif
+ opcode = RETURN_VALUE;
+ (void)(opcode);
frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(RETURN_VALUE);
}
TARGET(SEND) {
+ #if defined(Py_TAIL_CALL_INTERP)
+ int opcode;
+ #endif
+ opcode = SEND;
+ (void)(opcode);
frame->instr_ptr = next_instr;
next_instr += 2;
INSTRUCTION_STATS(SEND);
}
else {
PyStackRef_CLOSE(v);
- goto pop_1_error;
+ JUMP_TO_LABEL(pop_1_error);
}
}
stack_pointer += -1;
}
TARGET(SEND_GEN) {
+ #if defined(Py_TAIL_CALL_INTERP)
+ int opcode;
+ #endif
+ opcode = SEND_GEN;
+ (void)(opcode);
+ _Py_CODEUNIT* const this_instr = next_instr;
+ (void)this_instr;
frame->instr_ptr = next_instr;
next_instr += 2;
INSTRUCTION_STATS(SEND_GEN);
/* Skip 1 cache entry */
// _CHECK_PEP_523
{
- DEOPT_IF(tstate->interp->eval_frame, SEND);
+ if (tstate->interp->eval_frame) {
+ UPDATE_MISS_STATS(SEND);
+ assert(_PyOpcode_Deopt[opcode] == (SEND));
+ JUMP_TO_PREDICTED(SEND);
+ }
}
// _SEND_GEN_FRAME
{
v = stack_pointer[-1];
receiver = stack_pointer[-2];
PyGenObject *gen = (PyGenObject *)PyStackRef_AsPyObjectBorrow(receiver);
- DEOPT_IF(Py_TYPE(gen) != &PyGen_Type && Py_TYPE(gen) != &PyCoro_Type, SEND);
- DEOPT_IF(gen->gi_frame_state >= FRAME_EXECUTING, SEND);
+ if (Py_TYPE(gen) != &PyGen_Type && Py_TYPE(gen) != &PyCoro_Type) {
+ UPDATE_MISS_STATS(SEND);
+ assert(_PyOpcode_Deopt[opcode] == (SEND));
+ JUMP_TO_PREDICTED(SEND);
+ }
+ if (gen->gi_frame_state >= FRAME_EXECUTING) {
+ UPDATE_MISS_STATS(SEND);
+ assert(_PyOpcode_Deopt[opcode] == (SEND));
+ JUMP_TO_PREDICTED(SEND);
+ }
STAT_INC(SEND, hit);
gen_frame = &gen->gi_iframe;
_PyFrame_StackPush(gen_frame, v);
}
TARGET(SETUP_ANNOTATIONS) {
+ #if defined(Py_TAIL_CALL_INTERP)
+ int opcode;
+ #endif
+ opcode = SETUP_ANNOTATIONS;
+ (void)(opcode);
frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(SETUP_ANNOTATIONS);
_PyErr_Format(tstate, PyExc_SystemError,
"no locals found when setting up annotations");
stack_pointer = _PyFrame_GetStackPointer(frame);
- goto error;
+ JUMP_TO_LABEL(error);
}
/* check if __annotations__ in locals()... */
_PyFrame_SetStackPointer(frame, stack_pointer);
int err = PyMapping_GetOptionalItem(LOCALS(), &_Py_ID(__annotations__), &ann_dict);
stack_pointer = _PyFrame_GetStackPointer(frame);
if (err < 0) {
- goto error;
+ JUMP_TO_LABEL(error);
}
if (ann_dict == NULL) {
_PyFrame_SetStackPointer(frame, stack_pointer);
ann_dict = PyDict_New();
stack_pointer = _PyFrame_GetStackPointer(frame);
if (ann_dict == NULL) {
- goto error;
+ JUMP_TO_LABEL(error);
}
_PyFrame_SetStackPointer(frame, stack_pointer);
err = PyObject_SetItem(LOCALS(), &_Py_ID(__annotations__),
Py_DECREF(ann_dict);
stack_pointer = _PyFrame_GetStackPointer(frame);
if (err) {
- goto error;
+ JUMP_TO_LABEL(error);
}
}
else {
}
TARGET(SET_ADD) {
+ #if defined(Py_TAIL_CALL_INTERP)
+ int opcode;
+ #endif
+ opcode = SET_ADD;
+ (void)(opcode);
frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(SET_ADD);
stack_pointer = _PyFrame_GetStackPointer(frame);
PyStackRef_CLOSE(v);
if (err) {
- goto pop_1_error;
+ JUMP_TO_LABEL(pop_1_error);
}
stack_pointer += -1;
assert(WITHIN_STACK_BOUNDS());
}
TARGET(SET_FUNCTION_ATTRIBUTE) {
+ #if defined(Py_TAIL_CALL_INTERP)
+ int opcode;
+ #endif
+ opcode = SET_FUNCTION_ATTRIBUTE;
+ (void)(opcode);
frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(SET_FUNCTION_ATTRIBUTE);
}
TARGET(SET_UPDATE) {
+ #if defined(Py_TAIL_CALL_INTERP)
+ int opcode;
+ #endif
+ opcode = SET_UPDATE;
+ (void)(opcode);
frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(SET_UPDATE);
stack_pointer = _PyFrame_GetStackPointer(frame);
PyStackRef_CLOSE(iterable);
if (err < 0) {
- goto pop_1_error;
+ JUMP_TO_LABEL(pop_1_error);
}
stack_pointer += -1;
assert(WITHIN_STACK_BOUNDS());
}
TARGET(STORE_ATTR) {
+ #if defined(Py_TAIL_CALL_INTERP)
+ int opcode;
+ #endif
+ opcode = STORE_ATTR;
+ (void)(opcode);
frame->instr_ptr = next_instr;
next_instr += 5;
INSTRUCTION_STATS(STORE_ATTR);
PyStackRef_CLOSE(v);
PyStackRef_CLOSE(owner);
if (err) {
- goto pop_2_error;
+ JUMP_TO_LABEL(pop_2_error);
}
}
stack_pointer += -2;
}
TARGET(STORE_ATTR_INSTANCE_VALUE) {
- _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr;
+ #if defined(Py_TAIL_CALL_INTERP)
+ int opcode;
+ #endif
+ opcode = STORE_ATTR_INSTANCE_VALUE;
+ (void)(opcode);
+ _Py_CODEUNIT* const this_instr = next_instr;
(void)this_instr;
+ frame->instr_ptr = next_instr;
next_instr += 5;
INSTRUCTION_STATS(STORE_ATTR_INSTANCE_VALUE);
static_assert(INLINE_CACHE_ENTRIES_STORE_ATTR == 4, "incorrect cache size");
uint32_t type_version = read_u32(&this_instr[2].cache);
PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner);
assert(type_version != 0);
- DEOPT_IF(!LOCK_OBJECT(owner_o), STORE_ATTR);
+ if (!LOCK_OBJECT(owner_o)) {
+ UPDATE_MISS_STATS(STORE_ATTR);
+ assert(_PyOpcode_Deopt[opcode] == (STORE_ATTR));
+ JUMP_TO_PREDICTED(STORE_ATTR);
+ }
PyTypeObject *tp = Py_TYPE(owner_o);
if (FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version) {
UNLOCK_OBJECT(owner_o);
- DEOPT_IF(true, STORE_ATTR);
+ if (true) {
+ UPDATE_MISS_STATS(STORE_ATTR);
+ assert(_PyOpcode_Deopt[opcode] == (STORE_ATTR));
+ JUMP_TO_PREDICTED(STORE_ATTR);
+ }
}
}
// _GUARD_DORV_NO_DICT
if (_PyObject_GetManagedDict(owner_o) ||
!FT_ATOMIC_LOAD_UINT8(_PyObject_InlineValues(owner_o)->valid)) {
UNLOCK_OBJECT(owner_o);
- DEOPT_IF(true, STORE_ATTR);
+ if (true) {
+ UPDATE_MISS_STATS(STORE_ATTR);
+ assert(_PyOpcode_Deopt[opcode] == (STORE_ATTR));
+ JUMP_TO_PREDICTED(STORE_ATTR);
+ }
}
}
// _STORE_ATTR_INSTANCE_VALUE
}
TARGET(STORE_ATTR_SLOT) {
- _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr;
+ #if defined(Py_TAIL_CALL_INTERP)
+ int opcode;
+ #endif
+ opcode = STORE_ATTR_SLOT;
+ (void)(opcode);
+ _Py_CODEUNIT* const this_instr = next_instr;
(void)this_instr;
+ frame->instr_ptr = next_instr;
next_instr += 5;
INSTRUCTION_STATS(STORE_ATTR_SLOT);
static_assert(INLINE_CACHE_ENTRIES_STORE_ATTR == 4, "incorrect cache size");
uint32_t type_version = read_u32(&this_instr[2].cache);
PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner));
assert(type_version != 0);
- DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, STORE_ATTR);
+ if (FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version) {
+ UPDATE_MISS_STATS(STORE_ATTR);
+ assert(_PyOpcode_Deopt[opcode] == (STORE_ATTR));
+ JUMP_TO_PREDICTED(STORE_ATTR);
+ }
}
// _STORE_ATTR_SLOT
{
value = stack_pointer[-2];
uint16_t index = read_u16(&this_instr[4].cache);
PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner);
- DEOPT_IF(!LOCK_OBJECT(owner_o), STORE_ATTR);
+ if (!LOCK_OBJECT(owner_o)) {
+ UPDATE_MISS_STATS(STORE_ATTR);
+ assert(_PyOpcode_Deopt[opcode] == (STORE_ATTR));
+ JUMP_TO_PREDICTED(STORE_ATTR);
+ }
char *addr = (char *)owner_o + index;
STAT_INC(STORE_ATTR, hit);
PyObject *old_value = *(PyObject **)addr;
}
TARGET(STORE_ATTR_WITH_HINT) {
- _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr;
+ #if defined(Py_TAIL_CALL_INTERP)
+ int opcode;
+ #endif
+ opcode = STORE_ATTR_WITH_HINT;
+ (void)(opcode);
+ _Py_CODEUNIT* const this_instr = next_instr;
(void)this_instr;
+ frame->instr_ptr = next_instr;
next_instr += 5;
INSTRUCTION_STATS(STORE_ATTR_WITH_HINT);
static_assert(INLINE_CACHE_ENTRIES_STORE_ATTR == 4, "incorrect cache size");
uint32_t type_version = read_u32(&this_instr[2].cache);
PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner));
assert(type_version != 0);
- DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, STORE_ATTR);
+ if (FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version) {
+ UPDATE_MISS_STATS(STORE_ATTR);
+ assert(_PyOpcode_Deopt[opcode] == (STORE_ATTR));
+ JUMP_TO_PREDICTED(STORE_ATTR);
+ }
}
// _STORE_ATTR_WITH_HINT
{
PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner);
assert(Py_TYPE(owner_o)->tp_flags & Py_TPFLAGS_MANAGED_DICT);
PyDictObject *dict = _PyObject_GetManagedDict(owner_o);
- DEOPT_IF(dict == NULL, STORE_ATTR);
- DEOPT_IF(!LOCK_OBJECT(dict), STORE_ATTR);
+ if (dict == NULL) {
+ UPDATE_MISS_STATS(STORE_ATTR);
+ assert(_PyOpcode_Deopt[opcode] == (STORE_ATTR));
+ JUMP_TO_PREDICTED(STORE_ATTR);
+ }
+ if (!LOCK_OBJECT(dict)) {
+ UPDATE_MISS_STATS(STORE_ATTR);
+ assert(_PyOpcode_Deopt[opcode] == (STORE_ATTR));
+ JUMP_TO_PREDICTED(STORE_ATTR);
+ }
#ifdef Py_GIL_DISABLED
if (dict != _PyObject_GetManagedDict(owner_o)) {
UNLOCK_OBJECT(dict);
- DEOPT_IF(true, STORE_ATTR);
+ if (true) {
+ UPDATE_MISS_STATS(STORE_ATTR);
+ assert(_PyOpcode_Deopt[opcode] == (STORE_ATTR));
+ JUMP_TO_PREDICTED(STORE_ATTR);
+ }
}
#endif
assert(PyDict_CheckExact((PyObject *)dict));
if (hint >= (size_t)dict->ma_keys->dk_nentries ||
!DK_IS_UNICODE(dict->ma_keys)) {
UNLOCK_OBJECT(dict);
- DEOPT_IF(true, STORE_ATTR);
+ if (true) {
+ UPDATE_MISS_STATS(STORE_ATTR);
+ assert(_PyOpcode_Deopt[opcode] == (STORE_ATTR));
+ JUMP_TO_PREDICTED(STORE_ATTR);
+ }
}
PyDictUnicodeEntry *ep = DK_UNICODE_ENTRIES(dict->ma_keys) + hint;
if (ep->me_key != name) {
UNLOCK_OBJECT(dict);
- DEOPT_IF(true, STORE_ATTR);
+ if (true) {
+ UPDATE_MISS_STATS(STORE_ATTR);
+ assert(_PyOpcode_Deopt[opcode] == (STORE_ATTR));
+ JUMP_TO_PREDICTED(STORE_ATTR);
+ }
}
PyObject *old_value = ep->me_value;
if (old_value == NULL) {
UNLOCK_OBJECT(dict);
- DEOPT_IF(true, STORE_ATTR);
+ if (true) {
+ UPDATE_MISS_STATS(STORE_ATTR);
+ assert(_PyOpcode_Deopt[opcode] == (STORE_ATTR));
+ JUMP_TO_PREDICTED(STORE_ATTR);
+ }
}
_PyFrame_SetStackPointer(frame, stack_pointer);
_PyDict_NotifyEvent(tstate->interp, PyDict_EVENT_MODIFIED, dict, name, PyStackRef_AsPyObjectBorrow(value));
}
TARGET(STORE_DEREF) {
+ #if defined(Py_TAIL_CALL_INTERP)
+ int opcode;
+ #endif
+ opcode = STORE_DEREF;
+ (void)(opcode);
frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(STORE_DEREF);
}
TARGET(STORE_FAST) {
+ #if defined(Py_TAIL_CALL_INTERP)
+ int opcode;
+ #endif
+ opcode = STORE_FAST;
+ (void)(opcode);
frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(STORE_FAST);
}
TARGET(STORE_FAST_LOAD_FAST) {
+ #if defined(Py_TAIL_CALL_INTERP)
+ int opcode;
+ #endif
+ opcode = STORE_FAST_LOAD_FAST;
+ (void)(opcode);
frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(STORE_FAST_LOAD_FAST);
}
TARGET(STORE_FAST_STORE_FAST) {
+ #if defined(Py_TAIL_CALL_INTERP)
+ int opcode;
+ #endif
+ opcode = STORE_FAST_STORE_FAST;
+ (void)(opcode);
frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(STORE_FAST_STORE_FAST);
}
TARGET(STORE_GLOBAL) {
+ #if defined(Py_TAIL_CALL_INTERP)
+ int opcode;
+ #endif
+ opcode = STORE_GLOBAL;
+ (void)(opcode);
frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(STORE_GLOBAL);
stack_pointer = _PyFrame_GetStackPointer(frame);
PyStackRef_CLOSE(v);
if (err) {
- goto pop_1_error;
+ JUMP_TO_LABEL(pop_1_error);
}
stack_pointer += -1;
assert(WITHIN_STACK_BOUNDS());
}
TARGET(STORE_NAME) {
+ #if defined(Py_TAIL_CALL_INTERP)
+ int opcode;
+ #endif
+ opcode = STORE_NAME;
+ (void)(opcode);
frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(STORE_NAME);
"no locals found when storing %R", name);
stack_pointer = _PyFrame_GetStackPointer(frame);
PyStackRef_CLOSE(v);
- goto pop_1_error;
+ JUMP_TO_LABEL(pop_1_error);
}
if (PyDict_CheckExact(ns)) {
_PyFrame_SetStackPointer(frame, stack_pointer);
}
PyStackRef_CLOSE(v);
if (err) {
- goto pop_1_error;
+ JUMP_TO_LABEL(pop_1_error);
}
stack_pointer += -1;
assert(WITHIN_STACK_BOUNDS());
}
TARGET(STORE_SLICE) {
+ #if defined(Py_TAIL_CALL_INTERP)
+ int opcode;
+ #endif
+ opcode = STORE_SLICE;
+ (void)(opcode);
frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(STORE_SLICE);
PyStackRef_CLOSE(v);
PyStackRef_CLOSE(container);
if (err) {
- goto pop_4_error;
+ JUMP_TO_LABEL(pop_4_error);
}
}
stack_pointer += -4;
}
TARGET(STORE_SUBSCR) {
+ #if defined(Py_TAIL_CALL_INTERP)
+ int opcode;
+ #endif
+ opcode = STORE_SUBSCR;
+ (void)(opcode);
frame->instr_ptr = next_instr;
next_instr += 2;
INSTRUCTION_STATS(STORE_SUBSCR);
PyStackRef_CLOSE(container);
PyStackRef_CLOSE(sub);
if (err) {
- goto pop_3_error;
+ JUMP_TO_LABEL(pop_3_error);
}
}
stack_pointer += -3;
}
TARGET(STORE_SUBSCR_DICT) {
+ #if defined(Py_TAIL_CALL_INTERP)
+ int opcode;
+ #endif
+ opcode = STORE_SUBSCR_DICT;
+ (void)(opcode);
+ _Py_CODEUNIT* const this_instr = next_instr;
+ (void)this_instr;
frame->instr_ptr = next_instr;
next_instr += 2;
INSTRUCTION_STATS(STORE_SUBSCR_DICT);
dict_st = stack_pointer[-2];
value = stack_pointer[-3];
PyObject *dict = PyStackRef_AsPyObjectBorrow(dict_st);
- DEOPT_IF(!PyDict_CheckExact(dict), STORE_SUBSCR);
+ if (!PyDict_CheckExact(dict)) {
+ UPDATE_MISS_STATS(STORE_SUBSCR);
+ assert(_PyOpcode_Deopt[opcode] == (STORE_SUBSCR));
+ JUMP_TO_PREDICTED(STORE_SUBSCR);
+ }
STAT_INC(STORE_SUBSCR, hit);
_PyFrame_SetStackPointer(frame, stack_pointer);
int err = _PyDict_SetItem_Take2((PyDictObject *)dict,
PyStackRef_CLOSE(dict_st);
stack_pointer = _PyFrame_GetStackPointer(frame);
if (err) {
- goto error;
+ JUMP_TO_LABEL(error);
}
DISPATCH();
}
TARGET(STORE_SUBSCR_LIST_INT) {
+ #if defined(Py_TAIL_CALL_INTERP)
+ int opcode;
+ #endif
+ opcode = STORE_SUBSCR_LIST_INT;
+ (void)(opcode);
+ _Py_CODEUNIT* const this_instr = next_instr;
+ (void)this_instr;
frame->instr_ptr = next_instr;
next_instr += 2;
INSTRUCTION_STATS(STORE_SUBSCR_LIST_INT);
value = stack_pointer[-3];
PyObject *sub = PyStackRef_AsPyObjectBorrow(sub_st);
PyObject *list = PyStackRef_AsPyObjectBorrow(list_st);
- DEOPT_IF(!PyLong_CheckExact(sub), STORE_SUBSCR);
- DEOPT_IF(!PyList_CheckExact(list), STORE_SUBSCR);
+ if (!PyLong_CheckExact(sub)) {
+ UPDATE_MISS_STATS(STORE_SUBSCR);
+ assert(_PyOpcode_Deopt[opcode] == (STORE_SUBSCR));
+ JUMP_TO_PREDICTED(STORE_SUBSCR);
+ }
+ if (!PyList_CheckExact(list)) {
+ UPDATE_MISS_STATS(STORE_SUBSCR);
+ assert(_PyOpcode_Deopt[opcode] == (STORE_SUBSCR));
+ JUMP_TO_PREDICTED(STORE_SUBSCR);
+ }
// Ensure nonnegative, zero-or-one-digit ints.
- DEOPT_IF(!_PyLong_IsNonNegativeCompact((PyLongObject *)sub), STORE_SUBSCR);
+ if (!_PyLong_IsNonNegativeCompact((PyLongObject *)sub)) {
+ UPDATE_MISS_STATS(STORE_SUBSCR);
+ assert(_PyOpcode_Deopt[opcode] == (STORE_SUBSCR));
+ JUMP_TO_PREDICTED(STORE_SUBSCR);
+ }
Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0];
- DEOPT_IF(!LOCK_OBJECT(list), STORE_SUBSCR);
+ if (!LOCK_OBJECT(list)) {
+ UPDATE_MISS_STATS(STORE_SUBSCR);
+ assert(_PyOpcode_Deopt[opcode] == (STORE_SUBSCR));
+ JUMP_TO_PREDICTED(STORE_SUBSCR);
+ }
// Ensure index < len(list)
if (index >= PyList_GET_SIZE(list)) {
UNLOCK_OBJECT(list);
- DEOPT_IF(true, STORE_SUBSCR);
+ if (true) {
+ UPDATE_MISS_STATS(STORE_SUBSCR);
+ assert(_PyOpcode_Deopt[opcode] == (STORE_SUBSCR));
+ JUMP_TO_PREDICTED(STORE_SUBSCR);
+ }
}
STAT_INC(STORE_SUBSCR, hit);
PyObject *old_value = PyList_GET_ITEM(list, index);
}
TARGET(SWAP) {
+ #if defined(Py_TAIL_CALL_INTERP)
+ int opcode;
+ #endif
+ opcode = SWAP;
+ (void)(opcode);
frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(SWAP);
}
TARGET(TO_BOOL) {
+ #if defined(Py_TAIL_CALL_INTERP)
+ int opcode;
+ #endif
+ opcode = TO_BOOL;
+ (void)(opcode);
frame->instr_ptr = next_instr;
next_instr += 4;
INSTRUCTION_STATS(TO_BOOL);
stack_pointer = _PyFrame_GetStackPointer(frame);
PyStackRef_CLOSE(value);
if (err < 0) {
- goto pop_1_error;
+ JUMP_TO_LABEL(pop_1_error);
}
res = err ? PyStackRef_True : PyStackRef_False;
}
}
TARGET(TO_BOOL_ALWAYS_TRUE) {
- _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr;
+ #if defined(Py_TAIL_CALL_INTERP)
+ int opcode;
+ #endif
+ opcode = TO_BOOL_ALWAYS_TRUE;
+ (void)(opcode);
+ _Py_CODEUNIT* const this_instr = next_instr;
(void)this_instr;
+ frame->instr_ptr = next_instr;
next_instr += 4;
INSTRUCTION_STATS(TO_BOOL_ALWAYS_TRUE);
static_assert(INLINE_CACHE_ENTRIES_TO_BOOL == 3, "incorrect cache size");
uint32_t type_version = read_u32(&this_instr[2].cache);
PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner));
assert(type_version != 0);
- DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, TO_BOOL);
+ if (FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version) {
+ UPDATE_MISS_STATS(TO_BOOL);
+ assert(_PyOpcode_Deopt[opcode] == (TO_BOOL));
+ JUMP_TO_PREDICTED(TO_BOOL);
+ }
}
// _REPLACE_WITH_TRUE
{
}
TARGET(TO_BOOL_BOOL) {
+ #if defined(Py_TAIL_CALL_INTERP)
+ int opcode;
+ #endif
+ opcode = TO_BOOL_BOOL;
+ (void)(opcode);
+ _Py_CODEUNIT* const this_instr = next_instr;
+ (void)this_instr;
frame->instr_ptr = next_instr;
next_instr += 4;
INSTRUCTION_STATS(TO_BOOL_BOOL);
/* Skip 1 cache entry */
/* Skip 2 cache entries */
value = stack_pointer[-1];
- DEOPT_IF(!PyStackRef_BoolCheck(value), TO_BOOL);
+ if (!PyStackRef_BoolCheck(value)) {
+ UPDATE_MISS_STATS(TO_BOOL);
+ assert(_PyOpcode_Deopt[opcode] == (TO_BOOL));
+ JUMP_TO_PREDICTED(TO_BOOL);
+ }
STAT_INC(TO_BOOL, hit);
DISPATCH();
}
TARGET(TO_BOOL_INT) {
+ #if defined(Py_TAIL_CALL_INTERP)
+ int opcode;
+ #endif
+ opcode = TO_BOOL_INT;
+ (void)(opcode);
+ _Py_CODEUNIT* const this_instr = next_instr;
+ (void)this_instr;
frame->instr_ptr = next_instr;
next_instr += 4;
INSTRUCTION_STATS(TO_BOOL_INT);
/* Skip 2 cache entries */
value = stack_pointer[-1];
PyObject *value_o = PyStackRef_AsPyObjectBorrow(value);
- DEOPT_IF(!PyLong_CheckExact(value_o), TO_BOOL);
+ if (!PyLong_CheckExact(value_o)) {
+ UPDATE_MISS_STATS(TO_BOOL);
+ assert(_PyOpcode_Deopt[opcode] == (TO_BOOL));
+ JUMP_TO_PREDICTED(TO_BOOL);
+ }
STAT_INC(TO_BOOL, hit);
if (_PyLong_IsZero((PyLongObject *)value_o)) {
assert(_Py_IsImmortal(value_o));
}
TARGET(TO_BOOL_LIST) {
+ #if defined(Py_TAIL_CALL_INTERP)
+ int opcode;
+ #endif
+ opcode = TO_BOOL_LIST;
+ (void)(opcode);
+ _Py_CODEUNIT* const this_instr = next_instr;
+ (void)this_instr;
frame->instr_ptr = next_instr;
next_instr += 4;
INSTRUCTION_STATS(TO_BOOL_LIST);
/* Skip 2 cache entries */
value = stack_pointer[-1];
PyObject *value_o = PyStackRef_AsPyObjectBorrow(value);
- DEOPT_IF(!PyList_CheckExact(value_o), TO_BOOL);
+ if (!PyList_CheckExact(value_o)) {
+ UPDATE_MISS_STATS(TO_BOOL);
+ assert(_PyOpcode_Deopt[opcode] == (TO_BOOL));
+ JUMP_TO_PREDICTED(TO_BOOL);
+ }
STAT_INC(TO_BOOL, hit);
res = PyList_GET_SIZE(value_o) ? PyStackRef_True : PyStackRef_False;
PyStackRef_CLOSE(value);
}
TARGET(TO_BOOL_NONE) {
+ #if defined(Py_TAIL_CALL_INTERP)
+ int opcode;
+ #endif
+ opcode = TO_BOOL_NONE;
+ (void)(opcode);
+ _Py_CODEUNIT* const this_instr = next_instr;
+ (void)this_instr;
frame->instr_ptr = next_instr;
next_instr += 4;
INSTRUCTION_STATS(TO_BOOL_NONE);
/* Skip 2 cache entries */
value = stack_pointer[-1];
// This one is a bit weird, because we expect *some* failures:
- DEOPT_IF(!PyStackRef_IsNone(value), TO_BOOL);
+ if (!PyStackRef_IsNone(value)) {
+ UPDATE_MISS_STATS(TO_BOOL);
+ assert(_PyOpcode_Deopt[opcode] == (TO_BOOL));
+ JUMP_TO_PREDICTED(TO_BOOL);
+ }
STAT_INC(TO_BOOL, hit);
res = PyStackRef_False;
stack_pointer[-1] = res;
}
TARGET(TO_BOOL_STR) {
+ #if defined(Py_TAIL_CALL_INTERP)
+ int opcode;
+ #endif
+ opcode = TO_BOOL_STR;
+ (void)(opcode);
+ _Py_CODEUNIT* const this_instr = next_instr;
+ (void)this_instr;
frame->instr_ptr = next_instr;
next_instr += 4;
INSTRUCTION_STATS(TO_BOOL_STR);
/* Skip 2 cache entries */
value = stack_pointer[-1];
PyObject *value_o = PyStackRef_AsPyObjectBorrow(value);
- DEOPT_IF(!PyUnicode_CheckExact(value_o), TO_BOOL);
+ if (!PyUnicode_CheckExact(value_o)) {
+ UPDATE_MISS_STATS(TO_BOOL);
+ assert(_PyOpcode_Deopt[opcode] == (TO_BOOL));
+ JUMP_TO_PREDICTED(TO_BOOL);
+ }
STAT_INC(TO_BOOL, hit);
if (value_o == &_Py_STR(empty)) {
assert(_Py_IsImmortal(value_o));
}
TARGET(UNARY_INVERT) {
+ #if defined(Py_TAIL_CALL_INTERP)
+ int opcode;
+ #endif
+ opcode = UNARY_INVERT;
+ (void)(opcode);
frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(UNARY_INVERT);
stack_pointer = _PyFrame_GetStackPointer(frame);
PyStackRef_CLOSE(value);
if (res_o == NULL) {
- goto pop_1_error;
+ JUMP_TO_LABEL(pop_1_error);
}
res = PyStackRef_FromPyObjectSteal(res_o);
stack_pointer[-1] = res;
}
TARGET(UNARY_NEGATIVE) {
+ #if defined(Py_TAIL_CALL_INTERP)
+ int opcode;
+ #endif
+ opcode = UNARY_NEGATIVE;
+ (void)(opcode);
frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(UNARY_NEGATIVE);
stack_pointer = _PyFrame_GetStackPointer(frame);
PyStackRef_CLOSE(value);
if (res_o == NULL) {
- goto pop_1_error;
+ JUMP_TO_LABEL(pop_1_error);
}
res = PyStackRef_FromPyObjectSteal(res_o);
stack_pointer[-1] = res;
}
TARGET(UNARY_NOT) {
+ #if defined(Py_TAIL_CALL_INTERP)
+ int opcode;
+ #endif
+ opcode = UNARY_NOT;
+ (void)(opcode);
frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(UNARY_NOT);
}
TARGET(UNPACK_EX) {
+ #if defined(Py_TAIL_CALL_INTERP)
+ int opcode;
+ #endif
+ opcode = UNPACK_EX;
+ (void)(opcode);
frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(UNPACK_EX);
stack_pointer = _PyFrame_GetStackPointer(frame);
PyStackRef_CLOSE(seq);
if (res == 0) {
- goto pop_1_error;
+ JUMP_TO_LABEL(pop_1_error);
}
stack_pointer += (oparg & 0xFF) + (oparg >> 8);
assert(WITHIN_STACK_BOUNDS());
}
TARGET(UNPACK_SEQUENCE) {
+ #if defined(Py_TAIL_CALL_INTERP)
+ int opcode;
+ #endif
+ opcode = UNPACK_SEQUENCE;
+ (void)(opcode);
frame->instr_ptr = next_instr;
next_instr += 2;
INSTRUCTION_STATS(UNPACK_SEQUENCE);
stack_pointer = _PyFrame_GetStackPointer(frame);
PyStackRef_CLOSE(seq);
if (res == 0) {
- goto pop_1_error;
+ JUMP_TO_LABEL(pop_1_error);
}
}
stack_pointer += -1 + oparg;
}
TARGET(UNPACK_SEQUENCE_LIST) {
+ #if defined(Py_TAIL_CALL_INTERP)
+ int opcode;
+ #endif
+ opcode = UNPACK_SEQUENCE_LIST;
+ (void)(opcode);
+ _Py_CODEUNIT* const this_instr = next_instr;
+ (void)this_instr;
frame->instr_ptr = next_instr;
next_instr += 2;
INSTRUCTION_STATS(UNPACK_SEQUENCE_LIST);
seq = stack_pointer[-1];
values = &stack_pointer[-1];
PyObject *seq_o = PyStackRef_AsPyObjectBorrow(seq);
- DEOPT_IF(!PyList_CheckExact(seq_o), UNPACK_SEQUENCE);
- DEOPT_IF(!LOCK_OBJECT(seq_o), UNPACK_SEQUENCE);
+ if (!PyList_CheckExact(seq_o)) {
+ UPDATE_MISS_STATS(UNPACK_SEQUENCE);
+ assert(_PyOpcode_Deopt[opcode] == (UNPACK_SEQUENCE));
+ JUMP_TO_PREDICTED(UNPACK_SEQUENCE);
+ }
+ if (!LOCK_OBJECT(seq_o)) {
+ UPDATE_MISS_STATS(UNPACK_SEQUENCE);
+ assert(_PyOpcode_Deopt[opcode] == (UNPACK_SEQUENCE));
+ JUMP_TO_PREDICTED(UNPACK_SEQUENCE);
+ }
if (PyList_GET_SIZE(seq_o) != oparg) {
UNLOCK_OBJECT(seq_o);
- DEOPT_IF(true, UNPACK_SEQUENCE);
+ if (true) {
+ UPDATE_MISS_STATS(UNPACK_SEQUENCE);
+ assert(_PyOpcode_Deopt[opcode] == (UNPACK_SEQUENCE));
+ JUMP_TO_PREDICTED(UNPACK_SEQUENCE);
+ }
}
STAT_INC(UNPACK_SEQUENCE, hit);
PyObject **items = _PyList_ITEMS(seq_o);
}
TARGET(UNPACK_SEQUENCE_TUPLE) {
+ #if defined(Py_TAIL_CALL_INTERP)
+ int opcode;
+ #endif
+ opcode = UNPACK_SEQUENCE_TUPLE;
+ (void)(opcode);
+ _Py_CODEUNIT* const this_instr = next_instr;
+ (void)this_instr;
frame->instr_ptr = next_instr;
next_instr += 2;
INSTRUCTION_STATS(UNPACK_SEQUENCE_TUPLE);
seq = stack_pointer[-1];
values = &stack_pointer[-1];
PyObject *seq_o = PyStackRef_AsPyObjectBorrow(seq);
- DEOPT_IF(!PyTuple_CheckExact(seq_o), UNPACK_SEQUENCE);
- DEOPT_IF(PyTuple_GET_SIZE(seq_o) != oparg, UNPACK_SEQUENCE);
+ if (!PyTuple_CheckExact(seq_o)) {
+ UPDATE_MISS_STATS(UNPACK_SEQUENCE);
+ assert(_PyOpcode_Deopt[opcode] == (UNPACK_SEQUENCE));
+ JUMP_TO_PREDICTED(UNPACK_SEQUENCE);
+ }
+ if (PyTuple_GET_SIZE(seq_o) != oparg) {
+ UPDATE_MISS_STATS(UNPACK_SEQUENCE);
+ assert(_PyOpcode_Deopt[opcode] == (UNPACK_SEQUENCE));
+ JUMP_TO_PREDICTED(UNPACK_SEQUENCE);
+ }
STAT_INC(UNPACK_SEQUENCE, hit);
PyObject **items = _PyTuple_ITEMS(seq_o);
for (int i = oparg; --i >= 0; ) {
}
TARGET(UNPACK_SEQUENCE_TWO_TUPLE) {
+ #if defined(Py_TAIL_CALL_INTERP)
+ int opcode;
+ #endif
+ opcode = UNPACK_SEQUENCE_TWO_TUPLE;
+ (void)(opcode);
+ _Py_CODEUNIT* const this_instr = next_instr;
+ (void)this_instr;
frame->instr_ptr = next_instr;
next_instr += 2;
INSTRUCTION_STATS(UNPACK_SEQUENCE_TWO_TUPLE);
seq = stack_pointer[-1];
assert(oparg == 2);
PyObject *seq_o = PyStackRef_AsPyObjectBorrow(seq);
- DEOPT_IF(!PyTuple_CheckExact(seq_o), UNPACK_SEQUENCE);
- DEOPT_IF(PyTuple_GET_SIZE(seq_o) != 2, UNPACK_SEQUENCE);
+ if (!PyTuple_CheckExact(seq_o)) {
+ UPDATE_MISS_STATS(UNPACK_SEQUENCE);
+ assert(_PyOpcode_Deopt[opcode] == (UNPACK_SEQUENCE));
+ JUMP_TO_PREDICTED(UNPACK_SEQUENCE);
+ }
+ if (PyTuple_GET_SIZE(seq_o) != 2) {
+ UPDATE_MISS_STATS(UNPACK_SEQUENCE);
+ assert(_PyOpcode_Deopt[opcode] == (UNPACK_SEQUENCE));
+ JUMP_TO_PREDICTED(UNPACK_SEQUENCE);
+ }
STAT_INC(UNPACK_SEQUENCE, hit);
val0 = PyStackRef_FromPyObjectNew(PyTuple_GET_ITEM(seq_o, 0));
val1 = PyStackRef_FromPyObjectNew(PyTuple_GET_ITEM(seq_o, 1));
}
TARGET(WITH_EXCEPT_START) {
+ #if defined(Py_TAIL_CALL_INTERP)
+ int opcode;
+ #endif
+ opcode = WITH_EXCEPT_START;
+ (void)(opcode);
frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(WITH_EXCEPT_START);
(3 + has_self) | PY_VECTORCALL_ARGUMENTS_OFFSET, NULL);
stack_pointer = _PyFrame_GetStackPointer(frame);
if (res_o == NULL) {
- goto error;
+ JUMP_TO_LABEL(error);
}
res = PyStackRef_FromPyObjectSteal(res_o);
stack_pointer[0] = res;
}
TARGET(YIELD_VALUE) {
+ #if defined(Py_TAIL_CALL_INTERP)
+ int opcode;
+ #endif
+ opcode = YIELD_VALUE;
+ (void)(opcode);
frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(YIELD_VALUE);
}
/* END INSTRUCTIONS */
+#ifndef Py_TAIL_CALL_INTERP
#if USE_COMPUTED_GOTOS
_unknown_opcode:
#else
next_instr points the current instruction without TARGET(). */
opcode = next_instr->op.code;
_PyErr_Format(tstate, PyExc_SystemError,
- "%U:%d: unknown opcode %d",
- _PyFrame_GetCode(frame)->co_filename,
- PyUnstable_InterpreterFrame_GetLine(frame),
- opcode);
- goto error;
+ "%U:%d: unknown opcode %d",
+ _PyFrame_GetCode(frame)->co_filename,
+ PyUnstable_InterpreterFrame_GetLine(frame),
+ opcode);
+JUMP_TO_LABEL(error);
+
}
/* This should never be reached. Every opcode should end with DISPATCH()
or goto error. */
Py_UNREACHABLE();
+#endif /* Py_TAIL_CALL_INTERP */
/* BEGIN LABELS */
- pop_4_error:
+ LABEL(pop_4_error)
{
STACK_SHRINK(4);
- goto error;
+ JUMP_TO_LABEL(error);
}
- pop_3_error:
+ LABEL(pop_3_error)
{
STACK_SHRINK(3);
- goto error;
+ JUMP_TO_LABEL(error);
}
- pop_2_error:
+ LABEL(pop_2_error)
{
STACK_SHRINK(2);
- goto error;
+ JUMP_TO_LABEL(error);
}
- pop_1_error:
+ LABEL(pop_1_error)
{
STACK_SHRINK(1);
- goto error;
+ JUMP_TO_LABEL(error);
}
- error:
+ LABEL(error)
{
/* Double-check exception status. */
#ifdef NDEBUG
_PyEval_MonitorRaise(tstate, frame, next_instr-1);
stack_pointer = _PyFrame_GetStackPointer(frame);
_PyFrame_SetStackPointer(frame, stack_pointer);
- goto exception_unwind;
+ JUMP_TO_LABEL(exception_unwind);
}
- exception_unwind:
+ LABEL(exception_unwind)
{
/* STACK SPILLED */
/* We can't use frame->instr_ptr here, as RERAISE may have set it */
PyStackRef_XCLOSE(ref);
}
monitor_unwind(tstate, frame, next_instr-1);
- goto exit_unwind;
+ JUMP_TO_LABEL(exit_unwind);
}
assert(STACK_LEVEL() >= level);
_PyStackRef *new_top = _PyFrame_Stackbase(frame) + level;
int frame_lasti = _PyInterpreterFrame_LASTI(frame);
PyObject *lasti = PyLong_FromLong(frame_lasti);
if (lasti == NULL) {
- goto exception_unwind;
+ JUMP_TO_LABEL(exception_unwind);
}
_PyFrame_StackPush(frame, PyStackRef_FromPyObjectSteal(lasti));
}
next_instr = _PyFrame_GetBytecode(frame) + handler;
int err = monitor_handled(tstate, frame, next_instr, exc);
if (err < 0) {
- goto exception_unwind;
+ JUMP_TO_LABEL(exception_unwind);
}
/* Resume normal execution */
#ifdef LLTRACE
}
#endif
stack_pointer = _PyFrame_GetStackPointer(frame);
+ #ifdef Py_TAIL_CALL_INTERP
+ int opcode;
+ #endif
DISPATCH();
}
- exit_unwind:
+ LABEL(exit_unwind)
{
/* STACK SPILLED */
assert(_PyErr_Occurred(tstate));
}
next_instr = frame->instr_ptr;
stack_pointer = _PyFrame_GetStackPointer(frame);
- goto error;
+ JUMP_TO_LABEL(error);
}
- start_frame:
+ LABEL(start_frame)
{
/* STACK SPILLED */
int too_deep = _Py_EnterRecursivePy(tstate);
if (too_deep) {
- goto exit_unwind;
+ JUMP_TO_LABEL(exit_unwind);
}
next_instr = frame->instr_ptr;
#ifdef LLTRACE
int lltrace = maybe_lltrace_resume_frame(frame, GLOBALS());
frame->lltrace = lltrace;
if (lltrace < 0) {
- goto exit_unwind;
+ JUMP_TO_LABEL(exit_unwind);
}
}
#endif
assert(!_PyErr_Occurred(tstate));
#endif
stack_pointer = _PyFrame_GetStackPointer(frame);
+ #ifdef Py_TAIL_CALL_INTERP
+ int opcode;
+ #endif
DISPATCH();
}
projects / thirdparty / Python / cpython.git / commitdiff
