Splits the "cold" path, deopts and exits, from the "hot" path, reducing the size of most jitted instructions, at the cost of slower exits.
PyCodeObject *code; // Weak (NULL if no corresponding ENTER_EXECUTOR).
} _PyVMData;
+#define UOP_FORMAT_TARGET 0
+#define UOP_FORMAT_EXIT 1
+#define UOP_FORMAT_JUMP 2
+#define UOP_FORMAT_UNUSED 3
+
+/* Depending on the format,
+ * the 32 bits between the oparg and operand are:
+ * UOP_FORMAT_TARGET:
+ * uint32_t target;
+ * UOP_FORMAT_EXIT
+ * uint16_t exit_index;
+ * uint16_t error_target;
+ * UOP_FORMAT_JUMP
+ * uint16_t jump_target;
+ * uint16_t error_target;
+ */
typedef struct {
- uint16_t opcode;
+ uint16_t opcode:14;
+ uint16_t format:2;
uint16_t oparg;
union {
uint32_t target;
- uint32_t exit_index;
+ struct {
+ union {
+ uint16_t exit_index;
+ uint16_t jump_target;
+ };
+ uint16_t error_target;
+ };
};
uint64_t operand; // A cache entry
} _PyUOpInstruction;
+static inline uint32_t uop_get_target(const _PyUOpInstruction *inst)
+{
+ assert(inst->format == UOP_FORMAT_TARGET);
+ return inst->target;
+}
+
+static inline uint16_t uop_get_exit_index(const _PyUOpInstruction *inst)
+{
+ assert(inst->format == UOP_FORMAT_EXIT);
+ return inst->exit_index;
+}
+
+static inline uint16_t uop_get_jump_target(const _PyUOpInstruction *inst)
+{
+ assert(inst->format == UOP_FORMAT_JUMP);
+ return inst->jump_target;
+}
+
+static inline uint16_t uop_get_error_target(const _PyUOpInstruction *inst)
+{
+ assert(inst->format != UOP_FORMAT_TARGET);
+ return inst->error_target;
+}
+
typedef struct _exit_data {
uint32_t target;
int16_t temperature;
#define HAS_PURE_FLAG (2048)
#define HAS_PASSTHROUGH_FLAG (4096)
#define HAS_OPARG_AND_1_FLAG (8192)
+#define HAS_ERROR_NO_POP_FLAG (16384)
#define OPCODE_HAS_ARG(OP) (_PyOpcode_opcode_metadata[OP].flags & (HAS_ARG_FLAG))
#define OPCODE_HAS_CONST(OP) (_PyOpcode_opcode_metadata[OP].flags & (HAS_CONST_FLAG))
#define OPCODE_HAS_NAME(OP) (_PyOpcode_opcode_metadata[OP].flags & (HAS_NAME_FLAG))
#define OPCODE_HAS_PURE(OP) (_PyOpcode_opcode_metadata[OP].flags & (HAS_PURE_FLAG))
#define OPCODE_HAS_PASSTHROUGH(OP) (_PyOpcode_opcode_metadata[OP].flags & (HAS_PASSTHROUGH_FLAG))
#define OPCODE_HAS_OPARG_AND_1(OP) (_PyOpcode_opcode_metadata[OP].flags & (HAS_OPARG_AND_1_FLAG))
+#define OPCODE_HAS_ERROR_NO_POP(OP) (_PyOpcode_opcode_metadata[OP].flags & (HAS_ERROR_NO_POP_FLAG))
#define OPARG_FULL 0
#define OPARG_CACHE_1 1
extern const struct opcode_metadata _PyOpcode_opcode_metadata[268];
#ifdef NEED_OPCODE_METADATA
const struct opcode_metadata _PyOpcode_opcode_metadata[268] = {
- [BEFORE_ASYNC_WITH] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG },
- [BEFORE_WITH] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG },
+ [BEFORE_ASYNC_WITH] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG },
+ [BEFORE_WITH] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG },
[BINARY_OP] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG },
- [BINARY_OP_ADD_FLOAT] = { true, INSTR_FMT_IXC, HAS_DEOPT_FLAG | HAS_EXIT_FLAG },
- [BINARY_OP_ADD_INT] = { true, INSTR_FMT_IXC, HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG },
- [BINARY_OP_ADD_UNICODE] = { true, INSTR_FMT_IXC, HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG },
+ [BINARY_OP_ADD_FLOAT] = { true, INSTR_FMT_IXC, HAS_EXIT_FLAG },
+ [BINARY_OP_ADD_INT] = { true, INSTR_FMT_IXC, HAS_EXIT_FLAG | HAS_ERROR_FLAG },
+ [BINARY_OP_ADD_UNICODE] = { true, INSTR_FMT_IXC, HAS_EXIT_FLAG | HAS_ERROR_FLAG },
[BINARY_OP_INPLACE_ADD_UNICODE] = { true, INSTR_FMT_IXC, HAS_LOCAL_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG },
- [BINARY_OP_MULTIPLY_FLOAT] = { true, INSTR_FMT_IXC, HAS_DEOPT_FLAG | HAS_EXIT_FLAG },
- [BINARY_OP_MULTIPLY_INT] = { true, INSTR_FMT_IXC, HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG },
- [BINARY_OP_SUBTRACT_FLOAT] = { true, INSTR_FMT_IXC, HAS_DEOPT_FLAG | HAS_EXIT_FLAG },
- [BINARY_OP_SUBTRACT_INT] = { true, INSTR_FMT_IXC, HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG },
+ [BINARY_OP_MULTIPLY_FLOAT] = { true, INSTR_FMT_IXC, HAS_EXIT_FLAG },
+ [BINARY_OP_MULTIPLY_INT] = { true, INSTR_FMT_IXC, HAS_EXIT_FLAG | HAS_ERROR_FLAG },
+ [BINARY_OP_SUBTRACT_FLOAT] = { true, INSTR_FMT_IXC, HAS_EXIT_FLAG },
+ [BINARY_OP_SUBTRACT_INT] = { true, INSTR_FMT_IXC, HAS_EXIT_FLAG | HAS_ERROR_FLAG },
[BINARY_SLICE] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG },
[BINARY_SUBSCR] = { true, INSTR_FMT_IXC, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG },
[BINARY_SUBSCR_DICT] = { true, INSTR_FMT_IXC, HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG },
[BUILD_CONST_KEY_MAP] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG },
[BUILD_LIST] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG },
[BUILD_MAP] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG },
- [BUILD_SET] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG },
+ [BUILD_SET] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG },
[BUILD_SLICE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG },
[BUILD_STRING] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG },
[BUILD_TUPLE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG },
[CACHE] = { true, INSTR_FMT_IX, HAS_ESCAPES_FLAG },
- [CALL] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG },
- [CALL_ALLOC_AND_ENTER_INIT] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG },
+ [CALL] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG },
+ [CALL_ALLOC_AND_ENTER_INIT] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG },
[CALL_BOUND_METHOD_EXACT_ARGS] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG },
[CALL_BUILTIN_CLASS] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG },
[CALL_BUILTIN_FAST] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG },
[CALL_BUILTIN_FAST_WITH_KEYWORDS] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG },
[CALL_BUILTIN_O] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG },
- [CALL_FUNCTION_EX] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG },
+ [CALL_FUNCTION_EX] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG },
[CALL_INTRINSIC_1] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG },
[CALL_INTRINSIC_2] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG },
- [CALL_ISINSTANCE] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG },
- [CALL_KW] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG },
- [CALL_LEN] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG },
- [CALL_LIST_APPEND] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG },
+ [CALL_ISINSTANCE] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG },
+ [CALL_KW] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG },
+ [CALL_LEN] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG },
+ [CALL_LIST_APPEND] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG },
[CALL_METHOD_DESCRIPTOR_FAST] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG },
[CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG },
[CALL_METHOD_DESCRIPTOR_NOARGS] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG },
[CALL_TYPE_1] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_DEOPT_FLAG },
[CHECK_EG_MATCH] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG },
[CHECK_EXC_MATCH] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG },
- [CLEANUP_THROW] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG },
+ [CLEANUP_THROW] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG },
[COMPARE_OP] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG },
- [COMPARE_OP_FLOAT] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ESCAPES_FLAG },
+ [COMPARE_OP_FLOAT] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_EXIT_FLAG | HAS_ESCAPES_FLAG },
[COMPARE_OP_INT] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG },
- [COMPARE_OP_STR] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG },
+ [COMPARE_OP_STR] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_EXIT_FLAG },
[CONTAINS_OP] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG },
[CONTAINS_OP_DICT] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG },
[CONTAINS_OP_SET] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG },
[COPY] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_PURE_FLAG },
[COPY_FREE_VARS] = { true, INSTR_FMT_IB, HAS_ARG_FLAG },
[DELETE_ATTR] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG },
- [DELETE_DEREF] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_FREE_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG },
- [DELETE_FAST] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_LOCAL_FLAG | HAS_ERROR_FLAG },
- [DELETE_GLOBAL] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG },
- [DELETE_NAME] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG },
+ [DELETE_DEREF] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_FREE_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG },
+ [DELETE_FAST] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_LOCAL_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG },
+ [DELETE_GLOBAL] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG },
+ [DELETE_NAME] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG },
[DELETE_SUBSCR] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG },
[DICT_MERGE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG },
[DICT_UPDATE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG },
- [END_ASYNC_FOR] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG },
+ [END_ASYNC_FOR] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG },
[END_FOR] = { true, INSTR_FMT_IX, HAS_PURE_FLAG },
[END_SEND] = { true, INSTR_FMT_IX, HAS_PURE_FLAG },
[ENTER_EXECUTOR] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG },
- [EXIT_INIT_CHECK] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG },
+ [EXIT_INIT_CHECK] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG },
[EXTENDED_ARG] = { true, INSTR_FMT_IB, HAS_ARG_FLAG },
[FORMAT_SIMPLE] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG },
[FORMAT_WITH_SPEC] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG },
- [FOR_ITER] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG },
+ [FOR_ITER] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG },
[FOR_ITER_GEN] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG },
[FOR_ITER_LIST] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_DEOPT_FLAG },
[FOR_ITER_RANGE] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG },
[FOR_ITER_TUPLE] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_DEOPT_FLAG },
[GET_AITER] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG },
- [GET_ANEXT] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG },
+ [GET_ANEXT] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG },
[GET_AWAITABLE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG },
[GET_ITER] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG },
[GET_LEN] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG },
- [GET_YIELD_FROM_ITER] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG },
+ [GET_YIELD_FROM_ITER] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG },
[IMPORT_FROM] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG },
[IMPORT_NAME] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG },
[INSTRUMENTED_CALL] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG },
[INSTRUMENTED_CALL_FUNCTION_EX] = { true, INSTR_FMT_IX, 0 },
[INSTRUMENTED_CALL_KW] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG },
- [INSTRUMENTED_END_FOR] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG },
- [INSTRUMENTED_END_SEND] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG },
- [INSTRUMENTED_FOR_ITER] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG },
+ [INSTRUMENTED_END_FOR] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG },
+ [INSTRUMENTED_END_SEND] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG },
+ [INSTRUMENTED_FOR_ITER] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG },
[INSTRUMENTED_INSTRUCTION] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG },
[INSTRUMENTED_JUMP_BACKWARD] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG },
[INSTRUMENTED_JUMP_FORWARD] = { true, INSTR_FMT_IB, HAS_ARG_FLAG },
[INSTRUMENTED_POP_JUMP_IF_NONE] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG },
[INSTRUMENTED_POP_JUMP_IF_NOT_NONE] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG },
[INSTRUMENTED_POP_JUMP_IF_TRUE] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG },
- [INSTRUMENTED_RESUME] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG },
- [INSTRUMENTED_RETURN_CONST] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_CONST_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG },
- [INSTRUMENTED_RETURN_VALUE] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG },
- [INSTRUMENTED_YIELD_VALUE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG },
+ [INSTRUMENTED_RESUME] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG },
+ [INSTRUMENTED_RETURN_CONST] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_CONST_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG },
+ [INSTRUMENTED_RETURN_VALUE] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG },
+ [INSTRUMENTED_YIELD_VALUE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG },
[INTERPRETER_EXIT] = { true, INSTR_FMT_IX, HAS_ESCAPES_FLAG },
[IS_OP] = { true, INSTR_FMT_IB, HAS_ARG_FLAG },
[JUMP_BACKWARD] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_EVAL_BREAK_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG },
[LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG },
[LOAD_ATTR_INSTANCE_VALUE] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG },
[LOAD_ATTR_METHOD_LAZY_DICT] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG },
- [LOAD_ATTR_METHOD_NO_DICT] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG },
+ [LOAD_ATTR_METHOD_NO_DICT] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_EXIT_FLAG },
[LOAD_ATTR_METHOD_WITH_VALUES] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG },
[LOAD_ATTR_MODULE] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_DEOPT_FLAG },
- [LOAD_ATTR_NONDESCRIPTOR_NO_DICT] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG },
+ [LOAD_ATTR_NONDESCRIPTOR_NO_DICT] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_EXIT_FLAG },
[LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG },
[LOAD_ATTR_PROPERTY] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG },
[LOAD_ATTR_SLOT] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG },
[LOAD_DEREF] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_FREE_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG },
[LOAD_FAST] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_LOCAL_FLAG | HAS_PURE_FLAG },
[LOAD_FAST_AND_CLEAR] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_LOCAL_FLAG },
- [LOAD_FAST_CHECK] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_LOCAL_FLAG | HAS_ERROR_FLAG },
+ [LOAD_FAST_CHECK] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_LOCAL_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG },
[LOAD_FAST_LOAD_FAST] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_LOCAL_FLAG },
- [LOAD_FROM_DICT_OR_DEREF] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_FREE_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG },
- [LOAD_FROM_DICT_OR_GLOBALS] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG },
+ [LOAD_FROM_DICT_OR_DEREF] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_FREE_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG },
+ [LOAD_FROM_DICT_OR_GLOBALS] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG },
[LOAD_GLOBAL] = { true, INSTR_FMT_IBC000, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG },
[LOAD_GLOBAL_BUILTIN] = { true, INSTR_FMT_IBC000, HAS_ARG_FLAG | HAS_DEOPT_FLAG },
[LOAD_GLOBAL_MODULE] = { true, INSTR_FMT_IBC000, HAS_ARG_FLAG | HAS_DEOPT_FLAG },
[LOAD_LOCALS] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG },
- [LOAD_NAME] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG },
+ [LOAD_NAME] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG },
[LOAD_SUPER_ATTR] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG },
[LOAD_SUPER_ATTR_ATTR] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG },
[LOAD_SUPER_ATTR_METHOD] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG },
- [MAKE_CELL] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_FREE_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG },
- [MAKE_FUNCTION] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG },
+ [MAKE_CELL] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_FREE_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG },
+ [MAKE_FUNCTION] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG },
[MAP_ADD] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG },
[MATCH_CLASS] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG },
[MATCH_KEYS] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG },
[POP_TOP] = { true, INSTR_FMT_IX, HAS_PURE_FLAG },
[PUSH_EXC_INFO] = { true, INSTR_FMT_IX, 0 },
[PUSH_NULL] = { true, INSTR_FMT_IX, HAS_PURE_FLAG },
- [RAISE_VARARGS] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG },
- [RERAISE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG },
+ [RAISE_VARARGS] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG },
+ [RERAISE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG },
[RESERVED] = { true, INSTR_FMT_IX, HAS_ESCAPES_FLAG },
[RESUME] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG },
[RESUME_CHECK] = { true, INSTR_FMT_IX, HAS_DEOPT_FLAG },
[RETURN_CONST] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_CONST_FLAG | HAS_ESCAPES_FLAG },
- [RETURN_GENERATOR] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG },
+ [RETURN_GENERATOR] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG },
[RETURN_VALUE] = { true, INSTR_FMT_IX, HAS_ESCAPES_FLAG },
- [SEND] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG },
+ [SEND] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG },
[SEND_GEN] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG },
[SETUP_ANNOTATIONS] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG },
[SET_ADD] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG },
[SET_UPDATE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG },
[STORE_ATTR] = { true, INSTR_FMT_IBC000, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG },
[STORE_ATTR_INSTANCE_VALUE] = { true, INSTR_FMT_IXC000, HAS_DEOPT_FLAG | HAS_EXIT_FLAG },
- [STORE_ATTR_SLOT] = { true, INSTR_FMT_IXC000, HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ESCAPES_FLAG },
+ [STORE_ATTR_SLOT] = { true, INSTR_FMT_IXC000, HAS_EXIT_FLAG | HAS_ESCAPES_FLAG },
[STORE_ATTR_WITH_HINT] = { true, INSTR_FMT_IBC000, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG },
[STORE_DEREF] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_FREE_FLAG | HAS_ESCAPES_FLAG },
[STORE_FAST] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_LOCAL_FLAG },
[STORE_SUBSCR_LIST_INT] = { true, INSTR_FMT_IXC, HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG },
[SWAP] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_PURE_FLAG },
[TO_BOOL] = { true, INSTR_FMT_IXC00, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG },
- [TO_BOOL_ALWAYS_TRUE] = { true, INSTR_FMT_IXC00, HAS_DEOPT_FLAG | HAS_EXIT_FLAG },
- [TO_BOOL_BOOL] = { true, INSTR_FMT_IXC00, HAS_DEOPT_FLAG | HAS_EXIT_FLAG },
- [TO_BOOL_INT] = { true, INSTR_FMT_IXC00, HAS_DEOPT_FLAG | HAS_EXIT_FLAG },
- [TO_BOOL_LIST] = { true, INSTR_FMT_IXC00, HAS_DEOPT_FLAG | HAS_EXIT_FLAG },
- [TO_BOOL_NONE] = { true, INSTR_FMT_IXC00, HAS_DEOPT_FLAG | HAS_EXIT_FLAG },
- [TO_BOOL_STR] = { true, INSTR_FMT_IXC00, HAS_DEOPT_FLAG | HAS_EXIT_FLAG },
+ [TO_BOOL_ALWAYS_TRUE] = { true, INSTR_FMT_IXC00, HAS_EXIT_FLAG },
+ [TO_BOOL_BOOL] = { true, INSTR_FMT_IXC00, HAS_EXIT_FLAG },
+ [TO_BOOL_INT] = { true, INSTR_FMT_IXC00, HAS_EXIT_FLAG },
+ [TO_BOOL_LIST] = { true, INSTR_FMT_IXC00, HAS_EXIT_FLAG },
+ [TO_BOOL_NONE] = { true, INSTR_FMT_IXC00, HAS_EXIT_FLAG },
+ [TO_BOOL_STR] = { true, INSTR_FMT_IXC00, HAS_EXIT_FLAG },
[UNARY_INVERT] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG },
[UNARY_NEGATIVE] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG },
[UNARY_NOT] = { true, INSTR_FMT_IX, HAS_PURE_FLAG },
#ifdef NEED_OPCODE_METADATA
const struct opcode_macro_expansion
_PyOpcode_macro_expansion[256] = {
- [BEFORE_ASYNC_WITH] = { .nuops = 1, .uops = { { _BEFORE_ASYNC_WITH, 0, 0 } } },
- [BEFORE_WITH] = { .nuops = 1, .uops = { { _BEFORE_WITH, 0, 0 } } },
[BINARY_OP] = { .nuops = 1, .uops = { { _BINARY_OP, 0, 0 } } },
[BINARY_OP_ADD_FLOAT] = { .nuops = 2, .uops = { { _GUARD_BOTH_FLOAT, 0, 0 }, { _BINARY_OP_ADD_FLOAT, 0, 0 } } },
[BINARY_OP_ADD_INT] = { .nuops = 2, .uops = { { _GUARD_BOTH_INT, 0, 0 }, { _BINARY_OP_ADD_INT, 0, 0 } } },
[BUILD_CONST_KEY_MAP] = { .nuops = 1, .uops = { { _BUILD_CONST_KEY_MAP, 0, 0 } } },
[BUILD_LIST] = { .nuops = 1, .uops = { { _BUILD_LIST, 0, 0 } } },
[BUILD_MAP] = { .nuops = 1, .uops = { { _BUILD_MAP, 0, 0 } } },
- [BUILD_SET] = { .nuops = 1, .uops = { { _BUILD_SET, 0, 0 } } },
[BUILD_SLICE] = { .nuops = 1, .uops = { { _BUILD_SLICE, 0, 0 } } },
[BUILD_STRING] = { .nuops = 1, .uops = { { _BUILD_STRING, 0, 0 } } },
[BUILD_TUPLE] = { .nuops = 1, .uops = { { _BUILD_TUPLE, 0, 0 } } },
[LOAD_GLOBAL_BUILTIN] = { .nuops = 3, .uops = { { _GUARD_GLOBALS_VERSION, 1, 1 }, { _GUARD_BUILTINS_VERSION, 1, 2 }, { _LOAD_GLOBAL_BUILTINS, 1, 3 } } },
[LOAD_GLOBAL_MODULE] = { .nuops = 2, .uops = { { _GUARD_GLOBALS_VERSION, 1, 1 }, { _LOAD_GLOBAL_MODULE, 1, 3 } } },
[LOAD_LOCALS] = { .nuops = 1, .uops = { { _LOAD_LOCALS, 0, 0 } } },
- [LOAD_NAME] = { .nuops = 1, .uops = { { _LOAD_NAME, 0, 0 } } },
[LOAD_SUPER_ATTR_ATTR] = { .nuops = 1, .uops = { { _LOAD_SUPER_ATTR_ATTR, 0, 0 } } },
[LOAD_SUPER_ATTR_METHOD] = { .nuops = 1, .uops = { { _LOAD_SUPER_ATTR_METHOD, 0, 0 } } },
[MAKE_CELL] = { .nuops = 1, .uops = { { _MAKE_CELL, 0, 0 } } },
#include <stdbool.h>
// This is the length of the trace we project initially.
-#define UOP_MAX_TRACE_LENGTH 512
+#define UOP_MAX_TRACE_LENGTH 800
#define TRACE_STACK_SIZE 5
#define _DELETE_GLOBAL DELETE_GLOBAL
#define _DELETE_NAME DELETE_NAME
#define _DELETE_SUBSCR DELETE_SUBSCR
+#define _DEOPT 341
#define _DICT_MERGE DICT_MERGE
#define _DICT_UPDATE DICT_UPDATE
#define _END_SEND END_SEND
+#define _ERROR_POP_N 342
#define _EXIT_INIT_CHECK EXIT_INIT_CHECK
-#define _FATAL_ERROR 341
+#define _FATAL_ERROR 343
#define _FORMAT_SIMPLE FORMAT_SIMPLE
#define _FORMAT_WITH_SPEC FORMAT_WITH_SPEC
-#define _FOR_ITER 342
+#define _FOR_ITER 344
#define _FOR_ITER_GEN FOR_ITER_GEN
-#define _FOR_ITER_TIER_TWO 343
+#define _FOR_ITER_TIER_TWO 345
#define _GET_AITER GET_AITER
#define _GET_ANEXT GET_ANEXT
#define _GET_AWAITABLE GET_AWAITABLE
#define _GET_ITER GET_ITER
#define _GET_LEN GET_LEN
#define _GET_YIELD_FROM_ITER GET_YIELD_FROM_ITER
-#define _GUARD_BOTH_FLOAT 344
-#define _GUARD_BOTH_INT 345
-#define _GUARD_BOTH_UNICODE 346
-#define _GUARD_BUILTINS_VERSION 347
-#define _GUARD_DORV_VALUES 348
-#define _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT 349
-#define _GUARD_GLOBALS_VERSION 350
-#define _GUARD_IS_FALSE_POP 351
-#define _GUARD_IS_NONE_POP 352
-#define _GUARD_IS_NOT_NONE_POP 353
-#define _GUARD_IS_TRUE_POP 354
-#define _GUARD_KEYS_VERSION 355
-#define _GUARD_NOT_EXHAUSTED_LIST 356
-#define _GUARD_NOT_EXHAUSTED_RANGE 357
-#define _GUARD_NOT_EXHAUSTED_TUPLE 358
-#define _GUARD_TYPE_VERSION 359
-#define _INIT_CALL_BOUND_METHOD_EXACT_ARGS 360
-#define _INIT_CALL_PY_EXACT_ARGS 361
-#define _INIT_CALL_PY_EXACT_ARGS_0 362
-#define _INIT_CALL_PY_EXACT_ARGS_1 363
-#define _INIT_CALL_PY_EXACT_ARGS_2 364
-#define _INIT_CALL_PY_EXACT_ARGS_3 365
-#define _INIT_CALL_PY_EXACT_ARGS_4 366
+#define _GUARD_BOTH_FLOAT 346
+#define _GUARD_BOTH_INT 347
+#define _GUARD_BOTH_UNICODE 348
+#define _GUARD_BUILTINS_VERSION 349
+#define _GUARD_DORV_VALUES 350
+#define _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT 351
+#define _GUARD_GLOBALS_VERSION 352
+#define _GUARD_IS_FALSE_POP 353
+#define _GUARD_IS_NONE_POP 354
+#define _GUARD_IS_NOT_NONE_POP 355
+#define _GUARD_IS_TRUE_POP 356
+#define _GUARD_KEYS_VERSION 357
+#define _GUARD_NOT_EXHAUSTED_LIST 358
+#define _GUARD_NOT_EXHAUSTED_RANGE 359
+#define _GUARD_NOT_EXHAUSTED_TUPLE 360
+#define _GUARD_TYPE_VERSION 361
+#define _INIT_CALL_BOUND_METHOD_EXACT_ARGS 362
+#define _INIT_CALL_PY_EXACT_ARGS 363
+#define _INIT_CALL_PY_EXACT_ARGS_0 364
+#define _INIT_CALL_PY_EXACT_ARGS_1 365
+#define _INIT_CALL_PY_EXACT_ARGS_2 366
+#define _INIT_CALL_PY_EXACT_ARGS_3 367
+#define _INIT_CALL_PY_EXACT_ARGS_4 368
#define _INSTRUMENTED_CALL INSTRUMENTED_CALL
#define _INSTRUMENTED_CALL_FUNCTION_EX INSTRUMENTED_CALL_FUNCTION_EX
#define _INSTRUMENTED_CALL_KW INSTRUMENTED_CALL_KW
#define _INSTRUMENTED_RETURN_CONST INSTRUMENTED_RETURN_CONST
#define _INSTRUMENTED_RETURN_VALUE INSTRUMENTED_RETURN_VALUE
#define _INSTRUMENTED_YIELD_VALUE INSTRUMENTED_YIELD_VALUE
-#define _INTERNAL_INCREMENT_OPT_COUNTER 367
-#define _IS_NONE 368
+#define _INTERNAL_INCREMENT_OPT_COUNTER 369
+#define _IS_NONE 370
#define _IS_OP IS_OP
-#define _ITER_CHECK_LIST 369
-#define _ITER_CHECK_RANGE 370
-#define _ITER_CHECK_TUPLE 371
-#define _ITER_JUMP_LIST 372
-#define _ITER_JUMP_RANGE 373
-#define _ITER_JUMP_TUPLE 374
-#define _ITER_NEXT_LIST 375
-#define _ITER_NEXT_RANGE 376
-#define _ITER_NEXT_TUPLE 377
-#define _JUMP_TO_TOP 378
+#define _ITER_CHECK_LIST 371
+#define _ITER_CHECK_RANGE 372
+#define _ITER_CHECK_TUPLE 373
+#define _ITER_JUMP_LIST 374
+#define _ITER_JUMP_RANGE 375
+#define _ITER_JUMP_TUPLE 376
+#define _ITER_NEXT_LIST 377
+#define _ITER_NEXT_RANGE 378
+#define _ITER_NEXT_TUPLE 379
+#define _JUMP_TO_TOP 380
#define _LIST_APPEND LIST_APPEND
#define _LIST_EXTEND LIST_EXTEND
#define _LOAD_ASSERTION_ERROR LOAD_ASSERTION_ERROR
-#define _LOAD_ATTR 379
-#define _LOAD_ATTR_CLASS 380
-#define _LOAD_ATTR_CLASS_0 381
-#define _LOAD_ATTR_CLASS_1 382
+#define _LOAD_ATTR 381
+#define _LOAD_ATTR_CLASS 382
+#define _LOAD_ATTR_CLASS_0 383
+#define _LOAD_ATTR_CLASS_1 384
#define _LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN
-#define _LOAD_ATTR_INSTANCE_VALUE 383
-#define _LOAD_ATTR_INSTANCE_VALUE_0 384
-#define _LOAD_ATTR_INSTANCE_VALUE_1 385
-#define _LOAD_ATTR_METHOD_LAZY_DICT 386
-#define _LOAD_ATTR_METHOD_NO_DICT 387
-#define _LOAD_ATTR_METHOD_WITH_VALUES 388
-#define _LOAD_ATTR_MODULE 389
-#define _LOAD_ATTR_NONDESCRIPTOR_NO_DICT 390
-#define _LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES 391
+#define _LOAD_ATTR_INSTANCE_VALUE 385
+#define _LOAD_ATTR_INSTANCE_VALUE_0 386
+#define _LOAD_ATTR_INSTANCE_VALUE_1 387
+#define _LOAD_ATTR_METHOD_LAZY_DICT 388
+#define _LOAD_ATTR_METHOD_NO_DICT 389
+#define _LOAD_ATTR_METHOD_WITH_VALUES 390
+#define _LOAD_ATTR_MODULE 391
+#define _LOAD_ATTR_NONDESCRIPTOR_NO_DICT 392
+#define _LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES 393
#define _LOAD_ATTR_PROPERTY LOAD_ATTR_PROPERTY
-#define _LOAD_ATTR_SLOT 392
-#define _LOAD_ATTR_SLOT_0 393
-#define _LOAD_ATTR_SLOT_1 394
-#define _LOAD_ATTR_WITH_HINT 395
+#define _LOAD_ATTR_SLOT 394
+#define _LOAD_ATTR_SLOT_0 395
+#define _LOAD_ATTR_SLOT_1 396
+#define _LOAD_ATTR_WITH_HINT 397
#define _LOAD_BUILD_CLASS LOAD_BUILD_CLASS
#define _LOAD_CONST LOAD_CONST
-#define _LOAD_CONST_INLINE 396
-#define _LOAD_CONST_INLINE_BORROW 397
-#define _LOAD_CONST_INLINE_BORROW_WITH_NULL 398
-#define _LOAD_CONST_INLINE_WITH_NULL 399
+#define _LOAD_CONST_INLINE 398
+#define _LOAD_CONST_INLINE_BORROW 399
+#define _LOAD_CONST_INLINE_BORROW_WITH_NULL 400
+#define _LOAD_CONST_INLINE_WITH_NULL 401
#define _LOAD_DEREF LOAD_DEREF
-#define _LOAD_FAST 400
-#define _LOAD_FAST_0 401
-#define _LOAD_FAST_1 402
-#define _LOAD_FAST_2 403
-#define _LOAD_FAST_3 404
-#define _LOAD_FAST_4 405
-#define _LOAD_FAST_5 406
-#define _LOAD_FAST_6 407
-#define _LOAD_FAST_7 408
+#define _LOAD_FAST 402
+#define _LOAD_FAST_0 403
+#define _LOAD_FAST_1 404
+#define _LOAD_FAST_2 405
+#define _LOAD_FAST_3 406
+#define _LOAD_FAST_4 407
+#define _LOAD_FAST_5 408
+#define _LOAD_FAST_6 409
+#define _LOAD_FAST_7 410
#define _LOAD_FAST_AND_CLEAR LOAD_FAST_AND_CLEAR
#define _LOAD_FAST_CHECK LOAD_FAST_CHECK
#define _LOAD_FAST_LOAD_FAST LOAD_FAST_LOAD_FAST
#define _LOAD_FROM_DICT_OR_DEREF LOAD_FROM_DICT_OR_DEREF
#define _LOAD_FROM_DICT_OR_GLOBALS LOAD_FROM_DICT_OR_GLOBALS
-#define _LOAD_GLOBAL 409
-#define _LOAD_GLOBAL_BUILTINS 410
-#define _LOAD_GLOBAL_MODULE 411
+#define _LOAD_GLOBAL 411
+#define _LOAD_GLOBAL_BUILTINS 412
+#define _LOAD_GLOBAL_MODULE 413
#define _LOAD_LOCALS LOAD_LOCALS
#define _LOAD_NAME LOAD_NAME
#define _LOAD_SUPER_ATTR_ATTR LOAD_SUPER_ATTR_ATTR
#define _MATCH_SEQUENCE MATCH_SEQUENCE
#define _NOP NOP
#define _POP_EXCEPT POP_EXCEPT
-#define _POP_FRAME 412
-#define _POP_JUMP_IF_FALSE 413
-#define _POP_JUMP_IF_TRUE 414
+#define _POP_FRAME 414
+#define _POP_JUMP_IF_FALSE 415
+#define _POP_JUMP_IF_TRUE 416
#define _POP_TOP POP_TOP
-#define _POP_TOP_LOAD_CONST_INLINE_BORROW 415
+#define _POP_TOP_LOAD_CONST_INLINE_BORROW 417
#define _PUSH_EXC_INFO PUSH_EXC_INFO
-#define _PUSH_FRAME 416
+#define _PUSH_FRAME 418
#define _PUSH_NULL PUSH_NULL
-#define _REPLACE_WITH_TRUE 417
+#define _REPLACE_WITH_TRUE 419
#define _RESUME_CHECK RESUME_CHECK
-#define _SAVE_RETURN_OFFSET 418
-#define _SEND 419
+#define _SAVE_RETURN_OFFSET 420
+#define _SEND 421
#define _SEND_GEN SEND_GEN
#define _SETUP_ANNOTATIONS SETUP_ANNOTATIONS
#define _SET_ADD SET_ADD
#define _SET_FUNCTION_ATTRIBUTE SET_FUNCTION_ATTRIBUTE
#define _SET_UPDATE SET_UPDATE
-#define _START_EXECUTOR 420
-#define _STORE_ATTR 421
-#define _STORE_ATTR_INSTANCE_VALUE 422
-#define _STORE_ATTR_SLOT 423
+#define _SIDE_EXIT 422
+#define _START_EXECUTOR 423
+#define _STORE_ATTR 424
+#define _STORE_ATTR_INSTANCE_VALUE 425
+#define _STORE_ATTR_SLOT 426
#define _STORE_ATTR_WITH_HINT STORE_ATTR_WITH_HINT
#define _STORE_DEREF STORE_DEREF
-#define _STORE_FAST 424
-#define _STORE_FAST_0 425
-#define _STORE_FAST_1 426
-#define _STORE_FAST_2 427
-#define _STORE_FAST_3 428
-#define _STORE_FAST_4 429
-#define _STORE_FAST_5 430
-#define _STORE_FAST_6 431
-#define _STORE_FAST_7 432
+#define _STORE_FAST 427
+#define _STORE_FAST_0 428
+#define _STORE_FAST_1 429
+#define _STORE_FAST_2 430
+#define _STORE_FAST_3 431
+#define _STORE_FAST_4 432
+#define _STORE_FAST_5 433
+#define _STORE_FAST_6 434
+#define _STORE_FAST_7 435
#define _STORE_FAST_LOAD_FAST STORE_FAST_LOAD_FAST
#define _STORE_FAST_STORE_FAST STORE_FAST_STORE_FAST
#define _STORE_GLOBAL STORE_GLOBAL
#define _STORE_NAME STORE_NAME
#define _STORE_SLICE STORE_SLICE
-#define _STORE_SUBSCR 433
+#define _STORE_SUBSCR 436
#define _STORE_SUBSCR_DICT STORE_SUBSCR_DICT
#define _STORE_SUBSCR_LIST_INT STORE_SUBSCR_LIST_INT
#define _SWAP SWAP
-#define _TO_BOOL 434
+#define _TO_BOOL 437
#define _TO_BOOL_BOOL TO_BOOL_BOOL
#define _TO_BOOL_INT TO_BOOL_INT
#define _TO_BOOL_LIST TO_BOOL_LIST
#define _UNARY_NEGATIVE UNARY_NEGATIVE
#define _UNARY_NOT UNARY_NOT
#define _UNPACK_EX UNPACK_EX
-#define _UNPACK_SEQUENCE 435
+#define _UNPACK_SEQUENCE 438
#define _UNPACK_SEQUENCE_LIST UNPACK_SEQUENCE_LIST
#define _UNPACK_SEQUENCE_TUPLE UNPACK_SEQUENCE_TUPLE
#define _UNPACK_SEQUENCE_TWO_TUPLE UNPACK_SEQUENCE_TWO_TUPLE
#define _WITH_EXCEPT_START WITH_EXCEPT_START
-#define MAX_UOP_ID 435
+#define MAX_UOP_ID 438
#ifdef __cplusplus
}
extern const uint8_t _PyUop_Replication[MAX_UOP_ID+1];
extern const char * const _PyOpcode_uop_name[MAX_UOP_ID+1];
+extern int _PyUop_num_popped(int opcode, int oparg);
+
#ifdef NEED_OPCODE_METADATA
const uint16_t _PyUop_Flags[MAX_UOP_ID+1] = {
[_NOP] = HAS_PURE_FLAG,
[_RESUME_CHECK] = HAS_DEOPT_FLAG,
- [_LOAD_FAST_CHECK] = HAS_ARG_FLAG | HAS_LOCAL_FLAG | HAS_ERROR_FLAG,
+ [_LOAD_FAST_CHECK] = HAS_ARG_FLAG | HAS_LOCAL_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG,
[_LOAD_FAST_0] = HAS_LOCAL_FLAG | HAS_PURE_FLAG,
[_LOAD_FAST_1] = HAS_LOCAL_FLAG | HAS_PURE_FLAG,
[_LOAD_FAST_2] = HAS_LOCAL_FLAG | HAS_PURE_FLAG,
[_UNARY_NEGATIVE] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG,
[_UNARY_NOT] = HAS_PURE_FLAG,
[_TO_BOOL] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG,
- [_TO_BOOL_BOOL] = HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_PASSTHROUGH_FLAG,
- [_TO_BOOL_INT] = HAS_DEOPT_FLAG | HAS_EXIT_FLAG,
- [_TO_BOOL_LIST] = HAS_DEOPT_FLAG | HAS_EXIT_FLAG,
- [_TO_BOOL_NONE] = HAS_DEOPT_FLAG | HAS_EXIT_FLAG,
- [_TO_BOOL_STR] = HAS_DEOPT_FLAG | HAS_EXIT_FLAG,
+ [_TO_BOOL_BOOL] = HAS_EXIT_FLAG | HAS_PASSTHROUGH_FLAG,
+ [_TO_BOOL_INT] = HAS_EXIT_FLAG,
+ [_TO_BOOL_LIST] = HAS_EXIT_FLAG,
+ [_TO_BOOL_NONE] = HAS_EXIT_FLAG,
+ [_TO_BOOL_STR] = HAS_EXIT_FLAG,
[_REPLACE_WITH_TRUE] = 0,
[_UNARY_INVERT] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG,
- [_GUARD_BOTH_INT] = HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_PASSTHROUGH_FLAG,
+ [_GUARD_BOTH_INT] = HAS_EXIT_FLAG | HAS_PASSTHROUGH_FLAG,
[_BINARY_OP_MULTIPLY_INT] = HAS_ERROR_FLAG | HAS_PURE_FLAG,
[_BINARY_OP_ADD_INT] = HAS_ERROR_FLAG | HAS_PURE_FLAG,
[_BINARY_OP_SUBTRACT_INT] = HAS_ERROR_FLAG | HAS_PURE_FLAG,
- [_GUARD_BOTH_FLOAT] = HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_PASSTHROUGH_FLAG,
+ [_GUARD_BOTH_FLOAT] = HAS_EXIT_FLAG | HAS_PASSTHROUGH_FLAG,
[_BINARY_OP_MULTIPLY_FLOAT] = HAS_PURE_FLAG,
[_BINARY_OP_ADD_FLOAT] = HAS_PURE_FLAG,
[_BINARY_OP_SUBTRACT_FLOAT] = HAS_PURE_FLAG,
- [_GUARD_BOTH_UNICODE] = HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_PASSTHROUGH_FLAG,
+ [_GUARD_BOTH_UNICODE] = HAS_EXIT_FLAG | HAS_PASSTHROUGH_FLAG,
[_BINARY_OP_ADD_UNICODE] = HAS_ERROR_FLAG | HAS_PURE_FLAG,
[_BINARY_SUBSCR] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG,
[_BINARY_SLICE] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG,
[_CALL_INTRINSIC_2] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG,
[_POP_FRAME] = HAS_ESCAPES_FLAG,
[_GET_AITER] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG,
- [_GET_ANEXT] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG,
+ [_GET_ANEXT] = HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG,
[_GET_AWAITABLE] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG,
[_POP_EXCEPT] = HAS_ESCAPES_FLAG,
[_LOAD_ASSERTION_ERROR] = 0,
[_LOAD_BUILD_CLASS] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG,
[_STORE_NAME] = HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG,
- [_DELETE_NAME] = HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG,
+ [_DELETE_NAME] = HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG,
[_UNPACK_SEQUENCE] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG,
[_UNPACK_SEQUENCE_TWO_TUPLE] = HAS_ARG_FLAG | HAS_DEOPT_FLAG,
[_UNPACK_SEQUENCE_TUPLE] = HAS_ARG_FLAG | HAS_DEOPT_FLAG,
[_STORE_ATTR] = HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG,
[_DELETE_ATTR] = HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG,
[_STORE_GLOBAL] = HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG,
- [_DELETE_GLOBAL] = HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG,
+ [_DELETE_GLOBAL] = HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG,
[_LOAD_LOCALS] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG,
- [_LOAD_FROM_DICT_OR_GLOBALS] = HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG,
- [_LOAD_NAME] = HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG,
+ [_LOAD_FROM_DICT_OR_GLOBALS] = HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG,
[_LOAD_GLOBAL] = HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG,
[_GUARD_GLOBALS_VERSION] = HAS_DEOPT_FLAG,
[_GUARD_BUILTINS_VERSION] = HAS_DEOPT_FLAG,
[_LOAD_GLOBAL_MODULE] = HAS_ARG_FLAG | HAS_DEOPT_FLAG,
[_LOAD_GLOBAL_BUILTINS] = HAS_ARG_FLAG | HAS_DEOPT_FLAG,
- [_DELETE_FAST] = HAS_ARG_FLAG | HAS_LOCAL_FLAG | HAS_ERROR_FLAG,
- [_MAKE_CELL] = HAS_ARG_FLAG | HAS_FREE_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG,
- [_DELETE_DEREF] = HAS_ARG_FLAG | HAS_FREE_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG,
- [_LOAD_FROM_DICT_OR_DEREF] = HAS_ARG_FLAG | HAS_FREE_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG,
+ [_DELETE_FAST] = HAS_ARG_FLAG | HAS_LOCAL_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG,
+ [_MAKE_CELL] = HAS_ARG_FLAG | HAS_FREE_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG,
+ [_DELETE_DEREF] = HAS_ARG_FLAG | HAS_FREE_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG,
+ [_LOAD_FROM_DICT_OR_DEREF] = HAS_ARG_FLAG | HAS_FREE_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG,
[_LOAD_DEREF] = HAS_ARG_FLAG | HAS_FREE_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG,
[_STORE_DEREF] = HAS_ARG_FLAG | HAS_FREE_FLAG | HAS_ESCAPES_FLAG,
[_COPY_FREE_VARS] = HAS_ARG_FLAG,
[_BUILD_LIST] = HAS_ARG_FLAG | HAS_ERROR_FLAG,
[_LIST_EXTEND] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG,
[_SET_UPDATE] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG,
- [_BUILD_SET] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG,
[_BUILD_MAP] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG,
[_SETUP_ANNOTATIONS] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG,
[_BUILD_CONST_KEY_MAP] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG,
[_LOAD_SUPER_ATTR_ATTR] = HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG,
[_LOAD_SUPER_ATTR_METHOD] = HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG,
[_LOAD_ATTR] = HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG,
- [_GUARD_TYPE_VERSION] = HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_PASSTHROUGH_FLAG,
+ [_GUARD_TYPE_VERSION] = HAS_EXIT_FLAG | HAS_PASSTHROUGH_FLAG,
[_CHECK_MANAGED_OBJECT_HAS_VALUES] = HAS_DEOPT_FLAG | HAS_PASSTHROUGH_FLAG,
[_LOAD_ATTR_INSTANCE_VALUE_0] = HAS_DEOPT_FLAG,
[_LOAD_ATTR_INSTANCE_VALUE_1] = HAS_DEOPT_FLAG,
[_MATCH_SEQUENCE] = 0,
[_MATCH_KEYS] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG,
[_GET_ITER] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG,
- [_GET_YIELD_FROM_ITER] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG,
- [_FOR_ITER_TIER_TWO] = HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG,
+ [_GET_YIELD_FROM_ITER] = HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG,
+ [_FOR_ITER_TIER_TWO] = HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG,
[_ITER_CHECK_LIST] = HAS_DEOPT_FLAG | HAS_PASSTHROUGH_FLAG,
[_GUARD_NOT_EXHAUSTED_LIST] = HAS_DEOPT_FLAG | HAS_PASSTHROUGH_FLAG,
[_ITER_NEXT_LIST] = 0,
[_ITER_CHECK_RANGE] = HAS_DEOPT_FLAG | HAS_PASSTHROUGH_FLAG,
[_GUARD_NOT_EXHAUSTED_RANGE] = HAS_DEOPT_FLAG | HAS_PASSTHROUGH_FLAG,
[_ITER_NEXT_RANGE] = HAS_ERROR_FLAG,
- [_BEFORE_ASYNC_WITH] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG,
- [_BEFORE_WITH] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG,
[_WITH_EXCEPT_START] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG,
[_PUSH_EXC_INFO] = 0,
[_GUARD_DORV_VALUES_INST_ATTR_FROM_DICT] = HAS_DEOPT_FLAG | HAS_PASSTHROUGH_FLAG,
[_CALL_TYPE_1] = HAS_ARG_FLAG | HAS_DEOPT_FLAG,
[_CALL_STR_1] = HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG,
[_CALL_TUPLE_1] = HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG,
- [_EXIT_INIT_CHECK] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG,
+ [_EXIT_INIT_CHECK] = HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG,
[_CALL_BUILTIN_CLASS] = HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG,
[_CALL_BUILTIN_O] = HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG,
[_CALL_BUILTIN_FAST] = HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG,
[_CALL_BUILTIN_FAST_WITH_KEYWORDS] = HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG,
- [_CALL_LEN] = HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG,
- [_CALL_ISINSTANCE] = HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG,
+ [_CALL_LEN] = HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG,
+ [_CALL_ISINSTANCE] = HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG,
[_CALL_METHOD_DESCRIPTOR_O] = HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG,
[_CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS] = HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG,
[_CALL_METHOD_DESCRIPTOR_NOARGS] = HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG,
[_CALL_METHOD_DESCRIPTOR_FAST] = HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG,
- [_MAKE_FUNCTION] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG,
+ [_MAKE_FUNCTION] = HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG,
[_SET_FUNCTION_ATTRIBUTE] = HAS_ARG_FLAG | HAS_ESCAPES_FLAG,
[_BUILD_SLICE] = HAS_ARG_FLAG | HAS_ERROR_FLAG,
[_CONVERT_VALUE] = HAS_ARG_FLAG | HAS_ERROR_FLAG,
[_COPY] = HAS_ARG_FLAG | HAS_PURE_FLAG,
[_BINARY_OP] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG,
[_SWAP] = HAS_ARG_FLAG | HAS_PURE_FLAG,
- [_GUARD_IS_TRUE_POP] = HAS_DEOPT_FLAG | HAS_EXIT_FLAG,
- [_GUARD_IS_FALSE_POP] = HAS_DEOPT_FLAG | HAS_EXIT_FLAG,
- [_GUARD_IS_NONE_POP] = HAS_DEOPT_FLAG | HAS_EXIT_FLAG,
- [_GUARD_IS_NOT_NONE_POP] = HAS_DEOPT_FLAG | HAS_EXIT_FLAG,
+ [_GUARD_IS_TRUE_POP] = HAS_EXIT_FLAG,
+ [_GUARD_IS_FALSE_POP] = HAS_EXIT_FLAG,
+ [_GUARD_IS_NONE_POP] = HAS_EXIT_FLAG,
+ [_GUARD_IS_NOT_NONE_POP] = HAS_EXIT_FLAG,
[_JUMP_TO_TOP] = HAS_EVAL_BREAK_FLAG,
[_SET_IP] = 0,
[_SAVE_RETURN_OFFSET] = HAS_ARG_FLAG,
- [_EXIT_TRACE] = HAS_DEOPT_FLAG | HAS_EXIT_FLAG,
+ [_EXIT_TRACE] = HAS_EXIT_FLAG,
[_CHECK_VALIDITY] = HAS_DEOPT_FLAG,
[_LOAD_CONST_INLINE] = HAS_PURE_FLAG,
[_LOAD_CONST_INLINE_BORROW] = HAS_PURE_FLAG,
[_LOAD_CONST_INLINE_BORROW_WITH_NULL] = HAS_PURE_FLAG,
[_CHECK_FUNCTION] = HAS_DEOPT_FLAG,
[_INTERNAL_INCREMENT_OPT_COUNTER] = 0,
- [_COLD_EXIT] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG,
+ [_COLD_EXIT] = HAS_ARG_FLAG | HAS_ESCAPES_FLAG,
[_START_EXECUTOR] = 0,
[_FATAL_ERROR] = HAS_ESCAPES_FLAG,
[_CHECK_VALIDITY_AND_SET_IP] = HAS_DEOPT_FLAG,
+ [_DEOPT] = 0,
+ [_SIDE_EXIT] = 0,
+ [_ERROR_POP_N] = HAS_ARG_FLAG,
};
const uint8_t _PyUop_Replication[MAX_UOP_ID+1] = {
};
const char *const _PyOpcode_uop_name[MAX_UOP_ID+1] = {
- [_BEFORE_ASYNC_WITH] = "_BEFORE_ASYNC_WITH",
- [_BEFORE_WITH] = "_BEFORE_WITH",
[_BINARY_OP] = "_BINARY_OP",
[_BINARY_OP_ADD_FLOAT] = "_BINARY_OP_ADD_FLOAT",
[_BINARY_OP_ADD_INT] = "_BINARY_OP_ADD_INT",
[_BUILD_CONST_KEY_MAP] = "_BUILD_CONST_KEY_MAP",
[_BUILD_LIST] = "_BUILD_LIST",
[_BUILD_MAP] = "_BUILD_MAP",
- [_BUILD_SET] = "_BUILD_SET",
[_BUILD_SLICE] = "_BUILD_SLICE",
[_BUILD_STRING] = "_BUILD_STRING",
[_BUILD_TUPLE] = "_BUILD_TUPLE",
[_DELETE_GLOBAL] = "_DELETE_GLOBAL",
[_DELETE_NAME] = "_DELETE_NAME",
[_DELETE_SUBSCR] = "_DELETE_SUBSCR",
+ [_DEOPT] = "_DEOPT",
[_DICT_MERGE] = "_DICT_MERGE",
[_DICT_UPDATE] = "_DICT_UPDATE",
[_END_SEND] = "_END_SEND",
+ [_ERROR_POP_N] = "_ERROR_POP_N",
[_EXIT_INIT_CHECK] = "_EXIT_INIT_CHECK",
[_EXIT_TRACE] = "_EXIT_TRACE",
[_FATAL_ERROR] = "_FATAL_ERROR",
[_LOAD_GLOBAL_BUILTINS] = "_LOAD_GLOBAL_BUILTINS",
[_LOAD_GLOBAL_MODULE] = "_LOAD_GLOBAL_MODULE",
[_LOAD_LOCALS] = "_LOAD_LOCALS",
- [_LOAD_NAME] = "_LOAD_NAME",
[_LOAD_SUPER_ATTR_ATTR] = "_LOAD_SUPER_ATTR_ATTR",
[_LOAD_SUPER_ATTR_METHOD] = "_LOAD_SUPER_ATTR_METHOD",
[_MAKE_CELL] = "_MAKE_CELL",
[_SET_FUNCTION_ATTRIBUTE] = "_SET_FUNCTION_ATTRIBUTE",
[_SET_IP] = "_SET_IP",
[_SET_UPDATE] = "_SET_UPDATE",
+ [_SIDE_EXIT] = "_SIDE_EXIT",
[_START_EXECUTOR] = "_START_EXECUTOR",
[_STORE_ATTR] = "_STORE_ATTR",
[_STORE_ATTR_INSTANCE_VALUE] = "_STORE_ATTR_INSTANCE_VALUE",
[_UNPACK_SEQUENCE_TWO_TUPLE] = "_UNPACK_SEQUENCE_TWO_TUPLE",
[_WITH_EXCEPT_START] = "_WITH_EXCEPT_START",
};
+int _PyUop_num_popped(int opcode, int oparg)
+{
+ switch(opcode) {
+ case _NOP:
+ return 0;
+ case _RESUME_CHECK:
+ return 0;
+ case _LOAD_FAST_CHECK:
+ return 0;
+ case _LOAD_FAST_0:
+ return 0;
+ case _LOAD_FAST_1:
+ return 0;
+ case _LOAD_FAST_2:
+ return 0;
+ case _LOAD_FAST_3:
+ return 0;
+ case _LOAD_FAST_4:
+ return 0;
+ case _LOAD_FAST_5:
+ return 0;
+ case _LOAD_FAST_6:
+ return 0;
+ case _LOAD_FAST_7:
+ return 0;
+ case _LOAD_FAST:
+ return 0;
+ case _LOAD_FAST_AND_CLEAR:
+ return 0;
+ case _LOAD_FAST_LOAD_FAST:
+ return 0;
+ case _LOAD_CONST:
+ return 0;
+ case _STORE_FAST_0:
+ return 1;
+ case _STORE_FAST_1:
+ return 1;
+ case _STORE_FAST_2:
+ return 1;
+ case _STORE_FAST_3:
+ return 1;
+ case _STORE_FAST_4:
+ return 1;
+ case _STORE_FAST_5:
+ return 1;
+ case _STORE_FAST_6:
+ return 1;
+ case _STORE_FAST_7:
+ return 1;
+ case _STORE_FAST:
+ return 1;
+ case _STORE_FAST_LOAD_FAST:
+ return 1;
+ case _STORE_FAST_STORE_FAST:
+ return 2;
+ case _POP_TOP:
+ return 1;
+ case _PUSH_NULL:
+ return 0;
+ case _END_SEND:
+ return 2;
+ case _UNARY_NEGATIVE:
+ return 1;
+ case _UNARY_NOT:
+ return 1;
+ case _TO_BOOL:
+ return 1;
+ case _TO_BOOL_BOOL:
+ return 1;
+ case _TO_BOOL_INT:
+ return 1;
+ case _TO_BOOL_LIST:
+ return 1;
+ case _TO_BOOL_NONE:
+ return 1;
+ case _TO_BOOL_STR:
+ return 1;
+ case _REPLACE_WITH_TRUE:
+ return 1;
+ case _UNARY_INVERT:
+ return 1;
+ case _GUARD_BOTH_INT:
+ return 2;
+ case _BINARY_OP_MULTIPLY_INT:
+ return 2;
+ case _BINARY_OP_ADD_INT:
+ return 2;
+ case _BINARY_OP_SUBTRACT_INT:
+ return 2;
+ case _GUARD_BOTH_FLOAT:
+ return 2;
+ case _BINARY_OP_MULTIPLY_FLOAT:
+ return 2;
+ case _BINARY_OP_ADD_FLOAT:
+ return 2;
+ case _BINARY_OP_SUBTRACT_FLOAT:
+ return 2;
+ case _GUARD_BOTH_UNICODE:
+ return 2;
+ case _BINARY_OP_ADD_UNICODE:
+ return 2;
+ case _BINARY_SUBSCR:
+ return 2;
+ case _BINARY_SLICE:
+ return 3;
+ case _STORE_SLICE:
+ return 4;
+ case _BINARY_SUBSCR_LIST_INT:
+ return 2;
+ case _BINARY_SUBSCR_STR_INT:
+ return 2;
+ case _BINARY_SUBSCR_TUPLE_INT:
+ return 2;
+ case _BINARY_SUBSCR_DICT:
+ return 2;
+ case _LIST_APPEND:
+ return 2 + (oparg-1);
+ case _SET_ADD:
+ return 2 + (oparg-1);
+ case _STORE_SUBSCR:
+ return 3;
+ case _STORE_SUBSCR_LIST_INT:
+ return 3;
+ case _STORE_SUBSCR_DICT:
+ return 3;
+ case _DELETE_SUBSCR:
+ return 2;
+ case _CALL_INTRINSIC_1:
+ return 1;
+ case _CALL_INTRINSIC_2:
+ return 2;
+ case _POP_FRAME:
+ return 1;
+ case _GET_AITER:
+ return 1;
+ case _GET_ANEXT:
+ return 1;
+ case _GET_AWAITABLE:
+ return 1;
+ case _POP_EXCEPT:
+ return 1;
+ case _LOAD_ASSERTION_ERROR:
+ return 0;
+ case _LOAD_BUILD_CLASS:
+ return 0;
+ case _STORE_NAME:
+ return 1;
+ case _DELETE_NAME:
+ return 0;
+ case _UNPACK_SEQUENCE:
+ return 1;
+ case _UNPACK_SEQUENCE_TWO_TUPLE:
+ return 1;
+ case _UNPACK_SEQUENCE_TUPLE:
+ return 1;
+ case _UNPACK_SEQUENCE_LIST:
+ return 1;
+ case _UNPACK_EX:
+ return 1;
+ case _STORE_ATTR:
+ return 2;
+ case _DELETE_ATTR:
+ return 1;
+ case _STORE_GLOBAL:
+ return 1;
+ case _DELETE_GLOBAL:
+ return 0;
+ case _LOAD_LOCALS:
+ return 0;
+ case _LOAD_FROM_DICT_OR_GLOBALS:
+ return 1;
+ case _LOAD_GLOBAL:
+ return 0;
+ case _GUARD_GLOBALS_VERSION:
+ return 0;
+ case _GUARD_BUILTINS_VERSION:
+ return 0;
+ case _LOAD_GLOBAL_MODULE:
+ return 0;
+ case _LOAD_GLOBAL_BUILTINS:
+ return 0;
+ case _DELETE_FAST:
+ return 0;
+ case _MAKE_CELL:
+ return 0;
+ case _DELETE_DEREF:
+ return 0;
+ case _LOAD_FROM_DICT_OR_DEREF:
+ return 1;
+ case _LOAD_DEREF:
+ return 0;
+ case _STORE_DEREF:
+ return 1;
+ case _COPY_FREE_VARS:
+ return 0;
+ case _BUILD_STRING:
+ return oparg;
+ case _BUILD_TUPLE:
+ return oparg;
+ case _BUILD_LIST:
+ return oparg;
+ case _LIST_EXTEND:
+ return 2 + (oparg-1);
+ case _SET_UPDATE:
+ return 2 + (oparg-1);
+ case _BUILD_MAP:
+ return oparg*2;
+ case _SETUP_ANNOTATIONS:
+ return 0;
+ case _BUILD_CONST_KEY_MAP:
+ return 1 + oparg;
+ case _DICT_UPDATE:
+ return 2 + (oparg - 1);
+ case _DICT_MERGE:
+ return 5 + (oparg - 1);
+ case _MAP_ADD:
+ return 3 + (oparg - 1);
+ case _LOAD_SUPER_ATTR_ATTR:
+ return 3;
+ case _LOAD_SUPER_ATTR_METHOD:
+ return 3;
+ case _LOAD_ATTR:
+ return 1;
+ case _GUARD_TYPE_VERSION:
+ return 1;
+ case _CHECK_MANAGED_OBJECT_HAS_VALUES:
+ return 1;
+ case _LOAD_ATTR_INSTANCE_VALUE_0:
+ return 1;
+ case _LOAD_ATTR_INSTANCE_VALUE_1:
+ return 1;
+ case _LOAD_ATTR_INSTANCE_VALUE:
+ return 1;
+ case _CHECK_ATTR_MODULE:
+ return 1;
+ case _LOAD_ATTR_MODULE:
+ return 1;
+ case _CHECK_ATTR_WITH_HINT:
+ return 1;
+ case _LOAD_ATTR_WITH_HINT:
+ return 1;
+ case _LOAD_ATTR_SLOT_0:
+ return 1;
+ case _LOAD_ATTR_SLOT_1:
+ return 1;
+ case _LOAD_ATTR_SLOT:
+ return 1;
+ case _CHECK_ATTR_CLASS:
+ return 1;
+ case _LOAD_ATTR_CLASS_0:
+ return 1;
+ case _LOAD_ATTR_CLASS_1:
+ return 1;
+ case _LOAD_ATTR_CLASS:
+ return 1;
+ case _GUARD_DORV_VALUES:
+ return 1;
+ case _STORE_ATTR_INSTANCE_VALUE:
+ return 2;
+ case _STORE_ATTR_SLOT:
+ return 2;
+ case _COMPARE_OP:
+ return 2;
+ case _COMPARE_OP_FLOAT:
+ return 2;
+ case _COMPARE_OP_INT:
+ return 2;
+ case _COMPARE_OP_STR:
+ return 2;
+ case _IS_OP:
+ return 2;
+ case _CONTAINS_OP:
+ return 2;
+ case _CONTAINS_OP_SET:
+ return 2;
+ case _CONTAINS_OP_DICT:
+ return 2;
+ case _CHECK_EG_MATCH:
+ return 2;
+ case _CHECK_EXC_MATCH:
+ return 2;
+ case _IS_NONE:
+ return 1;
+ case _GET_LEN:
+ return 1;
+ case _MATCH_CLASS:
+ return 3;
+ case _MATCH_MAPPING:
+ return 1;
+ case _MATCH_SEQUENCE:
+ return 1;
+ case _MATCH_KEYS:
+ return 2;
+ case _GET_ITER:
+ return 1;
+ case _GET_YIELD_FROM_ITER:
+ return 1;
+ case _FOR_ITER_TIER_TWO:
+ return 1;
+ case _ITER_CHECK_LIST:
+ return 1;
+ case _GUARD_NOT_EXHAUSTED_LIST:
+ return 1;
+ case _ITER_NEXT_LIST:
+ return 1;
+ case _ITER_CHECK_TUPLE:
+ return 1;
+ case _GUARD_NOT_EXHAUSTED_TUPLE:
+ return 1;
+ case _ITER_NEXT_TUPLE:
+ return 1;
+ case _ITER_CHECK_RANGE:
+ return 1;
+ case _GUARD_NOT_EXHAUSTED_RANGE:
+ return 1;
+ case _ITER_NEXT_RANGE:
+ return 1;
+ case _WITH_EXCEPT_START:
+ return 4;
+ case _PUSH_EXC_INFO:
+ return 1;
+ case _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT:
+ return 1;
+ case _GUARD_KEYS_VERSION:
+ return 1;
+ case _LOAD_ATTR_METHOD_WITH_VALUES:
+ return 1;
+ case _LOAD_ATTR_METHOD_NO_DICT:
+ return 1;
+ case _LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES:
+ return 1;
+ case _LOAD_ATTR_NONDESCRIPTOR_NO_DICT:
+ return 1;
+ case _CHECK_ATTR_METHOD_LAZY_DICT:
+ return 1;
+ case _LOAD_ATTR_METHOD_LAZY_DICT:
+ return 1;
+ case _CHECK_PERIODIC:
+ return 0;
+ case _CHECK_CALL_BOUND_METHOD_EXACT_ARGS:
+ return 2 + oparg;
+ case _INIT_CALL_BOUND_METHOD_EXACT_ARGS:
+ return 2 + oparg;
+ case _CHECK_PEP_523:
+ return 0;
+ case _CHECK_FUNCTION_EXACT_ARGS:
+ return 2 + oparg;
+ case _CHECK_STACK_SPACE:
+ return 2 + oparg;
+ case _INIT_CALL_PY_EXACT_ARGS_0:
+ return 2 + oparg;
+ case _INIT_CALL_PY_EXACT_ARGS_1:
+ return 2 + oparg;
+ case _INIT_CALL_PY_EXACT_ARGS_2:
+ return 2 + oparg;
+ case _INIT_CALL_PY_EXACT_ARGS_3:
+ return 2 + oparg;
+ case _INIT_CALL_PY_EXACT_ARGS_4:
+ return 2 + oparg;
+ case _INIT_CALL_PY_EXACT_ARGS:
+ return 2 + oparg;
+ case _PUSH_FRAME:
+ return 1;
+ case _CALL_TYPE_1:
+ return 3;
+ case _CALL_STR_1:
+ return 3;
+ case _CALL_TUPLE_1:
+ return 3;
+ case _EXIT_INIT_CHECK:
+ return 1;
+ case _CALL_BUILTIN_CLASS:
+ return 2 + oparg;
+ case _CALL_BUILTIN_O:
+ return 2 + oparg;
+ case _CALL_BUILTIN_FAST:
+ return 2 + oparg;
+ case _CALL_BUILTIN_FAST_WITH_KEYWORDS:
+ return 2 + oparg;
+ case _CALL_LEN:
+ return 2 + oparg;
+ case _CALL_ISINSTANCE:
+ return 2 + oparg;
+ case _CALL_METHOD_DESCRIPTOR_O:
+ return 2 + oparg;
+ case _CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS:
+ return 2 + oparg;
+ case _CALL_METHOD_DESCRIPTOR_NOARGS:
+ return 2 + oparg;
+ case _CALL_METHOD_DESCRIPTOR_FAST:
+ return 2 + oparg;
+ case _MAKE_FUNCTION:
+ return 1;
+ case _SET_FUNCTION_ATTRIBUTE:
+ return 2;
+ case _BUILD_SLICE:
+ return 2 + ((oparg == 3) ? 1 : 0);
+ case _CONVERT_VALUE:
+ return 1;
+ case _FORMAT_SIMPLE:
+ return 1;
+ case _FORMAT_WITH_SPEC:
+ return 2;
+ case _COPY:
+ return 1 + (oparg-1);
+ case _BINARY_OP:
+ return 2;
+ case _SWAP:
+ return 2 + (oparg-2);
+ case _GUARD_IS_TRUE_POP:
+ return 1;
+ case _GUARD_IS_FALSE_POP:
+ return 1;
+ case _GUARD_IS_NONE_POP:
+ return 1;
+ case _GUARD_IS_NOT_NONE_POP:
+ return 1;
+ case _JUMP_TO_TOP:
+ return 0;
+ case _SET_IP:
+ return 0;
+ case _SAVE_RETURN_OFFSET:
+ return 0;
+ case _EXIT_TRACE:
+ return 0;
+ case _CHECK_VALIDITY:
+ return 0;
+ case _LOAD_CONST_INLINE:
+ return 0;
+ case _LOAD_CONST_INLINE_BORROW:
+ return 0;
+ case _POP_TOP_LOAD_CONST_INLINE_BORROW:
+ return 1;
+ case _LOAD_CONST_INLINE_WITH_NULL:
+ return 0;
+ case _LOAD_CONST_INLINE_BORROW_WITH_NULL:
+ return 0;
+ case _CHECK_FUNCTION:
+ return 0;
+ case _INTERNAL_INCREMENT_OPT_COUNTER:
+ return 1;
+ case _COLD_EXIT:
+ return 0;
+ case _START_EXECUTOR:
+ return 0;
+ case _FATAL_ERROR:
+ return 0;
+ case _CHECK_VALIDITY_AND_SET_IP:
+ return 0;
+ case _DEOPT:
+ return 0;
+ case _SIDE_EXIT:
+ return 0;
+ case _ERROR_POP_N:
+ return oparg;
+ default:
+ return -1;
+ }
+}
+
#endif // NEED_OPCODE_METADATA
uintptr_t code_version = _PyFrame_GetCode(frame)->_co_instrumentation_version;
if (code_version != global_version) {
if (_Py_Instrument(_PyFrame_GetCode(frame), tstate->interp)) {
- GOTO_ERROR(error);
+ ERROR_NO_POP();
}
next_instr = this_instr;
}
inst(LOAD_FAST_CHECK, (-- value)) {
value = GETLOCAL(oparg);
- ERROR_IF(value == NULL, unbound_local_error);
+ if (value == NULL) {
+ _PyEval_FormatExcCheckArg(tstate, PyExc_UnboundLocalError,
+ UNBOUNDLOCAL_ERROR_MSG,
+ PyTuple_GetItem(_PyFrame_GetCode(frame)->co_localsplusnames, oparg)
+ );
+ ERROR_IF(1, error);
+ }
Py_INCREF(value);
}
if (PyGen_Check(receiver)) {
PyErr_SetObject(PyExc_StopIteration, value);
if (monitor_stop_iteration(tstate, frame, this_instr)) {
- GOTO_ERROR(error);
+ ERROR_NO_POP();
}
PyErr_SetRaisedException(NULL);
}
if (PyGen_Check(receiver) || PyCoro_CheckExact(receiver)) {
PyErr_SetObject(PyExc_StopIteration, value);
if (monitor_stop_iteration(tstate, frame, this_instr)) {
- GOTO_ERROR(error);
+ ERROR_NO_POP();
}
PyErr_SetRaisedException(NULL);
}
int err = _Py_call_instrumentation_arg(
tstate, PY_MONITORING_EVENT_PY_RETURN,
frame, this_instr, retval);
- if (err) GOTO_ERROR(error);
+ if (err) ERROR_NO_POP();
STACK_SHRINK(1);
assert(EMPTY());
_PyFrame_SetStackPointer(frame, stack_pointer);
int err = _Py_call_instrumentation_arg(
tstate, PY_MONITORING_EVENT_PY_RETURN,
frame, this_instr, retval);
- if (err) GOTO_ERROR(error);
+ if (err) ERROR_NO_POP();
Py_INCREF(retval);
assert(EMPTY());
_PyFrame_SetStackPointer(frame, stack_pointer);
if (PyAsyncGen_CheckExact(aiter)) {
awaitable = type->tp_as_async->am_anext(aiter);
if (awaitable == NULL) {
- GOTO_ERROR(error);
+ ERROR_NO_POP();
}
} else {
if (type->tp_as_async != NULL){
if (getter != NULL) {
next_iter = (*getter)(aiter);
if (next_iter == NULL) {
- GOTO_ERROR(error);
+ ERROR_NO_POP();
}
}
else {
"'async for' requires an iterator with "
"__anext__ method, got %.100s",
type->tp_name);
- GOTO_ERROR(error);
+ ERROR_NO_POP();
}
awaitable = _PyCoro_GetAwaitableIter(next_iter);
Py_TYPE(next_iter)->tp_name);
Py_DECREF(next_iter);
- GOTO_ERROR(error);
+ ERROR_NO_POP();
} else {
Py_DECREF(next_iter);
}
JUMPBY(oparg);
}
else {
- GOTO_ERROR(error);
+ ERROR_NO_POP();
}
}
Py_DECREF(v);
int err = _Py_call_instrumentation_arg(
tstate, PY_MONITORING_EVENT_PY_YIELD,
frame, this_instr, retval);
- if (err) GOTO_ERROR(error);
+ if (err) ERROR_NO_POP();
tstate->exc_info = gen->gi_exc_state.previous_item;
gen->gi_exc_state.previous_item = NULL;
_Py_LeaveRecursiveCallPy(tstate);
else {
assert(PyLong_Check(lasti));
_PyErr_SetString(tstate, PyExc_SystemError, "lasti is not an int");
- GOTO_ERROR(error);
+ ERROR_NO_POP();
}
}
assert(exc && PyExceptionInstance_Check(exc));
if (ns == NULL) {
_PyErr_Format(tstate, PyExc_SystemError,
"no locals when deleting %R", name);
- GOTO_ERROR(error);
+ ERROR_NO_POP();
}
err = PyObject_DelItem(ns, name);
// Can't use ERROR_IF here.
_PyEval_FormatExcCheckArg(tstate, PyExc_NameError,
NAME_ERROR_MSG,
name);
- GOTO_ERROR(error);
+ ERROR_NO_POP();
}
}
int err = PyDict_Pop(GLOBALS(), name, NULL);
// Can't use ERROR_IF here.
if (err < 0) {
- GOTO_ERROR(error);
+ ERROR_NO_POP();
}
if (err == 0) {
_PyEval_FormatExcCheckArg(tstate, PyExc_NameError,
NAME_ERROR_MSG, name);
- GOTO_ERROR(error);
+ ERROR_NO_POP();
}
}
inst(LOAD_FROM_DICT_OR_GLOBALS, (mod_or_class_dict -- v)) {
PyObject *name = GETITEM(FRAME_CO_NAMES, oparg);
if (PyMapping_GetOptionalItem(mod_or_class_dict, name, &v) < 0) {
- GOTO_ERROR(error);
+ ERROR_NO_POP();
}
if (v == NULL) {
if (PyDict_GetItemRef(GLOBALS(), name, &v) < 0) {
- GOTO_ERROR(error);
+ ERROR_NO_POP();
}
if (v == NULL) {
if (PyMapping_GetOptionalItem(BUILTINS(), name, &v) < 0) {
- GOTO_ERROR(error);
+ ERROR_NO_POP();
}
if (v == NULL) {
_PyEval_FormatExcCheckArg(
tstate, PyExc_NameError,
NAME_ERROR_MSG, name);
- GOTO_ERROR(error);
+ ERROR_NO_POP();
}
}
}
}
PyObject *name = GETITEM(FRAME_CO_NAMES, oparg);
if (PyMapping_GetOptionalItem(mod_or_class_dict, name, &v) < 0) {
- GOTO_ERROR(error);
+ ERROR_NO_POP();
}
if (v == NULL) {
if (PyDict_GetItemRef(GLOBALS(), name, &v) < 0) {
- GOTO_ERROR(error);
+ ERROR_NO_POP();
}
if (v == NULL) {
if (PyMapping_GetOptionalItem(BUILTINS(), name, &v) < 0) {
- GOTO_ERROR(error);
+ ERROR_NO_POP();
}
if (v == NULL) {
_PyEval_FormatExcCheckArg(
tstate, PyExc_NameError,
NAME_ERROR_MSG, name);
- GOTO_ERROR(error);
+ ERROR_NO_POP();
}
}
}
inst(DELETE_FAST, (--)) {
PyObject *v = GETLOCAL(oparg);
- ERROR_IF(v == NULL, unbound_local_error);
+ if (v == NULL) {
+ _PyEval_FormatExcCheckArg(tstate, PyExc_UnboundLocalError,
+ UNBOUNDLOCAL_ERROR_MSG,
+ PyTuple_GetItem(_PyFrame_GetCode(frame)->co_localsplusnames, oparg)
+ );
+ ERROR_IF(1, error);
+ }
SETLOCAL(oparg, NULL);
}
PyObject *initial = GETLOCAL(oparg);
PyObject *cell = PyCell_New(initial);
if (cell == NULL) {
- GOTO_ERROR(error);
+ ERROR_NO_POP();
}
SETLOCAL(oparg, cell);
}
// Fortunately we don't need its superpower.
if (oldobj == NULL) {
_PyEval_FormatExcUnbound(tstate, _PyFrame_GetCode(frame), oparg);
- GOTO_ERROR(error);
+ ERROR_NO_POP();
}
PyCell_SET(cell, NULL);
Py_DECREF(oldobj);
assert(oparg >= 0 && oparg < _PyFrame_GetCode(frame)->co_nlocalsplus);
name = PyTuple_GET_ITEM(_PyFrame_GetCode(frame)->co_localsplusnames, oparg);
if (PyMapping_GetOptionalItem(class_dict, name, &value) < 0) {
- GOTO_ERROR(error);
+ ERROR_NO_POP();
}
if (!value) {
PyObject *cell = GETLOCAL(oparg);
value = PyCell_GET(cell);
if (value == NULL) {
_PyEval_FormatExcUnbound(tstate, _PyFrame_GetCode(frame), oparg);
- GOTO_ERROR(error);
+ ERROR_NO_POP();
}
Py_INCREF(value);
}
inst(BUILD_SET, (values[oparg] -- set)) {
set = PySet_New(NULL);
if (set == NULL)
- GOTO_ERROR(error);
+ ERROR_NO_POP();
int err = 0;
for (int i = 0; i < oparg; i++) {
PyObject *item = values[i];
}
inst(BUILD_CONST_KEY_MAP, (values[oparg], keys -- map)) {
- if (!PyTuple_CheckExact(keys) ||
- PyTuple_GET_SIZE(keys) != (Py_ssize_t)oparg) {
- _PyErr_SetString(tstate, PyExc_SystemError,
- "bad BUILD_CONST_KEY_MAP keys argument");
- GOTO_ERROR(error); // Pop the keys and values.
- }
+ assert(PyTuple_CheckExact(keys));
+ assert(PyTuple_GET_SIZE(keys) == (Py_ssize_t)oparg);
map = _PyDict_FromItems(
&PyTuple_GET_ITEM(keys, 0), 1,
values, 1, oparg);
_PyErr_SetString(tstate, PyExc_TypeError,
"cannot 'yield from' a coroutine object "
"in a non-coroutine generator");
- GOTO_ERROR(error);
+ ERROR_NO_POP();
}
iter = iterable;
}
/* `iterable` is not a generator. */
iter = PyObject_GetIter(iterable);
if (iter == NULL) {
- GOTO_ERROR(error);
+ ERROR_NO_POP();
}
DECREF_INPUTS();
}
if (next == NULL) {
if (_PyErr_Occurred(tstate)) {
if (!_PyErr_ExceptionMatches(tstate, PyExc_StopIteration)) {
- GOTO_ERROR(error);
+ ERROR_NO_POP();
}
monitor_raise(tstate, frame, this_instr);
_PyErr_Clear(tstate);
if (next == NULL) {
if (_PyErr_Occurred(tstate)) {
if (!_PyErr_ExceptionMatches(tstate, PyExc_StopIteration)) {
- GOTO_ERROR(error);
+ ERROR_NO_POP();
}
_PyErr_Clear(tstate);
}
else {
if (_PyErr_Occurred(tstate)) {
if (!_PyErr_ExceptionMatches(tstate, PyExc_StopIteration)) {
- GOTO_ERROR(error);
+ ERROR_NO_POP();
}
monitor_raise(tstate, frame, this_instr);
_PyErr_Clear(tstate);
"asynchronous context manager protocol",
Py_TYPE(mgr)->tp_name);
}
- GOTO_ERROR(error);
+ ERROR_NO_POP();
}
exit = _PyObject_LookupSpecial(mgr, &_Py_ID(__aexit__));
if (exit == NULL) {
Py_TYPE(mgr)->tp_name);
}
Py_DECREF(enter);
- GOTO_ERROR(error);
+ ERROR_NO_POP();
}
DECREF_INPUTS();
res = PyObject_CallNoArgs(enter);
"context manager protocol",
Py_TYPE(mgr)->tp_name);
}
- GOTO_ERROR(error);
+ ERROR_NO_POP();
}
exit = _PyObject_LookupSpecial(mgr, &_Py_ID(__exit__));
if (exit == NULL) {
Py_TYPE(mgr)->tp_name);
}
Py_DECREF(enter);
- GOTO_ERROR(error);
+ ERROR_NO_POP();
}
DECREF_INPUTS();
res = PyObject_CallNoArgs(enter);
// The frame has stolen all the arguments from the stack,
// so there is no need to clean them up.
if (new_frame == NULL) {
- GOTO_ERROR(error);
+ ERROR_NO_POP();
}
frame->return_offset = (uint16_t)(next_instr - this_instr);
DISPATCH_INLINED(new_frame);
STAT_INC(CALL, hit);
PyObject *self = _PyType_NewManagedObject(tp);
if (self == NULL) {
- GOTO_ERROR(error);
+ ERROR_NO_POP();
}
Py_DECREF(tp);
_PyInterpreterFrame *shim = _PyFrame_PushTrampolineUnchecked(
PyErr_Format(PyExc_TypeError,
"__init__() should return None, not '%.200s'",
Py_TYPE(should_be_none)->tp_name);
- GOTO_ERROR(error);
+ ERROR_NO_POP();
}
}
PyObject *arg = args[0];
Py_ssize_t len_i = PyObject_Length(arg);
if (len_i < 0) {
- GOTO_ERROR(error);
+ ERROR_NO_POP();
}
res = PyLong_FromSsize_t(len_i);
assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL));
PyObject *inst = args[0];
int retval = PyObject_IsInstance(inst, cls);
if (retval < 0) {
- GOTO_ERROR(error);
+ ERROR_NO_POP();
}
res = PyBool_FromLong(retval);
assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL));
// The frame has stolen all the arguments from the stack,
// so there is no need to clean them up.
if (new_frame == NULL) {
- GOTO_ERROR(error);
+ ERROR_NO_POP();
}
assert(next_instr - this_instr == 1);
frame->return_offset = 1;
assert(kwargs == NULL || PyDict_CheckExact(kwargs));
if (!PyTuple_CheckExact(callargs)) {
if (check_args_iterable(tstate, func, callargs) < 0) {
- GOTO_ERROR(error);
+ ERROR_NO_POP();
}
PyObject *tuple = PySequence_Tuple(callargs);
if (tuple == NULL) {
- GOTO_ERROR(error);
+ ERROR_NO_POP();
}
Py_SETREF(callargs, tuple);
}
int err = _Py_call_instrumentation_2args(
tstate, PY_MONITORING_EVENT_CALL,
frame, this_instr, func, arg);
- if (err) GOTO_ERROR(error);
+ if (err) ERROR_NO_POP();
result = PyObject_Call(func, callargs, kwargs);
if (!PyFunction_Check(func) && !PyMethod_Check(func)) {
// Need to manually shrink the stack since we exit with DISPATCH_INLINED.
STACK_SHRINK(oparg + 3);
if (new_frame == NULL) {
- GOTO_ERROR(error);
+ ERROR_NO_POP();
}
assert(next_instr - this_instr == 1);
frame->return_offset = 1;
Py_DECREF(codeobj);
if (func_obj == NULL) {
- GOTO_ERROR(error);
+ ERROR_NO_POP();
}
_PyFunction_SetVersion(
PyFunctionObject *func = (PyFunctionObject *)frame->f_funcobj;
PyGenObject *gen = (PyGenObject *)_Py_MakeCoro(func);
if (gen == NULL) {
- GOTO_ERROR(error);
+ ERROR_NO_POP();
}
assert(EMPTY());
_PyFrame_SetStackPointer(frame, stack_pointer);
if (optimized < 0) {
Py_DECREF(previous);
tstate->previous_executor = Py_None;
- ERROR_IF(1, error);
+ GOTO_UNWIND();
}
GOTO_TIER_ONE(target);
}
frame->instr_ptr = (_Py_CODEUNIT *)instr_ptr;
}
+ tier2 op(_DEOPT, (--)) {
+ EXIT_TO_TIER1();
+ }
+
+ tier2 op(_SIDE_EXIT, (--)) {
+ EXIT_TO_TRACE();
+ }
+
+ tier2 op(_ERROR_POP_N, (unused[oparg] --)) {
+ SYNC_SP();
+ GOTO_UNWIND();
+ }
+
// END BYTECODES //
}
return 0;
}
-
static const _Py_CODEUNIT _Py_INTERPRETER_TRAMPOLINE_INSTRUCTIONS[] = {
/* Put a NOP at the start, so that the IP points into
* the code, rather than before it */
or goto error. */
Py_UNREACHABLE();
-unbound_local_error:
- {
- _PyEval_FormatExcCheckArg(tstate, PyExc_UnboundLocalError,
- UNBOUNDLOCAL_ERROR_MSG,
- PyTuple_GetItem(_PyFrame_GetCode(frame)->co_localsplusnames, oparg)
- );
- goto error;
- }
-
pop_4_error:
STACK_SHRINK(1);
pop_3_error:
#undef GOTO_ERROR
#define GOTO_ERROR(LABEL) goto LABEL ## _tier_two
-#undef DEOPT_IF
-#define DEOPT_IF(COND, INSTNAME) \
- if ((COND)) { \
- goto deoptimize;\
- }
-
#ifdef Py_STATS
// Disable these macros that apply to Tier 1 stats when we are in Tier 2
#undef STAT_INC
#endif
assert(next_uop->opcode == _START_EXECUTOR || next_uop->opcode == _COLD_EXIT);
+tier2_dispatch:
for (;;) {
uopcode = next_uop->opcode;
#ifdef Py_DEBUG
}
}
-// Jump here from ERROR_IF(..., unbound_local_error)
-unbound_local_error_tier_two:
- _PyEval_FormatExcCheckArg(tstate, PyExc_UnboundLocalError,
- UNBOUNDLOCAL_ERROR_MSG,
- PyTuple_GetItem(_PyFrame_GetCode(frame)->co_localsplusnames, oparg)
- );
- goto error_tier_two;
-
-// JUMP to any of these from ERROR_IF(..., error)
-pop_4_error_tier_two:
- STACK_SHRINK(1);
-pop_3_error_tier_two:
- STACK_SHRINK(1);
-pop_2_error_tier_two:
- STACK_SHRINK(1);
-pop_1_error_tier_two:
- STACK_SHRINK(1);
-error_tier_two:
+jump_to_error_target:
#ifdef Py_DEBUG
if (lltrace >= 2) {
printf("Error: [UOp ");
_PyOpcode_OpName[frame->instr_ptr->op.code]);
}
#endif
+ assert (next_uop[-1].format == UOP_FORMAT_JUMP);
+ uint16_t target = uop_get_error_target(&next_uop[-1]);
+ next_uop = current_executor->trace + target;
+ goto tier2_dispatch;
+
+error_tier_two:
OPT_HIST(trace_uop_execution_counter, trace_run_length_hist);
+ assert(next_uop[-1].format == UOP_FORMAT_TARGET);
frame->return_offset = 0; // Don't leave this random
_PyFrame_SetStackPointer(frame, stack_pointer);
Py_DECREF(current_executor);
tstate->previous_executor = NULL;
goto resume_with_error;
-// Jump here from DEOPT_IF()
-deoptimize:
+jump_to_jump_target:
+ assert(next_uop[-1].format == UOP_FORMAT_JUMP);
+ target = uop_get_jump_target(&next_uop[-1]);
+ next_uop = current_executor->trace + target;
+ goto tier2_dispatch;
+
+exit_to_tier1:
+ assert(next_uop[-1].format == UOP_FORMAT_TARGET);
next_instr = next_uop[-1].target + _PyCode_CODE(_PyFrame_GetCode(frame));
#ifdef Py_DEBUG
if (lltrace >= 2) {
tstate->previous_executor = NULL;
DISPATCH();
-// Jump here from EXIT_IF()
-side_exit:
+exit_to_trace:
+ assert(next_uop[-1].format == UOP_FORMAT_EXIT);
OPT_HIST(trace_uop_execution_counter, trace_run_length_hist);
UOP_STAT_INC(uopcode, miss);
uint32_t exit_index = next_uop[-1].exit_index;
#define CURRENT_OPARG() (next_uop[-1].oparg)
#define CURRENT_OPERAND() (next_uop[-1].operand)
+
+#define JUMP_TO_JUMP_TARGET() goto jump_to_jump_target
+#define JUMP_TO_ERROR() goto jump_to_error_target
+#define GOTO_UNWIND() goto error_tier_two
+#define EXIT_TO_TRACE() goto exit_to_trace
+#define EXIT_TO_TIER1() goto exit_to_tier1
case _RESUME_CHECK: {
#if defined(__EMSCRIPTEN__)
- if (_Py_emscripten_signal_clock == 0) goto deoptimize;
+ if (_Py_emscripten_signal_clock == 0) JUMP_TO_JUMP_TARGET();
_Py_emscripten_signal_clock -= Py_EMSCRIPTEN_SIGNAL_HANDLING;
#endif
uintptr_t eval_breaker = _Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker);
uintptr_t version = _PyFrame_GetCode(frame)->_co_instrumentation_version;
assert((version & _PY_EVAL_EVENTS_MASK) == 0);
- if (eval_breaker != version) goto deoptimize;
+ if (eval_breaker != version) JUMP_TO_JUMP_TARGET();
break;
}
- /* _INSTRUMENTED_RESUME is not a viable micro-op for tier 2 */
+ /* _INSTRUMENTED_RESUME is not a viable micro-op for tier 2 because it is instrumented */
case _LOAD_FAST_CHECK: {
PyObject *value;
oparg = CURRENT_OPARG();
value = GETLOCAL(oparg);
- if (value == NULL) goto unbound_local_error_tier_two;
+ if (value == NULL) {
+ _PyEval_FormatExcCheckArg(tstate, PyExc_UnboundLocalError,
+ UNBOUNDLOCAL_ERROR_MSG,
+ PyTuple_GetItem(_PyFrame_GetCode(frame)->co_localsplusnames, oparg)
+ );
+ if (1) JUMP_TO_ERROR();
+ }
Py_INCREF(value);
stack_pointer[0] = value;
stack_pointer += 1;
value = stack_pointer[-1];
res = PyNumber_Negative(value);
Py_DECREF(value);
- if (res == NULL) goto pop_1_error_tier_two;
+ if (res == NULL) JUMP_TO_ERROR();
stack_pointer[-1] = res;
break;
}
value = stack_pointer[-1];
int err = PyObject_IsTrue(value);
Py_DECREF(value);
- if (err < 0) goto pop_1_error_tier_two;
+ if (err < 0) JUMP_TO_ERROR();
res = err ? Py_True : Py_False;
stack_pointer[-1] = res;
break;
case _TO_BOOL_BOOL: {
PyObject *value;
value = stack_pointer[-1];
- if (!PyBool_Check(value)) goto side_exit;
+ if (!PyBool_Check(value)) JUMP_TO_JUMP_TARGET();
STAT_INC(TO_BOOL, hit);
break;
}
PyObject *value;
PyObject *res;
value = stack_pointer[-1];
- if (!PyLong_CheckExact(value)) goto side_exit;
+ if (!PyLong_CheckExact(value)) JUMP_TO_JUMP_TARGET();
STAT_INC(TO_BOOL, hit);
if (_PyLong_IsZero((PyLongObject *)value)) {
assert(_Py_IsImmortal(value));
PyObject *value;
PyObject *res;
value = stack_pointer[-1];
- if (!PyList_CheckExact(value)) goto side_exit;
+ if (!PyList_CheckExact(value)) JUMP_TO_JUMP_TARGET();
STAT_INC(TO_BOOL, hit);
res = Py_SIZE(value) ? Py_True : Py_False;
Py_DECREF(value);
PyObject *res;
value = stack_pointer[-1];
// This one is a bit weird, because we expect *some* failures:
- if (!Py_IsNone(value)) goto side_exit;
+ if (!Py_IsNone(value)) JUMP_TO_JUMP_TARGET();
STAT_INC(TO_BOOL, hit);
res = Py_False;
stack_pointer[-1] = res;
PyObject *value;
PyObject *res;
value = stack_pointer[-1];
- if (!PyUnicode_CheckExact(value)) goto side_exit;
+ if (!PyUnicode_CheckExact(value)) JUMP_TO_JUMP_TARGET();
STAT_INC(TO_BOOL, hit);
if (value == &_Py_STR(empty)) {
assert(_Py_IsImmortal(value));
value = stack_pointer[-1];
res = PyNumber_Invert(value);
Py_DECREF(value);
- if (res == NULL) goto pop_1_error_tier_two;
+ if (res == NULL) JUMP_TO_ERROR();
stack_pointer[-1] = res;
break;
}
PyObject *left;
right = stack_pointer[-1];
left = stack_pointer[-2];
- if (!PyLong_CheckExact(left)) goto side_exit;
- if (!PyLong_CheckExact(right)) goto side_exit;
+ if (!PyLong_CheckExact(left)) JUMP_TO_JUMP_TARGET();
+ if (!PyLong_CheckExact(right)) JUMP_TO_JUMP_TARGET();
break;
}
res = _PyLong_Multiply((PyLongObject *)left, (PyLongObject *)right);
_Py_DECREF_SPECIALIZED(right, (destructor)PyObject_Free);
_Py_DECREF_SPECIALIZED(left, (destructor)PyObject_Free);
- if (res == NULL) goto pop_2_error_tier_two;
+ if (res == NULL) JUMP_TO_ERROR();
stack_pointer[-2] = res;
stack_pointer += -1;
break;
res = _PyLong_Add((PyLongObject *)left, (PyLongObject *)right);
_Py_DECREF_SPECIALIZED(right, (destructor)PyObject_Free);
_Py_DECREF_SPECIALIZED(left, (destructor)PyObject_Free);
- if (res == NULL) goto pop_2_error_tier_two;
+ if (res == NULL) JUMP_TO_ERROR();
stack_pointer[-2] = res;
stack_pointer += -1;
break;
res = _PyLong_Subtract((PyLongObject *)left, (PyLongObject *)right);
_Py_DECREF_SPECIALIZED(right, (destructor)PyObject_Free);
_Py_DECREF_SPECIALIZED(left, (destructor)PyObject_Free);
- if (res == NULL) goto pop_2_error_tier_two;
+ if (res == NULL) JUMP_TO_ERROR();
stack_pointer[-2] = res;
stack_pointer += -1;
break;
PyObject *left;
right = stack_pointer[-1];
left = stack_pointer[-2];
- if (!PyFloat_CheckExact(left)) goto side_exit;
- if (!PyFloat_CheckExact(right)) goto side_exit;
+ if (!PyFloat_CheckExact(left)) JUMP_TO_JUMP_TARGET();
+ if (!PyFloat_CheckExact(right)) JUMP_TO_JUMP_TARGET();
break;
}
PyObject *left;
right = stack_pointer[-1];
left = stack_pointer[-2];
- if (!PyUnicode_CheckExact(left)) goto side_exit;
- if (!PyUnicode_CheckExact(right)) goto side_exit;
+ if (!PyUnicode_CheckExact(left)) JUMP_TO_JUMP_TARGET();
+ if (!PyUnicode_CheckExact(right)) JUMP_TO_JUMP_TARGET();
break;
}
res = PyUnicode_Concat(left, right);
_Py_DECREF_SPECIALIZED(left, _PyUnicode_ExactDealloc);
_Py_DECREF_SPECIALIZED(right, _PyUnicode_ExactDealloc);
- if (res == NULL) goto pop_2_error_tier_two;
+ if (res == NULL) JUMP_TO_ERROR();
stack_pointer[-2] = res;
stack_pointer += -1;
break;
res = PyObject_GetItem(container, sub);
Py_DECREF(container);
Py_DECREF(sub);
- if (res == NULL) goto pop_2_error_tier_two;
+ if (res == NULL) JUMP_TO_ERROR();
stack_pointer[-2] = res;
stack_pointer += -1;
break;
Py_DECREF(slice);
}
Py_DECREF(container);
- if (res == NULL) goto pop_3_error_tier_two;
+ if (res == NULL) JUMP_TO_ERROR();
stack_pointer[-3] = res;
stack_pointer += -2;
break;
}
Py_DECREF(v);
Py_DECREF(container);
- if (err) goto pop_4_error_tier_two;
+ if (err) JUMP_TO_ERROR();
stack_pointer += -4;
break;
}
PyObject *res;
sub = stack_pointer[-1];
list = stack_pointer[-2];
- if (!PyLong_CheckExact(sub)) goto deoptimize;
- if (!PyList_CheckExact(list)) goto deoptimize;
+ if (!PyLong_CheckExact(sub)) JUMP_TO_JUMP_TARGET();
+ if (!PyList_CheckExact(list)) JUMP_TO_JUMP_TARGET();
// Deopt unless 0 <= sub < PyList_Size(list)
- if (!_PyLong_IsNonNegativeCompact((PyLongObject *)sub)) goto deoptimize;
+ if (!_PyLong_IsNonNegativeCompact((PyLongObject *)sub)) JUMP_TO_JUMP_TARGET();
Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0];
- if (index >= PyList_GET_SIZE(list)) goto deoptimize;
+ if (index >= PyList_GET_SIZE(list)) JUMP_TO_JUMP_TARGET();
STAT_INC(BINARY_SUBSCR, hit);
res = PyList_GET_ITEM(list, index);
assert(res != NULL);
PyObject *res;
sub = stack_pointer[-1];
str = stack_pointer[-2];
- if (!PyLong_CheckExact(sub)) goto deoptimize;
- if (!PyUnicode_CheckExact(str)) goto deoptimize;
- if (!_PyLong_IsNonNegativeCompact((PyLongObject *)sub)) goto deoptimize;
+ if (!PyLong_CheckExact(sub)) JUMP_TO_JUMP_TARGET();
+ if (!PyUnicode_CheckExact(str)) JUMP_TO_JUMP_TARGET();
+ if (!_PyLong_IsNonNegativeCompact((PyLongObject *)sub)) JUMP_TO_JUMP_TARGET();
Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0];
- if (PyUnicode_GET_LENGTH(str) <= index) goto deoptimize;
+ if (PyUnicode_GET_LENGTH(str) <= index) JUMP_TO_JUMP_TARGET();
// Specialize for reading an ASCII character from any string:
Py_UCS4 c = PyUnicode_READ_CHAR(str, index);
- if (Py_ARRAY_LENGTH(_Py_SINGLETON(strings).ascii) <= c) goto deoptimize;
+ if (Py_ARRAY_LENGTH(_Py_SINGLETON(strings).ascii) <= c) JUMP_TO_JUMP_TARGET();
STAT_INC(BINARY_SUBSCR, hit);
res = (PyObject*)&_Py_SINGLETON(strings).ascii[c];
_Py_DECREF_SPECIALIZED(sub, (destructor)PyObject_Free);
PyObject *res;
sub = stack_pointer[-1];
tuple = stack_pointer[-2];
- if (!PyLong_CheckExact(sub)) goto deoptimize;
- if (!PyTuple_CheckExact(tuple)) goto deoptimize;
+ if (!PyLong_CheckExact(sub)) JUMP_TO_JUMP_TARGET();
+ if (!PyTuple_CheckExact(tuple)) JUMP_TO_JUMP_TARGET();
// Deopt unless 0 <= sub < PyTuple_Size(list)
- if (!_PyLong_IsNonNegativeCompact((PyLongObject *)sub)) goto deoptimize;
+ if (!_PyLong_IsNonNegativeCompact((PyLongObject *)sub)) JUMP_TO_JUMP_TARGET();
Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0];
- if (index >= PyTuple_GET_SIZE(tuple)) goto deoptimize;
+ if (index >= PyTuple_GET_SIZE(tuple)) JUMP_TO_JUMP_TARGET();
STAT_INC(BINARY_SUBSCR, hit);
res = PyTuple_GET_ITEM(tuple, index);
assert(res != NULL);
PyObject *res;
sub = stack_pointer[-1];
dict = stack_pointer[-2];
- if (!PyDict_CheckExact(dict)) goto deoptimize;
+ if (!PyDict_CheckExact(dict)) JUMP_TO_JUMP_TARGET();
STAT_INC(BINARY_SUBSCR, hit);
int rc = PyDict_GetItemRef(dict, sub, &res);
if (rc == 0) {
}
Py_DECREF(dict);
Py_DECREF(sub);
- if (rc <= 0) goto pop_2_error_tier_two;
+ if (rc <= 0) JUMP_TO_ERROR();
// not found or error
stack_pointer[-2] = res;
stack_pointer += -1;
break;
}
- /* _BINARY_SUBSCR_GETITEM is not a viable micro-op for tier 2 */
+ /* _BINARY_SUBSCR_GETITEM is not a viable micro-op for tier 2 because it uses the 'this_instr' variable */
case _LIST_APPEND: {
PyObject *v;
oparg = CURRENT_OPARG();
v = stack_pointer[-1];
list = stack_pointer[-2 - (oparg-1)];
- if (_PyList_AppendTakeRef((PyListObject *)list, v) < 0) goto pop_1_error_tier_two;
+ if (_PyList_AppendTakeRef((PyListObject *)list, v) < 0) JUMP_TO_ERROR();
stack_pointer += -1;
break;
}
set = stack_pointer[-2 - (oparg-1)];
int err = PySet_Add(set, v);
Py_DECREF(v);
- if (err) goto pop_1_error_tier_two;
+ if (err) JUMP_TO_ERROR();
stack_pointer += -1;
break;
}
Py_DECREF(v);
Py_DECREF(container);
Py_DECREF(sub);
- if (err) goto pop_3_error_tier_two;
+ if (err) JUMP_TO_ERROR();
stack_pointer += -3;
break;
}
sub = stack_pointer[-1];
list = stack_pointer[-2];
value = stack_pointer[-3];
- if (!PyLong_CheckExact(sub)) goto deoptimize;
- if (!PyList_CheckExact(list)) goto deoptimize;
+ if (!PyLong_CheckExact(sub)) JUMP_TO_JUMP_TARGET();
+ if (!PyList_CheckExact(list)) JUMP_TO_JUMP_TARGET();
// Ensure nonnegative, zero-or-one-digit ints.
- if (!_PyLong_IsNonNegativeCompact((PyLongObject *)sub)) goto deoptimize;
+ if (!_PyLong_IsNonNegativeCompact((PyLongObject *)sub)) JUMP_TO_JUMP_TARGET();
Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0];
// Ensure index < len(list)
- if (index >= PyList_GET_SIZE(list)) goto deoptimize;
+ if (index >= PyList_GET_SIZE(list)) JUMP_TO_JUMP_TARGET();
STAT_INC(STORE_SUBSCR, hit);
PyObject *old_value = PyList_GET_ITEM(list, index);
PyList_SET_ITEM(list, index, value);
sub = stack_pointer[-1];
dict = stack_pointer[-2];
value = stack_pointer[-3];
- if (!PyDict_CheckExact(dict)) goto deoptimize;
+ if (!PyDict_CheckExact(dict)) JUMP_TO_JUMP_TARGET();
STAT_INC(STORE_SUBSCR, hit);
int err = _PyDict_SetItem_Take2((PyDictObject *)dict, sub, value);
Py_DECREF(dict);
- if (err) goto pop_3_error_tier_two;
+ if (err) JUMP_TO_ERROR();
stack_pointer += -3;
break;
}
int err = PyObject_DelItem(container, sub);
Py_DECREF(container);
Py_DECREF(sub);
- if (err) goto pop_2_error_tier_two;
+ if (err) JUMP_TO_ERROR();
stack_pointer += -2;
break;
}
assert(oparg <= MAX_INTRINSIC_1);
res = _PyIntrinsics_UnaryFunctions[oparg].func(tstate, value);
Py_DECREF(value);
- if (res == NULL) goto pop_1_error_tier_two;
+ if (res == NULL) JUMP_TO_ERROR();
stack_pointer[-1] = res;
break;
}
res = _PyIntrinsics_BinaryFunctions[oparg].func(tstate, value2, value1);
Py_DECREF(value2);
Py_DECREF(value1);
- if (res == NULL) goto pop_2_error_tier_two;
+ if (res == NULL) JUMP_TO_ERROR();
stack_pointer[-2] = res;
stack_pointer += -1;
break;
break;
}
- /* _INSTRUMENTED_RETURN_VALUE is not a viable micro-op for tier 2 */
+ /* _INSTRUMENTED_RETURN_VALUE is not a viable micro-op for tier 2 because it is instrumented */
- /* _INSTRUMENTED_RETURN_CONST is not a viable micro-op for tier 2 */
+ /* _INSTRUMENTED_RETURN_CONST is not a viable micro-op for tier 2 because it is instrumented */
case _GET_AITER: {
PyObject *obj;
"__aiter__ method, got %.100s",
type->tp_name);
Py_DECREF(obj);
- if (true) goto pop_1_error_tier_two;
+ if (true) JUMP_TO_ERROR();
}
iter = (*getter)(obj);
Py_DECREF(obj);
- if (iter == NULL) goto pop_1_error_tier_two;
+ if (iter == NULL) JUMP_TO_ERROR();
if (Py_TYPE(iter)->tp_as_async == NULL ||
Py_TYPE(iter)->tp_as_async->am_anext == NULL) {
_PyErr_Format(tstate, PyExc_TypeError,
"that does not implement __anext__: %.100s",
Py_TYPE(iter)->tp_name);
Py_DECREF(iter);
- if (true) goto pop_1_error_tier_two;
+ if (true) JUMP_TO_ERROR();
}
stack_pointer[-1] = iter;
break;
if (PyAsyncGen_CheckExact(aiter)) {
awaitable = type->tp_as_async->am_anext(aiter);
if (awaitable == NULL) {
- GOTO_ERROR(error);
+ JUMP_TO_ERROR();
}
} else {
if (type->tp_as_async != NULL){
if (getter != NULL) {
next_iter = (*getter)(aiter);
if (next_iter == NULL) {
- GOTO_ERROR(error);
+ JUMP_TO_ERROR();
}
}
else {
"'async for' requires an iterator with "
"__anext__ method, got %.100s",
type->tp_name);
- GOTO_ERROR(error);
+ JUMP_TO_ERROR();
}
awaitable = _PyCoro_GetAwaitableIter(next_iter);
if (awaitable == NULL) {
"from __anext__: %.100s",
Py_TYPE(next_iter)->tp_name);
Py_DECREF(next_iter);
- GOTO_ERROR(error);
+ JUMP_TO_ERROR();
} else {
Py_DECREF(next_iter);
}
/* The code below jumps to `error` if `iter` is NULL. */
}
}
- if (iter == NULL) goto pop_1_error_tier_two;
+ if (iter == NULL) JUMP_TO_ERROR();
stack_pointer[-1] = iter;
break;
}
- /* _SEND is not a viable micro-op for tier 2 */
+ /* _SEND is not a viable micro-op for tier 2 because it uses the 'this_instr' variable */
- /* _SEND_GEN is not a viable micro-op for tier 2 */
+ /* _SEND_GEN is not a viable micro-op for tier 2 because it uses the 'this_instr' variable */
- /* _INSTRUMENTED_YIELD_VALUE is not a viable micro-op for tier 2 */
+ /* _INSTRUMENTED_YIELD_VALUE is not a viable micro-op for tier 2 because it is instrumented */
case _POP_EXCEPT: {
PyObject *exc_value;
case _LOAD_BUILD_CLASS: {
PyObject *bc;
- if (PyMapping_GetOptionalItem(BUILTINS(), &_Py_ID(__build_class__), &bc) < 0) goto error_tier_two;
+ if (PyMapping_GetOptionalItem(BUILTINS(), &_Py_ID(__build_class__), &bc) < 0) JUMP_TO_ERROR();
if (bc == NULL) {
_PyErr_SetString(tstate, PyExc_NameError,
"__build_class__ not found");
- if (true) goto error_tier_two;
+ if (true) JUMP_TO_ERROR();
}
stack_pointer[0] = bc;
stack_pointer += 1;
_PyErr_Format(tstate, PyExc_SystemError,
"no locals found when storing %R", name);
Py_DECREF(v);
- if (true) goto pop_1_error_tier_two;
+ if (true) JUMP_TO_ERROR();
}
if (PyDict_CheckExact(ns))
err = PyDict_SetItem(ns, name, v);
else
err = PyObject_SetItem(ns, name, v);
Py_DECREF(v);
- if (err) goto pop_1_error_tier_two;
+ if (err) JUMP_TO_ERROR();
stack_pointer += -1;
break;
}
if (ns == NULL) {
_PyErr_Format(tstate, PyExc_SystemError,
"no locals when deleting %R", name);
- GOTO_ERROR(error);
+ JUMP_TO_ERROR();
}
err = PyObject_DelItem(ns, name);
// Can't use ERROR_IF here.
_PyEval_FormatExcCheckArg(tstate, PyExc_NameError,
NAME_ERROR_MSG,
name);
- GOTO_ERROR(error);
+ JUMP_TO_ERROR();
}
break;
}
PyObject **top = stack_pointer + oparg - 1;
int res = _PyEval_UnpackIterable(tstate, seq, oparg, -1, top);
Py_DECREF(seq);
- if (res == 0) goto pop_1_error_tier_two;
+ if (res == 0) JUMP_TO_ERROR();
stack_pointer += -1 + oparg;
break;
}
oparg = CURRENT_OPARG();
seq = stack_pointer[-1];
assert(oparg == 2);
- if (!PyTuple_CheckExact(seq)) goto deoptimize;
- if (PyTuple_GET_SIZE(seq) != 2) goto deoptimize;
+ if (!PyTuple_CheckExact(seq)) JUMP_TO_JUMP_TARGET();
+ if (PyTuple_GET_SIZE(seq) != 2) JUMP_TO_JUMP_TARGET();
STAT_INC(UNPACK_SEQUENCE, hit);
val0 = Py_NewRef(PyTuple_GET_ITEM(seq, 0));
val1 = Py_NewRef(PyTuple_GET_ITEM(seq, 1));
oparg = CURRENT_OPARG();
seq = stack_pointer[-1];
values = &stack_pointer[-1];
- if (!PyTuple_CheckExact(seq)) goto deoptimize;
- if (PyTuple_GET_SIZE(seq) != oparg) goto deoptimize;
+ if (!PyTuple_CheckExact(seq)) JUMP_TO_JUMP_TARGET();
+ if (PyTuple_GET_SIZE(seq) != oparg) JUMP_TO_JUMP_TARGET();
STAT_INC(UNPACK_SEQUENCE, hit);
PyObject **items = _PyTuple_ITEMS(seq);
for (int i = oparg; --i >= 0; ) {
oparg = CURRENT_OPARG();
seq = stack_pointer[-1];
values = &stack_pointer[-1];
- if (!PyList_CheckExact(seq)) goto deoptimize;
- if (PyList_GET_SIZE(seq) != oparg) goto deoptimize;
+ if (!PyList_CheckExact(seq)) JUMP_TO_JUMP_TARGET();
+ if (PyList_GET_SIZE(seq) != oparg) JUMP_TO_JUMP_TARGET();
STAT_INC(UNPACK_SEQUENCE, hit);
PyObject **items = _PyList_ITEMS(seq);
for (int i = oparg; --i >= 0; ) {
PyObject **top = stack_pointer + totalargs - 1;
int res = _PyEval_UnpackIterable(tstate, seq, oparg & 0xFF, oparg >> 8, top);
Py_DECREF(seq);
- if (res == 0) goto pop_1_error_tier_two;
+ if (res == 0) JUMP_TO_ERROR();
stack_pointer += (oparg >> 8) + (oparg & 0xFF);
break;
}
int err = PyObject_SetAttr(owner, name, v);
Py_DECREF(v);
Py_DECREF(owner);
- if (err) goto pop_2_error_tier_two;
+ if (err) JUMP_TO_ERROR();
stack_pointer += -2;
break;
}
PyObject *name = GETITEM(FRAME_CO_NAMES, oparg);
int err = PyObject_DelAttr(owner, name);
Py_DECREF(owner);
- if (err) goto pop_1_error_tier_two;
+ if (err) JUMP_TO_ERROR();
stack_pointer += -1;
break;
}
PyObject *name = GETITEM(FRAME_CO_NAMES, oparg);
int err = PyDict_SetItem(GLOBALS(), name, v);
Py_DECREF(v);
- if (err) goto pop_1_error_tier_two;
+ if (err) JUMP_TO_ERROR();
stack_pointer += -1;
break;
}
int err = PyDict_Pop(GLOBALS(), name, NULL);
// Can't use ERROR_IF here.
if (err < 0) {
- GOTO_ERROR(error);
+ JUMP_TO_ERROR();
}
if (err == 0) {
_PyEval_FormatExcCheckArg(tstate, PyExc_NameError,
NAME_ERROR_MSG, name);
- GOTO_ERROR(error);
+ JUMP_TO_ERROR();
}
break;
}
if (locals == NULL) {
_PyErr_SetString(tstate, PyExc_SystemError,
"no locals found");
- if (true) goto error_tier_two;
+ if (true) JUMP_TO_ERROR();
}
Py_INCREF(locals);
stack_pointer[0] = locals;
mod_or_class_dict = stack_pointer[-1];
PyObject *name = GETITEM(FRAME_CO_NAMES, oparg);
if (PyMapping_GetOptionalItem(mod_or_class_dict, name, &v) < 0) {
- GOTO_ERROR(error);
+ JUMP_TO_ERROR();
}
if (v == NULL) {
if (PyDict_GetItemRef(GLOBALS(), name, &v) < 0) {
- GOTO_ERROR(error);
+ JUMP_TO_ERROR();
}
if (v == NULL) {
if (PyMapping_GetOptionalItem(BUILTINS(), name, &v) < 0) {
- GOTO_ERROR(error);
+ JUMP_TO_ERROR();
}
if (v == NULL) {
_PyEval_FormatExcCheckArg(
tstate, PyExc_NameError,
NAME_ERROR_MSG, name);
- GOTO_ERROR(error);
+ JUMP_TO_ERROR();
}
}
}
break;
}
- case _LOAD_NAME: {
- PyObject *v;
- oparg = CURRENT_OPARG();
- PyObject *mod_or_class_dict = LOCALS();
- if (mod_or_class_dict == NULL) {
- _PyErr_SetString(tstate, PyExc_SystemError,
- "no locals found");
- if (true) goto error_tier_two;
- }
- PyObject *name = GETITEM(FRAME_CO_NAMES, oparg);
- if (PyMapping_GetOptionalItem(mod_or_class_dict, name, &v) < 0) {
- GOTO_ERROR(error);
- }
- if (v == NULL) {
- if (PyDict_GetItemRef(GLOBALS(), name, &v) < 0) {
- GOTO_ERROR(error);
- }
- if (v == NULL) {
- if (PyMapping_GetOptionalItem(BUILTINS(), name, &v) < 0) {
- GOTO_ERROR(error);
- }
- if (v == NULL) {
- _PyEval_FormatExcCheckArg(
- tstate, PyExc_NameError,
- NAME_ERROR_MSG, name);
- GOTO_ERROR(error);
- }
- }
- }
- stack_pointer[0] = v;
- stack_pointer += 1;
- break;
- }
+ /* _LOAD_NAME is not a viable micro-op for tier 2 because it has both popping and not-popping errors */
case _LOAD_GLOBAL: {
PyObject *res;
_PyEval_FormatExcCheckArg(tstate, PyExc_NameError,
NAME_ERROR_MSG, name);
}
- if (true) goto error_tier_two;
+ if (true) JUMP_TO_ERROR();
}
Py_INCREF(res);
}
else {
/* Slow-path if globals or builtins is not a dict */
/* namespace 1: globals */
- if (PyMapping_GetOptionalItem(GLOBALS(), name, &res) < 0) goto error_tier_two;
+ if (PyMapping_GetOptionalItem(GLOBALS(), name, &res) < 0) JUMP_TO_ERROR();
if (res == NULL) {
/* namespace 2: builtins */
- if (PyMapping_GetOptionalItem(BUILTINS(), name, &res) < 0) goto error_tier_two;
+ if (PyMapping_GetOptionalItem(BUILTINS(), name, &res) < 0) JUMP_TO_ERROR();
if (res == NULL) {
_PyEval_FormatExcCheckArg(
tstate, PyExc_NameError,
NAME_ERROR_MSG, name);
- if (true) goto error_tier_two;
+ if (true) JUMP_TO_ERROR();
}
}
}
case _GUARD_GLOBALS_VERSION: {
uint16_t version = (uint16_t)CURRENT_OPERAND();
PyDictObject *dict = (PyDictObject *)GLOBALS();
- if (!PyDict_CheckExact(dict)) goto deoptimize;
- if (dict->ma_keys->dk_version != version) goto deoptimize;
+ if (!PyDict_CheckExact(dict)) JUMP_TO_JUMP_TARGET();
+ if (dict->ma_keys->dk_version != version) JUMP_TO_JUMP_TARGET();
assert(DK_IS_UNICODE(dict->ma_keys));
break;
}
case _GUARD_BUILTINS_VERSION: {
uint16_t version = (uint16_t)CURRENT_OPERAND();
PyDictObject *dict = (PyDictObject *)BUILTINS();
- if (!PyDict_CheckExact(dict)) goto deoptimize;
- if (dict->ma_keys->dk_version != version) goto deoptimize;
+ if (!PyDict_CheckExact(dict)) JUMP_TO_JUMP_TARGET();
+ if (dict->ma_keys->dk_version != version) JUMP_TO_JUMP_TARGET();
assert(DK_IS_UNICODE(dict->ma_keys));
break;
}
PyDictObject *dict = (PyDictObject *)GLOBALS();
PyDictUnicodeEntry *entries = DK_UNICODE_ENTRIES(dict->ma_keys);
res = entries[index].me_value;
- if (res == NULL) goto deoptimize;
+ if (res == NULL) JUMP_TO_JUMP_TARGET();
Py_INCREF(res);
STAT_INC(LOAD_GLOBAL, hit);
null = NULL;
PyDictObject *bdict = (PyDictObject *)BUILTINS();
PyDictUnicodeEntry *entries = DK_UNICODE_ENTRIES(bdict->ma_keys);
res = entries[index].me_value;
- if (res == NULL) goto deoptimize;
+ if (res == NULL) JUMP_TO_JUMP_TARGET();
Py_INCREF(res);
STAT_INC(LOAD_GLOBAL, hit);
null = NULL;
case _DELETE_FAST: {
oparg = CURRENT_OPARG();
PyObject *v = GETLOCAL(oparg);
- if (v == NULL) goto unbound_local_error_tier_two;
+ if (v == NULL) {
+ _PyEval_FormatExcCheckArg(tstate, PyExc_UnboundLocalError,
+ UNBOUNDLOCAL_ERROR_MSG,
+ PyTuple_GetItem(_PyFrame_GetCode(frame)->co_localsplusnames, oparg)
+ );
+ if (1) JUMP_TO_ERROR();
+ }
SETLOCAL(oparg, NULL);
break;
}
PyObject *initial = GETLOCAL(oparg);
PyObject *cell = PyCell_New(initial);
if (cell == NULL) {
- GOTO_ERROR(error);
+ JUMP_TO_ERROR();
}
SETLOCAL(oparg, cell);
break;
// Fortunately we don't need its superpower.
if (oldobj == NULL) {
_PyEval_FormatExcUnbound(tstate, _PyFrame_GetCode(frame), oparg);
- GOTO_ERROR(error);
+ JUMP_TO_ERROR();
}
PyCell_SET(cell, NULL);
Py_DECREF(oldobj);
assert(oparg >= 0 && oparg < _PyFrame_GetCode(frame)->co_nlocalsplus);
name = PyTuple_GET_ITEM(_PyFrame_GetCode(frame)->co_localsplusnames, oparg);
if (PyMapping_GetOptionalItem(class_dict, name, &value) < 0) {
- GOTO_ERROR(error);
+ JUMP_TO_ERROR();
}
if (!value) {
PyObject *cell = GETLOCAL(oparg);
value = PyCell_GET(cell);
if (value == NULL) {
_PyEval_FormatExcUnbound(tstate, _PyFrame_GetCode(frame), oparg);
- GOTO_ERROR(error);
+ JUMP_TO_ERROR();
}
Py_INCREF(value);
}
value = PyCell_GET(cell);
if (value == NULL) {
_PyEval_FormatExcUnbound(tstate, _PyFrame_GetCode(frame), oparg);
- if (true) goto error_tier_two;
+ if (true) JUMP_TO_ERROR();
}
Py_INCREF(value);
stack_pointer[0] = value;
for (int _i = oparg; --_i >= 0;) {
Py_DECREF(pieces[_i]);
}
- if (str == NULL) { stack_pointer += -oparg; goto error_tier_two; }
+ if (str == NULL) JUMP_TO_ERROR();
stack_pointer[-oparg] = str;
stack_pointer += 1 - oparg;
break;
oparg = CURRENT_OPARG();
values = &stack_pointer[-oparg];
tup = _PyTuple_FromArraySteal(values, oparg);
- if (tup == NULL) { stack_pointer += -oparg; goto error_tier_two; }
+ if (tup == NULL) JUMP_TO_ERROR();
stack_pointer[-oparg] = tup;
stack_pointer += 1 - oparg;
break;
oparg = CURRENT_OPARG();
values = &stack_pointer[-oparg];
list = _PyList_FromArraySteal(values, oparg);
- if (list == NULL) { stack_pointer += -oparg; goto error_tier_two; }
+ if (list == NULL) JUMP_TO_ERROR();
stack_pointer[-oparg] = list;
stack_pointer += 1 - oparg;
break;
Py_TYPE(iterable)->tp_name);
}
Py_DECREF(iterable);
- if (true) goto pop_1_error_tier_two;
+ if (true) JUMP_TO_ERROR();
}
assert(Py_IsNone(none_val));
Py_DECREF(iterable);
set = stack_pointer[-2 - (oparg-1)];
int err = _PySet_Update(set, iterable);
Py_DECREF(iterable);
- if (err < 0) goto pop_1_error_tier_two;
+ if (err < 0) JUMP_TO_ERROR();
stack_pointer += -1;
break;
}
- case _BUILD_SET: {
- PyObject **values;
- PyObject *set;
- oparg = CURRENT_OPARG();
- values = &stack_pointer[-oparg];
- set = PySet_New(NULL);
- if (set == NULL)
- GOTO_ERROR(error);
- int err = 0;
- for (int i = 0; i < oparg; i++) {
- PyObject *item = values[i];
- if (err == 0)
- err = PySet_Add(set, item);
- Py_DECREF(item);
- }
- if (err != 0) {
- Py_DECREF(set);
- if (true) { stack_pointer += -oparg; goto error_tier_two; }
- }
- stack_pointer[-oparg] = set;
- stack_pointer += 1 - oparg;
- break;
- }
+ /* _BUILD_SET is not a viable micro-op for tier 2 because it has both popping and not-popping errors */
case _BUILD_MAP: {
PyObject **values;
for (int _i = oparg*2; --_i >= 0;) {
Py_DECREF(values[_i]);
}
- if (map == NULL) { stack_pointer += -oparg*2; goto error_tier_two; }
+ if (map == NULL) JUMP_TO_ERROR();
stack_pointer[-oparg*2] = map;
stack_pointer += 1 - oparg*2;
break;
if (LOCALS() == NULL) {
_PyErr_Format(tstate, PyExc_SystemError,
"no locals found when setting up annotations");
- if (true) goto error_tier_two;
+ if (true) JUMP_TO_ERROR();
}
/* check if __annotations__ in locals()... */
- if (PyMapping_GetOptionalItem(LOCALS(), &_Py_ID(__annotations__), &ann_dict) < 0) goto error_tier_two;
+ if (PyMapping_GetOptionalItem(LOCALS(), &_Py_ID(__annotations__), &ann_dict) < 0) JUMP_TO_ERROR();
if (ann_dict == NULL) {
ann_dict = PyDict_New();
- if (ann_dict == NULL) goto error_tier_two;
+ if (ann_dict == NULL) JUMP_TO_ERROR();
err = PyObject_SetItem(LOCALS(), &_Py_ID(__annotations__),
ann_dict);
Py_DECREF(ann_dict);
- if (err) goto error_tier_two;
+ if (err) JUMP_TO_ERROR();
}
else {
Py_DECREF(ann_dict);
oparg = CURRENT_OPARG();
keys = stack_pointer[-1];
values = &stack_pointer[-1 - oparg];
- if (!PyTuple_CheckExact(keys) ||
- PyTuple_GET_SIZE(keys) != (Py_ssize_t)oparg) {
- _PyErr_SetString(tstate, PyExc_SystemError,
- "bad BUILD_CONST_KEY_MAP keys argument");
- GOTO_ERROR(error); // Pop the keys and values.
- }
+ assert(PyTuple_CheckExact(keys));
+ assert(PyTuple_GET_SIZE(keys) == (Py_ssize_t)oparg);
map = _PyDict_FromItems(
&PyTuple_GET_ITEM(keys, 0), 1,
values, 1, oparg);
Py_DECREF(values[_i]);
}
Py_DECREF(keys);
- if (map == NULL) { stack_pointer += -1 - oparg; goto error_tier_two; }
+ if (map == NULL) JUMP_TO_ERROR();
stack_pointer[-1 - oparg] = map;
stack_pointer += -oparg;
break;
Py_TYPE(update)->tp_name);
}
Py_DECREF(update);
- if (true) goto pop_1_error_tier_two;
+ if (true) JUMP_TO_ERROR();
}
Py_DECREF(update);
stack_pointer += -1;
if (_PyDict_MergeEx(dict, update, 2) < 0) {
_PyEval_FormatKwargsError(tstate, callable, update);
Py_DECREF(update);
- if (true) goto pop_1_error_tier_two;
+ if (true) JUMP_TO_ERROR();
}
Py_DECREF(update);
stack_pointer += -1;
assert(PyDict_CheckExact(dict));
/* dict[key] = value */
// Do not DECREF INPUTS because the function steals the references
- if (_PyDict_SetItem_Take2((PyDictObject *)dict, key, value) != 0) goto pop_2_error_tier_two;
+ if (_PyDict_SetItem_Take2((PyDictObject *)dict, key, value) != 0) JUMP_TO_ERROR();
stack_pointer += -2;
break;
}
- /* _INSTRUMENTED_LOAD_SUPER_ATTR is not a viable micro-op for tier 2 */
+ /* _INSTRUMENTED_LOAD_SUPER_ATTR is not a viable micro-op for tier 2 because it is instrumented */
case _LOAD_SUPER_ATTR_ATTR: {
PyObject *self;
class = stack_pointer[-2];
global_super = stack_pointer[-3];
assert(!(oparg & 1));
- if (global_super != (PyObject *)&PySuper_Type) goto deoptimize;
- if (!PyType_Check(class)) goto deoptimize;
+ if (global_super != (PyObject *)&PySuper_Type) JUMP_TO_JUMP_TARGET();
+ if (!PyType_Check(class)) JUMP_TO_JUMP_TARGET();
STAT_INC(LOAD_SUPER_ATTR, hit);
PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 2);
attr = _PySuper_Lookup((PyTypeObject *)class, self, name, NULL);
Py_DECREF(global_super);
Py_DECREF(class);
Py_DECREF(self);
- if (attr == NULL) goto pop_3_error_tier_two;
+ if (attr == NULL) JUMP_TO_ERROR();
stack_pointer[-3] = attr;
stack_pointer += -2;
break;
class = stack_pointer[-2];
global_super = stack_pointer[-3];
assert(oparg & 1);
- if (global_super != (PyObject *)&PySuper_Type) goto deoptimize;
- if (!PyType_Check(class)) goto deoptimize;
+ if (global_super != (PyObject *)&PySuper_Type) JUMP_TO_JUMP_TARGET();
+ if (!PyType_Check(class)) JUMP_TO_JUMP_TARGET();
STAT_INC(LOAD_SUPER_ATTR, hit);
PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 2);
PyTypeObject *cls = (PyTypeObject *)class;
Py_DECREF(class);
if (attr == NULL) {
Py_DECREF(self);
- if (true) goto pop_3_error_tier_two;
+ if (true) JUMP_TO_ERROR();
}
if (method_found) {
self_or_null = self; // transfer ownership
meth | NULL | arg1 | ... | argN
*/
Py_DECREF(owner);
- if (attr == NULL) goto pop_1_error_tier_two;
+ if (attr == NULL) JUMP_TO_ERROR();
self_or_null = NULL;
}
}
/* Classic, pushes one value. */
attr = PyObject_GetAttr(owner, name);
Py_DECREF(owner);
- if (attr == NULL) goto pop_1_error_tier_two;
+ if (attr == NULL) JUMP_TO_ERROR();
}
stack_pointer[-1] = attr;
if (oparg & 1) stack_pointer[0] = self_or_null;
uint32_t type_version = (uint32_t)CURRENT_OPERAND();
PyTypeObject *tp = Py_TYPE(owner);
assert(type_version != 0);
- if (tp->tp_version_tag != type_version) goto side_exit;
+ if (tp->tp_version_tag != type_version) JUMP_TO_JUMP_TARGET();
break;
}
assert(Py_TYPE(owner)->tp_dictoffset < 0);
assert(Py_TYPE(owner)->tp_flags & Py_TPFLAGS_MANAGED_DICT);
PyDictOrValues *dorv = _PyObject_DictOrValuesPointer(owner);
- if (!_PyDictOrValues_IsValues(*dorv) && !_PyObject_MakeInstanceAttributesFromDict(owner, dorv)) goto deoptimize;
+ if (!_PyDictOrValues_IsValues(*dorv) && !_PyObject_MakeInstanceAttributesFromDict(owner, dorv)) JUMP_TO_JUMP_TARGET();
break;
}
uint16_t index = (uint16_t)CURRENT_OPERAND();
PyDictOrValues dorv = *_PyObject_DictOrValuesPointer(owner);
attr = _PyDictOrValues_GetValues(dorv)->values[index];
- if (attr == NULL) goto deoptimize;
+ if (attr == NULL) JUMP_TO_JUMP_TARGET();
STAT_INC(LOAD_ATTR, hit);
Py_INCREF(attr);
null = NULL;
uint16_t index = (uint16_t)CURRENT_OPERAND();
PyDictOrValues dorv = *_PyObject_DictOrValuesPointer(owner);
attr = _PyDictOrValues_GetValues(dorv)->values[index];
- if (attr == NULL) goto deoptimize;
+ if (attr == NULL) JUMP_TO_JUMP_TARGET();
STAT_INC(LOAD_ATTR, hit);
Py_INCREF(attr);
null = NULL;
PyObject *owner;
owner = stack_pointer[-1];
uint32_t dict_version = (uint32_t)CURRENT_OPERAND();
- if (!PyModule_CheckExact(owner)) goto deoptimize;
+ if (!PyModule_CheckExact(owner)) JUMP_TO_JUMP_TARGET();
PyDictObject *dict = (PyDictObject *)((PyModuleObject *)owner)->md_dict;
assert(dict != NULL);
- if (dict->ma_keys->dk_version != dict_version) goto deoptimize;
+ if (dict->ma_keys->dk_version != dict_version) JUMP_TO_JUMP_TARGET();
break;
}
assert(index < dict->ma_keys->dk_nentries);
PyDictUnicodeEntry *ep = DK_UNICODE_ENTRIES(dict->ma_keys) + index;
attr = ep->me_value;
- if (attr == NULL) goto deoptimize;
+ if (attr == NULL) JUMP_TO_JUMP_TARGET();
STAT_INC(LOAD_ATTR, hit);
Py_INCREF(attr);
null = NULL;
owner = stack_pointer[-1];
assert(Py_TYPE(owner)->tp_flags & Py_TPFLAGS_MANAGED_DICT);
PyDictOrValues dorv = *_PyObject_DictOrValuesPointer(owner);
- if (_PyDictOrValues_IsValues(dorv)) goto deoptimize;
+ if (_PyDictOrValues_IsValues(dorv)) JUMP_TO_JUMP_TARGET();
PyDictObject *dict = (PyDictObject *)_PyDictOrValues_GetDict(dorv);
- if (dict == NULL) goto deoptimize;
+ if (dict == NULL) JUMP_TO_JUMP_TARGET();
assert(PyDict_CheckExact((PyObject *)dict));
break;
}
uint16_t hint = (uint16_t)CURRENT_OPERAND();
PyDictOrValues dorv = *_PyObject_DictOrValuesPointer(owner);
PyDictObject *dict = (PyDictObject *)_PyDictOrValues_GetDict(dorv);
- if (hint >= (size_t)dict->ma_keys->dk_nentries) goto deoptimize;
+ if (hint >= (size_t)dict->ma_keys->dk_nentries) JUMP_TO_JUMP_TARGET();
PyObject *name = GETITEM(FRAME_CO_NAMES, oparg>>1);
if (DK_IS_UNICODE(dict->ma_keys)) {
PyDictUnicodeEntry *ep = DK_UNICODE_ENTRIES(dict->ma_keys) + hint;
- if (ep->me_key != name) goto deoptimize;
+ if (ep->me_key != name) JUMP_TO_JUMP_TARGET();
attr = ep->me_value;
}
else {
PyDictKeyEntry *ep = DK_ENTRIES(dict->ma_keys) + hint;
- if (ep->me_key != name) goto deoptimize;
+ if (ep->me_key != name) JUMP_TO_JUMP_TARGET();
attr = ep->me_value;
}
- if (attr == NULL) goto deoptimize;
+ if (attr == NULL) JUMP_TO_JUMP_TARGET();
STAT_INC(LOAD_ATTR, hit);
Py_INCREF(attr);
null = NULL;
uint16_t index = (uint16_t)CURRENT_OPERAND();
char *addr = (char *)owner + index;
attr = *(PyObject **)addr;
- if (attr == NULL) goto deoptimize;
+ if (attr == NULL) JUMP_TO_JUMP_TARGET();
STAT_INC(LOAD_ATTR, hit);
Py_INCREF(attr);
null = NULL;
uint16_t index = (uint16_t)CURRENT_OPERAND();
char *addr = (char *)owner + index;
attr = *(PyObject **)addr;
- if (attr == NULL) goto deoptimize;
+ if (attr == NULL) JUMP_TO_JUMP_TARGET();
STAT_INC(LOAD_ATTR, hit);
Py_INCREF(attr);
null = NULL;
PyObject *owner;
owner = stack_pointer[-1];
uint32_t type_version = (uint32_t)CURRENT_OPERAND();
- if (!PyType_Check(owner)) goto deoptimize;
+ if (!PyType_Check(owner)) JUMP_TO_JUMP_TARGET();
assert(type_version != 0);
- if (((PyTypeObject *)owner)->tp_version_tag != type_version) goto deoptimize;
+ if (((PyTypeObject *)owner)->tp_version_tag != type_version) JUMP_TO_JUMP_TARGET();
break;
}
/* _LOAD_ATTR_CLASS is split on (oparg & 1) */
- /* _LOAD_ATTR_PROPERTY is not a viable micro-op for tier 2 */
+ /* _LOAD_ATTR_PROPERTY is not a viable micro-op for tier 2 because it uses the 'this_instr' variable */
- /* _LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN is not a viable micro-op for tier 2 */
+ /* _LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN is not a viable micro-op for tier 2 because it uses the 'this_instr' variable */
case _GUARD_DORV_VALUES: {
PyObject *owner;
owner = stack_pointer[-1];
assert(Py_TYPE(owner)->tp_flags & Py_TPFLAGS_MANAGED_DICT);
PyDictOrValues dorv = *_PyObject_DictOrValuesPointer(owner);
- if (!_PyDictOrValues_IsValues(dorv)) goto deoptimize;
+ if (!_PyDictOrValues_IsValues(dorv)) JUMP_TO_JUMP_TARGET();
break;
}
break;
}
- /* _STORE_ATTR_WITH_HINT is not a viable micro-op for tier 2 */
+ /* _STORE_ATTR_WITH_HINT is not a viable micro-op for tier 2 because it has unused cache entries */
case _STORE_ATTR_SLOT: {
PyObject *owner;
res = PyObject_RichCompare(left, right, oparg >> 5);
Py_DECREF(left);
Py_DECREF(right);
- if (res == NULL) goto pop_2_error_tier_two;
+ if (res == NULL) JUMP_TO_ERROR();
if (oparg & 16) {
int res_bool = PyObject_IsTrue(res);
Py_DECREF(res);
- if (res_bool < 0) goto pop_2_error_tier_two;
+ if (res_bool < 0) JUMP_TO_ERROR();
res = res_bool ? Py_True : Py_False;
}
stack_pointer[-2] = res;
oparg = CURRENT_OPARG();
right = stack_pointer[-1];
left = stack_pointer[-2];
- if (!_PyLong_IsCompact((PyLongObject *)left)) goto deoptimize;
- if (!_PyLong_IsCompact((PyLongObject *)right)) goto deoptimize;
+ if (!_PyLong_IsCompact((PyLongObject *)left)) JUMP_TO_JUMP_TARGET();
+ if (!_PyLong_IsCompact((PyLongObject *)right)) JUMP_TO_JUMP_TARGET();
STAT_INC(COMPARE_OP, hit);
assert(_PyLong_DigitCount((PyLongObject *)left) <= 1 &&
_PyLong_DigitCount((PyLongObject *)right) <= 1);
int res = PySequence_Contains(right, left);
Py_DECREF(left);
Py_DECREF(right);
- if (res < 0) goto pop_2_error_tier_two;
+ if (res < 0) JUMP_TO_ERROR();
b = (res ^ oparg) ? Py_True : Py_False;
stack_pointer[-2] = b;
stack_pointer += -1;
oparg = CURRENT_OPARG();
right = stack_pointer[-1];
left = stack_pointer[-2];
- if (!(PySet_CheckExact(right) || PyFrozenSet_CheckExact(right))) goto deoptimize;
+ if (!(PySet_CheckExact(right) || PyFrozenSet_CheckExact(right))) JUMP_TO_JUMP_TARGET();
STAT_INC(CONTAINS_OP, hit);
// Note: both set and frozenset use the same seq_contains method!
int res = _PySet_Contains((PySetObject *)right, left);
Py_DECREF(left);
Py_DECREF(right);
- if (res < 0) goto pop_2_error_tier_two;
+ if (res < 0) JUMP_TO_ERROR();
b = (res ^ oparg) ? Py_True : Py_False;
stack_pointer[-2] = b;
stack_pointer += -1;
oparg = CURRENT_OPARG();
right = stack_pointer[-1];
left = stack_pointer[-2];
- if (!PyDict_CheckExact(right)) goto deoptimize;
+ if (!PyDict_CheckExact(right)) JUMP_TO_JUMP_TARGET();
STAT_INC(CONTAINS_OP, hit);
int res = PyDict_Contains(right, left);
Py_DECREF(left);
Py_DECREF(right);
- if (res < 0) goto pop_2_error_tier_two;
+ if (res < 0) JUMP_TO_ERROR();
b = (res ^ oparg) ? Py_True : Py_False;
stack_pointer[-2] = b;
stack_pointer += -1;
if (_PyEval_CheckExceptStarTypeValid(tstate, match_type) < 0) {
Py_DECREF(exc_value);
Py_DECREF(match_type);
- if (true) goto pop_2_error_tier_two;
+ if (true) JUMP_TO_ERROR();
}
match = NULL;
rest = NULL;
&match, &rest);
Py_DECREF(exc_value);
Py_DECREF(match_type);
- if (res < 0) goto pop_2_error_tier_two;
+ if (res < 0) JUMP_TO_ERROR();
assert((match == NULL) == (rest == NULL));
- if (match == NULL) goto pop_2_error_tier_two;
+ if (match == NULL) JUMP_TO_ERROR();
if (!Py_IsNone(match)) {
PyErr_SetHandledException(match);
}
assert(PyExceptionInstance_Check(left));
if (_PyEval_CheckExceptTypeValid(tstate, right) < 0) {
Py_DECREF(right);
- if (true) goto pop_1_error_tier_two;
+ if (true) JUMP_TO_ERROR();
}
int res = PyErr_GivenExceptionMatches(left, right);
Py_DECREF(right);
break;
}
- /* _POP_JUMP_IF_FALSE is not a viable micro-op for tier 2 */
+ /* _POP_JUMP_IF_FALSE is not a viable micro-op for tier 2 because it is replaced */
- /* _POP_JUMP_IF_TRUE is not a viable micro-op for tier 2 */
+ /* _POP_JUMP_IF_TRUE is not a viable micro-op for tier 2 because it is replaced */
case _IS_NONE: {
PyObject *value;
obj = stack_pointer[-1];
// PUSH(len(TOS))
Py_ssize_t len_i = PyObject_Length(obj);
- if (len_i < 0) goto error_tier_two;
+ if (len_i < 0) JUMP_TO_ERROR();
len_o = PyLong_FromSsize_t(len_i);
- if (len_o == NULL) goto error_tier_two;
+ if (len_o == NULL) JUMP_TO_ERROR();
stack_pointer[0] = len_o;
stack_pointer += 1;
break;
assert(PyTuple_CheckExact(attrs)); // Success!
}
else {
- if (_PyErr_Occurred(tstate)) goto pop_3_error_tier_two;
+ if (_PyErr_Occurred(tstate)) JUMP_TO_ERROR();
// Error!
attrs = Py_None; // Failure!
}
subject = stack_pointer[-2];
// On successful match, PUSH(values). Otherwise, PUSH(None).
values_or_none = _PyEval_MatchKeys(tstate, subject, keys);
- if (values_or_none == NULL) goto error_tier_two;
+ if (values_or_none == NULL) JUMP_TO_ERROR();
stack_pointer[0] = values_or_none;
stack_pointer += 1;
break;
/* before: [obj]; after [getiter(obj)] */
iter = PyObject_GetIter(iterable);
Py_DECREF(iterable);
- if (iter == NULL) goto pop_1_error_tier_two;
+ if (iter == NULL) JUMP_TO_ERROR();
stack_pointer[-1] = iter;
break;
}
_PyErr_SetString(tstate, PyExc_TypeError,
"cannot 'yield from' a coroutine object "
"in a non-coroutine generator");
- GOTO_ERROR(error);
+ JUMP_TO_ERROR();
}
iter = iterable;
}
/* `iterable` is not a generator. */
iter = PyObject_GetIter(iterable);
if (iter == NULL) {
- GOTO_ERROR(error);
+ JUMP_TO_ERROR();
}
Py_DECREF(iterable);
}
break;
}
- /* _FOR_ITER is not a viable micro-op for tier 2 */
+ /* _FOR_ITER is not a viable micro-op for tier 2 because it is replaced */
case _FOR_ITER_TIER_TWO: {
PyObject *iter;
if (next == NULL) {
if (_PyErr_Occurred(tstate)) {
if (!_PyErr_ExceptionMatches(tstate, PyExc_StopIteration)) {
- GOTO_ERROR(error);
+ JUMP_TO_ERROR();
}
_PyErr_Clear(tstate);
}
Py_DECREF(iter);
STACK_SHRINK(1);
/* The translator sets the deopt target just past END_FOR */
- if (true) goto deoptimize;
+ if (true) JUMP_TO_JUMP_TARGET();
}
// Common case: no jump, leave it to the code generator
stack_pointer[0] = next;
break;
}
- /* _INSTRUMENTED_FOR_ITER is not a viable micro-op for tier 2 */
+ /* _INSTRUMENTED_FOR_ITER is not a viable micro-op for tier 2 because it is instrumented */
case _ITER_CHECK_LIST: {
PyObject *iter;
iter = stack_pointer[-1];
- if (Py_TYPE(iter) != &PyListIter_Type) goto deoptimize;
+ if (Py_TYPE(iter) != &PyListIter_Type) JUMP_TO_JUMP_TARGET();
break;
}
- /* _ITER_JUMP_LIST is not a viable micro-op for tier 2 */
+ /* _ITER_JUMP_LIST is not a viable micro-op for tier 2 because it is replaced */
case _GUARD_NOT_EXHAUSTED_LIST: {
PyObject *iter;
_PyListIterObject *it = (_PyListIterObject *)iter;
assert(Py_TYPE(iter) == &PyListIter_Type);
PyListObject *seq = it->it_seq;
- if (seq == NULL) goto deoptimize;
- if ((size_t)it->it_index >= (size_t)PyList_GET_SIZE(seq)) goto deoptimize;
+ if (seq == NULL) JUMP_TO_JUMP_TARGET();
+ if ((size_t)it->it_index >= (size_t)PyList_GET_SIZE(seq)) JUMP_TO_JUMP_TARGET();
break;
}
case _ITER_CHECK_TUPLE: {
PyObject *iter;
iter = stack_pointer[-1];
- if (Py_TYPE(iter) != &PyTupleIter_Type) goto deoptimize;
+ if (Py_TYPE(iter) != &PyTupleIter_Type) JUMP_TO_JUMP_TARGET();
break;
}
- /* _ITER_JUMP_TUPLE is not a viable micro-op for tier 2 */
+ /* _ITER_JUMP_TUPLE is not a viable micro-op for tier 2 because it is replaced */
case _GUARD_NOT_EXHAUSTED_TUPLE: {
PyObject *iter;
_PyTupleIterObject *it = (_PyTupleIterObject *)iter;
assert(Py_TYPE(iter) == &PyTupleIter_Type);
PyTupleObject *seq = it->it_seq;
- if (seq == NULL) goto deoptimize;
- if (it->it_index >= PyTuple_GET_SIZE(seq)) goto deoptimize;
+ if (seq == NULL) JUMP_TO_JUMP_TARGET();
+ if (it->it_index >= PyTuple_GET_SIZE(seq)) JUMP_TO_JUMP_TARGET();
break;
}
PyObject *iter;
iter = stack_pointer[-1];
_PyRangeIterObject *r = (_PyRangeIterObject *)iter;
- if (Py_TYPE(r) != &PyRangeIter_Type) goto deoptimize;
+ if (Py_TYPE(r) != &PyRangeIter_Type) JUMP_TO_JUMP_TARGET();
break;
}
- /* _ITER_JUMP_RANGE is not a viable micro-op for tier 2 */
+ /* _ITER_JUMP_RANGE is not a viable micro-op for tier 2 because it is replaced */
case _GUARD_NOT_EXHAUSTED_RANGE: {
PyObject *iter;
iter = stack_pointer[-1];
_PyRangeIterObject *r = (_PyRangeIterObject *)iter;
assert(Py_TYPE(r) == &PyRangeIter_Type);
- if (r->len <= 0) goto deoptimize;
+ if (r->len <= 0) JUMP_TO_JUMP_TARGET();
break;
}
r->start = value + r->step;
r->len--;
next = PyLong_FromLong(value);
- if (next == NULL) goto error_tier_two;
+ if (next == NULL) JUMP_TO_ERROR();
stack_pointer[0] = next;
stack_pointer += 1;
break;
}
- /* _FOR_ITER_GEN is not a viable micro-op for tier 2 */
+ /* _FOR_ITER_GEN is not a viable micro-op for tier 2 because it uses the 'this_instr' variable */
- case _BEFORE_ASYNC_WITH: {
- PyObject *mgr;
- PyObject *exit;
- PyObject *res;
- mgr = stack_pointer[-1];
- PyObject *enter = _PyObject_LookupSpecial(mgr, &_Py_ID(__aenter__));
- if (enter == NULL) {
- if (!_PyErr_Occurred(tstate)) {
- _PyErr_Format(tstate, PyExc_TypeError,
- "'%.200s' object does not support the "
- "asynchronous context manager protocol",
- Py_TYPE(mgr)->tp_name);
- }
- GOTO_ERROR(error);
- }
- exit = _PyObject_LookupSpecial(mgr, &_Py_ID(__aexit__));
- if (exit == NULL) {
- if (!_PyErr_Occurred(tstate)) {
- _PyErr_Format(tstate, PyExc_TypeError,
- "'%.200s' object does not support the "
- "asynchronous context manager protocol "
- "(missed __aexit__ method)",
- Py_TYPE(mgr)->tp_name);
- }
- Py_DECREF(enter);
- GOTO_ERROR(error);
- }
- Py_DECREF(mgr);
- res = PyObject_CallNoArgs(enter);
- Py_DECREF(enter);
- if (res == NULL) {
- Py_DECREF(exit);
- if (true) goto pop_1_error_tier_two;
- }
- stack_pointer[-1] = exit;
- stack_pointer[0] = res;
- stack_pointer += 1;
- break;
- }
+ /* _BEFORE_ASYNC_WITH is not a viable micro-op for tier 2 because it has both popping and not-popping errors */
- case _BEFORE_WITH: {
- PyObject *mgr;
- PyObject *exit;
- PyObject *res;
- mgr = stack_pointer[-1];
- /* pop the context manager, push its __exit__ and the
- * value returned from calling its __enter__
- */
- PyObject *enter = _PyObject_LookupSpecial(mgr, &_Py_ID(__enter__));
- if (enter == NULL) {
- if (!_PyErr_Occurred(tstate)) {
- _PyErr_Format(tstate, PyExc_TypeError,
- "'%.200s' object does not support the "
- "context manager protocol",
- Py_TYPE(mgr)->tp_name);
- }
- GOTO_ERROR(error);
- }
- exit = _PyObject_LookupSpecial(mgr, &_Py_ID(__exit__));
- if (exit == NULL) {
- if (!_PyErr_Occurred(tstate)) {
- _PyErr_Format(tstate, PyExc_TypeError,
- "'%.200s' object does not support the "
- "context manager protocol "
- "(missed __exit__ method)",
- Py_TYPE(mgr)->tp_name);
- }
- Py_DECREF(enter);
- GOTO_ERROR(error);
- }
- Py_DECREF(mgr);
- res = PyObject_CallNoArgs(enter);
- Py_DECREF(enter);
- if (res == NULL) {
- Py_DECREF(exit);
- if (true) goto pop_1_error_tier_two;
- }
- stack_pointer[-1] = exit;
- stack_pointer[0] = res;
- stack_pointer += 1;
- break;
- }
+ /* _BEFORE_WITH is not a viable micro-op for tier 2 because it has both popping and not-popping errors */
case _WITH_EXCEPT_START: {
PyObject *val;
PyObject *stack[4] = {NULL, exc, val, tb};
res = PyObject_Vectorcall(exit_func, stack + 1,
3 | PY_VECTORCALL_ARGUMENTS_OFFSET, NULL);
- if (res == NULL) goto error_tier_two;
+ if (res == NULL) JUMP_TO_ERROR();
stack_pointer[0] = res;
stack_pointer += 1;
break;
owner = stack_pointer[-1];
assert(Py_TYPE(owner)->tp_flags & Py_TPFLAGS_MANAGED_DICT);
PyDictOrValues *dorv = _PyObject_DictOrValuesPointer(owner);
- if (!_PyDictOrValues_IsValues(*dorv) && !_PyObject_MakeInstanceAttributesFromDict(owner, dorv)) goto deoptimize;
+ if (!_PyDictOrValues_IsValues(*dorv) && !_PyObject_MakeInstanceAttributesFromDict(owner, dorv)) JUMP_TO_JUMP_TARGET();
break;
}
uint32_t keys_version = (uint32_t)CURRENT_OPERAND();
PyTypeObject *owner_cls = Py_TYPE(owner);
PyHeapTypeObject *owner_heap_type = (PyHeapTypeObject *)owner_cls;
- if (owner_heap_type->ht_cached_keys->dk_version != keys_version) goto deoptimize;
+ if (owner_heap_type->ht_cached_keys->dk_version != keys_version) JUMP_TO_JUMP_TARGET();
break;
}
assert(dictoffset > 0);
PyObject *dict = *(PyObject **)((char *)owner + dictoffset);
/* This object has a __dict__, just not yet created */
- if (dict != NULL) goto deoptimize;
+ if (dict != NULL) JUMP_TO_JUMP_TARGET();
break;
}
break;
}
- /* _INSTRUMENTED_CALL is not a viable micro-op for tier 2 */
+ /* _INSTRUMENTED_CALL is not a viable micro-op for tier 2 because it is instrumented */
- /* _CALL is not a viable micro-op for tier 2 */
+ /* _CALL is not a viable micro-op for tier 2 because it uses the 'this_instr' variable */
case _CHECK_PERIODIC: {
CHECK_EVAL_BREAKER();
oparg = CURRENT_OPARG();
null = stack_pointer[-1 - oparg];
callable = stack_pointer[-2 - oparg];
- if (null != NULL) goto deoptimize;
- if (Py_TYPE(callable) != &PyMethod_Type) goto deoptimize;
+ if (null != NULL) JUMP_TO_JUMP_TARGET();
+ if (Py_TYPE(callable) != &PyMethod_Type) JUMP_TO_JUMP_TARGET();
break;
}
}
case _CHECK_PEP_523: {
- if (tstate->interp->eval_frame) goto deoptimize;
+ if (tstate->interp->eval_frame) JUMP_TO_JUMP_TARGET();
break;
}
self_or_null = stack_pointer[-1 - oparg];
callable = stack_pointer[-2 - oparg];
uint32_t func_version = (uint32_t)CURRENT_OPERAND();
- if (!PyFunction_Check(callable)) goto deoptimize;
+ if (!PyFunction_Check(callable)) JUMP_TO_JUMP_TARGET();
PyFunctionObject *func = (PyFunctionObject *)callable;
- if (func->func_version != func_version) goto deoptimize;
+ if (func->func_version != func_version) JUMP_TO_JUMP_TARGET();
PyCodeObject *code = (PyCodeObject *)func->func_code;
- if (code->co_argcount != oparg + (self_or_null != NULL)) goto deoptimize;
+ if (code->co_argcount != oparg + (self_or_null != NULL)) JUMP_TO_JUMP_TARGET();
break;
}
callable = stack_pointer[-2 - oparg];
PyFunctionObject *func = (PyFunctionObject *)callable;
PyCodeObject *code = (PyCodeObject *)func->func_code;
- if (!_PyThreadState_HasStackSpace(tstate, code->co_framesize)) goto deoptimize;
- if (tstate->py_recursion_remaining <= 1) goto deoptimize;
+ if (!_PyThreadState_HasStackSpace(tstate, code->co_framesize)) JUMP_TO_JUMP_TARGET();
+ if (tstate->py_recursion_remaining <= 1) JUMP_TO_JUMP_TARGET();
break;
}
break;
}
- /* _CALL_PY_WITH_DEFAULTS is not a viable micro-op for tier 2 */
+ /* _CALL_PY_WITH_DEFAULTS is not a viable micro-op for tier 2 because it uses the 'this_instr' variable */
case _CALL_TYPE_1: {
PyObject *arg;
null = stack_pointer[-2];
callable = stack_pointer[-3];
assert(oparg == 1);
- if (null != NULL) goto deoptimize;
- if (callable != (PyObject *)&PyType_Type) goto deoptimize;
+ if (null != NULL) JUMP_TO_JUMP_TARGET();
+ if (callable != (PyObject *)&PyType_Type) JUMP_TO_JUMP_TARGET();
STAT_INC(CALL, hit);
res = Py_NewRef(Py_TYPE(arg));
Py_DECREF(arg);
null = stack_pointer[-2];
callable = stack_pointer[-3];
assert(oparg == 1);
- if (null != NULL) goto deoptimize;
- if (callable != (PyObject *)&PyUnicode_Type) goto deoptimize;
+ if (null != NULL) JUMP_TO_JUMP_TARGET();
+ if (callable != (PyObject *)&PyUnicode_Type) JUMP_TO_JUMP_TARGET();
STAT_INC(CALL, hit);
res = PyObject_Str(arg);
Py_DECREF(arg);
- if (res == NULL) goto pop_3_error_tier_two;
+ if (res == NULL) JUMP_TO_ERROR();
stack_pointer[-3] = res;
stack_pointer += -2;
break;
null = stack_pointer[-2];
callable = stack_pointer[-3];
assert(oparg == 1);
- if (null != NULL) goto deoptimize;
- if (callable != (PyObject *)&PyTuple_Type) goto deoptimize;
+ if (null != NULL) JUMP_TO_JUMP_TARGET();
+ if (callable != (PyObject *)&PyTuple_Type) JUMP_TO_JUMP_TARGET();
STAT_INC(CALL, hit);
res = PySequence_Tuple(arg);
Py_DECREF(arg);
- if (res == NULL) goto pop_3_error_tier_two;
+ if (res == NULL) JUMP_TO_ERROR();
stack_pointer[-3] = res;
stack_pointer += -2;
break;
}
- /* _CALL_ALLOC_AND_ENTER_INIT is not a viable micro-op for tier 2 */
+ /* _CALL_ALLOC_AND_ENTER_INIT is not a viable micro-op for tier 2 because it uses the 'this_instr' variable */
case _EXIT_INIT_CHECK: {
PyObject *should_be_none;
PyErr_Format(PyExc_TypeError,
"__init__() should return None, not '%.200s'",
Py_TYPE(should_be_none)->tp_name);
- GOTO_ERROR(error);
+ JUMP_TO_ERROR();
}
stack_pointer += -1;
break;
args--;
total_args++;
}
- if (!PyType_Check(callable)) goto deoptimize;
+ if (!PyType_Check(callable)) JUMP_TO_JUMP_TARGET();
PyTypeObject *tp = (PyTypeObject *)callable;
- if (tp->tp_vectorcall == NULL) goto deoptimize;
+ if (tp->tp_vectorcall == NULL) JUMP_TO_JUMP_TARGET();
STAT_INC(CALL, hit);
res = tp->tp_vectorcall((PyObject *)tp, args, total_args, NULL);
/* Free the arguments. */
Py_DECREF(args[i]);
}
Py_DECREF(tp);
- if (res == NULL) { stack_pointer += -2 - oparg; goto error_tier_two; }
+ if (res == NULL) JUMP_TO_ERROR();
stack_pointer[-2 - oparg] = res;
stack_pointer += -1 - oparg;
break;
args--;
total_args++;
}
- if (total_args != 1) goto deoptimize;
- if (!PyCFunction_CheckExact(callable)) goto deoptimize;
- if (PyCFunction_GET_FLAGS(callable) != METH_O) goto deoptimize;
+ if (total_args != 1) JUMP_TO_JUMP_TARGET();
+ if (!PyCFunction_CheckExact(callable)) JUMP_TO_JUMP_TARGET();
+ if (PyCFunction_GET_FLAGS(callable) != METH_O) JUMP_TO_JUMP_TARGET();
// CPython promises to check all non-vectorcall function calls.
- if (tstate->c_recursion_remaining <= 0) goto deoptimize;
+ if (tstate->c_recursion_remaining <= 0) JUMP_TO_JUMP_TARGET();
STAT_INC(CALL, hit);
PyCFunction cfunc = PyCFunction_GET_FUNCTION(callable);
PyObject *arg = args[0];
assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL));
Py_DECREF(arg);
Py_DECREF(callable);
- if (res == NULL) { stack_pointer += -2 - oparg; goto error_tier_two; }
+ if (res == NULL) JUMP_TO_ERROR();
stack_pointer[-2 - oparg] = res;
stack_pointer += -1 - oparg;
break;
args--;
total_args++;
}
- if (!PyCFunction_CheckExact(callable)) goto deoptimize;
- if (PyCFunction_GET_FLAGS(callable) != METH_FASTCALL) goto deoptimize;
+ if (!PyCFunction_CheckExact(callable)) JUMP_TO_JUMP_TARGET();
+ if (PyCFunction_GET_FLAGS(callable) != METH_FASTCALL) JUMP_TO_JUMP_TARGET();
STAT_INC(CALL, hit);
PyCFunction cfunc = PyCFunction_GET_FUNCTION(callable);
/* res = func(self, args, nargs) */
Py_DECREF(args[i]);
}
Py_DECREF(callable);
- if (res == NULL) { stack_pointer += -2 - oparg; goto error_tier_two; }
+ if (res == NULL) JUMP_TO_ERROR();
stack_pointer[-2 - oparg] = res;
stack_pointer += -1 - oparg;
break;
args--;
total_args++;
}
- if (!PyCFunction_CheckExact(callable)) goto deoptimize;
- if (PyCFunction_GET_FLAGS(callable) != (METH_FASTCALL | METH_KEYWORDS)) goto deoptimize;
+ if (!PyCFunction_CheckExact(callable)) JUMP_TO_JUMP_TARGET();
+ if (PyCFunction_GET_FLAGS(callable) != (METH_FASTCALL | METH_KEYWORDS)) JUMP_TO_JUMP_TARGET();
STAT_INC(CALL, hit);
/* res = func(self, args, nargs, kwnames) */
PyCFunctionFastWithKeywords cfunc =
Py_DECREF(args[i]);
}
Py_DECREF(callable);
- if (res == NULL) { stack_pointer += -2 - oparg; goto error_tier_two; }
+ if (res == NULL) JUMP_TO_ERROR();
stack_pointer[-2 - oparg] = res;
stack_pointer += -1 - oparg;
break;
args--;
total_args++;
}
- if (total_args != 1) goto deoptimize;
+ if (total_args != 1) JUMP_TO_JUMP_TARGET();
PyInterpreterState *interp = tstate->interp;
- if (callable != interp->callable_cache.len) goto deoptimize;
+ if (callable != interp->callable_cache.len) JUMP_TO_JUMP_TARGET();
STAT_INC(CALL, hit);
PyObject *arg = args[0];
Py_ssize_t len_i = PyObject_Length(arg);
if (len_i < 0) {
- GOTO_ERROR(error);
+ JUMP_TO_ERROR();
}
res = PyLong_FromSsize_t(len_i);
assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL));
args--;
total_args++;
}
- if (total_args != 2) goto deoptimize;
+ if (total_args != 2) JUMP_TO_JUMP_TARGET();
PyInterpreterState *interp = tstate->interp;
- if (callable != interp->callable_cache.isinstance) goto deoptimize;
+ if (callable != interp->callable_cache.isinstance) JUMP_TO_JUMP_TARGET();
STAT_INC(CALL, hit);
PyObject *cls = args[1];
PyObject *inst = args[0];
int retval = PyObject_IsInstance(inst, cls);
if (retval < 0) {
- GOTO_ERROR(error);
+ JUMP_TO_ERROR();
}
res = PyBool_FromLong(retval);
assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL));
total_args++;
}
PyMethodDescrObject *method = (PyMethodDescrObject *)callable;
- if (total_args != 2) goto deoptimize;
- if (!Py_IS_TYPE(method, &PyMethodDescr_Type)) goto deoptimize;
+ if (total_args != 2) JUMP_TO_JUMP_TARGET();
+ if (!Py_IS_TYPE(method, &PyMethodDescr_Type)) JUMP_TO_JUMP_TARGET();
PyMethodDef *meth = method->d_method;
- if (meth->ml_flags != METH_O) goto deoptimize;
+ if (meth->ml_flags != METH_O) JUMP_TO_JUMP_TARGET();
// CPython promises to check all non-vectorcall function calls.
- if (tstate->c_recursion_remaining <= 0) goto deoptimize;
+ if (tstate->c_recursion_remaining <= 0) JUMP_TO_JUMP_TARGET();
PyObject *arg = args[1];
PyObject *self = args[0];
- if (!Py_IS_TYPE(self, method->d_common.d_type)) goto deoptimize;
+ if (!Py_IS_TYPE(self, method->d_common.d_type)) JUMP_TO_JUMP_TARGET();
STAT_INC(CALL, hit);
PyCFunction cfunc = meth->ml_meth;
_Py_EnterRecursiveCallTstateUnchecked(tstate);
Py_DECREF(self);
Py_DECREF(arg);
Py_DECREF(callable);
- if (res == NULL) { stack_pointer += -2 - oparg; goto error_tier_two; }
+ if (res == NULL) JUMP_TO_ERROR();
stack_pointer[-2 - oparg] = res;
stack_pointer += -1 - oparg;
break;
total_args++;
}
PyMethodDescrObject *method = (PyMethodDescrObject *)callable;
- if (!Py_IS_TYPE(method, &PyMethodDescr_Type)) goto deoptimize;
+ if (!Py_IS_TYPE(method, &PyMethodDescr_Type)) JUMP_TO_JUMP_TARGET();
PyMethodDef *meth = method->d_method;
- if (meth->ml_flags != (METH_FASTCALL|METH_KEYWORDS)) goto deoptimize;
+ if (meth->ml_flags != (METH_FASTCALL|METH_KEYWORDS)) JUMP_TO_JUMP_TARGET();
PyTypeObject *d_type = method->d_common.d_type;
PyObject *self = args[0];
- if (!Py_IS_TYPE(self, d_type)) goto deoptimize;
+ if (!Py_IS_TYPE(self, d_type)) JUMP_TO_JUMP_TARGET();
STAT_INC(CALL, hit);
int nargs = total_args - 1;
PyCFunctionFastWithKeywords cfunc =
Py_DECREF(args[i]);
}
Py_DECREF(callable);
- if (res == NULL) { stack_pointer += -2 - oparg; goto error_tier_two; }
+ if (res == NULL) JUMP_TO_ERROR();
stack_pointer[-2 - oparg] = res;
stack_pointer += -1 - oparg;
break;
args--;
total_args++;
}
- if (total_args != 1) goto deoptimize;
+ if (total_args != 1) JUMP_TO_JUMP_TARGET();
PyMethodDescrObject *method = (PyMethodDescrObject *)callable;
- if (!Py_IS_TYPE(method, &PyMethodDescr_Type)) goto deoptimize;
+ if (!Py_IS_TYPE(method, &PyMethodDescr_Type)) JUMP_TO_JUMP_TARGET();
PyMethodDef *meth = method->d_method;
PyObject *self = args[0];
- if (!Py_IS_TYPE(self, method->d_common.d_type)) goto deoptimize;
- if (meth->ml_flags != METH_NOARGS) goto deoptimize;
+ if (!Py_IS_TYPE(self, method->d_common.d_type)) JUMP_TO_JUMP_TARGET();
+ if (meth->ml_flags != METH_NOARGS) JUMP_TO_JUMP_TARGET();
// CPython promises to check all non-vectorcall function calls.
- if (tstate->c_recursion_remaining <= 0) goto deoptimize;
+ if (tstate->c_recursion_remaining <= 0) JUMP_TO_JUMP_TARGET();
STAT_INC(CALL, hit);
PyCFunction cfunc = meth->ml_meth;
_Py_EnterRecursiveCallTstateUnchecked(tstate);
assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL));
Py_DECREF(self);
Py_DECREF(callable);
- if (res == NULL) { stack_pointer += -2 - oparg; goto error_tier_two; }
+ if (res == NULL) JUMP_TO_ERROR();
stack_pointer[-2 - oparg] = res;
stack_pointer += -1 - oparg;
break;
}
PyMethodDescrObject *method = (PyMethodDescrObject *)callable;
/* Builtin METH_FASTCALL methods, without keywords */
- if (!Py_IS_TYPE(method, &PyMethodDescr_Type)) goto deoptimize;
+ if (!Py_IS_TYPE(method, &PyMethodDescr_Type)) JUMP_TO_JUMP_TARGET();
PyMethodDef *meth = method->d_method;
- if (meth->ml_flags != METH_FASTCALL) goto deoptimize;
+ if (meth->ml_flags != METH_FASTCALL) JUMP_TO_JUMP_TARGET();
PyObject *self = args[0];
- if (!Py_IS_TYPE(self, method->d_common.d_type)) goto deoptimize;
+ if (!Py_IS_TYPE(self, method->d_common.d_type)) JUMP_TO_JUMP_TARGET();
STAT_INC(CALL, hit);
PyCFunctionFast cfunc =
(PyCFunctionFast)(void(*)(void))meth->ml_meth;
Py_DECREF(args[i]);
}
Py_DECREF(callable);
- if (res == NULL) { stack_pointer += -2 - oparg; goto error_tier_two; }
+ if (res == NULL) JUMP_TO_ERROR();
stack_pointer[-2 - oparg] = res;
stack_pointer += -1 - oparg;
break;
}
- /* _INSTRUMENTED_CALL_KW is not a viable micro-op for tier 2 */
+ /* _INSTRUMENTED_CALL_KW is not a viable micro-op for tier 2 because it is instrumented */
- /* _CALL_KW is not a viable micro-op for tier 2 */
+ /* _CALL_KW is not a viable micro-op for tier 2 because it uses the 'this_instr' variable */
- /* _INSTRUMENTED_CALL_FUNCTION_EX is not a viable micro-op for tier 2 */
+ /* _INSTRUMENTED_CALL_FUNCTION_EX is not a viable micro-op for tier 2 because it is instrumented */
- /* _CALL_FUNCTION_EX is not a viable micro-op for tier 2 */
+ /* _CALL_FUNCTION_EX is not a viable micro-op for tier 2 because it uses the 'this_instr' variable */
case _MAKE_FUNCTION: {
PyObject *codeobj;
PyFunction_New(codeobj, GLOBALS());
Py_DECREF(codeobj);
if (func_obj == NULL) {
- GOTO_ERROR(error);
+ JUMP_TO_ERROR();
}
_PyFunction_SetVersion(
func_obj, ((PyCodeObject *)codeobj)->co_version);
Py_DECREF(start);
Py_DECREF(stop);
Py_XDECREF(step);
- if (slice == NULL) { stack_pointer += -2 - ((oparg == 3) ? 1 : 0); goto error_tier_two; }
+ if (slice == NULL) JUMP_TO_ERROR();
stack_pointer[-2 - ((oparg == 3) ? 1 : 0)] = slice;
stack_pointer += -1 - ((oparg == 3) ? 1 : 0);
break;
conv_fn = _PyEval_ConversionFuncs[oparg];
result = conv_fn(value);
Py_DECREF(value);
- if (result == NULL) goto pop_1_error_tier_two;
+ if (result == NULL) JUMP_TO_ERROR();
stack_pointer[-1] = result;
break;
}
if (!PyUnicode_CheckExact(value)) {
res = PyObject_Format(value, NULL);
Py_DECREF(value);
- if (res == NULL) goto pop_1_error_tier_two;
+ if (res == NULL) JUMP_TO_ERROR();
}
else {
res = value;
res = PyObject_Format(value, fmt_spec);
Py_DECREF(value);
Py_DECREF(fmt_spec);
- if (res == NULL) goto pop_2_error_tier_two;
+ if (res == NULL) JUMP_TO_ERROR();
stack_pointer[-2] = res;
stack_pointer += -1;
break;
res = _PyEval_BinaryOps[oparg](lhs, rhs);
Py_DECREF(lhs);
Py_DECREF(rhs);
- if (res == NULL) goto pop_2_error_tier_two;
+ if (res == NULL) JUMP_TO_ERROR();
stack_pointer[-2] = res;
stack_pointer += -1;
break;
break;
}
- /* _INSTRUMENTED_INSTRUCTION is not a viable micro-op for tier 2 */
+ /* _INSTRUMENTED_INSTRUCTION is not a viable micro-op for tier 2 because it is instrumented */
- /* _INSTRUMENTED_JUMP_FORWARD is not a viable micro-op for tier 2 */
+ /* _INSTRUMENTED_JUMP_FORWARD is not a viable micro-op for tier 2 because it is instrumented */
- /* _INSTRUMENTED_JUMP_BACKWARD is not a viable micro-op for tier 2 */
+ /* _INSTRUMENTED_JUMP_BACKWARD is not a viable micro-op for tier 2 because it is instrumented */
- /* _INSTRUMENTED_POP_JUMP_IF_TRUE is not a viable micro-op for tier 2 */
+ /* _INSTRUMENTED_POP_JUMP_IF_TRUE is not a viable micro-op for tier 2 because it is instrumented */
- /* _INSTRUMENTED_POP_JUMP_IF_FALSE is not a viable micro-op for tier 2 */
+ /* _INSTRUMENTED_POP_JUMP_IF_FALSE is not a viable micro-op for tier 2 because it is instrumented */
- /* _INSTRUMENTED_POP_JUMP_IF_NONE is not a viable micro-op for tier 2 */
+ /* _INSTRUMENTED_POP_JUMP_IF_NONE is not a viable micro-op for tier 2 because it is instrumented */
- /* _INSTRUMENTED_POP_JUMP_IF_NOT_NONE is not a viable micro-op for tier 2 */
+ /* _INSTRUMENTED_POP_JUMP_IF_NOT_NONE is not a viable micro-op for tier 2 because it is instrumented */
case _GUARD_IS_TRUE_POP: {
PyObject *flag;
flag = stack_pointer[-1];
stack_pointer += -1;
- if (!Py_IsTrue(flag)) goto side_exit;
+ if (!Py_IsTrue(flag)) JUMP_TO_JUMP_TARGET();
assert(Py_IsTrue(flag));
break;
}
PyObject *flag;
flag = stack_pointer[-1];
stack_pointer += -1;
- if (!Py_IsFalse(flag)) goto side_exit;
+ if (!Py_IsFalse(flag)) JUMP_TO_JUMP_TARGET();
assert(Py_IsFalse(flag));
break;
}
stack_pointer += -1;
if (!Py_IsNone(val)) {
Py_DECREF(val);
- if (1) goto side_exit;
+ if (1) JUMP_TO_JUMP_TARGET();
}
break;
}
PyObject *val;
val = stack_pointer[-1];
stack_pointer += -1;
- if (Py_IsNone(val)) goto side_exit;
+ if (Py_IsNone(val)) JUMP_TO_JUMP_TARGET();
Py_DECREF(val);
break;
}
}
case _EXIT_TRACE: {
- if (1) goto side_exit;
+ if (1) JUMP_TO_JUMP_TARGET();
break;
}
case _CHECK_VALIDITY: {
- if (!current_executor->vm_data.valid) goto deoptimize;
+ if (!current_executor->vm_data.valid) JUMP_TO_JUMP_TARGET();
break;
}
case _CHECK_FUNCTION: {
uint32_t func_version = (uint32_t)CURRENT_OPERAND();
assert(PyFunction_Check(frame->f_funcobj));
- if (((PyFunctionObject *)frame->f_funcobj)->func_version != func_version) goto deoptimize;
+ if (((PyFunctionObject *)frame->f_funcobj)->func_version != func_version) JUMP_TO_JUMP_TARGET();
break;
}
if (optimized < 0) {
Py_DECREF(previous);
tstate->previous_executor = Py_None;
- if (1) goto error_tier_two;
+ GOTO_UNWIND();
}
GOTO_TIER_ONE(target);
}
case _CHECK_VALIDITY_AND_SET_IP: {
PyObject *instr_ptr = (PyObject *)CURRENT_OPERAND();
- if (!current_executor->vm_data.valid) goto deoptimize;
+ if (!current_executor->vm_data.valid) JUMP_TO_JUMP_TARGET();
frame->instr_ptr = (_Py_CODEUNIT *)instr_ptr;
break;
}
+ case _DEOPT: {
+ EXIT_TO_TIER1();
+ break;
+ }
+
+ case _SIDE_EXIT: {
+ EXIT_TO_TRACE();
+ break;
+ }
+
+ case _ERROR_POP_N: {
+ oparg = CURRENT_OPARG();
+ stack_pointer += -oparg;
+ GOTO_UNWIND();
+ break;
+ }
+
#undef TIER_TWO
"asynchronous context manager protocol",
Py_TYPE(mgr)->tp_name);
}
- GOTO_ERROR(error);
+ goto error;
}
exit = _PyObject_LookupSpecial(mgr, &_Py_ID(__aexit__));
if (exit == NULL) {
Py_TYPE(mgr)->tp_name);
}
Py_DECREF(enter);
- GOTO_ERROR(error);
+ goto error;
}
Py_DECREF(mgr);
res = PyObject_CallNoArgs(enter);
"context manager protocol",
Py_TYPE(mgr)->tp_name);
}
- GOTO_ERROR(error);
+ goto error;
}
exit = _PyObject_LookupSpecial(mgr, &_Py_ID(__exit__));
if (exit == NULL) {
Py_TYPE(mgr)->tp_name);
}
Py_DECREF(enter);
- GOTO_ERROR(error);
+ goto error;
}
Py_DECREF(mgr);
res = PyObject_CallNoArgs(enter);
PyObject *map;
keys = stack_pointer[-1];
values = &stack_pointer[-1 - oparg];
- if (!PyTuple_CheckExact(keys) ||
- PyTuple_GET_SIZE(keys) != (Py_ssize_t)oparg) {
- _PyErr_SetString(tstate, PyExc_SystemError,
- "bad BUILD_CONST_KEY_MAP keys argument");
- GOTO_ERROR(error); // Pop the keys and values.
- }
+ assert(PyTuple_CheckExact(keys));
+ assert(PyTuple_GET_SIZE(keys) == (Py_ssize_t)oparg);
map = _PyDict_FromItems(
&PyTuple_GET_ITEM(keys, 0), 1,
values, 1, oparg);
values = &stack_pointer[-oparg];
set = PySet_New(NULL);
if (set == NULL)
- GOTO_ERROR(error);
+ goto error;
int err = 0;
for (int i = 0; i < oparg; i++) {
PyObject *item = values[i];
// The frame has stolen all the arguments from the stack,
// so there is no need to clean them up.
if (new_frame == NULL) {
- GOTO_ERROR(error);
+ goto error;
}
frame->return_offset = (uint16_t)(next_instr - this_instr);
DISPATCH_INLINED(new_frame);
STAT_INC(CALL, hit);
PyObject *self = _PyType_NewManagedObject(tp);
if (self == NULL) {
- GOTO_ERROR(error);
+ goto error;
}
Py_DECREF(tp);
_PyInterpreterFrame *shim = _PyFrame_PushTrampolineUnchecked(
assert(kwargs == NULL || PyDict_CheckExact(kwargs));
if (!PyTuple_CheckExact(callargs)) {
if (check_args_iterable(tstate, func, callargs) < 0) {
- GOTO_ERROR(error);
+ goto error;
}
PyObject *tuple = PySequence_Tuple(callargs);
if (tuple == NULL) {
- GOTO_ERROR(error);
+ goto error;
}
Py_SETREF(callargs, tuple);
}
int err = _Py_call_instrumentation_2args(
tstate, PY_MONITORING_EVENT_CALL,
frame, this_instr, func, arg);
- if (err) GOTO_ERROR(error);
+ if (err) goto error;
result = PyObject_Call(func, callargs, kwargs);
if (!PyFunction_Check(func) && !PyMethod_Check(func)) {
if (result == NULL) {
// Need to manually shrink the stack since we exit with DISPATCH_INLINED.
STACK_SHRINK(oparg + 3);
if (new_frame == NULL) {
- GOTO_ERROR(error);
+ goto error;
}
assert(next_instr - this_instr == 1);
frame->return_offset = 1;
PyObject *inst = args[0];
int retval = PyObject_IsInstance(inst, cls);
if (retval < 0) {
- GOTO_ERROR(error);
+ goto error;
}
res = PyBool_FromLong(retval);
assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL));
// The frame has stolen all the arguments from the stack,
// so there is no need to clean them up.
if (new_frame == NULL) {
- GOTO_ERROR(error);
+ goto error;
}
assert(next_instr - this_instr == 1);
frame->return_offset = 1;
PyObject *arg = args[0];
Py_ssize_t len_i = PyObject_Length(arg);
if (len_i < 0) {
- GOTO_ERROR(error);
+ goto error;
}
res = PyLong_FromSsize_t(len_i);
assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL));
// Fortunately we don't need its superpower.
if (oldobj == NULL) {
_PyEval_FormatExcUnbound(tstate, _PyFrame_GetCode(frame), oparg);
- GOTO_ERROR(error);
+ goto error;
}
PyCell_SET(cell, NULL);
Py_DECREF(oldobj);
next_instr += 1;
INSTRUCTION_STATS(DELETE_FAST);
PyObject *v = GETLOCAL(oparg);
- if (v == NULL) goto unbound_local_error;
+ if (v == NULL) {
+ _PyEval_FormatExcCheckArg(tstate, PyExc_UnboundLocalError,
+ UNBOUNDLOCAL_ERROR_MSG,
+ PyTuple_GetItem(_PyFrame_GetCode(frame)->co_localsplusnames, oparg)
+ );
+ if (1) goto error;
+ }
SETLOCAL(oparg, NULL);
DISPATCH();
}
int err = PyDict_Pop(GLOBALS(), name, NULL);
// Can't use ERROR_IF here.
if (err < 0) {
- GOTO_ERROR(error);
+ goto error;
}
if (err == 0) {
_PyEval_FormatExcCheckArg(tstate, PyExc_NameError,
NAME_ERROR_MSG, name);
- GOTO_ERROR(error);
+ goto error;
}
DISPATCH();
}
if (ns == NULL) {
_PyErr_Format(tstate, PyExc_SystemError,
"no locals when deleting %R", name);
- GOTO_ERROR(error);
+ goto error;
}
err = PyObject_DelItem(ns, name);
// Can't use ERROR_IF here.
_PyEval_FormatExcCheckArg(tstate, PyExc_NameError,
NAME_ERROR_MSG,
name);
- GOTO_ERROR(error);
+ goto error;
}
DISPATCH();
}
PyErr_Format(PyExc_TypeError,
"__init__() should return None, not '%.200s'",
Py_TYPE(should_be_none)->tp_name);
- GOTO_ERROR(error);
+ goto error;
}
stack_pointer += -1;
DISPATCH();
if (next == NULL) {
if (_PyErr_Occurred(tstate)) {
if (!_PyErr_ExceptionMatches(tstate, PyExc_StopIteration)) {
- GOTO_ERROR(error);
+ goto error;
}
monitor_raise(tstate, frame, this_instr);
_PyErr_Clear(tstate);
if (PyAsyncGen_CheckExact(aiter)) {
awaitable = type->tp_as_async->am_anext(aiter);
if (awaitable == NULL) {
- GOTO_ERROR(error);
+ goto error;
}
} else {
if (type->tp_as_async != NULL){
if (getter != NULL) {
next_iter = (*getter)(aiter);
if (next_iter == NULL) {
- GOTO_ERROR(error);
+ goto error;
}
}
else {
"'async for' requires an iterator with "
"__anext__ method, got %.100s",
type->tp_name);
- GOTO_ERROR(error);
+ goto error;
}
awaitable = _PyCoro_GetAwaitableIter(next_iter);
if (awaitable == NULL) {
"from __anext__: %.100s",
Py_TYPE(next_iter)->tp_name);
Py_DECREF(next_iter);
- GOTO_ERROR(error);
+ goto error;
} else {
Py_DECREF(next_iter);
}
_PyErr_SetString(tstate, PyExc_TypeError,
"cannot 'yield from' a coroutine object "
"in a non-coroutine generator");
- GOTO_ERROR(error);
+ goto error;
}
iter = iterable;
}
/* `iterable` is not a generator. */
iter = PyObject_GetIter(iterable);
if (iter == NULL) {
- GOTO_ERROR(error);
+ goto error;
}
Py_DECREF(iterable);
}
if (PyGen_Check(receiver)) {
PyErr_SetObject(PyExc_StopIteration, value);
if (monitor_stop_iteration(tstate, frame, this_instr)) {
- GOTO_ERROR(error);
+ goto error;
}
PyErr_SetRaisedException(NULL);
}
if (PyGen_Check(receiver) || PyCoro_CheckExact(receiver)) {
PyErr_SetObject(PyExc_StopIteration, value);
if (monitor_stop_iteration(tstate, frame, this_instr)) {
- GOTO_ERROR(error);
+ goto error;
}
PyErr_SetRaisedException(NULL);
}
else {
if (_PyErr_Occurred(tstate)) {
if (!_PyErr_ExceptionMatches(tstate, PyExc_StopIteration)) {
- GOTO_ERROR(error);
+ goto error;
}
monitor_raise(tstate, frame, this_instr);
_PyErr_Clear(tstate);
uintptr_t code_version = _PyFrame_GetCode(frame)->_co_instrumentation_version;
if (code_version != global_version) {
if (_Py_Instrument(_PyFrame_GetCode(frame), tstate->interp)) {
- GOTO_ERROR(error);
+ goto error;
}
next_instr = this_instr;
}
int err = _Py_call_instrumentation_arg(
tstate, PY_MONITORING_EVENT_PY_RETURN,
frame, this_instr, retval);
- if (err) GOTO_ERROR(error);
+ if (err) goto error;
Py_INCREF(retval);
assert(EMPTY());
_PyFrame_SetStackPointer(frame, stack_pointer);
int err = _Py_call_instrumentation_arg(
tstate, PY_MONITORING_EVENT_PY_RETURN,
frame, this_instr, retval);
- if (err) GOTO_ERROR(error);
+ if (err) goto error;
STACK_SHRINK(1);
assert(EMPTY());
_PyFrame_SetStackPointer(frame, stack_pointer);
int err = _Py_call_instrumentation_arg(
tstate, PY_MONITORING_EVENT_PY_YIELD,
frame, this_instr, retval);
- if (err) GOTO_ERROR(error);
+ if (err) goto error;
tstate->exc_info = gen->gi_exc_state.previous_item;
gen->gi_exc_state.previous_item = NULL;
_Py_LeaveRecursiveCallPy(tstate);
INSTRUCTION_STATS(LOAD_FAST_CHECK);
PyObject *value;
value = GETLOCAL(oparg);
- if (value == NULL) goto unbound_local_error;
+ if (value == NULL) {
+ _PyEval_FormatExcCheckArg(tstate, PyExc_UnboundLocalError,
+ UNBOUNDLOCAL_ERROR_MSG,
+ PyTuple_GetItem(_PyFrame_GetCode(frame)->co_localsplusnames, oparg)
+ );
+ if (1) goto error;
+ }
Py_INCREF(value);
stack_pointer[0] = value;
stack_pointer += 1;
assert(oparg >= 0 && oparg < _PyFrame_GetCode(frame)->co_nlocalsplus);
name = PyTuple_GET_ITEM(_PyFrame_GetCode(frame)->co_localsplusnames, oparg);
if (PyMapping_GetOptionalItem(class_dict, name, &value) < 0) {
- GOTO_ERROR(error);
+ goto error;
}
if (!value) {
PyObject *cell = GETLOCAL(oparg);
value = PyCell_GET(cell);
if (value == NULL) {
_PyEval_FormatExcUnbound(tstate, _PyFrame_GetCode(frame), oparg);
- GOTO_ERROR(error);
+ goto error;
}
Py_INCREF(value);
}
mod_or_class_dict = stack_pointer[-1];
PyObject *name = GETITEM(FRAME_CO_NAMES, oparg);
if (PyMapping_GetOptionalItem(mod_or_class_dict, name, &v) < 0) {
- GOTO_ERROR(error);
+ goto error;
}
if (v == NULL) {
if (PyDict_GetItemRef(GLOBALS(), name, &v) < 0) {
- GOTO_ERROR(error);
+ goto error;
}
if (v == NULL) {
if (PyMapping_GetOptionalItem(BUILTINS(), name, &v) < 0) {
- GOTO_ERROR(error);
+ goto error;
}
if (v == NULL) {
_PyEval_FormatExcCheckArg(
tstate, PyExc_NameError,
NAME_ERROR_MSG, name);
- GOTO_ERROR(error);
+ goto error;
}
}
}
}
PyObject *name = GETITEM(FRAME_CO_NAMES, oparg);
if (PyMapping_GetOptionalItem(mod_or_class_dict, name, &v) < 0) {
- GOTO_ERROR(error);
+ goto error;
}
if (v == NULL) {
if (PyDict_GetItemRef(GLOBALS(), name, &v) < 0) {
- GOTO_ERROR(error);
+ goto error;
}
if (v == NULL) {
if (PyMapping_GetOptionalItem(BUILTINS(), name, &v) < 0) {
- GOTO_ERROR(error);
+ goto error;
}
if (v == NULL) {
_PyEval_FormatExcCheckArg(
tstate, PyExc_NameError,
NAME_ERROR_MSG, name);
- GOTO_ERROR(error);
+ goto error;
}
}
}
PyObject *initial = GETLOCAL(oparg);
PyObject *cell = PyCell_New(initial);
if (cell == NULL) {
- GOTO_ERROR(error);
+ goto error;
}
SETLOCAL(oparg, cell);
DISPATCH();
PyFunction_New(codeobj, GLOBALS());
Py_DECREF(codeobj);
if (func_obj == NULL) {
- GOTO_ERROR(error);
+ goto error;
}
_PyFunction_SetVersion(
func_obj, ((PyCodeObject *)codeobj)->co_version);
else {
assert(PyLong_Check(lasti));
_PyErr_SetString(tstate, PyExc_SystemError, "lasti is not an int");
- GOTO_ERROR(error);
+ goto error;
}
}
assert(exc && PyExceptionInstance_Check(exc));
PyFunctionObject *func = (PyFunctionObject *)frame->f_funcobj;
PyGenObject *gen = (PyGenObject *)_Py_MakeCoro(func);
if (gen == NULL) {
- GOTO_ERROR(error);
+ goto error;
}
assert(EMPTY());
_PyFrame_SetStackPointer(frame, stack_pointer);
JUMPBY(oparg);
}
else {
- GOTO_ERROR(error);
+ goto error;
}
}
Py_DECREF(v);
_PyJIT_Compile(_PyExecutorObject *executor, const _PyUOpInstruction *trace, size_t length)
{
// Loop once to find the total compiled size:
- size_t code_size = 0;
- size_t data_size = 0;
+ uint32_t instruction_starts[UOP_MAX_TRACE_LENGTH];
+ uint32_t code_size = 0;
+ uint32_t data_size = 0;
for (size_t i = 0; i < length; i++) {
_PyUOpInstruction *instruction = (_PyUOpInstruction *)&trace[i];
const StencilGroup *group = &stencil_groups[instruction->opcode];
+ instruction_starts[i] = code_size;
code_size += group->code.body_size;
data_size += group->data.body_size;
}
// Loop again to emit the code:
unsigned char *code = memory;
unsigned char *data = memory + code_size;
- unsigned char *top = code;
- if (trace[0].opcode == _START_EXECUTOR) {
- // Don't want to execute this more than once:
- top += stencil_groups[_START_EXECUTOR].code.body_size;
- }
+ assert(trace[0].opcode == _START_EXECUTOR || trace[0].opcode == _COLD_EXIT);
for (size_t i = 0; i < length; i++) {
_PyUOpInstruction *instruction = (_PyUOpInstruction *)&trace[i];
const StencilGroup *group = &stencil_groups[instruction->opcode];
patches[HoleValue_EXECUTOR] = (uint64_t)executor;
patches[HoleValue_OPARG] = instruction->oparg;
patches[HoleValue_OPERAND] = instruction->operand;
- patches[HoleValue_TARGET] = instruction->target;
- patches[HoleValue_TOP] = (uint64_t)top;
+ switch (instruction->format) {
+ case UOP_FORMAT_TARGET:
+ patches[HoleValue_TARGET] = instruction->target;
+ break;
+ case UOP_FORMAT_EXIT:
+ assert(instruction->exit_index < executor->exit_count);
+ patches[HoleValue_EXIT_INDEX] = instruction->exit_index;
+ if (instruction->error_target < length) {
+ patches[HoleValue_ERROR_TARGET] = (uint64_t)memory + instruction_starts[instruction->error_target];
+ }
+ break;
+ case UOP_FORMAT_JUMP:
+ assert(instruction->jump_target < length);
+ patches[HoleValue_JUMP_TARGET] = (uint64_t)memory + instruction_starts[instruction->jump_target];
+ if (instruction->error_target < length) {
+ patches[HoleValue_ERROR_TARGET] = (uint64_t)memory + instruction_starts[instruction->error_target];
+ }
+ break;
+ default:
+ assert(0);
+ Py_FatalError("Illegal instruction format");
+ }
+ patches[HoleValue_TOP] = (uint64_t)memory + instruction_starts[1];
patches[HoleValue_ZERO] = 0;
emit(group, patches);
code += group->code.body_size;
}
static _PyExecutorObject *
-make_executor_from_uops(_PyUOpInstruction *buffer, const _PyBloomFilter *dependencies);
+make_executor_from_uops(_PyUOpInstruction *buffer, int length, const _PyBloomFilter *dependencies);
static int
init_cold_exit_executor(_PyExecutorObject *executor, int oparg);
+/* It is impossible for the number of exits to reach 1/4 of the total length,
+ * as the number of exits cannot reach 1/3 of the number of non-exits, due to
+ * the presence of CHECK_VALIDITY checks and instructions to produce the values
+ * being checked in exits. */
+#define COLD_EXIT_COUNT (UOP_MAX_TRACE_LENGTH/4)
+
static int cold_exits_initialized = 0;
-static _PyExecutorObject COLD_EXITS[UOP_MAX_TRACE_LENGTH] = { 0 };
+static _PyExecutorObject COLD_EXITS[COLD_EXIT_COUNT] = { 0 };
static const _PyBloomFilter EMPTY_FILTER = { 0 };
}
else if (cold_exits_initialized == 0) {
cold_exits_initialized = 1;
- for (int i = 0; i < UOP_MAX_TRACE_LENGTH; i++) {
+ for (int i = 0; i < COLD_EXIT_COUNT; i++) {
if (init_cold_exit_executor(&COLD_EXITS[i], i)) {
return NULL;
}
else {
printf("%s", name);
}
- printf(" (%d, target=%d, operand=%#" PRIx64 ")",
- uop->oparg,
- uop->target,
- (uint64_t)uop->operand);
+ switch(uop->format) {
+ case UOP_FORMAT_TARGET:
+ printf(" (%d, target=%d, operand=%#" PRIx64,
+ uop->oparg,
+ uop->target,
+ (uint64_t)uop->operand);
+ break;
+ case UOP_FORMAT_JUMP:
+ printf(" (%d, jump_target=%d, operand=%#" PRIx64,
+ uop->oparg,
+ uop->jump_target,
+ (uint64_t)uop->operand);
+ break;
+ case UOP_FORMAT_EXIT:
+ printf(" (%d, exit_index=%d, operand=%#" PRIx64,
+ uop->oparg,
+ uop->exit_index,
+ (uint64_t)uop->operand);
+ break;
+ default:
+ printf(" (%d, Unknown format)", uop->oparg);
+ }
+ if (_PyUop_Flags[uop->opcode] & HAS_ERROR_FLAG) {
+ printf(", error_target=%d", uop->error_target);
+ }
+
+ printf(")");
}
#endif
#endif
-// Beware: Macro arg order differs from struct member order
+static inline int
+add_to_trace(
+ _PyUOpInstruction *trace,
+ int trace_length,
+ uint16_t opcode,
+ uint16_t oparg,
+ uint64_t operand,
+ uint32_t target)
+{
+ trace[trace_length].opcode = opcode;
+ trace[trace_length].format = UOP_FORMAT_TARGET;
+ trace[trace_length].target = target;
+ trace[trace_length].oparg = oparg;
+ trace[trace_length].operand = operand;
+ return trace_length + 1;
+}
+
#ifdef Py_DEBUG
#define ADD_TO_TRACE(OPCODE, OPARG, OPERAND, TARGET) \
assert(trace_length < max_length); \
- trace[trace_length].opcode = (OPCODE); \
- trace[trace_length].oparg = (OPARG); \
- trace[trace_length].target = (TARGET); \
- trace[trace_length].operand = (OPERAND); \
+ trace_length = add_to_trace(trace, trace_length, (OPCODE), (OPARG), (OPERAND), (TARGET)); \
if (lltrace >= 2) { \
printf("%4d ADD_TO_TRACE: ", trace_length); \
- _PyUOpPrint(&trace[trace_length]); \
+ _PyUOpPrint(&trace[trace_length-1]); \
printf("\n"); \
- } \
- trace_length++;
+ }
#else
#define ADD_TO_TRACE(OPCODE, OPARG, OPERAND, TARGET) \
assert(trace_length < max_length); \
- trace[trace_length].opcode = (OPCODE); \
- trace[trace_length].oparg = (OPARG); \
- trace[trace_length].target = (TARGET); \
- trace[trace_length].operand = (OPERAND); \
- trace_length++;
+ trace_length = add_to_trace(trace, trace_length, (OPCODE), (OPARG), (OPERAND), (TARGET));
#endif
#define INSTR_IP(INSTR, CODE) \
if (trace_stack_depth >= TRACE_STACK_SIZE) { \
DPRINTF(2, "Trace stack overflow\n"); \
OPT_STAT_INC(trace_stack_overflow); \
- ADD_TO_TRACE(uop, oparg, operand, target); \
- ADD_TO_TRACE(_EXIT_TRACE, 0, 0, 0); \
+ trace_length = 0; \
goto done; \
} \
assert(func == NULL || func->func_code == (PyObject *)code); \
assert(func == NULL || func->func_code == (PyObject *)code); \
instr = trace_stack[trace_stack_depth].instr;
-/* Returns 1 on success,
+/* Returns the length of the trace on success,
* 0 if it failed to produce a worthwhile trace,
* and -1 on an error.
*/
_Py_BloomFilter_Add(dependencies, initial_code);
_Py_CODEUNIT *initial_instr = instr;
int trace_length = 0;
- int max_length = buffer_size;
+ // Leave space for possible trailing _EXIT_TRACE
+ int max_length = buffer_size-2;
struct {
PyFunctionObject *func;
PyCodeObject *code;
PyUnicode_AsUTF8(code->co_filename),
code->co_firstlineno,
2 * INSTR_IP(initial_instr, code));
+ ADD_TO_TRACE(_START_EXECUTOR, 0, (uintptr_t)instr, INSTR_IP(instr, code));
uint32_t target = 0;
top: // Jump here after _PUSH_FRAME or likely branches
for (;;) {
target = INSTR_IP(instr, code);
- RESERVE_RAW(2, "epilogue"); // Always need space for _SET_IP, _CHECK_VALIDITY and _EXIT_TRACE
+ RESERVE_RAW(2, "_CHECK_VALIDITY_AND_SET_IP");
ADD_TO_TRACE(_CHECK_VALIDITY_AND_SET_IP, 0, (uintptr_t)instr, target);
+ // Need space for _DEOPT
+ max_length--;
uint32_t opcode = instr->op.code;
uint32_t oparg = instr->op.arg;
continue;
}
else {
- if (OPCODE_HAS_DEOPT(opcode)) {
+ if (OPCODE_HAS_EXIT(opcode) || OPCODE_HAS_DEOPT(opcode)) {
opcode = _PyOpcode_Deopt[opcode];
}
+ assert(!OPCODE_HAS_EXIT(opcode));
assert(!OPCODE_HAS_DEOPT(opcode));
}
}
+ if (OPCODE_HAS_EXIT(opcode)) {
+ // Make space for exit code
+ max_length--;
+ }
+ if (OPCODE_HAS_ERROR(opcode)) {
+ // Make space for error code
+ max_length--;
+ }
switch (opcode) {
case POP_JUMP_IF_NONE:
case POP_JUMP_IF_NOT_NONE:
DPRINTF(2, "Jump likely (%04x = %d bits), continue at byte offset %d\n",
instr[1].cache, bitcount, 2 * INSTR_IP(target_instr, code));
instr = target_instr;
- ADD_TO_TRACE(uopcode, max_length, 0, INSTR_IP(next_instr, code));
+ ADD_TO_TRACE(uopcode, 0, 0, INSTR_IP(next_instr, code));
goto top;
}
- ADD_TO_TRACE(uopcode, max_length, 0, INSTR_IP(target_instr, code));
+ ADD_TO_TRACE(uopcode, 0, 0, INSTR_IP(target_instr, code));
break;
}
progress_needed ? "no progress" : "too short");
return 0;
}
- ADD_TO_TRACE(_EXIT_TRACE, 0, 0, target);
+ if (trace[trace_length-1].opcode != _JUMP_TO_TOP) {
+ ADD_TO_TRACE(_EXIT_TRACE, 0, 0, target);
+ }
DPRINTF(1,
"Created a proto-trace for %s (%s:%d) at byte offset %d -- length %d\n",
PyUnicode_AsUTF8(code->co_qualname),
code->co_firstlineno,
2 * INSTR_IP(initial_instr, code),
trace_length);
- OPT_HIST(trace_length + buffer_size - max_length, trace_length_hist);
- return 1;
+ OPT_HIST(trace_length, trace_length_hist);
+ return trace_length;
}
#undef RESERVE
#define SET_BIT(array, bit) (array[(bit)>>5] |= (1<<((bit)&31)))
#define BIT_IS_SET(array, bit) (array[(bit)>>5] & (1<<((bit)&31)))
-/* Count the number of used uops, and mark them in the bit vector `used`.
- * This can be done in a single pass using simple reachability analysis,
- * as there are no backward jumps.
- * NOPs are excluded from the count.
+/* Count the number of unused uops and exits
*/
static int
-compute_used(_PyUOpInstruction *buffer, uint32_t *used, int *exit_count_ptr)
+count_exits(_PyUOpInstruction *buffer, int length)
{
- int count = 0;
int exit_count = 0;
- SET_BIT(used, 0);
- for (int i = 0; i < UOP_MAX_TRACE_LENGTH; i++) {
- if (!BIT_IS_SET(used, i)) {
- continue;
- }
- count++;
+ for (int i = 0; i < length; i++) {
int opcode = buffer[i].opcode;
- if (_PyUop_Flags[opcode] & HAS_EXIT_FLAG) {
+ if (opcode == _SIDE_EXIT) {
exit_count++;
}
- if (opcode == _JUMP_TO_TOP || opcode == _EXIT_TRACE) {
- continue;
+ }
+ return exit_count;
+}
+
+static void make_exit(_PyUOpInstruction *inst, int opcode, int target)
+{
+ inst->opcode = opcode;
+ inst->oparg = 0;
+ inst->format = UOP_FORMAT_TARGET;
+ inst->target = target;
+}
+
+/* Convert implicit exits, errors and deopts
+ * into explicit ones. */
+static int
+prepare_for_execution(_PyUOpInstruction *buffer, int length)
+{
+ int32_t current_jump = -1;
+ int32_t current_jump_target = -1;
+ int32_t current_error = -1;
+ int32_t current_error_target = -1;
+ int32_t current_popped = -1;
+ /* Leaving in NOPs slows down the interpreter and messes up the stats */
+ _PyUOpInstruction *copy_to = &buffer[0];
+ for (int i = 0; i < length; i++) {
+ _PyUOpInstruction *inst = &buffer[i];
+ if (inst->opcode != _NOP) {
+ if (copy_to != inst) {
+ *copy_to = *inst;
+ }
+ copy_to++;
}
- /* All other micro-ops fall through, so i+1 is reachable */
- SET_BIT(used, i+1);
- assert(opcode <= MAX_UOP_ID);
- if (_PyUop_Flags[opcode] & HAS_JUMP_FLAG) {
- /* Mark target as reachable */
- SET_BIT(used, buffer[i].oparg);
+ }
+ length = (int)(copy_to - buffer);
+ int next_spare = length;
+ for (int i = 0; i < length; i++) {
+ _PyUOpInstruction *inst = &buffer[i];
+ int opcode = inst->opcode;
+ int32_t target = (int32_t)uop_get_target(inst);
+ if (_PyUop_Flags[opcode] & (HAS_EXIT_FLAG | HAS_DEOPT_FLAG)) {
+ if (target != current_jump_target) {
+ uint16_t exit_op = (_PyUop_Flags[opcode] & HAS_EXIT_FLAG) ? _SIDE_EXIT : _DEOPT;
+ make_exit(&buffer[next_spare], exit_op, target);
+ current_jump_target = target;
+ current_jump = next_spare;
+ next_spare++;
+ }
+ buffer[i].jump_target = current_jump;
+ buffer[i].format = UOP_FORMAT_JUMP;
}
- if (opcode == NOP) {
- count--;
- UNSET_BIT(used, i);
+ if (_PyUop_Flags[opcode] & HAS_ERROR_FLAG) {
+ int popped = (_PyUop_Flags[opcode] & HAS_ERROR_NO_POP_FLAG) ?
+ 0 : _PyUop_num_popped(opcode, inst->oparg);
+ if (target != current_error_target || popped != current_popped) {
+ current_popped = popped;
+ current_error = next_spare;
+ current_error_target = target;
+ make_exit(&buffer[next_spare], _ERROR_POP_N, 0);
+ buffer[next_spare].oparg = popped;
+ next_spare++;
+ }
+ buffer[i].error_target = current_error;
+ if (buffer[i].format == UOP_FORMAT_TARGET) {
+ buffer[i].format = UOP_FORMAT_JUMP;
+ buffer[i].jump_target = 0;
+ }
}
}
- *exit_count_ptr = exit_count;
- return count;
+ return next_spare;
}
/* Executor side exits */
return res;
}
+#ifdef Py_DEBUG
+
+#define CHECK(PRED) \
+if (!(PRED)) { \
+ printf(#PRED " at %d\n", i); \
+ assert(0); \
+}
+
+static int
+target_unused(int opcode)
+{
+ return (_PyUop_Flags[opcode] & (HAS_ERROR_FLAG | HAS_EXIT_FLAG | HAS_DEOPT_FLAG)) == 0;
+}
+
+static void
+sanity_check(_PyExecutorObject *executor)
+{
+ for (uint32_t i = 0; i < executor->exit_count; i++) {
+ _PyExitData *exit = &executor->exits[i];
+ CHECK(exit->target < (1 << 25));
+ }
+ bool ended = false;
+ uint32_t i = 0;
+ CHECK(executor->trace[0].opcode == _START_EXECUTOR || executor->trace[0].opcode == _COLD_EXIT);
+ for (; i < executor->code_size; i++) {
+ const _PyUOpInstruction *inst = &executor->trace[i];
+ uint16_t opcode = inst->opcode;
+ CHECK(opcode <= MAX_UOP_ID);
+ CHECK(_PyOpcode_uop_name[opcode] != NULL);
+ switch(inst->format) {
+ case UOP_FORMAT_TARGET:
+ CHECK(target_unused(opcode));
+ break;
+ case UOP_FORMAT_EXIT:
+ CHECK(opcode == _SIDE_EXIT);
+ CHECK(inst->exit_index < executor->exit_count);
+ break;
+ case UOP_FORMAT_JUMP:
+ CHECK(inst->jump_target < executor->code_size);
+ break;
+ case UOP_FORMAT_UNUSED:
+ CHECK(0);
+ break;
+ }
+ if (_PyUop_Flags[opcode] & HAS_ERROR_FLAG) {
+ CHECK(inst->format == UOP_FORMAT_JUMP);
+ CHECK(inst->error_target < executor->code_size);
+ }
+ if (opcode == _JUMP_TO_TOP || opcode == _EXIT_TRACE || opcode == _COLD_EXIT) {
+ ended = true;
+ i++;
+ break;
+ }
+ }
+ CHECK(ended);
+ for (; i < executor->code_size; i++) {
+ const _PyUOpInstruction *inst = &executor->trace[i];
+ uint16_t opcode = inst->opcode;
+ CHECK(
+ opcode == _DEOPT ||
+ opcode == _SIDE_EXIT ||
+ opcode == _ERROR_POP_N);
+ if (opcode == _SIDE_EXIT) {
+ CHECK(inst->format == UOP_FORMAT_EXIT);
+ }
+ }
+}
+
+#undef CHECK
+#endif
+
/* Makes an executor from a buffer of uops.
* Account for the buffer having gaps and NOPs by computing a "used"
* bit vector and only copying the used uops. Here "used" means reachable
* and not a NOP.
*/
static _PyExecutorObject *
-make_executor_from_uops(_PyUOpInstruction *buffer, const _PyBloomFilter *dependencies)
+make_executor_from_uops(_PyUOpInstruction *buffer, int length, const _PyBloomFilter *dependencies)
{
- uint32_t used[(UOP_MAX_TRACE_LENGTH + 31)/32] = { 0 };
- int exit_count;
- int length = compute_used(buffer, used, &exit_count);
- length += 1; // For _START_EXECUTOR
+ int exit_count = count_exits(buffer, length);
_PyExecutorObject *executor = allocate_executor(exit_count, length);
if (executor == NULL) {
return NULL;
}
- OPT_HIST(length, optimized_trace_length_hist);
/* Initialize exits */
+ assert(exit_count < COLD_EXIT_COUNT);
for (int i = 0; i < exit_count; i++) {
executor->exits[i].executor = &COLD_EXITS[i];
executor->exits[i].temperature = 0;
}
int next_exit = exit_count-1;
- _PyUOpInstruction *dest = (_PyUOpInstruction *)&executor->trace[length-1];
- /* Scan backwards, so that we see the destinations of jumps before the jumps themselves. */
- for (int i = UOP_MAX_TRACE_LENGTH-1; i >= 0; i--) {
- if (!BIT_IS_SET(used, i)) {
- continue;
- }
- *dest = buffer[i];
+ _PyUOpInstruction *dest = (_PyUOpInstruction *)&executor->trace[length];
+ assert(buffer[0].opcode == _START_EXECUTOR);
+ buffer[0].operand = (uint64_t)executor;
+ for (int i = length-1; i >= 0; i--) {
int opcode = buffer[i].opcode;
- if (opcode == _POP_JUMP_IF_FALSE ||
- opcode == _POP_JUMP_IF_TRUE)
- {
- /* The oparg of the target will already have been set to its new offset */
- int oparg = dest->oparg;
- dest->oparg = buffer[oparg].oparg;
- }
- if (_PyUop_Flags[opcode] & HAS_EXIT_FLAG) {
+ dest--;
+ *dest = buffer[i];
+ assert(opcode != _POP_JUMP_IF_FALSE && opcode != _POP_JUMP_IF_TRUE);
+ if (opcode == _SIDE_EXIT) {
executor->exits[next_exit].target = buffer[i].target;
dest->exit_index = next_exit;
+ dest->format = UOP_FORMAT_EXIT;
next_exit--;
}
- /* Set the oparg to be the destination offset,
- * so that we can set the oparg of earlier jumps correctly. */
- buffer[i].oparg = (uint16_t)(dest - executor->trace);
- dest--;
}
assert(next_exit == -1);
assert(dest == executor->trace);
- dest->opcode = _START_EXECUTOR;
+ assert(dest->opcode == _START_EXECUTOR);
dest->oparg = 0;
dest->target = 0;
- dest->operand = (uintptr_t)executor;
_Py_ExecutorInit(executor, dependencies);
#ifdef Py_DEBUG
char *python_lltrace = Py_GETENV("PYTHON_LLTRACE");
printf("\n");
}
}
+ sanity_check(executor);
#endif
#ifdef _Py_JIT
executor->jit_code = NULL;
for (int i = 0; i < BLOOM_FILTER_WORDS; i++) {
assert(executor->vm_data.bloom.bits[i] == 0);
}
+#ifdef Py_DEBUG
+ sanity_check(executor);
+#endif
#ifdef _Py_JIT
executor->jit_code = NULL;
executor->jit_size = 0;
return 0;
}
+#ifdef Py_STATS
+/* Returns the effective trace length.
+ * Ignores NOPs and trailing exit and error handling.*/
+int effective_trace_length(_PyUOpInstruction *buffer, int length)
+{
+ int nop_count = 0;
+ for (int i = 0; i < length; i++) {
+ int opcode = buffer[i].opcode;
+ if (opcode == _NOP) {
+ nop_count++;
+ }
+ if (opcode == _EXIT_TRACE ||
+ opcode == _JUMP_TO_TOP ||
+ opcode == _COLD_EXIT) {
+ return i+1-nop_count;
+ }
+ }
+ Py_FatalError("No terminating instruction");
+ Py_UNREACHABLE();
+}
+#endif
+
static int
uop_optimize(
_PyOptimizerObject *self,
_Py_BloomFilter_Init(&dependencies);
_PyUOpInstruction buffer[UOP_MAX_TRACE_LENGTH];
OPT_STAT_INC(attempts);
- int err = translate_bytecode_to_trace(frame, instr, buffer, UOP_MAX_TRACE_LENGTH, &dependencies);
- if (err <= 0) {
+ int length = translate_bytecode_to_trace(frame, instr, buffer, UOP_MAX_TRACE_LENGTH, &dependencies);
+ if (length <= 0) {
// Error or nothing translated
- return err;
+ return length;
}
+ assert(length < UOP_MAX_TRACE_LENGTH);
OPT_STAT_INC(traces_created);
char *env_var = Py_GETENV("PYTHON_UOPS_OPTIMIZE");
if (env_var == NULL || *env_var == '\0' || *env_var > '0') {
- err = _Py_uop_analyze_and_optimize(frame, buffer,
- UOP_MAX_TRACE_LENGTH,
+ length = _Py_uop_analyze_and_optimize(frame, buffer,
+ length,
curr_stackentries, &dependencies);
- if (err <= 0) {
- return err;
+ if (length <= 0) {
+ return length;
}
}
- assert(err == 1);
+ assert(length < UOP_MAX_TRACE_LENGTH);
+ assert(length >= 1);
/* Fix up */
- for (int pc = 0; pc < UOP_MAX_TRACE_LENGTH; pc++) {
+ for (int pc = 0; pc < length; pc++) {
int opcode = buffer[pc].opcode;
int oparg = buffer[pc].oparg;
if (_PyUop_Flags[opcode] & HAS_OPARG_AND_1_FLAG) {
assert(_PyOpcode_uop_name[buffer[pc].opcode]);
assert(strncmp(_PyOpcode_uop_name[buffer[pc].opcode], _PyOpcode_uop_name[opcode], strlen(_PyOpcode_uop_name[opcode])) == 0);
}
- _PyExecutorObject *executor = make_executor_from_uops(buffer, &dependencies);
+ OPT_HIST(effective_trace_length(buffer, length), optimized_trace_length_hist);
+ length = prepare_for_execution(buffer, length);
+ assert(length <= UOP_MAX_TRACE_LENGTH);
+ _PyExecutorObject *executor = make_executor_from_uops(buffer, length, &dependencies);
if (executor == NULL) {
return -1;
}
+ assert(length <= UOP_MAX_TRACE_LENGTH);
*exec_ptr = executor;
return 1;
}
return 0;
}
_Py_CODEUNIT *target = instr + 1 + _PyOpcode_Caches[JUMP_BACKWARD] - oparg;
- _PyUOpInstruction buffer[3] = {
+ _PyUOpInstruction buffer[5] = {
+ { .opcode = _START_EXECUTOR },
{ .opcode = _LOAD_CONST_INLINE_BORROW, .operand = (uintptr_t)self },
{ .opcode = _INTERNAL_INCREMENT_OPT_COUNTER },
- { .opcode = _EXIT_TRACE, .target = (uint32_t)(target - _PyCode_CODE(code)) }
+ { .opcode = _EXIT_TRACE, .jump_target = 4, .format=UOP_FORMAT_JUMP },
+ { .opcode = _SIDE_EXIT, .target = (uint32_t)(target - _PyCode_CODE(code)), .format=UOP_FORMAT_TARGET }
};
- _PyExecutorObject *executor = make_executor_from_uops(buffer, &EMPTY_FILTER);
+ _PyExecutorObject *executor = make_executor_from_uops(buffer, 5, &EMPTY_FILTER);
if (executor == NULL) {
return -1;
}
ctx->curr_frame_depth++;
ctx->frame = frame;
- for (_PyUOpInstruction *this_instr = trace;
- this_instr < trace + trace_len && !op_is_end(this_instr->opcode);
- this_instr++) {
+ _PyUOpInstruction *this_instr = NULL;
+ for (int i = 0; i < trace_len; i++) {
+ this_instr = &trace[i];
int oparg = this_instr->oparg;
opcode = this_instr->opcode;
ctx->frame->stack_pointer = stack_pointer;
assert(STACK_LEVEL() >= 0);
}
-
_Py_uop_abstractcontext_fini(ctx);
- return 1;
+ return trace_len;
out_of_space:
DPRINTF(3, "\n");
/* Cannot optimize further, but there would be no benefit
* in retrying later */
_Py_uop_abstractcontext_fini(ctx);
- return 1;
+ return trace_len;
}
-static void
+static int
remove_unneeded_uops(_PyUOpInstruction *buffer, int buffer_size)
{
/* Remove _SET_IP and _CHECK_VALIDITY where possible.
}
case _JUMP_TO_TOP:
case _EXIT_TRACE:
- return;
+ return pc + 1;
default:
{
bool needs_ip = false;
}
}
}
+ Py_FatalError("No terminating instruction");
+ Py_UNREACHABLE();
}
static void
// 0 - failure, no error raised, just fall back to Tier 1
// -1 - failure, and raise error
-// 1 - optimizer success
+// > 0 - length of optimized trace
int
_Py_uop_analyze_and_optimize(
_PyInterpreterFrame *frame,
_PyUOpInstruction *buffer,
- int buffer_size,
+ int length,
int curr_stacklen,
_PyBloomFilter *dependencies
)
{
OPT_STAT_INC(optimizer_attempts);
- int err = remove_globals(frame, buffer, buffer_size, dependencies);
- if (err == 0) {
- goto not_ready;
- }
- if (err < 0) {
- goto error;
+ int err = remove_globals(frame, buffer, length, dependencies);
+ if (err <= 0) {
+ return err;
}
- peephole_opt(frame, buffer, buffer_size);
+ peephole_opt(frame, buffer, length);
- err = optimize_uops(
+ length = optimize_uops(
_PyFrame_GetCode(frame), buffer,
- buffer_size, curr_stacklen, dependencies);
+ length, curr_stacklen, dependencies);
- if (err == 0) {
- goto not_ready;
+ if (length <= 0) {
+ return length;
}
- assert(err == 1);
- remove_unneeded_uops(buffer, buffer_size);
+ length = remove_unneeded_uops(buffer, length);
+ assert(length > 0);
OPT_STAT_INC(optimizer_successes);
- return 1;
-not_ready:
- return 0;
-error:
- return -1;
+ return length;
}
break;
}
- case _LOAD_NAME: {
- _Py_UopsSymbol *v;
- v = sym_new_not_null(ctx);
- if (v == NULL) goto out_of_space;
- stack_pointer[0] = v;
- stack_pointer += 1;
- break;
- }
+ /* _LOAD_NAME is not a viable micro-op for tier 2 */
case _LOAD_GLOBAL: {
_Py_UopsSymbol *res;
break;
}
- case _BUILD_SET: {
- _Py_UopsSymbol *set;
- set = sym_new_not_null(ctx);
- if (set == NULL) goto out_of_space;
- stack_pointer[-oparg] = set;
- stack_pointer += 1 - oparg;
- break;
- }
+ /* _BUILD_SET is not a viable micro-op for tier 2 */
case _BUILD_MAP: {
_Py_UopsSymbol *map;
/* _FOR_ITER_GEN is not a viable micro-op for tier 2 */
- case _BEFORE_ASYNC_WITH: {
- _Py_UopsSymbol *exit;
- _Py_UopsSymbol *res;
- exit = sym_new_not_null(ctx);
- if (exit == NULL) goto out_of_space;
- res = sym_new_not_null(ctx);
- if (res == NULL) goto out_of_space;
- stack_pointer[-1] = exit;
- stack_pointer[0] = res;
- stack_pointer += 1;
- break;
- }
+ /* _BEFORE_ASYNC_WITH is not a viable micro-op for tier 2 */
- case _BEFORE_WITH: {
- _Py_UopsSymbol *exit;
- _Py_UopsSymbol *res;
- exit = sym_new_not_null(ctx);
- if (exit == NULL) goto out_of_space;
- res = sym_new_not_null(ctx);
- if (res == NULL) goto out_of_space;
- stack_pointer[-1] = exit;
- stack_pointer[0] = res;
- stack_pointer += 1;
- break;
- }
+ /* _BEFORE_WITH is not a viable micro-op for tier 2 */
case _WITH_EXCEPT_START: {
_Py_UopsSymbol *res;
break;
}
+ case _DEOPT: {
+ break;
+ }
+
+ case _SIDE_EXIT: {
+ break;
+ }
+
+ case _ERROR_POP_N: {
+ stack_pointer += -oparg;
+ break;
+ }
+
@dataclass
class Properties:
escapes: bool
- infallible: bool
+ error_with_pop: bool
+ error_without_pop: bool
deopts: bool
oparg: bool
jumps: bool
def from_list(properties: list["Properties"]) -> "Properties":
return Properties(
escapes=any(p.escapes for p in properties),
- infallible=all(p.infallible for p in properties),
+ error_with_pop=any(p.error_with_pop for p in properties),
+ error_without_pop=any(p.error_without_pop for p in properties),
deopts=any(p.deopts for p in properties),
oparg=any(p.oparg for p in properties),
jumps=any(p.jumps for p in properties),
passthrough=all(p.passthrough for p in properties),
)
+ @property
+ def infallible(self) -> bool:
+ return not self.error_with_pop and not self.error_without_pop
+
+
SKIP_PROPERTIES = Properties(
escapes=False,
- infallible=True,
+ error_with_pop=False,
+ error_without_pop=False,
deopts=False,
oparg=False,
jumps=False,
self._size = sum(c.size for c in self.caches)
return self._size
- def is_viable(self) -> bool:
+ def why_not_viable(self) -> str | None:
if self.name == "_SAVE_RETURN_OFFSET":
- return True # Adjusts next_instr, but only in tier 1 code
- if self.properties.needs_this:
- return False
+ return None # Adjusts next_instr, but only in tier 1 code
if "INSTRUMENTED" in self.name:
- return False
+ return "is instrumented"
if "replaced" in self.annotations:
- return False
+ return "is replaced"
if self.name in ("INTERPRETER_EXIT", "JUMP_BACKWARD"):
- return False
+ return "has tier 1 control flow"
+ if self.properties.needs_this:
+ return "uses the 'this_instr' variable"
if len([c for c in self.caches if c.name != "unused"]) > 1:
- return False
- return True
+ return "has unused cache entries"
+ if self.properties.error_with_pop and self.properties.error_without_pop:
+ return "has both popping and not-popping errors"
+ if self.properties.eval_breaker:
+ if self.properties.error_with_pop or self.properties.error_without_pop:
+ return "has error handling and eval-breaker check"
+ if self.properties.side_exit:
+ return "exits and eval-breaker check"
+ if self.properties.deopts:
+ return "deopts and eval-breaker check"
+ return None
+
+ def is_viable(self) -> bool:
+ return self.why_not_viable() is None
def is_super(self) -> bool:
for tkn in self.body:
return int(token.text[-1])
return None
-def is_infallible(op: parser.InstDef) -> bool:
- return not (
+def has_error_with_pop(op: parser.InstDef) -> bool:
+ return (
variable_used(op, "ERROR_IF")
- or variable_used(op, "error")
+ or variable_used(op, "pop_1_error")
+ or variable_used(op, "exception_unwind")
+ or variable_used(op, "resume_with_error")
+ )
+
+def has_error_without_pop(op: parser.InstDef) -> bool:
+ return (
+ variable_used(op, "ERROR_NO_POP")
or variable_used(op, "pop_1_error")
or variable_used(op, "exception_unwind")
or variable_used(op, "resume_with_error")
tkn.column,
op.name,
)
- infallible = is_infallible(op)
+ error_with_pop = has_error_with_pop(op)
+ error_without_pop = has_error_without_pop(op)
+ infallible = not error_with_pop and not error_without_pop
passthrough = stack_effect_only_peeks(op) and infallible
return Properties(
escapes=makes_escaping_api_call(op),
- infallible=infallible,
- deopts=deopts_if or exits_if,
+ error_with_pop=error_with_pop,
+ error_without_pop=error_without_pop,
+ deopts=deopts_if,
side_exit=exits_if,
oparg=variable_used(op, "oparg"),
jumps=variable_used(op, "JUMPBY"),
out.emit(close)
+def replace_error_no_pop(
+ out: CWriter,
+ tkn: Token,
+ tkn_iter: Iterator[Token],
+ uop: Uop,
+ stack: Stack,
+ inst: Instruction | None,
+) -> None:
+ next(tkn_iter) # LPAREN
+ next(tkn_iter) # RPAREN
+ next(tkn_iter) # Semi colon
+ out.emit_at("goto error;", tkn)
+
+
def replace_decrefs(
out: CWriter,
tkn: Token,
"EXIT_IF": replace_deopt,
"DEOPT_IF": replace_deopt,
"ERROR_IF": replace_error,
+ "ERROR_NO_POP": replace_error_no_pop,
"DECREF_INPUTS": replace_decrefs,
"CHECK_EVAL_BREAKER": replace_check_eval_breaker,
"SYNC_SP": replace_sync_sp,
flags.append("HAS_EXIT_FLAG")
if not p.infallible:
flags.append("HAS_ERROR_FLAG")
+ if p.error_without_pop:
+ flags.append("HAS_ERROR_NO_POP_FLAG")
if p.escapes:
flags.append("HAS_ESCAPES_FLAG")
if p.pure:
"PURE",
"PASSTHROUGH",
"OPARG_AND_1",
+ "ERROR_NO_POP",
]
label = next(tkn_iter).text
next(tkn_iter) # RPAREN
next(tkn_iter) # Semi colon
- out.emit(") ")
- c_offset = stack.peek_offset.to_c()
- try:
- offset = -int(c_offset)
- close = ";\n"
- except ValueError:
- offset = None
- out.emit(f"{{ stack_pointer += {c_offset}; ")
- close = "; }\n"
- out.emit("goto ")
- if offset:
- out.emit(f"pop_{offset}_")
- out.emit(label + "_tier_two")
- out.emit(close)
+ out.emit(") JUMP_TO_ERROR();\n")
+
+def tier2_replace_error_no_pop(
+ out: CWriter,
+ tkn: Token,
+ tkn_iter: Iterator[Token],
+ uop: Uop,
+ stack: Stack,
+ inst: Instruction | None,
+) -> None:
+ next(tkn_iter) # LPAREN
+ next(tkn_iter) # RPAREN
+ next(tkn_iter) # Semi colon
+ out.emit_at("JUMP_TO_ERROR();", tkn)
def tier2_replace_deopt(
out: CWriter,
out.emit(next(tkn_iter))
emit_to(out, tkn_iter, "RPAREN")
next(tkn_iter) # Semi colon
- out.emit(") goto deoptimize;\n")
+ out.emit(") JUMP_TO_JUMP_TARGET();\n")
def tier2_replace_exit_if(
out.emit(next(tkn_iter))
emit_to(out, tkn_iter, "RPAREN")
next(tkn_iter) # Semi colon
- out.emit(") goto side_exit;\n")
+ out.emit(") JUMP_TO_JUMP_TARGET();\n")
def tier2_replace_oparg(
TIER2_REPLACEMENT_FUNCTIONS = REPLACEMENT_FUNCTIONS.copy()
TIER2_REPLACEMENT_FUNCTIONS["ERROR_IF"] = tier2_replace_error
+TIER2_REPLACEMENT_FUNCTIONS["ERROR_NO_POP"] = tier2_replace_error_no_pop
TIER2_REPLACEMENT_FUNCTIONS["DEOPT_IF"] = tier2_replace_deopt
TIER2_REPLACEMENT_FUNCTIONS["oparg"] = tier2_replace_oparg
TIER2_REPLACEMENT_FUNCTIONS["EXIT_IF"] = tier2_replace_exit_if
continue
if uop.is_super():
continue
- if not uop.is_viable():
- out.emit(f"/* {uop.name} is not a viable micro-op for tier 2 */\n\n")
+ why_not_viable = uop.why_not_viable()
+ if why_not_viable is not None:
+ out.emit(f"/* {uop.name} is not a viable micro-op for tier 2 because it {why_not_viable} */\n\n")
continue
out.emit(f"case {uop.name}: {{\n")
declare_variables(uop, out)
write_header,
cflags,
)
+from stack import Stack
from cwriter import CWriter
from typing import TextIO
-
DEFAULT_OUTPUT = ROOT / "Include/internal/pycore_uop_metadata.h"
out.emit("extern const uint16_t _PyUop_Flags[MAX_UOP_ID+1];\n")
out.emit("extern const uint8_t _PyUop_Replication[MAX_UOP_ID+1];\n")
out.emit("extern const char * const _PyOpcode_uop_name[MAX_UOP_ID+1];\n\n")
+ out.emit("extern int _PyUop_num_popped(int opcode, int oparg);\n\n")
out.emit("#ifdef NEED_OPCODE_METADATA\n")
out.emit("const uint16_t _PyUop_Flags[MAX_UOP_ID+1] = {\n")
for uop in analysis.uops.values():
if uop.is_viable() and uop.properties.tier != 1:
out.emit(f'[{uop.name}] = "{uop.name}",\n')
out.emit("};\n")
+ out.emit("int _PyUop_num_popped(int opcode, int oparg)\n{\n")
+ out.emit("switch(opcode) {\n")
+ for uop in analysis.uops.values():
+ if uop.is_viable() and uop.properties.tier != 1:
+ stack = Stack()
+ for var in reversed(uop.stack.inputs):
+ stack.pop(var)
+ popped = (-stack.base_offset).to_c()
+ out.emit(f"case {uop.name}:\n")
+ out.emit(f" return {popped};\n")
+ out.emit("default:\n")
+ out.emit(" return -1;\n")
+ out.emit("}\n")
+ out.emit("}\n\n")
out.emit("#endif // NEED_OPCODE_METADATA\n\n")
OPERAND = enum.auto()
# The current uop's target (exposed as _JIT_TARGET):
TARGET = enum.auto()
+ # The base address of the machine code for the jump target (exposed as _JIT_JUMP_TARGET):
+ JUMP_TARGET = enum.auto()
+ # The base address of the machine code for the error jump target (exposed as _JIT_ERROR_TARGET):
+ ERROR_TARGET = enum.auto()
+ # The index of the exit to be jumped through (exposed as _JIT_EXIT_INDEX):
+ EXIT_INDEX = enum.auto()
# The base address of the machine code for the first uop (exposed as _JIT_TOP):
TOP = enum.auto()
# A hardcoded value of zero (used for symbol lookups):
TYPE NAME = (TYPE)(uint64_t)&ALIAS;
#define PATCH_JUMP(ALIAS) \
+do { \
PyAPI_DATA(void) ALIAS; \
__attribute__((musttail)) \
- return ((jit_func)&ALIAS)(frame, stack_pointer, tstate);
+ return ((jit_func)&ALIAS)(frame, stack_pointer, tstate); \
+} while (0)
+
+#undef JUMP_TO_JUMP_TARGET
+#define JUMP_TO_JUMP_TARGET() PATCH_JUMP(_JIT_JUMP_TARGET)
+
+#undef JUMP_TO_ERROR
+#define JUMP_TO_ERROR() PATCH_JUMP(_JIT_ERROR_TARGET)
_Py_CODEUNIT *
_JIT_ENTRY(_PyInterpreterFrame *frame, PyObject **stack_pointer, PyThreadState *tstate)
PATCH_VALUE(uint16_t, _oparg, _JIT_OPARG)
PATCH_VALUE(uint64_t, _operand, _JIT_OPERAND)
PATCH_VALUE(uint32_t, _target, _JIT_TARGET)
+ PATCH_VALUE(uint16_t, _exit_index, _JIT_EXIT_INDEX)
// The actual instruction definitions (only one will be used):
if (opcode == _JUMP_TO_TOP) {
CHECK_EVAL_BREAKER();
}
PATCH_JUMP(_JIT_CONTINUE);
// Labels that the instruction implementations expect to exist:
-unbound_local_error_tier_two:
- _PyEval_FormatExcCheckArg(
- tstate, PyExc_UnboundLocalError, UNBOUNDLOCAL_ERROR_MSG,
- PyTuple_GetItem(_PyFrame_GetCode(frame)->co_localsplusnames, oparg));
- goto error_tier_two;
-pop_4_error_tier_two:
- STACK_SHRINK(1);
-pop_3_error_tier_two:
- STACK_SHRINK(1);
-pop_2_error_tier_two:
- STACK_SHRINK(1);
-pop_1_error_tier_two:
- STACK_SHRINK(1);
+
error_tier_two:
tstate->previous_executor = (PyObject *)current_executor;
GOTO_TIER_ONE(NULL);
-deoptimize:
+exit_to_tier1:
tstate->previous_executor = (PyObject *)current_executor;
GOTO_TIER_ONE(_PyCode_CODE(_PyFrame_GetCode(frame)) + _target);
-side_exit:
+exit_to_trace:
{
- _PyExitData *exit = ¤t_executor->exits[_target];
+ _PyExitData *exit = ¤t_executor->exits[_exit_index];
Py_INCREF(exit->executor);
tstate->previous_executor = (PyObject *)current_executor;
GOTO_TIER_TWO(exit->executor);
projects / thirdparty / Python / cpython.git / commitdiff
