From: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Thu, 23 Oct 2025 23:13:48 +0000 (+0100) Subject: Rewrite the tracing JIT to use a common opcode handler X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=cedd7af91afc1b1f43d32b3419dfc6a70a75fd43;p=thirdparty%2FPython%2Fcpython.git Rewrite the tracing JIT to use a common opcode handler --- diff --git a/Include/internal/pycore_interp_structs.h b/Include/internal/pycore_interp_structs.h index ac1bcdf74ae4..c2cddfe47d3f 100644 --- a/Include/internal/pycore_interp_structs.h +++ b/Include/internal/pycore_interp_structs.h @@ -759,18 +759,23 @@ typedef _Py_CODEUNIT *(*_PyJitEntryFuncPtr)(struct _PyExecutorObject *exec, _PyI typedef struct _PyJitTracerState { bool dependencies_still_valid; + bool do_not_specialize; + bool dynamic_jump_taken; int code_max_size; int code_curr_size; int initial_stack_depth; int initial_chain_depth; + int prev_instr_oparg; + int prev_instr_stacklevel; _PyUOpInstruction *code_buffer; _Py_CODEUNIT *insert_exec_instr; _Py_CODEUNIT *close_loop_instr; - _Py_CODEUNIT *last_specialized_instr; PyCodeObject *initial_code; // Strong PyFunctionObject *initial_func; // Strong - struct _PyExitData *previous_exit; - _PyInterpreterFrame *current_frame; + _Py_CODEUNIT *prev_instr; + PyCodeObject *prev_instr_code; // Strong + struct _PyExitData *prev_exit; + _PyInterpreterFrame *prev_instr_frame; _PyBloomFilter dependencies; } _PyJitTracerState; diff --git a/Include/internal/pycore_magic_number.h b/Include/internal/pycore_magic_number.h index 7ec7bd1c6955..09c6b1f5b392 100644 --- a/Include/internal/pycore_magic_number.h +++ b/Include/internal/pycore_magic_number.h @@ -286,6 +286,7 @@ Known values: Python 3.15a1 3653 (Fix handling of opcodes that may leave operands on the stack when optimizing LOAD_FAST) Python 3.15a1 3654 (Fix missing exception handlers in logical expression) Python 3.15a1 3655 (Fix miscompilation of some module-level annotations) + Python 3.15a2 3656 (Add RECORD_PREVIOUS_INST for a tracing JIT) Python 3.16 will start with 3700 @@ -299,7 +300,7 @@ PC/launcher.c must also be updated. */ -#define PYC_MAGIC_NUMBER 3655 +#define PYC_MAGIC_NUMBER 3656 /* This is equivalent to converting PYC_MAGIC_NUMBER to 2 bytes (little-endian) and then appending b'\r\n'. */ #define PYC_MAGIC_NUMBER_TOKEN \ diff --git a/Include/internal/pycore_opcode_metadata.h b/Include/internal/pycore_opcode_metadata.h index e8e0c59874a5..288396f34fc3 100644 --- a/Include/internal/pycore_opcode_metadata.h +++ b/Include/internal/pycore_opcode_metadata.h @@ -412,6 +412,8 @@ int _PyOpcode_num_popped(int opcode, int oparg) { return 0; case RAISE_VARARGS: return oparg; + case RECORD_PREVIOUS_INST: + return 0; case RERAISE: return 1 + oparg; case RESERVED: @@ -895,6 +897,8 @@ int _PyOpcode_num_pushed(int opcode, int oparg) { return 1; case RAISE_VARARGS: return 0; + case RECORD_PREVIOUS_INST: + return 0; case RERAISE: return oparg; case RESERVED: @@ -1249,6 +1253,7 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[267] = { [PUSH_EXC_INFO] = { true, INSTR_FMT_IX, 0 }, [PUSH_NULL] = { true, INSTR_FMT_IX, HAS_PURE_FLAG }, [RAISE_VARARGS] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG }, + [RECORD_PREVIOUS_INST] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG | HAS_NO_SAVE_IP_FLAG }, [RERAISE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, [RESERVED] = { true, INSTR_FMT_IX, 0 }, [RESUME] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, @@ -1695,6 +1700,7 @@ const char *_PyOpcode_OpName[267] = { [PUSH_EXC_INFO] = "PUSH_EXC_INFO", [PUSH_NULL] = "PUSH_NULL", [RAISE_VARARGS] = "RAISE_VARARGS", + [RECORD_PREVIOUS_INST] = "RECORD_PREVIOUS_INST", [RERAISE] = "RERAISE", [RESERVED] = "RESERVED", [RESUME] = "RESUME", @@ -1805,7 +1811,6 @@ const uint8_t _PyOpcode_NeedsGuardIp[256] = { extern const uint8_t _PyOpcode_Deopt[256]; #ifdef NEED_OPCODE_METADATA const uint8_t _PyOpcode_Deopt[256] = { - [121] = 121, [122] = 122, [123] = 123, [124] = 124, @@ -2017,6 +2022,7 @@ const uint8_t _PyOpcode_Deopt[256] = { [PUSH_EXC_INFO] = PUSH_EXC_INFO, [PUSH_NULL] = PUSH_NULL, [RAISE_VARARGS] = RAISE_VARARGS, + [RECORD_PREVIOUS_INST] = RECORD_PREVIOUS_INST, [RERAISE] = RERAISE, [RESERVED] = RESERVED, [RESUME] = RESUME, @@ -2066,7 +2072,6 @@ const uint8_t _PyOpcode_Deopt[256] = { #endif // NEED_OPCODE_METADATA #define EXTRA_CASES \ - case 121: \ case 122: \ case 123: \ case 124: \ diff --git a/Include/internal/pycore_optimizer.h b/Include/internal/pycore_optimizer.h index 9c3aaf1edaf5..ca46249bde98 100644 --- a/Include/internal/pycore_optimizer.h +++ b/Include/internal/pycore_optimizer.h @@ -358,22 +358,12 @@ PyAPI_FUNC(int) _PyDumpExecutors(FILE *out); extern void _Py_ClearExecutorDeletionList(PyInterpreterState *interp); #endif -int -_PyJit_translate_single_bytecode_to_trace( - PyThreadState *tstate, - _PyInterpreterFrame *frame, - _Py_CODEUNIT *this_instr, - _Py_CODEUNIT *next_instr, - PyCodeObject *code, - PyFunctionObject *func, - int old_stack_level, - int opcode, - int oparg, - int jump_taken); +int _PyJit_translate_single_bytecode_to_trace(PyThreadState *tstate, _PyInterpreterFrame *frame, _Py_CODEUNIT *next_instr); void -_PyJit_InitializeTracing(PyThreadState *tstate, _PyInterpreterFrame *frame, _Py_CODEUNIT *insert_exec_instr, - _Py_CODEUNIT *close_loop_instr, int curr_stackdepth, int chain_depth, _PyExitData *exit); +_PyJit_InitializeTracing(PyThreadState *tstate, _PyInterpreterFrame *frame, + _Py_CODEUNIT *curr_instr, _Py_CODEUNIT *insert_exec_instr, + _Py_CODEUNIT *close_loop_instr, int curr_stackdepth, int chain_depth, _PyExitData *exit, int oparg); void _PyJit_FinalizeTracing(PyThreadState *tstate); diff --git a/Include/opcode_ids.h b/Include/opcode_ids.h index 1d5c74adefcd..1c7468456dbd 100644 --- a/Include/opcode_ids.h +++ b/Include/opcode_ids.h @@ -115,22 +115,23 @@ extern "C" { #define POP_JUMP_IF_NOT_NONE 102 #define POP_JUMP_IF_TRUE 103 #define RAISE_VARARGS 104 -#define RERAISE 105 -#define SEND 106 -#define SET_ADD 107 -#define SET_FUNCTION_ATTRIBUTE 108 -#define SET_UPDATE 109 -#define STORE_ATTR 110 -#define STORE_DEREF 111 -#define STORE_FAST 112 -#define STORE_FAST_LOAD_FAST 113 -#define STORE_FAST_STORE_FAST 114 -#define STORE_GLOBAL 115 -#define STORE_NAME 116 -#define SWAP 117 -#define UNPACK_EX 118 -#define UNPACK_SEQUENCE 119 -#define YIELD_VALUE 120 +#define RECORD_PREVIOUS_INST 105 +#define RERAISE 106 +#define SEND 107 +#define SET_ADD 108 +#define SET_FUNCTION_ATTRIBUTE 109 +#define SET_UPDATE 110 +#define STORE_ATTR 111 +#define STORE_DEREF 112 +#define STORE_FAST 113 +#define STORE_FAST_LOAD_FAST 114 +#define STORE_FAST_STORE_FAST 115 +#define STORE_GLOBAL 116 +#define STORE_NAME 117 +#define SWAP 118 +#define UNPACK_EX 119 +#define UNPACK_SEQUENCE 120 +#define YIELD_VALUE 121 #define RESUME 128 #define BINARY_OP_ADD_FLOAT 129 #define BINARY_OP_ADD_INT 130 diff --git a/Lib/_opcode_metadata.py b/Lib/_opcode_metadata.py index f168d169a329..b6bce074f8e5 100644 --- a/Lib/_opcode_metadata.py +++ b/Lib/_opcode_metadata.py @@ -312,22 +312,23 @@ opmap = { 'POP_JUMP_IF_NOT_NONE': 102, 'POP_JUMP_IF_TRUE': 103, 'RAISE_VARARGS': 104, - 'RERAISE': 105, - 'SEND': 106, - 'SET_ADD': 107, - 'SET_FUNCTION_ATTRIBUTE': 108, - 'SET_UPDATE': 109, - 'STORE_ATTR': 110, - 'STORE_DEREF': 111, - 'STORE_FAST': 112, - 'STORE_FAST_LOAD_FAST': 113, - 'STORE_FAST_STORE_FAST': 114, - 'STORE_GLOBAL': 115, - 'STORE_NAME': 116, - 'SWAP': 117, - 'UNPACK_EX': 118, - 'UNPACK_SEQUENCE': 119, - 'YIELD_VALUE': 120, + 'RECORD_PREVIOUS_INST': 105, + 'RERAISE': 106, + 'SEND': 107, + 'SET_ADD': 108, + 'SET_FUNCTION_ATTRIBUTE': 109, + 'SET_UPDATE': 110, + 'STORE_ATTR': 111, + 'STORE_DEREF': 112, + 'STORE_FAST': 113, + 'STORE_FAST_LOAD_FAST': 114, + 'STORE_FAST_STORE_FAST': 115, + 'STORE_GLOBAL': 116, + 'STORE_NAME': 117, + 'SWAP': 118, + 'UNPACK_EX': 119, + 'UNPACK_SEQUENCE': 120, + 'YIELD_VALUE': 121, 'INSTRUMENTED_END_FOR': 234, 'INSTRUMENTED_POP_ITER': 235, 'INSTRUMENTED_END_SEND': 236, diff --git a/Lib/test/test_sys.py b/Lib/test/test_sys.py index c5e50dffb139..98dd466849e5 100644 --- a/Lib/test/test_sys.py +++ b/Lib/test/test_sys.py @@ -2255,7 +2255,7 @@ class TestSysJIT(unittest.TestCase): # 1 extra iteration for tracing. for i in range(_testinternalcapi.TIER2_THRESHOLD + 2): # Careful, doing this in the reverse order breaks tracing: - expected = {enabled} and i >= _testinternalcapi.TIER2_THRESHOLD + expected = {enabled} and i >= _testinternalcapi.TIER2_THRESHOLD + 1 assert sys._jit.is_active() is expected frame_2_jit(expected) assert sys._jit.is_active() is expected diff --git a/Makefile.pre.in b/Makefile.pre.in index 2db96b744757..19423c11545c 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -2141,7 +2141,7 @@ Objects/mimalloc/page.o: $(srcdir)/Objects/mimalloc/page-queue.c regen-cases: \ regen-opcode-ids regen-opcode-targets regen-uop-ids regen-opcode-metadata-py \ regen-generated-cases regen-executor-cases regen-optimizer-cases \ - regen-opcode-metadata regen-uop-metadata regen-tracer-cases + regen-opcode-metadata regen-uop-metadata .PHONY: regen-opcode-ids regen-opcode-ids: @@ -2187,13 +2187,6 @@ regen-optimizer-cases: $(srcdir)/Python/bytecodes.c $(UPDATE_FILE) $(srcdir)/Python/optimizer_cases.c.h $(srcdir)/Python/optimizer_cases.c.h.new -.PHONY: regen-tracer-cases -regen-tracer-cases: - $(PYTHON_FOR_REGEN) $(srcdir)/Tools/cases_generator/tracer_generator.py \ - -o $(srcdir)/Python/generated_tracer_cases.c.h.new $(srcdir)/Python/bytecodes.c - $(UPDATE_FILE) $(srcdir)/Python/generated_tracer_cases.c.h $(srcdir)/Python/generated_tracer_cases.c.h.new - - .PHONY: regen-opcode-metadata regen-opcode-metadata: $(PYTHON_FOR_REGEN) $(srcdir)/Tools/cases_generator/opcode_metadata_generator.py \ diff --git a/Python/bytecodes.c b/Python/bytecodes.c index 388da83dbc24..73ee9fdb8fcc 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -45,6 +45,8 @@ #define USE_COMPUTED_GOTOS 0 #include "ceval_macros.h" +#include "ceval_macros.h" +#include "ceval_macros.h" #include "../Include/internal/pycore_code.h" #include "../Include/internal/pycore_stackref.h" @@ -2985,14 +2987,9 @@ dummy_func( oparg >>= 8; insert_exec_at--; } - _PyJit_InitializeTracing(tstate, frame, insert_exec_at, next_instr, STACK_LEVEL(), 0, NULL); + _PyJit_InitializeTracing(tstate, frame, this_instr, insert_exec_at, next_instr, STACK_LEVEL(), 0, NULL, oparg); ENTER_TRACING(); } - int _jump_taken = false; - PyCodeObject *old_code = _PyFrame_GetCode(frame); - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - int _old_stack_level = 0; - TRACING_DISPATCH(); } else { ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); @@ -3057,9 +3054,6 @@ dummy_func( } assert(executor != tstate->interp->cold_executor); tstate->jit_exit = NULL; - #if TRACING_JIT - RECORD_TRACE_NO_DISPATCH(); - #endif TIER1_TO_TIER2(executor); #else Py_FatalError("ENTER_EXECUTOR is not supported in this build"); @@ -5224,6 +5218,25 @@ dummy_func( Py_FatalError("Executing RESERVED instruction."); } + tier1 no_save_ip inst(RECORD_PREVIOUS_INST, (--)) { + assert(IS_JIT_TRACING()); + int opcode = next_instr->op.code; + int full = !_PyJit_translate_single_bytecode_to_trace(tstate, frame, next_instr); + if (full) { + LEAVE_TRACING(); + int err = bail_tracing_and_jit(tstate, frame); + ERROR_IF(err < 0); + DISPATCH_GOTO_NON_TRACING(); + } + tstate->interp->jit_state.do_not_specialize = false; + PyCodeObject *prev_code = (PyCodeObject *)Py_NewRef(_PyFrame_GetCode(frame)); + Py_SETREF(tstate->interp->jit_state.prev_instr_code, prev_code); + tstate->interp->jit_state.prev_instr = next_instr; + tstate->interp->jit_state.prev_instr_frame = frame; + tstate->interp->jit_state.prev_instr_oparg = oparg; + tstate->interp->jit_state.prev_instr_stacklevel = STACK_LEVEL(); + DISPATCH_GOTO_NON_TRACING(); + } ///////// Tier-2 only opcodes ///////// op (_GUARD_IS_TRUE_POP, (flag -- )) { @@ -5456,7 +5469,10 @@ dummy_func( _PyExecutorObject *previous_executor = _PyExecutor_FromExit(exit); assert(tstate->current_executor == (PyObject *)previous_executor); int chain_depth = previous_executor->vm_data.chain_depth + 1; - _PyJit_InitializeTracing(tstate, frame, target, target, STACK_LEVEL(), chain_depth, exit); + // Note: it's safe to use target->op.arg here instead of the oparg given by EXTENDED_ARG. + // The invariant in the optimizer is the deopt target always points back to the first EXTENDED_ARG. + // So setting it to anything else is wrong. + _PyJit_InitializeTracing(tstate, frame, target, target, target, STACK_LEVEL(), chain_depth, exit, target->op.arg); exit->temperature = initial_temperature_backoff_counter(); GOTO_TIER_ONE(target, 1); } diff --git a/Python/ceval.c b/Python/ceval.c index ada9a9fd736c..f4bf7b244dc7 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -984,17 +984,6 @@ _PyObjectArray_Free(PyObject **array, PyObject **scratch) } } -#if _Py_TIER2 -// 1 for trace full, 0 for successful write. -static inline int -add_to_code_trace(PyThreadState *tstate, _PyInterpreterFrame *frame, PyCodeObject *old_code, PyFunctionObject *old_func, int old_stack_level, _Py_CODEUNIT *this_instr, _Py_CODEUNIT *next_instr, int opcode, int oparg, int jump_taken) -{ - assert(frame != NULL); - assert(tstate->interp->jit_state.code_curr_size < UOP_MAX_TRACE_LENGTH); - return !_PyJit_translate_single_bytecode_to_trace(tstate, frame, this_instr, next_instr, old_code, old_func, old_stack_level, opcode, oparg, jump_taken); -} - - // 0 for success, -1 for error. static int bail_tracing_and_jit(PyThreadState *tstate, _PyInterpreterFrame *frame) @@ -1007,7 +996,6 @@ bail_tracing_and_jit(PyThreadState *tstate, _PyInterpreterFrame *frame) _PyJit_FinalizeTracing(tstate); return err; } -#endif /* _PyEval_EvalFrameDefault is too large to optimize for speed with PGO on MSVC. */ @@ -1025,7 +1013,6 @@ bail_tracing_and_jit(PyThreadState *tstate, _PyInterpreterFrame *frame) #if _Py_TAIL_CALL_INTERP #include "opcode_targets.h" #include "generated_cases.c.h" -#include "generated_tracer_cases.c.h" #endif #if (defined(__GNUC__) && __GNUC__ >= 10 && !defined(__clang__)) && defined(__x86_64__) @@ -1154,7 +1141,6 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int #else goto start_frame; # include "generated_cases.c.h" - #include "generated_tracer_cases.c.h" #endif diff --git a/Python/ceval_macros.h b/Python/ceval_macros.h index e19a614f28de..bfc84e8c6623 100644 --- a/Python/ceval_macros.h +++ b/Python/ceval_macros.h @@ -94,6 +94,10 @@ do { \ Py_MUSTTAIL return (((py_tail_call_funcptr *)instruction_funcptr_table)[opcode])(TAIL_CALL_ARGS); \ } while (0) +# define DISPATCH_GOTO_NON_TRACING() \ + do { \ + Py_MUSTTAIL return (((py_tail_call_funcptr *)DISPATCH_TABLE)[opcode])(TAIL_CALL_ARGS); \ + } while (0) # define JUMP_TO_LABEL(name) \ do { \ Py_MUSTTAIL return (_TAIL_CALL_##name)(TAIL_CALL_ARGS); \ @@ -117,7 +121,7 @@ # define TARGET(op) TARGET_##op: # define TRACING_TARGET(op) TARGET_TRACING_##op: # define DISPATCH_GOTO() goto *opcode_targets[opcode] - +# define DISPATCH_GOTO_NON_TRACING() goto *DISPATCH_TABLE[opcode]; # define JUMP_TO_LABEL(name) goto name; # define JUMP_TO_PREDICTED(name) goto PREDICTED_##name; # define LABEL(name) name: @@ -129,35 +133,14 @@ # define LABEL(name) name: #endif -#define TRACING_JUMP_TO_LABEL(label) \ - RECORD_DYNAMIC_JUMP_TAKEN() \ - RECORD_TRACE_NO_DISPATCH() \ - assert(!IS_JIT_TRACING()); \ - JUMP_TO_LABEL(label); - #if _Py_TAIL_CALL_INTERP || USE_COMPUTED_GOTOS # define IS_JIT_TRACING() (DISPATCH_TABLE_VAR == TRACING_DISPATCH_TABLE) -// tstate->interp->jit_state.last_specialized_instr != this_instr is required to not get stuck in infinite -// specialization loops due to specialization failure. -# define IS_JIT_TRACING_MAKING_PROGRESS() (IS_JIT_TRACING() && tstate->interp->jit_state.last_specialized_instr != this_instr) +// Required to not get stuck in infinite pecialization loops due to specialization failure. +# define IS_JIT_TRACING_MAKING_PROGRESS() (IS_JIT_TRACING() && !tstate->interp->jit_state.do_not_specialize) # define ENTER_TRACING() \ DISPATCH_TABLE_VAR = TRACING_DISPATCH_TABLE; # define LEAVE_TRACING() \ DISPATCH_TABLE_VAR = DISPATCH_TABLE; -# define RECORD_TRACE_NO_DISPATCH() do { \ - int err = 0; \ - _PyFrame_SetStackPointer(frame, stack_pointer); \ - /* We need to check once more here in case it swapped out halfway. */ \ - if (IS_JIT_TRACING()) { \ - int full = add_to_code_trace(tstate, frame, old_code, old_func, _old_stack_level, this_instr, next_instr, opcode, oparg, _jump_taken); \ - if (full) { \ - LEAVE_TRACING(); \ - err = bail_tracing_and_jit(tstate, frame); \ - } \ - } \ - stack_pointer = _PyFrame_GetStackPointer(frame); \ - if (err < 0) { JUMP_TO_LABEL(error); } \ - } while (0); #endif @@ -197,21 +180,22 @@ do { \ DISPATCH_GOTO(); \ } +#define DISPATCH_NON_TRACING() \ + { \ + assert(frame->stackpointer == NULL); \ + NEXTOPARG(); \ + PRE_DISPATCH_GOTO(); \ + DISPATCH_GOTO_NON_TRACING(); \ + } + +#if _Py_TIER2 #define DISPATCH_SAME_OPARG() \ { \ + tstate->interp->jit_state.do_not_specialize = true; \ opcode = next_instr->op.code; \ PRE_DISPATCH_GOTO(); \ - DISPATCH_GOTO(); \ + DISPATCH_GOTO_NON_TRACING(); \ } - -#define TRACING_SPECIALIZE_DISPATCH_SAME_OPARG() \ -{ \ - tstate->interp->jit_state.last_specialized_instr = this_instr; \ - opcode = next_instr->op.code; \ - PRE_DISPATCH_GOTO(); \ - DISPATCH_GOTO(); \ -} - #define DISPATCH_INLINED(NEW_FRAME) \ do { \ assert(tstate->interp->eval_frame == NULL); \ @@ -221,21 +205,24 @@ do { \ CALL_STAT_INC(inlined_py_calls); \ JUMP_TO_LABEL(start_frame); \ } while (0) - -#define TRACING_DISPATCH_INLINED(NEW_FRAME) \ - tstate->interp->jit_state.last_specialized_instr = this_instr; \ - RECORD_TRACE_NO_DISPATCH(); \ - DISPATCH_INLINED(NEW_FRAME); - -#define TRACING_DISPATCH() \ +#else +#define DISPATCH_SAME_OPARG() \ { \ - tstate->interp->jit_state.last_specialized_instr = this_instr; \ - assert(frame->stackpointer == NULL); \ - RECORD_TRACE_NO_DISPATCH(); \ - NEXTOPARG(); \ + opcode = next_instr->op.code; \ PRE_DISPATCH_GOTO(); \ DISPATCH_GOTO(); \ } +#define DISPATCH_INLINED(NEW_FRAME) \ + do { \ + assert(tstate->interp->eval_frame == NULL); \ + _PyFrame_SetStackPointer(frame, stack_pointer); \ + assert((NEW_FRAME)->previous == frame); \ + frame = tstate->current_frame = (NEW_FRAME); \ + CALL_STAT_INC(inlined_py_calls); \ + JUMP_TO_LABEL(start_frame); \ + } while (0) +#endif + /* Tuple access macros */ @@ -368,7 +355,7 @@ GETITEM(PyObject *v, Py_ssize_t i) { #define RECORD_BRANCH_TAKEN(bitset, flag) #endif -#define RECORD_DYNAMIC_JUMP_TAKEN() _jump_taken = 1; +#define RECORD_DYNAMIC_JUMP_TAKEN() tstate->interp->jit_state.dynamic_jump_taken = true; #define UNBOUNDLOCAL_ERROR_MSG \ "cannot access local variable '%s' where it is not associated with a value" @@ -426,6 +413,7 @@ _PyFrame_SetStackPointer(frame, stack_pointer) #define TIER1_TO_TIER2(EXECUTOR) \ do { \ + LEAVE_TRACING(); \ OPT_STAT_INC(traces_executed); \ next_instr = _Py_jit_entry((EXECUTOR), frame, stack_pointer, tstate); \ frame = tstate->current_frame; \ @@ -440,6 +428,7 @@ do { \ assert(next_instr->op.code != ENTER_EXECUTOR); \ assert(tstate->interp->jit_state.code_curr_size == 2); \ ENTER_TRACING(); \ + DISPATCH_NON_TRACING(); \ } \ DISPATCH(); \ } while (0) diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h index 72927d8378c3..f839b9579ef3 100644 --- a/Python/executor_cases.c.h +++ b/Python/executor_cases.c.h @@ -7501,7 +7501,7 @@ _PyExecutorObject *previous_executor = _PyExecutor_FromExit(exit); assert(tstate->current_executor == (PyObject *)previous_executor); int chain_depth = previous_executor->vm_data.chain_depth + 1; - _PyJit_InitializeTracing(tstate, frame, target, target, STACK_LEVEL(), chain_depth, exit); + _PyJit_InitializeTracing(tstate, frame, target, target, target, STACK_LEVEL(), chain_depth, exit, target->op.arg); exit->temperature = initial_temperature_backoff_counter(); GOTO_TIER_ONE(target, 1); } diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index 4fb61a06cdfe..516d42a3972d 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -5493,9 +5493,6 @@ } assert(executor != tstate->interp->cold_executor); tstate->jit_exit = NULL; - #if TRACING_JIT - RECORD_TRACE_NO_DISPATCH(); - #endif TIER1_TO_TIER2(executor); #else Py_FatalError("ENTER_EXECUTOR is not supported in this build"); @@ -5656,6 +5653,7 @@ JUMP_TO_LABEL(error); } JUMPBY(oparg + 1); + RECORD_DYNAMIC_JUMP_TAKEN(); stack_pointer[-1] = null_or_index; DISPATCH(); } @@ -5784,6 +5782,7 @@ if ((size_t)PyStackRef_UntagInt(null_or_index) >= (size_t)PyList_GET_SIZE(list_o)) { null_or_index = PyStackRef_TagInt(-1); JUMPBY(oparg + 1); + RECORD_DYNAMIC_JUMP_TAKEN(); stack_pointer[-1] = null_or_index; DISPATCH(); } @@ -5864,6 +5863,7 @@ STAT_INC(FOR_ITER, hit); if (r->len <= 0) { JUMPBY(oparg + 1); + RECORD_DYNAMIC_JUMP_TAKEN(); DISPATCH(); } } @@ -5926,6 +5926,7 @@ if ((size_t)PyStackRef_UntagInt(null_or_index) >= (size_t)PyTuple_GET_SIZE(tuple_o)) { null_or_index = PyStackRef_TagInt(-1); JUMPBY(oparg + 1); + RECORD_DYNAMIC_JUMP_TAKEN(); stack_pointer[-1] = null_or_index; DISPATCH(); } @@ -7676,14 +7677,9 @@ oparg >>= 8; insert_exec_at--; } - _PyJit_InitializeTracing(tstate, frame, insert_exec_at, next_instr, STACK_LEVEL(), 0, NULL); + _PyJit_InitializeTracing(tstate, frame, this_instr, insert_exec_at, next_instr, STACK_LEVEL(), 0, NULL, oparg); ENTER_TRACING(); } - int _jump_taken = false; - PyCodeObject *old_code = _PyFrame_GetCode(frame); - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - int _old_stack_level = 0; - TRACING_DISPATCH(); } else { ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); @@ -10200,6 +10196,36 @@ JUMP_TO_LABEL(error); } + + TARGET(RECORD_PREVIOUS_INST) { + INSTRUCTION_STATS(RECORD_PREVIOUS_INST); + assert(IS_JIT_TRACING()); + int opcode = next_instr->op.code; + _PyFrame_SetStackPointer(frame, stack_pointer); + int full = !_PyJit_translate_single_bytecode_to_trace(tstate, frame, next_instr); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (full) { + LEAVE_TRACING(); + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = bail_tracing_and_jit(tstate, frame); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err < 0) { + JUMP_TO_LABEL(error); + } + DISPATCH_GOTO_NON_TRACING(); + } + tstate->interp->jit_state.do_not_specialize = false; + PyCodeObject *prev_code = (PyCodeObject *)Py_NewRef(_PyFrame_GetCode(frame)); + _PyFrame_SetStackPointer(frame, stack_pointer); + Py_SETREF(tstate->interp->jit_state.prev_instr_code, prev_code); + stack_pointer = _PyFrame_GetStackPointer(frame); + tstate->interp->jit_state.prev_instr = next_instr; + tstate->interp->jit_state.prev_instr_frame = frame; + tstate->interp->jit_state.prev_instr_oparg = oparg; + tstate->interp->jit_state.prev_instr_stacklevel = STACK_LEVEL(); + DISPATCH_GOTO_NON_TRACING(); + } + TARGET(RERAISE) { #if _Py_TAIL_CALL_INTERP int opcode = RERAISE; @@ -10517,6 +10543,7 @@ if (err == 0) { assert(retval_o != NULL); JUMPBY(oparg); + RECORD_DYNAMIC_JUMP_TAKEN(); } else { stack_pointer += -1; diff --git a/Python/generated_tracer_cases.c.h b/Python/generated_tracer_cases.c.h deleted file mode 100644 index 932a57a22e51..000000000000 --- a/Python/generated_tracer_cases.c.h +++ /dev/null @@ -1,14556 +0,0 @@ -// This file is generated by Tools/cases_generator/tracer_generator.py -// from: -// Python/bytecodes.c -// Do not edit! - #define TIER_ONE 1 - #define TRACING_JIT 1 - #ifdef _Py_TIER2 /* BEGIN TRACING INSTRUCTIONS */ - - - TRACING_TARGET(BINARY_OP) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = BINARY_OP; - (void)(opcode); - #endif - frame->instr_ptr = next_instr; - next_instr += 6; - INSTRUCTION_STATS(BINARY_OP); - PREDICTED_TRACING_BINARY_OP:; - _Py_CODEUNIT* const this_instr = next_instr - 6; - (void)this_instr; - opcode = BINARY_OP; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; - (void)(_old_stack_level); - _PyStackRef lhs; - _PyStackRef rhs; - _PyStackRef res; - // _SPECIALIZE_BINARY_OP - { - rhs = stack_pointer[-1]; - lhs = stack_pointer[-2]; - uint16_t counter = read_u16(&this_instr[1].cache); - (void)counter; - #if ENABLE_SPECIALIZATION_FT - if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { - next_instr = this_instr; - _PyFrame_SetStackPointer(frame, stack_pointer); - _Py_Specialize_BinaryOp(lhs, rhs, next_instr, oparg, LOCALS_ARRAY); - stack_pointer = _PyFrame_GetStackPointer(frame); - TRACING_SPECIALIZE_DISPATCH_SAME_OPARG() - } - OPCODE_DEFERRED_INC(BINARY_OP); - ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); - #endif /* ENABLE_SPECIALIZATION_FT */ - assert(NB_ADD <= oparg); - assert(oparg <= NB_OPARG_LAST); - } - /* Skip 4 cache entries */ - // _BINARY_OP - { - PyObject *lhs_o = PyStackRef_AsPyObjectBorrow(lhs); - PyObject *rhs_o = PyStackRef_AsPyObjectBorrow(rhs); - assert(_PyEval_BinaryOps[oparg]); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *res_o = _PyEval_BinaryOps[oparg](lhs_o, rhs_o); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (res_o == NULL) { - TRACING_JUMP_TO_LABEL(error); - } - res = PyStackRef_FromPyObjectSteal(res_o); - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyStackRef tmp = lhs; - lhs = res; - stack_pointer[-2] = lhs; - PyStackRef_CLOSE(tmp); - tmp = rhs; - rhs = PyStackRef_NULL; - stack_pointer[-1] = rhs; - PyStackRef_CLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - } - TRACING_DISPATCH(); - } - - TRACING_TARGET(BINARY_OP_ADD_FLOAT) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = BINARY_OP_ADD_FLOAT; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 6; - INSTRUCTION_STATS(BINARY_OP_ADD_FLOAT); - opcode = BINARY_OP_ADD_FLOAT; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; - (void)(_old_stack_level); - static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5, "incorrect cache size"); - _PyStackRef value; - _PyStackRef left; - _PyStackRef right; - _PyStackRef res; - // _GUARD_TOS_FLOAT - { - value = stack_pointer[-1]; - PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); - if (!PyFloat_CheckExact(value_o)) { - UPDATE_MISS_STATS(BINARY_OP); - assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); - JUMP_TO_PREDICTED(TRACING_BINARY_OP); - } - } - // _GUARD_NOS_FLOAT - { - left = stack_pointer[-2]; - PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); - if (!PyFloat_CheckExact(left_o)) { - UPDATE_MISS_STATS(BINARY_OP); - assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); - JUMP_TO_PREDICTED(TRACING_BINARY_OP); - } - } - /* Skip 5 cache entries */ - // _BINARY_OP_ADD_FLOAT - { - right = value; - PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); - PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - assert(PyFloat_CheckExact(left_o)); - assert(PyFloat_CheckExact(right_o)); - STAT_INC(BINARY_OP, hit); - double dres = - ((PyFloatObject *)left_o)->ob_fval + - ((PyFloatObject *)right_o)->ob_fval; - res = _PyFloat_FromDouble_ConsumeInputs(left, right, dres); - if (PyStackRef_IsNull(res)) { - TRACING_JUMP_TO_LABEL(pop_2_error); - } - } - stack_pointer[-2] = res; - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - TRACING_DISPATCH(); - } - - TRACING_TARGET(BINARY_OP_ADD_INT) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = BINARY_OP_ADD_INT; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 6; - INSTRUCTION_STATS(BINARY_OP_ADD_INT); - opcode = BINARY_OP_ADD_INT; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; - (void)(_old_stack_level); - static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5, "incorrect cache size"); - _PyStackRef value; - _PyStackRef left; - _PyStackRef right; - _PyStackRef res; - // _GUARD_TOS_INT - { - value = stack_pointer[-1]; - PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); - if (!_PyLong_CheckExactAndCompact(value_o)) { - UPDATE_MISS_STATS(BINARY_OP); - assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); - JUMP_TO_PREDICTED(TRACING_BINARY_OP); - } - } - // _GUARD_NOS_INT - { - left = stack_pointer[-2]; - PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); - if (!_PyLong_CheckExactAndCompact(left_o)) { - UPDATE_MISS_STATS(BINARY_OP); - assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); - JUMP_TO_PREDICTED(TRACING_BINARY_OP); - } - } - /* Skip 5 cache entries */ - // _BINARY_OP_ADD_INT - { - right = value; - PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); - PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - assert(PyLong_CheckExact(left_o)); - assert(PyLong_CheckExact(right_o)); - assert(_PyLong_BothAreCompact((PyLongObject *)left_o, (PyLongObject *)right_o)); - STAT_INC(BINARY_OP, hit); - res = _PyCompactLong_Add((PyLongObject *)left_o, (PyLongObject *)right_o); - if (PyStackRef_IsNull(res)) { - UPDATE_MISS_STATS(BINARY_OP); - assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); - JUMP_TO_PREDICTED(TRACING_BINARY_OP); - } - PyStackRef_CLOSE_SPECIALIZED(right, _PyLong_ExactDealloc); - PyStackRef_CLOSE_SPECIALIZED(left, _PyLong_ExactDealloc); - } - stack_pointer[-2] = res; - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - TRACING_DISPATCH(); - } - - TRACING_TARGET(BINARY_OP_ADD_UNICODE) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = BINARY_OP_ADD_UNICODE; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 6; - INSTRUCTION_STATS(BINARY_OP_ADD_UNICODE); - opcode = BINARY_OP_ADD_UNICODE; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; - (void)(_old_stack_level); - static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5, "incorrect cache size"); - _PyStackRef value; - _PyStackRef nos; - _PyStackRef left; - _PyStackRef right; - _PyStackRef res; - // _GUARD_TOS_UNICODE - { - value = stack_pointer[-1]; - PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); - if (!PyUnicode_CheckExact(value_o)) { - UPDATE_MISS_STATS(BINARY_OP); - assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); - JUMP_TO_PREDICTED(TRACING_BINARY_OP); - } - } - // _GUARD_NOS_UNICODE - { - nos = stack_pointer[-2]; - PyObject *o = PyStackRef_AsPyObjectBorrow(nos); - if (!PyUnicode_CheckExact(o)) { - UPDATE_MISS_STATS(BINARY_OP); - assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); - JUMP_TO_PREDICTED(TRACING_BINARY_OP); - } - } - /* Skip 5 cache entries */ - // _BINARY_OP_ADD_UNICODE - { - right = value; - left = nos; - PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); - PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - assert(PyUnicode_CheckExact(left_o)); - assert(PyUnicode_CheckExact(right_o)); - STAT_INC(BINARY_OP, hit); - PyObject *res_o = PyUnicode_Concat(left_o, right_o); - PyStackRef_CLOSE_SPECIALIZED(right, _PyUnicode_ExactDealloc); - PyStackRef_CLOSE_SPECIALIZED(left, _PyUnicode_ExactDealloc); - if (res_o == NULL) { - TRACING_JUMP_TO_LABEL(pop_2_error); - } - res = PyStackRef_FromPyObjectSteal(res_o); - } - stack_pointer[-2] = res; - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - TRACING_DISPATCH(); - } - - TRACING_TARGET(BINARY_OP_EXTEND) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = BINARY_OP_EXTEND; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 6; - INSTRUCTION_STATS(BINARY_OP_EXTEND); - opcode = BINARY_OP_EXTEND; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; - (void)(_old_stack_level); - static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5, "incorrect cache size"); - _PyStackRef left; - _PyStackRef right; - _PyStackRef res; - /* Skip 1 cache entry */ - // _GUARD_BINARY_OP_EXTEND - { - right = stack_pointer[-1]; - left = stack_pointer[-2]; - PyObject *descr = read_obj(&this_instr[2].cache); - PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); - PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - _PyBinaryOpSpecializationDescr *d = (_PyBinaryOpSpecializationDescr*)descr; - assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5); - assert(d && d->guard); - _PyFrame_SetStackPointer(frame, stack_pointer); - int res = d->guard(left_o, right_o); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (!res) { - UPDATE_MISS_STATS(BINARY_OP); - assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); - JUMP_TO_PREDICTED(TRACING_BINARY_OP); - } - } - /* Skip -4 cache entry */ - // _BINARY_OP_EXTEND - { - PyObject *descr = read_obj(&this_instr[2].cache); - PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); - PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5); - _PyBinaryOpSpecializationDescr *d = (_PyBinaryOpSpecializationDescr*)descr; - STAT_INC(BINARY_OP, hit); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *res_o = d->action(left_o, right_o); - _PyStackRef tmp = right; - right = PyStackRef_NULL; - stack_pointer[-1] = right; - PyStackRef_CLOSE(tmp); - tmp = left; - left = PyStackRef_NULL; - stack_pointer[-2] = left; - PyStackRef_CLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); - res = PyStackRef_FromPyObjectSteal(res_o); - } - stack_pointer[0] = res; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - TRACING_DISPATCH(); - } - - TRACING_TARGET(BINARY_OP_INPLACE_ADD_UNICODE) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = BINARY_OP_INPLACE_ADD_UNICODE; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 6; - INSTRUCTION_STATS(BINARY_OP_INPLACE_ADD_UNICODE); - opcode = BINARY_OP_INPLACE_ADD_UNICODE; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; - (void)(_old_stack_level); - static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5, "incorrect cache size"); - _PyStackRef value; - _PyStackRef nos; - _PyStackRef left; - _PyStackRef right; - // _GUARD_TOS_UNICODE - { - value = stack_pointer[-1]; - PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); - if (!PyUnicode_CheckExact(value_o)) { - UPDATE_MISS_STATS(BINARY_OP); - assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); - JUMP_TO_PREDICTED(TRACING_BINARY_OP); - } - } - // _GUARD_NOS_UNICODE - { - nos = stack_pointer[-2]; - PyObject *o = PyStackRef_AsPyObjectBorrow(nos); - if (!PyUnicode_CheckExact(o)) { - UPDATE_MISS_STATS(BINARY_OP); - assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); - JUMP_TO_PREDICTED(TRACING_BINARY_OP); - } - } - /* Skip 5 cache entries */ - // _BINARY_OP_INPLACE_ADD_UNICODE - { - right = value; - left = nos; - PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); - assert(PyUnicode_CheckExact(left_o)); - assert(PyUnicode_CheckExact(PyStackRef_AsPyObjectBorrow(right))); - int next_oparg; - #if TIER_ONE - assert(next_instr->op.code == STORE_FAST); - next_oparg = next_instr->op.arg; - #else - next_oparg = (int)CURRENT_OPERAND0(); - #endif - _PyStackRef *target_local = &GETLOCAL(next_oparg); - assert(PyUnicode_CheckExact(left_o)); - if (PyStackRef_AsPyObjectBorrow(*target_local) != left_o) { - UPDATE_MISS_STATS(BINARY_OP); - assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); - JUMP_TO_PREDICTED(TRACING_BINARY_OP); - } - STAT_INC(BINARY_OP, hit); - assert(Py_REFCNT(left_o) >= 2 || !PyStackRef_IsHeapSafe(left)); - PyStackRef_CLOSE_SPECIALIZED(left, _PyUnicode_ExactDealloc); - PyObject *temp = PyStackRef_AsPyObjectSteal(*target_local); - PyObject *right_o = PyStackRef_AsPyObjectSteal(right); - stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyUnicode_Append(&temp, right_o); - stack_pointer = _PyFrame_GetStackPointer(frame); - *target_local = PyStackRef_FromPyObjectSteal(temp); - _PyFrame_SetStackPointer(frame, stack_pointer); - Py_DECREF(right_o); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (PyStackRef_IsNull(*target_local)) { - TRACING_JUMP_TO_LABEL(error); - } - #if TIER_ONE - - assert(next_instr->op.code == STORE_FAST); - SKIP_OVER(1); - #endif - } - TRACING_DISPATCH(); - } - - TRACING_TARGET(BINARY_OP_MULTIPLY_FLOAT) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = BINARY_OP_MULTIPLY_FLOAT; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 6; - INSTRUCTION_STATS(BINARY_OP_MULTIPLY_FLOAT); - opcode = BINARY_OP_MULTIPLY_FLOAT; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; - (void)(_old_stack_level); - static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5, "incorrect cache size"); - _PyStackRef value; - _PyStackRef left; - _PyStackRef right; - _PyStackRef res; - // _GUARD_TOS_FLOAT - { - value = stack_pointer[-1]; - PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); - if (!PyFloat_CheckExact(value_o)) { - UPDATE_MISS_STATS(BINARY_OP); - assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); - JUMP_TO_PREDICTED(TRACING_BINARY_OP); - } - } - // _GUARD_NOS_FLOAT - { - left = stack_pointer[-2]; - PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); - if (!PyFloat_CheckExact(left_o)) { - UPDATE_MISS_STATS(BINARY_OP); - assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); - JUMP_TO_PREDICTED(TRACING_BINARY_OP); - } - } - /* Skip 5 cache entries */ - // _BINARY_OP_MULTIPLY_FLOAT - { - right = value; - PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); - PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - assert(PyFloat_CheckExact(left_o)); - assert(PyFloat_CheckExact(right_o)); - STAT_INC(BINARY_OP, hit); - double dres = - ((PyFloatObject *)left_o)->ob_fval * - ((PyFloatObject *)right_o)->ob_fval; - res = _PyFloat_FromDouble_ConsumeInputs(left, right, dres); - if (PyStackRef_IsNull(res)) { - TRACING_JUMP_TO_LABEL(pop_2_error); - } - } - stack_pointer[-2] = res; - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - TRACING_DISPATCH(); - } - - TRACING_TARGET(BINARY_OP_MULTIPLY_INT) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = BINARY_OP_MULTIPLY_INT; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 6; - INSTRUCTION_STATS(BINARY_OP_MULTIPLY_INT); - opcode = BINARY_OP_MULTIPLY_INT; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; - (void)(_old_stack_level); - static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5, "incorrect cache size"); - _PyStackRef value; - _PyStackRef left; - _PyStackRef right; - _PyStackRef res; - // _GUARD_TOS_INT - { - value = stack_pointer[-1]; - PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); - if (!_PyLong_CheckExactAndCompact(value_o)) { - UPDATE_MISS_STATS(BINARY_OP); - assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); - JUMP_TO_PREDICTED(TRACING_BINARY_OP); - } - } - // _GUARD_NOS_INT - { - left = stack_pointer[-2]; - PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); - if (!_PyLong_CheckExactAndCompact(left_o)) { - UPDATE_MISS_STATS(BINARY_OP); - assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); - JUMP_TO_PREDICTED(TRACING_BINARY_OP); - } - } - /* Skip 5 cache entries */ - // _BINARY_OP_MULTIPLY_INT - { - right = value; - PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); - PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - assert(PyLong_CheckExact(left_o)); - assert(PyLong_CheckExact(right_o)); - assert(_PyLong_BothAreCompact((PyLongObject *)left_o, (PyLongObject *)right_o)); - STAT_INC(BINARY_OP, hit); - res = _PyCompactLong_Multiply((PyLongObject *)left_o, (PyLongObject *)right_o); - if (PyStackRef_IsNull(res)) { - UPDATE_MISS_STATS(BINARY_OP); - assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); - JUMP_TO_PREDICTED(TRACING_BINARY_OP); - } - PyStackRef_CLOSE_SPECIALIZED(right, _PyLong_ExactDealloc); - PyStackRef_CLOSE_SPECIALIZED(left, _PyLong_ExactDealloc); - } - stack_pointer[-2] = res; - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - TRACING_DISPATCH(); - } - - TRACING_TARGET(BINARY_OP_SUBSCR_DICT) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = BINARY_OP_SUBSCR_DICT; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 6; - INSTRUCTION_STATS(BINARY_OP_SUBSCR_DICT); - opcode = BINARY_OP_SUBSCR_DICT; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; - (void)(_old_stack_level); - static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5, "incorrect cache size"); - _PyStackRef nos; - _PyStackRef dict_st; - _PyStackRef sub_st; - _PyStackRef res; - // _GUARD_NOS_DICT - { - nos = stack_pointer[-2]; - PyObject *o = PyStackRef_AsPyObjectBorrow(nos); - if (!PyDict_CheckExact(o)) { - UPDATE_MISS_STATS(BINARY_OP); - assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); - JUMP_TO_PREDICTED(TRACING_BINARY_OP); - } - } - /* Skip 5 cache entries */ - // _BINARY_OP_SUBSCR_DICT - { - sub_st = stack_pointer[-1]; - dict_st = nos; - PyObject *sub = PyStackRef_AsPyObjectBorrow(sub_st); - PyObject *dict = PyStackRef_AsPyObjectBorrow(dict_st); - assert(PyDict_CheckExact(dict)); - STAT_INC(BINARY_OP, hit); - PyObject *res_o; - _PyFrame_SetStackPointer(frame, stack_pointer); - int rc = PyDict_GetItemRef(dict, sub, &res_o); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (rc == 0) { - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyErr_SetKeyError(sub); - stack_pointer = _PyFrame_GetStackPointer(frame); - } - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyStackRef tmp = sub_st; - sub_st = PyStackRef_NULL; - stack_pointer[-1] = sub_st; - PyStackRef_CLOSE(tmp); - tmp = dict_st; - dict_st = PyStackRef_NULL; - stack_pointer[-2] = dict_st; - PyStackRef_CLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); - if (rc <= 0) { - TRACING_JUMP_TO_LABEL(error); - } - res = PyStackRef_FromPyObjectSteal(res_o); - } - stack_pointer[0] = res; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - TRACING_DISPATCH(); - } - - TRACING_TARGET(BINARY_OP_SUBSCR_GETITEM) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = BINARY_OP_SUBSCR_GETITEM; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 6; - INSTRUCTION_STATS(BINARY_OP_SUBSCR_GETITEM); - opcode = BINARY_OP_SUBSCR_GETITEM; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; - (void)(_old_stack_level); - static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5, "incorrect cache size"); - _PyStackRef container; - _PyStackRef getitem; - _PyStackRef sub; - _PyStackRef new_frame; - /* Skip 5 cache entries */ - // _CHECK_PEP_523 - { - if (tstate->interp->eval_frame) { - UPDATE_MISS_STATS(BINARY_OP); - assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); - JUMP_TO_PREDICTED(TRACING_BINARY_OP); - } - } - // _BINARY_OP_SUBSCR_CHECK_FUNC - { - container = stack_pointer[-2]; - PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(container)); - if (!PyType_HasFeature(tp, Py_TPFLAGS_HEAPTYPE)) { - UPDATE_MISS_STATS(BINARY_OP); - assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); - JUMP_TO_PREDICTED(TRACING_BINARY_OP); - } - PyHeapTypeObject *ht = (PyHeapTypeObject *)tp; - PyObject *getitem_o = FT_ATOMIC_LOAD_PTR_ACQUIRE(ht->_spec_cache.getitem); - if (getitem_o == NULL) { - UPDATE_MISS_STATS(BINARY_OP); - assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); - JUMP_TO_PREDICTED(TRACING_BINARY_OP); - } - assert(PyFunction_Check(getitem_o)); - uint32_t cached_version = FT_ATOMIC_LOAD_UINT32_RELAXED(ht->_spec_cache.getitem_version); - if (((PyFunctionObject *)getitem_o)->func_version != cached_version) { - UPDATE_MISS_STATS(BINARY_OP); - assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); - JUMP_TO_PREDICTED(TRACING_BINARY_OP); - } - PyCodeObject *code = (PyCodeObject *)PyFunction_GET_CODE(getitem_o); - assert(code->co_argcount == 2); - if (!_PyThreadState_HasStackSpace(tstate, code->co_framesize)) { - UPDATE_MISS_STATS(BINARY_OP); - assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); - JUMP_TO_PREDICTED(TRACING_BINARY_OP); - } - getitem = PyStackRef_FromPyObjectNew(getitem_o); - STAT_INC(BINARY_OP, hit); - } - // _BINARY_OP_SUBSCR_INIT_CALL - { - sub = stack_pointer[-1]; - _PyInterpreterFrame* pushed_frame = _PyFrame_PushUnchecked(tstate, getitem, 2, frame); - pushed_frame->localsplus[0] = container; - pushed_frame->localsplus[1] = sub; - frame->return_offset = 6u ; - new_frame = PyStackRef_Wrap(pushed_frame); - } - // _PUSH_FRAME - { - assert(tstate->interp->eval_frame == NULL); - _PyInterpreterFrame *temp = PyStackRef_Unwrap(new_frame); - stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - assert(temp->previous == frame || temp->previous->previous == frame); - CALL_STAT_INC(inlined_py_calls); - frame = tstate->current_frame = temp; - tstate->py_recursion_remaining--; - LOAD_SP(); - LOAD_IP(0); - LLTRACE_RESUME_FRAME(); - } - TRACING_DISPATCH(); - } - - TRACING_TARGET(BINARY_OP_SUBSCR_LIST_INT) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = BINARY_OP_SUBSCR_LIST_INT; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 6; - INSTRUCTION_STATS(BINARY_OP_SUBSCR_LIST_INT); - opcode = BINARY_OP_SUBSCR_LIST_INT; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; - (void)(_old_stack_level); - static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5, "incorrect cache size"); - _PyStackRef value; - _PyStackRef nos; - _PyStackRef list_st; - _PyStackRef sub_st; - _PyStackRef res; - // _GUARD_TOS_INT - { - value = stack_pointer[-1]; - PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); - if (!_PyLong_CheckExactAndCompact(value_o)) { - UPDATE_MISS_STATS(BINARY_OP); - assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); - JUMP_TO_PREDICTED(TRACING_BINARY_OP); - } - } - // _GUARD_NOS_LIST - { - nos = stack_pointer[-2]; - PyObject *o = PyStackRef_AsPyObjectBorrow(nos); - if (!PyList_CheckExact(o)) { - UPDATE_MISS_STATS(BINARY_OP); - assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); - JUMP_TO_PREDICTED(TRACING_BINARY_OP); - } - } - /* Skip 5 cache entries */ - // _BINARY_OP_SUBSCR_LIST_INT - { - sub_st = value; - list_st = nos; - PyObject *sub = PyStackRef_AsPyObjectBorrow(sub_st); - PyObject *list = PyStackRef_AsPyObjectBorrow(list_st); - assert(PyLong_CheckExact(sub)); - assert(PyList_CheckExact(list)); - if (!_PyLong_IsNonNegativeCompact((PyLongObject *)sub)) { - UPDATE_MISS_STATS(BINARY_OP); - assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); - JUMP_TO_PREDICTED(TRACING_BINARY_OP); - } - Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0]; - #ifdef Py_GIL_DISABLED - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *res_o = _PyList_GetItemRef((PyListObject*)list, index); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (res_o == NULL) { - UPDATE_MISS_STATS(BINARY_OP); - assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); - JUMP_TO_PREDICTED(TRACING_BINARY_OP); - } - STAT_INC(BINARY_OP, hit); - res = PyStackRef_FromPyObjectSteal(res_o); - #else - if (index >= PyList_GET_SIZE(list)) { - UPDATE_MISS_STATS(BINARY_OP); - assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); - JUMP_TO_PREDICTED(TRACING_BINARY_OP); - } - STAT_INC(BINARY_OP, hit); - PyObject *res_o = PyList_GET_ITEM(list, index); - assert(res_o != NULL); - res = PyStackRef_FromPyObjectNew(res_o); - #endif - STAT_INC(BINARY_OP, hit); - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyStackRef tmp = list_st; - list_st = res; - stack_pointer[-2] = list_st; - PyStackRef_CLOSE(tmp); - tmp = sub_st; - sub_st = PyStackRef_NULL; - stack_pointer[-1] = sub_st; - PyStackRef_CLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - } - TRACING_DISPATCH(); - } - - TRACING_TARGET(BINARY_OP_SUBSCR_LIST_SLICE) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = BINARY_OP_SUBSCR_LIST_SLICE; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 6; - INSTRUCTION_STATS(BINARY_OP_SUBSCR_LIST_SLICE); - opcode = BINARY_OP_SUBSCR_LIST_SLICE; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; - (void)(_old_stack_level); - static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5, "incorrect cache size"); - _PyStackRef tos; - _PyStackRef nos; - _PyStackRef list_st; - _PyStackRef sub_st; - _PyStackRef res; - // _GUARD_TOS_SLICE - { - tos = stack_pointer[-1]; - PyObject *o = PyStackRef_AsPyObjectBorrow(tos); - if (!PySlice_Check(o)) { - UPDATE_MISS_STATS(BINARY_OP); - assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); - JUMP_TO_PREDICTED(TRACING_BINARY_OP); - } - } - // _GUARD_NOS_LIST - { - nos = stack_pointer[-2]; - PyObject *o = PyStackRef_AsPyObjectBorrow(nos); - if (!PyList_CheckExact(o)) { - UPDATE_MISS_STATS(BINARY_OP); - assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); - JUMP_TO_PREDICTED(TRACING_BINARY_OP); - } - } - /* Skip 5 cache entries */ - // _BINARY_OP_SUBSCR_LIST_SLICE - { - sub_st = tos; - list_st = nos; - PyObject *sub = PyStackRef_AsPyObjectBorrow(sub_st); - PyObject *list = PyStackRef_AsPyObjectBorrow(list_st); - assert(PySlice_Check(sub)); - assert(PyList_CheckExact(list)); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *res_o = _PyList_SliceSubscript(list, sub); - stack_pointer = _PyFrame_GetStackPointer(frame); - STAT_INC(BINARY_OP, hit); - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyStackRef tmp = sub_st; - sub_st = PyStackRef_NULL; - stack_pointer[-1] = sub_st; - PyStackRef_CLOSE(tmp); - tmp = list_st; - list_st = PyStackRef_NULL; - stack_pointer[-2] = list_st; - PyStackRef_CLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); - if (res_o == NULL) { - TRACING_JUMP_TO_LABEL(error); - } - res = PyStackRef_FromPyObjectSteal(res_o); - } - stack_pointer[0] = res; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - TRACING_DISPATCH(); - } - - TRACING_TARGET(BINARY_OP_SUBSCR_STR_INT) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = BINARY_OP_SUBSCR_STR_INT; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 6; - INSTRUCTION_STATS(BINARY_OP_SUBSCR_STR_INT); - opcode = BINARY_OP_SUBSCR_STR_INT; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; - (void)(_old_stack_level); - static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5, "incorrect cache size"); - _PyStackRef value; - _PyStackRef nos; - _PyStackRef str_st; - _PyStackRef sub_st; - _PyStackRef res; - // _GUARD_TOS_INT - { - value = stack_pointer[-1]; - PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); - if (!_PyLong_CheckExactAndCompact(value_o)) { - UPDATE_MISS_STATS(BINARY_OP); - assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); - JUMP_TO_PREDICTED(TRACING_BINARY_OP); - } - } - // _GUARD_NOS_UNICODE - { - nos = stack_pointer[-2]; - PyObject *o = PyStackRef_AsPyObjectBorrow(nos); - if (!PyUnicode_CheckExact(o)) { - UPDATE_MISS_STATS(BINARY_OP); - assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); - JUMP_TO_PREDICTED(TRACING_BINARY_OP); - } - } - /* Skip 5 cache entries */ - // _BINARY_OP_SUBSCR_STR_INT - { - sub_st = value; - str_st = nos; - PyObject *sub = PyStackRef_AsPyObjectBorrow(sub_st); - PyObject *str = PyStackRef_AsPyObjectBorrow(str_st); - assert(PyLong_CheckExact(sub)); - assert(PyUnicode_CheckExact(str)); - if (!_PyLong_IsNonNegativeCompact((PyLongObject *)sub)) { - UPDATE_MISS_STATS(BINARY_OP); - assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); - JUMP_TO_PREDICTED(TRACING_BINARY_OP); - } - Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0]; - if (PyUnicode_GET_LENGTH(str) <= index) { - UPDATE_MISS_STATS(BINARY_OP); - assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); - JUMP_TO_PREDICTED(TRACING_BINARY_OP); - } - Py_UCS4 c = PyUnicode_READ_CHAR(str, index); - if (Py_ARRAY_LENGTH(_Py_SINGLETON(strings).ascii) <= c) { - UPDATE_MISS_STATS(BINARY_OP); - assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); - JUMP_TO_PREDICTED(TRACING_BINARY_OP); - } - STAT_INC(BINARY_OP, hit); - PyObject *res_o = (PyObject*)&_Py_SINGLETON(strings).ascii[c]; - PyStackRef_CLOSE_SPECIALIZED(sub_st, _PyLong_ExactDealloc); - stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(str_st); - stack_pointer = _PyFrame_GetStackPointer(frame); - res = PyStackRef_FromPyObjectBorrow(res_o); - } - stack_pointer[0] = res; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - TRACING_DISPATCH(); - } - - TRACING_TARGET(BINARY_OP_SUBSCR_TUPLE_INT) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = BINARY_OP_SUBSCR_TUPLE_INT; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 6; - INSTRUCTION_STATS(BINARY_OP_SUBSCR_TUPLE_INT); - opcode = BINARY_OP_SUBSCR_TUPLE_INT; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; - (void)(_old_stack_level); - static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5, "incorrect cache size"); - _PyStackRef value; - _PyStackRef nos; - _PyStackRef tuple_st; - _PyStackRef sub_st; - _PyStackRef res; - // _GUARD_TOS_INT - { - value = stack_pointer[-1]; - PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); - if (!_PyLong_CheckExactAndCompact(value_o)) { - UPDATE_MISS_STATS(BINARY_OP); - assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); - JUMP_TO_PREDICTED(TRACING_BINARY_OP); - } - } - // _GUARD_NOS_TUPLE - { - nos = stack_pointer[-2]; - PyObject *o = PyStackRef_AsPyObjectBorrow(nos); - if (!PyTuple_CheckExact(o)) { - UPDATE_MISS_STATS(BINARY_OP); - assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); - JUMP_TO_PREDICTED(TRACING_BINARY_OP); - } - } - /* Skip 5 cache entries */ - // _BINARY_OP_SUBSCR_TUPLE_INT - { - sub_st = value; - tuple_st = nos; - PyObject *sub = PyStackRef_AsPyObjectBorrow(sub_st); - PyObject *tuple = PyStackRef_AsPyObjectBorrow(tuple_st); - assert(PyLong_CheckExact(sub)); - assert(PyTuple_CheckExact(tuple)); - if (!_PyLong_IsNonNegativeCompact((PyLongObject *)sub)) { - UPDATE_MISS_STATS(BINARY_OP); - assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); - JUMP_TO_PREDICTED(TRACING_BINARY_OP); - } - Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0]; - if (index >= PyTuple_GET_SIZE(tuple)) { - UPDATE_MISS_STATS(BINARY_OP); - assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); - JUMP_TO_PREDICTED(TRACING_BINARY_OP); - } - STAT_INC(BINARY_OP, hit); - PyObject *res_o = PyTuple_GET_ITEM(tuple, index); - assert(res_o != NULL); - PyStackRef_CLOSE_SPECIALIZED(sub_st, _PyLong_ExactDealloc); - res = PyStackRef_FromPyObjectNew(res_o); - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyStackRef tmp = tuple_st; - tuple_st = res; - stack_pointer[-1] = tuple_st; - PyStackRef_CLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); - } - TRACING_DISPATCH(); - } - - TRACING_TARGET(BINARY_OP_SUBTRACT_FLOAT) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = BINARY_OP_SUBTRACT_FLOAT; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 6; - INSTRUCTION_STATS(BINARY_OP_SUBTRACT_FLOAT); - opcode = BINARY_OP_SUBTRACT_FLOAT; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; - (void)(_old_stack_level); - static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5, "incorrect cache size"); - _PyStackRef value; - _PyStackRef left; - _PyStackRef right; - _PyStackRef res; - // _GUARD_TOS_FLOAT - { - value = stack_pointer[-1]; - PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); - if (!PyFloat_CheckExact(value_o)) { - UPDATE_MISS_STATS(BINARY_OP); - assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); - JUMP_TO_PREDICTED(TRACING_BINARY_OP); - } - } - // _GUARD_NOS_FLOAT - { - left = stack_pointer[-2]; - PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); - if (!PyFloat_CheckExact(left_o)) { - UPDATE_MISS_STATS(BINARY_OP); - assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); - JUMP_TO_PREDICTED(TRACING_BINARY_OP); - } - } - /* Skip 5 cache entries */ - // _BINARY_OP_SUBTRACT_FLOAT - { - right = value; - PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); - PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - assert(PyFloat_CheckExact(left_o)); - assert(PyFloat_CheckExact(right_o)); - STAT_INC(BINARY_OP, hit); - double dres = - ((PyFloatObject *)left_o)->ob_fval - - ((PyFloatObject *)right_o)->ob_fval; - res = _PyFloat_FromDouble_ConsumeInputs(left, right, dres); - if (PyStackRef_IsNull(res)) { - TRACING_JUMP_TO_LABEL(pop_2_error); - } - } - stack_pointer[-2] = res; - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - TRACING_DISPATCH(); - } - - TRACING_TARGET(BINARY_OP_SUBTRACT_INT) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = BINARY_OP_SUBTRACT_INT; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 6; - INSTRUCTION_STATS(BINARY_OP_SUBTRACT_INT); - opcode = BINARY_OP_SUBTRACT_INT; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; - (void)(_old_stack_level); - static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5, "incorrect cache size"); - _PyStackRef value; - _PyStackRef left; - _PyStackRef right; - _PyStackRef res; - // _GUARD_TOS_INT - { - value = stack_pointer[-1]; - PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); - if (!_PyLong_CheckExactAndCompact(value_o)) { - UPDATE_MISS_STATS(BINARY_OP); - assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); - JUMP_TO_PREDICTED(TRACING_BINARY_OP); - } - } - // _GUARD_NOS_INT - { - left = stack_pointer[-2]; - PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); - if (!_PyLong_CheckExactAndCompact(left_o)) { - UPDATE_MISS_STATS(BINARY_OP); - assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); - JUMP_TO_PREDICTED(TRACING_BINARY_OP); - } - } - /* Skip 5 cache entries */ - // _BINARY_OP_SUBTRACT_INT - { - right = value; - PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); - PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - assert(PyLong_CheckExact(left_o)); - assert(PyLong_CheckExact(right_o)); - assert(_PyLong_BothAreCompact((PyLongObject *)left_o, (PyLongObject *)right_o)); - STAT_INC(BINARY_OP, hit); - res = _PyCompactLong_Subtract((PyLongObject *)left_o, (PyLongObject *)right_o); - if (PyStackRef_IsNull(res)) { - UPDATE_MISS_STATS(BINARY_OP); - assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); - JUMP_TO_PREDICTED(TRACING_BINARY_OP); - } - PyStackRef_CLOSE_SPECIALIZED(right, _PyLong_ExactDealloc); - PyStackRef_CLOSE_SPECIALIZED(left, _PyLong_ExactDealloc); - } - stack_pointer[-2] = res; - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - TRACING_DISPATCH(); - } - - TRACING_TARGET(BINARY_SLICE) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = BINARY_SLICE; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(BINARY_SLICE); - opcode = BINARY_SLICE; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; - (void)(_old_stack_level); - _PyStackRef container; - _PyStackRef start; - _PyStackRef stop; - _PyStackRef res; - // _SPECIALIZE_BINARY_SLICE - { - #if ENABLE_SPECIALIZATION - OPCODE_DEFERRED_INC(BINARY_SLICE); - #endif /* ENABLE_SPECIALIZATION */ - } - // _BINARY_SLICE - { - stop = stack_pointer[-1]; - start = stack_pointer[-2]; - container = stack_pointer[-3]; - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *slice = _PyBuildSlice_ConsumeRefs(PyStackRef_AsPyObjectSteal(start), - PyStackRef_AsPyObjectSteal(stop)); - stack_pointer = _PyFrame_GetStackPointer(frame); - PyObject *res_o; - if (slice == NULL) { - res_o = NULL; - } - else { - stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - res_o = PyObject_GetItem(PyStackRef_AsPyObjectBorrow(container), slice); - Py_DECREF(slice); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += 2; - } - stack_pointer += -3; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(container); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (res_o == NULL) { - TRACING_JUMP_TO_LABEL(error); - } - res = PyStackRef_FromPyObjectSteal(res_o); - } - stack_pointer[0] = res; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - TRACING_DISPATCH(); - } - - TRACING_TARGET(BUILD_INTERPOLATION) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = BUILD_INTERPOLATION; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(BUILD_INTERPOLATION); - opcode = BUILD_INTERPOLATION; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; - (void)(_old_stack_level); - _PyStackRef value; - _PyStackRef str; - _PyStackRef *format; - _PyStackRef interpolation; - format = &stack_pointer[-(oparg & 1)]; - str = stack_pointer[-1 - (oparg & 1)]; - value = stack_pointer[-2 - (oparg & 1)]; - PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); - PyObject *str_o = PyStackRef_AsPyObjectBorrow(str); - int conversion = oparg >> 2; - PyObject *format_o; - if (oparg & 1) { - format_o = PyStackRef_AsPyObjectBorrow(format[0]); - } - else { - format_o = &_Py_STR(empty); - } - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *interpolation_o = _PyInterpolation_Build(value_o, str_o, conversion, format_o); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (oparg & 1) { - stack_pointer += -(oparg & 1); - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(format[0]); - stack_pointer = _PyFrame_GetStackPointer(frame); - } - else { - stack_pointer += -(oparg & 1); - } - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(str); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(value); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (interpolation_o == NULL) { - TRACING_JUMP_TO_LABEL(error); - } - interpolation = PyStackRef_FromPyObjectSteal(interpolation_o); - stack_pointer[0] = interpolation; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - TRACING_DISPATCH(); - } - - TRACING_TARGET(BUILD_LIST) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = BUILD_LIST; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(BUILD_LIST); - opcode = BUILD_LIST; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; - (void)(_old_stack_level); - _PyStackRef *values; - _PyStackRef list; - values = &stack_pointer[-oparg]; - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *list_o = _PyList_FromStackRefStealOnSuccess(values, oparg); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (list_o == NULL) { - TRACING_JUMP_TO_LABEL(error); - } - list = PyStackRef_FromPyObjectStealMortal(list_o); - stack_pointer[-oparg] = list; - stack_pointer += 1 - oparg; - assert(WITHIN_STACK_BOUNDS()); - TRACING_DISPATCH(); - } - - TRACING_TARGET(BUILD_MAP) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = BUILD_MAP; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(BUILD_MAP); - opcode = BUILD_MAP; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; - (void)(_old_stack_level); - _PyStackRef *values; - _PyStackRef map; - values = &stack_pointer[-oparg*2]; - STACKREFS_TO_PYOBJECTS(values, oparg*2, values_o); - if (CONVERSION_FAILED(values_o)) { - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyStackRef tmp; - for (int _i = oparg*2; --_i >= 0;) { - tmp = values[_i]; - values[_i] = PyStackRef_NULL; - PyStackRef_CLOSE(tmp); - } - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -oparg*2; - assert(WITHIN_STACK_BOUNDS()); - TRACING_JUMP_TO_LABEL(error); - } - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *map_o = _PyDict_FromItems( - values_o, 2, - values_o+1, 2, - oparg); - stack_pointer = _PyFrame_GetStackPointer(frame); - STACKREFS_TO_PYOBJECTS_CLEANUP(values_o); - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyStackRef tmp; - for (int _i = oparg*2; --_i >= 0;) { - tmp = values[_i]; - values[_i] = PyStackRef_NULL; - PyStackRef_CLOSE(tmp); - } - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -oparg*2; - assert(WITHIN_STACK_BOUNDS()); - if (map_o == NULL) { - TRACING_JUMP_TO_LABEL(error); - } - map = PyStackRef_FromPyObjectStealMortal(map_o); - stack_pointer[0] = map; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - TRACING_DISPATCH(); - } - - TRACING_TARGET(BUILD_SET) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = BUILD_SET; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(BUILD_SET); - opcode = BUILD_SET; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; - (void)(_old_stack_level); - _PyStackRef *values; - _PyStackRef set; - values = &stack_pointer[-oparg]; - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *set_o = PySet_New(NULL); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (set_o == NULL) { - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyStackRef tmp; - for (int _i = oparg; --_i >= 0;) { - tmp = values[_i]; - values[_i] = PyStackRef_NULL; - PyStackRef_CLOSE(tmp); - } - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -oparg; - assert(WITHIN_STACK_BOUNDS()); - TRACING_JUMP_TO_LABEL(error); - } - int err = 0; - for (Py_ssize_t i = 0; i < oparg; i++) { - _PyStackRef value = values[i]; - values[i] = PyStackRef_NULL; - if (err == 0) { - _PyFrame_SetStackPointer(frame, stack_pointer); - err = _PySet_AddTakeRef((PySetObject *)set_o, PyStackRef_AsPyObjectSteal(value)); - stack_pointer = _PyFrame_GetStackPointer(frame); - } - else { - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(value); - stack_pointer = _PyFrame_GetStackPointer(frame); - } - } - if (err) { - stack_pointer += -oparg; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - Py_DECREF(set_o); - stack_pointer = _PyFrame_GetStackPointer(frame); - TRACING_JUMP_TO_LABEL(error); - } - set = PyStackRef_FromPyObjectStealMortal(set_o); - stack_pointer[-oparg] = set; - stack_pointer += 1 - oparg; - assert(WITHIN_STACK_BOUNDS()); - TRACING_DISPATCH(); - } - - TRACING_TARGET(BUILD_SLICE) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = BUILD_SLICE; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(BUILD_SLICE); - opcode = BUILD_SLICE; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; - (void)(_old_stack_level); - _PyStackRef *args; - _PyStackRef slice; - args = &stack_pointer[-oparg]; - PyObject *start_o = PyStackRef_AsPyObjectBorrow(args[0]); - PyObject *stop_o = PyStackRef_AsPyObjectBorrow(args[1]); - PyObject *step_o = oparg == 3 ? PyStackRef_AsPyObjectBorrow(args[2]) : NULL; - PyObject *slice_o = PySlice_New(start_o, stop_o, step_o); - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyStackRef tmp; - for (int _i = oparg; --_i >= 0;) { - tmp = args[_i]; - args[_i] = PyStackRef_NULL; - PyStackRef_CLOSE(tmp); - } - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -oparg; - assert(WITHIN_STACK_BOUNDS()); - if (slice_o == NULL) { - TRACING_JUMP_TO_LABEL(error); - } - slice = PyStackRef_FromPyObjectStealMortal(slice_o); - stack_pointer[0] = slice; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - TRACING_DISPATCH(); - } - - TRACING_TARGET(BUILD_STRING) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = BUILD_STRING; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(BUILD_STRING); - opcode = BUILD_STRING; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; - (void)(_old_stack_level); - _PyStackRef *pieces; - _PyStackRef str; - pieces = &stack_pointer[-oparg]; - STACKREFS_TO_PYOBJECTS(pieces, oparg, pieces_o); - if (CONVERSION_FAILED(pieces_o)) { - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyStackRef tmp; - for (int _i = oparg; --_i >= 0;) { - tmp = pieces[_i]; - pieces[_i] = PyStackRef_NULL; - PyStackRef_CLOSE(tmp); - } - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -oparg; - assert(WITHIN_STACK_BOUNDS()); - TRACING_JUMP_TO_LABEL(error); - } - PyObject *str_o = _PyUnicode_JoinArray(&_Py_STR(empty), pieces_o, oparg); - STACKREFS_TO_PYOBJECTS_CLEANUP(pieces_o); - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyStackRef tmp; - for (int _i = oparg; --_i >= 0;) { - tmp = pieces[_i]; - pieces[_i] = PyStackRef_NULL; - PyStackRef_CLOSE(tmp); - } - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -oparg; - assert(WITHIN_STACK_BOUNDS()); - if (str_o == NULL) { - TRACING_JUMP_TO_LABEL(error); - } - str = PyStackRef_FromPyObjectSteal(str_o); - stack_pointer[0] = str; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - TRACING_DISPATCH(); - } - - TRACING_TARGET(BUILD_TEMPLATE) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = BUILD_TEMPLATE; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(BUILD_TEMPLATE); - opcode = BUILD_TEMPLATE; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; - (void)(_old_stack_level); - _PyStackRef strings; - _PyStackRef interpolations; - _PyStackRef template; - interpolations = stack_pointer[-1]; - strings = stack_pointer[-2]; - PyObject *strings_o = PyStackRef_AsPyObjectBorrow(strings); - PyObject *interpolations_o = PyStackRef_AsPyObjectBorrow(interpolations); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *template_o = _PyTemplate_Build(strings_o, interpolations_o); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(interpolations); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(strings); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (template_o == NULL) { - TRACING_JUMP_TO_LABEL(error); - } - template = PyStackRef_FromPyObjectSteal(template_o); - stack_pointer[0] = template; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - TRACING_DISPATCH(); - } - - TRACING_TARGET(BUILD_TUPLE) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = BUILD_TUPLE; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(BUILD_TUPLE); - opcode = BUILD_TUPLE; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; - (void)(_old_stack_level); - _PyStackRef *values; - _PyStackRef tup; - values = &stack_pointer[-oparg]; - PyObject *tup_o = _PyTuple_FromStackRefStealOnSuccess(values, oparg); - if (tup_o == NULL) { - TRACING_JUMP_TO_LABEL(error); - } - tup = PyStackRef_FromPyObjectStealMortal(tup_o); - stack_pointer[-oparg] = tup; - stack_pointer += 1 - oparg; - assert(WITHIN_STACK_BOUNDS()); - TRACING_DISPATCH(); - } - - TRACING_TARGET(CACHE) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = CACHE; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(CACHE); - opcode = CACHE; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; - (void)(_old_stack_level); - assert(0 && "Executing a cache."); - Py_FatalError("Executing a cache."); - TRACING_DISPATCH(); - } - - TRACING_TARGET(CALL) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = CALL; - (void)(opcode); - #endif - frame->instr_ptr = next_instr; - next_instr += 4; - INSTRUCTION_STATS(CALL); - PREDICTED_TRACING_CALL:; - _Py_CODEUNIT* const this_instr = next_instr - 4; - (void)this_instr; - opcode = CALL; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; - (void)(_old_stack_level); - _PyStackRef callable; - _PyStackRef self_or_null; - _PyStackRef *args; - _PyStackRef res; - // _SPECIALIZE_CALL - { - self_or_null = stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; - uint16_t counter = read_u16(&this_instr[1].cache); - (void)counter; - #if ENABLE_SPECIALIZATION_FT - if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { - next_instr = this_instr; - _PyFrame_SetStackPointer(frame, stack_pointer); - _Py_Specialize_Call(callable, next_instr, oparg + !PyStackRef_IsNull(self_or_null)); - stack_pointer = _PyFrame_GetStackPointer(frame); - TRACING_SPECIALIZE_DISPATCH_SAME_OPARG() - } - OPCODE_DEFERRED_INC(CALL); - ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); - #endif /* ENABLE_SPECIALIZATION_FT */ - } - /* Skip 2 cache entries */ - // _MAYBE_EXPAND_METHOD - { - if (PyStackRef_TYPE(callable) == &PyMethod_Type && PyStackRef_IsNull(self_or_null)) { - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); - PyObject *self = ((PyMethodObject *)callable_o)->im_self; - self_or_null = PyStackRef_FromPyObjectNew(self); - PyObject *method = ((PyMethodObject *)callable_o)->im_func; - _PyStackRef temp = callable; - callable = PyStackRef_FromPyObjectNew(method); - stack_pointer[-2 - oparg] = callable; - stack_pointer[-1 - oparg] = self_or_null; - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(temp); - stack_pointer = _PyFrame_GetStackPointer(frame); - } - } - // _DO_CALL - { - args = &stack_pointer[-oparg]; - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); - int total_args = oparg; - _PyStackRef *arguments = args; - if (!PyStackRef_IsNull(self_or_null)) { - arguments--; - total_args++; - } - if (Py_TYPE(callable_o) == &PyFunction_Type && - tstate->interp->eval_frame == NULL && - ((PyFunctionObject *)callable_o)->vectorcall == _PyFunction_Vectorcall) - { - int code_flags = ((PyCodeObject*)PyFunction_GET_CODE(callable_o))->co_flags; - PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable_o)); - stack_pointer[-2 - oparg] = callable; - stack_pointer[-1 - oparg] = self_or_null; - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyInterpreterFrame *new_frame = _PyEvalFramePushAndInit( - tstate, callable, locals, - arguments, total_args, NULL, frame - ); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -2 - oparg; - assert(WITHIN_STACK_BOUNDS()); - if (new_frame == NULL) { - TRACING_JUMP_TO_LABEL(error); - } - frame->return_offset = 4u ; - TRACING_DISPATCH_INLINED(new_frame); - } - STACKREFS_TO_PYOBJECTS(arguments, total_args, args_o); - if (CONVERSION_FAILED(args_o)) { - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyStackRef tmp; - for (int _i = oparg; --_i >= 0;) { - tmp = args[_i]; - args[_i] = PyStackRef_NULL; - stack_pointer[-2 - oparg] = callable; - stack_pointer[-1 - oparg] = self_or_null; - PyStackRef_CLOSE(tmp); - } - tmp = self_or_null; - self_or_null = PyStackRef_NULL; - stack_pointer[-1 - oparg] = self_or_null; - PyStackRef_XCLOSE(tmp); - tmp = callable; - callable = PyStackRef_NULL; - stack_pointer[-2 - oparg] = callable; - PyStackRef_CLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -2 - oparg; - assert(WITHIN_STACK_BOUNDS()); - TRACING_JUMP_TO_LABEL(error); - } - stack_pointer[-2 - oparg] = callable; - stack_pointer[-1 - oparg] = self_or_null; - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *res_o = PyObject_Vectorcall( - callable_o, args_o, - total_args | PY_VECTORCALL_ARGUMENTS_OFFSET, - NULL); - stack_pointer = _PyFrame_GetStackPointer(frame); - STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); - if (opcode == INSTRUMENTED_CALL) { - PyObject *arg = total_args == 0 ? - &_PyInstrumentation_MISSING : PyStackRef_AsPyObjectBorrow(arguments[0]); - if (res_o == NULL) { - _PyFrame_SetStackPointer(frame, stack_pointer); - _Py_call_instrumentation_exc2( - tstate, PY_MONITORING_EVENT_C_RAISE, - frame, this_instr, callable_o, arg); - stack_pointer = _PyFrame_GetStackPointer(frame); - } - else { - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = _Py_call_instrumentation_2args( - tstate, PY_MONITORING_EVENT_C_RETURN, - frame, this_instr, callable_o, arg); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err < 0) { - _PyFrame_SetStackPointer(frame, stack_pointer); - Py_CLEAR(res_o); - stack_pointer = _PyFrame_GetStackPointer(frame); - } - } - } - assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyStackRef tmp; - for (int _i = oparg; --_i >= 0;) { - tmp = args[_i]; - args[_i] = PyStackRef_NULL; - PyStackRef_CLOSE(tmp); - } - tmp = self_or_null; - self_or_null = PyStackRef_NULL; - stack_pointer[-1 - oparg] = self_or_null; - PyStackRef_XCLOSE(tmp); - tmp = callable; - callable = PyStackRef_NULL; - stack_pointer[-2 - oparg] = callable; - PyStackRef_CLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -2 - oparg; - assert(WITHIN_STACK_BOUNDS()); - if (res_o == NULL) { - TRACING_JUMP_TO_LABEL(error); - } - res = PyStackRef_FromPyObjectSteal(res_o); - } - // _CHECK_PERIODIC_AT_END - { - stack_pointer[0] = res; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = check_periodics(tstate); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err != 0) { - TRACING_JUMP_TO_LABEL(error); - } - } - TRACING_DISPATCH(); - } - - TRACING_TARGET(CALL_ALLOC_AND_ENTER_INIT) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = CALL_ALLOC_AND_ENTER_INIT; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 4; - INSTRUCTION_STATS(CALL_ALLOC_AND_ENTER_INIT); - opcode = CALL_ALLOC_AND_ENTER_INIT; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; - (void)(_old_stack_level); - static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); - _PyStackRef callable; - _PyStackRef self_or_null; - _PyStackRef init; - _PyStackRef self; - _PyStackRef *args; - _PyStackRef init_frame; - _PyStackRef new_frame; - /* Skip 1 cache entry */ - // _CHECK_PEP_523 - { - if (tstate->interp->eval_frame) { - UPDATE_MISS_STATS(CALL); - assert(_PyOpcode_Deopt[opcode] == (CALL)); - JUMP_TO_PREDICTED(TRACING_CALL); - } - } - // _CHECK_AND_ALLOCATE_OBJECT - { - self_or_null = stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; - uint32_t type_version = read_u32(&this_instr[2].cache); - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); - if (!PyStackRef_IsNull(self_or_null)) { - UPDATE_MISS_STATS(CALL); - assert(_PyOpcode_Deopt[opcode] == (CALL)); - JUMP_TO_PREDICTED(TRACING_CALL); - } - if (!PyType_Check(callable_o)) { - UPDATE_MISS_STATS(CALL); - assert(_PyOpcode_Deopt[opcode] == (CALL)); - JUMP_TO_PREDICTED(TRACING_CALL); - } - PyTypeObject *tp = (PyTypeObject *)callable_o; - if (FT_ATOMIC_LOAD_UINT32_RELAXED(tp->tp_version_tag) != type_version) { - UPDATE_MISS_STATS(CALL); - assert(_PyOpcode_Deopt[opcode] == (CALL)); - JUMP_TO_PREDICTED(TRACING_CALL); - } - assert(tp->tp_new == PyBaseObject_Type.tp_new); - assert(tp->tp_flags & Py_TPFLAGS_HEAPTYPE); - assert(tp->tp_alloc == PyType_GenericAlloc); - PyHeapTypeObject *cls = (PyHeapTypeObject *)callable_o; - PyFunctionObject *init_func = (PyFunctionObject *)FT_ATOMIC_LOAD_PTR_ACQUIRE(cls->_spec_cache.init); - PyCodeObject *code = (PyCodeObject *)init_func->func_code; - if (!_PyThreadState_HasStackSpace(tstate, code->co_framesize + _Py_InitCleanup.co_framesize)) { - UPDATE_MISS_STATS(CALL); - assert(_PyOpcode_Deopt[opcode] == (CALL)); - JUMP_TO_PREDICTED(TRACING_CALL); - } - STAT_INC(CALL, hit); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *self_o = PyType_GenericAlloc(tp, 0); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (self_o == NULL) { - TRACING_JUMP_TO_LABEL(error); - } - self_or_null = PyStackRef_FromPyObjectSteal(self_o); - _PyStackRef temp = callable; - callable = PyStackRef_FromPyObjectNew(init_func); - stack_pointer[-2 - oparg] = callable; - stack_pointer[-1 - oparg] = self_or_null; - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(temp); - stack_pointer = _PyFrame_GetStackPointer(frame); - } - // _CREATE_INIT_FRAME - { - args = &stack_pointer[-oparg]; - self = self_or_null; - init = callable; - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyInterpreterFrame *shim = _PyFrame_PushTrampolineUnchecked( - tstate, (PyCodeObject *)&_Py_InitCleanup, 1, frame); - stack_pointer = _PyFrame_GetStackPointer(frame); - assert(_PyFrame_GetBytecode(shim)[0].op.code == EXIT_INIT_CHECK); - assert(_PyFrame_GetBytecode(shim)[1].op.code == RETURN_VALUE); - shim->localsplus[0] = PyStackRef_DUP(self); - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyInterpreterFrame *temp = _PyEvalFramePushAndInit( - tstate, init, NULL, args-1, oparg+1, NULL, shim); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -2 - oparg; - assert(WITHIN_STACK_BOUNDS()); - if (temp == NULL) { - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyEval_FrameClearAndPop(tstate, shim); - stack_pointer = _PyFrame_GetStackPointer(frame); - TRACING_JUMP_TO_LABEL(error); - } - frame->return_offset = 1 + INLINE_CACHE_ENTRIES_CALL; - tstate->py_recursion_remaining--; - init_frame = PyStackRef_Wrap(temp); - } - // _PUSH_FRAME - { - new_frame = init_frame; - assert(tstate->interp->eval_frame == NULL); - _PyInterpreterFrame *temp = PyStackRef_Unwrap(new_frame); - _PyFrame_SetStackPointer(frame, stack_pointer); - assert(temp->previous == frame || temp->previous->previous == frame); - CALL_STAT_INC(inlined_py_calls); - frame = tstate->current_frame = temp; - tstate->py_recursion_remaining--; - LOAD_SP(); - LOAD_IP(0); - LLTRACE_RESUME_FRAME(); - } - TRACING_DISPATCH(); - } - - TRACING_TARGET(CALL_BOUND_METHOD_EXACT_ARGS) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = CALL_BOUND_METHOD_EXACT_ARGS; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 4; - INSTRUCTION_STATS(CALL_BOUND_METHOD_EXACT_ARGS); - opcode = CALL_BOUND_METHOD_EXACT_ARGS; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; - (void)(_old_stack_level); - static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); - _PyStackRef callable; - _PyStackRef null; - _PyStackRef self_or_null; - _PyStackRef *args; - _PyStackRef new_frame; - /* Skip 1 cache entry */ - // _CHECK_PEP_523 - { - if (tstate->interp->eval_frame) { - UPDATE_MISS_STATS(CALL); - assert(_PyOpcode_Deopt[opcode] == (CALL)); - JUMP_TO_PREDICTED(TRACING_CALL); - } - } - // _CHECK_CALL_BOUND_METHOD_EXACT_ARGS - { - null = stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; - if (!PyStackRef_IsNull(null)) { - UPDATE_MISS_STATS(CALL); - assert(_PyOpcode_Deopt[opcode] == (CALL)); - JUMP_TO_PREDICTED(TRACING_CALL); - } - if (Py_TYPE(PyStackRef_AsPyObjectBorrow(callable)) != &PyMethod_Type) { - UPDATE_MISS_STATS(CALL); - assert(_PyOpcode_Deopt[opcode] == (CALL)); - JUMP_TO_PREDICTED(TRACING_CALL); - } - } - // _INIT_CALL_BOUND_METHOD_EXACT_ARGS - { - self_or_null = null; - assert(PyStackRef_IsNull(self_or_null)); - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); - STAT_INC(CALL, hit); - self_or_null = PyStackRef_FromPyObjectNew(((PyMethodObject *)callable_o)->im_self); - _PyStackRef temp = callable; - callable = PyStackRef_FromPyObjectNew(((PyMethodObject *)callable_o)->im_func); - stack_pointer[-2 - oparg] = callable; - stack_pointer[-1 - oparg] = self_or_null; - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(temp); - stack_pointer = _PyFrame_GetStackPointer(frame); - } - // flush - // _CHECK_FUNCTION_VERSION - { - uint32_t func_version = read_u32(&this_instr[2].cache); - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); - if (!PyFunction_Check(callable_o)) { - UPDATE_MISS_STATS(CALL); - assert(_PyOpcode_Deopt[opcode] == (CALL)); - JUMP_TO_PREDICTED(TRACING_CALL); - } - PyFunctionObject *func = (PyFunctionObject *)callable_o; - if (func->func_version != func_version) { - UPDATE_MISS_STATS(CALL); - assert(_PyOpcode_Deopt[opcode] == (CALL)); - JUMP_TO_PREDICTED(TRACING_CALL); - } - } - // _CHECK_FUNCTION_EXACT_ARGS - { - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); - assert(PyFunction_Check(callable_o)); - PyFunctionObject *func = (PyFunctionObject *)callable_o; - PyCodeObject *code = (PyCodeObject *)func->func_code; - if (code->co_argcount != oparg + (!PyStackRef_IsNull(self_or_null))) { - UPDATE_MISS_STATS(CALL); - assert(_PyOpcode_Deopt[opcode] == (CALL)); - JUMP_TO_PREDICTED(TRACING_CALL); - } - } - // _CHECK_STACK_SPACE - { - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); - PyFunctionObject *func = (PyFunctionObject *)callable_o; - PyCodeObject *code = (PyCodeObject *)func->func_code; - if (!_PyThreadState_HasStackSpace(tstate, code->co_framesize)) { - UPDATE_MISS_STATS(CALL); - assert(_PyOpcode_Deopt[opcode] == (CALL)); - JUMP_TO_PREDICTED(TRACING_CALL); - } - } - // _CHECK_RECURSION_REMAINING - { - if (tstate->py_recursion_remaining <= 1) { - UPDATE_MISS_STATS(CALL); - assert(_PyOpcode_Deopt[opcode] == (CALL)); - JUMP_TO_PREDICTED(TRACING_CALL); - } - } - // _INIT_CALL_PY_EXACT_ARGS - { - args = &stack_pointer[-oparg]; - int has_self = !PyStackRef_IsNull(self_or_null); - STAT_INC(CALL, hit); - _PyInterpreterFrame *pushed_frame = _PyFrame_PushUnchecked(tstate, callable, oparg + has_self, frame); - _PyStackRef *first_non_self_local = pushed_frame->localsplus + has_self; - pushed_frame->localsplus[0] = self_or_null; - for (int i = 0; i < oparg; i++) { - first_non_self_local[i] = args[i]; - } - new_frame = PyStackRef_Wrap(pushed_frame); - } - // _SAVE_RETURN_OFFSET - { - #if TIER_ONE - frame->return_offset = (uint16_t)(next_instr - this_instr); - #endif - #if TIER_TWO - frame->return_offset = oparg; - #endif - } - // _PUSH_FRAME - { - assert(tstate->interp->eval_frame == NULL); - _PyInterpreterFrame *temp = PyStackRef_Unwrap(new_frame); - stack_pointer += -2 - oparg; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - assert(temp->previous == frame || temp->previous->previous == frame); - CALL_STAT_INC(inlined_py_calls); - frame = tstate->current_frame = temp; - tstate->py_recursion_remaining--; - LOAD_SP(); - LOAD_IP(0); - LLTRACE_RESUME_FRAME(); - } - TRACING_DISPATCH(); - } - - TRACING_TARGET(CALL_BOUND_METHOD_GENERAL) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = CALL_BOUND_METHOD_GENERAL; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 4; - INSTRUCTION_STATS(CALL_BOUND_METHOD_GENERAL); - opcode = CALL_BOUND_METHOD_GENERAL; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; - (void)(_old_stack_level); - static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); - _PyStackRef callable; - _PyStackRef null; - _PyStackRef self_or_null; - _PyStackRef *args; - _PyStackRef new_frame; - /* Skip 1 cache entry */ - // _CHECK_PEP_523 - { - if (tstate->interp->eval_frame) { - UPDATE_MISS_STATS(CALL); - assert(_PyOpcode_Deopt[opcode] == (CALL)); - JUMP_TO_PREDICTED(TRACING_CALL); - } - } - // _CHECK_METHOD_VERSION - { - null = stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; - uint32_t func_version = read_u32(&this_instr[2].cache); - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); - if (Py_TYPE(callable_o) != &PyMethod_Type) { - UPDATE_MISS_STATS(CALL); - assert(_PyOpcode_Deopt[opcode] == (CALL)); - JUMP_TO_PREDICTED(TRACING_CALL); - } - PyObject *func = ((PyMethodObject *)callable_o)->im_func; - if (!PyFunction_Check(func)) { - UPDATE_MISS_STATS(CALL); - assert(_PyOpcode_Deopt[opcode] == (CALL)); - JUMP_TO_PREDICTED(TRACING_CALL); - } - if (((PyFunctionObject *)func)->func_version != func_version) { - UPDATE_MISS_STATS(CALL); - assert(_PyOpcode_Deopt[opcode] == (CALL)); - JUMP_TO_PREDICTED(TRACING_CALL); - } - if (!PyStackRef_IsNull(null)) { - UPDATE_MISS_STATS(CALL); - assert(_PyOpcode_Deopt[opcode] == (CALL)); - JUMP_TO_PREDICTED(TRACING_CALL); - } - } - // _EXPAND_METHOD - { - self_or_null = null; - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); - assert(PyStackRef_IsNull(self_or_null)); - assert(Py_TYPE(callable_o) == &PyMethod_Type); - self_or_null = PyStackRef_FromPyObjectNew(((PyMethodObject *)callable_o)->im_self); - _PyStackRef temp = callable; - callable = PyStackRef_FromPyObjectNew(((PyMethodObject *)callable_o)->im_func); - assert(PyStackRef_FunctionCheck(callable)); - stack_pointer[-2 - oparg] = callable; - stack_pointer[-1 - oparg] = self_or_null; - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(temp); - stack_pointer = _PyFrame_GetStackPointer(frame); - } - // flush - // _CHECK_RECURSION_REMAINING - { - if (tstate->py_recursion_remaining <= 1) { - UPDATE_MISS_STATS(CALL); - assert(_PyOpcode_Deopt[opcode] == (CALL)); - JUMP_TO_PREDICTED(TRACING_CALL); - } - } - // _PY_FRAME_GENERAL - { - args = &stack_pointer[-oparg]; - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); - int total_args = oparg; - if (!PyStackRef_IsNull(self_or_null)) { - args--; - total_args++; - } - assert(Py_TYPE(callable_o) == &PyFunction_Type); - int code_flags = ((PyCodeObject*)PyFunction_GET_CODE(callable_o))->co_flags; - PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable_o)); - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyInterpreterFrame *temp = _PyEvalFramePushAndInit( - tstate, callable, locals, - args, total_args, NULL, frame - ); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -2 - oparg; - assert(WITHIN_STACK_BOUNDS()); - if (temp == NULL) { - TRACING_JUMP_TO_LABEL(error); - } - new_frame = PyStackRef_Wrap(temp); - } - // _SAVE_RETURN_OFFSET - { - #if TIER_ONE - frame->return_offset = (uint16_t)(next_instr - this_instr); - #endif - #if TIER_TWO - frame->return_offset = oparg; - #endif - } - // _PUSH_FRAME - { - assert(tstate->interp->eval_frame == NULL); - _PyInterpreterFrame *temp = PyStackRef_Unwrap(new_frame); - _PyFrame_SetStackPointer(frame, stack_pointer); - assert(temp->previous == frame || temp->previous->previous == frame); - CALL_STAT_INC(inlined_py_calls); - frame = tstate->current_frame = temp; - tstate->py_recursion_remaining--; - LOAD_SP(); - LOAD_IP(0); - LLTRACE_RESUME_FRAME(); - } - TRACING_DISPATCH(); - } - - TRACING_TARGET(CALL_BUILTIN_CLASS) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = CALL_BUILTIN_CLASS; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 4; - INSTRUCTION_STATS(CALL_BUILTIN_CLASS); - opcode = CALL_BUILTIN_CLASS; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; - (void)(_old_stack_level); - static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); - _PyStackRef callable; - _PyStackRef self_or_null; - _PyStackRef *args; - _PyStackRef res; - /* Skip 1 cache entry */ - /* Skip 2 cache entries */ - // _CALL_BUILTIN_CLASS - { - args = &stack_pointer[-oparg]; - self_or_null = stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); - if (!PyType_Check(callable_o)) { - UPDATE_MISS_STATS(CALL); - assert(_PyOpcode_Deopt[opcode] == (CALL)); - JUMP_TO_PREDICTED(TRACING_CALL); - } - PyTypeObject *tp = (PyTypeObject *)callable_o; - int total_args = oparg; - _PyStackRef *arguments = args; - if (!PyStackRef_IsNull(self_or_null)) { - arguments--; - total_args++; - } - if (tp->tp_vectorcall == NULL) { - UPDATE_MISS_STATS(CALL); - assert(_PyOpcode_Deopt[opcode] == (CALL)); - JUMP_TO_PREDICTED(TRACING_CALL); - } - STAT_INC(CALL, hit); - STACKREFS_TO_PYOBJECTS(arguments, total_args, args_o); - if (CONVERSION_FAILED(args_o)) { - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyStackRef tmp; - for (int _i = oparg; --_i >= 0;) { - tmp = args[_i]; - args[_i] = PyStackRef_NULL; - PyStackRef_CLOSE(tmp); - } - tmp = self_or_null; - self_or_null = PyStackRef_NULL; - stack_pointer[-1 - oparg] = self_or_null; - PyStackRef_XCLOSE(tmp); - tmp = callable; - callable = PyStackRef_NULL; - stack_pointer[-2 - oparg] = callable; - PyStackRef_CLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -2 - oparg; - assert(WITHIN_STACK_BOUNDS()); - TRACING_JUMP_TO_LABEL(error); - } - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *res_o = tp->tp_vectorcall((PyObject *)tp, args_o, total_args, NULL); - stack_pointer = _PyFrame_GetStackPointer(frame); - STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyStackRef tmp; - for (int _i = oparg; --_i >= 0;) { - tmp = args[_i]; - args[_i] = PyStackRef_NULL; - PyStackRef_CLOSE(tmp); - } - tmp = self_or_null; - self_or_null = PyStackRef_NULL; - stack_pointer[-1 - oparg] = self_or_null; - PyStackRef_XCLOSE(tmp); - tmp = callable; - callable = PyStackRef_NULL; - stack_pointer[-2 - oparg] = callable; - PyStackRef_CLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -2 - oparg; - assert(WITHIN_STACK_BOUNDS()); - if (res_o == NULL) { - TRACING_JUMP_TO_LABEL(error); - } - res = PyStackRef_FromPyObjectSteal(res_o); - } - // _CHECK_PERIODIC_AT_END - { - stack_pointer[0] = res; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = check_periodics(tstate); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err != 0) { - TRACING_JUMP_TO_LABEL(error); - } - } - TRACING_DISPATCH(); - } - - TRACING_TARGET(CALL_BUILTIN_FAST) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = CALL_BUILTIN_FAST; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 4; - INSTRUCTION_STATS(CALL_BUILTIN_FAST); - opcode = CALL_BUILTIN_FAST; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; - (void)(_old_stack_level); - static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); - _PyStackRef callable; - _PyStackRef self_or_null; - _PyStackRef *args; - _PyStackRef res; - /* Skip 1 cache entry */ - /* Skip 2 cache entries */ - // _CALL_BUILTIN_FAST - { - args = &stack_pointer[-oparg]; - self_or_null = stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); - int total_args = oparg; - _PyStackRef *arguments = args; - if (!PyStackRef_IsNull(self_or_null)) { - arguments--; - total_args++; - } - if (!PyCFunction_CheckExact(callable_o)) { - UPDATE_MISS_STATS(CALL); - assert(_PyOpcode_Deopt[opcode] == (CALL)); - JUMP_TO_PREDICTED(TRACING_CALL); - } - if (PyCFunction_GET_FLAGS(callable_o) != METH_FASTCALL) { - UPDATE_MISS_STATS(CALL); - assert(_PyOpcode_Deopt[opcode] == (CALL)); - JUMP_TO_PREDICTED(TRACING_CALL); - } - STAT_INC(CALL, hit); - PyCFunction cfunc = PyCFunction_GET_FUNCTION(callable_o); - STACKREFS_TO_PYOBJECTS(arguments, total_args, args_o); - if (CONVERSION_FAILED(args_o)) { - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyStackRef tmp; - for (int _i = oparg; --_i >= 0;) { - tmp = args[_i]; - args[_i] = PyStackRef_NULL; - PyStackRef_CLOSE(tmp); - } - tmp = self_or_null; - self_or_null = PyStackRef_NULL; - stack_pointer[-1 - oparg] = self_or_null; - PyStackRef_XCLOSE(tmp); - tmp = callable; - callable = PyStackRef_NULL; - stack_pointer[-2 - oparg] = callable; - PyStackRef_CLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -2 - oparg; - assert(WITHIN_STACK_BOUNDS()); - TRACING_JUMP_TO_LABEL(error); - } - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *res_o = _PyCFunctionFast_CAST(cfunc)( - PyCFunction_GET_SELF(callable_o), - args_o, - total_args); - stack_pointer = _PyFrame_GetStackPointer(frame); - STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); - assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyStackRef tmp; - for (int _i = oparg; --_i >= 0;) { - tmp = args[_i]; - args[_i] = PyStackRef_NULL; - PyStackRef_CLOSE(tmp); - } - tmp = self_or_null; - self_or_null = PyStackRef_NULL; - stack_pointer[-1 - oparg] = self_or_null; - PyStackRef_XCLOSE(tmp); - tmp = callable; - callable = PyStackRef_NULL; - stack_pointer[-2 - oparg] = callable; - PyStackRef_CLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -2 - oparg; - assert(WITHIN_STACK_BOUNDS()); - if (res_o == NULL) { - TRACING_JUMP_TO_LABEL(error); - } - res = PyStackRef_FromPyObjectSteal(res_o); - } - // _CHECK_PERIODIC_AT_END - { - stack_pointer[0] = res; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = check_periodics(tstate); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err != 0) { - TRACING_JUMP_TO_LABEL(error); - } - } - TRACING_DISPATCH(); - } - - TRACING_TARGET(CALL_BUILTIN_FAST_WITH_KEYWORDS) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = CALL_BUILTIN_FAST_WITH_KEYWORDS; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 4; - INSTRUCTION_STATS(CALL_BUILTIN_FAST_WITH_KEYWORDS); - opcode = CALL_BUILTIN_FAST_WITH_KEYWORDS; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; - (void)(_old_stack_level); - static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); - _PyStackRef callable; - _PyStackRef self_or_null; - _PyStackRef *args; - _PyStackRef res; - /* Skip 1 cache entry */ - /* Skip 2 cache entries */ - // _CALL_BUILTIN_FAST_WITH_KEYWORDS - { - args = &stack_pointer[-oparg]; - self_or_null = stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); - int total_args = oparg; - _PyStackRef *arguments = args; - if (!PyStackRef_IsNull(self_or_null)) { - arguments--; - total_args++; - } - if (!PyCFunction_CheckExact(callable_o)) { - UPDATE_MISS_STATS(CALL); - assert(_PyOpcode_Deopt[opcode] == (CALL)); - JUMP_TO_PREDICTED(TRACING_CALL); - } - if (PyCFunction_GET_FLAGS(callable_o) != (METH_FASTCALL | METH_KEYWORDS)) { - UPDATE_MISS_STATS(CALL); - assert(_PyOpcode_Deopt[opcode] == (CALL)); - JUMP_TO_PREDICTED(TRACING_CALL); - } - STAT_INC(CALL, hit); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyCFunctionFastWithKeywords cfunc = - _PyCFunctionFastWithKeywords_CAST(PyCFunction_GET_FUNCTION(callable_o)); - stack_pointer = _PyFrame_GetStackPointer(frame); - STACKREFS_TO_PYOBJECTS(arguments, total_args, args_o); - if (CONVERSION_FAILED(args_o)) { - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyStackRef tmp; - for (int _i = oparg; --_i >= 0;) { - tmp = args[_i]; - args[_i] = PyStackRef_NULL; - PyStackRef_CLOSE(tmp); - } - tmp = self_or_null; - self_or_null = PyStackRef_NULL; - stack_pointer[-1 - oparg] = self_or_null; - PyStackRef_XCLOSE(tmp); - tmp = callable; - callable = PyStackRef_NULL; - stack_pointer[-2 - oparg] = callable; - PyStackRef_CLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -2 - oparg; - assert(WITHIN_STACK_BOUNDS()); - TRACING_JUMP_TO_LABEL(error); - } - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *res_o = cfunc(PyCFunction_GET_SELF(callable_o), args_o, total_args, NULL); - stack_pointer = _PyFrame_GetStackPointer(frame); - STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); - assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyStackRef tmp; - for (int _i = oparg; --_i >= 0;) { - tmp = args[_i]; - args[_i] = PyStackRef_NULL; - PyStackRef_CLOSE(tmp); - } - tmp = self_or_null; - self_or_null = PyStackRef_NULL; - stack_pointer[-1 - oparg] = self_or_null; - PyStackRef_XCLOSE(tmp); - tmp = callable; - callable = PyStackRef_NULL; - stack_pointer[-2 - oparg] = callable; - PyStackRef_CLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -2 - oparg; - assert(WITHIN_STACK_BOUNDS()); - if (res_o == NULL) { - TRACING_JUMP_TO_LABEL(error); - } - res = PyStackRef_FromPyObjectSteal(res_o); - } - // _CHECK_PERIODIC_AT_END - { - stack_pointer[0] = res; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = check_periodics(tstate); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err != 0) { - TRACING_JUMP_TO_LABEL(error); - } - } - TRACING_DISPATCH(); - } - - TRACING_TARGET(CALL_BUILTIN_O) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = CALL_BUILTIN_O; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 4; - INSTRUCTION_STATS(CALL_BUILTIN_O); - opcode = CALL_BUILTIN_O; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; - (void)(_old_stack_level); - static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); - _PyStackRef callable; - _PyStackRef self_or_null; - _PyStackRef *args; - _PyStackRef res; - /* Skip 1 cache entry */ - /* Skip 2 cache entries */ - // _CALL_BUILTIN_O - { - args = &stack_pointer[-oparg]; - self_or_null = stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); - int total_args = oparg; - if (!PyStackRef_IsNull(self_or_null)) { - args--; - total_args++; - } - if (total_args != 1) { - UPDATE_MISS_STATS(CALL); - assert(_PyOpcode_Deopt[opcode] == (CALL)); - JUMP_TO_PREDICTED(TRACING_CALL); - } - if (!PyCFunction_CheckExact(callable_o)) { - UPDATE_MISS_STATS(CALL); - assert(_PyOpcode_Deopt[opcode] == (CALL)); - JUMP_TO_PREDICTED(TRACING_CALL); - } - if (PyCFunction_GET_FLAGS(callable_o) != METH_O) { - UPDATE_MISS_STATS(CALL); - assert(_PyOpcode_Deopt[opcode] == (CALL)); - JUMP_TO_PREDICTED(TRACING_CALL); - } - if (_Py_ReachedRecursionLimit(tstate)) { - UPDATE_MISS_STATS(CALL); - assert(_PyOpcode_Deopt[opcode] == (CALL)); - JUMP_TO_PREDICTED(TRACING_CALL); - } - STAT_INC(CALL, hit); - PyCFunction cfunc = PyCFunction_GET_FUNCTION(callable_o); - _PyStackRef arg = args[0]; - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *res_o = _PyCFunction_TrampolineCall(cfunc, PyCFunction_GET_SELF(callable_o), PyStackRef_AsPyObjectBorrow(arg)); - stack_pointer = _PyFrame_GetStackPointer(frame); - _Py_LeaveRecursiveCallTstate(tstate); - assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(arg); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -2 - oparg; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(callable); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (res_o == NULL) { - TRACING_JUMP_TO_LABEL(error); - } - res = PyStackRef_FromPyObjectSteal(res_o); - } - // _CHECK_PERIODIC_AT_END - { - stack_pointer[0] = res; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = check_periodics(tstate); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err != 0) { - TRACING_JUMP_TO_LABEL(error); - } - } - TRACING_DISPATCH(); - } - - TRACING_TARGET(CALL_FUNCTION_EX) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = CALL_FUNCTION_EX; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(CALL_FUNCTION_EX); - opcode = CALL_FUNCTION_EX; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; - (void)(_old_stack_level); - _PyStackRef func; - _PyStackRef callargs; - _PyStackRef func_st; - _PyStackRef null; - _PyStackRef callargs_st; - _PyStackRef kwargs_st; - _PyStackRef result; - // _MAKE_CALLARGS_A_TUPLE - { - callargs = stack_pointer[-2]; - func = stack_pointer[-4]; - PyObject *callargs_o = PyStackRef_AsPyObjectBorrow(callargs); - if (!PyTuple_CheckExact(callargs_o)) { - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = _Py_Check_ArgsIterable(tstate, PyStackRef_AsPyObjectBorrow(func), callargs_o); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err < 0) { - TRACING_JUMP_TO_LABEL(error); - } - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *tuple_o = PySequence_Tuple(callargs_o); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (tuple_o == NULL) { - TRACING_JUMP_TO_LABEL(error); - } - _PyStackRef temp = callargs; - callargs = PyStackRef_FromPyObjectSteal(tuple_o); - stack_pointer[-2] = callargs; - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(temp); - stack_pointer = _PyFrame_GetStackPointer(frame); - } - } - // _DO_CALL_FUNCTION_EX - { - kwargs_st = stack_pointer[-1]; - callargs_st = callargs; - null = stack_pointer[-3]; - func_st = func; - (void)null; - PyObject *func = PyStackRef_AsPyObjectBorrow(func_st); - EVAL_CALL_STAT_INC_IF_FUNCTION(EVAL_CALL_FUNCTION_EX, func); - PyObject *result_o; - assert(!_PyErr_Occurred(tstate)); - if (opcode == INSTRUMENTED_CALL_FUNCTION_EX) { - PyObject *callargs = PyStackRef_AsPyObjectBorrow(callargs_st); - PyObject *kwargs = PyStackRef_AsPyObjectBorrow(kwargs_st); - assert(kwargs == NULL || PyDict_CheckExact(kwargs)); - assert(PyTuple_CheckExact(callargs)); - PyObject *arg = PyTuple_GET_SIZE(callargs) > 0 ? - PyTuple_GET_ITEM(callargs, 0) : &_PyInstrumentation_MISSING; - stack_pointer[-2] = callargs_st; - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = _Py_call_instrumentation_2args( - tstate, PY_MONITORING_EVENT_CALL, - frame, this_instr, func, arg); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err) { - TRACING_JUMP_TO_LABEL(error); - } - _PyFrame_SetStackPointer(frame, stack_pointer); - result_o = PyObject_Call(func, callargs, kwargs); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (!PyFunction_Check(func) && !PyMethod_Check(func)) { - if (result_o == NULL) { - _PyFrame_SetStackPointer(frame, stack_pointer); - _Py_call_instrumentation_exc2( - tstate, PY_MONITORING_EVENT_C_RAISE, - frame, this_instr, func, arg); - stack_pointer = _PyFrame_GetStackPointer(frame); - } - else { - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = _Py_call_instrumentation_2args( - tstate, PY_MONITORING_EVENT_C_RETURN, - frame, this_instr, func, arg); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err < 0) { - _PyFrame_SetStackPointer(frame, stack_pointer); - Py_CLEAR(result_o); - stack_pointer = _PyFrame_GetStackPointer(frame); - } - } - } - } - else { - if (Py_TYPE(func) == &PyFunction_Type && - tstate->interp->eval_frame == NULL && - ((PyFunctionObject *)func)->vectorcall == _PyFunction_Vectorcall) { - PyObject *callargs = PyStackRef_AsPyObjectSteal(callargs_st); - assert(PyTuple_CheckExact(callargs)); - PyObject *kwargs = PyStackRef_IsNull(kwargs_st) ? NULL : PyStackRef_AsPyObjectSteal(kwargs_st); - assert(kwargs == NULL || PyDict_CheckExact(kwargs)); - Py_ssize_t nargs = PyTuple_GET_SIZE(callargs); - int code_flags = ((PyCodeObject *)PyFunction_GET_CODE(func))->co_flags; - PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(func)); - stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyInterpreterFrame *new_frame = _PyEvalFramePushAndInit_Ex( - tstate, func_st, locals, - nargs, callargs, kwargs, frame); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); - if (new_frame == NULL) { - TRACING_JUMP_TO_LABEL(error); - } - assert( 1u == 1); - frame->return_offset = 1; - TRACING_DISPATCH_INLINED(new_frame); - } - PyObject *callargs = PyStackRef_AsPyObjectBorrow(callargs_st); - assert(PyTuple_CheckExact(callargs)); - PyObject *kwargs = PyStackRef_AsPyObjectBorrow(kwargs_st); - assert(kwargs == NULL || PyDict_CheckExact(kwargs)); - stack_pointer[-2] = callargs_st; - _PyFrame_SetStackPointer(frame, stack_pointer); - result_o = PyObject_Call(func, callargs, kwargs); - stack_pointer = _PyFrame_GetStackPointer(frame); - } - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_XCLOSE(kwargs_st); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(callargs_st); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(func_st); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (result_o == NULL) { - TRACING_JUMP_TO_LABEL(error); - } - result = PyStackRef_FromPyObjectSteal(result_o); - } - // _CHECK_PERIODIC_AT_END - { - stack_pointer[0] = result; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = check_periodics(tstate); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err != 0) { - TRACING_JUMP_TO_LABEL(error); - } - } - TRACING_DISPATCH(); - } - - TRACING_TARGET(CALL_INTRINSIC_1) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = CALL_INTRINSIC_1; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(CALL_INTRINSIC_1); - opcode = CALL_INTRINSIC_1; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; - (void)(_old_stack_level); - _PyStackRef value; - _PyStackRef res; - value = stack_pointer[-1]; - assert(oparg <= MAX_INTRINSIC_1); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *res_o = _PyIntrinsics_UnaryFunctions[oparg].func(tstate, PyStackRef_AsPyObjectBorrow(value)); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(value); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (res_o == NULL) { - TRACING_JUMP_TO_LABEL(error); - } - res = PyStackRef_FromPyObjectSteal(res_o); - stack_pointer[0] = res; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - TRACING_DISPATCH(); - } - - TRACING_TARGET(CALL_INTRINSIC_2) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = CALL_INTRINSIC_2; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(CALL_INTRINSIC_2); - opcode = CALL_INTRINSIC_2; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; - (void)(_old_stack_level); - _PyStackRef value2_st; - _PyStackRef value1_st; - _PyStackRef res; - value1_st = stack_pointer[-1]; - value2_st = stack_pointer[-2]; - assert(oparg <= MAX_INTRINSIC_2); - PyObject *value1 = PyStackRef_AsPyObjectBorrow(value1_st); - PyObject *value2 = PyStackRef_AsPyObjectBorrow(value2_st); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *res_o = _PyIntrinsics_BinaryFunctions[oparg].func(tstate, value2, value1); - _PyStackRef tmp = value1_st; - value1_st = PyStackRef_NULL; - stack_pointer[-1] = value1_st; - PyStackRef_CLOSE(tmp); - tmp = value2_st; - value2_st = PyStackRef_NULL; - stack_pointer[-2] = value2_st; - PyStackRef_CLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); - if (res_o == NULL) { - TRACING_JUMP_TO_LABEL(error); - } - res = PyStackRef_FromPyObjectSteal(res_o); - stack_pointer[0] = res; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - TRACING_DISPATCH(); - } - - TRACING_TARGET(CALL_ISINSTANCE) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = CALL_ISINSTANCE; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 4; - INSTRUCTION_STATS(CALL_ISINSTANCE); - opcode = CALL_ISINSTANCE; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; - (void)(_old_stack_level); - static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); - _PyStackRef null; - _PyStackRef callable; - _PyStackRef instance; - _PyStackRef cls; - _PyStackRef res; - /* Skip 1 cache entry */ - /* Skip 2 cache entries */ - // _GUARD_THIRD_NULL - { - null = stack_pointer[-3]; - if (!PyStackRef_IsNull(null)) { - UPDATE_MISS_STATS(CALL); - assert(_PyOpcode_Deopt[opcode] == (CALL)); - JUMP_TO_PREDICTED(TRACING_CALL); - } - } - // _GUARD_CALLABLE_ISINSTANCE - { - callable = stack_pointer[-4]; - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); - PyInterpreterState *interp = tstate->interp; - if (callable_o != interp->callable_cache.isinstance) { - UPDATE_MISS_STATS(CALL); - assert(_PyOpcode_Deopt[opcode] == (CALL)); - JUMP_TO_PREDICTED(TRACING_CALL); - } - } - // _CALL_ISINSTANCE - { - cls = stack_pointer[-1]; - instance = stack_pointer[-2]; - STAT_INC(CALL, hit); - PyObject *inst_o = PyStackRef_AsPyObjectBorrow(instance); - PyObject *cls_o = PyStackRef_AsPyObjectBorrow(cls); - _PyFrame_SetStackPointer(frame, stack_pointer); - int retval = PyObject_IsInstance(inst_o, cls_o); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (retval < 0) { - TRACING_JUMP_TO_LABEL(error); - } - (void)null; - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(cls); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(instance); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(callable); - stack_pointer = _PyFrame_GetStackPointer(frame); - res = retval ? PyStackRef_True : PyStackRef_False; - assert((!PyStackRef_IsNull(res)) ^ (_PyErr_Occurred(tstate) != NULL)); - } - stack_pointer[0] = res; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - TRACING_DISPATCH(); - } - - TRACING_TARGET(CALL_KW) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = CALL_KW; - (void)(opcode); - #endif - frame->instr_ptr = next_instr; - next_instr += 4; - INSTRUCTION_STATS(CALL_KW); - PREDICTED_TRACING_CALL_KW:; - _Py_CODEUNIT* const this_instr = next_instr - 4; - (void)this_instr; - opcode = CALL_KW; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; - (void)(_old_stack_level); - _PyStackRef callable; - _PyStackRef self_or_null; - _PyStackRef *args; - _PyStackRef kwnames; - _PyStackRef res; - // _SPECIALIZE_CALL_KW - { - self_or_null = stack_pointer[-2 - oparg]; - callable = stack_pointer[-3 - oparg]; - uint16_t counter = read_u16(&this_instr[1].cache); - (void)counter; - #if ENABLE_SPECIALIZATION_FT - if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { - next_instr = this_instr; - _PyFrame_SetStackPointer(frame, stack_pointer); - _Py_Specialize_CallKw(callable, next_instr, oparg + !PyStackRef_IsNull(self_or_null)); - stack_pointer = _PyFrame_GetStackPointer(frame); - TRACING_SPECIALIZE_DISPATCH_SAME_OPARG() - } - OPCODE_DEFERRED_INC(CALL_KW); - ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); - #endif /* ENABLE_SPECIALIZATION_FT */ - } - /* Skip 2 cache entries */ - // _MAYBE_EXPAND_METHOD_KW - { - if (PyStackRef_TYPE(callable) == &PyMethod_Type && PyStackRef_IsNull(self_or_null)) { - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); - PyObject *self = ((PyMethodObject *)callable_o)->im_self; - self_or_null = PyStackRef_FromPyObjectNew(self); - PyObject *method = ((PyMethodObject *)callable_o)->im_func; - _PyStackRef temp = callable; - callable = PyStackRef_FromPyObjectNew(method); - stack_pointer[-3 - oparg] = callable; - stack_pointer[-2 - oparg] = self_or_null; - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(temp); - stack_pointer = _PyFrame_GetStackPointer(frame); - } - } - // _DO_CALL_KW - { - kwnames = stack_pointer[-1]; - args = &stack_pointer[-1 - oparg]; - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); - PyObject *kwnames_o = PyStackRef_AsPyObjectBorrow(kwnames); - int total_args = oparg; - _PyStackRef *arguments = args; - if (!PyStackRef_IsNull(self_or_null)) { - arguments--; - total_args++; - } - int positional_args = total_args - (int)PyTuple_GET_SIZE(kwnames_o); - if (Py_TYPE(callable_o) == &PyFunction_Type && - tstate->interp->eval_frame == NULL && - ((PyFunctionObject *)callable_o)->vectorcall == _PyFunction_Vectorcall) - { - int code_flags = ((PyCodeObject*)PyFunction_GET_CODE(callable_o))->co_flags; - PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable_o)); - stack_pointer[-3 - oparg] = callable; - stack_pointer[-2 - oparg] = self_or_null; - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyInterpreterFrame *new_frame = _PyEvalFramePushAndInit( - tstate, callable, locals, - arguments, positional_args, kwnames_o, frame - ); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -3 - oparg; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(kwnames); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (new_frame == NULL) { - TRACING_JUMP_TO_LABEL(error); - } - assert( 4u == 1 + INLINE_CACHE_ENTRIES_CALL_KW); - frame->return_offset = 4u ; - TRACING_DISPATCH_INLINED(new_frame); - } - STACKREFS_TO_PYOBJECTS(arguments, total_args, args_o); - if (CONVERSION_FAILED(args_o)) { - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyStackRef tmp = kwnames; - kwnames = PyStackRef_NULL; - stack_pointer[-3 - oparg] = callable; - stack_pointer[-2 - oparg] = self_or_null; - stack_pointer[-1] = kwnames; - PyStackRef_CLOSE(tmp); - for (int _i = oparg; --_i >= 0;) { - tmp = args[_i]; - args[_i] = PyStackRef_NULL; - PyStackRef_CLOSE(tmp); - } - tmp = self_or_null; - self_or_null = PyStackRef_NULL; - stack_pointer[-2 - oparg] = self_or_null; - PyStackRef_XCLOSE(tmp); - tmp = callable; - callable = PyStackRef_NULL; - stack_pointer[-3 - oparg] = callable; - PyStackRef_CLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -3 - oparg; - assert(WITHIN_STACK_BOUNDS()); - TRACING_JUMP_TO_LABEL(error); - } - stack_pointer[-3 - oparg] = callable; - stack_pointer[-2 - oparg] = self_or_null; - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *res_o = PyObject_Vectorcall( - callable_o, args_o, - positional_args | PY_VECTORCALL_ARGUMENTS_OFFSET, - kwnames_o); - stack_pointer = _PyFrame_GetStackPointer(frame); - STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); - if (opcode == INSTRUMENTED_CALL_KW) { - PyObject *arg = total_args == 0 ? - &_PyInstrumentation_MISSING : PyStackRef_AsPyObjectBorrow(arguments[0]); - if (res_o == NULL) { - _PyFrame_SetStackPointer(frame, stack_pointer); - _Py_call_instrumentation_exc2( - tstate, PY_MONITORING_EVENT_C_RAISE, - frame, this_instr, callable_o, arg); - stack_pointer = _PyFrame_GetStackPointer(frame); - } - else { - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = _Py_call_instrumentation_2args( - tstate, PY_MONITORING_EVENT_C_RETURN, - frame, this_instr, callable_o, arg); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err < 0) { - _PyFrame_SetStackPointer(frame, stack_pointer); - Py_CLEAR(res_o); - stack_pointer = _PyFrame_GetStackPointer(frame); - } - } - } - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyStackRef tmp = kwnames; - kwnames = PyStackRef_NULL; - stack_pointer[-1] = kwnames; - PyStackRef_CLOSE(tmp); - for (int _i = oparg; --_i >= 0;) { - tmp = args[_i]; - args[_i] = PyStackRef_NULL; - PyStackRef_CLOSE(tmp); - } - tmp = self_or_null; - self_or_null = PyStackRef_NULL; - stack_pointer[-2 - oparg] = self_or_null; - PyStackRef_XCLOSE(tmp); - tmp = callable; - callable = PyStackRef_NULL; - stack_pointer[-3 - oparg] = callable; - PyStackRef_CLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -3 - oparg; - assert(WITHIN_STACK_BOUNDS()); - if (res_o == NULL) { - TRACING_JUMP_TO_LABEL(error); - } - res = PyStackRef_FromPyObjectSteal(res_o); - } - stack_pointer[0] = res; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - TRACING_DISPATCH(); - } - - TRACING_TARGET(CALL_KW_BOUND_METHOD) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = CALL_KW_BOUND_METHOD; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 4; - INSTRUCTION_STATS(CALL_KW_BOUND_METHOD); - opcode = CALL_KW_BOUND_METHOD; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; - (void)(_old_stack_level); - static_assert(INLINE_CACHE_ENTRIES_CALL_KW == 3, "incorrect cache size"); - _PyStackRef callable; - _PyStackRef null; - _PyStackRef self_or_null; - _PyStackRef *args; - _PyStackRef kwnames; - _PyStackRef new_frame; - /* Skip 1 cache entry */ - // _CHECK_PEP_523 - { - if (tstate->interp->eval_frame) { - UPDATE_MISS_STATS(CALL_KW); - assert(_PyOpcode_Deopt[opcode] == (CALL_KW)); - JUMP_TO_PREDICTED(TRACING_CALL_KW); - } - } - // _CHECK_METHOD_VERSION_KW - { - null = stack_pointer[-2 - oparg]; - callable = stack_pointer[-3 - oparg]; - uint32_t func_version = read_u32(&this_instr[2].cache); - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); - if (Py_TYPE(callable_o) != &PyMethod_Type) { - UPDATE_MISS_STATS(CALL_KW); - assert(_PyOpcode_Deopt[opcode] == (CALL_KW)); - JUMP_TO_PREDICTED(TRACING_CALL_KW); - } - PyObject *func = ((PyMethodObject *)callable_o)->im_func; - if (!PyFunction_Check(func)) { - UPDATE_MISS_STATS(CALL_KW); - assert(_PyOpcode_Deopt[opcode] == (CALL_KW)); - JUMP_TO_PREDICTED(TRACING_CALL_KW); - } - if (((PyFunctionObject *)func)->func_version != func_version) { - UPDATE_MISS_STATS(CALL_KW); - assert(_PyOpcode_Deopt[opcode] == (CALL_KW)); - JUMP_TO_PREDICTED(TRACING_CALL_KW); - } - if (!PyStackRef_IsNull(null)) { - UPDATE_MISS_STATS(CALL_KW); - assert(_PyOpcode_Deopt[opcode] == (CALL_KW)); - JUMP_TO_PREDICTED(TRACING_CALL_KW); - } - } - // _EXPAND_METHOD_KW - { - self_or_null = null; - assert(PyStackRef_IsNull(self_or_null)); - _PyStackRef callable_s = callable; - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); - assert(Py_TYPE(callable_o) == &PyMethod_Type); - self_or_null = PyStackRef_FromPyObjectNew(((PyMethodObject *)callable_o)->im_self); - callable = PyStackRef_FromPyObjectNew(((PyMethodObject *)callable_o)->im_func); - assert(PyStackRef_FunctionCheck(callable)); - stack_pointer[-3 - oparg] = callable; - stack_pointer[-2 - oparg] = self_or_null; - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(callable_s); - stack_pointer = _PyFrame_GetStackPointer(frame); - } - // flush - // _PY_FRAME_KW - { - kwnames = stack_pointer[-1]; - args = &stack_pointer[-1 - oparg]; - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); - int total_args = oparg; - _PyStackRef *arguments = args; - if (!PyStackRef_IsNull(self_or_null)) { - arguments--; - total_args++; - } - PyObject *kwnames_o = PyStackRef_AsPyObjectBorrow(kwnames); - int positional_args = total_args - (int)PyTuple_GET_SIZE(kwnames_o); - assert(Py_TYPE(callable_o) == &PyFunction_Type); - int code_flags = ((PyCodeObject*)PyFunction_GET_CODE(callable_o))->co_flags; - PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable_o)); - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyInterpreterFrame *temp = _PyEvalFramePushAndInit( - tstate, callable, locals, - arguments, positional_args, kwnames_o, frame - ); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(kwnames); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -2 - oparg; - assert(WITHIN_STACK_BOUNDS()); - if (temp == NULL) { - TRACING_JUMP_TO_LABEL(error); - } - new_frame = PyStackRef_Wrap(temp); - } - // _SAVE_RETURN_OFFSET - { - #if TIER_ONE - frame->return_offset = (uint16_t)(next_instr - this_instr); - #endif - #if TIER_TWO - frame->return_offset = oparg; - #endif - } - // _PUSH_FRAME - { - assert(tstate->interp->eval_frame == NULL); - _PyInterpreterFrame *temp = PyStackRef_Unwrap(new_frame); - _PyFrame_SetStackPointer(frame, stack_pointer); - assert(temp->previous == frame || temp->previous->previous == frame); - CALL_STAT_INC(inlined_py_calls); - frame = tstate->current_frame = temp; - tstate->py_recursion_remaining--; - LOAD_SP(); - LOAD_IP(0); - LLTRACE_RESUME_FRAME(); - } - TRACING_DISPATCH(); - } - - TRACING_TARGET(CALL_KW_NON_PY) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = CALL_KW_NON_PY; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 4; - INSTRUCTION_STATS(CALL_KW_NON_PY); - opcode = CALL_KW_NON_PY; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; - (void)(_old_stack_level); - static_assert(INLINE_CACHE_ENTRIES_CALL_KW == 3, "incorrect cache size"); - _PyStackRef callable; - _PyStackRef self_or_null; - _PyStackRef *args; - _PyStackRef kwnames; - _PyStackRef res; - /* Skip 1 cache entry */ - /* Skip 2 cache entries */ - // _CHECK_IS_NOT_PY_CALLABLE_KW - { - callable = stack_pointer[-3 - oparg]; - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); - if (PyFunction_Check(callable_o)) { - UPDATE_MISS_STATS(CALL_KW); - assert(_PyOpcode_Deopt[opcode] == (CALL_KW)); - JUMP_TO_PREDICTED(TRACING_CALL_KW); - } - if (Py_TYPE(callable_o) == &PyMethod_Type) { - UPDATE_MISS_STATS(CALL_KW); - assert(_PyOpcode_Deopt[opcode] == (CALL_KW)); - JUMP_TO_PREDICTED(TRACING_CALL_KW); - } - } - // _CALL_KW_NON_PY - { - kwnames = stack_pointer[-1]; - args = &stack_pointer[-1 - oparg]; - self_or_null = stack_pointer[-2 - oparg]; - #if TIER_ONE - assert(opcode != INSTRUMENTED_CALL); - #endif - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); - int total_args = oparg; - _PyStackRef *arguments = args; - if (!PyStackRef_IsNull(self_or_null)) { - arguments--; - total_args++; - } - STACKREFS_TO_PYOBJECTS(arguments, total_args, args_o); - if (CONVERSION_FAILED(args_o)) { - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyStackRef tmp = kwnames; - kwnames = PyStackRef_NULL; - stack_pointer[-1] = kwnames; - PyStackRef_CLOSE(tmp); - for (int _i = oparg; --_i >= 0;) { - tmp = args[_i]; - args[_i] = PyStackRef_NULL; - PyStackRef_CLOSE(tmp); - } - tmp = self_or_null; - self_or_null = PyStackRef_NULL; - stack_pointer[-2 - oparg] = self_or_null; - PyStackRef_XCLOSE(tmp); - tmp = callable; - callable = PyStackRef_NULL; - stack_pointer[-3 - oparg] = callable; - PyStackRef_CLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -3 - oparg; - assert(WITHIN_STACK_BOUNDS()); - TRACING_JUMP_TO_LABEL(error); - } - PyObject *kwnames_o = PyStackRef_AsPyObjectBorrow(kwnames); - int positional_args = total_args - (int)PyTuple_GET_SIZE(kwnames_o); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *res_o = PyObject_Vectorcall( - callable_o, args_o, - positional_args | PY_VECTORCALL_ARGUMENTS_OFFSET, - kwnames_o); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(kwnames); - stack_pointer = _PyFrame_GetStackPointer(frame); - STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); - assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyStackRef tmp; - for (int _i = oparg; --_i >= 0;) { - tmp = args[_i]; - args[_i] = PyStackRef_NULL; - PyStackRef_CLOSE(tmp); - } - tmp = self_or_null; - self_or_null = PyStackRef_NULL; - stack_pointer[-1 - oparg] = self_or_null; - PyStackRef_XCLOSE(tmp); - tmp = callable; - callable = PyStackRef_NULL; - stack_pointer[-2 - oparg] = callable; - PyStackRef_CLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -2 - oparg; - assert(WITHIN_STACK_BOUNDS()); - if (res_o == NULL) { - TRACING_JUMP_TO_LABEL(error); - } - res = PyStackRef_FromPyObjectSteal(res_o); - } - // _CHECK_PERIODIC_AT_END - { - stack_pointer[0] = res; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = check_periodics(tstate); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err != 0) { - TRACING_JUMP_TO_LABEL(error); - } - } - TRACING_DISPATCH(); - } - - TRACING_TARGET(CALL_KW_PY) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = CALL_KW_PY; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 4; - INSTRUCTION_STATS(CALL_KW_PY); - opcode = CALL_KW_PY; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; - (void)(_old_stack_level); - static_assert(INLINE_CACHE_ENTRIES_CALL_KW == 3, "incorrect cache size"); - _PyStackRef callable; - _PyStackRef self_or_null; - _PyStackRef *args; - _PyStackRef kwnames; - _PyStackRef new_frame; - /* Skip 1 cache entry */ - // _CHECK_PEP_523 - { - if (tstate->interp->eval_frame) { - UPDATE_MISS_STATS(CALL_KW); - assert(_PyOpcode_Deopt[opcode] == (CALL_KW)); - JUMP_TO_PREDICTED(TRACING_CALL_KW); - } - } - // _CHECK_FUNCTION_VERSION_KW - { - callable = stack_pointer[-3 - oparg]; - uint32_t func_version = read_u32(&this_instr[2].cache); - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); - if (!PyFunction_Check(callable_o)) { - UPDATE_MISS_STATS(CALL_KW); - assert(_PyOpcode_Deopt[opcode] == (CALL_KW)); - JUMP_TO_PREDICTED(TRACING_CALL_KW); - } - PyFunctionObject *func = (PyFunctionObject *)callable_o; - if (func->func_version != func_version) { - UPDATE_MISS_STATS(CALL_KW); - assert(_PyOpcode_Deopt[opcode] == (CALL_KW)); - JUMP_TO_PREDICTED(TRACING_CALL_KW); - } - } - // _CHECK_RECURSION_REMAINING - { - if (tstate->py_recursion_remaining <= 1) { - UPDATE_MISS_STATS(CALL_KW); - assert(_PyOpcode_Deopt[opcode] == (CALL_KW)); - JUMP_TO_PREDICTED(TRACING_CALL_KW); - } - } - // _PY_FRAME_KW - { - kwnames = stack_pointer[-1]; - args = &stack_pointer[-1 - oparg]; - self_or_null = stack_pointer[-2 - oparg]; - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); - int total_args = oparg; - _PyStackRef *arguments = args; - if (!PyStackRef_IsNull(self_or_null)) { - arguments--; - total_args++; - } - PyObject *kwnames_o = PyStackRef_AsPyObjectBorrow(kwnames); - int positional_args = total_args - (int)PyTuple_GET_SIZE(kwnames_o); - assert(Py_TYPE(callable_o) == &PyFunction_Type); - int code_flags = ((PyCodeObject*)PyFunction_GET_CODE(callable_o))->co_flags; - PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable_o)); - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyInterpreterFrame *temp = _PyEvalFramePushAndInit( - tstate, callable, locals, - arguments, positional_args, kwnames_o, frame - ); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(kwnames); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -2 - oparg; - assert(WITHIN_STACK_BOUNDS()); - if (temp == NULL) { - TRACING_JUMP_TO_LABEL(error); - } - new_frame = PyStackRef_Wrap(temp); - } - // _SAVE_RETURN_OFFSET - { - #if TIER_ONE - frame->return_offset = (uint16_t)(next_instr - this_instr); - #endif - #if TIER_TWO - frame->return_offset = oparg; - #endif - } - // _PUSH_FRAME - { - assert(tstate->interp->eval_frame == NULL); - _PyInterpreterFrame *temp = PyStackRef_Unwrap(new_frame); - _PyFrame_SetStackPointer(frame, stack_pointer); - assert(temp->previous == frame || temp->previous->previous == frame); - CALL_STAT_INC(inlined_py_calls); - frame = tstate->current_frame = temp; - tstate->py_recursion_remaining--; - LOAD_SP(); - LOAD_IP(0); - LLTRACE_RESUME_FRAME(); - } - TRACING_DISPATCH(); - } - - TRACING_TARGET(CALL_LEN) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = CALL_LEN; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 4; - INSTRUCTION_STATS(CALL_LEN); - opcode = CALL_LEN; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; - (void)(_old_stack_level); - static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); - _PyStackRef null; - _PyStackRef callable; - _PyStackRef arg; - _PyStackRef res; - /* Skip 1 cache entry */ - /* Skip 2 cache entries */ - // _GUARD_NOS_NULL - { - null = stack_pointer[-2]; - if (!PyStackRef_IsNull(null)) { - UPDATE_MISS_STATS(CALL); - assert(_PyOpcode_Deopt[opcode] == (CALL)); - JUMP_TO_PREDICTED(TRACING_CALL); - } - } - // _GUARD_CALLABLE_LEN - { - callable = stack_pointer[-3]; - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); - PyInterpreterState *interp = tstate->interp; - if (callable_o != interp->callable_cache.len) { - UPDATE_MISS_STATS(CALL); - assert(_PyOpcode_Deopt[opcode] == (CALL)); - JUMP_TO_PREDICTED(TRACING_CALL); - } - } - // _CALL_LEN - { - arg = stack_pointer[-1]; - (void)null; - STAT_INC(CALL, hit); - PyObject *arg_o = PyStackRef_AsPyObjectBorrow(arg); - _PyFrame_SetStackPointer(frame, stack_pointer); - Py_ssize_t len_i = PyObject_Length(arg_o); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (len_i < 0) { - TRACING_JUMP_TO_LABEL(error); - } - PyObject *res_o = PyLong_FromSsize_t(len_i); - assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - if (res_o == NULL) { - TRACING_JUMP_TO_LABEL(error); - } - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(arg); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(callable); - stack_pointer = _PyFrame_GetStackPointer(frame); - res = PyStackRef_FromPyObjectSteal(res_o); - } - stack_pointer[0] = res; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - TRACING_DISPATCH(); - } - - TRACING_TARGET(CALL_LIST_APPEND) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = CALL_LIST_APPEND; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 4; - INSTRUCTION_STATS(CALL_LIST_APPEND); - opcode = CALL_LIST_APPEND; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; - (void)(_old_stack_level); - static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); - _PyStackRef callable; - _PyStackRef nos; - _PyStackRef self; - _PyStackRef arg; - /* Skip 1 cache entry */ - /* Skip 2 cache entries */ - // _GUARD_CALLABLE_LIST_APPEND - { - callable = stack_pointer[-3]; - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); - PyInterpreterState *interp = tstate->interp; - if (callable_o != interp->callable_cache.list_append) { - UPDATE_MISS_STATS(CALL); - assert(_PyOpcode_Deopt[opcode] == (CALL)); - JUMP_TO_PREDICTED(TRACING_CALL); - } - } - // _GUARD_NOS_NOT_NULL - { - nos = stack_pointer[-2]; - PyObject *o = PyStackRef_AsPyObjectBorrow(nos); - if (o == NULL) { - UPDATE_MISS_STATS(CALL); - assert(_PyOpcode_Deopt[opcode] == (CALL)); - JUMP_TO_PREDICTED(TRACING_CALL); - } - } - // _GUARD_NOS_LIST - { - PyObject *o = PyStackRef_AsPyObjectBorrow(nos); - if (!PyList_CheckExact(o)) { - UPDATE_MISS_STATS(CALL); - assert(_PyOpcode_Deopt[opcode] == (CALL)); - JUMP_TO_PREDICTED(TRACING_CALL); - } - } - // _CALL_LIST_APPEND - { - arg = stack_pointer[-1]; - self = nos; - assert(oparg == 1); - PyObject *self_o = PyStackRef_AsPyObjectBorrow(self); - if (!PyList_CheckExact(self_o)) { - UPDATE_MISS_STATS(CALL); - assert(_PyOpcode_Deopt[opcode] == (CALL)); - JUMP_TO_PREDICTED(TRACING_CALL); - } - if (!LOCK_OBJECT(self_o)) { - UPDATE_MISS_STATS(CALL); - assert(_PyOpcode_Deopt[opcode] == (CALL)); - JUMP_TO_PREDICTED(TRACING_CALL); - } - STAT_INC(CALL, hit); - int err = _PyList_AppendTakeRef((PyListObject *)self_o, PyStackRef_AsPyObjectSteal(arg)); - UNLOCK_OBJECT(self_o); - stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(self); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(callable); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err) { - TRACING_JUMP_TO_LABEL(error); - } - #if TIER_ONE - - assert(next_instr->op.code == POP_TOP); - SKIP_OVER(1); - #endif - } - TRACING_DISPATCH(); - } - - TRACING_TARGET(CALL_METHOD_DESCRIPTOR_FAST) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = CALL_METHOD_DESCRIPTOR_FAST; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 4; - INSTRUCTION_STATS(CALL_METHOD_DESCRIPTOR_FAST); - opcode = CALL_METHOD_DESCRIPTOR_FAST; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; - (void)(_old_stack_level); - static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); - _PyStackRef callable; - _PyStackRef self_or_null; - _PyStackRef *args; - _PyStackRef res; - /* Skip 1 cache entry */ - /* Skip 2 cache entries */ - // _CALL_METHOD_DESCRIPTOR_FAST - { - args = &stack_pointer[-oparg]; - self_or_null = stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); - int total_args = oparg; - _PyStackRef *arguments = args; - if (!PyStackRef_IsNull(self_or_null)) { - arguments--; - total_args++; - } - if (total_args == 0) { - UPDATE_MISS_STATS(CALL); - assert(_PyOpcode_Deopt[opcode] == (CALL)); - JUMP_TO_PREDICTED(TRACING_CALL); - } - PyMethodDescrObject *method = (PyMethodDescrObject *)callable_o; - if (!Py_IS_TYPE(method, &PyMethodDescr_Type)) { - UPDATE_MISS_STATS(CALL); - assert(_PyOpcode_Deopt[opcode] == (CALL)); - JUMP_TO_PREDICTED(TRACING_CALL); - } - PyMethodDef *meth = method->d_method; - if (meth->ml_flags != METH_FASTCALL) { - UPDATE_MISS_STATS(CALL); - assert(_PyOpcode_Deopt[opcode] == (CALL)); - JUMP_TO_PREDICTED(TRACING_CALL); - } - PyObject *self = PyStackRef_AsPyObjectBorrow(arguments[0]); - assert(self != NULL); - if (!Py_IS_TYPE(self, method->d_common.d_type)) { - UPDATE_MISS_STATS(CALL); - assert(_PyOpcode_Deopt[opcode] == (CALL)); - JUMP_TO_PREDICTED(TRACING_CALL); - } - STAT_INC(CALL, hit); - int nargs = total_args - 1; - STACKREFS_TO_PYOBJECTS(arguments, total_args, args_o); - if (CONVERSION_FAILED(args_o)) { - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyStackRef tmp; - for (int _i = oparg; --_i >= 0;) { - tmp = args[_i]; - args[_i] = PyStackRef_NULL; - PyStackRef_CLOSE(tmp); - } - tmp = self_or_null; - self_or_null = PyStackRef_NULL; - stack_pointer[-1 - oparg] = self_or_null; - PyStackRef_XCLOSE(tmp); - tmp = callable; - callable = PyStackRef_NULL; - stack_pointer[-2 - oparg] = callable; - PyStackRef_CLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -2 - oparg; - assert(WITHIN_STACK_BOUNDS()); - TRACING_JUMP_TO_LABEL(error); - } - _PyFrame_SetStackPointer(frame, stack_pointer); - PyCFunctionFast cfunc = _PyCFunctionFast_CAST(meth->ml_meth); - PyObject *res_o = cfunc(self, (args_o + 1), nargs); - stack_pointer = _PyFrame_GetStackPointer(frame); - STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); - assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyStackRef tmp; - for (int _i = oparg; --_i >= 0;) { - tmp = args[_i]; - args[_i] = PyStackRef_NULL; - PyStackRef_CLOSE(tmp); - } - tmp = self_or_null; - self_or_null = PyStackRef_NULL; - stack_pointer[-1 - oparg] = self_or_null; - PyStackRef_XCLOSE(tmp); - tmp = callable; - callable = PyStackRef_NULL; - stack_pointer[-2 - oparg] = callable; - PyStackRef_CLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -2 - oparg; - assert(WITHIN_STACK_BOUNDS()); - if (res_o == NULL) { - TRACING_JUMP_TO_LABEL(error); - } - res = PyStackRef_FromPyObjectSteal(res_o); - } - // _CHECK_PERIODIC_AT_END - { - stack_pointer[0] = res; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = check_periodics(tstate); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err != 0) { - TRACING_JUMP_TO_LABEL(error); - } - } - TRACING_DISPATCH(); - } - - TRACING_TARGET(CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 4; - INSTRUCTION_STATS(CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS); - opcode = CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; - (void)(_old_stack_level); - static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); - _PyStackRef callable; - _PyStackRef self_or_null; - _PyStackRef *args; - _PyStackRef res; - /* Skip 1 cache entry */ - /* Skip 2 cache entries */ - // _CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS - { - args = &stack_pointer[-oparg]; - self_or_null = stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); - int total_args = oparg; - _PyStackRef *arguments = args; - if (!PyStackRef_IsNull(self_or_null)) { - arguments--; - total_args++; - } - if (total_args == 0) { - UPDATE_MISS_STATS(CALL); - assert(_PyOpcode_Deopt[opcode] == (CALL)); - JUMP_TO_PREDICTED(TRACING_CALL); - } - PyMethodDescrObject *method = (PyMethodDescrObject *)callable_o; - if (!Py_IS_TYPE(method, &PyMethodDescr_Type)) { - UPDATE_MISS_STATS(CALL); - assert(_PyOpcode_Deopt[opcode] == (CALL)); - JUMP_TO_PREDICTED(TRACING_CALL); - } - PyMethodDef *meth = method->d_method; - if (meth->ml_flags != (METH_FASTCALL|METH_KEYWORDS)) { - UPDATE_MISS_STATS(CALL); - assert(_PyOpcode_Deopt[opcode] == (CALL)); - JUMP_TO_PREDICTED(TRACING_CALL); - } - PyTypeObject *d_type = method->d_common.d_type; - PyObject *self = PyStackRef_AsPyObjectBorrow(arguments[0]); - assert(self != NULL); - if (!Py_IS_TYPE(self, d_type)) { - UPDATE_MISS_STATS(CALL); - assert(_PyOpcode_Deopt[opcode] == (CALL)); - JUMP_TO_PREDICTED(TRACING_CALL); - } - STAT_INC(CALL, hit); - int nargs = total_args - 1; - STACKREFS_TO_PYOBJECTS(arguments, total_args, args_o); - if (CONVERSION_FAILED(args_o)) { - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyStackRef tmp; - for (int _i = oparg; --_i >= 0;) { - tmp = args[_i]; - args[_i] = PyStackRef_NULL; - PyStackRef_CLOSE(tmp); - } - tmp = self_or_null; - self_or_null = PyStackRef_NULL; - stack_pointer[-1 - oparg] = self_or_null; - PyStackRef_XCLOSE(tmp); - tmp = callable; - callable = PyStackRef_NULL; - stack_pointer[-2 - oparg] = callable; - PyStackRef_CLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -2 - oparg; - assert(WITHIN_STACK_BOUNDS()); - TRACING_JUMP_TO_LABEL(error); - } - _PyFrame_SetStackPointer(frame, stack_pointer); - PyCFunctionFastWithKeywords cfunc = - _PyCFunctionFastWithKeywords_CAST(meth->ml_meth); - PyObject *res_o = cfunc(self, (args_o + 1), nargs, NULL); - stack_pointer = _PyFrame_GetStackPointer(frame); - STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); - assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyStackRef tmp; - for (int _i = oparg; --_i >= 0;) { - tmp = args[_i]; - args[_i] = PyStackRef_NULL; - PyStackRef_CLOSE(tmp); - } - tmp = self_or_null; - self_or_null = PyStackRef_NULL; - stack_pointer[-1 - oparg] = self_or_null; - PyStackRef_XCLOSE(tmp); - tmp = callable; - callable = PyStackRef_NULL; - stack_pointer[-2 - oparg] = callable; - PyStackRef_CLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -2 - oparg; - assert(WITHIN_STACK_BOUNDS()); - if (res_o == NULL) { - TRACING_JUMP_TO_LABEL(error); - } - res = PyStackRef_FromPyObjectSteal(res_o); - } - // _CHECK_PERIODIC_AT_END - { - stack_pointer[0] = res; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = check_periodics(tstate); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err != 0) { - TRACING_JUMP_TO_LABEL(error); - } - } - TRACING_DISPATCH(); - } - - TRACING_TARGET(CALL_METHOD_DESCRIPTOR_NOARGS) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = CALL_METHOD_DESCRIPTOR_NOARGS; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 4; - INSTRUCTION_STATS(CALL_METHOD_DESCRIPTOR_NOARGS); - opcode = CALL_METHOD_DESCRIPTOR_NOARGS; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; - (void)(_old_stack_level); - static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); - _PyStackRef callable; - _PyStackRef self_or_null; - _PyStackRef *args; - _PyStackRef res; - /* Skip 1 cache entry */ - /* Skip 2 cache entries */ - // _CALL_METHOD_DESCRIPTOR_NOARGS - { - args = &stack_pointer[-oparg]; - self_or_null = stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; - assert(oparg == 0 || oparg == 1); - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); - int total_args = oparg; - if (!PyStackRef_IsNull(self_or_null)) { - args--; - total_args++; - } - if (total_args != 1) { - UPDATE_MISS_STATS(CALL); - assert(_PyOpcode_Deopt[opcode] == (CALL)); - JUMP_TO_PREDICTED(TRACING_CALL); - } - PyMethodDescrObject *method = (PyMethodDescrObject *)callable_o; - if (!Py_IS_TYPE(method, &PyMethodDescr_Type)) { - UPDATE_MISS_STATS(CALL); - assert(_PyOpcode_Deopt[opcode] == (CALL)); - JUMP_TO_PREDICTED(TRACING_CALL); - } - PyMethodDef *meth = method->d_method; - _PyStackRef self_stackref = args[0]; - PyObject *self = PyStackRef_AsPyObjectBorrow(self_stackref); - if (!Py_IS_TYPE(self, method->d_common.d_type)) { - UPDATE_MISS_STATS(CALL); - assert(_PyOpcode_Deopt[opcode] == (CALL)); - JUMP_TO_PREDICTED(TRACING_CALL); - } - if (meth->ml_flags != METH_NOARGS) { - UPDATE_MISS_STATS(CALL); - assert(_PyOpcode_Deopt[opcode] == (CALL)); - JUMP_TO_PREDICTED(TRACING_CALL); - } - if (_Py_ReachedRecursionLimit(tstate)) { - UPDATE_MISS_STATS(CALL); - assert(_PyOpcode_Deopt[opcode] == (CALL)); - JUMP_TO_PREDICTED(TRACING_CALL); - } - STAT_INC(CALL, hit); - PyCFunction cfunc = meth->ml_meth; - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *res_o = _PyCFunction_TrampolineCall(cfunc, self, NULL); - stack_pointer = _PyFrame_GetStackPointer(frame); - _Py_LeaveRecursiveCallTstate(tstate); - assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(self_stackref); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -2 - oparg; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(callable); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (res_o == NULL) { - TRACING_JUMP_TO_LABEL(error); - } - res = PyStackRef_FromPyObjectSteal(res_o); - } - // _CHECK_PERIODIC_AT_END - { - stack_pointer[0] = res; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = check_periodics(tstate); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err != 0) { - TRACING_JUMP_TO_LABEL(error); - } - } - TRACING_DISPATCH(); - } - - TRACING_TARGET(CALL_METHOD_DESCRIPTOR_O) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = CALL_METHOD_DESCRIPTOR_O; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 4; - INSTRUCTION_STATS(CALL_METHOD_DESCRIPTOR_O); - opcode = CALL_METHOD_DESCRIPTOR_O; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; - (void)(_old_stack_level); - static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); - _PyStackRef callable; - _PyStackRef self_or_null; - _PyStackRef *args; - _PyStackRef res; - /* Skip 1 cache entry */ - /* Skip 2 cache entries */ - // _CALL_METHOD_DESCRIPTOR_O - { - args = &stack_pointer[-oparg]; - self_or_null = stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); - int total_args = oparg; - _PyStackRef *arguments = args; - if (!PyStackRef_IsNull(self_or_null)) { - arguments--; - total_args++; - } - PyMethodDescrObject *method = (PyMethodDescrObject *)callable_o; - if (total_args != 2) { - UPDATE_MISS_STATS(CALL); - assert(_PyOpcode_Deopt[opcode] == (CALL)); - JUMP_TO_PREDICTED(TRACING_CALL); - } - if (!Py_IS_TYPE(method, &PyMethodDescr_Type)) { - UPDATE_MISS_STATS(CALL); - assert(_PyOpcode_Deopt[opcode] == (CALL)); - JUMP_TO_PREDICTED(TRACING_CALL); - } - PyMethodDef *meth = method->d_method; - if (meth->ml_flags != METH_O) { - UPDATE_MISS_STATS(CALL); - assert(_PyOpcode_Deopt[opcode] == (CALL)); - JUMP_TO_PREDICTED(TRACING_CALL); - } - if (_Py_ReachedRecursionLimit(tstate)) { - UPDATE_MISS_STATS(CALL); - assert(_PyOpcode_Deopt[opcode] == (CALL)); - JUMP_TO_PREDICTED(TRACING_CALL); - } - _PyStackRef arg_stackref = arguments[1]; - _PyStackRef self_stackref = arguments[0]; - if (!Py_IS_TYPE(PyStackRef_AsPyObjectBorrow(self_stackref), - method->d_common.d_type)) { - UPDATE_MISS_STATS(CALL); - assert(_PyOpcode_Deopt[opcode] == (CALL)); - JUMP_TO_PREDICTED(TRACING_CALL); - } - STAT_INC(CALL, hit); - PyCFunction cfunc = meth->ml_meth; - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *res_o = _PyCFunction_TrampolineCall(cfunc, - PyStackRef_AsPyObjectBorrow(self_stackref), - PyStackRef_AsPyObjectBorrow(arg_stackref)); - stack_pointer = _PyFrame_GetStackPointer(frame); - _Py_LeaveRecursiveCallTstate(tstate); - assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyStackRef tmp; - for (int _i = oparg; --_i >= 0;) { - tmp = args[_i]; - args[_i] = PyStackRef_NULL; - PyStackRef_CLOSE(tmp); - } - tmp = self_or_null; - self_or_null = PyStackRef_NULL; - stack_pointer[-1 - oparg] = self_or_null; - PyStackRef_XCLOSE(tmp); - tmp = callable; - callable = PyStackRef_NULL; - stack_pointer[-2 - oparg] = callable; - PyStackRef_CLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -2 - oparg; - assert(WITHIN_STACK_BOUNDS()); - if (res_o == NULL) { - TRACING_JUMP_TO_LABEL(error); - } - res = PyStackRef_FromPyObjectSteal(res_o); - } - // _CHECK_PERIODIC_AT_END - { - stack_pointer[0] = res; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = check_periodics(tstate); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err != 0) { - TRACING_JUMP_TO_LABEL(error); - } - } - TRACING_DISPATCH(); - } - - TRACING_TARGET(CALL_NON_PY_GENERAL) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = CALL_NON_PY_GENERAL; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 4; - INSTRUCTION_STATS(CALL_NON_PY_GENERAL); - opcode = CALL_NON_PY_GENERAL; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; - (void)(_old_stack_level); - static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); - _PyStackRef callable; - _PyStackRef self_or_null; - _PyStackRef *args; - _PyStackRef res; - /* Skip 1 cache entry */ - /* Skip 2 cache entries */ - // _CHECK_IS_NOT_PY_CALLABLE - { - callable = stack_pointer[-2 - oparg]; - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); - if (PyFunction_Check(callable_o)) { - UPDATE_MISS_STATS(CALL); - assert(_PyOpcode_Deopt[opcode] == (CALL)); - JUMP_TO_PREDICTED(TRACING_CALL); - } - if (Py_TYPE(callable_o) == &PyMethod_Type) { - UPDATE_MISS_STATS(CALL); - assert(_PyOpcode_Deopt[opcode] == (CALL)); - JUMP_TO_PREDICTED(TRACING_CALL); - } - } - // _CALL_NON_PY_GENERAL - { - args = &stack_pointer[-oparg]; - self_or_null = stack_pointer[-1 - oparg]; - #if TIER_ONE - assert(opcode != INSTRUMENTED_CALL); - #endif - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); - int total_args = oparg; - _PyStackRef *arguments = args; - if (!PyStackRef_IsNull(self_or_null)) { - arguments--; - total_args++; - } - STACKREFS_TO_PYOBJECTS(arguments, total_args, args_o); - if (CONVERSION_FAILED(args_o)) { - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyStackRef tmp; - for (int _i = oparg; --_i >= 0;) { - tmp = args[_i]; - args[_i] = PyStackRef_NULL; - PyStackRef_CLOSE(tmp); - } - tmp = self_or_null; - self_or_null = PyStackRef_NULL; - stack_pointer[-1 - oparg] = self_or_null; - PyStackRef_XCLOSE(tmp); - tmp = callable; - callable = PyStackRef_NULL; - stack_pointer[-2 - oparg] = callable; - PyStackRef_CLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -2 - oparg; - assert(WITHIN_STACK_BOUNDS()); - TRACING_JUMP_TO_LABEL(error); - } - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *res_o = PyObject_Vectorcall( - callable_o, args_o, - total_args | PY_VECTORCALL_ARGUMENTS_OFFSET, - NULL); - stack_pointer = _PyFrame_GetStackPointer(frame); - STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); - assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyStackRef tmp; - for (int _i = oparg; --_i >= 0;) { - tmp = args[_i]; - args[_i] = PyStackRef_NULL; - PyStackRef_CLOSE(tmp); - } - tmp = self_or_null; - self_or_null = PyStackRef_NULL; - stack_pointer[-1 - oparg] = self_or_null; - PyStackRef_XCLOSE(tmp); - tmp = callable; - callable = PyStackRef_NULL; - stack_pointer[-2 - oparg] = callable; - PyStackRef_CLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -2 - oparg; - assert(WITHIN_STACK_BOUNDS()); - if (res_o == NULL) { - TRACING_JUMP_TO_LABEL(error); - } - res = PyStackRef_FromPyObjectSteal(res_o); - } - // _CHECK_PERIODIC_AT_END - { - stack_pointer[0] = res; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = check_periodics(tstate); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err != 0) { - TRACING_JUMP_TO_LABEL(error); - } - } - TRACING_DISPATCH(); - } - - TRACING_TARGET(CALL_PY_EXACT_ARGS) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = CALL_PY_EXACT_ARGS; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 4; - INSTRUCTION_STATS(CALL_PY_EXACT_ARGS); - opcode = CALL_PY_EXACT_ARGS; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; - (void)(_old_stack_level); - static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); - _PyStackRef callable; - _PyStackRef self_or_null; - _PyStackRef *args; - _PyStackRef new_frame; - /* Skip 1 cache entry */ - // _CHECK_PEP_523 - { - if (tstate->interp->eval_frame) { - UPDATE_MISS_STATS(CALL); - assert(_PyOpcode_Deopt[opcode] == (CALL)); - JUMP_TO_PREDICTED(TRACING_CALL); - } - } - // _CHECK_FUNCTION_VERSION - { - callable = stack_pointer[-2 - oparg]; - uint32_t func_version = read_u32(&this_instr[2].cache); - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); - if (!PyFunction_Check(callable_o)) { - UPDATE_MISS_STATS(CALL); - assert(_PyOpcode_Deopt[opcode] == (CALL)); - JUMP_TO_PREDICTED(TRACING_CALL); - } - PyFunctionObject *func = (PyFunctionObject *)callable_o; - if (func->func_version != func_version) { - UPDATE_MISS_STATS(CALL); - assert(_PyOpcode_Deopt[opcode] == (CALL)); - JUMP_TO_PREDICTED(TRACING_CALL); - } - } - // _CHECK_FUNCTION_EXACT_ARGS - { - self_or_null = stack_pointer[-1 - oparg]; - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); - assert(PyFunction_Check(callable_o)); - PyFunctionObject *func = (PyFunctionObject *)callable_o; - PyCodeObject *code = (PyCodeObject *)func->func_code; - if (code->co_argcount != oparg + (!PyStackRef_IsNull(self_or_null))) { - UPDATE_MISS_STATS(CALL); - assert(_PyOpcode_Deopt[opcode] == (CALL)); - JUMP_TO_PREDICTED(TRACING_CALL); - } - } - // _CHECK_STACK_SPACE - { - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); - PyFunctionObject *func = (PyFunctionObject *)callable_o; - PyCodeObject *code = (PyCodeObject *)func->func_code; - if (!_PyThreadState_HasStackSpace(tstate, code->co_framesize)) { - UPDATE_MISS_STATS(CALL); - assert(_PyOpcode_Deopt[opcode] == (CALL)); - JUMP_TO_PREDICTED(TRACING_CALL); - } - } - // _CHECK_RECURSION_REMAINING - { - if (tstate->py_recursion_remaining <= 1) { - UPDATE_MISS_STATS(CALL); - assert(_PyOpcode_Deopt[opcode] == (CALL)); - JUMP_TO_PREDICTED(TRACING_CALL); - } - } - // _INIT_CALL_PY_EXACT_ARGS - { - args = &stack_pointer[-oparg]; - int has_self = !PyStackRef_IsNull(self_or_null); - STAT_INC(CALL, hit); - _PyInterpreterFrame *pushed_frame = _PyFrame_PushUnchecked(tstate, callable, oparg + has_self, frame); - _PyStackRef *first_non_self_local = pushed_frame->localsplus + has_self; - pushed_frame->localsplus[0] = self_or_null; - for (int i = 0; i < oparg; i++) { - first_non_self_local[i] = args[i]; - } - new_frame = PyStackRef_Wrap(pushed_frame); - } - // _SAVE_RETURN_OFFSET - { - #if TIER_ONE - frame->return_offset = (uint16_t)(next_instr - this_instr); - #endif - #if TIER_TWO - frame->return_offset = oparg; - #endif - } - // _PUSH_FRAME - { - assert(tstate->interp->eval_frame == NULL); - _PyInterpreterFrame *temp = PyStackRef_Unwrap(new_frame); - stack_pointer += -2 - oparg; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - assert(temp->previous == frame || temp->previous->previous == frame); - CALL_STAT_INC(inlined_py_calls); - frame = tstate->current_frame = temp; - tstate->py_recursion_remaining--; - LOAD_SP(); - LOAD_IP(0); - LLTRACE_RESUME_FRAME(); - } - TRACING_DISPATCH(); - } - - TRACING_TARGET(CALL_PY_GENERAL) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = CALL_PY_GENERAL; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 4; - INSTRUCTION_STATS(CALL_PY_GENERAL); - opcode = CALL_PY_GENERAL; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; - (void)(_old_stack_level); - static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); - _PyStackRef callable; - _PyStackRef self_or_null; - _PyStackRef *args; - _PyStackRef new_frame; - /* Skip 1 cache entry */ - // _CHECK_PEP_523 - { - if (tstate->interp->eval_frame) { - UPDATE_MISS_STATS(CALL); - assert(_PyOpcode_Deopt[opcode] == (CALL)); - JUMP_TO_PREDICTED(TRACING_CALL); - } - } - // _CHECK_FUNCTION_VERSION - { - callable = stack_pointer[-2 - oparg]; - uint32_t func_version = read_u32(&this_instr[2].cache); - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); - if (!PyFunction_Check(callable_o)) { - UPDATE_MISS_STATS(CALL); - assert(_PyOpcode_Deopt[opcode] == (CALL)); - JUMP_TO_PREDICTED(TRACING_CALL); - } - PyFunctionObject *func = (PyFunctionObject *)callable_o; - if (func->func_version != func_version) { - UPDATE_MISS_STATS(CALL); - assert(_PyOpcode_Deopt[opcode] == (CALL)); - JUMP_TO_PREDICTED(TRACING_CALL); - } - } - // _CHECK_RECURSION_REMAINING - { - if (tstate->py_recursion_remaining <= 1) { - UPDATE_MISS_STATS(CALL); - assert(_PyOpcode_Deopt[opcode] == (CALL)); - JUMP_TO_PREDICTED(TRACING_CALL); - } - } - // _PY_FRAME_GENERAL - { - args = &stack_pointer[-oparg]; - self_or_null = stack_pointer[-1 - oparg]; - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); - int total_args = oparg; - if (!PyStackRef_IsNull(self_or_null)) { - args--; - total_args++; - } - assert(Py_TYPE(callable_o) == &PyFunction_Type); - int code_flags = ((PyCodeObject*)PyFunction_GET_CODE(callable_o))->co_flags; - PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable_o)); - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyInterpreterFrame *temp = _PyEvalFramePushAndInit( - tstate, callable, locals, - args, total_args, NULL, frame - ); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -2 - oparg; - assert(WITHIN_STACK_BOUNDS()); - if (temp == NULL) { - TRACING_JUMP_TO_LABEL(error); - } - new_frame = PyStackRef_Wrap(temp); - } - // _SAVE_RETURN_OFFSET - { - #if TIER_ONE - frame->return_offset = (uint16_t)(next_instr - this_instr); - #endif - #if TIER_TWO - frame->return_offset = oparg; - #endif - } - // _PUSH_FRAME - { - assert(tstate->interp->eval_frame == NULL); - _PyInterpreterFrame *temp = PyStackRef_Unwrap(new_frame); - _PyFrame_SetStackPointer(frame, stack_pointer); - assert(temp->previous == frame || temp->previous->previous == frame); - CALL_STAT_INC(inlined_py_calls); - frame = tstate->current_frame = temp; - tstate->py_recursion_remaining--; - LOAD_SP(); - LOAD_IP(0); - LLTRACE_RESUME_FRAME(); - } - TRACING_DISPATCH(); - } - - TRACING_TARGET(CALL_STR_1) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = CALL_STR_1; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 4; - INSTRUCTION_STATS(CALL_STR_1); - opcode = CALL_STR_1; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; - (void)(_old_stack_level); - static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); - _PyStackRef null; - _PyStackRef callable; - _PyStackRef arg; - _PyStackRef res; - /* Skip 1 cache entry */ - /* Skip 2 cache entries */ - // _GUARD_NOS_NULL - { - null = stack_pointer[-2]; - if (!PyStackRef_IsNull(null)) { - UPDATE_MISS_STATS(CALL); - assert(_PyOpcode_Deopt[opcode] == (CALL)); - JUMP_TO_PREDICTED(TRACING_CALL); - } - } - // _GUARD_CALLABLE_STR_1 - { - callable = stack_pointer[-3]; - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); - if (callable_o != (PyObject *)&PyUnicode_Type) { - UPDATE_MISS_STATS(CALL); - assert(_PyOpcode_Deopt[opcode] == (CALL)); - JUMP_TO_PREDICTED(TRACING_CALL); - } - } - // _CALL_STR_1 - { - arg = stack_pointer[-1]; - PyObject *arg_o = PyStackRef_AsPyObjectBorrow(arg); - assert(oparg == 1); - STAT_INC(CALL, hit); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *res_o = PyObject_Str(arg_o); - stack_pointer = _PyFrame_GetStackPointer(frame); - (void)callable; - (void)null; - stack_pointer += -3; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(arg); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (res_o == NULL) { - TRACING_JUMP_TO_LABEL(error); - } - res = PyStackRef_FromPyObjectSteal(res_o); - } - // _CHECK_PERIODIC_AT_END - { - stack_pointer[0] = res; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = check_periodics(tstate); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err != 0) { - TRACING_JUMP_TO_LABEL(error); - } - } - TRACING_DISPATCH(); - } - - TRACING_TARGET(CALL_TUPLE_1) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = CALL_TUPLE_1; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 4; - INSTRUCTION_STATS(CALL_TUPLE_1); - opcode = CALL_TUPLE_1; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; - (void)(_old_stack_level); - static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); - _PyStackRef null; - _PyStackRef callable; - _PyStackRef arg; - _PyStackRef res; - /* Skip 1 cache entry */ - /* Skip 2 cache entries */ - // _GUARD_NOS_NULL - { - null = stack_pointer[-2]; - if (!PyStackRef_IsNull(null)) { - UPDATE_MISS_STATS(CALL); - assert(_PyOpcode_Deopt[opcode] == (CALL)); - JUMP_TO_PREDICTED(TRACING_CALL); - } - } - // _GUARD_CALLABLE_TUPLE_1 - { - callable = stack_pointer[-3]; - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); - if (callable_o != (PyObject *)&PyTuple_Type) { - UPDATE_MISS_STATS(CALL); - assert(_PyOpcode_Deopt[opcode] == (CALL)); - JUMP_TO_PREDICTED(TRACING_CALL); - } - } - // _CALL_TUPLE_1 - { - arg = stack_pointer[-1]; - PyObject *arg_o = PyStackRef_AsPyObjectBorrow(arg); - assert(oparg == 1); - STAT_INC(CALL, hit); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *res_o = PySequence_Tuple(arg_o); - stack_pointer = _PyFrame_GetStackPointer(frame); - (void)callable; - (void)null; - stack_pointer += -3; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(arg); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (res_o == NULL) { - TRACING_JUMP_TO_LABEL(error); - } - res = PyStackRef_FromPyObjectSteal(res_o); - } - // _CHECK_PERIODIC_AT_END - { - stack_pointer[0] = res; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = check_periodics(tstate); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err != 0) { - TRACING_JUMP_TO_LABEL(error); - } - } - TRACING_DISPATCH(); - } - - TRACING_TARGET(CALL_TYPE_1) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = CALL_TYPE_1; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 4; - INSTRUCTION_STATS(CALL_TYPE_1); - opcode = CALL_TYPE_1; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; - (void)(_old_stack_level); - static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); - _PyStackRef null; - _PyStackRef callable; - _PyStackRef arg; - _PyStackRef res; - /* Skip 1 cache entry */ - /* Skip 2 cache entries */ - // _GUARD_NOS_NULL - { - null = stack_pointer[-2]; - if (!PyStackRef_IsNull(null)) { - UPDATE_MISS_STATS(CALL); - assert(_PyOpcode_Deopt[opcode] == (CALL)); - JUMP_TO_PREDICTED(TRACING_CALL); - } - } - // _GUARD_CALLABLE_TYPE_1 - { - callable = stack_pointer[-3]; - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); - if (callable_o != (PyObject *)&PyType_Type) { - UPDATE_MISS_STATS(CALL); - assert(_PyOpcode_Deopt[opcode] == (CALL)); - JUMP_TO_PREDICTED(TRACING_CALL); - } - } - // _CALL_TYPE_1 - { - arg = stack_pointer[-1]; - PyObject *arg_o = PyStackRef_AsPyObjectBorrow(arg); - assert(oparg == 1); - (void)callable; - (void)null; - STAT_INC(CALL, hit); - res = PyStackRef_FromPyObjectNew(Py_TYPE(arg_o)); - stack_pointer[-3] = res; - stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(arg); - stack_pointer = _PyFrame_GetStackPointer(frame); - } - TRACING_DISPATCH(); - } - - TRACING_TARGET(CHECK_EG_MATCH) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = CHECK_EG_MATCH; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(CHECK_EG_MATCH); - opcode = CHECK_EG_MATCH; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; - (void)(_old_stack_level); - _PyStackRef exc_value_st; - _PyStackRef match_type_st; - _PyStackRef rest; - _PyStackRef match; - match_type_st = stack_pointer[-1]; - exc_value_st = stack_pointer[-2]; - PyObject *exc_value = PyStackRef_AsPyObjectBorrow(exc_value_st); - PyObject *match_type = PyStackRef_AsPyObjectBorrow(match_type_st); - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = _PyEval_CheckExceptStarTypeValid(tstate, match_type); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err < 0) { - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyStackRef tmp = match_type_st; - match_type_st = PyStackRef_NULL; - stack_pointer[-1] = match_type_st; - PyStackRef_CLOSE(tmp); - tmp = exc_value_st; - exc_value_st = PyStackRef_NULL; - stack_pointer[-2] = exc_value_st; - PyStackRef_CLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); - TRACING_JUMP_TO_LABEL(error); - } - PyObject *match_o = NULL; - PyObject *rest_o = NULL; - _PyFrame_SetStackPointer(frame, stack_pointer); - int res = _PyEval_ExceptionGroupMatch(frame, exc_value, match_type, - &match_o, &rest_o); - _PyStackRef tmp = match_type_st; - match_type_st = PyStackRef_NULL; - stack_pointer[-1] = match_type_st; - PyStackRef_CLOSE(tmp); - tmp = exc_value_st; - exc_value_st = PyStackRef_NULL; - stack_pointer[-2] = exc_value_st; - PyStackRef_CLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); - if (res < 0) { - TRACING_JUMP_TO_LABEL(error); - } - assert((match_o == NULL) == (rest_o == NULL)); - if (match_o == NULL) { - TRACING_JUMP_TO_LABEL(error); - } - if (!Py_IsNone(match_o)) { - _PyFrame_SetStackPointer(frame, stack_pointer); - PyErr_SetHandledException(match_o); - stack_pointer = _PyFrame_GetStackPointer(frame); - } - rest = PyStackRef_FromPyObjectSteal(rest_o); - match = PyStackRef_FromPyObjectSteal(match_o); - stack_pointer[0] = rest; - stack_pointer[1] = match; - stack_pointer += 2; - assert(WITHIN_STACK_BOUNDS()); - TRACING_DISPATCH(); - } - - TRACING_TARGET(CHECK_EXC_MATCH) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = CHECK_EXC_MATCH; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(CHECK_EXC_MATCH); - opcode = CHECK_EXC_MATCH; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; - (void)(_old_stack_level); - _PyStackRef left; - _PyStackRef right; - _PyStackRef b; - right = stack_pointer[-1]; - left = stack_pointer[-2]; - PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); - PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - assert(PyExceptionInstance_Check(left_o)); - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = _PyEval_CheckExceptTypeValid(tstate, right_o); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err < 0) { - TRACING_JUMP_TO_LABEL(error); - } - _PyFrame_SetStackPointer(frame, stack_pointer); - int res = PyErr_GivenExceptionMatches(left_o, right_o); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(right); - stack_pointer = _PyFrame_GetStackPointer(frame); - b = res ? PyStackRef_True : PyStackRef_False; - stack_pointer[0] = b; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - TRACING_DISPATCH(); - } - - TRACING_TARGET(CLEANUP_THROW) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = CLEANUP_THROW; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(CLEANUP_THROW); - opcode = CLEANUP_THROW; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; - (void)(_old_stack_level); - _PyStackRef sub_iter; - _PyStackRef last_sent_val; - _PyStackRef exc_value_st; - _PyStackRef none; - _PyStackRef value; - exc_value_st = stack_pointer[-1]; - last_sent_val = stack_pointer[-2]; - sub_iter = stack_pointer[-3]; - PyObject *exc_value = PyStackRef_AsPyObjectBorrow(exc_value_st); - #if !_Py_TAIL_CALL_INTERP - assert(throwflag); - #endif - assert(exc_value && PyExceptionInstance_Check(exc_value)); - _PyFrame_SetStackPointer(frame, stack_pointer); - int matches = PyErr_GivenExceptionMatches(exc_value, PyExc_StopIteration); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (matches) { - value = PyStackRef_FromPyObjectNew(((PyStopIterationObject *)exc_value)->value); - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyStackRef tmp = sub_iter; - sub_iter = value; - stack_pointer[-3] = sub_iter; - PyStackRef_CLOSE(tmp); - tmp = exc_value_st; - exc_value_st = PyStackRef_NULL; - stack_pointer[-1] = exc_value_st; - PyStackRef_CLOSE(tmp); - tmp = last_sent_val; - last_sent_val = PyStackRef_NULL; - stack_pointer[-2] = last_sent_val; - PyStackRef_CLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -3; - assert(WITHIN_STACK_BOUNDS()); - none = PyStackRef_None; - } - else { - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyErr_SetRaisedException(tstate, Py_NewRef(exc_value)); - monitor_reraise(tstate, frame, this_instr); - TRACING_JUMP_TO_LABEL(exception_unwind); - } - stack_pointer[0] = none; - stack_pointer[1] = value; - stack_pointer += 2; - assert(WITHIN_STACK_BOUNDS()); - TRACING_DISPATCH(); - } - - TRACING_TARGET(COMPARE_OP) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = COMPARE_OP; - (void)(opcode); - #endif - frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(COMPARE_OP); - PREDICTED_TRACING_COMPARE_OP:; - _Py_CODEUNIT* const this_instr = next_instr - 2; - (void)this_instr; - opcode = COMPARE_OP; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; - (void)(_old_stack_level); - _PyStackRef left; - _PyStackRef right; - _PyStackRef res; - // _SPECIALIZE_COMPARE_OP - { - right = stack_pointer[-1]; - left = stack_pointer[-2]; - uint16_t counter = read_u16(&this_instr[1].cache); - (void)counter; - #if ENABLE_SPECIALIZATION_FT - if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { - next_instr = this_instr; - _PyFrame_SetStackPointer(frame, stack_pointer); - _Py_Specialize_CompareOp(left, right, next_instr, oparg); - stack_pointer = _PyFrame_GetStackPointer(frame); - TRACING_SPECIALIZE_DISPATCH_SAME_OPARG() - } - OPCODE_DEFERRED_INC(COMPARE_OP); - ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); - #endif /* ENABLE_SPECIALIZATION_FT */ - } - // _COMPARE_OP - { - PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); - PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - assert((oparg >> 5) <= Py_GE); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *res_o = PyObject_RichCompare(left_o, right_o, oparg >> 5); - _PyStackRef tmp = right; - right = PyStackRef_NULL; - stack_pointer[-1] = right; - PyStackRef_CLOSE(tmp); - tmp = left; - left = PyStackRef_NULL; - stack_pointer[-2] = left; - PyStackRef_CLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); - if (res_o == NULL) { - TRACING_JUMP_TO_LABEL(error); - } - if (oparg & 16) { - _PyFrame_SetStackPointer(frame, stack_pointer); - int res_bool = PyObject_IsTrue(res_o); - Py_DECREF(res_o); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (res_bool < 0) { - TRACING_JUMP_TO_LABEL(error); - } - res = res_bool ? PyStackRef_True : PyStackRef_False; - } - else { - res = PyStackRef_FromPyObjectSteal(res_o); - } - } - stack_pointer[0] = res; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - TRACING_DISPATCH(); - } - - TRACING_TARGET(COMPARE_OP_FLOAT) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = COMPARE_OP_FLOAT; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(COMPARE_OP_FLOAT); - opcode = COMPARE_OP_FLOAT; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; - (void)(_old_stack_level); - static_assert(INLINE_CACHE_ENTRIES_COMPARE_OP == 1, "incorrect cache size"); - _PyStackRef value; - _PyStackRef left; - _PyStackRef right; - _PyStackRef res; - // _GUARD_TOS_FLOAT - { - value = stack_pointer[-1]; - PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); - if (!PyFloat_CheckExact(value_o)) { - UPDATE_MISS_STATS(COMPARE_OP); - assert(_PyOpcode_Deopt[opcode] == (COMPARE_OP)); - JUMP_TO_PREDICTED(TRACING_COMPARE_OP); - } - } - // _GUARD_NOS_FLOAT - { - left = stack_pointer[-2]; - PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); - if (!PyFloat_CheckExact(left_o)) { - UPDATE_MISS_STATS(COMPARE_OP); - assert(_PyOpcode_Deopt[opcode] == (COMPARE_OP)); - JUMP_TO_PREDICTED(TRACING_COMPARE_OP); - } - } - /* Skip 1 cache entry */ - // _COMPARE_OP_FLOAT - { - right = value; - PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); - PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - STAT_INC(COMPARE_OP, hit); - double dleft = PyFloat_AS_DOUBLE(left_o); - double dright = PyFloat_AS_DOUBLE(right_o); - int sign_ish = COMPARISON_BIT(dleft, dright); - PyStackRef_CLOSE_SPECIALIZED(left, _PyFloat_ExactDealloc); - PyStackRef_CLOSE_SPECIALIZED(right, _PyFloat_ExactDealloc); - res = (sign_ish & oparg) ? PyStackRef_True : PyStackRef_False; - } - stack_pointer[-2] = res; - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - TRACING_DISPATCH(); - } - - TRACING_TARGET(COMPARE_OP_INT) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = COMPARE_OP_INT; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(COMPARE_OP_INT); - opcode = COMPARE_OP_INT; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; - (void)(_old_stack_level); - static_assert(INLINE_CACHE_ENTRIES_COMPARE_OP == 1, "incorrect cache size"); - _PyStackRef value; - _PyStackRef left; - _PyStackRef right; - _PyStackRef res; - // _GUARD_TOS_INT - { - value = stack_pointer[-1]; - PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); - if (!_PyLong_CheckExactAndCompact(value_o)) { - UPDATE_MISS_STATS(COMPARE_OP); - assert(_PyOpcode_Deopt[opcode] == (COMPARE_OP)); - JUMP_TO_PREDICTED(TRACING_COMPARE_OP); - } - } - // _GUARD_NOS_INT - { - left = stack_pointer[-2]; - PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); - if (!_PyLong_CheckExactAndCompact(left_o)) { - UPDATE_MISS_STATS(COMPARE_OP); - assert(_PyOpcode_Deopt[opcode] == (COMPARE_OP)); - JUMP_TO_PREDICTED(TRACING_COMPARE_OP); - } - } - /* Skip 1 cache entry */ - // _COMPARE_OP_INT - { - right = value; - PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); - PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - assert(_PyLong_IsCompact((PyLongObject *)left_o)); - assert(_PyLong_IsCompact((PyLongObject *)right_o)); - STAT_INC(COMPARE_OP, hit); - assert(_PyLong_DigitCount((PyLongObject *)left_o) <= 1 && - _PyLong_DigitCount((PyLongObject *)right_o) <= 1); - Py_ssize_t ileft = _PyLong_CompactValue((PyLongObject *)left_o); - Py_ssize_t iright = _PyLong_CompactValue((PyLongObject *)right_o); - int sign_ish = COMPARISON_BIT(ileft, iright); - PyStackRef_CLOSE_SPECIALIZED(left, _PyLong_ExactDealloc); - PyStackRef_CLOSE_SPECIALIZED(right, _PyLong_ExactDealloc); - res = (sign_ish & oparg) ? PyStackRef_True : PyStackRef_False; - } - stack_pointer[-2] = res; - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - TRACING_DISPATCH(); - } - - TRACING_TARGET(COMPARE_OP_STR) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = COMPARE_OP_STR; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(COMPARE_OP_STR); - opcode = COMPARE_OP_STR; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; - (void)(_old_stack_level); - static_assert(INLINE_CACHE_ENTRIES_COMPARE_OP == 1, "incorrect cache size"); - _PyStackRef value; - _PyStackRef nos; - _PyStackRef left; - _PyStackRef right; - _PyStackRef res; - // _GUARD_TOS_UNICODE - { - value = stack_pointer[-1]; - PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); - if (!PyUnicode_CheckExact(value_o)) { - UPDATE_MISS_STATS(COMPARE_OP); - assert(_PyOpcode_Deopt[opcode] == (COMPARE_OP)); - JUMP_TO_PREDICTED(TRACING_COMPARE_OP); - } - } - // _GUARD_NOS_UNICODE - { - nos = stack_pointer[-2]; - PyObject *o = PyStackRef_AsPyObjectBorrow(nos); - if (!PyUnicode_CheckExact(o)) { - UPDATE_MISS_STATS(COMPARE_OP); - assert(_PyOpcode_Deopt[opcode] == (COMPARE_OP)); - JUMP_TO_PREDICTED(TRACING_COMPARE_OP); - } - } - /* Skip 1 cache entry */ - // _COMPARE_OP_STR - { - right = value; - left = nos; - PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); - PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - STAT_INC(COMPARE_OP, hit); - int eq = _PyUnicode_Equal(left_o, right_o); - assert((oparg >> 5) == Py_EQ || (oparg >> 5) == Py_NE); - PyStackRef_CLOSE_SPECIALIZED(left, _PyUnicode_ExactDealloc); - PyStackRef_CLOSE_SPECIALIZED(right, _PyUnicode_ExactDealloc); - assert(eq == 0 || eq == 1); - assert((oparg & 0xf) == COMPARISON_NOT_EQUALS || (oparg & 0xf) == COMPARISON_EQUALS); - assert(COMPARISON_NOT_EQUALS + 1 == COMPARISON_EQUALS); - res = ((COMPARISON_NOT_EQUALS + eq) & oparg) ? PyStackRef_True : PyStackRef_False; - } - stack_pointer[-2] = res; - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - TRACING_DISPATCH(); - } - - TRACING_TARGET(CONTAINS_OP) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = CONTAINS_OP; - (void)(opcode); - #endif - frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(CONTAINS_OP); - PREDICTED_TRACING_CONTAINS_OP:; - _Py_CODEUNIT* const this_instr = next_instr - 2; - (void)this_instr; - opcode = CONTAINS_OP; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; - (void)(_old_stack_level); - _PyStackRef right; - _PyStackRef left; - _PyStackRef b; - // _SPECIALIZE_CONTAINS_OP - { - right = stack_pointer[-1]; - uint16_t counter = read_u16(&this_instr[1].cache); - (void)counter; - #if ENABLE_SPECIALIZATION_FT - if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { - next_instr = this_instr; - _PyFrame_SetStackPointer(frame, stack_pointer); - _Py_Specialize_ContainsOp(right, next_instr); - stack_pointer = _PyFrame_GetStackPointer(frame); - TRACING_SPECIALIZE_DISPATCH_SAME_OPARG() - } - OPCODE_DEFERRED_INC(CONTAINS_OP); - ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); - #endif /* ENABLE_SPECIALIZATION_FT */ - } - // _CONTAINS_OP - { - left = stack_pointer[-2]; - PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); - PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - _PyFrame_SetStackPointer(frame, stack_pointer); - int res = PySequence_Contains(right_o, left_o); - _PyStackRef tmp = right; - right = PyStackRef_NULL; - stack_pointer[-1] = right; - PyStackRef_CLOSE(tmp); - tmp = left; - left = PyStackRef_NULL; - stack_pointer[-2] = left; - PyStackRef_CLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); - if (res < 0) { - TRACING_JUMP_TO_LABEL(error); - } - b = (res ^ oparg) ? PyStackRef_True : PyStackRef_False; - } - stack_pointer[0] = b; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - TRACING_DISPATCH(); - } - - TRACING_TARGET(CONTAINS_OP_DICT) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = CONTAINS_OP_DICT; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(CONTAINS_OP_DICT); - opcode = CONTAINS_OP_DICT; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; - (void)(_old_stack_level); - static_assert(INLINE_CACHE_ENTRIES_CONTAINS_OP == 1, "incorrect cache size"); - _PyStackRef tos; - _PyStackRef left; - _PyStackRef right; - _PyStackRef b; - // _GUARD_TOS_DICT - { - tos = stack_pointer[-1]; - PyObject *o = PyStackRef_AsPyObjectBorrow(tos); - if (!PyDict_CheckExact(o)) { - UPDATE_MISS_STATS(CONTAINS_OP); - assert(_PyOpcode_Deopt[opcode] == (CONTAINS_OP)); - JUMP_TO_PREDICTED(TRACING_CONTAINS_OP); - } - } - /* Skip 1 cache entry */ - // _CONTAINS_OP_DICT - { - right = tos; - left = stack_pointer[-2]; - PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); - PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - assert(PyDict_CheckExact(right_o)); - STAT_INC(CONTAINS_OP, hit); - _PyFrame_SetStackPointer(frame, stack_pointer); - int res = PyDict_Contains(right_o, left_o); - _PyStackRef tmp = right; - right = PyStackRef_NULL; - stack_pointer[-1] = right; - PyStackRef_CLOSE(tmp); - tmp = left; - left = PyStackRef_NULL; - stack_pointer[-2] = left; - PyStackRef_CLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); - if (res < 0) { - TRACING_JUMP_TO_LABEL(error); - } - b = (res ^ oparg) ? PyStackRef_True : PyStackRef_False; - } - stack_pointer[0] = b; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - TRACING_DISPATCH(); - } - - TRACING_TARGET(CONTAINS_OP_SET) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = CONTAINS_OP_SET; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(CONTAINS_OP_SET); - opcode = CONTAINS_OP_SET; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; - (void)(_old_stack_level); - static_assert(INLINE_CACHE_ENTRIES_CONTAINS_OP == 1, "incorrect cache size"); - _PyStackRef tos; - _PyStackRef left; - _PyStackRef right; - _PyStackRef b; - // _GUARD_TOS_ANY_SET - { - tos = stack_pointer[-1]; - PyObject *o = PyStackRef_AsPyObjectBorrow(tos); - if (!PyAnySet_CheckExact(o)) { - UPDATE_MISS_STATS(CONTAINS_OP); - assert(_PyOpcode_Deopt[opcode] == (CONTAINS_OP)); - JUMP_TO_PREDICTED(TRACING_CONTAINS_OP); - } - } - /* Skip 1 cache entry */ - // _CONTAINS_OP_SET - { - right = tos; - left = stack_pointer[-2]; - PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); - PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - assert(PyAnySet_CheckExact(right_o)); - STAT_INC(CONTAINS_OP, hit); - _PyFrame_SetStackPointer(frame, stack_pointer); - int res = _PySet_Contains((PySetObject *)right_o, left_o); - _PyStackRef tmp = right; - right = PyStackRef_NULL; - stack_pointer[-1] = right; - PyStackRef_CLOSE(tmp); - tmp = left; - left = PyStackRef_NULL; - stack_pointer[-2] = left; - PyStackRef_CLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); - if (res < 0) { - TRACING_JUMP_TO_LABEL(error); - } - b = (res ^ oparg) ? PyStackRef_True : PyStackRef_False; - } - stack_pointer[0] = b; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - TRACING_DISPATCH(); - } - - TRACING_TARGET(CONVERT_VALUE) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = CONVERT_VALUE; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(CONVERT_VALUE); - opcode = CONVERT_VALUE; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; - (void)(_old_stack_level); - _PyStackRef value; - _PyStackRef result; - value = stack_pointer[-1]; - conversion_func conv_fn; - assert(oparg >= FVC_STR && oparg <= FVC_ASCII); - conv_fn = _PyEval_ConversionFuncs[oparg]; - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *result_o = conv_fn(PyStackRef_AsPyObjectBorrow(value)); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(value); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (result_o == NULL) { - TRACING_JUMP_TO_LABEL(error); - } - result = PyStackRef_FromPyObjectSteal(result_o); - stack_pointer[0] = result; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - TRACING_DISPATCH(); - } - - TRACING_TARGET(COPY) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = COPY; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(COPY); - opcode = COPY; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; - (void)(_old_stack_level); - _PyStackRef bottom; - _PyStackRef top; - bottom = stack_pointer[-1 - (oparg-1)]; - top = PyStackRef_DUP(bottom); - stack_pointer[0] = top; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - TRACING_DISPATCH(); - } - - TRACING_TARGET(COPY_FREE_VARS) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = COPY_FREE_VARS; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(COPY_FREE_VARS); - opcode = COPY_FREE_VARS; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; - (void)(_old_stack_level); - PyCodeObject *co = _PyFrame_GetCode(frame); - assert(PyStackRef_FunctionCheck(frame->f_funcobj)); - PyFunctionObject *func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - PyObject *closure = func->func_closure; - assert(oparg == co->co_nfreevars); - int offset = co->co_nlocalsplus - oparg; - for (int i = 0; i < oparg; ++i) { - PyObject *o = PyTuple_GET_ITEM(closure, i); - frame->localsplus[offset + i] = PyStackRef_FromPyObjectNew(o); - } - TRACING_DISPATCH(); - } - - TRACING_TARGET(DELETE_ATTR) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = DELETE_ATTR; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(DELETE_ATTR); - opcode = DELETE_ATTR; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; - (void)(_old_stack_level); - _PyStackRef owner; - owner = stack_pointer[-1]; - PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = PyObject_DelAttr(PyStackRef_AsPyObjectBorrow(owner), name); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(owner); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err) { - TRACING_JUMP_TO_LABEL(error); - } - TRACING_DISPATCH(); - } - - TRACING_TARGET(DELETE_DEREF) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = DELETE_DEREF; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(DELETE_DEREF); - opcode = DELETE_DEREF; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; - (void)(_old_stack_level); - PyObject *cell = PyStackRef_AsPyObjectBorrow(GETLOCAL(oparg)); - PyObject *oldobj = PyCell_SwapTakeRef((PyCellObject *)cell, NULL); - if (oldobj == NULL) { - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyEval_FormatExcUnbound(tstate, _PyFrame_GetCode(frame), oparg); - stack_pointer = _PyFrame_GetStackPointer(frame); - TRACING_JUMP_TO_LABEL(error); - } - _PyFrame_SetStackPointer(frame, stack_pointer); - Py_DECREF(oldobj); - stack_pointer = _PyFrame_GetStackPointer(frame); - TRACING_DISPATCH(); - } - - TRACING_TARGET(DELETE_FAST) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = DELETE_FAST; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(DELETE_FAST); - opcode = DELETE_FAST; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; - (void)(_old_stack_level); - _PyStackRef v = GETLOCAL(oparg); - if (PyStackRef_IsNull(v)) { - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyEval_FormatExcCheckArg(tstate, PyExc_UnboundLocalError, - UNBOUNDLOCAL_ERROR_MSG, - PyTuple_GetItem(_PyFrame_GetCode(frame)->co_localsplusnames, oparg) - ); - stack_pointer = _PyFrame_GetStackPointer(frame); - TRACING_JUMP_TO_LABEL(error); - } - _PyStackRef tmp = GETLOCAL(oparg); - GETLOCAL(oparg) = PyStackRef_NULL; - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_XCLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); - TRACING_DISPATCH(); - } - - TRACING_TARGET(DELETE_GLOBAL) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = DELETE_GLOBAL; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(DELETE_GLOBAL); - opcode = DELETE_GLOBAL; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; - (void)(_old_stack_level); - PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = PyDict_Pop(GLOBALS(), name, NULL); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err < 0) { - TRACING_JUMP_TO_LABEL(error); - } - if (err == 0) { - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyEval_FormatExcCheckArg(tstate, PyExc_NameError, - NAME_ERROR_MSG, name); - stack_pointer = _PyFrame_GetStackPointer(frame); - TRACING_JUMP_TO_LABEL(error); - } - TRACING_DISPATCH(); - } - - TRACING_TARGET(DELETE_NAME) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = DELETE_NAME; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(DELETE_NAME); - opcode = DELETE_NAME; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; - (void)(_old_stack_level); - PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); - PyObject *ns = LOCALS(); - int err; - if (ns == NULL) { - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyErr_Format(tstate, PyExc_SystemError, - "no locals when deleting %R", name); - stack_pointer = _PyFrame_GetStackPointer(frame); - TRACING_JUMP_TO_LABEL(error); - } - _PyFrame_SetStackPointer(frame, stack_pointer); - err = PyObject_DelItem(ns, name); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err != 0) { - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyEval_FormatExcCheckArg(tstate, PyExc_NameError, - NAME_ERROR_MSG, - name); - stack_pointer = _PyFrame_GetStackPointer(frame); - TRACING_JUMP_TO_LABEL(error); - } - TRACING_DISPATCH(); - } - - TRACING_TARGET(DELETE_SUBSCR) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = DELETE_SUBSCR; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(DELETE_SUBSCR); - opcode = DELETE_SUBSCR; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; - (void)(_old_stack_level); - _PyStackRef container; - _PyStackRef sub; - sub = stack_pointer[-1]; - container = stack_pointer[-2]; - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = PyObject_DelItem(PyStackRef_AsPyObjectBorrow(container), - PyStackRef_AsPyObjectBorrow(sub)); - _PyStackRef tmp = sub; - sub = PyStackRef_NULL; - stack_pointer[-1] = sub; - PyStackRef_CLOSE(tmp); - tmp = container; - container = PyStackRef_NULL; - stack_pointer[-2] = container; - PyStackRef_CLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); - if (err) { - TRACING_JUMP_TO_LABEL(error); - } - TRACING_DISPATCH(); - } - - TRACING_TARGET(DICT_MERGE) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = DICT_MERGE; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(DICT_MERGE); - opcode = DICT_MERGE; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; - (void)(_old_stack_level); - _PyStackRef callable; - _PyStackRef dict; - _PyStackRef update; - update = stack_pointer[-1]; - dict = stack_pointer[-2 - (oparg - 1)]; - callable = stack_pointer[-5 - (oparg - 1)]; - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); - PyObject *dict_o = PyStackRef_AsPyObjectBorrow(dict); - PyObject *update_o = PyStackRef_AsPyObjectBorrow(update); - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = _PyDict_MergeEx(dict_o, update_o, 2); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err < 0) { - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyEval_FormatKwargsError(tstate, callable_o, update_o); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(update); - stack_pointer = _PyFrame_GetStackPointer(frame); - TRACING_JUMP_TO_LABEL(error); - } - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(update); - stack_pointer = _PyFrame_GetStackPointer(frame); - TRACING_DISPATCH(); - } - - TRACING_TARGET(DICT_UPDATE) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = DICT_UPDATE; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(DICT_UPDATE); - opcode = DICT_UPDATE; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; - (void)(_old_stack_level); - _PyStackRef dict; - _PyStackRef update; - update = stack_pointer[-1]; - dict = stack_pointer[-2 - (oparg - 1)]; - PyObject *dict_o = PyStackRef_AsPyObjectBorrow(dict); - PyObject *update_o = PyStackRef_AsPyObjectBorrow(update); - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = PyDict_Update(dict_o, update_o); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err < 0) { - _PyFrame_SetStackPointer(frame, stack_pointer); - int matches = _PyErr_ExceptionMatches(tstate, PyExc_AttributeError); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (matches) { - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyErr_Format(tstate, PyExc_TypeError, - "'%.200s' object is not a mapping", - Py_TYPE(update_o)->tp_name); - stack_pointer = _PyFrame_GetStackPointer(frame); - } - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(update); - stack_pointer = _PyFrame_GetStackPointer(frame); - TRACING_JUMP_TO_LABEL(error); - } - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(update); - stack_pointer = _PyFrame_GetStackPointer(frame); - TRACING_DISPATCH(); - } - - TRACING_TARGET(END_ASYNC_FOR) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = END_ASYNC_FOR; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(END_ASYNC_FOR); - opcode = END_ASYNC_FOR; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; - (void)(_old_stack_level); - _PyStackRef awaitable_st; - _PyStackRef exc_st; - exc_st = stack_pointer[-1]; - awaitable_st = stack_pointer[-2]; - JUMPBY(0); - (void)oparg; - PyObject *exc = PyStackRef_AsPyObjectBorrow(exc_st); - assert(exc && PyExceptionInstance_Check(exc)); - _PyFrame_SetStackPointer(frame, stack_pointer); - int matches = PyErr_GivenExceptionMatches(exc, PyExc_StopAsyncIteration); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (matches) { - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyStackRef tmp = exc_st; - exc_st = PyStackRef_NULL; - stack_pointer[-1] = exc_st; - PyStackRef_CLOSE(tmp); - tmp = awaitable_st; - awaitable_st = PyStackRef_NULL; - stack_pointer[-2] = awaitable_st; - PyStackRef_CLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); - } - else { - Py_INCREF(exc); - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyErr_SetRaisedException(tstate, exc); - monitor_reraise(tstate, frame, this_instr); - TRACING_JUMP_TO_LABEL(exception_unwind); - } - TRACING_DISPATCH(); - } - - TRACING_TARGET(END_FOR) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = END_FOR; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - next_instr += 1; - INSTRUCTION_STATS(END_FOR); - opcode = END_FOR; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; - (void)(_old_stack_level); - _PyStackRef value; - value = stack_pointer[-1]; - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(value); - stack_pointer = _PyFrame_GetStackPointer(frame); - TRACING_DISPATCH(); - } - - TRACING_TARGET(END_SEND) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = END_SEND; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(END_SEND); - opcode = END_SEND; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; - (void)(_old_stack_level); - _PyStackRef receiver; - _PyStackRef value; - _PyStackRef val; - value = stack_pointer[-1]; - receiver = stack_pointer[-2]; - val = value; - stack_pointer[-2] = val; - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(receiver); - stack_pointer = _PyFrame_GetStackPointer(frame); - TRACING_DISPATCH(); - } - - TRACING_TARGET(ENTER_EXECUTOR) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = ENTER_EXECUTOR; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(ENTER_EXECUTOR); - opcode = ENTER_EXECUTOR; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; - (void)(_old_stack_level); - #ifdef _Py_TIER2 - PyCodeObject *code = _PyFrame_GetCode(frame); - _PyExecutorObject *executor = code->co_executors->executors[oparg & 255]; - assert(executor->vm_data.index == INSTR_OFFSET() - 1); - assert(executor->vm_data.code == code); - assert(executor->vm_data.valid); - assert(tstate->current_executor == NULL); - if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { - opcode = executor->vm_data.opcode; - oparg = (oparg & ~255) | executor->vm_data.oparg; - next_instr = this_instr; - if (_PyOpcode_Caches[_PyOpcode_Deopt[opcode]]) { - PAUSE_ADAPTIVE_COUNTER(this_instr[1].counter); - } - DISPATCH_GOTO(); - } - assert(executor != tstate->interp->cold_executor); - tstate->jit_exit = NULL; - #if TRACING_JIT - RECORD_TRACE_NO_DISPATCH(); - #endif - TIER1_TO_TIER2(executor); - #else - Py_FatalError("ENTER_EXECUTOR is not supported in this build"); - #endif /* _Py_TIER2 */ - TRACING_DISPATCH(); - } - - TRACING_TARGET(EXIT_INIT_CHECK) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = EXIT_INIT_CHECK; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(EXIT_INIT_CHECK); - opcode = EXIT_INIT_CHECK; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; - (void)(_old_stack_level); - _PyStackRef should_be_none; - should_be_none = stack_pointer[-1]; - if (!PyStackRef_IsNone(should_be_none)) { - _PyFrame_SetStackPointer(frame, stack_pointer); - PyErr_Format(PyExc_TypeError, - "__init__() should return None, not '%.200s'", - Py_TYPE(PyStackRef_AsPyObjectBorrow(should_be_none))->tp_name); - stack_pointer = _PyFrame_GetStackPointer(frame); - TRACING_JUMP_TO_LABEL(error); - } - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - TRACING_DISPATCH(); - } - - TRACING_TARGET(EXTENDED_ARG) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = EXTENDED_ARG; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(EXTENDED_ARG); - opcode = EXTENDED_ARG; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; - (void)(_old_stack_level); - assert(oparg); - opcode = next_instr->op.code; - oparg = oparg << 8 | next_instr->op.arg; - PRE_DISPATCH_GOTO(); - DISPATCH_GOTO(); - } - - TRACING_TARGET(FORMAT_SIMPLE) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = FORMAT_SIMPLE; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(FORMAT_SIMPLE); - opcode = FORMAT_SIMPLE; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; - (void)(_old_stack_level); - _PyStackRef value; - _PyStackRef res; - value = stack_pointer[-1]; - PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); - if (!PyUnicode_CheckExact(value_o)) { - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *res_o = PyObject_Format(value_o, NULL); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(value); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (res_o == NULL) { - TRACING_JUMP_TO_LABEL(error); - } - res = PyStackRef_FromPyObjectSteal(res_o); - } - else { - res = value; - stack_pointer += -1; - } - stack_pointer[0] = res; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - TRACING_DISPATCH(); - } - - TRACING_TARGET(FORMAT_WITH_SPEC) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = FORMAT_WITH_SPEC; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(FORMAT_WITH_SPEC); - opcode = FORMAT_WITH_SPEC; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; - (void)(_old_stack_level); - _PyStackRef value; - _PyStackRef fmt_spec; - _PyStackRef res; - fmt_spec = stack_pointer[-1]; - value = stack_pointer[-2]; - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *res_o = PyObject_Format(PyStackRef_AsPyObjectBorrow(value), PyStackRef_AsPyObjectBorrow(fmt_spec)); - _PyStackRef tmp = fmt_spec; - fmt_spec = PyStackRef_NULL; - stack_pointer[-1] = fmt_spec; - PyStackRef_CLOSE(tmp); - tmp = value; - value = PyStackRef_NULL; - stack_pointer[-2] = value; - PyStackRef_CLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); - if (res_o == NULL) { - TRACING_JUMP_TO_LABEL(error); - } - res = PyStackRef_FromPyObjectSteal(res_o); - stack_pointer[0] = res; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - TRACING_DISPATCH(); - } - - TRACING_TARGET(FOR_ITER) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = FOR_ITER; - (void)(opcode); - #endif - frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(FOR_ITER); - PREDICTED_TRACING_FOR_ITER:; - _Py_CODEUNIT* const this_instr = next_instr - 2; - (void)this_instr; - opcode = FOR_ITER; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; - (void)(_old_stack_level); - _PyStackRef iter; - _PyStackRef null_or_index; - _PyStackRef next; - // _SPECIALIZE_FOR_ITER - { - null_or_index = stack_pointer[-1]; - iter = stack_pointer[-2]; - uint16_t counter = read_u16(&this_instr[1].cache); - (void)counter; - #if ENABLE_SPECIALIZATION_FT - if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { - next_instr = this_instr; - _PyFrame_SetStackPointer(frame, stack_pointer); - _Py_Specialize_ForIter(iter, null_or_index, next_instr, oparg); - stack_pointer = _PyFrame_GetStackPointer(frame); - TRACING_SPECIALIZE_DISPATCH_SAME_OPARG() - } - OPCODE_DEFERRED_INC(FOR_ITER); - ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); - #endif /* ENABLE_SPECIALIZATION_FT */ - } - // _FOR_ITER - { - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyStackRef item = _PyForIter_VirtualIteratorNext(tstate, frame, iter, &null_or_index); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (!PyStackRef_IsValid(item)) { - if (PyStackRef_IsError(item)) { - TRACING_JUMP_TO_LABEL(error); - } - JUMPBY(oparg + 1); - RECORD_DYNAMIC_JUMP_TAKEN(); - stack_pointer[-1] = null_or_index; - TRACING_DISPATCH(); - } - next = item; - } - stack_pointer[-1] = null_or_index; - stack_pointer[0] = next; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - TRACING_DISPATCH(); - } - - TRACING_TARGET(FOR_ITER_GEN) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = FOR_ITER_GEN; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(FOR_ITER_GEN); - opcode = FOR_ITER_GEN; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; - (void)(_old_stack_level); - static_assert(INLINE_CACHE_ENTRIES_FOR_ITER == 1, "incorrect cache size"); - _PyStackRef iter; - _PyStackRef gen_frame; - _PyStackRef new_frame; - /* Skip 1 cache entry */ - // _CHECK_PEP_523 - { - if (tstate->interp->eval_frame) { - UPDATE_MISS_STATS(FOR_ITER); - assert(_PyOpcode_Deopt[opcode] == (FOR_ITER)); - JUMP_TO_PREDICTED(TRACING_FOR_ITER); - } - } - // _FOR_ITER_GEN_FRAME - { - iter = stack_pointer[-2]; - PyGenObject *gen = (PyGenObject *)PyStackRef_AsPyObjectBorrow(iter); - if (Py_TYPE(gen) != &PyGen_Type) { - UPDATE_MISS_STATS(FOR_ITER); - assert(_PyOpcode_Deopt[opcode] == (FOR_ITER)); - JUMP_TO_PREDICTED(TRACING_FOR_ITER); - } - #ifdef Py_GIL_DISABLED - if (!_PyObject_IsUniquelyReferenced((PyObject *)gen)) { - UPDATE_MISS_STATS(FOR_ITER); - assert(_PyOpcode_Deopt[opcode] == (FOR_ITER)); - JUMP_TO_PREDICTED(TRACING_FOR_ITER); - } - #endif - if (gen->gi_frame_state >= FRAME_EXECUTING) { - UPDATE_MISS_STATS(FOR_ITER); - assert(_PyOpcode_Deopt[opcode] == (FOR_ITER)); - JUMP_TO_PREDICTED(TRACING_FOR_ITER); - } - STAT_INC(FOR_ITER, hit); - _PyInterpreterFrame *pushed_frame = &gen->gi_iframe; - _PyFrame_StackPush(pushed_frame, PyStackRef_None); - gen->gi_frame_state = FRAME_EXECUTING; - gen->gi_exc_state.previous_item = tstate->exc_info; - tstate->exc_info = &gen->gi_exc_state; - pushed_frame->previous = frame; - frame->return_offset = (uint16_t)( 2u + oparg); - gen_frame = PyStackRef_Wrap(pushed_frame); - } - // _PUSH_FRAME - { - new_frame = gen_frame; - assert(tstate->interp->eval_frame == NULL); - _PyInterpreterFrame *temp = PyStackRef_Unwrap(new_frame); - _PyFrame_SetStackPointer(frame, stack_pointer); - assert(temp->previous == frame || temp->previous->previous == frame); - CALL_STAT_INC(inlined_py_calls); - frame = tstate->current_frame = temp; - tstate->py_recursion_remaining--; - LOAD_SP(); - LOAD_IP(0); - LLTRACE_RESUME_FRAME(); - } - TRACING_DISPATCH(); - } - - TRACING_TARGET(FOR_ITER_LIST) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = FOR_ITER_LIST; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(FOR_ITER_LIST); - opcode = FOR_ITER_LIST; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; - (void)(_old_stack_level); - static_assert(INLINE_CACHE_ENTRIES_FOR_ITER == 1, "incorrect cache size"); - _PyStackRef iter; - _PyStackRef null_or_index; - _PyStackRef next; - /* Skip 1 cache entry */ - // _ITER_CHECK_LIST - { - null_or_index = stack_pointer[-1]; - iter = stack_pointer[-2]; - PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter); - if (Py_TYPE(iter_o) != &PyList_Type) { - UPDATE_MISS_STATS(FOR_ITER); - assert(_PyOpcode_Deopt[opcode] == (FOR_ITER)); - JUMP_TO_PREDICTED(TRACING_FOR_ITER); - } - assert(PyStackRef_IsTaggedInt(null_or_index)); - #ifdef Py_GIL_DISABLED - if (!_Py_IsOwnedByCurrentThread(iter_o) && !_PyObject_GC_IS_SHARED(iter_o)) { - UPDATE_MISS_STATS(FOR_ITER); - assert(_PyOpcode_Deopt[opcode] == (FOR_ITER)); - JUMP_TO_PREDICTED(TRACING_FOR_ITER); - } - #endif - } - // _ITER_JUMP_LIST - { - #ifdef Py_GIL_DISABLED - - #else - PyObject *list_o = PyStackRef_AsPyObjectBorrow(iter); - assert(Py_TYPE(list_o) == &PyList_Type); - STAT_INC(FOR_ITER, hit); - if ((size_t)PyStackRef_UntagInt(null_or_index) >= (size_t)PyList_GET_SIZE(list_o)) { - null_or_index = PyStackRef_TagInt(-1); - JUMPBY(oparg + 1); - RECORD_DYNAMIC_JUMP_TAKEN(); - stack_pointer[-1] = null_or_index; - TRACING_DISPATCH(); - } - #endif - } - // _ITER_NEXT_LIST - { - PyObject *list_o = PyStackRef_AsPyObjectBorrow(iter); - assert(PyList_CheckExact(list_o)); - #ifdef Py_GIL_DISABLED - assert(_Py_IsOwnedByCurrentThread(list_o) || - _PyObject_GC_IS_SHARED(list_o)); - STAT_INC(FOR_ITER, hit); - _PyFrame_SetStackPointer(frame, stack_pointer); - int result = _PyList_GetItemRefNoLock((PyListObject *)list_o, PyStackRef_UntagInt(null_or_index), &next); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (result < 0) { - UPDATE_MISS_STATS(FOR_ITER); - assert(_PyOpcode_Deopt[opcode] == (FOR_ITER)); - JUMP_TO_PREDICTED(TRACING_FOR_ITER); - } - if (result == 0) { - null_or_index = PyStackRef_TagInt(-1); - JUMPBY(oparg + 1); - stack_pointer[-1] = null_or_index; - TRACING_DISPATCH(); - } - #else - next = PyStackRef_FromPyObjectNew(PyList_GET_ITEM(list_o, PyStackRef_UntagInt(null_or_index))); - #endif - null_or_index = PyStackRef_IncrementTaggedIntNoOverflow(null_or_index); - } - stack_pointer[-1] = null_or_index; - stack_pointer[0] = next; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - TRACING_DISPATCH(); - } - - TRACING_TARGET(FOR_ITER_RANGE) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = FOR_ITER_RANGE; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(FOR_ITER_RANGE); - opcode = FOR_ITER_RANGE; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; - (void)(_old_stack_level); - static_assert(INLINE_CACHE_ENTRIES_FOR_ITER == 1, "incorrect cache size"); - _PyStackRef iter; - _PyStackRef next; - /* Skip 1 cache entry */ - // _ITER_CHECK_RANGE - { - iter = stack_pointer[-2]; - _PyRangeIterObject *r = (_PyRangeIterObject *)PyStackRef_AsPyObjectBorrow(iter); - if (Py_TYPE(r) != &PyRangeIter_Type) { - UPDATE_MISS_STATS(FOR_ITER); - assert(_PyOpcode_Deopt[opcode] == (FOR_ITER)); - JUMP_TO_PREDICTED(TRACING_FOR_ITER); - } - #ifdef Py_GIL_DISABLED - if (!_PyObject_IsUniquelyReferenced((PyObject *)r)) { - UPDATE_MISS_STATS(FOR_ITER); - assert(_PyOpcode_Deopt[opcode] == (FOR_ITER)); - JUMP_TO_PREDICTED(TRACING_FOR_ITER); - } - #endif - } - // _ITER_JUMP_RANGE - { - _PyRangeIterObject *r = (_PyRangeIterObject *)PyStackRef_AsPyObjectBorrow(iter); - assert(Py_TYPE(r) == &PyRangeIter_Type); - #ifdef Py_GIL_DISABLED - assert(_PyObject_IsUniquelyReferenced((PyObject *)r)); - #endif - STAT_INC(FOR_ITER, hit); - if (r->len <= 0) { - JUMPBY(oparg + 1); - RECORD_DYNAMIC_JUMP_TAKEN(); - TRACING_DISPATCH(); - } - } - // _ITER_NEXT_RANGE - { - _PyRangeIterObject *r = (_PyRangeIterObject *)PyStackRef_AsPyObjectBorrow(iter); - assert(Py_TYPE(r) == &PyRangeIter_Type); - #ifdef Py_GIL_DISABLED - assert(_PyObject_IsUniquelyReferenced((PyObject *)r)); - #endif - assert(r->len > 0); - long value = r->start; - r->start = value + r->step; - r->len--; - PyObject *res = PyLong_FromLong(value); - if (res == NULL) { - TRACING_JUMP_TO_LABEL(error); - } - next = PyStackRef_FromPyObjectSteal(res); - } - stack_pointer[0] = next; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - TRACING_DISPATCH(); - } - - TRACING_TARGET(FOR_ITER_TUPLE) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = FOR_ITER_TUPLE; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(FOR_ITER_TUPLE); - opcode = FOR_ITER_TUPLE; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; - (void)(_old_stack_level); - static_assert(INLINE_CACHE_ENTRIES_FOR_ITER == 1, "incorrect cache size"); - _PyStackRef iter; - _PyStackRef null_or_index; - _PyStackRef next; - /* Skip 1 cache entry */ - // _ITER_CHECK_TUPLE - { - null_or_index = stack_pointer[-1]; - iter = stack_pointer[-2]; - PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter); - if (Py_TYPE(iter_o) != &PyTuple_Type) { - UPDATE_MISS_STATS(FOR_ITER); - assert(_PyOpcode_Deopt[opcode] == (FOR_ITER)); - JUMP_TO_PREDICTED(TRACING_FOR_ITER); - } - assert(PyStackRef_IsTaggedInt(null_or_index)); - } - // _ITER_JUMP_TUPLE - { - PyObject *tuple_o = PyStackRef_AsPyObjectBorrow(iter); - (void)tuple_o; - assert(Py_TYPE(tuple_o) == &PyTuple_Type); - STAT_INC(FOR_ITER, hit); - if ((size_t)PyStackRef_UntagInt(null_or_index) >= (size_t)PyTuple_GET_SIZE(tuple_o)) { - null_or_index = PyStackRef_TagInt(-1); - JUMPBY(oparg + 1); - RECORD_DYNAMIC_JUMP_TAKEN(); - stack_pointer[-1] = null_or_index; - TRACING_DISPATCH(); - } - } - // _ITER_NEXT_TUPLE - { - PyObject *tuple_o = PyStackRef_AsPyObjectBorrow(iter); - assert(Py_TYPE(tuple_o) == &PyTuple_Type); - uintptr_t i = PyStackRef_UntagInt(null_or_index); - assert((size_t)i < (size_t)PyTuple_GET_SIZE(tuple_o)); - next = PyStackRef_FromPyObjectNew(PyTuple_GET_ITEM(tuple_o, i)); - null_or_index = PyStackRef_IncrementTaggedIntNoOverflow(null_or_index); - } - stack_pointer[-1] = null_or_index; - stack_pointer[0] = next; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - TRACING_DISPATCH(); - } - - TRACING_TARGET(GET_AITER) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = GET_AITER; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(GET_AITER); - opcode = GET_AITER; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; - (void)(_old_stack_level); - _PyStackRef obj; - _PyStackRef iter; - obj = stack_pointer[-1]; - unaryfunc getter = NULL; - PyObject *obj_o = PyStackRef_AsPyObjectBorrow(obj); - PyObject *iter_o; - PyTypeObject *type = Py_TYPE(obj_o); - if (type->tp_as_async != NULL) { - getter = type->tp_as_async->am_aiter; - } - if (getter == NULL) { - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyErr_Format(tstate, PyExc_TypeError, - "'async for' requires an object with " - "__aiter__ method, got %.100s", - type->tp_name); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(obj); - stack_pointer = _PyFrame_GetStackPointer(frame); - TRACING_JUMP_TO_LABEL(error); - } - _PyFrame_SetStackPointer(frame, stack_pointer); - iter_o = (*getter)(obj_o); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(obj); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (iter_o == NULL) { - TRACING_JUMP_TO_LABEL(error); - } - if (Py_TYPE(iter_o)->tp_as_async == NULL || - Py_TYPE(iter_o)->tp_as_async->am_anext == NULL) { - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyErr_Format(tstate, PyExc_TypeError, - "'async for' received an object from __aiter__ " - "that does not implement __anext__: %.100s", - Py_TYPE(iter_o)->tp_name); - Py_DECREF(iter_o); - stack_pointer = _PyFrame_GetStackPointer(frame); - TRACING_JUMP_TO_LABEL(error); - } - iter = PyStackRef_FromPyObjectSteal(iter_o); - stack_pointer[0] = iter; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - TRACING_DISPATCH(); - } - - TRACING_TARGET(GET_ANEXT) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = GET_ANEXT; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(GET_ANEXT); - opcode = GET_ANEXT; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; - (void)(_old_stack_level); - _PyStackRef aiter; - _PyStackRef awaitable; - aiter = stack_pointer[-1]; - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *awaitable_o = _PyEval_GetANext(PyStackRef_AsPyObjectBorrow(aiter)); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (awaitable_o == NULL) { - TRACING_JUMP_TO_LABEL(error); - } - awaitable = PyStackRef_FromPyObjectSteal(awaitable_o); - stack_pointer[0] = awaitable; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - TRACING_DISPATCH(); - } - - TRACING_TARGET(GET_AWAITABLE) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = GET_AWAITABLE; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(GET_AWAITABLE); - opcode = GET_AWAITABLE; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; - (void)(_old_stack_level); - _PyStackRef iterable; - _PyStackRef iter; - iterable = stack_pointer[-1]; - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *iter_o = _PyEval_GetAwaitable(PyStackRef_AsPyObjectBorrow(iterable), oparg); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(iterable); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (iter_o == NULL) { - TRACING_JUMP_TO_LABEL(error); - } - iter = PyStackRef_FromPyObjectSteal(iter_o); - stack_pointer[0] = iter; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - TRACING_DISPATCH(); - } - - TRACING_TARGET(GET_ITER) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = GET_ITER; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(GET_ITER); - opcode = GET_ITER; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; - (void)(_old_stack_level); - _PyStackRef iterable; - _PyStackRef iter; - _PyStackRef index_or_null; - iterable = stack_pointer[-1]; - #ifdef Py_STATS - _PyFrame_SetStackPointer(frame, stack_pointer); - _Py_GatherStats_GetIter(iterable); - stack_pointer = _PyFrame_GetStackPointer(frame); - #endif - - PyTypeObject *tp = PyStackRef_TYPE(iterable); - if (tp == &PyTuple_Type || tp == &PyList_Type) { - iter = iterable; - index_or_null = PyStackRef_TagInt(0); - } - else { - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *iter_o = PyObject_GetIter(PyStackRef_AsPyObjectBorrow(iterable)); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(iterable); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (iter_o == NULL) { - TRACING_JUMP_TO_LABEL(error); - } - iter = PyStackRef_FromPyObjectSteal(iter_o); - index_or_null = PyStackRef_NULL; - stack_pointer += 1; - } - stack_pointer[-1] = iter; - stack_pointer[0] = index_or_null; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - TRACING_DISPATCH(); - } - - TRACING_TARGET(GET_LEN) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = GET_LEN; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(GET_LEN); - opcode = GET_LEN; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; - (void)(_old_stack_level); - _PyStackRef obj; - _PyStackRef len; - obj = stack_pointer[-1]; - _PyFrame_SetStackPointer(frame, stack_pointer); - Py_ssize_t len_i = PyObject_Length(PyStackRef_AsPyObjectBorrow(obj)); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (len_i < 0) { - TRACING_JUMP_TO_LABEL(error); - } - PyObject *len_o = PyLong_FromSsize_t(len_i); - if (len_o == NULL) { - TRACING_JUMP_TO_LABEL(error); - } - len = PyStackRef_FromPyObjectSteal(len_o); - stack_pointer[0] = len; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - TRACING_DISPATCH(); - } - - TRACING_TARGET(GET_YIELD_FROM_ITER) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = GET_YIELD_FROM_ITER; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(GET_YIELD_FROM_ITER); - opcode = GET_YIELD_FROM_ITER; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; - (void)(_old_stack_level); - _PyStackRef iterable; - _PyStackRef iter; - iterable = stack_pointer[-1]; - PyObject *iterable_o = PyStackRef_AsPyObjectBorrow(iterable); - if (PyCoro_CheckExact(iterable_o)) { - if (!(_PyFrame_GetCode(frame)->co_flags & (CO_COROUTINE | CO_ITERABLE_COROUTINE))) { - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyErr_SetString(tstate, PyExc_TypeError, - "cannot 'yield from' a coroutine object " - "in a non-coroutine generator"); - stack_pointer = _PyFrame_GetStackPointer(frame); - TRACING_JUMP_TO_LABEL(error); - } - iter = iterable; - } - else if (PyGen_CheckExact(iterable_o)) { - iter = iterable; - } - else { - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *iter_o = PyObject_GetIter(iterable_o); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (iter_o == NULL) { - TRACING_JUMP_TO_LABEL(error); - } - iter = PyStackRef_FromPyObjectSteal(iter_o); - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyStackRef tmp = iterable; - iterable = iter; - stack_pointer[-1] = iterable; - PyStackRef_CLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); - } - stack_pointer[-1] = iter; - TRACING_DISPATCH(); - } - - TRACING_TARGET(IMPORT_FROM) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = IMPORT_FROM; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(IMPORT_FROM); - opcode = IMPORT_FROM; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; - (void)(_old_stack_level); - _PyStackRef from; - _PyStackRef res; - from = stack_pointer[-1]; - PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *res_o = _PyEval_ImportFrom(tstate, PyStackRef_AsPyObjectBorrow(from), name); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (res_o == NULL) { - TRACING_JUMP_TO_LABEL(error); - } - res = PyStackRef_FromPyObjectSteal(res_o); - stack_pointer[0] = res; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - TRACING_DISPATCH(); - } - - TRACING_TARGET(IMPORT_NAME) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = IMPORT_NAME; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(IMPORT_NAME); - opcode = IMPORT_NAME; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; - (void)(_old_stack_level); - _PyStackRef level; - _PyStackRef fromlist; - _PyStackRef res; - fromlist = stack_pointer[-1]; - level = stack_pointer[-2]; - PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *res_o = _PyEval_ImportName(tstate, frame, name, - PyStackRef_AsPyObjectBorrow(fromlist), - PyStackRef_AsPyObjectBorrow(level)); - _PyStackRef tmp = fromlist; - fromlist = PyStackRef_NULL; - stack_pointer[-1] = fromlist; - PyStackRef_CLOSE(tmp); - tmp = level; - level = PyStackRef_NULL; - stack_pointer[-2] = level; - PyStackRef_CLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); - if (res_o == NULL) { - TRACING_JUMP_TO_LABEL(error); - } - res = PyStackRef_FromPyObjectSteal(res_o); - stack_pointer[0] = res; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - TRACING_DISPATCH(); - } - - TRACING_TARGET(INSTRUMENTED_CALL) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = INSTRUMENTED_CALL; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 4; - INSTRUCTION_STATS(INSTRUMENTED_CALL); - opcode = INSTRUMENTED_CALL; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; - (void)(_old_stack_level); - _PyStackRef callable; - _PyStackRef self_or_null; - _PyStackRef func; - _PyStackRef maybe_self; - _PyStackRef *args; - _PyStackRef res; - /* Skip 3 cache entries */ - // _MAYBE_EXPAND_METHOD - { - self_or_null = stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; - if (PyStackRef_TYPE(callable) == &PyMethod_Type && PyStackRef_IsNull(self_or_null)) { - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); - PyObject *self = ((PyMethodObject *)callable_o)->im_self; - self_or_null = PyStackRef_FromPyObjectNew(self); - PyObject *method = ((PyMethodObject *)callable_o)->im_func; - _PyStackRef temp = callable; - callable = PyStackRef_FromPyObjectNew(method); - stack_pointer[-2 - oparg] = callable; - stack_pointer[-1 - oparg] = self_or_null; - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(temp); - stack_pointer = _PyFrame_GetStackPointer(frame); - } - } - // _MONITOR_CALL - { - args = &stack_pointer[-oparg]; - maybe_self = self_or_null; - func = callable; - int is_meth = !PyStackRef_IsNull(maybe_self); - PyObject *function = PyStackRef_AsPyObjectBorrow(func); - PyObject *arg0; - if (is_meth) { - arg0 = PyStackRef_AsPyObjectBorrow(maybe_self); - } - else if (oparg) { - arg0 = PyStackRef_AsPyObjectBorrow(args[0]); - } - else { - arg0 = &_PyInstrumentation_MISSING; - } - stack_pointer[-2 - oparg] = func; - stack_pointer[-1 - oparg] = maybe_self; - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = _Py_call_instrumentation_2args( - tstate, PY_MONITORING_EVENT_CALL, - frame, this_instr, function, arg0 - ); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err) { - TRACING_JUMP_TO_LABEL(error); - } - } - // _DO_CALL - { - args = &stack_pointer[-oparg]; - self_or_null = stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); - int total_args = oparg; - _PyStackRef *arguments = args; - if (!PyStackRef_IsNull(self_or_null)) { - arguments--; - total_args++; - } - if (Py_TYPE(callable_o) == &PyFunction_Type && - tstate->interp->eval_frame == NULL && - ((PyFunctionObject *)callable_o)->vectorcall == _PyFunction_Vectorcall) - { - int code_flags = ((PyCodeObject*)PyFunction_GET_CODE(callable_o))->co_flags; - PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable_o)); - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyInterpreterFrame *new_frame = _PyEvalFramePushAndInit( - tstate, callable, locals, - arguments, total_args, NULL, frame - ); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -2 - oparg; - assert(WITHIN_STACK_BOUNDS()); - if (new_frame == NULL) { - TRACING_JUMP_TO_LABEL(error); - } - frame->return_offset = 4u ; - TRACING_DISPATCH_INLINED(new_frame); - } - STACKREFS_TO_PYOBJECTS(arguments, total_args, args_o); - if (CONVERSION_FAILED(args_o)) { - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyStackRef tmp; - for (int _i = oparg; --_i >= 0;) { - tmp = args[_i]; - args[_i] = PyStackRef_NULL; - PyStackRef_CLOSE(tmp); - } - tmp = self_or_null; - self_or_null = PyStackRef_NULL; - stack_pointer[-1 - oparg] = self_or_null; - PyStackRef_XCLOSE(tmp); - tmp = callable; - callable = PyStackRef_NULL; - stack_pointer[-2 - oparg] = callable; - PyStackRef_CLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -2 - oparg; - assert(WITHIN_STACK_BOUNDS()); - TRACING_JUMP_TO_LABEL(error); - } - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *res_o = PyObject_Vectorcall( - callable_o, args_o, - total_args | PY_VECTORCALL_ARGUMENTS_OFFSET, - NULL); - stack_pointer = _PyFrame_GetStackPointer(frame); - STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); - if (opcode == INSTRUMENTED_CALL) { - PyObject *arg = total_args == 0 ? - &_PyInstrumentation_MISSING : PyStackRef_AsPyObjectBorrow(arguments[0]); - if (res_o == NULL) { - _PyFrame_SetStackPointer(frame, stack_pointer); - _Py_call_instrumentation_exc2( - tstate, PY_MONITORING_EVENT_C_RAISE, - frame, this_instr, callable_o, arg); - stack_pointer = _PyFrame_GetStackPointer(frame); - } - else { - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = _Py_call_instrumentation_2args( - tstate, PY_MONITORING_EVENT_C_RETURN, - frame, this_instr, callable_o, arg); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err < 0) { - _PyFrame_SetStackPointer(frame, stack_pointer); - Py_CLEAR(res_o); - stack_pointer = _PyFrame_GetStackPointer(frame); - } - } - } - assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyStackRef tmp; - for (int _i = oparg; --_i >= 0;) { - tmp = args[_i]; - args[_i] = PyStackRef_NULL; - PyStackRef_CLOSE(tmp); - } - tmp = self_or_null; - self_or_null = PyStackRef_NULL; - stack_pointer[-1 - oparg] = self_or_null; - PyStackRef_XCLOSE(tmp); - tmp = callable; - callable = PyStackRef_NULL; - stack_pointer[-2 - oparg] = callable; - PyStackRef_CLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -2 - oparg; - assert(WITHIN_STACK_BOUNDS()); - if (res_o == NULL) { - TRACING_JUMP_TO_LABEL(error); - } - res = PyStackRef_FromPyObjectSteal(res_o); - } - // _CHECK_PERIODIC_AT_END - { - stack_pointer[0] = res; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = check_periodics(tstate); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err != 0) { - TRACING_JUMP_TO_LABEL(error); - } - } - TRACING_DISPATCH(); - } - - TRACING_TARGET(INSTRUMENTED_CALL_FUNCTION_EX) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = INSTRUMENTED_CALL_FUNCTION_EX; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(INSTRUMENTED_CALL_FUNCTION_EX); - opcode = INSTRUMENTED_CALL_FUNCTION_EX; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; - (void)(_old_stack_level); - _PyStackRef func; - _PyStackRef callargs; - _PyStackRef func_st; - _PyStackRef null; - _PyStackRef callargs_st; - _PyStackRef kwargs_st; - _PyStackRef result; - // _MAKE_CALLARGS_A_TUPLE - { - callargs = stack_pointer[-2]; - func = stack_pointer[-4]; - PyObject *callargs_o = PyStackRef_AsPyObjectBorrow(callargs); - if (!PyTuple_CheckExact(callargs_o)) { - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = _Py_Check_ArgsIterable(tstate, PyStackRef_AsPyObjectBorrow(func), callargs_o); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err < 0) { - TRACING_JUMP_TO_LABEL(error); - } - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *tuple_o = PySequence_Tuple(callargs_o); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (tuple_o == NULL) { - TRACING_JUMP_TO_LABEL(error); - } - _PyStackRef temp = callargs; - callargs = PyStackRef_FromPyObjectSteal(tuple_o); - stack_pointer[-2] = callargs; - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(temp); - stack_pointer = _PyFrame_GetStackPointer(frame); - } - } - // _DO_CALL_FUNCTION_EX - { - kwargs_st = stack_pointer[-1]; - callargs_st = callargs; - null = stack_pointer[-3]; - func_st = func; - (void)null; - PyObject *func = PyStackRef_AsPyObjectBorrow(func_st); - EVAL_CALL_STAT_INC_IF_FUNCTION(EVAL_CALL_FUNCTION_EX, func); - PyObject *result_o; - assert(!_PyErr_Occurred(tstate)); - if (opcode == INSTRUMENTED_CALL_FUNCTION_EX) { - PyObject *callargs = PyStackRef_AsPyObjectBorrow(callargs_st); - PyObject *kwargs = PyStackRef_AsPyObjectBorrow(kwargs_st); - assert(kwargs == NULL || PyDict_CheckExact(kwargs)); - assert(PyTuple_CheckExact(callargs)); - PyObject *arg = PyTuple_GET_SIZE(callargs) > 0 ? - PyTuple_GET_ITEM(callargs, 0) : &_PyInstrumentation_MISSING; - stack_pointer[-2] = callargs_st; - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = _Py_call_instrumentation_2args( - tstate, PY_MONITORING_EVENT_CALL, - frame, this_instr, func, arg); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err) { - TRACING_JUMP_TO_LABEL(error); - } - _PyFrame_SetStackPointer(frame, stack_pointer); - result_o = PyObject_Call(func, callargs, kwargs); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (!PyFunction_Check(func) && !PyMethod_Check(func)) { - if (result_o == NULL) { - _PyFrame_SetStackPointer(frame, stack_pointer); - _Py_call_instrumentation_exc2( - tstate, PY_MONITORING_EVENT_C_RAISE, - frame, this_instr, func, arg); - stack_pointer = _PyFrame_GetStackPointer(frame); - } - else { - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = _Py_call_instrumentation_2args( - tstate, PY_MONITORING_EVENT_C_RETURN, - frame, this_instr, func, arg); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err < 0) { - _PyFrame_SetStackPointer(frame, stack_pointer); - Py_CLEAR(result_o); - stack_pointer = _PyFrame_GetStackPointer(frame); - } - } - } - } - else { - if (Py_TYPE(func) == &PyFunction_Type && - tstate->interp->eval_frame == NULL && - ((PyFunctionObject *)func)->vectorcall == _PyFunction_Vectorcall) { - PyObject *callargs = PyStackRef_AsPyObjectSteal(callargs_st); - assert(PyTuple_CheckExact(callargs)); - PyObject *kwargs = PyStackRef_IsNull(kwargs_st) ? NULL : PyStackRef_AsPyObjectSteal(kwargs_st); - assert(kwargs == NULL || PyDict_CheckExact(kwargs)); - Py_ssize_t nargs = PyTuple_GET_SIZE(callargs); - int code_flags = ((PyCodeObject *)PyFunction_GET_CODE(func))->co_flags; - PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(func)); - stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyInterpreterFrame *new_frame = _PyEvalFramePushAndInit_Ex( - tstate, func_st, locals, - nargs, callargs, kwargs, frame); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); - if (new_frame == NULL) { - TRACING_JUMP_TO_LABEL(error); - } - assert( 1u == 1); - frame->return_offset = 1; - TRACING_DISPATCH_INLINED(new_frame); - } - PyObject *callargs = PyStackRef_AsPyObjectBorrow(callargs_st); - assert(PyTuple_CheckExact(callargs)); - PyObject *kwargs = PyStackRef_AsPyObjectBorrow(kwargs_st); - assert(kwargs == NULL || PyDict_CheckExact(kwargs)); - stack_pointer[-2] = callargs_st; - _PyFrame_SetStackPointer(frame, stack_pointer); - result_o = PyObject_Call(func, callargs, kwargs); - stack_pointer = _PyFrame_GetStackPointer(frame); - } - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_XCLOSE(kwargs_st); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(callargs_st); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(func_st); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (result_o == NULL) { - TRACING_JUMP_TO_LABEL(error); - } - result = PyStackRef_FromPyObjectSteal(result_o); - } - // _CHECK_PERIODIC_AT_END - { - stack_pointer[0] = result; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = check_periodics(tstate); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err != 0) { - TRACING_JUMP_TO_LABEL(error); - } - } - TRACING_DISPATCH(); - } - - TRACING_TARGET(INSTRUMENTED_CALL_KW) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = INSTRUMENTED_CALL_KW; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 4; - INSTRUCTION_STATS(INSTRUMENTED_CALL_KW); - opcode = INSTRUMENTED_CALL_KW; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; - (void)(_old_stack_level); - _PyStackRef callable; - _PyStackRef self_or_null; - _PyStackRef *args; - _PyStackRef kwnames; - _PyStackRef res; - /* Skip 1 cache entry */ - /* Skip 2 cache entries */ - // _MAYBE_EXPAND_METHOD_KW - { - self_or_null = stack_pointer[-2 - oparg]; - callable = stack_pointer[-3 - oparg]; - if (PyStackRef_TYPE(callable) == &PyMethod_Type && PyStackRef_IsNull(self_or_null)) { - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); - PyObject *self = ((PyMethodObject *)callable_o)->im_self; - self_or_null = PyStackRef_FromPyObjectNew(self); - PyObject *method = ((PyMethodObject *)callable_o)->im_func; - _PyStackRef temp = callable; - callable = PyStackRef_FromPyObjectNew(method); - stack_pointer[-3 - oparg] = callable; - stack_pointer[-2 - oparg] = self_or_null; - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(temp); - stack_pointer = _PyFrame_GetStackPointer(frame); - } - } - // _MONITOR_CALL_KW - { - args = &stack_pointer[-1 - oparg]; - int is_meth = !PyStackRef_IsNull(self_or_null); - PyObject *arg; - if (is_meth) { - arg = PyStackRef_AsPyObjectBorrow(self_or_null); - } - else if (args) { - arg = PyStackRef_AsPyObjectBorrow(args[0]); - } - else { - arg = &_PyInstrumentation_MISSING; - } - PyObject *function = PyStackRef_AsPyObjectBorrow(callable); - stack_pointer[-3 - oparg] = callable; - stack_pointer[-2 - oparg] = self_or_null; - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = _Py_call_instrumentation_2args( - tstate, PY_MONITORING_EVENT_CALL, - frame, this_instr, function, arg); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err) { - TRACING_JUMP_TO_LABEL(error); - } - } - // _DO_CALL_KW - { - kwnames = stack_pointer[-1]; - args = &stack_pointer[-1 - oparg]; - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); - PyObject *kwnames_o = PyStackRef_AsPyObjectBorrow(kwnames); - int total_args = oparg; - _PyStackRef *arguments = args; - if (!PyStackRef_IsNull(self_or_null)) { - arguments--; - total_args++; - } - int positional_args = total_args - (int)PyTuple_GET_SIZE(kwnames_o); - if (Py_TYPE(callable_o) == &PyFunction_Type && - tstate->interp->eval_frame == NULL && - ((PyFunctionObject *)callable_o)->vectorcall == _PyFunction_Vectorcall) - { - int code_flags = ((PyCodeObject*)PyFunction_GET_CODE(callable_o))->co_flags; - PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable_o)); - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyInterpreterFrame *new_frame = _PyEvalFramePushAndInit( - tstate, callable, locals, - arguments, positional_args, kwnames_o, frame - ); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -3 - oparg; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(kwnames); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (new_frame == NULL) { - TRACING_JUMP_TO_LABEL(error); - } - assert( 4u == 1 + INLINE_CACHE_ENTRIES_CALL_KW); - frame->return_offset = 4u ; - TRACING_DISPATCH_INLINED(new_frame); - } - STACKREFS_TO_PYOBJECTS(arguments, total_args, args_o); - if (CONVERSION_FAILED(args_o)) { - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyStackRef tmp = kwnames; - kwnames = PyStackRef_NULL; - stack_pointer[-1] = kwnames; - PyStackRef_CLOSE(tmp); - for (int _i = oparg; --_i >= 0;) { - tmp = args[_i]; - args[_i] = PyStackRef_NULL; - PyStackRef_CLOSE(tmp); - } - tmp = self_or_null; - self_or_null = PyStackRef_NULL; - stack_pointer[-2 - oparg] = self_or_null; - PyStackRef_XCLOSE(tmp); - tmp = callable; - callable = PyStackRef_NULL; - stack_pointer[-3 - oparg] = callable; - PyStackRef_CLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -3 - oparg; - assert(WITHIN_STACK_BOUNDS()); - TRACING_JUMP_TO_LABEL(error); - } - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *res_o = PyObject_Vectorcall( - callable_o, args_o, - positional_args | PY_VECTORCALL_ARGUMENTS_OFFSET, - kwnames_o); - stack_pointer = _PyFrame_GetStackPointer(frame); - STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); - if (opcode == INSTRUMENTED_CALL_KW) { - PyObject *arg = total_args == 0 ? - &_PyInstrumentation_MISSING : PyStackRef_AsPyObjectBorrow(arguments[0]); - if (res_o == NULL) { - _PyFrame_SetStackPointer(frame, stack_pointer); - _Py_call_instrumentation_exc2( - tstate, PY_MONITORING_EVENT_C_RAISE, - frame, this_instr, callable_o, arg); - stack_pointer = _PyFrame_GetStackPointer(frame); - } - else { - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = _Py_call_instrumentation_2args( - tstate, PY_MONITORING_EVENT_C_RETURN, - frame, this_instr, callable_o, arg); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err < 0) { - _PyFrame_SetStackPointer(frame, stack_pointer); - Py_CLEAR(res_o); - stack_pointer = _PyFrame_GetStackPointer(frame); - } - } - } - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyStackRef tmp = kwnames; - kwnames = PyStackRef_NULL; - stack_pointer[-1] = kwnames; - PyStackRef_CLOSE(tmp); - for (int _i = oparg; --_i >= 0;) { - tmp = args[_i]; - args[_i] = PyStackRef_NULL; - PyStackRef_CLOSE(tmp); - } - tmp = self_or_null; - self_or_null = PyStackRef_NULL; - stack_pointer[-2 - oparg] = self_or_null; - PyStackRef_XCLOSE(tmp); - tmp = callable; - callable = PyStackRef_NULL; - stack_pointer[-3 - oparg] = callable; - PyStackRef_CLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -3 - oparg; - assert(WITHIN_STACK_BOUNDS()); - if (res_o == NULL) { - TRACING_JUMP_TO_LABEL(error); - } - res = PyStackRef_FromPyObjectSteal(res_o); - } - stack_pointer[0] = res; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - TRACING_DISPATCH(); - } - - TRACING_TARGET(INSTRUMENTED_END_ASYNC_FOR) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = INSTRUMENTED_END_ASYNC_FOR; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(INSTRUMENTED_END_ASYNC_FOR); - opcode = INSTRUMENTED_END_ASYNC_FOR; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; - (void)(_old_stack_level); - _PyStackRef awaitable_st; - _PyStackRef exc_st; - // _MONITOR_END_ASYNC_FOR - { - assert((next_instr-oparg)->op.code == END_SEND || (next_instr-oparg)->op.code >= MIN_INSTRUMENTED_OPCODE); - INSTRUMENTED_JUMP(next_instr-oparg, this_instr+1, PY_MONITORING_EVENT_BRANCH_RIGHT); - } - // _END_ASYNC_FOR - { - exc_st = stack_pointer[-1]; - awaitable_st = stack_pointer[-2]; - JUMPBY(0); - (void)oparg; - PyObject *exc = PyStackRef_AsPyObjectBorrow(exc_st); - assert(exc && PyExceptionInstance_Check(exc)); - _PyFrame_SetStackPointer(frame, stack_pointer); - int matches = PyErr_GivenExceptionMatches(exc, PyExc_StopAsyncIteration); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (matches) { - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyStackRef tmp = exc_st; - exc_st = PyStackRef_NULL; - stack_pointer[-1] = exc_st; - PyStackRef_CLOSE(tmp); - tmp = awaitable_st; - awaitable_st = PyStackRef_NULL; - stack_pointer[-2] = awaitable_st; - PyStackRef_CLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); - } - else { - Py_INCREF(exc); - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyErr_SetRaisedException(tstate, exc); - monitor_reraise(tstate, frame, this_instr); - TRACING_JUMP_TO_LABEL(exception_unwind); - } - } - TRACING_DISPATCH(); - } - - TRACING_TARGET(INSTRUMENTED_END_FOR) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = INSTRUMENTED_END_FOR; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - next_instr += 1; - INSTRUCTION_STATS(INSTRUMENTED_END_FOR); - opcode = INSTRUMENTED_END_FOR; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; - (void)(_old_stack_level); - _PyStackRef receiver; - _PyStackRef value; - value = stack_pointer[-1]; - receiver = stack_pointer[-3]; - if (PyStackRef_GenCheck(receiver)) { - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = monitor_stop_iteration(tstate, frame, this_instr, PyStackRef_AsPyObjectBorrow(value)); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err) { - TRACING_JUMP_TO_LABEL(error); - } - } - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(value); - stack_pointer = _PyFrame_GetStackPointer(frame); - TRACING_DISPATCH(); - } - - TRACING_TARGET(INSTRUMENTED_END_SEND) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = INSTRUMENTED_END_SEND; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(INSTRUMENTED_END_SEND); - opcode = INSTRUMENTED_END_SEND; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; - (void)(_old_stack_level); - _PyStackRef receiver; - _PyStackRef value; - _PyStackRef val; - value = stack_pointer[-1]; - receiver = stack_pointer[-2]; - PyObject *receiver_o = PyStackRef_AsPyObjectBorrow(receiver); - if (PyGen_Check(receiver_o) || PyCoro_CheckExact(receiver_o)) { - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = monitor_stop_iteration(tstate, frame, this_instr, PyStackRef_AsPyObjectBorrow(value)); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err) { - TRACING_JUMP_TO_LABEL(error); - } - } - val = value; - stack_pointer[-2] = val; - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(receiver); - stack_pointer = _PyFrame_GetStackPointer(frame); - TRACING_DISPATCH(); - } - - TRACING_TARGET(INSTRUMENTED_FOR_ITER) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = INSTRUMENTED_FOR_ITER; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(INSTRUMENTED_FOR_ITER); - opcode = INSTRUMENTED_FOR_ITER; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; - (void)(_old_stack_level); - _PyStackRef iter; - _PyStackRef null_or_index; - _PyStackRef next; - /* Skip 1 cache entry */ - null_or_index = stack_pointer[-1]; - iter = stack_pointer[-2]; - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyStackRef item = _PyForIter_VirtualIteratorNext(tstate, frame, iter, &null_or_index); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (!PyStackRef_IsValid(item)) { - if (PyStackRef_IsError(item)) { - TRACING_JUMP_TO_LABEL(error); - } - JUMPBY(oparg + 1); - stack_pointer[-1] = null_or_index; - TRACING_DISPATCH(); - } - next = item; - INSTRUMENTED_JUMP(this_instr, next_instr, PY_MONITORING_EVENT_BRANCH_LEFT); - stack_pointer[-1] = null_or_index; - stack_pointer[0] = next; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - TRACING_DISPATCH(); - } - - TRACING_TARGET(INSTRUMENTED_INSTRUCTION) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = INSTRUMENTED_INSTRUCTION; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(INSTRUMENTED_INSTRUCTION); - opcode = INSTRUMENTED_INSTRUCTION; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; - (void)(_old_stack_level); - _PyFrame_SetStackPointer(frame, stack_pointer); - int next_opcode = _Py_call_instrumentation_instruction( - tstate, frame, this_instr); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (next_opcode < 0) { - TRACING_JUMP_TO_LABEL(error); - } - next_instr = this_instr; - if (_PyOpcode_Caches[next_opcode]) { - PAUSE_ADAPTIVE_COUNTER(next_instr[1].counter); - } - assert(next_opcode > 0 && next_opcode < 256); - opcode = next_opcode; - DISPATCH_GOTO(); - } - - TRACING_TARGET(INSTRUMENTED_JUMP_BACKWARD) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = INSTRUMENTED_JUMP_BACKWARD; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(INSTRUMENTED_JUMP_BACKWARD); - opcode = INSTRUMENTED_JUMP_BACKWARD; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; - (void)(_old_stack_level); - /* Skip 1 cache entry */ - // _CHECK_PERIODIC - { - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = check_periodics(tstate); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err != 0) { - TRACING_JUMP_TO_LABEL(error); - } - } - // _MONITOR_JUMP_BACKWARD - { - INSTRUMENTED_JUMP(this_instr, next_instr - oparg, PY_MONITORING_EVENT_JUMP); - } - TRACING_DISPATCH(); - } - - TRACING_TARGET(INSTRUMENTED_JUMP_FORWARD) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = INSTRUMENTED_JUMP_FORWARD; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(INSTRUMENTED_JUMP_FORWARD); - opcode = INSTRUMENTED_JUMP_FORWARD; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; - (void)(_old_stack_level); - INSTRUMENTED_JUMP(this_instr, next_instr + oparg, PY_MONITORING_EVENT_JUMP); - TRACING_DISPATCH(); - } - - TRACING_TARGET(INSTRUMENTED_LINE) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = INSTRUMENTED_LINE; - (void)(opcode); - #endif - _Py_CODEUNIT* const prev_instr = frame->instr_ptr; - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(INSTRUMENTED_LINE); - opcode = INSTRUMENTED_LINE; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; - (void)(_old_stack_level); - int original_opcode = 0; - if (tstate->tracing) { - PyCodeObject *code = _PyFrame_GetCode(frame); - int index = (int)(this_instr - _PyFrame_GetBytecode(frame)); - original_opcode = code->_co_monitoring->lines->data[index*code->_co_monitoring->lines->bytes_per_entry]; - next_instr = this_instr; - } else { - _PyFrame_SetStackPointer(frame, stack_pointer); - original_opcode = _Py_call_instrumentation_line( - tstate, frame, this_instr, prev_instr); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (original_opcode < 0) { - next_instr = this_instr+1; - TRACING_JUMP_TO_LABEL(error); - } - next_instr = frame->instr_ptr; - if (next_instr != this_instr) { - TRACING_DISPATCH(); - } - } - if (_PyOpcode_Caches[original_opcode]) { - _PyBinaryOpCache *cache = (_PyBinaryOpCache *)(next_instr+1); - PAUSE_ADAPTIVE_COUNTER(cache->counter); - } - opcode = original_opcode; - DISPATCH_GOTO(); - } - - TRACING_TARGET(INSTRUMENTED_LOAD_SUPER_ATTR) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = INSTRUMENTED_LOAD_SUPER_ATTR; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(INSTRUMENTED_LOAD_SUPER_ATTR); - opcode = INSTRUMENTED_LOAD_SUPER_ATTR; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; - (void)(_old_stack_level); - _PyStackRef global_super_st; - _PyStackRef class_st; - _PyStackRef self_st; - _PyStackRef attr; - _PyStackRef *null; - /* Skip 1 cache entry */ - // _LOAD_SUPER_ATTR - { - self_st = stack_pointer[-1]; - class_st = stack_pointer[-2]; - global_super_st = stack_pointer[-3]; - PyObject *global_super = PyStackRef_AsPyObjectBorrow(global_super_st); - PyObject *class = PyStackRef_AsPyObjectBorrow(class_st); - PyObject *self = PyStackRef_AsPyObjectBorrow(self_st); - if (opcode == INSTRUMENTED_LOAD_SUPER_ATTR) { - PyObject *arg = oparg & 2 ? class : &_PyInstrumentation_MISSING; - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = _Py_call_instrumentation_2args( - tstate, PY_MONITORING_EVENT_CALL, - frame, this_instr, global_super, arg); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err) { - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyStackRef tmp = self_st; - self_st = PyStackRef_NULL; - stack_pointer[-1] = self_st; - PyStackRef_CLOSE(tmp); - tmp = class_st; - class_st = PyStackRef_NULL; - stack_pointer[-2] = class_st; - PyStackRef_CLOSE(tmp); - tmp = global_super_st; - global_super_st = PyStackRef_NULL; - stack_pointer[-3] = global_super_st; - PyStackRef_CLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -3; - assert(WITHIN_STACK_BOUNDS()); - TRACING_JUMP_TO_LABEL(error); - } - } - PyObject *stack[] = {class, self}; - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *super = PyObject_Vectorcall(global_super, stack, oparg & 2, NULL); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (opcode == INSTRUMENTED_LOAD_SUPER_ATTR) { - PyObject *arg = oparg & 2 ? class : &_PyInstrumentation_MISSING; - if (super == NULL) { - _PyFrame_SetStackPointer(frame, stack_pointer); - _Py_call_instrumentation_exc2( - tstate, PY_MONITORING_EVENT_C_RAISE, - frame, this_instr, global_super, arg); - stack_pointer = _PyFrame_GetStackPointer(frame); - } - else { - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = _Py_call_instrumentation_2args( - tstate, PY_MONITORING_EVENT_C_RETURN, - frame, this_instr, global_super, arg); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err < 0) { - _PyFrame_SetStackPointer(frame, stack_pointer); - Py_CLEAR(super); - stack_pointer = _PyFrame_GetStackPointer(frame); - } - } - } - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyStackRef tmp = self_st; - self_st = PyStackRef_NULL; - stack_pointer[-1] = self_st; - PyStackRef_CLOSE(tmp); - tmp = class_st; - class_st = PyStackRef_NULL; - stack_pointer[-2] = class_st; - PyStackRef_CLOSE(tmp); - tmp = global_super_st; - global_super_st = PyStackRef_NULL; - stack_pointer[-3] = global_super_st; - PyStackRef_CLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -3; - assert(WITHIN_STACK_BOUNDS()); - if (super == NULL) { - TRACING_JUMP_TO_LABEL(error); - } - PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 2); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *attr_o = PyObject_GetAttr(super, name); - Py_DECREF(super); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (attr_o == NULL) { - TRACING_JUMP_TO_LABEL(error); - } - attr = PyStackRef_FromPyObjectSteal(attr_o); - } - // _PUSH_NULL_CONDITIONAL - { - null = &stack_pointer[1]; - if (oparg & 1) { - null[0] = PyStackRef_NULL; - } - } - stack_pointer[0] = attr; - stack_pointer += 1 + (oparg & 1); - assert(WITHIN_STACK_BOUNDS()); - TRACING_DISPATCH(); - } - - TRACING_TARGET(INSTRUMENTED_NOT_TAKEN) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = INSTRUMENTED_NOT_TAKEN; - (void)(opcode); - #endif - _Py_CODEUNIT* const prev_instr = frame->instr_ptr; - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(INSTRUMENTED_NOT_TAKEN); - opcode = INSTRUMENTED_NOT_TAKEN; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; - (void)(_old_stack_level); - (void)this_instr; - INSTRUMENTED_JUMP(prev_instr, next_instr, PY_MONITORING_EVENT_BRANCH_LEFT); - TRACING_DISPATCH(); - } - - TRACING_TARGET(INSTRUMENTED_POP_ITER) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = INSTRUMENTED_POP_ITER; - (void)(opcode); - #endif - _Py_CODEUNIT* const prev_instr = frame->instr_ptr; - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(INSTRUMENTED_POP_ITER); - opcode = INSTRUMENTED_POP_ITER; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; - (void)(_old_stack_level); - _PyStackRef iter; - _PyStackRef index_or_null; - index_or_null = stack_pointer[-1]; - iter = stack_pointer[-2]; - (void)index_or_null; - INSTRUMENTED_JUMP(prev_instr, this_instr+1, PY_MONITORING_EVENT_BRANCH_RIGHT); - stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(iter); - stack_pointer = _PyFrame_GetStackPointer(frame); - TRACING_DISPATCH(); - } - - TRACING_TARGET(INSTRUMENTED_POP_JUMP_IF_FALSE) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = INSTRUMENTED_POP_JUMP_IF_FALSE; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(INSTRUMENTED_POP_JUMP_IF_FALSE); - opcode = INSTRUMENTED_POP_JUMP_IF_FALSE; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; - (void)(_old_stack_level); - _PyStackRef cond; - /* Skip 1 cache entry */ - cond = stack_pointer[-1]; - assert(PyStackRef_BoolCheck(cond)); - int jump = PyStackRef_IsFalse(cond); - RECORD_BRANCH_TAKEN(this_instr[1].cache, jump); - if (jump) { - INSTRUMENTED_JUMP(this_instr, next_instr + oparg, PY_MONITORING_EVENT_BRANCH_RIGHT); - } - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - TRACING_DISPATCH(); - } - - TRACING_TARGET(INSTRUMENTED_POP_JUMP_IF_NONE) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = INSTRUMENTED_POP_JUMP_IF_NONE; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(INSTRUMENTED_POP_JUMP_IF_NONE); - opcode = INSTRUMENTED_POP_JUMP_IF_NONE; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; - (void)(_old_stack_level); - _PyStackRef value; - /* Skip 1 cache entry */ - value = stack_pointer[-1]; - int jump = PyStackRef_IsNone(value); - RECORD_BRANCH_TAKEN(this_instr[1].cache, jump); - if (jump) { - INSTRUMENTED_JUMP(this_instr, next_instr + oparg, PY_MONITORING_EVENT_BRANCH_RIGHT); - } - else { - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(value); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += 1; - } - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - TRACING_DISPATCH(); - } - - TRACING_TARGET(INSTRUMENTED_POP_JUMP_IF_NOT_NONE) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = INSTRUMENTED_POP_JUMP_IF_NOT_NONE; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(INSTRUMENTED_POP_JUMP_IF_NOT_NONE); - opcode = INSTRUMENTED_POP_JUMP_IF_NOT_NONE; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; - (void)(_old_stack_level); - _PyStackRef value; - /* Skip 1 cache entry */ - value = stack_pointer[-1]; - int jump = !PyStackRef_IsNone(value); - RECORD_BRANCH_TAKEN(this_instr[1].cache, jump); - if (jump) { - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(value); - stack_pointer = _PyFrame_GetStackPointer(frame); - INSTRUMENTED_JUMP(this_instr, next_instr + oparg, PY_MONITORING_EVENT_BRANCH_RIGHT); - } - else { - stack_pointer += -1; - } - TRACING_DISPATCH(); - } - - TRACING_TARGET(INSTRUMENTED_POP_JUMP_IF_TRUE) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = INSTRUMENTED_POP_JUMP_IF_TRUE; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(INSTRUMENTED_POP_JUMP_IF_TRUE); - opcode = INSTRUMENTED_POP_JUMP_IF_TRUE; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; - (void)(_old_stack_level); - _PyStackRef cond; - /* Skip 1 cache entry */ - cond = stack_pointer[-1]; - assert(PyStackRef_BoolCheck(cond)); - int jump = PyStackRef_IsTrue(cond); - RECORD_BRANCH_TAKEN(this_instr[1].cache, jump); - if (jump) { - INSTRUMENTED_JUMP(this_instr, next_instr + oparg, PY_MONITORING_EVENT_BRANCH_RIGHT); - } - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - TRACING_DISPATCH(); - } - - TRACING_TARGET(INSTRUMENTED_RESUME) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = INSTRUMENTED_RESUME; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(INSTRUMENTED_RESUME); - opcode = INSTRUMENTED_RESUME; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; - (void)(_old_stack_level); - // _LOAD_BYTECODE - { - #ifdef Py_GIL_DISABLED - if (frame->tlbc_index != - ((_PyThreadStateImpl *)tstate)->tlbc_index) { - _PyFrame_SetStackPointer(frame, stack_pointer); - _Py_CODEUNIT *bytecode = - _PyEval_GetExecutableCode(tstate, _PyFrame_GetCode(frame)); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (bytecode == NULL) { - TRACING_JUMP_TO_LABEL(error); - } - ptrdiff_t off = this_instr - _PyFrame_GetBytecode(frame); - frame->tlbc_index = ((_PyThreadStateImpl *)tstate)->tlbc_index; - frame->instr_ptr = bytecode + off; - next_instr = frame->instr_ptr; - TRACING_DISPATCH(); - } - #endif - } - // _MAYBE_INSTRUMENT - { - #ifdef Py_GIL_DISABLED - - int check_instrumentation = 1; - #else - int check_instrumentation = (tstate->tracing == 0); - #endif - if (check_instrumentation) { - uintptr_t global_version = _Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & ~_PY_EVAL_EVENTS_MASK; - uintptr_t code_version = FT_ATOMIC_LOAD_UINTPTR_ACQUIRE(_PyFrame_GetCode(frame)->_co_instrumentation_version); - if (code_version != global_version) { - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = _Py_Instrument(_PyFrame_GetCode(frame), tstate->interp); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err) { - TRACING_JUMP_TO_LABEL(error); - } - next_instr = this_instr; - TRACING_DISPATCH(); - } - } - } - // _CHECK_PERIODIC_IF_NOT_YIELD_FROM - { - if ((oparg & RESUME_OPARG_LOCATION_MASK) < RESUME_AFTER_YIELD_FROM) { - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = check_periodics(tstate); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err != 0) { - TRACING_JUMP_TO_LABEL(error); - } - } - } - // _MONITOR_RESUME - { - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = _Py_call_instrumentation( - tstate, oparg > 0, frame, this_instr); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err) { - TRACING_JUMP_TO_LABEL(error); - } - if (frame->instr_ptr != this_instr) { - next_instr = frame->instr_ptr; - } - } - TRACING_DISPATCH(); - } - - TRACING_TARGET(INSTRUMENTED_RETURN_VALUE) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = INSTRUMENTED_RETURN_VALUE; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(INSTRUMENTED_RETURN_VALUE); - opcode = INSTRUMENTED_RETURN_VALUE; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; - (void)(_old_stack_level); - _PyStackRef val; - _PyStackRef retval; - _PyStackRef res; - // _RETURN_VALUE_EVENT - { - val = stack_pointer[-1]; - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = _Py_call_instrumentation_arg( - tstate, PY_MONITORING_EVENT_PY_RETURN, - frame, this_instr, PyStackRef_AsPyObjectBorrow(val)); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err) { - TRACING_JUMP_TO_LABEL(error); - } - } - // _RETURN_VALUE - { - retval = val; - assert(frame->owner != FRAME_OWNED_BY_INTERPRETER); - _PyStackRef temp = PyStackRef_MakeHeapSafe(retval); - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - assert(STACK_LEVEL() == 0); - _Py_LeaveRecursiveCallPy(tstate); - _PyInterpreterFrame *dying = frame; - frame = tstate->current_frame = dying->previous; - _PyEval_FrameClearAndPop(tstate, dying); - stack_pointer = _PyFrame_GetStackPointer(frame); - #if TIER_ONE - LOAD_IP(frame->return_offset); - #endif - #if TIER_TWO - TIER2_STORE_IP(frame->return_offset); - #endif - res = temp; - LLTRACE_RESUME_FRAME(); - } - stack_pointer[0] = res; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - TRACING_DISPATCH(); - } - - TRACING_TARGET(INSTRUMENTED_YIELD_VALUE) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = INSTRUMENTED_YIELD_VALUE; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(INSTRUMENTED_YIELD_VALUE); - opcode = INSTRUMENTED_YIELD_VALUE; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; - (void)(_old_stack_level); - _PyStackRef val; - _PyStackRef retval; - _PyStackRef value; - // _YIELD_VALUE_EVENT - { - val = stack_pointer[-1]; - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = _Py_call_instrumentation_arg( - tstate, PY_MONITORING_EVENT_PY_YIELD, - frame, this_instr, PyStackRef_AsPyObjectBorrow(val)); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err) { - TRACING_JUMP_TO_LABEL(error); - } - if (frame->instr_ptr != this_instr) { - next_instr = frame->instr_ptr; - TRACING_DISPATCH(); - } - } - // _YIELD_VALUE - { - retval = val; - assert(frame->owner != FRAME_OWNED_BY_INTERPRETER); - frame->instr_ptr++; - PyGenObject *gen = _PyGen_GetGeneratorFromFrame(frame); - assert(FRAME_SUSPENDED_YIELD_FROM == FRAME_SUSPENDED + 1); - assert(oparg == 0 || oparg == 1); - gen->gi_frame_state = FRAME_SUSPENDED + oparg; - _PyStackRef temp = retval; - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - tstate->exc_info = gen->gi_exc_state.previous_item; - gen->gi_exc_state.previous_item = NULL; - _Py_LeaveRecursiveCallPy(tstate); - _PyInterpreterFrame *gen_frame = frame; - frame = tstate->current_frame = frame->previous; - gen_frame->previous = NULL; - assert(INLINE_CACHE_ENTRIES_SEND == INLINE_CACHE_ENTRIES_FOR_ITER); - #if TIER_ONE - assert(frame->instr_ptr->op.code == INSTRUMENTED_LINE || - frame->instr_ptr->op.code == INSTRUMENTED_INSTRUCTION || - _PyOpcode_Deopt[frame->instr_ptr->op.code] == SEND || - _PyOpcode_Deopt[frame->instr_ptr->op.code] == FOR_ITER || - _PyOpcode_Deopt[frame->instr_ptr->op.code] == INTERPRETER_EXIT || - _PyOpcode_Deopt[frame->instr_ptr->op.code] == ENTER_EXECUTOR); - #endif - stack_pointer = _PyFrame_GetStackPointer(frame); - #if TIER_ONE - LOAD_IP(1 + INLINE_CACHE_ENTRIES_SEND); - #endif - #if TIER_TWO - TIER2_STORE_IP(1 + INLINE_CACHE_ENTRIES_SEND); - #endif - value = PyStackRef_MakeHeapSafe(temp); - LLTRACE_RESUME_FRAME(); - } - stack_pointer[0] = value; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - TRACING_DISPATCH(); - } - - TRACING_TARGET(INTERPRETER_EXIT) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = INTERPRETER_EXIT; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(INTERPRETER_EXIT); - opcode = INTERPRETER_EXIT; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; - (void)(_old_stack_level); - _PyStackRef retval; - retval = stack_pointer[-1]; - assert(frame->owner == FRAME_OWNED_BY_INTERPRETER); - assert(_PyFrame_IsIncomplete(frame)); - tstate->current_frame = frame->previous; - assert(!_PyErr_Occurred(tstate)); - PyObject *result = PyStackRef_AsPyObjectSteal(retval); - #if !_Py_TAIL_CALL_INTERP - assert(frame == &entry.frame); - #endif - #ifdef _Py_TIER2 - _PyStackRef executor = frame->localsplus[0]; - assert(tstate->current_executor == NULL); - if (!PyStackRef_IsNull(executor)) { - tstate->current_executor = PyStackRef_AsPyObjectBorrow(executor); - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(executor); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += 1; - } - #endif - LLTRACE_RESUME_FRAME(); - return result; - } - - TRACING_TARGET(IS_OP) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = IS_OP; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(IS_OP); - opcode = IS_OP; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; - (void)(_old_stack_level); - _PyStackRef left; - _PyStackRef right; - _PyStackRef b; - right = stack_pointer[-1]; - left = stack_pointer[-2]; - int res = Py_Is(PyStackRef_AsPyObjectBorrow(left), PyStackRef_AsPyObjectBorrow(right)) ^ oparg; - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyStackRef tmp = right; - right = PyStackRef_NULL; - stack_pointer[-1] = right; - PyStackRef_CLOSE(tmp); - tmp = left; - left = PyStackRef_NULL; - stack_pointer[-2] = left; - PyStackRef_CLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); - b = res ? PyStackRef_True : PyStackRef_False; - stack_pointer[0] = b; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - TRACING_DISPATCH(); - } - - TRACING_TARGET(JUMP_BACKWARD) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = JUMP_BACKWARD; - (void)(opcode); - #endif - frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(JUMP_BACKWARD); - PREDICTED_TRACING_JUMP_BACKWARD:; - _Py_CODEUNIT* const this_instr = next_instr - 2; - (void)this_instr; - opcode = JUMP_BACKWARD; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; - (void)(_old_stack_level); - /* Skip 1 cache entry */ - // _SPECIALIZE_JUMP_BACKWARD - { - #if ENABLE_SPECIALIZATION_FT - if (this_instr->op.code == JUMP_BACKWARD) { - uint8_t desired = tstate->interp->jit ? JUMP_BACKWARD_JIT : JUMP_BACKWARD_NO_JIT; - FT_ATOMIC_STORE_UINT8_RELAXED(this_instr->op.code, desired); - next_instr = this_instr; - DISPATCH_SAME_OPARG() - } - #endif - } - // _CHECK_PERIODIC - { - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = check_periodics(tstate); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err != 0) { - TRACING_JUMP_TO_LABEL(error); - } - } - // _JUMP_BACKWARD_NO_INTERRUPT - { - assert(oparg <= INSTR_OFFSET()); - JUMPBY(-oparg); - } - TRACING_DISPATCH(); - } - - TRACING_TARGET(JUMP_BACKWARD_JIT) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = JUMP_BACKWARD_JIT; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(JUMP_BACKWARD_JIT); - opcode = JUMP_BACKWARD_JIT; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; - (void)(_old_stack_level); - static_assert(1 == 1, "incorrect cache size"); - /* Skip 1 cache entry */ - // _CHECK_PERIODIC - { - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = check_periodics(tstate); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err != 0) { - TRACING_JUMP_TO_LABEL(error); - } - } - // _JUMP_BACKWARD_NO_INTERRUPT - { - assert(oparg <= INSTR_OFFSET()); - JUMPBY(-oparg); - } - // _JIT - { - #ifdef _Py_TIER2 - _Py_BackoffCounter counter = this_instr[1].counter; - if (!IS_JIT_TRACING() && backoff_counter_triggers(counter) && - this_instr->op.code == JUMP_BACKWARD_JIT && - next_instr->op.code != ENTER_EXECUTOR) { - if (tstate->interp->jit_state.code_buffer == NULL) { - _PyFrame_SetStackPointer(frame, stack_pointer); - tstate->interp->jit_state.code_buffer = (_PyUOpInstruction *)_PyObject_VirtualAlloc(UOP_BUFFER_SIZE); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (tstate->interp->jit_state.code_buffer == NULL) { - TRACING_DISPATCH(); - } - } - int _is_sys_tracing = (tstate->c_tracefunc != NULL) || (tstate->c_profilefunc != NULL); - if (!_is_sys_tracing) { - _Py_CODEUNIT *insert_exec_at = this_instr; - while (oparg > 255) { - oparg >>= 8; - insert_exec_at--; - } - _PyJit_InitializeTracing(tstate, frame, insert_exec_at, next_instr, STACK_LEVEL(), 0, NULL); - ENTER_TRACING(); - } - int _jump_taken = false; - PyCodeObject *old_code = _PyFrame_GetCode(frame); - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - int _old_stack_level = 0; - TRACING_DISPATCH(); - } - else { - ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); - } - #endif - } - TRACING_DISPATCH(); - } - - TRACING_TARGET(JUMP_BACKWARD_NO_INTERRUPT) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = JUMP_BACKWARD_NO_INTERRUPT; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(JUMP_BACKWARD_NO_INTERRUPT); - opcode = JUMP_BACKWARD_NO_INTERRUPT; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; - (void)(_old_stack_level); - assert(oparg <= INSTR_OFFSET()); - JUMPBY(-oparg); - TRACING_DISPATCH(); - } - - TRACING_TARGET(JUMP_BACKWARD_NO_JIT) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = JUMP_BACKWARD_NO_JIT; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(JUMP_BACKWARD_NO_JIT); - opcode = JUMP_BACKWARD_NO_JIT; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; - (void)(_old_stack_level); - static_assert(1 == 1, "incorrect cache size"); - /* Skip 1 cache entry */ - // _CHECK_PERIODIC - { - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = check_periodics(tstate); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err != 0) { - TRACING_JUMP_TO_LABEL(error); - } - } - // _JUMP_BACKWARD_NO_INTERRUPT - { - assert(oparg <= INSTR_OFFSET()); - JUMPBY(-oparg); - } - TRACING_DISPATCH(); - } - - TRACING_TARGET(JUMP_FORWARD) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = JUMP_FORWARD; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(JUMP_FORWARD); - opcode = JUMP_FORWARD; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; - (void)(_old_stack_level); - JUMPBY(oparg); - TRACING_DISPATCH(); - } - - TRACING_TARGET(LIST_APPEND) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = LIST_APPEND; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(LIST_APPEND); - opcode = LIST_APPEND; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; - (void)(_old_stack_level); - _PyStackRef list; - _PyStackRef v; - v = stack_pointer[-1]; - list = stack_pointer[-2 - (oparg-1)]; - int err = _PyList_AppendTakeRef((PyListObject *)PyStackRef_AsPyObjectBorrow(list), - PyStackRef_AsPyObjectSteal(v)); - if (err < 0) { - TRACING_JUMP_TO_LABEL(pop_1_error); - } - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - TRACING_DISPATCH(); - } - - TRACING_TARGET(LIST_EXTEND) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = LIST_EXTEND; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(LIST_EXTEND); - opcode = LIST_EXTEND; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; - (void)(_old_stack_level); - _PyStackRef list_st; - _PyStackRef iterable_st; - iterable_st = stack_pointer[-1]; - list_st = stack_pointer[-2 - (oparg-1)]; - PyObject *list = PyStackRef_AsPyObjectBorrow(list_st); - PyObject *iterable = PyStackRef_AsPyObjectBorrow(iterable_st); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *none_val = _PyList_Extend((PyListObject *)list, iterable); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (none_val == NULL) { - _PyFrame_SetStackPointer(frame, stack_pointer); - int matches = _PyErr_ExceptionMatches(tstate, PyExc_TypeError); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (matches && - (Py_TYPE(iterable)->tp_iter == NULL && !PySequence_Check(iterable))) - { - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyErr_Clear(tstate); - _PyErr_Format(tstate, PyExc_TypeError, - "Value after * must be an iterable, not %.200s", - Py_TYPE(iterable)->tp_name); - stack_pointer = _PyFrame_GetStackPointer(frame); - } - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(iterable_st); - stack_pointer = _PyFrame_GetStackPointer(frame); - TRACING_JUMP_TO_LABEL(error); - } - assert(Py_IsNone(none_val)); - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(iterable_st); - stack_pointer = _PyFrame_GetStackPointer(frame); - TRACING_DISPATCH(); - } - - TRACING_TARGET(LOAD_ATTR) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = LOAD_ATTR; - (void)(opcode); - #endif - frame->instr_ptr = next_instr; - next_instr += 10; - INSTRUCTION_STATS(LOAD_ATTR); - PREDICTED_TRACING_LOAD_ATTR:; - _Py_CODEUNIT* const this_instr = next_instr - 10; - (void)this_instr; - opcode = LOAD_ATTR; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; - (void)(_old_stack_level); - _PyStackRef owner; - _PyStackRef *attr; - _PyStackRef *self_or_null; - // _SPECIALIZE_LOAD_ATTR - { - owner = stack_pointer[-1]; - uint16_t counter = read_u16(&this_instr[1].cache); - (void)counter; - #if ENABLE_SPECIALIZATION_FT - if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { - PyObject *name = GETITEM(FRAME_CO_NAMES, oparg>>1); - next_instr = this_instr; - _PyFrame_SetStackPointer(frame, stack_pointer); - _Py_Specialize_LoadAttr(owner, next_instr, name); - stack_pointer = _PyFrame_GetStackPointer(frame); - TRACING_SPECIALIZE_DISPATCH_SAME_OPARG() - } - OPCODE_DEFERRED_INC(LOAD_ATTR); - ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); - #endif /* ENABLE_SPECIALIZATION_FT */ - } - /* Skip 8 cache entries */ - // _LOAD_ATTR - { - attr = &stack_pointer[-1]; - self_or_null = &stack_pointer[0]; - PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 1); - if (oparg & 1) { - *attr = PyStackRef_NULL; - _PyFrame_SetStackPointer(frame, stack_pointer); - int is_meth = _PyObject_GetMethodStackRef(tstate, PyStackRef_AsPyObjectBorrow(owner), name, attr); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (is_meth) { - assert(!PyStackRef_IsNull(*attr)); - self_or_null[0] = owner; - } - else { - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(owner); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (PyStackRef_IsNull(*attr)) { - TRACING_JUMP_TO_LABEL(error); - } - self_or_null[0] = PyStackRef_NULL; - stack_pointer += 1; - } - } - else { - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *attr_o = PyObject_GetAttr(PyStackRef_AsPyObjectBorrow(owner), name); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(owner); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (attr_o == NULL) { - TRACING_JUMP_TO_LABEL(error); - } - *attr = PyStackRef_FromPyObjectSteal(attr_o); - stack_pointer += 1; - } - } - stack_pointer += (oparg&1); - assert(WITHIN_STACK_BOUNDS()); - TRACING_DISPATCH(); - } - - TRACING_TARGET(LOAD_ATTR_CLASS) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = LOAD_ATTR_CLASS; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 10; - INSTRUCTION_STATS(LOAD_ATTR_CLASS); - opcode = LOAD_ATTR_CLASS; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; - (void)(_old_stack_level); - static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); - _PyStackRef owner; - _PyStackRef attr; - _PyStackRef *null; - /* Skip 1 cache entry */ - // _CHECK_ATTR_CLASS - { - owner = stack_pointer[-1]; - uint32_t type_version = read_u32(&this_instr[2].cache); - PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); - if (!PyType_Check(owner_o)) { - UPDATE_MISS_STATS(LOAD_ATTR); - assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); - JUMP_TO_PREDICTED(TRACING_LOAD_ATTR); - } - assert(type_version != 0); - if (FT_ATOMIC_LOAD_UINT_RELAXED(((PyTypeObject *)owner_o)->tp_version_tag) != type_version) { - UPDATE_MISS_STATS(LOAD_ATTR); - assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); - JUMP_TO_PREDICTED(TRACING_LOAD_ATTR); - } - } - /* Skip 2 cache entries */ - // _LOAD_ATTR_CLASS - { - PyObject *descr = read_obj(&this_instr[6].cache); - STAT_INC(LOAD_ATTR, hit); - assert(descr != NULL); - attr = PyStackRef_FromPyObjectNew(descr); - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyStackRef tmp = owner; - owner = attr; - stack_pointer[-1] = owner; - PyStackRef_CLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); - } - // _PUSH_NULL_CONDITIONAL - { - null = &stack_pointer[0]; - if (oparg & 1) { - null[0] = PyStackRef_NULL; - } - } - stack_pointer += (oparg & 1); - assert(WITHIN_STACK_BOUNDS()); - TRACING_DISPATCH(); - } - - TRACING_TARGET(LOAD_ATTR_CLASS_WITH_METACLASS_CHECK) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = LOAD_ATTR_CLASS_WITH_METACLASS_CHECK; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 10; - INSTRUCTION_STATS(LOAD_ATTR_CLASS_WITH_METACLASS_CHECK); - opcode = LOAD_ATTR_CLASS_WITH_METACLASS_CHECK; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; - (void)(_old_stack_level); - static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); - _PyStackRef owner; - _PyStackRef attr; - _PyStackRef *null; - /* Skip 1 cache entry */ - // _CHECK_ATTR_CLASS - { - owner = stack_pointer[-1]; - uint32_t type_version = read_u32(&this_instr[2].cache); - PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); - if (!PyType_Check(owner_o)) { - UPDATE_MISS_STATS(LOAD_ATTR); - assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); - JUMP_TO_PREDICTED(TRACING_LOAD_ATTR); - } - assert(type_version != 0); - if (FT_ATOMIC_LOAD_UINT_RELAXED(((PyTypeObject *)owner_o)->tp_version_tag) != type_version) { - UPDATE_MISS_STATS(LOAD_ATTR); - assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); - JUMP_TO_PREDICTED(TRACING_LOAD_ATTR); - } - } - // _GUARD_TYPE_VERSION - { - uint32_t type_version = read_u32(&this_instr[4].cache); - PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); - assert(type_version != 0); - if (FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version) { - UPDATE_MISS_STATS(LOAD_ATTR); - assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); - JUMP_TO_PREDICTED(TRACING_LOAD_ATTR); - } - } - // _LOAD_ATTR_CLASS - { - PyObject *descr = read_obj(&this_instr[6].cache); - STAT_INC(LOAD_ATTR, hit); - assert(descr != NULL); - attr = PyStackRef_FromPyObjectNew(descr); - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyStackRef tmp = owner; - owner = attr; - stack_pointer[-1] = owner; - PyStackRef_CLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); - } - // _PUSH_NULL_CONDITIONAL - { - null = &stack_pointer[0]; - if (oparg & 1) { - null[0] = PyStackRef_NULL; - } - } - stack_pointer += (oparg & 1); - assert(WITHIN_STACK_BOUNDS()); - TRACING_DISPATCH(); - } - - TRACING_TARGET(LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 10; - INSTRUCTION_STATS(LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN); - opcode = LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; - (void)(_old_stack_level); - static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); - _PyStackRef owner; - /* Skip 1 cache entry */ - owner = stack_pointer[-1]; - uint32_t type_version = read_u32(&this_instr[2].cache); - uint32_t func_version = read_u32(&this_instr[4].cache); - PyObject *getattribute = read_obj(&this_instr[6].cache); - PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); - assert((oparg & 1) == 0); - if (tstate->interp->eval_frame) { - UPDATE_MISS_STATS(LOAD_ATTR); - assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); - JUMP_TO_PREDICTED(TRACING_LOAD_ATTR); - } - PyTypeObject *cls = Py_TYPE(owner_o); - assert(type_version != 0); - if (FT_ATOMIC_LOAD_UINT_RELAXED(cls->tp_version_tag) != type_version) { - UPDATE_MISS_STATS(LOAD_ATTR); - assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); - JUMP_TO_PREDICTED(TRACING_LOAD_ATTR); - } - assert(Py_IS_TYPE(getattribute, &PyFunction_Type)); - PyFunctionObject *f = (PyFunctionObject *)getattribute; - assert(func_version != 0); - if (f->func_version != func_version) { - UPDATE_MISS_STATS(LOAD_ATTR); - assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); - JUMP_TO_PREDICTED(TRACING_LOAD_ATTR); - } - PyCodeObject *code = (PyCodeObject *)f->func_code; - assert(code->co_argcount == 2); - if (!_PyThreadState_HasStackSpace(tstate, code->co_framesize)) { - UPDATE_MISS_STATS(LOAD_ATTR); - assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); - JUMP_TO_PREDICTED(TRACING_LOAD_ATTR); - } - STAT_INC(LOAD_ATTR, hit); - PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 1); - _PyInterpreterFrame *new_frame = _PyFrame_PushUnchecked( - tstate, PyStackRef_FromPyObjectNew(f), 2, frame); - new_frame->localsplus[0] = owner; - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - new_frame->localsplus[1] = PyStackRef_FromPyObjectNew(name); - frame->return_offset = 10u ; - TRACING_DISPATCH_INLINED(new_frame); - } - - TRACING_TARGET(LOAD_ATTR_INSTANCE_VALUE) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = LOAD_ATTR_INSTANCE_VALUE; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 10; - INSTRUCTION_STATS(LOAD_ATTR_INSTANCE_VALUE); - opcode = LOAD_ATTR_INSTANCE_VALUE; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; - (void)(_old_stack_level); - static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); - _PyStackRef owner; - _PyStackRef attr; - _PyStackRef *null; - /* Skip 1 cache entry */ - // _GUARD_TYPE_VERSION - { - owner = stack_pointer[-1]; - uint32_t type_version = read_u32(&this_instr[2].cache); - PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); - assert(type_version != 0); - if (FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version) { - UPDATE_MISS_STATS(LOAD_ATTR); - assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); - JUMP_TO_PREDICTED(TRACING_LOAD_ATTR); - } - } - // _CHECK_MANAGED_OBJECT_HAS_VALUES - { - PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); - assert(Py_TYPE(owner_o)->tp_dictoffset < 0); - assert(Py_TYPE(owner_o)->tp_flags & Py_TPFLAGS_INLINE_VALUES); - if (!FT_ATOMIC_LOAD_UINT8(_PyObject_InlineValues(owner_o)->valid)) { - UPDATE_MISS_STATS(LOAD_ATTR); - assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); - JUMP_TO_PREDICTED(TRACING_LOAD_ATTR); - } - } - // _LOAD_ATTR_INSTANCE_VALUE - { - uint16_t offset = read_u16(&this_instr[4].cache); - PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); - PyObject **value_ptr = (PyObject**)(((char *)owner_o) + offset); - PyObject *attr_o = FT_ATOMIC_LOAD_PTR_ACQUIRE(*value_ptr); - if (attr_o == NULL) { - UPDATE_MISS_STATS(LOAD_ATTR); - assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); - JUMP_TO_PREDICTED(TRACING_LOAD_ATTR); - } - #ifdef Py_GIL_DISABLED - int increfed = _Py_TryIncrefCompareStackRef(value_ptr, attr_o, &attr); - if (!increfed) { - if (true) { - UPDATE_MISS_STATS(LOAD_ATTR); - assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); - JUMP_TO_PREDICTED(TRACING_LOAD_ATTR); - } - } - #else - attr = PyStackRef_FromPyObjectNew(attr_o); - #endif - STAT_INC(LOAD_ATTR, hit); - stack_pointer[-1] = attr; - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(owner); - stack_pointer = _PyFrame_GetStackPointer(frame); - } - /* Skip 5 cache entries */ - // _PUSH_NULL_CONDITIONAL - { - null = &stack_pointer[0]; - if (oparg & 1) { - null[0] = PyStackRef_NULL; - } - } - stack_pointer += (oparg & 1); - assert(WITHIN_STACK_BOUNDS()); - TRACING_DISPATCH(); - } - - TRACING_TARGET(LOAD_ATTR_METHOD_LAZY_DICT) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = LOAD_ATTR_METHOD_LAZY_DICT; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 10; - INSTRUCTION_STATS(LOAD_ATTR_METHOD_LAZY_DICT); - opcode = LOAD_ATTR_METHOD_LAZY_DICT; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; - (void)(_old_stack_level); - static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); - _PyStackRef owner; - _PyStackRef attr; - _PyStackRef self; - /* Skip 1 cache entry */ - // _GUARD_TYPE_VERSION - { - owner = stack_pointer[-1]; - uint32_t type_version = read_u32(&this_instr[2].cache); - PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); - assert(type_version != 0); - if (FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version) { - UPDATE_MISS_STATS(LOAD_ATTR); - assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); - JUMP_TO_PREDICTED(TRACING_LOAD_ATTR); - } - } - // _CHECK_ATTR_METHOD_LAZY_DICT - { - uint16_t dictoffset = read_u16(&this_instr[4].cache); - char *ptr = ((char *)PyStackRef_AsPyObjectBorrow(owner)) + MANAGED_DICT_OFFSET + dictoffset; - PyObject *dict = FT_ATOMIC_LOAD_PTR_ACQUIRE(*(PyObject **)ptr); - if (dict != NULL) { - UPDATE_MISS_STATS(LOAD_ATTR); - assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); - JUMP_TO_PREDICTED(TRACING_LOAD_ATTR); - } - } - /* Skip 1 cache entry */ - // _LOAD_ATTR_METHOD_LAZY_DICT - { - PyObject *descr = read_obj(&this_instr[6].cache); - assert(oparg & 1); - STAT_INC(LOAD_ATTR, hit); - assert(descr != NULL); - assert(_PyType_HasFeature(Py_TYPE(descr), Py_TPFLAGS_METHOD_DESCRIPTOR)); - attr = PyStackRef_FromPyObjectNew(descr); - self = owner; - } - stack_pointer[-1] = attr; - stack_pointer[0] = self; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - TRACING_DISPATCH(); - } - - TRACING_TARGET(LOAD_ATTR_METHOD_NO_DICT) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = LOAD_ATTR_METHOD_NO_DICT; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 10; - INSTRUCTION_STATS(LOAD_ATTR_METHOD_NO_DICT); - opcode = LOAD_ATTR_METHOD_NO_DICT; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; - (void)(_old_stack_level); - static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); - _PyStackRef owner; - _PyStackRef attr; - _PyStackRef self; - /* Skip 1 cache entry */ - // _GUARD_TYPE_VERSION - { - owner = stack_pointer[-1]; - uint32_t type_version = read_u32(&this_instr[2].cache); - PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); - assert(type_version != 0); - if (FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version) { - UPDATE_MISS_STATS(LOAD_ATTR); - assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); - JUMP_TO_PREDICTED(TRACING_LOAD_ATTR); - } - } - /* Skip 2 cache entries */ - // _LOAD_ATTR_METHOD_NO_DICT - { - PyObject *descr = read_obj(&this_instr[6].cache); - assert(oparg & 1); - assert(Py_TYPE(PyStackRef_AsPyObjectBorrow(owner))->tp_dictoffset == 0); - STAT_INC(LOAD_ATTR, hit); - assert(descr != NULL); - assert(_PyType_HasFeature(Py_TYPE(descr), Py_TPFLAGS_METHOD_DESCRIPTOR)); - attr = PyStackRef_FromPyObjectNew(descr); - self = owner; - } - stack_pointer[-1] = attr; - stack_pointer[0] = self; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - TRACING_DISPATCH(); - } - - TRACING_TARGET(LOAD_ATTR_METHOD_WITH_VALUES) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = LOAD_ATTR_METHOD_WITH_VALUES; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 10; - INSTRUCTION_STATS(LOAD_ATTR_METHOD_WITH_VALUES); - opcode = LOAD_ATTR_METHOD_WITH_VALUES; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; - (void)(_old_stack_level); - static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); - _PyStackRef owner; - _PyStackRef attr; - _PyStackRef self; - /* Skip 1 cache entry */ - // _GUARD_TYPE_VERSION - { - owner = stack_pointer[-1]; - uint32_t type_version = read_u32(&this_instr[2].cache); - PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); - assert(type_version != 0); - if (FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version) { - UPDATE_MISS_STATS(LOAD_ATTR); - assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); - JUMP_TO_PREDICTED(TRACING_LOAD_ATTR); - } - } - // _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT - { - PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); - assert(Py_TYPE(owner_o)->tp_flags & Py_TPFLAGS_INLINE_VALUES); - PyDictValues *ivs = _PyObject_InlineValues(owner_o); - if (!FT_ATOMIC_LOAD_UINT8(ivs->valid)) { - UPDATE_MISS_STATS(LOAD_ATTR); - assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); - JUMP_TO_PREDICTED(TRACING_LOAD_ATTR); - } - } - // _GUARD_KEYS_VERSION - { - uint32_t keys_version = read_u32(&this_instr[4].cache); - PyTypeObject *owner_cls = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); - PyHeapTypeObject *owner_heap_type = (PyHeapTypeObject *)owner_cls; - PyDictKeysObject *keys = owner_heap_type->ht_cached_keys; - if (FT_ATOMIC_LOAD_UINT32_RELAXED(keys->dk_version) != keys_version) { - UPDATE_MISS_STATS(LOAD_ATTR); - assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); - JUMP_TO_PREDICTED(TRACING_LOAD_ATTR); - } - } - // _LOAD_ATTR_METHOD_WITH_VALUES - { - PyObject *descr = read_obj(&this_instr[6].cache); - assert(oparg & 1); - STAT_INC(LOAD_ATTR, hit); - assert(descr != NULL); - assert(_PyType_HasFeature(Py_TYPE(descr), Py_TPFLAGS_METHOD_DESCRIPTOR)); - attr = PyStackRef_FromPyObjectNew(descr); - self = owner; - } - stack_pointer[-1] = attr; - stack_pointer[0] = self; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - TRACING_DISPATCH(); - } - - TRACING_TARGET(LOAD_ATTR_MODULE) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = LOAD_ATTR_MODULE; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 10; - INSTRUCTION_STATS(LOAD_ATTR_MODULE); - opcode = LOAD_ATTR_MODULE; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; - (void)(_old_stack_level); - static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); - _PyStackRef owner; - _PyStackRef attr; - _PyStackRef *null; - /* Skip 1 cache entry */ - // _LOAD_ATTR_MODULE - { - owner = stack_pointer[-1]; - uint32_t dict_version = read_u32(&this_instr[2].cache); - uint16_t index = read_u16(&this_instr[4].cache); - PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); - if (Py_TYPE(owner_o)->tp_getattro != PyModule_Type.tp_getattro) { - UPDATE_MISS_STATS(LOAD_ATTR); - assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); - JUMP_TO_PREDICTED(TRACING_LOAD_ATTR); - } - PyDictObject *dict = (PyDictObject *)((PyModuleObject *)owner_o)->md_dict; - assert(dict != NULL); - PyDictKeysObject *keys = FT_ATOMIC_LOAD_PTR_ACQUIRE(dict->ma_keys); - if (FT_ATOMIC_LOAD_UINT32_RELAXED(keys->dk_version) != dict_version) { - UPDATE_MISS_STATS(LOAD_ATTR); - assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); - JUMP_TO_PREDICTED(TRACING_LOAD_ATTR); - } - assert(keys->dk_kind == DICT_KEYS_UNICODE); - assert(index < FT_ATOMIC_LOAD_SSIZE_RELAXED(keys->dk_nentries)); - PyDictUnicodeEntry *ep = DK_UNICODE_ENTRIES(keys) + index; - PyObject *attr_o = FT_ATOMIC_LOAD_PTR_RELAXED(ep->me_value); - if (attr_o == NULL) { - UPDATE_MISS_STATS(LOAD_ATTR); - assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); - JUMP_TO_PREDICTED(TRACING_LOAD_ATTR); - } - #ifdef Py_GIL_DISABLED - int increfed = _Py_TryIncrefCompareStackRef(&ep->me_value, attr_o, &attr); - if (!increfed) { - if (true) { - UPDATE_MISS_STATS(LOAD_ATTR); - assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); - JUMP_TO_PREDICTED(TRACING_LOAD_ATTR); - } - } - #else - attr = PyStackRef_FromPyObjectNew(attr_o); - #endif - STAT_INC(LOAD_ATTR, hit); - stack_pointer[-1] = attr; - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(owner); - stack_pointer = _PyFrame_GetStackPointer(frame); - } - /* Skip 5 cache entries */ - // _PUSH_NULL_CONDITIONAL - { - null = &stack_pointer[0]; - if (oparg & 1) { - null[0] = PyStackRef_NULL; - } - } - stack_pointer += (oparg & 1); - assert(WITHIN_STACK_BOUNDS()); - TRACING_DISPATCH(); - } - - TRACING_TARGET(LOAD_ATTR_NONDESCRIPTOR_NO_DICT) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = LOAD_ATTR_NONDESCRIPTOR_NO_DICT; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 10; - INSTRUCTION_STATS(LOAD_ATTR_NONDESCRIPTOR_NO_DICT); - opcode = LOAD_ATTR_NONDESCRIPTOR_NO_DICT; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; - (void)(_old_stack_level); - static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); - _PyStackRef owner; - _PyStackRef attr; - /* Skip 1 cache entry */ - // _GUARD_TYPE_VERSION - { - owner = stack_pointer[-1]; - uint32_t type_version = read_u32(&this_instr[2].cache); - PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); - assert(type_version != 0); - if (FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version) { - UPDATE_MISS_STATS(LOAD_ATTR); - assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); - JUMP_TO_PREDICTED(TRACING_LOAD_ATTR); - } - } - /* Skip 2 cache entries */ - // _LOAD_ATTR_NONDESCRIPTOR_NO_DICT - { - PyObject *descr = read_obj(&this_instr[6].cache); - assert((oparg & 1) == 0); - assert(Py_TYPE(PyStackRef_AsPyObjectBorrow(owner))->tp_dictoffset == 0); - STAT_INC(LOAD_ATTR, hit); - assert(descr != NULL); - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(owner); - stack_pointer = _PyFrame_GetStackPointer(frame); - attr = PyStackRef_FromPyObjectNew(descr); - } - stack_pointer[0] = attr; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - TRACING_DISPATCH(); - } - - TRACING_TARGET(LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 10; - INSTRUCTION_STATS(LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES); - opcode = LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; - (void)(_old_stack_level); - static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); - _PyStackRef owner; - _PyStackRef attr; - /* Skip 1 cache entry */ - // _GUARD_TYPE_VERSION - { - owner = stack_pointer[-1]; - uint32_t type_version = read_u32(&this_instr[2].cache); - PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); - assert(type_version != 0); - if (FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version) { - UPDATE_MISS_STATS(LOAD_ATTR); - assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); - JUMP_TO_PREDICTED(TRACING_LOAD_ATTR); - } - } - // _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT - { - PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); - assert(Py_TYPE(owner_o)->tp_flags & Py_TPFLAGS_INLINE_VALUES); - PyDictValues *ivs = _PyObject_InlineValues(owner_o); - if (!FT_ATOMIC_LOAD_UINT8(ivs->valid)) { - UPDATE_MISS_STATS(LOAD_ATTR); - assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); - JUMP_TO_PREDICTED(TRACING_LOAD_ATTR); - } - } - // _GUARD_KEYS_VERSION - { - uint32_t keys_version = read_u32(&this_instr[4].cache); - PyTypeObject *owner_cls = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); - PyHeapTypeObject *owner_heap_type = (PyHeapTypeObject *)owner_cls; - PyDictKeysObject *keys = owner_heap_type->ht_cached_keys; - if (FT_ATOMIC_LOAD_UINT32_RELAXED(keys->dk_version) != keys_version) { - UPDATE_MISS_STATS(LOAD_ATTR); - assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); - JUMP_TO_PREDICTED(TRACING_LOAD_ATTR); - } - } - // _LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES - { - PyObject *descr = read_obj(&this_instr[6].cache); - assert((oparg & 1) == 0); - STAT_INC(LOAD_ATTR, hit); - assert(descr != NULL); - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(owner); - stack_pointer = _PyFrame_GetStackPointer(frame); - attr = PyStackRef_FromPyObjectNew(descr); - } - stack_pointer[0] = attr; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - TRACING_DISPATCH(); - } - - TRACING_TARGET(LOAD_ATTR_PROPERTY) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = LOAD_ATTR_PROPERTY; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 10; - INSTRUCTION_STATS(LOAD_ATTR_PROPERTY); - opcode = LOAD_ATTR_PROPERTY; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; - (void)(_old_stack_level); - static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); - _PyStackRef owner; - _PyStackRef new_frame; - /* Skip 1 cache entry */ - // _CHECK_PEP_523 - { - if (tstate->interp->eval_frame) { - UPDATE_MISS_STATS(LOAD_ATTR); - assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); - JUMP_TO_PREDICTED(TRACING_LOAD_ATTR); - } - } - // _GUARD_TYPE_VERSION - { - owner = stack_pointer[-1]; - uint32_t type_version = read_u32(&this_instr[2].cache); - PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); - assert(type_version != 0); - if (FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version) { - UPDATE_MISS_STATS(LOAD_ATTR); - assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); - JUMP_TO_PREDICTED(TRACING_LOAD_ATTR); - } - } - /* Skip 2 cache entries */ - // _LOAD_ATTR_PROPERTY_FRAME - { - PyObject *fget = read_obj(&this_instr[6].cache); - assert((oparg & 1) == 0); - assert(Py_IS_TYPE(fget, &PyFunction_Type)); - PyFunctionObject *f = (PyFunctionObject *)fget; - PyCodeObject *code = (PyCodeObject *)f->func_code; - if ((code->co_flags & (CO_VARKEYWORDS | CO_VARARGS | CO_OPTIMIZED)) != CO_OPTIMIZED) { - UPDATE_MISS_STATS(LOAD_ATTR); - assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); - JUMP_TO_PREDICTED(TRACING_LOAD_ATTR); - } - if (code->co_kwonlyargcount) { - UPDATE_MISS_STATS(LOAD_ATTR); - assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); - JUMP_TO_PREDICTED(TRACING_LOAD_ATTR); - } - if (code->co_argcount != 1) { - UPDATE_MISS_STATS(LOAD_ATTR); - assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); - JUMP_TO_PREDICTED(TRACING_LOAD_ATTR); - } - if (!_PyThreadState_HasStackSpace(tstate, code->co_framesize)) { - UPDATE_MISS_STATS(LOAD_ATTR); - assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); - JUMP_TO_PREDICTED(TRACING_LOAD_ATTR); - } - STAT_INC(LOAD_ATTR, hit); - _PyInterpreterFrame *pushed_frame = _PyFrame_PushUnchecked(tstate, PyStackRef_FromPyObjectNew(fget), 1, frame); - pushed_frame->localsplus[0] = owner; - new_frame = PyStackRef_Wrap(pushed_frame); - } - // _SAVE_RETURN_OFFSET - { - #if TIER_ONE - frame->return_offset = (uint16_t)(next_instr - this_instr); - #endif - #if TIER_TWO - frame->return_offset = oparg; - #endif - } - // _PUSH_FRAME - { - assert(tstate->interp->eval_frame == NULL); - _PyInterpreterFrame *temp = PyStackRef_Unwrap(new_frame); - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - assert(temp->previous == frame || temp->previous->previous == frame); - CALL_STAT_INC(inlined_py_calls); - frame = tstate->current_frame = temp; - tstate->py_recursion_remaining--; - LOAD_SP(); - LOAD_IP(0); - LLTRACE_RESUME_FRAME(); - } - TRACING_DISPATCH(); - } - - TRACING_TARGET(LOAD_ATTR_SLOT) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = LOAD_ATTR_SLOT; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 10; - INSTRUCTION_STATS(LOAD_ATTR_SLOT); - opcode = LOAD_ATTR_SLOT; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; - (void)(_old_stack_level); - static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); - _PyStackRef owner; - _PyStackRef attr; - _PyStackRef *null; - /* Skip 1 cache entry */ - // _GUARD_TYPE_VERSION - { - owner = stack_pointer[-1]; - uint32_t type_version = read_u32(&this_instr[2].cache); - PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); - assert(type_version != 0); - if (FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version) { - UPDATE_MISS_STATS(LOAD_ATTR); - assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); - JUMP_TO_PREDICTED(TRACING_LOAD_ATTR); - } - } - // _LOAD_ATTR_SLOT - { - uint16_t index = read_u16(&this_instr[4].cache); - PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); - PyObject **addr = (PyObject **)((char *)owner_o + index); - PyObject *attr_o = FT_ATOMIC_LOAD_PTR(*addr); - if (attr_o == NULL) { - UPDATE_MISS_STATS(LOAD_ATTR); - assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); - JUMP_TO_PREDICTED(TRACING_LOAD_ATTR); - } - #ifdef Py_GIL_DISABLED - int increfed = _Py_TryIncrefCompareStackRef(addr, attr_o, &attr); - if (!increfed) { - UPDATE_MISS_STATS(LOAD_ATTR); - assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); - JUMP_TO_PREDICTED(TRACING_LOAD_ATTR); - } - #else - attr = PyStackRef_FromPyObjectNew(attr_o); - #endif - STAT_INC(LOAD_ATTR, hit); - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyStackRef tmp = owner; - owner = attr; - stack_pointer[-1] = owner; - PyStackRef_CLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); - } - /* Skip 5 cache entries */ - // _PUSH_NULL_CONDITIONAL - { - null = &stack_pointer[0]; - if (oparg & 1) { - null[0] = PyStackRef_NULL; - } - } - stack_pointer += (oparg & 1); - assert(WITHIN_STACK_BOUNDS()); - TRACING_DISPATCH(); - } - - TRACING_TARGET(LOAD_ATTR_WITH_HINT) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = LOAD_ATTR_WITH_HINT; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 10; - INSTRUCTION_STATS(LOAD_ATTR_WITH_HINT); - opcode = LOAD_ATTR_WITH_HINT; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; - (void)(_old_stack_level); - static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); - _PyStackRef owner; - _PyStackRef attr; - _PyStackRef *null; - /* Skip 1 cache entry */ - // _GUARD_TYPE_VERSION - { - owner = stack_pointer[-1]; - uint32_t type_version = read_u32(&this_instr[2].cache); - PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); - assert(type_version != 0); - if (FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version) { - UPDATE_MISS_STATS(LOAD_ATTR); - assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); - JUMP_TO_PREDICTED(TRACING_LOAD_ATTR); - } - } - // _LOAD_ATTR_WITH_HINT - { - uint16_t hint = read_u16(&this_instr[4].cache); - PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); - assert(Py_TYPE(owner_o)->tp_flags & Py_TPFLAGS_MANAGED_DICT); - PyDictObject *dict = _PyObject_GetManagedDict(owner_o); - if (dict == NULL) { - UPDATE_MISS_STATS(LOAD_ATTR); - assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); - JUMP_TO_PREDICTED(TRACING_LOAD_ATTR); - } - PyDictKeysObject *dk = FT_ATOMIC_LOAD_PTR(dict->ma_keys); - assert(PyDict_CheckExact((PyObject *)dict)); - #ifdef Py_GIL_DISABLED - if (!_Py_IsOwnedByCurrentThread((PyObject *)dict) && !_PyObject_GC_IS_SHARED(dict)) { - UPDATE_MISS_STATS(LOAD_ATTR); - assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); - JUMP_TO_PREDICTED(TRACING_LOAD_ATTR); - } - #endif - PyObject *attr_o; - if (hint >= (size_t)FT_ATOMIC_LOAD_SSIZE_RELAXED(dk->dk_nentries)) { - if (true) { - UPDATE_MISS_STATS(LOAD_ATTR); - assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); - JUMP_TO_PREDICTED(TRACING_LOAD_ATTR); - } - } - PyObject *name = GETITEM(FRAME_CO_NAMES, oparg>>1); - if (dk->dk_kind != DICT_KEYS_UNICODE) { - if (true) { - UPDATE_MISS_STATS(LOAD_ATTR); - assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); - JUMP_TO_PREDICTED(TRACING_LOAD_ATTR); - } - } - PyDictUnicodeEntry *ep = DK_UNICODE_ENTRIES(dk) + hint; - if (FT_ATOMIC_LOAD_PTR_RELAXED(ep->me_key) != name) { - if (true) { - UPDATE_MISS_STATS(LOAD_ATTR); - assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); - JUMP_TO_PREDICTED(TRACING_LOAD_ATTR); - } - } - attr_o = FT_ATOMIC_LOAD_PTR(ep->me_value); - if (attr_o == NULL) { - if (true) { - UPDATE_MISS_STATS(LOAD_ATTR); - assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); - JUMP_TO_PREDICTED(TRACING_LOAD_ATTR); - } - } - STAT_INC(LOAD_ATTR, hit); - #ifdef Py_GIL_DISABLED - int increfed = _Py_TryIncrefCompareStackRef(&ep->me_value, attr_o, &attr); - if (!increfed) { - if (true) { - UPDATE_MISS_STATS(LOAD_ATTR); - assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); - JUMP_TO_PREDICTED(TRACING_LOAD_ATTR); - } - } - #else - attr = PyStackRef_FromPyObjectNew(attr_o); - #endif - stack_pointer[-1] = attr; - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(owner); - stack_pointer = _PyFrame_GetStackPointer(frame); - } - /* Skip 5 cache entries */ - // _PUSH_NULL_CONDITIONAL - { - null = &stack_pointer[0]; - if (oparg & 1) { - null[0] = PyStackRef_NULL; - } - } - stack_pointer += (oparg & 1); - assert(WITHIN_STACK_BOUNDS()); - TRACING_DISPATCH(); - } - - TRACING_TARGET(LOAD_BUILD_CLASS) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = LOAD_BUILD_CLASS; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(LOAD_BUILD_CLASS); - opcode = LOAD_BUILD_CLASS; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; - (void)(_old_stack_level); - _PyStackRef bc; - PyObject *bc_o; - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = PyMapping_GetOptionalItem(BUILTINS(), &_Py_ID(__build_class__), &bc_o); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err < 0) { - TRACING_JUMP_TO_LABEL(error); - } - if (bc_o == NULL) { - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyErr_SetString(tstate, PyExc_NameError, - "__build_class__ not found"); - stack_pointer = _PyFrame_GetStackPointer(frame); - TRACING_JUMP_TO_LABEL(error); - } - bc = PyStackRef_FromPyObjectSteal(bc_o); - stack_pointer[0] = bc; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - TRACING_DISPATCH(); - } - - TRACING_TARGET(LOAD_COMMON_CONSTANT) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = LOAD_COMMON_CONSTANT; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(LOAD_COMMON_CONSTANT); - opcode = LOAD_COMMON_CONSTANT; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; - (void)(_old_stack_level); - _PyStackRef value; - assert(oparg < NUM_COMMON_CONSTANTS); - value = PyStackRef_FromPyObjectNew(tstate->interp->common_consts[oparg]); - stack_pointer[0] = value; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - TRACING_DISPATCH(); - } - - TRACING_TARGET(LOAD_CONST) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = LOAD_CONST; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(LOAD_CONST); - opcode = LOAD_CONST; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; - (void)(_old_stack_level); - _PyStackRef value; - PyObject *obj = GETITEM(FRAME_CO_CONSTS, oparg); - value = PyStackRef_FromPyObjectBorrow(obj); - stack_pointer[0] = value; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - TRACING_DISPATCH(); - } - - TRACING_TARGET(LOAD_DEREF) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = LOAD_DEREF; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(LOAD_DEREF); - opcode = LOAD_DEREF; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; - (void)(_old_stack_level); - _PyStackRef value; - PyCellObject *cell = (PyCellObject *)PyStackRef_AsPyObjectBorrow(GETLOCAL(oparg)); - _PyFrame_SetStackPointer(frame, stack_pointer); - value = _PyCell_GetStackRef(cell); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (PyStackRef_IsNull(value)) { - stack_pointer[0] = value; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyEval_FormatExcUnbound(tstate, _PyFrame_GetCode(frame), oparg); - stack_pointer = _PyFrame_GetStackPointer(frame); - TRACING_JUMP_TO_LABEL(error); - } - stack_pointer[0] = value; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - TRACING_DISPATCH(); - } - - TRACING_TARGET(LOAD_FAST) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = LOAD_FAST; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(LOAD_FAST); - opcode = LOAD_FAST; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; - (void)(_old_stack_level); - _PyStackRef value; - assert(!PyStackRef_IsNull(GETLOCAL(oparg))); - value = PyStackRef_DUP(GETLOCAL(oparg)); - stack_pointer[0] = value; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - TRACING_DISPATCH(); - } - - TRACING_TARGET(LOAD_FAST_AND_CLEAR) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = LOAD_FAST_AND_CLEAR; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(LOAD_FAST_AND_CLEAR); - opcode = LOAD_FAST_AND_CLEAR; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; - (void)(_old_stack_level); - _PyStackRef value; - value = GETLOCAL(oparg); - GETLOCAL(oparg) = PyStackRef_NULL; - stack_pointer[0] = value; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - TRACING_DISPATCH(); - } - - TRACING_TARGET(LOAD_FAST_BORROW) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = LOAD_FAST_BORROW; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(LOAD_FAST_BORROW); - opcode = LOAD_FAST_BORROW; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; - (void)(_old_stack_level); - _PyStackRef value; - assert(!PyStackRef_IsNull(GETLOCAL(oparg))); - value = PyStackRef_Borrow(GETLOCAL(oparg)); - stack_pointer[0] = value; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - TRACING_DISPATCH(); - } - - TRACING_TARGET(LOAD_FAST_BORROW_LOAD_FAST_BORROW) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = LOAD_FAST_BORROW_LOAD_FAST_BORROW; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(LOAD_FAST_BORROW_LOAD_FAST_BORROW); - opcode = LOAD_FAST_BORROW_LOAD_FAST_BORROW; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; - (void)(_old_stack_level); - _PyStackRef value1; - _PyStackRef value2; - uint32_t oparg1 = oparg >> 4; - uint32_t oparg2 = oparg & 15; - value1 = PyStackRef_Borrow(GETLOCAL(oparg1)); - value2 = PyStackRef_Borrow(GETLOCAL(oparg2)); - stack_pointer[0] = value1; - stack_pointer[1] = value2; - stack_pointer += 2; - assert(WITHIN_STACK_BOUNDS()); - TRACING_DISPATCH(); - } - - TRACING_TARGET(LOAD_FAST_CHECK) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = LOAD_FAST_CHECK; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(LOAD_FAST_CHECK); - opcode = LOAD_FAST_CHECK; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; - (void)(_old_stack_level); - _PyStackRef value; - _PyStackRef value_s = GETLOCAL(oparg); - if (PyStackRef_IsNull(value_s)) { - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyEval_FormatExcCheckArg(tstate, PyExc_UnboundLocalError, - UNBOUNDLOCAL_ERROR_MSG, - PyTuple_GetItem(_PyFrame_GetCode(frame)->co_localsplusnames, oparg) - ); - stack_pointer = _PyFrame_GetStackPointer(frame); - TRACING_JUMP_TO_LABEL(error); - } - value = PyStackRef_DUP(value_s); - stack_pointer[0] = value; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - TRACING_DISPATCH(); - } - - TRACING_TARGET(LOAD_FAST_LOAD_FAST) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = LOAD_FAST_LOAD_FAST; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(LOAD_FAST_LOAD_FAST); - opcode = LOAD_FAST_LOAD_FAST; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; - (void)(_old_stack_level); - _PyStackRef value1; - _PyStackRef value2; - uint32_t oparg1 = oparg >> 4; - uint32_t oparg2 = oparg & 15; - value1 = PyStackRef_DUP(GETLOCAL(oparg1)); - value2 = PyStackRef_DUP(GETLOCAL(oparg2)); - stack_pointer[0] = value1; - stack_pointer[1] = value2; - stack_pointer += 2; - assert(WITHIN_STACK_BOUNDS()); - TRACING_DISPATCH(); - } - - TRACING_TARGET(LOAD_FROM_DICT_OR_DEREF) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = LOAD_FROM_DICT_OR_DEREF; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(LOAD_FROM_DICT_OR_DEREF); - opcode = LOAD_FROM_DICT_OR_DEREF; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; - (void)(_old_stack_level); - _PyStackRef class_dict_st; - _PyStackRef value; - class_dict_st = stack_pointer[-1]; - PyObject *value_o; - PyObject *name; - PyObject *class_dict = PyStackRef_AsPyObjectBorrow(class_dict_st); - assert(class_dict); - assert(oparg >= 0 && oparg < _PyFrame_GetCode(frame)->co_nlocalsplus); - name = PyTuple_GET_ITEM(_PyFrame_GetCode(frame)->co_localsplusnames, oparg); - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = PyMapping_GetOptionalItem(class_dict, name, &value_o); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err < 0) { - TRACING_JUMP_TO_LABEL(error); - } - if (!value_o) { - PyCellObject *cell = (PyCellObject *)PyStackRef_AsPyObjectBorrow(GETLOCAL(oparg)); - value_o = PyCell_GetRef(cell); - if (value_o == NULL) { - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyEval_FormatExcUnbound(tstate, _PyFrame_GetCode(frame), oparg); - stack_pointer = _PyFrame_GetStackPointer(frame); - TRACING_JUMP_TO_LABEL(error); - } - } - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(class_dict_st); - stack_pointer = _PyFrame_GetStackPointer(frame); - value = PyStackRef_FromPyObjectSteal(value_o); - stack_pointer[0] = value; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - TRACING_DISPATCH(); - } - - TRACING_TARGET(LOAD_FROM_DICT_OR_GLOBALS) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = LOAD_FROM_DICT_OR_GLOBALS; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(LOAD_FROM_DICT_OR_GLOBALS); - opcode = LOAD_FROM_DICT_OR_GLOBALS; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; - (void)(_old_stack_level); - _PyStackRef mod_or_class_dict; - _PyStackRef v; - mod_or_class_dict = stack_pointer[-1]; - PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); - PyObject *v_o; - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = PyMapping_GetOptionalItem(PyStackRef_AsPyObjectBorrow(mod_or_class_dict), name, &v_o); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(mod_or_class_dict); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err < 0) { - TRACING_JUMP_TO_LABEL(error); - } - if (v_o == NULL) { - if (PyDict_CheckExact(GLOBALS()) - && PyDict_CheckExact(BUILTINS())) - { - _PyFrame_SetStackPointer(frame, stack_pointer); - v_o = _PyDict_LoadGlobal((PyDictObject *)GLOBALS(), - (PyDictObject *)BUILTINS(), - name); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (v_o == NULL) { - if (!_PyErr_Occurred(tstate)) { - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyEval_FormatExcCheckArg(tstate, PyExc_NameError, - NAME_ERROR_MSG, name); - stack_pointer = _PyFrame_GetStackPointer(frame); - } - TRACING_JUMP_TO_LABEL(error); - } - } - else { - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = PyMapping_GetOptionalItem(GLOBALS(), name, &v_o); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err < 0) { - TRACING_JUMP_TO_LABEL(error); - } - if (v_o == NULL) { - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = PyMapping_GetOptionalItem(BUILTINS(), name, &v_o); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err < 0) { - TRACING_JUMP_TO_LABEL(error); - } - if (v_o == NULL) { - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyEval_FormatExcCheckArg( - tstate, PyExc_NameError, - NAME_ERROR_MSG, name); - stack_pointer = _PyFrame_GetStackPointer(frame); - TRACING_JUMP_TO_LABEL(error); - } - } - } - } - v = PyStackRef_FromPyObjectSteal(v_o); - stack_pointer[0] = v; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - TRACING_DISPATCH(); - } - - TRACING_TARGET(LOAD_GLOBAL) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = LOAD_GLOBAL; - (void)(opcode); - #endif - frame->instr_ptr = next_instr; - next_instr += 5; - INSTRUCTION_STATS(LOAD_GLOBAL); - PREDICTED_TRACING_LOAD_GLOBAL:; - _Py_CODEUNIT* const this_instr = next_instr - 5; - (void)this_instr; - opcode = LOAD_GLOBAL; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; - (void)(_old_stack_level); - _PyStackRef *res; - _PyStackRef *null; - // _SPECIALIZE_LOAD_GLOBAL - { - uint16_t counter = read_u16(&this_instr[1].cache); - (void)counter; - #if ENABLE_SPECIALIZATION_FT - if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { - PyObject *name = GETITEM(FRAME_CO_NAMES, oparg>>1); - next_instr = this_instr; - _PyFrame_SetStackPointer(frame, stack_pointer); - _Py_Specialize_LoadGlobal(GLOBALS(), BUILTINS(), next_instr, name); - stack_pointer = _PyFrame_GetStackPointer(frame); - TRACING_SPECIALIZE_DISPATCH_SAME_OPARG() - } - OPCODE_DEFERRED_INC(LOAD_GLOBAL); - ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); - #endif /* ENABLE_SPECIALIZATION_FT */ - } - /* Skip 1 cache entry */ - /* Skip 1 cache entry */ - /* Skip 1 cache entry */ - // _LOAD_GLOBAL - { - res = &stack_pointer[0]; - PyObject *name = GETITEM(FRAME_CO_NAMES, oparg>>1); - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyEval_LoadGlobalStackRef(GLOBALS(), BUILTINS(), name, res); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (PyStackRef_IsNull(*res)) { - TRACING_JUMP_TO_LABEL(error); - } - } - // _PUSH_NULL_CONDITIONAL - { - null = &stack_pointer[1]; - if (oparg & 1) { - null[0] = PyStackRef_NULL; - } - } - stack_pointer += 1 + (oparg & 1); - assert(WITHIN_STACK_BOUNDS()); - TRACING_DISPATCH(); - } - - TRACING_TARGET(LOAD_GLOBAL_BUILTIN) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = LOAD_GLOBAL_BUILTIN; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 5; - INSTRUCTION_STATS(LOAD_GLOBAL_BUILTIN); - opcode = LOAD_GLOBAL_BUILTIN; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; - (void)(_old_stack_level); - static_assert(INLINE_CACHE_ENTRIES_LOAD_GLOBAL == 4, "incorrect cache size"); - _PyStackRef res; - _PyStackRef *null; - /* Skip 1 cache entry */ - // _GUARD_GLOBALS_VERSION - { - uint16_t version = read_u16(&this_instr[2].cache); - PyDictObject *dict = (PyDictObject *)GLOBALS(); - if (!PyDict_CheckExact(dict)) { - UPDATE_MISS_STATS(LOAD_GLOBAL); - assert(_PyOpcode_Deopt[opcode] == (LOAD_GLOBAL)); - JUMP_TO_PREDICTED(TRACING_LOAD_GLOBAL); - } - PyDictKeysObject *keys = FT_ATOMIC_LOAD_PTR_ACQUIRE(dict->ma_keys); - if (FT_ATOMIC_LOAD_UINT32_RELAXED(keys->dk_version) != version) { - UPDATE_MISS_STATS(LOAD_GLOBAL); - assert(_PyOpcode_Deopt[opcode] == (LOAD_GLOBAL)); - JUMP_TO_PREDICTED(TRACING_LOAD_GLOBAL); - } - assert(DK_IS_UNICODE(keys)); - } - // _LOAD_GLOBAL_BUILTINS - { - uint16_t version = read_u16(&this_instr[3].cache); - uint16_t index = read_u16(&this_instr[4].cache); - PyDictObject *dict = (PyDictObject *)BUILTINS(); - if (!PyDict_CheckExact(dict)) { - UPDATE_MISS_STATS(LOAD_GLOBAL); - assert(_PyOpcode_Deopt[opcode] == (LOAD_GLOBAL)); - JUMP_TO_PREDICTED(TRACING_LOAD_GLOBAL); - } - PyDictKeysObject *keys = FT_ATOMIC_LOAD_PTR_ACQUIRE(dict->ma_keys); - if (FT_ATOMIC_LOAD_UINT32_RELAXED(keys->dk_version) != version) { - UPDATE_MISS_STATS(LOAD_GLOBAL); - assert(_PyOpcode_Deopt[opcode] == (LOAD_GLOBAL)); - JUMP_TO_PREDICTED(TRACING_LOAD_GLOBAL); - } - assert(DK_IS_UNICODE(keys)); - PyDictUnicodeEntry *entries = DK_UNICODE_ENTRIES(keys); - PyObject *res_o = FT_ATOMIC_LOAD_PTR_RELAXED(entries[index].me_value); - if (res_o == NULL) { - UPDATE_MISS_STATS(LOAD_GLOBAL); - assert(_PyOpcode_Deopt[opcode] == (LOAD_GLOBAL)); - JUMP_TO_PREDICTED(TRACING_LOAD_GLOBAL); - } - #if Py_GIL_DISABLED - int increfed = _Py_TryIncrefCompareStackRef(&entries[index].me_value, res_o, &res); - if (!increfed) { - UPDATE_MISS_STATS(LOAD_GLOBAL); - assert(_PyOpcode_Deopt[opcode] == (LOAD_GLOBAL)); - JUMP_TO_PREDICTED(TRACING_LOAD_GLOBAL); - } - #else - res = PyStackRef_FromPyObjectNew(res_o); - #endif - STAT_INC(LOAD_GLOBAL, hit); - } - // _PUSH_NULL_CONDITIONAL - { - null = &stack_pointer[1]; - if (oparg & 1) { - null[0] = PyStackRef_NULL; - } - } - stack_pointer[0] = res; - stack_pointer += 1 + (oparg & 1); - assert(WITHIN_STACK_BOUNDS()); - TRACING_DISPATCH(); - } - - TRACING_TARGET(LOAD_GLOBAL_MODULE) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = LOAD_GLOBAL_MODULE; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 5; - INSTRUCTION_STATS(LOAD_GLOBAL_MODULE); - opcode = LOAD_GLOBAL_MODULE; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; - (void)(_old_stack_level); - static_assert(INLINE_CACHE_ENTRIES_LOAD_GLOBAL == 4, "incorrect cache size"); - _PyStackRef res; - _PyStackRef *null; - /* Skip 1 cache entry */ - // _NOP - { - } - // _LOAD_GLOBAL_MODULE - { - uint16_t version = read_u16(&this_instr[2].cache); - uint16_t index = read_u16(&this_instr[4].cache); - PyDictObject *dict = (PyDictObject *)GLOBALS(); - if (!PyDict_CheckExact(dict)) { - UPDATE_MISS_STATS(LOAD_GLOBAL); - assert(_PyOpcode_Deopt[opcode] == (LOAD_GLOBAL)); - JUMP_TO_PREDICTED(TRACING_LOAD_GLOBAL); - } - PyDictKeysObject *keys = FT_ATOMIC_LOAD_PTR_ACQUIRE(dict->ma_keys); - if (FT_ATOMIC_LOAD_UINT32_RELAXED(keys->dk_version) != version) { - UPDATE_MISS_STATS(LOAD_GLOBAL); - assert(_PyOpcode_Deopt[opcode] == (LOAD_GLOBAL)); - JUMP_TO_PREDICTED(TRACING_LOAD_GLOBAL); - } - assert(DK_IS_UNICODE(keys)); - PyDictUnicodeEntry *entries = DK_UNICODE_ENTRIES(keys); - assert(index < DK_SIZE(keys)); - PyObject *res_o = FT_ATOMIC_LOAD_PTR_RELAXED(entries[index].me_value); - if (res_o == NULL) { - UPDATE_MISS_STATS(LOAD_GLOBAL); - assert(_PyOpcode_Deopt[opcode] == (LOAD_GLOBAL)); - JUMP_TO_PREDICTED(TRACING_LOAD_GLOBAL); - } - #if Py_GIL_DISABLED - int increfed = _Py_TryIncrefCompareStackRef(&entries[index].me_value, res_o, &res); - if (!increfed) { - UPDATE_MISS_STATS(LOAD_GLOBAL); - assert(_PyOpcode_Deopt[opcode] == (LOAD_GLOBAL)); - JUMP_TO_PREDICTED(TRACING_LOAD_GLOBAL); - } - #else - res = PyStackRef_FromPyObjectNew(res_o); - #endif - STAT_INC(LOAD_GLOBAL, hit); - } - // _PUSH_NULL_CONDITIONAL - { - null = &stack_pointer[1]; - if (oparg & 1) { - null[0] = PyStackRef_NULL; - } - } - stack_pointer[0] = res; - stack_pointer += 1 + (oparg & 1); - assert(WITHIN_STACK_BOUNDS()); - TRACING_DISPATCH(); - } - - TRACING_TARGET(LOAD_LOCALS) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = LOAD_LOCALS; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(LOAD_LOCALS); - opcode = LOAD_LOCALS; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; - (void)(_old_stack_level); - _PyStackRef locals; - PyObject *l = LOCALS(); - if (l == NULL) { - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyErr_SetString(tstate, PyExc_SystemError, - "no locals found"); - stack_pointer = _PyFrame_GetStackPointer(frame); - TRACING_JUMP_TO_LABEL(error); - } - locals = PyStackRef_FromPyObjectNew(l); - stack_pointer[0] = locals; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - TRACING_DISPATCH(); - } - - TRACING_TARGET(LOAD_NAME) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = LOAD_NAME; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(LOAD_NAME); - opcode = LOAD_NAME; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; - (void)(_old_stack_level); - _PyStackRef v; - PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *v_o = _PyEval_LoadName(tstate, frame, name); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (v_o == NULL) { - TRACING_JUMP_TO_LABEL(error); - } - v = PyStackRef_FromPyObjectSteal(v_o); - stack_pointer[0] = v; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - TRACING_DISPATCH(); - } - - TRACING_TARGET(LOAD_SMALL_INT) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = LOAD_SMALL_INT; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(LOAD_SMALL_INT); - opcode = LOAD_SMALL_INT; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; - (void)(_old_stack_level); - _PyStackRef value; - assert(oparg < _PY_NSMALLPOSINTS); - PyObject *obj = (PyObject *)&_PyLong_SMALL_INTS[_PY_NSMALLNEGINTS + oparg]; - value = PyStackRef_FromPyObjectBorrow(obj); - stack_pointer[0] = value; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - TRACING_DISPATCH(); - } - - TRACING_TARGET(LOAD_SPECIAL) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = LOAD_SPECIAL; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(LOAD_SPECIAL); - opcode = LOAD_SPECIAL; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; - (void)(_old_stack_level); - _PyStackRef self; - _PyStackRef *method_and_self; - // _INSERT_NULL - { - self = stack_pointer[-1]; - method_and_self = &stack_pointer[-1]; - method_and_self[1] = self; - method_and_self[0] = PyStackRef_NULL; - } - // _LOAD_SPECIAL - { - method_and_self = &stack_pointer[-1]; - PyObject *name = _Py_SpecialMethods[oparg].name; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = _PyObject_LookupSpecialMethod(name, method_and_self); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err <= 0) { - if (err == 0) { - PyObject *owner = PyStackRef_AsPyObjectBorrow(method_and_self[1]); - _PyFrame_SetStackPointer(frame, stack_pointer); - const char *errfmt = _PyEval_SpecialMethodCanSuggest(owner, oparg) - ? _Py_SpecialMethods[oparg].error_suggestion - : _Py_SpecialMethods[oparg].error; - stack_pointer = _PyFrame_GetStackPointer(frame); - assert(!_PyErr_Occurred(tstate)); - assert(errfmt != NULL); - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyErr_Format(tstate, PyExc_TypeError, errfmt, owner); - stack_pointer = _PyFrame_GetStackPointer(frame); - } - TRACING_JUMP_TO_LABEL(error); - } - } - TRACING_DISPATCH(); - } - - TRACING_TARGET(LOAD_SUPER_ATTR) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = LOAD_SUPER_ATTR; - (void)(opcode); - #endif - frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(LOAD_SUPER_ATTR); - PREDICTED_TRACING_LOAD_SUPER_ATTR:; - _Py_CODEUNIT* const this_instr = next_instr - 2; - (void)this_instr; - opcode = LOAD_SUPER_ATTR; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; - (void)(_old_stack_level); - _PyStackRef global_super_st; - _PyStackRef class_st; - _PyStackRef self_st; - _PyStackRef attr; - _PyStackRef *null; - // _SPECIALIZE_LOAD_SUPER_ATTR - { - class_st = stack_pointer[-2]; - global_super_st = stack_pointer[-3]; - uint16_t counter = read_u16(&this_instr[1].cache); - (void)counter; - #if ENABLE_SPECIALIZATION_FT - int load_method = oparg & 1; - if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { - next_instr = this_instr; - _PyFrame_SetStackPointer(frame, stack_pointer); - _Py_Specialize_LoadSuperAttr(global_super_st, class_st, next_instr, load_method); - stack_pointer = _PyFrame_GetStackPointer(frame); - TRACING_SPECIALIZE_DISPATCH_SAME_OPARG() - } - OPCODE_DEFERRED_INC(LOAD_SUPER_ATTR); - ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); - #endif /* ENABLE_SPECIALIZATION_FT */ - } - // _LOAD_SUPER_ATTR - { - self_st = stack_pointer[-1]; - PyObject *global_super = PyStackRef_AsPyObjectBorrow(global_super_st); - PyObject *class = PyStackRef_AsPyObjectBorrow(class_st); - PyObject *self = PyStackRef_AsPyObjectBorrow(self_st); - if (opcode == INSTRUMENTED_LOAD_SUPER_ATTR) { - PyObject *arg = oparg & 2 ? class : &_PyInstrumentation_MISSING; - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = _Py_call_instrumentation_2args( - tstate, PY_MONITORING_EVENT_CALL, - frame, this_instr, global_super, arg); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err) { - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyStackRef tmp = self_st; - self_st = PyStackRef_NULL; - stack_pointer[-1] = self_st; - PyStackRef_CLOSE(tmp); - tmp = class_st; - class_st = PyStackRef_NULL; - stack_pointer[-2] = class_st; - PyStackRef_CLOSE(tmp); - tmp = global_super_st; - global_super_st = PyStackRef_NULL; - stack_pointer[-3] = global_super_st; - PyStackRef_CLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -3; - assert(WITHIN_STACK_BOUNDS()); - TRACING_JUMP_TO_LABEL(error); - } - } - PyObject *stack[] = {class, self}; - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *super = PyObject_Vectorcall(global_super, stack, oparg & 2, NULL); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (opcode == INSTRUMENTED_LOAD_SUPER_ATTR) { - PyObject *arg = oparg & 2 ? class : &_PyInstrumentation_MISSING; - if (super == NULL) { - _PyFrame_SetStackPointer(frame, stack_pointer); - _Py_call_instrumentation_exc2( - tstate, PY_MONITORING_EVENT_C_RAISE, - frame, this_instr, global_super, arg); - stack_pointer = _PyFrame_GetStackPointer(frame); - } - else { - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = _Py_call_instrumentation_2args( - tstate, PY_MONITORING_EVENT_C_RETURN, - frame, this_instr, global_super, arg); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err < 0) { - _PyFrame_SetStackPointer(frame, stack_pointer); - Py_CLEAR(super); - stack_pointer = _PyFrame_GetStackPointer(frame); - } - } - } - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyStackRef tmp = self_st; - self_st = PyStackRef_NULL; - stack_pointer[-1] = self_st; - PyStackRef_CLOSE(tmp); - tmp = class_st; - class_st = PyStackRef_NULL; - stack_pointer[-2] = class_st; - PyStackRef_CLOSE(tmp); - tmp = global_super_st; - global_super_st = PyStackRef_NULL; - stack_pointer[-3] = global_super_st; - PyStackRef_CLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -3; - assert(WITHIN_STACK_BOUNDS()); - if (super == NULL) { - TRACING_JUMP_TO_LABEL(error); - } - PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 2); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *attr_o = PyObject_GetAttr(super, name); - Py_DECREF(super); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (attr_o == NULL) { - TRACING_JUMP_TO_LABEL(error); - } - attr = PyStackRef_FromPyObjectSteal(attr_o); - } - // _PUSH_NULL_CONDITIONAL - { - null = &stack_pointer[1]; - if (oparg & 1) { - null[0] = PyStackRef_NULL; - } - } - stack_pointer[0] = attr; - stack_pointer += 1 + (oparg & 1); - assert(WITHIN_STACK_BOUNDS()); - TRACING_DISPATCH(); - } - - TRACING_TARGET(LOAD_SUPER_ATTR_ATTR) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = LOAD_SUPER_ATTR_ATTR; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(LOAD_SUPER_ATTR_ATTR); - opcode = LOAD_SUPER_ATTR_ATTR; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; - (void)(_old_stack_level); - static_assert(INLINE_CACHE_ENTRIES_LOAD_SUPER_ATTR == 1, "incorrect cache size"); - _PyStackRef global_super_st; - _PyStackRef class_st; - _PyStackRef self_st; - _PyStackRef attr_st; - /* Skip 1 cache entry */ - self_st = stack_pointer[-1]; - class_st = stack_pointer[-2]; - global_super_st = stack_pointer[-3]; - PyObject *global_super = PyStackRef_AsPyObjectBorrow(global_super_st); - PyObject *class = PyStackRef_AsPyObjectBorrow(class_st); - PyObject *self = PyStackRef_AsPyObjectBorrow(self_st); - assert(!(oparg & 1)); - if (global_super != (PyObject *)&PySuper_Type) { - UPDATE_MISS_STATS(LOAD_SUPER_ATTR); - assert(_PyOpcode_Deopt[opcode] == (LOAD_SUPER_ATTR)); - JUMP_TO_PREDICTED(TRACING_LOAD_SUPER_ATTR); - } - if (!PyType_Check(class)) { - UPDATE_MISS_STATS(LOAD_SUPER_ATTR); - assert(_PyOpcode_Deopt[opcode] == (LOAD_SUPER_ATTR)); - JUMP_TO_PREDICTED(TRACING_LOAD_SUPER_ATTR); - } - STAT_INC(LOAD_SUPER_ATTR, hit); - PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 2); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *attr = _PySuper_Lookup((PyTypeObject *)class, self, name, NULL); - _PyStackRef tmp = self_st; - self_st = PyStackRef_NULL; - stack_pointer[-1] = self_st; - PyStackRef_CLOSE(tmp); - tmp = class_st; - class_st = PyStackRef_NULL; - stack_pointer[-2] = class_st; - PyStackRef_CLOSE(tmp); - tmp = global_super_st; - global_super_st = PyStackRef_NULL; - stack_pointer[-3] = global_super_st; - PyStackRef_CLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -3; - assert(WITHIN_STACK_BOUNDS()); - if (attr == NULL) { - TRACING_JUMP_TO_LABEL(error); - } - attr_st = PyStackRef_FromPyObjectSteal(attr); - stack_pointer[0] = attr_st; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - TRACING_DISPATCH(); - } - - TRACING_TARGET(LOAD_SUPER_ATTR_METHOD) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = LOAD_SUPER_ATTR_METHOD; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(LOAD_SUPER_ATTR_METHOD); - opcode = LOAD_SUPER_ATTR_METHOD; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; - (void)(_old_stack_level); - static_assert(INLINE_CACHE_ENTRIES_LOAD_SUPER_ATTR == 1, "incorrect cache size"); - _PyStackRef global_super_st; - _PyStackRef class_st; - _PyStackRef self_st; - _PyStackRef attr; - _PyStackRef self_or_null; - /* Skip 1 cache entry */ - self_st = stack_pointer[-1]; - class_st = stack_pointer[-2]; - global_super_st = stack_pointer[-3]; - PyObject *global_super = PyStackRef_AsPyObjectBorrow(global_super_st); - PyObject *class = PyStackRef_AsPyObjectBorrow(class_st); - PyObject *self = PyStackRef_AsPyObjectBorrow(self_st); - assert(oparg & 1); - if (global_super != (PyObject *)&PySuper_Type) { - UPDATE_MISS_STATS(LOAD_SUPER_ATTR); - assert(_PyOpcode_Deopt[opcode] == (LOAD_SUPER_ATTR)); - JUMP_TO_PREDICTED(TRACING_LOAD_SUPER_ATTR); - } - if (!PyType_Check(class)) { - UPDATE_MISS_STATS(LOAD_SUPER_ATTR); - assert(_PyOpcode_Deopt[opcode] == (LOAD_SUPER_ATTR)); - JUMP_TO_PREDICTED(TRACING_LOAD_SUPER_ATTR); - } - STAT_INC(LOAD_SUPER_ATTR, hit); - PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 2); - PyTypeObject *cls = (PyTypeObject *)class; - int method_found = 0; - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *attr_o = _PySuper_Lookup(cls, self, name, - Py_TYPE(self)->tp_getattro == PyObject_GenericGetAttr ? &method_found : NULL); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (attr_o == NULL) { - TRACING_JUMP_TO_LABEL(error); - } - if (method_found) { - self_or_null = self_st; - } else { - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(self_st); - stack_pointer = _PyFrame_GetStackPointer(frame); - self_or_null = PyStackRef_NULL; - stack_pointer += 1; - } - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyStackRef tmp = global_super_st; - global_super_st = self_or_null; - stack_pointer[-2] = global_super_st; - PyStackRef_CLOSE(tmp); - tmp = class_st; - class_st = PyStackRef_NULL; - stack_pointer[-1] = class_st; - PyStackRef_CLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); - attr = PyStackRef_FromPyObjectSteal(attr_o); - stack_pointer[0] = attr; - stack_pointer[1] = self_or_null; - stack_pointer += 2; - assert(WITHIN_STACK_BOUNDS()); - TRACING_DISPATCH(); - } - - TRACING_TARGET(MAKE_CELL) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = MAKE_CELL; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(MAKE_CELL); - opcode = MAKE_CELL; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; - (void)(_old_stack_level); - PyObject *initial = PyStackRef_AsPyObjectBorrow(GETLOCAL(oparg)); - PyObject *cell = PyCell_New(initial); - if (cell == NULL) { - TRACING_JUMP_TO_LABEL(error); - } - _PyStackRef tmp = GETLOCAL(oparg); - GETLOCAL(oparg) = PyStackRef_FromPyObjectSteal(cell); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_XCLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); - TRACING_DISPATCH(); - } - - TRACING_TARGET(MAKE_FUNCTION) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = MAKE_FUNCTION; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(MAKE_FUNCTION); - opcode = MAKE_FUNCTION; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; - (void)(_old_stack_level); - _PyStackRef codeobj_st; - _PyStackRef func; - codeobj_st = stack_pointer[-1]; - PyObject *codeobj = PyStackRef_AsPyObjectBorrow(codeobj_st); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyFunctionObject *func_obj = (PyFunctionObject *) - PyFunction_New(codeobj, GLOBALS()); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(codeobj_st); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (func_obj == NULL) { - TRACING_JUMP_TO_LABEL(error); - } - _PyFunction_SetVersion( - func_obj, ((PyCodeObject *)codeobj)->co_version); - func = PyStackRef_FromPyObjectSteal((PyObject *)func_obj); - stack_pointer[0] = func; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - TRACING_DISPATCH(); - } - - TRACING_TARGET(MAP_ADD) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = MAP_ADD; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(MAP_ADD); - opcode = MAP_ADD; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; - (void)(_old_stack_level); - _PyStackRef dict_st; - _PyStackRef key; - _PyStackRef value; - value = stack_pointer[-1]; - key = stack_pointer[-2]; - dict_st = stack_pointer[-3 - (oparg - 1)]; - PyObject *dict = PyStackRef_AsPyObjectBorrow(dict_st); - assert(PyDict_CheckExact(dict)); - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = _PyDict_SetItem_Take2( - (PyDictObject *)dict, - PyStackRef_AsPyObjectSteal(key), - PyStackRef_AsPyObjectSteal(value) - ); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err != 0) { - TRACING_JUMP_TO_LABEL(pop_2_error); - } - stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); - TRACING_DISPATCH(); - } - - TRACING_TARGET(MATCH_CLASS) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = MATCH_CLASS; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(MATCH_CLASS); - opcode = MATCH_CLASS; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; - (void)(_old_stack_level); - _PyStackRef subject; - _PyStackRef type; - _PyStackRef names; - _PyStackRef attrs; - names = stack_pointer[-1]; - type = stack_pointer[-2]; - subject = stack_pointer[-3]; - assert(PyTuple_CheckExact(PyStackRef_AsPyObjectBorrow(names))); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *attrs_o = _PyEval_MatchClass(tstate, - PyStackRef_AsPyObjectBorrow(subject), - PyStackRef_AsPyObjectBorrow(type), oparg, - PyStackRef_AsPyObjectBorrow(names)); - _PyStackRef tmp = names; - names = PyStackRef_NULL; - stack_pointer[-1] = names; - PyStackRef_CLOSE(tmp); - tmp = type; - type = PyStackRef_NULL; - stack_pointer[-2] = type; - PyStackRef_CLOSE(tmp); - tmp = subject; - subject = PyStackRef_NULL; - stack_pointer[-3] = subject; - PyStackRef_CLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -3; - assert(WITHIN_STACK_BOUNDS()); - if (attrs_o) { - assert(PyTuple_CheckExact(attrs_o)); - attrs = PyStackRef_FromPyObjectSteal(attrs_o); - } - else { - if (_PyErr_Occurred(tstate)) { - TRACING_JUMP_TO_LABEL(error); - } - attrs = PyStackRef_None; - } - stack_pointer[0] = attrs; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - TRACING_DISPATCH(); - } - - TRACING_TARGET(MATCH_KEYS) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = MATCH_KEYS; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(MATCH_KEYS); - opcode = MATCH_KEYS; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; - (void)(_old_stack_level); - _PyStackRef subject; - _PyStackRef keys; - _PyStackRef values_or_none; - keys = stack_pointer[-1]; - subject = stack_pointer[-2]; - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *values_or_none_o = _PyEval_MatchKeys(tstate, - PyStackRef_AsPyObjectBorrow(subject), PyStackRef_AsPyObjectBorrow(keys)); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (values_or_none_o == NULL) { - TRACING_JUMP_TO_LABEL(error); - } - values_or_none = PyStackRef_FromPyObjectSteal(values_or_none_o); - stack_pointer[0] = values_or_none; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - TRACING_DISPATCH(); - } - - TRACING_TARGET(MATCH_MAPPING) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = MATCH_MAPPING; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(MATCH_MAPPING); - opcode = MATCH_MAPPING; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; - (void)(_old_stack_level); - _PyStackRef subject; - _PyStackRef res; - subject = stack_pointer[-1]; - int match = PyStackRef_TYPE(subject)->tp_flags & Py_TPFLAGS_MAPPING; - res = match ? PyStackRef_True : PyStackRef_False; - stack_pointer[0] = res; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - TRACING_DISPATCH(); - } - - TRACING_TARGET(MATCH_SEQUENCE) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = MATCH_SEQUENCE; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(MATCH_SEQUENCE); - opcode = MATCH_SEQUENCE; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; - (void)(_old_stack_level); - _PyStackRef subject; - _PyStackRef res; - subject = stack_pointer[-1]; - int match = PyStackRef_TYPE(subject)->tp_flags & Py_TPFLAGS_SEQUENCE; - res = match ? PyStackRef_True : PyStackRef_False; - stack_pointer[0] = res; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - TRACING_DISPATCH(); - } - - TRACING_TARGET(NOP) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = NOP; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(NOP); - opcode = NOP; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; - (void)(_old_stack_level); - TRACING_DISPATCH(); - } - - TRACING_TARGET(NOT_TAKEN) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = NOT_TAKEN; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(NOT_TAKEN); - opcode = NOT_TAKEN; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; - (void)(_old_stack_level); - TRACING_DISPATCH(); - } - - TRACING_TARGET(POP_EXCEPT) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = POP_EXCEPT; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(POP_EXCEPT); - opcode = POP_EXCEPT; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; - (void)(_old_stack_level); - _PyStackRef exc_value; - exc_value = stack_pointer[-1]; - _PyErr_StackItem *exc_info = tstate->exc_info; - _PyFrame_SetStackPointer(frame, stack_pointer); - Py_XSETREF(exc_info->exc_value, - PyStackRef_IsNone(exc_value) - ? NULL : PyStackRef_AsPyObjectSteal(exc_value)); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - TRACING_DISPATCH(); - } - - TRACING_TARGET(POP_ITER) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = POP_ITER; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(POP_ITER); - opcode = POP_ITER; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; - (void)(_old_stack_level); - _PyStackRef iter; - _PyStackRef index_or_null; - index_or_null = stack_pointer[-1]; - iter = stack_pointer[-2]; - (void)index_or_null; - stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(iter); - stack_pointer = _PyFrame_GetStackPointer(frame); - TRACING_DISPATCH(); - } - - TRACING_TARGET(POP_JUMP_IF_FALSE) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = POP_JUMP_IF_FALSE; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(POP_JUMP_IF_FALSE); - opcode = POP_JUMP_IF_FALSE; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; - (void)(_old_stack_level); - _PyStackRef cond; - /* Skip 1 cache entry */ - cond = stack_pointer[-1]; - assert(PyStackRef_BoolCheck(cond)); - int flag = PyStackRef_IsFalse(cond); - RECORD_BRANCH_TAKEN(this_instr[1].cache, flag); - JUMPBY(flag ? oparg : next_instr->op.code == NOT_TAKEN); - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - TRACING_DISPATCH(); - } - - TRACING_TARGET(POP_JUMP_IF_NONE) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = POP_JUMP_IF_NONE; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(POP_JUMP_IF_NONE); - opcode = POP_JUMP_IF_NONE; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; - (void)(_old_stack_level); - _PyStackRef value; - _PyStackRef b; - _PyStackRef cond; - /* Skip 1 cache entry */ - // _IS_NONE - { - value = stack_pointer[-1]; - if (PyStackRef_IsNone(value)) { - b = PyStackRef_True; - } - else { - b = PyStackRef_False; - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyStackRef tmp = value; - value = b; - stack_pointer[-1] = value; - PyStackRef_CLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); - } - } - // _POP_JUMP_IF_TRUE - { - cond = b; - assert(PyStackRef_BoolCheck(cond)); - int flag = PyStackRef_IsTrue(cond); - RECORD_BRANCH_TAKEN(this_instr[1].cache, flag); - JUMPBY(flag ? oparg : next_instr->op.code == NOT_TAKEN); - } - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - TRACING_DISPATCH(); - } - - TRACING_TARGET(POP_JUMP_IF_NOT_NONE) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = POP_JUMP_IF_NOT_NONE; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(POP_JUMP_IF_NOT_NONE); - opcode = POP_JUMP_IF_NOT_NONE; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; - (void)(_old_stack_level); - _PyStackRef value; - _PyStackRef b; - _PyStackRef cond; - /* Skip 1 cache entry */ - // _IS_NONE - { - value = stack_pointer[-1]; - if (PyStackRef_IsNone(value)) { - b = PyStackRef_True; - } - else { - b = PyStackRef_False; - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyStackRef tmp = value; - value = b; - stack_pointer[-1] = value; - PyStackRef_CLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); - } - } - // _POP_JUMP_IF_FALSE - { - cond = b; - assert(PyStackRef_BoolCheck(cond)); - int flag = PyStackRef_IsFalse(cond); - RECORD_BRANCH_TAKEN(this_instr[1].cache, flag); - JUMPBY(flag ? oparg : next_instr->op.code == NOT_TAKEN); - } - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - TRACING_DISPATCH(); - } - - TRACING_TARGET(POP_JUMP_IF_TRUE) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = POP_JUMP_IF_TRUE; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(POP_JUMP_IF_TRUE); - opcode = POP_JUMP_IF_TRUE; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; - (void)(_old_stack_level); - _PyStackRef cond; - /* Skip 1 cache entry */ - cond = stack_pointer[-1]; - assert(PyStackRef_BoolCheck(cond)); - int flag = PyStackRef_IsTrue(cond); - RECORD_BRANCH_TAKEN(this_instr[1].cache, flag); - JUMPBY(flag ? oparg : next_instr->op.code == NOT_TAKEN); - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - TRACING_DISPATCH(); - } - - TRACING_TARGET(POP_TOP) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = POP_TOP; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(POP_TOP); - opcode = POP_TOP; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; - (void)(_old_stack_level); - _PyStackRef value; - value = stack_pointer[-1]; - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_XCLOSE(value); - stack_pointer = _PyFrame_GetStackPointer(frame); - TRACING_DISPATCH(); - } - - TRACING_TARGET(PUSH_EXC_INFO) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = PUSH_EXC_INFO; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(PUSH_EXC_INFO); - opcode = PUSH_EXC_INFO; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; - (void)(_old_stack_level); - _PyStackRef exc; - _PyStackRef prev_exc; - _PyStackRef new_exc; - exc = stack_pointer[-1]; - _PyErr_StackItem *exc_info = tstate->exc_info; - if (exc_info->exc_value != NULL) { - prev_exc = PyStackRef_FromPyObjectSteal(exc_info->exc_value); - } - else { - prev_exc = PyStackRef_None; - } - assert(PyStackRef_ExceptionInstanceCheck(exc)); - exc_info->exc_value = PyStackRef_AsPyObjectNew(exc); - new_exc = exc; - stack_pointer[-1] = prev_exc; - stack_pointer[0] = new_exc; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - TRACING_DISPATCH(); - } - - TRACING_TARGET(PUSH_NULL) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = PUSH_NULL; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(PUSH_NULL); - opcode = PUSH_NULL; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; - (void)(_old_stack_level); - _PyStackRef res; - res = PyStackRef_NULL; - stack_pointer[0] = res; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - TRACING_DISPATCH(); - } - - TRACING_TARGET(RAISE_VARARGS) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = RAISE_VARARGS; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(RAISE_VARARGS); - opcode = RAISE_VARARGS; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; - (void)(_old_stack_level); - _PyStackRef *args; - args = &stack_pointer[-oparg]; - assert(oparg < 3); - PyObject *cause = oparg == 2 ? PyStackRef_AsPyObjectSteal(args[1]) : NULL; - PyObject *exc = oparg > 0 ? PyStackRef_AsPyObjectSteal(args[0]) : NULL; - stack_pointer += -oparg; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = do_raise(tstate, exc, cause); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err) { - assert(oparg == 0); - _PyFrame_SetStackPointer(frame, stack_pointer); - monitor_reraise(tstate, frame, this_instr); - TRACING_JUMP_TO_LABEL(exception_unwind); - } - TRACING_JUMP_TO_LABEL(error); - } - - TRACING_TARGET(RERAISE) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = RERAISE; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(RERAISE); - opcode = RERAISE; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; - (void)(_old_stack_level); - _PyStackRef *values; - _PyStackRef exc_st; - exc_st = stack_pointer[-1]; - values = &stack_pointer[-1 - oparg]; - PyObject *exc = PyStackRef_AsPyObjectSteal(exc_st); - assert(oparg >= 0 && oparg <= 2); - if (oparg) { - frame->instr_ptr = _PyFrame_GetBytecode(frame) + PyStackRef_UntagInt(values[0]); - } - assert(exc && PyExceptionInstance_Check(exc)); - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyErr_SetRaisedException(tstate, exc); - monitor_reraise(tstate, frame, this_instr); - TRACING_JUMP_TO_LABEL(exception_unwind); - } - - TRACING_TARGET(RESERVED) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = RESERVED; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(RESERVED); - opcode = RESERVED; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; - (void)(_old_stack_level); - assert(0 && "Executing RESERVED instruction."); - Py_FatalError("Executing RESERVED instruction."); - TRACING_DISPATCH(); - } - - TRACING_TARGET(RESUME) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = RESUME; - (void)(opcode); - #endif - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(RESUME); - PREDICTED_TRACING_RESUME:; - _Py_CODEUNIT* const this_instr = next_instr - 1; - (void)this_instr; - opcode = RESUME; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; - (void)(_old_stack_level); - // _LOAD_BYTECODE - { - #ifdef Py_GIL_DISABLED - if (frame->tlbc_index != - ((_PyThreadStateImpl *)tstate)->tlbc_index) { - _PyFrame_SetStackPointer(frame, stack_pointer); - _Py_CODEUNIT *bytecode = - _PyEval_GetExecutableCode(tstate, _PyFrame_GetCode(frame)); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (bytecode == NULL) { - TRACING_JUMP_TO_LABEL(error); - } - ptrdiff_t off = this_instr - _PyFrame_GetBytecode(frame); - frame->tlbc_index = ((_PyThreadStateImpl *)tstate)->tlbc_index; - frame->instr_ptr = bytecode + off; - next_instr = frame->instr_ptr; - TRACING_DISPATCH(); - } - #endif - } - // _MAYBE_INSTRUMENT - { - #ifdef Py_GIL_DISABLED - - int check_instrumentation = 1; - #else - int check_instrumentation = (tstate->tracing == 0); - #endif - if (check_instrumentation) { - uintptr_t global_version = _Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & ~_PY_EVAL_EVENTS_MASK; - uintptr_t code_version = FT_ATOMIC_LOAD_UINTPTR_ACQUIRE(_PyFrame_GetCode(frame)->_co_instrumentation_version); - if (code_version != global_version) { - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = _Py_Instrument(_PyFrame_GetCode(frame), tstate->interp); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err) { - TRACING_JUMP_TO_LABEL(error); - } - next_instr = this_instr; - TRACING_DISPATCH(); - } - } - } - // _QUICKEN_RESUME - { - #if ENABLE_SPECIALIZATION_FT - if (tstate->tracing == 0 && this_instr->op.code == RESUME) { - FT_ATOMIC_STORE_UINT8_RELAXED(this_instr->op.code, RESUME_CHECK); - } - #endif /* ENABLE_SPECIALIZATION_FT */ - } - // _CHECK_PERIODIC_IF_NOT_YIELD_FROM - { - if ((oparg & RESUME_OPARG_LOCATION_MASK) < RESUME_AFTER_YIELD_FROM) { - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = check_periodics(tstate); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err != 0) { - TRACING_JUMP_TO_LABEL(error); - } - } - } - TRACING_DISPATCH(); - } - - TRACING_TARGET(RESUME_CHECK) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = RESUME_CHECK; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(RESUME_CHECK); - opcode = RESUME_CHECK; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; - (void)(_old_stack_level); - static_assert(0 == 0, "incorrect cache size"); - #if defined(__EMSCRIPTEN__) - if (_Py_emscripten_signal_clock == 0) { - UPDATE_MISS_STATS(RESUME); - assert(_PyOpcode_Deopt[opcode] == (RESUME)); - JUMP_TO_PREDICTED(TRACING_RESUME); - } - _Py_emscripten_signal_clock -= Py_EMSCRIPTEN_SIGNAL_HANDLING; - #endif - uintptr_t eval_breaker = _Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker); - uintptr_t version = FT_ATOMIC_LOAD_UINTPTR_ACQUIRE(_PyFrame_GetCode(frame)->_co_instrumentation_version); - assert((version & _PY_EVAL_EVENTS_MASK) == 0); - if (eval_breaker != version) { - UPDATE_MISS_STATS(RESUME); - assert(_PyOpcode_Deopt[opcode] == (RESUME)); - JUMP_TO_PREDICTED(TRACING_RESUME); - } - #ifdef Py_GIL_DISABLED - if (frame->tlbc_index != - ((_PyThreadStateImpl *)tstate)->tlbc_index) { - UPDATE_MISS_STATS(RESUME); - assert(_PyOpcode_Deopt[opcode] == (RESUME)); - JUMP_TO_PREDICTED(TRACING_RESUME); - } - #endif - TRACING_DISPATCH(); - } - - TRACING_TARGET(RETURN_GENERATOR) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = RETURN_GENERATOR; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(RETURN_GENERATOR); - opcode = RETURN_GENERATOR; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; - (void)(_old_stack_level); - _PyStackRef res; - assert(PyStackRef_FunctionCheck(frame->f_funcobj)); - PyFunctionObject *func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyGenObject *gen = (PyGenObject *)_Py_MakeCoro(func); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (gen == NULL) { - TRACING_JUMP_TO_LABEL(error); - } - assert(STACK_LEVEL() == 0); - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyInterpreterFrame *gen_frame = &gen->gi_iframe; - frame->instr_ptr++; - _PyFrame_Copy(frame, gen_frame); - assert(frame->frame_obj == NULL); - gen->gi_frame_state = FRAME_CREATED; - gen_frame->owner = FRAME_OWNED_BY_GENERATOR; - _Py_LeaveRecursiveCallPy(tstate); - _PyInterpreterFrame *prev = frame->previous; - _PyThreadState_PopFrame(tstate, frame); - frame = tstate->current_frame = prev; - #if TIER_ONE - LOAD_IP(frame->return_offset); - #endif - #if TIER_TWO - TIER2_STORE_IP(frame->return_offset); - #endif - stack_pointer = _PyFrame_GetStackPointer(frame); - res = PyStackRef_FromPyObjectStealMortal((PyObject *)gen); - LLTRACE_RESUME_FRAME(); - stack_pointer[0] = res; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - TRACING_DISPATCH(); - } - - TRACING_TARGET(RETURN_VALUE) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = RETURN_VALUE; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(RETURN_VALUE); - opcode = RETURN_VALUE; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; - (void)(_old_stack_level); - _PyStackRef retval; - _PyStackRef res; - retval = stack_pointer[-1]; - assert(frame->owner != FRAME_OWNED_BY_INTERPRETER); - _PyStackRef temp = PyStackRef_MakeHeapSafe(retval); - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - assert(STACK_LEVEL() == 0); - _Py_LeaveRecursiveCallPy(tstate); - _PyInterpreterFrame *dying = frame; - frame = tstate->current_frame = dying->previous; - _PyEval_FrameClearAndPop(tstate, dying); - stack_pointer = _PyFrame_GetStackPointer(frame); - #if TIER_ONE - LOAD_IP(frame->return_offset); - #endif - #if TIER_TWO - TIER2_STORE_IP(frame->return_offset); - #endif - res = temp; - LLTRACE_RESUME_FRAME(); - stack_pointer[0] = res; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - TRACING_DISPATCH(); - } - - TRACING_TARGET(SEND) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = SEND; - (void)(opcode); - #endif - frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(SEND); - PREDICTED_TRACING_SEND:; - _Py_CODEUNIT* const this_instr = next_instr - 2; - (void)this_instr; - opcode = SEND; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; - (void)(_old_stack_level); - _PyStackRef receiver; - _PyStackRef v; - _PyStackRef retval; - // _SPECIALIZE_SEND - { - receiver = stack_pointer[-2]; - uint16_t counter = read_u16(&this_instr[1].cache); - (void)counter; - #if ENABLE_SPECIALIZATION_FT - if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { - next_instr = this_instr; - _PyFrame_SetStackPointer(frame, stack_pointer); - _Py_Specialize_Send(receiver, next_instr); - stack_pointer = _PyFrame_GetStackPointer(frame); - TRACING_SPECIALIZE_DISPATCH_SAME_OPARG() - } - OPCODE_DEFERRED_INC(SEND); - ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); - #endif /* ENABLE_SPECIALIZATION_FT */ - } - // _SEND - { - v = stack_pointer[-1]; - PyObject *receiver_o = PyStackRef_AsPyObjectBorrow(receiver); - PyObject *retval_o; - assert(frame->owner != FRAME_OWNED_BY_INTERPRETER); - if ((tstate->interp->eval_frame == NULL) && - (Py_TYPE(receiver_o) == &PyGen_Type || Py_TYPE(receiver_o) == &PyCoro_Type) && - ((PyGenObject *)receiver_o)->gi_frame_state < FRAME_EXECUTING) - { - PyGenObject *gen = (PyGenObject *)receiver_o; - _PyInterpreterFrame *gen_frame = &gen->gi_iframe; - _PyFrame_StackPush(gen_frame, PyStackRef_MakeHeapSafe(v)); - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - gen->gi_frame_state = FRAME_EXECUTING; - gen->gi_exc_state.previous_item = tstate->exc_info; - tstate->exc_info = &gen->gi_exc_state; - assert( 2u + oparg <= UINT16_MAX); - frame->return_offset = (uint16_t)( 2u + oparg); - assert(gen_frame->previous == NULL); - gen_frame->previous = frame; - TRACING_DISPATCH_INLINED(gen_frame); - } - if (PyStackRef_IsNone(v) && PyIter_Check(receiver_o)) { - _PyFrame_SetStackPointer(frame, stack_pointer); - retval_o = Py_TYPE(receiver_o)->tp_iternext(receiver_o); - stack_pointer = _PyFrame_GetStackPointer(frame); - } - else { - _PyFrame_SetStackPointer(frame, stack_pointer); - retval_o = PyObject_CallMethodOneArg(receiver_o, - &_Py_ID(send), - PyStackRef_AsPyObjectBorrow(v)); - stack_pointer = _PyFrame_GetStackPointer(frame); - } - if (retval_o == NULL) { - _PyFrame_SetStackPointer(frame, stack_pointer); - int matches = _PyErr_ExceptionMatches(tstate, PyExc_StopIteration); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (matches) { - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyEval_MonitorRaise(tstate, frame, this_instr); - stack_pointer = _PyFrame_GetStackPointer(frame); - } - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = _PyGen_FetchStopIterationValue(&retval_o); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err == 0) { - assert(retval_o != NULL); - JUMPBY(oparg); - RECORD_DYNAMIC_JUMP_TAKEN(); - } - else { - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(v); - stack_pointer = _PyFrame_GetStackPointer(frame); - TRACING_JUMP_TO_LABEL(error); - } - } - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(v); - stack_pointer = _PyFrame_GetStackPointer(frame); - retval = PyStackRef_FromPyObjectSteal(retval_o); - } - stack_pointer[0] = retval; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - TRACING_DISPATCH(); - } - - TRACING_TARGET(SEND_GEN) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = SEND_GEN; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(SEND_GEN); - opcode = SEND_GEN; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; - (void)(_old_stack_level); - static_assert(INLINE_CACHE_ENTRIES_SEND == 1, "incorrect cache size"); - _PyStackRef receiver; - _PyStackRef v; - _PyStackRef gen_frame; - _PyStackRef new_frame; - /* Skip 1 cache entry */ - // _CHECK_PEP_523 - { - if (tstate->interp->eval_frame) { - UPDATE_MISS_STATS(SEND); - assert(_PyOpcode_Deopt[opcode] == (SEND)); - JUMP_TO_PREDICTED(TRACING_SEND); - } - } - // _SEND_GEN_FRAME - { - v = stack_pointer[-1]; - receiver = stack_pointer[-2]; - PyGenObject *gen = (PyGenObject *)PyStackRef_AsPyObjectBorrow(receiver); - if (Py_TYPE(gen) != &PyGen_Type && Py_TYPE(gen) != &PyCoro_Type) { - UPDATE_MISS_STATS(SEND); - assert(_PyOpcode_Deopt[opcode] == (SEND)); - JUMP_TO_PREDICTED(TRACING_SEND); - } - if (gen->gi_frame_state >= FRAME_EXECUTING) { - UPDATE_MISS_STATS(SEND); - assert(_PyOpcode_Deopt[opcode] == (SEND)); - JUMP_TO_PREDICTED(TRACING_SEND); - } - STAT_INC(SEND, hit); - _PyInterpreterFrame *pushed_frame = &gen->gi_iframe; - _PyFrame_StackPush(pushed_frame, PyStackRef_MakeHeapSafe(v)); - gen->gi_frame_state = FRAME_EXECUTING; - gen->gi_exc_state.previous_item = tstate->exc_info; - tstate->exc_info = &gen->gi_exc_state; - assert( 2u + oparg <= UINT16_MAX); - frame->return_offset = (uint16_t)( 2u + oparg); - pushed_frame->previous = frame; - gen_frame = PyStackRef_Wrap(pushed_frame); - } - // _PUSH_FRAME - { - new_frame = gen_frame; - assert(tstate->interp->eval_frame == NULL); - _PyInterpreterFrame *temp = PyStackRef_Unwrap(new_frame); - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - assert(temp->previous == frame || temp->previous->previous == frame); - CALL_STAT_INC(inlined_py_calls); - frame = tstate->current_frame = temp; - tstate->py_recursion_remaining--; - LOAD_SP(); - LOAD_IP(0); - LLTRACE_RESUME_FRAME(); - } - TRACING_DISPATCH(); - } - - TRACING_TARGET(SETUP_ANNOTATIONS) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = SETUP_ANNOTATIONS; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(SETUP_ANNOTATIONS); - opcode = SETUP_ANNOTATIONS; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; - (void)(_old_stack_level); - PyObject *ann_dict; - if (LOCALS() == NULL) { - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyErr_Format(tstate, PyExc_SystemError, - "no locals found when setting up annotations"); - stack_pointer = _PyFrame_GetStackPointer(frame); - TRACING_JUMP_TO_LABEL(error); - } - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = PyMapping_GetOptionalItem(LOCALS(), &_Py_ID(__annotations__), &ann_dict); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err < 0) { - TRACING_JUMP_TO_LABEL(error); - } - if (ann_dict == NULL) { - _PyFrame_SetStackPointer(frame, stack_pointer); - ann_dict = PyDict_New(); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (ann_dict == NULL) { - TRACING_JUMP_TO_LABEL(error); - } - _PyFrame_SetStackPointer(frame, stack_pointer); - err = PyObject_SetItem(LOCALS(), &_Py_ID(__annotations__), - ann_dict); - Py_DECREF(ann_dict); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err) { - TRACING_JUMP_TO_LABEL(error); - } - } - else { - _PyFrame_SetStackPointer(frame, stack_pointer); - Py_DECREF(ann_dict); - stack_pointer = _PyFrame_GetStackPointer(frame); - } - TRACING_DISPATCH(); - } - - TRACING_TARGET(SET_ADD) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = SET_ADD; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(SET_ADD); - opcode = SET_ADD; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; - (void)(_old_stack_level); - _PyStackRef set; - _PyStackRef v; - v = stack_pointer[-1]; - set = stack_pointer[-2 - (oparg-1)]; - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = _PySet_AddTakeRef((PySetObject *)PyStackRef_AsPyObjectBorrow(set), - PyStackRef_AsPyObjectSteal(v)); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err) { - TRACING_JUMP_TO_LABEL(pop_1_error); - } - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - TRACING_DISPATCH(); - } - - TRACING_TARGET(SET_FUNCTION_ATTRIBUTE) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = SET_FUNCTION_ATTRIBUTE; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(SET_FUNCTION_ATTRIBUTE); - opcode = SET_FUNCTION_ATTRIBUTE; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; - (void)(_old_stack_level); - _PyStackRef attr_st; - _PyStackRef func_in; - _PyStackRef func_out; - func_in = stack_pointer[-1]; - attr_st = stack_pointer[-2]; - PyObject *func = PyStackRef_AsPyObjectBorrow(func_in); - PyObject *attr = PyStackRef_AsPyObjectSteal(attr_st); - func_out = func_in; - assert(PyFunction_Check(func)); - size_t offset = _Py_FunctionAttributeOffsets[oparg]; - assert(offset != 0); - PyObject **ptr = (PyObject **)(((char *)func) + offset); - assert(*ptr == NULL); - *ptr = attr; - stack_pointer[-2] = func_out; - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - TRACING_DISPATCH(); - } - - TRACING_TARGET(SET_UPDATE) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = SET_UPDATE; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(SET_UPDATE); - opcode = SET_UPDATE; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; - (void)(_old_stack_level); - _PyStackRef set; - _PyStackRef iterable; - iterable = stack_pointer[-1]; - set = stack_pointer[-2 - (oparg-1)]; - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = _PySet_Update(PyStackRef_AsPyObjectBorrow(set), - PyStackRef_AsPyObjectBorrow(iterable)); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(iterable); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err < 0) { - TRACING_JUMP_TO_LABEL(error); - } - TRACING_DISPATCH(); - } - - TRACING_TARGET(STORE_ATTR) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = STORE_ATTR; - (void)(opcode); - #endif - frame->instr_ptr = next_instr; - next_instr += 5; - INSTRUCTION_STATS(STORE_ATTR); - PREDICTED_TRACING_STORE_ATTR:; - _Py_CODEUNIT* const this_instr = next_instr - 5; - (void)this_instr; - opcode = STORE_ATTR; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; - (void)(_old_stack_level); - _PyStackRef owner; - _PyStackRef v; - // _SPECIALIZE_STORE_ATTR - { - owner = stack_pointer[-1]; - uint16_t counter = read_u16(&this_instr[1].cache); - (void)counter; - #if ENABLE_SPECIALIZATION_FT - if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { - PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); - next_instr = this_instr; - _PyFrame_SetStackPointer(frame, stack_pointer); - _Py_Specialize_StoreAttr(owner, next_instr, name); - stack_pointer = _PyFrame_GetStackPointer(frame); - TRACING_SPECIALIZE_DISPATCH_SAME_OPARG() - } - OPCODE_DEFERRED_INC(STORE_ATTR); - ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); - #endif /* ENABLE_SPECIALIZATION_FT */ - } - /* Skip 3 cache entries */ - // _STORE_ATTR - { - v = stack_pointer[-2]; - PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = PyObject_SetAttr(PyStackRef_AsPyObjectBorrow(owner), - name, PyStackRef_AsPyObjectBorrow(v)); - _PyStackRef tmp = owner; - owner = PyStackRef_NULL; - stack_pointer[-1] = owner; - PyStackRef_CLOSE(tmp); - tmp = v; - v = PyStackRef_NULL; - stack_pointer[-2] = v; - PyStackRef_CLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); - if (err) { - TRACING_JUMP_TO_LABEL(error); - } - } - TRACING_DISPATCH(); - } - - TRACING_TARGET(STORE_ATTR_INSTANCE_VALUE) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = STORE_ATTR_INSTANCE_VALUE; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 5; - INSTRUCTION_STATS(STORE_ATTR_INSTANCE_VALUE); - opcode = STORE_ATTR_INSTANCE_VALUE; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; - (void)(_old_stack_level); - static_assert(INLINE_CACHE_ENTRIES_STORE_ATTR == 4, "incorrect cache size"); - _PyStackRef owner; - _PyStackRef value; - /* Skip 1 cache entry */ - // _GUARD_TYPE_VERSION_AND_LOCK - { - owner = stack_pointer[-1]; - uint32_t type_version = read_u32(&this_instr[2].cache); - PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); - assert(type_version != 0); - if (!LOCK_OBJECT(owner_o)) { - UPDATE_MISS_STATS(STORE_ATTR); - assert(_PyOpcode_Deopt[opcode] == (STORE_ATTR)); - JUMP_TO_PREDICTED(TRACING_STORE_ATTR); - } - PyTypeObject *tp = Py_TYPE(owner_o); - if (FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version) { - UNLOCK_OBJECT(owner_o); - if (true) { - UPDATE_MISS_STATS(STORE_ATTR); - assert(_PyOpcode_Deopt[opcode] == (STORE_ATTR)); - JUMP_TO_PREDICTED(TRACING_STORE_ATTR); - } - } - } - // _GUARD_DORV_NO_DICT - { - PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); - assert(Py_TYPE(owner_o)->tp_dictoffset < 0); - assert(Py_TYPE(owner_o)->tp_flags & Py_TPFLAGS_INLINE_VALUES); - if (_PyObject_GetManagedDict(owner_o) || - !FT_ATOMIC_LOAD_UINT8(_PyObject_InlineValues(owner_o)->valid)) { - UNLOCK_OBJECT(owner_o); - if (true) { - UPDATE_MISS_STATS(STORE_ATTR); - assert(_PyOpcode_Deopt[opcode] == (STORE_ATTR)); - JUMP_TO_PREDICTED(TRACING_STORE_ATTR); - } - } - } - // _STORE_ATTR_INSTANCE_VALUE - { - value = stack_pointer[-2]; - uint16_t offset = read_u16(&this_instr[4].cache); - PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); - STAT_INC(STORE_ATTR, hit); - assert(_PyObject_GetManagedDict(owner_o) == NULL); - PyObject **value_ptr = (PyObject**)(((char *)owner_o) + offset); - PyObject *old_value = *value_ptr; - FT_ATOMIC_STORE_PTR_RELEASE(*value_ptr, PyStackRef_AsPyObjectSteal(value)); - if (old_value == NULL) { - PyDictValues *values = _PyObject_InlineValues(owner_o); - Py_ssize_t index = value_ptr - values->values; - _PyDictValues_AddToInsertionOrder(values, index); - } - UNLOCK_OBJECT(owner_o); - stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(owner); - Py_XDECREF(old_value); - stack_pointer = _PyFrame_GetStackPointer(frame); - } - TRACING_DISPATCH(); - } - - TRACING_TARGET(STORE_ATTR_SLOT) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = STORE_ATTR_SLOT; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 5; - INSTRUCTION_STATS(STORE_ATTR_SLOT); - opcode = STORE_ATTR_SLOT; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; - (void)(_old_stack_level); - static_assert(INLINE_CACHE_ENTRIES_STORE_ATTR == 4, "incorrect cache size"); - _PyStackRef owner; - _PyStackRef value; - /* Skip 1 cache entry */ - // _GUARD_TYPE_VERSION - { - owner = stack_pointer[-1]; - uint32_t type_version = read_u32(&this_instr[2].cache); - PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); - assert(type_version != 0); - if (FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version) { - UPDATE_MISS_STATS(STORE_ATTR); - assert(_PyOpcode_Deopt[opcode] == (STORE_ATTR)); - JUMP_TO_PREDICTED(TRACING_STORE_ATTR); - } - } - // _STORE_ATTR_SLOT - { - value = stack_pointer[-2]; - uint16_t index = read_u16(&this_instr[4].cache); - PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); - if (!LOCK_OBJECT(owner_o)) { - UPDATE_MISS_STATS(STORE_ATTR); - assert(_PyOpcode_Deopt[opcode] == (STORE_ATTR)); - JUMP_TO_PREDICTED(TRACING_STORE_ATTR); - } - char *addr = (char *)owner_o + index; - STAT_INC(STORE_ATTR, hit); - PyObject *old_value = *(PyObject **)addr; - FT_ATOMIC_STORE_PTR_RELEASE(*(PyObject **)addr, PyStackRef_AsPyObjectSteal(value)); - UNLOCK_OBJECT(owner_o); - stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(owner); - Py_XDECREF(old_value); - stack_pointer = _PyFrame_GetStackPointer(frame); - } - TRACING_DISPATCH(); - } - - TRACING_TARGET(STORE_ATTR_WITH_HINT) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = STORE_ATTR_WITH_HINT; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 5; - INSTRUCTION_STATS(STORE_ATTR_WITH_HINT); - opcode = STORE_ATTR_WITH_HINT; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; - (void)(_old_stack_level); - static_assert(INLINE_CACHE_ENTRIES_STORE_ATTR == 4, "incorrect cache size"); - _PyStackRef owner; - _PyStackRef value; - /* Skip 1 cache entry */ - // _GUARD_TYPE_VERSION - { - owner = stack_pointer[-1]; - uint32_t type_version = read_u32(&this_instr[2].cache); - PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); - assert(type_version != 0); - if (FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version) { - UPDATE_MISS_STATS(STORE_ATTR); - assert(_PyOpcode_Deopt[opcode] == (STORE_ATTR)); - JUMP_TO_PREDICTED(TRACING_STORE_ATTR); - } - } - // _STORE_ATTR_WITH_HINT - { - value = stack_pointer[-2]; - uint16_t hint = read_u16(&this_instr[4].cache); - PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); - assert(Py_TYPE(owner_o)->tp_flags & Py_TPFLAGS_MANAGED_DICT); - PyDictObject *dict = _PyObject_GetManagedDict(owner_o); - if (dict == NULL) { - UPDATE_MISS_STATS(STORE_ATTR); - assert(_PyOpcode_Deopt[opcode] == (STORE_ATTR)); - JUMP_TO_PREDICTED(TRACING_STORE_ATTR); - } - if (!LOCK_OBJECT(dict)) { - UPDATE_MISS_STATS(STORE_ATTR); - assert(_PyOpcode_Deopt[opcode] == (STORE_ATTR)); - JUMP_TO_PREDICTED(TRACING_STORE_ATTR); - } - assert(PyDict_CheckExact((PyObject *)dict)); - PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); - if (hint >= (size_t)dict->ma_keys->dk_nentries || - !DK_IS_UNICODE(dict->ma_keys)) { - UNLOCK_OBJECT(dict); - if (true) { - UPDATE_MISS_STATS(STORE_ATTR); - assert(_PyOpcode_Deopt[opcode] == (STORE_ATTR)); - JUMP_TO_PREDICTED(TRACING_STORE_ATTR); - } - } - PyDictUnicodeEntry *ep = DK_UNICODE_ENTRIES(dict->ma_keys) + hint; - if (ep->me_key != name) { - UNLOCK_OBJECT(dict); - if (true) { - UPDATE_MISS_STATS(STORE_ATTR); - assert(_PyOpcode_Deopt[opcode] == (STORE_ATTR)); - JUMP_TO_PREDICTED(TRACING_STORE_ATTR); - } - } - PyObject *old_value = ep->me_value; - if (old_value == NULL) { - UNLOCK_OBJECT(dict); - if (true) { - UPDATE_MISS_STATS(STORE_ATTR); - assert(_PyOpcode_Deopt[opcode] == (STORE_ATTR)); - JUMP_TO_PREDICTED(TRACING_STORE_ATTR); - } - } - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyDict_NotifyEvent(tstate->interp, PyDict_EVENT_MODIFIED, dict, name, PyStackRef_AsPyObjectBorrow(value)); - stack_pointer = _PyFrame_GetStackPointer(frame); - FT_ATOMIC_STORE_PTR_RELEASE(ep->me_value, PyStackRef_AsPyObjectSteal(value)); - UNLOCK_OBJECT(dict); - STAT_INC(STORE_ATTR, hit); - stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(owner); - Py_XDECREF(old_value); - stack_pointer = _PyFrame_GetStackPointer(frame); - } - TRACING_DISPATCH(); - } - - TRACING_TARGET(STORE_DEREF) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = STORE_DEREF; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(STORE_DEREF); - opcode = STORE_DEREF; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; - (void)(_old_stack_level); - _PyStackRef v; - v = stack_pointer[-1]; - PyCellObject *cell = (PyCellObject *)PyStackRef_AsPyObjectBorrow(GETLOCAL(oparg)); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyCell_SetTakeRef(cell, PyStackRef_AsPyObjectSteal(v)); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - TRACING_DISPATCH(); - } - - TRACING_TARGET(STORE_FAST) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = STORE_FAST; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(STORE_FAST); - opcode = STORE_FAST; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; - (void)(_old_stack_level); - _PyStackRef value; - value = stack_pointer[-1]; - _PyStackRef tmp = GETLOCAL(oparg); - GETLOCAL(oparg) = value; - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_XCLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); - TRACING_DISPATCH(); - } - - TRACING_TARGET(STORE_FAST_LOAD_FAST) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = STORE_FAST_LOAD_FAST; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(STORE_FAST_LOAD_FAST); - opcode = STORE_FAST_LOAD_FAST; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; - (void)(_old_stack_level); - _PyStackRef value1; - _PyStackRef value2; - value1 = stack_pointer[-1]; - uint32_t oparg1 = oparg >> 4; - uint32_t oparg2 = oparg & 15; - _PyStackRef tmp = GETLOCAL(oparg1); - GETLOCAL(oparg1) = value1; - value2 = PyStackRef_DUP(GETLOCAL(oparg2)); - stack_pointer[-1] = value2; - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_XCLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); - TRACING_DISPATCH(); - } - - TRACING_TARGET(STORE_FAST_STORE_FAST) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = STORE_FAST_STORE_FAST; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(STORE_FAST_STORE_FAST); - opcode = STORE_FAST_STORE_FAST; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; - (void)(_old_stack_level); - _PyStackRef value2; - _PyStackRef value1; - value1 = stack_pointer[-1]; - value2 = stack_pointer[-2]; - uint32_t oparg1 = oparg >> 4; - uint32_t oparg2 = oparg & 15; - _PyStackRef tmp = GETLOCAL(oparg1); - GETLOCAL(oparg1) = value1; - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_XCLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); - tmp = GETLOCAL(oparg2); - GETLOCAL(oparg2) = value2; - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_XCLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); - TRACING_DISPATCH(); - } - - TRACING_TARGET(STORE_GLOBAL) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = STORE_GLOBAL; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(STORE_GLOBAL); - opcode = STORE_GLOBAL; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; - (void)(_old_stack_level); - _PyStackRef v; - v = stack_pointer[-1]; - PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = PyDict_SetItem(GLOBALS(), name, PyStackRef_AsPyObjectBorrow(v)); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(v); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err) { - TRACING_JUMP_TO_LABEL(error); - } - TRACING_DISPATCH(); - } - - TRACING_TARGET(STORE_NAME) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = STORE_NAME; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(STORE_NAME); - opcode = STORE_NAME; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; - (void)(_old_stack_level); - _PyStackRef v; - v = stack_pointer[-1]; - PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); - PyObject *ns = LOCALS(); - int err; - if (ns == NULL) { - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyErr_Format(tstate, PyExc_SystemError, - "no locals found when storing %R", name); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(v); - stack_pointer = _PyFrame_GetStackPointer(frame); - TRACING_JUMP_TO_LABEL(error); - } - if (PyDict_CheckExact(ns)) { - _PyFrame_SetStackPointer(frame, stack_pointer); - err = PyDict_SetItem(ns, name, PyStackRef_AsPyObjectBorrow(v)); - stack_pointer = _PyFrame_GetStackPointer(frame); - } - else { - _PyFrame_SetStackPointer(frame, stack_pointer); - err = PyObject_SetItem(ns, name, PyStackRef_AsPyObjectBorrow(v)); - stack_pointer = _PyFrame_GetStackPointer(frame); - } - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(v); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err) { - TRACING_JUMP_TO_LABEL(error); - } - TRACING_DISPATCH(); - } - - TRACING_TARGET(STORE_SLICE) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = STORE_SLICE; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(STORE_SLICE); - opcode = STORE_SLICE; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; - (void)(_old_stack_level); - _PyStackRef v; - _PyStackRef container; - _PyStackRef start; - _PyStackRef stop; - // _SPECIALIZE_STORE_SLICE - { - #if ENABLE_SPECIALIZATION - OPCODE_DEFERRED_INC(STORE_SLICE); - #endif /* ENABLE_SPECIALIZATION */ - } - // _STORE_SLICE - { - stop = stack_pointer[-1]; - start = stack_pointer[-2]; - container = stack_pointer[-3]; - v = stack_pointer[-4]; - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *slice = _PyBuildSlice_ConsumeRefs(PyStackRef_AsPyObjectSteal(start), - PyStackRef_AsPyObjectSteal(stop)); - stack_pointer = _PyFrame_GetStackPointer(frame); - int err; - if (slice == NULL) { - err = 1; - } - else { - stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - err = PyObject_SetItem(PyStackRef_AsPyObjectBorrow(container), slice, PyStackRef_AsPyObjectBorrow(v)); - Py_DECREF(slice); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += 2; - } - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyStackRef tmp = container; - container = PyStackRef_NULL; - stack_pointer[-3] = container; - PyStackRef_CLOSE(tmp); - tmp = v; - v = PyStackRef_NULL; - stack_pointer[-4] = v; - PyStackRef_CLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -4; - assert(WITHIN_STACK_BOUNDS()); - if (err) { - TRACING_JUMP_TO_LABEL(error); - } - } - TRACING_DISPATCH(); - } - - TRACING_TARGET(STORE_SUBSCR) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = STORE_SUBSCR; - (void)(opcode); - #endif - frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(STORE_SUBSCR); - PREDICTED_TRACING_STORE_SUBSCR:; - _Py_CODEUNIT* const this_instr = next_instr - 2; - (void)this_instr; - opcode = STORE_SUBSCR; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; - (void)(_old_stack_level); - _PyStackRef container; - _PyStackRef sub; - _PyStackRef v; - // _SPECIALIZE_STORE_SUBSCR - { - sub = stack_pointer[-1]; - container = stack_pointer[-2]; - uint16_t counter = read_u16(&this_instr[1].cache); - (void)counter; - #if ENABLE_SPECIALIZATION_FT - if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { - next_instr = this_instr; - _PyFrame_SetStackPointer(frame, stack_pointer); - _Py_Specialize_StoreSubscr(container, sub, next_instr); - stack_pointer = _PyFrame_GetStackPointer(frame); - TRACING_SPECIALIZE_DISPATCH_SAME_OPARG() - } - OPCODE_DEFERRED_INC(STORE_SUBSCR); - ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); - #endif /* ENABLE_SPECIALIZATION_FT */ - } - // _STORE_SUBSCR - { - v = stack_pointer[-3]; - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = PyObject_SetItem(PyStackRef_AsPyObjectBorrow(container), PyStackRef_AsPyObjectBorrow(sub), PyStackRef_AsPyObjectBorrow(v)); - _PyStackRef tmp = sub; - sub = PyStackRef_NULL; - stack_pointer[-1] = sub; - PyStackRef_CLOSE(tmp); - tmp = container; - container = PyStackRef_NULL; - stack_pointer[-2] = container; - PyStackRef_CLOSE(tmp); - tmp = v; - v = PyStackRef_NULL; - stack_pointer[-3] = v; - PyStackRef_CLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -3; - assert(WITHIN_STACK_BOUNDS()); - if (err) { - TRACING_JUMP_TO_LABEL(error); - } - } - TRACING_DISPATCH(); - } - - TRACING_TARGET(STORE_SUBSCR_DICT) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = STORE_SUBSCR_DICT; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(STORE_SUBSCR_DICT); - opcode = STORE_SUBSCR_DICT; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; - (void)(_old_stack_level); - static_assert(INLINE_CACHE_ENTRIES_STORE_SUBSCR == 1, "incorrect cache size"); - _PyStackRef nos; - _PyStackRef value; - _PyStackRef dict_st; - _PyStackRef sub; - // _GUARD_NOS_DICT - { - nos = stack_pointer[-2]; - PyObject *o = PyStackRef_AsPyObjectBorrow(nos); - if (!PyDict_CheckExact(o)) { - UPDATE_MISS_STATS(STORE_SUBSCR); - assert(_PyOpcode_Deopt[opcode] == (STORE_SUBSCR)); - JUMP_TO_PREDICTED(TRACING_STORE_SUBSCR); - } - } - /* Skip 1 cache entry */ - // _STORE_SUBSCR_DICT - { - sub = stack_pointer[-1]; - dict_st = nos; - value = stack_pointer[-3]; - PyObject *dict = PyStackRef_AsPyObjectBorrow(dict_st); - assert(PyDict_CheckExact(dict)); - STAT_INC(STORE_SUBSCR, hit); - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = _PyDict_SetItem_Take2((PyDictObject *)dict, - PyStackRef_AsPyObjectSteal(sub), - PyStackRef_AsPyObjectSteal(value)); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -3; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(dict_st); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err) { - TRACING_JUMP_TO_LABEL(error); - } - } - TRACING_DISPATCH(); - } - - TRACING_TARGET(STORE_SUBSCR_LIST_INT) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = STORE_SUBSCR_LIST_INT; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(STORE_SUBSCR_LIST_INT); - opcode = STORE_SUBSCR_LIST_INT; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; - (void)(_old_stack_level); - static_assert(INLINE_CACHE_ENTRIES_STORE_SUBSCR == 1, "incorrect cache size"); - _PyStackRef value; - _PyStackRef nos; - _PyStackRef list_st; - _PyStackRef sub_st; - // _GUARD_TOS_INT - { - value = stack_pointer[-1]; - PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); - if (!_PyLong_CheckExactAndCompact(value_o)) { - UPDATE_MISS_STATS(STORE_SUBSCR); - assert(_PyOpcode_Deopt[opcode] == (STORE_SUBSCR)); - JUMP_TO_PREDICTED(TRACING_STORE_SUBSCR); - } - } - // _GUARD_NOS_LIST - { - nos = stack_pointer[-2]; - PyObject *o = PyStackRef_AsPyObjectBorrow(nos); - if (!PyList_CheckExact(o)) { - UPDATE_MISS_STATS(STORE_SUBSCR); - assert(_PyOpcode_Deopt[opcode] == (STORE_SUBSCR)); - JUMP_TO_PREDICTED(TRACING_STORE_SUBSCR); - } - } - /* Skip 1 cache entry */ - // _STORE_SUBSCR_LIST_INT - { - sub_st = value; - list_st = nos; - value = stack_pointer[-3]; - PyObject *sub = PyStackRef_AsPyObjectBorrow(sub_st); - PyObject *list = PyStackRef_AsPyObjectBorrow(list_st); - assert(PyLong_CheckExact(sub)); - assert(PyList_CheckExact(list)); - if (!_PyLong_IsNonNegativeCompact((PyLongObject *)sub)) { - UPDATE_MISS_STATS(STORE_SUBSCR); - assert(_PyOpcode_Deopt[opcode] == (STORE_SUBSCR)); - JUMP_TO_PREDICTED(TRACING_STORE_SUBSCR); - } - Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0]; - if (!LOCK_OBJECT(list)) { - UPDATE_MISS_STATS(STORE_SUBSCR); - assert(_PyOpcode_Deopt[opcode] == (STORE_SUBSCR)); - JUMP_TO_PREDICTED(TRACING_STORE_SUBSCR); - } - if (index >= PyList_GET_SIZE(list)) { - UNLOCK_OBJECT(list); - if (true) { - UPDATE_MISS_STATS(STORE_SUBSCR); - assert(_PyOpcode_Deopt[opcode] == (STORE_SUBSCR)); - JUMP_TO_PREDICTED(TRACING_STORE_SUBSCR); - } - } - STAT_INC(STORE_SUBSCR, hit); - PyObject *old_value = PyList_GET_ITEM(list, index); - FT_ATOMIC_STORE_PTR_RELEASE(_PyList_ITEMS(list)[index], - PyStackRef_AsPyObjectSteal(value)); - assert(old_value != NULL); - UNLOCK_OBJECT(list); - PyStackRef_CLOSE_SPECIALIZED(sub_st, _PyLong_ExactDealloc); - stack_pointer += -3; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(list_st); - Py_DECREF(old_value); - stack_pointer = _PyFrame_GetStackPointer(frame); - } - TRACING_DISPATCH(); - } - - TRACING_TARGET(SWAP) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = SWAP; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(SWAP); - opcode = SWAP; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; - (void)(_old_stack_level); - _PyStackRef bottom; - _PyStackRef top; - top = stack_pointer[-1]; - bottom = stack_pointer[-2 - (oparg-2)]; - _PyStackRef temp = bottom; - bottom = top; - top = temp; - stack_pointer[-2 - (oparg-2)] = bottom; - stack_pointer[-1] = top; - TRACING_DISPATCH(); - } - - TRACING_TARGET(TO_BOOL) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = TO_BOOL; - (void)(opcode); - #endif - frame->instr_ptr = next_instr; - next_instr += 4; - INSTRUCTION_STATS(TO_BOOL); - PREDICTED_TRACING_TO_BOOL:; - _Py_CODEUNIT* const this_instr = next_instr - 4; - (void)this_instr; - opcode = TO_BOOL; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; - (void)(_old_stack_level); - _PyStackRef value; - _PyStackRef res; - // _SPECIALIZE_TO_BOOL - { - value = stack_pointer[-1]; - uint16_t counter = read_u16(&this_instr[1].cache); - (void)counter; - #if ENABLE_SPECIALIZATION_FT - if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { - next_instr = this_instr; - _PyFrame_SetStackPointer(frame, stack_pointer); - _Py_Specialize_ToBool(value, next_instr); - stack_pointer = _PyFrame_GetStackPointer(frame); - TRACING_SPECIALIZE_DISPATCH_SAME_OPARG() - } - OPCODE_DEFERRED_INC(TO_BOOL); - ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); - #endif /* ENABLE_SPECIALIZATION_FT */ - } - /* Skip 2 cache entries */ - // _TO_BOOL - { - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = PyObject_IsTrue(PyStackRef_AsPyObjectBorrow(value)); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(value); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err < 0) { - TRACING_JUMP_TO_LABEL(error); - } - res = err ? PyStackRef_True : PyStackRef_False; - } - stack_pointer[0] = res; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - TRACING_DISPATCH(); - } - - TRACING_TARGET(TO_BOOL_ALWAYS_TRUE) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = TO_BOOL_ALWAYS_TRUE; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 4; - INSTRUCTION_STATS(TO_BOOL_ALWAYS_TRUE); - opcode = TO_BOOL_ALWAYS_TRUE; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; - (void)(_old_stack_level); - static_assert(INLINE_CACHE_ENTRIES_TO_BOOL == 3, "incorrect cache size"); - _PyStackRef owner; - _PyStackRef value; - _PyStackRef res; - /* Skip 1 cache entry */ - // _GUARD_TYPE_VERSION - { - owner = stack_pointer[-1]; - uint32_t type_version = read_u32(&this_instr[2].cache); - PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); - assert(type_version != 0); - if (FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version) { - UPDATE_MISS_STATS(TO_BOOL); - assert(_PyOpcode_Deopt[opcode] == (TO_BOOL)); - JUMP_TO_PREDICTED(TRACING_TO_BOOL); - } - } - // _REPLACE_WITH_TRUE - { - value = owner; - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(value); - stack_pointer = _PyFrame_GetStackPointer(frame); - res = PyStackRef_True; - } - stack_pointer[0] = res; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - TRACING_DISPATCH(); - } - - TRACING_TARGET(TO_BOOL_BOOL) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = TO_BOOL_BOOL; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 4; - INSTRUCTION_STATS(TO_BOOL_BOOL); - opcode = TO_BOOL_BOOL; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; - (void)(_old_stack_level); - static_assert(INLINE_CACHE_ENTRIES_TO_BOOL == 3, "incorrect cache size"); - _PyStackRef value; - /* Skip 1 cache entry */ - /* Skip 2 cache entries */ - value = stack_pointer[-1]; - if (!PyStackRef_BoolCheck(value)) { - UPDATE_MISS_STATS(TO_BOOL); - assert(_PyOpcode_Deopt[opcode] == (TO_BOOL)); - JUMP_TO_PREDICTED(TRACING_TO_BOOL); - } - STAT_INC(TO_BOOL, hit); - TRACING_DISPATCH(); - } - - TRACING_TARGET(TO_BOOL_INT) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = TO_BOOL_INT; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 4; - INSTRUCTION_STATS(TO_BOOL_INT); - opcode = TO_BOOL_INT; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; - (void)(_old_stack_level); - static_assert(INLINE_CACHE_ENTRIES_TO_BOOL == 3, "incorrect cache size"); - _PyStackRef value; - _PyStackRef res; - /* Skip 1 cache entry */ - /* Skip 2 cache entries */ - value = stack_pointer[-1]; - PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); - if (!PyLong_CheckExact(value_o)) { - UPDATE_MISS_STATS(TO_BOOL); - assert(_PyOpcode_Deopt[opcode] == (TO_BOOL)); - JUMP_TO_PREDICTED(TRACING_TO_BOOL); - } - STAT_INC(TO_BOOL, hit); - if (_PyLong_IsZero((PyLongObject *)value_o)) { - assert(_Py_IsImmortal(value_o)); - res = PyStackRef_False; - } - else { - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(value); - stack_pointer = _PyFrame_GetStackPointer(frame); - res = PyStackRef_True; - stack_pointer += 1; - } - stack_pointer[-1] = res; - TRACING_DISPATCH(); - } - - TRACING_TARGET(TO_BOOL_LIST) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = TO_BOOL_LIST; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 4; - INSTRUCTION_STATS(TO_BOOL_LIST); - opcode = TO_BOOL_LIST; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; - (void)(_old_stack_level); - static_assert(INLINE_CACHE_ENTRIES_TO_BOOL == 3, "incorrect cache size"); - _PyStackRef tos; - _PyStackRef value; - _PyStackRef res; - // _GUARD_TOS_LIST - { - tos = stack_pointer[-1]; - PyObject *o = PyStackRef_AsPyObjectBorrow(tos); - if (!PyList_CheckExact(o)) { - UPDATE_MISS_STATS(TO_BOOL); - assert(_PyOpcode_Deopt[opcode] == (TO_BOOL)); - JUMP_TO_PREDICTED(TRACING_TO_BOOL); - } - } - /* Skip 1 cache entry */ - /* Skip 2 cache entries */ - // _TO_BOOL_LIST - { - value = tos; - PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); - assert(PyList_CheckExact(value_o)); - STAT_INC(TO_BOOL, hit); - res = PyList_GET_SIZE(value_o) ? PyStackRef_True : PyStackRef_False; - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyStackRef tmp = value; - value = res; - stack_pointer[-1] = value; - PyStackRef_CLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); - } - TRACING_DISPATCH(); - } - - TRACING_TARGET(TO_BOOL_NONE) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = TO_BOOL_NONE; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 4; - INSTRUCTION_STATS(TO_BOOL_NONE); - opcode = TO_BOOL_NONE; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; - (void)(_old_stack_level); - static_assert(INLINE_CACHE_ENTRIES_TO_BOOL == 3, "incorrect cache size"); - _PyStackRef value; - _PyStackRef res; - /* Skip 1 cache entry */ - /* Skip 2 cache entries */ - value = stack_pointer[-1]; - if (!PyStackRef_IsNone(value)) { - UPDATE_MISS_STATS(TO_BOOL); - assert(_PyOpcode_Deopt[opcode] == (TO_BOOL)); - JUMP_TO_PREDICTED(TRACING_TO_BOOL); - } - STAT_INC(TO_BOOL, hit); - res = PyStackRef_False; - stack_pointer[-1] = res; - TRACING_DISPATCH(); - } - - TRACING_TARGET(TO_BOOL_STR) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = TO_BOOL_STR; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 4; - INSTRUCTION_STATS(TO_BOOL_STR); - opcode = TO_BOOL_STR; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; - (void)(_old_stack_level); - static_assert(INLINE_CACHE_ENTRIES_TO_BOOL == 3, "incorrect cache size"); - _PyStackRef value; - _PyStackRef res; - // _GUARD_TOS_UNICODE - { - value = stack_pointer[-1]; - PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); - if (!PyUnicode_CheckExact(value_o)) { - UPDATE_MISS_STATS(TO_BOOL); - assert(_PyOpcode_Deopt[opcode] == (TO_BOOL)); - JUMP_TO_PREDICTED(TRACING_TO_BOOL); - } - } - /* Skip 1 cache entry */ - /* Skip 2 cache entries */ - // _TO_BOOL_STR - { - STAT_INC(TO_BOOL, hit); - PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); - if (value_o == &_Py_STR(empty)) { - assert(_Py_IsImmortal(value_o)); - res = PyStackRef_False; - } - else { - assert(Py_SIZE(value_o)); - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(value); - stack_pointer = _PyFrame_GetStackPointer(frame); - res = PyStackRef_True; - stack_pointer += 1; - } - } - stack_pointer[-1] = res; - TRACING_DISPATCH(); - } - - TRACING_TARGET(UNARY_INVERT) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = UNARY_INVERT; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(UNARY_INVERT); - opcode = UNARY_INVERT; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; - (void)(_old_stack_level); - _PyStackRef value; - _PyStackRef res; - value = stack_pointer[-1]; - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *res_o = PyNumber_Invert(PyStackRef_AsPyObjectBorrow(value)); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(value); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (res_o == NULL) { - TRACING_JUMP_TO_LABEL(error); - } - res = PyStackRef_FromPyObjectSteal(res_o); - stack_pointer[0] = res; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - TRACING_DISPATCH(); - } - - TRACING_TARGET(UNARY_NEGATIVE) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = UNARY_NEGATIVE; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(UNARY_NEGATIVE); - opcode = UNARY_NEGATIVE; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; - (void)(_old_stack_level); - _PyStackRef value; - _PyStackRef res; - value = stack_pointer[-1]; - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *res_o = PyNumber_Negative(PyStackRef_AsPyObjectBorrow(value)); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(value); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (res_o == NULL) { - TRACING_JUMP_TO_LABEL(error); - } - res = PyStackRef_FromPyObjectSteal(res_o); - stack_pointer[0] = res; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - TRACING_DISPATCH(); - } - - TRACING_TARGET(UNARY_NOT) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = UNARY_NOT; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(UNARY_NOT); - opcode = UNARY_NOT; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; - (void)(_old_stack_level); - _PyStackRef value; - _PyStackRef res; - value = stack_pointer[-1]; - assert(PyStackRef_BoolCheck(value)); - res = PyStackRef_IsFalse(value) - ? PyStackRef_True : PyStackRef_False; - stack_pointer[-1] = res; - TRACING_DISPATCH(); - } - - TRACING_TARGET(UNPACK_EX) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = UNPACK_EX; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(UNPACK_EX); - opcode = UNPACK_EX; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; - (void)(_old_stack_level); - _PyStackRef seq; - _PyStackRef *top; - seq = stack_pointer[-1]; - top = &stack_pointer[(oparg & 0xFF) + (oparg >> 8)]; - PyObject *seq_o = PyStackRef_AsPyObjectSteal(seq); - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - int res = _PyEval_UnpackIterableStackRef(tstate, seq_o, oparg & 0xFF, oparg >> 8, top); - Py_DECREF(seq_o); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (res == 0) { - TRACING_JUMP_TO_LABEL(error); - } - stack_pointer += 1 + (oparg & 0xFF) + (oparg >> 8); - assert(WITHIN_STACK_BOUNDS()); - TRACING_DISPATCH(); - } - - TRACING_TARGET(UNPACK_SEQUENCE) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = UNPACK_SEQUENCE; - (void)(opcode); - #endif - frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(UNPACK_SEQUENCE); - PREDICTED_TRACING_UNPACK_SEQUENCE:; - _Py_CODEUNIT* const this_instr = next_instr - 2; - (void)this_instr; - opcode = UNPACK_SEQUENCE; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; - (void)(_old_stack_level); - _PyStackRef seq; - _PyStackRef *top; - // _SPECIALIZE_UNPACK_SEQUENCE - { - seq = stack_pointer[-1]; - uint16_t counter = read_u16(&this_instr[1].cache); - (void)counter; - #if ENABLE_SPECIALIZATION_FT - if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { - next_instr = this_instr; - _PyFrame_SetStackPointer(frame, stack_pointer); - _Py_Specialize_UnpackSequence(seq, next_instr, oparg); - stack_pointer = _PyFrame_GetStackPointer(frame); - TRACING_SPECIALIZE_DISPATCH_SAME_OPARG() - } - OPCODE_DEFERRED_INC(UNPACK_SEQUENCE); - ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); - #endif /* ENABLE_SPECIALIZATION_FT */ - (void)seq; - (void)counter; - } - // _UNPACK_SEQUENCE - { - top = &stack_pointer[-1 + oparg]; - PyObject *seq_o = PyStackRef_AsPyObjectSteal(seq); - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - int res = _PyEval_UnpackIterableStackRef(tstate, seq_o, oparg, -1, top); - Py_DECREF(seq_o); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (res == 0) { - TRACING_JUMP_TO_LABEL(error); - } - } - stack_pointer += oparg; - assert(WITHIN_STACK_BOUNDS()); - TRACING_DISPATCH(); - } - - TRACING_TARGET(UNPACK_SEQUENCE_LIST) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = UNPACK_SEQUENCE_LIST; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(UNPACK_SEQUENCE_LIST); - opcode = UNPACK_SEQUENCE_LIST; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; - (void)(_old_stack_level); - static_assert(INLINE_CACHE_ENTRIES_UNPACK_SEQUENCE == 1, "incorrect cache size"); - _PyStackRef tos; - _PyStackRef seq; - _PyStackRef *values; - // _GUARD_TOS_LIST - { - tos = stack_pointer[-1]; - PyObject *o = PyStackRef_AsPyObjectBorrow(tos); - if (!PyList_CheckExact(o)) { - UPDATE_MISS_STATS(UNPACK_SEQUENCE); - assert(_PyOpcode_Deopt[opcode] == (UNPACK_SEQUENCE)); - JUMP_TO_PREDICTED(TRACING_UNPACK_SEQUENCE); - } - } - /* Skip 1 cache entry */ - // _UNPACK_SEQUENCE_LIST - { - seq = tos; - values = &stack_pointer[-1]; - PyObject *seq_o = PyStackRef_AsPyObjectBorrow(seq); - assert(PyList_CheckExact(seq_o)); - if (!LOCK_OBJECT(seq_o)) { - UPDATE_MISS_STATS(UNPACK_SEQUENCE); - assert(_PyOpcode_Deopt[opcode] == (UNPACK_SEQUENCE)); - JUMP_TO_PREDICTED(TRACING_UNPACK_SEQUENCE); - } - if (PyList_GET_SIZE(seq_o) != oparg) { - UNLOCK_OBJECT(seq_o); - if (true) { - UPDATE_MISS_STATS(UNPACK_SEQUENCE); - assert(_PyOpcode_Deopt[opcode] == (UNPACK_SEQUENCE)); - JUMP_TO_PREDICTED(TRACING_UNPACK_SEQUENCE); - } - } - STAT_INC(UNPACK_SEQUENCE, hit); - PyObject **items = _PyList_ITEMS(seq_o); - for (int i = oparg; --i >= 0; ) { - *values++ = PyStackRef_FromPyObjectNew(items[i]); - } - UNLOCK_OBJECT(seq_o); - stack_pointer += -1 + oparg; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(seq); - stack_pointer = _PyFrame_GetStackPointer(frame); - } - TRACING_DISPATCH(); - } - - TRACING_TARGET(UNPACK_SEQUENCE_TUPLE) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = UNPACK_SEQUENCE_TUPLE; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(UNPACK_SEQUENCE_TUPLE); - opcode = UNPACK_SEQUENCE_TUPLE; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; - (void)(_old_stack_level); - static_assert(INLINE_CACHE_ENTRIES_UNPACK_SEQUENCE == 1, "incorrect cache size"); - _PyStackRef tos; - _PyStackRef seq; - _PyStackRef *values; - // _GUARD_TOS_TUPLE - { - tos = stack_pointer[-1]; - PyObject *o = PyStackRef_AsPyObjectBorrow(tos); - if (!PyTuple_CheckExact(o)) { - UPDATE_MISS_STATS(UNPACK_SEQUENCE); - assert(_PyOpcode_Deopt[opcode] == (UNPACK_SEQUENCE)); - JUMP_TO_PREDICTED(TRACING_UNPACK_SEQUENCE); - } - } - /* Skip 1 cache entry */ - // _UNPACK_SEQUENCE_TUPLE - { - seq = tos; - values = &stack_pointer[-1]; - PyObject *seq_o = PyStackRef_AsPyObjectBorrow(seq); - assert(PyTuple_CheckExact(seq_o)); - if (PyTuple_GET_SIZE(seq_o) != oparg) { - UPDATE_MISS_STATS(UNPACK_SEQUENCE); - assert(_PyOpcode_Deopt[opcode] == (UNPACK_SEQUENCE)); - JUMP_TO_PREDICTED(TRACING_UNPACK_SEQUENCE); - } - STAT_INC(UNPACK_SEQUENCE, hit); - PyObject **items = _PyTuple_ITEMS(seq_o); - for (int i = oparg; --i >= 0; ) { - *values++ = PyStackRef_FromPyObjectNew(items[i]); - } - stack_pointer += -1 + oparg; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(seq); - stack_pointer = _PyFrame_GetStackPointer(frame); - } - TRACING_DISPATCH(); - } - - TRACING_TARGET(UNPACK_SEQUENCE_TWO_TUPLE) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = UNPACK_SEQUENCE_TWO_TUPLE; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(UNPACK_SEQUENCE_TWO_TUPLE); - opcode = UNPACK_SEQUENCE_TWO_TUPLE; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; - (void)(_old_stack_level); - static_assert(INLINE_CACHE_ENTRIES_UNPACK_SEQUENCE == 1, "incorrect cache size"); - _PyStackRef tos; - _PyStackRef seq; - _PyStackRef val1; - _PyStackRef val0; - // _GUARD_TOS_TUPLE - { - tos = stack_pointer[-1]; - PyObject *o = PyStackRef_AsPyObjectBorrow(tos); - if (!PyTuple_CheckExact(o)) { - UPDATE_MISS_STATS(UNPACK_SEQUENCE); - assert(_PyOpcode_Deopt[opcode] == (UNPACK_SEQUENCE)); - JUMP_TO_PREDICTED(TRACING_UNPACK_SEQUENCE); - } - } - /* Skip 1 cache entry */ - // _UNPACK_SEQUENCE_TWO_TUPLE - { - seq = tos; - assert(oparg == 2); - PyObject *seq_o = PyStackRef_AsPyObjectBorrow(seq); - assert(PyTuple_CheckExact(seq_o)); - if (PyTuple_GET_SIZE(seq_o) != 2) { - UPDATE_MISS_STATS(UNPACK_SEQUENCE); - assert(_PyOpcode_Deopt[opcode] == (UNPACK_SEQUENCE)); - JUMP_TO_PREDICTED(TRACING_UNPACK_SEQUENCE); - } - STAT_INC(UNPACK_SEQUENCE, hit); - val0 = PyStackRef_FromPyObjectNew(PyTuple_GET_ITEM(seq_o, 0)); - val1 = PyStackRef_FromPyObjectNew(PyTuple_GET_ITEM(seq_o, 1)); - stack_pointer[-1] = val1; - stack_pointer[0] = val0; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(seq); - stack_pointer = _PyFrame_GetStackPointer(frame); - } - TRACING_DISPATCH(); - } - - TRACING_TARGET(WITH_EXCEPT_START) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = WITH_EXCEPT_START; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(WITH_EXCEPT_START); - opcode = WITH_EXCEPT_START; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; - (void)(_old_stack_level); - _PyStackRef exit_func; - _PyStackRef exit_self; - _PyStackRef lasti; - _PyStackRef val; - _PyStackRef res; - val = stack_pointer[-1]; - lasti = stack_pointer[-3]; - exit_self = stack_pointer[-4]; - exit_func = stack_pointer[-5]; - PyObject *exc, *tb; - PyObject *val_o = PyStackRef_AsPyObjectBorrow(val); - PyObject *exit_func_o = PyStackRef_AsPyObjectBorrow(exit_func); - assert(val_o && PyExceptionInstance_Check(val_o)); - exc = PyExceptionInstance_Class(val_o); - PyObject *original_tb = tb = PyException_GetTraceback(val_o); - if (tb == NULL) { - tb = Py_None; - } - assert(PyStackRef_IsTaggedInt(lasti)); - (void)lasti; - PyObject *stack[5] = {NULL, PyStackRef_AsPyObjectBorrow(exit_self), exc, val_o, tb}; - int has_self = !PyStackRef_IsNull(exit_self); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *res_o = PyObject_Vectorcall(exit_func_o, stack + 2 - has_self, - (3 + has_self) | PY_VECTORCALL_ARGUMENTS_OFFSET, NULL); - Py_XDECREF(original_tb); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (res_o == NULL) { - TRACING_JUMP_TO_LABEL(error); - } - res = PyStackRef_FromPyObjectSteal(res_o); - stack_pointer[0] = res; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - TRACING_DISPATCH(); - } - - TRACING_TARGET(YIELD_VALUE) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = YIELD_VALUE; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(YIELD_VALUE); - opcode = YIELD_VALUE; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; - (void)(_old_stack_level); - _PyStackRef retval; - _PyStackRef value; - retval = stack_pointer[-1]; - assert(frame->owner != FRAME_OWNED_BY_INTERPRETER); - frame->instr_ptr++; - PyGenObject *gen = _PyGen_GetGeneratorFromFrame(frame); - assert(FRAME_SUSPENDED_YIELD_FROM == FRAME_SUSPENDED + 1); - assert(oparg == 0 || oparg == 1); - gen->gi_frame_state = FRAME_SUSPENDED + oparg; - _PyStackRef temp = retval; - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - tstate->exc_info = gen->gi_exc_state.previous_item; - gen->gi_exc_state.previous_item = NULL; - _Py_LeaveRecursiveCallPy(tstate); - _PyInterpreterFrame *gen_frame = frame; - frame = tstate->current_frame = frame->previous; - gen_frame->previous = NULL; - assert(INLINE_CACHE_ENTRIES_SEND == INLINE_CACHE_ENTRIES_FOR_ITER); - #if TIER_ONE - assert(frame->instr_ptr->op.code == INSTRUMENTED_LINE || - frame->instr_ptr->op.code == INSTRUMENTED_INSTRUCTION || - _PyOpcode_Deopt[frame->instr_ptr->op.code] == SEND || - _PyOpcode_Deopt[frame->instr_ptr->op.code] == FOR_ITER || - _PyOpcode_Deopt[frame->instr_ptr->op.code] == INTERPRETER_EXIT || - _PyOpcode_Deopt[frame->instr_ptr->op.code] == ENTER_EXECUTOR); - #endif - stack_pointer = _PyFrame_GetStackPointer(frame); - #if TIER_ONE - LOAD_IP(1 + INLINE_CACHE_ENTRIES_SEND); - #endif - #if TIER_TWO - TIER2_STORE_IP(1 + INLINE_CACHE_ENTRIES_SEND); - #endif - value = PyStackRef_MakeHeapSafe(temp); - LLTRACE_RESUME_FRAME(); - stack_pointer[0] = value; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - TRACING_DISPATCH(); - } - #endif /* END TRACING INSTRUCTIONS */ - #undef TRACING_JIT - #undef TIER_ONE diff --git a/Python/opcode_targets.h b/Python/opcode_targets.h index 66abd6d4623e..0d28b3d4e690 100644 --- a/Python/opcode_targets.h +++ b/Python/opcode_targets.h @@ -105,6 +105,7 @@ static void *opcode_targets_table[256] = { &&TARGET_POP_JUMP_IF_NOT_NONE, &&TARGET_POP_JUMP_IF_TRUE, &&TARGET_RAISE_VARARGS, + &&TARGET_RECORD_PREVIOUS_INST, &&TARGET_RERAISE, &&TARGET_SEND, &&TARGET_SET_ADD, @@ -127,7 +128,6 @@ static void *opcode_targets_table[256] = { &&_unknown_opcode, &&_unknown_opcode, &&_unknown_opcode, - &&_unknown_opcode, &&TARGET_RESUME, &&TARGET_BINARY_OP_ADD_FLOAT, &&TARGET_BINARY_OP_ADD_INT, @@ -259,216 +259,216 @@ static void *opcode_targets_table[256] = { }; #if _Py_TIER2 static void *opcode_tracing_targets_table[256] = { - &&TARGET_TRACING_CACHE, - &&TARGET_TRACING_BINARY_SLICE, - &&TARGET_TRACING_BUILD_TEMPLATE, - &&TARGET_TRACING_BINARY_OP_INPLACE_ADD_UNICODE, - &&TARGET_TRACING_CALL_FUNCTION_EX, - &&TARGET_TRACING_CHECK_EG_MATCH, - &&TARGET_TRACING_CHECK_EXC_MATCH, - &&TARGET_TRACING_CLEANUP_THROW, - &&TARGET_TRACING_DELETE_SUBSCR, - &&TARGET_TRACING_END_FOR, - &&TARGET_TRACING_END_SEND, - &&TARGET_TRACING_EXIT_INIT_CHECK, - &&TARGET_TRACING_FORMAT_SIMPLE, - &&TARGET_TRACING_FORMAT_WITH_SPEC, - &&TARGET_TRACING_GET_AITER, - &&TARGET_TRACING_GET_ANEXT, - &&TARGET_TRACING_GET_ITER, - &&TARGET_TRACING_RESERVED, - &&TARGET_TRACING_GET_LEN, - &&TARGET_TRACING_GET_YIELD_FROM_ITER, - &&TARGET_TRACING_INTERPRETER_EXIT, - &&TARGET_TRACING_LOAD_BUILD_CLASS, - &&TARGET_TRACING_LOAD_LOCALS, - &&TARGET_TRACING_MAKE_FUNCTION, - &&TARGET_TRACING_MATCH_KEYS, - &&TARGET_TRACING_MATCH_MAPPING, - &&TARGET_TRACING_MATCH_SEQUENCE, - &&TARGET_TRACING_NOP, - &&TARGET_TRACING_NOT_TAKEN, - &&TARGET_TRACING_POP_EXCEPT, - &&TARGET_TRACING_POP_ITER, - &&TARGET_TRACING_POP_TOP, - &&TARGET_TRACING_PUSH_EXC_INFO, - &&TARGET_TRACING_PUSH_NULL, - &&TARGET_TRACING_RETURN_GENERATOR, - &&TARGET_TRACING_RETURN_VALUE, - &&TARGET_TRACING_SETUP_ANNOTATIONS, - &&TARGET_TRACING_STORE_SLICE, - &&TARGET_TRACING_STORE_SUBSCR, - &&TARGET_TRACING_TO_BOOL, - &&TARGET_TRACING_UNARY_INVERT, - &&TARGET_TRACING_UNARY_NEGATIVE, - &&TARGET_TRACING_UNARY_NOT, - &&TARGET_TRACING_WITH_EXCEPT_START, - &&TARGET_TRACING_BINARY_OP, - &&TARGET_TRACING_BUILD_INTERPOLATION, - &&TARGET_TRACING_BUILD_LIST, - &&TARGET_TRACING_BUILD_MAP, - &&TARGET_TRACING_BUILD_SET, - &&TARGET_TRACING_BUILD_SLICE, - &&TARGET_TRACING_BUILD_STRING, - &&TARGET_TRACING_BUILD_TUPLE, - &&TARGET_TRACING_CALL, - &&TARGET_TRACING_CALL_INTRINSIC_1, - &&TARGET_TRACING_CALL_INTRINSIC_2, - &&TARGET_TRACING_CALL_KW, - &&TARGET_TRACING_COMPARE_OP, - &&TARGET_TRACING_CONTAINS_OP, - &&TARGET_TRACING_CONVERT_VALUE, - &&TARGET_TRACING_COPY, - &&TARGET_TRACING_COPY_FREE_VARS, - &&TARGET_TRACING_DELETE_ATTR, - &&TARGET_TRACING_DELETE_DEREF, - &&TARGET_TRACING_DELETE_FAST, - &&TARGET_TRACING_DELETE_GLOBAL, - &&TARGET_TRACING_DELETE_NAME, - &&TARGET_TRACING_DICT_MERGE, - &&TARGET_TRACING_DICT_UPDATE, - &&TARGET_TRACING_END_ASYNC_FOR, - &&TARGET_TRACING_EXTENDED_ARG, - &&TARGET_TRACING_FOR_ITER, - &&TARGET_TRACING_GET_AWAITABLE, - &&TARGET_TRACING_IMPORT_FROM, - &&TARGET_TRACING_IMPORT_NAME, - &&TARGET_TRACING_IS_OP, - &&TARGET_TRACING_JUMP_BACKWARD, - &&TARGET_TRACING_JUMP_BACKWARD_NO_INTERRUPT, - &&TARGET_TRACING_JUMP_FORWARD, - &&TARGET_TRACING_LIST_APPEND, - &&TARGET_TRACING_LIST_EXTEND, - &&TARGET_TRACING_LOAD_ATTR, - &&TARGET_TRACING_LOAD_COMMON_CONSTANT, - &&TARGET_TRACING_LOAD_CONST, - &&TARGET_TRACING_LOAD_DEREF, - &&TARGET_TRACING_LOAD_FAST, - &&TARGET_TRACING_LOAD_FAST_AND_CLEAR, - &&TARGET_TRACING_LOAD_FAST_BORROW, - &&TARGET_TRACING_LOAD_FAST_BORROW_LOAD_FAST_BORROW, - &&TARGET_TRACING_LOAD_FAST_CHECK, - &&TARGET_TRACING_LOAD_FAST_LOAD_FAST, - &&TARGET_TRACING_LOAD_FROM_DICT_OR_DEREF, - &&TARGET_TRACING_LOAD_FROM_DICT_OR_GLOBALS, - &&TARGET_TRACING_LOAD_GLOBAL, - &&TARGET_TRACING_LOAD_NAME, - &&TARGET_TRACING_LOAD_SMALL_INT, - &&TARGET_TRACING_LOAD_SPECIAL, - &&TARGET_TRACING_LOAD_SUPER_ATTR, - &&TARGET_TRACING_MAKE_CELL, - &&TARGET_TRACING_MAP_ADD, - &&TARGET_TRACING_MATCH_CLASS, - &&TARGET_TRACING_POP_JUMP_IF_FALSE, - &&TARGET_TRACING_POP_JUMP_IF_NONE, - &&TARGET_TRACING_POP_JUMP_IF_NOT_NONE, - &&TARGET_TRACING_POP_JUMP_IF_TRUE, - &&TARGET_TRACING_RAISE_VARARGS, - &&TARGET_TRACING_RERAISE, - &&TARGET_TRACING_SEND, - &&TARGET_TRACING_SET_ADD, - &&TARGET_TRACING_SET_FUNCTION_ATTRIBUTE, - &&TARGET_TRACING_SET_UPDATE, - &&TARGET_TRACING_STORE_ATTR, - &&TARGET_TRACING_STORE_DEREF, - &&TARGET_TRACING_STORE_FAST, - &&TARGET_TRACING_STORE_FAST_LOAD_FAST, - &&TARGET_TRACING_STORE_FAST_STORE_FAST, - &&TARGET_TRACING_STORE_GLOBAL, - &&TARGET_TRACING_STORE_NAME, - &&TARGET_TRACING_SWAP, - &&TARGET_TRACING_UNPACK_EX, - &&TARGET_TRACING_UNPACK_SEQUENCE, - &&TARGET_TRACING_YIELD_VALUE, - &&_unknown_opcode, + &&TARGET_RECORD_PREVIOUS_INST, + &&TARGET_RECORD_PREVIOUS_INST, + &&TARGET_RECORD_PREVIOUS_INST, + &&TARGET_RECORD_PREVIOUS_INST, + &&TARGET_RECORD_PREVIOUS_INST, + &&TARGET_RECORD_PREVIOUS_INST, + &&TARGET_RECORD_PREVIOUS_INST, + &&TARGET_RECORD_PREVIOUS_INST, + &&TARGET_RECORD_PREVIOUS_INST, + &&TARGET_RECORD_PREVIOUS_INST, + &&TARGET_RECORD_PREVIOUS_INST, + &&TARGET_RECORD_PREVIOUS_INST, + &&TARGET_RECORD_PREVIOUS_INST, + &&TARGET_RECORD_PREVIOUS_INST, + &&TARGET_RECORD_PREVIOUS_INST, + &&TARGET_RECORD_PREVIOUS_INST, + &&TARGET_RECORD_PREVIOUS_INST, + &&TARGET_RECORD_PREVIOUS_INST, + &&TARGET_RECORD_PREVIOUS_INST, + &&TARGET_RECORD_PREVIOUS_INST, + &&TARGET_RECORD_PREVIOUS_INST, + &&TARGET_RECORD_PREVIOUS_INST, + &&TARGET_RECORD_PREVIOUS_INST, + &&TARGET_RECORD_PREVIOUS_INST, + &&TARGET_RECORD_PREVIOUS_INST, + &&TARGET_RECORD_PREVIOUS_INST, + &&TARGET_RECORD_PREVIOUS_INST, + &&TARGET_RECORD_PREVIOUS_INST, + &&TARGET_RECORD_PREVIOUS_INST, + &&TARGET_RECORD_PREVIOUS_INST, + &&TARGET_RECORD_PREVIOUS_INST, + &&TARGET_RECORD_PREVIOUS_INST, + &&TARGET_RECORD_PREVIOUS_INST, + &&TARGET_RECORD_PREVIOUS_INST, + &&TARGET_RECORD_PREVIOUS_INST, + &&TARGET_RECORD_PREVIOUS_INST, + &&TARGET_RECORD_PREVIOUS_INST, + &&TARGET_RECORD_PREVIOUS_INST, + &&TARGET_RECORD_PREVIOUS_INST, + &&TARGET_RECORD_PREVIOUS_INST, + &&TARGET_RECORD_PREVIOUS_INST, + &&TARGET_RECORD_PREVIOUS_INST, + &&TARGET_RECORD_PREVIOUS_INST, + &&TARGET_RECORD_PREVIOUS_INST, + &&TARGET_RECORD_PREVIOUS_INST, + &&TARGET_RECORD_PREVIOUS_INST, + &&TARGET_RECORD_PREVIOUS_INST, + &&TARGET_RECORD_PREVIOUS_INST, + &&TARGET_RECORD_PREVIOUS_INST, + &&TARGET_RECORD_PREVIOUS_INST, + &&TARGET_RECORD_PREVIOUS_INST, + &&TARGET_RECORD_PREVIOUS_INST, + &&TARGET_RECORD_PREVIOUS_INST, + &&TARGET_RECORD_PREVIOUS_INST, + &&TARGET_RECORD_PREVIOUS_INST, + &&TARGET_RECORD_PREVIOUS_INST, + &&TARGET_RECORD_PREVIOUS_INST, + &&TARGET_RECORD_PREVIOUS_INST, + &&TARGET_RECORD_PREVIOUS_INST, + &&TARGET_RECORD_PREVIOUS_INST, + &&TARGET_RECORD_PREVIOUS_INST, + &&TARGET_RECORD_PREVIOUS_INST, + &&TARGET_RECORD_PREVIOUS_INST, + &&TARGET_RECORD_PREVIOUS_INST, + &&TARGET_RECORD_PREVIOUS_INST, + &&TARGET_RECORD_PREVIOUS_INST, + &&TARGET_RECORD_PREVIOUS_INST, + &&TARGET_RECORD_PREVIOUS_INST, + &&TARGET_RECORD_PREVIOUS_INST, + &&TARGET_RECORD_PREVIOUS_INST, + &&TARGET_RECORD_PREVIOUS_INST, + &&TARGET_RECORD_PREVIOUS_INST, + &&TARGET_RECORD_PREVIOUS_INST, + &&TARGET_RECORD_PREVIOUS_INST, + &&TARGET_RECORD_PREVIOUS_INST, + &&TARGET_RECORD_PREVIOUS_INST, + &&TARGET_RECORD_PREVIOUS_INST, + &&TARGET_RECORD_PREVIOUS_INST, + &&TARGET_RECORD_PREVIOUS_INST, + &&TARGET_RECORD_PREVIOUS_INST, + &&TARGET_RECORD_PREVIOUS_INST, + &&TARGET_RECORD_PREVIOUS_INST, + &&TARGET_RECORD_PREVIOUS_INST, + &&TARGET_RECORD_PREVIOUS_INST, + &&TARGET_RECORD_PREVIOUS_INST, + &&TARGET_RECORD_PREVIOUS_INST, + &&TARGET_RECORD_PREVIOUS_INST, + &&TARGET_RECORD_PREVIOUS_INST, + &&TARGET_RECORD_PREVIOUS_INST, + &&TARGET_RECORD_PREVIOUS_INST, + &&TARGET_RECORD_PREVIOUS_INST, + &&TARGET_RECORD_PREVIOUS_INST, + &&TARGET_RECORD_PREVIOUS_INST, + &&TARGET_RECORD_PREVIOUS_INST, + &&TARGET_RECORD_PREVIOUS_INST, + &&TARGET_RECORD_PREVIOUS_INST, + &&TARGET_RECORD_PREVIOUS_INST, + &&TARGET_RECORD_PREVIOUS_INST, + &&TARGET_RECORD_PREVIOUS_INST, + &&TARGET_RECORD_PREVIOUS_INST, + &&TARGET_RECORD_PREVIOUS_INST, + &&TARGET_RECORD_PREVIOUS_INST, + &&TARGET_RECORD_PREVIOUS_INST, + &&TARGET_RECORD_PREVIOUS_INST, + &&TARGET_RECORD_PREVIOUS_INST, + &&TARGET_RECORD_PREVIOUS_INST, + &&TARGET_RECORD_PREVIOUS_INST, + &&TARGET_RECORD_PREVIOUS_INST, + &&TARGET_RECORD_PREVIOUS_INST, + &&TARGET_RECORD_PREVIOUS_INST, + &&TARGET_RECORD_PREVIOUS_INST, + &&TARGET_RECORD_PREVIOUS_INST, + &&TARGET_RECORD_PREVIOUS_INST, + &&TARGET_RECORD_PREVIOUS_INST, + &&TARGET_RECORD_PREVIOUS_INST, + &&TARGET_RECORD_PREVIOUS_INST, + &&TARGET_RECORD_PREVIOUS_INST, + &&TARGET_RECORD_PREVIOUS_INST, + &&TARGET_RECORD_PREVIOUS_INST, + &&TARGET_RECORD_PREVIOUS_INST, + &&TARGET_RECORD_PREVIOUS_INST, + &&TARGET_RECORD_PREVIOUS_INST, &&_unknown_opcode, &&_unknown_opcode, &&_unknown_opcode, &&_unknown_opcode, &&_unknown_opcode, &&_unknown_opcode, - &&TARGET_TRACING_RESUME, - &&TARGET_TRACING_BINARY_OP_ADD_FLOAT, - &&TARGET_TRACING_BINARY_OP_ADD_INT, - &&TARGET_TRACING_BINARY_OP_ADD_UNICODE, - &&TARGET_TRACING_BINARY_OP_EXTEND, - &&TARGET_TRACING_BINARY_OP_MULTIPLY_FLOAT, - &&TARGET_TRACING_BINARY_OP_MULTIPLY_INT, - &&TARGET_TRACING_BINARY_OP_SUBSCR_DICT, - &&TARGET_TRACING_BINARY_OP_SUBSCR_GETITEM, - &&TARGET_TRACING_BINARY_OP_SUBSCR_LIST_INT, - &&TARGET_TRACING_BINARY_OP_SUBSCR_LIST_SLICE, - &&TARGET_TRACING_BINARY_OP_SUBSCR_STR_INT, - &&TARGET_TRACING_BINARY_OP_SUBSCR_TUPLE_INT, - &&TARGET_TRACING_BINARY_OP_SUBTRACT_FLOAT, - &&TARGET_TRACING_BINARY_OP_SUBTRACT_INT, - &&TARGET_TRACING_CALL_ALLOC_AND_ENTER_INIT, - &&TARGET_TRACING_CALL_BOUND_METHOD_EXACT_ARGS, - &&TARGET_TRACING_CALL_BOUND_METHOD_GENERAL, - &&TARGET_TRACING_CALL_BUILTIN_CLASS, - &&TARGET_TRACING_CALL_BUILTIN_FAST, - &&TARGET_TRACING_CALL_BUILTIN_FAST_WITH_KEYWORDS, - &&TARGET_TRACING_CALL_BUILTIN_O, - &&TARGET_TRACING_CALL_ISINSTANCE, - &&TARGET_TRACING_CALL_KW_BOUND_METHOD, - &&TARGET_TRACING_CALL_KW_NON_PY, - &&TARGET_TRACING_CALL_KW_PY, - &&TARGET_TRACING_CALL_LEN, - &&TARGET_TRACING_CALL_LIST_APPEND, - &&TARGET_TRACING_CALL_METHOD_DESCRIPTOR_FAST, - &&TARGET_TRACING_CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS, - &&TARGET_TRACING_CALL_METHOD_DESCRIPTOR_NOARGS, - &&TARGET_TRACING_CALL_METHOD_DESCRIPTOR_O, - &&TARGET_TRACING_CALL_NON_PY_GENERAL, - &&TARGET_TRACING_CALL_PY_EXACT_ARGS, - &&TARGET_TRACING_CALL_PY_GENERAL, - &&TARGET_TRACING_CALL_STR_1, - &&TARGET_TRACING_CALL_TUPLE_1, - &&TARGET_TRACING_CALL_TYPE_1, - &&TARGET_TRACING_COMPARE_OP_FLOAT, - &&TARGET_TRACING_COMPARE_OP_INT, - &&TARGET_TRACING_COMPARE_OP_STR, - &&TARGET_TRACING_CONTAINS_OP_DICT, - &&TARGET_TRACING_CONTAINS_OP_SET, - &&TARGET_TRACING_FOR_ITER_GEN, - &&TARGET_TRACING_FOR_ITER_LIST, - &&TARGET_TRACING_FOR_ITER_RANGE, - &&TARGET_TRACING_FOR_ITER_TUPLE, - &&TARGET_TRACING_JUMP_BACKWARD_JIT, - &&TARGET_TRACING_JUMP_BACKWARD_NO_JIT, - &&TARGET_TRACING_LOAD_ATTR_CLASS, - &&TARGET_TRACING_LOAD_ATTR_CLASS_WITH_METACLASS_CHECK, - &&TARGET_TRACING_LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN, - &&TARGET_TRACING_LOAD_ATTR_INSTANCE_VALUE, - &&TARGET_TRACING_LOAD_ATTR_METHOD_LAZY_DICT, - &&TARGET_TRACING_LOAD_ATTR_METHOD_NO_DICT, - &&TARGET_TRACING_LOAD_ATTR_METHOD_WITH_VALUES, - &&TARGET_TRACING_LOAD_ATTR_MODULE, - &&TARGET_TRACING_LOAD_ATTR_NONDESCRIPTOR_NO_DICT, - &&TARGET_TRACING_LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES, - &&TARGET_TRACING_LOAD_ATTR_PROPERTY, - &&TARGET_TRACING_LOAD_ATTR_SLOT, - &&TARGET_TRACING_LOAD_ATTR_WITH_HINT, - &&TARGET_TRACING_LOAD_GLOBAL_BUILTIN, - &&TARGET_TRACING_LOAD_GLOBAL_MODULE, - &&TARGET_TRACING_LOAD_SUPER_ATTR_ATTR, - &&TARGET_TRACING_LOAD_SUPER_ATTR_METHOD, - &&TARGET_TRACING_RESUME_CHECK, - &&TARGET_TRACING_SEND_GEN, - &&TARGET_TRACING_STORE_ATTR_INSTANCE_VALUE, - &&TARGET_TRACING_STORE_ATTR_SLOT, - &&TARGET_TRACING_STORE_ATTR_WITH_HINT, - &&TARGET_TRACING_STORE_SUBSCR_DICT, - &&TARGET_TRACING_STORE_SUBSCR_LIST_INT, - &&TARGET_TRACING_TO_BOOL_ALWAYS_TRUE, - &&TARGET_TRACING_TO_BOOL_BOOL, - &&TARGET_TRACING_TO_BOOL_INT, - &&TARGET_TRACING_TO_BOOL_LIST, - &&TARGET_TRACING_TO_BOOL_NONE, - &&TARGET_TRACING_TO_BOOL_STR, - &&TARGET_TRACING_UNPACK_SEQUENCE_LIST, - &&TARGET_TRACING_UNPACK_SEQUENCE_TUPLE, - &&TARGET_TRACING_UNPACK_SEQUENCE_TWO_TUPLE, + &&TARGET_RECORD_PREVIOUS_INST, + &&TARGET_RECORD_PREVIOUS_INST, + &&TARGET_RECORD_PREVIOUS_INST, + &&TARGET_RECORD_PREVIOUS_INST, + &&TARGET_RECORD_PREVIOUS_INST, + &&TARGET_RECORD_PREVIOUS_INST, + &&TARGET_RECORD_PREVIOUS_INST, + &&TARGET_RECORD_PREVIOUS_INST, + &&TARGET_RECORD_PREVIOUS_INST, + &&TARGET_RECORD_PREVIOUS_INST, + &&TARGET_RECORD_PREVIOUS_INST, + &&TARGET_RECORD_PREVIOUS_INST, + &&TARGET_RECORD_PREVIOUS_INST, + &&TARGET_RECORD_PREVIOUS_INST, + &&TARGET_RECORD_PREVIOUS_INST, + &&TARGET_RECORD_PREVIOUS_INST, + &&TARGET_RECORD_PREVIOUS_INST, + &&TARGET_RECORD_PREVIOUS_INST, + &&TARGET_RECORD_PREVIOUS_INST, + &&TARGET_RECORD_PREVIOUS_INST, + &&TARGET_RECORD_PREVIOUS_INST, + &&TARGET_RECORD_PREVIOUS_INST, + &&TARGET_RECORD_PREVIOUS_INST, + &&TARGET_RECORD_PREVIOUS_INST, + &&TARGET_RECORD_PREVIOUS_INST, + &&TARGET_RECORD_PREVIOUS_INST, + &&TARGET_RECORD_PREVIOUS_INST, + &&TARGET_RECORD_PREVIOUS_INST, + &&TARGET_RECORD_PREVIOUS_INST, + &&TARGET_RECORD_PREVIOUS_INST, + &&TARGET_RECORD_PREVIOUS_INST, + &&TARGET_RECORD_PREVIOUS_INST, + &&TARGET_RECORD_PREVIOUS_INST, + &&TARGET_RECORD_PREVIOUS_INST, + &&TARGET_RECORD_PREVIOUS_INST, + &&TARGET_RECORD_PREVIOUS_INST, + &&TARGET_RECORD_PREVIOUS_INST, + &&TARGET_RECORD_PREVIOUS_INST, + &&TARGET_RECORD_PREVIOUS_INST, + &&TARGET_RECORD_PREVIOUS_INST, + &&TARGET_RECORD_PREVIOUS_INST, + &&TARGET_RECORD_PREVIOUS_INST, + &&TARGET_RECORD_PREVIOUS_INST, + &&TARGET_RECORD_PREVIOUS_INST, + &&TARGET_RECORD_PREVIOUS_INST, + &&TARGET_RECORD_PREVIOUS_INST, + &&TARGET_RECORD_PREVIOUS_INST, + &&TARGET_RECORD_PREVIOUS_INST, + &&TARGET_RECORD_PREVIOUS_INST, + &&TARGET_RECORD_PREVIOUS_INST, + &&TARGET_RECORD_PREVIOUS_INST, + &&TARGET_RECORD_PREVIOUS_INST, + &&TARGET_RECORD_PREVIOUS_INST, + &&TARGET_RECORD_PREVIOUS_INST, + &&TARGET_RECORD_PREVIOUS_INST, + &&TARGET_RECORD_PREVIOUS_INST, + &&TARGET_RECORD_PREVIOUS_INST, + &&TARGET_RECORD_PREVIOUS_INST, + &&TARGET_RECORD_PREVIOUS_INST, + &&TARGET_RECORD_PREVIOUS_INST, + &&TARGET_RECORD_PREVIOUS_INST, + &&TARGET_RECORD_PREVIOUS_INST, + &&TARGET_RECORD_PREVIOUS_INST, + &&TARGET_RECORD_PREVIOUS_INST, + &&TARGET_RECORD_PREVIOUS_INST, + &&TARGET_RECORD_PREVIOUS_INST, + &&TARGET_RECORD_PREVIOUS_INST, + &&TARGET_RECORD_PREVIOUS_INST, + &&TARGET_RECORD_PREVIOUS_INST, + &&TARGET_RECORD_PREVIOUS_INST, + &&TARGET_RECORD_PREVIOUS_INST, + &&TARGET_RECORD_PREVIOUS_INST, + &&TARGET_RECORD_PREVIOUS_INST, + &&TARGET_RECORD_PREVIOUS_INST, + &&TARGET_RECORD_PREVIOUS_INST, + &&TARGET_RECORD_PREVIOUS_INST, + &&TARGET_RECORD_PREVIOUS_INST, + &&TARGET_RECORD_PREVIOUS_INST, + &&TARGET_RECORD_PREVIOUS_INST, + &&TARGET_RECORD_PREVIOUS_INST, + &&TARGET_RECORD_PREVIOUS_INST, + &&TARGET_RECORD_PREVIOUS_INST, &&_unknown_opcode, &&_unknown_opcode, &&_unknown_opcode, @@ -493,28 +493,28 @@ static void *opcode_tracing_targets_table[256] = { &&_unknown_opcode, &&_unknown_opcode, &&_unknown_opcode, - &&TARGET_TRACING_INSTRUMENTED_END_FOR, - &&TARGET_TRACING_INSTRUMENTED_POP_ITER, - &&TARGET_TRACING_INSTRUMENTED_END_SEND, - &&TARGET_TRACING_INSTRUMENTED_FOR_ITER, - &&TARGET_TRACING_INSTRUMENTED_INSTRUCTION, - &&TARGET_TRACING_INSTRUMENTED_JUMP_FORWARD, - &&TARGET_TRACING_INSTRUMENTED_NOT_TAKEN, - &&TARGET_TRACING_INSTRUMENTED_POP_JUMP_IF_TRUE, - &&TARGET_TRACING_INSTRUMENTED_POP_JUMP_IF_FALSE, - &&TARGET_TRACING_INSTRUMENTED_POP_JUMP_IF_NONE, - &&TARGET_TRACING_INSTRUMENTED_POP_JUMP_IF_NOT_NONE, - &&TARGET_TRACING_INSTRUMENTED_RESUME, - &&TARGET_TRACING_INSTRUMENTED_RETURN_VALUE, - &&TARGET_TRACING_INSTRUMENTED_YIELD_VALUE, - &&TARGET_TRACING_INSTRUMENTED_END_ASYNC_FOR, - &&TARGET_TRACING_INSTRUMENTED_LOAD_SUPER_ATTR, - &&TARGET_TRACING_INSTRUMENTED_CALL, - &&TARGET_TRACING_INSTRUMENTED_CALL_KW, - &&TARGET_TRACING_INSTRUMENTED_CALL_FUNCTION_EX, - &&TARGET_TRACING_INSTRUMENTED_JUMP_BACKWARD, - &&TARGET_TRACING_INSTRUMENTED_LINE, - &&TARGET_TRACING_ENTER_EXECUTOR, + &&TARGET_RECORD_PREVIOUS_INST, + &&TARGET_RECORD_PREVIOUS_INST, + &&TARGET_RECORD_PREVIOUS_INST, + &&TARGET_RECORD_PREVIOUS_INST, + &&TARGET_RECORD_PREVIOUS_INST, + &&TARGET_RECORD_PREVIOUS_INST, + &&TARGET_RECORD_PREVIOUS_INST, + &&TARGET_RECORD_PREVIOUS_INST, + &&TARGET_RECORD_PREVIOUS_INST, + &&TARGET_RECORD_PREVIOUS_INST, + &&TARGET_RECORD_PREVIOUS_INST, + &&TARGET_RECORD_PREVIOUS_INST, + &&TARGET_RECORD_PREVIOUS_INST, + &&TARGET_RECORD_PREVIOUS_INST, + &&TARGET_RECORD_PREVIOUS_INST, + &&TARGET_RECORD_PREVIOUS_INST, + &&TARGET_RECORD_PREVIOUS_INST, + &&TARGET_RECORD_PREVIOUS_INST, + &&TARGET_RECORD_PREVIOUS_INST, + &&TARGET_RECORD_PREVIOUS_INST, + &&TARGET_RECORD_PREVIOUS_INST, + &&TARGET_RECORD_PREVIOUS_INST, }; #endif #else /* _Py_TAIL_CALL_INTERP */ @@ -530,455 +530,231 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_exit_unwind(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_start_frame(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_OP(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_BINARY_OP(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_OP_ADD_FLOAT(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_BINARY_OP_ADD_FLOAT(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_OP_ADD_INT(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_BINARY_OP_ADD_INT(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_OP_ADD_UNICODE(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_BINARY_OP_ADD_UNICODE(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_OP_EXTEND(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_BINARY_OP_EXTEND(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_OP_INPLACE_ADD_UNICODE(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_BINARY_OP_INPLACE_ADD_UNICODE(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_OP_MULTIPLY_FLOAT(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_BINARY_OP_MULTIPLY_FLOAT(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_OP_MULTIPLY_INT(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_BINARY_OP_MULTIPLY_INT(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_OP_SUBSCR_DICT(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_BINARY_OP_SUBSCR_DICT(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_OP_SUBSCR_GETITEM(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_BINARY_OP_SUBSCR_GETITEM(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_OP_SUBSCR_LIST_INT(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_BINARY_OP_SUBSCR_LIST_INT(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_OP_SUBSCR_LIST_SLICE(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_BINARY_OP_SUBSCR_LIST_SLICE(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_OP_SUBSCR_STR_INT(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_BINARY_OP_SUBSCR_STR_INT(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_OP_SUBSCR_TUPLE_INT(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_BINARY_OP_SUBSCR_TUPLE_INT(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_OP_SUBTRACT_FLOAT(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_BINARY_OP_SUBTRACT_FLOAT(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_OP_SUBTRACT_INT(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_BINARY_OP_SUBTRACT_INT(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_SLICE(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_BINARY_SLICE(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BUILD_INTERPOLATION(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_BUILD_INTERPOLATION(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BUILD_LIST(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_BUILD_LIST(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BUILD_MAP(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_BUILD_MAP(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BUILD_SET(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_BUILD_SET(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BUILD_SLICE(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_BUILD_SLICE(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BUILD_STRING(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_BUILD_STRING(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BUILD_TEMPLATE(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_BUILD_TEMPLATE(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BUILD_TUPLE(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_BUILD_TUPLE(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CACHE(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_CACHE(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_CALL(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_ALLOC_AND_ENTER_INIT(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_CALL_ALLOC_AND_ENTER_INIT(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_BOUND_METHOD_EXACT_ARGS(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_CALL_BOUND_METHOD_EXACT_ARGS(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_BOUND_METHOD_GENERAL(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_CALL_BOUND_METHOD_GENERAL(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_BUILTIN_CLASS(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_CALL_BUILTIN_CLASS(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_BUILTIN_FAST(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_CALL_BUILTIN_FAST(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_BUILTIN_FAST_WITH_KEYWORDS(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_CALL_BUILTIN_FAST_WITH_KEYWORDS(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_BUILTIN_O(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_CALL_BUILTIN_O(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_FUNCTION_EX(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_CALL_FUNCTION_EX(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_INTRINSIC_1(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_CALL_INTRINSIC_1(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_INTRINSIC_2(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_CALL_INTRINSIC_2(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_ISINSTANCE(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_CALL_ISINSTANCE(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_KW(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_CALL_KW(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_KW_BOUND_METHOD(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_CALL_KW_BOUND_METHOD(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_KW_NON_PY(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_CALL_KW_NON_PY(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_KW_PY(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_CALL_KW_PY(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_LEN(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_CALL_LEN(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_LIST_APPEND(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_CALL_LIST_APPEND(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_METHOD_DESCRIPTOR_FAST(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_CALL_METHOD_DESCRIPTOR_FAST(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_METHOD_DESCRIPTOR_NOARGS(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_CALL_METHOD_DESCRIPTOR_NOARGS(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_METHOD_DESCRIPTOR_O(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_CALL_METHOD_DESCRIPTOR_O(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_NON_PY_GENERAL(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_CALL_NON_PY_GENERAL(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_PY_EXACT_ARGS(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_CALL_PY_EXACT_ARGS(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_PY_GENERAL(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_CALL_PY_GENERAL(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_STR_1(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_CALL_STR_1(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_TUPLE_1(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_CALL_TUPLE_1(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_TYPE_1(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_CALL_TYPE_1(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CHECK_EG_MATCH(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_CHECK_EG_MATCH(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CHECK_EXC_MATCH(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_CHECK_EXC_MATCH(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CLEANUP_THROW(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_CLEANUP_THROW(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_COMPARE_OP(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_COMPARE_OP(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_COMPARE_OP_FLOAT(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_COMPARE_OP_FLOAT(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_COMPARE_OP_INT(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_COMPARE_OP_INT(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_COMPARE_OP_STR(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_COMPARE_OP_STR(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CONTAINS_OP(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_CONTAINS_OP(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CONTAINS_OP_DICT(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_CONTAINS_OP_DICT(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CONTAINS_OP_SET(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_CONTAINS_OP_SET(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CONVERT_VALUE(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_CONVERT_VALUE(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_COPY(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_COPY(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_COPY_FREE_VARS(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_COPY_FREE_VARS(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_DELETE_ATTR(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_DELETE_ATTR(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_DELETE_DEREF(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_DELETE_DEREF(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_DELETE_FAST(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_DELETE_FAST(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_DELETE_GLOBAL(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_DELETE_GLOBAL(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_DELETE_NAME(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_DELETE_NAME(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_DELETE_SUBSCR(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_DELETE_SUBSCR(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_DICT_MERGE(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_DICT_MERGE(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_DICT_UPDATE(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_DICT_UPDATE(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_END_ASYNC_FOR(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_END_ASYNC_FOR(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_END_FOR(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_END_FOR(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_END_SEND(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_END_SEND(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_ENTER_EXECUTOR(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_ENTER_EXECUTOR(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_EXIT_INIT_CHECK(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_EXIT_INIT_CHECK(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_EXTENDED_ARG(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_EXTENDED_ARG(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_FORMAT_SIMPLE(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_FORMAT_SIMPLE(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_FORMAT_WITH_SPEC(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_FORMAT_WITH_SPEC(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_FOR_ITER(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_FOR_ITER(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_FOR_ITER_GEN(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_FOR_ITER_GEN(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_FOR_ITER_LIST(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_FOR_ITER_LIST(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_FOR_ITER_RANGE(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_FOR_ITER_RANGE(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_FOR_ITER_TUPLE(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_FOR_ITER_TUPLE(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_GET_AITER(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_GET_AITER(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_GET_ANEXT(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_GET_ANEXT(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_GET_AWAITABLE(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_GET_AWAITABLE(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_GET_ITER(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_GET_ITER(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_GET_LEN(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_GET_LEN(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_GET_YIELD_FROM_ITER(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_GET_YIELD_FROM_ITER(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_IMPORT_FROM(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_IMPORT_FROM(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_IMPORT_NAME(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_IMPORT_NAME(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_CALL(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_INSTRUMENTED_CALL(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_CALL_FUNCTION_EX(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_INSTRUMENTED_CALL_FUNCTION_EX(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_CALL_KW(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_INSTRUMENTED_CALL_KW(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_END_ASYNC_FOR(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_INSTRUMENTED_END_ASYNC_FOR(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_END_FOR(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_INSTRUMENTED_END_FOR(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_END_SEND(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_INSTRUMENTED_END_SEND(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_FOR_ITER(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_INSTRUMENTED_FOR_ITER(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_INSTRUCTION(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_INSTRUMENTED_INSTRUCTION(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_JUMP_BACKWARD(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_INSTRUMENTED_JUMP_BACKWARD(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_JUMP_FORWARD(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_INSTRUMENTED_JUMP_FORWARD(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_LINE(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_INSTRUMENTED_LINE(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_LOAD_SUPER_ATTR(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_INSTRUMENTED_LOAD_SUPER_ATTR(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_NOT_TAKEN(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_INSTRUMENTED_NOT_TAKEN(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_POP_ITER(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_INSTRUMENTED_POP_ITER(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_POP_JUMP_IF_FALSE(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_INSTRUMENTED_POP_JUMP_IF_FALSE(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_POP_JUMP_IF_NONE(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_INSTRUMENTED_POP_JUMP_IF_NONE(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_POP_JUMP_IF_NOT_NONE(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_INSTRUMENTED_POP_JUMP_IF_NOT_NONE(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_POP_JUMP_IF_TRUE(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_INSTRUMENTED_POP_JUMP_IF_TRUE(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_RESUME(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_INSTRUMENTED_RESUME(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_RETURN_VALUE(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_INSTRUMENTED_RETURN_VALUE(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_YIELD_VALUE(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_INSTRUMENTED_YIELD_VALUE(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INTERPRETER_EXIT(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_INTERPRETER_EXIT(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_IS_OP(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_IS_OP(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_JUMP_BACKWARD(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_JUMP_BACKWARD(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_JUMP_BACKWARD_JIT(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_JUMP_BACKWARD_JIT(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_JUMP_BACKWARD_NO_INTERRUPT(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_JUMP_BACKWARD_NO_INTERRUPT(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_JUMP_BACKWARD_NO_JIT(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_JUMP_BACKWARD_NO_JIT(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_JUMP_FORWARD(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_JUMP_FORWARD(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LIST_APPEND(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_LIST_APPEND(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LIST_EXTEND(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_LIST_EXTEND(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_LOAD_ATTR(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_CLASS(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_LOAD_ATTR_CLASS(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_CLASS_WITH_METACLASS_CHECK(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_LOAD_ATTR_CLASS_WITH_METACLASS_CHECK(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_INSTANCE_VALUE(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_LOAD_ATTR_INSTANCE_VALUE(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_METHOD_LAZY_DICT(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_LOAD_ATTR_METHOD_LAZY_DICT(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_METHOD_NO_DICT(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_LOAD_ATTR_METHOD_NO_DICT(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_METHOD_WITH_VALUES(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_LOAD_ATTR_METHOD_WITH_VALUES(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_MODULE(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_LOAD_ATTR_MODULE(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_NONDESCRIPTOR_NO_DICT(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_LOAD_ATTR_NONDESCRIPTOR_NO_DICT(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_PROPERTY(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_LOAD_ATTR_PROPERTY(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_SLOT(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_LOAD_ATTR_SLOT(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_WITH_HINT(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_LOAD_ATTR_WITH_HINT(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_BUILD_CLASS(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_LOAD_BUILD_CLASS(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_COMMON_CONSTANT(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_LOAD_COMMON_CONSTANT(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_CONST(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_LOAD_CONST(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_DEREF(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_LOAD_DEREF(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_FAST(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_LOAD_FAST(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_FAST_AND_CLEAR(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_LOAD_FAST_AND_CLEAR(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_FAST_BORROW(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_LOAD_FAST_BORROW(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_FAST_BORROW_LOAD_FAST_BORROW(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_LOAD_FAST_BORROW_LOAD_FAST_BORROW(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_FAST_CHECK(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_LOAD_FAST_CHECK(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_FAST_LOAD_FAST(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_LOAD_FAST_LOAD_FAST(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_FROM_DICT_OR_DEREF(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_LOAD_FROM_DICT_OR_DEREF(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_FROM_DICT_OR_GLOBALS(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_LOAD_FROM_DICT_OR_GLOBALS(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_GLOBAL(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_LOAD_GLOBAL(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_GLOBAL_BUILTIN(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_LOAD_GLOBAL_BUILTIN(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_GLOBAL_MODULE(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_LOAD_GLOBAL_MODULE(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_LOCALS(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_LOAD_LOCALS(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_NAME(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_LOAD_NAME(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_SMALL_INT(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_LOAD_SMALL_INT(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_SPECIAL(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_LOAD_SPECIAL(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_SUPER_ATTR(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_LOAD_SUPER_ATTR(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_SUPER_ATTR_ATTR(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_LOAD_SUPER_ATTR_ATTR(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_SUPER_ATTR_METHOD(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_LOAD_SUPER_ATTR_METHOD(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_MAKE_CELL(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_MAKE_CELL(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_MAKE_FUNCTION(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_MAKE_FUNCTION(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_MAP_ADD(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_MAP_ADD(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_MATCH_CLASS(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_MATCH_CLASS(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_MATCH_KEYS(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_MATCH_KEYS(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_MATCH_MAPPING(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_MATCH_MAPPING(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_MATCH_SEQUENCE(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_MATCH_SEQUENCE(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_NOP(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_NOP(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_NOT_TAKEN(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_NOT_TAKEN(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_POP_EXCEPT(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_POP_EXCEPT(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_POP_ITER(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_POP_ITER(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_POP_JUMP_IF_FALSE(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_POP_JUMP_IF_FALSE(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_POP_JUMP_IF_NONE(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_POP_JUMP_IF_NONE(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_POP_JUMP_IF_NOT_NONE(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_POP_JUMP_IF_NOT_NONE(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_POP_JUMP_IF_TRUE(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_POP_JUMP_IF_TRUE(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_POP_TOP(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_POP_TOP(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_PUSH_EXC_INFO(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_PUSH_EXC_INFO(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_PUSH_NULL(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_PUSH_NULL(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_RAISE_VARARGS(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_RAISE_VARARGS(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_RECORD_PREVIOUS_INST(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_RERAISE(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_RERAISE(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_RESERVED(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_RESERVED(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_RESUME(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_RESUME(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_RESUME_CHECK(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_RESUME_CHECK(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_RETURN_GENERATOR(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_RETURN_GENERATOR(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_RETURN_VALUE(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_RETURN_VALUE(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_SEND(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_SEND(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_SEND_GEN(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_SEND_GEN(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_SETUP_ANNOTATIONS(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_SETUP_ANNOTATIONS(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_SET_ADD(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_SET_ADD(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_SET_FUNCTION_ATTRIBUTE(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_SET_FUNCTION_ATTRIBUTE(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_SET_UPDATE(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_SET_UPDATE(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_STORE_ATTR(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_STORE_ATTR(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_STORE_ATTR_INSTANCE_VALUE(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_STORE_ATTR_INSTANCE_VALUE(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_STORE_ATTR_SLOT(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_STORE_ATTR_SLOT(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_STORE_ATTR_WITH_HINT(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_STORE_ATTR_WITH_HINT(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_STORE_DEREF(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_STORE_DEREF(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_STORE_FAST(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_STORE_FAST(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_STORE_FAST_LOAD_FAST(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_STORE_FAST_LOAD_FAST(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_STORE_FAST_STORE_FAST(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_STORE_FAST_STORE_FAST(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_STORE_GLOBAL(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_STORE_GLOBAL(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_STORE_NAME(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_STORE_NAME(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_STORE_SLICE(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_STORE_SLICE(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_STORE_SUBSCR(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_STORE_SUBSCR(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_STORE_SUBSCR_DICT(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_STORE_SUBSCR_DICT(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_STORE_SUBSCR_LIST_INT(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_STORE_SUBSCR_LIST_INT(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_SWAP(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_SWAP(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TO_BOOL(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_TO_BOOL(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TO_BOOL_ALWAYS_TRUE(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_TO_BOOL_ALWAYS_TRUE(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TO_BOOL_BOOL(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_TO_BOOL_BOOL(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TO_BOOL_INT(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_TO_BOOL_INT(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TO_BOOL_LIST(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_TO_BOOL_LIST(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TO_BOOL_NONE(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_TO_BOOL_NONE(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TO_BOOL_STR(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_TO_BOOL_STR(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_UNARY_INVERT(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_UNARY_INVERT(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_UNARY_NEGATIVE(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_UNARY_NEGATIVE(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_UNARY_NOT(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_UNARY_NOT(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_UNPACK_EX(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_UNPACK_EX(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_UNPACK_SEQUENCE(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_UNPACK_SEQUENCE(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_UNPACK_SEQUENCE_LIST(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_UNPACK_SEQUENCE_LIST(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_UNPACK_SEQUENCE_TUPLE(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_UNPACK_SEQUENCE_TUPLE(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_UNPACK_SEQUENCE_TWO_TUPLE(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_UNPACK_SEQUENCE_TWO_TUPLE(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_WITH_EXCEPT_START(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_WITH_EXCEPT_START(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_YIELD_VALUE(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_YIELD_VALUE(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_UNKNOWN_OPCODE(TAIL_CALL_PARAMS) { int opcode = next_instr->op.code; @@ -1172,6 +948,7 @@ static py_tail_call_funcptr instruction_funcptr_handler_table[256] = { [PUSH_EXC_INFO] = _TAIL_CALL_PUSH_EXC_INFO, [PUSH_NULL] = _TAIL_CALL_PUSH_NULL, [RAISE_VARARGS] = _TAIL_CALL_RAISE_VARARGS, + [RECORD_PREVIOUS_INST] = _TAIL_CALL_RECORD_PREVIOUS_INST, [RERAISE] = _TAIL_CALL_RERAISE, [RESERVED] = _TAIL_CALL_RESERVED, [RESUME] = _TAIL_CALL_RESUME, @@ -1216,7 +993,6 @@ static py_tail_call_funcptr instruction_funcptr_handler_table[256] = { [UNPACK_SEQUENCE_TWO_TUPLE] = _TAIL_CALL_UNPACK_SEQUENCE_TWO_TUPLE, [WITH_EXCEPT_START] = _TAIL_CALL_WITH_EXCEPT_START, [YIELD_VALUE] = _TAIL_CALL_YIELD_VALUE, - [121] = _TAIL_CALL_UNKNOWN_OPCODE, [122] = _TAIL_CALL_UNKNOWN_OPCODE, [123] = _TAIL_CALL_UNKNOWN_OPCODE, [124] = _TAIL_CALL_UNKNOWN_OPCODE, @@ -1249,232 +1025,232 @@ static py_tail_call_funcptr instruction_funcptr_handler_table[256] = { [233] = _TAIL_CALL_UNKNOWN_OPCODE, }; static py_tail_call_funcptr instruction_funcptr_tracing_table[256] = { - [BINARY_OP] = _TAIL_CALL_TRACING_BINARY_OP, - [BINARY_OP_ADD_FLOAT] = _TAIL_CALL_TRACING_BINARY_OP_ADD_FLOAT, - [BINARY_OP_ADD_INT] = _TAIL_CALL_TRACING_BINARY_OP_ADD_INT, - [BINARY_OP_ADD_UNICODE] = _TAIL_CALL_TRACING_BINARY_OP_ADD_UNICODE, - [BINARY_OP_EXTEND] = _TAIL_CALL_TRACING_BINARY_OP_EXTEND, - [BINARY_OP_INPLACE_ADD_UNICODE] = _TAIL_CALL_TRACING_BINARY_OP_INPLACE_ADD_UNICODE, - [BINARY_OP_MULTIPLY_FLOAT] = _TAIL_CALL_TRACING_BINARY_OP_MULTIPLY_FLOAT, - [BINARY_OP_MULTIPLY_INT] = _TAIL_CALL_TRACING_BINARY_OP_MULTIPLY_INT, - [BINARY_OP_SUBSCR_DICT] = _TAIL_CALL_TRACING_BINARY_OP_SUBSCR_DICT, - [BINARY_OP_SUBSCR_GETITEM] = _TAIL_CALL_TRACING_BINARY_OP_SUBSCR_GETITEM, - [BINARY_OP_SUBSCR_LIST_INT] = _TAIL_CALL_TRACING_BINARY_OP_SUBSCR_LIST_INT, - [BINARY_OP_SUBSCR_LIST_SLICE] = _TAIL_CALL_TRACING_BINARY_OP_SUBSCR_LIST_SLICE, - [BINARY_OP_SUBSCR_STR_INT] = _TAIL_CALL_TRACING_BINARY_OP_SUBSCR_STR_INT, - [BINARY_OP_SUBSCR_TUPLE_INT] = _TAIL_CALL_TRACING_BINARY_OP_SUBSCR_TUPLE_INT, - [BINARY_OP_SUBTRACT_FLOAT] = _TAIL_CALL_TRACING_BINARY_OP_SUBTRACT_FLOAT, - [BINARY_OP_SUBTRACT_INT] = _TAIL_CALL_TRACING_BINARY_OP_SUBTRACT_INT, - [BINARY_SLICE] = _TAIL_CALL_TRACING_BINARY_SLICE, - [BUILD_INTERPOLATION] = _TAIL_CALL_TRACING_BUILD_INTERPOLATION, - [BUILD_LIST] = _TAIL_CALL_TRACING_BUILD_LIST, - [BUILD_MAP] = _TAIL_CALL_TRACING_BUILD_MAP, - [BUILD_SET] = _TAIL_CALL_TRACING_BUILD_SET, - [BUILD_SLICE] = _TAIL_CALL_TRACING_BUILD_SLICE, - [BUILD_STRING] = _TAIL_CALL_TRACING_BUILD_STRING, - [BUILD_TEMPLATE] = _TAIL_CALL_TRACING_BUILD_TEMPLATE, - [BUILD_TUPLE] = _TAIL_CALL_TRACING_BUILD_TUPLE, - [CACHE] = _TAIL_CALL_TRACING_CACHE, - [CALL] = _TAIL_CALL_TRACING_CALL, - [CALL_ALLOC_AND_ENTER_INIT] = _TAIL_CALL_TRACING_CALL_ALLOC_AND_ENTER_INIT, - [CALL_BOUND_METHOD_EXACT_ARGS] = _TAIL_CALL_TRACING_CALL_BOUND_METHOD_EXACT_ARGS, - [CALL_BOUND_METHOD_GENERAL] = _TAIL_CALL_TRACING_CALL_BOUND_METHOD_GENERAL, - [CALL_BUILTIN_CLASS] = _TAIL_CALL_TRACING_CALL_BUILTIN_CLASS, - [CALL_BUILTIN_FAST] = _TAIL_CALL_TRACING_CALL_BUILTIN_FAST, - [CALL_BUILTIN_FAST_WITH_KEYWORDS] = _TAIL_CALL_TRACING_CALL_BUILTIN_FAST_WITH_KEYWORDS, - [CALL_BUILTIN_O] = _TAIL_CALL_TRACING_CALL_BUILTIN_O, - [CALL_FUNCTION_EX] = _TAIL_CALL_TRACING_CALL_FUNCTION_EX, - [CALL_INTRINSIC_1] = _TAIL_CALL_TRACING_CALL_INTRINSIC_1, - [CALL_INTRINSIC_2] = _TAIL_CALL_TRACING_CALL_INTRINSIC_2, - [CALL_ISINSTANCE] = _TAIL_CALL_TRACING_CALL_ISINSTANCE, - [CALL_KW] = _TAIL_CALL_TRACING_CALL_KW, - [CALL_KW_BOUND_METHOD] = _TAIL_CALL_TRACING_CALL_KW_BOUND_METHOD, - [CALL_KW_NON_PY] = _TAIL_CALL_TRACING_CALL_KW_NON_PY, - [CALL_KW_PY] = _TAIL_CALL_TRACING_CALL_KW_PY, - [CALL_LEN] = _TAIL_CALL_TRACING_CALL_LEN, - [CALL_LIST_APPEND] = _TAIL_CALL_TRACING_CALL_LIST_APPEND, - [CALL_METHOD_DESCRIPTOR_FAST] = _TAIL_CALL_TRACING_CALL_METHOD_DESCRIPTOR_FAST, - [CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS] = _TAIL_CALL_TRACING_CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS, - [CALL_METHOD_DESCRIPTOR_NOARGS] = _TAIL_CALL_TRACING_CALL_METHOD_DESCRIPTOR_NOARGS, - [CALL_METHOD_DESCRIPTOR_O] = _TAIL_CALL_TRACING_CALL_METHOD_DESCRIPTOR_O, - [CALL_NON_PY_GENERAL] = _TAIL_CALL_TRACING_CALL_NON_PY_GENERAL, - [CALL_PY_EXACT_ARGS] = _TAIL_CALL_TRACING_CALL_PY_EXACT_ARGS, - [CALL_PY_GENERAL] = _TAIL_CALL_TRACING_CALL_PY_GENERAL, - [CALL_STR_1] = _TAIL_CALL_TRACING_CALL_STR_1, - [CALL_TUPLE_1] = _TAIL_CALL_TRACING_CALL_TUPLE_1, - [CALL_TYPE_1] = _TAIL_CALL_TRACING_CALL_TYPE_1, - [CHECK_EG_MATCH] = _TAIL_CALL_TRACING_CHECK_EG_MATCH, - [CHECK_EXC_MATCH] = _TAIL_CALL_TRACING_CHECK_EXC_MATCH, - [CLEANUP_THROW] = _TAIL_CALL_TRACING_CLEANUP_THROW, - [COMPARE_OP] = _TAIL_CALL_TRACING_COMPARE_OP, - [COMPARE_OP_FLOAT] = _TAIL_CALL_TRACING_COMPARE_OP_FLOAT, - [COMPARE_OP_INT] = _TAIL_CALL_TRACING_COMPARE_OP_INT, - [COMPARE_OP_STR] = _TAIL_CALL_TRACING_COMPARE_OP_STR, - [CONTAINS_OP] = _TAIL_CALL_TRACING_CONTAINS_OP, - [CONTAINS_OP_DICT] = _TAIL_CALL_TRACING_CONTAINS_OP_DICT, - [CONTAINS_OP_SET] = _TAIL_CALL_TRACING_CONTAINS_OP_SET, - [CONVERT_VALUE] = _TAIL_CALL_TRACING_CONVERT_VALUE, - [COPY] = _TAIL_CALL_TRACING_COPY, - [COPY_FREE_VARS] = _TAIL_CALL_TRACING_COPY_FREE_VARS, - [DELETE_ATTR] = _TAIL_CALL_TRACING_DELETE_ATTR, - [DELETE_DEREF] = _TAIL_CALL_TRACING_DELETE_DEREF, - [DELETE_FAST] = _TAIL_CALL_TRACING_DELETE_FAST, - [DELETE_GLOBAL] = _TAIL_CALL_TRACING_DELETE_GLOBAL, - [DELETE_NAME] = _TAIL_CALL_TRACING_DELETE_NAME, - [DELETE_SUBSCR] = _TAIL_CALL_TRACING_DELETE_SUBSCR, - [DICT_MERGE] = _TAIL_CALL_TRACING_DICT_MERGE, - [DICT_UPDATE] = _TAIL_CALL_TRACING_DICT_UPDATE, - [END_ASYNC_FOR] = _TAIL_CALL_TRACING_END_ASYNC_FOR, - [END_FOR] = _TAIL_CALL_TRACING_END_FOR, - [END_SEND] = _TAIL_CALL_TRACING_END_SEND, - [ENTER_EXECUTOR] = _TAIL_CALL_TRACING_ENTER_EXECUTOR, - [EXIT_INIT_CHECK] = _TAIL_CALL_TRACING_EXIT_INIT_CHECK, - [EXTENDED_ARG] = _TAIL_CALL_TRACING_EXTENDED_ARG, - [FORMAT_SIMPLE] = _TAIL_CALL_TRACING_FORMAT_SIMPLE, - [FORMAT_WITH_SPEC] = _TAIL_CALL_TRACING_FORMAT_WITH_SPEC, - [FOR_ITER] = _TAIL_CALL_TRACING_FOR_ITER, - [FOR_ITER_GEN] = _TAIL_CALL_TRACING_FOR_ITER_GEN, - [FOR_ITER_LIST] = _TAIL_CALL_TRACING_FOR_ITER_LIST, - [FOR_ITER_RANGE] = _TAIL_CALL_TRACING_FOR_ITER_RANGE, - [FOR_ITER_TUPLE] = _TAIL_CALL_TRACING_FOR_ITER_TUPLE, - [GET_AITER] = _TAIL_CALL_TRACING_GET_AITER, - [GET_ANEXT] = _TAIL_CALL_TRACING_GET_ANEXT, - [GET_AWAITABLE] = _TAIL_CALL_TRACING_GET_AWAITABLE, - [GET_ITER] = _TAIL_CALL_TRACING_GET_ITER, - [GET_LEN] = _TAIL_CALL_TRACING_GET_LEN, - [GET_YIELD_FROM_ITER] = _TAIL_CALL_TRACING_GET_YIELD_FROM_ITER, - [IMPORT_FROM] = _TAIL_CALL_TRACING_IMPORT_FROM, - [IMPORT_NAME] = _TAIL_CALL_TRACING_IMPORT_NAME, - [INSTRUMENTED_CALL] = _TAIL_CALL_TRACING_INSTRUMENTED_CALL, - [INSTRUMENTED_CALL_FUNCTION_EX] = _TAIL_CALL_TRACING_INSTRUMENTED_CALL_FUNCTION_EX, - [INSTRUMENTED_CALL_KW] = _TAIL_CALL_TRACING_INSTRUMENTED_CALL_KW, - [INSTRUMENTED_END_ASYNC_FOR] = _TAIL_CALL_TRACING_INSTRUMENTED_END_ASYNC_FOR, - [INSTRUMENTED_END_FOR] = _TAIL_CALL_TRACING_INSTRUMENTED_END_FOR, - [INSTRUMENTED_END_SEND] = _TAIL_CALL_TRACING_INSTRUMENTED_END_SEND, - [INSTRUMENTED_FOR_ITER] = _TAIL_CALL_TRACING_INSTRUMENTED_FOR_ITER, - [INSTRUMENTED_INSTRUCTION] = _TAIL_CALL_TRACING_INSTRUMENTED_INSTRUCTION, - [INSTRUMENTED_JUMP_BACKWARD] = _TAIL_CALL_TRACING_INSTRUMENTED_JUMP_BACKWARD, - [INSTRUMENTED_JUMP_FORWARD] = _TAIL_CALL_TRACING_INSTRUMENTED_JUMP_FORWARD, - [INSTRUMENTED_LINE] = _TAIL_CALL_TRACING_INSTRUMENTED_LINE, - [INSTRUMENTED_LOAD_SUPER_ATTR] = _TAIL_CALL_TRACING_INSTRUMENTED_LOAD_SUPER_ATTR, - [INSTRUMENTED_NOT_TAKEN] = _TAIL_CALL_TRACING_INSTRUMENTED_NOT_TAKEN, - [INSTRUMENTED_POP_ITER] = _TAIL_CALL_TRACING_INSTRUMENTED_POP_ITER, - [INSTRUMENTED_POP_JUMP_IF_FALSE] = _TAIL_CALL_TRACING_INSTRUMENTED_POP_JUMP_IF_FALSE, - [INSTRUMENTED_POP_JUMP_IF_NONE] = _TAIL_CALL_TRACING_INSTRUMENTED_POP_JUMP_IF_NONE, - [INSTRUMENTED_POP_JUMP_IF_NOT_NONE] = _TAIL_CALL_TRACING_INSTRUMENTED_POP_JUMP_IF_NOT_NONE, - [INSTRUMENTED_POP_JUMP_IF_TRUE] = _TAIL_CALL_TRACING_INSTRUMENTED_POP_JUMP_IF_TRUE, - [INSTRUMENTED_RESUME] = _TAIL_CALL_TRACING_INSTRUMENTED_RESUME, - [INSTRUMENTED_RETURN_VALUE] = _TAIL_CALL_TRACING_INSTRUMENTED_RETURN_VALUE, - [INSTRUMENTED_YIELD_VALUE] = _TAIL_CALL_TRACING_INSTRUMENTED_YIELD_VALUE, - [INTERPRETER_EXIT] = _TAIL_CALL_TRACING_INTERPRETER_EXIT, - [IS_OP] = _TAIL_CALL_TRACING_IS_OP, - [JUMP_BACKWARD] = _TAIL_CALL_TRACING_JUMP_BACKWARD, - [JUMP_BACKWARD_JIT] = _TAIL_CALL_TRACING_JUMP_BACKWARD_JIT, - [JUMP_BACKWARD_NO_INTERRUPT] = _TAIL_CALL_TRACING_JUMP_BACKWARD_NO_INTERRUPT, - [JUMP_BACKWARD_NO_JIT] = _TAIL_CALL_TRACING_JUMP_BACKWARD_NO_JIT, - [JUMP_FORWARD] = _TAIL_CALL_TRACING_JUMP_FORWARD, - [LIST_APPEND] = _TAIL_CALL_TRACING_LIST_APPEND, - [LIST_EXTEND] = _TAIL_CALL_TRACING_LIST_EXTEND, - [LOAD_ATTR] = _TAIL_CALL_TRACING_LOAD_ATTR, - [LOAD_ATTR_CLASS] = _TAIL_CALL_TRACING_LOAD_ATTR_CLASS, - [LOAD_ATTR_CLASS_WITH_METACLASS_CHECK] = _TAIL_CALL_TRACING_LOAD_ATTR_CLASS_WITH_METACLASS_CHECK, - [LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN] = _TAIL_CALL_TRACING_LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN, - [LOAD_ATTR_INSTANCE_VALUE] = _TAIL_CALL_TRACING_LOAD_ATTR_INSTANCE_VALUE, - [LOAD_ATTR_METHOD_LAZY_DICT] = _TAIL_CALL_TRACING_LOAD_ATTR_METHOD_LAZY_DICT, - [LOAD_ATTR_METHOD_NO_DICT] = _TAIL_CALL_TRACING_LOAD_ATTR_METHOD_NO_DICT, - [LOAD_ATTR_METHOD_WITH_VALUES] = _TAIL_CALL_TRACING_LOAD_ATTR_METHOD_WITH_VALUES, - [LOAD_ATTR_MODULE] = _TAIL_CALL_TRACING_LOAD_ATTR_MODULE, - [LOAD_ATTR_NONDESCRIPTOR_NO_DICT] = _TAIL_CALL_TRACING_LOAD_ATTR_NONDESCRIPTOR_NO_DICT, - [LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES] = _TAIL_CALL_TRACING_LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES, - [LOAD_ATTR_PROPERTY] = _TAIL_CALL_TRACING_LOAD_ATTR_PROPERTY, - [LOAD_ATTR_SLOT] = _TAIL_CALL_TRACING_LOAD_ATTR_SLOT, - [LOAD_ATTR_WITH_HINT] = _TAIL_CALL_TRACING_LOAD_ATTR_WITH_HINT, - [LOAD_BUILD_CLASS] = _TAIL_CALL_TRACING_LOAD_BUILD_CLASS, - [LOAD_COMMON_CONSTANT] = _TAIL_CALL_TRACING_LOAD_COMMON_CONSTANT, - [LOAD_CONST] = _TAIL_CALL_TRACING_LOAD_CONST, - [LOAD_DEREF] = _TAIL_CALL_TRACING_LOAD_DEREF, - [LOAD_FAST] = _TAIL_CALL_TRACING_LOAD_FAST, - [LOAD_FAST_AND_CLEAR] = _TAIL_CALL_TRACING_LOAD_FAST_AND_CLEAR, - [LOAD_FAST_BORROW] = _TAIL_CALL_TRACING_LOAD_FAST_BORROW, - [LOAD_FAST_BORROW_LOAD_FAST_BORROW] = _TAIL_CALL_TRACING_LOAD_FAST_BORROW_LOAD_FAST_BORROW, - [LOAD_FAST_CHECK] = _TAIL_CALL_TRACING_LOAD_FAST_CHECK, - [LOAD_FAST_LOAD_FAST] = _TAIL_CALL_TRACING_LOAD_FAST_LOAD_FAST, - [LOAD_FROM_DICT_OR_DEREF] = _TAIL_CALL_TRACING_LOAD_FROM_DICT_OR_DEREF, - [LOAD_FROM_DICT_OR_GLOBALS] = _TAIL_CALL_TRACING_LOAD_FROM_DICT_OR_GLOBALS, - [LOAD_GLOBAL] = _TAIL_CALL_TRACING_LOAD_GLOBAL, - [LOAD_GLOBAL_BUILTIN] = _TAIL_CALL_TRACING_LOAD_GLOBAL_BUILTIN, - [LOAD_GLOBAL_MODULE] = _TAIL_CALL_TRACING_LOAD_GLOBAL_MODULE, - [LOAD_LOCALS] = _TAIL_CALL_TRACING_LOAD_LOCALS, - [LOAD_NAME] = _TAIL_CALL_TRACING_LOAD_NAME, - [LOAD_SMALL_INT] = _TAIL_CALL_TRACING_LOAD_SMALL_INT, - [LOAD_SPECIAL] = _TAIL_CALL_TRACING_LOAD_SPECIAL, - [LOAD_SUPER_ATTR] = _TAIL_CALL_TRACING_LOAD_SUPER_ATTR, - [LOAD_SUPER_ATTR_ATTR] = _TAIL_CALL_TRACING_LOAD_SUPER_ATTR_ATTR, - [LOAD_SUPER_ATTR_METHOD] = _TAIL_CALL_TRACING_LOAD_SUPER_ATTR_METHOD, - [MAKE_CELL] = _TAIL_CALL_TRACING_MAKE_CELL, - [MAKE_FUNCTION] = _TAIL_CALL_TRACING_MAKE_FUNCTION, - [MAP_ADD] = _TAIL_CALL_TRACING_MAP_ADD, - [MATCH_CLASS] = _TAIL_CALL_TRACING_MATCH_CLASS, - [MATCH_KEYS] = _TAIL_CALL_TRACING_MATCH_KEYS, - [MATCH_MAPPING] = _TAIL_CALL_TRACING_MATCH_MAPPING, - [MATCH_SEQUENCE] = _TAIL_CALL_TRACING_MATCH_SEQUENCE, - [NOP] = _TAIL_CALL_TRACING_NOP, - [NOT_TAKEN] = _TAIL_CALL_TRACING_NOT_TAKEN, - [POP_EXCEPT] = _TAIL_CALL_TRACING_POP_EXCEPT, - [POP_ITER] = _TAIL_CALL_TRACING_POP_ITER, - [POP_JUMP_IF_FALSE] = _TAIL_CALL_TRACING_POP_JUMP_IF_FALSE, - [POP_JUMP_IF_NONE] = _TAIL_CALL_TRACING_POP_JUMP_IF_NONE, - [POP_JUMP_IF_NOT_NONE] = _TAIL_CALL_TRACING_POP_JUMP_IF_NOT_NONE, - [POP_JUMP_IF_TRUE] = _TAIL_CALL_TRACING_POP_JUMP_IF_TRUE, - [POP_TOP] = _TAIL_CALL_TRACING_POP_TOP, - [PUSH_EXC_INFO] = _TAIL_CALL_TRACING_PUSH_EXC_INFO, - [PUSH_NULL] = _TAIL_CALL_TRACING_PUSH_NULL, - [RAISE_VARARGS] = _TAIL_CALL_TRACING_RAISE_VARARGS, - [RERAISE] = _TAIL_CALL_TRACING_RERAISE, - [RESERVED] = _TAIL_CALL_TRACING_RESERVED, - [RESUME] = _TAIL_CALL_TRACING_RESUME, - [RESUME_CHECK] = _TAIL_CALL_TRACING_RESUME_CHECK, - [RETURN_GENERATOR] = _TAIL_CALL_TRACING_RETURN_GENERATOR, - [RETURN_VALUE] = _TAIL_CALL_TRACING_RETURN_VALUE, - [SEND] = _TAIL_CALL_TRACING_SEND, - [SEND_GEN] = _TAIL_CALL_TRACING_SEND_GEN, - [SETUP_ANNOTATIONS] = _TAIL_CALL_TRACING_SETUP_ANNOTATIONS, - [SET_ADD] = _TAIL_CALL_TRACING_SET_ADD, - [SET_FUNCTION_ATTRIBUTE] = _TAIL_CALL_TRACING_SET_FUNCTION_ATTRIBUTE, - [SET_UPDATE] = _TAIL_CALL_TRACING_SET_UPDATE, - [STORE_ATTR] = _TAIL_CALL_TRACING_STORE_ATTR, - [STORE_ATTR_INSTANCE_VALUE] = _TAIL_CALL_TRACING_STORE_ATTR_INSTANCE_VALUE, - [STORE_ATTR_SLOT] = _TAIL_CALL_TRACING_STORE_ATTR_SLOT, - [STORE_ATTR_WITH_HINT] = _TAIL_CALL_TRACING_STORE_ATTR_WITH_HINT, - [STORE_DEREF] = _TAIL_CALL_TRACING_STORE_DEREF, - [STORE_FAST] = _TAIL_CALL_TRACING_STORE_FAST, - [STORE_FAST_LOAD_FAST] = _TAIL_CALL_TRACING_STORE_FAST_LOAD_FAST, - [STORE_FAST_STORE_FAST] = _TAIL_CALL_TRACING_STORE_FAST_STORE_FAST, - [STORE_GLOBAL] = _TAIL_CALL_TRACING_STORE_GLOBAL, - [STORE_NAME] = _TAIL_CALL_TRACING_STORE_NAME, - [STORE_SLICE] = _TAIL_CALL_TRACING_STORE_SLICE, - [STORE_SUBSCR] = _TAIL_CALL_TRACING_STORE_SUBSCR, - [STORE_SUBSCR_DICT] = _TAIL_CALL_TRACING_STORE_SUBSCR_DICT, - [STORE_SUBSCR_LIST_INT] = _TAIL_CALL_TRACING_STORE_SUBSCR_LIST_INT, - [SWAP] = _TAIL_CALL_TRACING_SWAP, - [TO_BOOL] = _TAIL_CALL_TRACING_TO_BOOL, - [TO_BOOL_ALWAYS_TRUE] = _TAIL_CALL_TRACING_TO_BOOL_ALWAYS_TRUE, - [TO_BOOL_BOOL] = _TAIL_CALL_TRACING_TO_BOOL_BOOL, - [TO_BOOL_INT] = _TAIL_CALL_TRACING_TO_BOOL_INT, - [TO_BOOL_LIST] = _TAIL_CALL_TRACING_TO_BOOL_LIST, - [TO_BOOL_NONE] = _TAIL_CALL_TRACING_TO_BOOL_NONE, - [TO_BOOL_STR] = _TAIL_CALL_TRACING_TO_BOOL_STR, - [UNARY_INVERT] = _TAIL_CALL_TRACING_UNARY_INVERT, - [UNARY_NEGATIVE] = _TAIL_CALL_TRACING_UNARY_NEGATIVE, - [UNARY_NOT] = _TAIL_CALL_TRACING_UNARY_NOT, - [UNPACK_EX] = _TAIL_CALL_TRACING_UNPACK_EX, - [UNPACK_SEQUENCE] = _TAIL_CALL_TRACING_UNPACK_SEQUENCE, - [UNPACK_SEQUENCE_LIST] = _TAIL_CALL_TRACING_UNPACK_SEQUENCE_LIST, - [UNPACK_SEQUENCE_TUPLE] = _TAIL_CALL_TRACING_UNPACK_SEQUENCE_TUPLE, - [UNPACK_SEQUENCE_TWO_TUPLE] = _TAIL_CALL_TRACING_UNPACK_SEQUENCE_TWO_TUPLE, - [WITH_EXCEPT_START] = _TAIL_CALL_TRACING_WITH_EXCEPT_START, - [YIELD_VALUE] = _TAIL_CALL_TRACING_YIELD_VALUE, - [121] = _TAIL_CALL_UNKNOWN_OPCODE, + [BINARY_OP] = _TAIL_CALL_RECORD_PREVIOUS_INST, + [BINARY_OP_ADD_FLOAT] = _TAIL_CALL_RECORD_PREVIOUS_INST, + [BINARY_OP_ADD_INT] = _TAIL_CALL_RECORD_PREVIOUS_INST, + [BINARY_OP_ADD_UNICODE] = _TAIL_CALL_RECORD_PREVIOUS_INST, + [BINARY_OP_EXTEND] = _TAIL_CALL_RECORD_PREVIOUS_INST, + [BINARY_OP_INPLACE_ADD_UNICODE] = _TAIL_CALL_RECORD_PREVIOUS_INST, + [BINARY_OP_MULTIPLY_FLOAT] = _TAIL_CALL_RECORD_PREVIOUS_INST, + [BINARY_OP_MULTIPLY_INT] = _TAIL_CALL_RECORD_PREVIOUS_INST, + [BINARY_OP_SUBSCR_DICT] = _TAIL_CALL_RECORD_PREVIOUS_INST, + [BINARY_OP_SUBSCR_GETITEM] = _TAIL_CALL_RECORD_PREVIOUS_INST, + [BINARY_OP_SUBSCR_LIST_INT] = _TAIL_CALL_RECORD_PREVIOUS_INST, + [BINARY_OP_SUBSCR_LIST_SLICE] = _TAIL_CALL_RECORD_PREVIOUS_INST, + [BINARY_OP_SUBSCR_STR_INT] = _TAIL_CALL_RECORD_PREVIOUS_INST, + [BINARY_OP_SUBSCR_TUPLE_INT] = _TAIL_CALL_RECORD_PREVIOUS_INST, + [BINARY_OP_SUBTRACT_FLOAT] = _TAIL_CALL_RECORD_PREVIOUS_INST, + [BINARY_OP_SUBTRACT_INT] = _TAIL_CALL_RECORD_PREVIOUS_INST, + [BINARY_SLICE] = _TAIL_CALL_RECORD_PREVIOUS_INST, + [BUILD_INTERPOLATION] = _TAIL_CALL_RECORD_PREVIOUS_INST, + [BUILD_LIST] = _TAIL_CALL_RECORD_PREVIOUS_INST, + [BUILD_MAP] = _TAIL_CALL_RECORD_PREVIOUS_INST, + [BUILD_SET] = _TAIL_CALL_RECORD_PREVIOUS_INST, + [BUILD_SLICE] = _TAIL_CALL_RECORD_PREVIOUS_INST, + [BUILD_STRING] = _TAIL_CALL_RECORD_PREVIOUS_INST, + [BUILD_TEMPLATE] = _TAIL_CALL_RECORD_PREVIOUS_INST, + [BUILD_TUPLE] = _TAIL_CALL_RECORD_PREVIOUS_INST, + [CACHE] = _TAIL_CALL_RECORD_PREVIOUS_INST, + [CALL] = _TAIL_CALL_RECORD_PREVIOUS_INST, + [CALL_ALLOC_AND_ENTER_INIT] = _TAIL_CALL_RECORD_PREVIOUS_INST, + [CALL_BOUND_METHOD_EXACT_ARGS] = _TAIL_CALL_RECORD_PREVIOUS_INST, + [CALL_BOUND_METHOD_GENERAL] = _TAIL_CALL_RECORD_PREVIOUS_INST, + [CALL_BUILTIN_CLASS] = _TAIL_CALL_RECORD_PREVIOUS_INST, + [CALL_BUILTIN_FAST] = _TAIL_CALL_RECORD_PREVIOUS_INST, + [CALL_BUILTIN_FAST_WITH_KEYWORDS] = _TAIL_CALL_RECORD_PREVIOUS_INST, + [CALL_BUILTIN_O] = _TAIL_CALL_RECORD_PREVIOUS_INST, + [CALL_FUNCTION_EX] = _TAIL_CALL_RECORD_PREVIOUS_INST, + [CALL_INTRINSIC_1] = _TAIL_CALL_RECORD_PREVIOUS_INST, + [CALL_INTRINSIC_2] = _TAIL_CALL_RECORD_PREVIOUS_INST, + [CALL_ISINSTANCE] = _TAIL_CALL_RECORD_PREVIOUS_INST, + [CALL_KW] = _TAIL_CALL_RECORD_PREVIOUS_INST, + [CALL_KW_BOUND_METHOD] = _TAIL_CALL_RECORD_PREVIOUS_INST, + [CALL_KW_NON_PY] = _TAIL_CALL_RECORD_PREVIOUS_INST, + [CALL_KW_PY] = _TAIL_CALL_RECORD_PREVIOUS_INST, + [CALL_LEN] = _TAIL_CALL_RECORD_PREVIOUS_INST, + [CALL_LIST_APPEND] = _TAIL_CALL_RECORD_PREVIOUS_INST, + [CALL_METHOD_DESCRIPTOR_FAST] = _TAIL_CALL_RECORD_PREVIOUS_INST, + [CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS] = _TAIL_CALL_RECORD_PREVIOUS_INST, + [CALL_METHOD_DESCRIPTOR_NOARGS] = _TAIL_CALL_RECORD_PREVIOUS_INST, + [CALL_METHOD_DESCRIPTOR_O] = _TAIL_CALL_RECORD_PREVIOUS_INST, + [CALL_NON_PY_GENERAL] = _TAIL_CALL_RECORD_PREVIOUS_INST, + [CALL_PY_EXACT_ARGS] = _TAIL_CALL_RECORD_PREVIOUS_INST, + [CALL_PY_GENERAL] = _TAIL_CALL_RECORD_PREVIOUS_INST, + [CALL_STR_1] = _TAIL_CALL_RECORD_PREVIOUS_INST, + [CALL_TUPLE_1] = _TAIL_CALL_RECORD_PREVIOUS_INST, + [CALL_TYPE_1] = _TAIL_CALL_RECORD_PREVIOUS_INST, + [CHECK_EG_MATCH] = _TAIL_CALL_RECORD_PREVIOUS_INST, + [CHECK_EXC_MATCH] = _TAIL_CALL_RECORD_PREVIOUS_INST, + [CLEANUP_THROW] = _TAIL_CALL_RECORD_PREVIOUS_INST, + [COMPARE_OP] = _TAIL_CALL_RECORD_PREVIOUS_INST, + [COMPARE_OP_FLOAT] = _TAIL_CALL_RECORD_PREVIOUS_INST, + [COMPARE_OP_INT] = _TAIL_CALL_RECORD_PREVIOUS_INST, + [COMPARE_OP_STR] = _TAIL_CALL_RECORD_PREVIOUS_INST, + [CONTAINS_OP] = _TAIL_CALL_RECORD_PREVIOUS_INST, + [CONTAINS_OP_DICT] = _TAIL_CALL_RECORD_PREVIOUS_INST, + [CONTAINS_OP_SET] = _TAIL_CALL_RECORD_PREVIOUS_INST, + [CONVERT_VALUE] = _TAIL_CALL_RECORD_PREVIOUS_INST, + [COPY] = _TAIL_CALL_RECORD_PREVIOUS_INST, + [COPY_FREE_VARS] = _TAIL_CALL_RECORD_PREVIOUS_INST, + [DELETE_ATTR] = _TAIL_CALL_RECORD_PREVIOUS_INST, + [DELETE_DEREF] = _TAIL_CALL_RECORD_PREVIOUS_INST, + [DELETE_FAST] = _TAIL_CALL_RECORD_PREVIOUS_INST, + [DELETE_GLOBAL] = _TAIL_CALL_RECORD_PREVIOUS_INST, + [DELETE_NAME] = _TAIL_CALL_RECORD_PREVIOUS_INST, + [DELETE_SUBSCR] = _TAIL_CALL_RECORD_PREVIOUS_INST, + [DICT_MERGE] = _TAIL_CALL_RECORD_PREVIOUS_INST, + [DICT_UPDATE] = _TAIL_CALL_RECORD_PREVIOUS_INST, + [END_ASYNC_FOR] = _TAIL_CALL_RECORD_PREVIOUS_INST, + [END_FOR] = _TAIL_CALL_RECORD_PREVIOUS_INST, + [END_SEND] = _TAIL_CALL_RECORD_PREVIOUS_INST, + [ENTER_EXECUTOR] = _TAIL_CALL_RECORD_PREVIOUS_INST, + [EXIT_INIT_CHECK] = _TAIL_CALL_RECORD_PREVIOUS_INST, + [EXTENDED_ARG] = _TAIL_CALL_RECORD_PREVIOUS_INST, + [FORMAT_SIMPLE] = _TAIL_CALL_RECORD_PREVIOUS_INST, + [FORMAT_WITH_SPEC] = _TAIL_CALL_RECORD_PREVIOUS_INST, + [FOR_ITER] = _TAIL_CALL_RECORD_PREVIOUS_INST, + [FOR_ITER_GEN] = _TAIL_CALL_RECORD_PREVIOUS_INST, + [FOR_ITER_LIST] = _TAIL_CALL_RECORD_PREVIOUS_INST, + [FOR_ITER_RANGE] = _TAIL_CALL_RECORD_PREVIOUS_INST, + [FOR_ITER_TUPLE] = _TAIL_CALL_RECORD_PREVIOUS_INST, + [GET_AITER] = _TAIL_CALL_RECORD_PREVIOUS_INST, + [GET_ANEXT] = _TAIL_CALL_RECORD_PREVIOUS_INST, + [GET_AWAITABLE] = _TAIL_CALL_RECORD_PREVIOUS_INST, + [GET_ITER] = _TAIL_CALL_RECORD_PREVIOUS_INST, + [GET_LEN] = _TAIL_CALL_RECORD_PREVIOUS_INST, + [GET_YIELD_FROM_ITER] = _TAIL_CALL_RECORD_PREVIOUS_INST, + [IMPORT_FROM] = _TAIL_CALL_RECORD_PREVIOUS_INST, + [IMPORT_NAME] = _TAIL_CALL_RECORD_PREVIOUS_INST, + [INSTRUMENTED_CALL] = _TAIL_CALL_RECORD_PREVIOUS_INST, + [INSTRUMENTED_CALL_FUNCTION_EX] = _TAIL_CALL_RECORD_PREVIOUS_INST, + [INSTRUMENTED_CALL_KW] = _TAIL_CALL_RECORD_PREVIOUS_INST, + [INSTRUMENTED_END_ASYNC_FOR] = _TAIL_CALL_RECORD_PREVIOUS_INST, + [INSTRUMENTED_END_FOR] = _TAIL_CALL_RECORD_PREVIOUS_INST, + [INSTRUMENTED_END_SEND] = _TAIL_CALL_RECORD_PREVIOUS_INST, + [INSTRUMENTED_FOR_ITER] = _TAIL_CALL_RECORD_PREVIOUS_INST, + [INSTRUMENTED_INSTRUCTION] = _TAIL_CALL_RECORD_PREVIOUS_INST, + [INSTRUMENTED_JUMP_BACKWARD] = _TAIL_CALL_RECORD_PREVIOUS_INST, + [INSTRUMENTED_JUMP_FORWARD] = _TAIL_CALL_RECORD_PREVIOUS_INST, + [INSTRUMENTED_LINE] = _TAIL_CALL_RECORD_PREVIOUS_INST, + [INSTRUMENTED_LOAD_SUPER_ATTR] = _TAIL_CALL_RECORD_PREVIOUS_INST, + [INSTRUMENTED_NOT_TAKEN] = _TAIL_CALL_RECORD_PREVIOUS_INST, + [INSTRUMENTED_POP_ITER] = _TAIL_CALL_RECORD_PREVIOUS_INST, + [INSTRUMENTED_POP_JUMP_IF_FALSE] = _TAIL_CALL_RECORD_PREVIOUS_INST, + [INSTRUMENTED_POP_JUMP_IF_NONE] = _TAIL_CALL_RECORD_PREVIOUS_INST, + [INSTRUMENTED_POP_JUMP_IF_NOT_NONE] = _TAIL_CALL_RECORD_PREVIOUS_INST, + [INSTRUMENTED_POP_JUMP_IF_TRUE] = _TAIL_CALL_RECORD_PREVIOUS_INST, + [INSTRUMENTED_RESUME] = _TAIL_CALL_RECORD_PREVIOUS_INST, + [INSTRUMENTED_RETURN_VALUE] = _TAIL_CALL_RECORD_PREVIOUS_INST, + [INSTRUMENTED_YIELD_VALUE] = _TAIL_CALL_RECORD_PREVIOUS_INST, + [INTERPRETER_EXIT] = _TAIL_CALL_RECORD_PREVIOUS_INST, + [IS_OP] = _TAIL_CALL_RECORD_PREVIOUS_INST, + [JUMP_BACKWARD] = _TAIL_CALL_RECORD_PREVIOUS_INST, + [JUMP_BACKWARD_JIT] = _TAIL_CALL_RECORD_PREVIOUS_INST, + [JUMP_BACKWARD_NO_INTERRUPT] = _TAIL_CALL_RECORD_PREVIOUS_INST, + [JUMP_BACKWARD_NO_JIT] = _TAIL_CALL_RECORD_PREVIOUS_INST, + [JUMP_FORWARD] = _TAIL_CALL_RECORD_PREVIOUS_INST, + [LIST_APPEND] = _TAIL_CALL_RECORD_PREVIOUS_INST, + [LIST_EXTEND] = _TAIL_CALL_RECORD_PREVIOUS_INST, + [LOAD_ATTR] = _TAIL_CALL_RECORD_PREVIOUS_INST, + [LOAD_ATTR_CLASS] = _TAIL_CALL_RECORD_PREVIOUS_INST, + [LOAD_ATTR_CLASS_WITH_METACLASS_CHECK] = _TAIL_CALL_RECORD_PREVIOUS_INST, + [LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN] = _TAIL_CALL_RECORD_PREVIOUS_INST, + [LOAD_ATTR_INSTANCE_VALUE] = _TAIL_CALL_RECORD_PREVIOUS_INST, + [LOAD_ATTR_METHOD_LAZY_DICT] = _TAIL_CALL_RECORD_PREVIOUS_INST, + [LOAD_ATTR_METHOD_NO_DICT] = _TAIL_CALL_RECORD_PREVIOUS_INST, + [LOAD_ATTR_METHOD_WITH_VALUES] = _TAIL_CALL_RECORD_PREVIOUS_INST, + [LOAD_ATTR_MODULE] = _TAIL_CALL_RECORD_PREVIOUS_INST, + [LOAD_ATTR_NONDESCRIPTOR_NO_DICT] = _TAIL_CALL_RECORD_PREVIOUS_INST, + [LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES] = _TAIL_CALL_RECORD_PREVIOUS_INST, + [LOAD_ATTR_PROPERTY] = _TAIL_CALL_RECORD_PREVIOUS_INST, + [LOAD_ATTR_SLOT] = _TAIL_CALL_RECORD_PREVIOUS_INST, + [LOAD_ATTR_WITH_HINT] = _TAIL_CALL_RECORD_PREVIOUS_INST, + [LOAD_BUILD_CLASS] = _TAIL_CALL_RECORD_PREVIOUS_INST, + [LOAD_COMMON_CONSTANT] = _TAIL_CALL_RECORD_PREVIOUS_INST, + [LOAD_CONST] = _TAIL_CALL_RECORD_PREVIOUS_INST, + [LOAD_DEREF] = _TAIL_CALL_RECORD_PREVIOUS_INST, + [LOAD_FAST] = _TAIL_CALL_RECORD_PREVIOUS_INST, + [LOAD_FAST_AND_CLEAR] = _TAIL_CALL_RECORD_PREVIOUS_INST, + [LOAD_FAST_BORROW] = _TAIL_CALL_RECORD_PREVIOUS_INST, + [LOAD_FAST_BORROW_LOAD_FAST_BORROW] = _TAIL_CALL_RECORD_PREVIOUS_INST, + [LOAD_FAST_CHECK] = _TAIL_CALL_RECORD_PREVIOUS_INST, + [LOAD_FAST_LOAD_FAST] = _TAIL_CALL_RECORD_PREVIOUS_INST, + [LOAD_FROM_DICT_OR_DEREF] = _TAIL_CALL_RECORD_PREVIOUS_INST, + [LOAD_FROM_DICT_OR_GLOBALS] = _TAIL_CALL_RECORD_PREVIOUS_INST, + [LOAD_GLOBAL] = _TAIL_CALL_RECORD_PREVIOUS_INST, + [LOAD_GLOBAL_BUILTIN] = _TAIL_CALL_RECORD_PREVIOUS_INST, + [LOAD_GLOBAL_MODULE] = _TAIL_CALL_RECORD_PREVIOUS_INST, + [LOAD_LOCALS] = _TAIL_CALL_RECORD_PREVIOUS_INST, + [LOAD_NAME] = _TAIL_CALL_RECORD_PREVIOUS_INST, + [LOAD_SMALL_INT] = _TAIL_CALL_RECORD_PREVIOUS_INST, + [LOAD_SPECIAL] = _TAIL_CALL_RECORD_PREVIOUS_INST, + [LOAD_SUPER_ATTR] = _TAIL_CALL_RECORD_PREVIOUS_INST, + [LOAD_SUPER_ATTR_ATTR] = _TAIL_CALL_RECORD_PREVIOUS_INST, + [LOAD_SUPER_ATTR_METHOD] = _TAIL_CALL_RECORD_PREVIOUS_INST, + [MAKE_CELL] = _TAIL_CALL_RECORD_PREVIOUS_INST, + [MAKE_FUNCTION] = _TAIL_CALL_RECORD_PREVIOUS_INST, + [MAP_ADD] = _TAIL_CALL_RECORD_PREVIOUS_INST, + [MATCH_CLASS] = _TAIL_CALL_RECORD_PREVIOUS_INST, + [MATCH_KEYS] = _TAIL_CALL_RECORD_PREVIOUS_INST, + [MATCH_MAPPING] = _TAIL_CALL_RECORD_PREVIOUS_INST, + [MATCH_SEQUENCE] = _TAIL_CALL_RECORD_PREVIOUS_INST, + [NOP] = _TAIL_CALL_RECORD_PREVIOUS_INST, + [NOT_TAKEN] = _TAIL_CALL_RECORD_PREVIOUS_INST, + [POP_EXCEPT] = _TAIL_CALL_RECORD_PREVIOUS_INST, + [POP_ITER] = _TAIL_CALL_RECORD_PREVIOUS_INST, + [POP_JUMP_IF_FALSE] = _TAIL_CALL_RECORD_PREVIOUS_INST, + [POP_JUMP_IF_NONE] = _TAIL_CALL_RECORD_PREVIOUS_INST, + [POP_JUMP_IF_NOT_NONE] = _TAIL_CALL_RECORD_PREVIOUS_INST, + [POP_JUMP_IF_TRUE] = _TAIL_CALL_RECORD_PREVIOUS_INST, + [POP_TOP] = _TAIL_CALL_RECORD_PREVIOUS_INST, + [PUSH_EXC_INFO] = _TAIL_CALL_RECORD_PREVIOUS_INST, + [PUSH_NULL] = _TAIL_CALL_RECORD_PREVIOUS_INST, + [RAISE_VARARGS] = _TAIL_CALL_RECORD_PREVIOUS_INST, + [RECORD_PREVIOUS_INST] = _TAIL_CALL_RECORD_PREVIOUS_INST, + [RERAISE] = _TAIL_CALL_RECORD_PREVIOUS_INST, + [RESERVED] = _TAIL_CALL_RECORD_PREVIOUS_INST, + [RESUME] = _TAIL_CALL_RECORD_PREVIOUS_INST, + [RESUME_CHECK] = _TAIL_CALL_RECORD_PREVIOUS_INST, + [RETURN_GENERATOR] = _TAIL_CALL_RECORD_PREVIOUS_INST, + [RETURN_VALUE] = _TAIL_CALL_RECORD_PREVIOUS_INST, + [SEND] = _TAIL_CALL_RECORD_PREVIOUS_INST, + [SEND_GEN] = _TAIL_CALL_RECORD_PREVIOUS_INST, + [SETUP_ANNOTATIONS] = _TAIL_CALL_RECORD_PREVIOUS_INST, + [SET_ADD] = _TAIL_CALL_RECORD_PREVIOUS_INST, + [SET_FUNCTION_ATTRIBUTE] = _TAIL_CALL_RECORD_PREVIOUS_INST, + [SET_UPDATE] = _TAIL_CALL_RECORD_PREVIOUS_INST, + [STORE_ATTR] = _TAIL_CALL_RECORD_PREVIOUS_INST, + [STORE_ATTR_INSTANCE_VALUE] = _TAIL_CALL_RECORD_PREVIOUS_INST, + [STORE_ATTR_SLOT] = _TAIL_CALL_RECORD_PREVIOUS_INST, + [STORE_ATTR_WITH_HINT] = _TAIL_CALL_RECORD_PREVIOUS_INST, + [STORE_DEREF] = _TAIL_CALL_RECORD_PREVIOUS_INST, + [STORE_FAST] = _TAIL_CALL_RECORD_PREVIOUS_INST, + [STORE_FAST_LOAD_FAST] = _TAIL_CALL_RECORD_PREVIOUS_INST, + [STORE_FAST_STORE_FAST] = _TAIL_CALL_RECORD_PREVIOUS_INST, + [STORE_GLOBAL] = _TAIL_CALL_RECORD_PREVIOUS_INST, + [STORE_NAME] = _TAIL_CALL_RECORD_PREVIOUS_INST, + [STORE_SLICE] = _TAIL_CALL_RECORD_PREVIOUS_INST, + [STORE_SUBSCR] = _TAIL_CALL_RECORD_PREVIOUS_INST, + [STORE_SUBSCR_DICT] = _TAIL_CALL_RECORD_PREVIOUS_INST, + [STORE_SUBSCR_LIST_INT] = _TAIL_CALL_RECORD_PREVIOUS_INST, + [SWAP] = _TAIL_CALL_RECORD_PREVIOUS_INST, + [TO_BOOL] = _TAIL_CALL_RECORD_PREVIOUS_INST, + [TO_BOOL_ALWAYS_TRUE] = _TAIL_CALL_RECORD_PREVIOUS_INST, + [TO_BOOL_BOOL] = _TAIL_CALL_RECORD_PREVIOUS_INST, + [TO_BOOL_INT] = _TAIL_CALL_RECORD_PREVIOUS_INST, + [TO_BOOL_LIST] = _TAIL_CALL_RECORD_PREVIOUS_INST, + [TO_BOOL_NONE] = _TAIL_CALL_RECORD_PREVIOUS_INST, + [TO_BOOL_STR] = _TAIL_CALL_RECORD_PREVIOUS_INST, + [UNARY_INVERT] = _TAIL_CALL_RECORD_PREVIOUS_INST, + [UNARY_NEGATIVE] = _TAIL_CALL_RECORD_PREVIOUS_INST, + [UNARY_NOT] = _TAIL_CALL_RECORD_PREVIOUS_INST, + [UNPACK_EX] = _TAIL_CALL_RECORD_PREVIOUS_INST, + [UNPACK_SEQUENCE] = _TAIL_CALL_RECORD_PREVIOUS_INST, + [UNPACK_SEQUENCE_LIST] = _TAIL_CALL_RECORD_PREVIOUS_INST, + [UNPACK_SEQUENCE_TUPLE] = _TAIL_CALL_RECORD_PREVIOUS_INST, + [UNPACK_SEQUENCE_TWO_TUPLE] = _TAIL_CALL_RECORD_PREVIOUS_INST, + [WITH_EXCEPT_START] = _TAIL_CALL_RECORD_PREVIOUS_INST, + [YIELD_VALUE] = _TAIL_CALL_RECORD_PREVIOUS_INST, [122] = _TAIL_CALL_UNKNOWN_OPCODE, [123] = _TAIL_CALL_UNKNOWN_OPCODE, [124] = _TAIL_CALL_UNKNOWN_OPCODE, diff --git a/Python/optimizer.c b/Python/optimizer.c index 4d9e9e0ae423..a3d847b6eab8 100644 --- a/Python/optimizer.c +++ b/Python/optimizer.c @@ -172,7 +172,7 @@ _PyOptimizer_Optimize( executor->vm_data.code = NULL; } if (chain_depth > 0) { - _PyExitData *prev_exit = tstate->interp->jit_state.previous_exit; + _PyExitData *prev_exit = tstate->interp->jit_state.prev_exit; assert(prev_exit != NULL); prev_exit->executor = executor;; } @@ -554,16 +554,16 @@ int _PyJit_translate_single_bytecode_to_trace( PyThreadState *tstate, _PyInterpreterFrame *frame, - _Py_CODEUNIT *this_instr, - _Py_CODEUNIT *next_instr, - PyCodeObject *old_code, - PyFunctionObject *func, - int old_stack_level, - int opcode, - int oparg, - int jump_taken) + _Py_CODEUNIT *next_instr) { + PyCodeObject *old_code = tstate->interp->jit_state.prev_instr_code; + // Something else finalized the trace. This can happen in multi-threaded scenarios as our trace + // addition from bytecode execution to here is not atomic. + // Though in GIL builds, the GIL protects the rest. + if (old_code == NULL) { + return 0; + } bool progress_needed = (tstate->interp->jit_state.initial_chain_depth % MAX_CHAIN_DEPTH) == 0;; _PyBloomFilter *dependencies = &tstate->interp->jit_state.dependencies; _Py_BloomFilter_Add(dependencies, old_code); @@ -583,6 +583,7 @@ _PyJit_translate_single_bytecode_to_trace( lltrace = *python_lltrace - '0'; // TODO: Parse an int and all that } #endif + _Py_CODEUNIT *this_instr = tstate->interp->jit_state.prev_instr; _Py_CODEUNIT *target_instr = this_instr; uint32_t target = 0; @@ -590,11 +591,17 @@ _PyJit_translate_single_bytecode_to_trace( // Rewind EXTENDED_ARG so that we see the whole thing. // We must point to the first EXTENDED_ARG when deopting. + int oparg = tstate->interp->jit_state.prev_instr_oparg; + int opcode = this_instr->op.code; int rewind_oparg = oparg; while (rewind_oparg > 255) { rewind_oparg >>= 8; target--; } + + bool needs_guard_ip = _PyOpcode_NeedsGuardIp[opcode]; + DPRINTF(2, "%p %d: %s(%d) %d\n", old_code, target, _PyOpcode_OpName[opcode], oparg, needs_guard_ip); + #ifdef Py_DEBUG if (oparg > 255) { assert(_Py_GetBaseCodeUnit(old_code, target).op.code == EXTENDED_ARG); @@ -605,44 +612,52 @@ _PyJit_translate_single_bytecode_to_trace( goto done; } - DPRINTF(2, "%p %d: %s(%d) %d\n", old_code, target, _PyOpcode_OpName[opcode], oparg, progress_needed); + int old_stack_level = tstate->interp->jit_state.prev_instr_oparg; + // Strange control-flow, unsupported opcode, etc. + if (tstate->interp->jit_state.dynamic_jump_taken) { + goto unsupported; + } - bool needs_guard_ip = _PyOpcode_NeedsGuardIp[opcode]; + // This happens when a recursive call happens that we can't trace. Such as Python -> C -> Python calls + // If we haven't guarded the IP, then it's untraceable. + if (frame != tstate->interp->jit_state.prev_instr_frame && !needs_guard_ip) { + goto unsupported; + } - // Strange control-flow, unsupported opcode, etc. - if (jump_taken || - // This happens when a recursive call happens that we can't trace. Such as Python -> C -> Python calls - // If we haven't guarded the IP, then it's untraceable. - (frame != tstate->interp->jit_state.current_frame && !needs_guard_ip) || - (oparg > 0xFFFF) || - // TODO (gh-140277): The constituent use one extra stack slot. So we need to check for headroom. - (opcode == BINARY_OP_SUBSCR_GETITEM && old_stack_level + 1 > old_code->co_stacksize)|| - // Exception stuff, could be handled in the future maybe? - opcode == WITH_EXCEPT_START || opcode == RERAISE || opcode == CLEANUP_THROW || opcode == PUSH_EXC_INFO || - frame->owner >= FRAME_OWNED_BY_INTERPRETER - ) { + if (oparg > 0xFFFF) { + goto unsupported; + } + + // TODO (gh-140277): The constituent use one extra stack slot. So we need to check for headroom. + if (opcode == BINARY_OP_SUBSCR_GETITEM && old_stack_level + 1 > old_code->co_stacksize) { + goto unsupported; + } + + if (opcode == WITH_EXCEPT_START || opcode == RERAISE || opcode == CLEANUP_THROW || opcode == PUSH_EXC_INFO) { + goto unsupported; + } + + if (frame->owner >= FRAME_OWNED_BY_INTERPRETER) { unsupported: - { - // Rewind to previous instruction and replace with _EXIT_TRACE. - _PyUOpInstruction *curr = &trace[trace_length-1]; - while (curr->opcode != _SET_IP && trace_length > 2) { - trace_length--; - curr = &trace[trace_length-1]; - } - assert(curr->opcode == _SET_IP || trace_length == 2); - if (curr->opcode == _SET_IP) { - int32_t old_target = (int32_t)uop_get_target(curr); - curr++; - trace_length++; - curr->opcode = _EXIT_TRACE; - curr->format = UOP_FORMAT_TARGET; - curr->target = old_target; - } - goto done; - } + { + // Rewind to previous instruction and replace with _EXIT_TRACE. + _PyUOpInstruction *curr = &trace[trace_length-1]; + while (curr->opcode != _SET_IP && trace_length > 2) { + trace_length--; + curr = &trace[trace_length-1]; + } + assert(curr->opcode == _SET_IP || trace_length == 2); + if (curr->opcode == _SET_IP) { + int32_t old_target = (int32_t)uop_get_target(curr); + curr++; + trace_length++; + curr->opcode = _EXIT_TRACE; + curr->format = UOP_FORMAT_TARGET; + curr->target = old_target; + } + goto done; } - - tstate->interp->jit_state.current_frame = frame; + } if (opcode == NOP) { return 1; @@ -721,7 +736,9 @@ _PyJit_translate_single_bytecode_to_trace( { if ((next_instr != tstate->interp->jit_state.close_loop_instr) && (next_instr != tstate->interp->jit_state.insert_exec_instr) && - tstate->interp->jit_state.code_curr_size > 5) { + tstate->interp->jit_state.code_curr_size > 5 && + // These are coroutines, and we want to unroll those usually. + opcode != JUMP_BACKWARD_NO_INTERRUPT) { // We encountered a JUMP_BACKWARD but not to the top of our own loop. // We don't want to continue tracing as we might get stuck in the // inner loop. Instead, end the trace where the executor of the @@ -846,9 +863,9 @@ _PyJit_translate_single_bytecode_to_trace( } if (uop == _BINARY_OP_INPLACE_ADD_UNICODE) { assert(i + 1 == nuops); - _Py_CODEUNIT *next_instr = target_instr + 1 + _PyOpcode_Caches[_PyOpcode_Deopt[opcode]]; - assert(next_instr->op.code == STORE_FAST); - operand = next_instr->op.arg; + _Py_CODEUNIT *next = target_instr + 1 + _PyOpcode_Caches[_PyOpcode_Deopt[opcode]]; + assert(next->op.code == STORE_FAST); + operand = next->op.arg; } // All other instructions ADD_TO_TRACE(uop, oparg, operand, target); @@ -889,7 +906,7 @@ full: } void -_PyJit_InitializeTracing(PyThreadState *tstate, _PyInterpreterFrame *frame, _Py_CODEUNIT *insert_exec_instr, _Py_CODEUNIT *close_loop_instr, int curr_stackdepth, int chain_depth, _PyExitData *exit) +_PyJit_InitializeTracing(PyThreadState *tstate, _PyInterpreterFrame *frame, _Py_CODEUNIT *curr_instr, _Py_CODEUNIT *insert_exec_instr, _Py_CODEUNIT *close_loop_instr, int curr_stackdepth, int chain_depth, _PyExitData *exit, int oparg) { PyCodeObject *code = _PyFrame_GetCode(frame); #ifdef Py_DEBUG @@ -914,13 +931,19 @@ _PyJit_InitializeTracing(PyThreadState *tstate, _PyInterpreterFrame *frame, _Py_ tstate->interp->jit_state.close_loop_instr = close_loop_instr; tstate->interp->jit_state.initial_code = (PyCodeObject *)Py_NewRef(code); tstate->interp->jit_state.initial_func = (PyFunctionObject *)Py_NewRef(_PyFrame_GetFunction(frame)); - tstate->interp->jit_state.previous_exit = exit; - _Py_BloomFilter_Init(&tstate->interp->jit_state.dependencies); + tstate->interp->jit_state.prev_exit = exit; tstate->interp->jit_state.initial_stack_depth = curr_stackdepth; tstate->interp->jit_state.initial_chain_depth = chain_depth % MAX_CHAIN_DEPTH; - tstate->interp->jit_state.current_frame = frame; + tstate->interp->jit_state.prev_instr_frame = frame; tstate->interp->jit_state.dependencies_still_valid = true; - tstate->interp->jit_state.last_specialized_instr = NULL; + tstate->interp->jit_state.do_not_specialize = false; + tstate->interp->jit_state.prev_instr_code = (PyCodeObject *)Py_NewRef(_PyFrame_GetCode(frame)); + tstate->interp->jit_state.prev_instr = curr_instr; + tstate->interp->jit_state.prev_instr_frame = frame; + tstate->interp->jit_state.prev_instr_oparg = oparg; + tstate->interp->jit_state.prev_instr_stacklevel = curr_stackdepth; + tstate->interp->jit_state.dynamic_jump_taken = false; + _Py_BloomFilter_Init(&tstate->interp->jit_state.dependencies); } void @@ -928,6 +951,7 @@ _PyJit_FinalizeTracing(PyThreadState *tstate) { Py_CLEAR(tstate->interp->jit_state.initial_code); Py_CLEAR(tstate->interp->jit_state.initial_func); + Py_CLEAR(tstate->interp->jit_state.prev_instr_code); tstate->interp->jit_state.code_curr_size = 2; tstate->interp->jit_state.code_max_size = UOP_MAX_TRACE_LENGTH - 1; } diff --git a/Tools/cases_generator/generators_common.py b/Tools/cases_generator/generators_common.py index 3edefb110834..4a66b7d51180 100644 --- a/Tools/cases_generator/generators_common.py +++ b/Tools/cases_generator/generators_common.py @@ -128,7 +128,6 @@ class Emitter: "DISPATCH": self.dispatch, "INSTRUCTION_SIZE": self.instruction_size, "stack_pointer": self.stack_pointer, - "RECORD_DYNAMIC_JUMP_TAKEN": self.record_dynamic_jump_taken, } self.out = out self.labels = labels @@ -476,19 +475,6 @@ class Emitter: self.out.emit(f" {uop.instruction_size}u ") return True - def record_dynamic_jump_taken( - self, - tkn: Token, - tkn_iter: TokenIterator, - uop: CodeSection, - storage: Storage, - inst: Instruction | None, - ) -> bool: - next(tkn_iter); - next(tkn_iter); - next(tkn_iter); - return True - def _print_storage(self, reason:str, storage: Storage) -> None: if DEBUG: self.out.start_line() diff --git a/Tools/cases_generator/target_generator.py b/Tools/cases_generator/target_generator.py index 8e5dc10f759a..209411ca6f50 100644 --- a/Tools/cases_generator/target_generator.py +++ b/Tools/cases_generator/target_generator.py @@ -34,7 +34,7 @@ def write_opcode_targets(analysis: Analysis, out: CWriter) -> None: targets = ["&&_unknown_opcode,\n"] * 256 for name, op in analysis.opmap.items(): if op < 256: - targets[op] = f"&&TARGET_TRACING_{name},\n" + targets[op] = f"&&TARGET_RECORD_PREVIOUS_INST,\n" out.emit("#if _Py_TIER2\n") out.emit("static void *opcode_tracing_targets_table[256] = {\n") for target in targets: @@ -58,10 +58,9 @@ def write_tailcall_dispatch_table(analysis: Analysis, out: CWriter) -> None: out.emit(f"{function_proto(name)};\n") out.emit("\n") - # Emit function prototypes for opcode handlers. + # Emit function prototypes for opcode handlers. for name in sorted(analysis.instructions.keys()): out.emit(f"{function_proto(name)};\n") - out.emit(f"{function_proto('TRACING_' + name)};\n") out.emit("\n") # Emit unknown opcode handler. @@ -85,7 +84,7 @@ def write_tailcall_dispatch_table(analysis: Analysis, out: CWriter) -> None: # Emit the tracing dispatch table. out.emit("static py_tail_call_funcptr instruction_funcptr_tracing_table[256] = {\n") for name in sorted(analysis.instructions.keys()): - out.emit(f"[{name}] = _TAIL_CALL_TRACING_{name},\n") + out.emit(f"[{name}] = _TAIL_CALL_RECORD_PREVIOUS_INST,\n") named_values = analysis.opmap.values() for rest in range(256): if rest not in named_values: diff --git a/Tools/cases_generator/tier1_generator.py b/Tools/cases_generator/tier1_generator.py index 25a7442a0ef6..a7961f4fbc7c 100644 --- a/Tools/cases_generator/tier1_generator.py +++ b/Tools/cases_generator/tier1_generator.py @@ -216,12 +216,35 @@ def get_popped(inst: Instruction, analysis: Analysis) -> str: stack = get_stack_effect(inst) return (-stack.base_offset).to_c() + +def generate_record_previous_inst(out: CWriter, emitter: Emitter, inst: Instruction) -> None: + out.emit("\n") + out.emit(f"TARGET(RECORD_PREVIOUS_INST) {{\n") + out.emit(f"INSTRUCTION_STATS(RECORD_PREVIOUS_INST);\n") + declare_variables(inst, out) + offset = 1 # The instruction itself + stack = Stack() + for part in inst.parts: + # Only emit braces if more than one uop + insert_braces = len([p for p in inst.parts if isinstance(p, Uop)]) > 1 + reachable, offset, stack = write_uop(part, emitter, offset, stack, inst, insert_braces) + out.start_line() + if reachable: # type: ignore[possibly-undefined] + stack.flush(out) + out.emit(f"DISPATCH();\n") + out.start_line() + out.emit("}") + out.emit("\n") + def generate_tier1_cases( analysis: Analysis, out: CWriter, emitter: Emitter ) -> None: out.emit("\n") for name, inst in sorted(analysis.instructions.items()): out.emit("\n") + if name == "RECORD_PREVIOUS_INST": + generate_record_previous_inst(out, emitter, inst) + continue out.emit(f"TARGET({name}) {{\n") popped = get_popped(inst, analysis) # We need to ifdef it because this breaks platforms diff --git a/Tools/cases_generator/tracer_generator.py b/Tools/cases_generator/tracer_generator.py deleted file mode 100644 index 1b7d9a5acea6..000000000000 --- a/Tools/cases_generator/tracer_generator.py +++ /dev/null @@ -1,222 +0,0 @@ -import argparse - -from analyzer import ( - Analysis, - Instruction, - Uop, - Label, - CodeSection, - Part, - analyze_files, - Skip, - Flush, - analysis_error, - StackItem, -) -from generators_common import ( - DEFAULT_INPUT, - ROOT, - write_header, - type_and_null, - Emitter, - TokenIterator, - always_true, - emit_to, - ReplacementFunctionType, -) -from cwriter import CWriter -from typing import TextIO -from lexer import Token -from stack import Local, Stack, StackError, get_stack_effect, Storage -from tier1_generator import get_popped, declare_variables, write_uop - -DEFAULT_OUTPUT = ROOT / "Python/generated_tracer_cases.c.h" - -class TracerEmitter(Emitter): - out: CWriter - labels: dict[str, Label] - _replacers: dict[str, ReplacementFunctionType] - cannot_escape: bool - - def __init__(self, out: CWriter, labels: dict[str, Label], cannot_escape: bool = False): - super().__init__(out, labels, cannot_escape, jump_prefix="TRACING_") - self._replacers = { - **self._replacers, - "DISPATCH": self.dispatch, - "DISPATCH_INLINED": self.dispatch_inlined, - "DISPATCH_SAME_OPARG": self.dispatch_same_oparg, - } - - def dispatch( - self, - tkn: Token, - tkn_iter: TokenIterator, - uop: CodeSection, - storage: Storage, - inst: Instruction | None, - ) -> bool: - if storage.spilled: - raise analysis_error("stack_pointer needs reloading before dispatch", tkn) - storage.stack.flush(self.out) - self.out.start_line() - self.emit("TRACING_DISPATCH") - return False - - def dispatch_inlined( - self, - tkn: Token, - tkn_iter: TokenIterator, - uop: CodeSection, - storage: Storage, - inst: Instruction | None, - ) -> bool: - if storage.spilled: - raise analysis_error("stack_pointer needs reloading before dispatch", tkn) - storage.stack.flush(self.out) - self.out.start_line() - self.emit("TRACING_DISPATCH_INLINED") - return False - - def dispatch_same_oparg( - self, - tkn: Token, - tkn_iter: TokenIterator, - uop: CodeSection, - storage: Storage, - inst: Instruction | None, - ) -> bool: - if storage.spilled: - raise analysis_error("stack_pointer needs reloading before dispatch", tkn) - storage.stack.flush(self.out) - self.out.start_line() - if isinstance(uop, Uop) and "specializing" in uop.annotations: - self.emit("TRACING_SPECIALIZE_DISPATCH_SAME_OPARG") - else: - self.emit(tkn) - emit_to(self.out, tkn_iter, "SEMI") - return False - - def record_dynamic_jump_taken( - self, - tkn: Token, - tkn_iter: TokenIterator, - uop: CodeSection, - storage: Storage, - inst: Instruction | None, - ) -> bool: - self.out.emit(tkn) - emit_to(self.out, tkn_iter, "SEMI") - self.out.emit(";\n") - return True - -def generate_tier1_tracer_cases( - analysis: Analysis, out: CWriter, emitter: Emitter -) -> None: - out.emit("\n") - for name, inst in sorted(analysis.instructions.items()): - out.emit("\n") - out.emit(f"TRACING_TARGET({name}) {{\n") - out.emit(f"assert(IS_JIT_TRACING());\n") - # We need to ifdef it because this breaks platforms - # without computed gotos/tail calling. - out.emit(f"#if _Py_TAIL_CALL_INTERP\n") - out.emit(f"int opcode = {name};\n") - out.emit(f"(void)(opcode);\n") - out.emit(f"#endif\n") - unused_guard = "(void)this_instr;\n" - if inst.properties.needs_prev: - out.emit(f"_Py_CODEUNIT* const prev_instr = frame->instr_ptr;\n") - if not inst.is_target: - out.emit(f"_Py_CODEUNIT* const this_instr = next_instr;\n") - out.emit(unused_guard) - if not inst.properties.no_save_ip: - out.emit(f"frame->instr_ptr = next_instr;\n") - - out.emit(f"next_instr += {inst.size};\n") - out.emit(f"INSTRUCTION_STATS({name});\n") - if inst.is_target: - out.emit(f"PREDICTED_TRACING_{name}:;\n") - out.emit(f"_Py_CODEUNIT* const this_instr = next_instr - {inst.size};\n") - out.emit(unused_guard) - # This is required so that the predicted ops reflect the correct opcode. - out.emit(f"opcode = {name};\n") - out.emit(f"PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable);\n") - out.emit(f"(void)old_code;\n") - out.emit(f"PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj);\n") - out.emit(f"(void)old_func;\n") - out.emit(f"int _jump_taken = false;\n") - out.emit(f"(void)_jump_taken;\n") - out.emit(f"int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0;\n") - out.emit(f"(void)(_old_stack_level);\n") - if inst.family is not None: - out.emit( - f"static_assert({inst.family.size} == {inst.size-1}" - ', "incorrect cache size");\n' - ) - declare_variables(inst, out) - offset = 1 # The instruction itself - stack = Stack() - for part in inst.parts: - # Only emit braces if more than one uop - insert_braces = len([p for p in inst.parts if isinstance(p, Uop)]) > 1 - reachable, offset, stack = write_uop(part, emitter, offset, stack, inst, insert_braces) - out.start_line() - if reachable: # type: ignore[possibly-undefined] - stack.flush(out) - out.emit(f"TRACING_DISPATCH();\n") - out.start_line() - out.emit("}") - out.emit("\n") - - -def generate_tracer_cases( - analysis: Analysis, out: CWriter -) -> None: - out.emit(f"#ifdef _Py_TIER2 /* BEGIN TRACING INSTRUCTIONS */\n") - generate_tier1_tracer_cases(analysis, out, TracerEmitter(out, analysis.labels)) - out.emit(f"#endif /* END TRACING INSTRUCTIONS */\n") - -def generate_tracer( - filenames: list[str], analysis: Analysis, outfile: TextIO, lines: bool -) -> None: - write_header(__file__, filenames, outfile) - out = CWriter(outfile, 2, lines) - out.emit("#define TIER_ONE 1\n") - out.emit("#define TRACING_JIT 1\n") - generate_tracer_cases(analysis, out) - out.emit("#undef TRACING_JIT\n") - out.emit("#undef TIER_ONE\n") - -# For use in unittest -def generate_tracer_from_files( - filenames: list[str], outfilename: str, lines: bool -) -> None: - data = analyze_files(filenames) - with open(outfilename, "w") as outfile: - generate_tracer(filenames, data, outfile, lines) - - -arg_parser = argparse.ArgumentParser( - description="Generate the code for the interpreter switch.", - formatter_class=argparse.ArgumentDefaultsHelpFormatter, -) - -arg_parser.add_argument( - "-o", "--output", type=str, help="Generated code", default=DEFAULT_OUTPUT -) - -arg_parser.add_argument( - "-l", "--emit-line-directives", help="Emit #line directives", action="store_true" -) - -arg_parser.add_argument( - "input", nargs=argparse.REMAINDER, help="Instruction definition file(s)" -) - -if __name__ == "__main__": - args = arg_parser.parse_args() - if len(args.input) == 0: - args.input.append(DEFAULT_INPUT) - data = analyze_files(args.input) - with open(args.output, "w") as outfile: - generate_tracer(args.input, data, outfile, args.emit_line_directives)