From: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Sat, 18 Oct 2025 20:20:12 +0000 (+0100) Subject: Separate the tracer out into its own file X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=093578cce1858ba5156c0c9ab10449bf32fe569a;p=thirdparty%2FPython%2Fcpython.git Separate the tracer out into its own file --- diff --git a/Makefile.pre.in b/Makefile.pre.in index a2a5f10585d2..4182fdc46e85 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -2139,7 +2139,7 @@ Objects/mimalloc/page.o: $(srcdir)/Objects/mimalloc/page-queue.c regen-cases: \ regen-opcode-ids regen-opcode-targets regen-uop-ids regen-opcode-metadata-py \ regen-generated-cases regen-executor-cases regen-optimizer-cases \ - regen-opcode-metadata regen-uop-metadata + regen-opcode-metadata regen-uop-metadata regen-tracer-cases .PHONY: regen-opcode-ids regen-opcode-ids: @@ -2185,6 +2185,13 @@ regen-optimizer-cases: $(srcdir)/Python/bytecodes.c $(UPDATE_FILE) $(srcdir)/Python/optimizer_cases.c.h $(srcdir)/Python/optimizer_cases.c.h.new +.PHONY: regen-tracer-cases +regen-tracer-cases: + $(PYTHON_FOR_REGEN) $(srcdir)/Tools/cases_generator/tracer_generator.py \ + -o $(srcdir)/Python/generated_tracer_cases.c.h.new $(srcdir)/Python/bytecodes.c + $(UPDATE_FILE) $(srcdir)/Python/generated_tracer_cases.c.h $(srcdir)/Python/generated_tracer_cases.c.h.new + + .PHONY: regen-opcode-metadata regen-opcode-metadata: $(PYTHON_FOR_REGEN) $(srcdir)/Tools/cases_generator/opcode_metadata_generator.py \ diff --git a/Python/bytecodes.c b/Python/bytecodes.c index a4a75008c66c..5b47782ea600 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -2964,13 +2964,6 @@ dummy_func( if (!IS_JIT_TRACING() && backoff_counter_triggers(counter) && this_instr->op.code == JUMP_BACKWARD_JIT && next_instr->op.code != ENTER_EXECUTOR) { - _Py_CODEUNIT *start = this_instr; - /* Back up over EXTENDED_ARGs so optimizer sees the whole instruction */ - int curr_oparg = oparg; - while (curr_oparg > 255) { - curr_oparg >>= 8; - start--; - } if (tstate->interp->jit_tracer_code_buffer == NULL) { tstate->interp->jit_tracer_code_buffer = (_PyUOpInstruction *)_PyObject_VirtualAlloc(UOP_BUFFER_SIZE); if (tstate->interp->jit_tracer_code_buffer == NULL) { @@ -3046,9 +3039,9 @@ dummy_func( } assert(executor != tstate->interp->cold_executor); tstate->jit_exit = NULL; - if (IS_JIT_TRACING()) { - RECORD_TRACE_NO_DISPATCH(); - } + #if TRACING_JIT + RECORD_TRACE_NO_DISPATCH(); + #endif TIER1_TO_TIER2(executor); #else Py_FatalError("ENTER_EXECUTOR is not supported in this build"); diff --git a/Python/ceval.c b/Python/ceval.c index 844ca4297f41..560c0eb364c9 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -998,6 +998,7 @@ add_to_code_trace(PyThreadState *tstate, _PyInterpreterFrame *frame, PyCodeObjec #if _Py_TAIL_CALL_INTERP #include "opcode_targets.h" #include "generated_cases.c.h" +#include "generated_tracer_cases.c.h" #endif #if (defined(__GNUC__) && __GNUC__ >= 10 && !defined(__clang__)) && defined(__x86_64__) @@ -1126,6 +1127,7 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int #else goto start_frame; # include "generated_cases.c.h" + #include "generated_tracer_cases.c.h" #endif diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index 95b966cc3aeb..c47b6bec7b27 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -29,12 +29,6 @@ PREDICTED_BINARY_OP:; _Py_CODEUNIT* const this_instr = next_instr - 6; (void)this_instr; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; _PyStackRef lhs; _PyStackRef rhs; _PyStackRef res; @@ -97,12 +91,6 @@ frame->instr_ptr = next_instr; next_instr += 6; INSTRUCTION_STATS(BINARY_OP_ADD_FLOAT); - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5, "incorrect cache size"); _PyStackRef value; _PyStackRef left; @@ -161,12 +149,6 @@ frame->instr_ptr = next_instr; next_instr += 6; INSTRUCTION_STATS(BINARY_OP_ADD_INT); - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5, "incorrect cache size"); _PyStackRef value; _PyStackRef left; @@ -227,12 +209,6 @@ frame->instr_ptr = next_instr; next_instr += 6; INSTRUCTION_STATS(BINARY_OP_ADD_UNICODE); - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5, "incorrect cache size"); _PyStackRef value; _PyStackRef nos; @@ -293,12 +269,6 @@ frame->instr_ptr = next_instr; next_instr += 6; INSTRUCTION_STATS(BINARY_OP_EXTEND); - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5, "incorrect cache size"); _PyStackRef left; _PyStackRef right; @@ -363,12 +333,6 @@ frame->instr_ptr = next_instr; next_instr += 6; INSTRUCTION_STATS(BINARY_OP_INPLACE_ADD_UNICODE); - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5, "incorrect cache size"); _PyStackRef value; _PyStackRef nos; @@ -452,12 +416,6 @@ frame->instr_ptr = next_instr; next_instr += 6; INSTRUCTION_STATS(BINARY_OP_MULTIPLY_FLOAT); - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5, "incorrect cache size"); _PyStackRef value; _PyStackRef left; @@ -516,12 +474,6 @@ frame->instr_ptr = next_instr; next_instr += 6; INSTRUCTION_STATS(BINARY_OP_MULTIPLY_INT); - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5, "incorrect cache size"); _PyStackRef value; _PyStackRef left; @@ -582,12 +534,6 @@ frame->instr_ptr = next_instr; next_instr += 6; INSTRUCTION_STATS(BINARY_OP_SUBSCR_DICT); - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5, "incorrect cache size"); _PyStackRef nos; _PyStackRef dict_st; @@ -654,12 +600,6 @@ frame->instr_ptr = next_instr; next_instr += 6; INSTRUCTION_STATS(BINARY_OP_SUBSCR_GETITEM); - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5, "incorrect cache size"); _PyStackRef container; _PyStackRef getitem; @@ -744,12 +684,6 @@ frame->instr_ptr = next_instr; next_instr += 6; INSTRUCTION_STATS(BINARY_OP_SUBSCR_LIST_INT); - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5, "incorrect cache size"); _PyStackRef value; _PyStackRef nos; @@ -840,12 +774,6 @@ frame->instr_ptr = next_instr; next_instr += 6; INSTRUCTION_STATS(BINARY_OP_SUBSCR_LIST_SLICE); - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5, "incorrect cache size"); _PyStackRef tos; _PyStackRef nos; @@ -918,12 +846,6 @@ frame->instr_ptr = next_instr; next_instr += 6; INSTRUCTION_STATS(BINARY_OP_SUBSCR_STR_INT); - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5, "incorrect cache size"); _PyStackRef value; _PyStackRef nos; @@ -1002,12 +924,6 @@ frame->instr_ptr = next_instr; next_instr += 6; INSTRUCTION_STATS(BINARY_OP_SUBSCR_TUPLE_INT); - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5, "incorrect cache size"); _PyStackRef value; _PyStackRef nos; @@ -1081,12 +997,6 @@ frame->instr_ptr = next_instr; next_instr += 6; INSTRUCTION_STATS(BINARY_OP_SUBTRACT_FLOAT); - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5, "incorrect cache size"); _PyStackRef value; _PyStackRef left; @@ -1145,12 +1055,6 @@ frame->instr_ptr = next_instr; next_instr += 6; INSTRUCTION_STATS(BINARY_OP_SUBTRACT_INT); - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5, "incorrect cache size"); _PyStackRef value; _PyStackRef left; @@ -1209,12 +1113,6 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(BINARY_SLICE); - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; _PyStackRef container; _PyStackRef start; _PyStackRef stop; @@ -1271,12 +1169,6 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(BUILD_INTERPOLATION); - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; _PyStackRef value; _PyStackRef str; _PyStackRef *format; @@ -1335,12 +1227,6 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(BUILD_LIST); - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; _PyStackRef *values; _PyStackRef list; values = &stack_pointer[-oparg]; @@ -1365,12 +1251,6 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(BUILD_MAP); - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; _PyStackRef *values; _PyStackRef map; values = &stack_pointer[-oparg*2]; @@ -1423,12 +1303,6 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(BUILD_SET); - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; _PyStackRef *values; _PyStackRef set; values = &stack_pointer[-oparg]; @@ -1486,12 +1360,6 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(BUILD_SLICE); - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; _PyStackRef *args; _PyStackRef slice; args = &stack_pointer[-oparg]; @@ -1527,12 +1395,6 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(BUILD_STRING); - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; _PyStackRef *pieces; _PyStackRef str; pieces = &stack_pointer[-oparg]; @@ -1580,12 +1442,6 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(BUILD_TEMPLATE); - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; _PyStackRef strings; _PyStackRef interpolations; _PyStackRef template; @@ -1624,12 +1480,6 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(BUILD_TUPLE); - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; _PyStackRef *values; _PyStackRef tup; values = &stack_pointer[-oparg]; @@ -1652,12 +1502,6 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(CACHE); - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; assert(0 && "Executing a cache."); Py_FatalError("Executing a cache."); DISPATCH(); @@ -1674,12 +1518,6 @@ PREDICTED_CALL:; _Py_CODEUNIT* const this_instr = next_instr - 4; (void)this_instr; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; opcode = CALL; _PyStackRef callable; _PyStackRef self_or_null; @@ -1857,12 +1695,6 @@ frame->instr_ptr = next_instr; next_instr += 4; INSTRUCTION_STATS(CALL_ALLOC_AND_ENTER_INIT); - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); _PyStackRef callable; _PyStackRef self_or_null; @@ -1984,12 +1816,6 @@ frame->instr_ptr = next_instr; next_instr += 4; INSTRUCTION_STATS(CALL_BOUND_METHOD_EXACT_ARGS); - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); _PyStackRef callable; _PyStackRef null; @@ -2133,12 +1959,6 @@ frame->instr_ptr = next_instr; next_instr += 4; INSTRUCTION_STATS(CALL_BOUND_METHOD_GENERAL); - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); _PyStackRef callable; _PyStackRef null; @@ -2267,12 +2087,6 @@ frame->instr_ptr = next_instr; next_instr += 4; INSTRUCTION_STATS(CALL_BUILTIN_CLASS); - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); _PyStackRef callable; _PyStackRef self_or_null; @@ -2378,12 +2192,6 @@ frame->instr_ptr = next_instr; next_instr += 4; INSTRUCTION_STATS(CALL_BUILTIN_FAST); - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); _PyStackRef callable; _PyStackRef self_or_null; @@ -2493,12 +2301,6 @@ frame->instr_ptr = next_instr; next_instr += 4; INSTRUCTION_STATS(CALL_BUILTIN_FAST_WITH_KEYWORDS); - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); _PyStackRef callable; _PyStackRef self_or_null; @@ -2608,12 +2410,6 @@ frame->instr_ptr = next_instr; next_instr += 4; INSTRUCTION_STATS(CALL_BUILTIN_O); - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); _PyStackRef callable; _PyStackRef self_or_null; @@ -2698,12 +2494,6 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(CALL_FUNCTION_EX); - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; opcode = CALL_FUNCTION_EX; _PyStackRef func; _PyStackRef callargs; @@ -2869,12 +2659,6 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(CALL_INTRINSIC_1); - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; _PyStackRef value; _PyStackRef res; value = stack_pointer[-1]; @@ -2905,12 +2689,6 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(CALL_INTRINSIC_2); - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; _PyStackRef value2_st; _PyStackRef value1_st; _PyStackRef res; @@ -2952,12 +2730,6 @@ frame->instr_ptr = next_instr; next_instr += 4; INSTRUCTION_STATS(CALL_ISINSTANCE); - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); _PyStackRef null; _PyStackRef callable; @@ -3035,12 +2807,6 @@ PREDICTED_CALL_KW:; _Py_CODEUNIT* const this_instr = next_instr - 4; (void)this_instr; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; opcode = CALL_KW; _PyStackRef callable; _PyStackRef self_or_null; @@ -3222,12 +2988,6 @@ frame->instr_ptr = next_instr; next_instr += 4; INSTRUCTION_STATS(CALL_KW_BOUND_METHOD); - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; static_assert(INLINE_CACHE_ENTRIES_CALL_KW == 3, "incorrect cache size"); _PyStackRef callable; _PyStackRef null; @@ -3358,12 +3118,6 @@ frame->instr_ptr = next_instr; next_instr += 4; INSTRUCTION_STATS(CALL_KW_NON_PY); - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; opcode = CALL_KW_NON_PY; static_assert(INLINE_CACHE_ENTRIES_CALL_KW == 3, "incorrect cache size"); _PyStackRef callable; @@ -3491,12 +3245,6 @@ frame->instr_ptr = next_instr; next_instr += 4; INSTRUCTION_STATS(CALL_KW_PY); - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; static_assert(INLINE_CACHE_ENTRIES_CALL_KW == 3, "incorrect cache size"); _PyStackRef callable; _PyStackRef self_or_null; @@ -3607,12 +3355,6 @@ frame->instr_ptr = next_instr; next_instr += 4; INSTRUCTION_STATS(CALL_LEN); - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); _PyStackRef null; _PyStackRef callable; @@ -3685,12 +3427,6 @@ frame->instr_ptr = next_instr; next_instr += 4; INSTRUCTION_STATS(CALL_LIST_APPEND); - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); _PyStackRef callable; _PyStackRef nos; @@ -3779,12 +3515,6 @@ frame->instr_ptr = next_instr; next_instr += 4; INSTRUCTION_STATS(CALL_METHOD_DESCRIPTOR_FAST); - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); _PyStackRef callable; _PyStackRef self_or_null; @@ -3906,12 +3636,6 @@ frame->instr_ptr = next_instr; next_instr += 4; INSTRUCTION_STATS(CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS); - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); _PyStackRef callable; _PyStackRef self_or_null; @@ -4035,12 +3759,6 @@ frame->instr_ptr = next_instr; next_instr += 4; INSTRUCTION_STATS(CALL_METHOD_DESCRIPTOR_NOARGS); - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); _PyStackRef callable; _PyStackRef self_or_null; @@ -4134,12 +3852,6 @@ frame->instr_ptr = next_instr; next_instr += 4; INSTRUCTION_STATS(CALL_METHOD_DESCRIPTOR_O); - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); _PyStackRef callable; _PyStackRef self_or_null; @@ -4246,12 +3958,6 @@ frame->instr_ptr = next_instr; next_instr += 4; INSTRUCTION_STATS(CALL_NON_PY_GENERAL); - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; opcode = CALL_NON_PY_GENERAL; static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); _PyStackRef callable; @@ -4367,12 +4073,6 @@ frame->instr_ptr = next_instr; next_instr += 4; INSTRUCTION_STATS(CALL_PY_EXACT_ARGS); - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); _PyStackRef callable; _PyStackRef self_or_null; @@ -4486,12 +4186,6 @@ frame->instr_ptr = next_instr; next_instr += 4; INSTRUCTION_STATS(CALL_PY_GENERAL); - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); _PyStackRef callable; _PyStackRef self_or_null; @@ -4592,12 +4286,6 @@ frame->instr_ptr = next_instr; next_instr += 4; INSTRUCTION_STATS(CALL_STR_1); - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); _PyStackRef null; _PyStackRef callable; @@ -4670,12 +4358,6 @@ frame->instr_ptr = next_instr; next_instr += 4; INSTRUCTION_STATS(CALL_TUPLE_1); - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); _PyStackRef null; _PyStackRef callable; @@ -4748,12 +4430,6 @@ frame->instr_ptr = next_instr; next_instr += 4; INSTRUCTION_STATS(CALL_TYPE_1); - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); _PyStackRef null; _PyStackRef callable; @@ -4807,12 +4483,6 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(CHECK_EG_MATCH); - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; _PyStackRef exc_value_st; _PyStackRef match_type_st; _PyStackRef rest; @@ -4884,12 +4554,6 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(CHECK_EXC_MATCH); - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; _PyStackRef left; _PyStackRef right; _PyStackRef b; @@ -4929,12 +4593,6 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(CLEANUP_THROW); - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; _PyStackRef sub_iter; _PyStackRef last_sent_val; _PyStackRef exc_value_st; @@ -4995,12 +4653,6 @@ PREDICTED_COMPARE_OP:; _Py_CODEUNIT* const this_instr = next_instr - 2; (void)this_instr; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; _PyStackRef left; _PyStackRef right; _PyStackRef res; @@ -5073,12 +4725,6 @@ frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(COMPARE_OP_FLOAT); - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; static_assert(INLINE_CACHE_ENTRIES_COMPARE_OP == 1, "incorrect cache size"); _PyStackRef value; _PyStackRef left; @@ -5134,12 +4780,6 @@ frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(COMPARE_OP_INT); - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; static_assert(INLINE_CACHE_ENTRIES_COMPARE_OP == 1, "incorrect cache size"); _PyStackRef value; _PyStackRef left; @@ -5199,12 +4839,6 @@ frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(COMPARE_OP_STR); - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; static_assert(INLINE_CACHE_ENTRIES_COMPARE_OP == 1, "incorrect cache size"); _PyStackRef value; _PyStackRef nos; @@ -5265,12 +4899,6 @@ PREDICTED_CONTAINS_OP:; _Py_CODEUNIT* const this_instr = next_instr - 2; (void)this_instr; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; _PyStackRef right; _PyStackRef left; _PyStackRef b; @@ -5330,12 +4958,6 @@ frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(CONTAINS_OP_DICT); - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; static_assert(INLINE_CACHE_ENTRIES_CONTAINS_OP == 1, "incorrect cache size"); _PyStackRef tos; _PyStackRef left; @@ -5394,12 +5016,6 @@ frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(CONTAINS_OP_SET); - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; static_assert(INLINE_CACHE_ENTRIES_CONTAINS_OP == 1, "incorrect cache size"); _PyStackRef tos; _PyStackRef left; @@ -5456,12 +5072,6 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(CONVERT_VALUE); - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; _PyStackRef value; _PyStackRef result; value = stack_pointer[-1]; @@ -5494,12 +5104,6 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(COPY); - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; _PyStackRef bottom; _PyStackRef top; bottom = stack_pointer[-1 - (oparg-1)]; @@ -5518,12 +5122,6 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(COPY_FREE_VARS); - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; PyCodeObject *co = _PyFrame_GetCode(frame); assert(PyStackRef_FunctionCheck(frame->f_funcobj)); PyFunctionObject *func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); @@ -5545,12 +5143,6 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(DELETE_ATTR); - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; _PyStackRef owner; owner = stack_pointer[-1]; PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); @@ -5576,12 +5168,6 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(DELETE_DEREF); - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; PyObject *cell = PyStackRef_AsPyObjectBorrow(GETLOCAL(oparg)); PyObject *oldobj = PyCell_SwapTakeRef((PyCellObject *)cell, NULL); if (oldobj == NULL) { @@ -5604,12 +5190,6 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(DELETE_FAST); - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; _PyStackRef v = GETLOCAL(oparg); if (PyStackRef_IsNull(v)) { _PyFrame_SetStackPointer(frame, stack_pointer); @@ -5636,12 +5216,6 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(DELETE_GLOBAL); - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); _PyFrame_SetStackPointer(frame, stack_pointer); int err = PyDict_Pop(GLOBALS(), name, NULL); @@ -5667,12 +5241,6 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(DELETE_NAME); - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); PyObject *ns = LOCALS(); int err; @@ -5705,12 +5273,6 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(DELETE_SUBSCR); - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; _PyStackRef container; _PyStackRef sub; sub = stack_pointer[-1]; @@ -5743,12 +5305,6 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(DICT_MERGE); - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; _PyStackRef callable; _PyStackRef dict; _PyStackRef update; @@ -5788,12 +5344,6 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(DICT_UPDATE); - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; _PyStackRef dict; _PyStackRef update; update = stack_pointer[-1]; @@ -5839,12 +5389,6 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(END_ASYNC_FOR); - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; _PyStackRef awaitable_st; _PyStackRef exc_st; exc_st = stack_pointer[-1]; @@ -5887,12 +5431,6 @@ #endif next_instr += 1; INSTRUCTION_STATS(END_FOR); - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; _PyStackRef value; value = stack_pointer[-1]; stack_pointer += -1; @@ -5911,12 +5449,6 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(END_SEND); - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; _PyStackRef receiver; _PyStackRef value; _PyStackRef val; @@ -5942,12 +5474,6 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(ENTER_EXECUTOR); - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; opcode = ENTER_EXECUTOR; #ifdef _Py_TIER2 PyCodeObject *code = _PyFrame_GetCode(frame); @@ -5967,9 +5493,9 @@ } assert(executor != tstate->interp->cold_executor); tstate->jit_exit = NULL; - if (IS_JIT_TRACING()) { - RECORD_TRACE_NO_DISPATCH(); - } + #if TRACING_JIT + RECORD_TRACE_NO_DISPATCH(); + #endif TIER1_TO_TIER2(executor); #else Py_FatalError("ENTER_EXECUTOR is not supported in this build"); @@ -5985,12 +5511,6 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(EXIT_INIT_CHECK); - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; _PyStackRef should_be_none; should_be_none = stack_pointer[-1]; if (!PyStackRef_IsNone(should_be_none)) { @@ -6014,12 +5534,6 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(EXTENDED_ARG); - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; opcode = EXTENDED_ARG; assert(oparg); opcode = next_instr->op.code; @@ -6036,12 +5550,6 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(FORMAT_SIMPLE); - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; _PyStackRef value; _PyStackRef res; value = stack_pointer[-1]; @@ -6078,12 +5586,6 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(FORMAT_WITH_SPEC); - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; _PyStackRef value; _PyStackRef fmt_spec; _PyStackRef res; @@ -6123,12 +5625,6 @@ PREDICTED_FOR_ITER:; _Py_CODEUNIT* const this_instr = next_instr - 2; (void)this_instr; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; _PyStackRef iter; _PyStackRef null_or_index; _PyStackRef next; @@ -6182,12 +5678,6 @@ frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(FOR_ITER_GEN); - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; static_assert(INLINE_CACHE_ENTRIES_FOR_ITER == 1, "incorrect cache size"); _PyStackRef iter; _PyStackRef gen_frame; @@ -6259,12 +5749,6 @@ frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(FOR_ITER_LIST); - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; static_assert(INLINE_CACHE_ENTRIES_FOR_ITER == 1, "incorrect cache size"); _PyStackRef iter; _PyStackRef null_or_index; @@ -6349,12 +5833,6 @@ frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(FOR_ITER_RANGE); - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; static_assert(INLINE_CACHE_ENTRIES_FOR_ITER == 1, "incorrect cache size"); _PyStackRef iter; _PyStackRef next; @@ -6422,12 +5900,6 @@ frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(FOR_ITER_TUPLE); - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; static_assert(INLINE_CACHE_ENTRIES_FOR_ITER == 1, "incorrect cache size"); _PyStackRef iter; _PyStackRef null_or_index; @@ -6482,12 +5954,6 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(GET_AITER); - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; _PyStackRef obj; _PyStackRef iter; obj = stack_pointer[-1]; @@ -6549,12 +6015,6 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(GET_ANEXT); - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; _PyStackRef aiter; _PyStackRef awaitable; aiter = stack_pointer[-1]; @@ -6579,12 +6039,6 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(GET_AWAITABLE); - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; _PyStackRef iterable; _PyStackRef iter; iterable = stack_pointer[-1]; @@ -6614,12 +6068,6 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(GET_ITER); - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; _PyStackRef iterable; _PyStackRef iter; _PyStackRef index_or_null; @@ -6666,12 +6114,6 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(GET_LEN); - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; _PyStackRef obj; _PyStackRef len; obj = stack_pointer[-1]; @@ -6700,12 +6142,6 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(GET_YIELD_FROM_ITER); - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; _PyStackRef iterable; _PyStackRef iter; iterable = stack_pointer[-1]; @@ -6751,12 +6187,6 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(IMPORT_FROM); - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; _PyStackRef from; _PyStackRef res; from = stack_pointer[-1]; @@ -6782,12 +6212,6 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(IMPORT_NAME); - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; _PyStackRef level; _PyStackRef fromlist; _PyStackRef res; @@ -6829,12 +6253,6 @@ frame->instr_ptr = next_instr; next_instr += 4; INSTRUCTION_STATS(INSTRUMENTED_CALL); - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; opcode = INSTRUMENTED_CALL; _PyStackRef callable; _PyStackRef self_or_null; @@ -7023,12 +6441,6 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(INSTRUMENTED_CALL_FUNCTION_EX); - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; opcode = INSTRUMENTED_CALL_FUNCTION_EX; _PyStackRef func; _PyStackRef callargs; @@ -7196,12 +6608,6 @@ frame->instr_ptr = next_instr; next_instr += 4; INSTRUCTION_STATS(INSTRUMENTED_CALL_KW); - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; opcode = INSTRUMENTED_CALL_KW; _PyStackRef callable; _PyStackRef self_or_null; @@ -7388,12 +6794,6 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(INSTRUMENTED_END_ASYNC_FOR); - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; _PyStackRef awaitable_st; _PyStackRef exc_st; // _MONITOR_END_ASYNC_FOR @@ -7446,12 +6846,6 @@ (void)this_instr; next_instr += 1; INSTRUCTION_STATS(INSTRUMENTED_END_FOR); - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; _PyStackRef receiver; _PyStackRef value; value = stack_pointer[-1]; @@ -7482,12 +6876,6 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(INSTRUMENTED_END_SEND); - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; _PyStackRef receiver; _PyStackRef value; _PyStackRef val; @@ -7522,12 +6910,6 @@ frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(INSTRUMENTED_FOR_ITER); - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; _PyStackRef iter; _PyStackRef null_or_index; _PyStackRef next; @@ -7564,12 +6946,6 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(INSTRUMENTED_INSTRUCTION); - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; opcode = INSTRUMENTED_INSTRUCTION; _PyFrame_SetStackPointer(frame, stack_pointer); int next_opcode = _Py_call_instrumentation_instruction( @@ -7597,12 +6973,6 @@ frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(INSTRUMENTED_JUMP_BACKWARD); - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; /* Skip 1 cache entry */ // _CHECK_PERIODIC { @@ -7630,12 +7000,6 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(INSTRUMENTED_JUMP_FORWARD); - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; INSTRUMENTED_JUMP(this_instr, next_instr + oparg, PY_MONITORING_EVENT_JUMP); DISPATCH(); } @@ -7651,12 +7015,6 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(INSTRUMENTED_LINE); - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; opcode = INSTRUMENTED_LINE; int original_opcode = 0; if (tstate->tracing) { @@ -7696,12 +7054,6 @@ frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(INSTRUMENTED_LOAD_SUPER_ATTR); - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; opcode = INSTRUMENTED_LOAD_SUPER_ATTR; _PyStackRef global_super_st; _PyStackRef class_st; @@ -7823,12 +7175,6 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(INSTRUMENTED_NOT_TAKEN); - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; (void)this_instr; INSTRUMENTED_JUMP(prev_instr, next_instr, PY_MONITORING_EVENT_BRANCH_LEFT); DISPATCH(); @@ -7845,12 +7191,6 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(INSTRUMENTED_POP_ITER); - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; _PyStackRef iter; _PyStackRef index_or_null; index_or_null = stack_pointer[-1]; @@ -7875,12 +7215,6 @@ frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(INSTRUMENTED_POP_JUMP_IF_FALSE); - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; _PyStackRef cond; /* Skip 1 cache entry */ cond = stack_pointer[-1]; @@ -7905,12 +7239,6 @@ frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(INSTRUMENTED_POP_JUMP_IF_NONE); - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; _PyStackRef value; /* Skip 1 cache entry */ value = stack_pointer[-1]; @@ -7942,12 +7270,6 @@ frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(INSTRUMENTED_POP_JUMP_IF_NOT_NONE); - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; _PyStackRef value; /* Skip 1 cache entry */ value = stack_pointer[-1]; @@ -7977,12 +7299,6 @@ frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(INSTRUMENTED_POP_JUMP_IF_TRUE); - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; _PyStackRef cond; /* Skip 1 cache entry */ cond = stack_pointer[-1]; @@ -8007,12 +7323,6 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(INSTRUMENTED_RESUME); - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; // _LOAD_BYTECODE { #ifdef Py_GIL_DISABLED @@ -8093,12 +7403,6 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(INSTRUMENTED_RETURN_VALUE); - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; _PyStackRef val; _PyStackRef retval; _PyStackRef res; @@ -8151,12 +7455,6 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(INSTRUMENTED_YIELD_VALUE); - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; _PyStackRef val; _PyStackRef retval; _PyStackRef value; @@ -8226,12 +7524,6 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(INTERPRETER_EXIT); - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; _PyStackRef retval; retval = stack_pointer[-1]; assert(frame->owner == FRAME_OWNED_BY_INTERPRETER); @@ -8267,12 +7559,6 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(IS_OP); - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; _PyStackRef left; _PyStackRef right; _PyStackRef b; @@ -8309,12 +7595,6 @@ PREDICTED_JUMP_BACKWARD:; _Py_CODEUNIT* const this_instr = next_instr - 2; (void)this_instr; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; /* Skip 1 cache entry */ // _SPECIALIZE_JUMP_BACKWARD { @@ -8355,12 +7635,6 @@ frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(JUMP_BACKWARD_JIT); - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; static_assert(1 == 1, "incorrect cache size"); /* Skip 1 cache entry */ // _CHECK_PERIODIC @@ -8386,12 +7660,6 @@ if (!IS_JIT_TRACING() && backoff_counter_triggers(counter) && this_instr->op.code == JUMP_BACKWARD_JIT && next_instr->op.code != ENTER_EXECUTOR) { - _Py_CODEUNIT *start = this_instr; - int curr_oparg = oparg; - while (curr_oparg > 255) { - curr_oparg >>= 8; - start--; - } if (tstate->interp->jit_tracer_code_buffer == NULL) { _PyFrame_SetStackPointer(frame, stack_pointer); tstate->interp->jit_tracer_code_buffer = (_PyUOpInstruction *)_PyObject_VirtualAlloc(UOP_BUFFER_SIZE); @@ -8420,12 +7688,6 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(JUMP_BACKWARD_NO_INTERRUPT); - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; #if TIER_ONE assert(oparg <= INSTR_OFFSET()); #endif @@ -8441,12 +7703,6 @@ frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(JUMP_BACKWARD_NO_JIT); - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; static_assert(1 == 1, "incorrect cache size"); /* Skip 1 cache entry */ // _CHECK_PERIODIC @@ -8476,12 +7732,6 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(JUMP_FORWARD); - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; JUMPBY(oparg); DISPATCH(); } @@ -8494,12 +7744,6 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(LIST_APPEND); - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; _PyStackRef list; _PyStackRef v; v = stack_pointer[-1]; @@ -8522,12 +7766,6 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(LIST_EXTEND); - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; _PyStackRef list_st; _PyStackRef iterable_st; iterable_st = stack_pointer[-1]; @@ -8578,12 +7816,6 @@ PREDICTED_LOAD_ATTR:; _Py_CODEUNIT* const this_instr = next_instr - 10; (void)this_instr; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; _PyStackRef owner; _PyStackRef *attr; _PyStackRef *self_or_null; @@ -8664,12 +7896,6 @@ frame->instr_ptr = next_instr; next_instr += 10; INSTRUCTION_STATS(LOAD_ATTR_CLASS); - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); _PyStackRef owner; _PyStackRef attr; @@ -8728,12 +7954,6 @@ frame->instr_ptr = next_instr; next_instr += 10; INSTRUCTION_STATS(LOAD_ATTR_CLASS_WITH_METACLASS_CHECK); - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); _PyStackRef owner; _PyStackRef attr; @@ -8802,12 +8022,6 @@ frame->instr_ptr = next_instr; next_instr += 10; INSTRUCTION_STATS(LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN); - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); _PyStackRef owner; /* Skip 1 cache entry */ @@ -8866,12 +8080,6 @@ frame->instr_ptr = next_instr; next_instr += 10; INSTRUCTION_STATS(LOAD_ATTR_INSTANCE_VALUE); - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); _PyStackRef owner; _PyStackRef attr; @@ -8952,12 +8160,6 @@ frame->instr_ptr = next_instr; next_instr += 10; INSTRUCTION_STATS(LOAD_ATTR_METHOD_LAZY_DICT); - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); _PyStackRef owner; _PyStackRef attr; @@ -9014,12 +8216,6 @@ frame->instr_ptr = next_instr; next_instr += 10; INSTRUCTION_STATS(LOAD_ATTR_METHOD_NO_DICT); - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); _PyStackRef owner; _PyStackRef attr; @@ -9066,12 +8262,6 @@ frame->instr_ptr = next_instr; next_instr += 10; INSTRUCTION_STATS(LOAD_ATTR_METHOD_WITH_VALUES); - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); _PyStackRef owner; _PyStackRef attr; @@ -9139,12 +8329,6 @@ frame->instr_ptr = next_instr; next_instr += 10; INSTRUCTION_STATS(LOAD_ATTR_MODULE); - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); _PyStackRef owner; _PyStackRef attr; @@ -9219,12 +8403,6 @@ frame->instr_ptr = next_instr; next_instr += 10; INSTRUCTION_STATS(LOAD_ATTR_NONDESCRIPTOR_NO_DICT); - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); _PyStackRef owner; _PyStackRef attr; @@ -9272,12 +8450,6 @@ frame->instr_ptr = next_instr; next_instr += 10; INSTRUCTION_STATS(LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES); - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); _PyStackRef owner; _PyStackRef attr; @@ -9346,12 +8518,6 @@ frame->instr_ptr = next_instr; next_instr += 10; INSTRUCTION_STATS(LOAD_ATTR_PROPERTY); - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); _PyStackRef owner; _PyStackRef new_frame; @@ -9446,12 +8612,6 @@ frame->instr_ptr = next_instr; next_instr += 10; INSTRUCTION_STATS(LOAD_ATTR_SLOT); - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); _PyStackRef owner; _PyStackRef attr; @@ -9521,12 +8681,6 @@ frame->instr_ptr = next_instr; next_instr += 10; INSTRUCTION_STATS(LOAD_ATTR_WITH_HINT); - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); _PyStackRef owner; _PyStackRef attr; @@ -9635,12 +8789,6 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(LOAD_BUILD_CLASS); - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; _PyStackRef bc; PyObject *bc_o; _PyFrame_SetStackPointer(frame, stack_pointer); @@ -9671,12 +8819,6 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(LOAD_COMMON_CONSTANT); - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; _PyStackRef value; assert(oparg < NUM_COMMON_CONSTANTS); value = PyStackRef_FromPyObjectNew(tstate->interp->common_consts[oparg]); @@ -9694,12 +8836,6 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(LOAD_CONST); - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; _PyStackRef value; PyObject *obj = GETITEM(FRAME_CO_CONSTS, oparg); value = PyStackRef_FromPyObjectBorrow(obj); @@ -9717,12 +8853,6 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(LOAD_DEREF); - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; _PyStackRef value; PyCellObject *cell = (PyCellObject *)PyStackRef_AsPyObjectBorrow(GETLOCAL(oparg)); _PyFrame_SetStackPointer(frame, stack_pointer); @@ -9751,12 +8881,6 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(LOAD_FAST); - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; _PyStackRef value; assert(!PyStackRef_IsNull(GETLOCAL(oparg))); value = PyStackRef_DUP(GETLOCAL(oparg)); @@ -9774,12 +8898,6 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(LOAD_FAST_AND_CLEAR); - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; _PyStackRef value; value = GETLOCAL(oparg); GETLOCAL(oparg) = PyStackRef_NULL; @@ -9797,12 +8915,6 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(LOAD_FAST_BORROW); - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; _PyStackRef value; assert(!PyStackRef_IsNull(GETLOCAL(oparg))); value = PyStackRef_Borrow(GETLOCAL(oparg)); @@ -9820,12 +8932,6 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(LOAD_FAST_BORROW_LOAD_FAST_BORROW); - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; _PyStackRef value1; _PyStackRef value2; uint32_t oparg1 = oparg >> 4; @@ -9847,12 +8953,6 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(LOAD_FAST_CHECK); - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; _PyStackRef value; _PyStackRef value_s = GETLOCAL(oparg); if (PyStackRef_IsNull(value_s)) { @@ -9879,12 +8979,6 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(LOAD_FAST_LOAD_FAST); - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; _PyStackRef value1; _PyStackRef value2; uint32_t oparg1 = oparg >> 4; @@ -9906,12 +9000,6 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(LOAD_FROM_DICT_OR_DEREF); - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; _PyStackRef class_dict_st; _PyStackRef value; class_dict_st = stack_pointer[-1]; @@ -9957,12 +9045,6 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(LOAD_FROM_DICT_OR_GLOBALS); - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; _PyStackRef mod_or_class_dict; _PyStackRef v; mod_or_class_dict = stack_pointer[-1]; @@ -10041,12 +9123,6 @@ PREDICTED_LOAD_GLOBAL:; _Py_CODEUNIT* const this_instr = next_instr - 5; (void)this_instr; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; _PyStackRef *res; _PyStackRef *null; // _SPECIALIZE_LOAD_GLOBAL @@ -10102,12 +9178,6 @@ frame->instr_ptr = next_instr; next_instr += 5; INSTRUCTION_STATS(LOAD_GLOBAL_BUILTIN); - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; static_assert(INLINE_CACHE_ENTRIES_LOAD_GLOBAL == 4, "incorrect cache size"); _PyStackRef res; _PyStackRef *null; @@ -10188,12 +9258,6 @@ frame->instr_ptr = next_instr; next_instr += 5; INSTRUCTION_STATS(LOAD_GLOBAL_MODULE); - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; static_assert(INLINE_CACHE_ENTRIES_LOAD_GLOBAL == 4, "incorrect cache size"); _PyStackRef res; _PyStackRef *null; @@ -10259,12 +9323,6 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(LOAD_LOCALS); - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; _PyStackRef locals; PyObject *l = LOCALS(); if (l == NULL) { @@ -10289,12 +9347,6 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(LOAD_NAME); - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; _PyStackRef v; PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); _PyFrame_SetStackPointer(frame, stack_pointer); @@ -10318,12 +9370,6 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(LOAD_SMALL_INT); - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; _PyStackRef value; assert(oparg < _PY_NSMALLPOSINTS); PyObject *obj = (PyObject *)&_PyLong_SMALL_INTS[_PY_NSMALLNEGINTS + oparg]; @@ -10342,12 +9388,6 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(LOAD_SPECIAL); - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; _PyStackRef self; _PyStackRef *method_and_self; // _INSERT_NULL @@ -10397,12 +9437,6 @@ PREDICTED_LOAD_SUPER_ATTR:; _Py_CODEUNIT* const this_instr = next_instr - 2; (void)this_instr; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; opcode = LOAD_SUPER_ATTR; _PyStackRef global_super_st; _PyStackRef class_st; @@ -10539,12 +9573,6 @@ frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(LOAD_SUPER_ATTR_ATTR); - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; static_assert(INLINE_CACHE_ENTRIES_LOAD_SUPER_ATTR == 1, "incorrect cache size"); _PyStackRef global_super_st; _PyStackRef class_st; @@ -10607,12 +9635,6 @@ frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(LOAD_SUPER_ATTR_METHOD); - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; static_assert(INLINE_CACHE_ENTRIES_LOAD_SUPER_ATTR == 1, "incorrect cache size"); _PyStackRef global_super_st; _PyStackRef class_st; @@ -10689,12 +9711,6 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(MAKE_CELL); - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; PyObject *initial = PyStackRef_AsPyObjectBorrow(GETLOCAL(oparg)); PyObject *cell = PyCell_New(initial); if (cell == NULL) { @@ -10716,12 +9732,6 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(MAKE_FUNCTION); - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; _PyStackRef codeobj_st; _PyStackRef func; codeobj_st = stack_pointer[-1]; @@ -10755,12 +9765,6 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(MAP_ADD); - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; _PyStackRef dict_st; _PyStackRef key; _PyStackRef value; @@ -10792,12 +9796,6 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(MATCH_CLASS); - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; _PyStackRef subject; _PyStackRef type; _PyStackRef names; @@ -10850,12 +9848,6 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(MATCH_KEYS); - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; _PyStackRef subject; _PyStackRef keys; _PyStackRef values_or_none; @@ -10883,12 +9875,6 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(MATCH_MAPPING); - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; _PyStackRef subject; _PyStackRef res; subject = stack_pointer[-1]; @@ -10908,12 +9894,6 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(MATCH_SEQUENCE); - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; _PyStackRef subject; _PyStackRef res; subject = stack_pointer[-1]; @@ -10933,12 +9913,6 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(NOP); - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; DISPATCH(); } @@ -10950,12 +9924,6 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(NOT_TAKEN); - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; DISPATCH(); } @@ -10967,12 +9935,6 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(POP_EXCEPT); - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; _PyStackRef exc_value; exc_value = stack_pointer[-1]; _PyErr_StackItem *exc_info = tstate->exc_info; @@ -10994,12 +9956,6 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(POP_ITER); - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; _PyStackRef iter; _PyStackRef index_or_null; index_or_null = stack_pointer[-1]; @@ -11021,12 +9977,6 @@ frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(POP_JUMP_IF_FALSE); - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; _PyStackRef cond; /* Skip 1 cache entry */ cond = stack_pointer[-1]; @@ -11046,12 +9996,6 @@ frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(POP_JUMP_IF_NONE); - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; _PyStackRef value; _PyStackRef b; _PyStackRef cond; @@ -11092,12 +10036,6 @@ frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(POP_JUMP_IF_NOT_NONE); - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; _PyStackRef value; _PyStackRef b; _PyStackRef cond; @@ -11138,12 +10076,6 @@ frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(POP_JUMP_IF_TRUE); - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; _PyStackRef cond; /* Skip 1 cache entry */ cond = stack_pointer[-1]; @@ -11163,12 +10095,6 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(POP_TOP); - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; _PyStackRef value; value = stack_pointer[-1]; stack_pointer += -1; @@ -11187,12 +10113,6 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(PUSH_EXC_INFO); - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; _PyStackRef exc; _PyStackRef prev_exc; _PyStackRef new_exc; @@ -11222,12 +10142,6 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(PUSH_NULL); - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; _PyStackRef res; res = PyStackRef_NULL; stack_pointer[0] = res; @@ -11246,12 +10160,6 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(RAISE_VARARGS); - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; _PyStackRef *args; args = &stack_pointer[-oparg]; assert(oparg < 3); @@ -11281,12 +10189,6 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(RERAISE); - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; _PyStackRef *values; _PyStackRef exc_st; exc_st = stack_pointer[-1]; @@ -11313,12 +10215,6 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(RESERVED); - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; assert(0 && "Executing RESERVED instruction."); Py_FatalError("Executing RESERVED instruction."); DISPATCH(); @@ -11335,12 +10231,6 @@ PREDICTED_RESUME:; _Py_CODEUNIT* const this_instr = next_instr - 1; (void)this_instr; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; // _LOAD_BYTECODE { #ifdef Py_GIL_DISABLED @@ -11416,12 +10306,6 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(RESUME_CHECK); - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; static_assert(0 == 0, "incorrect cache size"); #if defined(__EMSCRIPTEN__) if (_Py_emscripten_signal_clock == 0) { @@ -11458,12 +10342,6 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(RETURN_GENERATOR); - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; _PyStackRef res; assert(PyStackRef_FunctionCheck(frame->f_funcobj)); PyFunctionObject *func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); @@ -11506,12 +10384,6 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(RETURN_VALUE); - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; _PyStackRef retval; _PyStackRef res; retval = stack_pointer[-1]; @@ -11549,12 +10421,6 @@ PREDICTED_SEND:; _Py_CODEUNIT* const this_instr = next_instr - 2; (void)this_instr; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; _PyStackRef receiver; _PyStackRef v; _PyStackRef retval; @@ -11659,12 +10525,6 @@ frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(SEND_GEN); - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; static_assert(INLINE_CACHE_ENTRIES_SEND == 1, "incorrect cache size"); _PyStackRef receiver; _PyStackRef v; @@ -11732,12 +10592,6 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(SETUP_ANNOTATIONS); - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; PyObject *ann_dict; if (LOCALS() == NULL) { _PyFrame_SetStackPointer(frame, stack_pointer); @@ -11784,12 +10638,6 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(SET_ADD); - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; _PyStackRef set; _PyStackRef v; v = stack_pointer[-1]; @@ -11814,12 +10662,6 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(SET_FUNCTION_ATTRIBUTE); - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; _PyStackRef attr_st; _PyStackRef func_in; _PyStackRef func_out; @@ -11848,12 +10690,6 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(SET_UPDATE); - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; _PyStackRef set; _PyStackRef iterable; iterable = stack_pointer[-1]; @@ -11884,12 +10720,6 @@ PREDICTED_STORE_ATTR:; _Py_CODEUNIT* const this_instr = next_instr - 5; (void)this_instr; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; _PyStackRef owner; _PyStackRef v; // _SPECIALIZE_STORE_ATTR @@ -11946,12 +10776,6 @@ frame->instr_ptr = next_instr; next_instr += 5; INSTRUCTION_STATS(STORE_ATTR_INSTANCE_VALUE); - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; static_assert(INLINE_CACHE_ENTRIES_STORE_ATTR == 4, "incorrect cache size"); _PyStackRef owner; _PyStackRef value; @@ -12028,12 +10852,6 @@ frame->instr_ptr = next_instr; next_instr += 5; INSTRUCTION_STATS(STORE_ATTR_SLOT); - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; static_assert(INLINE_CACHE_ENTRIES_STORE_ATTR == 4, "incorrect cache size"); _PyStackRef owner; _PyStackRef value; @@ -12085,12 +10903,6 @@ frame->instr_ptr = next_instr; next_instr += 5; INSTRUCTION_STATS(STORE_ATTR_WITH_HINT); - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; static_assert(INLINE_CACHE_ENTRIES_STORE_ATTR == 4, "incorrect cache size"); _PyStackRef owner; _PyStackRef value; @@ -12177,12 +10989,6 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(STORE_DEREF); - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; _PyStackRef v; v = stack_pointer[-1]; PyCellObject *cell = (PyCellObject *)PyStackRef_AsPyObjectBorrow(GETLOCAL(oparg)); @@ -12202,12 +11008,6 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(STORE_FAST); - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; _PyStackRef value; value = stack_pointer[-1]; _PyStackRef tmp = GETLOCAL(oparg); @@ -12228,12 +11028,6 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(STORE_FAST_LOAD_FAST); - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; _PyStackRef value1; _PyStackRef value2; value1 = stack_pointer[-1]; @@ -12257,12 +11051,6 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(STORE_FAST_STORE_FAST); - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; _PyStackRef value2; _PyStackRef value1; value1 = stack_pointer[-1]; @@ -12294,12 +11082,6 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(STORE_GLOBAL); - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; _PyStackRef v; v = stack_pointer[-1]; PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); @@ -12325,12 +11107,6 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(STORE_NAME); - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; _PyStackRef v; v = stack_pointer[-1]; PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); @@ -12377,12 +11153,6 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(STORE_SLICE); - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; _PyStackRef v; _PyStackRef container; _PyStackRef start; @@ -12446,12 +11216,6 @@ PREDICTED_STORE_SUBSCR:; _Py_CODEUNIT* const this_instr = next_instr - 2; (void)this_instr; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; _PyStackRef container; _PyStackRef sub; _PyStackRef v; @@ -12510,12 +11274,6 @@ frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(STORE_SUBSCR_DICT); - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; static_assert(INLINE_CACHE_ENTRIES_STORE_SUBSCR == 1, "incorrect cache size"); _PyStackRef nos; _PyStackRef value; @@ -12567,12 +11325,6 @@ frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(STORE_SUBSCR_LIST_INT); - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; static_assert(INLINE_CACHE_ENTRIES_STORE_SUBSCR == 1, "incorrect cache size"); _PyStackRef value; _PyStackRef nos; @@ -12652,12 +11404,6 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(SWAP); - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; _PyStackRef bottom; _PyStackRef top; top = stack_pointer[-1]; @@ -12681,12 +11427,6 @@ PREDICTED_TO_BOOL:; _Py_CODEUNIT* const this_instr = next_instr - 4; (void)this_instr; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; _PyStackRef value; _PyStackRef res; // _SPECIALIZE_TO_BOOL @@ -12738,12 +11478,6 @@ frame->instr_ptr = next_instr; next_instr += 4; INSTRUCTION_STATS(TO_BOOL_ALWAYS_TRUE); - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; static_assert(INLINE_CACHE_ENTRIES_TO_BOOL == 3, "incorrect cache size"); _PyStackRef owner; _PyStackRef value; @@ -12787,12 +11521,6 @@ frame->instr_ptr = next_instr; next_instr += 4; INSTRUCTION_STATS(TO_BOOL_BOOL); - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; static_assert(INLINE_CACHE_ENTRIES_TO_BOOL == 3, "incorrect cache size"); _PyStackRef value; /* Skip 1 cache entry */ @@ -12817,12 +11545,6 @@ frame->instr_ptr = next_instr; next_instr += 4; INSTRUCTION_STATS(TO_BOOL_INT); - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; static_assert(INLINE_CACHE_ENTRIES_TO_BOOL == 3, "incorrect cache size"); _PyStackRef value; _PyStackRef res; @@ -12863,12 +11585,6 @@ frame->instr_ptr = next_instr; next_instr += 4; INSTRUCTION_STATS(TO_BOOL_LIST); - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; static_assert(INLINE_CACHE_ENTRIES_TO_BOOL == 3, "incorrect cache size"); _PyStackRef tos; _PyStackRef value; @@ -12912,12 +11628,6 @@ frame->instr_ptr = next_instr; next_instr += 4; INSTRUCTION_STATS(TO_BOOL_NONE); - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; static_assert(INLINE_CACHE_ENTRIES_TO_BOOL == 3, "incorrect cache size"); _PyStackRef value; _PyStackRef res; @@ -12945,12 +11655,6 @@ frame->instr_ptr = next_instr; next_instr += 4; INSTRUCTION_STATS(TO_BOOL_STR); - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; static_assert(INLINE_CACHE_ENTRIES_TO_BOOL == 3, "incorrect cache size"); _PyStackRef value; _PyStackRef res; @@ -12997,12 +11701,6 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(UNARY_INVERT); - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; _PyStackRef value; _PyStackRef res; value = stack_pointer[-1]; @@ -13032,12 +11730,6 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(UNARY_NEGATIVE); - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; _PyStackRef value; _PyStackRef res; value = stack_pointer[-1]; @@ -13067,12 +11759,6 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(UNARY_NOT); - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; _PyStackRef value; _PyStackRef res; value = stack_pointer[-1]; @@ -13091,12 +11777,6 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(UNPACK_EX); - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; _PyStackRef seq; _PyStackRef *top; seq = stack_pointer[-1]; @@ -13127,12 +11807,6 @@ PREDICTED_UNPACK_SEQUENCE:; _Py_CODEUNIT* const this_instr = next_instr - 2; (void)this_instr; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; _PyStackRef seq; _PyStackRef *top; // _SPECIALIZE_UNPACK_SEQUENCE @@ -13183,12 +11857,6 @@ frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(UNPACK_SEQUENCE_LIST); - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; static_assert(INLINE_CACHE_ENTRIES_UNPACK_SEQUENCE == 1, "incorrect cache size"); _PyStackRef tos; _PyStackRef seq; @@ -13248,12 +11916,6 @@ frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(UNPACK_SEQUENCE_TUPLE); - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; static_assert(INLINE_CACHE_ENTRIES_UNPACK_SEQUENCE == 1, "incorrect cache size"); _PyStackRef tos; _PyStackRef seq; @@ -13304,12 +11966,6 @@ frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(UNPACK_SEQUENCE_TWO_TUPLE); - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; static_assert(INLINE_CACHE_ENTRIES_UNPACK_SEQUENCE == 1, "incorrect cache size"); _PyStackRef tos; _PyStackRef seq; @@ -13359,12 +12015,6 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(WITH_EXCEPT_START); - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; _PyStackRef exit_func; _PyStackRef exit_self; _PyStackRef lasti; @@ -13410,12 +12060,6 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(YIELD_VALUE); - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; _PyStackRef retval; _PyStackRef value; retval = stack_pointer[-1]; @@ -13456,14104 +12100,6 @@ assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } - #ifdef _Py_TIER2 /* BEGIN TRACING INSTRUCTIONS */ - - - TRACING_TARGET(BINARY_OP) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = BINARY_OP; - (void)(opcode); - #endif - frame->instr_ptr = next_instr; - next_instr += 6; - INSTRUCTION_STATS(BINARY_OP); - PREDICTED_TRACING_BINARY_OP:; - _Py_CODEUNIT* const this_instr = next_instr - 6; - (void)this_instr; - opcode = BINARY_OP; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - _PyStackRef lhs; - _PyStackRef rhs; - _PyStackRef res; - // _SPECIALIZE_BINARY_OP - { - rhs = stack_pointer[-1]; - lhs = stack_pointer[-2]; - uint16_t counter = read_u16(&this_instr[1].cache); - (void)counter; - #if ENABLE_SPECIALIZATION_FT - if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { - next_instr = this_instr; - _PyFrame_SetStackPointer(frame, stack_pointer); - _Py_Specialize_BinaryOp(lhs, rhs, next_instr, oparg, LOCALS_ARRAY); - stack_pointer = _PyFrame_GetStackPointer(frame); - DISPATCH_SAME_OPARG(); - } - OPCODE_DEFERRED_INC(BINARY_OP); - ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); - #endif /* ENABLE_SPECIALIZATION_FT */ - assert(NB_ADD <= oparg); - assert(oparg <= NB_OPARG_LAST); - } - /* Skip 4 cache entries */ - // _BINARY_OP - { - PyObject *lhs_o = PyStackRef_AsPyObjectBorrow(lhs); - PyObject *rhs_o = PyStackRef_AsPyObjectBorrow(rhs); - assert(_PyEval_BinaryOps[oparg]); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *res_o = _PyEval_BinaryOps[oparg](lhs_o, rhs_o); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (res_o == NULL) { - TRACING_JUMP_TO_LABEL(error); - } - res = PyStackRef_FromPyObjectSteal(res_o); - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyStackRef tmp = lhs; - lhs = res; - stack_pointer[-2] = lhs; - PyStackRef_CLOSE(tmp); - tmp = rhs; - rhs = PyStackRef_NULL; - stack_pointer[-1] = rhs; - PyStackRef_CLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - } - TRACING_DISPATCH(); - } - - TRACING_TARGET(BINARY_OP_ADD_FLOAT) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = BINARY_OP_ADD_FLOAT; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 6; - INSTRUCTION_STATS(BINARY_OP_ADD_FLOAT); - opcode = BINARY_OP_ADD_FLOAT; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5, "incorrect cache size"); - _PyStackRef value; - _PyStackRef left; - _PyStackRef right; - _PyStackRef res; - // _GUARD_TOS_FLOAT - { - value = stack_pointer[-1]; - PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); - if (!PyFloat_CheckExact(value_o)) { - UPDATE_MISS_STATS(BINARY_OP); - assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); - JUMP_TO_PREDICTED(TRACING_BINARY_OP); - } - } - // _GUARD_NOS_FLOAT - { - left = stack_pointer[-2]; - PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); - if (!PyFloat_CheckExact(left_o)) { - UPDATE_MISS_STATS(BINARY_OP); - assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); - JUMP_TO_PREDICTED(TRACING_BINARY_OP); - } - } - /* Skip 5 cache entries */ - // _BINARY_OP_ADD_FLOAT - { - right = value; - PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); - PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - assert(PyFloat_CheckExact(left_o)); - assert(PyFloat_CheckExact(right_o)); - STAT_INC(BINARY_OP, hit); - double dres = - ((PyFloatObject *)left_o)->ob_fval + - ((PyFloatObject *)right_o)->ob_fval; - res = _PyFloat_FromDouble_ConsumeInputs(left, right, dres); - if (PyStackRef_IsNull(res)) { - TRACING_JUMP_TO_LABEL(pop_2_error); - } - } - stack_pointer[-2] = res; - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - TRACING_DISPATCH(); - } - - TRACING_TARGET(BINARY_OP_ADD_INT) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = BINARY_OP_ADD_INT; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 6; - INSTRUCTION_STATS(BINARY_OP_ADD_INT); - opcode = BINARY_OP_ADD_INT; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5, "incorrect cache size"); - _PyStackRef value; - _PyStackRef left; - _PyStackRef right; - _PyStackRef res; - // _GUARD_TOS_INT - { - value = stack_pointer[-1]; - PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); - if (!_PyLong_CheckExactAndCompact(value_o)) { - UPDATE_MISS_STATS(BINARY_OP); - assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); - JUMP_TO_PREDICTED(TRACING_BINARY_OP); - } - } - // _GUARD_NOS_INT - { - left = stack_pointer[-2]; - PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); - if (!_PyLong_CheckExactAndCompact(left_o)) { - UPDATE_MISS_STATS(BINARY_OP); - assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); - JUMP_TO_PREDICTED(TRACING_BINARY_OP); - } - } - /* Skip 5 cache entries */ - // _BINARY_OP_ADD_INT - { - right = value; - PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); - PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - assert(PyLong_CheckExact(left_o)); - assert(PyLong_CheckExact(right_o)); - assert(_PyLong_BothAreCompact((PyLongObject *)left_o, (PyLongObject *)right_o)); - STAT_INC(BINARY_OP, hit); - res = _PyCompactLong_Add((PyLongObject *)left_o, (PyLongObject *)right_o); - if (PyStackRef_IsNull(res)) { - UPDATE_MISS_STATS(BINARY_OP); - assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); - JUMP_TO_PREDICTED(TRACING_BINARY_OP); - } - PyStackRef_CLOSE_SPECIALIZED(right, _PyLong_ExactDealloc); - PyStackRef_CLOSE_SPECIALIZED(left, _PyLong_ExactDealloc); - } - stack_pointer[-2] = res; - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - TRACING_DISPATCH(); - } - - TRACING_TARGET(BINARY_OP_ADD_UNICODE) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = BINARY_OP_ADD_UNICODE; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 6; - INSTRUCTION_STATS(BINARY_OP_ADD_UNICODE); - opcode = BINARY_OP_ADD_UNICODE; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5, "incorrect cache size"); - _PyStackRef value; - _PyStackRef nos; - _PyStackRef left; - _PyStackRef right; - _PyStackRef res; - // _GUARD_TOS_UNICODE - { - value = stack_pointer[-1]; - PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); - if (!PyUnicode_CheckExact(value_o)) { - UPDATE_MISS_STATS(BINARY_OP); - assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); - JUMP_TO_PREDICTED(TRACING_BINARY_OP); - } - } - // _GUARD_NOS_UNICODE - { - nos = stack_pointer[-2]; - PyObject *o = PyStackRef_AsPyObjectBorrow(nos); - if (!PyUnicode_CheckExact(o)) { - UPDATE_MISS_STATS(BINARY_OP); - assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); - JUMP_TO_PREDICTED(TRACING_BINARY_OP); - } - } - /* Skip 5 cache entries */ - // _BINARY_OP_ADD_UNICODE - { - right = value; - left = nos; - PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); - PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - assert(PyUnicode_CheckExact(left_o)); - assert(PyUnicode_CheckExact(right_o)); - STAT_INC(BINARY_OP, hit); - PyObject *res_o = PyUnicode_Concat(left_o, right_o); - PyStackRef_CLOSE_SPECIALIZED(right, _PyUnicode_ExactDealloc); - PyStackRef_CLOSE_SPECIALIZED(left, _PyUnicode_ExactDealloc); - if (res_o == NULL) { - TRACING_JUMP_TO_LABEL(pop_2_error); - } - res = PyStackRef_FromPyObjectSteal(res_o); - } - stack_pointer[-2] = res; - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - TRACING_DISPATCH(); - } - - TRACING_TARGET(BINARY_OP_EXTEND) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = BINARY_OP_EXTEND; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 6; - INSTRUCTION_STATS(BINARY_OP_EXTEND); - opcode = BINARY_OP_EXTEND; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5, "incorrect cache size"); - _PyStackRef left; - _PyStackRef right; - _PyStackRef res; - /* Skip 1 cache entry */ - // _GUARD_BINARY_OP_EXTEND - { - right = stack_pointer[-1]; - left = stack_pointer[-2]; - PyObject *descr = read_obj(&this_instr[2].cache); - PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); - PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - _PyBinaryOpSpecializationDescr *d = (_PyBinaryOpSpecializationDescr*)descr; - assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5); - assert(d && d->guard); - _PyFrame_SetStackPointer(frame, stack_pointer); - int res = d->guard(left_o, right_o); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (!res) { - UPDATE_MISS_STATS(BINARY_OP); - assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); - JUMP_TO_PREDICTED(TRACING_BINARY_OP); - } - } - /* Skip -4 cache entry */ - // _BINARY_OP_EXTEND - { - PyObject *descr = read_obj(&this_instr[2].cache); - PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); - PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5); - _PyBinaryOpSpecializationDescr *d = (_PyBinaryOpSpecializationDescr*)descr; - STAT_INC(BINARY_OP, hit); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *res_o = d->action(left_o, right_o); - _PyStackRef tmp = right; - right = PyStackRef_NULL; - stack_pointer[-1] = right; - PyStackRef_CLOSE(tmp); - tmp = left; - left = PyStackRef_NULL; - stack_pointer[-2] = left; - PyStackRef_CLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); - res = PyStackRef_FromPyObjectSteal(res_o); - } - stack_pointer[0] = res; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - TRACING_DISPATCH(); - } - - TRACING_TARGET(BINARY_OP_INPLACE_ADD_UNICODE) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = BINARY_OP_INPLACE_ADD_UNICODE; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 6; - INSTRUCTION_STATS(BINARY_OP_INPLACE_ADD_UNICODE); - opcode = BINARY_OP_INPLACE_ADD_UNICODE; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5, "incorrect cache size"); - _PyStackRef value; - _PyStackRef nos; - _PyStackRef left; - _PyStackRef right; - // _GUARD_TOS_UNICODE - { - value = stack_pointer[-1]; - PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); - if (!PyUnicode_CheckExact(value_o)) { - UPDATE_MISS_STATS(BINARY_OP); - assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); - JUMP_TO_PREDICTED(TRACING_BINARY_OP); - } - } - // _GUARD_NOS_UNICODE - { - nos = stack_pointer[-2]; - PyObject *o = PyStackRef_AsPyObjectBorrow(nos); - if (!PyUnicode_CheckExact(o)) { - UPDATE_MISS_STATS(BINARY_OP); - assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); - JUMP_TO_PREDICTED(TRACING_BINARY_OP); - } - } - /* Skip 5 cache entries */ - // _BINARY_OP_INPLACE_ADD_UNICODE - { - right = value; - left = nos; - PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); - assert(PyUnicode_CheckExact(left_o)); - assert(PyUnicode_CheckExact(PyStackRef_AsPyObjectBorrow(right))); - int next_oparg; - #if TIER_ONE - assert(next_instr->op.code == STORE_FAST); - next_oparg = next_instr->op.arg; - #else - next_oparg = (int)CURRENT_OPERAND0(); - #endif - _PyStackRef *target_local = &GETLOCAL(next_oparg); - assert(PyUnicode_CheckExact(left_o)); - if (PyStackRef_AsPyObjectBorrow(*target_local) != left_o) { - UPDATE_MISS_STATS(BINARY_OP); - assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); - JUMP_TO_PREDICTED(TRACING_BINARY_OP); - } - STAT_INC(BINARY_OP, hit); - assert(Py_REFCNT(left_o) >= 2 || !PyStackRef_IsHeapSafe(left)); - PyStackRef_CLOSE_SPECIALIZED(left, _PyUnicode_ExactDealloc); - PyObject *temp = PyStackRef_AsPyObjectSteal(*target_local); - PyObject *right_o = PyStackRef_AsPyObjectSteal(right); - stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyUnicode_Append(&temp, right_o); - stack_pointer = _PyFrame_GetStackPointer(frame); - *target_local = PyStackRef_FromPyObjectSteal(temp); - _PyFrame_SetStackPointer(frame, stack_pointer); - Py_DECREF(right_o); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (PyStackRef_IsNull(*target_local)) { - TRACING_JUMP_TO_LABEL(error); - } - #if TIER_ONE - - assert(next_instr->op.code == STORE_FAST); - SKIP_OVER(1); - #endif - } - TRACING_DISPATCH(); - } - - TRACING_TARGET(BINARY_OP_MULTIPLY_FLOAT) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = BINARY_OP_MULTIPLY_FLOAT; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 6; - INSTRUCTION_STATS(BINARY_OP_MULTIPLY_FLOAT); - opcode = BINARY_OP_MULTIPLY_FLOAT; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5, "incorrect cache size"); - _PyStackRef value; - _PyStackRef left; - _PyStackRef right; - _PyStackRef res; - // _GUARD_TOS_FLOAT - { - value = stack_pointer[-1]; - PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); - if (!PyFloat_CheckExact(value_o)) { - UPDATE_MISS_STATS(BINARY_OP); - assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); - JUMP_TO_PREDICTED(TRACING_BINARY_OP); - } - } - // _GUARD_NOS_FLOAT - { - left = stack_pointer[-2]; - PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); - if (!PyFloat_CheckExact(left_o)) { - UPDATE_MISS_STATS(BINARY_OP); - assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); - JUMP_TO_PREDICTED(TRACING_BINARY_OP); - } - } - /* Skip 5 cache entries */ - // _BINARY_OP_MULTIPLY_FLOAT - { - right = value; - PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); - PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - assert(PyFloat_CheckExact(left_o)); - assert(PyFloat_CheckExact(right_o)); - STAT_INC(BINARY_OP, hit); - double dres = - ((PyFloatObject *)left_o)->ob_fval * - ((PyFloatObject *)right_o)->ob_fval; - res = _PyFloat_FromDouble_ConsumeInputs(left, right, dres); - if (PyStackRef_IsNull(res)) { - TRACING_JUMP_TO_LABEL(pop_2_error); - } - } - stack_pointer[-2] = res; - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - TRACING_DISPATCH(); - } - - TRACING_TARGET(BINARY_OP_MULTIPLY_INT) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = BINARY_OP_MULTIPLY_INT; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 6; - INSTRUCTION_STATS(BINARY_OP_MULTIPLY_INT); - opcode = BINARY_OP_MULTIPLY_INT; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5, "incorrect cache size"); - _PyStackRef value; - _PyStackRef left; - _PyStackRef right; - _PyStackRef res; - // _GUARD_TOS_INT - { - value = stack_pointer[-1]; - PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); - if (!_PyLong_CheckExactAndCompact(value_o)) { - UPDATE_MISS_STATS(BINARY_OP); - assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); - JUMP_TO_PREDICTED(TRACING_BINARY_OP); - } - } - // _GUARD_NOS_INT - { - left = stack_pointer[-2]; - PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); - if (!_PyLong_CheckExactAndCompact(left_o)) { - UPDATE_MISS_STATS(BINARY_OP); - assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); - JUMP_TO_PREDICTED(TRACING_BINARY_OP); - } - } - /* Skip 5 cache entries */ - // _BINARY_OP_MULTIPLY_INT - { - right = value; - PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); - PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - assert(PyLong_CheckExact(left_o)); - assert(PyLong_CheckExact(right_o)); - assert(_PyLong_BothAreCompact((PyLongObject *)left_o, (PyLongObject *)right_o)); - STAT_INC(BINARY_OP, hit); - res = _PyCompactLong_Multiply((PyLongObject *)left_o, (PyLongObject *)right_o); - if (PyStackRef_IsNull(res)) { - UPDATE_MISS_STATS(BINARY_OP); - assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); - JUMP_TO_PREDICTED(TRACING_BINARY_OP); - } - PyStackRef_CLOSE_SPECIALIZED(right, _PyLong_ExactDealloc); - PyStackRef_CLOSE_SPECIALIZED(left, _PyLong_ExactDealloc); - } - stack_pointer[-2] = res; - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - TRACING_DISPATCH(); - } - - TRACING_TARGET(BINARY_OP_SUBSCR_DICT) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = BINARY_OP_SUBSCR_DICT; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 6; - INSTRUCTION_STATS(BINARY_OP_SUBSCR_DICT); - opcode = BINARY_OP_SUBSCR_DICT; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5, "incorrect cache size"); - _PyStackRef nos; - _PyStackRef dict_st; - _PyStackRef sub_st; - _PyStackRef res; - // _GUARD_NOS_DICT - { - nos = stack_pointer[-2]; - PyObject *o = PyStackRef_AsPyObjectBorrow(nos); - if (!PyDict_CheckExact(o)) { - UPDATE_MISS_STATS(BINARY_OP); - assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); - JUMP_TO_PREDICTED(TRACING_BINARY_OP); - } - } - /* Skip 5 cache entries */ - // _BINARY_OP_SUBSCR_DICT - { - sub_st = stack_pointer[-1]; - dict_st = nos; - PyObject *sub = PyStackRef_AsPyObjectBorrow(sub_st); - PyObject *dict = PyStackRef_AsPyObjectBorrow(dict_st); - assert(PyDict_CheckExact(dict)); - STAT_INC(BINARY_OP, hit); - PyObject *res_o; - _PyFrame_SetStackPointer(frame, stack_pointer); - int rc = PyDict_GetItemRef(dict, sub, &res_o); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (rc == 0) { - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyErr_SetKeyError(sub); - stack_pointer = _PyFrame_GetStackPointer(frame); - } - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyStackRef tmp = sub_st; - sub_st = PyStackRef_NULL; - stack_pointer[-1] = sub_st; - PyStackRef_CLOSE(tmp); - tmp = dict_st; - dict_st = PyStackRef_NULL; - stack_pointer[-2] = dict_st; - PyStackRef_CLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); - if (rc <= 0) { - TRACING_JUMP_TO_LABEL(error); - } - res = PyStackRef_FromPyObjectSteal(res_o); - } - stack_pointer[0] = res; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - TRACING_DISPATCH(); - } - - TRACING_TARGET(BINARY_OP_SUBSCR_GETITEM) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = BINARY_OP_SUBSCR_GETITEM; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 6; - INSTRUCTION_STATS(BINARY_OP_SUBSCR_GETITEM); - opcode = BINARY_OP_SUBSCR_GETITEM; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5, "incorrect cache size"); - _PyStackRef container; - _PyStackRef getitem; - _PyStackRef sub; - _PyStackRef new_frame; - /* Skip 5 cache entries */ - // _CHECK_PEP_523 - { - if (tstate->interp->eval_frame) { - UPDATE_MISS_STATS(BINARY_OP); - assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); - JUMP_TO_PREDICTED(TRACING_BINARY_OP); - } - } - // _BINARY_OP_SUBSCR_CHECK_FUNC - { - container = stack_pointer[-2]; - PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(container)); - if (!PyType_HasFeature(tp, Py_TPFLAGS_HEAPTYPE)) { - UPDATE_MISS_STATS(BINARY_OP); - assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); - JUMP_TO_PREDICTED(TRACING_BINARY_OP); - } - PyHeapTypeObject *ht = (PyHeapTypeObject *)tp; - PyObject *getitem_o = FT_ATOMIC_LOAD_PTR_ACQUIRE(ht->_spec_cache.getitem); - if (getitem_o == NULL) { - UPDATE_MISS_STATS(BINARY_OP); - assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); - JUMP_TO_PREDICTED(TRACING_BINARY_OP); - } - assert(PyFunction_Check(getitem_o)); - uint32_t cached_version = FT_ATOMIC_LOAD_UINT32_RELAXED(ht->_spec_cache.getitem_version); - if (((PyFunctionObject *)getitem_o)->func_version != cached_version) { - UPDATE_MISS_STATS(BINARY_OP); - assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); - JUMP_TO_PREDICTED(TRACING_BINARY_OP); - } - PyCodeObject *code = (PyCodeObject *)PyFunction_GET_CODE(getitem_o); - assert(code->co_argcount == 2); - if (!_PyThreadState_HasStackSpace(tstate, code->co_framesize)) { - UPDATE_MISS_STATS(BINARY_OP); - assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); - JUMP_TO_PREDICTED(TRACING_BINARY_OP); - } - getitem = PyStackRef_FromPyObjectNew(getitem_o); - STAT_INC(BINARY_OP, hit); - } - // _BINARY_OP_SUBSCR_INIT_CALL - { - sub = stack_pointer[-1]; - _PyInterpreterFrame* pushed_frame = _PyFrame_PushUnchecked(tstate, getitem, 2, frame); - pushed_frame->localsplus[0] = container; - pushed_frame->localsplus[1] = sub; - frame->return_offset = 6u ; - new_frame = PyStackRef_Wrap(pushed_frame); - } - // _PUSH_FRAME - { - assert(tstate->interp->eval_frame == NULL); - _PyInterpreterFrame *temp = PyStackRef_Unwrap(new_frame); - stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - assert(temp->previous == frame || temp->previous->previous == frame); - CALL_STAT_INC(inlined_py_calls); - frame = tstate->current_frame = temp; - tstate->py_recursion_remaining--; - LOAD_SP(); - LOAD_IP(0); - LLTRACE_RESUME_FRAME(); - } - TRACING_DISPATCH(); - } - - TRACING_TARGET(BINARY_OP_SUBSCR_LIST_INT) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = BINARY_OP_SUBSCR_LIST_INT; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 6; - INSTRUCTION_STATS(BINARY_OP_SUBSCR_LIST_INT); - opcode = BINARY_OP_SUBSCR_LIST_INT; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5, "incorrect cache size"); - _PyStackRef value; - _PyStackRef nos; - _PyStackRef list_st; - _PyStackRef sub_st; - _PyStackRef res; - // _GUARD_TOS_INT - { - value = stack_pointer[-1]; - PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); - if (!_PyLong_CheckExactAndCompact(value_o)) { - UPDATE_MISS_STATS(BINARY_OP); - assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); - JUMP_TO_PREDICTED(TRACING_BINARY_OP); - } - } - // _GUARD_NOS_LIST - { - nos = stack_pointer[-2]; - PyObject *o = PyStackRef_AsPyObjectBorrow(nos); - if (!PyList_CheckExact(o)) { - UPDATE_MISS_STATS(BINARY_OP); - assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); - JUMP_TO_PREDICTED(TRACING_BINARY_OP); - } - } - /* Skip 5 cache entries */ - // _BINARY_OP_SUBSCR_LIST_INT - { - sub_st = value; - list_st = nos; - PyObject *sub = PyStackRef_AsPyObjectBorrow(sub_st); - PyObject *list = PyStackRef_AsPyObjectBorrow(list_st); - assert(PyLong_CheckExact(sub)); - assert(PyList_CheckExact(list)); - if (!_PyLong_IsNonNegativeCompact((PyLongObject *)sub)) { - UPDATE_MISS_STATS(BINARY_OP); - assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); - JUMP_TO_PREDICTED(TRACING_BINARY_OP); - } - Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0]; - #ifdef Py_GIL_DISABLED - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *res_o = _PyList_GetItemRef((PyListObject*)list, index); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (res_o == NULL) { - UPDATE_MISS_STATS(BINARY_OP); - assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); - JUMP_TO_PREDICTED(TRACING_BINARY_OP); - } - STAT_INC(BINARY_OP, hit); - res = PyStackRef_FromPyObjectSteal(res_o); - #else - if (index >= PyList_GET_SIZE(list)) { - UPDATE_MISS_STATS(BINARY_OP); - assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); - JUMP_TO_PREDICTED(TRACING_BINARY_OP); - } - STAT_INC(BINARY_OP, hit); - PyObject *res_o = PyList_GET_ITEM(list, index); - assert(res_o != NULL); - res = PyStackRef_FromPyObjectNew(res_o); - #endif - STAT_INC(BINARY_OP, hit); - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyStackRef tmp = list_st; - list_st = res; - stack_pointer[-2] = list_st; - PyStackRef_CLOSE(tmp); - tmp = sub_st; - sub_st = PyStackRef_NULL; - stack_pointer[-1] = sub_st; - PyStackRef_CLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - } - TRACING_DISPATCH(); - } - - TRACING_TARGET(BINARY_OP_SUBSCR_LIST_SLICE) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = BINARY_OP_SUBSCR_LIST_SLICE; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 6; - INSTRUCTION_STATS(BINARY_OP_SUBSCR_LIST_SLICE); - opcode = BINARY_OP_SUBSCR_LIST_SLICE; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5, "incorrect cache size"); - _PyStackRef tos; - _PyStackRef nos; - _PyStackRef list_st; - _PyStackRef sub_st; - _PyStackRef res; - // _GUARD_TOS_SLICE - { - tos = stack_pointer[-1]; - PyObject *o = PyStackRef_AsPyObjectBorrow(tos); - if (!PySlice_Check(o)) { - UPDATE_MISS_STATS(BINARY_OP); - assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); - JUMP_TO_PREDICTED(TRACING_BINARY_OP); - } - } - // _GUARD_NOS_LIST - { - nos = stack_pointer[-2]; - PyObject *o = PyStackRef_AsPyObjectBorrow(nos); - if (!PyList_CheckExact(o)) { - UPDATE_MISS_STATS(BINARY_OP); - assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); - JUMP_TO_PREDICTED(TRACING_BINARY_OP); - } - } - /* Skip 5 cache entries */ - // _BINARY_OP_SUBSCR_LIST_SLICE - { - sub_st = tos; - list_st = nos; - PyObject *sub = PyStackRef_AsPyObjectBorrow(sub_st); - PyObject *list = PyStackRef_AsPyObjectBorrow(list_st); - assert(PySlice_Check(sub)); - assert(PyList_CheckExact(list)); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *res_o = _PyList_SliceSubscript(list, sub); - stack_pointer = _PyFrame_GetStackPointer(frame); - STAT_INC(BINARY_OP, hit); - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyStackRef tmp = sub_st; - sub_st = PyStackRef_NULL; - stack_pointer[-1] = sub_st; - PyStackRef_CLOSE(tmp); - tmp = list_st; - list_st = PyStackRef_NULL; - stack_pointer[-2] = list_st; - PyStackRef_CLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); - if (res_o == NULL) { - TRACING_JUMP_TO_LABEL(error); - } - res = PyStackRef_FromPyObjectSteal(res_o); - } - stack_pointer[0] = res; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - TRACING_DISPATCH(); - } - - TRACING_TARGET(BINARY_OP_SUBSCR_STR_INT) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = BINARY_OP_SUBSCR_STR_INT; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 6; - INSTRUCTION_STATS(BINARY_OP_SUBSCR_STR_INT); - opcode = BINARY_OP_SUBSCR_STR_INT; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5, "incorrect cache size"); - _PyStackRef value; - _PyStackRef nos; - _PyStackRef str_st; - _PyStackRef sub_st; - _PyStackRef res; - // _GUARD_TOS_INT - { - value = stack_pointer[-1]; - PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); - if (!_PyLong_CheckExactAndCompact(value_o)) { - UPDATE_MISS_STATS(BINARY_OP); - assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); - JUMP_TO_PREDICTED(TRACING_BINARY_OP); - } - } - // _GUARD_NOS_UNICODE - { - nos = stack_pointer[-2]; - PyObject *o = PyStackRef_AsPyObjectBorrow(nos); - if (!PyUnicode_CheckExact(o)) { - UPDATE_MISS_STATS(BINARY_OP); - assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); - JUMP_TO_PREDICTED(TRACING_BINARY_OP); - } - } - /* Skip 5 cache entries */ - // _BINARY_OP_SUBSCR_STR_INT - { - sub_st = value; - str_st = nos; - PyObject *sub = PyStackRef_AsPyObjectBorrow(sub_st); - PyObject *str = PyStackRef_AsPyObjectBorrow(str_st); - assert(PyLong_CheckExact(sub)); - assert(PyUnicode_CheckExact(str)); - if (!_PyLong_IsNonNegativeCompact((PyLongObject *)sub)) { - UPDATE_MISS_STATS(BINARY_OP); - assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); - JUMP_TO_PREDICTED(TRACING_BINARY_OP); - } - Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0]; - if (PyUnicode_GET_LENGTH(str) <= index) { - UPDATE_MISS_STATS(BINARY_OP); - assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); - JUMP_TO_PREDICTED(TRACING_BINARY_OP); - } - Py_UCS4 c = PyUnicode_READ_CHAR(str, index); - if (Py_ARRAY_LENGTH(_Py_SINGLETON(strings).ascii) <= c) { - UPDATE_MISS_STATS(BINARY_OP); - assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); - JUMP_TO_PREDICTED(TRACING_BINARY_OP); - } - STAT_INC(BINARY_OP, hit); - PyObject *res_o = (PyObject*)&_Py_SINGLETON(strings).ascii[c]; - PyStackRef_CLOSE_SPECIALIZED(sub_st, _PyLong_ExactDealloc); - stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(str_st); - stack_pointer = _PyFrame_GetStackPointer(frame); - res = PyStackRef_FromPyObjectBorrow(res_o); - } - stack_pointer[0] = res; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - TRACING_DISPATCH(); - } - - TRACING_TARGET(BINARY_OP_SUBSCR_TUPLE_INT) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = BINARY_OP_SUBSCR_TUPLE_INT; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 6; - INSTRUCTION_STATS(BINARY_OP_SUBSCR_TUPLE_INT); - opcode = BINARY_OP_SUBSCR_TUPLE_INT; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5, "incorrect cache size"); - _PyStackRef value; - _PyStackRef nos; - _PyStackRef tuple_st; - _PyStackRef sub_st; - _PyStackRef res; - // _GUARD_TOS_INT - { - value = stack_pointer[-1]; - PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); - if (!_PyLong_CheckExactAndCompact(value_o)) { - UPDATE_MISS_STATS(BINARY_OP); - assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); - JUMP_TO_PREDICTED(TRACING_BINARY_OP); - } - } - // _GUARD_NOS_TUPLE - { - nos = stack_pointer[-2]; - PyObject *o = PyStackRef_AsPyObjectBorrow(nos); - if (!PyTuple_CheckExact(o)) { - UPDATE_MISS_STATS(BINARY_OP); - assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); - JUMP_TO_PREDICTED(TRACING_BINARY_OP); - } - } - /* Skip 5 cache entries */ - // _BINARY_OP_SUBSCR_TUPLE_INT - { - sub_st = value; - tuple_st = nos; - PyObject *sub = PyStackRef_AsPyObjectBorrow(sub_st); - PyObject *tuple = PyStackRef_AsPyObjectBorrow(tuple_st); - assert(PyLong_CheckExact(sub)); - assert(PyTuple_CheckExact(tuple)); - if (!_PyLong_IsNonNegativeCompact((PyLongObject *)sub)) { - UPDATE_MISS_STATS(BINARY_OP); - assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); - JUMP_TO_PREDICTED(TRACING_BINARY_OP); - } - Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0]; - if (index >= PyTuple_GET_SIZE(tuple)) { - UPDATE_MISS_STATS(BINARY_OP); - assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); - JUMP_TO_PREDICTED(TRACING_BINARY_OP); - } - STAT_INC(BINARY_OP, hit); - PyObject *res_o = PyTuple_GET_ITEM(tuple, index); - assert(res_o != NULL); - PyStackRef_CLOSE_SPECIALIZED(sub_st, _PyLong_ExactDealloc); - res = PyStackRef_FromPyObjectNew(res_o); - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyStackRef tmp = tuple_st; - tuple_st = res; - stack_pointer[-1] = tuple_st; - PyStackRef_CLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); - } - TRACING_DISPATCH(); - } - - TRACING_TARGET(BINARY_OP_SUBTRACT_FLOAT) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = BINARY_OP_SUBTRACT_FLOAT; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 6; - INSTRUCTION_STATS(BINARY_OP_SUBTRACT_FLOAT); - opcode = BINARY_OP_SUBTRACT_FLOAT; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5, "incorrect cache size"); - _PyStackRef value; - _PyStackRef left; - _PyStackRef right; - _PyStackRef res; - // _GUARD_TOS_FLOAT - { - value = stack_pointer[-1]; - PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); - if (!PyFloat_CheckExact(value_o)) { - UPDATE_MISS_STATS(BINARY_OP); - assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); - JUMP_TO_PREDICTED(TRACING_BINARY_OP); - } - } - // _GUARD_NOS_FLOAT - { - left = stack_pointer[-2]; - PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); - if (!PyFloat_CheckExact(left_o)) { - UPDATE_MISS_STATS(BINARY_OP); - assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); - JUMP_TO_PREDICTED(TRACING_BINARY_OP); - } - } - /* Skip 5 cache entries */ - // _BINARY_OP_SUBTRACT_FLOAT - { - right = value; - PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); - PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - assert(PyFloat_CheckExact(left_o)); - assert(PyFloat_CheckExact(right_o)); - STAT_INC(BINARY_OP, hit); - double dres = - ((PyFloatObject *)left_o)->ob_fval - - ((PyFloatObject *)right_o)->ob_fval; - res = _PyFloat_FromDouble_ConsumeInputs(left, right, dres); - if (PyStackRef_IsNull(res)) { - TRACING_JUMP_TO_LABEL(pop_2_error); - } - } - stack_pointer[-2] = res; - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - TRACING_DISPATCH(); - } - - TRACING_TARGET(BINARY_OP_SUBTRACT_INT) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = BINARY_OP_SUBTRACT_INT; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 6; - INSTRUCTION_STATS(BINARY_OP_SUBTRACT_INT); - opcode = BINARY_OP_SUBTRACT_INT; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5, "incorrect cache size"); - _PyStackRef value; - _PyStackRef left; - _PyStackRef right; - _PyStackRef res; - // _GUARD_TOS_INT - { - value = stack_pointer[-1]; - PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); - if (!_PyLong_CheckExactAndCompact(value_o)) { - UPDATE_MISS_STATS(BINARY_OP); - assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); - JUMP_TO_PREDICTED(TRACING_BINARY_OP); - } - } - // _GUARD_NOS_INT - { - left = stack_pointer[-2]; - PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); - if (!_PyLong_CheckExactAndCompact(left_o)) { - UPDATE_MISS_STATS(BINARY_OP); - assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); - JUMP_TO_PREDICTED(TRACING_BINARY_OP); - } - } - /* Skip 5 cache entries */ - // _BINARY_OP_SUBTRACT_INT - { - right = value; - PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); - PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - assert(PyLong_CheckExact(left_o)); - assert(PyLong_CheckExact(right_o)); - assert(_PyLong_BothAreCompact((PyLongObject *)left_o, (PyLongObject *)right_o)); - STAT_INC(BINARY_OP, hit); - res = _PyCompactLong_Subtract((PyLongObject *)left_o, (PyLongObject *)right_o); - if (PyStackRef_IsNull(res)) { - UPDATE_MISS_STATS(BINARY_OP); - assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); - JUMP_TO_PREDICTED(TRACING_BINARY_OP); - } - PyStackRef_CLOSE_SPECIALIZED(right, _PyLong_ExactDealloc); - PyStackRef_CLOSE_SPECIALIZED(left, _PyLong_ExactDealloc); - } - stack_pointer[-2] = res; - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - TRACING_DISPATCH(); - } - - TRACING_TARGET(BINARY_SLICE) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = BINARY_SLICE; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(BINARY_SLICE); - opcode = BINARY_SLICE; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - _PyStackRef container; - _PyStackRef start; - _PyStackRef stop; - _PyStackRef res; - // _SPECIALIZE_BINARY_SLICE - { - #if ENABLE_SPECIALIZATION - OPCODE_DEFERRED_INC(BINARY_SLICE); - #endif /* ENABLE_SPECIALIZATION */ - } - // _BINARY_SLICE - { - stop = stack_pointer[-1]; - start = stack_pointer[-2]; - container = stack_pointer[-3]; - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *slice = _PyBuildSlice_ConsumeRefs(PyStackRef_AsPyObjectSteal(start), - PyStackRef_AsPyObjectSteal(stop)); - stack_pointer = _PyFrame_GetStackPointer(frame); - PyObject *res_o; - if (slice == NULL) { - res_o = NULL; - } - else { - stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - res_o = PyObject_GetItem(PyStackRef_AsPyObjectBorrow(container), slice); - Py_DECREF(slice); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += 2; - } - stack_pointer += -3; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(container); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (res_o == NULL) { - TRACING_JUMP_TO_LABEL(error); - } - res = PyStackRef_FromPyObjectSteal(res_o); - } - stack_pointer[0] = res; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - TRACING_DISPATCH(); - } - - TRACING_TARGET(BUILD_INTERPOLATION) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = BUILD_INTERPOLATION; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(BUILD_INTERPOLATION); - opcode = BUILD_INTERPOLATION; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - _PyStackRef value; - _PyStackRef str; - _PyStackRef *format; - _PyStackRef interpolation; - format = &stack_pointer[-(oparg & 1)]; - str = stack_pointer[-1 - (oparg & 1)]; - value = stack_pointer[-2 - (oparg & 1)]; - PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); - PyObject *str_o = PyStackRef_AsPyObjectBorrow(str); - int conversion = oparg >> 2; - PyObject *format_o; - if (oparg & 1) { - format_o = PyStackRef_AsPyObjectBorrow(format[0]); - } - else { - format_o = &_Py_STR(empty); - } - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *interpolation_o = _PyInterpolation_Build(value_o, str_o, conversion, format_o); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (oparg & 1) { - stack_pointer += -(oparg & 1); - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(format[0]); - stack_pointer = _PyFrame_GetStackPointer(frame); - } - else { - stack_pointer += -(oparg & 1); - } - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(str); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(value); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (interpolation_o == NULL) { - TRACING_JUMP_TO_LABEL(error); - } - interpolation = PyStackRef_FromPyObjectSteal(interpolation_o); - stack_pointer[0] = interpolation; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - TRACING_DISPATCH(); - } - - TRACING_TARGET(BUILD_LIST) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = BUILD_LIST; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(BUILD_LIST); - opcode = BUILD_LIST; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - _PyStackRef *values; - _PyStackRef list; - values = &stack_pointer[-oparg]; - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *list_o = _PyList_FromStackRefStealOnSuccess(values, oparg); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (list_o == NULL) { - TRACING_JUMP_TO_LABEL(error); - } - list = PyStackRef_FromPyObjectStealMortal(list_o); - stack_pointer[-oparg] = list; - stack_pointer += 1 - oparg; - assert(WITHIN_STACK_BOUNDS()); - TRACING_DISPATCH(); - } - - TRACING_TARGET(BUILD_MAP) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = BUILD_MAP; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(BUILD_MAP); - opcode = BUILD_MAP; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - _PyStackRef *values; - _PyStackRef map; - values = &stack_pointer[-oparg*2]; - STACKREFS_TO_PYOBJECTS(values, oparg*2, values_o); - if (CONVERSION_FAILED(values_o)) { - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyStackRef tmp; - for (int _i = oparg*2; --_i >= 0;) { - tmp = values[_i]; - values[_i] = PyStackRef_NULL; - PyStackRef_CLOSE(tmp); - } - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -oparg*2; - assert(WITHIN_STACK_BOUNDS()); - TRACING_JUMP_TO_LABEL(error); - } - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *map_o = _PyDict_FromItems( - values_o, 2, - values_o+1, 2, - oparg); - stack_pointer = _PyFrame_GetStackPointer(frame); - STACKREFS_TO_PYOBJECTS_CLEANUP(values_o); - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyStackRef tmp; - for (int _i = oparg*2; --_i >= 0;) { - tmp = values[_i]; - values[_i] = PyStackRef_NULL; - PyStackRef_CLOSE(tmp); - } - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -oparg*2; - assert(WITHIN_STACK_BOUNDS()); - if (map_o == NULL) { - TRACING_JUMP_TO_LABEL(error); - } - map = PyStackRef_FromPyObjectStealMortal(map_o); - stack_pointer[0] = map; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - TRACING_DISPATCH(); - } - - TRACING_TARGET(BUILD_SET) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = BUILD_SET; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(BUILD_SET); - opcode = BUILD_SET; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - _PyStackRef *values; - _PyStackRef set; - values = &stack_pointer[-oparg]; - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *set_o = PySet_New(NULL); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (set_o == NULL) { - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyStackRef tmp; - for (int _i = oparg; --_i >= 0;) { - tmp = values[_i]; - values[_i] = PyStackRef_NULL; - PyStackRef_CLOSE(tmp); - } - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -oparg; - assert(WITHIN_STACK_BOUNDS()); - TRACING_JUMP_TO_LABEL(error); - } - int err = 0; - for (Py_ssize_t i = 0; i < oparg; i++) { - _PyStackRef value = values[i]; - values[i] = PyStackRef_NULL; - if (err == 0) { - _PyFrame_SetStackPointer(frame, stack_pointer); - err = _PySet_AddTakeRef((PySetObject *)set_o, PyStackRef_AsPyObjectSteal(value)); - stack_pointer = _PyFrame_GetStackPointer(frame); - } - else { - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(value); - stack_pointer = _PyFrame_GetStackPointer(frame); - } - } - if (err) { - stack_pointer += -oparg; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - Py_DECREF(set_o); - stack_pointer = _PyFrame_GetStackPointer(frame); - TRACING_JUMP_TO_LABEL(error); - } - set = PyStackRef_FromPyObjectStealMortal(set_o); - stack_pointer[-oparg] = set; - stack_pointer += 1 - oparg; - assert(WITHIN_STACK_BOUNDS()); - TRACING_DISPATCH(); - } - - TRACING_TARGET(BUILD_SLICE) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = BUILD_SLICE; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(BUILD_SLICE); - opcode = BUILD_SLICE; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - _PyStackRef *args; - _PyStackRef slice; - args = &stack_pointer[-oparg]; - PyObject *start_o = PyStackRef_AsPyObjectBorrow(args[0]); - PyObject *stop_o = PyStackRef_AsPyObjectBorrow(args[1]); - PyObject *step_o = oparg == 3 ? PyStackRef_AsPyObjectBorrow(args[2]) : NULL; - PyObject *slice_o = PySlice_New(start_o, stop_o, step_o); - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyStackRef tmp; - for (int _i = oparg; --_i >= 0;) { - tmp = args[_i]; - args[_i] = PyStackRef_NULL; - PyStackRef_CLOSE(tmp); - } - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -oparg; - assert(WITHIN_STACK_BOUNDS()); - if (slice_o == NULL) { - TRACING_JUMP_TO_LABEL(error); - } - slice = PyStackRef_FromPyObjectStealMortal(slice_o); - stack_pointer[0] = slice; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - TRACING_DISPATCH(); - } - - TRACING_TARGET(BUILD_STRING) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = BUILD_STRING; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(BUILD_STRING); - opcode = BUILD_STRING; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - _PyStackRef *pieces; - _PyStackRef str; - pieces = &stack_pointer[-oparg]; - STACKREFS_TO_PYOBJECTS(pieces, oparg, pieces_o); - if (CONVERSION_FAILED(pieces_o)) { - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyStackRef tmp; - for (int _i = oparg; --_i >= 0;) { - tmp = pieces[_i]; - pieces[_i] = PyStackRef_NULL; - PyStackRef_CLOSE(tmp); - } - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -oparg; - assert(WITHIN_STACK_BOUNDS()); - TRACING_JUMP_TO_LABEL(error); - } - PyObject *str_o = _PyUnicode_JoinArray(&_Py_STR(empty), pieces_o, oparg); - STACKREFS_TO_PYOBJECTS_CLEANUP(pieces_o); - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyStackRef tmp; - for (int _i = oparg; --_i >= 0;) { - tmp = pieces[_i]; - pieces[_i] = PyStackRef_NULL; - PyStackRef_CLOSE(tmp); - } - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -oparg; - assert(WITHIN_STACK_BOUNDS()); - if (str_o == NULL) { - TRACING_JUMP_TO_LABEL(error); - } - str = PyStackRef_FromPyObjectSteal(str_o); - stack_pointer[0] = str; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - TRACING_DISPATCH(); - } - - TRACING_TARGET(BUILD_TEMPLATE) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = BUILD_TEMPLATE; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(BUILD_TEMPLATE); - opcode = BUILD_TEMPLATE; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - _PyStackRef strings; - _PyStackRef interpolations; - _PyStackRef template; - interpolations = stack_pointer[-1]; - strings = stack_pointer[-2]; - PyObject *strings_o = PyStackRef_AsPyObjectBorrow(strings); - PyObject *interpolations_o = PyStackRef_AsPyObjectBorrow(interpolations); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *template_o = _PyTemplate_Build(strings_o, interpolations_o); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(interpolations); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(strings); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (template_o == NULL) { - TRACING_JUMP_TO_LABEL(error); - } - template = PyStackRef_FromPyObjectSteal(template_o); - stack_pointer[0] = template; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - TRACING_DISPATCH(); - } - - TRACING_TARGET(BUILD_TUPLE) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = BUILD_TUPLE; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(BUILD_TUPLE); - opcode = BUILD_TUPLE; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - _PyStackRef *values; - _PyStackRef tup; - values = &stack_pointer[-oparg]; - PyObject *tup_o = _PyTuple_FromStackRefStealOnSuccess(values, oparg); - if (tup_o == NULL) { - TRACING_JUMP_TO_LABEL(error); - } - tup = PyStackRef_FromPyObjectStealMortal(tup_o); - stack_pointer[-oparg] = tup; - stack_pointer += 1 - oparg; - assert(WITHIN_STACK_BOUNDS()); - TRACING_DISPATCH(); - } - - TRACING_TARGET(CACHE) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = CACHE; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(CACHE); - opcode = CACHE; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - assert(0 && "Executing a cache."); - Py_FatalError("Executing a cache."); - TRACING_DISPATCH(); - } - - TRACING_TARGET(CALL) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = CALL; - (void)(opcode); - #endif - frame->instr_ptr = next_instr; - next_instr += 4; - INSTRUCTION_STATS(CALL); - PREDICTED_TRACING_CALL:; - _Py_CODEUNIT* const this_instr = next_instr - 4; - (void)this_instr; - opcode = CALL; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - opcode = CALL; - _PyStackRef callable; - _PyStackRef self_or_null; - _PyStackRef *args; - _PyStackRef res; - // _SPECIALIZE_CALL - { - self_or_null = stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; - uint16_t counter = read_u16(&this_instr[1].cache); - (void)counter; - #if ENABLE_SPECIALIZATION_FT - if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { - next_instr = this_instr; - _PyFrame_SetStackPointer(frame, stack_pointer); - _Py_Specialize_Call(callable, next_instr, oparg + !PyStackRef_IsNull(self_or_null)); - stack_pointer = _PyFrame_GetStackPointer(frame); - DISPATCH_SAME_OPARG(); - } - OPCODE_DEFERRED_INC(CALL); - ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); - #endif /* ENABLE_SPECIALIZATION_FT */ - } - /* Skip 2 cache entries */ - // _MAYBE_EXPAND_METHOD - { - if (PyStackRef_TYPE(callable) == &PyMethod_Type && PyStackRef_IsNull(self_or_null)) { - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); - PyObject *self = ((PyMethodObject *)callable_o)->im_self; - self_or_null = PyStackRef_FromPyObjectNew(self); - PyObject *method = ((PyMethodObject *)callable_o)->im_func; - _PyStackRef temp = callable; - callable = PyStackRef_FromPyObjectNew(method); - stack_pointer[-2 - oparg] = callable; - stack_pointer[-1 - oparg] = self_or_null; - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(temp); - stack_pointer = _PyFrame_GetStackPointer(frame); - } - } - // _DO_CALL - { - args = &stack_pointer[-oparg]; - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); - int total_args = oparg; - _PyStackRef *arguments = args; - if (!PyStackRef_IsNull(self_or_null)) { - arguments--; - total_args++; - } - if (Py_TYPE(callable_o) == &PyFunction_Type && - tstate->interp->eval_frame == NULL && - ((PyFunctionObject *)callable_o)->vectorcall == _PyFunction_Vectorcall) - { - int code_flags = ((PyCodeObject*)PyFunction_GET_CODE(callable_o))->co_flags; - PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable_o)); - stack_pointer[-2 - oparg] = callable; - stack_pointer[-1 - oparg] = self_or_null; - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyInterpreterFrame *new_frame = _PyEvalFramePushAndInit( - tstate, callable, locals, - arguments, total_args, NULL, frame - ); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -2 - oparg; - assert(WITHIN_STACK_BOUNDS()); - if (new_frame == NULL) { - TRACING_JUMP_TO_LABEL(error); - } - frame->return_offset = 4u ; - TRACING_DISPATCH_INLINED(new_frame); - } - STACKREFS_TO_PYOBJECTS(arguments, total_args, args_o); - if (CONVERSION_FAILED(args_o)) { - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyStackRef tmp; - for (int _i = oparg; --_i >= 0;) { - tmp = args[_i]; - args[_i] = PyStackRef_NULL; - stack_pointer[-2 - oparg] = callable; - stack_pointer[-1 - oparg] = self_or_null; - PyStackRef_CLOSE(tmp); - } - tmp = self_or_null; - self_or_null = PyStackRef_NULL; - stack_pointer[-1 - oparg] = self_or_null; - PyStackRef_XCLOSE(tmp); - tmp = callable; - callable = PyStackRef_NULL; - stack_pointer[-2 - oparg] = callable; - PyStackRef_CLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -2 - oparg; - assert(WITHIN_STACK_BOUNDS()); - TRACING_JUMP_TO_LABEL(error); - } - stack_pointer[-2 - oparg] = callable; - stack_pointer[-1 - oparg] = self_or_null; - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *res_o = PyObject_Vectorcall( - callable_o, args_o, - total_args | PY_VECTORCALL_ARGUMENTS_OFFSET, - NULL); - stack_pointer = _PyFrame_GetStackPointer(frame); - STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); - if (opcode == INSTRUMENTED_CALL) { - PyObject *arg = total_args == 0 ? - &_PyInstrumentation_MISSING : PyStackRef_AsPyObjectBorrow(arguments[0]); - if (res_o == NULL) { - _PyFrame_SetStackPointer(frame, stack_pointer); - _Py_call_instrumentation_exc2( - tstate, PY_MONITORING_EVENT_C_RAISE, - frame, this_instr, callable_o, arg); - stack_pointer = _PyFrame_GetStackPointer(frame); - } - else { - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = _Py_call_instrumentation_2args( - tstate, PY_MONITORING_EVENT_C_RETURN, - frame, this_instr, callable_o, arg); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err < 0) { - _PyFrame_SetStackPointer(frame, stack_pointer); - Py_CLEAR(res_o); - stack_pointer = _PyFrame_GetStackPointer(frame); - } - } - } - assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyStackRef tmp; - for (int _i = oparg; --_i >= 0;) { - tmp = args[_i]; - args[_i] = PyStackRef_NULL; - PyStackRef_CLOSE(tmp); - } - tmp = self_or_null; - self_or_null = PyStackRef_NULL; - stack_pointer[-1 - oparg] = self_or_null; - PyStackRef_XCLOSE(tmp); - tmp = callable; - callable = PyStackRef_NULL; - stack_pointer[-2 - oparg] = callable; - PyStackRef_CLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -2 - oparg; - assert(WITHIN_STACK_BOUNDS()); - if (res_o == NULL) { - TRACING_JUMP_TO_LABEL(error); - } - res = PyStackRef_FromPyObjectSteal(res_o); - } - // _CHECK_PERIODIC_AT_END - { - stack_pointer[0] = res; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = check_periodics(tstate); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err != 0) { - TRACING_JUMP_TO_LABEL(error); - } - } - TRACING_DISPATCH(); - } - - TRACING_TARGET(CALL_ALLOC_AND_ENTER_INIT) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = CALL_ALLOC_AND_ENTER_INIT; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 4; - INSTRUCTION_STATS(CALL_ALLOC_AND_ENTER_INIT); - opcode = CALL_ALLOC_AND_ENTER_INIT; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); - _PyStackRef callable; - _PyStackRef self_or_null; - _PyStackRef init; - _PyStackRef self; - _PyStackRef *args; - _PyStackRef init_frame; - _PyStackRef new_frame; - /* Skip 1 cache entry */ - // _CHECK_PEP_523 - { - if (tstate->interp->eval_frame) { - UPDATE_MISS_STATS(CALL); - assert(_PyOpcode_Deopt[opcode] == (CALL)); - JUMP_TO_PREDICTED(TRACING_CALL); - } - } - // _CHECK_AND_ALLOCATE_OBJECT - { - self_or_null = stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; - uint32_t type_version = read_u32(&this_instr[2].cache); - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); - if (!PyStackRef_IsNull(self_or_null)) { - UPDATE_MISS_STATS(CALL); - assert(_PyOpcode_Deopt[opcode] == (CALL)); - JUMP_TO_PREDICTED(TRACING_CALL); - } - if (!PyType_Check(callable_o)) { - UPDATE_MISS_STATS(CALL); - assert(_PyOpcode_Deopt[opcode] == (CALL)); - JUMP_TO_PREDICTED(TRACING_CALL); - } - PyTypeObject *tp = (PyTypeObject *)callable_o; - if (FT_ATOMIC_LOAD_UINT32_RELAXED(tp->tp_version_tag) != type_version) { - UPDATE_MISS_STATS(CALL); - assert(_PyOpcode_Deopt[opcode] == (CALL)); - JUMP_TO_PREDICTED(TRACING_CALL); - } - assert(tp->tp_new == PyBaseObject_Type.tp_new); - assert(tp->tp_flags & Py_TPFLAGS_HEAPTYPE); - assert(tp->tp_alloc == PyType_GenericAlloc); - PyHeapTypeObject *cls = (PyHeapTypeObject *)callable_o; - PyFunctionObject *init_func = (PyFunctionObject *)FT_ATOMIC_LOAD_PTR_ACQUIRE(cls->_spec_cache.init); - PyCodeObject *code = (PyCodeObject *)init_func->func_code; - if (!_PyThreadState_HasStackSpace(tstate, code->co_framesize + _Py_InitCleanup.co_framesize)) { - UPDATE_MISS_STATS(CALL); - assert(_PyOpcode_Deopt[opcode] == (CALL)); - JUMP_TO_PREDICTED(TRACING_CALL); - } - STAT_INC(CALL, hit); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *self_o = PyType_GenericAlloc(tp, 0); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (self_o == NULL) { - TRACING_JUMP_TO_LABEL(error); - } - self_or_null = PyStackRef_FromPyObjectSteal(self_o); - _PyStackRef temp = callable; - callable = PyStackRef_FromPyObjectNew(init_func); - stack_pointer[-2 - oparg] = callable; - stack_pointer[-1 - oparg] = self_or_null; - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(temp); - stack_pointer = _PyFrame_GetStackPointer(frame); - } - // _CREATE_INIT_FRAME - { - args = &stack_pointer[-oparg]; - self = self_or_null; - init = callable; - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyInterpreterFrame *shim = _PyFrame_PushTrampolineUnchecked( - tstate, (PyCodeObject *)&_Py_InitCleanup, 1, frame); - stack_pointer = _PyFrame_GetStackPointer(frame); - assert(_PyFrame_GetBytecode(shim)[0].op.code == EXIT_INIT_CHECK); - assert(_PyFrame_GetBytecode(shim)[1].op.code == RETURN_VALUE); - shim->localsplus[0] = PyStackRef_DUP(self); - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyInterpreterFrame *temp = _PyEvalFramePushAndInit( - tstate, init, NULL, args-1, oparg+1, NULL, shim); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -2 - oparg; - assert(WITHIN_STACK_BOUNDS()); - if (temp == NULL) { - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyEval_FrameClearAndPop(tstate, shim); - stack_pointer = _PyFrame_GetStackPointer(frame); - TRACING_JUMP_TO_LABEL(error); - } - frame->return_offset = 1 + INLINE_CACHE_ENTRIES_CALL; - tstate->py_recursion_remaining--; - init_frame = PyStackRef_Wrap(temp); - } - // _PUSH_FRAME - { - new_frame = init_frame; - assert(tstate->interp->eval_frame == NULL); - _PyInterpreterFrame *temp = PyStackRef_Unwrap(new_frame); - _PyFrame_SetStackPointer(frame, stack_pointer); - assert(temp->previous == frame || temp->previous->previous == frame); - CALL_STAT_INC(inlined_py_calls); - frame = tstate->current_frame = temp; - tstate->py_recursion_remaining--; - LOAD_SP(); - LOAD_IP(0); - LLTRACE_RESUME_FRAME(); - } - TRACING_DISPATCH(); - } - - TRACING_TARGET(CALL_BOUND_METHOD_EXACT_ARGS) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = CALL_BOUND_METHOD_EXACT_ARGS; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 4; - INSTRUCTION_STATS(CALL_BOUND_METHOD_EXACT_ARGS); - opcode = CALL_BOUND_METHOD_EXACT_ARGS; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); - _PyStackRef callable; - _PyStackRef null; - _PyStackRef self_or_null; - _PyStackRef *args; - _PyStackRef new_frame; - /* Skip 1 cache entry */ - // _CHECK_PEP_523 - { - if (tstate->interp->eval_frame) { - UPDATE_MISS_STATS(CALL); - assert(_PyOpcode_Deopt[opcode] == (CALL)); - JUMP_TO_PREDICTED(TRACING_CALL); - } - } - // _CHECK_CALL_BOUND_METHOD_EXACT_ARGS - { - null = stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; - if (!PyStackRef_IsNull(null)) { - UPDATE_MISS_STATS(CALL); - assert(_PyOpcode_Deopt[opcode] == (CALL)); - JUMP_TO_PREDICTED(TRACING_CALL); - } - if (Py_TYPE(PyStackRef_AsPyObjectBorrow(callable)) != &PyMethod_Type) { - UPDATE_MISS_STATS(CALL); - assert(_PyOpcode_Deopt[opcode] == (CALL)); - JUMP_TO_PREDICTED(TRACING_CALL); - } - } - // _INIT_CALL_BOUND_METHOD_EXACT_ARGS - { - self_or_null = null; - assert(PyStackRef_IsNull(self_or_null)); - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); - STAT_INC(CALL, hit); - self_or_null = PyStackRef_FromPyObjectNew(((PyMethodObject *)callable_o)->im_self); - _PyStackRef temp = callable; - callable = PyStackRef_FromPyObjectNew(((PyMethodObject *)callable_o)->im_func); - stack_pointer[-2 - oparg] = callable; - stack_pointer[-1 - oparg] = self_or_null; - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(temp); - stack_pointer = _PyFrame_GetStackPointer(frame); - } - // flush - // _CHECK_FUNCTION_VERSION - { - uint32_t func_version = read_u32(&this_instr[2].cache); - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); - if (!PyFunction_Check(callable_o)) { - UPDATE_MISS_STATS(CALL); - assert(_PyOpcode_Deopt[opcode] == (CALL)); - JUMP_TO_PREDICTED(TRACING_CALL); - } - PyFunctionObject *func = (PyFunctionObject *)callable_o; - if (func->func_version != func_version) { - UPDATE_MISS_STATS(CALL); - assert(_PyOpcode_Deopt[opcode] == (CALL)); - JUMP_TO_PREDICTED(TRACING_CALL); - } - } - // _CHECK_FUNCTION_EXACT_ARGS - { - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); - assert(PyFunction_Check(callable_o)); - PyFunctionObject *func = (PyFunctionObject *)callable_o; - PyCodeObject *code = (PyCodeObject *)func->func_code; - if (code->co_argcount != oparg + (!PyStackRef_IsNull(self_or_null))) { - UPDATE_MISS_STATS(CALL); - assert(_PyOpcode_Deopt[opcode] == (CALL)); - JUMP_TO_PREDICTED(TRACING_CALL); - } - } - // _CHECK_STACK_SPACE - { - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); - PyFunctionObject *func = (PyFunctionObject *)callable_o; - PyCodeObject *code = (PyCodeObject *)func->func_code; - if (!_PyThreadState_HasStackSpace(tstate, code->co_framesize)) { - UPDATE_MISS_STATS(CALL); - assert(_PyOpcode_Deopt[opcode] == (CALL)); - JUMP_TO_PREDICTED(TRACING_CALL); - } - } - // _CHECK_RECURSION_REMAINING - { - if (tstate->py_recursion_remaining <= 1) { - UPDATE_MISS_STATS(CALL); - assert(_PyOpcode_Deopt[opcode] == (CALL)); - JUMP_TO_PREDICTED(TRACING_CALL); - } - } - // _INIT_CALL_PY_EXACT_ARGS - { - args = &stack_pointer[-oparg]; - int has_self = !PyStackRef_IsNull(self_or_null); - STAT_INC(CALL, hit); - _PyInterpreterFrame *pushed_frame = _PyFrame_PushUnchecked(tstate, callable, oparg + has_self, frame); - _PyStackRef *first_non_self_local = pushed_frame->localsplus + has_self; - pushed_frame->localsplus[0] = self_or_null; - for (int i = 0; i < oparg; i++) { - first_non_self_local[i] = args[i]; - } - new_frame = PyStackRef_Wrap(pushed_frame); - } - // _SAVE_RETURN_OFFSET - { - #if TIER_ONE - frame->return_offset = (uint16_t)(next_instr - this_instr); - #endif - #if TIER_TWO - frame->return_offset = oparg; - #endif - } - // _PUSH_FRAME - { - assert(tstate->interp->eval_frame == NULL); - _PyInterpreterFrame *temp = PyStackRef_Unwrap(new_frame); - stack_pointer += -2 - oparg; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - assert(temp->previous == frame || temp->previous->previous == frame); - CALL_STAT_INC(inlined_py_calls); - frame = tstate->current_frame = temp; - tstate->py_recursion_remaining--; - LOAD_SP(); - LOAD_IP(0); - LLTRACE_RESUME_FRAME(); - } - TRACING_DISPATCH(); - } - - TRACING_TARGET(CALL_BOUND_METHOD_GENERAL) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = CALL_BOUND_METHOD_GENERAL; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 4; - INSTRUCTION_STATS(CALL_BOUND_METHOD_GENERAL); - opcode = CALL_BOUND_METHOD_GENERAL; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); - _PyStackRef callable; - _PyStackRef null; - _PyStackRef self_or_null; - _PyStackRef *args; - _PyStackRef new_frame; - /* Skip 1 cache entry */ - // _CHECK_PEP_523 - { - if (tstate->interp->eval_frame) { - UPDATE_MISS_STATS(CALL); - assert(_PyOpcode_Deopt[opcode] == (CALL)); - JUMP_TO_PREDICTED(TRACING_CALL); - } - } - // _CHECK_METHOD_VERSION - { - null = stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; - uint32_t func_version = read_u32(&this_instr[2].cache); - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); - if (Py_TYPE(callable_o) != &PyMethod_Type) { - UPDATE_MISS_STATS(CALL); - assert(_PyOpcode_Deopt[opcode] == (CALL)); - JUMP_TO_PREDICTED(TRACING_CALL); - } - PyObject *func = ((PyMethodObject *)callable_o)->im_func; - if (!PyFunction_Check(func)) { - UPDATE_MISS_STATS(CALL); - assert(_PyOpcode_Deopt[opcode] == (CALL)); - JUMP_TO_PREDICTED(TRACING_CALL); - } - if (((PyFunctionObject *)func)->func_version != func_version) { - UPDATE_MISS_STATS(CALL); - assert(_PyOpcode_Deopt[opcode] == (CALL)); - JUMP_TO_PREDICTED(TRACING_CALL); - } - if (!PyStackRef_IsNull(null)) { - UPDATE_MISS_STATS(CALL); - assert(_PyOpcode_Deopt[opcode] == (CALL)); - JUMP_TO_PREDICTED(TRACING_CALL); - } - } - // _EXPAND_METHOD - { - self_or_null = null; - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); - assert(PyStackRef_IsNull(self_or_null)); - assert(Py_TYPE(callable_o) == &PyMethod_Type); - self_or_null = PyStackRef_FromPyObjectNew(((PyMethodObject *)callable_o)->im_self); - _PyStackRef temp = callable; - callable = PyStackRef_FromPyObjectNew(((PyMethodObject *)callable_o)->im_func); - assert(PyStackRef_FunctionCheck(callable)); - stack_pointer[-2 - oparg] = callable; - stack_pointer[-1 - oparg] = self_or_null; - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(temp); - stack_pointer = _PyFrame_GetStackPointer(frame); - } - // flush - // _CHECK_RECURSION_REMAINING - { - if (tstate->py_recursion_remaining <= 1) { - UPDATE_MISS_STATS(CALL); - assert(_PyOpcode_Deopt[opcode] == (CALL)); - JUMP_TO_PREDICTED(TRACING_CALL); - } - } - // _PY_FRAME_GENERAL - { - args = &stack_pointer[-oparg]; - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); - int total_args = oparg; - if (!PyStackRef_IsNull(self_or_null)) { - args--; - total_args++; - } - assert(Py_TYPE(callable_o) == &PyFunction_Type); - int code_flags = ((PyCodeObject*)PyFunction_GET_CODE(callable_o))->co_flags; - PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable_o)); - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyInterpreterFrame *temp = _PyEvalFramePushAndInit( - tstate, callable, locals, - args, total_args, NULL, frame - ); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -2 - oparg; - assert(WITHIN_STACK_BOUNDS()); - if (temp == NULL) { - TRACING_JUMP_TO_LABEL(error); - } - new_frame = PyStackRef_Wrap(temp); - } - // _SAVE_RETURN_OFFSET - { - #if TIER_ONE - frame->return_offset = (uint16_t)(next_instr - this_instr); - #endif - #if TIER_TWO - frame->return_offset = oparg; - #endif - } - // _PUSH_FRAME - { - assert(tstate->interp->eval_frame == NULL); - _PyInterpreterFrame *temp = PyStackRef_Unwrap(new_frame); - _PyFrame_SetStackPointer(frame, stack_pointer); - assert(temp->previous == frame || temp->previous->previous == frame); - CALL_STAT_INC(inlined_py_calls); - frame = tstate->current_frame = temp; - tstate->py_recursion_remaining--; - LOAD_SP(); - LOAD_IP(0); - LLTRACE_RESUME_FRAME(); - } - TRACING_DISPATCH(); - } - - TRACING_TARGET(CALL_BUILTIN_CLASS) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = CALL_BUILTIN_CLASS; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 4; - INSTRUCTION_STATS(CALL_BUILTIN_CLASS); - opcode = CALL_BUILTIN_CLASS; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); - _PyStackRef callable; - _PyStackRef self_or_null; - _PyStackRef *args; - _PyStackRef res; - /* Skip 1 cache entry */ - /* Skip 2 cache entries */ - // _CALL_BUILTIN_CLASS - { - args = &stack_pointer[-oparg]; - self_or_null = stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); - if (!PyType_Check(callable_o)) { - UPDATE_MISS_STATS(CALL); - assert(_PyOpcode_Deopt[opcode] == (CALL)); - JUMP_TO_PREDICTED(TRACING_CALL); - } - PyTypeObject *tp = (PyTypeObject *)callable_o; - int total_args = oparg; - _PyStackRef *arguments = args; - if (!PyStackRef_IsNull(self_or_null)) { - arguments--; - total_args++; - } - if (tp->tp_vectorcall == NULL) { - UPDATE_MISS_STATS(CALL); - assert(_PyOpcode_Deopt[opcode] == (CALL)); - JUMP_TO_PREDICTED(TRACING_CALL); - } - STAT_INC(CALL, hit); - STACKREFS_TO_PYOBJECTS(arguments, total_args, args_o); - if (CONVERSION_FAILED(args_o)) { - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyStackRef tmp; - for (int _i = oparg; --_i >= 0;) { - tmp = args[_i]; - args[_i] = PyStackRef_NULL; - PyStackRef_CLOSE(tmp); - } - tmp = self_or_null; - self_or_null = PyStackRef_NULL; - stack_pointer[-1 - oparg] = self_or_null; - PyStackRef_XCLOSE(tmp); - tmp = callable; - callable = PyStackRef_NULL; - stack_pointer[-2 - oparg] = callable; - PyStackRef_CLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -2 - oparg; - assert(WITHIN_STACK_BOUNDS()); - TRACING_JUMP_TO_LABEL(error); - } - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *res_o = tp->tp_vectorcall((PyObject *)tp, args_o, total_args, NULL); - stack_pointer = _PyFrame_GetStackPointer(frame); - STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyStackRef tmp; - for (int _i = oparg; --_i >= 0;) { - tmp = args[_i]; - args[_i] = PyStackRef_NULL; - PyStackRef_CLOSE(tmp); - } - tmp = self_or_null; - self_or_null = PyStackRef_NULL; - stack_pointer[-1 - oparg] = self_or_null; - PyStackRef_XCLOSE(tmp); - tmp = callable; - callable = PyStackRef_NULL; - stack_pointer[-2 - oparg] = callable; - PyStackRef_CLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -2 - oparg; - assert(WITHIN_STACK_BOUNDS()); - if (res_o == NULL) { - TRACING_JUMP_TO_LABEL(error); - } - res = PyStackRef_FromPyObjectSteal(res_o); - } - // _CHECK_PERIODIC_AT_END - { - stack_pointer[0] = res; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = check_periodics(tstate); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err != 0) { - TRACING_JUMP_TO_LABEL(error); - } - } - TRACING_DISPATCH(); - } - - TRACING_TARGET(CALL_BUILTIN_FAST) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = CALL_BUILTIN_FAST; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 4; - INSTRUCTION_STATS(CALL_BUILTIN_FAST); - opcode = CALL_BUILTIN_FAST; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); - _PyStackRef callable; - _PyStackRef self_or_null; - _PyStackRef *args; - _PyStackRef res; - /* Skip 1 cache entry */ - /* Skip 2 cache entries */ - // _CALL_BUILTIN_FAST - { - args = &stack_pointer[-oparg]; - self_or_null = stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); - int total_args = oparg; - _PyStackRef *arguments = args; - if (!PyStackRef_IsNull(self_or_null)) { - arguments--; - total_args++; - } - if (!PyCFunction_CheckExact(callable_o)) { - UPDATE_MISS_STATS(CALL); - assert(_PyOpcode_Deopt[opcode] == (CALL)); - JUMP_TO_PREDICTED(TRACING_CALL); - } - if (PyCFunction_GET_FLAGS(callable_o) != METH_FASTCALL) { - UPDATE_MISS_STATS(CALL); - assert(_PyOpcode_Deopt[opcode] == (CALL)); - JUMP_TO_PREDICTED(TRACING_CALL); - } - STAT_INC(CALL, hit); - PyCFunction cfunc = PyCFunction_GET_FUNCTION(callable_o); - STACKREFS_TO_PYOBJECTS(arguments, total_args, args_o); - if (CONVERSION_FAILED(args_o)) { - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyStackRef tmp; - for (int _i = oparg; --_i >= 0;) { - tmp = args[_i]; - args[_i] = PyStackRef_NULL; - PyStackRef_CLOSE(tmp); - } - tmp = self_or_null; - self_or_null = PyStackRef_NULL; - stack_pointer[-1 - oparg] = self_or_null; - PyStackRef_XCLOSE(tmp); - tmp = callable; - callable = PyStackRef_NULL; - stack_pointer[-2 - oparg] = callable; - PyStackRef_CLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -2 - oparg; - assert(WITHIN_STACK_BOUNDS()); - TRACING_JUMP_TO_LABEL(error); - } - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *res_o = _PyCFunctionFast_CAST(cfunc)( - PyCFunction_GET_SELF(callable_o), - args_o, - total_args); - stack_pointer = _PyFrame_GetStackPointer(frame); - STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); - assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyStackRef tmp; - for (int _i = oparg; --_i >= 0;) { - tmp = args[_i]; - args[_i] = PyStackRef_NULL; - PyStackRef_CLOSE(tmp); - } - tmp = self_or_null; - self_or_null = PyStackRef_NULL; - stack_pointer[-1 - oparg] = self_or_null; - PyStackRef_XCLOSE(tmp); - tmp = callable; - callable = PyStackRef_NULL; - stack_pointer[-2 - oparg] = callable; - PyStackRef_CLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -2 - oparg; - assert(WITHIN_STACK_BOUNDS()); - if (res_o == NULL) { - TRACING_JUMP_TO_LABEL(error); - } - res = PyStackRef_FromPyObjectSteal(res_o); - } - // _CHECK_PERIODIC_AT_END - { - stack_pointer[0] = res; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = check_periodics(tstate); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err != 0) { - TRACING_JUMP_TO_LABEL(error); - } - } - TRACING_DISPATCH(); - } - - TRACING_TARGET(CALL_BUILTIN_FAST_WITH_KEYWORDS) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = CALL_BUILTIN_FAST_WITH_KEYWORDS; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 4; - INSTRUCTION_STATS(CALL_BUILTIN_FAST_WITH_KEYWORDS); - opcode = CALL_BUILTIN_FAST_WITH_KEYWORDS; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); - _PyStackRef callable; - _PyStackRef self_or_null; - _PyStackRef *args; - _PyStackRef res; - /* Skip 1 cache entry */ - /* Skip 2 cache entries */ - // _CALL_BUILTIN_FAST_WITH_KEYWORDS - { - args = &stack_pointer[-oparg]; - self_or_null = stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); - int total_args = oparg; - _PyStackRef *arguments = args; - if (!PyStackRef_IsNull(self_or_null)) { - arguments--; - total_args++; - } - if (!PyCFunction_CheckExact(callable_o)) { - UPDATE_MISS_STATS(CALL); - assert(_PyOpcode_Deopt[opcode] == (CALL)); - JUMP_TO_PREDICTED(TRACING_CALL); - } - if (PyCFunction_GET_FLAGS(callable_o) != (METH_FASTCALL | METH_KEYWORDS)) { - UPDATE_MISS_STATS(CALL); - assert(_PyOpcode_Deopt[opcode] == (CALL)); - JUMP_TO_PREDICTED(TRACING_CALL); - } - STAT_INC(CALL, hit); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyCFunctionFastWithKeywords cfunc = - _PyCFunctionFastWithKeywords_CAST(PyCFunction_GET_FUNCTION(callable_o)); - stack_pointer = _PyFrame_GetStackPointer(frame); - STACKREFS_TO_PYOBJECTS(arguments, total_args, args_o); - if (CONVERSION_FAILED(args_o)) { - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyStackRef tmp; - for (int _i = oparg; --_i >= 0;) { - tmp = args[_i]; - args[_i] = PyStackRef_NULL; - PyStackRef_CLOSE(tmp); - } - tmp = self_or_null; - self_or_null = PyStackRef_NULL; - stack_pointer[-1 - oparg] = self_or_null; - PyStackRef_XCLOSE(tmp); - tmp = callable; - callable = PyStackRef_NULL; - stack_pointer[-2 - oparg] = callable; - PyStackRef_CLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -2 - oparg; - assert(WITHIN_STACK_BOUNDS()); - TRACING_JUMP_TO_LABEL(error); - } - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *res_o = cfunc(PyCFunction_GET_SELF(callable_o), args_o, total_args, NULL); - stack_pointer = _PyFrame_GetStackPointer(frame); - STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); - assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyStackRef tmp; - for (int _i = oparg; --_i >= 0;) { - tmp = args[_i]; - args[_i] = PyStackRef_NULL; - PyStackRef_CLOSE(tmp); - } - tmp = self_or_null; - self_or_null = PyStackRef_NULL; - stack_pointer[-1 - oparg] = self_or_null; - PyStackRef_XCLOSE(tmp); - tmp = callable; - callable = PyStackRef_NULL; - stack_pointer[-2 - oparg] = callable; - PyStackRef_CLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -2 - oparg; - assert(WITHIN_STACK_BOUNDS()); - if (res_o == NULL) { - TRACING_JUMP_TO_LABEL(error); - } - res = PyStackRef_FromPyObjectSteal(res_o); - } - // _CHECK_PERIODIC_AT_END - { - stack_pointer[0] = res; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = check_periodics(tstate); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err != 0) { - TRACING_JUMP_TO_LABEL(error); - } - } - TRACING_DISPATCH(); - } - - TRACING_TARGET(CALL_BUILTIN_O) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = CALL_BUILTIN_O; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 4; - INSTRUCTION_STATS(CALL_BUILTIN_O); - opcode = CALL_BUILTIN_O; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); - _PyStackRef callable; - _PyStackRef self_or_null; - _PyStackRef *args; - _PyStackRef res; - /* Skip 1 cache entry */ - /* Skip 2 cache entries */ - // _CALL_BUILTIN_O - { - args = &stack_pointer[-oparg]; - self_or_null = stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); - int total_args = oparg; - if (!PyStackRef_IsNull(self_or_null)) { - args--; - total_args++; - } - if (total_args != 1) { - UPDATE_MISS_STATS(CALL); - assert(_PyOpcode_Deopt[opcode] == (CALL)); - JUMP_TO_PREDICTED(TRACING_CALL); - } - if (!PyCFunction_CheckExact(callable_o)) { - UPDATE_MISS_STATS(CALL); - assert(_PyOpcode_Deopt[opcode] == (CALL)); - JUMP_TO_PREDICTED(TRACING_CALL); - } - if (PyCFunction_GET_FLAGS(callable_o) != METH_O) { - UPDATE_MISS_STATS(CALL); - assert(_PyOpcode_Deopt[opcode] == (CALL)); - JUMP_TO_PREDICTED(TRACING_CALL); - } - if (_Py_ReachedRecursionLimit(tstate)) { - UPDATE_MISS_STATS(CALL); - assert(_PyOpcode_Deopt[opcode] == (CALL)); - JUMP_TO_PREDICTED(TRACING_CALL); - } - STAT_INC(CALL, hit); - PyCFunction cfunc = PyCFunction_GET_FUNCTION(callable_o); - _PyStackRef arg = args[0]; - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *res_o = _PyCFunction_TrampolineCall(cfunc, PyCFunction_GET_SELF(callable_o), PyStackRef_AsPyObjectBorrow(arg)); - stack_pointer = _PyFrame_GetStackPointer(frame); - _Py_LeaveRecursiveCallTstate(tstate); - assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(arg); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -2 - oparg; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(callable); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (res_o == NULL) { - TRACING_JUMP_TO_LABEL(error); - } - res = PyStackRef_FromPyObjectSteal(res_o); - } - // _CHECK_PERIODIC_AT_END - { - stack_pointer[0] = res; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = check_periodics(tstate); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err != 0) { - TRACING_JUMP_TO_LABEL(error); - } - } - TRACING_DISPATCH(); - } - - TRACING_TARGET(CALL_FUNCTION_EX) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = CALL_FUNCTION_EX; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(CALL_FUNCTION_EX); - opcode = CALL_FUNCTION_EX; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - opcode = CALL_FUNCTION_EX; - _PyStackRef func; - _PyStackRef callargs; - _PyStackRef func_st; - _PyStackRef null; - _PyStackRef callargs_st; - _PyStackRef kwargs_st; - _PyStackRef result; - // _MAKE_CALLARGS_A_TUPLE - { - callargs = stack_pointer[-2]; - func = stack_pointer[-4]; - PyObject *callargs_o = PyStackRef_AsPyObjectBorrow(callargs); - if (!PyTuple_CheckExact(callargs_o)) { - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = _Py_Check_ArgsIterable(tstate, PyStackRef_AsPyObjectBorrow(func), callargs_o); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err < 0) { - TRACING_JUMP_TO_LABEL(error); - } - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *tuple_o = PySequence_Tuple(callargs_o); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (tuple_o == NULL) { - TRACING_JUMP_TO_LABEL(error); - } - _PyStackRef temp = callargs; - callargs = PyStackRef_FromPyObjectSteal(tuple_o); - stack_pointer[-2] = callargs; - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(temp); - stack_pointer = _PyFrame_GetStackPointer(frame); - } - } - // _DO_CALL_FUNCTION_EX - { - kwargs_st = stack_pointer[-1]; - callargs_st = callargs; - null = stack_pointer[-3]; - func_st = func; - (void)null; - PyObject *func = PyStackRef_AsPyObjectBorrow(func_st); - EVAL_CALL_STAT_INC_IF_FUNCTION(EVAL_CALL_FUNCTION_EX, func); - PyObject *result_o; - assert(!_PyErr_Occurred(tstate)); - if (opcode == INSTRUMENTED_CALL_FUNCTION_EX) { - PyObject *callargs = PyStackRef_AsPyObjectBorrow(callargs_st); - PyObject *kwargs = PyStackRef_AsPyObjectBorrow(kwargs_st); - assert(kwargs == NULL || PyDict_CheckExact(kwargs)); - assert(PyTuple_CheckExact(callargs)); - PyObject *arg = PyTuple_GET_SIZE(callargs) > 0 ? - PyTuple_GET_ITEM(callargs, 0) : &_PyInstrumentation_MISSING; - stack_pointer[-2] = callargs_st; - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = _Py_call_instrumentation_2args( - tstate, PY_MONITORING_EVENT_CALL, - frame, this_instr, func, arg); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err) { - TRACING_JUMP_TO_LABEL(error); - } - _PyFrame_SetStackPointer(frame, stack_pointer); - result_o = PyObject_Call(func, callargs, kwargs); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (!PyFunction_Check(func) && !PyMethod_Check(func)) { - if (result_o == NULL) { - _PyFrame_SetStackPointer(frame, stack_pointer); - _Py_call_instrumentation_exc2( - tstate, PY_MONITORING_EVENT_C_RAISE, - frame, this_instr, func, arg); - stack_pointer = _PyFrame_GetStackPointer(frame); - } - else { - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = _Py_call_instrumentation_2args( - tstate, PY_MONITORING_EVENT_C_RETURN, - frame, this_instr, func, arg); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err < 0) { - _PyFrame_SetStackPointer(frame, stack_pointer); - Py_CLEAR(result_o); - stack_pointer = _PyFrame_GetStackPointer(frame); - } - } - } - } - else { - if (Py_TYPE(func) == &PyFunction_Type && - tstate->interp->eval_frame == NULL && - ((PyFunctionObject *)func)->vectorcall == _PyFunction_Vectorcall) { - PyObject *callargs = PyStackRef_AsPyObjectSteal(callargs_st); - assert(PyTuple_CheckExact(callargs)); - PyObject *kwargs = PyStackRef_IsNull(kwargs_st) ? NULL : PyStackRef_AsPyObjectSteal(kwargs_st); - assert(kwargs == NULL || PyDict_CheckExact(kwargs)); - Py_ssize_t nargs = PyTuple_GET_SIZE(callargs); - int code_flags = ((PyCodeObject *)PyFunction_GET_CODE(func))->co_flags; - PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(func)); - stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyInterpreterFrame *new_frame = _PyEvalFramePushAndInit_Ex( - tstate, func_st, locals, - nargs, callargs, kwargs, frame); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); - if (new_frame == NULL) { - TRACING_JUMP_TO_LABEL(error); - } - assert( 1u == 1); - frame->return_offset = 1; - TRACING_DISPATCH_INLINED(new_frame); - } - PyObject *callargs = PyStackRef_AsPyObjectBorrow(callargs_st); - assert(PyTuple_CheckExact(callargs)); - PyObject *kwargs = PyStackRef_AsPyObjectBorrow(kwargs_st); - assert(kwargs == NULL || PyDict_CheckExact(kwargs)); - stack_pointer[-2] = callargs_st; - _PyFrame_SetStackPointer(frame, stack_pointer); - result_o = PyObject_Call(func, callargs, kwargs); - stack_pointer = _PyFrame_GetStackPointer(frame); - } - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_XCLOSE(kwargs_st); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(callargs_st); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(func_st); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (result_o == NULL) { - TRACING_JUMP_TO_LABEL(error); - } - result = PyStackRef_FromPyObjectSteal(result_o); - } - // _CHECK_PERIODIC_AT_END - { - stack_pointer[0] = result; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = check_periodics(tstate); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err != 0) { - TRACING_JUMP_TO_LABEL(error); - } - } - TRACING_DISPATCH(); - } - - TRACING_TARGET(CALL_INTRINSIC_1) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = CALL_INTRINSIC_1; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(CALL_INTRINSIC_1); - opcode = CALL_INTRINSIC_1; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - _PyStackRef value; - _PyStackRef res; - value = stack_pointer[-1]; - assert(oparg <= MAX_INTRINSIC_1); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *res_o = _PyIntrinsics_UnaryFunctions[oparg].func(tstate, PyStackRef_AsPyObjectBorrow(value)); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(value); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (res_o == NULL) { - TRACING_JUMP_TO_LABEL(error); - } - res = PyStackRef_FromPyObjectSteal(res_o); - stack_pointer[0] = res; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - TRACING_DISPATCH(); - } - - TRACING_TARGET(CALL_INTRINSIC_2) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = CALL_INTRINSIC_2; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(CALL_INTRINSIC_2); - opcode = CALL_INTRINSIC_2; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - _PyStackRef value2_st; - _PyStackRef value1_st; - _PyStackRef res; - value1_st = stack_pointer[-1]; - value2_st = stack_pointer[-2]; - assert(oparg <= MAX_INTRINSIC_2); - PyObject *value1 = PyStackRef_AsPyObjectBorrow(value1_st); - PyObject *value2 = PyStackRef_AsPyObjectBorrow(value2_st); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *res_o = _PyIntrinsics_BinaryFunctions[oparg].func(tstate, value2, value1); - _PyStackRef tmp = value1_st; - value1_st = PyStackRef_NULL; - stack_pointer[-1] = value1_st; - PyStackRef_CLOSE(tmp); - tmp = value2_st; - value2_st = PyStackRef_NULL; - stack_pointer[-2] = value2_st; - PyStackRef_CLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); - if (res_o == NULL) { - TRACING_JUMP_TO_LABEL(error); - } - res = PyStackRef_FromPyObjectSteal(res_o); - stack_pointer[0] = res; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - TRACING_DISPATCH(); - } - - TRACING_TARGET(CALL_ISINSTANCE) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = CALL_ISINSTANCE; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 4; - INSTRUCTION_STATS(CALL_ISINSTANCE); - opcode = CALL_ISINSTANCE; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); - _PyStackRef null; - _PyStackRef callable; - _PyStackRef instance; - _PyStackRef cls; - _PyStackRef res; - /* Skip 1 cache entry */ - /* Skip 2 cache entries */ - // _GUARD_THIRD_NULL - { - null = stack_pointer[-3]; - if (!PyStackRef_IsNull(null)) { - UPDATE_MISS_STATS(CALL); - assert(_PyOpcode_Deopt[opcode] == (CALL)); - JUMP_TO_PREDICTED(TRACING_CALL); - } - } - // _GUARD_CALLABLE_ISINSTANCE - { - callable = stack_pointer[-4]; - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); - PyInterpreterState *interp = tstate->interp; - if (callable_o != interp->callable_cache.isinstance) { - UPDATE_MISS_STATS(CALL); - assert(_PyOpcode_Deopt[opcode] == (CALL)); - JUMP_TO_PREDICTED(TRACING_CALL); - } - } - // _CALL_ISINSTANCE - { - cls = stack_pointer[-1]; - instance = stack_pointer[-2]; - STAT_INC(CALL, hit); - PyObject *inst_o = PyStackRef_AsPyObjectBorrow(instance); - PyObject *cls_o = PyStackRef_AsPyObjectBorrow(cls); - _PyFrame_SetStackPointer(frame, stack_pointer); - int retval = PyObject_IsInstance(inst_o, cls_o); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (retval < 0) { - TRACING_JUMP_TO_LABEL(error); - } - (void)null; - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(cls); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(instance); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(callable); - stack_pointer = _PyFrame_GetStackPointer(frame); - res = retval ? PyStackRef_True : PyStackRef_False; - assert((!PyStackRef_IsNull(res)) ^ (_PyErr_Occurred(tstate) != NULL)); - } - stack_pointer[0] = res; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - TRACING_DISPATCH(); - } - - TRACING_TARGET(CALL_KW) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = CALL_KW; - (void)(opcode); - #endif - frame->instr_ptr = next_instr; - next_instr += 4; - INSTRUCTION_STATS(CALL_KW); - PREDICTED_TRACING_CALL_KW:; - _Py_CODEUNIT* const this_instr = next_instr - 4; - (void)this_instr; - opcode = CALL_KW; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - opcode = CALL_KW; - _PyStackRef callable; - _PyStackRef self_or_null; - _PyStackRef *args; - _PyStackRef kwnames; - _PyStackRef res; - // _SPECIALIZE_CALL_KW - { - self_or_null = stack_pointer[-2 - oparg]; - callable = stack_pointer[-3 - oparg]; - uint16_t counter = read_u16(&this_instr[1].cache); - (void)counter; - #if ENABLE_SPECIALIZATION_FT - if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { - next_instr = this_instr; - _PyFrame_SetStackPointer(frame, stack_pointer); - _Py_Specialize_CallKw(callable, next_instr, oparg + !PyStackRef_IsNull(self_or_null)); - stack_pointer = _PyFrame_GetStackPointer(frame); - DISPATCH_SAME_OPARG(); - } - OPCODE_DEFERRED_INC(CALL_KW); - ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); - #endif /* ENABLE_SPECIALIZATION_FT */ - } - /* Skip 2 cache entries */ - // _MAYBE_EXPAND_METHOD_KW - { - if (PyStackRef_TYPE(callable) == &PyMethod_Type && PyStackRef_IsNull(self_or_null)) { - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); - PyObject *self = ((PyMethodObject *)callable_o)->im_self; - self_or_null = PyStackRef_FromPyObjectNew(self); - PyObject *method = ((PyMethodObject *)callable_o)->im_func; - _PyStackRef temp = callable; - callable = PyStackRef_FromPyObjectNew(method); - stack_pointer[-3 - oparg] = callable; - stack_pointer[-2 - oparg] = self_or_null; - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(temp); - stack_pointer = _PyFrame_GetStackPointer(frame); - } - } - // _DO_CALL_KW - { - kwnames = stack_pointer[-1]; - args = &stack_pointer[-1 - oparg]; - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); - PyObject *kwnames_o = PyStackRef_AsPyObjectBorrow(kwnames); - int total_args = oparg; - _PyStackRef *arguments = args; - if (!PyStackRef_IsNull(self_or_null)) { - arguments--; - total_args++; - } - int positional_args = total_args - (int)PyTuple_GET_SIZE(kwnames_o); - if (Py_TYPE(callable_o) == &PyFunction_Type && - tstate->interp->eval_frame == NULL && - ((PyFunctionObject *)callable_o)->vectorcall == _PyFunction_Vectorcall) - { - int code_flags = ((PyCodeObject*)PyFunction_GET_CODE(callable_o))->co_flags; - PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable_o)); - stack_pointer[-3 - oparg] = callable; - stack_pointer[-2 - oparg] = self_or_null; - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyInterpreterFrame *new_frame = _PyEvalFramePushAndInit( - tstate, callable, locals, - arguments, positional_args, kwnames_o, frame - ); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -3 - oparg; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(kwnames); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (new_frame == NULL) { - TRACING_JUMP_TO_LABEL(error); - } - assert( 4u == 1 + INLINE_CACHE_ENTRIES_CALL_KW); - frame->return_offset = 4u ; - TRACING_DISPATCH_INLINED(new_frame); - } - STACKREFS_TO_PYOBJECTS(arguments, total_args, args_o); - if (CONVERSION_FAILED(args_o)) { - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyStackRef tmp = kwnames; - kwnames = PyStackRef_NULL; - stack_pointer[-3 - oparg] = callable; - stack_pointer[-2 - oparg] = self_or_null; - stack_pointer[-1] = kwnames; - PyStackRef_CLOSE(tmp); - for (int _i = oparg; --_i >= 0;) { - tmp = args[_i]; - args[_i] = PyStackRef_NULL; - PyStackRef_CLOSE(tmp); - } - tmp = self_or_null; - self_or_null = PyStackRef_NULL; - stack_pointer[-2 - oparg] = self_or_null; - PyStackRef_XCLOSE(tmp); - tmp = callable; - callable = PyStackRef_NULL; - stack_pointer[-3 - oparg] = callable; - PyStackRef_CLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -3 - oparg; - assert(WITHIN_STACK_BOUNDS()); - TRACING_JUMP_TO_LABEL(error); - } - stack_pointer[-3 - oparg] = callable; - stack_pointer[-2 - oparg] = self_or_null; - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *res_o = PyObject_Vectorcall( - callable_o, args_o, - positional_args | PY_VECTORCALL_ARGUMENTS_OFFSET, - kwnames_o); - stack_pointer = _PyFrame_GetStackPointer(frame); - STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); - if (opcode == INSTRUMENTED_CALL_KW) { - PyObject *arg = total_args == 0 ? - &_PyInstrumentation_MISSING : PyStackRef_AsPyObjectBorrow(arguments[0]); - if (res_o == NULL) { - _PyFrame_SetStackPointer(frame, stack_pointer); - _Py_call_instrumentation_exc2( - tstate, PY_MONITORING_EVENT_C_RAISE, - frame, this_instr, callable_o, arg); - stack_pointer = _PyFrame_GetStackPointer(frame); - } - else { - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = _Py_call_instrumentation_2args( - tstate, PY_MONITORING_EVENT_C_RETURN, - frame, this_instr, callable_o, arg); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err < 0) { - _PyFrame_SetStackPointer(frame, stack_pointer); - Py_CLEAR(res_o); - stack_pointer = _PyFrame_GetStackPointer(frame); - } - } - } - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyStackRef tmp = kwnames; - kwnames = PyStackRef_NULL; - stack_pointer[-1] = kwnames; - PyStackRef_CLOSE(tmp); - for (int _i = oparg; --_i >= 0;) { - tmp = args[_i]; - args[_i] = PyStackRef_NULL; - PyStackRef_CLOSE(tmp); - } - tmp = self_or_null; - self_or_null = PyStackRef_NULL; - stack_pointer[-2 - oparg] = self_or_null; - PyStackRef_XCLOSE(tmp); - tmp = callable; - callable = PyStackRef_NULL; - stack_pointer[-3 - oparg] = callable; - PyStackRef_CLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -3 - oparg; - assert(WITHIN_STACK_BOUNDS()); - if (res_o == NULL) { - TRACING_JUMP_TO_LABEL(error); - } - res = PyStackRef_FromPyObjectSteal(res_o); - } - stack_pointer[0] = res; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - TRACING_DISPATCH(); - } - - TRACING_TARGET(CALL_KW_BOUND_METHOD) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = CALL_KW_BOUND_METHOD; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 4; - INSTRUCTION_STATS(CALL_KW_BOUND_METHOD); - opcode = CALL_KW_BOUND_METHOD; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - static_assert(INLINE_CACHE_ENTRIES_CALL_KW == 3, "incorrect cache size"); - _PyStackRef callable; - _PyStackRef null; - _PyStackRef self_or_null; - _PyStackRef *args; - _PyStackRef kwnames; - _PyStackRef new_frame; - /* Skip 1 cache entry */ - // _CHECK_PEP_523 - { - if (tstate->interp->eval_frame) { - UPDATE_MISS_STATS(CALL_KW); - assert(_PyOpcode_Deopt[opcode] == (CALL_KW)); - JUMP_TO_PREDICTED(TRACING_CALL_KW); - } - } - // _CHECK_METHOD_VERSION_KW - { - null = stack_pointer[-2 - oparg]; - callable = stack_pointer[-3 - oparg]; - uint32_t func_version = read_u32(&this_instr[2].cache); - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); - if (Py_TYPE(callable_o) != &PyMethod_Type) { - UPDATE_MISS_STATS(CALL_KW); - assert(_PyOpcode_Deopt[opcode] == (CALL_KW)); - JUMP_TO_PREDICTED(TRACING_CALL_KW); - } - PyObject *func = ((PyMethodObject *)callable_o)->im_func; - if (!PyFunction_Check(func)) { - UPDATE_MISS_STATS(CALL_KW); - assert(_PyOpcode_Deopt[opcode] == (CALL_KW)); - JUMP_TO_PREDICTED(TRACING_CALL_KW); - } - if (((PyFunctionObject *)func)->func_version != func_version) { - UPDATE_MISS_STATS(CALL_KW); - assert(_PyOpcode_Deopt[opcode] == (CALL_KW)); - JUMP_TO_PREDICTED(TRACING_CALL_KW); - } - if (!PyStackRef_IsNull(null)) { - UPDATE_MISS_STATS(CALL_KW); - assert(_PyOpcode_Deopt[opcode] == (CALL_KW)); - JUMP_TO_PREDICTED(TRACING_CALL_KW); - } - } - // _EXPAND_METHOD_KW - { - self_or_null = null; - assert(PyStackRef_IsNull(self_or_null)); - _PyStackRef callable_s = callable; - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); - assert(Py_TYPE(callable_o) == &PyMethod_Type); - self_or_null = PyStackRef_FromPyObjectNew(((PyMethodObject *)callable_o)->im_self); - callable = PyStackRef_FromPyObjectNew(((PyMethodObject *)callable_o)->im_func); - assert(PyStackRef_FunctionCheck(callable)); - stack_pointer[-3 - oparg] = callable; - stack_pointer[-2 - oparg] = self_or_null; - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(callable_s); - stack_pointer = _PyFrame_GetStackPointer(frame); - } - // flush - // _PY_FRAME_KW - { - kwnames = stack_pointer[-1]; - args = &stack_pointer[-1 - oparg]; - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); - int total_args = oparg; - _PyStackRef *arguments = args; - if (!PyStackRef_IsNull(self_or_null)) { - arguments--; - total_args++; - } - PyObject *kwnames_o = PyStackRef_AsPyObjectBorrow(kwnames); - int positional_args = total_args - (int)PyTuple_GET_SIZE(kwnames_o); - assert(Py_TYPE(callable_o) == &PyFunction_Type); - int code_flags = ((PyCodeObject*)PyFunction_GET_CODE(callable_o))->co_flags; - PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable_o)); - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyInterpreterFrame *temp = _PyEvalFramePushAndInit( - tstate, callable, locals, - arguments, positional_args, kwnames_o, frame - ); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(kwnames); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -2 - oparg; - assert(WITHIN_STACK_BOUNDS()); - if (temp == NULL) { - TRACING_JUMP_TO_LABEL(error); - } - new_frame = PyStackRef_Wrap(temp); - } - // _SAVE_RETURN_OFFSET - { - #if TIER_ONE - frame->return_offset = (uint16_t)(next_instr - this_instr); - #endif - #if TIER_TWO - frame->return_offset = oparg; - #endif - } - // _PUSH_FRAME - { - assert(tstate->interp->eval_frame == NULL); - _PyInterpreterFrame *temp = PyStackRef_Unwrap(new_frame); - _PyFrame_SetStackPointer(frame, stack_pointer); - assert(temp->previous == frame || temp->previous->previous == frame); - CALL_STAT_INC(inlined_py_calls); - frame = tstate->current_frame = temp; - tstate->py_recursion_remaining--; - LOAD_SP(); - LOAD_IP(0); - LLTRACE_RESUME_FRAME(); - } - TRACING_DISPATCH(); - } - - TRACING_TARGET(CALL_KW_NON_PY) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = CALL_KW_NON_PY; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 4; - INSTRUCTION_STATS(CALL_KW_NON_PY); - opcode = CALL_KW_NON_PY; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - opcode = CALL_KW_NON_PY; - static_assert(INLINE_CACHE_ENTRIES_CALL_KW == 3, "incorrect cache size"); - _PyStackRef callable; - _PyStackRef self_or_null; - _PyStackRef *args; - _PyStackRef kwnames; - _PyStackRef res; - /* Skip 1 cache entry */ - /* Skip 2 cache entries */ - // _CHECK_IS_NOT_PY_CALLABLE_KW - { - callable = stack_pointer[-3 - oparg]; - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); - if (PyFunction_Check(callable_o)) { - UPDATE_MISS_STATS(CALL_KW); - assert(_PyOpcode_Deopt[opcode] == (CALL_KW)); - JUMP_TO_PREDICTED(TRACING_CALL_KW); - } - if (Py_TYPE(callable_o) == &PyMethod_Type) { - UPDATE_MISS_STATS(CALL_KW); - assert(_PyOpcode_Deopt[opcode] == (CALL_KW)); - JUMP_TO_PREDICTED(TRACING_CALL_KW); - } - } - // _CALL_KW_NON_PY - { - kwnames = stack_pointer[-1]; - args = &stack_pointer[-1 - oparg]; - self_or_null = stack_pointer[-2 - oparg]; - #if TIER_ONE - assert(opcode != INSTRUMENTED_CALL); - #endif - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); - int total_args = oparg; - _PyStackRef *arguments = args; - if (!PyStackRef_IsNull(self_or_null)) { - arguments--; - total_args++; - } - STACKREFS_TO_PYOBJECTS(arguments, total_args, args_o); - if (CONVERSION_FAILED(args_o)) { - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyStackRef tmp = kwnames; - kwnames = PyStackRef_NULL; - stack_pointer[-1] = kwnames; - PyStackRef_CLOSE(tmp); - for (int _i = oparg; --_i >= 0;) { - tmp = args[_i]; - args[_i] = PyStackRef_NULL; - PyStackRef_CLOSE(tmp); - } - tmp = self_or_null; - self_or_null = PyStackRef_NULL; - stack_pointer[-2 - oparg] = self_or_null; - PyStackRef_XCLOSE(tmp); - tmp = callable; - callable = PyStackRef_NULL; - stack_pointer[-3 - oparg] = callable; - PyStackRef_CLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -3 - oparg; - assert(WITHIN_STACK_BOUNDS()); - TRACING_JUMP_TO_LABEL(error); - } - PyObject *kwnames_o = PyStackRef_AsPyObjectBorrow(kwnames); - int positional_args = total_args - (int)PyTuple_GET_SIZE(kwnames_o); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *res_o = PyObject_Vectorcall( - callable_o, args_o, - positional_args | PY_VECTORCALL_ARGUMENTS_OFFSET, - kwnames_o); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(kwnames); - stack_pointer = _PyFrame_GetStackPointer(frame); - STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); - assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyStackRef tmp; - for (int _i = oparg; --_i >= 0;) { - tmp = args[_i]; - args[_i] = PyStackRef_NULL; - PyStackRef_CLOSE(tmp); - } - tmp = self_or_null; - self_or_null = PyStackRef_NULL; - stack_pointer[-1 - oparg] = self_or_null; - PyStackRef_XCLOSE(tmp); - tmp = callable; - callable = PyStackRef_NULL; - stack_pointer[-2 - oparg] = callable; - PyStackRef_CLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -2 - oparg; - assert(WITHIN_STACK_BOUNDS()); - if (res_o == NULL) { - TRACING_JUMP_TO_LABEL(error); - } - res = PyStackRef_FromPyObjectSteal(res_o); - } - // _CHECK_PERIODIC_AT_END - { - stack_pointer[0] = res; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = check_periodics(tstate); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err != 0) { - TRACING_JUMP_TO_LABEL(error); - } - } - TRACING_DISPATCH(); - } - - TRACING_TARGET(CALL_KW_PY) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = CALL_KW_PY; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 4; - INSTRUCTION_STATS(CALL_KW_PY); - opcode = CALL_KW_PY; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - static_assert(INLINE_CACHE_ENTRIES_CALL_KW == 3, "incorrect cache size"); - _PyStackRef callable; - _PyStackRef self_or_null; - _PyStackRef *args; - _PyStackRef kwnames; - _PyStackRef new_frame; - /* Skip 1 cache entry */ - // _CHECK_PEP_523 - { - if (tstate->interp->eval_frame) { - UPDATE_MISS_STATS(CALL_KW); - assert(_PyOpcode_Deopt[opcode] == (CALL_KW)); - JUMP_TO_PREDICTED(TRACING_CALL_KW); - } - } - // _CHECK_FUNCTION_VERSION_KW - { - callable = stack_pointer[-3 - oparg]; - uint32_t func_version = read_u32(&this_instr[2].cache); - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); - if (!PyFunction_Check(callable_o)) { - UPDATE_MISS_STATS(CALL_KW); - assert(_PyOpcode_Deopt[opcode] == (CALL_KW)); - JUMP_TO_PREDICTED(TRACING_CALL_KW); - } - PyFunctionObject *func = (PyFunctionObject *)callable_o; - if (func->func_version != func_version) { - UPDATE_MISS_STATS(CALL_KW); - assert(_PyOpcode_Deopt[opcode] == (CALL_KW)); - JUMP_TO_PREDICTED(TRACING_CALL_KW); - } - } - // _CHECK_RECURSION_REMAINING - { - if (tstate->py_recursion_remaining <= 1) { - UPDATE_MISS_STATS(CALL_KW); - assert(_PyOpcode_Deopt[opcode] == (CALL_KW)); - JUMP_TO_PREDICTED(TRACING_CALL_KW); - } - } - // _PY_FRAME_KW - { - kwnames = stack_pointer[-1]; - args = &stack_pointer[-1 - oparg]; - self_or_null = stack_pointer[-2 - oparg]; - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); - int total_args = oparg; - _PyStackRef *arguments = args; - if (!PyStackRef_IsNull(self_or_null)) { - arguments--; - total_args++; - } - PyObject *kwnames_o = PyStackRef_AsPyObjectBorrow(kwnames); - int positional_args = total_args - (int)PyTuple_GET_SIZE(kwnames_o); - assert(Py_TYPE(callable_o) == &PyFunction_Type); - int code_flags = ((PyCodeObject*)PyFunction_GET_CODE(callable_o))->co_flags; - PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable_o)); - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyInterpreterFrame *temp = _PyEvalFramePushAndInit( - tstate, callable, locals, - arguments, positional_args, kwnames_o, frame - ); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(kwnames); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -2 - oparg; - assert(WITHIN_STACK_BOUNDS()); - if (temp == NULL) { - TRACING_JUMP_TO_LABEL(error); - } - new_frame = PyStackRef_Wrap(temp); - } - // _SAVE_RETURN_OFFSET - { - #if TIER_ONE - frame->return_offset = (uint16_t)(next_instr - this_instr); - #endif - #if TIER_TWO - frame->return_offset = oparg; - #endif - } - // _PUSH_FRAME - { - assert(tstate->interp->eval_frame == NULL); - _PyInterpreterFrame *temp = PyStackRef_Unwrap(new_frame); - _PyFrame_SetStackPointer(frame, stack_pointer); - assert(temp->previous == frame || temp->previous->previous == frame); - CALL_STAT_INC(inlined_py_calls); - frame = tstate->current_frame = temp; - tstate->py_recursion_remaining--; - LOAD_SP(); - LOAD_IP(0); - LLTRACE_RESUME_FRAME(); - } - TRACING_DISPATCH(); - } - - TRACING_TARGET(CALL_LEN) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = CALL_LEN; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 4; - INSTRUCTION_STATS(CALL_LEN); - opcode = CALL_LEN; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); - _PyStackRef null; - _PyStackRef callable; - _PyStackRef arg; - _PyStackRef res; - /* Skip 1 cache entry */ - /* Skip 2 cache entries */ - // _GUARD_NOS_NULL - { - null = stack_pointer[-2]; - if (!PyStackRef_IsNull(null)) { - UPDATE_MISS_STATS(CALL); - assert(_PyOpcode_Deopt[opcode] == (CALL)); - JUMP_TO_PREDICTED(TRACING_CALL); - } - } - // _GUARD_CALLABLE_LEN - { - callable = stack_pointer[-3]; - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); - PyInterpreterState *interp = tstate->interp; - if (callable_o != interp->callable_cache.len) { - UPDATE_MISS_STATS(CALL); - assert(_PyOpcode_Deopt[opcode] == (CALL)); - JUMP_TO_PREDICTED(TRACING_CALL); - } - } - // _CALL_LEN - { - arg = stack_pointer[-1]; - (void)null; - STAT_INC(CALL, hit); - PyObject *arg_o = PyStackRef_AsPyObjectBorrow(arg); - _PyFrame_SetStackPointer(frame, stack_pointer); - Py_ssize_t len_i = PyObject_Length(arg_o); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (len_i < 0) { - TRACING_JUMP_TO_LABEL(error); - } - PyObject *res_o = PyLong_FromSsize_t(len_i); - assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - if (res_o == NULL) { - TRACING_JUMP_TO_LABEL(error); - } - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(arg); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(callable); - stack_pointer = _PyFrame_GetStackPointer(frame); - res = PyStackRef_FromPyObjectSteal(res_o); - } - stack_pointer[0] = res; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - TRACING_DISPATCH(); - } - - TRACING_TARGET(CALL_LIST_APPEND) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = CALL_LIST_APPEND; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 4; - INSTRUCTION_STATS(CALL_LIST_APPEND); - opcode = CALL_LIST_APPEND; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); - _PyStackRef callable; - _PyStackRef nos; - _PyStackRef self; - _PyStackRef arg; - /* Skip 1 cache entry */ - /* Skip 2 cache entries */ - // _GUARD_CALLABLE_LIST_APPEND - { - callable = stack_pointer[-3]; - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); - PyInterpreterState *interp = tstate->interp; - if (callable_o != interp->callable_cache.list_append) { - UPDATE_MISS_STATS(CALL); - assert(_PyOpcode_Deopt[opcode] == (CALL)); - JUMP_TO_PREDICTED(TRACING_CALL); - } - } - // _GUARD_NOS_NOT_NULL - { - nos = stack_pointer[-2]; - PyObject *o = PyStackRef_AsPyObjectBorrow(nos); - if (o == NULL) { - UPDATE_MISS_STATS(CALL); - assert(_PyOpcode_Deopt[opcode] == (CALL)); - JUMP_TO_PREDICTED(TRACING_CALL); - } - } - // _GUARD_NOS_LIST - { - PyObject *o = PyStackRef_AsPyObjectBorrow(nos); - if (!PyList_CheckExact(o)) { - UPDATE_MISS_STATS(CALL); - assert(_PyOpcode_Deopt[opcode] == (CALL)); - JUMP_TO_PREDICTED(TRACING_CALL); - } - } - // _CALL_LIST_APPEND - { - arg = stack_pointer[-1]; - self = nos; - assert(oparg == 1); - PyObject *self_o = PyStackRef_AsPyObjectBorrow(self); - if (!PyList_CheckExact(self_o)) { - UPDATE_MISS_STATS(CALL); - assert(_PyOpcode_Deopt[opcode] == (CALL)); - JUMP_TO_PREDICTED(TRACING_CALL); - } - if (!LOCK_OBJECT(self_o)) { - UPDATE_MISS_STATS(CALL); - assert(_PyOpcode_Deopt[opcode] == (CALL)); - JUMP_TO_PREDICTED(TRACING_CALL); - } - STAT_INC(CALL, hit); - int err = _PyList_AppendTakeRef((PyListObject *)self_o, PyStackRef_AsPyObjectSteal(arg)); - UNLOCK_OBJECT(self_o); - stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(self); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(callable); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err) { - TRACING_JUMP_TO_LABEL(error); - } - #if TIER_ONE - - assert(next_instr->op.code == POP_TOP); - SKIP_OVER(1); - #endif - } - TRACING_DISPATCH(); - } - - TRACING_TARGET(CALL_METHOD_DESCRIPTOR_FAST) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = CALL_METHOD_DESCRIPTOR_FAST; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 4; - INSTRUCTION_STATS(CALL_METHOD_DESCRIPTOR_FAST); - opcode = CALL_METHOD_DESCRIPTOR_FAST; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); - _PyStackRef callable; - _PyStackRef self_or_null; - _PyStackRef *args; - _PyStackRef res; - /* Skip 1 cache entry */ - /* Skip 2 cache entries */ - // _CALL_METHOD_DESCRIPTOR_FAST - { - args = &stack_pointer[-oparg]; - self_or_null = stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); - int total_args = oparg; - _PyStackRef *arguments = args; - if (!PyStackRef_IsNull(self_or_null)) { - arguments--; - total_args++; - } - if (total_args == 0) { - UPDATE_MISS_STATS(CALL); - assert(_PyOpcode_Deopt[opcode] == (CALL)); - JUMP_TO_PREDICTED(TRACING_CALL); - } - PyMethodDescrObject *method = (PyMethodDescrObject *)callable_o; - if (!Py_IS_TYPE(method, &PyMethodDescr_Type)) { - UPDATE_MISS_STATS(CALL); - assert(_PyOpcode_Deopt[opcode] == (CALL)); - JUMP_TO_PREDICTED(TRACING_CALL); - } - PyMethodDef *meth = method->d_method; - if (meth->ml_flags != METH_FASTCALL) { - UPDATE_MISS_STATS(CALL); - assert(_PyOpcode_Deopt[opcode] == (CALL)); - JUMP_TO_PREDICTED(TRACING_CALL); - } - PyObject *self = PyStackRef_AsPyObjectBorrow(arguments[0]); - assert(self != NULL); - if (!Py_IS_TYPE(self, method->d_common.d_type)) { - UPDATE_MISS_STATS(CALL); - assert(_PyOpcode_Deopt[opcode] == (CALL)); - JUMP_TO_PREDICTED(TRACING_CALL); - } - STAT_INC(CALL, hit); - int nargs = total_args - 1; - STACKREFS_TO_PYOBJECTS(arguments, total_args, args_o); - if (CONVERSION_FAILED(args_o)) { - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyStackRef tmp; - for (int _i = oparg; --_i >= 0;) { - tmp = args[_i]; - args[_i] = PyStackRef_NULL; - PyStackRef_CLOSE(tmp); - } - tmp = self_or_null; - self_or_null = PyStackRef_NULL; - stack_pointer[-1 - oparg] = self_or_null; - PyStackRef_XCLOSE(tmp); - tmp = callable; - callable = PyStackRef_NULL; - stack_pointer[-2 - oparg] = callable; - PyStackRef_CLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -2 - oparg; - assert(WITHIN_STACK_BOUNDS()); - TRACING_JUMP_TO_LABEL(error); - } - _PyFrame_SetStackPointer(frame, stack_pointer); - PyCFunctionFast cfunc = _PyCFunctionFast_CAST(meth->ml_meth); - PyObject *res_o = cfunc(self, (args_o + 1), nargs); - stack_pointer = _PyFrame_GetStackPointer(frame); - STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); - assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyStackRef tmp; - for (int _i = oparg; --_i >= 0;) { - tmp = args[_i]; - args[_i] = PyStackRef_NULL; - PyStackRef_CLOSE(tmp); - } - tmp = self_or_null; - self_or_null = PyStackRef_NULL; - stack_pointer[-1 - oparg] = self_or_null; - PyStackRef_XCLOSE(tmp); - tmp = callable; - callable = PyStackRef_NULL; - stack_pointer[-2 - oparg] = callable; - PyStackRef_CLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -2 - oparg; - assert(WITHIN_STACK_BOUNDS()); - if (res_o == NULL) { - TRACING_JUMP_TO_LABEL(error); - } - res = PyStackRef_FromPyObjectSteal(res_o); - } - // _CHECK_PERIODIC_AT_END - { - stack_pointer[0] = res; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = check_periodics(tstate); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err != 0) { - TRACING_JUMP_TO_LABEL(error); - } - } - TRACING_DISPATCH(); - } - - TRACING_TARGET(CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 4; - INSTRUCTION_STATS(CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS); - opcode = CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); - _PyStackRef callable; - _PyStackRef self_or_null; - _PyStackRef *args; - _PyStackRef res; - /* Skip 1 cache entry */ - /* Skip 2 cache entries */ - // _CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS - { - args = &stack_pointer[-oparg]; - self_or_null = stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); - int total_args = oparg; - _PyStackRef *arguments = args; - if (!PyStackRef_IsNull(self_or_null)) { - arguments--; - total_args++; - } - if (total_args == 0) { - UPDATE_MISS_STATS(CALL); - assert(_PyOpcode_Deopt[opcode] == (CALL)); - JUMP_TO_PREDICTED(TRACING_CALL); - } - PyMethodDescrObject *method = (PyMethodDescrObject *)callable_o; - if (!Py_IS_TYPE(method, &PyMethodDescr_Type)) { - UPDATE_MISS_STATS(CALL); - assert(_PyOpcode_Deopt[opcode] == (CALL)); - JUMP_TO_PREDICTED(TRACING_CALL); - } - PyMethodDef *meth = method->d_method; - if (meth->ml_flags != (METH_FASTCALL|METH_KEYWORDS)) { - UPDATE_MISS_STATS(CALL); - assert(_PyOpcode_Deopt[opcode] == (CALL)); - JUMP_TO_PREDICTED(TRACING_CALL); - } - PyTypeObject *d_type = method->d_common.d_type; - PyObject *self = PyStackRef_AsPyObjectBorrow(arguments[0]); - assert(self != NULL); - if (!Py_IS_TYPE(self, d_type)) { - UPDATE_MISS_STATS(CALL); - assert(_PyOpcode_Deopt[opcode] == (CALL)); - JUMP_TO_PREDICTED(TRACING_CALL); - } - STAT_INC(CALL, hit); - int nargs = total_args - 1; - STACKREFS_TO_PYOBJECTS(arguments, total_args, args_o); - if (CONVERSION_FAILED(args_o)) { - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyStackRef tmp; - for (int _i = oparg; --_i >= 0;) { - tmp = args[_i]; - args[_i] = PyStackRef_NULL; - PyStackRef_CLOSE(tmp); - } - tmp = self_or_null; - self_or_null = PyStackRef_NULL; - stack_pointer[-1 - oparg] = self_or_null; - PyStackRef_XCLOSE(tmp); - tmp = callable; - callable = PyStackRef_NULL; - stack_pointer[-2 - oparg] = callable; - PyStackRef_CLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -2 - oparg; - assert(WITHIN_STACK_BOUNDS()); - TRACING_JUMP_TO_LABEL(error); - } - _PyFrame_SetStackPointer(frame, stack_pointer); - PyCFunctionFastWithKeywords cfunc = - _PyCFunctionFastWithKeywords_CAST(meth->ml_meth); - PyObject *res_o = cfunc(self, (args_o + 1), nargs, NULL); - stack_pointer = _PyFrame_GetStackPointer(frame); - STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); - assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyStackRef tmp; - for (int _i = oparg; --_i >= 0;) { - tmp = args[_i]; - args[_i] = PyStackRef_NULL; - PyStackRef_CLOSE(tmp); - } - tmp = self_or_null; - self_or_null = PyStackRef_NULL; - stack_pointer[-1 - oparg] = self_or_null; - PyStackRef_XCLOSE(tmp); - tmp = callable; - callable = PyStackRef_NULL; - stack_pointer[-2 - oparg] = callable; - PyStackRef_CLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -2 - oparg; - assert(WITHIN_STACK_BOUNDS()); - if (res_o == NULL) { - TRACING_JUMP_TO_LABEL(error); - } - res = PyStackRef_FromPyObjectSteal(res_o); - } - // _CHECK_PERIODIC_AT_END - { - stack_pointer[0] = res; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = check_periodics(tstate); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err != 0) { - TRACING_JUMP_TO_LABEL(error); - } - } - TRACING_DISPATCH(); - } - - TRACING_TARGET(CALL_METHOD_DESCRIPTOR_NOARGS) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = CALL_METHOD_DESCRIPTOR_NOARGS; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 4; - INSTRUCTION_STATS(CALL_METHOD_DESCRIPTOR_NOARGS); - opcode = CALL_METHOD_DESCRIPTOR_NOARGS; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); - _PyStackRef callable; - _PyStackRef self_or_null; - _PyStackRef *args; - _PyStackRef res; - /* Skip 1 cache entry */ - /* Skip 2 cache entries */ - // _CALL_METHOD_DESCRIPTOR_NOARGS - { - args = &stack_pointer[-oparg]; - self_or_null = stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; - assert(oparg == 0 || oparg == 1); - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); - int total_args = oparg; - if (!PyStackRef_IsNull(self_or_null)) { - args--; - total_args++; - } - if (total_args != 1) { - UPDATE_MISS_STATS(CALL); - assert(_PyOpcode_Deopt[opcode] == (CALL)); - JUMP_TO_PREDICTED(TRACING_CALL); - } - PyMethodDescrObject *method = (PyMethodDescrObject *)callable_o; - if (!Py_IS_TYPE(method, &PyMethodDescr_Type)) { - UPDATE_MISS_STATS(CALL); - assert(_PyOpcode_Deopt[opcode] == (CALL)); - JUMP_TO_PREDICTED(TRACING_CALL); - } - PyMethodDef *meth = method->d_method; - _PyStackRef self_stackref = args[0]; - PyObject *self = PyStackRef_AsPyObjectBorrow(self_stackref); - if (!Py_IS_TYPE(self, method->d_common.d_type)) { - UPDATE_MISS_STATS(CALL); - assert(_PyOpcode_Deopt[opcode] == (CALL)); - JUMP_TO_PREDICTED(TRACING_CALL); - } - if (meth->ml_flags != METH_NOARGS) { - UPDATE_MISS_STATS(CALL); - assert(_PyOpcode_Deopt[opcode] == (CALL)); - JUMP_TO_PREDICTED(TRACING_CALL); - } - if (_Py_ReachedRecursionLimit(tstate)) { - UPDATE_MISS_STATS(CALL); - assert(_PyOpcode_Deopt[opcode] == (CALL)); - JUMP_TO_PREDICTED(TRACING_CALL); - } - STAT_INC(CALL, hit); - PyCFunction cfunc = meth->ml_meth; - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *res_o = _PyCFunction_TrampolineCall(cfunc, self, NULL); - stack_pointer = _PyFrame_GetStackPointer(frame); - _Py_LeaveRecursiveCallTstate(tstate); - assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(self_stackref); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -2 - oparg; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(callable); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (res_o == NULL) { - TRACING_JUMP_TO_LABEL(error); - } - res = PyStackRef_FromPyObjectSteal(res_o); - } - // _CHECK_PERIODIC_AT_END - { - stack_pointer[0] = res; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = check_periodics(tstate); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err != 0) { - TRACING_JUMP_TO_LABEL(error); - } - } - TRACING_DISPATCH(); - } - - TRACING_TARGET(CALL_METHOD_DESCRIPTOR_O) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = CALL_METHOD_DESCRIPTOR_O; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 4; - INSTRUCTION_STATS(CALL_METHOD_DESCRIPTOR_O); - opcode = CALL_METHOD_DESCRIPTOR_O; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); - _PyStackRef callable; - _PyStackRef self_or_null; - _PyStackRef *args; - _PyStackRef res; - /* Skip 1 cache entry */ - /* Skip 2 cache entries */ - // _CALL_METHOD_DESCRIPTOR_O - { - args = &stack_pointer[-oparg]; - self_or_null = stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); - int total_args = oparg; - _PyStackRef *arguments = args; - if (!PyStackRef_IsNull(self_or_null)) { - arguments--; - total_args++; - } - PyMethodDescrObject *method = (PyMethodDescrObject *)callable_o; - if (total_args != 2) { - UPDATE_MISS_STATS(CALL); - assert(_PyOpcode_Deopt[opcode] == (CALL)); - JUMP_TO_PREDICTED(TRACING_CALL); - } - if (!Py_IS_TYPE(method, &PyMethodDescr_Type)) { - UPDATE_MISS_STATS(CALL); - assert(_PyOpcode_Deopt[opcode] == (CALL)); - JUMP_TO_PREDICTED(TRACING_CALL); - } - PyMethodDef *meth = method->d_method; - if (meth->ml_flags != METH_O) { - UPDATE_MISS_STATS(CALL); - assert(_PyOpcode_Deopt[opcode] == (CALL)); - JUMP_TO_PREDICTED(TRACING_CALL); - } - if (_Py_ReachedRecursionLimit(tstate)) { - UPDATE_MISS_STATS(CALL); - assert(_PyOpcode_Deopt[opcode] == (CALL)); - JUMP_TO_PREDICTED(TRACING_CALL); - } - _PyStackRef arg_stackref = arguments[1]; - _PyStackRef self_stackref = arguments[0]; - if (!Py_IS_TYPE(PyStackRef_AsPyObjectBorrow(self_stackref), - method->d_common.d_type)) { - UPDATE_MISS_STATS(CALL); - assert(_PyOpcode_Deopt[opcode] == (CALL)); - JUMP_TO_PREDICTED(TRACING_CALL); - } - STAT_INC(CALL, hit); - PyCFunction cfunc = meth->ml_meth; - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *res_o = _PyCFunction_TrampolineCall(cfunc, - PyStackRef_AsPyObjectBorrow(self_stackref), - PyStackRef_AsPyObjectBorrow(arg_stackref)); - stack_pointer = _PyFrame_GetStackPointer(frame); - _Py_LeaveRecursiveCallTstate(tstate); - assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyStackRef tmp; - for (int _i = oparg; --_i >= 0;) { - tmp = args[_i]; - args[_i] = PyStackRef_NULL; - PyStackRef_CLOSE(tmp); - } - tmp = self_or_null; - self_or_null = PyStackRef_NULL; - stack_pointer[-1 - oparg] = self_or_null; - PyStackRef_XCLOSE(tmp); - tmp = callable; - callable = PyStackRef_NULL; - stack_pointer[-2 - oparg] = callable; - PyStackRef_CLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -2 - oparg; - assert(WITHIN_STACK_BOUNDS()); - if (res_o == NULL) { - TRACING_JUMP_TO_LABEL(error); - } - res = PyStackRef_FromPyObjectSteal(res_o); - } - // _CHECK_PERIODIC_AT_END - { - stack_pointer[0] = res; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = check_periodics(tstate); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err != 0) { - TRACING_JUMP_TO_LABEL(error); - } - } - TRACING_DISPATCH(); - } - - TRACING_TARGET(CALL_NON_PY_GENERAL) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = CALL_NON_PY_GENERAL; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 4; - INSTRUCTION_STATS(CALL_NON_PY_GENERAL); - opcode = CALL_NON_PY_GENERAL; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - opcode = CALL_NON_PY_GENERAL; - static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); - _PyStackRef callable; - _PyStackRef self_or_null; - _PyStackRef *args; - _PyStackRef res; - /* Skip 1 cache entry */ - /* Skip 2 cache entries */ - // _CHECK_IS_NOT_PY_CALLABLE - { - callable = stack_pointer[-2 - oparg]; - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); - if (PyFunction_Check(callable_o)) { - UPDATE_MISS_STATS(CALL); - assert(_PyOpcode_Deopt[opcode] == (CALL)); - JUMP_TO_PREDICTED(TRACING_CALL); - } - if (Py_TYPE(callable_o) == &PyMethod_Type) { - UPDATE_MISS_STATS(CALL); - assert(_PyOpcode_Deopt[opcode] == (CALL)); - JUMP_TO_PREDICTED(TRACING_CALL); - } - } - // _CALL_NON_PY_GENERAL - { - args = &stack_pointer[-oparg]; - self_or_null = stack_pointer[-1 - oparg]; - #if TIER_ONE - assert(opcode != INSTRUMENTED_CALL); - #endif - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); - int total_args = oparg; - _PyStackRef *arguments = args; - if (!PyStackRef_IsNull(self_or_null)) { - arguments--; - total_args++; - } - STACKREFS_TO_PYOBJECTS(arguments, total_args, args_o); - if (CONVERSION_FAILED(args_o)) { - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyStackRef tmp; - for (int _i = oparg; --_i >= 0;) { - tmp = args[_i]; - args[_i] = PyStackRef_NULL; - PyStackRef_CLOSE(tmp); - } - tmp = self_or_null; - self_or_null = PyStackRef_NULL; - stack_pointer[-1 - oparg] = self_or_null; - PyStackRef_XCLOSE(tmp); - tmp = callable; - callable = PyStackRef_NULL; - stack_pointer[-2 - oparg] = callable; - PyStackRef_CLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -2 - oparg; - assert(WITHIN_STACK_BOUNDS()); - TRACING_JUMP_TO_LABEL(error); - } - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *res_o = PyObject_Vectorcall( - callable_o, args_o, - total_args | PY_VECTORCALL_ARGUMENTS_OFFSET, - NULL); - stack_pointer = _PyFrame_GetStackPointer(frame); - STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); - assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyStackRef tmp; - for (int _i = oparg; --_i >= 0;) { - tmp = args[_i]; - args[_i] = PyStackRef_NULL; - PyStackRef_CLOSE(tmp); - } - tmp = self_or_null; - self_or_null = PyStackRef_NULL; - stack_pointer[-1 - oparg] = self_or_null; - PyStackRef_XCLOSE(tmp); - tmp = callable; - callable = PyStackRef_NULL; - stack_pointer[-2 - oparg] = callable; - PyStackRef_CLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -2 - oparg; - assert(WITHIN_STACK_BOUNDS()); - if (res_o == NULL) { - TRACING_JUMP_TO_LABEL(error); - } - res = PyStackRef_FromPyObjectSteal(res_o); - } - // _CHECK_PERIODIC_AT_END - { - stack_pointer[0] = res; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = check_periodics(tstate); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err != 0) { - TRACING_JUMP_TO_LABEL(error); - } - } - TRACING_DISPATCH(); - } - - TRACING_TARGET(CALL_PY_EXACT_ARGS) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = CALL_PY_EXACT_ARGS; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 4; - INSTRUCTION_STATS(CALL_PY_EXACT_ARGS); - opcode = CALL_PY_EXACT_ARGS; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); - _PyStackRef callable; - _PyStackRef self_or_null; - _PyStackRef *args; - _PyStackRef new_frame; - /* Skip 1 cache entry */ - // _CHECK_PEP_523 - { - if (tstate->interp->eval_frame) { - UPDATE_MISS_STATS(CALL); - assert(_PyOpcode_Deopt[opcode] == (CALL)); - JUMP_TO_PREDICTED(TRACING_CALL); - } - } - // _CHECK_FUNCTION_VERSION - { - callable = stack_pointer[-2 - oparg]; - uint32_t func_version = read_u32(&this_instr[2].cache); - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); - if (!PyFunction_Check(callable_o)) { - UPDATE_MISS_STATS(CALL); - assert(_PyOpcode_Deopt[opcode] == (CALL)); - JUMP_TO_PREDICTED(TRACING_CALL); - } - PyFunctionObject *func = (PyFunctionObject *)callable_o; - if (func->func_version != func_version) { - UPDATE_MISS_STATS(CALL); - assert(_PyOpcode_Deopt[opcode] == (CALL)); - JUMP_TO_PREDICTED(TRACING_CALL); - } - } - // _CHECK_FUNCTION_EXACT_ARGS - { - self_or_null = stack_pointer[-1 - oparg]; - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); - assert(PyFunction_Check(callable_o)); - PyFunctionObject *func = (PyFunctionObject *)callable_o; - PyCodeObject *code = (PyCodeObject *)func->func_code; - if (code->co_argcount != oparg + (!PyStackRef_IsNull(self_or_null))) { - UPDATE_MISS_STATS(CALL); - assert(_PyOpcode_Deopt[opcode] == (CALL)); - JUMP_TO_PREDICTED(TRACING_CALL); - } - } - // _CHECK_STACK_SPACE - { - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); - PyFunctionObject *func = (PyFunctionObject *)callable_o; - PyCodeObject *code = (PyCodeObject *)func->func_code; - if (!_PyThreadState_HasStackSpace(tstate, code->co_framesize)) { - UPDATE_MISS_STATS(CALL); - assert(_PyOpcode_Deopt[opcode] == (CALL)); - JUMP_TO_PREDICTED(TRACING_CALL); - } - } - // _CHECK_RECURSION_REMAINING - { - if (tstate->py_recursion_remaining <= 1) { - UPDATE_MISS_STATS(CALL); - assert(_PyOpcode_Deopt[opcode] == (CALL)); - JUMP_TO_PREDICTED(TRACING_CALL); - } - } - // _INIT_CALL_PY_EXACT_ARGS - { - args = &stack_pointer[-oparg]; - int has_self = !PyStackRef_IsNull(self_or_null); - STAT_INC(CALL, hit); - _PyInterpreterFrame *pushed_frame = _PyFrame_PushUnchecked(tstate, callable, oparg + has_self, frame); - _PyStackRef *first_non_self_local = pushed_frame->localsplus + has_self; - pushed_frame->localsplus[0] = self_or_null; - for (int i = 0; i < oparg; i++) { - first_non_self_local[i] = args[i]; - } - new_frame = PyStackRef_Wrap(pushed_frame); - } - // _SAVE_RETURN_OFFSET - { - #if TIER_ONE - frame->return_offset = (uint16_t)(next_instr - this_instr); - #endif - #if TIER_TWO - frame->return_offset = oparg; - #endif - } - // _PUSH_FRAME - { - assert(tstate->interp->eval_frame == NULL); - _PyInterpreterFrame *temp = PyStackRef_Unwrap(new_frame); - stack_pointer += -2 - oparg; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - assert(temp->previous == frame || temp->previous->previous == frame); - CALL_STAT_INC(inlined_py_calls); - frame = tstate->current_frame = temp; - tstate->py_recursion_remaining--; - LOAD_SP(); - LOAD_IP(0); - LLTRACE_RESUME_FRAME(); - } - TRACING_DISPATCH(); - } - - TRACING_TARGET(CALL_PY_GENERAL) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = CALL_PY_GENERAL; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 4; - INSTRUCTION_STATS(CALL_PY_GENERAL); - opcode = CALL_PY_GENERAL; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); - _PyStackRef callable; - _PyStackRef self_or_null; - _PyStackRef *args; - _PyStackRef new_frame; - /* Skip 1 cache entry */ - // _CHECK_PEP_523 - { - if (tstate->interp->eval_frame) { - UPDATE_MISS_STATS(CALL); - assert(_PyOpcode_Deopt[opcode] == (CALL)); - JUMP_TO_PREDICTED(TRACING_CALL); - } - } - // _CHECK_FUNCTION_VERSION - { - callable = stack_pointer[-2 - oparg]; - uint32_t func_version = read_u32(&this_instr[2].cache); - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); - if (!PyFunction_Check(callable_o)) { - UPDATE_MISS_STATS(CALL); - assert(_PyOpcode_Deopt[opcode] == (CALL)); - JUMP_TO_PREDICTED(TRACING_CALL); - } - PyFunctionObject *func = (PyFunctionObject *)callable_o; - if (func->func_version != func_version) { - UPDATE_MISS_STATS(CALL); - assert(_PyOpcode_Deopt[opcode] == (CALL)); - JUMP_TO_PREDICTED(TRACING_CALL); - } - } - // _CHECK_RECURSION_REMAINING - { - if (tstate->py_recursion_remaining <= 1) { - UPDATE_MISS_STATS(CALL); - assert(_PyOpcode_Deopt[opcode] == (CALL)); - JUMP_TO_PREDICTED(TRACING_CALL); - } - } - // _PY_FRAME_GENERAL - { - args = &stack_pointer[-oparg]; - self_or_null = stack_pointer[-1 - oparg]; - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); - int total_args = oparg; - if (!PyStackRef_IsNull(self_or_null)) { - args--; - total_args++; - } - assert(Py_TYPE(callable_o) == &PyFunction_Type); - int code_flags = ((PyCodeObject*)PyFunction_GET_CODE(callable_o))->co_flags; - PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable_o)); - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyInterpreterFrame *temp = _PyEvalFramePushAndInit( - tstate, callable, locals, - args, total_args, NULL, frame - ); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -2 - oparg; - assert(WITHIN_STACK_BOUNDS()); - if (temp == NULL) { - TRACING_JUMP_TO_LABEL(error); - } - new_frame = PyStackRef_Wrap(temp); - } - // _SAVE_RETURN_OFFSET - { - #if TIER_ONE - frame->return_offset = (uint16_t)(next_instr - this_instr); - #endif - #if TIER_TWO - frame->return_offset = oparg; - #endif - } - // _PUSH_FRAME - { - assert(tstate->interp->eval_frame == NULL); - _PyInterpreterFrame *temp = PyStackRef_Unwrap(new_frame); - _PyFrame_SetStackPointer(frame, stack_pointer); - assert(temp->previous == frame || temp->previous->previous == frame); - CALL_STAT_INC(inlined_py_calls); - frame = tstate->current_frame = temp; - tstate->py_recursion_remaining--; - LOAD_SP(); - LOAD_IP(0); - LLTRACE_RESUME_FRAME(); - } - TRACING_DISPATCH(); - } - - TRACING_TARGET(CALL_STR_1) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = CALL_STR_1; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 4; - INSTRUCTION_STATS(CALL_STR_1); - opcode = CALL_STR_1; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); - _PyStackRef null; - _PyStackRef callable; - _PyStackRef arg; - _PyStackRef res; - /* Skip 1 cache entry */ - /* Skip 2 cache entries */ - // _GUARD_NOS_NULL - { - null = stack_pointer[-2]; - if (!PyStackRef_IsNull(null)) { - UPDATE_MISS_STATS(CALL); - assert(_PyOpcode_Deopt[opcode] == (CALL)); - JUMP_TO_PREDICTED(TRACING_CALL); - } - } - // _GUARD_CALLABLE_STR_1 - { - callable = stack_pointer[-3]; - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); - if (callable_o != (PyObject *)&PyUnicode_Type) { - UPDATE_MISS_STATS(CALL); - assert(_PyOpcode_Deopt[opcode] == (CALL)); - JUMP_TO_PREDICTED(TRACING_CALL); - } - } - // _CALL_STR_1 - { - arg = stack_pointer[-1]; - PyObject *arg_o = PyStackRef_AsPyObjectBorrow(arg); - assert(oparg == 1); - STAT_INC(CALL, hit); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *res_o = PyObject_Str(arg_o); - stack_pointer = _PyFrame_GetStackPointer(frame); - (void)callable; - (void)null; - stack_pointer += -3; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(arg); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (res_o == NULL) { - TRACING_JUMP_TO_LABEL(error); - } - res = PyStackRef_FromPyObjectSteal(res_o); - } - // _CHECK_PERIODIC_AT_END - { - stack_pointer[0] = res; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = check_periodics(tstate); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err != 0) { - TRACING_JUMP_TO_LABEL(error); - } - } - TRACING_DISPATCH(); - } - - TRACING_TARGET(CALL_TUPLE_1) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = CALL_TUPLE_1; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 4; - INSTRUCTION_STATS(CALL_TUPLE_1); - opcode = CALL_TUPLE_1; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); - _PyStackRef null; - _PyStackRef callable; - _PyStackRef arg; - _PyStackRef res; - /* Skip 1 cache entry */ - /* Skip 2 cache entries */ - // _GUARD_NOS_NULL - { - null = stack_pointer[-2]; - if (!PyStackRef_IsNull(null)) { - UPDATE_MISS_STATS(CALL); - assert(_PyOpcode_Deopt[opcode] == (CALL)); - JUMP_TO_PREDICTED(TRACING_CALL); - } - } - // _GUARD_CALLABLE_TUPLE_1 - { - callable = stack_pointer[-3]; - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); - if (callable_o != (PyObject *)&PyTuple_Type) { - UPDATE_MISS_STATS(CALL); - assert(_PyOpcode_Deopt[opcode] == (CALL)); - JUMP_TO_PREDICTED(TRACING_CALL); - } - } - // _CALL_TUPLE_1 - { - arg = stack_pointer[-1]; - PyObject *arg_o = PyStackRef_AsPyObjectBorrow(arg); - assert(oparg == 1); - STAT_INC(CALL, hit); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *res_o = PySequence_Tuple(arg_o); - stack_pointer = _PyFrame_GetStackPointer(frame); - (void)callable; - (void)null; - stack_pointer += -3; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(arg); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (res_o == NULL) { - TRACING_JUMP_TO_LABEL(error); - } - res = PyStackRef_FromPyObjectSteal(res_o); - } - // _CHECK_PERIODIC_AT_END - { - stack_pointer[0] = res; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = check_periodics(tstate); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err != 0) { - TRACING_JUMP_TO_LABEL(error); - } - } - TRACING_DISPATCH(); - } - - TRACING_TARGET(CALL_TYPE_1) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = CALL_TYPE_1; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 4; - INSTRUCTION_STATS(CALL_TYPE_1); - opcode = CALL_TYPE_1; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); - _PyStackRef null; - _PyStackRef callable; - _PyStackRef arg; - _PyStackRef res; - /* Skip 1 cache entry */ - /* Skip 2 cache entries */ - // _GUARD_NOS_NULL - { - null = stack_pointer[-2]; - if (!PyStackRef_IsNull(null)) { - UPDATE_MISS_STATS(CALL); - assert(_PyOpcode_Deopt[opcode] == (CALL)); - JUMP_TO_PREDICTED(TRACING_CALL); - } - } - // _GUARD_CALLABLE_TYPE_1 - { - callable = stack_pointer[-3]; - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); - if (callable_o != (PyObject *)&PyType_Type) { - UPDATE_MISS_STATS(CALL); - assert(_PyOpcode_Deopt[opcode] == (CALL)); - JUMP_TO_PREDICTED(TRACING_CALL); - } - } - // _CALL_TYPE_1 - { - arg = stack_pointer[-1]; - PyObject *arg_o = PyStackRef_AsPyObjectBorrow(arg); - assert(oparg == 1); - (void)callable; - (void)null; - STAT_INC(CALL, hit); - res = PyStackRef_FromPyObjectNew(Py_TYPE(arg_o)); - stack_pointer[-3] = res; - stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(arg); - stack_pointer = _PyFrame_GetStackPointer(frame); - } - TRACING_DISPATCH(); - } - - TRACING_TARGET(CHECK_EG_MATCH) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = CHECK_EG_MATCH; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(CHECK_EG_MATCH); - opcode = CHECK_EG_MATCH; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - _PyStackRef exc_value_st; - _PyStackRef match_type_st; - _PyStackRef rest; - _PyStackRef match; - match_type_st = stack_pointer[-1]; - exc_value_st = stack_pointer[-2]; - PyObject *exc_value = PyStackRef_AsPyObjectBorrow(exc_value_st); - PyObject *match_type = PyStackRef_AsPyObjectBorrow(match_type_st); - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = _PyEval_CheckExceptStarTypeValid(tstate, match_type); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err < 0) { - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyStackRef tmp = match_type_st; - match_type_st = PyStackRef_NULL; - stack_pointer[-1] = match_type_st; - PyStackRef_CLOSE(tmp); - tmp = exc_value_st; - exc_value_st = PyStackRef_NULL; - stack_pointer[-2] = exc_value_st; - PyStackRef_CLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); - TRACING_JUMP_TO_LABEL(error); - } - PyObject *match_o = NULL; - PyObject *rest_o = NULL; - _PyFrame_SetStackPointer(frame, stack_pointer); - int res = _PyEval_ExceptionGroupMatch(frame, exc_value, match_type, - &match_o, &rest_o); - _PyStackRef tmp = match_type_st; - match_type_st = PyStackRef_NULL; - stack_pointer[-1] = match_type_st; - PyStackRef_CLOSE(tmp); - tmp = exc_value_st; - exc_value_st = PyStackRef_NULL; - stack_pointer[-2] = exc_value_st; - PyStackRef_CLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); - if (res < 0) { - TRACING_JUMP_TO_LABEL(error); - } - assert((match_o == NULL) == (rest_o == NULL)); - if (match_o == NULL) { - TRACING_JUMP_TO_LABEL(error); - } - if (!Py_IsNone(match_o)) { - _PyFrame_SetStackPointer(frame, stack_pointer); - PyErr_SetHandledException(match_o); - stack_pointer = _PyFrame_GetStackPointer(frame); - } - rest = PyStackRef_FromPyObjectSteal(rest_o); - match = PyStackRef_FromPyObjectSteal(match_o); - stack_pointer[0] = rest; - stack_pointer[1] = match; - stack_pointer += 2; - assert(WITHIN_STACK_BOUNDS()); - TRACING_DISPATCH(); - } - - TRACING_TARGET(CHECK_EXC_MATCH) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = CHECK_EXC_MATCH; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(CHECK_EXC_MATCH); - opcode = CHECK_EXC_MATCH; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - _PyStackRef left; - _PyStackRef right; - _PyStackRef b; - right = stack_pointer[-1]; - left = stack_pointer[-2]; - PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); - PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - assert(PyExceptionInstance_Check(left_o)); - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = _PyEval_CheckExceptTypeValid(tstate, right_o); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err < 0) { - TRACING_JUMP_TO_LABEL(error); - } - _PyFrame_SetStackPointer(frame, stack_pointer); - int res = PyErr_GivenExceptionMatches(left_o, right_o); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(right); - stack_pointer = _PyFrame_GetStackPointer(frame); - b = res ? PyStackRef_True : PyStackRef_False; - stack_pointer[0] = b; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - TRACING_DISPATCH(); - } - - TRACING_TARGET(CLEANUP_THROW) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = CLEANUP_THROW; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(CLEANUP_THROW); - opcode = CLEANUP_THROW; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - _PyStackRef sub_iter; - _PyStackRef last_sent_val; - _PyStackRef exc_value_st; - _PyStackRef none; - _PyStackRef value; - exc_value_st = stack_pointer[-1]; - last_sent_val = stack_pointer[-2]; - sub_iter = stack_pointer[-3]; - PyObject *exc_value = PyStackRef_AsPyObjectBorrow(exc_value_st); - #if !_Py_TAIL_CALL_INTERP - assert(throwflag); - #endif - assert(exc_value && PyExceptionInstance_Check(exc_value)); - _PyFrame_SetStackPointer(frame, stack_pointer); - int matches = PyErr_GivenExceptionMatches(exc_value, PyExc_StopIteration); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (matches) { - value = PyStackRef_FromPyObjectNew(((PyStopIterationObject *)exc_value)->value); - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyStackRef tmp = sub_iter; - sub_iter = value; - stack_pointer[-3] = sub_iter; - PyStackRef_CLOSE(tmp); - tmp = exc_value_st; - exc_value_st = PyStackRef_NULL; - stack_pointer[-1] = exc_value_st; - PyStackRef_CLOSE(tmp); - tmp = last_sent_val; - last_sent_val = PyStackRef_NULL; - stack_pointer[-2] = last_sent_val; - PyStackRef_CLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -3; - assert(WITHIN_STACK_BOUNDS()); - none = PyStackRef_None; - } - else { - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyErr_SetRaisedException(tstate, Py_NewRef(exc_value)); - monitor_reraise(tstate, frame, this_instr); - TRACING_JUMP_TO_LABEL(exception_unwind); - } - stack_pointer[0] = none; - stack_pointer[1] = value; - stack_pointer += 2; - assert(WITHIN_STACK_BOUNDS()); - TRACING_DISPATCH(); - } - - TRACING_TARGET(COMPARE_OP) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = COMPARE_OP; - (void)(opcode); - #endif - frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(COMPARE_OP); - PREDICTED_TRACING_COMPARE_OP:; - _Py_CODEUNIT* const this_instr = next_instr - 2; - (void)this_instr; - opcode = COMPARE_OP; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - _PyStackRef left; - _PyStackRef right; - _PyStackRef res; - // _SPECIALIZE_COMPARE_OP - { - right = stack_pointer[-1]; - left = stack_pointer[-2]; - uint16_t counter = read_u16(&this_instr[1].cache); - (void)counter; - #if ENABLE_SPECIALIZATION_FT - if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { - next_instr = this_instr; - _PyFrame_SetStackPointer(frame, stack_pointer); - _Py_Specialize_CompareOp(left, right, next_instr, oparg); - stack_pointer = _PyFrame_GetStackPointer(frame); - DISPATCH_SAME_OPARG(); - } - OPCODE_DEFERRED_INC(COMPARE_OP); - ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); - #endif /* ENABLE_SPECIALIZATION_FT */ - } - // _COMPARE_OP - { - PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); - PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - assert((oparg >> 5) <= Py_GE); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *res_o = PyObject_RichCompare(left_o, right_o, oparg >> 5); - _PyStackRef tmp = right; - right = PyStackRef_NULL; - stack_pointer[-1] = right; - PyStackRef_CLOSE(tmp); - tmp = left; - left = PyStackRef_NULL; - stack_pointer[-2] = left; - PyStackRef_CLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); - if (res_o == NULL) { - TRACING_JUMP_TO_LABEL(error); - } - if (oparg & 16) { - _PyFrame_SetStackPointer(frame, stack_pointer); - int res_bool = PyObject_IsTrue(res_o); - Py_DECREF(res_o); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (res_bool < 0) { - TRACING_JUMP_TO_LABEL(error); - } - res = res_bool ? PyStackRef_True : PyStackRef_False; - } - else { - res = PyStackRef_FromPyObjectSteal(res_o); - } - } - stack_pointer[0] = res; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - TRACING_DISPATCH(); - } - - TRACING_TARGET(COMPARE_OP_FLOAT) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = COMPARE_OP_FLOAT; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(COMPARE_OP_FLOAT); - opcode = COMPARE_OP_FLOAT; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - static_assert(INLINE_CACHE_ENTRIES_COMPARE_OP == 1, "incorrect cache size"); - _PyStackRef value; - _PyStackRef left; - _PyStackRef right; - _PyStackRef res; - // _GUARD_TOS_FLOAT - { - value = stack_pointer[-1]; - PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); - if (!PyFloat_CheckExact(value_o)) { - UPDATE_MISS_STATS(COMPARE_OP); - assert(_PyOpcode_Deopt[opcode] == (COMPARE_OP)); - JUMP_TO_PREDICTED(TRACING_COMPARE_OP); - } - } - // _GUARD_NOS_FLOAT - { - left = stack_pointer[-2]; - PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); - if (!PyFloat_CheckExact(left_o)) { - UPDATE_MISS_STATS(COMPARE_OP); - assert(_PyOpcode_Deopt[opcode] == (COMPARE_OP)); - JUMP_TO_PREDICTED(TRACING_COMPARE_OP); - } - } - /* Skip 1 cache entry */ - // _COMPARE_OP_FLOAT - { - right = value; - PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); - PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - STAT_INC(COMPARE_OP, hit); - double dleft = PyFloat_AS_DOUBLE(left_o); - double dright = PyFloat_AS_DOUBLE(right_o); - int sign_ish = COMPARISON_BIT(dleft, dright); - PyStackRef_CLOSE_SPECIALIZED(left, _PyFloat_ExactDealloc); - PyStackRef_CLOSE_SPECIALIZED(right, _PyFloat_ExactDealloc); - res = (sign_ish & oparg) ? PyStackRef_True : PyStackRef_False; - } - stack_pointer[-2] = res; - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - TRACING_DISPATCH(); - } - - TRACING_TARGET(COMPARE_OP_INT) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = COMPARE_OP_INT; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(COMPARE_OP_INT); - opcode = COMPARE_OP_INT; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - static_assert(INLINE_CACHE_ENTRIES_COMPARE_OP == 1, "incorrect cache size"); - _PyStackRef value; - _PyStackRef left; - _PyStackRef right; - _PyStackRef res; - // _GUARD_TOS_INT - { - value = stack_pointer[-1]; - PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); - if (!_PyLong_CheckExactAndCompact(value_o)) { - UPDATE_MISS_STATS(COMPARE_OP); - assert(_PyOpcode_Deopt[opcode] == (COMPARE_OP)); - JUMP_TO_PREDICTED(TRACING_COMPARE_OP); - } - } - // _GUARD_NOS_INT - { - left = stack_pointer[-2]; - PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); - if (!_PyLong_CheckExactAndCompact(left_o)) { - UPDATE_MISS_STATS(COMPARE_OP); - assert(_PyOpcode_Deopt[opcode] == (COMPARE_OP)); - JUMP_TO_PREDICTED(TRACING_COMPARE_OP); - } - } - /* Skip 1 cache entry */ - // _COMPARE_OP_INT - { - right = value; - PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); - PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - assert(_PyLong_IsCompact((PyLongObject *)left_o)); - assert(_PyLong_IsCompact((PyLongObject *)right_o)); - STAT_INC(COMPARE_OP, hit); - assert(_PyLong_DigitCount((PyLongObject *)left_o) <= 1 && - _PyLong_DigitCount((PyLongObject *)right_o) <= 1); - Py_ssize_t ileft = _PyLong_CompactValue((PyLongObject *)left_o); - Py_ssize_t iright = _PyLong_CompactValue((PyLongObject *)right_o); - int sign_ish = COMPARISON_BIT(ileft, iright); - PyStackRef_CLOSE_SPECIALIZED(left, _PyLong_ExactDealloc); - PyStackRef_CLOSE_SPECIALIZED(right, _PyLong_ExactDealloc); - res = (sign_ish & oparg) ? PyStackRef_True : PyStackRef_False; - } - stack_pointer[-2] = res; - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - TRACING_DISPATCH(); - } - - TRACING_TARGET(COMPARE_OP_STR) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = COMPARE_OP_STR; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(COMPARE_OP_STR); - opcode = COMPARE_OP_STR; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - static_assert(INLINE_CACHE_ENTRIES_COMPARE_OP == 1, "incorrect cache size"); - _PyStackRef value; - _PyStackRef nos; - _PyStackRef left; - _PyStackRef right; - _PyStackRef res; - // _GUARD_TOS_UNICODE - { - value = stack_pointer[-1]; - PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); - if (!PyUnicode_CheckExact(value_o)) { - UPDATE_MISS_STATS(COMPARE_OP); - assert(_PyOpcode_Deopt[opcode] == (COMPARE_OP)); - JUMP_TO_PREDICTED(TRACING_COMPARE_OP); - } - } - // _GUARD_NOS_UNICODE - { - nos = stack_pointer[-2]; - PyObject *o = PyStackRef_AsPyObjectBorrow(nos); - if (!PyUnicode_CheckExact(o)) { - UPDATE_MISS_STATS(COMPARE_OP); - assert(_PyOpcode_Deopt[opcode] == (COMPARE_OP)); - JUMP_TO_PREDICTED(TRACING_COMPARE_OP); - } - } - /* Skip 1 cache entry */ - // _COMPARE_OP_STR - { - right = value; - left = nos; - PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); - PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - STAT_INC(COMPARE_OP, hit); - int eq = _PyUnicode_Equal(left_o, right_o); - assert((oparg >> 5) == Py_EQ || (oparg >> 5) == Py_NE); - PyStackRef_CLOSE_SPECIALIZED(left, _PyUnicode_ExactDealloc); - PyStackRef_CLOSE_SPECIALIZED(right, _PyUnicode_ExactDealloc); - assert(eq == 0 || eq == 1); - assert((oparg & 0xf) == COMPARISON_NOT_EQUALS || (oparg & 0xf) == COMPARISON_EQUALS); - assert(COMPARISON_NOT_EQUALS + 1 == COMPARISON_EQUALS); - res = ((COMPARISON_NOT_EQUALS + eq) & oparg) ? PyStackRef_True : PyStackRef_False; - } - stack_pointer[-2] = res; - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - TRACING_DISPATCH(); - } - - TRACING_TARGET(CONTAINS_OP) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = CONTAINS_OP; - (void)(opcode); - #endif - frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(CONTAINS_OP); - PREDICTED_TRACING_CONTAINS_OP:; - _Py_CODEUNIT* const this_instr = next_instr - 2; - (void)this_instr; - opcode = CONTAINS_OP; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - _PyStackRef right; - _PyStackRef left; - _PyStackRef b; - // _SPECIALIZE_CONTAINS_OP - { - right = stack_pointer[-1]; - uint16_t counter = read_u16(&this_instr[1].cache); - (void)counter; - #if ENABLE_SPECIALIZATION_FT - if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { - next_instr = this_instr; - _PyFrame_SetStackPointer(frame, stack_pointer); - _Py_Specialize_ContainsOp(right, next_instr); - stack_pointer = _PyFrame_GetStackPointer(frame); - DISPATCH_SAME_OPARG(); - } - OPCODE_DEFERRED_INC(CONTAINS_OP); - ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); - #endif /* ENABLE_SPECIALIZATION_FT */ - } - // _CONTAINS_OP - { - left = stack_pointer[-2]; - PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); - PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - _PyFrame_SetStackPointer(frame, stack_pointer); - int res = PySequence_Contains(right_o, left_o); - _PyStackRef tmp = right; - right = PyStackRef_NULL; - stack_pointer[-1] = right; - PyStackRef_CLOSE(tmp); - tmp = left; - left = PyStackRef_NULL; - stack_pointer[-2] = left; - PyStackRef_CLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); - if (res < 0) { - TRACING_JUMP_TO_LABEL(error); - } - b = (res ^ oparg) ? PyStackRef_True : PyStackRef_False; - } - stack_pointer[0] = b; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - TRACING_DISPATCH(); - } - - TRACING_TARGET(CONTAINS_OP_DICT) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = CONTAINS_OP_DICT; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(CONTAINS_OP_DICT); - opcode = CONTAINS_OP_DICT; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - static_assert(INLINE_CACHE_ENTRIES_CONTAINS_OP == 1, "incorrect cache size"); - _PyStackRef tos; - _PyStackRef left; - _PyStackRef right; - _PyStackRef b; - // _GUARD_TOS_DICT - { - tos = stack_pointer[-1]; - PyObject *o = PyStackRef_AsPyObjectBorrow(tos); - if (!PyDict_CheckExact(o)) { - UPDATE_MISS_STATS(CONTAINS_OP); - assert(_PyOpcode_Deopt[opcode] == (CONTAINS_OP)); - JUMP_TO_PREDICTED(TRACING_CONTAINS_OP); - } - } - /* Skip 1 cache entry */ - // _CONTAINS_OP_DICT - { - right = tos; - left = stack_pointer[-2]; - PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); - PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - assert(PyDict_CheckExact(right_o)); - STAT_INC(CONTAINS_OP, hit); - _PyFrame_SetStackPointer(frame, stack_pointer); - int res = PyDict_Contains(right_o, left_o); - _PyStackRef tmp = right; - right = PyStackRef_NULL; - stack_pointer[-1] = right; - PyStackRef_CLOSE(tmp); - tmp = left; - left = PyStackRef_NULL; - stack_pointer[-2] = left; - PyStackRef_CLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); - if (res < 0) { - TRACING_JUMP_TO_LABEL(error); - } - b = (res ^ oparg) ? PyStackRef_True : PyStackRef_False; - } - stack_pointer[0] = b; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - TRACING_DISPATCH(); - } - - TRACING_TARGET(CONTAINS_OP_SET) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = CONTAINS_OP_SET; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(CONTAINS_OP_SET); - opcode = CONTAINS_OP_SET; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - static_assert(INLINE_CACHE_ENTRIES_CONTAINS_OP == 1, "incorrect cache size"); - _PyStackRef tos; - _PyStackRef left; - _PyStackRef right; - _PyStackRef b; - // _GUARD_TOS_ANY_SET - { - tos = stack_pointer[-1]; - PyObject *o = PyStackRef_AsPyObjectBorrow(tos); - if (!PyAnySet_CheckExact(o)) { - UPDATE_MISS_STATS(CONTAINS_OP); - assert(_PyOpcode_Deopt[opcode] == (CONTAINS_OP)); - JUMP_TO_PREDICTED(TRACING_CONTAINS_OP); - } - } - /* Skip 1 cache entry */ - // _CONTAINS_OP_SET - { - right = tos; - left = stack_pointer[-2]; - PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); - PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - assert(PyAnySet_CheckExact(right_o)); - STAT_INC(CONTAINS_OP, hit); - _PyFrame_SetStackPointer(frame, stack_pointer); - int res = _PySet_Contains((PySetObject *)right_o, left_o); - _PyStackRef tmp = right; - right = PyStackRef_NULL; - stack_pointer[-1] = right; - PyStackRef_CLOSE(tmp); - tmp = left; - left = PyStackRef_NULL; - stack_pointer[-2] = left; - PyStackRef_CLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); - if (res < 0) { - TRACING_JUMP_TO_LABEL(error); - } - b = (res ^ oparg) ? PyStackRef_True : PyStackRef_False; - } - stack_pointer[0] = b; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - TRACING_DISPATCH(); - } - - TRACING_TARGET(CONVERT_VALUE) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = CONVERT_VALUE; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(CONVERT_VALUE); - opcode = CONVERT_VALUE; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - _PyStackRef value; - _PyStackRef result; - value = stack_pointer[-1]; - conversion_func conv_fn; - assert(oparg >= FVC_STR && oparg <= FVC_ASCII); - conv_fn = _PyEval_ConversionFuncs[oparg]; - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *result_o = conv_fn(PyStackRef_AsPyObjectBorrow(value)); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(value); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (result_o == NULL) { - TRACING_JUMP_TO_LABEL(error); - } - result = PyStackRef_FromPyObjectSteal(result_o); - stack_pointer[0] = result; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - TRACING_DISPATCH(); - } - - TRACING_TARGET(COPY) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = COPY; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(COPY); - opcode = COPY; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - _PyStackRef bottom; - _PyStackRef top; - bottom = stack_pointer[-1 - (oparg-1)]; - top = PyStackRef_DUP(bottom); - stack_pointer[0] = top; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - TRACING_DISPATCH(); - } - - TRACING_TARGET(COPY_FREE_VARS) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = COPY_FREE_VARS; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(COPY_FREE_VARS); - opcode = COPY_FREE_VARS; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - PyCodeObject *co = _PyFrame_GetCode(frame); - assert(PyStackRef_FunctionCheck(frame->f_funcobj)); - PyFunctionObject *func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - PyObject *closure = func->func_closure; - assert(oparg == co->co_nfreevars); - int offset = co->co_nlocalsplus - oparg; - for (int i = 0; i < oparg; ++i) { - PyObject *o = PyTuple_GET_ITEM(closure, i); - frame->localsplus[offset + i] = PyStackRef_FromPyObjectNew(o); - } - TRACING_DISPATCH(); - } - - TRACING_TARGET(DELETE_ATTR) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = DELETE_ATTR; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(DELETE_ATTR); - opcode = DELETE_ATTR; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - _PyStackRef owner; - owner = stack_pointer[-1]; - PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = PyObject_DelAttr(PyStackRef_AsPyObjectBorrow(owner), name); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(owner); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err) { - TRACING_JUMP_TO_LABEL(error); - } - TRACING_DISPATCH(); - } - - TRACING_TARGET(DELETE_DEREF) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = DELETE_DEREF; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(DELETE_DEREF); - opcode = DELETE_DEREF; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - PyObject *cell = PyStackRef_AsPyObjectBorrow(GETLOCAL(oparg)); - PyObject *oldobj = PyCell_SwapTakeRef((PyCellObject *)cell, NULL); - if (oldobj == NULL) { - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyEval_FormatExcUnbound(tstate, _PyFrame_GetCode(frame), oparg); - stack_pointer = _PyFrame_GetStackPointer(frame); - TRACING_JUMP_TO_LABEL(error); - } - _PyFrame_SetStackPointer(frame, stack_pointer); - Py_DECREF(oldobj); - stack_pointer = _PyFrame_GetStackPointer(frame); - TRACING_DISPATCH(); - } - - TRACING_TARGET(DELETE_FAST) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = DELETE_FAST; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(DELETE_FAST); - opcode = DELETE_FAST; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - _PyStackRef v = GETLOCAL(oparg); - if (PyStackRef_IsNull(v)) { - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyEval_FormatExcCheckArg(tstate, PyExc_UnboundLocalError, - UNBOUNDLOCAL_ERROR_MSG, - PyTuple_GetItem(_PyFrame_GetCode(frame)->co_localsplusnames, oparg) - ); - stack_pointer = _PyFrame_GetStackPointer(frame); - TRACING_JUMP_TO_LABEL(error); - } - _PyStackRef tmp = GETLOCAL(oparg); - GETLOCAL(oparg) = PyStackRef_NULL; - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_XCLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); - TRACING_DISPATCH(); - } - - TRACING_TARGET(DELETE_GLOBAL) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = DELETE_GLOBAL; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(DELETE_GLOBAL); - opcode = DELETE_GLOBAL; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = PyDict_Pop(GLOBALS(), name, NULL); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err < 0) { - TRACING_JUMP_TO_LABEL(error); - } - if (err == 0) { - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyEval_FormatExcCheckArg(tstate, PyExc_NameError, - NAME_ERROR_MSG, name); - stack_pointer = _PyFrame_GetStackPointer(frame); - TRACING_JUMP_TO_LABEL(error); - } - TRACING_DISPATCH(); - } - - TRACING_TARGET(DELETE_NAME) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = DELETE_NAME; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(DELETE_NAME); - opcode = DELETE_NAME; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); - PyObject *ns = LOCALS(); - int err; - if (ns == NULL) { - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyErr_Format(tstate, PyExc_SystemError, - "no locals when deleting %R", name); - stack_pointer = _PyFrame_GetStackPointer(frame); - TRACING_JUMP_TO_LABEL(error); - } - _PyFrame_SetStackPointer(frame, stack_pointer); - err = PyObject_DelItem(ns, name); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err != 0) { - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyEval_FormatExcCheckArg(tstate, PyExc_NameError, - NAME_ERROR_MSG, - name); - stack_pointer = _PyFrame_GetStackPointer(frame); - TRACING_JUMP_TO_LABEL(error); - } - TRACING_DISPATCH(); - } - - TRACING_TARGET(DELETE_SUBSCR) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = DELETE_SUBSCR; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(DELETE_SUBSCR); - opcode = DELETE_SUBSCR; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - _PyStackRef container; - _PyStackRef sub; - sub = stack_pointer[-1]; - container = stack_pointer[-2]; - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = PyObject_DelItem(PyStackRef_AsPyObjectBorrow(container), - PyStackRef_AsPyObjectBorrow(sub)); - _PyStackRef tmp = sub; - sub = PyStackRef_NULL; - stack_pointer[-1] = sub; - PyStackRef_CLOSE(tmp); - tmp = container; - container = PyStackRef_NULL; - stack_pointer[-2] = container; - PyStackRef_CLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); - if (err) { - TRACING_JUMP_TO_LABEL(error); - } - TRACING_DISPATCH(); - } - - TRACING_TARGET(DICT_MERGE) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = DICT_MERGE; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(DICT_MERGE); - opcode = DICT_MERGE; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - _PyStackRef callable; - _PyStackRef dict; - _PyStackRef update; - update = stack_pointer[-1]; - dict = stack_pointer[-2 - (oparg - 1)]; - callable = stack_pointer[-5 - (oparg - 1)]; - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); - PyObject *dict_o = PyStackRef_AsPyObjectBorrow(dict); - PyObject *update_o = PyStackRef_AsPyObjectBorrow(update); - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = _PyDict_MergeEx(dict_o, update_o, 2); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err < 0) { - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyEval_FormatKwargsError(tstate, callable_o, update_o); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(update); - stack_pointer = _PyFrame_GetStackPointer(frame); - TRACING_JUMP_TO_LABEL(error); - } - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(update); - stack_pointer = _PyFrame_GetStackPointer(frame); - TRACING_DISPATCH(); - } - - TRACING_TARGET(DICT_UPDATE) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = DICT_UPDATE; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(DICT_UPDATE); - opcode = DICT_UPDATE; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - _PyStackRef dict; - _PyStackRef update; - update = stack_pointer[-1]; - dict = stack_pointer[-2 - (oparg - 1)]; - PyObject *dict_o = PyStackRef_AsPyObjectBorrow(dict); - PyObject *update_o = PyStackRef_AsPyObjectBorrow(update); - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = PyDict_Update(dict_o, update_o); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err < 0) { - _PyFrame_SetStackPointer(frame, stack_pointer); - int matches = _PyErr_ExceptionMatches(tstate, PyExc_AttributeError); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (matches) { - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyErr_Format(tstate, PyExc_TypeError, - "'%.200s' object is not a mapping", - Py_TYPE(update_o)->tp_name); - stack_pointer = _PyFrame_GetStackPointer(frame); - } - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(update); - stack_pointer = _PyFrame_GetStackPointer(frame); - TRACING_JUMP_TO_LABEL(error); - } - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(update); - stack_pointer = _PyFrame_GetStackPointer(frame); - TRACING_DISPATCH(); - } - - TRACING_TARGET(END_ASYNC_FOR) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = END_ASYNC_FOR; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(END_ASYNC_FOR); - opcode = END_ASYNC_FOR; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - _PyStackRef awaitable_st; - _PyStackRef exc_st; - exc_st = stack_pointer[-1]; - awaitable_st = stack_pointer[-2]; - JUMPBY(0); - (void)oparg; - PyObject *exc = PyStackRef_AsPyObjectBorrow(exc_st); - assert(exc && PyExceptionInstance_Check(exc)); - _PyFrame_SetStackPointer(frame, stack_pointer); - int matches = PyErr_GivenExceptionMatches(exc, PyExc_StopAsyncIteration); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (matches) { - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyStackRef tmp = exc_st; - exc_st = PyStackRef_NULL; - stack_pointer[-1] = exc_st; - PyStackRef_CLOSE(tmp); - tmp = awaitable_st; - awaitable_st = PyStackRef_NULL; - stack_pointer[-2] = awaitable_st; - PyStackRef_CLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); - } - else { - Py_INCREF(exc); - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyErr_SetRaisedException(tstate, exc); - monitor_reraise(tstate, frame, this_instr); - TRACING_JUMP_TO_LABEL(exception_unwind); - } - TRACING_DISPATCH(); - } - - TRACING_TARGET(END_FOR) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = END_FOR; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - next_instr += 1; - INSTRUCTION_STATS(END_FOR); - opcode = END_FOR; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - _PyStackRef value; - value = stack_pointer[-1]; - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(value); - stack_pointer = _PyFrame_GetStackPointer(frame); - TRACING_DISPATCH(); - } - - TRACING_TARGET(END_SEND) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = END_SEND; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(END_SEND); - opcode = END_SEND; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - _PyStackRef receiver; - _PyStackRef value; - _PyStackRef val; - value = stack_pointer[-1]; - receiver = stack_pointer[-2]; - val = value; - stack_pointer[-2] = val; - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(receiver); - stack_pointer = _PyFrame_GetStackPointer(frame); - TRACING_DISPATCH(); - } - - TRACING_TARGET(ENTER_EXECUTOR) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = ENTER_EXECUTOR; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(ENTER_EXECUTOR); - opcode = ENTER_EXECUTOR; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - opcode = ENTER_EXECUTOR; - #ifdef _Py_TIER2 - PyCodeObject *code = _PyFrame_GetCode(frame); - _PyExecutorObject *executor = code->co_executors->executors[oparg & 255]; - assert(executor->vm_data.index == INSTR_OFFSET() - 1); - assert(executor->vm_data.code == code); - assert(executor->vm_data.valid); - assert(tstate->current_executor == NULL); - if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { - opcode = executor->vm_data.opcode; - oparg = (oparg & ~255) | executor->vm_data.oparg; - next_instr = this_instr; - if (_PyOpcode_Caches[_PyOpcode_Deopt[opcode]]) { - PAUSE_ADAPTIVE_COUNTER(this_instr[1].counter); - } - DISPATCH_GOTO(); - } - assert(executor != tstate->interp->cold_executor); - tstate->jit_exit = NULL; - if (IS_JIT_TRACING()) { - RECORD_TRACE_NO_DISPATCH(); - } - TIER1_TO_TIER2(executor); - #else - Py_FatalError("ENTER_EXECUTOR is not supported in this build"); - #endif /* _Py_TIER2 */ - TRACING_DISPATCH(); - } - - TRACING_TARGET(EXIT_INIT_CHECK) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = EXIT_INIT_CHECK; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(EXIT_INIT_CHECK); - opcode = EXIT_INIT_CHECK; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - _PyStackRef should_be_none; - should_be_none = stack_pointer[-1]; - if (!PyStackRef_IsNone(should_be_none)) { - _PyFrame_SetStackPointer(frame, stack_pointer); - PyErr_Format(PyExc_TypeError, - "__init__() should return None, not '%.200s'", - Py_TYPE(PyStackRef_AsPyObjectBorrow(should_be_none))->tp_name); - stack_pointer = _PyFrame_GetStackPointer(frame); - TRACING_JUMP_TO_LABEL(error); - } - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - TRACING_DISPATCH(); - } - - TRACING_TARGET(EXTENDED_ARG) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = EXTENDED_ARG; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(EXTENDED_ARG); - opcode = EXTENDED_ARG; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - opcode = EXTENDED_ARG; - assert(oparg); - opcode = next_instr->op.code; - oparg = oparg << 8 | next_instr->op.arg; - PRE_DISPATCH_GOTO(); - DISPATCH_GOTO(); - } - - TRACING_TARGET(FORMAT_SIMPLE) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = FORMAT_SIMPLE; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(FORMAT_SIMPLE); - opcode = FORMAT_SIMPLE; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - _PyStackRef value; - _PyStackRef res; - value = stack_pointer[-1]; - PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); - if (!PyUnicode_CheckExact(value_o)) { - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *res_o = PyObject_Format(value_o, NULL); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(value); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (res_o == NULL) { - TRACING_JUMP_TO_LABEL(error); - } - res = PyStackRef_FromPyObjectSteal(res_o); - } - else { - res = value; - stack_pointer += -1; - } - stack_pointer[0] = res; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - TRACING_DISPATCH(); - } - - TRACING_TARGET(FORMAT_WITH_SPEC) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = FORMAT_WITH_SPEC; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(FORMAT_WITH_SPEC); - opcode = FORMAT_WITH_SPEC; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - _PyStackRef value; - _PyStackRef fmt_spec; - _PyStackRef res; - fmt_spec = stack_pointer[-1]; - value = stack_pointer[-2]; - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *res_o = PyObject_Format(PyStackRef_AsPyObjectBorrow(value), PyStackRef_AsPyObjectBorrow(fmt_spec)); - _PyStackRef tmp = fmt_spec; - fmt_spec = PyStackRef_NULL; - stack_pointer[-1] = fmt_spec; - PyStackRef_CLOSE(tmp); - tmp = value; - value = PyStackRef_NULL; - stack_pointer[-2] = value; - PyStackRef_CLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); - if (res_o == NULL) { - TRACING_JUMP_TO_LABEL(error); - } - res = PyStackRef_FromPyObjectSteal(res_o); - stack_pointer[0] = res; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - TRACING_DISPATCH(); - } - - TRACING_TARGET(FOR_ITER) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = FOR_ITER; - (void)(opcode); - #endif - frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(FOR_ITER); - PREDICTED_TRACING_FOR_ITER:; - _Py_CODEUNIT* const this_instr = next_instr - 2; - (void)this_instr; - opcode = FOR_ITER; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - _PyStackRef iter; - _PyStackRef null_or_index; - _PyStackRef next; - // _SPECIALIZE_FOR_ITER - { - null_or_index = stack_pointer[-1]; - iter = stack_pointer[-2]; - uint16_t counter = read_u16(&this_instr[1].cache); - (void)counter; - #if ENABLE_SPECIALIZATION_FT - if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { - next_instr = this_instr; - _PyFrame_SetStackPointer(frame, stack_pointer); - _Py_Specialize_ForIter(iter, null_or_index, next_instr, oparg); - stack_pointer = _PyFrame_GetStackPointer(frame); - DISPATCH_SAME_OPARG(); - } - OPCODE_DEFERRED_INC(FOR_ITER); - ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); - #endif /* ENABLE_SPECIALIZATION_FT */ - } - // _FOR_ITER - { - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyStackRef item = _PyForIter_VirtualIteratorNext(tstate, frame, iter, &null_or_index); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (!PyStackRef_IsValid(item)) { - if (PyStackRef_IsError(item)) { - TRACING_JUMP_TO_LABEL(error); - } - JUMPBY(oparg + 1); - RECORD_JUMP_TAKEN(); - stack_pointer[-1] = null_or_index; - TRACING_DISPATCH(); - } - next = item; - } - stack_pointer[-1] = null_or_index; - stack_pointer[0] = next; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - TRACING_DISPATCH(); - } - - TRACING_TARGET(FOR_ITER_GEN) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = FOR_ITER_GEN; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(FOR_ITER_GEN); - opcode = FOR_ITER_GEN; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - static_assert(INLINE_CACHE_ENTRIES_FOR_ITER == 1, "incorrect cache size"); - _PyStackRef iter; - _PyStackRef gen_frame; - _PyStackRef new_frame; - /* Skip 1 cache entry */ - // _CHECK_PEP_523 - { - if (tstate->interp->eval_frame) { - UPDATE_MISS_STATS(FOR_ITER); - assert(_PyOpcode_Deopt[opcode] == (FOR_ITER)); - JUMP_TO_PREDICTED(TRACING_FOR_ITER); - } - } - // _FOR_ITER_GEN_FRAME - { - iter = stack_pointer[-2]; - PyGenObject *gen = (PyGenObject *)PyStackRef_AsPyObjectBorrow(iter); - if (Py_TYPE(gen) != &PyGen_Type) { - UPDATE_MISS_STATS(FOR_ITER); - assert(_PyOpcode_Deopt[opcode] == (FOR_ITER)); - JUMP_TO_PREDICTED(TRACING_FOR_ITER); - } - #ifdef Py_GIL_DISABLED - if (!_PyObject_IsUniquelyReferenced((PyObject *)gen)) { - UPDATE_MISS_STATS(FOR_ITER); - assert(_PyOpcode_Deopt[opcode] == (FOR_ITER)); - JUMP_TO_PREDICTED(TRACING_FOR_ITER); - } - #endif - if (gen->gi_frame_state >= FRAME_EXECUTING) { - UPDATE_MISS_STATS(FOR_ITER); - assert(_PyOpcode_Deopt[opcode] == (FOR_ITER)); - JUMP_TO_PREDICTED(TRACING_FOR_ITER); - } - STAT_INC(FOR_ITER, hit); - _PyInterpreterFrame *pushed_frame = &gen->gi_iframe; - _PyFrame_StackPush(pushed_frame, PyStackRef_None); - gen->gi_frame_state = FRAME_EXECUTING; - gen->gi_exc_state.previous_item = tstate->exc_info; - tstate->exc_info = &gen->gi_exc_state; - pushed_frame->previous = frame; - frame->return_offset = (uint16_t)( 2u + oparg); - gen_frame = PyStackRef_Wrap(pushed_frame); - } - // _PUSH_FRAME - { - new_frame = gen_frame; - assert(tstate->interp->eval_frame == NULL); - _PyInterpreterFrame *temp = PyStackRef_Unwrap(new_frame); - _PyFrame_SetStackPointer(frame, stack_pointer); - assert(temp->previous == frame || temp->previous->previous == frame); - CALL_STAT_INC(inlined_py_calls); - frame = tstate->current_frame = temp; - tstate->py_recursion_remaining--; - LOAD_SP(); - LOAD_IP(0); - LLTRACE_RESUME_FRAME(); - } - TRACING_DISPATCH(); - } - - TRACING_TARGET(FOR_ITER_LIST) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = FOR_ITER_LIST; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(FOR_ITER_LIST); - opcode = FOR_ITER_LIST; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - static_assert(INLINE_CACHE_ENTRIES_FOR_ITER == 1, "incorrect cache size"); - _PyStackRef iter; - _PyStackRef null_or_index; - _PyStackRef next; - /* Skip 1 cache entry */ - // _ITER_CHECK_LIST - { - null_or_index = stack_pointer[-1]; - iter = stack_pointer[-2]; - PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter); - if (Py_TYPE(iter_o) != &PyList_Type) { - UPDATE_MISS_STATS(FOR_ITER); - assert(_PyOpcode_Deopt[opcode] == (FOR_ITER)); - JUMP_TO_PREDICTED(TRACING_FOR_ITER); - } - assert(PyStackRef_IsTaggedInt(null_or_index)); - #ifdef Py_GIL_DISABLED - if (!_Py_IsOwnedByCurrentThread(iter_o) && !_PyObject_GC_IS_SHARED(iter_o)) { - UPDATE_MISS_STATS(FOR_ITER); - assert(_PyOpcode_Deopt[opcode] == (FOR_ITER)); - JUMP_TO_PREDICTED(TRACING_FOR_ITER); - } - #endif - } - // _ITER_JUMP_LIST - { - #ifdef Py_GIL_DISABLED - - #else - PyObject *list_o = PyStackRef_AsPyObjectBorrow(iter); - assert(Py_TYPE(list_o) == &PyList_Type); - STAT_INC(FOR_ITER, hit); - if ((size_t)PyStackRef_UntagInt(null_or_index) >= (size_t)PyList_GET_SIZE(list_o)) { - null_or_index = PyStackRef_TagInt(-1); - JUMPBY(oparg + 1); - RECORD_JUMP_TAKEN(); - stack_pointer[-1] = null_or_index; - TRACING_DISPATCH(); - } - #endif - } - // _ITER_NEXT_LIST - { - PyObject *list_o = PyStackRef_AsPyObjectBorrow(iter); - assert(PyList_CheckExact(list_o)); - #ifdef Py_GIL_DISABLED - assert(_Py_IsOwnedByCurrentThread(list_o) || - _PyObject_GC_IS_SHARED(list_o)); - STAT_INC(FOR_ITER, hit); - _PyFrame_SetStackPointer(frame, stack_pointer); - int result = _PyList_GetItemRefNoLock((PyListObject *)list_o, PyStackRef_UntagInt(null_or_index), &next); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (result < 0) { - UPDATE_MISS_STATS(FOR_ITER); - assert(_PyOpcode_Deopt[opcode] == (FOR_ITER)); - JUMP_TO_PREDICTED(TRACING_FOR_ITER); - } - if (result == 0) { - null_or_index = PyStackRef_TagInt(-1); - JUMPBY(oparg + 1); - stack_pointer[-1] = null_or_index; - TRACING_DISPATCH(); - } - #else - next = PyStackRef_FromPyObjectNew(PyList_GET_ITEM(list_o, PyStackRef_UntagInt(null_or_index))); - #endif - null_or_index = PyStackRef_IncrementTaggedIntNoOverflow(null_or_index); - } - stack_pointer[-1] = null_or_index; - stack_pointer[0] = next; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - TRACING_DISPATCH(); - } - - TRACING_TARGET(FOR_ITER_RANGE) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = FOR_ITER_RANGE; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(FOR_ITER_RANGE); - opcode = FOR_ITER_RANGE; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - static_assert(INLINE_CACHE_ENTRIES_FOR_ITER == 1, "incorrect cache size"); - _PyStackRef iter; - _PyStackRef next; - /* Skip 1 cache entry */ - // _ITER_CHECK_RANGE - { - iter = stack_pointer[-2]; - _PyRangeIterObject *r = (_PyRangeIterObject *)PyStackRef_AsPyObjectBorrow(iter); - if (Py_TYPE(r) != &PyRangeIter_Type) { - UPDATE_MISS_STATS(FOR_ITER); - assert(_PyOpcode_Deopt[opcode] == (FOR_ITER)); - JUMP_TO_PREDICTED(TRACING_FOR_ITER); - } - #ifdef Py_GIL_DISABLED - if (!_PyObject_IsUniquelyReferenced((PyObject *)r)) { - UPDATE_MISS_STATS(FOR_ITER); - assert(_PyOpcode_Deopt[opcode] == (FOR_ITER)); - JUMP_TO_PREDICTED(TRACING_FOR_ITER); - } - #endif - } - // _ITER_JUMP_RANGE - { - _PyRangeIterObject *r = (_PyRangeIterObject *)PyStackRef_AsPyObjectBorrow(iter); - assert(Py_TYPE(r) == &PyRangeIter_Type); - #ifdef Py_GIL_DISABLED - assert(_PyObject_IsUniquelyReferenced((PyObject *)r)); - #endif - STAT_INC(FOR_ITER, hit); - if (r->len <= 0) { - JUMPBY(oparg + 1); - RECORD_JUMP_TAKEN(); - TRACING_DISPATCH(); - } - } - // _ITER_NEXT_RANGE - { - _PyRangeIterObject *r = (_PyRangeIterObject *)PyStackRef_AsPyObjectBorrow(iter); - assert(Py_TYPE(r) == &PyRangeIter_Type); - #ifdef Py_GIL_DISABLED - assert(_PyObject_IsUniquelyReferenced((PyObject *)r)); - #endif - assert(r->len > 0); - long value = r->start; - r->start = value + r->step; - r->len--; - PyObject *res = PyLong_FromLong(value); - if (res == NULL) { - TRACING_JUMP_TO_LABEL(error); - } - next = PyStackRef_FromPyObjectSteal(res); - } - stack_pointer[0] = next; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - TRACING_DISPATCH(); - } - - TRACING_TARGET(FOR_ITER_TUPLE) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = FOR_ITER_TUPLE; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(FOR_ITER_TUPLE); - opcode = FOR_ITER_TUPLE; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - static_assert(INLINE_CACHE_ENTRIES_FOR_ITER == 1, "incorrect cache size"); - _PyStackRef iter; - _PyStackRef null_or_index; - _PyStackRef next; - /* Skip 1 cache entry */ - // _ITER_CHECK_TUPLE - { - null_or_index = stack_pointer[-1]; - iter = stack_pointer[-2]; - PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter); - if (Py_TYPE(iter_o) != &PyTuple_Type) { - UPDATE_MISS_STATS(FOR_ITER); - assert(_PyOpcode_Deopt[opcode] == (FOR_ITER)); - JUMP_TO_PREDICTED(TRACING_FOR_ITER); - } - assert(PyStackRef_IsTaggedInt(null_or_index)); - } - // _ITER_JUMP_TUPLE - { - PyObject *tuple_o = PyStackRef_AsPyObjectBorrow(iter); - (void)tuple_o; - assert(Py_TYPE(tuple_o) == &PyTuple_Type); - STAT_INC(FOR_ITER, hit); - if ((size_t)PyStackRef_UntagInt(null_or_index) >= (size_t)PyTuple_GET_SIZE(tuple_o)) { - null_or_index = PyStackRef_TagInt(-1); - JUMPBY(oparg + 1); - RECORD_JUMP_TAKEN(); - stack_pointer[-1] = null_or_index; - TRACING_DISPATCH(); - } - } - // _ITER_NEXT_TUPLE - { - PyObject *tuple_o = PyStackRef_AsPyObjectBorrow(iter); - assert(Py_TYPE(tuple_o) == &PyTuple_Type); - uintptr_t i = PyStackRef_UntagInt(null_or_index); - assert((size_t)i < (size_t)PyTuple_GET_SIZE(tuple_o)); - next = PyStackRef_FromPyObjectNew(PyTuple_GET_ITEM(tuple_o, i)); - null_or_index = PyStackRef_IncrementTaggedIntNoOverflow(null_or_index); - } - stack_pointer[-1] = null_or_index; - stack_pointer[0] = next; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - TRACING_DISPATCH(); - } - - TRACING_TARGET(GET_AITER) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = GET_AITER; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(GET_AITER); - opcode = GET_AITER; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - _PyStackRef obj; - _PyStackRef iter; - obj = stack_pointer[-1]; - unaryfunc getter = NULL; - PyObject *obj_o = PyStackRef_AsPyObjectBorrow(obj); - PyObject *iter_o; - PyTypeObject *type = Py_TYPE(obj_o); - if (type->tp_as_async != NULL) { - getter = type->tp_as_async->am_aiter; - } - if (getter == NULL) { - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyErr_Format(tstate, PyExc_TypeError, - "'async for' requires an object with " - "__aiter__ method, got %.100s", - type->tp_name); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(obj); - stack_pointer = _PyFrame_GetStackPointer(frame); - TRACING_JUMP_TO_LABEL(error); - } - _PyFrame_SetStackPointer(frame, stack_pointer); - iter_o = (*getter)(obj_o); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(obj); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (iter_o == NULL) { - TRACING_JUMP_TO_LABEL(error); - } - if (Py_TYPE(iter_o)->tp_as_async == NULL || - Py_TYPE(iter_o)->tp_as_async->am_anext == NULL) { - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyErr_Format(tstate, PyExc_TypeError, - "'async for' received an object from __aiter__ " - "that does not implement __anext__: %.100s", - Py_TYPE(iter_o)->tp_name); - Py_DECREF(iter_o); - stack_pointer = _PyFrame_GetStackPointer(frame); - TRACING_JUMP_TO_LABEL(error); - } - iter = PyStackRef_FromPyObjectSteal(iter_o); - stack_pointer[0] = iter; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - TRACING_DISPATCH(); - } - - TRACING_TARGET(GET_ANEXT) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = GET_ANEXT; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(GET_ANEXT); - opcode = GET_ANEXT; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - _PyStackRef aiter; - _PyStackRef awaitable; - aiter = stack_pointer[-1]; - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *awaitable_o = _PyEval_GetANext(PyStackRef_AsPyObjectBorrow(aiter)); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (awaitable_o == NULL) { - TRACING_JUMP_TO_LABEL(error); - } - awaitable = PyStackRef_FromPyObjectSteal(awaitable_o); - stack_pointer[0] = awaitable; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - TRACING_DISPATCH(); - } - - TRACING_TARGET(GET_AWAITABLE) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = GET_AWAITABLE; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(GET_AWAITABLE); - opcode = GET_AWAITABLE; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - _PyStackRef iterable; - _PyStackRef iter; - iterable = stack_pointer[-1]; - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *iter_o = _PyEval_GetAwaitable(PyStackRef_AsPyObjectBorrow(iterable), oparg); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(iterable); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (iter_o == NULL) { - TRACING_JUMP_TO_LABEL(error); - } - iter = PyStackRef_FromPyObjectSteal(iter_o); - stack_pointer[0] = iter; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - TRACING_DISPATCH(); - } - - TRACING_TARGET(GET_ITER) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = GET_ITER; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(GET_ITER); - opcode = GET_ITER; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - _PyStackRef iterable; - _PyStackRef iter; - _PyStackRef index_or_null; - iterable = stack_pointer[-1]; - #ifdef Py_STATS - _PyFrame_SetStackPointer(frame, stack_pointer); - _Py_GatherStats_GetIter(iterable); - stack_pointer = _PyFrame_GetStackPointer(frame); - #endif - - PyTypeObject *tp = PyStackRef_TYPE(iterable); - if (tp == &PyTuple_Type || tp == &PyList_Type) { - iter = iterable; - index_or_null = PyStackRef_TagInt(0); - } - else { - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *iter_o = PyObject_GetIter(PyStackRef_AsPyObjectBorrow(iterable)); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(iterable); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (iter_o == NULL) { - TRACING_JUMP_TO_LABEL(error); - } - iter = PyStackRef_FromPyObjectSteal(iter_o); - index_or_null = PyStackRef_NULL; - stack_pointer += 1; - } - stack_pointer[-1] = iter; - stack_pointer[0] = index_or_null; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - TRACING_DISPATCH(); - } - - TRACING_TARGET(GET_LEN) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = GET_LEN; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(GET_LEN); - opcode = GET_LEN; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - _PyStackRef obj; - _PyStackRef len; - obj = stack_pointer[-1]; - _PyFrame_SetStackPointer(frame, stack_pointer); - Py_ssize_t len_i = PyObject_Length(PyStackRef_AsPyObjectBorrow(obj)); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (len_i < 0) { - TRACING_JUMP_TO_LABEL(error); - } - PyObject *len_o = PyLong_FromSsize_t(len_i); - if (len_o == NULL) { - TRACING_JUMP_TO_LABEL(error); - } - len = PyStackRef_FromPyObjectSteal(len_o); - stack_pointer[0] = len; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - TRACING_DISPATCH(); - } - - TRACING_TARGET(GET_YIELD_FROM_ITER) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = GET_YIELD_FROM_ITER; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(GET_YIELD_FROM_ITER); - opcode = GET_YIELD_FROM_ITER; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - _PyStackRef iterable; - _PyStackRef iter; - iterable = stack_pointer[-1]; - PyObject *iterable_o = PyStackRef_AsPyObjectBorrow(iterable); - if (PyCoro_CheckExact(iterable_o)) { - if (!(_PyFrame_GetCode(frame)->co_flags & (CO_COROUTINE | CO_ITERABLE_COROUTINE))) { - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyErr_SetString(tstate, PyExc_TypeError, - "cannot 'yield from' a coroutine object " - "in a non-coroutine generator"); - stack_pointer = _PyFrame_GetStackPointer(frame); - TRACING_JUMP_TO_LABEL(error); - } - iter = iterable; - } - else if (PyGen_CheckExact(iterable_o)) { - iter = iterable; - } - else { - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *iter_o = PyObject_GetIter(iterable_o); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (iter_o == NULL) { - TRACING_JUMP_TO_LABEL(error); - } - iter = PyStackRef_FromPyObjectSteal(iter_o); - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyStackRef tmp = iterable; - iterable = iter; - stack_pointer[-1] = iterable; - PyStackRef_CLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); - } - stack_pointer[-1] = iter; - TRACING_DISPATCH(); - } - - TRACING_TARGET(IMPORT_FROM) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = IMPORT_FROM; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(IMPORT_FROM); - opcode = IMPORT_FROM; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - _PyStackRef from; - _PyStackRef res; - from = stack_pointer[-1]; - PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *res_o = _PyEval_ImportFrom(tstate, PyStackRef_AsPyObjectBorrow(from), name); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (res_o == NULL) { - TRACING_JUMP_TO_LABEL(error); - } - res = PyStackRef_FromPyObjectSteal(res_o); - stack_pointer[0] = res; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - TRACING_DISPATCH(); - } - - TRACING_TARGET(IMPORT_NAME) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = IMPORT_NAME; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(IMPORT_NAME); - opcode = IMPORT_NAME; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - _PyStackRef level; - _PyStackRef fromlist; - _PyStackRef res; - fromlist = stack_pointer[-1]; - level = stack_pointer[-2]; - PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *res_o = _PyEval_ImportName(tstate, frame, name, - PyStackRef_AsPyObjectBorrow(fromlist), - PyStackRef_AsPyObjectBorrow(level)); - _PyStackRef tmp = fromlist; - fromlist = PyStackRef_NULL; - stack_pointer[-1] = fromlist; - PyStackRef_CLOSE(tmp); - tmp = level; - level = PyStackRef_NULL; - stack_pointer[-2] = level; - PyStackRef_CLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); - if (res_o == NULL) { - TRACING_JUMP_TO_LABEL(error); - } - res = PyStackRef_FromPyObjectSteal(res_o); - stack_pointer[0] = res; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - TRACING_DISPATCH(); - } - - TRACING_TARGET(INSTRUMENTED_CALL) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = INSTRUMENTED_CALL; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 4; - INSTRUCTION_STATS(INSTRUMENTED_CALL); - opcode = INSTRUMENTED_CALL; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - opcode = INSTRUMENTED_CALL; - _PyStackRef callable; - _PyStackRef self_or_null; - _PyStackRef func; - _PyStackRef maybe_self; - _PyStackRef *args; - _PyStackRef res; - /* Skip 3 cache entries */ - // _MAYBE_EXPAND_METHOD - { - self_or_null = stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; - if (PyStackRef_TYPE(callable) == &PyMethod_Type && PyStackRef_IsNull(self_or_null)) { - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); - PyObject *self = ((PyMethodObject *)callable_o)->im_self; - self_or_null = PyStackRef_FromPyObjectNew(self); - PyObject *method = ((PyMethodObject *)callable_o)->im_func; - _PyStackRef temp = callable; - callable = PyStackRef_FromPyObjectNew(method); - stack_pointer[-2 - oparg] = callable; - stack_pointer[-1 - oparg] = self_or_null; - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(temp); - stack_pointer = _PyFrame_GetStackPointer(frame); - } - } - // _MONITOR_CALL - { - args = &stack_pointer[-oparg]; - maybe_self = self_or_null; - func = callable; - int is_meth = !PyStackRef_IsNull(maybe_self); - PyObject *function = PyStackRef_AsPyObjectBorrow(func); - PyObject *arg0; - if (is_meth) { - arg0 = PyStackRef_AsPyObjectBorrow(maybe_self); - } - else if (oparg) { - arg0 = PyStackRef_AsPyObjectBorrow(args[0]); - } - else { - arg0 = &_PyInstrumentation_MISSING; - } - stack_pointer[-2 - oparg] = func; - stack_pointer[-1 - oparg] = maybe_self; - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = _Py_call_instrumentation_2args( - tstate, PY_MONITORING_EVENT_CALL, - frame, this_instr, function, arg0 - ); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err) { - TRACING_JUMP_TO_LABEL(error); - } - } - // _DO_CALL - { - args = &stack_pointer[-oparg]; - self_or_null = stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); - int total_args = oparg; - _PyStackRef *arguments = args; - if (!PyStackRef_IsNull(self_or_null)) { - arguments--; - total_args++; - } - if (Py_TYPE(callable_o) == &PyFunction_Type && - tstate->interp->eval_frame == NULL && - ((PyFunctionObject *)callable_o)->vectorcall == _PyFunction_Vectorcall) - { - int code_flags = ((PyCodeObject*)PyFunction_GET_CODE(callable_o))->co_flags; - PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable_o)); - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyInterpreterFrame *new_frame = _PyEvalFramePushAndInit( - tstate, callable, locals, - arguments, total_args, NULL, frame - ); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -2 - oparg; - assert(WITHIN_STACK_BOUNDS()); - if (new_frame == NULL) { - TRACING_JUMP_TO_LABEL(error); - } - frame->return_offset = 4u ; - TRACING_DISPATCH_INLINED(new_frame); - } - STACKREFS_TO_PYOBJECTS(arguments, total_args, args_o); - if (CONVERSION_FAILED(args_o)) { - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyStackRef tmp; - for (int _i = oparg; --_i >= 0;) { - tmp = args[_i]; - args[_i] = PyStackRef_NULL; - PyStackRef_CLOSE(tmp); - } - tmp = self_or_null; - self_or_null = PyStackRef_NULL; - stack_pointer[-1 - oparg] = self_or_null; - PyStackRef_XCLOSE(tmp); - tmp = callable; - callable = PyStackRef_NULL; - stack_pointer[-2 - oparg] = callable; - PyStackRef_CLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -2 - oparg; - assert(WITHIN_STACK_BOUNDS()); - TRACING_JUMP_TO_LABEL(error); - } - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *res_o = PyObject_Vectorcall( - callable_o, args_o, - total_args | PY_VECTORCALL_ARGUMENTS_OFFSET, - NULL); - stack_pointer = _PyFrame_GetStackPointer(frame); - STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); - if (opcode == INSTRUMENTED_CALL) { - PyObject *arg = total_args == 0 ? - &_PyInstrumentation_MISSING : PyStackRef_AsPyObjectBorrow(arguments[0]); - if (res_o == NULL) { - _PyFrame_SetStackPointer(frame, stack_pointer); - _Py_call_instrumentation_exc2( - tstate, PY_MONITORING_EVENT_C_RAISE, - frame, this_instr, callable_o, arg); - stack_pointer = _PyFrame_GetStackPointer(frame); - } - else { - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = _Py_call_instrumentation_2args( - tstate, PY_MONITORING_EVENT_C_RETURN, - frame, this_instr, callable_o, arg); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err < 0) { - _PyFrame_SetStackPointer(frame, stack_pointer); - Py_CLEAR(res_o); - stack_pointer = _PyFrame_GetStackPointer(frame); - } - } - } - assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyStackRef tmp; - for (int _i = oparg; --_i >= 0;) { - tmp = args[_i]; - args[_i] = PyStackRef_NULL; - PyStackRef_CLOSE(tmp); - } - tmp = self_or_null; - self_or_null = PyStackRef_NULL; - stack_pointer[-1 - oparg] = self_or_null; - PyStackRef_XCLOSE(tmp); - tmp = callable; - callable = PyStackRef_NULL; - stack_pointer[-2 - oparg] = callable; - PyStackRef_CLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -2 - oparg; - assert(WITHIN_STACK_BOUNDS()); - if (res_o == NULL) { - TRACING_JUMP_TO_LABEL(error); - } - res = PyStackRef_FromPyObjectSteal(res_o); - } - // _CHECK_PERIODIC_AT_END - { - stack_pointer[0] = res; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = check_periodics(tstate); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err != 0) { - TRACING_JUMP_TO_LABEL(error); - } - } - TRACING_DISPATCH(); - } - - TRACING_TARGET(INSTRUMENTED_CALL_FUNCTION_EX) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = INSTRUMENTED_CALL_FUNCTION_EX; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(INSTRUMENTED_CALL_FUNCTION_EX); - opcode = INSTRUMENTED_CALL_FUNCTION_EX; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - opcode = INSTRUMENTED_CALL_FUNCTION_EX; - _PyStackRef func; - _PyStackRef callargs; - _PyStackRef func_st; - _PyStackRef null; - _PyStackRef callargs_st; - _PyStackRef kwargs_st; - _PyStackRef result; - // _MAKE_CALLARGS_A_TUPLE - { - callargs = stack_pointer[-2]; - func = stack_pointer[-4]; - PyObject *callargs_o = PyStackRef_AsPyObjectBorrow(callargs); - if (!PyTuple_CheckExact(callargs_o)) { - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = _Py_Check_ArgsIterable(tstate, PyStackRef_AsPyObjectBorrow(func), callargs_o); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err < 0) { - TRACING_JUMP_TO_LABEL(error); - } - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *tuple_o = PySequence_Tuple(callargs_o); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (tuple_o == NULL) { - TRACING_JUMP_TO_LABEL(error); - } - _PyStackRef temp = callargs; - callargs = PyStackRef_FromPyObjectSteal(tuple_o); - stack_pointer[-2] = callargs; - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(temp); - stack_pointer = _PyFrame_GetStackPointer(frame); - } - } - // _DO_CALL_FUNCTION_EX - { - kwargs_st = stack_pointer[-1]; - callargs_st = callargs; - null = stack_pointer[-3]; - func_st = func; - (void)null; - PyObject *func = PyStackRef_AsPyObjectBorrow(func_st); - EVAL_CALL_STAT_INC_IF_FUNCTION(EVAL_CALL_FUNCTION_EX, func); - PyObject *result_o; - assert(!_PyErr_Occurred(tstate)); - if (opcode == INSTRUMENTED_CALL_FUNCTION_EX) { - PyObject *callargs = PyStackRef_AsPyObjectBorrow(callargs_st); - PyObject *kwargs = PyStackRef_AsPyObjectBorrow(kwargs_st); - assert(kwargs == NULL || PyDict_CheckExact(kwargs)); - assert(PyTuple_CheckExact(callargs)); - PyObject *arg = PyTuple_GET_SIZE(callargs) > 0 ? - PyTuple_GET_ITEM(callargs, 0) : &_PyInstrumentation_MISSING; - stack_pointer[-2] = callargs_st; - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = _Py_call_instrumentation_2args( - tstate, PY_MONITORING_EVENT_CALL, - frame, this_instr, func, arg); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err) { - TRACING_JUMP_TO_LABEL(error); - } - _PyFrame_SetStackPointer(frame, stack_pointer); - result_o = PyObject_Call(func, callargs, kwargs); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (!PyFunction_Check(func) && !PyMethod_Check(func)) { - if (result_o == NULL) { - _PyFrame_SetStackPointer(frame, stack_pointer); - _Py_call_instrumentation_exc2( - tstate, PY_MONITORING_EVENT_C_RAISE, - frame, this_instr, func, arg); - stack_pointer = _PyFrame_GetStackPointer(frame); - } - else { - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = _Py_call_instrumentation_2args( - tstate, PY_MONITORING_EVENT_C_RETURN, - frame, this_instr, func, arg); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err < 0) { - _PyFrame_SetStackPointer(frame, stack_pointer); - Py_CLEAR(result_o); - stack_pointer = _PyFrame_GetStackPointer(frame); - } - } - } - } - else { - if (Py_TYPE(func) == &PyFunction_Type && - tstate->interp->eval_frame == NULL && - ((PyFunctionObject *)func)->vectorcall == _PyFunction_Vectorcall) { - PyObject *callargs = PyStackRef_AsPyObjectSteal(callargs_st); - assert(PyTuple_CheckExact(callargs)); - PyObject *kwargs = PyStackRef_IsNull(kwargs_st) ? NULL : PyStackRef_AsPyObjectSteal(kwargs_st); - assert(kwargs == NULL || PyDict_CheckExact(kwargs)); - Py_ssize_t nargs = PyTuple_GET_SIZE(callargs); - int code_flags = ((PyCodeObject *)PyFunction_GET_CODE(func))->co_flags; - PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(func)); - stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyInterpreterFrame *new_frame = _PyEvalFramePushAndInit_Ex( - tstate, func_st, locals, - nargs, callargs, kwargs, frame); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); - if (new_frame == NULL) { - TRACING_JUMP_TO_LABEL(error); - } - assert( 1u == 1); - frame->return_offset = 1; - TRACING_DISPATCH_INLINED(new_frame); - } - PyObject *callargs = PyStackRef_AsPyObjectBorrow(callargs_st); - assert(PyTuple_CheckExact(callargs)); - PyObject *kwargs = PyStackRef_AsPyObjectBorrow(kwargs_st); - assert(kwargs == NULL || PyDict_CheckExact(kwargs)); - stack_pointer[-2] = callargs_st; - _PyFrame_SetStackPointer(frame, stack_pointer); - result_o = PyObject_Call(func, callargs, kwargs); - stack_pointer = _PyFrame_GetStackPointer(frame); - } - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_XCLOSE(kwargs_st); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(callargs_st); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(func_st); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (result_o == NULL) { - TRACING_JUMP_TO_LABEL(error); - } - result = PyStackRef_FromPyObjectSteal(result_o); - } - // _CHECK_PERIODIC_AT_END - { - stack_pointer[0] = result; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = check_periodics(tstate); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err != 0) { - TRACING_JUMP_TO_LABEL(error); - } - } - TRACING_DISPATCH(); - } - - TRACING_TARGET(INSTRUMENTED_CALL_KW) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = INSTRUMENTED_CALL_KW; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 4; - INSTRUCTION_STATS(INSTRUMENTED_CALL_KW); - opcode = INSTRUMENTED_CALL_KW; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - opcode = INSTRUMENTED_CALL_KW; - _PyStackRef callable; - _PyStackRef self_or_null; - _PyStackRef *args; - _PyStackRef kwnames; - _PyStackRef res; - /* Skip 1 cache entry */ - /* Skip 2 cache entries */ - // _MAYBE_EXPAND_METHOD_KW - { - self_or_null = stack_pointer[-2 - oparg]; - callable = stack_pointer[-3 - oparg]; - if (PyStackRef_TYPE(callable) == &PyMethod_Type && PyStackRef_IsNull(self_or_null)) { - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); - PyObject *self = ((PyMethodObject *)callable_o)->im_self; - self_or_null = PyStackRef_FromPyObjectNew(self); - PyObject *method = ((PyMethodObject *)callable_o)->im_func; - _PyStackRef temp = callable; - callable = PyStackRef_FromPyObjectNew(method); - stack_pointer[-3 - oparg] = callable; - stack_pointer[-2 - oparg] = self_or_null; - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(temp); - stack_pointer = _PyFrame_GetStackPointer(frame); - } - } - // _MONITOR_CALL_KW - { - args = &stack_pointer[-1 - oparg]; - int is_meth = !PyStackRef_IsNull(self_or_null); - PyObject *arg; - if (is_meth) { - arg = PyStackRef_AsPyObjectBorrow(self_or_null); - } - else if (args) { - arg = PyStackRef_AsPyObjectBorrow(args[0]); - } - else { - arg = &_PyInstrumentation_MISSING; - } - PyObject *function = PyStackRef_AsPyObjectBorrow(callable); - stack_pointer[-3 - oparg] = callable; - stack_pointer[-2 - oparg] = self_or_null; - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = _Py_call_instrumentation_2args( - tstate, PY_MONITORING_EVENT_CALL, - frame, this_instr, function, arg); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err) { - TRACING_JUMP_TO_LABEL(error); - } - } - // _DO_CALL_KW - { - kwnames = stack_pointer[-1]; - args = &stack_pointer[-1 - oparg]; - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); - PyObject *kwnames_o = PyStackRef_AsPyObjectBorrow(kwnames); - int total_args = oparg; - _PyStackRef *arguments = args; - if (!PyStackRef_IsNull(self_or_null)) { - arguments--; - total_args++; - } - int positional_args = total_args - (int)PyTuple_GET_SIZE(kwnames_o); - if (Py_TYPE(callable_o) == &PyFunction_Type && - tstate->interp->eval_frame == NULL && - ((PyFunctionObject *)callable_o)->vectorcall == _PyFunction_Vectorcall) - { - int code_flags = ((PyCodeObject*)PyFunction_GET_CODE(callable_o))->co_flags; - PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable_o)); - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyInterpreterFrame *new_frame = _PyEvalFramePushAndInit( - tstate, callable, locals, - arguments, positional_args, kwnames_o, frame - ); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -3 - oparg; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(kwnames); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (new_frame == NULL) { - TRACING_JUMP_TO_LABEL(error); - } - assert( 4u == 1 + INLINE_CACHE_ENTRIES_CALL_KW); - frame->return_offset = 4u ; - TRACING_DISPATCH_INLINED(new_frame); - } - STACKREFS_TO_PYOBJECTS(arguments, total_args, args_o); - if (CONVERSION_FAILED(args_o)) { - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyStackRef tmp = kwnames; - kwnames = PyStackRef_NULL; - stack_pointer[-1] = kwnames; - PyStackRef_CLOSE(tmp); - for (int _i = oparg; --_i >= 0;) { - tmp = args[_i]; - args[_i] = PyStackRef_NULL; - PyStackRef_CLOSE(tmp); - } - tmp = self_or_null; - self_or_null = PyStackRef_NULL; - stack_pointer[-2 - oparg] = self_or_null; - PyStackRef_XCLOSE(tmp); - tmp = callable; - callable = PyStackRef_NULL; - stack_pointer[-3 - oparg] = callable; - PyStackRef_CLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -3 - oparg; - assert(WITHIN_STACK_BOUNDS()); - TRACING_JUMP_TO_LABEL(error); - } - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *res_o = PyObject_Vectorcall( - callable_o, args_o, - positional_args | PY_VECTORCALL_ARGUMENTS_OFFSET, - kwnames_o); - stack_pointer = _PyFrame_GetStackPointer(frame); - STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); - if (opcode == INSTRUMENTED_CALL_KW) { - PyObject *arg = total_args == 0 ? - &_PyInstrumentation_MISSING : PyStackRef_AsPyObjectBorrow(arguments[0]); - if (res_o == NULL) { - _PyFrame_SetStackPointer(frame, stack_pointer); - _Py_call_instrumentation_exc2( - tstate, PY_MONITORING_EVENT_C_RAISE, - frame, this_instr, callable_o, arg); - stack_pointer = _PyFrame_GetStackPointer(frame); - } - else { - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = _Py_call_instrumentation_2args( - tstate, PY_MONITORING_EVENT_C_RETURN, - frame, this_instr, callable_o, arg); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err < 0) { - _PyFrame_SetStackPointer(frame, stack_pointer); - Py_CLEAR(res_o); - stack_pointer = _PyFrame_GetStackPointer(frame); - } - } - } - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyStackRef tmp = kwnames; - kwnames = PyStackRef_NULL; - stack_pointer[-1] = kwnames; - PyStackRef_CLOSE(tmp); - for (int _i = oparg; --_i >= 0;) { - tmp = args[_i]; - args[_i] = PyStackRef_NULL; - PyStackRef_CLOSE(tmp); - } - tmp = self_or_null; - self_or_null = PyStackRef_NULL; - stack_pointer[-2 - oparg] = self_or_null; - PyStackRef_XCLOSE(tmp); - tmp = callable; - callable = PyStackRef_NULL; - stack_pointer[-3 - oparg] = callable; - PyStackRef_CLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -3 - oparg; - assert(WITHIN_STACK_BOUNDS()); - if (res_o == NULL) { - TRACING_JUMP_TO_LABEL(error); - } - res = PyStackRef_FromPyObjectSteal(res_o); - } - stack_pointer[0] = res; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - TRACING_DISPATCH(); - } - - TRACING_TARGET(INSTRUMENTED_END_ASYNC_FOR) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = INSTRUMENTED_END_ASYNC_FOR; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(INSTRUMENTED_END_ASYNC_FOR); - opcode = INSTRUMENTED_END_ASYNC_FOR; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - _PyStackRef awaitable_st; - _PyStackRef exc_st; - // _MONITOR_END_ASYNC_FOR - { - assert((next_instr-oparg)->op.code == END_SEND || (next_instr-oparg)->op.code >= MIN_INSTRUMENTED_OPCODE); - INSTRUMENTED_JUMP(next_instr-oparg, this_instr+1, PY_MONITORING_EVENT_BRANCH_RIGHT); - } - // _END_ASYNC_FOR - { - exc_st = stack_pointer[-1]; - awaitable_st = stack_pointer[-2]; - JUMPBY(0); - (void)oparg; - PyObject *exc = PyStackRef_AsPyObjectBorrow(exc_st); - assert(exc && PyExceptionInstance_Check(exc)); - _PyFrame_SetStackPointer(frame, stack_pointer); - int matches = PyErr_GivenExceptionMatches(exc, PyExc_StopAsyncIteration); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (matches) { - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyStackRef tmp = exc_st; - exc_st = PyStackRef_NULL; - stack_pointer[-1] = exc_st; - PyStackRef_CLOSE(tmp); - tmp = awaitable_st; - awaitable_st = PyStackRef_NULL; - stack_pointer[-2] = awaitable_st; - PyStackRef_CLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); - } - else { - Py_INCREF(exc); - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyErr_SetRaisedException(tstate, exc); - monitor_reraise(tstate, frame, this_instr); - TRACING_JUMP_TO_LABEL(exception_unwind); - } - } - TRACING_DISPATCH(); - } - - TRACING_TARGET(INSTRUMENTED_END_FOR) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = INSTRUMENTED_END_FOR; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - next_instr += 1; - INSTRUCTION_STATS(INSTRUMENTED_END_FOR); - opcode = INSTRUMENTED_END_FOR; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - _PyStackRef receiver; - _PyStackRef value; - value = stack_pointer[-1]; - receiver = stack_pointer[-3]; - if (PyStackRef_GenCheck(receiver)) { - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = monitor_stop_iteration(tstate, frame, this_instr, PyStackRef_AsPyObjectBorrow(value)); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err) { - TRACING_JUMP_TO_LABEL(error); - } - } - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(value); - stack_pointer = _PyFrame_GetStackPointer(frame); - TRACING_DISPATCH(); - } - - TRACING_TARGET(INSTRUMENTED_END_SEND) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = INSTRUMENTED_END_SEND; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(INSTRUMENTED_END_SEND); - opcode = INSTRUMENTED_END_SEND; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - _PyStackRef receiver; - _PyStackRef value; - _PyStackRef val; - value = stack_pointer[-1]; - receiver = stack_pointer[-2]; - PyObject *receiver_o = PyStackRef_AsPyObjectBorrow(receiver); - if (PyGen_Check(receiver_o) || PyCoro_CheckExact(receiver_o)) { - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = monitor_stop_iteration(tstate, frame, this_instr, PyStackRef_AsPyObjectBorrow(value)); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err) { - TRACING_JUMP_TO_LABEL(error); - } - } - val = value; - stack_pointer[-2] = val; - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(receiver); - stack_pointer = _PyFrame_GetStackPointer(frame); - TRACING_DISPATCH(); - } - - TRACING_TARGET(INSTRUMENTED_FOR_ITER) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = INSTRUMENTED_FOR_ITER; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(INSTRUMENTED_FOR_ITER); - opcode = INSTRUMENTED_FOR_ITER; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - _PyStackRef iter; - _PyStackRef null_or_index; - _PyStackRef next; - /* Skip 1 cache entry */ - null_or_index = stack_pointer[-1]; - iter = stack_pointer[-2]; - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyStackRef item = _PyForIter_VirtualIteratorNext(tstate, frame, iter, &null_or_index); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (!PyStackRef_IsValid(item)) { - if (PyStackRef_IsError(item)) { - TRACING_JUMP_TO_LABEL(error); - } - JUMPBY(oparg + 1); - stack_pointer[-1] = null_or_index; - TRACING_DISPATCH(); - } - next = item; - INSTRUMENTED_JUMP(this_instr, next_instr, PY_MONITORING_EVENT_BRANCH_LEFT); - stack_pointer[-1] = null_or_index; - stack_pointer[0] = next; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - TRACING_DISPATCH(); - } - - TRACING_TARGET(INSTRUMENTED_INSTRUCTION) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = INSTRUMENTED_INSTRUCTION; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(INSTRUMENTED_INSTRUCTION); - opcode = INSTRUMENTED_INSTRUCTION; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - opcode = INSTRUMENTED_INSTRUCTION; - _PyFrame_SetStackPointer(frame, stack_pointer); - int next_opcode = _Py_call_instrumentation_instruction( - tstate, frame, this_instr); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (next_opcode < 0) { - TRACING_JUMP_TO_LABEL(error); - } - next_instr = this_instr; - if (_PyOpcode_Caches[next_opcode]) { - PAUSE_ADAPTIVE_COUNTER(next_instr[1].counter); - } - assert(next_opcode > 0 && next_opcode < 256); - opcode = next_opcode; - DISPATCH_GOTO(); - } - - TRACING_TARGET(INSTRUMENTED_JUMP_BACKWARD) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = INSTRUMENTED_JUMP_BACKWARD; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(INSTRUMENTED_JUMP_BACKWARD); - opcode = INSTRUMENTED_JUMP_BACKWARD; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - /* Skip 1 cache entry */ - // _CHECK_PERIODIC - { - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = check_periodics(tstate); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err != 0) { - TRACING_JUMP_TO_LABEL(error); - } - } - // _MONITOR_JUMP_BACKWARD - { - INSTRUMENTED_JUMP(this_instr, next_instr - oparg, PY_MONITORING_EVENT_JUMP); - } - TRACING_DISPATCH(); - } - - TRACING_TARGET(INSTRUMENTED_JUMP_FORWARD) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = INSTRUMENTED_JUMP_FORWARD; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(INSTRUMENTED_JUMP_FORWARD); - opcode = INSTRUMENTED_JUMP_FORWARD; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - INSTRUMENTED_JUMP(this_instr, next_instr + oparg, PY_MONITORING_EVENT_JUMP); - TRACING_DISPATCH(); - } - - TRACING_TARGET(INSTRUMENTED_LINE) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = INSTRUMENTED_LINE; - (void)(opcode); - #endif - _Py_CODEUNIT* const prev_instr = frame->instr_ptr; - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(INSTRUMENTED_LINE); - opcode = INSTRUMENTED_LINE; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - opcode = INSTRUMENTED_LINE; - int original_opcode = 0; - if (tstate->tracing) { - PyCodeObject *code = _PyFrame_GetCode(frame); - int index = (int)(this_instr - _PyFrame_GetBytecode(frame)); - original_opcode = code->_co_monitoring->lines->data[index*code->_co_monitoring->lines->bytes_per_entry]; - next_instr = this_instr; - } else { - _PyFrame_SetStackPointer(frame, stack_pointer); - original_opcode = _Py_call_instrumentation_line( - tstate, frame, this_instr, prev_instr); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (original_opcode < 0) { - next_instr = this_instr+1; - TRACING_JUMP_TO_LABEL(error); - } - next_instr = frame->instr_ptr; - if (next_instr != this_instr) { - TRACING_DISPATCH(); - } - } - if (_PyOpcode_Caches[original_opcode]) { - _PyBinaryOpCache *cache = (_PyBinaryOpCache *)(next_instr+1); - PAUSE_ADAPTIVE_COUNTER(cache->counter); - } - opcode = original_opcode; - DISPATCH_GOTO(); - } - - TRACING_TARGET(INSTRUMENTED_LOAD_SUPER_ATTR) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = INSTRUMENTED_LOAD_SUPER_ATTR; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(INSTRUMENTED_LOAD_SUPER_ATTR); - opcode = INSTRUMENTED_LOAD_SUPER_ATTR; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - opcode = INSTRUMENTED_LOAD_SUPER_ATTR; - _PyStackRef global_super_st; - _PyStackRef class_st; - _PyStackRef self_st; - _PyStackRef attr; - _PyStackRef *null; - /* Skip 1 cache entry */ - // _LOAD_SUPER_ATTR - { - self_st = stack_pointer[-1]; - class_st = stack_pointer[-2]; - global_super_st = stack_pointer[-3]; - PyObject *global_super = PyStackRef_AsPyObjectBorrow(global_super_st); - PyObject *class = PyStackRef_AsPyObjectBorrow(class_st); - PyObject *self = PyStackRef_AsPyObjectBorrow(self_st); - if (opcode == INSTRUMENTED_LOAD_SUPER_ATTR) { - PyObject *arg = oparg & 2 ? class : &_PyInstrumentation_MISSING; - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = _Py_call_instrumentation_2args( - tstate, PY_MONITORING_EVENT_CALL, - frame, this_instr, global_super, arg); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err) { - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyStackRef tmp = self_st; - self_st = PyStackRef_NULL; - stack_pointer[-1] = self_st; - PyStackRef_CLOSE(tmp); - tmp = class_st; - class_st = PyStackRef_NULL; - stack_pointer[-2] = class_st; - PyStackRef_CLOSE(tmp); - tmp = global_super_st; - global_super_st = PyStackRef_NULL; - stack_pointer[-3] = global_super_st; - PyStackRef_CLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -3; - assert(WITHIN_STACK_BOUNDS()); - TRACING_JUMP_TO_LABEL(error); - } - } - PyObject *stack[] = {class, self}; - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *super = PyObject_Vectorcall(global_super, stack, oparg & 2, NULL); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (opcode == INSTRUMENTED_LOAD_SUPER_ATTR) { - PyObject *arg = oparg & 2 ? class : &_PyInstrumentation_MISSING; - if (super == NULL) { - _PyFrame_SetStackPointer(frame, stack_pointer); - _Py_call_instrumentation_exc2( - tstate, PY_MONITORING_EVENT_C_RAISE, - frame, this_instr, global_super, arg); - stack_pointer = _PyFrame_GetStackPointer(frame); - } - else { - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = _Py_call_instrumentation_2args( - tstate, PY_MONITORING_EVENT_C_RETURN, - frame, this_instr, global_super, arg); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err < 0) { - _PyFrame_SetStackPointer(frame, stack_pointer); - Py_CLEAR(super); - stack_pointer = _PyFrame_GetStackPointer(frame); - } - } - } - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyStackRef tmp = self_st; - self_st = PyStackRef_NULL; - stack_pointer[-1] = self_st; - PyStackRef_CLOSE(tmp); - tmp = class_st; - class_st = PyStackRef_NULL; - stack_pointer[-2] = class_st; - PyStackRef_CLOSE(tmp); - tmp = global_super_st; - global_super_st = PyStackRef_NULL; - stack_pointer[-3] = global_super_st; - PyStackRef_CLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -3; - assert(WITHIN_STACK_BOUNDS()); - if (super == NULL) { - TRACING_JUMP_TO_LABEL(error); - } - PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 2); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *attr_o = PyObject_GetAttr(super, name); - Py_DECREF(super); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (attr_o == NULL) { - TRACING_JUMP_TO_LABEL(error); - } - attr = PyStackRef_FromPyObjectSteal(attr_o); - } - // _PUSH_NULL_CONDITIONAL - { - null = &stack_pointer[1]; - if (oparg & 1) { - null[0] = PyStackRef_NULL; - } - } - stack_pointer[0] = attr; - stack_pointer += 1 + (oparg & 1); - assert(WITHIN_STACK_BOUNDS()); - TRACING_DISPATCH(); - } - - TRACING_TARGET(INSTRUMENTED_NOT_TAKEN) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = INSTRUMENTED_NOT_TAKEN; - (void)(opcode); - #endif - _Py_CODEUNIT* const prev_instr = frame->instr_ptr; - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(INSTRUMENTED_NOT_TAKEN); - opcode = INSTRUMENTED_NOT_TAKEN; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - (void)this_instr; - INSTRUMENTED_JUMP(prev_instr, next_instr, PY_MONITORING_EVENT_BRANCH_LEFT); - TRACING_DISPATCH(); - } - - TRACING_TARGET(INSTRUMENTED_POP_ITER) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = INSTRUMENTED_POP_ITER; - (void)(opcode); - #endif - _Py_CODEUNIT* const prev_instr = frame->instr_ptr; - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(INSTRUMENTED_POP_ITER); - opcode = INSTRUMENTED_POP_ITER; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - _PyStackRef iter; - _PyStackRef index_or_null; - index_or_null = stack_pointer[-1]; - iter = stack_pointer[-2]; - (void)index_or_null; - INSTRUMENTED_JUMP(prev_instr, this_instr+1, PY_MONITORING_EVENT_BRANCH_RIGHT); - stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(iter); - stack_pointer = _PyFrame_GetStackPointer(frame); - TRACING_DISPATCH(); - } - - TRACING_TARGET(INSTRUMENTED_POP_JUMP_IF_FALSE) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = INSTRUMENTED_POP_JUMP_IF_FALSE; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(INSTRUMENTED_POP_JUMP_IF_FALSE); - opcode = INSTRUMENTED_POP_JUMP_IF_FALSE; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - _PyStackRef cond; - /* Skip 1 cache entry */ - cond = stack_pointer[-1]; - assert(PyStackRef_BoolCheck(cond)); - int jump = PyStackRef_IsFalse(cond); - RECORD_BRANCH_TAKEN(this_instr[1].cache, jump); - if (jump) { - INSTRUMENTED_JUMP(this_instr, next_instr + oparg, PY_MONITORING_EVENT_BRANCH_RIGHT); - } - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - TRACING_DISPATCH(); - } - - TRACING_TARGET(INSTRUMENTED_POP_JUMP_IF_NONE) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = INSTRUMENTED_POP_JUMP_IF_NONE; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(INSTRUMENTED_POP_JUMP_IF_NONE); - opcode = INSTRUMENTED_POP_JUMP_IF_NONE; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - _PyStackRef value; - /* Skip 1 cache entry */ - value = stack_pointer[-1]; - int jump = PyStackRef_IsNone(value); - RECORD_BRANCH_TAKEN(this_instr[1].cache, jump); - if (jump) { - INSTRUMENTED_JUMP(this_instr, next_instr + oparg, PY_MONITORING_EVENT_BRANCH_RIGHT); - } - else { - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(value); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += 1; - } - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - TRACING_DISPATCH(); - } - - TRACING_TARGET(INSTRUMENTED_POP_JUMP_IF_NOT_NONE) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = INSTRUMENTED_POP_JUMP_IF_NOT_NONE; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(INSTRUMENTED_POP_JUMP_IF_NOT_NONE); - opcode = INSTRUMENTED_POP_JUMP_IF_NOT_NONE; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - _PyStackRef value; - /* Skip 1 cache entry */ - value = stack_pointer[-1]; - int jump = !PyStackRef_IsNone(value); - RECORD_BRANCH_TAKEN(this_instr[1].cache, jump); - if (jump) { - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(value); - stack_pointer = _PyFrame_GetStackPointer(frame); - INSTRUMENTED_JUMP(this_instr, next_instr + oparg, PY_MONITORING_EVENT_BRANCH_RIGHT); - } - else { - stack_pointer += -1; - } - TRACING_DISPATCH(); - } - - TRACING_TARGET(INSTRUMENTED_POP_JUMP_IF_TRUE) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = INSTRUMENTED_POP_JUMP_IF_TRUE; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(INSTRUMENTED_POP_JUMP_IF_TRUE); - opcode = INSTRUMENTED_POP_JUMP_IF_TRUE; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - _PyStackRef cond; - /* Skip 1 cache entry */ - cond = stack_pointer[-1]; - assert(PyStackRef_BoolCheck(cond)); - int jump = PyStackRef_IsTrue(cond); - RECORD_BRANCH_TAKEN(this_instr[1].cache, jump); - if (jump) { - INSTRUMENTED_JUMP(this_instr, next_instr + oparg, PY_MONITORING_EVENT_BRANCH_RIGHT); - } - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - TRACING_DISPATCH(); - } - - TRACING_TARGET(INSTRUMENTED_RESUME) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = INSTRUMENTED_RESUME; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(INSTRUMENTED_RESUME); - opcode = INSTRUMENTED_RESUME; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - // _LOAD_BYTECODE - { - #ifdef Py_GIL_DISABLED - if (frame->tlbc_index != - ((_PyThreadStateImpl *)tstate)->tlbc_index) { - _PyFrame_SetStackPointer(frame, stack_pointer); - _Py_CODEUNIT *bytecode = - _PyEval_GetExecutableCode(tstate, _PyFrame_GetCode(frame)); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (bytecode == NULL) { - TRACING_JUMP_TO_LABEL(error); - } - ptrdiff_t off = this_instr - _PyFrame_GetBytecode(frame); - frame->tlbc_index = ((_PyThreadStateImpl *)tstate)->tlbc_index; - frame->instr_ptr = bytecode + off; - next_instr = frame->instr_ptr; - TRACING_DISPATCH(); - } - #endif - } - // _MAYBE_INSTRUMENT - { - #ifdef Py_GIL_DISABLED - - int check_instrumentation = 1; - #else - int check_instrumentation = (tstate->tracing == 0); - #endif - if (check_instrumentation) { - uintptr_t global_version = _Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & ~_PY_EVAL_EVENTS_MASK; - uintptr_t code_version = FT_ATOMIC_LOAD_UINTPTR_ACQUIRE(_PyFrame_GetCode(frame)->_co_instrumentation_version); - if (code_version != global_version) { - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = _Py_Instrument(_PyFrame_GetCode(frame), tstate->interp); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err) { - TRACING_JUMP_TO_LABEL(error); - } - next_instr = this_instr; - TRACING_DISPATCH(); - } - } - } - // _CHECK_PERIODIC_IF_NOT_YIELD_FROM - { - if ((oparg & RESUME_OPARG_LOCATION_MASK) < RESUME_AFTER_YIELD_FROM) { - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = check_periodics(tstate); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err != 0) { - TRACING_JUMP_TO_LABEL(error); - } - } - } - // _MONITOR_RESUME - { - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = _Py_call_instrumentation( - tstate, oparg > 0, frame, this_instr); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err) { - TRACING_JUMP_TO_LABEL(error); - } - if (frame->instr_ptr != this_instr) { - next_instr = frame->instr_ptr; - } - } - TRACING_DISPATCH(); - } - - TRACING_TARGET(INSTRUMENTED_RETURN_VALUE) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = INSTRUMENTED_RETURN_VALUE; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(INSTRUMENTED_RETURN_VALUE); - opcode = INSTRUMENTED_RETURN_VALUE; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - _PyStackRef val; - _PyStackRef retval; - _PyStackRef res; - // _RETURN_VALUE_EVENT - { - val = stack_pointer[-1]; - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = _Py_call_instrumentation_arg( - tstate, PY_MONITORING_EVENT_PY_RETURN, - frame, this_instr, PyStackRef_AsPyObjectBorrow(val)); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err) { - TRACING_JUMP_TO_LABEL(error); - } - } - // _RETURN_VALUE - { - retval = val; - assert(frame->owner != FRAME_OWNED_BY_INTERPRETER); - _PyStackRef temp = PyStackRef_MakeHeapSafe(retval); - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - assert(STACK_LEVEL() == 0); - _Py_LeaveRecursiveCallPy(tstate); - _PyInterpreterFrame *dying = frame; - frame = tstate->current_frame = dying->previous; - _PyEval_FrameClearAndPop(tstate, dying); - stack_pointer = _PyFrame_GetStackPointer(frame); - LOAD_IP(frame->return_offset); - #if TIER_TWO - frame->instr_ptr += frame->return_offset; - #endif - res = temp; - LLTRACE_RESUME_FRAME(); - } - stack_pointer[0] = res; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - TRACING_DISPATCH(); - } - - TRACING_TARGET(INSTRUMENTED_YIELD_VALUE) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = INSTRUMENTED_YIELD_VALUE; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(INSTRUMENTED_YIELD_VALUE); - opcode = INSTRUMENTED_YIELD_VALUE; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - _PyStackRef val; - _PyStackRef retval; - _PyStackRef value; - // _YIELD_VALUE_EVENT - { - val = stack_pointer[-1]; - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = _Py_call_instrumentation_arg( - tstate, PY_MONITORING_EVENT_PY_YIELD, - frame, this_instr, PyStackRef_AsPyObjectBorrow(val)); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err) { - TRACING_JUMP_TO_LABEL(error); - } - if (frame->instr_ptr != this_instr) { - next_instr = frame->instr_ptr; - TRACING_DISPATCH(); - } - } - // _YIELD_VALUE - { - retval = val; - assert(frame->owner != FRAME_OWNED_BY_INTERPRETER); - frame->instr_ptr++; - PyGenObject *gen = _PyGen_GetGeneratorFromFrame(frame); - assert(FRAME_SUSPENDED_YIELD_FROM == FRAME_SUSPENDED + 1); - assert(oparg == 0 || oparg == 1); - gen->gi_frame_state = FRAME_SUSPENDED + oparg; - _PyStackRef temp = retval; - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - tstate->exc_info = gen->gi_exc_state.previous_item; - gen->gi_exc_state.previous_item = NULL; - _Py_LeaveRecursiveCallPy(tstate); - _PyInterpreterFrame *gen_frame = frame; - frame = tstate->current_frame = frame->previous; - gen_frame->previous = NULL; - assert(INLINE_CACHE_ENTRIES_SEND == INLINE_CACHE_ENTRIES_FOR_ITER); - #if TIER_ONE - assert(frame->instr_ptr->op.code == INSTRUMENTED_LINE || - frame->instr_ptr->op.code == INSTRUMENTED_INSTRUCTION || - _PyOpcode_Deopt[frame->instr_ptr->op.code] == SEND || - _PyOpcode_Deopt[frame->instr_ptr->op.code] == FOR_ITER || - _PyOpcode_Deopt[frame->instr_ptr->op.code] == INTERPRETER_EXIT || - _PyOpcode_Deopt[frame->instr_ptr->op.code] == ENTER_EXECUTOR); - #endif - stack_pointer = _PyFrame_GetStackPointer(frame); - LOAD_IP(1 + INLINE_CACHE_ENTRIES_SEND); - #if TIER_TWO - frame->instr_ptr += (1 + INLINE_CACHE_ENTRIES_SEND); - #endif - value = PyStackRef_MakeHeapSafe(temp); - LLTRACE_RESUME_FRAME(); - } - stack_pointer[0] = value; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - TRACING_DISPATCH(); - } - - TRACING_TARGET(INTERPRETER_EXIT) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = INTERPRETER_EXIT; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(INTERPRETER_EXIT); - opcode = INTERPRETER_EXIT; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - _PyStackRef retval; - retval = stack_pointer[-1]; - assert(frame->owner == FRAME_OWNED_BY_INTERPRETER); - assert(_PyFrame_IsIncomplete(frame)); - tstate->current_frame = frame->previous; - assert(!_PyErr_Occurred(tstate)); - PyObject *result = PyStackRef_AsPyObjectSteal(retval); - #if !_Py_TAIL_CALL_INTERP - assert(frame == &entry.frame); - #endif - #ifdef _Py_TIER2 - _PyStackRef executor = frame->localsplus[0]; - assert(tstate->current_executor == NULL); - if (!PyStackRef_IsNull(executor)) { - tstate->current_executor = PyStackRef_AsPyObjectBorrow(executor); - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(executor); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += 1; - } - #endif - LLTRACE_RESUME_FRAME(); - return result; - } - - TRACING_TARGET(IS_OP) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = IS_OP; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(IS_OP); - opcode = IS_OP; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - _PyStackRef left; - _PyStackRef right; - _PyStackRef b; - right = stack_pointer[-1]; - left = stack_pointer[-2]; - int res = Py_Is(PyStackRef_AsPyObjectBorrow(left), PyStackRef_AsPyObjectBorrow(right)) ^ oparg; - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyStackRef tmp = right; - right = PyStackRef_NULL; - stack_pointer[-1] = right; - PyStackRef_CLOSE(tmp); - tmp = left; - left = PyStackRef_NULL; - stack_pointer[-2] = left; - PyStackRef_CLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); - b = res ? PyStackRef_True : PyStackRef_False; - stack_pointer[0] = b; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - TRACING_DISPATCH(); - } - - TRACING_TARGET(JUMP_BACKWARD) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = JUMP_BACKWARD; - (void)(opcode); - #endif - frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(JUMP_BACKWARD); - PREDICTED_TRACING_JUMP_BACKWARD:; - _Py_CODEUNIT* const this_instr = next_instr - 2; - (void)this_instr; - opcode = JUMP_BACKWARD; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - /* Skip 1 cache entry */ - // _SPECIALIZE_JUMP_BACKWARD - { - #if ENABLE_SPECIALIZATION - if (this_instr->op.code == JUMP_BACKWARD) { - this_instr->op.code = (tstate->interp->jit && !PyStackRef_IsNull(frame->f_funcobj)) ? JUMP_BACKWARD_JIT : JUMP_BACKWARD_NO_JIT; - next_instr = this_instr; - DISPATCH_SAME_OPARG(); - } - #endif - } - // _CHECK_PERIODIC - { - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = check_periodics(tstate); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err != 0) { - TRACING_JUMP_TO_LABEL(error); - } - } - // _JUMP_BACKWARD_NO_INTERRUPT - { - #if TIER_ONE - assert(oparg <= INSTR_OFFSET()); - #endif - JUMPBY(-oparg); - } - TRACING_DISPATCH(); - } - - TRACING_TARGET(JUMP_BACKWARD_JIT) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = JUMP_BACKWARD_JIT; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(JUMP_BACKWARD_JIT); - opcode = JUMP_BACKWARD_JIT; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - static_assert(1 == 1, "incorrect cache size"); - /* Skip 1 cache entry */ - // _CHECK_PERIODIC - { - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = check_periodics(tstate); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err != 0) { - TRACING_JUMP_TO_LABEL(error); - } - } - // _JUMP_BACKWARD_NO_INTERRUPT - { - #if TIER_ONE - assert(oparg <= INSTR_OFFSET()); - #endif - JUMPBY(-oparg); - } - // _JIT - { - #ifdef _Py_TIER2 - _Py_BackoffCounter counter = this_instr[1].counter; - if (!IS_JIT_TRACING() && backoff_counter_triggers(counter) && - this_instr->op.code == JUMP_BACKWARD_JIT && - next_instr->op.code != ENTER_EXECUTOR) { - _Py_CODEUNIT *start = this_instr; - int curr_oparg = oparg; - while (curr_oparg > 255) { - curr_oparg >>= 8; - start--; - } - if (tstate->interp->jit_tracer_code_buffer == NULL) { - _PyFrame_SetStackPointer(frame, stack_pointer); - tstate->interp->jit_tracer_code_buffer = (_PyUOpInstruction *)_PyObject_VirtualAlloc(UOP_BUFFER_SIZE); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (tstate->interp->jit_tracer_code_buffer == NULL) { - TRACING_DISPATCH(); - } - } - _PyJIT_InitializeTracing(tstate, frame, this_instr, STACK_LEVEL(), 0, NULL); - ENTER_TRACING(); - TRACING_DISPATCH(); - } - else { - ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); - } - #endif - } - TRACING_DISPATCH(); - } - - TRACING_TARGET(JUMP_BACKWARD_NO_INTERRUPT) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = JUMP_BACKWARD_NO_INTERRUPT; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(JUMP_BACKWARD_NO_INTERRUPT); - opcode = JUMP_BACKWARD_NO_INTERRUPT; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - #if TIER_ONE - assert(oparg <= INSTR_OFFSET()); - #endif - JUMPBY(-oparg); - TRACING_DISPATCH(); - } - - TRACING_TARGET(JUMP_BACKWARD_NO_JIT) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = JUMP_BACKWARD_NO_JIT; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(JUMP_BACKWARD_NO_JIT); - opcode = JUMP_BACKWARD_NO_JIT; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - static_assert(1 == 1, "incorrect cache size"); - /* Skip 1 cache entry */ - // _CHECK_PERIODIC - { - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = check_periodics(tstate); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err != 0) { - TRACING_JUMP_TO_LABEL(error); - } - } - // _JUMP_BACKWARD_NO_INTERRUPT - { - #if TIER_ONE - assert(oparg <= INSTR_OFFSET()); - #endif - JUMPBY(-oparg); - } - TRACING_DISPATCH(); - } - - TRACING_TARGET(JUMP_FORWARD) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = JUMP_FORWARD; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(JUMP_FORWARD); - opcode = JUMP_FORWARD; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - JUMPBY(oparg); - TRACING_DISPATCH(); - } - - TRACING_TARGET(LIST_APPEND) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = LIST_APPEND; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(LIST_APPEND); - opcode = LIST_APPEND; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - _PyStackRef list; - _PyStackRef v; - v = stack_pointer[-1]; - list = stack_pointer[-2 - (oparg-1)]; - int err = _PyList_AppendTakeRef((PyListObject *)PyStackRef_AsPyObjectBorrow(list), - PyStackRef_AsPyObjectSteal(v)); - if (err < 0) { - TRACING_JUMP_TO_LABEL(pop_1_error); - } - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - TRACING_DISPATCH(); - } - - TRACING_TARGET(LIST_EXTEND) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = LIST_EXTEND; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(LIST_EXTEND); - opcode = LIST_EXTEND; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - _PyStackRef list_st; - _PyStackRef iterable_st; - iterable_st = stack_pointer[-1]; - list_st = stack_pointer[-2 - (oparg-1)]; - PyObject *list = PyStackRef_AsPyObjectBorrow(list_st); - PyObject *iterable = PyStackRef_AsPyObjectBorrow(iterable_st); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *none_val = _PyList_Extend((PyListObject *)list, iterable); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (none_val == NULL) { - _PyFrame_SetStackPointer(frame, stack_pointer); - int matches = _PyErr_ExceptionMatches(tstate, PyExc_TypeError); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (matches && - (Py_TYPE(iterable)->tp_iter == NULL && !PySequence_Check(iterable))) - { - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyErr_Clear(tstate); - _PyErr_Format(tstate, PyExc_TypeError, - "Value after * must be an iterable, not %.200s", - Py_TYPE(iterable)->tp_name); - stack_pointer = _PyFrame_GetStackPointer(frame); - } - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(iterable_st); - stack_pointer = _PyFrame_GetStackPointer(frame); - TRACING_JUMP_TO_LABEL(error); - } - assert(Py_IsNone(none_val)); - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(iterable_st); - stack_pointer = _PyFrame_GetStackPointer(frame); - TRACING_DISPATCH(); - } - - TRACING_TARGET(LOAD_ATTR) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = LOAD_ATTR; - (void)(opcode); - #endif - frame->instr_ptr = next_instr; - next_instr += 10; - INSTRUCTION_STATS(LOAD_ATTR); - PREDICTED_TRACING_LOAD_ATTR:; - _Py_CODEUNIT* const this_instr = next_instr - 10; - (void)this_instr; - opcode = LOAD_ATTR; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - _PyStackRef owner; - _PyStackRef *attr; - _PyStackRef *self_or_null; - // _SPECIALIZE_LOAD_ATTR - { - owner = stack_pointer[-1]; - uint16_t counter = read_u16(&this_instr[1].cache); - (void)counter; - #if ENABLE_SPECIALIZATION_FT - if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { - PyObject *name = GETITEM(FRAME_CO_NAMES, oparg>>1); - next_instr = this_instr; - _PyFrame_SetStackPointer(frame, stack_pointer); - _Py_Specialize_LoadAttr(owner, next_instr, name); - stack_pointer = _PyFrame_GetStackPointer(frame); - DISPATCH_SAME_OPARG(); - } - OPCODE_DEFERRED_INC(LOAD_ATTR); - ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); - #endif /* ENABLE_SPECIALIZATION_FT */ - } - /* Skip 8 cache entries */ - // _LOAD_ATTR - { - attr = &stack_pointer[-1]; - self_or_null = &stack_pointer[0]; - PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 1); - if (oparg & 1) { - *attr = PyStackRef_NULL; - _PyFrame_SetStackPointer(frame, stack_pointer); - int is_meth = _PyObject_GetMethodStackRef(tstate, PyStackRef_AsPyObjectBorrow(owner), name, attr); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (is_meth) { - assert(!PyStackRef_IsNull(*attr)); - self_or_null[0] = owner; - } - else { - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(owner); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (PyStackRef_IsNull(*attr)) { - TRACING_JUMP_TO_LABEL(error); - } - self_or_null[0] = PyStackRef_NULL; - stack_pointer += 1; - } - } - else { - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *attr_o = PyObject_GetAttr(PyStackRef_AsPyObjectBorrow(owner), name); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(owner); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (attr_o == NULL) { - TRACING_JUMP_TO_LABEL(error); - } - *attr = PyStackRef_FromPyObjectSteal(attr_o); - stack_pointer += 1; - } - } - stack_pointer += (oparg&1); - assert(WITHIN_STACK_BOUNDS()); - TRACING_DISPATCH(); - } - - TRACING_TARGET(LOAD_ATTR_CLASS) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = LOAD_ATTR_CLASS; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 10; - INSTRUCTION_STATS(LOAD_ATTR_CLASS); - opcode = LOAD_ATTR_CLASS; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); - _PyStackRef owner; - _PyStackRef attr; - _PyStackRef *null; - /* Skip 1 cache entry */ - // _CHECK_ATTR_CLASS - { - owner = stack_pointer[-1]; - uint32_t type_version = read_u32(&this_instr[2].cache); - PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); - if (!PyType_Check(owner_o)) { - UPDATE_MISS_STATS(LOAD_ATTR); - assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); - JUMP_TO_PREDICTED(TRACING_LOAD_ATTR); - } - assert(type_version != 0); - if (FT_ATOMIC_LOAD_UINT_RELAXED(((PyTypeObject *)owner_o)->tp_version_tag) != type_version) { - UPDATE_MISS_STATS(LOAD_ATTR); - assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); - JUMP_TO_PREDICTED(TRACING_LOAD_ATTR); - } - } - /* Skip 2 cache entries */ - // _LOAD_ATTR_CLASS - { - PyObject *descr = read_obj(&this_instr[6].cache); - STAT_INC(LOAD_ATTR, hit); - assert(descr != NULL); - attr = PyStackRef_FromPyObjectNew(descr); - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyStackRef tmp = owner; - owner = attr; - stack_pointer[-1] = owner; - PyStackRef_CLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); - } - // _PUSH_NULL_CONDITIONAL - { - null = &stack_pointer[0]; - if (oparg & 1) { - null[0] = PyStackRef_NULL; - } - } - stack_pointer += (oparg & 1); - assert(WITHIN_STACK_BOUNDS()); - TRACING_DISPATCH(); - } - - TRACING_TARGET(LOAD_ATTR_CLASS_WITH_METACLASS_CHECK) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = LOAD_ATTR_CLASS_WITH_METACLASS_CHECK; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 10; - INSTRUCTION_STATS(LOAD_ATTR_CLASS_WITH_METACLASS_CHECK); - opcode = LOAD_ATTR_CLASS_WITH_METACLASS_CHECK; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); - _PyStackRef owner; - _PyStackRef attr; - _PyStackRef *null; - /* Skip 1 cache entry */ - // _CHECK_ATTR_CLASS - { - owner = stack_pointer[-1]; - uint32_t type_version = read_u32(&this_instr[2].cache); - PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); - if (!PyType_Check(owner_o)) { - UPDATE_MISS_STATS(LOAD_ATTR); - assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); - JUMP_TO_PREDICTED(TRACING_LOAD_ATTR); - } - assert(type_version != 0); - if (FT_ATOMIC_LOAD_UINT_RELAXED(((PyTypeObject *)owner_o)->tp_version_tag) != type_version) { - UPDATE_MISS_STATS(LOAD_ATTR); - assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); - JUMP_TO_PREDICTED(TRACING_LOAD_ATTR); - } - } - // _GUARD_TYPE_VERSION - { - uint32_t type_version = read_u32(&this_instr[4].cache); - PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); - assert(type_version != 0); - if (FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version) { - UPDATE_MISS_STATS(LOAD_ATTR); - assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); - JUMP_TO_PREDICTED(TRACING_LOAD_ATTR); - } - } - // _LOAD_ATTR_CLASS - { - PyObject *descr = read_obj(&this_instr[6].cache); - STAT_INC(LOAD_ATTR, hit); - assert(descr != NULL); - attr = PyStackRef_FromPyObjectNew(descr); - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyStackRef tmp = owner; - owner = attr; - stack_pointer[-1] = owner; - PyStackRef_CLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); - } - // _PUSH_NULL_CONDITIONAL - { - null = &stack_pointer[0]; - if (oparg & 1) { - null[0] = PyStackRef_NULL; - } - } - stack_pointer += (oparg & 1); - assert(WITHIN_STACK_BOUNDS()); - TRACING_DISPATCH(); - } - - TRACING_TARGET(LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 10; - INSTRUCTION_STATS(LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN); - opcode = LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); - _PyStackRef owner; - /* Skip 1 cache entry */ - owner = stack_pointer[-1]; - uint32_t type_version = read_u32(&this_instr[2].cache); - uint32_t func_version = read_u32(&this_instr[4].cache); - PyObject *getattribute = read_obj(&this_instr[6].cache); - PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); - assert((oparg & 1) == 0); - if (tstate->interp->eval_frame) { - UPDATE_MISS_STATS(LOAD_ATTR); - assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); - JUMP_TO_PREDICTED(TRACING_LOAD_ATTR); - } - PyTypeObject *cls = Py_TYPE(owner_o); - assert(type_version != 0); - if (FT_ATOMIC_LOAD_UINT_RELAXED(cls->tp_version_tag) != type_version) { - UPDATE_MISS_STATS(LOAD_ATTR); - assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); - JUMP_TO_PREDICTED(TRACING_LOAD_ATTR); - } - assert(Py_IS_TYPE(getattribute, &PyFunction_Type)); - PyFunctionObject *f = (PyFunctionObject *)getattribute; - assert(func_version != 0); - if (f->func_version != func_version) { - UPDATE_MISS_STATS(LOAD_ATTR); - assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); - JUMP_TO_PREDICTED(TRACING_LOAD_ATTR); - } - PyCodeObject *code = (PyCodeObject *)f->func_code; - assert(code->co_argcount == 2); - if (!_PyThreadState_HasStackSpace(tstate, code->co_framesize)) { - UPDATE_MISS_STATS(LOAD_ATTR); - assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); - JUMP_TO_PREDICTED(TRACING_LOAD_ATTR); - } - STAT_INC(LOAD_ATTR, hit); - PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 1); - _PyInterpreterFrame *new_frame = _PyFrame_PushUnchecked( - tstate, PyStackRef_FromPyObjectNew(f), 2, frame); - new_frame->localsplus[0] = owner; - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - new_frame->localsplus[1] = PyStackRef_FromPyObjectNew(name); - frame->return_offset = 10u ; - TRACING_DISPATCH_INLINED(new_frame); - } - - TRACING_TARGET(LOAD_ATTR_INSTANCE_VALUE) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = LOAD_ATTR_INSTANCE_VALUE; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 10; - INSTRUCTION_STATS(LOAD_ATTR_INSTANCE_VALUE); - opcode = LOAD_ATTR_INSTANCE_VALUE; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); - _PyStackRef owner; - _PyStackRef attr; - _PyStackRef *null; - /* Skip 1 cache entry */ - // _GUARD_TYPE_VERSION - { - owner = stack_pointer[-1]; - uint32_t type_version = read_u32(&this_instr[2].cache); - PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); - assert(type_version != 0); - if (FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version) { - UPDATE_MISS_STATS(LOAD_ATTR); - assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); - JUMP_TO_PREDICTED(TRACING_LOAD_ATTR); - } - } - // _CHECK_MANAGED_OBJECT_HAS_VALUES - { - PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); - assert(Py_TYPE(owner_o)->tp_dictoffset < 0); - assert(Py_TYPE(owner_o)->tp_flags & Py_TPFLAGS_INLINE_VALUES); - if (!FT_ATOMIC_LOAD_UINT8(_PyObject_InlineValues(owner_o)->valid)) { - UPDATE_MISS_STATS(LOAD_ATTR); - assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); - JUMP_TO_PREDICTED(TRACING_LOAD_ATTR); - } - } - // _LOAD_ATTR_INSTANCE_VALUE - { - uint16_t offset = read_u16(&this_instr[4].cache); - PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); - PyObject **value_ptr = (PyObject**)(((char *)owner_o) + offset); - PyObject *attr_o = FT_ATOMIC_LOAD_PTR_ACQUIRE(*value_ptr); - if (attr_o == NULL) { - UPDATE_MISS_STATS(LOAD_ATTR); - assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); - JUMP_TO_PREDICTED(TRACING_LOAD_ATTR); - } - #ifdef Py_GIL_DISABLED - int increfed = _Py_TryIncrefCompareStackRef(value_ptr, attr_o, &attr); - if (!increfed) { - if (true) { - UPDATE_MISS_STATS(LOAD_ATTR); - assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); - JUMP_TO_PREDICTED(TRACING_LOAD_ATTR); - } - } - #else - attr = PyStackRef_FromPyObjectNew(attr_o); - #endif - STAT_INC(LOAD_ATTR, hit); - stack_pointer[-1] = attr; - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(owner); - stack_pointer = _PyFrame_GetStackPointer(frame); - } - /* Skip 5 cache entries */ - // _PUSH_NULL_CONDITIONAL - { - null = &stack_pointer[0]; - if (oparg & 1) { - null[0] = PyStackRef_NULL; - } - } - stack_pointer += (oparg & 1); - assert(WITHIN_STACK_BOUNDS()); - TRACING_DISPATCH(); - } - - TRACING_TARGET(LOAD_ATTR_METHOD_LAZY_DICT) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = LOAD_ATTR_METHOD_LAZY_DICT; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 10; - INSTRUCTION_STATS(LOAD_ATTR_METHOD_LAZY_DICT); - opcode = LOAD_ATTR_METHOD_LAZY_DICT; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); - _PyStackRef owner; - _PyStackRef attr; - _PyStackRef self; - /* Skip 1 cache entry */ - // _GUARD_TYPE_VERSION - { - owner = stack_pointer[-1]; - uint32_t type_version = read_u32(&this_instr[2].cache); - PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); - assert(type_version != 0); - if (FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version) { - UPDATE_MISS_STATS(LOAD_ATTR); - assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); - JUMP_TO_PREDICTED(TRACING_LOAD_ATTR); - } - } - // _CHECK_ATTR_METHOD_LAZY_DICT - { - uint16_t dictoffset = read_u16(&this_instr[4].cache); - char *ptr = ((char *)PyStackRef_AsPyObjectBorrow(owner)) + MANAGED_DICT_OFFSET + dictoffset; - PyObject *dict = FT_ATOMIC_LOAD_PTR_ACQUIRE(*(PyObject **)ptr); - if (dict != NULL) { - UPDATE_MISS_STATS(LOAD_ATTR); - assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); - JUMP_TO_PREDICTED(TRACING_LOAD_ATTR); - } - } - /* Skip 1 cache entry */ - // _LOAD_ATTR_METHOD_LAZY_DICT - { - PyObject *descr = read_obj(&this_instr[6].cache); - assert(oparg & 1); - STAT_INC(LOAD_ATTR, hit); - assert(descr != NULL); - assert(_PyType_HasFeature(Py_TYPE(descr), Py_TPFLAGS_METHOD_DESCRIPTOR)); - attr = PyStackRef_FromPyObjectNew(descr); - self = owner; - } - stack_pointer[-1] = attr; - stack_pointer[0] = self; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - TRACING_DISPATCH(); - } - - TRACING_TARGET(LOAD_ATTR_METHOD_NO_DICT) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = LOAD_ATTR_METHOD_NO_DICT; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 10; - INSTRUCTION_STATS(LOAD_ATTR_METHOD_NO_DICT); - opcode = LOAD_ATTR_METHOD_NO_DICT; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); - _PyStackRef owner; - _PyStackRef attr; - _PyStackRef self; - /* Skip 1 cache entry */ - // _GUARD_TYPE_VERSION - { - owner = stack_pointer[-1]; - uint32_t type_version = read_u32(&this_instr[2].cache); - PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); - assert(type_version != 0); - if (FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version) { - UPDATE_MISS_STATS(LOAD_ATTR); - assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); - JUMP_TO_PREDICTED(TRACING_LOAD_ATTR); - } - } - /* Skip 2 cache entries */ - // _LOAD_ATTR_METHOD_NO_DICT - { - PyObject *descr = read_obj(&this_instr[6].cache); - assert(oparg & 1); - assert(Py_TYPE(PyStackRef_AsPyObjectBorrow(owner))->tp_dictoffset == 0); - STAT_INC(LOAD_ATTR, hit); - assert(descr != NULL); - assert(_PyType_HasFeature(Py_TYPE(descr), Py_TPFLAGS_METHOD_DESCRIPTOR)); - attr = PyStackRef_FromPyObjectNew(descr); - self = owner; - } - stack_pointer[-1] = attr; - stack_pointer[0] = self; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - TRACING_DISPATCH(); - } - - TRACING_TARGET(LOAD_ATTR_METHOD_WITH_VALUES) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = LOAD_ATTR_METHOD_WITH_VALUES; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 10; - INSTRUCTION_STATS(LOAD_ATTR_METHOD_WITH_VALUES); - opcode = LOAD_ATTR_METHOD_WITH_VALUES; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); - _PyStackRef owner; - _PyStackRef attr; - _PyStackRef self; - /* Skip 1 cache entry */ - // _GUARD_TYPE_VERSION - { - owner = stack_pointer[-1]; - uint32_t type_version = read_u32(&this_instr[2].cache); - PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); - assert(type_version != 0); - if (FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version) { - UPDATE_MISS_STATS(LOAD_ATTR); - assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); - JUMP_TO_PREDICTED(TRACING_LOAD_ATTR); - } - } - // _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT - { - PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); - assert(Py_TYPE(owner_o)->tp_flags & Py_TPFLAGS_INLINE_VALUES); - PyDictValues *ivs = _PyObject_InlineValues(owner_o); - if (!FT_ATOMIC_LOAD_UINT8(ivs->valid)) { - UPDATE_MISS_STATS(LOAD_ATTR); - assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); - JUMP_TO_PREDICTED(TRACING_LOAD_ATTR); - } - } - // _GUARD_KEYS_VERSION - { - uint32_t keys_version = read_u32(&this_instr[4].cache); - PyTypeObject *owner_cls = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); - PyHeapTypeObject *owner_heap_type = (PyHeapTypeObject *)owner_cls; - PyDictKeysObject *keys = owner_heap_type->ht_cached_keys; - if (FT_ATOMIC_LOAD_UINT32_RELAXED(keys->dk_version) != keys_version) { - UPDATE_MISS_STATS(LOAD_ATTR); - assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); - JUMP_TO_PREDICTED(TRACING_LOAD_ATTR); - } - } - // _LOAD_ATTR_METHOD_WITH_VALUES - { - PyObject *descr = read_obj(&this_instr[6].cache); - assert(oparg & 1); - STAT_INC(LOAD_ATTR, hit); - assert(descr != NULL); - assert(_PyType_HasFeature(Py_TYPE(descr), Py_TPFLAGS_METHOD_DESCRIPTOR)); - attr = PyStackRef_FromPyObjectNew(descr); - self = owner; - } - stack_pointer[-1] = attr; - stack_pointer[0] = self; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - TRACING_DISPATCH(); - } - - TRACING_TARGET(LOAD_ATTR_MODULE) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = LOAD_ATTR_MODULE; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 10; - INSTRUCTION_STATS(LOAD_ATTR_MODULE); - opcode = LOAD_ATTR_MODULE; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); - _PyStackRef owner; - _PyStackRef attr; - _PyStackRef *null; - /* Skip 1 cache entry */ - // _LOAD_ATTR_MODULE - { - owner = stack_pointer[-1]; - uint32_t dict_version = read_u32(&this_instr[2].cache); - uint16_t index = read_u16(&this_instr[4].cache); - PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); - if (Py_TYPE(owner_o)->tp_getattro != PyModule_Type.tp_getattro) { - UPDATE_MISS_STATS(LOAD_ATTR); - assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); - JUMP_TO_PREDICTED(TRACING_LOAD_ATTR); - } - PyDictObject *dict = (PyDictObject *)((PyModuleObject *)owner_o)->md_dict; - assert(dict != NULL); - PyDictKeysObject *keys = FT_ATOMIC_LOAD_PTR_ACQUIRE(dict->ma_keys); - if (FT_ATOMIC_LOAD_UINT32_RELAXED(keys->dk_version) != dict_version) { - UPDATE_MISS_STATS(LOAD_ATTR); - assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); - JUMP_TO_PREDICTED(TRACING_LOAD_ATTR); - } - assert(keys->dk_kind == DICT_KEYS_UNICODE); - assert(index < FT_ATOMIC_LOAD_SSIZE_RELAXED(keys->dk_nentries)); - PyDictUnicodeEntry *ep = DK_UNICODE_ENTRIES(keys) + index; - PyObject *attr_o = FT_ATOMIC_LOAD_PTR_RELAXED(ep->me_value); - if (attr_o == NULL) { - UPDATE_MISS_STATS(LOAD_ATTR); - assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); - JUMP_TO_PREDICTED(TRACING_LOAD_ATTR); - } - #ifdef Py_GIL_DISABLED - int increfed = _Py_TryIncrefCompareStackRef(&ep->me_value, attr_o, &attr); - if (!increfed) { - if (true) { - UPDATE_MISS_STATS(LOAD_ATTR); - assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); - JUMP_TO_PREDICTED(TRACING_LOAD_ATTR); - } - } - #else - attr = PyStackRef_FromPyObjectNew(attr_o); - #endif - STAT_INC(LOAD_ATTR, hit); - stack_pointer[-1] = attr; - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(owner); - stack_pointer = _PyFrame_GetStackPointer(frame); - } - /* Skip 5 cache entries */ - // _PUSH_NULL_CONDITIONAL - { - null = &stack_pointer[0]; - if (oparg & 1) { - null[0] = PyStackRef_NULL; - } - } - stack_pointer += (oparg & 1); - assert(WITHIN_STACK_BOUNDS()); - TRACING_DISPATCH(); - } - - TRACING_TARGET(LOAD_ATTR_NONDESCRIPTOR_NO_DICT) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = LOAD_ATTR_NONDESCRIPTOR_NO_DICT; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 10; - INSTRUCTION_STATS(LOAD_ATTR_NONDESCRIPTOR_NO_DICT); - opcode = LOAD_ATTR_NONDESCRIPTOR_NO_DICT; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); - _PyStackRef owner; - _PyStackRef attr; - /* Skip 1 cache entry */ - // _GUARD_TYPE_VERSION - { - owner = stack_pointer[-1]; - uint32_t type_version = read_u32(&this_instr[2].cache); - PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); - assert(type_version != 0); - if (FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version) { - UPDATE_MISS_STATS(LOAD_ATTR); - assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); - JUMP_TO_PREDICTED(TRACING_LOAD_ATTR); - } - } - /* Skip 2 cache entries */ - // _LOAD_ATTR_NONDESCRIPTOR_NO_DICT - { - PyObject *descr = read_obj(&this_instr[6].cache); - assert((oparg & 1) == 0); - assert(Py_TYPE(PyStackRef_AsPyObjectBorrow(owner))->tp_dictoffset == 0); - STAT_INC(LOAD_ATTR, hit); - assert(descr != NULL); - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(owner); - stack_pointer = _PyFrame_GetStackPointer(frame); - attr = PyStackRef_FromPyObjectNew(descr); - } - stack_pointer[0] = attr; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - TRACING_DISPATCH(); - } - - TRACING_TARGET(LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 10; - INSTRUCTION_STATS(LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES); - opcode = LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); - _PyStackRef owner; - _PyStackRef attr; - /* Skip 1 cache entry */ - // _GUARD_TYPE_VERSION - { - owner = stack_pointer[-1]; - uint32_t type_version = read_u32(&this_instr[2].cache); - PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); - assert(type_version != 0); - if (FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version) { - UPDATE_MISS_STATS(LOAD_ATTR); - assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); - JUMP_TO_PREDICTED(TRACING_LOAD_ATTR); - } - } - // _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT - { - PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); - assert(Py_TYPE(owner_o)->tp_flags & Py_TPFLAGS_INLINE_VALUES); - PyDictValues *ivs = _PyObject_InlineValues(owner_o); - if (!FT_ATOMIC_LOAD_UINT8(ivs->valid)) { - UPDATE_MISS_STATS(LOAD_ATTR); - assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); - JUMP_TO_PREDICTED(TRACING_LOAD_ATTR); - } - } - // _GUARD_KEYS_VERSION - { - uint32_t keys_version = read_u32(&this_instr[4].cache); - PyTypeObject *owner_cls = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); - PyHeapTypeObject *owner_heap_type = (PyHeapTypeObject *)owner_cls; - PyDictKeysObject *keys = owner_heap_type->ht_cached_keys; - if (FT_ATOMIC_LOAD_UINT32_RELAXED(keys->dk_version) != keys_version) { - UPDATE_MISS_STATS(LOAD_ATTR); - assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); - JUMP_TO_PREDICTED(TRACING_LOAD_ATTR); - } - } - // _LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES - { - PyObject *descr = read_obj(&this_instr[6].cache); - assert((oparg & 1) == 0); - STAT_INC(LOAD_ATTR, hit); - assert(descr != NULL); - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(owner); - stack_pointer = _PyFrame_GetStackPointer(frame); - attr = PyStackRef_FromPyObjectNew(descr); - } - stack_pointer[0] = attr; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - TRACING_DISPATCH(); - } - - TRACING_TARGET(LOAD_ATTR_PROPERTY) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = LOAD_ATTR_PROPERTY; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 10; - INSTRUCTION_STATS(LOAD_ATTR_PROPERTY); - opcode = LOAD_ATTR_PROPERTY; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); - _PyStackRef owner; - _PyStackRef new_frame; - /* Skip 1 cache entry */ - // _CHECK_PEP_523 - { - if (tstate->interp->eval_frame) { - UPDATE_MISS_STATS(LOAD_ATTR); - assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); - JUMP_TO_PREDICTED(TRACING_LOAD_ATTR); - } - } - // _GUARD_TYPE_VERSION - { - owner = stack_pointer[-1]; - uint32_t type_version = read_u32(&this_instr[2].cache); - PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); - assert(type_version != 0); - if (FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version) { - UPDATE_MISS_STATS(LOAD_ATTR); - assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); - JUMP_TO_PREDICTED(TRACING_LOAD_ATTR); - } - } - /* Skip 2 cache entries */ - // _LOAD_ATTR_PROPERTY_FRAME - { - PyObject *fget = read_obj(&this_instr[6].cache); - assert((oparg & 1) == 0); - assert(Py_IS_TYPE(fget, &PyFunction_Type)); - PyFunctionObject *f = (PyFunctionObject *)fget; - PyCodeObject *code = (PyCodeObject *)f->func_code; - if ((code->co_flags & (CO_VARKEYWORDS | CO_VARARGS | CO_OPTIMIZED)) != CO_OPTIMIZED) { - UPDATE_MISS_STATS(LOAD_ATTR); - assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); - JUMP_TO_PREDICTED(TRACING_LOAD_ATTR); - } - if (code->co_kwonlyargcount) { - UPDATE_MISS_STATS(LOAD_ATTR); - assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); - JUMP_TO_PREDICTED(TRACING_LOAD_ATTR); - } - if (code->co_argcount != 1) { - UPDATE_MISS_STATS(LOAD_ATTR); - assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); - JUMP_TO_PREDICTED(TRACING_LOAD_ATTR); - } - if (!_PyThreadState_HasStackSpace(tstate, code->co_framesize)) { - UPDATE_MISS_STATS(LOAD_ATTR); - assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); - JUMP_TO_PREDICTED(TRACING_LOAD_ATTR); - } - STAT_INC(LOAD_ATTR, hit); - _PyInterpreterFrame *pushed_frame = _PyFrame_PushUnchecked(tstate, PyStackRef_FromPyObjectNew(fget), 1, frame); - pushed_frame->localsplus[0] = owner; - new_frame = PyStackRef_Wrap(pushed_frame); - } - // _SAVE_RETURN_OFFSET - { - #if TIER_ONE - frame->return_offset = (uint16_t)(next_instr - this_instr); - #endif - #if TIER_TWO - frame->return_offset = oparg; - #endif - } - // _PUSH_FRAME - { - assert(tstate->interp->eval_frame == NULL); - _PyInterpreterFrame *temp = PyStackRef_Unwrap(new_frame); - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - assert(temp->previous == frame || temp->previous->previous == frame); - CALL_STAT_INC(inlined_py_calls); - frame = tstate->current_frame = temp; - tstate->py_recursion_remaining--; - LOAD_SP(); - LOAD_IP(0); - LLTRACE_RESUME_FRAME(); - } - TRACING_DISPATCH(); - } - - TRACING_TARGET(LOAD_ATTR_SLOT) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = LOAD_ATTR_SLOT; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 10; - INSTRUCTION_STATS(LOAD_ATTR_SLOT); - opcode = LOAD_ATTR_SLOT; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); - _PyStackRef owner; - _PyStackRef attr; - _PyStackRef *null; - /* Skip 1 cache entry */ - // _GUARD_TYPE_VERSION - { - owner = stack_pointer[-1]; - uint32_t type_version = read_u32(&this_instr[2].cache); - PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); - assert(type_version != 0); - if (FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version) { - UPDATE_MISS_STATS(LOAD_ATTR); - assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); - JUMP_TO_PREDICTED(TRACING_LOAD_ATTR); - } - } - // _LOAD_ATTR_SLOT - { - uint16_t index = read_u16(&this_instr[4].cache); - PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); - PyObject **addr = (PyObject **)((char *)owner_o + index); - PyObject *attr_o = FT_ATOMIC_LOAD_PTR(*addr); - if (attr_o == NULL) { - UPDATE_MISS_STATS(LOAD_ATTR); - assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); - JUMP_TO_PREDICTED(TRACING_LOAD_ATTR); - } - #ifdef Py_GIL_DISABLED - int increfed = _Py_TryIncrefCompareStackRef(addr, attr_o, &attr); - if (!increfed) { - UPDATE_MISS_STATS(LOAD_ATTR); - assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); - JUMP_TO_PREDICTED(TRACING_LOAD_ATTR); - } - #else - attr = PyStackRef_FromPyObjectNew(attr_o); - #endif - STAT_INC(LOAD_ATTR, hit); - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyStackRef tmp = owner; - owner = attr; - stack_pointer[-1] = owner; - PyStackRef_CLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); - } - /* Skip 5 cache entries */ - // _PUSH_NULL_CONDITIONAL - { - null = &stack_pointer[0]; - if (oparg & 1) { - null[0] = PyStackRef_NULL; - } - } - stack_pointer += (oparg & 1); - assert(WITHIN_STACK_BOUNDS()); - TRACING_DISPATCH(); - } - - TRACING_TARGET(LOAD_ATTR_WITH_HINT) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = LOAD_ATTR_WITH_HINT; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 10; - INSTRUCTION_STATS(LOAD_ATTR_WITH_HINT); - opcode = LOAD_ATTR_WITH_HINT; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); - _PyStackRef owner; - _PyStackRef attr; - _PyStackRef *null; - /* Skip 1 cache entry */ - // _GUARD_TYPE_VERSION - { - owner = stack_pointer[-1]; - uint32_t type_version = read_u32(&this_instr[2].cache); - PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); - assert(type_version != 0); - if (FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version) { - UPDATE_MISS_STATS(LOAD_ATTR); - assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); - JUMP_TO_PREDICTED(TRACING_LOAD_ATTR); - } - } - // _LOAD_ATTR_WITH_HINT - { - uint16_t hint = read_u16(&this_instr[4].cache); - PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); - assert(Py_TYPE(owner_o)->tp_flags & Py_TPFLAGS_MANAGED_DICT); - PyDictObject *dict = _PyObject_GetManagedDict(owner_o); - if (dict == NULL) { - UPDATE_MISS_STATS(LOAD_ATTR); - assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); - JUMP_TO_PREDICTED(TRACING_LOAD_ATTR); - } - PyDictKeysObject *dk = FT_ATOMIC_LOAD_PTR(dict->ma_keys); - assert(PyDict_CheckExact((PyObject *)dict)); - #ifdef Py_GIL_DISABLED - if (!_Py_IsOwnedByCurrentThread((PyObject *)dict) && !_PyObject_GC_IS_SHARED(dict)) { - UPDATE_MISS_STATS(LOAD_ATTR); - assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); - JUMP_TO_PREDICTED(TRACING_LOAD_ATTR); - } - #endif - PyObject *attr_o; - if (hint >= (size_t)FT_ATOMIC_LOAD_SSIZE_RELAXED(dk->dk_nentries)) { - if (true) { - UPDATE_MISS_STATS(LOAD_ATTR); - assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); - JUMP_TO_PREDICTED(TRACING_LOAD_ATTR); - } - } - PyObject *name = GETITEM(FRAME_CO_NAMES, oparg>>1); - if (dk->dk_kind != DICT_KEYS_UNICODE) { - if (true) { - UPDATE_MISS_STATS(LOAD_ATTR); - assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); - JUMP_TO_PREDICTED(TRACING_LOAD_ATTR); - } - } - PyDictUnicodeEntry *ep = DK_UNICODE_ENTRIES(dk) + hint; - if (FT_ATOMIC_LOAD_PTR_RELAXED(ep->me_key) != name) { - if (true) { - UPDATE_MISS_STATS(LOAD_ATTR); - assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); - JUMP_TO_PREDICTED(TRACING_LOAD_ATTR); - } - } - attr_o = FT_ATOMIC_LOAD_PTR(ep->me_value); - if (attr_o == NULL) { - if (true) { - UPDATE_MISS_STATS(LOAD_ATTR); - assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); - JUMP_TO_PREDICTED(TRACING_LOAD_ATTR); - } - } - STAT_INC(LOAD_ATTR, hit); - #ifdef Py_GIL_DISABLED - int increfed = _Py_TryIncrefCompareStackRef(&ep->me_value, attr_o, &attr); - if (!increfed) { - if (true) { - UPDATE_MISS_STATS(LOAD_ATTR); - assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); - JUMP_TO_PREDICTED(TRACING_LOAD_ATTR); - } - } - #else - attr = PyStackRef_FromPyObjectNew(attr_o); - #endif - stack_pointer[-1] = attr; - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(owner); - stack_pointer = _PyFrame_GetStackPointer(frame); - } - /* Skip 5 cache entries */ - // _PUSH_NULL_CONDITIONAL - { - null = &stack_pointer[0]; - if (oparg & 1) { - null[0] = PyStackRef_NULL; - } - } - stack_pointer += (oparg & 1); - assert(WITHIN_STACK_BOUNDS()); - TRACING_DISPATCH(); - } - - TRACING_TARGET(LOAD_BUILD_CLASS) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = LOAD_BUILD_CLASS; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(LOAD_BUILD_CLASS); - opcode = LOAD_BUILD_CLASS; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - _PyStackRef bc; - PyObject *bc_o; - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = PyMapping_GetOptionalItem(BUILTINS(), &_Py_ID(__build_class__), &bc_o); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err < 0) { - TRACING_JUMP_TO_LABEL(error); - } - if (bc_o == NULL) { - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyErr_SetString(tstate, PyExc_NameError, - "__build_class__ not found"); - stack_pointer = _PyFrame_GetStackPointer(frame); - TRACING_JUMP_TO_LABEL(error); - } - bc = PyStackRef_FromPyObjectSteal(bc_o); - stack_pointer[0] = bc; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - TRACING_DISPATCH(); - } - - TRACING_TARGET(LOAD_COMMON_CONSTANT) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = LOAD_COMMON_CONSTANT; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(LOAD_COMMON_CONSTANT); - opcode = LOAD_COMMON_CONSTANT; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - _PyStackRef value; - assert(oparg < NUM_COMMON_CONSTANTS); - value = PyStackRef_FromPyObjectNew(tstate->interp->common_consts[oparg]); - stack_pointer[0] = value; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - TRACING_DISPATCH(); - } - - TRACING_TARGET(LOAD_CONST) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = LOAD_CONST; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(LOAD_CONST); - opcode = LOAD_CONST; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - _PyStackRef value; - PyObject *obj = GETITEM(FRAME_CO_CONSTS, oparg); - value = PyStackRef_FromPyObjectBorrow(obj); - stack_pointer[0] = value; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - TRACING_DISPATCH(); - } - - TRACING_TARGET(LOAD_DEREF) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = LOAD_DEREF; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(LOAD_DEREF); - opcode = LOAD_DEREF; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - _PyStackRef value; - PyCellObject *cell = (PyCellObject *)PyStackRef_AsPyObjectBorrow(GETLOCAL(oparg)); - _PyFrame_SetStackPointer(frame, stack_pointer); - value = _PyCell_GetStackRef(cell); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (PyStackRef_IsNull(value)) { - stack_pointer[0] = value; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyEval_FormatExcUnbound(tstate, _PyFrame_GetCode(frame), oparg); - stack_pointer = _PyFrame_GetStackPointer(frame); - TRACING_JUMP_TO_LABEL(error); - } - stack_pointer[0] = value; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - TRACING_DISPATCH(); - } - - TRACING_TARGET(LOAD_FAST) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = LOAD_FAST; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(LOAD_FAST); - opcode = LOAD_FAST; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - _PyStackRef value; - assert(!PyStackRef_IsNull(GETLOCAL(oparg))); - value = PyStackRef_DUP(GETLOCAL(oparg)); - stack_pointer[0] = value; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - TRACING_DISPATCH(); - } - - TRACING_TARGET(LOAD_FAST_AND_CLEAR) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = LOAD_FAST_AND_CLEAR; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(LOAD_FAST_AND_CLEAR); - opcode = LOAD_FAST_AND_CLEAR; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - _PyStackRef value; - value = GETLOCAL(oparg); - GETLOCAL(oparg) = PyStackRef_NULL; - stack_pointer[0] = value; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - TRACING_DISPATCH(); - } - - TRACING_TARGET(LOAD_FAST_BORROW) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = LOAD_FAST_BORROW; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(LOAD_FAST_BORROW); - opcode = LOAD_FAST_BORROW; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - _PyStackRef value; - assert(!PyStackRef_IsNull(GETLOCAL(oparg))); - value = PyStackRef_Borrow(GETLOCAL(oparg)); - stack_pointer[0] = value; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - TRACING_DISPATCH(); - } - - TRACING_TARGET(LOAD_FAST_BORROW_LOAD_FAST_BORROW) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = LOAD_FAST_BORROW_LOAD_FAST_BORROW; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(LOAD_FAST_BORROW_LOAD_FAST_BORROW); - opcode = LOAD_FAST_BORROW_LOAD_FAST_BORROW; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - _PyStackRef value1; - _PyStackRef value2; - uint32_t oparg1 = oparg >> 4; - uint32_t oparg2 = oparg & 15; - value1 = PyStackRef_Borrow(GETLOCAL(oparg1)); - value2 = PyStackRef_Borrow(GETLOCAL(oparg2)); - stack_pointer[0] = value1; - stack_pointer[1] = value2; - stack_pointer += 2; - assert(WITHIN_STACK_BOUNDS()); - TRACING_DISPATCH(); - } - - TRACING_TARGET(LOAD_FAST_CHECK) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = LOAD_FAST_CHECK; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(LOAD_FAST_CHECK); - opcode = LOAD_FAST_CHECK; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - _PyStackRef value; - _PyStackRef value_s = GETLOCAL(oparg); - if (PyStackRef_IsNull(value_s)) { - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyEval_FormatExcCheckArg(tstate, PyExc_UnboundLocalError, - UNBOUNDLOCAL_ERROR_MSG, - PyTuple_GetItem(_PyFrame_GetCode(frame)->co_localsplusnames, oparg) - ); - stack_pointer = _PyFrame_GetStackPointer(frame); - TRACING_JUMP_TO_LABEL(error); - } - value = PyStackRef_DUP(value_s); - stack_pointer[0] = value; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - TRACING_DISPATCH(); - } - - TRACING_TARGET(LOAD_FAST_LOAD_FAST) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = LOAD_FAST_LOAD_FAST; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(LOAD_FAST_LOAD_FAST); - opcode = LOAD_FAST_LOAD_FAST; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - _PyStackRef value1; - _PyStackRef value2; - uint32_t oparg1 = oparg >> 4; - uint32_t oparg2 = oparg & 15; - value1 = PyStackRef_DUP(GETLOCAL(oparg1)); - value2 = PyStackRef_DUP(GETLOCAL(oparg2)); - stack_pointer[0] = value1; - stack_pointer[1] = value2; - stack_pointer += 2; - assert(WITHIN_STACK_BOUNDS()); - TRACING_DISPATCH(); - } - - TRACING_TARGET(LOAD_FROM_DICT_OR_DEREF) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = LOAD_FROM_DICT_OR_DEREF; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(LOAD_FROM_DICT_OR_DEREF); - opcode = LOAD_FROM_DICT_OR_DEREF; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - _PyStackRef class_dict_st; - _PyStackRef value; - class_dict_st = stack_pointer[-1]; - PyObject *value_o; - PyObject *name; - PyObject *class_dict = PyStackRef_AsPyObjectBorrow(class_dict_st); - assert(class_dict); - assert(oparg >= 0 && oparg < _PyFrame_GetCode(frame)->co_nlocalsplus); - name = PyTuple_GET_ITEM(_PyFrame_GetCode(frame)->co_localsplusnames, oparg); - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = PyMapping_GetOptionalItem(class_dict, name, &value_o); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err < 0) { - TRACING_JUMP_TO_LABEL(error); - } - if (!value_o) { - PyCellObject *cell = (PyCellObject *)PyStackRef_AsPyObjectBorrow(GETLOCAL(oparg)); - value_o = PyCell_GetRef(cell); - if (value_o == NULL) { - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyEval_FormatExcUnbound(tstate, _PyFrame_GetCode(frame), oparg); - stack_pointer = _PyFrame_GetStackPointer(frame); - TRACING_JUMP_TO_LABEL(error); - } - } - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(class_dict_st); - stack_pointer = _PyFrame_GetStackPointer(frame); - value = PyStackRef_FromPyObjectSteal(value_o); - stack_pointer[0] = value; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - TRACING_DISPATCH(); - } - - TRACING_TARGET(LOAD_FROM_DICT_OR_GLOBALS) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = LOAD_FROM_DICT_OR_GLOBALS; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(LOAD_FROM_DICT_OR_GLOBALS); - opcode = LOAD_FROM_DICT_OR_GLOBALS; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - _PyStackRef mod_or_class_dict; - _PyStackRef v; - mod_or_class_dict = stack_pointer[-1]; - PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); - PyObject *v_o; - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = PyMapping_GetOptionalItem(PyStackRef_AsPyObjectBorrow(mod_or_class_dict), name, &v_o); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(mod_or_class_dict); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err < 0) { - TRACING_JUMP_TO_LABEL(error); - } - if (v_o == NULL) { - if (PyDict_CheckExact(GLOBALS()) - && PyDict_CheckExact(BUILTINS())) - { - _PyFrame_SetStackPointer(frame, stack_pointer); - v_o = _PyDict_LoadGlobal((PyDictObject *)GLOBALS(), - (PyDictObject *)BUILTINS(), - name); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (v_o == NULL) { - if (!_PyErr_Occurred(tstate)) { - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyEval_FormatExcCheckArg(tstate, PyExc_NameError, - NAME_ERROR_MSG, name); - stack_pointer = _PyFrame_GetStackPointer(frame); - } - TRACING_JUMP_TO_LABEL(error); - } - } - else { - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = PyMapping_GetOptionalItem(GLOBALS(), name, &v_o); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err < 0) { - TRACING_JUMP_TO_LABEL(error); - } - if (v_o == NULL) { - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = PyMapping_GetOptionalItem(BUILTINS(), name, &v_o); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err < 0) { - TRACING_JUMP_TO_LABEL(error); - } - if (v_o == NULL) { - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyEval_FormatExcCheckArg( - tstate, PyExc_NameError, - NAME_ERROR_MSG, name); - stack_pointer = _PyFrame_GetStackPointer(frame); - TRACING_JUMP_TO_LABEL(error); - } - } - } - } - v = PyStackRef_FromPyObjectSteal(v_o); - stack_pointer[0] = v; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - TRACING_DISPATCH(); - } - - TRACING_TARGET(LOAD_GLOBAL) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = LOAD_GLOBAL; - (void)(opcode); - #endif - frame->instr_ptr = next_instr; - next_instr += 5; - INSTRUCTION_STATS(LOAD_GLOBAL); - PREDICTED_TRACING_LOAD_GLOBAL:; - _Py_CODEUNIT* const this_instr = next_instr - 5; - (void)this_instr; - opcode = LOAD_GLOBAL; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - _PyStackRef *res; - _PyStackRef *null; - // _SPECIALIZE_LOAD_GLOBAL - { - uint16_t counter = read_u16(&this_instr[1].cache); - (void)counter; - #if ENABLE_SPECIALIZATION_FT - if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { - PyObject *name = GETITEM(FRAME_CO_NAMES, oparg>>1); - next_instr = this_instr; - _PyFrame_SetStackPointer(frame, stack_pointer); - _Py_Specialize_LoadGlobal(GLOBALS(), BUILTINS(), next_instr, name); - stack_pointer = _PyFrame_GetStackPointer(frame); - DISPATCH_SAME_OPARG(); - } - OPCODE_DEFERRED_INC(LOAD_GLOBAL); - ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); - #endif /* ENABLE_SPECIALIZATION_FT */ - } - /* Skip 1 cache entry */ - /* Skip 1 cache entry */ - /* Skip 1 cache entry */ - // _LOAD_GLOBAL - { - res = &stack_pointer[0]; - PyObject *name = GETITEM(FRAME_CO_NAMES, oparg>>1); - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyEval_LoadGlobalStackRef(GLOBALS(), BUILTINS(), name, res); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (PyStackRef_IsNull(*res)) { - TRACING_JUMP_TO_LABEL(error); - } - } - // _PUSH_NULL_CONDITIONAL - { - null = &stack_pointer[1]; - if (oparg & 1) { - null[0] = PyStackRef_NULL; - } - } - stack_pointer += 1 + (oparg & 1); - assert(WITHIN_STACK_BOUNDS()); - TRACING_DISPATCH(); - } - - TRACING_TARGET(LOAD_GLOBAL_BUILTIN) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = LOAD_GLOBAL_BUILTIN; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 5; - INSTRUCTION_STATS(LOAD_GLOBAL_BUILTIN); - opcode = LOAD_GLOBAL_BUILTIN; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - static_assert(INLINE_CACHE_ENTRIES_LOAD_GLOBAL == 4, "incorrect cache size"); - _PyStackRef res; - _PyStackRef *null; - /* Skip 1 cache entry */ - // _GUARD_GLOBALS_VERSION - { - uint16_t version = read_u16(&this_instr[2].cache); - PyDictObject *dict = (PyDictObject *)GLOBALS(); - if (!PyDict_CheckExact(dict)) { - UPDATE_MISS_STATS(LOAD_GLOBAL); - assert(_PyOpcode_Deopt[opcode] == (LOAD_GLOBAL)); - JUMP_TO_PREDICTED(TRACING_LOAD_GLOBAL); - } - PyDictKeysObject *keys = FT_ATOMIC_LOAD_PTR_ACQUIRE(dict->ma_keys); - if (FT_ATOMIC_LOAD_UINT32_RELAXED(keys->dk_version) != version) { - UPDATE_MISS_STATS(LOAD_GLOBAL); - assert(_PyOpcode_Deopt[opcode] == (LOAD_GLOBAL)); - JUMP_TO_PREDICTED(TRACING_LOAD_GLOBAL); - } - assert(DK_IS_UNICODE(keys)); - } - // _LOAD_GLOBAL_BUILTINS - { - uint16_t version = read_u16(&this_instr[3].cache); - uint16_t index = read_u16(&this_instr[4].cache); - PyDictObject *dict = (PyDictObject *)BUILTINS(); - if (!PyDict_CheckExact(dict)) { - UPDATE_MISS_STATS(LOAD_GLOBAL); - assert(_PyOpcode_Deopt[opcode] == (LOAD_GLOBAL)); - JUMP_TO_PREDICTED(TRACING_LOAD_GLOBAL); - } - PyDictKeysObject *keys = FT_ATOMIC_LOAD_PTR_ACQUIRE(dict->ma_keys); - if (FT_ATOMIC_LOAD_UINT32_RELAXED(keys->dk_version) != version) { - UPDATE_MISS_STATS(LOAD_GLOBAL); - assert(_PyOpcode_Deopt[opcode] == (LOAD_GLOBAL)); - JUMP_TO_PREDICTED(TRACING_LOAD_GLOBAL); - } - assert(DK_IS_UNICODE(keys)); - PyDictUnicodeEntry *entries = DK_UNICODE_ENTRIES(keys); - PyObject *res_o = FT_ATOMIC_LOAD_PTR_RELAXED(entries[index].me_value); - if (res_o == NULL) { - UPDATE_MISS_STATS(LOAD_GLOBAL); - assert(_PyOpcode_Deopt[opcode] == (LOAD_GLOBAL)); - JUMP_TO_PREDICTED(TRACING_LOAD_GLOBAL); - } - #if Py_GIL_DISABLED - int increfed = _Py_TryIncrefCompareStackRef(&entries[index].me_value, res_o, &res); - if (!increfed) { - UPDATE_MISS_STATS(LOAD_GLOBAL); - assert(_PyOpcode_Deopt[opcode] == (LOAD_GLOBAL)); - JUMP_TO_PREDICTED(TRACING_LOAD_GLOBAL); - } - #else - res = PyStackRef_FromPyObjectNew(res_o); - #endif - STAT_INC(LOAD_GLOBAL, hit); - } - // _PUSH_NULL_CONDITIONAL - { - null = &stack_pointer[1]; - if (oparg & 1) { - null[0] = PyStackRef_NULL; - } - } - stack_pointer[0] = res; - stack_pointer += 1 + (oparg & 1); - assert(WITHIN_STACK_BOUNDS()); - TRACING_DISPATCH(); - } - - TRACING_TARGET(LOAD_GLOBAL_MODULE) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = LOAD_GLOBAL_MODULE; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 5; - INSTRUCTION_STATS(LOAD_GLOBAL_MODULE); - opcode = LOAD_GLOBAL_MODULE; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - static_assert(INLINE_CACHE_ENTRIES_LOAD_GLOBAL == 4, "incorrect cache size"); - _PyStackRef res; - _PyStackRef *null; - /* Skip 1 cache entry */ - // _NOP - { - } - // _LOAD_GLOBAL_MODULE - { - uint16_t version = read_u16(&this_instr[2].cache); - uint16_t index = read_u16(&this_instr[4].cache); - PyDictObject *dict = (PyDictObject *)GLOBALS(); - if (!PyDict_CheckExact(dict)) { - UPDATE_MISS_STATS(LOAD_GLOBAL); - assert(_PyOpcode_Deopt[opcode] == (LOAD_GLOBAL)); - JUMP_TO_PREDICTED(TRACING_LOAD_GLOBAL); - } - PyDictKeysObject *keys = FT_ATOMIC_LOAD_PTR_ACQUIRE(dict->ma_keys); - if (FT_ATOMIC_LOAD_UINT32_RELAXED(keys->dk_version) != version) { - UPDATE_MISS_STATS(LOAD_GLOBAL); - assert(_PyOpcode_Deopt[opcode] == (LOAD_GLOBAL)); - JUMP_TO_PREDICTED(TRACING_LOAD_GLOBAL); - } - assert(DK_IS_UNICODE(keys)); - PyDictUnicodeEntry *entries = DK_UNICODE_ENTRIES(keys); - assert(index < DK_SIZE(keys)); - PyObject *res_o = FT_ATOMIC_LOAD_PTR_RELAXED(entries[index].me_value); - if (res_o == NULL) { - UPDATE_MISS_STATS(LOAD_GLOBAL); - assert(_PyOpcode_Deopt[opcode] == (LOAD_GLOBAL)); - JUMP_TO_PREDICTED(TRACING_LOAD_GLOBAL); - } - #if Py_GIL_DISABLED - int increfed = _Py_TryIncrefCompareStackRef(&entries[index].me_value, res_o, &res); - if (!increfed) { - UPDATE_MISS_STATS(LOAD_GLOBAL); - assert(_PyOpcode_Deopt[opcode] == (LOAD_GLOBAL)); - JUMP_TO_PREDICTED(TRACING_LOAD_GLOBAL); - } - #else - res = PyStackRef_FromPyObjectNew(res_o); - #endif - STAT_INC(LOAD_GLOBAL, hit); - } - // _PUSH_NULL_CONDITIONAL - { - null = &stack_pointer[1]; - if (oparg & 1) { - null[0] = PyStackRef_NULL; - } - } - stack_pointer[0] = res; - stack_pointer += 1 + (oparg & 1); - assert(WITHIN_STACK_BOUNDS()); - TRACING_DISPATCH(); - } - - TRACING_TARGET(LOAD_LOCALS) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = LOAD_LOCALS; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(LOAD_LOCALS); - opcode = LOAD_LOCALS; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - _PyStackRef locals; - PyObject *l = LOCALS(); - if (l == NULL) { - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyErr_SetString(tstate, PyExc_SystemError, - "no locals found"); - stack_pointer = _PyFrame_GetStackPointer(frame); - TRACING_JUMP_TO_LABEL(error); - } - locals = PyStackRef_FromPyObjectNew(l); - stack_pointer[0] = locals; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - TRACING_DISPATCH(); - } - - TRACING_TARGET(LOAD_NAME) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = LOAD_NAME; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(LOAD_NAME); - opcode = LOAD_NAME; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - _PyStackRef v; - PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *v_o = _PyEval_LoadName(tstate, frame, name); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (v_o == NULL) { - TRACING_JUMP_TO_LABEL(error); - } - v = PyStackRef_FromPyObjectSteal(v_o); - stack_pointer[0] = v; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - TRACING_DISPATCH(); - } - - TRACING_TARGET(LOAD_SMALL_INT) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = LOAD_SMALL_INT; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(LOAD_SMALL_INT); - opcode = LOAD_SMALL_INT; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - _PyStackRef value; - assert(oparg < _PY_NSMALLPOSINTS); - PyObject *obj = (PyObject *)&_PyLong_SMALL_INTS[_PY_NSMALLNEGINTS + oparg]; - value = PyStackRef_FromPyObjectBorrow(obj); - stack_pointer[0] = value; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - TRACING_DISPATCH(); - } - - TRACING_TARGET(LOAD_SPECIAL) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = LOAD_SPECIAL; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(LOAD_SPECIAL); - opcode = LOAD_SPECIAL; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - _PyStackRef self; - _PyStackRef *method_and_self; - // _INSERT_NULL - { - self = stack_pointer[-1]; - method_and_self = &stack_pointer[-1]; - method_and_self[1] = self; - method_and_self[0] = PyStackRef_NULL; - } - // _LOAD_SPECIAL - { - method_and_self = &stack_pointer[-1]; - PyObject *name = _Py_SpecialMethods[oparg].name; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = _PyObject_LookupSpecialMethod(name, method_and_self); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err <= 0) { - if (err == 0) { - PyObject *owner = PyStackRef_AsPyObjectBorrow(method_and_self[1]); - _PyFrame_SetStackPointer(frame, stack_pointer); - const char *errfmt = _PyEval_SpecialMethodCanSuggest(owner, oparg) - ? _Py_SpecialMethods[oparg].error_suggestion - : _Py_SpecialMethods[oparg].error; - stack_pointer = _PyFrame_GetStackPointer(frame); - assert(!_PyErr_Occurred(tstate)); - assert(errfmt != NULL); - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyErr_Format(tstate, PyExc_TypeError, errfmt, owner); - stack_pointer = _PyFrame_GetStackPointer(frame); - } - TRACING_JUMP_TO_LABEL(error); - } - } - TRACING_DISPATCH(); - } - - TRACING_TARGET(LOAD_SUPER_ATTR) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = LOAD_SUPER_ATTR; - (void)(opcode); - #endif - frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(LOAD_SUPER_ATTR); - PREDICTED_TRACING_LOAD_SUPER_ATTR:; - _Py_CODEUNIT* const this_instr = next_instr - 2; - (void)this_instr; - opcode = LOAD_SUPER_ATTR; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - opcode = LOAD_SUPER_ATTR; - _PyStackRef global_super_st; - _PyStackRef class_st; - _PyStackRef self_st; - _PyStackRef attr; - _PyStackRef *null; - // _SPECIALIZE_LOAD_SUPER_ATTR - { - class_st = stack_pointer[-2]; - global_super_st = stack_pointer[-3]; - uint16_t counter = read_u16(&this_instr[1].cache); - (void)counter; - #if ENABLE_SPECIALIZATION_FT - int load_method = oparg & 1; - if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { - next_instr = this_instr; - _PyFrame_SetStackPointer(frame, stack_pointer); - _Py_Specialize_LoadSuperAttr(global_super_st, class_st, next_instr, load_method); - stack_pointer = _PyFrame_GetStackPointer(frame); - DISPATCH_SAME_OPARG(); - } - OPCODE_DEFERRED_INC(LOAD_SUPER_ATTR); - ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); - #endif /* ENABLE_SPECIALIZATION_FT */ - } - // _LOAD_SUPER_ATTR - { - self_st = stack_pointer[-1]; - PyObject *global_super = PyStackRef_AsPyObjectBorrow(global_super_st); - PyObject *class = PyStackRef_AsPyObjectBorrow(class_st); - PyObject *self = PyStackRef_AsPyObjectBorrow(self_st); - if (opcode == INSTRUMENTED_LOAD_SUPER_ATTR) { - PyObject *arg = oparg & 2 ? class : &_PyInstrumentation_MISSING; - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = _Py_call_instrumentation_2args( - tstate, PY_MONITORING_EVENT_CALL, - frame, this_instr, global_super, arg); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err) { - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyStackRef tmp = self_st; - self_st = PyStackRef_NULL; - stack_pointer[-1] = self_st; - PyStackRef_CLOSE(tmp); - tmp = class_st; - class_st = PyStackRef_NULL; - stack_pointer[-2] = class_st; - PyStackRef_CLOSE(tmp); - tmp = global_super_st; - global_super_st = PyStackRef_NULL; - stack_pointer[-3] = global_super_st; - PyStackRef_CLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -3; - assert(WITHIN_STACK_BOUNDS()); - TRACING_JUMP_TO_LABEL(error); - } - } - PyObject *stack[] = {class, self}; - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *super = PyObject_Vectorcall(global_super, stack, oparg & 2, NULL); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (opcode == INSTRUMENTED_LOAD_SUPER_ATTR) { - PyObject *arg = oparg & 2 ? class : &_PyInstrumentation_MISSING; - if (super == NULL) { - _PyFrame_SetStackPointer(frame, stack_pointer); - _Py_call_instrumentation_exc2( - tstate, PY_MONITORING_EVENT_C_RAISE, - frame, this_instr, global_super, arg); - stack_pointer = _PyFrame_GetStackPointer(frame); - } - else { - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = _Py_call_instrumentation_2args( - tstate, PY_MONITORING_EVENT_C_RETURN, - frame, this_instr, global_super, arg); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err < 0) { - _PyFrame_SetStackPointer(frame, stack_pointer); - Py_CLEAR(super); - stack_pointer = _PyFrame_GetStackPointer(frame); - } - } - } - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyStackRef tmp = self_st; - self_st = PyStackRef_NULL; - stack_pointer[-1] = self_st; - PyStackRef_CLOSE(tmp); - tmp = class_st; - class_st = PyStackRef_NULL; - stack_pointer[-2] = class_st; - PyStackRef_CLOSE(tmp); - tmp = global_super_st; - global_super_st = PyStackRef_NULL; - stack_pointer[-3] = global_super_st; - PyStackRef_CLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -3; - assert(WITHIN_STACK_BOUNDS()); - if (super == NULL) { - TRACING_JUMP_TO_LABEL(error); - } - PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 2); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *attr_o = PyObject_GetAttr(super, name); - Py_DECREF(super); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (attr_o == NULL) { - TRACING_JUMP_TO_LABEL(error); - } - attr = PyStackRef_FromPyObjectSteal(attr_o); - } - // _PUSH_NULL_CONDITIONAL - { - null = &stack_pointer[1]; - if (oparg & 1) { - null[0] = PyStackRef_NULL; - } - } - stack_pointer[0] = attr; - stack_pointer += 1 + (oparg & 1); - assert(WITHIN_STACK_BOUNDS()); - TRACING_DISPATCH(); - } - - TRACING_TARGET(LOAD_SUPER_ATTR_ATTR) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = LOAD_SUPER_ATTR_ATTR; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(LOAD_SUPER_ATTR_ATTR); - opcode = LOAD_SUPER_ATTR_ATTR; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - static_assert(INLINE_CACHE_ENTRIES_LOAD_SUPER_ATTR == 1, "incorrect cache size"); - _PyStackRef global_super_st; - _PyStackRef class_st; - _PyStackRef self_st; - _PyStackRef attr_st; - /* Skip 1 cache entry */ - self_st = stack_pointer[-1]; - class_st = stack_pointer[-2]; - global_super_st = stack_pointer[-3]; - PyObject *global_super = PyStackRef_AsPyObjectBorrow(global_super_st); - PyObject *class = PyStackRef_AsPyObjectBorrow(class_st); - PyObject *self = PyStackRef_AsPyObjectBorrow(self_st); - assert(!(oparg & 1)); - if (global_super != (PyObject *)&PySuper_Type) { - UPDATE_MISS_STATS(LOAD_SUPER_ATTR); - assert(_PyOpcode_Deopt[opcode] == (LOAD_SUPER_ATTR)); - JUMP_TO_PREDICTED(TRACING_LOAD_SUPER_ATTR); - } - if (!PyType_Check(class)) { - UPDATE_MISS_STATS(LOAD_SUPER_ATTR); - assert(_PyOpcode_Deopt[opcode] == (LOAD_SUPER_ATTR)); - JUMP_TO_PREDICTED(TRACING_LOAD_SUPER_ATTR); - } - STAT_INC(LOAD_SUPER_ATTR, hit); - PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 2); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *attr = _PySuper_Lookup((PyTypeObject *)class, self, name, NULL); - _PyStackRef tmp = self_st; - self_st = PyStackRef_NULL; - stack_pointer[-1] = self_st; - PyStackRef_CLOSE(tmp); - tmp = class_st; - class_st = PyStackRef_NULL; - stack_pointer[-2] = class_st; - PyStackRef_CLOSE(tmp); - tmp = global_super_st; - global_super_st = PyStackRef_NULL; - stack_pointer[-3] = global_super_st; - PyStackRef_CLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -3; - assert(WITHIN_STACK_BOUNDS()); - if (attr == NULL) { - TRACING_JUMP_TO_LABEL(error); - } - attr_st = PyStackRef_FromPyObjectSteal(attr); - stack_pointer[0] = attr_st; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - TRACING_DISPATCH(); - } - - TRACING_TARGET(LOAD_SUPER_ATTR_METHOD) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = LOAD_SUPER_ATTR_METHOD; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(LOAD_SUPER_ATTR_METHOD); - opcode = LOAD_SUPER_ATTR_METHOD; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - static_assert(INLINE_CACHE_ENTRIES_LOAD_SUPER_ATTR == 1, "incorrect cache size"); - _PyStackRef global_super_st; - _PyStackRef class_st; - _PyStackRef self_st; - _PyStackRef attr; - _PyStackRef self_or_null; - /* Skip 1 cache entry */ - self_st = stack_pointer[-1]; - class_st = stack_pointer[-2]; - global_super_st = stack_pointer[-3]; - PyObject *global_super = PyStackRef_AsPyObjectBorrow(global_super_st); - PyObject *class = PyStackRef_AsPyObjectBorrow(class_st); - PyObject *self = PyStackRef_AsPyObjectBorrow(self_st); - assert(oparg & 1); - if (global_super != (PyObject *)&PySuper_Type) { - UPDATE_MISS_STATS(LOAD_SUPER_ATTR); - assert(_PyOpcode_Deopt[opcode] == (LOAD_SUPER_ATTR)); - JUMP_TO_PREDICTED(TRACING_LOAD_SUPER_ATTR); - } - if (!PyType_Check(class)) { - UPDATE_MISS_STATS(LOAD_SUPER_ATTR); - assert(_PyOpcode_Deopt[opcode] == (LOAD_SUPER_ATTR)); - JUMP_TO_PREDICTED(TRACING_LOAD_SUPER_ATTR); - } - STAT_INC(LOAD_SUPER_ATTR, hit); - PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 2); - PyTypeObject *cls = (PyTypeObject *)class; - int method_found = 0; - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *attr_o = _PySuper_Lookup(cls, self, name, - Py_TYPE(self)->tp_getattro == PyObject_GenericGetAttr ? &method_found : NULL); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (attr_o == NULL) { - TRACING_JUMP_TO_LABEL(error); - } - if (method_found) { - self_or_null = self_st; - } else { - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(self_st); - stack_pointer = _PyFrame_GetStackPointer(frame); - self_or_null = PyStackRef_NULL; - stack_pointer += 1; - } - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyStackRef tmp = global_super_st; - global_super_st = self_or_null; - stack_pointer[-2] = global_super_st; - PyStackRef_CLOSE(tmp); - tmp = class_st; - class_st = PyStackRef_NULL; - stack_pointer[-1] = class_st; - PyStackRef_CLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); - attr = PyStackRef_FromPyObjectSteal(attr_o); - stack_pointer[0] = attr; - stack_pointer[1] = self_or_null; - stack_pointer += 2; - assert(WITHIN_STACK_BOUNDS()); - TRACING_DISPATCH(); - } - - TRACING_TARGET(MAKE_CELL) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = MAKE_CELL; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(MAKE_CELL); - opcode = MAKE_CELL; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - PyObject *initial = PyStackRef_AsPyObjectBorrow(GETLOCAL(oparg)); - PyObject *cell = PyCell_New(initial); - if (cell == NULL) { - TRACING_JUMP_TO_LABEL(error); - } - _PyStackRef tmp = GETLOCAL(oparg); - GETLOCAL(oparg) = PyStackRef_FromPyObjectSteal(cell); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_XCLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); - TRACING_DISPATCH(); - } - - TRACING_TARGET(MAKE_FUNCTION) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = MAKE_FUNCTION; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(MAKE_FUNCTION); - opcode = MAKE_FUNCTION; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - _PyStackRef codeobj_st; - _PyStackRef func; - codeobj_st = stack_pointer[-1]; - PyObject *codeobj = PyStackRef_AsPyObjectBorrow(codeobj_st); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyFunctionObject *func_obj = (PyFunctionObject *) - PyFunction_New(codeobj, GLOBALS()); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(codeobj_st); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (func_obj == NULL) { - TRACING_JUMP_TO_LABEL(error); - } - _PyFunction_SetVersion( - func_obj, ((PyCodeObject *)codeobj)->co_version); - func = PyStackRef_FromPyObjectSteal((PyObject *)func_obj); - stack_pointer[0] = func; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - TRACING_DISPATCH(); - } - - TRACING_TARGET(MAP_ADD) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = MAP_ADD; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(MAP_ADD); - opcode = MAP_ADD; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - _PyStackRef dict_st; - _PyStackRef key; - _PyStackRef value; - value = stack_pointer[-1]; - key = stack_pointer[-2]; - dict_st = stack_pointer[-3 - (oparg - 1)]; - PyObject *dict = PyStackRef_AsPyObjectBorrow(dict_st); - assert(PyDict_CheckExact(dict)); - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = _PyDict_SetItem_Take2( - (PyDictObject *)dict, - PyStackRef_AsPyObjectSteal(key), - PyStackRef_AsPyObjectSteal(value) - ); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err != 0) { - TRACING_JUMP_TO_LABEL(pop_2_error); - } - stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); - TRACING_DISPATCH(); - } - - TRACING_TARGET(MATCH_CLASS) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = MATCH_CLASS; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(MATCH_CLASS); - opcode = MATCH_CLASS; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - _PyStackRef subject; - _PyStackRef type; - _PyStackRef names; - _PyStackRef attrs; - names = stack_pointer[-1]; - type = stack_pointer[-2]; - subject = stack_pointer[-3]; - assert(PyTuple_CheckExact(PyStackRef_AsPyObjectBorrow(names))); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *attrs_o = _PyEval_MatchClass(tstate, - PyStackRef_AsPyObjectBorrow(subject), - PyStackRef_AsPyObjectBorrow(type), oparg, - PyStackRef_AsPyObjectBorrow(names)); - _PyStackRef tmp = names; - names = PyStackRef_NULL; - stack_pointer[-1] = names; - PyStackRef_CLOSE(tmp); - tmp = type; - type = PyStackRef_NULL; - stack_pointer[-2] = type; - PyStackRef_CLOSE(tmp); - tmp = subject; - subject = PyStackRef_NULL; - stack_pointer[-3] = subject; - PyStackRef_CLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -3; - assert(WITHIN_STACK_BOUNDS()); - if (attrs_o) { - assert(PyTuple_CheckExact(attrs_o)); - attrs = PyStackRef_FromPyObjectSteal(attrs_o); - } - else { - if (_PyErr_Occurred(tstate)) { - TRACING_JUMP_TO_LABEL(error); - } - attrs = PyStackRef_None; - } - stack_pointer[0] = attrs; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - TRACING_DISPATCH(); - } - - TRACING_TARGET(MATCH_KEYS) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = MATCH_KEYS; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(MATCH_KEYS); - opcode = MATCH_KEYS; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - _PyStackRef subject; - _PyStackRef keys; - _PyStackRef values_or_none; - keys = stack_pointer[-1]; - subject = stack_pointer[-2]; - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *values_or_none_o = _PyEval_MatchKeys(tstate, - PyStackRef_AsPyObjectBorrow(subject), PyStackRef_AsPyObjectBorrow(keys)); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (values_or_none_o == NULL) { - TRACING_JUMP_TO_LABEL(error); - } - values_or_none = PyStackRef_FromPyObjectSteal(values_or_none_o); - stack_pointer[0] = values_or_none; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - TRACING_DISPATCH(); - } - - TRACING_TARGET(MATCH_MAPPING) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = MATCH_MAPPING; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(MATCH_MAPPING); - opcode = MATCH_MAPPING; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - _PyStackRef subject; - _PyStackRef res; - subject = stack_pointer[-1]; - int match = PyStackRef_TYPE(subject)->tp_flags & Py_TPFLAGS_MAPPING; - res = match ? PyStackRef_True : PyStackRef_False; - stack_pointer[0] = res; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - TRACING_DISPATCH(); - } - - TRACING_TARGET(MATCH_SEQUENCE) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = MATCH_SEQUENCE; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(MATCH_SEQUENCE); - opcode = MATCH_SEQUENCE; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - _PyStackRef subject; - _PyStackRef res; - subject = stack_pointer[-1]; - int match = PyStackRef_TYPE(subject)->tp_flags & Py_TPFLAGS_SEQUENCE; - res = match ? PyStackRef_True : PyStackRef_False; - stack_pointer[0] = res; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - TRACING_DISPATCH(); - } - - TRACING_TARGET(NOP) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = NOP; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(NOP); - opcode = NOP; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - TRACING_DISPATCH(); - } - - TRACING_TARGET(NOT_TAKEN) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = NOT_TAKEN; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(NOT_TAKEN); - opcode = NOT_TAKEN; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - TRACING_DISPATCH(); - } - - TRACING_TARGET(POP_EXCEPT) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = POP_EXCEPT; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(POP_EXCEPT); - opcode = POP_EXCEPT; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - _PyStackRef exc_value; - exc_value = stack_pointer[-1]; - _PyErr_StackItem *exc_info = tstate->exc_info; - _PyFrame_SetStackPointer(frame, stack_pointer); - Py_XSETREF(exc_info->exc_value, - PyStackRef_IsNone(exc_value) - ? NULL : PyStackRef_AsPyObjectSteal(exc_value)); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - TRACING_DISPATCH(); - } - - TRACING_TARGET(POP_ITER) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = POP_ITER; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(POP_ITER); - opcode = POP_ITER; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - _PyStackRef iter; - _PyStackRef index_or_null; - index_or_null = stack_pointer[-1]; - iter = stack_pointer[-2]; - (void)index_or_null; - stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(iter); - stack_pointer = _PyFrame_GetStackPointer(frame); - TRACING_DISPATCH(); - } - - TRACING_TARGET(POP_JUMP_IF_FALSE) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = POP_JUMP_IF_FALSE; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(POP_JUMP_IF_FALSE); - opcode = POP_JUMP_IF_FALSE; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - _PyStackRef cond; - /* Skip 1 cache entry */ - cond = stack_pointer[-1]; - assert(PyStackRef_BoolCheck(cond)); - int flag = PyStackRef_IsFalse(cond); - JUMPBY(flag ? oparg : next_instr->op.code == NOT_TAKEN); - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - TRACING_DISPATCH(); - } - - TRACING_TARGET(POP_JUMP_IF_NONE) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = POP_JUMP_IF_NONE; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(POP_JUMP_IF_NONE); - opcode = POP_JUMP_IF_NONE; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - _PyStackRef value; - _PyStackRef b; - _PyStackRef cond; - /* Skip 1 cache entry */ - // _IS_NONE - { - value = stack_pointer[-1]; - if (PyStackRef_IsNone(value)) { - b = PyStackRef_True; - } - else { - b = PyStackRef_False; - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyStackRef tmp = value; - value = b; - stack_pointer[-1] = value; - PyStackRef_CLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); - } - } - // _POP_JUMP_IF_TRUE - { - cond = b; - assert(PyStackRef_BoolCheck(cond)); - int flag = PyStackRef_IsTrue(cond); - JUMPBY(flag ? oparg : next_instr->op.code == NOT_TAKEN); - } - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - TRACING_DISPATCH(); - } - - TRACING_TARGET(POP_JUMP_IF_NOT_NONE) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = POP_JUMP_IF_NOT_NONE; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(POP_JUMP_IF_NOT_NONE); - opcode = POP_JUMP_IF_NOT_NONE; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - _PyStackRef value; - _PyStackRef b; - _PyStackRef cond; - /* Skip 1 cache entry */ - // _IS_NONE - { - value = stack_pointer[-1]; - if (PyStackRef_IsNone(value)) { - b = PyStackRef_True; - } - else { - b = PyStackRef_False; - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyStackRef tmp = value; - value = b; - stack_pointer[-1] = value; - PyStackRef_CLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); - } - } - // _POP_JUMP_IF_FALSE - { - cond = b; - assert(PyStackRef_BoolCheck(cond)); - int flag = PyStackRef_IsFalse(cond); - JUMPBY(flag ? oparg : next_instr->op.code == NOT_TAKEN); - } - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - TRACING_DISPATCH(); - } - - TRACING_TARGET(POP_JUMP_IF_TRUE) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = POP_JUMP_IF_TRUE; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(POP_JUMP_IF_TRUE); - opcode = POP_JUMP_IF_TRUE; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - _PyStackRef cond; - /* Skip 1 cache entry */ - cond = stack_pointer[-1]; - assert(PyStackRef_BoolCheck(cond)); - int flag = PyStackRef_IsTrue(cond); - JUMPBY(flag ? oparg : next_instr->op.code == NOT_TAKEN); - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - TRACING_DISPATCH(); - } - - TRACING_TARGET(POP_TOP) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = POP_TOP; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(POP_TOP); - opcode = POP_TOP; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - _PyStackRef value; - value = stack_pointer[-1]; - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_XCLOSE(value); - stack_pointer = _PyFrame_GetStackPointer(frame); - TRACING_DISPATCH(); - } - - TRACING_TARGET(PUSH_EXC_INFO) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = PUSH_EXC_INFO; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(PUSH_EXC_INFO); - opcode = PUSH_EXC_INFO; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - _PyStackRef exc; - _PyStackRef prev_exc; - _PyStackRef new_exc; - exc = stack_pointer[-1]; - _PyErr_StackItem *exc_info = tstate->exc_info; - if (exc_info->exc_value != NULL) { - prev_exc = PyStackRef_FromPyObjectSteal(exc_info->exc_value); - } - else { - prev_exc = PyStackRef_None; - } - assert(PyStackRef_ExceptionInstanceCheck(exc)); - exc_info->exc_value = PyStackRef_AsPyObjectNew(exc); - new_exc = exc; - stack_pointer[-1] = prev_exc; - stack_pointer[0] = new_exc; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - TRACING_DISPATCH(); - } - - TRACING_TARGET(PUSH_NULL) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = PUSH_NULL; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(PUSH_NULL); - opcode = PUSH_NULL; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - _PyStackRef res; - res = PyStackRef_NULL; - stack_pointer[0] = res; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - TRACING_DISPATCH(); - } - - TRACING_TARGET(RAISE_VARARGS) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = RAISE_VARARGS; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(RAISE_VARARGS); - opcode = RAISE_VARARGS; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - _PyStackRef *args; - args = &stack_pointer[-oparg]; - assert(oparg < 3); - PyObject *cause = oparg == 2 ? PyStackRef_AsPyObjectSteal(args[1]) : NULL; - PyObject *exc = oparg > 0 ? PyStackRef_AsPyObjectSteal(args[0]) : NULL; - stack_pointer += -oparg; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = do_raise(tstate, exc, cause); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err) { - assert(oparg == 0); - _PyFrame_SetStackPointer(frame, stack_pointer); - monitor_reraise(tstate, frame, this_instr); - TRACING_JUMP_TO_LABEL(exception_unwind); - } - TRACING_JUMP_TO_LABEL(error); - } - - TRACING_TARGET(RERAISE) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = RERAISE; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(RERAISE); - opcode = RERAISE; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - _PyStackRef *values; - _PyStackRef exc_st; - exc_st = stack_pointer[-1]; - values = &stack_pointer[-1 - oparg]; - PyObject *exc = PyStackRef_AsPyObjectSteal(exc_st); - assert(oparg >= 0 && oparg <= 2); - if (oparg) { - frame->instr_ptr = _PyFrame_GetBytecode(frame) + PyStackRef_UntagInt(values[0]); - } - assert(exc && PyExceptionInstance_Check(exc)); - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyErr_SetRaisedException(tstate, exc); - monitor_reraise(tstate, frame, this_instr); - TRACING_JUMP_TO_LABEL(exception_unwind); - } - - TRACING_TARGET(RESERVED) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = RESERVED; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(RESERVED); - opcode = RESERVED; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - assert(0 && "Executing RESERVED instruction."); - Py_FatalError("Executing RESERVED instruction."); - TRACING_DISPATCH(); - } - - TRACING_TARGET(RESUME) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = RESUME; - (void)(opcode); - #endif - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(RESUME); - PREDICTED_TRACING_RESUME:; - _Py_CODEUNIT* const this_instr = next_instr - 1; - (void)this_instr; - opcode = RESUME; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - // _LOAD_BYTECODE - { - #ifdef Py_GIL_DISABLED - if (frame->tlbc_index != - ((_PyThreadStateImpl *)tstate)->tlbc_index) { - _PyFrame_SetStackPointer(frame, stack_pointer); - _Py_CODEUNIT *bytecode = - _PyEval_GetExecutableCode(tstate, _PyFrame_GetCode(frame)); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (bytecode == NULL) { - TRACING_JUMP_TO_LABEL(error); - } - ptrdiff_t off = this_instr - _PyFrame_GetBytecode(frame); - frame->tlbc_index = ((_PyThreadStateImpl *)tstate)->tlbc_index; - frame->instr_ptr = bytecode + off; - next_instr = frame->instr_ptr; - TRACING_DISPATCH(); - } - #endif - } - // _MAYBE_INSTRUMENT - { - #ifdef Py_GIL_DISABLED - - int check_instrumentation = 1; - #else - int check_instrumentation = (tstate->tracing == 0); - #endif - if (check_instrumentation) { - uintptr_t global_version = _Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & ~_PY_EVAL_EVENTS_MASK; - uintptr_t code_version = FT_ATOMIC_LOAD_UINTPTR_ACQUIRE(_PyFrame_GetCode(frame)->_co_instrumentation_version); - if (code_version != global_version) { - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = _Py_Instrument(_PyFrame_GetCode(frame), tstate->interp); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err) { - TRACING_JUMP_TO_LABEL(error); - } - next_instr = this_instr; - TRACING_DISPATCH(); - } - } - } - // _QUICKEN_RESUME - { - #if ENABLE_SPECIALIZATION_FT - if (tstate->tracing == 0 && this_instr->op.code == RESUME) { - FT_ATOMIC_STORE_UINT8_RELAXED(this_instr->op.code, RESUME_CHECK); - } - #endif /* ENABLE_SPECIALIZATION_FT */ - } - // _CHECK_PERIODIC_IF_NOT_YIELD_FROM - { - if ((oparg & RESUME_OPARG_LOCATION_MASK) < RESUME_AFTER_YIELD_FROM) { - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = check_periodics(tstate); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err != 0) { - TRACING_JUMP_TO_LABEL(error); - } - } - } - TRACING_DISPATCH(); - } - - TRACING_TARGET(RESUME_CHECK) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = RESUME_CHECK; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(RESUME_CHECK); - opcode = RESUME_CHECK; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - static_assert(0 == 0, "incorrect cache size"); - #if defined(__EMSCRIPTEN__) - if (_Py_emscripten_signal_clock == 0) { - UPDATE_MISS_STATS(RESUME); - assert(_PyOpcode_Deopt[opcode] == (RESUME)); - JUMP_TO_PREDICTED(TRACING_RESUME); - } - _Py_emscripten_signal_clock -= Py_EMSCRIPTEN_SIGNAL_HANDLING; - #endif - uintptr_t eval_breaker = _Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker); - uintptr_t version = FT_ATOMIC_LOAD_UINTPTR_ACQUIRE(_PyFrame_GetCode(frame)->_co_instrumentation_version); - assert((version & _PY_EVAL_EVENTS_MASK) == 0); - if (eval_breaker != version) { - UPDATE_MISS_STATS(RESUME); - assert(_PyOpcode_Deopt[opcode] == (RESUME)); - JUMP_TO_PREDICTED(TRACING_RESUME); - } - #ifdef Py_GIL_DISABLED - if (frame->tlbc_index != - ((_PyThreadStateImpl *)tstate)->tlbc_index) { - UPDATE_MISS_STATS(RESUME); - assert(_PyOpcode_Deopt[opcode] == (RESUME)); - JUMP_TO_PREDICTED(TRACING_RESUME); - } - #endif - TRACING_DISPATCH(); - } - - TRACING_TARGET(RETURN_GENERATOR) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = RETURN_GENERATOR; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(RETURN_GENERATOR); - opcode = RETURN_GENERATOR; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - _PyStackRef res; - assert(PyStackRef_FunctionCheck(frame->f_funcobj)); - PyFunctionObject *func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyGenObject *gen = (PyGenObject *)_Py_MakeCoro(func); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (gen == NULL) { - TRACING_JUMP_TO_LABEL(error); - } - assert(STACK_LEVEL() == 0); - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyInterpreterFrame *gen_frame = &gen->gi_iframe; - frame->instr_ptr++; - _PyFrame_Copy(frame, gen_frame); - assert(frame->frame_obj == NULL); - gen->gi_frame_state = FRAME_CREATED; - gen_frame->owner = FRAME_OWNED_BY_GENERATOR; - _Py_LeaveRecursiveCallPy(tstate); - _PyInterpreterFrame *prev = frame->previous; - _PyThreadState_PopFrame(tstate, frame); - frame = tstate->current_frame = prev; - LOAD_IP(frame->return_offset); - #if TIER_TWO - frame->instr_ptr += (frame->return_offset); - #endif - stack_pointer = _PyFrame_GetStackPointer(frame); - res = PyStackRef_FromPyObjectStealMortal((PyObject *)gen); - LLTRACE_RESUME_FRAME(); - stack_pointer[0] = res; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - TRACING_DISPATCH(); - } - - TRACING_TARGET(RETURN_VALUE) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = RETURN_VALUE; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(RETURN_VALUE); - opcode = RETURN_VALUE; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - _PyStackRef retval; - _PyStackRef res; - retval = stack_pointer[-1]; - assert(frame->owner != FRAME_OWNED_BY_INTERPRETER); - _PyStackRef temp = PyStackRef_MakeHeapSafe(retval); - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - assert(STACK_LEVEL() == 0); - _Py_LeaveRecursiveCallPy(tstate); - _PyInterpreterFrame *dying = frame; - frame = tstate->current_frame = dying->previous; - _PyEval_FrameClearAndPop(tstate, dying); - stack_pointer = _PyFrame_GetStackPointer(frame); - LOAD_IP(frame->return_offset); - #if TIER_TWO - frame->instr_ptr += frame->return_offset; - #endif - res = temp; - LLTRACE_RESUME_FRAME(); - stack_pointer[0] = res; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - TRACING_DISPATCH(); - } - - TRACING_TARGET(SEND) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = SEND; - (void)(opcode); - #endif - frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(SEND); - PREDICTED_TRACING_SEND:; - _Py_CODEUNIT* const this_instr = next_instr - 2; - (void)this_instr; - opcode = SEND; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - _PyStackRef receiver; - _PyStackRef v; - _PyStackRef retval; - // _SPECIALIZE_SEND - { - receiver = stack_pointer[-2]; - uint16_t counter = read_u16(&this_instr[1].cache); - (void)counter; - #if ENABLE_SPECIALIZATION_FT - if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { - next_instr = this_instr; - _PyFrame_SetStackPointer(frame, stack_pointer); - _Py_Specialize_Send(receiver, next_instr); - stack_pointer = _PyFrame_GetStackPointer(frame); - DISPATCH_SAME_OPARG(); - } - OPCODE_DEFERRED_INC(SEND); - ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); - #endif /* ENABLE_SPECIALIZATION_FT */ - } - // _SEND - { - v = stack_pointer[-1]; - PyObject *receiver_o = PyStackRef_AsPyObjectBorrow(receiver); - PyObject *retval_o; - assert(frame->owner != FRAME_OWNED_BY_INTERPRETER); - if ((tstate->interp->eval_frame == NULL) && - (Py_TYPE(receiver_o) == &PyGen_Type || Py_TYPE(receiver_o) == &PyCoro_Type) && - ((PyGenObject *)receiver_o)->gi_frame_state < FRAME_EXECUTING) - { - PyGenObject *gen = (PyGenObject *)receiver_o; - _PyInterpreterFrame *gen_frame = &gen->gi_iframe; - _PyFrame_StackPush(gen_frame, PyStackRef_MakeHeapSafe(v)); - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - gen->gi_frame_state = FRAME_EXECUTING; - gen->gi_exc_state.previous_item = tstate->exc_info; - tstate->exc_info = &gen->gi_exc_state; - assert( 2u + oparg <= UINT16_MAX); - frame->return_offset = (uint16_t)( 2u + oparg); - assert(gen_frame->previous == NULL); - gen_frame->previous = frame; - TRACING_DISPATCH_INLINED(gen_frame); - } - if (PyStackRef_IsNone(v) && PyIter_Check(receiver_o)) { - _PyFrame_SetStackPointer(frame, stack_pointer); - retval_o = Py_TYPE(receiver_o)->tp_iternext(receiver_o); - stack_pointer = _PyFrame_GetStackPointer(frame); - } - else { - _PyFrame_SetStackPointer(frame, stack_pointer); - retval_o = PyObject_CallMethodOneArg(receiver_o, - &_Py_ID(send), - PyStackRef_AsPyObjectBorrow(v)); - stack_pointer = _PyFrame_GetStackPointer(frame); - } - if (retval_o == NULL) { - _PyFrame_SetStackPointer(frame, stack_pointer); - int matches = _PyErr_ExceptionMatches(tstate, PyExc_StopIteration); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (matches) { - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyEval_MonitorRaise(tstate, frame, this_instr); - stack_pointer = _PyFrame_GetStackPointer(frame); - } - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = _PyGen_FetchStopIterationValue(&retval_o); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err == 0) { - assert(retval_o != NULL); - JUMPBY(oparg); - } - else { - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(v); - stack_pointer = _PyFrame_GetStackPointer(frame); - TRACING_JUMP_TO_LABEL(error); - } - } - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(v); - stack_pointer = _PyFrame_GetStackPointer(frame); - retval = PyStackRef_FromPyObjectSteal(retval_o); - } - stack_pointer[0] = retval; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - TRACING_DISPATCH(); - } - - TRACING_TARGET(SEND_GEN) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = SEND_GEN; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(SEND_GEN); - opcode = SEND_GEN; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - static_assert(INLINE_CACHE_ENTRIES_SEND == 1, "incorrect cache size"); - _PyStackRef receiver; - _PyStackRef v; - _PyStackRef gen_frame; - _PyStackRef new_frame; - /* Skip 1 cache entry */ - // _CHECK_PEP_523 - { - if (tstate->interp->eval_frame) { - UPDATE_MISS_STATS(SEND); - assert(_PyOpcode_Deopt[opcode] == (SEND)); - JUMP_TO_PREDICTED(TRACING_SEND); - } - } - // _SEND_GEN_FRAME - { - v = stack_pointer[-1]; - receiver = stack_pointer[-2]; - PyGenObject *gen = (PyGenObject *)PyStackRef_AsPyObjectBorrow(receiver); - if (Py_TYPE(gen) != &PyGen_Type && Py_TYPE(gen) != &PyCoro_Type) { - UPDATE_MISS_STATS(SEND); - assert(_PyOpcode_Deopt[opcode] == (SEND)); - JUMP_TO_PREDICTED(TRACING_SEND); - } - if (gen->gi_frame_state >= FRAME_EXECUTING) { - UPDATE_MISS_STATS(SEND); - assert(_PyOpcode_Deopt[opcode] == (SEND)); - JUMP_TO_PREDICTED(TRACING_SEND); - } - STAT_INC(SEND, hit); - _PyInterpreterFrame *pushed_frame = &gen->gi_iframe; - _PyFrame_StackPush(pushed_frame, PyStackRef_MakeHeapSafe(v)); - gen->gi_frame_state = FRAME_EXECUTING; - gen->gi_exc_state.previous_item = tstate->exc_info; - tstate->exc_info = &gen->gi_exc_state; - assert( 2u + oparg <= UINT16_MAX); - frame->return_offset = (uint16_t)( 2u + oparg); - pushed_frame->previous = frame; - gen_frame = PyStackRef_Wrap(pushed_frame); - } - // _PUSH_FRAME - { - new_frame = gen_frame; - assert(tstate->interp->eval_frame == NULL); - _PyInterpreterFrame *temp = PyStackRef_Unwrap(new_frame); - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - assert(temp->previous == frame || temp->previous->previous == frame); - CALL_STAT_INC(inlined_py_calls); - frame = tstate->current_frame = temp; - tstate->py_recursion_remaining--; - LOAD_SP(); - LOAD_IP(0); - LLTRACE_RESUME_FRAME(); - } - TRACING_DISPATCH(); - } - - TRACING_TARGET(SETUP_ANNOTATIONS) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = SETUP_ANNOTATIONS; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(SETUP_ANNOTATIONS); - opcode = SETUP_ANNOTATIONS; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - PyObject *ann_dict; - if (LOCALS() == NULL) { - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyErr_Format(tstate, PyExc_SystemError, - "no locals found when setting up annotations"); - stack_pointer = _PyFrame_GetStackPointer(frame); - TRACING_JUMP_TO_LABEL(error); - } - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = PyMapping_GetOptionalItem(LOCALS(), &_Py_ID(__annotations__), &ann_dict); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err < 0) { - TRACING_JUMP_TO_LABEL(error); - } - if (ann_dict == NULL) { - _PyFrame_SetStackPointer(frame, stack_pointer); - ann_dict = PyDict_New(); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (ann_dict == NULL) { - TRACING_JUMP_TO_LABEL(error); - } - _PyFrame_SetStackPointer(frame, stack_pointer); - err = PyObject_SetItem(LOCALS(), &_Py_ID(__annotations__), - ann_dict); - Py_DECREF(ann_dict); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err) { - TRACING_JUMP_TO_LABEL(error); - } - } - else { - _PyFrame_SetStackPointer(frame, stack_pointer); - Py_DECREF(ann_dict); - stack_pointer = _PyFrame_GetStackPointer(frame); - } - TRACING_DISPATCH(); - } - - TRACING_TARGET(SET_ADD) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = SET_ADD; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(SET_ADD); - opcode = SET_ADD; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - _PyStackRef set; - _PyStackRef v; - v = stack_pointer[-1]; - set = stack_pointer[-2 - (oparg-1)]; - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = _PySet_AddTakeRef((PySetObject *)PyStackRef_AsPyObjectBorrow(set), - PyStackRef_AsPyObjectSteal(v)); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err) { - TRACING_JUMP_TO_LABEL(pop_1_error); - } - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - TRACING_DISPATCH(); - } - - TRACING_TARGET(SET_FUNCTION_ATTRIBUTE) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = SET_FUNCTION_ATTRIBUTE; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(SET_FUNCTION_ATTRIBUTE); - opcode = SET_FUNCTION_ATTRIBUTE; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - _PyStackRef attr_st; - _PyStackRef func_in; - _PyStackRef func_out; - func_in = stack_pointer[-1]; - attr_st = stack_pointer[-2]; - PyObject *func = PyStackRef_AsPyObjectBorrow(func_in); - PyObject *attr = PyStackRef_AsPyObjectSteal(attr_st); - func_out = func_in; - assert(PyFunction_Check(func)); - size_t offset = _Py_FunctionAttributeOffsets[oparg]; - assert(offset != 0); - PyObject **ptr = (PyObject **)(((char *)func) + offset); - assert(*ptr == NULL); - *ptr = attr; - stack_pointer[-2] = func_out; - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - TRACING_DISPATCH(); - } - - TRACING_TARGET(SET_UPDATE) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = SET_UPDATE; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(SET_UPDATE); - opcode = SET_UPDATE; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - _PyStackRef set; - _PyStackRef iterable; - iterable = stack_pointer[-1]; - set = stack_pointer[-2 - (oparg-1)]; - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = _PySet_Update(PyStackRef_AsPyObjectBorrow(set), - PyStackRef_AsPyObjectBorrow(iterable)); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(iterable); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err < 0) { - TRACING_JUMP_TO_LABEL(error); - } - TRACING_DISPATCH(); - } - - TRACING_TARGET(STORE_ATTR) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = STORE_ATTR; - (void)(opcode); - #endif - frame->instr_ptr = next_instr; - next_instr += 5; - INSTRUCTION_STATS(STORE_ATTR); - PREDICTED_TRACING_STORE_ATTR:; - _Py_CODEUNIT* const this_instr = next_instr - 5; - (void)this_instr; - opcode = STORE_ATTR; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - _PyStackRef owner; - _PyStackRef v; - // _SPECIALIZE_STORE_ATTR - { - owner = stack_pointer[-1]; - uint16_t counter = read_u16(&this_instr[1].cache); - (void)counter; - #if ENABLE_SPECIALIZATION_FT - if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { - PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); - next_instr = this_instr; - _PyFrame_SetStackPointer(frame, stack_pointer); - _Py_Specialize_StoreAttr(owner, next_instr, name); - stack_pointer = _PyFrame_GetStackPointer(frame); - DISPATCH_SAME_OPARG(); - } - OPCODE_DEFERRED_INC(STORE_ATTR); - ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); - #endif /* ENABLE_SPECIALIZATION_FT */ - } - /* Skip 3 cache entries */ - // _STORE_ATTR - { - v = stack_pointer[-2]; - PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = PyObject_SetAttr(PyStackRef_AsPyObjectBorrow(owner), - name, PyStackRef_AsPyObjectBorrow(v)); - _PyStackRef tmp = owner; - owner = PyStackRef_NULL; - stack_pointer[-1] = owner; - PyStackRef_CLOSE(tmp); - tmp = v; - v = PyStackRef_NULL; - stack_pointer[-2] = v; - PyStackRef_CLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); - if (err) { - TRACING_JUMP_TO_LABEL(error); - } - } - TRACING_DISPATCH(); - } - - TRACING_TARGET(STORE_ATTR_INSTANCE_VALUE) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = STORE_ATTR_INSTANCE_VALUE; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 5; - INSTRUCTION_STATS(STORE_ATTR_INSTANCE_VALUE); - opcode = STORE_ATTR_INSTANCE_VALUE; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - static_assert(INLINE_CACHE_ENTRIES_STORE_ATTR == 4, "incorrect cache size"); - _PyStackRef owner; - _PyStackRef value; - /* Skip 1 cache entry */ - // _GUARD_TYPE_VERSION_AND_LOCK - { - owner = stack_pointer[-1]; - uint32_t type_version = read_u32(&this_instr[2].cache); - PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); - assert(type_version != 0); - if (!LOCK_OBJECT(owner_o)) { - UPDATE_MISS_STATS(STORE_ATTR); - assert(_PyOpcode_Deopt[opcode] == (STORE_ATTR)); - JUMP_TO_PREDICTED(TRACING_STORE_ATTR); - } - PyTypeObject *tp = Py_TYPE(owner_o); - if (FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version) { - UNLOCK_OBJECT(owner_o); - if (true) { - UPDATE_MISS_STATS(STORE_ATTR); - assert(_PyOpcode_Deopt[opcode] == (STORE_ATTR)); - JUMP_TO_PREDICTED(TRACING_STORE_ATTR); - } - } - } - // _GUARD_DORV_NO_DICT - { - PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); - assert(Py_TYPE(owner_o)->tp_dictoffset < 0); - assert(Py_TYPE(owner_o)->tp_flags & Py_TPFLAGS_INLINE_VALUES); - if (_PyObject_GetManagedDict(owner_o) || - !FT_ATOMIC_LOAD_UINT8(_PyObject_InlineValues(owner_o)->valid)) { - UNLOCK_OBJECT(owner_o); - if (true) { - UPDATE_MISS_STATS(STORE_ATTR); - assert(_PyOpcode_Deopt[opcode] == (STORE_ATTR)); - JUMP_TO_PREDICTED(TRACING_STORE_ATTR); - } - } - } - // _STORE_ATTR_INSTANCE_VALUE - { - value = stack_pointer[-2]; - uint16_t offset = read_u16(&this_instr[4].cache); - PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); - STAT_INC(STORE_ATTR, hit); - assert(_PyObject_GetManagedDict(owner_o) == NULL); - PyObject **value_ptr = (PyObject**)(((char *)owner_o) + offset); - PyObject *old_value = *value_ptr; - FT_ATOMIC_STORE_PTR_RELEASE(*value_ptr, PyStackRef_AsPyObjectSteal(value)); - if (old_value == NULL) { - PyDictValues *values = _PyObject_InlineValues(owner_o); - Py_ssize_t index = value_ptr - values->values; - _PyDictValues_AddToInsertionOrder(values, index); - } - UNLOCK_OBJECT(owner_o); - stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(owner); - Py_XDECREF(old_value); - stack_pointer = _PyFrame_GetStackPointer(frame); - } - TRACING_DISPATCH(); - } - - TRACING_TARGET(STORE_ATTR_SLOT) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = STORE_ATTR_SLOT; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 5; - INSTRUCTION_STATS(STORE_ATTR_SLOT); - opcode = STORE_ATTR_SLOT; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - static_assert(INLINE_CACHE_ENTRIES_STORE_ATTR == 4, "incorrect cache size"); - _PyStackRef owner; - _PyStackRef value; - /* Skip 1 cache entry */ - // _GUARD_TYPE_VERSION - { - owner = stack_pointer[-1]; - uint32_t type_version = read_u32(&this_instr[2].cache); - PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); - assert(type_version != 0); - if (FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version) { - UPDATE_MISS_STATS(STORE_ATTR); - assert(_PyOpcode_Deopt[opcode] == (STORE_ATTR)); - JUMP_TO_PREDICTED(TRACING_STORE_ATTR); - } - } - // _STORE_ATTR_SLOT - { - value = stack_pointer[-2]; - uint16_t index = read_u16(&this_instr[4].cache); - PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); - if (!LOCK_OBJECT(owner_o)) { - UPDATE_MISS_STATS(STORE_ATTR); - assert(_PyOpcode_Deopt[opcode] == (STORE_ATTR)); - JUMP_TO_PREDICTED(TRACING_STORE_ATTR); - } - char *addr = (char *)owner_o + index; - STAT_INC(STORE_ATTR, hit); - PyObject *old_value = *(PyObject **)addr; - FT_ATOMIC_STORE_PTR_RELEASE(*(PyObject **)addr, PyStackRef_AsPyObjectSteal(value)); - UNLOCK_OBJECT(owner_o); - stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(owner); - Py_XDECREF(old_value); - stack_pointer = _PyFrame_GetStackPointer(frame); - } - TRACING_DISPATCH(); - } - - TRACING_TARGET(STORE_ATTR_WITH_HINT) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = STORE_ATTR_WITH_HINT; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 5; - INSTRUCTION_STATS(STORE_ATTR_WITH_HINT); - opcode = STORE_ATTR_WITH_HINT; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - static_assert(INLINE_CACHE_ENTRIES_STORE_ATTR == 4, "incorrect cache size"); - _PyStackRef owner; - _PyStackRef value; - /* Skip 1 cache entry */ - // _GUARD_TYPE_VERSION - { - owner = stack_pointer[-1]; - uint32_t type_version = read_u32(&this_instr[2].cache); - PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); - assert(type_version != 0); - if (FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version) { - UPDATE_MISS_STATS(STORE_ATTR); - assert(_PyOpcode_Deopt[opcode] == (STORE_ATTR)); - JUMP_TO_PREDICTED(TRACING_STORE_ATTR); - } - } - // _STORE_ATTR_WITH_HINT - { - value = stack_pointer[-2]; - uint16_t hint = read_u16(&this_instr[4].cache); - PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); - assert(Py_TYPE(owner_o)->tp_flags & Py_TPFLAGS_MANAGED_DICT); - PyDictObject *dict = _PyObject_GetManagedDict(owner_o); - if (dict == NULL) { - UPDATE_MISS_STATS(STORE_ATTR); - assert(_PyOpcode_Deopt[opcode] == (STORE_ATTR)); - JUMP_TO_PREDICTED(TRACING_STORE_ATTR); - } - if (!LOCK_OBJECT(dict)) { - UPDATE_MISS_STATS(STORE_ATTR); - assert(_PyOpcode_Deopt[opcode] == (STORE_ATTR)); - JUMP_TO_PREDICTED(TRACING_STORE_ATTR); - } - assert(PyDict_CheckExact((PyObject *)dict)); - PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); - if (hint >= (size_t)dict->ma_keys->dk_nentries || - !DK_IS_UNICODE(dict->ma_keys)) { - UNLOCK_OBJECT(dict); - if (true) { - UPDATE_MISS_STATS(STORE_ATTR); - assert(_PyOpcode_Deopt[opcode] == (STORE_ATTR)); - JUMP_TO_PREDICTED(TRACING_STORE_ATTR); - } - } - PyDictUnicodeEntry *ep = DK_UNICODE_ENTRIES(dict->ma_keys) + hint; - if (ep->me_key != name) { - UNLOCK_OBJECT(dict); - if (true) { - UPDATE_MISS_STATS(STORE_ATTR); - assert(_PyOpcode_Deopt[opcode] == (STORE_ATTR)); - JUMP_TO_PREDICTED(TRACING_STORE_ATTR); - } - } - PyObject *old_value = ep->me_value; - if (old_value == NULL) { - UNLOCK_OBJECT(dict); - if (true) { - UPDATE_MISS_STATS(STORE_ATTR); - assert(_PyOpcode_Deopt[opcode] == (STORE_ATTR)); - JUMP_TO_PREDICTED(TRACING_STORE_ATTR); - } - } - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyDict_NotifyEvent(tstate->interp, PyDict_EVENT_MODIFIED, dict, name, PyStackRef_AsPyObjectBorrow(value)); - stack_pointer = _PyFrame_GetStackPointer(frame); - FT_ATOMIC_STORE_PTR_RELEASE(ep->me_value, PyStackRef_AsPyObjectSteal(value)); - UNLOCK_OBJECT(dict); - STAT_INC(STORE_ATTR, hit); - stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(owner); - Py_XDECREF(old_value); - stack_pointer = _PyFrame_GetStackPointer(frame); - } - TRACING_DISPATCH(); - } - - TRACING_TARGET(STORE_DEREF) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = STORE_DEREF; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(STORE_DEREF); - opcode = STORE_DEREF; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - _PyStackRef v; - v = stack_pointer[-1]; - PyCellObject *cell = (PyCellObject *)PyStackRef_AsPyObjectBorrow(GETLOCAL(oparg)); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyCell_SetTakeRef(cell, PyStackRef_AsPyObjectSteal(v)); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - TRACING_DISPATCH(); - } - - TRACING_TARGET(STORE_FAST) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = STORE_FAST; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(STORE_FAST); - opcode = STORE_FAST; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - _PyStackRef value; - value = stack_pointer[-1]; - _PyStackRef tmp = GETLOCAL(oparg); - GETLOCAL(oparg) = value; - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_XCLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); - TRACING_DISPATCH(); - } - - TRACING_TARGET(STORE_FAST_LOAD_FAST) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = STORE_FAST_LOAD_FAST; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(STORE_FAST_LOAD_FAST); - opcode = STORE_FAST_LOAD_FAST; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - _PyStackRef value1; - _PyStackRef value2; - value1 = stack_pointer[-1]; - uint32_t oparg1 = oparg >> 4; - uint32_t oparg2 = oparg & 15; - _PyStackRef tmp = GETLOCAL(oparg1); - GETLOCAL(oparg1) = value1; - value2 = PyStackRef_DUP(GETLOCAL(oparg2)); - stack_pointer[-1] = value2; - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_XCLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); - TRACING_DISPATCH(); - } - - TRACING_TARGET(STORE_FAST_STORE_FAST) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = STORE_FAST_STORE_FAST; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(STORE_FAST_STORE_FAST); - opcode = STORE_FAST_STORE_FAST; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - _PyStackRef value2; - _PyStackRef value1; - value1 = stack_pointer[-1]; - value2 = stack_pointer[-2]; - uint32_t oparg1 = oparg >> 4; - uint32_t oparg2 = oparg & 15; - _PyStackRef tmp = GETLOCAL(oparg1); - GETLOCAL(oparg1) = value1; - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_XCLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); - tmp = GETLOCAL(oparg2); - GETLOCAL(oparg2) = value2; - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_XCLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); - TRACING_DISPATCH(); - } - - TRACING_TARGET(STORE_GLOBAL) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = STORE_GLOBAL; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(STORE_GLOBAL); - opcode = STORE_GLOBAL; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - _PyStackRef v; - v = stack_pointer[-1]; - PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = PyDict_SetItem(GLOBALS(), name, PyStackRef_AsPyObjectBorrow(v)); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(v); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err) { - TRACING_JUMP_TO_LABEL(error); - } - TRACING_DISPATCH(); - } - - TRACING_TARGET(STORE_NAME) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = STORE_NAME; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(STORE_NAME); - opcode = STORE_NAME; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - _PyStackRef v; - v = stack_pointer[-1]; - PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); - PyObject *ns = LOCALS(); - int err; - if (ns == NULL) { - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyErr_Format(tstate, PyExc_SystemError, - "no locals found when storing %R", name); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(v); - stack_pointer = _PyFrame_GetStackPointer(frame); - TRACING_JUMP_TO_LABEL(error); - } - if (PyDict_CheckExact(ns)) { - _PyFrame_SetStackPointer(frame, stack_pointer); - err = PyDict_SetItem(ns, name, PyStackRef_AsPyObjectBorrow(v)); - stack_pointer = _PyFrame_GetStackPointer(frame); - } - else { - _PyFrame_SetStackPointer(frame, stack_pointer); - err = PyObject_SetItem(ns, name, PyStackRef_AsPyObjectBorrow(v)); - stack_pointer = _PyFrame_GetStackPointer(frame); - } - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(v); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err) { - TRACING_JUMP_TO_LABEL(error); - } - TRACING_DISPATCH(); - } - - TRACING_TARGET(STORE_SLICE) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = STORE_SLICE; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(STORE_SLICE); - opcode = STORE_SLICE; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - _PyStackRef v; - _PyStackRef container; - _PyStackRef start; - _PyStackRef stop; - // _SPECIALIZE_STORE_SLICE - { - #if ENABLE_SPECIALIZATION - OPCODE_DEFERRED_INC(STORE_SLICE); - #endif /* ENABLE_SPECIALIZATION */ - } - // _STORE_SLICE - { - stop = stack_pointer[-1]; - start = stack_pointer[-2]; - container = stack_pointer[-3]; - v = stack_pointer[-4]; - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *slice = _PyBuildSlice_ConsumeRefs(PyStackRef_AsPyObjectSteal(start), - PyStackRef_AsPyObjectSteal(stop)); - stack_pointer = _PyFrame_GetStackPointer(frame); - int err; - if (slice == NULL) { - err = 1; - } - else { - stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - err = PyObject_SetItem(PyStackRef_AsPyObjectBorrow(container), slice, PyStackRef_AsPyObjectBorrow(v)); - Py_DECREF(slice); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += 2; - } - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyStackRef tmp = container; - container = PyStackRef_NULL; - stack_pointer[-3] = container; - PyStackRef_CLOSE(tmp); - tmp = v; - v = PyStackRef_NULL; - stack_pointer[-4] = v; - PyStackRef_CLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -4; - assert(WITHIN_STACK_BOUNDS()); - if (err) { - TRACING_JUMP_TO_LABEL(error); - } - } - TRACING_DISPATCH(); - } - - TRACING_TARGET(STORE_SUBSCR) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = STORE_SUBSCR; - (void)(opcode); - #endif - frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(STORE_SUBSCR); - PREDICTED_TRACING_STORE_SUBSCR:; - _Py_CODEUNIT* const this_instr = next_instr - 2; - (void)this_instr; - opcode = STORE_SUBSCR; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - _PyStackRef container; - _PyStackRef sub; - _PyStackRef v; - // _SPECIALIZE_STORE_SUBSCR - { - sub = stack_pointer[-1]; - container = stack_pointer[-2]; - uint16_t counter = read_u16(&this_instr[1].cache); - (void)counter; - #if ENABLE_SPECIALIZATION_FT - if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { - next_instr = this_instr; - _PyFrame_SetStackPointer(frame, stack_pointer); - _Py_Specialize_StoreSubscr(container, sub, next_instr); - stack_pointer = _PyFrame_GetStackPointer(frame); - DISPATCH_SAME_OPARG(); - } - OPCODE_DEFERRED_INC(STORE_SUBSCR); - ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); - #endif /* ENABLE_SPECIALIZATION_FT */ - } - // _STORE_SUBSCR - { - v = stack_pointer[-3]; - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = PyObject_SetItem(PyStackRef_AsPyObjectBorrow(container), PyStackRef_AsPyObjectBorrow(sub), PyStackRef_AsPyObjectBorrow(v)); - _PyStackRef tmp = sub; - sub = PyStackRef_NULL; - stack_pointer[-1] = sub; - PyStackRef_CLOSE(tmp); - tmp = container; - container = PyStackRef_NULL; - stack_pointer[-2] = container; - PyStackRef_CLOSE(tmp); - tmp = v; - v = PyStackRef_NULL; - stack_pointer[-3] = v; - PyStackRef_CLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -3; - assert(WITHIN_STACK_BOUNDS()); - if (err) { - TRACING_JUMP_TO_LABEL(error); - } - } - TRACING_DISPATCH(); - } - - TRACING_TARGET(STORE_SUBSCR_DICT) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = STORE_SUBSCR_DICT; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(STORE_SUBSCR_DICT); - opcode = STORE_SUBSCR_DICT; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - static_assert(INLINE_CACHE_ENTRIES_STORE_SUBSCR == 1, "incorrect cache size"); - _PyStackRef nos; - _PyStackRef value; - _PyStackRef dict_st; - _PyStackRef sub; - // _GUARD_NOS_DICT - { - nos = stack_pointer[-2]; - PyObject *o = PyStackRef_AsPyObjectBorrow(nos); - if (!PyDict_CheckExact(o)) { - UPDATE_MISS_STATS(STORE_SUBSCR); - assert(_PyOpcode_Deopt[opcode] == (STORE_SUBSCR)); - JUMP_TO_PREDICTED(TRACING_STORE_SUBSCR); - } - } - /* Skip 1 cache entry */ - // _STORE_SUBSCR_DICT - { - sub = stack_pointer[-1]; - dict_st = nos; - value = stack_pointer[-3]; - PyObject *dict = PyStackRef_AsPyObjectBorrow(dict_st); - assert(PyDict_CheckExact(dict)); - STAT_INC(STORE_SUBSCR, hit); - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = _PyDict_SetItem_Take2((PyDictObject *)dict, - PyStackRef_AsPyObjectSteal(sub), - PyStackRef_AsPyObjectSteal(value)); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -3; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(dict_st); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err) { - TRACING_JUMP_TO_LABEL(error); - } - } - TRACING_DISPATCH(); - } - - TRACING_TARGET(STORE_SUBSCR_LIST_INT) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = STORE_SUBSCR_LIST_INT; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(STORE_SUBSCR_LIST_INT); - opcode = STORE_SUBSCR_LIST_INT; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - static_assert(INLINE_CACHE_ENTRIES_STORE_SUBSCR == 1, "incorrect cache size"); - _PyStackRef value; - _PyStackRef nos; - _PyStackRef list_st; - _PyStackRef sub_st; - // _GUARD_TOS_INT - { - value = stack_pointer[-1]; - PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); - if (!_PyLong_CheckExactAndCompact(value_o)) { - UPDATE_MISS_STATS(STORE_SUBSCR); - assert(_PyOpcode_Deopt[opcode] == (STORE_SUBSCR)); - JUMP_TO_PREDICTED(TRACING_STORE_SUBSCR); - } - } - // _GUARD_NOS_LIST - { - nos = stack_pointer[-2]; - PyObject *o = PyStackRef_AsPyObjectBorrow(nos); - if (!PyList_CheckExact(o)) { - UPDATE_MISS_STATS(STORE_SUBSCR); - assert(_PyOpcode_Deopt[opcode] == (STORE_SUBSCR)); - JUMP_TO_PREDICTED(TRACING_STORE_SUBSCR); - } - } - /* Skip 1 cache entry */ - // _STORE_SUBSCR_LIST_INT - { - sub_st = value; - list_st = nos; - value = stack_pointer[-3]; - PyObject *sub = PyStackRef_AsPyObjectBorrow(sub_st); - PyObject *list = PyStackRef_AsPyObjectBorrow(list_st); - assert(PyLong_CheckExact(sub)); - assert(PyList_CheckExact(list)); - if (!_PyLong_IsNonNegativeCompact((PyLongObject *)sub)) { - UPDATE_MISS_STATS(STORE_SUBSCR); - assert(_PyOpcode_Deopt[opcode] == (STORE_SUBSCR)); - JUMP_TO_PREDICTED(TRACING_STORE_SUBSCR); - } - Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0]; - if (!LOCK_OBJECT(list)) { - UPDATE_MISS_STATS(STORE_SUBSCR); - assert(_PyOpcode_Deopt[opcode] == (STORE_SUBSCR)); - JUMP_TO_PREDICTED(TRACING_STORE_SUBSCR); - } - if (index >= PyList_GET_SIZE(list)) { - UNLOCK_OBJECT(list); - if (true) { - UPDATE_MISS_STATS(STORE_SUBSCR); - assert(_PyOpcode_Deopt[opcode] == (STORE_SUBSCR)); - JUMP_TO_PREDICTED(TRACING_STORE_SUBSCR); - } - } - STAT_INC(STORE_SUBSCR, hit); - PyObject *old_value = PyList_GET_ITEM(list, index); - FT_ATOMIC_STORE_PTR_RELEASE(_PyList_ITEMS(list)[index], - PyStackRef_AsPyObjectSteal(value)); - assert(old_value != NULL); - UNLOCK_OBJECT(list); - PyStackRef_CLOSE_SPECIALIZED(sub_st, _PyLong_ExactDealloc); - stack_pointer += -3; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(list_st); - Py_DECREF(old_value); - stack_pointer = _PyFrame_GetStackPointer(frame); - } - TRACING_DISPATCH(); - } - - TRACING_TARGET(SWAP) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = SWAP; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(SWAP); - opcode = SWAP; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - _PyStackRef bottom; - _PyStackRef top; - top = stack_pointer[-1]; - bottom = stack_pointer[-2 - (oparg-2)]; - _PyStackRef temp = bottom; - bottom = top; - top = temp; - stack_pointer[-2 - (oparg-2)] = bottom; - stack_pointer[-1] = top; - TRACING_DISPATCH(); - } - - TRACING_TARGET(TO_BOOL) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = TO_BOOL; - (void)(opcode); - #endif - frame->instr_ptr = next_instr; - next_instr += 4; - INSTRUCTION_STATS(TO_BOOL); - PREDICTED_TRACING_TO_BOOL:; - _Py_CODEUNIT* const this_instr = next_instr - 4; - (void)this_instr; - opcode = TO_BOOL; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - _PyStackRef value; - _PyStackRef res; - // _SPECIALIZE_TO_BOOL - { - value = stack_pointer[-1]; - uint16_t counter = read_u16(&this_instr[1].cache); - (void)counter; - #if ENABLE_SPECIALIZATION_FT - if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { - next_instr = this_instr; - _PyFrame_SetStackPointer(frame, stack_pointer); - _Py_Specialize_ToBool(value, next_instr); - stack_pointer = _PyFrame_GetStackPointer(frame); - DISPATCH_SAME_OPARG(); - } - OPCODE_DEFERRED_INC(TO_BOOL); - ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); - #endif /* ENABLE_SPECIALIZATION_FT */ - } - /* Skip 2 cache entries */ - // _TO_BOOL - { - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = PyObject_IsTrue(PyStackRef_AsPyObjectBorrow(value)); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(value); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err < 0) { - TRACING_JUMP_TO_LABEL(error); - } - res = err ? PyStackRef_True : PyStackRef_False; - } - stack_pointer[0] = res; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - TRACING_DISPATCH(); - } - - TRACING_TARGET(TO_BOOL_ALWAYS_TRUE) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = TO_BOOL_ALWAYS_TRUE; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 4; - INSTRUCTION_STATS(TO_BOOL_ALWAYS_TRUE); - opcode = TO_BOOL_ALWAYS_TRUE; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - static_assert(INLINE_CACHE_ENTRIES_TO_BOOL == 3, "incorrect cache size"); - _PyStackRef owner; - _PyStackRef value; - _PyStackRef res; - /* Skip 1 cache entry */ - // _GUARD_TYPE_VERSION - { - owner = stack_pointer[-1]; - uint32_t type_version = read_u32(&this_instr[2].cache); - PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); - assert(type_version != 0); - if (FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version) { - UPDATE_MISS_STATS(TO_BOOL); - assert(_PyOpcode_Deopt[opcode] == (TO_BOOL)); - JUMP_TO_PREDICTED(TRACING_TO_BOOL); - } - } - // _REPLACE_WITH_TRUE - { - value = owner; - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(value); - stack_pointer = _PyFrame_GetStackPointer(frame); - res = PyStackRef_True; - } - stack_pointer[0] = res; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - TRACING_DISPATCH(); - } - - TRACING_TARGET(TO_BOOL_BOOL) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = TO_BOOL_BOOL; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 4; - INSTRUCTION_STATS(TO_BOOL_BOOL); - opcode = TO_BOOL_BOOL; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - static_assert(INLINE_CACHE_ENTRIES_TO_BOOL == 3, "incorrect cache size"); - _PyStackRef value; - /* Skip 1 cache entry */ - /* Skip 2 cache entries */ - value = stack_pointer[-1]; - if (!PyStackRef_BoolCheck(value)) { - UPDATE_MISS_STATS(TO_BOOL); - assert(_PyOpcode_Deopt[opcode] == (TO_BOOL)); - JUMP_TO_PREDICTED(TRACING_TO_BOOL); - } - STAT_INC(TO_BOOL, hit); - TRACING_DISPATCH(); - } - - TRACING_TARGET(TO_BOOL_INT) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = TO_BOOL_INT; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 4; - INSTRUCTION_STATS(TO_BOOL_INT); - opcode = TO_BOOL_INT; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - static_assert(INLINE_CACHE_ENTRIES_TO_BOOL == 3, "incorrect cache size"); - _PyStackRef value; - _PyStackRef res; - /* Skip 1 cache entry */ - /* Skip 2 cache entries */ - value = stack_pointer[-1]; - PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); - if (!PyLong_CheckExact(value_o)) { - UPDATE_MISS_STATS(TO_BOOL); - assert(_PyOpcode_Deopt[opcode] == (TO_BOOL)); - JUMP_TO_PREDICTED(TRACING_TO_BOOL); - } - STAT_INC(TO_BOOL, hit); - if (_PyLong_IsZero((PyLongObject *)value_o)) { - assert(_Py_IsImmortal(value_o)); - res = PyStackRef_False; - } - else { - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(value); - stack_pointer = _PyFrame_GetStackPointer(frame); - res = PyStackRef_True; - stack_pointer += 1; - } - stack_pointer[-1] = res; - TRACING_DISPATCH(); - } - - TRACING_TARGET(TO_BOOL_LIST) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = TO_BOOL_LIST; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 4; - INSTRUCTION_STATS(TO_BOOL_LIST); - opcode = TO_BOOL_LIST; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - static_assert(INLINE_CACHE_ENTRIES_TO_BOOL == 3, "incorrect cache size"); - _PyStackRef tos; - _PyStackRef value; - _PyStackRef res; - // _GUARD_TOS_LIST - { - tos = stack_pointer[-1]; - PyObject *o = PyStackRef_AsPyObjectBorrow(tos); - if (!PyList_CheckExact(o)) { - UPDATE_MISS_STATS(TO_BOOL); - assert(_PyOpcode_Deopt[opcode] == (TO_BOOL)); - JUMP_TO_PREDICTED(TRACING_TO_BOOL); - } - } - /* Skip 1 cache entry */ - /* Skip 2 cache entries */ - // _TO_BOOL_LIST - { - value = tos; - PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); - assert(PyList_CheckExact(value_o)); - STAT_INC(TO_BOOL, hit); - res = PyList_GET_SIZE(value_o) ? PyStackRef_True : PyStackRef_False; - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyStackRef tmp = value; - value = res; - stack_pointer[-1] = value; - PyStackRef_CLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); - } - TRACING_DISPATCH(); - } - - TRACING_TARGET(TO_BOOL_NONE) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = TO_BOOL_NONE; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 4; - INSTRUCTION_STATS(TO_BOOL_NONE); - opcode = TO_BOOL_NONE; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - static_assert(INLINE_CACHE_ENTRIES_TO_BOOL == 3, "incorrect cache size"); - _PyStackRef value; - _PyStackRef res; - /* Skip 1 cache entry */ - /* Skip 2 cache entries */ - value = stack_pointer[-1]; - if (!PyStackRef_IsNone(value)) { - UPDATE_MISS_STATS(TO_BOOL); - assert(_PyOpcode_Deopt[opcode] == (TO_BOOL)); - JUMP_TO_PREDICTED(TRACING_TO_BOOL); - } - STAT_INC(TO_BOOL, hit); - res = PyStackRef_False; - stack_pointer[-1] = res; - TRACING_DISPATCH(); - } - - TRACING_TARGET(TO_BOOL_STR) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = TO_BOOL_STR; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 4; - INSTRUCTION_STATS(TO_BOOL_STR); - opcode = TO_BOOL_STR; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - static_assert(INLINE_CACHE_ENTRIES_TO_BOOL == 3, "incorrect cache size"); - _PyStackRef value; - _PyStackRef res; - // _GUARD_TOS_UNICODE - { - value = stack_pointer[-1]; - PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); - if (!PyUnicode_CheckExact(value_o)) { - UPDATE_MISS_STATS(TO_BOOL); - assert(_PyOpcode_Deopt[opcode] == (TO_BOOL)); - JUMP_TO_PREDICTED(TRACING_TO_BOOL); - } - } - /* Skip 1 cache entry */ - /* Skip 2 cache entries */ - // _TO_BOOL_STR - { - STAT_INC(TO_BOOL, hit); - PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); - if (value_o == &_Py_STR(empty)) { - assert(_Py_IsImmortal(value_o)); - res = PyStackRef_False; - } - else { - assert(Py_SIZE(value_o)); - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(value); - stack_pointer = _PyFrame_GetStackPointer(frame); - res = PyStackRef_True; - stack_pointer += 1; - } - } - stack_pointer[-1] = res; - TRACING_DISPATCH(); - } - - TRACING_TARGET(UNARY_INVERT) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = UNARY_INVERT; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(UNARY_INVERT); - opcode = UNARY_INVERT; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - _PyStackRef value; - _PyStackRef res; - value = stack_pointer[-1]; - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *res_o = PyNumber_Invert(PyStackRef_AsPyObjectBorrow(value)); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(value); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (res_o == NULL) { - TRACING_JUMP_TO_LABEL(error); - } - res = PyStackRef_FromPyObjectSteal(res_o); - stack_pointer[0] = res; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - TRACING_DISPATCH(); - } - - TRACING_TARGET(UNARY_NEGATIVE) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = UNARY_NEGATIVE; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(UNARY_NEGATIVE); - opcode = UNARY_NEGATIVE; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - _PyStackRef value; - _PyStackRef res; - value = stack_pointer[-1]; - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *res_o = PyNumber_Negative(PyStackRef_AsPyObjectBorrow(value)); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(value); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (res_o == NULL) { - TRACING_JUMP_TO_LABEL(error); - } - res = PyStackRef_FromPyObjectSteal(res_o); - stack_pointer[0] = res; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - TRACING_DISPATCH(); - } - - TRACING_TARGET(UNARY_NOT) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = UNARY_NOT; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(UNARY_NOT); - opcode = UNARY_NOT; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - _PyStackRef value; - _PyStackRef res; - value = stack_pointer[-1]; - assert(PyStackRef_BoolCheck(value)); - res = PyStackRef_IsFalse(value) - ? PyStackRef_True : PyStackRef_False; - stack_pointer[-1] = res; - TRACING_DISPATCH(); - } - - TRACING_TARGET(UNPACK_EX) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = UNPACK_EX; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(UNPACK_EX); - opcode = UNPACK_EX; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - _PyStackRef seq; - _PyStackRef *top; - seq = stack_pointer[-1]; - top = &stack_pointer[(oparg & 0xFF) + (oparg >> 8)]; - PyObject *seq_o = PyStackRef_AsPyObjectSteal(seq); - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - int res = _PyEval_UnpackIterableStackRef(tstate, seq_o, oparg & 0xFF, oparg >> 8, top); - Py_DECREF(seq_o); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (res == 0) { - TRACING_JUMP_TO_LABEL(error); - } - stack_pointer += 1 + (oparg & 0xFF) + (oparg >> 8); - assert(WITHIN_STACK_BOUNDS()); - TRACING_DISPATCH(); - } - - TRACING_TARGET(UNPACK_SEQUENCE) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = UNPACK_SEQUENCE; - (void)(opcode); - #endif - frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(UNPACK_SEQUENCE); - PREDICTED_TRACING_UNPACK_SEQUENCE:; - _Py_CODEUNIT* const this_instr = next_instr - 2; - (void)this_instr; - opcode = UNPACK_SEQUENCE; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - _PyStackRef seq; - _PyStackRef *top; - // _SPECIALIZE_UNPACK_SEQUENCE - { - seq = stack_pointer[-1]; - uint16_t counter = read_u16(&this_instr[1].cache); - (void)counter; - #if ENABLE_SPECIALIZATION_FT - if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { - next_instr = this_instr; - _PyFrame_SetStackPointer(frame, stack_pointer); - _Py_Specialize_UnpackSequence(seq, next_instr, oparg); - stack_pointer = _PyFrame_GetStackPointer(frame); - DISPATCH_SAME_OPARG(); - } - OPCODE_DEFERRED_INC(UNPACK_SEQUENCE); - ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); - #endif /* ENABLE_SPECIALIZATION_FT */ - (void)seq; - (void)counter; - } - // _UNPACK_SEQUENCE - { - top = &stack_pointer[-1 + oparg]; - PyObject *seq_o = PyStackRef_AsPyObjectSteal(seq); - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - int res = _PyEval_UnpackIterableStackRef(tstate, seq_o, oparg, -1, top); - Py_DECREF(seq_o); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (res == 0) { - TRACING_JUMP_TO_LABEL(error); - } - } - stack_pointer += oparg; - assert(WITHIN_STACK_BOUNDS()); - TRACING_DISPATCH(); - } - - TRACING_TARGET(UNPACK_SEQUENCE_LIST) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = UNPACK_SEQUENCE_LIST; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(UNPACK_SEQUENCE_LIST); - opcode = UNPACK_SEQUENCE_LIST; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - static_assert(INLINE_CACHE_ENTRIES_UNPACK_SEQUENCE == 1, "incorrect cache size"); - _PyStackRef tos; - _PyStackRef seq; - _PyStackRef *values; - // _GUARD_TOS_LIST - { - tos = stack_pointer[-1]; - PyObject *o = PyStackRef_AsPyObjectBorrow(tos); - if (!PyList_CheckExact(o)) { - UPDATE_MISS_STATS(UNPACK_SEQUENCE); - assert(_PyOpcode_Deopt[opcode] == (UNPACK_SEQUENCE)); - JUMP_TO_PREDICTED(TRACING_UNPACK_SEQUENCE); - } - } - /* Skip 1 cache entry */ - // _UNPACK_SEQUENCE_LIST - { - seq = tos; - values = &stack_pointer[-1]; - PyObject *seq_o = PyStackRef_AsPyObjectBorrow(seq); - assert(PyList_CheckExact(seq_o)); - if (!LOCK_OBJECT(seq_o)) { - UPDATE_MISS_STATS(UNPACK_SEQUENCE); - assert(_PyOpcode_Deopt[opcode] == (UNPACK_SEQUENCE)); - JUMP_TO_PREDICTED(TRACING_UNPACK_SEQUENCE); - } - if (PyList_GET_SIZE(seq_o) != oparg) { - UNLOCK_OBJECT(seq_o); - if (true) { - UPDATE_MISS_STATS(UNPACK_SEQUENCE); - assert(_PyOpcode_Deopt[opcode] == (UNPACK_SEQUENCE)); - JUMP_TO_PREDICTED(TRACING_UNPACK_SEQUENCE); - } - } - STAT_INC(UNPACK_SEQUENCE, hit); - PyObject **items = _PyList_ITEMS(seq_o); - for (int i = oparg; --i >= 0; ) { - *values++ = PyStackRef_FromPyObjectNew(items[i]); - } - UNLOCK_OBJECT(seq_o); - stack_pointer += -1 + oparg; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(seq); - stack_pointer = _PyFrame_GetStackPointer(frame); - } - TRACING_DISPATCH(); - } - - TRACING_TARGET(UNPACK_SEQUENCE_TUPLE) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = UNPACK_SEQUENCE_TUPLE; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(UNPACK_SEQUENCE_TUPLE); - opcode = UNPACK_SEQUENCE_TUPLE; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - static_assert(INLINE_CACHE_ENTRIES_UNPACK_SEQUENCE == 1, "incorrect cache size"); - _PyStackRef tos; - _PyStackRef seq; - _PyStackRef *values; - // _GUARD_TOS_TUPLE - { - tos = stack_pointer[-1]; - PyObject *o = PyStackRef_AsPyObjectBorrow(tos); - if (!PyTuple_CheckExact(o)) { - UPDATE_MISS_STATS(UNPACK_SEQUENCE); - assert(_PyOpcode_Deopt[opcode] == (UNPACK_SEQUENCE)); - JUMP_TO_PREDICTED(TRACING_UNPACK_SEQUENCE); - } - } - /* Skip 1 cache entry */ - // _UNPACK_SEQUENCE_TUPLE - { - seq = tos; - values = &stack_pointer[-1]; - PyObject *seq_o = PyStackRef_AsPyObjectBorrow(seq); - assert(PyTuple_CheckExact(seq_o)); - if (PyTuple_GET_SIZE(seq_o) != oparg) { - UPDATE_MISS_STATS(UNPACK_SEQUENCE); - assert(_PyOpcode_Deopt[opcode] == (UNPACK_SEQUENCE)); - JUMP_TO_PREDICTED(TRACING_UNPACK_SEQUENCE); - } - STAT_INC(UNPACK_SEQUENCE, hit); - PyObject **items = _PyTuple_ITEMS(seq_o); - for (int i = oparg; --i >= 0; ) { - *values++ = PyStackRef_FromPyObjectNew(items[i]); - } - stack_pointer += -1 + oparg; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(seq); - stack_pointer = _PyFrame_GetStackPointer(frame); - } - TRACING_DISPATCH(); - } - - TRACING_TARGET(UNPACK_SEQUENCE_TWO_TUPLE) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = UNPACK_SEQUENCE_TWO_TUPLE; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(UNPACK_SEQUENCE_TWO_TUPLE); - opcode = UNPACK_SEQUENCE_TWO_TUPLE; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - static_assert(INLINE_CACHE_ENTRIES_UNPACK_SEQUENCE == 1, "incorrect cache size"); - _PyStackRef tos; - _PyStackRef seq; - _PyStackRef val1; - _PyStackRef val0; - // _GUARD_TOS_TUPLE - { - tos = stack_pointer[-1]; - PyObject *o = PyStackRef_AsPyObjectBorrow(tos); - if (!PyTuple_CheckExact(o)) { - UPDATE_MISS_STATS(UNPACK_SEQUENCE); - assert(_PyOpcode_Deopt[opcode] == (UNPACK_SEQUENCE)); - JUMP_TO_PREDICTED(TRACING_UNPACK_SEQUENCE); - } - } - /* Skip 1 cache entry */ - // _UNPACK_SEQUENCE_TWO_TUPLE - { - seq = tos; - assert(oparg == 2); - PyObject *seq_o = PyStackRef_AsPyObjectBorrow(seq); - assert(PyTuple_CheckExact(seq_o)); - if (PyTuple_GET_SIZE(seq_o) != 2) { - UPDATE_MISS_STATS(UNPACK_SEQUENCE); - assert(_PyOpcode_Deopt[opcode] == (UNPACK_SEQUENCE)); - JUMP_TO_PREDICTED(TRACING_UNPACK_SEQUENCE); - } - STAT_INC(UNPACK_SEQUENCE, hit); - val0 = PyStackRef_FromPyObjectNew(PyTuple_GET_ITEM(seq_o, 0)); - val1 = PyStackRef_FromPyObjectNew(PyTuple_GET_ITEM(seq_o, 1)); - stack_pointer[-1] = val1; - stack_pointer[0] = val0; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(seq); - stack_pointer = _PyFrame_GetStackPointer(frame); - } - TRACING_DISPATCH(); - } - - TRACING_TARGET(WITH_EXCEPT_START) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = WITH_EXCEPT_START; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(WITH_EXCEPT_START); - opcode = WITH_EXCEPT_START; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - _PyStackRef exit_func; - _PyStackRef exit_self; - _PyStackRef lasti; - _PyStackRef val; - _PyStackRef res; - val = stack_pointer[-1]; - lasti = stack_pointer[-3]; - exit_self = stack_pointer[-4]; - exit_func = stack_pointer[-5]; - PyObject *exc, *tb; - PyObject *val_o = PyStackRef_AsPyObjectBorrow(val); - PyObject *exit_func_o = PyStackRef_AsPyObjectBorrow(exit_func); - assert(val_o && PyExceptionInstance_Check(val_o)); - exc = PyExceptionInstance_Class(val_o); - PyObject *original_tb = tb = PyException_GetTraceback(val_o); - if (tb == NULL) { - tb = Py_None; - } - assert(PyStackRef_IsTaggedInt(lasti)); - (void)lasti; - PyObject *stack[5] = {NULL, PyStackRef_AsPyObjectBorrow(exit_self), exc, val_o, tb}; - int has_self = !PyStackRef_IsNull(exit_self); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *res_o = PyObject_Vectorcall(exit_func_o, stack + 2 - has_self, - (3 + has_self) | PY_VECTORCALL_ARGUMENTS_OFFSET, NULL); - Py_XDECREF(original_tb); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (res_o == NULL) { - TRACING_JUMP_TO_LABEL(error); - } - res = PyStackRef_FromPyObjectSteal(res_o); - stack_pointer[0] = res; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - TRACING_DISPATCH(); - } - - TRACING_TARGET(YIELD_VALUE) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = YIELD_VALUE; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(YIELD_VALUE); - opcode = YIELD_VALUE; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - _PyStackRef retval; - _PyStackRef value; - retval = stack_pointer[-1]; - assert(frame->owner != FRAME_OWNED_BY_INTERPRETER); - frame->instr_ptr++; - PyGenObject *gen = _PyGen_GetGeneratorFromFrame(frame); - assert(FRAME_SUSPENDED_YIELD_FROM == FRAME_SUSPENDED + 1); - assert(oparg == 0 || oparg == 1); - gen->gi_frame_state = FRAME_SUSPENDED + oparg; - _PyStackRef temp = retval; - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - tstate->exc_info = gen->gi_exc_state.previous_item; - gen->gi_exc_state.previous_item = NULL; - _Py_LeaveRecursiveCallPy(tstate); - _PyInterpreterFrame *gen_frame = frame; - frame = tstate->current_frame = frame->previous; - gen_frame->previous = NULL; - assert(INLINE_CACHE_ENTRIES_SEND == INLINE_CACHE_ENTRIES_FOR_ITER); - #if TIER_ONE - assert(frame->instr_ptr->op.code == INSTRUMENTED_LINE || - frame->instr_ptr->op.code == INSTRUMENTED_INSTRUCTION || - _PyOpcode_Deopt[frame->instr_ptr->op.code] == SEND || - _PyOpcode_Deopt[frame->instr_ptr->op.code] == FOR_ITER || - _PyOpcode_Deopt[frame->instr_ptr->op.code] == INTERPRETER_EXIT || - _PyOpcode_Deopt[frame->instr_ptr->op.code] == ENTER_EXECUTOR); - #endif - stack_pointer = _PyFrame_GetStackPointer(frame); - LOAD_IP(1 + INLINE_CACHE_ENTRIES_SEND); - #if TIER_TWO - frame->instr_ptr += (1 + INLINE_CACHE_ENTRIES_SEND); - #endif - value = PyStackRef_MakeHeapSafe(temp); - LLTRACE_RESUME_FRAME(); - stack_pointer[0] = value; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - TRACING_DISPATCH(); - } - #endif /* END TRACING INSTRUCTIONS */ /* END INSTRUCTIONS */ #if !_Py_TAIL_CALL_INTERP diff --git a/Python/generated_tracer_cases.c.h b/Python/generated_tracer_cases.c.h new file mode 100644 index 000000000000..1871af274084 --- /dev/null +++ b/Python/generated_tracer_cases.c.h @@ -0,0 +1,14100 @@ +// This file is generated by Tools/cases_generator/tracer_generator.py +// from: +// Python/bytecodes.c +// Do not edit! + #define TIER_ONE 1 + #define TRACING_JIT 1 + #ifdef _Py_TIER2 /* BEGIN TRACING INSTRUCTIONS */ + + + TRACING_TARGET(BINARY_OP) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = BINARY_OP; + (void)(opcode); + #endif + frame->instr_ptr = next_instr; + next_instr += 6; + INSTRUCTION_STATS(BINARY_OP); + PREDICTED_TRACING_BINARY_OP:; + _Py_CODEUNIT* const this_instr = next_instr - 6; + (void)this_instr; + opcode = BINARY_OP; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + _PyStackRef lhs; + _PyStackRef rhs; + _PyStackRef res; + // _SPECIALIZE_BINARY_OP + { + rhs = stack_pointer[-1]; + lhs = stack_pointer[-2]; + uint16_t counter = read_u16(&this_instr[1].cache); + (void)counter; + #if ENABLE_SPECIALIZATION_FT + if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { + next_instr = this_instr; + _PyFrame_SetStackPointer(frame, stack_pointer); + _Py_Specialize_BinaryOp(lhs, rhs, next_instr, oparg, LOCALS_ARRAY); + stack_pointer = _PyFrame_GetStackPointer(frame); + DISPATCH_SAME_OPARG(); + } + OPCODE_DEFERRED_INC(BINARY_OP); + ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); + #endif /* ENABLE_SPECIALIZATION_FT */ + assert(NB_ADD <= oparg); + assert(oparg <= NB_OPARG_LAST); + } + /* Skip 4 cache entries */ + // _BINARY_OP + { + PyObject *lhs_o = PyStackRef_AsPyObjectBorrow(lhs); + PyObject *rhs_o = PyStackRef_AsPyObjectBorrow(rhs); + assert(_PyEval_BinaryOps[oparg]); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *res_o = _PyEval_BinaryOps[oparg](lhs_o, rhs_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (res_o == NULL) { + TRACING_JUMP_TO_LABEL(error); + } + res = PyStackRef_FromPyObjectSteal(res_o); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyStackRef tmp = lhs; + lhs = res; + stack_pointer[-2] = lhs; + PyStackRef_CLOSE(tmp); + tmp = rhs; + rhs = PyStackRef_NULL; + stack_pointer[-1] = rhs; + PyStackRef_CLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + } + TRACING_DISPATCH(); + } + + TRACING_TARGET(BINARY_OP_ADD_FLOAT) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = BINARY_OP_ADD_FLOAT; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 6; + INSTRUCTION_STATS(BINARY_OP_ADD_FLOAT); + opcode = BINARY_OP_ADD_FLOAT; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5, "incorrect cache size"); + _PyStackRef value; + _PyStackRef left; + _PyStackRef right; + _PyStackRef res; + // _GUARD_TOS_FLOAT + { + value = stack_pointer[-1]; + PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); + if (!PyFloat_CheckExact(value_o)) { + UPDATE_MISS_STATS(BINARY_OP); + assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); + JUMP_TO_PREDICTED(TRACING_BINARY_OP); + } + } + // _GUARD_NOS_FLOAT + { + left = stack_pointer[-2]; + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + if (!PyFloat_CheckExact(left_o)) { + UPDATE_MISS_STATS(BINARY_OP); + assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); + JUMP_TO_PREDICTED(TRACING_BINARY_OP); + } + } + /* Skip 5 cache entries */ + // _BINARY_OP_ADD_FLOAT + { + right = value; + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + assert(PyFloat_CheckExact(left_o)); + assert(PyFloat_CheckExact(right_o)); + STAT_INC(BINARY_OP, hit); + double dres = + ((PyFloatObject *)left_o)->ob_fval + + ((PyFloatObject *)right_o)->ob_fval; + res = _PyFloat_FromDouble_ConsumeInputs(left, right, dres); + if (PyStackRef_IsNull(res)) { + TRACING_JUMP_TO_LABEL(pop_2_error); + } + } + stack_pointer[-2] = res; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + TRACING_DISPATCH(); + } + + TRACING_TARGET(BINARY_OP_ADD_INT) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = BINARY_OP_ADD_INT; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 6; + INSTRUCTION_STATS(BINARY_OP_ADD_INT); + opcode = BINARY_OP_ADD_INT; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5, "incorrect cache size"); + _PyStackRef value; + _PyStackRef left; + _PyStackRef right; + _PyStackRef res; + // _GUARD_TOS_INT + { + value = stack_pointer[-1]; + PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); + if (!_PyLong_CheckExactAndCompact(value_o)) { + UPDATE_MISS_STATS(BINARY_OP); + assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); + JUMP_TO_PREDICTED(TRACING_BINARY_OP); + } + } + // _GUARD_NOS_INT + { + left = stack_pointer[-2]; + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + if (!_PyLong_CheckExactAndCompact(left_o)) { + UPDATE_MISS_STATS(BINARY_OP); + assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); + JUMP_TO_PREDICTED(TRACING_BINARY_OP); + } + } + /* Skip 5 cache entries */ + // _BINARY_OP_ADD_INT + { + right = value; + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + assert(PyLong_CheckExact(left_o)); + assert(PyLong_CheckExact(right_o)); + assert(_PyLong_BothAreCompact((PyLongObject *)left_o, (PyLongObject *)right_o)); + STAT_INC(BINARY_OP, hit); + res = _PyCompactLong_Add((PyLongObject *)left_o, (PyLongObject *)right_o); + if (PyStackRef_IsNull(res)) { + UPDATE_MISS_STATS(BINARY_OP); + assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); + JUMP_TO_PREDICTED(TRACING_BINARY_OP); + } + PyStackRef_CLOSE_SPECIALIZED(right, _PyLong_ExactDealloc); + PyStackRef_CLOSE_SPECIALIZED(left, _PyLong_ExactDealloc); + } + stack_pointer[-2] = res; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + TRACING_DISPATCH(); + } + + TRACING_TARGET(BINARY_OP_ADD_UNICODE) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = BINARY_OP_ADD_UNICODE; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 6; + INSTRUCTION_STATS(BINARY_OP_ADD_UNICODE); + opcode = BINARY_OP_ADD_UNICODE; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5, "incorrect cache size"); + _PyStackRef value; + _PyStackRef nos; + _PyStackRef left; + _PyStackRef right; + _PyStackRef res; + // _GUARD_TOS_UNICODE + { + value = stack_pointer[-1]; + PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); + if (!PyUnicode_CheckExact(value_o)) { + UPDATE_MISS_STATS(BINARY_OP); + assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); + JUMP_TO_PREDICTED(TRACING_BINARY_OP); + } + } + // _GUARD_NOS_UNICODE + { + nos = stack_pointer[-2]; + PyObject *o = PyStackRef_AsPyObjectBorrow(nos); + if (!PyUnicode_CheckExact(o)) { + UPDATE_MISS_STATS(BINARY_OP); + assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); + JUMP_TO_PREDICTED(TRACING_BINARY_OP); + } + } + /* Skip 5 cache entries */ + // _BINARY_OP_ADD_UNICODE + { + right = value; + left = nos; + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + assert(PyUnicode_CheckExact(left_o)); + assert(PyUnicode_CheckExact(right_o)); + STAT_INC(BINARY_OP, hit); + PyObject *res_o = PyUnicode_Concat(left_o, right_o); + PyStackRef_CLOSE_SPECIALIZED(right, _PyUnicode_ExactDealloc); + PyStackRef_CLOSE_SPECIALIZED(left, _PyUnicode_ExactDealloc); + if (res_o == NULL) { + TRACING_JUMP_TO_LABEL(pop_2_error); + } + res = PyStackRef_FromPyObjectSteal(res_o); + } + stack_pointer[-2] = res; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + TRACING_DISPATCH(); + } + + TRACING_TARGET(BINARY_OP_EXTEND) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = BINARY_OP_EXTEND; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 6; + INSTRUCTION_STATS(BINARY_OP_EXTEND); + opcode = BINARY_OP_EXTEND; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5, "incorrect cache size"); + _PyStackRef left; + _PyStackRef right; + _PyStackRef res; + /* Skip 1 cache entry */ + // _GUARD_BINARY_OP_EXTEND + { + right = stack_pointer[-1]; + left = stack_pointer[-2]; + PyObject *descr = read_obj(&this_instr[2].cache); + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + _PyBinaryOpSpecializationDescr *d = (_PyBinaryOpSpecializationDescr*)descr; + assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5); + assert(d && d->guard); + _PyFrame_SetStackPointer(frame, stack_pointer); + int res = d->guard(left_o, right_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (!res) { + UPDATE_MISS_STATS(BINARY_OP); + assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); + JUMP_TO_PREDICTED(TRACING_BINARY_OP); + } + } + /* Skip -4 cache entry */ + // _BINARY_OP_EXTEND + { + PyObject *descr = read_obj(&this_instr[2].cache); + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5); + _PyBinaryOpSpecializationDescr *d = (_PyBinaryOpSpecializationDescr*)descr; + STAT_INC(BINARY_OP, hit); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *res_o = d->action(left_o, right_o); + _PyStackRef tmp = right; + right = PyStackRef_NULL; + stack_pointer[-1] = right; + PyStackRef_CLOSE(tmp); + tmp = left; + left = PyStackRef_NULL; + stack_pointer[-2] = left; + PyStackRef_CLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); + res = PyStackRef_FromPyObjectSteal(res_o); + } + stack_pointer[0] = res; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + TRACING_DISPATCH(); + } + + TRACING_TARGET(BINARY_OP_INPLACE_ADD_UNICODE) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = BINARY_OP_INPLACE_ADD_UNICODE; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 6; + INSTRUCTION_STATS(BINARY_OP_INPLACE_ADD_UNICODE); + opcode = BINARY_OP_INPLACE_ADD_UNICODE; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5, "incorrect cache size"); + _PyStackRef value; + _PyStackRef nos; + _PyStackRef left; + _PyStackRef right; + // _GUARD_TOS_UNICODE + { + value = stack_pointer[-1]; + PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); + if (!PyUnicode_CheckExact(value_o)) { + UPDATE_MISS_STATS(BINARY_OP); + assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); + JUMP_TO_PREDICTED(TRACING_BINARY_OP); + } + } + // _GUARD_NOS_UNICODE + { + nos = stack_pointer[-2]; + PyObject *o = PyStackRef_AsPyObjectBorrow(nos); + if (!PyUnicode_CheckExact(o)) { + UPDATE_MISS_STATS(BINARY_OP); + assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); + JUMP_TO_PREDICTED(TRACING_BINARY_OP); + } + } + /* Skip 5 cache entries */ + // _BINARY_OP_INPLACE_ADD_UNICODE + { + right = value; + left = nos; + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + assert(PyUnicode_CheckExact(left_o)); + assert(PyUnicode_CheckExact(PyStackRef_AsPyObjectBorrow(right))); + int next_oparg; + #if TIER_ONE + assert(next_instr->op.code == STORE_FAST); + next_oparg = next_instr->op.arg; + #else + next_oparg = (int)CURRENT_OPERAND0(); + #endif + _PyStackRef *target_local = &GETLOCAL(next_oparg); + assert(PyUnicode_CheckExact(left_o)); + if (PyStackRef_AsPyObjectBorrow(*target_local) != left_o) { + UPDATE_MISS_STATS(BINARY_OP); + assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); + JUMP_TO_PREDICTED(TRACING_BINARY_OP); + } + STAT_INC(BINARY_OP, hit); + assert(Py_REFCNT(left_o) >= 2 || !PyStackRef_IsHeapSafe(left)); + PyStackRef_CLOSE_SPECIALIZED(left, _PyUnicode_ExactDealloc); + PyObject *temp = PyStackRef_AsPyObjectSteal(*target_local); + PyObject *right_o = PyStackRef_AsPyObjectSteal(right); + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyUnicode_Append(&temp, right_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + *target_local = PyStackRef_FromPyObjectSteal(temp); + _PyFrame_SetStackPointer(frame, stack_pointer); + Py_DECREF(right_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (PyStackRef_IsNull(*target_local)) { + TRACING_JUMP_TO_LABEL(error); + } + #if TIER_ONE + + assert(next_instr->op.code == STORE_FAST); + SKIP_OVER(1); + #endif + } + TRACING_DISPATCH(); + } + + TRACING_TARGET(BINARY_OP_MULTIPLY_FLOAT) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = BINARY_OP_MULTIPLY_FLOAT; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 6; + INSTRUCTION_STATS(BINARY_OP_MULTIPLY_FLOAT); + opcode = BINARY_OP_MULTIPLY_FLOAT; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5, "incorrect cache size"); + _PyStackRef value; + _PyStackRef left; + _PyStackRef right; + _PyStackRef res; + // _GUARD_TOS_FLOAT + { + value = stack_pointer[-1]; + PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); + if (!PyFloat_CheckExact(value_o)) { + UPDATE_MISS_STATS(BINARY_OP); + assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); + JUMP_TO_PREDICTED(TRACING_BINARY_OP); + } + } + // _GUARD_NOS_FLOAT + { + left = stack_pointer[-2]; + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + if (!PyFloat_CheckExact(left_o)) { + UPDATE_MISS_STATS(BINARY_OP); + assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); + JUMP_TO_PREDICTED(TRACING_BINARY_OP); + } + } + /* Skip 5 cache entries */ + // _BINARY_OP_MULTIPLY_FLOAT + { + right = value; + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + assert(PyFloat_CheckExact(left_o)); + assert(PyFloat_CheckExact(right_o)); + STAT_INC(BINARY_OP, hit); + double dres = + ((PyFloatObject *)left_o)->ob_fval * + ((PyFloatObject *)right_o)->ob_fval; + res = _PyFloat_FromDouble_ConsumeInputs(left, right, dres); + if (PyStackRef_IsNull(res)) { + TRACING_JUMP_TO_LABEL(pop_2_error); + } + } + stack_pointer[-2] = res; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + TRACING_DISPATCH(); + } + + TRACING_TARGET(BINARY_OP_MULTIPLY_INT) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = BINARY_OP_MULTIPLY_INT; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 6; + INSTRUCTION_STATS(BINARY_OP_MULTIPLY_INT); + opcode = BINARY_OP_MULTIPLY_INT; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5, "incorrect cache size"); + _PyStackRef value; + _PyStackRef left; + _PyStackRef right; + _PyStackRef res; + // _GUARD_TOS_INT + { + value = stack_pointer[-1]; + PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); + if (!_PyLong_CheckExactAndCompact(value_o)) { + UPDATE_MISS_STATS(BINARY_OP); + assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); + JUMP_TO_PREDICTED(TRACING_BINARY_OP); + } + } + // _GUARD_NOS_INT + { + left = stack_pointer[-2]; + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + if (!_PyLong_CheckExactAndCompact(left_o)) { + UPDATE_MISS_STATS(BINARY_OP); + assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); + JUMP_TO_PREDICTED(TRACING_BINARY_OP); + } + } + /* Skip 5 cache entries */ + // _BINARY_OP_MULTIPLY_INT + { + right = value; + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + assert(PyLong_CheckExact(left_o)); + assert(PyLong_CheckExact(right_o)); + assert(_PyLong_BothAreCompact((PyLongObject *)left_o, (PyLongObject *)right_o)); + STAT_INC(BINARY_OP, hit); + res = _PyCompactLong_Multiply((PyLongObject *)left_o, (PyLongObject *)right_o); + if (PyStackRef_IsNull(res)) { + UPDATE_MISS_STATS(BINARY_OP); + assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); + JUMP_TO_PREDICTED(TRACING_BINARY_OP); + } + PyStackRef_CLOSE_SPECIALIZED(right, _PyLong_ExactDealloc); + PyStackRef_CLOSE_SPECIALIZED(left, _PyLong_ExactDealloc); + } + stack_pointer[-2] = res; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + TRACING_DISPATCH(); + } + + TRACING_TARGET(BINARY_OP_SUBSCR_DICT) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = BINARY_OP_SUBSCR_DICT; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 6; + INSTRUCTION_STATS(BINARY_OP_SUBSCR_DICT); + opcode = BINARY_OP_SUBSCR_DICT; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5, "incorrect cache size"); + _PyStackRef nos; + _PyStackRef dict_st; + _PyStackRef sub_st; + _PyStackRef res; + // _GUARD_NOS_DICT + { + nos = stack_pointer[-2]; + PyObject *o = PyStackRef_AsPyObjectBorrow(nos); + if (!PyDict_CheckExact(o)) { + UPDATE_MISS_STATS(BINARY_OP); + assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); + JUMP_TO_PREDICTED(TRACING_BINARY_OP); + } + } + /* Skip 5 cache entries */ + // _BINARY_OP_SUBSCR_DICT + { + sub_st = stack_pointer[-1]; + dict_st = nos; + PyObject *sub = PyStackRef_AsPyObjectBorrow(sub_st); + PyObject *dict = PyStackRef_AsPyObjectBorrow(dict_st); + assert(PyDict_CheckExact(dict)); + STAT_INC(BINARY_OP, hit); + PyObject *res_o; + _PyFrame_SetStackPointer(frame, stack_pointer); + int rc = PyDict_GetItemRef(dict, sub, &res_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (rc == 0) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyErr_SetKeyError(sub); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyStackRef tmp = sub_st; + sub_st = PyStackRef_NULL; + stack_pointer[-1] = sub_st; + PyStackRef_CLOSE(tmp); + tmp = dict_st; + dict_st = PyStackRef_NULL; + stack_pointer[-2] = dict_st; + PyStackRef_CLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); + if (rc <= 0) { + TRACING_JUMP_TO_LABEL(error); + } + res = PyStackRef_FromPyObjectSteal(res_o); + } + stack_pointer[0] = res; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + TRACING_DISPATCH(); + } + + TRACING_TARGET(BINARY_OP_SUBSCR_GETITEM) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = BINARY_OP_SUBSCR_GETITEM; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 6; + INSTRUCTION_STATS(BINARY_OP_SUBSCR_GETITEM); + opcode = BINARY_OP_SUBSCR_GETITEM; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5, "incorrect cache size"); + _PyStackRef container; + _PyStackRef getitem; + _PyStackRef sub; + _PyStackRef new_frame; + /* Skip 5 cache entries */ + // _CHECK_PEP_523 + { + if (tstate->interp->eval_frame) { + UPDATE_MISS_STATS(BINARY_OP); + assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); + JUMP_TO_PREDICTED(TRACING_BINARY_OP); + } + } + // _BINARY_OP_SUBSCR_CHECK_FUNC + { + container = stack_pointer[-2]; + PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(container)); + if (!PyType_HasFeature(tp, Py_TPFLAGS_HEAPTYPE)) { + UPDATE_MISS_STATS(BINARY_OP); + assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); + JUMP_TO_PREDICTED(TRACING_BINARY_OP); + } + PyHeapTypeObject *ht = (PyHeapTypeObject *)tp; + PyObject *getitem_o = FT_ATOMIC_LOAD_PTR_ACQUIRE(ht->_spec_cache.getitem); + if (getitem_o == NULL) { + UPDATE_MISS_STATS(BINARY_OP); + assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); + JUMP_TO_PREDICTED(TRACING_BINARY_OP); + } + assert(PyFunction_Check(getitem_o)); + uint32_t cached_version = FT_ATOMIC_LOAD_UINT32_RELAXED(ht->_spec_cache.getitem_version); + if (((PyFunctionObject *)getitem_o)->func_version != cached_version) { + UPDATE_MISS_STATS(BINARY_OP); + assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); + JUMP_TO_PREDICTED(TRACING_BINARY_OP); + } + PyCodeObject *code = (PyCodeObject *)PyFunction_GET_CODE(getitem_o); + assert(code->co_argcount == 2); + if (!_PyThreadState_HasStackSpace(tstate, code->co_framesize)) { + UPDATE_MISS_STATS(BINARY_OP); + assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); + JUMP_TO_PREDICTED(TRACING_BINARY_OP); + } + getitem = PyStackRef_FromPyObjectNew(getitem_o); + STAT_INC(BINARY_OP, hit); + } + // _BINARY_OP_SUBSCR_INIT_CALL + { + sub = stack_pointer[-1]; + _PyInterpreterFrame* pushed_frame = _PyFrame_PushUnchecked(tstate, getitem, 2, frame); + pushed_frame->localsplus[0] = container; + pushed_frame->localsplus[1] = sub; + frame->return_offset = 6u ; + new_frame = PyStackRef_Wrap(pushed_frame); + } + // _PUSH_FRAME + { + assert(tstate->interp->eval_frame == NULL); + _PyInterpreterFrame *temp = PyStackRef_Unwrap(new_frame); + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + assert(temp->previous == frame || temp->previous->previous == frame); + CALL_STAT_INC(inlined_py_calls); + frame = tstate->current_frame = temp; + tstate->py_recursion_remaining--; + LOAD_SP(); + LOAD_IP(0); + LLTRACE_RESUME_FRAME(); + } + TRACING_DISPATCH(); + } + + TRACING_TARGET(BINARY_OP_SUBSCR_LIST_INT) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = BINARY_OP_SUBSCR_LIST_INT; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 6; + INSTRUCTION_STATS(BINARY_OP_SUBSCR_LIST_INT); + opcode = BINARY_OP_SUBSCR_LIST_INT; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5, "incorrect cache size"); + _PyStackRef value; + _PyStackRef nos; + _PyStackRef list_st; + _PyStackRef sub_st; + _PyStackRef res; + // _GUARD_TOS_INT + { + value = stack_pointer[-1]; + PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); + if (!_PyLong_CheckExactAndCompact(value_o)) { + UPDATE_MISS_STATS(BINARY_OP); + assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); + JUMP_TO_PREDICTED(TRACING_BINARY_OP); + } + } + // _GUARD_NOS_LIST + { + nos = stack_pointer[-2]; + PyObject *o = PyStackRef_AsPyObjectBorrow(nos); + if (!PyList_CheckExact(o)) { + UPDATE_MISS_STATS(BINARY_OP); + assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); + JUMP_TO_PREDICTED(TRACING_BINARY_OP); + } + } + /* Skip 5 cache entries */ + // _BINARY_OP_SUBSCR_LIST_INT + { + sub_st = value; + list_st = nos; + PyObject *sub = PyStackRef_AsPyObjectBorrow(sub_st); + PyObject *list = PyStackRef_AsPyObjectBorrow(list_st); + assert(PyLong_CheckExact(sub)); + assert(PyList_CheckExact(list)); + if (!_PyLong_IsNonNegativeCompact((PyLongObject *)sub)) { + UPDATE_MISS_STATS(BINARY_OP); + assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); + JUMP_TO_PREDICTED(TRACING_BINARY_OP); + } + Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0]; + #ifdef Py_GIL_DISABLED + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *res_o = _PyList_GetItemRef((PyListObject*)list, index); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (res_o == NULL) { + UPDATE_MISS_STATS(BINARY_OP); + assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); + JUMP_TO_PREDICTED(TRACING_BINARY_OP); + } + STAT_INC(BINARY_OP, hit); + res = PyStackRef_FromPyObjectSteal(res_o); + #else + if (index >= PyList_GET_SIZE(list)) { + UPDATE_MISS_STATS(BINARY_OP); + assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); + JUMP_TO_PREDICTED(TRACING_BINARY_OP); + } + STAT_INC(BINARY_OP, hit); + PyObject *res_o = PyList_GET_ITEM(list, index); + assert(res_o != NULL); + res = PyStackRef_FromPyObjectNew(res_o); + #endif + STAT_INC(BINARY_OP, hit); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyStackRef tmp = list_st; + list_st = res; + stack_pointer[-2] = list_st; + PyStackRef_CLOSE(tmp); + tmp = sub_st; + sub_st = PyStackRef_NULL; + stack_pointer[-1] = sub_st; + PyStackRef_CLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + } + TRACING_DISPATCH(); + } + + TRACING_TARGET(BINARY_OP_SUBSCR_LIST_SLICE) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = BINARY_OP_SUBSCR_LIST_SLICE; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 6; + INSTRUCTION_STATS(BINARY_OP_SUBSCR_LIST_SLICE); + opcode = BINARY_OP_SUBSCR_LIST_SLICE; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5, "incorrect cache size"); + _PyStackRef tos; + _PyStackRef nos; + _PyStackRef list_st; + _PyStackRef sub_st; + _PyStackRef res; + // _GUARD_TOS_SLICE + { + tos = stack_pointer[-1]; + PyObject *o = PyStackRef_AsPyObjectBorrow(tos); + if (!PySlice_Check(o)) { + UPDATE_MISS_STATS(BINARY_OP); + assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); + JUMP_TO_PREDICTED(TRACING_BINARY_OP); + } + } + // _GUARD_NOS_LIST + { + nos = stack_pointer[-2]; + PyObject *o = PyStackRef_AsPyObjectBorrow(nos); + if (!PyList_CheckExact(o)) { + UPDATE_MISS_STATS(BINARY_OP); + assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); + JUMP_TO_PREDICTED(TRACING_BINARY_OP); + } + } + /* Skip 5 cache entries */ + // _BINARY_OP_SUBSCR_LIST_SLICE + { + sub_st = tos; + list_st = nos; + PyObject *sub = PyStackRef_AsPyObjectBorrow(sub_st); + PyObject *list = PyStackRef_AsPyObjectBorrow(list_st); + assert(PySlice_Check(sub)); + assert(PyList_CheckExact(list)); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *res_o = _PyList_SliceSubscript(list, sub); + stack_pointer = _PyFrame_GetStackPointer(frame); + STAT_INC(BINARY_OP, hit); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyStackRef tmp = sub_st; + sub_st = PyStackRef_NULL; + stack_pointer[-1] = sub_st; + PyStackRef_CLOSE(tmp); + tmp = list_st; + list_st = PyStackRef_NULL; + stack_pointer[-2] = list_st; + PyStackRef_CLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); + if (res_o == NULL) { + TRACING_JUMP_TO_LABEL(error); + } + res = PyStackRef_FromPyObjectSteal(res_o); + } + stack_pointer[0] = res; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + TRACING_DISPATCH(); + } + + TRACING_TARGET(BINARY_OP_SUBSCR_STR_INT) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = BINARY_OP_SUBSCR_STR_INT; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 6; + INSTRUCTION_STATS(BINARY_OP_SUBSCR_STR_INT); + opcode = BINARY_OP_SUBSCR_STR_INT; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5, "incorrect cache size"); + _PyStackRef value; + _PyStackRef nos; + _PyStackRef str_st; + _PyStackRef sub_st; + _PyStackRef res; + // _GUARD_TOS_INT + { + value = stack_pointer[-1]; + PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); + if (!_PyLong_CheckExactAndCompact(value_o)) { + UPDATE_MISS_STATS(BINARY_OP); + assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); + JUMP_TO_PREDICTED(TRACING_BINARY_OP); + } + } + // _GUARD_NOS_UNICODE + { + nos = stack_pointer[-2]; + PyObject *o = PyStackRef_AsPyObjectBorrow(nos); + if (!PyUnicode_CheckExact(o)) { + UPDATE_MISS_STATS(BINARY_OP); + assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); + JUMP_TO_PREDICTED(TRACING_BINARY_OP); + } + } + /* Skip 5 cache entries */ + // _BINARY_OP_SUBSCR_STR_INT + { + sub_st = value; + str_st = nos; + PyObject *sub = PyStackRef_AsPyObjectBorrow(sub_st); + PyObject *str = PyStackRef_AsPyObjectBorrow(str_st); + assert(PyLong_CheckExact(sub)); + assert(PyUnicode_CheckExact(str)); + if (!_PyLong_IsNonNegativeCompact((PyLongObject *)sub)) { + UPDATE_MISS_STATS(BINARY_OP); + assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); + JUMP_TO_PREDICTED(TRACING_BINARY_OP); + } + Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0]; + if (PyUnicode_GET_LENGTH(str) <= index) { + UPDATE_MISS_STATS(BINARY_OP); + assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); + JUMP_TO_PREDICTED(TRACING_BINARY_OP); + } + Py_UCS4 c = PyUnicode_READ_CHAR(str, index); + if (Py_ARRAY_LENGTH(_Py_SINGLETON(strings).ascii) <= c) { + UPDATE_MISS_STATS(BINARY_OP); + assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); + JUMP_TO_PREDICTED(TRACING_BINARY_OP); + } + STAT_INC(BINARY_OP, hit); + PyObject *res_o = (PyObject*)&_Py_SINGLETON(strings).ascii[c]; + PyStackRef_CLOSE_SPECIALIZED(sub_st, _PyLong_ExactDealloc); + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(str_st); + stack_pointer = _PyFrame_GetStackPointer(frame); + res = PyStackRef_FromPyObjectBorrow(res_o); + } + stack_pointer[0] = res; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + TRACING_DISPATCH(); + } + + TRACING_TARGET(BINARY_OP_SUBSCR_TUPLE_INT) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = BINARY_OP_SUBSCR_TUPLE_INT; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 6; + INSTRUCTION_STATS(BINARY_OP_SUBSCR_TUPLE_INT); + opcode = BINARY_OP_SUBSCR_TUPLE_INT; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5, "incorrect cache size"); + _PyStackRef value; + _PyStackRef nos; + _PyStackRef tuple_st; + _PyStackRef sub_st; + _PyStackRef res; + // _GUARD_TOS_INT + { + value = stack_pointer[-1]; + PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); + if (!_PyLong_CheckExactAndCompact(value_o)) { + UPDATE_MISS_STATS(BINARY_OP); + assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); + JUMP_TO_PREDICTED(TRACING_BINARY_OP); + } + } + // _GUARD_NOS_TUPLE + { + nos = stack_pointer[-2]; + PyObject *o = PyStackRef_AsPyObjectBorrow(nos); + if (!PyTuple_CheckExact(o)) { + UPDATE_MISS_STATS(BINARY_OP); + assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); + JUMP_TO_PREDICTED(TRACING_BINARY_OP); + } + } + /* Skip 5 cache entries */ + // _BINARY_OP_SUBSCR_TUPLE_INT + { + sub_st = value; + tuple_st = nos; + PyObject *sub = PyStackRef_AsPyObjectBorrow(sub_st); + PyObject *tuple = PyStackRef_AsPyObjectBorrow(tuple_st); + assert(PyLong_CheckExact(sub)); + assert(PyTuple_CheckExact(tuple)); + if (!_PyLong_IsNonNegativeCompact((PyLongObject *)sub)) { + UPDATE_MISS_STATS(BINARY_OP); + assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); + JUMP_TO_PREDICTED(TRACING_BINARY_OP); + } + Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0]; + if (index >= PyTuple_GET_SIZE(tuple)) { + UPDATE_MISS_STATS(BINARY_OP); + assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); + JUMP_TO_PREDICTED(TRACING_BINARY_OP); + } + STAT_INC(BINARY_OP, hit); + PyObject *res_o = PyTuple_GET_ITEM(tuple, index); + assert(res_o != NULL); + PyStackRef_CLOSE_SPECIALIZED(sub_st, _PyLong_ExactDealloc); + res = PyStackRef_FromPyObjectNew(res_o); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyStackRef tmp = tuple_st; + tuple_st = res; + stack_pointer[-1] = tuple_st; + PyStackRef_CLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + TRACING_DISPATCH(); + } + + TRACING_TARGET(BINARY_OP_SUBTRACT_FLOAT) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = BINARY_OP_SUBTRACT_FLOAT; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 6; + INSTRUCTION_STATS(BINARY_OP_SUBTRACT_FLOAT); + opcode = BINARY_OP_SUBTRACT_FLOAT; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5, "incorrect cache size"); + _PyStackRef value; + _PyStackRef left; + _PyStackRef right; + _PyStackRef res; + // _GUARD_TOS_FLOAT + { + value = stack_pointer[-1]; + PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); + if (!PyFloat_CheckExact(value_o)) { + UPDATE_MISS_STATS(BINARY_OP); + assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); + JUMP_TO_PREDICTED(TRACING_BINARY_OP); + } + } + // _GUARD_NOS_FLOAT + { + left = stack_pointer[-2]; + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + if (!PyFloat_CheckExact(left_o)) { + UPDATE_MISS_STATS(BINARY_OP); + assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); + JUMP_TO_PREDICTED(TRACING_BINARY_OP); + } + } + /* Skip 5 cache entries */ + // _BINARY_OP_SUBTRACT_FLOAT + { + right = value; + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + assert(PyFloat_CheckExact(left_o)); + assert(PyFloat_CheckExact(right_o)); + STAT_INC(BINARY_OP, hit); + double dres = + ((PyFloatObject *)left_o)->ob_fval - + ((PyFloatObject *)right_o)->ob_fval; + res = _PyFloat_FromDouble_ConsumeInputs(left, right, dres); + if (PyStackRef_IsNull(res)) { + TRACING_JUMP_TO_LABEL(pop_2_error); + } + } + stack_pointer[-2] = res; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + TRACING_DISPATCH(); + } + + TRACING_TARGET(BINARY_OP_SUBTRACT_INT) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = BINARY_OP_SUBTRACT_INT; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 6; + INSTRUCTION_STATS(BINARY_OP_SUBTRACT_INT); + opcode = BINARY_OP_SUBTRACT_INT; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5, "incorrect cache size"); + _PyStackRef value; + _PyStackRef left; + _PyStackRef right; + _PyStackRef res; + // _GUARD_TOS_INT + { + value = stack_pointer[-1]; + PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); + if (!_PyLong_CheckExactAndCompact(value_o)) { + UPDATE_MISS_STATS(BINARY_OP); + assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); + JUMP_TO_PREDICTED(TRACING_BINARY_OP); + } + } + // _GUARD_NOS_INT + { + left = stack_pointer[-2]; + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + if (!_PyLong_CheckExactAndCompact(left_o)) { + UPDATE_MISS_STATS(BINARY_OP); + assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); + JUMP_TO_PREDICTED(TRACING_BINARY_OP); + } + } + /* Skip 5 cache entries */ + // _BINARY_OP_SUBTRACT_INT + { + right = value; + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + assert(PyLong_CheckExact(left_o)); + assert(PyLong_CheckExact(right_o)); + assert(_PyLong_BothAreCompact((PyLongObject *)left_o, (PyLongObject *)right_o)); + STAT_INC(BINARY_OP, hit); + res = _PyCompactLong_Subtract((PyLongObject *)left_o, (PyLongObject *)right_o); + if (PyStackRef_IsNull(res)) { + UPDATE_MISS_STATS(BINARY_OP); + assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); + JUMP_TO_PREDICTED(TRACING_BINARY_OP); + } + PyStackRef_CLOSE_SPECIALIZED(right, _PyLong_ExactDealloc); + PyStackRef_CLOSE_SPECIALIZED(left, _PyLong_ExactDealloc); + } + stack_pointer[-2] = res; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + TRACING_DISPATCH(); + } + + TRACING_TARGET(BINARY_SLICE) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = BINARY_SLICE; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(BINARY_SLICE); + opcode = BINARY_SLICE; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + _PyStackRef container; + _PyStackRef start; + _PyStackRef stop; + _PyStackRef res; + // _SPECIALIZE_BINARY_SLICE + { + #if ENABLE_SPECIALIZATION + OPCODE_DEFERRED_INC(BINARY_SLICE); + #endif /* ENABLE_SPECIALIZATION */ + } + // _BINARY_SLICE + { + stop = stack_pointer[-1]; + start = stack_pointer[-2]; + container = stack_pointer[-3]; + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *slice = _PyBuildSlice_ConsumeRefs(PyStackRef_AsPyObjectSteal(start), + PyStackRef_AsPyObjectSteal(stop)); + stack_pointer = _PyFrame_GetStackPointer(frame); + PyObject *res_o; + if (slice == NULL) { + res_o = NULL; + } + else { + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + res_o = PyObject_GetItem(PyStackRef_AsPyObjectBorrow(container), slice); + Py_DECREF(slice); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += 2; + } + stack_pointer += -3; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(container); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (res_o == NULL) { + TRACING_JUMP_TO_LABEL(error); + } + res = PyStackRef_FromPyObjectSteal(res_o); + } + stack_pointer[0] = res; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + TRACING_DISPATCH(); + } + + TRACING_TARGET(BUILD_INTERPOLATION) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = BUILD_INTERPOLATION; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(BUILD_INTERPOLATION); + opcode = BUILD_INTERPOLATION; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + _PyStackRef value; + _PyStackRef str; + _PyStackRef *format; + _PyStackRef interpolation; + format = &stack_pointer[-(oparg & 1)]; + str = stack_pointer[-1 - (oparg & 1)]; + value = stack_pointer[-2 - (oparg & 1)]; + PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); + PyObject *str_o = PyStackRef_AsPyObjectBorrow(str); + int conversion = oparg >> 2; + PyObject *format_o; + if (oparg & 1) { + format_o = PyStackRef_AsPyObjectBorrow(format[0]); + } + else { + format_o = &_Py_STR(empty); + } + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *interpolation_o = _PyInterpolation_Build(value_o, str_o, conversion, format_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (oparg & 1) { + stack_pointer += -(oparg & 1); + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(format[0]); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + else { + stack_pointer += -(oparg & 1); + } + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(str); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(value); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (interpolation_o == NULL) { + TRACING_JUMP_TO_LABEL(error); + } + interpolation = PyStackRef_FromPyObjectSteal(interpolation_o); + stack_pointer[0] = interpolation; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + TRACING_DISPATCH(); + } + + TRACING_TARGET(BUILD_LIST) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = BUILD_LIST; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(BUILD_LIST); + opcode = BUILD_LIST; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + _PyStackRef *values; + _PyStackRef list; + values = &stack_pointer[-oparg]; + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *list_o = _PyList_FromStackRefStealOnSuccess(values, oparg); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (list_o == NULL) { + TRACING_JUMP_TO_LABEL(error); + } + list = PyStackRef_FromPyObjectStealMortal(list_o); + stack_pointer[-oparg] = list; + stack_pointer += 1 - oparg; + assert(WITHIN_STACK_BOUNDS()); + TRACING_DISPATCH(); + } + + TRACING_TARGET(BUILD_MAP) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = BUILD_MAP; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(BUILD_MAP); + opcode = BUILD_MAP; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + _PyStackRef *values; + _PyStackRef map; + values = &stack_pointer[-oparg*2]; + STACKREFS_TO_PYOBJECTS(values, oparg*2, values_o); + if (CONVERSION_FAILED(values_o)) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyStackRef tmp; + for (int _i = oparg*2; --_i >= 0;) { + tmp = values[_i]; + values[_i] = PyStackRef_NULL; + PyStackRef_CLOSE(tmp); + } + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -oparg*2; + assert(WITHIN_STACK_BOUNDS()); + TRACING_JUMP_TO_LABEL(error); + } + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *map_o = _PyDict_FromItems( + values_o, 2, + values_o+1, 2, + oparg); + stack_pointer = _PyFrame_GetStackPointer(frame); + STACKREFS_TO_PYOBJECTS_CLEANUP(values_o); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyStackRef tmp; + for (int _i = oparg*2; --_i >= 0;) { + tmp = values[_i]; + values[_i] = PyStackRef_NULL; + PyStackRef_CLOSE(tmp); + } + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -oparg*2; + assert(WITHIN_STACK_BOUNDS()); + if (map_o == NULL) { + TRACING_JUMP_TO_LABEL(error); + } + map = PyStackRef_FromPyObjectStealMortal(map_o); + stack_pointer[0] = map; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + TRACING_DISPATCH(); + } + + TRACING_TARGET(BUILD_SET) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = BUILD_SET; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(BUILD_SET); + opcode = BUILD_SET; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + _PyStackRef *values; + _PyStackRef set; + values = &stack_pointer[-oparg]; + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *set_o = PySet_New(NULL); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (set_o == NULL) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyStackRef tmp; + for (int _i = oparg; --_i >= 0;) { + tmp = values[_i]; + values[_i] = PyStackRef_NULL; + PyStackRef_CLOSE(tmp); + } + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -oparg; + assert(WITHIN_STACK_BOUNDS()); + TRACING_JUMP_TO_LABEL(error); + } + int err = 0; + for (Py_ssize_t i = 0; i < oparg; i++) { + _PyStackRef value = values[i]; + values[i] = PyStackRef_NULL; + if (err == 0) { + _PyFrame_SetStackPointer(frame, stack_pointer); + err = _PySet_AddTakeRef((PySetObject *)set_o, PyStackRef_AsPyObjectSteal(value)); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + else { + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(value); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + } + if (err) { + stack_pointer += -oparg; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + Py_DECREF(set_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + TRACING_JUMP_TO_LABEL(error); + } + set = PyStackRef_FromPyObjectStealMortal(set_o); + stack_pointer[-oparg] = set; + stack_pointer += 1 - oparg; + assert(WITHIN_STACK_BOUNDS()); + TRACING_DISPATCH(); + } + + TRACING_TARGET(BUILD_SLICE) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = BUILD_SLICE; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(BUILD_SLICE); + opcode = BUILD_SLICE; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + _PyStackRef *args; + _PyStackRef slice; + args = &stack_pointer[-oparg]; + PyObject *start_o = PyStackRef_AsPyObjectBorrow(args[0]); + PyObject *stop_o = PyStackRef_AsPyObjectBorrow(args[1]); + PyObject *step_o = oparg == 3 ? PyStackRef_AsPyObjectBorrow(args[2]) : NULL; + PyObject *slice_o = PySlice_New(start_o, stop_o, step_o); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyStackRef tmp; + for (int _i = oparg; --_i >= 0;) { + tmp = args[_i]; + args[_i] = PyStackRef_NULL; + PyStackRef_CLOSE(tmp); + } + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -oparg; + assert(WITHIN_STACK_BOUNDS()); + if (slice_o == NULL) { + TRACING_JUMP_TO_LABEL(error); + } + slice = PyStackRef_FromPyObjectStealMortal(slice_o); + stack_pointer[0] = slice; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + TRACING_DISPATCH(); + } + + TRACING_TARGET(BUILD_STRING) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = BUILD_STRING; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(BUILD_STRING); + opcode = BUILD_STRING; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + _PyStackRef *pieces; + _PyStackRef str; + pieces = &stack_pointer[-oparg]; + STACKREFS_TO_PYOBJECTS(pieces, oparg, pieces_o); + if (CONVERSION_FAILED(pieces_o)) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyStackRef tmp; + for (int _i = oparg; --_i >= 0;) { + tmp = pieces[_i]; + pieces[_i] = PyStackRef_NULL; + PyStackRef_CLOSE(tmp); + } + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -oparg; + assert(WITHIN_STACK_BOUNDS()); + TRACING_JUMP_TO_LABEL(error); + } + PyObject *str_o = _PyUnicode_JoinArray(&_Py_STR(empty), pieces_o, oparg); + STACKREFS_TO_PYOBJECTS_CLEANUP(pieces_o); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyStackRef tmp; + for (int _i = oparg; --_i >= 0;) { + tmp = pieces[_i]; + pieces[_i] = PyStackRef_NULL; + PyStackRef_CLOSE(tmp); + } + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -oparg; + assert(WITHIN_STACK_BOUNDS()); + if (str_o == NULL) { + TRACING_JUMP_TO_LABEL(error); + } + str = PyStackRef_FromPyObjectSteal(str_o); + stack_pointer[0] = str; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + TRACING_DISPATCH(); + } + + TRACING_TARGET(BUILD_TEMPLATE) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = BUILD_TEMPLATE; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(BUILD_TEMPLATE); + opcode = BUILD_TEMPLATE; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + _PyStackRef strings; + _PyStackRef interpolations; + _PyStackRef template; + interpolations = stack_pointer[-1]; + strings = stack_pointer[-2]; + PyObject *strings_o = PyStackRef_AsPyObjectBorrow(strings); + PyObject *interpolations_o = PyStackRef_AsPyObjectBorrow(interpolations); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *template_o = _PyTemplate_Build(strings_o, interpolations_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(interpolations); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(strings); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (template_o == NULL) { + TRACING_JUMP_TO_LABEL(error); + } + template = PyStackRef_FromPyObjectSteal(template_o); + stack_pointer[0] = template; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + TRACING_DISPATCH(); + } + + TRACING_TARGET(BUILD_TUPLE) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = BUILD_TUPLE; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(BUILD_TUPLE); + opcode = BUILD_TUPLE; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + _PyStackRef *values; + _PyStackRef tup; + values = &stack_pointer[-oparg]; + PyObject *tup_o = _PyTuple_FromStackRefStealOnSuccess(values, oparg); + if (tup_o == NULL) { + TRACING_JUMP_TO_LABEL(error); + } + tup = PyStackRef_FromPyObjectStealMortal(tup_o); + stack_pointer[-oparg] = tup; + stack_pointer += 1 - oparg; + assert(WITHIN_STACK_BOUNDS()); + TRACING_DISPATCH(); + } + + TRACING_TARGET(CACHE) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = CACHE; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(CACHE); + opcode = CACHE; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + assert(0 && "Executing a cache."); + Py_FatalError("Executing a cache."); + TRACING_DISPATCH(); + } + + TRACING_TARGET(CALL) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = CALL; + (void)(opcode); + #endif + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(CALL); + PREDICTED_TRACING_CALL:; + _Py_CODEUNIT* const this_instr = next_instr - 4; + (void)this_instr; + opcode = CALL; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + opcode = CALL; + _PyStackRef callable; + _PyStackRef self_or_null; + _PyStackRef *args; + _PyStackRef res; + // _SPECIALIZE_CALL + { + self_or_null = stack_pointer[-1 - oparg]; + callable = stack_pointer[-2 - oparg]; + uint16_t counter = read_u16(&this_instr[1].cache); + (void)counter; + #if ENABLE_SPECIALIZATION_FT + if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { + next_instr = this_instr; + _PyFrame_SetStackPointer(frame, stack_pointer); + _Py_Specialize_Call(callable, next_instr, oparg + !PyStackRef_IsNull(self_or_null)); + stack_pointer = _PyFrame_GetStackPointer(frame); + DISPATCH_SAME_OPARG(); + } + OPCODE_DEFERRED_INC(CALL); + ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); + #endif /* ENABLE_SPECIALIZATION_FT */ + } + /* Skip 2 cache entries */ + // _MAYBE_EXPAND_METHOD + { + if (PyStackRef_TYPE(callable) == &PyMethod_Type && PyStackRef_IsNull(self_or_null)) { + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + PyObject *self = ((PyMethodObject *)callable_o)->im_self; + self_or_null = PyStackRef_FromPyObjectNew(self); + PyObject *method = ((PyMethodObject *)callable_o)->im_func; + _PyStackRef temp = callable; + callable = PyStackRef_FromPyObjectNew(method); + stack_pointer[-2 - oparg] = callable; + stack_pointer[-1 - oparg] = self_or_null; + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(temp); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + } + // _DO_CALL + { + args = &stack_pointer[-oparg]; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + int total_args = oparg; + _PyStackRef *arguments = args; + if (!PyStackRef_IsNull(self_or_null)) { + arguments--; + total_args++; + } + if (Py_TYPE(callable_o) == &PyFunction_Type && + tstate->interp->eval_frame == NULL && + ((PyFunctionObject *)callable_o)->vectorcall == _PyFunction_Vectorcall) + { + int code_flags = ((PyCodeObject*)PyFunction_GET_CODE(callable_o))->co_flags; + PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable_o)); + stack_pointer[-2 - oparg] = callable; + stack_pointer[-1 - oparg] = self_or_null; + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyInterpreterFrame *new_frame = _PyEvalFramePushAndInit( + tstate, callable, locals, + arguments, total_args, NULL, frame + ); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -2 - oparg; + assert(WITHIN_STACK_BOUNDS()); + if (new_frame == NULL) { + TRACING_JUMP_TO_LABEL(error); + } + frame->return_offset = 4u ; + TRACING_DISPATCH_INLINED(new_frame); + } + STACKREFS_TO_PYOBJECTS(arguments, total_args, args_o); + if (CONVERSION_FAILED(args_o)) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyStackRef tmp; + for (int _i = oparg; --_i >= 0;) { + tmp = args[_i]; + args[_i] = PyStackRef_NULL; + stack_pointer[-2 - oparg] = callable; + stack_pointer[-1 - oparg] = self_or_null; + PyStackRef_CLOSE(tmp); + } + tmp = self_or_null; + self_or_null = PyStackRef_NULL; + stack_pointer[-1 - oparg] = self_or_null; + PyStackRef_XCLOSE(tmp); + tmp = callable; + callable = PyStackRef_NULL; + stack_pointer[-2 - oparg] = callable; + PyStackRef_CLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -2 - oparg; + assert(WITHIN_STACK_BOUNDS()); + TRACING_JUMP_TO_LABEL(error); + } + stack_pointer[-2 - oparg] = callable; + stack_pointer[-1 - oparg] = self_or_null; + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *res_o = PyObject_Vectorcall( + callable_o, args_o, + total_args | PY_VECTORCALL_ARGUMENTS_OFFSET, + NULL); + stack_pointer = _PyFrame_GetStackPointer(frame); + STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); + if (opcode == INSTRUMENTED_CALL) { + PyObject *arg = total_args == 0 ? + &_PyInstrumentation_MISSING : PyStackRef_AsPyObjectBorrow(arguments[0]); + if (res_o == NULL) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _Py_call_instrumentation_exc2( + tstate, PY_MONITORING_EVENT_C_RAISE, + frame, this_instr, callable_o, arg); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + else { + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = _Py_call_instrumentation_2args( + tstate, PY_MONITORING_EVENT_C_RETURN, + frame, this_instr, callable_o, arg); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err < 0) { + _PyFrame_SetStackPointer(frame, stack_pointer); + Py_CLEAR(res_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + } + } + assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyStackRef tmp; + for (int _i = oparg; --_i >= 0;) { + tmp = args[_i]; + args[_i] = PyStackRef_NULL; + PyStackRef_CLOSE(tmp); + } + tmp = self_or_null; + self_or_null = PyStackRef_NULL; + stack_pointer[-1 - oparg] = self_or_null; + PyStackRef_XCLOSE(tmp); + tmp = callable; + callable = PyStackRef_NULL; + stack_pointer[-2 - oparg] = callable; + PyStackRef_CLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -2 - oparg; + assert(WITHIN_STACK_BOUNDS()); + if (res_o == NULL) { + TRACING_JUMP_TO_LABEL(error); + } + res = PyStackRef_FromPyObjectSteal(res_o); + } + // _CHECK_PERIODIC_AT_END + { + stack_pointer[0] = res; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = check_periodics(tstate); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err != 0) { + TRACING_JUMP_TO_LABEL(error); + } + } + TRACING_DISPATCH(); + } + + TRACING_TARGET(CALL_ALLOC_AND_ENTER_INIT) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = CALL_ALLOC_AND_ENTER_INIT; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(CALL_ALLOC_AND_ENTER_INIT); + opcode = CALL_ALLOC_AND_ENTER_INIT; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); + _PyStackRef callable; + _PyStackRef self_or_null; + _PyStackRef init; + _PyStackRef self; + _PyStackRef *args; + _PyStackRef init_frame; + _PyStackRef new_frame; + /* Skip 1 cache entry */ + // _CHECK_PEP_523 + { + if (tstate->interp->eval_frame) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(TRACING_CALL); + } + } + // _CHECK_AND_ALLOCATE_OBJECT + { + self_or_null = stack_pointer[-1 - oparg]; + callable = stack_pointer[-2 - oparg]; + uint32_t type_version = read_u32(&this_instr[2].cache); + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + if (!PyStackRef_IsNull(self_or_null)) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(TRACING_CALL); + } + if (!PyType_Check(callable_o)) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(TRACING_CALL); + } + PyTypeObject *tp = (PyTypeObject *)callable_o; + if (FT_ATOMIC_LOAD_UINT32_RELAXED(tp->tp_version_tag) != type_version) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(TRACING_CALL); + } + assert(tp->tp_new == PyBaseObject_Type.tp_new); + assert(tp->tp_flags & Py_TPFLAGS_HEAPTYPE); + assert(tp->tp_alloc == PyType_GenericAlloc); + PyHeapTypeObject *cls = (PyHeapTypeObject *)callable_o; + PyFunctionObject *init_func = (PyFunctionObject *)FT_ATOMIC_LOAD_PTR_ACQUIRE(cls->_spec_cache.init); + PyCodeObject *code = (PyCodeObject *)init_func->func_code; + if (!_PyThreadState_HasStackSpace(tstate, code->co_framesize + _Py_InitCleanup.co_framesize)) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(TRACING_CALL); + } + STAT_INC(CALL, hit); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *self_o = PyType_GenericAlloc(tp, 0); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (self_o == NULL) { + TRACING_JUMP_TO_LABEL(error); + } + self_or_null = PyStackRef_FromPyObjectSteal(self_o); + _PyStackRef temp = callable; + callable = PyStackRef_FromPyObjectNew(init_func); + stack_pointer[-2 - oparg] = callable; + stack_pointer[-1 - oparg] = self_or_null; + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(temp); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + // _CREATE_INIT_FRAME + { + args = &stack_pointer[-oparg]; + self = self_or_null; + init = callable; + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyInterpreterFrame *shim = _PyFrame_PushTrampolineUnchecked( + tstate, (PyCodeObject *)&_Py_InitCleanup, 1, frame); + stack_pointer = _PyFrame_GetStackPointer(frame); + assert(_PyFrame_GetBytecode(shim)[0].op.code == EXIT_INIT_CHECK); + assert(_PyFrame_GetBytecode(shim)[1].op.code == RETURN_VALUE); + shim->localsplus[0] = PyStackRef_DUP(self); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyInterpreterFrame *temp = _PyEvalFramePushAndInit( + tstate, init, NULL, args-1, oparg+1, NULL, shim); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -2 - oparg; + assert(WITHIN_STACK_BOUNDS()); + if (temp == NULL) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyEval_FrameClearAndPop(tstate, shim); + stack_pointer = _PyFrame_GetStackPointer(frame); + TRACING_JUMP_TO_LABEL(error); + } + frame->return_offset = 1 + INLINE_CACHE_ENTRIES_CALL; + tstate->py_recursion_remaining--; + init_frame = PyStackRef_Wrap(temp); + } + // _PUSH_FRAME + { + new_frame = init_frame; + assert(tstate->interp->eval_frame == NULL); + _PyInterpreterFrame *temp = PyStackRef_Unwrap(new_frame); + _PyFrame_SetStackPointer(frame, stack_pointer); + assert(temp->previous == frame || temp->previous->previous == frame); + CALL_STAT_INC(inlined_py_calls); + frame = tstate->current_frame = temp; + tstate->py_recursion_remaining--; + LOAD_SP(); + LOAD_IP(0); + LLTRACE_RESUME_FRAME(); + } + TRACING_DISPATCH(); + } + + TRACING_TARGET(CALL_BOUND_METHOD_EXACT_ARGS) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = CALL_BOUND_METHOD_EXACT_ARGS; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(CALL_BOUND_METHOD_EXACT_ARGS); + opcode = CALL_BOUND_METHOD_EXACT_ARGS; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); + _PyStackRef callable; + _PyStackRef null; + _PyStackRef self_or_null; + _PyStackRef *args; + _PyStackRef new_frame; + /* Skip 1 cache entry */ + // _CHECK_PEP_523 + { + if (tstate->interp->eval_frame) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(TRACING_CALL); + } + } + // _CHECK_CALL_BOUND_METHOD_EXACT_ARGS + { + null = stack_pointer[-1 - oparg]; + callable = stack_pointer[-2 - oparg]; + if (!PyStackRef_IsNull(null)) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(TRACING_CALL); + } + if (Py_TYPE(PyStackRef_AsPyObjectBorrow(callable)) != &PyMethod_Type) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(TRACING_CALL); + } + } + // _INIT_CALL_BOUND_METHOD_EXACT_ARGS + { + self_or_null = null; + assert(PyStackRef_IsNull(self_or_null)); + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + STAT_INC(CALL, hit); + self_or_null = PyStackRef_FromPyObjectNew(((PyMethodObject *)callable_o)->im_self); + _PyStackRef temp = callable; + callable = PyStackRef_FromPyObjectNew(((PyMethodObject *)callable_o)->im_func); + stack_pointer[-2 - oparg] = callable; + stack_pointer[-1 - oparg] = self_or_null; + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(temp); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + // flush + // _CHECK_FUNCTION_VERSION + { + uint32_t func_version = read_u32(&this_instr[2].cache); + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + if (!PyFunction_Check(callable_o)) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(TRACING_CALL); + } + PyFunctionObject *func = (PyFunctionObject *)callable_o; + if (func->func_version != func_version) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(TRACING_CALL); + } + } + // _CHECK_FUNCTION_EXACT_ARGS + { + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + assert(PyFunction_Check(callable_o)); + PyFunctionObject *func = (PyFunctionObject *)callable_o; + PyCodeObject *code = (PyCodeObject *)func->func_code; + if (code->co_argcount != oparg + (!PyStackRef_IsNull(self_or_null))) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(TRACING_CALL); + } + } + // _CHECK_STACK_SPACE + { + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + PyFunctionObject *func = (PyFunctionObject *)callable_o; + PyCodeObject *code = (PyCodeObject *)func->func_code; + if (!_PyThreadState_HasStackSpace(tstate, code->co_framesize)) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(TRACING_CALL); + } + } + // _CHECK_RECURSION_REMAINING + { + if (tstate->py_recursion_remaining <= 1) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(TRACING_CALL); + } + } + // _INIT_CALL_PY_EXACT_ARGS + { + args = &stack_pointer[-oparg]; + int has_self = !PyStackRef_IsNull(self_or_null); + STAT_INC(CALL, hit); + _PyInterpreterFrame *pushed_frame = _PyFrame_PushUnchecked(tstate, callable, oparg + has_self, frame); + _PyStackRef *first_non_self_local = pushed_frame->localsplus + has_self; + pushed_frame->localsplus[0] = self_or_null; + for (int i = 0; i < oparg; i++) { + first_non_self_local[i] = args[i]; + } + new_frame = PyStackRef_Wrap(pushed_frame); + } + // _SAVE_RETURN_OFFSET + { + #if TIER_ONE + frame->return_offset = (uint16_t)(next_instr - this_instr); + #endif + #if TIER_TWO + frame->return_offset = oparg; + #endif + } + // _PUSH_FRAME + { + assert(tstate->interp->eval_frame == NULL); + _PyInterpreterFrame *temp = PyStackRef_Unwrap(new_frame); + stack_pointer += -2 - oparg; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + assert(temp->previous == frame || temp->previous->previous == frame); + CALL_STAT_INC(inlined_py_calls); + frame = tstate->current_frame = temp; + tstate->py_recursion_remaining--; + LOAD_SP(); + LOAD_IP(0); + LLTRACE_RESUME_FRAME(); + } + TRACING_DISPATCH(); + } + + TRACING_TARGET(CALL_BOUND_METHOD_GENERAL) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = CALL_BOUND_METHOD_GENERAL; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(CALL_BOUND_METHOD_GENERAL); + opcode = CALL_BOUND_METHOD_GENERAL; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); + _PyStackRef callable; + _PyStackRef null; + _PyStackRef self_or_null; + _PyStackRef *args; + _PyStackRef new_frame; + /* Skip 1 cache entry */ + // _CHECK_PEP_523 + { + if (tstate->interp->eval_frame) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(TRACING_CALL); + } + } + // _CHECK_METHOD_VERSION + { + null = stack_pointer[-1 - oparg]; + callable = stack_pointer[-2 - oparg]; + uint32_t func_version = read_u32(&this_instr[2].cache); + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + if (Py_TYPE(callable_o) != &PyMethod_Type) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(TRACING_CALL); + } + PyObject *func = ((PyMethodObject *)callable_o)->im_func; + if (!PyFunction_Check(func)) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(TRACING_CALL); + } + if (((PyFunctionObject *)func)->func_version != func_version) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(TRACING_CALL); + } + if (!PyStackRef_IsNull(null)) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(TRACING_CALL); + } + } + // _EXPAND_METHOD + { + self_or_null = null; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + assert(PyStackRef_IsNull(self_or_null)); + assert(Py_TYPE(callable_o) == &PyMethod_Type); + self_or_null = PyStackRef_FromPyObjectNew(((PyMethodObject *)callable_o)->im_self); + _PyStackRef temp = callable; + callable = PyStackRef_FromPyObjectNew(((PyMethodObject *)callable_o)->im_func); + assert(PyStackRef_FunctionCheck(callable)); + stack_pointer[-2 - oparg] = callable; + stack_pointer[-1 - oparg] = self_or_null; + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(temp); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + // flush + // _CHECK_RECURSION_REMAINING + { + if (tstate->py_recursion_remaining <= 1) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(TRACING_CALL); + } + } + // _PY_FRAME_GENERAL + { + args = &stack_pointer[-oparg]; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + int total_args = oparg; + if (!PyStackRef_IsNull(self_or_null)) { + args--; + total_args++; + } + assert(Py_TYPE(callable_o) == &PyFunction_Type); + int code_flags = ((PyCodeObject*)PyFunction_GET_CODE(callable_o))->co_flags; + PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable_o)); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyInterpreterFrame *temp = _PyEvalFramePushAndInit( + tstate, callable, locals, + args, total_args, NULL, frame + ); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -2 - oparg; + assert(WITHIN_STACK_BOUNDS()); + if (temp == NULL) { + TRACING_JUMP_TO_LABEL(error); + } + new_frame = PyStackRef_Wrap(temp); + } + // _SAVE_RETURN_OFFSET + { + #if TIER_ONE + frame->return_offset = (uint16_t)(next_instr - this_instr); + #endif + #if TIER_TWO + frame->return_offset = oparg; + #endif + } + // _PUSH_FRAME + { + assert(tstate->interp->eval_frame == NULL); + _PyInterpreterFrame *temp = PyStackRef_Unwrap(new_frame); + _PyFrame_SetStackPointer(frame, stack_pointer); + assert(temp->previous == frame || temp->previous->previous == frame); + CALL_STAT_INC(inlined_py_calls); + frame = tstate->current_frame = temp; + tstate->py_recursion_remaining--; + LOAD_SP(); + LOAD_IP(0); + LLTRACE_RESUME_FRAME(); + } + TRACING_DISPATCH(); + } + + TRACING_TARGET(CALL_BUILTIN_CLASS) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = CALL_BUILTIN_CLASS; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(CALL_BUILTIN_CLASS); + opcode = CALL_BUILTIN_CLASS; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); + _PyStackRef callable; + _PyStackRef self_or_null; + _PyStackRef *args; + _PyStackRef res; + /* Skip 1 cache entry */ + /* Skip 2 cache entries */ + // _CALL_BUILTIN_CLASS + { + args = &stack_pointer[-oparg]; + self_or_null = stack_pointer[-1 - oparg]; + callable = stack_pointer[-2 - oparg]; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + if (!PyType_Check(callable_o)) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(TRACING_CALL); + } + PyTypeObject *tp = (PyTypeObject *)callable_o; + int total_args = oparg; + _PyStackRef *arguments = args; + if (!PyStackRef_IsNull(self_or_null)) { + arguments--; + total_args++; + } + if (tp->tp_vectorcall == NULL) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(TRACING_CALL); + } + STAT_INC(CALL, hit); + STACKREFS_TO_PYOBJECTS(arguments, total_args, args_o); + if (CONVERSION_FAILED(args_o)) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyStackRef tmp; + for (int _i = oparg; --_i >= 0;) { + tmp = args[_i]; + args[_i] = PyStackRef_NULL; + PyStackRef_CLOSE(tmp); + } + tmp = self_or_null; + self_or_null = PyStackRef_NULL; + stack_pointer[-1 - oparg] = self_or_null; + PyStackRef_XCLOSE(tmp); + tmp = callable; + callable = PyStackRef_NULL; + stack_pointer[-2 - oparg] = callable; + PyStackRef_CLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -2 - oparg; + assert(WITHIN_STACK_BOUNDS()); + TRACING_JUMP_TO_LABEL(error); + } + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *res_o = tp->tp_vectorcall((PyObject *)tp, args_o, total_args, NULL); + stack_pointer = _PyFrame_GetStackPointer(frame); + STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyStackRef tmp; + for (int _i = oparg; --_i >= 0;) { + tmp = args[_i]; + args[_i] = PyStackRef_NULL; + PyStackRef_CLOSE(tmp); + } + tmp = self_or_null; + self_or_null = PyStackRef_NULL; + stack_pointer[-1 - oparg] = self_or_null; + PyStackRef_XCLOSE(tmp); + tmp = callable; + callable = PyStackRef_NULL; + stack_pointer[-2 - oparg] = callable; + PyStackRef_CLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -2 - oparg; + assert(WITHIN_STACK_BOUNDS()); + if (res_o == NULL) { + TRACING_JUMP_TO_LABEL(error); + } + res = PyStackRef_FromPyObjectSteal(res_o); + } + // _CHECK_PERIODIC_AT_END + { + stack_pointer[0] = res; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = check_periodics(tstate); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err != 0) { + TRACING_JUMP_TO_LABEL(error); + } + } + TRACING_DISPATCH(); + } + + TRACING_TARGET(CALL_BUILTIN_FAST) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = CALL_BUILTIN_FAST; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(CALL_BUILTIN_FAST); + opcode = CALL_BUILTIN_FAST; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); + _PyStackRef callable; + _PyStackRef self_or_null; + _PyStackRef *args; + _PyStackRef res; + /* Skip 1 cache entry */ + /* Skip 2 cache entries */ + // _CALL_BUILTIN_FAST + { + args = &stack_pointer[-oparg]; + self_or_null = stack_pointer[-1 - oparg]; + callable = stack_pointer[-2 - oparg]; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + int total_args = oparg; + _PyStackRef *arguments = args; + if (!PyStackRef_IsNull(self_or_null)) { + arguments--; + total_args++; + } + if (!PyCFunction_CheckExact(callable_o)) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(TRACING_CALL); + } + if (PyCFunction_GET_FLAGS(callable_o) != METH_FASTCALL) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(TRACING_CALL); + } + STAT_INC(CALL, hit); + PyCFunction cfunc = PyCFunction_GET_FUNCTION(callable_o); + STACKREFS_TO_PYOBJECTS(arguments, total_args, args_o); + if (CONVERSION_FAILED(args_o)) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyStackRef tmp; + for (int _i = oparg; --_i >= 0;) { + tmp = args[_i]; + args[_i] = PyStackRef_NULL; + PyStackRef_CLOSE(tmp); + } + tmp = self_or_null; + self_or_null = PyStackRef_NULL; + stack_pointer[-1 - oparg] = self_or_null; + PyStackRef_XCLOSE(tmp); + tmp = callable; + callable = PyStackRef_NULL; + stack_pointer[-2 - oparg] = callable; + PyStackRef_CLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -2 - oparg; + assert(WITHIN_STACK_BOUNDS()); + TRACING_JUMP_TO_LABEL(error); + } + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *res_o = _PyCFunctionFast_CAST(cfunc)( + PyCFunction_GET_SELF(callable_o), + args_o, + total_args); + stack_pointer = _PyFrame_GetStackPointer(frame); + STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); + assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyStackRef tmp; + for (int _i = oparg; --_i >= 0;) { + tmp = args[_i]; + args[_i] = PyStackRef_NULL; + PyStackRef_CLOSE(tmp); + } + tmp = self_or_null; + self_or_null = PyStackRef_NULL; + stack_pointer[-1 - oparg] = self_or_null; + PyStackRef_XCLOSE(tmp); + tmp = callable; + callable = PyStackRef_NULL; + stack_pointer[-2 - oparg] = callable; + PyStackRef_CLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -2 - oparg; + assert(WITHIN_STACK_BOUNDS()); + if (res_o == NULL) { + TRACING_JUMP_TO_LABEL(error); + } + res = PyStackRef_FromPyObjectSteal(res_o); + } + // _CHECK_PERIODIC_AT_END + { + stack_pointer[0] = res; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = check_periodics(tstate); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err != 0) { + TRACING_JUMP_TO_LABEL(error); + } + } + TRACING_DISPATCH(); + } + + TRACING_TARGET(CALL_BUILTIN_FAST_WITH_KEYWORDS) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = CALL_BUILTIN_FAST_WITH_KEYWORDS; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(CALL_BUILTIN_FAST_WITH_KEYWORDS); + opcode = CALL_BUILTIN_FAST_WITH_KEYWORDS; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); + _PyStackRef callable; + _PyStackRef self_or_null; + _PyStackRef *args; + _PyStackRef res; + /* Skip 1 cache entry */ + /* Skip 2 cache entries */ + // _CALL_BUILTIN_FAST_WITH_KEYWORDS + { + args = &stack_pointer[-oparg]; + self_or_null = stack_pointer[-1 - oparg]; + callable = stack_pointer[-2 - oparg]; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + int total_args = oparg; + _PyStackRef *arguments = args; + if (!PyStackRef_IsNull(self_or_null)) { + arguments--; + total_args++; + } + if (!PyCFunction_CheckExact(callable_o)) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(TRACING_CALL); + } + if (PyCFunction_GET_FLAGS(callable_o) != (METH_FASTCALL | METH_KEYWORDS)) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(TRACING_CALL); + } + STAT_INC(CALL, hit); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyCFunctionFastWithKeywords cfunc = + _PyCFunctionFastWithKeywords_CAST(PyCFunction_GET_FUNCTION(callable_o)); + stack_pointer = _PyFrame_GetStackPointer(frame); + STACKREFS_TO_PYOBJECTS(arguments, total_args, args_o); + if (CONVERSION_FAILED(args_o)) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyStackRef tmp; + for (int _i = oparg; --_i >= 0;) { + tmp = args[_i]; + args[_i] = PyStackRef_NULL; + PyStackRef_CLOSE(tmp); + } + tmp = self_or_null; + self_or_null = PyStackRef_NULL; + stack_pointer[-1 - oparg] = self_or_null; + PyStackRef_XCLOSE(tmp); + tmp = callable; + callable = PyStackRef_NULL; + stack_pointer[-2 - oparg] = callable; + PyStackRef_CLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -2 - oparg; + assert(WITHIN_STACK_BOUNDS()); + TRACING_JUMP_TO_LABEL(error); + } + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *res_o = cfunc(PyCFunction_GET_SELF(callable_o), args_o, total_args, NULL); + stack_pointer = _PyFrame_GetStackPointer(frame); + STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); + assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyStackRef tmp; + for (int _i = oparg; --_i >= 0;) { + tmp = args[_i]; + args[_i] = PyStackRef_NULL; + PyStackRef_CLOSE(tmp); + } + tmp = self_or_null; + self_or_null = PyStackRef_NULL; + stack_pointer[-1 - oparg] = self_or_null; + PyStackRef_XCLOSE(tmp); + tmp = callable; + callable = PyStackRef_NULL; + stack_pointer[-2 - oparg] = callable; + PyStackRef_CLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -2 - oparg; + assert(WITHIN_STACK_BOUNDS()); + if (res_o == NULL) { + TRACING_JUMP_TO_LABEL(error); + } + res = PyStackRef_FromPyObjectSteal(res_o); + } + // _CHECK_PERIODIC_AT_END + { + stack_pointer[0] = res; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = check_periodics(tstate); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err != 0) { + TRACING_JUMP_TO_LABEL(error); + } + } + TRACING_DISPATCH(); + } + + TRACING_TARGET(CALL_BUILTIN_O) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = CALL_BUILTIN_O; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(CALL_BUILTIN_O); + opcode = CALL_BUILTIN_O; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); + _PyStackRef callable; + _PyStackRef self_or_null; + _PyStackRef *args; + _PyStackRef res; + /* Skip 1 cache entry */ + /* Skip 2 cache entries */ + // _CALL_BUILTIN_O + { + args = &stack_pointer[-oparg]; + self_or_null = stack_pointer[-1 - oparg]; + callable = stack_pointer[-2 - oparg]; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + int total_args = oparg; + if (!PyStackRef_IsNull(self_or_null)) { + args--; + total_args++; + } + if (total_args != 1) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(TRACING_CALL); + } + if (!PyCFunction_CheckExact(callable_o)) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(TRACING_CALL); + } + if (PyCFunction_GET_FLAGS(callable_o) != METH_O) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(TRACING_CALL); + } + if (_Py_ReachedRecursionLimit(tstate)) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(TRACING_CALL); + } + STAT_INC(CALL, hit); + PyCFunction cfunc = PyCFunction_GET_FUNCTION(callable_o); + _PyStackRef arg = args[0]; + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *res_o = _PyCFunction_TrampolineCall(cfunc, PyCFunction_GET_SELF(callable_o), PyStackRef_AsPyObjectBorrow(arg)); + stack_pointer = _PyFrame_GetStackPointer(frame); + _Py_LeaveRecursiveCallTstate(tstate); + assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(arg); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -2 - oparg; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(callable); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (res_o == NULL) { + TRACING_JUMP_TO_LABEL(error); + } + res = PyStackRef_FromPyObjectSteal(res_o); + } + // _CHECK_PERIODIC_AT_END + { + stack_pointer[0] = res; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = check_periodics(tstate); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err != 0) { + TRACING_JUMP_TO_LABEL(error); + } + } + TRACING_DISPATCH(); + } + + TRACING_TARGET(CALL_FUNCTION_EX) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = CALL_FUNCTION_EX; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(CALL_FUNCTION_EX); + opcode = CALL_FUNCTION_EX; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + opcode = CALL_FUNCTION_EX; + _PyStackRef func; + _PyStackRef callargs; + _PyStackRef func_st; + _PyStackRef null; + _PyStackRef callargs_st; + _PyStackRef kwargs_st; + _PyStackRef result; + // _MAKE_CALLARGS_A_TUPLE + { + callargs = stack_pointer[-2]; + func = stack_pointer[-4]; + PyObject *callargs_o = PyStackRef_AsPyObjectBorrow(callargs); + if (!PyTuple_CheckExact(callargs_o)) { + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = _Py_Check_ArgsIterable(tstate, PyStackRef_AsPyObjectBorrow(func), callargs_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err < 0) { + TRACING_JUMP_TO_LABEL(error); + } + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *tuple_o = PySequence_Tuple(callargs_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (tuple_o == NULL) { + TRACING_JUMP_TO_LABEL(error); + } + _PyStackRef temp = callargs; + callargs = PyStackRef_FromPyObjectSteal(tuple_o); + stack_pointer[-2] = callargs; + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(temp); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + } + // _DO_CALL_FUNCTION_EX + { + kwargs_st = stack_pointer[-1]; + callargs_st = callargs; + null = stack_pointer[-3]; + func_st = func; + (void)null; + PyObject *func = PyStackRef_AsPyObjectBorrow(func_st); + EVAL_CALL_STAT_INC_IF_FUNCTION(EVAL_CALL_FUNCTION_EX, func); + PyObject *result_o; + assert(!_PyErr_Occurred(tstate)); + if (opcode == INSTRUMENTED_CALL_FUNCTION_EX) { + PyObject *callargs = PyStackRef_AsPyObjectBorrow(callargs_st); + PyObject *kwargs = PyStackRef_AsPyObjectBorrow(kwargs_st); + assert(kwargs == NULL || PyDict_CheckExact(kwargs)); + assert(PyTuple_CheckExact(callargs)); + PyObject *arg = PyTuple_GET_SIZE(callargs) > 0 ? + PyTuple_GET_ITEM(callargs, 0) : &_PyInstrumentation_MISSING; + stack_pointer[-2] = callargs_st; + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = _Py_call_instrumentation_2args( + tstate, PY_MONITORING_EVENT_CALL, + frame, this_instr, func, arg); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err) { + TRACING_JUMP_TO_LABEL(error); + } + _PyFrame_SetStackPointer(frame, stack_pointer); + result_o = PyObject_Call(func, callargs, kwargs); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (!PyFunction_Check(func) && !PyMethod_Check(func)) { + if (result_o == NULL) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _Py_call_instrumentation_exc2( + tstate, PY_MONITORING_EVENT_C_RAISE, + frame, this_instr, func, arg); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + else { + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = _Py_call_instrumentation_2args( + tstate, PY_MONITORING_EVENT_C_RETURN, + frame, this_instr, func, arg); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err < 0) { + _PyFrame_SetStackPointer(frame, stack_pointer); + Py_CLEAR(result_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + } + } + } + else { + if (Py_TYPE(func) == &PyFunction_Type && + tstate->interp->eval_frame == NULL && + ((PyFunctionObject *)func)->vectorcall == _PyFunction_Vectorcall) { + PyObject *callargs = PyStackRef_AsPyObjectSteal(callargs_st); + assert(PyTuple_CheckExact(callargs)); + PyObject *kwargs = PyStackRef_IsNull(kwargs_st) ? NULL : PyStackRef_AsPyObjectSteal(kwargs_st); + assert(kwargs == NULL || PyDict_CheckExact(kwargs)); + Py_ssize_t nargs = PyTuple_GET_SIZE(callargs); + int code_flags = ((PyCodeObject *)PyFunction_GET_CODE(func))->co_flags; + PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(func)); + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyInterpreterFrame *new_frame = _PyEvalFramePushAndInit_Ex( + tstate, func_st, locals, + nargs, callargs, kwargs, frame); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); + if (new_frame == NULL) { + TRACING_JUMP_TO_LABEL(error); + } + assert( 1u == 1); + frame->return_offset = 1; + TRACING_DISPATCH_INLINED(new_frame); + } + PyObject *callargs = PyStackRef_AsPyObjectBorrow(callargs_st); + assert(PyTuple_CheckExact(callargs)); + PyObject *kwargs = PyStackRef_AsPyObjectBorrow(kwargs_st); + assert(kwargs == NULL || PyDict_CheckExact(kwargs)); + stack_pointer[-2] = callargs_st; + _PyFrame_SetStackPointer(frame, stack_pointer); + result_o = PyObject_Call(func, callargs, kwargs); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_XCLOSE(kwargs_st); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(callargs_st); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(func_st); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (result_o == NULL) { + TRACING_JUMP_TO_LABEL(error); + } + result = PyStackRef_FromPyObjectSteal(result_o); + } + // _CHECK_PERIODIC_AT_END + { + stack_pointer[0] = result; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = check_periodics(tstate); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err != 0) { + TRACING_JUMP_TO_LABEL(error); + } + } + TRACING_DISPATCH(); + } + + TRACING_TARGET(CALL_INTRINSIC_1) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = CALL_INTRINSIC_1; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(CALL_INTRINSIC_1); + opcode = CALL_INTRINSIC_1; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + _PyStackRef value; + _PyStackRef res; + value = stack_pointer[-1]; + assert(oparg <= MAX_INTRINSIC_1); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *res_o = _PyIntrinsics_UnaryFunctions[oparg].func(tstate, PyStackRef_AsPyObjectBorrow(value)); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(value); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (res_o == NULL) { + TRACING_JUMP_TO_LABEL(error); + } + res = PyStackRef_FromPyObjectSteal(res_o); + stack_pointer[0] = res; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + TRACING_DISPATCH(); + } + + TRACING_TARGET(CALL_INTRINSIC_2) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = CALL_INTRINSIC_2; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(CALL_INTRINSIC_2); + opcode = CALL_INTRINSIC_2; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + _PyStackRef value2_st; + _PyStackRef value1_st; + _PyStackRef res; + value1_st = stack_pointer[-1]; + value2_st = stack_pointer[-2]; + assert(oparg <= MAX_INTRINSIC_2); + PyObject *value1 = PyStackRef_AsPyObjectBorrow(value1_st); + PyObject *value2 = PyStackRef_AsPyObjectBorrow(value2_st); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *res_o = _PyIntrinsics_BinaryFunctions[oparg].func(tstate, value2, value1); + _PyStackRef tmp = value1_st; + value1_st = PyStackRef_NULL; + stack_pointer[-1] = value1_st; + PyStackRef_CLOSE(tmp); + tmp = value2_st; + value2_st = PyStackRef_NULL; + stack_pointer[-2] = value2_st; + PyStackRef_CLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); + if (res_o == NULL) { + TRACING_JUMP_TO_LABEL(error); + } + res = PyStackRef_FromPyObjectSteal(res_o); + stack_pointer[0] = res; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + TRACING_DISPATCH(); + } + + TRACING_TARGET(CALL_ISINSTANCE) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = CALL_ISINSTANCE; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(CALL_ISINSTANCE); + opcode = CALL_ISINSTANCE; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); + _PyStackRef null; + _PyStackRef callable; + _PyStackRef instance; + _PyStackRef cls; + _PyStackRef res; + /* Skip 1 cache entry */ + /* Skip 2 cache entries */ + // _GUARD_THIRD_NULL + { + null = stack_pointer[-3]; + if (!PyStackRef_IsNull(null)) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(TRACING_CALL); + } + } + // _GUARD_CALLABLE_ISINSTANCE + { + callable = stack_pointer[-4]; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + PyInterpreterState *interp = tstate->interp; + if (callable_o != interp->callable_cache.isinstance) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(TRACING_CALL); + } + } + // _CALL_ISINSTANCE + { + cls = stack_pointer[-1]; + instance = stack_pointer[-2]; + STAT_INC(CALL, hit); + PyObject *inst_o = PyStackRef_AsPyObjectBorrow(instance); + PyObject *cls_o = PyStackRef_AsPyObjectBorrow(cls); + _PyFrame_SetStackPointer(frame, stack_pointer); + int retval = PyObject_IsInstance(inst_o, cls_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (retval < 0) { + TRACING_JUMP_TO_LABEL(error); + } + (void)null; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(cls); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(instance); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(callable); + stack_pointer = _PyFrame_GetStackPointer(frame); + res = retval ? PyStackRef_True : PyStackRef_False; + assert((!PyStackRef_IsNull(res)) ^ (_PyErr_Occurred(tstate) != NULL)); + } + stack_pointer[0] = res; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + TRACING_DISPATCH(); + } + + TRACING_TARGET(CALL_KW) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = CALL_KW; + (void)(opcode); + #endif + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(CALL_KW); + PREDICTED_TRACING_CALL_KW:; + _Py_CODEUNIT* const this_instr = next_instr - 4; + (void)this_instr; + opcode = CALL_KW; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + opcode = CALL_KW; + _PyStackRef callable; + _PyStackRef self_or_null; + _PyStackRef *args; + _PyStackRef kwnames; + _PyStackRef res; + // _SPECIALIZE_CALL_KW + { + self_or_null = stack_pointer[-2 - oparg]; + callable = stack_pointer[-3 - oparg]; + uint16_t counter = read_u16(&this_instr[1].cache); + (void)counter; + #if ENABLE_SPECIALIZATION_FT + if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { + next_instr = this_instr; + _PyFrame_SetStackPointer(frame, stack_pointer); + _Py_Specialize_CallKw(callable, next_instr, oparg + !PyStackRef_IsNull(self_or_null)); + stack_pointer = _PyFrame_GetStackPointer(frame); + DISPATCH_SAME_OPARG(); + } + OPCODE_DEFERRED_INC(CALL_KW); + ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); + #endif /* ENABLE_SPECIALIZATION_FT */ + } + /* Skip 2 cache entries */ + // _MAYBE_EXPAND_METHOD_KW + { + if (PyStackRef_TYPE(callable) == &PyMethod_Type && PyStackRef_IsNull(self_or_null)) { + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + PyObject *self = ((PyMethodObject *)callable_o)->im_self; + self_or_null = PyStackRef_FromPyObjectNew(self); + PyObject *method = ((PyMethodObject *)callable_o)->im_func; + _PyStackRef temp = callable; + callable = PyStackRef_FromPyObjectNew(method); + stack_pointer[-3 - oparg] = callable; + stack_pointer[-2 - oparg] = self_or_null; + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(temp); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + } + // _DO_CALL_KW + { + kwnames = stack_pointer[-1]; + args = &stack_pointer[-1 - oparg]; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + PyObject *kwnames_o = PyStackRef_AsPyObjectBorrow(kwnames); + int total_args = oparg; + _PyStackRef *arguments = args; + if (!PyStackRef_IsNull(self_or_null)) { + arguments--; + total_args++; + } + int positional_args = total_args - (int)PyTuple_GET_SIZE(kwnames_o); + if (Py_TYPE(callable_o) == &PyFunction_Type && + tstate->interp->eval_frame == NULL && + ((PyFunctionObject *)callable_o)->vectorcall == _PyFunction_Vectorcall) + { + int code_flags = ((PyCodeObject*)PyFunction_GET_CODE(callable_o))->co_flags; + PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable_o)); + stack_pointer[-3 - oparg] = callable; + stack_pointer[-2 - oparg] = self_or_null; + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyInterpreterFrame *new_frame = _PyEvalFramePushAndInit( + tstate, callable, locals, + arguments, positional_args, kwnames_o, frame + ); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -3 - oparg; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(kwnames); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (new_frame == NULL) { + TRACING_JUMP_TO_LABEL(error); + } + assert( 4u == 1 + INLINE_CACHE_ENTRIES_CALL_KW); + frame->return_offset = 4u ; + TRACING_DISPATCH_INLINED(new_frame); + } + STACKREFS_TO_PYOBJECTS(arguments, total_args, args_o); + if (CONVERSION_FAILED(args_o)) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyStackRef tmp = kwnames; + kwnames = PyStackRef_NULL; + stack_pointer[-3 - oparg] = callable; + stack_pointer[-2 - oparg] = self_or_null; + stack_pointer[-1] = kwnames; + PyStackRef_CLOSE(tmp); + for (int _i = oparg; --_i >= 0;) { + tmp = args[_i]; + args[_i] = PyStackRef_NULL; + PyStackRef_CLOSE(tmp); + } + tmp = self_or_null; + self_or_null = PyStackRef_NULL; + stack_pointer[-2 - oparg] = self_or_null; + PyStackRef_XCLOSE(tmp); + tmp = callable; + callable = PyStackRef_NULL; + stack_pointer[-3 - oparg] = callable; + PyStackRef_CLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -3 - oparg; + assert(WITHIN_STACK_BOUNDS()); + TRACING_JUMP_TO_LABEL(error); + } + stack_pointer[-3 - oparg] = callable; + stack_pointer[-2 - oparg] = self_or_null; + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *res_o = PyObject_Vectorcall( + callable_o, args_o, + positional_args | PY_VECTORCALL_ARGUMENTS_OFFSET, + kwnames_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); + if (opcode == INSTRUMENTED_CALL_KW) { + PyObject *arg = total_args == 0 ? + &_PyInstrumentation_MISSING : PyStackRef_AsPyObjectBorrow(arguments[0]); + if (res_o == NULL) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _Py_call_instrumentation_exc2( + tstate, PY_MONITORING_EVENT_C_RAISE, + frame, this_instr, callable_o, arg); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + else { + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = _Py_call_instrumentation_2args( + tstate, PY_MONITORING_EVENT_C_RETURN, + frame, this_instr, callable_o, arg); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err < 0) { + _PyFrame_SetStackPointer(frame, stack_pointer); + Py_CLEAR(res_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + } + } + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyStackRef tmp = kwnames; + kwnames = PyStackRef_NULL; + stack_pointer[-1] = kwnames; + PyStackRef_CLOSE(tmp); + for (int _i = oparg; --_i >= 0;) { + tmp = args[_i]; + args[_i] = PyStackRef_NULL; + PyStackRef_CLOSE(tmp); + } + tmp = self_or_null; + self_or_null = PyStackRef_NULL; + stack_pointer[-2 - oparg] = self_or_null; + PyStackRef_XCLOSE(tmp); + tmp = callable; + callable = PyStackRef_NULL; + stack_pointer[-3 - oparg] = callable; + PyStackRef_CLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -3 - oparg; + assert(WITHIN_STACK_BOUNDS()); + if (res_o == NULL) { + TRACING_JUMP_TO_LABEL(error); + } + res = PyStackRef_FromPyObjectSteal(res_o); + } + stack_pointer[0] = res; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + TRACING_DISPATCH(); + } + + TRACING_TARGET(CALL_KW_BOUND_METHOD) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = CALL_KW_BOUND_METHOD; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(CALL_KW_BOUND_METHOD); + opcode = CALL_KW_BOUND_METHOD; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + static_assert(INLINE_CACHE_ENTRIES_CALL_KW == 3, "incorrect cache size"); + _PyStackRef callable; + _PyStackRef null; + _PyStackRef self_or_null; + _PyStackRef *args; + _PyStackRef kwnames; + _PyStackRef new_frame; + /* Skip 1 cache entry */ + // _CHECK_PEP_523 + { + if (tstate->interp->eval_frame) { + UPDATE_MISS_STATS(CALL_KW); + assert(_PyOpcode_Deopt[opcode] == (CALL_KW)); + JUMP_TO_PREDICTED(TRACING_CALL_KW); + } + } + // _CHECK_METHOD_VERSION_KW + { + null = stack_pointer[-2 - oparg]; + callable = stack_pointer[-3 - oparg]; + uint32_t func_version = read_u32(&this_instr[2].cache); + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + if (Py_TYPE(callable_o) != &PyMethod_Type) { + UPDATE_MISS_STATS(CALL_KW); + assert(_PyOpcode_Deopt[opcode] == (CALL_KW)); + JUMP_TO_PREDICTED(TRACING_CALL_KW); + } + PyObject *func = ((PyMethodObject *)callable_o)->im_func; + if (!PyFunction_Check(func)) { + UPDATE_MISS_STATS(CALL_KW); + assert(_PyOpcode_Deopt[opcode] == (CALL_KW)); + JUMP_TO_PREDICTED(TRACING_CALL_KW); + } + if (((PyFunctionObject *)func)->func_version != func_version) { + UPDATE_MISS_STATS(CALL_KW); + assert(_PyOpcode_Deopt[opcode] == (CALL_KW)); + JUMP_TO_PREDICTED(TRACING_CALL_KW); + } + if (!PyStackRef_IsNull(null)) { + UPDATE_MISS_STATS(CALL_KW); + assert(_PyOpcode_Deopt[opcode] == (CALL_KW)); + JUMP_TO_PREDICTED(TRACING_CALL_KW); + } + } + // _EXPAND_METHOD_KW + { + self_or_null = null; + assert(PyStackRef_IsNull(self_or_null)); + _PyStackRef callable_s = callable; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + assert(Py_TYPE(callable_o) == &PyMethod_Type); + self_or_null = PyStackRef_FromPyObjectNew(((PyMethodObject *)callable_o)->im_self); + callable = PyStackRef_FromPyObjectNew(((PyMethodObject *)callable_o)->im_func); + assert(PyStackRef_FunctionCheck(callable)); + stack_pointer[-3 - oparg] = callable; + stack_pointer[-2 - oparg] = self_or_null; + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(callable_s); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + // flush + // _PY_FRAME_KW + { + kwnames = stack_pointer[-1]; + args = &stack_pointer[-1 - oparg]; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + int total_args = oparg; + _PyStackRef *arguments = args; + if (!PyStackRef_IsNull(self_or_null)) { + arguments--; + total_args++; + } + PyObject *kwnames_o = PyStackRef_AsPyObjectBorrow(kwnames); + int positional_args = total_args - (int)PyTuple_GET_SIZE(kwnames_o); + assert(Py_TYPE(callable_o) == &PyFunction_Type); + int code_flags = ((PyCodeObject*)PyFunction_GET_CODE(callable_o))->co_flags; + PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable_o)); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyInterpreterFrame *temp = _PyEvalFramePushAndInit( + tstate, callable, locals, + arguments, positional_args, kwnames_o, frame + ); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(kwnames); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -2 - oparg; + assert(WITHIN_STACK_BOUNDS()); + if (temp == NULL) { + TRACING_JUMP_TO_LABEL(error); + } + new_frame = PyStackRef_Wrap(temp); + } + // _SAVE_RETURN_OFFSET + { + #if TIER_ONE + frame->return_offset = (uint16_t)(next_instr - this_instr); + #endif + #if TIER_TWO + frame->return_offset = oparg; + #endif + } + // _PUSH_FRAME + { + assert(tstate->interp->eval_frame == NULL); + _PyInterpreterFrame *temp = PyStackRef_Unwrap(new_frame); + _PyFrame_SetStackPointer(frame, stack_pointer); + assert(temp->previous == frame || temp->previous->previous == frame); + CALL_STAT_INC(inlined_py_calls); + frame = tstate->current_frame = temp; + tstate->py_recursion_remaining--; + LOAD_SP(); + LOAD_IP(0); + LLTRACE_RESUME_FRAME(); + } + TRACING_DISPATCH(); + } + + TRACING_TARGET(CALL_KW_NON_PY) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = CALL_KW_NON_PY; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(CALL_KW_NON_PY); + opcode = CALL_KW_NON_PY; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + opcode = CALL_KW_NON_PY; + static_assert(INLINE_CACHE_ENTRIES_CALL_KW == 3, "incorrect cache size"); + _PyStackRef callable; + _PyStackRef self_or_null; + _PyStackRef *args; + _PyStackRef kwnames; + _PyStackRef res; + /* Skip 1 cache entry */ + /* Skip 2 cache entries */ + // _CHECK_IS_NOT_PY_CALLABLE_KW + { + callable = stack_pointer[-3 - oparg]; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + if (PyFunction_Check(callable_o)) { + UPDATE_MISS_STATS(CALL_KW); + assert(_PyOpcode_Deopt[opcode] == (CALL_KW)); + JUMP_TO_PREDICTED(TRACING_CALL_KW); + } + if (Py_TYPE(callable_o) == &PyMethod_Type) { + UPDATE_MISS_STATS(CALL_KW); + assert(_PyOpcode_Deopt[opcode] == (CALL_KW)); + JUMP_TO_PREDICTED(TRACING_CALL_KW); + } + } + // _CALL_KW_NON_PY + { + kwnames = stack_pointer[-1]; + args = &stack_pointer[-1 - oparg]; + self_or_null = stack_pointer[-2 - oparg]; + #if TIER_ONE + assert(opcode != INSTRUMENTED_CALL); + #endif + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + int total_args = oparg; + _PyStackRef *arguments = args; + if (!PyStackRef_IsNull(self_or_null)) { + arguments--; + total_args++; + } + STACKREFS_TO_PYOBJECTS(arguments, total_args, args_o); + if (CONVERSION_FAILED(args_o)) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyStackRef tmp = kwnames; + kwnames = PyStackRef_NULL; + stack_pointer[-1] = kwnames; + PyStackRef_CLOSE(tmp); + for (int _i = oparg; --_i >= 0;) { + tmp = args[_i]; + args[_i] = PyStackRef_NULL; + PyStackRef_CLOSE(tmp); + } + tmp = self_or_null; + self_or_null = PyStackRef_NULL; + stack_pointer[-2 - oparg] = self_or_null; + PyStackRef_XCLOSE(tmp); + tmp = callable; + callable = PyStackRef_NULL; + stack_pointer[-3 - oparg] = callable; + PyStackRef_CLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -3 - oparg; + assert(WITHIN_STACK_BOUNDS()); + TRACING_JUMP_TO_LABEL(error); + } + PyObject *kwnames_o = PyStackRef_AsPyObjectBorrow(kwnames); + int positional_args = total_args - (int)PyTuple_GET_SIZE(kwnames_o); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *res_o = PyObject_Vectorcall( + callable_o, args_o, + positional_args | PY_VECTORCALL_ARGUMENTS_OFFSET, + kwnames_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(kwnames); + stack_pointer = _PyFrame_GetStackPointer(frame); + STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); + assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyStackRef tmp; + for (int _i = oparg; --_i >= 0;) { + tmp = args[_i]; + args[_i] = PyStackRef_NULL; + PyStackRef_CLOSE(tmp); + } + tmp = self_or_null; + self_or_null = PyStackRef_NULL; + stack_pointer[-1 - oparg] = self_or_null; + PyStackRef_XCLOSE(tmp); + tmp = callable; + callable = PyStackRef_NULL; + stack_pointer[-2 - oparg] = callable; + PyStackRef_CLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -2 - oparg; + assert(WITHIN_STACK_BOUNDS()); + if (res_o == NULL) { + TRACING_JUMP_TO_LABEL(error); + } + res = PyStackRef_FromPyObjectSteal(res_o); + } + // _CHECK_PERIODIC_AT_END + { + stack_pointer[0] = res; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = check_periodics(tstate); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err != 0) { + TRACING_JUMP_TO_LABEL(error); + } + } + TRACING_DISPATCH(); + } + + TRACING_TARGET(CALL_KW_PY) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = CALL_KW_PY; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(CALL_KW_PY); + opcode = CALL_KW_PY; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + static_assert(INLINE_CACHE_ENTRIES_CALL_KW == 3, "incorrect cache size"); + _PyStackRef callable; + _PyStackRef self_or_null; + _PyStackRef *args; + _PyStackRef kwnames; + _PyStackRef new_frame; + /* Skip 1 cache entry */ + // _CHECK_PEP_523 + { + if (tstate->interp->eval_frame) { + UPDATE_MISS_STATS(CALL_KW); + assert(_PyOpcode_Deopt[opcode] == (CALL_KW)); + JUMP_TO_PREDICTED(TRACING_CALL_KW); + } + } + // _CHECK_FUNCTION_VERSION_KW + { + callable = stack_pointer[-3 - oparg]; + uint32_t func_version = read_u32(&this_instr[2].cache); + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + if (!PyFunction_Check(callable_o)) { + UPDATE_MISS_STATS(CALL_KW); + assert(_PyOpcode_Deopt[opcode] == (CALL_KW)); + JUMP_TO_PREDICTED(TRACING_CALL_KW); + } + PyFunctionObject *func = (PyFunctionObject *)callable_o; + if (func->func_version != func_version) { + UPDATE_MISS_STATS(CALL_KW); + assert(_PyOpcode_Deopt[opcode] == (CALL_KW)); + JUMP_TO_PREDICTED(TRACING_CALL_KW); + } + } + // _CHECK_RECURSION_REMAINING + { + if (tstate->py_recursion_remaining <= 1) { + UPDATE_MISS_STATS(CALL_KW); + assert(_PyOpcode_Deopt[opcode] == (CALL_KW)); + JUMP_TO_PREDICTED(TRACING_CALL_KW); + } + } + // _PY_FRAME_KW + { + kwnames = stack_pointer[-1]; + args = &stack_pointer[-1 - oparg]; + self_or_null = stack_pointer[-2 - oparg]; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + int total_args = oparg; + _PyStackRef *arguments = args; + if (!PyStackRef_IsNull(self_or_null)) { + arguments--; + total_args++; + } + PyObject *kwnames_o = PyStackRef_AsPyObjectBorrow(kwnames); + int positional_args = total_args - (int)PyTuple_GET_SIZE(kwnames_o); + assert(Py_TYPE(callable_o) == &PyFunction_Type); + int code_flags = ((PyCodeObject*)PyFunction_GET_CODE(callable_o))->co_flags; + PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable_o)); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyInterpreterFrame *temp = _PyEvalFramePushAndInit( + tstate, callable, locals, + arguments, positional_args, kwnames_o, frame + ); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(kwnames); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -2 - oparg; + assert(WITHIN_STACK_BOUNDS()); + if (temp == NULL) { + TRACING_JUMP_TO_LABEL(error); + } + new_frame = PyStackRef_Wrap(temp); + } + // _SAVE_RETURN_OFFSET + { + #if TIER_ONE + frame->return_offset = (uint16_t)(next_instr - this_instr); + #endif + #if TIER_TWO + frame->return_offset = oparg; + #endif + } + // _PUSH_FRAME + { + assert(tstate->interp->eval_frame == NULL); + _PyInterpreterFrame *temp = PyStackRef_Unwrap(new_frame); + _PyFrame_SetStackPointer(frame, stack_pointer); + assert(temp->previous == frame || temp->previous->previous == frame); + CALL_STAT_INC(inlined_py_calls); + frame = tstate->current_frame = temp; + tstate->py_recursion_remaining--; + LOAD_SP(); + LOAD_IP(0); + LLTRACE_RESUME_FRAME(); + } + TRACING_DISPATCH(); + } + + TRACING_TARGET(CALL_LEN) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = CALL_LEN; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(CALL_LEN); + opcode = CALL_LEN; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); + _PyStackRef null; + _PyStackRef callable; + _PyStackRef arg; + _PyStackRef res; + /* Skip 1 cache entry */ + /* Skip 2 cache entries */ + // _GUARD_NOS_NULL + { + null = stack_pointer[-2]; + if (!PyStackRef_IsNull(null)) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(TRACING_CALL); + } + } + // _GUARD_CALLABLE_LEN + { + callable = stack_pointer[-3]; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + PyInterpreterState *interp = tstate->interp; + if (callable_o != interp->callable_cache.len) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(TRACING_CALL); + } + } + // _CALL_LEN + { + arg = stack_pointer[-1]; + (void)null; + STAT_INC(CALL, hit); + PyObject *arg_o = PyStackRef_AsPyObjectBorrow(arg); + _PyFrame_SetStackPointer(frame, stack_pointer); + Py_ssize_t len_i = PyObject_Length(arg_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (len_i < 0) { + TRACING_JUMP_TO_LABEL(error); + } + PyObject *res_o = PyLong_FromSsize_t(len_i); + assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); + if (res_o == NULL) { + TRACING_JUMP_TO_LABEL(error); + } + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(arg); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(callable); + stack_pointer = _PyFrame_GetStackPointer(frame); + res = PyStackRef_FromPyObjectSteal(res_o); + } + stack_pointer[0] = res; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + TRACING_DISPATCH(); + } + + TRACING_TARGET(CALL_LIST_APPEND) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = CALL_LIST_APPEND; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(CALL_LIST_APPEND); + opcode = CALL_LIST_APPEND; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); + _PyStackRef callable; + _PyStackRef nos; + _PyStackRef self; + _PyStackRef arg; + /* Skip 1 cache entry */ + /* Skip 2 cache entries */ + // _GUARD_CALLABLE_LIST_APPEND + { + callable = stack_pointer[-3]; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + PyInterpreterState *interp = tstate->interp; + if (callable_o != interp->callable_cache.list_append) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(TRACING_CALL); + } + } + // _GUARD_NOS_NOT_NULL + { + nos = stack_pointer[-2]; + PyObject *o = PyStackRef_AsPyObjectBorrow(nos); + if (o == NULL) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(TRACING_CALL); + } + } + // _GUARD_NOS_LIST + { + PyObject *o = PyStackRef_AsPyObjectBorrow(nos); + if (!PyList_CheckExact(o)) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(TRACING_CALL); + } + } + // _CALL_LIST_APPEND + { + arg = stack_pointer[-1]; + self = nos; + assert(oparg == 1); + PyObject *self_o = PyStackRef_AsPyObjectBorrow(self); + if (!PyList_CheckExact(self_o)) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(TRACING_CALL); + } + if (!LOCK_OBJECT(self_o)) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(TRACING_CALL); + } + STAT_INC(CALL, hit); + int err = _PyList_AppendTakeRef((PyListObject *)self_o, PyStackRef_AsPyObjectSteal(arg)); + UNLOCK_OBJECT(self_o); + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(self); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(callable); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err) { + TRACING_JUMP_TO_LABEL(error); + } + #if TIER_ONE + + assert(next_instr->op.code == POP_TOP); + SKIP_OVER(1); + #endif + } + TRACING_DISPATCH(); + } + + TRACING_TARGET(CALL_METHOD_DESCRIPTOR_FAST) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = CALL_METHOD_DESCRIPTOR_FAST; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(CALL_METHOD_DESCRIPTOR_FAST); + opcode = CALL_METHOD_DESCRIPTOR_FAST; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); + _PyStackRef callable; + _PyStackRef self_or_null; + _PyStackRef *args; + _PyStackRef res; + /* Skip 1 cache entry */ + /* Skip 2 cache entries */ + // _CALL_METHOD_DESCRIPTOR_FAST + { + args = &stack_pointer[-oparg]; + self_or_null = stack_pointer[-1 - oparg]; + callable = stack_pointer[-2 - oparg]; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + int total_args = oparg; + _PyStackRef *arguments = args; + if (!PyStackRef_IsNull(self_or_null)) { + arguments--; + total_args++; + } + if (total_args == 0) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(TRACING_CALL); + } + PyMethodDescrObject *method = (PyMethodDescrObject *)callable_o; + if (!Py_IS_TYPE(method, &PyMethodDescr_Type)) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(TRACING_CALL); + } + PyMethodDef *meth = method->d_method; + if (meth->ml_flags != METH_FASTCALL) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(TRACING_CALL); + } + PyObject *self = PyStackRef_AsPyObjectBorrow(arguments[0]); + assert(self != NULL); + if (!Py_IS_TYPE(self, method->d_common.d_type)) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(TRACING_CALL); + } + STAT_INC(CALL, hit); + int nargs = total_args - 1; + STACKREFS_TO_PYOBJECTS(arguments, total_args, args_o); + if (CONVERSION_FAILED(args_o)) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyStackRef tmp; + for (int _i = oparg; --_i >= 0;) { + tmp = args[_i]; + args[_i] = PyStackRef_NULL; + PyStackRef_CLOSE(tmp); + } + tmp = self_or_null; + self_or_null = PyStackRef_NULL; + stack_pointer[-1 - oparg] = self_or_null; + PyStackRef_XCLOSE(tmp); + tmp = callable; + callable = PyStackRef_NULL; + stack_pointer[-2 - oparg] = callable; + PyStackRef_CLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -2 - oparg; + assert(WITHIN_STACK_BOUNDS()); + TRACING_JUMP_TO_LABEL(error); + } + _PyFrame_SetStackPointer(frame, stack_pointer); + PyCFunctionFast cfunc = _PyCFunctionFast_CAST(meth->ml_meth); + PyObject *res_o = cfunc(self, (args_o + 1), nargs); + stack_pointer = _PyFrame_GetStackPointer(frame); + STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); + assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyStackRef tmp; + for (int _i = oparg; --_i >= 0;) { + tmp = args[_i]; + args[_i] = PyStackRef_NULL; + PyStackRef_CLOSE(tmp); + } + tmp = self_or_null; + self_or_null = PyStackRef_NULL; + stack_pointer[-1 - oparg] = self_or_null; + PyStackRef_XCLOSE(tmp); + tmp = callable; + callable = PyStackRef_NULL; + stack_pointer[-2 - oparg] = callable; + PyStackRef_CLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -2 - oparg; + assert(WITHIN_STACK_BOUNDS()); + if (res_o == NULL) { + TRACING_JUMP_TO_LABEL(error); + } + res = PyStackRef_FromPyObjectSteal(res_o); + } + // _CHECK_PERIODIC_AT_END + { + stack_pointer[0] = res; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = check_periodics(tstate); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err != 0) { + TRACING_JUMP_TO_LABEL(error); + } + } + TRACING_DISPATCH(); + } + + TRACING_TARGET(CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS); + opcode = CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); + _PyStackRef callable; + _PyStackRef self_or_null; + _PyStackRef *args; + _PyStackRef res; + /* Skip 1 cache entry */ + /* Skip 2 cache entries */ + // _CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS + { + args = &stack_pointer[-oparg]; + self_or_null = stack_pointer[-1 - oparg]; + callable = stack_pointer[-2 - oparg]; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + int total_args = oparg; + _PyStackRef *arguments = args; + if (!PyStackRef_IsNull(self_or_null)) { + arguments--; + total_args++; + } + if (total_args == 0) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(TRACING_CALL); + } + PyMethodDescrObject *method = (PyMethodDescrObject *)callable_o; + if (!Py_IS_TYPE(method, &PyMethodDescr_Type)) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(TRACING_CALL); + } + PyMethodDef *meth = method->d_method; + if (meth->ml_flags != (METH_FASTCALL|METH_KEYWORDS)) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(TRACING_CALL); + } + PyTypeObject *d_type = method->d_common.d_type; + PyObject *self = PyStackRef_AsPyObjectBorrow(arguments[0]); + assert(self != NULL); + if (!Py_IS_TYPE(self, d_type)) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(TRACING_CALL); + } + STAT_INC(CALL, hit); + int nargs = total_args - 1; + STACKREFS_TO_PYOBJECTS(arguments, total_args, args_o); + if (CONVERSION_FAILED(args_o)) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyStackRef tmp; + for (int _i = oparg; --_i >= 0;) { + tmp = args[_i]; + args[_i] = PyStackRef_NULL; + PyStackRef_CLOSE(tmp); + } + tmp = self_or_null; + self_or_null = PyStackRef_NULL; + stack_pointer[-1 - oparg] = self_or_null; + PyStackRef_XCLOSE(tmp); + tmp = callable; + callable = PyStackRef_NULL; + stack_pointer[-2 - oparg] = callable; + PyStackRef_CLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -2 - oparg; + assert(WITHIN_STACK_BOUNDS()); + TRACING_JUMP_TO_LABEL(error); + } + _PyFrame_SetStackPointer(frame, stack_pointer); + PyCFunctionFastWithKeywords cfunc = + _PyCFunctionFastWithKeywords_CAST(meth->ml_meth); + PyObject *res_o = cfunc(self, (args_o + 1), nargs, NULL); + stack_pointer = _PyFrame_GetStackPointer(frame); + STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); + assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyStackRef tmp; + for (int _i = oparg; --_i >= 0;) { + tmp = args[_i]; + args[_i] = PyStackRef_NULL; + PyStackRef_CLOSE(tmp); + } + tmp = self_or_null; + self_or_null = PyStackRef_NULL; + stack_pointer[-1 - oparg] = self_or_null; + PyStackRef_XCLOSE(tmp); + tmp = callable; + callable = PyStackRef_NULL; + stack_pointer[-2 - oparg] = callable; + PyStackRef_CLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -2 - oparg; + assert(WITHIN_STACK_BOUNDS()); + if (res_o == NULL) { + TRACING_JUMP_TO_LABEL(error); + } + res = PyStackRef_FromPyObjectSteal(res_o); + } + // _CHECK_PERIODIC_AT_END + { + stack_pointer[0] = res; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = check_periodics(tstate); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err != 0) { + TRACING_JUMP_TO_LABEL(error); + } + } + TRACING_DISPATCH(); + } + + TRACING_TARGET(CALL_METHOD_DESCRIPTOR_NOARGS) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = CALL_METHOD_DESCRIPTOR_NOARGS; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(CALL_METHOD_DESCRIPTOR_NOARGS); + opcode = CALL_METHOD_DESCRIPTOR_NOARGS; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); + _PyStackRef callable; + _PyStackRef self_or_null; + _PyStackRef *args; + _PyStackRef res; + /* Skip 1 cache entry */ + /* Skip 2 cache entries */ + // _CALL_METHOD_DESCRIPTOR_NOARGS + { + args = &stack_pointer[-oparg]; + self_or_null = stack_pointer[-1 - oparg]; + callable = stack_pointer[-2 - oparg]; + assert(oparg == 0 || oparg == 1); + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + int total_args = oparg; + if (!PyStackRef_IsNull(self_or_null)) { + args--; + total_args++; + } + if (total_args != 1) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(TRACING_CALL); + } + PyMethodDescrObject *method = (PyMethodDescrObject *)callable_o; + if (!Py_IS_TYPE(method, &PyMethodDescr_Type)) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(TRACING_CALL); + } + PyMethodDef *meth = method->d_method; + _PyStackRef self_stackref = args[0]; + PyObject *self = PyStackRef_AsPyObjectBorrow(self_stackref); + if (!Py_IS_TYPE(self, method->d_common.d_type)) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(TRACING_CALL); + } + if (meth->ml_flags != METH_NOARGS) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(TRACING_CALL); + } + if (_Py_ReachedRecursionLimit(tstate)) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(TRACING_CALL); + } + STAT_INC(CALL, hit); + PyCFunction cfunc = meth->ml_meth; + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *res_o = _PyCFunction_TrampolineCall(cfunc, self, NULL); + stack_pointer = _PyFrame_GetStackPointer(frame); + _Py_LeaveRecursiveCallTstate(tstate); + assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(self_stackref); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -2 - oparg; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(callable); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (res_o == NULL) { + TRACING_JUMP_TO_LABEL(error); + } + res = PyStackRef_FromPyObjectSteal(res_o); + } + // _CHECK_PERIODIC_AT_END + { + stack_pointer[0] = res; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = check_periodics(tstate); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err != 0) { + TRACING_JUMP_TO_LABEL(error); + } + } + TRACING_DISPATCH(); + } + + TRACING_TARGET(CALL_METHOD_DESCRIPTOR_O) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = CALL_METHOD_DESCRIPTOR_O; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(CALL_METHOD_DESCRIPTOR_O); + opcode = CALL_METHOD_DESCRIPTOR_O; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); + _PyStackRef callable; + _PyStackRef self_or_null; + _PyStackRef *args; + _PyStackRef res; + /* Skip 1 cache entry */ + /* Skip 2 cache entries */ + // _CALL_METHOD_DESCRIPTOR_O + { + args = &stack_pointer[-oparg]; + self_or_null = stack_pointer[-1 - oparg]; + callable = stack_pointer[-2 - oparg]; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + int total_args = oparg; + _PyStackRef *arguments = args; + if (!PyStackRef_IsNull(self_or_null)) { + arguments--; + total_args++; + } + PyMethodDescrObject *method = (PyMethodDescrObject *)callable_o; + if (total_args != 2) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(TRACING_CALL); + } + if (!Py_IS_TYPE(method, &PyMethodDescr_Type)) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(TRACING_CALL); + } + PyMethodDef *meth = method->d_method; + if (meth->ml_flags != METH_O) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(TRACING_CALL); + } + if (_Py_ReachedRecursionLimit(tstate)) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(TRACING_CALL); + } + _PyStackRef arg_stackref = arguments[1]; + _PyStackRef self_stackref = arguments[0]; + if (!Py_IS_TYPE(PyStackRef_AsPyObjectBorrow(self_stackref), + method->d_common.d_type)) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(TRACING_CALL); + } + STAT_INC(CALL, hit); + PyCFunction cfunc = meth->ml_meth; + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *res_o = _PyCFunction_TrampolineCall(cfunc, + PyStackRef_AsPyObjectBorrow(self_stackref), + PyStackRef_AsPyObjectBorrow(arg_stackref)); + stack_pointer = _PyFrame_GetStackPointer(frame); + _Py_LeaveRecursiveCallTstate(tstate); + assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyStackRef tmp; + for (int _i = oparg; --_i >= 0;) { + tmp = args[_i]; + args[_i] = PyStackRef_NULL; + PyStackRef_CLOSE(tmp); + } + tmp = self_or_null; + self_or_null = PyStackRef_NULL; + stack_pointer[-1 - oparg] = self_or_null; + PyStackRef_XCLOSE(tmp); + tmp = callable; + callable = PyStackRef_NULL; + stack_pointer[-2 - oparg] = callable; + PyStackRef_CLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -2 - oparg; + assert(WITHIN_STACK_BOUNDS()); + if (res_o == NULL) { + TRACING_JUMP_TO_LABEL(error); + } + res = PyStackRef_FromPyObjectSteal(res_o); + } + // _CHECK_PERIODIC_AT_END + { + stack_pointer[0] = res; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = check_periodics(tstate); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err != 0) { + TRACING_JUMP_TO_LABEL(error); + } + } + TRACING_DISPATCH(); + } + + TRACING_TARGET(CALL_NON_PY_GENERAL) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = CALL_NON_PY_GENERAL; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(CALL_NON_PY_GENERAL); + opcode = CALL_NON_PY_GENERAL; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + opcode = CALL_NON_PY_GENERAL; + static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); + _PyStackRef callable; + _PyStackRef self_or_null; + _PyStackRef *args; + _PyStackRef res; + /* Skip 1 cache entry */ + /* Skip 2 cache entries */ + // _CHECK_IS_NOT_PY_CALLABLE + { + callable = stack_pointer[-2 - oparg]; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + if (PyFunction_Check(callable_o)) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(TRACING_CALL); + } + if (Py_TYPE(callable_o) == &PyMethod_Type) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(TRACING_CALL); + } + } + // _CALL_NON_PY_GENERAL + { + args = &stack_pointer[-oparg]; + self_or_null = stack_pointer[-1 - oparg]; + #if TIER_ONE + assert(opcode != INSTRUMENTED_CALL); + #endif + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + int total_args = oparg; + _PyStackRef *arguments = args; + if (!PyStackRef_IsNull(self_or_null)) { + arguments--; + total_args++; + } + STACKREFS_TO_PYOBJECTS(arguments, total_args, args_o); + if (CONVERSION_FAILED(args_o)) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyStackRef tmp; + for (int _i = oparg; --_i >= 0;) { + tmp = args[_i]; + args[_i] = PyStackRef_NULL; + PyStackRef_CLOSE(tmp); + } + tmp = self_or_null; + self_or_null = PyStackRef_NULL; + stack_pointer[-1 - oparg] = self_or_null; + PyStackRef_XCLOSE(tmp); + tmp = callable; + callable = PyStackRef_NULL; + stack_pointer[-2 - oparg] = callable; + PyStackRef_CLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -2 - oparg; + assert(WITHIN_STACK_BOUNDS()); + TRACING_JUMP_TO_LABEL(error); + } + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *res_o = PyObject_Vectorcall( + callable_o, args_o, + total_args | PY_VECTORCALL_ARGUMENTS_OFFSET, + NULL); + stack_pointer = _PyFrame_GetStackPointer(frame); + STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); + assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyStackRef tmp; + for (int _i = oparg; --_i >= 0;) { + tmp = args[_i]; + args[_i] = PyStackRef_NULL; + PyStackRef_CLOSE(tmp); + } + tmp = self_or_null; + self_or_null = PyStackRef_NULL; + stack_pointer[-1 - oparg] = self_or_null; + PyStackRef_XCLOSE(tmp); + tmp = callable; + callable = PyStackRef_NULL; + stack_pointer[-2 - oparg] = callable; + PyStackRef_CLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -2 - oparg; + assert(WITHIN_STACK_BOUNDS()); + if (res_o == NULL) { + TRACING_JUMP_TO_LABEL(error); + } + res = PyStackRef_FromPyObjectSteal(res_o); + } + // _CHECK_PERIODIC_AT_END + { + stack_pointer[0] = res; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = check_periodics(tstate); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err != 0) { + TRACING_JUMP_TO_LABEL(error); + } + } + TRACING_DISPATCH(); + } + + TRACING_TARGET(CALL_PY_EXACT_ARGS) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = CALL_PY_EXACT_ARGS; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(CALL_PY_EXACT_ARGS); + opcode = CALL_PY_EXACT_ARGS; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); + _PyStackRef callable; + _PyStackRef self_or_null; + _PyStackRef *args; + _PyStackRef new_frame; + /* Skip 1 cache entry */ + // _CHECK_PEP_523 + { + if (tstate->interp->eval_frame) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(TRACING_CALL); + } + } + // _CHECK_FUNCTION_VERSION + { + callable = stack_pointer[-2 - oparg]; + uint32_t func_version = read_u32(&this_instr[2].cache); + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + if (!PyFunction_Check(callable_o)) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(TRACING_CALL); + } + PyFunctionObject *func = (PyFunctionObject *)callable_o; + if (func->func_version != func_version) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(TRACING_CALL); + } + } + // _CHECK_FUNCTION_EXACT_ARGS + { + self_or_null = stack_pointer[-1 - oparg]; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + assert(PyFunction_Check(callable_o)); + PyFunctionObject *func = (PyFunctionObject *)callable_o; + PyCodeObject *code = (PyCodeObject *)func->func_code; + if (code->co_argcount != oparg + (!PyStackRef_IsNull(self_or_null))) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(TRACING_CALL); + } + } + // _CHECK_STACK_SPACE + { + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + PyFunctionObject *func = (PyFunctionObject *)callable_o; + PyCodeObject *code = (PyCodeObject *)func->func_code; + if (!_PyThreadState_HasStackSpace(tstate, code->co_framesize)) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(TRACING_CALL); + } + } + // _CHECK_RECURSION_REMAINING + { + if (tstate->py_recursion_remaining <= 1) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(TRACING_CALL); + } + } + // _INIT_CALL_PY_EXACT_ARGS + { + args = &stack_pointer[-oparg]; + int has_self = !PyStackRef_IsNull(self_or_null); + STAT_INC(CALL, hit); + _PyInterpreterFrame *pushed_frame = _PyFrame_PushUnchecked(tstate, callable, oparg + has_self, frame); + _PyStackRef *first_non_self_local = pushed_frame->localsplus + has_self; + pushed_frame->localsplus[0] = self_or_null; + for (int i = 0; i < oparg; i++) { + first_non_self_local[i] = args[i]; + } + new_frame = PyStackRef_Wrap(pushed_frame); + } + // _SAVE_RETURN_OFFSET + { + #if TIER_ONE + frame->return_offset = (uint16_t)(next_instr - this_instr); + #endif + #if TIER_TWO + frame->return_offset = oparg; + #endif + } + // _PUSH_FRAME + { + assert(tstate->interp->eval_frame == NULL); + _PyInterpreterFrame *temp = PyStackRef_Unwrap(new_frame); + stack_pointer += -2 - oparg; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + assert(temp->previous == frame || temp->previous->previous == frame); + CALL_STAT_INC(inlined_py_calls); + frame = tstate->current_frame = temp; + tstate->py_recursion_remaining--; + LOAD_SP(); + LOAD_IP(0); + LLTRACE_RESUME_FRAME(); + } + TRACING_DISPATCH(); + } + + TRACING_TARGET(CALL_PY_GENERAL) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = CALL_PY_GENERAL; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(CALL_PY_GENERAL); + opcode = CALL_PY_GENERAL; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); + _PyStackRef callable; + _PyStackRef self_or_null; + _PyStackRef *args; + _PyStackRef new_frame; + /* Skip 1 cache entry */ + // _CHECK_PEP_523 + { + if (tstate->interp->eval_frame) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(TRACING_CALL); + } + } + // _CHECK_FUNCTION_VERSION + { + callable = stack_pointer[-2 - oparg]; + uint32_t func_version = read_u32(&this_instr[2].cache); + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + if (!PyFunction_Check(callable_o)) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(TRACING_CALL); + } + PyFunctionObject *func = (PyFunctionObject *)callable_o; + if (func->func_version != func_version) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(TRACING_CALL); + } + } + // _CHECK_RECURSION_REMAINING + { + if (tstate->py_recursion_remaining <= 1) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(TRACING_CALL); + } + } + // _PY_FRAME_GENERAL + { + args = &stack_pointer[-oparg]; + self_or_null = stack_pointer[-1 - oparg]; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + int total_args = oparg; + if (!PyStackRef_IsNull(self_or_null)) { + args--; + total_args++; + } + assert(Py_TYPE(callable_o) == &PyFunction_Type); + int code_flags = ((PyCodeObject*)PyFunction_GET_CODE(callable_o))->co_flags; + PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable_o)); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyInterpreterFrame *temp = _PyEvalFramePushAndInit( + tstate, callable, locals, + args, total_args, NULL, frame + ); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -2 - oparg; + assert(WITHIN_STACK_BOUNDS()); + if (temp == NULL) { + TRACING_JUMP_TO_LABEL(error); + } + new_frame = PyStackRef_Wrap(temp); + } + // _SAVE_RETURN_OFFSET + { + #if TIER_ONE + frame->return_offset = (uint16_t)(next_instr - this_instr); + #endif + #if TIER_TWO + frame->return_offset = oparg; + #endif + } + // _PUSH_FRAME + { + assert(tstate->interp->eval_frame == NULL); + _PyInterpreterFrame *temp = PyStackRef_Unwrap(new_frame); + _PyFrame_SetStackPointer(frame, stack_pointer); + assert(temp->previous == frame || temp->previous->previous == frame); + CALL_STAT_INC(inlined_py_calls); + frame = tstate->current_frame = temp; + tstate->py_recursion_remaining--; + LOAD_SP(); + LOAD_IP(0); + LLTRACE_RESUME_FRAME(); + } + TRACING_DISPATCH(); + } + + TRACING_TARGET(CALL_STR_1) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = CALL_STR_1; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(CALL_STR_1); + opcode = CALL_STR_1; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); + _PyStackRef null; + _PyStackRef callable; + _PyStackRef arg; + _PyStackRef res; + /* Skip 1 cache entry */ + /* Skip 2 cache entries */ + // _GUARD_NOS_NULL + { + null = stack_pointer[-2]; + if (!PyStackRef_IsNull(null)) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(TRACING_CALL); + } + } + // _GUARD_CALLABLE_STR_1 + { + callable = stack_pointer[-3]; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + if (callable_o != (PyObject *)&PyUnicode_Type) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(TRACING_CALL); + } + } + // _CALL_STR_1 + { + arg = stack_pointer[-1]; + PyObject *arg_o = PyStackRef_AsPyObjectBorrow(arg); + assert(oparg == 1); + STAT_INC(CALL, hit); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *res_o = PyObject_Str(arg_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + (void)callable; + (void)null; + stack_pointer += -3; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(arg); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (res_o == NULL) { + TRACING_JUMP_TO_LABEL(error); + } + res = PyStackRef_FromPyObjectSteal(res_o); + } + // _CHECK_PERIODIC_AT_END + { + stack_pointer[0] = res; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = check_periodics(tstate); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err != 0) { + TRACING_JUMP_TO_LABEL(error); + } + } + TRACING_DISPATCH(); + } + + TRACING_TARGET(CALL_TUPLE_1) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = CALL_TUPLE_1; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(CALL_TUPLE_1); + opcode = CALL_TUPLE_1; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); + _PyStackRef null; + _PyStackRef callable; + _PyStackRef arg; + _PyStackRef res; + /* Skip 1 cache entry */ + /* Skip 2 cache entries */ + // _GUARD_NOS_NULL + { + null = stack_pointer[-2]; + if (!PyStackRef_IsNull(null)) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(TRACING_CALL); + } + } + // _GUARD_CALLABLE_TUPLE_1 + { + callable = stack_pointer[-3]; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + if (callable_o != (PyObject *)&PyTuple_Type) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(TRACING_CALL); + } + } + // _CALL_TUPLE_1 + { + arg = stack_pointer[-1]; + PyObject *arg_o = PyStackRef_AsPyObjectBorrow(arg); + assert(oparg == 1); + STAT_INC(CALL, hit); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *res_o = PySequence_Tuple(arg_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + (void)callable; + (void)null; + stack_pointer += -3; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(arg); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (res_o == NULL) { + TRACING_JUMP_TO_LABEL(error); + } + res = PyStackRef_FromPyObjectSteal(res_o); + } + // _CHECK_PERIODIC_AT_END + { + stack_pointer[0] = res; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = check_periodics(tstate); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err != 0) { + TRACING_JUMP_TO_LABEL(error); + } + } + TRACING_DISPATCH(); + } + + TRACING_TARGET(CALL_TYPE_1) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = CALL_TYPE_1; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(CALL_TYPE_1); + opcode = CALL_TYPE_1; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); + _PyStackRef null; + _PyStackRef callable; + _PyStackRef arg; + _PyStackRef res; + /* Skip 1 cache entry */ + /* Skip 2 cache entries */ + // _GUARD_NOS_NULL + { + null = stack_pointer[-2]; + if (!PyStackRef_IsNull(null)) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(TRACING_CALL); + } + } + // _GUARD_CALLABLE_TYPE_1 + { + callable = stack_pointer[-3]; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + if (callable_o != (PyObject *)&PyType_Type) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(TRACING_CALL); + } + } + // _CALL_TYPE_1 + { + arg = stack_pointer[-1]; + PyObject *arg_o = PyStackRef_AsPyObjectBorrow(arg); + assert(oparg == 1); + (void)callable; + (void)null; + STAT_INC(CALL, hit); + res = PyStackRef_FromPyObjectNew(Py_TYPE(arg_o)); + stack_pointer[-3] = res; + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(arg); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + TRACING_DISPATCH(); + } + + TRACING_TARGET(CHECK_EG_MATCH) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = CHECK_EG_MATCH; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(CHECK_EG_MATCH); + opcode = CHECK_EG_MATCH; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + _PyStackRef exc_value_st; + _PyStackRef match_type_st; + _PyStackRef rest; + _PyStackRef match; + match_type_st = stack_pointer[-1]; + exc_value_st = stack_pointer[-2]; + PyObject *exc_value = PyStackRef_AsPyObjectBorrow(exc_value_st); + PyObject *match_type = PyStackRef_AsPyObjectBorrow(match_type_st); + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = _PyEval_CheckExceptStarTypeValid(tstate, match_type); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err < 0) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyStackRef tmp = match_type_st; + match_type_st = PyStackRef_NULL; + stack_pointer[-1] = match_type_st; + PyStackRef_CLOSE(tmp); + tmp = exc_value_st; + exc_value_st = PyStackRef_NULL; + stack_pointer[-2] = exc_value_st; + PyStackRef_CLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); + TRACING_JUMP_TO_LABEL(error); + } + PyObject *match_o = NULL; + PyObject *rest_o = NULL; + _PyFrame_SetStackPointer(frame, stack_pointer); + int res = _PyEval_ExceptionGroupMatch(frame, exc_value, match_type, + &match_o, &rest_o); + _PyStackRef tmp = match_type_st; + match_type_st = PyStackRef_NULL; + stack_pointer[-1] = match_type_st; + PyStackRef_CLOSE(tmp); + tmp = exc_value_st; + exc_value_st = PyStackRef_NULL; + stack_pointer[-2] = exc_value_st; + PyStackRef_CLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); + if (res < 0) { + TRACING_JUMP_TO_LABEL(error); + } + assert((match_o == NULL) == (rest_o == NULL)); + if (match_o == NULL) { + TRACING_JUMP_TO_LABEL(error); + } + if (!Py_IsNone(match_o)) { + _PyFrame_SetStackPointer(frame, stack_pointer); + PyErr_SetHandledException(match_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + rest = PyStackRef_FromPyObjectSteal(rest_o); + match = PyStackRef_FromPyObjectSteal(match_o); + stack_pointer[0] = rest; + stack_pointer[1] = match; + stack_pointer += 2; + assert(WITHIN_STACK_BOUNDS()); + TRACING_DISPATCH(); + } + + TRACING_TARGET(CHECK_EXC_MATCH) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = CHECK_EXC_MATCH; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(CHECK_EXC_MATCH); + opcode = CHECK_EXC_MATCH; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + _PyStackRef left; + _PyStackRef right; + _PyStackRef b; + right = stack_pointer[-1]; + left = stack_pointer[-2]; + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + assert(PyExceptionInstance_Check(left_o)); + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = _PyEval_CheckExceptTypeValid(tstate, right_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err < 0) { + TRACING_JUMP_TO_LABEL(error); + } + _PyFrame_SetStackPointer(frame, stack_pointer); + int res = PyErr_GivenExceptionMatches(left_o, right_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(right); + stack_pointer = _PyFrame_GetStackPointer(frame); + b = res ? PyStackRef_True : PyStackRef_False; + stack_pointer[0] = b; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + TRACING_DISPATCH(); + } + + TRACING_TARGET(CLEANUP_THROW) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = CLEANUP_THROW; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(CLEANUP_THROW); + opcode = CLEANUP_THROW; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + _PyStackRef sub_iter; + _PyStackRef last_sent_val; + _PyStackRef exc_value_st; + _PyStackRef none; + _PyStackRef value; + exc_value_st = stack_pointer[-1]; + last_sent_val = stack_pointer[-2]; + sub_iter = stack_pointer[-3]; + PyObject *exc_value = PyStackRef_AsPyObjectBorrow(exc_value_st); + #if !_Py_TAIL_CALL_INTERP + assert(throwflag); + #endif + assert(exc_value && PyExceptionInstance_Check(exc_value)); + _PyFrame_SetStackPointer(frame, stack_pointer); + int matches = PyErr_GivenExceptionMatches(exc_value, PyExc_StopIteration); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (matches) { + value = PyStackRef_FromPyObjectNew(((PyStopIterationObject *)exc_value)->value); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyStackRef tmp = sub_iter; + sub_iter = value; + stack_pointer[-3] = sub_iter; + PyStackRef_CLOSE(tmp); + tmp = exc_value_st; + exc_value_st = PyStackRef_NULL; + stack_pointer[-1] = exc_value_st; + PyStackRef_CLOSE(tmp); + tmp = last_sent_val; + last_sent_val = PyStackRef_NULL; + stack_pointer[-2] = last_sent_val; + PyStackRef_CLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -3; + assert(WITHIN_STACK_BOUNDS()); + none = PyStackRef_None; + } + else { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyErr_SetRaisedException(tstate, Py_NewRef(exc_value)); + monitor_reraise(tstate, frame, this_instr); + TRACING_JUMP_TO_LABEL(exception_unwind); + } + stack_pointer[0] = none; + stack_pointer[1] = value; + stack_pointer += 2; + assert(WITHIN_STACK_BOUNDS()); + TRACING_DISPATCH(); + } + + TRACING_TARGET(COMPARE_OP) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = COMPARE_OP; + (void)(opcode); + #endif + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(COMPARE_OP); + PREDICTED_TRACING_COMPARE_OP:; + _Py_CODEUNIT* const this_instr = next_instr - 2; + (void)this_instr; + opcode = COMPARE_OP; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + _PyStackRef left; + _PyStackRef right; + _PyStackRef res; + // _SPECIALIZE_COMPARE_OP + { + right = stack_pointer[-1]; + left = stack_pointer[-2]; + uint16_t counter = read_u16(&this_instr[1].cache); + (void)counter; + #if ENABLE_SPECIALIZATION_FT + if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { + next_instr = this_instr; + _PyFrame_SetStackPointer(frame, stack_pointer); + _Py_Specialize_CompareOp(left, right, next_instr, oparg); + stack_pointer = _PyFrame_GetStackPointer(frame); + DISPATCH_SAME_OPARG(); + } + OPCODE_DEFERRED_INC(COMPARE_OP); + ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); + #endif /* ENABLE_SPECIALIZATION_FT */ + } + // _COMPARE_OP + { + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + assert((oparg >> 5) <= Py_GE); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *res_o = PyObject_RichCompare(left_o, right_o, oparg >> 5); + _PyStackRef tmp = right; + right = PyStackRef_NULL; + stack_pointer[-1] = right; + PyStackRef_CLOSE(tmp); + tmp = left; + left = PyStackRef_NULL; + stack_pointer[-2] = left; + PyStackRef_CLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); + if (res_o == NULL) { + TRACING_JUMP_TO_LABEL(error); + } + if (oparg & 16) { + _PyFrame_SetStackPointer(frame, stack_pointer); + int res_bool = PyObject_IsTrue(res_o); + Py_DECREF(res_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (res_bool < 0) { + TRACING_JUMP_TO_LABEL(error); + } + res = res_bool ? PyStackRef_True : PyStackRef_False; + } + else { + res = PyStackRef_FromPyObjectSteal(res_o); + } + } + stack_pointer[0] = res; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + TRACING_DISPATCH(); + } + + TRACING_TARGET(COMPARE_OP_FLOAT) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = COMPARE_OP_FLOAT; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(COMPARE_OP_FLOAT); + opcode = COMPARE_OP_FLOAT; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + static_assert(INLINE_CACHE_ENTRIES_COMPARE_OP == 1, "incorrect cache size"); + _PyStackRef value; + _PyStackRef left; + _PyStackRef right; + _PyStackRef res; + // _GUARD_TOS_FLOAT + { + value = stack_pointer[-1]; + PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); + if (!PyFloat_CheckExact(value_o)) { + UPDATE_MISS_STATS(COMPARE_OP); + assert(_PyOpcode_Deopt[opcode] == (COMPARE_OP)); + JUMP_TO_PREDICTED(TRACING_COMPARE_OP); + } + } + // _GUARD_NOS_FLOAT + { + left = stack_pointer[-2]; + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + if (!PyFloat_CheckExact(left_o)) { + UPDATE_MISS_STATS(COMPARE_OP); + assert(_PyOpcode_Deopt[opcode] == (COMPARE_OP)); + JUMP_TO_PREDICTED(TRACING_COMPARE_OP); + } + } + /* Skip 1 cache entry */ + // _COMPARE_OP_FLOAT + { + right = value; + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + STAT_INC(COMPARE_OP, hit); + double dleft = PyFloat_AS_DOUBLE(left_o); + double dright = PyFloat_AS_DOUBLE(right_o); + int sign_ish = COMPARISON_BIT(dleft, dright); + PyStackRef_CLOSE_SPECIALIZED(left, _PyFloat_ExactDealloc); + PyStackRef_CLOSE_SPECIALIZED(right, _PyFloat_ExactDealloc); + res = (sign_ish & oparg) ? PyStackRef_True : PyStackRef_False; + } + stack_pointer[-2] = res; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + TRACING_DISPATCH(); + } + + TRACING_TARGET(COMPARE_OP_INT) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = COMPARE_OP_INT; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(COMPARE_OP_INT); + opcode = COMPARE_OP_INT; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + static_assert(INLINE_CACHE_ENTRIES_COMPARE_OP == 1, "incorrect cache size"); + _PyStackRef value; + _PyStackRef left; + _PyStackRef right; + _PyStackRef res; + // _GUARD_TOS_INT + { + value = stack_pointer[-1]; + PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); + if (!_PyLong_CheckExactAndCompact(value_o)) { + UPDATE_MISS_STATS(COMPARE_OP); + assert(_PyOpcode_Deopt[opcode] == (COMPARE_OP)); + JUMP_TO_PREDICTED(TRACING_COMPARE_OP); + } + } + // _GUARD_NOS_INT + { + left = stack_pointer[-2]; + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + if (!_PyLong_CheckExactAndCompact(left_o)) { + UPDATE_MISS_STATS(COMPARE_OP); + assert(_PyOpcode_Deopt[opcode] == (COMPARE_OP)); + JUMP_TO_PREDICTED(TRACING_COMPARE_OP); + } + } + /* Skip 1 cache entry */ + // _COMPARE_OP_INT + { + right = value; + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + assert(_PyLong_IsCompact((PyLongObject *)left_o)); + assert(_PyLong_IsCompact((PyLongObject *)right_o)); + STAT_INC(COMPARE_OP, hit); + assert(_PyLong_DigitCount((PyLongObject *)left_o) <= 1 && + _PyLong_DigitCount((PyLongObject *)right_o) <= 1); + Py_ssize_t ileft = _PyLong_CompactValue((PyLongObject *)left_o); + Py_ssize_t iright = _PyLong_CompactValue((PyLongObject *)right_o); + int sign_ish = COMPARISON_BIT(ileft, iright); + PyStackRef_CLOSE_SPECIALIZED(left, _PyLong_ExactDealloc); + PyStackRef_CLOSE_SPECIALIZED(right, _PyLong_ExactDealloc); + res = (sign_ish & oparg) ? PyStackRef_True : PyStackRef_False; + } + stack_pointer[-2] = res; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + TRACING_DISPATCH(); + } + + TRACING_TARGET(COMPARE_OP_STR) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = COMPARE_OP_STR; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(COMPARE_OP_STR); + opcode = COMPARE_OP_STR; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + static_assert(INLINE_CACHE_ENTRIES_COMPARE_OP == 1, "incorrect cache size"); + _PyStackRef value; + _PyStackRef nos; + _PyStackRef left; + _PyStackRef right; + _PyStackRef res; + // _GUARD_TOS_UNICODE + { + value = stack_pointer[-1]; + PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); + if (!PyUnicode_CheckExact(value_o)) { + UPDATE_MISS_STATS(COMPARE_OP); + assert(_PyOpcode_Deopt[opcode] == (COMPARE_OP)); + JUMP_TO_PREDICTED(TRACING_COMPARE_OP); + } + } + // _GUARD_NOS_UNICODE + { + nos = stack_pointer[-2]; + PyObject *o = PyStackRef_AsPyObjectBorrow(nos); + if (!PyUnicode_CheckExact(o)) { + UPDATE_MISS_STATS(COMPARE_OP); + assert(_PyOpcode_Deopt[opcode] == (COMPARE_OP)); + JUMP_TO_PREDICTED(TRACING_COMPARE_OP); + } + } + /* Skip 1 cache entry */ + // _COMPARE_OP_STR + { + right = value; + left = nos; + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + STAT_INC(COMPARE_OP, hit); + int eq = _PyUnicode_Equal(left_o, right_o); + assert((oparg >> 5) == Py_EQ || (oparg >> 5) == Py_NE); + PyStackRef_CLOSE_SPECIALIZED(left, _PyUnicode_ExactDealloc); + PyStackRef_CLOSE_SPECIALIZED(right, _PyUnicode_ExactDealloc); + assert(eq == 0 || eq == 1); + assert((oparg & 0xf) == COMPARISON_NOT_EQUALS || (oparg & 0xf) == COMPARISON_EQUALS); + assert(COMPARISON_NOT_EQUALS + 1 == COMPARISON_EQUALS); + res = ((COMPARISON_NOT_EQUALS + eq) & oparg) ? PyStackRef_True : PyStackRef_False; + } + stack_pointer[-2] = res; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + TRACING_DISPATCH(); + } + + TRACING_TARGET(CONTAINS_OP) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = CONTAINS_OP; + (void)(opcode); + #endif + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(CONTAINS_OP); + PREDICTED_TRACING_CONTAINS_OP:; + _Py_CODEUNIT* const this_instr = next_instr - 2; + (void)this_instr; + opcode = CONTAINS_OP; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + _PyStackRef right; + _PyStackRef left; + _PyStackRef b; + // _SPECIALIZE_CONTAINS_OP + { + right = stack_pointer[-1]; + uint16_t counter = read_u16(&this_instr[1].cache); + (void)counter; + #if ENABLE_SPECIALIZATION_FT + if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { + next_instr = this_instr; + _PyFrame_SetStackPointer(frame, stack_pointer); + _Py_Specialize_ContainsOp(right, next_instr); + stack_pointer = _PyFrame_GetStackPointer(frame); + DISPATCH_SAME_OPARG(); + } + OPCODE_DEFERRED_INC(CONTAINS_OP); + ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); + #endif /* ENABLE_SPECIALIZATION_FT */ + } + // _CONTAINS_OP + { + left = stack_pointer[-2]; + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + _PyFrame_SetStackPointer(frame, stack_pointer); + int res = PySequence_Contains(right_o, left_o); + _PyStackRef tmp = right; + right = PyStackRef_NULL; + stack_pointer[-1] = right; + PyStackRef_CLOSE(tmp); + tmp = left; + left = PyStackRef_NULL; + stack_pointer[-2] = left; + PyStackRef_CLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); + if (res < 0) { + TRACING_JUMP_TO_LABEL(error); + } + b = (res ^ oparg) ? PyStackRef_True : PyStackRef_False; + } + stack_pointer[0] = b; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + TRACING_DISPATCH(); + } + + TRACING_TARGET(CONTAINS_OP_DICT) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = CONTAINS_OP_DICT; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(CONTAINS_OP_DICT); + opcode = CONTAINS_OP_DICT; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + static_assert(INLINE_CACHE_ENTRIES_CONTAINS_OP == 1, "incorrect cache size"); + _PyStackRef tos; + _PyStackRef left; + _PyStackRef right; + _PyStackRef b; + // _GUARD_TOS_DICT + { + tos = stack_pointer[-1]; + PyObject *o = PyStackRef_AsPyObjectBorrow(tos); + if (!PyDict_CheckExact(o)) { + UPDATE_MISS_STATS(CONTAINS_OP); + assert(_PyOpcode_Deopt[opcode] == (CONTAINS_OP)); + JUMP_TO_PREDICTED(TRACING_CONTAINS_OP); + } + } + /* Skip 1 cache entry */ + // _CONTAINS_OP_DICT + { + right = tos; + left = stack_pointer[-2]; + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + assert(PyDict_CheckExact(right_o)); + STAT_INC(CONTAINS_OP, hit); + _PyFrame_SetStackPointer(frame, stack_pointer); + int res = PyDict_Contains(right_o, left_o); + _PyStackRef tmp = right; + right = PyStackRef_NULL; + stack_pointer[-1] = right; + PyStackRef_CLOSE(tmp); + tmp = left; + left = PyStackRef_NULL; + stack_pointer[-2] = left; + PyStackRef_CLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); + if (res < 0) { + TRACING_JUMP_TO_LABEL(error); + } + b = (res ^ oparg) ? PyStackRef_True : PyStackRef_False; + } + stack_pointer[0] = b; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + TRACING_DISPATCH(); + } + + TRACING_TARGET(CONTAINS_OP_SET) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = CONTAINS_OP_SET; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(CONTAINS_OP_SET); + opcode = CONTAINS_OP_SET; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + static_assert(INLINE_CACHE_ENTRIES_CONTAINS_OP == 1, "incorrect cache size"); + _PyStackRef tos; + _PyStackRef left; + _PyStackRef right; + _PyStackRef b; + // _GUARD_TOS_ANY_SET + { + tos = stack_pointer[-1]; + PyObject *o = PyStackRef_AsPyObjectBorrow(tos); + if (!PyAnySet_CheckExact(o)) { + UPDATE_MISS_STATS(CONTAINS_OP); + assert(_PyOpcode_Deopt[opcode] == (CONTAINS_OP)); + JUMP_TO_PREDICTED(TRACING_CONTAINS_OP); + } + } + /* Skip 1 cache entry */ + // _CONTAINS_OP_SET + { + right = tos; + left = stack_pointer[-2]; + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + assert(PyAnySet_CheckExact(right_o)); + STAT_INC(CONTAINS_OP, hit); + _PyFrame_SetStackPointer(frame, stack_pointer); + int res = _PySet_Contains((PySetObject *)right_o, left_o); + _PyStackRef tmp = right; + right = PyStackRef_NULL; + stack_pointer[-1] = right; + PyStackRef_CLOSE(tmp); + tmp = left; + left = PyStackRef_NULL; + stack_pointer[-2] = left; + PyStackRef_CLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); + if (res < 0) { + TRACING_JUMP_TO_LABEL(error); + } + b = (res ^ oparg) ? PyStackRef_True : PyStackRef_False; + } + stack_pointer[0] = b; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + TRACING_DISPATCH(); + } + + TRACING_TARGET(CONVERT_VALUE) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = CONVERT_VALUE; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(CONVERT_VALUE); + opcode = CONVERT_VALUE; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + _PyStackRef value; + _PyStackRef result; + value = stack_pointer[-1]; + conversion_func conv_fn; + assert(oparg >= FVC_STR && oparg <= FVC_ASCII); + conv_fn = _PyEval_ConversionFuncs[oparg]; + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *result_o = conv_fn(PyStackRef_AsPyObjectBorrow(value)); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(value); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (result_o == NULL) { + TRACING_JUMP_TO_LABEL(error); + } + result = PyStackRef_FromPyObjectSteal(result_o); + stack_pointer[0] = result; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + TRACING_DISPATCH(); + } + + TRACING_TARGET(COPY) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = COPY; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(COPY); + opcode = COPY; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + _PyStackRef bottom; + _PyStackRef top; + bottom = stack_pointer[-1 - (oparg-1)]; + top = PyStackRef_DUP(bottom); + stack_pointer[0] = top; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + TRACING_DISPATCH(); + } + + TRACING_TARGET(COPY_FREE_VARS) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = COPY_FREE_VARS; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(COPY_FREE_VARS); + opcode = COPY_FREE_VARS; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + PyCodeObject *co = _PyFrame_GetCode(frame); + assert(PyStackRef_FunctionCheck(frame->f_funcobj)); + PyFunctionObject *func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + PyObject *closure = func->func_closure; + assert(oparg == co->co_nfreevars); + int offset = co->co_nlocalsplus - oparg; + for (int i = 0; i < oparg; ++i) { + PyObject *o = PyTuple_GET_ITEM(closure, i); + frame->localsplus[offset + i] = PyStackRef_FromPyObjectNew(o); + } + TRACING_DISPATCH(); + } + + TRACING_TARGET(DELETE_ATTR) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = DELETE_ATTR; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(DELETE_ATTR); + opcode = DELETE_ATTR; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + _PyStackRef owner; + owner = stack_pointer[-1]; + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = PyObject_DelAttr(PyStackRef_AsPyObjectBorrow(owner), name); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(owner); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err) { + TRACING_JUMP_TO_LABEL(error); + } + TRACING_DISPATCH(); + } + + TRACING_TARGET(DELETE_DEREF) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = DELETE_DEREF; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(DELETE_DEREF); + opcode = DELETE_DEREF; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + PyObject *cell = PyStackRef_AsPyObjectBorrow(GETLOCAL(oparg)); + PyObject *oldobj = PyCell_SwapTakeRef((PyCellObject *)cell, NULL); + if (oldobj == NULL) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyEval_FormatExcUnbound(tstate, _PyFrame_GetCode(frame), oparg); + stack_pointer = _PyFrame_GetStackPointer(frame); + TRACING_JUMP_TO_LABEL(error); + } + _PyFrame_SetStackPointer(frame, stack_pointer); + Py_DECREF(oldobj); + stack_pointer = _PyFrame_GetStackPointer(frame); + TRACING_DISPATCH(); + } + + TRACING_TARGET(DELETE_FAST) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = DELETE_FAST; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(DELETE_FAST); + opcode = DELETE_FAST; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + _PyStackRef v = GETLOCAL(oparg); + if (PyStackRef_IsNull(v)) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyEval_FormatExcCheckArg(tstate, PyExc_UnboundLocalError, + UNBOUNDLOCAL_ERROR_MSG, + PyTuple_GetItem(_PyFrame_GetCode(frame)->co_localsplusnames, oparg) + ); + stack_pointer = _PyFrame_GetStackPointer(frame); + TRACING_JUMP_TO_LABEL(error); + } + _PyStackRef tmp = GETLOCAL(oparg); + GETLOCAL(oparg) = PyStackRef_NULL; + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_XCLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); + TRACING_DISPATCH(); + } + + TRACING_TARGET(DELETE_GLOBAL) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = DELETE_GLOBAL; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(DELETE_GLOBAL); + opcode = DELETE_GLOBAL; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = PyDict_Pop(GLOBALS(), name, NULL); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err < 0) { + TRACING_JUMP_TO_LABEL(error); + } + if (err == 0) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyEval_FormatExcCheckArg(tstate, PyExc_NameError, + NAME_ERROR_MSG, name); + stack_pointer = _PyFrame_GetStackPointer(frame); + TRACING_JUMP_TO_LABEL(error); + } + TRACING_DISPATCH(); + } + + TRACING_TARGET(DELETE_NAME) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = DELETE_NAME; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(DELETE_NAME); + opcode = DELETE_NAME; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); + PyObject *ns = LOCALS(); + int err; + if (ns == NULL) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyErr_Format(tstate, PyExc_SystemError, + "no locals when deleting %R", name); + stack_pointer = _PyFrame_GetStackPointer(frame); + TRACING_JUMP_TO_LABEL(error); + } + _PyFrame_SetStackPointer(frame, stack_pointer); + err = PyObject_DelItem(ns, name); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err != 0) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyEval_FormatExcCheckArg(tstate, PyExc_NameError, + NAME_ERROR_MSG, + name); + stack_pointer = _PyFrame_GetStackPointer(frame); + TRACING_JUMP_TO_LABEL(error); + } + TRACING_DISPATCH(); + } + + TRACING_TARGET(DELETE_SUBSCR) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = DELETE_SUBSCR; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(DELETE_SUBSCR); + opcode = DELETE_SUBSCR; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + _PyStackRef container; + _PyStackRef sub; + sub = stack_pointer[-1]; + container = stack_pointer[-2]; + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = PyObject_DelItem(PyStackRef_AsPyObjectBorrow(container), + PyStackRef_AsPyObjectBorrow(sub)); + _PyStackRef tmp = sub; + sub = PyStackRef_NULL; + stack_pointer[-1] = sub; + PyStackRef_CLOSE(tmp); + tmp = container; + container = PyStackRef_NULL; + stack_pointer[-2] = container; + PyStackRef_CLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); + if (err) { + TRACING_JUMP_TO_LABEL(error); + } + TRACING_DISPATCH(); + } + + TRACING_TARGET(DICT_MERGE) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = DICT_MERGE; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(DICT_MERGE); + opcode = DICT_MERGE; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + _PyStackRef callable; + _PyStackRef dict; + _PyStackRef update; + update = stack_pointer[-1]; + dict = stack_pointer[-2 - (oparg - 1)]; + callable = stack_pointer[-5 - (oparg - 1)]; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + PyObject *dict_o = PyStackRef_AsPyObjectBorrow(dict); + PyObject *update_o = PyStackRef_AsPyObjectBorrow(update); + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = _PyDict_MergeEx(dict_o, update_o, 2); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err < 0) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyEval_FormatKwargsError(tstate, callable_o, update_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(update); + stack_pointer = _PyFrame_GetStackPointer(frame); + TRACING_JUMP_TO_LABEL(error); + } + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(update); + stack_pointer = _PyFrame_GetStackPointer(frame); + TRACING_DISPATCH(); + } + + TRACING_TARGET(DICT_UPDATE) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = DICT_UPDATE; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(DICT_UPDATE); + opcode = DICT_UPDATE; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + _PyStackRef dict; + _PyStackRef update; + update = stack_pointer[-1]; + dict = stack_pointer[-2 - (oparg - 1)]; + PyObject *dict_o = PyStackRef_AsPyObjectBorrow(dict); + PyObject *update_o = PyStackRef_AsPyObjectBorrow(update); + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = PyDict_Update(dict_o, update_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err < 0) { + _PyFrame_SetStackPointer(frame, stack_pointer); + int matches = _PyErr_ExceptionMatches(tstate, PyExc_AttributeError); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (matches) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyErr_Format(tstate, PyExc_TypeError, + "'%.200s' object is not a mapping", + Py_TYPE(update_o)->tp_name); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(update); + stack_pointer = _PyFrame_GetStackPointer(frame); + TRACING_JUMP_TO_LABEL(error); + } + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(update); + stack_pointer = _PyFrame_GetStackPointer(frame); + TRACING_DISPATCH(); + } + + TRACING_TARGET(END_ASYNC_FOR) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = END_ASYNC_FOR; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(END_ASYNC_FOR); + opcode = END_ASYNC_FOR; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + _PyStackRef awaitable_st; + _PyStackRef exc_st; + exc_st = stack_pointer[-1]; + awaitable_st = stack_pointer[-2]; + JUMPBY(0); + (void)oparg; + PyObject *exc = PyStackRef_AsPyObjectBorrow(exc_st); + assert(exc && PyExceptionInstance_Check(exc)); + _PyFrame_SetStackPointer(frame, stack_pointer); + int matches = PyErr_GivenExceptionMatches(exc, PyExc_StopAsyncIteration); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (matches) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyStackRef tmp = exc_st; + exc_st = PyStackRef_NULL; + stack_pointer[-1] = exc_st; + PyStackRef_CLOSE(tmp); + tmp = awaitable_st; + awaitable_st = PyStackRef_NULL; + stack_pointer[-2] = awaitable_st; + PyStackRef_CLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); + } + else { + Py_INCREF(exc); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyErr_SetRaisedException(tstate, exc); + monitor_reraise(tstate, frame, this_instr); + TRACING_JUMP_TO_LABEL(exception_unwind); + } + TRACING_DISPATCH(); + } + + TRACING_TARGET(END_FOR) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = END_FOR; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + next_instr += 1; + INSTRUCTION_STATS(END_FOR); + opcode = END_FOR; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + _PyStackRef value; + value = stack_pointer[-1]; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(value); + stack_pointer = _PyFrame_GetStackPointer(frame); + TRACING_DISPATCH(); + } + + TRACING_TARGET(END_SEND) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = END_SEND; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(END_SEND); + opcode = END_SEND; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + _PyStackRef receiver; + _PyStackRef value; + _PyStackRef val; + value = stack_pointer[-1]; + receiver = stack_pointer[-2]; + val = value; + stack_pointer[-2] = val; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(receiver); + stack_pointer = _PyFrame_GetStackPointer(frame); + TRACING_DISPATCH(); + } + + TRACING_TARGET(ENTER_EXECUTOR) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = ENTER_EXECUTOR; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(ENTER_EXECUTOR); + opcode = ENTER_EXECUTOR; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + opcode = ENTER_EXECUTOR; + #ifdef _Py_TIER2 + PyCodeObject *code = _PyFrame_GetCode(frame); + _PyExecutorObject *executor = code->co_executors->executors[oparg & 255]; + assert(executor->vm_data.index == INSTR_OFFSET() - 1); + assert(executor->vm_data.code == code); + assert(executor->vm_data.valid); + assert(tstate->current_executor == NULL); + if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { + opcode = executor->vm_data.opcode; + oparg = (oparg & ~255) | executor->vm_data.oparg; + next_instr = this_instr; + if (_PyOpcode_Caches[_PyOpcode_Deopt[opcode]]) { + PAUSE_ADAPTIVE_COUNTER(this_instr[1].counter); + } + DISPATCH_GOTO(); + } + assert(executor != tstate->interp->cold_executor); + tstate->jit_exit = NULL; + #if TRACING_JIT + RECORD_TRACE_NO_DISPATCH(); + #endif + TIER1_TO_TIER2(executor); + #else + Py_FatalError("ENTER_EXECUTOR is not supported in this build"); + #endif /* _Py_TIER2 */ + TRACING_DISPATCH(); + } + + TRACING_TARGET(EXIT_INIT_CHECK) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = EXIT_INIT_CHECK; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(EXIT_INIT_CHECK); + opcode = EXIT_INIT_CHECK; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + _PyStackRef should_be_none; + should_be_none = stack_pointer[-1]; + if (!PyStackRef_IsNone(should_be_none)) { + _PyFrame_SetStackPointer(frame, stack_pointer); + PyErr_Format(PyExc_TypeError, + "__init__() should return None, not '%.200s'", + Py_TYPE(PyStackRef_AsPyObjectBorrow(should_be_none))->tp_name); + stack_pointer = _PyFrame_GetStackPointer(frame); + TRACING_JUMP_TO_LABEL(error); + } + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + TRACING_DISPATCH(); + } + + TRACING_TARGET(EXTENDED_ARG) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = EXTENDED_ARG; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(EXTENDED_ARG); + opcode = EXTENDED_ARG; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + opcode = EXTENDED_ARG; + assert(oparg); + opcode = next_instr->op.code; + oparg = oparg << 8 | next_instr->op.arg; + PRE_DISPATCH_GOTO(); + DISPATCH_GOTO(); + } + + TRACING_TARGET(FORMAT_SIMPLE) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = FORMAT_SIMPLE; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(FORMAT_SIMPLE); + opcode = FORMAT_SIMPLE; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + _PyStackRef value; + _PyStackRef res; + value = stack_pointer[-1]; + PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); + if (!PyUnicode_CheckExact(value_o)) { + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *res_o = PyObject_Format(value_o, NULL); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(value); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (res_o == NULL) { + TRACING_JUMP_TO_LABEL(error); + } + res = PyStackRef_FromPyObjectSteal(res_o); + } + else { + res = value; + stack_pointer += -1; + } + stack_pointer[0] = res; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + TRACING_DISPATCH(); + } + + TRACING_TARGET(FORMAT_WITH_SPEC) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = FORMAT_WITH_SPEC; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(FORMAT_WITH_SPEC); + opcode = FORMAT_WITH_SPEC; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + _PyStackRef value; + _PyStackRef fmt_spec; + _PyStackRef res; + fmt_spec = stack_pointer[-1]; + value = stack_pointer[-2]; + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *res_o = PyObject_Format(PyStackRef_AsPyObjectBorrow(value), PyStackRef_AsPyObjectBorrow(fmt_spec)); + _PyStackRef tmp = fmt_spec; + fmt_spec = PyStackRef_NULL; + stack_pointer[-1] = fmt_spec; + PyStackRef_CLOSE(tmp); + tmp = value; + value = PyStackRef_NULL; + stack_pointer[-2] = value; + PyStackRef_CLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); + if (res_o == NULL) { + TRACING_JUMP_TO_LABEL(error); + } + res = PyStackRef_FromPyObjectSteal(res_o); + stack_pointer[0] = res; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + TRACING_DISPATCH(); + } + + TRACING_TARGET(FOR_ITER) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = FOR_ITER; + (void)(opcode); + #endif + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(FOR_ITER); + PREDICTED_TRACING_FOR_ITER:; + _Py_CODEUNIT* const this_instr = next_instr - 2; + (void)this_instr; + opcode = FOR_ITER; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + _PyStackRef iter; + _PyStackRef null_or_index; + _PyStackRef next; + // _SPECIALIZE_FOR_ITER + { + null_or_index = stack_pointer[-1]; + iter = stack_pointer[-2]; + uint16_t counter = read_u16(&this_instr[1].cache); + (void)counter; + #if ENABLE_SPECIALIZATION_FT + if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { + next_instr = this_instr; + _PyFrame_SetStackPointer(frame, stack_pointer); + _Py_Specialize_ForIter(iter, null_or_index, next_instr, oparg); + stack_pointer = _PyFrame_GetStackPointer(frame); + DISPATCH_SAME_OPARG(); + } + OPCODE_DEFERRED_INC(FOR_ITER); + ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); + #endif /* ENABLE_SPECIALIZATION_FT */ + } + // _FOR_ITER + { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyStackRef item = _PyForIter_VirtualIteratorNext(tstate, frame, iter, &null_or_index); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (!PyStackRef_IsValid(item)) { + if (PyStackRef_IsError(item)) { + TRACING_JUMP_TO_LABEL(error); + } + JUMPBY(oparg + 1); + RECORD_JUMP_TAKEN(); + stack_pointer[-1] = null_or_index; + TRACING_DISPATCH(); + } + next = item; + } + stack_pointer[-1] = null_or_index; + stack_pointer[0] = next; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + TRACING_DISPATCH(); + } + + TRACING_TARGET(FOR_ITER_GEN) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = FOR_ITER_GEN; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(FOR_ITER_GEN); + opcode = FOR_ITER_GEN; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + static_assert(INLINE_CACHE_ENTRIES_FOR_ITER == 1, "incorrect cache size"); + _PyStackRef iter; + _PyStackRef gen_frame; + _PyStackRef new_frame; + /* Skip 1 cache entry */ + // _CHECK_PEP_523 + { + if (tstate->interp->eval_frame) { + UPDATE_MISS_STATS(FOR_ITER); + assert(_PyOpcode_Deopt[opcode] == (FOR_ITER)); + JUMP_TO_PREDICTED(TRACING_FOR_ITER); + } + } + // _FOR_ITER_GEN_FRAME + { + iter = stack_pointer[-2]; + PyGenObject *gen = (PyGenObject *)PyStackRef_AsPyObjectBorrow(iter); + if (Py_TYPE(gen) != &PyGen_Type) { + UPDATE_MISS_STATS(FOR_ITER); + assert(_PyOpcode_Deopt[opcode] == (FOR_ITER)); + JUMP_TO_PREDICTED(TRACING_FOR_ITER); + } + #ifdef Py_GIL_DISABLED + if (!_PyObject_IsUniquelyReferenced((PyObject *)gen)) { + UPDATE_MISS_STATS(FOR_ITER); + assert(_PyOpcode_Deopt[opcode] == (FOR_ITER)); + JUMP_TO_PREDICTED(TRACING_FOR_ITER); + } + #endif + if (gen->gi_frame_state >= FRAME_EXECUTING) { + UPDATE_MISS_STATS(FOR_ITER); + assert(_PyOpcode_Deopt[opcode] == (FOR_ITER)); + JUMP_TO_PREDICTED(TRACING_FOR_ITER); + } + STAT_INC(FOR_ITER, hit); + _PyInterpreterFrame *pushed_frame = &gen->gi_iframe; + _PyFrame_StackPush(pushed_frame, PyStackRef_None); + gen->gi_frame_state = FRAME_EXECUTING; + gen->gi_exc_state.previous_item = tstate->exc_info; + tstate->exc_info = &gen->gi_exc_state; + pushed_frame->previous = frame; + frame->return_offset = (uint16_t)( 2u + oparg); + gen_frame = PyStackRef_Wrap(pushed_frame); + } + // _PUSH_FRAME + { + new_frame = gen_frame; + assert(tstate->interp->eval_frame == NULL); + _PyInterpreterFrame *temp = PyStackRef_Unwrap(new_frame); + _PyFrame_SetStackPointer(frame, stack_pointer); + assert(temp->previous == frame || temp->previous->previous == frame); + CALL_STAT_INC(inlined_py_calls); + frame = tstate->current_frame = temp; + tstate->py_recursion_remaining--; + LOAD_SP(); + LOAD_IP(0); + LLTRACE_RESUME_FRAME(); + } + TRACING_DISPATCH(); + } + + TRACING_TARGET(FOR_ITER_LIST) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = FOR_ITER_LIST; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(FOR_ITER_LIST); + opcode = FOR_ITER_LIST; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + static_assert(INLINE_CACHE_ENTRIES_FOR_ITER == 1, "incorrect cache size"); + _PyStackRef iter; + _PyStackRef null_or_index; + _PyStackRef next; + /* Skip 1 cache entry */ + // _ITER_CHECK_LIST + { + null_or_index = stack_pointer[-1]; + iter = stack_pointer[-2]; + PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter); + if (Py_TYPE(iter_o) != &PyList_Type) { + UPDATE_MISS_STATS(FOR_ITER); + assert(_PyOpcode_Deopt[opcode] == (FOR_ITER)); + JUMP_TO_PREDICTED(TRACING_FOR_ITER); + } + assert(PyStackRef_IsTaggedInt(null_or_index)); + #ifdef Py_GIL_DISABLED + if (!_Py_IsOwnedByCurrentThread(iter_o) && !_PyObject_GC_IS_SHARED(iter_o)) { + UPDATE_MISS_STATS(FOR_ITER); + assert(_PyOpcode_Deopt[opcode] == (FOR_ITER)); + JUMP_TO_PREDICTED(TRACING_FOR_ITER); + } + #endif + } + // _ITER_JUMP_LIST + { + #ifdef Py_GIL_DISABLED + + #else + PyObject *list_o = PyStackRef_AsPyObjectBorrow(iter); + assert(Py_TYPE(list_o) == &PyList_Type); + STAT_INC(FOR_ITER, hit); + if ((size_t)PyStackRef_UntagInt(null_or_index) >= (size_t)PyList_GET_SIZE(list_o)) { + null_or_index = PyStackRef_TagInt(-1); + JUMPBY(oparg + 1); + RECORD_JUMP_TAKEN(); + stack_pointer[-1] = null_or_index; + TRACING_DISPATCH(); + } + #endif + } + // _ITER_NEXT_LIST + { + PyObject *list_o = PyStackRef_AsPyObjectBorrow(iter); + assert(PyList_CheckExact(list_o)); + #ifdef Py_GIL_DISABLED + assert(_Py_IsOwnedByCurrentThread(list_o) || + _PyObject_GC_IS_SHARED(list_o)); + STAT_INC(FOR_ITER, hit); + _PyFrame_SetStackPointer(frame, stack_pointer); + int result = _PyList_GetItemRefNoLock((PyListObject *)list_o, PyStackRef_UntagInt(null_or_index), &next); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (result < 0) { + UPDATE_MISS_STATS(FOR_ITER); + assert(_PyOpcode_Deopt[opcode] == (FOR_ITER)); + JUMP_TO_PREDICTED(TRACING_FOR_ITER); + } + if (result == 0) { + null_or_index = PyStackRef_TagInt(-1); + JUMPBY(oparg + 1); + stack_pointer[-1] = null_or_index; + TRACING_DISPATCH(); + } + #else + next = PyStackRef_FromPyObjectNew(PyList_GET_ITEM(list_o, PyStackRef_UntagInt(null_or_index))); + #endif + null_or_index = PyStackRef_IncrementTaggedIntNoOverflow(null_or_index); + } + stack_pointer[-1] = null_or_index; + stack_pointer[0] = next; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + TRACING_DISPATCH(); + } + + TRACING_TARGET(FOR_ITER_RANGE) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = FOR_ITER_RANGE; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(FOR_ITER_RANGE); + opcode = FOR_ITER_RANGE; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + static_assert(INLINE_CACHE_ENTRIES_FOR_ITER == 1, "incorrect cache size"); + _PyStackRef iter; + _PyStackRef next; + /* Skip 1 cache entry */ + // _ITER_CHECK_RANGE + { + iter = stack_pointer[-2]; + _PyRangeIterObject *r = (_PyRangeIterObject *)PyStackRef_AsPyObjectBorrow(iter); + if (Py_TYPE(r) != &PyRangeIter_Type) { + UPDATE_MISS_STATS(FOR_ITER); + assert(_PyOpcode_Deopt[opcode] == (FOR_ITER)); + JUMP_TO_PREDICTED(TRACING_FOR_ITER); + } + #ifdef Py_GIL_DISABLED + if (!_PyObject_IsUniquelyReferenced((PyObject *)r)) { + UPDATE_MISS_STATS(FOR_ITER); + assert(_PyOpcode_Deopt[opcode] == (FOR_ITER)); + JUMP_TO_PREDICTED(TRACING_FOR_ITER); + } + #endif + } + // _ITER_JUMP_RANGE + { + _PyRangeIterObject *r = (_PyRangeIterObject *)PyStackRef_AsPyObjectBorrow(iter); + assert(Py_TYPE(r) == &PyRangeIter_Type); + #ifdef Py_GIL_DISABLED + assert(_PyObject_IsUniquelyReferenced((PyObject *)r)); + #endif + STAT_INC(FOR_ITER, hit); + if (r->len <= 0) { + JUMPBY(oparg + 1); + RECORD_JUMP_TAKEN(); + TRACING_DISPATCH(); + } + } + // _ITER_NEXT_RANGE + { + _PyRangeIterObject *r = (_PyRangeIterObject *)PyStackRef_AsPyObjectBorrow(iter); + assert(Py_TYPE(r) == &PyRangeIter_Type); + #ifdef Py_GIL_DISABLED + assert(_PyObject_IsUniquelyReferenced((PyObject *)r)); + #endif + assert(r->len > 0); + long value = r->start; + r->start = value + r->step; + r->len--; + PyObject *res = PyLong_FromLong(value); + if (res == NULL) { + TRACING_JUMP_TO_LABEL(error); + } + next = PyStackRef_FromPyObjectSteal(res); + } + stack_pointer[0] = next; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + TRACING_DISPATCH(); + } + + TRACING_TARGET(FOR_ITER_TUPLE) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = FOR_ITER_TUPLE; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(FOR_ITER_TUPLE); + opcode = FOR_ITER_TUPLE; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + static_assert(INLINE_CACHE_ENTRIES_FOR_ITER == 1, "incorrect cache size"); + _PyStackRef iter; + _PyStackRef null_or_index; + _PyStackRef next; + /* Skip 1 cache entry */ + // _ITER_CHECK_TUPLE + { + null_or_index = stack_pointer[-1]; + iter = stack_pointer[-2]; + PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter); + if (Py_TYPE(iter_o) != &PyTuple_Type) { + UPDATE_MISS_STATS(FOR_ITER); + assert(_PyOpcode_Deopt[opcode] == (FOR_ITER)); + JUMP_TO_PREDICTED(TRACING_FOR_ITER); + } + assert(PyStackRef_IsTaggedInt(null_or_index)); + } + // _ITER_JUMP_TUPLE + { + PyObject *tuple_o = PyStackRef_AsPyObjectBorrow(iter); + (void)tuple_o; + assert(Py_TYPE(tuple_o) == &PyTuple_Type); + STAT_INC(FOR_ITER, hit); + if ((size_t)PyStackRef_UntagInt(null_or_index) >= (size_t)PyTuple_GET_SIZE(tuple_o)) { + null_or_index = PyStackRef_TagInt(-1); + JUMPBY(oparg + 1); + RECORD_JUMP_TAKEN(); + stack_pointer[-1] = null_or_index; + TRACING_DISPATCH(); + } + } + // _ITER_NEXT_TUPLE + { + PyObject *tuple_o = PyStackRef_AsPyObjectBorrow(iter); + assert(Py_TYPE(tuple_o) == &PyTuple_Type); + uintptr_t i = PyStackRef_UntagInt(null_or_index); + assert((size_t)i < (size_t)PyTuple_GET_SIZE(tuple_o)); + next = PyStackRef_FromPyObjectNew(PyTuple_GET_ITEM(tuple_o, i)); + null_or_index = PyStackRef_IncrementTaggedIntNoOverflow(null_or_index); + } + stack_pointer[-1] = null_or_index; + stack_pointer[0] = next; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + TRACING_DISPATCH(); + } + + TRACING_TARGET(GET_AITER) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = GET_AITER; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(GET_AITER); + opcode = GET_AITER; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + _PyStackRef obj; + _PyStackRef iter; + obj = stack_pointer[-1]; + unaryfunc getter = NULL; + PyObject *obj_o = PyStackRef_AsPyObjectBorrow(obj); + PyObject *iter_o; + PyTypeObject *type = Py_TYPE(obj_o); + if (type->tp_as_async != NULL) { + getter = type->tp_as_async->am_aiter; + } + if (getter == NULL) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyErr_Format(tstate, PyExc_TypeError, + "'async for' requires an object with " + "__aiter__ method, got %.100s", + type->tp_name); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(obj); + stack_pointer = _PyFrame_GetStackPointer(frame); + TRACING_JUMP_TO_LABEL(error); + } + _PyFrame_SetStackPointer(frame, stack_pointer); + iter_o = (*getter)(obj_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(obj); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (iter_o == NULL) { + TRACING_JUMP_TO_LABEL(error); + } + if (Py_TYPE(iter_o)->tp_as_async == NULL || + Py_TYPE(iter_o)->tp_as_async->am_anext == NULL) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyErr_Format(tstate, PyExc_TypeError, + "'async for' received an object from __aiter__ " + "that does not implement __anext__: %.100s", + Py_TYPE(iter_o)->tp_name); + Py_DECREF(iter_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + TRACING_JUMP_TO_LABEL(error); + } + iter = PyStackRef_FromPyObjectSteal(iter_o); + stack_pointer[0] = iter; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + TRACING_DISPATCH(); + } + + TRACING_TARGET(GET_ANEXT) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = GET_ANEXT; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(GET_ANEXT); + opcode = GET_ANEXT; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + _PyStackRef aiter; + _PyStackRef awaitable; + aiter = stack_pointer[-1]; + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *awaitable_o = _PyEval_GetANext(PyStackRef_AsPyObjectBorrow(aiter)); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (awaitable_o == NULL) { + TRACING_JUMP_TO_LABEL(error); + } + awaitable = PyStackRef_FromPyObjectSteal(awaitable_o); + stack_pointer[0] = awaitable; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + TRACING_DISPATCH(); + } + + TRACING_TARGET(GET_AWAITABLE) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = GET_AWAITABLE; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(GET_AWAITABLE); + opcode = GET_AWAITABLE; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + _PyStackRef iterable; + _PyStackRef iter; + iterable = stack_pointer[-1]; + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *iter_o = _PyEval_GetAwaitable(PyStackRef_AsPyObjectBorrow(iterable), oparg); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(iterable); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (iter_o == NULL) { + TRACING_JUMP_TO_LABEL(error); + } + iter = PyStackRef_FromPyObjectSteal(iter_o); + stack_pointer[0] = iter; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + TRACING_DISPATCH(); + } + + TRACING_TARGET(GET_ITER) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = GET_ITER; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(GET_ITER); + opcode = GET_ITER; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + _PyStackRef iterable; + _PyStackRef iter; + _PyStackRef index_or_null; + iterable = stack_pointer[-1]; + #ifdef Py_STATS + _PyFrame_SetStackPointer(frame, stack_pointer); + _Py_GatherStats_GetIter(iterable); + stack_pointer = _PyFrame_GetStackPointer(frame); + #endif + + PyTypeObject *tp = PyStackRef_TYPE(iterable); + if (tp == &PyTuple_Type || tp == &PyList_Type) { + iter = iterable; + index_or_null = PyStackRef_TagInt(0); + } + else { + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *iter_o = PyObject_GetIter(PyStackRef_AsPyObjectBorrow(iterable)); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(iterable); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (iter_o == NULL) { + TRACING_JUMP_TO_LABEL(error); + } + iter = PyStackRef_FromPyObjectSteal(iter_o); + index_or_null = PyStackRef_NULL; + stack_pointer += 1; + } + stack_pointer[-1] = iter; + stack_pointer[0] = index_or_null; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + TRACING_DISPATCH(); + } + + TRACING_TARGET(GET_LEN) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = GET_LEN; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(GET_LEN); + opcode = GET_LEN; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + _PyStackRef obj; + _PyStackRef len; + obj = stack_pointer[-1]; + _PyFrame_SetStackPointer(frame, stack_pointer); + Py_ssize_t len_i = PyObject_Length(PyStackRef_AsPyObjectBorrow(obj)); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (len_i < 0) { + TRACING_JUMP_TO_LABEL(error); + } + PyObject *len_o = PyLong_FromSsize_t(len_i); + if (len_o == NULL) { + TRACING_JUMP_TO_LABEL(error); + } + len = PyStackRef_FromPyObjectSteal(len_o); + stack_pointer[0] = len; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + TRACING_DISPATCH(); + } + + TRACING_TARGET(GET_YIELD_FROM_ITER) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = GET_YIELD_FROM_ITER; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(GET_YIELD_FROM_ITER); + opcode = GET_YIELD_FROM_ITER; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + _PyStackRef iterable; + _PyStackRef iter; + iterable = stack_pointer[-1]; + PyObject *iterable_o = PyStackRef_AsPyObjectBorrow(iterable); + if (PyCoro_CheckExact(iterable_o)) { + if (!(_PyFrame_GetCode(frame)->co_flags & (CO_COROUTINE | CO_ITERABLE_COROUTINE))) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyErr_SetString(tstate, PyExc_TypeError, + "cannot 'yield from' a coroutine object " + "in a non-coroutine generator"); + stack_pointer = _PyFrame_GetStackPointer(frame); + TRACING_JUMP_TO_LABEL(error); + } + iter = iterable; + } + else if (PyGen_CheckExact(iterable_o)) { + iter = iterable; + } + else { + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *iter_o = PyObject_GetIter(iterable_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (iter_o == NULL) { + TRACING_JUMP_TO_LABEL(error); + } + iter = PyStackRef_FromPyObjectSteal(iter_o); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyStackRef tmp = iterable; + iterable = iter; + stack_pointer[-1] = iterable; + PyStackRef_CLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + stack_pointer[-1] = iter; + TRACING_DISPATCH(); + } + + TRACING_TARGET(IMPORT_FROM) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = IMPORT_FROM; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(IMPORT_FROM); + opcode = IMPORT_FROM; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + _PyStackRef from; + _PyStackRef res; + from = stack_pointer[-1]; + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *res_o = _PyEval_ImportFrom(tstate, PyStackRef_AsPyObjectBorrow(from), name); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (res_o == NULL) { + TRACING_JUMP_TO_LABEL(error); + } + res = PyStackRef_FromPyObjectSteal(res_o); + stack_pointer[0] = res; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + TRACING_DISPATCH(); + } + + TRACING_TARGET(IMPORT_NAME) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = IMPORT_NAME; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(IMPORT_NAME); + opcode = IMPORT_NAME; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + _PyStackRef level; + _PyStackRef fromlist; + _PyStackRef res; + fromlist = stack_pointer[-1]; + level = stack_pointer[-2]; + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *res_o = _PyEval_ImportName(tstate, frame, name, + PyStackRef_AsPyObjectBorrow(fromlist), + PyStackRef_AsPyObjectBorrow(level)); + _PyStackRef tmp = fromlist; + fromlist = PyStackRef_NULL; + stack_pointer[-1] = fromlist; + PyStackRef_CLOSE(tmp); + tmp = level; + level = PyStackRef_NULL; + stack_pointer[-2] = level; + PyStackRef_CLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); + if (res_o == NULL) { + TRACING_JUMP_TO_LABEL(error); + } + res = PyStackRef_FromPyObjectSteal(res_o); + stack_pointer[0] = res; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + TRACING_DISPATCH(); + } + + TRACING_TARGET(INSTRUMENTED_CALL) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = INSTRUMENTED_CALL; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(INSTRUMENTED_CALL); + opcode = INSTRUMENTED_CALL; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + opcode = INSTRUMENTED_CALL; + _PyStackRef callable; + _PyStackRef self_or_null; + _PyStackRef func; + _PyStackRef maybe_self; + _PyStackRef *args; + _PyStackRef res; + /* Skip 3 cache entries */ + // _MAYBE_EXPAND_METHOD + { + self_or_null = stack_pointer[-1 - oparg]; + callable = stack_pointer[-2 - oparg]; + if (PyStackRef_TYPE(callable) == &PyMethod_Type && PyStackRef_IsNull(self_or_null)) { + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + PyObject *self = ((PyMethodObject *)callable_o)->im_self; + self_or_null = PyStackRef_FromPyObjectNew(self); + PyObject *method = ((PyMethodObject *)callable_o)->im_func; + _PyStackRef temp = callable; + callable = PyStackRef_FromPyObjectNew(method); + stack_pointer[-2 - oparg] = callable; + stack_pointer[-1 - oparg] = self_or_null; + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(temp); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + } + // _MONITOR_CALL + { + args = &stack_pointer[-oparg]; + maybe_self = self_or_null; + func = callable; + int is_meth = !PyStackRef_IsNull(maybe_self); + PyObject *function = PyStackRef_AsPyObjectBorrow(func); + PyObject *arg0; + if (is_meth) { + arg0 = PyStackRef_AsPyObjectBorrow(maybe_self); + } + else if (oparg) { + arg0 = PyStackRef_AsPyObjectBorrow(args[0]); + } + else { + arg0 = &_PyInstrumentation_MISSING; + } + stack_pointer[-2 - oparg] = func; + stack_pointer[-1 - oparg] = maybe_self; + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = _Py_call_instrumentation_2args( + tstate, PY_MONITORING_EVENT_CALL, + frame, this_instr, function, arg0 + ); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err) { + TRACING_JUMP_TO_LABEL(error); + } + } + // _DO_CALL + { + args = &stack_pointer[-oparg]; + self_or_null = stack_pointer[-1 - oparg]; + callable = stack_pointer[-2 - oparg]; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + int total_args = oparg; + _PyStackRef *arguments = args; + if (!PyStackRef_IsNull(self_or_null)) { + arguments--; + total_args++; + } + if (Py_TYPE(callable_o) == &PyFunction_Type && + tstate->interp->eval_frame == NULL && + ((PyFunctionObject *)callable_o)->vectorcall == _PyFunction_Vectorcall) + { + int code_flags = ((PyCodeObject*)PyFunction_GET_CODE(callable_o))->co_flags; + PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable_o)); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyInterpreterFrame *new_frame = _PyEvalFramePushAndInit( + tstate, callable, locals, + arguments, total_args, NULL, frame + ); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -2 - oparg; + assert(WITHIN_STACK_BOUNDS()); + if (new_frame == NULL) { + TRACING_JUMP_TO_LABEL(error); + } + frame->return_offset = 4u ; + TRACING_DISPATCH_INLINED(new_frame); + } + STACKREFS_TO_PYOBJECTS(arguments, total_args, args_o); + if (CONVERSION_FAILED(args_o)) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyStackRef tmp; + for (int _i = oparg; --_i >= 0;) { + tmp = args[_i]; + args[_i] = PyStackRef_NULL; + PyStackRef_CLOSE(tmp); + } + tmp = self_or_null; + self_or_null = PyStackRef_NULL; + stack_pointer[-1 - oparg] = self_or_null; + PyStackRef_XCLOSE(tmp); + tmp = callable; + callable = PyStackRef_NULL; + stack_pointer[-2 - oparg] = callable; + PyStackRef_CLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -2 - oparg; + assert(WITHIN_STACK_BOUNDS()); + TRACING_JUMP_TO_LABEL(error); + } + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *res_o = PyObject_Vectorcall( + callable_o, args_o, + total_args | PY_VECTORCALL_ARGUMENTS_OFFSET, + NULL); + stack_pointer = _PyFrame_GetStackPointer(frame); + STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); + if (opcode == INSTRUMENTED_CALL) { + PyObject *arg = total_args == 0 ? + &_PyInstrumentation_MISSING : PyStackRef_AsPyObjectBorrow(arguments[0]); + if (res_o == NULL) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _Py_call_instrumentation_exc2( + tstate, PY_MONITORING_EVENT_C_RAISE, + frame, this_instr, callable_o, arg); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + else { + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = _Py_call_instrumentation_2args( + tstate, PY_MONITORING_EVENT_C_RETURN, + frame, this_instr, callable_o, arg); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err < 0) { + _PyFrame_SetStackPointer(frame, stack_pointer); + Py_CLEAR(res_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + } + } + assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyStackRef tmp; + for (int _i = oparg; --_i >= 0;) { + tmp = args[_i]; + args[_i] = PyStackRef_NULL; + PyStackRef_CLOSE(tmp); + } + tmp = self_or_null; + self_or_null = PyStackRef_NULL; + stack_pointer[-1 - oparg] = self_or_null; + PyStackRef_XCLOSE(tmp); + tmp = callable; + callable = PyStackRef_NULL; + stack_pointer[-2 - oparg] = callable; + PyStackRef_CLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -2 - oparg; + assert(WITHIN_STACK_BOUNDS()); + if (res_o == NULL) { + TRACING_JUMP_TO_LABEL(error); + } + res = PyStackRef_FromPyObjectSteal(res_o); + } + // _CHECK_PERIODIC_AT_END + { + stack_pointer[0] = res; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = check_periodics(tstate); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err != 0) { + TRACING_JUMP_TO_LABEL(error); + } + } + TRACING_DISPATCH(); + } + + TRACING_TARGET(INSTRUMENTED_CALL_FUNCTION_EX) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = INSTRUMENTED_CALL_FUNCTION_EX; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(INSTRUMENTED_CALL_FUNCTION_EX); + opcode = INSTRUMENTED_CALL_FUNCTION_EX; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + opcode = INSTRUMENTED_CALL_FUNCTION_EX; + _PyStackRef func; + _PyStackRef callargs; + _PyStackRef func_st; + _PyStackRef null; + _PyStackRef callargs_st; + _PyStackRef kwargs_st; + _PyStackRef result; + // _MAKE_CALLARGS_A_TUPLE + { + callargs = stack_pointer[-2]; + func = stack_pointer[-4]; + PyObject *callargs_o = PyStackRef_AsPyObjectBorrow(callargs); + if (!PyTuple_CheckExact(callargs_o)) { + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = _Py_Check_ArgsIterable(tstate, PyStackRef_AsPyObjectBorrow(func), callargs_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err < 0) { + TRACING_JUMP_TO_LABEL(error); + } + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *tuple_o = PySequence_Tuple(callargs_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (tuple_o == NULL) { + TRACING_JUMP_TO_LABEL(error); + } + _PyStackRef temp = callargs; + callargs = PyStackRef_FromPyObjectSteal(tuple_o); + stack_pointer[-2] = callargs; + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(temp); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + } + // _DO_CALL_FUNCTION_EX + { + kwargs_st = stack_pointer[-1]; + callargs_st = callargs; + null = stack_pointer[-3]; + func_st = func; + (void)null; + PyObject *func = PyStackRef_AsPyObjectBorrow(func_st); + EVAL_CALL_STAT_INC_IF_FUNCTION(EVAL_CALL_FUNCTION_EX, func); + PyObject *result_o; + assert(!_PyErr_Occurred(tstate)); + if (opcode == INSTRUMENTED_CALL_FUNCTION_EX) { + PyObject *callargs = PyStackRef_AsPyObjectBorrow(callargs_st); + PyObject *kwargs = PyStackRef_AsPyObjectBorrow(kwargs_st); + assert(kwargs == NULL || PyDict_CheckExact(kwargs)); + assert(PyTuple_CheckExact(callargs)); + PyObject *arg = PyTuple_GET_SIZE(callargs) > 0 ? + PyTuple_GET_ITEM(callargs, 0) : &_PyInstrumentation_MISSING; + stack_pointer[-2] = callargs_st; + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = _Py_call_instrumentation_2args( + tstate, PY_MONITORING_EVENT_CALL, + frame, this_instr, func, arg); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err) { + TRACING_JUMP_TO_LABEL(error); + } + _PyFrame_SetStackPointer(frame, stack_pointer); + result_o = PyObject_Call(func, callargs, kwargs); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (!PyFunction_Check(func) && !PyMethod_Check(func)) { + if (result_o == NULL) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _Py_call_instrumentation_exc2( + tstate, PY_MONITORING_EVENT_C_RAISE, + frame, this_instr, func, arg); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + else { + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = _Py_call_instrumentation_2args( + tstate, PY_MONITORING_EVENT_C_RETURN, + frame, this_instr, func, arg); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err < 0) { + _PyFrame_SetStackPointer(frame, stack_pointer); + Py_CLEAR(result_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + } + } + } + else { + if (Py_TYPE(func) == &PyFunction_Type && + tstate->interp->eval_frame == NULL && + ((PyFunctionObject *)func)->vectorcall == _PyFunction_Vectorcall) { + PyObject *callargs = PyStackRef_AsPyObjectSteal(callargs_st); + assert(PyTuple_CheckExact(callargs)); + PyObject *kwargs = PyStackRef_IsNull(kwargs_st) ? NULL : PyStackRef_AsPyObjectSteal(kwargs_st); + assert(kwargs == NULL || PyDict_CheckExact(kwargs)); + Py_ssize_t nargs = PyTuple_GET_SIZE(callargs); + int code_flags = ((PyCodeObject *)PyFunction_GET_CODE(func))->co_flags; + PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(func)); + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyInterpreterFrame *new_frame = _PyEvalFramePushAndInit_Ex( + tstate, func_st, locals, + nargs, callargs, kwargs, frame); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); + if (new_frame == NULL) { + TRACING_JUMP_TO_LABEL(error); + } + assert( 1u == 1); + frame->return_offset = 1; + TRACING_DISPATCH_INLINED(new_frame); + } + PyObject *callargs = PyStackRef_AsPyObjectBorrow(callargs_st); + assert(PyTuple_CheckExact(callargs)); + PyObject *kwargs = PyStackRef_AsPyObjectBorrow(kwargs_st); + assert(kwargs == NULL || PyDict_CheckExact(kwargs)); + stack_pointer[-2] = callargs_st; + _PyFrame_SetStackPointer(frame, stack_pointer); + result_o = PyObject_Call(func, callargs, kwargs); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_XCLOSE(kwargs_st); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(callargs_st); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(func_st); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (result_o == NULL) { + TRACING_JUMP_TO_LABEL(error); + } + result = PyStackRef_FromPyObjectSteal(result_o); + } + // _CHECK_PERIODIC_AT_END + { + stack_pointer[0] = result; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = check_periodics(tstate); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err != 0) { + TRACING_JUMP_TO_LABEL(error); + } + } + TRACING_DISPATCH(); + } + + TRACING_TARGET(INSTRUMENTED_CALL_KW) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = INSTRUMENTED_CALL_KW; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(INSTRUMENTED_CALL_KW); + opcode = INSTRUMENTED_CALL_KW; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + opcode = INSTRUMENTED_CALL_KW; + _PyStackRef callable; + _PyStackRef self_or_null; + _PyStackRef *args; + _PyStackRef kwnames; + _PyStackRef res; + /* Skip 1 cache entry */ + /* Skip 2 cache entries */ + // _MAYBE_EXPAND_METHOD_KW + { + self_or_null = stack_pointer[-2 - oparg]; + callable = stack_pointer[-3 - oparg]; + if (PyStackRef_TYPE(callable) == &PyMethod_Type && PyStackRef_IsNull(self_or_null)) { + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + PyObject *self = ((PyMethodObject *)callable_o)->im_self; + self_or_null = PyStackRef_FromPyObjectNew(self); + PyObject *method = ((PyMethodObject *)callable_o)->im_func; + _PyStackRef temp = callable; + callable = PyStackRef_FromPyObjectNew(method); + stack_pointer[-3 - oparg] = callable; + stack_pointer[-2 - oparg] = self_or_null; + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(temp); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + } + // _MONITOR_CALL_KW + { + args = &stack_pointer[-1 - oparg]; + int is_meth = !PyStackRef_IsNull(self_or_null); + PyObject *arg; + if (is_meth) { + arg = PyStackRef_AsPyObjectBorrow(self_or_null); + } + else if (args) { + arg = PyStackRef_AsPyObjectBorrow(args[0]); + } + else { + arg = &_PyInstrumentation_MISSING; + } + PyObject *function = PyStackRef_AsPyObjectBorrow(callable); + stack_pointer[-3 - oparg] = callable; + stack_pointer[-2 - oparg] = self_or_null; + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = _Py_call_instrumentation_2args( + tstate, PY_MONITORING_EVENT_CALL, + frame, this_instr, function, arg); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err) { + TRACING_JUMP_TO_LABEL(error); + } + } + // _DO_CALL_KW + { + kwnames = stack_pointer[-1]; + args = &stack_pointer[-1 - oparg]; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + PyObject *kwnames_o = PyStackRef_AsPyObjectBorrow(kwnames); + int total_args = oparg; + _PyStackRef *arguments = args; + if (!PyStackRef_IsNull(self_or_null)) { + arguments--; + total_args++; + } + int positional_args = total_args - (int)PyTuple_GET_SIZE(kwnames_o); + if (Py_TYPE(callable_o) == &PyFunction_Type && + tstate->interp->eval_frame == NULL && + ((PyFunctionObject *)callable_o)->vectorcall == _PyFunction_Vectorcall) + { + int code_flags = ((PyCodeObject*)PyFunction_GET_CODE(callable_o))->co_flags; + PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable_o)); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyInterpreterFrame *new_frame = _PyEvalFramePushAndInit( + tstate, callable, locals, + arguments, positional_args, kwnames_o, frame + ); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -3 - oparg; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(kwnames); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (new_frame == NULL) { + TRACING_JUMP_TO_LABEL(error); + } + assert( 4u == 1 + INLINE_CACHE_ENTRIES_CALL_KW); + frame->return_offset = 4u ; + TRACING_DISPATCH_INLINED(new_frame); + } + STACKREFS_TO_PYOBJECTS(arguments, total_args, args_o); + if (CONVERSION_FAILED(args_o)) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyStackRef tmp = kwnames; + kwnames = PyStackRef_NULL; + stack_pointer[-1] = kwnames; + PyStackRef_CLOSE(tmp); + for (int _i = oparg; --_i >= 0;) { + tmp = args[_i]; + args[_i] = PyStackRef_NULL; + PyStackRef_CLOSE(tmp); + } + tmp = self_or_null; + self_or_null = PyStackRef_NULL; + stack_pointer[-2 - oparg] = self_or_null; + PyStackRef_XCLOSE(tmp); + tmp = callable; + callable = PyStackRef_NULL; + stack_pointer[-3 - oparg] = callable; + PyStackRef_CLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -3 - oparg; + assert(WITHIN_STACK_BOUNDS()); + TRACING_JUMP_TO_LABEL(error); + } + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *res_o = PyObject_Vectorcall( + callable_o, args_o, + positional_args | PY_VECTORCALL_ARGUMENTS_OFFSET, + kwnames_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); + if (opcode == INSTRUMENTED_CALL_KW) { + PyObject *arg = total_args == 0 ? + &_PyInstrumentation_MISSING : PyStackRef_AsPyObjectBorrow(arguments[0]); + if (res_o == NULL) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _Py_call_instrumentation_exc2( + tstate, PY_MONITORING_EVENT_C_RAISE, + frame, this_instr, callable_o, arg); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + else { + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = _Py_call_instrumentation_2args( + tstate, PY_MONITORING_EVENT_C_RETURN, + frame, this_instr, callable_o, arg); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err < 0) { + _PyFrame_SetStackPointer(frame, stack_pointer); + Py_CLEAR(res_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + } + } + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyStackRef tmp = kwnames; + kwnames = PyStackRef_NULL; + stack_pointer[-1] = kwnames; + PyStackRef_CLOSE(tmp); + for (int _i = oparg; --_i >= 0;) { + tmp = args[_i]; + args[_i] = PyStackRef_NULL; + PyStackRef_CLOSE(tmp); + } + tmp = self_or_null; + self_or_null = PyStackRef_NULL; + stack_pointer[-2 - oparg] = self_or_null; + PyStackRef_XCLOSE(tmp); + tmp = callable; + callable = PyStackRef_NULL; + stack_pointer[-3 - oparg] = callable; + PyStackRef_CLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -3 - oparg; + assert(WITHIN_STACK_BOUNDS()); + if (res_o == NULL) { + TRACING_JUMP_TO_LABEL(error); + } + res = PyStackRef_FromPyObjectSteal(res_o); + } + stack_pointer[0] = res; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + TRACING_DISPATCH(); + } + + TRACING_TARGET(INSTRUMENTED_END_ASYNC_FOR) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = INSTRUMENTED_END_ASYNC_FOR; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(INSTRUMENTED_END_ASYNC_FOR); + opcode = INSTRUMENTED_END_ASYNC_FOR; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + _PyStackRef awaitable_st; + _PyStackRef exc_st; + // _MONITOR_END_ASYNC_FOR + { + assert((next_instr-oparg)->op.code == END_SEND || (next_instr-oparg)->op.code >= MIN_INSTRUMENTED_OPCODE); + INSTRUMENTED_JUMP(next_instr-oparg, this_instr+1, PY_MONITORING_EVENT_BRANCH_RIGHT); + } + // _END_ASYNC_FOR + { + exc_st = stack_pointer[-1]; + awaitable_st = stack_pointer[-2]; + JUMPBY(0); + (void)oparg; + PyObject *exc = PyStackRef_AsPyObjectBorrow(exc_st); + assert(exc && PyExceptionInstance_Check(exc)); + _PyFrame_SetStackPointer(frame, stack_pointer); + int matches = PyErr_GivenExceptionMatches(exc, PyExc_StopAsyncIteration); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (matches) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyStackRef tmp = exc_st; + exc_st = PyStackRef_NULL; + stack_pointer[-1] = exc_st; + PyStackRef_CLOSE(tmp); + tmp = awaitable_st; + awaitable_st = PyStackRef_NULL; + stack_pointer[-2] = awaitable_st; + PyStackRef_CLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); + } + else { + Py_INCREF(exc); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyErr_SetRaisedException(tstate, exc); + monitor_reraise(tstate, frame, this_instr); + TRACING_JUMP_TO_LABEL(exception_unwind); + } + } + TRACING_DISPATCH(); + } + + TRACING_TARGET(INSTRUMENTED_END_FOR) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = INSTRUMENTED_END_FOR; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + next_instr += 1; + INSTRUCTION_STATS(INSTRUMENTED_END_FOR); + opcode = INSTRUMENTED_END_FOR; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + _PyStackRef receiver; + _PyStackRef value; + value = stack_pointer[-1]; + receiver = stack_pointer[-3]; + if (PyStackRef_GenCheck(receiver)) { + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = monitor_stop_iteration(tstate, frame, this_instr, PyStackRef_AsPyObjectBorrow(value)); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err) { + TRACING_JUMP_TO_LABEL(error); + } + } + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(value); + stack_pointer = _PyFrame_GetStackPointer(frame); + TRACING_DISPATCH(); + } + + TRACING_TARGET(INSTRUMENTED_END_SEND) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = INSTRUMENTED_END_SEND; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(INSTRUMENTED_END_SEND); + opcode = INSTRUMENTED_END_SEND; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + _PyStackRef receiver; + _PyStackRef value; + _PyStackRef val; + value = stack_pointer[-1]; + receiver = stack_pointer[-2]; + PyObject *receiver_o = PyStackRef_AsPyObjectBorrow(receiver); + if (PyGen_Check(receiver_o) || PyCoro_CheckExact(receiver_o)) { + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = monitor_stop_iteration(tstate, frame, this_instr, PyStackRef_AsPyObjectBorrow(value)); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err) { + TRACING_JUMP_TO_LABEL(error); + } + } + val = value; + stack_pointer[-2] = val; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(receiver); + stack_pointer = _PyFrame_GetStackPointer(frame); + TRACING_DISPATCH(); + } + + TRACING_TARGET(INSTRUMENTED_FOR_ITER) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = INSTRUMENTED_FOR_ITER; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(INSTRUMENTED_FOR_ITER); + opcode = INSTRUMENTED_FOR_ITER; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + _PyStackRef iter; + _PyStackRef null_or_index; + _PyStackRef next; + /* Skip 1 cache entry */ + null_or_index = stack_pointer[-1]; + iter = stack_pointer[-2]; + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyStackRef item = _PyForIter_VirtualIteratorNext(tstate, frame, iter, &null_or_index); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (!PyStackRef_IsValid(item)) { + if (PyStackRef_IsError(item)) { + TRACING_JUMP_TO_LABEL(error); + } + JUMPBY(oparg + 1); + stack_pointer[-1] = null_or_index; + TRACING_DISPATCH(); + } + next = item; + INSTRUMENTED_JUMP(this_instr, next_instr, PY_MONITORING_EVENT_BRANCH_LEFT); + stack_pointer[-1] = null_or_index; + stack_pointer[0] = next; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + TRACING_DISPATCH(); + } + + TRACING_TARGET(INSTRUMENTED_INSTRUCTION) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = INSTRUMENTED_INSTRUCTION; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(INSTRUMENTED_INSTRUCTION); + opcode = INSTRUMENTED_INSTRUCTION; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + opcode = INSTRUMENTED_INSTRUCTION; + _PyFrame_SetStackPointer(frame, stack_pointer); + int next_opcode = _Py_call_instrumentation_instruction( + tstate, frame, this_instr); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (next_opcode < 0) { + TRACING_JUMP_TO_LABEL(error); + } + next_instr = this_instr; + if (_PyOpcode_Caches[next_opcode]) { + PAUSE_ADAPTIVE_COUNTER(next_instr[1].counter); + } + assert(next_opcode > 0 && next_opcode < 256); + opcode = next_opcode; + DISPATCH_GOTO(); + } + + TRACING_TARGET(INSTRUMENTED_JUMP_BACKWARD) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = INSTRUMENTED_JUMP_BACKWARD; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(INSTRUMENTED_JUMP_BACKWARD); + opcode = INSTRUMENTED_JUMP_BACKWARD; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + /* Skip 1 cache entry */ + // _CHECK_PERIODIC + { + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = check_periodics(tstate); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err != 0) { + TRACING_JUMP_TO_LABEL(error); + } + } + // _MONITOR_JUMP_BACKWARD + { + INSTRUMENTED_JUMP(this_instr, next_instr - oparg, PY_MONITORING_EVENT_JUMP); + } + TRACING_DISPATCH(); + } + + TRACING_TARGET(INSTRUMENTED_JUMP_FORWARD) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = INSTRUMENTED_JUMP_FORWARD; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(INSTRUMENTED_JUMP_FORWARD); + opcode = INSTRUMENTED_JUMP_FORWARD; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + INSTRUMENTED_JUMP(this_instr, next_instr + oparg, PY_MONITORING_EVENT_JUMP); + TRACING_DISPATCH(); + } + + TRACING_TARGET(INSTRUMENTED_LINE) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = INSTRUMENTED_LINE; + (void)(opcode); + #endif + _Py_CODEUNIT* const prev_instr = frame->instr_ptr; + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(INSTRUMENTED_LINE); + opcode = INSTRUMENTED_LINE; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + opcode = INSTRUMENTED_LINE; + int original_opcode = 0; + if (tstate->tracing) { + PyCodeObject *code = _PyFrame_GetCode(frame); + int index = (int)(this_instr - _PyFrame_GetBytecode(frame)); + original_opcode = code->_co_monitoring->lines->data[index*code->_co_monitoring->lines->bytes_per_entry]; + next_instr = this_instr; + } else { + _PyFrame_SetStackPointer(frame, stack_pointer); + original_opcode = _Py_call_instrumentation_line( + tstate, frame, this_instr, prev_instr); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (original_opcode < 0) { + next_instr = this_instr+1; + TRACING_JUMP_TO_LABEL(error); + } + next_instr = frame->instr_ptr; + if (next_instr != this_instr) { + TRACING_DISPATCH(); + } + } + if (_PyOpcode_Caches[original_opcode]) { + _PyBinaryOpCache *cache = (_PyBinaryOpCache *)(next_instr+1); + PAUSE_ADAPTIVE_COUNTER(cache->counter); + } + opcode = original_opcode; + DISPATCH_GOTO(); + } + + TRACING_TARGET(INSTRUMENTED_LOAD_SUPER_ATTR) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = INSTRUMENTED_LOAD_SUPER_ATTR; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(INSTRUMENTED_LOAD_SUPER_ATTR); + opcode = INSTRUMENTED_LOAD_SUPER_ATTR; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + opcode = INSTRUMENTED_LOAD_SUPER_ATTR; + _PyStackRef global_super_st; + _PyStackRef class_st; + _PyStackRef self_st; + _PyStackRef attr; + _PyStackRef *null; + /* Skip 1 cache entry */ + // _LOAD_SUPER_ATTR + { + self_st = stack_pointer[-1]; + class_st = stack_pointer[-2]; + global_super_st = stack_pointer[-3]; + PyObject *global_super = PyStackRef_AsPyObjectBorrow(global_super_st); + PyObject *class = PyStackRef_AsPyObjectBorrow(class_st); + PyObject *self = PyStackRef_AsPyObjectBorrow(self_st); + if (opcode == INSTRUMENTED_LOAD_SUPER_ATTR) { + PyObject *arg = oparg & 2 ? class : &_PyInstrumentation_MISSING; + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = _Py_call_instrumentation_2args( + tstate, PY_MONITORING_EVENT_CALL, + frame, this_instr, global_super, arg); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyStackRef tmp = self_st; + self_st = PyStackRef_NULL; + stack_pointer[-1] = self_st; + PyStackRef_CLOSE(tmp); + tmp = class_st; + class_st = PyStackRef_NULL; + stack_pointer[-2] = class_st; + PyStackRef_CLOSE(tmp); + tmp = global_super_st; + global_super_st = PyStackRef_NULL; + stack_pointer[-3] = global_super_st; + PyStackRef_CLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -3; + assert(WITHIN_STACK_BOUNDS()); + TRACING_JUMP_TO_LABEL(error); + } + } + PyObject *stack[] = {class, self}; + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *super = PyObject_Vectorcall(global_super, stack, oparg & 2, NULL); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (opcode == INSTRUMENTED_LOAD_SUPER_ATTR) { + PyObject *arg = oparg & 2 ? class : &_PyInstrumentation_MISSING; + if (super == NULL) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _Py_call_instrumentation_exc2( + tstate, PY_MONITORING_EVENT_C_RAISE, + frame, this_instr, global_super, arg); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + else { + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = _Py_call_instrumentation_2args( + tstate, PY_MONITORING_EVENT_C_RETURN, + frame, this_instr, global_super, arg); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err < 0) { + _PyFrame_SetStackPointer(frame, stack_pointer); + Py_CLEAR(super); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + } + } + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyStackRef tmp = self_st; + self_st = PyStackRef_NULL; + stack_pointer[-1] = self_st; + PyStackRef_CLOSE(tmp); + tmp = class_st; + class_st = PyStackRef_NULL; + stack_pointer[-2] = class_st; + PyStackRef_CLOSE(tmp); + tmp = global_super_st; + global_super_st = PyStackRef_NULL; + stack_pointer[-3] = global_super_st; + PyStackRef_CLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -3; + assert(WITHIN_STACK_BOUNDS()); + if (super == NULL) { + TRACING_JUMP_TO_LABEL(error); + } + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 2); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *attr_o = PyObject_GetAttr(super, name); + Py_DECREF(super); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (attr_o == NULL) { + TRACING_JUMP_TO_LABEL(error); + } + attr = PyStackRef_FromPyObjectSteal(attr_o); + } + // _PUSH_NULL_CONDITIONAL + { + null = &stack_pointer[1]; + if (oparg & 1) { + null[0] = PyStackRef_NULL; + } + } + stack_pointer[0] = attr; + stack_pointer += 1 + (oparg & 1); + assert(WITHIN_STACK_BOUNDS()); + TRACING_DISPATCH(); + } + + TRACING_TARGET(INSTRUMENTED_NOT_TAKEN) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = INSTRUMENTED_NOT_TAKEN; + (void)(opcode); + #endif + _Py_CODEUNIT* const prev_instr = frame->instr_ptr; + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(INSTRUMENTED_NOT_TAKEN); + opcode = INSTRUMENTED_NOT_TAKEN; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + (void)this_instr; + INSTRUMENTED_JUMP(prev_instr, next_instr, PY_MONITORING_EVENT_BRANCH_LEFT); + TRACING_DISPATCH(); + } + + TRACING_TARGET(INSTRUMENTED_POP_ITER) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = INSTRUMENTED_POP_ITER; + (void)(opcode); + #endif + _Py_CODEUNIT* const prev_instr = frame->instr_ptr; + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(INSTRUMENTED_POP_ITER); + opcode = INSTRUMENTED_POP_ITER; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + _PyStackRef iter; + _PyStackRef index_or_null; + index_or_null = stack_pointer[-1]; + iter = stack_pointer[-2]; + (void)index_or_null; + INSTRUMENTED_JUMP(prev_instr, this_instr+1, PY_MONITORING_EVENT_BRANCH_RIGHT); + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(iter); + stack_pointer = _PyFrame_GetStackPointer(frame); + TRACING_DISPATCH(); + } + + TRACING_TARGET(INSTRUMENTED_POP_JUMP_IF_FALSE) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = INSTRUMENTED_POP_JUMP_IF_FALSE; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(INSTRUMENTED_POP_JUMP_IF_FALSE); + opcode = INSTRUMENTED_POP_JUMP_IF_FALSE; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + _PyStackRef cond; + /* Skip 1 cache entry */ + cond = stack_pointer[-1]; + assert(PyStackRef_BoolCheck(cond)); + int jump = PyStackRef_IsFalse(cond); + RECORD_BRANCH_TAKEN(this_instr[1].cache, jump); + if (jump) { + INSTRUMENTED_JUMP(this_instr, next_instr + oparg, PY_MONITORING_EVENT_BRANCH_RIGHT); + } + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + TRACING_DISPATCH(); + } + + TRACING_TARGET(INSTRUMENTED_POP_JUMP_IF_NONE) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = INSTRUMENTED_POP_JUMP_IF_NONE; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(INSTRUMENTED_POP_JUMP_IF_NONE); + opcode = INSTRUMENTED_POP_JUMP_IF_NONE; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + _PyStackRef value; + /* Skip 1 cache entry */ + value = stack_pointer[-1]; + int jump = PyStackRef_IsNone(value); + RECORD_BRANCH_TAKEN(this_instr[1].cache, jump); + if (jump) { + INSTRUMENTED_JUMP(this_instr, next_instr + oparg, PY_MONITORING_EVENT_BRANCH_RIGHT); + } + else { + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(value); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += 1; + } + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + TRACING_DISPATCH(); + } + + TRACING_TARGET(INSTRUMENTED_POP_JUMP_IF_NOT_NONE) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = INSTRUMENTED_POP_JUMP_IF_NOT_NONE; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(INSTRUMENTED_POP_JUMP_IF_NOT_NONE); + opcode = INSTRUMENTED_POP_JUMP_IF_NOT_NONE; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + _PyStackRef value; + /* Skip 1 cache entry */ + value = stack_pointer[-1]; + int jump = !PyStackRef_IsNone(value); + RECORD_BRANCH_TAKEN(this_instr[1].cache, jump); + if (jump) { + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(value); + stack_pointer = _PyFrame_GetStackPointer(frame); + INSTRUMENTED_JUMP(this_instr, next_instr + oparg, PY_MONITORING_EVENT_BRANCH_RIGHT); + } + else { + stack_pointer += -1; + } + TRACING_DISPATCH(); + } + + TRACING_TARGET(INSTRUMENTED_POP_JUMP_IF_TRUE) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = INSTRUMENTED_POP_JUMP_IF_TRUE; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(INSTRUMENTED_POP_JUMP_IF_TRUE); + opcode = INSTRUMENTED_POP_JUMP_IF_TRUE; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + _PyStackRef cond; + /* Skip 1 cache entry */ + cond = stack_pointer[-1]; + assert(PyStackRef_BoolCheck(cond)); + int jump = PyStackRef_IsTrue(cond); + RECORD_BRANCH_TAKEN(this_instr[1].cache, jump); + if (jump) { + INSTRUMENTED_JUMP(this_instr, next_instr + oparg, PY_MONITORING_EVENT_BRANCH_RIGHT); + } + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + TRACING_DISPATCH(); + } + + TRACING_TARGET(INSTRUMENTED_RESUME) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = INSTRUMENTED_RESUME; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(INSTRUMENTED_RESUME); + opcode = INSTRUMENTED_RESUME; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + // _LOAD_BYTECODE + { + #ifdef Py_GIL_DISABLED + if (frame->tlbc_index != + ((_PyThreadStateImpl *)tstate)->tlbc_index) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _Py_CODEUNIT *bytecode = + _PyEval_GetExecutableCode(tstate, _PyFrame_GetCode(frame)); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (bytecode == NULL) { + TRACING_JUMP_TO_LABEL(error); + } + ptrdiff_t off = this_instr - _PyFrame_GetBytecode(frame); + frame->tlbc_index = ((_PyThreadStateImpl *)tstate)->tlbc_index; + frame->instr_ptr = bytecode + off; + next_instr = frame->instr_ptr; + TRACING_DISPATCH(); + } + #endif + } + // _MAYBE_INSTRUMENT + { + #ifdef Py_GIL_DISABLED + + int check_instrumentation = 1; + #else + int check_instrumentation = (tstate->tracing == 0); + #endif + if (check_instrumentation) { + uintptr_t global_version = _Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & ~_PY_EVAL_EVENTS_MASK; + uintptr_t code_version = FT_ATOMIC_LOAD_UINTPTR_ACQUIRE(_PyFrame_GetCode(frame)->_co_instrumentation_version); + if (code_version != global_version) { + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = _Py_Instrument(_PyFrame_GetCode(frame), tstate->interp); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err) { + TRACING_JUMP_TO_LABEL(error); + } + next_instr = this_instr; + TRACING_DISPATCH(); + } + } + } + // _CHECK_PERIODIC_IF_NOT_YIELD_FROM + { + if ((oparg & RESUME_OPARG_LOCATION_MASK) < RESUME_AFTER_YIELD_FROM) { + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = check_periodics(tstate); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err != 0) { + TRACING_JUMP_TO_LABEL(error); + } + } + } + // _MONITOR_RESUME + { + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = _Py_call_instrumentation( + tstate, oparg > 0, frame, this_instr); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err) { + TRACING_JUMP_TO_LABEL(error); + } + if (frame->instr_ptr != this_instr) { + next_instr = frame->instr_ptr; + } + } + TRACING_DISPATCH(); + } + + TRACING_TARGET(INSTRUMENTED_RETURN_VALUE) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = INSTRUMENTED_RETURN_VALUE; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(INSTRUMENTED_RETURN_VALUE); + opcode = INSTRUMENTED_RETURN_VALUE; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + _PyStackRef val; + _PyStackRef retval; + _PyStackRef res; + // _RETURN_VALUE_EVENT + { + val = stack_pointer[-1]; + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = _Py_call_instrumentation_arg( + tstate, PY_MONITORING_EVENT_PY_RETURN, + frame, this_instr, PyStackRef_AsPyObjectBorrow(val)); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err) { + TRACING_JUMP_TO_LABEL(error); + } + } + // _RETURN_VALUE + { + retval = val; + assert(frame->owner != FRAME_OWNED_BY_INTERPRETER); + _PyStackRef temp = PyStackRef_MakeHeapSafe(retval); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + assert(STACK_LEVEL() == 0); + _Py_LeaveRecursiveCallPy(tstate); + _PyInterpreterFrame *dying = frame; + frame = tstate->current_frame = dying->previous; + _PyEval_FrameClearAndPop(tstate, dying); + stack_pointer = _PyFrame_GetStackPointer(frame); + LOAD_IP(frame->return_offset); + #if TIER_TWO + frame->instr_ptr += frame->return_offset; + #endif + res = temp; + LLTRACE_RESUME_FRAME(); + } + stack_pointer[0] = res; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + TRACING_DISPATCH(); + } + + TRACING_TARGET(INSTRUMENTED_YIELD_VALUE) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = INSTRUMENTED_YIELD_VALUE; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(INSTRUMENTED_YIELD_VALUE); + opcode = INSTRUMENTED_YIELD_VALUE; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + _PyStackRef val; + _PyStackRef retval; + _PyStackRef value; + // _YIELD_VALUE_EVENT + { + val = stack_pointer[-1]; + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = _Py_call_instrumentation_arg( + tstate, PY_MONITORING_EVENT_PY_YIELD, + frame, this_instr, PyStackRef_AsPyObjectBorrow(val)); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err) { + TRACING_JUMP_TO_LABEL(error); + } + if (frame->instr_ptr != this_instr) { + next_instr = frame->instr_ptr; + TRACING_DISPATCH(); + } + } + // _YIELD_VALUE + { + retval = val; + assert(frame->owner != FRAME_OWNED_BY_INTERPRETER); + frame->instr_ptr++; + PyGenObject *gen = _PyGen_GetGeneratorFromFrame(frame); + assert(FRAME_SUSPENDED_YIELD_FROM == FRAME_SUSPENDED + 1); + assert(oparg == 0 || oparg == 1); + gen->gi_frame_state = FRAME_SUSPENDED + oparg; + _PyStackRef temp = retval; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + tstate->exc_info = gen->gi_exc_state.previous_item; + gen->gi_exc_state.previous_item = NULL; + _Py_LeaveRecursiveCallPy(tstate); + _PyInterpreterFrame *gen_frame = frame; + frame = tstate->current_frame = frame->previous; + gen_frame->previous = NULL; + assert(INLINE_CACHE_ENTRIES_SEND == INLINE_CACHE_ENTRIES_FOR_ITER); + #if TIER_ONE + assert(frame->instr_ptr->op.code == INSTRUMENTED_LINE || + frame->instr_ptr->op.code == INSTRUMENTED_INSTRUCTION || + _PyOpcode_Deopt[frame->instr_ptr->op.code] == SEND || + _PyOpcode_Deopt[frame->instr_ptr->op.code] == FOR_ITER || + _PyOpcode_Deopt[frame->instr_ptr->op.code] == INTERPRETER_EXIT || + _PyOpcode_Deopt[frame->instr_ptr->op.code] == ENTER_EXECUTOR); + #endif + stack_pointer = _PyFrame_GetStackPointer(frame); + LOAD_IP(1 + INLINE_CACHE_ENTRIES_SEND); + #if TIER_TWO + frame->instr_ptr += (1 + INLINE_CACHE_ENTRIES_SEND); + #endif + value = PyStackRef_MakeHeapSafe(temp); + LLTRACE_RESUME_FRAME(); + } + stack_pointer[0] = value; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + TRACING_DISPATCH(); + } + + TRACING_TARGET(INTERPRETER_EXIT) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = INTERPRETER_EXIT; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(INTERPRETER_EXIT); + opcode = INTERPRETER_EXIT; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + _PyStackRef retval; + retval = stack_pointer[-1]; + assert(frame->owner == FRAME_OWNED_BY_INTERPRETER); + assert(_PyFrame_IsIncomplete(frame)); + tstate->current_frame = frame->previous; + assert(!_PyErr_Occurred(tstate)); + PyObject *result = PyStackRef_AsPyObjectSteal(retval); + #if !_Py_TAIL_CALL_INTERP + assert(frame == &entry.frame); + #endif + #ifdef _Py_TIER2 + _PyStackRef executor = frame->localsplus[0]; + assert(tstate->current_executor == NULL); + if (!PyStackRef_IsNull(executor)) { + tstate->current_executor = PyStackRef_AsPyObjectBorrow(executor); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(executor); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += 1; + } + #endif + LLTRACE_RESUME_FRAME(); + return result; + } + + TRACING_TARGET(IS_OP) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = IS_OP; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(IS_OP); + opcode = IS_OP; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + _PyStackRef left; + _PyStackRef right; + _PyStackRef b; + right = stack_pointer[-1]; + left = stack_pointer[-2]; + int res = Py_Is(PyStackRef_AsPyObjectBorrow(left), PyStackRef_AsPyObjectBorrow(right)) ^ oparg; + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyStackRef tmp = right; + right = PyStackRef_NULL; + stack_pointer[-1] = right; + PyStackRef_CLOSE(tmp); + tmp = left; + left = PyStackRef_NULL; + stack_pointer[-2] = left; + PyStackRef_CLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); + b = res ? PyStackRef_True : PyStackRef_False; + stack_pointer[0] = b; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + TRACING_DISPATCH(); + } + + TRACING_TARGET(JUMP_BACKWARD) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = JUMP_BACKWARD; + (void)(opcode); + #endif + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(JUMP_BACKWARD); + PREDICTED_TRACING_JUMP_BACKWARD:; + _Py_CODEUNIT* const this_instr = next_instr - 2; + (void)this_instr; + opcode = JUMP_BACKWARD; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + /* Skip 1 cache entry */ + // _SPECIALIZE_JUMP_BACKWARD + { + #if ENABLE_SPECIALIZATION + if (this_instr->op.code == JUMP_BACKWARD) { + this_instr->op.code = (tstate->interp->jit && !PyStackRef_IsNull(frame->f_funcobj)) ? JUMP_BACKWARD_JIT : JUMP_BACKWARD_NO_JIT; + next_instr = this_instr; + DISPATCH_SAME_OPARG(); + } + #endif + } + // _CHECK_PERIODIC + { + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = check_periodics(tstate); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err != 0) { + TRACING_JUMP_TO_LABEL(error); + } + } + // _JUMP_BACKWARD_NO_INTERRUPT + { + #if TIER_ONE + assert(oparg <= INSTR_OFFSET()); + #endif + JUMPBY(-oparg); + } + TRACING_DISPATCH(); + } + + TRACING_TARGET(JUMP_BACKWARD_JIT) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = JUMP_BACKWARD_JIT; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(JUMP_BACKWARD_JIT); + opcode = JUMP_BACKWARD_JIT; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + static_assert(1 == 1, "incorrect cache size"); + /* Skip 1 cache entry */ + // _CHECK_PERIODIC + { + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = check_periodics(tstate); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err != 0) { + TRACING_JUMP_TO_LABEL(error); + } + } + // _JUMP_BACKWARD_NO_INTERRUPT + { + #if TIER_ONE + assert(oparg <= INSTR_OFFSET()); + #endif + JUMPBY(-oparg); + } + // _JIT + { + #ifdef _Py_TIER2 + _Py_BackoffCounter counter = this_instr[1].counter; + if (!IS_JIT_TRACING() && backoff_counter_triggers(counter) && + this_instr->op.code == JUMP_BACKWARD_JIT && + next_instr->op.code != ENTER_EXECUTOR) { + if (tstate->interp->jit_tracer_code_buffer == NULL) { + _PyFrame_SetStackPointer(frame, stack_pointer); + tstate->interp->jit_tracer_code_buffer = (_PyUOpInstruction *)_PyObject_VirtualAlloc(UOP_BUFFER_SIZE); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (tstate->interp->jit_tracer_code_buffer == NULL) { + TRACING_DISPATCH(); + } + } + _PyJIT_InitializeTracing(tstate, frame, this_instr, STACK_LEVEL(), 0, NULL); + ENTER_TRACING(); + TRACING_DISPATCH(); + } + else { + ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); + } + #endif + } + TRACING_DISPATCH(); + } + + TRACING_TARGET(JUMP_BACKWARD_NO_INTERRUPT) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = JUMP_BACKWARD_NO_INTERRUPT; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(JUMP_BACKWARD_NO_INTERRUPT); + opcode = JUMP_BACKWARD_NO_INTERRUPT; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + #if TIER_ONE + assert(oparg <= INSTR_OFFSET()); + #endif + JUMPBY(-oparg); + TRACING_DISPATCH(); + } + + TRACING_TARGET(JUMP_BACKWARD_NO_JIT) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = JUMP_BACKWARD_NO_JIT; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(JUMP_BACKWARD_NO_JIT); + opcode = JUMP_BACKWARD_NO_JIT; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + static_assert(1 == 1, "incorrect cache size"); + /* Skip 1 cache entry */ + // _CHECK_PERIODIC + { + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = check_periodics(tstate); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err != 0) { + TRACING_JUMP_TO_LABEL(error); + } + } + // _JUMP_BACKWARD_NO_INTERRUPT + { + #if TIER_ONE + assert(oparg <= INSTR_OFFSET()); + #endif + JUMPBY(-oparg); + } + TRACING_DISPATCH(); + } + + TRACING_TARGET(JUMP_FORWARD) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = JUMP_FORWARD; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(JUMP_FORWARD); + opcode = JUMP_FORWARD; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + JUMPBY(oparg); + TRACING_DISPATCH(); + } + + TRACING_TARGET(LIST_APPEND) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = LIST_APPEND; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(LIST_APPEND); + opcode = LIST_APPEND; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + _PyStackRef list; + _PyStackRef v; + v = stack_pointer[-1]; + list = stack_pointer[-2 - (oparg-1)]; + int err = _PyList_AppendTakeRef((PyListObject *)PyStackRef_AsPyObjectBorrow(list), + PyStackRef_AsPyObjectSteal(v)); + if (err < 0) { + TRACING_JUMP_TO_LABEL(pop_1_error); + } + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + TRACING_DISPATCH(); + } + + TRACING_TARGET(LIST_EXTEND) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = LIST_EXTEND; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(LIST_EXTEND); + opcode = LIST_EXTEND; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + _PyStackRef list_st; + _PyStackRef iterable_st; + iterable_st = stack_pointer[-1]; + list_st = stack_pointer[-2 - (oparg-1)]; + PyObject *list = PyStackRef_AsPyObjectBorrow(list_st); + PyObject *iterable = PyStackRef_AsPyObjectBorrow(iterable_st); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *none_val = _PyList_Extend((PyListObject *)list, iterable); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (none_val == NULL) { + _PyFrame_SetStackPointer(frame, stack_pointer); + int matches = _PyErr_ExceptionMatches(tstate, PyExc_TypeError); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (matches && + (Py_TYPE(iterable)->tp_iter == NULL && !PySequence_Check(iterable))) + { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyErr_Clear(tstate); + _PyErr_Format(tstate, PyExc_TypeError, + "Value after * must be an iterable, not %.200s", + Py_TYPE(iterable)->tp_name); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(iterable_st); + stack_pointer = _PyFrame_GetStackPointer(frame); + TRACING_JUMP_TO_LABEL(error); + } + assert(Py_IsNone(none_val)); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(iterable_st); + stack_pointer = _PyFrame_GetStackPointer(frame); + TRACING_DISPATCH(); + } + + TRACING_TARGET(LOAD_ATTR) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = LOAD_ATTR; + (void)(opcode); + #endif + frame->instr_ptr = next_instr; + next_instr += 10; + INSTRUCTION_STATS(LOAD_ATTR); + PREDICTED_TRACING_LOAD_ATTR:; + _Py_CODEUNIT* const this_instr = next_instr - 10; + (void)this_instr; + opcode = LOAD_ATTR; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + _PyStackRef owner; + _PyStackRef *attr; + _PyStackRef *self_or_null; + // _SPECIALIZE_LOAD_ATTR + { + owner = stack_pointer[-1]; + uint16_t counter = read_u16(&this_instr[1].cache); + (void)counter; + #if ENABLE_SPECIALIZATION_FT + if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg>>1); + next_instr = this_instr; + _PyFrame_SetStackPointer(frame, stack_pointer); + _Py_Specialize_LoadAttr(owner, next_instr, name); + stack_pointer = _PyFrame_GetStackPointer(frame); + DISPATCH_SAME_OPARG(); + } + OPCODE_DEFERRED_INC(LOAD_ATTR); + ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); + #endif /* ENABLE_SPECIALIZATION_FT */ + } + /* Skip 8 cache entries */ + // _LOAD_ATTR + { + attr = &stack_pointer[-1]; + self_or_null = &stack_pointer[0]; + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 1); + if (oparg & 1) { + *attr = PyStackRef_NULL; + _PyFrame_SetStackPointer(frame, stack_pointer); + int is_meth = _PyObject_GetMethodStackRef(tstate, PyStackRef_AsPyObjectBorrow(owner), name, attr); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (is_meth) { + assert(!PyStackRef_IsNull(*attr)); + self_or_null[0] = owner; + } + else { + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(owner); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (PyStackRef_IsNull(*attr)) { + TRACING_JUMP_TO_LABEL(error); + } + self_or_null[0] = PyStackRef_NULL; + stack_pointer += 1; + } + } + else { + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *attr_o = PyObject_GetAttr(PyStackRef_AsPyObjectBorrow(owner), name); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(owner); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (attr_o == NULL) { + TRACING_JUMP_TO_LABEL(error); + } + *attr = PyStackRef_FromPyObjectSteal(attr_o); + stack_pointer += 1; + } + } + stack_pointer += (oparg&1); + assert(WITHIN_STACK_BOUNDS()); + TRACING_DISPATCH(); + } + + TRACING_TARGET(LOAD_ATTR_CLASS) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = LOAD_ATTR_CLASS; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 10; + INSTRUCTION_STATS(LOAD_ATTR_CLASS); + opcode = LOAD_ATTR_CLASS; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); + _PyStackRef owner; + _PyStackRef attr; + _PyStackRef *null; + /* Skip 1 cache entry */ + // _CHECK_ATTR_CLASS + { + owner = stack_pointer[-1]; + uint32_t type_version = read_u32(&this_instr[2].cache); + PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); + if (!PyType_Check(owner_o)) { + UPDATE_MISS_STATS(LOAD_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + JUMP_TO_PREDICTED(TRACING_LOAD_ATTR); + } + assert(type_version != 0); + if (FT_ATOMIC_LOAD_UINT_RELAXED(((PyTypeObject *)owner_o)->tp_version_tag) != type_version) { + UPDATE_MISS_STATS(LOAD_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + JUMP_TO_PREDICTED(TRACING_LOAD_ATTR); + } + } + /* Skip 2 cache entries */ + // _LOAD_ATTR_CLASS + { + PyObject *descr = read_obj(&this_instr[6].cache); + STAT_INC(LOAD_ATTR, hit); + assert(descr != NULL); + attr = PyStackRef_FromPyObjectNew(descr); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyStackRef tmp = owner; + owner = attr; + stack_pointer[-1] = owner; + PyStackRef_CLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + // _PUSH_NULL_CONDITIONAL + { + null = &stack_pointer[0]; + if (oparg & 1) { + null[0] = PyStackRef_NULL; + } + } + stack_pointer += (oparg & 1); + assert(WITHIN_STACK_BOUNDS()); + TRACING_DISPATCH(); + } + + TRACING_TARGET(LOAD_ATTR_CLASS_WITH_METACLASS_CHECK) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = LOAD_ATTR_CLASS_WITH_METACLASS_CHECK; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 10; + INSTRUCTION_STATS(LOAD_ATTR_CLASS_WITH_METACLASS_CHECK); + opcode = LOAD_ATTR_CLASS_WITH_METACLASS_CHECK; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); + _PyStackRef owner; + _PyStackRef attr; + _PyStackRef *null; + /* Skip 1 cache entry */ + // _CHECK_ATTR_CLASS + { + owner = stack_pointer[-1]; + uint32_t type_version = read_u32(&this_instr[2].cache); + PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); + if (!PyType_Check(owner_o)) { + UPDATE_MISS_STATS(LOAD_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + JUMP_TO_PREDICTED(TRACING_LOAD_ATTR); + } + assert(type_version != 0); + if (FT_ATOMIC_LOAD_UINT_RELAXED(((PyTypeObject *)owner_o)->tp_version_tag) != type_version) { + UPDATE_MISS_STATS(LOAD_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + JUMP_TO_PREDICTED(TRACING_LOAD_ATTR); + } + } + // _GUARD_TYPE_VERSION + { + uint32_t type_version = read_u32(&this_instr[4].cache); + PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); + assert(type_version != 0); + if (FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version) { + UPDATE_MISS_STATS(LOAD_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + JUMP_TO_PREDICTED(TRACING_LOAD_ATTR); + } + } + // _LOAD_ATTR_CLASS + { + PyObject *descr = read_obj(&this_instr[6].cache); + STAT_INC(LOAD_ATTR, hit); + assert(descr != NULL); + attr = PyStackRef_FromPyObjectNew(descr); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyStackRef tmp = owner; + owner = attr; + stack_pointer[-1] = owner; + PyStackRef_CLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + // _PUSH_NULL_CONDITIONAL + { + null = &stack_pointer[0]; + if (oparg & 1) { + null[0] = PyStackRef_NULL; + } + } + stack_pointer += (oparg & 1); + assert(WITHIN_STACK_BOUNDS()); + TRACING_DISPATCH(); + } + + TRACING_TARGET(LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 10; + INSTRUCTION_STATS(LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN); + opcode = LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); + _PyStackRef owner; + /* Skip 1 cache entry */ + owner = stack_pointer[-1]; + uint32_t type_version = read_u32(&this_instr[2].cache); + uint32_t func_version = read_u32(&this_instr[4].cache); + PyObject *getattribute = read_obj(&this_instr[6].cache); + PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); + assert((oparg & 1) == 0); + if (tstate->interp->eval_frame) { + UPDATE_MISS_STATS(LOAD_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + JUMP_TO_PREDICTED(TRACING_LOAD_ATTR); + } + PyTypeObject *cls = Py_TYPE(owner_o); + assert(type_version != 0); + if (FT_ATOMIC_LOAD_UINT_RELAXED(cls->tp_version_tag) != type_version) { + UPDATE_MISS_STATS(LOAD_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + JUMP_TO_PREDICTED(TRACING_LOAD_ATTR); + } + assert(Py_IS_TYPE(getattribute, &PyFunction_Type)); + PyFunctionObject *f = (PyFunctionObject *)getattribute; + assert(func_version != 0); + if (f->func_version != func_version) { + UPDATE_MISS_STATS(LOAD_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + JUMP_TO_PREDICTED(TRACING_LOAD_ATTR); + } + PyCodeObject *code = (PyCodeObject *)f->func_code; + assert(code->co_argcount == 2); + if (!_PyThreadState_HasStackSpace(tstate, code->co_framesize)) { + UPDATE_MISS_STATS(LOAD_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + JUMP_TO_PREDICTED(TRACING_LOAD_ATTR); + } + STAT_INC(LOAD_ATTR, hit); + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 1); + _PyInterpreterFrame *new_frame = _PyFrame_PushUnchecked( + tstate, PyStackRef_FromPyObjectNew(f), 2, frame); + new_frame->localsplus[0] = owner; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + new_frame->localsplus[1] = PyStackRef_FromPyObjectNew(name); + frame->return_offset = 10u ; + TRACING_DISPATCH_INLINED(new_frame); + } + + TRACING_TARGET(LOAD_ATTR_INSTANCE_VALUE) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = LOAD_ATTR_INSTANCE_VALUE; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 10; + INSTRUCTION_STATS(LOAD_ATTR_INSTANCE_VALUE); + opcode = LOAD_ATTR_INSTANCE_VALUE; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); + _PyStackRef owner; + _PyStackRef attr; + _PyStackRef *null; + /* Skip 1 cache entry */ + // _GUARD_TYPE_VERSION + { + owner = stack_pointer[-1]; + uint32_t type_version = read_u32(&this_instr[2].cache); + PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); + assert(type_version != 0); + if (FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version) { + UPDATE_MISS_STATS(LOAD_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + JUMP_TO_PREDICTED(TRACING_LOAD_ATTR); + } + } + // _CHECK_MANAGED_OBJECT_HAS_VALUES + { + PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); + assert(Py_TYPE(owner_o)->tp_dictoffset < 0); + assert(Py_TYPE(owner_o)->tp_flags & Py_TPFLAGS_INLINE_VALUES); + if (!FT_ATOMIC_LOAD_UINT8(_PyObject_InlineValues(owner_o)->valid)) { + UPDATE_MISS_STATS(LOAD_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + JUMP_TO_PREDICTED(TRACING_LOAD_ATTR); + } + } + // _LOAD_ATTR_INSTANCE_VALUE + { + uint16_t offset = read_u16(&this_instr[4].cache); + PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); + PyObject **value_ptr = (PyObject**)(((char *)owner_o) + offset); + PyObject *attr_o = FT_ATOMIC_LOAD_PTR_ACQUIRE(*value_ptr); + if (attr_o == NULL) { + UPDATE_MISS_STATS(LOAD_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + JUMP_TO_PREDICTED(TRACING_LOAD_ATTR); + } + #ifdef Py_GIL_DISABLED + int increfed = _Py_TryIncrefCompareStackRef(value_ptr, attr_o, &attr); + if (!increfed) { + if (true) { + UPDATE_MISS_STATS(LOAD_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + JUMP_TO_PREDICTED(TRACING_LOAD_ATTR); + } + } + #else + attr = PyStackRef_FromPyObjectNew(attr_o); + #endif + STAT_INC(LOAD_ATTR, hit); + stack_pointer[-1] = attr; + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(owner); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + /* Skip 5 cache entries */ + // _PUSH_NULL_CONDITIONAL + { + null = &stack_pointer[0]; + if (oparg & 1) { + null[0] = PyStackRef_NULL; + } + } + stack_pointer += (oparg & 1); + assert(WITHIN_STACK_BOUNDS()); + TRACING_DISPATCH(); + } + + TRACING_TARGET(LOAD_ATTR_METHOD_LAZY_DICT) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = LOAD_ATTR_METHOD_LAZY_DICT; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 10; + INSTRUCTION_STATS(LOAD_ATTR_METHOD_LAZY_DICT); + opcode = LOAD_ATTR_METHOD_LAZY_DICT; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); + _PyStackRef owner; + _PyStackRef attr; + _PyStackRef self; + /* Skip 1 cache entry */ + // _GUARD_TYPE_VERSION + { + owner = stack_pointer[-1]; + uint32_t type_version = read_u32(&this_instr[2].cache); + PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); + assert(type_version != 0); + if (FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version) { + UPDATE_MISS_STATS(LOAD_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + JUMP_TO_PREDICTED(TRACING_LOAD_ATTR); + } + } + // _CHECK_ATTR_METHOD_LAZY_DICT + { + uint16_t dictoffset = read_u16(&this_instr[4].cache); + char *ptr = ((char *)PyStackRef_AsPyObjectBorrow(owner)) + MANAGED_DICT_OFFSET + dictoffset; + PyObject *dict = FT_ATOMIC_LOAD_PTR_ACQUIRE(*(PyObject **)ptr); + if (dict != NULL) { + UPDATE_MISS_STATS(LOAD_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + JUMP_TO_PREDICTED(TRACING_LOAD_ATTR); + } + } + /* Skip 1 cache entry */ + // _LOAD_ATTR_METHOD_LAZY_DICT + { + PyObject *descr = read_obj(&this_instr[6].cache); + assert(oparg & 1); + STAT_INC(LOAD_ATTR, hit); + assert(descr != NULL); + assert(_PyType_HasFeature(Py_TYPE(descr), Py_TPFLAGS_METHOD_DESCRIPTOR)); + attr = PyStackRef_FromPyObjectNew(descr); + self = owner; + } + stack_pointer[-1] = attr; + stack_pointer[0] = self; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + TRACING_DISPATCH(); + } + + TRACING_TARGET(LOAD_ATTR_METHOD_NO_DICT) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = LOAD_ATTR_METHOD_NO_DICT; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 10; + INSTRUCTION_STATS(LOAD_ATTR_METHOD_NO_DICT); + opcode = LOAD_ATTR_METHOD_NO_DICT; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); + _PyStackRef owner; + _PyStackRef attr; + _PyStackRef self; + /* Skip 1 cache entry */ + // _GUARD_TYPE_VERSION + { + owner = stack_pointer[-1]; + uint32_t type_version = read_u32(&this_instr[2].cache); + PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); + assert(type_version != 0); + if (FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version) { + UPDATE_MISS_STATS(LOAD_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + JUMP_TO_PREDICTED(TRACING_LOAD_ATTR); + } + } + /* Skip 2 cache entries */ + // _LOAD_ATTR_METHOD_NO_DICT + { + PyObject *descr = read_obj(&this_instr[6].cache); + assert(oparg & 1); + assert(Py_TYPE(PyStackRef_AsPyObjectBorrow(owner))->tp_dictoffset == 0); + STAT_INC(LOAD_ATTR, hit); + assert(descr != NULL); + assert(_PyType_HasFeature(Py_TYPE(descr), Py_TPFLAGS_METHOD_DESCRIPTOR)); + attr = PyStackRef_FromPyObjectNew(descr); + self = owner; + } + stack_pointer[-1] = attr; + stack_pointer[0] = self; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + TRACING_DISPATCH(); + } + + TRACING_TARGET(LOAD_ATTR_METHOD_WITH_VALUES) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = LOAD_ATTR_METHOD_WITH_VALUES; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 10; + INSTRUCTION_STATS(LOAD_ATTR_METHOD_WITH_VALUES); + opcode = LOAD_ATTR_METHOD_WITH_VALUES; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); + _PyStackRef owner; + _PyStackRef attr; + _PyStackRef self; + /* Skip 1 cache entry */ + // _GUARD_TYPE_VERSION + { + owner = stack_pointer[-1]; + uint32_t type_version = read_u32(&this_instr[2].cache); + PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); + assert(type_version != 0); + if (FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version) { + UPDATE_MISS_STATS(LOAD_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + JUMP_TO_PREDICTED(TRACING_LOAD_ATTR); + } + } + // _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT + { + PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); + assert(Py_TYPE(owner_o)->tp_flags & Py_TPFLAGS_INLINE_VALUES); + PyDictValues *ivs = _PyObject_InlineValues(owner_o); + if (!FT_ATOMIC_LOAD_UINT8(ivs->valid)) { + UPDATE_MISS_STATS(LOAD_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + JUMP_TO_PREDICTED(TRACING_LOAD_ATTR); + } + } + // _GUARD_KEYS_VERSION + { + uint32_t keys_version = read_u32(&this_instr[4].cache); + PyTypeObject *owner_cls = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); + PyHeapTypeObject *owner_heap_type = (PyHeapTypeObject *)owner_cls; + PyDictKeysObject *keys = owner_heap_type->ht_cached_keys; + if (FT_ATOMIC_LOAD_UINT32_RELAXED(keys->dk_version) != keys_version) { + UPDATE_MISS_STATS(LOAD_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + JUMP_TO_PREDICTED(TRACING_LOAD_ATTR); + } + } + // _LOAD_ATTR_METHOD_WITH_VALUES + { + PyObject *descr = read_obj(&this_instr[6].cache); + assert(oparg & 1); + STAT_INC(LOAD_ATTR, hit); + assert(descr != NULL); + assert(_PyType_HasFeature(Py_TYPE(descr), Py_TPFLAGS_METHOD_DESCRIPTOR)); + attr = PyStackRef_FromPyObjectNew(descr); + self = owner; + } + stack_pointer[-1] = attr; + stack_pointer[0] = self; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + TRACING_DISPATCH(); + } + + TRACING_TARGET(LOAD_ATTR_MODULE) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = LOAD_ATTR_MODULE; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 10; + INSTRUCTION_STATS(LOAD_ATTR_MODULE); + opcode = LOAD_ATTR_MODULE; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); + _PyStackRef owner; + _PyStackRef attr; + _PyStackRef *null; + /* Skip 1 cache entry */ + // _LOAD_ATTR_MODULE + { + owner = stack_pointer[-1]; + uint32_t dict_version = read_u32(&this_instr[2].cache); + uint16_t index = read_u16(&this_instr[4].cache); + PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); + if (Py_TYPE(owner_o)->tp_getattro != PyModule_Type.tp_getattro) { + UPDATE_MISS_STATS(LOAD_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + JUMP_TO_PREDICTED(TRACING_LOAD_ATTR); + } + PyDictObject *dict = (PyDictObject *)((PyModuleObject *)owner_o)->md_dict; + assert(dict != NULL); + PyDictKeysObject *keys = FT_ATOMIC_LOAD_PTR_ACQUIRE(dict->ma_keys); + if (FT_ATOMIC_LOAD_UINT32_RELAXED(keys->dk_version) != dict_version) { + UPDATE_MISS_STATS(LOAD_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + JUMP_TO_PREDICTED(TRACING_LOAD_ATTR); + } + assert(keys->dk_kind == DICT_KEYS_UNICODE); + assert(index < FT_ATOMIC_LOAD_SSIZE_RELAXED(keys->dk_nentries)); + PyDictUnicodeEntry *ep = DK_UNICODE_ENTRIES(keys) + index; + PyObject *attr_o = FT_ATOMIC_LOAD_PTR_RELAXED(ep->me_value); + if (attr_o == NULL) { + UPDATE_MISS_STATS(LOAD_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + JUMP_TO_PREDICTED(TRACING_LOAD_ATTR); + } + #ifdef Py_GIL_DISABLED + int increfed = _Py_TryIncrefCompareStackRef(&ep->me_value, attr_o, &attr); + if (!increfed) { + if (true) { + UPDATE_MISS_STATS(LOAD_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + JUMP_TO_PREDICTED(TRACING_LOAD_ATTR); + } + } + #else + attr = PyStackRef_FromPyObjectNew(attr_o); + #endif + STAT_INC(LOAD_ATTR, hit); + stack_pointer[-1] = attr; + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(owner); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + /* Skip 5 cache entries */ + // _PUSH_NULL_CONDITIONAL + { + null = &stack_pointer[0]; + if (oparg & 1) { + null[0] = PyStackRef_NULL; + } + } + stack_pointer += (oparg & 1); + assert(WITHIN_STACK_BOUNDS()); + TRACING_DISPATCH(); + } + + TRACING_TARGET(LOAD_ATTR_NONDESCRIPTOR_NO_DICT) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = LOAD_ATTR_NONDESCRIPTOR_NO_DICT; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 10; + INSTRUCTION_STATS(LOAD_ATTR_NONDESCRIPTOR_NO_DICT); + opcode = LOAD_ATTR_NONDESCRIPTOR_NO_DICT; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); + _PyStackRef owner; + _PyStackRef attr; + /* Skip 1 cache entry */ + // _GUARD_TYPE_VERSION + { + owner = stack_pointer[-1]; + uint32_t type_version = read_u32(&this_instr[2].cache); + PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); + assert(type_version != 0); + if (FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version) { + UPDATE_MISS_STATS(LOAD_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + JUMP_TO_PREDICTED(TRACING_LOAD_ATTR); + } + } + /* Skip 2 cache entries */ + // _LOAD_ATTR_NONDESCRIPTOR_NO_DICT + { + PyObject *descr = read_obj(&this_instr[6].cache); + assert((oparg & 1) == 0); + assert(Py_TYPE(PyStackRef_AsPyObjectBorrow(owner))->tp_dictoffset == 0); + STAT_INC(LOAD_ATTR, hit); + assert(descr != NULL); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(owner); + stack_pointer = _PyFrame_GetStackPointer(frame); + attr = PyStackRef_FromPyObjectNew(descr); + } + stack_pointer[0] = attr; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + TRACING_DISPATCH(); + } + + TRACING_TARGET(LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 10; + INSTRUCTION_STATS(LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES); + opcode = LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); + _PyStackRef owner; + _PyStackRef attr; + /* Skip 1 cache entry */ + // _GUARD_TYPE_VERSION + { + owner = stack_pointer[-1]; + uint32_t type_version = read_u32(&this_instr[2].cache); + PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); + assert(type_version != 0); + if (FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version) { + UPDATE_MISS_STATS(LOAD_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + JUMP_TO_PREDICTED(TRACING_LOAD_ATTR); + } + } + // _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT + { + PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); + assert(Py_TYPE(owner_o)->tp_flags & Py_TPFLAGS_INLINE_VALUES); + PyDictValues *ivs = _PyObject_InlineValues(owner_o); + if (!FT_ATOMIC_LOAD_UINT8(ivs->valid)) { + UPDATE_MISS_STATS(LOAD_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + JUMP_TO_PREDICTED(TRACING_LOAD_ATTR); + } + } + // _GUARD_KEYS_VERSION + { + uint32_t keys_version = read_u32(&this_instr[4].cache); + PyTypeObject *owner_cls = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); + PyHeapTypeObject *owner_heap_type = (PyHeapTypeObject *)owner_cls; + PyDictKeysObject *keys = owner_heap_type->ht_cached_keys; + if (FT_ATOMIC_LOAD_UINT32_RELAXED(keys->dk_version) != keys_version) { + UPDATE_MISS_STATS(LOAD_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + JUMP_TO_PREDICTED(TRACING_LOAD_ATTR); + } + } + // _LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES + { + PyObject *descr = read_obj(&this_instr[6].cache); + assert((oparg & 1) == 0); + STAT_INC(LOAD_ATTR, hit); + assert(descr != NULL); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(owner); + stack_pointer = _PyFrame_GetStackPointer(frame); + attr = PyStackRef_FromPyObjectNew(descr); + } + stack_pointer[0] = attr; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + TRACING_DISPATCH(); + } + + TRACING_TARGET(LOAD_ATTR_PROPERTY) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = LOAD_ATTR_PROPERTY; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 10; + INSTRUCTION_STATS(LOAD_ATTR_PROPERTY); + opcode = LOAD_ATTR_PROPERTY; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); + _PyStackRef owner; + _PyStackRef new_frame; + /* Skip 1 cache entry */ + // _CHECK_PEP_523 + { + if (tstate->interp->eval_frame) { + UPDATE_MISS_STATS(LOAD_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + JUMP_TO_PREDICTED(TRACING_LOAD_ATTR); + } + } + // _GUARD_TYPE_VERSION + { + owner = stack_pointer[-1]; + uint32_t type_version = read_u32(&this_instr[2].cache); + PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); + assert(type_version != 0); + if (FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version) { + UPDATE_MISS_STATS(LOAD_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + JUMP_TO_PREDICTED(TRACING_LOAD_ATTR); + } + } + /* Skip 2 cache entries */ + // _LOAD_ATTR_PROPERTY_FRAME + { + PyObject *fget = read_obj(&this_instr[6].cache); + assert((oparg & 1) == 0); + assert(Py_IS_TYPE(fget, &PyFunction_Type)); + PyFunctionObject *f = (PyFunctionObject *)fget; + PyCodeObject *code = (PyCodeObject *)f->func_code; + if ((code->co_flags & (CO_VARKEYWORDS | CO_VARARGS | CO_OPTIMIZED)) != CO_OPTIMIZED) { + UPDATE_MISS_STATS(LOAD_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + JUMP_TO_PREDICTED(TRACING_LOAD_ATTR); + } + if (code->co_kwonlyargcount) { + UPDATE_MISS_STATS(LOAD_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + JUMP_TO_PREDICTED(TRACING_LOAD_ATTR); + } + if (code->co_argcount != 1) { + UPDATE_MISS_STATS(LOAD_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + JUMP_TO_PREDICTED(TRACING_LOAD_ATTR); + } + if (!_PyThreadState_HasStackSpace(tstate, code->co_framesize)) { + UPDATE_MISS_STATS(LOAD_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + JUMP_TO_PREDICTED(TRACING_LOAD_ATTR); + } + STAT_INC(LOAD_ATTR, hit); + _PyInterpreterFrame *pushed_frame = _PyFrame_PushUnchecked(tstate, PyStackRef_FromPyObjectNew(fget), 1, frame); + pushed_frame->localsplus[0] = owner; + new_frame = PyStackRef_Wrap(pushed_frame); + } + // _SAVE_RETURN_OFFSET + { + #if TIER_ONE + frame->return_offset = (uint16_t)(next_instr - this_instr); + #endif + #if TIER_TWO + frame->return_offset = oparg; + #endif + } + // _PUSH_FRAME + { + assert(tstate->interp->eval_frame == NULL); + _PyInterpreterFrame *temp = PyStackRef_Unwrap(new_frame); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + assert(temp->previous == frame || temp->previous->previous == frame); + CALL_STAT_INC(inlined_py_calls); + frame = tstate->current_frame = temp; + tstate->py_recursion_remaining--; + LOAD_SP(); + LOAD_IP(0); + LLTRACE_RESUME_FRAME(); + } + TRACING_DISPATCH(); + } + + TRACING_TARGET(LOAD_ATTR_SLOT) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = LOAD_ATTR_SLOT; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 10; + INSTRUCTION_STATS(LOAD_ATTR_SLOT); + opcode = LOAD_ATTR_SLOT; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); + _PyStackRef owner; + _PyStackRef attr; + _PyStackRef *null; + /* Skip 1 cache entry */ + // _GUARD_TYPE_VERSION + { + owner = stack_pointer[-1]; + uint32_t type_version = read_u32(&this_instr[2].cache); + PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); + assert(type_version != 0); + if (FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version) { + UPDATE_MISS_STATS(LOAD_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + JUMP_TO_PREDICTED(TRACING_LOAD_ATTR); + } + } + // _LOAD_ATTR_SLOT + { + uint16_t index = read_u16(&this_instr[4].cache); + PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); + PyObject **addr = (PyObject **)((char *)owner_o + index); + PyObject *attr_o = FT_ATOMIC_LOAD_PTR(*addr); + if (attr_o == NULL) { + UPDATE_MISS_STATS(LOAD_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + JUMP_TO_PREDICTED(TRACING_LOAD_ATTR); + } + #ifdef Py_GIL_DISABLED + int increfed = _Py_TryIncrefCompareStackRef(addr, attr_o, &attr); + if (!increfed) { + UPDATE_MISS_STATS(LOAD_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + JUMP_TO_PREDICTED(TRACING_LOAD_ATTR); + } + #else + attr = PyStackRef_FromPyObjectNew(attr_o); + #endif + STAT_INC(LOAD_ATTR, hit); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyStackRef tmp = owner; + owner = attr; + stack_pointer[-1] = owner; + PyStackRef_CLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + /* Skip 5 cache entries */ + // _PUSH_NULL_CONDITIONAL + { + null = &stack_pointer[0]; + if (oparg & 1) { + null[0] = PyStackRef_NULL; + } + } + stack_pointer += (oparg & 1); + assert(WITHIN_STACK_BOUNDS()); + TRACING_DISPATCH(); + } + + TRACING_TARGET(LOAD_ATTR_WITH_HINT) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = LOAD_ATTR_WITH_HINT; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 10; + INSTRUCTION_STATS(LOAD_ATTR_WITH_HINT); + opcode = LOAD_ATTR_WITH_HINT; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); + _PyStackRef owner; + _PyStackRef attr; + _PyStackRef *null; + /* Skip 1 cache entry */ + // _GUARD_TYPE_VERSION + { + owner = stack_pointer[-1]; + uint32_t type_version = read_u32(&this_instr[2].cache); + PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); + assert(type_version != 0); + if (FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version) { + UPDATE_MISS_STATS(LOAD_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + JUMP_TO_PREDICTED(TRACING_LOAD_ATTR); + } + } + // _LOAD_ATTR_WITH_HINT + { + uint16_t hint = read_u16(&this_instr[4].cache); + PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); + assert(Py_TYPE(owner_o)->tp_flags & Py_TPFLAGS_MANAGED_DICT); + PyDictObject *dict = _PyObject_GetManagedDict(owner_o); + if (dict == NULL) { + UPDATE_MISS_STATS(LOAD_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + JUMP_TO_PREDICTED(TRACING_LOAD_ATTR); + } + PyDictKeysObject *dk = FT_ATOMIC_LOAD_PTR(dict->ma_keys); + assert(PyDict_CheckExact((PyObject *)dict)); + #ifdef Py_GIL_DISABLED + if (!_Py_IsOwnedByCurrentThread((PyObject *)dict) && !_PyObject_GC_IS_SHARED(dict)) { + UPDATE_MISS_STATS(LOAD_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + JUMP_TO_PREDICTED(TRACING_LOAD_ATTR); + } + #endif + PyObject *attr_o; + if (hint >= (size_t)FT_ATOMIC_LOAD_SSIZE_RELAXED(dk->dk_nentries)) { + if (true) { + UPDATE_MISS_STATS(LOAD_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + JUMP_TO_PREDICTED(TRACING_LOAD_ATTR); + } + } + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg>>1); + if (dk->dk_kind != DICT_KEYS_UNICODE) { + if (true) { + UPDATE_MISS_STATS(LOAD_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + JUMP_TO_PREDICTED(TRACING_LOAD_ATTR); + } + } + PyDictUnicodeEntry *ep = DK_UNICODE_ENTRIES(dk) + hint; + if (FT_ATOMIC_LOAD_PTR_RELAXED(ep->me_key) != name) { + if (true) { + UPDATE_MISS_STATS(LOAD_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + JUMP_TO_PREDICTED(TRACING_LOAD_ATTR); + } + } + attr_o = FT_ATOMIC_LOAD_PTR(ep->me_value); + if (attr_o == NULL) { + if (true) { + UPDATE_MISS_STATS(LOAD_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + JUMP_TO_PREDICTED(TRACING_LOAD_ATTR); + } + } + STAT_INC(LOAD_ATTR, hit); + #ifdef Py_GIL_DISABLED + int increfed = _Py_TryIncrefCompareStackRef(&ep->me_value, attr_o, &attr); + if (!increfed) { + if (true) { + UPDATE_MISS_STATS(LOAD_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + JUMP_TO_PREDICTED(TRACING_LOAD_ATTR); + } + } + #else + attr = PyStackRef_FromPyObjectNew(attr_o); + #endif + stack_pointer[-1] = attr; + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(owner); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + /* Skip 5 cache entries */ + // _PUSH_NULL_CONDITIONAL + { + null = &stack_pointer[0]; + if (oparg & 1) { + null[0] = PyStackRef_NULL; + } + } + stack_pointer += (oparg & 1); + assert(WITHIN_STACK_BOUNDS()); + TRACING_DISPATCH(); + } + + TRACING_TARGET(LOAD_BUILD_CLASS) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = LOAD_BUILD_CLASS; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(LOAD_BUILD_CLASS); + opcode = LOAD_BUILD_CLASS; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + _PyStackRef bc; + PyObject *bc_o; + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = PyMapping_GetOptionalItem(BUILTINS(), &_Py_ID(__build_class__), &bc_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err < 0) { + TRACING_JUMP_TO_LABEL(error); + } + if (bc_o == NULL) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyErr_SetString(tstate, PyExc_NameError, + "__build_class__ not found"); + stack_pointer = _PyFrame_GetStackPointer(frame); + TRACING_JUMP_TO_LABEL(error); + } + bc = PyStackRef_FromPyObjectSteal(bc_o); + stack_pointer[0] = bc; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + TRACING_DISPATCH(); + } + + TRACING_TARGET(LOAD_COMMON_CONSTANT) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = LOAD_COMMON_CONSTANT; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(LOAD_COMMON_CONSTANT); + opcode = LOAD_COMMON_CONSTANT; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + _PyStackRef value; + assert(oparg < NUM_COMMON_CONSTANTS); + value = PyStackRef_FromPyObjectNew(tstate->interp->common_consts[oparg]); + stack_pointer[0] = value; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + TRACING_DISPATCH(); + } + + TRACING_TARGET(LOAD_CONST) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = LOAD_CONST; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(LOAD_CONST); + opcode = LOAD_CONST; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + _PyStackRef value; + PyObject *obj = GETITEM(FRAME_CO_CONSTS, oparg); + value = PyStackRef_FromPyObjectBorrow(obj); + stack_pointer[0] = value; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + TRACING_DISPATCH(); + } + + TRACING_TARGET(LOAD_DEREF) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = LOAD_DEREF; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(LOAD_DEREF); + opcode = LOAD_DEREF; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + _PyStackRef value; + PyCellObject *cell = (PyCellObject *)PyStackRef_AsPyObjectBorrow(GETLOCAL(oparg)); + _PyFrame_SetStackPointer(frame, stack_pointer); + value = _PyCell_GetStackRef(cell); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (PyStackRef_IsNull(value)) { + stack_pointer[0] = value; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyEval_FormatExcUnbound(tstate, _PyFrame_GetCode(frame), oparg); + stack_pointer = _PyFrame_GetStackPointer(frame); + TRACING_JUMP_TO_LABEL(error); + } + stack_pointer[0] = value; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + TRACING_DISPATCH(); + } + + TRACING_TARGET(LOAD_FAST) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = LOAD_FAST; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(LOAD_FAST); + opcode = LOAD_FAST; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + _PyStackRef value; + assert(!PyStackRef_IsNull(GETLOCAL(oparg))); + value = PyStackRef_DUP(GETLOCAL(oparg)); + stack_pointer[0] = value; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + TRACING_DISPATCH(); + } + + TRACING_TARGET(LOAD_FAST_AND_CLEAR) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = LOAD_FAST_AND_CLEAR; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(LOAD_FAST_AND_CLEAR); + opcode = LOAD_FAST_AND_CLEAR; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + _PyStackRef value; + value = GETLOCAL(oparg); + GETLOCAL(oparg) = PyStackRef_NULL; + stack_pointer[0] = value; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + TRACING_DISPATCH(); + } + + TRACING_TARGET(LOAD_FAST_BORROW) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = LOAD_FAST_BORROW; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(LOAD_FAST_BORROW); + opcode = LOAD_FAST_BORROW; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + _PyStackRef value; + assert(!PyStackRef_IsNull(GETLOCAL(oparg))); + value = PyStackRef_Borrow(GETLOCAL(oparg)); + stack_pointer[0] = value; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + TRACING_DISPATCH(); + } + + TRACING_TARGET(LOAD_FAST_BORROW_LOAD_FAST_BORROW) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = LOAD_FAST_BORROW_LOAD_FAST_BORROW; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(LOAD_FAST_BORROW_LOAD_FAST_BORROW); + opcode = LOAD_FAST_BORROW_LOAD_FAST_BORROW; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + _PyStackRef value1; + _PyStackRef value2; + uint32_t oparg1 = oparg >> 4; + uint32_t oparg2 = oparg & 15; + value1 = PyStackRef_Borrow(GETLOCAL(oparg1)); + value2 = PyStackRef_Borrow(GETLOCAL(oparg2)); + stack_pointer[0] = value1; + stack_pointer[1] = value2; + stack_pointer += 2; + assert(WITHIN_STACK_BOUNDS()); + TRACING_DISPATCH(); + } + + TRACING_TARGET(LOAD_FAST_CHECK) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = LOAD_FAST_CHECK; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(LOAD_FAST_CHECK); + opcode = LOAD_FAST_CHECK; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + _PyStackRef value; + _PyStackRef value_s = GETLOCAL(oparg); + if (PyStackRef_IsNull(value_s)) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyEval_FormatExcCheckArg(tstate, PyExc_UnboundLocalError, + UNBOUNDLOCAL_ERROR_MSG, + PyTuple_GetItem(_PyFrame_GetCode(frame)->co_localsplusnames, oparg) + ); + stack_pointer = _PyFrame_GetStackPointer(frame); + TRACING_JUMP_TO_LABEL(error); + } + value = PyStackRef_DUP(value_s); + stack_pointer[0] = value; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + TRACING_DISPATCH(); + } + + TRACING_TARGET(LOAD_FAST_LOAD_FAST) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = LOAD_FAST_LOAD_FAST; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(LOAD_FAST_LOAD_FAST); + opcode = LOAD_FAST_LOAD_FAST; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + _PyStackRef value1; + _PyStackRef value2; + uint32_t oparg1 = oparg >> 4; + uint32_t oparg2 = oparg & 15; + value1 = PyStackRef_DUP(GETLOCAL(oparg1)); + value2 = PyStackRef_DUP(GETLOCAL(oparg2)); + stack_pointer[0] = value1; + stack_pointer[1] = value2; + stack_pointer += 2; + assert(WITHIN_STACK_BOUNDS()); + TRACING_DISPATCH(); + } + + TRACING_TARGET(LOAD_FROM_DICT_OR_DEREF) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = LOAD_FROM_DICT_OR_DEREF; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(LOAD_FROM_DICT_OR_DEREF); + opcode = LOAD_FROM_DICT_OR_DEREF; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + _PyStackRef class_dict_st; + _PyStackRef value; + class_dict_st = stack_pointer[-1]; + PyObject *value_o; + PyObject *name; + PyObject *class_dict = PyStackRef_AsPyObjectBorrow(class_dict_st); + assert(class_dict); + assert(oparg >= 0 && oparg < _PyFrame_GetCode(frame)->co_nlocalsplus); + name = PyTuple_GET_ITEM(_PyFrame_GetCode(frame)->co_localsplusnames, oparg); + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = PyMapping_GetOptionalItem(class_dict, name, &value_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err < 0) { + TRACING_JUMP_TO_LABEL(error); + } + if (!value_o) { + PyCellObject *cell = (PyCellObject *)PyStackRef_AsPyObjectBorrow(GETLOCAL(oparg)); + value_o = PyCell_GetRef(cell); + if (value_o == NULL) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyEval_FormatExcUnbound(tstate, _PyFrame_GetCode(frame), oparg); + stack_pointer = _PyFrame_GetStackPointer(frame); + TRACING_JUMP_TO_LABEL(error); + } + } + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(class_dict_st); + stack_pointer = _PyFrame_GetStackPointer(frame); + value = PyStackRef_FromPyObjectSteal(value_o); + stack_pointer[0] = value; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + TRACING_DISPATCH(); + } + + TRACING_TARGET(LOAD_FROM_DICT_OR_GLOBALS) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = LOAD_FROM_DICT_OR_GLOBALS; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(LOAD_FROM_DICT_OR_GLOBALS); + opcode = LOAD_FROM_DICT_OR_GLOBALS; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + _PyStackRef mod_or_class_dict; + _PyStackRef v; + mod_or_class_dict = stack_pointer[-1]; + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); + PyObject *v_o; + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = PyMapping_GetOptionalItem(PyStackRef_AsPyObjectBorrow(mod_or_class_dict), name, &v_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(mod_or_class_dict); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err < 0) { + TRACING_JUMP_TO_LABEL(error); + } + if (v_o == NULL) { + if (PyDict_CheckExact(GLOBALS()) + && PyDict_CheckExact(BUILTINS())) + { + _PyFrame_SetStackPointer(frame, stack_pointer); + v_o = _PyDict_LoadGlobal((PyDictObject *)GLOBALS(), + (PyDictObject *)BUILTINS(), + name); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (v_o == NULL) { + if (!_PyErr_Occurred(tstate)) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyEval_FormatExcCheckArg(tstate, PyExc_NameError, + NAME_ERROR_MSG, name); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + TRACING_JUMP_TO_LABEL(error); + } + } + else { + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = PyMapping_GetOptionalItem(GLOBALS(), name, &v_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err < 0) { + TRACING_JUMP_TO_LABEL(error); + } + if (v_o == NULL) { + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = PyMapping_GetOptionalItem(BUILTINS(), name, &v_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err < 0) { + TRACING_JUMP_TO_LABEL(error); + } + if (v_o == NULL) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyEval_FormatExcCheckArg( + tstate, PyExc_NameError, + NAME_ERROR_MSG, name); + stack_pointer = _PyFrame_GetStackPointer(frame); + TRACING_JUMP_TO_LABEL(error); + } + } + } + } + v = PyStackRef_FromPyObjectSteal(v_o); + stack_pointer[0] = v; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + TRACING_DISPATCH(); + } + + TRACING_TARGET(LOAD_GLOBAL) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = LOAD_GLOBAL; + (void)(opcode); + #endif + frame->instr_ptr = next_instr; + next_instr += 5; + INSTRUCTION_STATS(LOAD_GLOBAL); + PREDICTED_TRACING_LOAD_GLOBAL:; + _Py_CODEUNIT* const this_instr = next_instr - 5; + (void)this_instr; + opcode = LOAD_GLOBAL; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + _PyStackRef *res; + _PyStackRef *null; + // _SPECIALIZE_LOAD_GLOBAL + { + uint16_t counter = read_u16(&this_instr[1].cache); + (void)counter; + #if ENABLE_SPECIALIZATION_FT + if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg>>1); + next_instr = this_instr; + _PyFrame_SetStackPointer(frame, stack_pointer); + _Py_Specialize_LoadGlobal(GLOBALS(), BUILTINS(), next_instr, name); + stack_pointer = _PyFrame_GetStackPointer(frame); + DISPATCH_SAME_OPARG(); + } + OPCODE_DEFERRED_INC(LOAD_GLOBAL); + ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); + #endif /* ENABLE_SPECIALIZATION_FT */ + } + /* Skip 1 cache entry */ + /* Skip 1 cache entry */ + /* Skip 1 cache entry */ + // _LOAD_GLOBAL + { + res = &stack_pointer[0]; + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg>>1); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyEval_LoadGlobalStackRef(GLOBALS(), BUILTINS(), name, res); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (PyStackRef_IsNull(*res)) { + TRACING_JUMP_TO_LABEL(error); + } + } + // _PUSH_NULL_CONDITIONAL + { + null = &stack_pointer[1]; + if (oparg & 1) { + null[0] = PyStackRef_NULL; + } + } + stack_pointer += 1 + (oparg & 1); + assert(WITHIN_STACK_BOUNDS()); + TRACING_DISPATCH(); + } + + TRACING_TARGET(LOAD_GLOBAL_BUILTIN) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = LOAD_GLOBAL_BUILTIN; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 5; + INSTRUCTION_STATS(LOAD_GLOBAL_BUILTIN); + opcode = LOAD_GLOBAL_BUILTIN; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + static_assert(INLINE_CACHE_ENTRIES_LOAD_GLOBAL == 4, "incorrect cache size"); + _PyStackRef res; + _PyStackRef *null; + /* Skip 1 cache entry */ + // _GUARD_GLOBALS_VERSION + { + uint16_t version = read_u16(&this_instr[2].cache); + PyDictObject *dict = (PyDictObject *)GLOBALS(); + if (!PyDict_CheckExact(dict)) { + UPDATE_MISS_STATS(LOAD_GLOBAL); + assert(_PyOpcode_Deopt[opcode] == (LOAD_GLOBAL)); + JUMP_TO_PREDICTED(TRACING_LOAD_GLOBAL); + } + PyDictKeysObject *keys = FT_ATOMIC_LOAD_PTR_ACQUIRE(dict->ma_keys); + if (FT_ATOMIC_LOAD_UINT32_RELAXED(keys->dk_version) != version) { + UPDATE_MISS_STATS(LOAD_GLOBAL); + assert(_PyOpcode_Deopt[opcode] == (LOAD_GLOBAL)); + JUMP_TO_PREDICTED(TRACING_LOAD_GLOBAL); + } + assert(DK_IS_UNICODE(keys)); + } + // _LOAD_GLOBAL_BUILTINS + { + uint16_t version = read_u16(&this_instr[3].cache); + uint16_t index = read_u16(&this_instr[4].cache); + PyDictObject *dict = (PyDictObject *)BUILTINS(); + if (!PyDict_CheckExact(dict)) { + UPDATE_MISS_STATS(LOAD_GLOBAL); + assert(_PyOpcode_Deopt[opcode] == (LOAD_GLOBAL)); + JUMP_TO_PREDICTED(TRACING_LOAD_GLOBAL); + } + PyDictKeysObject *keys = FT_ATOMIC_LOAD_PTR_ACQUIRE(dict->ma_keys); + if (FT_ATOMIC_LOAD_UINT32_RELAXED(keys->dk_version) != version) { + UPDATE_MISS_STATS(LOAD_GLOBAL); + assert(_PyOpcode_Deopt[opcode] == (LOAD_GLOBAL)); + JUMP_TO_PREDICTED(TRACING_LOAD_GLOBAL); + } + assert(DK_IS_UNICODE(keys)); + PyDictUnicodeEntry *entries = DK_UNICODE_ENTRIES(keys); + PyObject *res_o = FT_ATOMIC_LOAD_PTR_RELAXED(entries[index].me_value); + if (res_o == NULL) { + UPDATE_MISS_STATS(LOAD_GLOBAL); + assert(_PyOpcode_Deopt[opcode] == (LOAD_GLOBAL)); + JUMP_TO_PREDICTED(TRACING_LOAD_GLOBAL); + } + #if Py_GIL_DISABLED + int increfed = _Py_TryIncrefCompareStackRef(&entries[index].me_value, res_o, &res); + if (!increfed) { + UPDATE_MISS_STATS(LOAD_GLOBAL); + assert(_PyOpcode_Deopt[opcode] == (LOAD_GLOBAL)); + JUMP_TO_PREDICTED(TRACING_LOAD_GLOBAL); + } + #else + res = PyStackRef_FromPyObjectNew(res_o); + #endif + STAT_INC(LOAD_GLOBAL, hit); + } + // _PUSH_NULL_CONDITIONAL + { + null = &stack_pointer[1]; + if (oparg & 1) { + null[0] = PyStackRef_NULL; + } + } + stack_pointer[0] = res; + stack_pointer += 1 + (oparg & 1); + assert(WITHIN_STACK_BOUNDS()); + TRACING_DISPATCH(); + } + + TRACING_TARGET(LOAD_GLOBAL_MODULE) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = LOAD_GLOBAL_MODULE; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 5; + INSTRUCTION_STATS(LOAD_GLOBAL_MODULE); + opcode = LOAD_GLOBAL_MODULE; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + static_assert(INLINE_CACHE_ENTRIES_LOAD_GLOBAL == 4, "incorrect cache size"); + _PyStackRef res; + _PyStackRef *null; + /* Skip 1 cache entry */ + // _NOP + { + } + // _LOAD_GLOBAL_MODULE + { + uint16_t version = read_u16(&this_instr[2].cache); + uint16_t index = read_u16(&this_instr[4].cache); + PyDictObject *dict = (PyDictObject *)GLOBALS(); + if (!PyDict_CheckExact(dict)) { + UPDATE_MISS_STATS(LOAD_GLOBAL); + assert(_PyOpcode_Deopt[opcode] == (LOAD_GLOBAL)); + JUMP_TO_PREDICTED(TRACING_LOAD_GLOBAL); + } + PyDictKeysObject *keys = FT_ATOMIC_LOAD_PTR_ACQUIRE(dict->ma_keys); + if (FT_ATOMIC_LOAD_UINT32_RELAXED(keys->dk_version) != version) { + UPDATE_MISS_STATS(LOAD_GLOBAL); + assert(_PyOpcode_Deopt[opcode] == (LOAD_GLOBAL)); + JUMP_TO_PREDICTED(TRACING_LOAD_GLOBAL); + } + assert(DK_IS_UNICODE(keys)); + PyDictUnicodeEntry *entries = DK_UNICODE_ENTRIES(keys); + assert(index < DK_SIZE(keys)); + PyObject *res_o = FT_ATOMIC_LOAD_PTR_RELAXED(entries[index].me_value); + if (res_o == NULL) { + UPDATE_MISS_STATS(LOAD_GLOBAL); + assert(_PyOpcode_Deopt[opcode] == (LOAD_GLOBAL)); + JUMP_TO_PREDICTED(TRACING_LOAD_GLOBAL); + } + #if Py_GIL_DISABLED + int increfed = _Py_TryIncrefCompareStackRef(&entries[index].me_value, res_o, &res); + if (!increfed) { + UPDATE_MISS_STATS(LOAD_GLOBAL); + assert(_PyOpcode_Deopt[opcode] == (LOAD_GLOBAL)); + JUMP_TO_PREDICTED(TRACING_LOAD_GLOBAL); + } + #else + res = PyStackRef_FromPyObjectNew(res_o); + #endif + STAT_INC(LOAD_GLOBAL, hit); + } + // _PUSH_NULL_CONDITIONAL + { + null = &stack_pointer[1]; + if (oparg & 1) { + null[0] = PyStackRef_NULL; + } + } + stack_pointer[0] = res; + stack_pointer += 1 + (oparg & 1); + assert(WITHIN_STACK_BOUNDS()); + TRACING_DISPATCH(); + } + + TRACING_TARGET(LOAD_LOCALS) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = LOAD_LOCALS; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(LOAD_LOCALS); + opcode = LOAD_LOCALS; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + _PyStackRef locals; + PyObject *l = LOCALS(); + if (l == NULL) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyErr_SetString(tstate, PyExc_SystemError, + "no locals found"); + stack_pointer = _PyFrame_GetStackPointer(frame); + TRACING_JUMP_TO_LABEL(error); + } + locals = PyStackRef_FromPyObjectNew(l); + stack_pointer[0] = locals; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + TRACING_DISPATCH(); + } + + TRACING_TARGET(LOAD_NAME) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = LOAD_NAME; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(LOAD_NAME); + opcode = LOAD_NAME; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + _PyStackRef v; + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *v_o = _PyEval_LoadName(tstate, frame, name); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (v_o == NULL) { + TRACING_JUMP_TO_LABEL(error); + } + v = PyStackRef_FromPyObjectSteal(v_o); + stack_pointer[0] = v; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + TRACING_DISPATCH(); + } + + TRACING_TARGET(LOAD_SMALL_INT) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = LOAD_SMALL_INT; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(LOAD_SMALL_INT); + opcode = LOAD_SMALL_INT; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + _PyStackRef value; + assert(oparg < _PY_NSMALLPOSINTS); + PyObject *obj = (PyObject *)&_PyLong_SMALL_INTS[_PY_NSMALLNEGINTS + oparg]; + value = PyStackRef_FromPyObjectBorrow(obj); + stack_pointer[0] = value; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + TRACING_DISPATCH(); + } + + TRACING_TARGET(LOAD_SPECIAL) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = LOAD_SPECIAL; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(LOAD_SPECIAL); + opcode = LOAD_SPECIAL; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + _PyStackRef self; + _PyStackRef *method_and_self; + // _INSERT_NULL + { + self = stack_pointer[-1]; + method_and_self = &stack_pointer[-1]; + method_and_self[1] = self; + method_and_self[0] = PyStackRef_NULL; + } + // _LOAD_SPECIAL + { + method_and_self = &stack_pointer[-1]; + PyObject *name = _Py_SpecialMethods[oparg].name; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = _PyObject_LookupSpecialMethod(name, method_and_self); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err <= 0) { + if (err == 0) { + PyObject *owner = PyStackRef_AsPyObjectBorrow(method_and_self[1]); + _PyFrame_SetStackPointer(frame, stack_pointer); + const char *errfmt = _PyEval_SpecialMethodCanSuggest(owner, oparg) + ? _Py_SpecialMethods[oparg].error_suggestion + : _Py_SpecialMethods[oparg].error; + stack_pointer = _PyFrame_GetStackPointer(frame); + assert(!_PyErr_Occurred(tstate)); + assert(errfmt != NULL); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyErr_Format(tstate, PyExc_TypeError, errfmt, owner); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + TRACING_JUMP_TO_LABEL(error); + } + } + TRACING_DISPATCH(); + } + + TRACING_TARGET(LOAD_SUPER_ATTR) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = LOAD_SUPER_ATTR; + (void)(opcode); + #endif + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(LOAD_SUPER_ATTR); + PREDICTED_TRACING_LOAD_SUPER_ATTR:; + _Py_CODEUNIT* const this_instr = next_instr - 2; + (void)this_instr; + opcode = LOAD_SUPER_ATTR; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + opcode = LOAD_SUPER_ATTR; + _PyStackRef global_super_st; + _PyStackRef class_st; + _PyStackRef self_st; + _PyStackRef attr; + _PyStackRef *null; + // _SPECIALIZE_LOAD_SUPER_ATTR + { + class_st = stack_pointer[-2]; + global_super_st = stack_pointer[-3]; + uint16_t counter = read_u16(&this_instr[1].cache); + (void)counter; + #if ENABLE_SPECIALIZATION_FT + int load_method = oparg & 1; + if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { + next_instr = this_instr; + _PyFrame_SetStackPointer(frame, stack_pointer); + _Py_Specialize_LoadSuperAttr(global_super_st, class_st, next_instr, load_method); + stack_pointer = _PyFrame_GetStackPointer(frame); + DISPATCH_SAME_OPARG(); + } + OPCODE_DEFERRED_INC(LOAD_SUPER_ATTR); + ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); + #endif /* ENABLE_SPECIALIZATION_FT */ + } + // _LOAD_SUPER_ATTR + { + self_st = stack_pointer[-1]; + PyObject *global_super = PyStackRef_AsPyObjectBorrow(global_super_st); + PyObject *class = PyStackRef_AsPyObjectBorrow(class_st); + PyObject *self = PyStackRef_AsPyObjectBorrow(self_st); + if (opcode == INSTRUMENTED_LOAD_SUPER_ATTR) { + PyObject *arg = oparg & 2 ? class : &_PyInstrumentation_MISSING; + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = _Py_call_instrumentation_2args( + tstate, PY_MONITORING_EVENT_CALL, + frame, this_instr, global_super, arg); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyStackRef tmp = self_st; + self_st = PyStackRef_NULL; + stack_pointer[-1] = self_st; + PyStackRef_CLOSE(tmp); + tmp = class_st; + class_st = PyStackRef_NULL; + stack_pointer[-2] = class_st; + PyStackRef_CLOSE(tmp); + tmp = global_super_st; + global_super_st = PyStackRef_NULL; + stack_pointer[-3] = global_super_st; + PyStackRef_CLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -3; + assert(WITHIN_STACK_BOUNDS()); + TRACING_JUMP_TO_LABEL(error); + } + } + PyObject *stack[] = {class, self}; + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *super = PyObject_Vectorcall(global_super, stack, oparg & 2, NULL); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (opcode == INSTRUMENTED_LOAD_SUPER_ATTR) { + PyObject *arg = oparg & 2 ? class : &_PyInstrumentation_MISSING; + if (super == NULL) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _Py_call_instrumentation_exc2( + tstate, PY_MONITORING_EVENT_C_RAISE, + frame, this_instr, global_super, arg); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + else { + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = _Py_call_instrumentation_2args( + tstate, PY_MONITORING_EVENT_C_RETURN, + frame, this_instr, global_super, arg); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err < 0) { + _PyFrame_SetStackPointer(frame, stack_pointer); + Py_CLEAR(super); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + } + } + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyStackRef tmp = self_st; + self_st = PyStackRef_NULL; + stack_pointer[-1] = self_st; + PyStackRef_CLOSE(tmp); + tmp = class_st; + class_st = PyStackRef_NULL; + stack_pointer[-2] = class_st; + PyStackRef_CLOSE(tmp); + tmp = global_super_st; + global_super_st = PyStackRef_NULL; + stack_pointer[-3] = global_super_st; + PyStackRef_CLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -3; + assert(WITHIN_STACK_BOUNDS()); + if (super == NULL) { + TRACING_JUMP_TO_LABEL(error); + } + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 2); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *attr_o = PyObject_GetAttr(super, name); + Py_DECREF(super); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (attr_o == NULL) { + TRACING_JUMP_TO_LABEL(error); + } + attr = PyStackRef_FromPyObjectSteal(attr_o); + } + // _PUSH_NULL_CONDITIONAL + { + null = &stack_pointer[1]; + if (oparg & 1) { + null[0] = PyStackRef_NULL; + } + } + stack_pointer[0] = attr; + stack_pointer += 1 + (oparg & 1); + assert(WITHIN_STACK_BOUNDS()); + TRACING_DISPATCH(); + } + + TRACING_TARGET(LOAD_SUPER_ATTR_ATTR) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = LOAD_SUPER_ATTR_ATTR; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(LOAD_SUPER_ATTR_ATTR); + opcode = LOAD_SUPER_ATTR_ATTR; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + static_assert(INLINE_CACHE_ENTRIES_LOAD_SUPER_ATTR == 1, "incorrect cache size"); + _PyStackRef global_super_st; + _PyStackRef class_st; + _PyStackRef self_st; + _PyStackRef attr_st; + /* Skip 1 cache entry */ + self_st = stack_pointer[-1]; + class_st = stack_pointer[-2]; + global_super_st = stack_pointer[-3]; + PyObject *global_super = PyStackRef_AsPyObjectBorrow(global_super_st); + PyObject *class = PyStackRef_AsPyObjectBorrow(class_st); + PyObject *self = PyStackRef_AsPyObjectBorrow(self_st); + assert(!(oparg & 1)); + if (global_super != (PyObject *)&PySuper_Type) { + UPDATE_MISS_STATS(LOAD_SUPER_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_SUPER_ATTR)); + JUMP_TO_PREDICTED(TRACING_LOAD_SUPER_ATTR); + } + if (!PyType_Check(class)) { + UPDATE_MISS_STATS(LOAD_SUPER_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_SUPER_ATTR)); + JUMP_TO_PREDICTED(TRACING_LOAD_SUPER_ATTR); + } + STAT_INC(LOAD_SUPER_ATTR, hit); + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 2); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *attr = _PySuper_Lookup((PyTypeObject *)class, self, name, NULL); + _PyStackRef tmp = self_st; + self_st = PyStackRef_NULL; + stack_pointer[-1] = self_st; + PyStackRef_CLOSE(tmp); + tmp = class_st; + class_st = PyStackRef_NULL; + stack_pointer[-2] = class_st; + PyStackRef_CLOSE(tmp); + tmp = global_super_st; + global_super_st = PyStackRef_NULL; + stack_pointer[-3] = global_super_st; + PyStackRef_CLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -3; + assert(WITHIN_STACK_BOUNDS()); + if (attr == NULL) { + TRACING_JUMP_TO_LABEL(error); + } + attr_st = PyStackRef_FromPyObjectSteal(attr); + stack_pointer[0] = attr_st; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + TRACING_DISPATCH(); + } + + TRACING_TARGET(LOAD_SUPER_ATTR_METHOD) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = LOAD_SUPER_ATTR_METHOD; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(LOAD_SUPER_ATTR_METHOD); + opcode = LOAD_SUPER_ATTR_METHOD; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + static_assert(INLINE_CACHE_ENTRIES_LOAD_SUPER_ATTR == 1, "incorrect cache size"); + _PyStackRef global_super_st; + _PyStackRef class_st; + _PyStackRef self_st; + _PyStackRef attr; + _PyStackRef self_or_null; + /* Skip 1 cache entry */ + self_st = stack_pointer[-1]; + class_st = stack_pointer[-2]; + global_super_st = stack_pointer[-3]; + PyObject *global_super = PyStackRef_AsPyObjectBorrow(global_super_st); + PyObject *class = PyStackRef_AsPyObjectBorrow(class_st); + PyObject *self = PyStackRef_AsPyObjectBorrow(self_st); + assert(oparg & 1); + if (global_super != (PyObject *)&PySuper_Type) { + UPDATE_MISS_STATS(LOAD_SUPER_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_SUPER_ATTR)); + JUMP_TO_PREDICTED(TRACING_LOAD_SUPER_ATTR); + } + if (!PyType_Check(class)) { + UPDATE_MISS_STATS(LOAD_SUPER_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_SUPER_ATTR)); + JUMP_TO_PREDICTED(TRACING_LOAD_SUPER_ATTR); + } + STAT_INC(LOAD_SUPER_ATTR, hit); + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 2); + PyTypeObject *cls = (PyTypeObject *)class; + int method_found = 0; + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *attr_o = _PySuper_Lookup(cls, self, name, + Py_TYPE(self)->tp_getattro == PyObject_GenericGetAttr ? &method_found : NULL); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (attr_o == NULL) { + TRACING_JUMP_TO_LABEL(error); + } + if (method_found) { + self_or_null = self_st; + } else { + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(self_st); + stack_pointer = _PyFrame_GetStackPointer(frame); + self_or_null = PyStackRef_NULL; + stack_pointer += 1; + } + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyStackRef tmp = global_super_st; + global_super_st = self_or_null; + stack_pointer[-2] = global_super_st; + PyStackRef_CLOSE(tmp); + tmp = class_st; + class_st = PyStackRef_NULL; + stack_pointer[-1] = class_st; + PyStackRef_CLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); + attr = PyStackRef_FromPyObjectSteal(attr_o); + stack_pointer[0] = attr; + stack_pointer[1] = self_or_null; + stack_pointer += 2; + assert(WITHIN_STACK_BOUNDS()); + TRACING_DISPATCH(); + } + + TRACING_TARGET(MAKE_CELL) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = MAKE_CELL; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(MAKE_CELL); + opcode = MAKE_CELL; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + PyObject *initial = PyStackRef_AsPyObjectBorrow(GETLOCAL(oparg)); + PyObject *cell = PyCell_New(initial); + if (cell == NULL) { + TRACING_JUMP_TO_LABEL(error); + } + _PyStackRef tmp = GETLOCAL(oparg); + GETLOCAL(oparg) = PyStackRef_FromPyObjectSteal(cell); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_XCLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); + TRACING_DISPATCH(); + } + + TRACING_TARGET(MAKE_FUNCTION) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = MAKE_FUNCTION; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(MAKE_FUNCTION); + opcode = MAKE_FUNCTION; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + _PyStackRef codeobj_st; + _PyStackRef func; + codeobj_st = stack_pointer[-1]; + PyObject *codeobj = PyStackRef_AsPyObjectBorrow(codeobj_st); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyFunctionObject *func_obj = (PyFunctionObject *) + PyFunction_New(codeobj, GLOBALS()); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(codeobj_st); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (func_obj == NULL) { + TRACING_JUMP_TO_LABEL(error); + } + _PyFunction_SetVersion( + func_obj, ((PyCodeObject *)codeobj)->co_version); + func = PyStackRef_FromPyObjectSteal((PyObject *)func_obj); + stack_pointer[0] = func; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + TRACING_DISPATCH(); + } + + TRACING_TARGET(MAP_ADD) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = MAP_ADD; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(MAP_ADD); + opcode = MAP_ADD; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + _PyStackRef dict_st; + _PyStackRef key; + _PyStackRef value; + value = stack_pointer[-1]; + key = stack_pointer[-2]; + dict_st = stack_pointer[-3 - (oparg - 1)]; + PyObject *dict = PyStackRef_AsPyObjectBorrow(dict_st); + assert(PyDict_CheckExact(dict)); + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = _PyDict_SetItem_Take2( + (PyDictObject *)dict, + PyStackRef_AsPyObjectSteal(key), + PyStackRef_AsPyObjectSteal(value) + ); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err != 0) { + TRACING_JUMP_TO_LABEL(pop_2_error); + } + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); + TRACING_DISPATCH(); + } + + TRACING_TARGET(MATCH_CLASS) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = MATCH_CLASS; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(MATCH_CLASS); + opcode = MATCH_CLASS; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + _PyStackRef subject; + _PyStackRef type; + _PyStackRef names; + _PyStackRef attrs; + names = stack_pointer[-1]; + type = stack_pointer[-2]; + subject = stack_pointer[-3]; + assert(PyTuple_CheckExact(PyStackRef_AsPyObjectBorrow(names))); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *attrs_o = _PyEval_MatchClass(tstate, + PyStackRef_AsPyObjectBorrow(subject), + PyStackRef_AsPyObjectBorrow(type), oparg, + PyStackRef_AsPyObjectBorrow(names)); + _PyStackRef tmp = names; + names = PyStackRef_NULL; + stack_pointer[-1] = names; + PyStackRef_CLOSE(tmp); + tmp = type; + type = PyStackRef_NULL; + stack_pointer[-2] = type; + PyStackRef_CLOSE(tmp); + tmp = subject; + subject = PyStackRef_NULL; + stack_pointer[-3] = subject; + PyStackRef_CLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -3; + assert(WITHIN_STACK_BOUNDS()); + if (attrs_o) { + assert(PyTuple_CheckExact(attrs_o)); + attrs = PyStackRef_FromPyObjectSteal(attrs_o); + } + else { + if (_PyErr_Occurred(tstate)) { + TRACING_JUMP_TO_LABEL(error); + } + attrs = PyStackRef_None; + } + stack_pointer[0] = attrs; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + TRACING_DISPATCH(); + } + + TRACING_TARGET(MATCH_KEYS) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = MATCH_KEYS; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(MATCH_KEYS); + opcode = MATCH_KEYS; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + _PyStackRef subject; + _PyStackRef keys; + _PyStackRef values_or_none; + keys = stack_pointer[-1]; + subject = stack_pointer[-2]; + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *values_or_none_o = _PyEval_MatchKeys(tstate, + PyStackRef_AsPyObjectBorrow(subject), PyStackRef_AsPyObjectBorrow(keys)); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (values_or_none_o == NULL) { + TRACING_JUMP_TO_LABEL(error); + } + values_or_none = PyStackRef_FromPyObjectSteal(values_or_none_o); + stack_pointer[0] = values_or_none; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + TRACING_DISPATCH(); + } + + TRACING_TARGET(MATCH_MAPPING) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = MATCH_MAPPING; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(MATCH_MAPPING); + opcode = MATCH_MAPPING; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + _PyStackRef subject; + _PyStackRef res; + subject = stack_pointer[-1]; + int match = PyStackRef_TYPE(subject)->tp_flags & Py_TPFLAGS_MAPPING; + res = match ? PyStackRef_True : PyStackRef_False; + stack_pointer[0] = res; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + TRACING_DISPATCH(); + } + + TRACING_TARGET(MATCH_SEQUENCE) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = MATCH_SEQUENCE; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(MATCH_SEQUENCE); + opcode = MATCH_SEQUENCE; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + _PyStackRef subject; + _PyStackRef res; + subject = stack_pointer[-1]; + int match = PyStackRef_TYPE(subject)->tp_flags & Py_TPFLAGS_SEQUENCE; + res = match ? PyStackRef_True : PyStackRef_False; + stack_pointer[0] = res; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + TRACING_DISPATCH(); + } + + TRACING_TARGET(NOP) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = NOP; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(NOP); + opcode = NOP; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + TRACING_DISPATCH(); + } + + TRACING_TARGET(NOT_TAKEN) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = NOT_TAKEN; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(NOT_TAKEN); + opcode = NOT_TAKEN; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + TRACING_DISPATCH(); + } + + TRACING_TARGET(POP_EXCEPT) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = POP_EXCEPT; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(POP_EXCEPT); + opcode = POP_EXCEPT; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + _PyStackRef exc_value; + exc_value = stack_pointer[-1]; + _PyErr_StackItem *exc_info = tstate->exc_info; + _PyFrame_SetStackPointer(frame, stack_pointer); + Py_XSETREF(exc_info->exc_value, + PyStackRef_IsNone(exc_value) + ? NULL : PyStackRef_AsPyObjectSteal(exc_value)); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + TRACING_DISPATCH(); + } + + TRACING_TARGET(POP_ITER) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = POP_ITER; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(POP_ITER); + opcode = POP_ITER; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + _PyStackRef iter; + _PyStackRef index_or_null; + index_or_null = stack_pointer[-1]; + iter = stack_pointer[-2]; + (void)index_or_null; + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(iter); + stack_pointer = _PyFrame_GetStackPointer(frame); + TRACING_DISPATCH(); + } + + TRACING_TARGET(POP_JUMP_IF_FALSE) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = POP_JUMP_IF_FALSE; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(POP_JUMP_IF_FALSE); + opcode = POP_JUMP_IF_FALSE; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + _PyStackRef cond; + /* Skip 1 cache entry */ + cond = stack_pointer[-1]; + assert(PyStackRef_BoolCheck(cond)); + int flag = PyStackRef_IsFalse(cond); + JUMPBY(flag ? oparg : next_instr->op.code == NOT_TAKEN); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + TRACING_DISPATCH(); + } + + TRACING_TARGET(POP_JUMP_IF_NONE) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = POP_JUMP_IF_NONE; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(POP_JUMP_IF_NONE); + opcode = POP_JUMP_IF_NONE; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + _PyStackRef value; + _PyStackRef b; + _PyStackRef cond; + /* Skip 1 cache entry */ + // _IS_NONE + { + value = stack_pointer[-1]; + if (PyStackRef_IsNone(value)) { + b = PyStackRef_True; + } + else { + b = PyStackRef_False; + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyStackRef tmp = value; + value = b; + stack_pointer[-1] = value; + PyStackRef_CLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + } + // _POP_JUMP_IF_TRUE + { + cond = b; + assert(PyStackRef_BoolCheck(cond)); + int flag = PyStackRef_IsTrue(cond); + JUMPBY(flag ? oparg : next_instr->op.code == NOT_TAKEN); + } + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + TRACING_DISPATCH(); + } + + TRACING_TARGET(POP_JUMP_IF_NOT_NONE) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = POP_JUMP_IF_NOT_NONE; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(POP_JUMP_IF_NOT_NONE); + opcode = POP_JUMP_IF_NOT_NONE; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + _PyStackRef value; + _PyStackRef b; + _PyStackRef cond; + /* Skip 1 cache entry */ + // _IS_NONE + { + value = stack_pointer[-1]; + if (PyStackRef_IsNone(value)) { + b = PyStackRef_True; + } + else { + b = PyStackRef_False; + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyStackRef tmp = value; + value = b; + stack_pointer[-1] = value; + PyStackRef_CLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + } + // _POP_JUMP_IF_FALSE + { + cond = b; + assert(PyStackRef_BoolCheck(cond)); + int flag = PyStackRef_IsFalse(cond); + JUMPBY(flag ? oparg : next_instr->op.code == NOT_TAKEN); + } + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + TRACING_DISPATCH(); + } + + TRACING_TARGET(POP_JUMP_IF_TRUE) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = POP_JUMP_IF_TRUE; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(POP_JUMP_IF_TRUE); + opcode = POP_JUMP_IF_TRUE; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + _PyStackRef cond; + /* Skip 1 cache entry */ + cond = stack_pointer[-1]; + assert(PyStackRef_BoolCheck(cond)); + int flag = PyStackRef_IsTrue(cond); + JUMPBY(flag ? oparg : next_instr->op.code == NOT_TAKEN); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + TRACING_DISPATCH(); + } + + TRACING_TARGET(POP_TOP) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = POP_TOP; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(POP_TOP); + opcode = POP_TOP; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + _PyStackRef value; + value = stack_pointer[-1]; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_XCLOSE(value); + stack_pointer = _PyFrame_GetStackPointer(frame); + TRACING_DISPATCH(); + } + + TRACING_TARGET(PUSH_EXC_INFO) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = PUSH_EXC_INFO; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(PUSH_EXC_INFO); + opcode = PUSH_EXC_INFO; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + _PyStackRef exc; + _PyStackRef prev_exc; + _PyStackRef new_exc; + exc = stack_pointer[-1]; + _PyErr_StackItem *exc_info = tstate->exc_info; + if (exc_info->exc_value != NULL) { + prev_exc = PyStackRef_FromPyObjectSteal(exc_info->exc_value); + } + else { + prev_exc = PyStackRef_None; + } + assert(PyStackRef_ExceptionInstanceCheck(exc)); + exc_info->exc_value = PyStackRef_AsPyObjectNew(exc); + new_exc = exc; + stack_pointer[-1] = prev_exc; + stack_pointer[0] = new_exc; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + TRACING_DISPATCH(); + } + + TRACING_TARGET(PUSH_NULL) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = PUSH_NULL; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(PUSH_NULL); + opcode = PUSH_NULL; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + _PyStackRef res; + res = PyStackRef_NULL; + stack_pointer[0] = res; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + TRACING_DISPATCH(); + } + + TRACING_TARGET(RAISE_VARARGS) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = RAISE_VARARGS; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(RAISE_VARARGS); + opcode = RAISE_VARARGS; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + _PyStackRef *args; + args = &stack_pointer[-oparg]; + assert(oparg < 3); + PyObject *cause = oparg == 2 ? PyStackRef_AsPyObjectSteal(args[1]) : NULL; + PyObject *exc = oparg > 0 ? PyStackRef_AsPyObjectSteal(args[0]) : NULL; + stack_pointer += -oparg; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = do_raise(tstate, exc, cause); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err) { + assert(oparg == 0); + _PyFrame_SetStackPointer(frame, stack_pointer); + monitor_reraise(tstate, frame, this_instr); + TRACING_JUMP_TO_LABEL(exception_unwind); + } + TRACING_JUMP_TO_LABEL(error); + } + + TRACING_TARGET(RERAISE) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = RERAISE; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(RERAISE); + opcode = RERAISE; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + _PyStackRef *values; + _PyStackRef exc_st; + exc_st = stack_pointer[-1]; + values = &stack_pointer[-1 - oparg]; + PyObject *exc = PyStackRef_AsPyObjectSteal(exc_st); + assert(oparg >= 0 && oparg <= 2); + if (oparg) { + frame->instr_ptr = _PyFrame_GetBytecode(frame) + PyStackRef_UntagInt(values[0]); + } + assert(exc && PyExceptionInstance_Check(exc)); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyErr_SetRaisedException(tstate, exc); + monitor_reraise(tstate, frame, this_instr); + TRACING_JUMP_TO_LABEL(exception_unwind); + } + + TRACING_TARGET(RESERVED) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = RESERVED; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(RESERVED); + opcode = RESERVED; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + assert(0 && "Executing RESERVED instruction."); + Py_FatalError("Executing RESERVED instruction."); + TRACING_DISPATCH(); + } + + TRACING_TARGET(RESUME) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = RESUME; + (void)(opcode); + #endif + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(RESUME); + PREDICTED_TRACING_RESUME:; + _Py_CODEUNIT* const this_instr = next_instr - 1; + (void)this_instr; + opcode = RESUME; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + // _LOAD_BYTECODE + { + #ifdef Py_GIL_DISABLED + if (frame->tlbc_index != + ((_PyThreadStateImpl *)tstate)->tlbc_index) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _Py_CODEUNIT *bytecode = + _PyEval_GetExecutableCode(tstate, _PyFrame_GetCode(frame)); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (bytecode == NULL) { + TRACING_JUMP_TO_LABEL(error); + } + ptrdiff_t off = this_instr - _PyFrame_GetBytecode(frame); + frame->tlbc_index = ((_PyThreadStateImpl *)tstate)->tlbc_index; + frame->instr_ptr = bytecode + off; + next_instr = frame->instr_ptr; + TRACING_DISPATCH(); + } + #endif + } + // _MAYBE_INSTRUMENT + { + #ifdef Py_GIL_DISABLED + + int check_instrumentation = 1; + #else + int check_instrumentation = (tstate->tracing == 0); + #endif + if (check_instrumentation) { + uintptr_t global_version = _Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & ~_PY_EVAL_EVENTS_MASK; + uintptr_t code_version = FT_ATOMIC_LOAD_UINTPTR_ACQUIRE(_PyFrame_GetCode(frame)->_co_instrumentation_version); + if (code_version != global_version) { + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = _Py_Instrument(_PyFrame_GetCode(frame), tstate->interp); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err) { + TRACING_JUMP_TO_LABEL(error); + } + next_instr = this_instr; + TRACING_DISPATCH(); + } + } + } + // _QUICKEN_RESUME + { + #if ENABLE_SPECIALIZATION_FT + if (tstate->tracing == 0 && this_instr->op.code == RESUME) { + FT_ATOMIC_STORE_UINT8_RELAXED(this_instr->op.code, RESUME_CHECK); + } + #endif /* ENABLE_SPECIALIZATION_FT */ + } + // _CHECK_PERIODIC_IF_NOT_YIELD_FROM + { + if ((oparg & RESUME_OPARG_LOCATION_MASK) < RESUME_AFTER_YIELD_FROM) { + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = check_periodics(tstate); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err != 0) { + TRACING_JUMP_TO_LABEL(error); + } + } + } + TRACING_DISPATCH(); + } + + TRACING_TARGET(RESUME_CHECK) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = RESUME_CHECK; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(RESUME_CHECK); + opcode = RESUME_CHECK; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + static_assert(0 == 0, "incorrect cache size"); + #if defined(__EMSCRIPTEN__) + if (_Py_emscripten_signal_clock == 0) { + UPDATE_MISS_STATS(RESUME); + assert(_PyOpcode_Deopt[opcode] == (RESUME)); + JUMP_TO_PREDICTED(TRACING_RESUME); + } + _Py_emscripten_signal_clock -= Py_EMSCRIPTEN_SIGNAL_HANDLING; + #endif + uintptr_t eval_breaker = _Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker); + uintptr_t version = FT_ATOMIC_LOAD_UINTPTR_ACQUIRE(_PyFrame_GetCode(frame)->_co_instrumentation_version); + assert((version & _PY_EVAL_EVENTS_MASK) == 0); + if (eval_breaker != version) { + UPDATE_MISS_STATS(RESUME); + assert(_PyOpcode_Deopt[opcode] == (RESUME)); + JUMP_TO_PREDICTED(TRACING_RESUME); + } + #ifdef Py_GIL_DISABLED + if (frame->tlbc_index != + ((_PyThreadStateImpl *)tstate)->tlbc_index) { + UPDATE_MISS_STATS(RESUME); + assert(_PyOpcode_Deopt[opcode] == (RESUME)); + JUMP_TO_PREDICTED(TRACING_RESUME); + } + #endif + TRACING_DISPATCH(); + } + + TRACING_TARGET(RETURN_GENERATOR) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = RETURN_GENERATOR; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(RETURN_GENERATOR); + opcode = RETURN_GENERATOR; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + _PyStackRef res; + assert(PyStackRef_FunctionCheck(frame->f_funcobj)); + PyFunctionObject *func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyGenObject *gen = (PyGenObject *)_Py_MakeCoro(func); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (gen == NULL) { + TRACING_JUMP_TO_LABEL(error); + } + assert(STACK_LEVEL() == 0); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyInterpreterFrame *gen_frame = &gen->gi_iframe; + frame->instr_ptr++; + _PyFrame_Copy(frame, gen_frame); + assert(frame->frame_obj == NULL); + gen->gi_frame_state = FRAME_CREATED; + gen_frame->owner = FRAME_OWNED_BY_GENERATOR; + _Py_LeaveRecursiveCallPy(tstate); + _PyInterpreterFrame *prev = frame->previous; + _PyThreadState_PopFrame(tstate, frame); + frame = tstate->current_frame = prev; + LOAD_IP(frame->return_offset); + #if TIER_TWO + frame->instr_ptr += (frame->return_offset); + #endif + stack_pointer = _PyFrame_GetStackPointer(frame); + res = PyStackRef_FromPyObjectStealMortal((PyObject *)gen); + LLTRACE_RESUME_FRAME(); + stack_pointer[0] = res; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + TRACING_DISPATCH(); + } + + TRACING_TARGET(RETURN_VALUE) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = RETURN_VALUE; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(RETURN_VALUE); + opcode = RETURN_VALUE; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + _PyStackRef retval; + _PyStackRef res; + retval = stack_pointer[-1]; + assert(frame->owner != FRAME_OWNED_BY_INTERPRETER); + _PyStackRef temp = PyStackRef_MakeHeapSafe(retval); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + assert(STACK_LEVEL() == 0); + _Py_LeaveRecursiveCallPy(tstate); + _PyInterpreterFrame *dying = frame; + frame = tstate->current_frame = dying->previous; + _PyEval_FrameClearAndPop(tstate, dying); + stack_pointer = _PyFrame_GetStackPointer(frame); + LOAD_IP(frame->return_offset); + #if TIER_TWO + frame->instr_ptr += frame->return_offset; + #endif + res = temp; + LLTRACE_RESUME_FRAME(); + stack_pointer[0] = res; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + TRACING_DISPATCH(); + } + + TRACING_TARGET(SEND) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = SEND; + (void)(opcode); + #endif + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(SEND); + PREDICTED_TRACING_SEND:; + _Py_CODEUNIT* const this_instr = next_instr - 2; + (void)this_instr; + opcode = SEND; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + _PyStackRef receiver; + _PyStackRef v; + _PyStackRef retval; + // _SPECIALIZE_SEND + { + receiver = stack_pointer[-2]; + uint16_t counter = read_u16(&this_instr[1].cache); + (void)counter; + #if ENABLE_SPECIALIZATION_FT + if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { + next_instr = this_instr; + _PyFrame_SetStackPointer(frame, stack_pointer); + _Py_Specialize_Send(receiver, next_instr); + stack_pointer = _PyFrame_GetStackPointer(frame); + DISPATCH_SAME_OPARG(); + } + OPCODE_DEFERRED_INC(SEND); + ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); + #endif /* ENABLE_SPECIALIZATION_FT */ + } + // _SEND + { + v = stack_pointer[-1]; + PyObject *receiver_o = PyStackRef_AsPyObjectBorrow(receiver); + PyObject *retval_o; + assert(frame->owner != FRAME_OWNED_BY_INTERPRETER); + if ((tstate->interp->eval_frame == NULL) && + (Py_TYPE(receiver_o) == &PyGen_Type || Py_TYPE(receiver_o) == &PyCoro_Type) && + ((PyGenObject *)receiver_o)->gi_frame_state < FRAME_EXECUTING) + { + PyGenObject *gen = (PyGenObject *)receiver_o; + _PyInterpreterFrame *gen_frame = &gen->gi_iframe; + _PyFrame_StackPush(gen_frame, PyStackRef_MakeHeapSafe(v)); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + gen->gi_frame_state = FRAME_EXECUTING; + gen->gi_exc_state.previous_item = tstate->exc_info; + tstate->exc_info = &gen->gi_exc_state; + assert( 2u + oparg <= UINT16_MAX); + frame->return_offset = (uint16_t)( 2u + oparg); + assert(gen_frame->previous == NULL); + gen_frame->previous = frame; + TRACING_DISPATCH_INLINED(gen_frame); + } + if (PyStackRef_IsNone(v) && PyIter_Check(receiver_o)) { + _PyFrame_SetStackPointer(frame, stack_pointer); + retval_o = Py_TYPE(receiver_o)->tp_iternext(receiver_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + else { + _PyFrame_SetStackPointer(frame, stack_pointer); + retval_o = PyObject_CallMethodOneArg(receiver_o, + &_Py_ID(send), + PyStackRef_AsPyObjectBorrow(v)); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + if (retval_o == NULL) { + _PyFrame_SetStackPointer(frame, stack_pointer); + int matches = _PyErr_ExceptionMatches(tstate, PyExc_StopIteration); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (matches) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyEval_MonitorRaise(tstate, frame, this_instr); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = _PyGen_FetchStopIterationValue(&retval_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err == 0) { + assert(retval_o != NULL); + JUMPBY(oparg); + } + else { + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(v); + stack_pointer = _PyFrame_GetStackPointer(frame); + TRACING_JUMP_TO_LABEL(error); + } + } + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(v); + stack_pointer = _PyFrame_GetStackPointer(frame); + retval = PyStackRef_FromPyObjectSteal(retval_o); + } + stack_pointer[0] = retval; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + TRACING_DISPATCH(); + } + + TRACING_TARGET(SEND_GEN) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = SEND_GEN; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(SEND_GEN); + opcode = SEND_GEN; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + static_assert(INLINE_CACHE_ENTRIES_SEND == 1, "incorrect cache size"); + _PyStackRef receiver; + _PyStackRef v; + _PyStackRef gen_frame; + _PyStackRef new_frame; + /* Skip 1 cache entry */ + // _CHECK_PEP_523 + { + if (tstate->interp->eval_frame) { + UPDATE_MISS_STATS(SEND); + assert(_PyOpcode_Deopt[opcode] == (SEND)); + JUMP_TO_PREDICTED(TRACING_SEND); + } + } + // _SEND_GEN_FRAME + { + v = stack_pointer[-1]; + receiver = stack_pointer[-2]; + PyGenObject *gen = (PyGenObject *)PyStackRef_AsPyObjectBorrow(receiver); + if (Py_TYPE(gen) != &PyGen_Type && Py_TYPE(gen) != &PyCoro_Type) { + UPDATE_MISS_STATS(SEND); + assert(_PyOpcode_Deopt[opcode] == (SEND)); + JUMP_TO_PREDICTED(TRACING_SEND); + } + if (gen->gi_frame_state >= FRAME_EXECUTING) { + UPDATE_MISS_STATS(SEND); + assert(_PyOpcode_Deopt[opcode] == (SEND)); + JUMP_TO_PREDICTED(TRACING_SEND); + } + STAT_INC(SEND, hit); + _PyInterpreterFrame *pushed_frame = &gen->gi_iframe; + _PyFrame_StackPush(pushed_frame, PyStackRef_MakeHeapSafe(v)); + gen->gi_frame_state = FRAME_EXECUTING; + gen->gi_exc_state.previous_item = tstate->exc_info; + tstate->exc_info = &gen->gi_exc_state; + assert( 2u + oparg <= UINT16_MAX); + frame->return_offset = (uint16_t)( 2u + oparg); + pushed_frame->previous = frame; + gen_frame = PyStackRef_Wrap(pushed_frame); + } + // _PUSH_FRAME + { + new_frame = gen_frame; + assert(tstate->interp->eval_frame == NULL); + _PyInterpreterFrame *temp = PyStackRef_Unwrap(new_frame); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + assert(temp->previous == frame || temp->previous->previous == frame); + CALL_STAT_INC(inlined_py_calls); + frame = tstate->current_frame = temp; + tstate->py_recursion_remaining--; + LOAD_SP(); + LOAD_IP(0); + LLTRACE_RESUME_FRAME(); + } + TRACING_DISPATCH(); + } + + TRACING_TARGET(SETUP_ANNOTATIONS) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = SETUP_ANNOTATIONS; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(SETUP_ANNOTATIONS); + opcode = SETUP_ANNOTATIONS; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + PyObject *ann_dict; + if (LOCALS() == NULL) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyErr_Format(tstate, PyExc_SystemError, + "no locals found when setting up annotations"); + stack_pointer = _PyFrame_GetStackPointer(frame); + TRACING_JUMP_TO_LABEL(error); + } + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = PyMapping_GetOptionalItem(LOCALS(), &_Py_ID(__annotations__), &ann_dict); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err < 0) { + TRACING_JUMP_TO_LABEL(error); + } + if (ann_dict == NULL) { + _PyFrame_SetStackPointer(frame, stack_pointer); + ann_dict = PyDict_New(); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (ann_dict == NULL) { + TRACING_JUMP_TO_LABEL(error); + } + _PyFrame_SetStackPointer(frame, stack_pointer); + err = PyObject_SetItem(LOCALS(), &_Py_ID(__annotations__), + ann_dict); + Py_DECREF(ann_dict); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err) { + TRACING_JUMP_TO_LABEL(error); + } + } + else { + _PyFrame_SetStackPointer(frame, stack_pointer); + Py_DECREF(ann_dict); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + TRACING_DISPATCH(); + } + + TRACING_TARGET(SET_ADD) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = SET_ADD; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(SET_ADD); + opcode = SET_ADD; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + _PyStackRef set; + _PyStackRef v; + v = stack_pointer[-1]; + set = stack_pointer[-2 - (oparg-1)]; + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = _PySet_AddTakeRef((PySetObject *)PyStackRef_AsPyObjectBorrow(set), + PyStackRef_AsPyObjectSteal(v)); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err) { + TRACING_JUMP_TO_LABEL(pop_1_error); + } + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + TRACING_DISPATCH(); + } + + TRACING_TARGET(SET_FUNCTION_ATTRIBUTE) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = SET_FUNCTION_ATTRIBUTE; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(SET_FUNCTION_ATTRIBUTE); + opcode = SET_FUNCTION_ATTRIBUTE; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + _PyStackRef attr_st; + _PyStackRef func_in; + _PyStackRef func_out; + func_in = stack_pointer[-1]; + attr_st = stack_pointer[-2]; + PyObject *func = PyStackRef_AsPyObjectBorrow(func_in); + PyObject *attr = PyStackRef_AsPyObjectSteal(attr_st); + func_out = func_in; + assert(PyFunction_Check(func)); + size_t offset = _Py_FunctionAttributeOffsets[oparg]; + assert(offset != 0); + PyObject **ptr = (PyObject **)(((char *)func) + offset); + assert(*ptr == NULL); + *ptr = attr; + stack_pointer[-2] = func_out; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + TRACING_DISPATCH(); + } + + TRACING_TARGET(SET_UPDATE) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = SET_UPDATE; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(SET_UPDATE); + opcode = SET_UPDATE; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + _PyStackRef set; + _PyStackRef iterable; + iterable = stack_pointer[-1]; + set = stack_pointer[-2 - (oparg-1)]; + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = _PySet_Update(PyStackRef_AsPyObjectBorrow(set), + PyStackRef_AsPyObjectBorrow(iterable)); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(iterable); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err < 0) { + TRACING_JUMP_TO_LABEL(error); + } + TRACING_DISPATCH(); + } + + TRACING_TARGET(STORE_ATTR) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = STORE_ATTR; + (void)(opcode); + #endif + frame->instr_ptr = next_instr; + next_instr += 5; + INSTRUCTION_STATS(STORE_ATTR); + PREDICTED_TRACING_STORE_ATTR:; + _Py_CODEUNIT* const this_instr = next_instr - 5; + (void)this_instr; + opcode = STORE_ATTR; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + _PyStackRef owner; + _PyStackRef v; + // _SPECIALIZE_STORE_ATTR + { + owner = stack_pointer[-1]; + uint16_t counter = read_u16(&this_instr[1].cache); + (void)counter; + #if ENABLE_SPECIALIZATION_FT + if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); + next_instr = this_instr; + _PyFrame_SetStackPointer(frame, stack_pointer); + _Py_Specialize_StoreAttr(owner, next_instr, name); + stack_pointer = _PyFrame_GetStackPointer(frame); + DISPATCH_SAME_OPARG(); + } + OPCODE_DEFERRED_INC(STORE_ATTR); + ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); + #endif /* ENABLE_SPECIALIZATION_FT */ + } + /* Skip 3 cache entries */ + // _STORE_ATTR + { + v = stack_pointer[-2]; + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = PyObject_SetAttr(PyStackRef_AsPyObjectBorrow(owner), + name, PyStackRef_AsPyObjectBorrow(v)); + _PyStackRef tmp = owner; + owner = PyStackRef_NULL; + stack_pointer[-1] = owner; + PyStackRef_CLOSE(tmp); + tmp = v; + v = PyStackRef_NULL; + stack_pointer[-2] = v; + PyStackRef_CLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); + if (err) { + TRACING_JUMP_TO_LABEL(error); + } + } + TRACING_DISPATCH(); + } + + TRACING_TARGET(STORE_ATTR_INSTANCE_VALUE) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = STORE_ATTR_INSTANCE_VALUE; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 5; + INSTRUCTION_STATS(STORE_ATTR_INSTANCE_VALUE); + opcode = STORE_ATTR_INSTANCE_VALUE; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + static_assert(INLINE_CACHE_ENTRIES_STORE_ATTR == 4, "incorrect cache size"); + _PyStackRef owner; + _PyStackRef value; + /* Skip 1 cache entry */ + // _GUARD_TYPE_VERSION_AND_LOCK + { + owner = stack_pointer[-1]; + uint32_t type_version = read_u32(&this_instr[2].cache); + PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); + assert(type_version != 0); + if (!LOCK_OBJECT(owner_o)) { + UPDATE_MISS_STATS(STORE_ATTR); + assert(_PyOpcode_Deopt[opcode] == (STORE_ATTR)); + JUMP_TO_PREDICTED(TRACING_STORE_ATTR); + } + PyTypeObject *tp = Py_TYPE(owner_o); + if (FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version) { + UNLOCK_OBJECT(owner_o); + if (true) { + UPDATE_MISS_STATS(STORE_ATTR); + assert(_PyOpcode_Deopt[opcode] == (STORE_ATTR)); + JUMP_TO_PREDICTED(TRACING_STORE_ATTR); + } + } + } + // _GUARD_DORV_NO_DICT + { + PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); + assert(Py_TYPE(owner_o)->tp_dictoffset < 0); + assert(Py_TYPE(owner_o)->tp_flags & Py_TPFLAGS_INLINE_VALUES); + if (_PyObject_GetManagedDict(owner_o) || + !FT_ATOMIC_LOAD_UINT8(_PyObject_InlineValues(owner_o)->valid)) { + UNLOCK_OBJECT(owner_o); + if (true) { + UPDATE_MISS_STATS(STORE_ATTR); + assert(_PyOpcode_Deopt[opcode] == (STORE_ATTR)); + JUMP_TO_PREDICTED(TRACING_STORE_ATTR); + } + } + } + // _STORE_ATTR_INSTANCE_VALUE + { + value = stack_pointer[-2]; + uint16_t offset = read_u16(&this_instr[4].cache); + PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); + STAT_INC(STORE_ATTR, hit); + assert(_PyObject_GetManagedDict(owner_o) == NULL); + PyObject **value_ptr = (PyObject**)(((char *)owner_o) + offset); + PyObject *old_value = *value_ptr; + FT_ATOMIC_STORE_PTR_RELEASE(*value_ptr, PyStackRef_AsPyObjectSteal(value)); + if (old_value == NULL) { + PyDictValues *values = _PyObject_InlineValues(owner_o); + Py_ssize_t index = value_ptr - values->values; + _PyDictValues_AddToInsertionOrder(values, index); + } + UNLOCK_OBJECT(owner_o); + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(owner); + Py_XDECREF(old_value); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + TRACING_DISPATCH(); + } + + TRACING_TARGET(STORE_ATTR_SLOT) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = STORE_ATTR_SLOT; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 5; + INSTRUCTION_STATS(STORE_ATTR_SLOT); + opcode = STORE_ATTR_SLOT; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + static_assert(INLINE_CACHE_ENTRIES_STORE_ATTR == 4, "incorrect cache size"); + _PyStackRef owner; + _PyStackRef value; + /* Skip 1 cache entry */ + // _GUARD_TYPE_VERSION + { + owner = stack_pointer[-1]; + uint32_t type_version = read_u32(&this_instr[2].cache); + PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); + assert(type_version != 0); + if (FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version) { + UPDATE_MISS_STATS(STORE_ATTR); + assert(_PyOpcode_Deopt[opcode] == (STORE_ATTR)); + JUMP_TO_PREDICTED(TRACING_STORE_ATTR); + } + } + // _STORE_ATTR_SLOT + { + value = stack_pointer[-2]; + uint16_t index = read_u16(&this_instr[4].cache); + PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); + if (!LOCK_OBJECT(owner_o)) { + UPDATE_MISS_STATS(STORE_ATTR); + assert(_PyOpcode_Deopt[opcode] == (STORE_ATTR)); + JUMP_TO_PREDICTED(TRACING_STORE_ATTR); + } + char *addr = (char *)owner_o + index; + STAT_INC(STORE_ATTR, hit); + PyObject *old_value = *(PyObject **)addr; + FT_ATOMIC_STORE_PTR_RELEASE(*(PyObject **)addr, PyStackRef_AsPyObjectSteal(value)); + UNLOCK_OBJECT(owner_o); + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(owner); + Py_XDECREF(old_value); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + TRACING_DISPATCH(); + } + + TRACING_TARGET(STORE_ATTR_WITH_HINT) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = STORE_ATTR_WITH_HINT; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 5; + INSTRUCTION_STATS(STORE_ATTR_WITH_HINT); + opcode = STORE_ATTR_WITH_HINT; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + static_assert(INLINE_CACHE_ENTRIES_STORE_ATTR == 4, "incorrect cache size"); + _PyStackRef owner; + _PyStackRef value; + /* Skip 1 cache entry */ + // _GUARD_TYPE_VERSION + { + owner = stack_pointer[-1]; + uint32_t type_version = read_u32(&this_instr[2].cache); + PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); + assert(type_version != 0); + if (FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version) { + UPDATE_MISS_STATS(STORE_ATTR); + assert(_PyOpcode_Deopt[opcode] == (STORE_ATTR)); + JUMP_TO_PREDICTED(TRACING_STORE_ATTR); + } + } + // _STORE_ATTR_WITH_HINT + { + value = stack_pointer[-2]; + uint16_t hint = read_u16(&this_instr[4].cache); + PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); + assert(Py_TYPE(owner_o)->tp_flags & Py_TPFLAGS_MANAGED_DICT); + PyDictObject *dict = _PyObject_GetManagedDict(owner_o); + if (dict == NULL) { + UPDATE_MISS_STATS(STORE_ATTR); + assert(_PyOpcode_Deopt[opcode] == (STORE_ATTR)); + JUMP_TO_PREDICTED(TRACING_STORE_ATTR); + } + if (!LOCK_OBJECT(dict)) { + UPDATE_MISS_STATS(STORE_ATTR); + assert(_PyOpcode_Deopt[opcode] == (STORE_ATTR)); + JUMP_TO_PREDICTED(TRACING_STORE_ATTR); + } + assert(PyDict_CheckExact((PyObject *)dict)); + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); + if (hint >= (size_t)dict->ma_keys->dk_nentries || + !DK_IS_UNICODE(dict->ma_keys)) { + UNLOCK_OBJECT(dict); + if (true) { + UPDATE_MISS_STATS(STORE_ATTR); + assert(_PyOpcode_Deopt[opcode] == (STORE_ATTR)); + JUMP_TO_PREDICTED(TRACING_STORE_ATTR); + } + } + PyDictUnicodeEntry *ep = DK_UNICODE_ENTRIES(dict->ma_keys) + hint; + if (ep->me_key != name) { + UNLOCK_OBJECT(dict); + if (true) { + UPDATE_MISS_STATS(STORE_ATTR); + assert(_PyOpcode_Deopt[opcode] == (STORE_ATTR)); + JUMP_TO_PREDICTED(TRACING_STORE_ATTR); + } + } + PyObject *old_value = ep->me_value; + if (old_value == NULL) { + UNLOCK_OBJECT(dict); + if (true) { + UPDATE_MISS_STATS(STORE_ATTR); + assert(_PyOpcode_Deopt[opcode] == (STORE_ATTR)); + JUMP_TO_PREDICTED(TRACING_STORE_ATTR); + } + } + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyDict_NotifyEvent(tstate->interp, PyDict_EVENT_MODIFIED, dict, name, PyStackRef_AsPyObjectBorrow(value)); + stack_pointer = _PyFrame_GetStackPointer(frame); + FT_ATOMIC_STORE_PTR_RELEASE(ep->me_value, PyStackRef_AsPyObjectSteal(value)); + UNLOCK_OBJECT(dict); + STAT_INC(STORE_ATTR, hit); + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(owner); + Py_XDECREF(old_value); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + TRACING_DISPATCH(); + } + + TRACING_TARGET(STORE_DEREF) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = STORE_DEREF; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(STORE_DEREF); + opcode = STORE_DEREF; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + _PyStackRef v; + v = stack_pointer[-1]; + PyCellObject *cell = (PyCellObject *)PyStackRef_AsPyObjectBorrow(GETLOCAL(oparg)); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyCell_SetTakeRef(cell, PyStackRef_AsPyObjectSteal(v)); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + TRACING_DISPATCH(); + } + + TRACING_TARGET(STORE_FAST) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = STORE_FAST; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(STORE_FAST); + opcode = STORE_FAST; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + _PyStackRef value; + value = stack_pointer[-1]; + _PyStackRef tmp = GETLOCAL(oparg); + GETLOCAL(oparg) = value; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_XCLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); + TRACING_DISPATCH(); + } + + TRACING_TARGET(STORE_FAST_LOAD_FAST) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = STORE_FAST_LOAD_FAST; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(STORE_FAST_LOAD_FAST); + opcode = STORE_FAST_LOAD_FAST; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + _PyStackRef value1; + _PyStackRef value2; + value1 = stack_pointer[-1]; + uint32_t oparg1 = oparg >> 4; + uint32_t oparg2 = oparg & 15; + _PyStackRef tmp = GETLOCAL(oparg1); + GETLOCAL(oparg1) = value1; + value2 = PyStackRef_DUP(GETLOCAL(oparg2)); + stack_pointer[-1] = value2; + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_XCLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); + TRACING_DISPATCH(); + } + + TRACING_TARGET(STORE_FAST_STORE_FAST) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = STORE_FAST_STORE_FAST; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(STORE_FAST_STORE_FAST); + opcode = STORE_FAST_STORE_FAST; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + _PyStackRef value2; + _PyStackRef value1; + value1 = stack_pointer[-1]; + value2 = stack_pointer[-2]; + uint32_t oparg1 = oparg >> 4; + uint32_t oparg2 = oparg & 15; + _PyStackRef tmp = GETLOCAL(oparg1); + GETLOCAL(oparg1) = value1; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_XCLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); + tmp = GETLOCAL(oparg2); + GETLOCAL(oparg2) = value2; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_XCLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); + TRACING_DISPATCH(); + } + + TRACING_TARGET(STORE_GLOBAL) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = STORE_GLOBAL; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(STORE_GLOBAL); + opcode = STORE_GLOBAL; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + _PyStackRef v; + v = stack_pointer[-1]; + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = PyDict_SetItem(GLOBALS(), name, PyStackRef_AsPyObjectBorrow(v)); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(v); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err) { + TRACING_JUMP_TO_LABEL(error); + } + TRACING_DISPATCH(); + } + + TRACING_TARGET(STORE_NAME) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = STORE_NAME; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(STORE_NAME); + opcode = STORE_NAME; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + _PyStackRef v; + v = stack_pointer[-1]; + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); + PyObject *ns = LOCALS(); + int err; + if (ns == NULL) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyErr_Format(tstate, PyExc_SystemError, + "no locals found when storing %R", name); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(v); + stack_pointer = _PyFrame_GetStackPointer(frame); + TRACING_JUMP_TO_LABEL(error); + } + if (PyDict_CheckExact(ns)) { + _PyFrame_SetStackPointer(frame, stack_pointer); + err = PyDict_SetItem(ns, name, PyStackRef_AsPyObjectBorrow(v)); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + else { + _PyFrame_SetStackPointer(frame, stack_pointer); + err = PyObject_SetItem(ns, name, PyStackRef_AsPyObjectBorrow(v)); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(v); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err) { + TRACING_JUMP_TO_LABEL(error); + } + TRACING_DISPATCH(); + } + + TRACING_TARGET(STORE_SLICE) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = STORE_SLICE; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(STORE_SLICE); + opcode = STORE_SLICE; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + _PyStackRef v; + _PyStackRef container; + _PyStackRef start; + _PyStackRef stop; + // _SPECIALIZE_STORE_SLICE + { + #if ENABLE_SPECIALIZATION + OPCODE_DEFERRED_INC(STORE_SLICE); + #endif /* ENABLE_SPECIALIZATION */ + } + // _STORE_SLICE + { + stop = stack_pointer[-1]; + start = stack_pointer[-2]; + container = stack_pointer[-3]; + v = stack_pointer[-4]; + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *slice = _PyBuildSlice_ConsumeRefs(PyStackRef_AsPyObjectSteal(start), + PyStackRef_AsPyObjectSteal(stop)); + stack_pointer = _PyFrame_GetStackPointer(frame); + int err; + if (slice == NULL) { + err = 1; + } + else { + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + err = PyObject_SetItem(PyStackRef_AsPyObjectBorrow(container), slice, PyStackRef_AsPyObjectBorrow(v)); + Py_DECREF(slice); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += 2; + } + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyStackRef tmp = container; + container = PyStackRef_NULL; + stack_pointer[-3] = container; + PyStackRef_CLOSE(tmp); + tmp = v; + v = PyStackRef_NULL; + stack_pointer[-4] = v; + PyStackRef_CLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -4; + assert(WITHIN_STACK_BOUNDS()); + if (err) { + TRACING_JUMP_TO_LABEL(error); + } + } + TRACING_DISPATCH(); + } + + TRACING_TARGET(STORE_SUBSCR) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = STORE_SUBSCR; + (void)(opcode); + #endif + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(STORE_SUBSCR); + PREDICTED_TRACING_STORE_SUBSCR:; + _Py_CODEUNIT* const this_instr = next_instr - 2; + (void)this_instr; + opcode = STORE_SUBSCR; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + _PyStackRef container; + _PyStackRef sub; + _PyStackRef v; + // _SPECIALIZE_STORE_SUBSCR + { + sub = stack_pointer[-1]; + container = stack_pointer[-2]; + uint16_t counter = read_u16(&this_instr[1].cache); + (void)counter; + #if ENABLE_SPECIALIZATION_FT + if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { + next_instr = this_instr; + _PyFrame_SetStackPointer(frame, stack_pointer); + _Py_Specialize_StoreSubscr(container, sub, next_instr); + stack_pointer = _PyFrame_GetStackPointer(frame); + DISPATCH_SAME_OPARG(); + } + OPCODE_DEFERRED_INC(STORE_SUBSCR); + ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); + #endif /* ENABLE_SPECIALIZATION_FT */ + } + // _STORE_SUBSCR + { + v = stack_pointer[-3]; + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = PyObject_SetItem(PyStackRef_AsPyObjectBorrow(container), PyStackRef_AsPyObjectBorrow(sub), PyStackRef_AsPyObjectBorrow(v)); + _PyStackRef tmp = sub; + sub = PyStackRef_NULL; + stack_pointer[-1] = sub; + PyStackRef_CLOSE(tmp); + tmp = container; + container = PyStackRef_NULL; + stack_pointer[-2] = container; + PyStackRef_CLOSE(tmp); + tmp = v; + v = PyStackRef_NULL; + stack_pointer[-3] = v; + PyStackRef_CLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -3; + assert(WITHIN_STACK_BOUNDS()); + if (err) { + TRACING_JUMP_TO_LABEL(error); + } + } + TRACING_DISPATCH(); + } + + TRACING_TARGET(STORE_SUBSCR_DICT) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = STORE_SUBSCR_DICT; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(STORE_SUBSCR_DICT); + opcode = STORE_SUBSCR_DICT; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + static_assert(INLINE_CACHE_ENTRIES_STORE_SUBSCR == 1, "incorrect cache size"); + _PyStackRef nos; + _PyStackRef value; + _PyStackRef dict_st; + _PyStackRef sub; + // _GUARD_NOS_DICT + { + nos = stack_pointer[-2]; + PyObject *o = PyStackRef_AsPyObjectBorrow(nos); + if (!PyDict_CheckExact(o)) { + UPDATE_MISS_STATS(STORE_SUBSCR); + assert(_PyOpcode_Deopt[opcode] == (STORE_SUBSCR)); + JUMP_TO_PREDICTED(TRACING_STORE_SUBSCR); + } + } + /* Skip 1 cache entry */ + // _STORE_SUBSCR_DICT + { + sub = stack_pointer[-1]; + dict_st = nos; + value = stack_pointer[-3]; + PyObject *dict = PyStackRef_AsPyObjectBorrow(dict_st); + assert(PyDict_CheckExact(dict)); + STAT_INC(STORE_SUBSCR, hit); + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = _PyDict_SetItem_Take2((PyDictObject *)dict, + PyStackRef_AsPyObjectSteal(sub), + PyStackRef_AsPyObjectSteal(value)); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -3; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(dict_st); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err) { + TRACING_JUMP_TO_LABEL(error); + } + } + TRACING_DISPATCH(); + } + + TRACING_TARGET(STORE_SUBSCR_LIST_INT) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = STORE_SUBSCR_LIST_INT; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(STORE_SUBSCR_LIST_INT); + opcode = STORE_SUBSCR_LIST_INT; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + static_assert(INLINE_CACHE_ENTRIES_STORE_SUBSCR == 1, "incorrect cache size"); + _PyStackRef value; + _PyStackRef nos; + _PyStackRef list_st; + _PyStackRef sub_st; + // _GUARD_TOS_INT + { + value = stack_pointer[-1]; + PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); + if (!_PyLong_CheckExactAndCompact(value_o)) { + UPDATE_MISS_STATS(STORE_SUBSCR); + assert(_PyOpcode_Deopt[opcode] == (STORE_SUBSCR)); + JUMP_TO_PREDICTED(TRACING_STORE_SUBSCR); + } + } + // _GUARD_NOS_LIST + { + nos = stack_pointer[-2]; + PyObject *o = PyStackRef_AsPyObjectBorrow(nos); + if (!PyList_CheckExact(o)) { + UPDATE_MISS_STATS(STORE_SUBSCR); + assert(_PyOpcode_Deopt[opcode] == (STORE_SUBSCR)); + JUMP_TO_PREDICTED(TRACING_STORE_SUBSCR); + } + } + /* Skip 1 cache entry */ + // _STORE_SUBSCR_LIST_INT + { + sub_st = value; + list_st = nos; + value = stack_pointer[-3]; + PyObject *sub = PyStackRef_AsPyObjectBorrow(sub_st); + PyObject *list = PyStackRef_AsPyObjectBorrow(list_st); + assert(PyLong_CheckExact(sub)); + assert(PyList_CheckExact(list)); + if (!_PyLong_IsNonNegativeCompact((PyLongObject *)sub)) { + UPDATE_MISS_STATS(STORE_SUBSCR); + assert(_PyOpcode_Deopt[opcode] == (STORE_SUBSCR)); + JUMP_TO_PREDICTED(TRACING_STORE_SUBSCR); + } + Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0]; + if (!LOCK_OBJECT(list)) { + UPDATE_MISS_STATS(STORE_SUBSCR); + assert(_PyOpcode_Deopt[opcode] == (STORE_SUBSCR)); + JUMP_TO_PREDICTED(TRACING_STORE_SUBSCR); + } + if (index >= PyList_GET_SIZE(list)) { + UNLOCK_OBJECT(list); + if (true) { + UPDATE_MISS_STATS(STORE_SUBSCR); + assert(_PyOpcode_Deopt[opcode] == (STORE_SUBSCR)); + JUMP_TO_PREDICTED(TRACING_STORE_SUBSCR); + } + } + STAT_INC(STORE_SUBSCR, hit); + PyObject *old_value = PyList_GET_ITEM(list, index); + FT_ATOMIC_STORE_PTR_RELEASE(_PyList_ITEMS(list)[index], + PyStackRef_AsPyObjectSteal(value)); + assert(old_value != NULL); + UNLOCK_OBJECT(list); + PyStackRef_CLOSE_SPECIALIZED(sub_st, _PyLong_ExactDealloc); + stack_pointer += -3; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(list_st); + Py_DECREF(old_value); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + TRACING_DISPATCH(); + } + + TRACING_TARGET(SWAP) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = SWAP; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(SWAP); + opcode = SWAP; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + _PyStackRef bottom; + _PyStackRef top; + top = stack_pointer[-1]; + bottom = stack_pointer[-2 - (oparg-2)]; + _PyStackRef temp = bottom; + bottom = top; + top = temp; + stack_pointer[-2 - (oparg-2)] = bottom; + stack_pointer[-1] = top; + TRACING_DISPATCH(); + } + + TRACING_TARGET(TO_BOOL) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = TO_BOOL; + (void)(opcode); + #endif + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(TO_BOOL); + PREDICTED_TRACING_TO_BOOL:; + _Py_CODEUNIT* const this_instr = next_instr - 4; + (void)this_instr; + opcode = TO_BOOL; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + _PyStackRef value; + _PyStackRef res; + // _SPECIALIZE_TO_BOOL + { + value = stack_pointer[-1]; + uint16_t counter = read_u16(&this_instr[1].cache); + (void)counter; + #if ENABLE_SPECIALIZATION_FT + if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { + next_instr = this_instr; + _PyFrame_SetStackPointer(frame, stack_pointer); + _Py_Specialize_ToBool(value, next_instr); + stack_pointer = _PyFrame_GetStackPointer(frame); + DISPATCH_SAME_OPARG(); + } + OPCODE_DEFERRED_INC(TO_BOOL); + ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); + #endif /* ENABLE_SPECIALIZATION_FT */ + } + /* Skip 2 cache entries */ + // _TO_BOOL + { + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = PyObject_IsTrue(PyStackRef_AsPyObjectBorrow(value)); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(value); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err < 0) { + TRACING_JUMP_TO_LABEL(error); + } + res = err ? PyStackRef_True : PyStackRef_False; + } + stack_pointer[0] = res; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + TRACING_DISPATCH(); + } + + TRACING_TARGET(TO_BOOL_ALWAYS_TRUE) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = TO_BOOL_ALWAYS_TRUE; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(TO_BOOL_ALWAYS_TRUE); + opcode = TO_BOOL_ALWAYS_TRUE; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + static_assert(INLINE_CACHE_ENTRIES_TO_BOOL == 3, "incorrect cache size"); + _PyStackRef owner; + _PyStackRef value; + _PyStackRef res; + /* Skip 1 cache entry */ + // _GUARD_TYPE_VERSION + { + owner = stack_pointer[-1]; + uint32_t type_version = read_u32(&this_instr[2].cache); + PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); + assert(type_version != 0); + if (FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version) { + UPDATE_MISS_STATS(TO_BOOL); + assert(_PyOpcode_Deopt[opcode] == (TO_BOOL)); + JUMP_TO_PREDICTED(TRACING_TO_BOOL); + } + } + // _REPLACE_WITH_TRUE + { + value = owner; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(value); + stack_pointer = _PyFrame_GetStackPointer(frame); + res = PyStackRef_True; + } + stack_pointer[0] = res; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + TRACING_DISPATCH(); + } + + TRACING_TARGET(TO_BOOL_BOOL) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = TO_BOOL_BOOL; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(TO_BOOL_BOOL); + opcode = TO_BOOL_BOOL; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + static_assert(INLINE_CACHE_ENTRIES_TO_BOOL == 3, "incorrect cache size"); + _PyStackRef value; + /* Skip 1 cache entry */ + /* Skip 2 cache entries */ + value = stack_pointer[-1]; + if (!PyStackRef_BoolCheck(value)) { + UPDATE_MISS_STATS(TO_BOOL); + assert(_PyOpcode_Deopt[opcode] == (TO_BOOL)); + JUMP_TO_PREDICTED(TRACING_TO_BOOL); + } + STAT_INC(TO_BOOL, hit); + TRACING_DISPATCH(); + } + + TRACING_TARGET(TO_BOOL_INT) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = TO_BOOL_INT; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(TO_BOOL_INT); + opcode = TO_BOOL_INT; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + static_assert(INLINE_CACHE_ENTRIES_TO_BOOL == 3, "incorrect cache size"); + _PyStackRef value; + _PyStackRef res; + /* Skip 1 cache entry */ + /* Skip 2 cache entries */ + value = stack_pointer[-1]; + PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); + if (!PyLong_CheckExact(value_o)) { + UPDATE_MISS_STATS(TO_BOOL); + assert(_PyOpcode_Deopt[opcode] == (TO_BOOL)); + JUMP_TO_PREDICTED(TRACING_TO_BOOL); + } + STAT_INC(TO_BOOL, hit); + if (_PyLong_IsZero((PyLongObject *)value_o)) { + assert(_Py_IsImmortal(value_o)); + res = PyStackRef_False; + } + else { + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(value); + stack_pointer = _PyFrame_GetStackPointer(frame); + res = PyStackRef_True; + stack_pointer += 1; + } + stack_pointer[-1] = res; + TRACING_DISPATCH(); + } + + TRACING_TARGET(TO_BOOL_LIST) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = TO_BOOL_LIST; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(TO_BOOL_LIST); + opcode = TO_BOOL_LIST; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + static_assert(INLINE_CACHE_ENTRIES_TO_BOOL == 3, "incorrect cache size"); + _PyStackRef tos; + _PyStackRef value; + _PyStackRef res; + // _GUARD_TOS_LIST + { + tos = stack_pointer[-1]; + PyObject *o = PyStackRef_AsPyObjectBorrow(tos); + if (!PyList_CheckExact(o)) { + UPDATE_MISS_STATS(TO_BOOL); + assert(_PyOpcode_Deopt[opcode] == (TO_BOOL)); + JUMP_TO_PREDICTED(TRACING_TO_BOOL); + } + } + /* Skip 1 cache entry */ + /* Skip 2 cache entries */ + // _TO_BOOL_LIST + { + value = tos; + PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); + assert(PyList_CheckExact(value_o)); + STAT_INC(TO_BOOL, hit); + res = PyList_GET_SIZE(value_o) ? PyStackRef_True : PyStackRef_False; + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyStackRef tmp = value; + value = res; + stack_pointer[-1] = value; + PyStackRef_CLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + TRACING_DISPATCH(); + } + + TRACING_TARGET(TO_BOOL_NONE) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = TO_BOOL_NONE; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(TO_BOOL_NONE); + opcode = TO_BOOL_NONE; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + static_assert(INLINE_CACHE_ENTRIES_TO_BOOL == 3, "incorrect cache size"); + _PyStackRef value; + _PyStackRef res; + /* Skip 1 cache entry */ + /* Skip 2 cache entries */ + value = stack_pointer[-1]; + if (!PyStackRef_IsNone(value)) { + UPDATE_MISS_STATS(TO_BOOL); + assert(_PyOpcode_Deopt[opcode] == (TO_BOOL)); + JUMP_TO_PREDICTED(TRACING_TO_BOOL); + } + STAT_INC(TO_BOOL, hit); + res = PyStackRef_False; + stack_pointer[-1] = res; + TRACING_DISPATCH(); + } + + TRACING_TARGET(TO_BOOL_STR) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = TO_BOOL_STR; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(TO_BOOL_STR); + opcode = TO_BOOL_STR; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + static_assert(INLINE_CACHE_ENTRIES_TO_BOOL == 3, "incorrect cache size"); + _PyStackRef value; + _PyStackRef res; + // _GUARD_TOS_UNICODE + { + value = stack_pointer[-1]; + PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); + if (!PyUnicode_CheckExact(value_o)) { + UPDATE_MISS_STATS(TO_BOOL); + assert(_PyOpcode_Deopt[opcode] == (TO_BOOL)); + JUMP_TO_PREDICTED(TRACING_TO_BOOL); + } + } + /* Skip 1 cache entry */ + /* Skip 2 cache entries */ + // _TO_BOOL_STR + { + STAT_INC(TO_BOOL, hit); + PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); + if (value_o == &_Py_STR(empty)) { + assert(_Py_IsImmortal(value_o)); + res = PyStackRef_False; + } + else { + assert(Py_SIZE(value_o)); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(value); + stack_pointer = _PyFrame_GetStackPointer(frame); + res = PyStackRef_True; + stack_pointer += 1; + } + } + stack_pointer[-1] = res; + TRACING_DISPATCH(); + } + + TRACING_TARGET(UNARY_INVERT) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = UNARY_INVERT; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(UNARY_INVERT); + opcode = UNARY_INVERT; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + _PyStackRef value; + _PyStackRef res; + value = stack_pointer[-1]; + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *res_o = PyNumber_Invert(PyStackRef_AsPyObjectBorrow(value)); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(value); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (res_o == NULL) { + TRACING_JUMP_TO_LABEL(error); + } + res = PyStackRef_FromPyObjectSteal(res_o); + stack_pointer[0] = res; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + TRACING_DISPATCH(); + } + + TRACING_TARGET(UNARY_NEGATIVE) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = UNARY_NEGATIVE; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(UNARY_NEGATIVE); + opcode = UNARY_NEGATIVE; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + _PyStackRef value; + _PyStackRef res; + value = stack_pointer[-1]; + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *res_o = PyNumber_Negative(PyStackRef_AsPyObjectBorrow(value)); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(value); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (res_o == NULL) { + TRACING_JUMP_TO_LABEL(error); + } + res = PyStackRef_FromPyObjectSteal(res_o); + stack_pointer[0] = res; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + TRACING_DISPATCH(); + } + + TRACING_TARGET(UNARY_NOT) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = UNARY_NOT; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(UNARY_NOT); + opcode = UNARY_NOT; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + _PyStackRef value; + _PyStackRef res; + value = stack_pointer[-1]; + assert(PyStackRef_BoolCheck(value)); + res = PyStackRef_IsFalse(value) + ? PyStackRef_True : PyStackRef_False; + stack_pointer[-1] = res; + TRACING_DISPATCH(); + } + + TRACING_TARGET(UNPACK_EX) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = UNPACK_EX; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(UNPACK_EX); + opcode = UNPACK_EX; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + _PyStackRef seq; + _PyStackRef *top; + seq = stack_pointer[-1]; + top = &stack_pointer[(oparg & 0xFF) + (oparg >> 8)]; + PyObject *seq_o = PyStackRef_AsPyObjectSteal(seq); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + int res = _PyEval_UnpackIterableStackRef(tstate, seq_o, oparg & 0xFF, oparg >> 8, top); + Py_DECREF(seq_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (res == 0) { + TRACING_JUMP_TO_LABEL(error); + } + stack_pointer += 1 + (oparg & 0xFF) + (oparg >> 8); + assert(WITHIN_STACK_BOUNDS()); + TRACING_DISPATCH(); + } + + TRACING_TARGET(UNPACK_SEQUENCE) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = UNPACK_SEQUENCE; + (void)(opcode); + #endif + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(UNPACK_SEQUENCE); + PREDICTED_TRACING_UNPACK_SEQUENCE:; + _Py_CODEUNIT* const this_instr = next_instr - 2; + (void)this_instr; + opcode = UNPACK_SEQUENCE; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + _PyStackRef seq; + _PyStackRef *top; + // _SPECIALIZE_UNPACK_SEQUENCE + { + seq = stack_pointer[-1]; + uint16_t counter = read_u16(&this_instr[1].cache); + (void)counter; + #if ENABLE_SPECIALIZATION_FT + if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { + next_instr = this_instr; + _PyFrame_SetStackPointer(frame, stack_pointer); + _Py_Specialize_UnpackSequence(seq, next_instr, oparg); + stack_pointer = _PyFrame_GetStackPointer(frame); + DISPATCH_SAME_OPARG(); + } + OPCODE_DEFERRED_INC(UNPACK_SEQUENCE); + ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); + #endif /* ENABLE_SPECIALIZATION_FT */ + (void)seq; + (void)counter; + } + // _UNPACK_SEQUENCE + { + top = &stack_pointer[-1 + oparg]; + PyObject *seq_o = PyStackRef_AsPyObjectSteal(seq); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + int res = _PyEval_UnpackIterableStackRef(tstate, seq_o, oparg, -1, top); + Py_DECREF(seq_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (res == 0) { + TRACING_JUMP_TO_LABEL(error); + } + } + stack_pointer += oparg; + assert(WITHIN_STACK_BOUNDS()); + TRACING_DISPATCH(); + } + + TRACING_TARGET(UNPACK_SEQUENCE_LIST) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = UNPACK_SEQUENCE_LIST; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(UNPACK_SEQUENCE_LIST); + opcode = UNPACK_SEQUENCE_LIST; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + static_assert(INLINE_CACHE_ENTRIES_UNPACK_SEQUENCE == 1, "incorrect cache size"); + _PyStackRef tos; + _PyStackRef seq; + _PyStackRef *values; + // _GUARD_TOS_LIST + { + tos = stack_pointer[-1]; + PyObject *o = PyStackRef_AsPyObjectBorrow(tos); + if (!PyList_CheckExact(o)) { + UPDATE_MISS_STATS(UNPACK_SEQUENCE); + assert(_PyOpcode_Deopt[opcode] == (UNPACK_SEQUENCE)); + JUMP_TO_PREDICTED(TRACING_UNPACK_SEQUENCE); + } + } + /* Skip 1 cache entry */ + // _UNPACK_SEQUENCE_LIST + { + seq = tos; + values = &stack_pointer[-1]; + PyObject *seq_o = PyStackRef_AsPyObjectBorrow(seq); + assert(PyList_CheckExact(seq_o)); + if (!LOCK_OBJECT(seq_o)) { + UPDATE_MISS_STATS(UNPACK_SEQUENCE); + assert(_PyOpcode_Deopt[opcode] == (UNPACK_SEQUENCE)); + JUMP_TO_PREDICTED(TRACING_UNPACK_SEQUENCE); + } + if (PyList_GET_SIZE(seq_o) != oparg) { + UNLOCK_OBJECT(seq_o); + if (true) { + UPDATE_MISS_STATS(UNPACK_SEQUENCE); + assert(_PyOpcode_Deopt[opcode] == (UNPACK_SEQUENCE)); + JUMP_TO_PREDICTED(TRACING_UNPACK_SEQUENCE); + } + } + STAT_INC(UNPACK_SEQUENCE, hit); + PyObject **items = _PyList_ITEMS(seq_o); + for (int i = oparg; --i >= 0; ) { + *values++ = PyStackRef_FromPyObjectNew(items[i]); + } + UNLOCK_OBJECT(seq_o); + stack_pointer += -1 + oparg; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(seq); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + TRACING_DISPATCH(); + } + + TRACING_TARGET(UNPACK_SEQUENCE_TUPLE) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = UNPACK_SEQUENCE_TUPLE; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(UNPACK_SEQUENCE_TUPLE); + opcode = UNPACK_SEQUENCE_TUPLE; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + static_assert(INLINE_CACHE_ENTRIES_UNPACK_SEQUENCE == 1, "incorrect cache size"); + _PyStackRef tos; + _PyStackRef seq; + _PyStackRef *values; + // _GUARD_TOS_TUPLE + { + tos = stack_pointer[-1]; + PyObject *o = PyStackRef_AsPyObjectBorrow(tos); + if (!PyTuple_CheckExact(o)) { + UPDATE_MISS_STATS(UNPACK_SEQUENCE); + assert(_PyOpcode_Deopt[opcode] == (UNPACK_SEQUENCE)); + JUMP_TO_PREDICTED(TRACING_UNPACK_SEQUENCE); + } + } + /* Skip 1 cache entry */ + // _UNPACK_SEQUENCE_TUPLE + { + seq = tos; + values = &stack_pointer[-1]; + PyObject *seq_o = PyStackRef_AsPyObjectBorrow(seq); + assert(PyTuple_CheckExact(seq_o)); + if (PyTuple_GET_SIZE(seq_o) != oparg) { + UPDATE_MISS_STATS(UNPACK_SEQUENCE); + assert(_PyOpcode_Deopt[opcode] == (UNPACK_SEQUENCE)); + JUMP_TO_PREDICTED(TRACING_UNPACK_SEQUENCE); + } + STAT_INC(UNPACK_SEQUENCE, hit); + PyObject **items = _PyTuple_ITEMS(seq_o); + for (int i = oparg; --i >= 0; ) { + *values++ = PyStackRef_FromPyObjectNew(items[i]); + } + stack_pointer += -1 + oparg; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(seq); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + TRACING_DISPATCH(); + } + + TRACING_TARGET(UNPACK_SEQUENCE_TWO_TUPLE) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = UNPACK_SEQUENCE_TWO_TUPLE; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(UNPACK_SEQUENCE_TWO_TUPLE); + opcode = UNPACK_SEQUENCE_TWO_TUPLE; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + static_assert(INLINE_CACHE_ENTRIES_UNPACK_SEQUENCE == 1, "incorrect cache size"); + _PyStackRef tos; + _PyStackRef seq; + _PyStackRef val1; + _PyStackRef val0; + // _GUARD_TOS_TUPLE + { + tos = stack_pointer[-1]; + PyObject *o = PyStackRef_AsPyObjectBorrow(tos); + if (!PyTuple_CheckExact(o)) { + UPDATE_MISS_STATS(UNPACK_SEQUENCE); + assert(_PyOpcode_Deopt[opcode] == (UNPACK_SEQUENCE)); + JUMP_TO_PREDICTED(TRACING_UNPACK_SEQUENCE); + } + } + /* Skip 1 cache entry */ + // _UNPACK_SEQUENCE_TWO_TUPLE + { + seq = tos; + assert(oparg == 2); + PyObject *seq_o = PyStackRef_AsPyObjectBorrow(seq); + assert(PyTuple_CheckExact(seq_o)); + if (PyTuple_GET_SIZE(seq_o) != 2) { + UPDATE_MISS_STATS(UNPACK_SEQUENCE); + assert(_PyOpcode_Deopt[opcode] == (UNPACK_SEQUENCE)); + JUMP_TO_PREDICTED(TRACING_UNPACK_SEQUENCE); + } + STAT_INC(UNPACK_SEQUENCE, hit); + val0 = PyStackRef_FromPyObjectNew(PyTuple_GET_ITEM(seq_o, 0)); + val1 = PyStackRef_FromPyObjectNew(PyTuple_GET_ITEM(seq_o, 1)); + stack_pointer[-1] = val1; + stack_pointer[0] = val0; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(seq); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + TRACING_DISPATCH(); + } + + TRACING_TARGET(WITH_EXCEPT_START) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = WITH_EXCEPT_START; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(WITH_EXCEPT_START); + opcode = WITH_EXCEPT_START; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + _PyStackRef exit_func; + _PyStackRef exit_self; + _PyStackRef lasti; + _PyStackRef val; + _PyStackRef res; + val = stack_pointer[-1]; + lasti = stack_pointer[-3]; + exit_self = stack_pointer[-4]; + exit_func = stack_pointer[-5]; + PyObject *exc, *tb; + PyObject *val_o = PyStackRef_AsPyObjectBorrow(val); + PyObject *exit_func_o = PyStackRef_AsPyObjectBorrow(exit_func); + assert(val_o && PyExceptionInstance_Check(val_o)); + exc = PyExceptionInstance_Class(val_o); + PyObject *original_tb = tb = PyException_GetTraceback(val_o); + if (tb == NULL) { + tb = Py_None; + } + assert(PyStackRef_IsTaggedInt(lasti)); + (void)lasti; + PyObject *stack[5] = {NULL, PyStackRef_AsPyObjectBorrow(exit_self), exc, val_o, tb}; + int has_self = !PyStackRef_IsNull(exit_self); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *res_o = PyObject_Vectorcall(exit_func_o, stack + 2 - has_self, + (3 + has_self) | PY_VECTORCALL_ARGUMENTS_OFFSET, NULL); + Py_XDECREF(original_tb); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (res_o == NULL) { + TRACING_JUMP_TO_LABEL(error); + } + res = PyStackRef_FromPyObjectSteal(res_o); + stack_pointer[0] = res; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + TRACING_DISPATCH(); + } + + TRACING_TARGET(YIELD_VALUE) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = YIELD_VALUE; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(YIELD_VALUE); + opcode = YIELD_VALUE; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + _PyStackRef retval; + _PyStackRef value; + retval = stack_pointer[-1]; + assert(frame->owner != FRAME_OWNED_BY_INTERPRETER); + frame->instr_ptr++; + PyGenObject *gen = _PyGen_GetGeneratorFromFrame(frame); + assert(FRAME_SUSPENDED_YIELD_FROM == FRAME_SUSPENDED + 1); + assert(oparg == 0 || oparg == 1); + gen->gi_frame_state = FRAME_SUSPENDED + oparg; + _PyStackRef temp = retval; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + tstate->exc_info = gen->gi_exc_state.previous_item; + gen->gi_exc_state.previous_item = NULL; + _Py_LeaveRecursiveCallPy(tstate); + _PyInterpreterFrame *gen_frame = frame; + frame = tstate->current_frame = frame->previous; + gen_frame->previous = NULL; + assert(INLINE_CACHE_ENTRIES_SEND == INLINE_CACHE_ENTRIES_FOR_ITER); + #if TIER_ONE + assert(frame->instr_ptr->op.code == INSTRUMENTED_LINE || + frame->instr_ptr->op.code == INSTRUMENTED_INSTRUCTION || + _PyOpcode_Deopt[frame->instr_ptr->op.code] == SEND || + _PyOpcode_Deopt[frame->instr_ptr->op.code] == FOR_ITER || + _PyOpcode_Deopt[frame->instr_ptr->op.code] == INTERPRETER_EXIT || + _PyOpcode_Deopt[frame->instr_ptr->op.code] == ENTER_EXECUTOR); + #endif + stack_pointer = _PyFrame_GetStackPointer(frame); + LOAD_IP(1 + INLINE_CACHE_ENTRIES_SEND); + #if TIER_TWO + frame->instr_ptr += (1 + INLINE_CACHE_ENTRIES_SEND); + #endif + value = PyStackRef_MakeHeapSafe(temp); + LLTRACE_RESUME_FRAME(); + stack_pointer[0] = value; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + TRACING_DISPATCH(); + } + #endif /* END TRACING INSTRUCTIONS */ + #undef TRACING_JIT + #undef TIER_ONE diff --git a/Tools/cases_generator/tier1_generator.py b/Tools/cases_generator/tier1_generator.py index 05243195807c..939035332cba 100644 --- a/Tools/cases_generator/tier1_generator.py +++ b/Tools/cases_generator/tier1_generator.py @@ -5,7 +5,6 @@ Writes the cases to generated_cases.c.h, which is #included in ceval.c. import argparse -from tracer_generator import generate_tracer_cases from analyzer import ( Analysis, Instruction, @@ -171,7 +170,6 @@ def generate_tier1( out = CWriter(outfile, 2, lines) emitter = Emitter(out, analysis.labels) generate_tier1_cases(analysis, out, emitter) - generate_tracer_cases(analysis, out) outfile.write(f""" {INSTRUCTION_END_MARKER} #if !_Py_TAIL_CALL_INTERP @@ -256,12 +254,12 @@ def generate_tier1_cases( if is_tracing: # This is required so that the predicted ops reflect the correct opcode. out.emit(f"opcode = {name};\n") - out.emit(f"PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable);\n") - out.emit(f"(void)old_code;\n") - out.emit(f"PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj);\n") - out.emit(f"(void)old_func;\n") - out.emit(f"int _jump_taken = false;\n") - out.emit(f"(void)_jump_taken;\n") + out.emit(f"PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable);\n") + out.emit(f"(void)old_code;\n") + out.emit(f"PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj);\n") + out.emit(f"(void)old_func;\n") + out.emit(f"int _jump_taken = false;\n") + out.emit(f"(void)_jump_taken;\n") if inst.properties.uses_opcode: out.emit(f"opcode = {name};\n") if inst.family is not None: diff --git a/Tools/cases_generator/tracer_generator.py b/Tools/cases_generator/tracer_generator.py index 61088ff185e2..ebc9a63664e8 100644 --- a/Tools/cases_generator/tracer_generator.py +++ b/Tools/cases_generator/tracer_generator.py @@ -28,6 +28,9 @@ from cwriter import CWriter from typing import TextIO from lexer import Token from stack import Local, Stack, StackError, get_stack_effect, Storage +from tier1_generator import generate_tier1_cases + +DEFAULT_OUTPUT = ROOT / "Python/generated_tracer_cases.c.h" class TracerEmitter(Emitter): out: CWriter @@ -115,8 +118,51 @@ class TracerEmitter(Emitter): def generate_tracer_cases( analysis: Analysis, out: CWriter ): - # Import inside to avoid circular imports. - from tier1_generator import generate_tier1_cases out.emit(f"#ifdef _Py_TIER2 /* BEGIN TRACING INSTRUCTIONS */\n") generate_tier1_cases(analysis, out, TracerEmitter(out, analysis.labels), is_tracing=True) - out.emit(f"#endif /* END TRACING INSTRUCTIONS */\n") \ No newline at end of file + out.emit(f"#endif /* END TRACING INSTRUCTIONS */\n") + +def generate_tracer( + filenames: list[str], analysis: Analysis, outfile: TextIO, lines: bool +) -> None: + write_header(__file__, filenames, outfile) + out = CWriter(outfile, 2, lines) + out.emit("#define TIER_ONE 1\n") + out.emit("#define TRACING_JIT 1\n") + generate_tracer_cases(analysis, out) + out.emit("#undef TRACING_JIT\n") + out.emit("#undef TIER_ONE\n") + +# For use in unittest +def generate_tracer_from_files( + filenames: list[str], outfilename: str, lines: bool +) -> None: + data = analyze_files(filenames) + with open(outfilename, "w") as outfile: + generate_tracer(filenames, data, outfile, lines) + + +arg_parser = argparse.ArgumentParser( + description="Generate the code for the interpreter switch.", + formatter_class=argparse.ArgumentDefaultsHelpFormatter, +) + +arg_parser.add_argument( + "-o", "--output", type=str, help="Generated code", default=DEFAULT_OUTPUT +) + +arg_parser.add_argument( + "-l", "--emit-line-directives", help="Emit #line directives", action="store_true" +) + +arg_parser.add_argument( + "input", nargs=argparse.REMAINDER, help="Instruction definition file(s)" +) + +if __name__ == "__main__": + args = arg_parser.parse_args() + if len(args.input) == 0: + args.input.append(DEFAULT_INPUT) + data = analyze_files(args.input) + with open(args.output, "w") as outfile: + generate_tracer(args.input, data, outfile, args.emit_line_directives) \ No newline at end of file