From: Mark Shannon Date: Thu, 18 Jun 2026 14:14:30 +0000 (+0100) Subject: GH-150516: Reduce the work done to spill and reload the stack around calls (GH-151587) X-Git-Url: http://git.ipfire.org/gitweb.cgi?a=commitdiff_plain;h=ad1513a263bc0b7831908d65a7bea869623abd9f;p=thirdparty%2FPython%2Fcpython.git GH-150516: Reduce the work done to spill and reload the stack around calls (GH-151587) --- diff --git a/Include/internal/pycore_interpframe.h b/Include/internal/pycore_interpframe.h index 0f4bf7d8a2f2..3608a7f3bce8 100644 --- a/Include/internal/pycore_interpframe.h +++ b/Include/internal/pycore_interpframe.h @@ -152,6 +152,7 @@ static inline void _PyFrame_Copy(_PyInterpreterFrame *src, _PyInterpreterFrame * // visited is GC bookkeeping for the current stack walk, not frame state. dest->visited = 0; #ifdef Py_DEBUG + dest->stackpointer_valid = src->stackpointer_valid; dest->lltrace = src->lltrace; #endif for (int i = 0; i < stacktop; i++) { @@ -207,6 +208,7 @@ _PyFrame_Initialize( frame->owner = FRAME_OWNED_BY_THREAD; frame->visited = 0; #ifdef Py_DEBUG + frame->stackpointer_valid = 1; frame->lltrace = 0; #endif @@ -230,26 +232,51 @@ _PyFrame_GetLocalsArray(_PyInterpreterFrame *frame) static inline _PyStackRef* _PyFrame_GetStackPointer(_PyInterpreterFrame *frame) { + return frame->stackpointer; +} + +static inline void +_PyFrame_SetStackPointer(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer) +{ + frame->stackpointer = stack_pointer; +} + +static inline void +_PyFrame_StackPointerValidate(_PyInterpreterFrame *frame) +{ +#ifdef Py_DEBUG +/* Avoid bloating the JIT code */ #ifndef _Py_JIT - assert(frame->stackpointer != NULL); + assert(frame->stackpointer_valid == 0); #endif - _PyStackRef *sp = frame->stackpointer; -#ifndef NDEBUG - frame->stackpointer = NULL; + frame->stackpointer_valid = 1; #endif - return sp; } static inline void -_PyFrame_SetStackPointer(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer) +_PyFrame_StackPointerInvalidate(_PyInterpreterFrame *frame) { +#ifdef Py_DEBUG /* Avoid bloating the JIT code */ #ifndef _Py_JIT - assert(frame->stackpointer == NULL); + assert(frame->stackpointer_valid == 1); +#endif + frame->stackpointer_valid = 0; #endif - frame->stackpointer = stack_pointer; } +static inline void +_PyFrame_StackAssertInvalid(_PyInterpreterFrame *frame) +{ +#ifdef Py_DEBUG +/* Avoid bloating the JIT code */ +#ifndef _Py_JIT + assert(frame->stackpointer_valid == 0); +#endif +#endif +} + + /* Determine whether a frame is incomplete. * A frame is incomplete if it is part way through * creating cell objects or a generator or coroutine. @@ -411,6 +438,7 @@ _PyFrame_PushTrampolineUnchecked(PyThreadState *tstate, PyCodeObject *code, int frame->owner = FRAME_OWNED_BY_THREAD; frame->visited = 0; #ifdef Py_DEBUG + frame->stackpointer_valid = 1; frame->lltrace = 0; #endif frame->return_offset = 0; diff --git a/Include/internal/pycore_interpframe_structs.h b/Include/internal/pycore_interpframe_structs.h index 38510685f409..4d267e35504b 100644 --- a/Include/internal/pycore_interpframe_structs.h +++ b/Include/internal/pycore_interpframe_structs.h @@ -44,7 +44,8 @@ struct _PyInterpreterFrame { char owner; #ifdef Py_DEBUG uint8_t visited:1; - uint8_t lltrace:7; + uint8_t stackpointer_valid:1; + uint8_t lltrace:6; #else uint8_t visited; #endif diff --git a/Include/internal/pycore_opcode_metadata.h b/Include/internal/pycore_opcode_metadata.h index 80ad440bac82..f145b7f034c5 100644 --- a/Include/internal/pycore_opcode_metadata.h +++ b/Include/internal/pycore_opcode_metadata.h @@ -1231,9 +1231,9 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[267] = { [INSTRUMENTED_POP_JUMP_IF_NONE] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_ESCAPES_FLAG }, [INSTRUMENTED_POP_JUMP_IF_NOT_NONE] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_ESCAPES_FLAG }, [INSTRUMENTED_POP_JUMP_IF_TRUE] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG }, - [INSTRUMENTED_RESUME] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, - [INSTRUMENTED_RETURN_VALUE] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG | HAS_NEEDS_GUARD_IP_FLAG }, - [INSTRUMENTED_YIELD_VALUE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG | HAS_NEEDS_GUARD_IP_FLAG }, + [INSTRUMENTED_RESUME] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG | HAS_SYNC_SP_FLAG }, + [INSTRUMENTED_RETURN_VALUE] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG | HAS_SYNC_SP_FLAG | HAS_NEEDS_GUARD_IP_FLAG }, + [INSTRUMENTED_YIELD_VALUE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG | HAS_SYNC_SP_FLAG | HAS_NEEDS_GUARD_IP_FLAG }, [INTERPRETER_EXIT] = { true, INSTR_FMT_IX, HAS_ESCAPES_FLAG }, [IS_OP] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ESCAPES_FLAG }, [JUMP_BACKWARD] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_EVAL_BREAK_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, @@ -1303,8 +1303,8 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[267] = { [RESUME] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, [RESUME_CHECK] = { true, INSTR_FMT_IXC, HAS_DEOPT_FLAG }, [RESUME_CHECK_JIT] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_DEOPT_FLAG }, - [RETURN_GENERATOR] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG | HAS_NEEDS_GUARD_IP_FLAG }, - [RETURN_VALUE] = { true, INSTR_FMT_IX, HAS_ESCAPES_FLAG | HAS_NEEDS_GUARD_IP_FLAG }, + [RETURN_GENERATOR] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG | HAS_SYNC_SP_FLAG | HAS_NEEDS_GUARD_IP_FLAG }, + [RETURN_VALUE] = { true, INSTR_FMT_IX, HAS_ESCAPES_FLAG | HAS_SYNC_SP_FLAG | HAS_NEEDS_GUARD_IP_FLAG }, [SEND] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG | HAS_SYNC_SP_FLAG | HAS_NEEDS_GUARD_IP_FLAG }, [SEND_ASYNC_GEN] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG | HAS_UNPREDICTABLE_JUMP_FLAG }, [SEND_GEN] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_SYNC_SP_FLAG | HAS_NEEDS_GUARD_IP_FLAG | HAS_RECORDS_VALUE_FLAG }, @@ -1345,7 +1345,7 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[267] = { [UNPACK_SEQUENCE_TUPLE] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_EXIT_FLAG | HAS_ESCAPES_FLAG }, [UNPACK_SEQUENCE_TWO_TUPLE] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_EXIT_FLAG | HAS_ESCAPES_FLAG }, [WITH_EXCEPT_START] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [YIELD_VALUE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ESCAPES_FLAG | HAS_NEEDS_GUARD_IP_FLAG }, + [YIELD_VALUE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ESCAPES_FLAG | HAS_SYNC_SP_FLAG | HAS_NEEDS_GUARD_IP_FLAG }, [ANNOTATIONS_PLACEHOLDER] = { true, -1, HAS_PURE_FLAG }, [JUMP] = { true, -1, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_EVAL_BREAK_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [JUMP_IF_FALSE] = { true, -1, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, diff --git a/Include/internal/pycore_optimizer.h b/Include/internal/pycore_optimizer.h index 8c35c4416fe3..3d60638649dc 100644 --- a/Include/internal/pycore_optimizer.h +++ b/Include/internal/pycore_optimizer.h @@ -312,7 +312,7 @@ int _Py_uop_analyze_and_optimize( _PyUOpInstruction *input, int trace_len, int curr_stackentries, _PyUOpInstruction *output, _PyBloomFilter *dependencies); -extern PyTypeObject _PyUOpExecutor_Type; +PyAPI_DATA(PyTypeObject) _PyUOpExecutor_Type; #define UOP_FORMAT_TARGET 0 diff --git a/Include/internal/pycore_uop_metadata.h b/Include/internal/pycore_uop_metadata.h index 45f407dae3dd..43498c57d8a9 100644 --- a/Include/internal/pycore_uop_metadata.h +++ b/Include/internal/pycore_uop_metadata.h @@ -163,7 +163,7 @@ const uint32_t _PyUop_Flags[MAX_UOP_ID+1] = { [_CALL_INTRINSIC_1] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG, [_CALL_INTRINSIC_2] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG, [_MAKE_HEAP_SAFE] = 0, - [_RETURN_VALUE] = HAS_ESCAPES_FLAG | HAS_NEEDS_GUARD_IP_FLAG, + [_RETURN_VALUE] = HAS_ESCAPES_FLAG | HAS_SYNC_SP_FLAG | HAS_NEEDS_GUARD_IP_FLAG, [_GET_AITER] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_GET_ANEXT] = HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG, [_GET_AWAITABLE] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, @@ -173,7 +173,7 @@ const uint32_t _PyUop_Flags[MAX_UOP_ID+1] = { [_SEND_VIRTUAL_TIER_TWO] = HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG, [_GUARD_3OS_ASYNC_GEN_ASEND] = HAS_EXIT_FLAG, [_SEND_ASYNC_GEN_TIER_TWO] = HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG, - [_YIELD_VALUE] = HAS_ARG_FLAG | HAS_ESCAPES_FLAG | HAS_NEEDS_GUARD_IP_FLAG, + [_YIELD_VALUE] = HAS_ARG_FLAG | HAS_ESCAPES_FLAG | HAS_SYNC_SP_FLAG | HAS_NEEDS_GUARD_IP_FLAG, [_POP_EXCEPT] = HAS_ESCAPES_FLAG, [_LOAD_COMMON_CONSTANT] = HAS_ARG_FLAG, [_LOAD_BUILD_CLASS] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, @@ -367,7 +367,7 @@ const uint32_t _PyUop_Flags[MAX_UOP_ID+1] = { [_CALL_FUNCTION_EX_NON_PY_GENERAL] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_MAKE_FUNCTION] = HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG, [_SET_FUNCTION_ATTRIBUTE] = HAS_ARG_FLAG, - [_RETURN_GENERATOR] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG | HAS_NEEDS_GUARD_IP_FLAG, + [_RETURN_GENERATOR] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG | HAS_SYNC_SP_FLAG | HAS_NEEDS_GUARD_IP_FLAG, [_BUILD_SLICE] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_CONVERT_VALUE] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_FORMAT_SIMPLE] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, @@ -3439,7 +3439,7 @@ const _PyUopCachingInfo _PyUop_Caching[MAX_UOP_ID+1] = { [_RETURN_GENERATOR] = { .best = { 0, 0, 0, 0 }, .entries = { - { 1, 0, _RETURN_GENERATOR_r01 }, + { 1, 1, _RETURN_GENERATOR_r01 }, { -1, -1, -1 }, { -1, -1, -1 }, { -1, -1, -1 }, diff --git a/Lib/test/test_generated_cases.py b/Lib/test/test_generated_cases.py index 0cc44ff95b59..8ce5370e922e 100644 --- a/Lib/test/test_generated_cases.py +++ b/Lib/test/test_generated_cases.py @@ -369,8 +369,9 @@ class TestGeneratedCases(unittest.TestCase): stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); escaping_call(); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); res = Py_None; stack_pointer[0] = res; stack_pointer += 1; @@ -392,8 +393,9 @@ class TestGeneratedCases(unittest.TestCase): res = Py_None; stack_pointer[-1] = res; _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); escaping_call(); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); DISPATCH(); } """ @@ -626,8 +628,9 @@ class TestGeneratedCases(unittest.TestCase): uint16_t counter = read_u16(&this_instr[1].cache); (void)counter; _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); op1(left, right); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } /* Skip 2 cache entries */ // OP2 @@ -635,9 +638,10 @@ class TestGeneratedCases(unittest.TestCase): arg2 = stack_pointer[-3]; uint32_t extra = read_u32(&this_instr[4].cache); (void)extra; - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); res = op2(arg2, left, right); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } stack_pointer[-3] = res; stack_pointer += -2; @@ -662,8 +666,9 @@ class TestGeneratedCases(unittest.TestCase): uint16_t counter = read_u16(&this_instr[1].cache); (void)counter; _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); op1(left, right); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); DISPATCH(); } @@ -685,8 +690,9 @@ class TestGeneratedCases(unittest.TestCase): left = stack_pointer[-2]; arg2 = stack_pointer[-3]; _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); res = op3(arg2, left, right); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); stack_pointer[-3] = res; stack_pointer += -2; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); @@ -1475,8 +1481,10 @@ class TestGeneratedCases(unittest.TestCase): next_instr += 1; INSTRUCTION_STATS(BALANCED); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); code(); stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); DISPATCH(); } """ @@ -1500,6 +1508,10 @@ class TestGeneratedCases(unittest.TestCase): frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(BALANCED); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); DISPATCH(); } """ @@ -1522,13 +1534,13 @@ class TestGeneratedCases(unittest.TestCase): frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(BALANCED); - _PyFrame_SetStackPointer(frame, stack_pointer); + // Explicit stack reload stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackAssertInvalid(frame); DISPATCH(); } """ - with self.assertRaises(SyntaxError): - self.run_cases_test(input, output) + self.run_cases_test(input, output) def test_stack_save_only(self): @@ -1614,16 +1626,19 @@ class TestGeneratedCases(unittest.TestCase): INSTRUCTION_STATS(OP); #ifdef Py_GIL_DISABLED _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); escaping_call(); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); #else _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); another_escaping_call(); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); #endif - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); yet_another_escaping_call(); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); DISPATCH(); } """ @@ -1735,8 +1750,9 @@ class TestGeneratedCases(unittest.TestCase): LABEL(my_label) { _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); do_thing(); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (complex) { JUMP_TO_LABEL(other_label); } @@ -1762,12 +1778,14 @@ class TestGeneratedCases(unittest.TestCase): LABEL(one) { stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); JUMP_TO_LABEL(two); } LABEL(two) { _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); JUMP_TO_LABEL(one); } """ @@ -1817,16 +1835,18 @@ class TestGeneratedCases(unittest.TestCase): LABEL(my_label_1) { _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); do_thing1(); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); JUMP_TO_LABEL(my_label_2); } LABEL(my_label_2) { _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); do_thing2(); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); JUMP_TO_LABEL(my_label_1); } """ @@ -1878,14 +1898,16 @@ class TestGeneratedCases(unittest.TestCase): _PyStackRef in; in = stack_pointer[-1]; _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); temp = use(in); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); in = temp; stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(in); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); DISPATCH(); } """ @@ -2058,6 +2080,42 @@ class TestGeneratedCases(unittest.TestCase): "non-recording, non-specializing uops"): self.run_cases_test(input, "") + def test_escaping_in_loop(self): + input = """ + inst(TEST_LOOP_SPILL, (a -- a)) { + int n = (int)oparg; + for (int i = 0; i < n; i++) { + escaping_inside_loop(a); + } + escaping_after_loop(a); + } + """ + output = """ + TARGET(TEST_LOOP_SPILL) { + #if _Py_TAIL_CALL_INTERP + int opcode = TEST_LOOP_SPILL; + (void)(opcode); + #endif + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(TEST_LOOP_SPILL); + _PyStackRef a; + a = stack_pointer[-1]; + int n = (int)oparg; + for (int i = 0; i < n; i++) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + escaping_inside_loop(a); + _PyFrame_StackPointerInvalidate(frame); + } + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + escaping_after_loop(a); + _PyFrame_StackPointerInvalidate(frame); + DISPATCH(); + } + """ + self.run_cases_test(input, output) class TestRecorderTableGeneration(unittest.TestCase): diff --git a/Modules/_testinternalcapi/interpreter.c b/Modules/_testinternalcapi/interpreter.c index 4afa028cd875..237635c5f2bc 100644 --- a/Modules/_testinternalcapi/interpreter.c +++ b/Modules/_testinternalcapi/interpreter.c @@ -92,6 +92,7 @@ Test_EvalFrame(PyThreadState *tstate, _PyInterpreterFrame *frame, int throwflag) entry.frame.return_offset = 0; #ifdef Py_DEBUG entry.frame.lltrace = 0; + entry.frame.stackpointer_valid = 1; #endif /* Push frame */ entry.frame.previous = tstate->current_frame; @@ -122,6 +123,7 @@ Test_EvalFrame(PyThreadState *tstate, _PyInterpreterFrame *frame, int throwflag) next_instr = frame->instr_ptr; monitor_throw(tstate, frame, next_instr); stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); #if _Py_TAIL_CALL_INTERP # if Py_STATS return _TAIL_CALL_error(frame, stack_pointer, tstate, next_instr, instruction_funcptr_handler_table, 0, lastopcode); diff --git a/Modules/_testinternalcapi/test_cases.c.h b/Modules/_testinternalcapi/test_cases.c.h index f2935310ffbd..76493327ca0a 100644 --- a/Modules/_testinternalcapi/test_cases.c.h +++ b/Modules/_testinternalcapi/test_cases.c.h @@ -45,8 +45,9 @@ if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { next_instr = this_instr; _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); _Py_Specialize_BinaryOp(lhs, rhs, next_instr, oparg, LOCALS_ARRAY); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); DISPATCH_SAME_OPARG(); } OPCODE_DEFERRED_INC(BINARY_OP); @@ -62,8 +63,9 @@ PyObject *rhs_o = PyStackRef_AsPyObjectBorrow(rhs); assert(_PyEval_BinaryOps[oparg]); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyObject *res_o = _PyEval_BinaryOps[oparg](lhs_o, rhs_o); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (res_o == NULL) { JUMP_TO_LABEL(error); } @@ -76,9 +78,10 @@ value = r; stack_pointer[-2] = res; stack_pointer[-1] = l; - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); PyStackRef_XCLOSE(value); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } // _POP_TOP { @@ -86,8 +89,9 @@ stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_XCLOSE(value); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } DISPATCH(); } @@ -344,10 +348,11 @@ assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5); assert(d != NULL); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); int match = (d->guard != NULL) ? d->guard(left_o, right_o) : (Py_TYPE(left_o) == d->lhs_type && Py_TYPE(right_o) == d->rhs_type); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (!match) { UPDATE_MISS_STATS(BINARY_OP); assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); @@ -363,9 +368,10 @@ assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5); _PyBinaryOpSpecializationDescr *d = (_PyBinaryOpSpecializationDescr*)descr; STAT_INC(BINARY_OP, hit); - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); PyObject *res_o = d->action(left_o, right_o); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (res_o == NULL) { JUMP_TO_LABEL(error); } @@ -381,9 +387,10 @@ value = r; stack_pointer[-2] = res; stack_pointer[-1] = l; - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); PyStackRef_XCLOSE(value); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } // _POP_TOP { @@ -391,8 +398,9 @@ stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_XCLOSE(value); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } DISPATCH(); } @@ -463,9 +471,13 @@ stack_pointer += -2; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyUnicode_Append(&temp, right_o); + _PyFrame_StackPointerInvalidate(frame); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); _Py_DECREF_SPECIALIZED(right_o, _PyUnicode_ExactDealloc); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); *target_local = PyStackRef_NULL; if (temp == NULL) { JUMP_TO_LABEL(error); @@ -670,8 +682,9 @@ assert(Py_TYPE(dict)->tp_as_mapping->mp_subscript == _PyDict_Subscript); STAT_INC(BINARY_OP, hit); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyObject *res_o = _PyDict_Subscript(dict, sub); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (res_o == NULL) { JUMP_TO_LABEL(error); } @@ -684,9 +697,10 @@ value = ss; stack_pointer[-2] = res; stack_pointer[-1] = ds; - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); PyStackRef_XCLOSE(value); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } // _POP_TOP { @@ -694,8 +708,9 @@ stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_XCLOSE(value); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } DISPATCH(); } @@ -773,12 +788,17 @@ stack_pointer += -2; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); assert(temp->previous == frame || temp->previous->previous == frame); CALL_STAT_INC(inlined_py_calls); frame = tstate->current_frame = temp; tstate->py_recursion_remaining--; - LOAD_SP(); + stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); LOAD_IP(0); + #ifdef Py_DEBUG + assert(frame->previous->stackpointer_valid == 1); + #endif DTRACE_FUNCTION_ENTRY(); LLTRACE_RESUME_FRAME(); } @@ -838,8 +858,9 @@ } #ifdef Py_GIL_DISABLED _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyObject *res_o = _PyList_GetItemRef((PyListObject*)list, index); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (res_o == NULL) { UPDATE_MISS_STATS(BINARY_OP); assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); @@ -873,8 +894,9 @@ stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_XCLOSE(value); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } DISPATCH(); } @@ -928,8 +950,9 @@ assert(PySlice_Check(sub)); assert(PyList_CheckExact(list)); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyObject *res_o = _PyList_SliceSubscript(list, sub); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); STAT_INC(BINARY_OP, hit); if (res_o == NULL) { JUMP_TO_LABEL(error); @@ -943,9 +966,10 @@ value = ss; stack_pointer[-2] = res; stack_pointer[-1] = ls; - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); PyStackRef_XCLOSE(value); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } // _POP_TOP { @@ -953,8 +977,9 @@ stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_XCLOSE(value); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } DISPATCH(); } @@ -1134,8 +1159,9 @@ stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_XCLOSE(value); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } DISPATCH(); } @@ -1406,18 +1432,21 @@ PyObject *res_o; if (PyList_CheckExact(container_o)) { _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); res_o = _PyList_BinarySlice(container_o, start_o, stop_o); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } else if (PyTuple_CheckExact(container_o)) { _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); res_o = _PyTuple_BinarySlice(container_o, start_o, stop_o); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } else if (PyUnicode_CheckExact(container_o)) { _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); res_o = _PyUnicode_BinarySlice(container_o, start_o, stop_o); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } else { PyObject *slice = PySlice_New(start_o, stop_o, NULL); @@ -1426,25 +1455,36 @@ } else { _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); res_o = PyObject_GetItem(container_o, slice); + _PyFrame_StackPointerInvalidate(frame); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); Py_DECREF(slice); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } } _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); _PyStackRef tmp = stop; stop = PyStackRef_NULL; stack_pointer[-1] = stop; PyStackRef_CLOSE(tmp); + _PyFrame_StackPointerInvalidate(frame); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); tmp = start; start = PyStackRef_NULL; stack_pointer[-2] = start; PyStackRef_CLOSE(tmp); + _PyFrame_StackPointerInvalidate(frame); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); tmp = container; container = PyStackRef_NULL; stack_pointer[-3] = container; PyStackRef_CLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); stack_pointer += -3; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); if (res_o == NULL) { @@ -1484,14 +1524,16 @@ format_o = &_Py_STR(empty); } _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyObject *interpolation_o = _PyInterpolation_Build(value_o, str_o, conversion, format_o); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (oparg & 1) { stack_pointer += -(oparg & 1); ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(format[0]); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } else { stack_pointer += -(oparg & 1); @@ -1499,13 +1541,15 @@ stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(str); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(value); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (interpolation_o == NULL) { JUMP_TO_LABEL(error); } @@ -1528,8 +1572,9 @@ _PyStackRef list; values = &stack_pointer[-oparg]; _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyObject *list_o = _PyList_FromStackRefStealOnSuccess(values, oparg); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (list_o == NULL) { JUMP_TO_LABEL(error); } @@ -1552,8 +1597,9 @@ _PyStackRef map; values = &stack_pointer[-oparg*2]; _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyObject *map_o = _Py_BuildMap_StackRefSteal(values, oparg); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (map_o == NULL) { stack_pointer += -oparg*2; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); @@ -1578,17 +1624,19 @@ _PyStackRef set; values = &stack_pointer[-oparg]; _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyObject *set_o = PySet_New(NULL); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (set_o == NULL) { - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); _PyStackRef tmp; for (int _i = oparg; --_i >= 0;) { tmp = values[_i]; values[_i] = PyStackRef_NULL; PyStackRef_CLOSE(tmp); } - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); stack_pointer += -oparg; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); JUMP_TO_LABEL(error); @@ -1598,22 +1646,25 @@ _PyStackRef value = values[i]; values[i] = PyStackRef_NULL; if (err == 0) { - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); err = _PySet_AddTakeRef((PySetObject *)set_o, PyStackRef_AsPyObjectSteal(value)); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } else { - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(value); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } } if (err) { stack_pointer += -oparg; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); Py_DECREF(set_o); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); JUMP_TO_LABEL(error); } set = PyStackRef_FromPyObjectStealMortal(set_o); @@ -1639,13 +1690,14 @@ PyObject *step_o = oparg == 3 ? PyStackRef_AsPyObjectBorrow(args[2]) : NULL; PyObject *slice_o = PySlice_New(start_o, stop_o, step_o); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); _PyStackRef tmp; for (int _i = oparg; --_i >= 0;) { tmp = args[_i]; args[_i] = PyStackRef_NULL; PyStackRef_CLOSE(tmp); } - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); stack_pointer += -oparg; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); if (slice_o == NULL) { @@ -1670,8 +1722,9 @@ _PyStackRef str; pieces = &stack_pointer[-oparg]; _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyObject *str_o = _Py_BuildString_StackRefSteal(pieces, oparg); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (str_o == NULL) { stack_pointer += -oparg; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); @@ -1700,18 +1753,21 @@ PyObject *strings_o = PyStackRef_AsPyObjectBorrow(strings); PyObject *interpolations_o = PyStackRef_AsPyObjectBorrow(interpolations); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyObject *template_o = _PyTemplate_Build(strings_o, interpolations_o); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(interpolations); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(strings); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (template_o == NULL) { JUMP_TO_LABEL(error); } @@ -1783,8 +1839,9 @@ if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { next_instr = this_instr; _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); _Py_Specialize_Call(callable, self_or_null, next_instr, oparg + !PyStackRef_IsNull(self_or_null)); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); DISPATCH_SAME_OPARG(); } OPCODE_DEFERRED_INC(CALL); @@ -1804,8 +1861,9 @@ stack_pointer[-2 - oparg] = callable; stack_pointer[-1 - oparg] = self_or_null; _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(temp); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } } // _DO_CALL @@ -1827,11 +1885,12 @@ stack_pointer[-2 - oparg] = callable; stack_pointer[-1 - oparg] = self_or_null; _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); _PyInterpreterFrame *new_frame = _PyEvalFramePushAndInit( tstate, callable, locals, arguments, total_args, NULL, frame ); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); stack_pointer += -2 - oparg; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); if (new_frame == NULL) { @@ -1843,6 +1902,7 @@ stack_pointer[-2 - oparg] = callable; stack_pointer[-1 - oparg] = self_or_null; _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyObject* res_o = _Py_VectorCallInstrumentation_StackRefSteal( callable, arguments, @@ -1852,7 +1912,7 @@ frame, this_instr, tstate); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (res_o == NULL) { stack_pointer += -2 - oparg; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); @@ -1866,8 +1926,9 @@ stack_pointer += -1 - oparg; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); int err = check_periodics(tstate); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (err != 0) { JUMP_TO_LABEL(error); } @@ -1952,8 +2013,9 @@ } STAT_INC(CALL, hit); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyObject *self_o = PyType_GenericAlloc(tp, 0); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (self_o == NULL) { JUMP_TO_LABEL(error); } @@ -1962,32 +2024,36 @@ callable = PyStackRef_FromPyObjectNew(init_func); stack_pointer[-2 - oparg] = callable; stack_pointer[-1 - oparg] = self_or_null; - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(temp); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } // _CREATE_INIT_FRAME { args = &stack_pointer[-oparg]; self = self_or_null; init = callable; - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); _PyInterpreterFrame *shim = _PyFrame_PushTrampolineUnchecked( tstate, (PyCodeObject *)&_Py_InitCleanup, 1, frame); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); assert(_PyFrame_GetBytecode(shim)[0].op.code == EXIT_INIT_CHECK); assert(_PyFrame_GetBytecode(shim)[1].op.code == RETURN_VALUE); shim->localsplus[0] = PyStackRef_DUP(self); - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); _PyInterpreterFrame *temp = _PyEvalFramePushAndInit( tstate, init, NULL, args-1, oparg+1, NULL, shim); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); stack_pointer += -2 - oparg; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); if (temp == NULL) { _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); _PyEval_FrameClearAndPop(tstate, shim); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); JUMP_TO_LABEL(error); } frame->return_offset = 1 + INLINE_CACHE_ENTRIES_CALL; @@ -2000,12 +2066,17 @@ assert(!IS_PEP523_HOOKED(tstate)); _PyInterpreterFrame *temp = PyStackRef_Unwrap(new_frame); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); assert(temp->previous == frame || temp->previous->previous == frame); CALL_STAT_INC(inlined_py_calls); frame = tstate->current_frame = temp; tstate->py_recursion_remaining--; - LOAD_SP(); + stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); LOAD_IP(0); + #ifdef Py_DEBUG + assert(frame->previous->stackpointer_valid == 1); + #endif DTRACE_FUNCTION_ENTRY(); LLTRACE_RESUME_FRAME(); } @@ -2064,8 +2135,9 @@ stack_pointer[-2 - oparg] = callable; stack_pointer[-1 - oparg] = self_or_null; _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(temp); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } // flush // _CHECK_FUNCTION_VERSION @@ -2144,12 +2216,17 @@ stack_pointer += -2 - oparg; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); assert(temp->previous == frame || temp->previous->previous == frame); CALL_STAT_INC(inlined_py_calls); frame = tstate->current_frame = temp; tstate->py_recursion_remaining--; - LOAD_SP(); + stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); LOAD_IP(0); + #ifdef Py_DEBUG + assert(frame->previous->stackpointer_valid == 1); + #endif DTRACE_FUNCTION_ENTRY(); LLTRACE_RESUME_FRAME(); } @@ -2222,8 +2299,9 @@ stack_pointer[-2 - oparg] = callable; stack_pointer[-1 - oparg] = self_or_null; _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(temp); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } // flush // _CHECK_RECURSION_REMAINING @@ -2246,12 +2324,13 @@ assert(Py_TYPE(callable_o) == &PyFunction_Type); int code_flags = ((PyCodeObject*)PyFunction_GET_CODE(callable_o))->co_flags; PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable_o)); - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); _PyInterpreterFrame *temp = _PyEvalFramePushAndInit( tstate, callable, locals, args, total_args, NULL, frame ); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); stack_pointer += -2 - oparg; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); if (temp == NULL) { @@ -2273,12 +2352,17 @@ assert(!IS_PEP523_HOOKED(tstate)); _PyInterpreterFrame *temp = PyStackRef_Unwrap(new_frame); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); assert(temp->previous == frame || temp->previous->previous == frame); CALL_STAT_INC(inlined_py_calls); frame = tstate->current_frame = temp; tstate->py_recursion_remaining--; - LOAD_SP(); + stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); LOAD_IP(0); + #ifdef Py_DEBUG + assert(frame->previous->stackpointer_valid == 1); + #endif DTRACE_FUNCTION_ENTRY(); LLTRACE_RESUME_FRAME(); } @@ -2330,27 +2414,30 @@ } STAT_INC(CALL, hit); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyObject *res_o = _Py_CallBuiltinClass_StackRef( callable, arguments, total_args); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (res_o == NULL) { JUMP_TO_LABEL(error); } _PyStackRef temp = callable; callable = PyStackRef_FromPyObjectSteal(res_o); stack_pointer[-2 - oparg] = callable; - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(temp); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } // _POP_TOP_OPARG { args = &stack_pointer[-oparg]; - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); _PyStackRef_CloseStack(args, oparg); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } // _POP_TOP { @@ -2358,14 +2445,16 @@ stack_pointer += -1 - oparg; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_XCLOSE(value); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } // _CHECK_PERIODIC_AT_END { - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); int err = check_periodics(tstate); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (err != 0) { JUMP_TO_LABEL(error); } @@ -2417,28 +2506,31 @@ } STAT_INC(CALL, hit); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyObject *res_o = _Py_BuiltinCallFast_StackRef( callable, arguments, total_args ); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (res_o == NULL) { JUMP_TO_LABEL(error); } _PyStackRef temp = callable; callable = PyStackRef_FromPyObjectSteal(res_o); stack_pointer[-2 - oparg] = callable; - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(temp); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } // _POP_TOP_OPARG { args = &stack_pointer[-oparg]; - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); _PyStackRef_CloseStack(args, oparg); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } // _POP_TOP { @@ -2446,14 +2538,16 @@ stack_pointer += -1 - oparg; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_XCLOSE(value); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } // _CHECK_PERIODIC_AT_END { - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); int err = check_periodics(tstate); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (err != 0) { JUMP_TO_LABEL(error); } @@ -2505,24 +2599,27 @@ } STAT_INC(CALL, hit); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyObject *res_o = _Py_BuiltinCallFastWithKeywords_StackRef(callable, arguments, total_args); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (res_o == NULL) { JUMP_TO_LABEL(error); } _PyStackRef temp = callable; callable = PyStackRef_FromPyObjectSteal(res_o); stack_pointer[-2 - oparg] = callable; - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(temp); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } // _POP_TOP_OPARG { args = &stack_pointer[-oparg]; - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); _PyStackRef_CloseStack(args, oparg); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } // _POP_TOP { @@ -2530,14 +2627,16 @@ stack_pointer += -1 - oparg; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_XCLOSE(value); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } // _CHECK_PERIODIC_AT_END { - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); int err = check_periodics(tstate); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (err != 0) { JUMP_TO_LABEL(error); } @@ -2609,8 +2708,9 @@ PyCFunction cfunc = PyCFunction_GET_FUNCTION(callable_o); _PyStackRef arg = args[0]; _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyObject *res_o = _PyCFunction_TrampolineCall(cfunc, PyCFunction_GET_SELF(callable_o), PyStackRef_AsPyObjectBorrow(arg)); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); _Py_LeaveRecursiveCallTstate(tstate); assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); if (res_o == NULL) { @@ -2628,8 +2728,9 @@ stack_pointer += -oparg; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_XCLOSE(value); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } // _POP_TOP { @@ -2637,14 +2738,16 @@ stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_XCLOSE(value); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } // _CHECK_PERIODIC_AT_END { - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); int err = check_periodics(tstate); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (err != 0) { JUMP_TO_LABEL(error); } @@ -2688,23 +2791,26 @@ PyObject *callargs_o = PyStackRef_AsPyObjectBorrow(callargs); if (!PyTuple_CheckExact(callargs_o)) { _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); int err = _Py_Check_ArgsIterable(tstate, PyStackRef_AsPyObjectBorrow(func), callargs_o); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (err < 0) { JUMP_TO_LABEL(error); } - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); PyObject *tuple_o = PySequence_Tuple(callargs_o); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (tuple_o == NULL) { JUMP_TO_LABEL(error); } _PyStackRef temp = callargs; callargs = PyStackRef_FromPyObjectSteal(tuple_o); stack_pointer[-2] = callargs; - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(temp); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } } // _CALL_FUNCTION_EX_NON_PY_GENERAL @@ -2721,23 +2827,27 @@ assert(kwargs == NULL || PyDict_CheckExact(kwargs)); stack_pointer[-2] = callargs_st; _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyObject *result_o = PyObject_Call(func, callargs, kwargs); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_XCLOSE(kwargs_st); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(callargs_st); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); stack_pointer += -2; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(func_st); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (result_o == NULL) { JUMP_TO_LABEL(error); } @@ -2749,8 +2859,9 @@ stack_pointer += 1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); int err = check_periodics(tstate); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (err != 0) { JUMP_TO_LABEL(error); } @@ -2792,23 +2903,26 @@ PyObject *callargs_o = PyStackRef_AsPyObjectBorrow(callargs); if (!PyTuple_CheckExact(callargs_o)) { _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); int err = _Py_Check_ArgsIterable(tstate, PyStackRef_AsPyObjectBorrow(func), callargs_o); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (err < 0) { JUMP_TO_LABEL(error); } - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); PyObject *tuple_o = PySequence_Tuple(callargs_o); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (tuple_o == NULL) { JUMP_TO_LABEL(error); } _PyStackRef temp = callargs; callargs = PyStackRef_FromPyObjectSteal(tuple_o); stack_pointer[-2] = callargs; - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(temp); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } } // _CHECK_IS_PY_CALLABLE_EX @@ -2843,10 +2957,11 @@ stack_pointer += -3; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); _PyInterpreterFrame *new_frame = _PyEvalFramePushAndInit_Ex( tstate, func_st, locals, nargs, callargs, kwargs, frame); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); if (new_frame == NULL) { @@ -2869,12 +2984,17 @@ assert(!IS_PEP523_HOOKED(tstate)); _PyInterpreterFrame *temp = PyStackRef_Unwrap(new_frame); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); assert(temp->previous == frame || temp->previous->previous == frame); CALL_STAT_INC(inlined_py_calls); frame = tstate->current_frame = temp; tstate->py_recursion_remaining--; - LOAD_SP(); + stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); LOAD_IP(0); + #ifdef Py_DEBUG + assert(frame->previous->stackpointer_valid == 1); + #endif DTRACE_FUNCTION_ENTRY(); LLTRACE_RESUME_FRAME(); } @@ -2909,8 +3029,9 @@ if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { next_instr = this_instr; _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); _Py_Specialize_CallFunctionEx(func, next_instr); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); DISPATCH_SAME_OPARG(); } OPCODE_DEFERRED_INC(CALL_FUNCTION_EX); @@ -2923,23 +3044,26 @@ PyObject *callargs_o = PyStackRef_AsPyObjectBorrow(callargs); if (!PyTuple_CheckExact(callargs_o)) { _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); int err = _Py_Check_ArgsIterable(tstate, PyStackRef_AsPyObjectBorrow(func), callargs_o); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (err < 0) { JUMP_TO_LABEL(error); } - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); PyObject *tuple_o = PySequence_Tuple(callargs_o); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (tuple_o == NULL) { JUMP_TO_LABEL(error); } _PyStackRef temp = callargs; callargs = PyStackRef_FromPyObjectSteal(tuple_o); stack_pointer[-2] = callargs; - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(temp); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } } // _DO_CALL_FUNCTION_EX @@ -2962,34 +3086,39 @@ PyTuple_GET_ITEM(callargs, 0) : &_PyInstrumentation_MISSING; stack_pointer[-2] = callargs_st; _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); int err = _Py_call_instrumentation_2args( tstate, PY_MONITORING_EVENT_CALL, frame, this_instr, func, arg); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (err) { JUMP_TO_LABEL(error); } - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); result_o = PyObject_Call(func, callargs, kwargs); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (!PyFunction_Check(func) && !PyMethod_Check(func)) { if (result_o == NULL) { - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); _Py_call_instrumentation_exc2( tstate, PY_MONITORING_EVENT_C_RAISE, frame, this_instr, func, arg); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } else { - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); int err = _Py_call_instrumentation_2args( tstate, PY_MONITORING_EVENT_C_RETURN, frame, this_instr, func, arg); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (err < 0) { - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); Py_CLEAR(result_o); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } } } @@ -3008,10 +3137,11 @@ stack_pointer += -2; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); _PyInterpreterFrame *new_frame = _PyEvalFramePushAndInit_Ex( tstate, func_st, locals, nargs, callargs, kwargs, frame); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); stack_pointer += -2; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); if (new_frame == NULL) { @@ -3027,24 +3157,28 @@ assert(kwargs == NULL || PyDict_CheckExact(kwargs)); stack_pointer[-2] = callargs_st; _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); result_o = PyObject_Call(func, callargs, kwargs); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_XCLOSE(kwargs_st); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(callargs_st); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); stack_pointer += -2; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(func_st); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (result_o == NULL) { JUMP_TO_LABEL(error); } @@ -3056,8 +3190,9 @@ stack_pointer += 1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); int err = check_periodics(tstate); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (err != 0) { JUMP_TO_LABEL(error); } @@ -3081,8 +3216,9 @@ value = stack_pointer[-1]; assert(oparg <= MAX_INTRINSIC_1); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyObject *res_o = _PyIntrinsics_UnaryFunctions[oparg].func(tstate, PyStackRef_AsPyObjectBorrow(value)); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (res_o == NULL) { JUMP_TO_LABEL(error); } @@ -3093,9 +3229,10 @@ { value = v; stack_pointer[-1] = res; - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); PyStackRef_XCLOSE(value); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } DISPATCH(); } @@ -3122,8 +3259,9 @@ PyObject *value1 = PyStackRef_AsPyObjectBorrow(value1_st); PyObject *value2 = PyStackRef_AsPyObjectBorrow(value2_st); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyObject *res_o = _PyIntrinsics_BinaryFunctions[oparg].func(tstate, value2, value1); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (res_o == NULL) { JUMP_TO_LABEL(error); } @@ -3136,9 +3274,10 @@ value = vs2; stack_pointer[-2] = res; stack_pointer[-1] = vs1; - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); PyStackRef_XCLOSE(value); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } // _POP_TOP { @@ -3146,8 +3285,9 @@ stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_XCLOSE(value); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } DISPATCH(); } @@ -3198,8 +3338,9 @@ PyObject *inst_o = PyStackRef_AsPyObjectBorrow(instance); PyObject *cls_o = PyStackRef_AsPyObjectBorrow(cls); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); int retval = PyObject_IsInstance(inst_o, cls_o); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (retval < 0) { JUMP_TO_LABEL(error); } @@ -3207,18 +3348,21 @@ stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(cls); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(instance); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); stack_pointer += -2; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(callable); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); res = retval ? PyStackRef_True : PyStackRef_False; assert((!PyStackRef_IsNull(res)) ^ (_PyErr_Occurred(tstate) != NULL)); } @@ -3255,8 +3399,9 @@ if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { next_instr = this_instr; _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); _Py_Specialize_CallKw(callable, next_instr, oparg + !PyStackRef_IsNull(self_or_null)); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); DISPATCH_SAME_OPARG(); } OPCODE_DEFERRED_INC(CALL_KW); @@ -3276,8 +3421,9 @@ stack_pointer[-3 - oparg] = callable; stack_pointer[-2 - oparg] = self_or_null; _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(temp); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } } // _DO_CALL_KW @@ -3302,16 +3448,18 @@ stack_pointer[-3 - oparg] = callable; stack_pointer[-2 - oparg] = self_or_null; _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); _PyInterpreterFrame *new_frame = _PyEvalFramePushAndInit( tstate, callable, locals, arguments, positional_args, kwnames_o, frame ); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); stack_pointer += -3 - oparg; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(kwnames); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (new_frame == NULL) { JUMP_TO_LABEL(error); } @@ -3322,6 +3470,7 @@ stack_pointer[-3 - oparg] = callable; stack_pointer[-2 - oparg] = self_or_null; _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyObject* res_o = _Py_VectorCallInstrumentation_StackRefSteal( callable, arguments, @@ -3331,7 +3480,7 @@ frame, this_instr, tstate); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (res_o == NULL) { stack_pointer += -3 - oparg; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); @@ -3412,8 +3561,9 @@ stack_pointer[-3 - oparg] = callable; stack_pointer[-2 - oparg] = self_or_null; _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(callable_s); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } // flush // _PY_FRAME_KW @@ -3432,17 +3582,19 @@ assert(Py_TYPE(callable_o) == &PyFunction_Type); int code_flags = ((PyCodeObject*)PyFunction_GET_CODE(callable_o))->co_flags; PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable_o)); - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); _PyInterpreterFrame *temp = _PyEvalFramePushAndInit( tstate, callable, locals, arguments, positional_args, kwnames_o, frame ); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(kwnames); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); stack_pointer += -2 - oparg; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); if (temp == NULL) { @@ -3464,12 +3616,17 @@ assert(!IS_PEP523_HOOKED(tstate)); _PyInterpreterFrame *temp = PyStackRef_Unwrap(new_frame); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); assert(temp->previous == frame || temp->previous->previous == frame); CALL_STAT_INC(inlined_py_calls); frame = tstate->current_frame = temp; tstate->py_recursion_remaining--; - LOAD_SP(); + stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); LOAD_IP(0); + #ifdef Py_DEBUG + assert(frame->previous->stackpointer_valid == 1); + #endif DTRACE_FUNCTION_ENTRY(); LLTRACE_RESUME_FRAME(); } @@ -3525,12 +3682,13 @@ total_args++; } _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyObject *res_o = _Py_VectorCall_StackRefSteal( callable, arguments, total_args, kwnames); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (res_o == NULL) { stack_pointer += -3 - oparg; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); @@ -3544,8 +3702,9 @@ stack_pointer += -2 - oparg; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); int err = check_periodics(tstate); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (err != 0) { JUMP_TO_LABEL(error); } @@ -3621,16 +3780,18 @@ int code_flags = ((PyCodeObject*)PyFunction_GET_CODE(callable_o))->co_flags; PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable_o)); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); _PyInterpreterFrame *temp = _PyEvalFramePushAndInit( tstate, callable, locals, arguments, positional_args, kwnames_o, frame ); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(kwnames); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); stack_pointer += -2 - oparg; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); if (temp == NULL) { @@ -3652,12 +3813,17 @@ assert(!IS_PEP523_HOOKED(tstate)); _PyInterpreterFrame *temp = PyStackRef_Unwrap(new_frame); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); assert(temp->previous == frame || temp->previous->previous == frame); CALL_STAT_INC(inlined_py_calls); frame = tstate->current_frame = temp; tstate->py_recursion_remaining--; - LOAD_SP(); + stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); LOAD_IP(0); + #ifdef Py_DEBUG + assert(frame->previous->stackpointer_valid == 1); + #endif DTRACE_FUNCTION_ENTRY(); LLTRACE_RESUME_FRAME(); } @@ -3710,8 +3876,9 @@ STAT_INC(CALL, hit); PyObject *arg_o = PyStackRef_AsPyObjectBorrow(arg); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); Py_ssize_t len_i = PyObject_Length(arg_o); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (len_i < 0) { JUMP_TO_LABEL(error); } @@ -3732,8 +3899,9 @@ stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_XCLOSE(value); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } // _POP_TOP { @@ -3741,8 +3909,9 @@ stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_XCLOSE(value); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } DISPATCH(); } @@ -3826,8 +3995,9 @@ stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_XCLOSE(value); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } // _POP_TOP { @@ -3835,8 +4005,9 @@ stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_XCLOSE(value); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } DISPATCH(); } @@ -3906,7 +4077,11 @@ assert(self != NULL); STAT_INC(CALL, hit); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyCFunctionFast cfunc = _PyCFunctionFast_CAST(method->d_method->ml_meth); + _PyFrame_StackPointerInvalidate(frame); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); PyObject *res_o = _PyCallMethodDescriptorFast_StackRef( callable, cfunc, @@ -3914,22 +4089,24 @@ arguments, total_args ); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (res_o == NULL) { JUMP_TO_LABEL(error); } _PyStackRef temp = callable; callable = PyStackRef_FromPyObjectSteal(res_o); stack_pointer[-2 - oparg] = callable; - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(temp); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } // _POP_TOP_OPARG { - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); _PyStackRef_CloseStack(args, oparg); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } // _POP_TOP { @@ -3937,14 +4114,16 @@ stack_pointer += -1 - oparg; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_XCLOSE(value); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } // _CHECK_PERIODIC_AT_END { - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); int err = check_periodics(tstate); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (err != 0) { JUMP_TO_LABEL(error); } @@ -4018,7 +4197,11 @@ assert(self != NULL); STAT_INC(CALL, hit); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyCFunctionFastWithKeywords cfunc = _PyCFunctionFastWithKeywords_CAST(method->d_method->ml_meth); + _PyFrame_StackPointerInvalidate(frame); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); PyObject *res_o = _PyCallMethodDescriptorFastWithKeywords_StackRef( callable, cfunc, @@ -4026,22 +4209,24 @@ arguments, total_args ); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (res_o == NULL) { JUMP_TO_LABEL(error); } _PyStackRef temp = callable; callable = PyStackRef_FromPyObjectSteal(res_o); stack_pointer[-2 - oparg] = callable; - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(temp); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } // _POP_TOP_OPARG { - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); _PyStackRef_CloseStack(args, oparg); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } // _POP_TOP { @@ -4049,14 +4234,16 @@ stack_pointer += -1 - oparg; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_XCLOSE(value); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } // _CHECK_PERIODIC_AT_END { - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); int err = check_periodics(tstate); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (err != 0) { JUMP_TO_LABEL(error); } @@ -4139,8 +4326,9 @@ STAT_INC(CALL, hit); PyCFunction cfunc = method->d_method->ml_meth; _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyObject *res_o = _PyCFunction_TrampolineCall(cfunc, self, NULL); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); _Py_LeaveRecursiveCallTstate(tstate); assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); if (res_o == NULL) { @@ -4158,8 +4346,9 @@ stack_pointer += -oparg; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_XCLOSE(value); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } // _POP_TOP { @@ -4167,14 +4356,16 @@ stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_XCLOSE(value); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } // _CHECK_PERIODIC_AT_END { - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); int err = check_periodics(tstate); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (err != 0) { JUMP_TO_LABEL(error); } @@ -4258,8 +4449,9 @@ PyObject *self = PyStackRef_AsPyObjectBorrow(arguments[0]); PyObject *arg = PyStackRef_AsPyObjectBorrow(arguments[1]); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyObject *res_o = _PyCFunction_TrampolineCall(cfunc, self, arg); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); _Py_LeaveRecursiveCallTstate(tstate); assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); if (res_o == NULL) { @@ -4279,8 +4471,9 @@ stack_pointer += 1 - oparg; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_XCLOSE(value); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } // _POP_TOP { @@ -4288,8 +4481,9 @@ stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_XCLOSE(value); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } // _POP_TOP { @@ -4297,14 +4491,16 @@ stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_XCLOSE(value); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } // _CHECK_PERIODIC_AT_END { - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); int err = check_periodics(tstate); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (err != 0) { JUMP_TO_LABEL(error); } @@ -4359,12 +4555,13 @@ total_args++; } _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyObject *res_o = _Py_VectorCall_StackRefSteal( callable, arguments, total_args, PyStackRef_NULL); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (res_o == NULL) { stack_pointer += -2 - oparg; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); @@ -4378,8 +4575,9 @@ stack_pointer += -1 - oparg; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); int err = check_periodics(tstate); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (err != 0) { JUMP_TO_LABEL(error); } @@ -4489,12 +4687,17 @@ stack_pointer += -2 - oparg; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); assert(temp->previous == frame || temp->previous->previous == frame); CALL_STAT_INC(inlined_py_calls); frame = tstate->current_frame = temp; tstate->py_recursion_remaining--; - LOAD_SP(); + stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); LOAD_IP(0); + #ifdef Py_DEBUG + assert(frame->previous->stackpointer_valid == 1); + #endif DTRACE_FUNCTION_ENTRY(); LLTRACE_RESUME_FRAME(); } @@ -4564,11 +4767,12 @@ int code_flags = ((PyCodeObject*)PyFunction_GET_CODE(callable_o))->co_flags; PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable_o)); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); _PyInterpreterFrame *temp = _PyEvalFramePushAndInit( tstate, callable, locals, args, total_args, NULL, frame ); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); stack_pointer += -2 - oparg; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); if (temp == NULL) { @@ -4590,12 +4794,17 @@ assert(!IS_PEP523_HOOKED(tstate)); _PyInterpreterFrame *temp = PyStackRef_Unwrap(new_frame); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); assert(temp->previous == frame || temp->previous->previous == frame); CALL_STAT_INC(inlined_py_calls); frame = tstate->current_frame = temp; tstate->py_recursion_remaining--; - LOAD_SP(); + stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); LOAD_IP(0); + #ifdef Py_DEBUG + assert(frame->previous->stackpointer_valid == 1); + #endif DTRACE_FUNCTION_ENTRY(); LLTRACE_RESUME_FRAME(); } @@ -4647,8 +4856,9 @@ assert(oparg == 1); STAT_INC(CALL, hit); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyObject *res_o = PyObject_Str(arg_o); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (res_o == NULL) { JUMP_TO_LABEL(error); } @@ -4662,14 +4872,16 @@ stack_pointer += -2; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_XCLOSE(value); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } // _CHECK_PERIODIC_AT_END { - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); int err = check_periodics(tstate); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (err != 0) { JUMP_TO_LABEL(error); } @@ -4722,8 +4934,9 @@ assert(oparg == 1); STAT_INC(CALL, hit); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyObject *res_o = PySequence_Tuple(arg_o); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (res_o == NULL) { JUMP_TO_LABEL(error); } @@ -4737,14 +4950,16 @@ stack_pointer += -2; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_XCLOSE(value); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } // _CHECK_PERIODIC_AT_END { - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); int err = check_periodics(tstate); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (err != 0) { JUMP_TO_LABEL(error); } @@ -4806,8 +5021,9 @@ stack_pointer += -2; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_XCLOSE(value); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } DISPATCH(); } @@ -4829,37 +5045,49 @@ PyObject *exc_value = PyStackRef_AsPyObjectBorrow(exc_value_st); PyObject *match_type = PyStackRef_AsPyObjectBorrow(match_type_st); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); int err = _PyEval_CheckExceptStarTypeValid(tstate, match_type); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (err < 0) { - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); _PyStackRef tmp = match_type_st; match_type_st = PyStackRef_NULL; stack_pointer[-1] = match_type_st; PyStackRef_CLOSE(tmp); + _PyFrame_StackPointerInvalidate(frame); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); tmp = exc_value_st; exc_value_st = PyStackRef_NULL; stack_pointer[-2] = exc_value_st; PyStackRef_CLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); stack_pointer += -2; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); JUMP_TO_LABEL(error); } PyObject *match_o = NULL; PyObject *rest_o = NULL; - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); int res = _PyEval_ExceptionGroupMatch(frame, exc_value, match_type, &match_o, &rest_o); + _PyFrame_StackPointerInvalidate(frame); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); _PyStackRef tmp = match_type_st; match_type_st = PyStackRef_NULL; stack_pointer[-1] = match_type_st; PyStackRef_CLOSE(tmp); + _PyFrame_StackPointerInvalidate(frame); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); tmp = exc_value_st; exc_value_st = PyStackRef_NULL; stack_pointer[-2] = exc_value_st; PyStackRef_CLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); stack_pointer += -2; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); if (res < 0) { @@ -4871,8 +5099,9 @@ } if (!Py_IsNone(match_o)) { _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyErr_SetHandledException(match_o); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } rest = PyStackRef_FromPyObjectSteal(rest_o); match = PyStackRef_FromPyObjectSteal(match_o); @@ -4900,19 +5129,22 @@ PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); assert(PyExceptionInstance_Check(left_o)); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); int err = _PyEval_CheckExceptTypeValid(tstate, right_o); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (err < 0) { JUMP_TO_LABEL(error); } - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); int res = PyErr_GivenExceptionMatches(left_o, right_o); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(right); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); b = res ? PyStackRef_True : PyStackRef_False; stack_pointer[0] = b; stack_pointer += 1; @@ -4947,37 +5179,55 @@ #endif assert(exc_value && PyExceptionInstance_Check(exc_value)); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); int matches = PyErr_GivenExceptionMatches(exc_value, PyExc_StopIteration); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (matches) { value = PyStackRef_FromPyObjectNew(((PyStopIterationObject *)exc_value)->value); - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); _PyStackRef tmp = sub_iter; sub_iter = value; stack_pointer[-4] = sub_iter; PyStackRef_CLOSE(tmp); + _PyFrame_StackPointerInvalidate(frame); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); tmp = exc_value_st; exc_value_st = PyStackRef_NULL; stack_pointer[-1] = exc_value_st; PyStackRef_CLOSE(tmp); + _PyFrame_StackPointerInvalidate(frame); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); tmp = last_sent_val; last_sent_val = PyStackRef_NULL; stack_pointer[-2] = last_sent_val; PyStackRef_CLOSE(tmp); + _PyFrame_StackPointerInvalidate(frame); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); tmp = null_in; null_in = PyStackRef_NULL; stack_pointer[-3] = null_in; PyStackRef_XCLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); stack_pointer += -4; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); null_out = null_in; none = PyStackRef_None; } else { - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); _PyErr_SetRaisedException(tstate, Py_NewRef(exc_value)); + _PyFrame_StackPointerInvalidate(frame); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); monitor_reraise(tstate, frame, this_instr); + _PyFrame_StackPointerInvalidate(frame); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); JUMP_TO_LABEL(exception_unwind); } stack_pointer[0] = none; @@ -5012,8 +5262,9 @@ if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { next_instr = this_instr; _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); _Py_Specialize_CompareOp(left, right, next_instr, oparg); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); DISPATCH_SAME_OPARG(); } OPCODE_DEFERRED_INC(COMPARE_OP); @@ -5026,16 +5277,23 @@ PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); assert((oparg >> 5) <= Py_GE); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyObject *res_o = PyObject_RichCompare(left_o, right_o, oparg >> 5); + _PyFrame_StackPointerInvalidate(frame); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); _PyStackRef tmp = right; right = PyStackRef_NULL; stack_pointer[-1] = right; PyStackRef_CLOSE(tmp); + _PyFrame_StackPointerInvalidate(frame); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); tmp = left; left = PyStackRef_NULL; stack_pointer[-2] = left; PyStackRef_CLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); stack_pointer += -2; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); if (res_o == NULL) { @@ -5043,9 +5301,13 @@ } if (oparg & 16) { _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); int res_bool = PyObject_IsTrue(res_o); + _PyFrame_StackPointerInvalidate(frame); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); Py_DECREF(res_o); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (res_bool < 0) { JUMP_TO_LABEL(error); } @@ -5302,8 +5564,9 @@ if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { next_instr = this_instr; _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); _Py_Specialize_ContainsOp(right, next_instr); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); DISPATCH_SAME_OPARG(); } OPCODE_DEFERRED_INC(CONTAINS_OP); @@ -5316,8 +5579,9 @@ PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); int res = PySequence_Contains(right_o, left_o); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (res < 0) { JUMP_TO_LABEL(error); } @@ -5330,9 +5594,10 @@ value = r; stack_pointer[-2] = b; stack_pointer[-1] = l; - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); PyStackRef_XCLOSE(value); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } // _POP_TOP { @@ -5340,8 +5605,9 @@ stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_XCLOSE(value); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } DISPATCH(); } @@ -5384,8 +5650,9 @@ assert(PyAnyDict_CheckExact(right_o)); STAT_INC(CONTAINS_OP, hit); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); int res = PyDict_Contains(right_o, left_o); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (res < 0) { JUMP_TO_LABEL(error); } @@ -5398,9 +5665,10 @@ value = r; stack_pointer[-2] = b; stack_pointer[-1] = l; - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); PyStackRef_XCLOSE(value); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } // _POP_TOP { @@ -5408,8 +5676,9 @@ stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_XCLOSE(value); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } DISPATCH(); } @@ -5452,8 +5721,9 @@ assert(PyAnySet_CheckExact(right_o)); STAT_INC(CONTAINS_OP, hit); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); int res = _PySet_Contains((PySetObject *)right_o, left_o); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (res < 0) { JUMP_TO_LABEL(error); } @@ -5466,9 +5736,10 @@ value = r; stack_pointer[-2] = b; stack_pointer[-1] = l; - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); PyStackRef_XCLOSE(value); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } // _POP_TOP { @@ -5476,8 +5747,9 @@ stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_XCLOSE(value); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } DISPATCH(); } @@ -5497,13 +5769,15 @@ assert(oparg >= FVC_STR && oparg <= FVC_ASCII); conv_fn = _PyEval_ConversionFuncs[oparg]; _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyObject *result_o = conv_fn(PyStackRef_AsPyObjectBorrow(value)); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(value); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (result_o == NULL) { JUMP_TO_LABEL(error); } @@ -5565,13 +5839,15 @@ owner = stack_pointer[-1]; PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); int err = PyObject_DelAttr(PyStackRef_AsPyObjectBorrow(owner), name); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(owner); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (err) { JUMP_TO_LABEL(error); } @@ -5590,13 +5866,15 @@ PyObject *oldobj = PyCell_SwapTakeRef((PyCellObject *)cell, NULL); if (oldobj == NULL) { _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); _PyEval_FormatExcUnbound(tstate, _PyFrame_GetCode(frame), oparg); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); JUMP_TO_LABEL(error); } _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); Py_DECREF(oldobj); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); DISPATCH(); } @@ -5611,18 +5889,20 @@ _PyStackRef v = GETLOCAL(oparg); if (PyStackRef_IsNull(v)) { _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); _PyEval_FormatExcCheckArg(tstate, PyExc_UnboundLocalError, UNBOUNDLOCAL_ERROR_MSG, PyTuple_GetItem(_PyFrame_GetCode(frame)->co_localsplusnames, oparg) ); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); JUMP_TO_LABEL(error); } _PyStackRef tmp = GETLOCAL(oparg); GETLOCAL(oparg) = PyStackRef_NULL; _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_XCLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); DISPATCH(); } @@ -5636,16 +5916,18 @@ INSTRUCTION_STATS(DELETE_GLOBAL); PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); int err = PyDict_Pop(GLOBALS(), name, NULL); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (err < 0) { JUMP_TO_LABEL(error); } if (err == 0) { - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); _PyEval_FormatExcCheckArg(tstate, PyExc_NameError, NAME_ERROR_MSG, name); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); JUMP_TO_LABEL(error); } DISPATCH(); @@ -5664,20 +5946,23 @@ int err; if (ns == NULL) { _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); _PyErr_Format(tstate, PyExc_SystemError, "no locals when deleting %R", name); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); JUMP_TO_LABEL(error); } _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); err = PyObject_DelItem(ns, name); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (err != 0) { - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); _PyEval_FormatExcCheckArg(tstate, PyExc_NameError, NAME_ERROR_MSG, name); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); JUMP_TO_LABEL(error); } DISPATCH(); @@ -5696,17 +5981,24 @@ sub = stack_pointer[-1]; container = stack_pointer[-2]; _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); int err = PyObject_DelItem(PyStackRef_AsPyObjectBorrow(container), PyStackRef_AsPyObjectBorrow(sub)); + _PyFrame_StackPointerInvalidate(frame); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); _PyStackRef tmp = sub; sub = PyStackRef_NULL; stack_pointer[-1] = sub; PyStackRef_CLOSE(tmp); + _PyFrame_StackPointerInvalidate(frame); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); tmp = container; container = PyStackRef_NULL; stack_pointer[-2] = container; PyStackRef_CLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); stack_pointer += -2; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); if (err) { @@ -5738,13 +6030,18 @@ PyObject *update_o = PyStackRef_AsPyObjectBorrow(update); PyObject *dupkey = NULL; _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); int err = _PyDict_MergeUniq(dict_o, update_o, &dupkey); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (err < 0) { - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); _PyEval_FormatKwargsError(tstate, callable_o, update_o, dupkey); + _PyFrame_StackPointerInvalidate(frame); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); Py_XDECREF(dupkey); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); JUMP_TO_LABEL(error); } u = update; @@ -5755,8 +6052,9 @@ stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_XCLOSE(value); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } DISPATCH(); } @@ -5780,27 +6078,37 @@ PyObject *dict_o = PyStackRef_AsPyObjectBorrow(dict); PyObject *update_o = PyStackRef_AsPyObjectBorrow(update); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); int err = PyDict_Update(dict_o, update_o); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (err < 0) { int matches = _PyErr_ExceptionMatches(tstate, PyExc_AttributeError); if (matches) { - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); PyObject *exc = _PyErr_GetRaisedException(tstate); + _PyFrame_StackPointerInvalidate(frame); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); int has_keys = PyObject_HasAttrWithError(update_o, &_Py_ID(keys)); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (has_keys == 0) { - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); _PyErr_Format(tstate, PyExc_TypeError, "'%T' object is not a mapping", update_o); + _PyFrame_StackPointerInvalidate(frame); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); Py_DECREF(exc); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } else { - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); _PyErr_ChainExceptions1(exc); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } } JUMP_TO_LABEL(error); @@ -5813,8 +6121,9 @@ stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_XCLOSE(value); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } DISPATCH(); } @@ -5838,27 +6147,39 @@ PyObject *exc = PyStackRef_AsPyObjectBorrow(exc_st); assert(exc && PyExceptionInstance_Check(exc)); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); int matches = PyErr_GivenExceptionMatches(exc, PyExc_StopAsyncIteration); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (matches) { - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); _PyStackRef tmp = exc_st; exc_st = PyStackRef_NULL; stack_pointer[-1] = exc_st; PyStackRef_CLOSE(tmp); + _PyFrame_StackPointerInvalidate(frame); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); tmp = awaitable_st; awaitable_st = PyStackRef_NULL; stack_pointer[-2] = awaitable_st; PyStackRef_CLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); stack_pointer += -2; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); } else { Py_INCREF(exc); - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); _PyErr_SetRaisedException(tstate, exc); + _PyFrame_StackPointerInvalidate(frame); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); monitor_reraise(tstate, frame, this_instr); + _PyFrame_StackPointerInvalidate(frame); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); JUMP_TO_LABEL(exception_unwind); } DISPATCH(); @@ -5876,8 +6197,9 @@ stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(value); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); DISPATCH(); } @@ -5902,8 +6224,9 @@ stack_pointer += -2; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(receiver); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); DISPATCH(); } @@ -5969,10 +6292,11 @@ should_be_none = stack_pointer[-1]; if (!PyStackRef_IsNone(should_be_none)) { _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyErr_Format(PyExc_TypeError, "__init__() should return None, not '%.200s'", Py_TYPE(PyStackRef_AsPyObjectBorrow(should_be_none))->tp_name); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); JUMP_TO_LABEL(error); } stack_pointer += -1; @@ -6010,13 +6334,15 @@ PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); if (!PyUnicode_CheckExact(value_o)) { _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyObject *res_o = PyObject_Format(value_o, NULL); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(value); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (res_o == NULL) { JUMP_TO_LABEL(error); } @@ -6046,16 +6372,23 @@ fmt_spec = stack_pointer[-1]; value = stack_pointer[-2]; _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyObject *res_o = PyObject_Format(PyStackRef_AsPyObjectBorrow(value), PyStackRef_AsPyObjectBorrow(fmt_spec)); + _PyFrame_StackPointerInvalidate(frame); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); _PyStackRef tmp = fmt_spec; fmt_spec = PyStackRef_NULL; stack_pointer[-1] = fmt_spec; PyStackRef_CLOSE(tmp); + _PyFrame_StackPointerInvalidate(frame); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); tmp = value; value = PyStackRef_NULL; stack_pointer[-2] = value; PyStackRef_CLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); stack_pointer += -2; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); if (res_o == NULL) { @@ -6092,8 +6425,9 @@ if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { next_instr = this_instr; _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); _Py_Specialize_ForIter(iter, null_or_index, next_instr, oparg); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); DISPATCH_SAME_OPARG(); } OPCODE_DEFERRED_INC(FOR_ITER); @@ -6103,8 +6437,9 @@ // _FOR_ITER { _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); _PyStackRef item = _PyForIter_VirtualIteratorNext(tstate, frame, iter, &null_or_index); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (!PyStackRef_IsValid(item)) { if (PyStackRef_IsError(item)) { JUMP_TO_LABEL(error); @@ -6174,12 +6509,17 @@ assert(!IS_PEP523_HOOKED(tstate)); _PyInterpreterFrame *temp = PyStackRef_Unwrap(new_frame); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); assert(temp->previous == frame || temp->previous->previous == frame); CALL_STAT_INC(inlined_py_calls); frame = tstate->current_frame = temp; tstate->py_recursion_remaining--; - LOAD_SP(); + stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); LOAD_IP(0); + #ifdef Py_DEBUG + assert(frame->previous->stackpointer_valid == 1); + #endif DTRACE_FUNCTION_ENTRY(); LLTRACE_RESUME_FRAME(); } @@ -6245,8 +6585,9 @@ _PyObject_GC_IS_SHARED(list_o)); STAT_INC(FOR_ITER, hit); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); int result = _PyList_GetItemRefNoLock((PyListObject *)list_o, PyStackRef_UntagInt(null_or_index), &next); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (result < 0) { UPDATE_MISS_STATS(FOR_ITER); assert(_PyOpcode_Deopt[opcode] == (FOR_ITER)); @@ -6423,8 +6764,9 @@ PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter); Py_ssize_t index = PyStackRef_UntagInt(null_or_index); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); _PyObjectIndexPair next_index = Py_TYPE(iter_o)->_tp_iteritem(iter_o, index); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); PyObject *next_o = next_index.object; index = next_index.index; if (next_o == NULL) { @@ -6464,38 +6806,46 @@ } if (getter == NULL) { _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); _PyErr_Format(tstate, PyExc_TypeError, "'async for' requires an object with " "__aiter__ method, got %.100s", type->tp_name); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(obj); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); JUMP_TO_LABEL(error); } _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); iter_o = (*getter)(obj_o); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(obj); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (iter_o == NULL) { JUMP_TO_LABEL(error); } if (Py_TYPE(iter_o)->tp_as_async == NULL || Py_TYPE(iter_o)->tp_as_async->am_anext == NULL) { - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); _PyErr_Format(tstate, PyExc_TypeError, "'async for' received an object from __aiter__ " "that does not implement __anext__: %.100s", Py_TYPE(iter_o)->tp_name); + _PyFrame_StackPointerInvalidate(frame); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); Py_DECREF(iter_o); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); JUMP_TO_LABEL(error); } iter = PyStackRef_FromPyObjectSteal(iter_o); @@ -6517,8 +6867,9 @@ _PyStackRef awaitable; aiter = stack_pointer[-1]; _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyObject *awaitable_o = _PyEval_GetANext(PyStackRef_AsPyObjectBorrow(aiter)); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (awaitable_o == NULL) { JUMP_TO_LABEL(error); } @@ -6541,13 +6892,15 @@ _PyStackRef iter; iterable = stack_pointer[-1]; _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyObject *iter_o = _PyEval_GetAwaitable(PyStackRef_AsPyObjectBorrow(iterable), oparg); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(iterable); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (iter_o == NULL) { JUMP_TO_LABEL(error); } @@ -6581,8 +6934,9 @@ if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { next_instr = this_instr; _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); _Py_Specialize_GetIter(iterable, next_instr); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); DISPATCH_SAME_OPARG(); } OPCODE_DEFERRED_INC(GET_ITER); @@ -6592,8 +6946,9 @@ // _GET_ITER { _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); _PyStackRef result = _PyEval_GetIter(iterable, &index_or_null, oparg); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (PyStackRef_IsError(result)) { JUMP_TO_LABEL(pop_1_error); } @@ -6688,8 +7043,9 @@ _PyStackRef len; obj = stack_pointer[-1]; _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); Py_ssize_t len_i = PyObject_Length(PyStackRef_AsPyObjectBorrow(obj)); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (len_i < 0) { JUMP_TO_LABEL(error); } @@ -6719,15 +7075,17 @@ PyObject *res_o; if (PyLazyImport_CheckExact(PyStackRef_AsPyObjectBorrow(from))) { _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); res_o = _PyEval_LazyImportFrom( tstate, frame, PyStackRef_AsPyObjectBorrow(from), name); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } else { _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); res_o = _PyEval_ImportFrom( tstate, PyStackRef_AsPyObjectBorrow(from), name); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } if (res_o == NULL) { JUMP_TO_LABEL(error); @@ -6756,31 +7114,37 @@ PyObject *res_o; if (!(oparg & 0x02)) { _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); res_o = _PyEval_LazyImportName(tstate, BUILTINS(), GLOBALS(), LOCALS(), name, PyStackRef_AsPyObjectBorrow(fromlist), PyStackRef_AsPyObjectBorrow(level), oparg & 0x01); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } else { _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); res_o = _PyEval_ImportName(tstate, BUILTINS(), GLOBALS(), LOCALS(), name, PyStackRef_AsPyObjectBorrow(fromlist), PyStackRef_AsPyObjectBorrow(level)); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); _PyStackRef tmp = fromlist; fromlist = PyStackRef_NULL; stack_pointer[-1] = fromlist; PyStackRef_CLOSE(tmp); + _PyFrame_StackPointerInvalidate(frame); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); tmp = level; level = PyStackRef_NULL; stack_pointer[-2] = level; PyStackRef_CLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); stack_pointer += -2; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); if (res_o == NULL) { @@ -6825,8 +7189,9 @@ stack_pointer[-2 - oparg] = callable; stack_pointer[-1 - oparg] = self_or_null; _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(temp); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } } // _MONITOR_CALL @@ -6849,11 +7214,12 @@ stack_pointer[-2 - oparg] = func; stack_pointer[-1 - oparg] = maybe_self; _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); int err = _Py_call_instrumentation_2args( tstate, PY_MONITORING_EVENT_CALL, frame, this_instr, function, arg0 ); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (err) { JUMP_TO_LABEL(error); } @@ -6876,12 +7242,13 @@ { int code_flags = ((PyCodeObject*)PyFunction_GET_CODE(callable_o))->co_flags; PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable_o)); - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); _PyInterpreterFrame *new_frame = _PyEvalFramePushAndInit( tstate, callable, locals, arguments, total_args, NULL, frame ); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); stack_pointer += -2 - oparg; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); if (new_frame == NULL) { @@ -6890,7 +7257,8 @@ frame->return_offset = 4u ; DISPATCH_INLINED(new_frame); } - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); PyObject* res_o = _Py_VectorCallInstrumentation_StackRefSteal( callable, arguments, @@ -6900,7 +7268,7 @@ frame, this_instr, tstate); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (res_o == NULL) { stack_pointer += -2 - oparg; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); @@ -6914,8 +7282,9 @@ stack_pointer += -1 - oparg; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); int err = check_periodics(tstate); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (err != 0) { JUMP_TO_LABEL(error); } @@ -6949,23 +7318,26 @@ PyObject *callargs_o = PyStackRef_AsPyObjectBorrow(callargs); if (!PyTuple_CheckExact(callargs_o)) { _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); int err = _Py_Check_ArgsIterable(tstate, PyStackRef_AsPyObjectBorrow(func), callargs_o); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (err < 0) { JUMP_TO_LABEL(error); } - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); PyObject *tuple_o = PySequence_Tuple(callargs_o); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (tuple_o == NULL) { JUMP_TO_LABEL(error); } _PyStackRef temp = callargs; callargs = PyStackRef_FromPyObjectSteal(tuple_o); stack_pointer[-2] = callargs; - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(temp); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } } // _DO_CALL_FUNCTION_EX @@ -6988,34 +7360,39 @@ PyTuple_GET_ITEM(callargs, 0) : &_PyInstrumentation_MISSING; stack_pointer[-2] = callargs_st; _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); int err = _Py_call_instrumentation_2args( tstate, PY_MONITORING_EVENT_CALL, frame, this_instr, func, arg); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (err) { JUMP_TO_LABEL(error); } - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); result_o = PyObject_Call(func, callargs, kwargs); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (!PyFunction_Check(func) && !PyMethod_Check(func)) { if (result_o == NULL) { - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); _Py_call_instrumentation_exc2( tstate, PY_MONITORING_EVENT_C_RAISE, frame, this_instr, func, arg); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } else { - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); int err = _Py_call_instrumentation_2args( tstate, PY_MONITORING_EVENT_C_RETURN, frame, this_instr, func, arg); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (err < 0) { - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); Py_CLEAR(result_o); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } } } @@ -7034,10 +7411,11 @@ stack_pointer += -2; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); _PyInterpreterFrame *new_frame = _PyEvalFramePushAndInit_Ex( tstate, func_st, locals, nargs, callargs, kwargs, frame); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); stack_pointer += -2; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); if (new_frame == NULL) { @@ -7053,24 +7431,28 @@ assert(kwargs == NULL || PyDict_CheckExact(kwargs)); stack_pointer[-2] = callargs_st; _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); result_o = PyObject_Call(func, callargs, kwargs); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_XCLOSE(kwargs_st); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(callargs_st); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); stack_pointer += -2; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(func_st); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (result_o == NULL) { JUMP_TO_LABEL(error); } @@ -7082,8 +7464,9 @@ stack_pointer += 1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); int err = check_periodics(tstate); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (err != 0) { JUMP_TO_LABEL(error); } @@ -7123,8 +7506,9 @@ stack_pointer[-3 - oparg] = callable; stack_pointer[-2 - oparg] = self_or_null; _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(temp); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } } // _MONITOR_CALL_KW @@ -7145,10 +7529,11 @@ stack_pointer[-3 - oparg] = callable; stack_pointer[-2 - oparg] = self_or_null; _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); int err = _Py_call_instrumentation_2args( tstate, PY_MONITORING_EVENT_CALL, frame, this_instr, function, arg); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (err) { JUMP_TO_LABEL(error); } @@ -7172,17 +7557,19 @@ { int code_flags = ((PyCodeObject*)PyFunction_GET_CODE(callable_o))->co_flags; PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable_o)); - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); _PyInterpreterFrame *new_frame = _PyEvalFramePushAndInit( tstate, callable, locals, arguments, positional_args, kwnames_o, frame ); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); stack_pointer += -3 - oparg; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(kwnames); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (new_frame == NULL) { JUMP_TO_LABEL(error); } @@ -7190,7 +7577,8 @@ frame->return_offset = 4u ; DISPATCH_INLINED(new_frame); } - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); PyObject* res_o = _Py_VectorCallInstrumentation_StackRefSteal( callable, arguments, @@ -7200,7 +7588,7 @@ frame, this_instr, tstate); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (res_o == NULL) { stack_pointer += -3 - oparg; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); @@ -7240,27 +7628,39 @@ PyObject *exc = PyStackRef_AsPyObjectBorrow(exc_st); assert(exc && PyExceptionInstance_Check(exc)); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); int matches = PyErr_GivenExceptionMatches(exc, PyExc_StopAsyncIteration); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (matches) { - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); _PyStackRef tmp = exc_st; exc_st = PyStackRef_NULL; stack_pointer[-1] = exc_st; PyStackRef_CLOSE(tmp); + _PyFrame_StackPointerInvalidate(frame); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); tmp = awaitable_st; awaitable_st = PyStackRef_NULL; stack_pointer[-2] = awaitable_st; PyStackRef_CLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); stack_pointer += -2; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); } else { Py_INCREF(exc); - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); _PyErr_SetRaisedException(tstate, exc); + _PyFrame_StackPointerInvalidate(frame); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); monitor_reraise(tstate, frame, this_instr); + _PyFrame_StackPointerInvalidate(frame); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); JUMP_TO_LABEL(exception_unwind); } } @@ -7282,8 +7682,9 @@ receiver = stack_pointer[-3]; if (PyStackRef_GenCheck(receiver)) { _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); int err = monitor_stop_iteration(tstate, frame, this_instr, PyStackRef_AsPyObjectBorrow(value)); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (err) { JUMP_TO_LABEL(error); } @@ -7291,8 +7692,9 @@ stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(value); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); DISPATCH(); } @@ -7316,8 +7718,9 @@ PyObject *receiver_o = PyStackRef_AsPyObjectBorrow(receiver); if (PyGen_Check(receiver_o) || PyCoro_CheckExact(receiver_o)) { _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); int err = monitor_stop_iteration(tstate, frame, this_instr, PyStackRef_AsPyObjectBorrow(value)); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (err) { JUMP_TO_LABEL(error); } @@ -7328,8 +7731,9 @@ stack_pointer += -2; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(receiver); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); DISPATCH(); } @@ -7350,8 +7754,9 @@ null_or_index = stack_pointer[-1]; iter = stack_pointer[-2]; _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); _PyStackRef item = _PyForIter_VirtualIteratorNext(tstate, frame, iter, &null_or_index); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (!PyStackRef_IsValid(item)) { if (PyStackRef_IsError(item)) { JUMP_TO_LABEL(error); @@ -7381,9 +7786,10 @@ INSTRUCTION_STATS(INSTRUMENTED_INSTRUCTION); opcode = INSTRUMENTED_INSTRUCTION; _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); int next_opcode = _Py_call_instrumentation_instruction( tstate, frame, this_instr); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (next_opcode < 0) { JUMP_TO_LABEL(error); } @@ -7410,8 +7816,9 @@ // _CHECK_PERIODIC { _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); int err = check_periodics(tstate); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (err != 0) { JUMP_TO_LABEL(error); } @@ -7457,9 +7864,13 @@ next_instr = this_instr; } else { _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); original_opcode = _Py_call_instrumentation_line( tstate, frame, this_instr, prev_instr); + _PyFrame_StackPointerInvalidate(frame); + // Explicit stack reload stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackAssertInvalid(frame); if (original_opcode < 0) { next_instr = this_instr+1; JUMP_TO_LABEL(error); @@ -7474,6 +7885,7 @@ PAUSE_ADAPTIVE_COUNTER(cache->counter); } opcode = original_opcode; + PRE_DISPATCH_GOTO(); DISPATCH_GOTO(); } @@ -7505,25 +7917,33 @@ if (opcode == INSTRUMENTED_LOAD_SUPER_ATTR) { PyObject *arg = oparg & 2 ? class : &_PyInstrumentation_MISSING; _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); int err = _Py_call_instrumentation_2args( tstate, PY_MONITORING_EVENT_CALL, frame, this_instr, global_super, arg); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (err) { - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); _PyStackRef tmp = self_st; self_st = PyStackRef_NULL; stack_pointer[-1] = self_st; PyStackRef_CLOSE(tmp); + _PyFrame_StackPointerInvalidate(frame); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); tmp = class_st; class_st = PyStackRef_NULL; stack_pointer[-2] = class_st; PyStackRef_CLOSE(tmp); + _PyFrame_StackPointerInvalidate(frame); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); tmp = global_super_st; global_super_st = PyStackRef_NULL; stack_pointer[-3] = global_super_st; PyStackRef_CLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); stack_pointer += -3; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); JUMP_TO_LABEL(error); @@ -7533,45 +7953,56 @@ { PyObject *stack[] = {class, self}; _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); super = PyObject_Vectorcall(global_super, stack, oparg & 2, NULL); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } if (opcode == INSTRUMENTED_LOAD_SUPER_ATTR) { PyObject *arg = oparg & 2 ? class : &_PyInstrumentation_MISSING; if (super == NULL) { - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); _Py_call_instrumentation_exc2( tstate, PY_MONITORING_EVENT_C_RAISE, frame, this_instr, global_super, arg); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } else { - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); int err = _Py_call_instrumentation_2args( tstate, PY_MONITORING_EVENT_C_RETURN, frame, this_instr, global_super, arg); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (err < 0) { - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); Py_CLEAR(super); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } } } - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); _PyStackRef tmp = self_st; self_st = PyStackRef_NULL; stack_pointer[-1] = self_st; PyStackRef_CLOSE(tmp); + _PyFrame_StackPointerInvalidate(frame); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); tmp = class_st; class_st = PyStackRef_NULL; stack_pointer[-2] = class_st; PyStackRef_CLOSE(tmp); + _PyFrame_StackPointerInvalidate(frame); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); tmp = global_super_st; global_super_st = PyStackRef_NULL; stack_pointer[-3] = global_super_st; PyStackRef_CLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); stack_pointer += -3; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); if (super == NULL) { @@ -7579,9 +8010,13 @@ } PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 2); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyObject *attr_o = PyObject_GetAttr(super, name); + _PyFrame_StackPointerInvalidate(frame); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); Py_DECREF(super); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (attr_o == NULL) { JUMP_TO_LABEL(error); } @@ -7636,8 +8071,9 @@ stack_pointer += -2; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(iter); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); DISPATCH(); } @@ -7687,8 +8123,9 @@ stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(value); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); stack_pointer += 1; } stack_pointer += -1; @@ -7715,8 +8152,9 @@ stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(value); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); INSTRUMENTED_JUMP(this_instr, next_instr + oparg, PY_MONITORING_EVENT_BRANCH_RIGHT); } else { @@ -7766,9 +8204,10 @@ if (frame->tlbc_index != ((_PyThreadStateImpl *)tstate)->tlbc_index) { _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); _Py_CODEUNIT *bytecode = _PyEval_GetExecutableCode(tstate, _PyFrame_GetCode(frame)); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (bytecode == NULL) { JUMP_TO_LABEL(error); } @@ -7793,8 +8232,9 @@ uintptr_t code_version = FT_ATOMIC_LOAD_UINTPTR_ACQUIRE(_PyFrame_GetCode(frame)->_co_instrumentation_version); if (code_version != global_version) { _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); int err = _Py_Instrument(_PyFrame_GetCode(frame), tstate->interp); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (err) { JUMP_TO_LABEL(error); } @@ -7808,8 +8248,9 @@ Test_EvalFrame_Resumes++; if ((oparg & RESUME_OPARG_LOCATION_MASK) < RESUME_AFTER_YIELD_FROM) { _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); int err = check_periodics(tstate); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (err != 0) { JUMP_TO_LABEL(error); } @@ -7818,9 +8259,13 @@ // _MONITOR_RESUME { _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); int err = _Py_call_instrumentation( tstate, oparg == 0 ? PY_MONITORING_EVENT_PY_START : PY_MONITORING_EVENT_PY_RESUME, frame, this_instr); + _PyFrame_StackPointerInvalidate(frame); + // Explicit stack reload stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackAssertInvalid(frame); if (err) { JUMP_TO_LABEL(error); } @@ -7849,10 +8294,11 @@ { val = stack_pointer[-1]; _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); int err = _Py_call_instrumentation_arg( tstate, PY_MONITORING_EVENT_PY_RETURN, frame, this_instr, PyStackRef_AsPyObjectBorrow(val)); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (err) { JUMP_TO_LABEL(error); } @@ -7870,6 +8316,7 @@ stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); assert(STACK_LEVEL() == 0); DTRACE_FUNCTION_RETURN(); _Py_LeaveRecursiveCallPy(tstate); @@ -7877,6 +8324,7 @@ frame = tstate->current_frame = dying->previous; _PyEval_FrameClearAndPop(tstate, dying); stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); LOAD_IP(frame->return_offset); res = temp; LLTRACE_RESUME_FRAME(); @@ -7905,14 +8353,21 @@ { val = stack_pointer[-1]; _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); int err = _Py_call_instrumentation_arg( tstate, PY_MONITORING_EVENT_PY_YIELD, frame, this_instr, PyStackRef_AsPyObjectBorrow(val)); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (err) { + // Explicit stack reload + stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackAssertInvalid(frame); JUMP_TO_LABEL(error); } if (frame->instr_ptr != this_instr) { + // Explicit stack reload + stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackAssertInvalid(frame); next_instr = frame->instr_ptr; DISPATCH(); } @@ -7934,6 +8389,7 @@ stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); DTRACE_FUNCTION_RETURN(); tstate->exc_info = gen->gi_exc_state.previous_item; gen->gi_exc_state.previous_item = NULL; @@ -7954,6 +8410,7 @@ } #endif stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); LOAD_IP(1 + INLINE_CACHE_ENTRIES_SEND); value = temp; LLTRACE_RESUME_FRAME(); @@ -7986,12 +8443,14 @@ _PyStackRef executor = frame->localsplus[0]; assert(tstate->current_executor == NULL); if (!PyStackRef_IsNull(executor)) { + assert(PyStackRef_TYPE(executor) == &_PyUOpExecutor_Type); tstate->current_executor = PyStackRef_AsPyObjectBorrow(executor); stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(executor); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); stack_pointer += 1; } #endif @@ -8028,8 +8487,9 @@ stack_pointer[-2] = b; stack_pointer[-1] = l; _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_XCLOSE(value); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } // _POP_TOP { @@ -8037,8 +8497,9 @@ stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_XCLOSE(value); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } DISPATCH(); } @@ -8069,8 +8530,9 @@ // _CHECK_PERIODIC { _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); int err = check_periodics(tstate); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (err != 0) { JUMP_TO_LABEL(error); } @@ -8098,8 +8560,9 @@ // _CHECK_PERIODIC { _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); int err = check_periodics(tstate); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (err != 0) { JUMP_TO_LABEL(error); } @@ -8166,8 +8629,9 @@ // _CHECK_PERIODIC { _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); int err = check_periodics(tstate); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (err != 0) { JUMP_TO_LABEL(error); } @@ -8233,19 +8697,24 @@ PyObject *list = PyStackRef_AsPyObjectBorrow(list_st); PyObject *iterable = PyStackRef_AsPyObjectBorrow(iterable_st); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyObject *none_val = _PyList_Extend((PyListObject *)list, iterable); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (none_val == NULL) { int matches = _PyErr_ExceptionMatches(tstate, PyExc_TypeError); if (matches && (Py_TYPE(iterable)->tp_iter == NULL && !PySequence_Check(iterable))) { - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); _PyErr_Clear(tstate); + _PyFrame_StackPointerInvalidate(frame); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); _PyErr_Format(tstate, PyExc_TypeError, "Value after * must be an iterable, not %.200s", Py_TYPE(iterable)->tp_name); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } JUMP_TO_LABEL(error); } @@ -8258,8 +8727,9 @@ stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_XCLOSE(value); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } DISPATCH(); } @@ -8288,8 +8758,9 @@ PyObject *name = GETITEM(FRAME_CO_NAMES, oparg>>1); next_instr = this_instr; _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); _Py_Specialize_LoadAttr(owner, next_instr, name); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); DISPATCH_SAME_OPARG(); } OPCODE_DEFERRED_INC(LOAD_ATTR); @@ -8303,22 +8774,25 @@ PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 1); if (oparg & 1) { _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); attr = _Py_LoadAttr_StackRefSteal(tstate, owner, name, self_or_null); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (PyStackRef_IsNull(attr)) { JUMP_TO_LABEL(pop_1_error); } } else { _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); attr = _PyObject_GetAttrStackRef(PyStackRef_AsPyObjectBorrow(owner), name); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); stack_pointer[-1] = attr; stack_pointer += (oparg&1); ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(owner); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (PyStackRef_IsNull(attr)) { JUMP_TO_LABEL(error); } @@ -8371,11 +8845,12 @@ assert(descr != NULL); attr = PyStackRef_FromPyObjectNew(descr); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); _PyStackRef tmp = owner; owner = attr; stack_pointer[-1] = owner; PyStackRef_CLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } // _PUSH_NULL_CONDITIONAL { @@ -8439,11 +8914,12 @@ assert(descr != NULL); attr = PyStackRef_FromPyObjectNew(descr); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); _PyStackRef tmp = owner; owner = attr; stack_pointer[-1] = owner; PyStackRef_CLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } // _PUSH_NULL_CONDITIONAL { @@ -8535,12 +9011,17 @@ stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); assert(temp->previous == frame || temp->previous->previous == frame); CALL_STAT_INC(inlined_py_calls); frame = tstate->current_frame = temp; tstate->py_recursion_remaining--; - LOAD_SP(); + stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); LOAD_IP(0); + #ifdef Py_DEBUG + assert(frame->previous->stackpointer_valid == 1); + #endif DTRACE_FUNCTION_ENTRY(); LLTRACE_RESUME_FRAME(); } @@ -8618,8 +9099,9 @@ value = o; stack_pointer[-1] = attr; _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_XCLOSE(value); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } /* Skip 5 cache entries */ // _PUSH_NULL_CONDITIONAL @@ -8857,8 +9339,9 @@ value = o; stack_pointer[-1] = attr; _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_XCLOSE(value); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } /* Skip 5 cache entries */ // _PUSH_NULL_CONDITIONAL @@ -8910,8 +9393,9 @@ stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(owner); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); attr = PyStackRef_FromPyObjectNew(descr); } stack_pointer[0] = attr; @@ -8967,8 +9451,9 @@ stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(owner); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); attr = PyStackRef_FromPyObjectNew(descr); } stack_pointer[0] = attr; @@ -9050,12 +9535,17 @@ stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); assert(temp->previous == frame || temp->previous->previous == frame); CALL_STAT_INC(inlined_py_calls); frame = tstate->current_frame = temp; tstate->py_recursion_remaining--; - LOAD_SP(); + stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); LOAD_IP(0); + #ifdef Py_DEBUG + assert(frame->previous->stackpointer_valid == 1); + #endif DTRACE_FUNCTION_ENTRY(); LLTRACE_RESUME_FRAME(); } @@ -9120,8 +9610,9 @@ value = o; stack_pointer[-1] = attr; _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_XCLOSE(value); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } /* Skip 5 cache entries */ // _PUSH_NULL_CONDITIONAL @@ -9237,8 +9728,9 @@ value = o; stack_pointer[-1] = attr; _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_XCLOSE(value); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } /* Skip 5 cache entries */ // _PUSH_NULL_CONDITIONAL @@ -9264,16 +9756,18 @@ _PyStackRef bc; int err; _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyObject *bc_o = _PyMapping_GetOptionalItem2(BUILTINS(), &_Py_ID(__build_class__), &err); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (err < 0) { JUMP_TO_LABEL(error); } if (bc_o == NULL) { - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); _PyErr_SetString(tstate, PyExc_NameError, "__build_class__ not found"); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); JUMP_TO_LABEL(error); } bc = PyStackRef_FromPyObjectSteal(bc_o); @@ -9329,15 +9823,17 @@ _PyStackRef value; PyCellObject *cell = (PyCellObject *)PyStackRef_AsPyObjectBorrow(GETLOCAL(oparg)); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); value = _PyCell_GetStackRef(cell); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (PyStackRef_IsNull(value)) { stack_pointer[0] = value; stack_pointer += 1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); _PyEval_FormatExcUnbound(tstate, _PyFrame_GetCode(frame), oparg); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); JUMP_TO_LABEL(error); } stack_pointer[0] = value; @@ -9430,11 +9926,12 @@ _PyStackRef value_s = GETLOCAL(oparg); if (PyStackRef_IsNull(value_s)) { _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); _PyEval_FormatExcCheckArg(tstate, PyExc_UnboundLocalError, UNBOUNDLOCAL_ERROR_MSG, PyTuple_GetItem(_PyFrame_GetCode(frame)->co_localsplusnames, oparg) ); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); JUMP_TO_LABEL(error); } value = PyStackRef_DUP(value_s); @@ -9483,8 +9980,9 @@ name = PyTuple_GET_ITEM(_PyFrame_GetCode(frame)->co_localsplusnames, oparg); int err; _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyObject* value_o = _PyMapping_GetOptionalItem2(class_dict, name, &err); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (err < 0) { JUMP_TO_LABEL(error); } @@ -9492,17 +9990,19 @@ PyCellObject *cell = (PyCellObject *)PyStackRef_AsPyObjectBorrow(GETLOCAL(oparg)); value_o = PyCell_GetRef(cell); if (value_o == NULL) { - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); _PyEval_FormatExcUnbound(tstate, _PyFrame_GetCode(frame), oparg); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); JUMP_TO_LABEL(error); } } stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(class_dict_st); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); value = PyStackRef_FromPyObjectSteal(value_o); stack_pointer[0] = value; stack_pointer += 1; @@ -9524,13 +10024,15 @@ PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); int err; _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyObject *v_o = _PyMapping_GetOptionalItem2(PyStackRef_AsPyObjectBorrow(mod_or_class_dict), name, &err); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(mod_or_class_dict); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (err < 0) { JUMP_TO_LABEL(error); } @@ -9538,58 +10040,71 @@ if (PyDict_CheckExact(GLOBALS()) && PyDict_CheckExact(BUILTINS())) { - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); v_o = _PyDict_LoadGlobal((PyDictObject *)GLOBALS(), (PyDictObject *)BUILTINS(), name); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (v_o == NULL) { if (!_PyErr_Occurred(tstate)) { - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); _PyEval_FormatExcCheckArg(tstate, PyExc_NameError, NAME_ERROR_MSG, name); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } JUMP_TO_LABEL(error); } if (PyLazyImport_CheckExact(v_o)) { - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); PyObject *l_v = _PyImport_LoadLazyImportTstate(tstate, v_o); + _PyFrame_StackPointerInvalidate(frame); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); Py_SETREF(v_o, l_v); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (v_o == NULL) { JUMP_TO_LABEL(error); } } } else { - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); v_o = _PyMapping_GetOptionalItem2(GLOBALS(), name, &err); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (err < 0) { JUMP_TO_LABEL(error); } if (v_o == NULL) { - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); v_o = _PyMapping_GetOptionalItem2(BUILTINS(), name, &err); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (err < 0) { JUMP_TO_LABEL(error); } if (v_o == NULL) { - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); _PyEval_FormatExcCheckArg( tstate, PyExc_NameError, NAME_ERROR_MSG, name); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); JUMP_TO_LABEL(error); } } if (PyLazyImport_CheckExact(v_o)) { - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); PyObject *l_v = _PyImport_LoadLazyImportTstate(tstate, v_o); + _PyFrame_StackPointerInvalidate(frame); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); Py_SETREF(v_o, l_v); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (v_o == NULL) { JUMP_TO_LABEL(error); } @@ -9625,8 +10140,9 @@ PyObject *name = GETITEM(FRAME_CO_NAMES, oparg>>1); next_instr = this_instr; _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); _Py_Specialize_LoadGlobal(GLOBALS(), BUILTINS(), next_instr, name); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); DISPATCH_SAME_OPARG(); } OPCODE_DEFERRED_INC(LOAD_GLOBAL); @@ -9641,8 +10157,9 @@ res = &stack_pointer[0]; PyObject *name = GETITEM(FRAME_CO_NAMES, oparg>>1); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); _PyEval_LoadGlobalStackRef(GLOBALS(), BUILTINS(), name, res); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (PyStackRef_IsNull(*res)) { JUMP_TO_LABEL(error); } @@ -9818,9 +10335,10 @@ PyObject *l = LOCALS(); if (l == NULL) { _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); _PyErr_SetString(tstate, PyExc_SystemError, "no locals found"); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); JUMP_TO_LABEL(error); } locals = PyStackRef_FromPyObjectNew(l); @@ -9841,34 +10359,43 @@ _PyStackRef v; PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyObject *v_o = _PyEval_LoadName(tstate, frame, name); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (v_o == NULL) { JUMP_TO_LABEL(error); } if (PyLazyImport_CheckExact(v_o)) { - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); PyObject *l_v = _PyImport_LoadLazyImportTstate(tstate, v_o); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (l_v == NULL) { - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); Py_DECREF(v_o); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); JUMP_TO_LABEL(error); } - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); int err = PyDict_SetItem(GLOBALS(), name, l_v); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (err < 0) { - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); Py_DECREF(v_o); + _PyFrame_StackPointerInvalidate(frame); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); Py_DECREF(l_v); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); JUMP_TO_LABEL(error); } - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); Py_SETREF(v_o, l_v); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } v = PyStackRef_FromPyObjectSteal(v_o); stack_pointer[0] = v; @@ -9919,21 +10446,24 @@ stack_pointer += 1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); int err = _PyObject_LookupSpecialMethod(name, method_and_self); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (err <= 0) { if (err == 0) { PyObject *owner = PyStackRef_AsPyObjectBorrow(method_and_self[1]); - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); const char *errfmt = _PyEval_SpecialMethodCanSuggest(owner, oparg) ? _Py_SpecialMethods[oparg].error_suggestion : _Py_SpecialMethods[oparg].error; - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); assert(!_PyErr_Occurred(tstate)); assert(errfmt != NULL); - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); _PyErr_Format(tstate, PyExc_TypeError, errfmt, owner); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } JUMP_TO_LABEL(error); } @@ -9969,8 +10499,9 @@ if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { next_instr = this_instr; _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); _Py_Specialize_LoadSuperAttr(global_super_st, class_st, next_instr, load_method); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); DISPATCH_SAME_OPARG(); } OPCODE_DEFERRED_INC(LOAD_SUPER_ATTR); @@ -9986,25 +10517,33 @@ if (opcode == INSTRUMENTED_LOAD_SUPER_ATTR) { PyObject *arg = oparg & 2 ? class : &_PyInstrumentation_MISSING; _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); int err = _Py_call_instrumentation_2args( tstate, PY_MONITORING_EVENT_CALL, frame, this_instr, global_super, arg); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (err) { - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); _PyStackRef tmp = self_st; self_st = PyStackRef_NULL; stack_pointer[-1] = self_st; PyStackRef_CLOSE(tmp); + _PyFrame_StackPointerInvalidate(frame); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); tmp = class_st; class_st = PyStackRef_NULL; stack_pointer[-2] = class_st; PyStackRef_CLOSE(tmp); + _PyFrame_StackPointerInvalidate(frame); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); tmp = global_super_st; global_super_st = PyStackRef_NULL; stack_pointer[-3] = global_super_st; PyStackRef_CLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); stack_pointer += -3; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); JUMP_TO_LABEL(error); @@ -10014,45 +10553,56 @@ { PyObject *stack[] = {class, self}; _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); super = PyObject_Vectorcall(global_super, stack, oparg & 2, NULL); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } if (opcode == INSTRUMENTED_LOAD_SUPER_ATTR) { PyObject *arg = oparg & 2 ? class : &_PyInstrumentation_MISSING; if (super == NULL) { - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); _Py_call_instrumentation_exc2( tstate, PY_MONITORING_EVENT_C_RAISE, frame, this_instr, global_super, arg); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } else { - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); int err = _Py_call_instrumentation_2args( tstate, PY_MONITORING_EVENT_C_RETURN, frame, this_instr, global_super, arg); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (err < 0) { - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); Py_CLEAR(super); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } } } - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); _PyStackRef tmp = self_st; self_st = PyStackRef_NULL; stack_pointer[-1] = self_st; PyStackRef_CLOSE(tmp); + _PyFrame_StackPointerInvalidate(frame); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); tmp = class_st; class_st = PyStackRef_NULL; stack_pointer[-2] = class_st; PyStackRef_CLOSE(tmp); + _PyFrame_StackPointerInvalidate(frame); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); tmp = global_super_st; global_super_st = PyStackRef_NULL; stack_pointer[-3] = global_super_st; PyStackRef_CLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); stack_pointer += -3; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); if (super == NULL) { @@ -10060,9 +10610,13 @@ } PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 2); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyObject *attr_o = PyObject_GetAttr(super, name); + _PyFrame_StackPointerInvalidate(frame); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); Py_DECREF(super); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (attr_o == NULL) { JUMP_TO_LABEL(error); } @@ -10117,20 +10671,30 @@ STAT_INC(LOAD_SUPER_ATTR, hit); PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 2); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyObject *attr = _PySuper_Lookup((PyTypeObject *)class, self, name, NULL); + _PyFrame_StackPointerInvalidate(frame); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); _PyStackRef tmp = self_st; self_st = PyStackRef_NULL; stack_pointer[-1] = self_st; PyStackRef_CLOSE(tmp); + _PyFrame_StackPointerInvalidate(frame); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); tmp = class_st; class_st = PyStackRef_NULL; stack_pointer[-2] = class_st; PyStackRef_CLOSE(tmp); + _PyFrame_StackPointerInvalidate(frame); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); tmp = global_super_st; global_super_st = PyStackRef_NULL; stack_pointer[-3] = global_super_st; PyStackRef_CLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); stack_pointer += -3; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); if (attr == NULL) { @@ -10191,9 +10755,10 @@ { int *method_found_ptr = &method_found; _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); attr_o = _PySuper_Lookup(cls, self, name, Py_TYPE(self)->tp_getattro == PyObject_GenericGetAttr ? method_found_ptr : NULL); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } if (attr_o == NULL) { JUMP_TO_LABEL(error); @@ -10204,23 +10769,28 @@ stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(self_st); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); self_or_null = PyStackRef_NULL; stack_pointer += 1; } stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); _PyStackRef tmp = global_super_st; global_super_st = self_or_null; stack_pointer[-2] = global_super_st; PyStackRef_CLOSE(tmp); + _PyFrame_StackPointerInvalidate(frame); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); tmp = class_st; class_st = PyStackRef_NULL; stack_pointer[-1] = class_st; PyStackRef_CLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); stack_pointer += -2; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); attr = PyStackRef_FromPyObjectSteal(attr_o); @@ -10248,8 +10818,9 @@ _PyStackRef tmp = GETLOCAL(oparg); GETLOCAL(oparg) = PyStackRef_FromPyObjectSteal(cell); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_XCLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); DISPATCH(); } @@ -10270,9 +10841,10 @@ codeobj_st = stack_pointer[-1]; PyObject *codeobj = PyStackRef_AsPyObjectBorrow(codeobj_st); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyFunctionObject *func_obj = (PyFunctionObject *) PyFunction_New(codeobj, GLOBALS()); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (func_obj == NULL) { JUMP_TO_LABEL(error); } @@ -10285,9 +10857,10 @@ { value = co; stack_pointer[-1] = func; - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); PyStackRef_XCLOSE(value); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } DISPATCH(); } @@ -10309,12 +10882,13 @@ PyObject *dict = PyStackRef_AsPyObjectBorrow(dict_st); assert(PyDict_CheckExact(dict)); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); int err = _PyDict_SetItem_Take2( (PyDictObject *)dict, PyStackRef_AsPyObjectSteal(key), PyStackRef_AsPyObjectSteal(value) ); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (err != 0) { JUMP_TO_LABEL(pop_2_error); } @@ -10346,11 +10920,12 @@ subject = stack_pointer[-3]; assert(PyTuple_CheckExact(PyStackRef_AsPyObjectBorrow(names))); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyObject *attrs_o = _PyEval_MatchClass(tstate, PyStackRef_AsPyObjectBorrow(subject), PyStackRef_AsPyObjectBorrow(type), oparg, PyStackRef_AsPyObjectBorrow(names)); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (attrs_o) { assert(PyTuple_CheckExact(attrs_o)); attrs = PyStackRef_FromPyObjectSteal(attrs_o); @@ -10371,9 +10946,10 @@ stack_pointer[-3] = attrs; stack_pointer[-2] = s; stack_pointer[-1] = tp; - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); PyStackRef_XCLOSE(value); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } // _POP_TOP { @@ -10381,8 +10957,9 @@ stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_XCLOSE(value); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } // _POP_TOP { @@ -10390,8 +10967,9 @@ stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_XCLOSE(value); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } DISPATCH(); } @@ -10410,9 +10988,10 @@ keys = stack_pointer[-1]; subject = stack_pointer[-2]; _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyObject *values_or_none_o = _PyEval_MatchKeys(tstate, PyStackRef_AsPyObjectBorrow(subject), PyStackRef_AsPyObjectBorrow(keys)); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (values_or_none_o == NULL) { JUMP_TO_LABEL(error); } @@ -10495,10 +11074,11 @@ exc_value = stack_pointer[-1]; _PyErr_StackItem *exc_info = tstate->exc_info; _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); Py_XSETREF(exc_info->exc_value, PyStackRef_IsNone(exc_value) ? NULL : PyStackRef_AsPyObjectSteal(exc_value)); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); @@ -10520,8 +11100,9 @@ stack_pointer += -2; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(iter); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); DISPATCH(); } @@ -10570,11 +11151,12 @@ else { b = PyStackRef_False; _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); _PyStackRef tmp = value; value = b; stack_pointer[-1] = value; PyStackRef_CLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } } // _POP_JUMP_IF_TRUE @@ -10613,11 +11195,12 @@ else { b = PyStackRef_False; _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); _PyStackRef tmp = value; value = b; stack_pointer[-1] = value; PyStackRef_CLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } } // _POP_JUMP_IF_FALSE @@ -10668,8 +11251,9 @@ stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_XCLOSE(value); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); DISPATCH(); } @@ -10736,12 +11320,17 @@ stack_pointer += -oparg; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); int err = do_raise(tstate, exc, cause); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (err) { assert(oparg == 0); - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); monitor_reraise(tstate, frame, this_instr); + _PyFrame_StackPointerInvalidate(frame); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); JUMP_TO_LABEL(exception_unwind); } JUMP_TO_LABEL(error); @@ -10770,8 +11359,15 @@ stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); _PyErr_SetRaisedException(tstate, exc); + _PyFrame_StackPointerInvalidate(frame); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); monitor_reraise(tstate, frame, this_instr); + _PyFrame_StackPointerInvalidate(frame); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); JUMP_TO_LABEL(exception_unwind); } @@ -10805,9 +11401,10 @@ if (frame->tlbc_index != ((_PyThreadStateImpl *)tstate)->tlbc_index) { _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); _Py_CODEUNIT *bytecode = _PyEval_GetExecutableCode(tstate, _PyFrame_GetCode(frame)); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (bytecode == NULL) { JUMP_TO_LABEL(error); } @@ -10832,8 +11429,9 @@ uintptr_t code_version = FT_ATOMIC_LOAD_UINTPTR_ACQUIRE(_PyFrame_GetCode(frame)->_co_instrumentation_version); if (code_version != global_version) { _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); int err = _Py_Instrument(_PyFrame_GetCode(frame), tstate->interp); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (err) { JUMP_TO_LABEL(error); } @@ -10847,16 +11445,18 @@ uint16_t counter = read_u16(&this_instr[1].cache); (void)counter; _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); _Py_Specialize_Resume(this_instr, tstate, frame); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } // _CHECK_PERIODIC_IF_NOT_YIELD_FROM { Test_EvalFrame_Resumes++; if ((oparg & RESUME_OPARG_LOCATION_MASK) < RESUME_AFTER_YIELD_FROM) { - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); int err = check_periodics(tstate); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (err != 0) { JUMP_TO_LABEL(error); } @@ -10986,13 +11586,15 @@ assert(PyStackRef_FunctionCheck(frame->f_funcobj)); PyFunctionObject *func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyGenObject *gen = (PyGenObject *)_Py_MakeCoro(func); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (gen == NULL) { JUMP_TO_LABEL(error); } assert(STACK_LEVEL() <= 2); - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); _PyInterpreterFrame *gen_frame = &gen->gi_iframe; frame->instr_ptr++; _PyFrame_Copy(frame, gen_frame); @@ -11006,6 +11608,7 @@ frame = tstate->current_frame = prev; LOAD_IP(frame->return_offset); stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); res = PyStackRef_FromPyObjectStealMortal((PyObject *)gen); LLTRACE_RESUME_FRAME(); stack_pointer[0] = res; @@ -11038,6 +11641,7 @@ stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); assert(STACK_LEVEL() == 0); DTRACE_FUNCTION_RETURN(); _Py_LeaveRecursiveCallPy(tstate); @@ -11045,6 +11649,7 @@ frame = tstate->current_frame = dying->previous; _PyEval_FrameClearAndPop(tstate, dying); stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); LOAD_IP(frame->return_offset); res = temp; LLTRACE_RESUME_FRAME(); @@ -11079,8 +11684,9 @@ if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { next_instr = this_instr; _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); _Py_Specialize_Send(receiver, next_instr); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); DISPATCH_SAME_OPARG(); } OPCODE_DEFERRED_INC(SEND); @@ -11112,8 +11718,9 @@ } if (!PyStackRef_IsNull(null_or_index) && PyStackRef_IsNone(v)) { _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); _PyStackRef item = _PyForIter_VirtualIteratorNext(tstate, frame, receiver, &null_or_index); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (!PyStackRef_IsValid(item)) { if (PyStackRef_IsError(item)) { JUMP_TO_LABEL(error); @@ -11127,16 +11734,18 @@ else { PyObject *v_o = PyStackRef_AsPyObjectBorrow(v); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PySendResultPair res = _PyIter_Send(receiver_o, v_o); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (res.kind == PYGEN_ERROR) { JUMP_TO_LABEL(error); } stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(v); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); retval = PyStackRef_FromPyObjectSteal(res.object); if (res.kind == PYGEN_RETURN) { JUMPBY(oparg); @@ -11186,16 +11795,18 @@ PyObject *val = PyStackRef_AsPyObjectBorrow(v); PyObject *retval_o; _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PySendResult what = _PyAsyncGenASend_Send(iter_o, val, &retval_o); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (what == PYGEN_ERROR) { JUMP_TO_LABEL(error); } stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(v); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); asend = iter; null_out = null_in; retval = PyStackRef_FromPyObjectSteal(retval_o); @@ -11268,12 +11879,17 @@ stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); assert(temp->previous == frame || temp->previous->previous == frame); CALL_STAT_INC(inlined_py_calls); frame = tstate->current_frame = temp; tstate->py_recursion_remaining--; - LOAD_SP(); + stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); LOAD_IP(0); + #ifdef Py_DEBUG + assert(frame->previous->stackpointer_valid == 1); + #endif DTRACE_FUNCTION_ENTRY(); LLTRACE_RESUME_FRAME(); } @@ -11324,8 +11940,9 @@ PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter); Py_ssize_t index = PyStackRef_UntagInt(null_or_index); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); _PyObjectIndexPair next_index = Py_TYPE(iter_o)->_tp_iteritem(iter_o, index); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); PyObject *next_o = next_index.object; index = next_index.index; if (next_o == NULL) { @@ -11354,38 +11971,46 @@ INSTRUCTION_STATS(SETUP_ANNOTATIONS); if (LOCALS() == NULL) { _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); _PyErr_Format(tstate, PyExc_SystemError, "no locals found when setting up annotations"); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); JUMP_TO_LABEL(error); } int err; _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyObject* ann_dict = _PyMapping_GetOptionalItem2(LOCALS(), &_Py_ID(__annotations__), &err); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (err < 0) { JUMP_TO_LABEL(error); } if (ann_dict == NULL) { - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); ann_dict = PyDict_New(); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (ann_dict == NULL) { JUMP_TO_LABEL(error); } - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); err = PyObject_SetItem(LOCALS(), &_Py_ID(__annotations__), ann_dict); + _PyFrame_StackPointerInvalidate(frame); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); Py_DECREF(ann_dict); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (err) { JUMP_TO_LABEL(error); } } else { - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); Py_DECREF(ann_dict); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } DISPATCH(); } @@ -11403,9 +12028,10 @@ v = stack_pointer[-1]; set = stack_pointer[-2 - (oparg-1)]; _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); int err = _PySet_AddTakeRef((PySetObject *)PyStackRef_AsPyObjectBorrow(set), PyStackRef_AsPyObjectSteal(v)); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (err) { JUMP_TO_LABEL(pop_1_error); } @@ -11459,9 +12085,10 @@ iterable = stack_pointer[-1]; set = stack_pointer[-2 - (oparg-1)]; _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); int err = _PySet_Update(PyStackRef_AsPyObjectBorrow(set), PyStackRef_AsPyObjectBorrow(iterable)); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (err < 0) { JUMP_TO_LABEL(error); } @@ -11473,8 +12100,9 @@ stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_XCLOSE(value); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } DISPATCH(); } @@ -11502,8 +12130,9 @@ PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); next_instr = this_instr; _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); _Py_Specialize_StoreAttr(owner, next_instr, name); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); DISPATCH_SAME_OPARG(); } OPCODE_DEFERRED_INC(STORE_ATTR); @@ -11516,17 +12145,24 @@ v = stack_pointer[-2]; PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); int err = PyObject_SetAttr(PyStackRef_AsPyObjectBorrow(owner), name, PyStackRef_AsPyObjectBorrow(v)); + _PyFrame_StackPointerInvalidate(frame); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); _PyStackRef tmp = owner; owner = PyStackRef_NULL; stack_pointer[-1] = owner; PyStackRef_CLOSE(tmp); + _PyFrame_StackPointerInvalidate(frame); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); tmp = v; v = PyStackRef_NULL; stack_pointer[-2] = v; PyStackRef_CLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); stack_pointer += -2; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); if (err) { @@ -11612,8 +12248,9 @@ stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); Py_XDECREF(old_value); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } // _POP_TOP { @@ -11621,8 +12258,9 @@ stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_XCLOSE(value); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } DISPATCH(); } @@ -11674,8 +12312,9 @@ stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); Py_XDECREF(old_value); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } // _POP_TOP { @@ -11683,8 +12322,9 @@ stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_XCLOSE(value); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } DISPATCH(); } @@ -11763,8 +12403,9 @@ } } _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); _PyDict_NotifyEvent(PyDict_EVENT_MODIFIED, dict, name, PyStackRef_AsPyObjectBorrow(value)); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); FT_ATOMIC_STORE_PTR_RELEASE(ep->me_value, PyStackRef_AsPyObjectSteal(value)); UNLOCK_OBJECT(dict); STAT_INC(STORE_ATTR, hit); @@ -11773,8 +12414,9 @@ stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); Py_XDECREF(old_value); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } // _POP_TOP { @@ -11782,8 +12424,9 @@ stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_XCLOSE(value); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } DISPATCH(); } @@ -11800,8 +12443,9 @@ v = stack_pointer[-1]; PyCellObject *cell = (PyCellObject *)PyStackRef_AsPyObjectBorrow(GETLOCAL(oparg)); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyCell_SetTakeRef(cell, PyStackRef_AsPyObjectSteal(v)); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); @@ -11830,8 +12474,9 @@ stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_XCLOSE(value); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } DISPATCH(); } @@ -11854,8 +12499,9 @@ value2 = PyStackRef_DUP(GETLOCAL(oparg2)); stack_pointer[-1] = value2; _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_XCLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); DISPATCH(); } @@ -11878,15 +12524,17 @@ stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_XCLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); tmp = GETLOCAL(oparg2); GETLOCAL(oparg2) = value2; stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_XCLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); DISPATCH(); } @@ -11902,13 +12550,15 @@ v = stack_pointer[-1]; PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); int err = PyDict_SetItem(GLOBALS(), name, PyStackRef_AsPyObjectBorrow(v)); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(v); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (err) { JUMP_TO_LABEL(error); } @@ -11930,31 +12580,36 @@ int err; if (ns == NULL) { _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); _PyErr_Format(tstate, PyExc_SystemError, "no locals found when storing %R", name); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(v); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); JUMP_TO_LABEL(error); } if (PyDict_CheckExact(ns)) { _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); err = PyDict_SetItem(ns, name, PyStackRef_AsPyObjectBorrow(v)); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } else { _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); err = PyObject_SetItem(ns, name, PyStackRef_AsPyObjectBorrow(v)); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(v); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (err) { JUMP_TO_LABEL(error); } @@ -11986,10 +12641,11 @@ container = stack_pointer[-3]; v = stack_pointer[-4]; _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyObject *slice = _PyBuildSlice_ConsumeRefs(PyStackRef_AsPyObjectSteal(start), PyStackRef_AsPyObjectSteal(stop), Py_None); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); int err; if (slice == NULL) { err = 1; @@ -11998,21 +12654,29 @@ stack_pointer += -2; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); err = PyObject_SetItem(PyStackRef_AsPyObjectBorrow(container), slice, PyStackRef_AsPyObjectBorrow(v)); + _PyFrame_StackPointerInvalidate(frame); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); Py_DECREF(slice); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); stack_pointer += 2; } _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); _PyStackRef tmp = container; container = PyStackRef_NULL; stack_pointer[-3] = container; PyStackRef_CLOSE(tmp); + _PyFrame_StackPointerInvalidate(frame); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); tmp = v; v = PyStackRef_NULL; stack_pointer[-4] = v; PyStackRef_CLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); stack_pointer += -4; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); if (err) { @@ -12046,8 +12710,9 @@ if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { next_instr = this_instr; _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); _Py_Specialize_StoreSubscr(container, sub, next_instr); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); DISPATCH_SAME_OPARG(); } OPCODE_DEFERRED_INC(STORE_SUBSCR); @@ -12058,20 +12723,30 @@ { v = stack_pointer[-3]; _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); int err = PyObject_SetItem(PyStackRef_AsPyObjectBorrow(container), PyStackRef_AsPyObjectBorrow(sub), PyStackRef_AsPyObjectBorrow(v)); + _PyFrame_StackPointerInvalidate(frame); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); _PyStackRef tmp = sub; sub = PyStackRef_NULL; stack_pointer[-1] = sub; PyStackRef_CLOSE(tmp); + _PyFrame_StackPointerInvalidate(frame); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); tmp = container; container = PyStackRef_NULL; stack_pointer[-2] = container; PyStackRef_CLOSE(tmp); + _PyFrame_StackPointerInvalidate(frame); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); tmp = v; v = PyStackRef_NULL; stack_pointer[-3] = v; PyStackRef_CLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); stack_pointer += -3; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); if (err) { @@ -12122,16 +12797,18 @@ assert(Py_TYPE(dict)->tp_as_mapping->mp_ass_subscript == _PyDict_StoreSubscript); STAT_INC(STORE_SUBSCR, hit); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); int err = _PyDict_SetItem_Take2((PyDictObject *)dict, PyStackRef_AsPyObjectSteal(sub), PyStackRef_AsPyObjectSteal(value)); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (err) { stack_pointer += -3; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(dict_st); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); JUMP_TO_LABEL(error); } st = dict_st; @@ -12142,8 +12819,9 @@ stack_pointer += -3; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_XCLOSE(value); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } DISPATCH(); } @@ -12226,8 +12904,9 @@ stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); Py_DECREF(old_value); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } // _POP_TOP_INT { @@ -12241,8 +12920,9 @@ stack_pointer += -2; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_XCLOSE(value); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } DISPATCH(); } @@ -12289,8 +12969,9 @@ if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { next_instr = this_instr; _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); _Py_Specialize_ToBool(value, next_instr); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); DISPATCH_SAME_OPARG(); } OPCODE_DEFERRED_INC(TO_BOOL); @@ -12301,13 +12982,15 @@ // _TO_BOOL { _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); int err = PyObject_IsTrue(PyStackRef_AsPyObjectBorrow(value)); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(value); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (err < 0) { JUMP_TO_LABEL(error); } @@ -12358,8 +13041,9 @@ value = v; stack_pointer[-1] = res; _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_XCLOSE(value); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } DISPATCH(); } @@ -12472,8 +13156,9 @@ value = v; stack_pointer[-1] = res; _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_XCLOSE(value); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } DISPATCH(); } @@ -12577,30 +13262,34 @@ _PyJitTracerState *tracer = _tstate->jit_tracer_state; assert(tracer != NULL); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); int full = !_PyJit_translate_single_bytecode_to_trace(tstate, frame, next_instr, stop_tracing ? _DEOPT : 0); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (full) { LEAVE_TRACING(); - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); int err = stop_tracing_and_jit(tstate, frame); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (err < 0) { JUMP_TO_LABEL(error); } DISPATCH(); } for (int i = 0; i < tracer->prev_state.recorded_count; i++) { - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); Py_CLEAR(tracer->prev_state.recorded_values[i]); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } tracer->prev_state.recorded_count = 0; tracer->prev_state.instr = next_instr; PyObject *prev_code = PyStackRef_AsPyObjectBorrow(frame->f_executable); if (tracer->prev_state.instr_code != (PyCodeObject *)prev_code) { - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); Py_SETREF(tracer->prev_state.instr_code, (PyCodeObject*)Py_NewRef((prev_code))); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } tracer->prev_state.instr_frame = frame; tracer->prev_state.instr_oparg = oparg; @@ -12639,8 +13328,9 @@ { value = stack_pointer[-1]; _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyObject *res_o = PyNumber_Invert(PyStackRef_AsPyObjectBorrow(value)); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (res_o == NULL) { JUMP_TO_LABEL(error); } @@ -12651,9 +13341,10 @@ { value = v; stack_pointer[-1] = res; - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); PyStackRef_XCLOSE(value); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } DISPATCH(); } @@ -12673,8 +13364,9 @@ { value = stack_pointer[-1]; _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyObject *res_o = PyNumber_Negative(PyStackRef_AsPyObjectBorrow(value)); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (res_o == NULL) { JUMP_TO_LABEL(error); } @@ -12685,9 +13377,10 @@ { value = v; stack_pointer[-1] = res; - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); PyStackRef_XCLOSE(value); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } DISPATCH(); } @@ -12726,9 +13419,13 @@ stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); int res = _PyEval_UnpackIterableStackRef(tstate, seq_o, oparg & 0xFF, oparg >> 8, top); + _PyFrame_StackPointerInvalidate(frame); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); Py_DECREF(seq_o); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (res == 0) { JUMP_TO_LABEL(error); } @@ -12759,8 +13456,9 @@ if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { next_instr = this_instr; _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); _Py_Specialize_UnpackSequence(seq, next_instr, oparg); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); DISPATCH_SAME_OPARG(); } OPCODE_DEFERRED_INC(UNPACK_SEQUENCE); @@ -12776,9 +13474,13 @@ stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); int res = _PyEval_UnpackIterableStackRef(tstate, seq_o, oparg, -1, top); + _PyFrame_StackPointerInvalidate(frame); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); Py_DECREF(seq_o); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (res == 0) { JUMP_TO_LABEL(error); } @@ -12841,8 +13543,9 @@ stack_pointer += -1 + oparg; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(seq); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } DISPATCH(); } @@ -12891,8 +13594,9 @@ stack_pointer += -1 + oparg; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(seq); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } DISPATCH(); } @@ -12942,8 +13646,9 @@ stack_pointer += 1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(seq); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } DISPATCH(); } @@ -12981,13 +13686,15 @@ PyObject *stack[5] = {NULL, PyStackRef_AsPyObjectBorrow(exit_self), exc, val_o, tb}; int has_self = !PyStackRef_IsNull(exit_self); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); res_o = PyObject_Vectorcall(exit_func_o, stack + 2 - has_self, (3 + has_self) | PY_VECTORCALL_ARGUMENTS_OFFSET, NULL); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); Py_XDECREF(original_tb); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (res_o == NULL) { JUMP_TO_LABEL(error); } @@ -13026,6 +13733,7 @@ stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); DTRACE_FUNCTION_RETURN(); tstate->exc_info = gen->gi_exc_state.previous_item; gen->gi_exc_state.previous_item = NULL; @@ -13046,6 +13754,7 @@ } #endif stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); LOAD_IP(1 + INLINE_CACHE_ENTRIES_SEND); value = temp; LLTRACE_RESUME_FRAME(); @@ -13098,32 +13807,42 @@ JUMP_TO_LABEL(error); LABEL(error) { + _PyFrame_StackAssertInvalid(frame); #ifdef NDEBUG if (!_PyErr_Occurred(tstate)) { _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); _PyErr_SetString(tstate, PyExc_SystemError, "error return without exception set"); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } #else assert(_PyErr_Occurred(tstate)); #endif _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); STOP_TRACING(); stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); assert(frame->owner != FRAME_OWNED_BY_INTERPRETER); if (!_PyFrame_IsIncomplete(frame)) { - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); PyFrameObject *f = _PyFrame_GetFrameObject(frame); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (f != NULL) { - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); PyTraceBack_Here(f); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } } - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); _PyEval_MonitorRaise(tstate, frame, next_instr-1); + _PyFrame_StackPointerInvalidate(frame); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); JUMP_TO_LABEL(exception_unwind); } @@ -13168,6 +13887,7 @@ JUMP_TO_LABEL(error); } #endif stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); #if _Py_TAIL_CALL_INTERP int opcode; #endif @@ -13207,11 +13927,15 @@ JUMP_TO_LABEL(error); } next_instr = frame->instr_ptr; stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); JUMP_TO_LABEL(error); } LABEL(start_frame) { + #ifdef Py_DEBUG + assert(frame->stackpointer_valid == 1); + #endif int too_deep = _Py_EnterRecursivePy(tstate); if (too_deep) { JUMP_TO_LABEL(exit_unwind_notrace); @@ -13227,6 +13951,7 @@ JUMP_TO_LABEL(error); assert(!_PyErr_Occurred(tstate)); #endif stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); #if _Py_TAIL_CALL_INTERP int opcode; #endif @@ -13242,12 +13967,14 @@ JUMP_TO_LABEL(error); assert(IS_JIT_TRACING()); int opcode = next_instr->op.code; _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); _PyJit_translate_single_bytecode_to_trace(tstate, frame, NULL, _EXIT_TRACE); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); LEAVE_TRACING(); - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); int err = stop_tracing_and_jit(tstate, frame); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (err < 0) { JUMP_TO_LABEL(error); } diff --git a/Python/bytecodes.c b/Python/bytecodes.c index 51a188a6120f..8afa0be70235 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -250,6 +250,7 @@ dummy_func( op(_MONITOR_RESUME, (--)) { int err = _Py_call_instrumentation( tstate, oparg == 0 ? PY_MONITORING_EVENT_PY_START : PY_MONITORING_EVENT_PY_RESUME, frame, this_instr); + RELOAD_STACK(); ERROR_IF(err); if (frame->instr_ptr != this_instr) { /* Instrumentation has jumped */ @@ -1548,6 +1549,7 @@ dummy_func( _PyStackRef executor = frame->localsplus[0]; assert(tstate->current_executor == NULL); if (!PyStackRef_IsNull(executor)) { + assert(PyStackRef_TYPE(executor) == &_PyUOpExecutor_Type); tstate->current_executor = PyStackRef_AsPyObjectBorrow(executor); PyStackRef_CLOSE(executor); } @@ -1890,9 +1892,11 @@ dummy_func( tstate, PY_MONITORING_EVENT_PY_YIELD, frame, this_instr, PyStackRef_AsPyObjectBorrow(val)); if (err) { + RELOAD_STACK(); ERROR_NO_POP(); } if (frame->instr_ptr != this_instr) { + RELOAD_STACK(); next_instr = frame->instr_ptr; DISPATCH(); } @@ -4656,14 +4660,16 @@ dummy_func( assert(!IS_PEP523_HOOKED(tstate)); _PyInterpreterFrame *temp = PyStackRef_Unwrap(new_frame); DEAD(new_frame); - SYNC_SP(); - _PyFrame_SetStackPointer(frame, stack_pointer); + SAVE_STACK(); assert(temp->previous == frame || temp->previous->previous == frame); CALL_STAT_INC(inlined_py_calls); frame = tstate->current_frame = temp; tstate->py_recursion_remaining--; - LOAD_SP(); + RELOAD_STACK(); LOAD_IP(0); + #ifdef Py_DEBUG + assert(frame->previous->stackpointer_valid == 1); + #endif DTRACE_FUNCTION_ENTRY(); LLTRACE_RESUME_FRAME(); } @@ -5980,13 +5986,13 @@ dummy_func( } else { original_opcode = _Py_call_instrumentation_line( tstate, frame, this_instr, prev_instr); + RELOAD_STACK(); if (original_opcode < 0) { next_instr = this_instr+1; goto error; } next_instr = frame->instr_ptr; if (next_instr != this_instr) { - SYNC_SP(); DISPATCH(); } } @@ -5997,6 +6003,7 @@ dummy_func( PAUSE_ADAPTIVE_COUNTER(cache->counter); } opcode = original_opcode; + PRE_DISPATCH_GOTO(); DISPATCH_GOTO(); } @@ -6477,6 +6484,7 @@ dummy_func( } label(error) { + _PyFrame_StackAssertInvalid(frame); /* Double-check exception status. */ #ifdef NDEBUG if (!_PyErr_Occurred(tstate)) { @@ -6503,9 +6511,7 @@ dummy_func( } spilled label(exception_unwind) { - SAVE_STACK(); STOP_TRACING(); - RELOAD_STACK(); /* We can't use frame->instr_ptr here, as RERAISE may have set it */ int offset = INSTR_OFFSET()-1; int level, handler, lasti; @@ -6597,6 +6603,9 @@ dummy_func( } spilled label(start_frame) { + #ifdef Py_DEBUG + assert(frame->stackpointer_valid == 1); + #endif int too_deep = _Py_EnterRecursivePy(tstate); if (too_deep) { goto exit_unwind_notrace; diff --git a/Python/ceval.c b/Python/ceval.c index 32d31bee660a..fbea1f67a36f 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -1250,6 +1250,7 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int entry.frame.return_offset = 0; #ifdef Py_DEBUG entry.frame.lltrace = 0; + entry.frame.stackpointer_valid = 1; #endif /* Push frame */ entry.frame.previous = tstate->current_frame; @@ -1258,6 +1259,7 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int entry.frame.localsplus[0] = PyStackRef_NULL; #ifdef _Py_TIER2 if (tstate->current_executor != NULL) { + assert(Py_TYPE(tstate->current_executor) == &_PyUOpExecutor_Type); entry.frame.localsplus[0] = PyStackRef_FromPyObjectNew(tstate->current_executor); tstate->current_executor = NULL; } @@ -1286,6 +1288,7 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int next_instr = frame->instr_ptr; monitor_throw(tstate, frame, next_instr); stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); #if _Py_TAIL_CALL_INTERP # if Py_STATS return _TAIL_CALL_error(frame, stack_pointer, tstate, next_instr, instruction_funcptr_handler_table, 0, lastopcode); diff --git a/Python/ceval_macros.h b/Python/ceval_macros.h index 37479c4cb4eb..0de4477426af 100644 --- a/Python/ceval_macros.h +++ b/Python/ceval_macros.h @@ -198,7 +198,7 @@ do { \ /* Do interpreter dispatch accounting for tracing and instrumentation */ #define DISPATCH() \ { \ - assert(frame->stackpointer == NULL); \ + _PyFrame_StackAssertInvalid(frame); \ NEXTOPARG(); \ PRE_DISPATCH_GOTO(); \ DISPATCH_GOTO(); \ @@ -206,7 +206,7 @@ do { \ #define DISPATCH_NON_TRACING() \ { \ - assert(frame->stackpointer == NULL); \ + _PyFrame_StackAssertInvalid(frame); \ NEXTOPARG(); \ PRE_DISPATCH_GOTO(); \ DISPATCH_GOTO_NON_TRACING(); \ @@ -223,6 +223,7 @@ do { \ do { \ assert(!IS_PEP523_HOOKED(tstate)); \ _PyFrame_SetStackPointer(frame, stack_pointer); \ + _PyFrame_StackPointerValidate(frame); \ assert((NEW_FRAME)->previous == frame); \ frame = tstate->current_frame = (NEW_FRAME); \ CALL_STAT_INC(inlined_py_calls); \ @@ -290,7 +291,7 @@ GETITEM(PyObject *v, Py_ssize_t i) { STAT_INC(opcode, miss); \ STAT_INC((INSTNAME), miss); \ /* The counter is always the first cache entry: */ \ - if (ADAPTIVE_COUNTER_TRIGGERS(next_instr->cache)) { \ + if (ADAPTIVE_COUNTER_TRIGGERS(next_instr->cache)) { \ STAT_INC((INSTNAME), deopt); \ } \ } while (0) diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h index e3d13f9f9c61..63b77c9e1527 100644 --- a/Python/executor_cases.c.h +++ b/Python/executor_cases.c.h @@ -56,8 +56,9 @@ CHECK_CURRENT_CACHED_VALUES(0); ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); int err = check_periodics(tstate); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (err != 0) { SET_CURRENT_CACHED_VALUES(0); JUMP_TO_ERROR(); @@ -78,8 +79,9 @@ oparg = CURRENT_OPARG(); if ((oparg & RESUME_OPARG_LOCATION_MASK) < RESUME_AFTER_YIELD_FROM) { _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); int err = check_periodics(tstate); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (err != 0) { SET_CURRENT_CACHED_VALUES(0); JUMP_TO_ERROR(); @@ -263,11 +265,12 @@ _PyStackRef value_s = GETLOCAL(oparg); if (PyStackRef_IsNull(value_s)) { _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); _PyEval_FormatExcCheckArg(tstate, PyExc_UnboundLocalError, UNBOUNDLOCAL_ERROR_MSG, PyTuple_GetItem(_PyFrame_GetCode(frame)->co_localsplusnames, oparg) ); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); SET_CURRENT_CACHED_VALUES(0); JUMP_TO_ERROR(); } @@ -290,11 +293,12 @@ stack_pointer += 1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); _PyEval_FormatExcCheckArg(tstate, PyExc_UnboundLocalError, UNBOUNDLOCAL_ERROR_MSG, PyTuple_GetItem(_PyFrame_GetCode(frame)->co_localsplusnames, oparg) ); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); SET_CURRENT_CACHED_VALUES(0); JUMP_TO_ERROR(); } @@ -320,11 +324,12 @@ stack_pointer += 2; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); _PyEval_FormatExcCheckArg(tstate, PyExc_UnboundLocalError, UNBOUNDLOCAL_ERROR_MSG, PyTuple_GetItem(_PyFrame_GetCode(frame)->co_localsplusnames, oparg) ); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); SET_CURRENT_CACHED_VALUES(0); JUMP_TO_ERROR(); } @@ -2251,8 +2256,9 @@ _PyStackRef _stack_item_0 = _tos_cache0; value = _stack_item_0; _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_XCLOSE(value); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); _tos_cache0 = PyStackRef_ZERO_BITS; _tos_cache1 = PyStackRef_ZERO_BITS; _tos_cache2 = PyStackRef_ZERO_BITS; @@ -2504,8 +2510,9 @@ oparg = CURRENT_OPARG(); args = &stack_pointer[-oparg]; _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); _PyStackRef_CloseStack(args, oparg); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); _tos_cache0 = PyStackRef_ZERO_BITS; _tos_cache1 = PyStackRef_ZERO_BITS; _tos_cache2 = PyStackRef_ZERO_BITS; @@ -2562,8 +2569,9 @@ _PyStackRef _stack_item_0 = _tos_cache0; value = _stack_item_0; _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(value); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); _tos_cache0 = PyStackRef_ZERO_BITS; _tos_cache1 = PyStackRef_ZERO_BITS; _tos_cache2 = PyStackRef_ZERO_BITS; @@ -2583,8 +2591,9 @@ iter = _stack_item_0; (void)index_or_null; _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(iter); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); _tos_cache0 = PyStackRef_ZERO_BITS; _tos_cache1 = PyStackRef_ZERO_BITS; _tos_cache2 = PyStackRef_ZERO_BITS; @@ -2612,8 +2621,9 @@ stack_pointer += 1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(receiver); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); _tos_cache0 = val; _tos_cache1 = PyStackRef_ZERO_BITS; _tos_cache2 = PyStackRef_ZERO_BITS; @@ -2636,8 +2646,9 @@ stack_pointer += 1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyObject *res_o = PyNumber_Negative(PyStackRef_AsPyObjectBorrow(value)); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (res_o == NULL) { SET_CURRENT_CACHED_VALUES(0); JUMP_TO_ERROR(); @@ -2808,13 +2819,15 @@ stack_pointer += 1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); int err = PyObject_IsTrue(PyStackRef_AsPyObjectBorrow(value)); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(value); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (err < 0) { SET_CURRENT_CACHED_VALUES(0); JUMP_TO_ERROR(); @@ -3803,8 +3816,9 @@ stack_pointer += 1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyObject *res_o = PyNumber_Invert(PyStackRef_AsPyObjectBorrow(value)); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (res_o == NULL) { SET_CURRENT_CACHED_VALUES(0); JUMP_TO_ERROR(); @@ -6025,9 +6039,10 @@ stack_pointer += 2; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyErr_SetString(PyExc_ZeroDivisionError, "float division by zero"); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); SET_CURRENT_CACHED_VALUES(0); JUMP_TO_ERROR(); } @@ -6378,9 +6393,13 @@ PyObject *right_o = PyStackRef_AsPyObjectSteal(right); PyStackRef_CLOSE_SPECIALIZED(left, _PyUnicode_ExactDealloc); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyUnicode_Append(&temp, right_o); + _PyFrame_StackPointerInvalidate(frame); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); _Py_DECREF_SPECIALIZED(right_o, _PyUnicode_ExactDealloc); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); *target_local = PyStackRef_NULL; if (temp == NULL) { SET_CURRENT_CACHED_VALUES(0); @@ -6625,10 +6644,11 @@ stack_pointer += 2; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); int match = (d->guard != NULL) ? d->guard(left_o, right_o) : (Py_TYPE(left_o) == d->lhs_type && Py_TYPE(right_o) == d->rhs_type); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (!match) { UOP_STAT_INC(uopcode, miss); _tos_cache1 = right; @@ -6671,8 +6691,9 @@ stack_pointer += 2; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyObject *res_o = d->action(left_o, right_o); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (res_o == NULL) { SET_CURRENT_CACHED_VALUES(0); JUMP_TO_ERROR(); @@ -6717,8 +6738,9 @@ stack_pointer += 3; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); res_o = _PyList_BinarySlice(container_o, start_o, stop_o); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } else if (PyTuple_CheckExact(container_o)) { stack_pointer[0] = container; @@ -6727,8 +6749,9 @@ stack_pointer += 3; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); res_o = _PyTuple_BinarySlice(container_o, start_o, stop_o); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } else if (PyUnicode_CheckExact(container_o)) { stack_pointer[0] = container; @@ -6737,8 +6760,9 @@ stack_pointer += 3; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); res_o = _PyUnicode_BinarySlice(container_o, start_o, stop_o); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } else { PyObject *slice = PySlice_New(start_o, stop_o, NULL); @@ -6752,29 +6776,40 @@ stack_pointer += 3; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); res_o = PyObject_GetItem(container_o, slice); + _PyFrame_StackPointerInvalidate(frame); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); Py_DECREF(slice); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); stack_pointer += -3; } stack_pointer += 3; } _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); _PyStackRef tmp = stop; stop = PyStackRef_NULL; stack_pointer[-3] = container; stack_pointer[-2] = start; stack_pointer[-1] = stop; PyStackRef_CLOSE(tmp); + _PyFrame_StackPointerInvalidate(frame); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); tmp = start; start = PyStackRef_NULL; stack_pointer[-2] = start; PyStackRef_CLOSE(tmp); + _PyFrame_StackPointerInvalidate(frame); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); tmp = container; container = PyStackRef_NULL; stack_pointer[-3] = container; PyStackRef_CLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); stack_pointer += -3; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); if (res_o == NULL) { @@ -6810,10 +6845,11 @@ stack_pointer += 3; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyObject *slice = _PyBuildSlice_ConsumeRefs(PyStackRef_AsPyObjectSteal(start), PyStackRef_AsPyObjectSteal(stop), Py_None); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); int err; if (slice == NULL) { err = 1; @@ -6822,21 +6858,29 @@ stack_pointer += -2; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); err = PyObject_SetItem(PyStackRef_AsPyObjectBorrow(container), slice, PyStackRef_AsPyObjectBorrow(v)); + _PyFrame_StackPointerInvalidate(frame); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); Py_DECREF(slice); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); stack_pointer += 2; } _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); _PyStackRef tmp = container; container = PyStackRef_NULL; stack_pointer[-3] = container; PyStackRef_CLOSE(tmp); + _PyFrame_StackPointerInvalidate(frame); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); tmp = v; v = PyStackRef_NULL; stack_pointer[-4] = v; PyStackRef_CLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); stack_pointer += -4; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); if (err) { @@ -6877,8 +6921,9 @@ stack_pointer += 2; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyObject *res_o = _PyList_GetItemRef((PyListObject*)list, index); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (res_o == NULL) { UOP_STAT_INC(uopcode, miss); _tos_cache1 = sub_st; @@ -6936,8 +6981,9 @@ stack_pointer += 2; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyObject *res_o = _PyList_SliceSubscript(list, sub); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); STAT_INC(BINARY_OP, hit); if (res_o == NULL) { SET_CURRENT_CACHED_VALUES(0); @@ -7973,8 +8019,9 @@ stack_pointer += 2; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyObject *res_o = _PyDict_SubscriptKnownHash(dict, sub, (Py_hash_t)hash); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (res_o == NULL) { SET_CURRENT_CACHED_VALUES(0); JUMP_TO_ERROR(); @@ -8013,8 +8060,9 @@ stack_pointer += 2; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyObject *res_o = _PyDict_Subscript(dict, sub); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (res_o == NULL) { SET_CURRENT_CACHED_VALUES(0); JUMP_TO_ERROR(); @@ -8217,9 +8265,10 @@ stack_pointer += 1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); int err = _PySet_AddTakeRef((PySetObject *)PyStackRef_AsPyObjectBorrow(set), PyStackRef_AsPyObjectSteal(v)); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (err) { stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); @@ -8254,20 +8303,30 @@ stack_pointer += 3; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); int err = PyObject_SetItem(PyStackRef_AsPyObjectBorrow(container), PyStackRef_AsPyObjectBorrow(sub), PyStackRef_AsPyObjectBorrow(v)); + _PyFrame_StackPointerInvalidate(frame); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); _PyStackRef tmp = sub; sub = PyStackRef_NULL; stack_pointer[-1] = sub; PyStackRef_CLOSE(tmp); + _PyFrame_StackPointerInvalidate(frame); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); tmp = container; container = PyStackRef_NULL; stack_pointer[-2] = container; PyStackRef_CLOSE(tmp); + _PyFrame_StackPointerInvalidate(frame); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); tmp = v; v = PyStackRef_NULL; stack_pointer[-3] = v; PyStackRef_CLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); stack_pointer += -3; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); if (err) { @@ -8337,8 +8396,9 @@ stack_pointer += 2; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); Py_DECREF(old_value); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); _tos_cache1 = ss; _tos_cache0 = ls; _tos_cache2 = PyStackRef_ZERO_BITS; @@ -8371,16 +8431,18 @@ stack_pointer += 3; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); int err = _PyDict_SetItem_Take2((PyDictObject *)dict, PyStackRef_AsPyObjectSteal(sub), PyStackRef_AsPyObjectSteal(value)); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (err) { stack_pointer += -3; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(dict_st); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); SET_CURRENT_CACHED_VALUES(0); JUMP_TO_ERROR(); } @@ -8418,17 +8480,19 @@ stack_pointer += 3; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); int err = _PyDict_SetItem_Take2_KnownHash((PyDictObject *)dict, PyStackRef_AsPyObjectSteal(sub), PyStackRef_AsPyObjectSteal(value), (Py_hash_t)hash); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (err) { stack_pointer += -3; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(dict_st); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); SET_CURRENT_CACHED_VALUES(0); JUMP_TO_ERROR(); } @@ -8457,17 +8521,24 @@ stack_pointer += 2; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); int err = PyObject_DelItem(PyStackRef_AsPyObjectBorrow(container), PyStackRef_AsPyObjectBorrow(sub)); + _PyFrame_StackPointerInvalidate(frame); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); _PyStackRef tmp = sub; sub = PyStackRef_NULL; stack_pointer[-1] = sub; PyStackRef_CLOSE(tmp); + _PyFrame_StackPointerInvalidate(frame); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); tmp = container; container = PyStackRef_NULL; stack_pointer[-2] = container; PyStackRef_CLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); stack_pointer += -2; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); if (err) { @@ -8496,8 +8567,9 @@ stack_pointer += 1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyObject *res_o = _PyIntrinsics_UnaryFunctions[oparg].func(tstate, PyStackRef_AsPyObjectBorrow(value)); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (res_o == NULL) { SET_CURRENT_CACHED_VALUES(0); JUMP_TO_ERROR(); @@ -8535,8 +8607,9 @@ stack_pointer += 2; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyObject *res_o = _PyIntrinsics_BinaryFunctions[oparg].func(tstate, value2, value1); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (res_o == NULL) { SET_CURRENT_CACHED_VALUES(0); JUMP_TO_ERROR(); @@ -8623,6 +8696,7 @@ assert(frame->owner != FRAME_OWNED_BY_INTERPRETER); _PyStackRef temp = retval; _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); assert(STACK_LEVEL() == 0); DTRACE_FUNCTION_RETURN(); _Py_LeaveRecursiveCallPy(tstate); @@ -8630,6 +8704,7 @@ frame = tstate->current_frame = dying->previous; _PyEval_FrameClearAndPop(tstate, dying); stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); LOAD_IP(frame->return_offset); res = temp; LLTRACE_RESUME_FRAME(); @@ -8660,16 +8735,18 @@ stack_pointer += 1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); _PyErr_Format(tstate, PyExc_TypeError, "'async for' requires an object with " "__aiter__ method, got %.100s", type->tp_name); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(obj); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); SET_CURRENT_CACHED_VALUES(0); JUMP_TO_ERROR(); } @@ -8677,26 +8754,32 @@ stack_pointer += 1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); iter_o = (*getter)(obj_o); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(obj); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (iter_o == NULL) { SET_CURRENT_CACHED_VALUES(0); JUMP_TO_ERROR(); } if (Py_TYPE(iter_o)->tp_as_async == NULL || Py_TYPE(iter_o)->tp_as_async->am_anext == NULL) { - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); _PyErr_Format(tstate, PyExc_TypeError, "'async for' received an object from __aiter__ " "that does not implement __anext__: %.100s", Py_TYPE(iter_o)->tp_name); + _PyFrame_StackPointerInvalidate(frame); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); Py_DECREF(iter_o); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); SET_CURRENT_CACHED_VALUES(0); JUMP_TO_ERROR(); } @@ -8720,8 +8803,9 @@ stack_pointer += 1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyObject *awaitable_o = _PyEval_GetANext(PyStackRef_AsPyObjectBorrow(aiter)); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (awaitable_o == NULL) { SET_CURRENT_CACHED_VALUES(0); JUMP_TO_ERROR(); @@ -8749,13 +8833,15 @@ stack_pointer += 1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyObject *iter_o = _PyEval_GetAwaitable(PyStackRef_AsPyObjectBorrow(iterable), oparg); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(iterable); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (iter_o == NULL) { SET_CURRENT_CACHED_VALUES(0); JUMP_TO_ERROR(); @@ -9286,8 +9372,9 @@ stack_pointer += 3; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PySendResult what = _PyAsyncGenASend_Send(iter_o, val, &retval_o); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (what == PYGEN_ERROR) { SET_CURRENT_CACHED_VALUES(0); JUMP_TO_ERROR(); @@ -9295,8 +9382,9 @@ stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(v); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); asend = iter; null_out = null_in; retval = PyStackRef_FromPyObjectSteal(retval_o); @@ -9335,6 +9423,7 @@ assert(oparg == 0 || oparg == 1); _PyStackRef temp = retval; _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); DTRACE_FUNCTION_RETURN(); tstate->exc_info = gen->gi_exc_state.previous_item; gen->gi_exc_state.previous_item = NULL; @@ -9355,6 +9444,7 @@ } #endif stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); LOAD_IP(1 + INLINE_CACHE_ENTRIES_SEND); value = temp; LLTRACE_RESUME_FRAME(); @@ -9377,10 +9467,11 @@ stack_pointer += 1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); Py_XSETREF(exc_info->exc_value, PyStackRef_IsNone(exc_value) ? NULL : PyStackRef_AsPyObjectSteal(exc_value)); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); _tos_cache0 = PyStackRef_ZERO_BITS; _tos_cache1 = PyStackRef_ZERO_BITS; _tos_cache2 = PyStackRef_ZERO_BITS; @@ -9442,17 +9533,19 @@ _PyStackRef bc; int err; _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyObject *bc_o = _PyMapping_GetOptionalItem2(BUILTINS(), &_Py_ID(__build_class__), &err); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (err < 0) { SET_CURRENT_CACHED_VALUES(0); JUMP_TO_ERROR(); } if (bc_o == NULL) { - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); _PyErr_SetString(tstate, PyExc_NameError, "__build_class__ not found"); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); SET_CURRENT_CACHED_VALUES(0); JUMP_TO_ERROR(); } @@ -9480,14 +9573,16 @@ stack_pointer += 1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); _PyErr_Format(tstate, PyExc_SystemError, "no locals found when storing %R", name); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(v); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); SET_CURRENT_CACHED_VALUES(0); JUMP_TO_ERROR(); } @@ -9496,22 +9591,25 @@ stack_pointer += 1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); err = PyDict_SetItem(ns, name, PyStackRef_AsPyObjectBorrow(v)); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } else { stack_pointer[0] = v; stack_pointer += 1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); err = PyObject_SetItem(ns, name, PyStackRef_AsPyObjectBorrow(v)); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(v); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (err) { SET_CURRENT_CACHED_VALUES(0); JUMP_TO_ERROR(); @@ -9533,21 +9631,24 @@ int err; if (ns == NULL) { _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); _PyErr_Format(tstate, PyExc_SystemError, "no locals when deleting %R", name); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); SET_CURRENT_CACHED_VALUES(0); JUMP_TO_ERROR(); } _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); err = PyObject_DelItem(ns, name); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (err != 0) { - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); _PyEval_FormatExcCheckArg(tstate, PyExc_NameError, NAME_ERROR_MSG, name); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); SET_CURRENT_CACHED_VALUES(0); JUMP_TO_ERROR(); } @@ -9570,9 +9671,13 @@ top = &stack_pointer[oparg]; PyObject *seq_o = PyStackRef_AsPyObjectSteal(seq); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); int res = _PyEval_UnpackIterableStackRef(tstate, seq_o, oparg, -1, top); + _PyFrame_StackPointerInvalidate(frame); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); Py_DECREF(seq_o); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (res == 0) { SET_CURRENT_CACHED_VALUES(0); JUMP_TO_ERROR(); @@ -9613,8 +9718,9 @@ stack_pointer += 2; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(seq); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); _tos_cache1 = val0; _tos_cache0 = val1; _tos_cache2 = PyStackRef_ZERO_BITS; @@ -9765,8 +9871,9 @@ stack_pointer += oparg; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(seq); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); _tos_cache0 = PyStackRef_ZERO_BITS; _tos_cache1 = PyStackRef_ZERO_BITS; _tos_cache2 = PyStackRef_ZERO_BITS; @@ -9837,8 +9944,9 @@ stack_pointer += oparg; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(seq); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); _tos_cache0 = PyStackRef_ZERO_BITS; _tos_cache1 = PyStackRef_ZERO_BITS; _tos_cache2 = PyStackRef_ZERO_BITS; @@ -9858,9 +9966,13 @@ top = &stack_pointer[1 + (oparg & 0xFF) + (oparg >> 8)]; PyObject *seq_o = PyStackRef_AsPyObjectSteal(seq); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); int res = _PyEval_UnpackIterableStackRef(tstate, seq_o, oparg & 0xFF, oparg >> 8, top); + _PyFrame_StackPointerInvalidate(frame); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); Py_DECREF(seq_o); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (res == 0) { SET_CURRENT_CACHED_VALUES(0); JUMP_TO_ERROR(); @@ -9891,17 +10003,24 @@ stack_pointer += 2; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); int err = PyObject_SetAttr(PyStackRef_AsPyObjectBorrow(owner), name, PyStackRef_AsPyObjectBorrow(v)); + _PyFrame_StackPointerInvalidate(frame); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); _PyStackRef tmp = owner; owner = PyStackRef_NULL; stack_pointer[-1] = owner; PyStackRef_CLOSE(tmp); + _PyFrame_StackPointerInvalidate(frame); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); tmp = v; v = PyStackRef_NULL; stack_pointer[-2] = v; PyStackRef_CLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); stack_pointer += -2; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); if (err) { @@ -9928,13 +10047,15 @@ stack_pointer += 1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); int err = PyObject_DelAttr(PyStackRef_AsPyObjectBorrow(owner), name); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(owner); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (err) { SET_CURRENT_CACHED_VALUES(0); JUMP_TO_ERROR(); @@ -9959,13 +10080,15 @@ stack_pointer += 1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); int err = PyDict_SetItem(GLOBALS(), name, PyStackRef_AsPyObjectBorrow(v)); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(v); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (err) { SET_CURRENT_CACHED_VALUES(0); JUMP_TO_ERROR(); @@ -9984,17 +10107,19 @@ oparg = CURRENT_OPARG(); PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); int err = PyDict_Pop(GLOBALS(), name, NULL); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (err < 0) { SET_CURRENT_CACHED_VALUES(0); JUMP_TO_ERROR(); } if (err == 0) { - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); _PyEval_FormatExcCheckArg(tstate, PyExc_NameError, NAME_ERROR_MSG, name); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); SET_CURRENT_CACHED_VALUES(0); JUMP_TO_ERROR(); } @@ -10013,9 +10138,10 @@ PyObject *l = LOCALS(); if (l == NULL) { _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); _PyErr_SetString(tstate, PyExc_SystemError, "no locals found"); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); SET_CURRENT_CACHED_VALUES(0); JUMP_TO_ERROR(); } @@ -10037,9 +10163,10 @@ stack_pointer += 1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); _PyErr_SetString(tstate, PyExc_SystemError, "no locals found"); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); SET_CURRENT_CACHED_VALUES(0); JUMP_TO_ERROR(); } @@ -10064,9 +10191,10 @@ stack_pointer += 2; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); _PyErr_SetString(tstate, PyExc_SystemError, "no locals found"); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); SET_CURRENT_CACHED_VALUES(0); JUMP_TO_ERROR(); } @@ -10088,37 +10216,46 @@ oparg = CURRENT_OPARG(); PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyObject *v_o = _PyEval_LoadName(tstate, frame, name); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (v_o == NULL) { SET_CURRENT_CACHED_VALUES(0); JUMP_TO_ERROR(); } if (PyLazyImport_CheckExact(v_o)) { - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); PyObject *l_v = _PyImport_LoadLazyImportTstate(tstate, v_o); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (l_v == NULL) { - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); Py_DECREF(v_o); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); SET_CURRENT_CACHED_VALUES(0); JUMP_TO_ERROR(); } - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); int err = PyDict_SetItem(GLOBALS(), name, l_v); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (err < 0) { - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); Py_DECREF(v_o); + _PyFrame_StackPointerInvalidate(frame); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); Py_DECREF(l_v); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); SET_CURRENT_CACHED_VALUES(0); JUMP_TO_ERROR(); } - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); Py_SETREF(v_o, l_v); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } v = PyStackRef_FromPyObjectSteal(v_o); _tos_cache0 = v; @@ -10137,8 +10274,9 @@ res = &stack_pointer[0]; PyObject *name = GETITEM(FRAME_CO_NAMES, oparg>>1); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); _PyEval_LoadGlobalStackRef(GLOBALS(), BUILTINS(), name, res); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (PyStackRef_IsNull(*res)) { SET_CURRENT_CACHED_VALUES(0); JUMP_TO_ERROR(); @@ -10379,19 +10517,21 @@ _PyStackRef v = GETLOCAL(oparg); if (PyStackRef_IsNull(v)) { _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); _PyEval_FormatExcCheckArg(tstate, PyExc_UnboundLocalError, UNBOUNDLOCAL_ERROR_MSG, PyTuple_GetItem(_PyFrame_GetCode(frame)->co_localsplusnames, oparg) ); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); SET_CURRENT_CACHED_VALUES(0); JUMP_TO_ERROR(); } _PyStackRef tmp = GETLOCAL(oparg); GETLOCAL(oparg) = PyStackRef_NULL; _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_XCLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); _tos_cache0 = PyStackRef_ZERO_BITS; _tos_cache1 = PyStackRef_ZERO_BITS; _tos_cache2 = PyStackRef_ZERO_BITS; @@ -10413,8 +10553,9 @@ _PyStackRef tmp = GETLOCAL(oparg); GETLOCAL(oparg) = PyStackRef_FromPyObjectSteal(cell); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_XCLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); _tos_cache0 = PyStackRef_ZERO_BITS; _tos_cache1 = PyStackRef_ZERO_BITS; _tos_cache2 = PyStackRef_ZERO_BITS; @@ -10431,14 +10572,16 @@ PyObject *oldobj = PyCell_SwapTakeRef((PyCellObject *)cell, NULL); if (oldobj == NULL) { _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); _PyEval_FormatExcUnbound(tstate, _PyFrame_GetCode(frame), oparg); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); SET_CURRENT_CACHED_VALUES(0); JUMP_TO_ERROR(); } _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); Py_DECREF(oldobj); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); _tos_cache0 = PyStackRef_ZERO_BITS; _tos_cache1 = PyStackRef_ZERO_BITS; _tos_cache2 = PyStackRef_ZERO_BITS; @@ -10465,8 +10608,9 @@ stack_pointer += 1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyObject* value_o = _PyMapping_GetOptionalItem2(class_dict, name, &err); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (err < 0) { SET_CURRENT_CACHED_VALUES(0); JUMP_TO_ERROR(); @@ -10475,9 +10619,10 @@ PyCellObject *cell = (PyCellObject *)PyStackRef_AsPyObjectBorrow(GETLOCAL(oparg)); value_o = PyCell_GetRef(cell); if (value_o == NULL) { - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); _PyEval_FormatExcUnbound(tstate, _PyFrame_GetCode(frame), oparg); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); SET_CURRENT_CACHED_VALUES(0); JUMP_TO_ERROR(); } @@ -10485,8 +10630,9 @@ stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(class_dict_st); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); value = PyStackRef_FromPyObjectSteal(value_o); _tos_cache0 = value; _tos_cache1 = PyStackRef_ZERO_BITS; @@ -10503,15 +10649,17 @@ oparg = CURRENT_OPARG(); PyCellObject *cell = (PyCellObject *)PyStackRef_AsPyObjectBorrow(GETLOCAL(oparg)); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); value = _PyCell_GetStackRef(cell); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (PyStackRef_IsNull(value)) { stack_pointer[0] = value; stack_pointer += 1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); _PyEval_FormatExcUnbound(tstate, _PyFrame_GetCode(frame), oparg); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); SET_CURRENT_CACHED_VALUES(0); JUMP_TO_ERROR(); } @@ -10535,8 +10683,9 @@ stack_pointer += 1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyCell_SetTakeRef(cell, PyStackRef_AsPyObjectSteal(v)); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); _tos_cache0 = PyStackRef_ZERO_BITS; _tos_cache1 = PyStackRef_ZERO_BITS; _tos_cache2 = PyStackRef_ZERO_BITS; @@ -10643,8 +10792,9 @@ oparg = CURRENT_OPARG(); pieces = &stack_pointer[-oparg]; _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyObject *str_o = _Py_BuildString_StackRefSteal(pieces, oparg); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (str_o == NULL) { stack_pointer += -oparg; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); @@ -10684,14 +10834,16 @@ format_o = &_Py_STR(empty); } _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyObject *interpolation_o = _PyInterpolation_Build(value_o, str_o, conversion, format_o); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (oparg & 1) { stack_pointer += -(oparg & 1); ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(format[0]); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } else { stack_pointer += -(oparg & 1); @@ -10699,13 +10851,15 @@ stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(str); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(value); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (interpolation_o == NULL) { SET_CURRENT_CACHED_VALUES(0); JUMP_TO_ERROR(); @@ -10736,18 +10890,21 @@ stack_pointer += 2; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyObject *template_o = _PyTemplate_Build(strings_o, interpolations_o); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(interpolations); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(strings); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (template_o == NULL) { SET_CURRENT_CACHED_VALUES(0); JUMP_TO_ERROR(); @@ -10790,8 +10947,9 @@ oparg = CURRENT_OPARG(); values = &stack_pointer[-oparg]; _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyObject *list_o = _PyList_FromStackRefStealOnSuccess(values, oparg); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (list_o == NULL) { SET_CURRENT_CACHED_VALUES(0); JUMP_TO_ERROR(); @@ -10823,19 +10981,24 @@ stack_pointer += 1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyObject *none_val = _PyList_Extend((PyListObject *)list, iterable); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (none_val == NULL) { int matches = _PyErr_ExceptionMatches(tstate, PyExc_TypeError); if (matches && (Py_TYPE(iterable)->tp_iter == NULL && !PySequence_Check(iterable))) { - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); _PyErr_Clear(tstate); + _PyFrame_StackPointerInvalidate(frame); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); _PyErr_Format(tstate, PyExc_TypeError, "Value after * must be an iterable, not %.200s", Py_TYPE(iterable)->tp_name); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } SET_CURRENT_CACHED_VALUES(0); JUMP_TO_ERROR(); @@ -10866,9 +11029,10 @@ stack_pointer += 1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); int err = _PySet_Update(PyStackRef_AsPyObjectBorrow(set), PyStackRef_AsPyObjectBorrow(iterable)); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (err < 0) { SET_CURRENT_CACHED_VALUES(0); JUMP_TO_ERROR(); @@ -10892,17 +11056,19 @@ oparg = CURRENT_OPARG(); values = &stack_pointer[-oparg]; _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyObject *set_o = PySet_New(NULL); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (set_o == NULL) { - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); _PyStackRef tmp; for (int _i = oparg; --_i >= 0;) { tmp = values[_i]; values[_i] = PyStackRef_NULL; PyStackRef_CLOSE(tmp); } - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); stack_pointer += -oparg; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); SET_CURRENT_CACHED_VALUES(0); @@ -10913,22 +11079,25 @@ _PyStackRef value = values[i]; values[i] = PyStackRef_NULL; if (err == 0) { - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); err = _PySet_AddTakeRef((PySetObject *)set_o, PyStackRef_AsPyObjectSteal(value)); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } else { - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(value); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } } if (err) { stack_pointer += -oparg; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); Py_DECREF(set_o); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); SET_CURRENT_CACHED_VALUES(0); JUMP_TO_ERROR(); } @@ -10951,8 +11120,9 @@ oparg = CURRENT_OPARG(); values = &stack_pointer[-oparg*2]; _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyObject *map_o = _Py_BuildMap_StackRefSteal(values, oparg); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (map_o == NULL) { stack_pointer += -oparg*2; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); @@ -10975,42 +11145,50 @@ ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); if (LOCALS() == NULL) { _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); _PyErr_Format(tstate, PyExc_SystemError, "no locals found when setting up annotations"); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); SET_CURRENT_CACHED_VALUES(0); JUMP_TO_ERROR(); } int err; _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyObject* ann_dict = _PyMapping_GetOptionalItem2(LOCALS(), &_Py_ID(__annotations__), &err); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (err < 0) { SET_CURRENT_CACHED_VALUES(0); JUMP_TO_ERROR(); } if (ann_dict == NULL) { - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); ann_dict = PyDict_New(); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (ann_dict == NULL) { SET_CURRENT_CACHED_VALUES(0); JUMP_TO_ERROR(); } - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); err = PyObject_SetItem(LOCALS(), &_Py_ID(__annotations__), ann_dict); + _PyFrame_StackPointerInvalidate(frame); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); Py_DECREF(ann_dict); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (err) { SET_CURRENT_CACHED_VALUES(0); JUMP_TO_ERROR(); } } else { - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); Py_DECREF(ann_dict); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } _tos_cache0 = PyStackRef_ZERO_BITS; _tos_cache1 = PyStackRef_ZERO_BITS; @@ -11036,27 +11214,37 @@ stack_pointer += 1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); int err = PyDict_Update(dict_o, update_o); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (err < 0) { int matches = _PyErr_ExceptionMatches(tstate, PyExc_AttributeError); if (matches) { - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); PyObject *exc = _PyErr_GetRaisedException(tstate); + _PyFrame_StackPointerInvalidate(frame); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); int has_keys = PyObject_HasAttrWithError(update_o, &_Py_ID(keys)); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (has_keys == 0) { - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); _PyErr_Format(tstate, PyExc_TypeError, "'%T' object is not a mapping", update_o); + _PyFrame_StackPointerInvalidate(frame); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); Py_DECREF(exc); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } else { - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); _PyErr_ChainExceptions1(exc); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } } SET_CURRENT_CACHED_VALUES(0); @@ -11093,13 +11281,18 @@ stack_pointer += 1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); int err = _PyDict_MergeUniq(dict_o, update_o, &dupkey); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (err < 0) { - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); _PyEval_FormatKwargsError(tstate, callable_o, update_o, dupkey); + _PyFrame_StackPointerInvalidate(frame); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); Py_XDECREF(dupkey); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); SET_CURRENT_CACHED_VALUES(0); JUMP_TO_ERROR(); } @@ -11133,12 +11326,13 @@ stack_pointer += 2; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); int err = _PyDict_SetItem_Take2( (PyDictObject *)dict, PyStackRef_AsPyObjectSteal(key), PyStackRef_AsPyObjectSteal(value) ); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (err != 0) { stack_pointer += -2; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); @@ -11197,20 +11391,30 @@ stack_pointer += 3; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyObject *attr = _PySuper_Lookup((PyTypeObject *)class, self, name, NULL); + _PyFrame_StackPointerInvalidate(frame); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); _PyStackRef tmp = self_st; self_st = PyStackRef_NULL; stack_pointer[-1] = self_st; PyStackRef_CLOSE(tmp); + _PyFrame_StackPointerInvalidate(frame); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); tmp = class_st; class_st = PyStackRef_NULL; stack_pointer[-2] = class_st; PyStackRef_CLOSE(tmp); + _PyFrame_StackPointerInvalidate(frame); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); tmp = global_super_st; global_super_st = PyStackRef_NULL; stack_pointer[-3] = global_super_st; PyStackRef_CLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); stack_pointer += -3; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); if (attr == NULL) { @@ -11519,9 +11723,10 @@ stack_pointer += 3; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); attr_o = _PySuper_Lookup(cls, self, name, Py_TYPE(self)->tp_getattro == PyObject_GenericGetAttr ? method_found_ptr : NULL); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } if (attr_o == NULL) { SET_CURRENT_CACHED_VALUES(0); @@ -11533,23 +11738,28 @@ stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(self_st); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); self_or_null = PyStackRef_NULL; stack_pointer += 1; } stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); _PyStackRef tmp = global_super_st; global_super_st = self_or_null; stack_pointer[-2] = global_super_st; PyStackRef_CLOSE(tmp); + _PyFrame_StackPointerInvalidate(frame); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); tmp = class_st; class_st = PyStackRef_NULL; stack_pointer[-1] = class_st; PyStackRef_CLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); stack_pointer += -2; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); attr = PyStackRef_FromPyObjectSteal(attr_o); @@ -11577,8 +11787,9 @@ stack_pointer += 1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); attr = _Py_LoadAttr_StackRefSteal(tstate, owner, name, self_or_null); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (PyStackRef_IsNull(attr)) { stack_pointer[-1] = attr; stack_pointer += (oparg&1); @@ -11592,14 +11803,16 @@ stack_pointer += 1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); attr = _PyObject_GetAttrStackRef(PyStackRef_AsPyObjectBorrow(owner), name); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); stack_pointer[-1] = attr; stack_pointer += (oparg&1); ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(owner); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (PyStackRef_IsNull(attr)) { SET_CURRENT_CACHED_VALUES(0); JUMP_TO_ERROR(); @@ -12522,11 +12735,12 @@ stack_pointer += 1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); _PyStackRef tmp = owner; owner = attr; stack_pointer[-1] = owner; PyStackRef_CLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); _tos_cache0 = attr; _tos_cache1 = PyStackRef_ZERO_BITS; _tos_cache2 = PyStackRef_ZERO_BITS; @@ -12872,8 +13086,9 @@ stack_pointer += 1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); Py_XDECREF(old_value); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); _tos_cache0 = o; _tos_cache1 = PyStackRef_ZERO_BITS; _tos_cache2 = PyStackRef_ZERO_BITS; @@ -13034,8 +13249,9 @@ stack_pointer += 2; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); _PyDict_NotifyEvent(PyDict_EVENT_MODIFIED, dict, name, PyStackRef_AsPyObjectBorrow(value)); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); FT_ATOMIC_STORE_PTR_RELEASE(ep->me_value, PyStackRef_AsPyObjectSteal(value)); UNLOCK_OBJECT(dict); STAT_INC(STORE_ATTR, hit); @@ -13044,8 +13260,9 @@ stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); Py_XDECREF(old_value); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); _tos_cache0 = o; _tos_cache1 = PyStackRef_ZERO_BITS; _tos_cache2 = PyStackRef_ZERO_BITS; @@ -13085,8 +13302,9 @@ stack_pointer += 1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); Py_XDECREF(old_value); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); _tos_cache0 = o; _tos_cache1 = PyStackRef_ZERO_BITS; _tos_cache2 = PyStackRef_ZERO_BITS; @@ -13116,16 +13334,23 @@ stack_pointer += 2; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyObject *res_o = PyObject_RichCompare(left_o, right_o, oparg >> 5); + _PyFrame_StackPointerInvalidate(frame); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); _PyStackRef tmp = right; right = PyStackRef_NULL; stack_pointer[-1] = right; PyStackRef_CLOSE(tmp); + _PyFrame_StackPointerInvalidate(frame); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); tmp = left; left = PyStackRef_NULL; stack_pointer[-2] = left; PyStackRef_CLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); stack_pointer += -2; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); if (res_o == NULL) { @@ -13134,9 +13359,13 @@ } if (oparg & 16) { _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); int res_bool = PyObject_IsTrue(res_o); + _PyFrame_StackPointerInvalidate(frame); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); Py_DECREF(res_o); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (res_bool < 0) { SET_CURRENT_CACHED_VALUES(0); JUMP_TO_ERROR(); @@ -13407,8 +13636,9 @@ stack_pointer += 2; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); int res = PySequence_Contains(right_o, left_o); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (res < 0) { SET_CURRENT_CACHED_VALUES(0); JUMP_TO_ERROR(); @@ -13703,8 +13933,9 @@ stack_pointer += 2; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); int res = _PySet_Contains((PySetObject *)right_o, left_o); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (res < 0) { SET_CURRENT_CACHED_VALUES(0); JUMP_TO_ERROR(); @@ -13744,8 +13975,9 @@ stack_pointer += 2; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); int res = PyDict_Contains(right_o, left_o); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (res < 0) { SET_CURRENT_CACHED_VALUES(0); JUMP_TO_ERROR(); @@ -13781,19 +14013,24 @@ stack_pointer += 2; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); int err = _PyEval_CheckExceptStarTypeValid(tstate, match_type); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (err < 0) { - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); _PyStackRef tmp = match_type_st; match_type_st = PyStackRef_NULL; stack_pointer[-1] = match_type_st; PyStackRef_CLOSE(tmp); + _PyFrame_StackPointerInvalidate(frame); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); tmp = exc_value_st; exc_value_st = PyStackRef_NULL; stack_pointer[-2] = exc_value_st; PyStackRef_CLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); stack_pointer += -2; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); SET_CURRENT_CACHED_VALUES(0); @@ -13801,18 +14038,25 @@ } PyObject *match_o = NULL; PyObject *rest_o = NULL; - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); int res = _PyEval_ExceptionGroupMatch(frame, exc_value, match_type, &match_o, &rest_o); + _PyFrame_StackPointerInvalidate(frame); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); _PyStackRef tmp = match_type_st; match_type_st = PyStackRef_NULL; stack_pointer[-1] = match_type_st; PyStackRef_CLOSE(tmp); + _PyFrame_StackPointerInvalidate(frame); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); tmp = exc_value_st; exc_value_st = PyStackRef_NULL; stack_pointer[-2] = exc_value_st; PyStackRef_CLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); stack_pointer += -2; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); if (res < 0) { @@ -13826,8 +14070,9 @@ } if (!Py_IsNone(match_o)) { _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyErr_SetHandledException(match_o); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } rest = PyStackRef_FromPyObjectSteal(rest_o); match = PyStackRef_FromPyObjectSteal(match_o); @@ -13857,20 +14102,23 @@ stack_pointer += 2; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); int err = _PyEval_CheckExceptTypeValid(tstate, right_o); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (err < 0) { SET_CURRENT_CACHED_VALUES(0); JUMP_TO_ERROR(); } - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); int res = PyErr_GivenExceptionMatches(left_o, right_o); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(right); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); b = res ? PyStackRef_True : PyStackRef_False; _tos_cache1 = b; _tos_cache0 = left; @@ -13901,12 +14149,13 @@ stack_pointer += 2; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); res_o = _PyEval_LazyImportName(tstate, BUILTINS(), GLOBALS(), LOCALS(), name, PyStackRef_AsPyObjectBorrow(fromlist), PyStackRef_AsPyObjectBorrow(level), oparg & 0x01); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } else { stack_pointer[0] = level; @@ -13914,22 +14163,27 @@ stack_pointer += 2; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); res_o = _PyEval_ImportName(tstate, BUILTINS(), GLOBALS(), LOCALS(), name, PyStackRef_AsPyObjectBorrow(fromlist), PyStackRef_AsPyObjectBorrow(level)); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); _PyStackRef tmp = fromlist; fromlist = PyStackRef_NULL; stack_pointer[-1] = fromlist; PyStackRef_CLOSE(tmp); + _PyFrame_StackPointerInvalidate(frame); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); tmp = level; level = PyStackRef_NULL; stack_pointer[-2] = level; PyStackRef_CLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); stack_pointer += -2; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); if (res_o == NULL) { @@ -13960,18 +14214,20 @@ stack_pointer += 1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); res_o = _PyEval_LazyImportFrom( tstate, frame, PyStackRef_AsPyObjectBorrow(from), name); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } else { stack_pointer[0] = from; stack_pointer += 1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); res_o = _PyEval_ImportFrom( tstate, PyStackRef_AsPyObjectBorrow(from), name); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } if (res_o == NULL) { stack_pointer[-1] = from; @@ -14009,11 +14265,12 @@ stack_pointer += 1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); _PyStackRef tmp = value; value = b; stack_pointer[-1] = value; PyStackRef_CLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); stack_pointer += -1; } _tos_cache0 = b; @@ -14037,8 +14294,9 @@ stack_pointer += 1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); Py_ssize_t len_i = PyObject_Length(PyStackRef_AsPyObjectBorrow(obj)); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (len_i < 0) { SET_CURRENT_CACHED_VALUES(0); JUMP_TO_ERROR(); @@ -14083,11 +14341,12 @@ stack_pointer += 3; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyObject *attrs_o = _PyEval_MatchClass(tstate, PyStackRef_AsPyObjectBorrow(subject), PyStackRef_AsPyObjectBorrow(type), oparg, PyStackRef_AsPyObjectBorrow(names)); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (attrs_o) { assert(PyTuple_CheckExact(attrs_o)); attrs = PyStackRef_FromPyObjectSteal(attrs_o); @@ -14230,9 +14489,10 @@ stack_pointer += 2; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyObject *values_or_none_o = _PyEval_MatchKeys(tstate, PyStackRef_AsPyObjectBorrow(subject), PyStackRef_AsPyObjectBorrow(keys)); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (values_or_none_o == NULL) { SET_CURRENT_CACHED_VALUES(0); JUMP_TO_ERROR(); @@ -14261,8 +14521,9 @@ stack_pointer += 1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); _PyStackRef result = _PyEval_GetIter(iterable, &index_or_null, oparg); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (PyStackRef_IsError(result)) { stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); @@ -14509,13 +14770,15 @@ stack_pointer += 1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyObject *iter_o = PyObject_GetIter(PyStackRef_AsPyObjectBorrow(iterable)); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(iterable); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (iter_o == NULL) { SET_CURRENT_CACHED_VALUES(0); JUMP_TO_ERROR(); @@ -14547,8 +14810,9 @@ stack_pointer += 2; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); _PyStackRef item = _PyForIter_VirtualIteratorNext(tstate, frame, iter, &null_or_index); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (!PyStackRef_IsValid(item)) { if (PyStackRef_IsError(item)) { stack_pointer[-1] = null_or_index; @@ -14685,15 +14949,20 @@ stack_pointer += 2; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyObject *item = iternext_v(PyStackRef_AsPyObjectBorrow(iter)); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (item == NULL) { if (_PyErr_Occurred(tstate)) { if (_PyErr_ExceptionMatches(tstate, PyExc_StopIteration)) { - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); _PyEval_MonitorRaise(tstate, frame, frame->instr_ptr); + _PyFrame_StackPointerInvalidate(frame); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); _PyErr_Clear(tstate); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } else { SET_CURRENT_CACHED_VALUES(0); @@ -14911,8 +15180,9 @@ stack_pointer += 2; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); _PyObjectIndexPair next_index = Py_TYPE(iter_o)->_tp_iteritem(iter_o, index); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); PyObject *next_o = next_index.object; index = next_index.index; if (next_o == NULL) { @@ -15213,8 +15483,9 @@ stack_pointer += 2; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); int result = _PyList_GetItemRefNoLock((PyListObject *)list_o, PyStackRef_UntagInt(null_or_index), &next); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (result <= 0) { UOP_STAT_INC(uopcode, miss); _tos_cache1 = null_or_index; @@ -15976,21 +16247,24 @@ method_and_self = &stack_pointer[-2]; PyObject *name = _Py_SpecialMethods[oparg].name; _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); int err = _PyObject_LookupSpecialMethod(name, method_and_self); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (err <= 0) { if (err == 0) { PyObject *owner = PyStackRef_AsPyObjectBorrow(method_and_self[1]); - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); const char *errfmt = _PyEval_SpecialMethodCanSuggest(owner, oparg) ? _Py_SpecialMethods[oparg].error_suggestion : _Py_SpecialMethods[oparg].error; - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); assert(!_PyErr_Occurred(tstate)); assert(errfmt != NULL); - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); _PyErr_Format(tstate, PyExc_TypeError, errfmt, owner); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } SET_CURRENT_CACHED_VALUES(0); JUMP_TO_ERROR(); @@ -16039,13 +16313,15 @@ stack_pointer += 3; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); res_o = PyObject_Vectorcall(exit_func_o, stack + 2 - has_self, (3 + has_self) | PY_VECTORCALL_ARGUMENTS_OFFSET, NULL); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); Py_XDECREF(original_tb); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (res_o == NULL) { SET_CURRENT_CACHED_VALUES(0); JUMP_TO_ERROR(); @@ -16392,8 +16668,9 @@ STAT_INC(LOAD_ATTR, hit); assert(descr != NULL); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(owner); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); attr = PyStackRef_FromPyObjectNew(descr); _tos_cache0 = attr; _tos_cache1 = PyStackRef_ZERO_BITS; @@ -16417,8 +16694,9 @@ STAT_INC(LOAD_ATTR, hit); assert(descr != NULL); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(owner); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); attr = PyStackRef_FromPyObjectNew(descr); _tos_cache0 = attr; _tos_cache1 = PyStackRef_ZERO_BITS; @@ -16611,8 +16889,9 @@ stack_pointer[-2 - oparg] = callable; stack_pointer[-1 - oparg] = self_or_null; _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(temp); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } _tos_cache0 = PyStackRef_ZERO_BITS; _tos_cache1 = PyStackRef_ZERO_BITS; @@ -16649,11 +16928,12 @@ int code_flags = ((PyCodeObject*)PyFunction_GET_CODE(callable_o))->co_flags; PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable_o)); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); _PyInterpreterFrame *temp = _PyEvalFramePushAndInit( tstate, callable, locals, args, total_args, NULL, frame ); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); stack_pointer += -2 - oparg; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); if (temp == NULL) { @@ -16833,8 +17113,9 @@ stack_pointer[-2 - oparg] = callable; stack_pointer[-1 - oparg] = self_or_null; _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(temp); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); _tos_cache0 = PyStackRef_ZERO_BITS; _tos_cache1 = PyStackRef_ZERO_BITS; _tos_cache2 = PyStackRef_ZERO_BITS; @@ -16886,12 +17167,13 @@ total_args++; } _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyObject *res_o = _Py_VectorCall_StackRefSteal( callable, arguments, total_args, PyStackRef_NULL); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (res_o == NULL) { stack_pointer += -2 - oparg; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); @@ -16949,8 +17231,9 @@ stack_pointer[-2 - oparg] = callable; stack_pointer[-1 - oparg] = self_or_null; _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(temp); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); _tos_cache0 = PyStackRef_ZERO_BITS; _tos_cache1 = PyStackRef_ZERO_BITS; _tos_cache2 = PyStackRef_ZERO_BITS; @@ -17322,12 +17605,17 @@ assert(!IS_PEP523_HOOKED(tstate)); _PyInterpreterFrame *temp = PyStackRef_Unwrap(new_frame); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); assert(temp->previous == frame || temp->previous->previous == frame); CALL_STAT_INC(inlined_py_calls); frame = tstate->current_frame = temp; tstate->py_recursion_remaining--; - LOAD_SP(); + stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); LOAD_IP(0); + #ifdef Py_DEBUG + assert(frame->previous->stackpointer_valid == 1); + #endif DTRACE_FUNCTION_ENTRY(); LLTRACE_RESUME_FRAME(); SET_CURRENT_CACHED_VALUES(0); @@ -17810,8 +18098,9 @@ stack_pointer += 3; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyObject *res_o = PyObject_Str(arg_o); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (res_o == NULL) { SET_CURRENT_CACHED_VALUES(0); JUMP_TO_ERROR(); @@ -17942,8 +18231,9 @@ stack_pointer += 3; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyObject *res_o = PySequence_Tuple(arg_o); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (res_o == NULL) { SET_CURRENT_CACHED_VALUES(0); JUMP_TO_ERROR(); @@ -18016,8 +18306,9 @@ } STAT_INC(CALL, hit); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyObject *self_o = PyType_GenericAlloc(tp, 0); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (self_o == NULL) { SET_CURRENT_CACHED_VALUES(0); JUMP_TO_ERROR(); @@ -18027,9 +18318,10 @@ callable = PyStackRef_FromPyObjectNew(init_func); stack_pointer[-2 - oparg] = callable; stack_pointer[-1 - oparg] = self_or_null; - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(temp); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); _tos_cache0 = PyStackRef_ZERO_BITS; _tos_cache1 = PyStackRef_ZERO_BITS; _tos_cache2 = PyStackRef_ZERO_BITS; @@ -18050,22 +18342,25 @@ self = stack_pointer[-1 - oparg]; init = stack_pointer[-2 - oparg]; _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); _PyInterpreterFrame *shim = _PyFrame_PushTrampolineUnchecked( tstate, (PyCodeObject *)&_Py_InitCleanup, 1, frame); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); assert(_PyFrame_GetBytecode(shim)[0].op.code == EXIT_INIT_CHECK); assert(_PyFrame_GetBytecode(shim)[1].op.code == RETURN_VALUE); shim->localsplus[0] = PyStackRef_DUP(self); - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); _PyInterpreterFrame *temp = _PyEvalFramePushAndInit( tstate, init, NULL, args-1, oparg+1, NULL, shim); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); stack_pointer += -2 - oparg; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); if (temp == NULL) { _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); _PyEval_FrameClearAndPop(tstate, shim); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); SET_CURRENT_CACHED_VALUES(0); JUMP_TO_ERROR(); } @@ -18091,10 +18386,11 @@ stack_pointer += 1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyErr_Format(PyExc_TypeError, "__init__() should return None, not '%.200s'", Py_TYPE(PyStackRef_AsPyObjectBorrow(should_be_none))->tp_name); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); SET_CURRENT_CACHED_VALUES(0); JUMP_TO_ERROR(); } @@ -18147,11 +18443,12 @@ } STAT_INC(CALL, hit); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyObject *res_o = _Py_CallBuiltinClass_StackRef( callable, arguments, total_args); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (res_o == NULL) { SET_CURRENT_CACHED_VALUES(0); JUMP_TO_ERROR(); @@ -18159,9 +18456,10 @@ _PyStackRef temp = callable; callable = PyStackRef_FromPyObjectSteal(res_o); stack_pointer[-2 - oparg] = callable; - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(temp); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); _tos_cache0 = PyStackRef_ZERO_BITS; _tos_cache1 = PyStackRef_ZERO_BITS; _tos_cache2 = PyStackRef_ZERO_BITS; @@ -18224,8 +18522,9 @@ PyCFunction cfunc = PyCFunction_GET_FUNCTION(callable_o); _PyStackRef arg = args[0]; _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyObject *res_o = _PyCFunction_TrampolineCall(cfunc, PyCFunction_GET_SELF(callable_o), PyStackRef_AsPyObjectBorrow(arg)); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); _Py_LeaveRecursiveCallTstate(tstate); assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); if (res_o == NULL) { @@ -18285,12 +18584,13 @@ } STAT_INC(CALL, hit); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyObject *res_o = _Py_BuiltinCallFast_StackRef( callable, arguments, total_args ); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (res_o == NULL) { SET_CURRENT_CACHED_VALUES(0); JUMP_TO_ERROR(); @@ -18298,9 +18598,10 @@ _PyStackRef temp = callable; callable = PyStackRef_FromPyObjectSteal(res_o); stack_pointer[-2 - oparg] = callable; - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(temp); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); _tos_cache0 = PyStackRef_ZERO_BITS; _tos_cache1 = PyStackRef_ZERO_BITS; _tos_cache2 = PyStackRef_ZERO_BITS; @@ -18349,8 +18650,9 @@ } STAT_INC(CALL, hit); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyObject *res_o = _Py_BuiltinCallFastWithKeywords_StackRef(callable, arguments, total_args); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (res_o == NULL) { SET_CURRENT_CACHED_VALUES(0); JUMP_TO_ERROR(); @@ -18358,9 +18660,10 @@ _PyStackRef temp = callable; callable = PyStackRef_FromPyObjectSteal(res_o); stack_pointer[-2 - oparg] = callable; - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(temp); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); _tos_cache0 = PyStackRef_ZERO_BITS; _tos_cache1 = PyStackRef_ZERO_BITS; _tos_cache2 = PyStackRef_ZERO_BITS; @@ -18488,8 +18791,9 @@ stack_pointer += 3; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); Py_ssize_t len_i = PyObject_Length(arg_o); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (len_i < 0) { SET_CURRENT_CACHED_VALUES(0); JUMP_TO_ERROR(); @@ -18635,8 +18939,9 @@ stack_pointer += 3; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); int retval = PyObject_IsInstance(inst_o, cls_o); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (retval < 0) { SET_CURRENT_CACHED_VALUES(0); JUMP_TO_ERROR(); @@ -18645,18 +18950,21 @@ stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(cls); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(instance); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); stack_pointer += -2; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(callable); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); res = retval ? PyStackRef_True : PyStackRef_False; assert((!PyStackRef_IsNull(res)) ^ (_PyErr_Occurred(tstate) != NULL)); _tos_cache0 = res; @@ -19012,8 +19320,9 @@ PyObject *self = PyStackRef_AsPyObjectBorrow(arguments[0]); PyObject *arg = PyStackRef_AsPyObjectBorrow(arguments[1]); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyObject *res_o = _PyCFunction_TrampolineCall(cfunc, self, arg); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); _Py_LeaveRecursiveCallTstate(tstate); assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); if (res_o == NULL) { @@ -19124,8 +19433,9 @@ PyObject *self = PyStackRef_AsPyObjectBorrow(args[0]); PyObject *arg = PyStackRef_AsPyObjectBorrow(args[1]); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyObject *res_o = _PyCFunction_TrampolineCall(cfunc_v, self, arg); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); _Py_LeaveRecursiveCallTstate(tstate); assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); if (res_o == NULL) { @@ -19213,7 +19523,11 @@ assert(self != NULL); STAT_INC(CALL, hit); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyCFunctionFastWithKeywords cfunc = _PyCFunctionFastWithKeywords_CAST(method->d_method->ml_meth); + _PyFrame_StackPointerInvalidate(frame); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); PyObject *res_o = _PyCallMethodDescriptorFastWithKeywords_StackRef( callable, cfunc, @@ -19221,7 +19535,7 @@ arguments, total_args ); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (res_o == NULL) { SET_CURRENT_CACHED_VALUES(0); JUMP_TO_ERROR(); @@ -19229,9 +19543,10 @@ _PyStackRef temp = callable; callable = PyStackRef_FromPyObjectSteal(res_o); stack_pointer[-2 - oparg] = callable; - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(temp); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); _tos_cache0 = PyStackRef_ZERO_BITS; _tos_cache1 = PyStackRef_ZERO_BITS; _tos_cache2 = PyStackRef_ZERO_BITS; @@ -19254,7 +19569,11 @@ PyObject *self = PyStackRef_AsPyObjectBorrow(self_st); STAT_INC(CALL, hit); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); volatile PyCFunctionFastWithKeywords cfunc_v = _PyCFunctionFastWithKeywords_CAST(cfunc); + _PyFrame_StackPointerInvalidate(frame); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); PyObject *res_o = _PyCallMethodDescriptorFastWithKeywords_StackRef( callable, cfunc_v, @@ -19262,7 +19581,7 @@ args - 1, oparg + 1 ); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (res_o == NULL) { SET_CURRENT_CACHED_VALUES(0); JUMP_TO_ERROR(); @@ -19270,9 +19589,10 @@ _PyStackRef temp = callable; callable = PyStackRef_FromPyObjectSteal(res_o); stack_pointer[-2 - oparg] = callable; - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(temp); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); _tos_cache0 = PyStackRef_ZERO_BITS; _tos_cache1 = PyStackRef_ZERO_BITS; _tos_cache2 = PyStackRef_ZERO_BITS; @@ -19348,8 +19668,9 @@ STAT_INC(CALL, hit); PyCFunction cfunc = method->d_method->ml_meth; _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyObject *res_o = _PyCFunction_TrampolineCall(cfunc, self, NULL); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); _Py_LeaveRecursiveCallTstate(tstate); assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); if (res_o == NULL) { @@ -19387,8 +19708,9 @@ STAT_INC(CALL, hit); volatile PyCFunction cfunc_v = (PyCFunction)cfunc; _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyObject *res_o = _PyCFunction_TrampolineCall(cfunc_v, self, NULL); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); _Py_LeaveRecursiveCallTstate(tstate); assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); if (res_o == NULL) { @@ -19473,7 +19795,11 @@ assert(self != NULL); STAT_INC(CALL, hit); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyCFunctionFast cfunc = _PyCFunctionFast_CAST(method->d_method->ml_meth); + _PyFrame_StackPointerInvalidate(frame); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); PyObject *res_o = _PyCallMethodDescriptorFast_StackRef( callable, cfunc, @@ -19481,7 +19807,7 @@ arguments, total_args ); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (res_o == NULL) { SET_CURRENT_CACHED_VALUES(0); JUMP_TO_ERROR(); @@ -19489,9 +19815,10 @@ _PyStackRef temp = callable; callable = PyStackRef_FromPyObjectSteal(res_o); stack_pointer[-2 - oparg] = callable; - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(temp); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); _tos_cache0 = PyStackRef_ZERO_BITS; _tos_cache1 = PyStackRef_ZERO_BITS; _tos_cache2 = PyStackRef_ZERO_BITS; @@ -19515,7 +19842,11 @@ assert(self != NULL); STAT_INC(CALL, hit); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); volatile PyCFunctionFast cfunc_v = _PyCFunctionFast_CAST(cfunc); + _PyFrame_StackPointerInvalidate(frame); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); PyObject *res_o = _PyCallMethodDescriptorFast_StackRef( callable, cfunc_v, @@ -19523,7 +19854,7 @@ args - 1, oparg + 1 ); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (res_o == NULL) { SET_CURRENT_CACHED_VALUES(0); JUMP_TO_ERROR(); @@ -19531,9 +19862,10 @@ _PyStackRef temp = callable; callable = PyStackRef_FromPyObjectSteal(res_o); stack_pointer[-2 - oparg] = callable; - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(temp); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); _tos_cache0 = PyStackRef_ZERO_BITS; _tos_cache1 = PyStackRef_ZERO_BITS; _tos_cache2 = PyStackRef_ZERO_BITS; @@ -19566,8 +19898,9 @@ stack_pointer += 1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(temp); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); stack_pointer += -1; } _tos_cache0 = _stack_item_0; @@ -19612,16 +19945,18 @@ stack_pointer += 1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); _PyInterpreterFrame *temp = _PyEvalFramePushAndInit( tstate, callable, locals, arguments, positional_args, kwnames_o, frame ); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(kwnames); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); stack_pointer += -2 - oparg; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); if (temp == NULL) { @@ -19729,8 +20064,9 @@ stack_pointer += 1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(callable_s); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); _tos_cache0 = _stack_item_0; _tos_cache1 = PyStackRef_ZERO_BITS; _tos_cache2 = PyStackRef_ZERO_BITS; @@ -19794,12 +20130,13 @@ stack_pointer += 1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyObject *res_o = _Py_VectorCall_StackRefSteal( callable, arguments, total_args, kwnames); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (res_o == NULL) { stack_pointer += -3 - oparg; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); @@ -19835,15 +20172,17 @@ stack_pointer += 3; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); int err = _Py_Check_ArgsIterable(tstate, PyStackRef_AsPyObjectBorrow(func), callargs_o); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (err < 0) { SET_CURRENT_CACHED_VALUES(0); JUMP_TO_ERROR(); } - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); PyObject *tuple_o = PySequence_Tuple(callargs_o); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (tuple_o == NULL) { SET_CURRENT_CACHED_VALUES(0); JUMP_TO_ERROR(); @@ -19851,9 +20190,10 @@ _PyStackRef temp = callargs; callargs = PyStackRef_FromPyObjectSteal(tuple_o); stack_pointer[-2] = callargs; - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(temp); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); stack_pointer += -3; } _tos_cache2 = _stack_item_2; @@ -20013,10 +20353,11 @@ stack_pointer += 1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); _PyInterpreterFrame *new_frame = _PyEvalFramePushAndInit_Ex( tstate, func_st, locals, nargs, callargs, kwargs, frame); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); stack_pointer += -2; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); if (new_frame == NULL) { @@ -20153,23 +20494,27 @@ stack_pointer += 3; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyObject *result_o = PyObject_Call(func, callargs, kwargs); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_XCLOSE(kwargs_st); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(callargs_st); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); stack_pointer += -2; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(func_st); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (result_o == NULL) { SET_CURRENT_CACHED_VALUES(0); JUMP_TO_ERROR(); @@ -20196,9 +20541,10 @@ stack_pointer += 1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyFunctionObject *func_obj = (PyFunctionObject *) PyFunction_New(codeobj, GLOBALS()); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (func_obj == NULL) { SET_CURRENT_CACHED_VALUES(0); JUMP_TO_ERROR(); @@ -20331,14 +20677,16 @@ assert(PyStackRef_FunctionCheck(frame->f_funcobj)); PyFunctionObject *func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyGenObject *gen = (PyGenObject *)_Py_MakeCoro(func); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (gen == NULL) { SET_CURRENT_CACHED_VALUES(0); JUMP_TO_ERROR(); } assert(STACK_LEVEL() <= 2); - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); _PyInterpreterFrame *gen_frame = &gen->gi_iframe; frame->instr_ptr++; _PyFrame_Copy(frame, gen_frame); @@ -20352,6 +20700,7 @@ frame = tstate->current_frame = prev; LOAD_IP(frame->return_offset); stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); res = PyStackRef_FromPyObjectStealMortal((PyObject *)gen); LLTRACE_RESUME_FRAME(); _tos_cache0 = res; @@ -20374,13 +20723,14 @@ PyObject *step_o = oparg == 3 ? PyStackRef_AsPyObjectBorrow(args[2]) : NULL; PyObject *slice_o = PySlice_New(start_o, stop_o, step_o); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); _PyStackRef tmp; for (int _i = oparg; --_i >= 0;) { tmp = args[_i]; args[_i] = PyStackRef_NULL; PyStackRef_CLOSE(tmp); } - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); stack_pointer += -oparg; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); if (slice_o == NULL) { @@ -20411,13 +20761,15 @@ stack_pointer += 1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyObject *result_o = conv_fn(PyStackRef_AsPyObjectBorrow(value)); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(value); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (result_o == NULL) { SET_CURRENT_CACHED_VALUES(0); JUMP_TO_ERROR(); @@ -20444,13 +20796,15 @@ stack_pointer += 1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyObject *res_o = PyObject_Format(value_o, NULL); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(value); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (res_o == NULL) { SET_CURRENT_CACHED_VALUES(0); JUMP_TO_ERROR(); @@ -20483,16 +20837,23 @@ stack_pointer += 2; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyObject *res_o = PyObject_Format(PyStackRef_AsPyObjectBorrow(value), PyStackRef_AsPyObjectBorrow(fmt_spec)); + _PyFrame_StackPointerInvalidate(frame); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); _PyStackRef tmp = fmt_spec; fmt_spec = PyStackRef_NULL; stack_pointer[-1] = fmt_spec; PyStackRef_CLOSE(tmp); + _PyFrame_StackPointerInvalidate(frame); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); tmp = value; value = PyStackRef_NULL; stack_pointer[-2] = value; PyStackRef_CLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); stack_pointer += -2; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); if (res_o == NULL) { @@ -20716,8 +21077,9 @@ stack_pointer += 2; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyObject *res_o = _PyEval_BinaryOps[oparg](lhs_o, rhs_o); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (res_o == NULL) { SET_CURRENT_CACHED_VALUES(0); JUMP_TO_ERROR(); @@ -22177,8 +22539,9 @@ stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(val); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (1) { UOP_STAT_INC(uopcode, miss); SET_CURRENT_CACHED_VALUES(0); @@ -22201,8 +22564,9 @@ int is_none = PyStackRef_IsNone(val); if (!is_none) { _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(val); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (1) { UOP_STAT_INC(uopcode, miss); SET_CURRENT_CACHED_VALUES(0); @@ -22227,8 +22591,9 @@ stack_pointer += 1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(val); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (1) { UOP_STAT_INC(uopcode, miss); _tos_cache0 = _stack_item_0; @@ -22259,8 +22624,9 @@ stack_pointer += 2; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(val); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (1) { UOP_STAT_INC(uopcode, miss); _tos_cache1 = _stack_item_1; @@ -22286,8 +22652,9 @@ val = _stack_item_0; int is_none = PyStackRef_IsNone(val); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(val); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (is_none) { UOP_STAT_INC(uopcode, miss); SET_CURRENT_CACHED_VALUES(0); @@ -22524,13 +22891,20 @@ OPT_HIST(trace_uop_execution_counter, trace_run_length_hist); if (frame->lltrace >= 3) { _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); printf("SIDE EXIT: [UOp "); + _PyFrame_StackPointerInvalidate(frame); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); _PyUOpPrint(&next_uop[-1]); + _PyFrame_StackPointerInvalidate(frame); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); printf(", exit %tu, temp %d, target %d -> %s, is_control_flow %d]\n", exit - current_executor->exits, exit->temperature.value_and_backoff, (int)(target - _PyFrame_GetBytecode(frame)), _PyOpcode_OpName[target->op.code], exit->is_control_flow); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } #endif tstate->jit_exit = exit; @@ -22554,13 +22928,20 @@ stack_pointer += 1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); printf("SIDE EXIT: [UOp "); + _PyFrame_StackPointerInvalidate(frame); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); _PyUOpPrint(&next_uop[-1]); + _PyFrame_StackPointerInvalidate(frame); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); printf(", exit %tu, temp %d, target %d -> %s, is_control_flow %d]\n", exit - current_executor->exits, exit->temperature.value_and_backoff, (int)(target - _PyFrame_GetBytecode(frame)), _PyOpcode_OpName[target->op.code], exit->is_control_flow); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); stack_pointer += -1; } #endif @@ -22590,13 +22971,20 @@ stack_pointer += 2; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); printf("SIDE EXIT: [UOp "); + _PyFrame_StackPointerInvalidate(frame); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); _PyUOpPrint(&next_uop[-1]); + _PyFrame_StackPointerInvalidate(frame); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); printf(", exit %tu, temp %d, target %d -> %s, is_control_flow %d]\n", exit - current_executor->exits, exit->temperature.value_and_backoff, (int)(target - _PyFrame_GetBytecode(frame)), _PyOpcode_OpName[target->op.code], exit->is_control_flow); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); stack_pointer += -2; } #endif @@ -22629,13 +23017,20 @@ stack_pointer += 3; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); printf("SIDE EXIT: [UOp "); + _PyFrame_StackPointerInvalidate(frame); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); _PyUOpPrint(&next_uop[-1]); + _PyFrame_StackPointerInvalidate(frame); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); printf(", exit %tu, temp %d, target %d -> %s, is_control_flow %d]\n", exit - current_executor->exits, exit->temperature.value_and_backoff, (int)(target - _PyFrame_GetBytecode(frame)), _PyOpcode_OpName[target->op.code], exit->is_control_flow); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); stack_pointer += -3; } #endif @@ -22659,13 +23054,20 @@ OPT_HIST(trace_uop_execution_counter, trace_run_length_hist); if (frame->lltrace >= 3) { _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); printf("DYNAMIC EXIT: [UOp "); + _PyFrame_StackPointerInvalidate(frame); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); _PyUOpPrint(&next_uop[-1]); + _PyFrame_StackPointerInvalidate(frame); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); printf(", exit %tu, temp %d, target %d -> %s]\n", exit - current_executor->exits, exit->temperature.value_and_backoff, (int)(target - _PyFrame_GetBytecode(frame)), _PyOpcode_OpName[target->op.code]); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } #endif SET_CURRENT_CACHED_VALUES(0); @@ -22686,13 +23088,20 @@ stack_pointer += 1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); printf("DYNAMIC EXIT: [UOp "); + _PyFrame_StackPointerInvalidate(frame); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); _PyUOpPrint(&next_uop[-1]); + _PyFrame_StackPointerInvalidate(frame); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); printf(", exit %tu, temp %d, target %d -> %s]\n", exit - current_executor->exits, exit->temperature.value_and_backoff, (int)(target - _PyFrame_GetBytecode(frame)), _PyOpcode_OpName[target->op.code]); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); stack_pointer += -1; } #endif @@ -22719,13 +23128,20 @@ stack_pointer += 2; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); printf("DYNAMIC EXIT: [UOp "); + _PyFrame_StackPointerInvalidate(frame); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); _PyUOpPrint(&next_uop[-1]); + _PyFrame_StackPointerInvalidate(frame); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); printf(", exit %tu, temp %d, target %d -> %s]\n", exit - current_executor->exits, exit->temperature.value_and_backoff, (int)(target - _PyFrame_GetBytecode(frame)), _PyOpcode_OpName[target->op.code]); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); stack_pointer += -2; } #endif @@ -22755,13 +23171,20 @@ stack_pointer += 3; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); printf("DYNAMIC EXIT: [UOp "); + _PyFrame_StackPointerInvalidate(frame); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); _PyUOpPrint(&next_uop[-1]); + _PyFrame_StackPointerInvalidate(frame); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); printf(", exit %tu, temp %d, target %d -> %s]\n", exit - current_executor->exits, exit->temperature.value_and_backoff, (int)(target - _PyFrame_GetBytecode(frame)), _PyOpcode_OpName[target->op.code]); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); stack_pointer += -3; } #endif @@ -23037,8 +23460,9 @@ assert(tstate->jit_exit->executor == current_executor); assert(tstate->current_executor == executor); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); _PyExecutor_ClearExit(tstate->jit_exit); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (true) { UOP_STAT_INC(uopcode, miss); SET_CURRENT_CACHED_VALUES(0); @@ -23204,8 +23628,9 @@ CHECK_CURRENT_CACHED_VALUES(0); ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); int err = _Py_HandlePending(tstate); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); SET_CURRENT_CACHED_VALUES(0); GOTO_TIER_ONE(err ? NULL : _PyFrame_GetBytecode(frame) + CURRENT_TARGET()); } @@ -23218,8 +23643,9 @@ stack_pointer += 1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); int err = _Py_HandlePending(tstate); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); SET_CURRENT_CACHED_VALUES(0); GOTO_TIER_ONE(err ? NULL : _PyFrame_GetBytecode(frame) + CURRENT_TARGET()); } @@ -23234,8 +23660,9 @@ stack_pointer += 2; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); int err = _Py_HandlePending(tstate); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); SET_CURRENT_CACHED_VALUES(0); GOTO_TIER_ONE(err ? NULL : _PyFrame_GetBytecode(frame) + CURRENT_TARGET()); } @@ -23252,8 +23679,9 @@ stack_pointer += 3; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); int err = _Py_HandlePending(tstate); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); SET_CURRENT_CACHED_VALUES(0); GOTO_TIER_ONE(err ? NULL : _PyFrame_GetBytecode(frame) + CURRENT_TARGET()); } diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index a5590d992b39..2cdf48c559a2 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -45,8 +45,9 @@ if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { next_instr = this_instr; _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); _Py_Specialize_BinaryOp(lhs, rhs, next_instr, oparg, LOCALS_ARRAY); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); DISPATCH_SAME_OPARG(); } OPCODE_DEFERRED_INC(BINARY_OP); @@ -62,8 +63,9 @@ PyObject *rhs_o = PyStackRef_AsPyObjectBorrow(rhs); assert(_PyEval_BinaryOps[oparg]); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyObject *res_o = _PyEval_BinaryOps[oparg](lhs_o, rhs_o); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (res_o == NULL) { JUMP_TO_LABEL(error); } @@ -76,9 +78,10 @@ value = r; stack_pointer[-2] = res; stack_pointer[-1] = l; - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); PyStackRef_XCLOSE(value); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } // _POP_TOP { @@ -86,8 +89,9 @@ stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_XCLOSE(value); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } DISPATCH(); } @@ -344,10 +348,11 @@ assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5); assert(d != NULL); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); int match = (d->guard != NULL) ? d->guard(left_o, right_o) : (Py_TYPE(left_o) == d->lhs_type && Py_TYPE(right_o) == d->rhs_type); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (!match) { UPDATE_MISS_STATS(BINARY_OP); assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); @@ -363,9 +368,10 @@ assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5); _PyBinaryOpSpecializationDescr *d = (_PyBinaryOpSpecializationDescr*)descr; STAT_INC(BINARY_OP, hit); - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); PyObject *res_o = d->action(left_o, right_o); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (res_o == NULL) { JUMP_TO_LABEL(error); } @@ -381,9 +387,10 @@ value = r; stack_pointer[-2] = res; stack_pointer[-1] = l; - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); PyStackRef_XCLOSE(value); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } // _POP_TOP { @@ -391,8 +398,9 @@ stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_XCLOSE(value); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } DISPATCH(); } @@ -463,9 +471,13 @@ stack_pointer += -2; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyUnicode_Append(&temp, right_o); + _PyFrame_StackPointerInvalidate(frame); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); _Py_DECREF_SPECIALIZED(right_o, _PyUnicode_ExactDealloc); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); *target_local = PyStackRef_NULL; if (temp == NULL) { JUMP_TO_LABEL(error); @@ -670,8 +682,9 @@ assert(Py_TYPE(dict)->tp_as_mapping->mp_subscript == _PyDict_Subscript); STAT_INC(BINARY_OP, hit); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyObject *res_o = _PyDict_Subscript(dict, sub); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (res_o == NULL) { JUMP_TO_LABEL(error); } @@ -684,9 +697,10 @@ value = ss; stack_pointer[-2] = res; stack_pointer[-1] = ds; - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); PyStackRef_XCLOSE(value); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } // _POP_TOP { @@ -694,8 +708,9 @@ stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_XCLOSE(value); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } DISPATCH(); } @@ -773,12 +788,17 @@ stack_pointer += -2; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); assert(temp->previous == frame || temp->previous->previous == frame); CALL_STAT_INC(inlined_py_calls); frame = tstate->current_frame = temp; tstate->py_recursion_remaining--; - LOAD_SP(); + stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); LOAD_IP(0); + #ifdef Py_DEBUG + assert(frame->previous->stackpointer_valid == 1); + #endif DTRACE_FUNCTION_ENTRY(); LLTRACE_RESUME_FRAME(); } @@ -838,8 +858,9 @@ } #ifdef Py_GIL_DISABLED _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyObject *res_o = _PyList_GetItemRef((PyListObject*)list, index); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (res_o == NULL) { UPDATE_MISS_STATS(BINARY_OP); assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); @@ -873,8 +894,9 @@ stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_XCLOSE(value); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } DISPATCH(); } @@ -928,8 +950,9 @@ assert(PySlice_Check(sub)); assert(PyList_CheckExact(list)); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyObject *res_o = _PyList_SliceSubscript(list, sub); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); STAT_INC(BINARY_OP, hit); if (res_o == NULL) { JUMP_TO_LABEL(error); @@ -943,9 +966,10 @@ value = ss; stack_pointer[-2] = res; stack_pointer[-1] = ls; - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); PyStackRef_XCLOSE(value); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } // _POP_TOP { @@ -953,8 +977,9 @@ stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_XCLOSE(value); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } DISPATCH(); } @@ -1134,8 +1159,9 @@ stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_XCLOSE(value); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } DISPATCH(); } @@ -1406,18 +1432,21 @@ PyObject *res_o; if (PyList_CheckExact(container_o)) { _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); res_o = _PyList_BinarySlice(container_o, start_o, stop_o); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } else if (PyTuple_CheckExact(container_o)) { _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); res_o = _PyTuple_BinarySlice(container_o, start_o, stop_o); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } else if (PyUnicode_CheckExact(container_o)) { _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); res_o = _PyUnicode_BinarySlice(container_o, start_o, stop_o); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } else { PyObject *slice = PySlice_New(start_o, stop_o, NULL); @@ -1426,25 +1455,36 @@ } else { _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); res_o = PyObject_GetItem(container_o, slice); + _PyFrame_StackPointerInvalidate(frame); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); Py_DECREF(slice); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } } _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); _PyStackRef tmp = stop; stop = PyStackRef_NULL; stack_pointer[-1] = stop; PyStackRef_CLOSE(tmp); + _PyFrame_StackPointerInvalidate(frame); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); tmp = start; start = PyStackRef_NULL; stack_pointer[-2] = start; PyStackRef_CLOSE(tmp); + _PyFrame_StackPointerInvalidate(frame); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); tmp = container; container = PyStackRef_NULL; stack_pointer[-3] = container; PyStackRef_CLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); stack_pointer += -3; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); if (res_o == NULL) { @@ -1484,14 +1524,16 @@ format_o = &_Py_STR(empty); } _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyObject *interpolation_o = _PyInterpolation_Build(value_o, str_o, conversion, format_o); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (oparg & 1) { stack_pointer += -(oparg & 1); ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(format[0]); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } else { stack_pointer += -(oparg & 1); @@ -1499,13 +1541,15 @@ stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(str); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(value); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (interpolation_o == NULL) { JUMP_TO_LABEL(error); } @@ -1528,8 +1572,9 @@ _PyStackRef list; values = &stack_pointer[-oparg]; _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyObject *list_o = _PyList_FromStackRefStealOnSuccess(values, oparg); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (list_o == NULL) { JUMP_TO_LABEL(error); } @@ -1552,8 +1597,9 @@ _PyStackRef map; values = &stack_pointer[-oparg*2]; _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyObject *map_o = _Py_BuildMap_StackRefSteal(values, oparg); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (map_o == NULL) { stack_pointer += -oparg*2; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); @@ -1578,17 +1624,19 @@ _PyStackRef set; values = &stack_pointer[-oparg]; _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyObject *set_o = PySet_New(NULL); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (set_o == NULL) { - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); _PyStackRef tmp; for (int _i = oparg; --_i >= 0;) { tmp = values[_i]; values[_i] = PyStackRef_NULL; PyStackRef_CLOSE(tmp); } - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); stack_pointer += -oparg; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); JUMP_TO_LABEL(error); @@ -1598,22 +1646,25 @@ _PyStackRef value = values[i]; values[i] = PyStackRef_NULL; if (err == 0) { - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); err = _PySet_AddTakeRef((PySetObject *)set_o, PyStackRef_AsPyObjectSteal(value)); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } else { - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(value); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } } if (err) { stack_pointer += -oparg; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); Py_DECREF(set_o); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); JUMP_TO_LABEL(error); } set = PyStackRef_FromPyObjectStealMortal(set_o); @@ -1639,13 +1690,14 @@ PyObject *step_o = oparg == 3 ? PyStackRef_AsPyObjectBorrow(args[2]) : NULL; PyObject *slice_o = PySlice_New(start_o, stop_o, step_o); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); _PyStackRef tmp; for (int _i = oparg; --_i >= 0;) { tmp = args[_i]; args[_i] = PyStackRef_NULL; PyStackRef_CLOSE(tmp); } - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); stack_pointer += -oparg; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); if (slice_o == NULL) { @@ -1670,8 +1722,9 @@ _PyStackRef str; pieces = &stack_pointer[-oparg]; _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyObject *str_o = _Py_BuildString_StackRefSteal(pieces, oparg); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (str_o == NULL) { stack_pointer += -oparg; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); @@ -1700,18 +1753,21 @@ PyObject *strings_o = PyStackRef_AsPyObjectBorrow(strings); PyObject *interpolations_o = PyStackRef_AsPyObjectBorrow(interpolations); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyObject *template_o = _PyTemplate_Build(strings_o, interpolations_o); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(interpolations); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(strings); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (template_o == NULL) { JUMP_TO_LABEL(error); } @@ -1783,8 +1839,9 @@ if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { next_instr = this_instr; _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); _Py_Specialize_Call(callable, self_or_null, next_instr, oparg + !PyStackRef_IsNull(self_or_null)); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); DISPATCH_SAME_OPARG(); } OPCODE_DEFERRED_INC(CALL); @@ -1804,8 +1861,9 @@ stack_pointer[-2 - oparg] = callable; stack_pointer[-1 - oparg] = self_or_null; _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(temp); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } } // _DO_CALL @@ -1827,11 +1885,12 @@ stack_pointer[-2 - oparg] = callable; stack_pointer[-1 - oparg] = self_or_null; _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); _PyInterpreterFrame *new_frame = _PyEvalFramePushAndInit( tstate, callable, locals, arguments, total_args, NULL, frame ); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); stack_pointer += -2 - oparg; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); if (new_frame == NULL) { @@ -1843,6 +1902,7 @@ stack_pointer[-2 - oparg] = callable; stack_pointer[-1 - oparg] = self_or_null; _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyObject* res_o = _Py_VectorCallInstrumentation_StackRefSteal( callable, arguments, @@ -1852,7 +1912,7 @@ frame, this_instr, tstate); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (res_o == NULL) { stack_pointer += -2 - oparg; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); @@ -1866,8 +1926,9 @@ stack_pointer += -1 - oparg; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); int err = check_periodics(tstate); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (err != 0) { JUMP_TO_LABEL(error); } @@ -1952,8 +2013,9 @@ } STAT_INC(CALL, hit); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyObject *self_o = PyType_GenericAlloc(tp, 0); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (self_o == NULL) { JUMP_TO_LABEL(error); } @@ -1962,32 +2024,36 @@ callable = PyStackRef_FromPyObjectNew(init_func); stack_pointer[-2 - oparg] = callable; stack_pointer[-1 - oparg] = self_or_null; - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(temp); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } // _CREATE_INIT_FRAME { args = &stack_pointer[-oparg]; self = self_or_null; init = callable; - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); _PyInterpreterFrame *shim = _PyFrame_PushTrampolineUnchecked( tstate, (PyCodeObject *)&_Py_InitCleanup, 1, frame); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); assert(_PyFrame_GetBytecode(shim)[0].op.code == EXIT_INIT_CHECK); assert(_PyFrame_GetBytecode(shim)[1].op.code == RETURN_VALUE); shim->localsplus[0] = PyStackRef_DUP(self); - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); _PyInterpreterFrame *temp = _PyEvalFramePushAndInit( tstate, init, NULL, args-1, oparg+1, NULL, shim); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); stack_pointer += -2 - oparg; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); if (temp == NULL) { _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); _PyEval_FrameClearAndPop(tstate, shim); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); JUMP_TO_LABEL(error); } frame->return_offset = 1 + INLINE_CACHE_ENTRIES_CALL; @@ -2000,12 +2066,17 @@ assert(!IS_PEP523_HOOKED(tstate)); _PyInterpreterFrame *temp = PyStackRef_Unwrap(new_frame); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); assert(temp->previous == frame || temp->previous->previous == frame); CALL_STAT_INC(inlined_py_calls); frame = tstate->current_frame = temp; tstate->py_recursion_remaining--; - LOAD_SP(); + stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); LOAD_IP(0); + #ifdef Py_DEBUG + assert(frame->previous->stackpointer_valid == 1); + #endif DTRACE_FUNCTION_ENTRY(); LLTRACE_RESUME_FRAME(); } @@ -2064,8 +2135,9 @@ stack_pointer[-2 - oparg] = callable; stack_pointer[-1 - oparg] = self_or_null; _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(temp); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } // flush // _CHECK_FUNCTION_VERSION @@ -2144,12 +2216,17 @@ stack_pointer += -2 - oparg; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); assert(temp->previous == frame || temp->previous->previous == frame); CALL_STAT_INC(inlined_py_calls); frame = tstate->current_frame = temp; tstate->py_recursion_remaining--; - LOAD_SP(); + stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); LOAD_IP(0); + #ifdef Py_DEBUG + assert(frame->previous->stackpointer_valid == 1); + #endif DTRACE_FUNCTION_ENTRY(); LLTRACE_RESUME_FRAME(); } @@ -2222,8 +2299,9 @@ stack_pointer[-2 - oparg] = callable; stack_pointer[-1 - oparg] = self_or_null; _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(temp); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } // flush // _CHECK_RECURSION_REMAINING @@ -2246,12 +2324,13 @@ assert(Py_TYPE(callable_o) == &PyFunction_Type); int code_flags = ((PyCodeObject*)PyFunction_GET_CODE(callable_o))->co_flags; PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable_o)); - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); _PyInterpreterFrame *temp = _PyEvalFramePushAndInit( tstate, callable, locals, args, total_args, NULL, frame ); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); stack_pointer += -2 - oparg; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); if (temp == NULL) { @@ -2273,12 +2352,17 @@ assert(!IS_PEP523_HOOKED(tstate)); _PyInterpreterFrame *temp = PyStackRef_Unwrap(new_frame); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); assert(temp->previous == frame || temp->previous->previous == frame); CALL_STAT_INC(inlined_py_calls); frame = tstate->current_frame = temp; tstate->py_recursion_remaining--; - LOAD_SP(); + stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); LOAD_IP(0); + #ifdef Py_DEBUG + assert(frame->previous->stackpointer_valid == 1); + #endif DTRACE_FUNCTION_ENTRY(); LLTRACE_RESUME_FRAME(); } @@ -2330,27 +2414,30 @@ } STAT_INC(CALL, hit); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyObject *res_o = _Py_CallBuiltinClass_StackRef( callable, arguments, total_args); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (res_o == NULL) { JUMP_TO_LABEL(error); } _PyStackRef temp = callable; callable = PyStackRef_FromPyObjectSteal(res_o); stack_pointer[-2 - oparg] = callable; - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(temp); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } // _POP_TOP_OPARG { args = &stack_pointer[-oparg]; - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); _PyStackRef_CloseStack(args, oparg); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } // _POP_TOP { @@ -2358,14 +2445,16 @@ stack_pointer += -1 - oparg; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_XCLOSE(value); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } // _CHECK_PERIODIC_AT_END { - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); int err = check_periodics(tstate); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (err != 0) { JUMP_TO_LABEL(error); } @@ -2417,28 +2506,31 @@ } STAT_INC(CALL, hit); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyObject *res_o = _Py_BuiltinCallFast_StackRef( callable, arguments, total_args ); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (res_o == NULL) { JUMP_TO_LABEL(error); } _PyStackRef temp = callable; callable = PyStackRef_FromPyObjectSteal(res_o); stack_pointer[-2 - oparg] = callable; - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(temp); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } // _POP_TOP_OPARG { args = &stack_pointer[-oparg]; - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); _PyStackRef_CloseStack(args, oparg); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } // _POP_TOP { @@ -2446,14 +2538,16 @@ stack_pointer += -1 - oparg; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_XCLOSE(value); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } // _CHECK_PERIODIC_AT_END { - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); int err = check_periodics(tstate); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (err != 0) { JUMP_TO_LABEL(error); } @@ -2505,24 +2599,27 @@ } STAT_INC(CALL, hit); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyObject *res_o = _Py_BuiltinCallFastWithKeywords_StackRef(callable, arguments, total_args); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (res_o == NULL) { JUMP_TO_LABEL(error); } _PyStackRef temp = callable; callable = PyStackRef_FromPyObjectSteal(res_o); stack_pointer[-2 - oparg] = callable; - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(temp); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } // _POP_TOP_OPARG { args = &stack_pointer[-oparg]; - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); _PyStackRef_CloseStack(args, oparg); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } // _POP_TOP { @@ -2530,14 +2627,16 @@ stack_pointer += -1 - oparg; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_XCLOSE(value); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } // _CHECK_PERIODIC_AT_END { - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); int err = check_periodics(tstate); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (err != 0) { JUMP_TO_LABEL(error); } @@ -2609,8 +2708,9 @@ PyCFunction cfunc = PyCFunction_GET_FUNCTION(callable_o); _PyStackRef arg = args[0]; _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyObject *res_o = _PyCFunction_TrampolineCall(cfunc, PyCFunction_GET_SELF(callable_o), PyStackRef_AsPyObjectBorrow(arg)); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); _Py_LeaveRecursiveCallTstate(tstate); assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); if (res_o == NULL) { @@ -2628,8 +2728,9 @@ stack_pointer += -oparg; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_XCLOSE(value); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } // _POP_TOP { @@ -2637,14 +2738,16 @@ stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_XCLOSE(value); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } // _CHECK_PERIODIC_AT_END { - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); int err = check_periodics(tstate); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (err != 0) { JUMP_TO_LABEL(error); } @@ -2688,23 +2791,26 @@ PyObject *callargs_o = PyStackRef_AsPyObjectBorrow(callargs); if (!PyTuple_CheckExact(callargs_o)) { _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); int err = _Py_Check_ArgsIterable(tstate, PyStackRef_AsPyObjectBorrow(func), callargs_o); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (err < 0) { JUMP_TO_LABEL(error); } - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); PyObject *tuple_o = PySequence_Tuple(callargs_o); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (tuple_o == NULL) { JUMP_TO_LABEL(error); } _PyStackRef temp = callargs; callargs = PyStackRef_FromPyObjectSteal(tuple_o); stack_pointer[-2] = callargs; - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(temp); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } } // _CALL_FUNCTION_EX_NON_PY_GENERAL @@ -2721,23 +2827,27 @@ assert(kwargs == NULL || PyDict_CheckExact(kwargs)); stack_pointer[-2] = callargs_st; _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyObject *result_o = PyObject_Call(func, callargs, kwargs); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_XCLOSE(kwargs_st); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(callargs_st); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); stack_pointer += -2; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(func_st); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (result_o == NULL) { JUMP_TO_LABEL(error); } @@ -2749,8 +2859,9 @@ stack_pointer += 1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); int err = check_periodics(tstate); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (err != 0) { JUMP_TO_LABEL(error); } @@ -2792,23 +2903,26 @@ PyObject *callargs_o = PyStackRef_AsPyObjectBorrow(callargs); if (!PyTuple_CheckExact(callargs_o)) { _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); int err = _Py_Check_ArgsIterable(tstate, PyStackRef_AsPyObjectBorrow(func), callargs_o); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (err < 0) { JUMP_TO_LABEL(error); } - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); PyObject *tuple_o = PySequence_Tuple(callargs_o); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (tuple_o == NULL) { JUMP_TO_LABEL(error); } _PyStackRef temp = callargs; callargs = PyStackRef_FromPyObjectSteal(tuple_o); stack_pointer[-2] = callargs; - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(temp); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } } // _CHECK_IS_PY_CALLABLE_EX @@ -2843,10 +2957,11 @@ stack_pointer += -3; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); _PyInterpreterFrame *new_frame = _PyEvalFramePushAndInit_Ex( tstate, func_st, locals, nargs, callargs, kwargs, frame); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); if (new_frame == NULL) { @@ -2869,12 +2984,17 @@ assert(!IS_PEP523_HOOKED(tstate)); _PyInterpreterFrame *temp = PyStackRef_Unwrap(new_frame); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); assert(temp->previous == frame || temp->previous->previous == frame); CALL_STAT_INC(inlined_py_calls); frame = tstate->current_frame = temp; tstate->py_recursion_remaining--; - LOAD_SP(); + stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); LOAD_IP(0); + #ifdef Py_DEBUG + assert(frame->previous->stackpointer_valid == 1); + #endif DTRACE_FUNCTION_ENTRY(); LLTRACE_RESUME_FRAME(); } @@ -2909,8 +3029,9 @@ if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { next_instr = this_instr; _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); _Py_Specialize_CallFunctionEx(func, next_instr); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); DISPATCH_SAME_OPARG(); } OPCODE_DEFERRED_INC(CALL_FUNCTION_EX); @@ -2923,23 +3044,26 @@ PyObject *callargs_o = PyStackRef_AsPyObjectBorrow(callargs); if (!PyTuple_CheckExact(callargs_o)) { _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); int err = _Py_Check_ArgsIterable(tstate, PyStackRef_AsPyObjectBorrow(func), callargs_o); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (err < 0) { JUMP_TO_LABEL(error); } - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); PyObject *tuple_o = PySequence_Tuple(callargs_o); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (tuple_o == NULL) { JUMP_TO_LABEL(error); } _PyStackRef temp = callargs; callargs = PyStackRef_FromPyObjectSteal(tuple_o); stack_pointer[-2] = callargs; - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(temp); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } } // _DO_CALL_FUNCTION_EX @@ -2962,34 +3086,39 @@ PyTuple_GET_ITEM(callargs, 0) : &_PyInstrumentation_MISSING; stack_pointer[-2] = callargs_st; _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); int err = _Py_call_instrumentation_2args( tstate, PY_MONITORING_EVENT_CALL, frame, this_instr, func, arg); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (err) { JUMP_TO_LABEL(error); } - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); result_o = PyObject_Call(func, callargs, kwargs); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (!PyFunction_Check(func) && !PyMethod_Check(func)) { if (result_o == NULL) { - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); _Py_call_instrumentation_exc2( tstate, PY_MONITORING_EVENT_C_RAISE, frame, this_instr, func, arg); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } else { - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); int err = _Py_call_instrumentation_2args( tstate, PY_MONITORING_EVENT_C_RETURN, frame, this_instr, func, arg); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (err < 0) { - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); Py_CLEAR(result_o); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } } } @@ -3008,10 +3137,11 @@ stack_pointer += -2; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); _PyInterpreterFrame *new_frame = _PyEvalFramePushAndInit_Ex( tstate, func_st, locals, nargs, callargs, kwargs, frame); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); stack_pointer += -2; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); if (new_frame == NULL) { @@ -3027,24 +3157,28 @@ assert(kwargs == NULL || PyDict_CheckExact(kwargs)); stack_pointer[-2] = callargs_st; _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); result_o = PyObject_Call(func, callargs, kwargs); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_XCLOSE(kwargs_st); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(callargs_st); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); stack_pointer += -2; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(func_st); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (result_o == NULL) { JUMP_TO_LABEL(error); } @@ -3056,8 +3190,9 @@ stack_pointer += 1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); int err = check_periodics(tstate); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (err != 0) { JUMP_TO_LABEL(error); } @@ -3081,8 +3216,9 @@ value = stack_pointer[-1]; assert(oparg <= MAX_INTRINSIC_1); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyObject *res_o = _PyIntrinsics_UnaryFunctions[oparg].func(tstate, PyStackRef_AsPyObjectBorrow(value)); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (res_o == NULL) { JUMP_TO_LABEL(error); } @@ -3093,9 +3229,10 @@ { value = v; stack_pointer[-1] = res; - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); PyStackRef_XCLOSE(value); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } DISPATCH(); } @@ -3122,8 +3259,9 @@ PyObject *value1 = PyStackRef_AsPyObjectBorrow(value1_st); PyObject *value2 = PyStackRef_AsPyObjectBorrow(value2_st); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyObject *res_o = _PyIntrinsics_BinaryFunctions[oparg].func(tstate, value2, value1); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (res_o == NULL) { JUMP_TO_LABEL(error); } @@ -3136,9 +3274,10 @@ value = vs2; stack_pointer[-2] = res; stack_pointer[-1] = vs1; - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); PyStackRef_XCLOSE(value); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } // _POP_TOP { @@ -3146,8 +3285,9 @@ stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_XCLOSE(value); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } DISPATCH(); } @@ -3198,8 +3338,9 @@ PyObject *inst_o = PyStackRef_AsPyObjectBorrow(instance); PyObject *cls_o = PyStackRef_AsPyObjectBorrow(cls); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); int retval = PyObject_IsInstance(inst_o, cls_o); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (retval < 0) { JUMP_TO_LABEL(error); } @@ -3207,18 +3348,21 @@ stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(cls); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(instance); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); stack_pointer += -2; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(callable); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); res = retval ? PyStackRef_True : PyStackRef_False; assert((!PyStackRef_IsNull(res)) ^ (_PyErr_Occurred(tstate) != NULL)); } @@ -3255,8 +3399,9 @@ if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { next_instr = this_instr; _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); _Py_Specialize_CallKw(callable, next_instr, oparg + !PyStackRef_IsNull(self_or_null)); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); DISPATCH_SAME_OPARG(); } OPCODE_DEFERRED_INC(CALL_KW); @@ -3276,8 +3421,9 @@ stack_pointer[-3 - oparg] = callable; stack_pointer[-2 - oparg] = self_or_null; _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(temp); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } } // _DO_CALL_KW @@ -3302,16 +3448,18 @@ stack_pointer[-3 - oparg] = callable; stack_pointer[-2 - oparg] = self_or_null; _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); _PyInterpreterFrame *new_frame = _PyEvalFramePushAndInit( tstate, callable, locals, arguments, positional_args, kwnames_o, frame ); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); stack_pointer += -3 - oparg; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(kwnames); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (new_frame == NULL) { JUMP_TO_LABEL(error); } @@ -3322,6 +3470,7 @@ stack_pointer[-3 - oparg] = callable; stack_pointer[-2 - oparg] = self_or_null; _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyObject* res_o = _Py_VectorCallInstrumentation_StackRefSteal( callable, arguments, @@ -3331,7 +3480,7 @@ frame, this_instr, tstate); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (res_o == NULL) { stack_pointer += -3 - oparg; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); @@ -3412,8 +3561,9 @@ stack_pointer[-3 - oparg] = callable; stack_pointer[-2 - oparg] = self_or_null; _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(callable_s); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } // flush // _PY_FRAME_KW @@ -3432,17 +3582,19 @@ assert(Py_TYPE(callable_o) == &PyFunction_Type); int code_flags = ((PyCodeObject*)PyFunction_GET_CODE(callable_o))->co_flags; PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable_o)); - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); _PyInterpreterFrame *temp = _PyEvalFramePushAndInit( tstate, callable, locals, arguments, positional_args, kwnames_o, frame ); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(kwnames); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); stack_pointer += -2 - oparg; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); if (temp == NULL) { @@ -3464,12 +3616,17 @@ assert(!IS_PEP523_HOOKED(tstate)); _PyInterpreterFrame *temp = PyStackRef_Unwrap(new_frame); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); assert(temp->previous == frame || temp->previous->previous == frame); CALL_STAT_INC(inlined_py_calls); frame = tstate->current_frame = temp; tstate->py_recursion_remaining--; - LOAD_SP(); + stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); LOAD_IP(0); + #ifdef Py_DEBUG + assert(frame->previous->stackpointer_valid == 1); + #endif DTRACE_FUNCTION_ENTRY(); LLTRACE_RESUME_FRAME(); } @@ -3525,12 +3682,13 @@ total_args++; } _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyObject *res_o = _Py_VectorCall_StackRefSteal( callable, arguments, total_args, kwnames); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (res_o == NULL) { stack_pointer += -3 - oparg; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); @@ -3544,8 +3702,9 @@ stack_pointer += -2 - oparg; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); int err = check_periodics(tstate); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (err != 0) { JUMP_TO_LABEL(error); } @@ -3621,16 +3780,18 @@ int code_flags = ((PyCodeObject*)PyFunction_GET_CODE(callable_o))->co_flags; PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable_o)); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); _PyInterpreterFrame *temp = _PyEvalFramePushAndInit( tstate, callable, locals, arguments, positional_args, kwnames_o, frame ); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(kwnames); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); stack_pointer += -2 - oparg; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); if (temp == NULL) { @@ -3652,12 +3813,17 @@ assert(!IS_PEP523_HOOKED(tstate)); _PyInterpreterFrame *temp = PyStackRef_Unwrap(new_frame); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); assert(temp->previous == frame || temp->previous->previous == frame); CALL_STAT_INC(inlined_py_calls); frame = tstate->current_frame = temp; tstate->py_recursion_remaining--; - LOAD_SP(); + stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); LOAD_IP(0); + #ifdef Py_DEBUG + assert(frame->previous->stackpointer_valid == 1); + #endif DTRACE_FUNCTION_ENTRY(); LLTRACE_RESUME_FRAME(); } @@ -3710,8 +3876,9 @@ STAT_INC(CALL, hit); PyObject *arg_o = PyStackRef_AsPyObjectBorrow(arg); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); Py_ssize_t len_i = PyObject_Length(arg_o); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (len_i < 0) { JUMP_TO_LABEL(error); } @@ -3732,8 +3899,9 @@ stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_XCLOSE(value); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } // _POP_TOP { @@ -3741,8 +3909,9 @@ stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_XCLOSE(value); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } DISPATCH(); } @@ -3826,8 +3995,9 @@ stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_XCLOSE(value); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } // _POP_TOP { @@ -3835,8 +4005,9 @@ stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_XCLOSE(value); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } DISPATCH(); } @@ -3906,7 +4077,11 @@ assert(self != NULL); STAT_INC(CALL, hit); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyCFunctionFast cfunc = _PyCFunctionFast_CAST(method->d_method->ml_meth); + _PyFrame_StackPointerInvalidate(frame); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); PyObject *res_o = _PyCallMethodDescriptorFast_StackRef( callable, cfunc, @@ -3914,22 +4089,24 @@ arguments, total_args ); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (res_o == NULL) { JUMP_TO_LABEL(error); } _PyStackRef temp = callable; callable = PyStackRef_FromPyObjectSteal(res_o); stack_pointer[-2 - oparg] = callable; - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(temp); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } // _POP_TOP_OPARG { - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); _PyStackRef_CloseStack(args, oparg); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } // _POP_TOP { @@ -3937,14 +4114,16 @@ stack_pointer += -1 - oparg; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_XCLOSE(value); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } // _CHECK_PERIODIC_AT_END { - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); int err = check_periodics(tstate); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (err != 0) { JUMP_TO_LABEL(error); } @@ -4018,7 +4197,11 @@ assert(self != NULL); STAT_INC(CALL, hit); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyCFunctionFastWithKeywords cfunc = _PyCFunctionFastWithKeywords_CAST(method->d_method->ml_meth); + _PyFrame_StackPointerInvalidate(frame); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); PyObject *res_o = _PyCallMethodDescriptorFastWithKeywords_StackRef( callable, cfunc, @@ -4026,22 +4209,24 @@ arguments, total_args ); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (res_o == NULL) { JUMP_TO_LABEL(error); } _PyStackRef temp = callable; callable = PyStackRef_FromPyObjectSteal(res_o); stack_pointer[-2 - oparg] = callable; - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(temp); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } // _POP_TOP_OPARG { - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); _PyStackRef_CloseStack(args, oparg); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } // _POP_TOP { @@ -4049,14 +4234,16 @@ stack_pointer += -1 - oparg; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_XCLOSE(value); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } // _CHECK_PERIODIC_AT_END { - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); int err = check_periodics(tstate); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (err != 0) { JUMP_TO_LABEL(error); } @@ -4139,8 +4326,9 @@ STAT_INC(CALL, hit); PyCFunction cfunc = method->d_method->ml_meth; _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyObject *res_o = _PyCFunction_TrampolineCall(cfunc, self, NULL); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); _Py_LeaveRecursiveCallTstate(tstate); assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); if (res_o == NULL) { @@ -4158,8 +4346,9 @@ stack_pointer += -oparg; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_XCLOSE(value); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } // _POP_TOP { @@ -4167,14 +4356,16 @@ stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_XCLOSE(value); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } // _CHECK_PERIODIC_AT_END { - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); int err = check_periodics(tstate); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (err != 0) { JUMP_TO_LABEL(error); } @@ -4258,8 +4449,9 @@ PyObject *self = PyStackRef_AsPyObjectBorrow(arguments[0]); PyObject *arg = PyStackRef_AsPyObjectBorrow(arguments[1]); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyObject *res_o = _PyCFunction_TrampolineCall(cfunc, self, arg); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); _Py_LeaveRecursiveCallTstate(tstate); assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); if (res_o == NULL) { @@ -4279,8 +4471,9 @@ stack_pointer += 1 - oparg; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_XCLOSE(value); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } // _POP_TOP { @@ -4288,8 +4481,9 @@ stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_XCLOSE(value); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } // _POP_TOP { @@ -4297,14 +4491,16 @@ stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_XCLOSE(value); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } // _CHECK_PERIODIC_AT_END { - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); int err = check_periodics(tstate); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (err != 0) { JUMP_TO_LABEL(error); } @@ -4359,12 +4555,13 @@ total_args++; } _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyObject *res_o = _Py_VectorCall_StackRefSteal( callable, arguments, total_args, PyStackRef_NULL); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (res_o == NULL) { stack_pointer += -2 - oparg; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); @@ -4378,8 +4575,9 @@ stack_pointer += -1 - oparg; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); int err = check_periodics(tstate); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (err != 0) { JUMP_TO_LABEL(error); } @@ -4489,12 +4687,17 @@ stack_pointer += -2 - oparg; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); assert(temp->previous == frame || temp->previous->previous == frame); CALL_STAT_INC(inlined_py_calls); frame = tstate->current_frame = temp; tstate->py_recursion_remaining--; - LOAD_SP(); + stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); LOAD_IP(0); + #ifdef Py_DEBUG + assert(frame->previous->stackpointer_valid == 1); + #endif DTRACE_FUNCTION_ENTRY(); LLTRACE_RESUME_FRAME(); } @@ -4564,11 +4767,12 @@ int code_flags = ((PyCodeObject*)PyFunction_GET_CODE(callable_o))->co_flags; PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable_o)); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); _PyInterpreterFrame *temp = _PyEvalFramePushAndInit( tstate, callable, locals, args, total_args, NULL, frame ); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); stack_pointer += -2 - oparg; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); if (temp == NULL) { @@ -4590,12 +4794,17 @@ assert(!IS_PEP523_HOOKED(tstate)); _PyInterpreterFrame *temp = PyStackRef_Unwrap(new_frame); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); assert(temp->previous == frame || temp->previous->previous == frame); CALL_STAT_INC(inlined_py_calls); frame = tstate->current_frame = temp; tstate->py_recursion_remaining--; - LOAD_SP(); + stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); LOAD_IP(0); + #ifdef Py_DEBUG + assert(frame->previous->stackpointer_valid == 1); + #endif DTRACE_FUNCTION_ENTRY(); LLTRACE_RESUME_FRAME(); } @@ -4647,8 +4856,9 @@ assert(oparg == 1); STAT_INC(CALL, hit); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyObject *res_o = PyObject_Str(arg_o); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (res_o == NULL) { JUMP_TO_LABEL(error); } @@ -4662,14 +4872,16 @@ stack_pointer += -2; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_XCLOSE(value); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } // _CHECK_PERIODIC_AT_END { - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); int err = check_periodics(tstate); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (err != 0) { JUMP_TO_LABEL(error); } @@ -4722,8 +4934,9 @@ assert(oparg == 1); STAT_INC(CALL, hit); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyObject *res_o = PySequence_Tuple(arg_o); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (res_o == NULL) { JUMP_TO_LABEL(error); } @@ -4737,14 +4950,16 @@ stack_pointer += -2; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_XCLOSE(value); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } // _CHECK_PERIODIC_AT_END { - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); int err = check_periodics(tstate); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (err != 0) { JUMP_TO_LABEL(error); } @@ -4806,8 +5021,9 @@ stack_pointer += -2; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_XCLOSE(value); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } DISPATCH(); } @@ -4829,37 +5045,49 @@ PyObject *exc_value = PyStackRef_AsPyObjectBorrow(exc_value_st); PyObject *match_type = PyStackRef_AsPyObjectBorrow(match_type_st); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); int err = _PyEval_CheckExceptStarTypeValid(tstate, match_type); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (err < 0) { - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); _PyStackRef tmp = match_type_st; match_type_st = PyStackRef_NULL; stack_pointer[-1] = match_type_st; PyStackRef_CLOSE(tmp); + _PyFrame_StackPointerInvalidate(frame); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); tmp = exc_value_st; exc_value_st = PyStackRef_NULL; stack_pointer[-2] = exc_value_st; PyStackRef_CLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); stack_pointer += -2; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); JUMP_TO_LABEL(error); } PyObject *match_o = NULL; PyObject *rest_o = NULL; - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); int res = _PyEval_ExceptionGroupMatch(frame, exc_value, match_type, &match_o, &rest_o); + _PyFrame_StackPointerInvalidate(frame); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); _PyStackRef tmp = match_type_st; match_type_st = PyStackRef_NULL; stack_pointer[-1] = match_type_st; PyStackRef_CLOSE(tmp); + _PyFrame_StackPointerInvalidate(frame); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); tmp = exc_value_st; exc_value_st = PyStackRef_NULL; stack_pointer[-2] = exc_value_st; PyStackRef_CLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); stack_pointer += -2; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); if (res < 0) { @@ -4871,8 +5099,9 @@ } if (!Py_IsNone(match_o)) { _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyErr_SetHandledException(match_o); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } rest = PyStackRef_FromPyObjectSteal(rest_o); match = PyStackRef_FromPyObjectSteal(match_o); @@ -4900,19 +5129,22 @@ PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); assert(PyExceptionInstance_Check(left_o)); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); int err = _PyEval_CheckExceptTypeValid(tstate, right_o); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (err < 0) { JUMP_TO_LABEL(error); } - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); int res = PyErr_GivenExceptionMatches(left_o, right_o); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(right); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); b = res ? PyStackRef_True : PyStackRef_False; stack_pointer[0] = b; stack_pointer += 1; @@ -4947,37 +5179,55 @@ #endif assert(exc_value && PyExceptionInstance_Check(exc_value)); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); int matches = PyErr_GivenExceptionMatches(exc_value, PyExc_StopIteration); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (matches) { value = PyStackRef_FromPyObjectNew(((PyStopIterationObject *)exc_value)->value); - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); _PyStackRef tmp = sub_iter; sub_iter = value; stack_pointer[-4] = sub_iter; PyStackRef_CLOSE(tmp); + _PyFrame_StackPointerInvalidate(frame); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); tmp = exc_value_st; exc_value_st = PyStackRef_NULL; stack_pointer[-1] = exc_value_st; PyStackRef_CLOSE(tmp); + _PyFrame_StackPointerInvalidate(frame); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); tmp = last_sent_val; last_sent_val = PyStackRef_NULL; stack_pointer[-2] = last_sent_val; PyStackRef_CLOSE(tmp); + _PyFrame_StackPointerInvalidate(frame); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); tmp = null_in; null_in = PyStackRef_NULL; stack_pointer[-3] = null_in; PyStackRef_XCLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); stack_pointer += -4; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); null_out = null_in; none = PyStackRef_None; } else { - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); _PyErr_SetRaisedException(tstate, Py_NewRef(exc_value)); + _PyFrame_StackPointerInvalidate(frame); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); monitor_reraise(tstate, frame, this_instr); + _PyFrame_StackPointerInvalidate(frame); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); JUMP_TO_LABEL(exception_unwind); } stack_pointer[0] = none; @@ -5012,8 +5262,9 @@ if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { next_instr = this_instr; _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); _Py_Specialize_CompareOp(left, right, next_instr, oparg); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); DISPATCH_SAME_OPARG(); } OPCODE_DEFERRED_INC(COMPARE_OP); @@ -5026,16 +5277,23 @@ PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); assert((oparg >> 5) <= Py_GE); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyObject *res_o = PyObject_RichCompare(left_o, right_o, oparg >> 5); + _PyFrame_StackPointerInvalidate(frame); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); _PyStackRef tmp = right; right = PyStackRef_NULL; stack_pointer[-1] = right; PyStackRef_CLOSE(tmp); + _PyFrame_StackPointerInvalidate(frame); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); tmp = left; left = PyStackRef_NULL; stack_pointer[-2] = left; PyStackRef_CLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); stack_pointer += -2; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); if (res_o == NULL) { @@ -5043,9 +5301,13 @@ } if (oparg & 16) { _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); int res_bool = PyObject_IsTrue(res_o); + _PyFrame_StackPointerInvalidate(frame); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); Py_DECREF(res_o); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (res_bool < 0) { JUMP_TO_LABEL(error); } @@ -5302,8 +5564,9 @@ if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { next_instr = this_instr; _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); _Py_Specialize_ContainsOp(right, next_instr); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); DISPATCH_SAME_OPARG(); } OPCODE_DEFERRED_INC(CONTAINS_OP); @@ -5316,8 +5579,9 @@ PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); int res = PySequence_Contains(right_o, left_o); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (res < 0) { JUMP_TO_LABEL(error); } @@ -5330,9 +5594,10 @@ value = r; stack_pointer[-2] = b; stack_pointer[-1] = l; - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); PyStackRef_XCLOSE(value); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } // _POP_TOP { @@ -5340,8 +5605,9 @@ stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_XCLOSE(value); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } DISPATCH(); } @@ -5384,8 +5650,9 @@ assert(PyAnyDict_CheckExact(right_o)); STAT_INC(CONTAINS_OP, hit); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); int res = PyDict_Contains(right_o, left_o); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (res < 0) { JUMP_TO_LABEL(error); } @@ -5398,9 +5665,10 @@ value = r; stack_pointer[-2] = b; stack_pointer[-1] = l; - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); PyStackRef_XCLOSE(value); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } // _POP_TOP { @@ -5408,8 +5676,9 @@ stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_XCLOSE(value); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } DISPATCH(); } @@ -5452,8 +5721,9 @@ assert(PyAnySet_CheckExact(right_o)); STAT_INC(CONTAINS_OP, hit); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); int res = _PySet_Contains((PySetObject *)right_o, left_o); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (res < 0) { JUMP_TO_LABEL(error); } @@ -5466,9 +5736,10 @@ value = r; stack_pointer[-2] = b; stack_pointer[-1] = l; - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); PyStackRef_XCLOSE(value); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } // _POP_TOP { @@ -5476,8 +5747,9 @@ stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_XCLOSE(value); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } DISPATCH(); } @@ -5497,13 +5769,15 @@ assert(oparg >= FVC_STR && oparg <= FVC_ASCII); conv_fn = _PyEval_ConversionFuncs[oparg]; _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyObject *result_o = conv_fn(PyStackRef_AsPyObjectBorrow(value)); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(value); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (result_o == NULL) { JUMP_TO_LABEL(error); } @@ -5565,13 +5839,15 @@ owner = stack_pointer[-1]; PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); int err = PyObject_DelAttr(PyStackRef_AsPyObjectBorrow(owner), name); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(owner); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (err) { JUMP_TO_LABEL(error); } @@ -5590,13 +5866,15 @@ PyObject *oldobj = PyCell_SwapTakeRef((PyCellObject *)cell, NULL); if (oldobj == NULL) { _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); _PyEval_FormatExcUnbound(tstate, _PyFrame_GetCode(frame), oparg); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); JUMP_TO_LABEL(error); } _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); Py_DECREF(oldobj); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); DISPATCH(); } @@ -5611,18 +5889,20 @@ _PyStackRef v = GETLOCAL(oparg); if (PyStackRef_IsNull(v)) { _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); _PyEval_FormatExcCheckArg(tstate, PyExc_UnboundLocalError, UNBOUNDLOCAL_ERROR_MSG, PyTuple_GetItem(_PyFrame_GetCode(frame)->co_localsplusnames, oparg) ); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); JUMP_TO_LABEL(error); } _PyStackRef tmp = GETLOCAL(oparg); GETLOCAL(oparg) = PyStackRef_NULL; _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_XCLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); DISPATCH(); } @@ -5636,16 +5916,18 @@ INSTRUCTION_STATS(DELETE_GLOBAL); PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); int err = PyDict_Pop(GLOBALS(), name, NULL); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (err < 0) { JUMP_TO_LABEL(error); } if (err == 0) { - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); _PyEval_FormatExcCheckArg(tstate, PyExc_NameError, NAME_ERROR_MSG, name); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); JUMP_TO_LABEL(error); } DISPATCH(); @@ -5664,20 +5946,23 @@ int err; if (ns == NULL) { _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); _PyErr_Format(tstate, PyExc_SystemError, "no locals when deleting %R", name); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); JUMP_TO_LABEL(error); } _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); err = PyObject_DelItem(ns, name); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (err != 0) { - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); _PyEval_FormatExcCheckArg(tstate, PyExc_NameError, NAME_ERROR_MSG, name); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); JUMP_TO_LABEL(error); } DISPATCH(); @@ -5696,17 +5981,24 @@ sub = stack_pointer[-1]; container = stack_pointer[-2]; _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); int err = PyObject_DelItem(PyStackRef_AsPyObjectBorrow(container), PyStackRef_AsPyObjectBorrow(sub)); + _PyFrame_StackPointerInvalidate(frame); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); _PyStackRef tmp = sub; sub = PyStackRef_NULL; stack_pointer[-1] = sub; PyStackRef_CLOSE(tmp); + _PyFrame_StackPointerInvalidate(frame); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); tmp = container; container = PyStackRef_NULL; stack_pointer[-2] = container; PyStackRef_CLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); stack_pointer += -2; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); if (err) { @@ -5738,13 +6030,18 @@ PyObject *update_o = PyStackRef_AsPyObjectBorrow(update); PyObject *dupkey = NULL; _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); int err = _PyDict_MergeUniq(dict_o, update_o, &dupkey); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (err < 0) { - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); _PyEval_FormatKwargsError(tstate, callable_o, update_o, dupkey); + _PyFrame_StackPointerInvalidate(frame); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); Py_XDECREF(dupkey); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); JUMP_TO_LABEL(error); } u = update; @@ -5755,8 +6052,9 @@ stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_XCLOSE(value); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } DISPATCH(); } @@ -5780,27 +6078,37 @@ PyObject *dict_o = PyStackRef_AsPyObjectBorrow(dict); PyObject *update_o = PyStackRef_AsPyObjectBorrow(update); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); int err = PyDict_Update(dict_o, update_o); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (err < 0) { int matches = _PyErr_ExceptionMatches(tstate, PyExc_AttributeError); if (matches) { - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); PyObject *exc = _PyErr_GetRaisedException(tstate); + _PyFrame_StackPointerInvalidate(frame); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); int has_keys = PyObject_HasAttrWithError(update_o, &_Py_ID(keys)); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (has_keys == 0) { - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); _PyErr_Format(tstate, PyExc_TypeError, "'%T' object is not a mapping", update_o); + _PyFrame_StackPointerInvalidate(frame); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); Py_DECREF(exc); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } else { - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); _PyErr_ChainExceptions1(exc); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } } JUMP_TO_LABEL(error); @@ -5813,8 +6121,9 @@ stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_XCLOSE(value); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } DISPATCH(); } @@ -5838,27 +6147,39 @@ PyObject *exc = PyStackRef_AsPyObjectBorrow(exc_st); assert(exc && PyExceptionInstance_Check(exc)); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); int matches = PyErr_GivenExceptionMatches(exc, PyExc_StopAsyncIteration); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (matches) { - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); _PyStackRef tmp = exc_st; exc_st = PyStackRef_NULL; stack_pointer[-1] = exc_st; PyStackRef_CLOSE(tmp); + _PyFrame_StackPointerInvalidate(frame); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); tmp = awaitable_st; awaitable_st = PyStackRef_NULL; stack_pointer[-2] = awaitable_st; PyStackRef_CLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); stack_pointer += -2; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); } else { Py_INCREF(exc); - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); _PyErr_SetRaisedException(tstate, exc); + _PyFrame_StackPointerInvalidate(frame); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); monitor_reraise(tstate, frame, this_instr); + _PyFrame_StackPointerInvalidate(frame); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); JUMP_TO_LABEL(exception_unwind); } DISPATCH(); @@ -5876,8 +6197,9 @@ stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(value); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); DISPATCH(); } @@ -5902,8 +6224,9 @@ stack_pointer += -2; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(receiver); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); DISPATCH(); } @@ -5969,10 +6292,11 @@ should_be_none = stack_pointer[-1]; if (!PyStackRef_IsNone(should_be_none)) { _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyErr_Format(PyExc_TypeError, "__init__() should return None, not '%.200s'", Py_TYPE(PyStackRef_AsPyObjectBorrow(should_be_none))->tp_name); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); JUMP_TO_LABEL(error); } stack_pointer += -1; @@ -6010,13 +6334,15 @@ PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); if (!PyUnicode_CheckExact(value_o)) { _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyObject *res_o = PyObject_Format(value_o, NULL); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(value); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (res_o == NULL) { JUMP_TO_LABEL(error); } @@ -6046,16 +6372,23 @@ fmt_spec = stack_pointer[-1]; value = stack_pointer[-2]; _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyObject *res_o = PyObject_Format(PyStackRef_AsPyObjectBorrow(value), PyStackRef_AsPyObjectBorrow(fmt_spec)); + _PyFrame_StackPointerInvalidate(frame); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); _PyStackRef tmp = fmt_spec; fmt_spec = PyStackRef_NULL; stack_pointer[-1] = fmt_spec; PyStackRef_CLOSE(tmp); + _PyFrame_StackPointerInvalidate(frame); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); tmp = value; value = PyStackRef_NULL; stack_pointer[-2] = value; PyStackRef_CLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); stack_pointer += -2; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); if (res_o == NULL) { @@ -6092,8 +6425,9 @@ if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { next_instr = this_instr; _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); _Py_Specialize_ForIter(iter, null_or_index, next_instr, oparg); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); DISPATCH_SAME_OPARG(); } OPCODE_DEFERRED_INC(FOR_ITER); @@ -6103,8 +6437,9 @@ // _FOR_ITER { _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); _PyStackRef item = _PyForIter_VirtualIteratorNext(tstate, frame, iter, &null_or_index); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (!PyStackRef_IsValid(item)) { if (PyStackRef_IsError(item)) { JUMP_TO_LABEL(error); @@ -6174,12 +6509,17 @@ assert(!IS_PEP523_HOOKED(tstate)); _PyInterpreterFrame *temp = PyStackRef_Unwrap(new_frame); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); assert(temp->previous == frame || temp->previous->previous == frame); CALL_STAT_INC(inlined_py_calls); frame = tstate->current_frame = temp; tstate->py_recursion_remaining--; - LOAD_SP(); + stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); LOAD_IP(0); + #ifdef Py_DEBUG + assert(frame->previous->stackpointer_valid == 1); + #endif DTRACE_FUNCTION_ENTRY(); LLTRACE_RESUME_FRAME(); } @@ -6245,8 +6585,9 @@ _PyObject_GC_IS_SHARED(list_o)); STAT_INC(FOR_ITER, hit); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); int result = _PyList_GetItemRefNoLock((PyListObject *)list_o, PyStackRef_UntagInt(null_or_index), &next); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (result < 0) { UPDATE_MISS_STATS(FOR_ITER); assert(_PyOpcode_Deopt[opcode] == (FOR_ITER)); @@ -6423,8 +6764,9 @@ PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter); Py_ssize_t index = PyStackRef_UntagInt(null_or_index); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); _PyObjectIndexPair next_index = Py_TYPE(iter_o)->_tp_iteritem(iter_o, index); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); PyObject *next_o = next_index.object; index = next_index.index; if (next_o == NULL) { @@ -6464,38 +6806,46 @@ } if (getter == NULL) { _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); _PyErr_Format(tstate, PyExc_TypeError, "'async for' requires an object with " "__aiter__ method, got %.100s", type->tp_name); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(obj); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); JUMP_TO_LABEL(error); } _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); iter_o = (*getter)(obj_o); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(obj); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (iter_o == NULL) { JUMP_TO_LABEL(error); } if (Py_TYPE(iter_o)->tp_as_async == NULL || Py_TYPE(iter_o)->tp_as_async->am_anext == NULL) { - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); _PyErr_Format(tstate, PyExc_TypeError, "'async for' received an object from __aiter__ " "that does not implement __anext__: %.100s", Py_TYPE(iter_o)->tp_name); + _PyFrame_StackPointerInvalidate(frame); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); Py_DECREF(iter_o); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); JUMP_TO_LABEL(error); } iter = PyStackRef_FromPyObjectSteal(iter_o); @@ -6517,8 +6867,9 @@ _PyStackRef awaitable; aiter = stack_pointer[-1]; _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyObject *awaitable_o = _PyEval_GetANext(PyStackRef_AsPyObjectBorrow(aiter)); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (awaitable_o == NULL) { JUMP_TO_LABEL(error); } @@ -6541,13 +6892,15 @@ _PyStackRef iter; iterable = stack_pointer[-1]; _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyObject *iter_o = _PyEval_GetAwaitable(PyStackRef_AsPyObjectBorrow(iterable), oparg); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(iterable); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (iter_o == NULL) { JUMP_TO_LABEL(error); } @@ -6581,8 +6934,9 @@ if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { next_instr = this_instr; _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); _Py_Specialize_GetIter(iterable, next_instr); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); DISPATCH_SAME_OPARG(); } OPCODE_DEFERRED_INC(GET_ITER); @@ -6592,8 +6946,9 @@ // _GET_ITER { _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); _PyStackRef result = _PyEval_GetIter(iterable, &index_or_null, oparg); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (PyStackRef_IsError(result)) { JUMP_TO_LABEL(pop_1_error); } @@ -6688,8 +7043,9 @@ _PyStackRef len; obj = stack_pointer[-1]; _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); Py_ssize_t len_i = PyObject_Length(PyStackRef_AsPyObjectBorrow(obj)); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (len_i < 0) { JUMP_TO_LABEL(error); } @@ -6719,15 +7075,17 @@ PyObject *res_o; if (PyLazyImport_CheckExact(PyStackRef_AsPyObjectBorrow(from))) { _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); res_o = _PyEval_LazyImportFrom( tstate, frame, PyStackRef_AsPyObjectBorrow(from), name); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } else { _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); res_o = _PyEval_ImportFrom( tstate, PyStackRef_AsPyObjectBorrow(from), name); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } if (res_o == NULL) { JUMP_TO_LABEL(error); @@ -6756,31 +7114,37 @@ PyObject *res_o; if (!(oparg & 0x02)) { _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); res_o = _PyEval_LazyImportName(tstate, BUILTINS(), GLOBALS(), LOCALS(), name, PyStackRef_AsPyObjectBorrow(fromlist), PyStackRef_AsPyObjectBorrow(level), oparg & 0x01); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } else { _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); res_o = _PyEval_ImportName(tstate, BUILTINS(), GLOBALS(), LOCALS(), name, PyStackRef_AsPyObjectBorrow(fromlist), PyStackRef_AsPyObjectBorrow(level)); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); _PyStackRef tmp = fromlist; fromlist = PyStackRef_NULL; stack_pointer[-1] = fromlist; PyStackRef_CLOSE(tmp); + _PyFrame_StackPointerInvalidate(frame); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); tmp = level; level = PyStackRef_NULL; stack_pointer[-2] = level; PyStackRef_CLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); stack_pointer += -2; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); if (res_o == NULL) { @@ -6825,8 +7189,9 @@ stack_pointer[-2 - oparg] = callable; stack_pointer[-1 - oparg] = self_or_null; _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(temp); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } } // _MONITOR_CALL @@ -6849,11 +7214,12 @@ stack_pointer[-2 - oparg] = func; stack_pointer[-1 - oparg] = maybe_self; _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); int err = _Py_call_instrumentation_2args( tstate, PY_MONITORING_EVENT_CALL, frame, this_instr, function, arg0 ); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (err) { JUMP_TO_LABEL(error); } @@ -6876,12 +7242,13 @@ { int code_flags = ((PyCodeObject*)PyFunction_GET_CODE(callable_o))->co_flags; PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable_o)); - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); _PyInterpreterFrame *new_frame = _PyEvalFramePushAndInit( tstate, callable, locals, arguments, total_args, NULL, frame ); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); stack_pointer += -2 - oparg; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); if (new_frame == NULL) { @@ -6890,7 +7257,8 @@ frame->return_offset = 4u ; DISPATCH_INLINED(new_frame); } - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); PyObject* res_o = _Py_VectorCallInstrumentation_StackRefSteal( callable, arguments, @@ -6900,7 +7268,7 @@ frame, this_instr, tstate); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (res_o == NULL) { stack_pointer += -2 - oparg; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); @@ -6914,8 +7282,9 @@ stack_pointer += -1 - oparg; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); int err = check_periodics(tstate); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (err != 0) { JUMP_TO_LABEL(error); } @@ -6949,23 +7318,26 @@ PyObject *callargs_o = PyStackRef_AsPyObjectBorrow(callargs); if (!PyTuple_CheckExact(callargs_o)) { _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); int err = _Py_Check_ArgsIterable(tstate, PyStackRef_AsPyObjectBorrow(func), callargs_o); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (err < 0) { JUMP_TO_LABEL(error); } - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); PyObject *tuple_o = PySequence_Tuple(callargs_o); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (tuple_o == NULL) { JUMP_TO_LABEL(error); } _PyStackRef temp = callargs; callargs = PyStackRef_FromPyObjectSteal(tuple_o); stack_pointer[-2] = callargs; - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(temp); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } } // _DO_CALL_FUNCTION_EX @@ -6988,34 +7360,39 @@ PyTuple_GET_ITEM(callargs, 0) : &_PyInstrumentation_MISSING; stack_pointer[-2] = callargs_st; _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); int err = _Py_call_instrumentation_2args( tstate, PY_MONITORING_EVENT_CALL, frame, this_instr, func, arg); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (err) { JUMP_TO_LABEL(error); } - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); result_o = PyObject_Call(func, callargs, kwargs); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (!PyFunction_Check(func) && !PyMethod_Check(func)) { if (result_o == NULL) { - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); _Py_call_instrumentation_exc2( tstate, PY_MONITORING_EVENT_C_RAISE, frame, this_instr, func, arg); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } else { - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); int err = _Py_call_instrumentation_2args( tstate, PY_MONITORING_EVENT_C_RETURN, frame, this_instr, func, arg); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (err < 0) { - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); Py_CLEAR(result_o); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } } } @@ -7034,10 +7411,11 @@ stack_pointer += -2; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); _PyInterpreterFrame *new_frame = _PyEvalFramePushAndInit_Ex( tstate, func_st, locals, nargs, callargs, kwargs, frame); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); stack_pointer += -2; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); if (new_frame == NULL) { @@ -7053,24 +7431,28 @@ assert(kwargs == NULL || PyDict_CheckExact(kwargs)); stack_pointer[-2] = callargs_st; _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); result_o = PyObject_Call(func, callargs, kwargs); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_XCLOSE(kwargs_st); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(callargs_st); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); stack_pointer += -2; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(func_st); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (result_o == NULL) { JUMP_TO_LABEL(error); } @@ -7082,8 +7464,9 @@ stack_pointer += 1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); int err = check_periodics(tstate); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (err != 0) { JUMP_TO_LABEL(error); } @@ -7123,8 +7506,9 @@ stack_pointer[-3 - oparg] = callable; stack_pointer[-2 - oparg] = self_or_null; _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(temp); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } } // _MONITOR_CALL_KW @@ -7145,10 +7529,11 @@ stack_pointer[-3 - oparg] = callable; stack_pointer[-2 - oparg] = self_or_null; _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); int err = _Py_call_instrumentation_2args( tstate, PY_MONITORING_EVENT_CALL, frame, this_instr, function, arg); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (err) { JUMP_TO_LABEL(error); } @@ -7172,17 +7557,19 @@ { int code_flags = ((PyCodeObject*)PyFunction_GET_CODE(callable_o))->co_flags; PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable_o)); - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); _PyInterpreterFrame *new_frame = _PyEvalFramePushAndInit( tstate, callable, locals, arguments, positional_args, kwnames_o, frame ); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); stack_pointer += -3 - oparg; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(kwnames); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (new_frame == NULL) { JUMP_TO_LABEL(error); } @@ -7190,7 +7577,8 @@ frame->return_offset = 4u ; DISPATCH_INLINED(new_frame); } - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); PyObject* res_o = _Py_VectorCallInstrumentation_StackRefSteal( callable, arguments, @@ -7200,7 +7588,7 @@ frame, this_instr, tstate); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (res_o == NULL) { stack_pointer += -3 - oparg; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); @@ -7240,27 +7628,39 @@ PyObject *exc = PyStackRef_AsPyObjectBorrow(exc_st); assert(exc && PyExceptionInstance_Check(exc)); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); int matches = PyErr_GivenExceptionMatches(exc, PyExc_StopAsyncIteration); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (matches) { - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); _PyStackRef tmp = exc_st; exc_st = PyStackRef_NULL; stack_pointer[-1] = exc_st; PyStackRef_CLOSE(tmp); + _PyFrame_StackPointerInvalidate(frame); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); tmp = awaitable_st; awaitable_st = PyStackRef_NULL; stack_pointer[-2] = awaitable_st; PyStackRef_CLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); stack_pointer += -2; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); } else { Py_INCREF(exc); - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); _PyErr_SetRaisedException(tstate, exc); + _PyFrame_StackPointerInvalidate(frame); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); monitor_reraise(tstate, frame, this_instr); + _PyFrame_StackPointerInvalidate(frame); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); JUMP_TO_LABEL(exception_unwind); } } @@ -7282,8 +7682,9 @@ receiver = stack_pointer[-3]; if (PyStackRef_GenCheck(receiver)) { _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); int err = monitor_stop_iteration(tstate, frame, this_instr, PyStackRef_AsPyObjectBorrow(value)); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (err) { JUMP_TO_LABEL(error); } @@ -7291,8 +7692,9 @@ stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(value); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); DISPATCH(); } @@ -7316,8 +7718,9 @@ PyObject *receiver_o = PyStackRef_AsPyObjectBorrow(receiver); if (PyGen_Check(receiver_o) || PyCoro_CheckExact(receiver_o)) { _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); int err = monitor_stop_iteration(tstate, frame, this_instr, PyStackRef_AsPyObjectBorrow(value)); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (err) { JUMP_TO_LABEL(error); } @@ -7328,8 +7731,9 @@ stack_pointer += -2; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(receiver); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); DISPATCH(); } @@ -7350,8 +7754,9 @@ null_or_index = stack_pointer[-1]; iter = stack_pointer[-2]; _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); _PyStackRef item = _PyForIter_VirtualIteratorNext(tstate, frame, iter, &null_or_index); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (!PyStackRef_IsValid(item)) { if (PyStackRef_IsError(item)) { JUMP_TO_LABEL(error); @@ -7381,9 +7786,10 @@ INSTRUCTION_STATS(INSTRUMENTED_INSTRUCTION); opcode = INSTRUMENTED_INSTRUCTION; _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); int next_opcode = _Py_call_instrumentation_instruction( tstate, frame, this_instr); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (next_opcode < 0) { JUMP_TO_LABEL(error); } @@ -7410,8 +7816,9 @@ // _CHECK_PERIODIC { _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); int err = check_periodics(tstate); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (err != 0) { JUMP_TO_LABEL(error); } @@ -7457,9 +7864,13 @@ next_instr = this_instr; } else { _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); original_opcode = _Py_call_instrumentation_line( tstate, frame, this_instr, prev_instr); + _PyFrame_StackPointerInvalidate(frame); + // Explicit stack reload stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackAssertInvalid(frame); if (original_opcode < 0) { next_instr = this_instr+1; JUMP_TO_LABEL(error); @@ -7474,6 +7885,7 @@ PAUSE_ADAPTIVE_COUNTER(cache->counter); } opcode = original_opcode; + PRE_DISPATCH_GOTO(); DISPATCH_GOTO(); } @@ -7505,25 +7917,33 @@ if (opcode == INSTRUMENTED_LOAD_SUPER_ATTR) { PyObject *arg = oparg & 2 ? class : &_PyInstrumentation_MISSING; _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); int err = _Py_call_instrumentation_2args( tstate, PY_MONITORING_EVENT_CALL, frame, this_instr, global_super, arg); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (err) { - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); _PyStackRef tmp = self_st; self_st = PyStackRef_NULL; stack_pointer[-1] = self_st; PyStackRef_CLOSE(tmp); + _PyFrame_StackPointerInvalidate(frame); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); tmp = class_st; class_st = PyStackRef_NULL; stack_pointer[-2] = class_st; PyStackRef_CLOSE(tmp); + _PyFrame_StackPointerInvalidate(frame); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); tmp = global_super_st; global_super_st = PyStackRef_NULL; stack_pointer[-3] = global_super_st; PyStackRef_CLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); stack_pointer += -3; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); JUMP_TO_LABEL(error); @@ -7533,45 +7953,56 @@ { PyObject *stack[] = {class, self}; _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); super = PyObject_Vectorcall(global_super, stack, oparg & 2, NULL); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } if (opcode == INSTRUMENTED_LOAD_SUPER_ATTR) { PyObject *arg = oparg & 2 ? class : &_PyInstrumentation_MISSING; if (super == NULL) { - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); _Py_call_instrumentation_exc2( tstate, PY_MONITORING_EVENT_C_RAISE, frame, this_instr, global_super, arg); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } else { - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); int err = _Py_call_instrumentation_2args( tstate, PY_MONITORING_EVENT_C_RETURN, frame, this_instr, global_super, arg); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (err < 0) { - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); Py_CLEAR(super); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } } } - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); _PyStackRef tmp = self_st; self_st = PyStackRef_NULL; stack_pointer[-1] = self_st; PyStackRef_CLOSE(tmp); + _PyFrame_StackPointerInvalidate(frame); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); tmp = class_st; class_st = PyStackRef_NULL; stack_pointer[-2] = class_st; PyStackRef_CLOSE(tmp); + _PyFrame_StackPointerInvalidate(frame); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); tmp = global_super_st; global_super_st = PyStackRef_NULL; stack_pointer[-3] = global_super_st; PyStackRef_CLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); stack_pointer += -3; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); if (super == NULL) { @@ -7579,9 +8010,13 @@ } PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 2); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyObject *attr_o = PyObject_GetAttr(super, name); + _PyFrame_StackPointerInvalidate(frame); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); Py_DECREF(super); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (attr_o == NULL) { JUMP_TO_LABEL(error); } @@ -7636,8 +8071,9 @@ stack_pointer += -2; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(iter); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); DISPATCH(); } @@ -7687,8 +8123,9 @@ stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(value); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); stack_pointer += 1; } stack_pointer += -1; @@ -7715,8 +8152,9 @@ stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(value); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); INSTRUMENTED_JUMP(this_instr, next_instr + oparg, PY_MONITORING_EVENT_BRANCH_RIGHT); } else { @@ -7766,9 +8204,10 @@ if (frame->tlbc_index != ((_PyThreadStateImpl *)tstate)->tlbc_index) { _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); _Py_CODEUNIT *bytecode = _PyEval_GetExecutableCode(tstate, _PyFrame_GetCode(frame)); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (bytecode == NULL) { JUMP_TO_LABEL(error); } @@ -7793,8 +8232,9 @@ uintptr_t code_version = FT_ATOMIC_LOAD_UINTPTR_ACQUIRE(_PyFrame_GetCode(frame)->_co_instrumentation_version); if (code_version != global_version) { _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); int err = _Py_Instrument(_PyFrame_GetCode(frame), tstate->interp); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (err) { JUMP_TO_LABEL(error); } @@ -7807,8 +8247,9 @@ { if ((oparg & RESUME_OPARG_LOCATION_MASK) < RESUME_AFTER_YIELD_FROM) { _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); int err = check_periodics(tstate); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (err != 0) { JUMP_TO_LABEL(error); } @@ -7817,9 +8258,13 @@ // _MONITOR_RESUME { _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); int err = _Py_call_instrumentation( tstate, oparg == 0 ? PY_MONITORING_EVENT_PY_START : PY_MONITORING_EVENT_PY_RESUME, frame, this_instr); + _PyFrame_StackPointerInvalidate(frame); + // Explicit stack reload stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackAssertInvalid(frame); if (err) { JUMP_TO_LABEL(error); } @@ -7848,10 +8293,11 @@ { val = stack_pointer[-1]; _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); int err = _Py_call_instrumentation_arg( tstate, PY_MONITORING_EVENT_PY_RETURN, frame, this_instr, PyStackRef_AsPyObjectBorrow(val)); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (err) { JUMP_TO_LABEL(error); } @@ -7869,6 +8315,7 @@ stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); assert(STACK_LEVEL() == 0); DTRACE_FUNCTION_RETURN(); _Py_LeaveRecursiveCallPy(tstate); @@ -7876,6 +8323,7 @@ frame = tstate->current_frame = dying->previous; _PyEval_FrameClearAndPop(tstate, dying); stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); LOAD_IP(frame->return_offset); res = temp; LLTRACE_RESUME_FRAME(); @@ -7904,14 +8352,21 @@ { val = stack_pointer[-1]; _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); int err = _Py_call_instrumentation_arg( tstate, PY_MONITORING_EVENT_PY_YIELD, frame, this_instr, PyStackRef_AsPyObjectBorrow(val)); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (err) { + // Explicit stack reload + stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackAssertInvalid(frame); JUMP_TO_LABEL(error); } if (frame->instr_ptr != this_instr) { + // Explicit stack reload + stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackAssertInvalid(frame); next_instr = frame->instr_ptr; DISPATCH(); } @@ -7933,6 +8388,7 @@ stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); DTRACE_FUNCTION_RETURN(); tstate->exc_info = gen->gi_exc_state.previous_item; gen->gi_exc_state.previous_item = NULL; @@ -7953,6 +8409,7 @@ } #endif stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); LOAD_IP(1 + INLINE_CACHE_ENTRIES_SEND); value = temp; LLTRACE_RESUME_FRAME(); @@ -7985,12 +8442,14 @@ _PyStackRef executor = frame->localsplus[0]; assert(tstate->current_executor == NULL); if (!PyStackRef_IsNull(executor)) { + assert(PyStackRef_TYPE(executor) == &_PyUOpExecutor_Type); tstate->current_executor = PyStackRef_AsPyObjectBorrow(executor); stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(executor); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); stack_pointer += 1; } #endif @@ -8027,8 +8486,9 @@ stack_pointer[-2] = b; stack_pointer[-1] = l; _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_XCLOSE(value); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } // _POP_TOP { @@ -8036,8 +8496,9 @@ stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_XCLOSE(value); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } DISPATCH(); } @@ -8068,8 +8529,9 @@ // _CHECK_PERIODIC { _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); int err = check_periodics(tstate); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (err != 0) { JUMP_TO_LABEL(error); } @@ -8097,8 +8559,9 @@ // _CHECK_PERIODIC { _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); int err = check_periodics(tstate); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (err != 0) { JUMP_TO_LABEL(error); } @@ -8165,8 +8628,9 @@ // _CHECK_PERIODIC { _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); int err = check_periodics(tstate); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (err != 0) { JUMP_TO_LABEL(error); } @@ -8232,19 +8696,24 @@ PyObject *list = PyStackRef_AsPyObjectBorrow(list_st); PyObject *iterable = PyStackRef_AsPyObjectBorrow(iterable_st); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyObject *none_val = _PyList_Extend((PyListObject *)list, iterable); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (none_val == NULL) { int matches = _PyErr_ExceptionMatches(tstate, PyExc_TypeError); if (matches && (Py_TYPE(iterable)->tp_iter == NULL && !PySequence_Check(iterable))) { - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); _PyErr_Clear(tstate); + _PyFrame_StackPointerInvalidate(frame); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); _PyErr_Format(tstate, PyExc_TypeError, "Value after * must be an iterable, not %.200s", Py_TYPE(iterable)->tp_name); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } JUMP_TO_LABEL(error); } @@ -8257,8 +8726,9 @@ stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_XCLOSE(value); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } DISPATCH(); } @@ -8287,8 +8757,9 @@ PyObject *name = GETITEM(FRAME_CO_NAMES, oparg>>1); next_instr = this_instr; _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); _Py_Specialize_LoadAttr(owner, next_instr, name); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); DISPATCH_SAME_OPARG(); } OPCODE_DEFERRED_INC(LOAD_ATTR); @@ -8302,22 +8773,25 @@ PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 1); if (oparg & 1) { _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); attr = _Py_LoadAttr_StackRefSteal(tstate, owner, name, self_or_null); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (PyStackRef_IsNull(attr)) { JUMP_TO_LABEL(pop_1_error); } } else { _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); attr = _PyObject_GetAttrStackRef(PyStackRef_AsPyObjectBorrow(owner), name); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); stack_pointer[-1] = attr; stack_pointer += (oparg&1); ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(owner); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (PyStackRef_IsNull(attr)) { JUMP_TO_LABEL(error); } @@ -8370,11 +8844,12 @@ assert(descr != NULL); attr = PyStackRef_FromPyObjectNew(descr); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); _PyStackRef tmp = owner; owner = attr; stack_pointer[-1] = owner; PyStackRef_CLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } // _PUSH_NULL_CONDITIONAL { @@ -8438,11 +8913,12 @@ assert(descr != NULL); attr = PyStackRef_FromPyObjectNew(descr); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); _PyStackRef tmp = owner; owner = attr; stack_pointer[-1] = owner; PyStackRef_CLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } // _PUSH_NULL_CONDITIONAL { @@ -8534,12 +9010,17 @@ stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); assert(temp->previous == frame || temp->previous->previous == frame); CALL_STAT_INC(inlined_py_calls); frame = tstate->current_frame = temp; tstate->py_recursion_remaining--; - LOAD_SP(); + stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); LOAD_IP(0); + #ifdef Py_DEBUG + assert(frame->previous->stackpointer_valid == 1); + #endif DTRACE_FUNCTION_ENTRY(); LLTRACE_RESUME_FRAME(); } @@ -8617,8 +9098,9 @@ value = o; stack_pointer[-1] = attr; _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_XCLOSE(value); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } /* Skip 5 cache entries */ // _PUSH_NULL_CONDITIONAL @@ -8856,8 +9338,9 @@ value = o; stack_pointer[-1] = attr; _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_XCLOSE(value); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } /* Skip 5 cache entries */ // _PUSH_NULL_CONDITIONAL @@ -8909,8 +9392,9 @@ stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(owner); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); attr = PyStackRef_FromPyObjectNew(descr); } stack_pointer[0] = attr; @@ -8966,8 +9450,9 @@ stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(owner); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); attr = PyStackRef_FromPyObjectNew(descr); } stack_pointer[0] = attr; @@ -9049,12 +9534,17 @@ stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); assert(temp->previous == frame || temp->previous->previous == frame); CALL_STAT_INC(inlined_py_calls); frame = tstate->current_frame = temp; tstate->py_recursion_remaining--; - LOAD_SP(); + stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); LOAD_IP(0); + #ifdef Py_DEBUG + assert(frame->previous->stackpointer_valid == 1); + #endif DTRACE_FUNCTION_ENTRY(); LLTRACE_RESUME_FRAME(); } @@ -9119,8 +9609,9 @@ value = o; stack_pointer[-1] = attr; _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_XCLOSE(value); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } /* Skip 5 cache entries */ // _PUSH_NULL_CONDITIONAL @@ -9236,8 +9727,9 @@ value = o; stack_pointer[-1] = attr; _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_XCLOSE(value); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } /* Skip 5 cache entries */ // _PUSH_NULL_CONDITIONAL @@ -9263,16 +9755,18 @@ _PyStackRef bc; int err; _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyObject *bc_o = _PyMapping_GetOptionalItem2(BUILTINS(), &_Py_ID(__build_class__), &err); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (err < 0) { JUMP_TO_LABEL(error); } if (bc_o == NULL) { - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); _PyErr_SetString(tstate, PyExc_NameError, "__build_class__ not found"); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); JUMP_TO_LABEL(error); } bc = PyStackRef_FromPyObjectSteal(bc_o); @@ -9327,15 +9821,17 @@ _PyStackRef value; PyCellObject *cell = (PyCellObject *)PyStackRef_AsPyObjectBorrow(GETLOCAL(oparg)); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); value = _PyCell_GetStackRef(cell); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (PyStackRef_IsNull(value)) { stack_pointer[0] = value; stack_pointer += 1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); _PyEval_FormatExcUnbound(tstate, _PyFrame_GetCode(frame), oparg); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); JUMP_TO_LABEL(error); } stack_pointer[0] = value; @@ -9428,11 +9924,12 @@ _PyStackRef value_s = GETLOCAL(oparg); if (PyStackRef_IsNull(value_s)) { _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); _PyEval_FormatExcCheckArg(tstate, PyExc_UnboundLocalError, UNBOUNDLOCAL_ERROR_MSG, PyTuple_GetItem(_PyFrame_GetCode(frame)->co_localsplusnames, oparg) ); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); JUMP_TO_LABEL(error); } value = PyStackRef_DUP(value_s); @@ -9481,8 +9978,9 @@ name = PyTuple_GET_ITEM(_PyFrame_GetCode(frame)->co_localsplusnames, oparg); int err; _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyObject* value_o = _PyMapping_GetOptionalItem2(class_dict, name, &err); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (err < 0) { JUMP_TO_LABEL(error); } @@ -9490,17 +9988,19 @@ PyCellObject *cell = (PyCellObject *)PyStackRef_AsPyObjectBorrow(GETLOCAL(oparg)); value_o = PyCell_GetRef(cell); if (value_o == NULL) { - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); _PyEval_FormatExcUnbound(tstate, _PyFrame_GetCode(frame), oparg); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); JUMP_TO_LABEL(error); } } stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(class_dict_st); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); value = PyStackRef_FromPyObjectSteal(value_o); stack_pointer[0] = value; stack_pointer += 1; @@ -9522,13 +10022,15 @@ PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); int err; _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyObject *v_o = _PyMapping_GetOptionalItem2(PyStackRef_AsPyObjectBorrow(mod_or_class_dict), name, &err); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(mod_or_class_dict); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (err < 0) { JUMP_TO_LABEL(error); } @@ -9536,58 +10038,71 @@ if (PyDict_CheckExact(GLOBALS()) && PyDict_CheckExact(BUILTINS())) { - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); v_o = _PyDict_LoadGlobal((PyDictObject *)GLOBALS(), (PyDictObject *)BUILTINS(), name); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (v_o == NULL) { if (!_PyErr_Occurred(tstate)) { - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); _PyEval_FormatExcCheckArg(tstate, PyExc_NameError, NAME_ERROR_MSG, name); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } JUMP_TO_LABEL(error); } if (PyLazyImport_CheckExact(v_o)) { - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); PyObject *l_v = _PyImport_LoadLazyImportTstate(tstate, v_o); + _PyFrame_StackPointerInvalidate(frame); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); Py_SETREF(v_o, l_v); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (v_o == NULL) { JUMP_TO_LABEL(error); } } } else { - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); v_o = _PyMapping_GetOptionalItem2(GLOBALS(), name, &err); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (err < 0) { JUMP_TO_LABEL(error); } if (v_o == NULL) { - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); v_o = _PyMapping_GetOptionalItem2(BUILTINS(), name, &err); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (err < 0) { JUMP_TO_LABEL(error); } if (v_o == NULL) { - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); _PyEval_FormatExcCheckArg( tstate, PyExc_NameError, NAME_ERROR_MSG, name); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); JUMP_TO_LABEL(error); } } if (PyLazyImport_CheckExact(v_o)) { - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); PyObject *l_v = _PyImport_LoadLazyImportTstate(tstate, v_o); + _PyFrame_StackPointerInvalidate(frame); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); Py_SETREF(v_o, l_v); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (v_o == NULL) { JUMP_TO_LABEL(error); } @@ -9623,8 +10138,9 @@ PyObject *name = GETITEM(FRAME_CO_NAMES, oparg>>1); next_instr = this_instr; _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); _Py_Specialize_LoadGlobal(GLOBALS(), BUILTINS(), next_instr, name); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); DISPATCH_SAME_OPARG(); } OPCODE_DEFERRED_INC(LOAD_GLOBAL); @@ -9639,8 +10155,9 @@ res = &stack_pointer[0]; PyObject *name = GETITEM(FRAME_CO_NAMES, oparg>>1); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); _PyEval_LoadGlobalStackRef(GLOBALS(), BUILTINS(), name, res); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (PyStackRef_IsNull(*res)) { JUMP_TO_LABEL(error); } @@ -9816,9 +10333,10 @@ PyObject *l = LOCALS(); if (l == NULL) { _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); _PyErr_SetString(tstate, PyExc_SystemError, "no locals found"); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); JUMP_TO_LABEL(error); } locals = PyStackRef_FromPyObjectNew(l); @@ -9839,34 +10357,43 @@ _PyStackRef v; PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyObject *v_o = _PyEval_LoadName(tstate, frame, name); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (v_o == NULL) { JUMP_TO_LABEL(error); } if (PyLazyImport_CheckExact(v_o)) { - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); PyObject *l_v = _PyImport_LoadLazyImportTstate(tstate, v_o); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (l_v == NULL) { - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); Py_DECREF(v_o); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); JUMP_TO_LABEL(error); } - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); int err = PyDict_SetItem(GLOBALS(), name, l_v); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (err < 0) { - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); Py_DECREF(v_o); + _PyFrame_StackPointerInvalidate(frame); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); Py_DECREF(l_v); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); JUMP_TO_LABEL(error); } - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); Py_SETREF(v_o, l_v); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } v = PyStackRef_FromPyObjectSteal(v_o); stack_pointer[0] = v; @@ -9917,21 +10444,24 @@ stack_pointer += 1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); int err = _PyObject_LookupSpecialMethod(name, method_and_self); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (err <= 0) { if (err == 0) { PyObject *owner = PyStackRef_AsPyObjectBorrow(method_and_self[1]); - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); const char *errfmt = _PyEval_SpecialMethodCanSuggest(owner, oparg) ? _Py_SpecialMethods[oparg].error_suggestion : _Py_SpecialMethods[oparg].error; - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); assert(!_PyErr_Occurred(tstate)); assert(errfmt != NULL); - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); _PyErr_Format(tstate, PyExc_TypeError, errfmt, owner); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } JUMP_TO_LABEL(error); } @@ -9967,8 +10497,9 @@ if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { next_instr = this_instr; _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); _Py_Specialize_LoadSuperAttr(global_super_st, class_st, next_instr, load_method); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); DISPATCH_SAME_OPARG(); } OPCODE_DEFERRED_INC(LOAD_SUPER_ATTR); @@ -9984,25 +10515,33 @@ if (opcode == INSTRUMENTED_LOAD_SUPER_ATTR) { PyObject *arg = oparg & 2 ? class : &_PyInstrumentation_MISSING; _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); int err = _Py_call_instrumentation_2args( tstate, PY_MONITORING_EVENT_CALL, frame, this_instr, global_super, arg); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (err) { - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); _PyStackRef tmp = self_st; self_st = PyStackRef_NULL; stack_pointer[-1] = self_st; PyStackRef_CLOSE(tmp); + _PyFrame_StackPointerInvalidate(frame); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); tmp = class_st; class_st = PyStackRef_NULL; stack_pointer[-2] = class_st; PyStackRef_CLOSE(tmp); + _PyFrame_StackPointerInvalidate(frame); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); tmp = global_super_st; global_super_st = PyStackRef_NULL; stack_pointer[-3] = global_super_st; PyStackRef_CLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); stack_pointer += -3; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); JUMP_TO_LABEL(error); @@ -10012,45 +10551,56 @@ { PyObject *stack[] = {class, self}; _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); super = PyObject_Vectorcall(global_super, stack, oparg & 2, NULL); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } if (opcode == INSTRUMENTED_LOAD_SUPER_ATTR) { PyObject *arg = oparg & 2 ? class : &_PyInstrumentation_MISSING; if (super == NULL) { - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); _Py_call_instrumentation_exc2( tstate, PY_MONITORING_EVENT_C_RAISE, frame, this_instr, global_super, arg); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } else { - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); int err = _Py_call_instrumentation_2args( tstate, PY_MONITORING_EVENT_C_RETURN, frame, this_instr, global_super, arg); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (err < 0) { - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); Py_CLEAR(super); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } } } - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); _PyStackRef tmp = self_st; self_st = PyStackRef_NULL; stack_pointer[-1] = self_st; PyStackRef_CLOSE(tmp); + _PyFrame_StackPointerInvalidate(frame); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); tmp = class_st; class_st = PyStackRef_NULL; stack_pointer[-2] = class_st; PyStackRef_CLOSE(tmp); + _PyFrame_StackPointerInvalidate(frame); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); tmp = global_super_st; global_super_st = PyStackRef_NULL; stack_pointer[-3] = global_super_st; PyStackRef_CLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); stack_pointer += -3; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); if (super == NULL) { @@ -10058,9 +10608,13 @@ } PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 2); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyObject *attr_o = PyObject_GetAttr(super, name); + _PyFrame_StackPointerInvalidate(frame); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); Py_DECREF(super); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (attr_o == NULL) { JUMP_TO_LABEL(error); } @@ -10115,20 +10669,30 @@ STAT_INC(LOAD_SUPER_ATTR, hit); PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 2); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyObject *attr = _PySuper_Lookup((PyTypeObject *)class, self, name, NULL); + _PyFrame_StackPointerInvalidate(frame); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); _PyStackRef tmp = self_st; self_st = PyStackRef_NULL; stack_pointer[-1] = self_st; PyStackRef_CLOSE(tmp); + _PyFrame_StackPointerInvalidate(frame); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); tmp = class_st; class_st = PyStackRef_NULL; stack_pointer[-2] = class_st; PyStackRef_CLOSE(tmp); + _PyFrame_StackPointerInvalidate(frame); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); tmp = global_super_st; global_super_st = PyStackRef_NULL; stack_pointer[-3] = global_super_st; PyStackRef_CLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); stack_pointer += -3; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); if (attr == NULL) { @@ -10189,9 +10753,10 @@ { int *method_found_ptr = &method_found; _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); attr_o = _PySuper_Lookup(cls, self, name, Py_TYPE(self)->tp_getattro == PyObject_GenericGetAttr ? method_found_ptr : NULL); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } if (attr_o == NULL) { JUMP_TO_LABEL(error); @@ -10202,23 +10767,28 @@ stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(self_st); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); self_or_null = PyStackRef_NULL; stack_pointer += 1; } stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); _PyStackRef tmp = global_super_st; global_super_st = self_or_null; stack_pointer[-2] = global_super_st; PyStackRef_CLOSE(tmp); + _PyFrame_StackPointerInvalidate(frame); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); tmp = class_st; class_st = PyStackRef_NULL; stack_pointer[-1] = class_st; PyStackRef_CLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); stack_pointer += -2; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); attr = PyStackRef_FromPyObjectSteal(attr_o); @@ -10246,8 +10816,9 @@ _PyStackRef tmp = GETLOCAL(oparg); GETLOCAL(oparg) = PyStackRef_FromPyObjectSteal(cell); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_XCLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); DISPATCH(); } @@ -10268,9 +10839,10 @@ codeobj_st = stack_pointer[-1]; PyObject *codeobj = PyStackRef_AsPyObjectBorrow(codeobj_st); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyFunctionObject *func_obj = (PyFunctionObject *) PyFunction_New(codeobj, GLOBALS()); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (func_obj == NULL) { JUMP_TO_LABEL(error); } @@ -10283,9 +10855,10 @@ { value = co; stack_pointer[-1] = func; - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); PyStackRef_XCLOSE(value); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } DISPATCH(); } @@ -10307,12 +10880,13 @@ PyObject *dict = PyStackRef_AsPyObjectBorrow(dict_st); assert(PyDict_CheckExact(dict)); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); int err = _PyDict_SetItem_Take2( (PyDictObject *)dict, PyStackRef_AsPyObjectSteal(key), PyStackRef_AsPyObjectSteal(value) ); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (err != 0) { JUMP_TO_LABEL(pop_2_error); } @@ -10344,11 +10918,12 @@ subject = stack_pointer[-3]; assert(PyTuple_CheckExact(PyStackRef_AsPyObjectBorrow(names))); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyObject *attrs_o = _PyEval_MatchClass(tstate, PyStackRef_AsPyObjectBorrow(subject), PyStackRef_AsPyObjectBorrow(type), oparg, PyStackRef_AsPyObjectBorrow(names)); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (attrs_o) { assert(PyTuple_CheckExact(attrs_o)); attrs = PyStackRef_FromPyObjectSteal(attrs_o); @@ -10369,9 +10944,10 @@ stack_pointer[-3] = attrs; stack_pointer[-2] = s; stack_pointer[-1] = tp; - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); PyStackRef_XCLOSE(value); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } // _POP_TOP { @@ -10379,8 +10955,9 @@ stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_XCLOSE(value); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } // _POP_TOP { @@ -10388,8 +10965,9 @@ stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_XCLOSE(value); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } DISPATCH(); } @@ -10408,9 +10986,10 @@ keys = stack_pointer[-1]; subject = stack_pointer[-2]; _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyObject *values_or_none_o = _PyEval_MatchKeys(tstate, PyStackRef_AsPyObjectBorrow(subject), PyStackRef_AsPyObjectBorrow(keys)); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (values_or_none_o == NULL) { JUMP_TO_LABEL(error); } @@ -10493,10 +11072,11 @@ exc_value = stack_pointer[-1]; _PyErr_StackItem *exc_info = tstate->exc_info; _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); Py_XSETREF(exc_info->exc_value, PyStackRef_IsNone(exc_value) ? NULL : PyStackRef_AsPyObjectSteal(exc_value)); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); @@ -10518,8 +11098,9 @@ stack_pointer += -2; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(iter); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); DISPATCH(); } @@ -10568,11 +11149,12 @@ else { b = PyStackRef_False; _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); _PyStackRef tmp = value; value = b; stack_pointer[-1] = value; PyStackRef_CLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } } // _POP_JUMP_IF_TRUE @@ -10611,11 +11193,12 @@ else { b = PyStackRef_False; _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); _PyStackRef tmp = value; value = b; stack_pointer[-1] = value; PyStackRef_CLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } } // _POP_JUMP_IF_FALSE @@ -10666,8 +11249,9 @@ stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_XCLOSE(value); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); DISPATCH(); } @@ -10734,12 +11318,17 @@ stack_pointer += -oparg; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); int err = do_raise(tstate, exc, cause); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (err) { assert(oparg == 0); - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); monitor_reraise(tstate, frame, this_instr); + _PyFrame_StackPointerInvalidate(frame); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); JUMP_TO_LABEL(exception_unwind); } JUMP_TO_LABEL(error); @@ -10768,8 +11357,15 @@ stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); _PyErr_SetRaisedException(tstate, exc); + _PyFrame_StackPointerInvalidate(frame); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); monitor_reraise(tstate, frame, this_instr); + _PyFrame_StackPointerInvalidate(frame); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); JUMP_TO_LABEL(exception_unwind); } @@ -10803,9 +11399,10 @@ if (frame->tlbc_index != ((_PyThreadStateImpl *)tstate)->tlbc_index) { _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); _Py_CODEUNIT *bytecode = _PyEval_GetExecutableCode(tstate, _PyFrame_GetCode(frame)); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (bytecode == NULL) { JUMP_TO_LABEL(error); } @@ -10830,8 +11427,9 @@ uintptr_t code_version = FT_ATOMIC_LOAD_UINTPTR_ACQUIRE(_PyFrame_GetCode(frame)->_co_instrumentation_version); if (code_version != global_version) { _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); int err = _Py_Instrument(_PyFrame_GetCode(frame), tstate->interp); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (err) { JUMP_TO_LABEL(error); } @@ -10845,15 +11443,17 @@ uint16_t counter = read_u16(&this_instr[1].cache); (void)counter; _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); _Py_Specialize_Resume(this_instr, tstate, frame); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } // _CHECK_PERIODIC_IF_NOT_YIELD_FROM { if ((oparg & RESUME_OPARG_LOCATION_MASK) < RESUME_AFTER_YIELD_FROM) { - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); int err = check_periodics(tstate); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (err != 0) { JUMP_TO_LABEL(error); } @@ -10983,13 +11583,15 @@ assert(PyStackRef_FunctionCheck(frame->f_funcobj)); PyFunctionObject *func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyGenObject *gen = (PyGenObject *)_Py_MakeCoro(func); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (gen == NULL) { JUMP_TO_LABEL(error); } assert(STACK_LEVEL() <= 2); - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); _PyInterpreterFrame *gen_frame = &gen->gi_iframe; frame->instr_ptr++; _PyFrame_Copy(frame, gen_frame); @@ -11003,6 +11605,7 @@ frame = tstate->current_frame = prev; LOAD_IP(frame->return_offset); stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); res = PyStackRef_FromPyObjectStealMortal((PyObject *)gen); LLTRACE_RESUME_FRAME(); stack_pointer[0] = res; @@ -11035,6 +11638,7 @@ stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); assert(STACK_LEVEL() == 0); DTRACE_FUNCTION_RETURN(); _Py_LeaveRecursiveCallPy(tstate); @@ -11042,6 +11646,7 @@ frame = tstate->current_frame = dying->previous; _PyEval_FrameClearAndPop(tstate, dying); stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); LOAD_IP(frame->return_offset); res = temp; LLTRACE_RESUME_FRAME(); @@ -11076,8 +11681,9 @@ if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { next_instr = this_instr; _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); _Py_Specialize_Send(receiver, next_instr); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); DISPATCH_SAME_OPARG(); } OPCODE_DEFERRED_INC(SEND); @@ -11109,8 +11715,9 @@ } if (!PyStackRef_IsNull(null_or_index) && PyStackRef_IsNone(v)) { _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); _PyStackRef item = _PyForIter_VirtualIteratorNext(tstate, frame, receiver, &null_or_index); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (!PyStackRef_IsValid(item)) { if (PyStackRef_IsError(item)) { JUMP_TO_LABEL(error); @@ -11124,16 +11731,18 @@ else { PyObject *v_o = PyStackRef_AsPyObjectBorrow(v); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PySendResultPair res = _PyIter_Send(receiver_o, v_o); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (res.kind == PYGEN_ERROR) { JUMP_TO_LABEL(error); } stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(v); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); retval = PyStackRef_FromPyObjectSteal(res.object); if (res.kind == PYGEN_RETURN) { JUMPBY(oparg); @@ -11183,16 +11792,18 @@ PyObject *val = PyStackRef_AsPyObjectBorrow(v); PyObject *retval_o; _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PySendResult what = _PyAsyncGenASend_Send(iter_o, val, &retval_o); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (what == PYGEN_ERROR) { JUMP_TO_LABEL(error); } stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(v); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); asend = iter; null_out = null_in; retval = PyStackRef_FromPyObjectSteal(retval_o); @@ -11265,12 +11876,17 @@ stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); assert(temp->previous == frame || temp->previous->previous == frame); CALL_STAT_INC(inlined_py_calls); frame = tstate->current_frame = temp; tstate->py_recursion_remaining--; - LOAD_SP(); + stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); LOAD_IP(0); + #ifdef Py_DEBUG + assert(frame->previous->stackpointer_valid == 1); + #endif DTRACE_FUNCTION_ENTRY(); LLTRACE_RESUME_FRAME(); } @@ -11321,8 +11937,9 @@ PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter); Py_ssize_t index = PyStackRef_UntagInt(null_or_index); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); _PyObjectIndexPair next_index = Py_TYPE(iter_o)->_tp_iteritem(iter_o, index); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); PyObject *next_o = next_index.object; index = next_index.index; if (next_o == NULL) { @@ -11351,38 +11968,46 @@ INSTRUCTION_STATS(SETUP_ANNOTATIONS); if (LOCALS() == NULL) { _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); _PyErr_Format(tstate, PyExc_SystemError, "no locals found when setting up annotations"); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); JUMP_TO_LABEL(error); } int err; _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyObject* ann_dict = _PyMapping_GetOptionalItem2(LOCALS(), &_Py_ID(__annotations__), &err); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (err < 0) { JUMP_TO_LABEL(error); } if (ann_dict == NULL) { - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); ann_dict = PyDict_New(); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (ann_dict == NULL) { JUMP_TO_LABEL(error); } - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); err = PyObject_SetItem(LOCALS(), &_Py_ID(__annotations__), ann_dict); + _PyFrame_StackPointerInvalidate(frame); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); Py_DECREF(ann_dict); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (err) { JUMP_TO_LABEL(error); } } else { - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); Py_DECREF(ann_dict); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } DISPATCH(); } @@ -11400,9 +12025,10 @@ v = stack_pointer[-1]; set = stack_pointer[-2 - (oparg-1)]; _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); int err = _PySet_AddTakeRef((PySetObject *)PyStackRef_AsPyObjectBorrow(set), PyStackRef_AsPyObjectSteal(v)); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (err) { JUMP_TO_LABEL(pop_1_error); } @@ -11456,9 +12082,10 @@ iterable = stack_pointer[-1]; set = stack_pointer[-2 - (oparg-1)]; _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); int err = _PySet_Update(PyStackRef_AsPyObjectBorrow(set), PyStackRef_AsPyObjectBorrow(iterable)); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (err < 0) { JUMP_TO_LABEL(error); } @@ -11470,8 +12097,9 @@ stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_XCLOSE(value); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } DISPATCH(); } @@ -11499,8 +12127,9 @@ PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); next_instr = this_instr; _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); _Py_Specialize_StoreAttr(owner, next_instr, name); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); DISPATCH_SAME_OPARG(); } OPCODE_DEFERRED_INC(STORE_ATTR); @@ -11513,17 +12142,24 @@ v = stack_pointer[-2]; PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); int err = PyObject_SetAttr(PyStackRef_AsPyObjectBorrow(owner), name, PyStackRef_AsPyObjectBorrow(v)); + _PyFrame_StackPointerInvalidate(frame); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); _PyStackRef tmp = owner; owner = PyStackRef_NULL; stack_pointer[-1] = owner; PyStackRef_CLOSE(tmp); + _PyFrame_StackPointerInvalidate(frame); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); tmp = v; v = PyStackRef_NULL; stack_pointer[-2] = v; PyStackRef_CLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); stack_pointer += -2; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); if (err) { @@ -11609,8 +12245,9 @@ stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); Py_XDECREF(old_value); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } // _POP_TOP { @@ -11618,8 +12255,9 @@ stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_XCLOSE(value); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } DISPATCH(); } @@ -11671,8 +12309,9 @@ stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); Py_XDECREF(old_value); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } // _POP_TOP { @@ -11680,8 +12319,9 @@ stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_XCLOSE(value); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } DISPATCH(); } @@ -11760,8 +12400,9 @@ } } _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); _PyDict_NotifyEvent(PyDict_EVENT_MODIFIED, dict, name, PyStackRef_AsPyObjectBorrow(value)); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); FT_ATOMIC_STORE_PTR_RELEASE(ep->me_value, PyStackRef_AsPyObjectSteal(value)); UNLOCK_OBJECT(dict); STAT_INC(STORE_ATTR, hit); @@ -11770,8 +12411,9 @@ stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); Py_XDECREF(old_value); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } // _POP_TOP { @@ -11779,8 +12421,9 @@ stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_XCLOSE(value); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } DISPATCH(); } @@ -11797,8 +12440,9 @@ v = stack_pointer[-1]; PyCellObject *cell = (PyCellObject *)PyStackRef_AsPyObjectBorrow(GETLOCAL(oparg)); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyCell_SetTakeRef(cell, PyStackRef_AsPyObjectSteal(v)); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); @@ -11827,8 +12471,9 @@ stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_XCLOSE(value); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } DISPATCH(); } @@ -11851,8 +12496,9 @@ value2 = PyStackRef_DUP(GETLOCAL(oparg2)); stack_pointer[-1] = value2; _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_XCLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); DISPATCH(); } @@ -11875,15 +12521,17 @@ stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_XCLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); tmp = GETLOCAL(oparg2); GETLOCAL(oparg2) = value2; stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_XCLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); DISPATCH(); } @@ -11899,13 +12547,15 @@ v = stack_pointer[-1]; PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); int err = PyDict_SetItem(GLOBALS(), name, PyStackRef_AsPyObjectBorrow(v)); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(v); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (err) { JUMP_TO_LABEL(error); } @@ -11927,31 +12577,36 @@ int err; if (ns == NULL) { _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); _PyErr_Format(tstate, PyExc_SystemError, "no locals found when storing %R", name); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(v); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); JUMP_TO_LABEL(error); } if (PyDict_CheckExact(ns)) { _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); err = PyDict_SetItem(ns, name, PyStackRef_AsPyObjectBorrow(v)); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } else { _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); err = PyObject_SetItem(ns, name, PyStackRef_AsPyObjectBorrow(v)); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(v); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (err) { JUMP_TO_LABEL(error); } @@ -11983,10 +12638,11 @@ container = stack_pointer[-3]; v = stack_pointer[-4]; _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyObject *slice = _PyBuildSlice_ConsumeRefs(PyStackRef_AsPyObjectSteal(start), PyStackRef_AsPyObjectSteal(stop), Py_None); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); int err; if (slice == NULL) { err = 1; @@ -11995,21 +12651,29 @@ stack_pointer += -2; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); err = PyObject_SetItem(PyStackRef_AsPyObjectBorrow(container), slice, PyStackRef_AsPyObjectBorrow(v)); + _PyFrame_StackPointerInvalidate(frame); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); Py_DECREF(slice); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); stack_pointer += 2; } _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); _PyStackRef tmp = container; container = PyStackRef_NULL; stack_pointer[-3] = container; PyStackRef_CLOSE(tmp); + _PyFrame_StackPointerInvalidate(frame); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); tmp = v; v = PyStackRef_NULL; stack_pointer[-4] = v; PyStackRef_CLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); stack_pointer += -4; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); if (err) { @@ -12043,8 +12707,9 @@ if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { next_instr = this_instr; _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); _Py_Specialize_StoreSubscr(container, sub, next_instr); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); DISPATCH_SAME_OPARG(); } OPCODE_DEFERRED_INC(STORE_SUBSCR); @@ -12055,20 +12720,30 @@ { v = stack_pointer[-3]; _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); int err = PyObject_SetItem(PyStackRef_AsPyObjectBorrow(container), PyStackRef_AsPyObjectBorrow(sub), PyStackRef_AsPyObjectBorrow(v)); + _PyFrame_StackPointerInvalidate(frame); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); _PyStackRef tmp = sub; sub = PyStackRef_NULL; stack_pointer[-1] = sub; PyStackRef_CLOSE(tmp); + _PyFrame_StackPointerInvalidate(frame); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); tmp = container; container = PyStackRef_NULL; stack_pointer[-2] = container; PyStackRef_CLOSE(tmp); + _PyFrame_StackPointerInvalidate(frame); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); tmp = v; v = PyStackRef_NULL; stack_pointer[-3] = v; PyStackRef_CLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); stack_pointer += -3; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); if (err) { @@ -12119,16 +12794,18 @@ assert(Py_TYPE(dict)->tp_as_mapping->mp_ass_subscript == _PyDict_StoreSubscript); STAT_INC(STORE_SUBSCR, hit); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); int err = _PyDict_SetItem_Take2((PyDictObject *)dict, PyStackRef_AsPyObjectSteal(sub), PyStackRef_AsPyObjectSteal(value)); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (err) { stack_pointer += -3; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(dict_st); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); JUMP_TO_LABEL(error); } st = dict_st; @@ -12139,8 +12816,9 @@ stack_pointer += -3; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_XCLOSE(value); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } DISPATCH(); } @@ -12223,8 +12901,9 @@ stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); Py_DECREF(old_value); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } // _POP_TOP_INT { @@ -12238,8 +12917,9 @@ stack_pointer += -2; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_XCLOSE(value); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } DISPATCH(); } @@ -12286,8 +12966,9 @@ if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { next_instr = this_instr; _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); _Py_Specialize_ToBool(value, next_instr); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); DISPATCH_SAME_OPARG(); } OPCODE_DEFERRED_INC(TO_BOOL); @@ -12298,13 +12979,15 @@ // _TO_BOOL { _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); int err = PyObject_IsTrue(PyStackRef_AsPyObjectBorrow(value)); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(value); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (err < 0) { JUMP_TO_LABEL(error); } @@ -12355,8 +13038,9 @@ value = v; stack_pointer[-1] = res; _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_XCLOSE(value); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } DISPATCH(); } @@ -12469,8 +13153,9 @@ value = v; stack_pointer[-1] = res; _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_XCLOSE(value); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } DISPATCH(); } @@ -12574,30 +13259,34 @@ _PyJitTracerState *tracer = _tstate->jit_tracer_state; assert(tracer != NULL); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); int full = !_PyJit_translate_single_bytecode_to_trace(tstate, frame, next_instr, stop_tracing ? _DEOPT : 0); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (full) { LEAVE_TRACING(); - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); int err = stop_tracing_and_jit(tstate, frame); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (err < 0) { JUMP_TO_LABEL(error); } DISPATCH(); } for (int i = 0; i < tracer->prev_state.recorded_count; i++) { - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); Py_CLEAR(tracer->prev_state.recorded_values[i]); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } tracer->prev_state.recorded_count = 0; tracer->prev_state.instr = next_instr; PyObject *prev_code = PyStackRef_AsPyObjectBorrow(frame->f_executable); if (tracer->prev_state.instr_code != (PyCodeObject *)prev_code) { - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); Py_SETREF(tracer->prev_state.instr_code, (PyCodeObject*)Py_NewRef((prev_code))); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } tracer->prev_state.instr_frame = frame; tracer->prev_state.instr_oparg = oparg; @@ -12636,8 +13325,9 @@ { value = stack_pointer[-1]; _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyObject *res_o = PyNumber_Invert(PyStackRef_AsPyObjectBorrow(value)); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (res_o == NULL) { JUMP_TO_LABEL(error); } @@ -12648,9 +13338,10 @@ { value = v; stack_pointer[-1] = res; - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); PyStackRef_XCLOSE(value); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } DISPATCH(); } @@ -12670,8 +13361,9 @@ { value = stack_pointer[-1]; _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyObject *res_o = PyNumber_Negative(PyStackRef_AsPyObjectBorrow(value)); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (res_o == NULL) { JUMP_TO_LABEL(error); } @@ -12682,9 +13374,10 @@ { value = v; stack_pointer[-1] = res; - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); PyStackRef_XCLOSE(value); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } DISPATCH(); } @@ -12723,9 +13416,13 @@ stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); int res = _PyEval_UnpackIterableStackRef(tstate, seq_o, oparg & 0xFF, oparg >> 8, top); + _PyFrame_StackPointerInvalidate(frame); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); Py_DECREF(seq_o); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (res == 0) { JUMP_TO_LABEL(error); } @@ -12756,8 +13453,9 @@ if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { next_instr = this_instr; _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); _Py_Specialize_UnpackSequence(seq, next_instr, oparg); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); DISPATCH_SAME_OPARG(); } OPCODE_DEFERRED_INC(UNPACK_SEQUENCE); @@ -12773,9 +13471,13 @@ stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); int res = _PyEval_UnpackIterableStackRef(tstate, seq_o, oparg, -1, top); + _PyFrame_StackPointerInvalidate(frame); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); Py_DECREF(seq_o); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (res == 0) { JUMP_TO_LABEL(error); } @@ -12838,8 +13540,9 @@ stack_pointer += -1 + oparg; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(seq); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } DISPATCH(); } @@ -12888,8 +13591,9 @@ stack_pointer += -1 + oparg; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(seq); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } DISPATCH(); } @@ -12939,8 +13643,9 @@ stack_pointer += 1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(seq); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } DISPATCH(); } @@ -12978,13 +13683,15 @@ PyObject *stack[5] = {NULL, PyStackRef_AsPyObjectBorrow(exit_self), exc, val_o, tb}; int has_self = !PyStackRef_IsNull(exit_self); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); res_o = PyObject_Vectorcall(exit_func_o, stack + 2 - has_self, (3 + has_self) | PY_VECTORCALL_ARGUMENTS_OFFSET, NULL); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); Py_XDECREF(original_tb); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (res_o == NULL) { JUMP_TO_LABEL(error); } @@ -13023,6 +13730,7 @@ stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); DTRACE_FUNCTION_RETURN(); tstate->exc_info = gen->gi_exc_state.previous_item; gen->gi_exc_state.previous_item = NULL; @@ -13043,6 +13751,7 @@ } #endif stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); LOAD_IP(1 + INLINE_CACHE_ENTRIES_SEND); value = temp; LLTRACE_RESUME_FRAME(); @@ -13095,32 +13804,42 @@ JUMP_TO_LABEL(error); LABEL(error) { + _PyFrame_StackAssertInvalid(frame); #ifdef NDEBUG if (!_PyErr_Occurred(tstate)) { _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); _PyErr_SetString(tstate, PyExc_SystemError, "error return without exception set"); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } #else assert(_PyErr_Occurred(tstate)); #endif _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); STOP_TRACING(); stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); assert(frame->owner != FRAME_OWNED_BY_INTERPRETER); if (!_PyFrame_IsIncomplete(frame)) { - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); PyFrameObject *f = _PyFrame_GetFrameObject(frame); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (f != NULL) { - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); PyTraceBack_Here(f); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } } - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); _PyEval_MonitorRaise(tstate, frame, next_instr-1); + _PyFrame_StackPointerInvalidate(frame); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); JUMP_TO_LABEL(exception_unwind); } @@ -13165,6 +13884,7 @@ JUMP_TO_LABEL(error); } #endif stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); #if _Py_TAIL_CALL_INTERP int opcode; #endif @@ -13204,11 +13924,15 @@ JUMP_TO_LABEL(error); } next_instr = frame->instr_ptr; stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); JUMP_TO_LABEL(error); } LABEL(start_frame) { + #ifdef Py_DEBUG + assert(frame->stackpointer_valid == 1); + #endif int too_deep = _Py_EnterRecursivePy(tstate); if (too_deep) { JUMP_TO_LABEL(exit_unwind_notrace); @@ -13224,6 +13948,7 @@ JUMP_TO_LABEL(error); assert(!_PyErr_Occurred(tstate)); #endif stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); #if _Py_TAIL_CALL_INTERP int opcode; #endif @@ -13239,12 +13964,14 @@ JUMP_TO_LABEL(error); assert(IS_JIT_TRACING()); int opcode = next_instr->op.code; _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); _PyJit_translate_single_bytecode_to_trace(tstate, frame, NULL, _EXIT_TRACE); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); LEAVE_TRACING(); - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); int err = stop_tracing_and_jit(tstate, frame); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (err < 0) { JUMP_TO_LABEL(error); } diff --git a/Tools/cases_generator/analyzer.py b/Tools/cases_generator/analyzer.py index 22a321b4953d..4eed4d8d60e4 100644 --- a/Tools/cases_generator/analyzer.py +++ b/Tools/cases_generator/analyzer.py @@ -657,6 +657,7 @@ NON_ESCAPING_FUNCTIONS = ( "_PyFrame_PushUnchecked", "_PyFrame_SetStackPointer", "_PyFrame_StackPush", + "_PyFrame_StackAssertInvalid", "_PyFunction_SetVersion", "_PyGen_GetGeneratorFromFrame", "gen_try_set_executing", @@ -1006,7 +1007,7 @@ def compute_properties(op: parser.CodeDef) -> Properties: eval_breaker="CHECK_PERIODIC" in op.name, needs_this=variable_used(op, "this_instr"), always_exits=always_exits(op), - sync_sp=variable_used(op, "SYNC_SP"), + sync_sp=variable_used(op, "SYNC_SP") or variable_used(op, "SAVE_STACK") or variable_used(op, "RELOAD_STACK"), uses_co_consts=variable_used(op, "FRAME_CO_CONSTS"), uses_co_names=variable_used(op, "FRAME_CO_NAMES"), uses_locals=variable_used(op, "GETLOCAL") and not has_free, diff --git a/Tools/cases_generator/cwriter.py b/Tools/cases_generator/cwriter.py index 6636755db55e..de6247185824 100644 --- a/Tools/cases_generator/cwriter.py +++ b/Tools/cases_generator/cwriter.py @@ -39,7 +39,6 @@ class CWriter: self.newline = False def emit_at(self, txt: str, where: Token) -> None: - self.maybe_write_spill() self.set_position(where) self.out.write(txt) @@ -116,7 +115,6 @@ class CWriter: self.last_token = None def emit(self, txt: str | Token) -> None: - self.maybe_write_spill() if isinstance(txt, Token): self.emit_token(txt) elif isinstance(txt, str): @@ -130,28 +128,6 @@ class CWriter: self.newline = True self.last_token = None - def emit_spill(self) -> None: - if self.pending_reload: - self.pending_reload = False - return - assert not self.pending_spill - self.pending_spill = True - - def maybe_write_spill(self) -> None: - if self.pending_spill: - self.pending_spill = False - self.emit_str("_PyFrame_SetStackPointer(frame, stack_pointer);\n") - elif self.pending_reload: - self.pending_reload = False - self.emit_str("stack_pointer = _PyFrame_GetStackPointer(frame);\n") - - def emit_reload(self) -> None: - if self.pending_spill: - self.pending_spill = False - return - assert not self.pending_reload - self.pending_reload = True - @contextlib.contextmanager def header_guard(self, name: str) -> Iterator[None]: self.out.write( diff --git a/Tools/cases_generator/generators_common.py b/Tools/cases_generator/generators_common.py index 8c66ad4885cc..ccff95a961ce 100644 --- a/Tools/cases_generator/generators_common.py +++ b/Tools/cases_generator/generators_common.py @@ -503,7 +503,8 @@ class Emitter: next(tkn_iter) next(tkn_iter) next(tkn_iter) - self.emit_reload(storage) + storage.flush(self.out) + storage.full_reload(self.out) return True def instruction_size(self, @@ -709,7 +710,11 @@ class Emitter: self.out.emit(stmt.for_) for tkn in stmt.header: self.out.emit(tkn) - return self._emit_stmt(stmt.body, uop, storage, inst) + reachable, brace, body_storage = self._emit_stmt(stmt.body, uop, storage.copy(), inst) + body_storage.merge(storage, self.out) + if brace is not None: + self.out.emit(brace) + return reachable, None, storage def emit_WhileStmt( self, diff --git a/Tools/cases_generator/optimizer_generator.py b/Tools/cases_generator/optimizer_generator.py index aa914783f7cd..a32e6457cded 100644 --- a/Tools/cases_generator/optimizer_generator.py +++ b/Tools/cases_generator/optimizer_generator.py @@ -112,6 +112,7 @@ def decref_inputs( def emit_default(out: CWriter, uop: Uop, stack: Stack) -> None: + assert stack.physical_sp is not None null = CWriter.null() for var in reversed(uop.stack.inputs): stack.pop(var, null) @@ -153,6 +154,19 @@ class OptimizerEmitter(Emitter): def emit_reload(self, storage: Storage) -> None: pass + def reload_stack( + self, + tkn: Token, + tkn_iter: TokenIterator, + uop: CodeSection, + storage: Storage, + inst: Instruction | None, + ) -> bool: + next(tkn_iter) + next(tkn_iter) + next(tkn_iter) + return True + def goto_label(self, goto: Token, label: Token, storage: Storage) -> None: self.out.emit(goto) self.out.emit(label) diff --git a/Tools/cases_generator/stack.py b/Tools/cases_generator/stack.py index efc534fb607b..8044ac10dd0a 100644 --- a/Tools/cases_generator/stack.py +++ b/Tools/cases_generator/stack.py @@ -218,7 +218,8 @@ def array_or_scalar(var: StackItem | Local) -> str: class Stack: def __init__(self, check_stack_bounds: bool = False) -> None: self.base_offset = PointerOffset.zero() - self.physical_sp = PointerOffset.zero() + self.physical_sp:PointerOffset | None = PointerOffset.zero() + self.frame_sp:PointerOffset | None = None self.logical_sp = PointerOffset.zero() self.variables: list[Local] = [] self.check_stack_bounds = check_stack_bounds @@ -239,6 +240,7 @@ class Stack: raise StackError(f"Dropping live value '{var.name}'") def pop(self, var: StackItem, out: CWriter) -> Local: + assert self.physical_sp is not None if self.variables: top = self.variables[-1] if var.is_array() != top.is_array() or top.size != var.size: @@ -299,6 +301,8 @@ class Stack: out.emit(f"stack_pointer[{stack_offset.to_c()}] = {var.name};\n") def _save_physical_sp(self, out: CWriter) -> None: + if self.physical_sp is None: + return if self.physical_sp != self.logical_sp: diff = self.logical_sp - self.physical_sp out.start_line() @@ -311,6 +315,7 @@ class Stack: out.start_line() var_offset = self.base_offset for var in self.variables: + assert self.physical_sp is not None if ( var.in_local and not var.memory_offset and @@ -324,6 +329,8 @@ class Stack: var_offset = var_offset.push(var.item) def stack_bound_check(self, out: CWriter) -> None: + if self.physical_sp is None: + return if not self.check_stack_bounds: return if self.physical_sp != self.logical_sp: @@ -338,6 +345,44 @@ class Stack: self._save_physical_sp(out) out.start_line() + def emit_spill(self, out: CWriter) -> None: + if self.frame_sp != self.physical_sp: + out.emit_str("_PyFrame_SetStackPointer(frame, stack_pointer);\n") + self.frame_sp = self.physical_sp + else: + out.emit_str("assert(stack_pointer == _PyFrame_GetStackPointer(frame));\n") + out.emit_str("_PyFrame_StackPointerValidate(frame);\n"); + + def expected_frame_pointer(self) -> str: + assert self.frame_sp is not None + assert self.physical_sp is not None + diff = self.frame_sp - self.physical_sp + return f"(stack_pointer + {diff.to_c()})" + + def emit_reload(self, out: CWriter) -> None: + if self.physical_sp is None: + out.emit_str("stack_pointer = _PyFrame_GetStackPointer(frame);\n") + if self.frame_sp is None: + self.frame_sp = PointerOffset.zero() + self.physical_sp = self.frame_sp + out.emit_str("_PyFrame_StackPointerInvalidate(frame);\n"); + + def emit_fullreload(self, out: CWriter, spilled: bool = True) -> None: + self._print(out) + while self.variables: + top = self.variables.pop() + if not top.in_memory(): + raise StackError(f"Cannot reload with live variable: {top.name}") + self.base_offset = PointerOffset.zero() + self.physical_sp = PointerOffset.zero() + self.frame_sp = PointerOffset.zero() + self.logical_sp = PointerOffset.zero() + out.emit_str("stack_pointer = _PyFrame_GetStackPointer(frame);\n") + if spilled: + out.emit_str("_PyFrame_StackPointerInvalidate(frame);\n") + else: + out.emit_str("_PyFrame_StackAssertInvalid(frame);\n") + def is_flushed(self) -> bool: for var in self.variables: if not var.in_memory(): @@ -345,12 +390,15 @@ class Stack: return self.physical_sp == self.logical_sp def sp_offset(self) -> str: + assert self.physical_sp is not None return (self.physical_sp - self.logical_sp).to_c() def as_comment(self) -> str: variables = ", ".join([v.compact_str() for v in self.variables]) + sp = "undefined" if self.physical_sp is None else self.physical_sp.to_c() + fp = "undefined" if self.frame_sp is None else self.frame_sp.to_c() return ( - f"/* Variables=[{variables}]; base={self.base_offset.to_c()}; sp={self.physical_sp.to_c()}; logical_sp={self.logical_sp.to_c()} */" + f"/* Variables=[{variables}]; base={self.base_offset.to_c()}; sp={sp}; logical_sp={self.logical_sp.to_c()}; frame_sp={fp} */" ) def _print(self, out: CWriter) -> None: @@ -362,6 +410,7 @@ class Stack: other.base_offset = self.base_offset other.physical_sp = self.physical_sp other.logical_sp = self.logical_sp + other.frame_sp = self.frame_sp other.variables = [var.copy() for var in self.variables] other.check_stack_bounds = self.check_stack_bounds return other @@ -379,12 +428,18 @@ class Stack: def align(self, other: "Stack", out: CWriter) -> None: if self.logical_sp != other.logical_sp: raise StackError("Cannot align stacks: differing logical top") + if self.physical_sp is None and other.physical_sp is None: + return + if self.physical_sp is None or other.physical_sp is None: + raise StackError("Cannot align stacks: stack_pointer is partly undefined") if self.physical_sp == other.physical_sp: return diff = other.physical_sp - self.physical_sp out.start_line() out.emit(f"stack_pointer += {diff.to_c()};\n") self.physical_sp = other.physical_sp + if self.frame_sp != other.frame_sp: + self.frame_sp = None def merge(self, other: "Stack", out: CWriter) -> None: if len(self.variables) != len(other.variables): @@ -402,6 +457,9 @@ class Stack: raise StackError(f"Mismatched stack depths for {self_var.name}: {self_var.memory_offset} and {other_var.memory_offset}") elif other_var.memory_offset is None: self_var.memory_offset = None + if self.frame_sp != other.frame_sp: + self.frame_sp = None + other.frame_sp = None def stacks(inst: Instruction | PseudoInstruction) -> Iterator[StackEffect]: @@ -525,7 +583,7 @@ class Storage: assert self.spilled >= 0 if self.spilled == 0: out.start_line() - out.emit_spill() + self.stack.emit_spill(out) self.spilled += 1 def save_inputs(self, out: CWriter) -> None: @@ -534,7 +592,7 @@ class Storage: self.clear_dead_inputs() self.stack.flush(out) out.start_line() - out.emit_spill() + self.stack.emit_spill(out) self.spilled += 1 def reload(self, out: CWriter) -> None: @@ -544,10 +602,25 @@ class Storage: self.spilled -= 1 if self.spilled == 0: out.start_line() - out.emit_reload() + self.stack.emit_reload(out) + + def full_reload(self, out: CWriter) -> None: + if self.spilled == 0: + # Already spilled, presumably an explicit request + # to reload the stack from memory + out.start_line() + out.emit_str("// Explicit stack reload\n"); + self.stack.emit_fullreload(out, spilled=False) + else: + assert self.spilled > 0 + self.spilled -= 1 + implicit = True + self.stack.emit_fullreload(out) @staticmethod def for_uop(stack: Stack, uop: Uop, out: CWriter, check_liveness: bool = True) -> "Storage": + if stack.physical_sp is None: + raise StackError("stack pointer must be defined for uop") inputs: list[Local] = [] peeks: list[Local] = [] for input in reversed(uop.stack.inputs): @@ -632,7 +705,7 @@ class Storage: def push_outputs(self) -> None: if self.spilled: - raise StackError(f"Unbalanced stack spills") + raise StackError(f"Too many stack spills") self.clear_inputs("at the end of the micro-op") if len(self.inputs) > self.peeks and self.check_liveness: raise StackError(f"Input variable '{self.inputs[-1].name}' is still live") diff --git a/Tools/cases_generator/tier1_generator.py b/Tools/cases_generator/tier1_generator.py index ebc914a6837e..0d477cb83c5d 100644 --- a/Tools/cases_generator/tier1_generator.py +++ b/Tools/cases_generator/tier1_generator.py @@ -205,6 +205,7 @@ def generate_tier1_labels( storage = Storage(Stack(), [], [], 0, False) if label.spilled: storage.spilled = 1 + storage.stack.physical_sp = None emitter.emit_tokens(label, storage, None) emitter.emit("\n\n")