From: Kumar Aditya Date: Sat, 11 Apr 2026 04:59:38 +0000 (+0530) Subject: gh-148171: convert more variadic uops to leave input on stack in JIT (#148361) X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;ds=sidebyside;p=thirdparty%2FPython%2Fcpython.git gh-148171: convert more variadic uops to leave input on stack in JIT (#148361) --- diff --git a/Include/internal/pycore_ceval.h b/Include/internal/pycore_ceval.h index c12f72216b31..8c29fd0bde69 100644 --- a/Include/internal/pycore_ceval.h +++ b/Include/internal/pycore_ceval.h @@ -445,7 +445,7 @@ _Py_BuiltinCallFastWithKeywords_StackRefSteal( int total_args); PyAPI_FUNC(PyObject *) -_PyCallMethodDescriptorFast_StackRefSteal( +_PyCallMethodDescriptorFast_StackRef( _PyStackRef callable, PyCFunctionFast cfunc, PyObject *self, @@ -453,7 +453,7 @@ _PyCallMethodDescriptorFast_StackRefSteal( int total_args); PyAPI_FUNC(PyObject *) -_PyCallMethodDescriptorFastWithKeywords_StackRefSteal( +_PyCallMethodDescriptorFastWithKeywords_StackRef( _PyStackRef callable, PyCFunctionFastWithKeywords cfunc, PyObject *self, diff --git a/Include/internal/pycore_opcode_metadata.h b/Include/internal/pycore_opcode_metadata.h index 3810c886e565..c498f7127ba7 100644 --- a/Include/internal/pycore_opcode_metadata.h +++ b/Include/internal/pycore_opcode_metadata.h @@ -1139,8 +1139,8 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[267] = { [CALL_KW_PY] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG | HAS_SYNC_SP_FLAG | HAS_NEEDS_GUARD_IP_FLAG }, [CALL_LEN] = { true, INSTR_FMT_IXC00, HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, [CALL_LIST_APPEND] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, - [CALL_METHOD_DESCRIPTOR_FAST] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG | HAS_RECORDS_VALUE_FLAG }, + [CALL_METHOD_DESCRIPTOR_FAST] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, + [CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG | HAS_RECORDS_VALUE_FLAG }, [CALL_METHOD_DESCRIPTOR_NOARGS] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG | HAS_RECORDS_VALUE_FLAG }, [CALL_METHOD_DESCRIPTOR_O] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG | HAS_RECORDS_VALUE_FLAG }, [CALL_NON_PY_GENERAL] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG | HAS_RECORDS_VALUE_FLAG }, @@ -1388,8 +1388,8 @@ _PyOpcode_macro_expansion[256] = { [CALL_KW_PY] = { .nuops = 6, .uops = { { _CHECK_PEP_523, OPARG_SIMPLE, 1 }, { _CHECK_FUNCTION_VERSION_KW, 2, 1 }, { _CHECK_RECURSION_REMAINING, OPARG_SIMPLE, 3 }, { _PY_FRAME_KW, OPARG_SIMPLE, 3 }, { _SAVE_RETURN_OFFSET, OPARG_SAVE_RETURN_OFFSET, 3 }, { _PUSH_FRAME, OPARG_SIMPLE, 3 } } }, [CALL_LEN] = { .nuops = 5, .uops = { { _GUARD_NOS_NULL, OPARG_SIMPLE, 3 }, { _GUARD_CALLABLE_LEN, OPARG_SIMPLE, 3 }, { _CALL_LEN, OPARG_SIMPLE, 3 }, { _POP_TOP, OPARG_SIMPLE, 3 }, { _POP_TOP, OPARG_SIMPLE, 3 } } }, [CALL_LIST_APPEND] = { .nuops = 6, .uops = { { _GUARD_CALLABLE_LIST_APPEND, OPARG_SIMPLE, 3 }, { _GUARD_NOS_NOT_NULL, OPARG_SIMPLE, 3 }, { _GUARD_NOS_LIST, OPARG_SIMPLE, 3 }, { _CALL_LIST_APPEND, OPARG_SIMPLE, 3 }, { _POP_TOP, OPARG_SIMPLE, 3 }, { _POP_TOP, OPARG_SIMPLE, 3 } } }, - [CALL_METHOD_DESCRIPTOR_FAST] = { .nuops = 3, .uops = { { _GUARD_CALLABLE_METHOD_DESCRIPTOR_FAST, OPARG_SIMPLE, 3 }, { _CALL_METHOD_DESCRIPTOR_FAST, OPARG_SIMPLE, 3 }, { _CHECK_PERIODIC_AT_END, OPARG_REPLACED, 3 } } }, - [CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS] = { .nuops = 4, .uops = { { _RECORD_CALLABLE, OPARG_SIMPLE, 0 }, { _GUARD_CALLABLE_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS, OPARG_SIMPLE, 3 }, { _CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS, OPARG_SIMPLE, 3 }, { _CHECK_PERIODIC_AT_END, OPARG_REPLACED, 3 } } }, + [CALL_METHOD_DESCRIPTOR_FAST] = { .nuops = 5, .uops = { { _GUARD_CALLABLE_METHOD_DESCRIPTOR_FAST, OPARG_SIMPLE, 3 }, { _CALL_METHOD_DESCRIPTOR_FAST, OPARG_SIMPLE, 3 }, { _POP_TOP_OPARG, OPARG_SIMPLE, 3 }, { _POP_TOP, OPARG_SIMPLE, 3 }, { _CHECK_PERIODIC_AT_END, OPARG_REPLACED, 3 } } }, + [CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS] = { .nuops = 6, .uops = { { _RECORD_CALLABLE, OPARG_SIMPLE, 0 }, { _GUARD_CALLABLE_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS, OPARG_SIMPLE, 3 }, { _CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS, OPARG_SIMPLE, 3 }, { _POP_TOP_OPARG, OPARG_SIMPLE, 3 }, { _POP_TOP, OPARG_SIMPLE, 3 }, { _CHECK_PERIODIC_AT_END, OPARG_REPLACED, 3 } } }, [CALL_METHOD_DESCRIPTOR_NOARGS] = { .nuops = 7, .uops = { { _RECORD_CALLABLE, OPARG_SIMPLE, 0 }, { _GUARD_CALLABLE_METHOD_DESCRIPTOR_NOARGS, OPARG_SIMPLE, 3 }, { _CHECK_RECURSION_LIMIT, OPARG_SIMPLE, 3 }, { _CALL_METHOD_DESCRIPTOR_NOARGS, OPARG_SIMPLE, 3 }, { _POP_TOP, OPARG_SIMPLE, 3 }, { _POP_TOP, OPARG_SIMPLE, 3 }, { _CHECK_PERIODIC_AT_END, OPARG_REPLACED, 3 } } }, [CALL_METHOD_DESCRIPTOR_O] = { .nuops = 8, .uops = { { _RECORD_CALLABLE, OPARG_SIMPLE, 0 }, { _GUARD_CALLABLE_METHOD_DESCRIPTOR_O, OPARG_SIMPLE, 3 }, { _CHECK_RECURSION_LIMIT, OPARG_SIMPLE, 3 }, { _CALL_METHOD_DESCRIPTOR_O, OPARG_SIMPLE, 3 }, { _POP_TOP, OPARG_SIMPLE, 3 }, { _POP_TOP, OPARG_SIMPLE, 3 }, { _POP_TOP, OPARG_SIMPLE, 3 }, { _CHECK_PERIODIC_AT_END, OPARG_REPLACED, 3 } } }, [CALL_NON_PY_GENERAL] = { .nuops = 4, .uops = { { _RECORD_CALLABLE, OPARG_SIMPLE, 0 }, { _CHECK_IS_NOT_PY_CALLABLE, OPARG_SIMPLE, 3 }, { _CALL_NON_PY_GENERAL, OPARG_SIMPLE, 3 }, { _CHECK_PERIODIC_AT_END, OPARG_REPLACED, 3 } } }, diff --git a/Include/internal/pycore_uop_ids.h b/Include/internal/pycore_uop_ids.h index 12b2bf32d777..232fde3e6f80 100644 --- a/Include/internal/pycore_uop_ids.h +++ b/Include/internal/pycore_uop_ids.h @@ -521,10 +521,10 @@ extern "C" { #define _CALL_LIST_APPEND_r13 731 #define _CALL_LIST_APPEND_r23 732 #define _CALL_LIST_APPEND_r33 733 -#define _CALL_METHOD_DESCRIPTOR_FAST_r01 734 -#define _CALL_METHOD_DESCRIPTOR_FAST_INLINE_r01 735 -#define _CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS_r01 736 -#define _CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS_INLINE_r01 737 +#define _CALL_METHOD_DESCRIPTOR_FAST_r00 734 +#define _CALL_METHOD_DESCRIPTOR_FAST_INLINE_r00 735 +#define _CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS_r00 736 +#define _CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS_INLINE_r00 737 #define _CALL_METHOD_DESCRIPTOR_NOARGS_r03 738 #define _CALL_METHOD_DESCRIPTOR_NOARGS_INLINE_r03 739 #define _CALL_METHOD_DESCRIPTOR_O_r03 740 diff --git a/Include/internal/pycore_uop_metadata.h b/Include/internal/pycore_uop_metadata.h index b6d784707844..9200d345309b 100644 --- a/Include/internal/pycore_uop_metadata.h +++ b/Include/internal/pycore_uop_metadata.h @@ -326,14 +326,14 @@ const uint32_t _PyUop_Flags[MAX_UOP_ID+1] = { [_CHECK_RECURSION_LIMIT] = HAS_EXIT_FLAG, [_CALL_METHOD_DESCRIPTOR_O_INLINE] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG, [_GUARD_CALLABLE_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS] = HAS_ARG_FLAG | HAS_EXIT_FLAG, - [_CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, - [_CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS_INLINE] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, + [_CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG, + [_CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS_INLINE] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG, [_GUARD_CALLABLE_METHOD_DESCRIPTOR_NOARGS] = HAS_ARG_FLAG | HAS_EXIT_FLAG, [_CALL_METHOD_DESCRIPTOR_NOARGS] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG, [_CALL_METHOD_DESCRIPTOR_NOARGS_INLINE] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG, [_GUARD_CALLABLE_METHOD_DESCRIPTOR_FAST] = HAS_ARG_FLAG | HAS_EXIT_FLAG, - [_CALL_METHOD_DESCRIPTOR_FAST] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, - [_CALL_METHOD_DESCRIPTOR_FAST_INLINE] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, + [_CALL_METHOD_DESCRIPTOR_FAST] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG, + [_CALL_METHOD_DESCRIPTOR_FAST_INLINE] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG, [_MAYBE_EXPAND_METHOD_KW] = HAS_ARG_FLAG | HAS_ESCAPES_FLAG, [_PY_FRAME_KW] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG | HAS_SYNC_SP_FLAG, [_CHECK_FUNCTION_VERSION_KW] = HAS_ARG_FLAG | HAS_EXIT_FLAG, @@ -3056,7 +3056,7 @@ const _PyUopCachingInfo _PyUop_Caching[MAX_UOP_ID+1] = { [_CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS] = { .best = { 0, 0, 0, 0 }, .entries = { - { 1, 0, _CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS_r01 }, + { 0, 0, _CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS_r00 }, { -1, -1, -1 }, { -1, -1, -1 }, { -1, -1, -1 }, @@ -3065,7 +3065,7 @@ const _PyUopCachingInfo _PyUop_Caching[MAX_UOP_ID+1] = { [_CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS_INLINE] = { .best = { 0, 0, 0, 0 }, .entries = { - { 1, 0, _CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS_INLINE_r01 }, + { 0, 0, _CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS_INLINE_r00 }, { -1, -1, -1 }, { -1, -1, -1 }, { -1, -1, -1 }, @@ -3110,7 +3110,7 @@ const _PyUopCachingInfo _PyUop_Caching[MAX_UOP_ID+1] = { [_CALL_METHOD_DESCRIPTOR_FAST] = { .best = { 0, 0, 0, 0 }, .entries = { - { 1, 0, _CALL_METHOD_DESCRIPTOR_FAST_r01 }, + { 0, 0, _CALL_METHOD_DESCRIPTOR_FAST_r00 }, { -1, -1, -1 }, { -1, -1, -1 }, { -1, -1, -1 }, @@ -3119,7 +3119,7 @@ const _PyUopCachingInfo _PyUop_Caching[MAX_UOP_ID+1] = { [_CALL_METHOD_DESCRIPTOR_FAST_INLINE] = { .best = { 0, 0, 0, 0 }, .entries = { - { 1, 0, _CALL_METHOD_DESCRIPTOR_FAST_INLINE_r01 }, + { 0, 0, _CALL_METHOD_DESCRIPTOR_FAST_INLINE_r00 }, { -1, -1, -1 }, { -1, -1, -1 }, { -1, -1, -1 }, @@ -4462,14 +4462,14 @@ const uint16_t _PyUop_Uncached[MAX_UOP_REGS_ID+1] = { [_CHECK_RECURSION_LIMIT_r33] = _CHECK_RECURSION_LIMIT, [_CALL_METHOD_DESCRIPTOR_O_INLINE_r03] = _CALL_METHOD_DESCRIPTOR_O_INLINE, [_GUARD_CALLABLE_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS_r00] = _GUARD_CALLABLE_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS, - [_CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS_r01] = _CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS, - [_CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS_INLINE_r01] = _CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS_INLINE, + [_CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS_r00] = _CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS, + [_CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS_INLINE_r00] = _CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS_INLINE, [_GUARD_CALLABLE_METHOD_DESCRIPTOR_NOARGS_r00] = _GUARD_CALLABLE_METHOD_DESCRIPTOR_NOARGS, [_CALL_METHOD_DESCRIPTOR_NOARGS_r03] = _CALL_METHOD_DESCRIPTOR_NOARGS, [_CALL_METHOD_DESCRIPTOR_NOARGS_INLINE_r03] = _CALL_METHOD_DESCRIPTOR_NOARGS_INLINE, [_GUARD_CALLABLE_METHOD_DESCRIPTOR_FAST_r00] = _GUARD_CALLABLE_METHOD_DESCRIPTOR_FAST, - [_CALL_METHOD_DESCRIPTOR_FAST_r01] = _CALL_METHOD_DESCRIPTOR_FAST, - [_CALL_METHOD_DESCRIPTOR_FAST_INLINE_r01] = _CALL_METHOD_DESCRIPTOR_FAST_INLINE, + [_CALL_METHOD_DESCRIPTOR_FAST_r00] = _CALL_METHOD_DESCRIPTOR_FAST, + [_CALL_METHOD_DESCRIPTOR_FAST_INLINE_r00] = _CALL_METHOD_DESCRIPTOR_FAST_INLINE, [_MAYBE_EXPAND_METHOD_KW_r11] = _MAYBE_EXPAND_METHOD_KW, [_PY_FRAME_KW_r11] = _PY_FRAME_KW, [_CHECK_FUNCTION_VERSION_KW_r11] = _CHECK_FUNCTION_VERSION_KW, @@ -4851,13 +4851,13 @@ const char *const _PyOpcode_uop_name[MAX_UOP_REGS_ID+1] = { [_CALL_LIST_APPEND_r23] = "_CALL_LIST_APPEND_r23", [_CALL_LIST_APPEND_r33] = "_CALL_LIST_APPEND_r33", [_CALL_METHOD_DESCRIPTOR_FAST] = "_CALL_METHOD_DESCRIPTOR_FAST", - [_CALL_METHOD_DESCRIPTOR_FAST_r01] = "_CALL_METHOD_DESCRIPTOR_FAST_r01", + [_CALL_METHOD_DESCRIPTOR_FAST_r00] = "_CALL_METHOD_DESCRIPTOR_FAST_r00", [_CALL_METHOD_DESCRIPTOR_FAST_INLINE] = "_CALL_METHOD_DESCRIPTOR_FAST_INLINE", - [_CALL_METHOD_DESCRIPTOR_FAST_INLINE_r01] = "_CALL_METHOD_DESCRIPTOR_FAST_INLINE_r01", + [_CALL_METHOD_DESCRIPTOR_FAST_INLINE_r00] = "_CALL_METHOD_DESCRIPTOR_FAST_INLINE_r00", [_CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS] = "_CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS", - [_CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS_r01] = "_CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS_r01", + [_CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS_r00] = "_CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS_r00", [_CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS_INLINE] = "_CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS_INLINE", - [_CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS_INLINE_r01] = "_CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS_INLINE_r01", + [_CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS_INLINE_r00] = "_CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS_INLINE_r00", [_CALL_METHOD_DESCRIPTOR_NOARGS] = "_CALL_METHOD_DESCRIPTOR_NOARGS", [_CALL_METHOD_DESCRIPTOR_NOARGS_r03] = "_CALL_METHOD_DESCRIPTOR_NOARGS_r03", [_CALL_METHOD_DESCRIPTOR_NOARGS_INLINE] = "_CALL_METHOD_DESCRIPTOR_NOARGS_INLINE", @@ -6554,9 +6554,9 @@ int _PyUop_num_popped(int opcode, int oparg) case _GUARD_CALLABLE_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS: return 0; case _CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS: - return 2 + oparg; + return 0; case _CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS_INLINE: - return 1 + oparg; + return 0; case _GUARD_CALLABLE_METHOD_DESCRIPTOR_NOARGS: return 0; case _CALL_METHOD_DESCRIPTOR_NOARGS: @@ -6566,9 +6566,9 @@ int _PyUop_num_popped(int opcode, int oparg) case _GUARD_CALLABLE_METHOD_DESCRIPTOR_FAST: return 0; case _CALL_METHOD_DESCRIPTOR_FAST: - return 2 + oparg; + return 0; case _CALL_METHOD_DESCRIPTOR_FAST_INLINE: - return 1 + oparg; + return 0; case _MAYBE_EXPAND_METHOD_KW: return 0; case _PY_FRAME_KW: diff --git a/Modules/_testinternalcapi/test_cases.c.h b/Modules/_testinternalcapi/test_cases.c.h index 4d9eb4998649..93fa20520108 100644 --- a/Modules/_testinternalcapi/test_cases.c.h +++ b/Modules/_testinternalcapi/test_cases.c.h @@ -3798,7 +3798,7 @@ _PyStackRef callable; _PyStackRef self_or_null; _PyStackRef *args; - _PyStackRef res; + _PyStackRef value; /* Skip 1 cache entry */ /* Skip 2 cache entries */ // _GUARD_CALLABLE_METHOD_DESCRIPTOR_FAST @@ -3850,7 +3850,7 @@ STAT_INC(CALL, hit); _PyFrame_SetStackPointer(frame, stack_pointer); PyCFunctionFast cfunc = _PyCFunctionFast_CAST(method->d_method->ml_meth); - PyObject *res_o = _PyCallMethodDescriptorFast_StackRefSteal( + PyObject *res_o = _PyCallMethodDescriptorFast_StackRef( callable, cfunc, self, @@ -3859,17 +3859,32 @@ ); stack_pointer = _PyFrame_GetStackPointer(frame); if (res_o == NULL) { - stack_pointer += -2 - oparg; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); JUMP_TO_LABEL(error); } - res = PyStackRef_FromPyObjectSteal(res_o); + _PyStackRef temp = callable; + callable = PyStackRef_FromPyObjectSteal(res_o); + stack_pointer[-2 - oparg] = callable; + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(temp); + stack_pointer = _PyFrame_GetStackPointer(frame); } - // _CHECK_PERIODIC_AT_END + // _POP_TOP_OPARG { - stack_pointer[-2 - oparg] = res; + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyStackRef_CloseStack(args, oparg); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + // _POP_TOP + { + value = self_or_null; stack_pointer += -1 - oparg; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_XCLOSE(value); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + // _CHECK_PERIODIC_AT_END + { _PyFrame_SetStackPointer(frame, stack_pointer); int err = check_periodics(tstate); stack_pointer = _PyFrame_GetStackPointer(frame); @@ -3894,7 +3909,7 @@ _PyStackRef callable; _PyStackRef self_or_null; _PyStackRef *args; - _PyStackRef res; + _PyStackRef value; /* Skip 1 cache entry */ /* Skip 2 cache entries */ // _GUARD_CALLABLE_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS @@ -3947,7 +3962,7 @@ STAT_INC(CALL, hit); _PyFrame_SetStackPointer(frame, stack_pointer); PyCFunctionFastWithKeywords cfunc = _PyCFunctionFastWithKeywords_CAST(method->d_method->ml_meth); - PyObject *res_o = _PyCallMethodDescriptorFastWithKeywords_StackRefSteal( + PyObject *res_o = _PyCallMethodDescriptorFastWithKeywords_StackRef( callable, cfunc, self, @@ -3956,17 +3971,32 @@ ); stack_pointer = _PyFrame_GetStackPointer(frame); if (res_o == NULL) { - stack_pointer += -2 - oparg; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); JUMP_TO_LABEL(error); } - res = PyStackRef_FromPyObjectSteal(res_o); + _PyStackRef temp = callable; + callable = PyStackRef_FromPyObjectSteal(res_o); + stack_pointer[-2 - oparg] = callable; + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(temp); + stack_pointer = _PyFrame_GetStackPointer(frame); } - // _CHECK_PERIODIC_AT_END + // _POP_TOP_OPARG { - stack_pointer[-2 - oparg] = res; + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyStackRef_CloseStack(args, oparg); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + // _POP_TOP + { + value = self_or_null; stack_pointer += -1 - oparg; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_XCLOSE(value); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + // _CHECK_PERIODIC_AT_END + { _PyFrame_SetStackPointer(frame, stack_pointer); int err = check_periodics(tstate); stack_pointer = _PyFrame_GetStackPointer(frame); diff --git a/Python/bytecodes.c b/Python/bytecodes.c index 5124c19e501c..52f75559317d 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -4907,7 +4907,7 @@ dummy_func( EXIT_IF(!Py_IS_TYPE(self, method->d_common.d_type)); } - op(_CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS, (callable, self_or_null, args[oparg] -- res)) { + op(_CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS, (callable, self_or_null, args[oparg] -- callable, self_or_null, args[oparg])) { PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); PyMethodDescrObject *method = (PyMethodDescrObject *)callable_o; @@ -4921,36 +4921,38 @@ dummy_func( assert(self != NULL); STAT_INC(CALL, hit); PyCFunctionFastWithKeywords cfunc = _PyCFunctionFastWithKeywords_CAST(method->d_method->ml_meth); - PyObject *res_o = _PyCallMethodDescriptorFastWithKeywords_StackRefSteal( + PyObject *res_o = _PyCallMethodDescriptorFastWithKeywords_StackRef( callable, cfunc, self, arguments, total_args ); - DEAD(args); - DEAD(self_or_null); - DEAD(callable); - ERROR_IF(res_o == NULL); - res = PyStackRef_FromPyObjectSteal(res_o); + if (res_o == NULL) { + ERROR_NO_POP(); + } + _PyStackRef temp = callable; + callable = PyStackRef_FromPyObjectSteal(res_o); + PyStackRef_CLOSE(temp); } - tier2 op(_CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS_INLINE, (callable, args[oparg], cfunc/4 -- res)) { - PyObject *self = PyStackRef_AsPyObjectBorrow(args[0]); - assert(self != NULL); + tier2 op(_CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS_INLINE, (callable, self_st, args[oparg], cfunc/4 -- callable, self_st, args[oparg])) { + PyObject *self = PyStackRef_AsPyObjectBorrow(self_st); STAT_INC(CALL, hit); volatile PyCFunctionFastWithKeywords cfunc_v = _PyCFunctionFastWithKeywords_CAST(cfunc); - PyObject *res_o = _PyCallMethodDescriptorFastWithKeywords_StackRefSteal( + PyObject *res_o = _PyCallMethodDescriptorFastWithKeywords_StackRef( callable, cfunc_v, self, - args, - oparg + args - 1, + oparg + 1 ); - DEAD(args); - DEAD(callable); - ERROR_IF(res_o == NULL); - res = PyStackRef_FromPyObjectSteal(res_o); + if (res_o == NULL) { + ERROR_NO_POP(); + } + _PyStackRef temp = callable; + callable = PyStackRef_FromPyObjectSteal(res_o); + PyStackRef_CLOSE(temp); } macro(CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS) = @@ -4959,6 +4961,8 @@ dummy_func( unused/2 + _GUARD_CALLABLE_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS + _CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS + + _POP_TOP_OPARG + + POP_TOP + _CHECK_PERIODIC_AT_END; op(_GUARD_CALLABLE_METHOD_DESCRIPTOR_NOARGS, (callable, self_or_null, args[oparg] -- callable, self_or_null, args[oparg])) { @@ -5045,7 +5049,7 @@ dummy_func( EXIT_IF(!Py_IS_TYPE(self, method->d_common.d_type)); } - op(_CALL_METHOD_DESCRIPTOR_FAST, (callable, self_or_null, args[oparg] -- res)) { + op(_CALL_METHOD_DESCRIPTOR_FAST, (callable, self_or_null, args[oparg] -- callable, self_or_null, args[oparg])) { PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); PyMethodDescrObject *method = (PyMethodDescrObject *)callable_o; @@ -5059,36 +5063,39 @@ dummy_func( assert(self != NULL); STAT_INC(CALL, hit); PyCFunctionFast cfunc = _PyCFunctionFast_CAST(method->d_method->ml_meth); - PyObject *res_o = _PyCallMethodDescriptorFast_StackRefSteal( + PyObject *res_o = _PyCallMethodDescriptorFast_StackRef( callable, cfunc, self, arguments, total_args ); - DEAD(args); - DEAD(self_or_null); - DEAD(callable); - ERROR_IF(res_o == NULL); - res = PyStackRef_FromPyObjectSteal(res_o); + if (res_o == NULL) { + ERROR_NO_POP(); + } + _PyStackRef temp = callable; + callable = PyStackRef_FromPyObjectSteal(res_o); + PyStackRef_CLOSE(temp); } - tier2 op(_CALL_METHOD_DESCRIPTOR_FAST_INLINE, (callable, args[oparg], cfunc/4 -- res)) { - PyObject *self = PyStackRef_AsPyObjectBorrow(args[0]); + tier2 op(_CALL_METHOD_DESCRIPTOR_FAST_INLINE, (callable, self_st, args[oparg], cfunc/4 -- callable, self_st, args[oparg])) { + PyObject *self = PyStackRef_AsPyObjectBorrow(self_st); assert(self != NULL); STAT_INC(CALL, hit); volatile PyCFunctionFast cfunc_v = _PyCFunctionFast_CAST(cfunc); - PyObject *res_o = _PyCallMethodDescriptorFast_StackRefSteal( + PyObject *res_o = _PyCallMethodDescriptorFast_StackRef( callable, cfunc_v, self, - args, - oparg + args - 1, + oparg + 1 ); - DEAD(args); - DEAD(callable); - ERROR_IF(res_o == NULL); - res = PyStackRef_FromPyObjectSteal(res_o); + if (res_o == NULL) { + ERROR_NO_POP(); + } + _PyStackRef temp = callable; + callable = PyStackRef_FromPyObjectSteal(res_o); + PyStackRef_CLOSE(temp); } macro(CALL_METHOD_DESCRIPTOR_FAST) = @@ -5096,6 +5103,8 @@ dummy_func( unused/2 + _GUARD_CALLABLE_METHOD_DESCRIPTOR_FAST + _CALL_METHOD_DESCRIPTOR_FAST + + _POP_TOP_OPARG + + POP_TOP + _CHECK_PERIODIC_AT_END; // Cache layout: counter/1, func_version/2 diff --git a/Python/ceval.c b/Python/ceval.c index 3f024ad67f2c..6ca5849d1168 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -862,7 +862,7 @@ cleanup: } PyObject * -_PyCallMethodDescriptorFast_StackRefSteal( +_PyCallMethodDescriptorFast_StackRef( _PyStackRef callable, PyCFunctionFast cfunc, PyObject *self, @@ -872,28 +872,18 @@ _PyCallMethodDescriptorFast_StackRefSteal( PyObject *res; STACKREFS_TO_PYOBJECTS(arguments, total_args, args_o); if (CONVERSION_FAILED(args_o)) { - res = NULL; - goto cleanup; + return NULL; } assert(self == PyStackRef_AsPyObjectBorrow(arguments[0])); res = cfunc(self, (args_o + 1), total_args - 1); STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); assert((res != NULL) ^ (PyErr_Occurred() != NULL)); -cleanup: - // arguments is a pointer into the GC visible stack, - // so we must NULL out values as we clear them. - for (int i = total_args-1; i >= 0; i--) { - _PyStackRef tmp = arguments[i]; - arguments[i] = PyStackRef_NULL; - PyStackRef_CLOSE(tmp); - } - PyStackRef_CLOSE(callable); return res; } PyObject * -_PyCallMethodDescriptorFastWithKeywords_StackRefSteal( +_PyCallMethodDescriptorFastWithKeywords_StackRef( _PyStackRef callable, PyCFunctionFastWithKeywords cfunc, PyObject *self, @@ -903,23 +893,13 @@ _PyCallMethodDescriptorFastWithKeywords_StackRefSteal( PyObject *res; STACKREFS_TO_PYOBJECTS(arguments, total_args, args_o); if (CONVERSION_FAILED(args_o)) { - res = NULL; - goto cleanup; + return NULL; } assert(self == PyStackRef_AsPyObjectBorrow(arguments[0])); res = cfunc(self, (args_o + 1), total_args-1, NULL); STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); assert((res != NULL) ^ (PyErr_Occurred() != NULL)); -cleanup: - // arguments is a pointer into the GC visible stack, - // so we must NULL out values as we clear them. - for (int i = total_args-1; i >= 0; i--) { - _PyStackRef tmp = arguments[i]; - arguments[i] = PyStackRef_NULL; - PyStackRef_CLOSE(tmp); - } - PyStackRef_CLOSE(callable); return res; } diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h index 20f255689a75..04fc4322ead6 100644 --- a/Python/executor_cases.c.h +++ b/Python/executor_cases.c.h @@ -17540,13 +17540,12 @@ break; } - case _CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS_r01: { + case _CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS_r00: { CHECK_CURRENT_CACHED_VALUES(0); assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); _PyStackRef *args; _PyStackRef self_or_null; _PyStackRef callable; - _PyStackRef res; oparg = CURRENT_OPARG(); args = &stack_pointer[-oparg]; self_or_null = stack_pointer[-1 - oparg]; @@ -17564,7 +17563,7 @@ STAT_INC(CALL, hit); _PyFrame_SetStackPointer(frame, stack_pointer); PyCFunctionFastWithKeywords cfunc = _PyCFunctionFastWithKeywords_CAST(method->d_method->ml_meth); - PyObject *res_o = _PyCallMethodDescriptorFastWithKeywords_StackRefSteal( + PyObject *res_o = _PyCallMethodDescriptorFastWithKeywords_StackRef( callable, cfunc, self, @@ -17573,58 +17572,60 @@ ); stack_pointer = _PyFrame_GetStackPointer(frame); if (res_o == NULL) { - stack_pointer += -2 - oparg; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); SET_CURRENT_CACHED_VALUES(0); JUMP_TO_ERROR(); } - res = PyStackRef_FromPyObjectSteal(res_o); - _tos_cache0 = res; + _PyStackRef temp = callable; + callable = PyStackRef_FromPyObjectSteal(res_o); + stack_pointer[-2 - oparg] = callable; + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(temp); + stack_pointer = _PyFrame_GetStackPointer(frame); + _tos_cache0 = PyStackRef_ZERO_BITS; _tos_cache1 = PyStackRef_ZERO_BITS; _tos_cache2 = PyStackRef_ZERO_BITS; - SET_CURRENT_CACHED_VALUES(1); - stack_pointer += -2 - oparg; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + SET_CURRENT_CACHED_VALUES(0); assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); break; } - case _CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS_INLINE_r01: { + case _CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS_INLINE_r00: { CHECK_CURRENT_CACHED_VALUES(0); assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); _PyStackRef *args; + _PyStackRef self_st; _PyStackRef callable; - _PyStackRef res; oparg = CURRENT_OPARG(); args = &stack_pointer[-oparg]; - callable = stack_pointer[-1 - oparg]; + self_st = stack_pointer[-1 - oparg]; + callable = stack_pointer[-2 - oparg]; PyObject *cfunc = (PyObject *)CURRENT_OPERAND0_64(); - PyObject *self = PyStackRef_AsPyObjectBorrow(args[0]); - assert(self != NULL); + PyObject *self = PyStackRef_AsPyObjectBorrow(self_st); STAT_INC(CALL, hit); _PyFrame_SetStackPointer(frame, stack_pointer); volatile PyCFunctionFastWithKeywords cfunc_v = _PyCFunctionFastWithKeywords_CAST(cfunc); - PyObject *res_o = _PyCallMethodDescriptorFastWithKeywords_StackRefSteal( + PyObject *res_o = _PyCallMethodDescriptorFastWithKeywords_StackRef( callable, cfunc_v, self, - args, - oparg + args - 1, + oparg + 1 ); stack_pointer = _PyFrame_GetStackPointer(frame); if (res_o == NULL) { - stack_pointer += -1 - oparg; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); SET_CURRENT_CACHED_VALUES(0); JUMP_TO_ERROR(); } - res = PyStackRef_FromPyObjectSteal(res_o); - _tos_cache0 = res; + _PyStackRef temp = callable; + callable = PyStackRef_FromPyObjectSteal(res_o); + stack_pointer[-2 - oparg] = callable; + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(temp); + stack_pointer = _PyFrame_GetStackPointer(frame); + _tos_cache0 = PyStackRef_ZERO_BITS; _tos_cache1 = PyStackRef_ZERO_BITS; _tos_cache2 = PyStackRef_ZERO_BITS; - SET_CURRENT_CACHED_VALUES(1); - stack_pointer += -1 - oparg; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + SET_CURRENT_CACHED_VALUES(0); assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); break; } @@ -17799,13 +17800,12 @@ break; } - case _CALL_METHOD_DESCRIPTOR_FAST_r01: { + case _CALL_METHOD_DESCRIPTOR_FAST_r00: { CHECK_CURRENT_CACHED_VALUES(0); assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); _PyStackRef *args; _PyStackRef self_or_null; _PyStackRef callable; - _PyStackRef res; oparg = CURRENT_OPARG(); args = &stack_pointer[-oparg]; self_or_null = stack_pointer[-1 - oparg]; @@ -17823,7 +17823,7 @@ STAT_INC(CALL, hit); _PyFrame_SetStackPointer(frame, stack_pointer); PyCFunctionFast cfunc = _PyCFunctionFast_CAST(method->d_method->ml_meth); - PyObject *res_o = _PyCallMethodDescriptorFast_StackRefSteal( + PyObject *res_o = _PyCallMethodDescriptorFast_StackRef( callable, cfunc, self, @@ -17832,58 +17832,61 @@ ); stack_pointer = _PyFrame_GetStackPointer(frame); if (res_o == NULL) { - stack_pointer += -2 - oparg; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); SET_CURRENT_CACHED_VALUES(0); JUMP_TO_ERROR(); } - res = PyStackRef_FromPyObjectSteal(res_o); - _tos_cache0 = res; + _PyStackRef temp = callable; + callable = PyStackRef_FromPyObjectSteal(res_o); + stack_pointer[-2 - oparg] = callable; + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(temp); + stack_pointer = _PyFrame_GetStackPointer(frame); + _tos_cache0 = PyStackRef_ZERO_BITS; _tos_cache1 = PyStackRef_ZERO_BITS; _tos_cache2 = PyStackRef_ZERO_BITS; - SET_CURRENT_CACHED_VALUES(1); - stack_pointer += -2 - oparg; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + SET_CURRENT_CACHED_VALUES(0); assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); break; } - case _CALL_METHOD_DESCRIPTOR_FAST_INLINE_r01: { + case _CALL_METHOD_DESCRIPTOR_FAST_INLINE_r00: { CHECK_CURRENT_CACHED_VALUES(0); assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); _PyStackRef *args; + _PyStackRef self_st; _PyStackRef callable; - _PyStackRef res; oparg = CURRENT_OPARG(); args = &stack_pointer[-oparg]; - callable = stack_pointer[-1 - oparg]; + self_st = stack_pointer[-1 - oparg]; + callable = stack_pointer[-2 - oparg]; PyObject *cfunc = (PyObject *)CURRENT_OPERAND0_64(); - PyObject *self = PyStackRef_AsPyObjectBorrow(args[0]); + PyObject *self = PyStackRef_AsPyObjectBorrow(self_st); assert(self != NULL); STAT_INC(CALL, hit); _PyFrame_SetStackPointer(frame, stack_pointer); volatile PyCFunctionFast cfunc_v = _PyCFunctionFast_CAST(cfunc); - PyObject *res_o = _PyCallMethodDescriptorFast_StackRefSteal( + PyObject *res_o = _PyCallMethodDescriptorFast_StackRef( callable, cfunc_v, self, - args, - oparg + args - 1, + oparg + 1 ); stack_pointer = _PyFrame_GetStackPointer(frame); if (res_o == NULL) { - stack_pointer += -1 - oparg; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); SET_CURRENT_CACHED_VALUES(0); JUMP_TO_ERROR(); } - res = PyStackRef_FromPyObjectSteal(res_o); - _tos_cache0 = res; + _PyStackRef temp = callable; + callable = PyStackRef_FromPyObjectSteal(res_o); + stack_pointer[-2 - oparg] = callable; + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(temp); + stack_pointer = _PyFrame_GetStackPointer(frame); + _tos_cache0 = PyStackRef_ZERO_BITS; _tos_cache1 = PyStackRef_ZERO_BITS; _tos_cache2 = PyStackRef_ZERO_BITS; - SET_CURRENT_CACHED_VALUES(1); - stack_pointer += -1 - oparg; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + SET_CURRENT_CACHED_VALUES(0); assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); break; } diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index db7ee9e00914..e3ffd89ebb10 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -3798,7 +3798,7 @@ _PyStackRef callable; _PyStackRef self_or_null; _PyStackRef *args; - _PyStackRef res; + _PyStackRef value; /* Skip 1 cache entry */ /* Skip 2 cache entries */ // _GUARD_CALLABLE_METHOD_DESCRIPTOR_FAST @@ -3850,7 +3850,7 @@ STAT_INC(CALL, hit); _PyFrame_SetStackPointer(frame, stack_pointer); PyCFunctionFast cfunc = _PyCFunctionFast_CAST(method->d_method->ml_meth); - PyObject *res_o = _PyCallMethodDescriptorFast_StackRefSteal( + PyObject *res_o = _PyCallMethodDescriptorFast_StackRef( callable, cfunc, self, @@ -3859,17 +3859,32 @@ ); stack_pointer = _PyFrame_GetStackPointer(frame); if (res_o == NULL) { - stack_pointer += -2 - oparg; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); JUMP_TO_LABEL(error); } - res = PyStackRef_FromPyObjectSteal(res_o); + _PyStackRef temp = callable; + callable = PyStackRef_FromPyObjectSteal(res_o); + stack_pointer[-2 - oparg] = callable; + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(temp); + stack_pointer = _PyFrame_GetStackPointer(frame); } - // _CHECK_PERIODIC_AT_END + // _POP_TOP_OPARG { - stack_pointer[-2 - oparg] = res; + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyStackRef_CloseStack(args, oparg); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + // _POP_TOP + { + value = self_or_null; stack_pointer += -1 - oparg; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_XCLOSE(value); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + // _CHECK_PERIODIC_AT_END + { _PyFrame_SetStackPointer(frame, stack_pointer); int err = check_periodics(tstate); stack_pointer = _PyFrame_GetStackPointer(frame); @@ -3894,7 +3909,7 @@ _PyStackRef callable; _PyStackRef self_or_null; _PyStackRef *args; - _PyStackRef res; + _PyStackRef value; /* Skip 1 cache entry */ /* Skip 2 cache entries */ // _GUARD_CALLABLE_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS @@ -3947,7 +3962,7 @@ STAT_INC(CALL, hit); _PyFrame_SetStackPointer(frame, stack_pointer); PyCFunctionFastWithKeywords cfunc = _PyCFunctionFastWithKeywords_CAST(method->d_method->ml_meth); - PyObject *res_o = _PyCallMethodDescriptorFastWithKeywords_StackRefSteal( + PyObject *res_o = _PyCallMethodDescriptorFastWithKeywords_StackRef( callable, cfunc, self, @@ -3956,17 +3971,32 @@ ); stack_pointer = _PyFrame_GetStackPointer(frame); if (res_o == NULL) { - stack_pointer += -2 - oparg; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); JUMP_TO_LABEL(error); } - res = PyStackRef_FromPyObjectSteal(res_o); + _PyStackRef temp = callable; + callable = PyStackRef_FromPyObjectSteal(res_o); + stack_pointer[-2 - oparg] = callable; + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(temp); + stack_pointer = _PyFrame_GetStackPointer(frame); } - // _CHECK_PERIODIC_AT_END + // _POP_TOP_OPARG { - stack_pointer[-2 - oparg] = res; + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyStackRef_CloseStack(args, oparg); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + // _POP_TOP + { + value = self_or_null; stack_pointer += -1 - oparg; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_XCLOSE(value); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + // _CHECK_PERIODIC_AT_END + { _PyFrame_SetStackPointer(frame, stack_pointer); int err = check_periodics(tstate); stack_pointer = _PyFrame_GetStackPointer(frame); diff --git a/Python/optimizer_bytecodes.c b/Python/optimizer_bytecodes.c index 0708fed6c5c8..5d8f1d2c249c 100644 --- a/Python/optimizer_bytecodes.c +++ b/Python/optimizer_bytecodes.c @@ -1467,26 +1467,26 @@ dummy_func(void) { } } - op(_CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS, (callable, self_or_null, args[oparg] -- res)) { + op(_CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS, (callable, self_or_null, args[oparg] -- callable, self_or_null, args[oparg])) { PyObject *callable_o = sym_get_const(ctx, callable); if (callable_o && Py_IS_TYPE(callable_o, &PyMethodDescr_Type) && sym_is_not_null(self_or_null)) { PyMethodDescrObject *method = (PyMethodDescrObject *)callable_o; PyCFunction cfunc = method->d_method->ml_meth; - ADD_OP(_CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS_INLINE, oparg + 1, (uintptr_t)cfunc); + ADD_OP(_CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS_INLINE, oparg, (uintptr_t)cfunc); } - res = sym_new_not_null(ctx); + callable = sym_new_not_null(ctx); } - op(_CALL_METHOD_DESCRIPTOR_FAST, (callable, self_or_null, args[oparg] -- res)) { + op(_CALL_METHOD_DESCRIPTOR_FAST, (callable, self_or_null, args[oparg] -- callable, self_or_null, args[oparg])) { PyObject *callable_o = sym_get_const(ctx, callable); if (callable_o && Py_IS_TYPE(callable_o, &PyMethodDescr_Type) && sym_is_not_null(self_or_null)) { PyMethodDescrObject *method = (PyMethodDescrObject *)callable_o; PyCFunction cfunc = method->d_method->ml_meth; - ADD_OP(_CALL_METHOD_DESCRIPTOR_FAST_INLINE, oparg + 1, (uintptr_t)cfunc); + ADD_OP(_CALL_METHOD_DESCRIPTOR_FAST_INLINE, oparg, (uintptr_t)cfunc); } - res = sym_new_not_null(ctx); + callable = sym_new_not_null(ctx); } op(_GUARD_CALLABLE_METHOD_DESCRIPTOR_FAST, (callable, self_or_null, args[oparg] -- callable, self_or_null, args[oparg])) { diff --git a/Python/optimizer_cases.c.h b/Python/optimizer_cases.c.h index 93b60f64e14a..d3967a32ad3d 100644 --- a/Python/optimizer_cases.c.h +++ b/Python/optimizer_cases.c.h @@ -4318,7 +4318,6 @@ case _CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS: { JitOptRef self_or_null; JitOptRef callable; - JitOptRef res; self_or_null = stack_pointer[-1 - oparg]; callable = stack_pointer[-2 - oparg]; PyObject *callable_o = sym_get_const(ctx, callable); @@ -4326,23 +4325,14 @@ && sym_is_not_null(self_or_null)) { PyMethodDescrObject *method = (PyMethodDescrObject *)callable_o; PyCFunction cfunc = method->d_method->ml_meth; - ADD_OP(_CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS_INLINE, oparg + 1, (uintptr_t)cfunc); + ADD_OP(_CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS_INLINE, oparg, (uintptr_t)cfunc); } - res = sym_new_not_null(ctx); - CHECK_STACK_BOUNDS(-1 - oparg); - stack_pointer[-2 - oparg] = res; - stack_pointer += -1 - oparg; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + callable = sym_new_not_null(ctx); + stack_pointer[-2 - oparg] = callable; break; } case _CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS_INLINE: { - JitOptRef res; - res = sym_new_not_null(ctx); - CHECK_STACK_BOUNDS(-oparg); - stack_pointer[-1 - oparg] = res; - stack_pointer += -oparg; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -4471,7 +4461,6 @@ case _CALL_METHOD_DESCRIPTOR_FAST: { JitOptRef self_or_null; JitOptRef callable; - JitOptRef res; self_or_null = stack_pointer[-1 - oparg]; callable = stack_pointer[-2 - oparg]; PyObject *callable_o = sym_get_const(ctx, callable); @@ -4479,23 +4468,14 @@ && sym_is_not_null(self_or_null)) { PyMethodDescrObject *method = (PyMethodDescrObject *)callable_o; PyCFunction cfunc = method->d_method->ml_meth; - ADD_OP(_CALL_METHOD_DESCRIPTOR_FAST_INLINE, oparg + 1, (uintptr_t)cfunc); + ADD_OP(_CALL_METHOD_DESCRIPTOR_FAST_INLINE, oparg, (uintptr_t)cfunc); } - res = sym_new_not_null(ctx); - CHECK_STACK_BOUNDS(-1 - oparg); - stack_pointer[-2 - oparg] = res; - stack_pointer += -1 - oparg; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + callable = sym_new_not_null(ctx); + stack_pointer[-2 - oparg] = callable; break; } case _CALL_METHOD_DESCRIPTOR_FAST_INLINE: { - JitOptRef res; - res = sym_new_not_null(ctx); - CHECK_STACK_BOUNDS(-oparg); - stack_pointer[-1 - oparg] = res; - stack_pointer += -oparg; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; }