Includes a slight improvement to `DECREF_INPUTS()`.
assert(oparg & 1);
}
- // stack effect: (__0, __array[oparg] -- )
- inst(CALL_BOUND_METHOD_EXACT_ARGS) {
- DEOPT_IF(is_method(stack_pointer, oparg), CALL);
- PyObject *function = PEEK(oparg + 1);
- DEOPT_IF(Py_TYPE(function) != &PyMethod_Type, CALL);
- STAT_INC(CALL, hit);
- PyObject *self = ((PyMethodObject *)function)->im_self;
- PEEK(oparg + 1) = Py_NewRef(self);
- PyObject *meth = ((PyMethodObject *)function)->im_func;
- PEEK(oparg + 2) = Py_NewRef(meth);
- Py_DECREF(function);
- GO_TO_INSTRUCTION(CALL_PY_EXACT_ARGS);
- }
-
inst(KW_NAMES, (--)) {
assert(kwnames == NULL);
assert(oparg < PyTuple_GET_SIZE(consts));
kwnames = GETITEM(consts, oparg);
}
- // stack effect: (__0, __array[oparg] -- )
- inst(CALL) {
+ // Cache layout: counter/1, func_version/2, min_args/1
+ // Neither CALL_INTRINSIC_1 nor CALL_FUNCTION_EX are members!
+ family(call, INLINE_CACHE_ENTRIES_CALL) = {
+ CALL,
+ CALL_BOUND_METHOD_EXACT_ARGS,
+ CALL_PY_EXACT_ARGS,
+ CALL_PY_WITH_DEFAULTS,
+ CALL_NO_KW_TYPE_1,
+ CALL_NO_KW_STR_1,
+ CALL_NO_KW_TUPLE_1,
+ CALL_BUILTIN_CLASS,
+ CALL_NO_KW_BUILTIN_O,
+ CALL_NO_KW_BUILTIN_FAST,
+ CALL_BUILTIN_FAST_WITH_KEYWORDS,
+ CALL_NO_KW_LEN,
+ CALL_NO_KW_ISINSTANCE,
+ CALL_NO_KW_LIST_APPEND,
+ CALL_NO_KW_METHOD_DESCRIPTOR_O,
+ CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS,
+ CALL_NO_KW_METHOD_DESCRIPTOR_NOARGS,
+ CALL_NO_KW_METHOD_DESCRIPTOR_FAST,
+ };
+
+ // On entry, the stack is either
+ // [NULL, callable, arg1, arg2, ...]
+ // or
+ // [method, self, arg1, arg2, ...]
+ // (Some args may be keywords, see KW_NAMES, which sets 'kwnames'.)
+ // On exit, the stack is [result].
+ // When calling Python, inline the call using DISPATCH_INLINED().
+ inst(CALL, (unused/1, unused/2, unused/1, method, callable, args[oparg] -- res)) {
+ int is_meth = method != NULL;
+ int total_args = oparg;
+ if (is_meth) {
+ callable = method;
+ args--;
+ total_args++;
+ }
#if ENABLE_SPECIALIZATION
_PyCallCache *cache = (_PyCallCache *)next_instr;
if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) {
assert(cframe.use_tracing == 0);
- int is_meth = is_method(stack_pointer, oparg);
- int nargs = oparg + is_meth;
- PyObject *callable = PEEK(nargs + 1);
next_instr--;
- _Py_Specialize_Call(callable, next_instr, nargs, kwnames);
+ _Py_Specialize_Call(callable, next_instr, total_args, kwnames);
DISPATCH_SAME_OPARG();
}
STAT_INC(CALL, deferred);
DECREMENT_ADAPTIVE_COUNTER(cache->counter);
#endif /* ENABLE_SPECIALIZATION */
- int total_args, is_meth;
- is_meth = is_method(stack_pointer, oparg);
- PyObject *function = PEEK(oparg + 1);
- if (!is_meth && Py_TYPE(function) == &PyMethod_Type) {
- PyObject *self = ((PyMethodObject *)function)->im_self;
- PEEK(oparg+1) = Py_NewRef(self);
- PyObject *meth = ((PyMethodObject *)function)->im_func;
- PEEK(oparg+2) = Py_NewRef(meth);
- Py_DECREF(function);
- is_meth = 1;
- }
- total_args = oparg + is_meth;
- function = PEEK(total_args + 1);
+ if (!is_meth && Py_TYPE(callable) == &PyMethod_Type) {
+ is_meth = 1; // For consistenct; it's dead, though
+ args--;
+ total_args++;
+ PyObject *self = ((PyMethodObject *)callable)->im_self;
+ args[0] = Py_NewRef(self);
+ method = ((PyMethodObject *)callable)->im_func;
+ args[-1] = Py_NewRef(method);
+ Py_DECREF(callable);
+ callable = method;
+ }
int positional_args = total_args - KWNAMES_LEN();
// Check if the call can be inlined or not
- if (Py_TYPE(function) == &PyFunction_Type &&
+ if (Py_TYPE(callable) == &PyFunction_Type &&
tstate->interp->eval_frame == NULL &&
- ((PyFunctionObject *)function)->vectorcall == _PyFunction_Vectorcall)
+ ((PyFunctionObject *)callable)->vectorcall == _PyFunction_Vectorcall)
{
- int code_flags = ((PyCodeObject*)PyFunction_GET_CODE(function))->co_flags;
- PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(function));
- STACK_SHRINK(total_args);
+ int code_flags = ((PyCodeObject*)PyFunction_GET_CODE(callable))->co_flags;
+ PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable));
_PyInterpreterFrame *new_frame = _PyEvalFramePushAndInit(
- tstate, (PyFunctionObject *)function, locals,
- stack_pointer, positional_args, kwnames
+ tstate, (PyFunctionObject *)callable, locals,
+ args, positional_args, kwnames
);
kwnames = NULL;
- STACK_SHRINK(2-is_meth);
+ // Manipulate stack directly since we leave using DISPATCH_INLINED().
+ STACK_SHRINK(oparg + 2);
// The frame has stolen all the arguments from the stack,
// so there is no need to clean them up.
if (new_frame == NULL) {
DISPATCH_INLINED(new_frame);
}
/* Callable is not a normal Python function */
- PyObject *res;
if (cframe.use_tracing) {
res = trace_call_function(
- tstate, function, stack_pointer-total_args,
+ tstate, callable, args,
positional_args, kwnames);
}
else {
res = PyObject_Vectorcall(
- function, stack_pointer-total_args,
+ callable, args,
positional_args | PY_VECTORCALL_ARGUMENTS_OFFSET,
kwnames);
}
kwnames = NULL;
assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL));
- Py_DECREF(function);
- /* Clear the stack */
- STACK_SHRINK(total_args);
+ Py_DECREF(callable);
for (int i = 0; i < total_args; i++) {
- Py_DECREF(stack_pointer[i]);
+ Py_DECREF(args[i]);
}
- STACK_SHRINK(2-is_meth);
- PUSH(res);
- if (res == NULL) {
- goto error;
- }
- JUMPBY(INLINE_CACHE_ENTRIES_CALL);
+ ERROR_IF(res == NULL, error);
CHECK_EVAL_BREAKER();
}
- // stack effect: (__0, __array[oparg] -- )
- inst(CALL_PY_EXACT_ARGS) {
+ // Start out with [NULL, bound_method, arg1, arg2, ...]
+ // Transform to [callable, self, arg1, arg2, ...]
+ // Then fall through to CALL_PY_EXACT_ARGS
+ inst(CALL_BOUND_METHOD_EXACT_ARGS, (unused/1, unused/2, unused/1, method, callable, unused[oparg] -- unused)) {
+ DEOPT_IF(method != NULL, CALL);
+ DEOPT_IF(Py_TYPE(callable) != &PyMethod_Type, CALL);
+ STAT_INC(CALL, hit);
+ PyObject *self = ((PyMethodObject *)callable)->im_self;
+ PEEK(oparg + 1) = Py_NewRef(self); // callable
+ PyObject *meth = ((PyMethodObject *)callable)->im_func;
+ PEEK(oparg + 2) = Py_NewRef(meth); // method
+ Py_DECREF(callable);
+ GO_TO_INSTRUCTION(CALL_PY_EXACT_ARGS);
+ }
+
+ inst(CALL_PY_EXACT_ARGS, (unused/1, func_version/2, unused/1, method, callable, args[oparg] -- unused)) {
assert(kwnames == NULL);
DEOPT_IF(tstate->interp->eval_frame, CALL);
- _PyCallCache *cache = (_PyCallCache *)next_instr;
- int is_meth = is_method(stack_pointer, oparg);
- int argcount = oparg + is_meth;
- PyObject *callable = PEEK(argcount + 1);
+ int is_meth = method != NULL;
+ int argcount = oparg;
+ if (is_meth) {
+ callable = method;
+ args--;
+ argcount++;
+ }
DEOPT_IF(!PyFunction_Check(callable), CALL);
PyFunctionObject *func = (PyFunctionObject *)callable;
- DEOPT_IF(func->func_version != read_u32(cache->func_version), CALL);
+ DEOPT_IF(func->func_version != func_version, CALL);
PyCodeObject *code = (PyCodeObject *)func->func_code;
DEOPT_IF(code->co_argcount != argcount, CALL);
DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize), CALL);
STAT_INC(CALL, hit);
_PyInterpreterFrame *new_frame = _PyFrame_PushUnchecked(tstate, func, argcount);
- STACK_SHRINK(argcount);
for (int i = 0; i < argcount; i++) {
- new_frame->localsplus[i] = stack_pointer[i];
+ new_frame->localsplus[i] = args[i];
}
- STACK_SHRINK(2-is_meth);
+ // Manipulate stack directly since we leave using DISPATCH_INLINED().
+ STACK_SHRINK(oparg + 2);
JUMPBY(INLINE_CACHE_ENTRIES_CALL);
DISPATCH_INLINED(new_frame);
}
- // stack effect: (__0, __array[oparg] -- )
- inst(CALL_PY_WITH_DEFAULTS) {
+ inst(CALL_PY_WITH_DEFAULTS, (unused/1, func_version/2, min_args/1, method, callable, args[oparg] -- unused)) {
assert(kwnames == NULL);
DEOPT_IF(tstate->interp->eval_frame, CALL);
- _PyCallCache *cache = (_PyCallCache *)next_instr;
- int is_meth = is_method(stack_pointer, oparg);
- int argcount = oparg + is_meth;
- PyObject *callable = PEEK(argcount + 1);
+ int is_meth = method != NULL;
+ int argcount = oparg;
+ if (is_meth) {
+ callable = method;
+ args--;
+ argcount++;
+ }
DEOPT_IF(!PyFunction_Check(callable), CALL);
PyFunctionObject *func = (PyFunctionObject *)callable;
- DEOPT_IF(func->func_version != read_u32(cache->func_version), CALL);
+ DEOPT_IF(func->func_version != func_version, CALL);
PyCodeObject *code = (PyCodeObject *)func->func_code;
DEOPT_IF(argcount > code->co_argcount, CALL);
- int minargs = cache->min_args;
- DEOPT_IF(argcount < minargs, CALL);
+ DEOPT_IF(argcount < min_args, CALL);
DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize), CALL);
STAT_INC(CALL, hit);
_PyInterpreterFrame *new_frame = _PyFrame_PushUnchecked(tstate, func, code->co_argcount);
- STACK_SHRINK(argcount);
for (int i = 0; i < argcount; i++) {
- new_frame->localsplus[i] = stack_pointer[i];
+ new_frame->localsplus[i] = args[i];
}
for (int i = argcount; i < code->co_argcount; i++) {
- PyObject *def = PyTuple_GET_ITEM(func->func_defaults,
- i - minargs);
+ PyObject *def = PyTuple_GET_ITEM(func->func_defaults, i - min_args);
new_frame->localsplus[i] = Py_NewRef(def);
}
- STACK_SHRINK(2-is_meth);
+ // Manipulate stack and cache directly since we leave using DISPATCH_INLINED().
+ STACK_SHRINK(oparg + 2);
JUMPBY(INLINE_CACHE_ENTRIES_CALL);
DISPATCH_INLINED(new_frame);
}
- // stack effect: (__0, __array[oparg] -- )
- inst(CALL_NO_KW_TYPE_1) {
+ inst(CALL_NO_KW_TYPE_1, (unused/1, unused/2, unused/1, null, callable, args[oparg] -- res)) {
assert(kwnames == NULL);
assert(cframe.use_tracing == 0);
assert(oparg == 1);
- DEOPT_IF(is_method(stack_pointer, 1), CALL);
- PyObject *obj = TOP();
- PyObject *callable = SECOND();
+ DEOPT_IF(null != NULL, CALL);
+ PyObject *obj = args[0];
DEOPT_IF(callable != (PyObject *)&PyType_Type, CALL);
STAT_INC(CALL, hit);
- JUMPBY(INLINE_CACHE_ENTRIES_CALL);
- PyObject *res = Py_NewRef(Py_TYPE(obj));
- Py_DECREF(callable);
+ res = Py_NewRef(Py_TYPE(obj));
Py_DECREF(obj);
- STACK_SHRINK(2);
- SET_TOP(res);
+ Py_DECREF(&PyType_Type); // I.e., callable
}
- // stack effect: (__0, __array[oparg] -- )
- inst(CALL_NO_KW_STR_1) {
+ inst(CALL_NO_KW_STR_1, (unused/1, unused/2, unused/1, null, callable, args[oparg] -- res)) {
assert(kwnames == NULL);
assert(cframe.use_tracing == 0);
assert(oparg == 1);
- DEOPT_IF(is_method(stack_pointer, 1), CALL);
- PyObject *callable = PEEK(2);
+ DEOPT_IF(null != NULL, CALL);
DEOPT_IF(callable != (PyObject *)&PyUnicode_Type, CALL);
STAT_INC(CALL, hit);
- PyObject *arg = TOP();
- PyObject *res = PyObject_Str(arg);
+ PyObject *arg = args[0];
+ res = PyObject_Str(arg);
Py_DECREF(arg);
- Py_DECREF(&PyUnicode_Type);
- STACK_SHRINK(2);
- SET_TOP(res);
- if (res == NULL) {
- goto error;
- }
- JUMPBY(INLINE_CACHE_ENTRIES_CALL);
+ Py_DECREF(&PyUnicode_Type); // I.e., callable
+ ERROR_IF(res == NULL, error);
CHECK_EVAL_BREAKER();
}
- // stack effect: (__0, __array[oparg] -- )
- inst(CALL_NO_KW_TUPLE_1) {
+ inst(CALL_NO_KW_TUPLE_1, (unused/1, unused/2, unused/1, null, callable, args[oparg] -- res)) {
assert(kwnames == NULL);
assert(oparg == 1);
- DEOPT_IF(is_method(stack_pointer, 1), CALL);
- PyObject *callable = PEEK(2);
+ DEOPT_IF(null != NULL, CALL);
DEOPT_IF(callable != (PyObject *)&PyTuple_Type, CALL);
STAT_INC(CALL, hit);
- PyObject *arg = TOP();
- PyObject *res = PySequence_Tuple(arg);
+ PyObject *arg = args[0];
+ res = PySequence_Tuple(arg);
Py_DECREF(arg);
- Py_DECREF(&PyTuple_Type);
- STACK_SHRINK(2);
- SET_TOP(res);
- if (res == NULL) {
- goto error;
- }
- JUMPBY(INLINE_CACHE_ENTRIES_CALL);
+ Py_DECREF(&PyTuple_Type); // I.e., tuple
+ ERROR_IF(res == NULL, error);
CHECK_EVAL_BREAKER();
}
- // stack effect: (__0, __array[oparg] -- )
- inst(CALL_BUILTIN_CLASS) {
- int is_meth = is_method(stack_pointer, oparg);
- int total_args = oparg + is_meth;
+ inst(CALL_BUILTIN_CLASS, (unused/1, unused/2, unused/1, method, callable, args[oparg] -- res)) {
+ int is_meth = method != NULL;
+ int total_args = oparg;
+ if (is_meth) {
+ callable = method;
+ args--;
+ total_args++;
+ }
int kwnames_len = KWNAMES_LEN();
- PyObject *callable = PEEK(total_args + 1);
DEOPT_IF(!PyType_Check(callable), CALL);
PyTypeObject *tp = (PyTypeObject *)callable;
DEOPT_IF(tp->tp_vectorcall == NULL, CALL);
STAT_INC(CALL, hit);
- STACK_SHRINK(total_args);
- PyObject *res = tp->tp_vectorcall((PyObject *)tp, stack_pointer,
- total_args-kwnames_len, kwnames);
+ res = tp->tp_vectorcall((PyObject *)tp, args,
+ total_args - kwnames_len, kwnames);
kwnames = NULL;
/* Free the arguments. */
for (int i = 0; i < total_args; i++) {
- Py_DECREF(stack_pointer[i]);
+ Py_DECREF(args[i]);
}
Py_DECREF(tp);
- STACK_SHRINK(1-is_meth);
- SET_TOP(res);
- if (res == NULL) {
- goto error;
- }
- JUMPBY(INLINE_CACHE_ENTRIES_CALL);
+ ERROR_IF(res == NULL, error);
CHECK_EVAL_BREAKER();
}
- // stack effect: (__0, __array[oparg] -- )
- inst(CALL_NO_KW_BUILTIN_O) {
+ inst(CALL_NO_KW_BUILTIN_O, (unused/1, unused/2, unused/1, method, callable, args[oparg] -- res)) {
assert(cframe.use_tracing == 0);
/* Builtin METH_O functions */
assert(kwnames == NULL);
- int is_meth = is_method(stack_pointer, oparg);
- int total_args = oparg + is_meth;
+ int is_meth = method != NULL;
+ int total_args = oparg;
+ if (is_meth) {
+ callable = method;
+ args--;
+ total_args++;
+ }
DEOPT_IF(total_args != 1, CALL);
- PyObject *callable = PEEK(total_args + 1);
DEOPT_IF(!PyCFunction_CheckExact(callable), CALL);
DEOPT_IF(PyCFunction_GET_FLAGS(callable) != METH_O, CALL);
STAT_INC(CALL, hit);
if (_Py_EnterRecursiveCallTstate(tstate, " while calling a Python object")) {
goto error;
}
- PyObject *arg = TOP();
- PyObject *res = _PyCFunction_TrampolineCall(cfunc, PyCFunction_GET_SELF(callable), arg);
+ PyObject *arg = args[0];
+ res = _PyCFunction_TrampolineCall(cfunc, PyCFunction_GET_SELF(callable), arg);
_Py_LeaveRecursiveCallTstate(tstate);
assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL));
Py_DECREF(arg);
Py_DECREF(callable);
- STACK_SHRINK(2-is_meth);
- SET_TOP(res);
- if (res == NULL) {
- goto error;
- }
- JUMPBY(INLINE_CACHE_ENTRIES_CALL);
+ ERROR_IF(res == NULL, error);
CHECK_EVAL_BREAKER();
}
- // stack effect: (__0, __array[oparg] -- )
- inst(CALL_NO_KW_BUILTIN_FAST) {
+ inst(CALL_NO_KW_BUILTIN_FAST, (unused/1, unused/2, unused/1, method, callable, args[oparg] -- res)) {
assert(cframe.use_tracing == 0);
/* Builtin METH_FASTCALL functions, without keywords */
assert(kwnames == NULL);
- int is_meth = is_method(stack_pointer, oparg);
- int total_args = oparg + is_meth;
- PyObject *callable = PEEK(total_args + 1);
+ int is_meth = method != NULL;
+ int total_args = oparg;
+ if (is_meth) {
+ callable = method;
+ args--;
+ total_args++;
+ }
DEOPT_IF(!PyCFunction_CheckExact(callable), CALL);
- DEOPT_IF(PyCFunction_GET_FLAGS(callable) != METH_FASTCALL,
- CALL);
+ DEOPT_IF(PyCFunction_GET_FLAGS(callable) != METH_FASTCALL, CALL);
STAT_INC(CALL, hit);
PyCFunction cfunc = PyCFunction_GET_FUNCTION(callable);
- STACK_SHRINK(total_args);
/* res = func(self, args, nargs) */
- PyObject *res = ((_PyCFunctionFast)(void(*)(void))cfunc)(
+ res = ((_PyCFunctionFast)(void(*)(void))cfunc)(
PyCFunction_GET_SELF(callable),
- stack_pointer,
+ args,
total_args);
assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL));
/* Free the arguments. */
for (int i = 0; i < total_args; i++) {
- Py_DECREF(stack_pointer[i]);
+ Py_DECREF(args[i]);
}
- STACK_SHRINK(2-is_meth);
- PUSH(res);
Py_DECREF(callable);
- if (res == NULL) {
+ ERROR_IF(res == NULL, error);
/* Not deopting because this doesn't mean our optimization was
wrong. `res` can be NULL for valid reasons. Eg. getattr(x,
'invalid'). In those cases an exception is set, so we must
handle it.
*/
- goto error;
- }
- JUMPBY(INLINE_CACHE_ENTRIES_CALL);
CHECK_EVAL_BREAKER();
}
- // stack effect: (__0, __array[oparg] -- )
- inst(CALL_BUILTIN_FAST_WITH_KEYWORDS) {
+ inst(CALL_BUILTIN_FAST_WITH_KEYWORDS, (unused/1, unused/2, unused/1, method, callable, args[oparg] -- res)) {
assert(cframe.use_tracing == 0);
/* Builtin METH_FASTCALL | METH_KEYWORDS functions */
- int is_meth = is_method(stack_pointer, oparg);
- int total_args = oparg + is_meth;
- PyObject *callable = PEEK(total_args + 1);
+ int is_meth = method != NULL;
+ int total_args = oparg;
+ if (is_meth) {
+ callable = method;
+ args--;
+ total_args++;
+ }
DEOPT_IF(!PyCFunction_CheckExact(callable), CALL);
DEOPT_IF(PyCFunction_GET_FLAGS(callable) !=
(METH_FASTCALL | METH_KEYWORDS), CALL);
STAT_INC(CALL, hit);
- STACK_SHRINK(total_args);
/* res = func(self, args, nargs, kwnames) */
_PyCFunctionFastWithKeywords cfunc =
(_PyCFunctionFastWithKeywords)(void(*)(void))
PyCFunction_GET_FUNCTION(callable);
- PyObject *res = cfunc(
+ res = cfunc(
PyCFunction_GET_SELF(callable),
- stack_pointer,
+ args,
total_args - KWNAMES_LEN(),
kwnames
);
/* Free the arguments. */
for (int i = 0; i < total_args; i++) {
- Py_DECREF(stack_pointer[i]);
+ Py_DECREF(args[i]);
}
- STACK_SHRINK(2-is_meth);
- PUSH(res);
Py_DECREF(callable);
- if (res == NULL) {
- goto error;
- }
- JUMPBY(INLINE_CACHE_ENTRIES_CALL);
+ ERROR_IF(res == NULL, error);
CHECK_EVAL_BREAKER();
}
- // stack effect: (__0, __array[oparg] -- )
- inst(CALL_NO_KW_LEN) {
+ inst(CALL_NO_KW_LEN, (unused/1, unused/2, unused/1, method, callable, args[oparg] -- res)) {
assert(cframe.use_tracing == 0);
assert(kwnames == NULL);
/* len(o) */
- int is_meth = is_method(stack_pointer, oparg);
- int total_args = oparg + is_meth;
+ int is_meth = method != NULL;
+ int total_args = oparg;
+ if (is_meth) {
+ callable = method;
+ args--;
+ total_args++;
+ }
DEOPT_IF(total_args != 1, CALL);
- PyObject *callable = PEEK(total_args + 1);
PyInterpreterState *interp = _PyInterpreterState_GET();
DEOPT_IF(callable != interp->callable_cache.len, CALL);
STAT_INC(CALL, hit);
- PyObject *arg = TOP();
+ PyObject *arg = args[0];
Py_ssize_t len_i = PyObject_Length(arg);
if (len_i < 0) {
goto error;
}
- PyObject *res = PyLong_FromSsize_t(len_i);
+ res = PyLong_FromSsize_t(len_i);
assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL));
- STACK_SHRINK(2-is_meth);
- SET_TOP(res);
Py_DECREF(callable);
Py_DECREF(arg);
- if (res == NULL) {
- goto error;
- }
- JUMPBY(INLINE_CACHE_ENTRIES_CALL);
+ ERROR_IF(res == NULL, error);
}
- // stack effect: (__0, __array[oparg] -- )
- inst(CALL_NO_KW_ISINSTANCE) {
+ inst(CALL_NO_KW_ISINSTANCE, (unused/1, unused/2, unused/1, method, callable, args[oparg] -- res)) {
assert(cframe.use_tracing == 0);
assert(kwnames == NULL);
/* isinstance(o, o2) */
- int is_meth = is_method(stack_pointer, oparg);
- int total_args = oparg + is_meth;
- PyObject *callable = PEEK(total_args + 1);
+ int is_meth = method != NULL;
+ int total_args = oparg;
+ if (is_meth) {
+ callable = method;
+ args--;
+ total_args++;
+ }
DEOPT_IF(total_args != 2, CALL);
PyInterpreterState *interp = _PyInterpreterState_GET();
DEOPT_IF(callable != interp->callable_cache.isinstance, CALL);
STAT_INC(CALL, hit);
- PyObject *cls = POP();
- PyObject *inst = TOP();
+ PyObject *cls = args[1];
+ PyObject *inst = args[0];
int retval = PyObject_IsInstance(inst, cls);
if (retval < 0) {
- Py_DECREF(cls);
goto error;
}
- PyObject *res = PyBool_FromLong(retval);
+ res = PyBool_FromLong(retval);
assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL));
- STACK_SHRINK(2-is_meth);
- SET_TOP(res);
Py_DECREF(inst);
Py_DECREF(cls);
Py_DECREF(callable);
- if (res == NULL) {
- goto error;
- }
- JUMPBY(INLINE_CACHE_ENTRIES_CALL);
+ ERROR_IF(res == NULL, error);
}
- // stack effect: (__0, __array[oparg] -- )
- inst(CALL_NO_KW_LIST_APPEND) {
+ // This is secretly a super-instruction
+ inst(CALL_NO_KW_LIST_APPEND, (unused/1, unused/2, unused/1, method, self, args[oparg] -- unused)) {
assert(cframe.use_tracing == 0);
assert(kwnames == NULL);
assert(oparg == 1);
- PyObject *callable = PEEK(3);
+ assert(method != NULL);
PyInterpreterState *interp = _PyInterpreterState_GET();
- DEOPT_IF(callable != interp->callable_cache.list_append, CALL);
- PyObject *list = SECOND();
- DEOPT_IF(!PyList_Check(list), CALL);
+ DEOPT_IF(method != interp->callable_cache.list_append, CALL);
+ DEOPT_IF(!PyList_Check(self), CALL);
STAT_INC(CALL, hit);
- PyObject *arg = POP();
- if (_PyList_AppendTakeRef((PyListObject *)list, arg) < 0) {
- goto error;
+ if (_PyList_AppendTakeRef((PyListObject *)self, args[0]) < 0) {
+ goto pop_1_error; // Since arg is DECREF'ed already
}
- STACK_SHRINK(2);
- Py_DECREF(list);
- Py_DECREF(callable);
+ Py_DECREF(self);
+ Py_DECREF(method);
+ STACK_SHRINK(3);
// CALL + POP_TOP
JUMPBY(INLINE_CACHE_ENTRIES_CALL + 1);
assert(_Py_OPCODE(next_instr[-1]) == POP_TOP);
+ DISPATCH();
}
- // stack effect: (__0, __array[oparg] -- )
- inst(CALL_NO_KW_METHOD_DESCRIPTOR_O) {
+ inst(CALL_NO_KW_METHOD_DESCRIPTOR_O, (unused/1, unused/2, unused/1, method, unused, args[oparg] -- res)) {
assert(kwnames == NULL);
- int is_meth = is_method(stack_pointer, oparg);
- int total_args = oparg + is_meth;
+ int is_meth = method != NULL;
+ int total_args = oparg;
+ if (is_meth) {
+ args--;
+ total_args++;
+ }
PyMethodDescrObject *callable =
(PyMethodDescrObject *)PEEK(total_args + 1);
DEOPT_IF(total_args != 2, CALL);
DEOPT_IF(!Py_IS_TYPE(callable, &PyMethodDescr_Type), CALL);
PyMethodDef *meth = callable->d_method;
DEOPT_IF(meth->ml_flags != METH_O, CALL);
- PyObject *arg = TOP();
- PyObject *self = SECOND();
+ PyObject *arg = args[1];
+ PyObject *self = args[0];
DEOPT_IF(!Py_IS_TYPE(self, callable->d_common.d_type), CALL);
STAT_INC(CALL, hit);
PyCFunction cfunc = meth->ml_meth;
if (_Py_EnterRecursiveCallTstate(tstate, " while calling a Python object")) {
goto error;
}
- PyObject *res = _PyCFunction_TrampolineCall(cfunc, self, arg);
+ res = _PyCFunction_TrampolineCall(cfunc, self, arg);
_Py_LeaveRecursiveCallTstate(tstate);
assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL));
Py_DECREF(self);
Py_DECREF(arg);
- STACK_SHRINK(oparg + 1);
- SET_TOP(res);
Py_DECREF(callable);
- if (res == NULL) {
- goto error;
- }
- JUMPBY(INLINE_CACHE_ENTRIES_CALL);
+ ERROR_IF(res == NULL, error);
CHECK_EVAL_BREAKER();
}
- // stack effect: (__0, __array[oparg] -- )
- inst(CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS) {
- int is_meth = is_method(stack_pointer, oparg);
- int total_args = oparg + is_meth;
+ inst(CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS, (unused/1, unused/2, unused/1, method, unused, args[oparg] -- res)) {
+ int is_meth = method != NULL;
+ int total_args = oparg;
+ if (is_meth) {
+ args--;
+ total_args++;
+ }
PyMethodDescrObject *callable =
(PyMethodDescrObject *)PEEK(total_args + 1);
DEOPT_IF(!Py_IS_TYPE(callable, &PyMethodDescr_Type), CALL);
PyMethodDef *meth = callable->d_method;
DEOPT_IF(meth->ml_flags != (METH_FASTCALL|METH_KEYWORDS), CALL);
PyTypeObject *d_type = callable->d_common.d_type;
- PyObject *self = PEEK(total_args);
+ PyObject *self = args[0];
DEOPT_IF(!Py_IS_TYPE(self, d_type), CALL);
STAT_INC(CALL, hit);
- int nargs = total_args-1;
- STACK_SHRINK(nargs);
+ int nargs = total_args - 1;
_PyCFunctionFastWithKeywords cfunc =
(_PyCFunctionFastWithKeywords)(void(*)(void))meth->ml_meth;
- PyObject *res = cfunc(self, stack_pointer, nargs - KWNAMES_LEN(),
- kwnames);
+ res = cfunc(self, args + 1, nargs - KWNAMES_LEN(), kwnames);
assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL));
kwnames = NULL;
/* Free the arguments. */
- for (int i = 0; i < nargs; i++) {
- Py_DECREF(stack_pointer[i]);
+ for (int i = 0; i < total_args; i++) {
+ Py_DECREF(args[i]);
}
- Py_DECREF(self);
- STACK_SHRINK(2-is_meth);
- SET_TOP(res);
Py_DECREF(callable);
- if (res == NULL) {
- goto error;
- }
- JUMPBY(INLINE_CACHE_ENTRIES_CALL);
+ ERROR_IF(res == NULL, error);
CHECK_EVAL_BREAKER();
}
- // stack effect: (__0, __array[oparg] -- )
- inst(CALL_NO_KW_METHOD_DESCRIPTOR_NOARGS) {
+ inst(CALL_NO_KW_METHOD_DESCRIPTOR_NOARGS, (unused/1, unused/2, unused/1, method, unused, args[oparg] -- res)) {
assert(kwnames == NULL);
assert(oparg == 0 || oparg == 1);
- int is_meth = is_method(stack_pointer, oparg);
- int total_args = oparg + is_meth;
+ int is_meth = method != NULL;
+ int total_args = oparg;
+ if (is_meth) {
+ args--;
+ total_args++;
+ }
DEOPT_IF(total_args != 1, CALL);
PyMethodDescrObject *callable = (PyMethodDescrObject *)SECOND();
DEOPT_IF(!Py_IS_TYPE(callable, &PyMethodDescr_Type), CALL);
PyMethodDef *meth = callable->d_method;
- PyObject *self = TOP();
+ PyObject *self = args[0];
DEOPT_IF(!Py_IS_TYPE(self, callable->d_common.d_type), CALL);
DEOPT_IF(meth->ml_flags != METH_NOARGS, CALL);
STAT_INC(CALL, hit);
if (_Py_EnterRecursiveCallTstate(tstate, " while calling a Python object")) {
goto error;
}
- PyObject *res = _PyCFunction_TrampolineCall(cfunc, self, NULL);
+ res = _PyCFunction_TrampolineCall(cfunc, self, NULL);
_Py_LeaveRecursiveCallTstate(tstate);
assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL));
Py_DECREF(self);
- STACK_SHRINK(oparg + 1);
- SET_TOP(res);
Py_DECREF(callable);
- if (res == NULL) {
- goto error;
- }
- JUMPBY(INLINE_CACHE_ENTRIES_CALL);
+ ERROR_IF(res == NULL, error);
CHECK_EVAL_BREAKER();
}
- // stack effect: (__0, __array[oparg] -- )
- inst(CALL_NO_KW_METHOD_DESCRIPTOR_FAST) {
+ inst(CALL_NO_KW_METHOD_DESCRIPTOR_FAST, (unused/1, unused/2, unused/1, method, unused, args[oparg] -- res)) {
assert(kwnames == NULL);
- int is_meth = is_method(stack_pointer, oparg);
- int total_args = oparg + is_meth;
+ int is_meth = method != NULL;
+ int total_args = oparg;
+ if (is_meth) {
+ args--;
+ total_args++;
+ }
PyMethodDescrObject *callable =
(PyMethodDescrObject *)PEEK(total_args + 1);
/* Builtin METH_FASTCALL methods, without keywords */
DEOPT_IF(!Py_IS_TYPE(callable, &PyMethodDescr_Type), CALL);
PyMethodDef *meth = callable->d_method;
DEOPT_IF(meth->ml_flags != METH_FASTCALL, CALL);
- PyObject *self = PEEK(total_args);
+ PyObject *self = args[0];
DEOPT_IF(!Py_IS_TYPE(self, callable->d_common.d_type), CALL);
STAT_INC(CALL, hit);
_PyCFunctionFast cfunc =
(_PyCFunctionFast)(void(*)(void))meth->ml_meth;
- int nargs = total_args-1;
- STACK_SHRINK(nargs);
- PyObject *res = cfunc(self, stack_pointer, nargs);
+ int nargs = total_args - 1;
+ res = cfunc(self, args + 1, nargs);
assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL));
/* Clear the stack of the arguments. */
- for (int i = 0; i < nargs; i++) {
- Py_DECREF(stack_pointer[i]);
+ for (int i = 0; i < total_args; i++) {
+ Py_DECREF(args[i]);
}
- Py_DECREF(self);
- STACK_SHRINK(2-is_meth);
- SET_TOP(res);
Py_DECREF(callable);
- if (res == NULL) {
- goto error;
- }
- JUMPBY(INLINE_CACHE_ENTRIES_CALL);
+ ERROR_IF(res == NULL, error);
CHECK_EVAL_BREAKER();
}
// Future families go below this point //
-family(call, INLINE_CACHE_ENTRIES_CALL) = {
- CALL, CALL_PY_EXACT_ARGS,
- CALL_PY_WITH_DEFAULTS, CALL_BOUND_METHOD_EXACT_ARGS, CALL_BUILTIN_CLASS,
- CALL_BUILTIN_FAST_WITH_KEYWORDS, CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS, CALL_NO_KW_BUILTIN_FAST,
- CALL_NO_KW_BUILTIN_O, CALL_NO_KW_ISINSTANCE, CALL_NO_KW_LEN,
- CALL_NO_KW_LIST_APPEND, CALL_NO_KW_METHOD_DESCRIPTOR_FAST, CALL_NO_KW_METHOD_DESCRIPTOR_NOARGS,
- CALL_NO_KW_METHOD_DESCRIPTOR_O, CALL_NO_KW_STR_1, CALL_NO_KW_TUPLE_1,
- CALL_NO_KW_TYPE_1 };
family(store_fast) = { STORE_FAST, STORE_FAST__LOAD_FAST, STORE_FAST__STORE_FAST };
}
-// GH-89279: Must be a macro to be sure it's inlined by MSVC.
-#define is_method(stack_pointer, args) (PEEK((args)+2) != NULL)
-
-#define KWNAMES_LEN() \
- (kwnames == NULL ? 0 : ((int)PyTuple_GET_SIZE(kwnames)))
-
/* Disable unused label warnings. They are handy for debugging, even
if computed gotos aren't used. */
} while (0);
#define NAME_ERROR_MSG "name '%.200s' is not defined"
+
+#define KWNAMES_LEN() \
+ (kwnames == NULL ? 0 : ((int)PyTuple_GET_SIZE(kwnames)))
DISPATCH();
}
- TARGET(CALL_BOUND_METHOD_EXACT_ARGS) {
- DEOPT_IF(is_method(stack_pointer, oparg), CALL);
- PyObject *function = PEEK(oparg + 1);
- DEOPT_IF(Py_TYPE(function) != &PyMethod_Type, CALL);
- STAT_INC(CALL, hit);
- PyObject *self = ((PyMethodObject *)function)->im_self;
- PEEK(oparg + 1) = Py_NewRef(self);
- PyObject *meth = ((PyMethodObject *)function)->im_func;
- PEEK(oparg + 2) = Py_NewRef(meth);
- Py_DECREF(function);
- GO_TO_INSTRUCTION(CALL_PY_EXACT_ARGS);
- }
-
TARGET(KW_NAMES) {
assert(kwnames == NULL);
assert(oparg < PyTuple_GET_SIZE(consts));
TARGET(CALL) {
PREDICTED(CALL);
+ static_assert(INLINE_CACHE_ENTRIES_CALL == 4, "incorrect cache size");
+ PyObject **args = &PEEK(oparg);
+ PyObject *callable = PEEK(1 + oparg);
+ PyObject *method = PEEK(2 + oparg);
+ PyObject *res;
+ int is_meth = method != NULL;
+ int total_args = oparg;
+ if (is_meth) {
+ callable = method;
+ args--;
+ total_args++;
+ }
#if ENABLE_SPECIALIZATION
_PyCallCache *cache = (_PyCallCache *)next_instr;
if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) {
assert(cframe.use_tracing == 0);
- int is_meth = is_method(stack_pointer, oparg);
- int nargs = oparg + is_meth;
- PyObject *callable = PEEK(nargs + 1);
next_instr--;
- _Py_Specialize_Call(callable, next_instr, nargs, kwnames);
+ _Py_Specialize_Call(callable, next_instr, total_args, kwnames);
DISPATCH_SAME_OPARG();
}
STAT_INC(CALL, deferred);
DECREMENT_ADAPTIVE_COUNTER(cache->counter);
#endif /* ENABLE_SPECIALIZATION */
- int total_args, is_meth;
- is_meth = is_method(stack_pointer, oparg);
- PyObject *function = PEEK(oparg + 1);
- if (!is_meth && Py_TYPE(function) == &PyMethod_Type) {
- PyObject *self = ((PyMethodObject *)function)->im_self;
- PEEK(oparg+1) = Py_NewRef(self);
- PyObject *meth = ((PyMethodObject *)function)->im_func;
- PEEK(oparg+2) = Py_NewRef(meth);
- Py_DECREF(function);
- is_meth = 1;
- }
- total_args = oparg + is_meth;
- function = PEEK(total_args + 1);
+ if (!is_meth && Py_TYPE(callable) == &PyMethod_Type) {
+ is_meth = 1; // For consistenct; it's dead, though
+ args--;
+ total_args++;
+ PyObject *self = ((PyMethodObject *)callable)->im_self;
+ args[0] = Py_NewRef(self);
+ method = ((PyMethodObject *)callable)->im_func;
+ args[-1] = Py_NewRef(method);
+ Py_DECREF(callable);
+ callable = method;
+ }
int positional_args = total_args - KWNAMES_LEN();
// Check if the call can be inlined or not
- if (Py_TYPE(function) == &PyFunction_Type &&
+ if (Py_TYPE(callable) == &PyFunction_Type &&
tstate->interp->eval_frame == NULL &&
- ((PyFunctionObject *)function)->vectorcall == _PyFunction_Vectorcall)
+ ((PyFunctionObject *)callable)->vectorcall == _PyFunction_Vectorcall)
{
- int code_flags = ((PyCodeObject*)PyFunction_GET_CODE(function))->co_flags;
- PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(function));
- STACK_SHRINK(total_args);
+ int code_flags = ((PyCodeObject*)PyFunction_GET_CODE(callable))->co_flags;
+ PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable));
_PyInterpreterFrame *new_frame = _PyEvalFramePushAndInit(
- tstate, (PyFunctionObject *)function, locals,
- stack_pointer, positional_args, kwnames
+ tstate, (PyFunctionObject *)callable, locals,
+ args, positional_args, kwnames
);
kwnames = NULL;
- STACK_SHRINK(2-is_meth);
+ // Manipulate stack directly since we leave using DISPATCH_INLINED().
+ STACK_SHRINK(oparg + 2);
// The frame has stolen all the arguments from the stack,
// so there is no need to clean them up.
if (new_frame == NULL) {
DISPATCH_INLINED(new_frame);
}
/* Callable is not a normal Python function */
- PyObject *res;
if (cframe.use_tracing) {
res = trace_call_function(
- tstate, function, stack_pointer-total_args,
+ tstate, callable, args,
positional_args, kwnames);
}
else {
res = PyObject_Vectorcall(
- function, stack_pointer-total_args,
+ callable, args,
positional_args | PY_VECTORCALL_ARGUMENTS_OFFSET,
kwnames);
}
kwnames = NULL;
assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL));
- Py_DECREF(function);
- /* Clear the stack */
- STACK_SHRINK(total_args);
+ Py_DECREF(callable);
for (int i = 0; i < total_args; i++) {
- Py_DECREF(stack_pointer[i]);
+ Py_DECREF(args[i]);
}
- STACK_SHRINK(2-is_meth);
- PUSH(res);
- if (res == NULL) {
- goto error;
- }
- JUMPBY(INLINE_CACHE_ENTRIES_CALL);
+ if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; }
+ STACK_SHRINK(oparg);
+ STACK_SHRINK(1);
+ POKE(1, res);
+ JUMPBY(4);
CHECK_EVAL_BREAKER();
DISPATCH();
}
+ TARGET(CALL_BOUND_METHOD_EXACT_ARGS) {
+ PyObject *callable = PEEK(1 + oparg);
+ PyObject *method = PEEK(2 + oparg);
+ DEOPT_IF(method != NULL, CALL);
+ DEOPT_IF(Py_TYPE(callable) != &PyMethod_Type, CALL);
+ STAT_INC(CALL, hit);
+ PyObject *self = ((PyMethodObject *)callable)->im_self;
+ PEEK(oparg + 1) = Py_NewRef(self); // callable
+ PyObject *meth = ((PyMethodObject *)callable)->im_func;
+ PEEK(oparg + 2) = Py_NewRef(meth); // method
+ Py_DECREF(callable);
+ GO_TO_INSTRUCTION(CALL_PY_EXACT_ARGS);
+ }
+
TARGET(CALL_PY_EXACT_ARGS) {
PREDICTED(CALL_PY_EXACT_ARGS);
+ PyObject **args = &PEEK(oparg);
+ PyObject *callable = PEEK(1 + oparg);
+ PyObject *method = PEEK(2 + oparg);
+ uint32_t func_version = read_u32(&next_instr[1].cache);
assert(kwnames == NULL);
DEOPT_IF(tstate->interp->eval_frame, CALL);
- _PyCallCache *cache = (_PyCallCache *)next_instr;
- int is_meth = is_method(stack_pointer, oparg);
- int argcount = oparg + is_meth;
- PyObject *callable = PEEK(argcount + 1);
+ int is_meth = method != NULL;
+ int argcount = oparg;
+ if (is_meth) {
+ callable = method;
+ args--;
+ argcount++;
+ }
DEOPT_IF(!PyFunction_Check(callable), CALL);
PyFunctionObject *func = (PyFunctionObject *)callable;
- DEOPT_IF(func->func_version != read_u32(cache->func_version), CALL);
+ DEOPT_IF(func->func_version != func_version, CALL);
PyCodeObject *code = (PyCodeObject *)func->func_code;
DEOPT_IF(code->co_argcount != argcount, CALL);
DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize), CALL);
STAT_INC(CALL, hit);
_PyInterpreterFrame *new_frame = _PyFrame_PushUnchecked(tstate, func, argcount);
- STACK_SHRINK(argcount);
for (int i = 0; i < argcount; i++) {
- new_frame->localsplus[i] = stack_pointer[i];
+ new_frame->localsplus[i] = args[i];
}
- STACK_SHRINK(2-is_meth);
+ // Manipulate stack directly since we leave using DISPATCH_INLINED().
+ STACK_SHRINK(oparg + 2);
JUMPBY(INLINE_CACHE_ENTRIES_CALL);
DISPATCH_INLINED(new_frame);
}
TARGET(CALL_PY_WITH_DEFAULTS) {
+ PyObject **args = &PEEK(oparg);
+ PyObject *callable = PEEK(1 + oparg);
+ PyObject *method = PEEK(2 + oparg);
+ uint32_t func_version = read_u32(&next_instr[1].cache);
+ uint16_t min_args = read_u16(&next_instr[3].cache);
assert(kwnames == NULL);
DEOPT_IF(tstate->interp->eval_frame, CALL);
- _PyCallCache *cache = (_PyCallCache *)next_instr;
- int is_meth = is_method(stack_pointer, oparg);
- int argcount = oparg + is_meth;
- PyObject *callable = PEEK(argcount + 1);
+ int is_meth = method != NULL;
+ int argcount = oparg;
+ if (is_meth) {
+ callable = method;
+ args--;
+ argcount++;
+ }
DEOPT_IF(!PyFunction_Check(callable), CALL);
PyFunctionObject *func = (PyFunctionObject *)callable;
- DEOPT_IF(func->func_version != read_u32(cache->func_version), CALL);
+ DEOPT_IF(func->func_version != func_version, CALL);
PyCodeObject *code = (PyCodeObject *)func->func_code;
DEOPT_IF(argcount > code->co_argcount, CALL);
- int minargs = cache->min_args;
- DEOPT_IF(argcount < minargs, CALL);
+ DEOPT_IF(argcount < min_args, CALL);
DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize), CALL);
STAT_INC(CALL, hit);
_PyInterpreterFrame *new_frame = _PyFrame_PushUnchecked(tstate, func, code->co_argcount);
- STACK_SHRINK(argcount);
for (int i = 0; i < argcount; i++) {
- new_frame->localsplus[i] = stack_pointer[i];
+ new_frame->localsplus[i] = args[i];
}
for (int i = argcount; i < code->co_argcount; i++) {
- PyObject *def = PyTuple_GET_ITEM(func->func_defaults,
- i - minargs);
+ PyObject *def = PyTuple_GET_ITEM(func->func_defaults, i - min_args);
new_frame->localsplus[i] = Py_NewRef(def);
}
- STACK_SHRINK(2-is_meth);
+ // Manipulate stack and cache directly since we leave using DISPATCH_INLINED().
+ STACK_SHRINK(oparg + 2);
JUMPBY(INLINE_CACHE_ENTRIES_CALL);
DISPATCH_INLINED(new_frame);
}
TARGET(CALL_NO_KW_TYPE_1) {
+ PyObject **args = &PEEK(oparg);
+ PyObject *callable = PEEK(1 + oparg);
+ PyObject *null = PEEK(2 + oparg);
+ PyObject *res;
assert(kwnames == NULL);
assert(cframe.use_tracing == 0);
assert(oparg == 1);
- DEOPT_IF(is_method(stack_pointer, 1), CALL);
- PyObject *obj = TOP();
- PyObject *callable = SECOND();
+ DEOPT_IF(null != NULL, CALL);
+ PyObject *obj = args[0];
DEOPT_IF(callable != (PyObject *)&PyType_Type, CALL);
STAT_INC(CALL, hit);
- JUMPBY(INLINE_CACHE_ENTRIES_CALL);
- PyObject *res = Py_NewRef(Py_TYPE(obj));
- Py_DECREF(callable);
+ res = Py_NewRef(Py_TYPE(obj));
Py_DECREF(obj);
- STACK_SHRINK(2);
- SET_TOP(res);
+ Py_DECREF(&PyType_Type); // I.e., callable
+ STACK_SHRINK(oparg);
+ STACK_SHRINK(1);
+ POKE(1, res);
+ JUMPBY(4);
DISPATCH();
}
TARGET(CALL_NO_KW_STR_1) {
+ PyObject **args = &PEEK(oparg);
+ PyObject *callable = PEEK(1 + oparg);
+ PyObject *null = PEEK(2 + oparg);
+ PyObject *res;
assert(kwnames == NULL);
assert(cframe.use_tracing == 0);
assert(oparg == 1);
- DEOPT_IF(is_method(stack_pointer, 1), CALL);
- PyObject *callable = PEEK(2);
+ DEOPT_IF(null != NULL, CALL);
DEOPT_IF(callable != (PyObject *)&PyUnicode_Type, CALL);
STAT_INC(CALL, hit);
- PyObject *arg = TOP();
- PyObject *res = PyObject_Str(arg);
+ PyObject *arg = args[0];
+ res = PyObject_Str(arg);
Py_DECREF(arg);
- Py_DECREF(&PyUnicode_Type);
- STACK_SHRINK(2);
- SET_TOP(res);
- if (res == NULL) {
- goto error;
- }
- JUMPBY(INLINE_CACHE_ENTRIES_CALL);
+ Py_DECREF(&PyUnicode_Type); // I.e., callable
+ if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; }
+ STACK_SHRINK(oparg);
+ STACK_SHRINK(1);
+ POKE(1, res);
+ JUMPBY(4);
CHECK_EVAL_BREAKER();
DISPATCH();
}
TARGET(CALL_NO_KW_TUPLE_1) {
+ PyObject **args = &PEEK(oparg);
+ PyObject *callable = PEEK(1 + oparg);
+ PyObject *null = PEEK(2 + oparg);
+ PyObject *res;
assert(kwnames == NULL);
assert(oparg == 1);
- DEOPT_IF(is_method(stack_pointer, 1), CALL);
- PyObject *callable = PEEK(2);
+ DEOPT_IF(null != NULL, CALL);
DEOPT_IF(callable != (PyObject *)&PyTuple_Type, CALL);
STAT_INC(CALL, hit);
- PyObject *arg = TOP();
- PyObject *res = PySequence_Tuple(arg);
+ PyObject *arg = args[0];
+ res = PySequence_Tuple(arg);
Py_DECREF(arg);
- Py_DECREF(&PyTuple_Type);
- STACK_SHRINK(2);
- SET_TOP(res);
- if (res == NULL) {
- goto error;
- }
- JUMPBY(INLINE_CACHE_ENTRIES_CALL);
+ Py_DECREF(&PyTuple_Type); // I.e., tuple
+ if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; }
+ STACK_SHRINK(oparg);
+ STACK_SHRINK(1);
+ POKE(1, res);
+ JUMPBY(4);
CHECK_EVAL_BREAKER();
DISPATCH();
}
TARGET(CALL_BUILTIN_CLASS) {
- int is_meth = is_method(stack_pointer, oparg);
- int total_args = oparg + is_meth;
+ PyObject **args = &PEEK(oparg);
+ PyObject *callable = PEEK(1 + oparg);
+ PyObject *method = PEEK(2 + oparg);
+ PyObject *res;
+ int is_meth = method != NULL;
+ int total_args = oparg;
+ if (is_meth) {
+ callable = method;
+ args--;
+ total_args++;
+ }
int kwnames_len = KWNAMES_LEN();
- PyObject *callable = PEEK(total_args + 1);
DEOPT_IF(!PyType_Check(callable), CALL);
PyTypeObject *tp = (PyTypeObject *)callable;
DEOPT_IF(tp->tp_vectorcall == NULL, CALL);
STAT_INC(CALL, hit);
- STACK_SHRINK(total_args);
- PyObject *res = tp->tp_vectorcall((PyObject *)tp, stack_pointer,
- total_args-kwnames_len, kwnames);
+ res = tp->tp_vectorcall((PyObject *)tp, args,
+ total_args - kwnames_len, kwnames);
kwnames = NULL;
/* Free the arguments. */
for (int i = 0; i < total_args; i++) {
- Py_DECREF(stack_pointer[i]);
+ Py_DECREF(args[i]);
}
Py_DECREF(tp);
- STACK_SHRINK(1-is_meth);
- SET_TOP(res);
- if (res == NULL) {
- goto error;
- }
- JUMPBY(INLINE_CACHE_ENTRIES_CALL);
+ if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; }
+ STACK_SHRINK(oparg);
+ STACK_SHRINK(1);
+ POKE(1, res);
+ JUMPBY(4);
CHECK_EVAL_BREAKER();
DISPATCH();
}
TARGET(CALL_NO_KW_BUILTIN_O) {
+ PyObject **args = &PEEK(oparg);
+ PyObject *callable = PEEK(1 + oparg);
+ PyObject *method = PEEK(2 + oparg);
+ PyObject *res;
assert(cframe.use_tracing == 0);
/* Builtin METH_O functions */
assert(kwnames == NULL);
- int is_meth = is_method(stack_pointer, oparg);
- int total_args = oparg + is_meth;
+ int is_meth = method != NULL;
+ int total_args = oparg;
+ if (is_meth) {
+ callable = method;
+ args--;
+ total_args++;
+ }
DEOPT_IF(total_args != 1, CALL);
- PyObject *callable = PEEK(total_args + 1);
DEOPT_IF(!PyCFunction_CheckExact(callable), CALL);
DEOPT_IF(PyCFunction_GET_FLAGS(callable) != METH_O, CALL);
STAT_INC(CALL, hit);
if (_Py_EnterRecursiveCallTstate(tstate, " while calling a Python object")) {
goto error;
}
- PyObject *arg = TOP();
- PyObject *res = _PyCFunction_TrampolineCall(cfunc, PyCFunction_GET_SELF(callable), arg);
+ PyObject *arg = args[0];
+ res = _PyCFunction_TrampolineCall(cfunc, PyCFunction_GET_SELF(callable), arg);
_Py_LeaveRecursiveCallTstate(tstate);
assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL));
Py_DECREF(arg);
Py_DECREF(callable);
- STACK_SHRINK(2-is_meth);
- SET_TOP(res);
- if (res == NULL) {
- goto error;
- }
- JUMPBY(INLINE_CACHE_ENTRIES_CALL);
+ if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; }
+ STACK_SHRINK(oparg);
+ STACK_SHRINK(1);
+ POKE(1, res);
+ JUMPBY(4);
CHECK_EVAL_BREAKER();
DISPATCH();
}
TARGET(CALL_NO_KW_BUILTIN_FAST) {
+ PyObject **args = &PEEK(oparg);
+ PyObject *callable = PEEK(1 + oparg);
+ PyObject *method = PEEK(2 + oparg);
+ PyObject *res;
assert(cframe.use_tracing == 0);
/* Builtin METH_FASTCALL functions, without keywords */
assert(kwnames == NULL);
- int is_meth = is_method(stack_pointer, oparg);
- int total_args = oparg + is_meth;
- PyObject *callable = PEEK(total_args + 1);
+ int is_meth = method != NULL;
+ int total_args = oparg;
+ if (is_meth) {
+ callable = method;
+ args--;
+ total_args++;
+ }
DEOPT_IF(!PyCFunction_CheckExact(callable), CALL);
- DEOPT_IF(PyCFunction_GET_FLAGS(callable) != METH_FASTCALL,
- CALL);
+ DEOPT_IF(PyCFunction_GET_FLAGS(callable) != METH_FASTCALL, CALL);
STAT_INC(CALL, hit);
PyCFunction cfunc = PyCFunction_GET_FUNCTION(callable);
- STACK_SHRINK(total_args);
/* res = func(self, args, nargs) */
- PyObject *res = ((_PyCFunctionFast)(void(*)(void))cfunc)(
+ res = ((_PyCFunctionFast)(void(*)(void))cfunc)(
PyCFunction_GET_SELF(callable),
- stack_pointer,
+ args,
total_args);
assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL));
/* Free the arguments. */
for (int i = 0; i < total_args; i++) {
- Py_DECREF(stack_pointer[i]);
+ Py_DECREF(args[i]);
}
- STACK_SHRINK(2-is_meth);
- PUSH(res);
Py_DECREF(callable);
- if (res == NULL) {
+ if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; }
/* Not deopting because this doesn't mean our optimization was
wrong. `res` can be NULL for valid reasons. Eg. getattr(x,
'invalid'). In those cases an exception is set, so we must
handle it.
*/
- goto error;
- }
- JUMPBY(INLINE_CACHE_ENTRIES_CALL);
+ STACK_SHRINK(oparg);
+ STACK_SHRINK(1);
+ POKE(1, res);
+ JUMPBY(4);
CHECK_EVAL_BREAKER();
DISPATCH();
}
TARGET(CALL_BUILTIN_FAST_WITH_KEYWORDS) {
+ PyObject **args = &PEEK(oparg);
+ PyObject *callable = PEEK(1 + oparg);
+ PyObject *method = PEEK(2 + oparg);
+ PyObject *res;
assert(cframe.use_tracing == 0);
/* Builtin METH_FASTCALL | METH_KEYWORDS functions */
- int is_meth = is_method(stack_pointer, oparg);
- int total_args = oparg + is_meth;
- PyObject *callable = PEEK(total_args + 1);
+ int is_meth = method != NULL;
+ int total_args = oparg;
+ if (is_meth) {
+ callable = method;
+ args--;
+ total_args++;
+ }
DEOPT_IF(!PyCFunction_CheckExact(callable), CALL);
DEOPT_IF(PyCFunction_GET_FLAGS(callable) !=
(METH_FASTCALL | METH_KEYWORDS), CALL);
STAT_INC(CALL, hit);
- STACK_SHRINK(total_args);
/* res = func(self, args, nargs, kwnames) */
_PyCFunctionFastWithKeywords cfunc =
(_PyCFunctionFastWithKeywords)(void(*)(void))
PyCFunction_GET_FUNCTION(callable);
- PyObject *res = cfunc(
+ res = cfunc(
PyCFunction_GET_SELF(callable),
- stack_pointer,
+ args,
total_args - KWNAMES_LEN(),
kwnames
);
/* Free the arguments. */
for (int i = 0; i < total_args; i++) {
- Py_DECREF(stack_pointer[i]);
+ Py_DECREF(args[i]);
}
- STACK_SHRINK(2-is_meth);
- PUSH(res);
Py_DECREF(callable);
- if (res == NULL) {
- goto error;
- }
- JUMPBY(INLINE_CACHE_ENTRIES_CALL);
+ if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; }
+ STACK_SHRINK(oparg);
+ STACK_SHRINK(1);
+ POKE(1, res);
+ JUMPBY(4);
CHECK_EVAL_BREAKER();
DISPATCH();
}
TARGET(CALL_NO_KW_LEN) {
+ PyObject **args = &PEEK(oparg);
+ PyObject *callable = PEEK(1 + oparg);
+ PyObject *method = PEEK(2 + oparg);
+ PyObject *res;
assert(cframe.use_tracing == 0);
assert(kwnames == NULL);
/* len(o) */
- int is_meth = is_method(stack_pointer, oparg);
- int total_args = oparg + is_meth;
+ int is_meth = method != NULL;
+ int total_args = oparg;
+ if (is_meth) {
+ callable = method;
+ args--;
+ total_args++;
+ }
DEOPT_IF(total_args != 1, CALL);
- PyObject *callable = PEEK(total_args + 1);
PyInterpreterState *interp = _PyInterpreterState_GET();
DEOPT_IF(callable != interp->callable_cache.len, CALL);
STAT_INC(CALL, hit);
- PyObject *arg = TOP();
+ PyObject *arg = args[0];
Py_ssize_t len_i = PyObject_Length(arg);
if (len_i < 0) {
goto error;
}
- PyObject *res = PyLong_FromSsize_t(len_i);
+ res = PyLong_FromSsize_t(len_i);
assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL));
- STACK_SHRINK(2-is_meth);
- SET_TOP(res);
Py_DECREF(callable);
Py_DECREF(arg);
- if (res == NULL) {
- goto error;
- }
- JUMPBY(INLINE_CACHE_ENTRIES_CALL);
+ if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; }
+ STACK_SHRINK(oparg);
+ STACK_SHRINK(1);
+ POKE(1, res);
+ JUMPBY(4);
DISPATCH();
}
TARGET(CALL_NO_KW_ISINSTANCE) {
+ PyObject **args = &PEEK(oparg);
+ PyObject *callable = PEEK(1 + oparg);
+ PyObject *method = PEEK(2 + oparg);
+ PyObject *res;
assert(cframe.use_tracing == 0);
assert(kwnames == NULL);
/* isinstance(o, o2) */
- int is_meth = is_method(stack_pointer, oparg);
- int total_args = oparg + is_meth;
- PyObject *callable = PEEK(total_args + 1);
+ int is_meth = method != NULL;
+ int total_args = oparg;
+ if (is_meth) {
+ callable = method;
+ args--;
+ total_args++;
+ }
DEOPT_IF(total_args != 2, CALL);
PyInterpreterState *interp = _PyInterpreterState_GET();
DEOPT_IF(callable != interp->callable_cache.isinstance, CALL);
STAT_INC(CALL, hit);
- PyObject *cls = POP();
- PyObject *inst = TOP();
+ PyObject *cls = args[1];
+ PyObject *inst = args[0];
int retval = PyObject_IsInstance(inst, cls);
if (retval < 0) {
- Py_DECREF(cls);
goto error;
}
- PyObject *res = PyBool_FromLong(retval);
+ res = PyBool_FromLong(retval);
assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL));
- STACK_SHRINK(2-is_meth);
- SET_TOP(res);
Py_DECREF(inst);
Py_DECREF(cls);
Py_DECREF(callable);
- if (res == NULL) {
- goto error;
- }
- JUMPBY(INLINE_CACHE_ENTRIES_CALL);
+ if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; }
+ STACK_SHRINK(oparg);
+ STACK_SHRINK(1);
+ POKE(1, res);
+ JUMPBY(4);
DISPATCH();
}
TARGET(CALL_NO_KW_LIST_APPEND) {
+ PyObject **args = &PEEK(oparg);
+ PyObject *self = PEEK(1 + oparg);
+ PyObject *method = PEEK(2 + oparg);
assert(cframe.use_tracing == 0);
assert(kwnames == NULL);
assert(oparg == 1);
- PyObject *callable = PEEK(3);
+ assert(method != NULL);
PyInterpreterState *interp = _PyInterpreterState_GET();
- DEOPT_IF(callable != interp->callable_cache.list_append, CALL);
- PyObject *list = SECOND();
- DEOPT_IF(!PyList_Check(list), CALL);
+ DEOPT_IF(method != interp->callable_cache.list_append, CALL);
+ DEOPT_IF(!PyList_Check(self), CALL);
STAT_INC(CALL, hit);
- PyObject *arg = POP();
- if (_PyList_AppendTakeRef((PyListObject *)list, arg) < 0) {
- goto error;
+ if (_PyList_AppendTakeRef((PyListObject *)self, args[0]) < 0) {
+ goto pop_1_error; // Since arg is DECREF'ed already
}
- STACK_SHRINK(2);
- Py_DECREF(list);
- Py_DECREF(callable);
+ Py_DECREF(self);
+ Py_DECREF(method);
+ STACK_SHRINK(3);
// CALL + POP_TOP
JUMPBY(INLINE_CACHE_ENTRIES_CALL + 1);
assert(_Py_OPCODE(next_instr[-1]) == POP_TOP);
}
TARGET(CALL_NO_KW_METHOD_DESCRIPTOR_O) {
+ PyObject **args = &PEEK(oparg);
+ PyObject *method = PEEK(2 + oparg);
+ PyObject *res;
assert(kwnames == NULL);
- int is_meth = is_method(stack_pointer, oparg);
- int total_args = oparg + is_meth;
+ int is_meth = method != NULL;
+ int total_args = oparg;
+ if (is_meth) {
+ args--;
+ total_args++;
+ }
PyMethodDescrObject *callable =
(PyMethodDescrObject *)PEEK(total_args + 1);
DEOPT_IF(total_args != 2, CALL);
DEOPT_IF(!Py_IS_TYPE(callable, &PyMethodDescr_Type), CALL);
PyMethodDef *meth = callable->d_method;
DEOPT_IF(meth->ml_flags != METH_O, CALL);
- PyObject *arg = TOP();
- PyObject *self = SECOND();
+ PyObject *arg = args[1];
+ PyObject *self = args[0];
DEOPT_IF(!Py_IS_TYPE(self, callable->d_common.d_type), CALL);
STAT_INC(CALL, hit);
PyCFunction cfunc = meth->ml_meth;
if (_Py_EnterRecursiveCallTstate(tstate, " while calling a Python object")) {
goto error;
}
- PyObject *res = _PyCFunction_TrampolineCall(cfunc, self, arg);
+ res = _PyCFunction_TrampolineCall(cfunc, self, arg);
_Py_LeaveRecursiveCallTstate(tstate);
assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL));
Py_DECREF(self);
Py_DECREF(arg);
- STACK_SHRINK(oparg + 1);
- SET_TOP(res);
Py_DECREF(callable);
- if (res == NULL) {
- goto error;
- }
- JUMPBY(INLINE_CACHE_ENTRIES_CALL);
+ if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; }
+ STACK_SHRINK(oparg);
+ STACK_SHRINK(1);
+ POKE(1, res);
+ JUMPBY(4);
CHECK_EVAL_BREAKER();
DISPATCH();
}
TARGET(CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS) {
- int is_meth = is_method(stack_pointer, oparg);
- int total_args = oparg + is_meth;
+ PyObject **args = &PEEK(oparg);
+ PyObject *method = PEEK(2 + oparg);
+ PyObject *res;
+ int is_meth = method != NULL;
+ int total_args = oparg;
+ if (is_meth) {
+ args--;
+ total_args++;
+ }
PyMethodDescrObject *callable =
(PyMethodDescrObject *)PEEK(total_args + 1);
DEOPT_IF(!Py_IS_TYPE(callable, &PyMethodDescr_Type), CALL);
PyMethodDef *meth = callable->d_method;
DEOPT_IF(meth->ml_flags != (METH_FASTCALL|METH_KEYWORDS), CALL);
PyTypeObject *d_type = callable->d_common.d_type;
- PyObject *self = PEEK(total_args);
+ PyObject *self = args[0];
DEOPT_IF(!Py_IS_TYPE(self, d_type), CALL);
STAT_INC(CALL, hit);
- int nargs = total_args-1;
- STACK_SHRINK(nargs);
+ int nargs = total_args - 1;
_PyCFunctionFastWithKeywords cfunc =
(_PyCFunctionFastWithKeywords)(void(*)(void))meth->ml_meth;
- PyObject *res = cfunc(self, stack_pointer, nargs - KWNAMES_LEN(),
- kwnames);
+ res = cfunc(self, args + 1, nargs - KWNAMES_LEN(), kwnames);
assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL));
kwnames = NULL;
/* Free the arguments. */
- for (int i = 0; i < nargs; i++) {
- Py_DECREF(stack_pointer[i]);
+ for (int i = 0; i < total_args; i++) {
+ Py_DECREF(args[i]);
}
- Py_DECREF(self);
- STACK_SHRINK(2-is_meth);
- SET_TOP(res);
Py_DECREF(callable);
- if (res == NULL) {
- goto error;
- }
- JUMPBY(INLINE_CACHE_ENTRIES_CALL);
+ if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; }
+ STACK_SHRINK(oparg);
+ STACK_SHRINK(1);
+ POKE(1, res);
+ JUMPBY(4);
CHECK_EVAL_BREAKER();
DISPATCH();
}
TARGET(CALL_NO_KW_METHOD_DESCRIPTOR_NOARGS) {
+ PyObject **args = &PEEK(oparg);
+ PyObject *method = PEEK(2 + oparg);
+ PyObject *res;
assert(kwnames == NULL);
assert(oparg == 0 || oparg == 1);
- int is_meth = is_method(stack_pointer, oparg);
- int total_args = oparg + is_meth;
+ int is_meth = method != NULL;
+ int total_args = oparg;
+ if (is_meth) {
+ args--;
+ total_args++;
+ }
DEOPT_IF(total_args != 1, CALL);
PyMethodDescrObject *callable = (PyMethodDescrObject *)SECOND();
DEOPT_IF(!Py_IS_TYPE(callable, &PyMethodDescr_Type), CALL);
PyMethodDef *meth = callable->d_method;
- PyObject *self = TOP();
+ PyObject *self = args[0];
DEOPT_IF(!Py_IS_TYPE(self, callable->d_common.d_type), CALL);
DEOPT_IF(meth->ml_flags != METH_NOARGS, CALL);
STAT_INC(CALL, hit);
if (_Py_EnterRecursiveCallTstate(tstate, " while calling a Python object")) {
goto error;
}
- PyObject *res = _PyCFunction_TrampolineCall(cfunc, self, NULL);
+ res = _PyCFunction_TrampolineCall(cfunc, self, NULL);
_Py_LeaveRecursiveCallTstate(tstate);
assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL));
Py_DECREF(self);
- STACK_SHRINK(oparg + 1);
- SET_TOP(res);
Py_DECREF(callable);
- if (res == NULL) {
- goto error;
- }
- JUMPBY(INLINE_CACHE_ENTRIES_CALL);
+ if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; }
+ STACK_SHRINK(oparg);
+ STACK_SHRINK(1);
+ POKE(1, res);
+ JUMPBY(4);
CHECK_EVAL_BREAKER();
DISPATCH();
}
TARGET(CALL_NO_KW_METHOD_DESCRIPTOR_FAST) {
+ PyObject **args = &PEEK(oparg);
+ PyObject *method = PEEK(2 + oparg);
+ PyObject *res;
assert(kwnames == NULL);
- int is_meth = is_method(stack_pointer, oparg);
- int total_args = oparg + is_meth;
+ int is_meth = method != NULL;
+ int total_args = oparg;
+ if (is_meth) {
+ args--;
+ total_args++;
+ }
PyMethodDescrObject *callable =
(PyMethodDescrObject *)PEEK(total_args + 1);
/* Builtin METH_FASTCALL methods, without keywords */
DEOPT_IF(!Py_IS_TYPE(callable, &PyMethodDescr_Type), CALL);
PyMethodDef *meth = callable->d_method;
DEOPT_IF(meth->ml_flags != METH_FASTCALL, CALL);
- PyObject *self = PEEK(total_args);
+ PyObject *self = args[0];
DEOPT_IF(!Py_IS_TYPE(self, callable->d_common.d_type), CALL);
STAT_INC(CALL, hit);
_PyCFunctionFast cfunc =
(_PyCFunctionFast)(void(*)(void))meth->ml_meth;
- int nargs = total_args-1;
- STACK_SHRINK(nargs);
- PyObject *res = cfunc(self, stack_pointer, nargs);
+ int nargs = total_args - 1;
+ res = cfunc(self, args + 1, nargs);
assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL));
/* Clear the stack of the arguments. */
- for (int i = 0; i < nargs; i++) {
- Py_DECREF(stack_pointer[i]);
+ for (int i = 0; i < total_args; i++) {
+ Py_DECREF(args[i]);
}
- Py_DECREF(self);
- STACK_SHRINK(2-is_meth);
- SET_TOP(res);
Py_DECREF(callable);
- if (res == NULL) {
- goto error;
- }
- JUMPBY(INLINE_CACHE_ENTRIES_CALL);
+ if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; }
+ STACK_SHRINK(oparg);
+ STACK_SHRINK(1);
+ POKE(1, res);
+ JUMPBY(4);
CHECK_EVAL_BREAKER();
DISPATCH();
}
return 1;
case LOAD_ATTR_METHOD_LAZY_DICT:
return 1;
- case CALL_BOUND_METHOD_EXACT_ARGS:
- return -1;
case KW_NAMES:
return 0;
case CALL:
- return -1;
+ return oparg + 2;
+ case CALL_BOUND_METHOD_EXACT_ARGS:
+ return oparg + 2;
case CALL_PY_EXACT_ARGS:
- return -1;
+ return oparg + 2;
case CALL_PY_WITH_DEFAULTS:
- return -1;
+ return oparg + 2;
case CALL_NO_KW_TYPE_1:
- return -1;
+ return oparg + 2;
case CALL_NO_KW_STR_1:
- return -1;
+ return oparg + 2;
case CALL_NO_KW_TUPLE_1:
- return -1;
+ return oparg + 2;
case CALL_BUILTIN_CLASS:
- return -1;
+ return oparg + 2;
case CALL_NO_KW_BUILTIN_O:
- return -1;
+ return oparg + 2;
case CALL_NO_KW_BUILTIN_FAST:
- return -1;
+ return oparg + 2;
case CALL_BUILTIN_FAST_WITH_KEYWORDS:
- return -1;
+ return oparg + 2;
case CALL_NO_KW_LEN:
- return -1;
+ return oparg + 2;
case CALL_NO_KW_ISINSTANCE:
- return -1;
+ return oparg + 2;
case CALL_NO_KW_LIST_APPEND:
- return -1;
+ return oparg + 2;
case CALL_NO_KW_METHOD_DESCRIPTOR_O:
- return -1;
+ return oparg + 2;
case CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS:
- return -1;
+ return oparg + 2;
case CALL_NO_KW_METHOD_DESCRIPTOR_NOARGS:
- return -1;
+ return oparg + 2;
case CALL_NO_KW_METHOD_DESCRIPTOR_FAST:
- return -1;
+ return oparg + 2;
case CALL_FUNCTION_EX:
return ((oparg & 1) ? 1 : 0) + 3;
case MAKE_FUNCTION:
return ((oparg & 1) ? 1 : 0) + 1;
case LOAD_ATTR_METHOD_LAZY_DICT:
return ((oparg & 1) ? 1 : 0) + 1;
- case CALL_BOUND_METHOD_EXACT_ARGS:
- return -1;
case KW_NAMES:
return 0;
case CALL:
- return -1;
+ return 1;
+ case CALL_BOUND_METHOD_EXACT_ARGS:
+ return 1;
case CALL_PY_EXACT_ARGS:
- return -1;
+ return 1;
case CALL_PY_WITH_DEFAULTS:
- return -1;
+ return 1;
case CALL_NO_KW_TYPE_1:
- return -1;
+ return 1;
case CALL_NO_KW_STR_1:
- return -1;
+ return 1;
case CALL_NO_KW_TUPLE_1:
- return -1;
+ return 1;
case CALL_BUILTIN_CLASS:
- return -1;
+ return 1;
case CALL_NO_KW_BUILTIN_O:
- return -1;
+ return 1;
case CALL_NO_KW_BUILTIN_FAST:
- return -1;
+ return 1;
case CALL_BUILTIN_FAST_WITH_KEYWORDS:
- return -1;
+ return 1;
case CALL_NO_KW_LEN:
- return -1;
+ return 1;
case CALL_NO_KW_ISINSTANCE:
- return -1;
+ return 1;
case CALL_NO_KW_LIST_APPEND:
- return -1;
+ return 1;
case CALL_NO_KW_METHOD_DESCRIPTOR_O:
- return -1;
+ return 1;
case CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS:
- return -1;
+ return 1;
case CALL_NO_KW_METHOD_DESCRIPTOR_NOARGS:
- return -1;
+ return 1;
case CALL_NO_KW_METHOD_DESCRIPTOR_FAST:
- return -1;
+ return 1;
case CALL_FUNCTION_EX:
return 1;
case MAKE_FUNCTION:
[LOAD_ATTR_METHOD_WITH_VALUES] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IBC00000000 },
[LOAD_ATTR_METHOD_NO_DICT] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IBC00000000 },
[LOAD_ATTR_METHOD_LAZY_DICT] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IBC00000000 },
- [CALL_BOUND_METHOD_EXACT_ARGS] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IB },
[KW_NAMES] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IB },
- [CALL] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IB },
- [CALL_PY_EXACT_ARGS] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IB },
- [CALL_PY_WITH_DEFAULTS] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IB },
- [CALL_NO_KW_TYPE_1] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IB },
- [CALL_NO_KW_STR_1] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IB },
- [CALL_NO_KW_TUPLE_1] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IB },
- [CALL_BUILTIN_CLASS] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IB },
- [CALL_NO_KW_BUILTIN_O] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IB },
- [CALL_NO_KW_BUILTIN_FAST] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IB },
- [CALL_BUILTIN_FAST_WITH_KEYWORDS] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IB },
- [CALL_NO_KW_LEN] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IB },
- [CALL_NO_KW_ISINSTANCE] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IB },
- [CALL_NO_KW_LIST_APPEND] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IB },
- [CALL_NO_KW_METHOD_DESCRIPTOR_O] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IB },
- [CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IB },
- [CALL_NO_KW_METHOD_DESCRIPTOR_NOARGS] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IB },
- [CALL_NO_KW_METHOD_DESCRIPTOR_FAST] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IB },
+ [CALL] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IBC000 },
+ [CALL_BOUND_METHOD_EXACT_ARGS] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IBC000 },
+ [CALL_PY_EXACT_ARGS] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IBC000 },
+ [CALL_PY_WITH_DEFAULTS] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IBC000 },
+ [CALL_NO_KW_TYPE_1] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IBC000 },
+ [CALL_NO_KW_STR_1] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IBC000 },
+ [CALL_NO_KW_TUPLE_1] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IBC000 },
+ [CALL_BUILTIN_CLASS] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IBC000 },
+ [CALL_NO_KW_BUILTIN_O] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IBC000 },
+ [CALL_NO_KW_BUILTIN_FAST] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IBC000 },
+ [CALL_BUILTIN_FAST_WITH_KEYWORDS] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IBC000 },
+ [CALL_NO_KW_LEN] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IBC000 },
+ [CALL_NO_KW_ISINSTANCE] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IBC000 },
+ [CALL_NO_KW_LIST_APPEND] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IBC000 },
+ [CALL_NO_KW_METHOD_DESCRIPTOR_O] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IBC000 },
+ [CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IBC000 },
+ [CALL_NO_KW_METHOD_DESCRIPTOR_NOARGS] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IBC000 },
+ [CALL_NO_KW_METHOD_DESCRIPTOR_FAST] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IBC000 },
[CALL_FUNCTION_EX] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IB },
[MAKE_FUNCTION] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IB },
[RETURN_GENERATOR] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IX },
# Write the body, substituting a goto for ERROR_IF() and other stuff
assert dedent <= 0
extra = " " * -dedent
+ names_to_skip = self.unmoved_names | frozenset({UNUSED, "null"})
for line in self.block_text:
if m := re.match(r"(\s*)ERROR_IF\((.+), (\w+)\);\s*(?://.*)?$", line):
space, cond, label = m.groups()
+ space = extra + space
# ERROR_IF() must pop the inputs from the stack.
# The code block is responsible for DECREF()ing them.
# NOTE: If the label doesn't exist, just add it to ceval.c.
symbolic = ""
if symbolic:
out.write_raw(
- f"{extra}{space}if ({cond}) {{ STACK_SHRINK({symbolic}); goto {label}; }}\n"
+ f"{space}if ({cond}) {{ STACK_SHRINK({symbolic}); goto {label}; }}\n"
)
else:
- out.write_raw(f"{extra}{space}if ({cond}) goto {label};\n")
+ out.write_raw(f"{space}if ({cond}) goto {label};\n")
elif m := re.match(r"(\s*)DECREF_INPUTS\(\);\s*(?://.*)?$", line):
if not self.register:
- space = m.group(1)
+ space = extra + m.group(1)
for ieff in self.input_effects:
- if ieff.name not in self.unmoved_names:
- out.write_raw(f"{extra}{space}Py_DECREF({ieff.name});\n")
+ if ieff.name in names_to_skip:
+ continue
+ if ieff.size:
+ out.write_raw(
+ f"{space}for (int _i = {ieff.size}; --_i >= 0;) {{\n"
+ )
+ out.write_raw(f"{space} Py_DECREF({ieff.name}[_i]);\n")
+ out.write_raw(f"{space}}}\n")
+ else:
+ decref = "XDECREF" if ieff.cond else "DECREF"
+ out.write_raw(f"{space}Py_{decref}({ieff.name});\n")
else:
out.write_raw(extra + line)
projects / thirdparty / Python / cpython.git / commitdiff
