int _Py_Specialize_LoadMethod(PyObject *owner, _Py_CODEUNIT *instr, PyObject *name, SpecializedCacheEntry *cache);
int _Py_Specialize_BinarySubscr(PyObject *sub, PyObject *container, _Py_CODEUNIT *instr, SpecializedCacheEntry *cache);
int _Py_Specialize_StoreSubscr(PyObject *container, PyObject *sub, _Py_CODEUNIT *instr);
-int _Py_Specialize_CallNoKw(PyObject *callable, _Py_CODEUNIT *instr, int nargs,
+int _Py_Specialize_Call(PyObject *callable, _Py_CODEUNIT *instr, int nargs,
+ PyObject *kwnames, SpecializedCacheEntry *cache);
+int _Py_Specialize_Precall(PyObject *callable, _Py_CODEUNIT *instr, int nargs,
PyObject *kwnames, SpecializedCacheEntry *cache, PyObject *builtins);
void _Py_Specialize_BinaryOp(PyObject *lhs, PyObject *rhs, _Py_CODEUNIT *instr,
SpecializedCacheEntry *cache);
#define STORE_SUBSCR_LIST_INT 28
#define STORE_SUBSCR_DICT 29
#define CALL_ADAPTIVE 34
-#define CALL_BUILTIN_CLASS 36
-#define CALL_NO_KW_BUILTIN_O 37
-#define CALL_NO_KW_BUILTIN_FAST 38
-#define CALL_BUILTIN_FAST_WITH_KEYWORDS 39
-#define CALL_NO_KW_LEN 40
-#define CALL_NO_KW_ISINSTANCE 41
-#define CALL_PY_EXACT_ARGS 42
-#define CALL_PY_WITH_DEFAULTS 43
-#define CALL_NO_KW_LIST_APPEND 44
-#define CALL_NO_KW_METHOD_DESCRIPTOR_O 45
-#define CALL_NO_KW_METHOD_DESCRIPTOR_NOARGS 46
-#define CALL_NO_KW_STR_1 47
-#define CALL_NO_KW_TUPLE_1 48
-#define CALL_NO_KW_TYPE_1 55
-#define CALL_NO_KW_METHOD_DESCRIPTOR_FAST 56
-#define JUMP_ABSOLUTE_QUICK 57
-#define LOAD_ATTR_ADAPTIVE 58
-#define LOAD_ATTR_INSTANCE_VALUE 59
-#define LOAD_ATTR_WITH_HINT 62
-#define LOAD_ATTR_SLOT 63
-#define LOAD_ATTR_MODULE 64
-#define LOAD_GLOBAL_ADAPTIVE 65
-#define LOAD_GLOBAL_MODULE 66
-#define LOAD_GLOBAL_BUILTIN 67
-#define LOAD_METHOD_ADAPTIVE 72
-#define LOAD_METHOD_CACHED 76
-#define LOAD_METHOD_CLASS 77
-#define LOAD_METHOD_MODULE 78
-#define LOAD_METHOD_NO_DICT 79
-#define RESUME_QUICK 80
-#define STORE_ATTR_ADAPTIVE 81
-#define STORE_ATTR_INSTANCE_VALUE 131
-#define STORE_ATTR_SLOT 140
-#define STORE_ATTR_WITH_HINT 141
-#define UNPACK_SEQUENCE_ADAPTIVE 143
-#define UNPACK_SEQUENCE_LIST 150
-#define UNPACK_SEQUENCE_TUPLE 153
-#define UNPACK_SEQUENCE_TWO_TUPLE 154
-#define LOAD_FAST__LOAD_FAST 158
-#define STORE_FAST__LOAD_FAST 159
-#define LOAD_FAST__LOAD_CONST 161
-#define LOAD_CONST__LOAD_FAST 167
-#define STORE_FAST__STORE_FAST 168
+#define CALL_PY_EXACT_ARGS 36
+#define CALL_PY_WITH_DEFAULTS 37
+#define JUMP_ABSOLUTE_QUICK 38
+#define LOAD_ATTR_ADAPTIVE 39
+#define LOAD_ATTR_INSTANCE_VALUE 40
+#define LOAD_ATTR_WITH_HINT 41
+#define LOAD_ATTR_SLOT 42
+#define LOAD_ATTR_MODULE 43
+#define LOAD_GLOBAL_ADAPTIVE 44
+#define LOAD_GLOBAL_MODULE 45
+#define LOAD_GLOBAL_BUILTIN 46
+#define LOAD_METHOD_ADAPTIVE 47
+#define LOAD_METHOD_CACHED 48
+#define LOAD_METHOD_CLASS 55
+#define LOAD_METHOD_MODULE 56
+#define LOAD_METHOD_NO_DICT 57
+#define PRECALL_ADAPTIVE 58
+#define PRECALL_BUILTIN_CLASS 59
+#define PRECALL_NO_KW_BUILTIN_O 62
+#define PRECALL_NO_KW_BUILTIN_FAST 63
+#define PRECALL_BUILTIN_FAST_WITH_KEYWORDS 64
+#define PRECALL_NO_KW_LEN 65
+#define PRECALL_NO_KW_ISINSTANCE 66
+#define PRECALL_NO_KW_LIST_APPEND 67
+#define PRECALL_NO_KW_METHOD_DESCRIPTOR_O 72
+#define PRECALL_NO_KW_METHOD_DESCRIPTOR_NOARGS 76
+#define PRECALL_NO_KW_STR_1 77
+#define PRECALL_NO_KW_TUPLE_1 78
+#define PRECALL_NO_KW_TYPE_1 79
+#define PRECALL_NO_KW_METHOD_DESCRIPTOR_FAST 80
+#define PRECALL_BOUND_METHOD 81
+#define PRECALL_PYFUNC 131
+#define RESUME_QUICK 140
+#define STORE_ATTR_ADAPTIVE 141
+#define STORE_ATTR_INSTANCE_VALUE 143
+#define STORE_ATTR_SLOT 150
+#define STORE_ATTR_WITH_HINT 153
+#define UNPACK_SEQUENCE_ADAPTIVE 154
+#define UNPACK_SEQUENCE_LIST 158
+#define UNPACK_SEQUENCE_TUPLE 159
+#define UNPACK_SEQUENCE_TWO_TUPLE 161
+#define LOAD_FAST__LOAD_FAST 167
+#define STORE_FAST__LOAD_FAST 168
+#define LOAD_FAST__LOAD_CONST 169
+#define LOAD_CONST__LOAD_FAST 170
+#define STORE_FAST__STORE_FAST 173
#define DO_TRACING 255
#ifdef NEED_OPCODE_JUMP_TABLES
static uint32_t _PyOpcode_RelativeJump[8] = {
"STORE_SUBSCR_LIST_INT",
"STORE_SUBSCR_DICT",
"CALL_ADAPTIVE",
- "CALL_BUILTIN_CLASS",
- "CALL_NO_KW_BUILTIN_O",
- "CALL_NO_KW_BUILTIN_FAST",
- "CALL_BUILTIN_FAST_WITH_KEYWORDS",
- "CALL_NO_KW_LEN",
- "CALL_NO_KW_ISINSTANCE",
"CALL_PY_EXACT_ARGS",
"CALL_PY_WITH_DEFAULTS",
- "CALL_NO_KW_LIST_APPEND",
- "CALL_NO_KW_METHOD_DESCRIPTOR_O",
- "CALL_NO_KW_METHOD_DESCRIPTOR_NOARGS",
- "CALL_NO_KW_STR_1",
- "CALL_NO_KW_TUPLE_1",
- "CALL_NO_KW_TYPE_1",
- "CALL_NO_KW_METHOD_DESCRIPTOR_FAST",
"JUMP_ABSOLUTE_QUICK",
"LOAD_ATTR_ADAPTIVE",
"LOAD_ATTR_INSTANCE_VALUE",
"LOAD_METHOD_CLASS",
"LOAD_METHOD_MODULE",
"LOAD_METHOD_NO_DICT",
+ "PRECALL_ADAPTIVE",
+ "PRECALL_BUILTIN_CLASS",
+ "PRECALL_NO_KW_BUILTIN_O",
+ "PRECALL_NO_KW_BUILTIN_FAST",
+ "PRECALL_BUILTIN_FAST_WITH_KEYWORDS",
+ "PRECALL_NO_KW_LEN",
+ "PRECALL_NO_KW_ISINSTANCE",
+ "PRECALL_NO_KW_LIST_APPEND",
+ "PRECALL_NO_KW_METHOD_DESCRIPTOR_O",
+ "PRECALL_NO_KW_METHOD_DESCRIPTOR_NOARGS",
+ "PRECALL_NO_KW_STR_1",
+ "PRECALL_NO_KW_TUPLE_1",
+ "PRECALL_NO_KW_TYPE_1",
+ "PRECALL_NO_KW_METHOD_DESCRIPTOR_FAST",
+ "PRECALL_BOUND_METHOD",
+ "PRECALL_PYFUNC",
"RESUME_QUICK",
"STORE_ATTR_ADAPTIVE",
"STORE_ATTR_INSTANCE_VALUE",
}
TARGET(PRECALL) {
+ PREDICTED(PRECALL);
/* Designed to work in tamdem with LOAD_METHOD. */
/* `meth` is NULL when LOAD_METHOD thinks that it's not
a method call.
PEEK(oparg+1) = self;
PEEK(oparg+2) = meth;
Py_DECREF(function);
- function = meth;
}
+ DISPATCH();
+ }
+
+ TARGET(PRECALL_BOUND_METHOD) {
+ SpecializedCacheEntry *cache = GET_CACHE();
+ int original_oparg = cache->adaptive.original_oparg;
+ int is_method = (PEEK(original_oparg + 2) != NULL);
+ DEOPT_IF(is_method, PRECALL);
+ PyObject *function = PEEK(original_oparg + 1);
+ DEOPT_IF(Py_TYPE(function) != &PyMethod_Type, PRECALL);
+ STAT_INC(PRECALL, hit);
+ PyObject *meth = ((PyMethodObject *)function)->im_func;
+ PyObject *self = ((PyMethodObject *)function)->im_self;
+ Py_INCREF(meth);
+ Py_INCREF(self);
+ PEEK(original_oparg+1) = self;
+ PEEK(original_oparg+2) = meth;
+ Py_DECREF(function);
+ DISPATCH();
+ }
+ TARGET(PRECALL_PYFUNC) {
+ SpecializedCacheEntry *cache = GET_CACHE();
+ int original_oparg = cache->adaptive.original_oparg;
+ int is_method = (PEEK(original_oparg + 2) != NULL);
+ int nargs = original_oparg + is_method;
+ PyObject *function = PEEK(nargs + 1);
+ DEOPT_IF(Py_TYPE(function) != &PyFunction_Type, PRECALL);
+ STAT_INC(PRECALL, hit);
DISPATCH();
}
DISPATCH();
}
- TARGET(CALL_ADAPTIVE) {
+ TARGET(PRECALL_ADAPTIVE) {
SpecializedCacheEntry *cache = GET_CACHE();
int original_oparg = cache->adaptive.original_oparg;
if (cache->adaptive.counter == 0) {
int is_meth = is_method(stack_pointer, original_oparg);
int nargs = original_oparg + is_meth;
PyObject *callable = PEEK(nargs + 1);
- int err = _Py_Specialize_CallNoKw(
+ int err = _Py_Specialize_Precall(
callable, next_instr, nargs,
call_shape.kwnames, cache, BUILTINS());
if (err < 0) {
}
DISPATCH();
}
+ else {
+ STAT_INC(PRECALL, deferred);
+ cache->adaptive.counter--;
+ oparg = original_oparg;
+ JUMP_TO_INSTRUCTION(PRECALL);
+ }
+ }
+
+ TARGET(CALL_ADAPTIVE) {
+ SpecializedCacheEntry *cache = GET_CACHE();
+ int original_oparg = cache->adaptive.original_oparg;
+ if (cache->adaptive.counter == 0) {
+ next_instr--;
+ int is_meth = is_method(stack_pointer, original_oparg);
+ int nargs = original_oparg + is_meth;
+ PyObject *callable = PEEK(nargs + 1);
+ int err = _Py_Specialize_Call(
+ callable, next_instr, nargs,
+ call_shape.kwnames, cache);
+ if (err < 0) {
+ goto error;
+ }
+ DISPATCH();
+ }
else {
STAT_INC(CALL, deferred);
cache->adaptive.counter--;
goto start_frame;
}
- TARGET(CALL_NO_KW_TYPE_1) {
+ TARGET(PRECALL_NO_KW_TYPE_1) {
assert(call_shape.kwnames == NULL);
assert(cframe.use_tracing == 0);
assert(GET_CACHE()->adaptive.original_oparg == 1);
- DEOPT_IF(is_method(stack_pointer, 1), CALL);
+ DEOPT_IF(is_method(stack_pointer, 1), PRECALL);
PyObject *obj = TOP();
PyObject *callable = SECOND();
- DEOPT_IF(callable != (PyObject *)&PyType_Type, CALL);
+ DEOPT_IF(callable != (PyObject *)&PyType_Type, PRECALL);
+ next_instr++; // Skip following call
+ STAT_INC(PRECALL, hit);
PyObject *res = Py_NewRef(Py_TYPE(obj));
Py_DECREF(callable);
Py_DECREF(obj);
NOTRACE_DISPATCH();
}
- TARGET(CALL_NO_KW_STR_1) {
+ TARGET(PRECALL_NO_KW_STR_1) {
assert(call_shape.kwnames == NULL);
assert(cframe.use_tracing == 0);
assert(GET_CACHE()->adaptive.original_oparg == 1);
+ DEOPT_IF(is_method(stack_pointer, 1), PRECALL);
PyObject *callable = PEEK(2);
- DEOPT_IF(!PyType_Check(callable), CALL);
- PyTypeObject *tp = (PyTypeObject *)callable;
- DEOPT_IF(is_method(stack_pointer, 1), CALL);
- DEOPT_IF(tp != &PyUnicode_Type, CALL);
- STAT_INC(CALL, hit);
+ DEOPT_IF(callable != (PyObject *)&PyUnicode_Type, PRECALL);
+ next_instr++; // Skip following call
+ STAT_INC(PRECALL, hit);
PyObject *arg = TOP();
PyObject *res = PyObject_Str(arg);
Py_DECREF(arg);
DISPATCH();
}
- TARGET(CALL_NO_KW_TUPLE_1) {
+ TARGET(PRECALL_NO_KW_TUPLE_1) {
assert(call_shape.kwnames == NULL);
assert(GET_CACHE()->adaptive.original_oparg == 1);
- int is_meth = is_method(stack_pointer, 1);
+ DEOPT_IF(is_method(stack_pointer, 1), PRECALL);
PyObject *callable = PEEK(2);
- DEOPT_IF(!PyType_Check(callable), CALL);
- PyTypeObject *tp = (PyTypeObject *)callable;
- DEOPT_IF(is_meth, CALL);
- DEOPT_IF(tp != &PyTuple_Type, CALL);
- STAT_INC(CALL, hit);
+ DEOPT_IF(callable != (PyObject *)&PyTuple_Type, PRECALL);
+ next_instr++; // Skip following call
+ STAT_INC(PRECALL, hit);
PyObject *arg = TOP();
PyObject *res = PySequence_Tuple(arg);
Py_DECREF(arg);
DISPATCH();
}
- TARGET(CALL_BUILTIN_CLASS) {
+ TARGET(PRECALL_BUILTIN_CLASS) {
int original_oparg = GET_CACHE()->adaptive.original_oparg;
int is_meth = is_method(stack_pointer, original_oparg);
int total_args = original_oparg + is_meth;
int kwnames_len = KWNAMES_LEN();
PyObject *callable = PEEK(total_args + 1);
- DEOPT_IF(!PyType_Check(callable), CALL);
+ DEOPT_IF(!PyType_Check(callable), PRECALL);
PyTypeObject *tp = (PyTypeObject *)callable;
- DEOPT_IF(tp->tp_vectorcall == NULL, CALL);
- STAT_INC(CALL, hit);
+ DEOPT_IF(tp->tp_vectorcall == NULL, PRECALL);
+ next_instr++; // Skip following call
+ STAT_INC(PRECALL, hit);
STACK_SHRINK(total_args);
PyObject *res = tp->tp_vectorcall((PyObject *)tp, stack_pointer,
total_args-kwnames_len, call_shape.kwnames);
DISPATCH();
}
- TARGET(CALL_NO_KW_BUILTIN_O) {
+ TARGET(PRECALL_NO_KW_BUILTIN_O) {
assert(cframe.use_tracing == 0);
/* Builtin METH_O functions */
assert(call_shape.kwnames == NULL);
int original_oparg = caches->adaptive.original_oparg;
int is_meth = is_method(stack_pointer, original_oparg);
int total_args = original_oparg + is_meth;
- DEOPT_IF(total_args != 1, CALL);
+ DEOPT_IF(total_args != 1, PRECALL);
PyObject *callable = PEEK(total_args + 1);
- DEOPT_IF(!PyCFunction_CheckExact(callable), CALL);
- DEOPT_IF(PyCFunction_GET_FLAGS(callable) != METH_O, CALL);
- STAT_INC(CALL, hit);
-
+ DEOPT_IF(!PyCFunction_CheckExact(callable), PRECALL);
+ DEOPT_IF(PyCFunction_GET_FLAGS(callable) != METH_O, PRECALL);
+ next_instr++; // Skip following call
+ STAT_INC(PRECALL, hit);
PyCFunction cfunc = PyCFunction_GET_FUNCTION(callable);
// This is slower but CPython promises to check all non-vectorcall
// function calls.
DISPATCH();
}
- TARGET(CALL_NO_KW_BUILTIN_FAST) {
+ TARGET(PRECALL_NO_KW_BUILTIN_FAST) {
assert(cframe.use_tracing == 0);
/* Builtin METH_FASTCALL functions, without keywords */
assert(call_shape.kwnames == NULL);
int is_meth = is_method(stack_pointer, original_oparg);
int total_args = original_oparg + is_meth;
PyObject *callable = PEEK(total_args + 1);
- DEOPT_IF(!PyCFunction_CheckExact(callable), CALL);
+ DEOPT_IF(!PyCFunction_CheckExact(callable), PRECALL);
DEOPT_IF(PyCFunction_GET_FLAGS(callable) != METH_FASTCALL,
- CALL);
- STAT_INC(CALL, hit);
-
+ PRECALL);
+ next_instr++; // Skip following call
+ STAT_INC(PRECALL, hit);
PyCFunction cfunc = PyCFunction_GET_FUNCTION(callable);
STACK_SHRINK(total_args);
/* res = func(self, args, nargs) */
DISPATCH();
}
- TARGET(CALL_BUILTIN_FAST_WITH_KEYWORDS) {
+ TARGET(PRECALL_BUILTIN_FAST_WITH_KEYWORDS) {
assert(cframe.use_tracing == 0);
/* Builtin METH_FASTCALL | METH_KEYWORDS functions */
SpecializedCacheEntry *caches = GET_CACHE();
int is_meth = is_method(stack_pointer, original_oparg);
int total_args = original_oparg + is_meth;
PyObject *callable = PEEK(total_args + 1);
- DEOPT_IF(!PyCFunction_CheckExact(callable), CALL);
+ DEOPT_IF(!PyCFunction_CheckExact(callable), PRECALL);
DEOPT_IF(PyCFunction_GET_FLAGS(callable) !=
- (METH_FASTCALL | METH_KEYWORDS), CALL);
- STAT_INC(CALL, hit);
+ (METH_FASTCALL | METH_KEYWORDS), PRECALL);
+ next_instr++; // Skip following call
+ STAT_INC(PRECALL, hit);
STACK_SHRINK(total_args);
/* res = func(self, args, nargs, kwnames) */
_PyCFunctionFastWithKeywords cfunc =
DISPATCH();
}
- TARGET(CALL_NO_KW_LEN) {
+ TARGET(PRECALL_NO_KW_LEN) {
assert(cframe.use_tracing == 0);
assert(call_shape.kwnames == NULL);
/* len(o) */
int original_oparg = caches->adaptive.original_oparg;
int is_meth = is_method(stack_pointer, original_oparg);
int total_args = original_oparg + is_meth;
- DEOPT_IF(total_args != 1, CALL);
+ DEOPT_IF(total_args != 1, PRECALL);
_PyObjectCache *cache1 = &caches[-1].obj;
PyObject *callable = PEEK(total_args + 1);
- DEOPT_IF(callable != cache1->obj, CALL);
- STAT_INC(CALL, hit);
-
+ DEOPT_IF(callable != cache1->obj, PRECALL);
+ next_instr++; // Skip following call
+ STAT_INC(PRECALL, hit);
PyObject *arg = TOP();
Py_ssize_t len_i = PyObject_Length(arg);
if (len_i < 0) {
DISPATCH();
}
- TARGET(CALL_NO_KW_ISINSTANCE) {
+ TARGET(PRECALL_NO_KW_ISINSTANCE) {
assert(cframe.use_tracing == 0);
assert(call_shape.kwnames == NULL);
/* isinstance(o, o2) */
int is_meth = is_method(stack_pointer, original_oparg);
int total_args = original_oparg + is_meth;
PyObject *callable = PEEK(total_args + 1);
- DEOPT_IF(total_args != 2, CALL);
+ DEOPT_IF(total_args != 2, PRECALL);
_PyObjectCache *cache1 = &caches[-1].obj;
- DEOPT_IF(callable != cache1->obj, CALL);
- STAT_INC(CALL, hit);
-
+ DEOPT_IF(callable != cache1->obj, PRECALL);
+ next_instr++; // Skip following call
+ STAT_INC(PRECALL, hit);
PyObject *cls = POP();
PyObject *inst = TOP();
int retval = PyObject_IsInstance(inst, cls);
DISPATCH();
}
- TARGET(CALL_NO_KW_LIST_APPEND) {
+ TARGET(PRECALL_NO_KW_LIST_APPEND) {
assert(cframe.use_tracing == 0);
assert(call_shape.kwnames == NULL);
+ assert(GET_CACHE()->adaptive.original_oparg == 1);
SpecializedCacheEntry *caches = GET_CACHE();
- int original_oparg = caches->adaptive.original_oparg;
_PyObjectCache *cache1 = &caches[-1].obj;
- int is_meth = is_method(stack_pointer, original_oparg);
- int total_args = original_oparg + is_meth;
- PyObject *callable = PEEK(total_args + 1);
- DEOPT_IF(total_args != 2, CALL);
- DEOPT_IF(callable != cache1->obj, CALL);
+ assert(cache1->obj != NULL);
+ PyObject *callable = PEEK(3);
+ DEOPT_IF(callable != cache1->obj, PRECALL);
PyObject *list = SECOND();
- DEOPT_IF(!PyList_Check(list), CALL);
- STAT_INC(CALL, hit);
+ DEOPT_IF(!PyList_Check(list), PRECALL);
+ STAT_INC(PRECALL, hit);
+ next_instr++; // Skip following call
PyObject *arg = TOP();
int err = PyList_Append(list, arg);
if (err) {
}
Py_DECREF(arg);
Py_DECREF(list);
- STACK_SHRINK(3-is_meth);
+ STACK_SHRINK(2);
Py_INCREF(Py_None);
SET_TOP(Py_None);
Py_DECREF(callable);
NOTRACE_DISPATCH();
}
- TARGET(CALL_NO_KW_METHOD_DESCRIPTOR_O) {
+ TARGET(PRECALL_NO_KW_METHOD_DESCRIPTOR_O) {
assert(call_shape.kwnames == NULL);
int original_oparg = GET_CACHE()->adaptive.original_oparg;
int is_meth = is_method(stack_pointer, original_oparg);
int total_args = original_oparg + is_meth;
PyObject *callable = PEEK(total_args + 1);
- DEOPT_IF(total_args != 2, CALL);
- DEOPT_IF(!Py_IS_TYPE(callable, &PyMethodDescr_Type), CALL);
+ DEOPT_IF(total_args != 2, PRECALL);
+ DEOPT_IF(!Py_IS_TYPE(callable, &PyMethodDescr_Type), PRECALL);
PyMethodDef *meth = ((PyMethodDescrObject *)callable)->d_method;
- DEOPT_IF(meth->ml_flags != METH_O, CALL);
- STAT_INC(CALL, hit);
+ DEOPT_IF(meth->ml_flags != METH_O, PRECALL);
+ next_instr++; // Skip following call
+ STAT_INC(PRECALL, hit);
PyCFunction cfunc = meth->ml_meth;
// This is slower but CPython promises to check all non-vectorcall
// function calls.
assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL));
Py_DECREF(self);
Py_DECREF(arg);
- STACK_SHRINK(3-is_meth);
+ STACK_SHRINK(original_oparg+1);
SET_TOP(res);
Py_DECREF(callable);
if (res == NULL) {
DISPATCH();
}
- TARGET(CALL_NO_KW_METHOD_DESCRIPTOR_NOARGS) {
+ TARGET(PRECALL_NO_KW_METHOD_DESCRIPTOR_NOARGS) {
assert(call_shape.kwnames == NULL);
int original_oparg = GET_CACHE()->adaptive.original_oparg;
+ assert(original_oparg == 0 || original_oparg == 1);
int is_meth = is_method(stack_pointer, original_oparg);
int total_args = original_oparg + is_meth;
- PyObject *callable = PEEK(total_args + 1);
- DEOPT_IF(total_args != 1, CALL);
- DEOPT_IF(!Py_IS_TYPE(callable, &PyMethodDescr_Type), CALL);
+ DEOPT_IF(total_args != 1, PRECALL);
+ PyObject *callable = SECOND();
+ DEOPT_IF(!Py_IS_TYPE(callable, &PyMethodDescr_Type), PRECALL);
PyMethodDef *meth = ((PyMethodDescrObject *)callable)->d_method;
- DEOPT_IF(meth->ml_flags != METH_NOARGS, CALL);
- STAT_INC(CALL, hit);
+ DEOPT_IF(meth->ml_flags != METH_NOARGS, PRECALL);
+ next_instr++; // Skip following call
+ STAT_INC(PRECALL, hit);
PyCFunction cfunc = meth->ml_meth;
// This is slower but CPython promises to check all non-vectorcall
// function calls.
_Py_LeaveRecursiveCall(tstate);
assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL));
Py_DECREF(self);
- STACK_SHRINK(2-is_meth);
+ STACK_SHRINK(original_oparg+1);
SET_TOP(res);
Py_DECREF(callable);
if (res == NULL) {
DISPATCH();
}
- TARGET(CALL_NO_KW_METHOD_DESCRIPTOR_FAST) {
+ TARGET(PRECALL_NO_KW_METHOD_DESCRIPTOR_FAST) {
assert(call_shape.kwnames == NULL);
int original_oparg = GET_CACHE()->adaptive.original_oparg;
int is_meth = is_method(stack_pointer, original_oparg);
int total_args = original_oparg + is_meth;
PyObject *callable = PEEK(total_args + 1);
/* Builtin METH_FASTCALL methods, without keywords */
- DEOPT_IF(!Py_IS_TYPE(callable, &PyMethodDescr_Type), CALL);
+ DEOPT_IF(!Py_IS_TYPE(callable, &PyMethodDescr_Type), PRECALL);
PyMethodDef *meth = ((PyMethodDescrObject *)callable)->d_method;
- DEOPT_IF(meth->ml_flags != METH_FASTCALL, CALL);
- STAT_INC(CALL, hit);
+ DEOPT_IF(meth->ml_flags != METH_FASTCALL, PRECALL);
+ next_instr++; // Skip following call
+ STAT_INC(PRECALL, hit);
_PyCFunctionFast cfunc = (_PyCFunctionFast)(void(*)(void))meth->ml_meth;
int nargs = total_args-1;
STACK_SHRINK(nargs);
MISS_WITH_CACHE(STORE_ATTR)
MISS_WITH_CACHE(LOAD_GLOBAL)
MISS_WITH_CACHE(LOAD_METHOD)
+MISS_WITH_CACHE(PRECALL)
MISS_WITH_CACHE(CALL)
MISS_WITH_CACHE(BINARY_OP)
MISS_WITH_CACHE(COMPARE_OP)
&&TARGET_MATCH_KEYS,
&&TARGET_CALL_ADAPTIVE,
&&TARGET_PUSH_EXC_INFO,
- &&TARGET_CALL_BUILTIN_CLASS,
- &&TARGET_CALL_NO_KW_BUILTIN_O,
- &&TARGET_CALL_NO_KW_BUILTIN_FAST,
- &&TARGET_CALL_BUILTIN_FAST_WITH_KEYWORDS,
- &&TARGET_CALL_NO_KW_LEN,
- &&TARGET_CALL_NO_KW_ISINSTANCE,
&&TARGET_CALL_PY_EXACT_ARGS,
&&TARGET_CALL_PY_WITH_DEFAULTS,
- &&TARGET_CALL_NO_KW_LIST_APPEND,
- &&TARGET_CALL_NO_KW_METHOD_DESCRIPTOR_O,
- &&TARGET_CALL_NO_KW_METHOD_DESCRIPTOR_NOARGS,
- &&TARGET_CALL_NO_KW_STR_1,
- &&TARGET_CALL_NO_KW_TUPLE_1,
- &&TARGET_WITH_EXCEPT_START,
- &&TARGET_GET_AITER,
- &&TARGET_GET_ANEXT,
- &&TARGET_BEFORE_ASYNC_WITH,
- &&TARGET_BEFORE_WITH,
- &&TARGET_END_ASYNC_FOR,
- &&TARGET_CALL_NO_KW_TYPE_1,
- &&TARGET_CALL_NO_KW_METHOD_DESCRIPTOR_FAST,
&&TARGET_JUMP_ABSOLUTE_QUICK,
&&TARGET_LOAD_ATTR_ADAPTIVE,
&&TARGET_LOAD_ATTR_INSTANCE_VALUE,
- &&TARGET_STORE_SUBSCR,
- &&TARGET_DELETE_SUBSCR,
&&TARGET_LOAD_ATTR_WITH_HINT,
&&TARGET_LOAD_ATTR_SLOT,
&&TARGET_LOAD_ATTR_MODULE,
&&TARGET_LOAD_GLOBAL_ADAPTIVE,
&&TARGET_LOAD_GLOBAL_MODULE,
&&TARGET_LOAD_GLOBAL_BUILTIN,
+ &&TARGET_LOAD_METHOD_ADAPTIVE,
+ &&TARGET_LOAD_METHOD_CACHED,
+ &&TARGET_WITH_EXCEPT_START,
+ &&TARGET_GET_AITER,
+ &&TARGET_GET_ANEXT,
+ &&TARGET_BEFORE_ASYNC_WITH,
+ &&TARGET_BEFORE_WITH,
+ &&TARGET_END_ASYNC_FOR,
+ &&TARGET_LOAD_METHOD_CLASS,
+ &&TARGET_LOAD_METHOD_MODULE,
+ &&TARGET_LOAD_METHOD_NO_DICT,
+ &&TARGET_PRECALL_ADAPTIVE,
+ &&TARGET_PRECALL_BUILTIN_CLASS,
+ &&TARGET_STORE_SUBSCR,
+ &&TARGET_DELETE_SUBSCR,
+ &&TARGET_PRECALL_NO_KW_BUILTIN_O,
+ &&TARGET_PRECALL_NO_KW_BUILTIN_FAST,
+ &&TARGET_PRECALL_BUILTIN_FAST_WITH_KEYWORDS,
+ &&TARGET_PRECALL_NO_KW_LEN,
+ &&TARGET_PRECALL_NO_KW_ISINSTANCE,
+ &&TARGET_PRECALL_NO_KW_LIST_APPEND,
&&TARGET_GET_ITER,
&&TARGET_GET_YIELD_FROM_ITER,
&&TARGET_PRINT_EXPR,
&&TARGET_LOAD_BUILD_CLASS,
- &&TARGET_LOAD_METHOD_ADAPTIVE,
+ &&TARGET_PRECALL_NO_KW_METHOD_DESCRIPTOR_O,
&&TARGET_GET_AWAITABLE,
&&TARGET_LOAD_ASSERTION_ERROR,
&&TARGET_RETURN_GENERATOR,
- &&TARGET_LOAD_METHOD_CACHED,
- &&TARGET_LOAD_METHOD_CLASS,
- &&TARGET_LOAD_METHOD_MODULE,
- &&TARGET_LOAD_METHOD_NO_DICT,
- &&TARGET_RESUME_QUICK,
- &&TARGET_STORE_ATTR_ADAPTIVE,
+ &&TARGET_PRECALL_NO_KW_METHOD_DESCRIPTOR_NOARGS,
+ &&TARGET_PRECALL_NO_KW_STR_1,
+ &&TARGET_PRECALL_NO_KW_TUPLE_1,
+ &&TARGET_PRECALL_NO_KW_TYPE_1,
+ &&TARGET_PRECALL_NO_KW_METHOD_DESCRIPTOR_FAST,
+ &&TARGET_PRECALL_BOUND_METHOD,
&&TARGET_LIST_TO_TUPLE,
&&TARGET_RETURN_VALUE,
&&TARGET_IMPORT_STAR,
&&TARGET_POP_JUMP_IF_NOT_NONE,
&&TARGET_POP_JUMP_IF_NONE,
&&TARGET_RAISE_VARARGS,
- &&TARGET_STORE_ATTR_INSTANCE_VALUE,
+ &&TARGET_PRECALL_PYFUNC,
&&TARGET_MAKE_FUNCTION,
&&TARGET_BUILD_SLICE,
&&TARGET_JUMP_NO_INTERRUPT,
&&TARGET_LOAD_DEREF,
&&TARGET_STORE_DEREF,
&&TARGET_DELETE_DEREF,
- &&TARGET_STORE_ATTR_SLOT,
- &&TARGET_STORE_ATTR_WITH_HINT,
+ &&TARGET_RESUME_QUICK,
+ &&TARGET_STORE_ATTR_ADAPTIVE,
&&TARGET_CALL_FUNCTION_EX,
- &&TARGET_UNPACK_SEQUENCE_ADAPTIVE,
+ &&TARGET_STORE_ATTR_INSTANCE_VALUE,
&&TARGET_EXTENDED_ARG,
&&TARGET_LIST_APPEND,
&&TARGET_SET_ADD,
&&TARGET_MAP_ADD,
&&TARGET_LOAD_CLASSDEREF,
&&TARGET_COPY_FREE_VARS,
- &&TARGET_UNPACK_SEQUENCE_LIST,
+ &&TARGET_STORE_ATTR_SLOT,
&&TARGET_RESUME,
&&TARGET_MATCH_CLASS,
- &&TARGET_UNPACK_SEQUENCE_TUPLE,
- &&TARGET_UNPACK_SEQUENCE_TWO_TUPLE,
+ &&TARGET_STORE_ATTR_WITH_HINT,
+ &&TARGET_UNPACK_SEQUENCE_ADAPTIVE,
&&TARGET_FORMAT_VALUE,
&&TARGET_BUILD_CONST_KEY_MAP,
&&TARGET_BUILD_STRING,
- &&TARGET_LOAD_FAST__LOAD_FAST,
- &&TARGET_STORE_FAST__LOAD_FAST,
+ &&TARGET_UNPACK_SEQUENCE_LIST,
+ &&TARGET_UNPACK_SEQUENCE_TUPLE,
&&TARGET_LOAD_METHOD,
- &&TARGET_LOAD_FAST__LOAD_CONST,
+ &&TARGET_UNPACK_SEQUENCE_TWO_TUPLE,
&&TARGET_LIST_EXTEND,
&&TARGET_SET_UPDATE,
&&TARGET_DICT_MERGE,
&&TARGET_DICT_UPDATE,
&&TARGET_PRECALL,
+ &&TARGET_LOAD_FAST__LOAD_FAST,
+ &&TARGET_STORE_FAST__LOAD_FAST,
+ &&TARGET_LOAD_FAST__LOAD_CONST,
&&TARGET_LOAD_CONST__LOAD_FAST,
- &&TARGET_STORE_FAST__STORE_FAST,
- &&_unknown_opcode,
- &&_unknown_opcode,
&&TARGET_CALL,
&&TARGET_KW_NAMES,
- &&_unknown_opcode,
+ &&TARGET_STORE_FAST__STORE_FAST,
&&_unknown_opcode,
&&_unknown_opcode,
&&_unknown_opcode,
[BINARY_SUBSCR] = BINARY_SUBSCR_ADAPTIVE,
[STORE_SUBSCR] = STORE_SUBSCR_ADAPTIVE,
[CALL] = CALL_ADAPTIVE,
+ [PRECALL] = PRECALL_ADAPTIVE,
[STORE_ATTR] = STORE_ATTR_ADAPTIVE,
[BINARY_OP] = BINARY_OP_ADAPTIVE,
[COMPARE_OP] = COMPARE_OP_ADAPTIVE,
[BINARY_SUBSCR] = 2, /* _PyAdaptiveEntry, _PyObjectCache */
[STORE_SUBSCR] = 0,
[CALL] = 2, /* _PyAdaptiveEntry and _PyObjectCache/_PyCallCache */
+ [PRECALL] = 2, /* _PyAdaptiveEntry and _PyObjectCache/_PyCallCache */
[STORE_ATTR] = 2, /* _PyAdaptiveEntry and _PyAttrCache */
[BINARY_OP] = 1, // _PyAdaptiveEntry
[COMPARE_OP] = 1, /* _PyAdaptiveEntry */
PyObject *callable, _Py_CODEUNIT *instr,
int nargs, PyObject *kwnames, SpecializedCacheEntry *cache)
{
+ assert(_Py_OPCODE(*instr) == PRECALL_ADAPTIVE);
PyTypeObject *tp = _PyType_CAST(callable);
if (tp->tp_new == PyBaseObject_Type.tp_new) {
- SPECIALIZATION_FAIL(CALL, SPEC_FAIL_CALL_PYTHON_CLASS);
+ SPECIALIZATION_FAIL(PRECALL, SPEC_FAIL_CALL_PYTHON_CLASS);
return -1;
}
if (tp->tp_flags & Py_TPFLAGS_IMMUTABLETYPE) {
- if (nargs == 1 && kwnames == NULL) {
+ if (nargs == 1 && kwnames == NULL && cache->adaptive.original_oparg == 1) {
if (tp == &PyUnicode_Type) {
- *instr = _Py_MAKECODEUNIT(CALL_NO_KW_STR_1, _Py_OPARG(*instr));
+ *instr = _Py_MAKECODEUNIT(PRECALL_NO_KW_STR_1, _Py_OPARG(*instr));
return 0;
}
else if (tp == &PyType_Type) {
- *instr = _Py_MAKECODEUNIT(CALL_NO_KW_TYPE_1, _Py_OPARG(*instr));
+ *instr = _Py_MAKECODEUNIT(PRECALL_NO_KW_TYPE_1, _Py_OPARG(*instr));
return 0;
}
else if (tp == &PyTuple_Type) {
- *instr = _Py_MAKECODEUNIT(CALL_NO_KW_TUPLE_1, _Py_OPARG(*instr));
+ *instr = _Py_MAKECODEUNIT(PRECALL_NO_KW_TUPLE_1, _Py_OPARG(*instr));
return 0;
}
}
if (tp->tp_vectorcall != NULL) {
- *instr = _Py_MAKECODEUNIT(CALL_BUILTIN_CLASS, _Py_OPARG(*instr));
+ *instr = _Py_MAKECODEUNIT(PRECALL_BUILTIN_CLASS, _Py_OPARG(*instr));
return 0;
}
- SPECIALIZATION_FAIL(CALL, tp == &PyUnicode_Type ?
+ SPECIALIZATION_FAIL(PRECALL, tp == &PyUnicode_Type ?
SPEC_FAIL_CALL_STR : SPEC_FAIL_CALL_CLASS_NO_VECTORCALL);
return -1;
}
- SPECIALIZATION_FAIL(CALL, SPEC_FAIL_CALL_CLASS_MUTABLE);
+ SPECIALIZATION_FAIL(PRECALL, SPEC_FAIL_CALL_CLASS_MUTABLE);
return -1;
}
PyMethodDescrObject *descr, _Py_CODEUNIT *instr,
int nargs, PyObject *kwnames, SpecializedCacheEntry *cache)
{
+ assert(_Py_OPCODE(*instr) == PRECALL_ADAPTIVE);
if (kwnames) {
- SPECIALIZATION_FAIL(CALL, SPEC_FAIL_CALL_KWNAMES);
+ SPECIALIZATION_FAIL(PRECALL, SPEC_FAIL_CALL_KWNAMES);
return -1;
}
if (_list_append == NULL) {
&_Py_ID(append));
}
assert(_list_append != NULL);
- if (nargs == 2 && descr == _list_append) {
- assert(_Py_OPCODE(instr[-1]) == PRECALL);
+ if (nargs == 2 && descr == _list_append && cache->adaptive.original_oparg == 1) {
cache[-1].obj.obj = (PyObject *)_list_append;
- *instr = _Py_MAKECODEUNIT(CALL_NO_KW_LIST_APPEND, _Py_OPARG(*instr));
+ *instr = _Py_MAKECODEUNIT(PRECALL_NO_KW_LIST_APPEND, _Py_OPARG(*instr));
return 0;
}
METH_KEYWORDS | METH_METHOD)) {
case METH_NOARGS: {
if (nargs != 1) {
- SPECIALIZATION_FAIL(CALL, SPEC_FAIL_WRONG_NUMBER_ARGUMENTS);
+ SPECIALIZATION_FAIL(PRECALL, SPEC_FAIL_WRONG_NUMBER_ARGUMENTS);
return -1;
}
- *instr = _Py_MAKECODEUNIT(CALL_NO_KW_METHOD_DESCRIPTOR_NOARGS,
+ *instr = _Py_MAKECODEUNIT(PRECALL_NO_KW_METHOD_DESCRIPTOR_NOARGS,
_Py_OPARG(*instr));
return 0;
}
case METH_O: {
if (nargs != 2) {
- SPECIALIZATION_FAIL(CALL, SPEC_FAIL_OUT_OF_RANGE);
+ SPECIALIZATION_FAIL(PRECALL, SPEC_FAIL_OUT_OF_RANGE);
return -1;
}
- *instr = _Py_MAKECODEUNIT(CALL_NO_KW_METHOD_DESCRIPTOR_O,
+ *instr = _Py_MAKECODEUNIT(PRECALL_NO_KW_METHOD_DESCRIPTOR_O,
_Py_OPARG(*instr));
return 0;
}
case METH_FASTCALL: {
- *instr = _Py_MAKECODEUNIT(CALL_NO_KW_METHOD_DESCRIPTOR_FAST,
+ *instr = _Py_MAKECODEUNIT(PRECALL_NO_KW_METHOD_DESCRIPTOR_FAST,
_Py_OPARG(*instr));
return 0;
}
}
- SPECIALIZATION_FAIL(CALL, builtin_call_fail_kind(descr->d_method->ml_flags));
+ SPECIALIZATION_FAIL(PRECALL, builtin_call_fail_kind(descr->d_method->ml_flags));
return -1;
}
PyFunctionObject *func, _Py_CODEUNIT *instr,
int nargs, PyObject *kwnames, SpecializedCacheEntry *cache)
{
+ assert(_Py_OPCODE(*instr) == CALL_ADAPTIVE);
_PyCallCache *cache1 = &cache[-1].call;
PyCodeObject *code = (PyCodeObject *)func->func_code;
int kind = function_kind(code);
specialize_c_call(PyObject *callable, _Py_CODEUNIT *instr, int nargs,
PyObject *kwnames, SpecializedCacheEntry *cache, PyObject *builtins)
{
+ assert(_Py_OPCODE(*instr) == PRECALL_ADAPTIVE);
_PyObjectCache *cache1 = &cache[-1].obj;
if (PyCFunction_GET_FUNCTION(callable) == NULL) {
return 1;
METH_KEYWORDS | METH_METHOD)) {
case METH_O: {
if (kwnames) {
- SPECIALIZATION_FAIL(CALL, SPEC_FAIL_CALL_KWNAMES);
+ SPECIALIZATION_FAIL(PRECALL, SPEC_FAIL_CALL_KWNAMES);
return -1;
}
if (nargs != 1) {
- SPECIALIZATION_FAIL(CALL, SPEC_FAIL_WRONG_NUMBER_ARGUMENTS);
+ SPECIALIZATION_FAIL(PRECALL, SPEC_FAIL_WRONG_NUMBER_ARGUMENTS);
return 1;
}
/* len(o) */
PyObject *builtin_len = PyDict_GetItemString(builtins, "len");
if (callable == builtin_len) {
cache1->obj = builtin_len; // borrowed
- *instr = _Py_MAKECODEUNIT(CALL_NO_KW_LEN,
+ *instr = _Py_MAKECODEUNIT(PRECALL_NO_KW_LEN,
_Py_OPARG(*instr));
return 0;
}
- *instr = _Py_MAKECODEUNIT(CALL_NO_KW_BUILTIN_O,
+ *instr = _Py_MAKECODEUNIT(PRECALL_NO_KW_BUILTIN_O,
_Py_OPARG(*instr));
return 0;
}
case METH_FASTCALL: {
if (kwnames) {
- SPECIALIZATION_FAIL(CALL, SPEC_FAIL_CALL_KWNAMES);
+ SPECIALIZATION_FAIL(PRECALL, SPEC_FAIL_CALL_KWNAMES);
return -1;
}
if (nargs == 2) {
builtins, "isinstance");
if (callable == builtin_isinstance) {
cache1->obj = builtin_isinstance; // borrowed
- *instr = _Py_MAKECODEUNIT(CALL_NO_KW_ISINSTANCE,
+ *instr = _Py_MAKECODEUNIT(PRECALL_NO_KW_ISINSTANCE,
_Py_OPARG(*instr));
return 0;
}
}
- *instr = _Py_MAKECODEUNIT(CALL_NO_KW_BUILTIN_FAST,
+ *instr = _Py_MAKECODEUNIT(PRECALL_NO_KW_BUILTIN_FAST,
_Py_OPARG(*instr));
return 0;
}
case METH_FASTCALL | METH_KEYWORDS: {
- *instr = _Py_MAKECODEUNIT(CALL_BUILTIN_FAST_WITH_KEYWORDS,
+ *instr = _Py_MAKECODEUNIT(PRECALL_BUILTIN_FAST_WITH_KEYWORDS,
_Py_OPARG(*instr));
return 0;
}
default:
- SPECIALIZATION_FAIL(CALL,
+ SPECIALIZATION_FAIL(PRECALL,
builtin_call_fail_kind(PyCFunction_GET_FLAGS(callable)));
return 1;
}
}
#endif
-/* TODO:
- - Specialize calling classes.
-*/
+
int
-_Py_Specialize_CallNoKw(
+_Py_Specialize_Precall(
PyObject *callable, _Py_CODEUNIT *instr,
int nargs, PyObject *kwnames,
SpecializedCacheEntry *cache, PyObject *builtins)
fail = specialize_c_call(callable, instr, nargs, kwnames, cache, builtins);
}
else if (PyFunction_Check(callable)) {
- fail = specialize_py_call((PyFunctionObject *)callable, instr, nargs, kwnames, cache);
+ *instr = _Py_MAKECODEUNIT(PRECALL_PYFUNC, _Py_OPARG(*instr));
+ fail = 0;
}
else if (PyType_Check(callable)) {
fail = specialize_class_call(callable, instr, nargs, kwnames, cache);
fail = specialize_method_descriptor(
(PyMethodDescrObject *)callable, instr, nargs, kwnames, cache);
}
+ else if (Py_TYPE(callable) == &PyMethod_Type) {
+ *instr = _Py_MAKECODEUNIT(PRECALL_BOUND_METHOD, _Py_OPARG(*instr));
+ fail = 0;
+ }
+ else {
+ SPECIALIZATION_FAIL(CALL, call_fail_kind(callable));
+ fail = -1;
+ }
+ if (fail) {
+ STAT_INC(CALL, failure);
+ assert(!PyErr_Occurred());
+ cache_backoff(cache0);
+ }
+ else {
+ STAT_INC(CALL, success);
+ assert(!PyErr_Occurred());
+ cache0->counter = initial_counter_value();
+ }
+ return 0;
+}
+
+
+/* TODO:
+ - Specialize calling classes.
+*/
+int
+_Py_Specialize_Call(
+ PyObject *callable, _Py_CODEUNIT *instr,
+ int nargs, PyObject *kwnames,
+ SpecializedCacheEntry *cache)
+{
+ _PyAdaptiveEntry *cache0 = &cache->adaptive;
+ int fail;
+ if (PyFunction_Check(callable)) {
+ fail = specialize_py_call((PyFunctionObject *)callable, instr, nargs, kwnames, cache);
+ }
else {
SPECIALIZATION_FAIL(CALL, call_fail_kind(callable));
fail = -1;
projects / thirdparty / Python / cpython.git / commitdiff
