extern int _PyRunRemoteDebugger(PyThreadState *tstate);
#endif
-_PyStackRef _PyForIter_NextWithIndex(PyObject *seq, _PyStackRef index);
+PyAPI_FUNC(_PyStackRef)
+_PyForIter_VirtualIteratorNext(PyThreadState* tstate, struct _PyInterpreterFrame* frame, _PyStackRef iter, _PyStackRef *index_ptr);
#ifdef __cplusplus
}
extern void _Py_stackref_associate(PyInterpreterState *interp, PyObject *obj, _PyStackRef ref);
static const _PyStackRef PyStackRef_NULL = { .index = 0 };
+static const _PyStackRef PyStackRef_ERROR = { .index = 2 };
// Use the first 3 even numbers for None, True and False.
// Odd numbers are reserved for (tagged) integers
-#define PyStackRef_None ((_PyStackRef){ .index = 2 } )
-#define PyStackRef_False ((_PyStackRef){ .index = 4 })
-#define PyStackRef_True ((_PyStackRef){ .index = 6 })
+#define PyStackRef_None ((_PyStackRef){ .index = 4 } )
+#define PyStackRef_False ((_PyStackRef){ .index = 6 })
+#define PyStackRef_True ((_PyStackRef){ .index = 8 })
-#define INITIAL_STACKREF_INDEX 8
+#define INITIAL_STACKREF_INDEX 10
static inline int
PyStackRef_IsNull(_PyStackRef ref)
return ref.index == 0;
}
+static inline bool
+PyStackRef_IsError(_PyStackRef ref)
+{
+ return ref.index == 2;
+}
+
+static inline bool
+PyStackRef_IsValid(_PyStackRef ref)
+{
+ /* Invalid values are ERROR and NULL */
+ return !PyStackRef_IsError(ref) && !PyStackRef_IsNull(ref);
+}
+
static inline int
PyStackRef_IsTrue(_PyStackRef ref)
{
static inline PyObject *
_PyStackRef_AsPyObjectBorrow(_PyStackRef ref, const char *filename, int linenumber)
{
+ assert(!PyStackRef_IsError(ref));
assert(!PyStackRef_IsTaggedInt(ref));
_Py_stackref_record_borrow(ref, filename, linenumber);
return _Py_stackref_get_object(ref);
static inline void
_PyStackRef_XCLOSE(_PyStackRef ref, const char *filename, int linenumber)
{
+ assert(!PyStackRef_IsError(ref));
if (PyStackRef_IsNull(ref)) {
return;
}
static inline _PyStackRef
_PyStackRef_DUP(_PyStackRef ref, const char *filename, int linenumber)
{
+ assert(!PyStackRef_IsError(ref));
if (PyStackRef_IsTaggedInt(ref)) {
return ref;
}
#else
#define Py_INT_TAG 3
+#define Py_TAG_INVALID 2
#define Py_TAG_REFCNT 1
#define Py_TAG_BITS 3
+static const _PyStackRef PyStackRef_ERROR = { .bits = Py_TAG_INVALID };
+
+static inline bool
+PyStackRef_IsError(_PyStackRef ref)
+{
+ return ref.bits == Py_TAG_INVALID;
+}
+
+static inline bool
+PyStackRef_IsValid(_PyStackRef ref)
+{
+ /* Invalid values are ERROR and NULL */
+ return ref.bits >= Py_INT_TAG;
+}
+
static inline bool
PyStackRef_IsTaggedInt(_PyStackRef i)
{
static const _PyStackRef PyStackRef_NULL = { .bits = Py_TAG_DEFERRED};
+
#define PyStackRef_IsNull(stackref) ((stackref).bits == PyStackRef_NULL.bits)
#define PyStackRef_True ((_PyStackRef){.bits = ((uintptr_t)&_Py_TrueStruct) | Py_TAG_DEFERRED })
#define PyStackRef_False ((_PyStackRef){.bits = ((uintptr_t)&_Py_FalseStruct) | Py_TAG_DEFERRED })
self.assertIsNotNone(ex)
self.assertIn("_RETURN_GENERATOR", get_opnames(ex))
+ def test_for_iter(self):
+ def testfunc(n):
+ t = 0
+ for i in set(range(n)):
+ t += i
+ return t
+ res, ex = self._run_with_optimizer(testfunc, TIER2_THRESHOLD)
+ self.assertEqual(res, TIER2_THRESHOLD * (TIER2_THRESHOLD - 1) // 2)
+ self.assertIsNotNone(ex)
+ self.assertIn("_FOR_ITER_TIER_TWO", get_opnames(ex))
+
@unittest.skip("Tracing into generators currently isn't supported.")
def test_for_iter_gen(self):
def gen(n):
}
replaced op(_FOR_ITER, (iter, null_or_index -- iter, null_or_index, next)) {
- /* before: [iter]; after: [iter, iter()] *or* [] (and jump over END_FOR.) */
- PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter);
- if (PyStackRef_IsTaggedInt(null_or_index)) {
- next = _PyForIter_NextWithIndex(iter_o, null_or_index);
- if (PyStackRef_IsNull(next)) {
- JUMPBY(oparg + 1);
- DISPATCH();
- }
- null_or_index = PyStackRef_IncrementTaggedIntNoOverflow(null_or_index);
- }
- else {
- PyObject *next_o = (*Py_TYPE(iter_o)->tp_iternext)(iter_o);
- if (next_o == NULL) {
- if (_PyErr_Occurred(tstate)) {
- int matches = _PyErr_ExceptionMatches(tstate, PyExc_StopIteration);
- if (!matches) {
- ERROR_NO_POP();
- }
- _PyEval_MonitorRaise(tstate, frame, this_instr);
- _PyErr_Clear(tstate);
- }
- /* iterator ended normally */
- assert(next_instr[oparg].op.code == END_FOR ||
- next_instr[oparg].op.code == INSTRUMENTED_END_FOR);
- /* Jump forward oparg, then skip following END_FOR */
- JUMPBY(oparg + 1);
- DISPATCH();
+ _PyStackRef item = _PyForIter_VirtualIteratorNext(tstate, frame, iter, &null_or_index);
+ if (!PyStackRef_IsValid(item)) {
+ if (PyStackRef_IsError(item)) {
+ ERROR_NO_POP();
}
- next = PyStackRef_FromPyObjectSteal(next_o);
+ // Jump forward by oparg and skip the following END_FOR
+ JUMPBY(oparg + 1);
+ DISPATCH();
}
+ next = item;
}
op(_FOR_ITER_TIER_TWO, (iter, null_or_index -- iter, null_or_index, next)) {
- /* before: [iter]; after: [iter, iter()] *or* [] (and jump over END_FOR.) */
- PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter);
- EXIT_IF(!PyStackRef_IsNull(null_or_index));
- PyObject *next_o = (*Py_TYPE(iter_o)->tp_iternext)(iter_o);
- if (next_o == NULL) {
- if (_PyErr_Occurred(tstate)) {
- int matches = _PyErr_ExceptionMatches(tstate, PyExc_StopIteration);
- if (!matches) {
- ERROR_NO_POP();
- }
- _PyEval_MonitorRaise(tstate, frame, frame->instr_ptr);
- _PyErr_Clear(tstate);
+ _PyStackRef item = _PyForIter_VirtualIteratorNext(tstate, frame, iter, &null_or_index);
+ if (!PyStackRef_IsValid(item)) {
+ if (PyStackRef_IsError(item)) {
+ ERROR_NO_POP();
}
/* iterator ended normally */
/* The translator sets the deopt target just past the matching END_FOR */
EXIT_IF(true);
}
- next = PyStackRef_FromPyObjectSteal(next_o);
- // Common case: no jump, leave it to the code generator
+ next = item;
}
+
macro(FOR_ITER) = _SPECIALIZE_FOR_ITER + _FOR_ITER;
inst(INSTRUMENTED_FOR_ITER, (unused/1, iter, null_or_index -- iter, null_or_index, next)) {
- PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter);
- if (PyStackRef_IsTaggedInt(null_or_index)) {
- next = _PyForIter_NextWithIndex(iter_o, null_or_index);
- if (PyStackRef_IsNull(next)) {
- JUMPBY(oparg + 1);
- DISPATCH();
- }
- null_or_index = PyStackRef_IncrementTaggedIntNoOverflow(null_or_index);
- INSTRUMENTED_JUMP(this_instr, next_instr, PY_MONITORING_EVENT_BRANCH_LEFT);
- }
- else {
- PyObject *next_o = (*Py_TYPE(iter_o)->tp_iternext)(iter_o);
- if (next_o != NULL) {
- next = PyStackRef_FromPyObjectSteal(next_o);
- INSTRUMENTED_JUMP(this_instr, next_instr, PY_MONITORING_EVENT_BRANCH_LEFT);
- }
- else {
- if (_PyErr_Occurred(tstate)) {
- int matches = _PyErr_ExceptionMatches(tstate, PyExc_StopIteration);
- if (!matches) {
- ERROR_NO_POP();
- }
- _PyEval_MonitorRaise(tstate, frame, this_instr);
- _PyErr_Clear(tstate);
- }
- /* iterator ended normally */
- assert(next_instr[oparg].op.code == END_FOR ||
- next_instr[oparg].op.code == INSTRUMENTED_END_FOR);
- /* Skip END_FOR */
- JUMPBY(oparg + 1);
- DISPATCH();
+ _PyStackRef item = _PyForIter_VirtualIteratorNext(tstate, frame, iter, &null_or_index);
+ if (!PyStackRef_IsValid(item)) {
+ if (PyStackRef_IsError(item)) {
+ ERROR_NO_POP();
}
+ // Jump forward by oparg and skip the following END_FOR
+ JUMPBY(oparg + 1);
+ DISPATCH();
}
+ next = item;
+ INSTRUMENTED_JUMP(this_instr, next_instr, PY_MONITORING_EVENT_BRANCH_LEFT);
}
-
op(_ITER_CHECK_LIST, (iter, null_or_index -- iter, null_or_index)) {
PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter);
EXIT_IF(Py_TYPE(iter_o) != &PyList_Type);
return value;
}
-_PyStackRef
-_PyForIter_NextWithIndex(PyObject *seq, _PyStackRef index)
+static _PyStackRef
+foriter_next(PyObject *seq, _PyStackRef index)
{
assert(PyStackRef_IsTaggedInt(index));
assert(PyTuple_CheckExact(seq) || PyList_CheckExact(seq));
return PyStackRef_FromPyObjectSteal(item);
}
+_PyStackRef _PyForIter_VirtualIteratorNext(PyThreadState* tstate, _PyInterpreterFrame* frame, _PyStackRef iter, _PyStackRef* index_ptr)
+{
+ PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter);
+ _PyStackRef index = *index_ptr;
+ if (PyStackRef_IsTaggedInt(index)) {
+ *index_ptr = PyStackRef_IncrementTaggedIntNoOverflow(index);
+ return foriter_next(iter_o, index);
+ }
+ PyObject *next_o = (*Py_TYPE(iter_o)->tp_iternext)(iter_o);
+ if (next_o == NULL) {
+ if (_PyErr_Occurred(tstate)) {
+ if (_PyErr_ExceptionMatches(tstate, PyExc_StopIteration)) {
+ _PyEval_MonitorRaise(tstate, frame, frame->instr_ptr);
+ _PyErr_Clear(tstate);
+ }
+ else {
+ return PyStackRef_ERROR;
+ }
+ }
+ return PyStackRef_NULL;
+ }
+ return PyStackRef_FromPyObjectSteal(next_o);
+}
+
/* Check if a 'cls' provides the given special method. */
static inline int
type_has_special_method(PyTypeObject *cls, PyObject *name)
_PyStackRef next;
null_or_index = stack_pointer[-1];
iter = stack_pointer[-2];
- PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter);
- if (!PyStackRef_IsNull(null_or_index)) {
- UOP_STAT_INC(uopcode, miss);
- JUMP_TO_JUMP_TARGET();
- }
_PyFrame_SetStackPointer(frame, stack_pointer);
- PyObject *next_o = (*Py_TYPE(iter_o)->tp_iternext)(iter_o);
+ _PyStackRef item = _PyForIter_VirtualIteratorNext(tstate, frame, iter, &null_or_index);
stack_pointer = _PyFrame_GetStackPointer(frame);
- if (next_o == NULL) {
- if (_PyErr_Occurred(tstate)) {
- _PyFrame_SetStackPointer(frame, stack_pointer);
- int matches = _PyErr_ExceptionMatches(tstate, PyExc_StopIteration);
- stack_pointer = _PyFrame_GetStackPointer(frame);
- if (!matches) {
- JUMP_TO_ERROR();
- }
- _PyFrame_SetStackPointer(frame, stack_pointer);
- _PyEval_MonitorRaise(tstate, frame, frame->instr_ptr);
- _PyErr_Clear(tstate);
- stack_pointer = _PyFrame_GetStackPointer(frame);
+ if (!PyStackRef_IsValid(item)) {
+ if (PyStackRef_IsError(item)) {
+ JUMP_TO_ERROR();
}
if (true) {
UOP_STAT_INC(uopcode, miss);
JUMP_TO_JUMP_TARGET();
}
}
- next = PyStackRef_FromPyObjectSteal(next_o);
+ next = item;
+ stack_pointer[-1] = null_or_index;
stack_pointer[0] = next;
stack_pointer += 1;
assert(WITHIN_STACK_BOUNDS());
}
// _FOR_ITER
{
- PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter);
- if (PyStackRef_IsTaggedInt(null_or_index)) {
- _PyFrame_SetStackPointer(frame, stack_pointer);
- next = _PyForIter_NextWithIndex(iter_o, null_or_index);
- stack_pointer = _PyFrame_GetStackPointer(frame);
- if (PyStackRef_IsNull(next)) {
- JUMPBY(oparg + 1);
- DISPATCH();
- }
- null_or_index = PyStackRef_IncrementTaggedIntNoOverflow(null_or_index);
- }
- else {
- _PyFrame_SetStackPointer(frame, stack_pointer);
- PyObject *next_o = (*Py_TYPE(iter_o)->tp_iternext)(iter_o);
- stack_pointer = _PyFrame_GetStackPointer(frame);
- if (next_o == NULL) {
- if (_PyErr_Occurred(tstate)) {
- _PyFrame_SetStackPointer(frame, stack_pointer);
- int matches = _PyErr_ExceptionMatches(tstate, PyExc_StopIteration);
- stack_pointer = _PyFrame_GetStackPointer(frame);
- if (!matches) {
- JUMP_TO_LABEL(error);
- }
- _PyFrame_SetStackPointer(frame, stack_pointer);
- _PyEval_MonitorRaise(tstate, frame, this_instr);
- _PyErr_Clear(tstate);
- stack_pointer = _PyFrame_GetStackPointer(frame);
- }
- assert(next_instr[oparg].op.code == END_FOR ||
- next_instr[oparg].op.code == INSTRUMENTED_END_FOR);
- JUMPBY(oparg + 1);
- DISPATCH();
+ _PyFrame_SetStackPointer(frame, stack_pointer);
+ _PyStackRef item = _PyForIter_VirtualIteratorNext(tstate, frame, iter, &null_or_index);
+ stack_pointer = _PyFrame_GetStackPointer(frame);
+ if (!PyStackRef_IsValid(item)) {
+ if (PyStackRef_IsError(item)) {
+ JUMP_TO_LABEL(error);
}
- next = PyStackRef_FromPyObjectSteal(next_o);
+ JUMPBY(oparg + 1);
+ stack_pointer[-1] = null_or_index;
+ DISPATCH();
}
+ next = item;
}
stack_pointer[-1] = null_or_index;
stack_pointer[0] = next;
/* Skip 1 cache entry */
null_or_index = stack_pointer[-1];
iter = stack_pointer[-2];
- PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter);
- if (PyStackRef_IsTaggedInt(null_or_index)) {
- _PyFrame_SetStackPointer(frame, stack_pointer);
- next = _PyForIter_NextWithIndex(iter_o, null_or_index);
- stack_pointer = _PyFrame_GetStackPointer(frame);
- if (PyStackRef_IsNull(next)) {
- JUMPBY(oparg + 1);
- DISPATCH();
- }
- null_or_index = PyStackRef_IncrementTaggedIntNoOverflow(null_or_index);
- INSTRUMENTED_JUMP(this_instr, next_instr, PY_MONITORING_EVENT_BRANCH_LEFT);
- }
- else {
- _PyFrame_SetStackPointer(frame, stack_pointer);
- PyObject *next_o = (*Py_TYPE(iter_o)->tp_iternext)(iter_o);
- stack_pointer = _PyFrame_GetStackPointer(frame);
- if (next_o != NULL) {
- next = PyStackRef_FromPyObjectSteal(next_o);
- INSTRUMENTED_JUMP(this_instr, next_instr, PY_MONITORING_EVENT_BRANCH_LEFT);
- }
- else {
- if (_PyErr_Occurred(tstate)) {
- _PyFrame_SetStackPointer(frame, stack_pointer);
- int matches = _PyErr_ExceptionMatches(tstate, PyExc_StopIteration);
- stack_pointer = _PyFrame_GetStackPointer(frame);
- if (!matches) {
- JUMP_TO_LABEL(error);
- }
- _PyFrame_SetStackPointer(frame, stack_pointer);
- _PyEval_MonitorRaise(tstate, frame, this_instr);
- _PyErr_Clear(tstate);
- stack_pointer = _PyFrame_GetStackPointer(frame);
- }
- assert(next_instr[oparg].op.code == END_FOR ||
- next_instr[oparg].op.code == INSTRUMENTED_END_FOR);
- JUMPBY(oparg + 1);
- DISPATCH();
+ _PyFrame_SetStackPointer(frame, stack_pointer);
+ _PyStackRef item = _PyForIter_VirtualIteratorNext(tstate, frame, iter, &null_or_index);
+ stack_pointer = _PyFrame_GetStackPointer(frame);
+ if (!PyStackRef_IsValid(item)) {
+ if (PyStackRef_IsError(item)) {
+ JUMP_TO_LABEL(error);
}
+ JUMPBY(oparg + 1);
+ stack_pointer[-1] = null_or_index;
+ DISPATCH();
}
+ next = item;
+ INSTRUMENTED_JUMP(this_instr, next_instr, PY_MONITORING_EVENT_BRANCH_LEFT);
stack_pointer[-1] = null_or_index;
stack_pointer[0] = next;
stack_pointer += 1;
value = sym_new_unknown(ctx);
}
+ op(_GET_ITER, (iterable -- iter, index_or_null)) {
+ if (sym_matches_type(iterable, &PyTuple_Type) || sym_matches_type(iterable, &PyList_Type)) {
+ iter = iterable;
+ index_or_null = sym_new_not_null(ctx);
+ }
+ else {
+ iter = sym_new_not_null(ctx);
+ index_or_null = sym_new_unknown(ctx);
+ }
+ }
+
op(_FOR_ITER_GEN_FRAME, (unused, unused -- unused, unused, gen_frame: _Py_UOpsAbstractFrame*)) {
gen_frame = NULL;
/* We are about to hit the end of the trace */
}
case _GET_ITER: {
+ JitOptSymbol *iterable;
JitOptSymbol *iter;
JitOptSymbol *index_or_null;
- iter = sym_new_not_null(ctx);
- index_or_null = sym_new_not_null(ctx);
+ iterable = stack_pointer[-1];
+ if (sym_matches_type(iterable, &PyTuple_Type) || sym_matches_type(iterable, &PyList_Type)) {
+ iter = iterable;
+ index_or_null = sym_new_not_null(ctx);
+ }
+ else {
+ iter = sym_new_not_null(ctx);
+ index_or_null = sym_new_unknown(ctx);
+ }
stack_pointer[-1] = iter;
stack_pointer[0] = index_or_null;
stack_pointer += 1;
PyObject *
_Py_stackref_get_object(_PyStackRef ref)
{
+ assert(!PyStackRef_IsError(ref));
if (ref.index == 0) {
return NULL;
}
PyObject *
_Py_stackref_close(_PyStackRef ref, const char *filename, int linenumber)
{
+ assert(!PyStackRef_IsError(ref));
PyInterpreterState *interp = PyInterpreterState_Get();
if (ref.index >= interp->next_stackref) {
_Py_FatalErrorFormat(__func__, "Invalid StackRef with ID %" PRIu64 " at %s:%d\n", (void *)ref.index, filename, linenumber);
void
_Py_stackref_record_borrow(_PyStackRef ref, const char *filename, int linenumber)
{
+ assert(!PyStackRef_IsError(ref));
if (ref.index < INITIAL_STACKREF_INDEX) {
return;
}
void
_Py_stackref_associate(PyInterpreterState *interp, PyObject *obj, _PyStackRef ref)
{
+ assert(!PyStackRef_IsError(ref));
assert(ref.index < INITIAL_STACKREF_INDEX);
TableEntry *entry = make_table_entry(obj, "builtin-object", 0);
if (entry == NULL) {
"PyStackRef_UntagInt",
"PyStackRef_IncrementTaggedIntNoOverflow",
"PyStackRef_IsNullOrInt",
+ "PyStackRef_IsError",
+ "PyStackRef_IsValid",
)
f"Expected '{undefined}' to be defined before '{out.name}'"
else:
undefined = out.name
- while len(self.outputs) > self.peeks and not self.needs_defining(self.outputs[0]):
+ while len(self.outputs) > self.peeks and not self.needs_defining(self.outputs[self.peeks]):
out = self.outputs.pop(self.peeks)
self.stack.push(out)
projects / thirdparty / Python / cpython.git / commitdiff
