This PR sets up tagged pointers for CPython.
The general idea is to create a separate struct _PyStackRef for everything on the evaluation stack to store the bits. This forces the C compiler to warn us if we try to cast things or pull things out of the struct directly.
Only for free threading: We tag the low bit if something is deferred - that means we skip incref and decref operations on it. This behavior may change in the future if Mark's plans to defer all objects in the interpreter loop pans out.
This implies a strict stack reference discipline is required. ALL incref and decref operations on stackrefs must use the stackref variants. It is unsafe to untag something then do normal incref/decref ops on it.
The new incref and decref variants are called dup and close. They mimic a "handle" API operating on these stackrefs.
Please read Include/internal/pycore_stackref.h for more information!
---------
Co-authored-by: Mark Shannon <9448417+markshannon@users.noreply.github.com>
PyAPI_FUNC(void) _PyEval_FormatKwargsError(PyThreadState *tstate, PyObject *func, PyObject *kwargs);
PyAPI_FUNC(PyObject *)_PyEval_MatchClass(PyThreadState *tstate, PyObject *subject, PyObject *type, Py_ssize_t nargs, PyObject *kwargs);
PyAPI_FUNC(PyObject *)_PyEval_MatchKeys(PyThreadState *tstate, PyObject *map, PyObject *keys);
-PyAPI_FUNC(int) _PyEval_UnpackIterable(PyThreadState *tstate, PyObject *v, int argcnt, int argcntafter, PyObject **sp);
+PyAPI_FUNC(int) _PyEval_UnpackIterableStackRef(PyThreadState *tstate, _PyStackRef v, int argcnt, int argcntafter, _PyStackRef *sp);
PyAPI_FUNC(void) _PyEval_FrameClearAndPop(PyThreadState *tstate, _PyInterpreterFrame *frame);
+PyAPI_FUNC(PyObject **) _PyObjectArray_FromStackRefArray(_PyStackRef *input, Py_ssize_t nargs, PyObject **scratch);
+
+PyAPI_FUNC(void) _PyObjectArray_Free(PyObject **array, PyObject **scratch);
/* Bits that can be set in PyThreadState.eval_breaker */
# error "this header requires Py_BUILD_CORE define"
#endif
+#include "pycore_stackref.h" // _PyStackRef
#include "pycore_lock.h" // PyMutex
#include "pycore_backoff.h" // _Py_BackoffCounter
/* Specialization functions */
-extern void _Py_Specialize_LoadSuperAttr(PyObject *global_super, PyObject *cls,
+extern void _Py_Specialize_LoadSuperAttr(_PyStackRef global_super, _PyStackRef cls,
_Py_CODEUNIT *instr, int load_method);
-extern void _Py_Specialize_LoadAttr(PyObject *owner, _Py_CODEUNIT *instr,
+extern void _Py_Specialize_LoadAttr(_PyStackRef owner, _Py_CODEUNIT *instr,
PyObject *name);
-extern void _Py_Specialize_StoreAttr(PyObject *owner, _Py_CODEUNIT *instr,
+extern void _Py_Specialize_StoreAttr(_PyStackRef owner, _Py_CODEUNIT *instr,
PyObject *name);
extern void _Py_Specialize_LoadGlobal(PyObject *globals, PyObject *builtins,
_Py_CODEUNIT *instr, PyObject *name);
-extern void _Py_Specialize_BinarySubscr(PyObject *sub, PyObject *container,
+extern void _Py_Specialize_BinarySubscr(_PyStackRef sub, _PyStackRef container,
_Py_CODEUNIT *instr);
-extern void _Py_Specialize_StoreSubscr(PyObject *container, PyObject *sub,
+extern void _Py_Specialize_StoreSubscr(_PyStackRef container, _PyStackRef sub,
_Py_CODEUNIT *instr);
-extern void _Py_Specialize_Call(PyObject *callable, _Py_CODEUNIT *instr,
+extern void _Py_Specialize_Call(_PyStackRef callable, _Py_CODEUNIT *instr,
int nargs);
-extern void _Py_Specialize_BinaryOp(PyObject *lhs, PyObject *rhs, _Py_CODEUNIT *instr,
- int oparg, PyObject **locals);
-extern void _Py_Specialize_CompareOp(PyObject *lhs, PyObject *rhs,
+extern void _Py_Specialize_BinaryOp(_PyStackRef lhs, _PyStackRef rhs, _Py_CODEUNIT *instr,
+ int oparg, _PyStackRef *locals);
+extern void _Py_Specialize_CompareOp(_PyStackRef lhs, _PyStackRef rhs,
_Py_CODEUNIT *instr, int oparg);
-extern void _Py_Specialize_UnpackSequence(PyObject *seq, _Py_CODEUNIT *instr,
+extern void _Py_Specialize_UnpackSequence(_PyStackRef seq, _Py_CODEUNIT *instr,
int oparg);
-extern void _Py_Specialize_ForIter(PyObject *iter, _Py_CODEUNIT *instr, int oparg);
-extern void _Py_Specialize_Send(PyObject *receiver, _Py_CODEUNIT *instr);
-extern void _Py_Specialize_ToBool(PyObject *value, _Py_CODEUNIT *instr);
-extern void _Py_Specialize_ContainsOp(PyObject *value, _Py_CODEUNIT *instr);
+extern void _Py_Specialize_ForIter(_PyStackRef iter, _Py_CODEUNIT *instr, int oparg);
+extern void _Py_Specialize_Send(_PyStackRef receiver, _Py_CODEUNIT *instr);
+extern void _Py_Specialize_ToBool(_PyStackRef value, _Py_CODEUNIT *instr);
+extern void _Py_Specialize_ContainsOp(_PyStackRef value, _Py_CODEUNIT *instr);
#ifdef Py_STATS
#include <stdbool.h>
#include <stddef.h> // offsetof()
#include "pycore_code.h" // STATS
+#include "pycore_stackref.h" // _PyStackRef
/* See Objects/frame_layout.md for an explanation of the frame stack
* including explanation of the PyFrameObject and _PyInterpreterFrame
uint16_t return_offset; /* Only relevant during a function call */
char owner;
/* Locals and stack */
- PyObject *localsplus[1];
+ _PyStackRef localsplus[1];
} _PyInterpreterFrame;
#define _PyInterpreterFrame_LASTI(IF) \
return (PyCodeObject *)f->f_executable;
}
-static inline PyObject **_PyFrame_Stackbase(_PyInterpreterFrame *f) {
- return f->localsplus + _PyFrame_GetCode(f)->co_nlocalsplus;
+static inline _PyStackRef *_PyFrame_Stackbase(_PyInterpreterFrame *f) {
+ return (f->localsplus + _PyFrame_GetCode(f)->co_nlocalsplus);
}
-static inline PyObject *_PyFrame_StackPeek(_PyInterpreterFrame *f) {
+static inline _PyStackRef _PyFrame_StackPeek(_PyInterpreterFrame *f) {
assert(f->stacktop > _PyFrame_GetCode(f)->co_nlocalsplus);
- assert(f->localsplus[f->stacktop-1] != NULL);
+ assert(!PyStackRef_IsNull(f->localsplus[f->stacktop-1]));
return f->localsplus[f->stacktop-1];
}
-static inline PyObject *_PyFrame_StackPop(_PyInterpreterFrame *f) {
+static inline _PyStackRef _PyFrame_StackPop(_PyInterpreterFrame *f) {
assert(f->stacktop > _PyFrame_GetCode(f)->co_nlocalsplus);
f->stacktop--;
return f->localsplus[f->stacktop];
}
-static inline void _PyFrame_StackPush(_PyInterpreterFrame *f, PyObject *value) {
+static inline void _PyFrame_StackPush(_PyInterpreterFrame *f, _PyStackRef value) {
f->localsplus[f->stacktop] = value;
f->stacktop++;
}
frame->owner = FRAME_OWNED_BY_THREAD;
for (int i = null_locals_from; i < code->co_nlocalsplus; i++) {
- frame->localsplus[i] = NULL;
+ frame->localsplus[i] = PyStackRef_NULL;
}
}
/* Gets the pointer to the locals array
* that precedes this frame.
*/
-static inline PyObject**
+static inline _PyStackRef*
_PyFrame_GetLocalsArray(_PyInterpreterFrame *frame)
{
return frame->localsplus;
Having stacktop <= 0 ensures that invalid
values are not visible to the cycle GC.
We choose -1 rather than 0 to assist debugging. */
-static inline PyObject**
+static inline _PyStackRef*
_PyFrame_GetStackPointer(_PyInterpreterFrame *frame)
{
- PyObject **sp = frame->localsplus + frame->stacktop;
+ _PyStackRef *sp = frame->localsplus + frame->stacktop;
frame->stacktop = -1;
return sp;
}
static inline void
-_PyFrame_SetStackPointer(_PyInterpreterFrame *frame, PyObject **stack_pointer)
+_PyFrame_SetStackPointer(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer)
{
frame->stacktop = (int)(stack_pointer - frame->localsplus);
}
PyAPI_FUNC(_PyInterpreterFrame *)
_PyEvalFramePushAndInit(PyThreadState *tstate, PyFunctionObject *func,
- PyObject *locals, PyObject* const* args,
+ PyObject *locals, _PyStackRef const* args,
size_t argcount, PyObject *kwnames);
#ifdef __cplusplus
#ifdef _Py_JIT
-typedef _Py_CODEUNIT *(*jit_func)(_PyInterpreterFrame *frame, PyObject **stack_pointer, PyThreadState *tstate);
+typedef _Py_CODEUNIT *(*jit_func)(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate);
int _PyJIT_Compile(_PyExecutorObject *executor, const _PyUOpInstruction *trace, size_t length);
void _PyJIT_Free(_PyExecutorObject *executor);
op = NULL; \
} while (0)
-// Mark an object as supporting deferred reference counting. This is a no-op
-// in the default (with GIL) build. Objects that use deferred reference
-// counting should be tracked by the GC so that they are eventually collected.
-extern void _PyObject_SetDeferredRefcount(PyObject *op);
-
-static inline int
-_PyObject_HasDeferredRefcount(PyObject *op)
-{
-#ifdef Py_GIL_DISABLED
- return _PyObject_HAS_GC_BITS(op, _PyGC_BITS_DEFERRED);
-#else
- return 0;
-#endif
-}
-
#if !defined(Py_GIL_DISABLED)
static inline void
_Py_DECREF_SPECIALIZED(PyObject *op, const destructor destruct)
--- /dev/null
+#ifndef Py_INTERNAL_OBJECT_DEFERRED_H
+#define Py_INTERNAL_OBJECT_DEFERRED_H
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#include "pycore_gc.h"
+
+#ifndef Py_BUILD_CORE
+# error "this header requires Py_BUILD_CORE define"
+#endif
+
+// Mark an object as supporting deferred reference counting. This is a no-op
+// in the default (with GIL) build. Objects that use deferred reference
+// counting should be tracked by the GC so that they are eventually collected.
+extern void _PyObject_SetDeferredRefcount(PyObject *op);
+
+static inline int
+_PyObject_HasDeferredRefcount(PyObject *op)
+{
+#ifdef Py_GIL_DISABLED
+ return _PyObject_HAS_GC_BITS(op, _PyGC_BITS_DEFERRED);
+#else
+ return 0;
+#endif
+}
+
+#ifdef __cplusplus
+}
+#endif
+#endif // !Py_INTERNAL_OBJECT_DEFERRED_H
PyAPI_FUNC(PyObject *) _Py_uop_symbols_test(PyObject *self, PyObject *ignored);
-PyAPI_FUNC(int) _PyOptimizer_Optimize(struct _PyInterpreterFrame *frame, _Py_CODEUNIT *start, PyObject **stack_pointer, _PyExecutorObject **exec_ptr);
+PyAPI_FUNC(int) _PyOptimizer_Optimize(struct _PyInterpreterFrame *frame, _Py_CODEUNIT *start, _PyStackRef *stack_pointer, _PyExecutorObject **exec_ptr);
#ifdef __cplusplus
}
# error "this header requires Py_BUILD_CORE define"
#endif
+#include "pycore_object_deferred.h"
+
#include <stddef.h>
+/*
+ This file introduces a new API for handling references on the stack, called
+ _PyStackRef. This API is inspired by HPy.
+
+ There are 3 main operations, that convert _PyStackRef to PyObject* and
+ vice versa:
+
+ 1. Borrow (discouraged)
+ 2. Steal
+ 3. New
+
+ Borrow means that the reference is converted without any change in ownership.
+ This is discouraged because it makes verification much harder. It also makes
+ unboxed integers harder in the future.
+
+ Steal means that ownership is transferred to something else. The total
+ number of references to the object stays the same.
+
+ New creates a new reference from the old reference. The old reference
+ is still valid.
+
+ With these 3 API, a strict stack discipline must be maintained. All
+ _PyStackRef must be operated on by the new reference operations:
+
+ 1. DUP
+ 2. CLOSE
+
+ DUP is roughly equivalent to Py_NewRef. It creates a new reference from an old
+ reference. The old reference remains unchanged.
+
+ CLOSE is roughly equivalent to Py_DECREF. It destroys a reference.
+
+ Note that it is unsafe to borrow a _PyStackRef and then do normal
+ CPython refcounting operations on it!
+*/
+
typedef union {
uintptr_t bits;
} _PyStackRef;
-static const _PyStackRef Py_STACKREF_NULL = { .bits = 0 };
#define Py_TAG_DEFERRED (1)
+#define Py_TAG_PTR (0)
+#define Py_TAG_BITS (1)
+
+#ifdef Py_GIL_DISABLED
+ static const _PyStackRef PyStackRef_NULL = { .bits = 0 | Py_TAG_DEFERRED};
+#else
+ static const _PyStackRef PyStackRef_NULL = { .bits = 0 };
+#endif
+
+#define PyStackRef_IsNull(stackref) ((stackref).bits == PyStackRef_NULL.bits)
+
+
+#ifdef Py_GIL_DISABLED
+# define PyStackRef_True ((_PyStackRef){.bits = ((uintptr_t)&_Py_TrueStruct) | Py_TAG_DEFERRED })
+#else
+# define PyStackRef_True ((_PyStackRef){.bits = ((uintptr_t)&_Py_TrueStruct) })
+#endif
+
+#ifdef Py_GIL_DISABLED
+# define PyStackRef_False ((_PyStackRef){.bits = ((uintptr_t)&_Py_FalseStruct) | Py_TAG_DEFERRED })
+#else
+# define PyStackRef_False ((_PyStackRef){.bits = ((uintptr_t)&_Py_FalseStruct) })
+#endif
+
+#ifdef Py_GIL_DISABLED
+# define PyStackRef_None ((_PyStackRef){.bits = ((uintptr_t)&_Py_NoneStruct) | Py_TAG_DEFERRED })
+#else
+# define PyStackRef_None ((_PyStackRef){.bits = ((uintptr_t)&_Py_NoneStruct) })
+#endif
+
+
+static inline int
+PyStackRef_Is(_PyStackRef a, _PyStackRef b) {
+ return a.bits == b.bits;
+}
+
+static inline int
+PyStackRef_IsDeferred(_PyStackRef ref)
+{
+ return ((ref.bits & Py_TAG_BITS) == Py_TAG_DEFERRED);
+}
+
// Gets a PyObject * from a _PyStackRef
-#if defined(Py_GIL_DISABLED)
static inline PyObject *
-PyStackRef_Get(_PyStackRef tagged)
+PyStackRef_AsPyObjectBorrow(_PyStackRef stackref)
{
- PyObject *cleared = ((PyObject *)((tagged).bits & (~Py_TAG_DEFERRED)));
+#ifdef Py_GIL_DISABLED
+ PyObject *cleared = ((PyObject *)((stackref).bits & (~Py_TAG_BITS)));
return cleared;
+#else
+ return ((PyObject *)(stackref).bits);
+#endif
}
+
+// Converts a PyStackRef back to a PyObject *, stealing the
+// PyStackRef.
+static inline PyObject *
+PyStackRef_AsPyObjectSteal(_PyStackRef stackref)
+{
+#ifdef Py_GIL_DISABLED
+ if (!PyStackRef_IsNull(stackref) && PyStackRef_IsDeferred(stackref)) {
+ return Py_NewRef(PyStackRef_AsPyObjectBorrow(stackref));
+ }
+ return PyStackRef_AsPyObjectBorrow(stackref);
#else
-# define PyStackRef_Get(tagged) ((PyObject *)((tagged).bits))
+ return PyStackRef_AsPyObjectBorrow(stackref);
#endif
+}
+
+// Converts a PyStackRef back to a PyObject *, converting the
+// stackref to a new reference.
+static inline PyObject *
+PyStackRef_AsPyObjectNew(_PyStackRef stackref)
+{
+ return Py_NewRef(PyStackRef_AsPyObjectBorrow(stackref));
+}
+
+static inline PyTypeObject *
+PyStackRef_TYPE(_PyStackRef stackref)
+{
+ return Py_TYPE(PyStackRef_AsPyObjectBorrow(stackref));
+}
-// Converts a PyObject * to a PyStackRef, stealing the reference.
-#if defined(Py_GIL_DISABLED)
+// Converts a PyObject * to a PyStackRef, stealing the reference
static inline _PyStackRef
-_PyStackRef_StealRef(PyObject *obj)
+_PyStackRef_FromPyObjectSteal(PyObject *obj)
{
+#ifdef Py_GIL_DISABLED
// Make sure we don't take an already tagged value.
- assert(((uintptr_t)obj & Py_TAG_DEFERRED) == 0);
- return ((_PyStackRef){.bits = ((uintptr_t)(obj))});
-}
-# define PyStackRef_StealRef(obj) _PyStackRef_StealRef(_PyObject_CAST(obj))
+ assert(((uintptr_t)obj & Py_TAG_BITS) == 0);
+ int tag = (obj == NULL || _Py_IsImmortal(obj)) ? (Py_TAG_DEFERRED) : Py_TAG_PTR;
+ return ((_PyStackRef){.bits = ((uintptr_t)(obj)) | tag});
#else
-# define PyStackRef_StealRef(obj) ((_PyStackRef){.bits = ((uintptr_t)(obj))})
+ return ((_PyStackRef){.bits = ((uintptr_t)(obj))});
#endif
+}
+
+#define PyStackRef_FromPyObjectSteal(obj) _PyStackRef_FromPyObjectSteal(_PyObject_CAST(obj))
+
// Converts a PyObject * to a PyStackRef, with a new reference
-#if defined(Py_GIL_DISABLED)
static inline _PyStackRef
-_PyStackRef_NewRefDeferred(PyObject *obj)
+PyStackRef_FromPyObjectNew(PyObject *obj)
{
+#ifdef Py_GIL_DISABLED
// Make sure we don't take an already tagged value.
- assert(((uintptr_t)obj & Py_TAG_DEFERRED) == 0);
+ assert(((uintptr_t)obj & Py_TAG_BITS) == 0);
assert(obj != NULL);
- if (_PyObject_HasDeferredRefcount(obj)) {
+ // TODO (gh-117139): Add deferred objects later.
+ if (_Py_IsImmortal(obj)) {
return (_PyStackRef){ .bits = (uintptr_t)obj | Py_TAG_DEFERRED };
}
else {
- return (_PyStackRef){ .bits = (uintptr_t)Py_NewRef(obj) };
+ return (_PyStackRef){ .bits = (uintptr_t)(Py_NewRef(obj)) | Py_TAG_PTR };
}
-}
-# define PyStackRef_NewRefDeferred(obj) _PyStackRef_NewRefDeferred(_PyObject_CAST(obj))
#else
-# define PyStackRef_NewRefDeferred(obj) PyStackRef_NewRef(((_PyStackRef){.bits = ((uintptr_t)(obj))}))
+ return ((_PyStackRef){ .bits = (uintptr_t)(Py_NewRef(obj)) });
#endif
+}
-#if defined(Py_GIL_DISABLED)
+#define PyStackRef_FromPyObjectNew(obj) PyStackRef_FromPyObjectNew(_PyObject_CAST(obj))
+
+// Same as PyStackRef_FromPyObjectNew but only for immortal objects.
static inline _PyStackRef
-_PyStackRef_XNewRefDeferred(PyObject *obj)
+PyStackRef_FromPyObjectImmortal(PyObject *obj)
{
+#ifdef Py_GIL_DISABLED
// Make sure we don't take an already tagged value.
- assert(((uintptr_t)obj & Py_TAG_DEFERRED) == 0);
- if (obj == NULL) {
- return Py_STACKREF_NULL;
- }
- return _PyStackRef_NewRefDeferred(obj);
-}
-# define PyStackRef_XNewRefDeferred(obj) _PyStackRef_XNewRefDeferred(_PyObject_CAST(obj))
-#else
-# define PyStackRef_XNewRefDeferred(obj) PyStackRef_XNewRef(((_PyStackRef){.bits = ((uintptr_t)(obj))}))
-#endif
-
-// Converts a PyStackRef back to a PyObject *.
-#if defined(Py_GIL_DISABLED)
-static inline PyObject *
-PyStackRef_StealObject(_PyStackRef tagged)
-{
- if ((tagged.bits & Py_TAG_DEFERRED) == Py_TAG_DEFERRED) {
- assert(_PyObject_HasDeferredRefcount(PyStackRef_Get(tagged)));
- return Py_NewRef(PyStackRef_Get(tagged));
- }
- return PyStackRef_Get(tagged);
-}
+ assert(((uintptr_t)obj & Py_TAG_BITS) == 0);
+ assert(obj != NULL);
+ assert(_Py_IsImmortal(obj));
+ return (_PyStackRef){ .bits = (uintptr_t)obj | Py_TAG_DEFERRED };
#else
-# define PyStackRef_StealObject(tagged) PyStackRef_Get(tagged)
+ assert(_Py_IsImmortal(obj));
+ return ((_PyStackRef){ .bits = (uintptr_t)(obj) });
#endif
-
-static inline void
-_Py_untag_stack_borrowed(PyObject **dst, const _PyStackRef *src, size_t length)
-{
- for (size_t i = 0; i < length; i++) {
- dst[i] = PyStackRef_Get(src[i]);
- }
-}
-
-static inline void
-_Py_untag_stack_steal(PyObject **dst, const _PyStackRef *src, size_t length)
-{
- for (size_t i = 0; i < length; i++) {
- dst[i] = PyStackRef_StealObject(src[i]);
- }
}
+#define PyStackRef_FromPyObjectImmortal(obj) PyStackRef_FromPyObjectImmortal(_PyObject_CAST(obj))
-#define PyStackRef_XSETREF(dst, src) \
- do { \
- _PyStackRef *_tmp_dst_ptr = &(dst); \
- _PyStackRef _tmp_old_dst = (*_tmp_dst_ptr); \
- *_tmp_dst_ptr = (src); \
- PyStackRef_XDECREF(_tmp_old_dst); \
- } while (0)
-
-#define PyStackRef_SETREF(dst, src) \
- do { \
- _PyStackRef *_tmp_dst_ptr = &(dst); \
- _PyStackRef _tmp_old_dst = (*_tmp_dst_ptr); \
- *_tmp_dst_ptr = (src); \
- PyStackRef_DECREF(_tmp_old_dst); \
- } while (0)
#define PyStackRef_CLEAR(op) \
do { \
_PyStackRef *_tmp_op_ptr = &(op); \
_PyStackRef _tmp_old_op = (*_tmp_op_ptr); \
- if (_tmp_old_op.bits != Py_STACKREF_NULL.bits) { \
- *_tmp_op_ptr = Py_STACKREF_NULL; \
- PyStackRef_DECREF(_tmp_old_op); \
+ if (!PyStackRef_IsNull(_tmp_old_op)) { \
+ *_tmp_op_ptr = PyStackRef_NULL; \
+ PyStackRef_CLOSE(_tmp_old_op); \
} \
} while (0)
-#if defined(Py_GIL_DISABLED)
static inline void
-PyStackRef_DECREF(_PyStackRef tagged)
+PyStackRef_CLOSE(_PyStackRef stackref)
{
- if ((tagged.bits & Py_TAG_DEFERRED) == Py_TAG_DEFERRED) {
+#ifdef Py_GIL_DISABLED
+ if (PyStackRef_IsDeferred(stackref)) {
+ // No assert for being immortal or deferred here.
+ // The GC unsets deferred objects right before clearing.
return;
}
- Py_DECREF(PyStackRef_Get(tagged));
-}
+ Py_DECREF(PyStackRef_AsPyObjectBorrow(stackref));
#else
-# define PyStackRef_DECREF(op) Py_DECREF(PyStackRef_Get(op))
+ Py_DECREF(PyStackRef_AsPyObjectBorrow(stackref));
#endif
+}
-#if defined(Py_GIL_DISABLED)
-static inline void
-PyStackRef_INCREF(_PyStackRef tagged)
+#define PyStackRef_XCLOSE(stackref) \
+ do { \
+ _PyStackRef _tmp = (stackref); \
+ if (!PyStackRef_IsNull(_tmp)) { \
+ PyStackRef_CLOSE(_tmp); \
+ } \
+ } while (0);
+
+
+static inline _PyStackRef
+PyStackRef_DUP(_PyStackRef stackref)
{
- if ((tagged.bits & Py_TAG_DEFERRED) == Py_TAG_DEFERRED) {
- assert(_PyObject_HasDeferredRefcount(PyStackRef_Get(tagged)));
- return;
+#ifdef Py_GIL_DISABLED
+ if (PyStackRef_IsDeferred(stackref)) {
+ assert(PyStackRef_IsNull(stackref) ||
+ _Py_IsImmortal(PyStackRef_AsPyObjectBorrow(stackref)));
+ return stackref;
}
- Py_INCREF(PyStackRef_Get(tagged));
-}
+ Py_INCREF(PyStackRef_AsPyObjectBorrow(stackref));
+ return stackref;
#else
-# define PyStackRef_INCREF(op) Py_INCREF(PyStackRef_Get(op))
+ Py_INCREF(PyStackRef_AsPyObjectBorrow(stackref));
+ return stackref;
#endif
-
-static inline void
-PyStackRef_XDECREF(_PyStackRef op)
-{
- if (op.bits != Py_STACKREF_NULL.bits) {
- PyStackRef_DECREF(op);
- }
}
static inline _PyStackRef
-PyStackRef_NewRef(_PyStackRef obj)
+PyStackRef_XDUP(_PyStackRef stackref)
{
- PyStackRef_INCREF(obj);
- return obj;
+ if (!PyStackRef_IsNull(stackref)) {
+ return PyStackRef_DUP(stackref);
+ }
+ return stackref;
}
-static inline _PyStackRef
-PyStackRef_XNewRef(_PyStackRef obj)
+
+static inline void
+_PyObjectStack_FromStackRefStack(PyObject **dst, const _PyStackRef *src, size_t length)
{
- if (obj.bits == Py_STACKREF_NULL.bits) {
- return obj;
+ for (size_t i = 0; i < length; i++) {
+ dst[i] = PyStackRef_AsPyObjectBorrow(src[i]);
}
- return PyStackRef_NewRef(obj);
}
+
#ifdef __cplusplus
}
#endif
frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(OP);
- PyObject *value;
+ _PyStackRef value;
value = stack_pointer[-1];
spam();
stack_pointer += -1;
frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(OP);
- PyObject *res;
+ _PyStackRef res;
spam();
stack_pointer[0] = res;
stack_pointer += 1;
frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(OP);
- PyObject *value;
- PyObject *res;
+ _PyStackRef value;
+ _PyStackRef res;
value = stack_pointer[-1];
spam();
stack_pointer[-1] = res;
frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(OP);
- PyObject *right;
- PyObject *left;
- PyObject *res;
+ _PyStackRef right;
+ _PyStackRef left;
+ _PyStackRef res;
right = stack_pointer[-1];
left = stack_pointer[-2];
spam();
frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(OP);
- PyObject *right;
- PyObject *left;
- PyObject *result;
+ _PyStackRef right;
+ _PyStackRef left;
+ _PyStackRef result;
right = stack_pointer[-1];
left = stack_pointer[-2];
spam();
next_instr += 1;
INSTRUCTION_STATS(OP1);
PREDICTED(OP1);
- PyObject *arg;
- PyObject *rest;
+ _PyStackRef arg;
+ _PyStackRef rest;
arg = stack_pointer[-1];
stack_pointer[-1] = rest;
DISPATCH();
next_instr += 1;
INSTRUCTION_STATS(OP3);
static_assert(INLINE_CACHE_ENTRIES_OP1 == 0, "incorrect cache size");
- PyObject *arg;
- PyObject *res;
+ _PyStackRef arg;
+ _PyStackRef res;
arg = stack_pointer[-1];
DEOPT_IF(xxx, OP1);
stack_pointer[-1] = res;
frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(OP);
- PyObject *right;
- PyObject *left;
- PyObject *res;
+ _PyStackRef right;
+ _PyStackRef left;
+ _PyStackRef res;
right = stack_pointer[-1];
left = stack_pointer[-2];
if (cond) goto pop_2_label;
(void)this_instr;
next_instr += 4;
INSTRUCTION_STATS(OP);
- PyObject *value;
+ _PyStackRef value;
value = stack_pointer[-1];
uint16_t counter = read_u16(&this_instr[1].cache);
(void)counter;
PREDICTED(OP);
_Py_CODEUNIT *this_instr = next_instr - 6;
(void)this_instr;
- PyObject *right;
- PyObject *left;
- PyObject *arg2;
- PyObject *res;
+ _PyStackRef right;
+ _PyStackRef left;
+ _PyStackRef arg2;
+ _PyStackRef res;
// _OP1
right = stack_pointer[-1];
left = stack_pointer[-2];
(void)this_instr;
next_instr += 2;
INSTRUCTION_STATS(OP1);
- PyObject *right;
- PyObject *left;
+ _PyStackRef right;
+ _PyStackRef left;
right = stack_pointer[-1];
left = stack_pointer[-2];
uint16_t counter = read_u16(&this_instr[1].cache);
next_instr += 6;
INSTRUCTION_STATS(OP3);
static_assert(INLINE_CACHE_ENTRIES_OP == 5, "incorrect cache size");
- PyObject *right;
- PyObject *left;
- PyObject *arg2;
- PyObject *res;
+ _PyStackRef right;
+ _PyStackRef left;
+ _PyStackRef arg2;
+ _PyStackRef res;
/* Skip 5 cache entries */
right = stack_pointer[-1];
left = stack_pointer[-2];
frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(OP);
- PyObject *above;
- PyObject **values;
- PyObject *below;
+ _PyStackRef above;
+ _PyStackRef *values;
+ _PyStackRef below;
above = stack_pointer[-1];
values = &stack_pointer[-1 - oparg*2];
below = stack_pointer[-2 - oparg*2];
frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(OP);
- PyObject *below;
- PyObject **values;
- PyObject *above;
+ _PyStackRef below;
+ _PyStackRef *values;
+ _PyStackRef above;
values = &stack_pointer[-1];
spam(values, oparg);
stack_pointer[-2] = below;
frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(OP);
- PyObject **values;
- PyObject *above;
+ _PyStackRef *values;
+ _PyStackRef above;
values = &stack_pointer[-oparg];
spam(values, oparg);
stack_pointer[0] = above;
frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(OP);
- PyObject **values;
- PyObject *extra;
+ _PyStackRef *values;
+ _PyStackRef extra;
values = &stack_pointer[-oparg];
extra = stack_pointer[-1 - oparg];
if (oparg == 0) { stack_pointer += -1 - oparg; goto somewhere; }
frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(OP);
- PyObject *cc;
- PyObject *input = NULL;
- PyObject *aa;
- PyObject *xx;
- PyObject *output = NULL;
- PyObject *zz;
+ _PyStackRef cc;
+ _PyStackRef input = PyStackRef_NULL;
+ _PyStackRef aa;
+ _PyStackRef xx;
+ _PyStackRef output = PyStackRef_NULL;
+ _PyStackRef zz;
cc = stack_pointer[-1];
if ((oparg & 1) == 1) { input = stack_pointer[-1 - (((oparg & 1) == 1) ? 1 : 0)]; }
aa = stack_pointer[-2 - (((oparg & 1) == 1) ? 1 : 0)];
frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(M);
- PyObject *right;
- PyObject *middle;
- PyObject *left;
- PyObject *deep;
- PyObject *extra = NULL;
- PyObject *res;
+ _PyStackRef right;
+ _PyStackRef middle;
+ _PyStackRef left;
+ _PyStackRef deep;
+ _PyStackRef extra = PyStackRef_NULL;
+ _PyStackRef res;
// A
right = stack_pointer[-1];
middle = stack_pointer[-2];
frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(M);
- PyObject *val1;
- PyObject *val2;
+ _PyStackRef val1;
+ _PyStackRef val2;
// A
{
val1 = spam();
$(srcdir)/Include/internal/pycore_namespace.h \
$(srcdir)/Include/internal/pycore_object.h \
$(srcdir)/Include/internal/pycore_object_alloc.h \
+ $(srcdir)/Include/internal/pycore_object_deferred.h \
$(srcdir)/Include/internal/pycore_object_stack.h \
$(srcdir)/Include/internal/pycore_object_state.h \
$(srcdir)/Include/internal/pycore_obmalloc.h \
--- /dev/null
+Convert the Python evaluation stack to use internal stack references. The
+purpose is to support tagged pointers. In :pep:`703`, this will
+allow for its form of deferred reference counting. For both
+the default and free-threaded builds, this sets up the infrastructure
+for unboxed integers in the future.
static PyObject *
framelocalsproxy_getval(_PyInterpreterFrame *frame, PyCodeObject *co, int i)
{
- PyObject **fast = _PyFrame_GetLocalsArray(frame);
+ _PyStackRef *fast = _PyFrame_GetLocalsArray(frame);
_PyLocals_Kind kind = _PyLocals_GetKind(co->co_localspluskinds, i);
- PyObject *value = fast[i];
+ PyObject *value = PyStackRef_AsPyObjectBorrow(fast[i]);
PyObject *cell = NULL;
if (value == NULL) {
framelocalsproxy_setitem(PyObject *self, PyObject *key, PyObject *value)
{
/* Merge locals into fast locals */
- PyFrameObject* frame = ((PyFrameLocalsProxyObject*)self)->frame;
- PyObject** fast = _PyFrame_GetLocalsArray(frame->f_frame);
- PyCodeObject* co = _PyFrame_GetCode(frame->f_frame);
+ PyFrameObject *frame = ((PyFrameLocalsProxyObject*)self)->frame;
+ _PyStackRef *fast = _PyFrame_GetLocalsArray(frame->f_frame);
+ PyCodeObject *co = _PyFrame_GetCode(frame->f_frame);
if (value == NULL) {
PyErr_SetString(PyExc_TypeError, "cannot remove variables from FrameLocalsProxy");
_Py_Executors_InvalidateDependency(PyInterpreterState_Get(), co, 1);
_PyLocals_Kind kind = _PyLocals_GetKind(co->co_localspluskinds, i);
- PyObject *oldvalue = fast[i];
+ _PyStackRef oldvalue = fast[i];
PyObject *cell = NULL;
if (kind == CO_FAST_FREE) {
// The cell was set when the frame was created from
// the function's closure.
- assert(oldvalue != NULL && PyCell_Check(oldvalue));
- cell = oldvalue;
- } else if (kind & CO_FAST_CELL && oldvalue != NULL) {
- if (PyCell_Check(oldvalue)) {
- cell = oldvalue;
+ assert(oldvalue.bits != 0 && PyCell_Check(PyStackRef_AsPyObjectBorrow(oldvalue)));
+ cell = PyStackRef_AsPyObjectBorrow(oldvalue);
+ } else if (kind & CO_FAST_CELL && oldvalue.bits != 0) {
+ PyObject *as_obj = PyStackRef_AsPyObjectBorrow(oldvalue);
+ if (PyCell_Check(as_obj)) {
+ cell = as_obj;
}
}
if (cell != NULL) {
- oldvalue = PyCell_GET(cell);
- if (value != oldvalue) {
+ PyObject *oldvalue_o = PyCell_GET(cell);
+ if (value != oldvalue_o) {
PyCell_SET(cell, Py_XNewRef(value));
- Py_XDECREF(oldvalue);
+ Py_XDECREF(oldvalue_o);
}
- } else if (value != oldvalue) {
- Py_XSETREF(fast[i], Py_NewRef(value));
+ } else if (value != PyStackRef_AsPyObjectBorrow(oldvalue)) {
+ PyStackRef_XCLOSE(fast[i]);
+ fast[i] = PyStackRef_FromPyObjectNew(value);
}
return 0;
}
for (int i = 0; i < code->co_nlocalsplus; i++) {
// Counting every unbound local is overly-cautious, but a full flow
// analysis (like we do in the compiler) is probably too expensive:
- unbound += f->f_frame->localsplus[i] == NULL;
+ unbound += PyStackRef_IsNull(f->f_frame->localsplus[i]);
}
if (unbound) {
const char *e = "assigning None to %d unbound local%s";
// Do this in a second pass to avoid writing a bunch of Nones when
// warnings are being treated as errors and the previous bit raises:
for (int i = 0; i < code->co_nlocalsplus; i++) {
- if (f->f_frame->localsplus[i] == NULL) {
- f->f_frame->localsplus[i] = Py_NewRef(Py_None);
+ if (PyStackRef_IsNull(f->f_frame->localsplus[i])) {
+ f->f_frame->localsplus[i] = PyStackRef_None;
unbound--;
}
}
while (start_stack > best_stack) {
if (top_of_stack(start_stack) == Except) {
/* Pop exception stack as well as the evaluation stack */
- PyObject *exc = _PyFrame_StackPop(f->f_frame);
+ PyObject *exc = PyStackRef_AsPyObjectBorrow(_PyFrame_StackPop(f->f_frame));
assert(PyExceptionInstance_Check(exc) || exc == Py_None);
PyThreadState *tstate = _PyThreadState_GET();
Py_XSETREF(tstate->exc_info->exc_value, exc == Py_None ? NULL : exc);
}
else {
- PyObject *v = _PyFrame_StackPop(f->f_frame);
- Py_XDECREF(v);
+ PyStackRef_XCLOSE(_PyFrame_StackPop(f->f_frame));
}
start_stack = pop_value(start_stack);
}
frame->f_executable = NULL;
Py_CLEAR(frame->f_funcobj);
Py_CLEAR(frame->f_locals);
- PyObject **locals = _PyFrame_GetLocalsArray(frame);
+ _PyStackRef *locals = _PyFrame_GetLocalsArray(frame);
for (int i = 0; i < frame->stacktop; i++) {
- Py_CLEAR(locals[i]);
+ PyStackRef_CLEAR(locals[i]);
}
}
Py_CLEAR(f->f_back);
Py_CLEAR(f->f_extra_locals);
/* locals and stack */
- PyObject **locals = _PyFrame_GetLocalsArray(f->f_frame);
+ _PyStackRef *locals = _PyFrame_GetLocalsArray(f->f_frame);
assert(f->f_frame->stacktop >= 0);
for (int i = 0; i < f->f_frame->stacktop; i++) {
- Py_CLEAR(locals[i]);
+ PyStackRef_CLEAR(locals[i]);
}
f->f_frame->stacktop = 0;
Py_CLEAR(f->f_frame->f_locals);
int offset = PyUnstable_Code_GetFirstFree(co);
for (int i = 0; i < co->co_nfreevars; ++i) {
PyObject *o = PyTuple_GET_ITEM(closure, i);
- frame->localsplus[offset + i] = Py_NewRef(o);
+ frame->localsplus[offset + i] = PyStackRef_FromPyObjectNew(o);
}
// COPY_FREE_VARS doesn't have inline CACHEs, either:
frame->instr_ptr = _PyCode_CODE(_PyFrame_GetCode(frame));
return 0;
}
- PyObject *value = frame->localsplus[i];
+ PyObject *value = PyStackRef_AsPyObjectBorrow(frame->localsplus[i]);
if (frame->stacktop) {
if (kind & CO_FAST_FREE) {
// The cell was set by COPY_FREE_VARS.
/* Push arg onto the frame's value stack */
PyObject *arg_obj = arg ? arg : Py_None;
- _PyFrame_StackPush(frame, Py_NewRef(arg_obj));
+ _PyFrame_StackPush(frame, PyStackRef_FromPyObjectNew(arg_obj));
_PyErr_StackItem *prev_exc_info = tstate->exc_info;
gen->gi_exc_state.previous_item = prev_exc_info;
_PyInterpreterFrame *frame = &gen->gi_iframe;
assert(is_resume(frame->instr_ptr));
assert((frame->instr_ptr->op.arg & RESUME_OPARG_LOCATION_MASK) >= RESUME_AFTER_YIELD_FROM);
- return Py_NewRef(_PyFrame_StackPeek(frame));
+ return PyStackRef_AsPyObjectNew(_PyFrame_StackPeek(frame));
}
return NULL;
}
}
assert(_PyFrame_GetCode(cframe)->co_nlocalsplus > 0);
- PyObject *firstarg = _PyFrame_GetLocalsArray(cframe)[0];
+ PyObject *firstarg = PyStackRef_AsPyObjectBorrow(_PyFrame_GetLocalsArray(cframe)[0]);
// The first argument might be a cell.
if (firstarg != NULL && (_PyLocals_GetKind(co->co_localspluskinds, 0) & CO_FAST_CELL)) {
// "firstarg" is a cell here unless (very unlikely) super()
PyObject *name = PyTuple_GET_ITEM(co->co_localsplusnames, i);
assert(PyUnicode_Check(name));
if (_PyUnicode_Equal(name, &_Py_ID(__class__))) {
- PyObject *cell = _PyFrame_GetLocalsArray(cframe)[i];
+ PyObject *cell = PyStackRef_AsPyObjectBorrow(_PyFrame_GetLocalsArray(cframe)[i]);
if (cell == NULL || !PyCell_Check(cell)) {
PyErr_SetString(PyExc_RuntimeError,
"super(): bad __class__ cell");
<ClInclude Include="..\Include\internal\pycore_namespace.h" />
<ClInclude Include="..\Include\internal\pycore_object.h" />
<ClInclude Include="..\Include\internal\pycore_object_alloc.h" />
+ <ClInclude Include="..\Include\internal\pycore_object_deferred.h" />
<ClInclude Include="..\Include\internal\pycore_object_state.h" />
<ClInclude Include="..\Include\internal\pycore_obmalloc.h" />
<ClInclude Include="..\Include\internal\pycore_obmalloc_init.h" />
<ClInclude Include="..\Include\internal\pycore_object_alloc.h">
<Filter>Include\internal</Filter>
</ClInclude>
+ <ClInclude Include="..\Include\internal\pycore_object_deferred.h">
+ <Filter>Include\internal</Filter>
+ </ClInclude>
<ClInclude Include="..\Include\internal\pycore_object_state.h">
<Filter>Include\internal</Filter>
</ClInclude>
};
inst(LOAD_FAST_CHECK, (-- value)) {
- value = GETLOCAL(oparg);
- if (value == NULL) {
+ _PyStackRef value_s = GETLOCAL(oparg);
+ if (PyStackRef_IsNull(value_s)) {
_PyEval_FormatExcCheckArg(tstate, PyExc_UnboundLocalError,
UNBOUNDLOCAL_ERROR_MSG,
PyTuple_GetItem(_PyFrame_GetCode(frame)->co_localsplusnames, oparg)
);
ERROR_IF(1, error);
}
- Py_INCREF(value);
+ value = PyStackRef_DUP(value_s);
}
replicate(8) pure inst(LOAD_FAST, (-- value)) {
- value = GETLOCAL(oparg);
- assert(value != NULL);
- Py_INCREF(value);
+ assert(PyStackRef_AsPyObjectBorrow(GETLOCAL(oparg)) != NULL);
+ value = PyStackRef_DUP(GETLOCAL(oparg));
}
inst(LOAD_FAST_AND_CLEAR, (-- value)) {
value = GETLOCAL(oparg);
// do not use SETLOCAL here, it decrefs the old value
- GETLOCAL(oparg) = NULL;
+ GETLOCAL(oparg) = PyStackRef_NULL;
}
inst(LOAD_FAST_LOAD_FAST, ( -- value1, value2)) {
uint32_t oparg1 = oparg >> 4;
uint32_t oparg2 = oparg & 15;
- value1 = GETLOCAL(oparg1);
- value2 = GETLOCAL(oparg2);
- Py_INCREF(value1);
- Py_INCREF(value2);
+ value1 = PyStackRef_DUP(GETLOCAL(oparg1));
+ value2 = PyStackRef_DUP(GETLOCAL(oparg2));
}
pure inst(LOAD_CONST, (-- value)) {
- value = GETITEM(FRAME_CO_CONSTS, oparg);
- Py_INCREF(value);
+ value = PyStackRef_FromPyObjectNew(GETITEM(FRAME_CO_CONSTS, oparg));
}
replicate(8) inst(STORE_FAST, (value --)) {
uint32_t oparg1 = oparg >> 4;
uint32_t oparg2 = oparg & 15;
SETLOCAL(oparg1, value1);
- value2 = GETLOCAL(oparg2);
- Py_INCREF(value2);
+ value2 = PyStackRef_DUP(GETLOCAL(oparg2));
}
inst(STORE_FAST_STORE_FAST, (value2, value1 --)) {
}
pure inst(PUSH_NULL, (-- res)) {
- res = NULL;
+ res = PyStackRef_NULL;
}
macro(END_FOR) = POP_TOP;
tier1 inst(INSTRUMENTED_END_FOR, (receiver, value -- receiver)) {
/* Need to create a fake StopIteration error here,
* to conform to PEP 380 */
- if (PyGen_Check(receiver)) {
- if (monitor_stop_iteration(tstate, frame, this_instr, value)) {
+ if (PyGen_Check(PyStackRef_AsPyObjectBorrow(receiver))) {
+ if (monitor_stop_iteration(tstate, frame, this_instr, PyStackRef_AsPyObjectBorrow(value))) {
ERROR_NO_POP();
}
}
}
pure inst(END_SEND, (receiver, value -- value)) {
- Py_DECREF(receiver);
+ (void)receiver;
+ PyStackRef_CLOSE(receiver);
}
tier1 inst(INSTRUMENTED_END_SEND, (receiver, value -- value)) {
- if (PyGen_Check(receiver) || PyCoro_CheckExact(receiver)) {
- if (monitor_stop_iteration(tstate, frame, this_instr, value)) {
+ PyObject *receiver_o = PyStackRef_AsPyObjectBorrow(receiver);
+ if (PyGen_Check(receiver_o) || PyCoro_CheckExact(receiver_o)) {
+ if (monitor_stop_iteration(tstate, frame, this_instr, PyStackRef_AsPyObjectBorrow(value))) {
ERROR_NO_POP();
}
}
- Py_DECREF(receiver);
+ PyStackRef_CLOSE(receiver);
}
inst(UNARY_NEGATIVE, (value -- res)) {
- res = PyNumber_Negative(value);
+ PyObject *res_o = PyNumber_Negative(PyStackRef_AsPyObjectBorrow(value));
DECREF_INPUTS();
- ERROR_IF(res == NULL, error);
+ ERROR_IF(res_o == NULL, error);
+ res = PyStackRef_FromPyObjectSteal(res_o);
}
pure inst(UNARY_NOT, (value -- res)) {
- assert(PyBool_Check(value));
- res = Py_IsFalse(value) ? Py_True : Py_False;
+ assert(PyBool_Check(PyStackRef_AsPyObjectBorrow(value)));
+ res = PyStackRef_Is(value, PyStackRef_False)
+ ? PyStackRef_True : PyStackRef_False;
}
family(TO_BOOL, INLINE_CACHE_ENTRIES_TO_BOOL) = {
}
op(_TO_BOOL, (value -- res)) {
- int err = PyObject_IsTrue(value);
+ int err = PyObject_IsTrue(PyStackRef_AsPyObjectBorrow(value));
DECREF_INPUTS();
ERROR_IF(err < 0, error);
- res = err ? Py_True : Py_False;
+ res = err ? PyStackRef_True : PyStackRef_False;
}
macro(TO_BOOL) = _SPECIALIZE_TO_BOOL + unused/2 + _TO_BOOL;
inst(TO_BOOL_BOOL, (unused/1, unused/2, value -- value)) {
- EXIT_IF(!PyBool_Check(value));
+ EXIT_IF(!PyBool_Check(PyStackRef_AsPyObjectBorrow(value)));
STAT_INC(TO_BOOL, hit);
}
inst(TO_BOOL_INT, (unused/1, unused/2, value -- res)) {
- EXIT_IF(!PyLong_CheckExact(value));
+ PyObject *value_o = PyStackRef_AsPyObjectBorrow(value);
+ EXIT_IF(!PyLong_CheckExact(value_o));
STAT_INC(TO_BOOL, hit);
- if (_PyLong_IsZero((PyLongObject *)value)) {
- assert(_Py_IsImmortal(value));
- res = Py_False;
+ if (_PyLong_IsZero((PyLongObject *)value_o)) {
+ assert(_Py_IsImmortal(value_o));
+ res = PyStackRef_False;
}
else {
DECREF_INPUTS();
- res = Py_True;
+ res = PyStackRef_True;
}
}
inst(TO_BOOL_LIST, (unused/1, unused/2, value -- res)) {
- EXIT_IF(!PyList_CheckExact(value));
+ PyObject *value_o = PyStackRef_AsPyObjectBorrow(value);
+ EXIT_IF(!PyList_CheckExact(value_o));
STAT_INC(TO_BOOL, hit);
- res = Py_SIZE(value) ? Py_True : Py_False;
+ res = Py_SIZE(value_o) ? PyStackRef_True : PyStackRef_False;
DECREF_INPUTS();
}
inst(TO_BOOL_NONE, (unused/1, unused/2, value -- res)) {
// This one is a bit weird, because we expect *some* failures:
- EXIT_IF(!Py_IsNone(value));
+ EXIT_IF(!PyStackRef_Is(value, PyStackRef_None));
STAT_INC(TO_BOOL, hit);
- res = Py_False;
+ res = PyStackRef_False;
}
inst(TO_BOOL_STR, (unused/1, unused/2, value -- res)) {
- EXIT_IF(!PyUnicode_CheckExact(value));
+ PyObject *value_o = PyStackRef_AsPyObjectBorrow(value);
+ EXIT_IF(!PyUnicode_CheckExact(value_o));
STAT_INC(TO_BOOL, hit);
- if (value == &_Py_STR(empty)) {
- assert(_Py_IsImmortal(value));
- res = Py_False;
+ if (value_o == &_Py_STR(empty)) {
+ assert(_Py_IsImmortal(value_o));
+ res = PyStackRef_False;
}
else {
- assert(Py_SIZE(value));
+ assert(Py_SIZE(value_o));
DECREF_INPUTS();
- res = Py_True;
+ res = PyStackRef_True;
}
}
op(_REPLACE_WITH_TRUE, (value -- res)) {
- Py_DECREF(value);
- res = Py_True;
+ DECREF_INPUTS();
+ res = PyStackRef_True;
}
macro(TO_BOOL_ALWAYS_TRUE) =
_REPLACE_WITH_TRUE;
inst(UNARY_INVERT, (value -- res)) {
- res = PyNumber_Invert(value);
+ PyObject *res_o = PyNumber_Invert(PyStackRef_AsPyObjectBorrow(value));
DECREF_INPUTS();
- ERROR_IF(res == NULL, error);
+ ERROR_IF(res_o == NULL, error);
+ res = PyStackRef_FromPyObjectSteal(res_o);
}
family(BINARY_OP, INLINE_CACHE_ENTRIES_BINARY_OP) = {
};
op(_GUARD_BOTH_INT, (left, right -- left, right)) {
- EXIT_IF(!PyLong_CheckExact(left));
- EXIT_IF(!PyLong_CheckExact(right));
+ PyObject *left_o = PyStackRef_AsPyObjectBorrow(left);
+ PyObject *right_o = PyStackRef_AsPyObjectBorrow(right);
+ EXIT_IF(!PyLong_CheckExact(left_o));
+ EXIT_IF(!PyLong_CheckExact(right_o));
}
op(_GUARD_NOS_INT, (left, unused -- left, unused)) {
- EXIT_IF(!PyLong_CheckExact(left));
+ PyObject *left_o = PyStackRef_AsPyObjectBorrow(left);
+ EXIT_IF(!PyLong_CheckExact(left_o));
}
op(_GUARD_TOS_INT, (value -- value)) {
- EXIT_IF(!PyLong_CheckExact(value));
+ PyObject *value_o = PyStackRef_AsPyObjectBorrow(value);
+ EXIT_IF(!PyLong_CheckExact(value_o));
}
pure op(_BINARY_OP_MULTIPLY_INT, (left, right -- res)) {
+ PyObject *left_o = PyStackRef_AsPyObjectBorrow(left);
+ PyObject *right_o = PyStackRef_AsPyObjectBorrow(right);
+
STAT_INC(BINARY_OP, hit);
- res = _PyLong_Multiply((PyLongObject *)left, (PyLongObject *)right);
- _Py_DECREF_SPECIALIZED(right, (destructor)PyObject_Free);
- _Py_DECREF_SPECIALIZED(left, (destructor)PyObject_Free);
- ERROR_IF(res == NULL, error);
+ PyObject *res_o = _PyLong_Multiply((PyLongObject *)left_o, (PyLongObject *)right_o);
+ _Py_DECREF_SPECIALIZED(right_o, (destructor)PyObject_Free);
+ _Py_DECREF_SPECIALIZED(left_o, (destructor)PyObject_Free);
+ ERROR_IF(res_o == NULL, error);
+ res = PyStackRef_FromPyObjectSteal(res_o);
}
pure op(_BINARY_OP_ADD_INT, (left, right -- res)) {
+ PyObject *left_o = PyStackRef_AsPyObjectBorrow(left);
+ PyObject *right_o = PyStackRef_AsPyObjectBorrow(right);
+
STAT_INC(BINARY_OP, hit);
- res = _PyLong_Add((PyLongObject *)left, (PyLongObject *)right);
- _Py_DECREF_SPECIALIZED(right, (destructor)PyObject_Free);
- _Py_DECREF_SPECIALIZED(left, (destructor)PyObject_Free);
- ERROR_IF(res == NULL, error);
+ PyObject *res_o = _PyLong_Add((PyLongObject *)left_o, (PyLongObject *)right_o);
+ _Py_DECREF_SPECIALIZED(right_o, (destructor)PyObject_Free);
+ _Py_DECREF_SPECIALIZED(left_o, (destructor)PyObject_Free);
+ ERROR_IF(res_o == NULL, error);
+ res = PyStackRef_FromPyObjectSteal(res_o);
}
pure op(_BINARY_OP_SUBTRACT_INT, (left, right -- res)) {
+ PyObject *left_o = PyStackRef_AsPyObjectBorrow(left);
+ PyObject *right_o = PyStackRef_AsPyObjectBorrow(right);
+
STAT_INC(BINARY_OP, hit);
- res = _PyLong_Subtract((PyLongObject *)left, (PyLongObject *)right);
- _Py_DECREF_SPECIALIZED(right, (destructor)PyObject_Free);
- _Py_DECREF_SPECIALIZED(left, (destructor)PyObject_Free);
- ERROR_IF(res == NULL, error);
+ PyObject *res_o = _PyLong_Subtract((PyLongObject *)left_o, (PyLongObject *)right_o);
+ _Py_DECREF_SPECIALIZED(right_o, (destructor)PyObject_Free);
+ _Py_DECREF_SPECIALIZED(left_o, (destructor)PyObject_Free);;
+ ERROR_IF(res_o == NULL, error);
+ res = PyStackRef_FromPyObjectSteal(res_o);
}
macro(BINARY_OP_MULTIPLY_INT) =
_GUARD_BOTH_INT + unused/1 + _BINARY_OP_SUBTRACT_INT;
op(_GUARD_BOTH_FLOAT, (left, right -- left, right)) {
- EXIT_IF(!PyFloat_CheckExact(left));
- EXIT_IF(!PyFloat_CheckExact(right));
+ PyObject *left_o = PyStackRef_AsPyObjectBorrow(left);
+ PyObject *right_o = PyStackRef_AsPyObjectBorrow(right);
+ EXIT_IF(!PyFloat_CheckExact(left_o));
+ EXIT_IF(!PyFloat_CheckExact(right_o));
}
op(_GUARD_NOS_FLOAT, (left, unused -- left, unused)) {
- EXIT_IF(!PyFloat_CheckExact(left));
+ PyObject *left_o = PyStackRef_AsPyObjectBorrow(left);
+ EXIT_IF(!PyFloat_CheckExact(left_o));
}
op(_GUARD_TOS_FLOAT, (value -- value)) {
- EXIT_IF(!PyFloat_CheckExact(value));
+ PyObject *value_o = PyStackRef_AsPyObjectBorrow(value);
+ EXIT_IF(!PyFloat_CheckExact(value_o));
}
pure op(_BINARY_OP_MULTIPLY_FLOAT, (left, right -- res)) {
+ PyObject *left_o = PyStackRef_AsPyObjectBorrow(left);
+ PyObject *right_o = PyStackRef_AsPyObjectBorrow(right);
+
STAT_INC(BINARY_OP, hit);
double dres =
- ((PyFloatObject *)left)->ob_fval *
- ((PyFloatObject *)right)->ob_fval;
- DECREF_INPUTS_AND_REUSE_FLOAT(left, right, dres, res);
+ ((PyFloatObject *)left_o)->ob_fval *
+ ((PyFloatObject *)right_o)->ob_fval;
+ PyObject *res_o;
+ DECREF_INPUTS_AND_REUSE_FLOAT(left_o, right_o, dres, res_o);
+ res = PyStackRef_FromPyObjectSteal(res_o);
}
pure op(_BINARY_OP_ADD_FLOAT, (left, right -- res)) {
+ PyObject *left_o = PyStackRef_AsPyObjectBorrow(left);
+ PyObject *right_o = PyStackRef_AsPyObjectBorrow(right);
+
STAT_INC(BINARY_OP, hit);
double dres =
- ((PyFloatObject *)left)->ob_fval +
- ((PyFloatObject *)right)->ob_fval;
- DECREF_INPUTS_AND_REUSE_FLOAT(left, right, dres, res);
+ ((PyFloatObject *)left_o)->ob_fval +
+ ((PyFloatObject *)right_o)->ob_fval;
+ PyObject *res_o;
+ DECREF_INPUTS_AND_REUSE_FLOAT(left_o, right_o, dres, res_o);
+ res = PyStackRef_FromPyObjectSteal(res_o);
}
pure op(_BINARY_OP_SUBTRACT_FLOAT, (left, right -- res)) {
+ PyObject *left_o = PyStackRef_AsPyObjectBorrow(left);
+ PyObject *right_o = PyStackRef_AsPyObjectBorrow(right);
+
STAT_INC(BINARY_OP, hit);
double dres =
- ((PyFloatObject *)left)->ob_fval -
- ((PyFloatObject *)right)->ob_fval;
- DECREF_INPUTS_AND_REUSE_FLOAT(left, right, dres, res);
+ ((PyFloatObject *)left_o)->ob_fval -
+ ((PyFloatObject *)right_o)->ob_fval;
+ PyObject *res_o;
+ DECREF_INPUTS_AND_REUSE_FLOAT(left_o, right_o, dres, res_o);
+ res = PyStackRef_FromPyObjectSteal(res_o);
}
macro(BINARY_OP_MULTIPLY_FLOAT) =
_GUARD_BOTH_FLOAT + unused/1 + _BINARY_OP_SUBTRACT_FLOAT;
op(_GUARD_BOTH_UNICODE, (left, right -- left, right)) {
- EXIT_IF(!PyUnicode_CheckExact(left));
- EXIT_IF(!PyUnicode_CheckExact(right));
+ PyObject *left_o = PyStackRef_AsPyObjectBorrow(left);
+ PyObject *right_o = PyStackRef_AsPyObjectBorrow(right);
+
+ EXIT_IF(!PyUnicode_CheckExact(left_o));
+ EXIT_IF(!PyUnicode_CheckExact(right_o));
}
pure op(_BINARY_OP_ADD_UNICODE, (left, right -- res)) {
+ PyObject *left_o = PyStackRef_AsPyObjectBorrow(left);
+ PyObject *right_o = PyStackRef_AsPyObjectBorrow(right);
+
STAT_INC(BINARY_OP, hit);
- res = PyUnicode_Concat(left, right);
- _Py_DECREF_SPECIALIZED(left, _PyUnicode_ExactDealloc);
- _Py_DECREF_SPECIALIZED(right, _PyUnicode_ExactDealloc);
- ERROR_IF(res == NULL, error);
+ PyObject *res_o = PyUnicode_Concat(left_o, right_o);
+ _Py_DECREF_SPECIALIZED(left_o, _PyUnicode_ExactDealloc);
+ _Py_DECREF_SPECIALIZED(right_o, _PyUnicode_ExactDealloc);
+ ERROR_IF(res_o == NULL, error);
+ res = PyStackRef_FromPyObjectSteal(res_o);
}
macro(BINARY_OP_ADD_UNICODE) =
// specializations, but there is no output.
// At the end we just skip over the STORE_FAST.
tier1 op(_BINARY_OP_INPLACE_ADD_UNICODE, (left, right --)) {
+ PyObject *left_o = PyStackRef_AsPyObjectBorrow(left);
+ PyObject *right_o = PyStackRef_AsPyObjectBorrow(right);
+
assert(next_instr->op.code == STORE_FAST);
- PyObject **target_local = &GETLOCAL(next_instr->op.arg);
- DEOPT_IF(*target_local != left);
+ _PyStackRef *target_local = &GETLOCAL(next_instr->op.arg);
+ DEOPT_IF(!PyStackRef_Is(*target_local, left));
STAT_INC(BINARY_OP, hit);
/* Handle `left = left + right` or `left += right` for str.
*
* only the locals reference, so PyUnicode_Append knows
* that the string is safe to mutate.
*/
- assert(Py_REFCNT(left) >= 2);
- _Py_DECREF_NO_DEALLOC(left);
- PyUnicode_Append(target_local, right);
- _Py_DECREF_SPECIALIZED(right, _PyUnicode_ExactDealloc);
- ERROR_IF(*target_local == NULL, error);
+ assert(Py_REFCNT(left_o) >= 2);
+ _Py_DECREF_NO_DEALLOC(left_o);
+ PyObject *temp = PyStackRef_AsPyObjectBorrow(*target_local);
+ PyUnicode_Append(&temp, right_o);
+ *target_local = PyStackRef_FromPyObjectSteal(temp);
+ _Py_DECREF_SPECIALIZED(right_o, _PyUnicode_ExactDealloc);
+ ERROR_IF(PyStackRef_IsNull(*target_local), error);
// The STORE_FAST is already done.
assert(next_instr->op.code == STORE_FAST);
SKIP_OVER(1);
}
op(_BINARY_SUBSCR, (container, sub -- res)) {
- res = PyObject_GetItem(container, sub);
+ PyObject *container_o = PyStackRef_AsPyObjectBorrow(container);
+ PyObject *sub_o = PyStackRef_AsPyObjectBorrow(sub);
+
+ PyObject *res_o = PyObject_GetItem(container_o, sub_o);
DECREF_INPUTS();
- ERROR_IF(res == NULL, error);
+ ERROR_IF(res_o == NULL, error);
+ res = PyStackRef_FromPyObjectSteal(res_o);
}
macro(BINARY_SUBSCR) = _SPECIALIZE_BINARY_SUBSCR + _BINARY_SUBSCR;
inst(BINARY_SLICE, (container, start, stop -- res)) {
- PyObject *slice = _PyBuildSlice_ConsumeRefs(start, stop);
+ PyObject *slice = _PyBuildSlice_ConsumeRefs(PyStackRef_AsPyObjectSteal(start),
+ PyStackRef_AsPyObjectSteal(stop));
+ PyObject *res_o;
// Can't use ERROR_IF() here, because we haven't
// DECREF'ed container yet, and we still own slice.
if (slice == NULL) {
- res = NULL;
+ res_o = NULL;
}
else {
- res = PyObject_GetItem(container, slice);
+ res_o = PyObject_GetItem(PyStackRef_AsPyObjectBorrow(container), slice);
Py_DECREF(slice);
}
- Py_DECREF(container);
- ERROR_IF(res == NULL, error);
+ PyStackRef_CLOSE(container);
+ ERROR_IF(res_o == NULL, error);
+ res = PyStackRef_FromPyObjectSteal(res_o);
}
inst(STORE_SLICE, (v, container, start, stop -- )) {
- PyObject *slice = _PyBuildSlice_ConsumeRefs(start, stop);
+ PyObject *slice = _PyBuildSlice_ConsumeRefs(PyStackRef_AsPyObjectSteal(start),
+ PyStackRef_AsPyObjectSteal(stop));
int err;
if (slice == NULL) {
err = 1;
}
else {
- err = PyObject_SetItem(container, slice, v);
+ err = PyObject_SetItem(PyStackRef_AsPyObjectBorrow(container), slice, PyStackRef_AsPyObjectSteal(v));
Py_DECREF(slice);
}
- Py_DECREF(v);
- Py_DECREF(container);
+ PyStackRef_CLOSE(v);
+ PyStackRef_CLOSE(container);
ERROR_IF(err, error);
}
- inst(BINARY_SUBSCR_LIST_INT, (unused/1, list, sub -- res)) {
+ inst(BINARY_SUBSCR_LIST_INT, (unused/1, list_st, sub_st -- res)) {
+ PyObject *sub = PyStackRef_AsPyObjectBorrow(sub_st);
+ PyObject *list = PyStackRef_AsPyObjectBorrow(list_st);
+
DEOPT_IF(!PyLong_CheckExact(sub));
DEOPT_IF(!PyList_CheckExact(list));
Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0];
DEOPT_IF(index >= PyList_GET_SIZE(list));
STAT_INC(BINARY_SUBSCR, hit);
- res = PyList_GET_ITEM(list, index);
- assert(res != NULL);
- Py_INCREF(res);
+ PyObject *res_o = PyList_GET_ITEM(list, index);
+ assert(res_o != NULL);
+ Py_INCREF(res_o);
_Py_DECREF_SPECIALIZED(sub, (destructor)PyObject_Free);
- Py_DECREF(list);
+ PyStackRef_CLOSE(list_st);
+ res = PyStackRef_FromPyObjectSteal(res_o);
}
- inst(BINARY_SUBSCR_STR_INT, (unused/1, str, sub -- res)) {
+ inst(BINARY_SUBSCR_STR_INT, (unused/1, str_st, sub_st -- res)) {
+ PyObject *sub = PyStackRef_AsPyObjectBorrow(sub_st);
+ PyObject *str = PyStackRef_AsPyObjectBorrow(str_st);
+
DEOPT_IF(!PyLong_CheckExact(sub));
DEOPT_IF(!PyUnicode_CheckExact(str));
DEOPT_IF(!_PyLong_IsNonNegativeCompact((PyLongObject *)sub));
Py_UCS4 c = PyUnicode_READ_CHAR(str, index);
DEOPT_IF(Py_ARRAY_LENGTH(_Py_SINGLETON(strings).ascii) <= c);
STAT_INC(BINARY_SUBSCR, hit);
- res = (PyObject*)&_Py_SINGLETON(strings).ascii[c];
+ PyObject *res_o = (PyObject*)&_Py_SINGLETON(strings).ascii[c];
_Py_DECREF_SPECIALIZED(sub, (destructor)PyObject_Free);
- Py_DECREF(str);
+ PyStackRef_CLOSE(str_st);
+ res = PyStackRef_FromPyObjectSteal(res_o);
}
- inst(BINARY_SUBSCR_TUPLE_INT, (unused/1, tuple, sub -- res)) {
+ inst(BINARY_SUBSCR_TUPLE_INT, (unused/1, tuple_st, sub_st -- res)) {
+ PyObject *sub = PyStackRef_AsPyObjectBorrow(sub_st);
+ PyObject *tuple = PyStackRef_AsPyObjectBorrow(tuple_st);
+
DEOPT_IF(!PyLong_CheckExact(sub));
DEOPT_IF(!PyTuple_CheckExact(tuple));
Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0];
DEOPT_IF(index >= PyTuple_GET_SIZE(tuple));
STAT_INC(BINARY_SUBSCR, hit);
- res = PyTuple_GET_ITEM(tuple, index);
- assert(res != NULL);
- Py_INCREF(res);
+ PyObject *res_o = PyTuple_GET_ITEM(tuple, index);
+ assert(res_o != NULL);
+ Py_INCREF(res_o);
_Py_DECREF_SPECIALIZED(sub, (destructor)PyObject_Free);
- Py_DECREF(tuple);
+ PyStackRef_CLOSE(tuple_st);
+ res = PyStackRef_FromPyObjectSteal(res_o);
}
- inst(BINARY_SUBSCR_DICT, (unused/1, dict, sub -- res)) {
+ inst(BINARY_SUBSCR_DICT, (unused/1, dict_st, sub_st -- res)) {
+ PyObject *sub = PyStackRef_AsPyObjectBorrow(sub_st);
+ PyObject *dict = PyStackRef_AsPyObjectBorrow(dict_st);
+
DEOPT_IF(!PyDict_CheckExact(dict));
STAT_INC(BINARY_SUBSCR, hit);
- int rc = PyDict_GetItemRef(dict, sub, &res);
+ PyObject *res_o;
+ int rc = PyDict_GetItemRef(dict, sub, &res_o);
if (rc == 0) {
_PyErr_SetKeyError(sub);
}
DECREF_INPUTS();
ERROR_IF(rc <= 0, error); // not found or error
+ res = PyStackRef_FromPyObjectSteal(res_o);
}
- inst(BINARY_SUBSCR_GETITEM, (unused/1, container, sub -- unused)) {
+ inst(BINARY_SUBSCR_GETITEM, (unused/1, container_st, sub_st -- unused)) {
+ PyObject *container = PyStackRef_AsPyObjectBorrow(container_st);
+
DEOPT_IF(tstate->interp->eval_frame);
PyTypeObject *tp = Py_TYPE(container);
DEOPT_IF(!PyType_HasFeature(tp, Py_TPFLAGS_HEAPTYPE));
Py_INCREF(getitem);
_PyInterpreterFrame *new_frame = _PyFrame_PushUnchecked(tstate, getitem, 2);
STACK_SHRINK(2);
- new_frame->localsplus[0] = container;
- new_frame->localsplus[1] = sub;
+ new_frame->localsplus[0] = container_st;
+ new_frame->localsplus[1] = sub_st;
frame->return_offset = (uint16_t)(next_instr - this_instr);
DISPATCH_INLINED(new_frame);
}
inst(LIST_APPEND, (list, unused[oparg-1], v -- list, unused[oparg-1])) {
- ERROR_IF(_PyList_AppendTakeRef((PyListObject *)list, v) < 0, error);
+ ERROR_IF(_PyList_AppendTakeRef((PyListObject *)PyStackRef_AsPyObjectBorrow(list),
+ PyStackRef_AsPyObjectSteal(v)) < 0, error);
}
inst(SET_ADD, (set, unused[oparg-1], v -- set, unused[oparg-1])) {
- int err = PySet_Add(set, v);
+ int err = PySet_Add(PyStackRef_AsPyObjectBorrow(set),
+ PyStackRef_AsPyObjectSteal(v));
DECREF_INPUTS();
ERROR_IF(err, error);
}
op(_STORE_SUBSCR, (v, container, sub -- )) {
/* container[sub] = v */
- int err = PyObject_SetItem(container, sub, v);
+ int err = PyObject_SetItem(PyStackRef_AsPyObjectBorrow(container), PyStackRef_AsPyObjectSteal(sub), PyStackRef_AsPyObjectSteal(v));
DECREF_INPUTS();
ERROR_IF(err, error);
}
macro(STORE_SUBSCR) = _SPECIALIZE_STORE_SUBSCR + _STORE_SUBSCR;
- inst(STORE_SUBSCR_LIST_INT, (unused/1, value, list, sub -- )) {
+ inst(STORE_SUBSCR_LIST_INT, (unused/1, value, list_st, sub_st -- )) {
+ PyObject *sub = PyStackRef_AsPyObjectBorrow(sub_st);
+ PyObject *list = PyStackRef_AsPyObjectBorrow(list_st);
+
DEOPT_IF(!PyLong_CheckExact(sub));
DEOPT_IF(!PyList_CheckExact(list));
STAT_INC(STORE_SUBSCR, hit);
PyObject *old_value = PyList_GET_ITEM(list, index);
- PyList_SET_ITEM(list, index, value);
+ PyList_SET_ITEM(list, index, PyStackRef_AsPyObjectSteal(value));
assert(old_value != NULL);
Py_DECREF(old_value);
_Py_DECREF_SPECIALIZED(sub, (destructor)PyObject_Free);
- Py_DECREF(list);
+ PyStackRef_CLOSE(list_st);
}
- inst(STORE_SUBSCR_DICT, (unused/1, value, dict, sub -- )) {
+ inst(STORE_SUBSCR_DICT, (unused/1, value, dict_st, sub_st -- )) {
+ PyObject *sub = PyStackRef_AsPyObjectBorrow(sub_st);
+ PyObject *dict = PyStackRef_AsPyObjectBorrow(dict_st);
+
DEOPT_IF(!PyDict_CheckExact(dict));
STAT_INC(STORE_SUBSCR, hit);
- int err = _PyDict_SetItem_Take2((PyDictObject *)dict, sub, value);
- Py_DECREF(dict);
+ int err = _PyDict_SetItem_Take2((PyDictObject *)dict, sub, PyStackRef_AsPyObjectSteal(value));
+ PyStackRef_CLOSE(dict_st);
ERROR_IF(err, error);
}
inst(DELETE_SUBSCR, (container, sub --)) {
/* del container[sub] */
- int err = PyObject_DelItem(container, sub);
+ int err = PyObject_DelItem(PyStackRef_AsPyObjectBorrow(container),
+ PyStackRef_AsPyObjectBorrow(sub));
DECREF_INPUTS();
ERROR_IF(err, error);
}
inst(CALL_INTRINSIC_1, (value -- res)) {
assert(oparg <= MAX_INTRINSIC_1);
- res = _PyIntrinsics_UnaryFunctions[oparg].func(tstate, value);
+ PyObject *res_o = _PyIntrinsics_UnaryFunctions[oparg].func(tstate, PyStackRef_AsPyObjectBorrow(value));
DECREF_INPUTS();
- ERROR_IF(res == NULL, error);
+ ERROR_IF(res_o == NULL, error);
+ res = PyStackRef_FromPyObjectSteal(res_o);
}
- inst(CALL_INTRINSIC_2, (value2, value1 -- res)) {
+ inst(CALL_INTRINSIC_2, (value2_st, value1_st -- res)) {
assert(oparg <= MAX_INTRINSIC_2);
- res = _PyIntrinsics_BinaryFunctions[oparg].func(tstate, value2, value1);
+ PyObject *value1 = PyStackRef_AsPyObjectBorrow(value1_st);
+ PyObject *value2 = PyStackRef_AsPyObjectBorrow(value2_st);
+
+ PyObject *res_o = _PyIntrinsics_BinaryFunctions[oparg].func(tstate, value2, value1);
DECREF_INPUTS();
- ERROR_IF(res == NULL, error);
+ ERROR_IF(res_o == NULL, error);
+ res = PyStackRef_FromPyObjectSteal(res_o);
}
tier1 inst(RAISE_VARARGS, (args[oparg] -- )) {
PyObject *cause = NULL, *exc = NULL;
switch (oparg) {
case 2:
- cause = args[1];
+ cause = PyStackRef_AsPyObjectSteal(args[1]);
/* fall through */
case 1:
- exc = args[0];
+ exc = PyStackRef_AsPyObjectSteal(args[0]);
/* fall through */
case 0:
if (do_raise(tstate, exc, cause)) {
tstate->current_frame = frame->previous;
assert(!_PyErr_Occurred(tstate));
tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS;
- return retval;
+ return PyStackRef_AsPyObjectSteal(retval);
}
// The stack effect here is ambiguous.
inst(INSTRUMENTED_RETURN_VALUE, (retval --)) {
int err = _Py_call_instrumentation_arg(
tstate, PY_MONITORING_EVENT_PY_RETURN,
- frame, this_instr, retval);
+ frame, this_instr, PyStackRef_AsPyObjectBorrow(retval));
if (err) ERROR_NO_POP();
STACK_SHRINK(1);
assert(EMPTY());
_PyInterpreterFrame *dying = frame;
frame = tstate->current_frame = dying->previous;
_PyEval_FrameClearAndPop(tstate, dying);
- _PyFrame_StackPush(frame, retval);
+ _PyFrame_StackPush(frame, PyStackRef_FromPyObjectSteal(retval));
LOAD_IP(frame->return_offset);
goto resume_frame;
}
inst(GET_AITER, (obj -- iter)) {
unaryfunc getter = NULL;
- PyTypeObject *type = Py_TYPE(obj);
+ PyObject *obj_o = PyStackRef_AsPyObjectBorrow(obj);
+ PyObject *iter_o;
+ PyTypeObject *type = Py_TYPE(obj_o);
if (type->tp_as_async != NULL) {
getter = type->tp_as_async->am_aiter;
ERROR_IF(true, error);
}
- iter = (*getter)(obj);
+ iter_o = (*getter)(obj_o);
DECREF_INPUTS();
- ERROR_IF(iter == NULL, error);
+ ERROR_IF(iter_o == NULL, error);
- if (Py_TYPE(iter)->tp_as_async == NULL ||
- Py_TYPE(iter)->tp_as_async->am_anext == NULL) {
+ if (Py_TYPE(iter_o)->tp_as_async == NULL ||
+ Py_TYPE(iter_o)->tp_as_async->am_anext == NULL) {
_PyErr_Format(tstate, PyExc_TypeError,
"'async for' received an object from __aiter__ "
"that does not implement __anext__: %.100s",
- Py_TYPE(iter)->tp_name);
- Py_DECREF(iter);
+ Py_TYPE(iter_o)->tp_name);
+ Py_DECREF(iter_o);
ERROR_IF(true, error);
}
+ iter = PyStackRef_FromPyObjectSteal(iter_o);
}
inst(GET_ANEXT, (aiter -- aiter, awaitable)) {
unaryfunc getter = NULL;
PyObject *next_iter = NULL;
- PyTypeObject *type = Py_TYPE(aiter);
+ PyObject *awaitable_o;
+ PyObject *aiter_o = PyStackRef_AsPyObjectBorrow(aiter);
+ PyTypeObject *type = Py_TYPE(aiter_o);
- if (PyAsyncGen_CheckExact(aiter)) {
- awaitable = type->tp_as_async->am_anext(aiter);
- if (awaitable == NULL) {
+ if (PyAsyncGen_CheckExact(aiter_o)) {
+ awaitable_o = type->tp_as_async->am_anext(aiter_o);
+ if (awaitable_o == NULL) {
ERROR_NO_POP();
}
} else {
}
if (getter != NULL) {
- next_iter = (*getter)(aiter);
+ next_iter = (*getter)(aiter_o);
if (next_iter == NULL) {
ERROR_NO_POP();
}
ERROR_NO_POP();
}
- awaitable = _PyCoro_GetAwaitableIter(next_iter);
- if (awaitable == NULL) {
+ awaitable_o = _PyCoro_GetAwaitableIter(next_iter);
+ if (awaitable_o == NULL) {
_PyErr_FormatFromCause(
PyExc_TypeError,
"'async for' received an invalid object "
Py_DECREF(next_iter);
}
}
+ awaitable = PyStackRef_FromPyObjectSteal(awaitable_o);
}
inst(GET_AWAITABLE, (iterable -- iter)) {
- iter = _PyCoro_GetAwaitableIter(iterable);
+ PyObject *iter_o = _PyCoro_GetAwaitableIter(PyStackRef_AsPyObjectBorrow(iterable));
- if (iter == NULL) {
- _PyEval_FormatAwaitableError(tstate, Py_TYPE(iterable), oparg);
+ if (iter_o == NULL) {
+ _PyEval_FormatAwaitableError(tstate,
+ Py_TYPE(PyStackRef_AsPyObjectBorrow(iterable)), oparg);
}
DECREF_INPUTS();
- if (iter != NULL && PyCoro_CheckExact(iter)) {
- PyObject *yf = _PyGen_yf((PyGenObject*)iter);
+ if (iter_o != NULL && PyCoro_CheckExact(iter_o)) {
+ PyObject *yf = _PyGen_yf((PyGenObject*)iter_o);
if (yf != NULL) {
/* `iter` is a coroutine object that is being
awaited, `yf` is a pointer to the current awaitable
being awaited on. */
Py_DECREF(yf);
- Py_CLEAR(iter);
+ Py_CLEAR(iter_o);
_PyErr_SetString(tstate, PyExc_RuntimeError,
"coroutine is being awaited already");
/* The code below jumps to `error` if `iter` is NULL. */
}
}
- ERROR_IF(iter == NULL, error);
+ ERROR_IF(iter_o == NULL, error);
+ iter = PyStackRef_FromPyObjectSteal(iter_o);
}
family(SEND, INLINE_CACHE_ENTRIES_SEND) = {
}
op(_SEND, (receiver, v -- receiver, retval)) {
+ PyObject *receiver_o = PyStackRef_AsPyObjectBorrow(receiver);
+
+ PyObject *retval_o;
assert(frame != &entry_frame);
if ((tstate->interp->eval_frame == NULL) &&
- (Py_TYPE(receiver) == &PyGen_Type || Py_TYPE(receiver) == &PyCoro_Type) &&
- ((PyGenObject *)receiver)->gi_frame_state < FRAME_EXECUTING)
+ (Py_TYPE(receiver_o) == &PyGen_Type || Py_TYPE(receiver_o) == &PyCoro_Type) &&
+ ((PyGenObject *)receiver_o)->gi_frame_state < FRAME_EXECUTING)
{
- PyGenObject *gen = (PyGenObject *)receiver;
+ PyGenObject *gen = (PyGenObject *)receiver_o;
_PyInterpreterFrame *gen_frame = &gen->gi_iframe;
STACK_SHRINK(1);
_PyFrame_StackPush(gen_frame, v);
frame->return_offset = (uint16_t)(next_instr - this_instr + oparg);
DISPATCH_INLINED(gen_frame);
}
- if (Py_IsNone(v) && PyIter_Check(receiver)) {
- retval = Py_TYPE(receiver)->tp_iternext(receiver);
+ if (PyStackRef_Is(v, PyStackRef_None) && PyIter_Check(receiver_o)) {
+ retval_o = Py_TYPE(receiver_o)->tp_iternext(receiver_o);
}
else {
- retval = PyObject_CallMethodOneArg(receiver, &_Py_ID(send), v);
+ retval_o = PyObject_CallMethodOneArg(receiver_o,
+ &_Py_ID(send),
+ PyStackRef_AsPyObjectBorrow(v));
}
- if (retval == NULL) {
+ if (retval_o == NULL) {
if (_PyErr_ExceptionMatches(tstate, PyExc_StopIteration)
) {
monitor_raise(tstate, frame, this_instr);
}
- if (_PyGen_FetchStopIterationValue(&retval) == 0) {
- assert(retval != NULL);
+ if (_PyGen_FetchStopIterationValue(&retval_o) == 0) {
+ assert(retval_o != NULL);
JUMPBY(oparg);
}
else {
ERROR_NO_POP();
}
}
- Py_DECREF(v);
+ PyStackRef_CLOSE(v);
+ retval = PyStackRef_FromPyObjectSteal(retval_o);
}
macro(SEND) = _SPECIALIZE_SEND + _SEND;
inst(SEND_GEN, (unused/1, receiver, v -- receiver, unused)) {
DEOPT_IF(tstate->interp->eval_frame);
- PyGenObject *gen = (PyGenObject *)receiver;
+ PyGenObject *gen = (PyGenObject *)PyStackRef_AsPyObjectBorrow(receiver);
DEOPT_IF(Py_TYPE(gen) != &PyGen_Type && Py_TYPE(gen) != &PyCoro_Type);
DEOPT_IF(gen->gi_frame_state >= FRAME_EXECUTING);
STAT_INC(SEND, hit);
_PyFrame_SetStackPointer(frame, stack_pointer - 1);
int err = _Py_call_instrumentation_arg(
tstate, PY_MONITORING_EVENT_PY_YIELD,
- frame, this_instr, retval);
+ frame, this_instr, PyStackRef_AsPyObjectBorrow(retval));
if (err) ERROR_NO_POP();
tstate->exc_info = gen->gi_exc_state.previous_item;
gen->gi_exc_state.previous_item = NULL;
inst(POP_EXCEPT, (exc_value -- )) {
_PyErr_StackItem *exc_info = tstate->exc_info;
- Py_XSETREF(exc_info->exc_value, exc_value == Py_None ? NULL : exc_value);
+ Py_XSETREF(exc_info->exc_value,
+ PyStackRef_AsPyObjectBorrow(exc_value) == Py_None
+ ? NULL : PyStackRef_AsPyObjectSteal(exc_value));
}
- tier1 inst(RERAISE, (values[oparg], exc -- values[oparg])) {
+ tier1 inst(RERAISE, (values[oparg], exc_st -- values[oparg])) {
+ PyObject *exc = PyStackRef_AsPyObjectBorrow(exc_st);
+
assert(oparg >= 0 && oparg <= 2);
if (oparg) {
- PyObject *lasti = values[0];
+ PyObject *lasti = PyStackRef_AsPyObjectBorrow(values[0]);
if (PyLong_Check(lasti)) {
frame->instr_ptr = _PyCode_CODE(_PyFrame_GetCode(frame)) + PyLong_AsLong(lasti);
assert(!_PyErr_Occurred(tstate));
goto exception_unwind;
}
- tier1 inst(END_ASYNC_FOR, (awaitable, exc -- )) {
+ tier1 inst(END_ASYNC_FOR, (awaitable_st, exc_st -- )) {
+ PyObject *exc = PyStackRef_AsPyObjectBorrow(exc_st);
+
assert(exc && PyExceptionInstance_Check(exc));
if (PyErr_GivenExceptionMatches(exc, PyExc_StopAsyncIteration)) {
DECREF_INPUTS();
}
}
- tier1 inst(CLEANUP_THROW, (sub_iter, last_sent_val, exc_value -- none, value)) {
+ tier1 inst(CLEANUP_THROW, (sub_iter_st, last_sent_val_st, exc_value_st -- none, value)) {
+ PyObject *exc_value = PyStackRef_AsPyObjectBorrow(exc_value_st);
assert(throwflag);
assert(exc_value && PyExceptionInstance_Check(exc_value));
+
if (PyErr_GivenExceptionMatches(exc_value, PyExc_StopIteration)) {
- value = Py_NewRef(((PyStopIterationObject *)exc_value)->value);
+ value = PyStackRef_FromPyObjectNew(((PyStopIterationObject *)exc_value)->value);
DECREF_INPUTS();
- none = Py_None;
+ none = PyStackRef_None;
}
else {
_PyErr_SetRaisedException(tstate, Py_NewRef(exc_value));
// Keep in sync with _common_constants in opcode.py
switch(oparg) {
case CONSTANT_ASSERTIONERROR:
- value = PyExc_AssertionError;
+ value = PyStackRef_FromPyObjectImmortal(PyExc_AssertionError);
break;
case CONSTANT_NOTIMPLEMENTEDERROR:
- value = PyExc_NotImplementedError;
+ value = PyStackRef_FromPyObjectImmortal(PyExc_NotImplementedError);
break;
default:
Py_FatalError("bad LOAD_COMMON_CONSTANT oparg");
}
inst(LOAD_BUILD_CLASS, ( -- bc)) {
- ERROR_IF(PyMapping_GetOptionalItem(BUILTINS(), &_Py_ID(__build_class__), &bc) < 0, error);
- if (bc == NULL) {
+ PyObject *bc_o;
+ ERROR_IF(PyMapping_GetOptionalItem(BUILTINS(), &_Py_ID(__build_class__), &bc_o) < 0, error);
+ if (bc_o == NULL) {
_PyErr_SetString(tstate, PyExc_NameError,
"__build_class__ not found");
ERROR_IF(true, error);
}
+ bc = PyStackRef_FromPyObjectSteal(bc_o);
}
inst(STORE_NAME, (v -- )) {
ERROR_IF(true, error);
}
if (PyDict_CheckExact(ns))
- err = PyDict_SetItem(ns, name, v);
+ err = PyDict_SetItem(ns, name, PyStackRef_AsPyObjectSteal(v));
else
- err = PyObject_SetItem(ns, name, v);
+ err = PyObject_SetItem(ns, name, PyStackRef_AsPyObjectSteal(v));
DECREF_INPUTS();
ERROR_IF(err, error);
}
}
op(_UNPACK_SEQUENCE, (seq -- unused[oparg])) {
- PyObject **top = stack_pointer + oparg - 1;
- int res = _PyEval_UnpackIterable(tstate, seq, oparg, -1, top);
+ _PyStackRef *top = stack_pointer + oparg - 1;
+ int res = _PyEval_UnpackIterableStackRef(tstate, seq, oparg, -1, top);
DECREF_INPUTS();
ERROR_IF(res == 0, error);
}
inst(UNPACK_SEQUENCE_TWO_TUPLE, (unused/1, seq -- val1, val0)) {
assert(oparg == 2);
- DEOPT_IF(!PyTuple_CheckExact(seq));
- DEOPT_IF(PyTuple_GET_SIZE(seq) != 2);
+ PyObject *seq_o = PyStackRef_AsPyObjectBorrow(seq);
+ DEOPT_IF(!PyTuple_CheckExact(seq_o));
+ DEOPT_IF(PyTuple_GET_SIZE(seq_o) != 2);
STAT_INC(UNPACK_SEQUENCE, hit);
- val0 = Py_NewRef(PyTuple_GET_ITEM(seq, 0));
- val1 = Py_NewRef(PyTuple_GET_ITEM(seq, 1));
+ val0 = PyStackRef_FromPyObjectNew(PyTuple_GET_ITEM(seq_o, 0));
+ val1 = PyStackRef_FromPyObjectNew(PyTuple_GET_ITEM(seq_o, 1));
DECREF_INPUTS();
}
inst(UNPACK_SEQUENCE_TUPLE, (unused/1, seq -- values[oparg])) {
- DEOPT_IF(!PyTuple_CheckExact(seq));
- DEOPT_IF(PyTuple_GET_SIZE(seq) != oparg);
+ PyObject *seq_o = PyStackRef_AsPyObjectBorrow(seq);
+ DEOPT_IF(!PyTuple_CheckExact(seq_o));
+ DEOPT_IF(PyTuple_GET_SIZE(seq_o) != oparg);
STAT_INC(UNPACK_SEQUENCE, hit);
- PyObject **items = _PyTuple_ITEMS(seq);
+ PyObject **items = _PyTuple_ITEMS(seq_o);
for (int i = oparg; --i >= 0; ) {
- *values++ = Py_NewRef(items[i]);
+ *values++ = PyStackRef_FromPyObjectNew(items[i]);
}
DECREF_INPUTS();
}
inst(UNPACK_SEQUENCE_LIST, (unused/1, seq -- values[oparg])) {
- DEOPT_IF(!PyList_CheckExact(seq));
- DEOPT_IF(PyList_GET_SIZE(seq) != oparg);
+ PyObject *seq_o = PyStackRef_AsPyObjectBorrow(seq);
+ DEOPT_IF(!PyList_CheckExact(seq_o));
+ DEOPT_IF(PyList_GET_SIZE(seq_o) != oparg);
STAT_INC(UNPACK_SEQUENCE, hit);
- PyObject **items = _PyList_ITEMS(seq);
+ PyObject **items = _PyList_ITEMS(seq_o);
for (int i = oparg; --i >= 0; ) {
- *values++ = Py_NewRef(items[i]);
+ *values++ = PyStackRef_FromPyObjectNew(items[i]);
}
DECREF_INPUTS();
}
inst(UNPACK_EX, (seq -- unused[oparg & 0xFF], unused, unused[oparg >> 8])) {
int totalargs = 1 + (oparg & 0xFF) + (oparg >> 8);
- PyObject **top = stack_pointer + totalargs - 1;
- int res = _PyEval_UnpackIterable(tstate, seq, oparg & 0xFF, oparg >> 8, top);
+ _PyStackRef *top = stack_pointer + totalargs - 1;
+ int res = _PyEval_UnpackIterableStackRef(tstate, seq, oparg & 0xFF, oparg >> 8, top);
DECREF_INPUTS();
ERROR_IF(res == 0, error);
}
op(_STORE_ATTR, (v, owner --)) {
PyObject *name = GETITEM(FRAME_CO_NAMES, oparg);
- int err = PyObject_SetAttr(owner, name, v);
+ int err = PyObject_SetAttr(PyStackRef_AsPyObjectBorrow(owner),
+ name, PyStackRef_AsPyObjectSteal(v));
DECREF_INPUTS();
ERROR_IF(err, error);
}
inst(DELETE_ATTR, (owner --)) {
PyObject *name = GETITEM(FRAME_CO_NAMES, oparg);
- int err = PyObject_DelAttr(owner, name);
+ int err = PyObject_DelAttr(PyStackRef_AsPyObjectBorrow(owner), name);
DECREF_INPUTS();
ERROR_IF(err, error);
}
inst(STORE_GLOBAL, (v --)) {
PyObject *name = GETITEM(FRAME_CO_NAMES, oparg);
- int err = PyDict_SetItem(GLOBALS(), name, v);
+ int err = PyDict_SetItem(GLOBALS(), name, PyStackRef_AsPyObjectBorrow(v));
DECREF_INPUTS();
ERROR_IF(err, error);
}
}
inst(LOAD_LOCALS, ( -- locals)) {
- locals = LOCALS();
- if (locals == NULL) {
+ PyObject *l = LOCALS();
+ if (l == NULL) {
_PyErr_SetString(tstate, PyExc_SystemError,
"no locals found");
ERROR_IF(true, error);
}
- Py_INCREF(locals);
+ locals = PyStackRef_FromPyObjectNew(l);;
}
inst(LOAD_FROM_DICT_OR_GLOBALS, (mod_or_class_dict -- v)) {
PyObject *name = GETITEM(FRAME_CO_NAMES, oparg);
- if (PyMapping_GetOptionalItem(mod_or_class_dict, name, &v) < 0) {
+ PyObject *v_o;
+ if (PyMapping_GetOptionalItem(PyStackRef_AsPyObjectBorrow(mod_or_class_dict), name, &v_o) < 0) {
ERROR_NO_POP();
}
- if (v == NULL) {
+ if (v_o == NULL) {
if (PyDict_CheckExact(GLOBALS())
&& PyDict_CheckExact(BUILTINS()))
{
- v = _PyDict_LoadGlobal((PyDictObject *)GLOBALS(),
+ v_o = _PyDict_LoadGlobal((PyDictObject *)GLOBALS(),
(PyDictObject *)BUILTINS(),
name);
- if (v == NULL) {
+ if (v_o == NULL) {
if (!_PyErr_Occurred(tstate)) {
/* _PyDict_LoadGlobal() returns NULL without raising
* an exception if the key doesn't exist */
else {
/* Slow-path if globals or builtins is not a dict */
/* namespace 1: globals */
- ERROR_IF(PyMapping_GetOptionalItem(GLOBALS(), name, &v) < 0, error);
- if (v == NULL) {
+ ERROR_IF(PyMapping_GetOptionalItem(GLOBALS(), name, &v_o) < 0, error);
+ if (v_o == NULL) {
/* namespace 2: builtins */
- ERROR_IF(PyMapping_GetOptionalItem(BUILTINS(), name, &v) < 0, error);
- if (v == NULL) {
+ ERROR_IF(PyMapping_GetOptionalItem(BUILTINS(), name, &v_o) < 0, error);
+ if (v_o == NULL) {
_PyEval_FormatExcCheckArg(
tstate, PyExc_NameError,
NAME_ERROR_MSG, name);
}
}
DECREF_INPUTS();
+ v = PyStackRef_FromPyObjectSteal(v_o);
}
inst(LOAD_NAME, (-- v)) {
+ PyObject *v_o;
PyObject *mod_or_class_dict = LOCALS();
if (mod_or_class_dict == NULL) {
_PyErr_SetString(tstate, PyExc_SystemError,
ERROR_IF(true, error);
}
PyObject *name = GETITEM(FRAME_CO_NAMES, oparg);
- if (PyMapping_GetOptionalItem(mod_or_class_dict, name, &v) < 0) {
+ if (PyMapping_GetOptionalItem(mod_or_class_dict, name, &v_o) < 0) {
ERROR_NO_POP();
}
- if (v == NULL) {
- if (PyDict_GetItemRef(GLOBALS(), name, &v) < 0) {
+ if (v_o == NULL) {
+ if (PyDict_GetItemRef(GLOBALS(), name, &v_o) < 0) {
ERROR_NO_POP();
}
- if (v == NULL) {
- if (PyMapping_GetOptionalItem(BUILTINS(), name, &v) < 0) {
+ if (v_o == NULL) {
+ if (PyMapping_GetOptionalItem(BUILTINS(), name, &v_o) < 0) {
ERROR_NO_POP();
}
- if (v == NULL) {
+ if (v_o == NULL) {
_PyEval_FormatExcCheckArg(
tstate, PyExc_NameError,
NAME_ERROR_MSG, name);
}
}
}
+ v = PyStackRef_FromPyObjectSteal(v_o);
}
family(LOAD_GLOBAL, INLINE_CACHE_ENTRIES_LOAD_GLOBAL) = {
op(_LOAD_GLOBAL, ( -- res, null if (oparg & 1))) {
PyObject *name = GETITEM(FRAME_CO_NAMES, oparg>>1);
+ PyObject *res_o;
if (PyDict_CheckExact(GLOBALS())
&& PyDict_CheckExact(BUILTINS()))
{
- res = _PyDict_LoadGlobal((PyDictObject *)GLOBALS(),
+ res_o = _PyDict_LoadGlobal((PyDictObject *)GLOBALS(),
(PyDictObject *)BUILTINS(),
name);
- if (res == NULL) {
+ if (res_o == NULL) {
if (!_PyErr_Occurred(tstate)) {
/* _PyDict_LoadGlobal() returns NULL without raising
* an exception if the key doesn't exist */
else {
/* Slow-path if globals or builtins is not a dict */
/* namespace 1: globals */
- ERROR_IF(PyMapping_GetOptionalItem(GLOBALS(), name, &res) < 0, error);
- if (res == NULL) {
+ ERROR_IF(PyMapping_GetOptionalItem(GLOBALS(), name, &res_o) < 0, error);
+ if (res_o == NULL) {
/* namespace 2: builtins */
- ERROR_IF(PyMapping_GetOptionalItem(BUILTINS(), name, &res) < 0, error);
- if (res == NULL) {
+ ERROR_IF(PyMapping_GetOptionalItem(BUILTINS(), name, &res_o) < 0, error);
+ if (res_o == NULL) {
_PyEval_FormatExcCheckArg(
tstate, PyExc_NameError,
NAME_ERROR_MSG, name);
}
}
}
- null = NULL;
+ null = PyStackRef_NULL;
+ res = PyStackRef_FromPyObjectSteal(res_o);
}
macro(LOAD_GLOBAL) =
op(_LOAD_GLOBAL_MODULE, (index/1 -- res, null if (oparg & 1))) {
PyDictObject *dict = (PyDictObject *)GLOBALS();
PyDictUnicodeEntry *entries = DK_UNICODE_ENTRIES(dict->ma_keys);
- res = entries[index].me_value;
- DEOPT_IF(res == NULL);
- Py_INCREF(res);
+ PyObject *res_o = entries[index].me_value;
+ DEOPT_IF(res_o == NULL);
+ Py_INCREF(res_o);
STAT_INC(LOAD_GLOBAL, hit);
- null = NULL;
+ null = PyStackRef_NULL;
+ res = PyStackRef_FromPyObjectSteal(res_o);
}
op(_LOAD_GLOBAL_BUILTINS, (index/1 -- res, null if (oparg & 1))) {
PyDictObject *bdict = (PyDictObject *)BUILTINS();
PyDictUnicodeEntry *entries = DK_UNICODE_ENTRIES(bdict->ma_keys);
- res = entries[index].me_value;
- DEOPT_IF(res == NULL);
- Py_INCREF(res);
+ PyObject *res_o = entries[index].me_value;
+ DEOPT_IF(res_o == NULL);
+ Py_INCREF(res_o);
STAT_INC(LOAD_GLOBAL, hit);
- null = NULL;
+ null = PyStackRef_NULL;
+ res = PyStackRef_FromPyObjectSteal(res_o);
}
macro(LOAD_GLOBAL_MODULE) =
_LOAD_GLOBAL_BUILTINS;
inst(DELETE_FAST, (--)) {
- PyObject *v = GETLOCAL(oparg);
- if (v == NULL) {
+ _PyStackRef v = GETLOCAL(oparg);
+ if (PyStackRef_IsNull(v)) {
_PyEval_FormatExcCheckArg(tstate, PyExc_UnboundLocalError,
UNBOUNDLOCAL_ERROR_MSG,
PyTuple_GetItem(_PyFrame_GetCode(frame)->co_localsplusnames, oparg)
);
ERROR_IF(1, error);
}
- SETLOCAL(oparg, NULL);
+ SETLOCAL(oparg, PyStackRef_NULL);
}
inst(MAKE_CELL, (--)) {
// "initial" is probably NULL but not if it's an arg (or set
// via the f_locals proxy before MAKE_CELL has run).
- PyObject *initial = GETLOCAL(oparg);
+ PyObject *initial = PyStackRef_AsPyObjectBorrow(GETLOCAL(oparg));
PyObject *cell = PyCell_New(initial);
if (cell == NULL) {
ERROR_NO_POP();
}
- SETLOCAL(oparg, cell);
+ SETLOCAL(oparg, PyStackRef_FromPyObjectSteal(cell));
}
inst(DELETE_DEREF, (--)) {
- PyObject *cell = GETLOCAL(oparg);
+ PyObject *cell = PyStackRef_AsPyObjectBorrow(GETLOCAL(oparg));
// Can't use ERROR_IF here.
// Fortunately we don't need its superpower.
PyObject *oldobj = PyCell_SwapTakeRef((PyCellObject *)cell, NULL);
Py_DECREF(oldobj);
}
- inst(LOAD_FROM_DICT_OR_DEREF, (class_dict -- value)) {
+ inst(LOAD_FROM_DICT_OR_DEREF, (class_dict_st -- value)) {
+ PyObject *value_o;
PyObject *name;
+ PyObject *class_dict = PyStackRef_AsPyObjectBorrow(class_dict_st);
+
assert(class_dict);
assert(oparg >= 0 && oparg < _PyFrame_GetCode(frame)->co_nlocalsplus);
name = PyTuple_GET_ITEM(_PyFrame_GetCode(frame)->co_localsplusnames, oparg);
- if (PyMapping_GetOptionalItem(class_dict, name, &value) < 0) {
+ if (PyMapping_GetOptionalItem(class_dict, name, &value_o) < 0) {
ERROR_NO_POP();
}
- if (!value) {
- PyCellObject *cell = (PyCellObject *)GETLOCAL(oparg);
- value = PyCell_GetRef(cell);
- if (value == NULL) {
+ if (!value_o) {
+ PyCellObject *cell = (PyCellObject *)PyStackRef_AsPyObjectBorrow(GETLOCAL(oparg));
+ value_o = PyCell_GetRef(cell);
+ if (value_o == NULL) {
_PyEval_FormatExcUnbound(tstate, _PyFrame_GetCode(frame), oparg);
ERROR_NO_POP();
}
}
- Py_DECREF(class_dict);
+ PyStackRef_CLOSE(class_dict_st);
+ value = PyStackRef_FromPyObjectSteal(value_o);
}
inst(LOAD_DEREF, ( -- value)) {
- PyCellObject *cell = (PyCellObject *)GETLOCAL(oparg);
- value = PyCell_GetRef(cell);
- if (value == NULL) {
+ PyCellObject *cell = (PyCellObject *)PyStackRef_AsPyObjectBorrow(GETLOCAL(oparg));
+ PyObject *value_o = PyCell_GetRef(cell);
+ if (value_o == NULL) {
_PyEval_FormatExcUnbound(tstate, _PyFrame_GetCode(frame), oparg);
ERROR_IF(true, error);
}
+ value = PyStackRef_FromPyObjectSteal(value_o);
}
inst(STORE_DEREF, (v --)) {
- PyCellObject *cell = (PyCellObject *)GETLOCAL(oparg);
- PyCell_SetTakeRef(cell, v);
+ PyCellObject *cell = (PyCellObject *)PyStackRef_AsPyObjectBorrow(GETLOCAL(oparg));
+ PyCell_SetTakeRef(cell, PyStackRef_AsPyObjectSteal(v));
}
inst(COPY_FREE_VARS, (--)) {
int offset = co->co_nlocalsplus - oparg;
for (int i = 0; i < oparg; ++i) {
PyObject *o = PyTuple_GET_ITEM(closure, i);
- frame->localsplus[offset + i] = Py_NewRef(o);
+ frame->localsplus[offset + i] = PyStackRef_FromPyObjectNew(o);
}
}
inst(BUILD_STRING, (pieces[oparg] -- str)) {
- str = _PyUnicode_JoinArray(&_Py_STR(empty), pieces, oparg);
+ STACKREFS_TO_PYOBJECTS(pieces, oparg, pieces_o);
+ if (CONVERSION_FAILED(pieces_o)) {
+ DECREF_INPUTS();
+ ERROR_IF(true, error);
+ }
+ PyObject *str_o = _PyUnicode_JoinArray(&_Py_STR(empty), pieces_o, oparg);
+ STACKREFS_TO_PYOBJECTS_CLEANUP(pieces_o);
DECREF_INPUTS();
- ERROR_IF(str == NULL, error);
+ ERROR_IF(str_o == NULL, error);
+ str = PyStackRef_FromPyObjectSteal(str_o);
}
inst(BUILD_TUPLE, (values[oparg] -- tup)) {
- tup = _PyTuple_FromArraySteal(values, oparg);
- ERROR_IF(tup == NULL, error);
+ STACKREFS_TO_PYOBJECTS(values, oparg, values_o);
+ if (CONVERSION_FAILED(values_o)) {
+ DECREF_INPUTS();
+ ERROR_IF(true, error);
+ }
+ PyObject *tup_o = _PyTuple_FromArraySteal(values_o, oparg);
+ STACKREFS_TO_PYOBJECTS_CLEANUP(values_o);
+ ERROR_IF(tup_o == NULL, error);
+ tup = PyStackRef_FromPyObjectSteal(tup_o);
}
inst(BUILD_LIST, (values[oparg] -- list)) {
- list = _PyList_FromArraySteal(values, oparg);
- ERROR_IF(list == NULL, error);
+ STACKREFS_TO_PYOBJECTS(values, oparg, values_o);
+ if (CONVERSION_FAILED(values_o)) {
+ DECREF_INPUTS();
+ ERROR_IF(true, error);
+ }
+ PyObject *list_o = _PyList_FromArraySteal(values_o, oparg);
+ STACKREFS_TO_PYOBJECTS_CLEANUP(values_o);
+ ERROR_IF(list_o == NULL, error);
+ list = PyStackRef_FromPyObjectSteal(list_o);
}
- inst(LIST_EXTEND, (list, unused[oparg-1], iterable -- list, unused[oparg-1])) {
+ inst(LIST_EXTEND, (list_st, unused[oparg-1], iterable_st -- list_st, unused[oparg-1])) {
+ PyObject *list = PyStackRef_AsPyObjectBorrow(list_st);
+ PyObject *iterable = PyStackRef_AsPyObjectBorrow(iterable_st);
+
PyObject *none_val = _PyList_Extend((PyListObject *)list, iterable);
if (none_val == NULL) {
if (_PyErr_ExceptionMatches(tstate, PyExc_TypeError) &&
}
inst(SET_UPDATE, (set, unused[oparg-1], iterable -- set, unused[oparg-1])) {
- int err = _PySet_Update(set, iterable);
+ int err = _PySet_Update(PyStackRef_AsPyObjectBorrow(set),
+ PyStackRef_AsPyObjectBorrow(iterable));
DECREF_INPUTS();
ERROR_IF(err < 0, error);
}
inst(BUILD_SET, (values[oparg] -- set)) {
- set = PySet_New(NULL);
- if (set == NULL)
+ PyObject *set_o = PySet_New(NULL);
+ if (set_o == NULL) {
ERROR_NO_POP();
+ }
int err = 0;
for (int i = 0; i < oparg; i++) {
- PyObject *item = values[i];
- if (err == 0)
- err = PySet_Add(set, item);
+ PyObject *item = PyStackRef_AsPyObjectSteal(values[i]);
+ if (err == 0) {
+ err = PySet_Add(set_o, item);
+ }
Py_DECREF(item);
}
if (err != 0) {
- Py_DECREF(set);
+ Py_DECREF(set_o);
ERROR_IF(true, error);
}
+ set = PyStackRef_FromPyObjectSteal(set_o);
}
inst(BUILD_MAP, (values[oparg*2] -- map)) {
- map = _PyDict_FromItems(
- values, 2,
- values+1, 2,
+ STACKREFS_TO_PYOBJECTS(values, oparg*2, values_o);
+ if (CONVERSION_FAILED(values_o)) {
+ DECREF_INPUTS();
+ ERROR_IF(true, error);
+ }
+ PyObject *map_o = _PyDict_FromItems(
+ values_o, 2,
+ values_o+1, 2,
oparg);
+ STACKREFS_TO_PYOBJECTS_CLEANUP(values_o);
DECREF_INPUTS();
- ERROR_IF(map == NULL, error);
+ ERROR_IF(map_o == NULL, error);
+ map = PyStackRef_FromPyObjectSteal(map_o);
}
inst(SETUP_ANNOTATIONS, (--)) {
}
inst(BUILD_CONST_KEY_MAP, (values[oparg], keys -- map)) {
- assert(PyTuple_CheckExact(keys));
- assert(PyTuple_GET_SIZE(keys) == (Py_ssize_t)oparg);
- map = _PyDict_FromItems(
- &PyTuple_GET_ITEM(keys, 0), 1,
- values, 1, oparg);
+ PyObject *keys_o = PyStackRef_AsPyObjectBorrow(keys);
+
+ assert(PyTuple_CheckExact(keys_o));
+ assert(PyTuple_GET_SIZE(keys_o) == (Py_ssize_t)oparg);
+ STACKREFS_TO_PYOBJECTS(values, oparg, values_o);
+ if (CONVERSION_FAILED(values_o)) {
+ DECREF_INPUTS();
+ ERROR_IF(true, error);
+ }
+ PyObject *map_o = _PyDict_FromItems(
+ &PyTuple_GET_ITEM(keys_o, 0), 1,
+ values_o, 1, oparg);
+ STACKREFS_TO_PYOBJECTS_CLEANUP(values_o);
DECREF_INPUTS();
- ERROR_IF(map == NULL, error);
+ ERROR_IF(map_o == NULL, error);
+ map = PyStackRef_FromPyObjectSteal(map_o);
}
inst(DICT_UPDATE, (dict, unused[oparg - 1], update -- dict, unused[oparg - 1])) {
- if (PyDict_Update(dict, update) < 0) {
+ PyObject *dict_o = PyStackRef_AsPyObjectBorrow(dict);
+ PyObject *update_o = PyStackRef_AsPyObjectBorrow(update);
+
+ if (PyDict_Update(dict_o, update_o) < 0) {
if (_PyErr_ExceptionMatches(tstate, PyExc_AttributeError)) {
_PyErr_Format(tstate, PyExc_TypeError,
"'%.200s' object is not a mapping",
- Py_TYPE(update)->tp_name);
+ Py_TYPE(update_o)->tp_name);
}
DECREF_INPUTS();
ERROR_IF(true, error);
}
inst(DICT_MERGE, (callable, unused, unused, dict, unused[oparg - 1], update -- callable, unused, unused, dict, unused[oparg - 1])) {
- if (_PyDict_MergeEx(dict, update, 2) < 0) {
- _PyEval_FormatKwargsError(tstate, callable, update);
+ PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable);
+ PyObject *dict_o = PyStackRef_AsPyObjectBorrow(dict);
+ PyObject *update_o = PyStackRef_AsPyObjectBorrow(update);
+
+ if (_PyDict_MergeEx(dict_o, update_o, 2) < 0) {
+ _PyEval_FormatKwargsError(tstate, callable_o, update_o);
DECREF_INPUTS();
ERROR_IF(true, error);
}
DECREF_INPUTS();
}
- inst(MAP_ADD, (dict, unused[oparg - 1], key, value -- dict, unused[oparg - 1])) {
+ inst(MAP_ADD, (dict_st, unused[oparg - 1], key, value -- dict_st, unused[oparg - 1])) {
+ PyObject *dict = PyStackRef_AsPyObjectBorrow(dict_st);
assert(PyDict_CheckExact(dict));
/* dict[key] = value */
// Do not DECREF INPUTS because the function steals the references
- ERROR_IF(_PyDict_SetItem_Take2((PyDictObject *)dict, key, value) != 0, error);
+ ERROR_IF(_PyDict_SetItem_Take2((PyDictObject *)dict, PyStackRef_AsPyObjectSteal(key), PyStackRef_AsPyObjectSteal(value)) != 0, error);
}
inst(INSTRUMENTED_LOAD_SUPER_ATTR, (unused/1, unused, unused, unused -- unused, unused if (oparg & 1))) {
LOAD_SUPER_ATTR_METHOD,
};
- specializing op(_SPECIALIZE_LOAD_SUPER_ATTR, (counter/1, global_super, class, unused -- global_super, class, unused)) {
+ specializing op(_SPECIALIZE_LOAD_SUPER_ATTR, (counter/1, global_super_st, class_st, unused -- global_super_st, class_st, unused)) {
#if ENABLE_SPECIALIZATION
int load_method = oparg & 1;
if (ADAPTIVE_COUNTER_TRIGGERS(counter)) {
next_instr = this_instr;
- _Py_Specialize_LoadSuperAttr(global_super, class, next_instr, load_method);
+ _Py_Specialize_LoadSuperAttr(global_super_st, class_st, next_instr, load_method);
DISPATCH_SAME_OPARG();
}
STAT_INC(LOAD_SUPER_ATTR, deferred);
#endif /* ENABLE_SPECIALIZATION */
}
- tier1 op(_LOAD_SUPER_ATTR, (global_super, class, self -- attr, null if (oparg & 1))) {
+ tier1 op(_LOAD_SUPER_ATTR, (global_super_st, class_st, self_st -- attr, null if (oparg & 1))) {
+ PyObject *global_super = PyStackRef_AsPyObjectBorrow(global_super_st);
+ PyObject *class = PyStackRef_AsPyObjectBorrow(class_st);
+ PyObject *self = PyStackRef_AsPyObjectBorrow(self_st);
+
if (opcode == INSTRUMENTED_LOAD_SUPER_ATTR) {
PyObject *arg = oparg & 2 ? class : &_PyInstrumentation_MISSING;
int err = _Py_call_instrumentation_2args(
DECREF_INPUTS();
ERROR_IF(super == NULL, error);
PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 2);
- attr = PyObject_GetAttr(super, name);
+ attr = PyStackRef_FromPyObjectSteal(PyObject_GetAttr(super, name));
Py_DECREF(super);
- ERROR_IF(attr == NULL, error);
- null = NULL;
+ ERROR_IF(PyStackRef_IsNull(attr), error);
+ null = PyStackRef_NULL;
}
macro(LOAD_SUPER_ATTR) = _SPECIALIZE_LOAD_SUPER_ATTR + _LOAD_SUPER_ATTR;
- inst(LOAD_SUPER_ATTR_ATTR, (unused/1, global_super, class, self -- attr, unused if (0))) {
+ inst(LOAD_SUPER_ATTR_ATTR, (unused/1, global_super_st, class_st, self_st -- attr_st, unused if (0))) {
+ PyObject *global_super = PyStackRef_AsPyObjectBorrow(global_super_st);
+ PyObject *class = PyStackRef_AsPyObjectBorrow(class_st);
+ PyObject *self = PyStackRef_AsPyObjectBorrow(self_st);
+
assert(!(oparg & 1));
DEOPT_IF(global_super != (PyObject *)&PySuper_Type);
DEOPT_IF(!PyType_Check(class));
STAT_INC(LOAD_SUPER_ATTR, hit);
PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 2);
- attr = _PySuper_Lookup((PyTypeObject *)class, self, name, NULL);
+ PyObject *attr = _PySuper_Lookup((PyTypeObject *)class, self, name, NULL);
DECREF_INPUTS();
ERROR_IF(attr == NULL, error);
+ attr_st = PyStackRef_FromPyObjectSteal(attr);
}
- inst(LOAD_SUPER_ATTR_METHOD, (unused/1, global_super, class, self -- attr, self_or_null)) {
+ inst(LOAD_SUPER_ATTR_METHOD, (unused/1, global_super_st, class_st, self_st -- attr, self_or_null)) {
+ PyObject *global_super = PyStackRef_AsPyObjectBorrow(global_super_st);
+ PyObject *class = PyStackRef_AsPyObjectBorrow(class_st);
+ PyObject *self = PyStackRef_AsPyObjectBorrow(self_st);
+
assert(oparg & 1);
DEOPT_IF(global_super != (PyObject *)&PySuper_Type);
DEOPT_IF(!PyType_Check(class));
PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 2);
PyTypeObject *cls = (PyTypeObject *)class;
int method_found = 0;
- attr = _PySuper_Lookup(cls, self, name,
+ PyObject *attr_o = _PySuper_Lookup(cls, self, name,
Py_TYPE(self)->tp_getattro == PyObject_GenericGetAttr ? &method_found : NULL);
- Py_DECREF(global_super);
- Py_DECREF(class);
- if (attr == NULL) {
- Py_DECREF(self);
+ PyStackRef_CLOSE(global_super_st);
+ PyStackRef_CLOSE(class_st);
+ if (attr_o == NULL) {
+ PyStackRef_CLOSE(self_st);
ERROR_IF(true, error);
}
if (method_found) {
- self_or_null = self; // transfer ownership
+ self_or_null = self_st; // transfer ownership
} else {
- Py_DECREF(self);
- self_or_null = NULL;
+ PyStackRef_CLOSE(self_st);
+ self_or_null = PyStackRef_NULL;
}
+
+ attr = PyStackRef_FromPyObjectSteal(attr_o);
}
family(LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR) = {
op(_LOAD_ATTR, (owner -- attr, self_or_null if (oparg & 1))) {
PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 1);
+ PyObject *attr_o;
if (oparg & 1) {
/* Designed to work in tandem with CALL, pushes two values. */
- attr = NULL;
- if (_PyObject_GetMethod(owner, name, &attr)) {
+ attr_o = NULL;
+ if (_PyObject_GetMethod(PyStackRef_AsPyObjectBorrow(owner), name, &attr_o)) {
/* We can bypass temporary bound method object.
meth is unbound method and obj is self.
meth | self | arg1 | ... | argN
*/
- assert(attr != NULL); // No errors on this branch
+ assert(attr_o != NULL); // No errors on this branch
self_or_null = owner; // Transfer ownership
}
else {
meth | NULL | arg1 | ... | argN
*/
DECREF_INPUTS();
- ERROR_IF(attr == NULL, error);
- self_or_null = NULL;
+ ERROR_IF(attr_o == NULL, error);
+ self_or_null = PyStackRef_NULL;
}
}
else {
/* Classic, pushes one value. */
- attr = PyObject_GetAttr(owner, name);
+ attr_o = PyObject_GetAttr(PyStackRef_AsPyObjectBorrow(owner), name);
DECREF_INPUTS();
- ERROR_IF(attr == NULL, error);
+ ERROR_IF(attr_o == NULL, error);
}
+ attr = PyStackRef_FromPyObjectSteal(attr_o);
}
macro(LOAD_ATTR) =
_LOAD_ATTR;
op(_GUARD_TYPE_VERSION, (type_version/2, owner -- owner)) {
- PyTypeObject *tp = Py_TYPE(owner);
+ PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner));
assert(type_version != 0);
EXIT_IF(tp->tp_version_tag != type_version);
}
op(_CHECK_MANAGED_OBJECT_HAS_VALUES, (owner -- owner)) {
- assert(Py_TYPE(owner)->tp_dictoffset < 0);
- assert(Py_TYPE(owner)->tp_flags & Py_TPFLAGS_INLINE_VALUES);
- DEOPT_IF(!_PyObject_InlineValues(owner)->valid);
+ PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner);
+ assert(Py_TYPE(owner_o)->tp_dictoffset < 0);
+ assert(Py_TYPE(owner_o)->tp_flags & Py_TPFLAGS_INLINE_VALUES);
+ DEOPT_IF(!_PyObject_InlineValues(owner_o)->valid);
}
split op(_LOAD_ATTR_INSTANCE_VALUE, (index/1, owner -- attr, null if (oparg & 1))) {
- attr = _PyObject_InlineValues(owner)->values[index];
- DEOPT_IF(attr == NULL);
+ PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner);
+ PyObject *attr_o = _PyObject_InlineValues(owner_o)->values[index];
+ DEOPT_IF(attr_o == NULL);
STAT_INC(LOAD_ATTR, hit);
- Py_INCREF(attr);
- null = NULL;
+ Py_INCREF(attr_o);
+ null = PyStackRef_NULL;
+ attr = PyStackRef_FromPyObjectSteal(attr_o);
DECREF_INPUTS();
}
unused/5; // Skip over rest of cache
op(_CHECK_ATTR_MODULE, (dict_version/2, owner -- owner)) {
- DEOPT_IF(!PyModule_CheckExact(owner));
- PyDictObject *dict = (PyDictObject *)((PyModuleObject *)owner)->md_dict;
+ PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner);
+ DEOPT_IF(!PyModule_CheckExact(owner_o));
+ PyDictObject *dict = (PyDictObject *)((PyModuleObject *)owner_o)->md_dict;
assert(dict != NULL);
DEOPT_IF(dict->ma_keys->dk_version != dict_version);
}
op(_LOAD_ATTR_MODULE, (index/1, owner -- attr, null if (oparg & 1))) {
- PyDictObject *dict = (PyDictObject *)((PyModuleObject *)owner)->md_dict;
+ PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner);
+ PyDictObject *dict = (PyDictObject *)((PyModuleObject *)owner_o)->md_dict;
assert(dict->ma_keys->dk_kind == DICT_KEYS_UNICODE);
assert(index < dict->ma_keys->dk_nentries);
PyDictUnicodeEntry *ep = DK_UNICODE_ENTRIES(dict->ma_keys) + index;
- attr = ep->me_value;
- DEOPT_IF(attr == NULL);
+ PyObject *attr_o = ep->me_value;
+ DEOPT_IF(attr_o == NULL);
STAT_INC(LOAD_ATTR, hit);
- Py_INCREF(attr);
- null = NULL;
+ Py_INCREF(attr_o);
+ attr = PyStackRef_FromPyObjectSteal(attr_o);
+ null = PyStackRef_NULL;
DECREF_INPUTS();
}
unused/5;
op(_CHECK_ATTR_WITH_HINT, (owner -- owner)) {
- assert(Py_TYPE(owner)->tp_flags & Py_TPFLAGS_MANAGED_DICT);
- PyDictObject *dict = _PyObject_GetManagedDict(owner);
+ PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner);
+
+ assert(Py_TYPE(owner_o)->tp_flags & Py_TPFLAGS_MANAGED_DICT);
+ PyDictObject *dict = _PyObject_GetManagedDict(owner_o);
DEOPT_IF(dict == NULL);
assert(PyDict_CheckExact((PyObject *)dict));
}
op(_LOAD_ATTR_WITH_HINT, (hint/1, owner -- attr, null if (oparg & 1))) {
- PyDictObject *dict = _PyObject_GetManagedDict(owner);
+ PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner);
+ PyObject *attr_o;
+
+ PyDictObject *dict = _PyObject_GetManagedDict(owner_o);
DEOPT_IF(hint >= (size_t)dict->ma_keys->dk_nentries);
PyObject *name = GETITEM(FRAME_CO_NAMES, oparg>>1);
if (DK_IS_UNICODE(dict->ma_keys)) {
PyDictUnicodeEntry *ep = DK_UNICODE_ENTRIES(dict->ma_keys) + hint;
DEOPT_IF(ep->me_key != name);
- attr = ep->me_value;
+ attr_o = ep->me_value;
}
else {
PyDictKeyEntry *ep = DK_ENTRIES(dict->ma_keys) + hint;
DEOPT_IF(ep->me_key != name);
- attr = ep->me_value;
+ attr_o = ep->me_value;
}
- DEOPT_IF(attr == NULL);
+ DEOPT_IF(attr_o == NULL);
STAT_INC(LOAD_ATTR, hit);
- Py_INCREF(attr);
- null = NULL;
+ Py_INCREF(attr_o);
+ attr = PyStackRef_FromPyObjectSteal(attr_o);
+ null = PyStackRef_NULL;
DECREF_INPUTS();
}
unused/5;
split op(_LOAD_ATTR_SLOT, (index/1, owner -- attr, null if (oparg & 1))) {
- char *addr = (char *)owner + index;
- attr = *(PyObject **)addr;
- DEOPT_IF(attr == NULL);
+ PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner);
+
+ char *addr = (char *)owner_o + index;
+ PyObject *attr_o = *(PyObject **)addr;
+ DEOPT_IF(attr_o == NULL);
STAT_INC(LOAD_ATTR, hit);
- Py_INCREF(attr);
- null = NULL;
+ null = PyStackRef_NULL;
+ attr = PyStackRef_FromPyObjectNew(attr_o);
DECREF_INPUTS();
}
unused/5;
op(_CHECK_ATTR_CLASS, (type_version/2, owner -- owner)) {
- DEOPT_IF(!PyType_Check(owner));
+ PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner);
+
+ DEOPT_IF(!PyType_Check(owner_o));
assert(type_version != 0);
- DEOPT_IF(((PyTypeObject *)owner)->tp_version_tag != type_version);
+ DEOPT_IF(((PyTypeObject *)owner_o)->tp_version_tag != type_version);
}
split op(_LOAD_ATTR_CLASS, (descr/4, owner -- attr, null if (oparg & 1))) {
STAT_INC(LOAD_ATTR, hit);
assert(descr != NULL);
- attr = Py_NewRef(descr);
- null = NULL;
+ attr = PyStackRef_FromPyObjectNew(descr);
+ null = PyStackRef_NULL;
DECREF_INPUTS();
}
_LOAD_ATTR_CLASS;
inst(LOAD_ATTR_PROPERTY, (unused/1, type_version/2, func_version/2, fget/4, owner -- unused, unused if (0))) {
+ PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner);
+
assert((oparg & 1) == 0);
DEOPT_IF(tstate->interp->eval_frame);
- PyTypeObject *cls = Py_TYPE(owner);
+ PyTypeObject *cls = Py_TYPE(owner_o);
assert(type_version != 0);
DEOPT_IF(cls->tp_version_tag != type_version);
assert(Py_IS_TYPE(fget, &PyFunction_Type));
}
inst(LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN, (unused/1, type_version/2, func_version/2, getattribute/4, owner -- unused, unused if (0))) {
+ PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner);
+
assert((oparg & 1) == 0);
DEOPT_IF(tstate->interp->eval_frame);
- PyTypeObject *cls = Py_TYPE(owner);
+ PyTypeObject *cls = Py_TYPE(owner_o);
assert(type_version != 0);
DEOPT_IF(cls->tp_version_tag != type_version);
assert(Py_IS_TYPE(getattribute, &PyFunction_Type));
// Manipulate stack directly because we exit with DISPATCH_INLINED().
STACK_SHRINK(1);
new_frame->localsplus[0] = owner;
- new_frame->localsplus[1] = Py_NewRef(name);
+ new_frame->localsplus[1] = PyStackRef_FromPyObjectNew(name);
frame->return_offset = (uint16_t)(next_instr - this_instr);
DISPATCH_INLINED(new_frame);
}
op(_GUARD_DORV_NO_DICT, (owner -- owner)) {
- assert(Py_TYPE(owner)->tp_dictoffset < 0);
- assert(Py_TYPE(owner)->tp_flags & Py_TPFLAGS_INLINE_VALUES);
- DEOPT_IF(_PyObject_GetManagedDict(owner));
- DEOPT_IF(_PyObject_InlineValues(owner)->valid == 0);
+ PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner);
+
+ assert(Py_TYPE(owner_o)->tp_dictoffset < 0);
+ assert(Py_TYPE(owner_o)->tp_flags & Py_TPFLAGS_INLINE_VALUES);
+ DEOPT_IF(_PyObject_GetManagedDict(owner_o));
+ DEOPT_IF(_PyObject_InlineValues(owner_o)->valid == 0);
}
op(_STORE_ATTR_INSTANCE_VALUE, (index/1, value, owner --)) {
+ PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner);
+
STAT_INC(STORE_ATTR, hit);
- assert(_PyObject_GetManagedDict(owner) == NULL);
- PyDictValues *values = _PyObject_InlineValues(owner);
+ assert(_PyObject_GetManagedDict(owner_o) == NULL);
+ PyDictValues *values = _PyObject_InlineValues(owner_o);
PyObject *old_value = values->values[index];
- values->values[index] = value;
+ values->values[index] = PyStackRef_AsPyObjectSteal(value);
if (old_value == NULL) {
_PyDictValues_AddToInsertionOrder(values, index);
}
else {
Py_DECREF(old_value);
}
-
- Py_DECREF(owner);
+ PyStackRef_CLOSE(owner);
}
macro(STORE_ATTR_INSTANCE_VALUE) =
_STORE_ATTR_INSTANCE_VALUE;
op(_STORE_ATTR_WITH_HINT, (hint/1, value, owner --)) {
- assert(Py_TYPE(owner)->tp_flags & Py_TPFLAGS_MANAGED_DICT);
- PyDictObject *dict = _PyObject_GetManagedDict(owner);
+ PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner);
+ assert(Py_TYPE(owner_o)->tp_flags & Py_TPFLAGS_MANAGED_DICT);
+ PyDictObject *dict = _PyObject_GetManagedDict(owner_o);
DEOPT_IF(dict == NULL);
assert(PyDict_CheckExact((PyObject *)dict));
PyObject *name = GETITEM(FRAME_CO_NAMES, oparg);
DEOPT_IF(ep->me_key != name);
old_value = ep->me_value;
DEOPT_IF(old_value == NULL);
- new_version = _PyDict_NotifyEvent(tstate->interp, PyDict_EVENT_MODIFIED, dict, name, value);
- ep->me_value = value;
+ new_version = _PyDict_NotifyEvent(tstate->interp, PyDict_EVENT_MODIFIED, dict, name, PyStackRef_AsPyObjectBorrow(value));
+ ep->me_value = PyStackRef_AsPyObjectSteal(value);
}
else {
PyDictKeyEntry *ep = DK_ENTRIES(dict->ma_keys) + hint;
DEOPT_IF(ep->me_key != name);
old_value = ep->me_value;
DEOPT_IF(old_value == NULL);
- new_version = _PyDict_NotifyEvent(tstate->interp, PyDict_EVENT_MODIFIED, dict, name, value);
- ep->me_value = value;
+ new_version = _PyDict_NotifyEvent(tstate->interp, PyDict_EVENT_MODIFIED, dict, name, PyStackRef_AsPyObjectBorrow(value));
+ ep->me_value = PyStackRef_AsPyObjectSteal(value);
}
Py_DECREF(old_value);
STAT_INC(STORE_ATTR, hit);
/* Ensure dict is GC tracked if it needs to be */
- if (!_PyObject_GC_IS_TRACKED(dict) && _PyObject_GC_MAY_BE_TRACKED(value)) {
+ if (!_PyObject_GC_IS_TRACKED(dict) && _PyObject_GC_MAY_BE_TRACKED(PyStackRef_AsPyObjectBorrow(value))) {
_PyObject_GC_TRACK(dict);
}
/* PEP 509 */
dict->ma_version_tag = new_version;
- Py_DECREF(owner);
+ PyStackRef_CLOSE(owner);
}
macro(STORE_ATTR_WITH_HINT) =
_STORE_ATTR_WITH_HINT;
op(_STORE_ATTR_SLOT, (index/1, value, owner --)) {
- char *addr = (char *)owner + index;
+ PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner);
+
+ char *addr = (char *)owner_o + index;
STAT_INC(STORE_ATTR, hit);
PyObject *old_value = *(PyObject **)addr;
- *(PyObject **)addr = value;
+ *(PyObject **)addr = PyStackRef_AsPyObjectSteal(value);
Py_XDECREF(old_value);
- Py_DECREF(owner);
+ PyStackRef_CLOSE(owner);
}
macro(STORE_ATTR_SLOT) =
}
op(_COMPARE_OP, (left, right -- res)) {
+ PyObject *left_o = PyStackRef_AsPyObjectBorrow(left);
+ PyObject *right_o = PyStackRef_AsPyObjectBorrow(right);
+
assert((oparg >> 5) <= Py_GE);
- res = PyObject_RichCompare(left, right, oparg >> 5);
+ PyObject *res_o = PyObject_RichCompare(left_o, right_o, oparg >> 5);
DECREF_INPUTS();
- ERROR_IF(res == NULL, error);
+ ERROR_IF(res_o == NULL, error);
if (oparg & 16) {
- int res_bool = PyObject_IsTrue(res);
- Py_DECREF(res);
+ int res_bool = PyObject_IsTrue(res_o);
+ Py_DECREF(res_o);
ERROR_IF(res_bool < 0, error);
- res = res_bool ? Py_True : Py_False;
+ res = res_bool ? PyStackRef_True : PyStackRef_False;
+ }
+ else {
+ res = PyStackRef_FromPyObjectSteal(res_o);
}
}
_GUARD_BOTH_UNICODE + unused/1 + _COMPARE_OP_STR;
op(_COMPARE_OP_FLOAT, (left, right -- res)) {
+ PyObject *left_o = PyStackRef_AsPyObjectBorrow(left);
+ PyObject *right_o = PyStackRef_AsPyObjectBorrow(right);
+
STAT_INC(COMPARE_OP, hit);
- double dleft = PyFloat_AS_DOUBLE(left);
- double dright = PyFloat_AS_DOUBLE(right);
+ double dleft = PyFloat_AS_DOUBLE(left_o);
+ double dright = PyFloat_AS_DOUBLE(right_o);
// 1 if NaN, 2 if <, 4 if >, 8 if ==; this matches low four bits of the oparg
int sign_ish = COMPARISON_BIT(dleft, dright);
- _Py_DECREF_SPECIALIZED(left, _PyFloat_ExactDealloc);
- _Py_DECREF_SPECIALIZED(right, _PyFloat_ExactDealloc);
- res = (sign_ish & oparg) ? Py_True : Py_False;
+ _Py_DECREF_SPECIALIZED(left_o, _PyFloat_ExactDealloc);
+ _Py_DECREF_SPECIALIZED(right_o, _PyFloat_ExactDealloc);
+ res = (sign_ish & oparg) ? PyStackRef_True : PyStackRef_False;
// It's always a bool, so we don't care about oparg & 16.
}
// Similar to COMPARE_OP_FLOAT
op(_COMPARE_OP_INT, (left, right -- res)) {
- DEOPT_IF(!_PyLong_IsCompact((PyLongObject *)left));
- DEOPT_IF(!_PyLong_IsCompact((PyLongObject *)right));
+ PyObject *left_o = PyStackRef_AsPyObjectBorrow(left);
+ PyObject *right_o = PyStackRef_AsPyObjectBorrow(right);
+
+ DEOPT_IF(!_PyLong_IsCompact((PyLongObject *)left_o));
+ DEOPT_IF(!_PyLong_IsCompact((PyLongObject *)right_o));
STAT_INC(COMPARE_OP, hit);
- assert(_PyLong_DigitCount((PyLongObject *)left) <= 1 &&
- _PyLong_DigitCount((PyLongObject *)right) <= 1);
- Py_ssize_t ileft = _PyLong_CompactValue((PyLongObject *)left);
- Py_ssize_t iright = _PyLong_CompactValue((PyLongObject *)right);
+ assert(_PyLong_DigitCount((PyLongObject *)left_o) <= 1 &&
+ _PyLong_DigitCount((PyLongObject *)right_o) <= 1);
+ Py_ssize_t ileft = _PyLong_CompactValue((PyLongObject *)left_o);
+ Py_ssize_t iright = _PyLong_CompactValue((PyLongObject *)right_o);
// 2 if <, 4 if >, 8 if ==; this matches the low 4 bits of the oparg
int sign_ish = COMPARISON_BIT(ileft, iright);
- _Py_DECREF_SPECIALIZED(left, (destructor)PyObject_Free);
- _Py_DECREF_SPECIALIZED(right, (destructor)PyObject_Free);
- res = (sign_ish & oparg) ? Py_True : Py_False;
+ _Py_DECREF_SPECIALIZED(left_o, (destructor)PyObject_Free);
+ _Py_DECREF_SPECIALIZED(right_o, (destructor)PyObject_Free);
+ res = (sign_ish & oparg) ? PyStackRef_True : PyStackRef_False;
// It's always a bool, so we don't care about oparg & 16.
}
// Similar to COMPARE_OP_FLOAT, but for ==, != only
op(_COMPARE_OP_STR, (left, right -- res)) {
+ PyObject *left_o = PyStackRef_AsPyObjectBorrow(left);
+ PyObject *right_o = PyStackRef_AsPyObjectBorrow(right);
+
STAT_INC(COMPARE_OP, hit);
- int eq = _PyUnicode_Equal(left, right);
+ int eq = _PyUnicode_Equal(left_o, right_o);
assert((oparg >> 5) == Py_EQ || (oparg >> 5) == Py_NE);
- _Py_DECREF_SPECIALIZED(left, _PyUnicode_ExactDealloc);
- _Py_DECREF_SPECIALIZED(right, _PyUnicode_ExactDealloc);
+ _Py_DECREF_SPECIALIZED(left_o, _PyUnicode_ExactDealloc);
+ _Py_DECREF_SPECIALIZED(right_o, _PyUnicode_ExactDealloc);
assert(eq == 0 || eq == 1);
assert((oparg & 0xf) == COMPARISON_NOT_EQUALS || (oparg & 0xf) == COMPARISON_EQUALS);
assert(COMPARISON_NOT_EQUALS + 1 == COMPARISON_EQUALS);
- res = ((COMPARISON_NOT_EQUALS + eq) & oparg) ? Py_True : Py_False;
+ res = ((COMPARISON_NOT_EQUALS + eq) & oparg) ? PyStackRef_True : PyStackRef_False;
// It's always a bool, so we don't care about oparg & 16.
}
inst(IS_OP, (left, right -- b)) {
- int res = Py_Is(left, right) ^ oparg;
+#ifdef Py_GIL_DISABLED
+ // On free-threaded builds, objects are conditionally immortalized.
+ // So their bits don't always compare equally.
+ int res = Py_Is(PyStackRef_AsPyObjectBorrow(left), PyStackRef_AsPyObjectBorrow(right)) ^ oparg;
+#else
+ int res = PyStackRef_Is(left, right) ^ oparg;
+#endif
DECREF_INPUTS();
- b = res ? Py_True : Py_False;
+ b = res ? PyStackRef_True : PyStackRef_False;
}
family(CONTAINS_OP, INLINE_CACHE_ENTRIES_CONTAINS_OP) = {
};
op(_CONTAINS_OP, (left, right -- b)) {
- int res = PySequence_Contains(right, left);
+ PyObject *left_o = PyStackRef_AsPyObjectBorrow(left);
+ PyObject *right_o = PyStackRef_AsPyObjectBorrow(right);
+
+ int res = PySequence_Contains(right_o, left_o);
DECREF_INPUTS();
ERROR_IF(res < 0, error);
- b = (res ^ oparg) ? Py_True : Py_False;
+ b = (res ^ oparg) ? PyStackRef_True : PyStackRef_False;
}
specializing op(_SPECIALIZE_CONTAINS_OP, (counter/1, left, right -- left, right)) {
macro(CONTAINS_OP) = _SPECIALIZE_CONTAINS_OP + _CONTAINS_OP;
inst(CONTAINS_OP_SET, (unused/1, left, right -- b)) {
- DEOPT_IF(!(PySet_CheckExact(right) || PyFrozenSet_CheckExact(right)));
+ PyObject *left_o = PyStackRef_AsPyObjectBorrow(left);
+ PyObject *right_o = PyStackRef_AsPyObjectBorrow(right);
+
+ DEOPT_IF(!(PySet_CheckExact(right_o) || PyFrozenSet_CheckExact(right_o)));
STAT_INC(CONTAINS_OP, hit);
// Note: both set and frozenset use the same seq_contains method!
- int res = _PySet_Contains((PySetObject *)right, left);
+ int res = _PySet_Contains((PySetObject *)right_o, left_o);
DECREF_INPUTS();
ERROR_IF(res < 0, error);
- b = (res ^ oparg) ? Py_True : Py_False;
+ b = (res ^ oparg) ? PyStackRef_True : PyStackRef_False;
}
inst(CONTAINS_OP_DICT, (unused/1, left, right -- b)) {
- DEOPT_IF(!PyDict_CheckExact(right));
+ PyObject *left_o = PyStackRef_AsPyObjectBorrow(left);
+ PyObject *right_o = PyStackRef_AsPyObjectBorrow(right);
+
+ DEOPT_IF(!PyDict_CheckExact(right_o));
STAT_INC(CONTAINS_OP, hit);
- int res = PyDict_Contains(right, left);
+ int res = PyDict_Contains(right_o, left_o);
DECREF_INPUTS();
ERROR_IF(res < 0, error);
- b = (res ^ oparg) ? Py_True : Py_False;
+ b = (res ^ oparg) ? PyStackRef_True : PyStackRef_False;
}
- inst(CHECK_EG_MATCH, (exc_value, match_type -- rest, match)) {
+ inst(CHECK_EG_MATCH, (exc_value_st, match_type_st -- rest, match)) {
+ PyObject *exc_value = PyStackRef_AsPyObjectBorrow(exc_value_st);
+ PyObject *match_type = PyStackRef_AsPyObjectBorrow(match_type_st);
+
if (_PyEval_CheckExceptStarTypeValid(tstate, match_type) < 0) {
DECREF_INPUTS();
ERROR_IF(true, error);
}
- match = NULL;
- rest = NULL;
+ PyObject *match_o = NULL;
+ PyObject *rest_o = NULL;
int res = _PyEval_ExceptionGroupMatch(exc_value, match_type,
- &match, &rest);
+ &match_o, &rest_o);
DECREF_INPUTS();
ERROR_IF(res < 0, error);
- assert((match == NULL) == (rest == NULL));
- ERROR_IF(match == NULL, error);
+ assert((match_o == NULL) == (rest_o == NULL));
+ ERROR_IF(match_o == NULL, error);
- if (!Py_IsNone(match)) {
- PyErr_SetHandledException(match);
+ if (!Py_IsNone(match_o)) {
+ PyErr_SetHandledException(match_o);
}
+ rest = PyStackRef_FromPyObjectSteal(rest_o);
+ match = PyStackRef_FromPyObjectSteal(match_o);
}
inst(CHECK_EXC_MATCH, (left, right -- left, b)) {
- assert(PyExceptionInstance_Check(left));
- if (_PyEval_CheckExceptTypeValid(tstate, right) < 0) {
+ PyObject *left_o = PyStackRef_AsPyObjectBorrow(left);
+ PyObject *right_o = PyStackRef_AsPyObjectBorrow(right);
+
+ assert(PyExceptionInstance_Check(left_o));
+ if (_PyEval_CheckExceptTypeValid(tstate, right_o) < 0) {
DECREF_INPUTS();
ERROR_IF(true, error);
}
- int res = PyErr_GivenExceptionMatches(left, right);
+ int res = PyErr_GivenExceptionMatches(left_o, right_o);
DECREF_INPUTS();
- b = res ? Py_True : Py_False;
+ b = res ? PyStackRef_True : PyStackRef_False;
}
tier1 inst(IMPORT_NAME, (level, fromlist -- res)) {
PyObject *name = GETITEM(FRAME_CO_NAMES, oparg);
- res = import_name(tstate, frame, name, fromlist, level);
+ PyObject *res_o = import_name(tstate, frame, name,
+ PyStackRef_AsPyObjectBorrow(fromlist),
+ PyStackRef_AsPyObjectBorrow(level));
DECREF_INPUTS();
- ERROR_IF(res == NULL, error);
+ ERROR_IF(res_o == NULL, error);
+ res = PyStackRef_FromPyObjectSteal(res_o);
}
tier1 inst(IMPORT_FROM, (from -- from, res)) {
PyObject *name = GETITEM(FRAME_CO_NAMES, oparg);
- res = import_from(tstate, from, name);
- ERROR_IF(res == NULL, error);
+ PyObject *res_o = import_from(tstate, PyStackRef_AsPyObjectBorrow(from), name);
+ ERROR_IF(res_o == NULL, error);
+ res = PyStackRef_FromPyObjectSteal(res_o);
}
tier1 inst(JUMP_FORWARD, (--)) {
}
replaced op(_POP_JUMP_IF_FALSE, (cond -- )) {
- assert(PyBool_Check(cond));
- int flag = Py_IsFalse(cond);
+ assert(PyBool_Check(PyStackRef_AsPyObjectBorrow(cond)));
+ int flag = PyStackRef_Is(cond, PyStackRef_False);
#if ENABLE_SPECIALIZATION
this_instr[1].cache = (this_instr[1].cache << 1) | flag;
#endif
}
replaced op(_POP_JUMP_IF_TRUE, (cond -- )) {
- assert(PyBool_Check(cond));
- int flag = Py_IsTrue(cond);
+ assert(PyBool_Check(PyStackRef_AsPyObjectBorrow(cond)));
+ int flag = PyStackRef_Is(cond, PyStackRef_True);
#if ENABLE_SPECIALIZATION
this_instr[1].cache = (this_instr[1].cache << 1) | flag;
#endif
}
op(_IS_NONE, (value -- b)) {
- if (Py_IsNone(value)) {
- b = Py_True;
+ if (PyStackRef_Is(value, PyStackRef_None)) {
+ b = PyStackRef_True;
}
else {
- b = Py_False;
+ b = PyStackRef_False;
DECREF_INPUTS();
}
}
JUMPBY(-oparg);
}
- inst(GET_LEN, (obj -- obj, len_o)) {
+ inst(GET_LEN, (obj -- obj, len)) {
// PUSH(len(TOS))
- Py_ssize_t len_i = PyObject_Length(obj);
+ Py_ssize_t len_i = PyObject_Length(PyStackRef_AsPyObjectBorrow(obj));
ERROR_IF(len_i < 0, error);
- len_o = PyLong_FromSsize_t(len_i);
+ PyObject *len_o = PyLong_FromSsize_t(len_i);
ERROR_IF(len_o == NULL, error);
+ len = PyStackRef_FromPyObjectSteal(len_o);
}
inst(MATCH_CLASS, (subject, type, names -- attrs)) {
// Pop TOS and TOS1. Set TOS to a tuple of attributes on success, or
// None on failure.
- assert(PyTuple_CheckExact(names));
- attrs = _PyEval_MatchClass(tstate, subject, type, oparg, names);
+ assert(PyTuple_CheckExact(PyStackRef_AsPyObjectBorrow(names)));
+ PyObject *attrs_o = _PyEval_MatchClass(tstate,
+ PyStackRef_AsPyObjectBorrow(subject),
+ PyStackRef_AsPyObjectBorrow(type), oparg,
+ PyStackRef_AsPyObjectBorrow(names));
DECREF_INPUTS();
- if (attrs) {
- assert(PyTuple_CheckExact(attrs)); // Success!
+ if (attrs_o) {
+ assert(PyTuple_CheckExact(attrs_o)); // Success!
+ attrs = PyStackRef_FromPyObjectSteal(attrs_o);
}
else {
ERROR_IF(_PyErr_Occurred(tstate), error); // Error!
- attrs = Py_None; // Failure!
+ attrs = PyStackRef_None; // Failure!
}
}
inst(MATCH_MAPPING, (subject -- subject, res)) {
- int match = Py_TYPE(subject)->tp_flags & Py_TPFLAGS_MAPPING;
- res = match ? Py_True : Py_False;
+ int match = PyStackRef_TYPE(subject)->tp_flags & Py_TPFLAGS_MAPPING;
+ res = match ? PyStackRef_True : PyStackRef_False;
}
inst(MATCH_SEQUENCE, (subject -- subject, res)) {
- int match = Py_TYPE(subject)->tp_flags & Py_TPFLAGS_SEQUENCE;
- res = match ? Py_True : Py_False;
+ int match = PyStackRef_TYPE(subject)->tp_flags & Py_TPFLAGS_SEQUENCE;
+ res = match ? PyStackRef_True : PyStackRef_False;
}
inst(MATCH_KEYS, (subject, keys -- subject, keys, values_or_none)) {
// On successful match, PUSH(values). Otherwise, PUSH(None).
- values_or_none = _PyEval_MatchKeys(tstate, subject, keys);
- ERROR_IF(values_or_none == NULL, error);
+ PyObject *values_or_none_o = _PyEval_MatchKeys(tstate,
+ PyStackRef_AsPyObjectBorrow(subject), PyStackRef_AsPyObjectBorrow(keys));
+ ERROR_IF(values_or_none_o == NULL, error);
+ values_or_none = PyStackRef_FromPyObjectSteal(values_or_none_o);
}
inst(GET_ITER, (iterable -- iter)) {
/* before: [obj]; after [getiter(obj)] */
- iter = PyObject_GetIter(iterable);
+ iter = PyStackRef_FromPyObjectSteal(PyObject_GetIter(PyStackRef_AsPyObjectBorrow(iterable)));
DECREF_INPUTS();
- ERROR_IF(iter == NULL, error);
+ ERROR_IF(PyStackRef_IsNull(iter), error);
}
inst(GET_YIELD_FROM_ITER, (iterable -- iter)) {
/* before: [obj]; after [getiter(obj)] */
- if (PyCoro_CheckExact(iterable)) {
+ PyObject *iterable_o = PyStackRef_AsPyObjectBorrow(iterable);
+ if (PyCoro_CheckExact(iterable_o)) {
/* `iterable` is a coroutine */
if (!(_PyFrame_GetCode(frame)->co_flags & (CO_COROUTINE | CO_ITERABLE_COROUTINE))) {
/* and it is used in a 'yield from' expression of a
}
iter = iterable;
}
- else if (PyGen_CheckExact(iterable)) {
+ else if (PyGen_CheckExact(iterable_o)) {
iter = iterable;
}
else {
/* `iterable` is not a generator. */
- iter = PyObject_GetIter(iterable);
- if (iter == NULL) {
+ iter = PyStackRef_FromPyObjectSteal(PyObject_GetIter(iterable_o));
+ if (PyStackRef_IsNull(iter)) {
ERROR_NO_POP();
}
DECREF_INPUTS();
replaced op(_FOR_ITER, (iter -- iter, next)) {
/* before: [iter]; after: [iter, iter()] *or* [] (and jump over END_FOR.) */
- next = (*Py_TYPE(iter)->tp_iternext)(iter);
- if (next == NULL) {
+ PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter);
+ PyObject *next_o = (*Py_TYPE(iter_o)->tp_iternext)(iter_o);
+ if (next_o == NULL) {
+ next = PyStackRef_NULL;
if (_PyErr_Occurred(tstate)) {
if (!_PyErr_ExceptionMatches(tstate, PyExc_StopIteration)) {
ERROR_NO_POP();
/* iterator ended normally */
assert(next_instr[oparg].op.code == END_FOR ||
next_instr[oparg].op.code == INSTRUMENTED_END_FOR);
- Py_DECREF(iter);
+ PyStackRef_CLOSE(iter);
STACK_SHRINK(1);
/* Jump forward oparg, then skip following END_FOR and POP_TOP instruction */
JUMPBY(oparg + 2);
DISPATCH();
}
+ next = PyStackRef_FromPyObjectSteal(next_o);
// Common case: no jump, leave it to the code generator
}
op(_FOR_ITER_TIER_TWO, (iter -- iter, next)) {
/* before: [iter]; after: [iter, iter()] *or* [] (and jump over END_FOR.) */
- next = (*Py_TYPE(iter)->tp_iternext)(iter);
- if (next == NULL) {
+ PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter);
+ PyObject *next_o = (*Py_TYPE(iter_o)->tp_iternext)(iter_o);
+ if (next_o == NULL) {
if (_PyErr_Occurred(tstate)) {
if (!_PyErr_ExceptionMatches(tstate, PyExc_StopIteration)) {
ERROR_NO_POP();
/* The translator sets the deopt target just past the matching END_FOR */
DEOPT_IF(true);
}
+ next = PyStackRef_FromPyObjectSteal(next_o);
// Common case: no jump, leave it to the code generator
}
inst(INSTRUMENTED_FOR_ITER, (unused/1 -- )) {
_Py_CODEUNIT *target;
- PyObject *iter = TOP();
+ _PyStackRef iter_stackref = TOP();
+ PyObject *iter = PyStackRef_AsPyObjectBorrow(iter_stackref);
PyObject *next = (*Py_TYPE(iter)->tp_iternext)(iter);
if (next != NULL) {
- PUSH(next);
+ PUSH(PyStackRef_FromPyObjectSteal(next));
target = next_instr;
}
else {
assert(next_instr[oparg].op.code == END_FOR ||
next_instr[oparg].op.code == INSTRUMENTED_END_FOR);
STACK_SHRINK(1);
- Py_DECREF(iter);
+ PyStackRef_CLOSE(iter_stackref);
/* Skip END_FOR and POP_TOP */
target = next_instr + oparg + 2;
}
}
op(_ITER_CHECK_LIST, (iter -- iter)) {
- EXIT_IF(Py_TYPE(iter) != &PyListIter_Type);
+ EXIT_IF(Py_TYPE(PyStackRef_AsPyObjectBorrow(iter)) != &PyListIter_Type);
}
replaced op(_ITER_JUMP_LIST, (iter -- iter)) {
- _PyListIterObject *it = (_PyListIterObject *)iter;
- assert(Py_TYPE(iter) == &PyListIter_Type);
+ PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter);
+ _PyListIterObject *it = (_PyListIterObject *)iter_o;
+ assert(Py_TYPE(iter_o) == &PyListIter_Type);
STAT_INC(FOR_ITER, hit);
PyListObject *seq = it->it_seq;
if (seq == NULL || (size_t)it->it_index >= (size_t)PyList_GET_SIZE(seq)) {
Py_DECREF(seq);
}
#endif
- Py_DECREF(iter);
+ PyStackRef_CLOSE(iter);
STACK_SHRINK(1);
/* Jump forward oparg, then skip following END_FOR and POP_TOP instructions */
JUMPBY(oparg + 2);
// Only used by Tier 2
op(_GUARD_NOT_EXHAUSTED_LIST, (iter -- iter)) {
- _PyListIterObject *it = (_PyListIterObject *)iter;
- assert(Py_TYPE(iter) == &PyListIter_Type);
+ PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter);
+ _PyListIterObject *it = (_PyListIterObject *)iter_o;
+ assert(Py_TYPE(iter_o) == &PyListIter_Type);
PyListObject *seq = it->it_seq;
EXIT_IF(seq == NULL);
EXIT_IF((size_t)it->it_index >= (size_t)PyList_GET_SIZE(seq));
}
op(_ITER_NEXT_LIST, (iter -- iter, next)) {
- _PyListIterObject *it = (_PyListIterObject *)iter;
- assert(Py_TYPE(iter) == &PyListIter_Type);
+ PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter);
+ _PyListIterObject *it = (_PyListIterObject *)iter_o;
+ assert(Py_TYPE(iter_o) == &PyListIter_Type);
PyListObject *seq = it->it_seq;
assert(seq);
assert(it->it_index < PyList_GET_SIZE(seq));
- next = Py_NewRef(PyList_GET_ITEM(seq, it->it_index++));
+ next = PyStackRef_FromPyObjectNew(PyList_GET_ITEM(seq, it->it_index++));
}
macro(FOR_ITER_LIST) =
_ITER_NEXT_LIST;
op(_ITER_CHECK_TUPLE, (iter -- iter)) {
- EXIT_IF(Py_TYPE(iter) != &PyTupleIter_Type);
+ EXIT_IF(Py_TYPE(PyStackRef_AsPyObjectBorrow(iter)) != &PyTupleIter_Type);
}
replaced op(_ITER_JUMP_TUPLE, (iter -- iter)) {
- _PyTupleIterObject *it = (_PyTupleIterObject *)iter;
- assert(Py_TYPE(iter) == &PyTupleIter_Type);
+ PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter);
+ _PyTupleIterObject *it = (_PyTupleIterObject *)iter_o;
+ assert(Py_TYPE(iter_o) == &PyTupleIter_Type);
STAT_INC(FOR_ITER, hit);
PyTupleObject *seq = it->it_seq;
if (seq == NULL || it->it_index >= PyTuple_GET_SIZE(seq)) {
it->it_seq = NULL;
Py_DECREF(seq);
}
- Py_DECREF(iter);
+ PyStackRef_CLOSE(iter);
STACK_SHRINK(1);
/* Jump forward oparg, then skip following END_FOR and POP_TOP instructions */
JUMPBY(oparg + 2);
// Only used by Tier 2
op(_GUARD_NOT_EXHAUSTED_TUPLE, (iter -- iter)) {
- _PyTupleIterObject *it = (_PyTupleIterObject *)iter;
- assert(Py_TYPE(iter) == &PyTupleIter_Type);
+ PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter);
+ _PyTupleIterObject *it = (_PyTupleIterObject *)iter_o;
+ assert(Py_TYPE(iter_o) == &PyTupleIter_Type);
PyTupleObject *seq = it->it_seq;
EXIT_IF(seq == NULL);
EXIT_IF(it->it_index >= PyTuple_GET_SIZE(seq));
}
op(_ITER_NEXT_TUPLE, (iter -- iter, next)) {
- _PyTupleIterObject *it = (_PyTupleIterObject *)iter;
- assert(Py_TYPE(iter) == &PyTupleIter_Type);
+ PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter);
+ _PyTupleIterObject *it = (_PyTupleIterObject *)iter_o;
+ assert(Py_TYPE(iter_o) == &PyTupleIter_Type);
PyTupleObject *seq = it->it_seq;
assert(seq);
assert(it->it_index < PyTuple_GET_SIZE(seq));
- next = Py_NewRef(PyTuple_GET_ITEM(seq, it->it_index++));
+ next = PyStackRef_FromPyObjectNew(PyTuple_GET_ITEM(seq, it->it_index++));
}
macro(FOR_ITER_TUPLE) =
_ITER_NEXT_TUPLE;
op(_ITER_CHECK_RANGE, (iter -- iter)) {
- _PyRangeIterObject *r = (_PyRangeIterObject *)iter;
+ _PyRangeIterObject *r = (_PyRangeIterObject *)PyStackRef_AsPyObjectBorrow(iter);
EXIT_IF(Py_TYPE(r) != &PyRangeIter_Type);
}
replaced op(_ITER_JUMP_RANGE, (iter -- iter)) {
- _PyRangeIterObject *r = (_PyRangeIterObject *)iter;
+ _PyRangeIterObject *r = (_PyRangeIterObject *)PyStackRef_AsPyObjectBorrow(iter);
assert(Py_TYPE(r) == &PyRangeIter_Type);
STAT_INC(FOR_ITER, hit);
if (r->len <= 0) {
STACK_SHRINK(1);
- Py_DECREF(r);
+ PyStackRef_CLOSE(iter);
// Jump over END_FOR and POP_TOP instructions.
JUMPBY(oparg + 2);
DISPATCH();
// Only used by Tier 2
op(_GUARD_NOT_EXHAUSTED_RANGE, (iter -- iter)) {
- _PyRangeIterObject *r = (_PyRangeIterObject *)iter;
+ _PyRangeIterObject *r = (_PyRangeIterObject *)PyStackRef_AsPyObjectBorrow(iter);
assert(Py_TYPE(r) == &PyRangeIter_Type);
EXIT_IF(r->len <= 0);
}
op(_ITER_NEXT_RANGE, (iter -- iter, next)) {
- _PyRangeIterObject *r = (_PyRangeIterObject *)iter;
+ _PyRangeIterObject *r = (_PyRangeIterObject *)PyStackRef_AsPyObjectBorrow(iter);
assert(Py_TYPE(r) == &PyRangeIter_Type);
assert(r->len > 0);
long value = r->start;
r->start = value + r->step;
r->len--;
- next = PyLong_FromLong(value);
- ERROR_IF(next == NULL, error);
+ PyObject *res = PyLong_FromLong(value);
+ ERROR_IF(res == NULL, error);
+ next = PyStackRef_FromPyObjectSteal(res);
}
macro(FOR_ITER_RANGE) =
_ITER_NEXT_RANGE;
op(_FOR_ITER_GEN_FRAME, (iter -- iter, gen_frame: _PyInterpreterFrame*)) {
- PyGenObject *gen = (PyGenObject *)iter;
+ PyGenObject *gen = (PyGenObject *)PyStackRef_AsPyObjectBorrow(iter);
DEOPT_IF(Py_TYPE(gen) != &PyGen_Type);
DEOPT_IF(gen->gi_frame_state >= FRAME_EXECUTING);
STAT_INC(FOR_ITER, hit);
gen_frame = &gen->gi_iframe;
- _PyFrame_StackPush(gen_frame, Py_None);
+ _PyFrame_StackPush(gen_frame, PyStackRef_None);
gen->gi_frame_state = FRAME_EXECUTING;
gen->gi_exc_state.previous_item = tstate->exc_info;
tstate->exc_info = &gen->gi_exc_state;
inst(LOAD_SPECIAL, (owner -- attr, self_or_null)) {
assert(oparg <= SPECIAL_MAX);
+ PyObject *owner_o = PyStackRef_AsPyObjectSteal(owner);
PyObject *name = _Py_SpecialMethods[oparg].name;
- attr = _PyObject_LookupSpecialMethod(owner, name, &self_or_null);
- if (attr == NULL) {
+ PyObject *self_or_null_o;
+ attr = PyStackRef_FromPyObjectSteal(_PyObject_LookupSpecialMethod(owner_o, name, &self_or_null_o));
+ if (PyStackRef_IsNull(attr)) {
if (!_PyErr_Occurred(tstate)) {
_PyErr_Format(tstate, PyExc_TypeError,
_Py_SpecialMethods[oparg].error,
- Py_TYPE(owner)->tp_name);
+ Py_TYPE(owner_o)->tp_name);
}
}
- ERROR_IF(attr == NULL, error);
+ ERROR_IF(PyStackRef_IsNull(attr), error);
+ self_or_null = PyStackRef_FromPyObjectSteal(self_or_null_o);
}
inst(WITH_EXCEPT_START, (exit_func, exit_self, lasti, unused, val -- exit_func, exit_self, lasti, unused, val, res)) {
*/
PyObject *exc, *tb;
- assert(val && PyExceptionInstance_Check(val));
- exc = PyExceptionInstance_Class(val);
- tb = PyException_GetTraceback(val);
+ PyObject *val_o = PyStackRef_AsPyObjectBorrow(val);
+ PyObject *exit_func_o = PyStackRef_AsPyObjectBorrow(exit_func);
+
+ assert(val_o && PyExceptionInstance_Check(val_o));
+ exc = PyExceptionInstance_Class(val_o);
+ tb = PyException_GetTraceback(val_o);
if (tb == NULL) {
tb = Py_None;
}
else {
Py_DECREF(tb);
}
- assert(PyLong_Check(lasti));
+ assert(PyLong_Check(PyStackRef_AsPyObjectBorrow(lasti)));
(void)lasti; // Shut up compiler warning if asserts are off
- PyObject *stack[5] = {NULL, exit_self, exc, val, tb};
- int has_self = (exit_self != NULL);
- res = PyObject_Vectorcall(exit_func, stack + 2 - has_self,
- (3 + has_self) | PY_VECTORCALL_ARGUMENTS_OFFSET, NULL);
- ERROR_IF(res == NULL, error);
+ PyObject *stack[5] = {NULL, PyStackRef_AsPyObjectBorrow(exit_self), exc, val_o, tb};
+ int has_self = !PyStackRef_IsNull(exit_self);
+ res = PyStackRef_FromPyObjectSteal(PyObject_Vectorcall(exit_func_o, stack + 2 - has_self,
+ (3 + has_self) | PY_VECTORCALL_ARGUMENTS_OFFSET, NULL));
+ ERROR_IF(PyStackRef_IsNull(res), error);
}
pseudo(SETUP_FINALLY, (-- unused), (HAS_ARG)) = {
};
inst(PUSH_EXC_INFO, (new_exc -- prev_exc, new_exc)) {
+
_PyErr_StackItem *exc_info = tstate->exc_info;
if (exc_info->exc_value != NULL) {
- prev_exc = exc_info->exc_value;
+ prev_exc = PyStackRef_FromPyObjectSteal(exc_info->exc_value);
}
else {
- prev_exc = Py_None;
+ prev_exc = PyStackRef_None;
}
- assert(PyExceptionInstance_Check(new_exc));
- exc_info->exc_value = Py_NewRef(new_exc);
+ assert(PyExceptionInstance_Check(PyStackRef_AsPyObjectBorrow(new_exc)));
+ exc_info->exc_value = PyStackRef_AsPyObjectNew(new_exc);
}
op(_GUARD_DORV_VALUES_INST_ATTR_FROM_DICT, (owner -- owner)) {
- assert(Py_TYPE(owner)->tp_flags & Py_TPFLAGS_INLINE_VALUES);
- DEOPT_IF(!_PyObject_InlineValues(owner)->valid);
+ PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner);
+ assert(Py_TYPE(owner_o)->tp_flags & Py_TPFLAGS_INLINE_VALUES);
+ DEOPT_IF(!_PyObject_InlineValues(owner_o)->valid);
}
op(_GUARD_KEYS_VERSION, (keys_version/2, owner -- owner)) {
- PyTypeObject *owner_cls = Py_TYPE(owner);
+ PyTypeObject *owner_cls = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner));
PyHeapTypeObject *owner_heap_type = (PyHeapTypeObject *)owner_cls;
DEOPT_IF(owner_heap_type->ht_cached_keys->dk_version != keys_version);
}
/* Cached method object */
STAT_INC(LOAD_ATTR, hit);
assert(descr != NULL);
- attr = Py_NewRef(descr);
- assert(_PyType_HasFeature(Py_TYPE(attr), Py_TPFLAGS_METHOD_DESCRIPTOR));
+ assert(_PyType_HasFeature(Py_TYPE(descr), Py_TPFLAGS_METHOD_DESCRIPTOR));
+ attr = PyStackRef_FromPyObjectNew(descr);
self = owner;
}
op(_LOAD_ATTR_METHOD_NO_DICT, (descr/4, owner -- attr, self if (1))) {
assert(oparg & 1);
- assert(Py_TYPE(owner)->tp_dictoffset == 0);
+ assert(Py_TYPE(PyStackRef_AsPyObjectBorrow(owner))->tp_dictoffset == 0);
STAT_INC(LOAD_ATTR, hit);
assert(descr != NULL);
assert(_PyType_HasFeature(Py_TYPE(descr), Py_TPFLAGS_METHOD_DESCRIPTOR));
- attr = Py_NewRef(descr);
+ attr = PyStackRef_FromPyObjectNew(descr);
self = owner;
}
STAT_INC(LOAD_ATTR, hit);
assert(descr != NULL);
DECREF_INPUTS();
- attr = Py_NewRef(descr);
+ attr = PyStackRef_FromPyObjectNew(descr);
}
macro(LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES) =
op(_LOAD_ATTR_NONDESCRIPTOR_NO_DICT, (descr/4, owner -- attr, unused if (0))) {
assert((oparg & 1) == 0);
- assert(Py_TYPE(owner)->tp_dictoffset == 0);
+ assert(Py_TYPE(PyStackRef_AsPyObjectBorrow(owner))->tp_dictoffset == 0);
STAT_INC(LOAD_ATTR, hit);
assert(descr != NULL);
DECREF_INPUTS();
- attr = Py_NewRef(descr);
+ attr = PyStackRef_FromPyObjectNew(descr);
}
macro(LOAD_ATTR_NONDESCRIPTOR_NO_DICT) =
_LOAD_ATTR_NONDESCRIPTOR_NO_DICT;
op(_CHECK_ATTR_METHOD_LAZY_DICT, (dictoffset/1, owner -- owner)) {
- char *ptr = ((char *)owner) + MANAGED_DICT_OFFSET + dictoffset;
+ char *ptr = ((char *)PyStackRef_AsPyObjectBorrow(owner)) + MANAGED_DICT_OFFSET + dictoffset;
PyObject *dict = *(PyObject **)ptr;
/* This object has a __dict__, just not yet created */
DEOPT_IF(dict != NULL);
STAT_INC(LOAD_ATTR, hit);
assert(descr != NULL);
assert(_PyType_HasFeature(Py_TYPE(descr), Py_TPFLAGS_METHOD_DESCRIPTOR));
- attr = Py_NewRef(descr);
+ attr = PyStackRef_FromPyObjectNew(descr);
self = owner;
}
_LOAD_ATTR_METHOD_LAZY_DICT;
inst(INSTRUMENTED_CALL, (unused/3 -- )) {
- int is_meth = PEEK(oparg + 1) != NULL;
+ int is_meth = PyStackRef_AsPyObjectBorrow(PEEK(oparg + 1)) != NULL;
int total_args = oparg + is_meth;
- PyObject *function = PEEK(oparg + 2);
+ PyObject *function = PyStackRef_AsPyObjectBorrow(PEEK(oparg + 2));
PyObject *arg = total_args == 0 ?
- &_PyInstrumentation_MISSING : PEEK(total_args);
+ &_PyInstrumentation_MISSING : PyStackRef_AsPyObjectBorrow(PEEK(total_args));
int err = _Py_call_instrumentation_2args(
tstate, PY_MONITORING_EVENT_CALL,
frame, this_instr, function, arg);
#if ENABLE_SPECIALIZATION
if (ADAPTIVE_COUNTER_TRIGGERS(counter)) {
next_instr = this_instr;
- _Py_Specialize_Call(callable, next_instr, oparg + (self_or_null != NULL));
+ _Py_Specialize_Call(callable, next_instr, oparg + !PyStackRef_IsNull(self_or_null));
DISPATCH_SAME_OPARG();
}
STAT_INC(CALL, deferred);
// When calling Python, inline the call using DISPATCH_INLINED().
op(_CALL, (callable, self_or_null, args[oparg] -- res)) {
+ PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable);
+ PyObject *self_or_null_o = PyStackRef_AsPyObjectBorrow(self_or_null);
+
// oparg counts all of the args, but *not* self:
int total_args = oparg;
- if (self_or_null != NULL) {
+ if (self_or_null_o != NULL) {
args--;
total_args++;
}
- else if (Py_TYPE(callable) == &PyMethod_Type) {
+ else if (Py_TYPE(callable_o) == &PyMethod_Type) {
args--;
total_args++;
- PyObject *self = ((PyMethodObject *)callable)->im_self;
- args[0] = Py_NewRef(self);
- PyObject *method = ((PyMethodObject *)callable)->im_func;
- args[-1] = Py_NewRef(method);
- Py_DECREF(callable);
- callable = method;
+ PyObject *self = ((PyMethodObject *)callable_o)->im_self;
+ args[0] = PyStackRef_FromPyObjectNew(self);
+ PyObject *method = ((PyMethodObject *)callable_o)->im_func;
+ args[-1] = PyStackRef_FromPyObjectNew(method);
+ PyStackRef_CLOSE(callable);
+ callable_o = method;
+ callable = args[-1];
}
// Check if the call can be inlined or not
- if (Py_TYPE(callable) == &PyFunction_Type &&
+ if (Py_TYPE(callable_o) == &PyFunction_Type &&
tstate->interp->eval_frame == NULL &&
- ((PyFunctionObject *)callable)->vectorcall == _PyFunction_Vectorcall)
+ ((PyFunctionObject *)callable_o)->vectorcall == _PyFunction_Vectorcall)
{
- int code_flags = ((PyCodeObject*)PyFunction_GET_CODE(callable))->co_flags;
- PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable));
+ int code_flags = ((PyCodeObject*)PyFunction_GET_CODE(callable_o))->co_flags;
+ PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable_o));
_PyInterpreterFrame *new_frame = _PyEvalFramePushAndInit(
- tstate, (PyFunctionObject *)callable, locals,
+ tstate, (PyFunctionObject *)PyStackRef_AsPyObjectSteal(callable), locals,
args, total_args, NULL
);
// Manipulate stack directly since we leave using DISPATCH_INLINED().
DISPATCH_INLINED(new_frame);
}
/* Callable is not a normal Python function */
- res = PyObject_Vectorcall(
- callable, args,
+ STACKREFS_TO_PYOBJECTS(args, total_args, args_o);
+ if (CONVERSION_FAILED(args_o)) {
+ DECREF_INPUTS();
+ ERROR_IF(true, error);
+ }
+ PyObject *res_o = PyObject_Vectorcall(
+ callable_o, args_o,
total_args | PY_VECTORCALL_ARGUMENTS_OFFSET,
NULL);
+ STACKREFS_TO_PYOBJECTS_CLEANUP(args_o);
if (opcode == INSTRUMENTED_CALL) {
PyObject *arg = total_args == 0 ?
- &_PyInstrumentation_MISSING : args[0];
- if (res == NULL) {
+ &_PyInstrumentation_MISSING : PyStackRef_AsPyObjectBorrow(args[0]);
+ if (res_o == NULL) {
_Py_call_instrumentation_exc2(
tstate, PY_MONITORING_EVENT_C_RAISE,
- frame, this_instr, callable, arg);
+ frame, this_instr, callable_o, arg);
}
else {
int err = _Py_call_instrumentation_2args(
tstate, PY_MONITORING_EVENT_C_RETURN,
- frame, this_instr, callable, arg);
+ frame, this_instr, callable_o, arg);
if (err < 0) {
- Py_CLEAR(res);
+ Py_CLEAR(res_o);
}
}
}
- assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL));
- Py_DECREF(callable);
+ assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL));
+ PyStackRef_CLOSE(callable);
for (int i = 0; i < total_args; i++) {
- Py_DECREF(args[i]);
+ PyStackRef_CLOSE(args[i]);
}
- ERROR_IF(res == NULL, error);
+ ERROR_IF(res_o == NULL, error);
+ res = PyStackRef_FromPyObjectSteal(res_o);
}
op(_CHECK_PERIODIC, (--)) {
macro(CALL) = _SPECIALIZE_CALL + unused/2 + _CALL + _CHECK_PERIODIC;
op(_PY_FRAME_GENERAL, (callable, self_or_null, args[oparg] -- new_frame: _PyInterpreterFrame*)) {
+ PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable);
+ PyObject *self_or_null_o = PyStackRef_AsPyObjectBorrow(self_or_null);
+
// oparg counts all of the args, but *not* self:
int total_args = oparg;
- if (self_or_null != NULL) {
+ if (self_or_null_o != NULL) {
args--;
total_args++;
}
- assert(Py_TYPE(callable) == &PyFunction_Type);
- int code_flags = ((PyCodeObject*)PyFunction_GET_CODE(callable))->co_flags;
- PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable));
+ assert(Py_TYPE(callable_o) == &PyFunction_Type);
+ int code_flags = ((PyCodeObject*)PyFunction_GET_CODE(callable_o))->co_flags;
+ PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable_o));
new_frame = _PyEvalFramePushAndInit(
- tstate, (PyFunctionObject *)callable, locals,
+ tstate, (PyFunctionObject *)PyStackRef_AsPyObjectSteal(callable), locals,
args, total_args, NULL
);
// The frame has stolen all the arguments from the stack,
}
op(_CHECK_FUNCTION_VERSION, (func_version/2, callable, unused, unused[oparg] -- callable, unused, unused[oparg])) {
- EXIT_IF(!PyFunction_Check(callable));
- PyFunctionObject *func = (PyFunctionObject *)callable;
+ PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable);
+ EXIT_IF(!PyFunction_Check(callable_o));
+ PyFunctionObject *func = (PyFunctionObject *)callable_o;
EXIT_IF(func->func_version != func_version);
}
_PUSH_FRAME;
op(_CHECK_METHOD_VERSION, (func_version/2, callable, null, unused[oparg] -- callable, null, unused[oparg])) {
- EXIT_IF(Py_TYPE(callable) != &PyMethod_Type);
- PyObject *func = ((PyMethodObject *)callable)->im_func;
+ PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable);
+
+ EXIT_IF(Py_TYPE(callable_o) != &PyMethod_Type);
+ PyObject *func = ((PyMethodObject *)callable_o)->im_func;
EXIT_IF(!PyFunction_Check(func));
EXIT_IF(((PyFunctionObject *)func)->func_version != func_version);
- EXIT_IF(null != NULL);
+ EXIT_IF(!PyStackRef_IsNull(null));
}
op(_EXPAND_METHOD, (callable, null, unused[oparg] -- method, self, unused[oparg])) {
- assert(null == NULL);
- assert(Py_TYPE(callable) == &PyMethod_Type);
- self = ((PyMethodObject *)callable)->im_self;
- Py_INCREF(self);
+ PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable);
+
+ assert(PyStackRef_IsNull(null));
+ assert(Py_TYPE(callable_o) == &PyMethod_Type);
+ self = PyStackRef_FromPyObjectNew(((PyMethodObject *)callable_o)->im_self);
stack_pointer[-1 - oparg] = self; // Patch stack as it is used by _PY_FRAME_GENERAL
- method = ((PyMethodObject *)callable)->im_func;
- assert(PyFunction_Check(method));
- Py_INCREF(method);
- Py_DECREF(callable);
+ method = PyStackRef_FromPyObjectNew(((PyMethodObject *)callable_o)->im_func);
+ assert(PyFunction_Check(PyStackRef_AsPyObjectBorrow(method)));
+ PyStackRef_CLOSE(callable);
}
macro(CALL_BOUND_METHOD_GENERAL) =
_PUSH_FRAME;
op(_CHECK_IS_NOT_PY_CALLABLE, (callable, unused, unused[oparg] -- callable, unused, unused[oparg])) {
- EXIT_IF(PyFunction_Check(callable));
- EXIT_IF(Py_TYPE(callable) == &PyMethod_Type);
+ PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable);
+ EXIT_IF(PyFunction_Check(callable_o));
+ EXIT_IF(Py_TYPE(callable_o) == &PyMethod_Type);
}
op(_CALL_NON_PY_GENERAL, (callable, self_or_null, args[oparg] -- res)) {
#if TIER_ONE
assert(opcode != INSTRUMENTED_CALL);
#endif
+ PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable);
+ PyObject *self_or_null_o = PyStackRef_AsPyObjectBorrow(self_or_null);
+
int total_args = oparg;
- if (self_or_null != NULL) {
+ if (self_or_null_o != NULL) {
args--;
total_args++;
}
/* Callable is not a normal Python function */
- res = PyObject_Vectorcall(
- callable, args,
+ STACKREFS_TO_PYOBJECTS(args, total_args, args_o);
+ if (CONVERSION_FAILED(args_o)) {
+ DECREF_INPUTS();
+ ERROR_IF(true, error);
+ }
+ PyObject *res_o = PyObject_Vectorcall(
+ callable_o, args_o,
total_args | PY_VECTORCALL_ARGUMENTS_OFFSET,
NULL);
- assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL));
- Py_DECREF(callable);
+ STACKREFS_TO_PYOBJECTS_CLEANUP(args_o);
+ assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL));
+ PyStackRef_CLOSE(callable);
for (int i = 0; i < total_args; i++) {
- Py_DECREF(args[i]);
+ PyStackRef_CLOSE(args[i]);
}
- ERROR_IF(res == NULL, error);
+ ERROR_IF(res_o == NULL, error);
+ res = PyStackRef_FromPyObjectSteal(res_o);
}
macro(CALL_NON_PY_GENERAL) =
_CHECK_PERIODIC;
op(_CHECK_CALL_BOUND_METHOD_EXACT_ARGS, (callable, null, unused[oparg] -- callable, null, unused[oparg])) {
- EXIT_IF(null != NULL);
- EXIT_IF(Py_TYPE(callable) != &PyMethod_Type);
+ EXIT_IF(!PyStackRef_IsNull(null));
+ EXIT_IF(Py_TYPE(PyStackRef_AsPyObjectBorrow(callable)) != &PyMethod_Type);
}
op(_INIT_CALL_BOUND_METHOD_EXACT_ARGS, (callable, unused, unused[oparg] -- func, self, unused[oparg])) {
+ PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable);
STAT_INC(CALL, hit);
- self = Py_NewRef(((PyMethodObject *)callable)->im_self);
- stack_pointer[-1 - oparg] = self; // Patch stack as it is used by _INIT_CALL_PY_EXACT_ARGS
- func = Py_NewRef(((PyMethodObject *)callable)->im_func);
- stack_pointer[-2 - oparg] = func; // This is used by CALL, upon deoptimization
- Py_DECREF(callable);
+ stack_pointer[-1 - oparg] = PyStackRef_FromPyObjectNew(((PyMethodObject *)callable_o)->im_self); // Patch stack as it is used by _INIT_CALL_PY_EXACT_ARGS
+ stack_pointer[-2 - oparg] = PyStackRef_FromPyObjectNew(((PyMethodObject *)callable_o)->im_func); // This is used by CALL, upon deoptimization
+ self = stack_pointer[-1 - oparg];
+ func = stack_pointer[-2 - oparg];
+ PyStackRef_CLOSE(callable);
+ // self may be unused in tier 1, so silence warnings.
+ (void)self;
}
op(_CHECK_PEP_523, (--)) {
}
op(_CHECK_FUNCTION_EXACT_ARGS, (callable, self_or_null, unused[oparg] -- callable, self_or_null, unused[oparg])) {
- assert(PyFunction_Check(callable));
- PyFunctionObject *func = (PyFunctionObject *)callable;
+ PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable);
+ assert(PyFunction_Check(callable_o));
+ PyFunctionObject *func = (PyFunctionObject *)callable_o;
PyCodeObject *code = (PyCodeObject *)func->func_code;
- EXIT_IF(code->co_argcount != oparg + (self_or_null != NULL));
+ EXIT_IF(code->co_argcount != oparg + (!PyStackRef_IsNull(self_or_null)));
}
op(_CHECK_STACK_SPACE, (callable, unused, unused[oparg] -- callable, unused, unused[oparg])) {
- PyFunctionObject *func = (PyFunctionObject *)callable;
+ PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable);
+ PyFunctionObject *func = (PyFunctionObject *)callable_o;
PyCodeObject *code = (PyCodeObject *)func->func_code;
DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize));
DEOPT_IF(tstate->py_recursion_remaining <= 1);
}
replicate(5) pure op(_INIT_CALL_PY_EXACT_ARGS, (callable, self_or_null, args[oparg] -- new_frame: _PyInterpreterFrame*)) {
- int has_self = (self_or_null != NULL);
+ PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable);
+ int has_self = !PyStackRef_IsNull(self_or_null);
STAT_INC(CALL, hit);
- PyFunctionObject *func = (PyFunctionObject *)callable;
+ PyFunctionObject *func = (PyFunctionObject *)callable_o;
new_frame = _PyFrame_PushUnchecked(tstate, func, oparg + has_self);
- PyObject **first_non_self_local = new_frame->localsplus + has_self;
+ _PyStackRef *first_non_self_local = new_frame->localsplus + has_self;
new_frame->localsplus[0] = self_or_null;
for (int i = 0; i < oparg; i++) {
first_non_self_local[i] = args[i];
_PUSH_FRAME;
inst(CALL_TYPE_1, (unused/1, unused/2, callable, null, arg -- res)) {
+ PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable);
+ PyObject *arg_o = PyStackRef_AsPyObjectBorrow(arg);
+
assert(oparg == 1);
- DEOPT_IF(null != NULL);
- DEOPT_IF(callable != (PyObject *)&PyType_Type);
+ DEOPT_IF(!PyStackRef_IsNull(null));
+ DEOPT_IF(callable_o != (PyObject *)&PyType_Type);
STAT_INC(CALL, hit);
- res = Py_NewRef(Py_TYPE(arg));
- Py_DECREF(arg);
+ res = PyStackRef_FromPyObjectSteal(Py_NewRef(Py_TYPE(arg_o)));
+ PyStackRef_CLOSE(arg);
}
op(_CALL_STR_1, (callable, null, arg -- res)) {
+ PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable);
+ PyObject *arg_o = PyStackRef_AsPyObjectBorrow(arg);
+
assert(oparg == 1);
- DEOPT_IF(null != NULL);
- DEOPT_IF(callable != (PyObject *)&PyUnicode_Type);
+ DEOPT_IF(!PyStackRef_IsNull(null));
+ DEOPT_IF(callable_o != (PyObject *)&PyUnicode_Type);
STAT_INC(CALL, hit);
- res = PyObject_Str(arg);
- Py_DECREF(arg);
- ERROR_IF(res == NULL, error);
+ res = PyStackRef_FromPyObjectSteal(PyObject_Str(arg_o));
+ PyStackRef_CLOSE(arg);
+ ERROR_IF(PyStackRef_IsNull(res), error);
}
macro(CALL_STR_1) =
_CHECK_PERIODIC;
op(_CALL_TUPLE_1, (callable, null, arg -- res)) {
+ PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable);
+ PyObject *arg_o = PyStackRef_AsPyObjectBorrow(arg);
+
assert(oparg == 1);
- DEOPT_IF(null != NULL);
- DEOPT_IF(callable != (PyObject *)&PyTuple_Type);
+ DEOPT_IF(!PyStackRef_IsNull(null));
+ DEOPT_IF(callable_o != (PyObject *)&PyTuple_Type);
STAT_INC(CALL, hit);
- res = PySequence_Tuple(arg);
- Py_DECREF(arg);
- ERROR_IF(res == NULL, error);
+ res = PyStackRef_FromPyObjectSteal(PySequence_Tuple(arg_o));
+ PyStackRef_CLOSE(arg);
+ ERROR_IF(PyStackRef_IsNull(res), error);
}
macro(CALL_TUPLE_1) =
_CHECK_PERIODIC;
inst(CALL_ALLOC_AND_ENTER_INIT, (unused/1, unused/2, callable, null, args[oparg] -- unused)) {
+ PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable);
/* This instruction does the following:
* 1. Creates the object (by calling ``object.__new__``)
* 2. Pushes a shim frame to the frame stack (to cleanup after ``__init__``)
* 3. Pushes the frame for ``__init__`` to the frame stack
* */
_PyCallCache *cache = (_PyCallCache *)&this_instr[1];
- DEOPT_IF(null != NULL);
- DEOPT_IF(!PyType_Check(callable));
- PyTypeObject *tp = (PyTypeObject *)callable;
+ DEOPT_IF(!PyStackRef_IsNull(null));
+ DEOPT_IF(!PyType_Check(callable_o));
+ PyTypeObject *tp = (PyTypeObject *)callable_o;
DEOPT_IF(tp->tp_version_tag != read_u32(cache->func_version));
assert(tp->tp_flags & Py_TPFLAGS_INLINE_VALUES);
- PyHeapTypeObject *cls = (PyHeapTypeObject *)callable;
+ PyHeapTypeObject *cls = (PyHeapTypeObject *)callable_o;
PyFunctionObject *init = (PyFunctionObject *)cls->_spec_cache.init;
PyCodeObject *code = (PyCodeObject *)init->func_code;
DEOPT_IF(code->co_argcount != oparg+1);
if (self == NULL) {
ERROR_NO_POP();
}
- Py_DECREF(tp);
+ PyStackRef_CLOSE(callable);
_PyInterpreterFrame *shim = _PyFrame_PushTrampolineUnchecked(
tstate, (PyCodeObject *)&_Py_InitCleanup, 1);
assert(_PyCode_CODE((PyCodeObject *)shim->f_executable)[0].op.code == EXIT_INIT_CHECK);
/* Push self onto stack of shim */
Py_INCREF(self);
- shim->localsplus[0] = self;
+ shim->localsplus[0] = PyStackRef_FromPyObjectSteal(self);
Py_INCREF(init);
_PyInterpreterFrame *init_frame = _PyFrame_PushUnchecked(tstate, init, oparg+1);
/* Copy self followed by args to __init__ frame */
- init_frame->localsplus[0] = self;
+ init_frame->localsplus[0] = PyStackRef_FromPyObjectSteal(self);
for (int i = 0; i < oparg; i++) {
init_frame->localsplus[i+1] = args[i];
}
inst(EXIT_INIT_CHECK, (should_be_none -- )) {
assert(STACK_LEVEL() == 2);
- if (should_be_none != Py_None) {
+ if (!PyStackRef_Is(should_be_none, PyStackRef_None)) {
PyErr_Format(PyExc_TypeError,
"__init__() should return None, not '%.200s'",
- Py_TYPE(should_be_none)->tp_name);
+ Py_TYPE(PyStackRef_AsPyObjectBorrow(should_be_none))->tp_name);
ERROR_NO_POP();
}
}
op(_CALL_BUILTIN_CLASS, (callable, self_or_null, args[oparg] -- res)) {
+ PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable);
+
int total_args = oparg;
- if (self_or_null != NULL) {
+ if (!PyStackRef_IsNull(self_or_null)) {
args--;
total_args++;
}
- DEOPT_IF(!PyType_Check(callable));
- PyTypeObject *tp = (PyTypeObject *)callable;
+ DEOPT_IF(!PyType_Check(callable_o));
+ PyTypeObject *tp = (PyTypeObject *)callable_o;
DEOPT_IF(tp->tp_vectorcall == NULL);
STAT_INC(CALL, hit);
- res = tp->tp_vectorcall((PyObject *)tp, args, total_args, NULL);
+ STACKREFS_TO_PYOBJECTS(args, total_args, args_o);
+ if (CONVERSION_FAILED(args_o)) {
+ DECREF_INPUTS();
+ ERROR_IF(true, error);
+ }
+ PyObject *res_o = tp->tp_vectorcall((PyObject *)tp, args_o, total_args, NULL);
+ STACKREFS_TO_PYOBJECTS_CLEANUP(args_o);
/* Free the arguments. */
for (int i = 0; i < total_args; i++) {
- Py_DECREF(args[i]);
+ PyStackRef_CLOSE(args[i]);
}
- Py_DECREF(tp);
- ERROR_IF(res == NULL, error);
+ PyStackRef_CLOSE(callable);
+ ERROR_IF(res_o == NULL, error);
+ res = PyStackRef_FromPyObjectSteal(res_o);
}
macro(CALL_BUILTIN_CLASS) =
op(_CALL_BUILTIN_O, (callable, self_or_null, args[oparg] -- res)) {
/* Builtin METH_O functions */
+ PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable);
+
int total_args = oparg;
- if (self_or_null != NULL) {
+ if (!PyStackRef_IsNull(self_or_null)) {
args--;
total_args++;
}
DEOPT_IF(total_args != 1);
- DEOPT_IF(!PyCFunction_CheckExact(callable));
- DEOPT_IF(PyCFunction_GET_FLAGS(callable) != METH_O);
+ DEOPT_IF(!PyCFunction_CheckExact(callable_o));
+ DEOPT_IF(PyCFunction_GET_FLAGS(callable_o) != METH_O);
// CPython promises to check all non-vectorcall function calls.
DEOPT_IF(tstate->c_recursion_remaining <= 0);
STAT_INC(CALL, hit);
- PyCFunction cfunc = PyCFunction_GET_FUNCTION(callable);
- PyObject *arg = args[0];
+ PyCFunction cfunc = PyCFunction_GET_FUNCTION(callable_o);
+ _PyStackRef arg = args[0];
_Py_EnterRecursiveCallTstateUnchecked(tstate);
- res = _PyCFunction_TrampolineCall(cfunc, PyCFunction_GET_SELF(callable), arg);
+ PyObject *res_o = _PyCFunction_TrampolineCall(cfunc, PyCFunction_GET_SELF(callable_o), PyStackRef_AsPyObjectBorrow(arg));
_Py_LeaveRecursiveCallTstate(tstate);
- assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL));
+ assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL));
- Py_DECREF(arg);
- Py_DECREF(callable);
- ERROR_IF(res == NULL, error);
+ PyStackRef_CLOSE(arg);
+ PyStackRef_CLOSE(callable);
+ ERROR_IF(res_o == NULL, error);
+ res = PyStackRef_FromPyObjectSteal(res_o);
}
macro(CALL_BUILTIN_O) =
op(_CALL_BUILTIN_FAST, (callable, self_or_null, args[oparg] -- res)) {
/* Builtin METH_FASTCALL functions, without keywords */
+ PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable);
+
int total_args = oparg;
- if (self_or_null != NULL) {
+ if (!PyStackRef_IsNull(self_or_null)) {
args--;
total_args++;
}
- DEOPT_IF(!PyCFunction_CheckExact(callable));
- DEOPT_IF(PyCFunction_GET_FLAGS(callable) != METH_FASTCALL);
+ DEOPT_IF(!PyCFunction_CheckExact(callable_o));
+ DEOPT_IF(PyCFunction_GET_FLAGS(callable_o) != METH_FASTCALL);
STAT_INC(CALL, hit);
- PyCFunction cfunc = PyCFunction_GET_FUNCTION(callable);
+ PyCFunction cfunc = PyCFunction_GET_FUNCTION(callable_o);
/* res = func(self, args, nargs) */
- res = ((PyCFunctionFast)(void(*)(void))cfunc)(
- PyCFunction_GET_SELF(callable),
- args,
+ STACKREFS_TO_PYOBJECTS(args, total_args, args_o);
+ if (CONVERSION_FAILED(args_o)) {
+ DECREF_INPUTS();
+ ERROR_IF(true, error);
+ }
+ PyObject *res_o = ((PyCFunctionFast)(void(*)(void))cfunc)(
+ PyCFunction_GET_SELF(callable_o),
+ args_o,
total_args);
- assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL));
+ STACKREFS_TO_PYOBJECTS_CLEANUP(args_o);
+ assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL));
/* Free the arguments. */
for (int i = 0; i < total_args; i++) {
- Py_DECREF(args[i]);
+ PyStackRef_CLOSE(args[i]);
}
- Py_DECREF(callable);
- ERROR_IF(res == NULL, error);
+ PyStackRef_CLOSE(callable);
+ ERROR_IF(res_o == NULL, error);
+ res = PyStackRef_FromPyObjectSteal(res_o);
}
macro(CALL_BUILTIN_FAST) =
op(_CALL_BUILTIN_FAST_WITH_KEYWORDS, (callable, self_or_null, args[oparg] -- res)) {
/* Builtin METH_FASTCALL | METH_KEYWORDS functions */
+ PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable);
+
int total_args = oparg;
- if (self_or_null != NULL) {
+ if (!PyStackRef_IsNull(self_or_null)) {
args--;
total_args++;
}
- DEOPT_IF(!PyCFunction_CheckExact(callable));
- DEOPT_IF(PyCFunction_GET_FLAGS(callable) != (METH_FASTCALL | METH_KEYWORDS));
+ DEOPT_IF(!PyCFunction_CheckExact(callable_o));
+ DEOPT_IF(PyCFunction_GET_FLAGS(callable_o) != (METH_FASTCALL | METH_KEYWORDS));
STAT_INC(CALL, hit);
/* res = func(self, args, nargs, kwnames) */
PyCFunctionFastWithKeywords cfunc =
(PyCFunctionFastWithKeywords)(void(*)(void))
- PyCFunction_GET_FUNCTION(callable);
- res = cfunc(PyCFunction_GET_SELF(callable), args, total_args, NULL);
- assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL));
+ PyCFunction_GET_FUNCTION(callable_o);
+
+ STACKREFS_TO_PYOBJECTS(args, total_args, args_o);
+ if (CONVERSION_FAILED(args_o)) {
+ DECREF_INPUTS();
+ ERROR_IF(true, error);
+ }
+ PyObject *res_o = cfunc(PyCFunction_GET_SELF(callable_o), args_o, total_args, NULL);
+ STACKREFS_TO_PYOBJECTS_CLEANUP(args_o);
+
+ assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL));
/* Free the arguments. */
for (int i = 0; i < total_args; i++) {
- Py_DECREF(args[i]);
+ PyStackRef_CLOSE(args[i]);
}
- Py_DECREF(callable);
- ERROR_IF(res == NULL, error);
+ PyStackRef_CLOSE(callable);
+ ERROR_IF(res_o == NULL, error);
+ res = PyStackRef_FromPyObjectSteal(res_o);
}
macro(CALL_BUILTIN_FAST_WITH_KEYWORDS) =
inst(CALL_LEN, (unused/1, unused/2, callable, self_or_null, args[oparg] -- res)) {
/* len(o) */
+ PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable);
+
int total_args = oparg;
- if (self_or_null != NULL) {
+ if (!PyStackRef_IsNull(self_or_null)) {
args--;
total_args++;
}
DEOPT_IF(total_args != 1);
PyInterpreterState *interp = tstate->interp;
- DEOPT_IF(callable != interp->callable_cache.len);
+ DEOPT_IF(callable_o != interp->callable_cache.len);
STAT_INC(CALL, hit);
- PyObject *arg = args[0];
+ _PyStackRef arg_stackref = args[0];
+ PyObject *arg = PyStackRef_AsPyObjectBorrow(arg_stackref);
Py_ssize_t len_i = PyObject_Length(arg);
if (len_i < 0) {
ERROR_NO_POP();
}
- res = PyLong_FromSsize_t(len_i);
- assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL));
- if (res == NULL) {
+ PyObject *res_o = PyLong_FromSsize_t(len_i);
+ assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL));
+ if (res_o == NULL) {
GOTO_ERROR(error);
}
- Py_DECREF(callable);
- Py_DECREF(arg);
+ PyStackRef_CLOSE(callable);
+ PyStackRef_CLOSE(arg_stackref);
+ res = PyStackRef_FromPyObjectSteal(res_o);
}
inst(CALL_ISINSTANCE, (unused/1, unused/2, callable, self_or_null, args[oparg] -- res)) {
/* isinstance(o, o2) */
+ PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable);
+
int total_args = oparg;
- if (self_or_null != NULL) {
+ if (!PyStackRef_IsNull(self_or_null)) {
args--;
total_args++;
}
DEOPT_IF(total_args != 2);
PyInterpreterState *interp = tstate->interp;
- DEOPT_IF(callable != interp->callable_cache.isinstance);
+ DEOPT_IF(callable_o != interp->callable_cache.isinstance);
STAT_INC(CALL, hit);
- PyObject *cls = args[1];
- PyObject *inst = args[0];
- int retval = PyObject_IsInstance(inst, cls);
+ _PyStackRef cls_stackref = args[1];
+ _PyStackRef inst_stackref = args[0];
+ int retval = PyObject_IsInstance(PyStackRef_AsPyObjectBorrow(inst_stackref), PyStackRef_AsPyObjectBorrow(cls_stackref));
if (retval < 0) {
ERROR_NO_POP();
}
- res = PyBool_FromLong(retval);
- assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL));
- if (res == NULL) {
- GOTO_ERROR(error);
- }
- Py_DECREF(inst);
- Py_DECREF(cls);
- Py_DECREF(callable);
+ res = retval ? PyStackRef_True : PyStackRef_False;
+ assert((!PyStackRef_IsNull(res)) ^ (_PyErr_Occurred(tstate) != NULL));
+ PyStackRef_CLOSE(inst_stackref);
+ PyStackRef_CLOSE(cls_stackref);
+ PyStackRef_CLOSE(callable);
}
// This is secretly a super-instruction
tier1 inst(CALL_LIST_APPEND, (unused/1, unused/2, callable, self, arg -- unused)) {
assert(oparg == 1);
+ PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable);
+ PyObject *self_o = PyStackRef_AsPyObjectBorrow(self);
+
PyInterpreterState *interp = tstate->interp;
- DEOPT_IF(callable != interp->callable_cache.list_append);
- assert(self != NULL);
- DEOPT_IF(!PyList_Check(self));
+ DEOPT_IF(callable_o != interp->callable_cache.list_append);
+ assert(self_o != NULL);
+ DEOPT_IF(!PyList_Check(self_o));
STAT_INC(CALL, hit);
- if (_PyList_AppendTakeRef((PyListObject *)self, arg) < 0) {
+ if (_PyList_AppendTakeRef((PyListObject *)self_o, PyStackRef_AsPyObjectSteal(arg)) < 0) {
goto pop_1_error; // Since arg is DECREF'ed already
}
- Py_DECREF(self);
- Py_DECREF(callable);
+ PyStackRef_CLOSE(self);
+ PyStackRef_CLOSE(callable);
STACK_SHRINK(3);
// Skip POP_TOP
assert(next_instr->op.code == POP_TOP);
}
op(_CALL_METHOD_DESCRIPTOR_O, (callable, self_or_null, args[oparg] -- res)) {
+ PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable);
+
int total_args = oparg;
- if (self_or_null != NULL) {
+ if (!PyStackRef_IsNull(self_or_null)) {
args--;
total_args++;
}
- PyMethodDescrObject *method = (PyMethodDescrObject *)callable;
+
+ PyMethodDescrObject *method = (PyMethodDescrObject *)callable_o;
DEOPT_IF(total_args != 2);
DEOPT_IF(!Py_IS_TYPE(method, &PyMethodDescr_Type));
PyMethodDef *meth = method->d_method;
DEOPT_IF(meth->ml_flags != METH_O);
// CPython promises to check all non-vectorcall function calls.
DEOPT_IF(tstate->c_recursion_remaining <= 0);
- PyObject *arg = args[1];
- PyObject *self = args[0];
- DEOPT_IF(!Py_IS_TYPE(self, method->d_common.d_type));
+ _PyStackRef arg_stackref = args[1];
+ _PyStackRef self_stackref = args[0];
+ DEOPT_IF(!Py_IS_TYPE(PyStackRef_AsPyObjectBorrow(self_stackref),
+ method->d_common.d_type));
STAT_INC(CALL, hit);
PyCFunction cfunc = meth->ml_meth;
_Py_EnterRecursiveCallTstateUnchecked(tstate);
- res = _PyCFunction_TrampolineCall(cfunc, self, arg);
+ PyObject *res_o = _PyCFunction_TrampolineCall(cfunc,
+ PyStackRef_AsPyObjectBorrow(self_stackref),
+ PyStackRef_AsPyObjectBorrow(arg_stackref));
_Py_LeaveRecursiveCallTstate(tstate);
- assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL));
- Py_DECREF(self);
- Py_DECREF(arg);
- Py_DECREF(callable);
- ERROR_IF(res == NULL, error);
+ assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL));
+ PyStackRef_CLOSE(self_stackref);
+ PyStackRef_CLOSE(arg_stackref);
+ PyStackRef_CLOSE(callable);
+ ERROR_IF(res_o == NULL, error);
+ res = PyStackRef_FromPyObjectSteal(res_o);
}
macro(CALL_METHOD_DESCRIPTOR_O) =
_CHECK_PERIODIC;
op(_CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS, (callable, self_or_null, args[oparg] -- res)) {
+ PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable);
+
int total_args = oparg;
- if (self_or_null != NULL) {
+ if (!PyStackRef_IsNull(self_or_null)) {
args--;
total_args++;
}
- PyMethodDescrObject *method = (PyMethodDescrObject *)callable;
+ PyMethodDescrObject *method = (PyMethodDescrObject *)callable_o;
DEOPT_IF(!Py_IS_TYPE(method, &PyMethodDescr_Type));
PyMethodDef *meth = method->d_method;
DEOPT_IF(meth->ml_flags != (METH_FASTCALL|METH_KEYWORDS));
PyTypeObject *d_type = method->d_common.d_type;
- PyObject *self = args[0];
+ PyObject *self = PyStackRef_AsPyObjectBorrow(args[0]);
DEOPT_IF(!Py_IS_TYPE(self, d_type));
STAT_INC(CALL, hit);
int nargs = total_args - 1;
PyCFunctionFastWithKeywords cfunc =
(PyCFunctionFastWithKeywords)(void(*)(void))meth->ml_meth;
- res = cfunc(self, args + 1, nargs, NULL);
- assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL));
+
+ STACKREFS_TO_PYOBJECTS(args, nargs, args_o);
+ if (CONVERSION_FAILED(args_o)) {
+ DECREF_INPUTS();
+ ERROR_IF(true, error);
+ }
+ PyObject *res_o = cfunc(self, (args_o + 1), nargs, NULL);
+ STACKREFS_TO_PYOBJECTS_CLEANUP(args_o);
+ assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL));
/* Free the arguments. */
for (int i = 0; i < total_args; i++) {
- Py_DECREF(args[i]);
+ PyStackRef_CLOSE(args[i]);
}
- Py_DECREF(callable);
- ERROR_IF(res == NULL, error);
+ PyStackRef_CLOSE(callable);
+ ERROR_IF(res_o == NULL, error);
+ res = PyStackRef_FromPyObjectSteal(res_o);
}
macro(CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS) =
op(_CALL_METHOD_DESCRIPTOR_NOARGS, (callable, self_or_null, args[oparg] -- res)) {
assert(oparg == 0 || oparg == 1);
+ PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable);
+
int total_args = oparg;
- if (self_or_null != NULL) {
+ if (!PyStackRef_IsNull(self_or_null)) {
args--;
total_args++;
}
DEOPT_IF(total_args != 1);
- PyMethodDescrObject *method = (PyMethodDescrObject *)callable;
+ PyMethodDescrObject *method = (PyMethodDescrObject *)callable_o;
DEOPT_IF(!Py_IS_TYPE(method, &PyMethodDescr_Type));
PyMethodDef *meth = method->d_method;
- PyObject *self = args[0];
+ _PyStackRef self_stackref = args[0];
+ PyObject *self = PyStackRef_AsPyObjectBorrow(self_stackref);
DEOPT_IF(!Py_IS_TYPE(self, method->d_common.d_type));
DEOPT_IF(meth->ml_flags != METH_NOARGS);
// CPython promises to check all non-vectorcall function calls.
STAT_INC(CALL, hit);
PyCFunction cfunc = meth->ml_meth;
_Py_EnterRecursiveCallTstateUnchecked(tstate);
- res = _PyCFunction_TrampolineCall(cfunc, self, NULL);
+ PyObject *res_o = _PyCFunction_TrampolineCall(cfunc, self, NULL);
_Py_LeaveRecursiveCallTstate(tstate);
- assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL));
- Py_DECREF(self);
- Py_DECREF(callable);
- ERROR_IF(res == NULL, error);
+ assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL));
+ PyStackRef_CLOSE(self_stackref);
+ PyStackRef_CLOSE(callable);
+ ERROR_IF(res_o == NULL, error);
+ res = PyStackRef_FromPyObjectSteal(res_o);
}
macro(CALL_METHOD_DESCRIPTOR_NOARGS) =
_CHECK_PERIODIC;
op(_CALL_METHOD_DESCRIPTOR_FAST, (callable, self_or_null, args[oparg] -- res)) {
+ PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable);
+
int total_args = oparg;
- if (self_or_null != NULL) {
+ if (!PyStackRef_IsNull(self_or_null)) {
args--;
total_args++;
}
- PyMethodDescrObject *method = (PyMethodDescrObject *)callable;
+ PyMethodDescrObject *method = (PyMethodDescrObject *)callable_o;
/* Builtin METH_FASTCALL methods, without keywords */
DEOPT_IF(!Py_IS_TYPE(method, &PyMethodDescr_Type));
PyMethodDef *meth = method->d_method;
DEOPT_IF(meth->ml_flags != METH_FASTCALL);
- PyObject *self = args[0];
+ PyObject *self = PyStackRef_AsPyObjectBorrow(args[0]);
DEOPT_IF(!Py_IS_TYPE(self, method->d_common.d_type));
STAT_INC(CALL, hit);
PyCFunctionFast cfunc =
(PyCFunctionFast)(void(*)(void))meth->ml_meth;
int nargs = total_args - 1;
- res = cfunc(self, args + 1, nargs);
- assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL));
+
+ STACKREFS_TO_PYOBJECTS(args, nargs, args_o);
+ if (CONVERSION_FAILED(args_o)) {
+ DECREF_INPUTS();
+ ERROR_IF(true, error);
+ }
+ PyObject *res_o = cfunc(self, (args_o + 1), nargs);
+ STACKREFS_TO_PYOBJECTS_CLEANUP(args_o);
+ assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL));
+
/* Clear the stack of the arguments. */
for (int i = 0; i < total_args; i++) {
- Py_DECREF(args[i]);
+ PyStackRef_CLOSE(args[i]);
}
- Py_DECREF(callable);
- ERROR_IF(res == NULL, error);
+ PyStackRef_CLOSE(callable);
+ ERROR_IF(res_o == NULL, error);
+ res = PyStackRef_FromPyObjectSteal(res_o);
}
macro(CALL_METHOD_DESCRIPTOR_FAST) =
_CHECK_PERIODIC;
inst(INSTRUMENTED_CALL_KW, ( -- )) {
- int is_meth = PEEK(oparg + 2) != NULL;
+ int is_meth = !PyStackRef_IsNull(PEEK(oparg + 2));
int total_args = oparg + is_meth;
- PyObject *function = PEEK(oparg + 3);
+ PyObject *function = PyStackRef_AsPyObjectBorrow(PEEK(oparg + 3));
PyObject *arg = total_args == 0 ? &_PyInstrumentation_MISSING
- : PEEK(total_args + 1);
+ : PyStackRef_AsPyObjectBorrow(PEEK(total_args + 1));
int err = _Py_call_instrumentation_2args(
tstate, PY_MONITORING_EVENT_CALL,
frame, this_instr, function, arg);
}
inst(CALL_KW, (callable, self_or_null, args[oparg], kwnames -- res)) {
+ PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable);
+ PyObject *self_or_null_o = PyStackRef_AsPyObjectBorrow(self_or_null);
+ PyObject *kwnames_o = PyStackRef_AsPyObjectBorrow(kwnames);
+
// oparg counts all of the args, but *not* self:
int total_args = oparg;
- if (self_or_null != NULL) {
+ if (self_or_null_o != NULL) {
args--;
total_args++;
}
- if (self_or_null == NULL && Py_TYPE(callable) == &PyMethod_Type) {
+ if (self_or_null_o == NULL && Py_TYPE(callable_o) == &PyMethod_Type) {
args--;
total_args++;
- PyObject *self = ((PyMethodObject *)callable)->im_self;
- args[0] = Py_NewRef(self);
- PyObject *method = ((PyMethodObject *)callable)->im_func;
- args[-1] = Py_NewRef(method);
- Py_DECREF(callable);
- callable = method;
- }
- int positional_args = total_args - (int)PyTuple_GET_SIZE(kwnames);
+ PyObject *self = ((PyMethodObject *)callable_o)->im_self;
+ args[0] = PyStackRef_FromPyObjectNew(self);
+ PyObject *method = ((PyMethodObject *)callable_o)->im_func;
+ args[-1] = PyStackRef_FromPyObjectNew(method);
+ PyStackRef_CLOSE(callable);
+ callable_o = method;
+ callable = args[-1];
+ }
+ int positional_args = total_args - (int)PyTuple_GET_SIZE(kwnames_o);
// Check if the call can be inlined or not
- if (Py_TYPE(callable) == &PyFunction_Type &&
+ if (Py_TYPE(callable_o) == &PyFunction_Type &&
tstate->interp->eval_frame == NULL &&
- ((PyFunctionObject *)callable)->vectorcall == _PyFunction_Vectorcall)
+ ((PyFunctionObject *)callable_o)->vectorcall == _PyFunction_Vectorcall)
{
- int code_flags = ((PyCodeObject*)PyFunction_GET_CODE(callable))->co_flags;
- PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable));
+ int code_flags = ((PyCodeObject*)PyFunction_GET_CODE(callable_o))->co_flags;
+ PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable_o));
_PyInterpreterFrame *new_frame = _PyEvalFramePushAndInit(
- tstate, (PyFunctionObject *)callable, locals,
- args, positional_args, kwnames
+ tstate, (PyFunctionObject *)PyStackRef_AsPyObjectSteal(callable), locals,
+ args, positional_args, kwnames_o
);
- Py_DECREF(kwnames);
+ PyStackRef_CLOSE(kwnames);
// Manipulate stack directly since we leave using DISPATCH_INLINED().
STACK_SHRINK(oparg + 3);
// The frame has stolen all the arguments from the stack,
DISPATCH_INLINED(new_frame);
}
/* Callable is not a normal Python function */
- res = PyObject_Vectorcall(
- callable, args,
+ STACKREFS_TO_PYOBJECTS(args, total_args, args_o);
+ if (CONVERSION_FAILED(args_o)) {
+ DECREF_INPUTS();
+ ERROR_IF(true, error);
+ }
+ PyObject *res_o = PyObject_Vectorcall(
+ callable_o, args_o,
positional_args | PY_VECTORCALL_ARGUMENTS_OFFSET,
- kwnames);
+ kwnames_o);
+ STACKREFS_TO_PYOBJECTS_CLEANUP(args_o);
if (opcode == INSTRUMENTED_CALL_KW) {
PyObject *arg = total_args == 0 ?
- &_PyInstrumentation_MISSING : args[0];
- if (res == NULL) {
+ &_PyInstrumentation_MISSING : PyStackRef_AsPyObjectBorrow(args[0]);
+ if (res_o == NULL) {
_Py_call_instrumentation_exc2(
tstate, PY_MONITORING_EVENT_C_RAISE,
- frame, this_instr, callable, arg);
+ frame, this_instr, callable_o, arg);
}
else {
int err = _Py_call_instrumentation_2args(
tstate, PY_MONITORING_EVENT_C_RETURN,
- frame, this_instr, callable, arg);
+ frame, this_instr, callable_o, arg);
if (err < 0) {
- Py_CLEAR(res);
+ Py_CLEAR(res_o);
}
}
}
- Py_DECREF(kwnames);
- assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL));
- Py_DECREF(callable);
+ PyStackRef_CLOSE(kwnames);
+ assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL));
+ PyStackRef_CLOSE(callable);
for (int i = 0; i < total_args; i++) {
- Py_DECREF(args[i]);
+ PyStackRef_CLOSE(args[i]);
}
- ERROR_IF(res == NULL, error);
+ ERROR_IF(res_o == NULL, error);
+ res = PyStackRef_FromPyObjectSteal(res_o);
CHECK_EVAL_BREAKER();
}
GO_TO_INSTRUCTION(CALL_FUNCTION_EX);
}
- inst(CALL_FUNCTION_EX, (func, unused, callargs, kwargs if (oparg & 1) -- result)) {
+ inst(CALL_FUNCTION_EX, (func_st, unused, callargs_st, kwargs_st if (oparg & 1) -- result)) {
+ PyObject *func = PyStackRef_AsPyObjectBorrow(func_st);
+ PyObject *callargs = PyStackRef_AsPyObjectBorrow(callargs_st);
+ PyObject *kwargs = PyStackRef_AsPyObjectBorrow(kwargs_st);
+
// DICT_MERGE is called before this opcode if there are kwargs.
// It converts all dict subtypes in kwargs into regular dicts.
assert(kwargs == NULL || PyDict_CheckExact(kwargs));
if (tuple == NULL) {
ERROR_NO_POP();
}
- Py_SETREF(callargs, tuple);
+ PyStackRef_CLOSE(callargs_st);
+ callargs_st = PyStackRef_FromPyObjectSteal(tuple);
+ callargs = tuple;
}
assert(PyTuple_CheckExact(callargs));
EVAL_CALL_STAT_INC_IF_FUNCTION(EVAL_CALL_FUNCTION_EX, func);
tstate, PY_MONITORING_EVENT_CALL,
frame, this_instr, func, arg);
if (err) ERROR_NO_POP();
- result = PyObject_Call(func, callargs, kwargs);
+ result = PyStackRef_FromPyObjectSteal(PyObject_Call(func, callargs, kwargs));
if (!PyFunction_Check(func) && !PyMethod_Check(func)) {
- if (result == NULL) {
+ if (PyStackRef_IsNull(result)) {
_Py_call_instrumentation_exc2(
tstate, PY_MONITORING_EVENT_C_RAISE,
frame, this_instr, func, arg);
tstate, PY_MONITORING_EVENT_C_RETURN,
frame, this_instr, func, arg);
if (err < 0) {
- Py_CLEAR(result);
+ PyStackRef_CLEAR(result);
}
}
}
PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(func));
_PyInterpreterFrame *new_frame = _PyEvalFramePushAndInit_Ex(tstate,
- (PyFunctionObject *)func, locals,
+ (PyFunctionObject *)PyStackRef_AsPyObjectSteal(func_st), locals,
nargs, callargs, kwargs);
// Need to manually shrink the stack since we exit with DISPATCH_INLINED.
STACK_SHRINK(oparg + 3);
frame->return_offset = 1;
DISPATCH_INLINED(new_frame);
}
- result = PyObject_Call(func, callargs, kwargs);
+ result = PyStackRef_FromPyObjectSteal(PyObject_Call(func, callargs, kwargs));
}
DECREF_INPUTS();
- assert(PEEK(2 + (oparg & 1)) == NULL);
- ERROR_IF(result == NULL, error);
+ assert(PyStackRef_AsPyObjectBorrow(PEEK(2 + (oparg & 1))) == NULL);
+ ERROR_IF(PyStackRef_IsNull(result), error);
CHECK_EVAL_BREAKER();
}
- inst(MAKE_FUNCTION, (codeobj -- func)) {
+ inst(MAKE_FUNCTION, (codeobj_st -- func)) {
+ PyObject *codeobj = PyStackRef_AsPyObjectBorrow(codeobj_st);
PyFunctionObject *func_obj = (PyFunctionObject *)
PyFunction_New(codeobj, GLOBALS());
- Py_DECREF(codeobj);
+ PyStackRef_CLOSE(codeobj_st);
if (func_obj == NULL) {
ERROR_NO_POP();
}
_PyFunction_SetVersion(
func_obj, ((PyCodeObject *)codeobj)->co_version);
- func = (PyObject *)func_obj;
+ func = PyStackRef_FromPyObjectSteal((PyObject *)func_obj);
}
- inst(SET_FUNCTION_ATTRIBUTE, (attr, func -- func)) {
+ inst(SET_FUNCTION_ATTRIBUTE, (attr_st, func_st -- func_st)) {
+ PyObject *func = PyStackRef_AsPyObjectBorrow(func_st);
+ PyObject *attr = PyStackRef_AsPyObjectBorrow(attr_st);
+
assert(PyFunction_Check(func));
PyFunctionObject *func_obj = (PyFunctionObject *)func;
switch(oparg) {
gen->gi_frame_state = FRAME_CREATED;
gen_frame->owner = FRAME_OWNED_BY_GENERATOR;
_Py_LeaveRecursiveCallPy(tstate);
- res = (PyObject *)gen;
+ res = PyStackRef_FromPyObjectSteal((PyObject *)gen);
_PyInterpreterFrame *prev = frame->previous;
_PyThreadState_PopFrame(tstate, frame);
frame = tstate->current_frame = prev;
}
inst(BUILD_SLICE, (start, stop, step if (oparg == 3) -- slice)) {
- slice = PySlice_New(start, stop, step);
+ PyObject *start_o = PyStackRef_AsPyObjectBorrow(start);
+ PyObject *stop_o = PyStackRef_AsPyObjectBorrow(stop);
+ PyObject *step_o = PyStackRef_AsPyObjectBorrow(step);
+
+ PyObject *slice_o = PySlice_New(start_o, stop_o, step_o);
DECREF_INPUTS();
- ERROR_IF(slice == NULL, error);
+ ERROR_IF(slice_o == NULL, error);
+ slice = PyStackRef_FromPyObjectSteal(slice_o);
}
inst(CONVERT_VALUE, (value -- result)) {
conversion_func conv_fn;
assert(oparg >= FVC_STR && oparg <= FVC_ASCII);
conv_fn = _PyEval_ConversionFuncs[oparg];
- result = conv_fn(value);
- Py_DECREF(value);
- ERROR_IF(result == NULL, error);
+ PyObject *result_o = conv_fn(PyStackRef_AsPyObjectBorrow(value));
+ PyStackRef_CLOSE(value);
+ ERROR_IF(result_o == NULL, error);
+ result = PyStackRef_FromPyObjectSteal(result_o);
}
inst(FORMAT_SIMPLE, (value -- res)) {
+ PyObject *value_o = PyStackRef_AsPyObjectBorrow(value);
/* If value is a unicode object, then we know the result
* of format(value) is value itself. */
- if (!PyUnicode_CheckExact(value)) {
- res = PyObject_Format(value, NULL);
- Py_DECREF(value);
- ERROR_IF(res == NULL, error);
+ if (!PyUnicode_CheckExact(value_o)) {
+ res = PyStackRef_FromPyObjectSteal(PyObject_Format(value_o, NULL));
+ PyStackRef_CLOSE(value);
+ ERROR_IF(PyStackRef_IsNull(res), error);
}
else {
res = value;
}
inst(FORMAT_WITH_SPEC, (value, fmt_spec -- res)) {
- res = PyObject_Format(value, fmt_spec);
- Py_DECREF(value);
- Py_DECREF(fmt_spec);
- ERROR_IF(res == NULL, error);
+ PyObject *res_o = PyObject_Format(PyStackRef_AsPyObjectBorrow(value), PyStackRef_AsPyObjectBorrow(fmt_spec));
+ PyStackRef_CLOSE(value);
+ PyStackRef_CLOSE(fmt_spec);
+ ERROR_IF(res_o == NULL, error);
+ res = PyStackRef_FromPyObjectSteal(res_o);
}
pure inst(COPY, (bottom, unused[oparg-1] -- bottom, unused[oparg-1], top)) {
assert(oparg > 0);
- top = Py_NewRef(bottom);
+ top = PyStackRef_DUP(bottom);
}
specializing op(_SPECIALIZE_BINARY_OP, (counter/1, lhs, rhs -- lhs, rhs)) {
}
op(_BINARY_OP, (lhs, rhs -- res)) {
+ PyObject *lhs_o = PyStackRef_AsPyObjectBorrow(lhs);
+ PyObject *rhs_o = PyStackRef_AsPyObjectBorrow(rhs);
+
assert(_PyEval_BinaryOps[oparg]);
- res = _PyEval_BinaryOps[oparg](lhs, rhs);
+ PyObject *res_o = _PyEval_BinaryOps[oparg](lhs_o, rhs_o);
DECREF_INPUTS();
- ERROR_IF(res == NULL, error);
+ ERROR_IF(res_o == NULL, error);
+ res = PyStackRef_FromPyObjectSteal(res_o);
}
macro(BINARY_OP) = _SPECIALIZE_BINARY_OP + _BINARY_OP;
}
inst(INSTRUMENTED_POP_JUMP_IF_TRUE, (unused/1 -- )) {
- PyObject *cond = POP();
- assert(PyBool_Check(cond));
- int flag = Py_IsTrue(cond);
+ _PyStackRef cond = POP();
+ assert(PyBool_Check(PyStackRef_AsPyObjectBorrow(cond)));
+ int flag = PyStackRef_Is(cond, PyStackRef_True);
int offset = flag * oparg;
#if ENABLE_SPECIALIZATION
this_instr[1].cache = (this_instr[1].cache << 1) | flag;
}
inst(INSTRUMENTED_POP_JUMP_IF_FALSE, (unused/1 -- )) {
- PyObject *cond = POP();
- assert(PyBool_Check(cond));
- int flag = Py_IsFalse(cond);
+ _PyStackRef cond = POP();
+ assert(PyBool_Check(PyStackRef_AsPyObjectBorrow(cond)));
+ int flag = PyStackRef_Is(cond, PyStackRef_False);
int offset = flag * oparg;
#if ENABLE_SPECIALIZATION
this_instr[1].cache = (this_instr[1].cache << 1) | flag;
}
inst(INSTRUMENTED_POP_JUMP_IF_NONE, (unused/1 -- )) {
- PyObject *value = POP();
- int flag = Py_IsNone(value);
+ _PyStackRef value_stackref = POP();
+ int flag = PyStackRef_Is(value_stackref, PyStackRef_None);
int offset;
if (flag) {
offset = oparg;
}
else {
- Py_DECREF(value);
+ PyStackRef_CLOSE(value_stackref);
offset = 0;
}
#if ENABLE_SPECIALIZATION
}
inst(INSTRUMENTED_POP_JUMP_IF_NOT_NONE, (unused/1 -- )) {
- PyObject *value = POP();
+ _PyStackRef value_stackref = POP();
int offset;
- int nflag = Py_IsNone(value);
+ int nflag = PyStackRef_Is(value_stackref, PyStackRef_None);
if (nflag) {
offset = 0;
}
else {
- Py_DECREF(value);
+ PyStackRef_CLOSE(value_stackref);
offset = oparg;
}
#if ENABLE_SPECIALIZATION
op (_GUARD_IS_TRUE_POP, (flag -- )) {
SYNC_SP();
- EXIT_IF(!Py_IsTrue(flag));
- assert(Py_IsTrue(flag));
+ EXIT_IF(!PyStackRef_Is(flag, PyStackRef_True));
+ assert(PyStackRef_Is(flag, PyStackRef_True));
}
op (_GUARD_IS_FALSE_POP, (flag -- )) {
SYNC_SP();
- EXIT_IF(!Py_IsFalse(flag));
- assert(Py_IsFalse(flag));
+ EXIT_IF(!PyStackRef_Is(flag, PyStackRef_False));
+ assert(PyStackRef_Is(flag, PyStackRef_False));
}
op (_GUARD_IS_NONE_POP, (val -- )) {
SYNC_SP();
- if (!Py_IsNone(val)) {
- Py_DECREF(val);
+ if (!PyStackRef_Is(val, PyStackRef_None)) {
+ PyStackRef_CLOSE(val);
EXIT_IF(1);
}
}
op (_GUARD_IS_NOT_NONE_POP, (val -- )) {
SYNC_SP();
- EXIT_IF(Py_IsNone(val));
- Py_DECREF(val);
+ EXIT_IF(PyStackRef_Is(val, PyStackRef_None));
+ PyStackRef_CLOSE(val);
}
op(_JUMP_TO_TOP, (--)) {
}
tier2 pure op(_LOAD_CONST_INLINE, (ptr/4 -- value)) {
- value = Py_NewRef(ptr);
+ value = PyStackRef_FromPyObjectNew(ptr);
}
tier2 pure op(_LOAD_CONST_INLINE_BORROW, (ptr/4 -- value)) {
- value = ptr;
+ value = PyStackRef_FromPyObjectImmortal(ptr);
}
tier2 pure op (_POP_TOP_LOAD_CONST_INLINE_BORROW, (ptr/4, pop -- value)) {
- Py_DECREF(pop);
- value = ptr;
+ PyStackRef_CLOSE(pop);
+ value = PyStackRef_FromPyObjectImmortal(ptr);
}
tier2 pure op(_LOAD_CONST_INLINE_WITH_NULL, (ptr/4 -- value, null)) {
- value = Py_NewRef(ptr);
- null = NULL;
+ value = PyStackRef_FromPyObjectNew(ptr);
+ null = PyStackRef_NULL;
}
tier2 pure op(_LOAD_CONST_INLINE_BORROW_WITH_NULL, (ptr/4 -- value, null)) {
- value = ptr;
- null = NULL;
+ value = PyStackRef_FromPyObjectImmortal(ptr);
+ null = PyStackRef_NULL;
}
tier2 op(_CHECK_FUNCTION, (func_version/2 -- )) {
/* Internal -- for testing executors */
op(_INTERNAL_INCREMENT_OPT_COUNTER, (opt --)) {
- _PyCounterOptimizerObject *exe = (_PyCounterOptimizerObject *)opt;
+ _PyCounterOptimizerObject *exe = (_PyCounterOptimizerObject *)PyStackRef_AsPyObjectBorrow(opt);
exe->count++;
}
#include "opcode.h"
#include "pydtrace.h"
#include "setobject.h"
+#include "pycore_stackref.h"
#include <stdbool.h> // bool
#ifdef LLTRACE
static void
-dump_stack(_PyInterpreterFrame *frame, PyObject **stack_pointer)
+dump_stack(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer)
{
- PyObject **stack_base = _PyFrame_Stackbase(frame);
+ _PyStackRef *stack_base = _PyFrame_Stackbase(frame);
PyObject *exc = PyErr_GetRaisedException();
printf(" stack=[");
- for (PyObject **ptr = stack_base; ptr < stack_pointer; ptr++) {
+ for (_PyStackRef *ptr = stack_base; ptr < stack_pointer; ptr++) {
if (ptr != stack_base) {
printf(", ");
}
- if (*ptr == NULL) {
+ PyObject *obj = PyStackRef_AsPyObjectBorrow(*ptr);
+ if (obj == NULL) {
printf("<nil>");
continue;
}
if (
- *ptr == Py_None
- || PyBool_Check(*ptr)
- || PyLong_CheckExact(*ptr)
- || PyFloat_CheckExact(*ptr)
- || PyUnicode_CheckExact(*ptr)
+ obj == Py_None
+ || PyBool_Check(obj)
+ || PyLong_CheckExact(obj)
+ || PyFloat_CheckExact(obj)
+ || PyUnicode_CheckExact(obj)
) {
- if (PyObject_Print(*ptr, stdout, 0) == 0) {
+ if (PyObject_Print(obj, stdout, 0) == 0) {
continue;
}
PyErr_Clear();
}
// Don't call __repr__(), it might recurse into the interpreter.
- printf("<%s at %p>", Py_TYPE(*ptr)->tp_name, (void *)(*ptr));
+ printf("<%s at %p>", Py_TYPE(obj)->tp_name, (void *)(ptr->bits));
}
printf("]\n");
fflush(stdout);
static void
lltrace_instruction(_PyInterpreterFrame *frame,
- PyObject **stack_pointer,
+ _PyStackRef *stack_pointer,
_Py_CODEUNIT *next_instr,
int opcode,
int oparg)
#endif
+PyObject **
+_PyObjectArray_FromStackRefArray(_PyStackRef *input, Py_ssize_t nargs, PyObject **scratch)
+{
+ PyObject **result;
+ if (nargs > MAX_STACKREF_SCRATCH) {
+ // +1 in case PY_VECTORCALL_ARGUMENTS_OFFSET is set.
+ result = PyMem_Malloc((nargs + 1) * sizeof(PyObject *));
+ if (result == NULL) {
+ return NULL;
+ }
+ result++;
+ }
+ else {
+ result = scratch;
+ }
+ for (int i = 0; i < nargs; i++) {
+ result[i] = PyStackRef_AsPyObjectBorrow(input[i]);
+ }
+ return result;
+}
+
+void
+_PyObjectArray_Free(PyObject **array, PyObject **scratch)
+{
+ if (array != scratch) {
+ PyMem_Free(array);
+ }
+}
+
/* _PyEval_EvalFrameDefault() is a *big* function,
* so consume 3 units of C stack */
#define PY_EVAL_C_STACK_UNITS 2
/* Local "register" variables.
* These are cached values from the frame and code object. */
_Py_CODEUNIT *next_instr;
- PyObject **stack_pointer;
+ _PyStackRef *stack_pointer;
#if defined(_Py_TIER2) && !defined(_Py_JIT)
/* Tier 2 interpreter state */
assert(_PyErr_Occurred(tstate));
/* Pop remaining stack entries. */
- PyObject **stackbase = _PyFrame_Stackbase(frame);
+ _PyStackRef *stackbase = _PyFrame_Stackbase(frame);
while (stack_pointer > stackbase) {
- PyObject *o = POP();
- Py_XDECREF(o);
+ PyStackRef_XCLOSE(POP());
}
assert(STACK_LEVEL() == 0);
_PyFrame_SetStackPointer(frame, stack_pointer);
}
assert(STACK_LEVEL() >= level);
- PyObject **new_top = _PyFrame_Stackbase(frame) + level;
+ _PyStackRef *new_top = _PyFrame_Stackbase(frame) + level;
while (stack_pointer > new_top) {
- PyObject *v = POP();
- Py_XDECREF(v);
+ PyStackRef_XCLOSE(POP());
}
if (lasti) {
int frame_lasti = _PyInterpreterFrame_LASTI(frame);
if (lasti == NULL) {
goto exception_unwind;
}
- PUSH(lasti);
+ PUSH(PyStackRef_FromPyObjectSteal(lasti));
}
/* Make the raw exception data
so a program can emulate the
Python main loop. */
PyObject *exc = _PyErr_GetRaisedException(tstate);
- PUSH(exc);
+ PUSH(PyStackRef_FromPyObjectSteal(exc));
next_instr = _PyCode_CODE(_PyFrame_GetCode(frame)) + handler;
if (monitor_handled(tstate, frame, next_instr, exc) < 0) {
static void
missing_arguments(PyThreadState *tstate, PyCodeObject *co,
Py_ssize_t missing, Py_ssize_t defcount,
- PyObject **localsplus, PyObject *qualname)
+ _PyStackRef *localsplus, PyObject *qualname)
{
Py_ssize_t i, j = 0;
Py_ssize_t start, end;
end = start + co->co_kwonlyargcount;
}
for (i = start; i < end; i++) {
- if (localsplus[i] == NULL) {
+ if (PyStackRef_IsNull(localsplus[i])) {
PyObject *raw = PyTuple_GET_ITEM(co->co_localsplusnames, i);
PyObject *name = PyObject_Repr(raw);
if (name == NULL) {
static void
too_many_positional(PyThreadState *tstate, PyCodeObject *co,
Py_ssize_t given, PyObject *defaults,
- PyObject **localsplus, PyObject *qualname)
+ _PyStackRef *localsplus, PyObject *qualname)
{
int plural;
Py_ssize_t kwonly_given = 0;
assert((co->co_flags & CO_VARARGS) == 0);
/* Count missing keyword-only args. */
for (i = co_argcount; i < co_argcount + co->co_kwonlyargcount; i++) {
- if (localsplus[i] != NULL) {
+ if (PyStackRef_AsPyObjectBorrow(localsplus[i]) != NULL) {
kwonly_given++;
}
}
static int
initialize_locals(PyThreadState *tstate, PyFunctionObject *func,
- PyObject **localsplus, PyObject *const *args,
+ _PyStackRef *localsplus, _PyStackRef const *args,
Py_ssize_t argcount, PyObject *kwnames)
{
PyCodeObject *co = (PyCodeObject*)func->func_code;
if (co->co_flags & CO_VARARGS) {
i++;
}
- assert(localsplus[i] == NULL);
- localsplus[i] = kwdict;
+ assert(PyStackRef_IsNull(localsplus[i]));
+ localsplus[i] = PyStackRef_FromPyObjectSteal(kwdict);
}
else {
kwdict = NULL;
n = argcount;
}
for (j = 0; j < n; j++) {
- PyObject *x = args[j];
- assert(localsplus[j] == NULL);
- localsplus[j] = x;
+ assert(PyStackRef_IsNull(localsplus[j]));
+ localsplus[j] = args[j];
}
/* Pack other positional arguments into the *args argument */
}
else {
assert(args != NULL);
- u = _PyTuple_FromArraySteal(args + n, argcount - n);
+ STACKREFS_TO_PYOBJECTS((_PyStackRef *)args, argcount, args_o);
+ if (args_o == NULL) {
+ goto fail_pre_positional;
+ }
+ u = _PyTuple_FromArraySteal((args_o + n), argcount - n);
+ STACKREFS_TO_PYOBJECTS_CLEANUP(args_o);
}
if (u == NULL) {
goto fail_post_positional;
}
- assert(localsplus[total_args] == NULL);
- localsplus[total_args] = u;
+ assert(PyStackRef_AsPyObjectBorrow(localsplus[total_args]) == NULL);
+ localsplus[total_args] = PyStackRef_FromPyObjectSteal(u);
}
else if (argcount > n) {
/* Too many positional args. Error is reported later */
for (j = n; j < argcount; j++) {
- Py_DECREF(args[j]);
+ PyStackRef_CLOSE(args[j]);
}
}
for (i = 0; i < kwcount; i++) {
PyObject **co_varnames;
PyObject *keyword = PyTuple_GET_ITEM(kwnames, i);
- PyObject *value = args[i+argcount];
+ _PyStackRef value_stackref = args[i+argcount];
Py_ssize_t j;
if (keyword == NULL || !PyUnicode_Check(keyword)) {
goto kw_fail;
}
- if (PyDict_SetItem(kwdict, keyword, value) == -1) {
+ if (PyDict_SetItem(kwdict, keyword, PyStackRef_AsPyObjectSteal(value_stackref)) == -1) {
goto kw_fail;
}
- Py_DECREF(value);
+ PyStackRef_CLOSE(value_stackref);
continue;
kw_fail:
for (;i < kwcount; i++) {
- PyObject *value = args[i+argcount];
- Py_DECREF(value);
+ PyStackRef_CLOSE(args[i+argcount]);
}
goto fail_post_args;
kw_found:
- if (localsplus[j] != NULL) {
+ if (PyStackRef_AsPyObjectBorrow(localsplus[j]) != NULL) {
_PyErr_Format(tstate, PyExc_TypeError,
"%U() got multiple values for argument '%S'",
func->func_qualname, keyword);
goto kw_fail;
}
- localsplus[j] = value;
+ localsplus[j] = value_stackref;
}
}
Py_ssize_t m = co->co_argcount - defcount;
Py_ssize_t missing = 0;
for (i = argcount; i < m; i++) {
- if (localsplus[i] == NULL) {
+ if (PyStackRef_IsNull(localsplus[i])) {
missing++;
}
}
if (defcount) {
PyObject **defs = &PyTuple_GET_ITEM(func->func_defaults, 0);
for (; i < defcount; i++) {
- if (localsplus[m+i] == NULL) {
+ if (PyStackRef_AsPyObjectBorrow(localsplus[m+i]) == NULL) {
PyObject *def = defs[i];
- localsplus[m+i] = Py_NewRef(def);
+ localsplus[m+i] = PyStackRef_FromPyObjectNew(def);
}
}
}
if (co->co_kwonlyargcount > 0) {
Py_ssize_t missing = 0;
for (i = co->co_argcount; i < total_args; i++) {
- if (localsplus[i] != NULL)
+ if (PyStackRef_AsPyObjectBorrow(localsplus[i]) != NULL)
continue;
PyObject *varname = PyTuple_GET_ITEM(co->co_localsplusnames, i);
if (func->func_kwdefaults != NULL) {
goto fail_post_args;
}
if (def) {
- localsplus[i] = def;
+ localsplus[i] = PyStackRef_FromPyObjectSteal(def);
continue;
}
}
fail_pre_positional:
for (j = 0; j < argcount; j++) {
- Py_DECREF(args[j]);
+ PyStackRef_CLOSE(args[j]);
}
/* fall through */
fail_post_positional:
if (kwnames) {
Py_ssize_t kwcount = PyTuple_GET_SIZE(kwnames);
for (j = argcount; j < argcount+kwcount; j++) {
- Py_DECREF(args[j]);
+ PyStackRef_CLOSE(args[j]);
}
}
/* fall through */
/* Consumes references to func, locals and all the args */
_PyInterpreterFrame *
_PyEvalFramePushAndInit(PyThreadState *tstate, PyFunctionObject *func,
- PyObject *locals, PyObject* const* args,
+ PyObject *locals, _PyStackRef const* args,
size_t argcount, PyObject *kwnames)
{
PyCodeObject * code = (PyCodeObject *)func->func_code;
Py_DECREF(func);
Py_XDECREF(locals);
for (size_t i = 0; i < argcount; i++) {
- Py_DECREF(args[i]);
+ PyStackRef_CLOSE(args[i]);
}
if (kwnames) {
Py_ssize_t kwcount = PyTuple_GET_SIZE(kwnames);
for (Py_ssize_t i = 0; i < kwcount; i++) {
- Py_DECREF(args[i+argcount]);
+ PyStackRef_CLOSE(args[i+argcount]);
}
}
PyErr_NoMemory();
return NULL;
}
+static _PyInterpreterFrame *
+_PyEvalFramePushAndInit_UnTagged(PyThreadState *tstate, PyFunctionObject *func,
+ PyObject *locals, PyObject *const* args,
+ size_t argcount, PyObject *kwnames)
+{
+#if defined(Py_GIL_DISABLED)
+ size_t kw_count = kwnames == NULL ? 0 : PyTuple_GET_SIZE(kwnames);
+ size_t total_argcount = argcount + kw_count;
+ _PyStackRef *tagged_args_buffer = PyMem_Malloc(sizeof(_PyStackRef) * total_argcount);
+ if (tagged_args_buffer == NULL) {
+ PyErr_NoMemory();
+ return NULL;
+ }
+ for (size_t i = 0; i < argcount; i++) {
+ tagged_args_buffer[i] = PyStackRef_FromPyObjectSteal(args[i]);
+ }
+ for (size_t i = 0; i < kw_count; i++) {
+ tagged_args_buffer[argcount + i] = PyStackRef_FromPyObjectSteal(args[argcount + i]);
+ }
+ _PyInterpreterFrame *res = _PyEvalFramePushAndInit(tstate, func, locals, (_PyStackRef const *)tagged_args_buffer, argcount, kwnames);
+ PyMem_Free(tagged_args_buffer);
+ return res;
+#else
+ return _PyEvalFramePushAndInit(tstate, func, locals, (_PyStackRef const *)args, argcount, kwnames);
+#endif
+}
+
/* Same as _PyEvalFramePushAndInit but takes an args tuple and kwargs dict.
Steals references to func, callargs and kwargs.
*/
Py_INCREF(PyTuple_GET_ITEM(callargs, i));
}
}
- _PyInterpreterFrame *new_frame = _PyEvalFramePushAndInit(
+ _PyInterpreterFrame *new_frame = _PyEvalFramePushAndInit_UnTagged(
tstate, (PyFunctionObject *)func, locals,
newargs, nargs, kwnames
);
Py_INCREF(args[i+argcount]);
}
}
- _PyInterpreterFrame *frame = _PyEvalFramePushAndInit(
+ _PyInterpreterFrame *frame = _PyEvalFramePushAndInit_UnTagged(
tstate, func, locals, args, argcount, kwnames);
if (frame == NULL) {
return NULL;
*/
int
-_PyEval_UnpackIterable(PyThreadState *tstate, PyObject *v,
- int argcnt, int argcntafter, PyObject **sp)
+_PyEval_UnpackIterableStackRef(PyThreadState *tstate, _PyStackRef v_stackref,
+ int argcnt, int argcntafter, _PyStackRef *sp)
{
int i = 0, j = 0;
Py_ssize_t ll = 0;
PyObject *w;
PyObject *l = NULL; /* variable list */
+ PyObject *v = PyStackRef_AsPyObjectBorrow(v_stackref);
assert(v != NULL);
it = PyObject_GetIter(v);
}
goto Error;
}
- *--sp = w;
+ *--sp = PyStackRef_FromPyObjectSteal(w);
}
if (argcntafter == -1) {
l = PySequence_List(it);
if (l == NULL)
goto Error;
- *--sp = l;
+ *--sp = PyStackRef_FromPyObjectSteal(l);
i++;
ll = PyList_GET_SIZE(l);
/* Pop the "after-variable" args off the list. */
for (j = argcntafter; j > 0; j--, i++) {
- *--sp = PyList_GET_ITEM(l, ll - j);
+ *--sp = PyStackRef_FromPyObjectSteal(PyList_GET_ITEM(l, ll - j));
}
/* Resize the list. */
Py_SET_SIZE(l, ll - argcntafter);
return 1;
Error:
- for (; i > 0; i--, sp++)
- Py_DECREF(*sp);
+ for (; i > 0; i--, sp++) {
+ PyStackRef_CLOSE(*sp);
+ }
Py_XDECREF(it);
return 0;
}
This is because it is possible that during the DECREF the frame is
accessed by other code (e.g. a __del__ method or gc.collect()) and the
variable would be pointing to already-freed memory. */
-#define SETLOCAL(i, value) do { PyObject *tmp = GETLOCAL(i); \
+#define SETLOCAL(i, value) do { _PyStackRef tmp = GETLOCAL(i); \
GETLOCAL(i) = value; \
- Py_XDECREF(tmp); } while (0)
+ PyStackRef_XCLOSE(tmp); } while (0)
#define GO_TO_INSTRUCTION(op) goto PREDICT_ID(op)
#define EXIT_TO_TRACE() goto exit_to_trace
#define EXIT_TO_TIER1() goto exit_to_tier1
#define EXIT_TO_TIER1_DYNAMIC() goto exit_to_tier1_dynamic;
+
+/* Stackref macros */
+
+/* How much scratch space to give stackref to PyObject* conversion. */
+#define MAX_STACKREF_SCRATCH 10
+
+#ifdef Py_GIL_DISABLED
+#define STACKREFS_TO_PYOBJECTS(ARGS, ARG_COUNT, NAME) \
+ /* +1 because vectorcall might use -1 to write self */ \
+ PyObject *NAME##_temp[MAX_STACKREF_SCRATCH+1]; \
+ PyObject **NAME = _PyObjectArray_FromStackRefArray(ARGS, ARG_COUNT, NAME##_temp + 1);
+#else
+#define STACKREFS_TO_PYOBJECTS(ARGS, ARG_COUNT, NAME) \
+ PyObject **NAME = (PyObject **)ARGS; \
+ assert(NAME != NULL);
+#endif
+
+#ifdef Py_GIL_DISABLED
+#define STACKREFS_TO_PYOBJECTS_CLEANUP(NAME) \
+ /* +1 because we +1 previously */ \
+ _PyObjectArray_Free(NAME - 1, NAME##_temp);
+#else
+#define STACKREFS_TO_PYOBJECTS_CLEANUP(NAME) \
+ (void)(NAME);
+#endif
+
+#ifdef Py_GIL_DISABLED
+#define CONVERSION_FAILED(NAME) ((NAME) == NULL)
+#else
+#define CONVERSION_FAILED(NAME) (0)
+#endif
/* _INSTRUMENTED_RESUME is not a viable micro-op for tier 2 because it is instrumented */
case _LOAD_FAST_CHECK: {
- PyObject *value;
+ _PyStackRef value;
oparg = CURRENT_OPARG();
- value = GETLOCAL(oparg);
- if (value == NULL) {
+ _PyStackRef value_s = GETLOCAL(oparg);
+ if (PyStackRef_IsNull(value_s)) {
_PyEval_FormatExcCheckArg(tstate, PyExc_UnboundLocalError,
UNBOUNDLOCAL_ERROR_MSG,
PyTuple_GetItem(_PyFrame_GetCode(frame)->co_localsplusnames, oparg)
);
if (1) JUMP_TO_ERROR();
}
- Py_INCREF(value);
+ value = PyStackRef_DUP(value_s);
stack_pointer[0] = value;
stack_pointer += 1;
assert(WITHIN_STACK_BOUNDS());
}
case _LOAD_FAST_0: {
- PyObject *value;
+ _PyStackRef value;
oparg = 0;
assert(oparg == CURRENT_OPARG());
- value = GETLOCAL(oparg);
- assert(value != NULL);
- Py_INCREF(value);
+ assert(PyStackRef_AsPyObjectBorrow(GETLOCAL(oparg)) != NULL);
+ value = PyStackRef_DUP(GETLOCAL(oparg));
stack_pointer[0] = value;
stack_pointer += 1;
assert(WITHIN_STACK_BOUNDS());
}
case _LOAD_FAST_1: {
- PyObject *value;
+ _PyStackRef value;
oparg = 1;
assert(oparg == CURRENT_OPARG());
- value = GETLOCAL(oparg);
- assert(value != NULL);
- Py_INCREF(value);
+ assert(PyStackRef_AsPyObjectBorrow(GETLOCAL(oparg)) != NULL);
+ value = PyStackRef_DUP(GETLOCAL(oparg));
stack_pointer[0] = value;
stack_pointer += 1;
assert(WITHIN_STACK_BOUNDS());
}
case _LOAD_FAST_2: {
- PyObject *value;
+ _PyStackRef value;
oparg = 2;
assert(oparg == CURRENT_OPARG());
- value = GETLOCAL(oparg);
- assert(value != NULL);
- Py_INCREF(value);
+ assert(PyStackRef_AsPyObjectBorrow(GETLOCAL(oparg)) != NULL);
+ value = PyStackRef_DUP(GETLOCAL(oparg));
stack_pointer[0] = value;
stack_pointer += 1;
assert(WITHIN_STACK_BOUNDS());
}
case _LOAD_FAST_3: {
- PyObject *value;
+ _PyStackRef value;
oparg = 3;
assert(oparg == CURRENT_OPARG());
- value = GETLOCAL(oparg);
- assert(value != NULL);
- Py_INCREF(value);
+ assert(PyStackRef_AsPyObjectBorrow(GETLOCAL(oparg)) != NULL);
+ value = PyStackRef_DUP(GETLOCAL(oparg));
stack_pointer[0] = value;
stack_pointer += 1;
assert(WITHIN_STACK_BOUNDS());
}
case _LOAD_FAST_4: {
- PyObject *value;
+ _PyStackRef value;
oparg = 4;
assert(oparg == CURRENT_OPARG());
- value = GETLOCAL(oparg);
- assert(value != NULL);
- Py_INCREF(value);
+ assert(PyStackRef_AsPyObjectBorrow(GETLOCAL(oparg)) != NULL);
+ value = PyStackRef_DUP(GETLOCAL(oparg));
stack_pointer[0] = value;
stack_pointer += 1;
assert(WITHIN_STACK_BOUNDS());
}
case _LOAD_FAST_5: {
- PyObject *value;
+ _PyStackRef value;
oparg = 5;
assert(oparg == CURRENT_OPARG());
- value = GETLOCAL(oparg);
- assert(value != NULL);
- Py_INCREF(value);
+ assert(PyStackRef_AsPyObjectBorrow(GETLOCAL(oparg)) != NULL);
+ value = PyStackRef_DUP(GETLOCAL(oparg));
stack_pointer[0] = value;
stack_pointer += 1;
assert(WITHIN_STACK_BOUNDS());
}
case _LOAD_FAST_6: {
- PyObject *value;
+ _PyStackRef value;
oparg = 6;
assert(oparg == CURRENT_OPARG());
- value = GETLOCAL(oparg);
- assert(value != NULL);
- Py_INCREF(value);
+ assert(PyStackRef_AsPyObjectBorrow(GETLOCAL(oparg)) != NULL);
+ value = PyStackRef_DUP(GETLOCAL(oparg));
stack_pointer[0] = value;
stack_pointer += 1;
assert(WITHIN_STACK_BOUNDS());
}
case _LOAD_FAST_7: {
- PyObject *value;
+ _PyStackRef value;
oparg = 7;
assert(oparg == CURRENT_OPARG());
- value = GETLOCAL(oparg);
- assert(value != NULL);
- Py_INCREF(value);
+ assert(PyStackRef_AsPyObjectBorrow(GETLOCAL(oparg)) != NULL);
+ value = PyStackRef_DUP(GETLOCAL(oparg));
stack_pointer[0] = value;
stack_pointer += 1;
assert(WITHIN_STACK_BOUNDS());
}
case _LOAD_FAST: {
- PyObject *value;
+ _PyStackRef value;
oparg = CURRENT_OPARG();
- value = GETLOCAL(oparg);
- assert(value != NULL);
- Py_INCREF(value);
+ assert(PyStackRef_AsPyObjectBorrow(GETLOCAL(oparg)) != NULL);
+ value = PyStackRef_DUP(GETLOCAL(oparg));
stack_pointer[0] = value;
stack_pointer += 1;
assert(WITHIN_STACK_BOUNDS());
}
case _LOAD_FAST_AND_CLEAR: {
- PyObject *value;
+ _PyStackRef value;
oparg = CURRENT_OPARG();
value = GETLOCAL(oparg);
// do not use SETLOCAL here, it decrefs the old value
- GETLOCAL(oparg) = NULL;
+ GETLOCAL(oparg) = PyStackRef_NULL;
stack_pointer[0] = value;
stack_pointer += 1;
assert(WITHIN_STACK_BOUNDS());
}
case _LOAD_CONST: {
- PyObject *value;
+ _PyStackRef value;
oparg = CURRENT_OPARG();
- value = GETITEM(FRAME_CO_CONSTS, oparg);
- Py_INCREF(value);
+ value = PyStackRef_FromPyObjectNew(GETITEM(FRAME_CO_CONSTS, oparg));
stack_pointer[0] = value;
stack_pointer += 1;
assert(WITHIN_STACK_BOUNDS());
}
case _STORE_FAST_0: {
- PyObject *value;
+ _PyStackRef value;
oparg = 0;
assert(oparg == CURRENT_OPARG());
value = stack_pointer[-1];
}
case _STORE_FAST_1: {
- PyObject *value;
+ _PyStackRef value;
oparg = 1;
assert(oparg == CURRENT_OPARG());
value = stack_pointer[-1];
}
case _STORE_FAST_2: {
- PyObject *value;
+ _PyStackRef value;
oparg = 2;
assert(oparg == CURRENT_OPARG());
value = stack_pointer[-1];
}
case _STORE_FAST_3: {
- PyObject *value;
+ _PyStackRef value;
oparg = 3;
assert(oparg == CURRENT_OPARG());
value = stack_pointer[-1];
}
case _STORE_FAST_4: {
- PyObject *value;
+ _PyStackRef value;
oparg = 4;
assert(oparg == CURRENT_OPARG());
value = stack_pointer[-1];
}
case _STORE_FAST_5: {
- PyObject *value;
+ _PyStackRef value;
oparg = 5;
assert(oparg == CURRENT_OPARG());
value = stack_pointer[-1];
}
case _STORE_FAST_6: {
- PyObject *value;
+ _PyStackRef value;
oparg = 6;
assert(oparg == CURRENT_OPARG());
value = stack_pointer[-1];
}
case _STORE_FAST_7: {
- PyObject *value;
+ _PyStackRef value;
oparg = 7;
assert(oparg == CURRENT_OPARG());
value = stack_pointer[-1];
}
case _STORE_FAST: {
- PyObject *value;
+ _PyStackRef value;
oparg = CURRENT_OPARG();
value = stack_pointer[-1];
SETLOCAL(oparg, value);
}
case _POP_TOP: {
- PyObject *value;
+ _PyStackRef value;
value = stack_pointer[-1];
- Py_DECREF(value);
+ PyStackRef_CLOSE(value);
stack_pointer += -1;
assert(WITHIN_STACK_BOUNDS());
break;
}
case _PUSH_NULL: {
- PyObject *res;
- res = NULL;
+ _PyStackRef res;
+ res = PyStackRef_NULL;
stack_pointer[0] = res;
stack_pointer += 1;
assert(WITHIN_STACK_BOUNDS());
}
case _END_SEND: {
- PyObject *value;
- PyObject *receiver;
+ _PyStackRef value;
+ _PyStackRef receiver;
value = stack_pointer[-1];
receiver = stack_pointer[-2];
- Py_DECREF(receiver);
+ (void)receiver;
+ PyStackRef_CLOSE(receiver);
stack_pointer[-2] = value;
stack_pointer += -1;
assert(WITHIN_STACK_BOUNDS());
}
case _UNARY_NEGATIVE: {
- PyObject *value;
- PyObject *res;
+ _PyStackRef value;
+ _PyStackRef res;
value = stack_pointer[-1];
- res = PyNumber_Negative(value);
- Py_DECREF(value);
- if (res == NULL) JUMP_TO_ERROR();
+ PyObject *res_o = PyNumber_Negative(PyStackRef_AsPyObjectBorrow(value));
+ PyStackRef_CLOSE(value);
+ if (res_o == NULL) JUMP_TO_ERROR();
+ res = PyStackRef_FromPyObjectSteal(res_o);
stack_pointer[-1] = res;
break;
}
case _UNARY_NOT: {
- PyObject *value;
- PyObject *res;
+ _PyStackRef value;
+ _PyStackRef res;
value = stack_pointer[-1];
- assert(PyBool_Check(value));
- res = Py_IsFalse(value) ? Py_True : Py_False;
+ assert(PyBool_Check(PyStackRef_AsPyObjectBorrow(value)));
+ res = PyStackRef_Is(value, PyStackRef_False)
+ ? PyStackRef_True : PyStackRef_False;
stack_pointer[-1] = res;
break;
}
case _TO_BOOL: {
- PyObject *value;
- PyObject *res;
+ _PyStackRef value;
+ _PyStackRef res;
value = stack_pointer[-1];
- int err = PyObject_IsTrue(value);
- Py_DECREF(value);
+ int err = PyObject_IsTrue(PyStackRef_AsPyObjectBorrow(value));
+ PyStackRef_CLOSE(value);
if (err < 0) JUMP_TO_ERROR();
- res = err ? Py_True : Py_False;
+ res = err ? PyStackRef_True : PyStackRef_False;
stack_pointer[-1] = res;
break;
}
case _TO_BOOL_BOOL: {
- PyObject *value;
+ _PyStackRef value;
value = stack_pointer[-1];
- if (!PyBool_Check(value)) {
+ if (!PyBool_Check(PyStackRef_AsPyObjectBorrow(value))) {
UOP_STAT_INC(uopcode, miss);
JUMP_TO_JUMP_TARGET();
}
}
case _TO_BOOL_INT: {
- PyObject *value;
- PyObject *res;
+ _PyStackRef value;
+ _PyStackRef res;
value = stack_pointer[-1];
- if (!PyLong_CheckExact(value)) {
+ PyObject *value_o = PyStackRef_AsPyObjectBorrow(value);
+ if (!PyLong_CheckExact(value_o)) {
UOP_STAT_INC(uopcode, miss);
JUMP_TO_JUMP_TARGET();
}
STAT_INC(TO_BOOL, hit);
- if (_PyLong_IsZero((PyLongObject *)value)) {
- assert(_Py_IsImmortal(value));
- res = Py_False;
+ if (_PyLong_IsZero((PyLongObject *)value_o)) {
+ assert(_Py_IsImmortal(value_o));
+ res = PyStackRef_False;
}
else {
- Py_DECREF(value);
- res = Py_True;
+ PyStackRef_CLOSE(value);
+ res = PyStackRef_True;
}
stack_pointer[-1] = res;
break;
}
case _TO_BOOL_LIST: {
- PyObject *value;
- PyObject *res;
+ _PyStackRef value;
+ _PyStackRef res;
value = stack_pointer[-1];
- if (!PyList_CheckExact(value)) {
+ PyObject *value_o = PyStackRef_AsPyObjectBorrow(value);
+ if (!PyList_CheckExact(value_o)) {
UOP_STAT_INC(uopcode, miss);
JUMP_TO_JUMP_TARGET();
}
STAT_INC(TO_BOOL, hit);
- res = Py_SIZE(value) ? Py_True : Py_False;
- Py_DECREF(value);
+ res = Py_SIZE(value_o) ? PyStackRef_True : PyStackRef_False;
+ PyStackRef_CLOSE(value);
stack_pointer[-1] = res;
break;
}
case _TO_BOOL_NONE: {
- PyObject *value;
- PyObject *res;
+ _PyStackRef value;
+ _PyStackRef res;
value = stack_pointer[-1];
// This one is a bit weird, because we expect *some* failures:
- if (!Py_IsNone(value)) {
+ if (!PyStackRef_Is(value, PyStackRef_None)) {
UOP_STAT_INC(uopcode, miss);
JUMP_TO_JUMP_TARGET();
}
STAT_INC(TO_BOOL, hit);
- res = Py_False;
+ res = PyStackRef_False;
stack_pointer[-1] = res;
break;
}
case _TO_BOOL_STR: {
- PyObject *value;
- PyObject *res;
+ _PyStackRef value;
+ _PyStackRef res;
value = stack_pointer[-1];
- if (!PyUnicode_CheckExact(value)) {
+ PyObject *value_o = PyStackRef_AsPyObjectBorrow(value);
+ if (!PyUnicode_CheckExact(value_o)) {
UOP_STAT_INC(uopcode, miss);
JUMP_TO_JUMP_TARGET();
}
STAT_INC(TO_BOOL, hit);
- if (value == &_Py_STR(empty)) {
- assert(_Py_IsImmortal(value));
- res = Py_False;
+ if (value_o == &_Py_STR(empty)) {
+ assert(_Py_IsImmortal(value_o));
+ res = PyStackRef_False;
}
else {
- assert(Py_SIZE(value));
- Py_DECREF(value);
- res = Py_True;
+ assert(Py_SIZE(value_o));
+ PyStackRef_CLOSE(value);
+ res = PyStackRef_True;
}
stack_pointer[-1] = res;
break;
}
case _REPLACE_WITH_TRUE: {
- PyObject *value;
- PyObject *res;
+ _PyStackRef value;
+ _PyStackRef res;
value = stack_pointer[-1];
- Py_DECREF(value);
- res = Py_True;
+ PyStackRef_CLOSE(value);
+ res = PyStackRef_True;
stack_pointer[-1] = res;
break;
}
case _UNARY_INVERT: {
- PyObject *value;
- PyObject *res;
+ _PyStackRef value;
+ _PyStackRef res;
value = stack_pointer[-1];
- res = PyNumber_Invert(value);
- Py_DECREF(value);
- if (res == NULL) JUMP_TO_ERROR();
+ PyObject *res_o = PyNumber_Invert(PyStackRef_AsPyObjectBorrow(value));
+ PyStackRef_CLOSE(value);
+ if (res_o == NULL) JUMP_TO_ERROR();
+ res = PyStackRef_FromPyObjectSteal(res_o);
stack_pointer[-1] = res;
break;
}
case _GUARD_BOTH_INT: {
- PyObject *right;
- PyObject *left;
+ _PyStackRef right;
+ _PyStackRef left;
right = stack_pointer[-1];
left = stack_pointer[-2];
- if (!PyLong_CheckExact(left)) {
+ PyObject *left_o = PyStackRef_AsPyObjectBorrow(left);
+ PyObject *right_o = PyStackRef_AsPyObjectBorrow(right);
+ if (!PyLong_CheckExact(left_o)) {
UOP_STAT_INC(uopcode, miss);
JUMP_TO_JUMP_TARGET();
}
- if (!PyLong_CheckExact(right)) {
+ if (!PyLong_CheckExact(right_o)) {
UOP_STAT_INC(uopcode, miss);
JUMP_TO_JUMP_TARGET();
}
}
case _GUARD_NOS_INT: {
- PyObject *left;
+ _PyStackRef left;
left = stack_pointer[-2];
- if (!PyLong_CheckExact(left)) {
+ PyObject *left_o = PyStackRef_AsPyObjectBorrow(left);
+ if (!PyLong_CheckExact(left_o)) {
UOP_STAT_INC(uopcode, miss);
JUMP_TO_JUMP_TARGET();
}
}
case _GUARD_TOS_INT: {
- PyObject *value;
+ _PyStackRef value;
value = stack_pointer[-1];
- if (!PyLong_CheckExact(value)) {
+ PyObject *value_o = PyStackRef_AsPyObjectBorrow(value);
+ if (!PyLong_CheckExact(value_o)) {
UOP_STAT_INC(uopcode, miss);
JUMP_TO_JUMP_TARGET();
}
}
case _BINARY_OP_MULTIPLY_INT: {
- PyObject *right;
- PyObject *left;
- PyObject *res;
+ _PyStackRef right;
+ _PyStackRef left;
+ _PyStackRef res;
right = stack_pointer[-1];
left = stack_pointer[-2];
+ PyObject *left_o = PyStackRef_AsPyObjectBorrow(left);
+ PyObject *right_o = PyStackRef_AsPyObjectBorrow(right);
STAT_INC(BINARY_OP, hit);
- res = _PyLong_Multiply((PyLongObject *)left, (PyLongObject *)right);
- _Py_DECREF_SPECIALIZED(right, (destructor)PyObject_Free);
- _Py_DECREF_SPECIALIZED(left, (destructor)PyObject_Free);
- if (res == NULL) JUMP_TO_ERROR();
+ PyObject *res_o = _PyLong_Multiply((PyLongObject *)left_o, (PyLongObject *)right_o);
+ _Py_DECREF_SPECIALIZED(right_o, (destructor)PyObject_Free);
+ _Py_DECREF_SPECIALIZED(left_o, (destructor)PyObject_Free);
+ if (res_o == NULL) JUMP_TO_ERROR();
+ res = PyStackRef_FromPyObjectSteal(res_o);
stack_pointer[-2] = res;
stack_pointer += -1;
assert(WITHIN_STACK_BOUNDS());
}
case _BINARY_OP_ADD_INT: {
- PyObject *right;
- PyObject *left;
- PyObject *res;
+ _PyStackRef right;
+ _PyStackRef left;
+ _PyStackRef res;
right = stack_pointer[-1];
left = stack_pointer[-2];
+ PyObject *left_o = PyStackRef_AsPyObjectBorrow(left);
+ PyObject *right_o = PyStackRef_AsPyObjectBorrow(right);
STAT_INC(BINARY_OP, hit);
- res = _PyLong_Add((PyLongObject *)left, (PyLongObject *)right);
- _Py_DECREF_SPECIALIZED(right, (destructor)PyObject_Free);
- _Py_DECREF_SPECIALIZED(left, (destructor)PyObject_Free);
- if (res == NULL) JUMP_TO_ERROR();
+ PyObject *res_o = _PyLong_Add((PyLongObject *)left_o, (PyLongObject *)right_o);
+ _Py_DECREF_SPECIALIZED(right_o, (destructor)PyObject_Free);
+ _Py_DECREF_SPECIALIZED(left_o, (destructor)PyObject_Free);
+ if (res_o == NULL) JUMP_TO_ERROR();
+ res = PyStackRef_FromPyObjectSteal(res_o);
stack_pointer[-2] = res;
stack_pointer += -1;
assert(WITHIN_STACK_BOUNDS());
}
case _BINARY_OP_SUBTRACT_INT: {
- PyObject *right;
- PyObject *left;
- PyObject *res;
+ _PyStackRef right;
+ _PyStackRef left;
+ _PyStackRef res;
right = stack_pointer[-1];
left = stack_pointer[-2];
+ PyObject *left_o = PyStackRef_AsPyObjectBorrow(left);
+ PyObject *right_o = PyStackRef_AsPyObjectBorrow(right);
STAT_INC(BINARY_OP, hit);
- res = _PyLong_Subtract((PyLongObject *)left, (PyLongObject *)right);
- _Py_DECREF_SPECIALIZED(right, (destructor)PyObject_Free);
- _Py_DECREF_SPECIALIZED(left, (destructor)PyObject_Free);
- if (res == NULL) JUMP_TO_ERROR();
+ PyObject *res_o = _PyLong_Subtract((PyLongObject *)left_o, (PyLongObject *)right_o);
+ _Py_DECREF_SPECIALIZED(right_o, (destructor)PyObject_Free);
+ _Py_DECREF_SPECIALIZED(left_o, (destructor)PyObject_Free);;
+ if (res_o == NULL) JUMP_TO_ERROR();
+ res = PyStackRef_FromPyObjectSteal(res_o);
stack_pointer[-2] = res;
stack_pointer += -1;
assert(WITHIN_STACK_BOUNDS());
}
case _GUARD_BOTH_FLOAT: {
- PyObject *right;
- PyObject *left;
+ _PyStackRef right;
+ _PyStackRef left;
right = stack_pointer[-1];
left = stack_pointer[-2];
- if (!PyFloat_CheckExact(left)) {
+ PyObject *left_o = PyStackRef_AsPyObjectBorrow(left);
+ PyObject *right_o = PyStackRef_AsPyObjectBorrow(right);
+ if (!PyFloat_CheckExact(left_o)) {
UOP_STAT_INC(uopcode, miss);
JUMP_TO_JUMP_TARGET();
}
- if (!PyFloat_CheckExact(right)) {
+ if (!PyFloat_CheckExact(right_o)) {
UOP_STAT_INC(uopcode, miss);
JUMP_TO_JUMP_TARGET();
}
}
case _GUARD_NOS_FLOAT: {
- PyObject *left;
+ _PyStackRef left;
left = stack_pointer[-2];
- if (!PyFloat_CheckExact(left)) {
+ PyObject *left_o = PyStackRef_AsPyObjectBorrow(left);
+ if (!PyFloat_CheckExact(left_o)) {
UOP_STAT_INC(uopcode, miss);
JUMP_TO_JUMP_TARGET();
}
}
case _GUARD_TOS_FLOAT: {
- PyObject *value;
+ _PyStackRef value;
value = stack_pointer[-1];
- if (!PyFloat_CheckExact(value)) {
+ PyObject *value_o = PyStackRef_AsPyObjectBorrow(value);
+ if (!PyFloat_CheckExact(value_o)) {
UOP_STAT_INC(uopcode, miss);
JUMP_TO_JUMP_TARGET();
}
}
case _BINARY_OP_MULTIPLY_FLOAT: {
- PyObject *right;
- PyObject *left;
- PyObject *res;
+ _PyStackRef right;
+ _PyStackRef left;
+ _PyStackRef res;
right = stack_pointer[-1];
left = stack_pointer[-2];
+ PyObject *left_o = PyStackRef_AsPyObjectBorrow(left);
+ PyObject *right_o = PyStackRef_AsPyObjectBorrow(right);
STAT_INC(BINARY_OP, hit);
double dres =
- ((PyFloatObject *)left)->ob_fval *
- ((PyFloatObject *)right)->ob_fval;
- DECREF_INPUTS_AND_REUSE_FLOAT(left, right, dres, res);
+ ((PyFloatObject *)left_o)->ob_fval *
+ ((PyFloatObject *)right_o)->ob_fval;
+ PyObject *res_o;
+ DECREF_INPUTS_AND_REUSE_FLOAT(left_o, right_o, dres, res_o);
+ res = PyStackRef_FromPyObjectSteal(res_o);
stack_pointer[-2] = res;
stack_pointer += -1;
assert(WITHIN_STACK_BOUNDS());
}
case _BINARY_OP_ADD_FLOAT: {
- PyObject *right;
- PyObject *left;
- PyObject *res;
+ _PyStackRef right;
+ _PyStackRef left;
+ _PyStackRef res;
right = stack_pointer[-1];
left = stack_pointer[-2];
+ PyObject *left_o = PyStackRef_AsPyObjectBorrow(left);
+ PyObject *right_o = PyStackRef_AsPyObjectBorrow(right);
STAT_INC(BINARY_OP, hit);
double dres =
- ((PyFloatObject *)left)->ob_fval +
- ((PyFloatObject *)right)->ob_fval;
- DECREF_INPUTS_AND_REUSE_FLOAT(left, right, dres, res);
+ ((PyFloatObject *)left_o)->ob_fval +
+ ((PyFloatObject *)right_o)->ob_fval;
+ PyObject *res_o;
+ DECREF_INPUTS_AND_REUSE_FLOAT(left_o, right_o, dres, res_o);
+ res = PyStackRef_FromPyObjectSteal(res_o);
stack_pointer[-2] = res;
stack_pointer += -1;
assert(WITHIN_STACK_BOUNDS());
}
case _BINARY_OP_SUBTRACT_FLOAT: {
- PyObject *right;
- PyObject *left;
- PyObject *res;
+ _PyStackRef right;
+ _PyStackRef left;
+ _PyStackRef res;
right = stack_pointer[-1];
left = stack_pointer[-2];
+ PyObject *left_o = PyStackRef_AsPyObjectBorrow(left);
+ PyObject *right_o = PyStackRef_AsPyObjectBorrow(right);
STAT_INC(BINARY_OP, hit);
double dres =
- ((PyFloatObject *)left)->ob_fval -
- ((PyFloatObject *)right)->ob_fval;
- DECREF_INPUTS_AND_REUSE_FLOAT(left, right, dres, res);
+ ((PyFloatObject *)left_o)->ob_fval -
+ ((PyFloatObject *)right_o)->ob_fval;
+ PyObject *res_o;
+ DECREF_INPUTS_AND_REUSE_FLOAT(left_o, right_o, dres, res_o);
+ res = PyStackRef_FromPyObjectSteal(res_o);
stack_pointer[-2] = res;
stack_pointer += -1;
assert(WITHIN_STACK_BOUNDS());
}
case _GUARD_BOTH_UNICODE: {
- PyObject *right;
- PyObject *left;
+ _PyStackRef right;
+ _PyStackRef left;
right = stack_pointer[-1];
left = stack_pointer[-2];
- if (!PyUnicode_CheckExact(left)) {
+ PyObject *left_o = PyStackRef_AsPyObjectBorrow(left);
+ PyObject *right_o = PyStackRef_AsPyObjectBorrow(right);
+ if (!PyUnicode_CheckExact(left_o)) {
UOP_STAT_INC(uopcode, miss);
JUMP_TO_JUMP_TARGET();
}
- if (!PyUnicode_CheckExact(right)) {
+ if (!PyUnicode_CheckExact(right_o)) {
UOP_STAT_INC(uopcode, miss);
JUMP_TO_JUMP_TARGET();
}
}
case _BINARY_OP_ADD_UNICODE: {
- PyObject *right;
- PyObject *left;
- PyObject *res;
+ _PyStackRef right;
+ _PyStackRef left;
+ _PyStackRef res;
right = stack_pointer[-1];
left = stack_pointer[-2];
+ PyObject *left_o = PyStackRef_AsPyObjectBorrow(left);
+ PyObject *right_o = PyStackRef_AsPyObjectBorrow(right);
STAT_INC(BINARY_OP, hit);
- res = PyUnicode_Concat(left, right);
- _Py_DECREF_SPECIALIZED(left, _PyUnicode_ExactDealloc);
- _Py_DECREF_SPECIALIZED(right, _PyUnicode_ExactDealloc);
- if (res == NULL) JUMP_TO_ERROR();
+ PyObject *res_o = PyUnicode_Concat(left_o, right_o);
+ _Py_DECREF_SPECIALIZED(left_o, _PyUnicode_ExactDealloc);
+ _Py_DECREF_SPECIALIZED(right_o, _PyUnicode_ExactDealloc);
+ if (res_o == NULL) JUMP_TO_ERROR();
+ res = PyStackRef_FromPyObjectSteal(res_o);
stack_pointer[-2] = res;
stack_pointer += -1;
assert(WITHIN_STACK_BOUNDS());
}
case _BINARY_SUBSCR: {
- PyObject *sub;
- PyObject *container;
- PyObject *res;
+ _PyStackRef sub;
+ _PyStackRef container;
+ _PyStackRef res;
sub = stack_pointer[-1];
container = stack_pointer[-2];
- res = PyObject_GetItem(container, sub);
- Py_DECREF(container);
- Py_DECREF(sub);
- if (res == NULL) JUMP_TO_ERROR();
+ PyObject *container_o = PyStackRef_AsPyObjectBorrow(container);
+ PyObject *sub_o = PyStackRef_AsPyObjectBorrow(sub);
+ PyObject *res_o = PyObject_GetItem(container_o, sub_o);
+ PyStackRef_CLOSE(container);
+ PyStackRef_CLOSE(sub);
+ if (res_o == NULL) JUMP_TO_ERROR();
+ res = PyStackRef_FromPyObjectSteal(res_o);
stack_pointer[-2] = res;
stack_pointer += -1;
assert(WITHIN_STACK_BOUNDS());
}
case _BINARY_SLICE: {
- PyObject *stop;
- PyObject *start;
- PyObject *container;
- PyObject *res;
+ _PyStackRef stop;
+ _PyStackRef start;
+ _PyStackRef container;
+ _PyStackRef res;
stop = stack_pointer[-1];
start = stack_pointer[-2];
container = stack_pointer[-3];
- PyObject *slice = _PyBuildSlice_ConsumeRefs(start, stop);
+ PyObject *slice = _PyBuildSlice_ConsumeRefs(PyStackRef_AsPyObjectSteal(start),
+ PyStackRef_AsPyObjectSteal(stop));
+ PyObject *res_o;
// Can't use ERROR_IF() here, because we haven't
// DECREF'ed container yet, and we still own slice.
if (slice == NULL) {
- res = NULL;
+ res_o = NULL;
}
else {
- res = PyObject_GetItem(container, slice);
+ res_o = PyObject_GetItem(PyStackRef_AsPyObjectBorrow(container), slice);
Py_DECREF(slice);
}
- Py_DECREF(container);
- if (res == NULL) JUMP_TO_ERROR();
+ PyStackRef_CLOSE(container);
+ if (res_o == NULL) JUMP_TO_ERROR();
+ res = PyStackRef_FromPyObjectSteal(res_o);
stack_pointer[-3] = res;
stack_pointer += -2;
assert(WITHIN_STACK_BOUNDS());
}
case _STORE_SLICE: {
- PyObject *stop;
- PyObject *start;
- PyObject *container;
- PyObject *v;
+ _PyStackRef stop;
+ _PyStackRef start;
+ _PyStackRef container;
+ _PyStackRef v;
stop = stack_pointer[-1];
start = stack_pointer[-2];
container = stack_pointer[-3];
v = stack_pointer[-4];
- PyObject *slice = _PyBuildSlice_ConsumeRefs(start, stop);
+ PyObject *slice = _PyBuildSlice_ConsumeRefs(PyStackRef_AsPyObjectSteal(start),
+ PyStackRef_AsPyObjectSteal(stop));
int err;
if (slice == NULL) {
err = 1;
}
else {
- err = PyObject_SetItem(container, slice, v);
+ err = PyObject_SetItem(PyStackRef_AsPyObjectBorrow(container), slice, PyStackRef_AsPyObjectSteal(v));
Py_DECREF(slice);
}
- Py_DECREF(v);
- Py_DECREF(container);
+ PyStackRef_CLOSE(v);
+ PyStackRef_CLOSE(container);
if (err) JUMP_TO_ERROR();
stack_pointer += -4;
assert(WITHIN_STACK_BOUNDS());
}
case _BINARY_SUBSCR_LIST_INT: {
- PyObject *sub;
- PyObject *list;
- PyObject *res;
- sub = stack_pointer[-1];
- list = stack_pointer[-2];
+ _PyStackRef sub_st;
+ _PyStackRef list_st;
+ _PyStackRef res;
+ sub_st = stack_pointer[-1];
+ list_st = stack_pointer[-2];
+ PyObject *sub = PyStackRef_AsPyObjectBorrow(sub_st);
+ PyObject *list = PyStackRef_AsPyObjectBorrow(list_st);
if (!PyLong_CheckExact(sub)) {
UOP_STAT_INC(uopcode, miss);
JUMP_TO_JUMP_TARGET();
JUMP_TO_JUMP_TARGET();
}
STAT_INC(BINARY_SUBSCR, hit);
- res = PyList_GET_ITEM(list, index);
- assert(res != NULL);
- Py_INCREF(res);
+ PyObject *res_o = PyList_GET_ITEM(list, index);
+ assert(res_o != NULL);
+ Py_INCREF(res_o);
_Py_DECREF_SPECIALIZED(sub, (destructor)PyObject_Free);
- Py_DECREF(list);
+ PyStackRef_CLOSE(list_st);
+ res = PyStackRef_FromPyObjectSteal(res_o);
stack_pointer[-2] = res;
stack_pointer += -1;
assert(WITHIN_STACK_BOUNDS());
}
case _BINARY_SUBSCR_STR_INT: {
- PyObject *sub;
- PyObject *str;
- PyObject *res;
- sub = stack_pointer[-1];
- str = stack_pointer[-2];
+ _PyStackRef sub_st;
+ _PyStackRef str_st;
+ _PyStackRef res;
+ sub_st = stack_pointer[-1];
+ str_st = stack_pointer[-2];
+ PyObject *sub = PyStackRef_AsPyObjectBorrow(sub_st);
+ PyObject *str = PyStackRef_AsPyObjectBorrow(str_st);
if (!PyLong_CheckExact(sub)) {
UOP_STAT_INC(uopcode, miss);
JUMP_TO_JUMP_TARGET();
JUMP_TO_JUMP_TARGET();
}
STAT_INC(BINARY_SUBSCR, hit);
- res = (PyObject*)&_Py_SINGLETON(strings).ascii[c];
+ PyObject *res_o = (PyObject*)&_Py_SINGLETON(strings).ascii[c];
_Py_DECREF_SPECIALIZED(sub, (destructor)PyObject_Free);
- Py_DECREF(str);
+ PyStackRef_CLOSE(str_st);
+ res = PyStackRef_FromPyObjectSteal(res_o);
stack_pointer[-2] = res;
stack_pointer += -1;
assert(WITHIN_STACK_BOUNDS());
}
case _BINARY_SUBSCR_TUPLE_INT: {
- PyObject *sub;
- PyObject *tuple;
- PyObject *res;
- sub = stack_pointer[-1];
- tuple = stack_pointer[-2];
+ _PyStackRef sub_st;
+ _PyStackRef tuple_st;
+ _PyStackRef res;
+ sub_st = stack_pointer[-1];
+ tuple_st = stack_pointer[-2];
+ PyObject *sub = PyStackRef_AsPyObjectBorrow(sub_st);
+ PyObject *tuple = PyStackRef_AsPyObjectBorrow(tuple_st);
if (!PyLong_CheckExact(sub)) {
UOP_STAT_INC(uopcode, miss);
JUMP_TO_JUMP_TARGET();
JUMP_TO_JUMP_TARGET();
}
STAT_INC(BINARY_SUBSCR, hit);
- res = PyTuple_GET_ITEM(tuple, index);
- assert(res != NULL);
- Py_INCREF(res);
+ PyObject *res_o = PyTuple_GET_ITEM(tuple, index);
+ assert(res_o != NULL);
+ Py_INCREF(res_o);
_Py_DECREF_SPECIALIZED(sub, (destructor)PyObject_Free);
- Py_DECREF(tuple);
+ PyStackRef_CLOSE(tuple_st);
+ res = PyStackRef_FromPyObjectSteal(res_o);
stack_pointer[-2] = res;
stack_pointer += -1;
assert(WITHIN_STACK_BOUNDS());
}
case _BINARY_SUBSCR_DICT: {
- PyObject *sub;
- PyObject *dict;
- PyObject *res;
- sub = stack_pointer[-1];
- dict = stack_pointer[-2];
+ _PyStackRef sub_st;
+ _PyStackRef dict_st;
+ _PyStackRef res;
+ sub_st = stack_pointer[-1];
+ dict_st = stack_pointer[-2];
+ PyObject *sub = PyStackRef_AsPyObjectBorrow(sub_st);
+ PyObject *dict = PyStackRef_AsPyObjectBorrow(dict_st);
if (!PyDict_CheckExact(dict)) {
UOP_STAT_INC(uopcode, miss);
JUMP_TO_JUMP_TARGET();
}
STAT_INC(BINARY_SUBSCR, hit);
- int rc = PyDict_GetItemRef(dict, sub, &res);
+ PyObject *res_o;
+ int rc = PyDict_GetItemRef(dict, sub, &res_o);
if (rc == 0) {
_PyErr_SetKeyError(sub);
}
- Py_DECREF(dict);
- Py_DECREF(sub);
+ PyStackRef_CLOSE(dict_st);
+ PyStackRef_CLOSE(sub_st);
if (rc <= 0) JUMP_TO_ERROR();
// not found or error
+ res = PyStackRef_FromPyObjectSteal(res_o);
stack_pointer[-2] = res;
stack_pointer += -1;
assert(WITHIN_STACK_BOUNDS());
/* _BINARY_SUBSCR_GETITEM is not a viable micro-op for tier 2 because it uses the 'this_instr' variable */
case _LIST_APPEND: {
- PyObject *v;
- PyObject *list;
+ _PyStackRef v;
+ _PyStackRef list;
oparg = CURRENT_OPARG();
v = stack_pointer[-1];
list = stack_pointer[-2 - (oparg-1)];
- if (_PyList_AppendTakeRef((PyListObject *)list, v) < 0) JUMP_TO_ERROR();
+ if (_PyList_AppendTakeRef((PyListObject *)PyStackRef_AsPyObjectBorrow(list),
+ PyStackRef_AsPyObjectSteal(v)) < 0) JUMP_TO_ERROR();
stack_pointer += -1;
assert(WITHIN_STACK_BOUNDS());
break;
}
case _SET_ADD: {
- PyObject *v;
- PyObject *set;
+ _PyStackRef v;
+ _PyStackRef set;
oparg = CURRENT_OPARG();
v = stack_pointer[-1];
set = stack_pointer[-2 - (oparg-1)];
- int err = PySet_Add(set, v);
- Py_DECREF(v);
+ int err = PySet_Add(PyStackRef_AsPyObjectBorrow(set),
+ PyStackRef_AsPyObjectSteal(v));
+ PyStackRef_CLOSE(v);
if (err) JUMP_TO_ERROR();
stack_pointer += -1;
assert(WITHIN_STACK_BOUNDS());
}
case _STORE_SUBSCR: {
- PyObject *sub;
- PyObject *container;
- PyObject *v;
+ _PyStackRef sub;
+ _PyStackRef container;
+ _PyStackRef v;
sub = stack_pointer[-1];
container = stack_pointer[-2];
v = stack_pointer[-3];
/* container[sub] = v */
- int err = PyObject_SetItem(container, sub, v);
- Py_DECREF(v);
- Py_DECREF(container);
- Py_DECREF(sub);
+ int err = PyObject_SetItem(PyStackRef_AsPyObjectBorrow(container), PyStackRef_AsPyObjectSteal(sub), PyStackRef_AsPyObjectSteal(v));
+ PyStackRef_CLOSE(v);
+ PyStackRef_CLOSE(container);
+ PyStackRef_CLOSE(sub);
if (err) JUMP_TO_ERROR();
stack_pointer += -3;
assert(WITHIN_STACK_BOUNDS());
}
case _STORE_SUBSCR_LIST_INT: {
- PyObject *sub;
- PyObject *list;
- PyObject *value;
- sub = stack_pointer[-1];
- list = stack_pointer[-2];
+ _PyStackRef sub_st;
+ _PyStackRef list_st;
+ _PyStackRef value;
+ sub_st = stack_pointer[-1];
+ list_st = stack_pointer[-2];
value = stack_pointer[-3];
+ PyObject *sub = PyStackRef_AsPyObjectBorrow(sub_st);
+ PyObject *list = PyStackRef_AsPyObjectBorrow(list_st);
if (!PyLong_CheckExact(sub)) {
UOP_STAT_INC(uopcode, miss);
JUMP_TO_JUMP_TARGET();
}
STAT_INC(STORE_SUBSCR, hit);
PyObject *old_value = PyList_GET_ITEM(list, index);
- PyList_SET_ITEM(list, index, value);
+ PyList_SET_ITEM(list, index, PyStackRef_AsPyObjectSteal(value));
assert(old_value != NULL);
Py_DECREF(old_value);
_Py_DECREF_SPECIALIZED(sub, (destructor)PyObject_Free);
- Py_DECREF(list);
+ PyStackRef_CLOSE(list_st);
stack_pointer += -3;
assert(WITHIN_STACK_BOUNDS());
break;
}
case _STORE_SUBSCR_DICT: {
- PyObject *sub;
- PyObject *dict;
- PyObject *value;
- sub = stack_pointer[-1];
- dict = stack_pointer[-2];
+ _PyStackRef sub_st;
+ _PyStackRef dict_st;
+ _PyStackRef value;
+ sub_st = stack_pointer[-1];
+ dict_st = stack_pointer[-2];
value = stack_pointer[-3];
+ PyObject *sub = PyStackRef_AsPyObjectBorrow(sub_st);
+ PyObject *dict = PyStackRef_AsPyObjectBorrow(dict_st);
if (!PyDict_CheckExact(dict)) {
UOP_STAT_INC(uopcode, miss);
JUMP_TO_JUMP_TARGET();
}
STAT_INC(STORE_SUBSCR, hit);
- int err = _PyDict_SetItem_Take2((PyDictObject *)dict, sub, value);
- Py_DECREF(dict);
+ int err = _PyDict_SetItem_Take2((PyDictObject *)dict, sub, PyStackRef_AsPyObjectSteal(value));
+ PyStackRef_CLOSE(dict_st);
if (err) JUMP_TO_ERROR();
stack_pointer += -3;
assert(WITHIN_STACK_BOUNDS());
}
case _DELETE_SUBSCR: {
- PyObject *sub;
- PyObject *container;
+ _PyStackRef sub;
+ _PyStackRef container;
sub = stack_pointer[-1];
container = stack_pointer[-2];
/* del container[sub] */
- int err = PyObject_DelItem(container, sub);
- Py_DECREF(container);
- Py_DECREF(sub);
+ int err = PyObject_DelItem(PyStackRef_AsPyObjectBorrow(container),
+ PyStackRef_AsPyObjectBorrow(sub));
+ PyStackRef_CLOSE(container);
+ PyStackRef_CLOSE(sub);
if (err) JUMP_TO_ERROR();
stack_pointer += -2;
assert(WITHIN_STACK_BOUNDS());
}
case _CALL_INTRINSIC_1: {
- PyObject *value;
- PyObject *res;
+ _PyStackRef value;
+ _PyStackRef res;
oparg = CURRENT_OPARG();
value = stack_pointer[-1];
assert(oparg <= MAX_INTRINSIC_1);
- res = _PyIntrinsics_UnaryFunctions[oparg].func(tstate, value);
- Py_DECREF(value);
- if (res == NULL) JUMP_TO_ERROR();
+ PyObject *res_o = _PyIntrinsics_UnaryFunctions[oparg].func(tstate, PyStackRef_AsPyObjectBorrow(value));
+ PyStackRef_CLOSE(value);
+ if (res_o == NULL) JUMP_TO_ERROR();
+ res = PyStackRef_FromPyObjectSteal(res_o);
stack_pointer[-1] = res;
break;
}
case _CALL_INTRINSIC_2: {
- PyObject *value1;
- PyObject *value2;
- PyObject *res;
+ _PyStackRef value1_st;
+ _PyStackRef value2_st;
+ _PyStackRef res;
oparg = CURRENT_OPARG();
- value1 = stack_pointer[-1];
- value2 = stack_pointer[-2];
+ value1_st = stack_pointer[-1];
+ value2_st = stack_pointer[-2];
assert(oparg <= MAX_INTRINSIC_2);
- res = _PyIntrinsics_BinaryFunctions[oparg].func(tstate, value2, value1);
- Py_DECREF(value2);
- Py_DECREF(value1);
- if (res == NULL) JUMP_TO_ERROR();
+ PyObject *value1 = PyStackRef_AsPyObjectBorrow(value1_st);
+ PyObject *value2 = PyStackRef_AsPyObjectBorrow(value2_st);
+ PyObject *res_o = _PyIntrinsics_BinaryFunctions[oparg].func(tstate, value2, value1);
+ PyStackRef_CLOSE(value2_st);
+ PyStackRef_CLOSE(value1_st);
+ if (res_o == NULL) JUMP_TO_ERROR();
+ res = PyStackRef_FromPyObjectSteal(res_o);
stack_pointer[-2] = res;
stack_pointer += -1;
assert(WITHIN_STACK_BOUNDS());
}
case _RETURN_VALUE: {
- PyObject *retval;
- PyObject *res;
+ _PyStackRef retval;
+ _PyStackRef res;
retval = stack_pointer[-1];
#if TIER_ONE
assert(frame != &entry_frame);
/* _INSTRUMENTED_RETURN_CONST is not a viable micro-op for tier 2 because it is instrumented */
case _GET_AITER: {
- PyObject *obj;
- PyObject *iter;
+ _PyStackRef obj;
+ _PyStackRef iter;
obj = stack_pointer[-1];
unaryfunc getter = NULL;
- PyTypeObject *type = Py_TYPE(obj);
+ PyObject *obj_o = PyStackRef_AsPyObjectBorrow(obj);
+ PyObject *iter_o;
+ PyTypeObject *type = Py_TYPE(obj_o);
if (type->tp_as_async != NULL) {
getter = type->tp_as_async->am_aiter;
}
"'async for' requires an object with "
"__aiter__ method, got %.100s",
type->tp_name);
- Py_DECREF(obj);
+ PyStackRef_CLOSE(obj);
if (true) JUMP_TO_ERROR();
}
- iter = (*getter)(obj);
- Py_DECREF(obj);
- if (iter == NULL) JUMP_TO_ERROR();
- if (Py_TYPE(iter)->tp_as_async == NULL ||
- Py_TYPE(iter)->tp_as_async->am_anext == NULL) {
+ iter_o = (*getter)(obj_o);
+ PyStackRef_CLOSE(obj);
+ if (iter_o == NULL) JUMP_TO_ERROR();
+ if (Py_TYPE(iter_o)->tp_as_async == NULL ||
+ Py_TYPE(iter_o)->tp_as_async->am_anext == NULL) {
_PyErr_Format(tstate, PyExc_TypeError,
"'async for' received an object from __aiter__ "
"that does not implement __anext__: %.100s",
- Py_TYPE(iter)->tp_name);
- Py_DECREF(iter);
+ Py_TYPE(iter_o)->tp_name);
+ Py_DECREF(iter_o);
if (true) JUMP_TO_ERROR();
}
+ iter = PyStackRef_FromPyObjectSteal(iter_o);
stack_pointer[-1] = iter;
break;
}
case _GET_ANEXT: {
- PyObject *aiter;
- PyObject *awaitable;
+ _PyStackRef aiter;
+ _PyStackRef awaitable;
aiter = stack_pointer[-1];
unaryfunc getter = NULL;
PyObject *next_iter = NULL;
- PyTypeObject *type = Py_TYPE(aiter);
- if (PyAsyncGen_CheckExact(aiter)) {
- awaitable = type->tp_as_async->am_anext(aiter);
- if (awaitable == NULL) {
+ PyObject *awaitable_o;
+ PyObject *aiter_o = PyStackRef_AsPyObjectBorrow(aiter);
+ PyTypeObject *type = Py_TYPE(aiter_o);
+ if (PyAsyncGen_CheckExact(aiter_o)) {
+ awaitable_o = type->tp_as_async->am_anext(aiter_o);
+ if (awaitable_o == NULL) {
JUMP_TO_ERROR();
}
} else {
getter = type->tp_as_async->am_anext;
}
if (getter != NULL) {
- next_iter = (*getter)(aiter);
+ next_iter = (*getter)(aiter_o);
if (next_iter == NULL) {
JUMP_TO_ERROR();
}
type->tp_name);
JUMP_TO_ERROR();
}
- awaitable = _PyCoro_GetAwaitableIter(next_iter);
- if (awaitable == NULL) {
+ awaitable_o = _PyCoro_GetAwaitableIter(next_iter);
+ if (awaitable_o == NULL) {
_PyErr_FormatFromCause(
PyExc_TypeError,
"'async for' received an invalid object "
Py_DECREF(next_iter);
}
}
+ awaitable = PyStackRef_FromPyObjectSteal(awaitable_o);
stack_pointer[0] = awaitable;
stack_pointer += 1;
assert(WITHIN_STACK_BOUNDS());
}
case _GET_AWAITABLE: {
- PyObject *iterable;
- PyObject *iter;
+ _PyStackRef iterable;
+ _PyStackRef iter;
oparg = CURRENT_OPARG();
iterable = stack_pointer[-1];
- iter = _PyCoro_GetAwaitableIter(iterable);
- if (iter == NULL) {
- _PyEval_FormatAwaitableError(tstate, Py_TYPE(iterable), oparg);
- }
- Py_DECREF(iterable);
- if (iter != NULL && PyCoro_CheckExact(iter)) {
- PyObject *yf = _PyGen_yf((PyGenObject*)iter);
+ PyObject *iter_o = _PyCoro_GetAwaitableIter(PyStackRef_AsPyObjectBorrow(iterable));
+ if (iter_o == NULL) {
+ _PyEval_FormatAwaitableError(tstate,
+ Py_TYPE(PyStackRef_AsPyObjectBorrow(iterable)), oparg);
+ }
+ PyStackRef_CLOSE(iterable);
+ if (iter_o != NULL && PyCoro_CheckExact(iter_o)) {
+ PyObject *yf = _PyGen_yf((PyGenObject*)iter_o);
if (yf != NULL) {
/* `iter` is a coroutine object that is being
awaited, `yf` is a pointer to the current awaitable
being awaited on. */
Py_DECREF(yf);
- Py_CLEAR(iter);
+ Py_CLEAR(iter_o);
_PyErr_SetString(tstate, PyExc_RuntimeError,
"coroutine is being awaited already");
/* The code below jumps to `error` if `iter` is NULL. */
}
}
- if (iter == NULL) JUMP_TO_ERROR();
+ if (iter_o == NULL) JUMP_TO_ERROR();
+ iter = PyStackRef_FromPyObjectSteal(iter_o);
stack_pointer[-1] = iter;
break;
}
/* _INSTRUMENTED_YIELD_VALUE is not a viable micro-op for tier 2 because it is instrumented */
case _YIELD_VALUE: {
- PyObject *retval;
- PyObject *value;
+ _PyStackRef retval;
+ _PyStackRef value;
oparg = CURRENT_OPARG();
retval = stack_pointer[-1];
// NOTE: It's important that YIELD_VALUE never raises an exception!
}
case _POP_EXCEPT: {
- PyObject *exc_value;
+ _PyStackRef exc_value;
exc_value = stack_pointer[-1];
_PyErr_StackItem *exc_info = tstate->exc_info;
- Py_XSETREF(exc_info->exc_value, exc_value == Py_None ? NULL : exc_value);
+ Py_XSETREF(exc_info->exc_value,
+ PyStackRef_AsPyObjectBorrow(exc_value) == Py_None
+ ? NULL : PyStackRef_AsPyObjectSteal(exc_value));
stack_pointer += -1;
assert(WITHIN_STACK_BOUNDS());
break;
}
case _LOAD_COMMON_CONSTANT: {
- PyObject *value;
+ _PyStackRef value;
oparg = CURRENT_OPARG();
// Keep in sync with _common_constants in opcode.py
switch(oparg) {
case CONSTANT_ASSERTIONERROR:
- value = PyExc_AssertionError;
+ value = PyStackRef_FromPyObjectImmortal(PyExc_AssertionError);
break;
case CONSTANT_NOTIMPLEMENTEDERROR:
- value = PyExc_NotImplementedError;
+ value = PyStackRef_FromPyObjectImmortal(PyExc_NotImplementedError);
break;
default:
Py_FatalError("bad LOAD_COMMON_CONSTANT oparg");
}
case _LOAD_BUILD_CLASS: {
- PyObject *bc;
- if (PyMapping_GetOptionalItem(BUILTINS(), &_Py_ID(__build_class__), &bc) < 0) JUMP_TO_ERROR();
- if (bc == NULL) {
+ _PyStackRef bc;
+ PyObject *bc_o;
+ if (PyMapping_GetOptionalItem(BUILTINS(), &_Py_ID(__build_class__), &bc_o) < 0) JUMP_TO_ERROR();
+ if (bc_o == NULL) {
_PyErr_SetString(tstate, PyExc_NameError,
"__build_class__ not found");
if (true) JUMP_TO_ERROR();
}
+ bc = PyStackRef_FromPyObjectSteal(bc_o);
stack_pointer[0] = bc;
stack_pointer += 1;
assert(WITHIN_STACK_BOUNDS());
}
case _STORE_NAME: {
- PyObject *v;
+ _PyStackRef v;
oparg = CURRENT_OPARG();
v = stack_pointer[-1];
PyObject *name = GETITEM(FRAME_CO_NAMES, oparg);
if (ns == NULL) {
_PyErr_Format(tstate, PyExc_SystemError,
"no locals found when storing %R", name);
- Py_DECREF(v);
+ PyStackRef_CLOSE(v);
if (true) JUMP_TO_ERROR();
}
if (PyDict_CheckExact(ns))
- err = PyDict_SetItem(ns, name, v);
+ err = PyDict_SetItem(ns, name, PyStackRef_AsPyObjectSteal(v));
else
- err = PyObject_SetItem(ns, name, v);
- Py_DECREF(v);
+ err = PyObject_SetItem(ns, name, PyStackRef_AsPyObjectSteal(v));
+ PyStackRef_CLOSE(v);
if (err) JUMP_TO_ERROR();
stack_pointer += -1;
assert(WITHIN_STACK_BOUNDS());
}
case _UNPACK_SEQUENCE: {
- PyObject *seq;
+ _PyStackRef seq;
oparg = CURRENT_OPARG();
seq = stack_pointer[-1];
- PyObject **top = stack_pointer + oparg - 1;
- int res = _PyEval_UnpackIterable(tstate, seq, oparg, -1, top);
- Py_DECREF(seq);
+ _PyStackRef *top = stack_pointer + oparg - 1;
+ int res = _PyEval_UnpackIterableStackRef(tstate, seq, oparg, -1, top);
+ PyStackRef_CLOSE(seq);
if (res == 0) JUMP_TO_ERROR();
stack_pointer += -1 + oparg;
assert(WITHIN_STACK_BOUNDS());
}
case _UNPACK_SEQUENCE_TWO_TUPLE: {
- PyObject *seq;
- PyObject *val1;
- PyObject *val0;
+ _PyStackRef seq;
+ _PyStackRef val1;
+ _PyStackRef val0;
oparg = CURRENT_OPARG();
seq = stack_pointer[-1];
assert(oparg == 2);
- if (!PyTuple_CheckExact(seq)) {
+ PyObject *seq_o = PyStackRef_AsPyObjectBorrow(seq);
+ if (!PyTuple_CheckExact(seq_o)) {
UOP_STAT_INC(uopcode, miss);
JUMP_TO_JUMP_TARGET();
}
- if (PyTuple_GET_SIZE(seq) != 2) {
+ if (PyTuple_GET_SIZE(seq_o) != 2) {
UOP_STAT_INC(uopcode, miss);
JUMP_TO_JUMP_TARGET();
}
STAT_INC(UNPACK_SEQUENCE, hit);
- val0 = Py_NewRef(PyTuple_GET_ITEM(seq, 0));
- val1 = Py_NewRef(PyTuple_GET_ITEM(seq, 1));
- Py_DECREF(seq);
+ val0 = PyStackRef_FromPyObjectNew(PyTuple_GET_ITEM(seq_o, 0));
+ val1 = PyStackRef_FromPyObjectNew(PyTuple_GET_ITEM(seq_o, 1));
+ PyStackRef_CLOSE(seq);
stack_pointer[-1] = val1;
stack_pointer[0] = val0;
stack_pointer += 1;
}
case _UNPACK_SEQUENCE_TUPLE: {
- PyObject *seq;
- PyObject **values;
+ _PyStackRef seq;
+ _PyStackRef *values;
oparg = CURRENT_OPARG();
seq = stack_pointer[-1];
values = &stack_pointer[-1];
- if (!PyTuple_CheckExact(seq)) {
+ PyObject *seq_o = PyStackRef_AsPyObjectBorrow(seq);
+ if (!PyTuple_CheckExact(seq_o)) {
UOP_STAT_INC(uopcode, miss);
JUMP_TO_JUMP_TARGET();
}
- if (PyTuple_GET_SIZE(seq) != oparg) {
+ if (PyTuple_GET_SIZE(seq_o) != oparg) {
UOP_STAT_INC(uopcode, miss);
JUMP_TO_JUMP_TARGET();
}
STAT_INC(UNPACK_SEQUENCE, hit);
- PyObject **items = _PyTuple_ITEMS(seq);
+ PyObject **items = _PyTuple_ITEMS(seq_o);
for (int i = oparg; --i >= 0; ) {
- *values++ = Py_NewRef(items[i]);
+ *values++ = PyStackRef_FromPyObjectNew(items[i]);
}
- Py_DECREF(seq);
+ PyStackRef_CLOSE(seq);
stack_pointer += -1 + oparg;
assert(WITHIN_STACK_BOUNDS());
break;
}
case _UNPACK_SEQUENCE_LIST: {
- PyObject *seq;
- PyObject **values;
+ _PyStackRef seq;
+ _PyStackRef *values;
oparg = CURRENT_OPARG();
seq = stack_pointer[-1];
values = &stack_pointer[-1];
- if (!PyList_CheckExact(seq)) {
+ PyObject *seq_o = PyStackRef_AsPyObjectBorrow(seq);
+ if (!PyList_CheckExact(seq_o)) {
UOP_STAT_INC(uopcode, miss);
JUMP_TO_JUMP_TARGET();
}
- if (PyList_GET_SIZE(seq) != oparg) {
+ if (PyList_GET_SIZE(seq_o) != oparg) {
UOP_STAT_INC(uopcode, miss);
JUMP_TO_JUMP_TARGET();
}
STAT_INC(UNPACK_SEQUENCE, hit);
- PyObject **items = _PyList_ITEMS(seq);
+ PyObject **items = _PyList_ITEMS(seq_o);
for (int i = oparg; --i >= 0; ) {
- *values++ = Py_NewRef(items[i]);
+ *values++ = PyStackRef_FromPyObjectNew(items[i]);
}
- Py_DECREF(seq);
+ PyStackRef_CLOSE(seq);
stack_pointer += -1 + oparg;
assert(WITHIN_STACK_BOUNDS());
break;
}
case _UNPACK_EX: {
- PyObject *seq;
+ _PyStackRef seq;
oparg = CURRENT_OPARG();
seq = stack_pointer[-1];
int totalargs = 1 + (oparg & 0xFF) + (oparg >> 8);
- PyObject **top = stack_pointer + totalargs - 1;
- int res = _PyEval_UnpackIterable(tstate, seq, oparg & 0xFF, oparg >> 8, top);
- Py_DECREF(seq);
+ _PyStackRef *top = stack_pointer + totalargs - 1;
+ int res = _PyEval_UnpackIterableStackRef(tstate, seq, oparg & 0xFF, oparg >> 8, top);
+ PyStackRef_CLOSE(seq);
if (res == 0) JUMP_TO_ERROR();
stack_pointer += (oparg >> 8) + (oparg & 0xFF);
assert(WITHIN_STACK_BOUNDS());
}
case _STORE_ATTR: {
- PyObject *owner;
- PyObject *v;
+ _PyStackRef owner;
+ _PyStackRef v;
oparg = CURRENT_OPARG();
owner = stack_pointer[-1];
v = stack_pointer[-2];
PyObject *name = GETITEM(FRAME_CO_NAMES, oparg);
- int err = PyObject_SetAttr(owner, name, v);
- Py_DECREF(v);
- Py_DECREF(owner);
+ int err = PyObject_SetAttr(PyStackRef_AsPyObjectBorrow(owner),
+ name, PyStackRef_AsPyObjectSteal(v));
+ PyStackRef_CLOSE(v);
+ PyStackRef_CLOSE(owner);
if (err) JUMP_TO_ERROR();
stack_pointer += -2;
assert(WITHIN_STACK_BOUNDS());
}
case _DELETE_ATTR: {
- PyObject *owner;
+ _PyStackRef owner;
oparg = CURRENT_OPARG();
owner = stack_pointer[-1];
PyObject *name = GETITEM(FRAME_CO_NAMES, oparg);
- int err = PyObject_DelAttr(owner, name);
- Py_DECREF(owner);
+ int err = PyObject_DelAttr(PyStackRef_AsPyObjectBorrow(owner), name);
+ PyStackRef_CLOSE(owner);
if (err) JUMP_TO_ERROR();
stack_pointer += -1;
assert(WITHIN_STACK_BOUNDS());
}
case _STORE_GLOBAL: {
- PyObject *v;
+ _PyStackRef v;
oparg = CURRENT_OPARG();
v = stack_pointer[-1];
PyObject *name = GETITEM(FRAME_CO_NAMES, oparg);
- int err = PyDict_SetItem(GLOBALS(), name, v);
- Py_DECREF(v);
+ int err = PyDict_SetItem(GLOBALS(), name, PyStackRef_AsPyObjectBorrow(v));
+ PyStackRef_CLOSE(v);
if (err) JUMP_TO_ERROR();
stack_pointer += -1;
assert(WITHIN_STACK_BOUNDS());
}
case _LOAD_LOCALS: {
- PyObject *locals;
- locals = LOCALS();
- if (locals == NULL) {
+ _PyStackRef locals;
+ PyObject *l = LOCALS();
+ if (l == NULL) {
_PyErr_SetString(tstate, PyExc_SystemError,
"no locals found");
if (true) JUMP_TO_ERROR();
}
- Py_INCREF(locals);
+ locals = PyStackRef_FromPyObjectNew(l);;
stack_pointer[0] = locals;
stack_pointer += 1;
assert(WITHIN_STACK_BOUNDS());
/* _LOAD_NAME is not a viable micro-op for tier 2 because it has both popping and not-popping errors */
case _LOAD_GLOBAL: {
- PyObject *res;
- PyObject *null = NULL;
+ _PyStackRef res;
+ _PyStackRef null = PyStackRef_NULL;
oparg = CURRENT_OPARG();
PyObject *name = GETITEM(FRAME_CO_NAMES, oparg>>1);
+ PyObject *res_o;
if (PyDict_CheckExact(GLOBALS())
&& PyDict_CheckExact(BUILTINS()))
{
- res = _PyDict_LoadGlobal((PyDictObject *)GLOBALS(),
+ res_o = _PyDict_LoadGlobal((PyDictObject *)GLOBALS(),
(PyDictObject *)BUILTINS(),
name);
- if (res == NULL) {
+ if (res_o == NULL) {
if (!_PyErr_Occurred(tstate)) {
/* _PyDict_LoadGlobal() returns NULL without raising
* an exception if the key doesn't exist */
else {
/* Slow-path if globals or builtins is not a dict */
/* namespace 1: globals */
- if (PyMapping_GetOptionalItem(GLOBALS(), name, &res) < 0) JUMP_TO_ERROR();
- if (res == NULL) {
+ if (PyMapping_GetOptionalItem(GLOBALS(), name, &res_o) < 0) JUMP_TO_ERROR();
+ if (res_o == NULL) {
/* namespace 2: builtins */
- if (PyMapping_GetOptionalItem(BUILTINS(), name, &res) < 0) JUMP_TO_ERROR();
- if (res == NULL) {
+ if (PyMapping_GetOptionalItem(BUILTINS(), name, &res_o) < 0) JUMP_TO_ERROR();
+ if (res_o == NULL) {
_PyEval_FormatExcCheckArg(
tstate, PyExc_NameError,
NAME_ERROR_MSG, name);
}
}
}
- null = NULL;
+ null = PyStackRef_NULL;
+ res = PyStackRef_FromPyObjectSteal(res_o);
stack_pointer[0] = res;
if (oparg & 1) stack_pointer[1] = null;
stack_pointer += 1 + (oparg & 1);
}
case _LOAD_GLOBAL_MODULE: {
- PyObject *res;
- PyObject *null = NULL;
+ _PyStackRef res;
+ _PyStackRef null = PyStackRef_NULL;
oparg = CURRENT_OPARG();
uint16_t index = (uint16_t)CURRENT_OPERAND();
PyDictObject *dict = (PyDictObject *)GLOBALS();
PyDictUnicodeEntry *entries = DK_UNICODE_ENTRIES(dict->ma_keys);
- res = entries[index].me_value;
- if (res == NULL) {
+ PyObject *res_o = entries[index].me_value;
+ if (res_o == NULL) {
UOP_STAT_INC(uopcode, miss);
JUMP_TO_JUMP_TARGET();
}
- Py_INCREF(res);
+ Py_INCREF(res_o);
STAT_INC(LOAD_GLOBAL, hit);
- null = NULL;
+ null = PyStackRef_NULL;
+ res = PyStackRef_FromPyObjectSteal(res_o);
stack_pointer[0] = res;
if (oparg & 1) stack_pointer[1] = null;
stack_pointer += 1 + (oparg & 1);
}
case _LOAD_GLOBAL_BUILTINS: {
- PyObject *res;
- PyObject *null = NULL;
+ _PyStackRef res;
+ _PyStackRef null = PyStackRef_NULL;
oparg = CURRENT_OPARG();
uint16_t index = (uint16_t)CURRENT_OPERAND();
PyDictObject *bdict = (PyDictObject *)BUILTINS();
PyDictUnicodeEntry *entries = DK_UNICODE_ENTRIES(bdict->ma_keys);
- res = entries[index].me_value;
- if (res == NULL) {
+ PyObject *res_o = entries[index].me_value;
+ if (res_o == NULL) {
UOP_STAT_INC(uopcode, miss);
JUMP_TO_JUMP_TARGET();
}
- Py_INCREF(res);
+ Py_INCREF(res_o);
STAT_INC(LOAD_GLOBAL, hit);
- null = NULL;
+ null = PyStackRef_NULL;
+ res = PyStackRef_FromPyObjectSteal(res_o);
stack_pointer[0] = res;
if (oparg & 1) stack_pointer[1] = null;
stack_pointer += 1 + (oparg & 1);
case _DELETE_FAST: {
oparg = CURRENT_OPARG();
- PyObject *v = GETLOCAL(oparg);
- if (v == NULL) {
+ _PyStackRef v = GETLOCAL(oparg);
+ if (PyStackRef_IsNull(v)) {
_PyEval_FormatExcCheckArg(tstate, PyExc_UnboundLocalError,
UNBOUNDLOCAL_ERROR_MSG,
PyTuple_GetItem(_PyFrame_GetCode(frame)->co_localsplusnames, oparg)
);
if (1) JUMP_TO_ERROR();
}
- SETLOCAL(oparg, NULL);
+ SETLOCAL(oparg, PyStackRef_NULL);
break;
}
oparg = CURRENT_OPARG();
// "initial" is probably NULL but not if it's an arg (or set
// via the f_locals proxy before MAKE_CELL has run).
- PyObject *initial = GETLOCAL(oparg);
+ PyObject *initial = PyStackRef_AsPyObjectBorrow(GETLOCAL(oparg));
PyObject *cell = PyCell_New(initial);
if (cell == NULL) {
JUMP_TO_ERROR();
}
- SETLOCAL(oparg, cell);
+ SETLOCAL(oparg, PyStackRef_FromPyObjectSteal(cell));
break;
}
case _DELETE_DEREF: {
oparg = CURRENT_OPARG();
- PyObject *cell = GETLOCAL(oparg);
+ PyObject *cell = PyStackRef_AsPyObjectBorrow(GETLOCAL(oparg));
// Can't use ERROR_IF here.
// Fortunately we don't need its superpower.
PyObject *oldobj = PyCell_SwapTakeRef((PyCellObject *)cell, NULL);
}
case _LOAD_FROM_DICT_OR_DEREF: {
- PyObject *class_dict;
- PyObject *value;
+ _PyStackRef class_dict_st;
+ _PyStackRef value;
oparg = CURRENT_OPARG();
- class_dict = stack_pointer[-1];
+ class_dict_st = stack_pointer[-1];
+ PyObject *value_o;
PyObject *name;
+ PyObject *class_dict = PyStackRef_AsPyObjectBorrow(class_dict_st);
assert(class_dict);
assert(oparg >= 0 && oparg < _PyFrame_GetCode(frame)->co_nlocalsplus);
name = PyTuple_GET_ITEM(_PyFrame_GetCode(frame)->co_localsplusnames, oparg);
- if (PyMapping_GetOptionalItem(class_dict, name, &value) < 0) {
+ if (PyMapping_GetOptionalItem(class_dict, name, &value_o) < 0) {
JUMP_TO_ERROR();
}
- if (!value) {
- PyCellObject *cell = (PyCellObject *)GETLOCAL(oparg);
- value = PyCell_GetRef(cell);
- if (value == NULL) {
+ if (!value_o) {
+ PyCellObject *cell = (PyCellObject *)PyStackRef_AsPyObjectBorrow(GETLOCAL(oparg));
+ value_o = PyCell_GetRef(cell);
+ if (value_o == NULL) {
_PyEval_FormatExcUnbound(tstate, _PyFrame_GetCode(frame), oparg);
JUMP_TO_ERROR();
}
}
- Py_DECREF(class_dict);
+ PyStackRef_CLOSE(class_dict_st);
+ value = PyStackRef_FromPyObjectSteal(value_o);
stack_pointer[-1] = value;
break;
}
case _LOAD_DEREF: {
- PyObject *value;
+ _PyStackRef value;
oparg = CURRENT_OPARG();
- PyCellObject *cell = (PyCellObject *)GETLOCAL(oparg);
- value = PyCell_GetRef(cell);
- if (value == NULL) {
+ PyCellObject *cell = (PyCellObject *)PyStackRef_AsPyObjectBorrow(GETLOCAL(oparg));
+ PyObject *value_o = PyCell_GetRef(cell);
+ if (value_o == NULL) {
_PyEval_FormatExcUnbound(tstate, _PyFrame_GetCode(frame), oparg);
if (true) JUMP_TO_ERROR();
}
+ value = PyStackRef_FromPyObjectSteal(value_o);
stack_pointer[0] = value;
stack_pointer += 1;
assert(WITHIN_STACK_BOUNDS());
}
case _STORE_DEREF: {
- PyObject *v;
+ _PyStackRef v;
oparg = CURRENT_OPARG();
v = stack_pointer[-1];
- PyCellObject *cell = (PyCellObject *)GETLOCAL(oparg);
- PyCell_SetTakeRef(cell, v);
+ PyCellObject *cell = (PyCellObject *)PyStackRef_AsPyObjectBorrow(GETLOCAL(oparg));
+ PyCell_SetTakeRef(cell, PyStackRef_AsPyObjectSteal(v));
stack_pointer += -1;
assert(WITHIN_STACK_BOUNDS());
break;
int offset = co->co_nlocalsplus - oparg;
for (int i = 0; i < oparg; ++i) {
PyObject *o = PyTuple_GET_ITEM(closure, i);
- frame->localsplus[offset + i] = Py_NewRef(o);
+ frame->localsplus[offset + i] = PyStackRef_FromPyObjectNew(o);
}
break;
}
case _BUILD_STRING: {
- PyObject **pieces;
- PyObject *str;
+ _PyStackRef *pieces;
+ _PyStackRef str;
oparg = CURRENT_OPARG();
pieces = &stack_pointer[-oparg];
- str = _PyUnicode_JoinArray(&_Py_STR(empty), pieces, oparg);
+ STACKREFS_TO_PYOBJECTS(pieces, oparg, pieces_o);
+ if (CONVERSION_FAILED(pieces_o)) {
+ for (int _i = oparg; --_i >= 0;) {
+ PyStackRef_CLOSE(pieces[_i]);
+ }
+ if (true) JUMP_TO_ERROR();
+ }
+ PyObject *str_o = _PyUnicode_JoinArray(&_Py_STR(empty), pieces_o, oparg);
+ STACKREFS_TO_PYOBJECTS_CLEANUP(pieces_o);
for (int _i = oparg; --_i >= 0;) {
- Py_DECREF(pieces[_i]);
+ PyStackRef_CLOSE(pieces[_i]);
}
- if (str == NULL) JUMP_TO_ERROR();
+ if (str_o == NULL) JUMP_TO_ERROR();
+ str = PyStackRef_FromPyObjectSteal(str_o);
stack_pointer[-oparg] = str;
stack_pointer += 1 - oparg;
assert(WITHIN_STACK_BOUNDS());
}
case _BUILD_TUPLE: {
- PyObject **values;
- PyObject *tup;
+ _PyStackRef *values;
+ _PyStackRef tup;
oparg = CURRENT_OPARG();
values = &stack_pointer[-oparg];
- tup = _PyTuple_FromArraySteal(values, oparg);
- if (tup == NULL) JUMP_TO_ERROR();
+ STACKREFS_TO_PYOBJECTS(values, oparg, values_o);
+ if (CONVERSION_FAILED(values_o)) {
+ for (int _i = oparg; --_i >= 0;) {
+ PyStackRef_CLOSE(values[_i]);
+ }
+ if (true) JUMP_TO_ERROR();
+ }
+ PyObject *tup_o = _PyTuple_FromArraySteal(values_o, oparg);
+ STACKREFS_TO_PYOBJECTS_CLEANUP(values_o);
+ if (tup_o == NULL) JUMP_TO_ERROR();
+ tup = PyStackRef_FromPyObjectSteal(tup_o);
stack_pointer[-oparg] = tup;
stack_pointer += 1 - oparg;
assert(WITHIN_STACK_BOUNDS());
}
case _BUILD_LIST: {
- PyObject **values;
- PyObject *list;
+ _PyStackRef *values;
+ _PyStackRef list;
oparg = CURRENT_OPARG();
values = &stack_pointer[-oparg];
- list = _PyList_FromArraySteal(values, oparg);
- if (list == NULL) JUMP_TO_ERROR();
+ STACKREFS_TO_PYOBJECTS(values, oparg, values_o);
+ if (CONVERSION_FAILED(values_o)) {
+ for (int _i = oparg; --_i >= 0;) {
+ PyStackRef_CLOSE(values[_i]);
+ }
+ if (true) JUMP_TO_ERROR();
+ }
+ PyObject *list_o = _PyList_FromArraySteal(values_o, oparg);
+ STACKREFS_TO_PYOBJECTS_CLEANUP(values_o);
+ if (list_o == NULL) JUMP_TO_ERROR();
+ list = PyStackRef_FromPyObjectSteal(list_o);
stack_pointer[-oparg] = list;
stack_pointer += 1 - oparg;
assert(WITHIN_STACK_BOUNDS());
}
case _LIST_EXTEND: {
- PyObject *iterable;
- PyObject *list;
+ _PyStackRef iterable_st;
+ _PyStackRef list_st;
oparg = CURRENT_OPARG();
- iterable = stack_pointer[-1];
- list = stack_pointer[-2 - (oparg-1)];
+ iterable_st = stack_pointer[-1];
+ list_st = stack_pointer[-2 - (oparg-1)];
+ PyObject *list = PyStackRef_AsPyObjectBorrow(list_st);
+ PyObject *iterable = PyStackRef_AsPyObjectBorrow(iterable_st);
PyObject *none_val = _PyList_Extend((PyListObject *)list, iterable);
if (none_val == NULL) {
if (_PyErr_ExceptionMatches(tstate, PyExc_TypeError) &&
"Value after * must be an iterable, not %.200s",
Py_TYPE(iterable)->tp_name);
}
- Py_DECREF(iterable);
+ PyStackRef_CLOSE(iterable_st);
if (true) JUMP_TO_ERROR();
}
assert(Py_IsNone(none_val));
- Py_DECREF(iterable);
+ PyStackRef_CLOSE(iterable_st);
stack_pointer += -1;
assert(WITHIN_STACK_BOUNDS());
break;
}
case _SET_UPDATE: {
- PyObject *iterable;
- PyObject *set;
+ _PyStackRef iterable;
+ _PyStackRef set;
oparg = CURRENT_OPARG();
iterable = stack_pointer[-1];
set = stack_pointer[-2 - (oparg-1)];
- int err = _PySet_Update(set, iterable);
- Py_DECREF(iterable);
+ int err = _PySet_Update(PyStackRef_AsPyObjectBorrow(set),
+ PyStackRef_AsPyObjectBorrow(iterable));
+ PyStackRef_CLOSE(iterable);
if (err < 0) JUMP_TO_ERROR();
stack_pointer += -1;
assert(WITHIN_STACK_BOUNDS());
/* _BUILD_SET is not a viable micro-op for tier 2 because it has both popping and not-popping errors */
case _BUILD_MAP: {
- PyObject **values;
- PyObject *map;
+ _PyStackRef *values;
+ _PyStackRef map;
oparg = CURRENT_OPARG();
values = &stack_pointer[-oparg*2];
- map = _PyDict_FromItems(
- values, 2,
- values+1, 2,
- oparg);
+ STACKREFS_TO_PYOBJECTS(values, oparg*2, values_o);
+ if (CONVERSION_FAILED(values_o)) {
+ for (int _i = oparg*2; --_i >= 0;) {
+ PyStackRef_CLOSE(values[_i]);
+ }
+ if (true) JUMP_TO_ERROR();
+ }
+ PyObject *map_o = _PyDict_FromItems(
+ values_o, 2,
+ values_o+1, 2,
+ oparg);
+ STACKREFS_TO_PYOBJECTS_CLEANUP(values_o);
for (int _i = oparg*2; --_i >= 0;) {
- Py_DECREF(values[_i]);
+ PyStackRef_CLOSE(values[_i]);
}
- if (map == NULL) JUMP_TO_ERROR();
+ if (map_o == NULL) JUMP_TO_ERROR();
+ map = PyStackRef_FromPyObjectSteal(map_o);
stack_pointer[-oparg*2] = map;
stack_pointer += 1 - oparg*2;
assert(WITHIN_STACK_BOUNDS());
}
case _BUILD_CONST_KEY_MAP: {
- PyObject *keys;
- PyObject **values;
- PyObject *map;
+ _PyStackRef keys;
+ _PyStackRef *values;
+ _PyStackRef map;
oparg = CURRENT_OPARG();
keys = stack_pointer[-1];
values = &stack_pointer[-1 - oparg];
- assert(PyTuple_CheckExact(keys));
- assert(PyTuple_GET_SIZE(keys) == (Py_ssize_t)oparg);
- map = _PyDict_FromItems(
- &PyTuple_GET_ITEM(keys, 0), 1,
- values, 1, oparg);
+ PyObject *keys_o = PyStackRef_AsPyObjectBorrow(keys);
+ assert(PyTuple_CheckExact(keys_o));
+ assert(PyTuple_GET_SIZE(keys_o) == (Py_ssize_t)oparg);
+ STACKREFS_TO_PYOBJECTS(values, oparg, values_o);
+ if (CONVERSION_FAILED(values_o)) {
+ for (int _i = oparg; --_i >= 0;) {
+ PyStackRef_CLOSE(values[_i]);
+ }
+ PyStackRef_CLOSE(keys);
+ if (true) JUMP_TO_ERROR();
+ }
+ PyObject *map_o = _PyDict_FromItems(
+ &PyTuple_GET_ITEM(keys_o, 0), 1,
+ values_o, 1, oparg);
+ STACKREFS_TO_PYOBJECTS_CLEANUP(values_o);
for (int _i = oparg; --_i >= 0;) {
- Py_DECREF(values[_i]);
+ PyStackRef_CLOSE(values[_i]);
}
- Py_DECREF(keys);
- if (map == NULL) JUMP_TO_ERROR();
+ PyStackRef_CLOSE(keys);
+ if (map_o == NULL) JUMP_TO_ERROR();
+ map = PyStackRef_FromPyObjectSteal(map_o);
stack_pointer[-1 - oparg] = map;
stack_pointer += -oparg;
assert(WITHIN_STACK_BOUNDS());
}
case _DICT_UPDATE: {
- PyObject *update;
- PyObject *dict;
+ _PyStackRef update;
+ _PyStackRef dict;
oparg = CURRENT_OPARG();
update = stack_pointer[-1];
dict = stack_pointer[-2 - (oparg - 1)];
- if (PyDict_Update(dict, update) < 0) {
+ PyObject *dict_o = PyStackRef_AsPyObjectBorrow(dict);
+ PyObject *update_o = PyStackRef_AsPyObjectBorrow(update);
+ if (PyDict_Update(dict_o, update_o) < 0) {
if (_PyErr_ExceptionMatches(tstate, PyExc_AttributeError)) {
_PyErr_Format(tstate, PyExc_TypeError,
"'%.200s' object is not a mapping",
- Py_TYPE(update)->tp_name);
+ Py_TYPE(update_o)->tp_name);
}
- Py_DECREF(update);
+ PyStackRef_CLOSE(update);
if (true) JUMP_TO_ERROR();
}
- Py_DECREF(update);
+ PyStackRef_CLOSE(update);
stack_pointer += -1;
assert(WITHIN_STACK_BOUNDS());
break;
}
case _DICT_MERGE: {
- PyObject *update;
- PyObject *dict;
- PyObject *callable;
+ _PyStackRef update;
+ _PyStackRef dict;
+ _PyStackRef callable;
oparg = CURRENT_OPARG();
update = stack_pointer[-1];
dict = stack_pointer[-2 - (oparg - 1)];
callable = stack_pointer[-5 - (oparg - 1)];
- if (_PyDict_MergeEx(dict, update, 2) < 0) {
- _PyEval_FormatKwargsError(tstate, callable, update);
- Py_DECREF(update);
+ PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable);
+ PyObject *dict_o = PyStackRef_AsPyObjectBorrow(dict);
+ PyObject *update_o = PyStackRef_AsPyObjectBorrow(update);
+ if (_PyDict_MergeEx(dict_o, update_o, 2) < 0) {
+ _PyEval_FormatKwargsError(tstate, callable_o, update_o);
+ PyStackRef_CLOSE(update);
if (true) JUMP_TO_ERROR();
}
- Py_DECREF(update);
+ PyStackRef_CLOSE(update);
stack_pointer += -1;
assert(WITHIN_STACK_BOUNDS());
break;
}
case _MAP_ADD: {
- PyObject *value;
- PyObject *key;
- PyObject *dict;
+ _PyStackRef value;
+ _PyStackRef key;
+ _PyStackRef dict_st;
oparg = CURRENT_OPARG();
value = stack_pointer[-1];
key = stack_pointer[-2];
- dict = stack_pointer[-3 - (oparg - 1)];
+ dict_st = stack_pointer[-3 - (oparg - 1)];
+ PyObject *dict = PyStackRef_AsPyObjectBorrow(dict_st);
assert(PyDict_CheckExact(dict));
/* dict[key] = value */
// Do not DECREF INPUTS because the function steals the references
- if (_PyDict_SetItem_Take2((PyDictObject *)dict, key, value) != 0) JUMP_TO_ERROR();
+ if (_PyDict_SetItem_Take2((PyDictObject *)dict, PyStackRef_AsPyObjectSteal(key), PyStackRef_AsPyObjectSteal(value)) != 0) JUMP_TO_ERROR();
stack_pointer += -2;
assert(WITHIN_STACK_BOUNDS());
break;
/* _INSTRUMENTED_LOAD_SUPER_ATTR is not a viable micro-op for tier 2 because it is instrumented */
case _LOAD_SUPER_ATTR_ATTR: {
- PyObject *self;
- PyObject *class;
- PyObject *global_super;
- PyObject *attr;
+ _PyStackRef self_st;
+ _PyStackRef class_st;
+ _PyStackRef global_super_st;
+ _PyStackRef attr_st;
oparg = CURRENT_OPARG();
- self = stack_pointer[-1];
- class = stack_pointer[-2];
- global_super = stack_pointer[-3];
+ self_st = stack_pointer[-1];
+ class_st = stack_pointer[-2];
+ global_super_st = stack_pointer[-3];
+ PyObject *global_super = PyStackRef_AsPyObjectBorrow(global_super_st);
+ PyObject *class = PyStackRef_AsPyObjectBorrow(class_st);
+ PyObject *self = PyStackRef_AsPyObjectBorrow(self_st);
assert(!(oparg & 1));
if (global_super != (PyObject *)&PySuper_Type) {
UOP_STAT_INC(uopcode, miss);
}
STAT_INC(LOAD_SUPER_ATTR, hit);
PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 2);
- attr = _PySuper_Lookup((PyTypeObject *)class, self, name, NULL);
- Py_DECREF(global_super);
- Py_DECREF(class);
- Py_DECREF(self);
+ PyObject *attr = _PySuper_Lookup((PyTypeObject *)class, self, name, NULL);
+ PyStackRef_CLOSE(global_super_st);
+ PyStackRef_CLOSE(class_st);
+ PyStackRef_CLOSE(self_st);
if (attr == NULL) JUMP_TO_ERROR();
- stack_pointer[-3] = attr;
+ attr_st = PyStackRef_FromPyObjectSteal(attr);
+ stack_pointer[-3] = attr_st;
stack_pointer += -2;
assert(WITHIN_STACK_BOUNDS());
break;
}
case _LOAD_SUPER_ATTR_METHOD: {
- PyObject *self;
- PyObject *class;
- PyObject *global_super;
- PyObject *attr;
- PyObject *self_or_null;
+ _PyStackRef self_st;
+ _PyStackRef class_st;
+ _PyStackRef global_super_st;
+ _PyStackRef attr;
+ _PyStackRef self_or_null;
oparg = CURRENT_OPARG();
- self = stack_pointer[-1];
- class = stack_pointer[-2];
- global_super = stack_pointer[-3];
+ self_st = stack_pointer[-1];
+ class_st = stack_pointer[-2];
+ global_super_st = stack_pointer[-3];
+ PyObject *global_super = PyStackRef_AsPyObjectBorrow(global_super_st);
+ PyObject *class = PyStackRef_AsPyObjectBorrow(class_st);
+ PyObject *self = PyStackRef_AsPyObjectBorrow(self_st);
assert(oparg & 1);
if (global_super != (PyObject *)&PySuper_Type) {
UOP_STAT_INC(uopcode, miss);
PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 2);
PyTypeObject *cls = (PyTypeObject *)class;
int method_found = 0;
- attr = _PySuper_Lookup(cls, self, name,
- Py_TYPE(self)->tp_getattro == PyObject_GenericGetAttr ? &method_found : NULL);
- Py_DECREF(global_super);
- Py_DECREF(class);
- if (attr == NULL) {
- Py_DECREF(self);
+ PyObject *attr_o = _PySuper_Lookup(cls, self, name,
+ Py_TYPE(self)->tp_getattro == PyObject_GenericGetAttr ? &method_found : NULL);
+ PyStackRef_CLOSE(global_super_st);
+ PyStackRef_CLOSE(class_st);
+ if (attr_o == NULL) {
+ PyStackRef_CLOSE(self_st);
if (true) JUMP_TO_ERROR();
}
if (method_found) {
- self_or_null = self; // transfer ownership
+ self_or_null = self_st; // transfer ownership
} else {
- Py_DECREF(self);
- self_or_null = NULL;
+ PyStackRef_CLOSE(self_st);
+ self_or_null = PyStackRef_NULL;
}
+ attr = PyStackRef_FromPyObjectSteal(attr_o);
stack_pointer[-3] = attr;
stack_pointer[-2] = self_or_null;
stack_pointer += -1;
}
case _LOAD_ATTR: {
- PyObject *owner;
- PyObject *attr;
- PyObject *self_or_null = NULL;
+ _PyStackRef owner;
+ _PyStackRef attr;
+ _PyStackRef self_or_null = PyStackRef_NULL;
oparg = CURRENT_OPARG();
owner = stack_pointer[-1];
PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 1);
+ PyObject *attr_o;
if (oparg & 1) {
/* Designed to work in tandem with CALL, pushes two values. */
- attr = NULL;
- if (_PyObject_GetMethod(owner, name, &attr)) {
+ attr_o = NULL;
+ if (_PyObject_GetMethod(PyStackRef_AsPyObjectBorrow(owner), name, &attr_o)) {
/* We can bypass temporary bound method object.
meth is unbound method and obj is self.
meth | self | arg1 | ... | argN
*/
- assert(attr != NULL); // No errors on this branch
+ assert(attr_o != NULL); // No errors on this branch
self_or_null = owner; // Transfer ownership
}
else {
CALL that it's not a method call.
meth | NULL | arg1 | ... | argN
*/
- Py_DECREF(owner);
- if (attr == NULL) JUMP_TO_ERROR();
- self_or_null = NULL;
+ PyStackRef_CLOSE(owner);
+ if (attr_o == NULL) JUMP_TO_ERROR();
+ self_or_null = PyStackRef_NULL;
}
}
else {
/* Classic, pushes one value. */
- attr = PyObject_GetAttr(owner, name);
- Py_DECREF(owner);
- if (attr == NULL) JUMP_TO_ERROR();
+ attr_o = PyObject_GetAttr(PyStackRef_AsPyObjectBorrow(owner), name);
+ PyStackRef_CLOSE(owner);
+ if (attr_o == NULL) JUMP_TO_ERROR();
}
+ attr = PyStackRef_FromPyObjectSteal(attr_o);
stack_pointer[-1] = attr;
if (oparg & 1) stack_pointer[0] = self_or_null;
stack_pointer += (oparg & 1);
}
case _GUARD_TYPE_VERSION: {
- PyObject *owner;
+ _PyStackRef owner;
owner = stack_pointer[-1];
uint32_t type_version = (uint32_t)CURRENT_OPERAND();
- PyTypeObject *tp = Py_TYPE(owner);
+ PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner));
assert(type_version != 0);
if (tp->tp_version_tag != type_version) {
UOP_STAT_INC(uopcode, miss);
}
case _CHECK_MANAGED_OBJECT_HAS_VALUES: {
- PyObject *owner;
+ _PyStackRef owner;
owner = stack_pointer[-1];
- assert(Py_TYPE(owner)->tp_dictoffset < 0);
- assert(Py_TYPE(owner)->tp_flags & Py_TPFLAGS_INLINE_VALUES);
- if (!_PyObject_InlineValues(owner)->valid) {
+ PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner);
+ assert(Py_TYPE(owner_o)->tp_dictoffset < 0);
+ assert(Py_TYPE(owner_o)->tp_flags & Py_TPFLAGS_INLINE_VALUES);
+ if (!_PyObject_InlineValues(owner_o)->valid) {
UOP_STAT_INC(uopcode, miss);
JUMP_TO_JUMP_TARGET();
}
}
case _LOAD_ATTR_INSTANCE_VALUE_0: {
- PyObject *owner;
- PyObject *attr;
- PyObject *null = NULL;
+ _PyStackRef owner;
+ _PyStackRef attr;
+ _PyStackRef null = PyStackRef_NULL;
(void)null;
owner = stack_pointer[-1];
uint16_t index = (uint16_t)CURRENT_OPERAND();
- attr = _PyObject_InlineValues(owner)->values[index];
- if (attr == NULL) {
+ PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner);
+ PyObject *attr_o = _PyObject_InlineValues(owner_o)->values[index];
+ if (attr_o == NULL) {
UOP_STAT_INC(uopcode, miss);
JUMP_TO_JUMP_TARGET();
}
STAT_INC(LOAD_ATTR, hit);
- Py_INCREF(attr);
- null = NULL;
- Py_DECREF(owner);
+ Py_INCREF(attr_o);
+ null = PyStackRef_NULL;
+ attr = PyStackRef_FromPyObjectSteal(attr_o);
+ PyStackRef_CLOSE(owner);
stack_pointer[-1] = attr;
break;
}
case _LOAD_ATTR_INSTANCE_VALUE_1: {
- PyObject *owner;
- PyObject *attr;
- PyObject *null = NULL;
+ _PyStackRef owner;
+ _PyStackRef attr;
+ _PyStackRef null = PyStackRef_NULL;
(void)null;
owner = stack_pointer[-1];
uint16_t index = (uint16_t)CURRENT_OPERAND();
- attr = _PyObject_InlineValues(owner)->values[index];
- if (attr == NULL) {
+ PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner);
+ PyObject *attr_o = _PyObject_InlineValues(owner_o)->values[index];
+ if (attr_o == NULL) {
UOP_STAT_INC(uopcode, miss);
JUMP_TO_JUMP_TARGET();
}
STAT_INC(LOAD_ATTR, hit);
- Py_INCREF(attr);
- null = NULL;
- Py_DECREF(owner);
+ Py_INCREF(attr_o);
+ null = PyStackRef_NULL;
+ attr = PyStackRef_FromPyObjectSteal(attr_o);
+ PyStackRef_CLOSE(owner);
stack_pointer[-1] = attr;
stack_pointer[0] = null;
stack_pointer += 1;
/* _LOAD_ATTR_INSTANCE_VALUE is split on (oparg & 1) */
case _CHECK_ATTR_MODULE: {
- PyObject *owner;
+ _PyStackRef owner;
owner = stack_pointer[-1];
uint32_t dict_version = (uint32_t)CURRENT_OPERAND();
- if (!PyModule_CheckExact(owner)) {
+ PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner);
+ if (!PyModule_CheckExact(owner_o)) {
UOP_STAT_INC(uopcode, miss);
JUMP_TO_JUMP_TARGET();
}
- PyDictObject *dict = (PyDictObject *)((PyModuleObject *)owner)->md_dict;
+ PyDictObject *dict = (PyDictObject *)((PyModuleObject *)owner_o)->md_dict;
assert(dict != NULL);
if (dict->ma_keys->dk_version != dict_version) {
UOP_STAT_INC(uopcode, miss);
}
case _LOAD_ATTR_MODULE: {
- PyObject *owner;
- PyObject *attr;
- PyObject *null = NULL;
+ _PyStackRef owner;
+ _PyStackRef attr;
+ _PyStackRef null = PyStackRef_NULL;
oparg = CURRENT_OPARG();
owner = stack_pointer[-1];
uint16_t index = (uint16_t)CURRENT_OPERAND();
- PyDictObject *dict = (PyDictObject *)((PyModuleObject *)owner)->md_dict;
+ PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner);
+ PyDictObject *dict = (PyDictObject *)((PyModuleObject *)owner_o)->md_dict;
assert(dict->ma_keys->dk_kind == DICT_KEYS_UNICODE);
assert(index < dict->ma_keys->dk_nentries);
PyDictUnicodeEntry *ep = DK_UNICODE_ENTRIES(dict->ma_keys) + index;
- attr = ep->me_value;
- if (attr == NULL) {
+ PyObject *attr_o = ep->me_value;
+ if (attr_o == NULL) {
UOP_STAT_INC(uopcode, miss);
JUMP_TO_JUMP_TARGET();
}
STAT_INC(LOAD_ATTR, hit);
- Py_INCREF(attr);
- null = NULL;
- Py_DECREF(owner);
+ Py_INCREF(attr_o);
+ attr = PyStackRef_FromPyObjectSteal(attr_o);
+ null = PyStackRef_NULL;
+ PyStackRef_CLOSE(owner);
stack_pointer[-1] = attr;
if (oparg & 1) stack_pointer[0] = null;
stack_pointer += (oparg & 1);
}
case _CHECK_ATTR_WITH_HINT: {
- PyObject *owner;
+ _PyStackRef owner;
owner = stack_pointer[-1];
- assert(Py_TYPE(owner)->tp_flags & Py_TPFLAGS_MANAGED_DICT);
- PyDictObject *dict = _PyObject_GetManagedDict(owner);
+ PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner);
+ assert(Py_TYPE(owner_o)->tp_flags & Py_TPFLAGS_MANAGED_DICT);
+ PyDictObject *dict = _PyObject_GetManagedDict(owner_o);
if (dict == NULL) {
UOP_STAT_INC(uopcode, miss);
JUMP_TO_JUMP_TARGET();
}
case _LOAD_ATTR_WITH_HINT: {
- PyObject *owner;
- PyObject *attr;
- PyObject *null = NULL;
+ _PyStackRef owner;
+ _PyStackRef attr;
+ _PyStackRef null = PyStackRef_NULL;
oparg = CURRENT_OPARG();
owner = stack_pointer[-1];
uint16_t hint = (uint16_t)CURRENT_OPERAND();
- PyDictObject *dict = _PyObject_GetManagedDict(owner);
+ PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner);
+ PyObject *attr_o;
+ PyDictObject *dict = _PyObject_GetManagedDict(owner_o);
if (hint >= (size_t)dict->ma_keys->dk_nentries) {
UOP_STAT_INC(uopcode, miss);
JUMP_TO_JUMP_TARGET();
UOP_STAT_INC(uopcode, miss);
JUMP_TO_JUMP_TARGET();
}
- attr = ep->me_value;
+ attr_o = ep->me_value;
}
else {
PyDictKeyEntry *ep = DK_ENTRIES(dict->ma_keys) + hint;
UOP_STAT_INC(uopcode, miss);
JUMP_TO_JUMP_TARGET();
}
- attr = ep->me_value;
+ attr_o = ep->me_value;
}
- if (attr == NULL) {
+ if (attr_o == NULL) {
UOP_STAT_INC(uopcode, miss);
JUMP_TO_JUMP_TARGET();
}
STAT_INC(LOAD_ATTR, hit);
- Py_INCREF(attr);
- null = NULL;
- Py_DECREF(owner);
+ Py_INCREF(attr_o);
+ attr = PyStackRef_FromPyObjectSteal(attr_o);
+ null = PyStackRef_NULL;
+ PyStackRef_CLOSE(owner);
stack_pointer[-1] = attr;
if (oparg & 1) stack_pointer[0] = null;
stack_pointer += (oparg & 1);
}
case _LOAD_ATTR_SLOT_0: {
- PyObject *owner;
- PyObject *attr;
- PyObject *null = NULL;
+ _PyStackRef owner;
+ _PyStackRef attr;
+ _PyStackRef null = PyStackRef_NULL;
(void)null;
owner = stack_pointer[-1];
uint16_t index = (uint16_t)CURRENT_OPERAND();
- char *addr = (char *)owner + index;
- attr = *(PyObject **)addr;
- if (attr == NULL) {
+ PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner);
+ char *addr = (char *)owner_o + index;
+ PyObject *attr_o = *(PyObject **)addr;
+ if (attr_o == NULL) {
UOP_STAT_INC(uopcode, miss);
JUMP_TO_JUMP_TARGET();
}
STAT_INC(LOAD_ATTR, hit);
- Py_INCREF(attr);
- null = NULL;
- Py_DECREF(owner);
+ null = PyStackRef_NULL;
+ attr = PyStackRef_FromPyObjectNew(attr_o);
+ PyStackRef_CLOSE(owner);
stack_pointer[-1] = attr;
break;
}
case _LOAD_ATTR_SLOT_1: {
- PyObject *owner;
- PyObject *attr;
- PyObject *null = NULL;
+ _PyStackRef owner;
+ _PyStackRef attr;
+ _PyStackRef null = PyStackRef_NULL;
(void)null;
owner = stack_pointer[-1];
uint16_t index = (uint16_t)CURRENT_OPERAND();
- char *addr = (char *)owner + index;
- attr = *(PyObject **)addr;
- if (attr == NULL) {
+ PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner);
+ char *addr = (char *)owner_o + index;
+ PyObject *attr_o = *(PyObject **)addr;
+ if (attr_o == NULL) {
UOP_STAT_INC(uopcode, miss);
JUMP_TO_JUMP_TARGET();
}
STAT_INC(LOAD_ATTR, hit);
- Py_INCREF(attr);
- null = NULL;
- Py_DECREF(owner);
+ null = PyStackRef_NULL;
+ attr = PyStackRef_FromPyObjectNew(attr_o);
+ PyStackRef_CLOSE(owner);
stack_pointer[-1] = attr;
stack_pointer[0] = null;
stack_pointer += 1;
/* _LOAD_ATTR_SLOT is split on (oparg & 1) */
case _CHECK_ATTR_CLASS: {
- PyObject *owner;
+ _PyStackRef owner;
owner = stack_pointer[-1];
uint32_t type_version = (uint32_t)CURRENT_OPERAND();
- if (!PyType_Check(owner)) {
+ PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner);
+ if (!PyType_Check(owner_o)) {
UOP_STAT_INC(uopcode, miss);
JUMP_TO_JUMP_TARGET();
}
assert(type_version != 0);
- if (((PyTypeObject *)owner)->tp_version_tag != type_version) {
+ if (((PyTypeObject *)owner_o)->tp_version_tag != type_version) {
UOP_STAT_INC(uopcode, miss);
JUMP_TO_JUMP_TARGET();
}
}
case _LOAD_ATTR_CLASS_0: {
- PyObject *owner;
- PyObject *attr;
- PyObject *null = NULL;
+ _PyStackRef owner;
+ _PyStackRef attr;
+ _PyStackRef null = PyStackRef_NULL;
(void)null;
owner = stack_pointer[-1];
PyObject *descr = (PyObject *)CURRENT_OPERAND();
STAT_INC(LOAD_ATTR, hit);
assert(descr != NULL);
- attr = Py_NewRef(descr);
- null = NULL;
- Py_DECREF(owner);
+ attr = PyStackRef_FromPyObjectNew(descr);
+ null = PyStackRef_NULL;
+ PyStackRef_CLOSE(owner);
stack_pointer[-1] = attr;
break;
}
case _LOAD_ATTR_CLASS_1: {
- PyObject *owner;
- PyObject *attr;
- PyObject *null = NULL;
+ _PyStackRef owner;
+ _PyStackRef attr;
+ _PyStackRef null = PyStackRef_NULL;
(void)null;
owner = stack_pointer[-1];
PyObject *descr = (PyObject *)CURRENT_OPERAND();
STAT_INC(LOAD_ATTR, hit);
assert(descr != NULL);
- attr = Py_NewRef(descr);
- null = NULL;
- Py_DECREF(owner);
+ attr = PyStackRef_FromPyObjectNew(descr);
+ null = PyStackRef_NULL;
+ PyStackRef_CLOSE(owner);
stack_pointer[-1] = attr;
stack_pointer[0] = null;
stack_pointer += 1;
/* _LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN is not a viable micro-op for tier 2 because it uses the 'this_instr' variable */
case _GUARD_DORV_NO_DICT: {
- PyObject *owner;
+ _PyStackRef owner;
owner = stack_pointer[-1];
- assert(Py_TYPE(owner)->tp_dictoffset < 0);
- assert(Py_TYPE(owner)->tp_flags & Py_TPFLAGS_INLINE_VALUES);
- if (_PyObject_GetManagedDict(owner)) {
+ PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner);
+ assert(Py_TYPE(owner_o)->tp_dictoffset < 0);
+ assert(Py_TYPE(owner_o)->tp_flags & Py_TPFLAGS_INLINE_VALUES);
+ if (_PyObject_GetManagedDict(owner_o)) {
UOP_STAT_INC(uopcode, miss);
JUMP_TO_JUMP_TARGET();
}
- if (_PyObject_InlineValues(owner)->valid == 0) {
+ if (_PyObject_InlineValues(owner_o)->valid == 0) {
UOP_STAT_INC(uopcode, miss);
JUMP_TO_JUMP_TARGET();
}
}
case _STORE_ATTR_INSTANCE_VALUE: {
- PyObject *owner;
- PyObject *value;
+ _PyStackRef owner;
+ _PyStackRef value;
owner = stack_pointer[-1];
value = stack_pointer[-2];
uint16_t index = (uint16_t)CURRENT_OPERAND();
+ PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner);
STAT_INC(STORE_ATTR, hit);
- assert(_PyObject_GetManagedDict(owner) == NULL);
- PyDictValues *values = _PyObject_InlineValues(owner);
+ assert(_PyObject_GetManagedDict(owner_o) == NULL);
+ PyDictValues *values = _PyObject_InlineValues(owner_o);
PyObject *old_value = values->values[index];
- values->values[index] = value;
+ values->values[index] = PyStackRef_AsPyObjectSteal(value);
if (old_value == NULL) {
_PyDictValues_AddToInsertionOrder(values, index);
}
else {
Py_DECREF(old_value);
}
- Py_DECREF(owner);
+ PyStackRef_CLOSE(owner);
stack_pointer += -2;
assert(WITHIN_STACK_BOUNDS());
break;
}
case _STORE_ATTR_WITH_HINT: {
- PyObject *owner;
- PyObject *value;
+ _PyStackRef owner;
+ _PyStackRef value;
oparg = CURRENT_OPARG();
owner = stack_pointer[-1];
value = stack_pointer[-2];
uint16_t hint = (uint16_t)CURRENT_OPERAND();
- assert(Py_TYPE(owner)->tp_flags & Py_TPFLAGS_MANAGED_DICT);
- PyDictObject *dict = _PyObject_GetManagedDict(owner);
+ PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner);
+ assert(Py_TYPE(owner_o)->tp_flags & Py_TPFLAGS_MANAGED_DICT);
+ PyDictObject *dict = _PyObject_GetManagedDict(owner_o);
if (dict == NULL) {
UOP_STAT_INC(uopcode, miss);
JUMP_TO_JUMP_TARGET();
UOP_STAT_INC(uopcode, miss);
JUMP_TO_JUMP_TARGET();
}
- new_version = _PyDict_NotifyEvent(tstate->interp, PyDict_EVENT_MODIFIED, dict, name, value);
- ep->me_value = value;
+ new_version = _PyDict_NotifyEvent(tstate->interp, PyDict_EVENT_MODIFIED, dict, name, PyStackRef_AsPyObjectBorrow(value));
+ ep->me_value = PyStackRef_AsPyObjectSteal(value);
}
else {
PyDictKeyEntry *ep = DK_ENTRIES(dict->ma_keys) + hint;
UOP_STAT_INC(uopcode, miss);
JUMP_TO_JUMP_TARGET();
}
- new_version = _PyDict_NotifyEvent(tstate->interp, PyDict_EVENT_MODIFIED, dict, name, value);
- ep->me_value = value;
+ new_version = _PyDict_NotifyEvent(tstate->interp, PyDict_EVENT_MODIFIED, dict, name, PyStackRef_AsPyObjectBorrow(value));
+ ep->me_value = PyStackRef_AsPyObjectSteal(value);
}
Py_DECREF(old_value);
STAT_INC(STORE_ATTR, hit);
/* Ensure dict is GC tracked if it needs to be */
- if (!_PyObject_GC_IS_TRACKED(dict) && _PyObject_GC_MAY_BE_TRACKED(value)) {
+ if (!_PyObject_GC_IS_TRACKED(dict) && _PyObject_GC_MAY_BE_TRACKED(PyStackRef_AsPyObjectBorrow(value))) {
_PyObject_GC_TRACK(dict);
}
/* PEP 509 */
dict->ma_version_tag = new_version;
- Py_DECREF(owner);
+ PyStackRef_CLOSE(owner);
stack_pointer += -2;
assert(WITHIN_STACK_BOUNDS());
break;
}
case _STORE_ATTR_SLOT: {
- PyObject *owner;
- PyObject *value;
+ _PyStackRef owner;
+ _PyStackRef value;
owner = stack_pointer[-1];
value = stack_pointer[-2];
uint16_t index = (uint16_t)CURRENT_OPERAND();
- char *addr = (char *)owner + index;
+ PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner);
+ char *addr = (char *)owner_o + index;
STAT_INC(STORE_ATTR, hit);
PyObject *old_value = *(PyObject **)addr;
- *(PyObject **)addr = value;
+ *(PyObject **)addr = PyStackRef_AsPyObjectSteal(value);
Py_XDECREF(old_value);
- Py_DECREF(owner);
+ PyStackRef_CLOSE(owner);
stack_pointer += -2;
assert(WITHIN_STACK_BOUNDS());
break;
}
case _COMPARE_OP: {
- PyObject *right;
- PyObject *left;
- PyObject *res;
+ _PyStackRef right;
+ _PyStackRef left;
+ _PyStackRef res;
oparg = CURRENT_OPARG();
right = stack_pointer[-1];
left = stack_pointer[-2];
+ PyObject *left_o = PyStackRef_AsPyObjectBorrow(left);
+ PyObject *right_o = PyStackRef_AsPyObjectBorrow(right);
assert((oparg >> 5) <= Py_GE);
- res = PyObject_RichCompare(left, right, oparg >> 5);
- Py_DECREF(left);
- Py_DECREF(right);
- if (res == NULL) JUMP_TO_ERROR();
+ PyObject *res_o = PyObject_RichCompare(left_o, right_o, oparg >> 5);
+ PyStackRef_CLOSE(left);
+ PyStackRef_CLOSE(right);
+ if (res_o == NULL) JUMP_TO_ERROR();
if (oparg & 16) {
- int res_bool = PyObject_IsTrue(res);
- Py_DECREF(res);
+ int res_bool = PyObject_IsTrue(res_o);
+ Py_DECREF(res_o);
if (res_bool < 0) JUMP_TO_ERROR();
- res = res_bool ? Py_True : Py_False;
+ res = res_bool ? PyStackRef_True : PyStackRef_False;
+ }
+ else {
+ res = PyStackRef_FromPyObjectSteal(res_o);
}
stack_pointer[-2] = res;
stack_pointer += -1;
}
case _COMPARE_OP_FLOAT: {
- PyObject *right;
- PyObject *left;
- PyObject *res;
+ _PyStackRef right;
+ _PyStackRef left;
+ _PyStackRef res;
oparg = CURRENT_OPARG();
right = stack_pointer[-1];
left = stack_pointer[-2];
+ PyObject *left_o = PyStackRef_AsPyObjectBorrow(left);
+ PyObject *right_o = PyStackRef_AsPyObjectBorrow(right);
STAT_INC(COMPARE_OP, hit);
- double dleft = PyFloat_AS_DOUBLE(left);
- double dright = PyFloat_AS_DOUBLE(right);
+ double dleft = PyFloat_AS_DOUBLE(left_o);
+ double dright = PyFloat_AS_DOUBLE(right_o);
// 1 if NaN, 2 if <, 4 if >, 8 if ==; this matches low four bits of the oparg
int sign_ish = COMPARISON_BIT(dleft, dright);
- _Py_DECREF_SPECIALIZED(left, _PyFloat_ExactDealloc);
- _Py_DECREF_SPECIALIZED(right, _PyFloat_ExactDealloc);
- res = (sign_ish & oparg) ? Py_True : Py_False;
+ _Py_DECREF_SPECIALIZED(left_o, _PyFloat_ExactDealloc);
+ _Py_DECREF_SPECIALIZED(right_o, _PyFloat_ExactDealloc);
+ res = (sign_ish & oparg) ? PyStackRef_True : PyStackRef_False;
// It's always a bool, so we don't care about oparg & 16.
stack_pointer[-2] = res;
stack_pointer += -1;
}
case _COMPARE_OP_INT: {
- PyObject *right;
- PyObject *left;
- PyObject *res;
+ _PyStackRef right;
+ _PyStackRef left;
+ _PyStackRef res;
oparg = CURRENT_OPARG();
right = stack_pointer[-1];
left = stack_pointer[-2];
- if (!_PyLong_IsCompact((PyLongObject *)left)) {
+ PyObject *left_o = PyStackRef_AsPyObjectBorrow(left);
+ PyObject *right_o = PyStackRef_AsPyObjectBorrow(right);
+ if (!_PyLong_IsCompact((PyLongObject *)left_o)) {
UOP_STAT_INC(uopcode, miss);
JUMP_TO_JUMP_TARGET();
}
- if (!_PyLong_IsCompact((PyLongObject *)right)) {
+ if (!_PyLong_IsCompact((PyLongObject *)right_o)) {
UOP_STAT_INC(uopcode, miss);
JUMP_TO_JUMP_TARGET();
}
STAT_INC(COMPARE_OP, hit);
- assert(_PyLong_DigitCount((PyLongObject *)left) <= 1 &&
- _PyLong_DigitCount((PyLongObject *)right) <= 1);
- Py_ssize_t ileft = _PyLong_CompactValue((PyLongObject *)left);
- Py_ssize_t iright = _PyLong_CompactValue((PyLongObject *)right);
+ assert(_PyLong_DigitCount((PyLongObject *)left_o) <= 1 &&
+ _PyLong_DigitCount((PyLongObject *)right_o) <= 1);
+ Py_ssize_t ileft = _PyLong_CompactValue((PyLongObject *)left_o);
+ Py_ssize_t iright = _PyLong_CompactValue((PyLongObject *)right_o);
// 2 if <, 4 if >, 8 if ==; this matches the low 4 bits of the oparg
int sign_ish = COMPARISON_BIT(ileft, iright);
- _Py_DECREF_SPECIALIZED(left, (destructor)PyObject_Free);
- _Py_DECREF_SPECIALIZED(right, (destructor)PyObject_Free);
- res = (sign_ish & oparg) ? Py_True : Py_False;
+ _Py_DECREF_SPECIALIZED(left_o, (destructor)PyObject_Free);
+ _Py_DECREF_SPECIALIZED(right_o, (destructor)PyObject_Free);
+ res = (sign_ish & oparg) ? PyStackRef_True : PyStackRef_False;
// It's always a bool, so we don't care about oparg & 16.
stack_pointer[-2] = res;
stack_pointer += -1;
}
case _COMPARE_OP_STR: {
- PyObject *right;
- PyObject *left;
- PyObject *res;
+ _PyStackRef right;
+ _PyStackRef left;
+ _PyStackRef res;
oparg = CURRENT_OPARG();
right = stack_pointer[-1];
left = stack_pointer[-2];
+ PyObject *left_o = PyStackRef_AsPyObjectBorrow(left);
+ PyObject *right_o = PyStackRef_AsPyObjectBorrow(right);
STAT_INC(COMPARE_OP, hit);
- int eq = _PyUnicode_Equal(left, right);
+ int eq = _PyUnicode_Equal(left_o, right_o);
assert((oparg >> 5) == Py_EQ || (oparg >> 5) == Py_NE);
- _Py_DECREF_SPECIALIZED(left, _PyUnicode_ExactDealloc);
- _Py_DECREF_SPECIALIZED(right, _PyUnicode_ExactDealloc);
+ _Py_DECREF_SPECIALIZED(left_o, _PyUnicode_ExactDealloc);
+ _Py_DECREF_SPECIALIZED(right_o, _PyUnicode_ExactDealloc);
assert(eq == 0 || eq == 1);
assert((oparg & 0xf) == COMPARISON_NOT_EQUALS || (oparg & 0xf) == COMPARISON_EQUALS);
assert(COMPARISON_NOT_EQUALS + 1 == COMPARISON_EQUALS);
- res = ((COMPARISON_NOT_EQUALS + eq) & oparg) ? Py_True : Py_False;
+ res = ((COMPARISON_NOT_EQUALS + eq) & oparg) ? PyStackRef_True : PyStackRef_False;
// It's always a bool, so we don't care about oparg & 16.
stack_pointer[-2] = res;
stack_pointer += -1;
}
case _IS_OP: {
- PyObject *right;
- PyObject *left;
- PyObject *b;
+ _PyStackRef right;
+ _PyStackRef left;
+ _PyStackRef b;
oparg = CURRENT_OPARG();
right = stack_pointer[-1];
left = stack_pointer[-2];
- int res = Py_Is(left, right) ^ oparg;
- Py_DECREF(left);
- Py_DECREF(right);
- b = res ? Py_True : Py_False;
+ #ifdef Py_GIL_DISABLED
+ // On free-threaded builds, objects are conditionally immortalized.
+ // So their bits don't always compare equally.
+ int res = Py_Is(PyStackRef_AsPyObjectBorrow(left), PyStackRef_AsPyObjectBorrow(right)) ^ oparg;
+ #else
+ int res = PyStackRef_Is(left, right) ^ oparg;
+ #endif
+ PyStackRef_CLOSE(left);
+ PyStackRef_CLOSE(right);
+ b = res ? PyStackRef_True : PyStackRef_False;
stack_pointer[-2] = b;
stack_pointer += -1;
assert(WITHIN_STACK_BOUNDS());
}
case _CONTAINS_OP: {
- PyObject *right;
- PyObject *left;
- PyObject *b;
+ _PyStackRef right;
+ _PyStackRef left;
+ _PyStackRef b;
oparg = CURRENT_OPARG();
right = stack_pointer[-1];
left = stack_pointer[-2];
- int res = PySequence_Contains(right, left);
- Py_DECREF(left);
- Py_DECREF(right);
+ PyObject *left_o = PyStackRef_AsPyObjectBorrow(left);
+ PyObject *right_o = PyStackRef_AsPyObjectBorrow(right);
+ int res = PySequence_Contains(right_o, left_o);
+ PyStackRef_CLOSE(left);
+ PyStackRef_CLOSE(right);
if (res < 0) JUMP_TO_ERROR();
- b = (res ^ oparg) ? Py_True : Py_False;
+ b = (res ^ oparg) ? PyStackRef_True : PyStackRef_False;
stack_pointer[-2] = b;
stack_pointer += -1;
assert(WITHIN_STACK_BOUNDS());
}
case _CONTAINS_OP_SET: {
- PyObject *right;
- PyObject *left;
- PyObject *b;
+ _PyStackRef right;
+ _PyStackRef left;
+ _PyStackRef b;
oparg = CURRENT_OPARG();
right = stack_pointer[-1];
left = stack_pointer[-2];
- if (!(PySet_CheckExact(right) || PyFrozenSet_CheckExact(right))) {
+ PyObject *left_o = PyStackRef_AsPyObjectBorrow(left);
+ PyObject *right_o = PyStackRef_AsPyObjectBorrow(right);
+ if (!(PySet_CheckExact(right_o) || PyFrozenSet_CheckExact(right_o))) {
UOP_STAT_INC(uopcode, miss);
JUMP_TO_JUMP_TARGET();
}
STAT_INC(CONTAINS_OP, hit);
// Note: both set and frozenset use the same seq_contains method!
- int res = _PySet_Contains((PySetObject *)right, left);
- Py_DECREF(left);
- Py_DECREF(right);
+ int res = _PySet_Contains((PySetObject *)right_o, left_o);
+ PyStackRef_CLOSE(left);
+ PyStackRef_CLOSE(right);
if (res < 0) JUMP_TO_ERROR();
- b = (res ^ oparg) ? Py_True : Py_False;
+ b = (res ^ oparg) ? PyStackRef_True : PyStackRef_False;
stack_pointer[-2] = b;
stack_pointer += -1;
assert(WITHIN_STACK_BOUNDS());
}
case _CONTAINS_OP_DICT: {
- PyObject *right;
- PyObject *left;
- PyObject *b;
+ _PyStackRef right;
+ _PyStackRef left;
+ _PyStackRef b;
oparg = CURRENT_OPARG();
right = stack_pointer[-1];
left = stack_pointer[-2];
- if (!PyDict_CheckExact(right)) {
+ PyObject *left_o = PyStackRef_AsPyObjectBorrow(left);
+ PyObject *right_o = PyStackRef_AsPyObjectBorrow(right);
+ if (!PyDict_CheckExact(right_o)) {
UOP_STAT_INC(uopcode, miss);
JUMP_TO_JUMP_TARGET();
}
STAT_INC(CONTAINS_OP, hit);
- int res = PyDict_Contains(right, left);
- Py_DECREF(left);
- Py_DECREF(right);
+ int res = PyDict_Contains(right_o, left_o);
+ PyStackRef_CLOSE(left);
+ PyStackRef_CLOSE(right);
if (res < 0) JUMP_TO_ERROR();
- b = (res ^ oparg) ? Py_True : Py_False;
+ b = (res ^ oparg) ? PyStackRef_True : PyStackRef_False;
stack_pointer[-2] = b;
stack_pointer += -1;
assert(WITHIN_STACK_BOUNDS());
}
case _CHECK_EG_MATCH: {
- PyObject *match_type;
- PyObject *exc_value;
- PyObject *rest;
- PyObject *match;
- match_type = stack_pointer[-1];
- exc_value = stack_pointer[-2];
+ _PyStackRef match_type_st;
+ _PyStackRef exc_value_st;
+ _PyStackRef rest;
+ _PyStackRef match;
+ match_type_st = stack_pointer[-1];
+ exc_value_st = stack_pointer[-2];
+ PyObject *exc_value = PyStackRef_AsPyObjectBorrow(exc_value_st);
+ PyObject *match_type = PyStackRef_AsPyObjectBorrow(match_type_st);
if (_PyEval_CheckExceptStarTypeValid(tstate, match_type) < 0) {
- Py_DECREF(exc_value);
- Py_DECREF(match_type);
+ PyStackRef_CLOSE(exc_value_st);
+ PyStackRef_CLOSE(match_type_st);
if (true) JUMP_TO_ERROR();
}
- match = NULL;
- rest = NULL;
+ PyObject *match_o = NULL;
+ PyObject *rest_o = NULL;
int res = _PyEval_ExceptionGroupMatch(exc_value, match_type,
- &match, &rest);
- Py_DECREF(exc_value);
- Py_DECREF(match_type);
+ &match_o, &rest_o);
+ PyStackRef_CLOSE(exc_value_st);
+ PyStackRef_CLOSE(match_type_st);
if (res < 0) JUMP_TO_ERROR();
- assert((match == NULL) == (rest == NULL));
- if (match == NULL) JUMP_TO_ERROR();
- if (!Py_IsNone(match)) {
- PyErr_SetHandledException(match);
+ assert((match_o == NULL) == (rest_o == NULL));
+ if (match_o == NULL) JUMP_TO_ERROR();
+ if (!Py_IsNone(match_o)) {
+ PyErr_SetHandledException(match_o);
}
+ rest = PyStackRef_FromPyObjectSteal(rest_o);
+ match = PyStackRef_FromPyObjectSteal(match_o);
stack_pointer[-2] = rest;
stack_pointer[-1] = match;
break;
}
case _CHECK_EXC_MATCH: {
- PyObject *right;
- PyObject *left;
- PyObject *b;
+ _PyStackRef right;
+ _PyStackRef left;
+ _PyStackRef b;
right = stack_pointer[-1];
left = stack_pointer[-2];
- assert(PyExceptionInstance_Check(left));
- if (_PyEval_CheckExceptTypeValid(tstate, right) < 0) {
- Py_DECREF(right);
+ PyObject *left_o = PyStackRef_AsPyObjectBorrow(left);
+ PyObject *right_o = PyStackRef_AsPyObjectBorrow(right);
+ assert(PyExceptionInstance_Check(left_o));
+ if (_PyEval_CheckExceptTypeValid(tstate, right_o) < 0) {
+ PyStackRef_CLOSE(right);
if (true) JUMP_TO_ERROR();
}
- int res = PyErr_GivenExceptionMatches(left, right);
- Py_DECREF(right);
- b = res ? Py_True : Py_False;
+ int res = PyErr_GivenExceptionMatches(left_o, right_o);
+ PyStackRef_CLOSE(right);
+ b = res ? PyStackRef_True : PyStackRef_False;
stack_pointer[-1] = b;
break;
}
/* _POP_JUMP_IF_TRUE is not a viable micro-op for tier 2 because it is replaced */
case _IS_NONE: {
- PyObject *value;
- PyObject *b;
+ _PyStackRef value;
+ _PyStackRef b;
value = stack_pointer[-1];
- if (Py_IsNone(value)) {
- b = Py_True;
+ if (PyStackRef_Is(value, PyStackRef_None)) {
+ b = PyStackRef_True;
}
else {
- b = Py_False;
- Py_DECREF(value);
+ b = PyStackRef_False;
+ PyStackRef_CLOSE(value);
}
stack_pointer[-1] = b;
break;
}
case _GET_LEN: {
- PyObject *obj;
- PyObject *len_o;
+ _PyStackRef obj;
+ _PyStackRef len;
obj = stack_pointer[-1];
// PUSH(len(TOS))
- Py_ssize_t len_i = PyObject_Length(obj);
+ Py_ssize_t len_i = PyObject_Length(PyStackRef_AsPyObjectBorrow(obj));
if (len_i < 0) JUMP_TO_ERROR();
- len_o = PyLong_FromSsize_t(len_i);
+ PyObject *len_o = PyLong_FromSsize_t(len_i);
if (len_o == NULL) JUMP_TO_ERROR();
- stack_pointer[0] = len_o;
+ len = PyStackRef_FromPyObjectSteal(len_o);
+ stack_pointer[0] = len;
stack_pointer += 1;
assert(WITHIN_STACK_BOUNDS());
break;
}
case _MATCH_CLASS: {
- PyObject *names;
- PyObject *type;
- PyObject *subject;
- PyObject *attrs;
+ _PyStackRef names;
+ _PyStackRef type;
+ _PyStackRef subject;
+ _PyStackRef attrs;
oparg = CURRENT_OPARG();
names = stack_pointer[-1];
type = stack_pointer[-2];
subject = stack_pointer[-3];
// Pop TOS and TOS1. Set TOS to a tuple of attributes on success, or
// None on failure.
- assert(PyTuple_CheckExact(names));
- attrs = _PyEval_MatchClass(tstate, subject, type, oparg, names);
- Py_DECREF(subject);
- Py_DECREF(type);
- Py_DECREF(names);
- if (attrs) {
- assert(PyTuple_CheckExact(attrs)); // Success!
+ assert(PyTuple_CheckExact(PyStackRef_AsPyObjectBorrow(names)));
+ PyObject *attrs_o = _PyEval_MatchClass(tstate,
+ PyStackRef_AsPyObjectBorrow(subject),
+ PyStackRef_AsPyObjectBorrow(type), oparg,
+ PyStackRef_AsPyObjectBorrow(names));
+ PyStackRef_CLOSE(subject);
+ PyStackRef_CLOSE(type);
+ PyStackRef_CLOSE(names);
+ if (attrs_o) {
+ assert(PyTuple_CheckExact(attrs_o)); // Success!
+ attrs = PyStackRef_FromPyObjectSteal(attrs_o);
}
else {
if (_PyErr_Occurred(tstate)) JUMP_TO_ERROR();
// Error!
- attrs = Py_None; // Failure!
+ attrs = PyStackRef_None; // Failure!
}
stack_pointer[-3] = attrs;
stack_pointer += -2;
}
case _MATCH_MAPPING: {
- PyObject *subject;
- PyObject *res;
+ _PyStackRef subject;
+ _PyStackRef res;
subject = stack_pointer[-1];
- int match = Py_TYPE(subject)->tp_flags & Py_TPFLAGS_MAPPING;
- res = match ? Py_True : Py_False;
+ int match = PyStackRef_TYPE(subject)->tp_flags & Py_TPFLAGS_MAPPING;
+ res = match ? PyStackRef_True : PyStackRef_False;
stack_pointer[0] = res;
stack_pointer += 1;
assert(WITHIN_STACK_BOUNDS());
}
case _MATCH_SEQUENCE: {
- PyObject *subject;
- PyObject *res;
+ _PyStackRef subject;
+ _PyStackRef res;
subject = stack_pointer[-1];
- int match = Py_TYPE(subject)->tp_flags & Py_TPFLAGS_SEQUENCE;
- res = match ? Py_True : Py_False;
+ int match = PyStackRef_TYPE(subject)->tp_flags & Py_TPFLAGS_SEQUENCE;
+ res = match ? PyStackRef_True : PyStackRef_False;
stack_pointer[0] = res;
stack_pointer += 1;
assert(WITHIN_STACK_BOUNDS());
}
case _MATCH_KEYS: {
- PyObject *keys;
- PyObject *subject;
- PyObject *values_or_none;
+ _PyStackRef keys;
+ _PyStackRef subject;
+ _PyStackRef values_or_none;
keys = stack_pointer[-1];
subject = stack_pointer[-2];
// On successful match, PUSH(values). Otherwise, PUSH(None).
- values_or_none = _PyEval_MatchKeys(tstate, subject, keys);
- if (values_or_none == NULL) JUMP_TO_ERROR();
+ PyObject *values_or_none_o = _PyEval_MatchKeys(tstate,
+ PyStackRef_AsPyObjectBorrow(subject), PyStackRef_AsPyObjectBorrow(keys));
+ if (values_or_none_o == NULL) JUMP_TO_ERROR();
+ values_or_none = PyStackRef_FromPyObjectSteal(values_or_none_o);
stack_pointer[0] = values_or_none;
stack_pointer += 1;
assert(WITHIN_STACK_BOUNDS());
}
case _GET_ITER: {
- PyObject *iterable;
- PyObject *iter;
+ _PyStackRef iterable;
+ _PyStackRef iter;
iterable = stack_pointer[-1];
/* before: [obj]; after [getiter(obj)] */
- iter = PyObject_GetIter(iterable);
- Py_DECREF(iterable);
- if (iter == NULL) JUMP_TO_ERROR();
+ iter = PyStackRef_FromPyObjectSteal(PyObject_GetIter(PyStackRef_AsPyObjectBorrow(iterable)));
+ PyStackRef_CLOSE(iterable);
+ if (PyStackRef_IsNull(iter)) JUMP_TO_ERROR();
stack_pointer[-1] = iter;
break;
}
case _GET_YIELD_FROM_ITER: {
- PyObject *iterable;
- PyObject *iter;
+ _PyStackRef iterable;
+ _PyStackRef iter;
iterable = stack_pointer[-1];
/* before: [obj]; after [getiter(obj)] */
- if (PyCoro_CheckExact(iterable)) {
+ PyObject *iterable_o = PyStackRef_AsPyObjectBorrow(iterable);
+ if (PyCoro_CheckExact(iterable_o)) {
/* `iterable` is a coroutine */
if (!(_PyFrame_GetCode(frame)->co_flags & (CO_COROUTINE | CO_ITERABLE_COROUTINE))) {
/* and it is used in a 'yield from' expression of a
}
iter = iterable;
}
- else if (PyGen_CheckExact(iterable)) {
+ else if (PyGen_CheckExact(iterable_o)) {
iter = iterable;
}
else {
/* `iterable` is not a generator. */
- iter = PyObject_GetIter(iterable);
- if (iter == NULL) {
+ iter = PyStackRef_FromPyObjectSteal(PyObject_GetIter(iterable_o));
+ if (PyStackRef_IsNull(iter)) {
JUMP_TO_ERROR();
}
- Py_DECREF(iterable);
+ PyStackRef_CLOSE(iterable);
}
stack_pointer[-1] = iter;
break;
/* _FOR_ITER is not a viable micro-op for tier 2 because it is replaced */
case _FOR_ITER_TIER_TWO: {
- PyObject *iter;
- PyObject *next;
+ _PyStackRef iter;
+ _PyStackRef next;
iter = stack_pointer[-1];
/* before: [iter]; after: [iter, iter()] *or* [] (and jump over END_FOR.) */
- next = (*Py_TYPE(iter)->tp_iternext)(iter);
- if (next == NULL) {
+ PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter);
+ PyObject *next_o = (*Py_TYPE(iter_o)->tp_iternext)(iter_o);
+ if (next_o == NULL) {
if (_PyErr_Occurred(tstate)) {
if (!_PyErr_ExceptionMatches(tstate, PyExc_StopIteration)) {
JUMP_TO_ERROR();
JUMP_TO_JUMP_TARGET();
}
}
+ next = PyStackRef_FromPyObjectSteal(next_o);
// Common case: no jump, leave it to the code generator
stack_pointer[0] = next;
stack_pointer += 1;
/* _INSTRUMENTED_FOR_ITER is not a viable micro-op for tier 2 because it is instrumented */
case _ITER_CHECK_LIST: {
- PyObject *iter;
+ _PyStackRef iter;
iter = stack_pointer[-1];
- if (Py_TYPE(iter) != &PyListIter_Type) {
+ if (Py_TYPE(PyStackRef_AsPyObjectBorrow(iter)) != &PyListIter_Type) {
UOP_STAT_INC(uopcode, miss);
JUMP_TO_JUMP_TARGET();
}
/* _ITER_JUMP_LIST is not a viable micro-op for tier 2 because it is replaced */
case _GUARD_NOT_EXHAUSTED_LIST: {
- PyObject *iter;
+ _PyStackRef iter;
iter = stack_pointer[-1];
- _PyListIterObject *it = (_PyListIterObject *)iter;
- assert(Py_TYPE(iter) == &PyListIter_Type);
+ PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter);
+ _PyListIterObject *it = (_PyListIterObject *)iter_o;
+ assert(Py_TYPE(iter_o) == &PyListIter_Type);
PyListObject *seq = it->it_seq;
if (seq == NULL) {
UOP_STAT_INC(uopcode, miss);
}
case _ITER_NEXT_LIST: {
- PyObject *iter;
- PyObject *next;
+ _PyStackRef iter;
+ _PyStackRef next;
iter = stack_pointer[-1];
- _PyListIterObject *it = (_PyListIterObject *)iter;
- assert(Py_TYPE(iter) == &PyListIter_Type);
+ PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter);
+ _PyListIterObject *it = (_PyListIterObject *)iter_o;
+ assert(Py_TYPE(iter_o) == &PyListIter_Type);
PyListObject *seq = it->it_seq;
assert(seq);
assert(it->it_index < PyList_GET_SIZE(seq));
- next = Py_NewRef(PyList_GET_ITEM(seq, it->it_index++));
+ next = PyStackRef_FromPyObjectNew(PyList_GET_ITEM(seq, it->it_index++));
stack_pointer[0] = next;
stack_pointer += 1;
assert(WITHIN_STACK_BOUNDS());
}
case _ITER_CHECK_TUPLE: {
- PyObject *iter;
+ _PyStackRef iter;
iter = stack_pointer[-1];
- if (Py_TYPE(iter) != &PyTupleIter_Type) {
+ if (Py_TYPE(PyStackRef_AsPyObjectBorrow(iter)) != &PyTupleIter_Type) {
UOP_STAT_INC(uopcode, miss);
JUMP_TO_JUMP_TARGET();
}
/* _ITER_JUMP_TUPLE is not a viable micro-op for tier 2 because it is replaced */
case _GUARD_NOT_EXHAUSTED_TUPLE: {
- PyObject *iter;
+ _PyStackRef iter;
iter = stack_pointer[-1];
- _PyTupleIterObject *it = (_PyTupleIterObject *)iter;
- assert(Py_TYPE(iter) == &PyTupleIter_Type);
+ PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter);
+ _PyTupleIterObject *it = (_PyTupleIterObject *)iter_o;
+ assert(Py_TYPE(iter_o) == &PyTupleIter_Type);
PyTupleObject *seq = it->it_seq;
if (seq == NULL) {
UOP_STAT_INC(uopcode, miss);
}
case _ITER_NEXT_TUPLE: {
- PyObject *iter;
- PyObject *next;
+ _PyStackRef iter;
+ _PyStackRef next;
iter = stack_pointer[-1];
- _PyTupleIterObject *it = (_PyTupleIterObject *)iter;
- assert(Py_TYPE(iter) == &PyTupleIter_Type);
+ PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter);
+ _PyTupleIterObject *it = (_PyTupleIterObject *)iter_o;
+ assert(Py_TYPE(iter_o) == &PyTupleIter_Type);
PyTupleObject *seq = it->it_seq;
assert(seq);
assert(it->it_index < PyTuple_GET_SIZE(seq));
- next = Py_NewRef(PyTuple_GET_ITEM(seq, it->it_index++));
+ next = PyStackRef_FromPyObjectNew(PyTuple_GET_ITEM(seq, it->it_index++));
stack_pointer[0] = next;
stack_pointer += 1;
assert(WITHIN_STACK_BOUNDS());
}
case _ITER_CHECK_RANGE: {
- PyObject *iter;
+ _PyStackRef iter;
iter = stack_pointer[-1];
- _PyRangeIterObject *r = (_PyRangeIterObject *)iter;
+ _PyRangeIterObject *r = (_PyRangeIterObject *)PyStackRef_AsPyObjectBorrow(iter);
if (Py_TYPE(r) != &PyRangeIter_Type) {
UOP_STAT_INC(uopcode, miss);
JUMP_TO_JUMP_TARGET();
/* _ITER_JUMP_RANGE is not a viable micro-op for tier 2 because it is replaced */
case _GUARD_NOT_EXHAUSTED_RANGE: {
- PyObject *iter;
+ _PyStackRef iter;
iter = stack_pointer[-1];
- _PyRangeIterObject *r = (_PyRangeIterObject *)iter;
+ _PyRangeIterObject *r = (_PyRangeIterObject *)PyStackRef_AsPyObjectBorrow(iter);
assert(Py_TYPE(r) == &PyRangeIter_Type);
if (r->len <= 0) {
UOP_STAT_INC(uopcode, miss);
}
case _ITER_NEXT_RANGE: {
- PyObject *iter;
- PyObject *next;
+ _PyStackRef iter;
+ _PyStackRef next;
iter = stack_pointer[-1];
- _PyRangeIterObject *r = (_PyRangeIterObject *)iter;
+ _PyRangeIterObject *r = (_PyRangeIterObject *)PyStackRef_AsPyObjectBorrow(iter);
assert(Py_TYPE(r) == &PyRangeIter_Type);
assert(r->len > 0);
long value = r->start;
r->start = value + r->step;
r->len--;
- next = PyLong_FromLong(value);
- if (next == NULL) JUMP_TO_ERROR();
+ PyObject *res = PyLong_FromLong(value);
+ if (res == NULL) JUMP_TO_ERROR();
+ next = PyStackRef_FromPyObjectSteal(res);
stack_pointer[0] = next;
stack_pointer += 1;
assert(WITHIN_STACK_BOUNDS());
}
case _FOR_ITER_GEN_FRAME: {
- PyObject *iter;
+ _PyStackRef iter;
_PyInterpreterFrame *gen_frame;
oparg = CURRENT_OPARG();
iter = stack_pointer[-1];
- PyGenObject *gen = (PyGenObject *)iter;
+ PyGenObject *gen = (PyGenObject *)PyStackRef_AsPyObjectBorrow(iter);
if (Py_TYPE(gen) != &PyGen_Type) {
UOP_STAT_INC(uopcode, miss);
JUMP_TO_JUMP_TARGET();
}
STAT_INC(FOR_ITER, hit);
gen_frame = &gen->gi_iframe;
- _PyFrame_StackPush(gen_frame, Py_None);
+ _PyFrame_StackPush(gen_frame, PyStackRef_None);
gen->gi_frame_state = FRAME_EXECUTING;
gen->gi_exc_state.previous_item = tstate->exc_info;
tstate->exc_info = &gen->gi_exc_state;
// oparg is the return offset from the next instruction.
frame->return_offset = (uint16_t)(1 + INLINE_CACHE_ENTRIES_FOR_ITER + oparg);
- stack_pointer[0] = (PyObject *)gen_frame;
+ stack_pointer[0].bits = (uintptr_t)gen_frame;
stack_pointer += 1;
assert(WITHIN_STACK_BOUNDS());
break;
}
case _LOAD_SPECIAL: {
- PyObject *owner;
- PyObject *attr;
- PyObject *self_or_null;
+ _PyStackRef owner;
+ _PyStackRef attr;
+ _PyStackRef self_or_null;
oparg = CURRENT_OPARG();
owner = stack_pointer[-1];
assert(oparg <= SPECIAL_MAX);
+ PyObject *owner_o = PyStackRef_AsPyObjectSteal(owner);
PyObject *name = _Py_SpecialMethods[oparg].name;
- attr = _PyObject_LookupSpecialMethod(owner, name, &self_or_null);
- if (attr == NULL) {
+ PyObject *self_or_null_o;
+ attr = PyStackRef_FromPyObjectSteal(_PyObject_LookupSpecialMethod(owner_o, name, &self_or_null_o));
+ if (PyStackRef_IsNull(attr)) {
if (!_PyErr_Occurred(tstate)) {
_PyErr_Format(tstate, PyExc_TypeError,
_Py_SpecialMethods[oparg].error,
- Py_TYPE(owner)->tp_name);
+ Py_TYPE(owner_o)->tp_name);
}
}
- if (attr == NULL) JUMP_TO_ERROR();
+ if (PyStackRef_IsNull(attr)) JUMP_TO_ERROR();
+ self_or_null = PyStackRef_FromPyObjectSteal(self_or_null_o);
stack_pointer[-1] = attr;
stack_pointer[0] = self_or_null;
stack_pointer += 1;
}
case _WITH_EXCEPT_START: {
- PyObject *val;
- PyObject *lasti;
- PyObject *exit_self;
- PyObject *exit_func;
- PyObject *res;
+ _PyStackRef val;
+ _PyStackRef lasti;
+ _PyStackRef exit_self;
+ _PyStackRef exit_func;
+ _PyStackRef res;
val = stack_pointer[-1];
lasti = stack_pointer[-3];
exit_self = stack_pointer[-4];
Then we push the __exit__ return value.
*/
PyObject *exc, *tb;
- assert(val && PyExceptionInstance_Check(val));
- exc = PyExceptionInstance_Class(val);
- tb = PyException_GetTraceback(val);
+ PyObject *val_o = PyStackRef_AsPyObjectBorrow(val);
+ PyObject *exit_func_o = PyStackRef_AsPyObjectBorrow(exit_func);
+ assert(val_o && PyExceptionInstance_Check(val_o));
+ exc = PyExceptionInstance_Class(val_o);
+ tb = PyException_GetTraceback(val_o);
if (tb == NULL) {
tb = Py_None;
}
else {
Py_DECREF(tb);
}
- assert(PyLong_Check(lasti));
+ assert(PyLong_Check(PyStackRef_AsPyObjectBorrow(lasti)));
(void)lasti; // Shut up compiler warning if asserts are off
- PyObject *stack[5] = {NULL, exit_self, exc, val, tb};
- int has_self = (exit_self != NULL);
- res = PyObject_Vectorcall(exit_func, stack + 2 - has_self,
- (3 + has_self) | PY_VECTORCALL_ARGUMENTS_OFFSET, NULL);
- if (res == NULL) JUMP_TO_ERROR();
+ PyObject *stack[5] = {NULL, PyStackRef_AsPyObjectBorrow(exit_self), exc, val_o, tb};
+ int has_self = !PyStackRef_IsNull(exit_self);
+ res = PyStackRef_FromPyObjectSteal(PyObject_Vectorcall(exit_func_o, stack + 2 - has_self,
+ (3 + has_self) | PY_VECTORCALL_ARGUMENTS_OFFSET, NULL));
+ if (PyStackRef_IsNull(res)) JUMP_TO_ERROR();
stack_pointer[0] = res;
stack_pointer += 1;
assert(WITHIN_STACK_BOUNDS());
}
case _PUSH_EXC_INFO: {
- PyObject *new_exc;
- PyObject *prev_exc;
+ _PyStackRef new_exc;
+ _PyStackRef prev_exc;
new_exc = stack_pointer[-1];
_PyErr_StackItem *exc_info = tstate->exc_info;
if (exc_info->exc_value != NULL) {
- prev_exc = exc_info->exc_value;
+ prev_exc = PyStackRef_FromPyObjectSteal(exc_info->exc_value);
}
else {
- prev_exc = Py_None;
+ prev_exc = PyStackRef_None;
}
- assert(PyExceptionInstance_Check(new_exc));
- exc_info->exc_value = Py_NewRef(new_exc);
+ assert(PyExceptionInstance_Check(PyStackRef_AsPyObjectBorrow(new_exc)));
+ exc_info->exc_value = PyStackRef_AsPyObjectNew(new_exc);
stack_pointer[-1] = prev_exc;
stack_pointer[0] = new_exc;
stack_pointer += 1;
}
case _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT: {
- PyObject *owner;
+ _PyStackRef owner;
owner = stack_pointer[-1];
- assert(Py_TYPE(owner)->tp_flags & Py_TPFLAGS_INLINE_VALUES);
- if (!_PyObject_InlineValues(owner)->valid) {
+ PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner);
+ assert(Py_TYPE(owner_o)->tp_flags & Py_TPFLAGS_INLINE_VALUES);
+ if (!_PyObject_InlineValues(owner_o)->valid) {
UOP_STAT_INC(uopcode, miss);
JUMP_TO_JUMP_TARGET();
}
}
case _GUARD_KEYS_VERSION: {
- PyObject *owner;
+ _PyStackRef owner;
owner = stack_pointer[-1];
uint32_t keys_version = (uint32_t)CURRENT_OPERAND();
- PyTypeObject *owner_cls = Py_TYPE(owner);
+ PyTypeObject *owner_cls = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner));
PyHeapTypeObject *owner_heap_type = (PyHeapTypeObject *)owner_cls;
if (owner_heap_type->ht_cached_keys->dk_version != keys_version) {
UOP_STAT_INC(uopcode, miss);
}
case _LOAD_ATTR_METHOD_WITH_VALUES: {
- PyObject *owner;
- PyObject *attr;
- PyObject *self = NULL;
+ _PyStackRef owner;
+ _PyStackRef attr;
+ _PyStackRef self = PyStackRef_NULL;
oparg = CURRENT_OPARG();
owner = stack_pointer[-1];
PyObject *descr = (PyObject *)CURRENT_OPERAND();
/* Cached method object */
STAT_INC(LOAD_ATTR, hit);
assert(descr != NULL);
- attr = Py_NewRef(descr);
- assert(_PyType_HasFeature(Py_TYPE(attr), Py_TPFLAGS_METHOD_DESCRIPTOR));
+ assert(_PyType_HasFeature(Py_TYPE(descr), Py_TPFLAGS_METHOD_DESCRIPTOR));
+ attr = PyStackRef_FromPyObjectNew(descr);
self = owner;
stack_pointer[-1] = attr;
stack_pointer[0] = self;
}
case _LOAD_ATTR_METHOD_NO_DICT: {
- PyObject *owner;
- PyObject *attr;
- PyObject *self = NULL;
+ _PyStackRef owner;
+ _PyStackRef attr;
+ _PyStackRef self = PyStackRef_NULL;
oparg = CURRENT_OPARG();
owner = stack_pointer[-1];
PyObject *descr = (PyObject *)CURRENT_OPERAND();
assert(oparg & 1);
- assert(Py_TYPE(owner)->tp_dictoffset == 0);
+ assert(Py_TYPE(PyStackRef_AsPyObjectBorrow(owner))->tp_dictoffset == 0);
STAT_INC(LOAD_ATTR, hit);
assert(descr != NULL);
assert(_PyType_HasFeature(Py_TYPE(descr), Py_TPFLAGS_METHOD_DESCRIPTOR));
- attr = Py_NewRef(descr);
+ attr = PyStackRef_FromPyObjectNew(descr);
self = owner;
stack_pointer[-1] = attr;
stack_pointer[0] = self;
}
case _LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES: {
- PyObject *owner;
- PyObject *attr;
+ _PyStackRef owner;
+ _PyStackRef attr;
oparg = CURRENT_OPARG();
owner = stack_pointer[-1];
PyObject *descr = (PyObject *)CURRENT_OPERAND();
assert((oparg & 1) == 0);
STAT_INC(LOAD_ATTR, hit);
assert(descr != NULL);
- Py_DECREF(owner);
- attr = Py_NewRef(descr);
+ PyStackRef_CLOSE(owner);
+ attr = PyStackRef_FromPyObjectNew(descr);
stack_pointer[-1] = attr;
break;
}
case _LOAD_ATTR_NONDESCRIPTOR_NO_DICT: {
- PyObject *owner;
- PyObject *attr;
+ _PyStackRef owner;
+ _PyStackRef attr;
oparg = CURRENT_OPARG();
owner = stack_pointer[-1];
PyObject *descr = (PyObject *)CURRENT_OPERAND();
assert((oparg & 1) == 0);
- assert(Py_TYPE(owner)->tp_dictoffset == 0);
+ assert(Py_TYPE(PyStackRef_AsPyObjectBorrow(owner))->tp_dictoffset == 0);
STAT_INC(LOAD_ATTR, hit);
assert(descr != NULL);
- Py_DECREF(owner);
- attr = Py_NewRef(descr);
+ PyStackRef_CLOSE(owner);
+ attr = PyStackRef_FromPyObjectNew(descr);
stack_pointer[-1] = attr;
break;
}
case _CHECK_ATTR_METHOD_LAZY_DICT: {
- PyObject *owner;
+ _PyStackRef owner;
owner = stack_pointer[-1];
uint16_t dictoffset = (uint16_t)CURRENT_OPERAND();
- char *ptr = ((char *)owner) + MANAGED_DICT_OFFSET + dictoffset;
+ char *ptr = ((char *)PyStackRef_AsPyObjectBorrow(owner)) + MANAGED_DICT_OFFSET + dictoffset;
PyObject *dict = *(PyObject **)ptr;
/* This object has a __dict__, just not yet created */
if (dict != NULL) {
}
case _LOAD_ATTR_METHOD_LAZY_DICT: {
- PyObject *owner;
- PyObject *attr;
- PyObject *self = NULL;
+ _PyStackRef owner;
+ _PyStackRef attr;
+ _PyStackRef self = PyStackRef_NULL;
oparg = CURRENT_OPARG();
owner = stack_pointer[-1];
PyObject *descr = (PyObject *)CURRENT_OPERAND();
STAT_INC(LOAD_ATTR, hit);
assert(descr != NULL);
assert(_PyType_HasFeature(Py_TYPE(descr), Py_TPFLAGS_METHOD_DESCRIPTOR));
- attr = Py_NewRef(descr);
+ attr = PyStackRef_FromPyObjectNew(descr);
self = owner;
stack_pointer[-1] = attr;
stack_pointer[0] = self;
}
case _PY_FRAME_GENERAL: {
- PyObject **args;
- PyObject *self_or_null;
- PyObject *callable;
+ _PyStackRef *args;
+ _PyStackRef self_or_null;
+ _PyStackRef callable;
_PyInterpreterFrame *new_frame;
oparg = CURRENT_OPARG();
args = &stack_pointer[-oparg];
self_or_null = stack_pointer[-1 - oparg];
callable = stack_pointer[-2 - oparg];
+ PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable);
+ PyObject *self_or_null_o = PyStackRef_AsPyObjectBorrow(self_or_null);
// oparg counts all of the args, but *not* self:
int total_args = oparg;
- if (self_or_null != NULL) {
+ if (self_or_null_o != NULL) {
args--;
total_args++;
}
- assert(Py_TYPE(callable) == &PyFunction_Type);
- int code_flags = ((PyCodeObject*)PyFunction_GET_CODE(callable))->co_flags;
- PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable));
+ assert(Py_TYPE(callable_o) == &PyFunction_Type);
+ int code_flags = ((PyCodeObject*)PyFunction_GET_CODE(callable_o))->co_flags;
+ PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable_o));
new_frame = _PyEvalFramePushAndInit(
- tstate, (PyFunctionObject *)callable, locals,
+ tstate, (PyFunctionObject *)PyStackRef_AsPyObjectSteal(callable), locals,
args, total_args, NULL
);
// The frame has stolen all the arguments from the stack,
if (new_frame == NULL) {
JUMP_TO_ERROR();
}
- stack_pointer[0] = (PyObject *)new_frame;
+ stack_pointer[0].bits = (uintptr_t)new_frame;
stack_pointer += 1;
assert(WITHIN_STACK_BOUNDS());
break;
}
case _CHECK_FUNCTION_VERSION: {
- PyObject *callable;
+ _PyStackRef callable;
oparg = CURRENT_OPARG();
callable = stack_pointer[-2 - oparg];
uint32_t func_version = (uint32_t)CURRENT_OPERAND();
- if (!PyFunction_Check(callable)) {
+ PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable);
+ if (!PyFunction_Check(callable_o)) {
UOP_STAT_INC(uopcode, miss);
JUMP_TO_JUMP_TARGET();
}
- PyFunctionObject *func = (PyFunctionObject *)callable;
+ PyFunctionObject *func = (PyFunctionObject *)callable_o;
if (func->func_version != func_version) {
UOP_STAT_INC(uopcode, miss);
JUMP_TO_JUMP_TARGET();
}
case _CHECK_METHOD_VERSION: {
- PyObject *null;
- PyObject *callable;
+ _PyStackRef null;
+ _PyStackRef callable;
oparg = CURRENT_OPARG();
null = stack_pointer[-1 - oparg];
callable = stack_pointer[-2 - oparg];
uint32_t func_version = (uint32_t)CURRENT_OPERAND();
- if (Py_TYPE(callable) != &PyMethod_Type) {
+ PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable);
+ if (Py_TYPE(callable_o) != &PyMethod_Type) {
UOP_STAT_INC(uopcode, miss);
JUMP_TO_JUMP_TARGET();
}
- PyObject *func = ((PyMethodObject *)callable)->im_func;
+ PyObject *func = ((PyMethodObject *)callable_o)->im_func;
if (!PyFunction_Check(func)) {
UOP_STAT_INC(uopcode, miss);
JUMP_TO_JUMP_TARGET();
UOP_STAT_INC(uopcode, miss);
JUMP_TO_JUMP_TARGET();
}
- if (null != NULL) {
+ if (!PyStackRef_IsNull(null)) {
UOP_STAT_INC(uopcode, miss);
JUMP_TO_JUMP_TARGET();
}
}
case _EXPAND_METHOD: {
- PyObject *null;
- PyObject *callable;
- PyObject *method;
- PyObject *self;
+ _PyStackRef null;
+ _PyStackRef callable;
+ _PyStackRef method;
+ _PyStackRef self;
oparg = CURRENT_OPARG();
null = stack_pointer[-1 - oparg];
callable = stack_pointer[-2 - oparg];
- assert(null == NULL);
- assert(Py_TYPE(callable) == &PyMethod_Type);
- self = ((PyMethodObject *)callable)->im_self;
- Py_INCREF(self);
+ PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable);
+ assert(PyStackRef_IsNull(null));
+ assert(Py_TYPE(callable_o) == &PyMethod_Type);
+ self = PyStackRef_FromPyObjectNew(((PyMethodObject *)callable_o)->im_self);
stack_pointer[-1 - oparg] = self; // Patch stack as it is used by _PY_FRAME_GENERAL
- method = ((PyMethodObject *)callable)->im_func;
- assert(PyFunction_Check(method));
- Py_INCREF(method);
- Py_DECREF(callable);
+ method = PyStackRef_FromPyObjectNew(((PyMethodObject *)callable_o)->im_func);
+ assert(PyFunction_Check(PyStackRef_AsPyObjectBorrow(method)));
+ PyStackRef_CLOSE(callable);
stack_pointer[-2 - oparg] = method;
stack_pointer[-1 - oparg] = self;
break;
}
case _CHECK_IS_NOT_PY_CALLABLE: {
- PyObject *callable;
+ _PyStackRef callable;
oparg = CURRENT_OPARG();
callable = stack_pointer[-2 - oparg];
- if (PyFunction_Check(callable)) {
+ PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable);
+ if (PyFunction_Check(callable_o)) {
UOP_STAT_INC(uopcode, miss);
JUMP_TO_JUMP_TARGET();
}
- if (Py_TYPE(callable) == &PyMethod_Type) {
+ if (Py_TYPE(callable_o) == &PyMethod_Type) {
UOP_STAT_INC(uopcode, miss);
JUMP_TO_JUMP_TARGET();
}
}
case _CALL_NON_PY_GENERAL: {
- PyObject **args;
- PyObject *self_or_null;
- PyObject *callable;
- PyObject *res;
+ _PyStackRef *args;
+ _PyStackRef self_or_null;
+ _PyStackRef callable;
+ _PyStackRef res;
oparg = CURRENT_OPARG();
args = &stack_pointer[-oparg];
self_or_null = stack_pointer[-1 - oparg];
#if TIER_ONE
assert(opcode != INSTRUMENTED_CALL);
#endif
+ PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable);
+ PyObject *self_or_null_o = PyStackRef_AsPyObjectBorrow(self_or_null);
int total_args = oparg;
- if (self_or_null != NULL) {
+ if (self_or_null_o != NULL) {
args--;
total_args++;
}
/* Callable is not a normal Python function */
- res = PyObject_Vectorcall(
- callable, args,
- total_args | PY_VECTORCALL_ARGUMENTS_OFFSET,
- NULL);
- assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL));
- Py_DECREF(callable);
+ STACKREFS_TO_PYOBJECTS(args, total_args, args_o);
+ if (CONVERSION_FAILED(args_o)) {
+ PyStackRef_CLOSE(callable);
+ PyStackRef_CLOSE(self_or_null);
+ for (int _i = oparg; --_i >= 0;) {
+ PyStackRef_CLOSE(args[_i]);
+ }
+ if (true) JUMP_TO_ERROR();
+ }
+ PyObject *res_o = PyObject_Vectorcall(
+ callable_o, args_o,
+ total_args | PY_VECTORCALL_ARGUMENTS_OFFSET,
+ NULL);
+ STACKREFS_TO_PYOBJECTS_CLEANUP(args_o);
+ assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL));
+ PyStackRef_CLOSE(callable);
for (int i = 0; i < total_args; i++) {
- Py_DECREF(args[i]);
+ PyStackRef_CLOSE(args[i]);
}
- if (res == NULL) JUMP_TO_ERROR();
+ if (res_o == NULL) JUMP_TO_ERROR();
+ res = PyStackRef_FromPyObjectSteal(res_o);
stack_pointer[-2 - oparg] = res;
stack_pointer += -1 - oparg;
assert(WITHIN_STACK_BOUNDS());
}
case _CHECK_CALL_BOUND_METHOD_EXACT_ARGS: {
- PyObject *null;
- PyObject *callable;
+ _PyStackRef null;
+ _PyStackRef callable;
oparg = CURRENT_OPARG();
null = stack_pointer[-1 - oparg];
callable = stack_pointer[-2 - oparg];
- if (null != NULL) {
+ if (!PyStackRef_IsNull(null)) {
UOP_STAT_INC(uopcode, miss);
JUMP_TO_JUMP_TARGET();
}
- if (Py_TYPE(callable) != &PyMethod_Type) {
+ if (Py_TYPE(PyStackRef_AsPyObjectBorrow(callable)) != &PyMethod_Type) {
UOP_STAT_INC(uopcode, miss);
JUMP_TO_JUMP_TARGET();
}
}
case _INIT_CALL_BOUND_METHOD_EXACT_ARGS: {
- PyObject *callable;
- PyObject *func;
- PyObject *self;
+ _PyStackRef callable;
+ _PyStackRef func;
+ _PyStackRef self;
oparg = CURRENT_OPARG();
callable = stack_pointer[-2 - oparg];
+ PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable);
STAT_INC(CALL, hit);
- self = Py_NewRef(((PyMethodObject *)callable)->im_self);
- stack_pointer[-1 - oparg] = self; // Patch stack as it is used by _INIT_CALL_PY_EXACT_ARGS
- func = Py_NewRef(((PyMethodObject *)callable)->im_func);
- stack_pointer[-2 - oparg] = func; // This is used by CALL, upon deoptimization
- Py_DECREF(callable);
+ stack_pointer[-1 - oparg] = PyStackRef_FromPyObjectNew(((PyMethodObject *)callable_o)->im_self); // Patch stack as it is used by _INIT_CALL_PY_EXACT_ARGS
+ stack_pointer[-2 - oparg] = PyStackRef_FromPyObjectNew(((PyMethodObject *)callable_o)->im_func); // This is used by CALL, upon deoptimization
+ self = stack_pointer[-1 - oparg];
+ func = stack_pointer[-2 - oparg];
+ PyStackRef_CLOSE(callable);
+ // self may be unused in tier 1, so silence warnings.
+ (void)self;
stack_pointer[-2 - oparg] = func;
stack_pointer[-1 - oparg] = self;
break;
}
case _CHECK_FUNCTION_EXACT_ARGS: {
- PyObject *self_or_null;
- PyObject *callable;
+ _PyStackRef self_or_null;
+ _PyStackRef callable;
oparg = CURRENT_OPARG();
self_or_null = stack_pointer[-1 - oparg];
callable = stack_pointer[-2 - oparg];
- assert(PyFunction_Check(callable));
- PyFunctionObject *func = (PyFunctionObject *)callable;
+ PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable);
+ assert(PyFunction_Check(callable_o));
+ PyFunctionObject *func = (PyFunctionObject *)callable_o;
PyCodeObject *code = (PyCodeObject *)func->func_code;
- if (code->co_argcount != oparg + (self_or_null != NULL)) {
+ if (code->co_argcount != oparg + (!PyStackRef_IsNull(self_or_null))) {
UOP_STAT_INC(uopcode, miss);
JUMP_TO_JUMP_TARGET();
}
}
case _CHECK_STACK_SPACE: {
- PyObject *callable;
+ _PyStackRef callable;
oparg = CURRENT_OPARG();
callable = stack_pointer[-2 - oparg];
- PyFunctionObject *func = (PyFunctionObject *)callable;
+ PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable);
+ PyFunctionObject *func = (PyFunctionObject *)callable_o;
PyCodeObject *code = (PyCodeObject *)func->func_code;
if (!_PyThreadState_HasStackSpace(tstate, code->co_framesize)) {
UOP_STAT_INC(uopcode, miss);
}
case _INIT_CALL_PY_EXACT_ARGS_0: {
- PyObject **args;
- PyObject *self_or_null;
- PyObject *callable;
+ _PyStackRef *args;
+ _PyStackRef self_or_null;
+ _PyStackRef callable;
_PyInterpreterFrame *new_frame;
oparg = 0;
assert(oparg == CURRENT_OPARG());
args = &stack_pointer[-oparg];
self_or_null = stack_pointer[-1 - oparg];
callable = stack_pointer[-2 - oparg];
- int has_self = (self_or_null != NULL);
+ PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable);
+ int has_self = !PyStackRef_IsNull(self_or_null);
STAT_INC(CALL, hit);
- PyFunctionObject *func = (PyFunctionObject *)callable;
+ PyFunctionObject *func = (PyFunctionObject *)callable_o;
new_frame = _PyFrame_PushUnchecked(tstate, func, oparg + has_self);
- PyObject **first_non_self_local = new_frame->localsplus + has_self;
+ _PyStackRef *first_non_self_local = new_frame->localsplus + has_self;
new_frame->localsplus[0] = self_or_null;
for (int i = 0; i < oparg; i++) {
first_non_self_local[i] = args[i];
}
- stack_pointer[-2 - oparg] = (PyObject *)new_frame;
+ stack_pointer[-2 - oparg].bits = (uintptr_t)new_frame;
stack_pointer += -1 - oparg;
assert(WITHIN_STACK_BOUNDS());
break;
}
case _INIT_CALL_PY_EXACT_ARGS_1: {
- PyObject **args;
- PyObject *self_or_null;
- PyObject *callable;
+ _PyStackRef *args;
+ _PyStackRef self_or_null;
+ _PyStackRef callable;
_PyInterpreterFrame *new_frame;
oparg = 1;
assert(oparg == CURRENT_OPARG());
args = &stack_pointer[-oparg];
self_or_null = stack_pointer[-1 - oparg];
callable = stack_pointer[-2 - oparg];
- int has_self = (self_or_null != NULL);
+ PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable);
+ int has_self = !PyStackRef_IsNull(self_or_null);
STAT_INC(CALL, hit);
- PyFunctionObject *func = (PyFunctionObject *)callable;
+ PyFunctionObject *func = (PyFunctionObject *)callable_o;
new_frame = _PyFrame_PushUnchecked(tstate, func, oparg + has_self);
- PyObject **first_non_self_local = new_frame->localsplus + has_self;
+ _PyStackRef *first_non_self_local = new_frame->localsplus + has_self;
new_frame->localsplus[0] = self_or_null;
for (int i = 0; i < oparg; i++) {
first_non_self_local[i] = args[i];
}
- stack_pointer[-2 - oparg] = (PyObject *)new_frame;
+ stack_pointer[-2 - oparg].bits = (uintptr_t)new_frame;
stack_pointer += -1 - oparg;
assert(WITHIN_STACK_BOUNDS());
break;
}
case _INIT_CALL_PY_EXACT_ARGS_2: {
- PyObject **args;
- PyObject *self_or_null;
- PyObject *callable;
+ _PyStackRef *args;
+ _PyStackRef self_or_null;
+ _PyStackRef callable;
_PyInterpreterFrame *new_frame;
oparg = 2;
assert(oparg == CURRENT_OPARG());
args = &stack_pointer[-oparg];
self_or_null = stack_pointer[-1 - oparg];
callable = stack_pointer[-2 - oparg];
- int has_self = (self_or_null != NULL);
+ PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable);
+ int has_self = !PyStackRef_IsNull(self_or_null);
STAT_INC(CALL, hit);
- PyFunctionObject *func = (PyFunctionObject *)callable;
+ PyFunctionObject *func = (PyFunctionObject *)callable_o;
new_frame = _PyFrame_PushUnchecked(tstate, func, oparg + has_self);
- PyObject **first_non_self_local = new_frame->localsplus + has_self;
+ _PyStackRef *first_non_self_local = new_frame->localsplus + has_self;
new_frame->localsplus[0] = self_or_null;
for (int i = 0; i < oparg; i++) {
first_non_self_local[i] = args[i];
}
- stack_pointer[-2 - oparg] = (PyObject *)new_frame;
+ stack_pointer[-2 - oparg].bits = (uintptr_t)new_frame;
stack_pointer += -1 - oparg;
assert(WITHIN_STACK_BOUNDS());
break;
}
case _INIT_CALL_PY_EXACT_ARGS_3: {
- PyObject **args;
- PyObject *self_or_null;
- PyObject *callable;
+ _PyStackRef *args;
+ _PyStackRef self_or_null;
+ _PyStackRef callable;
_PyInterpreterFrame *new_frame;
oparg = 3;
assert(oparg == CURRENT_OPARG());
args = &stack_pointer[-oparg];
self_or_null = stack_pointer[-1 - oparg];
callable = stack_pointer[-2 - oparg];
- int has_self = (self_or_null != NULL);
+ PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable);
+ int has_self = !PyStackRef_IsNull(self_or_null);
STAT_INC(CALL, hit);
- PyFunctionObject *func = (PyFunctionObject *)callable;
+ PyFunctionObject *func = (PyFunctionObject *)callable_o;
new_frame = _PyFrame_PushUnchecked(tstate, func, oparg + has_self);
- PyObject **first_non_self_local = new_frame->localsplus + has_self;
+ _PyStackRef *first_non_self_local = new_frame->localsplus + has_self;
new_frame->localsplus[0] = self_or_null;
for (int i = 0; i < oparg; i++) {
first_non_self_local[i] = args[i];
}
- stack_pointer[-2 - oparg] = (PyObject *)new_frame;
+ stack_pointer[-2 - oparg].bits = (uintptr_t)new_frame;
stack_pointer += -1 - oparg;
assert(WITHIN_STACK_BOUNDS());
break;
}
case _INIT_CALL_PY_EXACT_ARGS_4: {
- PyObject **args;
- PyObject *self_or_null;
- PyObject *callable;
+ _PyStackRef *args;
+ _PyStackRef self_or_null;
+ _PyStackRef callable;
_PyInterpreterFrame *new_frame;
oparg = 4;
assert(oparg == CURRENT_OPARG());
args = &stack_pointer[-oparg];
self_or_null = stack_pointer[-1 - oparg];
callable = stack_pointer[-2 - oparg];
- int has_self = (self_or_null != NULL);
+ PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable);
+ int has_self = !PyStackRef_IsNull(self_or_null);
STAT_INC(CALL, hit);
- PyFunctionObject *func = (PyFunctionObject *)callable;
+ PyFunctionObject *func = (PyFunctionObject *)callable_o;
new_frame = _PyFrame_PushUnchecked(tstate, func, oparg + has_self);
- PyObject **first_non_self_local = new_frame->localsplus + has_self;
+ _PyStackRef *first_non_self_local = new_frame->localsplus + has_self;
new_frame->localsplus[0] = self_or_null;
for (int i = 0; i < oparg; i++) {
first_non_self_local[i] = args[i];
}
- stack_pointer[-2 - oparg] = (PyObject *)new_frame;
+ stack_pointer[-2 - oparg].bits = (uintptr_t)new_frame;
stack_pointer += -1 - oparg;
assert(WITHIN_STACK_BOUNDS());
break;
}
case _INIT_CALL_PY_EXACT_ARGS: {
- PyObject **args;
- PyObject *self_or_null;
- PyObject *callable;
+ _PyStackRef *args;
+ _PyStackRef self_or_null;
+ _PyStackRef callable;
_PyInterpreterFrame *new_frame;
oparg = CURRENT_OPARG();
args = &stack_pointer[-oparg];
self_or_null = stack_pointer[-1 - oparg];
callable = stack_pointer[-2 - oparg];
- int has_self = (self_or_null != NULL);
+ PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable);
+ int has_self = !PyStackRef_IsNull(self_or_null);
STAT_INC(CALL, hit);
- PyFunctionObject *func = (PyFunctionObject *)callable;
+ PyFunctionObject *func = (PyFunctionObject *)callable_o;
new_frame = _PyFrame_PushUnchecked(tstate, func, oparg + has_self);
- PyObject **first_non_self_local = new_frame->localsplus + has_self;
+ _PyStackRef *first_non_self_local = new_frame->localsplus + has_self;
new_frame->localsplus[0] = self_or_null;
for (int i = 0; i < oparg; i++) {
first_non_self_local[i] = args[i];
}
- stack_pointer[-2 - oparg] = (PyObject *)new_frame;
+ stack_pointer[-2 - oparg].bits = (uintptr_t)new_frame;
stack_pointer += -1 - oparg;
assert(WITHIN_STACK_BOUNDS());
break;
case _PUSH_FRAME: {
_PyInterpreterFrame *new_frame;
- new_frame = (_PyInterpreterFrame *)stack_pointer[-1];
+ new_frame = (_PyInterpreterFrame *)stack_pointer[-1].bits;
// Write it out explicitly because it's subtly different.
// Eventually this should be the only occurrence of this code.
assert(tstate->interp->eval_frame == NULL);
}
case _CALL_TYPE_1: {
- PyObject *arg;
- PyObject *null;
- PyObject *callable;
- PyObject *res;
+ _PyStackRef arg;
+ _PyStackRef null;
+ _PyStackRef callable;
+ _PyStackRef res;
oparg = CURRENT_OPARG();
arg = stack_pointer[-1];
null = stack_pointer[-2];
callable = stack_pointer[-3];
+ PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable);
+ PyObject *arg_o = PyStackRef_AsPyObjectBorrow(arg);
assert(oparg == 1);
- if (null != NULL) {
+ if (!PyStackRef_IsNull(null)) {
UOP_STAT_INC(uopcode, miss);
JUMP_TO_JUMP_TARGET();
}
- if (callable != (PyObject *)&PyType_Type) {
+ if (callable_o != (PyObject *)&PyType_Type) {
UOP_STAT_INC(uopcode, miss);
JUMP_TO_JUMP_TARGET();
}
STAT_INC(CALL, hit);
- res = Py_NewRef(Py_TYPE(arg));
- Py_DECREF(arg);
+ res = PyStackRef_FromPyObjectSteal(Py_NewRef(Py_TYPE(arg_o)));
+ PyStackRef_CLOSE(arg);
stack_pointer[-3] = res;
stack_pointer += -2;
assert(WITHIN_STACK_BOUNDS());
}
case _CALL_STR_1: {
- PyObject *arg;
- PyObject *null;
- PyObject *callable;
- PyObject *res;
+ _PyStackRef arg;
+ _PyStackRef null;
+ _PyStackRef callable;
+ _PyStackRef res;
oparg = CURRENT_OPARG();
arg = stack_pointer[-1];
null = stack_pointer[-2];
callable = stack_pointer[-3];
+ PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable);
+ PyObject *arg_o = PyStackRef_AsPyObjectBorrow(arg);
assert(oparg == 1);
- if (null != NULL) {
+ if (!PyStackRef_IsNull(null)) {
UOP_STAT_INC(uopcode, miss);
JUMP_TO_JUMP_TARGET();
}
- if (callable != (PyObject *)&PyUnicode_Type) {
+ if (callable_o != (PyObject *)&PyUnicode_Type) {
UOP_STAT_INC(uopcode, miss);
JUMP_TO_JUMP_TARGET();
}
STAT_INC(CALL, hit);
- res = PyObject_Str(arg);
- Py_DECREF(arg);
- if (res == NULL) JUMP_TO_ERROR();
+ res = PyStackRef_FromPyObjectSteal(PyObject_Str(arg_o));
+ PyStackRef_CLOSE(arg);
+ if (PyStackRef_IsNull(res)) JUMP_TO_ERROR();
stack_pointer[-3] = res;
stack_pointer += -2;
assert(WITHIN_STACK_BOUNDS());
}
case _CALL_TUPLE_1: {
- PyObject *arg;
- PyObject *null;
- PyObject *callable;
- PyObject *res;
+ _PyStackRef arg;
+ _PyStackRef null;
+ _PyStackRef callable;
+ _PyStackRef res;
oparg = CURRENT_OPARG();
arg = stack_pointer[-1];
null = stack_pointer[-2];
callable = stack_pointer[-3];
+ PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable);
+ PyObject *arg_o = PyStackRef_AsPyObjectBorrow(arg);
assert(oparg == 1);
- if (null != NULL) {
+ if (!PyStackRef_IsNull(null)) {
UOP_STAT_INC(uopcode, miss);
JUMP_TO_JUMP_TARGET();
}
- if (callable != (PyObject *)&PyTuple_Type) {
+ if (callable_o != (PyObject *)&PyTuple_Type) {
UOP_STAT_INC(uopcode, miss);
JUMP_TO_JUMP_TARGET();
}
STAT_INC(CALL, hit);
- res = PySequence_Tuple(arg);
- Py_DECREF(arg);
- if (res == NULL) JUMP_TO_ERROR();
+ res = PyStackRef_FromPyObjectSteal(PySequence_Tuple(arg_o));
+ PyStackRef_CLOSE(arg);
+ if (PyStackRef_IsNull(res)) JUMP_TO_ERROR();
stack_pointer[-3] = res;
stack_pointer += -2;
assert(WITHIN_STACK_BOUNDS());
/* _CALL_ALLOC_AND_ENTER_INIT is not a viable micro-op for tier 2 because it uses the 'this_instr' variable */
case _EXIT_INIT_CHECK: {
- PyObject *should_be_none;
+ _PyStackRef should_be_none;
should_be_none = stack_pointer[-1];
assert(STACK_LEVEL() == 2);
- if (should_be_none != Py_None) {
+ if (!PyStackRef_Is(should_be_none, PyStackRef_None)) {
PyErr_Format(PyExc_TypeError,
"__init__() should return None, not '%.200s'",
- Py_TYPE(should_be_none)->tp_name);
+ Py_TYPE(PyStackRef_AsPyObjectBorrow(should_be_none))->tp_name);
JUMP_TO_ERROR();
}
stack_pointer += -1;
}
case _CALL_BUILTIN_CLASS: {
- PyObject **args;
- PyObject *self_or_null;
- PyObject *callable;
- PyObject *res;
+ _PyStackRef *args;
+ _PyStackRef self_or_null;
+ _PyStackRef callable;
+ _PyStackRef res;
oparg = CURRENT_OPARG();
args = &stack_pointer[-oparg];
self_or_null = stack_pointer[-1 - oparg];
callable = stack_pointer[-2 - oparg];
+ PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable);
int total_args = oparg;
- if (self_or_null != NULL) {
+ if (!PyStackRef_IsNull(self_or_null)) {
args--;
total_args++;
}
- if (!PyType_Check(callable)) {
+ if (!PyType_Check(callable_o)) {
UOP_STAT_INC(uopcode, miss);
JUMP_TO_JUMP_TARGET();
}
- PyTypeObject *tp = (PyTypeObject *)callable;
+ PyTypeObject *tp = (PyTypeObject *)callable_o;
if (tp->tp_vectorcall == NULL) {
UOP_STAT_INC(uopcode, miss);
JUMP_TO_JUMP_TARGET();
}
STAT_INC(CALL, hit);
- res = tp->tp_vectorcall((PyObject *)tp, args, total_args, NULL);
+ STACKREFS_TO_PYOBJECTS(args, total_args, args_o);
+ if (CONVERSION_FAILED(args_o)) {
+ PyStackRef_CLOSE(callable);
+ PyStackRef_CLOSE(self_or_null);
+ for (int _i = oparg; --_i >= 0;) {
+ PyStackRef_CLOSE(args[_i]);
+ }
+ if (true) JUMP_TO_ERROR();
+ }
+ PyObject *res_o = tp->tp_vectorcall((PyObject *)tp, args_o, total_args, NULL);
+ STACKREFS_TO_PYOBJECTS_CLEANUP(args_o);
/* Free the arguments. */
for (int i = 0; i < total_args; i++) {
- Py_DECREF(args[i]);
+ PyStackRef_CLOSE(args[i]);
}
- Py_DECREF(tp);
- if (res == NULL) JUMP_TO_ERROR();
+ PyStackRef_CLOSE(callable);
+ if (res_o == NULL) JUMP_TO_ERROR();
+ res = PyStackRef_FromPyObjectSteal(res_o);
stack_pointer[-2 - oparg] = res;
stack_pointer += -1 - oparg;
assert(WITHIN_STACK_BOUNDS());
}
case _CALL_BUILTIN_O: {
- PyObject **args;
- PyObject *self_or_null;
- PyObject *callable;
- PyObject *res;
+ _PyStackRef *args;
+ _PyStackRef self_or_null;
+ _PyStackRef callable;
+ _PyStackRef res;
oparg = CURRENT_OPARG();
args = &stack_pointer[-oparg];
self_or_null = stack_pointer[-1 - oparg];
callable = stack_pointer[-2 - oparg];
/* Builtin METH_O functions */
+ PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable);
int total_args = oparg;
- if (self_or_null != NULL) {
+ if (!PyStackRef_IsNull(self_or_null)) {
args--;
total_args++;
}
UOP_STAT_INC(uopcode, miss);
JUMP_TO_JUMP_TARGET();
}
- if (!PyCFunction_CheckExact(callable)) {
+ if (!PyCFunction_CheckExact(callable_o)) {
UOP_STAT_INC(uopcode, miss);
JUMP_TO_JUMP_TARGET();
}
- if (PyCFunction_GET_FLAGS(callable) != METH_O) {
+ if (PyCFunction_GET_FLAGS(callable_o) != METH_O) {
UOP_STAT_INC(uopcode, miss);
JUMP_TO_JUMP_TARGET();
}
JUMP_TO_JUMP_TARGET();
}
STAT_INC(CALL, hit);
- PyCFunction cfunc = PyCFunction_GET_FUNCTION(callable);
- PyObject *arg = args[0];
+ PyCFunction cfunc = PyCFunction_GET_FUNCTION(callable_o);
+ _PyStackRef arg = args[0];
_Py_EnterRecursiveCallTstateUnchecked(tstate);
- res = _PyCFunction_TrampolineCall(cfunc, PyCFunction_GET_SELF(callable), arg);
+ PyObject *res_o = _PyCFunction_TrampolineCall(cfunc, PyCFunction_GET_SELF(callable_o), PyStackRef_AsPyObjectBorrow(arg));
_Py_LeaveRecursiveCallTstate(tstate);
- assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL));
- Py_DECREF(arg);
- Py_DECREF(callable);
- if (res == NULL) JUMP_TO_ERROR();
+ assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL));
+ PyStackRef_CLOSE(arg);
+ PyStackRef_CLOSE(callable);
+ if (res_o == NULL) JUMP_TO_ERROR();
+ res = PyStackRef_FromPyObjectSteal(res_o);
stack_pointer[-2 - oparg] = res;
stack_pointer += -1 - oparg;
assert(WITHIN_STACK_BOUNDS());
}
case _CALL_BUILTIN_FAST: {
- PyObject **args;
- PyObject *self_or_null;
- PyObject *callable;
- PyObject *res;
+ _PyStackRef *args;
+ _PyStackRef self_or_null;
+ _PyStackRef callable;
+ _PyStackRef res;
oparg = CURRENT_OPARG();
args = &stack_pointer[-oparg];
self_or_null = stack_pointer[-1 - oparg];
callable = stack_pointer[-2 - oparg];
/* Builtin METH_FASTCALL functions, without keywords */
+ PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable);
int total_args = oparg;
- if (self_or_null != NULL) {
+ if (!PyStackRef_IsNull(self_or_null)) {
args--;
total_args++;
}
- if (!PyCFunction_CheckExact(callable)) {
+ if (!PyCFunction_CheckExact(callable_o)) {
UOP_STAT_INC(uopcode, miss);
JUMP_TO_JUMP_TARGET();
}
- if (PyCFunction_GET_FLAGS(callable) != METH_FASTCALL) {
+ if (PyCFunction_GET_FLAGS(callable_o) != METH_FASTCALL) {
UOP_STAT_INC(uopcode, miss);
JUMP_TO_JUMP_TARGET();
}
STAT_INC(CALL, hit);
- PyCFunction cfunc = PyCFunction_GET_FUNCTION(callable);
+ PyCFunction cfunc = PyCFunction_GET_FUNCTION(callable_o);
/* res = func(self, args, nargs) */
- res = ((PyCFunctionFast)(void(*)(void))cfunc)(
- PyCFunction_GET_SELF(callable),
- args,
+ STACKREFS_TO_PYOBJECTS(args, total_args, args_o);
+ if (CONVERSION_FAILED(args_o)) {
+ PyStackRef_CLOSE(callable);
+ PyStackRef_CLOSE(self_or_null);
+ for (int _i = oparg; --_i >= 0;) {
+ PyStackRef_CLOSE(args[_i]);
+ }
+ if (true) JUMP_TO_ERROR();
+ }
+ PyObject *res_o = ((PyCFunctionFast)(void(*)(void))cfunc)(
+ PyCFunction_GET_SELF(callable_o),
+ args_o,
total_args);
- assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL));
+ STACKREFS_TO_PYOBJECTS_CLEANUP(args_o);
+ assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL));
/* Free the arguments. */
for (int i = 0; i < total_args; i++) {
- Py_DECREF(args[i]);
+ PyStackRef_CLOSE(args[i]);
}
- Py_DECREF(callable);
- if (res == NULL) JUMP_TO_ERROR();
+ PyStackRef_CLOSE(callable);
+ if (res_o == NULL) JUMP_TO_ERROR();
+ res = PyStackRef_FromPyObjectSteal(res_o);
stack_pointer[-2 - oparg] = res;
stack_pointer += -1 - oparg;
assert(WITHIN_STACK_BOUNDS());
}
case _CALL_BUILTIN_FAST_WITH_KEYWORDS: {
- PyObject **args;
- PyObject *self_or_null;
- PyObject *callable;
- PyObject *res;
+ _PyStackRef *args;
+ _PyStackRef self_or_null;
+ _PyStackRef callable;
+ _PyStackRef res;
oparg = CURRENT_OPARG();
args = &stack_pointer[-oparg];
self_or_null = stack_pointer[-1 - oparg];
callable = stack_pointer[-2 - oparg];
/* Builtin METH_FASTCALL | METH_KEYWORDS functions */
+ PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable);
int total_args = oparg;
- if (self_or_null != NULL) {
+ if (!PyStackRef_IsNull(self_or_null)) {
args--;
total_args++;
}
- if (!PyCFunction_CheckExact(callable)) {
+ if (!PyCFunction_CheckExact(callable_o)) {
UOP_STAT_INC(uopcode, miss);
JUMP_TO_JUMP_TARGET();
}
- if (PyCFunction_GET_FLAGS(callable) != (METH_FASTCALL | METH_KEYWORDS)) {
+ if (PyCFunction_GET_FLAGS(callable_o) != (METH_FASTCALL | METH_KEYWORDS)) {
UOP_STAT_INC(uopcode, miss);
JUMP_TO_JUMP_TARGET();
}
/* res = func(self, args, nargs, kwnames) */
PyCFunctionFastWithKeywords cfunc =
(PyCFunctionFastWithKeywords)(void(*)(void))
- PyCFunction_GET_FUNCTION(callable);
- res = cfunc(PyCFunction_GET_SELF(callable), args, total_args, NULL);
- assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL));
+ PyCFunction_GET_FUNCTION(callable_o);
+ STACKREFS_TO_PYOBJECTS(args, total_args, args_o);
+ if (CONVERSION_FAILED(args_o)) {
+ PyStackRef_CLOSE(callable);
+ PyStackRef_CLOSE(self_or_null);
+ for (int _i = oparg; --_i >= 0;) {
+ PyStackRef_CLOSE(args[_i]);
+ }
+ if (true) JUMP_TO_ERROR();
+ }
+ PyObject *res_o = cfunc(PyCFunction_GET_SELF(callable_o), args_o, total_args, NULL);
+ STACKREFS_TO_PYOBJECTS_CLEANUP(args_o);
+ assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL));
/* Free the arguments. */
for (int i = 0; i < total_args; i++) {
- Py_DECREF(args[i]);
+ PyStackRef_CLOSE(args[i]);
}
- Py_DECREF(callable);
- if (res == NULL) JUMP_TO_ERROR();
+ PyStackRef_CLOSE(callable);
+ if (res_o == NULL) JUMP_TO_ERROR();
+ res = PyStackRef_FromPyObjectSteal(res_o);
stack_pointer[-2 - oparg] = res;
stack_pointer += -1 - oparg;
assert(WITHIN_STACK_BOUNDS());
}
case _CALL_LEN: {
- PyObject **args;
- PyObject *self_or_null;
- PyObject *callable;
- PyObject *res;
+ _PyStackRef *args;
+ _PyStackRef self_or_null;
+ _PyStackRef callable;
+ _PyStackRef res;
oparg = CURRENT_OPARG();
args = &stack_pointer[-oparg];
self_or_null = stack_pointer[-1 - oparg];
callable = stack_pointer[-2 - oparg];
/* len(o) */
+ PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable);
int total_args = oparg;
- if (self_or_null != NULL) {
+ if (!PyStackRef_IsNull(self_or_null)) {
args--;
total_args++;
}
JUMP_TO_JUMP_TARGET();
}
PyInterpreterState *interp = tstate->interp;
- if (callable != interp->callable_cache.len) {
+ if (callable_o != interp->callable_cache.len) {
UOP_STAT_INC(uopcode, miss);
JUMP_TO_JUMP_TARGET();
}
STAT_INC(CALL, hit);
- PyObject *arg = args[0];
+ _PyStackRef arg_stackref = args[0];
+ PyObject *arg = PyStackRef_AsPyObjectBorrow(arg_stackref);
Py_ssize_t len_i = PyObject_Length(arg);
if (len_i < 0) {
JUMP_TO_ERROR();
}
- res = PyLong_FromSsize_t(len_i);
- assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL));
- if (res == NULL) {
+ PyObject *res_o = PyLong_FromSsize_t(len_i);
+ assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL));
+ if (res_o == NULL) {
GOTO_ERROR(error);
}
- Py_DECREF(callable);
- Py_DECREF(arg);
+ PyStackRef_CLOSE(callable);
+ PyStackRef_CLOSE(arg_stackref);
+ res = PyStackRef_FromPyObjectSteal(res_o);
stack_pointer[-2 - oparg] = res;
stack_pointer += -1 - oparg;
assert(WITHIN_STACK_BOUNDS());
}
case _CALL_ISINSTANCE: {
- PyObject **args;
- PyObject *self_or_null;
- PyObject *callable;
- PyObject *res;
+ _PyStackRef *args;
+ _PyStackRef self_or_null;
+ _PyStackRef callable;
+ _PyStackRef res;
oparg = CURRENT_OPARG();
args = &stack_pointer[-oparg];
self_or_null = stack_pointer[-1 - oparg];
callable = stack_pointer[-2 - oparg];
/* isinstance(o, o2) */
+ PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable);
int total_args = oparg;
- if (self_or_null != NULL) {
+ if (!PyStackRef_IsNull(self_or_null)) {
args--;
total_args++;
}
JUMP_TO_JUMP_TARGET();
}
PyInterpreterState *interp = tstate->interp;
- if (callable != interp->callable_cache.isinstance) {
+ if (callable_o != interp->callable_cache.isinstance) {
UOP_STAT_INC(uopcode, miss);
JUMP_TO_JUMP_TARGET();
}
STAT_INC(CALL, hit);
- PyObject *cls = args[1];
- PyObject *inst = args[0];
- int retval = PyObject_IsInstance(inst, cls);
+ _PyStackRef cls_stackref = args[1];
+ _PyStackRef inst_stackref = args[0];
+ int retval = PyObject_IsInstance(PyStackRef_AsPyObjectBorrow(inst_stackref), PyStackRef_AsPyObjectBorrow(cls_stackref));
if (retval < 0) {
JUMP_TO_ERROR();
}
- res = PyBool_FromLong(retval);
- assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL));
- if (res == NULL) {
- GOTO_ERROR(error);
- }
- Py_DECREF(inst);
- Py_DECREF(cls);
- Py_DECREF(callable);
+ res = retval ? PyStackRef_True : PyStackRef_False;
+ assert((!PyStackRef_IsNull(res)) ^ (_PyErr_Occurred(tstate) != NULL));
+ PyStackRef_CLOSE(inst_stackref);
+ PyStackRef_CLOSE(cls_stackref);
+ PyStackRef_CLOSE(callable);
stack_pointer[-2 - oparg] = res;
stack_pointer += -1 - oparg;
assert(WITHIN_STACK_BOUNDS());
}
case _CALL_METHOD_DESCRIPTOR_O: {
- PyObject **args;
- PyObject *self_or_null;
- PyObject *callable;
- PyObject *res;
+ _PyStackRef *args;
+ _PyStackRef self_or_null;
+ _PyStackRef callable;
+ _PyStackRef res;
oparg = CURRENT_OPARG();
args = &stack_pointer[-oparg];
self_or_null = stack_pointer[-1 - oparg];
callable = stack_pointer[-2 - oparg];
+ PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable);
int total_args = oparg;
- if (self_or_null != NULL) {
+ if (!PyStackRef_IsNull(self_or_null)) {
args--;
total_args++;
}
- PyMethodDescrObject *method = (PyMethodDescrObject *)callable;
+ PyMethodDescrObject *method = (PyMethodDescrObject *)callable_o;
if (total_args != 2) {
UOP_STAT_INC(uopcode, miss);
JUMP_TO_JUMP_TARGET();
UOP_STAT_INC(uopcode, miss);
JUMP_TO_JUMP_TARGET();
}
- PyObject *arg = args[1];
- PyObject *self = args[0];
- if (!Py_IS_TYPE(self, method->d_common.d_type)) {
+ _PyStackRef arg_stackref = args[1];
+ _PyStackRef self_stackref = args[0];
+ if (!Py_IS_TYPE(PyStackRef_AsPyObjectBorrow(self_stackref),
+ method->d_common.d_type)) {
UOP_STAT_INC(uopcode, miss);
JUMP_TO_JUMP_TARGET();
}
STAT_INC(CALL, hit);
PyCFunction cfunc = meth->ml_meth;
_Py_EnterRecursiveCallTstateUnchecked(tstate);
- res = _PyCFunction_TrampolineCall(cfunc, self, arg);
+ PyObject *res_o = _PyCFunction_TrampolineCall(cfunc,
+ PyStackRef_AsPyObjectBorrow(self_stackref),
+ PyStackRef_AsPyObjectBorrow(arg_stackref));
_Py_LeaveRecursiveCallTstate(tstate);
- assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL));
- Py_DECREF(self);
- Py_DECREF(arg);
- Py_DECREF(callable);
- if (res == NULL) JUMP_TO_ERROR();
+ assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL));
+ PyStackRef_CLOSE(self_stackref);
+ PyStackRef_CLOSE(arg_stackref);
+ PyStackRef_CLOSE(callable);
+ if (res_o == NULL) JUMP_TO_ERROR();
+ res = PyStackRef_FromPyObjectSteal(res_o);
stack_pointer[-2 - oparg] = res;
stack_pointer += -1 - oparg;
assert(WITHIN_STACK_BOUNDS());
}
case _CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS: {
- PyObject **args;
- PyObject *self_or_null;
- PyObject *callable;
- PyObject *res;
+ _PyStackRef *args;
+ _PyStackRef self_or_null;
+ _PyStackRef callable;
+ _PyStackRef res;
oparg = CURRENT_OPARG();
args = &stack_pointer[-oparg];
self_or_null = stack_pointer[-1 - oparg];
callable = stack_pointer[-2 - oparg];
+ PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable);
int total_args = oparg;
- if (self_or_null != NULL) {
+ if (!PyStackRef_IsNull(self_or_null)) {
args--;
total_args++;
}
- PyMethodDescrObject *method = (PyMethodDescrObject *)callable;
+ PyMethodDescrObject *method = (PyMethodDescrObject *)callable_o;
if (!Py_IS_TYPE(method, &PyMethodDescr_Type)) {
UOP_STAT_INC(uopcode, miss);
JUMP_TO_JUMP_TARGET();
JUMP_TO_JUMP_TARGET();
}
PyTypeObject *d_type = method->d_common.d_type;
- PyObject *self = args[0];
+ PyObject *self = PyStackRef_AsPyObjectBorrow(args[0]);
if (!Py_IS_TYPE(self, d_type)) {
UOP_STAT_INC(uopcode, miss);
JUMP_TO_JUMP_TARGET();
int nargs = total_args - 1;
PyCFunctionFastWithKeywords cfunc =
(PyCFunctionFastWithKeywords)(void(*)(void))meth->ml_meth;
- res = cfunc(self, args + 1, nargs, NULL);
- assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL));
+ STACKREFS_TO_PYOBJECTS(args, nargs, args_o);
+ if (CONVERSION_FAILED(args_o)) {
+ PyStackRef_CLOSE(callable);
+ PyStackRef_CLOSE(self_or_null);
+ for (int _i = oparg; --_i >= 0;) {
+ PyStackRef_CLOSE(args[_i]);
+ }
+ if (true) JUMP_TO_ERROR();
+ }
+ PyObject *res_o = cfunc(self, (args_o + 1), nargs, NULL);
+ STACKREFS_TO_PYOBJECTS_CLEANUP(args_o);
+ assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL));
/* Free the arguments. */
for (int i = 0; i < total_args; i++) {
- Py_DECREF(args[i]);
+ PyStackRef_CLOSE(args[i]);
}
- Py_DECREF(callable);
- if (res == NULL) JUMP_TO_ERROR();
+ PyStackRef_CLOSE(callable);
+ if (res_o == NULL) JUMP_TO_ERROR();
+ res = PyStackRef_FromPyObjectSteal(res_o);
stack_pointer[-2 - oparg] = res;
stack_pointer += -1 - oparg;
assert(WITHIN_STACK_BOUNDS());
}
case _CALL_METHOD_DESCRIPTOR_NOARGS: {
- PyObject **args;
- PyObject *self_or_null;
- PyObject *callable;
- PyObject *res;
+ _PyStackRef *args;
+ _PyStackRef self_or_null;
+ _PyStackRef callable;
+ _PyStackRef res;
oparg = CURRENT_OPARG();
args = &stack_pointer[-oparg];
self_or_null = stack_pointer[-1 - oparg];
callable = stack_pointer[-2 - oparg];
assert(oparg == 0 || oparg == 1);
+ PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable);
int total_args = oparg;
- if (self_or_null != NULL) {
+ if (!PyStackRef_IsNull(self_or_null)) {
args--;
total_args++;
}
UOP_STAT_INC(uopcode, miss);
JUMP_TO_JUMP_TARGET();
}
- PyMethodDescrObject *method = (PyMethodDescrObject *)callable;
+ PyMethodDescrObject *method = (PyMethodDescrObject *)callable_o;
if (!Py_IS_TYPE(method, &PyMethodDescr_Type)) {
UOP_STAT_INC(uopcode, miss);
JUMP_TO_JUMP_TARGET();
}
PyMethodDef *meth = method->d_method;
- PyObject *self = args[0];
+ _PyStackRef self_stackref = args[0];
+ PyObject *self = PyStackRef_AsPyObjectBorrow(self_stackref);
if (!Py_IS_TYPE(self, method->d_common.d_type)) {
UOP_STAT_INC(uopcode, miss);
JUMP_TO_JUMP_TARGET();
STAT_INC(CALL, hit);
PyCFunction cfunc = meth->ml_meth;
_Py_EnterRecursiveCallTstateUnchecked(tstate);
- res = _PyCFunction_TrampolineCall(cfunc, self, NULL);
+ PyObject *res_o = _PyCFunction_TrampolineCall(cfunc, self, NULL);
_Py_LeaveRecursiveCallTstate(tstate);
- assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL));
- Py_DECREF(self);
- Py_DECREF(callable);
- if (res == NULL) JUMP_TO_ERROR();
+ assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL));
+ PyStackRef_CLOSE(self_stackref);
+ PyStackRef_CLOSE(callable);
+ if (res_o == NULL) JUMP_TO_ERROR();
+ res = PyStackRef_FromPyObjectSteal(res_o);
stack_pointer[-2 - oparg] = res;
stack_pointer += -1 - oparg;
assert(WITHIN_STACK_BOUNDS());
}
case _CALL_METHOD_DESCRIPTOR_FAST: {
- PyObject **args;
- PyObject *self_or_null;
- PyObject *callable;
- PyObject *res;
+ _PyStackRef *args;
+ _PyStackRef self_or_null;
+ _PyStackRef callable;
+ _PyStackRef res;
oparg = CURRENT_OPARG();
args = &stack_pointer[-oparg];
self_or_null = stack_pointer[-1 - oparg];
callable = stack_pointer[-2 - oparg];
+ PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable);
int total_args = oparg;
- if (self_or_null != NULL) {
+ if (!PyStackRef_IsNull(self_or_null)) {
args--;
total_args++;
}
- PyMethodDescrObject *method = (PyMethodDescrObject *)callable;
+ PyMethodDescrObject *method = (PyMethodDescrObject *)callable_o;
/* Builtin METH_FASTCALL methods, without keywords */
if (!Py_IS_TYPE(method, &PyMethodDescr_Type)) {
UOP_STAT_INC(uopcode, miss);
UOP_STAT_INC(uopcode, miss);
JUMP_TO_JUMP_TARGET();
}
- PyObject *self = args[0];
+ PyObject *self = PyStackRef_AsPyObjectBorrow(args[0]);
if (!Py_IS_TYPE(self, method->d_common.d_type)) {
UOP_STAT_INC(uopcode, miss);
JUMP_TO_JUMP_TARGET();
PyCFunctionFast cfunc =
(PyCFunctionFast)(void(*)(void))meth->ml_meth;
int nargs = total_args - 1;
- res = cfunc(self, args + 1, nargs);
- assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL));
+ STACKREFS_TO_PYOBJECTS(args, nargs, args_o);
+ if (CONVERSION_FAILED(args_o)) {
+ PyStackRef_CLOSE(callable);
+ PyStackRef_CLOSE(self_or_null);
+ for (int _i = oparg; --_i >= 0;) {
+ PyStackRef_CLOSE(args[_i]);
+ }
+ if (true) JUMP_TO_ERROR();
+ }
+ PyObject *res_o = cfunc(self, (args_o + 1), nargs);
+ STACKREFS_TO_PYOBJECTS_CLEANUP(args_o);
+ assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL));
/* Clear the stack of the arguments. */
for (int i = 0; i < total_args; i++) {
- Py_DECREF(args[i]);
+ PyStackRef_CLOSE(args[i]);
}
- Py_DECREF(callable);
- if (res == NULL) JUMP_TO_ERROR();
+ PyStackRef_CLOSE(callable);
+ if (res_o == NULL) JUMP_TO_ERROR();
+ res = PyStackRef_FromPyObjectSteal(res_o);
stack_pointer[-2 - oparg] = res;
stack_pointer += -1 - oparg;
assert(WITHIN_STACK_BOUNDS());
/* _CALL_FUNCTION_EX is not a viable micro-op for tier 2 because it uses the 'this_instr' variable */
case _MAKE_FUNCTION: {
- PyObject *codeobj;
- PyObject *func;
- codeobj = stack_pointer[-1];
+ _PyStackRef codeobj_st;
+ _PyStackRef func;
+ codeobj_st = stack_pointer[-1];
+ PyObject *codeobj = PyStackRef_AsPyObjectBorrow(codeobj_st);
PyFunctionObject *func_obj = (PyFunctionObject *)
PyFunction_New(codeobj, GLOBALS());
- Py_DECREF(codeobj);
+ PyStackRef_CLOSE(codeobj_st);
if (func_obj == NULL) {
JUMP_TO_ERROR();
}
_PyFunction_SetVersion(
func_obj, ((PyCodeObject *)codeobj)->co_version);
- func = (PyObject *)func_obj;
+ func = PyStackRef_FromPyObjectSteal((PyObject *)func_obj);
stack_pointer[-1] = func;
break;
}
case _SET_FUNCTION_ATTRIBUTE: {
- PyObject *func;
- PyObject *attr;
+ _PyStackRef func_st;
+ _PyStackRef attr_st;
oparg = CURRENT_OPARG();
- func = stack_pointer[-1];
- attr = stack_pointer[-2];
+ func_st = stack_pointer[-1];
+ attr_st = stack_pointer[-2];
+ PyObject *func = PyStackRef_AsPyObjectBorrow(func_st);
+ PyObject *attr = PyStackRef_AsPyObjectBorrow(attr_st);
assert(PyFunction_Check(func));
PyFunctionObject *func_obj = (PyFunctionObject *)func;
switch(oparg) {
default:
Py_UNREACHABLE();
}
- stack_pointer[-2] = func;
+ stack_pointer[-2] = func_st;
stack_pointer += -1;
assert(WITHIN_STACK_BOUNDS());
break;
}
case _RETURN_GENERATOR: {
- PyObject *res;
+ _PyStackRef res;
assert(PyFunction_Check(frame->f_funcobj));
PyFunctionObject *func = (PyFunctionObject *)frame->f_funcobj;
PyGenObject *gen = (PyGenObject *)_Py_MakeCoro(func);
gen->gi_frame_state = FRAME_CREATED;
gen_frame->owner = FRAME_OWNED_BY_GENERATOR;
_Py_LeaveRecursiveCallPy(tstate);
- res = (PyObject *)gen;
+ res = PyStackRef_FromPyObjectSteal((PyObject *)gen);
_PyInterpreterFrame *prev = frame->previous;
_PyThreadState_PopFrame(tstate, frame);
frame = tstate->current_frame = prev;
}
case _BUILD_SLICE: {
- PyObject *step = NULL;
- PyObject *stop;
- PyObject *start;
- PyObject *slice;
+ _PyStackRef step = PyStackRef_NULL;
+ _PyStackRef stop;
+ _PyStackRef start;
+ _PyStackRef slice;
oparg = CURRENT_OPARG();
if (oparg == 3) { step = stack_pointer[-((oparg == 3) ? 1 : 0)]; }
stop = stack_pointer[-1 - ((oparg == 3) ? 1 : 0)];
start = stack_pointer[-2 - ((oparg == 3) ? 1 : 0)];
- slice = PySlice_New(start, stop, step);
- Py_DECREF(start);
- Py_DECREF(stop);
- Py_XDECREF(step);
- if (slice == NULL) JUMP_TO_ERROR();
+ PyObject *start_o = PyStackRef_AsPyObjectBorrow(start);
+ PyObject *stop_o = PyStackRef_AsPyObjectBorrow(stop);
+ PyObject *step_o = PyStackRef_AsPyObjectBorrow(step);
+ PyObject *slice_o = PySlice_New(start_o, stop_o, step_o);
+ PyStackRef_CLOSE(start);
+ PyStackRef_CLOSE(stop);
+ PyStackRef_XCLOSE(step);
+ if (slice_o == NULL) JUMP_TO_ERROR();
+ slice = PyStackRef_FromPyObjectSteal(slice_o);
stack_pointer[-2 - ((oparg == 3) ? 1 : 0)] = slice;
stack_pointer += -1 - ((oparg == 3) ? 1 : 0);
assert(WITHIN_STACK_BOUNDS());
}
case _CONVERT_VALUE: {
- PyObject *value;
- PyObject *result;
+ _PyStackRef value;
+ _PyStackRef result;
oparg = CURRENT_OPARG();
value = stack_pointer[-1];
conversion_func conv_fn;
assert(oparg >= FVC_STR && oparg <= FVC_ASCII);
conv_fn = _PyEval_ConversionFuncs[oparg];
- result = conv_fn(value);
- Py_DECREF(value);
- if (result == NULL) JUMP_TO_ERROR();
+ PyObject *result_o = conv_fn(PyStackRef_AsPyObjectBorrow(value));
+ PyStackRef_CLOSE(value);
+ if (result_o == NULL) JUMP_TO_ERROR();
+ result = PyStackRef_FromPyObjectSteal(result_o);
stack_pointer[-1] = result;
break;
}
case _FORMAT_SIMPLE: {
- PyObject *value;
- PyObject *res;
+ _PyStackRef value;
+ _PyStackRef res;
value = stack_pointer[-1];
+ PyObject *value_o = PyStackRef_AsPyObjectBorrow(value);
/* If value is a unicode object, then we know the result
* of format(value) is value itself. */
- if (!PyUnicode_CheckExact(value)) {
- res = PyObject_Format(value, NULL);
- Py_DECREF(value);
- if (res == NULL) JUMP_TO_ERROR();
+ if (!PyUnicode_CheckExact(value_o)) {
+ res = PyStackRef_FromPyObjectSteal(PyObject_Format(value_o, NULL));
+ PyStackRef_CLOSE(value);
+ if (PyStackRef_IsNull(res)) JUMP_TO_ERROR();
}
else {
res = value;
}
case _FORMAT_WITH_SPEC: {
- PyObject *fmt_spec;
- PyObject *value;
- PyObject *res;
+ _PyStackRef fmt_spec;
+ _PyStackRef value;
+ _PyStackRef res;
fmt_spec = stack_pointer[-1];
value = stack_pointer[-2];
- res = PyObject_Format(value, fmt_spec);
- Py_DECREF(value);
- Py_DECREF(fmt_spec);
- if (res == NULL) JUMP_TO_ERROR();
+ PyObject *res_o = PyObject_Format(PyStackRef_AsPyObjectBorrow(value), PyStackRef_AsPyObjectBorrow(fmt_spec));
+ PyStackRef_CLOSE(value);
+ PyStackRef_CLOSE(fmt_spec);
+ if (res_o == NULL) JUMP_TO_ERROR();
+ res = PyStackRef_FromPyObjectSteal(res_o);
stack_pointer[-2] = res;
stack_pointer += -1;
assert(WITHIN_STACK_BOUNDS());
}
case _COPY: {
- PyObject *bottom;
- PyObject *top;
+ _PyStackRef bottom;
+ _PyStackRef top;
oparg = CURRENT_OPARG();
bottom = stack_pointer[-1 - (oparg-1)];
assert(oparg > 0);
- top = Py_NewRef(bottom);
+ top = PyStackRef_DUP(bottom);
stack_pointer[0] = top;
stack_pointer += 1;
assert(WITHIN_STACK_BOUNDS());
}
case _BINARY_OP: {
- PyObject *rhs;
- PyObject *lhs;
- PyObject *res;
+ _PyStackRef rhs;
+ _PyStackRef lhs;
+ _PyStackRef res;
oparg = CURRENT_OPARG();
rhs = stack_pointer[-1];
lhs = stack_pointer[-2];
+ PyObject *lhs_o = PyStackRef_AsPyObjectBorrow(lhs);
+ PyObject *rhs_o = PyStackRef_AsPyObjectBorrow(rhs);
assert(_PyEval_BinaryOps[oparg]);
- res = _PyEval_BinaryOps[oparg](lhs, rhs);
- Py_DECREF(lhs);
- Py_DECREF(rhs);
- if (res == NULL) JUMP_TO_ERROR();
+ PyObject *res_o = _PyEval_BinaryOps[oparg](lhs_o, rhs_o);
+ PyStackRef_CLOSE(lhs);
+ PyStackRef_CLOSE(rhs);
+ if (res_o == NULL) JUMP_TO_ERROR();
+ res = PyStackRef_FromPyObjectSteal(res_o);
stack_pointer[-2] = res;
stack_pointer += -1;
assert(WITHIN_STACK_BOUNDS());
}
case _SWAP: {
- PyObject *top;
- PyObject *bottom;
+ _PyStackRef top;
+ _PyStackRef bottom;
oparg = CURRENT_OPARG();
top = stack_pointer[-1];
bottom = stack_pointer[-2 - (oparg-2)];
/* _INSTRUMENTED_POP_JUMP_IF_NOT_NONE is not a viable micro-op for tier 2 because it is instrumented */
case _GUARD_IS_TRUE_POP: {
- PyObject *flag;
+ _PyStackRef flag;
flag = stack_pointer[-1];
stack_pointer += -1;
assert(WITHIN_STACK_BOUNDS());
- if (!Py_IsTrue(flag)) {
+ if (!PyStackRef_Is(flag, PyStackRef_True)) {
UOP_STAT_INC(uopcode, miss);
JUMP_TO_JUMP_TARGET();
}
- assert(Py_IsTrue(flag));
+ assert(PyStackRef_Is(flag, PyStackRef_True));
break;
}
case _GUARD_IS_FALSE_POP: {
- PyObject *flag;
+ _PyStackRef flag;
flag = stack_pointer[-1];
stack_pointer += -1;
assert(WITHIN_STACK_BOUNDS());
- if (!Py_IsFalse(flag)) {
+ if (!PyStackRef_Is(flag, PyStackRef_False)) {
UOP_STAT_INC(uopcode, miss);
JUMP_TO_JUMP_TARGET();
}
- assert(Py_IsFalse(flag));
+ assert(PyStackRef_Is(flag, PyStackRef_False));
break;
}
case _GUARD_IS_NONE_POP: {
- PyObject *val;
+ _PyStackRef val;
val = stack_pointer[-1];
stack_pointer += -1;
assert(WITHIN_STACK_BOUNDS());
- if (!Py_IsNone(val)) {
- Py_DECREF(val);
+ if (!PyStackRef_Is(val, PyStackRef_None)) {
+ PyStackRef_CLOSE(val);
if (1) {
UOP_STAT_INC(uopcode, miss);
JUMP_TO_JUMP_TARGET();
}
case _GUARD_IS_NOT_NONE_POP: {
- PyObject *val;
+ _PyStackRef val;
val = stack_pointer[-1];
stack_pointer += -1;
assert(WITHIN_STACK_BOUNDS());
- if (Py_IsNone(val)) {
+ if (PyStackRef_Is(val, PyStackRef_None)) {
UOP_STAT_INC(uopcode, miss);
JUMP_TO_JUMP_TARGET();
}
- Py_DECREF(val);
+ PyStackRef_CLOSE(val);
break;
}
}
case _LOAD_CONST_INLINE: {
- PyObject *value;
+ _PyStackRef value;
PyObject *ptr = (PyObject *)CURRENT_OPERAND();
- value = Py_NewRef(ptr);
+ value = PyStackRef_FromPyObjectNew(ptr);
stack_pointer[0] = value;
stack_pointer += 1;
assert(WITHIN_STACK_BOUNDS());
}
case _LOAD_CONST_INLINE_BORROW: {
- PyObject *value;
+ _PyStackRef value;
PyObject *ptr = (PyObject *)CURRENT_OPERAND();
- value = ptr;
+ value = PyStackRef_FromPyObjectImmortal(ptr);
stack_pointer[0] = value;
stack_pointer += 1;
assert(WITHIN_STACK_BOUNDS());
}
case _POP_TOP_LOAD_CONST_INLINE_BORROW: {
- PyObject *pop;
- PyObject *value;
+ _PyStackRef pop;
+ _PyStackRef value;
pop = stack_pointer[-1];
PyObject *ptr = (PyObject *)CURRENT_OPERAND();
- Py_DECREF(pop);
- value = ptr;
+ PyStackRef_CLOSE(pop);
+ value = PyStackRef_FromPyObjectImmortal(ptr);
stack_pointer[-1] = value;
break;
}
case _LOAD_CONST_INLINE_WITH_NULL: {
- PyObject *value;
- PyObject *null;
+ _PyStackRef value;
+ _PyStackRef null;
PyObject *ptr = (PyObject *)CURRENT_OPERAND();
- value = Py_NewRef(ptr);
- null = NULL;
+ value = PyStackRef_FromPyObjectNew(ptr);
+ null = PyStackRef_NULL;
stack_pointer[0] = value;
stack_pointer[1] = null;
stack_pointer += 2;
}
case _LOAD_CONST_INLINE_BORROW_WITH_NULL: {
- PyObject *value;
- PyObject *null;
+ _PyStackRef value;
+ _PyStackRef null;
PyObject *ptr = (PyObject *)CURRENT_OPERAND();
- value = ptr;
- null = NULL;
+ value = PyStackRef_FromPyObjectImmortal(ptr);
+ null = PyStackRef_NULL;
stack_pointer[0] = value;
stack_pointer[1] = null;
stack_pointer += 2;
}
case _INTERNAL_INCREMENT_OPT_COUNTER: {
- PyObject *opt;
+ _PyStackRef opt;
opt = stack_pointer[-1];
- _PyCounterOptimizerObject *exe = (_PyCounterOptimizerObject *)opt;
+ _PyCounterOptimizerObject *exe = (_PyCounterOptimizerObject *)PyStackRef_AsPyObjectBorrow(opt);
exe->count++;
stack_pointer += -1;
assert(WITHIN_STACK_BOUNDS());
Py_VISIT(frame->f_funcobj);
Py_VISIT(_PyFrame_GetCode(frame));
/* locals */
- PyObject **locals = _PyFrame_GetLocalsArray(frame);
+ _PyStackRef *locals = _PyFrame_GetLocalsArray(frame);
int i = 0;
/* locals and stack */
for (; i <frame->stacktop; i++) {
- Py_VISIT(locals[i]);
+ Py_VISIT(PyStackRef_AsPyObjectBorrow(locals[i]));
}
return 0;
}
int stacktop = frame->stacktop;
frame->stacktop = 0;
for (int i = 0; i < stacktop; i++) {
- Py_XDECREF(frame->localsplus[i]);
+ PyStackRef_XCLOSE(frame->localsplus[i]);
}
Py_CLEAR(frame->f_locals);
}
PREDICTED(BINARY_OP);
_Py_CODEUNIT *this_instr = next_instr - 2;
(void)this_instr;
- PyObject *rhs;
- PyObject *lhs;
- PyObject *res;
+ _PyStackRef rhs;
+ _PyStackRef lhs;
+ _PyStackRef res;
// _SPECIALIZE_BINARY_OP
rhs = stack_pointer[-1];
lhs = stack_pointer[-2];
}
// _BINARY_OP
{
+ PyObject *lhs_o = PyStackRef_AsPyObjectBorrow(lhs);
+ PyObject *rhs_o = PyStackRef_AsPyObjectBorrow(rhs);
assert(_PyEval_BinaryOps[oparg]);
- res = _PyEval_BinaryOps[oparg](lhs, rhs);
- Py_DECREF(lhs);
- Py_DECREF(rhs);
- if (res == NULL) goto pop_2_error;
+ PyObject *res_o = _PyEval_BinaryOps[oparg](lhs_o, rhs_o);
+ PyStackRef_CLOSE(lhs);
+ PyStackRef_CLOSE(rhs);
+ if (res_o == NULL) goto pop_2_error;
+ res = PyStackRef_FromPyObjectSteal(res_o);
}
stack_pointer[-2] = res;
stack_pointer += -1;
next_instr += 2;
INSTRUCTION_STATS(BINARY_OP_ADD_FLOAT);
static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 1, "incorrect cache size");
- PyObject *right;
- PyObject *left;
- PyObject *res;
+ _PyStackRef right;
+ _PyStackRef left;
+ _PyStackRef res;
// _GUARD_BOTH_FLOAT
right = stack_pointer[-1];
left = stack_pointer[-2];
{
- DEOPT_IF(!PyFloat_CheckExact(left), BINARY_OP);
- DEOPT_IF(!PyFloat_CheckExact(right), BINARY_OP);
+ PyObject *left_o = PyStackRef_AsPyObjectBorrow(left);
+ PyObject *right_o = PyStackRef_AsPyObjectBorrow(right);
+ DEOPT_IF(!PyFloat_CheckExact(left_o), BINARY_OP);
+ DEOPT_IF(!PyFloat_CheckExact(right_o), BINARY_OP);
}
/* Skip 1 cache entry */
// _BINARY_OP_ADD_FLOAT
{
+ PyObject *left_o = PyStackRef_AsPyObjectBorrow(left);
+ PyObject *right_o = PyStackRef_AsPyObjectBorrow(right);
STAT_INC(BINARY_OP, hit);
double dres =
- ((PyFloatObject *)left)->ob_fval +
- ((PyFloatObject *)right)->ob_fval;
- DECREF_INPUTS_AND_REUSE_FLOAT(left, right, dres, res);
+ ((PyFloatObject *)left_o)->ob_fval +
+ ((PyFloatObject *)right_o)->ob_fval;
+ PyObject *res_o;
+ DECREF_INPUTS_AND_REUSE_FLOAT(left_o, right_o, dres, res_o);
+ res = PyStackRef_FromPyObjectSteal(res_o);
}
stack_pointer[-2] = res;
stack_pointer += -1;
next_instr += 2;
INSTRUCTION_STATS(BINARY_OP_ADD_INT);
static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 1, "incorrect cache size");
- PyObject *right;
- PyObject *left;
- PyObject *res;
+ _PyStackRef right;
+ _PyStackRef left;
+ _PyStackRef res;
// _GUARD_BOTH_INT
right = stack_pointer[-1];
left = stack_pointer[-2];
{
- DEOPT_IF(!PyLong_CheckExact(left), BINARY_OP);
- DEOPT_IF(!PyLong_CheckExact(right), BINARY_OP);
+ PyObject *left_o = PyStackRef_AsPyObjectBorrow(left);
+ PyObject *right_o = PyStackRef_AsPyObjectBorrow(right);
+ DEOPT_IF(!PyLong_CheckExact(left_o), BINARY_OP);
+ DEOPT_IF(!PyLong_CheckExact(right_o), BINARY_OP);
}
/* Skip 1 cache entry */
// _BINARY_OP_ADD_INT
{
+ PyObject *left_o = PyStackRef_AsPyObjectBorrow(left);
+ PyObject *right_o = PyStackRef_AsPyObjectBorrow(right);
STAT_INC(BINARY_OP, hit);
- res = _PyLong_Add((PyLongObject *)left, (PyLongObject *)right);
- _Py_DECREF_SPECIALIZED(right, (destructor)PyObject_Free);
- _Py_DECREF_SPECIALIZED(left, (destructor)PyObject_Free);
- if (res == NULL) goto pop_2_error;
+ PyObject *res_o = _PyLong_Add((PyLongObject *)left_o, (PyLongObject *)right_o);
+ _Py_DECREF_SPECIALIZED(right_o, (destructor)PyObject_Free);
+ _Py_DECREF_SPECIALIZED(left_o, (destructor)PyObject_Free);
+ if (res_o == NULL) goto pop_2_error;
+ res = PyStackRef_FromPyObjectSteal(res_o);
}
stack_pointer[-2] = res;
stack_pointer += -1;
next_instr += 2;
INSTRUCTION_STATS(BINARY_OP_ADD_UNICODE);
static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 1, "incorrect cache size");
- PyObject *right;
- PyObject *left;
- PyObject *res;
+ _PyStackRef right;
+ _PyStackRef left;
+ _PyStackRef res;
// _GUARD_BOTH_UNICODE
right = stack_pointer[-1];
left = stack_pointer[-2];
{
- DEOPT_IF(!PyUnicode_CheckExact(left), BINARY_OP);
- DEOPT_IF(!PyUnicode_CheckExact(right), BINARY_OP);
+ PyObject *left_o = PyStackRef_AsPyObjectBorrow(left);
+ PyObject *right_o = PyStackRef_AsPyObjectBorrow(right);
+ DEOPT_IF(!PyUnicode_CheckExact(left_o), BINARY_OP);
+ DEOPT_IF(!PyUnicode_CheckExact(right_o), BINARY_OP);
}
/* Skip 1 cache entry */
// _BINARY_OP_ADD_UNICODE
{
+ PyObject *left_o = PyStackRef_AsPyObjectBorrow(left);
+ PyObject *right_o = PyStackRef_AsPyObjectBorrow(right);
STAT_INC(BINARY_OP, hit);
- res = PyUnicode_Concat(left, right);
- _Py_DECREF_SPECIALIZED(left, _PyUnicode_ExactDealloc);
- _Py_DECREF_SPECIALIZED(right, _PyUnicode_ExactDealloc);
- if (res == NULL) goto pop_2_error;
+ PyObject *res_o = PyUnicode_Concat(left_o, right_o);
+ _Py_DECREF_SPECIALIZED(left_o, _PyUnicode_ExactDealloc);
+ _Py_DECREF_SPECIALIZED(right_o, _PyUnicode_ExactDealloc);
+ if (res_o == NULL) goto pop_2_error;
+ res = PyStackRef_FromPyObjectSteal(res_o);
}
stack_pointer[-2] = res;
stack_pointer += -1;
next_instr += 2;
INSTRUCTION_STATS(BINARY_OP_INPLACE_ADD_UNICODE);
static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 1, "incorrect cache size");
- PyObject *right;
- PyObject *left;
+ _PyStackRef right;
+ _PyStackRef left;
// _GUARD_BOTH_UNICODE
right = stack_pointer[-1];
left = stack_pointer[-2];
{
- DEOPT_IF(!PyUnicode_CheckExact(left), BINARY_OP);
- DEOPT_IF(!PyUnicode_CheckExact(right), BINARY_OP);
+ PyObject *left_o = PyStackRef_AsPyObjectBorrow(left);
+ PyObject *right_o = PyStackRef_AsPyObjectBorrow(right);
+ DEOPT_IF(!PyUnicode_CheckExact(left_o), BINARY_OP);
+ DEOPT_IF(!PyUnicode_CheckExact(right_o), BINARY_OP);
}
/* Skip 1 cache entry */
// _BINARY_OP_INPLACE_ADD_UNICODE
{
+ PyObject *left_o = PyStackRef_AsPyObjectBorrow(left);
+ PyObject *right_o = PyStackRef_AsPyObjectBorrow(right);
assert(next_instr->op.code == STORE_FAST);
- PyObject **target_local = &GETLOCAL(next_instr->op.arg);
- DEOPT_IF(*target_local != left, BINARY_OP);
+ _PyStackRef *target_local = &GETLOCAL(next_instr->op.arg);
+ DEOPT_IF(!PyStackRef_Is(*target_local, left), BINARY_OP);
STAT_INC(BINARY_OP, hit);
/* Handle `left = left + right` or `left += right` for str.
*
* only the locals reference, so PyUnicode_Append knows
* that the string is safe to mutate.
*/
- assert(Py_REFCNT(left) >= 2);
- _Py_DECREF_NO_DEALLOC(left);
- PyUnicode_Append(target_local, right);
- _Py_DECREF_SPECIALIZED(right, _PyUnicode_ExactDealloc);
- if (*target_local == NULL) goto pop_2_error;
+ assert(Py_REFCNT(left_o) >= 2);
+ _Py_DECREF_NO_DEALLOC(left_o);
+ PyObject *temp = PyStackRef_AsPyObjectBorrow(*target_local);
+ PyUnicode_Append(&temp, right_o);
+ *target_local = PyStackRef_FromPyObjectSteal(temp);
+ _Py_DECREF_SPECIALIZED(right_o, _PyUnicode_ExactDealloc);
+ if (PyStackRef_IsNull(*target_local)) goto pop_2_error;
// The STORE_FAST is already done.
assert(next_instr->op.code == STORE_FAST);
SKIP_OVER(1);
next_instr += 2;
INSTRUCTION_STATS(BINARY_OP_MULTIPLY_FLOAT);
static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 1, "incorrect cache size");
- PyObject *right;
- PyObject *left;
- PyObject *res;
+ _PyStackRef right;
+ _PyStackRef left;
+ _PyStackRef res;
// _GUARD_BOTH_FLOAT
right = stack_pointer[-1];
left = stack_pointer[-2];
{
- DEOPT_IF(!PyFloat_CheckExact(left), BINARY_OP);
- DEOPT_IF(!PyFloat_CheckExact(right), BINARY_OP);
+ PyObject *left_o = PyStackRef_AsPyObjectBorrow(left);
+ PyObject *right_o = PyStackRef_AsPyObjectBorrow(right);
+ DEOPT_IF(!PyFloat_CheckExact(left_o), BINARY_OP);
+ DEOPT_IF(!PyFloat_CheckExact(right_o), BINARY_OP);
}
/* Skip 1 cache entry */
// _BINARY_OP_MULTIPLY_FLOAT
{
+ PyObject *left_o = PyStackRef_AsPyObjectBorrow(left);
+ PyObject *right_o = PyStackRef_AsPyObjectBorrow(right);
STAT_INC(BINARY_OP, hit);
double dres =
- ((PyFloatObject *)left)->ob_fval *
- ((PyFloatObject *)right)->ob_fval;
- DECREF_INPUTS_AND_REUSE_FLOAT(left, right, dres, res);
+ ((PyFloatObject *)left_o)->ob_fval *
+ ((PyFloatObject *)right_o)->ob_fval;
+ PyObject *res_o;
+ DECREF_INPUTS_AND_REUSE_FLOAT(left_o, right_o, dres, res_o);
+ res = PyStackRef_FromPyObjectSteal(res_o);
}
stack_pointer[-2] = res;
stack_pointer += -1;
next_instr += 2;
INSTRUCTION_STATS(BINARY_OP_MULTIPLY_INT);
static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 1, "incorrect cache size");
- PyObject *right;
- PyObject *left;
- PyObject *res;
+ _PyStackRef right;
+ _PyStackRef left;
+ _PyStackRef res;
// _GUARD_BOTH_INT
right = stack_pointer[-1];
left = stack_pointer[-2];
{
- DEOPT_IF(!PyLong_CheckExact(left), BINARY_OP);
- DEOPT_IF(!PyLong_CheckExact(right), BINARY_OP);
+ PyObject *left_o = PyStackRef_AsPyObjectBorrow(left);
+ PyObject *right_o = PyStackRef_AsPyObjectBorrow(right);
+ DEOPT_IF(!PyLong_CheckExact(left_o), BINARY_OP);
+ DEOPT_IF(!PyLong_CheckExact(right_o), BINARY_OP);
}
/* Skip 1 cache entry */
// _BINARY_OP_MULTIPLY_INT
{
+ PyObject *left_o = PyStackRef_AsPyObjectBorrow(left);
+ PyObject *right_o = PyStackRef_AsPyObjectBorrow(right);
STAT_INC(BINARY_OP, hit);
- res = _PyLong_Multiply((PyLongObject *)left, (PyLongObject *)right);
- _Py_DECREF_SPECIALIZED(right, (destructor)PyObject_Free);
- _Py_DECREF_SPECIALIZED(left, (destructor)PyObject_Free);
- if (res == NULL) goto pop_2_error;
+ PyObject *res_o = _PyLong_Multiply((PyLongObject *)left_o, (PyLongObject *)right_o);
+ _Py_DECREF_SPECIALIZED(right_o, (destructor)PyObject_Free);
+ _Py_DECREF_SPECIALIZED(left_o, (destructor)PyObject_Free);
+ if (res_o == NULL) goto pop_2_error;
+ res = PyStackRef_FromPyObjectSteal(res_o);
}
stack_pointer[-2] = res;
stack_pointer += -1;
next_instr += 2;
INSTRUCTION_STATS(BINARY_OP_SUBTRACT_FLOAT);
static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 1, "incorrect cache size");
- PyObject *right;
- PyObject *left;
- PyObject *res;
+ _PyStackRef right;
+ _PyStackRef left;
+ _PyStackRef res;
// _GUARD_BOTH_FLOAT
right = stack_pointer[-1];
left = stack_pointer[-2];
{
- DEOPT_IF(!PyFloat_CheckExact(left), BINARY_OP);
- DEOPT_IF(!PyFloat_CheckExact(right), BINARY_OP);
+ PyObject *left_o = PyStackRef_AsPyObjectBorrow(left);
+ PyObject *right_o = PyStackRef_AsPyObjectBorrow(right);
+ DEOPT_IF(!PyFloat_CheckExact(left_o), BINARY_OP);
+ DEOPT_IF(!PyFloat_CheckExact(right_o), BINARY_OP);
}
/* Skip 1 cache entry */
// _BINARY_OP_SUBTRACT_FLOAT
{
+ PyObject *left_o = PyStackRef_AsPyObjectBorrow(left);
+ PyObject *right_o = PyStackRef_AsPyObjectBorrow(right);
STAT_INC(BINARY_OP, hit);
double dres =
- ((PyFloatObject *)left)->ob_fval -
- ((PyFloatObject *)right)->ob_fval;
- DECREF_INPUTS_AND_REUSE_FLOAT(left, right, dres, res);
+ ((PyFloatObject *)left_o)->ob_fval -
+ ((PyFloatObject *)right_o)->ob_fval;
+ PyObject *res_o;
+ DECREF_INPUTS_AND_REUSE_FLOAT(left_o, right_o, dres, res_o);
+ res = PyStackRef_FromPyObjectSteal(res_o);
}
stack_pointer[-2] = res;
stack_pointer += -1;
next_instr += 2;
INSTRUCTION_STATS(BINARY_OP_SUBTRACT_INT);
static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 1, "incorrect cache size");
- PyObject *right;
- PyObject *left;
- PyObject *res;
+ _PyStackRef right;
+ _PyStackRef left;
+ _PyStackRef res;
// _GUARD_BOTH_INT
right = stack_pointer[-1];
left = stack_pointer[-2];
{
- DEOPT_IF(!PyLong_CheckExact(left), BINARY_OP);
- DEOPT_IF(!PyLong_CheckExact(right), BINARY_OP);
+ PyObject *left_o = PyStackRef_AsPyObjectBorrow(left);
+ PyObject *right_o = PyStackRef_AsPyObjectBorrow(right);
+ DEOPT_IF(!PyLong_CheckExact(left_o), BINARY_OP);
+ DEOPT_IF(!PyLong_CheckExact(right_o), BINARY_OP);
}
/* Skip 1 cache entry */
// _BINARY_OP_SUBTRACT_INT
{
+ PyObject *left_o = PyStackRef_AsPyObjectBorrow(left);
+ PyObject *right_o = PyStackRef_AsPyObjectBorrow(right);
STAT_INC(BINARY_OP, hit);
- res = _PyLong_Subtract((PyLongObject *)left, (PyLongObject *)right);
- _Py_DECREF_SPECIALIZED(right, (destructor)PyObject_Free);
- _Py_DECREF_SPECIALIZED(left, (destructor)PyObject_Free);
- if (res == NULL) goto pop_2_error;
+ PyObject *res_o = _PyLong_Subtract((PyLongObject *)left_o, (PyLongObject *)right_o);
+ _Py_DECREF_SPECIALIZED(right_o, (destructor)PyObject_Free);
+ _Py_DECREF_SPECIALIZED(left_o, (destructor)PyObject_Free);;
+ if (res_o == NULL) goto pop_2_error;
+ res = PyStackRef_FromPyObjectSteal(res_o);
}
stack_pointer[-2] = res;
stack_pointer += -1;
frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(BINARY_SLICE);
- PyObject *stop;
- PyObject *start;
- PyObject *container;
- PyObject *res;
+ _PyStackRef stop;
+ _PyStackRef start;
+ _PyStackRef container;
+ _PyStackRef res;
stop = stack_pointer[-1];
start = stack_pointer[-2];
container = stack_pointer[-3];
- PyObject *slice = _PyBuildSlice_ConsumeRefs(start, stop);
+ PyObject *slice = _PyBuildSlice_ConsumeRefs(PyStackRef_AsPyObjectSteal(start),
+ PyStackRef_AsPyObjectSteal(stop));
+ PyObject *res_o;
// Can't use ERROR_IF() here, because we haven't
// DECREF'ed container yet, and we still own slice.
if (slice == NULL) {
- res = NULL;
+ res_o = NULL;
}
else {
- res = PyObject_GetItem(container, slice);
+ res_o = PyObject_GetItem(PyStackRef_AsPyObjectBorrow(container), slice);
Py_DECREF(slice);
}
- Py_DECREF(container);
- if (res == NULL) goto pop_3_error;
+ PyStackRef_CLOSE(container);
+ if (res_o == NULL) goto pop_3_error;
+ res = PyStackRef_FromPyObjectSteal(res_o);
stack_pointer[-3] = res;
stack_pointer += -2;
assert(WITHIN_STACK_BOUNDS());
PREDICTED(BINARY_SUBSCR);
_Py_CODEUNIT *this_instr = next_instr - 2;
(void)this_instr;
- PyObject *sub;
- PyObject *container;
- PyObject *res;
+ _PyStackRef sub;
+ _PyStackRef container;
+ _PyStackRef res;
// _SPECIALIZE_BINARY_SUBSCR
sub = stack_pointer[-1];
container = stack_pointer[-2];
}
// _BINARY_SUBSCR
{
- res = PyObject_GetItem(container, sub);
- Py_DECREF(container);
- Py_DECREF(sub);
- if (res == NULL) goto pop_2_error;
+ PyObject *container_o = PyStackRef_AsPyObjectBorrow(container);
+ PyObject *sub_o = PyStackRef_AsPyObjectBorrow(sub);
+ PyObject *res_o = PyObject_GetItem(container_o, sub_o);
+ PyStackRef_CLOSE(container);
+ PyStackRef_CLOSE(sub);
+ if (res_o == NULL) goto pop_2_error;
+ res = PyStackRef_FromPyObjectSteal(res_o);
}
stack_pointer[-2] = res;
stack_pointer += -1;
next_instr += 2;
INSTRUCTION_STATS(BINARY_SUBSCR_DICT);
static_assert(INLINE_CACHE_ENTRIES_BINARY_SUBSCR == 1, "incorrect cache size");
- PyObject *sub;
- PyObject *dict;
- PyObject *res;
+ _PyStackRef sub_st;
+ _PyStackRef dict_st;
+ _PyStackRef res;
/* Skip 1 cache entry */
- sub = stack_pointer[-1];
- dict = stack_pointer[-2];
+ sub_st = stack_pointer[-1];
+ dict_st = stack_pointer[-2];
+ PyObject *sub = PyStackRef_AsPyObjectBorrow(sub_st);
+ PyObject *dict = PyStackRef_AsPyObjectBorrow(dict_st);
DEOPT_IF(!PyDict_CheckExact(dict), BINARY_SUBSCR);
STAT_INC(BINARY_SUBSCR, hit);
- int rc = PyDict_GetItemRef(dict, sub, &res);
+ PyObject *res_o;
+ int rc = PyDict_GetItemRef(dict, sub, &res_o);
if (rc == 0) {
_PyErr_SetKeyError(sub);
}
- Py_DECREF(dict);
- Py_DECREF(sub);
+ PyStackRef_CLOSE(dict_st);
+ PyStackRef_CLOSE(sub_st);
if (rc <= 0) goto pop_2_error;
// not found or error
+ res = PyStackRef_FromPyObjectSteal(res_o);
stack_pointer[-2] = res;
stack_pointer += -1;
assert(WITHIN_STACK_BOUNDS());
next_instr += 2;
INSTRUCTION_STATS(BINARY_SUBSCR_GETITEM);
static_assert(INLINE_CACHE_ENTRIES_BINARY_SUBSCR == 1, "incorrect cache size");
- PyObject *sub;
- PyObject *container;
+ _PyStackRef sub_st;
+ _PyStackRef container_st;
/* Skip 1 cache entry */
- sub = stack_pointer[-1];
- container = stack_pointer[-2];
+ sub_st = stack_pointer[-1];
+ container_st = stack_pointer[-2];
+ PyObject *container = PyStackRef_AsPyObjectBorrow(container_st);
DEOPT_IF(tstate->interp->eval_frame, BINARY_SUBSCR);
PyTypeObject *tp = Py_TYPE(container);
DEOPT_IF(!PyType_HasFeature(tp, Py_TPFLAGS_HEAPTYPE), BINARY_SUBSCR);
Py_INCREF(getitem);
_PyInterpreterFrame *new_frame = _PyFrame_PushUnchecked(tstate, getitem, 2);
STACK_SHRINK(2);
- new_frame->localsplus[0] = container;
- new_frame->localsplus[1] = sub;
+ new_frame->localsplus[0] = container_st;
+ new_frame->localsplus[1] = sub_st;
frame->return_offset = (uint16_t)(next_instr - this_instr);
DISPATCH_INLINED(new_frame);
}
next_instr += 2;
INSTRUCTION_STATS(BINARY_SUBSCR_LIST_INT);
static_assert(INLINE_CACHE_ENTRIES_BINARY_SUBSCR == 1, "incorrect cache size");
- PyObject *sub;
- PyObject *list;
- PyObject *res;
+ _PyStackRef sub_st;
+ _PyStackRef list_st;
+ _PyStackRef res;
/* Skip 1 cache entry */
- sub = stack_pointer[-1];
- list = stack_pointer[-2];
+ sub_st = stack_pointer[-1];
+ list_st = stack_pointer[-2];
+ PyObject *sub = PyStackRef_AsPyObjectBorrow(sub_st);
+ PyObject *list = PyStackRef_AsPyObjectBorrow(list_st);
DEOPT_IF(!PyLong_CheckExact(sub), BINARY_SUBSCR);
DEOPT_IF(!PyList_CheckExact(list), BINARY_SUBSCR);
// Deopt unless 0 <= sub < PyList_Size(list)
Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0];
DEOPT_IF(index >= PyList_GET_SIZE(list), BINARY_SUBSCR);
STAT_INC(BINARY_SUBSCR, hit);
- res = PyList_GET_ITEM(list, index);
- assert(res != NULL);
- Py_INCREF(res);
+ PyObject *res_o = PyList_GET_ITEM(list, index);
+ assert(res_o != NULL);
+ Py_INCREF(res_o);
_Py_DECREF_SPECIALIZED(sub, (destructor)PyObject_Free);
- Py_DECREF(list);
+ PyStackRef_CLOSE(list_st);
+ res = PyStackRef_FromPyObjectSteal(res_o);
stack_pointer[-2] = res;
stack_pointer += -1;
assert(WITHIN_STACK_BOUNDS());
next_instr += 2;
INSTRUCTION_STATS(BINARY_SUBSCR_STR_INT);
static_assert(INLINE_CACHE_ENTRIES_BINARY_SUBSCR == 1, "incorrect cache size");
- PyObject *sub;
- PyObject *str;
- PyObject *res;
+ _PyStackRef sub_st;
+ _PyStackRef str_st;
+ _PyStackRef res;
/* Skip 1 cache entry */
- sub = stack_pointer[-1];
- str = stack_pointer[-2];
+ sub_st = stack_pointer[-1];
+ str_st = stack_pointer[-2];
+ PyObject *sub = PyStackRef_AsPyObjectBorrow(sub_st);
+ PyObject *str = PyStackRef_AsPyObjectBorrow(str_st);
DEOPT_IF(!PyLong_CheckExact(sub), BINARY_SUBSCR);
DEOPT_IF(!PyUnicode_CheckExact(str), BINARY_SUBSCR);
DEOPT_IF(!_PyLong_IsNonNegativeCompact((PyLongObject *)sub), BINARY_SUBSCR);
Py_UCS4 c = PyUnicode_READ_CHAR(str, index);
DEOPT_IF(Py_ARRAY_LENGTH(_Py_SINGLETON(strings).ascii) <= c, BINARY_SUBSCR);
STAT_INC(BINARY_SUBSCR, hit);
- res = (PyObject*)&_Py_SINGLETON(strings).ascii[c];
+ PyObject *res_o = (PyObject*)&_Py_SINGLETON(strings).ascii[c];
_Py_DECREF_SPECIALIZED(sub, (destructor)PyObject_Free);
- Py_DECREF(str);
+ PyStackRef_CLOSE(str_st);
+ res = PyStackRef_FromPyObjectSteal(res_o);
stack_pointer[-2] = res;
stack_pointer += -1;
assert(WITHIN_STACK_BOUNDS());
next_instr += 2;
INSTRUCTION_STATS(BINARY_SUBSCR_TUPLE_INT);
static_assert(INLINE_CACHE_ENTRIES_BINARY_SUBSCR == 1, "incorrect cache size");
- PyObject *sub;
- PyObject *tuple;
- PyObject *res;
+ _PyStackRef sub_st;
+ _PyStackRef tuple_st;
+ _PyStackRef res;
/* Skip 1 cache entry */
- sub = stack_pointer[-1];
- tuple = stack_pointer[-2];
+ sub_st = stack_pointer[-1];
+ tuple_st = stack_pointer[-2];
+ PyObject *sub = PyStackRef_AsPyObjectBorrow(sub_st);
+ PyObject *tuple = PyStackRef_AsPyObjectBorrow(tuple_st);
DEOPT_IF(!PyLong_CheckExact(sub), BINARY_SUBSCR);
DEOPT_IF(!PyTuple_CheckExact(tuple), BINARY_SUBSCR);
// Deopt unless 0 <= sub < PyTuple_Size(list)
Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0];
DEOPT_IF(index >= PyTuple_GET_SIZE(tuple), BINARY_SUBSCR);
STAT_INC(BINARY_SUBSCR, hit);
- res = PyTuple_GET_ITEM(tuple, index);
- assert(res != NULL);
- Py_INCREF(res);
+ PyObject *res_o = PyTuple_GET_ITEM(tuple, index);
+ assert(res_o != NULL);
+ Py_INCREF(res_o);
_Py_DECREF_SPECIALIZED(sub, (destructor)PyObject_Free);
- Py_DECREF(tuple);
+ PyStackRef_CLOSE(tuple_st);
+ res = PyStackRef_FromPyObjectSteal(res_o);
stack_pointer[-2] = res;
stack_pointer += -1;
assert(WITHIN_STACK_BOUNDS());
frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(BUILD_CONST_KEY_MAP);
- PyObject *keys;
- PyObject **values;
- PyObject *map;
+ _PyStackRef keys;
+ _PyStackRef *values;
+ _PyStackRef map;
keys = stack_pointer[-1];
values = &stack_pointer[-1 - oparg];
- assert(PyTuple_CheckExact(keys));
- assert(PyTuple_GET_SIZE(keys) == (Py_ssize_t)oparg);
- map = _PyDict_FromItems(
- &PyTuple_GET_ITEM(keys, 0), 1,
- values, 1, oparg);
+ PyObject *keys_o = PyStackRef_AsPyObjectBorrow(keys);
+ assert(PyTuple_CheckExact(keys_o));
+ assert(PyTuple_GET_SIZE(keys_o) == (Py_ssize_t)oparg);
+ STACKREFS_TO_PYOBJECTS(values, oparg, values_o);
+ if (CONVERSION_FAILED(values_o)) {
+ for (int _i = oparg; --_i >= 0;) {
+ PyStackRef_CLOSE(values[_i]);
+ }
+ PyStackRef_CLOSE(keys);
+ if (true) { stack_pointer += -1 - oparg; goto error; }
+ }
+ PyObject *map_o = _PyDict_FromItems(
+ &PyTuple_GET_ITEM(keys_o, 0), 1,
+ values_o, 1, oparg);
+ STACKREFS_TO_PYOBJECTS_CLEANUP(values_o);
for (int _i = oparg; --_i >= 0;) {
- Py_DECREF(values[_i]);
+ PyStackRef_CLOSE(values[_i]);
}
- Py_DECREF(keys);
- if (map == NULL) { stack_pointer += -1 - oparg; goto error; }
+ PyStackRef_CLOSE(keys);
+ if (map_o == NULL) { stack_pointer += -1 - oparg; goto error; }
+ map = PyStackRef_FromPyObjectSteal(map_o);
stack_pointer[-1 - oparg] = map;
stack_pointer += -oparg;
assert(WITHIN_STACK_BOUNDS());
frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(BUILD_LIST);
- PyObject **values;
- PyObject *list;
+ _PyStackRef *values;
+ _PyStackRef list;
values = &stack_pointer[-oparg];
- list = _PyList_FromArraySteal(values, oparg);
- if (list == NULL) { stack_pointer += -oparg; goto error; }
+ STACKREFS_TO_PYOBJECTS(values, oparg, values_o);
+ if (CONVERSION_FAILED(values_o)) {
+ for (int _i = oparg; --_i >= 0;) {
+ PyStackRef_CLOSE(values[_i]);
+ }
+ if (true) { stack_pointer += -oparg; goto error; }
+ }
+ PyObject *list_o = _PyList_FromArraySteal(values_o, oparg);
+ STACKREFS_TO_PYOBJECTS_CLEANUP(values_o);
+ if (list_o == NULL) { stack_pointer += -oparg; goto error; }
+ list = PyStackRef_FromPyObjectSteal(list_o);
stack_pointer[-oparg] = list;
stack_pointer += 1 - oparg;
assert(WITHIN_STACK_BOUNDS());
frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(BUILD_MAP);
- PyObject **values;
- PyObject *map;
+ _PyStackRef *values;
+ _PyStackRef map;
values = &stack_pointer[-oparg*2];
- map = _PyDict_FromItems(
- values, 2,
- values+1, 2,
- oparg);
+ STACKREFS_TO_PYOBJECTS(values, oparg*2, values_o);
+ if (CONVERSION_FAILED(values_o)) {
+ for (int _i = oparg*2; --_i >= 0;) {
+ PyStackRef_CLOSE(values[_i]);
+ }
+ if (true) { stack_pointer += -oparg*2; goto error; }
+ }
+ PyObject *map_o = _PyDict_FromItems(
+ values_o, 2,
+ values_o+1, 2,
+ oparg);
+ STACKREFS_TO_PYOBJECTS_CLEANUP(values_o);
for (int _i = oparg*2; --_i >= 0;) {
- Py_DECREF(values[_i]);
+ PyStackRef_CLOSE(values[_i]);
}
- if (map == NULL) { stack_pointer += -oparg*2; goto error; }
+ if (map_o == NULL) { stack_pointer += -oparg*2; goto error; }
+ map = PyStackRef_FromPyObjectSteal(map_o);
stack_pointer[-oparg*2] = map;
stack_pointer += 1 - oparg*2;
assert(WITHIN_STACK_BOUNDS());
frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(BUILD_SET);
- PyObject **values;
- PyObject *set;
+ _PyStackRef *values;
+ _PyStackRef set;
values = &stack_pointer[-oparg];
- set = PySet_New(NULL);
- if (set == NULL)
- goto error;
+ PyObject *set_o = PySet_New(NULL);
+ if (set_o == NULL) {
+ goto error;
+ }
int err = 0;
for (int i = 0; i < oparg; i++) {
- PyObject *item = values[i];
- if (err == 0)
- err = PySet_Add(set, item);
+ PyObject *item = PyStackRef_AsPyObjectSteal(values[i]);
+ if (err == 0) {
+ err = PySet_Add(set_o, item);
+ }
Py_DECREF(item);
}
if (err != 0) {
- Py_DECREF(set);
+ Py_DECREF(set_o);
if (true) { stack_pointer += -oparg; goto error; }
}
+ set = PyStackRef_FromPyObjectSteal(set_o);
stack_pointer[-oparg] = set;
stack_pointer += 1 - oparg;
assert(WITHIN_STACK_BOUNDS());
frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(BUILD_SLICE);
- PyObject *step = NULL;
- PyObject *stop;
- PyObject *start;
- PyObject *slice;
+ _PyStackRef step = PyStackRef_NULL;
+ _PyStackRef stop;
+ _PyStackRef start;
+ _PyStackRef slice;
if (oparg == 3) { step = stack_pointer[-((oparg == 3) ? 1 : 0)]; }
stop = stack_pointer[-1 - ((oparg == 3) ? 1 : 0)];
start = stack_pointer[-2 - ((oparg == 3) ? 1 : 0)];
- slice = PySlice_New(start, stop, step);
- Py_DECREF(start);
- Py_DECREF(stop);
- Py_XDECREF(step);
- if (slice == NULL) { stack_pointer += -2 - ((oparg == 3) ? 1 : 0); goto error; }
+ PyObject *start_o = PyStackRef_AsPyObjectBorrow(start);
+ PyObject *stop_o = PyStackRef_AsPyObjectBorrow(stop);
+ PyObject *step_o = PyStackRef_AsPyObjectBorrow(step);
+ PyObject *slice_o = PySlice_New(start_o, stop_o, step_o);
+ PyStackRef_CLOSE(start);
+ PyStackRef_CLOSE(stop);
+ PyStackRef_XCLOSE(step);
+ if (slice_o == NULL) { stack_pointer += -2 - ((oparg == 3) ? 1 : 0); goto error; }
+ slice = PyStackRef_FromPyObjectSteal(slice_o);
stack_pointer[-2 - ((oparg == 3) ? 1 : 0)] = slice;
stack_pointer += -1 - ((oparg == 3) ? 1 : 0);
assert(WITHIN_STACK_BOUNDS());
frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(BUILD_STRING);
- PyObject **pieces;
- PyObject *str;
+ _PyStackRef *pieces;
+ _PyStackRef str;
pieces = &stack_pointer[-oparg];
- str = _PyUnicode_JoinArray(&_Py_STR(empty), pieces, oparg);
+ STACKREFS_TO_PYOBJECTS(pieces, oparg, pieces_o);
+ if (CONVERSION_FAILED(pieces_o)) {
+ for (int _i = oparg; --_i >= 0;) {
+ PyStackRef_CLOSE(pieces[_i]);
+ }
+ if (true) { stack_pointer += -oparg; goto error; }
+ }
+ PyObject *str_o = _PyUnicode_JoinArray(&_Py_STR(empty), pieces_o, oparg);
+ STACKREFS_TO_PYOBJECTS_CLEANUP(pieces_o);
for (int _i = oparg; --_i >= 0;) {
- Py_DECREF(pieces[_i]);
+ PyStackRef_CLOSE(pieces[_i]);
}
- if (str == NULL) { stack_pointer += -oparg; goto error; }
+ if (str_o == NULL) { stack_pointer += -oparg; goto error; }
+ str = PyStackRef_FromPyObjectSteal(str_o);
stack_pointer[-oparg] = str;
stack_pointer += 1 - oparg;
assert(WITHIN_STACK_BOUNDS());
frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(BUILD_TUPLE);
- PyObject **values;
- PyObject *tup;
+ _PyStackRef *values;
+ _PyStackRef tup;
values = &stack_pointer[-oparg];
- tup = _PyTuple_FromArraySteal(values, oparg);
- if (tup == NULL) { stack_pointer += -oparg; goto error; }
+ STACKREFS_TO_PYOBJECTS(values, oparg, values_o);
+ if (CONVERSION_FAILED(values_o)) {
+ for (int _i = oparg; --_i >= 0;) {
+ PyStackRef_CLOSE(values[_i]);
+ }
+ if (true) { stack_pointer += -oparg; goto error; }
+ }
+ PyObject *tup_o = _PyTuple_FromArraySteal(values_o, oparg);
+ STACKREFS_TO_PYOBJECTS_CLEANUP(values_o);
+ if (tup_o == NULL) { stack_pointer += -oparg; goto error; }
+ tup = PyStackRef_FromPyObjectSteal(tup_o);
stack_pointer[-oparg] = tup;
stack_pointer += 1 - oparg;
assert(WITHIN_STACK_BOUNDS());
PREDICTED(CALL);
_Py_CODEUNIT *this_instr = next_instr - 4;
(void)this_instr;
- PyObject **args;
- PyObject *self_or_null;
- PyObject *callable;
- PyObject *res;
+ _PyStackRef *args;
+ _PyStackRef self_or_null;
+ _PyStackRef callable;
+ _PyStackRef res;
// _SPECIALIZE_CALL
args = &stack_pointer[-oparg];
self_or_null = stack_pointer[-1 - oparg];
#if ENABLE_SPECIALIZATION
if (ADAPTIVE_COUNTER_TRIGGERS(counter)) {
next_instr = this_instr;
- _Py_Specialize_Call(callable, next_instr, oparg + (self_or_null != NULL));
+ _Py_Specialize_Call(callable, next_instr, oparg + !PyStackRef_IsNull(self_or_null));
DISPATCH_SAME_OPARG();
}
STAT_INC(CALL, deferred);
/* Skip 2 cache entries */
// _CALL
{
+ PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable);
+ PyObject *self_or_null_o = PyStackRef_AsPyObjectBorrow(self_or_null);
// oparg counts all of the args, but *not* self:
int total_args = oparg;
- if (self_or_null != NULL) {
+ if (self_or_null_o != NULL) {
args--;
total_args++;
}
- else if (Py_TYPE(callable) == &PyMethod_Type) {
+ else if (Py_TYPE(callable_o) == &PyMethod_Type) {
args--;
total_args++;
- PyObject *self = ((PyMethodObject *)callable)->im_self;
- args[0] = Py_NewRef(self);
- PyObject *method = ((PyMethodObject *)callable)->im_func;
- args[-1] = Py_NewRef(method);
- Py_DECREF(callable);
- callable = method;
+ PyObject *self = ((PyMethodObject *)callable_o)->im_self;
+ args[0] = PyStackRef_FromPyObjectNew(self);
+ PyObject *method = ((PyMethodObject *)callable_o)->im_func;
+ args[-1] = PyStackRef_FromPyObjectNew(method);
+ PyStackRef_CLOSE(callable);
+ callable_o = method;
+ callable = args[-1];
}
// Check if the call can be inlined or not
- if (Py_TYPE(callable) == &PyFunction_Type &&
+ if (Py_TYPE(callable_o) == &PyFunction_Type &&
tstate->interp->eval_frame == NULL &&
- ((PyFunctionObject *)callable)->vectorcall == _PyFunction_Vectorcall)
+ ((PyFunctionObject *)callable_o)->vectorcall == _PyFunction_Vectorcall)
{
- int code_flags = ((PyCodeObject*)PyFunction_GET_CODE(callable))->co_flags;
- PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable));
+ int code_flags = ((PyCodeObject*)PyFunction_GET_CODE(callable_o))->co_flags;
+ PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable_o));
_PyInterpreterFrame *new_frame = _PyEvalFramePushAndInit(
- tstate, (PyFunctionObject *)callable, locals,
+ tstate, (PyFunctionObject *)PyStackRef_AsPyObjectSteal(callable), locals,
args, total_args, NULL
);
// Manipulate stack directly since we leave using DISPATCH_INLINED().
DISPATCH_INLINED(new_frame);
}
/* Callable is not a normal Python function */
- res = PyObject_Vectorcall(
- callable, args,
- total_args | PY_VECTORCALL_ARGUMENTS_OFFSET,
- NULL);
+ STACKREFS_TO_PYOBJECTS(args, total_args, args_o);
+ if (CONVERSION_FAILED(args_o)) {
+ PyStackRef_CLOSE(callable);
+ PyStackRef_CLOSE(self_or_null);
+ for (int _i = oparg; --_i >= 0;) {
+ PyStackRef_CLOSE(args[_i]);
+ }
+ if (true) { stack_pointer += -2 - oparg; goto error; }
+ }
+ PyObject *res_o = PyObject_Vectorcall(
+ callable_o, args_o,
+ total_args | PY_VECTORCALL_ARGUMENTS_OFFSET,
+ NULL);
+ STACKREFS_TO_PYOBJECTS_CLEANUP(args_o);
if (opcode == INSTRUMENTED_CALL) {
PyObject *arg = total_args == 0 ?
- &_PyInstrumentation_MISSING : args[0];
- if (res == NULL) {
+ &_PyInstrumentation_MISSING : PyStackRef_AsPyObjectBorrow(args[0]);
+ if (res_o == NULL) {
_Py_call_instrumentation_exc2(
tstate, PY_MONITORING_EVENT_C_RAISE,
- frame, this_instr, callable, arg);
+ frame, this_instr, callable_o, arg);
}
else {
int err = _Py_call_instrumentation_2args(
tstate, PY_MONITORING_EVENT_C_RETURN,
- frame, this_instr, callable, arg);
+ frame, this_instr, callable_o, arg);
if (err < 0) {
- Py_CLEAR(res);
+ Py_CLEAR(res_o);
}
}
}
- assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL));
- Py_DECREF(callable);
+ assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL));
+ PyStackRef_CLOSE(callable);
for (int i = 0; i < total_args; i++) {
- Py_DECREF(args[i]);
+ PyStackRef_CLOSE(args[i]);
}
- if (res == NULL) { stack_pointer += -2 - oparg; goto error; }
+ if (res_o == NULL) { stack_pointer += -2 - oparg; goto error; }
+ res = PyStackRef_FromPyObjectSteal(res_o);
}
// _CHECK_PERIODIC
{
next_instr += 4;
INSTRUCTION_STATS(CALL_ALLOC_AND_ENTER_INIT);
static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size");
- PyObject **args;
- PyObject *null;
- PyObject *callable;
+ _PyStackRef *args;
+ _PyStackRef null;
+ _PyStackRef callable;
/* Skip 1 cache entry */
/* Skip 2 cache entries */
args = &stack_pointer[-oparg];
null = stack_pointer[-1 - oparg];
callable = stack_pointer[-2 - oparg];
+ PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable);
/* This instruction does the following:
* 1. Creates the object (by calling ``object.__new__``)
* 2. Pushes a shim frame to the frame stack (to cleanup after ``__init__``)
* 3. Pushes the frame for ``__init__`` to the frame stack
* */
_PyCallCache *cache = (_PyCallCache *)&this_instr[1];
- DEOPT_IF(null != NULL, CALL);
- DEOPT_IF(!PyType_Check(callable), CALL);
- PyTypeObject *tp = (PyTypeObject *)callable;
+ DEOPT_IF(!PyStackRef_IsNull(null), CALL);
+ DEOPT_IF(!PyType_Check(callable_o), CALL);
+ PyTypeObject *tp = (PyTypeObject *)callable_o;
DEOPT_IF(tp->tp_version_tag != read_u32(cache->func_version), CALL);
assert(tp->tp_flags & Py_TPFLAGS_INLINE_VALUES);
- PyHeapTypeObject *cls = (PyHeapTypeObject *)callable;
+ PyHeapTypeObject *cls = (PyHeapTypeObject *)callable_o;
PyFunctionObject *init = (PyFunctionObject *)cls->_spec_cache.init;
PyCodeObject *code = (PyCodeObject *)init->func_code;
DEOPT_IF(code->co_argcount != oparg+1, CALL);
if (self == NULL) {
goto error;
}
- Py_DECREF(tp);
+ PyStackRef_CLOSE(callable);
_PyInterpreterFrame *shim = _PyFrame_PushTrampolineUnchecked(
tstate, (PyCodeObject *)&_Py_InitCleanup, 1);
assert(_PyCode_CODE((PyCodeObject *)shim->f_executable)[0].op.code == EXIT_INIT_CHECK);
/* Push self onto stack of shim */
Py_INCREF(self);
- shim->localsplus[0] = self;
+ shim->localsplus[0] = PyStackRef_FromPyObjectSteal(self);
Py_INCREF(init);
_PyInterpreterFrame *init_frame = _PyFrame_PushUnchecked(tstate, init, oparg+1);
/* Copy self followed by args to __init__ frame */
- init_frame->localsplus[0] = self;
+ init_frame->localsplus[0] = PyStackRef_FromPyObjectSteal(self);
for (int i = 0; i < oparg; i++) {
init_frame->localsplus[i+1] = args[i];
}
next_instr += 4;
INSTRUCTION_STATS(CALL_BOUND_METHOD_EXACT_ARGS);
static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size");
- PyObject *null;
- PyObject *callable;
- PyObject *func;
- PyObject *self;
- PyObject *self_or_null;
- PyObject **args;
+ _PyStackRef null;
+ _PyStackRef callable;
+ _PyStackRef func;
+ _PyStackRef self;
+ _PyStackRef self_or_null;
+ _PyStackRef *args;
_PyInterpreterFrame *new_frame;
/* Skip 1 cache entry */
// _CHECK_PEP_523
null = stack_pointer[-1 - oparg];
callable = stack_pointer[-2 - oparg];
{
- DEOPT_IF(null != NULL, CALL);
- DEOPT_IF(Py_TYPE(callable) != &PyMethod_Type, CALL);
+ DEOPT_IF(!PyStackRef_IsNull(null), CALL);
+ DEOPT_IF(Py_TYPE(PyStackRef_AsPyObjectBorrow(callable)) != &PyMethod_Type, CALL);
}
// _INIT_CALL_BOUND_METHOD_EXACT_ARGS
{
+ PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable);
STAT_INC(CALL, hit);
- self = Py_NewRef(((PyMethodObject *)callable)->im_self);
- stack_pointer[-1 - oparg] = self; // Patch stack as it is used by _INIT_CALL_PY_EXACT_ARGS
- func = Py_NewRef(((PyMethodObject *)callable)->im_func);
- stack_pointer[-2 - oparg] = func; // This is used by CALL, upon deoptimization
- Py_DECREF(callable);
+ stack_pointer[-1 - oparg] = PyStackRef_FromPyObjectNew(((PyMethodObject *)callable_o)->im_self); // Patch stack as it is used by _INIT_CALL_PY_EXACT_ARGS
+ stack_pointer[-2 - oparg] = PyStackRef_FromPyObjectNew(((PyMethodObject *)callable_o)->im_func); // This is used by CALL, upon deoptimization
+ self = stack_pointer[-1 - oparg];
+ func = stack_pointer[-2 - oparg];
+ PyStackRef_CLOSE(callable);
+ // self may be unused in tier 1, so silence warnings.
+ (void)self;
}
// _CHECK_FUNCTION_VERSION
callable = func;
{
uint32_t func_version = read_u32(&this_instr[2].cache);
- DEOPT_IF(!PyFunction_Check(callable), CALL);
- PyFunctionObject *func = (PyFunctionObject *)callable;
+ PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable);
+ DEOPT_IF(!PyFunction_Check(callable_o), CALL);
+ PyFunctionObject *func = (PyFunctionObject *)callable_o;
DEOPT_IF(func->func_version != func_version, CALL);
}
// _CHECK_FUNCTION_EXACT_ARGS
self_or_null = stack_pointer[-1 - oparg];
{
- assert(PyFunction_Check(callable));
- PyFunctionObject *func = (PyFunctionObject *)callable;
+ PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable);
+ assert(PyFunction_Check(callable_o));
+ PyFunctionObject *func = (PyFunctionObject *)callable_o;
PyCodeObject *code = (PyCodeObject *)func->func_code;
- DEOPT_IF(code->co_argcount != oparg + (self_or_null != NULL), CALL);
+ DEOPT_IF(code->co_argcount != oparg + (!PyStackRef_IsNull(self_or_null)), CALL);
}
// _CHECK_STACK_SPACE
{
- PyFunctionObject *func = (PyFunctionObject *)callable;
+ PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable);
+ PyFunctionObject *func = (PyFunctionObject *)callable_o;
PyCodeObject *code = (PyCodeObject *)func->func_code;
DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize), CALL);
DEOPT_IF(tstate->py_recursion_remaining <= 1, CALL);
args = &stack_pointer[-oparg];
self_or_null = stack_pointer[-1 - oparg];
{
- int has_self = (self_or_null != NULL);
+ PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable);
+ int has_self = !PyStackRef_IsNull(self_or_null);
STAT_INC(CALL, hit);
- PyFunctionObject *func = (PyFunctionObject *)callable;
+ PyFunctionObject *func = (PyFunctionObject *)callable_o;
new_frame = _PyFrame_PushUnchecked(tstate, func, oparg + has_self);
- PyObject **first_non_self_local = new_frame->localsplus + has_self;
+ _PyStackRef *first_non_self_local = new_frame->localsplus + has_self;
new_frame->localsplus[0] = self_or_null;
for (int i = 0; i < oparg; i++) {
first_non_self_local[i] = args[i];
next_instr += 4;
INSTRUCTION_STATS(CALL_BOUND_METHOD_GENERAL);
static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size");
- PyObject *null;
- PyObject *callable;
- PyObject *method;
- PyObject *self;
- PyObject **args;
- PyObject *self_or_null;
+ _PyStackRef null;
+ _PyStackRef callable;
+ _PyStackRef method;
+ _PyStackRef self;
+ _PyStackRef *args;
+ _PyStackRef self_or_null;
_PyInterpreterFrame *new_frame;
/* Skip 1 cache entry */
// _CHECK_PEP_523
callable = stack_pointer[-2 - oparg];
{
uint32_t func_version = read_u32(&this_instr[2].cache);
- DEOPT_IF(Py_TYPE(callable) != &PyMethod_Type, CALL);
- PyObject *func = ((PyMethodObject *)callable)->im_func;
+ PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable);
+ DEOPT_IF(Py_TYPE(callable_o) != &PyMethod_Type, CALL);
+ PyObject *func = ((PyMethodObject *)callable_o)->im_func;
DEOPT_IF(!PyFunction_Check(func), CALL);
DEOPT_IF(((PyFunctionObject *)func)->func_version != func_version, CALL);
- DEOPT_IF(null != NULL, CALL);
+ DEOPT_IF(!PyStackRef_IsNull(null), CALL);
}
// _EXPAND_METHOD
{
- assert(null == NULL);
- assert(Py_TYPE(callable) == &PyMethod_Type);
- self = ((PyMethodObject *)callable)->im_self;
- Py_INCREF(self);
+ PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable);
+ assert(PyStackRef_IsNull(null));
+ assert(Py_TYPE(callable_o) == &PyMethod_Type);
+ self = PyStackRef_FromPyObjectNew(((PyMethodObject *)callable_o)->im_self);
stack_pointer[-1 - oparg] = self; // Patch stack as it is used by _PY_FRAME_GENERAL
- method = ((PyMethodObject *)callable)->im_func;
- assert(PyFunction_Check(method));
- Py_INCREF(method);
- Py_DECREF(callable);
+ method = PyStackRef_FromPyObjectNew(((PyMethodObject *)callable_o)->im_func);
+ assert(PyFunction_Check(PyStackRef_AsPyObjectBorrow(method)));
+ PyStackRef_CLOSE(callable);
}
// _PY_FRAME_GENERAL
args = &stack_pointer[-oparg];
self_or_null = self;
callable = method;
{
+ PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable);
+ PyObject *self_or_null_o = PyStackRef_AsPyObjectBorrow(self_or_null);
// oparg counts all of the args, but *not* self:
int total_args = oparg;
- if (self_or_null != NULL) {
+ if (self_or_null_o != NULL) {
args--;
total_args++;
}
- assert(Py_TYPE(callable) == &PyFunction_Type);
- int code_flags = ((PyCodeObject*)PyFunction_GET_CODE(callable))->co_flags;
- PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable));
+ assert(Py_TYPE(callable_o) == &PyFunction_Type);
+ int code_flags = ((PyCodeObject*)PyFunction_GET_CODE(callable_o))->co_flags;
+ PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable_o));
new_frame = _PyEvalFramePushAndInit(
- tstate, (PyFunctionObject *)callable, locals,
+ tstate, (PyFunctionObject *)PyStackRef_AsPyObjectSteal(callable), locals,
args, total_args, NULL
);
// The frame has stolen all the arguments from the stack,
next_instr += 4;
INSTRUCTION_STATS(CALL_BUILTIN_CLASS);
static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size");
- PyObject **args;
- PyObject *self_or_null;
- PyObject *callable;
- PyObject *res;
+ _PyStackRef *args;
+ _PyStackRef self_or_null;
+ _PyStackRef callable;
+ _PyStackRef res;
/* Skip 1 cache entry */
/* Skip 2 cache entries */
// _CALL_BUILTIN_CLASS
self_or_null = stack_pointer[-1 - oparg];
callable = stack_pointer[-2 - oparg];
{
+ PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable);
int total_args = oparg;
- if (self_or_null != NULL) {
+ if (!PyStackRef_IsNull(self_or_null)) {
args--;
total_args++;
}
- DEOPT_IF(!PyType_Check(callable), CALL);
- PyTypeObject *tp = (PyTypeObject *)callable;
+ DEOPT_IF(!PyType_Check(callable_o), CALL);
+ PyTypeObject *tp = (PyTypeObject *)callable_o;
DEOPT_IF(tp->tp_vectorcall == NULL, CALL);
STAT_INC(CALL, hit);
- res = tp->tp_vectorcall((PyObject *)tp, args, total_args, NULL);
+ STACKREFS_TO_PYOBJECTS(args, total_args, args_o);
+ if (CONVERSION_FAILED(args_o)) {
+ PyStackRef_CLOSE(callable);
+ PyStackRef_CLOSE(self_or_null);
+ for (int _i = oparg; --_i >= 0;) {
+ PyStackRef_CLOSE(args[_i]);
+ }
+ if (true) { stack_pointer += -2 - oparg; goto error; }
+ }
+ PyObject *res_o = tp->tp_vectorcall((PyObject *)tp, args_o, total_args, NULL);
+ STACKREFS_TO_PYOBJECTS_CLEANUP(args_o);
/* Free the arguments. */
for (int i = 0; i < total_args; i++) {
- Py_DECREF(args[i]);
+ PyStackRef_CLOSE(args[i]);
}
- Py_DECREF(tp);
- if (res == NULL) { stack_pointer += -2 - oparg; goto error; }
+ PyStackRef_CLOSE(callable);
+ if (res_o == NULL) { stack_pointer += -2 - oparg; goto error; }
+ res = PyStackRef_FromPyObjectSteal(res_o);
}
// _CHECK_PERIODIC
{
next_instr += 4;
INSTRUCTION_STATS(CALL_BUILTIN_FAST);
static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size");
- PyObject **args;
- PyObject *self_or_null;
- PyObject *callable;
- PyObject *res;
+ _PyStackRef *args;
+ _PyStackRef self_or_null;
+ _PyStackRef callable;
+ _PyStackRef res;
/* Skip 1 cache entry */
/* Skip 2 cache entries */
// _CALL_BUILTIN_FAST
callable = stack_pointer[-2 - oparg];
{
/* Builtin METH_FASTCALL functions, without keywords */
+ PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable);
int total_args = oparg;
- if (self_or_null != NULL) {
+ if (!PyStackRef_IsNull(self_or_null)) {
args--;
total_args++;
}
- DEOPT_IF(!PyCFunction_CheckExact(callable), CALL);
- DEOPT_IF(PyCFunction_GET_FLAGS(callable) != METH_FASTCALL, CALL);
+ DEOPT_IF(!PyCFunction_CheckExact(callable_o), CALL);
+ DEOPT_IF(PyCFunction_GET_FLAGS(callable_o) != METH_FASTCALL, CALL);
STAT_INC(CALL, hit);
- PyCFunction cfunc = PyCFunction_GET_FUNCTION(callable);
+ PyCFunction cfunc = PyCFunction_GET_FUNCTION(callable_o);
/* res = func(self, args, nargs) */
- res = ((PyCFunctionFast)(void(*)(void))cfunc)(
- PyCFunction_GET_SELF(callable),
- args,
+ STACKREFS_TO_PYOBJECTS(args, total_args, args_o);
+ if (CONVERSION_FAILED(args_o)) {
+ PyStackRef_CLOSE(callable);
+ PyStackRef_CLOSE(self_or_null);
+ for (int _i = oparg; --_i >= 0;) {
+ PyStackRef_CLOSE(args[_i]);
+ }
+ if (true) { stack_pointer += -2 - oparg; goto error; }
+ }
+ PyObject *res_o = ((PyCFunctionFast)(void(*)(void))cfunc)(
+ PyCFunction_GET_SELF(callable_o),
+ args_o,
total_args);
- assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL));
+ STACKREFS_TO_PYOBJECTS_CLEANUP(args_o);
+ assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL));
/* Free the arguments. */
for (int i = 0; i < total_args; i++) {
- Py_DECREF(args[i]);
+ PyStackRef_CLOSE(args[i]);
}
- Py_DECREF(callable);
- if (res == NULL) { stack_pointer += -2 - oparg; goto error; }
+ PyStackRef_CLOSE(callable);
+ if (res_o == NULL) { stack_pointer += -2 - oparg; goto error; }
+ res = PyStackRef_FromPyObjectSteal(res_o);
}
// _CHECK_PERIODIC
{
next_instr += 4;
INSTRUCTION_STATS(CALL_BUILTIN_FAST_WITH_KEYWORDS);
static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size");
- PyObject **args;
- PyObject *self_or_null;
- PyObject *callable;
- PyObject *res;
+ _PyStackRef *args;
+ _PyStackRef self_or_null;
+ _PyStackRef callable;
+ _PyStackRef res;
/* Skip 1 cache entry */
/* Skip 2 cache entries */
// _CALL_BUILTIN_FAST_WITH_KEYWORDS
callable = stack_pointer[-2 - oparg];
{
/* Builtin METH_FASTCALL | METH_KEYWORDS functions */
+ PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable);
int total_args = oparg;
- if (self_or_null != NULL) {
+ if (!PyStackRef_IsNull(self_or_null)) {
args--;
total_args++;
}
- DEOPT_IF(!PyCFunction_CheckExact(callable), CALL);
- DEOPT_IF(PyCFunction_GET_FLAGS(callable) != (METH_FASTCALL | METH_KEYWORDS), CALL);
+ DEOPT_IF(!PyCFunction_CheckExact(callable_o), CALL);
+ DEOPT_IF(PyCFunction_GET_FLAGS(callable_o) != (METH_FASTCALL | METH_KEYWORDS), CALL);
STAT_INC(CALL, hit);
/* res = func(self, args, nargs, kwnames) */
PyCFunctionFastWithKeywords cfunc =
(PyCFunctionFastWithKeywords)(void(*)(void))
- PyCFunction_GET_FUNCTION(callable);
- res = cfunc(PyCFunction_GET_SELF(callable), args, total_args, NULL);
- assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL));
+ PyCFunction_GET_FUNCTION(callable_o);
+ STACKREFS_TO_PYOBJECTS(args, total_args, args_o);
+ if (CONVERSION_FAILED(args_o)) {
+ PyStackRef_CLOSE(callable);
+ PyStackRef_CLOSE(self_or_null);
+ for (int _i = oparg; --_i >= 0;) {
+ PyStackRef_CLOSE(args[_i]);
+ }
+ if (true) { stack_pointer += -2 - oparg; goto error; }
+ }
+ PyObject *res_o = cfunc(PyCFunction_GET_SELF(callable_o), args_o, total_args, NULL);
+ STACKREFS_TO_PYOBJECTS_CLEANUP(args_o);
+ assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL));
/* Free the arguments. */
for (int i = 0; i < total_args; i++) {
- Py_DECREF(args[i]);
+ PyStackRef_CLOSE(args[i]);
}
- Py_DECREF(callable);
- if (res == NULL) { stack_pointer += -2 - oparg; goto error; }
+ PyStackRef_CLOSE(callable);
+ if (res_o == NULL) { stack_pointer += -2 - oparg; goto error; }
+ res = PyStackRef_FromPyObjectSteal(res_o);
}
// _CHECK_PERIODIC
{
next_instr += 4;
INSTRUCTION_STATS(CALL_BUILTIN_O);
static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size");
- PyObject **args;
- PyObject *self_or_null;
- PyObject *callable;
- PyObject *res;
+ _PyStackRef *args;
+ _PyStackRef self_or_null;
+ _PyStackRef callable;
+ _PyStackRef res;
/* Skip 1 cache entry */
/* Skip 2 cache entries */
// _CALL_BUILTIN_O
callable = stack_pointer[-2 - oparg];
{
/* Builtin METH_O functions */
+ PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable);
int total_args = oparg;
- if (self_or_null != NULL) {
+ if (!PyStackRef_IsNull(self_or_null)) {
args--;
total_args++;
}
DEOPT_IF(total_args != 1, CALL);
- DEOPT_IF(!PyCFunction_CheckExact(callable), CALL);
- DEOPT_IF(PyCFunction_GET_FLAGS(callable) != METH_O, CALL);
+ DEOPT_IF(!PyCFunction_CheckExact(callable_o), CALL);
+ DEOPT_IF(PyCFunction_GET_FLAGS(callable_o) != METH_O, CALL);
// CPython promises to check all non-vectorcall function calls.
DEOPT_IF(tstate->c_recursion_remaining <= 0, CALL);
STAT_INC(CALL, hit);
- PyCFunction cfunc = PyCFunction_GET_FUNCTION(callable);
- PyObject *arg = args[0];
+ PyCFunction cfunc = PyCFunction_GET_FUNCTION(callable_o);
+ _PyStackRef arg = args[0];
_Py_EnterRecursiveCallTstateUnchecked(tstate);
- res = _PyCFunction_TrampolineCall(cfunc, PyCFunction_GET_SELF(callable), arg);
+ PyObject *res_o = _PyCFunction_TrampolineCall(cfunc, PyCFunction_GET_SELF(callable_o), PyStackRef_AsPyObjectBorrow(arg));
_Py_LeaveRecursiveCallTstate(tstate);
- assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL));
- Py_DECREF(arg);
- Py_DECREF(callable);
- if (res == NULL) { stack_pointer += -2 - oparg; goto error; }
+ assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL));
+ PyStackRef_CLOSE(arg);
+ PyStackRef_CLOSE(callable);
+ if (res_o == NULL) { stack_pointer += -2 - oparg; goto error; }
+ res = PyStackRef_FromPyObjectSteal(res_o);
}
// _CHECK_PERIODIC
{
PREDICTED(CALL_FUNCTION_EX);
_Py_CODEUNIT *this_instr = next_instr - 1;
(void)this_instr;
- PyObject *kwargs = NULL;
- PyObject *callargs;
- PyObject *func;
- PyObject *result;
- if (oparg & 1) { kwargs = stack_pointer[-(oparg & 1)]; }
- callargs = stack_pointer[-1 - (oparg & 1)];
- func = stack_pointer[-3 - (oparg & 1)];
+ _PyStackRef kwargs_st = PyStackRef_NULL;
+ _PyStackRef callargs_st;
+ _PyStackRef func_st;
+ _PyStackRef result;
+ if (oparg & 1) { kwargs_st = stack_pointer[-(oparg & 1)]; }
+ callargs_st = stack_pointer[-1 - (oparg & 1)];
+ func_st = stack_pointer[-3 - (oparg & 1)];
+ PyObject *func = PyStackRef_AsPyObjectBorrow(func_st);
+ PyObject *callargs = PyStackRef_AsPyObjectBorrow(callargs_st);
+ PyObject *kwargs = PyStackRef_AsPyObjectBorrow(kwargs_st);
// DICT_MERGE is called before this opcode if there are kwargs.
// It converts all dict subtypes in kwargs into regular dicts.
assert(kwargs == NULL || PyDict_CheckExact(kwargs));
if (tuple == NULL) {
goto error;
}
- Py_SETREF(callargs, tuple);
+ PyStackRef_CLOSE(callargs_st);
+ callargs_st = PyStackRef_FromPyObjectSteal(tuple);
+ callargs = tuple;
}
assert(PyTuple_CheckExact(callargs));
EVAL_CALL_STAT_INC_IF_FUNCTION(EVAL_CALL_FUNCTION_EX, func);
tstate, PY_MONITORING_EVENT_CALL,
frame, this_instr, func, arg);
if (err) goto error;
- result = PyObject_Call(func, callargs, kwargs);
+ result = PyStackRef_FromPyObjectSteal(PyObject_Call(func, callargs, kwargs));
if (!PyFunction_Check(func) && !PyMethod_Check(func)) {
- if (result == NULL) {
+ if (PyStackRef_IsNull(result)) {
_Py_call_instrumentation_exc2(
tstate, PY_MONITORING_EVENT_C_RAISE,
frame, this_instr, func, arg);
tstate, PY_MONITORING_EVENT_C_RETURN,
frame, this_instr, func, arg);
if (err < 0) {
- Py_CLEAR(result);
+ PyStackRef_CLEAR(result);
}
}
}
int code_flags = ((PyCodeObject *)PyFunction_GET_CODE(func))->co_flags;
PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(func));
_PyInterpreterFrame *new_frame = _PyEvalFramePushAndInit_Ex(tstate,
- (PyFunctionObject *)func, locals,
+ (PyFunctionObject *)PyStackRef_AsPyObjectSteal(func_st), locals,
nargs, callargs, kwargs);
// Need to manually shrink the stack since we exit with DISPATCH_INLINED.
STACK_SHRINK(oparg + 3);
frame->return_offset = 1;
DISPATCH_INLINED(new_frame);
}
- result = PyObject_Call(func, callargs, kwargs);
+ result = PyStackRef_FromPyObjectSteal(PyObject_Call(func, callargs, kwargs));
}
- Py_DECREF(func);
- Py_DECREF(callargs);
- Py_XDECREF(kwargs);
- assert(PEEK(2 + (oparg & 1)) == NULL);
- if (result == NULL) { stack_pointer += -3 - (oparg & 1); goto error; }
+ PyStackRef_CLOSE(func_st);
+ PyStackRef_CLOSE(callargs_st);
+ PyStackRef_XCLOSE(kwargs_st);
+ assert(PyStackRef_AsPyObjectBorrow(PEEK(2 + (oparg & 1))) == NULL);
+ if (PyStackRef_IsNull(result)) { stack_pointer += -3 - (oparg & 1); goto error; }
stack_pointer[-3 - (oparg & 1)] = result;
stack_pointer += -2 - (oparg & 1);
assert(WITHIN_STACK_BOUNDS());
frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(CALL_INTRINSIC_1);
- PyObject *value;
- PyObject *res;
+ _PyStackRef value;
+ _PyStackRef res;
value = stack_pointer[-1];
assert(oparg <= MAX_INTRINSIC_1);
- res = _PyIntrinsics_UnaryFunctions[oparg].func(tstate, value);
- Py_DECREF(value);
- if (res == NULL) goto pop_1_error;
+ PyObject *res_o = _PyIntrinsics_UnaryFunctions[oparg].func(tstate, PyStackRef_AsPyObjectBorrow(value));
+ PyStackRef_CLOSE(value);
+ if (res_o == NULL) goto pop_1_error;
+ res = PyStackRef_FromPyObjectSteal(res_o);
stack_pointer[-1] = res;
DISPATCH();
}
frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(CALL_INTRINSIC_2);
- PyObject *value1;
- PyObject *value2;
- PyObject *res;
- value1 = stack_pointer[-1];
- value2 = stack_pointer[-2];
+ _PyStackRef value1_st;
+ _PyStackRef value2_st;
+ _PyStackRef res;
+ value1_st = stack_pointer[-1];
+ value2_st = stack_pointer[-2];
assert(oparg <= MAX_INTRINSIC_2);
- res = _PyIntrinsics_BinaryFunctions[oparg].func(tstate, value2, value1);
- Py_DECREF(value2);
- Py_DECREF(value1);
- if (res == NULL) goto pop_2_error;
+ PyObject *value1 = PyStackRef_AsPyObjectBorrow(value1_st);
+ PyObject *value2 = PyStackRef_AsPyObjectBorrow(value2_st);
+ PyObject *res_o = _PyIntrinsics_BinaryFunctions[oparg].func(tstate, value2, value1);
+ PyStackRef_CLOSE(value2_st);
+ PyStackRef_CLOSE(value1_st);
+ if (res_o == NULL) goto pop_2_error;
+ res = PyStackRef_FromPyObjectSteal(res_o);
stack_pointer[-2] = res;
stack_pointer += -1;
assert(WITHIN_STACK_BOUNDS());
next_instr += 4;
INSTRUCTION_STATS(CALL_ISINSTANCE);
static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size");
- PyObject **args;
- PyObject *self_or_null;
- PyObject *callable;
- PyObject *res;
+ _PyStackRef *args;
+ _PyStackRef self_or_null;
+ _PyStackRef callable;
+ _PyStackRef res;
/* Skip 1 cache entry */
/* Skip 2 cache entries */
args = &stack_pointer[-oparg];
self_or_null = stack_pointer[-1 - oparg];
callable = stack_pointer[-2 - oparg];
/* isinstance(o, o2) */
+ PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable);
int total_args = oparg;
- if (self_or_null != NULL) {
+ if (!PyStackRef_IsNull(self_or_null)) {
args--;
total_args++;
}
DEOPT_IF(total_args != 2, CALL);
PyInterpreterState *interp = tstate->interp;
- DEOPT_IF(callable != interp->callable_cache.isinstance, CALL);
+ DEOPT_IF(callable_o != interp->callable_cache.isinstance, CALL);
STAT_INC(CALL, hit);
- PyObject *cls = args[1];
- PyObject *inst = args[0];
- int retval = PyObject_IsInstance(inst, cls);
+ _PyStackRef cls_stackref = args[1];
+ _PyStackRef inst_stackref = args[0];
+ int retval = PyObject_IsInstance(PyStackRef_AsPyObjectBorrow(inst_stackref), PyStackRef_AsPyObjectBorrow(cls_stackref));
if (retval < 0) {
goto error;
}
- res = PyBool_FromLong(retval);
- assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL));
- if (res == NULL) {
- GOTO_ERROR(error);
- }
- Py_DECREF(inst);
- Py_DECREF(cls);
- Py_DECREF(callable);
+ res = retval ? PyStackRef_True : PyStackRef_False;
+ assert((!PyStackRef_IsNull(res)) ^ (_PyErr_Occurred(tstate) != NULL));
+ PyStackRef_CLOSE(inst_stackref);
+ PyStackRef_CLOSE(cls_stackref);
+ PyStackRef_CLOSE(callable);
stack_pointer[-2 - oparg] = res;
stack_pointer += -1 - oparg;
assert(WITHIN_STACK_BOUNDS());
PREDICTED(CALL_KW);
_Py_CODEUNIT *this_instr = next_instr - 1;
(void)this_instr;
- PyObject *kwnames;
- PyObject **args;
- PyObject *self_or_null;
- PyObject *callable;
- PyObject *res;
+ _PyStackRef kwnames;
+ _PyStackRef *args;
+ _PyStackRef self_or_null;
+ _PyStackRef callable;
+ _PyStackRef res;
kwnames = stack_pointer[-1];
args = &stack_pointer[-1 - oparg];
self_or_null = stack_pointer[-2 - oparg];
callable = stack_pointer[-3 - oparg];
+ PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable);
+ PyObject *self_or_null_o = PyStackRef_AsPyObjectBorrow(self_or_null);
+ PyObject *kwnames_o = PyStackRef_AsPyObjectBorrow(kwnames);
// oparg counts all of the args, but *not* self:
int total_args = oparg;
- if (self_or_null != NULL) {
+ if (self_or_null_o != NULL) {
args--;
total_args++;
}
- if (self_or_null == NULL && Py_TYPE(callable) == &PyMethod_Type) {
+ if (self_or_null_o == NULL && Py_TYPE(callable_o) == &PyMethod_Type) {
args--;
total_args++;
- PyObject *self = ((PyMethodObject *)callable)->im_self;
- args[0] = Py_NewRef(self);
- PyObject *method = ((PyMethodObject *)callable)->im_func;
- args[-1] = Py_NewRef(method);
- Py_DECREF(callable);
- callable = method;
- }
- int positional_args = total_args - (int)PyTuple_GET_SIZE(kwnames);
+ PyObject *self = ((PyMethodObject *)callable_o)->im_self;
+ args[0] = PyStackRef_FromPyObjectNew(self);
+ PyObject *method = ((PyMethodObject *)callable_o)->im_func;
+ args[-1] = PyStackRef_FromPyObjectNew(method);
+ PyStackRef_CLOSE(callable);
+ callable_o = method;
+ callable = args[-1];
+ }
+ int positional_args = total_args - (int)PyTuple_GET_SIZE(kwnames_o);
// Check if the call can be inlined or not
- if (Py_TYPE(callable) == &PyFunction_Type &&
+ if (Py_TYPE(callable_o) == &PyFunction_Type &&
tstate->interp->eval_frame == NULL &&
- ((PyFunctionObject *)callable)->vectorcall == _PyFunction_Vectorcall)
+ ((PyFunctionObject *)callable_o)->vectorcall == _PyFunction_Vectorcall)
{
- int code_flags = ((PyCodeObject*)PyFunction_GET_CODE(callable))->co_flags;
- PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable));
+ int code_flags = ((PyCodeObject*)PyFunction_GET_CODE(callable_o))->co_flags;
+ PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable_o));
_PyInterpreterFrame *new_frame = _PyEvalFramePushAndInit(
- tstate, (PyFunctionObject *)callable, locals,
- args, positional_args, kwnames
+ tstate, (PyFunctionObject *)PyStackRef_AsPyObjectSteal(callable), locals,
+ args, positional_args, kwnames_o
);
- Py_DECREF(kwnames);
+ PyStackRef_CLOSE(kwnames);
// Manipulate stack directly since we leave using DISPATCH_INLINED().
STACK_SHRINK(oparg + 3);
// The frame has stolen all the arguments from the stack,
DISPATCH_INLINED(new_frame);
}
/* Callable is not a normal Python function */
- res = PyObject_Vectorcall(
- callable, args,
- positional_args | PY_VECTORCALL_ARGUMENTS_OFFSET,
- kwnames);
+ STACKREFS_TO_PYOBJECTS(args, total_args, args_o);
+ if (CONVERSION_FAILED(args_o)) {
+ PyStackRef_CLOSE(callable);
+ PyStackRef_CLOSE(self_or_null);
+ for (int _i = oparg; --_i >= 0;) {
+ PyStackRef_CLOSE(args[_i]);
+ }
+ PyStackRef_CLOSE(kwnames);
+ if (true) { stack_pointer += -3 - oparg; goto error; }
+ }
+ PyObject *res_o = PyObject_Vectorcall(
+ callable_o, args_o,
+ positional_args | PY_VECTORCALL_ARGUMENTS_OFFSET,
+ kwnames_o);
+ STACKREFS_TO_PYOBJECTS_CLEANUP(args_o);
if (opcode == INSTRUMENTED_CALL_KW) {
PyObject *arg = total_args == 0 ?
- &_PyInstrumentation_MISSING : args[0];
- if (res == NULL) {
+ &_PyInstrumentation_MISSING : PyStackRef_AsPyObjectBorrow(args[0]);
+ if (res_o == NULL) {
_Py_call_instrumentation_exc2(
tstate, PY_MONITORING_EVENT_C_RAISE,
- frame, this_instr, callable, arg);
+ frame, this_instr, callable_o, arg);
}
else {
int err = _Py_call_instrumentation_2args(
tstate, PY_MONITORING_EVENT_C_RETURN,
- frame, this_instr, callable, arg);
+ frame, this_instr, callable_o, arg);
if (err < 0) {
- Py_CLEAR(res);
+ Py_CLEAR(res_o);
}
}
}
- Py_DECREF(kwnames);
- assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL));
- Py_DECREF(callable);
+ PyStackRef_CLOSE(kwnames);
+ assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL));
+ PyStackRef_CLOSE(callable);
for (int i = 0; i < total_args; i++) {
- Py_DECREF(args[i]);
+ PyStackRef_CLOSE(args[i]);
}
- if (res == NULL) { stack_pointer += -3 - oparg; goto error; }
+ if (res_o == NULL) { stack_pointer += -3 - oparg; goto error; }
+ res = PyStackRef_FromPyObjectSteal(res_o);
stack_pointer[-3 - oparg] = res;
stack_pointer += -2 - oparg;
assert(WITHIN_STACK_BOUNDS());
next_instr += 4;
INSTRUCTION_STATS(CALL_LEN);
static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size");
- PyObject **args;
- PyObject *self_or_null;
- PyObject *callable;
- PyObject *res;
+ _PyStackRef *args;
+ _PyStackRef self_or_null;
+ _PyStackRef callable;
+ _PyStackRef res;
/* Skip 1 cache entry */
/* Skip 2 cache entries */
args = &stack_pointer[-oparg];
self_or_null = stack_pointer[-1 - oparg];
callable = stack_pointer[-2 - oparg];
/* len(o) */
+ PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable);
int total_args = oparg;
- if (self_or_null != NULL) {
+ if (!PyStackRef_IsNull(self_or_null)) {
args--;
total_args++;
}
DEOPT_IF(total_args != 1, CALL);
PyInterpreterState *interp = tstate->interp;
- DEOPT_IF(callable != interp->callable_cache.len, CALL);
+ DEOPT_IF(callable_o != interp->callable_cache.len, CALL);
STAT_INC(CALL, hit);
- PyObject *arg = args[0];
+ _PyStackRef arg_stackref = args[0];
+ PyObject *arg = PyStackRef_AsPyObjectBorrow(arg_stackref);
Py_ssize_t len_i = PyObject_Length(arg);
if (len_i < 0) {
goto error;
}
- res = PyLong_FromSsize_t(len_i);
- assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL));
- if (res == NULL) {
+ PyObject *res_o = PyLong_FromSsize_t(len_i);
+ assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL));
+ if (res_o == NULL) {
GOTO_ERROR(error);
}
- Py_DECREF(callable);
- Py_DECREF(arg);
+ PyStackRef_CLOSE(callable);
+ PyStackRef_CLOSE(arg_stackref);
+ res = PyStackRef_FromPyObjectSteal(res_o);
stack_pointer[-2 - oparg] = res;
stack_pointer += -1 - oparg;
assert(WITHIN_STACK_BOUNDS());
next_instr += 4;
INSTRUCTION_STATS(CALL_LIST_APPEND);
static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size");
- PyObject *arg;
- PyObject *self;
- PyObject *callable;
+ _PyStackRef arg;
+ _PyStackRef self;
+ _PyStackRef callable;
/* Skip 1 cache entry */
/* Skip 2 cache entries */
arg = stack_pointer[-1];
self = stack_pointer[-2];
callable = stack_pointer[-3];
assert(oparg == 1);
+ PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable);
+ PyObject *self_o = PyStackRef_AsPyObjectBorrow(self);
PyInterpreterState *interp = tstate->interp;
- DEOPT_IF(callable != interp->callable_cache.list_append, CALL);
- assert(self != NULL);
- DEOPT_IF(!PyList_Check(self), CALL);
+ DEOPT_IF(callable_o != interp->callable_cache.list_append, CALL);
+ assert(self_o != NULL);
+ DEOPT_IF(!PyList_Check(self_o), CALL);
STAT_INC(CALL, hit);
- if (_PyList_AppendTakeRef((PyListObject *)self, arg) < 0) {
+ if (_PyList_AppendTakeRef((PyListObject *)self_o, PyStackRef_AsPyObjectSteal(arg)) < 0) {
goto pop_1_error; // Since arg is DECREF'ed already
}
- Py_DECREF(self);
- Py_DECREF(callable);
+ PyStackRef_CLOSE(self);
+ PyStackRef_CLOSE(callable);
STACK_SHRINK(3);
// Skip POP_TOP
assert(next_instr->op.code == POP_TOP);
next_instr += 4;
INSTRUCTION_STATS(CALL_METHOD_DESCRIPTOR_FAST);
static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size");
- PyObject **args;
- PyObject *self_or_null;
- PyObject *callable;
- PyObject *res;
+ _PyStackRef *args;
+ _PyStackRef self_or_null;
+ _PyStackRef callable;
+ _PyStackRef res;
/* Skip 1 cache entry */
/* Skip 2 cache entries */
// _CALL_METHOD_DESCRIPTOR_FAST
self_or_null = stack_pointer[-1 - oparg];
callable = stack_pointer[-2 - oparg];
{
+ PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable);
int total_args = oparg;
- if (self_or_null != NULL) {
+ if (!PyStackRef_IsNull(self_or_null)) {
args--;
total_args++;
}
- PyMethodDescrObject *method = (PyMethodDescrObject *)callable;
+ PyMethodDescrObject *method = (PyMethodDescrObject *)callable_o;
/* Builtin METH_FASTCALL methods, without keywords */
DEOPT_IF(!Py_IS_TYPE(method, &PyMethodDescr_Type), CALL);
PyMethodDef *meth = method->d_method;
DEOPT_IF(meth->ml_flags != METH_FASTCALL, CALL);
- PyObject *self = args[0];
+ PyObject *self = PyStackRef_AsPyObjectBorrow(args[0]);
DEOPT_IF(!Py_IS_TYPE(self, method->d_common.d_type), CALL);
STAT_INC(CALL, hit);
PyCFunctionFast cfunc =
(PyCFunctionFast)(void(*)(void))meth->ml_meth;
int nargs = total_args - 1;
- res = cfunc(self, args + 1, nargs);
- assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL));
+ STACKREFS_TO_PYOBJECTS(args, nargs, args_o);
+ if (CONVERSION_FAILED(args_o)) {
+ PyStackRef_CLOSE(callable);
+ PyStackRef_CLOSE(self_or_null);
+ for (int _i = oparg; --_i >= 0;) {
+ PyStackRef_CLOSE(args[_i]);
+ }
+ if (true) { stack_pointer += -2 - oparg; goto error; }
+ }
+ PyObject *res_o = cfunc(self, (args_o + 1), nargs);
+ STACKREFS_TO_PYOBJECTS_CLEANUP(args_o);
+ assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL));
/* Clear the stack of the arguments. */
for (int i = 0; i < total_args; i++) {
- Py_DECREF(args[i]);
+ PyStackRef_CLOSE(args[i]);
}
- Py_DECREF(callable);
- if (res == NULL) { stack_pointer += -2 - oparg; goto error; }
+ PyStackRef_CLOSE(callable);
+ if (res_o == NULL) { stack_pointer += -2 - oparg; goto error; }
+ res = PyStackRef_FromPyObjectSteal(res_o);
}
// _CHECK_PERIODIC
{
next_instr += 4;
INSTRUCTION_STATS(CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS);
static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size");
- PyObject **args;
- PyObject *self_or_null;
- PyObject *callable;
- PyObject *res;
+ _PyStackRef *args;
+ _PyStackRef self_or_null;
+ _PyStackRef callable;
+ _PyStackRef res;
/* Skip 1 cache entry */
/* Skip 2 cache entries */
// _CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS
self_or_null = stack_pointer[-1 - oparg];
callable = stack_pointer[-2 - oparg];
{
+ PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable);
int total_args = oparg;
- if (self_or_null != NULL) {
+ if (!PyStackRef_IsNull(self_or_null)) {
args--;
total_args++;
}
- PyMethodDescrObject *method = (PyMethodDescrObject *)callable;
+ PyMethodDescrObject *method = (PyMethodDescrObject *)callable_o;
DEOPT_IF(!Py_IS_TYPE(method, &PyMethodDescr_Type), CALL);
PyMethodDef *meth = method->d_method;
DEOPT_IF(meth->ml_flags != (METH_FASTCALL|METH_KEYWORDS), CALL);
PyTypeObject *d_type = method->d_common.d_type;
- PyObject *self = args[0];
+ PyObject *self = PyStackRef_AsPyObjectBorrow(args[0]);
DEOPT_IF(!Py_IS_TYPE(self, d_type), CALL);
STAT_INC(CALL, hit);
int nargs = total_args - 1;
PyCFunctionFastWithKeywords cfunc =
(PyCFunctionFastWithKeywords)(void(*)(void))meth->ml_meth;
- res = cfunc(self, args + 1, nargs, NULL);
- assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL));
+ STACKREFS_TO_PYOBJECTS(args, nargs, args_o);
+ if (CONVERSION_FAILED(args_o)) {
+ PyStackRef_CLOSE(callable);
+ PyStackRef_CLOSE(self_or_null);
+ for (int _i = oparg; --_i >= 0;) {
+ PyStackRef_CLOSE(args[_i]);
+ }
+ if (true) { stack_pointer += -2 - oparg; goto error; }
+ }
+ PyObject *res_o = cfunc(self, (args_o + 1), nargs, NULL);
+ STACKREFS_TO_PYOBJECTS_CLEANUP(args_o);
+ assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL));
/* Free the arguments. */
for (int i = 0; i < total_args; i++) {
- Py_DECREF(args[i]);
+ PyStackRef_CLOSE(args[i]);
}
- Py_DECREF(callable);
- if (res == NULL) { stack_pointer += -2 - oparg; goto error; }
+ PyStackRef_CLOSE(callable);
+ if (res_o == NULL) { stack_pointer += -2 - oparg; goto error; }
+ res = PyStackRef_FromPyObjectSteal(res_o);
}
// _CHECK_PERIODIC
{
next_instr += 4;
INSTRUCTION_STATS(CALL_METHOD_DESCRIPTOR_NOARGS);
static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size");
- PyObject **args;
- PyObject *self_or_null;
- PyObject *callable;
- PyObject *res;
+ _PyStackRef *args;
+ _PyStackRef self_or_null;
+ _PyStackRef callable;
+ _PyStackRef res;
/* Skip 1 cache entry */
/* Skip 2 cache entries */
// _CALL_METHOD_DESCRIPTOR_NOARGS
callable = stack_pointer[-2 - oparg];
{
assert(oparg == 0 || oparg == 1);
+ PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable);
int total_args = oparg;
- if (self_or_null != NULL) {
+ if (!PyStackRef_IsNull(self_or_null)) {
args--;
total_args++;
}
DEOPT_IF(total_args != 1, CALL);
- PyMethodDescrObject *method = (PyMethodDescrObject *)callable;
+ PyMethodDescrObject *method = (PyMethodDescrObject *)callable_o;
DEOPT_IF(!Py_IS_TYPE(method, &PyMethodDescr_Type), CALL);
PyMethodDef *meth = method->d_method;
- PyObject *self = args[0];
+ _PyStackRef self_stackref = args[0];
+ PyObject *self = PyStackRef_AsPyObjectBorrow(self_stackref);
DEOPT_IF(!Py_IS_TYPE(self, method->d_common.d_type), CALL);
DEOPT_IF(meth->ml_flags != METH_NOARGS, CALL);
// CPython promises to check all non-vectorcall function calls.
STAT_INC(CALL, hit);
PyCFunction cfunc = meth->ml_meth;
_Py_EnterRecursiveCallTstateUnchecked(tstate);
- res = _PyCFunction_TrampolineCall(cfunc, self, NULL);
+ PyObject *res_o = _PyCFunction_TrampolineCall(cfunc, self, NULL);
_Py_LeaveRecursiveCallTstate(tstate);
- assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL));
- Py_DECREF(self);
- Py_DECREF(callable);
- if (res == NULL) { stack_pointer += -2 - oparg; goto error; }
+ assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL));
+ PyStackRef_CLOSE(self_stackref);
+ PyStackRef_CLOSE(callable);
+ if (res_o == NULL) { stack_pointer += -2 - oparg; goto error; }
+ res = PyStackRef_FromPyObjectSteal(res_o);
}
// _CHECK_PERIODIC
{
next_instr += 4;
INSTRUCTION_STATS(CALL_METHOD_DESCRIPTOR_O);
static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size");
- PyObject **args;
- PyObject *self_or_null;
- PyObject *callable;
- PyObject *res;
+ _PyStackRef *args;
+ _PyStackRef self_or_null;
+ _PyStackRef callable;
+ _PyStackRef res;
/* Skip 1 cache entry */
/* Skip 2 cache entries */
// _CALL_METHOD_DESCRIPTOR_O
self_or_null = stack_pointer[-1 - oparg];
callable = stack_pointer[-2 - oparg];
{
+ PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable);
int total_args = oparg;
- if (self_or_null != NULL) {
+ if (!PyStackRef_IsNull(self_or_null)) {
args--;
total_args++;
}
- PyMethodDescrObject *method = (PyMethodDescrObject *)callable;
+ PyMethodDescrObject *method = (PyMethodDescrObject *)callable_o;
DEOPT_IF(total_args != 2, CALL);
DEOPT_IF(!Py_IS_TYPE(method, &PyMethodDescr_Type), CALL);
PyMethodDef *meth = method->d_method;
DEOPT_IF(meth->ml_flags != METH_O, CALL);
// CPython promises to check all non-vectorcall function calls.
DEOPT_IF(tstate->c_recursion_remaining <= 0, CALL);
- PyObject *arg = args[1];
- PyObject *self = args[0];
- DEOPT_IF(!Py_IS_TYPE(self, method->d_common.d_type), CALL);
+ _PyStackRef arg_stackref = args[1];
+ _PyStackRef self_stackref = args[0];
+ DEOPT_IF(!Py_IS_TYPE(PyStackRef_AsPyObjectBorrow(self_stackref),
+ method->d_common.d_type), CALL);
STAT_INC(CALL, hit);
PyCFunction cfunc = meth->ml_meth;
_Py_EnterRecursiveCallTstateUnchecked(tstate);
- res = _PyCFunction_TrampolineCall(cfunc, self, arg);
+ PyObject *res_o = _PyCFunction_TrampolineCall(cfunc,
+ PyStackRef_AsPyObjectBorrow(self_stackref),
+ PyStackRef_AsPyObjectBorrow(arg_stackref));
_Py_LeaveRecursiveCallTstate(tstate);
- assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL));
- Py_DECREF(self);
- Py_DECREF(arg);
- Py_DECREF(callable);
- if (res == NULL) { stack_pointer += -2 - oparg; goto error; }
+ assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL));
+ PyStackRef_CLOSE(self_stackref);
+ PyStackRef_CLOSE(arg_stackref);
+ PyStackRef_CLOSE(callable);
+ if (res_o == NULL) { stack_pointer += -2 - oparg; goto error; }
+ res = PyStackRef_FromPyObjectSteal(res_o);
}
// _CHECK_PERIODIC
{
next_instr += 4;
INSTRUCTION_STATS(CALL_NON_PY_GENERAL);
static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size");
- PyObject *callable;
- PyObject **args;
- PyObject *self_or_null;
- PyObject *res;
+ _PyStackRef callable;
+ _PyStackRef *args;
+ _PyStackRef self_or_null;
+ _PyStackRef res;
/* Skip 1 cache entry */
/* Skip 2 cache entries */
// _CHECK_IS_NOT_PY_CALLABLE
callable = stack_pointer[-2 - oparg];
{
- DEOPT_IF(PyFunction_Check(callable), CALL);
- DEOPT_IF(Py_TYPE(callable) == &PyMethod_Type, CALL);
+ PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable);
+ DEOPT_IF(PyFunction_Check(callable_o), CALL);
+ DEOPT_IF(Py_TYPE(callable_o) == &PyMethod_Type, CALL);
}
// _CALL_NON_PY_GENERAL
args = &stack_pointer[-oparg];
#if TIER_ONE
assert(opcode != INSTRUMENTED_CALL);
#endif
+ PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable);
+ PyObject *self_or_null_o = PyStackRef_AsPyObjectBorrow(self_or_null);
int total_args = oparg;
- if (self_or_null != NULL) {
+ if (self_or_null_o != NULL) {
args--;
total_args++;
}
/* Callable is not a normal Python function */
- res = PyObject_Vectorcall(
- callable, args,
- total_args | PY_VECTORCALL_ARGUMENTS_OFFSET,
- NULL);
- assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL));
- Py_DECREF(callable);
+ STACKREFS_TO_PYOBJECTS(args, total_args, args_o);
+ if (CONVERSION_FAILED(args_o)) {
+ PyStackRef_CLOSE(callable);
+ PyStackRef_CLOSE(self_or_null);
+ for (int _i = oparg; --_i >= 0;) {
+ PyStackRef_CLOSE(args[_i]);
+ }
+ if (true) { stack_pointer += -2 - oparg; goto error; }
+ }
+ PyObject *res_o = PyObject_Vectorcall(
+ callable_o, args_o,
+ total_args | PY_VECTORCALL_ARGUMENTS_OFFSET,
+ NULL);
+ STACKREFS_TO_PYOBJECTS_CLEANUP(args_o);
+ assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL));
+ PyStackRef_CLOSE(callable);
for (int i = 0; i < total_args; i++) {
- Py_DECREF(args[i]);
+ PyStackRef_CLOSE(args[i]);
}
- if (res == NULL) { stack_pointer += -2 - oparg; goto error; }
+ if (res_o == NULL) { stack_pointer += -2 - oparg; goto error; }
+ res = PyStackRef_FromPyObjectSteal(res_o);
}
// _CHECK_PERIODIC
{
next_instr += 4;
INSTRUCTION_STATS(CALL_PY_EXACT_ARGS);
static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size");
- PyObject *callable;
- PyObject *self_or_null;
- PyObject **args;
+ _PyStackRef callable;
+ _PyStackRef self_or_null;
+ _PyStackRef *args;
_PyInterpreterFrame *new_frame;
/* Skip 1 cache entry */
// _CHECK_PEP_523
callable = stack_pointer[-2 - oparg];
{
uint32_t func_version = read_u32(&this_instr[2].cache);
- DEOPT_IF(!PyFunction_Check(callable), CALL);
- PyFunctionObject *func = (PyFunctionObject *)callable;
+ PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable);
+ DEOPT_IF(!PyFunction_Check(callable_o), CALL);
+ PyFunctionObject *func = (PyFunctionObject *)callable_o;
DEOPT_IF(func->func_version != func_version, CALL);
}
// _CHECK_FUNCTION_EXACT_ARGS
self_or_null = stack_pointer[-1 - oparg];
{
- assert(PyFunction_Check(callable));
- PyFunctionObject *func = (PyFunctionObject *)callable;
+ PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable);
+ assert(PyFunction_Check(callable_o));
+ PyFunctionObject *func = (PyFunctionObject *)callable_o;
PyCodeObject *code = (PyCodeObject *)func->func_code;
- DEOPT_IF(code->co_argcount != oparg + (self_or_null != NULL), CALL);
+ DEOPT_IF(code->co_argcount != oparg + (!PyStackRef_IsNull(self_or_null)), CALL);
}
// _CHECK_STACK_SPACE
{
- PyFunctionObject *func = (PyFunctionObject *)callable;
+ PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable);
+ PyFunctionObject *func = (PyFunctionObject *)callable_o;
PyCodeObject *code = (PyCodeObject *)func->func_code;
DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize), CALL);
DEOPT_IF(tstate->py_recursion_remaining <= 1, CALL);
args = &stack_pointer[-oparg];
self_or_null = stack_pointer[-1 - oparg];
{
- int has_self = (self_or_null != NULL);
+ PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable);
+ int has_self = !PyStackRef_IsNull(self_or_null);
STAT_INC(CALL, hit);
- PyFunctionObject *func = (PyFunctionObject *)callable;
+ PyFunctionObject *func = (PyFunctionObject *)callable_o;
new_frame = _PyFrame_PushUnchecked(tstate, func, oparg + has_self);
- PyObject **first_non_self_local = new_frame->localsplus + has_self;
+ _PyStackRef *first_non_self_local = new_frame->localsplus + has_self;
new_frame->localsplus[0] = self_or_null;
for (int i = 0; i < oparg; i++) {
first_non_self_local[i] = args[i];
next_instr += 4;
INSTRUCTION_STATS(CALL_PY_GENERAL);
static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size");
- PyObject *callable;
- PyObject **args;
- PyObject *self_or_null;
+ _PyStackRef callable;
+ _PyStackRef *args;
+ _PyStackRef self_or_null;
_PyInterpreterFrame *new_frame;
/* Skip 1 cache entry */
// _CHECK_PEP_523
callable = stack_pointer[-2 - oparg];
{
uint32_t func_version = read_u32(&this_instr[2].cache);
- DEOPT_IF(!PyFunction_Check(callable), CALL);
- PyFunctionObject *func = (PyFunctionObject *)callable;
+ PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable);
+ DEOPT_IF(!PyFunction_Check(callable_o), CALL);
+ PyFunctionObject *func = (PyFunctionObject *)callable_o;
DEOPT_IF(func->func_version != func_version, CALL);
}
// _PY_FRAME_GENERAL
args = &stack_pointer[-oparg];
self_or_null = stack_pointer[-1 - oparg];
{
+ PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable);
+ PyObject *self_or_null_o = PyStackRef_AsPyObjectBorrow(self_or_null);
// oparg counts all of the args, but *not* self:
int total_args = oparg;
- if (self_or_null != NULL) {
+ if (self_or_null_o != NULL) {
args--;
total_args++;
}
- assert(Py_TYPE(callable) == &PyFunction_Type);
- int code_flags = ((PyCodeObject*)PyFunction_GET_CODE(callable))->co_flags;
- PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable));
+ assert(Py_TYPE(callable_o) == &PyFunction_Type);
+ int code_flags = ((PyCodeObject*)PyFunction_GET_CODE(callable_o))->co_flags;
+ PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable_o));
new_frame = _PyEvalFramePushAndInit(
- tstate, (PyFunctionObject *)callable, locals,
+ tstate, (PyFunctionObject *)PyStackRef_AsPyObjectSteal(callable), locals,
args, total_args, NULL
);
// The frame has stolen all the arguments from the stack,
next_instr += 4;
INSTRUCTION_STATS(CALL_STR_1);
static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size");
- PyObject *arg;
- PyObject *null;
- PyObject *callable;
- PyObject *res;
+ _PyStackRef arg;
+ _PyStackRef null;
+ _PyStackRef callable;
+ _PyStackRef res;
/* Skip 1 cache entry */
/* Skip 2 cache entries */
// _CALL_STR_1
null = stack_pointer[-2];
callable = stack_pointer[-3];
{
+ PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable);
+ PyObject *arg_o = PyStackRef_AsPyObjectBorrow(arg);
assert(oparg == 1);
- DEOPT_IF(null != NULL, CALL);
- DEOPT_IF(callable != (PyObject *)&PyUnicode_Type, CALL);
+ DEOPT_IF(!PyStackRef_IsNull(null), CALL);
+ DEOPT_IF(callable_o != (PyObject *)&PyUnicode_Type, CALL);
STAT_INC(CALL, hit);
- res = PyObject_Str(arg);
- Py_DECREF(arg);
- if (res == NULL) goto pop_3_error;
+ res = PyStackRef_FromPyObjectSteal(PyObject_Str(arg_o));
+ PyStackRef_CLOSE(arg);
+ if (PyStackRef_IsNull(res)) goto pop_3_error;
}
// _CHECK_PERIODIC
{
next_instr += 4;
INSTRUCTION_STATS(CALL_TUPLE_1);
static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size");
- PyObject *arg;
- PyObject *null;
- PyObject *callable;
- PyObject *res;
+ _PyStackRef arg;
+ _PyStackRef null;
+ _PyStackRef callable;
+ _PyStackRef res;
/* Skip 1 cache entry */
/* Skip 2 cache entries */
// _CALL_TUPLE_1
null = stack_pointer[-2];
callable = stack_pointer[-3];
{
+ PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable);
+ PyObject *arg_o = PyStackRef_AsPyObjectBorrow(arg);
assert(oparg == 1);
- DEOPT_IF(null != NULL, CALL);
- DEOPT_IF(callable != (PyObject *)&PyTuple_Type, CALL);
+ DEOPT_IF(!PyStackRef_IsNull(null), CALL);
+ DEOPT_IF(callable_o != (PyObject *)&PyTuple_Type, CALL);
STAT_INC(CALL, hit);
- res = PySequence_Tuple(arg);
- Py_DECREF(arg);
- if (res == NULL) goto pop_3_error;
+ res = PyStackRef_FromPyObjectSteal(PySequence_Tuple(arg_o));
+ PyStackRef_CLOSE(arg);
+ if (PyStackRef_IsNull(res)) goto pop_3_error;
}
// _CHECK_PERIODIC
{
next_instr += 4;
INSTRUCTION_STATS(CALL_TYPE_1);
static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size");
- PyObject *arg;
- PyObject *null;
- PyObject *callable;
- PyObject *res;
+ _PyStackRef arg;
+ _PyStackRef null;
+ _PyStackRef callable;
+ _PyStackRef res;
/* Skip 1 cache entry */
/* Skip 2 cache entries */
arg = stack_pointer[-1];
null = stack_pointer[-2];
callable = stack_pointer[-3];
+ PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable);
+ PyObject *arg_o = PyStackRef_AsPyObjectBorrow(arg);
assert(oparg == 1);
- DEOPT_IF(null != NULL, CALL);
- DEOPT_IF(callable != (PyObject *)&PyType_Type, CALL);
+ DEOPT_IF(!PyStackRef_IsNull(null), CALL);
+ DEOPT_IF(callable_o != (PyObject *)&PyType_Type, CALL);
STAT_INC(CALL, hit);
- res = Py_NewRef(Py_TYPE(arg));
- Py_DECREF(arg);
+ res = PyStackRef_FromPyObjectSteal(Py_NewRef(Py_TYPE(arg_o)));
+ PyStackRef_CLOSE(arg);
stack_pointer[-3] = res;
stack_pointer += -2;
assert(WITHIN_STACK_BOUNDS());
frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(CHECK_EG_MATCH);
- PyObject *match_type;
- PyObject *exc_value;
- PyObject *rest;
- PyObject *match;
- match_type = stack_pointer[-1];
- exc_value = stack_pointer[-2];
+ _PyStackRef match_type_st;
+ _PyStackRef exc_value_st;
+ _PyStackRef rest;
+ _PyStackRef match;
+ match_type_st = stack_pointer[-1];
+ exc_value_st = stack_pointer[-2];
+ PyObject *exc_value = PyStackRef_AsPyObjectBorrow(exc_value_st);
+ PyObject *match_type = PyStackRef_AsPyObjectBorrow(match_type_st);
if (_PyEval_CheckExceptStarTypeValid(tstate, match_type) < 0) {
- Py_DECREF(exc_value);
- Py_DECREF(match_type);
+ PyStackRef_CLOSE(exc_value_st);
+ PyStackRef_CLOSE(match_type_st);
if (true) goto pop_2_error;
}
- match = NULL;
- rest = NULL;
+ PyObject *match_o = NULL;
+ PyObject *rest_o = NULL;
int res = _PyEval_ExceptionGroupMatch(exc_value, match_type,
- &match, &rest);
- Py_DECREF(exc_value);
- Py_DECREF(match_type);
+ &match_o, &rest_o);
+ PyStackRef_CLOSE(exc_value_st);
+ PyStackRef_CLOSE(match_type_st);
if (res < 0) goto pop_2_error;
- assert((match == NULL) == (rest == NULL));
- if (match == NULL) goto pop_2_error;
- if (!Py_IsNone(match)) {
- PyErr_SetHandledException(match);
+ assert((match_o == NULL) == (rest_o == NULL));
+ if (match_o == NULL) goto pop_2_error;
+ if (!Py_IsNone(match_o)) {
+ PyErr_SetHandledException(match_o);
}
+ rest = PyStackRef_FromPyObjectSteal(rest_o);
+ match = PyStackRef_FromPyObjectSteal(match_o);
stack_pointer[-2] = rest;
stack_pointer[-1] = match;
DISPATCH();
frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(CHECK_EXC_MATCH);
- PyObject *right;
- PyObject *left;
- PyObject *b;
+ _PyStackRef right;
+ _PyStackRef left;
+ _PyStackRef b;
right = stack_pointer[-1];
left = stack_pointer[-2];
- assert(PyExceptionInstance_Check(left));
- if (_PyEval_CheckExceptTypeValid(tstate, right) < 0) {
- Py_DECREF(right);
+ PyObject *left_o = PyStackRef_AsPyObjectBorrow(left);
+ PyObject *right_o = PyStackRef_AsPyObjectBorrow(right);
+ assert(PyExceptionInstance_Check(left_o));
+ if (_PyEval_CheckExceptTypeValid(tstate, right_o) < 0) {
+ PyStackRef_CLOSE(right);
if (true) goto pop_1_error;
}
- int res = PyErr_GivenExceptionMatches(left, right);
- Py_DECREF(right);
- b = res ? Py_True : Py_False;
+ int res = PyErr_GivenExceptionMatches(left_o, right_o);
+ PyStackRef_CLOSE(right);
+ b = res ? PyStackRef_True : PyStackRef_False;
stack_pointer[-1] = b;
DISPATCH();
}
(void)this_instr;
next_instr += 1;
INSTRUCTION_STATS(CLEANUP_THROW);
- PyObject *exc_value;
- PyObject *last_sent_val;
- PyObject *sub_iter;
- PyObject *none;
- PyObject *value;
- exc_value = stack_pointer[-1];
- last_sent_val = stack_pointer[-2];
- sub_iter = stack_pointer[-3];
+ _PyStackRef exc_value_st;
+ _PyStackRef last_sent_val_st;
+ _PyStackRef sub_iter_st;
+ _PyStackRef none;
+ _PyStackRef value;
+ exc_value_st = stack_pointer[-1];
+ last_sent_val_st = stack_pointer[-2];
+ sub_iter_st = stack_pointer[-3];
+ PyObject *exc_value = PyStackRef_AsPyObjectBorrow(exc_value_st);
assert(throwflag);
assert(exc_value && PyExceptionInstance_Check(exc_value));
if (PyErr_GivenExceptionMatches(exc_value, PyExc_StopIteration)) {
- value = Py_NewRef(((PyStopIterationObject *)exc_value)->value);
- Py_DECREF(sub_iter);
- Py_DECREF(last_sent_val);
- Py_DECREF(exc_value);
- none = Py_None;
+ value = PyStackRef_FromPyObjectNew(((PyStopIterationObject *)exc_value)->value);
+ PyStackRef_CLOSE(sub_iter_st);
+ PyStackRef_CLOSE(last_sent_val_st);
+ PyStackRef_CLOSE(exc_value_st);
+ none = PyStackRef_None;
}
else {
_PyErr_SetRaisedException(tstate, Py_NewRef(exc_value));
PREDICTED(COMPARE_OP);
_Py_CODEUNIT *this_instr = next_instr - 2;
(void)this_instr;
- PyObject *right;
- PyObject *left;
- PyObject *res;
+ _PyStackRef right;
+ _PyStackRef left;
+ _PyStackRef res;
// _SPECIALIZE_COMPARE_OP
right = stack_pointer[-1];
left = stack_pointer[-2];
}
// _COMPARE_OP
{
+ PyObject *left_o = PyStackRef_AsPyObjectBorrow(left);
+ PyObject *right_o = PyStackRef_AsPyObjectBorrow(right);
assert((oparg >> 5) <= Py_GE);
- res = PyObject_RichCompare(left, right, oparg >> 5);
- Py_DECREF(left);
- Py_DECREF(right);
- if (res == NULL) goto pop_2_error;
+ PyObject *res_o = PyObject_RichCompare(left_o, right_o, oparg >> 5);
+ PyStackRef_CLOSE(left);
+ PyStackRef_CLOSE(right);
+ if (res_o == NULL) goto pop_2_error;
if (oparg & 16) {
- int res_bool = PyObject_IsTrue(res);
- Py_DECREF(res);
+ int res_bool = PyObject_IsTrue(res_o);
+ Py_DECREF(res_o);
if (res_bool < 0) goto pop_2_error;
- res = res_bool ? Py_True : Py_False;
+ res = res_bool ? PyStackRef_True : PyStackRef_False;
+ }
+ else {
+ res = PyStackRef_FromPyObjectSteal(res_o);
}
}
stack_pointer[-2] = res;
next_instr += 2;
INSTRUCTION_STATS(COMPARE_OP_FLOAT);
static_assert(INLINE_CACHE_ENTRIES_COMPARE_OP == 1, "incorrect cache size");
- PyObject *right;
- PyObject *left;
- PyObject *res;
+ _PyStackRef right;
+ _PyStackRef left;
+ _PyStackRef res;
// _GUARD_BOTH_FLOAT
right = stack_pointer[-1];
left = stack_pointer[-2];
{
- DEOPT_IF(!PyFloat_CheckExact(left), COMPARE_OP);
- DEOPT_IF(!PyFloat_CheckExact(right), COMPARE_OP);
+ PyObject *left_o = PyStackRef_AsPyObjectBorrow(left);
+ PyObject *right_o = PyStackRef_AsPyObjectBorrow(right);
+ DEOPT_IF(!PyFloat_CheckExact(left_o), COMPARE_OP);
+ DEOPT_IF(!PyFloat_CheckExact(right_o), COMPARE_OP);
}
/* Skip 1 cache entry */
// _COMPARE_OP_FLOAT
{
+ PyObject *left_o = PyStackRef_AsPyObjectBorrow(left);
+ PyObject *right_o = PyStackRef_AsPyObjectBorrow(right);
STAT_INC(COMPARE_OP, hit);
- double dleft = PyFloat_AS_DOUBLE(left);
- double dright = PyFloat_AS_DOUBLE(right);
+ double dleft = PyFloat_AS_DOUBLE(left_o);
+ double dright = PyFloat_AS_DOUBLE(right_o);
// 1 if NaN, 2 if <, 4 if >, 8 if ==; this matches low four bits of the oparg
int sign_ish = COMPARISON_BIT(dleft, dright);
- _Py_DECREF_SPECIALIZED(left, _PyFloat_ExactDealloc);
- _Py_DECREF_SPECIALIZED(right, _PyFloat_ExactDealloc);
- res = (sign_ish & oparg) ? Py_True : Py_False;
+ _Py_DECREF_SPECIALIZED(left_o, _PyFloat_ExactDealloc);
+ _Py_DECREF_SPECIALIZED(right_o, _PyFloat_ExactDealloc);
+ res = (sign_ish & oparg) ? PyStackRef_True : PyStackRef_False;
// It's always a bool, so we don't care about oparg & 16.
}
stack_pointer[-2] = res;
next_instr += 2;
INSTRUCTION_STATS(COMPARE_OP_INT);
static_assert(INLINE_CACHE_ENTRIES_COMPARE_OP == 1, "incorrect cache size");
- PyObject *right;
- PyObject *left;
- PyObject *res;
+ _PyStackRef right;
+ _PyStackRef left;
+ _PyStackRef res;
// _GUARD_BOTH_INT
right = stack_pointer[-1];
left = stack_pointer[-2];
{
- DEOPT_IF(!PyLong_CheckExact(left), COMPARE_OP);
- DEOPT_IF(!PyLong_CheckExact(right), COMPARE_OP);
+ PyObject *left_o = PyStackRef_AsPyObjectBorrow(left);
+ PyObject *right_o = PyStackRef_AsPyObjectBorrow(right);
+ DEOPT_IF(!PyLong_CheckExact(left_o), COMPARE_OP);
+ DEOPT_IF(!PyLong_CheckExact(right_o), COMPARE_OP);
}
/* Skip 1 cache entry */
// _COMPARE_OP_INT
{
- DEOPT_IF(!_PyLong_IsCompact((PyLongObject *)left), COMPARE_OP);
- DEOPT_IF(!_PyLong_IsCompact((PyLongObject *)right), COMPARE_OP);
+ PyObject *left_o = PyStackRef_AsPyObjectBorrow(left);
+ PyObject *right_o = PyStackRef_AsPyObjectBorrow(right);
+ DEOPT_IF(!_PyLong_IsCompact((PyLongObject *)left_o), COMPARE_OP);
+ DEOPT_IF(!_PyLong_IsCompact((PyLongObject *)right_o), COMPARE_OP);
STAT_INC(COMPARE_OP, hit);
- assert(_PyLong_DigitCount((PyLongObject *)left) <= 1 &&
- _PyLong_DigitCount((PyLongObject *)right) <= 1);
- Py_ssize_t ileft = _PyLong_CompactValue((PyLongObject *)left);
- Py_ssize_t iright = _PyLong_CompactValue((PyLongObject *)right);
+ assert(_PyLong_DigitCount((PyLongObject *)left_o) <= 1 &&
+ _PyLong_DigitCount((PyLongObject *)right_o) <= 1);
+ Py_ssize_t ileft = _PyLong_CompactValue((PyLongObject *)left_o);
+ Py_ssize_t iright = _PyLong_CompactValue((PyLongObject *)right_o);
// 2 if <, 4 if >, 8 if ==; this matches the low 4 bits of the oparg
int sign_ish = COMPARISON_BIT(ileft, iright);
- _Py_DECREF_SPECIALIZED(left, (destructor)PyObject_Free);
- _Py_DECREF_SPECIALIZED(right, (destructor)PyObject_Free);
- res = (sign_ish & oparg) ? Py_True : Py_False;
+ _Py_DECREF_SPECIALIZED(left_o, (destructor)PyObject_Free);
+ _Py_DECREF_SPECIALIZED(right_o, (destructor)PyObject_Free);
+ res = (sign_ish & oparg) ? PyStackRef_True : PyStackRef_False;
// It's always a bool, so we don't care about oparg & 16.
}
stack_pointer[-2] = res;
next_instr += 2;
INSTRUCTION_STATS(COMPARE_OP_STR);
static_assert(INLINE_CACHE_ENTRIES_COMPARE_OP == 1, "incorrect cache size");
- PyObject *right;
- PyObject *left;
- PyObject *res;
+ _PyStackRef right;
+ _PyStackRef left;
+ _PyStackRef res;
// _GUARD_BOTH_UNICODE
right = stack_pointer[-1];
left = stack_pointer[-2];
{
- DEOPT_IF(!PyUnicode_CheckExact(left), COMPARE_OP);
- DEOPT_IF(!PyUnicode_CheckExact(right), COMPARE_OP);
+ PyObject *left_o = PyStackRef_AsPyObjectBorrow(left);
+ PyObject *right_o = PyStackRef_AsPyObjectBorrow(right);
+ DEOPT_IF(!PyUnicode_CheckExact(left_o), COMPARE_OP);
+ DEOPT_IF(!PyUnicode_CheckExact(right_o), COMPARE_OP);
}
/* Skip 1 cache entry */
// _COMPARE_OP_STR
{
+ PyObject *left_o = PyStackRef_AsPyObjectBorrow(left);
+ PyObject *right_o = PyStackRef_AsPyObjectBorrow(right);
STAT_INC(COMPARE_OP, hit);
- int eq = _PyUnicode_Equal(left, right);
+ int eq = _PyUnicode_Equal(left_o, right_o);
assert((oparg >> 5) == Py_EQ || (oparg >> 5) == Py_NE);
- _Py_DECREF_SPECIALIZED(left, _PyUnicode_ExactDealloc);
- _Py_DECREF_SPECIALIZED(right, _PyUnicode_ExactDealloc);
+ _Py_DECREF_SPECIALIZED(left_o, _PyUnicode_ExactDealloc);
+ _Py_DECREF_SPECIALIZED(right_o, _PyUnicode_ExactDealloc);
assert(eq == 0 || eq == 1);
assert((oparg & 0xf) == COMPARISON_NOT_EQUALS || (oparg & 0xf) == COMPARISON_EQUALS);
assert(COMPARISON_NOT_EQUALS + 1 == COMPARISON_EQUALS);
- res = ((COMPARISON_NOT_EQUALS + eq) & oparg) ? Py_True : Py_False;
+ res = ((COMPARISON_NOT_EQUALS + eq) & oparg) ? PyStackRef_True : PyStackRef_False;
// It's always a bool, so we don't care about oparg & 16.
}
stack_pointer[-2] = res;
PREDICTED(CONTAINS_OP);
_Py_CODEUNIT *this_instr = next_instr - 2;
(void)this_instr;
- PyObject *right;
- PyObject *left;
- PyObject *b;
+ _PyStackRef right;
+ _PyStackRef left;
+ _PyStackRef b;
// _SPECIALIZE_CONTAINS_OP
right = stack_pointer[-1];
left = stack_pointer[-2];
}
// _CONTAINS_OP
{
- int res = PySequence_Contains(right, left);
- Py_DECREF(left);
- Py_DECREF(right);
+ PyObject *left_o = PyStackRef_AsPyObjectBorrow(left);
+ PyObject *right_o = PyStackRef_AsPyObjectBorrow(right);
+ int res = PySequence_Contains(right_o, left_o);
+ PyStackRef_CLOSE(left);
+ PyStackRef_CLOSE(right);
if (res < 0) goto pop_2_error;
- b = (res ^ oparg) ? Py_True : Py_False;
+ b = (res ^ oparg) ? PyStackRef_True : PyStackRef_False;
}
stack_pointer[-2] = b;
stack_pointer += -1;
next_instr += 2;
INSTRUCTION_STATS(CONTAINS_OP_DICT);
static_assert(INLINE_CACHE_ENTRIES_CONTAINS_OP == 1, "incorrect cache size");
- PyObject *right;
- PyObject *left;
- PyObject *b;
+ _PyStackRef right;
+ _PyStackRef left;
+ _PyStackRef b;
/* Skip 1 cache entry */
right = stack_pointer[-1];
left = stack_pointer[-2];
- DEOPT_IF(!PyDict_CheckExact(right), CONTAINS_OP);
+ PyObject *left_o = PyStackRef_AsPyObjectBorrow(left);
+ PyObject *right_o = PyStackRef_AsPyObjectBorrow(right);
+ DEOPT_IF(!PyDict_CheckExact(right_o), CONTAINS_OP);
STAT_INC(CONTAINS_OP, hit);
- int res = PyDict_Contains(right, left);
- Py_DECREF(left);
- Py_DECREF(right);
+ int res = PyDict_Contains(right_o, left_o);
+ PyStackRef_CLOSE(left);
+ PyStackRef_CLOSE(right);
if (res < 0) goto pop_2_error;
- b = (res ^ oparg) ? Py_True : Py_False;
+ b = (res ^ oparg) ? PyStackRef_True : PyStackRef_False;
stack_pointer[-2] = b;
stack_pointer += -1;
assert(WITHIN_STACK_BOUNDS());
next_instr += 2;
INSTRUCTION_STATS(CONTAINS_OP_SET);
static_assert(INLINE_CACHE_ENTRIES_CONTAINS_OP == 1, "incorrect cache size");
- PyObject *right;
- PyObject *left;
- PyObject *b;
+ _PyStackRef right;
+ _PyStackRef left;
+ _PyStackRef b;
/* Skip 1 cache entry */
right = stack_pointer[-1];
left = stack_pointer[-2];
- DEOPT_IF(!(PySet_CheckExact(right) || PyFrozenSet_CheckExact(right)), CONTAINS_OP);
+ PyObject *left_o = PyStackRef_AsPyObjectBorrow(left);
+ PyObject *right_o = PyStackRef_AsPyObjectBorrow(right);
+ DEOPT_IF(!(PySet_CheckExact(right_o) || PyFrozenSet_CheckExact(right_o)), CONTAINS_OP);
STAT_INC(CONTAINS_OP, hit);
// Note: both set and frozenset use the same seq_contains method!
- int res = _PySet_Contains((PySetObject *)right, left);
- Py_DECREF(left);
- Py_DECREF(right);
+ int res = _PySet_Contains((PySetObject *)right_o, left_o);
+ PyStackRef_CLOSE(left);
+ PyStackRef_CLOSE(right);
if (res < 0) goto pop_2_error;
- b = (res ^ oparg) ? Py_True : Py_False;
+ b = (res ^ oparg) ? PyStackRef_True : PyStackRef_False;
stack_pointer[-2] = b;
stack_pointer += -1;
assert(WITHIN_STACK_BOUNDS());
frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(CONVERT_VALUE);
- PyObject *value;
- PyObject *result;
+ _PyStackRef value;
+ _PyStackRef result;
value = stack_pointer[-1];
conversion_func conv_fn;
assert(oparg >= FVC_STR && oparg <= FVC_ASCII);
conv_fn = _PyEval_ConversionFuncs[oparg];
- result = conv_fn(value);
- Py_DECREF(value);
- if (result == NULL) goto pop_1_error;
+ PyObject *result_o = conv_fn(PyStackRef_AsPyObjectBorrow(value));
+ PyStackRef_CLOSE(value);
+ if (result_o == NULL) goto pop_1_error;
+ result = PyStackRef_FromPyObjectSteal(result_o);
stack_pointer[-1] = result;
DISPATCH();
}
frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(COPY);
- PyObject *bottom;
- PyObject *top;
+ _PyStackRef bottom;
+ _PyStackRef top;
bottom = stack_pointer[-1 - (oparg-1)];
assert(oparg > 0);
- top = Py_NewRef(bottom);
+ top = PyStackRef_DUP(bottom);
stack_pointer[0] = top;
stack_pointer += 1;
assert(WITHIN_STACK_BOUNDS());
int offset = co->co_nlocalsplus - oparg;
for (int i = 0; i < oparg; ++i) {
PyObject *o = PyTuple_GET_ITEM(closure, i);
- frame->localsplus[offset + i] = Py_NewRef(o);
+ frame->localsplus[offset + i] = PyStackRef_FromPyObjectNew(o);
}
DISPATCH();
}
frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(DELETE_ATTR);
- PyObject *owner;
+ _PyStackRef owner;
owner = stack_pointer[-1];
PyObject *name = GETITEM(FRAME_CO_NAMES, oparg);
- int err = PyObject_DelAttr(owner, name);
- Py_DECREF(owner);
+ int err = PyObject_DelAttr(PyStackRef_AsPyObjectBorrow(owner), name);
+ PyStackRef_CLOSE(owner);
if (err) goto pop_1_error;
stack_pointer += -1;
assert(WITHIN_STACK_BOUNDS());
frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(DELETE_DEREF);
- PyObject *cell = GETLOCAL(oparg);
+ PyObject *cell = PyStackRef_AsPyObjectBorrow(GETLOCAL(oparg));
// Can't use ERROR_IF here.
// Fortunately we don't need its superpower.
PyObject *oldobj = PyCell_SwapTakeRef((PyCellObject *)cell, NULL);
frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(DELETE_FAST);
- PyObject *v = GETLOCAL(oparg);
- if (v == NULL) {
+ _PyStackRef v = GETLOCAL(oparg);
+ if (PyStackRef_IsNull(v)) {
_PyEval_FormatExcCheckArg(tstate, PyExc_UnboundLocalError,
UNBOUNDLOCAL_ERROR_MSG,
PyTuple_GetItem(_PyFrame_GetCode(frame)->co_localsplusnames, oparg)
);
if (1) goto error;
}
- SETLOCAL(oparg, NULL);
+ SETLOCAL(oparg, PyStackRef_NULL);
DISPATCH();
}
frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(DELETE_SUBSCR);
- PyObject *sub;
- PyObject *container;
+ _PyStackRef sub;
+ _PyStackRef container;
sub = stack_pointer[-1];
container = stack_pointer[-2];
/* del container[sub] */
- int err = PyObject_DelItem(container, sub);
- Py_DECREF(container);
- Py_DECREF(sub);
+ int err = PyObject_DelItem(PyStackRef_AsPyObjectBorrow(container),
+ PyStackRef_AsPyObjectBorrow(sub));
+ PyStackRef_CLOSE(container);
+ PyStackRef_CLOSE(sub);
if (err) goto pop_2_error;
stack_pointer += -2;
assert(WITHIN_STACK_BOUNDS());
frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(DICT_MERGE);
- PyObject *update;
- PyObject *dict;
- PyObject *callable;
+ _PyStackRef update;
+ _PyStackRef dict;
+ _PyStackRef callable;
update = stack_pointer[-1];
dict = stack_pointer[-2 - (oparg - 1)];
callable = stack_pointer[-5 - (oparg - 1)];
- if (_PyDict_MergeEx(dict, update, 2) < 0) {
- _PyEval_FormatKwargsError(tstate, callable, update);
- Py_DECREF(update);
+ PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable);
+ PyObject *dict_o = PyStackRef_AsPyObjectBorrow(dict);
+ PyObject *update_o = PyStackRef_AsPyObjectBorrow(update);
+ if (_PyDict_MergeEx(dict_o, update_o, 2) < 0) {
+ _PyEval_FormatKwargsError(tstate, callable_o, update_o);
+ PyStackRef_CLOSE(update);
if (true) goto pop_1_error;
}
- Py_DECREF(update);
+ PyStackRef_CLOSE(update);
stack_pointer += -1;
assert(WITHIN_STACK_BOUNDS());
DISPATCH();
frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(DICT_UPDATE);
- PyObject *update;
- PyObject *dict;
+ _PyStackRef update;
+ _PyStackRef dict;
update = stack_pointer[-1];
dict = stack_pointer[-2 - (oparg - 1)];
- if (PyDict_Update(dict, update) < 0) {
+ PyObject *dict_o = PyStackRef_AsPyObjectBorrow(dict);
+ PyObject *update_o = PyStackRef_AsPyObjectBorrow(update);
+ if (PyDict_Update(dict_o, update_o) < 0) {
if (_PyErr_ExceptionMatches(tstate, PyExc_AttributeError)) {
_PyErr_Format(tstate, PyExc_TypeError,
"'%.200s' object is not a mapping",
- Py_TYPE(update)->tp_name);
+ Py_TYPE(update_o)->tp_name);
}
- Py_DECREF(update);
+ PyStackRef_CLOSE(update);
if (true) goto pop_1_error;
}
- Py_DECREF(update);
+ PyStackRef_CLOSE(update);
stack_pointer += -1;
assert(WITHIN_STACK_BOUNDS());
DISPATCH();
(void)this_instr;
next_instr += 1;
INSTRUCTION_STATS(END_ASYNC_FOR);
- PyObject *exc;
- PyObject *awaitable;
- exc = stack_pointer[-1];
- awaitable = stack_pointer[-2];
+ _PyStackRef exc_st;
+ _PyStackRef awaitable_st;
+ exc_st = stack_pointer[-1];
+ awaitable_st = stack_pointer[-2];
+ PyObject *exc = PyStackRef_AsPyObjectBorrow(exc_st);
assert(exc && PyExceptionInstance_Check(exc));
if (PyErr_GivenExceptionMatches(exc, PyExc_StopAsyncIteration)) {
- Py_DECREF(awaitable);
- Py_DECREF(exc);
+ PyStackRef_CLOSE(awaitable_st);
+ PyStackRef_CLOSE(exc_st);
}
else {
Py_INCREF(exc);
frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(END_FOR);
- PyObject *value;
+ _PyStackRef value;
value = stack_pointer[-1];
- Py_DECREF(value);
+ PyStackRef_CLOSE(value);
stack_pointer += -1;
assert(WITHIN_STACK_BOUNDS());
DISPATCH();
frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(END_SEND);
- PyObject *value;
- PyObject *receiver;
+ _PyStackRef value;
+ _PyStackRef receiver;
value = stack_pointer[-1];
receiver = stack_pointer[-2];
- Py_DECREF(receiver);
+ (void)receiver;
+ PyStackRef_CLOSE(receiver);
stack_pointer[-2] = value;
stack_pointer += -1;
assert(WITHIN_STACK_BOUNDS());
frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(EXIT_INIT_CHECK);
- PyObject *should_be_none;
+ _PyStackRef should_be_none;
should_be_none = stack_pointer[-1];
assert(STACK_LEVEL() == 2);
- if (should_be_none != Py_None) {
+ if (!PyStackRef_Is(should_be_none, PyStackRef_None)) {
PyErr_Format(PyExc_TypeError,
"__init__() should return None, not '%.200s'",
- Py_TYPE(should_be_none)->tp_name);
+ Py_TYPE(PyStackRef_AsPyObjectBorrow(should_be_none))->tp_name);
goto error;
}
stack_pointer += -1;
frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(FORMAT_SIMPLE);
- PyObject *value;
- PyObject *res;
+ _PyStackRef value;
+ _PyStackRef res;
value = stack_pointer[-1];
+ PyObject *value_o = PyStackRef_AsPyObjectBorrow(value);
/* If value is a unicode object, then we know the result
* of format(value) is value itself. */
- if (!PyUnicode_CheckExact(value)) {
- res = PyObject_Format(value, NULL);
- Py_DECREF(value);
- if (res == NULL) goto pop_1_error;
+ if (!PyUnicode_CheckExact(value_o)) {
+ res = PyStackRef_FromPyObjectSteal(PyObject_Format(value_o, NULL));
+ PyStackRef_CLOSE(value);
+ if (PyStackRef_IsNull(res)) goto pop_1_error;
}
else {
res = value;
frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(FORMAT_WITH_SPEC);
- PyObject *fmt_spec;
- PyObject *value;
- PyObject *res;
+ _PyStackRef fmt_spec;
+ _PyStackRef value;
+ _PyStackRef res;
fmt_spec = stack_pointer[-1];
value = stack_pointer[-2];
- res = PyObject_Format(value, fmt_spec);
- Py_DECREF(value);
- Py_DECREF(fmt_spec);
- if (res == NULL) goto pop_2_error;
+ PyObject *res_o = PyObject_Format(PyStackRef_AsPyObjectBorrow(value), PyStackRef_AsPyObjectBorrow(fmt_spec));
+ PyStackRef_CLOSE(value);
+ PyStackRef_CLOSE(fmt_spec);
+ if (res_o == NULL) goto pop_2_error;
+ res = PyStackRef_FromPyObjectSteal(res_o);
stack_pointer[-2] = res;
stack_pointer += -1;
assert(WITHIN_STACK_BOUNDS());
PREDICTED(FOR_ITER);
_Py_CODEUNIT *this_instr = next_instr - 2;
(void)this_instr;
- PyObject *iter;
- PyObject *next;
+ _PyStackRef iter;
+ _PyStackRef next;
// _SPECIALIZE_FOR_ITER
iter = stack_pointer[-1];
{
// _FOR_ITER
{
/* before: [iter]; after: [iter, iter()] *or* [] (and jump over END_FOR.) */
- next = (*Py_TYPE(iter)->tp_iternext)(iter);
- if (next == NULL) {
+ PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter);
+ PyObject *next_o = (*Py_TYPE(iter_o)->tp_iternext)(iter_o);
+ if (next_o == NULL) {
+ next = PyStackRef_NULL;
if (_PyErr_Occurred(tstate)) {
if (!_PyErr_ExceptionMatches(tstate, PyExc_StopIteration)) {
goto error;
/* iterator ended normally */
assert(next_instr[oparg].op.code == END_FOR ||
next_instr[oparg].op.code == INSTRUMENTED_END_FOR);
- Py_DECREF(iter);
+ PyStackRef_CLOSE(iter);
STACK_SHRINK(1);
/* Jump forward oparg, then skip following END_FOR and POP_TOP instruction */
JUMPBY(oparg + 2);
DISPATCH();
}
+ next = PyStackRef_FromPyObjectSteal(next_o);
// Common case: no jump, leave it to the code generator
}
stack_pointer[0] = next;
next_instr += 2;
INSTRUCTION_STATS(FOR_ITER_GEN);
static_assert(INLINE_CACHE_ENTRIES_FOR_ITER == 1, "incorrect cache size");
- PyObject *iter;
+ _PyStackRef iter;
_PyInterpreterFrame *gen_frame;
_PyInterpreterFrame *new_frame;
/* Skip 1 cache entry */
// _FOR_ITER_GEN_FRAME
iter = stack_pointer[-1];
{
- PyGenObject *gen = (PyGenObject *)iter;
+ PyGenObject *gen = (PyGenObject *)PyStackRef_AsPyObjectBorrow(iter);
DEOPT_IF(Py_TYPE(gen) != &PyGen_Type, FOR_ITER);
DEOPT_IF(gen->gi_frame_state >= FRAME_EXECUTING, FOR_ITER);
STAT_INC(FOR_ITER, hit);
gen_frame = &gen->gi_iframe;
- _PyFrame_StackPush(gen_frame, Py_None);
+ _PyFrame_StackPush(gen_frame, PyStackRef_None);
gen->gi_frame_state = FRAME_EXECUTING;
gen->gi_exc_state.previous_item = tstate->exc_info;
tstate->exc_info = &gen->gi_exc_state;
next_instr += 2;
INSTRUCTION_STATS(FOR_ITER_LIST);
static_assert(INLINE_CACHE_ENTRIES_FOR_ITER == 1, "incorrect cache size");
- PyObject *iter;
- PyObject *next;
+ _PyStackRef iter;
+ _PyStackRef next;
/* Skip 1 cache entry */
// _ITER_CHECK_LIST
iter = stack_pointer[-1];
{
- DEOPT_IF(Py_TYPE(iter) != &PyListIter_Type, FOR_ITER);
+ DEOPT_IF(Py_TYPE(PyStackRef_AsPyObjectBorrow(iter)) != &PyListIter_Type, FOR_ITER);
}
// _ITER_JUMP_LIST
{
- _PyListIterObject *it = (_PyListIterObject *)iter;
- assert(Py_TYPE(iter) == &PyListIter_Type);
+ PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter);
+ _PyListIterObject *it = (_PyListIterObject *)iter_o;
+ assert(Py_TYPE(iter_o) == &PyListIter_Type);
STAT_INC(FOR_ITER, hit);
PyListObject *seq = it->it_seq;
if (seq == NULL || (size_t)it->it_index >= (size_t)PyList_GET_SIZE(seq)) {
Py_DECREF(seq);
}
#endif
- Py_DECREF(iter);
+ PyStackRef_CLOSE(iter);
STACK_SHRINK(1);
/* Jump forward oparg, then skip following END_FOR and POP_TOP instructions */
JUMPBY(oparg + 2);
}
// _ITER_NEXT_LIST
{
- _PyListIterObject *it = (_PyListIterObject *)iter;
- assert(Py_TYPE(iter) == &PyListIter_Type);
+ PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter);
+ _PyListIterObject *it = (_PyListIterObject *)iter_o;
+ assert(Py_TYPE(iter_o) == &PyListIter_Type);
PyListObject *seq = it->it_seq;
assert(seq);
assert(it->it_index < PyList_GET_SIZE(seq));
- next = Py_NewRef(PyList_GET_ITEM(seq, it->it_index++));
+ next = PyStackRef_FromPyObjectNew(PyList_GET_ITEM(seq, it->it_index++));
}
stack_pointer[0] = next;
stack_pointer += 1;
next_instr += 2;
INSTRUCTION_STATS(FOR_ITER_RANGE);
static_assert(INLINE_CACHE_ENTRIES_FOR_ITER == 1, "incorrect cache size");
- PyObject *iter;
- PyObject *next;
+ _PyStackRef iter;
+ _PyStackRef next;
/* Skip 1 cache entry */
// _ITER_CHECK_RANGE
iter = stack_pointer[-1];
{
- _PyRangeIterObject *r = (_PyRangeIterObject *)iter;
+ _PyRangeIterObject *r = (_PyRangeIterObject *)PyStackRef_AsPyObjectBorrow(iter);
DEOPT_IF(Py_TYPE(r) != &PyRangeIter_Type, FOR_ITER);
}
// _ITER_JUMP_RANGE
{
- _PyRangeIterObject *r = (_PyRangeIterObject *)iter;
+ _PyRangeIterObject *r = (_PyRangeIterObject *)PyStackRef_AsPyObjectBorrow(iter);
assert(Py_TYPE(r) == &PyRangeIter_Type);
STAT_INC(FOR_ITER, hit);
if (r->len <= 0) {
STACK_SHRINK(1);
- Py_DECREF(r);
+ PyStackRef_CLOSE(iter);
// Jump over END_FOR and POP_TOP instructions.
JUMPBY(oparg + 2);
DISPATCH();
}
// _ITER_NEXT_RANGE
{
- _PyRangeIterObject *r = (_PyRangeIterObject *)iter;
+ _PyRangeIterObject *r = (_PyRangeIterObject *)PyStackRef_AsPyObjectBorrow(iter);
assert(Py_TYPE(r) == &PyRangeIter_Type);
assert(r->len > 0);
long value = r->start;
r->start = value + r->step;
r->len--;
- next = PyLong_FromLong(value);
- if (next == NULL) goto error;
+ PyObject *res = PyLong_FromLong(value);
+ if (res == NULL) goto error;
+ next = PyStackRef_FromPyObjectSteal(res);
}
stack_pointer[0] = next;
stack_pointer += 1;
next_instr += 2;
INSTRUCTION_STATS(FOR_ITER_TUPLE);
static_assert(INLINE_CACHE_ENTRIES_FOR_ITER == 1, "incorrect cache size");
- PyObject *iter;
- PyObject *next;
+ _PyStackRef iter;
+ _PyStackRef next;
/* Skip 1 cache entry */
// _ITER_CHECK_TUPLE
iter = stack_pointer[-1];
{
- DEOPT_IF(Py_TYPE(iter) != &PyTupleIter_Type, FOR_ITER);
+ DEOPT_IF(Py_TYPE(PyStackRef_AsPyObjectBorrow(iter)) != &PyTupleIter_Type, FOR_ITER);
}
// _ITER_JUMP_TUPLE
{
- _PyTupleIterObject *it = (_PyTupleIterObject *)iter;
- assert(Py_TYPE(iter) == &PyTupleIter_Type);
+ PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter);
+ _PyTupleIterObject *it = (_PyTupleIterObject *)iter_o;
+ assert(Py_TYPE(iter_o) == &PyTupleIter_Type);
STAT_INC(FOR_ITER, hit);
PyTupleObject *seq = it->it_seq;
if (seq == NULL || it->it_index >= PyTuple_GET_SIZE(seq)) {
it->it_seq = NULL;
Py_DECREF(seq);
}
- Py_DECREF(iter);
+ PyStackRef_CLOSE(iter);
STACK_SHRINK(1);
/* Jump forward oparg, then skip following END_FOR and POP_TOP instructions */
JUMPBY(oparg + 2);
}
// _ITER_NEXT_TUPLE
{
- _PyTupleIterObject *it = (_PyTupleIterObject *)iter;
- assert(Py_TYPE(iter) == &PyTupleIter_Type);
+ PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter);
+ _PyTupleIterObject *it = (_PyTupleIterObject *)iter_o;
+ assert(Py_TYPE(iter_o) == &PyTupleIter_Type);
PyTupleObject *seq = it->it_seq;
assert(seq);
assert(it->it_index < PyTuple_GET_SIZE(seq));
- next = Py_NewRef(PyTuple_GET_ITEM(seq, it->it_index++));
+ next = PyStackRef_FromPyObjectNew(PyTuple_GET_ITEM(seq, it->it_index++));
}
stack_pointer[0] = next;
stack_pointer += 1;
frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(GET_AITER);
- PyObject *obj;
- PyObject *iter;
+ _PyStackRef obj;
+ _PyStackRef iter;
obj = stack_pointer[-1];
unaryfunc getter = NULL;
- PyTypeObject *type = Py_TYPE(obj);
+ PyObject *obj_o = PyStackRef_AsPyObjectBorrow(obj);
+ PyObject *iter_o;
+ PyTypeObject *type = Py_TYPE(obj_o);
if (type->tp_as_async != NULL) {
getter = type->tp_as_async->am_aiter;
}
"'async for' requires an object with "
"__aiter__ method, got %.100s",
type->tp_name);
- Py_DECREF(obj);
+ PyStackRef_CLOSE(obj);
if (true) goto pop_1_error;
}
- iter = (*getter)(obj);
- Py_DECREF(obj);
- if (iter == NULL) goto pop_1_error;
- if (Py_TYPE(iter)->tp_as_async == NULL ||
- Py_TYPE(iter)->tp_as_async->am_anext == NULL) {
+ iter_o = (*getter)(obj_o);
+ PyStackRef_CLOSE(obj);
+ if (iter_o == NULL) goto pop_1_error;
+ if (Py_TYPE(iter_o)->tp_as_async == NULL ||
+ Py_TYPE(iter_o)->tp_as_async->am_anext == NULL) {
_PyErr_Format(tstate, PyExc_TypeError,
"'async for' received an object from __aiter__ "
"that does not implement __anext__: %.100s",
- Py_TYPE(iter)->tp_name);
- Py_DECREF(iter);
+ Py_TYPE(iter_o)->tp_name);
+ Py_DECREF(iter_o);
if (true) goto pop_1_error;
}
+ iter = PyStackRef_FromPyObjectSteal(iter_o);
stack_pointer[-1] = iter;
DISPATCH();
}
frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(GET_ANEXT);
- PyObject *aiter;
- PyObject *awaitable;
+ _PyStackRef aiter;
+ _PyStackRef awaitable;
aiter = stack_pointer[-1];
unaryfunc getter = NULL;
PyObject *next_iter = NULL;
- PyTypeObject *type = Py_TYPE(aiter);
- if (PyAsyncGen_CheckExact(aiter)) {
- awaitable = type->tp_as_async->am_anext(aiter);
- if (awaitable == NULL) {
+ PyObject *awaitable_o;
+ PyObject *aiter_o = PyStackRef_AsPyObjectBorrow(aiter);
+ PyTypeObject *type = Py_TYPE(aiter_o);
+ if (PyAsyncGen_CheckExact(aiter_o)) {
+ awaitable_o = type->tp_as_async->am_anext(aiter_o);
+ if (awaitable_o == NULL) {
goto error;
}
} else {
getter = type->tp_as_async->am_anext;
}
if (getter != NULL) {
- next_iter = (*getter)(aiter);
+ next_iter = (*getter)(aiter_o);
if (next_iter == NULL) {
goto error;
}
type->tp_name);
goto error;
}
- awaitable = _PyCoro_GetAwaitableIter(next_iter);
- if (awaitable == NULL) {
+ awaitable_o = _PyCoro_GetAwaitableIter(next_iter);
+ if (awaitable_o == NULL) {
_PyErr_FormatFromCause(
PyExc_TypeError,
"'async for' received an invalid object "
Py_DECREF(next_iter);
}
}
+ awaitable = PyStackRef_FromPyObjectSteal(awaitable_o);
stack_pointer[0] = awaitable;
stack_pointer += 1;
assert(WITHIN_STACK_BOUNDS());
frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(GET_AWAITABLE);
- PyObject *iterable;
- PyObject *iter;
+ _PyStackRef iterable;
+ _PyStackRef iter;
iterable = stack_pointer[-1];
- iter = _PyCoro_GetAwaitableIter(iterable);
- if (iter == NULL) {
- _PyEval_FormatAwaitableError(tstate, Py_TYPE(iterable), oparg);
- }
- Py_DECREF(iterable);
- if (iter != NULL && PyCoro_CheckExact(iter)) {
- PyObject *yf = _PyGen_yf((PyGenObject*)iter);
+ PyObject *iter_o = _PyCoro_GetAwaitableIter(PyStackRef_AsPyObjectBorrow(iterable));
+ if (iter_o == NULL) {
+ _PyEval_FormatAwaitableError(tstate,
+ Py_TYPE(PyStackRef_AsPyObjectBorrow(iterable)), oparg);
+ }
+ PyStackRef_CLOSE(iterable);
+ if (iter_o != NULL && PyCoro_CheckExact(iter_o)) {
+ PyObject *yf = _PyGen_yf((PyGenObject*)iter_o);
if (yf != NULL) {
/* `iter` is a coroutine object that is being
awaited, `yf` is a pointer to the current awaitable
being awaited on. */
Py_DECREF(yf);
- Py_CLEAR(iter);
+ Py_CLEAR(iter_o);
_PyErr_SetString(tstate, PyExc_RuntimeError,
"coroutine is being awaited already");
/* The code below jumps to `error` if `iter` is NULL. */
}
}
- if (iter == NULL) goto pop_1_error;
+ if (iter_o == NULL) goto pop_1_error;
+ iter = PyStackRef_FromPyObjectSteal(iter_o);
stack_pointer[-1] = iter;
DISPATCH();
}
frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(GET_ITER);
- PyObject *iterable;
- PyObject *iter;
+ _PyStackRef iterable;
+ _PyStackRef iter;
iterable = stack_pointer[-1];
/* before: [obj]; after [getiter(obj)] */
- iter = PyObject_GetIter(iterable);
- Py_DECREF(iterable);
- if (iter == NULL) goto pop_1_error;
+ iter = PyStackRef_FromPyObjectSteal(PyObject_GetIter(PyStackRef_AsPyObjectBorrow(iterable)));
+ PyStackRef_CLOSE(iterable);
+ if (PyStackRef_IsNull(iter)) goto pop_1_error;
stack_pointer[-1] = iter;
DISPATCH();
}
frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(GET_LEN);
- PyObject *obj;
- PyObject *len_o;
+ _PyStackRef obj;
+ _PyStackRef len;
obj = stack_pointer[-1];
// PUSH(len(TOS))
- Py_ssize_t len_i = PyObject_Length(obj);
+ Py_ssize_t len_i = PyObject_Length(PyStackRef_AsPyObjectBorrow(obj));
if (len_i < 0) goto error;
- len_o = PyLong_FromSsize_t(len_i);
+ PyObject *len_o = PyLong_FromSsize_t(len_i);
if (len_o == NULL) goto error;
- stack_pointer[0] = len_o;
+ len = PyStackRef_FromPyObjectSteal(len_o);
+ stack_pointer[0] = len;
stack_pointer += 1;
assert(WITHIN_STACK_BOUNDS());
DISPATCH();
frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(GET_YIELD_FROM_ITER);
- PyObject *iterable;
- PyObject *iter;
+ _PyStackRef iterable;
+ _PyStackRef iter;
iterable = stack_pointer[-1];
/* before: [obj]; after [getiter(obj)] */
- if (PyCoro_CheckExact(iterable)) {
+ PyObject *iterable_o = PyStackRef_AsPyObjectBorrow(iterable);
+ if (PyCoro_CheckExact(iterable_o)) {
/* `iterable` is a coroutine */
if (!(_PyFrame_GetCode(frame)->co_flags & (CO_COROUTINE | CO_ITERABLE_COROUTINE))) {
/* and it is used in a 'yield from' expression of a
}
iter = iterable;
}
- else if (PyGen_CheckExact(iterable)) {
+ else if (PyGen_CheckExact(iterable_o)) {
iter = iterable;
}
else {
/* `iterable` is not a generator. */
- iter = PyObject_GetIter(iterable);
- if (iter == NULL) {
+ iter = PyStackRef_FromPyObjectSteal(PyObject_GetIter(iterable_o));
+ if (PyStackRef_IsNull(iter)) {
goto error;
}
- Py_DECREF(iterable);
+ PyStackRef_CLOSE(iterable);
}
stack_pointer[-1] = iter;
DISPATCH();
frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(IMPORT_FROM);
- PyObject *from;
- PyObject *res;
+ _PyStackRef from;
+ _PyStackRef res;
from = stack_pointer[-1];
PyObject *name = GETITEM(FRAME_CO_NAMES, oparg);
- res = import_from(tstate, from, name);
- if (res == NULL) goto error;
+ PyObject *res_o = import_from(tstate, PyStackRef_AsPyObjectBorrow(from), name);
+ if (res_o == NULL) goto error;
+ res = PyStackRef_FromPyObjectSteal(res_o);
stack_pointer[0] = res;
stack_pointer += 1;
assert(WITHIN_STACK_BOUNDS());
frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(IMPORT_NAME);
- PyObject *fromlist;
- PyObject *level;
- PyObject *res;
+ _PyStackRef fromlist;
+ _PyStackRef level;
+ _PyStackRef res;
fromlist = stack_pointer[-1];
level = stack_pointer[-2];
PyObject *name = GETITEM(FRAME_CO_NAMES, oparg);
- res = import_name(tstate, frame, name, fromlist, level);
- Py_DECREF(level);
- Py_DECREF(fromlist);
- if (res == NULL) goto pop_2_error;
+ PyObject *res_o = import_name(tstate, frame, name,
+ PyStackRef_AsPyObjectBorrow(fromlist),
+ PyStackRef_AsPyObjectBorrow(level));
+ PyStackRef_CLOSE(level);
+ PyStackRef_CLOSE(fromlist);
+ if (res_o == NULL) goto pop_2_error;
+ res = PyStackRef_FromPyObjectSteal(res_o);
stack_pointer[-2] = res;
stack_pointer += -1;
assert(WITHIN_STACK_BOUNDS());
next_instr += 4;
INSTRUCTION_STATS(INSTRUMENTED_CALL);
/* Skip 3 cache entries */
- int is_meth = PEEK(oparg + 1) != NULL;
+ int is_meth = PyStackRef_AsPyObjectBorrow(PEEK(oparg + 1)) != NULL;
int total_args = oparg + is_meth;
- PyObject *function = PEEK(oparg + 2);
+ PyObject *function = PyStackRef_AsPyObjectBorrow(PEEK(oparg + 2));
PyObject *arg = total_args == 0 ?
- &_PyInstrumentation_MISSING : PEEK(total_args);
+ &_PyInstrumentation_MISSING : PyStackRef_AsPyObjectBorrow(PEEK(total_args));
int err = _Py_call_instrumentation_2args(
tstate, PY_MONITORING_EVENT_CALL,
frame, this_instr, function, arg);
(void)this_instr;
next_instr += 1;
INSTRUCTION_STATS(INSTRUMENTED_CALL_KW);
- int is_meth = PEEK(oparg + 2) != NULL;
+ int is_meth = !PyStackRef_IsNull(PEEK(oparg + 2));
int total_args = oparg + is_meth;
- PyObject *function = PEEK(oparg + 3);
+ PyObject *function = PyStackRef_AsPyObjectBorrow(PEEK(oparg + 3));
PyObject *arg = total_args == 0 ? &_PyInstrumentation_MISSING
- : PEEK(total_args + 1);
+ : PyStackRef_AsPyObjectBorrow(PEEK(total_args + 1));
int err = _Py_call_instrumentation_2args(
tstate, PY_MONITORING_EVENT_CALL,
frame, this_instr, function, arg);
(void)this_instr;
next_instr += 1;
INSTRUCTION_STATS(INSTRUMENTED_END_FOR);
- PyObject *value;
- PyObject *receiver;
+ _PyStackRef value;
+ _PyStackRef receiver;
value = stack_pointer[-1];
receiver = stack_pointer[-2];
/* Need to create a fake StopIteration error here,
* to conform to PEP 380 */
- if (PyGen_Check(receiver)) {
- if (monitor_stop_iteration(tstate, frame, this_instr, value)) {
+ if (PyGen_Check(PyStackRef_AsPyObjectBorrow(receiver))) {
+ if (monitor_stop_iteration(tstate, frame, this_instr, PyStackRef_AsPyObjectBorrow(value))) {
goto error;
}
}
- Py_DECREF(value);
+ PyStackRef_CLOSE(value);
stack_pointer += -1;
assert(WITHIN_STACK_BOUNDS());
DISPATCH();
(void)this_instr;
next_instr += 1;
INSTRUCTION_STATS(INSTRUMENTED_END_SEND);
- PyObject *value;
- PyObject *receiver;
+ _PyStackRef value;
+ _PyStackRef receiver;
value = stack_pointer[-1];
receiver = stack_pointer[-2];
- if (PyGen_Check(receiver) || PyCoro_CheckExact(receiver)) {
- if (monitor_stop_iteration(tstate, frame, this_instr, value)) {
+ PyObject *receiver_o = PyStackRef_AsPyObjectBorrow(receiver);
+ if (PyGen_Check(receiver_o) || PyCoro_CheckExact(receiver_o)) {
+ if (monitor_stop_iteration(tstate, frame, this_instr, PyStackRef_AsPyObjectBorrow(value))) {
goto error;
}
}
- Py_DECREF(receiver);
+ PyStackRef_CLOSE(receiver);
stack_pointer[-2] = value;
stack_pointer += -1;
assert(WITHIN_STACK_BOUNDS());
INSTRUCTION_STATS(INSTRUMENTED_FOR_ITER);
/* Skip 1 cache entry */
_Py_CODEUNIT *target;
- PyObject *iter = TOP();
+ _PyStackRef iter_stackref = TOP();
+ PyObject *iter = PyStackRef_AsPyObjectBorrow(iter_stackref);
PyObject *next = (*Py_TYPE(iter)->tp_iternext)(iter);
if (next != NULL) {
- PUSH(next);
+ PUSH(PyStackRef_FromPyObjectSteal(next));
target = next_instr;
}
else {
assert(next_instr[oparg].op.code == END_FOR ||
next_instr[oparg].op.code == INSTRUMENTED_END_FOR);
STACK_SHRINK(1);
- Py_DECREF(iter);
+ PyStackRef_CLOSE(iter_stackref);
/* Skip END_FOR and POP_TOP */
target = next_instr + oparg + 2;
}
next_instr += 2;
INSTRUCTION_STATS(INSTRUMENTED_POP_JUMP_IF_FALSE);
/* Skip 1 cache entry */
- PyObject *cond = POP();
- assert(PyBool_Check(cond));
- int flag = Py_IsFalse(cond);
+ _PyStackRef cond = POP();
+ assert(PyBool_Check(PyStackRef_AsPyObjectBorrow(cond)));
+ int flag = PyStackRef_Is(cond, PyStackRef_False);
int offset = flag * oparg;
#if ENABLE_SPECIALIZATION
this_instr[1].cache = (this_instr[1].cache << 1) | flag;
next_instr += 2;
INSTRUCTION_STATS(INSTRUMENTED_POP_JUMP_IF_NONE);
/* Skip 1 cache entry */
- PyObject *value = POP();
- int flag = Py_IsNone(value);
+ _PyStackRef value_stackref = POP();
+ int flag = PyStackRef_Is(value_stackref, PyStackRef_None);
int offset;
if (flag) {
offset = oparg;
}
else {
- Py_DECREF(value);
+ PyStackRef_CLOSE(value_stackref);
offset = 0;
}
#if ENABLE_SPECIALIZATION
next_instr += 2;
INSTRUCTION_STATS(INSTRUMENTED_POP_JUMP_IF_NOT_NONE);
/* Skip 1 cache entry */
- PyObject *value = POP();
+ _PyStackRef value_stackref = POP();
int offset;
- int nflag = Py_IsNone(value);
+ int nflag = PyStackRef_Is(value_stackref, PyStackRef_None);
if (nflag) {
offset = 0;
}
else {
- Py_DECREF(value);
+ PyStackRef_CLOSE(value_stackref);
offset = oparg;
}
#if ENABLE_SPECIALIZATION
next_instr += 2;
INSTRUCTION_STATS(INSTRUMENTED_POP_JUMP_IF_TRUE);
/* Skip 1 cache entry */
- PyObject *cond = POP();
- assert(PyBool_Check(cond));
- int flag = Py_IsTrue(cond);
+ _PyStackRef cond = POP();
+ assert(PyBool_Check(PyStackRef_AsPyObjectBorrow(cond)));
+ int flag = PyStackRef_Is(cond, PyStackRef_True);
int offset = flag * oparg;
#if ENABLE_SPECIALIZATION
this_instr[1].cache = (this_instr[1].cache << 1) | flag;
_PyInterpreterFrame *dying = frame;
frame = tstate->current_frame = dying->previous;
_PyEval_FrameClearAndPop(tstate, dying);
- _PyFrame_StackPush(frame, retval);
+ _PyFrame_StackPush(frame, PyStackRef_FromPyObjectSteal(retval));
LOAD_IP(frame->return_offset);
goto resume_frame;
}
(void)this_instr;
next_instr += 1;
INSTRUCTION_STATS(INSTRUMENTED_RETURN_VALUE);
- PyObject *retval;
+ _PyStackRef retval;
retval = stack_pointer[-1];
int err = _Py_call_instrumentation_arg(
tstate, PY_MONITORING_EVENT_PY_RETURN,
- frame, this_instr, retval);
+ frame, this_instr, PyStackRef_AsPyObjectBorrow(retval));
if (err) goto error;
STACK_SHRINK(1);
assert(EMPTY());
(void)this_instr;
next_instr += 1;
INSTRUCTION_STATS(INSTRUMENTED_YIELD_VALUE);
- PyObject *retval;
+ _PyStackRef retval;
retval = stack_pointer[-1];
assert(frame != &entry_frame);
frame->instr_ptr = next_instr;
_PyFrame_SetStackPointer(frame, stack_pointer - 1);
int err = _Py_call_instrumentation_arg(
tstate, PY_MONITORING_EVENT_PY_YIELD,
- frame, this_instr, retval);
+ frame, this_instr, PyStackRef_AsPyObjectBorrow(retval));
if (err) goto error;
tstate->exc_info = gen->gi_exc_state.previous_item;
gen->gi_exc_state.previous_item = NULL;
frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(INTERPRETER_EXIT);
- PyObject *retval;
+ _PyStackRef retval;
retval = stack_pointer[-1];
assert(frame == &entry_frame);
assert(_PyFrame_IsIncomplete(frame));
tstate->current_frame = frame->previous;
assert(!_PyErr_Occurred(tstate));
tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS;
- return retval;
+ return PyStackRef_AsPyObjectSteal(retval);
}
TARGET(IS_OP) {
frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(IS_OP);
- PyObject *right;
- PyObject *left;
- PyObject *b;
+ _PyStackRef right;
+ _PyStackRef left;
+ _PyStackRef b;
right = stack_pointer[-1];
left = stack_pointer[-2];
- int res = Py_Is(left, right) ^ oparg;
- Py_DECREF(left);
- Py_DECREF(right);
- b = res ? Py_True : Py_False;
+ #ifdef Py_GIL_DISABLED
+ // On free-threaded builds, objects are conditionally immortalized.
+ // So their bits don't always compare equally.
+ int res = Py_Is(PyStackRef_AsPyObjectBorrow(left), PyStackRef_AsPyObjectBorrow(right)) ^ oparg;
+ #else
+ int res = PyStackRef_Is(left, right) ^ oparg;
+ #endif
+ PyStackRef_CLOSE(left);
+ PyStackRef_CLOSE(right);
+ b = res ? PyStackRef_True : PyStackRef_False;
stack_pointer[-2] = b;
stack_pointer += -1;
assert(WITHIN_STACK_BOUNDS());
frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(LIST_APPEND);
- PyObject *v;
- PyObject *list;
+ _PyStackRef v;
+ _PyStackRef list;
v = stack_pointer[-1];
list = stack_pointer[-2 - (oparg-1)];
- if (_PyList_AppendTakeRef((PyListObject *)list, v) < 0) goto pop_1_error;
+ if (_PyList_AppendTakeRef((PyListObject *)PyStackRef_AsPyObjectBorrow(list),
+ PyStackRef_AsPyObjectSteal(v)) < 0) goto pop_1_error;
stack_pointer += -1;
assert(WITHIN_STACK_BOUNDS());
DISPATCH();
frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(LIST_EXTEND);
- PyObject *iterable;
- PyObject *list;
- iterable = stack_pointer[-1];
- list = stack_pointer[-2 - (oparg-1)];
+ _PyStackRef iterable_st;
+ _PyStackRef list_st;
+ iterable_st = stack_pointer[-1];
+ list_st = stack_pointer[-2 - (oparg-1)];
+ PyObject *list = PyStackRef_AsPyObjectBorrow(list_st);
+ PyObject *iterable = PyStackRef_AsPyObjectBorrow(iterable_st);
PyObject *none_val = _PyList_Extend((PyListObject *)list, iterable);
if (none_val == NULL) {
if (_PyErr_ExceptionMatches(tstate, PyExc_TypeError) &&
"Value after * must be an iterable, not %.200s",
Py_TYPE(iterable)->tp_name);
}
- Py_DECREF(iterable);
+ PyStackRef_CLOSE(iterable_st);
if (true) goto pop_1_error;
}
assert(Py_IsNone(none_val));
- Py_DECREF(iterable);
+ PyStackRef_CLOSE(iterable_st);
stack_pointer += -1;
assert(WITHIN_STACK_BOUNDS());
DISPATCH();
PREDICTED(LOAD_ATTR);
_Py_CODEUNIT *this_instr = next_instr - 10;
(void)this_instr;
- PyObject *owner;
- PyObject *attr;
- PyObject *self_or_null = NULL;
+ _PyStackRef owner;
+ _PyStackRef attr;
+ _PyStackRef self_or_null = PyStackRef_NULL;
// _SPECIALIZE_LOAD_ATTR
owner = stack_pointer[-1];
{
// _LOAD_ATTR
{
PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 1);
+ PyObject *attr_o;
if (oparg & 1) {
/* Designed to work in tandem with CALL, pushes two values. */
- attr = NULL;
- if (_PyObject_GetMethod(owner, name, &attr)) {
+ attr_o = NULL;
+ if (_PyObject_GetMethod(PyStackRef_AsPyObjectBorrow(owner), name, &attr_o)) {
/* We can bypass temporary bound method object.
meth is unbound method and obj is self.
meth | self | arg1 | ... | argN
*/
- assert(attr != NULL); // No errors on this branch
+ assert(attr_o != NULL); // No errors on this branch
self_or_null = owner; // Transfer ownership
}
else {
CALL that it's not a method call.
meth | NULL | arg1 | ... | argN
*/
- Py_DECREF(owner);
- if (attr == NULL) goto pop_1_error;
- self_or_null = NULL;
+ PyStackRef_CLOSE(owner);
+ if (attr_o == NULL) goto pop_1_error;
+ self_or_null = PyStackRef_NULL;
}
}
else {
/* Classic, pushes one value. */
- attr = PyObject_GetAttr(owner, name);
- Py_DECREF(owner);
- if (attr == NULL) goto pop_1_error;
+ attr_o = PyObject_GetAttr(PyStackRef_AsPyObjectBorrow(owner), name);
+ PyStackRef_CLOSE(owner);
+ if (attr_o == NULL) goto pop_1_error;
}
+ attr = PyStackRef_FromPyObjectSteal(attr_o);
}
stack_pointer[-1] = attr;
if (oparg & 1) stack_pointer[0] = self_or_null;
next_instr += 10;
INSTRUCTION_STATS(LOAD_ATTR_CLASS);
static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size");
- PyObject *owner;
- PyObject *attr;
- PyObject *null = NULL;
+ _PyStackRef owner;
+ _PyStackRef attr;
+ _PyStackRef null = PyStackRef_NULL;
/* Skip 1 cache entry */
// _CHECK_ATTR_CLASS
owner = stack_pointer[-1];
{
uint32_t type_version = read_u32(&this_instr[2].cache);
- DEOPT_IF(!PyType_Check(owner), LOAD_ATTR);
+ PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner);
+ DEOPT_IF(!PyType_Check(owner_o), LOAD_ATTR);
assert(type_version != 0);
- DEOPT_IF(((PyTypeObject *)owner)->tp_version_tag != type_version, LOAD_ATTR);
+ DEOPT_IF(((PyTypeObject *)owner_o)->tp_version_tag != type_version, LOAD_ATTR);
}
/* Skip 2 cache entries */
// _LOAD_ATTR_CLASS
PyObject *descr = read_obj(&this_instr[6].cache);
STAT_INC(LOAD_ATTR, hit);
assert(descr != NULL);
- attr = Py_NewRef(descr);
- null = NULL;
- Py_DECREF(owner);
+ attr = PyStackRef_FromPyObjectNew(descr);
+ null = PyStackRef_NULL;
+ PyStackRef_CLOSE(owner);
}
stack_pointer[-1] = attr;
if (oparg & 1) stack_pointer[0] = null;
next_instr += 10;
INSTRUCTION_STATS(LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN);
static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size");
- PyObject *owner;
+ _PyStackRef owner;
/* Skip 1 cache entry */
owner = stack_pointer[-1];
uint32_t type_version = read_u32(&this_instr[2].cache);
uint32_t func_version = read_u32(&this_instr[4].cache);
PyObject *getattribute = read_obj(&this_instr[6].cache);
+ PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner);
assert((oparg & 1) == 0);
DEOPT_IF(tstate->interp->eval_frame, LOAD_ATTR);
- PyTypeObject *cls = Py_TYPE(owner);
+ PyTypeObject *cls = Py_TYPE(owner_o);
assert(type_version != 0);
DEOPT_IF(cls->tp_version_tag != type_version, LOAD_ATTR);
assert(Py_IS_TYPE(getattribute, &PyFunction_Type));
// Manipulate stack directly because we exit with DISPATCH_INLINED().
STACK_SHRINK(1);
new_frame->localsplus[0] = owner;
- new_frame->localsplus[1] = Py_NewRef(name);
+ new_frame->localsplus[1] = PyStackRef_FromPyObjectNew(name);
frame->return_offset = (uint16_t)(next_instr - this_instr);
DISPATCH_INLINED(new_frame);
}
next_instr += 10;
INSTRUCTION_STATS(LOAD_ATTR_INSTANCE_VALUE);
static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size");
- PyObject *owner;
- PyObject *attr;
- PyObject *null = NULL;
+ _PyStackRef owner;
+ _PyStackRef attr;
+ _PyStackRef null = PyStackRef_NULL;
/* Skip 1 cache entry */
// _GUARD_TYPE_VERSION
owner = stack_pointer[-1];
{
uint32_t type_version = read_u32(&this_instr[2].cache);
- PyTypeObject *tp = Py_TYPE(owner);
+ PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner));
assert(type_version != 0);
DEOPT_IF(tp->tp_version_tag != type_version, LOAD_ATTR);
}
// _CHECK_MANAGED_OBJECT_HAS_VALUES
{
- assert(Py_TYPE(owner)->tp_dictoffset < 0);
- assert(Py_TYPE(owner)->tp_flags & Py_TPFLAGS_INLINE_VALUES);
- DEOPT_IF(!_PyObject_InlineValues(owner)->valid, LOAD_ATTR);
+ PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner);
+ assert(Py_TYPE(owner_o)->tp_dictoffset < 0);
+ assert(Py_TYPE(owner_o)->tp_flags & Py_TPFLAGS_INLINE_VALUES);
+ DEOPT_IF(!_PyObject_InlineValues(owner_o)->valid, LOAD_ATTR);
}
// _LOAD_ATTR_INSTANCE_VALUE
{
uint16_t index = read_u16(&this_instr[4].cache);
- attr = _PyObject_InlineValues(owner)->values[index];
- DEOPT_IF(attr == NULL, LOAD_ATTR);
+ PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner);
+ PyObject *attr_o = _PyObject_InlineValues(owner_o)->values[index];
+ DEOPT_IF(attr_o == NULL, LOAD_ATTR);
STAT_INC(LOAD_ATTR, hit);
- Py_INCREF(attr);
- null = NULL;
- Py_DECREF(owner);
+ Py_INCREF(attr_o);
+ null = PyStackRef_NULL;
+ attr = PyStackRef_FromPyObjectSteal(attr_o);
+ PyStackRef_CLOSE(owner);
}
/* Skip 5 cache entries */
stack_pointer[-1] = attr;
next_instr += 10;
INSTRUCTION_STATS(LOAD_ATTR_METHOD_LAZY_DICT);
static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size");
- PyObject *owner;
- PyObject *attr;
- PyObject *self = NULL;
+ _PyStackRef owner;
+ _PyStackRef attr;
+ _PyStackRef self = PyStackRef_NULL;
/* Skip 1 cache entry */
// _GUARD_TYPE_VERSION
owner = stack_pointer[-1];
{
uint32_t type_version = read_u32(&this_instr[2].cache);
- PyTypeObject *tp = Py_TYPE(owner);
+ PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner));
assert(type_version != 0);
DEOPT_IF(tp->tp_version_tag != type_version, LOAD_ATTR);
}
// _CHECK_ATTR_METHOD_LAZY_DICT
{
uint16_t dictoffset = read_u16(&this_instr[4].cache);
- char *ptr = ((char *)owner) + MANAGED_DICT_OFFSET + dictoffset;
+ char *ptr = ((char *)PyStackRef_AsPyObjectBorrow(owner)) + MANAGED_DICT_OFFSET + dictoffset;
PyObject *dict = *(PyObject **)ptr;
/* This object has a __dict__, just not yet created */
DEOPT_IF(dict != NULL, LOAD_ATTR);
STAT_INC(LOAD_ATTR, hit);
assert(descr != NULL);
assert(_PyType_HasFeature(Py_TYPE(descr), Py_TPFLAGS_METHOD_DESCRIPTOR));
- attr = Py_NewRef(descr);
+ attr = PyStackRef_FromPyObjectNew(descr);
self = owner;
}
stack_pointer[-1] = attr;
next_instr += 10;
INSTRUCTION_STATS(LOAD_ATTR_METHOD_NO_DICT);
static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size");
- PyObject *owner;
- PyObject *attr;
- PyObject *self = NULL;
+ _PyStackRef owner;
+ _PyStackRef attr;
+ _PyStackRef self = PyStackRef_NULL;
/* Skip 1 cache entry */
// _GUARD_TYPE_VERSION
owner = stack_pointer[-1];
{
uint32_t type_version = read_u32(&this_instr[2].cache);
- PyTypeObject *tp = Py_TYPE(owner);
+ PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner));
assert(type_version != 0);
DEOPT_IF(tp->tp_version_tag != type_version, LOAD_ATTR);
}
{
PyObject *descr = read_obj(&this_instr[6].cache);
assert(oparg & 1);
- assert(Py_TYPE(owner)->tp_dictoffset == 0);
+ assert(Py_TYPE(PyStackRef_AsPyObjectBorrow(owner))->tp_dictoffset == 0);
STAT_INC(LOAD_ATTR, hit);
assert(descr != NULL);
assert(_PyType_HasFeature(Py_TYPE(descr), Py_TPFLAGS_METHOD_DESCRIPTOR));
- attr = Py_NewRef(descr);
+ attr = PyStackRef_FromPyObjectNew(descr);
self = owner;
}
stack_pointer[-1] = attr;
next_instr += 10;
INSTRUCTION_STATS(LOAD_ATTR_METHOD_WITH_VALUES);
static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size");
- PyObject *owner;
- PyObject *attr;
- PyObject *self = NULL;
+ _PyStackRef owner;
+ _PyStackRef attr;
+ _PyStackRef self = PyStackRef_NULL;
/* Skip 1 cache entry */
// _GUARD_TYPE_VERSION
owner = stack_pointer[-1];
{
uint32_t type_version = read_u32(&this_instr[2].cache);
- PyTypeObject *tp = Py_TYPE(owner);
+ PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner));
assert(type_version != 0);
DEOPT_IF(tp->tp_version_tag != type_version, LOAD_ATTR);
}
// _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT
{
- assert(Py_TYPE(owner)->tp_flags & Py_TPFLAGS_INLINE_VALUES);
- DEOPT_IF(!_PyObject_InlineValues(owner)->valid, LOAD_ATTR);
+ PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner);
+ assert(Py_TYPE(owner_o)->tp_flags & Py_TPFLAGS_INLINE_VALUES);
+ DEOPT_IF(!_PyObject_InlineValues(owner_o)->valid, LOAD_ATTR);
}
// _GUARD_KEYS_VERSION
{
uint32_t keys_version = read_u32(&this_instr[4].cache);
- PyTypeObject *owner_cls = Py_TYPE(owner);
+ PyTypeObject *owner_cls = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner));
PyHeapTypeObject *owner_heap_type = (PyHeapTypeObject *)owner_cls;
DEOPT_IF(owner_heap_type->ht_cached_keys->dk_version != keys_version, LOAD_ATTR);
}
/* Cached method object */
STAT_INC(LOAD_ATTR, hit);
assert(descr != NULL);
- attr = Py_NewRef(descr);
- assert(_PyType_HasFeature(Py_TYPE(attr), Py_TPFLAGS_METHOD_DESCRIPTOR));
+ assert(_PyType_HasFeature(Py_TYPE(descr), Py_TPFLAGS_METHOD_DESCRIPTOR));
+ attr = PyStackRef_FromPyObjectNew(descr);
self = owner;
}
stack_pointer[-1] = attr;
next_instr += 10;
INSTRUCTION_STATS(LOAD_ATTR_MODULE);
static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size");
- PyObject *owner;
- PyObject *attr;
- PyObject *null = NULL;
+ _PyStackRef owner;
+ _PyStackRef attr;
+ _PyStackRef null = PyStackRef_NULL;
/* Skip 1 cache entry */
// _CHECK_ATTR_MODULE
owner = stack_pointer[-1];
{
uint32_t dict_version = read_u32(&this_instr[2].cache);
- DEOPT_IF(!PyModule_CheckExact(owner), LOAD_ATTR);
- PyDictObject *dict = (PyDictObject *)((PyModuleObject *)owner)->md_dict;
+ PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner);
+ DEOPT_IF(!PyModule_CheckExact(owner_o), LOAD_ATTR);
+ PyDictObject *dict = (PyDictObject *)((PyModuleObject *)owner_o)->md_dict;
assert(dict != NULL);
DEOPT_IF(dict->ma_keys->dk_version != dict_version, LOAD_ATTR);
}
// _LOAD_ATTR_MODULE
{
uint16_t index = read_u16(&this_instr[4].cache);
- PyDictObject *dict = (PyDictObject *)((PyModuleObject *)owner)->md_dict;
+ PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner);
+ PyDictObject *dict = (PyDictObject *)((PyModuleObject *)owner_o)->md_dict;
assert(dict->ma_keys->dk_kind == DICT_KEYS_UNICODE);
assert(index < dict->ma_keys->dk_nentries);
PyDictUnicodeEntry *ep = DK_UNICODE_ENTRIES(dict->ma_keys) + index;
- attr = ep->me_value;
- DEOPT_IF(attr == NULL, LOAD_ATTR);
+ PyObject *attr_o = ep->me_value;
+ DEOPT_IF(attr_o == NULL, LOAD_ATTR);
STAT_INC(LOAD_ATTR, hit);
- Py_INCREF(attr);
- null = NULL;
- Py_DECREF(owner);
+ Py_INCREF(attr_o);
+ attr = PyStackRef_FromPyObjectSteal(attr_o);
+ null = PyStackRef_NULL;
+ PyStackRef_CLOSE(owner);
}
/* Skip 5 cache entries */
stack_pointer[-1] = attr;
next_instr += 10;
INSTRUCTION_STATS(LOAD_ATTR_NONDESCRIPTOR_NO_DICT);
static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size");
- PyObject *owner;
- PyObject *attr;
+ _PyStackRef owner;
+ _PyStackRef attr;
/* Skip 1 cache entry */
// _GUARD_TYPE_VERSION
owner = stack_pointer[-1];
{
uint32_t type_version = read_u32(&this_instr[2].cache);
- PyTypeObject *tp = Py_TYPE(owner);
+ PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner));
assert(type_version != 0);
DEOPT_IF(tp->tp_version_tag != type_version, LOAD_ATTR);
}
{
PyObject *descr = read_obj(&this_instr[6].cache);
assert((oparg & 1) == 0);
- assert(Py_TYPE(owner)->tp_dictoffset == 0);
+ assert(Py_TYPE(PyStackRef_AsPyObjectBorrow(owner))->tp_dictoffset == 0);
STAT_INC(LOAD_ATTR, hit);
assert(descr != NULL);
- Py_DECREF(owner);
- attr = Py_NewRef(descr);
+ PyStackRef_CLOSE(owner);
+ attr = PyStackRef_FromPyObjectNew(descr);
}
stack_pointer[-1] = attr;
DISPATCH();
next_instr += 10;
INSTRUCTION_STATS(LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES);
static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size");
- PyObject *owner;
- PyObject *attr;
+ _PyStackRef owner;
+ _PyStackRef attr;
/* Skip 1 cache entry */
// _GUARD_TYPE_VERSION
owner = stack_pointer[-1];
{
uint32_t type_version = read_u32(&this_instr[2].cache);
- PyTypeObject *tp = Py_TYPE(owner);
+ PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner));
assert(type_version != 0);
DEOPT_IF(tp->tp_version_tag != type_version, LOAD_ATTR);
}
// _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT
{
- assert(Py_TYPE(owner)->tp_flags & Py_TPFLAGS_INLINE_VALUES);
- DEOPT_IF(!_PyObject_InlineValues(owner)->valid, LOAD_ATTR);
+ PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner);
+ assert(Py_TYPE(owner_o)->tp_flags & Py_TPFLAGS_INLINE_VALUES);
+ DEOPT_IF(!_PyObject_InlineValues(owner_o)->valid, LOAD_ATTR);
}
// _GUARD_KEYS_VERSION
{
uint32_t keys_version = read_u32(&this_instr[4].cache);
- PyTypeObject *owner_cls = Py_TYPE(owner);
+ PyTypeObject *owner_cls = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner));
PyHeapTypeObject *owner_heap_type = (PyHeapTypeObject *)owner_cls;
DEOPT_IF(owner_heap_type->ht_cached_keys->dk_version != keys_version, LOAD_ATTR);
}
assert((oparg & 1) == 0);
STAT_INC(LOAD_ATTR, hit);
assert(descr != NULL);
- Py_DECREF(owner);
- attr = Py_NewRef(descr);
+ PyStackRef_CLOSE(owner);
+ attr = PyStackRef_FromPyObjectNew(descr);
}
stack_pointer[-1] = attr;
DISPATCH();
next_instr += 10;
INSTRUCTION_STATS(LOAD_ATTR_PROPERTY);
static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size");
- PyObject *owner;
+ _PyStackRef owner;
/* Skip 1 cache entry */
owner = stack_pointer[-1];
uint32_t type_version = read_u32(&this_instr[2].cache);
uint32_t func_version = read_u32(&this_instr[4].cache);
PyObject *fget = read_obj(&this_instr[6].cache);
+ PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner);
assert((oparg & 1) == 0);
DEOPT_IF(tstate->interp->eval_frame, LOAD_ATTR);
- PyTypeObject *cls = Py_TYPE(owner);
+ PyTypeObject *cls = Py_TYPE(owner_o);
assert(type_version != 0);
DEOPT_IF(cls->tp_version_tag != type_version, LOAD_ATTR);
assert(Py_IS_TYPE(fget, &PyFunction_Type));
next_instr += 10;
INSTRUCTION_STATS(LOAD_ATTR_SLOT);
static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size");
- PyObject *owner;
- PyObject *attr;
- PyObject *null = NULL;
+ _PyStackRef owner;
+ _PyStackRef attr;
+ _PyStackRef null = PyStackRef_NULL;
/* Skip 1 cache entry */
// _GUARD_TYPE_VERSION
owner = stack_pointer[-1];
{
uint32_t type_version = read_u32(&this_instr[2].cache);
- PyTypeObject *tp = Py_TYPE(owner);
+ PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner));
assert(type_version != 0);
DEOPT_IF(tp->tp_version_tag != type_version, LOAD_ATTR);
}
// _LOAD_ATTR_SLOT
{
uint16_t index = read_u16(&this_instr[4].cache);
- char *addr = (char *)owner + index;
- attr = *(PyObject **)addr;
- DEOPT_IF(attr == NULL, LOAD_ATTR);
+ PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner);
+ char *addr = (char *)owner_o + index;
+ PyObject *attr_o = *(PyObject **)addr;
+ DEOPT_IF(attr_o == NULL, LOAD_ATTR);
STAT_INC(LOAD_ATTR, hit);
- Py_INCREF(attr);
- null = NULL;
- Py_DECREF(owner);
+ null = PyStackRef_NULL;
+ attr = PyStackRef_FromPyObjectNew(attr_o);
+ PyStackRef_CLOSE(owner);
}
/* Skip 5 cache entries */
stack_pointer[-1] = attr;
next_instr += 10;
INSTRUCTION_STATS(LOAD_ATTR_WITH_HINT);
static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size");
- PyObject *owner;
- PyObject *attr;
- PyObject *null = NULL;
+ _PyStackRef owner;
+ _PyStackRef attr;
+ _PyStackRef null = PyStackRef_NULL;
/* Skip 1 cache entry */
// _GUARD_TYPE_VERSION
owner = stack_pointer[-1];
{
uint32_t type_version = read_u32(&this_instr[2].cache);
- PyTypeObject *tp = Py_TYPE(owner);
+ PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner));
assert(type_version != 0);
DEOPT_IF(tp->tp_version_tag != type_version, LOAD_ATTR);
}
// _CHECK_ATTR_WITH_HINT
{
- assert(Py_TYPE(owner)->tp_flags & Py_TPFLAGS_MANAGED_DICT);
- PyDictObject *dict = _PyObject_GetManagedDict(owner);
+ PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner);
+ assert(Py_TYPE(owner_o)->tp_flags & Py_TPFLAGS_MANAGED_DICT);
+ PyDictObject *dict = _PyObject_GetManagedDict(owner_o);
DEOPT_IF(dict == NULL, LOAD_ATTR);
assert(PyDict_CheckExact((PyObject *)dict));
}
// _LOAD_ATTR_WITH_HINT
{
uint16_t hint = read_u16(&this_instr[4].cache);
- PyDictObject *dict = _PyObject_GetManagedDict(owner);
+ PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner);
+ PyObject *attr_o;
+ PyDictObject *dict = _PyObject_GetManagedDict(owner_o);
DEOPT_IF(hint >= (size_t)dict->ma_keys->dk_nentries, LOAD_ATTR);
PyObject *name = GETITEM(FRAME_CO_NAMES, oparg>>1);
if (DK_IS_UNICODE(dict->ma_keys)) {
PyDictUnicodeEntry *ep = DK_UNICODE_ENTRIES(dict->ma_keys) + hint;
DEOPT_IF(ep->me_key != name, LOAD_ATTR);
- attr = ep->me_value;
+ attr_o = ep->me_value;
}
else {
PyDictKeyEntry *ep = DK_ENTRIES(dict->ma_keys) + hint;
DEOPT_IF(ep->me_key != name, LOAD_ATTR);
- attr = ep->me_value;
+ attr_o = ep->me_value;
}
- DEOPT_IF(attr == NULL, LOAD_ATTR);
+ DEOPT_IF(attr_o == NULL, LOAD_ATTR);
STAT_INC(LOAD_ATTR, hit);
- Py_INCREF(attr);
- null = NULL;
- Py_DECREF(owner);
+ Py_INCREF(attr_o);
+ attr = PyStackRef_FromPyObjectSteal(attr_o);
+ null = PyStackRef_NULL;
+ PyStackRef_CLOSE(owner);
}
/* Skip 5 cache entries */
stack_pointer[-1] = attr;
frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(LOAD_BUILD_CLASS);
- PyObject *bc;
- if (PyMapping_GetOptionalItem(BUILTINS(), &_Py_ID(__build_class__), &bc) < 0) goto error;
- if (bc == NULL) {
+ _PyStackRef bc;
+ PyObject *bc_o;
+ if (PyMapping_GetOptionalItem(BUILTINS(), &_Py_ID(__build_class__), &bc_o) < 0) goto error;
+ if (bc_o == NULL) {
_PyErr_SetString(tstate, PyExc_NameError,
"__build_class__ not found");
if (true) goto error;
}
+ bc = PyStackRef_FromPyObjectSteal(bc_o);
stack_pointer[0] = bc;
stack_pointer += 1;
assert(WITHIN_STACK_BOUNDS());
frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(LOAD_COMMON_CONSTANT);
- PyObject *value;
+ _PyStackRef value;
// Keep in sync with _common_constants in opcode.py
switch(oparg) {
case CONSTANT_ASSERTIONERROR:
- value = PyExc_AssertionError;
+ value = PyStackRef_FromPyObjectImmortal(PyExc_AssertionError);
break;
case CONSTANT_NOTIMPLEMENTEDERROR:
- value = PyExc_NotImplementedError;
+ value = PyStackRef_FromPyObjectImmortal(PyExc_NotImplementedError);
break;
default:
Py_FatalError("bad LOAD_COMMON_CONSTANT oparg");
frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(LOAD_CONST);
- PyObject *value;
- value = GETITEM(FRAME_CO_CONSTS, oparg);
- Py_INCREF(value);
+ _PyStackRef value;
+ value = PyStackRef_FromPyObjectNew(GETITEM(FRAME_CO_CONSTS, oparg));
stack_pointer[0] = value;
stack_pointer += 1;
assert(WITHIN_STACK_BOUNDS());
frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(LOAD_DEREF);
- PyObject *value;
- PyCellObject *cell = (PyCellObject *)GETLOCAL(oparg);
- value = PyCell_GetRef(cell);
- if (value == NULL) {
+ _PyStackRef value;
+ PyCellObject *cell = (PyCellObject *)PyStackRef_AsPyObjectBorrow(GETLOCAL(oparg));
+ PyObject *value_o = PyCell_GetRef(cell);
+ if (value_o == NULL) {
_PyEval_FormatExcUnbound(tstate, _PyFrame_GetCode(frame), oparg);
if (true) goto error;
}
+ value = PyStackRef_FromPyObjectSteal(value_o);
stack_pointer[0] = value;
stack_pointer += 1;
assert(WITHIN_STACK_BOUNDS());
frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(LOAD_FAST);
- PyObject *value;
- value = GETLOCAL(oparg);
- assert(value != NULL);
- Py_INCREF(value);
+ _PyStackRef value;
+ assert(PyStackRef_AsPyObjectBorrow(GETLOCAL(oparg)) != NULL);
+ value = PyStackRef_DUP(GETLOCAL(oparg));
stack_pointer[0] = value;
stack_pointer += 1;
assert(WITHIN_STACK_BOUNDS());
frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(LOAD_FAST_AND_CLEAR);
- PyObject *value;
+ _PyStackRef value;
value = GETLOCAL(oparg);
// do not use SETLOCAL here, it decrefs the old value
- GETLOCAL(oparg) = NULL;
+ GETLOCAL(oparg) = PyStackRef_NULL;
stack_pointer[0] = value;
stack_pointer += 1;
assert(WITHIN_STACK_BOUNDS());
frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(LOAD_FAST_CHECK);
- PyObject *value;
- value = GETLOCAL(oparg);
- if (value == NULL) {
+ _PyStackRef value;
+ _PyStackRef value_s = GETLOCAL(oparg);
+ if (PyStackRef_IsNull(value_s)) {
_PyEval_FormatExcCheckArg(tstate, PyExc_UnboundLocalError,
UNBOUNDLOCAL_ERROR_MSG,
PyTuple_GetItem(_PyFrame_GetCode(frame)->co_localsplusnames, oparg)
);
if (1) goto error;
}
- Py_INCREF(value);
+ value = PyStackRef_DUP(value_s);
stack_pointer[0] = value;
stack_pointer += 1;
assert(WITHIN_STACK_BOUNDS());
frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(LOAD_FAST_LOAD_FAST);
- PyObject *value1;
- PyObject *value2;
+ _PyStackRef value1;
+ _PyStackRef value2;
uint32_t oparg1 = oparg >> 4;
uint32_t oparg2 = oparg & 15;
- value1 = GETLOCAL(oparg1);
- value2 = GETLOCAL(oparg2);
- Py_INCREF(value1);
- Py_INCREF(value2);
+ value1 = PyStackRef_DUP(GETLOCAL(oparg1));
+ value2 = PyStackRef_DUP(GETLOCAL(oparg2));
stack_pointer[0] = value1;
stack_pointer[1] = value2;
stack_pointer += 2;
frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(LOAD_FROM_DICT_OR_DEREF);
- PyObject *class_dict;
- PyObject *value;
- class_dict = stack_pointer[-1];
+ _PyStackRef class_dict_st;
+ _PyStackRef value;
+ class_dict_st = stack_pointer[-1];
+ PyObject *value_o;
PyObject *name;
+ PyObject *class_dict = PyStackRef_AsPyObjectBorrow(class_dict_st);
assert(class_dict);
assert(oparg >= 0 && oparg < _PyFrame_GetCode(frame)->co_nlocalsplus);
name = PyTuple_GET_ITEM(_PyFrame_GetCode(frame)->co_localsplusnames, oparg);
- if (PyMapping_GetOptionalItem(class_dict, name, &value) < 0) {
+ if (PyMapping_GetOptionalItem(class_dict, name, &value_o) < 0) {
goto error;
}
- if (!value) {
- PyCellObject *cell = (PyCellObject *)GETLOCAL(oparg);
- value = PyCell_GetRef(cell);
- if (value == NULL) {
+ if (!value_o) {
+ PyCellObject *cell = (PyCellObject *)PyStackRef_AsPyObjectBorrow(GETLOCAL(oparg));
+ value_o = PyCell_GetRef(cell);
+ if (value_o == NULL) {
_PyEval_FormatExcUnbound(tstate, _PyFrame_GetCode(frame), oparg);
goto error;
}
}
- Py_DECREF(class_dict);
+ PyStackRef_CLOSE(class_dict_st);
+ value = PyStackRef_FromPyObjectSteal(value_o);
stack_pointer[-1] = value;
DISPATCH();
}
frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(LOAD_FROM_DICT_OR_GLOBALS);
- PyObject *mod_or_class_dict;
- PyObject *v;
+ _PyStackRef mod_or_class_dict;
+ _PyStackRef v;
mod_or_class_dict = stack_pointer[-1];
PyObject *name = GETITEM(FRAME_CO_NAMES, oparg);
- if (PyMapping_GetOptionalItem(mod_or_class_dict, name, &v) < 0) {
+ PyObject *v_o;
+ if (PyMapping_GetOptionalItem(PyStackRef_AsPyObjectBorrow(mod_or_class_dict), name, &v_o) < 0) {
goto error;
}
- if (v == NULL) {
+ if (v_o == NULL) {
if (PyDict_CheckExact(GLOBALS())
&& PyDict_CheckExact(BUILTINS()))
{
- v = _PyDict_LoadGlobal((PyDictObject *)GLOBALS(),
+ v_o = _PyDict_LoadGlobal((PyDictObject *)GLOBALS(),
(PyDictObject *)BUILTINS(),
name);
- if (v == NULL) {
+ if (v_o == NULL) {
if (!_PyErr_Occurred(tstate)) {
/* _PyDict_LoadGlobal() returns NULL without raising
* an exception if the key doesn't exist */
else {
/* Slow-path if globals or builtins is not a dict */
/* namespace 1: globals */
- if (PyMapping_GetOptionalItem(GLOBALS(), name, &v) < 0) goto pop_1_error;
- if (v == NULL) {
+ if (PyMapping_GetOptionalItem(GLOBALS(), name, &v_o) < 0) goto pop_1_error;
+ if (v_o == NULL) {
/* namespace 2: builtins */
- if (PyMapping_GetOptionalItem(BUILTINS(), name, &v) < 0) goto pop_1_error;
- if (v == NULL) {
+ if (PyMapping_GetOptionalItem(BUILTINS(), name, &v_o) < 0) goto pop_1_error;
+ if (v_o == NULL) {
_PyEval_FormatExcCheckArg(
tstate, PyExc_NameError,
NAME_ERROR_MSG, name);
}
}
}
- Py_DECREF(mod_or_class_dict);
+ PyStackRef_CLOSE(mod_or_class_dict);
+ v = PyStackRef_FromPyObjectSteal(v_o);
stack_pointer[-1] = v;
DISPATCH();
}
PREDICTED(LOAD_GLOBAL);
_Py_CODEUNIT *this_instr = next_instr - 5;
(void)this_instr;
- PyObject *res;
- PyObject *null = NULL;
+ _PyStackRef res;
+ _PyStackRef null = PyStackRef_NULL;
// _SPECIALIZE_LOAD_GLOBAL
{
uint16_t counter = read_u16(&this_instr[1].cache);
// _LOAD_GLOBAL
{
PyObject *name = GETITEM(FRAME_CO_NAMES, oparg>>1);
+ PyObject *res_o;
if (PyDict_CheckExact(GLOBALS())
&& PyDict_CheckExact(BUILTINS()))
{
- res = _PyDict_LoadGlobal((PyDictObject *)GLOBALS(),
+ res_o = _PyDict_LoadGlobal((PyDictObject *)GLOBALS(),
(PyDictObject *)BUILTINS(),
name);
- if (res == NULL) {
+ if (res_o == NULL) {
if (!_PyErr_Occurred(tstate)) {
/* _PyDict_LoadGlobal() returns NULL without raising
* an exception if the key doesn't exist */
else {
/* Slow-path if globals or builtins is not a dict */
/* namespace 1: globals */
- if (PyMapping_GetOptionalItem(GLOBALS(), name, &res) < 0) goto error;
- if (res == NULL) {
+ if (PyMapping_GetOptionalItem(GLOBALS(), name, &res_o) < 0) goto error;
+ if (res_o == NULL) {
/* namespace 2: builtins */
- if (PyMapping_GetOptionalItem(BUILTINS(), name, &res) < 0) goto error;
- if (res == NULL) {
+ if (PyMapping_GetOptionalItem(BUILTINS(), name, &res_o) < 0) goto error;
+ if (res_o == NULL) {
_PyEval_FormatExcCheckArg(
tstate, PyExc_NameError,
NAME_ERROR_MSG, name);
}
}
}
- null = NULL;
+ null = PyStackRef_NULL;
+ res = PyStackRef_FromPyObjectSteal(res_o);
}
stack_pointer[0] = res;
if (oparg & 1) stack_pointer[1] = null;
next_instr += 5;
INSTRUCTION_STATS(LOAD_GLOBAL_BUILTIN);
static_assert(INLINE_CACHE_ENTRIES_LOAD_GLOBAL == 4, "incorrect cache size");
- PyObject *res;
- PyObject *null = NULL;
+ _PyStackRef res;
+ _PyStackRef null = PyStackRef_NULL;
/* Skip 1 cache entry */
// _GUARD_GLOBALS_VERSION
{
uint16_t index = read_u16(&this_instr[4].cache);
PyDictObject *bdict = (PyDictObject *)BUILTINS();
PyDictUnicodeEntry *entries = DK_UNICODE_ENTRIES(bdict->ma_keys);
- res = entries[index].me_value;
- DEOPT_IF(res == NULL, LOAD_GLOBAL);
- Py_INCREF(res);
+ PyObject *res_o = entries[index].me_value;
+ DEOPT_IF(res_o == NULL, LOAD_GLOBAL);
+ Py_INCREF(res_o);
STAT_INC(LOAD_GLOBAL, hit);
- null = NULL;
+ null = PyStackRef_NULL;
+ res = PyStackRef_FromPyObjectSteal(res_o);
}
stack_pointer[0] = res;
if (oparg & 1) stack_pointer[1] = null;
next_instr += 5;
INSTRUCTION_STATS(LOAD_GLOBAL_MODULE);
static_assert(INLINE_CACHE_ENTRIES_LOAD_GLOBAL == 4, "incorrect cache size");
- PyObject *res;
- PyObject *null = NULL;
+ _PyStackRef res;
+ _PyStackRef null = PyStackRef_NULL;
/* Skip 1 cache entry */
// _GUARD_GLOBALS_VERSION
{
uint16_t index = read_u16(&this_instr[4].cache);
PyDictObject *dict = (PyDictObject *)GLOBALS();
PyDictUnicodeEntry *entries = DK_UNICODE_ENTRIES(dict->ma_keys);
- res = entries[index].me_value;
- DEOPT_IF(res == NULL, LOAD_GLOBAL);
- Py_INCREF(res);
+ PyObject *res_o = entries[index].me_value;
+ DEOPT_IF(res_o == NULL, LOAD_GLOBAL);
+ Py_INCREF(res_o);
STAT_INC(LOAD_GLOBAL, hit);
- null = NULL;
+ null = PyStackRef_NULL;
+ res = PyStackRef_FromPyObjectSteal(res_o);
}
stack_pointer[0] = res;
if (oparg & 1) stack_pointer[1] = null;
frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(LOAD_LOCALS);
- PyObject *locals;
- locals = LOCALS();
- if (locals == NULL) {
+ _PyStackRef locals;
+ PyObject *l = LOCALS();
+ if (l == NULL) {
_PyErr_SetString(tstate, PyExc_SystemError,
"no locals found");
if (true) goto error;
}
- Py_INCREF(locals);
+ locals = PyStackRef_FromPyObjectNew(l);;
stack_pointer[0] = locals;
stack_pointer += 1;
assert(WITHIN_STACK_BOUNDS());
frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(LOAD_NAME);
- PyObject *v;
+ _PyStackRef v;
+ PyObject *v_o;
PyObject *mod_or_class_dict = LOCALS();
if (mod_or_class_dict == NULL) {
_PyErr_SetString(tstate, PyExc_SystemError,
if (true) goto error;
}
PyObject *name = GETITEM(FRAME_CO_NAMES, oparg);
- if (PyMapping_GetOptionalItem(mod_or_class_dict, name, &v) < 0) {
+ if (PyMapping_GetOptionalItem(mod_or_class_dict, name, &v_o) < 0) {
goto error;
}
- if (v == NULL) {
- if (PyDict_GetItemRef(GLOBALS(), name, &v) < 0) {
+ if (v_o == NULL) {
+ if (PyDict_GetItemRef(GLOBALS(), name, &v_o) < 0) {
goto error;
}
- if (v == NULL) {
- if (PyMapping_GetOptionalItem(BUILTINS(), name, &v) < 0) {
+ if (v_o == NULL) {
+ if (PyMapping_GetOptionalItem(BUILTINS(), name, &v_o) < 0) {
goto error;
}
- if (v == NULL) {
+ if (v_o == NULL) {
_PyEval_FormatExcCheckArg(
tstate, PyExc_NameError,
NAME_ERROR_MSG, name);
}
}
}
+ v = PyStackRef_FromPyObjectSteal(v_o);
stack_pointer[0] = v;
stack_pointer += 1;
assert(WITHIN_STACK_BOUNDS());
frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(LOAD_SPECIAL);
- PyObject *owner;
- PyObject *attr;
- PyObject *self_or_null;
+ _PyStackRef owner;
+ _PyStackRef attr;
+ _PyStackRef self_or_null;
owner = stack_pointer[-1];
assert(oparg <= SPECIAL_MAX);
+ PyObject *owner_o = PyStackRef_AsPyObjectSteal(owner);
PyObject *name = _Py_SpecialMethods[oparg].name;
- attr = _PyObject_LookupSpecialMethod(owner, name, &self_or_null);
- if (attr == NULL) {
+ PyObject *self_or_null_o;
+ attr = PyStackRef_FromPyObjectSteal(_PyObject_LookupSpecialMethod(owner_o, name, &self_or_null_o));
+ if (PyStackRef_IsNull(attr)) {
if (!_PyErr_Occurred(tstate)) {
_PyErr_Format(tstate, PyExc_TypeError,
_Py_SpecialMethods[oparg].error,
- Py_TYPE(owner)->tp_name);
+ Py_TYPE(owner_o)->tp_name);
}
}
- if (attr == NULL) goto pop_1_error;
+ if (PyStackRef_IsNull(attr)) goto pop_1_error;
+ self_or_null = PyStackRef_FromPyObjectSteal(self_or_null_o);
stack_pointer[-1] = attr;
stack_pointer[0] = self_or_null;
stack_pointer += 1;
PREDICTED(LOAD_SUPER_ATTR);
_Py_CODEUNIT *this_instr = next_instr - 2;
(void)this_instr;
- PyObject *class;
- PyObject *global_super;
- PyObject *self;
- PyObject *attr;
- PyObject *null = NULL;
+ _PyStackRef class_st;
+ _PyStackRef global_super_st;
+ _PyStackRef self_st;
+ _PyStackRef attr;
+ _PyStackRef null = PyStackRef_NULL;
// _SPECIALIZE_LOAD_SUPER_ATTR
- class = stack_pointer[-2];
- global_super = stack_pointer[-3];
+ class_st = stack_pointer[-2];
+ global_super_st = stack_pointer[-3];
{
uint16_t counter = read_u16(&this_instr[1].cache);
(void)counter;
int load_method = oparg & 1;
if (ADAPTIVE_COUNTER_TRIGGERS(counter)) {
next_instr = this_instr;
- _Py_Specialize_LoadSuperAttr(global_super, class, next_instr, load_method);
+ _Py_Specialize_LoadSuperAttr(global_super_st, class_st, next_instr, load_method);
DISPATCH_SAME_OPARG();
}
STAT_INC(LOAD_SUPER_ATTR, deferred);
#endif /* ENABLE_SPECIALIZATION */
}
// _LOAD_SUPER_ATTR
- self = stack_pointer[-1];
+ self_st = stack_pointer[-1];
{
+ PyObject *global_super = PyStackRef_AsPyObjectBorrow(global_super_st);
+ PyObject *class = PyStackRef_AsPyObjectBorrow(class_st);
+ PyObject *self = PyStackRef_AsPyObjectBorrow(self_st);
if (opcode == INSTRUMENTED_LOAD_SUPER_ATTR) {
PyObject *arg = oparg & 2 ? class : &_PyInstrumentation_MISSING;
int err = _Py_call_instrumentation_2args(
}
}
}
- Py_DECREF(global_super);
- Py_DECREF(class);
- Py_DECREF(self);
+ PyStackRef_CLOSE(global_super_st);
+ PyStackRef_CLOSE(class_st);
+ PyStackRef_CLOSE(self_st);
if (super == NULL) goto pop_3_error;
PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 2);
- attr = PyObject_GetAttr(super, name);
+ attr = PyStackRef_FromPyObjectSteal(PyObject_GetAttr(super, name));
Py_DECREF(super);
- if (attr == NULL) goto pop_3_error;
- null = NULL;
+ if (PyStackRef_IsNull(attr)) goto pop_3_error;
+ null = PyStackRef_NULL;
}
stack_pointer[-3] = attr;
if (oparg & 1) stack_pointer[-2] = null;
next_instr += 2;
INSTRUCTION_STATS(LOAD_SUPER_ATTR_ATTR);
static_assert(INLINE_CACHE_ENTRIES_LOAD_SUPER_ATTR == 1, "incorrect cache size");
- PyObject *self;
- PyObject *class;
- PyObject *global_super;
- PyObject *attr;
+ _PyStackRef self_st;
+ _PyStackRef class_st;
+ _PyStackRef global_super_st;
+ _PyStackRef attr_st;
/* Skip 1 cache entry */
- self = stack_pointer[-1];
- class = stack_pointer[-2];
- global_super = stack_pointer[-3];
+ self_st = stack_pointer[-1];
+ class_st = stack_pointer[-2];
+ global_super_st = stack_pointer[-3];
+ PyObject *global_super = PyStackRef_AsPyObjectBorrow(global_super_st);
+ PyObject *class = PyStackRef_AsPyObjectBorrow(class_st);
+ PyObject *self = PyStackRef_AsPyObjectBorrow(self_st);
assert(!(oparg & 1));
DEOPT_IF(global_super != (PyObject *)&PySuper_Type, LOAD_SUPER_ATTR);
DEOPT_IF(!PyType_Check(class), LOAD_SUPER_ATTR);
STAT_INC(LOAD_SUPER_ATTR, hit);
PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 2);
- attr = _PySuper_Lookup((PyTypeObject *)class, self, name, NULL);
- Py_DECREF(global_super);
- Py_DECREF(class);
- Py_DECREF(self);
+ PyObject *attr = _PySuper_Lookup((PyTypeObject *)class, self, name, NULL);
+ PyStackRef_CLOSE(global_super_st);
+ PyStackRef_CLOSE(class_st);
+ PyStackRef_CLOSE(self_st);
if (attr == NULL) goto pop_3_error;
- stack_pointer[-3] = attr;
+ attr_st = PyStackRef_FromPyObjectSteal(attr);
+ stack_pointer[-3] = attr_st;
stack_pointer += -2;
assert(WITHIN_STACK_BOUNDS());
DISPATCH();
next_instr += 2;
INSTRUCTION_STATS(LOAD_SUPER_ATTR_METHOD);
static_assert(INLINE_CACHE_ENTRIES_LOAD_SUPER_ATTR == 1, "incorrect cache size");
- PyObject *self;
- PyObject *class;
- PyObject *global_super;
- PyObject *attr;
- PyObject *self_or_null;
+ _PyStackRef self_st;
+ _PyStackRef class_st;
+ _PyStackRef global_super_st;
+ _PyStackRef attr;
+ _PyStackRef self_or_null;
/* Skip 1 cache entry */
- self = stack_pointer[-1];
- class = stack_pointer[-2];
- global_super = stack_pointer[-3];
+ self_st = stack_pointer[-1];
+ class_st = stack_pointer[-2];
+ global_super_st = stack_pointer[-3];
+ PyObject *global_super = PyStackRef_AsPyObjectBorrow(global_super_st);
+ PyObject *class = PyStackRef_AsPyObjectBorrow(class_st);
+ PyObject *self = PyStackRef_AsPyObjectBorrow(self_st);
assert(oparg & 1);
DEOPT_IF(global_super != (PyObject *)&PySuper_Type, LOAD_SUPER_ATTR);
DEOPT_IF(!PyType_Check(class), LOAD_SUPER_ATTR);
PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 2);
PyTypeObject *cls = (PyTypeObject *)class;
int method_found = 0;
- attr = _PySuper_Lookup(cls, self, name,
- Py_TYPE(self)->tp_getattro == PyObject_GenericGetAttr ? &method_found : NULL);
- Py_DECREF(global_super);
- Py_DECREF(class);
- if (attr == NULL) {
- Py_DECREF(self);
+ PyObject *attr_o = _PySuper_Lookup(cls, self, name,
+ Py_TYPE(self)->tp_getattro == PyObject_GenericGetAttr ? &method_found : NULL);
+ PyStackRef_CLOSE(global_super_st);
+ PyStackRef_CLOSE(class_st);
+ if (attr_o == NULL) {
+ PyStackRef_CLOSE(self_st);
if (true) goto pop_3_error;
}
if (method_found) {
- self_or_null = self; // transfer ownership
+ self_or_null = self_st; // transfer ownership
} else {
- Py_DECREF(self);
- self_or_null = NULL;
+ PyStackRef_CLOSE(self_st);
+ self_or_null = PyStackRef_NULL;
}
+ attr = PyStackRef_FromPyObjectSteal(attr_o);
stack_pointer[-3] = attr;
stack_pointer[-2] = self_or_null;
stack_pointer += -1;
INSTRUCTION_STATS(MAKE_CELL);
// "initial" is probably NULL but not if it's an arg (or set
// via the f_locals proxy before MAKE_CELL has run).
- PyObject *initial = GETLOCAL(oparg);
+ PyObject *initial = PyStackRef_AsPyObjectBorrow(GETLOCAL(oparg));
PyObject *cell = PyCell_New(initial);
if (cell == NULL) {
goto error;
}
- SETLOCAL(oparg, cell);
+ SETLOCAL(oparg, PyStackRef_FromPyObjectSteal(cell));
DISPATCH();
}
frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(MAKE_FUNCTION);
- PyObject *codeobj;
- PyObject *func;
- codeobj = stack_pointer[-1];
+ _PyStackRef codeobj_st;
+ _PyStackRef func;
+ codeobj_st = stack_pointer[-1];
+ PyObject *codeobj = PyStackRef_AsPyObjectBorrow(codeobj_st);
PyFunctionObject *func_obj = (PyFunctionObject *)
PyFunction_New(codeobj, GLOBALS());
- Py_DECREF(codeobj);
+ PyStackRef_CLOSE(codeobj_st);
if (func_obj == NULL) {
goto error;
}
_PyFunction_SetVersion(
func_obj, ((PyCodeObject *)codeobj)->co_version);
- func = (PyObject *)func_obj;
+ func = PyStackRef_FromPyObjectSteal((PyObject *)func_obj);
stack_pointer[-1] = func;
DISPATCH();
}
frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(MAP_ADD);
- PyObject *value;
- PyObject *key;
- PyObject *dict;
+ _PyStackRef value;
+ _PyStackRef key;
+ _PyStackRef dict_st;
value = stack_pointer[-1];
key = stack_pointer[-2];
- dict = stack_pointer[-3 - (oparg - 1)];
+ dict_st = stack_pointer[-3 - (oparg - 1)];
+ PyObject *dict = PyStackRef_AsPyObjectBorrow(dict_st);
assert(PyDict_CheckExact(dict));
/* dict[key] = value */
// Do not DECREF INPUTS because the function steals the references
- if (_PyDict_SetItem_Take2((PyDictObject *)dict, key, value) != 0) goto pop_2_error;
+ if (_PyDict_SetItem_Take2((PyDictObject *)dict, PyStackRef_AsPyObjectSteal(key), PyStackRef_AsPyObjectSteal(value)) != 0) goto pop_2_error;
stack_pointer += -2;
assert(WITHIN_STACK_BOUNDS());
DISPATCH();
frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(MATCH_CLASS);
- PyObject *names;
- PyObject *type;
- PyObject *subject;
- PyObject *attrs;
+ _PyStackRef names;
+ _PyStackRef type;
+ _PyStackRef subject;
+ _PyStackRef attrs;
names = stack_pointer[-1];
type = stack_pointer[-2];
subject = stack_pointer[-3];
// Pop TOS and TOS1. Set TOS to a tuple of attributes on success, or
// None on failure.
- assert(PyTuple_CheckExact(names));
- attrs = _PyEval_MatchClass(tstate, subject, type, oparg, names);
- Py_DECREF(subject);
- Py_DECREF(type);
- Py_DECREF(names);
- if (attrs) {
- assert(PyTuple_CheckExact(attrs)); // Success!
+ assert(PyTuple_CheckExact(PyStackRef_AsPyObjectBorrow(names)));
+ PyObject *attrs_o = _PyEval_MatchClass(tstate,
+ PyStackRef_AsPyObjectBorrow(subject),
+ PyStackRef_AsPyObjectBorrow(type), oparg,
+ PyStackRef_AsPyObjectBorrow(names));
+ PyStackRef_CLOSE(subject);
+ PyStackRef_CLOSE(type);
+ PyStackRef_CLOSE(names);
+ if (attrs_o) {
+ assert(PyTuple_CheckExact(attrs_o)); // Success!
+ attrs = PyStackRef_FromPyObjectSteal(attrs_o);
}
else {
if (_PyErr_Occurred(tstate)) goto pop_3_error;
// Error!
- attrs = Py_None; // Failure!
+ attrs = PyStackRef_None; // Failure!
}
stack_pointer[-3] = attrs;
stack_pointer += -2;
frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(MATCH_KEYS);
- PyObject *keys;
- PyObject *subject;
- PyObject *values_or_none;
+ _PyStackRef keys;
+ _PyStackRef subject;
+ _PyStackRef values_or_none;
keys = stack_pointer[-1];
subject = stack_pointer[-2];
// On successful match, PUSH(values). Otherwise, PUSH(None).
- values_or_none = _PyEval_MatchKeys(tstate, subject, keys);
- if (values_or_none == NULL) goto error;
+ PyObject *values_or_none_o = _PyEval_MatchKeys(tstate,
+ PyStackRef_AsPyObjectBorrow(subject), PyStackRef_AsPyObjectBorrow(keys));
+ if (values_or_none_o == NULL) goto error;
+ values_or_none = PyStackRef_FromPyObjectSteal(values_or_none_o);
stack_pointer[0] = values_or_none;
stack_pointer += 1;
assert(WITHIN_STACK_BOUNDS());
frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(MATCH_MAPPING);
- PyObject *subject;
- PyObject *res;
+ _PyStackRef subject;
+ _PyStackRef res;
subject = stack_pointer[-1];
- int match = Py_TYPE(subject)->tp_flags & Py_TPFLAGS_MAPPING;
- res = match ? Py_True : Py_False;
+ int match = PyStackRef_TYPE(subject)->tp_flags & Py_TPFLAGS_MAPPING;
+ res = match ? PyStackRef_True : PyStackRef_False;
stack_pointer[0] = res;
stack_pointer += 1;
assert(WITHIN_STACK_BOUNDS());
frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(MATCH_SEQUENCE);
- PyObject *subject;
- PyObject *res;
+ _PyStackRef subject;
+ _PyStackRef res;
subject = stack_pointer[-1];
- int match = Py_TYPE(subject)->tp_flags & Py_TPFLAGS_SEQUENCE;
- res = match ? Py_True : Py_False;
+ int match = PyStackRef_TYPE(subject)->tp_flags & Py_TPFLAGS_SEQUENCE;
+ res = match ? PyStackRef_True : PyStackRef_False;
stack_pointer[0] = res;
stack_pointer += 1;
assert(WITHIN_STACK_BOUNDS());
frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(POP_EXCEPT);
- PyObject *exc_value;
+ _PyStackRef exc_value;
exc_value = stack_pointer[-1];
_PyErr_StackItem *exc_info = tstate->exc_info;
- Py_XSETREF(exc_info->exc_value, exc_value == Py_None ? NULL : exc_value);
+ Py_XSETREF(exc_info->exc_value,
+ PyStackRef_AsPyObjectBorrow(exc_value) == Py_None
+ ? NULL : PyStackRef_AsPyObjectSteal(exc_value));
stack_pointer += -1;
assert(WITHIN_STACK_BOUNDS());
DISPATCH();
(void)this_instr;
next_instr += 2;
INSTRUCTION_STATS(POP_JUMP_IF_FALSE);
- PyObject *cond;
+ _PyStackRef cond;
/* Skip 1 cache entry */
cond = stack_pointer[-1];
- assert(PyBool_Check(cond));
- int flag = Py_IsFalse(cond);
+ assert(PyBool_Check(PyStackRef_AsPyObjectBorrow(cond)));
+ int flag = PyStackRef_Is(cond, PyStackRef_False);
#if ENABLE_SPECIALIZATION
this_instr[1].cache = (this_instr[1].cache << 1) | flag;
#endif
(void)this_instr;
next_instr += 2;
INSTRUCTION_STATS(POP_JUMP_IF_NONE);
- PyObject *value;
- PyObject *b;
- PyObject *cond;
+ _PyStackRef value;
+ _PyStackRef b;
+ _PyStackRef cond;
/* Skip 1 cache entry */
// _IS_NONE
value = stack_pointer[-1];
{
- if (Py_IsNone(value)) {
- b = Py_True;
+ if (PyStackRef_Is(value, PyStackRef_None)) {
+ b = PyStackRef_True;
}
else {
- b = Py_False;
- Py_DECREF(value);
+ b = PyStackRef_False;
+ PyStackRef_CLOSE(value);
}
}
// _POP_JUMP_IF_TRUE
cond = b;
{
- assert(PyBool_Check(cond));
- int flag = Py_IsTrue(cond);
+ assert(PyBool_Check(PyStackRef_AsPyObjectBorrow(cond)));
+ int flag = PyStackRef_Is(cond, PyStackRef_True);
#if ENABLE_SPECIALIZATION
this_instr[1].cache = (this_instr[1].cache << 1) | flag;
#endif
(void)this_instr;
next_instr += 2;
INSTRUCTION_STATS(POP_JUMP_IF_NOT_NONE);
- PyObject *value;
- PyObject *b;
- PyObject *cond;
+ _PyStackRef value;
+ _PyStackRef b;
+ _PyStackRef cond;
/* Skip 1 cache entry */
// _IS_NONE
value = stack_pointer[-1];
{
- if (Py_IsNone(value)) {
- b = Py_True;
+ if (PyStackRef_Is(value, PyStackRef_None)) {
+ b = PyStackRef_True;
}
else {
- b = Py_False;
- Py_DECREF(value);
+ b = PyStackRef_False;
+ PyStackRef_CLOSE(value);
}
}
// _POP_JUMP_IF_FALSE
cond = b;
{
- assert(PyBool_Check(cond));
- int flag = Py_IsFalse(cond);
+ assert(PyBool_Check(PyStackRef_AsPyObjectBorrow(cond)));
+ int flag = PyStackRef_Is(cond, PyStackRef_False);
#if ENABLE_SPECIALIZATION
this_instr[1].cache = (this_instr[1].cache << 1) | flag;
#endif
(void)this_instr;
next_instr += 2;
INSTRUCTION_STATS(POP_JUMP_IF_TRUE);
- PyObject *cond;
+ _PyStackRef cond;
/* Skip 1 cache entry */
cond = stack_pointer[-1];
- assert(PyBool_Check(cond));
- int flag = Py_IsTrue(cond);
+ assert(PyBool_Check(PyStackRef_AsPyObjectBorrow(cond)));
+ int flag = PyStackRef_Is(cond, PyStackRef_True);
#if ENABLE_SPECIALIZATION
this_instr[1].cache = (this_instr[1].cache << 1) | flag;
#endif
frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(POP_TOP);
- PyObject *value;
+ _PyStackRef value;
value = stack_pointer[-1];
- Py_DECREF(value);
+ PyStackRef_CLOSE(value);
stack_pointer += -1;
assert(WITHIN_STACK_BOUNDS());
DISPATCH();
frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(PUSH_EXC_INFO);
- PyObject *new_exc;
- PyObject *prev_exc;
+ _PyStackRef new_exc;
+ _PyStackRef prev_exc;
new_exc = stack_pointer[-1];
_PyErr_StackItem *exc_info = tstate->exc_info;
if (exc_info->exc_value != NULL) {
- prev_exc = exc_info->exc_value;
+ prev_exc = PyStackRef_FromPyObjectSteal(exc_info->exc_value);
}
else {
- prev_exc = Py_None;
+ prev_exc = PyStackRef_None;
}
- assert(PyExceptionInstance_Check(new_exc));
- exc_info->exc_value = Py_NewRef(new_exc);
+ assert(PyExceptionInstance_Check(PyStackRef_AsPyObjectBorrow(new_exc)));
+ exc_info->exc_value = PyStackRef_AsPyObjectNew(new_exc);
stack_pointer[-1] = prev_exc;
stack_pointer[0] = new_exc;
stack_pointer += 1;
frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(PUSH_NULL);
- PyObject *res;
- res = NULL;
+ _PyStackRef res;
+ res = PyStackRef_NULL;
stack_pointer[0] = res;
stack_pointer += 1;
assert(WITHIN_STACK_BOUNDS());
(void)this_instr;
next_instr += 1;
INSTRUCTION_STATS(RAISE_VARARGS);
- PyObject **args;
+ _PyStackRef *args;
args = &stack_pointer[-oparg];
PyObject *cause = NULL, *exc = NULL;
switch (oparg) {
case 2:
- cause = args[1];
+ cause = PyStackRef_AsPyObjectSteal(args[1]);
/* fall through */
case 1:
- exc = args[0];
+ exc = PyStackRef_AsPyObjectSteal(args[0]);
/* fall through */
case 0:
if (do_raise(tstate, exc, cause)) {
(void)this_instr;
next_instr += 1;
INSTRUCTION_STATS(RERAISE);
- PyObject *exc;
- PyObject **values;
- exc = stack_pointer[-1];
+ _PyStackRef exc_st;
+ _PyStackRef *values;
+ exc_st = stack_pointer[-1];
values = &stack_pointer[-1 - oparg];
+ PyObject *exc = PyStackRef_AsPyObjectBorrow(exc_st);
assert(oparg >= 0 && oparg <= 2);
if (oparg) {
- PyObject *lasti = values[0];
+ PyObject *lasti = PyStackRef_AsPyObjectBorrow(values[0]);
if (PyLong_Check(lasti)) {
frame->instr_ptr = _PyCode_CODE(_PyFrame_GetCode(frame)) + PyLong_AsLong(lasti);
assert(!_PyErr_Occurred(tstate));
frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(RETURN_CONST);
- PyObject *value;
- PyObject *retval;
- PyObject *res;
+ _PyStackRef value;
+ _PyStackRef retval;
+ _PyStackRef res;
// _LOAD_CONST
{
- value = GETITEM(FRAME_CO_CONSTS, oparg);
- Py_INCREF(value);
+ value = PyStackRef_FromPyObjectNew(GETITEM(FRAME_CO_CONSTS, oparg));
}
// _RETURN_VALUE
retval = value;
frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(RETURN_GENERATOR);
- PyObject *res;
+ _PyStackRef res;
assert(PyFunction_Check(frame->f_funcobj));
PyFunctionObject *func = (PyFunctionObject *)frame->f_funcobj;
PyGenObject *gen = (PyGenObject *)_Py_MakeCoro(func);
gen->gi_frame_state = FRAME_CREATED;
gen_frame->owner = FRAME_OWNED_BY_GENERATOR;
_Py_LeaveRecursiveCallPy(tstate);
- res = (PyObject *)gen;
+ res = PyStackRef_FromPyObjectSteal((PyObject *)gen);
_PyInterpreterFrame *prev = frame->previous;
_PyThreadState_PopFrame(tstate, frame);
frame = tstate->current_frame = prev;
frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(RETURN_VALUE);
- PyObject *retval;
- PyObject *res;
+ _PyStackRef retval;
+ _PyStackRef res;
retval = stack_pointer[-1];
#if TIER_ONE
assert(frame != &entry_frame);
PREDICTED(SEND);
_Py_CODEUNIT *this_instr = next_instr - 2;
(void)this_instr;
- PyObject *receiver;
- PyObject *v;
- PyObject *retval;
+ _PyStackRef receiver;
+ _PyStackRef v;
+ _PyStackRef retval;
// _SPECIALIZE_SEND
receiver = stack_pointer[-2];
{
// _SEND
v = stack_pointer[-1];
{
+ PyObject *receiver_o = PyStackRef_AsPyObjectBorrow(receiver);
+ PyObject *retval_o;
assert(frame != &entry_frame);
if ((tstate->interp->eval_frame == NULL) &&
- (Py_TYPE(receiver) == &PyGen_Type || Py_TYPE(receiver) == &PyCoro_Type) &&
- ((PyGenObject *)receiver)->gi_frame_state < FRAME_EXECUTING)
+ (Py_TYPE(receiver_o) == &PyGen_Type || Py_TYPE(receiver_o) == &PyCoro_Type) &&
+ ((PyGenObject *)receiver_o)->gi_frame_state < FRAME_EXECUTING)
{
- PyGenObject *gen = (PyGenObject *)receiver;
+ PyGenObject *gen = (PyGenObject *)receiver_o;
_PyInterpreterFrame *gen_frame = &gen->gi_iframe;
STACK_SHRINK(1);
_PyFrame_StackPush(gen_frame, v);
frame->return_offset = (uint16_t)(next_instr - this_instr + oparg);
DISPATCH_INLINED(gen_frame);
}
- if (Py_IsNone(v) && PyIter_Check(receiver)) {
- retval = Py_TYPE(receiver)->tp_iternext(receiver);
+ if (PyStackRef_Is(v, PyStackRef_None) && PyIter_Check(receiver_o)) {
+ retval_o = Py_TYPE(receiver_o)->tp_iternext(receiver_o);
}
else {
- retval = PyObject_CallMethodOneArg(receiver, &_Py_ID(send), v);
+ retval_o = PyObject_CallMethodOneArg(receiver_o,
+ &_Py_ID(send),
+ PyStackRef_AsPyObjectBorrow(v));
}
- if (retval == NULL) {
+ if (retval_o == NULL) {
if (_PyErr_ExceptionMatches(tstate, PyExc_StopIteration)
) {
monitor_raise(tstate, frame, this_instr);
}
- if (_PyGen_FetchStopIterationValue(&retval) == 0) {
- assert(retval != NULL);
+ if (_PyGen_FetchStopIterationValue(&retval_o) == 0) {
+ assert(retval_o != NULL);
JUMPBY(oparg);
}
else {
goto error;
}
}
- Py_DECREF(v);
+ PyStackRef_CLOSE(v);
+ retval = PyStackRef_FromPyObjectSteal(retval_o);
}
stack_pointer[-1] = retval;
DISPATCH();
next_instr += 2;
INSTRUCTION_STATS(SEND_GEN);
static_assert(INLINE_CACHE_ENTRIES_SEND == 1, "incorrect cache size");
- PyObject *v;
- PyObject *receiver;
+ _PyStackRef v;
+ _PyStackRef receiver;
/* Skip 1 cache entry */
v = stack_pointer[-1];
receiver = stack_pointer[-2];
DEOPT_IF(tstate->interp->eval_frame, SEND);
- PyGenObject *gen = (PyGenObject *)receiver;
+ PyGenObject *gen = (PyGenObject *)PyStackRef_AsPyObjectBorrow(receiver);
DEOPT_IF(Py_TYPE(gen) != &PyGen_Type && Py_TYPE(gen) != &PyCoro_Type, SEND);
DEOPT_IF(gen->gi_frame_state >= FRAME_EXECUTING, SEND);
STAT_INC(SEND, hit);
frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(SET_ADD);
- PyObject *v;
- PyObject *set;
+ _PyStackRef v;
+ _PyStackRef set;
v = stack_pointer[-1];
set = stack_pointer[-2 - (oparg-1)];
- int err = PySet_Add(set, v);
- Py_DECREF(v);
+ int err = PySet_Add(PyStackRef_AsPyObjectBorrow(set),
+ PyStackRef_AsPyObjectSteal(v));
+ PyStackRef_CLOSE(v);
if (err) goto pop_1_error;
stack_pointer += -1;
assert(WITHIN_STACK_BOUNDS());
frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(SET_FUNCTION_ATTRIBUTE);
- PyObject *func;
- PyObject *attr;
- func = stack_pointer[-1];
- attr = stack_pointer[-2];
+ _PyStackRef func_st;
+ _PyStackRef attr_st;
+ func_st = stack_pointer[-1];
+ attr_st = stack_pointer[-2];
+ PyObject *func = PyStackRef_AsPyObjectBorrow(func_st);
+ PyObject *attr = PyStackRef_AsPyObjectBorrow(attr_st);
assert(PyFunction_Check(func));
PyFunctionObject *func_obj = (PyFunctionObject *)func;
switch(oparg) {
default:
Py_UNREACHABLE();
}
- stack_pointer[-2] = func;
+ stack_pointer[-2] = func_st;
stack_pointer += -1;
assert(WITHIN_STACK_BOUNDS());
DISPATCH();
frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(SET_UPDATE);
- PyObject *iterable;
- PyObject *set;
+ _PyStackRef iterable;
+ _PyStackRef set;
iterable = stack_pointer[-1];
set = stack_pointer[-2 - (oparg-1)];
- int err = _PySet_Update(set, iterable);
- Py_DECREF(iterable);
+ int err = _PySet_Update(PyStackRef_AsPyObjectBorrow(set),
+ PyStackRef_AsPyObjectBorrow(iterable));
+ PyStackRef_CLOSE(iterable);
if (err < 0) goto pop_1_error;
stack_pointer += -1;
assert(WITHIN_STACK_BOUNDS());
PREDICTED(STORE_ATTR);
_Py_CODEUNIT *this_instr = next_instr - 5;
(void)this_instr;
- PyObject *owner;
- PyObject *v;
+ _PyStackRef owner;
+ _PyStackRef v;
// _SPECIALIZE_STORE_ATTR
owner = stack_pointer[-1];
{
v = stack_pointer[-2];
{
PyObject *name = GETITEM(FRAME_CO_NAMES, oparg);
- int err = PyObject_SetAttr(owner, name, v);
- Py_DECREF(v);
- Py_DECREF(owner);
+ int err = PyObject_SetAttr(PyStackRef_AsPyObjectBorrow(owner),
+ name, PyStackRef_AsPyObjectSteal(v));
+ PyStackRef_CLOSE(v);
+ PyStackRef_CLOSE(owner);
if (err) goto pop_2_error;
}
stack_pointer += -2;
next_instr += 5;
INSTRUCTION_STATS(STORE_ATTR_INSTANCE_VALUE);
static_assert(INLINE_CACHE_ENTRIES_STORE_ATTR == 4, "incorrect cache size");
- PyObject *owner;
- PyObject *value;
+ _PyStackRef owner;
+ _PyStackRef value;
/* Skip 1 cache entry */
// _GUARD_TYPE_VERSION
owner = stack_pointer[-1];
{
uint32_t type_version = read_u32(&this_instr[2].cache);
- PyTypeObject *tp = Py_TYPE(owner);
+ PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner));
assert(type_version != 0);
DEOPT_IF(tp->tp_version_tag != type_version, STORE_ATTR);
}
// _GUARD_DORV_NO_DICT
{
- assert(Py_TYPE(owner)->tp_dictoffset < 0);
- assert(Py_TYPE(owner)->tp_flags & Py_TPFLAGS_INLINE_VALUES);
- DEOPT_IF(_PyObject_GetManagedDict(owner), STORE_ATTR);
- DEOPT_IF(_PyObject_InlineValues(owner)->valid == 0, STORE_ATTR);
+ PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner);
+ assert(Py_TYPE(owner_o)->tp_dictoffset < 0);
+ assert(Py_TYPE(owner_o)->tp_flags & Py_TPFLAGS_INLINE_VALUES);
+ DEOPT_IF(_PyObject_GetManagedDict(owner_o), STORE_ATTR);
+ DEOPT_IF(_PyObject_InlineValues(owner_o)->valid == 0, STORE_ATTR);
}
// _STORE_ATTR_INSTANCE_VALUE
value = stack_pointer[-2];
{
uint16_t index = read_u16(&this_instr[4].cache);
+ PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner);
STAT_INC(STORE_ATTR, hit);
- assert(_PyObject_GetManagedDict(owner) == NULL);
- PyDictValues *values = _PyObject_InlineValues(owner);
+ assert(_PyObject_GetManagedDict(owner_o) == NULL);
+ PyDictValues *values = _PyObject_InlineValues(owner_o);
PyObject *old_value = values->values[index];
- values->values[index] = value;
+ values->values[index] = PyStackRef_AsPyObjectSteal(value);
if (old_value == NULL) {
_PyDictValues_AddToInsertionOrder(values, index);
}
else {
Py_DECREF(old_value);
}
- Py_DECREF(owner);
+ PyStackRef_CLOSE(owner);
}
stack_pointer += -2;
assert(WITHIN_STACK_BOUNDS());
next_instr += 5;
INSTRUCTION_STATS(STORE_ATTR_SLOT);
static_assert(INLINE_CACHE_ENTRIES_STORE_ATTR == 4, "incorrect cache size");
- PyObject *owner;
- PyObject *value;
+ _PyStackRef owner;
+ _PyStackRef value;
/* Skip 1 cache entry */
// _GUARD_TYPE_VERSION
owner = stack_pointer[-1];
{
uint32_t type_version = read_u32(&this_instr[2].cache);
- PyTypeObject *tp = Py_TYPE(owner);
+ PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner));
assert(type_version != 0);
DEOPT_IF(tp->tp_version_tag != type_version, STORE_ATTR);
}
value = stack_pointer[-2];
{
uint16_t index = read_u16(&this_instr[4].cache);
- char *addr = (char *)owner + index;
+ PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner);
+ char *addr = (char *)owner_o + index;
STAT_INC(STORE_ATTR, hit);
PyObject *old_value = *(PyObject **)addr;
- *(PyObject **)addr = value;
+ *(PyObject **)addr = PyStackRef_AsPyObjectSteal(value);
Py_XDECREF(old_value);
- Py_DECREF(owner);
+ PyStackRef_CLOSE(owner);
}
stack_pointer += -2;
assert(WITHIN_STACK_BOUNDS());
next_instr += 5;
INSTRUCTION_STATS(STORE_ATTR_WITH_HINT);
static_assert(INLINE_CACHE_ENTRIES_STORE_ATTR == 4, "incorrect cache size");
- PyObject *owner;
- PyObject *value;
+ _PyStackRef owner;
+ _PyStackRef value;
/* Skip 1 cache entry */
// _GUARD_TYPE_VERSION
owner = stack_pointer[-1];
{
uint32_t type_version = read_u32(&this_instr[2].cache);
- PyTypeObject *tp = Py_TYPE(owner);
+ PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner));
assert(type_version != 0);
DEOPT_IF(tp->tp_version_tag != type_version, STORE_ATTR);
}
value = stack_pointer[-2];
{
uint16_t hint = read_u16(&this_instr[4].cache);
- assert(Py_TYPE(owner)->tp_flags & Py_TPFLAGS_MANAGED_DICT);
- PyDictObject *dict = _PyObject_GetManagedDict(owner);
+ PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner);
+ assert(Py_TYPE(owner_o)->tp_flags & Py_TPFLAGS_MANAGED_DICT);
+ PyDictObject *dict = _PyObject_GetManagedDict(owner_o);
DEOPT_IF(dict == NULL, STORE_ATTR);
assert(PyDict_CheckExact((PyObject *)dict));
PyObject *name = GETITEM(FRAME_CO_NAMES, oparg);
DEOPT_IF(ep->me_key != name, STORE_ATTR);
old_value = ep->me_value;
DEOPT_IF(old_value == NULL, STORE_ATTR);
- new_version = _PyDict_NotifyEvent(tstate->interp, PyDict_EVENT_MODIFIED, dict, name, value);
- ep->me_value = value;
+ new_version = _PyDict_NotifyEvent(tstate->interp, PyDict_EVENT_MODIFIED, dict, name, PyStackRef_AsPyObjectBorrow(value));
+ ep->me_value = PyStackRef_AsPyObjectSteal(value);
}
else {
PyDictKeyEntry *ep = DK_ENTRIES(dict->ma_keys) + hint;
DEOPT_IF(ep->me_key != name, STORE_ATTR);
old_value = ep->me_value;
DEOPT_IF(old_value == NULL, STORE_ATTR);
- new_version = _PyDict_NotifyEvent(tstate->interp, PyDict_EVENT_MODIFIED, dict, name, value);
- ep->me_value = value;
+ new_version = _PyDict_NotifyEvent(tstate->interp, PyDict_EVENT_MODIFIED, dict, name, PyStackRef_AsPyObjectBorrow(value));
+ ep->me_value = PyStackRef_AsPyObjectSteal(value);
}
Py_DECREF(old_value);
STAT_INC(STORE_ATTR, hit);
/* Ensure dict is GC tracked if it needs to be */
- if (!_PyObject_GC_IS_TRACKED(dict) && _PyObject_GC_MAY_BE_TRACKED(value)) {
+ if (!_PyObject_GC_IS_TRACKED(dict) && _PyObject_GC_MAY_BE_TRACKED(PyStackRef_AsPyObjectBorrow(value))) {
_PyObject_GC_TRACK(dict);
}
/* PEP 509 */
dict->ma_version_tag = new_version;
- Py_DECREF(owner);
+ PyStackRef_CLOSE(owner);
}
stack_pointer += -2;
assert(WITHIN_STACK_BOUNDS());
frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(STORE_DEREF);
- PyObject *v;
+ _PyStackRef v;
v = stack_pointer[-1];
- PyCellObject *cell = (PyCellObject *)GETLOCAL(oparg);
- PyCell_SetTakeRef(cell, v);
+ PyCellObject *cell = (PyCellObject *)PyStackRef_AsPyObjectBorrow(GETLOCAL(oparg));
+ PyCell_SetTakeRef(cell, PyStackRef_AsPyObjectSteal(v));
stack_pointer += -1;
assert(WITHIN_STACK_BOUNDS());
DISPATCH();
frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(STORE_FAST);
- PyObject *value;
+ _PyStackRef value;
value = stack_pointer[-1];
SETLOCAL(oparg, value);
stack_pointer += -1;
frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(STORE_FAST_LOAD_FAST);
- PyObject *value1;
- PyObject *value2;
+ _PyStackRef value1;
+ _PyStackRef value2;
value1 = stack_pointer[-1];
uint32_t oparg1 = oparg >> 4;
uint32_t oparg2 = oparg & 15;
SETLOCAL(oparg1, value1);
- value2 = GETLOCAL(oparg2);
- Py_INCREF(value2);
+ value2 = PyStackRef_DUP(GETLOCAL(oparg2));
stack_pointer[-1] = value2;
DISPATCH();
}
frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(STORE_FAST_STORE_FAST);
- PyObject *value1;
- PyObject *value2;
+ _PyStackRef value1;
+ _PyStackRef value2;
value1 = stack_pointer[-1];
value2 = stack_pointer[-2];
uint32_t oparg1 = oparg >> 4;
frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(STORE_GLOBAL);
- PyObject *v;
+ _PyStackRef v;
v = stack_pointer[-1];
PyObject *name = GETITEM(FRAME_CO_NAMES, oparg);
- int err = PyDict_SetItem(GLOBALS(), name, v);
- Py_DECREF(v);
+ int err = PyDict_SetItem(GLOBALS(), name, PyStackRef_AsPyObjectBorrow(v));
+ PyStackRef_CLOSE(v);
if (err) goto pop_1_error;
stack_pointer += -1;
assert(WITHIN_STACK_BOUNDS());
frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(STORE_NAME);
- PyObject *v;
+ _PyStackRef v;
v = stack_pointer[-1];
PyObject *name = GETITEM(FRAME_CO_NAMES, oparg);
PyObject *ns = LOCALS();
if (ns == NULL) {
_PyErr_Format(tstate, PyExc_SystemError,
"no locals found when storing %R", name);
- Py_DECREF(v);
+ PyStackRef_CLOSE(v);
if (true) goto pop_1_error;
}
if (PyDict_CheckExact(ns))
- err = PyDict_SetItem(ns, name, v);
+ err = PyDict_SetItem(ns, name, PyStackRef_AsPyObjectSteal(v));
else
- err = PyObject_SetItem(ns, name, v);
- Py_DECREF(v);
+ err = PyObject_SetItem(ns, name, PyStackRef_AsPyObjectSteal(v));
+ PyStackRef_CLOSE(v);
if (err) goto pop_1_error;
stack_pointer += -1;
assert(WITHIN_STACK_BOUNDS());
frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(STORE_SLICE);
- PyObject *stop;
- PyObject *start;
- PyObject *container;
- PyObject *v;
+ _PyStackRef stop;
+ _PyStackRef start;
+ _PyStackRef container;
+ _PyStackRef v;
stop = stack_pointer[-1];
start = stack_pointer[-2];
container = stack_pointer[-3];
v = stack_pointer[-4];
- PyObject *slice = _PyBuildSlice_ConsumeRefs(start, stop);
+ PyObject *slice = _PyBuildSlice_ConsumeRefs(PyStackRef_AsPyObjectSteal(start),
+ PyStackRef_AsPyObjectSteal(stop));
int err;
if (slice == NULL) {
err = 1;
}
else {
- err = PyObject_SetItem(container, slice, v);
+ err = PyObject_SetItem(PyStackRef_AsPyObjectBorrow(container), slice, PyStackRef_AsPyObjectSteal(v));
Py_DECREF(slice);
}
- Py_DECREF(v);
- Py_DECREF(container);
+ PyStackRef_CLOSE(v);
+ PyStackRef_CLOSE(container);
if (err) goto pop_4_error;
stack_pointer += -4;
assert(WITHIN_STACK_BOUNDS());
PREDICTED(STORE_SUBSCR);
_Py_CODEUNIT *this_instr = next_instr - 2;
(void)this_instr;
- PyObject *sub;
- PyObject *container;
- PyObject *v;
+ _PyStackRef sub;
+ _PyStackRef container;
+ _PyStackRef v;
// _SPECIALIZE_STORE_SUBSCR
sub = stack_pointer[-1];
container = stack_pointer[-2];
v = stack_pointer[-3];
{
/* container[sub] = v */
- int err = PyObject_SetItem(container, sub, v);
- Py_DECREF(v);
- Py_DECREF(container);
- Py_DECREF(sub);
+ int err = PyObject_SetItem(PyStackRef_AsPyObjectBorrow(container), PyStackRef_AsPyObjectSteal(sub), PyStackRef_AsPyObjectSteal(v));
+ PyStackRef_CLOSE(v);
+ PyStackRef_CLOSE(container);
+ PyStackRef_CLOSE(sub);
if (err) goto pop_3_error;
}
stack_pointer += -3;
next_instr += 2;
INSTRUCTION_STATS(STORE_SUBSCR_DICT);
static_assert(INLINE_CACHE_ENTRIES_STORE_SUBSCR == 1, "incorrect cache size");
- PyObject *sub;
- PyObject *dict;
- PyObject *value;
+ _PyStackRef sub_st;
+ _PyStackRef dict_st;
+ _PyStackRef value;
/* Skip 1 cache entry */
- sub = stack_pointer[-1];
- dict = stack_pointer[-2];
+ sub_st = stack_pointer[-1];
+ dict_st = stack_pointer[-2];
value = stack_pointer[-3];
+ PyObject *sub = PyStackRef_AsPyObjectBorrow(sub_st);
+ PyObject *dict = PyStackRef_AsPyObjectBorrow(dict_st);
DEOPT_IF(!PyDict_CheckExact(dict), STORE_SUBSCR);
STAT_INC(STORE_SUBSCR, hit);
- int err = _PyDict_SetItem_Take2((PyDictObject *)dict, sub, value);
- Py_DECREF(dict);
+ int err = _PyDict_SetItem_Take2((PyDictObject *)dict, sub, PyStackRef_AsPyObjectSteal(value));
+ PyStackRef_CLOSE(dict_st);
if (err) goto pop_3_error;
stack_pointer += -3;
assert(WITHIN_STACK_BOUNDS());
next_instr += 2;
INSTRUCTION_STATS(STORE_SUBSCR_LIST_INT);
static_assert(INLINE_CACHE_ENTRIES_STORE_SUBSCR == 1, "incorrect cache size");
- PyObject *sub;
- PyObject *list;
- PyObject *value;
+ _PyStackRef sub_st;
+ _PyStackRef list_st;
+ _PyStackRef value;
/* Skip 1 cache entry */
- sub = stack_pointer[-1];
- list = stack_pointer[-2];
+ sub_st = stack_pointer[-1];
+ list_st = stack_pointer[-2];
value = stack_pointer[-3];
+ PyObject *sub = PyStackRef_AsPyObjectBorrow(sub_st);
+ PyObject *list = PyStackRef_AsPyObjectBorrow(list_st);
DEOPT_IF(!PyLong_CheckExact(sub), STORE_SUBSCR);
DEOPT_IF(!PyList_CheckExact(list), STORE_SUBSCR);
// Ensure nonnegative, zero-or-one-digit ints.
DEOPT_IF(index >= PyList_GET_SIZE(list), STORE_SUBSCR);
STAT_INC(STORE_SUBSCR, hit);
PyObject *old_value = PyList_GET_ITEM(list, index);
- PyList_SET_ITEM(list, index, value);
+ PyList_SET_ITEM(list, index, PyStackRef_AsPyObjectSteal(value));
assert(old_value != NULL);
Py_DECREF(old_value);
_Py_DECREF_SPECIALIZED(sub, (destructor)PyObject_Free);
- Py_DECREF(list);
+ PyStackRef_CLOSE(list_st);
stack_pointer += -3;
assert(WITHIN_STACK_BOUNDS());
DISPATCH();
frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(SWAP);
- PyObject *top;
- PyObject *bottom;
+ _PyStackRef top;
+ _PyStackRef bottom;
top = stack_pointer[-1];
bottom = stack_pointer[-2 - (oparg-2)];
assert(oparg >= 2);
PREDICTED(TO_BOOL);
_Py_CODEUNIT *this_instr = next_instr - 4;
(void)this_instr;
- PyObject *value;
- PyObject *res;
+ _PyStackRef value;
+ _PyStackRef res;
// _SPECIALIZE_TO_BOOL
value = stack_pointer[-1];
{
/* Skip 2 cache entries */
// _TO_BOOL
{
- int err = PyObject_IsTrue(value);
- Py_DECREF(value);
+ int err = PyObject_IsTrue(PyStackRef_AsPyObjectBorrow(value));
+ PyStackRef_CLOSE(value);
if (err < 0) goto pop_1_error;
- res = err ? Py_True : Py_False;
+ res = err ? PyStackRef_True : PyStackRef_False;
}
stack_pointer[-1] = res;
DISPATCH();
next_instr += 4;
INSTRUCTION_STATS(TO_BOOL_ALWAYS_TRUE);
static_assert(INLINE_CACHE_ENTRIES_TO_BOOL == 3, "incorrect cache size");
- PyObject *owner;
- PyObject *value;
- PyObject *res;
+ _PyStackRef owner;
+ _PyStackRef value;
+ _PyStackRef res;
/* Skip 1 cache entry */
// _GUARD_TYPE_VERSION
owner = stack_pointer[-1];
{
uint32_t type_version = read_u32(&this_instr[2].cache);
- PyTypeObject *tp = Py_TYPE(owner);
+ PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner));
assert(type_version != 0);
DEOPT_IF(tp->tp_version_tag != type_version, TO_BOOL);
}
// _REPLACE_WITH_TRUE
value = owner;
{
- Py_DECREF(value);
- res = Py_True;
+ PyStackRef_CLOSE(value);
+ res = PyStackRef_True;
}
stack_pointer[-1] = res;
DISPATCH();
next_instr += 4;
INSTRUCTION_STATS(TO_BOOL_BOOL);
static_assert(INLINE_CACHE_ENTRIES_TO_BOOL == 3, "incorrect cache size");
- PyObject *value;
+ _PyStackRef value;
/* Skip 1 cache entry */
/* Skip 2 cache entries */
value = stack_pointer[-1];
- DEOPT_IF(!PyBool_Check(value), TO_BOOL);
+ DEOPT_IF(!PyBool_Check(PyStackRef_AsPyObjectBorrow(value)), TO_BOOL);
STAT_INC(TO_BOOL, hit);
DISPATCH();
}
next_instr += 4;
INSTRUCTION_STATS(TO_BOOL_INT);
static_assert(INLINE_CACHE_ENTRIES_TO_BOOL == 3, "incorrect cache size");
- PyObject *value;
- PyObject *res;
+ _PyStackRef value;
+ _PyStackRef res;
/* Skip 1 cache entry */
/* Skip 2 cache entries */
value = stack_pointer[-1];
- DEOPT_IF(!PyLong_CheckExact(value), TO_BOOL);
+ PyObject *value_o = PyStackRef_AsPyObjectBorrow(value);
+ DEOPT_IF(!PyLong_CheckExact(value_o), TO_BOOL);
STAT_INC(TO_BOOL, hit);
- if (_PyLong_IsZero((PyLongObject *)value)) {
- assert(_Py_IsImmortal(value));
- res = Py_False;
+ if (_PyLong_IsZero((PyLongObject *)value_o)) {
+ assert(_Py_IsImmortal(value_o));
+ res = PyStackRef_False;
}
else {
- Py_DECREF(value);
- res = Py_True;
+ PyStackRef_CLOSE(value);
+ res = PyStackRef_True;
}
stack_pointer[-1] = res;
DISPATCH();
next_instr += 4;
INSTRUCTION_STATS(TO_BOOL_LIST);
static_assert(INLINE_CACHE_ENTRIES_TO_BOOL == 3, "incorrect cache size");
- PyObject *value;
- PyObject *res;
+ _PyStackRef value;
+ _PyStackRef res;
/* Skip 1 cache entry */
/* Skip 2 cache entries */
value = stack_pointer[-1];
- DEOPT_IF(!PyList_CheckExact(value), TO_BOOL);
+ PyObject *value_o = PyStackRef_AsPyObjectBorrow(value);
+ DEOPT_IF(!PyList_CheckExact(value_o), TO_BOOL);
STAT_INC(TO_BOOL, hit);
- res = Py_SIZE(value) ? Py_True : Py_False;
- Py_DECREF(value);
+ res = Py_SIZE(value_o) ? PyStackRef_True : PyStackRef_False;
+ PyStackRef_CLOSE(value);
stack_pointer[-1] = res;
DISPATCH();
}
next_instr += 4;
INSTRUCTION_STATS(TO_BOOL_NONE);
static_assert(INLINE_CACHE_ENTRIES_TO_BOOL == 3, "incorrect cache size");
- PyObject *value;
- PyObject *res;
+ _PyStackRef value;
+ _PyStackRef res;
/* Skip 1 cache entry */
/* Skip 2 cache entries */
value = stack_pointer[-1];
// This one is a bit weird, because we expect *some* failures:
- DEOPT_IF(!Py_IsNone(value), TO_BOOL);
+ DEOPT_IF(!PyStackRef_Is(value, PyStackRef_None), TO_BOOL);
STAT_INC(TO_BOOL, hit);
- res = Py_False;
+ res = PyStackRef_False;
stack_pointer[-1] = res;
DISPATCH();
}
next_instr += 4;
INSTRUCTION_STATS(TO_BOOL_STR);
static_assert(INLINE_CACHE_ENTRIES_TO_BOOL == 3, "incorrect cache size");
- PyObject *value;
- PyObject *res;
+ _PyStackRef value;
+ _PyStackRef res;
/* Skip 1 cache entry */
/* Skip 2 cache entries */
value = stack_pointer[-1];
- DEOPT_IF(!PyUnicode_CheckExact(value), TO_BOOL);
+ PyObject *value_o = PyStackRef_AsPyObjectBorrow(value);
+ DEOPT_IF(!PyUnicode_CheckExact(value_o), TO_BOOL);
STAT_INC(TO_BOOL, hit);
- if (value == &_Py_STR(empty)) {
- assert(_Py_IsImmortal(value));
- res = Py_False;
+ if (value_o == &_Py_STR(empty)) {
+ assert(_Py_IsImmortal(value_o));
+ res = PyStackRef_False;
}
else {
- assert(Py_SIZE(value));
- Py_DECREF(value);
- res = Py_True;
+ assert(Py_SIZE(value_o));
+ PyStackRef_CLOSE(value);
+ res = PyStackRef_True;
}
stack_pointer[-1] = res;
DISPATCH();
frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(UNARY_INVERT);
- PyObject *value;
- PyObject *res;
+ _PyStackRef value;
+ _PyStackRef res;
value = stack_pointer[-1];
- res = PyNumber_Invert(value);
- Py_DECREF(value);
- if (res == NULL) goto pop_1_error;
+ PyObject *res_o = PyNumber_Invert(PyStackRef_AsPyObjectBorrow(value));
+ PyStackRef_CLOSE(value);
+ if (res_o == NULL) goto pop_1_error;
+ res = PyStackRef_FromPyObjectSteal(res_o);
stack_pointer[-1] = res;
DISPATCH();
}
frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(UNARY_NEGATIVE);
- PyObject *value;
- PyObject *res;
+ _PyStackRef value;
+ _PyStackRef res;
value = stack_pointer[-1];
- res = PyNumber_Negative(value);
- Py_DECREF(value);
- if (res == NULL) goto pop_1_error;
+ PyObject *res_o = PyNumber_Negative(PyStackRef_AsPyObjectBorrow(value));
+ PyStackRef_CLOSE(value);
+ if (res_o == NULL) goto pop_1_error;
+ res = PyStackRef_FromPyObjectSteal(res_o);
stack_pointer[-1] = res;
DISPATCH();
}
frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(UNARY_NOT);
- PyObject *value;
- PyObject *res;
+ _PyStackRef value;
+ _PyStackRef res;
value = stack_pointer[-1];
- assert(PyBool_Check(value));
- res = Py_IsFalse(value) ? Py_True : Py_False;
+ assert(PyBool_Check(PyStackRef_AsPyObjectBorrow(value)));
+ res = PyStackRef_Is(value, PyStackRef_False)
+ ? PyStackRef_True : PyStackRef_False;
stack_pointer[-1] = res;
DISPATCH();
}
frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(UNPACK_EX);
- PyObject *seq;
+ _PyStackRef seq;
seq = stack_pointer[-1];
int totalargs = 1 + (oparg & 0xFF) + (oparg >> 8);
- PyObject **top = stack_pointer + totalargs - 1;
- int res = _PyEval_UnpackIterable(tstate, seq, oparg & 0xFF, oparg >> 8, top);
- Py_DECREF(seq);
+ _PyStackRef *top = stack_pointer + totalargs - 1;
+ int res = _PyEval_UnpackIterableStackRef(tstate, seq, oparg & 0xFF, oparg >> 8, top);
+ PyStackRef_CLOSE(seq);
if (res == 0) goto pop_1_error;
stack_pointer += (oparg >> 8) + (oparg & 0xFF);
assert(WITHIN_STACK_BOUNDS());
PREDICTED(UNPACK_SEQUENCE);
_Py_CODEUNIT *this_instr = next_instr - 2;
(void)this_instr;
- PyObject *seq;
+ _PyStackRef seq;
// _SPECIALIZE_UNPACK_SEQUENCE
seq = stack_pointer[-1];
{
}
// _UNPACK_SEQUENCE
{
- PyObject **top = stack_pointer + oparg - 1;
- int res = _PyEval_UnpackIterable(tstate, seq, oparg, -1, top);
- Py_DECREF(seq);
+ _PyStackRef *top = stack_pointer + oparg - 1;
+ int res = _PyEval_UnpackIterableStackRef(tstate, seq, oparg, -1, top);
+ PyStackRef_CLOSE(seq);
if (res == 0) goto pop_1_error;
}
stack_pointer += -1 + oparg;
next_instr += 2;
INSTRUCTION_STATS(UNPACK_SEQUENCE_LIST);
static_assert(INLINE_CACHE_ENTRIES_UNPACK_SEQUENCE == 1, "incorrect cache size");
- PyObject *seq;
- PyObject **values;
+ _PyStackRef seq;
+ _PyStackRef *values;
/* Skip 1 cache entry */
seq = stack_pointer[-1];
values = &stack_pointer[-1];
- DEOPT_IF(!PyList_CheckExact(seq), UNPACK_SEQUENCE);
- DEOPT_IF(PyList_GET_SIZE(seq) != oparg, UNPACK_SEQUENCE);
+ PyObject *seq_o = PyStackRef_AsPyObjectBorrow(seq);
+ DEOPT_IF(!PyList_CheckExact(seq_o), UNPACK_SEQUENCE);
+ DEOPT_IF(PyList_GET_SIZE(seq_o) != oparg, UNPACK_SEQUENCE);
STAT_INC(UNPACK_SEQUENCE, hit);
- PyObject **items = _PyList_ITEMS(seq);
+ PyObject **items = _PyList_ITEMS(seq_o);
for (int i = oparg; --i >= 0; ) {
- *values++ = Py_NewRef(items[i]);
+ *values++ = PyStackRef_FromPyObjectNew(items[i]);
}
- Py_DECREF(seq);
+ PyStackRef_CLOSE(seq);
stack_pointer += -1 + oparg;
assert(WITHIN_STACK_BOUNDS());
DISPATCH();
next_instr += 2;
INSTRUCTION_STATS(UNPACK_SEQUENCE_TUPLE);
static_assert(INLINE_CACHE_ENTRIES_UNPACK_SEQUENCE == 1, "incorrect cache size");
- PyObject *seq;
- PyObject **values;
+ _PyStackRef seq;
+ _PyStackRef *values;
/* Skip 1 cache entry */
seq = stack_pointer[-1];
values = &stack_pointer[-1];
- DEOPT_IF(!PyTuple_CheckExact(seq), UNPACK_SEQUENCE);
- DEOPT_IF(PyTuple_GET_SIZE(seq) != oparg, UNPACK_SEQUENCE);
+ PyObject *seq_o = PyStackRef_AsPyObjectBorrow(seq);
+ DEOPT_IF(!PyTuple_CheckExact(seq_o), UNPACK_SEQUENCE);
+ DEOPT_IF(PyTuple_GET_SIZE(seq_o) != oparg, UNPACK_SEQUENCE);
STAT_INC(UNPACK_SEQUENCE, hit);
- PyObject **items = _PyTuple_ITEMS(seq);
+ PyObject **items = _PyTuple_ITEMS(seq_o);
for (int i = oparg; --i >= 0; ) {
- *values++ = Py_NewRef(items[i]);
+ *values++ = PyStackRef_FromPyObjectNew(items[i]);
}
- Py_DECREF(seq);
+ PyStackRef_CLOSE(seq);
stack_pointer += -1 + oparg;
assert(WITHIN_STACK_BOUNDS());
DISPATCH();
next_instr += 2;
INSTRUCTION_STATS(UNPACK_SEQUENCE_TWO_TUPLE);
static_assert(INLINE_CACHE_ENTRIES_UNPACK_SEQUENCE == 1, "incorrect cache size");
- PyObject *seq;
- PyObject *val1;
- PyObject *val0;
+ _PyStackRef seq;
+ _PyStackRef val1;
+ _PyStackRef val0;
/* Skip 1 cache entry */
seq = stack_pointer[-1];
assert(oparg == 2);
- DEOPT_IF(!PyTuple_CheckExact(seq), UNPACK_SEQUENCE);
- DEOPT_IF(PyTuple_GET_SIZE(seq) != 2, UNPACK_SEQUENCE);
+ PyObject *seq_o = PyStackRef_AsPyObjectBorrow(seq);
+ DEOPT_IF(!PyTuple_CheckExact(seq_o), UNPACK_SEQUENCE);
+ DEOPT_IF(PyTuple_GET_SIZE(seq_o) != 2, UNPACK_SEQUENCE);
STAT_INC(UNPACK_SEQUENCE, hit);
- val0 = Py_NewRef(PyTuple_GET_ITEM(seq, 0));
- val1 = Py_NewRef(PyTuple_GET_ITEM(seq, 1));
- Py_DECREF(seq);
+ val0 = PyStackRef_FromPyObjectNew(PyTuple_GET_ITEM(seq_o, 0));
+ val1 = PyStackRef_FromPyObjectNew(PyTuple_GET_ITEM(seq_o, 1));
+ PyStackRef_CLOSE(seq);
stack_pointer[-1] = val1;
stack_pointer[0] = val0;
stack_pointer += 1;
frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(WITH_EXCEPT_START);
- PyObject *val;
- PyObject *lasti;
- PyObject *exit_self;
- PyObject *exit_func;
- PyObject *res;
+ _PyStackRef val;
+ _PyStackRef lasti;
+ _PyStackRef exit_self;
+ _PyStackRef exit_func;
+ _PyStackRef res;
val = stack_pointer[-1];
lasti = stack_pointer[-3];
exit_self = stack_pointer[-4];
Then we push the __exit__ return value.
*/
PyObject *exc, *tb;
- assert(val && PyExceptionInstance_Check(val));
- exc = PyExceptionInstance_Class(val);
- tb = PyException_GetTraceback(val);
+ PyObject *val_o = PyStackRef_AsPyObjectBorrow(val);
+ PyObject *exit_func_o = PyStackRef_AsPyObjectBorrow(exit_func);
+ assert(val_o && PyExceptionInstance_Check(val_o));
+ exc = PyExceptionInstance_Class(val_o);
+ tb = PyException_GetTraceback(val_o);
if (tb == NULL) {
tb = Py_None;
}
else {
Py_DECREF(tb);
}
- assert(PyLong_Check(lasti));
+ assert(PyLong_Check(PyStackRef_AsPyObjectBorrow(lasti)));
(void)lasti; // Shut up compiler warning if asserts are off
- PyObject *stack[5] = {NULL, exit_self, exc, val, tb};
- int has_self = (exit_self != NULL);
- res = PyObject_Vectorcall(exit_func, stack + 2 - has_self,
- (3 + has_self) | PY_VECTORCALL_ARGUMENTS_OFFSET, NULL);
- if (res == NULL) goto error;
+ PyObject *stack[5] = {NULL, PyStackRef_AsPyObjectBorrow(exit_self), exc, val_o, tb};
+ int has_self = !PyStackRef_IsNull(exit_self);
+ res = PyStackRef_FromPyObjectSteal(PyObject_Vectorcall(exit_func_o, stack + 2 - has_self,
+ (3 + has_self) | PY_VECTORCALL_ARGUMENTS_OFFSET, NULL));
+ if (PyStackRef_IsNull(res)) goto error;
stack_pointer[0] = res;
stack_pointer += 1;
assert(WITHIN_STACK_BOUNDS());
frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(YIELD_VALUE);
- PyObject *retval;
- PyObject *value;
+ _PyStackRef retval;
+ _PyStackRef value;
retval = stack_pointer[-1];
// NOTE: It's important that YIELD_VALUE never raises an exception!
// The compiler treats any exception raised here as a failed close()
int
_PyOptimizer_Optimize(
_PyInterpreterFrame *frame, _Py_CODEUNIT *start,
- PyObject **stack_pointer, _PyExecutorObject **executor_ptr)
+ _PyStackRef *stack_pointer, _PyExecutorObject **executor_ptr)
{
PyCodeObject *code = _PyFrame_GetCode(frame);
assert(PyCode_Check(code));
_Py_CODEUNIT *target = instr + 1 + _PyOpcode_Caches[JUMP_BACKWARD] - oparg;
_PyUOpInstruction buffer[4] = {
{ .opcode = _START_EXECUTOR, .jump_target = 3, .format=UOP_FORMAT_JUMP },
- { .opcode = _LOAD_CONST_INLINE_BORROW, .operand = (uintptr_t)self },
+ { .opcode = _LOAD_CONST_INLINE, .operand = (uintptr_t)self },
{ .opcode = _INTERNAL_INCREMENT_OPT_COUNTER },
{ .opcode = _EXIT_TRACE, .target = (uint32_t)(target - _PyCode_CODE(code)), .format=UOP_FORMAT_TARGET }
};
/* _INSTRUMENTED_LOAD_SUPER_ATTR is not a viable micro-op for tier 2 */
case _LOAD_SUPER_ATTR_ATTR: {
- _Py_UopsSymbol *attr;
- attr = sym_new_not_null(ctx);
- stack_pointer[-3] = attr;
+ _Py_UopsSymbol *attr_st;
+ attr_st = sym_new_not_null(ctx);
+ stack_pointer[-3] = attr_st;
stack_pointer += -2;
assert(WITHIN_STACK_BOUNDS());
break;
}
case _GET_LEN: {
- _Py_UopsSymbol *len_o;
- len_o = sym_new_not_null(ctx);
- stack_pointer[0] = len_o;
+ _Py_UopsSymbol *len;
+ len = sym_new_not_null(ctx);
+ stack_pointer[0] = len;
stack_pointer += 1;
assert(WITHIN_STACK_BOUNDS());
break;
}
case _SET_FUNCTION_ATTRIBUTE: {
- _Py_UopsSymbol *func;
- func = sym_new_not_null(ctx);
- stack_pointer[-2] = func;
+ _Py_UopsSymbol *func_st;
+ func_st = sym_new_not_null(ctx);
+ stack_pointer[-2] = func_st;
stack_pointer += -1;
assert(WITHIN_STACK_BOUNDS());
break;
/* Attribute specialization */
void
-_Py_Specialize_LoadSuperAttr(PyObject *global_super, PyObject *cls, _Py_CODEUNIT *instr, int load_method) {
+_Py_Specialize_LoadSuperAttr(_PyStackRef global_super_st, _PyStackRef cls_st, _Py_CODEUNIT *instr, int load_method) {
+ PyObject *global_super = PyStackRef_AsPyObjectBorrow(global_super_st);
+ PyObject *cls = PyStackRef_AsPyObjectBorrow(cls_st);
+
assert(ENABLE_SPECIALIZATION);
assert(_PyOpcode_Caches[LOAD_SUPER_ATTR] == INLINE_CACHE_ENTRIES_LOAD_SUPER_ATTR);
_PySuperAttrCache *cache = (_PySuperAttrCache *)(instr + 1);
static int specialize_class_load_attr(PyObject* owner, _Py_CODEUNIT* instr, PyObject* name);
void
-_Py_Specialize_LoadAttr(PyObject *owner, _Py_CODEUNIT *instr, PyObject *name)
+_Py_Specialize_LoadAttr(_PyStackRef owner_st, _Py_CODEUNIT *instr, PyObject *name)
{
+ PyObject *owner = PyStackRef_AsPyObjectBorrow(owner_st);
+
assert(ENABLE_SPECIALIZATION);
assert(_PyOpcode_Caches[LOAD_ATTR] == INLINE_CACHE_ENTRIES_LOAD_ATTR);
_PyAttrCache *cache = (_PyAttrCache *)(instr + 1);
}
void
-_Py_Specialize_StoreAttr(PyObject *owner, _Py_CODEUNIT *instr, PyObject *name)
+_Py_Specialize_StoreAttr(_PyStackRef owner_st, _Py_CODEUNIT *instr, PyObject *name)
{
+ PyObject *owner = PyStackRef_AsPyObjectBorrow(owner_st);
+
assert(ENABLE_SPECIALIZATION);
assert(_PyOpcode_Caches[STORE_ATTR] == INLINE_CACHE_ENTRIES_STORE_ATTR);
_PyAttrCache *cache = (_PyAttrCache *)(instr + 1);
void
_Py_Specialize_BinarySubscr(
- PyObject *container, PyObject *sub, _Py_CODEUNIT *instr)
+ _PyStackRef container_st, _PyStackRef sub_st, _Py_CODEUNIT *instr)
{
+ PyObject *container = PyStackRef_AsPyObjectBorrow(container_st);
+ PyObject *sub = PyStackRef_AsPyObjectBorrow(sub_st);
+
assert(ENABLE_SPECIALIZATION);
assert(_PyOpcode_Caches[BINARY_SUBSCR] ==
INLINE_CACHE_ENTRIES_BINARY_SUBSCR);
}
void
-_Py_Specialize_StoreSubscr(PyObject *container, PyObject *sub, _Py_CODEUNIT *instr)
+_Py_Specialize_StoreSubscr(_PyStackRef container_st, _PyStackRef sub_st, _Py_CODEUNIT *instr)
{
+ PyObject *container = PyStackRef_AsPyObjectBorrow(container_st);
+ PyObject *sub = PyStackRef_AsPyObjectBorrow(sub_st);
+
assert(ENABLE_SPECIALIZATION);
_PyStoreSubscrCache *cache = (_PyStoreSubscrCache *)(instr + 1);
PyTypeObject *container_type = Py_TYPE(container);
}
void
-_Py_Specialize_Call(PyObject *callable, _Py_CODEUNIT *instr, int nargs)
+_Py_Specialize_Call(_PyStackRef callable_st, _Py_CODEUNIT *instr, int nargs)
{
+ PyObject *callable = PyStackRef_AsPyObjectBorrow(callable_st);
+
assert(ENABLE_SPECIALIZATION);
assert(_PyOpcode_Caches[CALL] == INLINE_CACHE_ENTRIES_CALL);
assert(_Py_OPCODE(*instr) != INSTRUMENTED_CALL);
#endif // Py_STATS
void
-_Py_Specialize_BinaryOp(PyObject *lhs, PyObject *rhs, _Py_CODEUNIT *instr,
- int oparg, PyObject **locals)
+_Py_Specialize_BinaryOp(_PyStackRef lhs_st, _PyStackRef rhs_st, _Py_CODEUNIT *instr,
+ int oparg, _PyStackRef *locals)
{
+ PyObject *lhs = PyStackRef_AsPyObjectBorrow(lhs_st);
+ PyObject *rhs = PyStackRef_AsPyObjectBorrow(rhs_st);
assert(ENABLE_SPECIALIZATION);
assert(_PyOpcode_Caches[BINARY_OP] == INLINE_CACHE_ENTRIES_BINARY_OP);
_PyBinaryOpCache *cache = (_PyBinaryOpCache *)(instr + 1);
if (PyUnicode_CheckExact(lhs)) {
_Py_CODEUNIT next = instr[INLINE_CACHE_ENTRIES_BINARY_OP + 1];
bool to_store = (next.op.code == STORE_FAST);
- if (to_store && locals[next.op.arg] == lhs) {
+ if (to_store && PyStackRef_AsPyObjectBorrow(locals[next.op.arg]) == lhs) {
instr->op.code = BINARY_OP_INPLACE_ADD_UNICODE;
goto success;
}
#endif // Py_STATS
void
-_Py_Specialize_CompareOp(PyObject *lhs, PyObject *rhs, _Py_CODEUNIT *instr,
+_Py_Specialize_CompareOp(_PyStackRef lhs_st, _PyStackRef rhs_st, _Py_CODEUNIT *instr,
int oparg)
{
+ PyObject *lhs = PyStackRef_AsPyObjectBorrow(lhs_st);
+ PyObject *rhs = PyStackRef_AsPyObjectBorrow(rhs_st);
+
assert(ENABLE_SPECIALIZATION);
assert(_PyOpcode_Caches[COMPARE_OP] == INLINE_CACHE_ENTRIES_COMPARE_OP);
// All of these specializations compute boolean values, so they're all valid
#endif // Py_STATS
void
-_Py_Specialize_UnpackSequence(PyObject *seq, _Py_CODEUNIT *instr, int oparg)
+_Py_Specialize_UnpackSequence(_PyStackRef seq_st, _Py_CODEUNIT *instr, int oparg)
{
+ PyObject *seq = PyStackRef_AsPyObjectBorrow(seq_st);
+
assert(ENABLE_SPECIALIZATION);
assert(_PyOpcode_Caches[UNPACK_SEQUENCE] ==
INLINE_CACHE_ENTRIES_UNPACK_SEQUENCE);
#endif // Py_STATS
void
-_Py_Specialize_ForIter(PyObject *iter, _Py_CODEUNIT *instr, int oparg)
+_Py_Specialize_ForIter(_PyStackRef iter, _Py_CODEUNIT *instr, int oparg)
{
assert(ENABLE_SPECIALIZATION);
assert(_PyOpcode_Caches[FOR_ITER] == INLINE_CACHE_ENTRIES_FOR_ITER);
_PyForIterCache *cache = (_PyForIterCache *)(instr + 1);
- PyTypeObject *tp = Py_TYPE(iter);
+ PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(iter));
if (tp == &PyListIter_Type) {
instr->op.code = FOR_ITER_LIST;
goto success;
}
void
-_Py_Specialize_Send(PyObject *receiver, _Py_CODEUNIT *instr)
+_Py_Specialize_Send(_PyStackRef receiver_st, _Py_CODEUNIT *instr)
{
+ PyObject *receiver = PyStackRef_AsPyObjectBorrow(receiver_st);
+
assert(ENABLE_SPECIALIZATION);
assert(_PyOpcode_Caches[SEND] == INLINE_CACHE_ENTRIES_SEND);
_PySendCache *cache = (_PySendCache *)(instr + 1);
}
void
-_Py_Specialize_ToBool(PyObject *value, _Py_CODEUNIT *instr)
+_Py_Specialize_ToBool(_PyStackRef value_o, _Py_CODEUNIT *instr)
{
assert(ENABLE_SPECIALIZATION);
assert(_PyOpcode_Caches[TO_BOOL] == INLINE_CACHE_ENTRIES_TO_BOOL);
_PyToBoolCache *cache = (_PyToBoolCache *)(instr + 1);
+ PyObject *value = PyStackRef_AsPyObjectBorrow(value_o);
if (PyBool_Check(value)) {
instr->op.code = TO_BOOL_BOOL;
goto success;
#endif // Py_STATS
void
-_Py_Specialize_ContainsOp(PyObject *value, _Py_CODEUNIT *instr)
+_Py_Specialize_ContainsOp(_PyStackRef value_st, _Py_CODEUNIT *instr)
{
+ PyObject *value = PyStackRef_AsPyObjectBorrow(value_st);
+
assert(ENABLE_SPECIALIZATION);
assert(_PyOpcode_Caches[CONTAINS_OP] == INLINE_CACHE_ENTRIES_COMPARE_OP);
_PyContainsOpCache *cache = (_PyContainsOpCache *)(instr + 1);
return f"{type}{self.name}{size}{cond} {self.peek}"
def is_array(self) -> bool:
- return self.type == "PyObject **"
+ return self.type == "_PyStackRef *"
@dataclass
NON_ESCAPING_FUNCTIONS = (
+ "PyStackRef_FromPyObjectSteal",
+ "PyStackRef_AsPyObjectBorrow",
+ "PyStackRef_AsPyObjectSteal",
+ "PyStackRef_CLOSE",
+ "PyStackRef_DUP",
+ "PyStackRef_CLEAR",
+ "PyStackRef_IsNull",
+ "PyStackRef_TYPE",
+ "PyStackRef_False",
+ "PyStackRef_True",
+ "PyStackRef_None",
+ "PyStackRef_Is",
+ "PyStackRef_FromPyObjectNew",
+ "PyStackRef_AsPyObjectNew",
+ "PyStackRef_FromPyObjectImmortal",
"Py_INCREF",
"_PyManagedDictPointer_IsValues",
"_PyObject_GetManagedDict",
"_PyFrame_SetStackPointer",
"_PyType_HasFeature",
"PyUnicode_Concat",
- "_PyList_FromArraySteal",
- "_PyTuple_FromArraySteal",
"PySlice_New",
"_Py_LeaveRecursiveCallPy",
"CALL_STAT_INC",
"PyFloat_AS_DOUBLE",
"_PyFrame_PushUnchecked",
"Py_FatalError",
+ "STACKREFS_TO_PYOBJECTS",
+ "STACKREFS_TO_PYOBJECTS_CLEANUP",
+ "CONVERSION_FAILED",
+ "_PyList_FromArraySteal",
+ "_PyTuple_FromArraySteal",
)
ESCAPING_FUNCTIONS = (
continue
if var.size != "1":
out.emit(f"for (int _i = {var.size}; --_i >= 0;) {{\n")
- out.emit(f"Py_DECREF({var.name}[_i]);\n")
+ out.emit(f"PyStackRef_CLOSE({var.name}[_i]);\n")
out.emit("}\n")
elif var.condition:
if var.condition == "1":
- out.emit(f"Py_DECREF({var.name});\n")
+ out.emit(f"PyStackRef_CLOSE({var.name});\n")
elif var.condition != "0":
- out.emit(f"Py_XDECREF({var.name});\n")
+ out.emit(f"PyStackRef_XCLOSE({var.name});\n")
else:
- out.emit(f"Py_DECREF({var.name});\n")
+ out.emit(f"PyStackRef_CLOSE({var.name});\n")
def replace_sync_sp(
is_override = override is not None
out.start_line()
for var in reversed(prototype.stack.inputs):
- res = stack.pop(var)
+ res = stack.pop(var, extract_bits=True)
if not skip_inputs:
out.emit(res)
if not prototype.properties.stores_sp:
if not var.peek or is_override:
out.emit(stack.push(var))
out.start_line()
- stack.flush(out, cast_type="_Py_UopsSymbol *")
+ stack.flush(out, cast_type="_Py_UopsSymbol *", extract_bits=True)
except SizeMismatch as ex:
raise analysis_error(ex.args[0], uop.body[0])
if not (size := self.expression()):
raise self.make_syntax_error("Expected expression")
self.require(lx.RBRACKET)
- type_text = "PyObject **"
+ type_text = "_PyStackRef *"
size_text = size.text.strip()
return StackEffect(tkn.text, type_text, cond_text, size_text)
return None
self.variables: list[StackItem] = []
self.defined: set[str] = set()
- def pop(self, var: StackItem) -> str:
+ def pop(self, var: StackItem, extract_bits: bool = False) -> str:
self.top_offset.pop(var)
if not var.peek:
self.peek_offset.pop(var)
else:
self.defined.add(var.name)
cast = f"({var.type})" if (not indirect and var.type) else ""
+ bits = ".bits" if cast and not extract_bits else ""
assign = (
- f"{var.name} = {cast}{indirect}stack_pointer[{self.base_offset.to_c()}];"
+ f"{var.name} = {cast}{indirect}stack_pointer[{self.base_offset.to_c()}]{bits};"
)
if var.condition:
if var.condition == "1":
self.top_offset.push(var)
return ""
- def flush(self, out: CWriter, cast_type: str = "PyObject *") -> None:
+ def flush(self, out: CWriter, cast_type: str = "uintptr_t", extract_bits: bool = False) -> None:
out.start_line()
for var in self.variables:
if not var.peek:
cast = f"({cast_type})" if var.type else ""
+ bits = ".bits" if cast and not extract_bits else ""
if var.name not in UNUSED and not var.is_array():
if var.condition:
if var.condition == "0":
elif var.condition != "1":
out.emit(f"if ({var.condition}) ")
out.emit(
- f"stack_pointer[{self.base_offset.to_c()}] = {cast}{var.name};\n"
+ f"stack_pointer[{self.base_offset.to_c()}]{bits} = {cast}{var.name};\n"
)
self.base_offset.push(var)
if self.base_offset.to_c() != self.top_offset.to_c():
if isinstance(uop, Uop):
for var in reversed(uop.stack.inputs):
if var.name not in variables:
- type = var.type if var.type else "PyObject *"
variables.add(var.name)
+ type, null = (var.type, "NULL") if var.type else ("_PyStackRef", "PyStackRef_NULL")
+ space = " " if type[-1].isalnum() else ""
if var.condition:
- out.emit(f"{type}{var.name} = NULL;\n")
+ out.emit(f"{type}{space}{var.name} = {null};\n")
else:
- out.emit(f"{type}{var.name};\n")
+ if var.is_array():
+ out.emit(f"{var.type}{space}{var.name};\n")
+ else:
+ out.emit(f"{type}{space}{var.name};\n")
for var in uop.stack.outputs:
if var.name not in variables:
variables.add(var.name)
- type = var.type if var.type else "PyObject *"
+ type, null = (var.type, "NULL") if var.type else ("_PyStackRef", "PyStackRef_NULL")
+ space = " " if type[-1].isalnum() else ""
if var.condition:
- out.emit(f"{type}{var.name} = NULL;\n")
+ out.emit(f"{type}{space}{var.name} = {null};\n")
else:
- out.emit(f"{type}{var.name};\n")
+ out.emit(f"{type}{space}{var.name};\n")
def write_uop(
) -> None:
if var.name in variables:
return
- type = var.type if var.type else "PyObject *"
variables.add(var.name)
+ type, null = (var.type, "NULL") if var.type else ("_PyStackRef", "PyStackRef_NULL")
+ space = " " if type[-1].isalnum() else ""
if var.condition:
- out.emit(f"{type}{var.name} = NULL;\n")
+ out.emit(f"{type}{space}{var.name} = {null};\n")
if uop.replicates:
# Replicas may not use all their conditional variables
# So avoid a compiler warning with a fake use
out.emit(f"(void){var.name};\n")
else:
- out.emit(f"{type}{var.name};\n")
+ out.emit(f"{type}{space}{var.name};\n")
def declare_variables(uop: Uop, out: CWriter) -> None:
hexdigits = "0123456789abcdef"
+USED_TAGS = 0b11
+
ENCODING = locale.getpreferredencoding()
FRAME_INFO_OPTIMIZED_OUT = '(frame information optimized out)'
_typename = 'PyObject'
def __init__(self, gdbval, cast_to=None):
+ # Clear the tagged pointer
+ gdbval = gdb.Value(int(gdbval) & (~USED_TAGS)).cast(gdbval.type)
if cast_to:
self._gdbval = gdbval.cast(cast_to)
else:
#include "pycore_setobject.h"
#include "pycore_sliceobject.h"
#include "pycore_descrobject.h"
+#include "pycore_stackref.h"
#include "ceval_macros.h"
#define WITHIN_STACK_BOUNDS() 1
_Py_CODEUNIT *
-_JIT_ENTRY(_PyInterpreterFrame *frame, PyObject **stack_pointer, PyThreadState *tstate)
+_JIT_ENTRY(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate)
{
// Locals that the instruction implementations expect to exist:
PATCH_VALUE(_PyExecutorObject *, current_executor, _JIT_EXECUTOR)
// The actual change is patched in while the JIT compiler is being built, in
// Tools/jit/_targets.py. On other platforms, this function compiles to nothing.
_Py_CODEUNIT *
-_ENTRY(_PyInterpreterFrame *frame, PyObject **stack_pointer, PyThreadState *tstate)
+_ENTRY(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate)
{
// This is subtle. The actual trace will return to us once it exits, so we
// need to make sure that we stay alive until then. If our trace side-exits
projects / thirdparty / Python / cpython.git / commitdiff
