// different frame, and it's accounted for by _PUSH_FRAME.
op(_POP_FRAME, (retval --)) {
assert(EMPTY());
- _PyFrame_SetStackPointer(frame, stack_pointer);
- _Py_LeaveRecursiveCallPy(tstate);
- // GH-99729: We need to unlink the frame *before* clearing it:
- _PyInterpreterFrame *dying = frame;
#if TIER_ONE
assert(frame != &entry_frame);
#endif
+ STORE_SP();
+ _Py_LeaveRecursiveCallPy(tstate);
+ // GH-99729: We need to unlink the frame *before* clearing it:
+ _PyInterpreterFrame *dying = frame;
frame = tstate->current_frame = dying->previous;
_PyEval_FrameClearAndPop(tstate, dying);
frame->prev_instr += frame->return_offset;
_PyFrame_StackPush(frame, retval);
- #if TIER_ONE
- goto resume_frame;
- #endif
- #if TIER_TWO
- stack_pointer = _PyFrame_GetStackPointer(frame);
- ip_offset = (_Py_CODEUNIT *)_PyFrame_GetCode(frame)->co_code_adaptive;
- #endif
+ LOAD_SP();
+ LOAD_IP();
+#if LLTRACE && TIER_ONE
+ lltrace = maybe_lltrace_resume_frame(frame, &entry_frame, GLOBALS());
+ if (lltrace < 0) {
+ goto exit_unwind;
+ }
+#endif
}
macro(RETURN_VALUE) =
fflush(stdout);
PyErr_SetRaisedException(exc);
}
+
+static int
+maybe_lltrace_resume_frame(_PyInterpreterFrame *frame, _PyInterpreterFrame *skip_frame, PyObject *globals)
+{
+ if (globals == NULL) {
+ return 0;
+ }
+ if (frame == skip_frame) {
+ return 0;
+ }
+ int r = PyDict_Contains(globals, &_Py_ID(__lltrace__));
+ if (r < 0) {
+ return -1;
+ }
+ int lltrace = r;
+ if (!lltrace) {
+ // When tracing executed uops, also trace bytecode
+ char *uop_debug = Py_GETENV("PYTHONUOPSDEBUG");
+ if (uop_debug != NULL && *uop_debug >= '0') {
+ lltrace = (*uop_debug - '0') >= 5; // TODO: Parse an int and all that
+ }
+ }
+ if (lltrace) {
+ lltrace_resume_frame(frame);
+ }
+ return lltrace;
+}
+
#endif
static void monitor_raise(PyThreadState *tstate,
return _PyEval_EvalFrame(tstate, f->f_frame, throwflag);
}
+#define TIER_ONE 1
#include "ceval_macros.h"
SET_LOCALS_FROM_FRAME();
#ifdef LLTRACE
- {
- if (frame != &entry_frame && GLOBALS()) {
- int r = PyDict_Contains(GLOBALS(), &_Py_ID(__lltrace__));
- if (r < 0) {
- goto exit_unwind;
- }
- lltrace = r;
- if (!lltrace) {
- // When tracing executed uops, also trace bytecode
- char *uop_debug = Py_GETENV("PYTHONUOPSDEBUG");
- if (uop_debug != NULL && *uop_debug >= '0') {
- lltrace = (*uop_debug - '0') >= 5; // TODO: Parse an int and all that
- }
- }
- }
- if (lltrace) {
- lltrace_resume_frame(frame);
- }
+ lltrace = maybe_lltrace_resume_frame(frame, &entry_frame, GLOBALS());
+ if (lltrace < 0) {
+ goto exit_unwind;
}
#endif
#endif
{
-#define TIER_ONE 1
#include "generated_cases.c.h"
/* INSTRUMENTED_LINE has to be here, rather than in bytecodes.c,
static inline void _Py_LeaveRecursiveCallPy(PyThreadState *tstate) {
tstate->py_recursion_remaining++;
}
+
+
+/* Implementation of "macros" that modify the instruction pointer,
+ * stack pointer, or frame pointer.
+ * These need to treated differently by tier 1 and 2. */
+
+#if TIER_ONE
+
+#define LOAD_IP() \
+do { next_instr = frame->prev_instr+1; } while (0)
+
+#define STORE_SP() \
+_PyFrame_SetStackPointer(frame, stack_pointer)
+
+#define LOAD_SP() \
+stack_pointer = _PyFrame_GetStackPointer(frame);
+
+#endif
+
+
+#if TIER_TWO
+
+#define LOAD_IP() \
+do { ip_offset = (_Py_CODEUNIT *)_PyFrame_GetCode(frame)->co_code_adaptive; } while (0)
+
+#define STORE_SP() \
+_PyFrame_SetStackPointer(frame, stack_pointer)
+
+#define LOAD_SP() \
+stack_pointer = _PyFrame_GetStackPointer(frame);
+
+#endif
+
+
+
+
#include "pycore_sliceobject.h"
#include "pycore_uops.h"
+#define TIER_TWO 2
#include "ceval_macros.h"
OBJECT_STAT_INC(optimization_uops_executed);
switch (opcode) {
-#define TIER_TWO 2
#include "executor_cases.c.h"
default:
retval = stack_pointer[-1];
STACK_SHRINK(1);
assert(EMPTY());
- _PyFrame_SetStackPointer(frame, stack_pointer);
- _Py_LeaveRecursiveCallPy(tstate);
- // GH-99729: We need to unlink the frame *before* clearing it:
- _PyInterpreterFrame *dying = frame;
#if TIER_ONE
assert(frame != &entry_frame);
#endif
+ STORE_SP();
+ _Py_LeaveRecursiveCallPy(tstate);
+ // GH-99729: We need to unlink the frame *before* clearing it:
+ _PyInterpreterFrame *dying = frame;
frame = tstate->current_frame = dying->previous;
_PyEval_FrameClearAndPop(tstate, dying);
frame->prev_instr += frame->return_offset;
_PyFrame_StackPush(frame, retval);
- #if TIER_ONE
- goto resume_frame;
- #endif
- #if TIER_TWO
- stack_pointer = _PyFrame_GetStackPointer(frame);
- ip_offset = (_Py_CODEUNIT *)_PyFrame_GetCode(frame)->co_code_adaptive;
- #endif
+ LOAD_SP();
+ LOAD_IP();
+#if LLTRACE && TIER_ONE
+ lltrace = maybe_lltrace_resume_frame(frame, &entry_frame, GLOBALS());
+ if (lltrace < 0) {
+ goto exit_unwind;
+ }
+#endif
break;
}
STACK_SHRINK(1);
{
assert(EMPTY());
- _PyFrame_SetStackPointer(frame, stack_pointer);
- _Py_LeaveRecursiveCallPy(tstate);
- // GH-99729: We need to unlink the frame *before* clearing it:
- _PyInterpreterFrame *dying = frame;
#if TIER_ONE
assert(frame != &entry_frame);
#endif
+ STORE_SP();
+ _Py_LeaveRecursiveCallPy(tstate);
+ // GH-99729: We need to unlink the frame *before* clearing it:
+ _PyInterpreterFrame *dying = frame;
frame = tstate->current_frame = dying->previous;
_PyEval_FrameClearAndPop(tstate, dying);
frame->prev_instr += frame->return_offset;
_PyFrame_StackPush(frame, retval);
- #if TIER_ONE
- goto resume_frame;
- #endif
- #if TIER_TWO
- stack_pointer = _PyFrame_GetStackPointer(frame);
- ip_offset = (_Py_CODEUNIT *)_PyFrame_GetCode(frame)->co_code_adaptive;
- #endif
+ LOAD_SP();
+ LOAD_IP();
+ #if LLTRACE && TIER_ONE
+ lltrace = maybe_lltrace_resume_frame(frame, &entry_frame, GLOBALS());
+ if (lltrace < 0) {
+ goto exit_unwind;
+ }
+ #endif
}
+ DISPATCH();
}
TARGET(INSTRUMENTED_RETURN_VALUE) {
retval = value;
{
assert(EMPTY());
- _PyFrame_SetStackPointer(frame, stack_pointer);
- _Py_LeaveRecursiveCallPy(tstate);
- // GH-99729: We need to unlink the frame *before* clearing it:
- _PyInterpreterFrame *dying = frame;
#if TIER_ONE
assert(frame != &entry_frame);
#endif
+ STORE_SP();
+ _Py_LeaveRecursiveCallPy(tstate);
+ // GH-99729: We need to unlink the frame *before* clearing it:
+ _PyInterpreterFrame *dying = frame;
frame = tstate->current_frame = dying->previous;
_PyEval_FrameClearAndPop(tstate, dying);
frame->prev_instr += frame->return_offset;
_PyFrame_StackPush(frame, retval);
- #if TIER_ONE
- goto resume_frame;
- #endif
- #if TIER_TWO
- stack_pointer = _PyFrame_GetStackPointer(frame);
- ip_offset = (_Py_CODEUNIT *)_PyFrame_GetCode(frame)->co_code_adaptive;
- #endif
+ LOAD_SP();
+ LOAD_IP();
+ #if LLTRACE && TIER_ONE
+ lltrace = maybe_lltrace_resume_frame(frame, &entry_frame, GLOBALS());
+ if (lltrace < 0) {
+ goto exit_unwind;
+ }
+ #endif
}
+ DISPATCH();
}
TARGET(INSTRUMENTED_RETURN_CONST) {
ip_offset = (_Py_CODEUNIT *)_PyFrame_GetCode(frame)->co_code_adaptive;
#endif
}
+ DISPATCH();
}
TARGET(CALL_PY_EXACT_ARGS) {
ip_offset = (_Py_CODEUNIT *)_PyFrame_GetCode(frame)->co_code_adaptive;
#endif
}
+ DISPATCH();
}
TARGET(CALL_PY_WITH_DEFAULTS) {
next_instr_is_set = write_components(parts, out, TIER_ONE, mac.cache_offset)
except AssertionError as err:
raise AssertionError(f"Error writing macro {mac.name}") from err
- if not parts[-1].instr.always_exits and not next_instr_is_set:
- if mac.cache_offset:
+ if not parts[-1].instr.always_exits:
+ if not next_instr_is_set and mac.cache_offset:
out.emit(f"next_instr += {mac.cache_offset};")
out.emit("DISPATCH();")