- There is no longer a separate Python/executor.c file.
- Conventions in Python/bytecodes.c are slightly different -- don't use `goto error`,
you must use `GOTO_ERROR(error)` (same for others like `unused_local_error`).
- The `TIER_ONE` and `TIER_TWO` symbols are only valid in the generated (.c.h) files.
- In Lib/test/support/__init__.py, `Py_C_RECURSION_LIMIT` is imported from `_testcapi`.
- On Windows, in debug mode, stack allocation grows from 8MiB to 12MiB.
- **Beware!** This changes the env vars to enable uops and their debugging
to `PYTHON_UOPS` and `PYTHON_LLTRACE`.
#For recursion tests, easily exceeds default recursion limit
EXCEEDS_RECURSION_LIMIT = 5000
-# The default C recursion limit (from Include/cpython/pystate.h).
-Py_C_RECURSION_LIMIT = 1500
+def _get_c_recursion_limit():
+ try:
+ import _testcapi
+ return _testcapi.Py_C_RECURSION_LIMIT
+ except (ImportError, AttributeError):
+ return 1500 # (from Include/cpython/pystate.h)
+
+# The default C recursion limit.
+Py_C_RECURSION_LIMIT = _get_c_recursion_limit()
#Windows doesn't have os.uname() but it doesn't support s390x.
skip_on_s390x = unittest.skipIf(hasattr(os, 'uname') and os.uname().machine == 's390x',
with open(self.temp_output_filename) as temp_output:
lines = temp_output.readlines()
- while lines and lines[0].startswith("// "):
+ while lines and lines[0].startswith(("// ", "#", " #", "\n")):
lines.pop(0)
+ while lines and lines[-1].startswith(("#", "\n")):
+ lines.pop(-1)
actual = "".join(lines)
- # if actual.rstrip() != expected.rstrip():
+ # if actual.strip() != expected.strip():
# print("Actual:")
# print(actual)
# print("Expected:")
# print(expected)
# print("End")
- self.assertEqual(actual.rstrip(), expected.rstrip())
+ self.assertEqual(actual.strip(), expected.strip())
def test_inst_no_args(self):
input = """
Python/crossinterp.o \
Python/dynamic_annotations.o \
Python/errors.o \
- Python/executor.o \
Python/flowgraph.o \
Python/frame.o \
Python/frozenmain.o \
$(srcdir)/Python/generated_cases.c.h \
$(srcdir)/Python/opcode_targets.h
-Python/executor.o: \
- $(srcdir)/Include/internal/pycore_opcode_metadata.h \
- $(srcdir)/Include/internal/pycore_optimizer.h \
- $(srcdir)/Python/ceval_macros.h \
- $(srcdir)/Python/executor_cases.c.h
-
Python/flowgraph.o: \
$(srcdir)/Include/internal/pycore_opcode_metadata.h
--- /dev/null
+Merge the Tier 1 (bytecode) and Tier 2 (micro-ops) interpreters together,
+moving the Tier 2 interpreter loop and switch into
+``_PyEval_EvalFrameDefault()`` in ``Python/ceval.c``.
+The ``Python/executor.c`` file is gone.
+Also the ``TIER_ONE`` and ``TIER_TWO`` macros are now handled
+by the code generator.
+
+**Beware!** This changes the environment variables to enable micro-ops and
+their debugging to ``PYTHON_UOPS`` and ``PYTHON_LLTRACE``.
<ClCompile Include="..\Python\dynamic_annotations.c" />
<ClCompile Include="..\Python\dynload_win.c" />
<ClCompile Include="..\Python\errors.c" />
- <ClCompile Include="..\Python\executor.c" />
<ClCompile Include="..\Python\fileutils.c" />
<ClCompile Include="..\Python\flowgraph.c" />
<ClCompile Include="..\Python\formatter_unicode.c" />
<ClCompile Include="..\Python\errors.c">
<Filter>Source Files</Filter>
</ClCompile>
- <ClCompile Include="..\Python\executor.c">
- <Filter>Source Files</Filter>
- </ClCompile>
<ClCompile Include="..\Objects\exceptions.c">
<Filter>Source Files</Filter>
</ClCompile>
<Link>
<SubSystem>Console</SubSystem>
<StackReserveSize Condition="$(Configuration) != 'Debug'">2000000</StackReserveSize>
- <StackReserveSize Condition="$(Configuration) == 'Debug'">8000000</StackReserveSize>
+ <StackReserveSize Condition="$(Configuration) == 'Debug'">12000000</StackReserveSize>
</Link>
</ItemDefinitionGroup>
<ItemGroup>
<ClCompile Include="..\Python\dynamic_annotations.c" />
<ClCompile Include="..\Python\dynload_win.c" />
<ClCompile Include="..\Python\errors.c" />
- <ClCompile Include="..\Python\executor.c" />
<ClCompile Include="..\Python\fileutils.c" />
<ClCompile Include="..\Python\flowgraph.c" />
<ClCompile Include="..\Python\formatter_unicode.c" />
<ClCompile Include="..\Python\errors.c">
<Filter>Python</Filter>
</ClCompile>
- <ClCompile Include="..\Python\executor.c">
- <Filter>Python</Filter>
- </ClCompile>
<ClCompile Include="..\Python\fileutils.c">
<Filter>Python</Filter>
</ClCompile>
static uint16_t invert, counter, index, hint;
#define unused 0 // Used in a macro def, can't be static
static uint32_t type_version;
+static _PyUOpExecutorObject *current_executor;
static PyObject *
dummy_func(
uintptr_t code_version = _PyFrame_GetCode(frame)->_co_instrumentation_version;
if (code_version != global_version) {
if (_Py_Instrument(_PyFrame_GetCode(frame), tstate->interp)) {
- goto error;
+ GOTO_ERROR(error);
}
next_instr = this_instr;
}
if (PyGen_Check(receiver)) {
PyErr_SetObject(PyExc_StopIteration, value);
if (monitor_stop_iteration(tstate, frame, this_instr)) {
- goto error;
+ GOTO_ERROR(error);
}
PyErr_SetRaisedException(NULL);
}
if (PyGen_Check(receiver) || PyCoro_CheckExact(receiver)) {
PyErr_SetObject(PyExc_StopIteration, value);
if (monitor_stop_iteration(tstate, frame, this_instr)) {
- goto error;
+ GOTO_ERROR(error);
}
PyErr_SetRaisedException(NULL);
}
int err = _Py_call_instrumentation_arg(
tstate, PY_MONITORING_EVENT_PY_RETURN,
frame, this_instr, retval);
- if (err) goto error;
+ if (err) GOTO_ERROR(error);
STACK_SHRINK(1);
assert(EMPTY());
_PyFrame_SetStackPointer(frame, stack_pointer);
int err = _Py_call_instrumentation_arg(
tstate, PY_MONITORING_EVENT_PY_RETURN,
frame, this_instr, retval);
- if (err) goto error;
+ if (err) GOTO_ERROR(error);
Py_INCREF(retval);
assert(EMPTY());
_PyFrame_SetStackPointer(frame, stack_pointer);
if (PyAsyncGen_CheckExact(aiter)) {
awaitable = type->tp_as_async->am_anext(aiter);
if (awaitable == NULL) {
- goto error;
+ GOTO_ERROR(error);
}
} else {
if (type->tp_as_async != NULL){
if (getter != NULL) {
next_iter = (*getter)(aiter);
if (next_iter == NULL) {
- goto error;
+ GOTO_ERROR(error);
}
}
else {
"'async for' requires an iterator with "
"__anext__ method, got %.100s",
type->tp_name);
- goto error;
+ GOTO_ERROR(error);
}
awaitable = _PyCoro_GetAwaitableIter(next_iter);
Py_TYPE(next_iter)->tp_name);
Py_DECREF(next_iter);
- goto error;
+ GOTO_ERROR(error);
} else {
Py_DECREF(next_iter);
}
JUMPBY(oparg);
}
else {
- goto error;
+ GOTO_ERROR(error);
}
}
Py_DECREF(v);
int err = _Py_call_instrumentation_arg(
tstate, PY_MONITORING_EVENT_PY_YIELD,
frame, this_instr, retval);
- if (err) goto error;
+ if (err) GOTO_ERROR(error);
tstate->exc_info = gen->gi_exc_state.previous_item;
gen->gi_exc_state.previous_item = NULL;
_Py_LeaveRecursiveCallPy(tstate);
else {
assert(PyLong_Check(lasti));
_PyErr_SetString(tstate, PyExc_SystemError, "lasti is not an int");
- goto error;
+ GOTO_ERROR(error);
}
}
assert(exc && PyExceptionInstance_Check(exc));
if (ns == NULL) {
_PyErr_Format(tstate, PyExc_SystemError,
"no locals when deleting %R", name);
- goto error;
+ GOTO_ERROR(error);
}
err = PyObject_DelItem(ns, name);
// Can't use ERROR_IF here.
_PyEval_FormatExcCheckArg(tstate, PyExc_NameError,
NAME_ERROR_MSG,
name);
- goto error;
+ GOTO_ERROR(error);
}
}
_PyEval_FormatExcCheckArg(tstate, PyExc_NameError,
NAME_ERROR_MSG, name);
}
- goto error;
+ GOTO_ERROR(error);
}
}
inst(LOAD_FROM_DICT_OR_GLOBALS, (mod_or_class_dict -- v)) {
PyObject *name = GETITEM(FRAME_CO_NAMES, oparg);
if (PyMapping_GetOptionalItem(mod_or_class_dict, name, &v) < 0) {
- goto error;
+ GOTO_ERROR(error);
}
if (v == NULL) {
v = PyDict_GetItemWithError(GLOBALS(), name);
Py_INCREF(v);
}
else if (_PyErr_Occurred(tstate)) {
- goto error;
+ GOTO_ERROR(error);
}
else {
if (PyMapping_GetOptionalItem(BUILTINS(), name, &v) < 0) {
- goto error;
+ GOTO_ERROR(error);
}
if (v == NULL) {
_PyEval_FormatExcCheckArg(
tstate, PyExc_NameError,
NAME_ERROR_MSG, name);
- goto error;
+ GOTO_ERROR(error);
}
}
}
}
PyObject *name = GETITEM(FRAME_CO_NAMES, oparg);
if (PyMapping_GetOptionalItem(mod_or_class_dict, name, &v) < 0) {
- goto error;
+ GOTO_ERROR(error);
}
if (v == NULL) {
v = PyDict_GetItemWithError(GLOBALS(), name);
Py_INCREF(v);
}
else if (_PyErr_Occurred(tstate)) {
- goto error;
+ GOTO_ERROR(error);
}
else {
if (PyMapping_GetOptionalItem(BUILTINS(), name, &v) < 0) {
- goto error;
+ GOTO_ERROR(error);
}
if (v == NULL) {
_PyEval_FormatExcCheckArg(
tstate, PyExc_NameError,
NAME_ERROR_MSG, name);
- goto error;
+ GOTO_ERROR(error);
}
}
}
PyObject *initial = GETLOCAL(oparg);
PyObject *cell = PyCell_New(initial);
if (cell == NULL) {
- goto error;
+ GOTO_ERROR(error);
}
SETLOCAL(oparg, cell);
}
// Fortunately we don't need its superpower.
if (oldobj == NULL) {
_PyEval_FormatExcUnbound(tstate, _PyFrame_GetCode(frame), oparg);
- goto error;
+ GOTO_ERROR(error);
}
PyCell_SET(cell, NULL);
Py_DECREF(oldobj);
name = PyTuple_GET_ITEM(_PyFrame_GetCode(frame)->co_localsplusnames, oparg);
if (PyMapping_GetOptionalItem(class_dict, name, &value) < 0) {
Py_DECREF(class_dict);
- goto error;
+ GOTO_ERROR(error);
}
Py_DECREF(class_dict);
if (!value) {
value = PyCell_GET(cell);
if (value == NULL) {
_PyEval_FormatExcUnbound(tstate, _PyFrame_GetCode(frame), oparg);
- goto error;
+ GOTO_ERROR(error);
}
Py_INCREF(value);
}
inst(BUILD_SET, (values[oparg] -- set)) {
set = PySet_New(NULL);
if (set == NULL)
- goto error;
+ GOTO_ERROR(error);
int err = 0;
for (int i = 0; i < oparg; i++) {
PyObject *item = values[i];
PyTuple_GET_SIZE(keys) != (Py_ssize_t)oparg) {
_PyErr_SetString(tstate, PyExc_SystemError,
"bad BUILD_CONST_KEY_MAP keys argument");
- goto error; // Pop the keys and values.
+ GOTO_ERROR(error); // Pop the keys and values.
}
map = _PyDict_FromItems(
&PyTuple_GET_ITEM(keys, 0), 1,
JUMPBY(1-original_oparg);
frame->instr_ptr = next_instr;
Py_INCREF(executor);
+ if (executor->execute == _PyUopExecute) {
+ current_executor = (_PyUOpExecutorObject *)executor;
+ GOTO_TIER_TWO();
+ }
frame = executor->execute(executor, frame, stack_pointer);
if (frame == NULL) {
frame = tstate->current_frame;
goto resume_with_error;
}
- next_instr = frame->instr_ptr;
- goto resume_frame;
+ goto enter_tier_one;
}
inst(POP_JUMP_IF_FALSE, (unused/1, cond -- )) {
_PyErr_SetString(tstate, PyExc_TypeError,
"cannot 'yield from' a coroutine object "
"in a non-coroutine generator");
- goto error;
+ GOTO_ERROR(error);
}
iter = iterable;
}
/* `iterable` is not a generator. */
iter = PyObject_GetIter(iterable);
if (iter == NULL) {
- goto error;
+ GOTO_ERROR(error);
}
DECREF_INPUTS();
}
if (next == NULL) {
if (_PyErr_Occurred(tstate)) {
if (!_PyErr_ExceptionMatches(tstate, PyExc_StopIteration)) {
- goto error;
+ GOTO_ERROR(error);
}
monitor_raise(tstate, frame, this_instr);
_PyErr_Clear(tstate);
else {
if (_PyErr_Occurred(tstate)) {
if (!_PyErr_ExceptionMatches(tstate, PyExc_StopIteration)) {
- goto error;
+ GOTO_ERROR(error);
}
monitor_raise(tstate, frame, this_instr);
_PyErr_Clear(tstate);
"asynchronous context manager protocol",
Py_TYPE(mgr)->tp_name);
}
- goto error;
+ GOTO_ERROR(error);
}
exit = _PyObject_LookupSpecial(mgr, &_Py_ID(__aexit__));
if (exit == NULL) {
Py_TYPE(mgr)->tp_name);
}
Py_DECREF(enter);
- goto error;
+ GOTO_ERROR(error);
}
DECREF_INPUTS();
res = _PyObject_CallNoArgs(enter);
"context manager protocol",
Py_TYPE(mgr)->tp_name);
}
- goto error;
+ GOTO_ERROR(error);
}
exit = _PyObject_LookupSpecial(mgr, &_Py_ID(__exit__));
if (exit == NULL) {
Py_TYPE(mgr)->tp_name);
}
Py_DECREF(enter);
- goto error;
+ GOTO_ERROR(error);
}
DECREF_INPUTS();
res = _PyObject_CallNoArgs(enter);
// The frame has stolen all the arguments from the stack,
// so there is no need to clean them up.
if (new_frame == NULL) {
- goto error;
+ GOTO_ERROR(error);
}
frame->return_offset = (uint16_t)(next_instr - this_instr);
DISPATCH_INLINED(new_frame);
STAT_INC(CALL, hit);
PyObject *self = _PyType_NewManagedObject(tp);
if (self == NULL) {
- goto error;
+ GOTO_ERROR(error);
}
Py_DECREF(tp);
_PyInterpreterFrame *shim = _PyFrame_PushTrampolineUnchecked(
PyErr_Format(PyExc_TypeError,
"__init__() should return None, not '%.200s'",
Py_TYPE(should_be_none)->tp_name);
- goto error;
+ GOTO_ERROR(error);
}
}
// This is slower but CPython promises to check all non-vectorcall
// function calls.
if (_Py_EnterRecursiveCallTstate(tstate, " while calling a Python object")) {
- goto error;
+ GOTO_ERROR(error);
}
PyObject *arg = args[0];
res = _PyCFunction_TrampolineCall(cfunc, PyCFunction_GET_SELF(callable), arg);
PyObject *arg = args[0];
Py_ssize_t len_i = PyObject_Length(arg);
if (len_i < 0) {
- goto error;
+ GOTO_ERROR(error);
}
res = PyLong_FromSsize_t(len_i);
assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL));
PyObject *inst = args[0];
int retval = PyObject_IsInstance(inst, cls);
if (retval < 0) {
- goto error;
+ GOTO_ERROR(error);
}
res = PyBool_FromLong(retval);
assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL));
// This is slower but CPython promises to check all non-vectorcall
// function calls.
if (_Py_EnterRecursiveCallTstate(tstate, " while calling a Python object")) {
- goto error;
+ GOTO_ERROR(error);
}
res = _PyCFunction_TrampolineCall(cfunc, self, arg);
_Py_LeaveRecursiveCallTstate(tstate);
// This is slower but CPython promises to check all non-vectorcall
// function calls.
if (_Py_EnterRecursiveCallTstate(tstate, " while calling a Python object")) {
- goto error;
+ GOTO_ERROR(error);
}
res = _PyCFunction_TrampolineCall(cfunc, self, NULL);
_Py_LeaveRecursiveCallTstate(tstate);
// The frame has stolen all the arguments from the stack,
// so there is no need to clean them up.
if (new_frame == NULL) {
- goto error;
+ GOTO_ERROR(error);
}
assert(next_instr - this_instr == 1);
frame->return_offset = 1;
assert(kwargs == NULL || PyDict_CheckExact(kwargs));
if (!PyTuple_CheckExact(callargs)) {
if (check_args_iterable(tstate, func, callargs) < 0) {
- goto error;
+ GOTO_ERROR(error);
}
PyObject *tuple = PySequence_Tuple(callargs);
if (tuple == NULL) {
- goto error;
+ GOTO_ERROR(error);
}
Py_SETREF(callargs, tuple);
}
int err = _Py_call_instrumentation_2args(
tstate, PY_MONITORING_EVENT_CALL,
frame, this_instr, func, arg);
- if (err) goto error;
+ if (err) GOTO_ERROR(error);
result = PyObject_Call(func, callargs, kwargs);
if (result == NULL) {
_Py_call_instrumentation_exc2(
// Need to manually shrink the stack since we exit with DISPATCH_INLINED.
STACK_SHRINK(oparg + 3);
if (new_frame == NULL) {
- goto error;
+ GOTO_ERROR(error);
}
assert(next_instr - this_instr == 1);
frame->return_offset = 1;
Py_DECREF(codeobj);
if (func_obj == NULL) {
- goto error;
+ GOTO_ERROR(error);
}
_PyFunction_SetVersion(
PyFunctionObject *func = (PyFunctionObject *)frame->f_funcobj;
PyGenObject *gen = (PyGenObject *)_Py_MakeCoro(func);
if (gen == NULL) {
- goto error;
+ GOTO_ERROR(error);
}
assert(EMPTY());
_PyFrame_SetStackPointer(frame, stack_pointer);
op(_POP_JUMP_IF_FALSE, (flag -- )) {
if (Py_IsFalse(flag)) {
- pc = oparg;
+ next_uop = current_executor->trace + oparg;
}
}
op(_POP_JUMP_IF_TRUE, (flag -- )) {
if (Py_IsTrue(flag)) {
- pc = oparg;
+ next_uop = current_executor->trace + oparg;
}
}
op(_JUMP_TO_TOP, (--)) {
- pc = 0;
+ next_uop = current_executor->trace;
CHECK_EVAL_BREAKER();
}
op(_SET_IP, (--)) {
TIER_TWO_ONLY
- frame->instr_ptr = ip_offset + oparg;
+ // TODO: Put the code pointer in `operand` to avoid indirection via `frame`
+ frame->instr_ptr = _PyCode_CODE(_PyFrame_GetCode(frame)) + oparg;
}
op(_SAVE_RETURN_OFFSET, (--)) {
op(_EXIT_TRACE, (--)) {
TIER_TWO_ONLY
- _PyFrame_SetStackPointer(frame, stack_pointer);
- Py_DECREF(self);
- OPT_HIST(trace_uop_execution_counter, trace_run_length_hist);
- return frame;
+ GOTO_TIER_ONE();
}
op(_INSERT, (unused[oparg], top -- top, unused[oparg])) {
if (r < 0) {
return -1;
}
- int lltrace = r;
+ int lltrace = r * 5; // Levels 1-4 only trace uops
if (!lltrace) {
- // When tracing executed uops, also trace bytecode
- char *uop_debug = Py_GETENV("PYTHONUOPSDEBUG");
- if (uop_debug != NULL && *uop_debug >= '0') {
- lltrace = (*uop_debug - '0') >= 5; // TODO: Parse an int and all that
+ // Can also be controlled by environment variable
+ char *python_lltrace = Py_GETENV("PYTHON_LLTRACE");
+ if (python_lltrace != NULL && *python_lltrace >= '0') {
+ lltrace = *python_lltrace - '0'; // TODO: Parse an int and all that
}
}
- if (lltrace) {
+ if (lltrace >= 5) {
lltrace_resume_frame(frame);
}
return lltrace;
return _PyEval_EvalFrame(tstate, f->f_frame, throwflag);
}
-#define TIER_ONE 1
#include "ceval_macros.h"
-
int _Py_CheckRecursiveCallPy(
PyThreadState *tstate)
{
#ifdef Py_STATS
int lastopcode = 0;
#endif
- // opcode is an 8-bit value to improve the code generated by MSVC
- // for the big switch below (in combination with the EXTRA_CASES macro).
- uint8_t opcode; /* Current opcode */
+ int opcode; /* Current opcode */
int oparg; /* Current opcode argument, if any */
#ifdef LLTRACE
int lltrace = 0;
goto resume_with_error;
}
+ /* State shared between Tier 1 and Tier 2 interpreter */
+ _PyUOpExecutorObject *current_executor = NULL;
+
/* Local "register" variables.
* These are cached values from the frame and code object. */
/* Start instructions */
#if !USE_COMPUTED_GOTOS
dispatch_opcode:
- switch (opcode)
+ // Cast to an 8-bit value to improve the code generated by MSVC
+ // (in combination with the EXTRA_CASES macro).
+ switch ((uint8_t)opcode)
#endif
{
}
/* Resume normal execution */
#ifdef LLTRACE
- if (lltrace) {
+ if (lltrace >= 5) {
lltrace_resume_frame(frame);
}
#endif
stack_pointer = _PyFrame_GetStackPointer(frame);
goto error;
+
+
+// The Tier 2 interpreter is also here!
+enter_tier_two:
+
+#undef LOAD_IP
+#define LOAD_IP(UNUSED) (void)0
+
+#undef GOTO_ERROR
+#define GOTO_ERROR(LABEL) goto LABEL ## _tier_two
+
+#undef DEOPT_IF
+#define DEOPT_IF(COND, INSTNAME) \
+ if ((COND)) { \
+ goto deoptimize;\
+ }
+
+#ifdef Py_STATS
+// Disable these macros that apply to Tier 1 stats when we are in Tier 2
+#undef STAT_INC
+#define STAT_INC(opname, name) ((void)0)
+#undef STAT_DEC
+#define STAT_DEC(opname, name) ((void)0)
+#undef CALL_STAT_INC
+#define CALL_STAT_INC(name) ((void)0)
+#endif
+
+#undef ENABLE_SPECIALIZATION
+#define ENABLE_SPECIALIZATION 0
+
+#ifdef Py_DEBUG
+ #define DPRINTF(level, ...) \
+ if (lltrace >= (level)) { printf(__VA_ARGS__); }
+#else
+ #define DPRINTF(level, ...)
+#endif
+
+ OPT_STAT_INC(traces_executed);
+ _PyUOpInstruction *next_uop = current_executor->trace;
+#ifdef Py_DEBUG
+ uint64_t operand; // Used by several DPRINTF() calls
+#endif
+#ifdef Py_STATS
+ uint64_t trace_uop_execution_counter = 0;
+#endif
+
+ for (;;) {
+ opcode = next_uop->opcode;
+ oparg = next_uop->oparg;
+#ifdef Py_DEBUG
+ operand = next_uop->operand;
+#endif
+ DPRINTF(3,
+ "%4d: uop %s, oparg %d, operand %" PRIu64 ", stack_level %d\n",
+ (int)(next_uop - current_executor->trace),
+ opcode < 256 ? _PyOpcode_OpName[opcode] : _PyOpcode_uop_name[opcode],
+ oparg,
+ operand,
+ (int)(stack_pointer - _PyFrame_Stackbase(frame)));
+ next_uop++;
+ OPT_STAT_INC(uops_executed);
+#ifdef Py_STATS
+ trace_uop_execution_counter++;
+#endif
+
+ switch (opcode) {
+
+#include "executor_cases.c.h"
+
+ default:
+#ifdef Py_DEBUG
+ {
+ fprintf(stderr, "Unknown uop %d, oparg %d, operand %" PRIu64 "\n",
+ opcode, oparg, operand);
+ Py_FatalError("Unknown uop");
+ }
+#else
+ Py_UNREACHABLE();
+#endif
+
+ }
+ }
+
+// Jump here from ERROR_IF(..., unbound_local_error)
+unbound_local_error_tier_two:
+ _PyEval_FormatExcCheckArg(tstate, PyExc_UnboundLocalError,
+ UNBOUNDLOCAL_ERROR_MSG,
+ PyTuple_GetItem(_PyFrame_GetCode(frame)->co_localsplusnames, oparg)
+ );
+ goto error_tier_two;
+
+// JUMP to any of these from ERROR_IF(..., error)
+pop_4_error_tier_two:
+ STACK_SHRINK(1);
+pop_3_error_tier_two:
+ STACK_SHRINK(1);
+pop_2_error_tier_two:
+ STACK_SHRINK(1);
+pop_1_error_tier_two:
+ STACK_SHRINK(1);
+error_tier_two:
+ DPRINTF(2, "Error: [Opcode %d, operand %" PRIu64 "]\n", opcode, operand);
+ OPT_HIST(trace_uop_execution_counter, trace_run_length_hist);
+ frame->return_offset = 0; // Don't leave this random
+ _PyFrame_SetStackPointer(frame, stack_pointer);
+ Py_DECREF(current_executor);
+ goto resume_with_error;
+
+// Jump here from DEOPT_IF()
+deoptimize:
+ // On DEOPT_IF we just repeat the last instruction.
+ // This presumes nothing was popped from the stack (nor pushed).
+ DPRINTF(2, "DEOPT: [Opcode %d, operand %" PRIu64 "]\n", opcode, operand);
+ OPT_HIST(trace_uop_execution_counter, trace_run_length_hist);
+ frame->return_offset = 0; // Dispatch to frame->instr_ptr
+ _PyFrame_SetStackPointer(frame, stack_pointer);
+ Py_DECREF(current_executor);
+ // Fall through
+// Jump here from ENTER_EXECUTOR
+enter_tier_one:
+ next_instr = frame->instr_ptr;
+ goto resume_frame;
+
+// Jump here from _EXIT_TRACE
+exit_trace:
+ _PyFrame_SetStackPointer(frame, stack_pointer);
+ Py_DECREF(current_executor);
+ OPT_HIST(trace_uop_execution_counter, trace_run_length_hist);
+ goto enter_tier_one;
}
#if defined(__GNUC__)
# pragma GCC diagnostic pop
-// Macros and other things needed by ceval.c, executor.c, and bytecodes.c
+// Macros and other things needed by ceval.c, and bytecodes.c
/* Computed GOTOs, or
the-optimization-commonly-but-improperly-known-as-"threaded code"
/* PRE_DISPATCH_GOTO() does lltrace if enabled. Normally a no-op */
#ifdef LLTRACE
-#define PRE_DISPATCH_GOTO() if (lltrace) { \
+#define PRE_DISPATCH_GOTO() if (lltrace >= 5) { \
lltrace_instruction(frame, stack_pointer, next_instr); }
#else
#define PRE_DISPATCH_GOTO() ((void)0)
goto start_frame; \
} while (0)
+// Use this instead of 'goto error' so Tier 2 can go to a different label
+#define GOTO_ERROR(LABEL) goto LABEL
+
#define CHECK_EVAL_BREAKER() \
_Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY(); \
if (_Py_atomic_load_uintptr_relaxed(&tstate->interp->ceval.eval_breaker) & _PY_EVAL_EVENTS_MASK) { \
if (_Py_HandlePending(tstate) != 0) { \
- goto error; \
+ GOTO_ERROR(error); \
} \
}
}\
else { \
result = PyFloat_FromDouble(dval); \
- if ((result) == NULL) goto error; \
+ if ((result) == NULL) GOTO_ERROR(error); \
_Py_DECREF_NO_DEALLOC(left); \
_Py_DECREF_NO_DEALLOC(right); \
} \
/* Implementation of "macros" that modify the instruction pointer,
* stack pointer, or frame pointer.
- * These need to treated differently by tier 1 and 2. */
-
-#if TIER_ONE
+ * These need to treated differently by tier 1 and 2.
+ * The Tier 1 version is here; Tier 2 is inlined in ceval.c. */
#define LOAD_IP(OFFSET) do { \
next_instr = frame->instr_ptr + (OFFSET); \
} while (0)
-#define STORE_SP() \
-_PyFrame_SetStackPointer(frame, stack_pointer)
-
-#define LOAD_SP() \
-stack_pointer = _PyFrame_GetStackPointer(frame);
-
-#endif
-
-
-#if TIER_TWO
-
-#define LOAD_IP(UNUSED) \
-do { ip_offset = (_Py_CODEUNIT *)_PyFrame_GetCode(frame)->co_code_adaptive; } while (0)
+/* There's no STORE_IP(), it's inlined by the code generator. */
#define STORE_SP() \
_PyFrame_SetStackPointer(frame, stack_pointer)
#define LOAD_SP() \
stack_pointer = _PyFrame_GetStackPointer(frame);
-#endif
-
-
+/* Tier-switching macros. */
+#define GOTO_TIER_TWO() goto enter_tier_two;
+#define GOTO_TIER_ONE() goto exit_trace;
+++ /dev/null
-#include "Python.h"
-
-#include "opcode.h"
-
-#include "pycore_bitutils.h"
-#include "pycore_call.h"
-#include "pycore_ceval.h"
-#include "pycore_dict.h"
-#include "pycore_emscripten_signal.h"
-#include "pycore_intrinsics.h"
-#include "pycore_long.h"
-#include "pycore_object.h"
-#include "pycore_opcode_metadata.h"
-#include "pycore_opcode_utils.h"
-#include "pycore_pyerrors.h"
-#include "pycore_range.h"
-#include "pycore_setobject.h" // _PySet_Update()
-#include "pycore_sliceobject.h"
-#include "pycore_uops.h"
-
-#define TIER_TWO 2
-#include "ceval_macros.h"
-
-
-#undef DEOPT_IF
-#define DEOPT_IF(COND, INSTNAME) \
- if ((COND)) { \
- UOP_STAT_INC(INSTNAME, miss); \
- goto deoptimize; \
- }
-
-#ifdef Py_STATS
-// Disable these macros that apply to Tier 1 stats when we are in Tier 2
-#undef STAT_INC
-#define STAT_INC(opname, name) ((void)0)
-#undef STAT_DEC
-#define STAT_DEC(opname, name) ((void)0)
-#undef CALL_STAT_INC
-#define CALL_STAT_INC(name) ((void)0)
-#endif
-
-#undef ENABLE_SPECIALIZATION
-#define ENABLE_SPECIALIZATION 0
-
-
-_PyInterpreterFrame *
-_PyUopExecute(_PyExecutorObject *executor, _PyInterpreterFrame *frame, PyObject **stack_pointer)
-{
-#ifdef Py_DEBUG
- char *uop_debug = Py_GETENV("PYTHONUOPSDEBUG");
- int lltrace = 0;
- if (uop_debug != NULL && *uop_debug >= '0') {
- lltrace = *uop_debug - '0'; // TODO: Parse an int and all that
- }
- #define DPRINTF(level, ...) \
- if (lltrace >= (level)) { printf(__VA_ARGS__); }
-#else
- #define DPRINTF(level, ...)
-#endif
-
- DPRINTF(3,
- "Entering _PyUopExecute for %s (%s:%d) at byte offset %ld\n",
- PyUnicode_AsUTF8(_PyFrame_GetCode(frame)->co_qualname),
- PyUnicode_AsUTF8(_PyFrame_GetCode(frame)->co_filename),
- _PyFrame_GetCode(frame)->co_firstlineno,
- 2 * (long)(frame->instr_ptr -
- (_Py_CODEUNIT *)_PyFrame_GetCode(frame)->co_code_adaptive));
-
- PyThreadState *tstate = _PyThreadState_GET();
- _PyUOpExecutorObject *self = (_PyUOpExecutorObject *)executor;
-
- CHECK_EVAL_BREAKER();
-
- OPT_STAT_INC(traces_executed);
- _Py_CODEUNIT *ip_offset = (_Py_CODEUNIT *)_PyFrame_GetCode(frame)->co_code_adaptive;
- int pc = 0;
- int opcode;
- int oparg;
- uint64_t operand;
-#ifdef Py_STATS
- uint64_t trace_uop_execution_counter = 0;
-#endif
-
- for (;;) {
- opcode = self->trace[pc].opcode;
- oparg = self->trace[pc].oparg;
- operand = self->trace[pc].operand;
- DPRINTF(3,
- "%4d: uop %s, oparg %d, operand %" PRIu64 ", stack_level %d\n",
- pc,
- opcode < 256 ? _PyOpcode_OpName[opcode] : _PyOpcode_uop_name[opcode],
- oparg,
- operand,
- (int)(stack_pointer - _PyFrame_Stackbase(frame)));
- pc++;
- OPT_STAT_INC(uops_executed);
- UOP_STAT_INC(opcode, execution_count);
-#ifdef Py_STATS
- trace_uop_execution_counter++;
-#endif
-
- switch (opcode) {
-
-#include "executor_cases.c.h"
-
- default:
- {
- fprintf(stderr, "Unknown uop %d, operand %" PRIu64 "\n", opcode, operand);
- Py_FatalError("Unknown uop");
- }
-
- }
- }
-
-unbound_local_error:
- _PyEval_FormatExcCheckArg(tstate, PyExc_UnboundLocalError,
- UNBOUNDLOCAL_ERROR_MSG,
- PyTuple_GetItem(_PyFrame_GetCode(frame)->co_localsplusnames, oparg)
- );
- goto error;
-
-pop_4_error:
- STACK_SHRINK(1);
-pop_3_error:
- STACK_SHRINK(1);
-pop_2_error:
- STACK_SHRINK(1);
-pop_1_error:
- STACK_SHRINK(1);
-error:
- // On ERROR_IF we return NULL as the frame.
- // The caller recovers the frame from tstate->current_frame.
- DPRINTF(2, "Error: [Opcode %d, operand %" PRIu64 "]\n", opcode, operand);
- OPT_HIST(trace_uop_execution_counter, trace_run_length_hist);
- frame->return_offset = 0; // Don't leave this random
- _PyFrame_SetStackPointer(frame, stack_pointer);
- Py_DECREF(self);
- return NULL;
-
-deoptimize:
- // On DEOPT_IF we just repeat the last instruction.
- // This presumes nothing was popped from the stack (nor pushed).
- DPRINTF(2, "DEOPT: [Opcode %d, operand %" PRIu64 "]\n", opcode, operand);
- OPT_HIST(trace_uop_execution_counter, trace_run_length_hist);
- frame->return_offset = 0; // Dispatch to frame->instr_ptr
- _PyFrame_SetStackPointer(frame, stack_pointer);
- Py_DECREF(self);
- return frame;
-}
// Python/bytecodes.c
// Do not edit!
+#ifdef TIER_ONE
+ #error "This file is for Tier 2 only"
+#endif
+#define TIER_TWO 2
+
case NOP: {
break;
}
case LOAD_FAST_CHECK: {
PyObject *value;
value = GETLOCAL(oparg);
- if (value == NULL) goto unbound_local_error;
+ if (value == NULL) goto unbound_local_error_tier_two;
Py_INCREF(value);
STACK_GROW(1);
stack_pointer[-1] = value;
value = stack_pointer[-1];
res = PyNumber_Negative(value);
Py_DECREF(value);
- if (res == NULL) goto pop_1_error;
+ if (res == NULL) goto pop_1_error_tier_two;
stack_pointer[-1] = res;
break;
}
value = stack_pointer[-1];
int err = PyObject_IsTrue(value);
Py_DECREF(value);
- if (err < 0) goto pop_1_error;
+ if (err < 0) goto pop_1_error_tier_two;
res = err ? Py_True : Py_False;
stack_pointer[-1] = res;
break;
PyObject *value;
PyObject *res;
value = stack_pointer[-1];
- uint32_t version = (uint32_t)operand;
+ uint32_t version = (uint32_t)next_uop[-1].operand;
// This one is a bit weird, because we expect *some* failures:
assert(version);
DEOPT_IF(Py_TYPE(value)->tp_version_tag != version, TO_BOOL);
value = stack_pointer[-1];
res = PyNumber_Invert(value);
Py_DECREF(value);
- if (res == NULL) goto pop_1_error;
+ if (res == NULL) goto pop_1_error_tier_two;
stack_pointer[-1] = res;
break;
}
res = _PyLong_Multiply((PyLongObject *)left, (PyLongObject *)right);
_Py_DECREF_SPECIALIZED(right, (destructor)PyObject_Free);
_Py_DECREF_SPECIALIZED(left, (destructor)PyObject_Free);
- if (res == NULL) goto pop_2_error;
+ if (res == NULL) goto pop_2_error_tier_two;
STACK_SHRINK(1);
stack_pointer[-1] = res;
break;
res = _PyLong_Add((PyLongObject *)left, (PyLongObject *)right);
_Py_DECREF_SPECIALIZED(right, (destructor)PyObject_Free);
_Py_DECREF_SPECIALIZED(left, (destructor)PyObject_Free);
- if (res == NULL) goto pop_2_error;
+ if (res == NULL) goto pop_2_error_tier_two;
STACK_SHRINK(1);
stack_pointer[-1] = res;
break;
res = _PyLong_Subtract((PyLongObject *)left, (PyLongObject *)right);
_Py_DECREF_SPECIALIZED(right, (destructor)PyObject_Free);
_Py_DECREF_SPECIALIZED(left, (destructor)PyObject_Free);
- if (res == NULL) goto pop_2_error;
+ if (res == NULL) goto pop_2_error_tier_two;
STACK_SHRINK(1);
stack_pointer[-1] = res;
break;
res = PyUnicode_Concat(left, right);
_Py_DECREF_SPECIALIZED(left, _PyUnicode_ExactDealloc);
_Py_DECREF_SPECIALIZED(right, _PyUnicode_ExactDealloc);
- if (res == NULL) goto pop_2_error;
+ if (res == NULL) goto pop_2_error_tier_two;
STACK_SHRINK(1);
stack_pointer[-1] = res;
break;
res = PyObject_GetItem(container, sub);
Py_DECREF(container);
Py_DECREF(sub);
- if (res == NULL) goto pop_2_error;
+ if (res == NULL) goto pop_2_error_tier_two;
STACK_SHRINK(1);
stack_pointer[-1] = res;
break;
Py_DECREF(slice);
}
Py_DECREF(container);
- if (res == NULL) goto pop_3_error;
+ if (res == NULL) goto pop_3_error_tier_two;
STACK_SHRINK(2);
stack_pointer[-1] = res;
break;
}
Py_DECREF(v);
Py_DECREF(container);
- if (err) goto pop_4_error;
+ if (err) goto pop_4_error_tier_two;
STACK_SHRINK(4);
break;
}
}
Py_DECREF(dict);
Py_DECREF(sub);
- if (true) goto pop_2_error;
+ if (true) goto pop_2_error_tier_two;
}
Py_INCREF(res); // Do this before DECREF'ing dict, sub
Py_DECREF(dict);
PyObject *list;
v = stack_pointer[-1];
list = stack_pointer[-2 - (oparg-1)];
- if (_PyList_AppendTakeRef((PyListObject *)list, v) < 0) goto pop_1_error;
+ if (_PyList_AppendTakeRef((PyListObject *)list, v) < 0) goto pop_1_error_tier_two;
STACK_SHRINK(1);
break;
}
set = stack_pointer[-2 - (oparg-1)];
int err = PySet_Add(set, v);
Py_DECREF(v);
- if (err) goto pop_1_error;
+ if (err) goto pop_1_error_tier_two;
STACK_SHRINK(1);
break;
}
Py_DECREF(v);
Py_DECREF(container);
Py_DECREF(sub);
- if (err) goto pop_3_error;
+ if (err) goto pop_3_error_tier_two;
STACK_SHRINK(3);
break;
}
STAT_INC(STORE_SUBSCR, hit);
int err = _PyDict_SetItem_Take2((PyDictObject *)dict, sub, value);
Py_DECREF(dict);
- if (err) goto pop_3_error;
+ if (err) goto pop_3_error_tier_two;
STACK_SHRINK(3);
break;
}
int err = PyObject_DelItem(container, sub);
Py_DECREF(container);
Py_DECREF(sub);
- if (err) goto pop_2_error;
+ if (err) goto pop_2_error_tier_two;
STACK_SHRINK(2);
break;
}
assert(oparg <= MAX_INTRINSIC_1);
res = _PyIntrinsics_UnaryFunctions[oparg].func(tstate, value);
Py_DECREF(value);
- if (res == NULL) goto pop_1_error;
+ if (res == NULL) goto pop_1_error_tier_two;
stack_pointer[-1] = res;
break;
}
res = _PyIntrinsics_BinaryFunctions[oparg].func(tstate, value2, value1);
Py_DECREF(value2);
Py_DECREF(value1);
- if (res == NULL) goto pop_2_error;
+ if (res == NULL) goto pop_2_error_tier_two;
STACK_SHRINK(1);
stack_pointer[-1] = res;
break;
"__aiter__ method, got %.100s",
type->tp_name);
Py_DECREF(obj);
- if (true) goto pop_1_error;
+ if (true) goto pop_1_error_tier_two;
}
iter = (*getter)(obj);
Py_DECREF(obj);
- if (iter == NULL) goto pop_1_error;
+ if (iter == NULL) goto pop_1_error_tier_two;
if (Py_TYPE(iter)->tp_as_async == NULL ||
Py_TYPE(iter)->tp_as_async->am_anext == NULL) {
"that does not implement __anext__: %.100s",
Py_TYPE(iter)->tp_name);
Py_DECREF(iter);
- if (true) goto pop_1_error;
+ if (true) goto pop_1_error_tier_two;
}
stack_pointer[-1] = iter;
break;
if (PyAsyncGen_CheckExact(aiter)) {
awaitable = type->tp_as_async->am_anext(aiter);
if (awaitable == NULL) {
- goto error;
+ GOTO_ERROR(error);
}
} else {
if (type->tp_as_async != NULL){
if (getter != NULL) {
next_iter = (*getter)(aiter);
if (next_iter == NULL) {
- goto error;
+ GOTO_ERROR(error);
}
}
else {
"'async for' requires an iterator with "
"__anext__ method, got %.100s",
type->tp_name);
- goto error;
+ GOTO_ERROR(error);
}
awaitable = _PyCoro_GetAwaitableIter(next_iter);
Py_TYPE(next_iter)->tp_name);
Py_DECREF(next_iter);
- goto error;
+ GOTO_ERROR(error);
} else {
Py_DECREF(next_iter);
}
}
}
- if (iter == NULL) goto pop_1_error;
+ if (iter == NULL) goto pop_1_error_tier_two;
stack_pointer[-1] = iter;
break;
}
case LOAD_BUILD_CLASS: {
PyObject *bc;
- if (PyMapping_GetOptionalItem(BUILTINS(), &_Py_ID(__build_class__), &bc) < 0) goto error;
+ if (PyMapping_GetOptionalItem(BUILTINS(), &_Py_ID(__build_class__), &bc) < 0) goto error_tier_two;
if (bc == NULL) {
_PyErr_SetString(tstate, PyExc_NameError,
"__build_class__ not found");
- if (true) goto error;
+ if (true) goto error_tier_two;
}
STACK_GROW(1);
stack_pointer[-1] = bc;
_PyErr_Format(tstate, PyExc_SystemError,
"no locals found when storing %R", name);
Py_DECREF(v);
- if (true) goto pop_1_error;
+ if (true) goto pop_1_error_tier_two;
}
if (PyDict_CheckExact(ns))
err = PyDict_SetItem(ns, name, v);
else
err = PyObject_SetItem(ns, name, v);
Py_DECREF(v);
- if (err) goto pop_1_error;
+ if (err) goto pop_1_error_tier_two;
STACK_SHRINK(1);
break;
}
if (ns == NULL) {
_PyErr_Format(tstate, PyExc_SystemError,
"no locals when deleting %R", name);
- goto error;
+ GOTO_ERROR(error);
}
err = PyObject_DelItem(ns, name);
// Can't use ERROR_IF here.
_PyEval_FormatExcCheckArg(tstate, PyExc_NameError,
NAME_ERROR_MSG,
name);
- goto error;
+ GOTO_ERROR(error);
}
break;
}
case _SPECIALIZE_UNPACK_SEQUENCE: {
PyObject *seq;
seq = stack_pointer[-1];
- uint16_t counter = (uint16_t)operand;
+ uint16_t counter = (uint16_t)next_uop[-1].operand;
#if ENABLE_SPECIALIZATION
if (ADAPTIVE_COUNTER_IS_ZERO(counter)) {
next_instr = this_instr;
PyObject **top = stack_pointer + oparg - 1;
int res = _PyEval_UnpackIterable(tstate, seq, oparg, -1, top);
Py_DECREF(seq);
- if (res == 0) goto pop_1_error;
+ if (res == 0) goto pop_1_error_tier_two;
STACK_SHRINK(1);
STACK_GROW(oparg);
break;
PyObject **top = stack_pointer + totalargs - 1;
int res = _PyEval_UnpackIterable(tstate, seq, oparg & 0xFF, oparg >> 8, top);
Py_DECREF(seq);
- if (res == 0) goto pop_1_error;
+ if (res == 0) goto pop_1_error_tier_two;
STACK_GROW((oparg & 0xFF) + (oparg >> 8));
break;
}
int err = PyObject_SetAttr(owner, name, v);
Py_DECREF(v);
Py_DECREF(owner);
- if (err) goto pop_2_error;
+ if (err) goto pop_2_error_tier_two;
STACK_SHRINK(2);
break;
}
PyObject *name = GETITEM(FRAME_CO_NAMES, oparg);
int err = PyObject_DelAttr(owner, name);
Py_DECREF(owner);
- if (err) goto pop_1_error;
+ if (err) goto pop_1_error_tier_two;
STACK_SHRINK(1);
break;
}
PyObject *name = GETITEM(FRAME_CO_NAMES, oparg);
int err = PyDict_SetItem(GLOBALS(), name, v);
Py_DECREF(v);
- if (err) goto pop_1_error;
+ if (err) goto pop_1_error_tier_two;
STACK_SHRINK(1);
break;
}
_PyEval_FormatExcCheckArg(tstate, PyExc_NameError,
NAME_ERROR_MSG, name);
}
- goto error;
+ GOTO_ERROR(error);
}
break;
}
if (locals == NULL) {
_PyErr_SetString(tstate, PyExc_SystemError,
"no locals found");
- if (true) goto error;
+ if (true) goto error_tier_two;
}
Py_INCREF(locals);
STACK_GROW(1);
mod_or_class_dict = stack_pointer[-1];
PyObject *name = GETITEM(FRAME_CO_NAMES, oparg);
if (PyMapping_GetOptionalItem(mod_or_class_dict, name, &v) < 0) {
- goto error;
+ GOTO_ERROR(error);
}
if (v == NULL) {
v = PyDict_GetItemWithError(GLOBALS(), name);
Py_INCREF(v);
}
else if (_PyErr_Occurred(tstate)) {
- goto error;
+ GOTO_ERROR(error);
}
else {
if (PyMapping_GetOptionalItem(BUILTINS(), name, &v) < 0) {
- goto error;
+ GOTO_ERROR(error);
}
if (v == NULL) {
_PyEval_FormatExcCheckArg(
tstate, PyExc_NameError,
NAME_ERROR_MSG, name);
- goto error;
+ GOTO_ERROR(error);
}
}
}
if (mod_or_class_dict == NULL) {
_PyErr_SetString(tstate, PyExc_SystemError,
"no locals found");
- if (true) goto error;
+ if (true) goto error_tier_two;
}
PyObject *name = GETITEM(FRAME_CO_NAMES, oparg);
if (PyMapping_GetOptionalItem(mod_or_class_dict, name, &v) < 0) {
- goto error;
+ GOTO_ERROR(error);
}
if (v == NULL) {
v = PyDict_GetItemWithError(GLOBALS(), name);
Py_INCREF(v);
}
else if (_PyErr_Occurred(tstate)) {
- goto error;
+ GOTO_ERROR(error);
}
else {
if (PyMapping_GetOptionalItem(BUILTINS(), name, &v) < 0) {
- goto error;
+ GOTO_ERROR(error);
}
if (v == NULL) {
_PyEval_FormatExcCheckArg(
tstate, PyExc_NameError,
NAME_ERROR_MSG, name);
- goto error;
+ GOTO_ERROR(error);
}
}
}
_PyEval_FormatExcCheckArg(tstate, PyExc_NameError,
NAME_ERROR_MSG, name);
}
- if (true) goto error;
+ if (true) goto error_tier_two;
}
Py_INCREF(res);
}
else {
/* Slow-path if globals or builtins is not a dict */
/* namespace 1: globals */
- if (PyMapping_GetOptionalItem(GLOBALS(), name, &res) < 0) goto error;
+ if (PyMapping_GetOptionalItem(GLOBALS(), name, &res) < 0) goto error_tier_two;
if (res == NULL) {
/* namespace 2: builtins */
- if (PyMapping_GetOptionalItem(BUILTINS(), name, &res) < 0) goto error;
+ if (PyMapping_GetOptionalItem(BUILTINS(), name, &res) < 0) goto error_tier_two;
if (res == NULL) {
_PyEval_FormatExcCheckArg(
tstate, PyExc_NameError,
NAME_ERROR_MSG, name);
- if (true) goto error;
+ if (true) goto error_tier_two;
}
}
}
}
case _GUARD_GLOBALS_VERSION: {
- uint16_t version = (uint16_t)operand;
+ uint16_t version = (uint16_t)next_uop[-1].operand;
PyDictObject *dict = (PyDictObject *)GLOBALS();
DEOPT_IF(!PyDict_CheckExact(dict), _GUARD_GLOBALS_VERSION);
DEOPT_IF(dict->ma_keys->dk_version != version, _GUARD_GLOBALS_VERSION);
}
case _GUARD_BUILTINS_VERSION: {
- uint16_t version = (uint16_t)operand;
+ uint16_t version = (uint16_t)next_uop[-1].operand;
PyDictObject *dict = (PyDictObject *)BUILTINS();
DEOPT_IF(!PyDict_CheckExact(dict), _GUARD_BUILTINS_VERSION);
DEOPT_IF(dict->ma_keys->dk_version != version, _GUARD_BUILTINS_VERSION);
case _LOAD_GLOBAL_MODULE: {
PyObject *res;
PyObject *null = NULL;
- uint16_t index = (uint16_t)operand;
+ uint16_t index = (uint16_t)next_uop[-1].operand;
PyDictObject *dict = (PyDictObject *)GLOBALS();
PyDictUnicodeEntry *entries = DK_UNICODE_ENTRIES(dict->ma_keys);
res = entries[index].me_value;
case _LOAD_GLOBAL_BUILTINS: {
PyObject *res;
PyObject *null = NULL;
- uint16_t index = (uint16_t)operand;
+ uint16_t index = (uint16_t)next_uop[-1].operand;
PyDictObject *bdict = (PyDictObject *)BUILTINS();
PyDictUnicodeEntry *entries = DK_UNICODE_ENTRIES(bdict->ma_keys);
res = entries[index].me_value;
case DELETE_FAST: {
PyObject *v = GETLOCAL(oparg);
- if (v == NULL) goto unbound_local_error;
+ if (v == NULL) goto unbound_local_error_tier_two;
SETLOCAL(oparg, NULL);
break;
}
PyObject *initial = GETLOCAL(oparg);
PyObject *cell = PyCell_New(initial);
if (cell == NULL) {
- goto error;
+ GOTO_ERROR(error);
}
SETLOCAL(oparg, cell);
break;
// Fortunately we don't need its superpower.
if (oldobj == NULL) {
_PyEval_FormatExcUnbound(tstate, _PyFrame_GetCode(frame), oparg);
- goto error;
+ GOTO_ERROR(error);
}
PyCell_SET(cell, NULL);
Py_DECREF(oldobj);
name = PyTuple_GET_ITEM(_PyFrame_GetCode(frame)->co_localsplusnames, oparg);
if (PyMapping_GetOptionalItem(class_dict, name, &value) < 0) {
Py_DECREF(class_dict);
- goto error;
+ GOTO_ERROR(error);
}
Py_DECREF(class_dict);
if (!value) {
value = PyCell_GET(cell);
if (value == NULL) {
_PyEval_FormatExcUnbound(tstate, _PyFrame_GetCode(frame), oparg);
- goto error;
+ GOTO_ERROR(error);
}
Py_INCREF(value);
}
value = PyCell_GET(cell);
if (value == NULL) {
_PyEval_FormatExcUnbound(tstate, _PyFrame_GetCode(frame), oparg);
- if (true) goto error;
+ if (true) goto error_tier_two;
}
Py_INCREF(value);
STACK_GROW(1);
for (int _i = oparg; --_i >= 0;) {
Py_DECREF(pieces[_i]);
}
- if (str == NULL) { STACK_SHRINK(oparg); goto error; }
+ if (str == NULL) { STACK_SHRINK(oparg); goto error_tier_two; }
STACK_SHRINK(oparg);
STACK_GROW(1);
stack_pointer[-1] = str;
PyObject *tup;
values = stack_pointer - oparg;
tup = _PyTuple_FromArraySteal(values, oparg);
- if (tup == NULL) { STACK_SHRINK(oparg); goto error; }
+ if (tup == NULL) { STACK_SHRINK(oparg); goto error_tier_two; }
STACK_SHRINK(oparg);
STACK_GROW(1);
stack_pointer[-1] = tup;
PyObject *list;
values = stack_pointer - oparg;
list = _PyList_FromArraySteal(values, oparg);
- if (list == NULL) { STACK_SHRINK(oparg); goto error; }
+ if (list == NULL) { STACK_SHRINK(oparg); goto error_tier_two; }
STACK_SHRINK(oparg);
STACK_GROW(1);
stack_pointer[-1] = list;
Py_TYPE(iterable)->tp_name);
}
Py_DECREF(iterable);
- if (true) goto pop_1_error;
+ if (true) goto pop_1_error_tier_two;
}
assert(Py_IsNone(none_val));
Py_DECREF(iterable);
set = stack_pointer[-2 - (oparg-1)];
int err = _PySet_Update(set, iterable);
Py_DECREF(iterable);
- if (err < 0) goto pop_1_error;
+ if (err < 0) goto pop_1_error_tier_two;
STACK_SHRINK(1);
break;
}
values = stack_pointer - oparg;
set = PySet_New(NULL);
if (set == NULL)
- goto error;
+ GOTO_ERROR(error);
int err = 0;
for (int i = 0; i < oparg; i++) {
PyObject *item = values[i];
}
if (err != 0) {
Py_DECREF(set);
- if (true) { STACK_SHRINK(oparg); goto error; }
+ if (true) { STACK_SHRINK(oparg); goto error_tier_two; }
}
STACK_SHRINK(oparg);
STACK_GROW(1);
for (int _i = oparg*2; --_i >= 0;) {
Py_DECREF(values[_i]);
}
- if (map == NULL) { STACK_SHRINK(oparg*2); goto error; }
+ if (map == NULL) { STACK_SHRINK(oparg*2); goto error_tier_two; }
STACK_SHRINK(oparg*2);
STACK_GROW(1);
stack_pointer[-1] = map;
if (LOCALS() == NULL) {
_PyErr_Format(tstate, PyExc_SystemError,
"no locals found when setting up annotations");
- if (true) goto error;
+ if (true) goto error_tier_two;
}
/* check if __annotations__ in locals()... */
if (PyDict_CheckExact(LOCALS())) {
ann_dict = _PyDict_GetItemWithError(LOCALS(),
&_Py_ID(__annotations__));
if (ann_dict == NULL) {
- if (_PyErr_Occurred(tstate)) goto error;
+ if (_PyErr_Occurred(tstate)) goto error_tier_two;
/* ...if not, create a new one */
ann_dict = PyDict_New();
- if (ann_dict == NULL) goto error;
+ if (ann_dict == NULL) goto error_tier_two;
err = PyDict_SetItem(LOCALS(), &_Py_ID(__annotations__),
ann_dict);
Py_DECREF(ann_dict);
- if (err) goto error;
+ if (err) goto error_tier_two;
}
}
else {
/* do the same if locals() is not a dict */
- if (PyMapping_GetOptionalItem(LOCALS(), &_Py_ID(__annotations__), &ann_dict) < 0) goto error;
+ if (PyMapping_GetOptionalItem(LOCALS(), &_Py_ID(__annotations__), &ann_dict) < 0) goto error_tier_two;
if (ann_dict == NULL) {
ann_dict = PyDict_New();
- if (ann_dict == NULL) goto error;
+ if (ann_dict == NULL) goto error_tier_two;
err = PyObject_SetItem(LOCALS(), &_Py_ID(__annotations__),
ann_dict);
Py_DECREF(ann_dict);
- if (err) goto error;
+ if (err) goto error_tier_two;
}
else {
Py_DECREF(ann_dict);
PyTuple_GET_SIZE(keys) != (Py_ssize_t)oparg) {
_PyErr_SetString(tstate, PyExc_SystemError,
"bad BUILD_CONST_KEY_MAP keys argument");
- goto error; // Pop the keys and values.
+ GOTO_ERROR(error); // Pop the keys and values.
}
map = _PyDict_FromItems(
&PyTuple_GET_ITEM(keys, 0), 1,
Py_DECREF(values[_i]);
}
Py_DECREF(keys);
- if (map == NULL) { STACK_SHRINK(oparg); goto pop_1_error; }
+ if (map == NULL) { STACK_SHRINK(oparg); goto pop_1_error_tier_two; }
STACK_SHRINK(oparg);
stack_pointer[-1] = map;
break;
Py_TYPE(update)->tp_name);
}
Py_DECREF(update);
- if (true) goto pop_1_error;
+ if (true) goto pop_1_error_tier_two;
}
Py_DECREF(update);
STACK_SHRINK(1);
if (_PyDict_MergeEx(dict, update, 2) < 0) {
_PyEval_FormatKwargsError(tstate, callable, update);
Py_DECREF(update);
- if (true) goto pop_1_error;
+ if (true) goto pop_1_error_tier_two;
}
Py_DECREF(update);
STACK_SHRINK(1);
assert(PyDict_CheckExact(dict));
/* dict[key] = value */
// Do not DECREF INPUTS because the function steals the references
- if (_PyDict_SetItem_Take2((PyDictObject *)dict, key, value) != 0) goto pop_2_error;
+ if (_PyDict_SetItem_Take2((PyDictObject *)dict, key, value) != 0) goto pop_2_error_tier_two;
STACK_SHRINK(2);
break;
}
Py_DECREF(global_super);
Py_DECREF(class);
Py_DECREF(self);
- if (attr == NULL) goto pop_3_error;
+ if (attr == NULL) goto pop_3_error_tier_two;
STACK_SHRINK(2);
stack_pointer[-1] = attr;
break;
Py_DECREF(class);
if (attr == NULL) {
Py_DECREF(self);
- if (true) goto pop_3_error;
+ if (true) goto pop_3_error_tier_two;
}
if (method_found) {
self_or_null = self; // transfer ownership
NULL | meth | arg1 | ... | argN
*/
Py_DECREF(owner);
- if (attr == NULL) goto pop_1_error;
+ if (attr == NULL) goto pop_1_error_tier_two;
self_or_null = NULL;
}
}
/* Classic, pushes one value. */
attr = PyObject_GetAttr(owner, name);
Py_DECREF(owner);
- if (attr == NULL) goto pop_1_error;
+ if (attr == NULL) goto pop_1_error_tier_two;
}
STACK_GROW(((oparg & 1) ? 1 : 0));
stack_pointer[-1 - (oparg & 1 ? 1 : 0)] = attr;
case _GUARD_TYPE_VERSION: {
PyObject *owner;
owner = stack_pointer[-1];
- uint32_t type_version = (uint32_t)operand;
+ uint32_t type_version = (uint32_t)next_uop[-1].operand;
PyTypeObject *tp = Py_TYPE(owner);
assert(type_version != 0);
DEOPT_IF(tp->tp_version_tag != type_version, _GUARD_TYPE_VERSION);
PyObject *attr;
PyObject *null = NULL;
owner = stack_pointer[-1];
- uint16_t index = (uint16_t)operand;
+ uint16_t index = (uint16_t)next_uop[-1].operand;
PyDictOrValues dorv = *_PyObject_DictOrValuesPointer(owner);
attr = _PyDictOrValues_GetValues(dorv)->values[index];
DEOPT_IF(attr == NULL, _LOAD_ATTR_INSTANCE_VALUE);
case _CHECK_ATTR_MODULE: {
PyObject *owner;
owner = stack_pointer[-1];
- uint32_t type_version = (uint32_t)operand;
+ uint32_t type_version = (uint32_t)next_uop[-1].operand;
DEOPT_IF(!PyModule_CheckExact(owner), _CHECK_ATTR_MODULE);
PyDictObject *dict = (PyDictObject *)((PyModuleObject *)owner)->md_dict;
assert(dict != NULL);
PyObject *attr;
PyObject *null = NULL;
owner = stack_pointer[-1];
- uint16_t index = (uint16_t)operand;
+ uint16_t index = (uint16_t)next_uop[-1].operand;
PyDictObject *dict = (PyDictObject *)((PyModuleObject *)owner)->md_dict;
assert(dict->ma_keys->dk_kind == DICT_KEYS_UNICODE);
assert(index < dict->ma_keys->dk_nentries);
PyObject *attr;
PyObject *null = NULL;
owner = stack_pointer[-1];
- uint16_t hint = (uint16_t)operand;
+ uint16_t hint = (uint16_t)next_uop[-1].operand;
PyDictOrValues dorv = *_PyObject_DictOrValuesPointer(owner);
PyDictObject *dict = (PyDictObject *)_PyDictOrValues_GetDict(dorv);
DEOPT_IF(hint >= (size_t)dict->ma_keys->dk_nentries, _LOAD_ATTR_WITH_HINT);
PyObject *attr;
PyObject *null = NULL;
owner = stack_pointer[-1];
- uint16_t index = (uint16_t)operand;
+ uint16_t index = (uint16_t)next_uop[-1].operand;
char *addr = (char *)owner + index;
attr = *(PyObject **)addr;
DEOPT_IF(attr == NULL, _LOAD_ATTR_SLOT);
case _CHECK_ATTR_CLASS: {
PyObject *owner;
owner = stack_pointer[-1];
- uint32_t type_version = (uint32_t)operand;
+ uint32_t type_version = (uint32_t)next_uop[-1].operand;
DEOPT_IF(!PyType_Check(owner), _CHECK_ATTR_CLASS);
assert(type_version != 0);
DEOPT_IF(((PyTypeObject *)owner)->tp_version_tag != type_version, _CHECK_ATTR_CLASS);
PyObject *attr;
PyObject *null = NULL;
owner = stack_pointer[-1];
- PyObject *descr = (PyObject *)operand;
+ PyObject *descr = (PyObject *)next_uop[-1].operand;
STAT_INC(LOAD_ATTR, hit);
assert(descr != NULL);
attr = Py_NewRef(descr);
PyObject *value;
owner = stack_pointer[-1];
value = stack_pointer[-2];
- uint16_t index = (uint16_t)operand;
+ uint16_t index = (uint16_t)next_uop[-1].operand;
PyDictOrValues dorv = *_PyObject_DictOrValuesPointer(owner);
STAT_INC(STORE_ATTR, hit);
PyDictValues *values = _PyDictOrValues_GetValues(dorv);
PyObject *value;
owner = stack_pointer[-1];
value = stack_pointer[-2];
- uint16_t index = (uint16_t)operand;
+ uint16_t index = (uint16_t)next_uop[-1].operand;
char *addr = (char *)owner + index;
STAT_INC(STORE_ATTR, hit);
PyObject *old_value = *(PyObject **)addr;
res = PyObject_RichCompare(left, right, oparg >> 5);
Py_DECREF(left);
Py_DECREF(right);
- if (res == NULL) goto pop_2_error;
+ if (res == NULL) goto pop_2_error_tier_two;
if (oparg & 16) {
int res_bool = PyObject_IsTrue(res);
Py_DECREF(res);
- if (res_bool < 0) goto pop_2_error;
+ if (res_bool < 0) goto pop_2_error_tier_two;
res = res_bool ? Py_True : Py_False;
}
STACK_SHRINK(1);
int res = PySequence_Contains(right, left);
Py_DECREF(left);
Py_DECREF(right);
- if (res < 0) goto pop_2_error;
+ if (res < 0) goto pop_2_error_tier_two;
b = (res ^ oparg) ? Py_True : Py_False;
STACK_SHRINK(1);
stack_pointer[-1] = b;
if (_PyEval_CheckExceptStarTypeValid(tstate, match_type) < 0) {
Py_DECREF(exc_value);
Py_DECREF(match_type);
- if (true) goto pop_2_error;
+ if (true) goto pop_2_error_tier_two;
}
match = NULL;
&match, &rest);
Py_DECREF(exc_value);
Py_DECREF(match_type);
- if (res < 0) goto pop_2_error;
+ if (res < 0) goto pop_2_error_tier_two;
assert((match == NULL) == (rest == NULL));
- if (match == NULL) goto pop_2_error;
+ if (match == NULL) goto pop_2_error_tier_two;
if (!Py_IsNone(match)) {
PyErr_SetHandledException(match);
assert(PyExceptionInstance_Check(left));
if (_PyEval_CheckExceptTypeValid(tstate, right) < 0) {
Py_DECREF(right);
- if (true) goto pop_1_error;
+ if (true) goto pop_1_error_tier_two;
}
int res = PyErr_GivenExceptionMatches(left, right);
obj = stack_pointer[-1];
// PUSH(len(TOS))
Py_ssize_t len_i = PyObject_Length(obj);
- if (len_i < 0) goto error;
+ if (len_i < 0) goto error_tier_two;
len_o = PyLong_FromSsize_t(len_i);
- if (len_o == NULL) goto error;
+ if (len_o == NULL) goto error_tier_two;
STACK_GROW(1);
stack_pointer[-1] = len_o;
break;
assert(PyTuple_CheckExact(attrs)); // Success!
}
else {
- if (_PyErr_Occurred(tstate)) goto pop_3_error;
+ if (_PyErr_Occurred(tstate)) goto pop_3_error_tier_two;
attrs = Py_None; // Failure!
}
STACK_SHRINK(2);
subject = stack_pointer[-2];
// On successful match, PUSH(values). Otherwise, PUSH(None).
values_or_none = _PyEval_MatchKeys(tstate, subject, keys);
- if (values_or_none == NULL) goto error;
+ if (values_or_none == NULL) goto error_tier_two;
STACK_GROW(1);
stack_pointer[-1] = values_or_none;
break;
/* before: [obj]; after [getiter(obj)] */
iter = PyObject_GetIter(iterable);
Py_DECREF(iterable);
- if (iter == NULL) goto pop_1_error;
+ if (iter == NULL) goto pop_1_error_tier_two;
stack_pointer[-1] = iter;
break;
}
_PyErr_SetString(tstate, PyExc_TypeError,
"cannot 'yield from' a coroutine object "
"in a non-coroutine generator");
- goto error;
+ GOTO_ERROR(error);
}
iter = iterable;
}
/* `iterable` is not a generator. */
iter = PyObject_GetIter(iterable);
if (iter == NULL) {
- goto error;
+ GOTO_ERROR(error);
}
Py_DECREF(iterable);
}
r->start = value + r->step;
r->len--;
next = PyLong_FromLong(value);
- if (next == NULL) goto error;
+ if (next == NULL) goto error_tier_two;
STACK_GROW(1);
stack_pointer[-1] = next;
break;
"asynchronous context manager protocol",
Py_TYPE(mgr)->tp_name);
}
- goto error;
+ GOTO_ERROR(error);
}
exit = _PyObject_LookupSpecial(mgr, &_Py_ID(__aexit__));
if (exit == NULL) {
Py_TYPE(mgr)->tp_name);
}
Py_DECREF(enter);
- goto error;
+ GOTO_ERROR(error);
}
Py_DECREF(mgr);
res = _PyObject_CallNoArgs(enter);
Py_DECREF(enter);
if (res == NULL) {
Py_DECREF(exit);
- if (true) goto pop_1_error;
+ if (true) goto pop_1_error_tier_two;
}
STACK_GROW(1);
stack_pointer[-2] = exit;
PyObject *stack[4] = {NULL, exc, val, tb};
res = PyObject_Vectorcall(exit_func, stack + 1,
3 | PY_VECTORCALL_ARGUMENTS_OFFSET, NULL);
- if (res == NULL) goto error;
+ if (res == NULL) goto error_tier_two;
STACK_GROW(1);
stack_pointer[-1] = res;
break;
case _GUARD_KEYS_VERSION: {
PyObject *owner;
owner = stack_pointer[-1];
- uint32_t keys_version = (uint32_t)operand;
+ uint32_t keys_version = (uint32_t)next_uop[-1].operand;
PyTypeObject *owner_cls = Py_TYPE(owner);
PyHeapTypeObject *owner_heap_type = (PyHeapTypeObject *)owner_cls;
DEOPT_IF(owner_heap_type->ht_cached_keys->dk_version != keys_version, _GUARD_KEYS_VERSION);
PyObject *attr;
PyObject *self;
owner = stack_pointer[-1];
- PyObject *descr = (PyObject *)operand;
+ PyObject *descr = (PyObject *)next_uop[-1].operand;
assert(oparg & 1);
/* Cached method object */
STAT_INC(LOAD_ATTR, hit);
PyObject *attr;
PyObject *self;
owner = stack_pointer[-1];
- PyObject *descr = (PyObject *)operand;
+ PyObject *descr = (PyObject *)next_uop[-1].operand;
assert(oparg & 1);
assert(Py_TYPE(owner)->tp_dictoffset == 0);
STAT_INC(LOAD_ATTR, hit);
PyObject *owner;
PyObject *attr;
owner = stack_pointer[-1];
- PyObject *descr = (PyObject *)operand;
+ PyObject *descr = (PyObject *)next_uop[-1].operand;
assert((oparg & 1) == 0);
STAT_INC(LOAD_ATTR, hit);
assert(descr != NULL);
PyObject *owner;
PyObject *attr;
owner = stack_pointer[-1];
- PyObject *descr = (PyObject *)operand;
+ PyObject *descr = (PyObject *)next_uop[-1].operand;
assert((oparg & 1) == 0);
assert(Py_TYPE(owner)->tp_dictoffset == 0);
STAT_INC(LOAD_ATTR, hit);
PyObject *attr;
PyObject *self;
owner = stack_pointer[-1];
- PyObject *descr = (PyObject *)operand;
+ PyObject *descr = (PyObject *)next_uop[-1].operand;
assert(oparg & 1);
STAT_INC(LOAD_ATTR, hit);
assert(descr != NULL);
PyObject *callable;
self_or_null = stack_pointer[-1 - oparg];
callable = stack_pointer[-2 - oparg];
- uint32_t func_version = (uint32_t)operand;
+ uint32_t func_version = (uint32_t)next_uop[-1].operand;
DEOPT_IF(!PyFunction_Check(callable), _CHECK_FUNCTION_EXACT_ARGS);
PyFunctionObject *func = (PyFunctionObject *)callable;
DEOPT_IF(func->func_version != func_version, _CHECK_FUNCTION_EXACT_ARGS);
res = PyObject_Str(arg);
Py_DECREF(arg);
Py_DECREF(&PyUnicode_Type); // I.e., callable
- if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; }
+ if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error_tier_two; }
STACK_SHRINK(oparg);
STACK_SHRINK(1);
stack_pointer[-1] = res;
res = PySequence_Tuple(arg);
Py_DECREF(arg);
Py_DECREF(&PyTuple_Type); // I.e., tuple
- if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; }
+ if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error_tier_two; }
STACK_SHRINK(oparg);
STACK_SHRINK(1);
stack_pointer[-1] = res;
PyErr_Format(PyExc_TypeError,
"__init__() should return None, not '%.200s'",
Py_TYPE(should_be_none)->tp_name);
- goto error;
+ GOTO_ERROR(error);
}
STACK_SHRINK(1);
break;
Py_DECREF(args[i]);
}
Py_DECREF(tp);
- if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; }
+ if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error_tier_two; }
STACK_SHRINK(oparg);
STACK_SHRINK(1);
stack_pointer[-1] = res;
// This is slower but CPython promises to check all non-vectorcall
// function calls.
if (_Py_EnterRecursiveCallTstate(tstate, " while calling a Python object")) {
- goto error;
+ GOTO_ERROR(error);
}
PyObject *arg = args[0];
res = _PyCFunction_TrampolineCall(cfunc, PyCFunction_GET_SELF(callable), arg);
Py_DECREF(arg);
Py_DECREF(callable);
- if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; }
+ if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error_tier_two; }
STACK_SHRINK(oparg);
STACK_SHRINK(1);
stack_pointer[-1] = res;
Py_DECREF(args[i]);
}
Py_DECREF(callable);
- if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; }
+ if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error_tier_two; }
/* Not deopting because this doesn't mean our optimization was
wrong. `res` can be NULL for valid reasons. Eg. getattr(x,
'invalid'). In those cases an exception is set, so we must
Py_DECREF(args[i]);
}
Py_DECREF(callable);
- if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; }
+ if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error_tier_two; }
STACK_SHRINK(oparg);
STACK_SHRINK(1);
stack_pointer[-1] = res;
PyObject *arg = args[0];
Py_ssize_t len_i = PyObject_Length(arg);
if (len_i < 0) {
- goto error;
+ GOTO_ERROR(error);
}
res = PyLong_FromSsize_t(len_i);
assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL));
Py_DECREF(callable);
Py_DECREF(arg);
- if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; }
+ if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error_tier_two; }
STACK_SHRINK(oparg);
STACK_SHRINK(1);
stack_pointer[-1] = res;
PyObject *inst = args[0];
int retval = PyObject_IsInstance(inst, cls);
if (retval < 0) {
- goto error;
+ GOTO_ERROR(error);
}
res = PyBool_FromLong(retval);
assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL));
Py_DECREF(inst);
Py_DECREF(cls);
Py_DECREF(callable);
- if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; }
+ if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error_tier_two; }
STACK_SHRINK(oparg);
STACK_SHRINK(1);
stack_pointer[-1] = res;
// This is slower but CPython promises to check all non-vectorcall
// function calls.
if (_Py_EnterRecursiveCallTstate(tstate, " while calling a Python object")) {
- goto error;
+ GOTO_ERROR(error);
}
res = _PyCFunction_TrampolineCall(cfunc, self, arg);
_Py_LeaveRecursiveCallTstate(tstate);
Py_DECREF(self);
Py_DECREF(arg);
Py_DECREF(callable);
- if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; }
+ if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error_tier_two; }
STACK_SHRINK(oparg);
STACK_SHRINK(1);
stack_pointer[-1] = res;
Py_DECREF(args[i]);
}
Py_DECREF(callable);
- if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; }
+ if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error_tier_two; }
STACK_SHRINK(oparg);
STACK_SHRINK(1);
stack_pointer[-1] = res;
// This is slower but CPython promises to check all non-vectorcall
// function calls.
if (_Py_EnterRecursiveCallTstate(tstate, " while calling a Python object")) {
- goto error;
+ GOTO_ERROR(error);
}
res = _PyCFunction_TrampolineCall(cfunc, self, NULL);
_Py_LeaveRecursiveCallTstate(tstate);
assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL));
Py_DECREF(self);
Py_DECREF(callable);
- if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; }
+ if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error_tier_two; }
STACK_SHRINK(oparg);
STACK_SHRINK(1);
stack_pointer[-1] = res;
Py_DECREF(args[i]);
}
Py_DECREF(callable);
- if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; }
+ if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error_tier_two; }
STACK_SHRINK(oparg);
STACK_SHRINK(1);
stack_pointer[-1] = res;
Py_DECREF(codeobj);
if (func_obj == NULL) {
- goto error;
+ GOTO_ERROR(error);
}
_PyFunction_SetVersion(
Py_DECREF(start);
Py_DECREF(stop);
Py_XDECREF(step);
- if (slice == NULL) { STACK_SHRINK(((oparg == 3) ? 1 : 0)); goto pop_2_error; }
+ if (slice == NULL) { STACK_SHRINK(((oparg == 3) ? 1 : 0)); goto pop_2_error_tier_two; }
STACK_SHRINK(((oparg == 3) ? 1 : 0));
STACK_SHRINK(1);
stack_pointer[-1] = slice;
conv_fn = CONVERSION_FUNCTIONS[oparg];
result = conv_fn(value);
Py_DECREF(value);
- if (result == NULL) goto pop_1_error;
+ if (result == NULL) goto pop_1_error_tier_two;
stack_pointer[-1] = result;
break;
}
if (!PyUnicode_CheckExact(value)) {
res = PyObject_Format(value, NULL);
Py_DECREF(value);
- if (res == NULL) goto pop_1_error;
+ if (res == NULL) goto pop_1_error_tier_two;
}
else {
res = value;
res = PyObject_Format(value, fmt_spec);
Py_DECREF(value);
Py_DECREF(fmt_spec);
- if (res == NULL) goto pop_2_error;
+ if (res == NULL) goto pop_2_error_tier_two;
STACK_SHRINK(1);
stack_pointer[-1] = res;
break;
res = _PyEval_BinaryOps[oparg](lhs, rhs);
Py_DECREF(lhs);
Py_DECREF(rhs);
- if (res == NULL) goto pop_2_error;
+ if (res == NULL) goto pop_2_error_tier_two;
STACK_SHRINK(1);
stack_pointer[-1] = res;
break;
PyObject *flag;
flag = stack_pointer[-1];
if (Py_IsFalse(flag)) {
- pc = oparg;
+ next_uop = current_executor->trace + oparg;
}
STACK_SHRINK(1);
break;
PyObject *flag;
flag = stack_pointer[-1];
if (Py_IsTrue(flag)) {
- pc = oparg;
+ next_uop = current_executor->trace + oparg;
}
STACK_SHRINK(1);
break;
}
case _JUMP_TO_TOP: {
- pc = 0;
+ next_uop = current_executor->trace;
CHECK_EVAL_BREAKER();
break;
}
case _SET_IP: {
TIER_TWO_ONLY
- frame->instr_ptr = ip_offset + oparg;
+ // TODO: Put the code pointer in `operand` to avoid indirection via `frame`
+ frame->instr_ptr = _PyCode_CODE(_PyFrame_GetCode(frame)) + oparg;
break;
}
case _EXIT_TRACE: {
TIER_TWO_ONLY
- _PyFrame_SetStackPointer(frame, stack_pointer);
- Py_DECREF(self);
- OPT_HIST(trace_uop_execution_counter, trace_run_length_hist);
- return frame;
+ GOTO_TIER_ONE();
break;
}
stack_pointer[-1 - oparg] = top;
break;
}
+
+#undef TIER_TWO
// Python/bytecodes.c
// Do not edit!
+#ifdef TIER_TWO
+ #error "This file is for Tier 1 only"
+#endif
+#define TIER_ONE 1
+
TARGET(NOP) {
frame->instr_ptr = next_instr;
next_instr += 1;
uintptr_t code_version = _PyFrame_GetCode(frame)->_co_instrumentation_version;
if (code_version != global_version) {
if (_Py_Instrument(_PyFrame_GetCode(frame), tstate->interp)) {
- goto error;
+ GOTO_ERROR(error);
}
next_instr = this_instr;
}
if (PyGen_Check(receiver)) {
PyErr_SetObject(PyExc_StopIteration, value);
if (monitor_stop_iteration(tstate, frame, this_instr)) {
- goto error;
+ GOTO_ERROR(error);
}
PyErr_SetRaisedException(NULL);
}
if (PyGen_Check(receiver) || PyCoro_CheckExact(receiver)) {
PyErr_SetObject(PyExc_StopIteration, value);
if (monitor_stop_iteration(tstate, frame, this_instr)) {
- goto error;
+ GOTO_ERROR(error);
}
PyErr_SetRaisedException(NULL);
}
int err = _Py_call_instrumentation_arg(
tstate, PY_MONITORING_EVENT_PY_RETURN,
frame, this_instr, retval);
- if (err) goto error;
+ if (err) GOTO_ERROR(error);
STACK_SHRINK(1);
assert(EMPTY());
_PyFrame_SetStackPointer(frame, stack_pointer);
int err = _Py_call_instrumentation_arg(
tstate, PY_MONITORING_EVENT_PY_RETURN,
frame, this_instr, retval);
- if (err) goto error;
+ if (err) GOTO_ERROR(error);
Py_INCREF(retval);
assert(EMPTY());
_PyFrame_SetStackPointer(frame, stack_pointer);
if (PyAsyncGen_CheckExact(aiter)) {
awaitable = type->tp_as_async->am_anext(aiter);
if (awaitable == NULL) {
- goto error;
+ GOTO_ERROR(error);
}
} else {
if (type->tp_as_async != NULL){
if (getter != NULL) {
next_iter = (*getter)(aiter);
if (next_iter == NULL) {
- goto error;
+ GOTO_ERROR(error);
}
}
else {
"'async for' requires an iterator with "
"__anext__ method, got %.100s",
type->tp_name);
- goto error;
+ GOTO_ERROR(error);
}
awaitable = _PyCoro_GetAwaitableIter(next_iter);
Py_TYPE(next_iter)->tp_name);
Py_DECREF(next_iter);
- goto error;
+ GOTO_ERROR(error);
} else {
Py_DECREF(next_iter);
}
JUMPBY(oparg);
}
else {
- goto error;
+ GOTO_ERROR(error);
}
}
Py_DECREF(v);
int err = _Py_call_instrumentation_arg(
tstate, PY_MONITORING_EVENT_PY_YIELD,
frame, this_instr, retval);
- if (err) goto error;
+ if (err) GOTO_ERROR(error);
tstate->exc_info = gen->gi_exc_state.previous_item;
gen->gi_exc_state.previous_item = NULL;
_Py_LeaveRecursiveCallPy(tstate);
else {
assert(PyLong_Check(lasti));
_PyErr_SetString(tstate, PyExc_SystemError, "lasti is not an int");
- goto error;
+ GOTO_ERROR(error);
}
}
assert(exc && PyExceptionInstance_Check(exc));
if (ns == NULL) {
_PyErr_Format(tstate, PyExc_SystemError,
"no locals when deleting %R", name);
- goto error;
+ GOTO_ERROR(error);
}
err = PyObject_DelItem(ns, name);
// Can't use ERROR_IF here.
_PyEval_FormatExcCheckArg(tstate, PyExc_NameError,
NAME_ERROR_MSG,
name);
- goto error;
+ GOTO_ERROR(error);
}
DISPATCH();
}
_PyEval_FormatExcCheckArg(tstate, PyExc_NameError,
NAME_ERROR_MSG, name);
}
- goto error;
+ GOTO_ERROR(error);
}
DISPATCH();
}
mod_or_class_dict = stack_pointer[-1];
PyObject *name = GETITEM(FRAME_CO_NAMES, oparg);
if (PyMapping_GetOptionalItem(mod_or_class_dict, name, &v) < 0) {
- goto error;
+ GOTO_ERROR(error);
}
if (v == NULL) {
v = PyDict_GetItemWithError(GLOBALS(), name);
Py_INCREF(v);
}
else if (_PyErr_Occurred(tstate)) {
- goto error;
+ GOTO_ERROR(error);
}
else {
if (PyMapping_GetOptionalItem(BUILTINS(), name, &v) < 0) {
- goto error;
+ GOTO_ERROR(error);
}
if (v == NULL) {
_PyEval_FormatExcCheckArg(
tstate, PyExc_NameError,
NAME_ERROR_MSG, name);
- goto error;
+ GOTO_ERROR(error);
}
}
}
}
PyObject *name = GETITEM(FRAME_CO_NAMES, oparg);
if (PyMapping_GetOptionalItem(mod_or_class_dict, name, &v) < 0) {
- goto error;
+ GOTO_ERROR(error);
}
if (v == NULL) {
v = PyDict_GetItemWithError(GLOBALS(), name);
Py_INCREF(v);
}
else if (_PyErr_Occurred(tstate)) {
- goto error;
+ GOTO_ERROR(error);
}
else {
if (PyMapping_GetOptionalItem(BUILTINS(), name, &v) < 0) {
- goto error;
+ GOTO_ERROR(error);
}
if (v == NULL) {
_PyEval_FormatExcCheckArg(
tstate, PyExc_NameError,
NAME_ERROR_MSG, name);
- goto error;
+ GOTO_ERROR(error);
}
}
}
PyObject *initial = GETLOCAL(oparg);
PyObject *cell = PyCell_New(initial);
if (cell == NULL) {
- goto error;
+ GOTO_ERROR(error);
}
SETLOCAL(oparg, cell);
DISPATCH();
// Fortunately we don't need its superpower.
if (oldobj == NULL) {
_PyEval_FormatExcUnbound(tstate, _PyFrame_GetCode(frame), oparg);
- goto error;
+ GOTO_ERROR(error);
}
PyCell_SET(cell, NULL);
Py_DECREF(oldobj);
name = PyTuple_GET_ITEM(_PyFrame_GetCode(frame)->co_localsplusnames, oparg);
if (PyMapping_GetOptionalItem(class_dict, name, &value) < 0) {
Py_DECREF(class_dict);
- goto error;
+ GOTO_ERROR(error);
}
Py_DECREF(class_dict);
if (!value) {
value = PyCell_GET(cell);
if (value == NULL) {
_PyEval_FormatExcUnbound(tstate, _PyFrame_GetCode(frame), oparg);
- goto error;
+ GOTO_ERROR(error);
}
Py_INCREF(value);
}
values = stack_pointer - oparg;
set = PySet_New(NULL);
if (set == NULL)
- goto error;
+ GOTO_ERROR(error);
int err = 0;
for (int i = 0; i < oparg; i++) {
PyObject *item = values[i];
PyTuple_GET_SIZE(keys) != (Py_ssize_t)oparg) {
_PyErr_SetString(tstate, PyExc_SystemError,
"bad BUILD_CONST_KEY_MAP keys argument");
- goto error; // Pop the keys and values.
+ GOTO_ERROR(error); // Pop the keys and values.
}
map = _PyDict_FromItems(
&PyTuple_GET_ITEM(keys, 0), 1,
JUMPBY(1-original_oparg);
frame->instr_ptr = next_instr;
Py_INCREF(executor);
+ if (executor->execute == _PyUopExecute) {
+ current_executor = (_PyUOpExecutorObject *)executor;
+ GOTO_TIER_TWO();
+ }
frame = executor->execute(executor, frame, stack_pointer);
if (frame == NULL) {
frame = tstate->current_frame;
goto resume_with_error;
}
- next_instr = frame->instr_ptr;
- goto resume_frame;
+ goto enter_tier_one;
}
TARGET(POP_JUMP_IF_FALSE) {
_PyErr_SetString(tstate, PyExc_TypeError,
"cannot 'yield from' a coroutine object "
"in a non-coroutine generator");
- goto error;
+ GOTO_ERROR(error);
}
iter = iterable;
}
/* `iterable` is not a generator. */
iter = PyObject_GetIter(iterable);
if (iter == NULL) {
- goto error;
+ GOTO_ERROR(error);
}
Py_DECREF(iterable);
}
if (next == NULL) {
if (_PyErr_Occurred(tstate)) {
if (!_PyErr_ExceptionMatches(tstate, PyExc_StopIteration)) {
- goto error;
+ GOTO_ERROR(error);
}
monitor_raise(tstate, frame, this_instr);
_PyErr_Clear(tstate);
else {
if (_PyErr_Occurred(tstate)) {
if (!_PyErr_ExceptionMatches(tstate, PyExc_StopIteration)) {
- goto error;
+ GOTO_ERROR(error);
}
monitor_raise(tstate, frame, this_instr);
_PyErr_Clear(tstate);
"asynchronous context manager protocol",
Py_TYPE(mgr)->tp_name);
}
- goto error;
+ GOTO_ERROR(error);
}
exit = _PyObject_LookupSpecial(mgr, &_Py_ID(__aexit__));
if (exit == NULL) {
Py_TYPE(mgr)->tp_name);
}
Py_DECREF(enter);
- goto error;
+ GOTO_ERROR(error);
}
Py_DECREF(mgr);
res = _PyObject_CallNoArgs(enter);
"context manager protocol",
Py_TYPE(mgr)->tp_name);
}
- goto error;
+ GOTO_ERROR(error);
}
exit = _PyObject_LookupSpecial(mgr, &_Py_ID(__exit__));
if (exit == NULL) {
Py_TYPE(mgr)->tp_name);
}
Py_DECREF(enter);
- goto error;
+ GOTO_ERROR(error);
}
Py_DECREF(mgr);
res = _PyObject_CallNoArgs(enter);
// The frame has stolen all the arguments from the stack,
// so there is no need to clean them up.
if (new_frame == NULL) {
- goto error;
+ GOTO_ERROR(error);
}
frame->return_offset = (uint16_t)(next_instr - this_instr);
DISPATCH_INLINED(new_frame);
STAT_INC(CALL, hit);
PyObject *self = _PyType_NewManagedObject(tp);
if (self == NULL) {
- goto error;
+ GOTO_ERROR(error);
}
Py_DECREF(tp);
_PyInterpreterFrame *shim = _PyFrame_PushTrampolineUnchecked(
PyErr_Format(PyExc_TypeError,
"__init__() should return None, not '%.200s'",
Py_TYPE(should_be_none)->tp_name);
- goto error;
+ GOTO_ERROR(error);
}
STACK_SHRINK(1);
DISPATCH();
// This is slower but CPython promises to check all non-vectorcall
// function calls.
if (_Py_EnterRecursiveCallTstate(tstate, " while calling a Python object")) {
- goto error;
+ GOTO_ERROR(error);
}
PyObject *arg = args[0];
res = _PyCFunction_TrampolineCall(cfunc, PyCFunction_GET_SELF(callable), arg);
PyObject *arg = args[0];
Py_ssize_t len_i = PyObject_Length(arg);
if (len_i < 0) {
- goto error;
+ GOTO_ERROR(error);
}
res = PyLong_FromSsize_t(len_i);
assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL));
PyObject *inst = args[0];
int retval = PyObject_IsInstance(inst, cls);
if (retval < 0) {
- goto error;
+ GOTO_ERROR(error);
}
res = PyBool_FromLong(retval);
assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL));
// This is slower but CPython promises to check all non-vectorcall
// function calls.
if (_Py_EnterRecursiveCallTstate(tstate, " while calling a Python object")) {
- goto error;
+ GOTO_ERROR(error);
}
res = _PyCFunction_TrampolineCall(cfunc, self, arg);
_Py_LeaveRecursiveCallTstate(tstate);
// This is slower but CPython promises to check all non-vectorcall
// function calls.
if (_Py_EnterRecursiveCallTstate(tstate, " while calling a Python object")) {
- goto error;
+ GOTO_ERROR(error);
}
res = _PyCFunction_TrampolineCall(cfunc, self, NULL);
_Py_LeaveRecursiveCallTstate(tstate);
// The frame has stolen all the arguments from the stack,
// so there is no need to clean them up.
if (new_frame == NULL) {
- goto error;
+ GOTO_ERROR(error);
}
assert(next_instr - this_instr == 1);
frame->return_offset = 1;
assert(kwargs == NULL || PyDict_CheckExact(kwargs));
if (!PyTuple_CheckExact(callargs)) {
if (check_args_iterable(tstate, func, callargs) < 0) {
- goto error;
+ GOTO_ERROR(error);
}
PyObject *tuple = PySequence_Tuple(callargs);
if (tuple == NULL) {
- goto error;
+ GOTO_ERROR(error);
}
Py_SETREF(callargs, tuple);
}
int err = _Py_call_instrumentation_2args(
tstate, PY_MONITORING_EVENT_CALL,
frame, this_instr, func, arg);
- if (err) goto error;
+ if (err) GOTO_ERROR(error);
result = PyObject_Call(func, callargs, kwargs);
if (result == NULL) {
_Py_call_instrumentation_exc2(
// Need to manually shrink the stack since we exit with DISPATCH_INLINED.
STACK_SHRINK(oparg + 3);
if (new_frame == NULL) {
- goto error;
+ GOTO_ERROR(error);
}
assert(next_instr - this_instr == 1);
frame->return_offset = 1;
Py_DECREF(codeobj);
if (func_obj == NULL) {
- goto error;
+ GOTO_ERROR(error);
}
_PyFunction_SetVersion(
PyFunctionObject *func = (PyFunctionObject *)frame->f_funcobj;
PyGenObject *gen = (PyGenObject *)_Py_MakeCoro(func);
if (gen == NULL) {
- goto error;
+ GOTO_ERROR(error);
}
assert(EMPTY());
_PyFrame_SetStackPointer(frame, stack_pointer);
assert(0 && "Executing RESERVED instruction.");
Py_UNREACHABLE();
}
+
+#undef TIER_ONE
int trace_stack_depth = 0;
#ifdef Py_DEBUG
- char *uop_debug = Py_GETENV("PYTHONUOPSDEBUG");
+ char *python_lltrace = Py_GETENV("PYTHON_LLTRACE");
int lltrace = 0;
- if (uop_debug != NULL && *uop_debug >= '0') {
- lltrace = *uop_debug - '0'; // TODO: Parse an int and all that
+ if (python_lltrace != NULL && *python_lltrace >= '0') {
+ lltrace = *python_lltrace - '0'; // TODO: Parse an int and all that
}
#endif
if (dest < last_instr) {
int new_trace_length = move_stubs(trace, dest, last_instr, trace_length);
#ifdef Py_DEBUG
- char *uop_debug = Py_GETENV("PYTHONUOPSDEBUG");
+ char *python_lltrace = Py_GETENV("PYTHON_LLTRACE");
int lltrace = 0;
- if (uop_debug != NULL && *uop_debug >= '0') {
- lltrace = *uop_debug - '0'; // TODO: Parse an int and all that
+ if (python_lltrace != NULL && *python_lltrace >= '0') {
+ lltrace = *python_lltrace - '0'; // TODO: Parse an int and all that
}
if (lltrace >= 2) {
printf("Optimized trace (length %d+%d = %d, saved %d):\n",
return 1;
}
+/* Dummy execute() function for Uop Executor.
+ * The actual implementation is inlined in ceval.c,
+ * in _PyEval_EvalFrameDefault(). */
+_PyInterpreterFrame *
+_PyUopExecute(_PyExecutorObject *executor, _PyInterpreterFrame *frame, PyObject **stack_pointer)
+{
+ Py_FatalError("Tier 2 is now inlined into Tier 1");
+}
+
static void
uop_opt_dealloc(PyObject *self) {
PyObject_Free(self);
// Turn on experimental tier 2 (uops-based) optimizer
if (is_main_interp) {
- char *envvar = Py_GETENV("PYTHONUOPS");
+ char *envvar = Py_GETENV("PYTHON_UOPS");
int enabled = envvar != NULL && *envvar > '0';
if (_Py_get_xoption(&config->xoptions, L"uops") != NULL) {
enabled = 1;
self.write_provenance_header()
+ self.out.write_raw("\n")
+ self.out.write_raw("#ifdef TIER_TWO\n")
+ self.out.write_raw(" #error \"This file is for Tier 1 only\"\n")
+ self.out.write_raw("#endif\n")
+ self.out.write_raw("#define TIER_ONE 1\n")
+
# Write and count instructions of all kinds
n_macros = 0
for thing in self.everything:
case _:
assert_never(thing)
+ self.out.write_raw("\n")
+ self.out.write_raw("#undef TIER_ONE\n")
+
print(
f"Wrote {n_macros} cases to {output_filename}",
file=sys.stderr,
with open(executor_filename, "w") as f:
self.out = Formatter(f, 8, emit_line_directives)
self.write_provenance_header()
+
+ self.out.write_raw("\n")
+ self.out.write_raw("#ifdef TIER_ONE\n")
+ self.out.write_raw(" #error \"This file is for Tier 2 only\"\n")
+ self.out.write_raw("#endif\n")
+ self.out.write_raw("#define TIER_TWO 2\n")
+
for instr in self.instrs.values():
if instr.is_viable_uop():
n_uops += 1
if instr.check_eval_breaker:
self.out.emit("CHECK_EVAL_BREAKER();")
self.out.emit("break;")
+
+ self.out.write_raw("\n")
+ self.out.write_raw("#undef TIER_TWO\n")
+
print(
f"Wrote {n_uops} cases to {executor_filename}",
file=sys.stderr,
f"{func}(&this_instr[{active.offset + 1}].cache);"
)
else:
- out.emit(f"{typ}{ceffect.name} = ({typ.strip()})operand;")
+ out.emit(f"{typ}{ceffect.name} = ({typ.strip()})next_uop[-1].operand;")
# Write the body, substituting a goto for ERROR_IF() and other stuff
assert dedent <= 0
ninputs, symbolic = list_effect_size(ieffs)
if ninputs:
label = f"pop_{ninputs}_{label}"
+ if tier == TIER_TWO:
+ label = label + "_tier_two"
if symbolic:
out.write_raw(
f"{space}if ({cond}) {{ STACK_SHRINK({symbolic}); goto {label}; }}\n"
projects / thirdparty / Python / cpython.git / commitdiff
