#define SPECIAL___AEXIT__ 3
#define SPECIAL_MAX 3
+struct _PyCode12 _PyCode_DEF(12);
+PyAPI_DATA(const struct _PyCode12) _PyEntryFrameCode;
+
#ifdef __cplusplus
}
#endif
* compare bytes and str which can raise a BytesWarning exception. */
extern PyObject* _PyCode_ConstantKey(PyObject *obj);
+#define NO_LOC_4 (128 | (PY_CODE_LOCATION_INFO_NONE << 3) | 3)
+
+static const PyBytesObject no_location = {
+ PyVarObject_HEAD_INIT(&PyBytes_Type, 1)
+ .ob_sval = { NO_LOC_4 }
+};
#ifdef __cplusplus
}
_PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(TextIOWrapper));
_PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(True));
_PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(WarningMessage));
+ _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(_PyEval_EvalFrameDefault));
_PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(_WindowsConsoleIO));
_PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(__IOBase_closed));
_PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(__abc_tpflags__));
STRUCT_FOR_ID(TextIOWrapper)
STRUCT_FOR_ID(True)
STRUCT_FOR_ID(WarningMessage)
+ STRUCT_FOR_ID(_PyEval_EvalFrameDefault)
STRUCT_FOR_ID(_WindowsConsoleIO)
STRUCT_FOR_ID(__IOBase_closed)
STRUCT_FOR_ID(__abc_tpflags__)
[END_ASYNC_FOR] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG | HAS_UNPREDICTABLE_JUMP_FLAG },
[END_FOR] = { true, INSTR_FMT_IX, HAS_ESCAPES_FLAG | HAS_NO_SAVE_IP_FLAG },
[END_SEND] = { true, INSTR_FMT_IX, HAS_ESCAPES_FLAG | HAS_PURE_FLAG },
- [ENTER_EXECUTOR] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG },
+ [ENTER_EXECUTOR] = { true, INSTR_FMT_IB, HAS_ARG_FLAG },
[EXIT_INIT_CHECK] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG },
[EXTENDED_ARG] = { true, INSTR_FMT_IB, HAS_ARG_FLAG },
[FORMAT_SIMPLE] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG },
[INSTRUMENTED_RESUME] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG },
[INSTRUMENTED_RETURN_VALUE] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG },
[INSTRUMENTED_YIELD_VALUE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG },
- [INTERPRETER_EXIT] = { true, INSTR_FMT_IX, HAS_ESCAPES_FLAG },
+ [INTERPRETER_EXIT] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG },
[IS_OP] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ESCAPES_FLAG },
[JUMP_BACKWARD] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_EVAL_BREAK_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG },
[JUMP_BACKWARD_JIT] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_EVAL_BREAK_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG },
INIT_ID(TextIOWrapper), \
INIT_ID(True), \
INIT_ID(WarningMessage), \
+ INIT_ID(_PyEval_EvalFrameDefault), \
INIT_ID(_WindowsConsoleIO), \
INIT_ID(__IOBase_closed), \
INIT_ID(__abc_tpflags__), \
_PyUnicode_InternStatic(interp, &string);
assert(_PyUnicode_CheckConsistency(string, 1));
assert(PyUnicode_GET_LENGTH(string) != 1);
+ string = &_Py_ID(_PyEval_EvalFrameDefault);
+ _PyUnicode_InternStatic(interp, &string);
+ assert(_PyUnicode_CheckConsistency(string, 1));
+ assert(PyUnicode_GET_LENGTH(string) != 1);
string = &_Py_ID(_WindowsConsoleIO);
_PyUnicode_InternStatic(interp, &string);
assert(_PyUnicode_CheckConsistency(string, 1));
op(_QUICKEN_RESUME, (--)) {
#if ENABLE_SPECIALIZATION_FT
- if (tstate->tracing == 0 && this_instr->op.code == RESUME) {
+ PyCodeObject *code = _PyFrame_GetCode(frame);
+ if (tstate->tracing == 0 && this_instr->op.code == RESUME &&
+ code != (PyCodeObject *)&_Py_InitCleanup && code != (PyCodeObject *)&_PyEntryFrameCode) {
FT_ATOMIC_STORE_UINT8_RELAXED(this_instr->op.code, RESUME_CHECK);
}
#endif /* ENABLE_SPECIALIZATION_FT */
tstate->current_frame = frame->previous;
assert(!_PyErr_Occurred(tstate));
PyObject *result = PyStackRef_AsPyObjectSteal(retval);
+ if (IS_JIT_TRACING()) {
+#if _Py_TIER2
+ _PyJit_translate_single_bytecode_to_trace(tstate, frame, next_instr);
+ LEAVE_TRACING();
+ int err = bail_tracing_and_jit(tstate, frame);
+ if (err < 0) {
+ Py_DECREF(result);
+ ERROR_IF(true);
+ }
+ return result;
+#endif
+ }
+ else {
#if !_Py_TAIL_CALL_INTERP
- assert(frame == &entry.frame);
+ assert(frame == &entry.frame);
#endif
#ifdef _Py_TIER2
- _PyStackRef executor = frame->localsplus[0];
- assert(tstate->current_executor == NULL);
- if (!PyStackRef_IsNull(executor)) {
- tstate->current_executor = PyStackRef_AsPyObjectBorrow(executor);
- PyStackRef_CLOSE(executor);
- }
+ _PyStackRef executor = frame->localsplus[0];
+ assert(tstate->current_executor == NULL);
+ if (!PyStackRef_IsNull(executor)) {
+ tstate->current_executor = PyStackRef_AsPyObjectBorrow(executor);
+ PyStackRef_CLOSE(executor);
+ }
#endif
- LLTRACE_RESUME_FRAME();
- return result;
+ LLTRACE_RESUME_FRAME();
+ return result;
+ }
}
// The stack effect here is a bit misleading.
tier1 inst(ENTER_EXECUTOR, (--)) {
#ifdef _Py_TIER2
- // We want to end any current trace here, before we possibly need
- // to start tracing new ones due to recursive traces in any inner C functions
- // in tier2 code.
- if (IS_JIT_TRACING()) {
- _PyJit_translate_single_bytecode_to_trace(tstate, frame, next_instr);
- LEAVE_TRACING();
- int err = bail_tracing_and_jit(tstate, frame);
- ERROR_IF(err < 0);
- }
PyCodeObject *code = _PyFrame_GetCode(frame);
_PyExecutorObject *executor = code->co_executors->executors[oparg & 255];
assert(executor->vm_data.index == INSTR_OFFSET() - 1);
/* If the eval breaker is set then stay in tier 1.
* This avoids any potentially infinite loops
* involving _RESUME_CHECK */
- if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) {
+ if (IS_JIT_TRACING() || _Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) {
opcode = executor->vm_data.opcode;
oparg = (oparg & ~255) | executor->vm_data.oparg;
next_instr = this_instr;
if (_PyOpcode_Caches[_PyOpcode_Deopt[opcode]]) {
PAUSE_ADAPTIVE_COUNTER(this_instr[1].counter);
}
- DISPATCH_GOTO();
+ if (IS_JIT_TRACING()) {
+ DISPATCH_GOTO_NON_TRACING();
+ }
+ else {
+ DISPATCH_GOTO();
+ }
}
assert(executor != tstate->interp->cold_executor);
tstate->jit_exit = NULL;
return 0;
}
-static const _Py_CODEUNIT _Py_INTERPRETER_TRAMPOLINE_INSTRUCTIONS[] = {
- /* Put a NOP at the start, so that the IP points into
- * the code, rather than before it */
- { .op.code = NOP, .op.arg = 0 },
- { .op.code = INTERPRETER_EXIT, .op.arg = 0 }, /* reached on return */
- { .op.code = NOP, .op.arg = 0 },
- { .op.code = INTERPRETER_EXIT, .op.arg = 0 }, /* reached on yield */
- { .op.code = RESUME, .op.arg = RESUME_OPARG_DEPTH1_MASK | RESUME_AT_FUNC_START }
+
+#ifdef Py_GIL_DISABLED
+static _PyCodeArray emtry_cleanup_tlbc = {
+ .size = 1,
+ .entries = {(char*) &_PyEntryFrameCode.co_code_adaptive},
+};
+#endif
+
+const struct _PyCode12 _PyEntryFrameCode = {
+ _PyVarObject_HEAD_INIT(&PyCode_Type, 5),
+ .co_consts = (PyObject *)&_Py_SINGLETON(tuple_empty),
+ .co_names = (PyObject *)&_Py_SINGLETON(tuple_empty),
+ .co_exceptiontable = (PyObject *)&_Py_SINGLETON(bytes_empty),
+ .co_flags = CO_OPTIMIZED | CO_NO_MONITORING_EVENTS,
+ .co_localsplusnames = (PyObject *)&_Py_SINGLETON(tuple_empty),
+ .co_localspluskinds = (PyObject *)&_Py_SINGLETON(bytes_empty),
+ .co_filename = &_Py_ID(_PyEval_EvalFrameDefault),
+ .co_name = &_Py_ID(_PyEval_EvalFrameDefault),
+ .co_qualname = &_Py_ID(_PyEval_EvalFrameDefault),
+ .co_linetable = (PyObject *)&no_location,
+ ._co_firsttraceable = 4,
+ .co_stacksize = 2,
+ .co_framesize = 2 + FRAME_SPECIALS_SIZE,
+#ifdef Py_GIL_DISABLED
+ .co_tlbc = &emtry_cleanup_tlbc,
+#endif
+ .co_code_adaptive = {
+ NOP, 0,
+ INTERPRETER_EXIT, 0, /* reached on return */
+ NOP, 0,
+ INTERPRETER_EXIT, 0, /* reached on yield */
+ RESUME, RESUME_OPARG_DEPTH1_MASK | RESUME_AT_FUNC_START
+ }
};
#ifdef Py_DEBUG
// Likewise, we hold a strong reference to the executor containing this exit, so the exit is guaranteed
// to be valid to access.
if (err <= 0) {
- exit->temperature = restart_backoff_counter(exit->temperature);
+ // Some opcodes will forever be unchanged. Don't ever bother specializing for them ever again.
+ if (tstate->interp->jit_state.prev_instr->op.code == INTERPRETER_EXIT) {
+ exit->temperature = initial_unreachable_backoff_counter();
+ }
+ else {
+ exit->temperature = restart_backoff_counter(exit->temperature);
+ }
}
else {
exit->temperature = initial_temperature_backoff_counter();
entry.frame.f_globals = (PyObject*)0xaaa3;
entry.frame.f_builtins = (PyObject*)0xaaa4;
#endif
- entry.frame.f_executable = PyStackRef_None;
- entry.frame.instr_ptr = (_Py_CODEUNIT *)_Py_INTERPRETER_TRAMPOLINE_INSTRUCTIONS + 1;
+ entry.frame.f_executable = PyStackRef_FromPyObjectBorrow((PyObject *)&_PyEntryFrameCode);
+ entry.frame.instr_ptr = ((_Py_CODEUNIT *)_PyEntryFrameCode.co_code_adaptive) + 1;
entry.frame.stackpointer = entry.stack;
entry.frame.owner = FRAME_OWNED_BY_INTERPRETER;
entry.frame.visited = 0;
INSTRUCTION_STATS(ENTER_EXECUTOR);
opcode = ENTER_EXECUTOR;
#ifdef _Py_TIER2
-
- if (IS_JIT_TRACING()) {
- _PyFrame_SetStackPointer(frame, stack_pointer);
- _PyJit_translate_single_bytecode_to_trace(tstate, frame, next_instr);
- stack_pointer = _PyFrame_GetStackPointer(frame);
- LEAVE_TRACING();
- _PyFrame_SetStackPointer(frame, stack_pointer);
- int err = bail_tracing_and_jit(tstate, frame);
- stack_pointer = _PyFrame_GetStackPointer(frame);
- if (err < 0) {
- JUMP_TO_LABEL(error);
- }
- }
PyCodeObject *code = _PyFrame_GetCode(frame);
_PyExecutorObject *executor = code->co_executors->executors[oparg & 255];
assert(executor->vm_data.index == INSTR_OFFSET() - 1);
assert(executor->vm_data.code == code);
assert(executor->vm_data.valid);
assert(tstate->current_executor == NULL);
- if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) {
+ if (IS_JIT_TRACING() || _Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) {
opcode = executor->vm_data.opcode;
oparg = (oparg & ~255) | executor->vm_data.oparg;
next_instr = this_instr;
if (_PyOpcode_Caches[_PyOpcode_Deopt[opcode]]) {
PAUSE_ADAPTIVE_COUNTER(this_instr[1].counter);
}
- DISPATCH_GOTO();
+ if (IS_JIT_TRACING()) {
+ DISPATCH_GOTO_NON_TRACING();
+ }
+ else {
+ DISPATCH_GOTO();
+ }
}
assert(executor != tstate->interp->cold_executor);
tstate->jit_exit = NULL;
tstate->current_frame = frame->previous;
assert(!_PyErr_Occurred(tstate));
PyObject *result = PyStackRef_AsPyObjectSteal(retval);
- #if !_Py_TAIL_CALL_INTERP
- assert(frame == &entry.frame);
- #endif
- #ifdef _Py_TIER2
- _PyStackRef executor = frame->localsplus[0];
- assert(tstate->current_executor == NULL);
- if (!PyStackRef_IsNull(executor)) {
- tstate->current_executor = PyStackRef_AsPyObjectBorrow(executor);
+ if (IS_JIT_TRACING()) {
+ #if _Py_TIER2
stack_pointer += -1;
assert(WITHIN_STACK_BOUNDS());
_PyFrame_SetStackPointer(frame, stack_pointer);
- PyStackRef_CLOSE(executor);
+ _PyJit_translate_single_bytecode_to_trace(tstate, frame, next_instr);
stack_pointer = _PyFrame_GetStackPointer(frame);
- stack_pointer += 1;
+ LEAVE_TRACING();
+ _PyFrame_SetStackPointer(frame, stack_pointer);
+ int err = bail_tracing_and_jit(tstate, frame);
+ stack_pointer = _PyFrame_GetStackPointer(frame);
+ if (err < 0) {
+ _PyFrame_SetStackPointer(frame, stack_pointer);
+ Py_DECREF(result);
+ stack_pointer = _PyFrame_GetStackPointer(frame);
+ JUMP_TO_LABEL(error);
+ }
+ return result;
+ #endif
+ }
+ else {
+ #if !_Py_TAIL_CALL_INTERP
+ assert(frame == &entry.frame);
+ #endif
+ #ifdef _Py_TIER2
+ _PyStackRef executor = frame->localsplus[0];
+ assert(tstate->current_executor == NULL);
+ if (!PyStackRef_IsNull(executor)) {
+ tstate->current_executor = PyStackRef_AsPyObjectBorrow(executor);
+ stack_pointer += -1;
+ assert(WITHIN_STACK_BOUNDS());
+ _PyFrame_SetStackPointer(frame, stack_pointer);
+ PyStackRef_CLOSE(executor);
+ stack_pointer = _PyFrame_GetStackPointer(frame);
+ stack_pointer += 1;
+ }
+ #endif
+ LLTRACE_RESUME_FRAME();
+ return result;
}
- #endif
- LLTRACE_RESUME_FRAME();
- return result;
}
TARGET(IS_OP) {
// _QUICKEN_RESUME
{
#if ENABLE_SPECIALIZATION_FT
- if (tstate->tracing == 0 && this_instr->op.code == RESUME) {
+ PyCodeObject *code = _PyFrame_GetCode(frame);
+ if (tstate->tracing == 0 && this_instr->op.code == RESUME &&
+ code != (PyCodeObject *)&_Py_InitCleanup && code != (PyCodeObject *)&_PyEntryFrameCode) {
FT_ATOMIC_STORE_UINT8_RELAXED(this_instr->op.code, RESUME_CHECK);
}
#endif /* ENABLE_SPECIALIZATION_FT */
static bool
has_space_for_executor(PyCodeObject *code, _Py_CODEUNIT *instr)
{
- if (code == (PyCodeObject *)&_Py_InitCleanup) {
+ if (code == (PyCodeObject *)&_Py_InitCleanup || code == (PyCodeObject *)&_PyEntryFrameCode) {
return false;
}
if (instr->op.code == ENTER_EXECUTOR) {
bool progress_needed = chain_depth == 0;
PyCodeObject *code = (PyCodeObject *)tstate->interp->jit_state.initial_code;
_Py_CODEUNIT *start = tstate->interp->jit_state.insert_exec_instr;
- // A recursive trace might've cleared the values. In that case, bail.
- if (code == NULL) {
- interp->compiling = false;
- return 0;
- }
if (progress_needed && !has_space_for_executor(code, start)) {
interp->compiling = false;
return 0;
}
if (opcode == ENTER_EXECUTOR) {
- int is_first_instr = tstate->interp->jit_state.close_loop_instr == next_instr || tstate->interp->jit_state.insert_exec_instr == next_instr;
- if (is_first_instr && tstate->interp->jit_state.code_curr_size > 5) {
- ADD_TO_TRACE(_JUMP_TO_TOP, 0, 0, 0);
- goto done;
- }
goto full;
}
if (opcode == WITH_EXCEPT_START || opcode == RERAISE || opcode == CLEANUP_THROW || opcode == PUSH_EXC_INFO) {
DPRINTF(2, "Unsupported: strange control-flow\n");
- goto unsupported;
- }
-
- if (frame->owner >= FRAME_OWNED_BY_INTERPRETER) {
- DPRINTF(2, "Unsupported: frame owned by interpreter\n");
unsupported:
{
// Rewind to previous instruction and replace with _EXIT_TRACE.
tstate->interp->jit_state.insert_exec_instr = insert_exec_instr;
tstate->interp->jit_state.close_loop_instr = close_loop_instr;
tstate->interp->jit_state.initial_code = (PyCodeObject *)Py_NewRef(code);
- tstate->interp->jit_state.initial_func = (PyFunctionObject *)Py_NewRef(_PyFrame_GetFunction(frame));
+ tstate->interp->jit_state.initial_func = (PyFunctionObject *)Py_XNewRef(PyStackRef_AsPyObjectBorrow(frame->f_funcobj));
tstate->interp->jit_state.prev_exit = exit;
tstate->interp->jit_state.initial_stack_depth = curr_stackdepth;
tstate->interp->jit_state.initial_chain_depth = chain_depth;
return;
}
_Py_CODEUNIT *instruction = &_PyCode_CODE(code)[executor->vm_data.index];
- assert(instruction->op.code == ENTER_EXECUTOR);
int index = instruction->op.arg;
- assert(code->co_executors->executors[index] == executor);
- instruction->op.code = executor->vm_data.opcode;
- instruction->op.arg = executor->vm_data.oparg;
+ // Due to a combination of re-entrancy and tracing, it's possible for an
+ // instruction to no longer be ENTER_EXECUTOR. In which case, no-op.
+ if (instruction->op.code == ENTER_EXECUTOR) {
+ assert(instruction->op.code == ENTER_EXECUTOR);
+ assert(code->co_executors->executors[index] == executor);
+ instruction->op.code = executor->vm_data.opcode;
+ instruction->op.arg = executor->vm_data.oparg;
+ }
executor->vm_data.code = NULL;
code->co_executors->executors[index] = NULL;
Py_DECREF(executor);
* be lifted.
*/
-#define NO_LOC_4 (128 | (PY_CODE_LOCATION_INFO_NONE << 3) | 3)
-
-static const PyBytesObject no_location = {
- PyVarObject_HEAD_INIT(&PyBytes_Type, 1)
- .ob_sval = { NO_LOC_4 }
-};
#ifdef Py_GIL_DISABLED
static _PyCodeArray init_cleanup_tlbc = {
projects / thirdparty / Python / cpython.git / commitdiff
