return counter.value_and_backoff < UNREACHABLE_BACKOFF;
}
+static inline _Py_BackoffCounter
+trigger_backoff_counter(void)
+{
+ _Py_BackoffCounter result;
+ result.value_and_backoff = 0;
+ return result;
+}
+
// Initial JUMP_BACKWARD counter.
// Must be larger than ADAPTIVE_COOLDOWN_VALUE, otherwise when JIT code is
// invalidated we may construct a new trace before the bytecode has properly
return make_backoff_counter(0, UNREACHABLE_BACKOFF);
}
-// Required to not get stuck in infinite specialization loops due to specialization failure.
-// We use 2 here as there are a few scenarios:
-// 1. Freshly specialized from unspecialized, in which case the counter will be 1.
-// 2. Re-specialized from deopt, in which case the counter will be 1.
-// 3. Deopt -> Specialize -> Deopt -> Specialize, in which case the counter will be 2.
-// We do not want the 3rd case.
-#define MAX_SPECIALIZATION_TRIES 2
-
#ifdef __cplusplus
}
#endif
int code_curr_size;
int instr_oparg;
int instr_stacklevel;
- int specialize_counter;
_Py_CODEUNIT *instr;
PyCodeObject *instr_code; // Strong
struct _PyInterpreterFrame *instr_frame;
? _Py_INTERPRETER_TRAMPOLINE_INSTRUCTIONS_PTR : _PyFrame_GetBytecode(frame))
+ exit->target;
OPT_HIST(trace_uop_execution_counter, trace_run_length_hist);
- if (frame->lltrace >= 2) {
+ if (frame->lltrace >= 3) {
printf("SIDE EXIT: [UOp ");
_PyUOpPrint(&next_uop[-1]);
printf(", exit %tu, temp %d, target %d -> %s, is_control_flow %d]\n",
_PyExitData *exit = (_PyExitData *)exit_p;
_Py_CODEUNIT *target = frame->instr_ptr;
OPT_HIST(trace_uop_execution_counter, trace_run_length_hist);
- if (frame->lltrace >= 2) {
+ if (frame->lltrace >= 3) {
printf("DYNAMIC EXIT: [UOp ");
_PyUOpPrint(&next_uop[-1]);
printf(", exit %tu, temp %d, target %d -> %s]\n",
else {
_tstate->jit_state.prev_state.instr = next_instr;
}
- _tstate->jit_state.prev_state.specialize_counter = 0;
PyObject *prev_code = PyStackRef_AsPyObjectBorrow(frame->f_executable);
if (_tstate->jit_state.prev_state.instr_code != (PyCodeObject *)prev_code) {
Py_SETREF(_tstate->jit_state.prev_state.instr_code, (PyCodeObject*)Py_NewRef((prev_code)));
_tstate->jit_state.prev_state.instr_frame = frame;
_tstate->jit_state.prev_state.instr_oparg = oparg;
_tstate->jit_state.prev_state.instr_stacklevel = PyStackRef_IsNone(frame->f_executable) ? 2 : STACK_LEVEL();
+ if (_PyOpcode_Caches[_PyOpcode_Deopt[opcode]]) {
+ (&next_instr[1])->counter = trigger_backoff_counter();
+ }
DISPATCH_GOTO_NON_TRACING();
#else
Py_FatalError("JIT label executed in non-jit build.");
#if (_Py_TAIL_CALL_INTERP || USE_COMPUTED_GOTOS) && _Py_TIER2
# define IS_JIT_TRACING() (DISPATCH_TABLE_VAR == TRACING_DISPATCH_TABLE)
-# define IS_JIT_TRACING_MAKING_PROGRESS() (IS_JIT_TRACING() && ((_PyThreadStateImpl *)tstate)->jit_state.prev_state.specialize_counter < MAX_SPECIALIZATION_TRIES)
# define ENTER_TRACING() \
DISPATCH_TABLE_VAR = TRACING_DISPATCH_TABLE;
# define LEAVE_TRACING() \
DISPATCH_TABLE_VAR = DISPATCH_TABLE;
#else
# define IS_JIT_TRACING() (0)
-# define IS_JIT_TRACING_MAKING_PROGRESS() (0)
# define ENTER_TRACING()
# define LEAVE_TRACING()
#endif
* which is always an integral type. */
// Force re-specialization when tracing a side exit to get good side exits.
#define ADAPTIVE_COUNTER_TRIGGERS(COUNTER) \
- backoff_counter_triggers(forge_backoff_counter((COUNTER))) || IS_JIT_TRACING_MAKING_PROGRESS()
+ backoff_counter_triggers(forge_backoff_counter((COUNTER)))
#define ADVANCE_ADAPTIVE_COUNTER(COUNTER) \
do { \
? _Py_INTERPRETER_TRAMPOLINE_INSTRUCTIONS_PTR : _PyFrame_GetBytecode(frame))
+ exit->target;
OPT_HIST(trace_uop_execution_counter, trace_run_length_hist);
- if (frame->lltrace >= 2) {
+ if (frame->lltrace >= 3) {
_PyFrame_SetStackPointer(frame, stack_pointer);
printf("SIDE EXIT: [UOp ");
_PyUOpPrint(&next_uop[-1]);
_PyExitData *exit = (_PyExitData *)exit_p;
_Py_CODEUNIT *target = frame->instr_ptr;
OPT_HIST(trace_uop_execution_counter, trace_run_length_hist);
- if (frame->lltrace >= 2) {
+ if (frame->lltrace >= 3) {
_PyFrame_SetStackPointer(frame, stack_pointer);
printf("DYNAMIC EXIT: [UOp ");
_PyUOpPrint(&next_uop[-1]);
_PyFrame_SetStackPointer(frame, stack_pointer);
_Py_Specialize_BinaryOp(lhs, rhs, next_instr, oparg, LOCALS_ARRAY);
stack_pointer = _PyFrame_GetStackPointer(frame);
- #if _Py_TIER2
- ((_PyThreadStateImpl *)tstate)->jit_state.prev_state.specialize_counter++;
- #endif
DISPATCH_SAME_OPARG();
}
OPCODE_DEFERRED_INC(BINARY_OP);
_PyFrame_SetStackPointer(frame, stack_pointer);
_Py_Specialize_Call(callable, next_instr, oparg + !PyStackRef_IsNull(self_or_null));
stack_pointer = _PyFrame_GetStackPointer(frame);
- #if _Py_TIER2
- ((_PyThreadStateImpl *)tstate)->jit_state.prev_state.specialize_counter++;
- #endif
DISPATCH_SAME_OPARG();
}
OPCODE_DEFERRED_INC(CALL);
_PyFrame_SetStackPointer(frame, stack_pointer);
_Py_Specialize_CallKw(callable, next_instr, oparg + !PyStackRef_IsNull(self_or_null));
stack_pointer = _PyFrame_GetStackPointer(frame);
- #if _Py_TIER2
- ((_PyThreadStateImpl *)tstate)->jit_state.prev_state.specialize_counter++;
- #endif
DISPATCH_SAME_OPARG();
}
OPCODE_DEFERRED_INC(CALL_KW);
_PyFrame_SetStackPointer(frame, stack_pointer);
_Py_Specialize_CompareOp(left, right, next_instr, oparg);
stack_pointer = _PyFrame_GetStackPointer(frame);
- #if _Py_TIER2
- ((_PyThreadStateImpl *)tstate)->jit_state.prev_state.specialize_counter++;
- #endif
DISPATCH_SAME_OPARG();
}
OPCODE_DEFERRED_INC(COMPARE_OP);
_PyFrame_SetStackPointer(frame, stack_pointer);
_Py_Specialize_ContainsOp(right, next_instr);
stack_pointer = _PyFrame_GetStackPointer(frame);
- #if _Py_TIER2
- ((_PyThreadStateImpl *)tstate)->jit_state.prev_state.specialize_counter++;
- #endif
DISPATCH_SAME_OPARG();
}
OPCODE_DEFERRED_INC(CONTAINS_OP);
_PyFrame_SetStackPointer(frame, stack_pointer);
_Py_Specialize_ForIter(iter, null_or_index, next_instr, oparg);
stack_pointer = _PyFrame_GetStackPointer(frame);
- #if _Py_TIER2
- ((_PyThreadStateImpl *)tstate)->jit_state.prev_state.specialize_counter++;
- #endif
DISPATCH_SAME_OPARG();
}
OPCODE_DEFERRED_INC(FOR_ITER);
uint8_t desired = tstate->interp->jit ? JUMP_BACKWARD_JIT : JUMP_BACKWARD_NO_JIT;
FT_ATOMIC_STORE_UINT8_RELAXED(this_instr->op.code, desired);
next_instr = this_instr;
- #if _Py_TIER2
- ((_PyThreadStateImpl *)tstate)->jit_state.prev_state.specialize_counter++;
- #endif
DISPATCH_SAME_OPARG();
}
#endif
_PyFrame_SetStackPointer(frame, stack_pointer);
_Py_Specialize_LoadAttr(owner, next_instr, name);
stack_pointer = _PyFrame_GetStackPointer(frame);
- #if _Py_TIER2
- ((_PyThreadStateImpl *)tstate)->jit_state.prev_state.specialize_counter++;
- #endif
DISPATCH_SAME_OPARG();
}
OPCODE_DEFERRED_INC(LOAD_ATTR);
_PyFrame_SetStackPointer(frame, stack_pointer);
_Py_Specialize_LoadGlobal(GLOBALS(), BUILTINS(), next_instr, name);
stack_pointer = _PyFrame_GetStackPointer(frame);
- #if _Py_TIER2
- ((_PyThreadStateImpl *)tstate)->jit_state.prev_state.specialize_counter++;
- #endif
DISPATCH_SAME_OPARG();
}
OPCODE_DEFERRED_INC(LOAD_GLOBAL);
_PyFrame_SetStackPointer(frame, stack_pointer);
_Py_Specialize_LoadSuperAttr(global_super_st, class_st, next_instr, load_method);
stack_pointer = _PyFrame_GetStackPointer(frame);
- #if _Py_TIER2
- ((_PyThreadStateImpl *)tstate)->jit_state.prev_state.specialize_counter++;
- #endif
DISPATCH_SAME_OPARG();
}
OPCODE_DEFERRED_INC(LOAD_SUPER_ATTR);
_PyFrame_SetStackPointer(frame, stack_pointer);
_Py_Specialize_Send(receiver, next_instr);
stack_pointer = _PyFrame_GetStackPointer(frame);
- #if _Py_TIER2
- ((_PyThreadStateImpl *)tstate)->jit_state.prev_state.specialize_counter++;
- #endif
DISPATCH_SAME_OPARG();
}
OPCODE_DEFERRED_INC(SEND);
_PyFrame_SetStackPointer(frame, stack_pointer);
_Py_Specialize_StoreAttr(owner, next_instr, name);
stack_pointer = _PyFrame_GetStackPointer(frame);
- #if _Py_TIER2
- ((_PyThreadStateImpl *)tstate)->jit_state.prev_state.specialize_counter++;
- #endif
DISPATCH_SAME_OPARG();
}
OPCODE_DEFERRED_INC(STORE_ATTR);
_PyFrame_SetStackPointer(frame, stack_pointer);
_Py_Specialize_StoreSubscr(container, sub, next_instr);
stack_pointer = _PyFrame_GetStackPointer(frame);
- #if _Py_TIER2
- ((_PyThreadStateImpl *)tstate)->jit_state.prev_state.specialize_counter++;
- #endif
DISPATCH_SAME_OPARG();
}
OPCODE_DEFERRED_INC(STORE_SUBSCR);
_PyFrame_SetStackPointer(frame, stack_pointer);
_Py_Specialize_ToBool(value, next_instr);
stack_pointer = _PyFrame_GetStackPointer(frame);
- #if _Py_TIER2
- ((_PyThreadStateImpl *)tstate)->jit_state.prev_state.specialize_counter++;
- #endif
DISPATCH_SAME_OPARG();
}
OPCODE_DEFERRED_INC(TO_BOOL);
_PyFrame_SetStackPointer(frame, stack_pointer);
_Py_Specialize_UnpackSequence(seq, next_instr, oparg);
stack_pointer = _PyFrame_GetStackPointer(frame);
- #if _Py_TIER2
- ((_PyThreadStateImpl *)tstate)->jit_state.prev_state.specialize_counter++;
- #endif
DISPATCH_SAME_OPARG();
}
OPCODE_DEFERRED_INC(UNPACK_SEQUENCE);
else {
_tstate->jit_state.prev_state.instr = next_instr;
}
- _tstate->jit_state.prev_state.specialize_counter = 0;
PyObject *prev_code = PyStackRef_AsPyObjectBorrow(frame->f_executable);
if (_tstate->jit_state.prev_state.instr_code != (PyCodeObject *)prev_code) {
_PyFrame_SetStackPointer(frame, stack_pointer);
_tstate->jit_state.prev_state.instr_frame = frame;
_tstate->jit_state.prev_state.instr_oparg = oparg;
_tstate->jit_state.prev_state.instr_stacklevel = PyStackRef_IsNone(frame->f_executable) ? 2 : STACK_LEVEL();
+ if (_PyOpcode_Caches[_PyOpcode_Deopt[opcode]]) {
+ (&next_instr[1])->counter = trigger_backoff_counter();
+ }
DISPATCH_GOTO_NON_TRACING();
#else
Py_FatalError("JIT label executed in non-jit build.");
if (_tstate->jit_state.initial_state.func == NULL) {
return 0;
}
+ assert(_tstate->jit_state.initial_state.func != NULL);
interp->compiling = true;
// The first executor in a chain and the MAX_CHAIN_DEPTH'th executor *must*
// make progress in order to avoid infinite loops or excessively-long
// We must point to the first EXTENDED_ARG when deopting.
int oparg = _tstate->jit_state.prev_state.instr_oparg;
int opcode = this_instr->op.code;
- // Failed specialization many times. Deopt!
- if (_tstate->jit_state.prev_state.specialize_counter >= MAX_SPECIALIZATION_TRIES) {
- opcode = _PyOpcode_Deopt[opcode];
- }
int rewind_oparg = oparg;
while (rewind_oparg > 255) {
rewind_oparg >>= 8;
_Py_CODEUNIT *computed_jump_instr = computed_next_instr_without_modifiers + oparg;
assert(next_instr == computed_next_instr || next_instr == computed_jump_instr);
int jump_happened = computed_jump_instr == next_instr;
+ assert(jump_happened == (target_instr[1].cache & 1));
uint32_t uopcode = BRANCH_TO_GUARD[opcode - POP_JUMP_IF_FALSE][jump_happened];
ADD_TO_TRACE(uopcode, 0, 0, INSTR_IP(jump_happened ? computed_next_instr : computed_jump_instr, old_code));
break;
return 0;
}
}
-
+ PyObject *func = PyStackRef_AsPyObjectBorrow(frame->f_funcobj);
+ if (func == NULL) {
+ return 0;
+ }
PyCodeObject *code = _PyFrame_GetCode(frame);
#ifdef Py_DEBUG
char *python_lltrace = Py_GETENV("PYTHON_LLTRACE");
_tstate->jit_state.initial_state.start_instr = start_instr;
_tstate->jit_state.initial_state.close_loop_instr = close_loop_instr;
_tstate->jit_state.initial_state.code = (PyCodeObject *)Py_NewRef(code);
- _tstate->jit_state.initial_state.func = (PyFunctionObject *)Py_XNewRef(PyStackRef_AsPyObjectBorrow(frame->f_funcobj));
+ _tstate->jit_state.initial_state.func = (PyFunctionObject *)Py_NewRef(func);
_tstate->jit_state.initial_state.exit = exit;
_tstate->jit_state.initial_state.stack_depth = curr_stackdepth;
_tstate->jit_state.initial_state.chain_depth = chain_depth;
_tstate->jit_state.prev_state.instr_frame = frame;
_tstate->jit_state.prev_state.dependencies_still_valid = true;
- _tstate->jit_state.prev_state.specialize_counter = 0;
_tstate->jit_state.prev_state.instr_code = (PyCodeObject *)Py_NewRef(_PyFrame_GetCode(frame));
_tstate->jit_state.prev_state.instr = curr_instr;
_tstate->jit_state.prev_state.instr_frame = frame;
_tstate->jit_state.prev_state.instr_is_super = false;
assert(curr_instr->op.code == JUMP_BACKWARD_JIT || (exit != NULL));
_tstate->jit_state.initial_state.jump_backward_instr = curr_instr;
- assert(curr_instr->op.code == JUMP_BACKWARD_JIT || (exit != NULL));
+
+ if (_PyOpcode_Caches[_PyOpcode_Deopt[close_loop_instr->op.code]]) {
+ close_loop_instr[1].counter = trigger_backoff_counter();
+ }
_Py_BloomFilter_Init(&_tstate->jit_state.prev_state.dependencies);
return 1;
}
assert(length < UOP_MAX_TRACE_LENGTH);
OPT_STAT_INC(traces_created);
if (!is_noopt) {
- length = _Py_uop_analyze_and_optimize(_tstate->jit_state.initial_state.func, buffer,
- length,
- curr_stackentries, &new_dependencies);
+ length = _Py_uop_analyze_and_optimize(
+ _tstate->jit_state.initial_state.func,
+ buffer,length,
+ curr_stackentries, &new_dependencies);
if (length <= 0) {
return length;
}
}
if (ctx->contradiction) {
// Attempted to push a "bottom" (contradiction) symbol onto the stack.
- // This means that the abstract interpreter has hit unreachable code.
+ // This means that the abstract interpreter has optimized to trace
+ // to an unreachable estate.
// We *could* generate an _EXIT_TRACE or _FATAL_ERROR here, but hitting
- // bottom indicates type instability, so we are probably better off
+ // bottom usually indicates an optimizer bug, so we are probably better off
// retrying later.
DPRINTF(3, "\n");
DPRINTF(1, "Hit bottom in abstract interpreter\n");
int already_bool = optimize_to_bool(this_instr, ctx, value, &value);
if (!already_bool) {
sym_set_type(value, &PyBool_Type);
- value = sym_new_truthiness(ctx, value, true);
}
}
int already_bool = optimize_to_bool(this_instr, ctx, value, &value);
if (!already_bool) {
sym_set_type(value, &PyBool_Type);
- value = sym_new_truthiness(ctx, value, true);
}
stack_pointer[-1] = value;
break;
bool pop = (next.op.code == POP_TOP);
int oparg = instr->op.arg;
if ((PyObject *)descr == list_append && oparg == 1 && pop) {
- specialize(instr, CALL_LIST_APPEND);
- return 0;
+ PyThreadState *tstate = PyThreadState_GET();
+ PyObject *self = PyStackRef_AsPyObjectBorrow(tstate->current_frame->stackpointer[-2]);
+ if (PyList_CheckExact(self)) {
+ specialize(instr, CALL_LIST_APPEND);
+ return 0;
+ }
}
specialize(instr, CALL_METHOD_DESCRIPTOR_O);
return 0;
"_PyJit_TryInitializeTracing",
"_Py_unset_eval_breaker_bit",
"_Py_set_eval_breaker_bit",
+ "trigger_backoff_counter",
)
"DISPATCH": self.dispatch,
"INSTRUCTION_SIZE": self.instruction_size,
"stack_pointer": self.stack_pointer,
- "DISPATCH_SAME_OPARG": self.dispatch_same_oparg,
}
self.out = out
self.labels = labels
self.emit(tkn)
return False
- def dispatch_same_oparg(
- self,
- tkn: Token,
- tkn_iter: TokenIterator,
- uop: CodeSection,
- storage: Storage,
- inst: Instruction | None,
- ) -> bool:
- assert isinstance(uop, Uop)
- assert "specializing" in uop.annotations, uop.name
- self.out.start_line()
- self.emit("#if _Py_TIER2\n")
- self.emit("((_PyThreadStateImpl *)tstate)->jit_state.prev_state.specialize_counter++;\n")
- self.emit("#endif\n")
- self.emit(tkn)
- emit_to(self.out, tkn_iter, "SEMI")
- self.emit(";\n")
- self.out.start_line()
- return False
-
def deopt_if(
self,
tkn: Token,