///////////////////// Experimental UOp Optimizer /////////////////////
static int executor_clear(PyObject *executor);
-static void unlink_executor(_PyExecutorObject *executor);
-
void
_PyExecutor_Free(_PyExecutorObject *self)
PyObject_GC_Del(self);
}
+static void executor_invalidate(PyObject *op);
+
+static void
+executor_clear_exits(_PyExecutorObject *executor)
+{
+ _PyExecutorObject *cold = _PyExecutor_GetColdExecutor();
+ _PyExecutorObject *cold_dynamic = _PyExecutor_GetColdDynamicExecutor();
+ for (uint32_t i = 0; i < executor->exit_count; i++) {
+ _PyExitData *exit = &executor->exits[i];
+ exit->temperature = initial_unreachable_backoff_counter();
+ _PyExecutorObject *old = executor->exits[i].executor;
+ exit->executor = exit->is_dynamic ? cold_dynamic : cold;
+ Py_DECREF(old);
+ }
+}
+
+
void
_Py_ClearExecutorDeletionList(PyInterpreterState *interp)
{
+ if (interp->executor_deletion_list_head == NULL) {
+ return;
+ }
_PyRuntimeState *runtime = &_PyRuntime;
HEAD_LOCK(runtime);
PyThreadState* ts = PyInterpreterState_ThreadHead(interp);
+ while (ts) {
+ _PyExecutorObject *current = (_PyExecutorObject *)ts->current_executor;
+ Py_XINCREF(current);
+ ts = ts->next;
+ }
HEAD_UNLOCK(runtime);
+ _PyExecutorObject *keep_list = NULL;
+ do {
+ _PyExecutorObject *exec = interp->executor_deletion_list_head;
+ interp->executor_deletion_list_head = exec->vm_data.links.next;
+ if (Py_REFCNT(exec) == 0) {
+ _PyExecutor_Free(exec);
+ } else {
+ exec->vm_data.links.next = keep_list;
+ keep_list = exec;
+ }
+ } while (interp->executor_deletion_list_head != NULL);
+ interp->executor_deletion_list_head = keep_list;
+ HEAD_LOCK(runtime);
+ ts = PyInterpreterState_ThreadHead(interp);
while (ts) {
_PyExecutorObject *current = (_PyExecutorObject *)ts->current_executor;
if (current != NULL) {
- /* Anything in this list will be unlinked, so we can reuse the
- * linked field as a reachability marker. */
- current->vm_data.linked = 1;
+ _Py_DECREF_NO_DEALLOC((PyObject *)current);
}
- HEAD_LOCK(runtime);
- ts = PyThreadState_Next(ts);
- HEAD_UNLOCK(runtime);
- }
- _PyExecutorObject **prev_to_next_ptr = &interp->executor_deletion_list_head;
- _PyExecutorObject *exec = *prev_to_next_ptr;
- while (exec != NULL) {
- if (exec->vm_data.linked) {
- // This executor is currently executing
- exec->vm_data.linked = 0;
- prev_to_next_ptr = &exec->vm_data.links.next;
- }
- else {
- *prev_to_next_ptr = exec->vm_data.links.next;
- _PyExecutor_Free(exec);
- }
- exec = *prev_to_next_ptr;
+ ts = ts->next;
}
- interp->executor_deletion_list_remaining_capacity = EXECUTOR_DELETE_LIST_MAX;
+ HEAD_UNLOCK(runtime);
}
static void
add_to_pending_deletion_list(_PyExecutorObject *self)
{
PyInterpreterState *interp = PyInterpreterState_Get();
+ self->vm_data.links.previous = NULL;
self->vm_data.links.next = interp->executor_deletion_list_head;
interp->executor_deletion_list_head = self;
- if (interp->executor_deletion_list_remaining_capacity > 0) {
- interp->executor_deletion_list_remaining_capacity--;
- }
- else {
- _Py_ClearExecutorDeletionList(interp);
- }
}
static void
uop_dealloc(PyObject *op) {
_PyExecutorObject *self = _PyExecutorObject_CAST(op);
- _PyObject_GC_UNTRACK(self);
+ executor_invalidate(op);
assert(self->vm_data.code == NULL);
- unlink_executor(self);
- // Once unlinked it becomes impossible to invalidate an executor, so do it here.
- self->vm_data.valid = 0;
add_to_pending_deletion_list(self);
}
head->vm_data.links.previous = executor;
interp->executor_list_head = executor;
}
- executor->vm_data.linked = true;
/* executor_list_head must be first in list */
assert(interp->executor_list_head->vm_data.links.previous == NULL);
}
static void
unlink_executor(_PyExecutorObject *executor)
{
- if (!executor->vm_data.linked) {
- return;
- }
_PyExecutorLinkListNode *links = &executor->vm_data.links;
- assert(executor->vm_data.valid);
_PyExecutorObject *next = links->next;
_PyExecutorObject *prev = links->previous;
if (next != NULL) {
assert(interp->executor_list_head == executor);
interp->executor_list_head = next;
}
- executor->vm_data.linked = false;
}
/* This must be called by optimizers before using the executor */
link_executor(executor);
}
-_PyExecutorObject *
-_PyExecutor_GetColdExecutor(void)
+static _PyExecutorObject *
+make_cold_executor(uint16_t opcode)
{
- PyInterpreterState *interp = _PyInterpreterState_GET();
- if (interp->cold_executor != NULL) {
- return interp->cold_executor;
- }
_PyExecutorObject *cold = allocate_executor(0, 1);
if (cold == NULL) {
Py_FatalError("Cannot allocate core JIT code");
}
- ((_PyUOpInstruction *)cold->trace)->opcode = _COLD_EXIT_r00;
-#ifdef _Py_JIT
- cold->jit_code = NULL;
- cold->jit_size = 0;
+ ((_PyUOpInstruction *)cold->trace)->opcode = opcode;
// This is initialized to true so we can prevent the executor
// from being immediately detected as cold and invalidated.
cold->vm_data.warm = true;
+#ifdef _Py_JIT
+ cold->jit_code = NULL;
+ cold->jit_size = 0;
if (_PyJIT_Compile(cold, cold->trace, 1)) {
Py_DECREF(cold);
Py_FatalError("Cannot allocate core JIT code");
}
#endif
_Py_SetImmortal((PyObject *)cold);
- interp->cold_executor = cold;
return cold;
}
_PyExecutorObject *
-_PyExecutor_GetColdDynamicExecutor(void)
+_PyExecutor_GetColdExecutor(void)
{
PyInterpreterState *interp = _PyInterpreterState_GET();
- if (interp->cold_dynamic_executor != NULL) {
- assert(interp->cold_dynamic_executor->trace[0].opcode == _COLD_DYNAMIC_EXIT_r00);
- return interp->cold_dynamic_executor;
- }
- _PyExecutorObject *cold = allocate_executor(0, 1);
- if (cold == NULL) {
- Py_FatalError("Cannot allocate core JIT code");
+ if (interp->cold_executor == NULL) {
+ return interp->cold_executor = make_cold_executor(_COLD_EXIT_r00);;
}
- ((_PyUOpInstruction *)cold->trace)->opcode = _COLD_DYNAMIC_EXIT_r00;
-#ifdef _Py_JIT
- cold->jit_code = NULL;
- cold->jit_size = 0;
- // This is initialized to true so we can prevent the executor
- // from being immediately detected as cold and invalidated.
- cold->vm_data.warm = true;
- if (_PyJIT_Compile(cold, cold->trace, 1)) {
- Py_DECREF(cold);
- Py_FatalError("Cannot allocate core JIT code");
+ return interp->cold_executor;
+}
+
+_PyExecutorObject *
+_PyExecutor_GetColdDynamicExecutor(void)
+{
+ PyInterpreterState *interp = _PyInterpreterState_GET();
+ if (interp->cold_dynamic_executor == NULL) {
+ interp->cold_dynamic_executor = make_cold_executor(_COLD_DYNAMIC_EXIT_r00);
}
-#endif
- _Py_SetImmortal((PyObject *)cold);
- interp->cold_dynamic_executor = cold;
- return cold;
+ return interp->cold_dynamic_executor;
}
void
Py_DECREF(executor);
}
-static int
-executor_clear(PyObject *op)
+/* Executors can be invalidated at any time,
+ even with a stop-the-world lock held.
+ Consequently it must not run arbitrary code,
+ including Py_DECREF with a non-executor. */
+static void
+executor_invalidate(PyObject *op)
{
_PyExecutorObject *executor = _PyExecutorObject_CAST(op);
if (!executor->vm_data.valid) {
- return 0;
+ return;
}
- assert(executor->vm_data.valid == 1);
- unlink_executor(executor);
executor->vm_data.valid = 0;
-
- /* It is possible for an executor to form a reference
- * cycle with itself, so decref'ing a side exit could
- * free the executor unless we hold a strong reference to it
- */
- _PyExecutorObject *cold = _PyExecutor_GetColdExecutor();
- Py_INCREF(executor);
- for (uint32_t i = 0; i < executor->exit_count; i++) {
- executor->exits[i].temperature = initial_unreachable_backoff_counter();
- _PyExecutorObject *e = executor->exits[i].executor;
- executor->exits[i].executor = cold;
- Py_DECREF(e);
- }
+ unlink_executor(executor);
+ executor_clear_exits(executor);
_Py_ExecutorDetach(executor);
- Py_DECREF(executor);
- return 0;
+ _PyObject_GC_UNTRACK(op);
+}
+
+static int
+executor_clear(PyObject *op)
+{
+ executor_invalidate(op);
}
void
if (invalidate == NULL) {
goto error;
}
- /* Clearing an executor can deallocate others, so we need to make a list of
+ /* Clearing an executor can clear others, so we need to make a list of
* executors to invalidate first */
for (_PyExecutorObject *exec = interp->executor_list_head; exec != NULL;) {
assert(exec->vm_data.valid);
}
for (Py_ssize_t i = 0; i < PyList_GET_SIZE(invalidate); i++) {
PyObject *exec = PyList_GET_ITEM(invalidate, i);
- executor_clear(exec);
+ executor_invalidate(exec);
if (is_invalidation) {
OPT_STAT_INC(executors_invalidated);
}
{
while (interp->executor_list_head) {
_PyExecutorObject *executor = interp->executor_list_head;
- assert(executor->vm_data.valid == 1 && executor->vm_data.linked == 1);
+ assert(executor->vm_data.valid);
if (executor->vm_data.code) {
// Clear the entire code object so its co_executors array be freed:
_PyCode_Clear_Executors(executor->vm_data.code);
}
else {
- executor_clear((PyObject *)executor);
+ executor_invalidate((PyObject *)executor);
}
if (is_invalidation) {
OPT_STAT_INC(executors_invalidated);
}
for (Py_ssize_t i = 0; i < PyList_GET_SIZE(invalidate); i++) {
PyObject *exec = PyList_GET_ITEM(invalidate, i);
- executor_clear(exec);
+ executor_invalidate(exec);
}
Py_DECREF(invalidate);
return;
projects / thirdparty / Python / cpython.git / commitdiff
