// interpreter at the same time. Only the "bound" thread may perform the
// transitions between "attached" and "detached" on its own PyThreadState.
//
-// The "gc" state is used to implement stop-the-world pauses, such as for
-// cyclic garbage collection. It is only used in `--disable-gil` builds. It is
-// similar to the "detached" state, but only the thread performing a
-// stop-the-world pause may transition threads between the "detached" and "gc"
-// states. A thread trying to "attach" from the "gc" state will block until
-// it is transitioned back to "detached" when the stop-the-world pause is
-// complete.
+// The "suspended" state is used to implement stop-the-world pauses, such as
+// for cyclic garbage collection. It is only used in `--disable-gil` builds.
+// The "suspended" state is similar to the "detached" state in that in both
+// states the thread is not allowed to call most Python APIs. However, unlike
+// the "detached" state, a thread may not transition itself out from the
+// "suspended" state. Only the thread performing a stop-the-world pause may
+// transition a thread from the "suspended" state back to the "detached" state.
//
// State transition diagram:
//
// (bound thread) (stop-the-world thread)
-// [attached] <-> [detached] <-> [gc]
+// [attached] <-> [detached] <-> [suspended]
+// | ^
+// +---------------------------->---------------------------+
+// (bound thread)
//
-// See `_PyThreadState_Attach()` and `_PyThreadState_Detach()`.
+// The (bound thread) and (stop-the-world thread) labels indicate which thread
+// is allowed to perform the transition.
#define _Py_THREAD_DETACHED 0
#define _Py_THREAD_ATTACHED 1
-#define _Py_THREAD_GC 2
+#define _Py_THREAD_SUSPENDED 2
/* Check if the current thread is the main thread.
//
// High-level code should generally call PyEval_RestoreThread() instead, which
// calls this function.
-void _PyThreadState_Attach(PyThreadState *tstate);
+extern void _PyThreadState_Attach(PyThreadState *tstate);
// Detaches the current thread from the interpreter.
//
// High-level code should generally call PyEval_SaveThread() instead, which
// calls this function.
-void _PyThreadState_Detach(PyThreadState *tstate);
+extern void _PyThreadState_Detach(PyThreadState *tstate);
+
+// Detaches the current thread to the "suspended" state if a stop-the-world
+// pause is in progress.
+//
+// If there is no stop-the-world pause in progress, then the thread switches
+// to the "detached" state.
+extern void _PyThreadState_Suspend(PyThreadState *tstate);
+
+// Perform a stop-the-world pause for all threads in the all interpreters.
+//
+// Threads in the "attached" state are paused and transitioned to the "GC"
+// state. Threads in the "detached" state switch to the "GC" state, preventing
+// them from reattaching until the stop-the-world pause is complete.
+//
+// NOTE: This is a no-op outside of Py_GIL_DISABLED builds.
+extern void _PyEval_StopTheWorldAll(_PyRuntimeState *runtime);
+extern void _PyEval_StartTheWorldAll(_PyRuntimeState *runtime);
+
+// Perform a stop-the-world pause for threads in the specified interpreter.
+//
+// NOTE: This is a no-op outside of Py_GIL_DISABLED builds.
+extern void _PyEval_StopTheWorld(PyInterpreterState *interp);
+extern void _PyEval_StartTheWorld(PyInterpreterState *interp);
static inline void
tstate->datastack_limit = NULL;
tstate->what_event = -1;
+ if (interp->stoptheworld.requested || _PyRuntime.stoptheworld.requested) {
+ // Start in the suspended state if there is an ongoing stop-the-world.
+ tstate->state = _Py_THREAD_SUSPENDED;
+ }
+
tstate->_status.initialized = 1;
}
// XXX Do it as early in the function as possible.
}
+static void
+decrement_stoptheworld_countdown(struct _stoptheworld_state *stw);
+
/* Common code for PyThreadState_Delete() and PyThreadState_DeleteCurrent() */
static void
tstate_delete_common(PyThreadState *tstate)
if (tstate->next) {
tstate->next->prev = tstate->prev;
}
+ if (tstate->state != _Py_THREAD_SUSPENDED) {
+ // Any ongoing stop-the-world request should not wait for us because
+ // our thread is getting deleted.
+ if (interp->stoptheworld.requested) {
+ decrement_stoptheworld_countdown(&interp->stoptheworld);
+ }
+ if (runtime->stoptheworld.requested) {
+ decrement_stoptheworld_countdown(&runtime->stoptheworld);
+ }
+ }
HEAD_UNLOCK(runtime);
// XXX Unbind in PyThreadState_Clear(), or earlier
{
#ifdef Py_GIL_DISABLED
int expected = _Py_THREAD_DETACHED;
- if (_Py_atomic_compare_exchange_int(
- &tstate->state,
- &expected,
- _Py_THREAD_ATTACHED)) {
- return 1;
- }
- return 0;
+ return _Py_atomic_compare_exchange_int(&tstate->state,
+ &expected,
+ _Py_THREAD_ATTACHED);
#else
assert(tstate->state == _Py_THREAD_DETACHED);
tstate->state = _Py_THREAD_ATTACHED;
#endif
}
+static void
+tstate_wait_attach(PyThreadState *tstate)
+{
+ do {
+ int expected = _Py_THREAD_SUSPENDED;
+
+ // Wait until we're switched out of SUSPENDED to DETACHED.
+ _PyParkingLot_Park(&tstate->state, &expected, sizeof(tstate->state),
+ /*timeout=*/-1, NULL, /*detach=*/0);
+
+ // Once we're back in DETACHED we can re-attach
+ } while (!tstate_try_attach(tstate));
+}
+
void
_PyThreadState_Attach(PyThreadState *tstate)
{
tstate_activate(tstate);
if (!tstate_try_attach(tstate)) {
- // TODO: Once stop-the-world GC is implemented for --disable-gil builds
- // this will need to wait until the GC completes. For now, this case
- // should never happen.
- Py_FatalError("thread attach failed");
+ tstate_wait_attach(tstate);
}
// Resume previous critical section. This acquires the lock(s) from the
#endif
}
-void
-_PyThreadState_Detach(PyThreadState *tstate)
+static void
+detach_thread(PyThreadState *tstate, int detached_state)
{
// XXX assert(tstate_is_alive(tstate) && tstate_is_bound(tstate));
assert(tstate->state == _Py_THREAD_ATTACHED);
if (tstate->critical_section != 0) {
_PyCriticalSection_SuspendAll(tstate);
}
- tstate_set_detached(tstate);
tstate_deactivate(tstate);
+ tstate_set_detached(tstate);
current_fast_clear(&_PyRuntime);
_PyEval_ReleaseLock(tstate->interp, tstate);
}
+void
+_PyThreadState_Detach(PyThreadState *tstate)
+{
+ detach_thread(tstate, _Py_THREAD_DETACHED);
+}
+
+void
+_PyThreadState_Suspend(PyThreadState *tstate)
+{
+ _PyRuntimeState *runtime = &_PyRuntime;
+
+ assert(tstate->state == _Py_THREAD_ATTACHED);
+
+ struct _stoptheworld_state *stw = NULL;
+ HEAD_LOCK(runtime);
+ if (runtime->stoptheworld.requested) {
+ stw = &runtime->stoptheworld;
+ }
+ else if (tstate->interp->stoptheworld.requested) {
+ stw = &tstate->interp->stoptheworld;
+ }
+ HEAD_UNLOCK(runtime);
+
+ if (stw == NULL) {
+ // Switch directly to "detached" if there is no active stop-the-world
+ // request.
+ detach_thread(tstate, _Py_THREAD_DETACHED);
+ return;
+ }
+
+ // Switch to "suspended" state.
+ detach_thread(tstate, _Py_THREAD_SUSPENDED);
+
+ // Decrease the count of remaining threads needing to park.
+ HEAD_LOCK(runtime);
+ decrement_stoptheworld_countdown(stw);
+ HEAD_UNLOCK(runtime);
+}
+
+// Decrease stop-the-world counter of remaining number of threads that need to
+// pause. If we are the final thread to pause, notify the requesting thread.
+static void
+decrement_stoptheworld_countdown(struct _stoptheworld_state *stw)
+{
+ assert(stw->thread_countdown > 0);
+ if (--stw->thread_countdown == 0) {
+ _PyEvent_Notify(&stw->stop_event);
+ }
+}
+
+#ifdef Py_GIL_DISABLED
+// Interpreter for _Py_FOR_EACH_THREAD(). For global stop-the-world events,
+// we start with the first interpreter and then iterate over all interpreters.
+// For per-interpreter stop-the-world events, we only operate on the one
+// interpreter.
+static PyInterpreterState *
+interp_for_stop_the_world(struct _stoptheworld_state *stw)
+{
+ return (stw->is_global
+ ? PyInterpreterState_Head()
+ : _Py_CONTAINER_OF(stw, PyInterpreterState, stoptheworld));
+}
+
+// Loops over threads for a stop-the-world event.
+// For global: all threads in all interpreters
+// For per-interpreter: all threads in the interpreter
+#define _Py_FOR_EACH_THREAD(stw, i, t) \
+ for (i = interp_for_stop_the_world((stw)); \
+ i != NULL; i = ((stw->is_global) ? i->next : NULL)) \
+ for (t = i->threads.head; t; t = t->next)
+
+
+// Try to transition threads atomically from the "detached" state to the
+// "gc stopped" state. Returns true if all threads are in the "gc stopped"
+static bool
+park_detached_threads(struct _stoptheworld_state *stw)
+{
+ int num_parked = 0;
+ PyInterpreterState *i;
+ PyThreadState *t;
+ _Py_FOR_EACH_THREAD(stw, i, t) {
+ int state = _Py_atomic_load_int_relaxed(&t->state);
+ if (state == _Py_THREAD_DETACHED) {
+ // Atomically transition to "suspended" if in "detached" state.
+ if (_Py_atomic_compare_exchange_int(&t->state,
+ &state, _Py_THREAD_SUSPENDED)) {
+ num_parked++;
+ }
+ }
+ else if (state == _Py_THREAD_ATTACHED && t != stw->requester) {
+ // TODO: set this per-thread, rather than per-interpreter.
+ _Py_set_eval_breaker_bit(t->interp, _PY_EVAL_PLEASE_STOP_BIT, 1);
+ }
+ }
+ stw->thread_countdown -= num_parked;
+ assert(stw->thread_countdown >= 0);
+ return num_parked > 0 && stw->thread_countdown == 0;
+}
+
+static void
+stop_the_world(struct _stoptheworld_state *stw)
+{
+ _PyRuntimeState *runtime = &_PyRuntime;
+
+ PyMutex_Lock(&stw->mutex);
+ if (stw->is_global) {
+ _PyRWMutex_Lock(&runtime->stoptheworld_mutex);
+ }
+ else {
+ _PyRWMutex_RLock(&runtime->stoptheworld_mutex);
+ }
+
+ HEAD_LOCK(runtime);
+ stw->requested = 1;
+ stw->thread_countdown = 0;
+ stw->stop_event = (PyEvent){0}; // zero-initialize (unset)
+ stw->requester = _PyThreadState_GET(); // may be NULL
+
+ PyInterpreterState *i;
+ PyThreadState *t;
+ _Py_FOR_EACH_THREAD(stw, i, t) {
+ if (t != stw->requester) {
+ // Count all the other threads (we don't wait on ourself).
+ stw->thread_countdown++;
+ }
+ }
+
+ if (stw->thread_countdown == 0) {
+ HEAD_UNLOCK(runtime);
+ stw->world_stopped = 1;
+ return;
+ }
+
+ for (;;) {
+ // Switch threads that are detached to the GC stopped state
+ bool stopped_all_threads = park_detached_threads(stw);
+ HEAD_UNLOCK(runtime);
+
+ if (stopped_all_threads) {
+ break;
+ }
+
+ _PyTime_t wait_ns = 1000*1000; // 1ms (arbitrary, may need tuning)
+ if (PyEvent_WaitTimed(&stw->stop_event, wait_ns)) {
+ assert(stw->thread_countdown == 0);
+ break;
+ }
+
+ HEAD_LOCK(runtime);
+ }
+ stw->world_stopped = 1;
+}
+
+static void
+start_the_world(struct _stoptheworld_state *stw)
+{
+ _PyRuntimeState *runtime = &_PyRuntime;
+ assert(PyMutex_IsLocked(&stw->mutex));
+
+ HEAD_LOCK(runtime);
+ stw->requested = 0;
+ stw->world_stopped = 0;
+ stw->requester = NULL;
+ // Switch threads back to the detached state.
+ PyInterpreterState *i;
+ PyThreadState *t;
+ _Py_FOR_EACH_THREAD(stw, i, t) {
+ if (t != stw->requester) {
+ assert(t->state == _Py_THREAD_SUSPENDED);
+ _Py_atomic_store_int(&t->state, _Py_THREAD_DETACHED);
+ _PyParkingLot_UnparkAll(&t->state);
+ }
+ }
+ HEAD_UNLOCK(runtime);
+ if (stw->is_global) {
+ _PyRWMutex_Unlock(&runtime->stoptheworld_mutex);
+ }
+ else {
+ _PyRWMutex_RUnlock(&runtime->stoptheworld_mutex);
+ }
+ PyMutex_Unlock(&stw->mutex);
+}
+#endif // Py_GIL_DISABLED
+
+void
+_PyEval_StopTheWorldAll(_PyRuntimeState *runtime)
+{
+#ifdef Py_GIL_DISABLED
+ stop_the_world(&runtime->stoptheworld);
+#endif
+}
+
+void
+_PyEval_StartTheWorldAll(_PyRuntimeState *runtime)
+{
+#ifdef Py_GIL_DISABLED
+ start_the_world(&runtime->stoptheworld);
+#endif
+}
+
+void
+_PyEval_StopTheWorld(PyInterpreterState *interp)
+{
+#ifdef Py_GIL_DISABLED
+ stop_the_world(&interp->stoptheworld);
+#endif
+}
+
+void
+_PyEval_StartTheWorld(PyInterpreterState *interp)
+{
+#ifdef Py_GIL_DISABLED
+ start_the_world(&interp->stoptheworld);
+#endif
+}
+
//----------
// other API
//----------
projects / thirdparty / Python / cpython.git / commitdiff
