PyAPI_FUNC(void) _PyRWMutex_Lock(_PyRWMutex *rwmutex);
PyAPI_FUNC(void) _PyRWMutex_Unlock(_PyRWMutex *rwmutex);
-// Similar to linux seqlock: https://en.wikipedia.org/wiki/Seqlock
-// We use a sequence number to lock the writer, an even sequence means we're unlocked, an odd
-// sequence means we're locked. Readers will read the sequence before attempting to read the
-// underlying data and then read the sequence number again after reading the data. If the
-// sequence has not changed the data is valid.
-//
-// Differs a little bit in that we use CAS on sequence as the lock, instead of a separate spin lock.
-// The writer can also detect that the undelering data has not changed and abandon the write
-// and restore the previous sequence.
-typedef struct {
- uint32_t sequence;
-} _PySeqLock;
-
-// Lock the sequence lock for the writer
-PyAPI_FUNC(void) _PySeqLock_LockWrite(_PySeqLock *seqlock);
-
-// Unlock the sequence lock and move to the next sequence number.
-PyAPI_FUNC(void) _PySeqLock_UnlockWrite(_PySeqLock *seqlock);
-
-// Abandon the current update indicating that no mutations have occurred
-// and restore the previous sequence value.
-PyAPI_FUNC(void) _PySeqLock_AbandonWrite(_PySeqLock *seqlock);
-
-// Begin a read operation and return the current sequence number.
-PyAPI_FUNC(uint32_t) _PySeqLock_BeginRead(_PySeqLock *seqlock);
-
-// End the read operation and confirm that the sequence number has not changed.
-// Returns 1 if the read was successful or 0 if the read should be retried.
-PyAPI_FUNC(int) _PySeqLock_EndRead(_PySeqLock *seqlock, uint32_t previous);
-
-// Check if the lock was held during a fork and clear the lock. Returns 1
-// if the lock was held and any associated data should be cleared.
-PyAPI_FUNC(int) _PySeqLock_AfterFork(_PySeqLock *seqlock);
-
#ifdef __cplusplus
}
#endif
#include "pycore_dict.h" // _PyDict_KeysSize()
#include "pycore_function.h" // _PyFunction_GetVersionForCurrentState()
#include "pycore_interpframe.h" // _PyInterpreterFrame
-#include "pycore_lock.h" // _PySeqLock_*
#include "pycore_long.h" // _PyLong_IsNegative(), _PyLong_GetOne()
#include "pycore_memoryobject.h" // _PyMemoryView_FromBufferProc()
#include "pycore_modsupport.h" // _PyArg_NoKwnames()
}
}
-#define SEQLOCK_IS_UPDATING(sequence) (sequence & 0x01)
-
-void _PySeqLock_LockWrite(_PySeqLock *seqlock)
-{
- // lock by moving to an odd sequence number
- uint32_t prev = _Py_atomic_load_uint32_relaxed(&seqlock->sequence);
- while (1) {
- if (SEQLOCK_IS_UPDATING(prev)) {
- // Someone else is currently updating the cache
- _Py_yield();
- prev = _Py_atomic_load_uint32_relaxed(&seqlock->sequence);
- }
- else if (_Py_atomic_compare_exchange_uint32(&seqlock->sequence, &prev, prev + 1)) {
- // We've locked the cache
- _Py_atomic_fence_release();
- break;
- }
- else {
- _Py_yield();
- }
- }
-}
-
-void _PySeqLock_AbandonWrite(_PySeqLock *seqlock)
-{
- uint32_t new_seq = _Py_atomic_load_uint32_relaxed(&seqlock->sequence) - 1;
- assert(!SEQLOCK_IS_UPDATING(new_seq));
- _Py_atomic_store_uint32(&seqlock->sequence, new_seq);
-}
-
-void _PySeqLock_UnlockWrite(_PySeqLock *seqlock)
-{
- uint32_t new_seq = _Py_atomic_load_uint32_relaxed(&seqlock->sequence) + 1;
- assert(!SEQLOCK_IS_UPDATING(new_seq));
- _Py_atomic_store_uint32(&seqlock->sequence, new_seq);
-}
-
-uint32_t _PySeqLock_BeginRead(_PySeqLock *seqlock)
-{
- uint32_t sequence = _Py_atomic_load_uint32_acquire(&seqlock->sequence);
- while (SEQLOCK_IS_UPDATING(sequence)) {
- _Py_yield();
- sequence = _Py_atomic_load_uint32_acquire(&seqlock->sequence);
- }
-
- return sequence;
-}
-
-int _PySeqLock_EndRead(_PySeqLock *seqlock, uint32_t previous)
-{
- // gh-121368: We need an explicit acquire fence here to ensure that
- // this load of the sequence number is not reordered before any loads
- // within the read lock.
- _Py_atomic_fence_acquire();
-
- if (_Py_atomic_load_uint32_relaxed(&seqlock->sequence) == previous) {
- return 1;
- }
-
- _Py_yield();
- return 0;
-}
-
-int _PySeqLock_AfterFork(_PySeqLock *seqlock)
-{
- // Synchronize again and validate that the entry hasn't been updated
- // while we were readying the values.
- if (SEQLOCK_IS_UPDATING(seqlock->sequence)) {
- seqlock->sequence = 0;
- return 1;
- }
-
- return 0;
-}
-
#undef PyMutex_Lock
void
PyMutex_Lock(PyMutex *m)