/*! \file isc/rwlock.h */
-#include <isc/atomic.h>
-#include <isc/condition.h>
#include <isc/lang.h>
#include <isc/types.h>
#include <isc/util.h>
typedef pthread_rwlock_t *isc_rwlock_t;
typedef pthread_rwlock_t isc__rwlock_t;
-#define isc_rwlock_init(rwl, rq, wq) \
- { \
- *rwl = malloc(sizeof(**rwl)); \
- isc__rwlock_init(*rwl, rq, wq); \
+#define isc_rwlock_init(rwl) \
+ { \
+ *rwl = malloc(sizeof(**rwl)); \
+ isc__rwlock_init(*rwl); \
}
#define isc_rwlock_lock(rwl, type) isc__rwlock_lock(*rwl, type)
#define isc_rwlock_trylock(rwl, type) isc__rwlock_trylock(*rwl, type)
typedef pthread_rwlock_t isc_rwlock_t;
typedef pthread_rwlock_t isc__rwlock_t;
-#define isc_rwlock_init(rwl, rq, wq) isc__rwlock_init(rwl, rq, wq)
+#define isc_rwlock_init(rwl) isc__rwlock_init(rwl)
#define isc_rwlock_lock(rwl, type) isc__rwlock_lock(rwl, type)
#define isc_rwlock_trylock(rwl, type) isc__rwlock_trylock(rwl, type)
#define isc_rwlock_unlock(rwl, type) isc__rwlock_unlock(rwl, type)
#endif /* ISC_TRACK_PTHREADS_OBJECTS */
-#define isc__rwlock_init(rwl, read_quota, write_quote) \
+#define isc__rwlock_init(rwl) \
{ \
int _ret = pthread_rwlock_init(rwl, NULL); \
PTHREADS_RUNTIME_CHECK(pthread_rwlock_init, _ret); \
PTHREADS_RUNTIME_CHECK(pthread_rwlock_destroy, _ret); \
}
+#define isc_rwlock_setworkers(workers)
+
#else /* USE_PTHREAD_RWLOCK */
+#include <isc/align.h>
+#include <isc/atomic.h>
+#include <isc/os.h>
+
struct isc_rwlock {
- /* Unlocked. */
- unsigned int magic;
- isc_mutex_t lock;
- atomic_int_fast32_t spins;
-
- /*
- * When some atomic instructions with hardware assistance are
- * available, rwlock will use those so that concurrent readers do not
- * interfere with each other through mutex as long as no writers
- * appear, massively reducing the lock overhead in the typical case.
- *
- * The basic algorithm of this approach is the "simple
- * writer-preference lock" shown in the following URL:
- * http://www.cs.rochester.edu/u/scott/synchronization/pseudocode/rw.html
- * but our implementation does not rely on the spin lock unlike the
- * original algorithm to be more portable as a user space application.
- */
-
- /* Read or modified atomically. */
- atomic_int_fast32_t write_requests;
- atomic_int_fast32_t write_completions;
- atomic_int_fast32_t cnt_and_flag;
-
- /* Locked by lock. */
- isc_condition_t readable;
- isc_condition_t writeable;
- unsigned int readers_waiting;
-
- /* Locked by rwlock itself. */
- atomic_uint_fast32_t write_granted;
-
- /* Unlocked. */
- unsigned int write_quota;
+ alignas(ISC_OS_CACHELINE_SIZE) atomic_uint_fast32_t readers_ingress;
+ alignas(ISC_OS_CACHELINE_SIZE) atomic_uint_fast32_t readers_egress;
+ alignas(ISC_OS_CACHELINE_SIZE) atomic_int_fast32_t writers_barrier;
+ alignas(ISC_OS_CACHELINE_SIZE) atomic_bool writers_lock;
};
typedef struct isc_rwlock isc_rwlock_t;
-typedef struct isc_rwlock isc__rwlock_t;
-#define isc_rwlock_init(rwl, rq, wq) isc__rwlock_init(rwl, rq, wq)
-#define isc_rwlock_lock(rwl, type) isc__rwlock_lock(rwl, type)
-#define isc_rwlock_trylock(rwl, type) isc__rwlock_trylock(rwl, type)
-#define isc_rwlock_unlock(rwl, type) isc__rwlock_unlock(rwl, type)
-#define isc_rwlock_tryupgrade(rwl) isc__rwlock_tryupgrade(rwl)
-#define isc_rwlock_destroy(rwl) isc__rwlock_destroy(rwl)
+void
+isc_rwlock_init(isc_rwlock_t *rwl);
void
-isc__rwlock_init(isc__rwlock_t *rwl, unsigned int read_quota,
- unsigned int write_quota);
+isc_rwlock_rdlock(isc_rwlock_t *rwl);
void
-isc__rwlock_lock(isc__rwlock_t *rwl, isc_rwlocktype_t type);
+isc_rwlock_wrlock(isc_rwlock_t *rwl);
+
+isc_result_t
+isc_rwlock_tryrdlock(isc_rwlock_t *rwl);
isc_result_t
-isc__rwlock_trylock(isc__rwlock_t *rwl, isc_rwlocktype_t type);
+isc_rwlock_trywrlock(isc_rwlock_t *rwl);
void
-isc__rwlock_unlock(isc__rwlock_t *rwl, isc_rwlocktype_t type);
+isc_rwlock_rdunlock(isc_rwlock_t *rwl);
+
+void
+isc_rwlock_wrunlock(isc_rwlock_t *rwl);
isc_result_t
-isc__rwlock_tryupgrade(isc__rwlock_t *rwl);
+isc_rwlock_tryupgrade(isc_rwlock_t *rwl);
+
+void
+isc_rwlock_downgrade(isc_rwlock_t *rwl);
void
-isc__rwlock_destroy(isc__rwlock_t *rwl);
+isc_rwlock_destroy(isc_rwlock_t *rwl);
+
+void
+isc_rwlock_setworkers(uint16_t workers);
+
+#define isc_rwlock_lock(rwl, type) \
+ { \
+ switch (type) { \
+ case isc_rwlocktype_read: \
+ isc_rwlock_rdlock(rwl); \
+ break; \
+ case isc_rwlocktype_write: \
+ isc_rwlock_wrlock(rwl); \
+ break; \
+ default: \
+ UNREACHABLE(); \
+ } \
+ }
+
+#define isc_rwlock_trylock(rwl, type) \
+ ({ \
+ int __result; \
+ switch (type) { \
+ case isc_rwlocktype_read: \
+ __result = isc_rwlock_tryrdlock(rwl); \
+ break; \
+ case isc_rwlocktype_write: \
+ __result = isc_rwlock_trywrlock(rwl); \
+ break; \
+ default: \
+ UNREACHABLE(); \
+ } \
+ __result; \
+ })
+
+#define isc_rwlock_unlock(rwl, type) \
+ { \
+ switch (type) { \
+ case isc_rwlocktype_read: \
+ isc_rwlock_rdunlock(rwl); \
+ break; \
+ case isc_rwlocktype_write: \
+ isc_rwlock_wrunlock(rwl); \
+ break; \
+ default: \
+ UNREACHABLE(); \
+ } \
+ }
#endif /* USE_PTHREAD_RWLOCK */
* information regarding copyright ownership.
*/
+/*
+ * Modified C-RW-WP Implementation from NUMA-Aware Reader-Writer Locks paper:
+ * http://dl.acm.org/citation.cfm?id=2442532
+ *
+ * This work is based on C++ code available from
+ * https://github.com/pramalhe/ConcurrencyFreaks/
+ *
+ * Copyright (c) 2014-2016, Pedro Ramalhete, Andreia Correia
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ * * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * * Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ * * Neither the name of Concurrency Freaks nor the
+ * names of its contributors may be used to endorse or promote products
+ * derived from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
+ * IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+ * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
+ * PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL <COPYRIGHT HOLDER>
+ * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
+ * THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
/*! \file */
#include <inttypes.h>
#include <stdbool.h>
#include <stddef.h>
+#include <stdlib.h>
+#include <unistd.h>
#include <isc/atomic.h>
-#include <isc/magic.h>
+#include <isc/hash.h>
#include <isc/pause.h>
-#include <isc/print.h>
#include <isc/rwlock.h>
+#include <isc/thread.h>
+#include <isc/tid.h>
#include <isc/util.h>
-#define RWLOCK_MAGIC ISC_MAGIC('R', 'W', 'L', 'k')
-#define VALID_RWLOCK(rwl) ISC_MAGIC_VALID(rwl, RWLOCK_MAGIC)
-
-#ifndef RWLOCK_DEFAULT_READ_QUOTA
-#define RWLOCK_DEFAULT_READ_QUOTA 4
-#endif /* ifndef RWLOCK_DEFAULT_READ_QUOTA */
+static atomic_uint_fast16_t isc__crwlock_workers = 128;
-#ifndef RWLOCK_DEFAULT_WRITE_QUOTA
-#define RWLOCK_DEFAULT_WRITE_QUOTA 4
-#endif /* ifndef RWLOCK_DEFAULT_WRITE_QUOTA */
+#define ISC_RWLOCK_UNLOCKED false
+#define ISC_RWLOCK_LOCKED true
-#ifndef RWLOCK_MAX_ADAPTIVE_COUNT
-#define RWLOCK_MAX_ADAPTIVE_COUNT 100
-#endif /* ifndef RWLOCK_MAX_ADAPTIVE_COUNT */
+/*
+ * See https://csce.ucmss.com/cr/books/2017/LFS/CSREA2017/FCS3701.pdf for
+ * guidance on patience level
+ */
+#ifndef RWLOCK_MAX_READER_PATIENCE
+#define RWLOCK_MAX_READER_PATIENCE 500
+#endif /* ifndef RWLOCK_MAX_READER_PATIENCE */
-#ifdef ISC_RWLOCK_TRACE
-#include <stdio.h> /* Required for fprintf/stderr. */
+static void
+read_indicator_wait_until_empty(isc_rwlock_t *rwl);
-#include <isc/thread.h> /* Required for isc_thread_self(). */
+#include <stdio.h>
static void
-print_lock(const char *operation, isc__rwlock_t *rwl, isc_rwlocktype_t type) {
- fprintf(stderr,
- "rwlock %p thread %" PRIuPTR " %s(%s): "
- "write_requests=%u, write_completions=%u, "
- "cnt_and_flag=0x%x, readers_waiting=%u, "
- "write_granted=%u, write_quota=%u\n",
- rwl, isc_thread_self(), operation,
- (type == isc_rwlocktype_read ? "read" : "write"),
- atomic_load_acquire(&rwl->write_requests),
- atomic_load_acquire(&rwl->write_completions),
- atomic_load_acquire(&rwl->cnt_and_flag), rwl->readers_waiting,
- atomic_load_acquire(&rwl->write_granted), rwl->write_quota);
+read_indicator_arrive(isc_rwlock_t *rwl) {
+ (void)atomic_fetch_add_release(&rwl->readers_ingress, 1);
}
-#endif /* ISC_RWLOCK_TRACE */
-
-void
-isc__rwlock_init(isc__rwlock_t *rwl, unsigned int read_quota,
- unsigned int write_quota) {
- REQUIRE(rwl != NULL);
- /*
- * In case there's trouble initializing, we zero magic now. If all
- * goes well, we'll set it to RWLOCK_MAGIC.
- */
- rwl->magic = 0;
-
- atomic_init(&rwl->spins, 0);
- atomic_init(&rwl->write_requests, 0);
- atomic_init(&rwl->write_completions, 0);
- atomic_init(&rwl->cnt_and_flag, 0);
- rwl->readers_waiting = 0;
- atomic_init(&rwl->write_granted, 0);
- if (read_quota != 0) {
- UNEXPECTED_ERROR("read quota is not supported");
- }
- if (write_quota == 0) {
- write_quota = RWLOCK_DEFAULT_WRITE_QUOTA;
- }
- rwl->write_quota = write_quota;
+static void
+read_indicator_depart(isc_rwlock_t *rwl) {
+ (void)atomic_fetch_add_release(&rwl->readers_egress, 1);
+}
- isc_mutex_init(&rwl->lock);
+static bool
+read_indicator_isempty(isc_rwlock_t *rwl) {
+ return (atomic_load_acquire(&rwl->readers_egress) ==
+ atomic_load_acquire(&rwl->readers_ingress));
+}
- isc_condition_init(&rwl->readable);
- isc_condition_init(&rwl->writeable);
+static void
+writers_barrier_raise(isc_rwlock_t *rwl) {
+ (void)atomic_fetch_add_release(&rwl->writers_barrier, 1);
+}
- rwl->magic = RWLOCK_MAGIC;
+static void
+writers_barrier_lower(isc_rwlock_t *rwl) {
+ (void)atomic_fetch_sub_release(&rwl->writers_barrier, 1);
}
-void
-isc__rwlock_destroy(isc__rwlock_t *rwl) {
- REQUIRE(VALID_RWLOCK(rwl));
-
- REQUIRE(atomic_load_acquire(&rwl->write_requests) ==
- atomic_load_acquire(&rwl->write_completions) &&
- atomic_load_acquire(&rwl->cnt_and_flag) == 0 &&
- rwl->readers_waiting == 0);
-
- rwl->magic = 0;
- isc_condition_destroy(&rwl->readable);
- isc_condition_destroy(&rwl->writeable);
- isc_mutex_destroy(&rwl->lock);
+static bool
+writers_barrier_israised(isc_rwlock_t *rwl) {
+ return (atomic_load_acquire(&rwl->writers_barrier) > 0);
}
-/*
- * When some architecture-dependent atomic operations are available,
- * rwlock can be more efficient than the generic algorithm defined below.
- * The basic algorithm is described in the following URL:
- * http://www.cs.rochester.edu/u/scott/synchronization/pseudocode/rw.html
- *
- * The key is to use the following integer variables modified atomically:
- * write_requests, write_completions, and cnt_and_flag.
- *
- * write_requests and write_completions act as a waiting queue for writers
- * in order to ensure the FIFO order. Both variables begin with the initial
- * value of 0. When a new writer tries to get a write lock, it increments
- * write_requests and gets the previous value of the variable as a "ticket".
- * When write_completions reaches the ticket number, the new writer can start
- * writing. When the writer completes its work, it increments
- * write_completions so that another new writer can start working. If the
- * write_requests is not equal to write_completions, it means a writer is now
- * working or waiting. In this case, a new readers cannot start reading, or
- * in other words, this algorithm basically prefers writers.
- *
- * cnt_and_flag is a "lock" shared by all readers and writers. This integer
- * variable is a kind of structure with two members: writer_flag (1 bit) and
- * reader_count (31 bits). The writer_flag shows whether a writer is working,
- * and the reader_count shows the number of readers currently working or almost
- * ready for working. A writer who has the current "ticket" tries to get the
- * lock by exclusively setting the writer_flag to 1, provided that the whole
- * 32-bit is 0 (meaning no readers or writers working). On the other hand,
- * a new reader tries to increment the "reader_count" field provided that
- * the writer_flag is 0 (meaning there is no writer working).
- *
- * If some of the above operations fail, the reader or the writer sleeps
- * until the related condition changes. When a working reader or writer
- * completes its work, some readers or writers are sleeping, and the condition
- * that suspended the reader or writer has changed, it wakes up the sleeping
- * readers or writers.
- *
- * As already noted, this algorithm basically prefers writers. In order to
- * prevent readers from starving, however, the algorithm also introduces the
- * "writer quota" (Q). When Q consecutive writers have completed their work,
- * suspending readers, the last writer will wake up the readers, even if a new
- * writer is waiting.
- *
- * Implementation specific note: due to the combination of atomic operations
- * and a mutex lock, ordering between the atomic operation and locks can be
- * very sensitive in some cases. In particular, it is generally very important
- * to check the atomic variable that requires a reader or writer to sleep after
- * locking the mutex and before actually sleeping; otherwise, it could be very
- * likely to cause a deadlock. For example, assume "var" is a variable
- * atomically modified, then the corresponding code would be:
- * if (var == need_sleep) {
- * LOCK(lock);
- * if (var == need_sleep)
- * WAIT(cond, lock);
- * UNLOCK(lock);
- * }
- * The second check is important, since "var" is protected by the atomic
- * operation, not by the mutex, and can be changed just before sleeping.
- * (The first "if" could be omitted, but this is also important in order to
- * make the code efficient by avoiding the use of the mutex unless it is
- * really necessary.)
- */
+static bool
+writers_lock_islocked(isc_rwlock_t *rwl) {
+ return (atomic_load_acquire(&rwl->writers_lock) == ISC_RWLOCK_LOCKED);
+}
-#define WRITER_ACTIVE 0x1
-#define READER_INCR 0x2
+static bool
+writers_lock_acquire(isc_rwlock_t *rwl) {
+ return (atomic_compare_exchange_weak_acq_rel(
+ &rwl->writers_lock, &(bool){ ISC_RWLOCK_UNLOCKED },
+ ISC_RWLOCK_LOCKED));
+}
static void
-rwlock_lock(isc__rwlock_t *rwl, isc_rwlocktype_t type) {
- int32_t cntflag;
-
- REQUIRE(VALID_RWLOCK(rwl));
-
-#ifdef ISC_RWLOCK_TRACE
- print_lock("prelock", rwl, type);
-#endif /* ifdef ISC_RWLOCK_TRACE */
-
- if (type == isc_rwlocktype_read) {
- if (atomic_load_acquire(&rwl->write_requests) !=
- atomic_load_acquire(&rwl->write_completions))
- {
- /* there is a waiting or active writer */
- LOCK(&rwl->lock);
- if (atomic_load_acquire(&rwl->write_requests) !=
- atomic_load_acquire(&rwl->write_completions))
- {
- rwl->readers_waiting++;
- WAIT(&rwl->readable, &rwl->lock);
- rwl->readers_waiting--;
- }
- UNLOCK(&rwl->lock);
- }
+writers_lock_release(isc_rwlock_t *rwl) {
+ REQUIRE(atomic_compare_exchange_strong_acq_rel(
+ &rwl->writers_lock, &(bool){ ISC_RWLOCK_LOCKED },
+ ISC_RWLOCK_UNLOCKED));
+}
- cntflag = atomic_fetch_add_release(&rwl->cnt_and_flag,
- READER_INCR);
- POST(cntflag);
- while (1) {
- if ((atomic_load_acquire(&rwl->cnt_and_flag) &
- WRITER_ACTIVE) == 0)
- {
- break;
- }
+#define ran_out_of_patience(cnt) (cnt >= RWLOCK_MAX_READER_PATIENCE)
- /* A writer is still working */
- LOCK(&rwl->lock);
- rwl->readers_waiting++;
- if ((atomic_load_acquire(&rwl->cnt_and_flag) &
- WRITER_ACTIVE) != 0)
- {
- WAIT(&rwl->readable, &rwl->lock);
- }
- rwl->readers_waiting--;
- UNLOCK(&rwl->lock);
-
- /*
- * Typically, the reader should be able to get a lock
- * at this stage:
- * (1) there should have been no pending writer when
- * the reader was trying to increment the
- * counter; otherwise, the writer should be in
- * the waiting queue, preventing the reader from
- * proceeding to this point.
- * (2) once the reader increments the counter, no
- * more writer can get a lock.
- * Still, it is possible another writer can work at
- * this point, e.g. in the following scenario:
- * A previous writer unlocks the writer lock.
- * This reader proceeds to point (1).
- * A new writer appears, and gets a new lock before
- * the reader increments the counter.
- * The reader then increments the counter.
- * The previous writer notices there is a waiting
- * reader who is almost ready, and wakes it up.
- * So, the reader needs to confirm whether it can now
- * read explicitly (thus we loop). Note that this is
- * not an infinite process, since the reader has
- * incremented the counter at this point.
- */
- }
-
- /*
- * If we are temporarily preferred to writers due to the writer
- * quota, reset the condition (race among readers doesn't
- * matter).
- */
- atomic_store_release(&rwl->write_granted, 0);
- } else {
- int32_t prev_writer;
-
- /* enter the waiting queue, and wait for our turn */
- prev_writer = atomic_fetch_add_release(&rwl->write_requests, 1);
- while (atomic_load_acquire(&rwl->write_completions) !=
- prev_writer)
- {
- LOCK(&rwl->lock);
- if (atomic_load_acquire(&rwl->write_completions) !=
- prev_writer)
- {
- WAIT(&rwl->writeable, &rwl->lock);
- UNLOCK(&rwl->lock);
- continue;
- }
- UNLOCK(&rwl->lock);
+void
+isc_rwlock_rdlock(isc_rwlock_t *rwl) {
+ uint32_t cnt = 0;
+ bool barrier_raised = false;
+
+ while (true) {
+ read_indicator_arrive(rwl);
+ if (!writers_lock_islocked(rwl)) {
+ /* Acquired lock in read-only mode */
break;
}
- while (!atomic_compare_exchange_weak_acq_rel(
- &rwl->cnt_and_flag, &(int_fast32_t){ 0 },
- WRITER_ACTIVE))
- {
- /* Another active reader or writer is working. */
- LOCK(&rwl->lock);
- if (atomic_load_acquire(&rwl->cnt_and_flag) != 0) {
- WAIT(&rwl->writeable, &rwl->lock);
+ /* Writer has acquired the lock, must reset to 0 and wait */
+ read_indicator_depart(rwl);
+
+ while (writers_lock_islocked(rwl)) {
+ isc_pause();
+ if (ran_out_of_patience(cnt++) && !barrier_raised) {
+ writers_barrier_raise(rwl);
+ barrier_raised = true;
}
- UNLOCK(&rwl->lock);
}
+ }
+ if (barrier_raised) {
+ writers_barrier_lower(rwl);
+ }
+}
+
+isc_result_t
+isc_rwlock_tryrdlock(isc_rwlock_t *rwl) {
+ read_indicator_arrive(rwl);
+ if (writers_lock_islocked(rwl)) {
+ /* Writer has acquired the lock, release the read lock */
+ read_indicator_depart(rwl);
- INSIST((atomic_load_acquire(&rwl->cnt_and_flag) &
- WRITER_ACTIVE));
- atomic_fetch_add_release(&rwl->write_granted, 1);
+ return (ISC_R_LOCKBUSY);
}
-#ifdef ISC_RWLOCK_TRACE
- print_lock("postlock", rwl, type);
-#endif /* ifdef ISC_RWLOCK_TRACE */
+ /* Acquired lock in read-only mode */
+ return (ISC_R_SUCCESS);
}
void
-isc__rwlock_lock(isc__rwlock_t *rwl, isc_rwlocktype_t type) {
- int32_t cnt = 0;
- int32_t spins = atomic_load_acquire(&rwl->spins) * 2 + 10;
- int32_t max_cnt = ISC_MAX(spins, RWLOCK_MAX_ADAPTIVE_COUNT);
-
- do {
- if (cnt++ >= max_cnt) {
- rwlock_lock(rwl, type);
- break;
- }
- isc_pause();
- } while (isc_rwlock_trylock(rwl, type) != ISC_R_SUCCESS);
-
- atomic_fetch_add_release(&rwl->spins, (cnt - spins) / 8);
+isc_rwlock_rdunlock(isc_rwlock_t *rwl) {
+ read_indicator_depart(rwl);
}
isc_result_t
-isc__rwlock_trylock(isc__rwlock_t *rwl, isc_rwlocktype_t type) {
- int32_t cntflag;
+isc_rwlock_tryupgrade(isc_rwlock_t *rwl) {
+ /* Write Barriers has been raised */
+ if (writers_barrier_israised(rwl)) {
+ return (ISC_R_LOCKBUSY);
+ }
- REQUIRE(VALID_RWLOCK(rwl));
+ /* Try to acquire the write-lock */
+ if (!writers_lock_acquire(rwl)) {
+ return (ISC_R_LOCKBUSY);
+ }
-#ifdef ISC_RWLOCK_TRACE
- print_lock("prelock", rwl, type);
-#endif /* ifdef ISC_RWLOCK_TRACE */
+ /* Unlock the read-lock */
+ read_indicator_depart(rwl);
- if (type == isc_rwlocktype_read) {
- /* If a writer is waiting or working, we fail. */
- if (atomic_load_acquire(&rwl->write_requests) !=
- atomic_load_acquire(&rwl->write_completions))
- {
- return (ISC_R_LOCKBUSY);
- }
+ if (!read_indicator_isempty(rwl)) {
+ /* Re-acquire the read-lock back */
+ read_indicator_arrive(rwl);
- /* Otherwise, be ready for reading. */
- cntflag = atomic_fetch_add_release(&rwl->cnt_and_flag,
- READER_INCR);
- if ((cntflag & WRITER_ACTIVE) != 0) {
- /*
- * A writer is working. We lose, and cancel the read
- * request.
- */
- cntflag = atomic_fetch_sub_release(&rwl->cnt_and_flag,
- READER_INCR);
- /*
- * If no other readers are waiting and we've suspended
- * new writers in this short period, wake them up.
- */
- if (cntflag == READER_INCR &&
- atomic_load_acquire(&rwl->write_completions) !=
- atomic_load_acquire(&rwl->write_requests))
- {
- LOCK(&rwl->lock);
- BROADCAST(&rwl->writeable);
- UNLOCK(&rwl->lock);
- }
+ /* Unlock the write-lock */
+ writers_lock_release(rwl);
+ return (ISC_R_LOCKBUSY);
+ }
+ return (ISC_R_SUCCESS);
+}
- return (ISC_R_LOCKBUSY);
- }
- } else {
- /* Try locking without entering the waiting queue. */
- int_fast32_t zero = 0;
- if (!atomic_compare_exchange_strong_acq_rel(
- &rwl->cnt_and_flag, &zero, WRITER_ACTIVE))
- {
- return (ISC_R_LOCKBUSY);
+static void
+read_indicator_wait_until_empty(isc_rwlock_t *rwl) {
+ /* Write-lock was acquired, now wait for running Readers to finish */
+ while (true) {
+ if (read_indicator_isempty(rwl)) {
+ break;
}
+ isc_pause();
+ }
+}
- /*
- * XXXJT: jump into the queue, possibly breaking the writer
- * order.
- */
- atomic_fetch_sub_release(&rwl->write_completions, 1);
- atomic_fetch_add_release(&rwl->write_granted, 1);
+void
+isc_rwlock_wrlock(isc_rwlock_t *rwl) {
+ /* Write Barriers has been raised, wait */
+ while (writers_barrier_israised(rwl)) {
+ isc_pause();
}
-#ifdef ISC_RWLOCK_TRACE
- print_lock("postlock", rwl, type);
-#endif /* ifdef ISC_RWLOCK_TRACE */
+ /* Try to acquire the write-lock */
+ while (!writers_lock_acquire(rwl)) {
+ isc_pause();
+ }
- return (ISC_R_SUCCESS);
+ read_indicator_wait_until_empty(rwl);
+}
+
+void
+isc_rwlock_wrunlock(isc_rwlock_t *rwl) {
+ writers_lock_release(rwl);
}
isc_result_t
-isc__rwlock_tryupgrade(isc__rwlock_t *rwl) {
- REQUIRE(VALID_RWLOCK(rwl));
-
- int_fast32_t reader_incr = READER_INCR;
-
- /* Try to acquire write access. */
- atomic_compare_exchange_strong_acq_rel(&rwl->cnt_and_flag, &reader_incr,
- WRITER_ACTIVE);
- /*
- * There must have been no writer, and there must have
- * been at least one reader.
- */
- INSIST((reader_incr & WRITER_ACTIVE) == 0 &&
- (reader_incr & ~WRITER_ACTIVE) != 0);
-
- if (reader_incr == READER_INCR) {
- /*
- * We are the only reader and have been upgraded.
- * Now jump into the head of the writer waiting queue.
- */
- atomic_fetch_sub_release(&rwl->write_completions, 1);
- } else {
+isc_rwlock_trywrlock(isc_rwlock_t *rwl) {
+ /* Write Barriers has been raised */
+ if (writers_barrier_israised(rwl)) {
+ return (ISC_R_LOCKBUSY);
+ }
+
+ /* Try to acquire the write-lock */
+ if (!writers_lock_acquire(rwl)) {
+ return (ISC_R_LOCKBUSY);
+ }
+
+ if (!read_indicator_isempty(rwl)) {
+ /* Unlock the write-lock */
+ writers_lock_release(rwl);
+
return (ISC_R_LOCKBUSY);
}
}
void
-isc__rwlock_unlock(isc__rwlock_t *rwl, isc_rwlocktype_t type) {
- int32_t prev_cnt;
-
- REQUIRE(VALID_RWLOCK(rwl));
-
-#ifdef ISC_RWLOCK_TRACE
- print_lock("preunlock", rwl, type);
-#endif /* ifdef ISC_RWLOCK_TRACE */
-
- if (type == isc_rwlocktype_read) {
- prev_cnt = atomic_fetch_sub_release(&rwl->cnt_and_flag,
- READER_INCR);
- /*
- * If we're the last reader and any writers are waiting, wake
- * them up. We need to wake up all of them to ensure the
- * FIFO order.
- */
- if (prev_cnt == READER_INCR &&
- atomic_load_acquire(&rwl->write_completions) !=
- atomic_load_acquire(&rwl->write_requests))
- {
- LOCK(&rwl->lock);
- BROADCAST(&rwl->writeable);
- UNLOCK(&rwl->lock);
- }
- } else {
- bool wakeup_writers = true;
-
- /*
- * Reset the flag, and (implicitly) tell other writers
- * we are done.
- */
- atomic_fetch_sub_release(&rwl->cnt_and_flag, WRITER_ACTIVE);
- atomic_fetch_add_release(&rwl->write_completions, 1);
-
- if ((atomic_load_acquire(&rwl->write_granted) >=
- rwl->write_quota) ||
- (atomic_load_acquire(&rwl->write_requests) ==
- atomic_load_acquire(&rwl->write_completions)) ||
- (atomic_load_acquire(&rwl->cnt_and_flag) & ~WRITER_ACTIVE))
- {
- /*
- * We have passed the write quota, no writer is
- * waiting, or some readers are almost ready, pending
- * possible writers. Note that the last case can
- * happen even if write_requests != write_completions
- * (which means a new writer in the queue), so we need
- * to catch the case explicitly.
- */
- LOCK(&rwl->lock);
- if (rwl->readers_waiting > 0) {
- wakeup_writers = false;
- BROADCAST(&rwl->readable);
- }
- UNLOCK(&rwl->lock);
- }
+isc_rwlock_downgrade(isc_rwlock_t *rwl) {
+ read_indicator_arrive(rwl);
- if ((atomic_load_acquire(&rwl->write_requests) !=
- atomic_load_acquire(&rwl->write_completions)) &&
- wakeup_writers)
- {
- LOCK(&rwl->lock);
- BROADCAST(&rwl->writeable);
- UNLOCK(&rwl->lock);
- }
- }
+ writers_lock_release(rwl);
+}
+
+void
+isc_rwlock_init(isc_rwlock_t *rwl) {
+ REQUIRE(rwl != NULL);
-#ifdef ISC_RWLOCK_TRACE
- print_lock("postunlock", rwl, type);
-#endif /* ifdef ISC_RWLOCK_TRACE */
+ atomic_init(&rwl->writers_lock, ISC_RWLOCK_UNLOCKED);
+ atomic_init(&rwl->writers_barrier, 0);
+ atomic_init(&rwl->readers_ingress, 0);
+ atomic_init(&rwl->readers_egress, 0);
+}
+
+void
+isc_rwlock_destroy(isc_rwlock_t *rwl) {
+ /* Check whether write lock has been unlocked */
+ REQUIRE(atomic_load(&rwl->writers_lock) == ISC_RWLOCK_UNLOCKED);
+ REQUIRE(read_indicator_isempty(rwl));
+}
+
+void
+isc_rwlock_setworkers(uint16_t workers) {
+ atomic_store(&isc__crwlock_workers, workers);
}
projects / thirdparty / bind9.git / commitdiff
