[thirdparty/kernel/stable.git] / kernel / locking / qrwlock.c

// SPDX-License-Identifier: GPL-2.0-or-later
/*
 * Queued read/write locks
 *
 * (C) Copyright 2013-2014 Hewlett-Packard Development Company, L.P.
 *
 * Authors: Waiman Long <waiman.long@hp.com>
 */
#include <linux/smp.h>
#include <linux/bug.h>
#include <linux/cpumask.h>
#include <linux/percpu.h>
#include <linux/hardirq.h>
#include <linux/spinlock.h>
#include <asm/qrwlock.h>

/**
 * queued_read_lock_slowpath - acquire read lock of a queue rwlock
 * @lock: Pointer to queue rwlock structure
 */
void queued_read_lock_slowpath(struct qrwlock *lock)
{
	/*
	 * Readers come here when they cannot get the lock without waiting
	 */
	if (unlikely(in_interrupt())) {
		/*
		 * Readers in interrupt context will get the lock immediately
		 * if the writer is just waiting (not holding the lock yet),
		 * so spin with ACQUIRE semantics until the lock is available
		 * without waiting in the queue.
		 */
		atomic_cond_read_acquire(&lock->cnts, !(VAL & _QW_LOCKED));
		return;
	}
	atomic_sub(_QR_BIAS, &lock->cnts);

	/*
	 * Put the reader into the wait queue
	 */
	arch_spin_lock(&lock->wait_lock);
	atomic_add(_QR_BIAS, &lock->cnts);

	/*
	 * The ACQUIRE semantics of the following spinning code ensure
	 * that accesses can't leak upwards out of our subsequent critical
	 * section in the case that the lock is currently held for write.
	 */
	atomic_cond_read_acquire(&lock->cnts, !(VAL & _QW_LOCKED));

	/*
	 * Signal the next one in queue to become queue head
	 */
	arch_spin_unlock(&lock->wait_lock);
}
EXPORT_SYMBOL(queued_read_lock_slowpath);

/**
 * queued_write_lock_slowpath - acquire write lock of a queue rwlock
 * @lock : Pointer to queue rwlock structure
 */
void queued_write_lock_slowpath(struct qrwlock *lock)
{
	/* Put the writer into the wait queue */
	arch_spin_lock(&lock->wait_lock);

	/* Try to acquire the lock directly if no reader is present */
	if (!atomic_read(&lock->cnts) &&
	    (atomic_cmpxchg_acquire(&lock->cnts, 0, _QW_LOCKED) == 0))
		goto unlock;

	/* Set the waiting flag to notify readers that a writer is pending */
	atomic_add(_QW_WAITING, &lock->cnts);

	/* When no more readers or writers, set the locked flag */
	do {
		atomic_cond_read_acquire(&lock->cnts, VAL == _QW_WAITING);
	} while (atomic_cmpxchg_relaxed(&lock->cnts, _QW_WAITING,
					_QW_LOCKED) != _QW_WAITING);
unlock:
	arch_spin_unlock(&lock->wait_lock);
}
EXPORT_SYMBOL(queued_write_lock_slowpath);
Commit	Line	Data
c942fddf	1	// SPDX-License-Identifier: GPL-2.0-or-later
70af2f8a	2	/*
c7114b4e	3	* Queued read/write locks
70af2f8a	4	*
70af2f8a WL	5	* (C) Copyright 2013-2014 Hewlett-Packard Development Company, L.P.
	6	*
	7	* Authors: Waiman Long <waiman.long@hp.com>
	8	*/
	9	#include <linux/smp.h>
	10	#include <linux/bug.h>
	11	#include <linux/cpumask.h>
	12	#include <linux/percpu.h>
	13	#include <linux/hardirq.h>
9ab6055f	14	#include <linux/spinlock.h>
70af2f8a WL	15	#include <asm/qrwlock.h>
70af2f8a WL	16
70af2f8a	17	/**
f7d71f20	18	* queued_read_lock_slowpath - acquire read lock of a queue rwlock
70af2f8a WL	19	* @lock: Pointer to queue rwlock structure
70af2f8a WL	20	*/
b519b56e	21	void queued_read_lock_slowpath(struct qrwlock *lock)
70af2f8a	22	{
70af2f8a WL	23	/*
	24	* Readers come here when they cannot get the lock without waiting
	25	*/
	26	if (unlikely(in_interrupt())) {
	27	/*
0e06e5be	28	* Readers in interrupt context will get the lock immediately
b519b56e WD	29	* if the writer is just waiting (not holding the lock yet),
	30	* so spin with ACQUIRE semantics until the lock is available
	31	* without waiting in the queue.
70af2f8a	32	*/
d1331661	33	atomic_cond_read_acquire(&lock->cnts, !(VAL & _QW_LOCKED));
70af2f8a WL	34	return;
	35	}
	36	atomic_sub(_QR_BIAS, &lock->cnts);
	37
	38	/*
	39	* Put the reader into the wait queue
	40	*/
6e1e5196	41	arch_spin_lock(&lock->wait_lock);
b519b56e	42	atomic_add(_QR_BIAS, &lock->cnts);
70af2f8a WL	43
70af2f8a WL	44	/*
77e430e3 WD	45	* The ACQUIRE semantics of the following spinning code ensure
	46	* that accesses can't leak upwards out of our subsequent critical
	47	* section in the case that the lock is currently held for write.
70af2f8a	48	*/
d1331661	49	atomic_cond_read_acquire(&lock->cnts, !(VAL & _QW_LOCKED));
70af2f8a WL	50
	51	/*
	52	* Signal the next one in queue to become queue head
	53	*/
6e1e5196	54	arch_spin_unlock(&lock->wait_lock);
70af2f8a	55	}
f7d71f20	56	EXPORT_SYMBOL(queued_read_lock_slowpath);
70af2f8a WL	57
70af2f8a WL	58	/**
f7d71f20	59	* queued_write_lock_slowpath - acquire write lock of a queue rwlock
70af2f8a WL	60	* @lock : Pointer to queue rwlock structure
70af2f8a WL	61	*/
f7d71f20	62	void queued_write_lock_slowpath(struct qrwlock *lock)
70af2f8a	63	{
70af2f8a	64	/* Put the writer into the wait queue */
6e1e5196	65	arch_spin_lock(&lock->wait_lock);
70af2f8a WL	66
	67	/* Try to acquire the lock directly if no reader is present */
	68	if (!atomic_read(&lock->cnts) &&
77e430e3	69	(atomic_cmpxchg_acquire(&lock->cnts, 0, _QW_LOCKED) == 0))
70af2f8a WL	70	goto unlock;
70af2f8a WL	71
d1331661 WD	72	/* Set the waiting flag to notify readers that a writer is pending */
d1331661 WD	73	atomic_add(_QW_WAITING, &lock->cnts);
70af2f8a	74
d1331661	75	/* When no more readers or writers, set the locked flag */
b519b56e WD	76	do {
	77	atomic_cond_read_acquire(&lock->cnts, VAL == _QW_WAITING);
	78	} while (atomic_cmpxchg_relaxed(&lock->cnts, _QW_WAITING,
	79	_QW_LOCKED) != _QW_WAITING);
70af2f8a	80	unlock:
6e1e5196	81	arch_spin_unlock(&lock->wait_lock);
70af2f8a	82	}
f7d71f20	83	EXPORT_SYMBOL(queued_write_lock_slowpath);