kernel/locking/rwsem.h

   1 /* SPDX-License-Identifier: GPL-2.0 */
   2 /*
   3  * The least significant 2 bits of the owner value has the following
   4  * meanings when set.
   5  *  - RWSEM_READER_OWNED (bit 0): The rwsem is owned by readers
   6  *  - RWSEM_ANONYMOUSLY_OWNED (bit 1): The rwsem is anonymously owned,
   7  *    i.e. the owner(s) cannot be readily determined. It can be reader
   8  *    owned or the owning writer is indeterminate.
   9  *
  10  * When a writer acquires a rwsem, it puts its task_struct pointer
  11  * into the owner field. It is cleared after an unlock.
  12  *
  13  * When a reader acquires a rwsem, it will also puts its task_struct
  14  * pointer into the owner field with both the RWSEM_READER_OWNED and
  15  * RWSEM_ANONYMOUSLY_OWNED bits set. On unlock, the owner field will
  16  * largely be left untouched. So for a free or reader-owned rwsem,
  17  * the owner value may contain information about the last reader that
  18  * acquires the rwsem. The anonymous bit is set because that particular
  19  * reader may or may not still own the lock.
  20  *
  21  * That information may be helpful in debugging cases where the system
  22  * seems to hang on a reader owned rwsem especially if only one reader
  23  * is involved. Ideally we would like to track all the readers that own
  24  * a rwsem, but the overhead is simply too big.
  25  */
  26 #define RWSEM_READER_OWNED      (1UL << 0)
  27 #define RWSEM_ANONYMOUSLY_OWNED (1UL << 1)
  28
  29 #ifdef CONFIG_DEBUG_RWSEMS
  30 # define DEBUG_RWSEMS_WARN_ON(c)        DEBUG_LOCKS_WARN_ON(c)
  31 #else
  32 # define DEBUG_RWSEMS_WARN_ON(c)
  33 #endif
  34
  35 #ifdef CONFIG_RWSEM_SPIN_ON_OWNER
  36 /*
  37  * All writes to owner are protected by WRITE_ONCE() to make sure that
  38  * store tearing can't happen as optimistic spinners may read and use
  39  * the owner value concurrently without lock. Read from owner, however,
  40  * may not need READ_ONCE() as long as the pointer value is only used
  41  * for comparison and isn't being dereferenced.
  42  */
  43 static inline void rwsem_set_owner(struct rw_semaphore *sem)
  44 {
  45         WRITE_ONCE(sem->owner, current);
  46 }
  47
  48 static inline void rwsem_clear_owner(struct rw_semaphore *sem)
  49 {
  50         WRITE_ONCE(sem->owner, NULL);
  51 }
  52
  53 /*
  54  * The task_struct pointer of the last owning reader will be left in
  55  * the owner field.
  56  *
  57  * Note that the owner value just indicates the task has owned the rwsem
  58  * previously, it may not be the real owner or one of the real owners
  59  * anymore when that field is examined, so take it with a grain of salt.
  60  */
  61 static inline void __rwsem_set_reader_owned(struct rw_semaphore *sem,
  62                                             struct task_struct *owner)
  63 {
  64         unsigned long val = (unsigned long)owner | RWSEM_READER_OWNED
  65                                                  | RWSEM_ANONYMOUSLY_OWNED;
  66
  67         WRITE_ONCE(sem->owner, (struct task_struct *)val);
  68 }
  69
  70 static inline void rwsem_set_reader_owned(struct rw_semaphore *sem)
  71 {
  72         __rwsem_set_reader_owned(sem, current);
  73 }
  74
  75 /*
  76  * Return true if the a rwsem waiter can spin on the rwsem's owner
  77  * and steal the lock, i.e. the lock is not anonymously owned.
  78  * N.B. !owner is considered spinnable.
  79  */
  80 static inline bool is_rwsem_owner_spinnable(struct task_struct *owner)
  81 {
  82         return !((unsigned long)owner & RWSEM_ANONYMOUSLY_OWNED);
  83 }
  84
  85 /*
  86  * Return true if rwsem is owned by an anonymous writer or readers.
  87  */
  88 static inline bool rwsem_has_anonymous_owner(struct task_struct *owner)
  89 {
  90         return (unsigned long)owner & RWSEM_ANONYMOUSLY_OWNED;
  91 }
  92
  93 #ifdef CONFIG_DEBUG_RWSEMS
  94 /*
  95  * With CONFIG_DEBUG_RWSEMS configured, it will make sure that if there
  96  * is a task pointer in owner of a reader-owned rwsem, it will be the
  97  * real owner or one of the real owners. The only exception is when the
  98  * unlock is done by up_read_non_owner().
  99  */
 100 #define rwsem_clear_reader_owned rwsem_clear_reader_owned
 101 static inline void rwsem_clear_reader_owned(struct rw_semaphore *sem)
 102 {
 103         unsigned long val = (unsigned long)current | RWSEM_READER_OWNED
 104                                                    | RWSEM_ANONYMOUSLY_OWNED;
 105         if (READ_ONCE(sem->owner) == (struct task_struct *)val)
 106                 cmpxchg_relaxed((unsigned long *)&sem->owner, val,
 107                                 RWSEM_READER_OWNED | RWSEM_ANONYMOUSLY_OWNED);
 108 }
 109 #endif
 110
 111 #else
 112 static inline void rwsem_set_owner(struct rw_semaphore *sem)
 113 {
 114 }
 115
 116 static inline void rwsem_clear_owner(struct rw_semaphore *sem)
 117 {
 118 }
 119
 120 static inline void __rwsem_set_reader_owned(struct rw_semaphore *sem,
 121                                            struct task_struct *owner)
 122 {
 123 }
 124
 125 static inline void rwsem_set_reader_owned(struct rw_semaphore *sem)
 126 {
 127 }
 128 #endif
 129
 130 #ifndef rwsem_clear_reader_owned
 131 static inline void rwsem_clear_reader_owned(struct rw_semaphore *sem)
 132 {
 133 }
 134 #endif