locking/semaphore: Remove the list_head from struct semaphore

Instead of embedding a list_head in struct semaphore, store a pointer to
the first waiter.  The list of waiters remains a doubly linked list so
we can efficiently add to the tail of the list and remove from the front
(or middle) of the list.

Some of the list manipulation becomes more complicated, but it's a
reasonable tradeoff on the slow paths to shrink data structures
which embed a semaphore.

Signed-off-by: Matthew Wilcox (Oracle) <willy@infradead.org>
Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>
Link: https://patch.msgid.link/20260305195545.3707590-3-willy@infradead.org

diff --git a/drivers/acpi/osl.c b/drivers/acpi/osl.c
index 5b777316b9acd1b6b8b6b403cda2e1240542a5f5..2af0db9210fe46af2ae7f2e642bdea92c5f2d161 100644 (file)
--- a/drivers/acpi/osl.c
+++ b/drivers/acpi/osl.c
@@ -1257,7 +1257,7 @@ acpi_status acpi_os_delete_semaphore(acpi_handle handle)
 
        ACPI_DEBUG_PRINT((ACPI_DB_MUTEX, "Deleting semaphore[%p].\n", handle));
 
-       BUG_ON(!list_empty(&sem->wait_list));
+       BUG_ON(sem->first_waiter);
        kfree(sem);
        sem = NULL;
 

diff --git a/include/linux/semaphore.h b/include/linux/semaphore.h
index 89706157e62234ad4fd2e3b2c3f930e171507f04..a4c8651ef0217ca384c1c1ce944d47a63f1fb18f 100644 (file)
--- a/include/linux/semaphore.h
+++ b/include/linux/semaphore.h
@@ -15,7 +15,7 @@
 struct semaphore {
        raw_spinlock_t          lock;
        unsigned int            count;
-       struct list_head        wait_list;
+       struct semaphore_waiter *first_waiter;
 
 #ifdef CONFIG_DETECT_HUNG_TASK_BLOCKER
        unsigned long           last_holder;
@@ -33,7 +33,7 @@ struct semaphore {
 {                                                                      \
        .lock           = __RAW_SPIN_LOCK_UNLOCKED((name).lock),        \
        .count          = n,                                            \
-       .wait_list      = LIST_HEAD_INIT((name).wait_list)              \
+       .first_waiter   = NULL                                          \
        __LAST_HOLDER_SEMAPHORE_INITIALIZER                             \
 }
 

diff --git a/kernel/locking/semaphore.c b/kernel/locking/semaphore.c
index 3ef032e22f7ef4a82fa6ad5bf6ee5ecf28e14c91..74d41433ba13ce8c26a08c65185bf1c197396684 100644 (file)
--- a/kernel/locking/semaphore.c
+++ b/kernel/locking/semaphore.c
@@ -21,7 +21,7 @@
  * too.
  *
  * The ->count variable represents how many more tasks can acquire this
- * semaphore.  If it's zero, there may be tasks waiting on the wait_list.
+ * semaphore.  If it's zero, there may be waiters.
  */
 
 #include <linux/compiler.h>
@@ -226,7 +226,7 @@ void __sched up(struct semaphore *sem)
 
        hung_task_sem_clear_if_holder(sem);
 
-       if (likely(list_empty(&sem->wait_list)))
+       if (likely(!sem->first_waiter))
                sem->count++;
        else
                __up(sem, &wake_q);
@@ -244,6 +244,21 @@ struct semaphore_waiter {
        bool up;
 };
 
+static inline
+void sem_del_waiter(struct semaphore *sem, struct semaphore_waiter *waiter)
+{
+       if (list_empty(&waiter->list)) {
+               sem->first_waiter = NULL;
+               return;
+       }
+
+       if (sem->first_waiter == waiter) {
+               sem->first_waiter = list_first_entry(&waiter->list,
+                                                    struct semaphore_waiter, list);
+       }
+       list_del(&waiter->list);
+}
+
 /*
  * Because this function is inlined, the 'state' parameter will be
  * constant, and thus optimised away by the compiler.  Likewise the
@@ -252,9 +267,15 @@ struct semaphore_waiter {
 static inline int __sched ___down_common(struct semaphore *sem, long state,
                                                                long timeout)
 {
-       struct semaphore_waiter waiter;
-
-       list_add_tail(&waiter.list, &sem->wait_list);
+       struct semaphore_waiter waiter, *first;
+
+       first = sem->first_waiter;
+       if (first) {
+               list_add_tail(&waiter.list, &first->list);
+       } else {
+               INIT_LIST_HEAD(&waiter.list);
+               sem->first_waiter = &waiter;
+       }
        waiter.task = current;
        waiter.up = false;
 
@@ -274,11 +295,11 @@ static inline int __sched ___down_common(struct semaphore *sem, long state,
        }
 
  timed_out:
-       list_del(&waiter.list);
+       sem_del_waiter(sem, &waiter);
        return -ETIME;
 
  interrupted:
-       list_del(&waiter.list);
+       sem_del_waiter(sem, &waiter);
        return -EINTR;
 }
 
@@ -321,9 +342,9 @@ static noinline int __sched __down_timeout(struct semaphore *sem, long timeout)
 static noinline void __sched __up(struct semaphore *sem,
                                  struct wake_q_head *wake_q)
 {
-       struct semaphore_waiter *waiter = list_first_entry(&sem->wait_list,
-                                               struct semaphore_waiter, list);
-       list_del(&waiter->list);
+       struct semaphore_waiter *waiter = sem->first_waiter;
+
+       sem_del_waiter(sem, waiter);
        waiter->up = true;
        wake_q_add(wake_q, waiter->task);
 }