posix-timers: Avoid false cacheline sharing

struct k_itimer has the hlist_node, which is used for lookup in the hash
bucket, and the timer lock in the same cache line.

That's obviously bad, if one CPU fiddles with a timer and the other is
walking the hash bucket on which that timer is queued.

Avoid this by restructuring struct k_itimer, so that the read mostly (only
modified during setup and teardown) fields are in the first cache line and
the lock and the rest of the fields which get written to are in cacheline
2-N.

Reduces cacheline contention in a test case of 64 processes creating and
accessing 20000 timers each by almost 30% according to perf.

Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
Reviewed-by: Frederic Weisbecker <frederic@kernel.org>
Link: https://lore.kernel.org/all/20250308155624.341108067@linutronix.de

diff --git a/include/linux/posix-timers.h b/include/linux/posix-timers.h
index e714a55e1ddd091107d01306a5074c8d2cf2f9c4..094ef5778c0f95706b74662e16966683e3ade5b8 100644 (file)
--- a/include/linux/posix-timers.h
+++ b/include/linux/posix-timers.h
@@ -177,23 +177,26 @@ static inline void posix_cputimers_init_work(void) { }
  * @rcu:               RCU head for freeing the timer.
  */
 struct k_itimer {
-       struct hlist_node       list;
-       struct hlist_node       ignored_list;
+       /* 1st cacheline contains read-mostly fields */
        struct hlist_node       t_hash;
-       spinlock_t              it_lock;
-       const struct k_clock    *kclock;
-       clockid_t               it_clock;
+       struct hlist_node       list;
        timer_t                 it_id;
+       clockid_t               it_clock;
+       int                     it_sigev_notify;
+       enum pid_type           it_pid_type;
+       struct signal_struct    *it_signal;
+       const struct k_clock    *kclock;
+
+       /* 2nd cacheline and above contain fields which are modified regularly */
+       spinlock_t              it_lock;
        int                     it_status;
        bool                    it_sig_periodic;
        s64                     it_overrun;
        s64                     it_overrun_last;
        unsigned int            it_signal_seq;
        unsigned int            it_sigqueue_seq;
-       int                     it_sigev_notify;
-       enum pid_type           it_pid_type;
        ktime_t                 it_interval;
-       struct signal_struct    *it_signal;
+       struct hlist_node       ignored_list;
        union {
                struct pid              *it_pid;
                struct task_struct      *it_process;
@@ -210,7 +213,7 @@ struct k_itimer {
                } alarm;
        } it;
        struct rcu_head         rcu;
-};
+} ____cacheline_aligned_in_smp;
 
 void run_posix_cpu_timers(void);
 void posix_cpu_timers_exit(struct task_struct *task);

diff --git a/kernel/time/posix-timers.c b/kernel/time/posix-timers.c
index 0c4cee38d965a29f61e3d3ed5b5f915255c3d25b..e4c92f42fa5745b07f4ce9647ea11e45d3af49b8 100644 (file)
--- a/kernel/time/posix-timers.c
+++ b/kernel/time/posix-timers.c
@@ -260,8 +260,8 @@ static int posix_get_hrtimer_res(clockid_t which_clock, struct timespec64 *tp)
 
 static __init int init_posix_timers(void)
 {
-       posix_timers_cache = kmem_cache_create("posix_timers_cache", sizeof(struct k_itimer), 0,
-                                              SLAB_ACCOUNT, NULL);
+       posix_timers_cache = kmem_cache_create("posix_timers_cache", sizeof(struct k_itimer),
+                                              __alignof__(struct k_itimer), SLAB_ACCOUNT, NULL);
        return 0;
 }
 __initcall(init_posix_timers);