rseq: Switch to fast path processing on exit to user

Now that all bits and pieces are in place, hook the RSEQ handling fast path
function into exit_to_user_mode_prepare() after the TIF work bits have been
handled. If case of fast path failure, TIF_NOTIFY_RESUME has been raised
and the caller needs to take another turn through the TIF handling slow
path.

This only works for architectures which use the generic entry code.
Architectures who still have their own incomplete hacks are not supported
and won't be.

This results in the following improvements:

  Kernel build	       Before		  After		      Reduction

  exit to user         80692981		  80514451
  signal checks:          32581		       121	       99%
  slowpath runs:        1201408   1.49%	       198 0.00%      100%
  fastpath runs:			    675941 0.84%       N/A
  id updates:           1233989   1.53%	     50541 0.06%       96%
  cs checks:            1125366   1.39%	         0 0.00%      100%
    cs cleared:         1125366      100%	 0            100%
    cs fixup:                 0        0%	 0

  RSEQ selftests      Before		  After		      Reduction

  exit to user:       386281778		  387373750
  signal checks:       35661203		          0           100%
  slowpath runs:      140542396 36.38%	        100  0.00%    100%
  fastpath runs:			    9509789  2.51%     N/A
  id updates:         176203599 45.62%	    9087994  2.35%     95%
  cs checks:          175587856 45.46%	    4728394  1.22%     98%
    cs cleared:       172359544   98.16%    1319307   27.90%   99%
    cs fixup:           3228312    1.84%    3409087   72.10%

The 'cs cleared' and 'cs fixup' percentages are not relative to the exit to
user invocations, they are relative to the actual 'cs check' invocations.

While some of this could have been avoided in the original code, like the
obvious clearing of CS when it's already clear, the main problem of going
through TIF_NOTIFY_RESUME cannot be solved. In some workloads the RSEQ
notify handler is invoked more than once before going out to user
space. Doing this once when everything has stabilized is the only solution
to avoid this.

The initial attempt to completely decouple it from the TIF work turned out
to be suboptimal for workloads, which do a lot of quick and short system
calls. Even if the fast path decision is only 4 instructions (including a
conditional branch), this adds up quickly and becomes measurable when the
rate for actually having to handle rseq is in the low single digit
percentage range of user/kernel transitions.

Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>
Signed-off-by: Ingo Molnar <mingo@kernel.org>
Reviewed-by: Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
Link: https://patch.msgid.link/20251027084307.701201365@linutronix.de

diff --git a/include/linux/irq-entry-common.h b/include/linux/irq-entry-common.h
index cb31fb84d7b40f047e0cd5d648e2ea5579c72d46..8f5ceeaaaea5918d48e3e4ccd7d41adbf664a6f9 100644 (file)
--- a/include/linux/irq-entry-common.h
+++ b/include/linux/irq-entry-common.h
@@ -197,11 +197,8 @@ static __always_inline void arch_exit_to_user_mode(void) { }
  */
 void arch_do_signal_or_restart(struct pt_regs *regs);
 
-/**
- * exit_to_user_mode_loop - do any pending work before leaving to user space
- */
-unsigned long exit_to_user_mode_loop(struct pt_regs *regs,
-                                    unsigned long ti_work);
+/* Handle pending TIF work */
+unsigned long exit_to_user_mode_loop(struct pt_regs *regs, unsigned long ti_work);
 
 /**
  * exit_to_user_mode_prepare - call exit_to_user_mode_loop() if required

diff --git a/include/linux/resume_user_mode.h b/include/linux/resume_user_mode.h
index dd3bf7da90a8ce034ea0c648c9864f4684e59b7c..bf92227c78d0dfc9ccb14f3ffe5aa68fc8777533 100644 (file)
--- a/include/linux/resume_user_mode.h
+++ b/include/linux/resume_user_mode.h
@@ -59,7 +59,7 @@ static inline void resume_user_mode_work(struct pt_regs *regs)
        mem_cgroup_handle_over_high(GFP_KERNEL);
        blkcg_maybe_throttle_current();
 
-       rseq_handle_notify_resume(regs);
+       rseq_handle_slowpath(regs);
 }
 
 #endif /* LINUX_RESUME_USER_MODE_H */

diff --git a/include/linux/rseq.h b/include/linux/rseq.h
index abfbeb42d1a235673785d03e1aa2391d9251c7e1..ded4baa34586a72db4b1d67b4c3ccb38175eab96 100644 (file)
--- a/include/linux/rseq.h
+++ b/include/linux/rseq.h
@@ -7,13 +7,19 @@
 
 #include <uapi/linux/rseq.h>
 
-void __rseq_handle_notify_resume(struct pt_regs *regs);
+void __rseq_handle_slowpath(struct pt_regs *regs);
 
-static inline void rseq_handle_notify_resume(struct pt_regs *regs)
+/* Invoked from resume_user_mode_work() */
+static inline void rseq_handle_slowpath(struct pt_regs *regs)
 {
-       /* '&' is intentional to spare one conditional branch */
-       if (current->rseq.event.sched_switch & current->rseq.event.has_rseq)
-               __rseq_handle_notify_resume(regs);
+       if (IS_ENABLED(CONFIG_GENERIC_ENTRY)) {
+               if (current->rseq.event.slowpath)
+                       __rseq_handle_slowpath(regs);
+       } else {
+               /* '&' is intentional to spare one conditional branch */
+               if (current->rseq.event.sched_switch & current->rseq.event.has_rseq)
+                       __rseq_handle_slowpath(regs);
+       }
 }
 
 void __rseq_signal_deliver(int sig, struct pt_regs *regs);
@@ -152,7 +158,7 @@ static inline void rseq_fork(struct task_struct *t, u64 clone_flags)
 }
 
 #else /* CONFIG_RSEQ */
-static inline void rseq_handle_notify_resume(struct pt_regs *regs) { }
+static inline void rseq_handle_slowpath(struct pt_regs *regs) { }
 static inline void rseq_signal_deliver(struct ksignal *ksig, struct pt_regs *regs) { }
 static inline void rseq_sched_switch_event(struct task_struct *t) { }
 static inline void rseq_sched_set_task_cpu(struct task_struct *t, unsigned int cpu) { }

diff --git a/init/Kconfig b/init/Kconfig
index bde40ab664e2450d0a60563dc7cec0ee1082acaa..d1c606ec632eb1917dd2c4b510ea800e61acca4e 100644 (file)
--- a/init/Kconfig
+++ b/init/Kconfig
@@ -1941,7 +1941,7 @@ config RSEQ_DEBUG_DEFAULT_ENABLE
 config DEBUG_RSEQ
        default n
        bool "Enable debugging of rseq() system call" if EXPERT
-       depends on RSEQ && DEBUG_KERNEL
+       depends on RSEQ && DEBUG_KERNEL && !GENERIC_ENTRY
        select RSEQ_DEBUG_DEFAULT_ENABLE
        help
          Enable extra debugging checks for the rseq system call.

diff --git a/kernel/entry/common.c b/kernel/entry/common.c
index 70a16db4cc0ae98207c6b70e86352d20ce5966a4..523a3e758af4623f94f48575a9bed6edfaf90c11 100644 (file)
--- a/kernel/entry/common.c
+++ b/kernel/entry/common.c
@@ -11,13 +11,8 @@
 /* Workaround to allow gradual conversion of architecture code */
 void __weak arch_do_signal_or_restart(struct pt_regs *regs) { }
 
-/**
- * exit_to_user_mode_loop - do any pending work before leaving to user space
- * @regs:      Pointer to pt_regs on entry stack
- * @ti_work:   TIF work flags as read by the caller
- */
-__always_inline unsigned long exit_to_user_mode_loop(struct pt_regs *regs,
-                                                    unsigned long ti_work)
+static __always_inline unsigned long __exit_to_user_mode_loop(struct pt_regs *regs,
+                                                             unsigned long ti_work)
 {
        /*
         * Before returning to user space ensure that all pending work
@@ -62,6 +57,23 @@ __always_inline unsigned long exit_to_user_mode_loop(struct pt_regs *regs,
        return ti_work;
 }
 
+/**
+ * exit_to_user_mode_loop - do any pending work before leaving to user space
+ * @regs:      Pointer to pt_regs on entry stack
+ * @ti_work:   TIF work flags as read by the caller
+ */
+__always_inline unsigned long exit_to_user_mode_loop(struct pt_regs *regs,
+                                                    unsigned long ti_work)
+{
+       for (;;) {
+               ti_work = __exit_to_user_mode_loop(regs, ti_work);
+
+               if (likely(!rseq_exit_to_user_mode_restart(regs)))
+                       return ti_work;
+               ti_work = read_thread_flags();
+       }
+}
+
 noinstr irqentry_state_t irqentry_enter(struct pt_regs *regs)
 {
        irqentry_state_t ret = {

diff --git a/kernel/rseq.c b/kernel/rseq.c
index c5d6336c69560981802e3bba749a468c3e99bf7e..395d8b002350a366f500909c07aaf4c08e558629 100644 (file)
--- a/kernel/rseq.c
+++ b/kernel/rseq.c
@@ -237,7 +237,11 @@ efault:
 
 static void rseq_slowpath_update_usr(struct pt_regs *regs)
 {
-       /* Preserve rseq state and user_irq state for exit to user */
+       /*
+        * Preserve rseq state and user_irq state. The generic entry code
+        * clears user_irq on the way out, the non-generic entry
+        * architectures are not having user_irq.
+        */
        const struct rseq_event evt_mask = { .has_rseq = true, .user_irq = true, };
        struct task_struct *t = current;
        struct rseq_ids ids;
@@ -289,7 +293,7 @@ static void rseq_slowpath_update_usr(struct pt_regs *regs)
        }
 }
 
-void __rseq_handle_notify_resume(struct pt_regs *regs)
+void __rseq_handle_slowpath(struct pt_regs *regs)
 {
        /*
         * If invoked from hypervisors before entering the guest via