powerpc: Add preempt lazy support

Define preempt lazy bit for Powerpc. Use bit 9 which is free and within
16 bit range of NEED_RESCHED, so compiler can issue single andi.

Since Powerpc doesn't use the generic entry/exit, add lazy check at exit
to user. CONFIG_PREEMPTION is defined for lazy/full/rt so use it for
return to kernel.

Ran a few benchmarks and db workload on Power10. Performance is close to
preempt=none/voluntary.

Since Powerpc systems can have large core count and large memory,
preempt lazy is going to be helpful in avoiding soft lockup issues.

Reviewed-by: Sebastian Andrzej Siewior <bigeasy@linutronix.de>
Reviewed-by: Ankur Arora <ankur.a.arora@oracle.com>
Signed-off-by: Shrikanth Hegde <sshegde@linux.ibm.com>
Signed-off-by: Madhavan Srinivasan <maddy@linux.ibm.com>
Link: https://patch.msgid.link/20241116192306.88217-2-sshegde@linux.ibm.com

diff --git a/arch/powerpc/Kconfig b/arch/powerpc/Kconfig
index a0ce777f97063bf858942c60654d8411bcf2a3dc..db9f7b2d07bf92882095fc6c1660c4823333b700 100644 (file)
--- a/arch/powerpc/Kconfig
+++ b/arch/powerpc/Kconfig
@@ -145,6 +145,7 @@ config PPC
        select ARCH_HAS_NON_OVERLAPPING_ADDRESS_SPACE
        select ARCH_HAS_PHYS_TO_DMA
        select ARCH_HAS_PMEM_API
+       select ARCH_HAS_PREEMPT_LAZY
        select ARCH_HAS_PTE_DEVMAP              if PPC_BOOK3S_64
        select ARCH_HAS_PTE_SPECIAL
        select ARCH_HAS_SCALED_CPUTIME          if VIRT_CPU_ACCOUNTING_NATIVE && PPC_BOOK3S_64

diff --git a/arch/powerpc/include/asm/thread_info.h b/arch/powerpc/include/asm/thread_info.h
index 6ebca2996f18f8ca9e8ae2f89146f35d78394cb1..2785c7462ebf7b936b24b382c712cc9aedf769d5 100644 (file)
--- a/arch/powerpc/include/asm/thread_info.h
+++ b/arch/powerpc/include/asm/thread_info.h
@@ -103,6 +103,7 @@ void arch_setup_new_exec(void);
 #define TIF_PATCH_PENDING      6       /* pending live patching update */
 #define TIF_SYSCALL_AUDIT      7       /* syscall auditing active */
 #define TIF_SINGLESTEP         8       /* singlestepping active */
+#define TIF_NEED_RESCHED_LAZY  9       /* Scheduler driven lazy preemption */
 #define TIF_SECCOMP            10      /* secure computing */
 #define TIF_RESTOREALL         11      /* Restore all regs (implies NOERROR) */
 #define TIF_NOERROR            12      /* Force successful syscall return */
@@ -122,6 +123,7 @@ void arch_setup_new_exec(void);
 #define _TIF_SYSCALL_TRACE     (1<<TIF_SYSCALL_TRACE)
 #define _TIF_SIGPENDING                (1<<TIF_SIGPENDING)
 #define _TIF_NEED_RESCHED      (1<<TIF_NEED_RESCHED)
+#define _TIF_NEED_RESCHED_LAZY (1<<TIF_NEED_RESCHED_LAZY)
 #define _TIF_NOTIFY_SIGNAL     (1<<TIF_NOTIFY_SIGNAL)
 #define _TIF_POLLING_NRFLAG    (1<<TIF_POLLING_NRFLAG)
 #define _TIF_32BIT             (1<<TIF_32BIT)
@@ -142,9 +144,10 @@ void arch_setup_new_exec(void);
                                 _TIF_SYSCALL_EMU)
 
 #define _TIF_USER_WORK_MASK    (_TIF_SIGPENDING | _TIF_NEED_RESCHED | \
-                                _TIF_NOTIFY_RESUME | _TIF_UPROBE | \
-                                _TIF_RESTORE_TM | _TIF_PATCH_PENDING | \
-                                _TIF_NOTIFY_SIGNAL)
+                                _TIF_NEED_RESCHED_LAZY | _TIF_NOTIFY_RESUME | \
+                                _TIF_UPROBE | _TIF_RESTORE_TM | \
+                                _TIF_PATCH_PENDING | _TIF_NOTIFY_SIGNAL)
+
 #define _TIF_PERSYSCALL_MASK   (_TIF_RESTOREALL|_TIF_NOERROR)
 
 /* Bits in local_flags */

diff --git a/arch/powerpc/kernel/interrupt.c b/arch/powerpc/kernel/interrupt.c
index af62ec974b9702cc0b310290bc262e6924222267..8f4acc55407b0420875e9cf842196d65573adb5d 100644 (file)
--- a/arch/powerpc/kernel/interrupt.c
+++ b/arch/powerpc/kernel/interrupt.c
@@ -185,7 +185,7 @@ again:
        ti_flags = read_thread_flags();
        while (unlikely(ti_flags & (_TIF_USER_WORK_MASK & ~_TIF_RESTORE_TM))) {
                local_irq_enable();
-               if (ti_flags & _TIF_NEED_RESCHED) {
+               if (ti_flags & (_TIF_NEED_RESCHED | _TIF_NEED_RESCHED_LAZY)) {
                        schedule();
                } else {
                        /*
@@ -396,7 +396,7 @@ notrace unsigned long interrupt_exit_kernel_prepare(struct pt_regs *regs)
                /* Returning to a kernel context with local irqs enabled. */
                WARN_ON_ONCE(!(regs->msr & MSR_EE));
 again:
-               if (IS_ENABLED(CONFIG_PREEMPT)) {
+               if (IS_ENABLED(CONFIG_PREEMPTION)) {
                        /* Return to preemptible kernel context */
                        if (unlikely(read_thread_flags() & _TIF_NEED_RESCHED)) {
                                if (preempt_count() == 0)