arm64: futex: Support futex with FEAT_LSUI

Current futex atomic operations are implemented using LL/SC instructions
while temporarily clearing PSTATE.PAN and setting PSTATE.TCO (if
KASAN_HW_TAGS is enabled). With Armv9.6, FEAT_LSUI provides atomic
instructions for user memory access in the kernel without the need for
PSTATE bits toggling.

Use the FEAT_LSUI instructions to implement the futex atomic operations.
Note that some futex operations do not have a matching LSUI instruction,
(eor or word-sized cmpxchg). For such cases, use cas{al}t to implement
the operation.

Signed-off-by: Yeoreum Yun <yeoreum.yun@arm.com>
[catalin.marinas@arm.com: add comment on -EAGAIN in __lsui_futex_cmpxchg()]
Signed-off-by: Catalin Marinas <catalin.marinas@arm.com>

diff --git a/arch/arm64/include/asm/futex.h b/arch/arm64/include/asm/futex.h
index 321242e40952cd47f31dd7c30bedbf506a74ba9f..d1d2ff9d323a41b8c20d967503f2eee3de2dea71 100644 (file)
--- a/arch/arm64/include/asm/futex.h
+++ b/arch/arm64/include/asm/futex.h
@@ -9,6 +9,7 @@
 #include <linux/uaccess.h>
 
 #include <asm/errno.h>
+#include <asm/lsui.h>
 
 #define FUTEX_MAX_LOOPS        128 /* What's the largest number you can think of? */
 
@@ -89,11 +90,166 @@ __llsc_futex_cmpxchg(u32 __user *uaddr, u32 oldval, u32 newval, u32 *oval)
        return ret;
 }
 
+#ifdef CONFIG_ARM64_LSUI
+
+/*
+ * Wrap LSUI instructions with uaccess_ttbr0_enable()/disable(), as
+ * PAN toggling is not required.
+ */
+
+#define LSUI_FUTEX_ATOMIC_OP(op, asm_op)                               \
+static __always_inline int                                             \
+__lsui_futex_atomic_##op(int oparg, u32 __user *uaddr, int *oval)      \
+{                                                                      \
+       int ret = 0;                                                    \
+       int oldval;                                                     \
+                                                                       \
+       uaccess_ttbr0_enable();                                         \
+                                                                       \
+       asm volatile("// __lsui_futex_atomic_" #op "\n"                 \
+       __LSUI_PREAMBLE                                                 \
+"1:    " #asm_op "al   %w[oparg], %w[oldval], %[uaddr]\n"              \
+"2:\n"                                                                 \
+       _ASM_EXTABLE_UACCESS_ERR(1b, 2b, %w[ret])                       \
+       : [ret] "+r" (ret), [uaddr] "+Q" (*uaddr),                      \
+         [oldval] "=r" (oldval)                                        \
+       : [oparg] "r" (oparg)                                           \
+       : "memory");                                                    \
+                                                                       \
+       uaccess_ttbr0_disable();                                        \
+                                                                       \
+       if (!ret)                                                       \
+               *oval = oldval;                                         \
+       return ret;                                                     \
+}
+
+LSUI_FUTEX_ATOMIC_OP(add, ldtadd)
+LSUI_FUTEX_ATOMIC_OP(or, ldtset)
+LSUI_FUTEX_ATOMIC_OP(andnot, ldtclr)
+LSUI_FUTEX_ATOMIC_OP(set, swpt)
+
+static __always_inline int
+__lsui_cmpxchg64(u64 __user *uaddr, u64 *oldval, u64 newval)
+{
+       int ret = 0;
+
+       uaccess_ttbr0_enable();
+
+       asm volatile("// __lsui_cmpxchg64\n"
+       __LSUI_PREAMBLE
+"1:    casalt  %[oldval], %[newval], %[uaddr]\n"
+"2:\n"
+       _ASM_EXTABLE_UACCESS_ERR(1b, 2b, %w[ret])
+       : [ret] "+r" (ret), [uaddr] "+Q" (*uaddr),
+         [oldval] "+r" (*oldval)
+       : [newval] "r" (newval)
+       : "memory");
+
+       uaccess_ttbr0_disable();
+
+       return ret;
+}
+
+static __always_inline int
+__lsui_cmpxchg32(u32 __user *uaddr, u32 oldval, u32 newval, u32 *oval)
+{
+       u64 __user *uaddr64;
+       bool futex_pos, other_pos;
+       u32 other, orig_other;
+       union {
+               u32 futex[2];
+               u64 raw;
+       } oval64, orig64, nval64;
+
+       uaddr64 = (u64 __user *)PTR_ALIGN_DOWN(uaddr, sizeof(u64));
+       futex_pos = !IS_ALIGNED((unsigned long)uaddr, sizeof(u64));
+       other_pos = !futex_pos;
+
+       oval64.futex[futex_pos] = oldval;
+       if (get_user(oval64.futex[other_pos], (u32 __user *)uaddr64 + other_pos))
+               return -EFAULT;
+
+       orig64.raw = oval64.raw;
+
+       nval64.futex[futex_pos] = newval;
+       nval64.futex[other_pos] = oval64.futex[other_pos];
+
+       if (__lsui_cmpxchg64(uaddr64, &oval64.raw, nval64.raw))
+               return -EFAULT;
+
+       oldval = oval64.futex[futex_pos];
+       other = oval64.futex[other_pos];
+       orig_other = orig64.futex[other_pos];
+
+       if (other != orig_other)
+               return -EAGAIN;
+
+       *oval = oldval;
+
+       return 0;
+}
+
+static __always_inline int
+__lsui_futex_atomic_and(int oparg, u32 __user *uaddr, int *oval)
+{
+       /*
+        * Undo the bitwise negation applied to the oparg passed from
+        * arch_futex_atomic_op_inuser() with FUTEX_OP_ANDN.
+        */
+       return __lsui_futex_atomic_andnot(~oparg, uaddr, oval);
+}
+
+static __always_inline int
+__lsui_futex_atomic_eor(int oparg, u32 __user *uaddr, int *oval)
+{
+       u32 oldval, newval, val;
+       int ret, i;
+
+       if (get_user(oldval, uaddr))
+               return -EFAULT;
+
+       /*
+        * there are no ldteor/stteor instructions...
+        */
+       for (i = 0; i < FUTEX_MAX_LOOPS; i++) {
+               newval = oldval ^ oparg;
+
+               ret = __lsui_cmpxchg32(uaddr, oldval, newval, &val);
+               switch (ret) {
+               case -EFAULT:
+                       return ret;
+               case -EAGAIN:
+                       continue;
+               }
+
+               if (val == oldval) {
+                       *oval = val;
+                       return 0;
+               }
+
+               oldval = val;
+       }
+
+       return -EAGAIN;
+}
+
+static __always_inline int
+__lsui_futex_cmpxchg(u32 __user *uaddr, u32 oldval, u32 newval, u32 *oval)
+{
+       /*
+        * Callers of futex_atomic_cmpxchg_inatomic() already retry on
+        * -EAGAIN, no need for another loop of max retries.
+        */
+       return __lsui_cmpxchg32(uaddr, oldval, newval, oval);
+}
+#endif /* CONFIG_ARM64_LSUI */
+
+
 #define FUTEX_ATOMIC_OP(op)                                            \
 static __always_inline int                                             \
 __futex_atomic_##op(int oparg, u32 __user *uaddr, int *oval)           \
 {                                                                      \
-       return __llsc_futex_atomic_##op(oparg, uaddr, oval);            \
+       return __lsui_llsc_body(futex_atomic_##op, oparg, uaddr, oval); \
 }
 
 FUTEX_ATOMIC_OP(add)
@@ -105,7 +261,7 @@ FUTEX_ATOMIC_OP(set)
 static __always_inline int
 __futex_cmpxchg(u32 __user *uaddr, u32 oldval, u32 newval, u32 *oval)
 {
-       return __llsc_futex_cmpxchg(uaddr, oldval, newval, oval);
+       return __lsui_llsc_body(futex_cmpxchg, uaddr, oldval, newval, oval);
 }
 
 static inline int

diff --git a/arch/arm64/include/asm/lsui.h b/arch/arm64/include/asm/lsui.h
new file mode 100644 (file)
index 0000000..8f0d819
--- /dev/null
+++ b/arch/arm64/include/asm/lsui.h
@@ -0,0 +1,27 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+#ifndef __ASM_LSUI_H
+#define __ASM_LSUI_H
+
+#include <linux/compiler_types.h>
+#include <linux/stringify.h>
+#include <asm/alternative.h>
+#include <asm/alternative-macros.h>
+#include <asm/cpucaps.h>
+
+#define __LSUI_PREAMBLE        ".arch_extension lsui\n"
+
+#ifdef CONFIG_ARM64_LSUI
+
+#define __lsui_llsc_body(op, ...)                                      \
+({                                                                     \
+       alternative_has_cap_unlikely(ARM64_HAS_LSUI) ?                  \
+               __lsui_##op(__VA_ARGS__) : __llsc_##op(__VA_ARGS__);    \
+})
+
+#else  /* CONFIG_ARM64_LSUI */
+
+#define __lsui_llsc_body(op, ...)      __llsc_##op(__VA_ARGS__)
+
+#endif /* CONFIG_ARM64_LSUI */
+
+#endif /* __ASM_LSUI_H */