LoongArch: Add 128-bit atomic cmpxchg support

Implement 128-bit atomic compare-and-exchange using LoongArch's
LL.D/SC.Q instructions.

At the same time, this fix the BPF scheduler test failures (scx_central
and scx_qmap) caused by kmalloc_nolock_noprof() returning NULL, due to
missing 128-bit atomics. The NULL returns lead to -ENOMEM errors during
scheduler initialization, causing test cases to fail.

Verified by testing with the scx_qmap scheduler
(located in tools/sched_ext/).

Building with `make` and running ./tools/sched_ext/build/bin/scx_qmap.

Link: https://git.kernel.org/pub/scm/linux/kernel/git/bpf/bpf.git/commit/?id=5fb750e8a9ae
Acked-by: Hengqi Chen <hengqi.chen@gmail.com>
Tested-by: Hengqi Chen <hengqi.chen@gmail.com>
Co-developed-by:: Xi Ruoyao <xry111@xry111.site>
Signed-off-by: Xi Ruoyao <xry111@xry111.site>
Signed-off-by: George Guo <guodongtai@kylinos.cn>
Signed-off-by: Huacai Chen <chenhuacai@loongson.cn>

diff --git a/arch/loongarch/Kconfig b/arch/loongarch/Kconfig
index ae9d954aa0525544811b83b24e6825b850cdbfd4..bf5a5beab366c69989755f7d6c4c8ee8bc7b60cb 100644 (file)
--- a/arch/loongarch/Kconfig
+++ b/arch/loongarch/Kconfig
@@ -114,6 +114,7 @@ config LOONGARCH
        select GENERIC_TIME_VSYSCALL
        select GPIOLIB
        select HAS_IOPORT
+       select HAVE_ALIGNED_STRUCT_PAGE
        select HAVE_ARCH_AUDITSYSCALL
        select HAVE_ARCH_BITREVERSE
        select HAVE_ARCH_JUMP_LABEL
@@ -130,6 +131,7 @@ config LOONGARCH
        select HAVE_ARCH_TRANSPARENT_HUGEPAGE
        select HAVE_ARCH_USERFAULTFD_MINOR if USERFAULTFD
        select HAVE_ASM_MODVERSIONS
+       select HAVE_CMPXCHG_DOUBLE
        select HAVE_CMPXCHG_LOCAL
        select HAVE_CONTEXT_TRACKING_USER
        select HAVE_C_RECORDMCOUNT

diff --git a/arch/loongarch/include/asm/cmpxchg.h b/arch/loongarch/include/asm/cmpxchg.h
index 0494c2ab553e95d77c17b501f7b062b026c9acaa..58cabab6d90d1a7f27beba820625fef6f695bcea 100644 (file)
--- a/arch/loongarch/include/asm/cmpxchg.h
+++ b/arch/loongarch/include/asm/cmpxchg.h
@@ -8,6 +8,7 @@
 #include <linux/bits.h>
 #include <linux/build_bug.h>
 #include <asm/barrier.h>
+#include <asm/cpu-features.h>
 
 #define __xchg_amo_asm(amswap_db, m, val)      \
 ({                                             \
@@ -236,6 +237,59 @@ __cmpxchg(volatile void *ptr, unsigned long old, unsigned long new, unsigned int
        BUILD_BUG_ON(sizeof(*(ptr)) != 8);                              \
        arch_cmpxchg((ptr), (o), (n));                                  \
   })
+
+union __u128_halves {
+       u128 full;
+       struct {
+               u64 low;
+               u64 high;
+       };
+};
+
+#define system_has_cmpxchg128()        cpu_opt(LOONGARCH_CPU_SCQ)
+
+#define __arch_cmpxchg128(ptr, old, new, llsc_mb)                      \
+({                                                                     \
+       union __u128_halves __old, __new, __ret;                        \
+       volatile u64 *__ptr = (volatile u64 *)(ptr);                    \
+                                                                       \
+       __old.full = (old);                                             \
+       __new.full = (new);                                             \
+                                                                       \
+       __asm__ __volatile__(                                           \
+       "1:   ll.d    %0, %3            # 128-bit cmpxchg low   \n"     \
+       llsc_mb                                                         \
+       "     ld.d    %1, %4            # 128-bit cmpxchg high  \n"     \
+       "     move    $t0, %0                                   \n"     \
+       "     move    $t1, %1                                   \n"     \
+       "     bne     %0, %z5, 2f                               \n"     \
+       "     bne     %1, %z6, 2f                               \n"     \
+       "     move    $t0, %z7                                  \n"     \
+       "     move    $t1, %z8                                  \n"     \
+       "2:   sc.q    $t0, $t1, %2                              \n"     \
+       "     beqz    $t0, 1b                                   \n"     \
+       llsc_mb                                                         \
+       : "=&r" (__ret.low), "=&r" (__ret.high)                         \
+       : "r" (__ptr),                                                  \
+         "ZC" (__ptr[0]), "m" (__ptr[1]),                              \
+         "Jr" (__old.low), "Jr" (__old.high),                          \
+         "Jr" (__new.low), "Jr" (__new.high)                           \
+       : "t0", "t1", "memory");                                        \
+                                                                       \
+       __ret.full;                                                     \
+})
+
+#define arch_cmpxchg128(ptr, o, n)                                     \
+({                                                                     \
+       BUILD_BUG_ON(sizeof(*(ptr)) != 16);                             \
+       __arch_cmpxchg128(ptr, o, n, __WEAK_LLSC_MB);                   \
+})
+
+#define arch_cmpxchg128_local(ptr, o, n)                               \
+({                                                                     \
+       BUILD_BUG_ON(sizeof(*(ptr)) != 16);                             \
+       __arch_cmpxchg128(ptr, o, n, "");                               \
+})
 #else
 #include <asm-generic/cmpxchg-local.h>
 #define arch_cmpxchg64_local(ptr, o, n) __generic_cmpxchg64_local((ptr), (o), (n))