riscv: Implement cmpxchg32/64() using Zacas

This adds runtime support for Zacas in cmpxchg operations.

Signed-off-by: Alexandre Ghiti <alexghiti@rivosinc.com>
Reviewed-by: Andrew Jones <ajones@ventanamicro.com>
Reviewed-by: Andrea Parri <parri.andrea@gmail.com>
Link: https://lore.kernel.org/r/20241103145153.105097-4-alexghiti@rivosinc.com
Signed-off-by: Palmer Dabbelt <palmer@rivosinc.com>

diff --git a/arch/riscv/Kconfig b/arch/riscv/Kconfig
index 62545946ecf432df5b41e235ba66438cd3743c06..3542efe3088b99edc638f3a74539abefaf25f767 100644 (file)
--- a/arch/riscv/Kconfig
+++ b/arch/riscv/Kconfig
@@ -632,6 +632,22 @@ config RISCV_ISA_ZAWRS
          use of these instructions in the kernel when the Zawrs extension is
          detected at boot.
 
+config TOOLCHAIN_HAS_ZACAS
+       bool
+       default y
+       depends on !64BIT || $(cc-option,-mabi=lp64 -march=rv64ima_zacas)
+       depends on !32BIT || $(cc-option,-mabi=ilp32 -march=rv32ima_zacas)
+       depends on AS_HAS_OPTION_ARCH
+
+config RISCV_ISA_ZACAS
+       bool "Zacas extension support for atomic CAS"
+       depends on TOOLCHAIN_HAS_ZACAS
+       depends on RISCV_ALTERNATIVE
+       default y
+       help
+         Enable the use of the Zacas ISA-extension to implement kernel atomic
+         cmpxchg operations when it is detected at boot.
+
          If you don't know what to do here, say Y.
 
 config TOOLCHAIN_HAS_ZBB

diff --git a/arch/riscv/Makefile b/arch/riscv/Makefile
index d469db9f46f426122a1d6a41861622171402763c..3700a15744130e1a3f319f3c999042a39b322f1b 100644 (file)
--- a/arch/riscv/Makefile
+++ b/arch/riscv/Makefile
@@ -82,6 +82,9 @@ else
 riscv-march-$(CONFIG_TOOLCHAIN_NEEDS_EXPLICIT_ZICSR_ZIFENCEI) := $(riscv-march-y)_zicsr_zifencei
 endif
 
+# Check if the toolchain supports Zacas
+riscv-march-$(CONFIG_TOOLCHAIN_HAS_ZACAS) := $(riscv-march-y)_zacas
+
 # Remove F,D,V from isa string for all. Keep extensions between "fd" and "v" by
 # matching non-v and non-multi-letter extensions out with the filter ([^v_]*)
 KBUILD_CFLAGS += -march=$(shell echo $(riscv-march-y) | sed -E 's/(rv32ima|rv64ima)fd([^v_]*)v?/\1\2/')

diff --git a/arch/riscv/include/asm/cmpxchg.h b/arch/riscv/include/asm/cmpxchg.h
index ac1d7df898ef4d8bda0394e533396dac6b098df1..39c1daf39f6af7fc1e433777a9ffa12bad41c7b6 100644 (file)
--- a/arch/riscv/include/asm/cmpxchg.h
+++ b/arch/riscv/include/asm/cmpxchg.h
@@ -12,6 +12,7 @@
 #include <asm/fence.h>
 #include <asm/hwcap.h>
 #include <asm/insn-def.h>
+#include <asm/cpufeature-macros.h>
 
 #define __arch_xchg_masked(sc_sfx, prepend, append, r, p, n)           \
 ({                                                                     \
@@ -137,24 +138,37 @@
        r = (__typeof__(*(p)))((__retx & __mask) >> __s);               \
 })
 
-#define __arch_cmpxchg(lr_sfx, sc_sfx, prepend, append, r, p, co, o, n)        \
+#define __arch_cmpxchg(lr_sfx, sc_cas_sfx, prepend, append, r, p, co, o, n)    \
 ({                                                                     \
-       register unsigned int __rc;                                     \
+       if (IS_ENABLED(CONFIG_RISCV_ISA_ZACAS) &&                       \
+           riscv_has_extension_unlikely(RISCV_ISA_EXT_ZACAS)) {        \
+               r = o;                                                  \
                                                                        \
-       __asm__ __volatile__ (                                          \
-               prepend                                                 \
-               "0:     lr" lr_sfx " %0, %2\n"                          \
-               "       bne  %0, %z3, 1f\n"                             \
-               "       sc" sc_sfx " %1, %z4, %2\n"                     \
-               "       bnez %1, 0b\n"                                  \
-               append                                                  \
-               "1:\n"                                                  \
-               : "=&r" (r), "=&r" (__rc), "+A" (*(p))                  \
-               : "rJ" (co o), "rJ" (n)                                 \
-               : "memory");                                            \
+               __asm__ __volatile__ (                                  \
+                       prepend                                         \
+                       "       amocas" sc_cas_sfx " %0, %z2, %1\n"     \
+                       append                                          \
+                       : "+&r" (r), "+A" (*(p))                        \
+                       : "rJ" (n)                                      \
+                       : "memory");                                    \
+       } else {                                                        \
+               register unsigned int __rc;                             \
+                                                                       \
+               __asm__ __volatile__ (                                  \
+                       prepend                                         \
+                       "0:     lr" lr_sfx " %0, %2\n"                  \
+                       "       bne  %0, %z3, 1f\n"                     \
+                       "       sc" sc_cas_sfx " %1, %z4, %2\n"         \
+                       "       bnez %1, 0b\n"                          \
+                       append                                          \
+                       "1:\n"                                          \
+                       : "=&r" (r), "=&r" (__rc), "+A" (*(p))          \
+                       : "rJ" (co o), "rJ" (n)                         \
+                       : "memory");                                    \
+       }                                                               \
 })
 
-#define _arch_cmpxchg(ptr, old, new, sc_sfx, prepend, append)          \
+#define _arch_cmpxchg(ptr, old, new, sc_cas_sfx, prepend, append)      \
 ({                                                                     \
        __typeof__(ptr) __ptr = (ptr);                                  \
        __typeof__(*(__ptr)) __old = (old);                             \
@@ -164,15 +178,15 @@
        switch (sizeof(*__ptr)) {                                       \
        case 1:                                                         \
        case 2:                                                         \
-               __arch_cmpxchg_masked(sc_sfx, prepend, append,          \
+               __arch_cmpxchg_masked(sc_cas_sfx, prepend, append,      \
                                        __ret, __ptr, __old, __new);    \
                break;                                                  \
        case 4:                                                         \
-               __arch_cmpxchg(".w", ".w" sc_sfx, prepend, append,      \
+               __arch_cmpxchg(".w", ".w" sc_cas_sfx, prepend, append,  \
                                __ret, __ptr, (long), __old, __new);    \
                break;                                                  \
        case 8:                                                         \
-               __arch_cmpxchg(".d", ".d" sc_sfx, prepend, append,      \
+               __arch_cmpxchg(".d", ".d" sc_cas_sfx, prepend, append,  \
                                __ret, __ptr, /**/, __old, __new);      \
                break;                                                  \
        default:                                                        \