riscv: Add support for Zicbop

Zicbop introduces cache blocks prefetching instructions, add the
necessary support for the kernel to use it in the coming commits.

Co-developed-by: Guo Ren <guoren@kernel.org>
Signed-off-by: Guo Ren <guoren@kernel.org>
Tested-by: Andrea Parri <parri.andrea@gmail.com>
Link: https://lore.kernel.org/r/20250421142441.395849-3-alexghiti@rivosinc.com
Signed-off-by: Alexandre Ghiti <alexghiti@rivosinc.com>
Signed-off-by: Palmer Dabbelt <palmer@dabbelt.com>

diff --git a/arch/riscv/Kconfig b/arch/riscv/Kconfig
index bbec87b7930999748b33df7a92aecdefcdfe624b..28765ce563dec9551d992c3a3666c02902042f69 100644 (file)
--- a/arch/riscv/Kconfig
+++ b/arch/riscv/Kconfig
@@ -842,6 +842,21 @@ config RISCV_ISA_ZICBOZ
 
           If you don't know what to do here, say Y.
 
+config RISCV_ISA_ZICBOP
+       bool "Zicbop extension support for cache block prefetch"
+       depends on MMU
+       depends on RISCV_ALTERNATIVE
+       default y
+       help
+         Adds support to dynamically detect the presence of the ZICBOP
+         extension (Cache Block Prefetch Operations) and enable its
+         usage.
+
+         The Zicbop extension can be used to prefetch cache blocks for
+         read/write fetch.
+
+         If you don't know what to do here, say Y.
+
 config TOOLCHAIN_NEEDS_EXPLICIT_ZICSR_ZIFENCEI
        def_bool y
        # https://sourceware.org/git/?p=binutils-gdb.git;a=commit;h=aed44286efa8ae8717a77d94b51ac3614e2ca6dc

diff --git a/arch/riscv/include/asm/barrier.h b/arch/riscv/include/asm/barrier.h
index e1d9bf1deca6852e962816af066c13f29c734231..b8c5726d86acb1e2e5c6b9fb63bcdf305b01b4f9 100644 (file)
--- a/arch/riscv/include/asm/barrier.h
+++ b/arch/riscv/include/asm/barrier.h
@@ -14,11 +14,6 @@
 #include <asm/cmpxchg.h>
 #include <asm/fence.h>
 
-#define nop()          __asm__ __volatile__ ("nop")
-#define __nops(n)      ".rept  " #n "\nnop\n.endr\n"
-#define nops(n)                __asm__ __volatile__ (__nops(n))
-
-
 /* These barriers need to enforce ordering on both devices or memory. */
 #define __mb()         RISCV_FENCE(iorw, iorw)
 #define __rmb()                RISCV_FENCE(ir, ir)

diff --git a/arch/riscv/include/asm/cacheflush.h b/arch/riscv/include/asm/cacheflush.h
index 8de73f91bfa3716603e181d59d51fcd9316d97cc..effa02c2e682faa9f56b5d466588dcc4e04c5dee 100644 (file)
--- a/arch/riscv/include/asm/cacheflush.h
+++ b/arch/riscv/include/asm/cacheflush.h
@@ -80,6 +80,7 @@ void flush_icache_mm(struct mm_struct *mm, bool local);
 
 extern unsigned int riscv_cbom_block_size;
 extern unsigned int riscv_cboz_block_size;
+extern unsigned int riscv_cbop_block_size;
 void riscv_init_cbo_blocksizes(void);
 
 #ifdef CONFIG_RISCV_DMA_NONCOHERENT

diff --git a/arch/riscv/include/asm/hwcap.h b/arch/riscv/include/asm/hwcap.h
index e3cbf203cdde7fd6803671e126d272adff10e056..affd63e11b0a344c33a73647351ac02a94e42981 100644 (file)
--- a/arch/riscv/include/asm/hwcap.h
+++ b/arch/riscv/include/asm/hwcap.h
@@ -105,6 +105,7 @@
 #define RISCV_ISA_EXT_ZVFBFWMA         96
 #define RISCV_ISA_EXT_ZAAMO            97
 #define RISCV_ISA_EXT_ZALRSC           98
+#define RISCV_ISA_EXT_ZICBOP           99
 
 #define RISCV_ISA_EXT_XLINUXENVCFG     127
 

diff --git a/arch/riscv/include/asm/insn-def.h b/arch/riscv/include/asm/insn-def.h
index 02c92c1657d2e46616fc731dc7076164f1d6bcdf..d5adbaec1d010d26c8c41a5b48c682d9efa3eb75 100644 (file)
--- a/arch/riscv/include/asm/insn-def.h
+++ b/arch/riscv/include/asm/insn-def.h
@@ -263,4 +263,10 @@
 
 #define RISCV_INSN_NOP4        _AC(0x00000013, U)
 
+#ifndef __ASSEMBLY__
+#define nop()           __asm__ __volatile__ ("nop")
+#define __nops(n)       ".rept  " #n "\nnop\n.endr\n"
+#define nops(n)         __asm__ __volatile__ (__nops(n))
+#endif
+
 #endif /* __ASM_INSN_DEF_H */

diff --git a/arch/riscv/include/asm/processor.h b/arch/riscv/include/asm/processor.h
index 5f56eb9d114a95fc1c1ecbe2de30abea5aebf73b..09d4c963399ac7233dd30c92650a8049cc8bdfe2 100644 (file)
--- a/arch/riscv/include/asm/processor.h
+++ b/arch/riscv/include/asm/processor.h
@@ -52,7 +52,6 @@
 #endif
 
 #ifndef __ASSEMBLY__
-#include <linux/cpumask.h>
 
 struct task_struct;
 struct pt_regs;

diff --git a/arch/riscv/kernel/cpufeature.c b/arch/riscv/kernel/cpufeature.c
index 2054f6c4b0ae1fd27a160e64a452cd755d7852c0..743d53415572e071fb22851161bd079ef3158b7c 100644 (file)
--- a/arch/riscv/kernel/cpufeature.c
+++ b/arch/riscv/kernel/cpufeature.c
@@ -32,6 +32,7 @@
 #define NUM_ALPHA_EXTS ('z' - 'a' + 1)
 
 static bool any_cpu_has_zicboz;
+static bool any_cpu_has_zicbop;
 static bool any_cpu_has_zicbom;
 
 unsigned long elf_hwcap __read_mostly;
@@ -119,6 +120,21 @@ static int riscv_ext_zicboz_validate(const struct riscv_isa_ext_data *data,
        return 0;
 }
 
+static int riscv_ext_zicbop_validate(const struct riscv_isa_ext_data *data,
+                                    const unsigned long *isa_bitmap)
+{
+       if (!riscv_cbop_block_size) {
+               pr_err("Zicbop detected in ISA string, disabling as no cbop-block-size found\n");
+               return -EINVAL;
+       }
+       if (!is_power_of_2(riscv_cbop_block_size)) {
+               pr_err("Zicbop disabled as cbop-block-size present, but is not a power-of-2\n");
+               return -EINVAL;
+       }
+       any_cpu_has_zicbop = true;
+       return 0;
+}
+
 static int riscv_ext_f_validate(const struct riscv_isa_ext_data *data,
                                const unsigned long *isa_bitmap)
 {
@@ -442,6 +458,7 @@ const struct riscv_isa_ext_data riscv_isa_ext[] = {
        __RISCV_ISA_EXT_SUPERSET_VALIDATE(v, RISCV_ISA_EXT_v, riscv_v_exts, riscv_ext_vector_float_validate),
        __RISCV_ISA_EXT_DATA(h, RISCV_ISA_EXT_h),
        __RISCV_ISA_EXT_SUPERSET_VALIDATE(zicbom, RISCV_ISA_EXT_ZICBOM, riscv_xlinuxenvcfg_exts, riscv_ext_zicbom_validate),
+       __RISCV_ISA_EXT_DATA_VALIDATE(zicbop, RISCV_ISA_EXT_ZICBOP, riscv_ext_zicbop_validate),
        __RISCV_ISA_EXT_SUPERSET_VALIDATE(zicboz, RISCV_ISA_EXT_ZICBOZ, riscv_xlinuxenvcfg_exts, riscv_ext_zicboz_validate),
        __RISCV_ISA_EXT_DATA(ziccrse, RISCV_ISA_EXT_ZICCRSE),
        __RISCV_ISA_EXT_DATA(zicntr, RISCV_ISA_EXT_ZICNTR),
@@ -1112,6 +1129,10 @@ void __init riscv_user_isa_enable(void)
                current->thread.envcfg |= ENVCFG_CBCFE;
        else if (any_cpu_has_zicbom)
                pr_warn("Zicbom disabled as it is unavailable on some harts\n");
+
+       if (!riscv_has_extension_unlikely(RISCV_ISA_EXT_ZICBOP) &&
+           any_cpu_has_zicbop)
+               pr_warn("Zicbop disabled as it is unavailable on some harts\n");
 }
 
 #ifdef CONFIG_RISCV_ALTERNATIVE

diff --git a/arch/riscv/mm/cacheflush.c b/arch/riscv/mm/cacheflush.c
index b8167272988723ecdda97a6aa528aab7cd55eada..6265052ef8b6bb9bfa24e80d39303e847dd10231 100644 (file)
--- a/arch/riscv/mm/cacheflush.c
+++ b/arch/riscv/mm/cacheflush.c
@@ -101,6 +101,9 @@ EXPORT_SYMBOL_GPL(riscv_cbom_block_size);
 unsigned int riscv_cboz_block_size;
 EXPORT_SYMBOL_GPL(riscv_cboz_block_size);
 
+unsigned int riscv_cbop_block_size;
+EXPORT_SYMBOL_GPL(riscv_cbop_block_size);
+
 static void __init cbo_get_block_size(struct device_node *node,
                                      const char *name, u32 *block_size,
                                      unsigned long *first_hartid)
@@ -125,8 +128,8 @@ static void __init cbo_get_block_size(struct device_node *node,
 
 void __init riscv_init_cbo_blocksizes(void)
 {
-       unsigned long cbom_hartid, cboz_hartid;
-       u32 cbom_block_size = 0, cboz_block_size = 0;
+       unsigned long cbom_hartid, cboz_hartid, cbop_hartid;
+       u32 cbom_block_size = 0, cboz_block_size = 0, cbop_block_size = 0;
        struct device_node *node;
        struct acpi_table_header *rhct;
        acpi_status status;
@@ -138,13 +141,15 @@ void __init riscv_init_cbo_blocksizes(void)
                                           &cbom_block_size, &cbom_hartid);
                        cbo_get_block_size(node, "riscv,cboz-block-size",
                                           &cboz_block_size, &cboz_hartid);
+                       cbo_get_block_size(node, "riscv,cbop-block-size",
+                                          &cbop_block_size, &cbop_hartid);
                }
        } else {
                status = acpi_get_table(ACPI_SIG_RHCT, 0, &rhct);
                if (ACPI_FAILURE(status))
                        return;
 
-               acpi_get_cbo_block_size(rhct, &cbom_block_size, &cboz_block_size, NULL);
+               acpi_get_cbo_block_size(rhct, &cbom_block_size, &cboz_block_size, &cbop_block_size);
                acpi_put_table((struct acpi_table_header *)rhct);
        }
 
@@ -153,6 +158,9 @@ void __init riscv_init_cbo_blocksizes(void)
 
        if (cboz_block_size)
                riscv_cboz_block_size = cboz_block_size;
+
+       if (cbop_block_size)
+               riscv_cbop_block_size = cbop_block_size;
 }
 
 #ifdef CONFIG_SMP