arm64: extable: Add fixup handling for uaccess CPY* instructions

A subsequent patch will use CPY* instructions to copy between user and
kernel memory. Add a new exception fixup type to avoid fixing up faults
on kernel memory accesses, in order to make it easier to debug kernel
bugs and to keep the same behavior as with regular loads/stores.

Signed-off-by: Kristina Martšenko <kristina.martsenko@arm.com>
Reviewed-by: Robin Murphy <robin.murphy@arm.com>
Link: https://lore.kernel.org/r/20250228170006.390100-2-kristina.martsenko@arm.com
Signed-off-by: Catalin Marinas <catalin.marinas@arm.com>

diff --git a/arch/arm64/include/asm/asm-extable.h b/arch/arm64/include/asm/asm-extable.h
index b8a5861dc7b77a937c8e05251890338e771f4dd2..292f2687a12e57ee74571bed475dedf9ee47228b 100644 (file)
--- a/arch/arm64/include/asm/asm-extable.h
+++ b/arch/arm64/include/asm/asm-extable.h
@@ -9,7 +9,8 @@
 #define EX_TYPE_BPF                    1
 #define EX_TYPE_UACCESS_ERR_ZERO       2
 #define EX_TYPE_KACCESS_ERR_ZERO       3
-#define EX_TYPE_LOAD_UNALIGNED_ZEROPAD 4
+#define EX_TYPE_UACCESS_CPY            4
+#define EX_TYPE_LOAD_UNALIGNED_ZEROPAD 5
 
 /* Data fields for EX_TYPE_UACCESS_ERR_ZERO */
 #define EX_DATA_REG_ERR_SHIFT  0
@@ -23,6 +24,9 @@
 #define EX_DATA_REG_ADDR_SHIFT 5
 #define EX_DATA_REG_ADDR       GENMASK(9, 5)
 
+/* Data fields for EX_TYPE_UACCESS_CPY */
+#define EX_DATA_UACCESS_WRITE  BIT(0)
+
 #ifdef __ASSEMBLY__
 
 #define __ASM_EXTABLE_RAW(insn, fixup, type, data)     \
@@ -69,6 +73,10 @@
        .endif
        .endm
 
+       .macro          _asm_extable_uaccess_cpy, insn, fixup, uaccess_is_write
+       __ASM_EXTABLE_RAW(\insn, \fixup, EX_TYPE_UACCESS_CPY, \uaccess_is_write)
+       .endm
+
 #else /* __ASSEMBLY__ */
 
 #include <linux/stringify.h>

diff --git a/arch/arm64/include/asm/extable.h b/arch/arm64/include/asm/extable.h
index 72b0e71cc3de88bbc3673d19feeb56e924ae2069..5892b8977710cfd08a85cac70a74a8c2bfa0d12f 100644 (file)
--- a/arch/arm64/include/asm/extable.h
+++ b/arch/arm64/include/asm/extable.h
@@ -45,5 +45,5 @@ bool ex_handler_bpf(const struct exception_table_entry *ex,
 }
 #endif /* !CONFIG_BPF_JIT */
 
-bool fixup_exception(struct pt_regs *regs);
+bool fixup_exception(struct pt_regs *regs, unsigned long esr);
 #endif

diff --git a/arch/arm64/mm/extable.c b/arch/arm64/mm/extable.c
index 228d681a871594f05acea36f572b31363f8000c6..afb5241e4d91cc2889f410d34f4c6d8928dc2af9 100644 (file)
--- a/arch/arm64/mm/extable.c
+++ b/arch/arm64/mm/extable.c
@@ -8,8 +8,18 @@
 #include <linux/uaccess.h>
 
 #include <asm/asm-extable.h>
+#include <asm/esr.h>
 #include <asm/ptrace.h>
 
+static bool cpy_faulted_on_uaccess(const struct exception_table_entry *ex,
+                                  unsigned long esr)
+{
+       bool uaccess_is_write = FIELD_GET(EX_DATA_UACCESS_WRITE, ex->data);
+       bool fault_on_write = esr & ESR_ELx_WNR;
+
+       return uaccess_is_write == fault_on_write;
+}
+
 static inline unsigned long
 get_ex_fixup(const struct exception_table_entry *ex)
 {
@@ -29,6 +39,17 @@ static bool ex_handler_uaccess_err_zero(const struct exception_table_entry *ex,
        return true;
 }
 
+static bool ex_handler_uaccess_cpy(const struct exception_table_entry *ex,
+                                  struct pt_regs *regs, unsigned long esr)
+{
+       /* Do not fix up faults on kernel memory accesses */
+       if (!cpy_faulted_on_uaccess(ex, esr))
+               return false;
+
+       regs->pc = get_ex_fixup(ex);
+       return true;
+}
+
 static bool
 ex_handler_load_unaligned_zeropad(const struct exception_table_entry *ex,
                                  struct pt_regs *regs)
@@ -56,7 +77,7 @@ ex_handler_load_unaligned_zeropad(const struct exception_table_entry *ex,
        return true;
 }
 
-bool fixup_exception(struct pt_regs *regs)
+bool fixup_exception(struct pt_regs *regs, unsigned long esr)
 {
        const struct exception_table_entry *ex;
 
@@ -70,6 +91,8 @@ bool fixup_exception(struct pt_regs *regs)
        case EX_TYPE_UACCESS_ERR_ZERO:
        case EX_TYPE_KACCESS_ERR_ZERO:
                return ex_handler_uaccess_err_zero(ex, regs);
+       case EX_TYPE_UACCESS_CPY:
+               return ex_handler_uaccess_cpy(ex, regs, esr);
        case EX_TYPE_LOAD_UNALIGNED_ZEROPAD:
                return ex_handler_load_unaligned_zeropad(ex, regs);
        }

diff --git a/arch/arm64/mm/fault.c b/arch/arm64/mm/fault.c
index ef63651099a9dc20f3e56260161681c2a7a1b6c2..da4854fc6150ae375ef55a33d4f5194ba78e5217 100644 (file)
--- a/arch/arm64/mm/fault.c
+++ b/arch/arm64/mm/fault.c
@@ -375,7 +375,7 @@ static void __do_kernel_fault(unsigned long addr, unsigned long esr,
         * Are we prepared to handle this kernel fault?
         * We are almost certainly not prepared to handle instruction faults.
         */
-       if (!is_el1_instruction_abort(esr) && fixup_exception(regs))
+       if (!is_el1_instruction_abort(esr) && fixup_exception(regs, esr))
                return;
 
        if (WARN_RATELIMIT(is_spurious_el1_translation_fault(addr, esr, regs),