arm64: head: move dynamic shadow call stack patching into early C runtime

Once we update the early kernel mapping code to only map the kernel once
with the right permissions, we can no longer perform code patching via
this mapping.

So move this code to an earlier stage of the boot, right after applying
the relocations.

Signed-off-by: Ard Biesheuvel <ardb@kernel.org>
Link: https://lore.kernel.org/r/20240214122845.2033971-54-ardb+git@google.com
Signed-off-by: Catalin Marinas <catalin.marinas@arm.com>

diff --git a/arch/arm64/include/asm/scs.h b/arch/arm64/include/asm/scs.h
index 3fdae5fe3142f625ffabd19ab26ea02637c06f8a..eca2ba5a6276dea683bfdce4c6e5c17713e0c7e3 100644 (file)
--- a/arch/arm64/include/asm/scs.h
+++ b/arch/arm64/include/asm/scs.h
@@ -72,8 +72,8 @@ static inline void dynamic_scs_init(void)
 static inline void dynamic_scs_init(void) {}
 #endif
 
-int scs_patch(const u8 eh_frame[], int size);
-asmlinkage void scs_patch_vmlinux(void);
+int __pi_scs_patch(const u8 eh_frame[], int size);
+asmlinkage void __pi_scs_patch_vmlinux(void);
 
 #endif /* __ASSEMBLY __ */
 

diff --git a/arch/arm64/kernel/Makefile b/arch/arm64/kernel/Makefile
index 4236f1e0fffa4f110e8a4fbd6d6cdf591917506a..14b4a179bad313ead93f620f6377a395ea79e535 100644 (file)
--- a/arch/arm64/kernel/Makefile
+++ b/arch/arm64/kernel/Makefile
@@ -71,14 +71,6 @@ obj-$(CONFIG_ARM64_PTR_AUTH)         += pointer_auth.o
 obj-$(CONFIG_ARM64_MTE)                        += mte.o
 obj-y                                  += vdso-wrap.o
 obj-$(CONFIG_COMPAT_VDSO)              += vdso32-wrap.o
-obj-$(CONFIG_UNWIND_PATCH_PAC_INTO_SCS)        += patch-scs.o
-
-# We need to prevent the SCS patching code from patching itself. Using
-# -mbranch-protection=none here to avoid the patchable PAC opcodes from being
-# generated triggers an issue with full LTO on Clang, which stops emitting PAC
-# instructions altogether. So disable LTO as well for the compilation unit.
-CFLAGS_patch-scs.o                     += -mbranch-protection=none
-CFLAGS_REMOVE_patch-scs.o              += $(CC_FLAGS_LTO)
 
 # Force dependency (vdso*-wrap.S includes vdso.so through incbin)
 $(obj)/vdso-wrap.o: $(obj)/vdso/vdso.so

diff --git a/arch/arm64/kernel/head.S b/arch/arm64/kernel/head.S
index 865ecc1f825571065b996fdf800c0a118f721d04..b320702032a7e0750807434ee997d133ccf3de62 100644 (file)
--- a/arch/arm64/kernel/head.S
+++ b/arch/arm64/kernel/head.S
@@ -490,9 +490,6 @@ SYM_FUNC_START_LOCAL(__primary_switched)
 #endif
 #if defined(CONFIG_KASAN_GENERIC) || defined(CONFIG_KASAN_SW_TAGS)
        bl      kasan_early_init
-#endif
-#ifdef CONFIG_UNWIND_PATCH_PAC_INTO_SCS
-       bl      scs_patch_vmlinux
 #endif
        mov     x0, x20
        bl      finalise_el2                    // Prefer VHE if possible
@@ -794,6 +791,11 @@ SYM_FUNC_START_LOCAL(__primary_switch)
 #ifdef CONFIG_RELOCATABLE
        mov     x0, x23
        bl      __pi_relocate_kernel
+#endif
+#ifdef CONFIG_UNWIND_PATCH_PAC_INTO_SCS
+       ldr     x0, =__eh_frame_start
+       ldr     x1, =__eh_frame_end
+       bl      __pi_scs_patch_vmlinux
 #endif
        ldr     x8, =__primary_switched
        adrp    x0, KERNEL_START                // __pa(KERNEL_START)

diff --git a/arch/arm64/kernel/module.c b/arch/arm64/kernel/module.c
index dd851297596e5e9372bf4f64f640283181e86395..47e0be610bb6a13f26997c73a46151006ecfbedc 100644 (file)
--- a/arch/arm64/kernel/module.c
+++ b/arch/arm64/kernel/module.c
@@ -595,7 +595,7 @@ int module_finalize(const Elf_Ehdr *hdr,
        if (scs_is_dynamic()) {
                s = find_section(hdr, sechdrs, ".init.eh_frame");
                if (s)
-                       scs_patch((void *)s->sh_addr, s->sh_size);
+                       __pi_scs_patch((void *)s->sh_addr, s->sh_size);
        }
 
        return module_init_ftrace_plt(hdr, sechdrs, me);

diff --git a/arch/arm64/kernel/pi/Makefile b/arch/arm64/kernel/pi/Makefile
index 7f6dfce893c3b88f69171d9ff1618fd197c3bb1b..a8b302245f15326ac601911053bf9ceefb5ce02d 100644 (file)
--- a/arch/arm64/kernel/pi/Makefile
+++ b/arch/arm64/kernel/pi/Makefile
@@ -38,7 +38,9 @@ $(obj)/lib-%.pi.o: OBJCOPYFLAGS += --prefix-alloc-sections=.init
 $(obj)/lib-%.o: $(srctree)/lib/%.c FORCE
        $(call if_changed_rule,cc_o_c)
 
-obj-y                          := idreg-override.pi.o lib-fdt.pi.o lib-fdt_ro.pi.o
-obj-$(CONFIG_RELOCATABLE)      += relocate.pi.o
-obj-$(CONFIG_RANDOMIZE_BASE)   += kaslr_early.pi.o
-extra-y                                := $(patsubst %.pi.o,%.o,$(obj-y))
+obj-y                                  := idreg-override.pi.o \
+                                          lib-fdt.pi.o lib-fdt_ro.pi.o
+obj-$(CONFIG_RELOCATABLE)              += relocate.pi.o
+obj-$(CONFIG_RANDOMIZE_BASE)           += kaslr_early.pi.o
+obj-$(CONFIG_UNWIND_PATCH_PAC_INTO_SCS)        += patch-scs.pi.o
+extra-y                                        := $(patsubst %.pi.o,%.o,$(obj-y))

diff --git a/arch/arm64/kernel/patch-scs.c b/arch/arm64/kernel/pi/patch-scs.c
similarity index 91%
rename from arch/arm64/kernel/patch-scs.c
rename to arch/arm64/kernel/pi/patch-scs.c
index a1fe4b4ff5917670d5e99c892a9efdc78be1b1d0..c65ef40d1e6b6b30a58099e577c7261a401f6f35 100644 (file)
--- a/arch/arm64/kernel/patch-scs.c
+++ b/arch/arm64/kernel/pi/patch-scs.c
@@ -4,14 +4,11 @@
  * Author: Ard Biesheuvel <ardb@google.com>
  */
 
-#include <linux/bug.h>
 #include <linux/errno.h>
 #include <linux/init.h>
 #include <linux/linkage.h>
-#include <linux/printk.h>
 #include <linux/types.h>
 
-#include <asm/cacheflush.h>
 #include <asm/scs.h>
 
 //
@@ -81,7 +78,11 @@ static void __always_inline scs_patch_loc(u64 loc)
                 */
                return;
        }
-       dcache_clean_pou(loc, loc + sizeof(u32));
+       if (IS_ENABLED(CONFIG_ARM64_WORKAROUND_CLEAN_CACHE))
+               asm("dc civac, %0" :: "r"(loc));
+       else
+               asm(ALTERNATIVE("dc cvau, %0", "nop", ARM64_HAS_CACHE_IDC)
+                   :: "r"(loc));
 }
 
 /*
@@ -128,10 +129,10 @@ struct eh_frame {
        };
 };
 
-static int noinstr scs_handle_fde_frame(const struct eh_frame *frame,
-                                       bool fde_has_augmentation_data,
-                                       int code_alignment_factor,
-                                       bool dry_run)
+static int scs_handle_fde_frame(const struct eh_frame *frame,
+                               bool fde_has_augmentation_data,
+                               int code_alignment_factor,
+                               bool dry_run)
 {
        int size = frame->size - offsetof(struct eh_frame, opcodes) + 4;
        u64 loc = (u64)offset_to_ptr(&frame->initial_loc);
@@ -198,14 +199,13 @@ static int noinstr scs_handle_fde_frame(const struct eh_frame *frame,
                        break;
 
                default:
-                       pr_err("unhandled opcode: %02x in FDE frame %lx\n", opcode[-1], (uintptr_t)frame);
                        return -ENOEXEC;
                }
        }
        return 0;
 }
 
-int noinstr scs_patch(const u8 eh_frame[], int size)
+int scs_patch(const u8 eh_frame[], int size)
 {
        const u8 *p = eh_frame;
 
@@ -251,12 +251,12 @@ int noinstr scs_patch(const u8 eh_frame[], int size)
        return 0;
 }
 
-asmlinkage void __init scs_patch_vmlinux(void)
+asmlinkage void __init scs_patch_vmlinux(const u8 start[], const u8 end[])
 {
        if (!should_patch_pac_into_scs())
                return;
 
-       WARN_ON(scs_patch(__eh_frame_start, __eh_frame_end - __eh_frame_start));
-       icache_inval_all_pou();
+       scs_patch(start, end - start);
+       asm("ic ialluis");
        isb();
 }