From: Greg Kroah-Hartman Date: Fri, 20 Nov 2020 07:19:14 +0000 (+0100) Subject: 5.4-stable patches X-Git-Tag: v4.4.245~23 X-Git-Url: http://git.ipfire.org/gitweb.cgi?a=commitdiff_plain;h=4f4f91b3b675e06cc9954775162d9ebc72d0205a;p=thirdparty%2Fkernel%2Fstable-queue.git 5.4-stable patches added patches: powerpc-64s-flush-l1d-after-user-accesses.patch powerpc-64s-flush-l1d-on-kernel-entry.patch powerpc-only-include-kup-radix.h-for-64-bit-book3s.patch selftests-powerpc-entry-flush-test.patch selftests-powerpc-rfi_flush-disable-entry-flush-if-present.patch --- diff --git a/queue-5.4/powerpc-64s-flush-l1d-after-user-accesses.patch b/queue-5.4/powerpc-64s-flush-l1d-after-user-accesses.patch new file mode 100644 index 00000000000..142c55c302a --- /dev/null +++ b/queue-5.4/powerpc-64s-flush-l1d-after-user-accesses.patch @@ -0,0 +1,543 @@ +From foo@baz Fri Nov 20 08:16:31 AM CET 2020 +From: Daniel Axtens +Date: Fri, 20 Nov 2020 10:35:14 +1100 +Subject: powerpc/64s: flush L1D after user accesses +To: stable@vger.kernel.org +Cc: dja@axtens.net +Message-ID: <20201119233516.368194-4-dja@axtens.net> + +From: Nicholas Piggin + +commit 9a32a7e78bd0cd9a9b6332cbdc345ee5ffd0c5de upstream. + +IBM Power9 processors can speculatively operate on data in the L1 cache +before it has been completely validated, via a way-prediction mechanism. It +is not possible for an attacker to determine the contents of impermissible +memory using this method, since these systems implement a combination of +hardware and software security measures to prevent scenarios where +protected data could be leaked. + +However these measures don't address the scenario where an attacker induces +the operating system to speculatively execute instructions using data that +the attacker controls. This can be used for example to speculatively bypass +"kernel user access prevention" techniques, as discovered by Anthony +Steinhauser of Google's Safeside Project. This is not an attack by itself, +but there is a possibility it could be used in conjunction with +side-channels or other weaknesses in the privileged code to construct an +attack. + +This issue can be mitigated by flushing the L1 cache between privilege +boundaries of concern. This patch flushes the L1 cache after user accesses. + +This is part of the fix for CVE-2020-4788. + +Signed-off-by: Nicholas Piggin +Signed-off-by: Daniel Axtens +Signed-off-by: Greg Kroah-Hartman +--- + Documentation/admin-guide/kernel-parameters.txt | 4 + + arch/powerpc/include/asm/book3s/64/kup-radix.h | 24 ++++--- + arch/powerpc/include/asm/exception-64s.h | 3 + arch/powerpc/include/asm/feature-fixups.h | 9 ++ + arch/powerpc/include/asm/kup.h | 17 +++-- + arch/powerpc/include/asm/security_features.h | 3 + arch/powerpc/include/asm/setup.h | 1 + arch/powerpc/kernel/exceptions-64s.S | 81 +++++++----------------- + arch/powerpc/kernel/setup_64.c | 62 ++++++++++++++++++ + arch/powerpc/kernel/vmlinux.lds.S | 7 ++ + arch/powerpc/lib/feature-fixups.c | 50 ++++++++++++++ + arch/powerpc/platforms/powernv/setup.c | 4 + + arch/powerpc/platforms/pseries/setup.c | 4 + + 13 files changed, 199 insertions(+), 70 deletions(-) + +--- a/Documentation/admin-guide/kernel-parameters.txt ++++ b/Documentation/admin-guide/kernel-parameters.txt +@@ -2668,6 +2668,7 @@ + tsx_async_abort=off [X86] + kvm.nx_huge_pages=off [X86] + no_entry_flush [PPC] ++ no_uaccess_flush [PPC] + + Exceptions: + This does not have any effect on +@@ -3041,6 +3042,9 @@ + nospec_store_bypass_disable + [HW] Disable all mitigations for the Speculative Store Bypass vulnerability + ++ no_uaccess_flush ++ [PPC] Don't flush the L1-D cache after accessing user data. ++ + noxsave [BUGS=X86] Disables x86 extended register state save + and restore using xsave. The kernel will fallback to + enabling legacy floating-point and sse state. +--- a/arch/powerpc/include/asm/book3s/64/kup-radix.h ++++ b/arch/powerpc/include/asm/book3s/64/kup-radix.h +@@ -54,6 +54,8 @@ + + #else /* !__ASSEMBLY__ */ + ++DECLARE_STATIC_KEY_FALSE(uaccess_flush_key); ++ + #ifdef CONFIG_PPC_KUAP + + #include +@@ -77,6 +79,17 @@ static inline void set_kuap(unsigned lon + isync(); + } + ++static inline bool ++bad_kuap_fault(struct pt_regs *regs, unsigned long address, bool is_write) ++{ ++ return WARN(mmu_has_feature(MMU_FTR_RADIX_KUAP) && ++ (regs->kuap & (is_write ? AMR_KUAP_BLOCK_WRITE : AMR_KUAP_BLOCK_READ)), ++ "Bug: %s fault blocked by AMR!", is_write ? "Write" : "Read"); ++} ++#else /* CONFIG_PPC_KUAP */ ++static inline void set_kuap(unsigned long value) { } ++#endif /* !CONFIG_PPC_KUAP */ ++ + static __always_inline void allow_user_access(void __user *to, const void __user *from, + unsigned long size, unsigned long dir) + { +@@ -94,17 +107,10 @@ static inline void prevent_user_access(v + unsigned long size, unsigned long dir) + { + set_kuap(AMR_KUAP_BLOCKED); ++ if (static_branch_unlikely(&uaccess_flush_key)) ++ do_uaccess_flush(); + } + +-static inline bool +-bad_kuap_fault(struct pt_regs *regs, unsigned long address, bool is_write) +-{ +- return WARN(mmu_has_feature(MMU_FTR_RADIX_KUAP) && +- (regs->kuap & (is_write ? AMR_KUAP_BLOCK_WRITE : AMR_KUAP_BLOCK_READ)), +- "Bug: %s fault blocked by AMR!", is_write ? "Write" : "Read"); +-} +-#endif /* CONFIG_PPC_KUAP */ +- + #endif /* __ASSEMBLY__ */ + + #endif /* _ASM_POWERPC_BOOK3S_64_KUP_RADIX_H */ +--- a/arch/powerpc/include/asm/exception-64s.h ++++ b/arch/powerpc/include/asm/exception-64s.h +@@ -134,6 +134,9 @@ + hrfid; \ + b hrfi_flush_fallback + ++#else /* __ASSEMBLY__ */ ++/* Prototype for function defined in exceptions-64s.S */ ++void do_uaccess_flush(void); + #endif /* __ASSEMBLY__ */ + + #endif /* _ASM_POWERPC_EXCEPTION_H */ +--- a/arch/powerpc/include/asm/feature-fixups.h ++++ b/arch/powerpc/include/asm/feature-fixups.h +@@ -205,6 +205,14 @@ label##3: \ + FTR_ENTRY_OFFSET 955b-956b; \ + .popsection; + ++#define UACCESS_FLUSH_FIXUP_SECTION \ ++959: \ ++ .pushsection __uaccess_flush_fixup,"a"; \ ++ .align 2; \ ++960: \ ++ FTR_ENTRY_OFFSET 959b-960b; \ ++ .popsection; ++ + #define ENTRY_FLUSH_FIXUP_SECTION \ + 957: \ + .pushsection __entry_flush_fixup,"a"; \ +@@ -248,6 +256,7 @@ extern long stf_barrier_fallback; + extern long entry_flush_fallback; + extern long __start___stf_entry_barrier_fixup, __stop___stf_entry_barrier_fixup; + extern long __start___stf_exit_barrier_fixup, __stop___stf_exit_barrier_fixup; ++extern long __start___uaccess_flush_fixup, __stop___uaccess_flush_fixup; + extern long __start___entry_flush_fixup, __stop___entry_flush_fixup; + extern long __start___rfi_flush_fixup, __stop___rfi_flush_fixup; + extern long __start___barrier_nospec_fixup, __stop___barrier_nospec_fixup; +--- a/arch/powerpc/include/asm/kup.h ++++ b/arch/powerpc/include/asm/kup.h +@@ -45,15 +45,24 @@ static inline void setup_kuep(bool disab + void setup_kuap(bool disabled); + #else + static inline void setup_kuap(bool disabled) { } +-static inline void allow_user_access(void __user *to, const void __user *from, +- unsigned long size, unsigned long dir) { } +-static inline void prevent_user_access(void __user *to, const void __user *from, +- unsigned long size, unsigned long dir) { } ++ + static inline bool + bad_kuap_fault(struct pt_regs *regs, unsigned long address, bool is_write) + { + return false; + } ++ ++/* ++ * book3s/64/kup-radix.h defines these functions for the !KUAP case to flush ++ * the L1D cache after user accesses. Only include the empty stubs for other ++ * platforms. ++ */ ++#ifndef CONFIG_PPC64 ++static inline void allow_user_access(void __user *to, const void __user *from, ++ unsigned long size, unsigned long dir) { } ++static inline void prevent_user_access(void __user *to, const void __user *from, ++ unsigned long size, unsigned long dir) { } ++#endif /* CONFIG_PPC64 */ + #endif /* CONFIG_PPC_KUAP */ + + static inline void allow_read_from_user(const void __user *from, unsigned long size) +--- a/arch/powerpc/include/asm/security_features.h ++++ b/arch/powerpc/include/asm/security_features.h +@@ -87,6 +87,8 @@ static inline bool security_ftr_enabled( + // The L1-D cache should be flushed when entering the kernel + #define SEC_FTR_L1D_FLUSH_ENTRY 0x0000000000004000ull + ++// The L1-D cache should be flushed after user accesses from the kernel ++#define SEC_FTR_L1D_FLUSH_UACCESS 0x0000000000008000ull + + // Features enabled by default + #define SEC_FTR_DEFAULT \ +@@ -94,6 +96,7 @@ static inline bool security_ftr_enabled( + SEC_FTR_L1D_FLUSH_PR | \ + SEC_FTR_BNDS_CHK_SPEC_BAR | \ + SEC_FTR_L1D_FLUSH_ENTRY | \ ++ SEC_FTR_L1D_FLUSH_UACCESS | \ + SEC_FTR_FAVOUR_SECURITY) + + #endif /* _ASM_POWERPC_SECURITY_FEATURES_H */ +--- a/arch/powerpc/include/asm/setup.h ++++ b/arch/powerpc/include/asm/setup.h +@@ -60,6 +60,7 @@ void setup_barrier_nospec(void); + #else + static inline void setup_barrier_nospec(void) { }; + #endif ++void do_uaccess_flush_fixups(enum l1d_flush_type types); + void do_entry_flush_fixups(enum l1d_flush_type types); + void do_barrier_nospec_fixups(bool enable); + extern bool barrier_nospec_enabled; +--- a/arch/powerpc/kernel/exceptions-64s.S ++++ b/arch/powerpc/kernel/exceptions-64s.S +@@ -2046,11 +2046,8 @@ TRAMP_REAL_BEGIN(stf_barrier_fallback) + .endr + blr + +-TRAMP_REAL_BEGIN(entry_flush_fallback) +- std r9,PACA_EXRFI+EX_R9(r13) +- std r10,PACA_EXRFI+EX_R10(r13) +- std r11,PACA_EXRFI+EX_R11(r13) +- mfctr r9 ++/* Clobbers r10, r11, ctr */ ++.macro L1D_DISPLACEMENT_FLUSH + ld r10,PACA_RFI_FLUSH_FALLBACK_AREA(r13) + ld r11,PACA_L1D_FLUSH_SIZE(r13) + srdi r11,r11,(7 + 3) /* 128 byte lines, unrolled 8x */ +@@ -2076,7 +2073,14 @@ TRAMP_REAL_BEGIN(entry_flush_fallback) + ld r11,(0x80 + 8)*7(r10) + addi r10,r10,0x80*8 + bdnz 1b ++.endm + ++TRAMP_REAL_BEGIN(entry_flush_fallback) ++ std r9,PACA_EXRFI+EX_R9(r13) ++ std r10,PACA_EXRFI+EX_R10(r13) ++ std r11,PACA_EXRFI+EX_R11(r13) ++ mfctr r9 ++ L1D_DISPLACEMENT_FLUSH + mtctr r9 + ld r9,PACA_EXRFI+EX_R9(r13) + ld r10,PACA_EXRFI+EX_R10(r13) +@@ -2092,32 +2096,7 @@ TRAMP_REAL_BEGIN(rfi_flush_fallback) + std r10,PACA_EXRFI+EX_R10(r13) + std r11,PACA_EXRFI+EX_R11(r13) + mfctr r9 +- ld r10,PACA_RFI_FLUSH_FALLBACK_AREA(r13) +- ld r11,PACA_L1D_FLUSH_SIZE(r13) +- srdi r11,r11,(7 + 3) /* 128 byte lines, unrolled 8x */ +- mtctr r11 +- DCBT_BOOK3S_STOP_ALL_STREAM_IDS(r11) /* Stop prefetch streams */ +- +- /* order ld/st prior to dcbt stop all streams with flushing */ +- sync +- +- /* +- * The load adresses are at staggered offsets within cachelines, +- * which suits some pipelines better (on others it should not +- * hurt). +- */ +-1: +- ld r11,(0x80 + 8)*0(r10) +- ld r11,(0x80 + 8)*1(r10) +- ld r11,(0x80 + 8)*2(r10) +- ld r11,(0x80 + 8)*3(r10) +- ld r11,(0x80 + 8)*4(r10) +- ld r11,(0x80 + 8)*5(r10) +- ld r11,(0x80 + 8)*6(r10) +- ld r11,(0x80 + 8)*7(r10) +- addi r10,r10,0x80*8 +- bdnz 1b +- ++ L1D_DISPLACEMENT_FLUSH + mtctr r9 + ld r9,PACA_EXRFI+EX_R9(r13) + ld r10,PACA_EXRFI+EX_R10(r13) +@@ -2135,32 +2114,7 @@ TRAMP_REAL_BEGIN(hrfi_flush_fallback) + std r10,PACA_EXRFI+EX_R10(r13) + std r11,PACA_EXRFI+EX_R11(r13) + mfctr r9 +- ld r10,PACA_RFI_FLUSH_FALLBACK_AREA(r13) +- ld r11,PACA_L1D_FLUSH_SIZE(r13) +- srdi r11,r11,(7 + 3) /* 128 byte lines, unrolled 8x */ +- mtctr r11 +- DCBT_BOOK3S_STOP_ALL_STREAM_IDS(r11) /* Stop prefetch streams */ +- +- /* order ld/st prior to dcbt stop all streams with flushing */ +- sync +- +- /* +- * The load adresses are at staggered offsets within cachelines, +- * which suits some pipelines better (on others it should not +- * hurt). +- */ +-1: +- ld r11,(0x80 + 8)*0(r10) +- ld r11,(0x80 + 8)*1(r10) +- ld r11,(0x80 + 8)*2(r10) +- ld r11,(0x80 + 8)*3(r10) +- ld r11,(0x80 + 8)*4(r10) +- ld r11,(0x80 + 8)*5(r10) +- ld r11,(0x80 + 8)*6(r10) +- ld r11,(0x80 + 8)*7(r10) +- addi r10,r10,0x80*8 +- bdnz 1b +- ++ L1D_DISPLACEMENT_FLUSH + mtctr r9 + ld r9,PACA_EXRFI+EX_R9(r13) + ld r10,PACA_EXRFI+EX_R10(r13) +@@ -2169,6 +2123,19 @@ TRAMP_REAL_BEGIN(hrfi_flush_fallback) + GET_SCRATCH0(r13); + hrfid + ++USE_TEXT_SECTION() ++ ++_GLOBAL(do_uaccess_flush) ++ UACCESS_FLUSH_FIXUP_SECTION ++ nop ++ nop ++ nop ++ blr ++ L1D_DISPLACEMENT_FLUSH ++ blr ++_ASM_NOKPROBE_SYMBOL(do_uaccess_flush) ++EXPORT_SYMBOL(do_uaccess_flush) ++ + /* + * Real mode exceptions actually use this too, but alternate + * instruction code patches (which end up in the common .text area) +--- a/arch/powerpc/kernel/setup_64.c ++++ b/arch/powerpc/kernel/setup_64.c +@@ -860,8 +860,12 @@ static enum l1d_flush_type enabled_flush + static void *l1d_flush_fallback_area; + static bool no_rfi_flush; + static bool no_entry_flush; ++static bool no_uaccess_flush; + bool rfi_flush; + bool entry_flush; ++bool uaccess_flush; ++DEFINE_STATIC_KEY_FALSE(uaccess_flush_key); ++EXPORT_SYMBOL(uaccess_flush_key); + + static int __init handle_no_rfi_flush(char *p) + { +@@ -879,6 +883,14 @@ static int __init handle_no_entry_flush( + } + early_param("no_entry_flush", handle_no_entry_flush); + ++static int __init handle_no_uaccess_flush(char *p) ++{ ++ pr_info("uaccess-flush: disabled on command line."); ++ no_uaccess_flush = true; ++ return 0; ++} ++early_param("no_uaccess_flush", handle_no_uaccess_flush); ++ + /* + * The RFI flush is not KPTI, but because users will see doco that says to use + * nopti we hijack that option here to also disable the RFI flush. +@@ -922,6 +934,20 @@ void entry_flush_enable(bool enable) + entry_flush = enable; + } + ++void uaccess_flush_enable(bool enable) ++{ ++ if (enable) { ++ do_uaccess_flush_fixups(enabled_flush_types); ++ static_branch_enable(&uaccess_flush_key); ++ on_each_cpu(do_nothing, NULL, 1); ++ } else { ++ static_branch_disable(&uaccess_flush_key); ++ do_uaccess_flush_fixups(L1D_FLUSH_NONE); ++ } ++ ++ uaccess_flush = enable; ++} ++ + static void __ref init_fallback_flush(void) + { + u64 l1d_size, limit; +@@ -993,6 +1019,15 @@ void setup_entry_flush(bool enable) + entry_flush_enable(enable); + } + ++void setup_uaccess_flush(bool enable) ++{ ++ if (cpu_mitigations_off()) ++ return; ++ ++ if (!no_uaccess_flush) ++ uaccess_flush_enable(true); ++} ++ + #ifdef CONFIG_DEBUG_FS + static int rfi_flush_set(void *data, u64 val) + { +@@ -1046,10 +1081,37 @@ static int entry_flush_get(void *data, u + + DEFINE_SIMPLE_ATTRIBUTE(fops_entry_flush, entry_flush_get, entry_flush_set, "%llu\n"); + ++static int uaccess_flush_set(void *data, u64 val) ++{ ++ bool enable; ++ ++ if (val == 1) ++ enable = true; ++ else if (val == 0) ++ enable = false; ++ else ++ return -EINVAL; ++ ++ /* Only do anything if we're changing state */ ++ if (enable != uaccess_flush) ++ uaccess_flush_enable(enable); ++ ++ return 0; ++} ++ ++static int uaccess_flush_get(void *data, u64 *val) ++{ ++ *val = uaccess_flush ? 1 : 0; ++ return 0; ++} ++ ++DEFINE_SIMPLE_ATTRIBUTE(fops_uaccess_flush, uaccess_flush_get, uaccess_flush_set, "%llu\n"); ++ + static __init int rfi_flush_debugfs_init(void) + { + debugfs_create_file("rfi_flush", 0600, powerpc_debugfs_root, NULL, &fops_rfi_flush); + debugfs_create_file("entry_flush", 0600, powerpc_debugfs_root, NULL, &fops_entry_flush); ++ debugfs_create_file("uaccess_flush", 0600, powerpc_debugfs_root, NULL, &fops_uaccess_flush); + return 0; + } + device_initcall(rfi_flush_debugfs_init); +--- a/arch/powerpc/kernel/vmlinux.lds.S ++++ b/arch/powerpc/kernel/vmlinux.lds.S +@@ -144,6 +144,13 @@ SECTIONS + } + + . = ALIGN(8); ++ __uaccess_flush_fixup : AT(ADDR(__uaccess_flush_fixup) - LOAD_OFFSET) { ++ __start___uaccess_flush_fixup = .; ++ *(__uaccess_flush_fixup) ++ __stop___uaccess_flush_fixup = .; ++ } ++ ++ . = ALIGN(8); + __entry_flush_fixup : AT(ADDR(__entry_flush_fixup) - LOAD_OFFSET) { + __start___entry_flush_fixup = .; + *(__entry_flush_fixup) +--- a/arch/powerpc/lib/feature-fixups.c ++++ b/arch/powerpc/lib/feature-fixups.c +@@ -228,6 +228,56 @@ void do_stf_barrier_fixups(enum stf_barr + do_stf_exit_barrier_fixups(types); + } + ++void do_uaccess_flush_fixups(enum l1d_flush_type types) ++{ ++ unsigned int instrs[4], *dest; ++ long *start, *end; ++ int i; ++ ++ start = PTRRELOC(&__start___uaccess_flush_fixup); ++ end = PTRRELOC(&__stop___uaccess_flush_fixup); ++ ++ instrs[0] = 0x60000000; /* nop */ ++ instrs[1] = 0x60000000; /* nop */ ++ instrs[2] = 0x60000000; /* nop */ ++ instrs[3] = 0x4e800020; /* blr */ ++ ++ i = 0; ++ if (types == L1D_FLUSH_FALLBACK) { ++ instrs[3] = 0x60000000; /* nop */ ++ /* fallthrough to fallback flush */ ++ } ++ ++ if (types & L1D_FLUSH_ORI) { ++ instrs[i++] = 0x63ff0000; /* ori 31,31,0 speculation barrier */ ++ instrs[i++] = 0x63de0000; /* ori 30,30,0 L1d flush*/ ++ } ++ ++ if (types & L1D_FLUSH_MTTRIG) ++ instrs[i++] = 0x7c12dba6; /* mtspr TRIG2,r0 (SPR #882) */ ++ ++ for (i = 0; start < end; start++, i++) { ++ dest = (void *)start + *start; ++ ++ pr_devel("patching dest %lx\n", (unsigned long)dest); ++ ++ patch_instruction(dest, instrs[0]); ++ ++ patch_instruction((dest + 1), instrs[1]); ++ patch_instruction((dest + 2), instrs[2]); ++ patch_instruction((dest + 3), instrs[3]); ++ } ++ ++ printk(KERN_DEBUG "uaccess-flush: patched %d locations (%s flush)\n", i, ++ (types == L1D_FLUSH_NONE) ? "no" : ++ (types == L1D_FLUSH_FALLBACK) ? "fallback displacement" : ++ (types & L1D_FLUSH_ORI) ? (types & L1D_FLUSH_MTTRIG) ++ ? "ori+mttrig type" ++ : "ori type" : ++ (types & L1D_FLUSH_MTTRIG) ? "mttrig type" ++ : "unknown"); ++} ++ + void do_entry_flush_fixups(enum l1d_flush_type types) + { + unsigned int instrs[3], *dest; +--- a/arch/powerpc/platforms/powernv/setup.c ++++ b/arch/powerpc/platforms/powernv/setup.c +@@ -139,6 +139,10 @@ static void pnv_setup_rfi_flush(void) + enable = security_ftr_enabled(SEC_FTR_FAVOUR_SECURITY) && + security_ftr_enabled(SEC_FTR_L1D_FLUSH_ENTRY); + setup_entry_flush(enable); ++ ++ enable = security_ftr_enabled(SEC_FTR_FAVOUR_SECURITY) && ++ security_ftr_enabled(SEC_FTR_L1D_FLUSH_UACCESS); ++ setup_uaccess_flush(enable); + } + + static void __init pnv_setup_arch(void) +--- a/arch/powerpc/platforms/pseries/setup.c ++++ b/arch/powerpc/platforms/pseries/setup.c +@@ -565,6 +565,10 @@ void pseries_setup_rfi_flush(void) + enable = security_ftr_enabled(SEC_FTR_FAVOUR_SECURITY) && + security_ftr_enabled(SEC_FTR_L1D_FLUSH_ENTRY); + setup_entry_flush(enable); ++ ++ enable = security_ftr_enabled(SEC_FTR_FAVOUR_SECURITY) && ++ security_ftr_enabled(SEC_FTR_L1D_FLUSH_UACCESS); ++ setup_uaccess_flush(enable); + } + + #ifdef CONFIG_PCI_IOV diff --git a/queue-5.4/powerpc-64s-flush-l1d-on-kernel-entry.patch b/queue-5.4/powerpc-64s-flush-l1d-on-kernel-entry.patch new file mode 100644 index 00000000000..99581eacb46 --- /dev/null +++ b/queue-5.4/powerpc-64s-flush-l1d-on-kernel-entry.patch @@ -0,0 +1,475 @@ +From foo@baz Fri Nov 20 08:16:31 AM CET 2020 +From: Daniel Axtens +Date: Fri, 20 Nov 2020 10:35:13 +1100 +Subject: powerpc/64s: flush L1D on kernel entry +To: stable@vger.kernel.org +Cc: dja@axtens.net +Message-ID: <20201119233516.368194-3-dja@axtens.net> + +From: Nicholas Piggin + +commit f79643787e0a0762d2409b7b8334e83f22d85695 upstream. + +[backporting note: we need to mark some exception handlers as out-of-line + because the flushing makes them take too much space -- dja] + +IBM Power9 processors can speculatively operate on data in the L1 cache +before it has been completely validated, via a way-prediction mechanism. It +is not possible for an attacker to determine the contents of impermissible +memory using this method, since these systems implement a combination of +hardware and software security measures to prevent scenarios where +protected data could be leaked. + +However these measures don't address the scenario where an attacker induces +the operating system to speculatively execute instructions using data that +the attacker controls. This can be used for example to speculatively bypass +"kernel user access prevention" techniques, as discovered by Anthony +Steinhauser of Google's Safeside Project. This is not an attack by itself, +but there is a possibility it could be used in conjunction with +side-channels or other weaknesses in the privileged code to construct an +attack. + +This issue can be mitigated by flushing the L1 cache between privilege +boundaries of concern. This patch flushes the L1 cache on kernel entry. + +This is part of the fix for CVE-2020-4788. + +Signed-off-by: Nicholas Piggin +Signed-off-by: Daniel Axtens +Signed-off-by: Greg Kroah-Hartman +--- + Documentation/admin-guide/kernel-parameters.txt | 3 + + arch/powerpc/include/asm/exception-64s.h | 9 +++ + arch/powerpc/include/asm/feature-fixups.h | 10 ++++ + arch/powerpc/include/asm/security_features.h | 4 + + arch/powerpc/include/asm/setup.h | 3 + + arch/powerpc/kernel/exceptions-64s.S | 49 +++++++++++++++++-- + arch/powerpc/kernel/setup_64.c | 60 +++++++++++++++++++++++- + arch/powerpc/kernel/vmlinux.lds.S | 7 ++ + arch/powerpc/lib/feature-fixups.c | 54 +++++++++++++++++++++ + arch/powerpc/platforms/powernv/setup.c | 11 ++++ + arch/powerpc/platforms/pseries/setup.c | 4 + + 11 files changed, 206 insertions(+), 8 deletions(-) + +--- a/Documentation/admin-guide/kernel-parameters.txt ++++ b/Documentation/admin-guide/kernel-parameters.txt +@@ -2667,6 +2667,7 @@ + mds=off [X86] + tsx_async_abort=off [X86] + kvm.nx_huge_pages=off [X86] ++ no_entry_flush [PPC] + + Exceptions: + This does not have any effect on +@@ -2989,6 +2990,8 @@ + + noefi Disable EFI runtime services support. + ++ no_entry_flush [PPC] Don't flush the L1-D cache when entering the kernel. ++ + noexec [IA-64] + + noexec [X86] +--- a/arch/powerpc/include/asm/exception-64s.h ++++ b/arch/powerpc/include/asm/exception-64s.h +@@ -61,11 +61,18 @@ + nop; \ + nop + ++#define ENTRY_FLUSH_SLOT \ ++ ENTRY_FLUSH_FIXUP_SECTION; \ ++ nop; \ ++ nop; \ ++ nop; ++ + /* + * r10 must be free to use, r13 must be paca + */ + #define INTERRUPT_TO_KERNEL \ +- STF_ENTRY_BARRIER_SLOT ++ STF_ENTRY_BARRIER_SLOT; \ ++ ENTRY_FLUSH_SLOT + + /* + * Macros for annotating the expected destination of (h)rfid +--- a/arch/powerpc/include/asm/feature-fixups.h ++++ b/arch/powerpc/include/asm/feature-fixups.h +@@ -205,6 +205,14 @@ label##3: \ + FTR_ENTRY_OFFSET 955b-956b; \ + .popsection; + ++#define ENTRY_FLUSH_FIXUP_SECTION \ ++957: \ ++ .pushsection __entry_flush_fixup,"a"; \ ++ .align 2; \ ++958: \ ++ FTR_ENTRY_OFFSET 957b-958b; \ ++ .popsection; ++ + #define RFI_FLUSH_FIXUP_SECTION \ + 951: \ + .pushsection __rfi_flush_fixup,"a"; \ +@@ -237,8 +245,10 @@ label##3: \ + #include + + extern long stf_barrier_fallback; ++extern long entry_flush_fallback; + extern long __start___stf_entry_barrier_fixup, __stop___stf_entry_barrier_fixup; + extern long __start___stf_exit_barrier_fixup, __stop___stf_exit_barrier_fixup; ++extern long __start___entry_flush_fixup, __stop___entry_flush_fixup; + extern long __start___rfi_flush_fixup, __stop___rfi_flush_fixup; + extern long __start___barrier_nospec_fixup, __stop___barrier_nospec_fixup; + extern long __start__btb_flush_fixup, __stop__btb_flush_fixup; +--- a/arch/powerpc/include/asm/security_features.h ++++ b/arch/powerpc/include/asm/security_features.h +@@ -84,12 +84,16 @@ static inline bool security_ftr_enabled( + // Software required to flush link stack on context switch + #define SEC_FTR_FLUSH_LINK_STACK 0x0000000000001000ull + ++// The L1-D cache should be flushed when entering the kernel ++#define SEC_FTR_L1D_FLUSH_ENTRY 0x0000000000004000ull ++ + + // Features enabled by default + #define SEC_FTR_DEFAULT \ + (SEC_FTR_L1D_FLUSH_HV | \ + SEC_FTR_L1D_FLUSH_PR | \ + SEC_FTR_BNDS_CHK_SPEC_BAR | \ ++ SEC_FTR_L1D_FLUSH_ENTRY | \ + SEC_FTR_FAVOUR_SECURITY) + + #endif /* _ASM_POWERPC_SECURITY_FEATURES_H */ +--- a/arch/powerpc/include/asm/setup.h ++++ b/arch/powerpc/include/asm/setup.h +@@ -52,12 +52,15 @@ enum l1d_flush_type { + }; + + void setup_rfi_flush(enum l1d_flush_type, bool enable); ++void setup_entry_flush(bool enable); ++void setup_uaccess_flush(bool enable); + void do_rfi_flush_fixups(enum l1d_flush_type types); + #ifdef CONFIG_PPC_BARRIER_NOSPEC + void setup_barrier_nospec(void); + #else + static inline void setup_barrier_nospec(void) { }; + #endif ++void do_entry_flush_fixups(enum l1d_flush_type types); + void do_barrier_nospec_fixups(bool enable); + extern bool barrier_nospec_enabled; + +--- a/arch/powerpc/kernel/exceptions-64s.S ++++ b/arch/powerpc/kernel/exceptions-64s.S +@@ -1150,7 +1150,7 @@ EXC_REAL_BEGIN(data_access, 0x300, 0x80) + INT_HANDLER data_access, 0x300, ool=1, dar=1, dsisr=1, kvm=1 + EXC_REAL_END(data_access, 0x300, 0x80) + EXC_VIRT_BEGIN(data_access, 0x4300, 0x80) +- INT_HANDLER data_access, 0x300, virt=1, dar=1, dsisr=1 ++ INT_HANDLER data_access, 0x300, ool=1, virt=1, dar=1, dsisr=1 + EXC_VIRT_END(data_access, 0x4300, 0x80) + INT_KVM_HANDLER data_access, 0x300, EXC_STD, PACA_EXGEN, 1 + EXC_COMMON_BEGIN(data_access_common) +@@ -1205,7 +1205,7 @@ ALT_MMU_FTR_SECTION_END_IFCLR(MMU_FTR_TY + + + EXC_REAL_BEGIN(instruction_access, 0x400, 0x80) +- INT_HANDLER instruction_access, 0x400, kvm=1 ++ INT_HANDLER instruction_access, 0x400, ool=1, kvm=1 + EXC_REAL_END(instruction_access, 0x400, 0x80) + EXC_VIRT_BEGIN(instruction_access, 0x4400, 0x80) + INT_HANDLER instruction_access, 0x400, virt=1 +@@ -1225,7 +1225,7 @@ ALT_MMU_FTR_SECTION_END_IFCLR(MMU_FTR_TY + + + EXC_REAL_BEGIN(instruction_access_slb, 0x480, 0x80) +- INT_HANDLER instruction_access_slb, 0x480, area=PACA_EXSLB, kvm=1 ++ INT_HANDLER instruction_access_slb, 0x480, ool=1, area=PACA_EXSLB, kvm=1 + EXC_REAL_END(instruction_access_slb, 0x480, 0x80) + EXC_VIRT_BEGIN(instruction_access_slb, 0x4480, 0x80) + INT_HANDLER instruction_access_slb, 0x480, virt=1, area=PACA_EXSLB +@@ -1365,17 +1365,17 @@ EXC_REAL_BEGIN(decrementer, 0x900, 0x80) + INT_HANDLER decrementer, 0x900, ool=1, bitmask=IRQS_DISABLED, kvm=1 + EXC_REAL_END(decrementer, 0x900, 0x80) + EXC_VIRT_BEGIN(decrementer, 0x4900, 0x80) +- INT_HANDLER decrementer, 0x900, virt=1, bitmask=IRQS_DISABLED ++ INT_HANDLER decrementer, 0x900, ool=1, virt=1, bitmask=IRQS_DISABLED + EXC_VIRT_END(decrementer, 0x4900, 0x80) + INT_KVM_HANDLER decrementer, 0x900, EXC_STD, PACA_EXGEN, 0 + EXC_COMMON_ASYNC(decrementer_common, 0x900, timer_interrupt) + + + EXC_REAL_BEGIN(hdecrementer, 0x980, 0x80) +- INT_HANDLER hdecrementer, 0x980, hsrr=EXC_HV, kvm=1 ++ INT_HANDLER hdecrementer, 0x980, ool=1, hsrr=EXC_HV, kvm=1 + EXC_REAL_END(hdecrementer, 0x980, 0x80) + EXC_VIRT_BEGIN(hdecrementer, 0x4980, 0x80) +- INT_HANDLER hdecrementer, 0x980, virt=1, hsrr=EXC_HV, kvm=1 ++ INT_HANDLER hdecrementer, 0x980, ool=1, virt=1, hsrr=EXC_HV, kvm=1 + EXC_VIRT_END(hdecrementer, 0x4980, 0x80) + INT_KVM_HANDLER hdecrementer, 0x980, EXC_HV, PACA_EXGEN, 0 + EXC_COMMON(hdecrementer_common, 0x980, hdec_interrupt) +@@ -2046,6 +2046,43 @@ TRAMP_REAL_BEGIN(stf_barrier_fallback) + .endr + blr + ++TRAMP_REAL_BEGIN(entry_flush_fallback) ++ std r9,PACA_EXRFI+EX_R9(r13) ++ std r10,PACA_EXRFI+EX_R10(r13) ++ std r11,PACA_EXRFI+EX_R11(r13) ++ mfctr r9 ++ ld r10,PACA_RFI_FLUSH_FALLBACK_AREA(r13) ++ ld r11,PACA_L1D_FLUSH_SIZE(r13) ++ srdi r11,r11,(7 + 3) /* 128 byte lines, unrolled 8x */ ++ mtctr r11 ++ DCBT_BOOK3S_STOP_ALL_STREAM_IDS(r11) /* Stop prefetch streams */ ++ ++ /* order ld/st prior to dcbt stop all streams with flushing */ ++ sync ++ ++ /* ++ * The load addresses are at staggered offsets within cachelines, ++ * which suits some pipelines better (on others it should not ++ * hurt). ++ */ ++1: ++ ld r11,(0x80 + 8)*0(r10) ++ ld r11,(0x80 + 8)*1(r10) ++ ld r11,(0x80 + 8)*2(r10) ++ ld r11,(0x80 + 8)*3(r10) ++ ld r11,(0x80 + 8)*4(r10) ++ ld r11,(0x80 + 8)*5(r10) ++ ld r11,(0x80 + 8)*6(r10) ++ ld r11,(0x80 + 8)*7(r10) ++ addi r10,r10,0x80*8 ++ bdnz 1b ++ ++ mtctr r9 ++ ld r9,PACA_EXRFI+EX_R9(r13) ++ ld r10,PACA_EXRFI+EX_R10(r13) ++ ld r11,PACA_EXRFI+EX_R11(r13) ++ blr ++ + TRAMP_REAL_BEGIN(rfi_flush_fallback) + SET_SCRATCH0(r13); + GET_PACA(r13); +--- a/arch/powerpc/kernel/setup_64.c ++++ b/arch/powerpc/kernel/setup_64.c +@@ -859,7 +859,9 @@ early_initcall(disable_hardlockup_detect + static enum l1d_flush_type enabled_flush_types; + static void *l1d_flush_fallback_area; + static bool no_rfi_flush; ++static bool no_entry_flush; + bool rfi_flush; ++bool entry_flush; + + static int __init handle_no_rfi_flush(char *p) + { +@@ -869,6 +871,14 @@ static int __init handle_no_rfi_flush(ch + } + early_param("no_rfi_flush", handle_no_rfi_flush); + ++static int __init handle_no_entry_flush(char *p) ++{ ++ pr_info("entry-flush: disabled on command line."); ++ no_entry_flush = true; ++ return 0; ++} ++early_param("no_entry_flush", handle_no_entry_flush); ++ + /* + * The RFI flush is not KPTI, but because users will see doco that says to use + * nopti we hijack that option here to also disable the RFI flush. +@@ -900,6 +910,18 @@ void rfi_flush_enable(bool enable) + rfi_flush = enable; + } + ++void entry_flush_enable(bool enable) ++{ ++ if (enable) { ++ do_entry_flush_fixups(enabled_flush_types); ++ on_each_cpu(do_nothing, NULL, 1); ++ } else { ++ do_entry_flush_fixups(L1D_FLUSH_NONE); ++ } ++ ++ entry_flush = enable; ++} ++ + static void __ref init_fallback_flush(void) + { + u64 l1d_size, limit; +@@ -958,10 +980,19 @@ void setup_rfi_flush(enum l1d_flush_type + + enabled_flush_types = types; + +- if (!no_rfi_flush && !cpu_mitigations_off()) ++ if (!cpu_mitigations_off() && !no_rfi_flush) + rfi_flush_enable(enable); + } + ++void setup_entry_flush(bool enable) ++{ ++ if (cpu_mitigations_off()) ++ return; ++ ++ if (!no_entry_flush) ++ entry_flush_enable(enable); ++} ++ + #ifdef CONFIG_DEBUG_FS + static int rfi_flush_set(void *data, u64 val) + { +@@ -989,9 +1020,36 @@ static int rfi_flush_get(void *data, u64 + + DEFINE_SIMPLE_ATTRIBUTE(fops_rfi_flush, rfi_flush_get, rfi_flush_set, "%llu\n"); + ++static int entry_flush_set(void *data, u64 val) ++{ ++ bool enable; ++ ++ if (val == 1) ++ enable = true; ++ else if (val == 0) ++ enable = false; ++ else ++ return -EINVAL; ++ ++ /* Only do anything if we're changing state */ ++ if (enable != entry_flush) ++ entry_flush_enable(enable); ++ ++ return 0; ++} ++ ++static int entry_flush_get(void *data, u64 *val) ++{ ++ *val = entry_flush ? 1 : 0; ++ return 0; ++} ++ ++DEFINE_SIMPLE_ATTRIBUTE(fops_entry_flush, entry_flush_get, entry_flush_set, "%llu\n"); ++ + static __init int rfi_flush_debugfs_init(void) + { + debugfs_create_file("rfi_flush", 0600, powerpc_debugfs_root, NULL, &fops_rfi_flush); ++ debugfs_create_file("entry_flush", 0600, powerpc_debugfs_root, NULL, &fops_entry_flush); + return 0; + } + device_initcall(rfi_flush_debugfs_init); +--- a/arch/powerpc/kernel/vmlinux.lds.S ++++ b/arch/powerpc/kernel/vmlinux.lds.S +@@ -144,6 +144,13 @@ SECTIONS + } + + . = ALIGN(8); ++ __entry_flush_fixup : AT(ADDR(__entry_flush_fixup) - LOAD_OFFSET) { ++ __start___entry_flush_fixup = .; ++ *(__entry_flush_fixup) ++ __stop___entry_flush_fixup = .; ++ } ++ ++ . = ALIGN(8); + __stf_exit_barrier_fixup : AT(ADDR(__stf_exit_barrier_fixup) - LOAD_OFFSET) { + __start___stf_exit_barrier_fixup = .; + *(__stf_exit_barrier_fixup) +--- a/arch/powerpc/lib/feature-fixups.c ++++ b/arch/powerpc/lib/feature-fixups.c +@@ -228,6 +228,60 @@ void do_stf_barrier_fixups(enum stf_barr + do_stf_exit_barrier_fixups(types); + } + ++void do_entry_flush_fixups(enum l1d_flush_type types) ++{ ++ unsigned int instrs[3], *dest; ++ long *start, *end; ++ int i; ++ ++ start = PTRRELOC(&__start___entry_flush_fixup); ++ end = PTRRELOC(&__stop___entry_flush_fixup); ++ ++ instrs[0] = 0x60000000; /* nop */ ++ instrs[1] = 0x60000000; /* nop */ ++ instrs[2] = 0x60000000; /* nop */ ++ ++ i = 0; ++ if (types == L1D_FLUSH_FALLBACK) { ++ instrs[i++] = 0x7d4802a6; /* mflr r10 */ ++ instrs[i++] = 0x60000000; /* branch patched below */ ++ instrs[i++] = 0x7d4803a6; /* mtlr r10 */ ++ } ++ ++ if (types & L1D_FLUSH_ORI) { ++ instrs[i++] = 0x63ff0000; /* ori 31,31,0 speculation barrier */ ++ instrs[i++] = 0x63de0000; /* ori 30,30,0 L1d flush*/ ++ } ++ ++ if (types & L1D_FLUSH_MTTRIG) ++ instrs[i++] = 0x7c12dba6; /* mtspr TRIG2,r0 (SPR #882) */ ++ ++ for (i = 0; start < end; start++, i++) { ++ dest = (void *)start + *start; ++ ++ pr_devel("patching dest %lx\n", (unsigned long)dest); ++ ++ patch_instruction(dest, instrs[0]); ++ ++ if (types == L1D_FLUSH_FALLBACK) ++ patch_branch((dest + 1), (unsigned long)&entry_flush_fallback, ++ BRANCH_SET_LINK); ++ else ++ patch_instruction((dest + 1), instrs[1]); ++ ++ patch_instruction((dest + 2), instrs[2]); ++ } ++ ++ printk(KERN_DEBUG "entry-flush: patched %d locations (%s flush)\n", i, ++ (types == L1D_FLUSH_NONE) ? "no" : ++ (types == L1D_FLUSH_FALLBACK) ? "fallback displacement" : ++ (types & L1D_FLUSH_ORI) ? (types & L1D_FLUSH_MTTRIG) ++ ? "ori+mttrig type" ++ : "ori type" : ++ (types & L1D_FLUSH_MTTRIG) ? "mttrig type" ++ : "unknown"); ++} ++ + void do_rfi_flush_fixups(enum l1d_flush_type types) + { + unsigned int instrs[3], *dest; +--- a/arch/powerpc/platforms/powernv/setup.c ++++ b/arch/powerpc/platforms/powernv/setup.c +@@ -122,12 +122,23 @@ static void pnv_setup_rfi_flush(void) + type = L1D_FLUSH_ORI; + } + ++ /* ++ * If we are non-Power9 bare metal, we don't need to flush on kernel ++ * entry: it fixes a P9 specific vulnerability. ++ */ ++ if (!pvr_version_is(PVR_POWER9)) ++ security_ftr_clear(SEC_FTR_L1D_FLUSH_ENTRY); ++ + enable = security_ftr_enabled(SEC_FTR_FAVOUR_SECURITY) && \ + (security_ftr_enabled(SEC_FTR_L1D_FLUSH_PR) || \ + security_ftr_enabled(SEC_FTR_L1D_FLUSH_HV)); + + setup_rfi_flush(type, enable); + setup_count_cache_flush(); ++ ++ enable = security_ftr_enabled(SEC_FTR_FAVOUR_SECURITY) && ++ security_ftr_enabled(SEC_FTR_L1D_FLUSH_ENTRY); ++ setup_entry_flush(enable); + } + + static void __init pnv_setup_arch(void) +--- a/arch/powerpc/platforms/pseries/setup.c ++++ b/arch/powerpc/platforms/pseries/setup.c +@@ -561,6 +561,10 @@ void pseries_setup_rfi_flush(void) + + setup_rfi_flush(types, enable); + setup_count_cache_flush(); ++ ++ enable = security_ftr_enabled(SEC_FTR_FAVOUR_SECURITY) && ++ security_ftr_enabled(SEC_FTR_L1D_FLUSH_ENTRY); ++ setup_entry_flush(enable); + } + + #ifdef CONFIG_PCI_IOV diff --git a/queue-5.4/powerpc-only-include-kup-radix.h-for-64-bit-book3s.patch b/queue-5.4/powerpc-only-include-kup-radix.h-for-64-bit-book3s.patch new file mode 100644 index 00000000000..f9dea7d78dc --- /dev/null +++ b/queue-5.4/powerpc-only-include-kup-radix.h-for-64-bit-book3s.patch @@ -0,0 +1,113 @@ +From foo@baz Fri Nov 20 08:16:31 AM CET 2020 +From: Daniel Axtens +Date: Fri, 20 Nov 2020 10:35:15 +1100 +Subject: powerpc: Only include kup-radix.h for 64-bit Book3S +To: stable@vger.kernel.org +Cc: dja@axtens.net +Message-ID: <20201119233516.368194-5-dja@axtens.net> + +From: Michael Ellerman + +commit 178d52c6e89c38d0553b0ac8b99927b11eb995b0 upstream. + +In kup.h we currently include kup-radix.h for all 64-bit builds, which +includes Book3S and Book3E. The latter doesn't make sense, Book3E +never uses the Radix MMU. + +This has worked up until now, but almost by accident, and the recent +uaccess flush changes introduced a build breakage on Book3E because of +the bad structure of the code. + +So disentangle things so that we only use kup-radix.h for Book3S. This +requires some more stubs in kup.h. + +Signed-off-by: Michael Ellerman +Signed-off-by: Daniel Axtens +Signed-off-by: Greg Kroah-Hartman +--- + arch/powerpc/include/asm/book3s/64/kup-radix.h | 5 +++-- + arch/powerpc/include/asm/kup.h | 14 +++++++++++--- + 2 files changed, 14 insertions(+), 5 deletions(-) + +--- a/arch/powerpc/include/asm/book3s/64/kup-radix.h ++++ b/arch/powerpc/include/asm/book3s/64/kup-radix.h +@@ -11,13 +11,12 @@ + + #ifdef __ASSEMBLY__ + +-.macro kuap_restore_amr gpr + #ifdef CONFIG_PPC_KUAP ++.macro kuap_restore_amr gpr + BEGIN_MMU_FTR_SECTION_NESTED(67) + ld \gpr, STACK_REGS_KUAP(r1) + mtspr SPRN_AMR, \gpr + END_MMU_FTR_SECTION_NESTED_IFSET(MMU_FTR_RADIX_KUAP, 67) +-#endif + .endm + + .macro kuap_check_amr gpr1, gpr2 +@@ -31,6 +30,7 @@ + END_MMU_FTR_SECTION_NESTED_IFSET(MMU_FTR_RADIX_KUAP, 67) + #endif + .endm ++#endif + + .macro kuap_save_amr_and_lock gpr1, gpr2, use_cr, msr_pr_cr + #ifdef CONFIG_PPC_KUAP +@@ -87,6 +87,7 @@ bad_kuap_fault(struct pt_regs *regs, uns + "Bug: %s fault blocked by AMR!", is_write ? "Write" : "Read"); + } + #else /* CONFIG_PPC_KUAP */ ++static inline void kuap_restore_amr(struct pt_regs *regs, unsigned long amr) { } + static inline void set_kuap(unsigned long value) { } + #endif /* !CONFIG_PPC_KUAP */ + +--- a/arch/powerpc/include/asm/kup.h ++++ b/arch/powerpc/include/asm/kup.h +@@ -6,7 +6,7 @@ + #define KUAP_WRITE 2 + #define KUAP_READ_WRITE (KUAP_READ | KUAP_WRITE) + +-#ifdef CONFIG_PPC64 ++#ifdef CONFIG_PPC_BOOK3S_64 + #include + #endif + #ifdef CONFIG_PPC_8xx +@@ -24,9 +24,15 @@ + .macro kuap_restore sp, current, gpr1, gpr2, gpr3 + .endm + ++.macro kuap_restore_amr gpr ++.endm ++ + .macro kuap_check current, gpr + .endm + ++.macro kuap_check_amr gpr1, gpr2 ++.endm ++ + #endif + + #else /* !__ASSEMBLY__ */ +@@ -52,17 +58,19 @@ bad_kuap_fault(struct pt_regs *regs, uns + return false; + } + ++static inline void kuap_check_amr(void) { } ++ + /* + * book3s/64/kup-radix.h defines these functions for the !KUAP case to flush + * the L1D cache after user accesses. Only include the empty stubs for other + * platforms. + */ +-#ifndef CONFIG_PPC64 ++#ifndef CONFIG_PPC_BOOK3S_64 + static inline void allow_user_access(void __user *to, const void __user *from, + unsigned long size, unsigned long dir) { } + static inline void prevent_user_access(void __user *to, const void __user *from, + unsigned long size, unsigned long dir) { } +-#endif /* CONFIG_PPC64 */ ++#endif /* CONFIG_PPC_BOOK3S_64 */ + #endif /* CONFIG_PPC_KUAP */ + + static inline void allow_read_from_user(const void __user *from, unsigned long size) diff --git a/queue-5.4/selftests-powerpc-entry-flush-test.patch b/queue-5.4/selftests-powerpc-entry-flush-test.patch new file mode 100644 index 00000000000..46e0ad32e5b --- /dev/null +++ b/queue-5.4/selftests-powerpc-entry-flush-test.patch @@ -0,0 +1,239 @@ +From foo@baz Fri Nov 20 08:16:31 AM CET 2020 +From: Daniel Axtens +Date: Fri, 20 Nov 2020 10:35:16 +1100 +Subject: selftests/powerpc: entry flush test +To: stable@vger.kernel.org +Cc: dja@axtens.net +Message-ID: <20201119233516.368194-6-dja@axtens.net> + +From: Daniel Axtens + +commit 89a83a0c69c81a25ce91002b90ca27ed86132a0a upstream. + +Add a test modelled on the RFI flush test which counts the number +of L1D misses doing a simple syscall with the entry flush on and off. + +For simplicity of backporting, this test duplicates a lot of code from +the upstream rfi_flush. This is cleaned up upstream, but we don't clean +it up here because it would involve bringing in even more commits. + +Signed-off-by: Daniel Axtens +Signed-off-by: Greg Kroah-Hartman +--- + arch/powerpc/kernel/setup_64.c | 2 + arch/powerpc/platforms/powernv/setup.c | 6 + tools/testing/selftests/powerpc/security/.gitignore | 1 + tools/testing/selftests/powerpc/security/Makefile | 2 + tools/testing/selftests/powerpc/security/entry_flush.c | 163 +++++++++++++++++ + 5 files changed, 170 insertions(+), 4 deletions(-) + create mode 100644 tools/testing/selftests/powerpc/security/entry_flush.c + +--- a/arch/powerpc/kernel/setup_64.c ++++ b/arch/powerpc/kernel/setup_64.c +@@ -1025,7 +1025,7 @@ void setup_uaccess_flush(bool enable) + return; + + if (!no_uaccess_flush) +- uaccess_flush_enable(true); ++ uaccess_flush_enable(enable); + } + + #ifdef CONFIG_DEBUG_FS +--- a/arch/powerpc/platforms/powernv/setup.c ++++ b/arch/powerpc/platforms/powernv/setup.c +@@ -124,10 +124,12 @@ static void pnv_setup_rfi_flush(void) + + /* + * If we are non-Power9 bare metal, we don't need to flush on kernel +- * entry: it fixes a P9 specific vulnerability. ++ * entry or after user access: they fix a P9 specific vulnerability. + */ +- if (!pvr_version_is(PVR_POWER9)) ++ if (!pvr_version_is(PVR_POWER9)) { + security_ftr_clear(SEC_FTR_L1D_FLUSH_ENTRY); ++ security_ftr_clear(SEC_FTR_L1D_FLUSH_UACCESS); ++ } + + enable = security_ftr_enabled(SEC_FTR_FAVOUR_SECURITY) && \ + (security_ftr_enabled(SEC_FTR_L1D_FLUSH_PR) || \ +--- a/tools/testing/selftests/powerpc/security/.gitignore ++++ b/tools/testing/selftests/powerpc/security/.gitignore +@@ -1 +1,2 @@ + rfi_flush ++entry_flush +--- a/tools/testing/selftests/powerpc/security/Makefile ++++ b/tools/testing/selftests/powerpc/security/Makefile +@@ -1,6 +1,6 @@ + # SPDX-License-Identifier: GPL-2.0+ + +-TEST_GEN_PROGS := rfi_flush ++TEST_GEN_PROGS := rfi_flush entry_flush + top_srcdir = ../../../../.. + + CFLAGS += -I../../../../../usr/include +--- /dev/null ++++ b/tools/testing/selftests/powerpc/security/entry_flush.c +@@ -0,0 +1,163 @@ ++// SPDX-License-Identifier: GPL-2.0+ ++ ++/* ++ * Copyright 2018 IBM Corporation. ++ */ ++ ++#define __SANE_USERSPACE_TYPES__ ++ ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include "utils.h" ++ ++#define CACHELINE_SIZE 128 ++ ++struct perf_event_read { ++ __u64 nr; ++ __u64 l1d_misses; ++}; ++ ++static inline __u64 load(void *addr) ++{ ++ __u64 tmp; ++ ++ asm volatile("ld %0,0(%1)" : "=r"(tmp) : "b"(addr)); ++ ++ return tmp; ++} ++ ++static void syscall_loop(char *p, unsigned long iterations, ++ unsigned long zero_size) ++{ ++ for (unsigned long i = 0; i < iterations; i++) { ++ for (unsigned long j = 0; j < zero_size; j += CACHELINE_SIZE) ++ load(p + j); ++ getppid(); ++ } ++} ++ ++int entry_flush_test(void) ++{ ++ char *p; ++ int repetitions = 10; ++ int fd, passes = 0, iter, rc = 0; ++ struct perf_event_read v; ++ __u64 l1d_misses_total = 0; ++ unsigned long iterations = 100000, zero_size = 24 * 1024; ++ unsigned long l1d_misses_expected; ++ int rfi_flush_orig; ++ int entry_flush, entry_flush_orig; ++ ++ SKIP_IF(geteuid() != 0); ++ ++ // The PMU event we use only works on Power7 or later ++ SKIP_IF(!have_hwcap(PPC_FEATURE_ARCH_2_06)); ++ ++ if (read_debugfs_file("powerpc/rfi_flush", &rfi_flush_orig) < 0) { ++ perror("Unable to read powerpc/rfi_flush debugfs file"); ++ SKIP_IF(1); ++ } ++ ++ if (read_debugfs_file("powerpc/entry_flush", &entry_flush_orig) < 0) { ++ perror("Unable to read powerpc/entry_flush debugfs file"); ++ SKIP_IF(1); ++ } ++ ++ if (rfi_flush_orig != 0) { ++ if (write_debugfs_file("powerpc/rfi_flush", 0) < 0) { ++ perror("error writing to powerpc/rfi_flush debugfs file"); ++ FAIL_IF(1); ++ } ++ } ++ ++ entry_flush = entry_flush_orig; ++ ++ fd = perf_event_open_counter(PERF_TYPE_RAW, /* L1d miss */ 0x400f0, -1); ++ FAIL_IF(fd < 0); ++ ++ p = (char *)memalign(zero_size, CACHELINE_SIZE); ++ ++ FAIL_IF(perf_event_enable(fd)); ++ ++ // disable L1 prefetching ++ set_dscr(1); ++ ++ iter = repetitions; ++ ++ /* ++ * We expect to see l1d miss for each cacheline access when entry_flush ++ * is set. Allow a small variation on this. ++ */ ++ l1d_misses_expected = iterations * (zero_size / CACHELINE_SIZE - 2); ++ ++again: ++ FAIL_IF(perf_event_reset(fd)); ++ ++ syscall_loop(p, iterations, zero_size); ++ ++ FAIL_IF(read(fd, &v, sizeof(v)) != sizeof(v)); ++ ++ if (entry_flush && v.l1d_misses >= l1d_misses_expected) ++ passes++; ++ else if (!entry_flush && v.l1d_misses < (l1d_misses_expected / 2)) ++ passes++; ++ ++ l1d_misses_total += v.l1d_misses; ++ ++ while (--iter) ++ goto again; ++ ++ if (passes < repetitions) { ++ printf("FAIL (L1D misses with entry_flush=%d: %llu %c %lu) [%d/%d failures]\n", ++ entry_flush, l1d_misses_total, entry_flush ? '<' : '>', ++ entry_flush ? repetitions * l1d_misses_expected : ++ repetitions * l1d_misses_expected / 2, ++ repetitions - passes, repetitions); ++ rc = 1; ++ } else ++ printf("PASS (L1D misses with entry_flush=%d: %llu %c %lu) [%d/%d pass]\n", ++ entry_flush, l1d_misses_total, entry_flush ? '>' : '<', ++ entry_flush ? repetitions * l1d_misses_expected : ++ repetitions * l1d_misses_expected / 2, ++ passes, repetitions); ++ ++ if (entry_flush == entry_flush_orig) { ++ entry_flush = !entry_flush_orig; ++ if (write_debugfs_file("powerpc/entry_flush", entry_flush) < 0) { ++ perror("error writing to powerpc/entry_flush debugfs file"); ++ return 1; ++ } ++ iter = repetitions; ++ l1d_misses_total = 0; ++ passes = 0; ++ goto again; ++ } ++ ++ perf_event_disable(fd); ++ close(fd); ++ ++ set_dscr(0); ++ ++ if (write_debugfs_file("powerpc/rfi_flush", rfi_flush_orig) < 0) { ++ perror("unable to restore original value of powerpc/rfi_flush debugfs file"); ++ return 1; ++ } ++ ++ if (write_debugfs_file("powerpc/entry_flush", entry_flush_orig) < 0) { ++ perror("unable to restore original value of powerpc/entry_flush debugfs file"); ++ return 1; ++ } ++ ++ return rc; ++} ++ ++int main(int argc, char *argv[]) ++{ ++ return test_harness(entry_flush_test, "entry_flush_test"); ++} diff --git a/queue-5.4/selftests-powerpc-rfi_flush-disable-entry-flush-if-present.patch b/queue-5.4/selftests-powerpc-rfi_flush-disable-entry-flush-if-present.patch new file mode 100644 index 00000000000..ea539d43641 --- /dev/null +++ b/queue-5.4/selftests-powerpc-rfi_flush-disable-entry-flush-if-present.patch @@ -0,0 +1,104 @@ +From foo@baz Fri Nov 20 08:16:31 AM CET 2020 +From: Daniel Axtens +Date: Fri, 20 Nov 2020 10:35:12 +1100 +Subject: selftests/powerpc: rfi_flush: disable entry flush if present +To: stable@vger.kernel.org +Cc: dja@axtens.net +Message-ID: <20201119233516.368194-2-dja@axtens.net> + +From: Russell Currey + +commit fcb48454c23c5679d1a2e252f127642e91b05cbe upstream. + +We are about to add an entry flush. The rfi (exit) flush test measures +the number of L1D flushes over a syscall with the RFI flush enabled and +disabled. But if the entry flush is also enabled, the effect of enabling +and disabling the RFI flush is masked. + +If there is a debugfs entry for the entry flush, disable it during the RFI +flush and restore it later. + +Reported-by: Spoorthy S +Signed-off-by: Russell Currey +Signed-off-by: Daniel Axtens +Signed-off-by: Greg Kroah-Hartman +--- + tools/testing/selftests/powerpc/security/rfi_flush.c | 35 +++++++++++++++---- + 1 file changed, 29 insertions(+), 6 deletions(-) + +--- a/tools/testing/selftests/powerpc/security/rfi_flush.c ++++ b/tools/testing/selftests/powerpc/security/rfi_flush.c +@@ -50,16 +50,30 @@ int rfi_flush_test(void) + __u64 l1d_misses_total = 0; + unsigned long iterations = 100000, zero_size = 24 * 1024; + unsigned long l1d_misses_expected; +- int rfi_flush_org, rfi_flush; ++ int rfi_flush_orig, rfi_flush; ++ int have_entry_flush, entry_flush_orig; + + SKIP_IF(geteuid() != 0); + +- if (read_debugfs_file("powerpc/rfi_flush", &rfi_flush_org)) { ++ if (read_debugfs_file("powerpc/rfi_flush", &rfi_flush_orig) < 0) { + perror("Unable to read powerpc/rfi_flush debugfs file"); + SKIP_IF(1); + } + +- rfi_flush = rfi_flush_org; ++ if (read_debugfs_file("powerpc/entry_flush", &entry_flush_orig) < 0) { ++ have_entry_flush = 0; ++ } else { ++ have_entry_flush = 1; ++ ++ if (entry_flush_orig != 0) { ++ if (write_debugfs_file("powerpc/entry_flush", 0) < 0) { ++ perror("error writing to powerpc/entry_flush debugfs file"); ++ return 1; ++ } ++ } ++ } ++ ++ rfi_flush = rfi_flush_orig; + + fd = perf_event_open_counter(PERF_TYPE_RAW, /* L1d miss */ 0x400f0, -1); + FAIL_IF(fd < 0); +@@ -68,6 +82,7 @@ int rfi_flush_test(void) + + FAIL_IF(perf_event_enable(fd)); + ++ // disable L1 prefetching + set_dscr(1); + + iter = repetitions; +@@ -109,8 +124,8 @@ again: + repetitions * l1d_misses_expected / 2, + passes, repetitions); + +- if (rfi_flush == rfi_flush_org) { +- rfi_flush = !rfi_flush_org; ++ if (rfi_flush == rfi_flush_orig) { ++ rfi_flush = !rfi_flush_orig; + if (write_debugfs_file("powerpc/rfi_flush", rfi_flush) < 0) { + perror("error writing to powerpc/rfi_flush debugfs file"); + return 1; +@@ -126,11 +141,19 @@ again: + + set_dscr(0); + +- if (write_debugfs_file("powerpc/rfi_flush", rfi_flush_org) < 0) { ++ if (write_debugfs_file("powerpc/rfi_flush", rfi_flush_orig) < 0) { + perror("unable to restore original value of powerpc/rfi_flush debugfs file"); + return 1; + } + ++ if (have_entry_flush) { ++ if (write_debugfs_file("powerpc/entry_flush", entry_flush_orig) < 0) { ++ perror("unable to restore original value of powerpc/entry_flush " ++ "debugfs file"); ++ return 1; ++ } ++ } ++ + return rc; + } + diff --git a/queue-5.4/series b/queue-5.4/series new file mode 100644 index 00000000000..66fc4adb43e --- /dev/null +++ b/queue-5.4/series @@ -0,0 +1,5 @@ +selftests-powerpc-rfi_flush-disable-entry-flush-if-present.patch +powerpc-64s-flush-l1d-on-kernel-entry.patch +powerpc-64s-flush-l1d-after-user-accesses.patch +powerpc-only-include-kup-radix.h-for-64-bit-book3s.patch +selftests-powerpc-entry-flush-test.patch