--- /dev/null
+From foo@baz Fri Nov 20 08:16:31 AM CET 2020
+From: Daniel Axtens <dja@axtens.net>
+Date: Fri, 20 Nov 2020 10:35:14 +1100
+Subject: powerpc/64s: flush L1D after user accesses
+To: stable@vger.kernel.org
+Cc: dja@axtens.net
+Message-ID: <20201119233516.368194-4-dja@axtens.net>
+
+From: Nicholas Piggin <npiggin@gmail.com>
+
+commit 9a32a7e78bd0cd9a9b6332cbdc345ee5ffd0c5de upstream.
+
+IBM Power9 processors can speculatively operate on data in the L1 cache
+before it has been completely validated, via a way-prediction mechanism. It
+is not possible for an attacker to determine the contents of impermissible
+memory using this method, since these systems implement a combination of
+hardware and software security measures to prevent scenarios where
+protected data could be leaked.
+
+However these measures don't address the scenario where an attacker induces
+the operating system to speculatively execute instructions using data that
+the attacker controls. This can be used for example to speculatively bypass
+"kernel user access prevention" techniques, as discovered by Anthony
+Steinhauser of Google's Safeside Project. This is not an attack by itself,
+but there is a possibility it could be used in conjunction with
+side-channels or other weaknesses in the privileged code to construct an
+attack.
+
+This issue can be mitigated by flushing the L1 cache between privilege
+boundaries of concern. This patch flushes the L1 cache after user accesses.
+
+This is part of the fix for CVE-2020-4788.
+
+Signed-off-by: Nicholas Piggin <npiggin@gmail.com>
+Signed-off-by: Daniel Axtens <dja@axtens.net>
+Signed-off-by: Greg Kroah-Hartman <gregkh@linuxfoundation.org>
+---
+ Documentation/admin-guide/kernel-parameters.txt | 4 +
+ arch/powerpc/include/asm/book3s/64/kup-radix.h | 24 ++++---
+ arch/powerpc/include/asm/exception-64s.h | 3
+ arch/powerpc/include/asm/feature-fixups.h | 9 ++
+ arch/powerpc/include/asm/kup.h | 17 +++--
+ arch/powerpc/include/asm/security_features.h | 3
+ arch/powerpc/include/asm/setup.h | 1
+ arch/powerpc/kernel/exceptions-64s.S | 81 +++++++-----------------
+ arch/powerpc/kernel/setup_64.c | 62 ++++++++++++++++++
+ arch/powerpc/kernel/vmlinux.lds.S | 7 ++
+ arch/powerpc/lib/feature-fixups.c | 50 ++++++++++++++
+ arch/powerpc/platforms/powernv/setup.c | 4 +
+ arch/powerpc/platforms/pseries/setup.c | 4 +
+ 13 files changed, 199 insertions(+), 70 deletions(-)
+
+--- a/Documentation/admin-guide/kernel-parameters.txt
++++ b/Documentation/admin-guide/kernel-parameters.txt
+@@ -2668,6 +2668,7 @@
+ tsx_async_abort=off [X86]
+ kvm.nx_huge_pages=off [X86]
+ no_entry_flush [PPC]
++ no_uaccess_flush [PPC]
+
+ Exceptions:
+ This does not have any effect on
+@@ -3041,6 +3042,9 @@
+ nospec_store_bypass_disable
+ [HW] Disable all mitigations for the Speculative Store Bypass vulnerability
+
++ no_uaccess_flush
++ [PPC] Don't flush the L1-D cache after accessing user data.
++
+ noxsave [BUGS=X86] Disables x86 extended register state save
+ and restore using xsave. The kernel will fallback to
+ enabling legacy floating-point and sse state.
+--- a/arch/powerpc/include/asm/book3s/64/kup-radix.h
++++ b/arch/powerpc/include/asm/book3s/64/kup-radix.h
+@@ -54,6 +54,8 @@
+
+ #else /* !__ASSEMBLY__ */
+
++DECLARE_STATIC_KEY_FALSE(uaccess_flush_key);
++
+ #ifdef CONFIG_PPC_KUAP
+
+ #include <asm/reg.h>
+@@ -77,6 +79,17 @@ static inline void set_kuap(unsigned lon
+ isync();
+ }
+
++static inline bool
++bad_kuap_fault(struct pt_regs *regs, unsigned long address, bool is_write)
++{
++ return WARN(mmu_has_feature(MMU_FTR_RADIX_KUAP) &&
++ (regs->kuap & (is_write ? AMR_KUAP_BLOCK_WRITE : AMR_KUAP_BLOCK_READ)),
++ "Bug: %s fault blocked by AMR!", is_write ? "Write" : "Read");
++}
++#else /* CONFIG_PPC_KUAP */
++static inline void set_kuap(unsigned long value) { }
++#endif /* !CONFIG_PPC_KUAP */
++
+ static __always_inline void allow_user_access(void __user *to, const void __user *from,
+ unsigned long size, unsigned long dir)
+ {
+@@ -94,17 +107,10 @@ static inline void prevent_user_access(v
+ unsigned long size, unsigned long dir)
+ {
+ set_kuap(AMR_KUAP_BLOCKED);
++ if (static_branch_unlikely(&uaccess_flush_key))
++ do_uaccess_flush();
+ }
+
+-static inline bool
+-bad_kuap_fault(struct pt_regs *regs, unsigned long address, bool is_write)
+-{
+- return WARN(mmu_has_feature(MMU_FTR_RADIX_KUAP) &&
+- (regs->kuap & (is_write ? AMR_KUAP_BLOCK_WRITE : AMR_KUAP_BLOCK_READ)),
+- "Bug: %s fault blocked by AMR!", is_write ? "Write" : "Read");
+-}
+-#endif /* CONFIG_PPC_KUAP */
+-
+ #endif /* __ASSEMBLY__ */
+
+ #endif /* _ASM_POWERPC_BOOK3S_64_KUP_RADIX_H */
+--- a/arch/powerpc/include/asm/exception-64s.h
++++ b/arch/powerpc/include/asm/exception-64s.h
+@@ -134,6 +134,9 @@
+ hrfid; \
+ b hrfi_flush_fallback
+
++#else /* __ASSEMBLY__ */
++/* Prototype for function defined in exceptions-64s.S */
++void do_uaccess_flush(void);
+ #endif /* __ASSEMBLY__ */
+
+ #endif /* _ASM_POWERPC_EXCEPTION_H */
+--- a/arch/powerpc/include/asm/feature-fixups.h
++++ b/arch/powerpc/include/asm/feature-fixups.h
+@@ -205,6 +205,14 @@ label##3: \
+ FTR_ENTRY_OFFSET 955b-956b; \
+ .popsection;
+
++#define UACCESS_FLUSH_FIXUP_SECTION \
++959: \
++ .pushsection __uaccess_flush_fixup,"a"; \
++ .align 2; \
++960: \
++ FTR_ENTRY_OFFSET 959b-960b; \
++ .popsection;
++
+ #define ENTRY_FLUSH_FIXUP_SECTION \
+ 957: \
+ .pushsection __entry_flush_fixup,"a"; \
+@@ -248,6 +256,7 @@ extern long stf_barrier_fallback;
+ extern long entry_flush_fallback;
+ extern long __start___stf_entry_barrier_fixup, __stop___stf_entry_barrier_fixup;
+ extern long __start___stf_exit_barrier_fixup, __stop___stf_exit_barrier_fixup;
++extern long __start___uaccess_flush_fixup, __stop___uaccess_flush_fixup;
+ extern long __start___entry_flush_fixup, __stop___entry_flush_fixup;
+ extern long __start___rfi_flush_fixup, __stop___rfi_flush_fixup;
+ extern long __start___barrier_nospec_fixup, __stop___barrier_nospec_fixup;
+--- a/arch/powerpc/include/asm/kup.h
++++ b/arch/powerpc/include/asm/kup.h
+@@ -45,15 +45,24 @@ static inline void setup_kuep(bool disab
+ void setup_kuap(bool disabled);
+ #else
+ static inline void setup_kuap(bool disabled) { }
+-static inline void allow_user_access(void __user *to, const void __user *from,
+- unsigned long size, unsigned long dir) { }
+-static inline void prevent_user_access(void __user *to, const void __user *from,
+- unsigned long size, unsigned long dir) { }
++
+ static inline bool
+ bad_kuap_fault(struct pt_regs *regs, unsigned long address, bool is_write)
+ {
+ return false;
+ }
++
++/*
++ * book3s/64/kup-radix.h defines these functions for the !KUAP case to flush
++ * the L1D cache after user accesses. Only include the empty stubs for other
++ * platforms.
++ */
++#ifndef CONFIG_PPC64
++static inline void allow_user_access(void __user *to, const void __user *from,
++ unsigned long size, unsigned long dir) { }
++static inline void prevent_user_access(void __user *to, const void __user *from,
++ unsigned long size, unsigned long dir) { }
++#endif /* CONFIG_PPC64 */
+ #endif /* CONFIG_PPC_KUAP */
+
+ static inline void allow_read_from_user(const void __user *from, unsigned long size)
+--- a/arch/powerpc/include/asm/security_features.h
++++ b/arch/powerpc/include/asm/security_features.h
+@@ -87,6 +87,8 @@ static inline bool security_ftr_enabled(
+ // The L1-D cache should be flushed when entering the kernel
+ #define SEC_FTR_L1D_FLUSH_ENTRY 0x0000000000004000ull
+
++// The L1-D cache should be flushed after user accesses from the kernel
++#define SEC_FTR_L1D_FLUSH_UACCESS 0x0000000000008000ull
+
+ // Features enabled by default
+ #define SEC_FTR_DEFAULT \
+@@ -94,6 +96,7 @@ static inline bool security_ftr_enabled(
+ SEC_FTR_L1D_FLUSH_PR | \
+ SEC_FTR_BNDS_CHK_SPEC_BAR | \
+ SEC_FTR_L1D_FLUSH_ENTRY | \
++ SEC_FTR_L1D_FLUSH_UACCESS | \
+ SEC_FTR_FAVOUR_SECURITY)
+
+ #endif /* _ASM_POWERPC_SECURITY_FEATURES_H */
+--- a/arch/powerpc/include/asm/setup.h
++++ b/arch/powerpc/include/asm/setup.h
+@@ -60,6 +60,7 @@ void setup_barrier_nospec(void);
+ #else
+ static inline void setup_barrier_nospec(void) { };
+ #endif
++void do_uaccess_flush_fixups(enum l1d_flush_type types);
+ void do_entry_flush_fixups(enum l1d_flush_type types);
+ void do_barrier_nospec_fixups(bool enable);
+ extern bool barrier_nospec_enabled;
+--- a/arch/powerpc/kernel/exceptions-64s.S
++++ b/arch/powerpc/kernel/exceptions-64s.S
+@@ -2046,11 +2046,8 @@ TRAMP_REAL_BEGIN(stf_barrier_fallback)
+ .endr
+ blr
+
+-TRAMP_REAL_BEGIN(entry_flush_fallback)
+- std r9,PACA_EXRFI+EX_R9(r13)
+- std r10,PACA_EXRFI+EX_R10(r13)
+- std r11,PACA_EXRFI+EX_R11(r13)
+- mfctr r9
++/* Clobbers r10, r11, ctr */
++.macro L1D_DISPLACEMENT_FLUSH
+ ld r10,PACA_RFI_FLUSH_FALLBACK_AREA(r13)
+ ld r11,PACA_L1D_FLUSH_SIZE(r13)
+ srdi r11,r11,(7 + 3) /* 128 byte lines, unrolled 8x */
+@@ -2076,7 +2073,14 @@ TRAMP_REAL_BEGIN(entry_flush_fallback)
+ ld r11,(0x80 + 8)*7(r10)
+ addi r10,r10,0x80*8
+ bdnz 1b
++.endm
+
++TRAMP_REAL_BEGIN(entry_flush_fallback)
++ std r9,PACA_EXRFI+EX_R9(r13)
++ std r10,PACA_EXRFI+EX_R10(r13)
++ std r11,PACA_EXRFI+EX_R11(r13)
++ mfctr r9
++ L1D_DISPLACEMENT_FLUSH
+ mtctr r9
+ ld r9,PACA_EXRFI+EX_R9(r13)
+ ld r10,PACA_EXRFI+EX_R10(r13)
+@@ -2092,32 +2096,7 @@ TRAMP_REAL_BEGIN(rfi_flush_fallback)
+ std r10,PACA_EXRFI+EX_R10(r13)
+ std r11,PACA_EXRFI+EX_R11(r13)
+ mfctr r9
+- ld r10,PACA_RFI_FLUSH_FALLBACK_AREA(r13)
+- ld r11,PACA_L1D_FLUSH_SIZE(r13)
+- srdi r11,r11,(7 + 3) /* 128 byte lines, unrolled 8x */
+- mtctr r11
+- DCBT_BOOK3S_STOP_ALL_STREAM_IDS(r11) /* Stop prefetch streams */
+-
+- /* order ld/st prior to dcbt stop all streams with flushing */
+- sync
+-
+- /*
+- * The load adresses are at staggered offsets within cachelines,
+- * which suits some pipelines better (on others it should not
+- * hurt).
+- */
+-1:
+- ld r11,(0x80 + 8)*0(r10)
+- ld r11,(0x80 + 8)*1(r10)
+- ld r11,(0x80 + 8)*2(r10)
+- ld r11,(0x80 + 8)*3(r10)
+- ld r11,(0x80 + 8)*4(r10)
+- ld r11,(0x80 + 8)*5(r10)
+- ld r11,(0x80 + 8)*6(r10)
+- ld r11,(0x80 + 8)*7(r10)
+- addi r10,r10,0x80*8
+- bdnz 1b
+-
++ L1D_DISPLACEMENT_FLUSH
+ mtctr r9
+ ld r9,PACA_EXRFI+EX_R9(r13)
+ ld r10,PACA_EXRFI+EX_R10(r13)
+@@ -2135,32 +2114,7 @@ TRAMP_REAL_BEGIN(hrfi_flush_fallback)
+ std r10,PACA_EXRFI+EX_R10(r13)
+ std r11,PACA_EXRFI+EX_R11(r13)
+ mfctr r9
+- ld r10,PACA_RFI_FLUSH_FALLBACK_AREA(r13)
+- ld r11,PACA_L1D_FLUSH_SIZE(r13)
+- srdi r11,r11,(7 + 3) /* 128 byte lines, unrolled 8x */
+- mtctr r11
+- DCBT_BOOK3S_STOP_ALL_STREAM_IDS(r11) /* Stop prefetch streams */
+-
+- /* order ld/st prior to dcbt stop all streams with flushing */
+- sync
+-
+- /*
+- * The load adresses are at staggered offsets within cachelines,
+- * which suits some pipelines better (on others it should not
+- * hurt).
+- */
+-1:
+- ld r11,(0x80 + 8)*0(r10)
+- ld r11,(0x80 + 8)*1(r10)
+- ld r11,(0x80 + 8)*2(r10)
+- ld r11,(0x80 + 8)*3(r10)
+- ld r11,(0x80 + 8)*4(r10)
+- ld r11,(0x80 + 8)*5(r10)
+- ld r11,(0x80 + 8)*6(r10)
+- ld r11,(0x80 + 8)*7(r10)
+- addi r10,r10,0x80*8
+- bdnz 1b
+-
++ L1D_DISPLACEMENT_FLUSH
+ mtctr r9
+ ld r9,PACA_EXRFI+EX_R9(r13)
+ ld r10,PACA_EXRFI+EX_R10(r13)
+@@ -2169,6 +2123,19 @@ TRAMP_REAL_BEGIN(hrfi_flush_fallback)
+ GET_SCRATCH0(r13);
+ hrfid
+
++USE_TEXT_SECTION()
++
++_GLOBAL(do_uaccess_flush)
++ UACCESS_FLUSH_FIXUP_SECTION
++ nop
++ nop
++ nop
++ blr
++ L1D_DISPLACEMENT_FLUSH
++ blr
++_ASM_NOKPROBE_SYMBOL(do_uaccess_flush)
++EXPORT_SYMBOL(do_uaccess_flush)
++
+ /*
+ * Real mode exceptions actually use this too, but alternate
+ * instruction code patches (which end up in the common .text area)
+--- a/arch/powerpc/kernel/setup_64.c
++++ b/arch/powerpc/kernel/setup_64.c
+@@ -860,8 +860,12 @@ static enum l1d_flush_type enabled_flush
+ static void *l1d_flush_fallback_area;
+ static bool no_rfi_flush;
+ static bool no_entry_flush;
++static bool no_uaccess_flush;
+ bool rfi_flush;
+ bool entry_flush;
++bool uaccess_flush;
++DEFINE_STATIC_KEY_FALSE(uaccess_flush_key);
++EXPORT_SYMBOL(uaccess_flush_key);
+
+ static int __init handle_no_rfi_flush(char *p)
+ {
+@@ -879,6 +883,14 @@ static int __init handle_no_entry_flush(
+ }
+ early_param("no_entry_flush", handle_no_entry_flush);
+
++static int __init handle_no_uaccess_flush(char *p)
++{
++ pr_info("uaccess-flush: disabled on command line.");
++ no_uaccess_flush = true;
++ return 0;
++}
++early_param("no_uaccess_flush", handle_no_uaccess_flush);
++
+ /*
+ * The RFI flush is not KPTI, but because users will see doco that says to use
+ * nopti we hijack that option here to also disable the RFI flush.
+@@ -922,6 +934,20 @@ void entry_flush_enable(bool enable)
+ entry_flush = enable;
+ }
+
++void uaccess_flush_enable(bool enable)
++{
++ if (enable) {
++ do_uaccess_flush_fixups(enabled_flush_types);
++ static_branch_enable(&uaccess_flush_key);
++ on_each_cpu(do_nothing, NULL, 1);
++ } else {
++ static_branch_disable(&uaccess_flush_key);
++ do_uaccess_flush_fixups(L1D_FLUSH_NONE);
++ }
++
++ uaccess_flush = enable;
++}
++
+ static void __ref init_fallback_flush(void)
+ {
+ u64 l1d_size, limit;
+@@ -993,6 +1019,15 @@ void setup_entry_flush(bool enable)
+ entry_flush_enable(enable);
+ }
+
++void setup_uaccess_flush(bool enable)
++{
++ if (cpu_mitigations_off())
++ return;
++
++ if (!no_uaccess_flush)
++ uaccess_flush_enable(true);
++}
++
+ #ifdef CONFIG_DEBUG_FS
+ static int rfi_flush_set(void *data, u64 val)
+ {
+@@ -1046,10 +1081,37 @@ static int entry_flush_get(void *data, u
+
+ DEFINE_SIMPLE_ATTRIBUTE(fops_entry_flush, entry_flush_get, entry_flush_set, "%llu\n");
+
++static int uaccess_flush_set(void *data, u64 val)
++{
++ bool enable;
++
++ if (val == 1)
++ enable = true;
++ else if (val == 0)
++ enable = false;
++ else
++ return -EINVAL;
++
++ /* Only do anything if we're changing state */
++ if (enable != uaccess_flush)
++ uaccess_flush_enable(enable);
++
++ return 0;
++}
++
++static int uaccess_flush_get(void *data, u64 *val)
++{
++ *val = uaccess_flush ? 1 : 0;
++ return 0;
++}
++
++DEFINE_SIMPLE_ATTRIBUTE(fops_uaccess_flush, uaccess_flush_get, uaccess_flush_set, "%llu\n");
++
+ static __init int rfi_flush_debugfs_init(void)
+ {
+ debugfs_create_file("rfi_flush", 0600, powerpc_debugfs_root, NULL, &fops_rfi_flush);
+ debugfs_create_file("entry_flush", 0600, powerpc_debugfs_root, NULL, &fops_entry_flush);
++ debugfs_create_file("uaccess_flush", 0600, powerpc_debugfs_root, NULL, &fops_uaccess_flush);
+ return 0;
+ }
+ device_initcall(rfi_flush_debugfs_init);
+--- a/arch/powerpc/kernel/vmlinux.lds.S
++++ b/arch/powerpc/kernel/vmlinux.lds.S
+@@ -144,6 +144,13 @@ SECTIONS
+ }
+
+ . = ALIGN(8);
++ __uaccess_flush_fixup : AT(ADDR(__uaccess_flush_fixup) - LOAD_OFFSET) {
++ __start___uaccess_flush_fixup = .;
++ *(__uaccess_flush_fixup)
++ __stop___uaccess_flush_fixup = .;
++ }
++
++ . = ALIGN(8);
+ __entry_flush_fixup : AT(ADDR(__entry_flush_fixup) - LOAD_OFFSET) {
+ __start___entry_flush_fixup = .;
+ *(__entry_flush_fixup)
+--- a/arch/powerpc/lib/feature-fixups.c
++++ b/arch/powerpc/lib/feature-fixups.c
+@@ -228,6 +228,56 @@ void do_stf_barrier_fixups(enum stf_barr
+ do_stf_exit_barrier_fixups(types);
+ }
+
++void do_uaccess_flush_fixups(enum l1d_flush_type types)
++{
++ unsigned int instrs[4], *dest;
++ long *start, *end;
++ int i;
++
++ start = PTRRELOC(&__start___uaccess_flush_fixup);
++ end = PTRRELOC(&__stop___uaccess_flush_fixup);
++
++ instrs[0] = 0x60000000; /* nop */
++ instrs[1] = 0x60000000; /* nop */
++ instrs[2] = 0x60000000; /* nop */
++ instrs[3] = 0x4e800020; /* blr */
++
++ i = 0;
++ if (types == L1D_FLUSH_FALLBACK) {
++ instrs[3] = 0x60000000; /* nop */
++ /* fallthrough to fallback flush */
++ }
++
++ if (types & L1D_FLUSH_ORI) {
++ instrs[i++] = 0x63ff0000; /* ori 31,31,0 speculation barrier */
++ instrs[i++] = 0x63de0000; /* ori 30,30,0 L1d flush*/
++ }
++
++ if (types & L1D_FLUSH_MTTRIG)
++ instrs[i++] = 0x7c12dba6; /* mtspr TRIG2,r0 (SPR #882) */
++
++ for (i = 0; start < end; start++, i++) {
++ dest = (void *)start + *start;
++
++ pr_devel("patching dest %lx\n", (unsigned long)dest);
++
++ patch_instruction(dest, instrs[0]);
++
++ patch_instruction((dest + 1), instrs[1]);
++ patch_instruction((dest + 2), instrs[2]);
++ patch_instruction((dest + 3), instrs[3]);
++ }
++
++ printk(KERN_DEBUG "uaccess-flush: patched %d locations (%s flush)\n", i,
++ (types == L1D_FLUSH_NONE) ? "no" :
++ (types == L1D_FLUSH_FALLBACK) ? "fallback displacement" :
++ (types & L1D_FLUSH_ORI) ? (types & L1D_FLUSH_MTTRIG)
++ ? "ori+mttrig type"
++ : "ori type" :
++ (types & L1D_FLUSH_MTTRIG) ? "mttrig type"
++ : "unknown");
++}
++
+ void do_entry_flush_fixups(enum l1d_flush_type types)
+ {
+ unsigned int instrs[3], *dest;
+--- a/arch/powerpc/platforms/powernv/setup.c
++++ b/arch/powerpc/platforms/powernv/setup.c
+@@ -139,6 +139,10 @@ static void pnv_setup_rfi_flush(void)
+ enable = security_ftr_enabled(SEC_FTR_FAVOUR_SECURITY) &&
+ security_ftr_enabled(SEC_FTR_L1D_FLUSH_ENTRY);
+ setup_entry_flush(enable);
++
++ enable = security_ftr_enabled(SEC_FTR_FAVOUR_SECURITY) &&
++ security_ftr_enabled(SEC_FTR_L1D_FLUSH_UACCESS);
++ setup_uaccess_flush(enable);
+ }
+
+ static void __init pnv_setup_arch(void)
+--- a/arch/powerpc/platforms/pseries/setup.c
++++ b/arch/powerpc/platforms/pseries/setup.c
+@@ -565,6 +565,10 @@ void pseries_setup_rfi_flush(void)
+ enable = security_ftr_enabled(SEC_FTR_FAVOUR_SECURITY) &&
+ security_ftr_enabled(SEC_FTR_L1D_FLUSH_ENTRY);
+ setup_entry_flush(enable);
++
++ enable = security_ftr_enabled(SEC_FTR_FAVOUR_SECURITY) &&
++ security_ftr_enabled(SEC_FTR_L1D_FLUSH_UACCESS);
++ setup_uaccess_flush(enable);
+ }
+
+ #ifdef CONFIG_PCI_IOV
--- /dev/null
+From foo@baz Fri Nov 20 08:16:31 AM CET 2020
+From: Daniel Axtens <dja@axtens.net>
+Date: Fri, 20 Nov 2020 10:35:13 +1100
+Subject: powerpc/64s: flush L1D on kernel entry
+To: stable@vger.kernel.org
+Cc: dja@axtens.net
+Message-ID: <20201119233516.368194-3-dja@axtens.net>
+
+From: Nicholas Piggin <npiggin@gmail.com>
+
+commit f79643787e0a0762d2409b7b8334e83f22d85695 upstream.
+
+[backporting note: we need to mark some exception handlers as out-of-line
+ because the flushing makes them take too much space -- dja]
+
+IBM Power9 processors can speculatively operate on data in the L1 cache
+before it has been completely validated, via a way-prediction mechanism. It
+is not possible for an attacker to determine the contents of impermissible
+memory using this method, since these systems implement a combination of
+hardware and software security measures to prevent scenarios where
+protected data could be leaked.
+
+However these measures don't address the scenario where an attacker induces
+the operating system to speculatively execute instructions using data that
+the attacker controls. This can be used for example to speculatively bypass
+"kernel user access prevention" techniques, as discovered by Anthony
+Steinhauser of Google's Safeside Project. This is not an attack by itself,
+but there is a possibility it could be used in conjunction with
+side-channels or other weaknesses in the privileged code to construct an
+attack.
+
+This issue can be mitigated by flushing the L1 cache between privilege
+boundaries of concern. This patch flushes the L1 cache on kernel entry.
+
+This is part of the fix for CVE-2020-4788.
+
+Signed-off-by: Nicholas Piggin <npiggin@gmail.com>
+Signed-off-by: Daniel Axtens <dja@axtens.net>
+Signed-off-by: Greg Kroah-Hartman <gregkh@linuxfoundation.org>
+---
+ Documentation/admin-guide/kernel-parameters.txt | 3 +
+ arch/powerpc/include/asm/exception-64s.h | 9 +++
+ arch/powerpc/include/asm/feature-fixups.h | 10 ++++
+ arch/powerpc/include/asm/security_features.h | 4 +
+ arch/powerpc/include/asm/setup.h | 3 +
+ arch/powerpc/kernel/exceptions-64s.S | 49 +++++++++++++++++--
+ arch/powerpc/kernel/setup_64.c | 60 +++++++++++++++++++++++-
+ arch/powerpc/kernel/vmlinux.lds.S | 7 ++
+ arch/powerpc/lib/feature-fixups.c | 54 +++++++++++++++++++++
+ arch/powerpc/platforms/powernv/setup.c | 11 ++++
+ arch/powerpc/platforms/pseries/setup.c | 4 +
+ 11 files changed, 206 insertions(+), 8 deletions(-)
+
+--- a/Documentation/admin-guide/kernel-parameters.txt
++++ b/Documentation/admin-guide/kernel-parameters.txt
+@@ -2667,6 +2667,7 @@
+ mds=off [X86]
+ tsx_async_abort=off [X86]
+ kvm.nx_huge_pages=off [X86]
++ no_entry_flush [PPC]
+
+ Exceptions:
+ This does not have any effect on
+@@ -2989,6 +2990,8 @@
+
+ noefi Disable EFI runtime services support.
+
++ no_entry_flush [PPC] Don't flush the L1-D cache when entering the kernel.
++
+ noexec [IA-64]
+
+ noexec [X86]
+--- a/arch/powerpc/include/asm/exception-64s.h
++++ b/arch/powerpc/include/asm/exception-64s.h
+@@ -61,11 +61,18 @@
+ nop; \
+ nop
+
++#define ENTRY_FLUSH_SLOT \
++ ENTRY_FLUSH_FIXUP_SECTION; \
++ nop; \
++ nop; \
++ nop;
++
+ /*
+ * r10 must be free to use, r13 must be paca
+ */
+ #define INTERRUPT_TO_KERNEL \
+- STF_ENTRY_BARRIER_SLOT
++ STF_ENTRY_BARRIER_SLOT; \
++ ENTRY_FLUSH_SLOT
+
+ /*
+ * Macros for annotating the expected destination of (h)rfid
+--- a/arch/powerpc/include/asm/feature-fixups.h
++++ b/arch/powerpc/include/asm/feature-fixups.h
+@@ -205,6 +205,14 @@ label##3: \
+ FTR_ENTRY_OFFSET 955b-956b; \
+ .popsection;
+
++#define ENTRY_FLUSH_FIXUP_SECTION \
++957: \
++ .pushsection __entry_flush_fixup,"a"; \
++ .align 2; \
++958: \
++ FTR_ENTRY_OFFSET 957b-958b; \
++ .popsection;
++
+ #define RFI_FLUSH_FIXUP_SECTION \
+ 951: \
+ .pushsection __rfi_flush_fixup,"a"; \
+@@ -237,8 +245,10 @@ label##3: \
+ #include <linux/types.h>
+
+ extern long stf_barrier_fallback;
++extern long entry_flush_fallback;
+ extern long __start___stf_entry_barrier_fixup, __stop___stf_entry_barrier_fixup;
+ extern long __start___stf_exit_barrier_fixup, __stop___stf_exit_barrier_fixup;
++extern long __start___entry_flush_fixup, __stop___entry_flush_fixup;
+ extern long __start___rfi_flush_fixup, __stop___rfi_flush_fixup;
+ extern long __start___barrier_nospec_fixup, __stop___barrier_nospec_fixup;
+ extern long __start__btb_flush_fixup, __stop__btb_flush_fixup;
+--- a/arch/powerpc/include/asm/security_features.h
++++ b/arch/powerpc/include/asm/security_features.h
+@@ -84,12 +84,16 @@ static inline bool security_ftr_enabled(
+ // Software required to flush link stack on context switch
+ #define SEC_FTR_FLUSH_LINK_STACK 0x0000000000001000ull
+
++// The L1-D cache should be flushed when entering the kernel
++#define SEC_FTR_L1D_FLUSH_ENTRY 0x0000000000004000ull
++
+
+ // Features enabled by default
+ #define SEC_FTR_DEFAULT \
+ (SEC_FTR_L1D_FLUSH_HV | \
+ SEC_FTR_L1D_FLUSH_PR | \
+ SEC_FTR_BNDS_CHK_SPEC_BAR | \
++ SEC_FTR_L1D_FLUSH_ENTRY | \
+ SEC_FTR_FAVOUR_SECURITY)
+
+ #endif /* _ASM_POWERPC_SECURITY_FEATURES_H */
+--- a/arch/powerpc/include/asm/setup.h
++++ b/arch/powerpc/include/asm/setup.h
+@@ -52,12 +52,15 @@ enum l1d_flush_type {
+ };
+
+ void setup_rfi_flush(enum l1d_flush_type, bool enable);
++void setup_entry_flush(bool enable);
++void setup_uaccess_flush(bool enable);
+ void do_rfi_flush_fixups(enum l1d_flush_type types);
+ #ifdef CONFIG_PPC_BARRIER_NOSPEC
+ void setup_barrier_nospec(void);
+ #else
+ static inline void setup_barrier_nospec(void) { };
+ #endif
++void do_entry_flush_fixups(enum l1d_flush_type types);
+ void do_barrier_nospec_fixups(bool enable);
+ extern bool barrier_nospec_enabled;
+
+--- a/arch/powerpc/kernel/exceptions-64s.S
++++ b/arch/powerpc/kernel/exceptions-64s.S
+@@ -1150,7 +1150,7 @@ EXC_REAL_BEGIN(data_access, 0x300, 0x80)
+ INT_HANDLER data_access, 0x300, ool=1, dar=1, dsisr=1, kvm=1
+ EXC_REAL_END(data_access, 0x300, 0x80)
+ EXC_VIRT_BEGIN(data_access, 0x4300, 0x80)
+- INT_HANDLER data_access, 0x300, virt=1, dar=1, dsisr=1
++ INT_HANDLER data_access, 0x300, ool=1, virt=1, dar=1, dsisr=1
+ EXC_VIRT_END(data_access, 0x4300, 0x80)
+ INT_KVM_HANDLER data_access, 0x300, EXC_STD, PACA_EXGEN, 1
+ EXC_COMMON_BEGIN(data_access_common)
+@@ -1205,7 +1205,7 @@ ALT_MMU_FTR_SECTION_END_IFCLR(MMU_FTR_TY
+
+
+ EXC_REAL_BEGIN(instruction_access, 0x400, 0x80)
+- INT_HANDLER instruction_access, 0x400, kvm=1
++ INT_HANDLER instruction_access, 0x400, ool=1, kvm=1
+ EXC_REAL_END(instruction_access, 0x400, 0x80)
+ EXC_VIRT_BEGIN(instruction_access, 0x4400, 0x80)
+ INT_HANDLER instruction_access, 0x400, virt=1
+@@ -1225,7 +1225,7 @@ ALT_MMU_FTR_SECTION_END_IFCLR(MMU_FTR_TY
+
+
+ EXC_REAL_BEGIN(instruction_access_slb, 0x480, 0x80)
+- INT_HANDLER instruction_access_slb, 0x480, area=PACA_EXSLB, kvm=1
++ INT_HANDLER instruction_access_slb, 0x480, ool=1, area=PACA_EXSLB, kvm=1
+ EXC_REAL_END(instruction_access_slb, 0x480, 0x80)
+ EXC_VIRT_BEGIN(instruction_access_slb, 0x4480, 0x80)
+ INT_HANDLER instruction_access_slb, 0x480, virt=1, area=PACA_EXSLB
+@@ -1365,17 +1365,17 @@ EXC_REAL_BEGIN(decrementer, 0x900, 0x80)
+ INT_HANDLER decrementer, 0x900, ool=1, bitmask=IRQS_DISABLED, kvm=1
+ EXC_REAL_END(decrementer, 0x900, 0x80)
+ EXC_VIRT_BEGIN(decrementer, 0x4900, 0x80)
+- INT_HANDLER decrementer, 0x900, virt=1, bitmask=IRQS_DISABLED
++ INT_HANDLER decrementer, 0x900, ool=1, virt=1, bitmask=IRQS_DISABLED
+ EXC_VIRT_END(decrementer, 0x4900, 0x80)
+ INT_KVM_HANDLER decrementer, 0x900, EXC_STD, PACA_EXGEN, 0
+ EXC_COMMON_ASYNC(decrementer_common, 0x900, timer_interrupt)
+
+
+ EXC_REAL_BEGIN(hdecrementer, 0x980, 0x80)
+- INT_HANDLER hdecrementer, 0x980, hsrr=EXC_HV, kvm=1
++ INT_HANDLER hdecrementer, 0x980, ool=1, hsrr=EXC_HV, kvm=1
+ EXC_REAL_END(hdecrementer, 0x980, 0x80)
+ EXC_VIRT_BEGIN(hdecrementer, 0x4980, 0x80)
+- INT_HANDLER hdecrementer, 0x980, virt=1, hsrr=EXC_HV, kvm=1
++ INT_HANDLER hdecrementer, 0x980, ool=1, virt=1, hsrr=EXC_HV, kvm=1
+ EXC_VIRT_END(hdecrementer, 0x4980, 0x80)
+ INT_KVM_HANDLER hdecrementer, 0x980, EXC_HV, PACA_EXGEN, 0
+ EXC_COMMON(hdecrementer_common, 0x980, hdec_interrupt)
+@@ -2046,6 +2046,43 @@ TRAMP_REAL_BEGIN(stf_barrier_fallback)
+ .endr
+ blr
+
++TRAMP_REAL_BEGIN(entry_flush_fallback)
++ std r9,PACA_EXRFI+EX_R9(r13)
++ std r10,PACA_EXRFI+EX_R10(r13)
++ std r11,PACA_EXRFI+EX_R11(r13)
++ mfctr r9
++ ld r10,PACA_RFI_FLUSH_FALLBACK_AREA(r13)
++ ld r11,PACA_L1D_FLUSH_SIZE(r13)
++ srdi r11,r11,(7 + 3) /* 128 byte lines, unrolled 8x */
++ mtctr r11
++ DCBT_BOOK3S_STOP_ALL_STREAM_IDS(r11) /* Stop prefetch streams */
++
++ /* order ld/st prior to dcbt stop all streams with flushing */
++ sync
++
++ /*
++ * The load addresses are at staggered offsets within cachelines,
++ * which suits some pipelines better (on others it should not
++ * hurt).
++ */
++1:
++ ld r11,(0x80 + 8)*0(r10)
++ ld r11,(0x80 + 8)*1(r10)
++ ld r11,(0x80 + 8)*2(r10)
++ ld r11,(0x80 + 8)*3(r10)
++ ld r11,(0x80 + 8)*4(r10)
++ ld r11,(0x80 + 8)*5(r10)
++ ld r11,(0x80 + 8)*6(r10)
++ ld r11,(0x80 + 8)*7(r10)
++ addi r10,r10,0x80*8
++ bdnz 1b
++
++ mtctr r9
++ ld r9,PACA_EXRFI+EX_R9(r13)
++ ld r10,PACA_EXRFI+EX_R10(r13)
++ ld r11,PACA_EXRFI+EX_R11(r13)
++ blr
++
+ TRAMP_REAL_BEGIN(rfi_flush_fallback)
+ SET_SCRATCH0(r13);
+ GET_PACA(r13);
+--- a/arch/powerpc/kernel/setup_64.c
++++ b/arch/powerpc/kernel/setup_64.c
+@@ -859,7 +859,9 @@ early_initcall(disable_hardlockup_detect
+ static enum l1d_flush_type enabled_flush_types;
+ static void *l1d_flush_fallback_area;
+ static bool no_rfi_flush;
++static bool no_entry_flush;
+ bool rfi_flush;
++bool entry_flush;
+
+ static int __init handle_no_rfi_flush(char *p)
+ {
+@@ -869,6 +871,14 @@ static int __init handle_no_rfi_flush(ch
+ }
+ early_param("no_rfi_flush", handle_no_rfi_flush);
+
++static int __init handle_no_entry_flush(char *p)
++{
++ pr_info("entry-flush: disabled on command line.");
++ no_entry_flush = true;
++ return 0;
++}
++early_param("no_entry_flush", handle_no_entry_flush);
++
+ /*
+ * The RFI flush is not KPTI, but because users will see doco that says to use
+ * nopti we hijack that option here to also disable the RFI flush.
+@@ -900,6 +910,18 @@ void rfi_flush_enable(bool enable)
+ rfi_flush = enable;
+ }
+
++void entry_flush_enable(bool enable)
++{
++ if (enable) {
++ do_entry_flush_fixups(enabled_flush_types);
++ on_each_cpu(do_nothing, NULL, 1);
++ } else {
++ do_entry_flush_fixups(L1D_FLUSH_NONE);
++ }
++
++ entry_flush = enable;
++}
++
+ static void __ref init_fallback_flush(void)
+ {
+ u64 l1d_size, limit;
+@@ -958,10 +980,19 @@ void setup_rfi_flush(enum l1d_flush_type
+
+ enabled_flush_types = types;
+
+- if (!no_rfi_flush && !cpu_mitigations_off())
++ if (!cpu_mitigations_off() && !no_rfi_flush)
+ rfi_flush_enable(enable);
+ }
+
++void setup_entry_flush(bool enable)
++{
++ if (cpu_mitigations_off())
++ return;
++
++ if (!no_entry_flush)
++ entry_flush_enable(enable);
++}
++
+ #ifdef CONFIG_DEBUG_FS
+ static int rfi_flush_set(void *data, u64 val)
+ {
+@@ -989,9 +1020,36 @@ static int rfi_flush_get(void *data, u64
+
+ DEFINE_SIMPLE_ATTRIBUTE(fops_rfi_flush, rfi_flush_get, rfi_flush_set, "%llu\n");
+
++static int entry_flush_set(void *data, u64 val)
++{
++ bool enable;
++
++ if (val == 1)
++ enable = true;
++ else if (val == 0)
++ enable = false;
++ else
++ return -EINVAL;
++
++ /* Only do anything if we're changing state */
++ if (enable != entry_flush)
++ entry_flush_enable(enable);
++
++ return 0;
++}
++
++static int entry_flush_get(void *data, u64 *val)
++{
++ *val = entry_flush ? 1 : 0;
++ return 0;
++}
++
++DEFINE_SIMPLE_ATTRIBUTE(fops_entry_flush, entry_flush_get, entry_flush_set, "%llu\n");
++
+ static __init int rfi_flush_debugfs_init(void)
+ {
+ debugfs_create_file("rfi_flush", 0600, powerpc_debugfs_root, NULL, &fops_rfi_flush);
++ debugfs_create_file("entry_flush", 0600, powerpc_debugfs_root, NULL, &fops_entry_flush);
+ return 0;
+ }
+ device_initcall(rfi_flush_debugfs_init);
+--- a/arch/powerpc/kernel/vmlinux.lds.S
++++ b/arch/powerpc/kernel/vmlinux.lds.S
+@@ -144,6 +144,13 @@ SECTIONS
+ }
+
+ . = ALIGN(8);
++ __entry_flush_fixup : AT(ADDR(__entry_flush_fixup) - LOAD_OFFSET) {
++ __start___entry_flush_fixup = .;
++ *(__entry_flush_fixup)
++ __stop___entry_flush_fixup = .;
++ }
++
++ . = ALIGN(8);
+ __stf_exit_barrier_fixup : AT(ADDR(__stf_exit_barrier_fixup) - LOAD_OFFSET) {
+ __start___stf_exit_barrier_fixup = .;
+ *(__stf_exit_barrier_fixup)
+--- a/arch/powerpc/lib/feature-fixups.c
++++ b/arch/powerpc/lib/feature-fixups.c
+@@ -228,6 +228,60 @@ void do_stf_barrier_fixups(enum stf_barr
+ do_stf_exit_barrier_fixups(types);
+ }
+
++void do_entry_flush_fixups(enum l1d_flush_type types)
++{
++ unsigned int instrs[3], *dest;
++ long *start, *end;
++ int i;
++
++ start = PTRRELOC(&__start___entry_flush_fixup);
++ end = PTRRELOC(&__stop___entry_flush_fixup);
++
++ instrs[0] = 0x60000000; /* nop */
++ instrs[1] = 0x60000000; /* nop */
++ instrs[2] = 0x60000000; /* nop */
++
++ i = 0;
++ if (types == L1D_FLUSH_FALLBACK) {
++ instrs[i++] = 0x7d4802a6; /* mflr r10 */
++ instrs[i++] = 0x60000000; /* branch patched below */
++ instrs[i++] = 0x7d4803a6; /* mtlr r10 */
++ }
++
++ if (types & L1D_FLUSH_ORI) {
++ instrs[i++] = 0x63ff0000; /* ori 31,31,0 speculation barrier */
++ instrs[i++] = 0x63de0000; /* ori 30,30,0 L1d flush*/
++ }
++
++ if (types & L1D_FLUSH_MTTRIG)
++ instrs[i++] = 0x7c12dba6; /* mtspr TRIG2,r0 (SPR #882) */
++
++ for (i = 0; start < end; start++, i++) {
++ dest = (void *)start + *start;
++
++ pr_devel("patching dest %lx\n", (unsigned long)dest);
++
++ patch_instruction(dest, instrs[0]);
++
++ if (types == L1D_FLUSH_FALLBACK)
++ patch_branch((dest + 1), (unsigned long)&entry_flush_fallback,
++ BRANCH_SET_LINK);
++ else
++ patch_instruction((dest + 1), instrs[1]);
++
++ patch_instruction((dest + 2), instrs[2]);
++ }
++
++ printk(KERN_DEBUG "entry-flush: patched %d locations (%s flush)\n", i,
++ (types == L1D_FLUSH_NONE) ? "no" :
++ (types == L1D_FLUSH_FALLBACK) ? "fallback displacement" :
++ (types & L1D_FLUSH_ORI) ? (types & L1D_FLUSH_MTTRIG)
++ ? "ori+mttrig type"
++ : "ori type" :
++ (types & L1D_FLUSH_MTTRIG) ? "mttrig type"
++ : "unknown");
++}
++
+ void do_rfi_flush_fixups(enum l1d_flush_type types)
+ {
+ unsigned int instrs[3], *dest;
+--- a/arch/powerpc/platforms/powernv/setup.c
++++ b/arch/powerpc/platforms/powernv/setup.c
+@@ -122,12 +122,23 @@ static void pnv_setup_rfi_flush(void)
+ type = L1D_FLUSH_ORI;
+ }
+
++ /*
++ * If we are non-Power9 bare metal, we don't need to flush on kernel
++ * entry: it fixes a P9 specific vulnerability.
++ */
++ if (!pvr_version_is(PVR_POWER9))
++ security_ftr_clear(SEC_FTR_L1D_FLUSH_ENTRY);
++
+ enable = security_ftr_enabled(SEC_FTR_FAVOUR_SECURITY) && \
+ (security_ftr_enabled(SEC_FTR_L1D_FLUSH_PR) || \
+ security_ftr_enabled(SEC_FTR_L1D_FLUSH_HV));
+
+ setup_rfi_flush(type, enable);
+ setup_count_cache_flush();
++
++ enable = security_ftr_enabled(SEC_FTR_FAVOUR_SECURITY) &&
++ security_ftr_enabled(SEC_FTR_L1D_FLUSH_ENTRY);
++ setup_entry_flush(enable);
+ }
+
+ static void __init pnv_setup_arch(void)
+--- a/arch/powerpc/platforms/pseries/setup.c
++++ b/arch/powerpc/platforms/pseries/setup.c
+@@ -561,6 +561,10 @@ void pseries_setup_rfi_flush(void)
+
+ setup_rfi_flush(types, enable);
+ setup_count_cache_flush();
++
++ enable = security_ftr_enabled(SEC_FTR_FAVOUR_SECURITY) &&
++ security_ftr_enabled(SEC_FTR_L1D_FLUSH_ENTRY);
++ setup_entry_flush(enable);
+ }
+
+ #ifdef CONFIG_PCI_IOV
--- /dev/null
+From foo@baz Fri Nov 20 08:16:31 AM CET 2020
+From: Daniel Axtens <dja@axtens.net>
+Date: Fri, 20 Nov 2020 10:35:15 +1100
+Subject: powerpc: Only include kup-radix.h for 64-bit Book3S
+To: stable@vger.kernel.org
+Cc: dja@axtens.net
+Message-ID: <20201119233516.368194-5-dja@axtens.net>
+
+From: Michael Ellerman <mpe@ellerman.id.au>
+
+commit 178d52c6e89c38d0553b0ac8b99927b11eb995b0 upstream.
+
+In kup.h we currently include kup-radix.h for all 64-bit builds, which
+includes Book3S and Book3E. The latter doesn't make sense, Book3E
+never uses the Radix MMU.
+
+This has worked up until now, but almost by accident, and the recent
+uaccess flush changes introduced a build breakage on Book3E because of
+the bad structure of the code.
+
+So disentangle things so that we only use kup-radix.h for Book3S. This
+requires some more stubs in kup.h.
+
+Signed-off-by: Michael Ellerman <mpe@ellerman.id.au>
+Signed-off-by: Daniel Axtens <dja@axtens.net>
+Signed-off-by: Greg Kroah-Hartman <gregkh@linuxfoundation.org>
+---
+ arch/powerpc/include/asm/book3s/64/kup-radix.h | 5 +++--
+ arch/powerpc/include/asm/kup.h | 14 +++++++++++---
+ 2 files changed, 14 insertions(+), 5 deletions(-)
+
+--- a/arch/powerpc/include/asm/book3s/64/kup-radix.h
++++ b/arch/powerpc/include/asm/book3s/64/kup-radix.h
+@@ -11,13 +11,12 @@
+
+ #ifdef __ASSEMBLY__
+
+-.macro kuap_restore_amr gpr
+ #ifdef CONFIG_PPC_KUAP
++.macro kuap_restore_amr gpr
+ BEGIN_MMU_FTR_SECTION_NESTED(67)
+ ld \gpr, STACK_REGS_KUAP(r1)
+ mtspr SPRN_AMR, \gpr
+ END_MMU_FTR_SECTION_NESTED_IFSET(MMU_FTR_RADIX_KUAP, 67)
+-#endif
+ .endm
+
+ .macro kuap_check_amr gpr1, gpr2
+@@ -31,6 +30,7 @@
+ END_MMU_FTR_SECTION_NESTED_IFSET(MMU_FTR_RADIX_KUAP, 67)
+ #endif
+ .endm
++#endif
+
+ .macro kuap_save_amr_and_lock gpr1, gpr2, use_cr, msr_pr_cr
+ #ifdef CONFIG_PPC_KUAP
+@@ -87,6 +87,7 @@ bad_kuap_fault(struct pt_regs *regs, uns
+ "Bug: %s fault blocked by AMR!", is_write ? "Write" : "Read");
+ }
+ #else /* CONFIG_PPC_KUAP */
++static inline void kuap_restore_amr(struct pt_regs *regs, unsigned long amr) { }
+ static inline void set_kuap(unsigned long value) { }
+ #endif /* !CONFIG_PPC_KUAP */
+
+--- a/arch/powerpc/include/asm/kup.h
++++ b/arch/powerpc/include/asm/kup.h
+@@ -6,7 +6,7 @@
+ #define KUAP_WRITE 2
+ #define KUAP_READ_WRITE (KUAP_READ | KUAP_WRITE)
+
+-#ifdef CONFIG_PPC64
++#ifdef CONFIG_PPC_BOOK3S_64
+ #include <asm/book3s/64/kup-radix.h>
+ #endif
+ #ifdef CONFIG_PPC_8xx
+@@ -24,9 +24,15 @@
+ .macro kuap_restore sp, current, gpr1, gpr2, gpr3
+ .endm
+
++.macro kuap_restore_amr gpr
++.endm
++
+ .macro kuap_check current, gpr
+ .endm
+
++.macro kuap_check_amr gpr1, gpr2
++.endm
++
+ #endif
+
+ #else /* !__ASSEMBLY__ */
+@@ -52,17 +58,19 @@ bad_kuap_fault(struct pt_regs *regs, uns
+ return false;
+ }
+
++static inline void kuap_check_amr(void) { }
++
+ /*
+ * book3s/64/kup-radix.h defines these functions for the !KUAP case to flush
+ * the L1D cache after user accesses. Only include the empty stubs for other
+ * platforms.
+ */
+-#ifndef CONFIG_PPC64
++#ifndef CONFIG_PPC_BOOK3S_64
+ static inline void allow_user_access(void __user *to, const void __user *from,
+ unsigned long size, unsigned long dir) { }
+ static inline void prevent_user_access(void __user *to, const void __user *from,
+ unsigned long size, unsigned long dir) { }
+-#endif /* CONFIG_PPC64 */
++#endif /* CONFIG_PPC_BOOK3S_64 */
+ #endif /* CONFIG_PPC_KUAP */
+
+ static inline void allow_read_from_user(const void __user *from, unsigned long size)
--- /dev/null
+From foo@baz Fri Nov 20 08:16:31 AM CET 2020
+From: Daniel Axtens <dja@axtens.net>
+Date: Fri, 20 Nov 2020 10:35:16 +1100
+Subject: selftests/powerpc: entry flush test
+To: stable@vger.kernel.org
+Cc: dja@axtens.net
+Message-ID: <20201119233516.368194-6-dja@axtens.net>
+
+From: Daniel Axtens <dja@axtens.net>
+
+commit 89a83a0c69c81a25ce91002b90ca27ed86132a0a upstream.
+
+Add a test modelled on the RFI flush test which counts the number
+of L1D misses doing a simple syscall with the entry flush on and off.
+
+For simplicity of backporting, this test duplicates a lot of code from
+the upstream rfi_flush. This is cleaned up upstream, but we don't clean
+it up here because it would involve bringing in even more commits.
+
+Signed-off-by: Daniel Axtens <dja@axtens.net>
+Signed-off-by: Greg Kroah-Hartman <gregkh@linuxfoundation.org>
+---
+ arch/powerpc/kernel/setup_64.c | 2
+ arch/powerpc/platforms/powernv/setup.c | 6
+ tools/testing/selftests/powerpc/security/.gitignore | 1
+ tools/testing/selftests/powerpc/security/Makefile | 2
+ tools/testing/selftests/powerpc/security/entry_flush.c | 163 +++++++++++++++++
+ 5 files changed, 170 insertions(+), 4 deletions(-)
+ create mode 100644 tools/testing/selftests/powerpc/security/entry_flush.c
+
+--- a/arch/powerpc/kernel/setup_64.c
++++ b/arch/powerpc/kernel/setup_64.c
+@@ -1025,7 +1025,7 @@ void setup_uaccess_flush(bool enable)
+ return;
+
+ if (!no_uaccess_flush)
+- uaccess_flush_enable(true);
++ uaccess_flush_enable(enable);
+ }
+
+ #ifdef CONFIG_DEBUG_FS
+--- a/arch/powerpc/platforms/powernv/setup.c
++++ b/arch/powerpc/platforms/powernv/setup.c
+@@ -124,10 +124,12 @@ static void pnv_setup_rfi_flush(void)
+
+ /*
+ * If we are non-Power9 bare metal, we don't need to flush on kernel
+- * entry: it fixes a P9 specific vulnerability.
++ * entry or after user access: they fix a P9 specific vulnerability.
+ */
+- if (!pvr_version_is(PVR_POWER9))
++ if (!pvr_version_is(PVR_POWER9)) {
+ security_ftr_clear(SEC_FTR_L1D_FLUSH_ENTRY);
++ security_ftr_clear(SEC_FTR_L1D_FLUSH_UACCESS);
++ }
+
+ enable = security_ftr_enabled(SEC_FTR_FAVOUR_SECURITY) && \
+ (security_ftr_enabled(SEC_FTR_L1D_FLUSH_PR) || \
+--- a/tools/testing/selftests/powerpc/security/.gitignore
++++ b/tools/testing/selftests/powerpc/security/.gitignore
+@@ -1 +1,2 @@
+ rfi_flush
++entry_flush
+--- a/tools/testing/selftests/powerpc/security/Makefile
++++ b/tools/testing/selftests/powerpc/security/Makefile
+@@ -1,6 +1,6 @@
+ # SPDX-License-Identifier: GPL-2.0+
+
+-TEST_GEN_PROGS := rfi_flush
++TEST_GEN_PROGS := rfi_flush entry_flush
+ top_srcdir = ../../../../..
+
+ CFLAGS += -I../../../../../usr/include
+--- /dev/null
++++ b/tools/testing/selftests/powerpc/security/entry_flush.c
+@@ -0,0 +1,163 @@
++// SPDX-License-Identifier: GPL-2.0+
++
++/*
++ * Copyright 2018 IBM Corporation.
++ */
++
++#define __SANE_USERSPACE_TYPES__
++
++#include <sys/types.h>
++#include <stdint.h>
++#include <malloc.h>
++#include <unistd.h>
++#include <signal.h>
++#include <stdlib.h>
++#include <string.h>
++#include <stdio.h>
++#include "utils.h"
++
++#define CACHELINE_SIZE 128
++
++struct perf_event_read {
++ __u64 nr;
++ __u64 l1d_misses;
++};
++
++static inline __u64 load(void *addr)
++{
++ __u64 tmp;
++
++ asm volatile("ld %0,0(%1)" : "=r"(tmp) : "b"(addr));
++
++ return tmp;
++}
++
++static void syscall_loop(char *p, unsigned long iterations,
++ unsigned long zero_size)
++{
++ for (unsigned long i = 0; i < iterations; i++) {
++ for (unsigned long j = 0; j < zero_size; j += CACHELINE_SIZE)
++ load(p + j);
++ getppid();
++ }
++}
++
++int entry_flush_test(void)
++{
++ char *p;
++ int repetitions = 10;
++ int fd, passes = 0, iter, rc = 0;
++ struct perf_event_read v;
++ __u64 l1d_misses_total = 0;
++ unsigned long iterations = 100000, zero_size = 24 * 1024;
++ unsigned long l1d_misses_expected;
++ int rfi_flush_orig;
++ int entry_flush, entry_flush_orig;
++
++ SKIP_IF(geteuid() != 0);
++
++ // The PMU event we use only works on Power7 or later
++ SKIP_IF(!have_hwcap(PPC_FEATURE_ARCH_2_06));
++
++ if (read_debugfs_file("powerpc/rfi_flush", &rfi_flush_orig) < 0) {
++ perror("Unable to read powerpc/rfi_flush debugfs file");
++ SKIP_IF(1);
++ }
++
++ if (read_debugfs_file("powerpc/entry_flush", &entry_flush_orig) < 0) {
++ perror("Unable to read powerpc/entry_flush debugfs file");
++ SKIP_IF(1);
++ }
++
++ if (rfi_flush_orig != 0) {
++ if (write_debugfs_file("powerpc/rfi_flush", 0) < 0) {
++ perror("error writing to powerpc/rfi_flush debugfs file");
++ FAIL_IF(1);
++ }
++ }
++
++ entry_flush = entry_flush_orig;
++
++ fd = perf_event_open_counter(PERF_TYPE_RAW, /* L1d miss */ 0x400f0, -1);
++ FAIL_IF(fd < 0);
++
++ p = (char *)memalign(zero_size, CACHELINE_SIZE);
++
++ FAIL_IF(perf_event_enable(fd));
++
++ // disable L1 prefetching
++ set_dscr(1);
++
++ iter = repetitions;
++
++ /*
++ * We expect to see l1d miss for each cacheline access when entry_flush
++ * is set. Allow a small variation on this.
++ */
++ l1d_misses_expected = iterations * (zero_size / CACHELINE_SIZE - 2);
++
++again:
++ FAIL_IF(perf_event_reset(fd));
++
++ syscall_loop(p, iterations, zero_size);
++
++ FAIL_IF(read(fd, &v, sizeof(v)) != sizeof(v));
++
++ if (entry_flush && v.l1d_misses >= l1d_misses_expected)
++ passes++;
++ else if (!entry_flush && v.l1d_misses < (l1d_misses_expected / 2))
++ passes++;
++
++ l1d_misses_total += v.l1d_misses;
++
++ while (--iter)
++ goto again;
++
++ if (passes < repetitions) {
++ printf("FAIL (L1D misses with entry_flush=%d: %llu %c %lu) [%d/%d failures]\n",
++ entry_flush, l1d_misses_total, entry_flush ? '<' : '>',
++ entry_flush ? repetitions * l1d_misses_expected :
++ repetitions * l1d_misses_expected / 2,
++ repetitions - passes, repetitions);
++ rc = 1;
++ } else
++ printf("PASS (L1D misses with entry_flush=%d: %llu %c %lu) [%d/%d pass]\n",
++ entry_flush, l1d_misses_total, entry_flush ? '>' : '<',
++ entry_flush ? repetitions * l1d_misses_expected :
++ repetitions * l1d_misses_expected / 2,
++ passes, repetitions);
++
++ if (entry_flush == entry_flush_orig) {
++ entry_flush = !entry_flush_orig;
++ if (write_debugfs_file("powerpc/entry_flush", entry_flush) < 0) {
++ perror("error writing to powerpc/entry_flush debugfs file");
++ return 1;
++ }
++ iter = repetitions;
++ l1d_misses_total = 0;
++ passes = 0;
++ goto again;
++ }
++
++ perf_event_disable(fd);
++ close(fd);
++
++ set_dscr(0);
++
++ if (write_debugfs_file("powerpc/rfi_flush", rfi_flush_orig) < 0) {
++ perror("unable to restore original value of powerpc/rfi_flush debugfs file");
++ return 1;
++ }
++
++ if (write_debugfs_file("powerpc/entry_flush", entry_flush_orig) < 0) {
++ perror("unable to restore original value of powerpc/entry_flush debugfs file");
++ return 1;
++ }
++
++ return rc;
++}
++
++int main(int argc, char *argv[])
++{
++ return test_harness(entry_flush_test, "entry_flush_test");
++}
--- /dev/null
+From foo@baz Fri Nov 20 08:16:31 AM CET 2020
+From: Daniel Axtens <dja@axtens.net>
+Date: Fri, 20 Nov 2020 10:35:12 +1100
+Subject: selftests/powerpc: rfi_flush: disable entry flush if present
+To: stable@vger.kernel.org
+Cc: dja@axtens.net
+Message-ID: <20201119233516.368194-2-dja@axtens.net>
+
+From: Russell Currey <ruscur@russell.cc>
+
+commit fcb48454c23c5679d1a2e252f127642e91b05cbe upstream.
+
+We are about to add an entry flush. The rfi (exit) flush test measures
+the number of L1D flushes over a syscall with the RFI flush enabled and
+disabled. But if the entry flush is also enabled, the effect of enabling
+and disabling the RFI flush is masked.
+
+If there is a debugfs entry for the entry flush, disable it during the RFI
+flush and restore it later.
+
+Reported-by: Spoorthy S <spoorts2@in.ibm.com>
+Signed-off-by: Russell Currey <ruscur@russell.cc>
+Signed-off-by: Daniel Axtens <dja@axtens.net>
+Signed-off-by: Greg Kroah-Hartman <gregkh@linuxfoundation.org>
+---
+ tools/testing/selftests/powerpc/security/rfi_flush.c | 35 +++++++++++++++----
+ 1 file changed, 29 insertions(+), 6 deletions(-)
+
+--- a/tools/testing/selftests/powerpc/security/rfi_flush.c
++++ b/tools/testing/selftests/powerpc/security/rfi_flush.c
+@@ -50,16 +50,30 @@ int rfi_flush_test(void)
+ __u64 l1d_misses_total = 0;
+ unsigned long iterations = 100000, zero_size = 24 * 1024;
+ unsigned long l1d_misses_expected;
+- int rfi_flush_org, rfi_flush;
++ int rfi_flush_orig, rfi_flush;
++ int have_entry_flush, entry_flush_orig;
+
+ SKIP_IF(geteuid() != 0);
+
+- if (read_debugfs_file("powerpc/rfi_flush", &rfi_flush_org)) {
++ if (read_debugfs_file("powerpc/rfi_flush", &rfi_flush_orig) < 0) {
+ perror("Unable to read powerpc/rfi_flush debugfs file");
+ SKIP_IF(1);
+ }
+
+- rfi_flush = rfi_flush_org;
++ if (read_debugfs_file("powerpc/entry_flush", &entry_flush_orig) < 0) {
++ have_entry_flush = 0;
++ } else {
++ have_entry_flush = 1;
++
++ if (entry_flush_orig != 0) {
++ if (write_debugfs_file("powerpc/entry_flush", 0) < 0) {
++ perror("error writing to powerpc/entry_flush debugfs file");
++ return 1;
++ }
++ }
++ }
++
++ rfi_flush = rfi_flush_orig;
+
+ fd = perf_event_open_counter(PERF_TYPE_RAW, /* L1d miss */ 0x400f0, -1);
+ FAIL_IF(fd < 0);
+@@ -68,6 +82,7 @@ int rfi_flush_test(void)
+
+ FAIL_IF(perf_event_enable(fd));
+
++ // disable L1 prefetching
+ set_dscr(1);
+
+ iter = repetitions;
+@@ -109,8 +124,8 @@ again:
+ repetitions * l1d_misses_expected / 2,
+ passes, repetitions);
+
+- if (rfi_flush == rfi_flush_org) {
+- rfi_flush = !rfi_flush_org;
++ if (rfi_flush == rfi_flush_orig) {
++ rfi_flush = !rfi_flush_orig;
+ if (write_debugfs_file("powerpc/rfi_flush", rfi_flush) < 0) {
+ perror("error writing to powerpc/rfi_flush debugfs file");
+ return 1;
+@@ -126,11 +141,19 @@ again:
+
+ set_dscr(0);
+
+- if (write_debugfs_file("powerpc/rfi_flush", rfi_flush_org) < 0) {
++ if (write_debugfs_file("powerpc/rfi_flush", rfi_flush_orig) < 0) {
+ perror("unable to restore original value of powerpc/rfi_flush debugfs file");
+ return 1;
+ }
+
++ if (have_entry_flush) {
++ if (write_debugfs_file("powerpc/entry_flush", entry_flush_orig) < 0) {
++ perror("unable to restore original value of powerpc/entry_flush "
++ "debugfs file");
++ return 1;
++ }
++ }
++
+ return rc;
+ }
+
--- /dev/null
+selftests-powerpc-rfi_flush-disable-entry-flush-if-present.patch
+powerpc-64s-flush-l1d-on-kernel-entry.patch
+powerpc-64s-flush-l1d-after-user-accesses.patch
+powerpc-only-include-kup-radix.h-for-64-bit-book3s.patch
+selftests-powerpc-entry-flush-test.patch
projects / thirdparty / kernel / stable-queue.git / commitdiff
