--- /dev/null
+From foo@baz Fri Nov 20 08:28:41 AM CET 2020
+From: Daniel Axtens <dja@axtens.net>
+Date: Fri, 20 Nov 2020 11:06:57 +1100
+Subject: powerpc/64s: Define MASKABLE_RELON_EXCEPTION_PSERIES_OOL
+To: stable@vger.kernel.org
+Cc: dja@axtens.net
+Message-ID: <20201120000704.374811-2-dja@axtens.net>
+
+From: Daniel Axtens <dja@axtens.net>
+
+Add a definition provided by mpe and fixed up for 4.4. It doesn't exist
+for 4.4 and we'd quite like to use it.
+
+Signed-off-by: Daniel Axtens <dja@axtens.net>
+Signed-off-by: Greg Kroah-Hartman <gregkh@linuxfoundation.org>
+---
+ arch/powerpc/include/asm/exception-64s.h | 6 ++++++
+ 1 file changed, 6 insertions(+)
+
+--- a/arch/powerpc/include/asm/exception-64s.h
++++ b/arch/powerpc/include/asm/exception-64s.h
+@@ -597,6 +597,12 @@ label##_relon_hv: \
+ EXCEPTION_PROLOG_1(PACA_EXGEN, SOFTEN_NOTEST_HV, vec); \
+ EXCEPTION_PROLOG_PSERIES_1(label##_common, EXC_HV);
+
++#define MASKABLE_RELON_EXCEPTION_PSERIES_OOL(vec, label) \
++ .globl label##_relon_pSeries; \
++label##_relon_pSeries: \
++ EXCEPTION_PROLOG_1(PACA_EXGEN, SOFTEN_NOTEST_PR, vec); \
++ EXCEPTION_PROLOG_PSERIES_1(label##_common, EXC_STD)
++
+ /*
+ * Our exception common code can be passed various "additions"
+ * to specify the behaviour of interrupts, whether to kick the
--- /dev/null
+From foo@baz Fri Nov 20 08:28:41 AM CET 2020
+From: Daniel Axtens <dja@axtens.net>
+Date: Fri, 20 Nov 2020 11:07:04 +1100
+Subject: powerpc/64s: flush L1D after user accesses
+To: stable@vger.kernel.org
+Cc: dja@axtens.net
+Message-ID: <20201120000704.374811-9-dja@axtens.net>
+
+From: Nicholas Piggin <npiggin@gmail.com>
+
+commit 9a32a7e78bd0cd9a9b6332cbdc345ee5ffd0c5de upstream.
+
+IBM Power9 processors can speculatively operate on data in the L1 cache before
+it has been completely validated, via a way-prediction mechanism. It is not possible
+for an attacker to determine the contents of impermissible memory using this method,
+since these systems implement a combination of hardware and software security measures
+to prevent scenarios where protected data could be leaked.
+
+However these measures don't address the scenario where an attacker induces
+the operating system to speculatively execute instructions using data that the
+attacker controls. This can be used for example to speculatively bypass "kernel
+user access prevention" techniques, as discovered by Anthony Steinhauser of
+Google's Safeside Project. This is not an attack by itself, but there is a possibility
+it could be used in conjunction with side-channels or other weaknesses in the
+privileged code to construct an attack.
+
+This issue can be mitigated by flushing the L1 cache between privilege boundaries
+of concern. This patch flushes the L1 cache after user accesses.
+
+This is part of the fix for CVE-2020-4788.
+
+Signed-off-by: Nicholas Piggin <npiggin@gmail.com>
+Signed-off-by: Daniel Axtens <dja@axtens.net>
+Signed-off-by: Greg Kroah-Hartman <gregkh@linuxfoundation.org>
+---
+ Documentation/kernel-parameters.txt | 4 +
+ arch/powerpc/include/asm/book3s/64/kup-radix.h | 23 ++++++
+ arch/powerpc/include/asm/feature-fixups.h | 9 ++
+ arch/powerpc/include/asm/kup.h | 4 +
+ arch/powerpc/include/asm/security_features.h | 3
+ arch/powerpc/include/asm/setup.h | 1
+ arch/powerpc/kernel/exceptions-64s.S | 86 +++++++------------------
+ arch/powerpc/kernel/ppc_ksyms.c | 7 ++
+ arch/powerpc/kernel/setup_64.c | 80 +++++++++++++++++++++++
+ arch/powerpc/kernel/vmlinux.lds.S | 7 ++
+ arch/powerpc/lib/feature-fixups.c | 50 ++++++++++++++
+ arch/powerpc/platforms/powernv/setup.c | 7 +-
+ arch/powerpc/platforms/pseries/setup.c | 4 +
+ 13 files changed, 224 insertions(+), 61 deletions(-)
+ create mode 100644 arch/powerpc/include/asm/book3s/64/kup-radix.h
+
+--- a/Documentation/kernel-parameters.txt
++++ b/Documentation/kernel-parameters.txt
+@@ -2197,6 +2197,7 @@ bytes respectively. Such letter suffixes
+ mds=off [X86]
+ tsx_async_abort=off [X86]
+ no_entry_flush [PPC]
++ no_uaccess_flush [PPC]
+
+ auto (default)
+ Mitigate all CPU vulnerabilities, but leave SMT
+@@ -2521,6 +2522,9 @@ bytes respectively. Such letter suffixes
+ nospec_store_bypass_disable
+ [HW] Disable all mitigations for the Speculative Store Bypass vulnerability
+
++ no_uaccess_flush
++ [PPC] Don't flush the L1-D cache after accessing user data.
++
+ noxsave [BUGS=X86] Disables x86 extended register state save
+ and restore using xsave. The kernel will fallback to
+ enabling legacy floating-point and sse state.
+--- /dev/null
++++ b/arch/powerpc/include/asm/book3s/64/kup-radix.h
+@@ -0,0 +1,23 @@
++/* SPDX-License-Identifier: GPL-2.0 */
++#ifndef _ASM_POWERPC_BOOK3S_64_KUP_RADIX_H
++#define _ASM_POWERPC_BOOK3S_64_KUP_RADIX_H
++#include <linux/jump_label.h>
++
++DECLARE_STATIC_KEY_FALSE(uaccess_flush_key);
++
++/* Prototype for function defined in exceptions-64s.S */
++void do_uaccess_flush(void);
++
++static __always_inline void allow_user_access(void __user *to, const void __user *from,
++ unsigned long size)
++{
++}
++
++static inline void prevent_user_access(void __user *to, const void __user *from,
++ unsigned long size)
++{
++ if (static_branch_unlikely(&uaccess_flush_key))
++ do_uaccess_flush();
++}
++
++#endif /* _ASM_POWERPC_BOOK3S_64_KUP_RADIX_H */
+--- a/arch/powerpc/include/asm/feature-fixups.h
++++ b/arch/powerpc/include/asm/feature-fixups.h
+@@ -200,6 +200,14 @@ label##3: \
+ FTR_ENTRY_OFFSET 955b-956b; \
+ .popsection;
+
++#define UACCESS_FLUSH_FIXUP_SECTION \
++959: \
++ .pushsection __uaccess_flush_fixup,"a"; \
++ .align 2; \
++960: \
++ FTR_ENTRY_OFFSET 959b-960b; \
++ .popsection;
++
+ #define ENTRY_FLUSH_FIXUP_SECTION \
+ 957: \
+ .pushsection __entry_flush_fixup,"a"; \
+@@ -242,6 +250,7 @@ extern long stf_barrier_fallback;
+ extern long entry_flush_fallback;
+ extern long __start___stf_entry_barrier_fixup, __stop___stf_entry_barrier_fixup;
+ extern long __start___stf_exit_barrier_fixup, __stop___stf_exit_barrier_fixup;
++extern long __start___uaccess_flush_fixup, __stop___uaccess_flush_fixup;
+ extern long __start___entry_flush_fixup, __stop___entry_flush_fixup;
+ extern long __start___rfi_flush_fixup, __stop___rfi_flush_fixup;
+ extern long __start___barrier_nospec_fixup, __stop___barrier_nospec_fixup;
+--- a/arch/powerpc/include/asm/kup.h
++++ b/arch/powerpc/include/asm/kup.h
+@@ -6,10 +6,14 @@
+
+ #include <asm/pgtable.h>
+
++#ifdef CONFIG_PPC_BOOK3S_64
++#include <asm/book3s/64/kup-radix.h>
++#else
+ static inline void allow_user_access(void __user *to, const void __user *from,
+ unsigned long size) { }
+ static inline void prevent_user_access(void __user *to, const void __user *from,
+ unsigned long size) { }
++#endif /* CONFIG_PPC_BOOK3S_64 */
+
+ static inline void allow_read_from_user(const void __user *from, unsigned long size)
+ {
+--- a/arch/powerpc/include/asm/security_features.h
++++ b/arch/powerpc/include/asm/security_features.h
+@@ -87,6 +87,8 @@ static inline bool security_ftr_enabled(
+ // The L1-D cache should be flushed when entering the kernel
+ #define SEC_FTR_L1D_FLUSH_ENTRY 0x0000000000004000ull
+
++// The L1-D cache should be flushed after user accesses from the kernel
++#define SEC_FTR_L1D_FLUSH_UACCESS 0x0000000000008000ull
+
+ // Features enabled by default
+ #define SEC_FTR_DEFAULT \
+@@ -94,6 +96,7 @@ static inline bool security_ftr_enabled(
+ SEC_FTR_L1D_FLUSH_PR | \
+ SEC_FTR_BNDS_CHK_SPEC_BAR | \
+ SEC_FTR_L1D_FLUSH_ENTRY | \
++ SEC_FTR_L1D_FLUSH_UACCESS | \
+ SEC_FTR_FAVOUR_SECURITY)
+
+ #endif /* _ASM_POWERPC_SECURITY_FEATURES_H */
+--- a/arch/powerpc/include/asm/setup.h
++++ b/arch/powerpc/include/asm/setup.h
+@@ -46,6 +46,7 @@ void setup_barrier_nospec(void);
+ #else
+ static inline void setup_barrier_nospec(void) { };
+ #endif
++void do_uaccess_flush_fixups(enum l1d_flush_type types);
+ void do_entry_flush_fixups(enum l1d_flush_type types);
+ void do_barrier_nospec_fixups(bool enable);
+ extern bool barrier_nospec_enabled;
+--- a/arch/powerpc/kernel/exceptions-64s.S
++++ b/arch/powerpc/kernel/exceptions-64s.S
+@@ -1630,14 +1630,9 @@ stf_barrier_fallback:
+ .endr
+ blr
+
+- .globl rfi_flush_fallback
+-rfi_flush_fallback:
+- SET_SCRATCH0(r13);
+- GET_PACA(r13);
+- std r9,PACA_EXRFI+EX_R9(r13)
+- std r10,PACA_EXRFI+EX_R10(r13)
+- std r11,PACA_EXRFI+EX_R11(r13)
+- mfctr r9
++
++/* Clobbers r10, r11, ctr */
++.macro L1D_DISPLACEMENT_FLUSH
+ ld r10,PACA_RFI_FLUSH_FALLBACK_AREA(r13)
+ ld r11,PACA_L1D_FLUSH_SIZE(r13)
+ srdi r11,r11,(7 + 3) /* 128 byte lines, unrolled 8x */
+@@ -1663,7 +1658,18 @@ rfi_flush_fallback:
+ ld r11,(0x80 + 8)*7(r10)
+ addi r10,r10,0x80*8
+ bdnz 1b
++.endm
++
+
++ .globl rfi_flush_fallback
++rfi_flush_fallback:
++ SET_SCRATCH0(r13);
++ GET_PACA(r13);
++ std r9,PACA_EXRFI+EX_R9(r13)
++ std r10,PACA_EXRFI+EX_R10(r13)
++ std r11,PACA_EXRFI+EX_R11(r13)
++ mfctr r9
++ L1D_DISPLACEMENT_FLUSH
+ mtctr r9
+ ld r9,PACA_EXRFI+EX_R9(r13)
+ ld r10,PACA_EXRFI+EX_R10(r13)
+@@ -1679,32 +1685,7 @@ hrfi_flush_fallback:
+ std r10,PACA_EXRFI+EX_R10(r13)
+ std r11,PACA_EXRFI+EX_R11(r13)
+ mfctr r9
+- ld r10,PACA_RFI_FLUSH_FALLBACK_AREA(r13)
+- ld r11,PACA_L1D_FLUSH_SIZE(r13)
+- srdi r11,r11,(7 + 3) /* 128 byte lines, unrolled 8x */
+- mtctr r11
+- DCBT_STOP_ALL_STREAM_IDS(r11) /* Stop prefetch streams */
+-
+- /* order ld/st prior to dcbt stop all streams with flushing */
+- sync
+-
+- /*
+- * The load adresses are at staggered offsets within cachelines,
+- * which suits some pipelines better (on others it should not
+- * hurt).
+- */
+-1:
+- ld r11,(0x80 + 8)*0(r10)
+- ld r11,(0x80 + 8)*1(r10)
+- ld r11,(0x80 + 8)*2(r10)
+- ld r11,(0x80 + 8)*3(r10)
+- ld r11,(0x80 + 8)*4(r10)
+- ld r11,(0x80 + 8)*5(r10)
+- ld r11,(0x80 + 8)*6(r10)
+- ld r11,(0x80 + 8)*7(r10)
+- addi r10,r10,0x80*8
+- bdnz 1b
+-
++ L1D_DISPLACEMENT_FLUSH
+ mtctr r9
+ ld r9,PACA_EXRFI+EX_R9(r13)
+ ld r10,PACA_EXRFI+EX_R10(r13)
+@@ -1718,38 +1699,14 @@ entry_flush_fallback:
+ std r10,PACA_EXRFI+EX_R10(r13)
+ std r11,PACA_EXRFI+EX_R11(r13)
+ mfctr r9
+- ld r10,PACA_RFI_FLUSH_FALLBACK_AREA(r13)
+- ld r11,PACA_L1D_FLUSH_SIZE(r13)
+- srdi r11,r11,(7 + 3) /* 128 byte lines, unrolled 8x */
+- mtctr r11
+- DCBT_STOP_ALL_STREAM_IDS(r11) /* Stop prefetch streams */
+-
+- /* order ld/st prior to dcbt stop all streams with flushing */
+- sync
+-
+- /*
+- * The load addresses are at staggered offsets within cachelines,
+- * which suits some pipelines better (on others it should not
+- * hurt).
+- */
+-1:
+- ld r11,(0x80 + 8)*0(r10)
+- ld r11,(0x80 + 8)*1(r10)
+- ld r11,(0x80 + 8)*2(r10)
+- ld r11,(0x80 + 8)*3(r10)
+- ld r11,(0x80 + 8)*4(r10)
+- ld r11,(0x80 + 8)*5(r10)
+- ld r11,(0x80 + 8)*6(r10)
+- ld r11,(0x80 + 8)*7(r10)
+- addi r10,r10,0x80*8
+- bdnz 1b
+-
++ L1D_DISPLACEMENT_FLUSH
+ mtctr r9
+ ld r9,PACA_EXRFI+EX_R9(r13)
+ ld r10,PACA_EXRFI+EX_R10(r13)
+ ld r11,PACA_EXRFI+EX_R11(r13)
+ blr
+
++
+ /*
+ * Hash table stuff
+ */
+@@ -1909,3 +1866,12 @@ END_FTR_SECTION_IFSET(CPU_FTR_CFAR)
+ 1: addi r3,r1,STACK_FRAME_OVERHEAD
+ bl kernel_bad_stack
+ b 1b
++
++_KPROBE(do_uaccess_flush)
++ UACCESS_FLUSH_FIXUP_SECTION
++ nop
++ nop
++ nop
++ blr
++ L1D_DISPLACEMENT_FLUSH
++ blr
+--- a/arch/powerpc/kernel/ppc_ksyms.c
++++ b/arch/powerpc/kernel/ppc_ksyms.c
+@@ -6,6 +6,9 @@
+ #include <asm/cacheflush.h>
+ #include <asm/epapr_hcalls.h>
+ #include <asm/uaccess.h>
++#ifdef CONFIG_PPC64
++#include <asm/book3s/64/kup-radix.h>
++#endif
+
+ EXPORT_SYMBOL(flush_dcache_range);
+ EXPORT_SYMBOL(flush_icache_range);
+@@ -46,3 +49,7 @@ EXPORT_SYMBOL(epapr_hypercall_start);
+ EXPORT_SYMBOL(current_stack_pointer);
+
+ EXPORT_SYMBOL(__arch_clear_user);
++
++#ifdef CONFIG_PPC64
++EXPORT_SYMBOL(do_uaccess_flush);
++#endif
+--- a/arch/powerpc/kernel/setup_64.c
++++ b/arch/powerpc/kernel/setup_64.c
+@@ -845,8 +845,12 @@ static enum l1d_flush_type enabled_flush
+ static void *l1d_flush_fallback_area;
+ static bool no_rfi_flush;
+ static bool no_entry_flush;
++static bool no_uaccess_flush;
+ bool rfi_flush;
+ bool entry_flush;
++bool uaccess_flush;
++DEFINE_STATIC_KEY_FALSE(uaccess_flush_key);
++EXPORT_SYMBOL(uaccess_flush_key);
+
+ static int __init handle_no_rfi_flush(char *p)
+ {
+@@ -864,6 +868,14 @@ static int __init handle_no_entry_flush(
+ }
+ early_param("no_entry_flush", handle_no_entry_flush);
+
++static int __init handle_no_uaccess_flush(char *p)
++{
++ pr_info("uaccess-flush: disabled on command line.");
++ no_uaccess_flush = true;
++ return 0;
++}
++early_param("no_uaccess_flush", handle_no_uaccess_flush);
++
+ /*
+ * The RFI flush is not KPTI, but because users will see doco that says to use
+ * nopti we hijack that option here to also disable the RFI flush.
+@@ -907,6 +919,23 @@ void entry_flush_enable(bool enable)
+ entry_flush = enable;
+ }
+
++void uaccess_flush_enable(bool enable)
++{
++ if (enable) {
++ do_uaccess_flush_fixups(enabled_flush_types);
++ if (static_key_initialized)
++ static_branch_enable(&uaccess_flush_key);
++ else
++ printk(KERN_DEBUG "uaccess-flush: deferring static key until after static key initialization\n");
++ on_each_cpu(do_nothing, NULL, 1);
++ } else {
++ static_branch_disable(&uaccess_flush_key);
++ do_uaccess_flush_fixups(L1D_FLUSH_NONE);
++ }
++
++ uaccess_flush = enable;
++}
++
+ static void __ref init_fallback_flush(void)
+ {
+ u64 l1d_size, limit;
+@@ -961,6 +990,15 @@ void setup_entry_flush(bool enable)
+ entry_flush_enable(enable);
+ }
+
++void setup_uaccess_flush(bool enable)
++{
++ if (cpu_mitigations_off())
++ return;
++
++ if (!no_uaccess_flush)
++ uaccess_flush_enable(enable);
++}
++
+ #ifdef CONFIG_DEBUG_FS
+ static int rfi_flush_set(void *data, u64 val)
+ {
+@@ -1014,12 +1052,54 @@ static int entry_flush_get(void *data, u
+
+ DEFINE_SIMPLE_ATTRIBUTE(fops_entry_flush, entry_flush_get, entry_flush_set, "%llu\n");
+
++static int uaccess_flush_set(void *data, u64 val)
++{
++ bool enable;
++
++ if (val == 1)
++ enable = true;
++ else if (val == 0)
++ enable = false;
++ else
++ return -EINVAL;
++
++ /* Only do anything if we're changing state */
++ if (enable != uaccess_flush)
++ uaccess_flush_enable(enable);
++
++ return 0;
++}
++
++static int uaccess_flush_get(void *data, u64 *val)
++{
++ *val = uaccess_flush ? 1 : 0;
++ return 0;
++}
++
++DEFINE_SIMPLE_ATTRIBUTE(fops_uaccess_flush, uaccess_flush_get, uaccess_flush_set, "%llu\n");
++
++
+ static __init int rfi_flush_debugfs_init(void)
+ {
+ debugfs_create_file("rfi_flush", 0600, powerpc_debugfs_root, NULL, &fops_rfi_flush);
+ debugfs_create_file("entry_flush", 0600, powerpc_debugfs_root, NULL, &fops_entry_flush);
++ debugfs_create_file("uaccess_flush", 0600, powerpc_debugfs_root, NULL, &fops_uaccess_flush);
+ return 0;
+ }
+ device_initcall(rfi_flush_debugfs_init);
+ #endif
++
++/*
++ * setup_uaccess_flush runs before jump_label_init, so we can't do the setup
++ * there. Do it now instead.
++ */
++static __init int uaccess_flush_static_key_init(void)
++{
++ if (uaccess_flush) {
++ printk(KERN_DEBUG "uaccess-flush: switching on static key\n");
++ static_branch_enable(&uaccess_flush_key);
++ }
++ return 0;
++}
++early_initcall(uaccess_flush_static_key_init);
+ #endif /* CONFIG_PPC_BOOK3S_64 */
+--- a/arch/powerpc/kernel/vmlinux.lds.S
++++ b/arch/powerpc/kernel/vmlinux.lds.S
+@@ -81,6 +81,13 @@ SECTIONS
+ }
+
+ . = ALIGN(8);
++ __uaccess_flush_fixup : AT(ADDR(__uaccess_flush_fixup) - LOAD_OFFSET) {
++ __start___uaccess_flush_fixup = .;
++ *(__uaccess_flush_fixup)
++ __stop___uaccess_flush_fixup = .;
++ }
++
++ . = ALIGN(8);
+ __entry_flush_fixup : AT(ADDR(__entry_flush_fixup) - LOAD_OFFSET) {
+ __start___entry_flush_fixup = .;
+ *(__entry_flush_fixup)
+--- a/arch/powerpc/lib/feature-fixups.c
++++ b/arch/powerpc/lib/feature-fixups.c
+@@ -229,6 +229,56 @@ void do_stf_barrier_fixups(enum stf_barr
+ do_stf_exit_barrier_fixups(types);
+ }
+
++void do_uaccess_flush_fixups(enum l1d_flush_type types)
++{
++ unsigned int instrs[4], *dest;
++ long *start, *end;
++ int i;
++
++ start = PTRRELOC(&__start___uaccess_flush_fixup);
++ end = PTRRELOC(&__stop___uaccess_flush_fixup);
++
++ instrs[0] = 0x60000000; /* nop */
++ instrs[1] = 0x60000000; /* nop */
++ instrs[2] = 0x60000000; /* nop */
++ instrs[3] = 0x4e800020; /* blr */
++
++ i = 0;
++ if (types == L1D_FLUSH_FALLBACK) {
++ instrs[3] = 0x60000000; /* nop */
++ /* fallthrough to fallback flush */
++ }
++
++ if (types & L1D_FLUSH_ORI) {
++ instrs[i++] = 0x63ff0000; /* ori 31,31,0 speculation barrier */
++ instrs[i++] = 0x63de0000; /* ori 30,30,0 L1d flush*/
++ }
++
++ if (types & L1D_FLUSH_MTTRIG)
++ instrs[i++] = 0x7c12dba6; /* mtspr TRIG2,r0 (SPR #882) */
++
++ for (i = 0; start < end; start++, i++) {
++ dest = (void *)start + *start;
++
++ pr_devel("patching dest %lx\n", (unsigned long)dest);
++
++ patch_instruction(dest, instrs[0]);
++
++ patch_instruction((dest + 1), instrs[1]);
++ patch_instruction((dest + 2), instrs[2]);
++ patch_instruction((dest + 3), instrs[3]);
++ }
++
++ printk(KERN_DEBUG "uaccess-flush: patched %d locations (%s flush)\n", i,
++ (types == L1D_FLUSH_NONE) ? "no" :
++ (types == L1D_FLUSH_FALLBACK) ? "fallback displacement" :
++ (types & L1D_FLUSH_ORI) ? (types & L1D_FLUSH_MTTRIG)
++ ? "ori+mttrig type"
++ : "ori type" :
++ (types & L1D_FLUSH_MTTRIG) ? "mttrig type"
++ : "unknown");
++}
++
+ void do_entry_flush_fixups(enum l1d_flush_type types)
+ {
+ unsigned int instrs[3], *dest;
+--- a/arch/powerpc/platforms/powernv/setup.c
++++ b/arch/powerpc/platforms/powernv/setup.c
+@@ -126,9 +126,10 @@ static void pnv_setup_rfi_flush(void)
+
+ /*
+ * 4.4 doesn't support Power9 bare metal, so we don't need to flush
+- * here - the flush fixes a P9 specific vulnerability.
++ * here - the flushes fix a P9 specific vulnerability.
+ */
+ security_ftr_clear(SEC_FTR_L1D_FLUSH_ENTRY);
++ security_ftr_clear(SEC_FTR_L1D_FLUSH_UACCESS);
+
+ enable = security_ftr_enabled(SEC_FTR_FAVOUR_SECURITY) && \
+ (security_ftr_enabled(SEC_FTR_L1D_FLUSH_PR) || \
+@@ -140,6 +141,10 @@ static void pnv_setup_rfi_flush(void)
+ enable = security_ftr_enabled(SEC_FTR_FAVOUR_SECURITY) &&
+ security_ftr_enabled(SEC_FTR_L1D_FLUSH_ENTRY);
+ setup_entry_flush(enable);
++
++ enable = security_ftr_enabled(SEC_FTR_FAVOUR_SECURITY) &&
++ security_ftr_enabled(SEC_FTR_L1D_FLUSH_UACCESS);
++ setup_uaccess_flush(enable);
+ }
+
+ static void __init pnv_setup_arch(void)
+--- a/arch/powerpc/platforms/pseries/setup.c
++++ b/arch/powerpc/platforms/pseries/setup.c
+@@ -588,6 +588,10 @@ void pseries_setup_rfi_flush(void)
+ enable = security_ftr_enabled(SEC_FTR_FAVOUR_SECURITY) &&
+ security_ftr_enabled(SEC_FTR_L1D_FLUSH_ENTRY);
+ setup_entry_flush(enable);
++
++ enable = security_ftr_enabled(SEC_FTR_FAVOUR_SECURITY) &&
++ security_ftr_enabled(SEC_FTR_L1D_FLUSH_UACCESS);
++ setup_uaccess_flush(enable);
+ }
+
+ static void __init pSeries_setup_arch(void)
--- /dev/null
+From foo@baz Fri Nov 20 08:28:41 AM CET 2020
+From: Daniel Axtens <dja@axtens.net>
+Date: Fri, 20 Nov 2020 11:06:59 +1100
+Subject: powerpc/64s: flush L1D on kernel entry
+To: stable@vger.kernel.org
+Cc: dja@axtens.net
+Message-ID: <20201120000704.374811-4-dja@axtens.net>
+
+From: Nicholas Piggin <npiggin@gmail.com>
+
+commit f79643787e0a0762d2409b7b8334e83f22d85695 upstream.
+
+IBM Power9 processors can speculatively operate on data in the L1 cache before
+it has been completely validated, via a way-prediction mechanism. It is not possible
+for an attacker to determine the contents of impermissible memory using this method,
+since these systems implement a combination of hardware and software security measures
+to prevent scenarios where protected data could be leaked.
+
+However these measures don't address the scenario where an attacker induces
+the operating system to speculatively execute instructions using data that the
+attacker controls. This can be used for example to speculatively bypass "kernel
+user access prevention" techniques, as discovered by Anthony Steinhauser of
+Google's Safeside Project. This is not an attack by itself, but there is a possibility
+it could be used in conjunction with side-channels or other weaknesses in the
+privileged code to construct an attack.
+
+This issue can be mitigated by flushing the L1 cache between privilege boundaries
+of concern. This patch flushes the L1 cache on kernel entry.
+
+This is part of the fix for CVE-2020-4788.
+
+Signed-off-by: Nicholas Piggin <npiggin@gmail.com>
+Signed-off-by: Daniel Axtens <dja@axtens.net>
+Signed-off-by: Greg Kroah-Hartman <gregkh@linuxfoundation.org>
+---
+ Documentation/kernel-parameters.txt | 3 +
+ arch/powerpc/include/asm/exception-64s.h | 9 +++-
+ arch/powerpc/include/asm/feature-fixups.h | 10 ++++
+ arch/powerpc/include/asm/security_features.h | 4 +
+ arch/powerpc/include/asm/setup.h | 3 +
+ arch/powerpc/kernel/exceptions-64s.S | 38 +++++++++++++++++
+ arch/powerpc/kernel/setup_64.c | 58 +++++++++++++++++++++++++++
+ arch/powerpc/kernel/vmlinux.lds.S | 7 +++
+ arch/powerpc/lib/feature-fixups.c | 54 +++++++++++++++++++++++++
+ arch/powerpc/platforms/powernv/setup.c | 10 ++++
+ arch/powerpc/platforms/pseries/setup.c | 4 +
+ 11 files changed, 199 insertions(+), 1 deletion(-)
+
+--- a/Documentation/kernel-parameters.txt
++++ b/Documentation/kernel-parameters.txt
+@@ -2196,6 +2196,7 @@ bytes respectively. Such letter suffixes
+ spec_store_bypass_disable=off [X86]
+ mds=off [X86]
+ tsx_async_abort=off [X86]
++ no_entry_flush [PPC]
+
+ auto (default)
+ Mitigate all CPU vulnerabilities, but leave SMT
+@@ -2476,6 +2477,8 @@ bytes respectively. Such letter suffixes
+
+ noefi Disable EFI runtime services support.
+
++ no_entry_flush [PPC] Don't flush the L1-D cache when entering the kernel.
++
+ noexec [IA-64]
+
+ noexec [X86]
+--- a/arch/powerpc/include/asm/exception-64s.h
++++ b/arch/powerpc/include/asm/exception-64s.h
+@@ -65,11 +65,18 @@
+ nop; \
+ nop
+
++#define ENTRY_FLUSH_SLOT \
++ ENTRY_FLUSH_FIXUP_SECTION; \
++ nop; \
++ nop; \
++ nop;
++
+ /*
+ * r10 must be free to use, r13 must be paca
+ */
+ #define INTERRUPT_TO_KERNEL \
+- STF_ENTRY_BARRIER_SLOT
++ STF_ENTRY_BARRIER_SLOT; \
++ ENTRY_FLUSH_SLOT
+
+ /*
+ * Macros for annotating the expected destination of (h)rfid
+--- a/arch/powerpc/include/asm/feature-fixups.h
++++ b/arch/powerpc/include/asm/feature-fixups.h
+@@ -200,6 +200,14 @@ label##3: \
+ FTR_ENTRY_OFFSET 955b-956b; \
+ .popsection;
+
++#define ENTRY_FLUSH_FIXUP_SECTION \
++957: \
++ .pushsection __entry_flush_fixup,"a"; \
++ .align 2; \
++958: \
++ FTR_ENTRY_OFFSET 957b-958b; \
++ .popsection;
++
+ #define RFI_FLUSH_FIXUP_SECTION \
+ 951: \
+ .pushsection __rfi_flush_fixup,"a"; \
+@@ -231,8 +239,10 @@ label##3: \
+ #ifndef __ASSEMBLY__
+
+ extern long stf_barrier_fallback;
++extern long entry_flush_fallback;
+ extern long __start___stf_entry_barrier_fixup, __stop___stf_entry_barrier_fixup;
+ extern long __start___stf_exit_barrier_fixup, __stop___stf_exit_barrier_fixup;
++extern long __start___entry_flush_fixup, __stop___entry_flush_fixup;
+ extern long __start___rfi_flush_fixup, __stop___rfi_flush_fixup;
+ extern long __start___barrier_nospec_fixup, __stop___barrier_nospec_fixup;
+ extern long __start__btb_flush_fixup, __stop__btb_flush_fixup;
+--- a/arch/powerpc/include/asm/security_features.h
++++ b/arch/powerpc/include/asm/security_features.h
+@@ -84,12 +84,16 @@ static inline bool security_ftr_enabled(
+ // Software required to flush link stack on context switch
+ #define SEC_FTR_FLUSH_LINK_STACK 0x0000000000001000ull
+
++// The L1-D cache should be flushed when entering the kernel
++#define SEC_FTR_L1D_FLUSH_ENTRY 0x0000000000004000ull
++
+
+ // Features enabled by default
+ #define SEC_FTR_DEFAULT \
+ (SEC_FTR_L1D_FLUSH_HV | \
+ SEC_FTR_L1D_FLUSH_PR | \
+ SEC_FTR_BNDS_CHK_SPEC_BAR | \
++ SEC_FTR_L1D_FLUSH_ENTRY | \
+ SEC_FTR_FAVOUR_SECURITY)
+
+ #endif /* _ASM_POWERPC_SECURITY_FEATURES_H */
+--- a/arch/powerpc/include/asm/setup.h
++++ b/arch/powerpc/include/asm/setup.h
+@@ -38,12 +38,15 @@ enum l1d_flush_type {
+ };
+
+ void setup_rfi_flush(enum l1d_flush_type, bool enable);
++void setup_entry_flush(bool enable);
++void setup_uaccess_flush(bool enable);
+ void do_rfi_flush_fixups(enum l1d_flush_type types);
+ #ifdef CONFIG_PPC_BARRIER_NOSPEC
+ void setup_barrier_nospec(void);
+ #else
+ static inline void setup_barrier_nospec(void) { };
+ #endif
++void do_entry_flush_fixups(enum l1d_flush_type types);
+ void do_barrier_nospec_fixups(bool enable);
+ extern bool barrier_nospec_enabled;
+
+--- a/arch/powerpc/kernel/exceptions-64s.S
++++ b/arch/powerpc/kernel/exceptions-64s.S
+@@ -1712,6 +1712,44 @@ hrfi_flush_fallback:
+ GET_SCRATCH0(r13);
+ hrfid
+
++ .globl entry_flush_fallback
++entry_flush_fallback:
++ std r9,PACA_EXRFI+EX_R9(r13)
++ std r10,PACA_EXRFI+EX_R10(r13)
++ std r11,PACA_EXRFI+EX_R11(r13)
++ mfctr r9
++ ld r10,PACA_RFI_FLUSH_FALLBACK_AREA(r13)
++ ld r11,PACA_L1D_FLUSH_SIZE(r13)
++ srdi r11,r11,(7 + 3) /* 128 byte lines, unrolled 8x */
++ mtctr r11
++ DCBT_STOP_ALL_STREAM_IDS(r11) /* Stop prefetch streams */
++
++ /* order ld/st prior to dcbt stop all streams with flushing */
++ sync
++
++ /*
++ * The load addresses are at staggered offsets within cachelines,
++ * which suits some pipelines better (on others it should not
++ * hurt).
++ */
++1:
++ ld r11,(0x80 + 8)*0(r10)
++ ld r11,(0x80 + 8)*1(r10)
++ ld r11,(0x80 + 8)*2(r10)
++ ld r11,(0x80 + 8)*3(r10)
++ ld r11,(0x80 + 8)*4(r10)
++ ld r11,(0x80 + 8)*5(r10)
++ ld r11,(0x80 + 8)*6(r10)
++ ld r11,(0x80 + 8)*7(r10)
++ addi r10,r10,0x80*8
++ bdnz 1b
++
++ mtctr r9
++ ld r9,PACA_EXRFI+EX_R9(r13)
++ ld r10,PACA_EXRFI+EX_R10(r13)
++ ld r11,PACA_EXRFI+EX_R11(r13)
++ blr
++
+ /*
+ * Hash table stuff
+ */
+--- a/arch/powerpc/kernel/setup_64.c
++++ b/arch/powerpc/kernel/setup_64.c
+@@ -844,7 +844,9 @@ early_initcall(disable_hardlockup_detect
+ static enum l1d_flush_type enabled_flush_types;
+ static void *l1d_flush_fallback_area;
+ static bool no_rfi_flush;
++static bool no_entry_flush;
+ bool rfi_flush;
++bool entry_flush;
+
+ static int __init handle_no_rfi_flush(char *p)
+ {
+@@ -854,6 +856,14 @@ static int __init handle_no_rfi_flush(ch
+ }
+ early_param("no_rfi_flush", handle_no_rfi_flush);
+
++static int __init handle_no_entry_flush(char *p)
++{
++ pr_info("entry-flush: disabled on command line.");
++ no_entry_flush = true;
++ return 0;
++}
++early_param("no_entry_flush", handle_no_entry_flush);
++
+ /*
+ * The RFI flush is not KPTI, but because users will see doco that says to use
+ * nopti we hijack that option here to also disable the RFI flush.
+@@ -885,6 +895,18 @@ void rfi_flush_enable(bool enable)
+ rfi_flush = enable;
+ }
+
++void entry_flush_enable(bool enable)
++{
++ if (enable) {
++ do_entry_flush_fixups(enabled_flush_types);
++ on_each_cpu(do_nothing, NULL, 1);
++ } else {
++ do_entry_flush_fixups(L1D_FLUSH_NONE);
++ }
++
++ entry_flush = enable;
++}
++
+ static void __ref init_fallback_flush(void)
+ {
+ u64 l1d_size, limit;
+@@ -930,6 +952,15 @@ void setup_rfi_flush(enum l1d_flush_type
+ rfi_flush_enable(enable);
+ }
+
++void setup_entry_flush(bool enable)
++{
++ if (cpu_mitigations_off())
++ return;
++
++ if (!no_entry_flush)
++ entry_flush_enable(enable);
++}
++
+ #ifdef CONFIG_DEBUG_FS
+ static int rfi_flush_set(void *data, u64 val)
+ {
+@@ -957,9 +988,36 @@ static int rfi_flush_get(void *data, u64
+
+ DEFINE_SIMPLE_ATTRIBUTE(fops_rfi_flush, rfi_flush_get, rfi_flush_set, "%llu\n");
+
++static int entry_flush_set(void *data, u64 val)
++{
++ bool enable;
++
++ if (val == 1)
++ enable = true;
++ else if (val == 0)
++ enable = false;
++ else
++ return -EINVAL;
++
++ /* Only do anything if we're changing state */
++ if (enable != entry_flush)
++ entry_flush_enable(enable);
++
++ return 0;
++}
++
++static int entry_flush_get(void *data, u64 *val)
++{
++ *val = entry_flush ? 1 : 0;
++ return 0;
++}
++
++DEFINE_SIMPLE_ATTRIBUTE(fops_entry_flush, entry_flush_get, entry_flush_set, "%llu\n");
++
+ static __init int rfi_flush_debugfs_init(void)
+ {
+ debugfs_create_file("rfi_flush", 0600, powerpc_debugfs_root, NULL, &fops_rfi_flush);
++ debugfs_create_file("entry_flush", 0600, powerpc_debugfs_root, NULL, &fops_entry_flush);
+ return 0;
+ }
+ device_initcall(rfi_flush_debugfs_init);
+--- a/arch/powerpc/kernel/vmlinux.lds.S
++++ b/arch/powerpc/kernel/vmlinux.lds.S
+@@ -81,6 +81,13 @@ SECTIONS
+ }
+
+ . = ALIGN(8);
++ __entry_flush_fixup : AT(ADDR(__entry_flush_fixup) - LOAD_OFFSET) {
++ __start___entry_flush_fixup = .;
++ *(__entry_flush_fixup)
++ __stop___entry_flush_fixup = .;
++ }
++
++ . = ALIGN(8);
+ __stf_exit_barrier_fixup : AT(ADDR(__stf_exit_barrier_fixup) - LOAD_OFFSET) {
+ __start___stf_exit_barrier_fixup = .;
+ *(__stf_exit_barrier_fixup)
+--- a/arch/powerpc/lib/feature-fixups.c
++++ b/arch/powerpc/lib/feature-fixups.c
+@@ -229,6 +229,60 @@ void do_stf_barrier_fixups(enum stf_barr
+ do_stf_exit_barrier_fixups(types);
+ }
+
++void do_entry_flush_fixups(enum l1d_flush_type types)
++{
++ unsigned int instrs[3], *dest;
++ long *start, *end;
++ int i;
++
++ start = PTRRELOC(&__start___entry_flush_fixup);
++ end = PTRRELOC(&__stop___entry_flush_fixup);
++
++ instrs[0] = 0x60000000; /* nop */
++ instrs[1] = 0x60000000; /* nop */
++ instrs[2] = 0x60000000; /* nop */
++
++ i = 0;
++ if (types == L1D_FLUSH_FALLBACK) {
++ instrs[i++] = 0x7d4802a6; /* mflr r10 */
++ instrs[i++] = 0x60000000; /* branch patched below */
++ instrs[i++] = 0x7d4803a6; /* mtlr r10 */
++ }
++
++ if (types & L1D_FLUSH_ORI) {
++ instrs[i++] = 0x63ff0000; /* ori 31,31,0 speculation barrier */
++ instrs[i++] = 0x63de0000; /* ori 30,30,0 L1d flush*/
++ }
++
++ if (types & L1D_FLUSH_MTTRIG)
++ instrs[i++] = 0x7c12dba6; /* mtspr TRIG2,r0 (SPR #882) */
++
++ for (i = 0; start < end; start++, i++) {
++ dest = (void *)start + *start;
++
++ pr_devel("patching dest %lx\n", (unsigned long)dest);
++
++ patch_instruction(dest, instrs[0]);
++
++ if (types == L1D_FLUSH_FALLBACK)
++ patch_branch((dest + 1), (unsigned long)&entry_flush_fallback,
++ BRANCH_SET_LINK);
++ else
++ patch_instruction((dest + 1), instrs[1]);
++
++ patch_instruction((dest + 2), instrs[2]);
++ }
++
++ printk(KERN_DEBUG "entry-flush: patched %d locations (%s flush)\n", i,
++ (types == L1D_FLUSH_NONE) ? "no" :
++ (types == L1D_FLUSH_FALLBACK) ? "fallback displacement" :
++ (types & L1D_FLUSH_ORI) ? (types & L1D_FLUSH_MTTRIG)
++ ? "ori+mttrig type"
++ : "ori type" :
++ (types & L1D_FLUSH_MTTRIG) ? "mttrig type"
++ : "unknown");
++}
++
+ void do_rfi_flush_fixups(enum l1d_flush_type types)
+ {
+ unsigned int instrs[3], *dest;
+--- a/arch/powerpc/platforms/powernv/setup.c
++++ b/arch/powerpc/platforms/powernv/setup.c
+@@ -124,12 +124,22 @@ static void pnv_setup_rfi_flush(void)
+ type = L1D_FLUSH_ORI;
+ }
+
++ /*
++ * 4.4 doesn't support Power9 bare metal, so we don't need to flush
++ * here - the flush fixes a P9 specific vulnerability.
++ */
++ security_ftr_clear(SEC_FTR_L1D_FLUSH_ENTRY);
++
+ enable = security_ftr_enabled(SEC_FTR_FAVOUR_SECURITY) && \
+ (security_ftr_enabled(SEC_FTR_L1D_FLUSH_PR) || \
+ security_ftr_enabled(SEC_FTR_L1D_FLUSH_HV));
+
+ setup_rfi_flush(type, enable);
+ setup_count_cache_flush();
++
++ enable = security_ftr_enabled(SEC_FTR_FAVOUR_SECURITY) &&
++ security_ftr_enabled(SEC_FTR_L1D_FLUSH_ENTRY);
++ setup_entry_flush(enable);
+ }
+
+ static void __init pnv_setup_arch(void)
+--- a/arch/powerpc/platforms/pseries/setup.c
++++ b/arch/powerpc/platforms/pseries/setup.c
+@@ -584,6 +584,10 @@ void pseries_setup_rfi_flush(void)
+
+ setup_rfi_flush(types, enable);
+ setup_count_cache_flush();
++
++ enable = security_ftr_enabled(SEC_FTR_FAVOUR_SECURITY) &&
++ security_ftr_enabled(SEC_FTR_L1D_FLUSH_ENTRY);
++ setup_entry_flush(enable);
+ }
+
+ static void __init pSeries_setup_arch(void)
--- /dev/null
+From foo@baz Fri Nov 20 08:28:41 AM CET 2020
+From: Daniel Axtens <dja@axtens.net>
+Date: Fri, 20 Nov 2020 11:06:58 +1100
+Subject: powerpc/64s: move some exception handlers out of line
+To: stable@vger.kernel.org
+Cc: dja@axtens.net
+Message-ID: <20201120000704.374811-3-dja@axtens.net>
+
+From: Daniel Axtens <dja@axtens.net>
+
+(backport only)
+
+We're about to grow the exception handlers, which will make a bunch of them
+no longer fit within the space available. We move them out of line.
+
+This is a fiddly and error-prone business, so in the interests of reviewability
+I haven't merged this in with the addition of the entry flush.
+
+Signed-off-by: Daniel Axtens <dja@axtens.net>
+Signed-off-by: Greg Kroah-Hartman <gregkh@linuxfoundation.org>
+---
+ arch/powerpc/kernel/exceptions-64s.S | 138 ++++++++++++++++++++++-------------
+ 1 file changed, 90 insertions(+), 48 deletions(-)
+
+--- a/arch/powerpc/kernel/exceptions-64s.S
++++ b/arch/powerpc/kernel/exceptions-64s.S
+@@ -202,8 +202,8 @@ ALT_FTR_SECTION_END_IFSET(CPU_FTR_HVMODE
+ data_access_pSeries:
+ HMT_MEDIUM_PPR_DISCARD
+ SET_SCRATCH0(r13)
+- EXCEPTION_PROLOG_PSERIES(PACA_EXGEN, data_access_common, EXC_STD,
+- KVMTEST, 0x300)
++ EXCEPTION_PROLOG_0(PACA_EXGEN)
++ b data_access_pSeries_ool
+
+ . = 0x380
+ .globl data_access_slb_pSeries
+@@ -211,31 +211,15 @@ data_access_slb_pSeries:
+ HMT_MEDIUM_PPR_DISCARD
+ SET_SCRATCH0(r13)
+ EXCEPTION_PROLOG_0(PACA_EXSLB)
+- EXCEPTION_PROLOG_1(PACA_EXSLB, KVMTEST, 0x380)
+- std r3,PACA_EXSLB+EX_R3(r13)
+- mfspr r3,SPRN_DAR
+-#ifdef __DISABLED__
+- /* Keep that around for when we re-implement dynamic VSIDs */
+- cmpdi r3,0
+- bge slb_miss_user_pseries
+-#endif /* __DISABLED__ */
+- mfspr r12,SPRN_SRR1
+-#ifndef CONFIG_RELOCATABLE
+- b slb_miss_realmode
+-#else
+- /*
+- * We can't just use a direct branch to slb_miss_realmode
+- * because the distance from here to there depends on where
+- * the kernel ends up being put.
+- */
+- mfctr r11
+- ld r10,PACAKBASE(r13)
+- LOAD_HANDLER(r10, slb_miss_realmode)
+- mtctr r10
+- bctr
+-#endif
++ b data_access_slb_pSeries_ool
+
+- STD_EXCEPTION_PSERIES(0x400, 0x400, instruction_access)
++ . = 0x400
++ .globl instruction_access_pSeries
++instruction_access_pSeries:
++ HMT_MEDIUM_PPR_DISCARD
++ SET_SCRATCH0(r13)
++ EXCEPTION_PROLOG_0(PACA_EXGEN)
++ b instruction_access_pSeries_ool
+
+ . = 0x480
+ .globl instruction_access_slb_pSeries
+@@ -243,24 +227,7 @@ instruction_access_slb_pSeries:
+ HMT_MEDIUM_PPR_DISCARD
+ SET_SCRATCH0(r13)
+ EXCEPTION_PROLOG_0(PACA_EXSLB)
+- EXCEPTION_PROLOG_1(PACA_EXSLB, KVMTEST_PR, 0x480)
+- std r3,PACA_EXSLB+EX_R3(r13)
+- mfspr r3,SPRN_SRR0 /* SRR0 is faulting address */
+-#ifdef __DISABLED__
+- /* Keep that around for when we re-implement dynamic VSIDs */
+- cmpdi r3,0
+- bge slb_miss_user_pseries
+-#endif /* __DISABLED__ */
+- mfspr r12,SPRN_SRR1
+-#ifndef CONFIG_RELOCATABLE
+- b slb_miss_realmode
+-#else
+- mfctr r11
+- ld r10,PACAKBASE(r13)
+- LOAD_HANDLER(r10, slb_miss_realmode)
+- mtctr r10
+- bctr
+-#endif
++ b instruction_access_slb_pSeries_ool
+
+ /* We open code these as we can't have a ". = x" (even with
+ * x = "." within a feature section
+@@ -291,13 +258,19 @@ hardware_interrupt_hv:
+ KVM_HANDLER_PR(PACA_EXGEN, EXC_STD, 0x800)
+
+ . = 0x900
+- .globl decrementer_pSeries
+-decrementer_pSeries:
++ .globl decrementer_trampoline
++decrementer_trampoline:
+ SET_SCRATCH0(r13)
+ EXCEPTION_PROLOG_0(PACA_EXGEN)
+ b decrementer_ool
+
+- STD_EXCEPTION_HV(0x980, 0x982, hdecrementer)
++ . = 0x980
++ .globl hdecrementer_trampoline
++hdecrementer_trampoline:
++ HMT_MEDIUM_PPR_DISCARD;
++ SET_SCRATCH0(r13);
++ EXCEPTION_PROLOG_0(PACA_EXGEN)
++ b hdecrementer_hv
+
+ MASKABLE_EXCEPTION_PSERIES(0xa00, 0xa00, doorbell_super)
+ KVM_HANDLER_PR(PACA_EXGEN, EXC_STD, 0xa00)
+@@ -545,6 +518,64 @@ machine_check_pSeries_0:
+ KVM_HANDLER_PR(PACA_EXGEN, EXC_STD, 0x900)
+ KVM_HANDLER(PACA_EXGEN, EXC_HV, 0x982)
+
++/* moved from 0x300 */
++ .globl data_access_pSeries_ool
++data_access_pSeries_ool:
++ EXCEPTION_PROLOG_1(PACA_EXGEN, KVMTEST, 0x300)
++ EXCEPTION_PROLOG_PSERIES_1(data_access_common, EXC_STD)
++
++ .globl data_access_slb_pSeries_ool
++data_access_slb_pSeries_ool:
++ EXCEPTION_PROLOG_1(PACA_EXSLB, KVMTEST, 0x380)
++ std r3,PACA_EXSLB+EX_R3(r13)
++ mfspr r3,SPRN_DAR
++#ifdef __DISABLED__
++ /* Keep that around for when we re-implement dynamic VSIDs */
++ cmpdi r3,0
++ bge slb_miss_user_pseries
++#endif /* __DISABLED__ */
++ mfspr r12,SPRN_SRR1
++#ifndef CONFIG_RELOCATABLE
++ b slb_miss_realmode
++#else
++ /*
++ * We can't just use a direct branch to slb_miss_realmode
++ * because the distance from here to there depends on where
++ * the kernel ends up being put.
++ */
++ mfctr r11
++ ld r10,PACAKBASE(r13)
++ LOAD_HANDLER(r10, slb_miss_realmode)
++ mtctr r10
++ bctr
++#endif
++
++ .globl instruction_access_pSeries_ool
++instruction_access_pSeries_ool:
++ EXCEPTION_PROLOG_1(PACA_EXGEN, KVMTEST_PR, 0x400)
++ EXCEPTION_PROLOG_PSERIES_1(instruction_access_common, EXC_STD)
++
++ .globl instruction_access_slb_pSeries_ool
++instruction_access_slb_pSeries_ool:
++ EXCEPTION_PROLOG_1(PACA_EXSLB, KVMTEST_PR, 0x480)
++ std r3,PACA_EXSLB+EX_R3(r13)
++ mfspr r3,SPRN_SRR0 /* SRR0 is faulting address */
++#ifdef __DISABLED__
++ /* Keep that around for when we re-implement dynamic VSIDs */
++ cmpdi r3,0
++ bge slb_miss_user_pseries
++#endif /* __DISABLED__ */
++ mfspr r12,SPRN_SRR1
++#ifndef CONFIG_RELOCATABLE
++ b slb_miss_realmode
++#else
++ mfctr r11
++ ld r10,PACAKBASE(r13)
++ LOAD_HANDLER(r10, slb_miss_realmode)
++ mtctr r10
++ bctr
++#endif
++
+ #ifdef CONFIG_PPC_DENORMALISATION
+ denorm_assist:
+ BEGIN_FTR_SECTION
+@@ -612,6 +643,7 @@ END_FTR_SECTION_IFSET(CPU_FTR_CFAR)
+ .align 7
+ /* moved from 0xe00 */
+ MASKABLE_EXCEPTION_OOL(0x900, decrementer)
++ STD_EXCEPTION_HV_OOL(0x982, hdecrementer)
+ STD_EXCEPTION_HV_OOL(0xe02, h_data_storage)
+ KVM_HANDLER_SKIP(PACA_EXGEN, EXC_HV, 0xe02)
+ STD_EXCEPTION_HV_OOL(0xe22, h_instr_storage)
+@@ -894,7 +926,15 @@ hardware_interrupt_relon_hv:
+ STD_RELON_EXCEPTION_PSERIES(0x4600, 0x600, alignment)
+ STD_RELON_EXCEPTION_PSERIES(0x4700, 0x700, program_check)
+ STD_RELON_EXCEPTION_PSERIES(0x4800, 0x800, fp_unavailable)
+- MASKABLE_RELON_EXCEPTION_PSERIES(0x4900, 0x900, decrementer)
++
++ . = 0x4900
++ .globl decrementer_relon_trampoline
++decrementer_relon_trampoline:
++ HMT_MEDIUM_PPR_DISCARD
++ SET_SCRATCH0(r13)
++ EXCEPTION_PROLOG_0(PACA_EXGEN)
++ b decrementer_relon_pSeries
++
+ STD_RELON_EXCEPTION_HV(0x4980, 0x982, hdecrementer)
+ MASKABLE_RELON_EXCEPTION_PSERIES(0x4a00, 0xa00, doorbell_super)
+ STD_RELON_EXCEPTION_PSERIES(0x4b00, 0xb00, trap_0b)
+@@ -1244,6 +1284,8 @@ END_FTR_SECTION_IFSET(CPU_FTR_VSX)
+ __end_handlers:
+
+ /* Equivalents to the above handlers for relocation-on interrupt vectors */
++ MASKABLE_RELON_EXCEPTION_PSERIES_OOL(0x900, decrementer)
++
+ STD_RELON_EXCEPTION_HV_OOL(0xe40, emulation_assist)
+ MASKABLE_RELON_EXCEPTION_HV_OOL(0xe80, h_doorbell)
+
--- /dev/null
+From foo@baz Fri Nov 20 08:28:41 AM CET 2020
+From: Daniel Axtens <dja@axtens.net>
+Date: Fri, 20 Nov 2020 11:07:00 +1100
+Subject: powerpc: Add a framework for user access tracking
+To: stable@vger.kernel.org
+Cc: dja@axtens.net
+Message-ID: <20201120000704.374811-5-dja@axtens.net>
+
+From: Christophe Leroy <christophe.leroy@c-s.fr>
+
+Backported from commit de78a9c42a79 ("powerpc: Add a framework
+for Kernel Userspace Access Protection"). Here we don't try to
+add the KUAP framework, we just want the helper functions
+because we want to put uaccess flush helpers in them.
+
+In terms of fixes, we don't need commit 1d8f739b07bd ("powerpc/kuap:
+Fix set direction in allow/prevent_user_access()") as we don't have
+real KUAP. Likewise as all our allows are noops and all our prevents
+are just flushes, we don't need commit 9dc086f1e9ef ("powerpc/futex:
+Fix incorrect user access blocking") The other 2 fixes we do need.
+
+The original description is:
+
+This patch implements a framework for Kernel Userspace Access
+Protection.
+
+Then subarches will have the possibility to provide their own
+implementation by providing setup_kuap() and
+allow/prevent_user_access().
+
+Some platforms will need to know the area accessed and whether it is
+accessed from read, write or both. Therefore source, destination and
+size and handed over to the two functions.
+
+mpe: Rename to allow/prevent rather than unlock/lock, and add
+read/write wrappers. Drop the 32-bit code for now until we have an
+implementation for it. Add kuap to pt_regs for 64-bit as well as
+32-bit. Don't split strings, use pr_crit_ratelimited().
+
+Signed-off-by: Christophe Leroy <christophe.leroy@c-s.fr>
+Signed-off-by: Russell Currey <ruscur@russell.cc>
+Signed-off-by: Michael Ellerman <mpe@ellerman.id.au>
+Signed-off-by: Daniel Axtens <dja@axtens.net>
+Signed-off-by: Greg Kroah-Hartman <gregkh@linuxfoundation.org>
+---
+ arch/powerpc/include/asm/futex.h | 4 +++
+ arch/powerpc/include/asm/kup.h | 36 ++++++++++++++++++++++++++++++
+ arch/powerpc/include/asm/uaccess.h | 38 +++++++++++++++++++++++++-------
+ arch/powerpc/lib/checksum_wrappers_64.c | 4 +++
+ 4 files changed, 74 insertions(+), 8 deletions(-)
+ create mode 100644 arch/powerpc/include/asm/kup.h
+
+--- a/arch/powerpc/include/asm/futex.h
++++ b/arch/powerpc/include/asm/futex.h
+@@ -36,6 +36,7 @@ static inline int arch_futex_atomic_op_i
+ {
+ int oldval = 0, ret;
+
++ allow_write_to_user(uaddr, sizeof(*uaddr));
+ pagefault_disable();
+
+ switch (op) {
+@@ -62,6 +63,7 @@ static inline int arch_futex_atomic_op_i
+
+ *oval = oldval;
+
++ prevent_write_to_user(uaddr, sizeof(*uaddr));
+ return ret;
+ }
+
+@@ -75,6 +77,7 @@ futex_atomic_cmpxchg_inatomic(u32 *uval,
+ if (!access_ok(VERIFY_WRITE, uaddr, sizeof(u32)))
+ return -EFAULT;
+
++ allow_write_to_user(uaddr, sizeof(*uaddr));
+ __asm__ __volatile__ (
+ PPC_ATOMIC_ENTRY_BARRIER
+ "1: lwarx %1,0,%3 # futex_atomic_cmpxchg_inatomic\n\
+@@ -97,6 +100,7 @@ futex_atomic_cmpxchg_inatomic(u32 *uval,
+ : "cc", "memory");
+
+ *uval = prev;
++ prevent_write_to_user(uaddr, sizeof(*uaddr));
+ return ret;
+ }
+
+--- /dev/null
++++ b/arch/powerpc/include/asm/kup.h
+@@ -0,0 +1,36 @@
++/* SPDX-License-Identifier: GPL-2.0 */
++#ifndef _ASM_POWERPC_KUP_H_
++#define _ASM_POWERPC_KUP_H_
++
++#ifndef __ASSEMBLY__
++
++#include <asm/pgtable.h>
++
++static inline void allow_user_access(void __user *to, const void __user *from,
++ unsigned long size) { }
++static inline void prevent_user_access(void __user *to, const void __user *from,
++ unsigned long size) { }
++
++static inline void allow_read_from_user(const void __user *from, unsigned long size)
++{
++ allow_user_access(NULL, from, size);
++}
++
++static inline void allow_write_to_user(void __user *to, unsigned long size)
++{
++ allow_user_access(to, NULL, size);
++}
++
++static inline void prevent_read_from_user(const void __user *from, unsigned long size)
++{
++ prevent_user_access(NULL, from, size);
++}
++
++static inline void prevent_write_to_user(void __user *to, unsigned long size)
++{
++ prevent_user_access(to, NULL, size);
++}
++
++#endif /* !__ASSEMBLY__ */
++
++#endif /* _ASM_POWERPC_KUP_H_ */
+--- a/arch/powerpc/include/asm/uaccess.h
++++ b/arch/powerpc/include/asm/uaccess.h
+@@ -9,6 +9,7 @@
+ #include <asm/asm-compat.h>
+ #include <asm/processor.h>
+ #include <asm/page.h>
++#include <asm/kup.h>
+
+ #define VERIFY_READ 0
+ #define VERIFY_WRITE 1
+@@ -164,6 +165,7 @@ extern long __put_user_bad(void);
+ #define __put_user_size(x, ptr, size, retval) \
+ do { \
+ retval = 0; \
++ allow_write_to_user(ptr, size); \
+ switch (size) { \
+ case 1: __put_user_asm(x, ptr, retval, "stb"); break; \
+ case 2: __put_user_asm(x, ptr, retval, "sth"); break; \
+@@ -171,6 +173,7 @@ do { \
+ case 8: __put_user_asm2(x, ptr, retval); break; \
+ default: __put_user_bad(); \
+ } \
++ prevent_write_to_user(ptr, size); \
+ } while (0)
+
+ #define __put_user_nocheck(x, ptr, size) \
+@@ -252,6 +255,7 @@ do { \
+ __chk_user_ptr(ptr); \
+ if (size > sizeof(x)) \
+ (x) = __get_user_bad(); \
++ allow_read_from_user(ptr, size); \
+ switch (size) { \
+ case 1: __get_user_asm(x, ptr, retval, "lbz"); break; \
+ case 2: __get_user_asm(x, ptr, retval, "lhz"); break; \
+@@ -259,6 +263,7 @@ do { \
+ case 8: __get_user_asm2(x, ptr, retval); break; \
+ default: (x) = __get_user_bad(); \
+ } \
++ prevent_read_from_user(ptr, size); \
+ } while (0)
+
+ #define __get_user_nocheck(x, ptr, size) \
+@@ -328,9 +333,14 @@ extern unsigned long __copy_tofrom_user(
+ static inline unsigned long copy_from_user(void *to,
+ const void __user *from, unsigned long n)
+ {
++ unsigned long ret;
++
+ if (likely(access_ok(VERIFY_READ, from, n))) {
++ allow_user_access(to, from, n);
+ barrier_nospec();
+- return __copy_tofrom_user((__force void __user *)to, from, n);
++ ret = __copy_tofrom_user((__force void __user *)to, from, n);
++ prevent_user_access(to, from, n);
++ return ret;
+ }
+ memset(to, 0, n);
+ return n;
+@@ -361,8 +371,9 @@ extern unsigned long copy_in_user(void _
+ static inline unsigned long __copy_from_user_inatomic(void *to,
+ const void __user *from, unsigned long n)
+ {
++ unsigned long ret;
+ if (__builtin_constant_p(n) && (n <= 8)) {
+- unsigned long ret = 1;
++ ret = 1;
+
+ switch (n) {
+ case 1:
+@@ -387,14 +398,18 @@ static inline unsigned long __copy_from_
+ }
+
+ barrier_nospec();
+- return __copy_tofrom_user((__force void __user *)to, from, n);
++ allow_read_from_user(from, n);
++ ret = __copy_tofrom_user((__force void __user *)to, from, n);
++ prevent_read_from_user(from, n);
++ return ret;
+ }
+
+ static inline unsigned long __copy_to_user_inatomic(void __user *to,
+ const void *from, unsigned long n)
+ {
++ unsigned long ret;
+ if (__builtin_constant_p(n) && (n <= 8)) {
+- unsigned long ret = 1;
++ ret = 1;
+
+ switch (n) {
+ case 1:
+@@ -414,7 +429,10 @@ static inline unsigned long __copy_to_us
+ return 0;
+ }
+
+- return __copy_tofrom_user(to, (__force const void __user *)from, n);
++ allow_write_to_user(to, n);
++ ret = __copy_tofrom_user(to, (__force const void __user *)from, n);
++ prevent_write_to_user(to, n);
++ return ret;
+ }
+
+ static inline unsigned long __copy_from_user(void *to,
+@@ -435,10 +453,14 @@ extern unsigned long __clear_user(void _
+
+ static inline unsigned long clear_user(void __user *addr, unsigned long size)
+ {
++ unsigned long ret = size;
+ might_fault();
+- if (likely(access_ok(VERIFY_WRITE, addr, size)))
+- return __clear_user(addr, size);
+- return size;
++ if (likely(access_ok(VERIFY_WRITE, addr, size))) {
++ allow_write_to_user(addr, size);
++ ret = __clear_user(addr, size);
++ prevent_write_to_user(addr, size);
++ }
++ return ret;
+ }
+
+ extern long strncpy_from_user(char *dst, const char __user *src, long count);
+--- a/arch/powerpc/lib/checksum_wrappers_64.c
++++ b/arch/powerpc/lib/checksum_wrappers_64.c
+@@ -29,6 +29,7 @@ __wsum csum_and_copy_from_user(const voi
+ unsigned int csum;
+
+ might_sleep();
++ allow_read_from_user(src, len);
+
+ *err_ptr = 0;
+
+@@ -60,6 +61,7 @@ __wsum csum_and_copy_from_user(const voi
+ }
+
+ out:
++ prevent_read_from_user(src, len);
+ return (__force __wsum)csum;
+ }
+ EXPORT_SYMBOL(csum_and_copy_from_user);
+@@ -70,6 +72,7 @@ __wsum csum_and_copy_to_user(const void
+ unsigned int csum;
+
+ might_sleep();
++ allow_write_to_user(dst, len);
+
+ *err_ptr = 0;
+
+@@ -97,6 +100,7 @@ __wsum csum_and_copy_to_user(const void
+ }
+
+ out:
++ prevent_write_to_user(dst, len);
+ return (__force __wsum)csum;
+ }
+ EXPORT_SYMBOL(csum_and_copy_to_user);
--- /dev/null
+From foo@baz Fri Nov 20 08:28:41 AM CET 2020
+From: Daniel Axtens <dja@axtens.net>
+Date: Fri, 20 Nov 2020 11:07:02 +1100
+Subject: powerpc: Fix __clear_user() with KUAP enabled
+To: stable@vger.kernel.org
+Cc: dja@axtens.net
+Message-ID: <20201120000704.374811-7-dja@axtens.net>
+
+From: Andrew Donnellan <ajd@linux.ibm.com>
+
+commit 61e3acd8c693a14fc69b824cb5b08d02cb90a6e7 upstream.
+
+The KUAP implementation adds calls in clear_user() to enable and
+disable access to userspace memory. However, it doesn't add these to
+__clear_user(), which is used in the ptrace regset code.
+
+As there's only one direct user of __clear_user() (the regset code),
+and the time taken to set the AMR for KUAP purposes is going to
+dominate the cost of a quick access_ok(), there's not much point
+having a separate path.
+
+Rename __clear_user() to __arch_clear_user(), and make __clear_user()
+just call clear_user().
+
+Reported-by: syzbot+f25ecf4b2982d8c7a640@syzkaller-ppc64.appspotmail.com
+Reported-by: Daniel Axtens <dja@axtens.net>
+Suggested-by: Michael Ellerman <mpe@ellerman.id.au>
+Fixes: de78a9c42a79 ("powerpc: Add a framework for Kernel Userspace Access Protection")
+Signed-off-by: Andrew Donnellan <ajd@linux.ibm.com>
+[mpe: Use __arch_clear_user() for the asm version like arm64 & nds32]
+Signed-off-by: Michael Ellerman <mpe@ellerman.id.au>
+Link: https://lore.kernel.org/r/20191209132221.15328-1-ajd@linux.ibm.com
+Signed-off-by: Daniel Axtens <dja@axtens.net>
+Signed-off-by: Greg Kroah-Hartman <gregkh@linuxfoundation.org>
+---
+ arch/powerpc/include/asm/uaccess.h | 9 +++++++--
+ arch/powerpc/kernel/ppc_ksyms.c | 3 +++
+ arch/powerpc/lib/string.S | 2 +-
+ arch/powerpc/lib/string_64.S | 4 ++--
+ 4 files changed, 13 insertions(+), 5 deletions(-)
+
+--- a/arch/powerpc/include/asm/uaccess.h
++++ b/arch/powerpc/include/asm/uaccess.h
+@@ -471,7 +471,7 @@ static inline unsigned long __copy_to_us
+ return __copy_to_user_inatomic(to, from, size);
+ }
+
+-extern unsigned long __clear_user(void __user *addr, unsigned long size);
++unsigned long __arch_clear_user(void __user *addr, unsigned long size);
+
+ static inline unsigned long clear_user(void __user *addr, unsigned long size)
+ {
+@@ -479,12 +479,17 @@ static inline unsigned long clear_user(v
+ might_fault();
+ if (likely(access_ok(VERIFY_WRITE, addr, size))) {
+ allow_write_to_user(addr, size);
+- ret = __clear_user(addr, size);
++ ret = __arch_clear_user(addr, size);
+ prevent_write_to_user(addr, size);
+ }
+ return ret;
+ }
+
++static inline unsigned long __clear_user(void __user *addr, unsigned long size)
++{
++ return clear_user(addr, size);
++}
++
+ extern long strncpy_from_user(char *dst, const char __user *src, long count);
+ extern __must_check long strlen_user(const char __user *str);
+ extern __must_check long strnlen_user(const char __user *str, long n);
+--- a/arch/powerpc/kernel/ppc_ksyms.c
++++ b/arch/powerpc/kernel/ppc_ksyms.c
+@@ -5,6 +5,7 @@
+ #include <asm/switch_to.h>
+ #include <asm/cacheflush.h>
+ #include <asm/epapr_hcalls.h>
++#include <asm/uaccess.h>
+
+ EXPORT_SYMBOL(flush_dcache_range);
+ EXPORT_SYMBOL(flush_icache_range);
+@@ -43,3 +44,5 @@ EXPORT_SYMBOL(epapr_hypercall_start);
+ #endif
+
+ EXPORT_SYMBOL(current_stack_pointer);
++
++EXPORT_SYMBOL(__arch_clear_user);
+--- a/arch/powerpc/lib/string.S
++++ b/arch/powerpc/lib/string.S
+@@ -122,7 +122,7 @@ _GLOBAL(memchr)
+ blr
+
+ #ifdef CONFIG_PPC32
+-_GLOBAL(__clear_user)
++_GLOBAL(__arch_clear_user)
+ addi r6,r3,-4
+ li r3,0
+ li r5,0
+--- a/arch/powerpc/lib/string_64.S
++++ b/arch/powerpc/lib/string_64.S
+@@ -27,7 +27,7 @@ PPC64_CACHES:
+ .section ".text"
+
+ /**
+- * __clear_user: - Zero a block of memory in user space, with less checking.
++ * __arch_clear_user: - Zero a block of memory in user space, with less checking.
+ * @to: Destination address, in user space.
+ * @n: Number of bytes to zero.
+ *
+@@ -77,7 +77,7 @@ err3; stb r0,0(r3)
+ mr r3,r4
+ blr
+
+-_GLOBAL_TOC(__clear_user)
++_GLOBAL_TOC(__arch_clear_user)
+ cmpdi r4,32
+ neg r6,r3
+ li r0,0
--- /dev/null
+From foo@baz Fri Nov 20 08:28:41 AM CET 2020
+From: Daniel Axtens <dja@axtens.net>
+Date: Fri, 20 Nov 2020 11:07:01 +1100
+Subject: powerpc: Implement user_access_begin and friends
+To: stable@vger.kernel.org
+Cc: dja@axtens.net
+Message-ID: <20201120000704.374811-6-dja@axtens.net>
+
+From: Christophe Leroy <christophe.leroy@c-s.fr>
+
+commit 5cd623333e7cf4e3a334c70529268b65f2a6c2c7 upstream.
+
+Today, when a function like strncpy_from_user() is called,
+the userspace access protection is de-activated and re-activated
+for every word read.
+
+By implementing user_access_begin and friends, the protection
+is de-activated at the beginning of the copy and re-activated at the
+end.
+
+Implement user_access_begin(), user_access_end() and
+unsafe_get_user(), unsafe_put_user() and unsafe_copy_to_user()
+
+For the time being, we keep user_access_save() and
+user_access_restore() as nops.
+
+Signed-off-by: Christophe Leroy <christophe.leroy@c-s.fr>
+Signed-off-by: Michael Ellerman <mpe@ellerman.id.au>
+Link: https://lore.kernel.org/r/36d4fbf9e56a75994aca4ee2214c77b26a5a8d35.1579866752.git.christophe.leroy@c-s.fr
+Signed-off-by: Daniel Axtens <dja@axtens.net>
+Signed-off-by: Greg Kroah-Hartman <gregkh@linuxfoundation.org>
+---
+ arch/powerpc/include/asm/uaccess.h | 60 ++++++++++++++++++++++++++++---------
+ 1 file changed, 46 insertions(+), 14 deletions(-)
+
+--- a/arch/powerpc/include/asm/uaccess.h
++++ b/arch/powerpc/include/asm/uaccess.h
+@@ -106,9 +106,14 @@ struct exception_table_entry {
+ __put_user_check((__typeof__(*(ptr)))(x), (ptr), sizeof(*(ptr)))
+
+ #define __get_user(x, ptr) \
+- __get_user_nocheck((x), (ptr), sizeof(*(ptr)))
++ __get_user_nocheck((x), (ptr), sizeof(*(ptr)), true)
+ #define __put_user(x, ptr) \
+- __put_user_nocheck((__typeof__(*(ptr)))(x), (ptr), sizeof(*(ptr)))
++ __put_user_nocheck((__typeof__(*(ptr)))(x), (ptr), sizeof(*(ptr)), true)
++
++#define __get_user_allowed(x, ptr) \
++ __get_user_nocheck((x), (ptr), sizeof(*(ptr)), false)
++#define __put_user_allowed(x, ptr) \
++ __put_user_nocheck((__typeof__(*(ptr)))(x), (ptr), sizeof(*(ptr)), false)
+
+ #define __get_user_inatomic(x, ptr) \
+ __get_user_nosleep((x), (ptr), sizeof(*(ptr)))
+@@ -162,10 +167,9 @@ extern long __put_user_bad(void);
+ : "r" (x), "b" (addr), "i" (-EFAULT), "0" (err))
+ #endif /* __powerpc64__ */
+
+-#define __put_user_size(x, ptr, size, retval) \
++#define __put_user_size_allowed(x, ptr, size, retval) \
+ do { \
+ retval = 0; \
+- allow_write_to_user(ptr, size); \
+ switch (size) { \
+ case 1: __put_user_asm(x, ptr, retval, "stb"); break; \
+ case 2: __put_user_asm(x, ptr, retval, "sth"); break; \
+@@ -173,17 +177,26 @@ do { \
+ case 8: __put_user_asm2(x, ptr, retval); break; \
+ default: __put_user_bad(); \
+ } \
++} while (0)
++
++#define __put_user_size(x, ptr, size, retval) \
++do { \
++ allow_write_to_user(ptr, size); \
++ __put_user_size_allowed(x, ptr, size, retval); \
+ prevent_write_to_user(ptr, size); \
+ } while (0)
+
+-#define __put_user_nocheck(x, ptr, size) \
++#define __put_user_nocheck(x, ptr, size, do_allow) \
+ ({ \
+ long __pu_err; \
+ __typeof__(*(ptr)) __user *__pu_addr = (ptr); \
+ if (!is_kernel_addr((unsigned long)__pu_addr)) \
+ might_fault(); \
+ __chk_user_ptr(ptr); \
+- __put_user_size((x), __pu_addr, (size), __pu_err); \
++ if (do_allow) \
++ __put_user_size((x), __pu_addr, (size), __pu_err); \
++ else \
++ __put_user_size_allowed((x), __pu_addr, (size), __pu_err); \
+ __pu_err; \
+ })
+
+@@ -249,13 +262,12 @@ extern long __get_user_bad(void);
+ : "b" (addr), "i" (-EFAULT), "0" (err))
+ #endif /* __powerpc64__ */
+
+-#define __get_user_size(x, ptr, size, retval) \
++#define __get_user_size_allowed(x, ptr, size, retval) \
+ do { \
+ retval = 0; \
+ __chk_user_ptr(ptr); \
+ if (size > sizeof(x)) \
+ (x) = __get_user_bad(); \
+- allow_read_from_user(ptr, size); \
+ switch (size) { \
+ case 1: __get_user_asm(x, ptr, retval, "lbz"); break; \
+ case 2: __get_user_asm(x, ptr, retval, "lhz"); break; \
+@@ -263,10 +275,16 @@ do { \
+ case 8: __get_user_asm2(x, ptr, retval); break; \
+ default: (x) = __get_user_bad(); \
+ } \
++} while (0)
++
++#define __get_user_size(x, ptr, size, retval) \
++do { \
++ allow_read_from_user(ptr, size); \
++ __get_user_size_allowed(x, ptr, size, retval); \
+ prevent_read_from_user(ptr, size); \
+ } while (0)
+
+-#define __get_user_nocheck(x, ptr, size) \
++#define __get_user_nocheck(x, ptr, size, do_allow) \
+ ({ \
+ long __gu_err; \
+ unsigned long __gu_val; \
+@@ -275,7 +293,10 @@ do { \
+ if (!is_kernel_addr((unsigned long)__gu_addr)) \
+ might_fault(); \
+ barrier_nospec(); \
+- __get_user_size(__gu_val, __gu_addr, (size), __gu_err); \
++ if (do_allow) \
++ __get_user_size(__gu_val, __gu_addr, (size), __gu_err); \
++ else \
++ __get_user_size_allowed(__gu_val, __gu_addr, (size), __gu_err); \
+ (x) = (__typeof__(*(ptr)))__gu_val; \
+ __gu_err; \
+ })
+@@ -408,21 +429,22 @@ static inline unsigned long __copy_to_us
+ const void *from, unsigned long n)
+ {
+ unsigned long ret;
++
+ if (__builtin_constant_p(n) && (n <= 8)) {
+ ret = 1;
+
+ switch (n) {
+ case 1:
+- __put_user_size(*(u8 *)from, (u8 __user *)to, 1, ret);
++ __put_user_size_allowed(*(u8 *)from, (u8 __user *)to, 1, ret);
+ break;
+ case 2:
+- __put_user_size(*(u16 *)from, (u16 __user *)to, 2, ret);
++ __put_user_size_allowed(*(u16 *)from, (u16 __user *)to, 2, ret);
+ break;
+ case 4:
+- __put_user_size(*(u32 *)from, (u32 __user *)to, 4, ret);
++ __put_user_size_allowed(*(u32 *)from, (u32 __user *)to, 4, ret);
+ break;
+ case 8:
+- __put_user_size(*(u64 *)from, (u64 __user *)to, 8, ret);
++ __put_user_size_allowed(*(u64 *)from, (u64 __user *)to, 8, ret);
+ break;
+ }
+ if (ret == 0)
+@@ -467,6 +489,16 @@ extern long strncpy_from_user(char *dst,
+ extern __must_check long strlen_user(const char __user *str);
+ extern __must_check long strnlen_user(const char __user *str, long n);
+
++
++#define user_access_begin() do { } while (0)
++#define user_access_end() prevent_user_access(NULL, NULL, ~0ul)
++
++#define unsafe_op_wrap(op, err) do { if (unlikely(op)) goto err; } while (0)
++#define unsafe_get_user(x, p, e) unsafe_op_wrap(__get_user_allowed(x, p), e)
++#define unsafe_put_user(x, p, e) unsafe_op_wrap(__put_user_allowed(x, p), e)
++#define unsafe_copy_to_user(d, s, l, e) \
++ unsafe_op_wrap(__copy_to_user_inatomic(d, s, l), e)
++
+ #endif /* __ASSEMBLY__ */
+ #endif /* __KERNEL__ */
+
--- /dev/null
+From foo@baz Fri Nov 20 08:28:41 AM CET 2020
+From: Daniel Axtens <dja@axtens.net>
+Date: Fri, 20 Nov 2020 11:07:03 +1100
+Subject: powerpc/uaccess: Evaluate macro arguments once, before user access is allowed
+To: stable@vger.kernel.org
+Cc: dja@axtens.net
+Message-ID: <20201120000704.374811-8-dja@axtens.net>
+
+From: Nicholas Piggin <npiggin@gmail.com>
+
+commit d02f6b7dab8228487268298ea1f21081c0b4b3eb upstream.
+
+get/put_user() can be called with nontrivial arguments. fs/proc/page.c
+has a good example:
+
+ if (put_user(stable_page_flags(ppage), out)) {
+
+stable_page_flags() is quite a lot of code, including spin locks in
+the page allocator.
+
+Ensure these arguments are evaluated before user access is allowed.
+
+This improves security by reducing code with access to userspace, but
+it also fixes a PREEMPT bug with KUAP on powerpc/64s:
+stable_page_flags() is currently called with AMR set to allow writes,
+it ends up calling spin_unlock(), which can call preempt_schedule. But
+the task switch code can not be called with AMR set (it relies on
+interrupts saving the register), so this blows up.
+
+It's fine if the code inside allow_user_access() is preemptible,
+because a timer or IPI will save the AMR, but it's not okay to
+explicitly cause a reschedule.
+
+Fixes: de78a9c42a79 ("powerpc: Add a framework for Kernel Userspace Access Protection")
+Signed-off-by: Nicholas Piggin <npiggin@gmail.com>
+Signed-off-by: Michael Ellerman <mpe@ellerman.id.au>
+Link: https://lore.kernel.org/r/20200407041245.600651-1-npiggin@gmail.com
+Signed-off-by: Daniel Axtens <dja@axtens.net>
+Signed-off-by: Greg Kroah-Hartman <gregkh@linuxfoundation.org>
+---
+ arch/powerpc/include/asm/uaccess.h | 49 ++++++++++++++++++++++++++-----------
+ 1 file changed, 35 insertions(+), 14 deletions(-)
+
+--- a/arch/powerpc/include/asm/uaccess.h
++++ b/arch/powerpc/include/asm/uaccess.h
+@@ -190,13 +190,17 @@ do { \
+ ({ \
+ long __pu_err; \
+ __typeof__(*(ptr)) __user *__pu_addr = (ptr); \
++ __typeof__(*(ptr)) __pu_val = (x); \
++ __typeof__(size) __pu_size = (size); \
++ \
+ if (!is_kernel_addr((unsigned long)__pu_addr)) \
+ might_fault(); \
+- __chk_user_ptr(ptr); \
++ __chk_user_ptr(__pu_addr); \
+ if (do_allow) \
+- __put_user_size((x), __pu_addr, (size), __pu_err); \
++ __put_user_size(__pu_val, __pu_addr, __pu_size, __pu_err); \
+ else \
+- __put_user_size_allowed((x), __pu_addr, (size), __pu_err); \
++ __put_user_size_allowed(__pu_val, __pu_addr, __pu_size, __pu_err); \
++ \
+ __pu_err; \
+ })
+
+@@ -204,9 +208,13 @@ do { \
+ ({ \
+ long __pu_err = -EFAULT; \
+ __typeof__(*(ptr)) __user *__pu_addr = (ptr); \
++ __typeof__(*(ptr)) __pu_val = (x); \
++ __typeof__(size) __pu_size = (size); \
++ \
+ might_fault(); \
+- if (access_ok(VERIFY_WRITE, __pu_addr, size)) \
+- __put_user_size((x), __pu_addr, (size), __pu_err); \
++ if (access_ok(VERIFY_WRITE, __pu_addr, __pu_size)) \
++ __put_user_size(__pu_val, __pu_addr, __pu_size, __pu_err); \
++ \
+ __pu_err; \
+ })
+
+@@ -214,8 +222,12 @@ do { \
+ ({ \
+ long __pu_err; \
+ __typeof__(*(ptr)) __user *__pu_addr = (ptr); \
+- __chk_user_ptr(ptr); \
+- __put_user_size((x), __pu_addr, (size), __pu_err); \
++ __typeof__(*(ptr)) __pu_val = (x); \
++ __typeof__(size) __pu_size = (size); \
++ \
++ __chk_user_ptr(__pu_addr); \
++ __put_user_size(__pu_val, __pu_addr, __pu_size, __pu_err); \
++ \
+ __pu_err; \
+ })
+
+@@ -289,15 +301,18 @@ do { \
+ long __gu_err; \
+ unsigned long __gu_val; \
+ __typeof__(*(ptr)) __user *__gu_addr = (ptr); \
+- __chk_user_ptr(ptr); \
++ __typeof__(size) __gu_size = (size); \
++ \
++ __chk_user_ptr(__gu_addr); \
+ if (!is_kernel_addr((unsigned long)__gu_addr)) \
+ might_fault(); \
+ barrier_nospec(); \
+ if (do_allow) \
+- __get_user_size(__gu_val, __gu_addr, (size), __gu_err); \
++ __get_user_size(__gu_val, __gu_addr, __gu_size, __gu_err); \
+ else \
+- __get_user_size_allowed(__gu_val, __gu_addr, (size), __gu_err); \
++ __get_user_size_allowed(__gu_val, __gu_addr, __gu_size, __gu_err); \
+ (x) = (__typeof__(*(ptr)))__gu_val; \
++ \
+ __gu_err; \
+ })
+
+@@ -322,12 +337,15 @@ do { \
+ long __gu_err = -EFAULT; \
+ unsigned long __gu_val = 0; \
+ __typeof__(*(ptr)) __user *__gu_addr = (ptr); \
++ __typeof__(size) __gu_size = (size); \
++ \
+ might_fault(); \
+- if (access_ok(VERIFY_READ, __gu_addr, (size))) { \
++ if (access_ok(VERIFY_READ, __gu_addr, __gu_size)) { \
+ barrier_nospec(); \
+- __get_user_size(__gu_val, __gu_addr, (size), __gu_err); \
++ __get_user_size(__gu_val, __gu_addr, __gu_size, __gu_err); \
+ } \
+ (x) = (__force __typeof__(*(ptr)))__gu_val; \
++ \
+ __gu_err; \
+ })
+
+@@ -336,10 +354,13 @@ do { \
+ long __gu_err; \
+ unsigned long __gu_val; \
+ __typeof__(*(ptr)) __user *__gu_addr = (ptr); \
+- __chk_user_ptr(ptr); \
++ __typeof__(size) __gu_size = (size); \
++ \
++ __chk_user_ptr(__gu_addr); \
+ barrier_nospec(); \
+- __get_user_size(__gu_val, __gu_addr, (size), __gu_err); \
++ __get_user_size(__gu_val, __gu_addr, __gu_size, __gu_err); \
+ (x) = (__force __typeof__(*(ptr)))__gu_val; \
++ \
+ __gu_err; \
+ })
+
--- /dev/null
+powerpc-64s-define-maskable_relon_exception_pseries_ool.patch
+powerpc-64s-move-some-exception-handlers-out-of-line.patch
+powerpc-64s-flush-l1d-on-kernel-entry.patch
+powerpc-add-a-framework-for-user-access-tracking.patch
+powerpc-implement-user_access_begin-and-friends.patch
+powerpc-fix-__clear_user-with-kuap-enabled.patch
+powerpc-uaccess-evaluate-macro-arguments-once-before-user-access-is-allowed.patch
+powerpc-64s-flush-l1d-after-user-accesses.patch
projects / thirdparty / kernel / stable-queue.git / commitdiff
