]>
Commit | Line | Data |
---|---|---|
4fa5b95e GKH |
1 | From foo@baz Mon 29 Apr 2019 11:38:37 AM CEST |
2 | From: Michael Ellerman <mpe@ellerman.id.au> | |
3 | Date: Mon, 22 Apr 2019 00:20:26 +1000 | |
4 | Subject: powerpc/64s: Add support for software count cache flush | |
5 | To: stable@vger.kernel.org, gregkh@linuxfoundation.org | |
6 | Cc: linuxppc-dev@ozlabs.org, diana.craciun@nxp.com, msuchanek@suse.de, npiggin@gmail.com, christophe.leroy@c-s.fr | |
7 | Message-ID: <20190421142037.21881-42-mpe@ellerman.id.au> | |
8 | ||
9 | From: Michael Ellerman <mpe@ellerman.id.au> | |
10 | ||
11 | commit ee13cb249fabdff8b90aaff61add347749280087 upstream. | |
12 | ||
13 | Some CPU revisions support a mode where the count cache needs to be | |
14 | flushed by software on context switch. Additionally some revisions may | |
15 | have a hardware accelerated flush, in which case the software flush | |
16 | sequence can be shortened. | |
17 | ||
18 | If we detect the appropriate flag from firmware we patch a branch | |
19 | into _switch() which takes us to a count cache flush sequence. | |
20 | ||
21 | That sequence in turn may be patched to return early if we detect that | |
22 | the CPU supports accelerating the flush sequence in hardware. | |
23 | ||
24 | Add debugfs support for reporting the state of the flush, as well as | |
25 | runtime disabling it. | |
26 | ||
27 | And modify the spectre_v2 sysfs file to report the state of the | |
28 | software flush. | |
29 | ||
30 | Signed-off-by: Michael Ellerman <mpe@ellerman.id.au> | |
31 | Signed-off-by: Greg Kroah-Hartman <gregkh@linuxfoundation.org> | |
32 | --- | |
33 | arch/powerpc/include/asm/asm-prototypes.h | 21 +++++ | |
34 | arch/powerpc/include/asm/security_features.h | 1 | |
35 | arch/powerpc/kernel/entry_64.S | 54 ++++++++++++++ | |
36 | arch/powerpc/kernel/security.c | 98 +++++++++++++++++++++++++-- | |
37 | 4 files changed, 169 insertions(+), 5 deletions(-) | |
38 | create mode 100644 arch/powerpc/include/asm/asm-prototypes.h | |
39 | ||
40 | --- /dev/null | |
41 | +++ b/arch/powerpc/include/asm/asm-prototypes.h | |
42 | @@ -0,0 +1,21 @@ | |
43 | +#ifndef _ASM_POWERPC_ASM_PROTOTYPES_H | |
44 | +#define _ASM_POWERPC_ASM_PROTOTYPES_H | |
45 | +/* | |
46 | + * This file is for prototypes of C functions that are only called | |
47 | + * from asm, and any associated variables. | |
48 | + * | |
49 | + * Copyright 2016, Daniel Axtens, IBM Corporation. | |
50 | + * | |
51 | + * This program is free software; you can redistribute it and/or | |
52 | + * modify it under the terms of the GNU General Public License | |
53 | + * as published by the Free Software Foundation; either version 2 | |
54 | + * of the License, or (at your option) any later version. | |
55 | + */ | |
56 | + | |
57 | +/* Patch sites */ | |
58 | +extern s32 patch__call_flush_count_cache; | |
59 | +extern s32 patch__flush_count_cache_return; | |
60 | + | |
61 | +extern long flush_count_cache; | |
62 | + | |
63 | +#endif /* _ASM_POWERPC_ASM_PROTOTYPES_H */ | |
64 | --- a/arch/powerpc/include/asm/security_features.h | |
65 | +++ b/arch/powerpc/include/asm/security_features.h | |
66 | @@ -22,6 +22,7 @@ enum stf_barrier_type { | |
67 | ||
68 | void setup_stf_barrier(void); | |
69 | void do_stf_barrier_fixups(enum stf_barrier_type types); | |
70 | +void setup_count_cache_flush(void); | |
71 | ||
72 | static inline void security_ftr_set(unsigned long feature) | |
73 | { | |
74 | --- a/arch/powerpc/kernel/entry_64.S | |
75 | +++ b/arch/powerpc/kernel/entry_64.S | |
76 | @@ -25,6 +25,7 @@ | |
77 | #include <asm/page.h> | |
78 | #include <asm/mmu.h> | |
79 | #include <asm/thread_info.h> | |
80 | +#include <asm/code-patching-asm.h> | |
81 | #include <asm/ppc_asm.h> | |
82 | #include <asm/asm-offsets.h> | |
83 | #include <asm/cputable.h> | |
84 | @@ -450,6 +451,57 @@ _GLOBAL(ret_from_kernel_thread) | |
85 | li r3,0 | |
86 | b .Lsyscall_exit | |
87 | ||
88 | +#ifdef CONFIG_PPC_BOOK3S_64 | |
89 | + | |
90 | +#define FLUSH_COUNT_CACHE \ | |
91 | +1: nop; \ | |
92 | + patch_site 1b, patch__call_flush_count_cache | |
93 | + | |
94 | + | |
95 | +#define BCCTR_FLUSH .long 0x4c400420 | |
96 | + | |
97 | +.macro nops number | |
98 | + .rept \number | |
99 | + nop | |
100 | + .endr | |
101 | +.endm | |
102 | + | |
103 | +.balign 32 | |
104 | +.global flush_count_cache | |
105 | +flush_count_cache: | |
106 | + /* Save LR into r9 */ | |
107 | + mflr r9 | |
108 | + | |
109 | + .rept 64 | |
110 | + bl .+4 | |
111 | + .endr | |
112 | + b 1f | |
113 | + nops 6 | |
114 | + | |
115 | + .balign 32 | |
116 | + /* Restore LR */ | |
117 | +1: mtlr r9 | |
118 | + li r9,0x7fff | |
119 | + mtctr r9 | |
120 | + | |
121 | + BCCTR_FLUSH | |
122 | + | |
123 | +2: nop | |
124 | + patch_site 2b patch__flush_count_cache_return | |
125 | + | |
126 | + nops 3 | |
127 | + | |
128 | + .rept 278 | |
129 | + .balign 32 | |
130 | + BCCTR_FLUSH | |
131 | + nops 7 | |
132 | + .endr | |
133 | + | |
134 | + blr | |
135 | +#else | |
136 | +#define FLUSH_COUNT_CACHE | |
137 | +#endif /* CONFIG_PPC_BOOK3S_64 */ | |
138 | + | |
139 | /* | |
140 | * This routine switches between two different tasks. The process | |
141 | * state of one is saved on its kernel stack. Then the state | |
142 | @@ -513,6 +565,8 @@ BEGIN_FTR_SECTION | |
143 | END_FTR_SECTION_IFSET(CPU_FTR_ARCH_207S) | |
144 | #endif | |
145 | ||
146 | + FLUSH_COUNT_CACHE | |
147 | + | |
148 | #ifdef CONFIG_SMP | |
149 | /* We need a sync somewhere here to make sure that if the | |
150 | * previous task gets rescheduled on another CPU, it sees all | |
151 | --- a/arch/powerpc/kernel/security.c | |
152 | +++ b/arch/powerpc/kernel/security.c | |
153 | @@ -10,12 +10,21 @@ | |
154 | #include <linux/seq_buf.h> | |
155 | ||
156 | #include <asm/debug.h> | |
157 | +#include <asm/asm-prototypes.h> | |
158 | +#include <asm/code-patching.h> | |
159 | #include <asm/security_features.h> | |
160 | #include <asm/setup.h> | |
161 | ||
162 | ||
163 | unsigned long powerpc_security_features __read_mostly = SEC_FTR_DEFAULT; | |
164 | ||
165 | +enum count_cache_flush_type { | |
166 | + COUNT_CACHE_FLUSH_NONE = 0x1, | |
167 | + COUNT_CACHE_FLUSH_SW = 0x2, | |
168 | + COUNT_CACHE_FLUSH_HW = 0x4, | |
169 | +}; | |
170 | +static enum count_cache_flush_type count_cache_flush_type; | |
171 | + | |
172 | bool barrier_nospec_enabled; | |
173 | static bool no_nospec; | |
174 | ||
175 | @@ -160,17 +169,29 @@ ssize_t cpu_show_spectre_v2(struct devic | |
176 | bcs = security_ftr_enabled(SEC_FTR_BCCTRL_SERIALISED); | |
177 | ccd = security_ftr_enabled(SEC_FTR_COUNT_CACHE_DISABLED); | |
178 | ||
179 | - if (bcs || ccd) { | |
180 | + if (bcs || ccd || count_cache_flush_type != COUNT_CACHE_FLUSH_NONE) { | |
181 | + bool comma = false; | |
182 | seq_buf_printf(&s, "Mitigation: "); | |
183 | ||
184 | - if (bcs) | |
185 | + if (bcs) { | |
186 | seq_buf_printf(&s, "Indirect branch serialisation (kernel only)"); | |
187 | + comma = true; | |
188 | + } | |
189 | + | |
190 | + if (ccd) { | |
191 | + if (comma) | |
192 | + seq_buf_printf(&s, ", "); | |
193 | + seq_buf_printf(&s, "Indirect branch cache disabled"); | |
194 | + comma = true; | |
195 | + } | |
196 | ||
197 | - if (bcs && ccd) | |
198 | + if (comma) | |
199 | seq_buf_printf(&s, ", "); | |
200 | ||
201 | - if (ccd) | |
202 | - seq_buf_printf(&s, "Indirect branch cache disabled"); | |
203 | + seq_buf_printf(&s, "Software count cache flush"); | |
204 | + | |
205 | + if (count_cache_flush_type == COUNT_CACHE_FLUSH_HW) | |
206 | + seq_buf_printf(&s, "(hardware accelerated)"); | |
207 | } else | |
208 | seq_buf_printf(&s, "Vulnerable"); | |
209 | ||
210 | @@ -325,4 +346,71 @@ static __init int stf_barrier_debugfs_in | |
211 | } | |
212 | device_initcall(stf_barrier_debugfs_init); | |
213 | #endif /* CONFIG_DEBUG_FS */ | |
214 | + | |
215 | +static void toggle_count_cache_flush(bool enable) | |
216 | +{ | |
217 | + if (!enable || !security_ftr_enabled(SEC_FTR_FLUSH_COUNT_CACHE)) { | |
218 | + patch_instruction_site(&patch__call_flush_count_cache, PPC_INST_NOP); | |
219 | + count_cache_flush_type = COUNT_CACHE_FLUSH_NONE; | |
220 | + pr_info("count-cache-flush: software flush disabled.\n"); | |
221 | + return; | |
222 | + } | |
223 | + | |
224 | + patch_branch_site(&patch__call_flush_count_cache, | |
225 | + (u64)&flush_count_cache, BRANCH_SET_LINK); | |
226 | + | |
227 | + if (!security_ftr_enabled(SEC_FTR_BCCTR_FLUSH_ASSIST)) { | |
228 | + count_cache_flush_type = COUNT_CACHE_FLUSH_SW; | |
229 | + pr_info("count-cache-flush: full software flush sequence enabled.\n"); | |
230 | + return; | |
231 | + } | |
232 | + | |
233 | + patch_instruction_site(&patch__flush_count_cache_return, PPC_INST_BLR); | |
234 | + count_cache_flush_type = COUNT_CACHE_FLUSH_HW; | |
235 | + pr_info("count-cache-flush: hardware assisted flush sequence enabled\n"); | |
236 | +} | |
237 | + | |
238 | +void setup_count_cache_flush(void) | |
239 | +{ | |
240 | + toggle_count_cache_flush(true); | |
241 | +} | |
242 | + | |
243 | +#ifdef CONFIG_DEBUG_FS | |
244 | +static int count_cache_flush_set(void *data, u64 val) | |
245 | +{ | |
246 | + bool enable; | |
247 | + | |
248 | + if (val == 1) | |
249 | + enable = true; | |
250 | + else if (val == 0) | |
251 | + enable = false; | |
252 | + else | |
253 | + return -EINVAL; | |
254 | + | |
255 | + toggle_count_cache_flush(enable); | |
256 | + | |
257 | + return 0; | |
258 | +} | |
259 | + | |
260 | +static int count_cache_flush_get(void *data, u64 *val) | |
261 | +{ | |
262 | + if (count_cache_flush_type == COUNT_CACHE_FLUSH_NONE) | |
263 | + *val = 0; | |
264 | + else | |
265 | + *val = 1; | |
266 | + | |
267 | + return 0; | |
268 | +} | |
269 | + | |
270 | +DEFINE_SIMPLE_ATTRIBUTE(fops_count_cache_flush, count_cache_flush_get, | |
271 | + count_cache_flush_set, "%llu\n"); | |
272 | + | |
273 | +static __init int count_cache_flush_debugfs_init(void) | |
274 | +{ | |
275 | + debugfs_create_file("count_cache_flush", 0600, powerpc_debugfs_root, | |
276 | + NULL, &fops_count_cache_flush); | |
277 | + return 0; | |
278 | +} | |
279 | +device_initcall(count_cache_flush_debugfs_init); | |
280 | +#endif /* CONFIG_DEBUG_FS */ | |
281 | #endif /* CONFIG_PPC_BOOK3S_64 */ |