]>
Commit | Line | Data |
---|---|---|
2cb7cef9 BS |
1 | From: Suresh Siddha <suresh.b.siddha@intel.com> |
2 | Subject: x64, x2apic/intr-remap: basic apic ops support | |
3 | References: fate #303948 and fate #303984 | |
4 | Patch-Mainline: queued for .28 | |
5 | Commit-ID: 1b374e4d6f8b3eb2fcd034fcc24ea8ba1dfde7aa | |
6 | ||
7 | Signed-off-by: Thomas Renninger <trenn@suse.de> | |
8 | ||
9 | Introduce basic apic operations which handle the apic programming. This | |
10 | will be used later to introduce another specific operations for x2apic. | |
11 | ||
12 | For the perfomance critial accesses like IPI's, EOI etc, we use the | |
13 | native operations as they are already referenced by different | |
14 | indirections like genapic, irq_chip etc. | |
15 | ||
16 | 64bit Paravirt ops can also define their apic operations accordingly. | |
17 | ||
18 | Signed-off-by: Suresh Siddha <suresh.b.siddha@intel.com> | |
19 | Cc: akpm@linux-foundation.org | |
20 | Cc: arjan@linux.intel.com | |
21 | Cc: andi@firstfloor.org | |
22 | Cc: ebiederm@xmission.com | |
23 | Cc: jbarnes@virtuousgeek.org | |
24 | Cc: steiner@sgi.com | |
25 | Signed-off-by: Ingo Molnar <mingo@elte.hu> | |
26 | ||
27 | --- | |
28 | arch/x86/kernel/apic_32.c | 6 ++++++ | |
29 | arch/x86/kernel/apic_64.c | 33 +++++++++++++++++++++++++++++++-- | |
30 | arch/x86/kernel/io_apic_64.c | 8 ++++---- | |
31 | arch/x86/kernel/paravirt.c | 6 ++++-- | |
32 | arch/x86/kernel/smpboot.c | 36 ++++++++++++++---------------------- | |
33 | include/asm-x86/apic.h | 35 +++++++++++++++++++++++++++++++---- | |
34 | include/asm-x86/ipi.h | 16 +++++++++++----- | |
35 | include/asm-x86/paravirt.h | 2 ++ | |
36 | include/asm-x86/smp.h | 2 +- | |
37 | 9 files changed, 104 insertions(+), 40 deletions(-) | |
38 | ||
39 | --- a/arch/x86/kernel/apic_32.c | |
40 | +++ b/arch/x86/kernel/apic_32.c | |
41 | @@ -145,6 +145,12 @@ static int modern_apic(void) | |
42 | return lapic_get_version() >= 0x14; | |
43 | } | |
44 | ||
45 | +void apic_icr_write(u32 low, u32 id) | |
46 | +{ | |
47 | + apic_write(APIC_ICR2, SET_APIC_DEST_FIELD(id)); | |
48 | + apic_write(APIC_ICR, low); | |
49 | +} | |
50 | + | |
51 | void apic_wait_icr_idle(void) | |
52 | { | |
53 | while (apic_read(APIC_ICR) & APIC_ICR_BUSY) | |
54 | --- a/arch/x86/kernel/apic_64.c | |
55 | +++ b/arch/x86/kernel/apic_64.c | |
56 | @@ -118,13 +118,13 @@ static int modern_apic(void) | |
57 | return lapic_get_version() >= 0x14; | |
58 | } | |
59 | ||
60 | -void apic_wait_icr_idle(void) | |
61 | +void xapic_wait_icr_idle(void) | |
62 | { | |
63 | while (apic_read(APIC_ICR) & APIC_ICR_BUSY) | |
64 | cpu_relax(); | |
65 | } | |
66 | ||
67 | -u32 safe_apic_wait_icr_idle(void) | |
68 | +u32 safe_xapic_wait_icr_idle(void) | |
69 | { | |
70 | u32 send_status; | |
71 | int timeout; | |
72 | @@ -140,6 +140,35 @@ u32 safe_apic_wait_icr_idle(void) | |
73 | return send_status; | |
74 | } | |
75 | ||
76 | +void xapic_icr_write(u32 low, u32 id) | |
77 | +{ | |
78 | + apic_write(APIC_ICR2, id << 24); | |
79 | + apic_write(APIC_ICR, low); | |
80 | +} | |
81 | + | |
82 | +u64 xapic_icr_read(void) | |
83 | +{ | |
84 | + u32 icr1, icr2; | |
85 | + | |
86 | + icr2 = apic_read(APIC_ICR2); | |
87 | + icr1 = apic_read(APIC_ICR); | |
88 | + | |
89 | + return (icr1 | ((u64)icr2 << 32)); | |
90 | +} | |
91 | + | |
92 | +static struct apic_ops xapic_ops = { | |
93 | + .read = native_apic_mem_read, | |
94 | + .write = native_apic_mem_write, | |
95 | + .icr_read = xapic_icr_read, | |
96 | + .icr_write = xapic_icr_write, | |
97 | + .wait_icr_idle = xapic_wait_icr_idle, | |
98 | + .safe_wait_icr_idle = safe_xapic_wait_icr_idle, | |
99 | +}; | |
100 | + | |
101 | +struct apic_ops __read_mostly *apic_ops = &xapic_ops; | |
102 | + | |
103 | +EXPORT_SYMBOL_GPL(apic_ops); | |
104 | + | |
105 | /** | |
106 | * enable_NMI_through_LVT0 - enable NMI through local vector table 0 | |
107 | */ | |
108 | --- a/arch/x86/kernel/io_apic_64.c | |
109 | +++ b/arch/x86/kernel/io_apic_64.c | |
110 | @@ -1156,6 +1156,7 @@ static __apicdebuginit void print_APIC_b | |
111 | void __apicdebuginit print_local_APIC(void * dummy) | |
112 | { | |
113 | unsigned int v, ver, maxlvt; | |
114 | + unsigned long icr; | |
115 | ||
116 | if (apic_verbosity == APIC_QUIET) | |
117 | return; | |
118 | @@ -1199,10 +1200,9 @@ void __apicdebuginit print_local_APIC(vo | |
119 | v = apic_read(APIC_ESR); | |
120 | printk(KERN_DEBUG "... APIC ESR: %08x\n", v); | |
121 | ||
122 | - v = apic_read(APIC_ICR); | |
123 | - printk(KERN_DEBUG "... APIC ICR: %08x\n", v); | |
124 | - v = apic_read(APIC_ICR2); | |
125 | - printk(KERN_DEBUG "... APIC ICR2: %08x\n", v); | |
126 | + icr = apic_icr_read(); | |
127 | + printk(KERN_DEBUG "... APIC ICR: %08x\n", icr); | |
128 | + printk(KERN_DEBUG "... APIC ICR2: %08x\n", icr >> 32); | |
129 | ||
130 | v = apic_read(APIC_LVTT); | |
131 | printk(KERN_DEBUG "... APIC LVTT: %08x\n", v); | |
132 | --- a/arch/x86/kernel/paravirt.c | |
133 | +++ b/arch/x86/kernel/paravirt.c | |
134 | @@ -373,8 +373,10 @@ struct pv_cpu_ops pv_cpu_ops = { | |
135 | ||
136 | struct pv_apic_ops pv_apic_ops = { | |
137 | #ifdef CONFIG_X86_LOCAL_APIC | |
138 | - .apic_write = native_apic_write, | |
139 | - .apic_read = native_apic_read, | |
140 | +#ifndef CONFIG_X86_64 | |
141 | + .apic_write = native_apic_mem_write, | |
142 | + .apic_read = native_apic_mem_read, | |
143 | +#endif | |
144 | .setup_boot_clock = setup_boot_APIC_clock, | |
145 | .setup_secondary_clock = setup_secondary_APIC_clock, | |
146 | .startup_ipi_hook = paravirt_nop, | |
147 | --- a/arch/x86/kernel/smpboot.c | |
148 | +++ b/arch/x86/kernel/smpboot.c | |
149 | @@ -123,7 +123,6 @@ EXPORT_PER_CPU_SYMBOL(cpu_info); | |
150 | ||
151 | static atomic_t init_deasserted; | |
152 | ||
153 | -static int boot_cpu_logical_apicid; | |
154 | ||
155 | /* representing cpus for which sibling maps can be computed */ | |
156 | static cpumask_t cpu_sibling_setup_map; | |
157 | @@ -165,6 +164,8 @@ static void unmap_cpu_to_node(int cpu) | |
158 | #endif | |
159 | ||
160 | #ifdef CONFIG_X86_32 | |
161 | +static int boot_cpu_logical_apicid; | |
162 | + | |
163 | u8 cpu_2_logical_apicid[NR_CPUS] __read_mostly = | |
164 | { [0 ... NR_CPUS-1] = BAD_APICID }; | |
165 | ||
166 | @@ -548,8 +549,7 @@ static inline void __inquire_remote_apic | |
167 | printk(KERN_CONT | |
168 | "a previous APIC delivery may have failed\n"); | |
169 | ||
170 | - apic_write(APIC_ICR2, SET_APIC_DEST_FIELD(apicid)); | |
171 | - apic_write(APIC_ICR, APIC_DM_REMRD | regs[i]); | |
172 | + apic_icr_write(APIC_DM_REMRD | regs[i], apicid); | |
173 | ||
174 | timeout = 0; | |
175 | do { | |
176 | @@ -580,12 +580,7 @@ wakeup_secondary_cpu(int logical_apicid, | |
177 | unsigned long send_status, accept_status = 0; | |
178 | int maxlvt; | |
179 | ||
180 | - /* Target chip */ | |
181 | - apic_write(APIC_ICR2, SET_APIC_DEST_FIELD(logical_apicid)); | |
182 | - | |
183 | - /* Boot on the stack */ | |
184 | - /* Kick the second */ | |
185 | - apic_write(APIC_ICR, APIC_DM_NMI | APIC_DEST_LOGICAL); | |
186 | + apic_icr_write(APIC_DM_NMI | APIC_DEST_LOGICAL, logical_apicid); | |
187 | ||
188 | pr_debug("Waiting for send to finish...\n"); | |
189 | send_status = safe_apic_wait_icr_idle(); | |
190 | @@ -635,16 +630,14 @@ wakeup_secondary_cpu(int phys_apicid, un | |
191 | ||
192 | pr_debug("Asserting INIT.\n"); | |
193 | ||
194 | - /* | |
195 | - * Turn INIT on target chip | |
196 | - */ | |
197 | - apic_write(APIC_ICR2, SET_APIC_DEST_FIELD(phys_apicid)); | |
198 | - | |
199 | + /* | |
200 | + * Turn INIT on target chip | |
201 | + */ | |
202 | /* | |
203 | * Send IPI | |
204 | */ | |
205 | - apic_write(APIC_ICR, | |
206 | - APIC_INT_LEVELTRIG | APIC_INT_ASSERT | APIC_DM_INIT); | |
207 | + apic_icr_write(APIC_INT_LEVELTRIG | APIC_INT_ASSERT | APIC_DM_INIT, | |
208 | + phys_apicid); | |
209 | ||
210 | pr_debug("Waiting for send to finish...\n"); | |
211 | send_status = safe_apic_wait_icr_idle(); | |
212 | @@ -654,10 +647,8 @@ wakeup_secondary_cpu(int phys_apicid, un | |
213 | pr_debug("Deasserting INIT.\n"); | |
214 | ||
215 | /* Target chip */ | |
216 | - apic_write(APIC_ICR2, SET_APIC_DEST_FIELD(phys_apicid)); | |
217 | - | |
218 | /* Send IPI */ | |
219 | - apic_write(APIC_ICR, APIC_INT_LEVELTRIG | APIC_DM_INIT); | |
220 | + apic_icr_write(APIC_INT_LEVELTRIG | APIC_DM_INIT, phys_apicid); | |
221 | ||
222 | pr_debug("Waiting for send to finish...\n"); | |
223 | send_status = safe_apic_wait_icr_idle(); | |
224 | @@ -700,11 +691,10 @@ wakeup_secondary_cpu(int phys_apicid, un | |
225 | */ | |
226 | ||
227 | /* Target chip */ | |
228 | - apic_write(APIC_ICR2, SET_APIC_DEST_FIELD(phys_apicid)); | |
229 | - | |
230 | /* Boot on the stack */ | |
231 | /* Kick the second */ | |
232 | - apic_write(APIC_ICR, APIC_DM_STARTUP | (start_eip >> 12)); | |
233 | + apic_icr_write(APIC_DM_STARTUP | (start_eip >> 12), | |
234 | + phys_apicid); | |
235 | ||
236 | /* | |
237 | * Give the other CPU some time to accept the IPI. | |
238 | @@ -1173,7 +1163,9 @@ void __init native_smp_prepare_cpus(unsi | |
239 | * Setup boot CPU information | |
240 | */ | |
241 | smp_store_cpu_info(0); /* Final full version of the data */ | |
242 | +#ifdef CONFIG_X86_32 | |
243 | boot_cpu_logical_apicid = logical_smp_processor_id(); | |
244 | +#endif | |
245 | current_thread_info()->cpu = 0; /* needed? */ | |
246 | set_cpu_sibling_map(0); | |
247 | ||
248 | --- a/include/asm-x86/apic.h | |
249 | +++ b/include/asm-x86/apic.h | |
250 | @@ -47,15 +47,17 @@ extern int disable_apic; | |
251 | #ifdef CONFIG_PARAVIRT | |
252 | #include <asm/paravirt.h> | |
253 | #else | |
254 | -#define apic_write native_apic_write | |
255 | -#define apic_read native_apic_read | |
256 | +#ifndef CONFIG_X86_64 | |
257 | +#define apic_write native_apic_mem_write | |
258 | +#define apic_read native_apic_mem_read | |
259 | +#endif | |
260 | #define setup_boot_clock setup_boot_APIC_clock | |
261 | #define setup_secondary_clock setup_secondary_APIC_clock | |
262 | #endif | |
263 | ||
264 | extern int is_vsmp_box(void); | |
265 | ||
266 | -static inline void native_apic_write(unsigned long reg, u32 v) | |
267 | +static inline void native_apic_mem_write(u32 reg, u32 v) | |
268 | { | |
269 | volatile u32 *addr = (volatile u32 *)(APIC_BASE + reg); | |
270 | ||
271 | @@ -64,13 +66,38 @@ static inline void native_apic_write(uns | |
272 | ASM_OUTPUT2("0" (v), "m" (*addr))); | |
273 | } | |
274 | ||
275 | -static inline u32 native_apic_read(unsigned long reg) | |
276 | +static inline u32 native_apic_mem_read(u32 reg) | |
277 | { | |
278 | return *((volatile u32 *)(APIC_BASE + reg)); | |
279 | } | |
280 | ||
281 | +#ifdef CONFIG_X86_32 | |
282 | extern void apic_wait_icr_idle(void); | |
283 | extern u32 safe_apic_wait_icr_idle(void); | |
284 | +extern void apic_icr_write(u32 low, u32 id); | |
285 | +#else | |
286 | + | |
287 | +struct apic_ops { | |
288 | + u32 (*read)(u32 reg); | |
289 | + void (*write)(u32 reg, u32 v); | |
290 | + void (*write_atomic)(u32 reg, u32 v); | |
291 | + u64 (*icr_read)(void); | |
292 | + void (*icr_write)(u32 low, u32 high); | |
293 | + void (*wait_icr_idle)(void); | |
294 | + u32 (*safe_wait_icr_idle)(void); | |
295 | +}; | |
296 | + | |
297 | +extern struct apic_ops *apic_ops; | |
298 | + | |
299 | +#define apic_read (apic_ops->read) | |
300 | +#define apic_write (apic_ops->write) | |
301 | +#define apic_write_atomic (apic_ops->write_atomic) | |
302 | +#define apic_icr_read (apic_ops->icr_read) | |
303 | +#define apic_icr_write (apic_ops->icr_write) | |
304 | +#define apic_wait_icr_idle (apic_ops->wait_icr_idle) | |
305 | +#define safe_apic_wait_icr_idle (apic_ops->safe_wait_icr_idle) | |
306 | +#endif | |
307 | + | |
308 | extern int get_physical_broadcast(void); | |
309 | ||
310 | static inline void ack_APIC_irq(void) | |
311 | --- a/include/asm-x86/ipi.h | |
312 | +++ b/include/asm-x86/ipi.h | |
313 | @@ -49,6 +49,12 @@ static inline int __prepare_ICR2(unsigne | |
314 | return SET_APIC_DEST_FIELD(mask); | |
315 | } | |
316 | ||
317 | +static inline void __xapic_wait_icr_idle(void) | |
318 | +{ | |
319 | + while (native_apic_mem_read(APIC_ICR) & APIC_ICR_BUSY) | |
320 | + cpu_relax(); | |
321 | +} | |
322 | + | |
323 | static inline void __send_IPI_shortcut(unsigned int shortcut, int vector, | |
324 | unsigned int dest) | |
325 | { | |
326 | @@ -64,7 +70,7 @@ static inline void __send_IPI_shortcut(u | |
327 | /* | |
328 | * Wait for idle. | |
329 | */ | |
330 | - apic_wait_icr_idle(); | |
331 | + __xapic_wait_icr_idle(); | |
332 | ||
333 | /* | |
334 | * No need to touch the target chip field | |
335 | @@ -74,7 +80,7 @@ static inline void __send_IPI_shortcut(u | |
336 | /* | |
337 | * Send the IPI. The write to APIC_ICR fires this off. | |
338 | */ | |
339 | - apic_write(APIC_ICR, cfg); | |
340 | + native_apic_mem_write(APIC_ICR, cfg); | |
341 | } | |
342 | ||
343 | /* | |
344 | @@ -92,13 +98,13 @@ static inline void __send_IPI_dest_field | |
345 | if (unlikely(vector == NMI_VECTOR)) | |
346 | safe_apic_wait_icr_idle(); | |
347 | else | |
348 | - apic_wait_icr_idle(); | |
349 | + __xapic_wait_icr_idle(); | |
350 | ||
351 | /* | |
352 | * prepare target chip field | |
353 | */ | |
354 | cfg = __prepare_ICR2(mask); | |
355 | - apic_write(APIC_ICR2, cfg); | |
356 | + native_apic_mem_write(APIC_ICR2, cfg); | |
357 | ||
358 | /* | |
359 | * program the ICR | |
360 | @@ -108,7 +114,7 @@ static inline void __send_IPI_dest_field | |
361 | /* | |
362 | * Send the IPI. The write to APIC_ICR fires this off. | |
363 | */ | |
364 | - apic_write(APIC_ICR, cfg); | |
365 | + native_apic_mem_write(APIC_ICR, cfg); | |
366 | } | |
367 | ||
368 | static inline void send_IPI_mask_sequence(cpumask_t mask, int vector) | |
369 | --- a/include/asm-x86/paravirt.h | |
370 | +++ b/include/asm-x86/paravirt.h | |
371 | @@ -901,6 +901,7 @@ static inline void slow_down_io(void) | |
372 | /* | |
373 | * Basic functions accessing APICs. | |
374 | */ | |
375 | +#ifndef CONFIG_X86_64 | |
376 | static inline void apic_write(unsigned long reg, u32 v) | |
377 | { | |
378 | PVOP_VCALL2(pv_apic_ops.apic_write, reg, v); | |
379 | @@ -910,6 +911,7 @@ static inline u32 apic_read(unsigned lon | |
380 | { | |
381 | return PVOP_CALL1(unsigned long, pv_apic_ops.apic_read, reg); | |
382 | } | |
383 | +#endif | |
384 | ||
385 | static inline void setup_boot_clock(void) | |
386 | { | |
387 | --- a/include/asm-x86/smp.h | |
388 | +++ b/include/asm-x86/smp.h | |
389 | @@ -165,13 +165,13 @@ extern int safe_smp_processor_id(void); | |
390 | ||
391 | #ifdef CONFIG_X86_LOCAL_APIC | |
392 | ||
393 | +#ifndef CONFIG_X86_64 | |
394 | static inline int logical_smp_processor_id(void) | |
395 | { | |
396 | /* we don't want to mark this access volatile - bad code generation */ | |
397 | return GET_APIC_LOGICAL_ID(*(u32 *)(APIC_BASE + APIC_LDR)); | |
398 | } | |
399 | ||
400 | -#ifndef CONFIG_X86_64 | |
401 | static inline unsigned int read_apic_id(void) | |
402 | { | |
403 | return *(u32 *)(APIC_BASE + APIC_ID); |