1 From: jbeulich@novell.com
2 Subject: fold per-CPU VIRQs onto a single IRQ each
3 Patch-mainline: obsolete
5 --- sle11-2009-10-16.orig/arch/x86/kernel/time_32-xen.c 2009-10-28 14:59:22.000000000 +0100
6 +++ sle11-2009-10-16/arch/x86/kernel/time_32-xen.c 2009-10-28 14:59:30.000000000 +0100
7 @@ -726,19 +726,17 @@ int xen_update_persistent_clock(void)
10 /* Dynamically-mapped IRQ. */
11 -DEFINE_PER_CPU(int, timer_irq);
12 +static int __read_mostly timer_irq = -1;
13 +static struct irqaction timer_action = {
14 + .handler = timer_interrupt,
15 + .flags = IRQF_DISABLED,
19 static void __init setup_cpu0_timer_irq(void)
21 - per_cpu(timer_irq, 0) =
22 - bind_virq_to_irqhandler(
26 - IRQF_DISABLED|IRQF_NOBALANCING,
29 - BUG_ON(per_cpu(timer_irq, 0) < 0);
30 + timer_irq = bind_virq_to_irqaction(VIRQ_TIMER, 0, &timer_action);
31 + BUG_ON(timer_irq < 0);
34 void __init time_init(void)
35 @@ -865,8 +863,6 @@ void xen_halt(void)
36 EXPORT_SYMBOL(xen_halt);
39 -static char timer_name[NR_CPUS][15];
41 int __cpuinit local_setup_timer(unsigned int cpu)
44 @@ -892,16 +888,10 @@ int __cpuinit local_setup_timer(unsigned
45 init_missing_ticks_accounting(cpu);
46 } while (read_seqretry(&xtime_lock, seq));
48 - sprintf(timer_name[cpu], "timer%u", cpu);
49 - irq = bind_virq_to_irqhandler(VIRQ_TIMER,
52 - IRQF_DISABLED|IRQF_NOBALANCING,
55 + irq = bind_virq_to_irqaction(VIRQ_TIMER, cpu, &timer_action);
58 - per_cpu(timer_irq, cpu) = irq;
59 + BUG_ON(timer_irq != irq);
63 @@ -909,7 +899,7 @@ int __cpuinit local_setup_timer(unsigned
64 void __cpuinit local_teardown_timer(unsigned int cpu)
67 - unbind_from_irqhandler(per_cpu(timer_irq, cpu), NULL);
68 + unbind_from_per_cpu_irq(timer_irq, cpu, &timer_action);
72 --- sle11-2009-10-16.orig/drivers/xen/core/evtchn.c 2009-06-04 10:47:21.000000000 +0200
73 +++ sle11-2009-10-16/drivers/xen/core/evtchn.c 2009-06-04 10:47:28.000000000 +0200
74 @@ -58,6 +58,23 @@ static DEFINE_SPINLOCK(irq_mapping_updat
75 static int evtchn_to_irq[NR_EVENT_CHANNELS] = {
76 [0 ... NR_EVENT_CHANNELS-1] = -1 };
78 +#if defined(CONFIG_SMP) && defined(CONFIG_X86)
79 +static struct per_cpu_irqaction {
80 + struct irqaction action; /* must be first */
81 + struct per_cpu_irqaction *next;
83 +} *virq_actions[NR_VIRQS];
84 +/* IRQ <-> VIRQ mapping. */
85 +static DECLARE_BITMAP(virq_per_cpu, NR_VIRQS) __read_mostly;
86 +static DEFINE_PER_CPU(int[NR_VIRQS], virq_to_evtchn);
87 +#define BUG_IF_VIRQ_PER_CPU(irq) \
88 + BUG_ON(type_from_irq(irq) == IRQT_VIRQ \
89 + && test_bit(index_from_irq(irq), virq_per_cpu))
91 +#define BUG_IF_VIRQ_PER_CPU(irq) ((void)(irq))
92 +#define PER_CPU_VIRQ_IRQ
95 /* IRQ <-> IPI mapping. */
98 @@ -114,13 +131,6 @@ static inline u32 mk_irq_info(u32 type,
99 * Accessors for packed IRQ information.
102 -#ifdef PER_CPU_IPI_IRQ
103 -static inline unsigned int evtchn_from_irq(int irq)
105 - return irq_info[irq] & ((1U << _EVTCHN_BITS) - 1);
109 static inline unsigned int index_from_irq(int irq)
111 return (irq_info[irq] >> _EVTCHN_BITS) & ((1U << _INDEX_BITS) - 1);
112 @@ -131,20 +141,35 @@ static inline unsigned int type_from_irq
113 return irq_info[irq] >> (32 - _IRQT_BITS);
116 -#ifndef PER_CPU_IPI_IRQ
117 static inline unsigned int evtchn_from_per_cpu_irq(unsigned int irq, unsigned int cpu)
119 - BUG_ON(type_from_irq(irq) != IRQT_IPI);
120 - return per_cpu(ipi_to_evtchn, cpu)[index_from_irq(irq)];
121 + switch (type_from_irq(irq)) {
122 +#ifndef PER_CPU_VIRQ_IRQ
124 + return per_cpu(virq_to_evtchn, cpu)[index_from_irq(irq)];
126 +#ifndef PER_CPU_IPI_IRQ
128 + return per_cpu(ipi_to_evtchn, cpu)[index_from_irq(irq)];
135 static inline unsigned int evtchn_from_irq(unsigned int irq)
137 - if (type_from_irq(irq) != IRQT_IPI)
138 - return irq_info[irq] & ((1U << _EVTCHN_BITS) - 1);
139 - return evtchn_from_per_cpu_irq(irq, smp_processor_id());
141 + switch (type_from_irq(irq)) {
142 +#ifndef PER_CPU_VIRQ_IRQ
145 +#ifndef PER_CPU_IPI_IRQ
148 + return evtchn_from_per_cpu_irq(irq, smp_processor_id());
150 + return irq_info[irq] & ((1U << _EVTCHN_BITS) - 1);
153 /* IRQ <-> VIRQ mapping. */
154 DEFINE_PER_CPU(int, virq_to_irq[NR_VIRQS]) = {[0 ... NR_VIRQS-1] = -1};
155 @@ -479,6 +504,14 @@ static int bind_virq_to_irq(unsigned int
156 evtchn = bind_virq.port;
158 evtchn_to_irq[evtchn] = irq;
159 +#ifndef PER_CPU_VIRQ_IRQ
163 + for_each_possible_cpu(cpu)
164 + per_cpu(virq_to_evtchn, cpu)[virq] = evtchn;
167 irq_info[irq] = mk_irq_info(IRQT_VIRQ, virq, evtchn);
169 per_cpu(virq_to_irq, cpu)[virq] = irq;
170 @@ -533,7 +566,9 @@ static void unbind_from_irq(unsigned int
172 int evtchn = evtchn_from_irq(irq);
174 + BUG_IF_VIRQ_PER_CPU(irq);
177 spin_lock(&irq_mapping_update_lock);
179 if ((--irq_bindcount[irq] == 0) && VALID_EVTCHN(evtchn)) {
180 @@ -546,6 +581,11 @@ static void unbind_from_irq(unsigned int
182 per_cpu(virq_to_irq, cpu_from_evtchn(evtchn))
183 [index_from_irq(irq)] = -1;
184 +#ifndef PER_CPU_VIRQ_IRQ
185 + for_each_possible_cpu(cpu)
186 + per_cpu(virq_to_evtchn, cpu)
187 + [index_from_irq(irq)] = 0;
190 #if defined(CONFIG_SMP) && defined(PER_CPU_IPI_IRQ)
192 @@ -571,11 +611,13 @@ static void unbind_from_irq(unsigned int
193 spin_unlock(&irq_mapping_update_lock);
196 -#if defined(CONFIG_SMP) && !defined(PER_CPU_IPI_IRQ)
197 -void unbind_from_per_cpu_irq(unsigned int irq, unsigned int cpu)
198 +#if defined(CONFIG_SMP) && (!defined(PER_CPU_IPI_IRQ) || !defined(PER_CPU_VIRQ_IRQ))
199 +void unbind_from_per_cpu_irq(unsigned int irq, unsigned int cpu,
200 + struct irqaction *action)
202 struct evtchn_close close;
203 int evtchn = evtchn_from_per_cpu_irq(irq, cpu);
204 + struct irqaction *free_action = NULL;
206 spin_lock(&irq_mapping_update_lock);
208 @@ -586,6 +628,32 @@ void unbind_from_per_cpu_irq(unsigned in
210 BUG_ON(irq_bindcount[irq] <= 1);
211 irq_bindcount[irq]--;
213 +#ifndef PER_CPU_VIRQ_IRQ
214 + if (type_from_irq(irq) == IRQT_VIRQ) {
215 + unsigned int virq = index_from_irq(irq);
216 + struct per_cpu_irqaction *cur, *prev = NULL;
218 + cur = virq_actions[virq];
220 + if (cur->action.dev_id == action) {
221 + cpu_clear(cpu, cur->cpus);
222 + if (cpus_empty(cur->cpus)) {
224 + prev->next = cur->next;
226 + virq_actions[virq] = cur->next;
227 + free_action = action;
229 + } else if (cpu_isset(cpu, cur->cpus))
231 + cur = (prev = cur)->next;
233 + if (!VALID_EVTCHN(evtchn))
238 cpu_clear(cpu, desc->affinity);
241 @@ -593,9 +661,16 @@ void unbind_from_per_cpu_irq(unsigned in
244 switch (type_from_irq(irq)) {
245 +#ifndef PER_CPU_VIRQ_IRQ
247 + per_cpu(virq_to_evtchn, cpu)[index_from_irq(irq)] = 0;
250 +#ifndef PER_CPU_IPI_IRQ
252 per_cpu(ipi_to_evtchn, cpu)[index_from_irq(irq)] = 0;
258 @@ -607,9 +682,16 @@ void unbind_from_per_cpu_irq(unsigned in
259 evtchn_to_irq[evtchn] = -1;
262 +#ifndef PER_CPU_VIRQ_IRQ
265 spin_unlock(&irq_mapping_update_lock);
268 + free_irq(irq, free_action);
270 -#endif /* CONFIG_SMP && !PER_CPU_IPI_IRQ */
271 +EXPORT_SYMBOL_GPL(unbind_from_per_cpu_irq);
272 +#endif /* CONFIG_SMP && (!PER_CPU_IPI_IRQ || !PER_CPU_VIRQ_IRQ) */
274 int bind_caller_port_to_irqhandler(
275 unsigned int caller_port,
276 @@ -691,6 +773,8 @@ int bind_virq_to_irqhandler(
280 + BUG_IF_VIRQ_PER_CPU(virq);
282 irq = bind_virq_to_irq(virq, cpu);
285 @@ -706,6 +790,108 @@ int bind_virq_to_irqhandler(
286 EXPORT_SYMBOL_GPL(bind_virq_to_irqhandler);
289 +#ifndef PER_CPU_VIRQ_IRQ
290 +int bind_virq_to_irqaction(
293 + struct irqaction *action)
295 + struct evtchn_bind_virq bind_virq;
296 + int evtchn, irq, retval = 0;
297 + struct per_cpu_irqaction *cur = NULL, *new;
299 + BUG_ON(!test_bit(virq, virq_per_cpu));
301 + if (action->dev_id)
304 + new = kzalloc(sizeof(*new), GFP_ATOMIC);
306 + new->action = *action;
307 + new->action.dev_id = action;
310 + spin_lock(&irq_mapping_update_lock);
312 + for (cur = virq_actions[virq]; cur; cur = cur->next)
313 + if (cur->action.dev_id == action)
317 + spin_unlock(&irq_mapping_update_lock);
320 + new->next = virq_actions[virq];
321 + virq_actions[virq] = cur = new;
324 + cpu_set(cpu, cur->cpus);
325 + action = &cur->action;
327 + if ((irq = per_cpu(virq_to_irq, cpu)[virq]) == -1) {
332 + if ((irq = find_unbound_irq(true)) < 0) {
334 + virq_actions[virq] = cur->next;
335 + spin_unlock(&irq_mapping_update_lock);
341 + /* Extra reference so count will never drop to zero. */
342 + irq_bindcount[irq]++;
344 + for_each_possible_cpu(cpu)
345 + per_cpu(virq_to_irq, cpu)[virq] = irq;
346 + irq_info[irq] = mk_irq_info(IRQT_VIRQ, virq, 0);
349 + evtchn = per_cpu(virq_to_evtchn, cpu)[virq];
350 + if (!VALID_EVTCHN(evtchn)) {
351 + bind_virq.virq = virq;
352 + bind_virq.vcpu = cpu;
353 + if (HYPERVISOR_event_channel_op(EVTCHNOP_bind_virq,
356 + evtchn = bind_virq.port;
357 + evtchn_to_irq[evtchn] = irq;
358 + per_cpu(virq_to_evtchn, cpu)[virq] = evtchn;
360 + bind_evtchn_to_cpu(evtchn, cpu);
363 + irq_bindcount[irq]++;
365 + spin_unlock(&irq_mapping_update_lock);
371 + unsigned long flags;
373 + local_irq_save(flags);
374 + unmask_evtchn(evtchn);
375 + local_irq_restore(flags);
377 + action->flags |= IRQF_PERCPU;
378 + retval = setup_irq(irq, action);
380 + unbind_from_per_cpu_irq(irq, cpu, cur->action.dev_id);
381 + BUG_ON(retval > 0);
388 +EXPORT_SYMBOL_GPL(bind_virq_to_irqaction);
391 #ifdef PER_CPU_IPI_IRQ
392 int bind_ipi_to_irqhandler(
394 @@ -784,7 +970,7 @@ int __cpuinit bind_ipi_to_irqaction(
395 action->flags |= IRQF_PERCPU;
396 retval = setup_irq(irq, action);
398 - unbind_from_per_cpu_irq(irq, cpu);
399 + unbind_from_per_cpu_irq(irq, cpu, NULL);
403 @@ -819,7 +1005,9 @@ static void rebind_irq_to_cpu(unsigned i
405 int evtchn = evtchn_from_irq(irq);
407 + BUG_IF_VIRQ_PER_CPU(irq);
410 if (VALID_EVTCHN(evtchn))
411 rebind_evtchn_to_cpu(evtchn, tcpu);
413 @@ -1098,7 +1286,9 @@ void notify_remote_via_irq(int irq)
415 int evtchn = evtchn_from_irq(irq);
417 + BUG_ON(type_from_irq(irq) == IRQT_VIRQ);
420 if (VALID_EVTCHN(evtchn))
421 notify_remote_via_evtchn(evtchn);
423 @@ -1106,6 +1296,7 @@ EXPORT_SYMBOL_GPL(notify_remote_via_irq)
425 int irq_to_evtchn_port(int irq)
427 + BUG_IF_VIRQ_PER_CPU(irq);
429 return evtchn_from_irq(irq);
431 @@ -1200,6 +1391,12 @@ static void restore_cpu_virqs(unsigned i
432 if ((irq = per_cpu(virq_to_irq, cpu)[virq]) == -1)
435 +#ifndef PER_CPU_VIRQ_IRQ
436 + if (test_bit(virq, virq_per_cpu)
437 + && !VALID_EVTCHN(per_cpu(virq_to_evtchn, cpu)[virq]))
441 BUG_ON(irq_info[irq] != mk_irq_info(IRQT_VIRQ, virq, 0));
443 /* Get a new binding from Xen. */
444 @@ -1212,7 +1409,19 @@ static void restore_cpu_virqs(unsigned i
446 /* Record the new mapping. */
447 evtchn_to_irq[evtchn] = irq;
448 +#ifdef PER_CPU_VIRQ_IRQ
449 irq_info[irq] = mk_irq_info(IRQT_VIRQ, virq, evtchn);
451 + if (test_bit(virq, virq_per_cpu))
452 + per_cpu(virq_to_evtchn, cpu)[virq] = evtchn;
456 + irq_info[irq] = mk_irq_info(IRQT_VIRQ, virq, evtchn);
457 + for_each_possible_cpu(cpu)
458 + per_cpu(virq_to_evtchn, cpu)[virq] = evtchn;
461 bind_evtchn_to_cpu(evtchn, cpu);
464 @@ -1267,7 +1476,11 @@ static int evtchn_resume(struct sys_devi
466 /* Avoid doing anything in the 'suspend cancelled' case. */
467 status.dom = DOMID_SELF;
468 +#ifdef PER_CPU_VIRQ_IRQ
469 status.port = evtchn_from_irq(__get_cpu_var(virq_to_irq)[VIRQ_TIMER]);
471 + status.port = __get_cpu_var(virq_to_evtchn)[VIRQ_TIMER];
473 if (HYPERVISOR_event_channel_op(EVTCHNOP_status, &status))
475 if (status.status == EVTCHNSTAT_virq
476 @@ -1407,6 +1620,15 @@ void __init xen_init_IRQ(void)
478 struct physdev_pirq_eoi_gmfn eoi_gmfn;
480 +#ifndef PER_CPU_VIRQ_IRQ
481 + __set_bit(VIRQ_TIMER, virq_per_cpu);
482 + __set_bit(VIRQ_DEBUG, virq_per_cpu);
483 + __set_bit(VIRQ_XENOPROF, virq_per_cpu);
485 + __set_bit(VIRQ_ITC, virq_per_cpu);
489 init_evtchn_cpu_bindings();
491 pirq_needs_eoi = alloc_bootmem_pages(sizeof(unsigned long)
492 --- sle11-2009-10-16.orig/drivers/xen/core/smpboot.c 2009-06-04 10:47:21.000000000 +0200
493 +++ sle11-2009-10-16/drivers/xen/core/smpboot.c 2009-06-04 10:47:28.000000000 +0200
494 @@ -163,11 +163,11 @@ static int __cpuinit xen_smp_intr_init(u
496 xen_spinlock_cleanup(cpu);
498 - unbind_from_per_cpu_irq(call1func_irq, cpu);
499 + unbind_from_per_cpu_irq(call1func_irq, cpu, NULL);
501 - unbind_from_per_cpu_irq(callfunc_irq, cpu);
502 + unbind_from_per_cpu_irq(callfunc_irq, cpu, NULL);
504 - unbind_from_per_cpu_irq(resched_irq, cpu);
505 + unbind_from_per_cpu_irq(resched_irq, cpu, NULL);
509 @@ -177,9 +177,9 @@ static void __cpuinit xen_smp_intr_exit(
511 local_teardown_timer(cpu);
513 - unbind_from_per_cpu_irq(resched_irq, cpu);
514 - unbind_from_per_cpu_irq(callfunc_irq, cpu);
515 - unbind_from_per_cpu_irq(call1func_irq, cpu);
516 + unbind_from_per_cpu_irq(resched_irq, cpu, NULL);
517 + unbind_from_per_cpu_irq(callfunc_irq, cpu, NULL);
518 + unbind_from_per_cpu_irq(call1func_irq, cpu, NULL);
519 xen_spinlock_cleanup(cpu);
522 --- sle11-2009-10-16.orig/drivers/xen/core/spinlock.c 2009-06-04 10:47:21.000000000 +0200
523 +++ sle11-2009-10-16/drivers/xen/core/spinlock.c 2009-06-04 10:47:28.000000000 +0200
524 @@ -57,7 +57,7 @@ int __cpuinit xen_spinlock_init(unsigned
526 void __cpuinit xen_spinlock_cleanup(unsigned int cpu)
528 - unbind_from_per_cpu_irq(spinlock_irq, cpu);
529 + unbind_from_per_cpu_irq(spinlock_irq, cpu, NULL);
532 int xen_spin_wait(raw_spinlock_t *lock, unsigned int token)
533 --- sle11-2009-10-16.orig/drivers/xen/netback/netback.c 2009-02-16 16:18:36.000000000 +0100
534 +++ sle11-2009-10-16/drivers/xen/netback/netback.c 2009-06-04 10:47:28.000000000 +0200
535 @@ -1561,6 +1561,12 @@ static irqreturn_t netif_be_dbg(int irq,
540 +static struct irqaction netif_be_dbg_action = {
541 + .handler = netif_be_dbg,
542 + .flags = IRQF_SHARED,
543 + .name = "net-be-dbg"
547 static int __init netback_init(void)
548 @@ -1620,12 +1626,9 @@ static int __init netback_init(void)
551 #ifdef NETBE_DEBUG_INTERRUPT
552 - (void)bind_virq_to_irqhandler(VIRQ_DEBUG,
558 + (void)bind_virq_to_irqaction(VIRQ_DEBUG,
560 + &netif_be_dbg_action);
564 --- sle11-2009-10-16.orig/drivers/xen/xenoprof/xenoprofile.c 2009-03-16 16:33:40.000000000 +0100
565 +++ sle11-2009-10-16/drivers/xen/xenoprof/xenoprofile.c 2009-06-04 10:47:28.000000000 +0200
566 @@ -210,6 +210,11 @@ static irqreturn_t xenoprof_ovf_interrup
570 +static struct irqaction ovf_action = {
571 + .handler = xenoprof_ovf_interrupt,
572 + .flags = IRQF_DISABLED,
576 static void unbind_virq(void)
578 @@ -217,7 +222,7 @@ static void unbind_virq(void)
580 for_each_online_cpu(i) {
581 if (ovf_irq[i] >= 0) {
582 - unbind_from_irqhandler(ovf_irq[i], NULL);
583 + unbind_from_per_cpu_irq(ovf_irq[i], i, &ovf_action);
587 @@ -230,12 +235,7 @@ static int bind_virq(void)
590 for_each_online_cpu(i) {
591 - result = bind_virq_to_irqhandler(VIRQ_XENOPROF,
593 - xenoprof_ovf_interrupt,
594 - IRQF_DISABLED|IRQF_NOBALANCING,
597 + result = bind_virq_to_irqaction(VIRQ_XENOPROF, i, &ovf_action);
601 --- sle11-2009-10-16.orig/include/xen/evtchn.h 2009-06-04 10:47:21.000000000 +0200
602 +++ sle11-2009-10-16/include/xen/evtchn.h 2009-06-04 10:47:28.000000000 +0200
603 @@ -78,6 +78,17 @@ int bind_virq_to_irqhandler(
604 unsigned long irqflags,
607 +#if defined(CONFIG_SMP) && defined(CONFIG_XEN) && defined(CONFIG_X86)
608 +int bind_virq_to_irqaction(
611 + struct irqaction *action);
613 +#define bind_virq_to_irqaction(virq, cpu, action) \
614 + bind_virq_to_irqhandler(virq, cpu, (action)->handler, \
615 + (action)->flags | IRQF_NOBALANCING, \
616 + (action)->name, action)
618 #if defined(CONFIG_SMP) && !defined(MODULE)
620 int bind_ipi_to_irqhandler(
621 @@ -102,9 +113,13 @@ int bind_ipi_to_irqaction(
623 void unbind_from_irqhandler(unsigned int irq, void *dev_id);
625 -#if defined(CONFIG_SMP) && !defined(MODULE) && defined(CONFIG_X86)
626 +#if defined(CONFIG_SMP) && defined(CONFIG_XEN) && defined(CONFIG_X86)
627 /* Specialized unbind function for per-CPU IRQs. */
628 -void unbind_from_per_cpu_irq(unsigned int irq, unsigned int cpu);
629 +void unbind_from_per_cpu_irq(unsigned int irq, unsigned int cpu,
630 + struct irqaction *);
632 +#define unbind_from_per_cpu_irq(irq, cpu, action) \
633 + unbind_from_irqhandler(irq, action)