]>
Commit | Line | Data |
---|---|---|
5e605b64 JA |
1 | /* |
2 | * Functions related to interrupt-poll handling in the block layer. This | |
3 | * is similar to NAPI for network devices. | |
4 | */ | |
5 | #include <linux/kernel.h> | |
6 | #include <linux/module.h> | |
7 | #include <linux/init.h> | |
8 | #include <linux/bio.h> | |
5e605b64 JA |
9 | #include <linux/interrupt.h> |
10 | #include <linux/cpu.h> | |
511cbce2 | 11 | #include <linux/irq_poll.h> |
5e605b64 JA |
12 | #include <linux/delay.h> |
13 | ||
511cbce2 | 14 | static unsigned int irq_poll_budget __read_mostly = 256; |
37867ae7 | 15 | |
5e605b64 JA |
16 | static DEFINE_PER_CPU(struct list_head, blk_cpu_iopoll); |
17 | ||
18 | /** | |
511cbce2 | 19 | * irq_poll_sched - Schedule a run of the iopoll handler |
5e605b64 JA |
20 | * @iop: The parent iopoll structure |
21 | * | |
22 | * Description: | |
511cbce2 | 23 | * Add this irq_poll structure to the pending poll list and trigger the |
1badcfbd | 24 | * raise of the blk iopoll softirq. The driver must already have gotten a |
511cbce2 | 25 | * successful return from irq_poll_sched_prep() before calling this. |
5e605b64 | 26 | **/ |
511cbce2 | 27 | void irq_poll_sched(struct irq_poll *iop) |
5e605b64 JA |
28 | { |
29 | unsigned long flags; | |
30 | ||
31 | local_irq_save(flags); | |
170d800a | 32 | list_add_tail(&iop->list, this_cpu_ptr(&blk_cpu_iopoll)); |
511cbce2 | 33 | __raise_softirq_irqoff(IRQ_POLL_SOFTIRQ); |
5e605b64 JA |
34 | local_irq_restore(flags); |
35 | } | |
511cbce2 | 36 | EXPORT_SYMBOL(irq_poll_sched); |
5e605b64 JA |
37 | |
38 | /** | |
511cbce2 | 39 | * __irq_poll_complete - Mark this @iop as un-polled again |
5e605b64 JA |
40 | * @iop: The parent iopoll structure |
41 | * | |
42 | * Description: | |
511cbce2 | 43 | * See irq_poll_complete(). This function must be called with interrupts |
1badcfbd | 44 | * disabled. |
5e605b64 | 45 | **/ |
511cbce2 | 46 | void __irq_poll_complete(struct irq_poll *iop) |
5e605b64 JA |
47 | { |
48 | list_del(&iop->list); | |
4e857c58 | 49 | smp_mb__before_atomic(); |
511cbce2 | 50 | clear_bit_unlock(IRQ_POLL_F_SCHED, &iop->state); |
5e605b64 | 51 | } |
511cbce2 | 52 | EXPORT_SYMBOL(__irq_poll_complete); |
5e605b64 JA |
53 | |
54 | /** | |
511cbce2 | 55 | * irq_poll_complete - Mark this @iop as un-polled again |
5e605b64 JA |
56 | * @iop: The parent iopoll structure |
57 | * | |
58 | * Description: | |
1badcfbd JA |
59 | * If a driver consumes less than the assigned budget in its run of the |
60 | * iopoll handler, it'll end the polled mode by calling this function. The | |
511cbce2 | 61 | * iopoll handler will not be invoked again before irq_poll_sched_prep() |
1badcfbd | 62 | * is called. |
5e605b64 | 63 | **/ |
511cbce2 | 64 | void irq_poll_complete(struct irq_poll *iop) |
5e605b64 JA |
65 | { |
66 | unsigned long flags; | |
67 | ||
68 | local_irq_save(flags); | |
511cbce2 | 69 | __irq_poll_complete(iop); |
5e605b64 JA |
70 | local_irq_restore(flags); |
71 | } | |
511cbce2 | 72 | EXPORT_SYMBOL(irq_poll_complete); |
5e605b64 | 73 | |
511cbce2 | 74 | static void irq_poll_softirq(struct softirq_action *h) |
5e605b64 | 75 | { |
170d800a | 76 | struct list_head *list = this_cpu_ptr(&blk_cpu_iopoll); |
511cbce2 | 77 | int rearm = 0, budget = irq_poll_budget; |
5e605b64 | 78 | unsigned long start_time = jiffies; |
5e605b64 JA |
79 | |
80 | local_irq_disable(); | |
81 | ||
82 | while (!list_empty(list)) { | |
511cbce2 | 83 | struct irq_poll *iop; |
5e605b64 JA |
84 | int work, weight; |
85 | ||
86 | /* | |
87 | * If softirq window is exhausted then punt. | |
88 | */ | |
89 | if (budget <= 0 || time_after(jiffies, start_time)) { | |
90 | rearm = 1; | |
91 | break; | |
92 | } | |
93 | ||
94 | local_irq_enable(); | |
95 | ||
96 | /* Even though interrupts have been re-enabled, this | |
97 | * access is safe because interrupts can only add new | |
98 | * entries to the tail of this list, and only ->poll() | |
99 | * calls can remove this head entry from the list. | |
100 | */ | |
511cbce2 | 101 | iop = list_entry(list->next, struct irq_poll, list); |
5e605b64 JA |
102 | |
103 | weight = iop->weight; | |
104 | work = 0; | |
511cbce2 | 105 | if (test_bit(IRQ_POLL_F_SCHED, &iop->state)) |
5e605b64 JA |
106 | work = iop->poll(iop, weight); |
107 | ||
108 | budget -= work; | |
109 | ||
110 | local_irq_disable(); | |
111 | ||
fca51d64 JA |
112 | /* |
113 | * Drivers must not modify the iopoll state, if they | |
114 | * consume their assigned weight (or more, some drivers can't | |
115 | * easily just stop processing, they have to complete an | |
116 | * entire mask of commands).In such cases this code | |
117 | * still "owns" the iopoll instance and therefore can | |
5e605b64 JA |
118 | * move the instance around on the list at-will. |
119 | */ | |
120 | if (work >= weight) { | |
511cbce2 CH |
121 | if (irq_poll_disable_pending(iop)) |
122 | __irq_poll_complete(iop); | |
5e605b64 JA |
123 | else |
124 | list_move_tail(&iop->list, list); | |
125 | } | |
126 | } | |
127 | ||
128 | if (rearm) | |
511cbce2 | 129 | __raise_softirq_irqoff(IRQ_POLL_SOFTIRQ); |
5e605b64 JA |
130 | |
131 | local_irq_enable(); | |
132 | } | |
133 | ||
134 | /** | |
511cbce2 | 135 | * irq_poll_disable - Disable iopoll on this @iop |
5e605b64 JA |
136 | * @iop: The parent iopoll structure |
137 | * | |
138 | * Description: | |
139 | * Disable io polling and wait for any pending callbacks to have completed. | |
140 | **/ | |
511cbce2 | 141 | void irq_poll_disable(struct irq_poll *iop) |
5e605b64 | 142 | { |
511cbce2 CH |
143 | set_bit(IRQ_POLL_F_DISABLE, &iop->state); |
144 | while (test_and_set_bit(IRQ_POLL_F_SCHED, &iop->state)) | |
5e605b64 | 145 | msleep(1); |
511cbce2 | 146 | clear_bit(IRQ_POLL_F_DISABLE, &iop->state); |
5e605b64 | 147 | } |
511cbce2 | 148 | EXPORT_SYMBOL(irq_poll_disable); |
5e605b64 JA |
149 | |
150 | /** | |
511cbce2 | 151 | * irq_poll_enable - Enable iopoll on this @iop |
5e605b64 JA |
152 | * @iop: The parent iopoll structure |
153 | * | |
154 | * Description: | |
1badcfbd JA |
155 | * Enable iopoll on this @iop. Note that the handler run will not be |
156 | * scheduled, it will only mark it as active. | |
5e605b64 | 157 | **/ |
511cbce2 | 158 | void irq_poll_enable(struct irq_poll *iop) |
5e605b64 | 159 | { |
511cbce2 | 160 | BUG_ON(!test_bit(IRQ_POLL_F_SCHED, &iop->state)); |
4e857c58 | 161 | smp_mb__before_atomic(); |
511cbce2 | 162 | clear_bit_unlock(IRQ_POLL_F_SCHED, &iop->state); |
5e605b64 | 163 | } |
511cbce2 | 164 | EXPORT_SYMBOL(irq_poll_enable); |
5e605b64 JA |
165 | |
166 | /** | |
511cbce2 | 167 | * irq_poll_init - Initialize this @iop |
5e605b64 JA |
168 | * @iop: The parent iopoll structure |
169 | * @weight: The default weight (or command completion budget) | |
170 | * @poll_fn: The handler to invoke | |
171 | * | |
172 | * Description: | |
511cbce2 CH |
173 | * Initialize this irq_poll structure. Before being actively used, the |
174 | * driver must call irq_poll_enable(). | |
5e605b64 | 175 | **/ |
511cbce2 | 176 | void irq_poll_init(struct irq_poll *iop, int weight, irq_poll_fn *poll_fn) |
5e605b64 JA |
177 | { |
178 | memset(iop, 0, sizeof(*iop)); | |
179 | INIT_LIST_HEAD(&iop->list); | |
180 | iop->weight = weight; | |
181 | iop->poll = poll_fn; | |
511cbce2 | 182 | set_bit(IRQ_POLL_F_SCHED, &iop->state); |
5e605b64 | 183 | } |
511cbce2 | 184 | EXPORT_SYMBOL(irq_poll_init); |
5e605b64 | 185 | |
511cbce2 | 186 | static int irq_poll_cpu_notify(struct notifier_block *self, |
0b776b06 | 187 | unsigned long action, void *hcpu) |
5e605b64 JA |
188 | { |
189 | /* | |
190 | * If a CPU goes away, splice its entries to the current CPU | |
191 | * and trigger a run of the softirq | |
192 | */ | |
193 | if (action == CPU_DEAD || action == CPU_DEAD_FROZEN) { | |
194 | int cpu = (unsigned long) hcpu; | |
195 | ||
196 | local_irq_disable(); | |
197 | list_splice_init(&per_cpu(blk_cpu_iopoll, cpu), | |
170d800a | 198 | this_cpu_ptr(&blk_cpu_iopoll)); |
511cbce2 | 199 | __raise_softirq_irqoff(IRQ_POLL_SOFTIRQ); |
5e605b64 JA |
200 | local_irq_enable(); |
201 | } | |
202 | ||
203 | return NOTIFY_OK; | |
204 | } | |
205 | ||
511cbce2 CH |
206 | static struct notifier_block irq_poll_cpu_notifier = { |
207 | .notifier_call = irq_poll_cpu_notify, | |
5e605b64 JA |
208 | }; |
209 | ||
511cbce2 | 210 | static __init int irq_poll_setup(void) |
5e605b64 JA |
211 | { |
212 | int i; | |
213 | ||
214 | for_each_possible_cpu(i) | |
215 | INIT_LIST_HEAD(&per_cpu(blk_cpu_iopoll, i)); | |
216 | ||
511cbce2 CH |
217 | open_softirq(IRQ_POLL_SOFTIRQ, irq_poll_softirq); |
218 | register_hotcpu_notifier(&irq_poll_cpu_notifier); | |
5e605b64 JA |
219 | return 0; |
220 | } | |
511cbce2 | 221 | subsys_initcall(irq_poll_setup); |