]>
Commit | Line | Data |
---|---|---|
4f999d05 | 1 | /* |
c2b38b27 | 2 | * Data plane event loop |
4f999d05 KW |
3 | * |
4 | * Copyright (c) 2003-2008 Fabrice Bellard | |
c2b38b27 | 5 | * Copyright (c) 2009-2017 QEMU contributors |
4f999d05 KW |
6 | * |
7 | * Permission is hereby granted, free of charge, to any person obtaining a copy | |
8 | * of this software and associated documentation files (the "Software"), to deal | |
9 | * in the Software without restriction, including without limitation the rights | |
10 | * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell | |
11 | * copies of the Software, and to permit persons to whom the Software is | |
12 | * furnished to do so, subject to the following conditions: | |
13 | * | |
14 | * The above copyright notice and this permission notice shall be included in | |
15 | * all copies or substantial portions of the Software. | |
16 | * | |
17 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR | |
18 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, | |
19 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL | |
20 | * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER | |
21 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, | |
22 | * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN | |
23 | * THE SOFTWARE. | |
24 | */ | |
25 | ||
d38ea87a | 26 | #include "qemu/osdep.h" |
da34e65c | 27 | #include "qapi/error.h" |
4f999d05 | 28 | #include "qemu-common.h" |
737e150e | 29 | #include "block/aio.h" |
9b34277d | 30 | #include "block/thread-pool.h" |
1de7afc9 | 31 | #include "qemu/main-loop.h" |
0ceb849b | 32 | #include "qemu/atomic.h" |
0187f5c9 | 33 | #include "block/raw-aio.h" |
0c330a73 PB |
34 | #include "qemu/coroutine_int.h" |
35 | #include "trace.h" | |
9a1e9481 | 36 | |
4f999d05 KW |
37 | /***********************************************************/ |
38 | /* bottom halves (can be seen as timers which expire ASAP) */ | |
39 | ||
40 | struct QEMUBH { | |
2f4dc3c1 | 41 | AioContext *ctx; |
4f999d05 KW |
42 | QEMUBHFunc *cb; |
43 | void *opaque; | |
4f999d05 | 44 | QEMUBH *next; |
9b47b17e SW |
45 | bool scheduled; |
46 | bool idle; | |
47 | bool deleted; | |
4f999d05 KW |
48 | }; |
49 | ||
5b8bb359 PB |
50 | void aio_bh_schedule_oneshot(AioContext *ctx, QEMUBHFunc *cb, void *opaque) |
51 | { | |
52 | QEMUBH *bh; | |
53 | bh = g_new(QEMUBH, 1); | |
54 | *bh = (QEMUBH){ | |
55 | .ctx = ctx, | |
56 | .cb = cb, | |
57 | .opaque = opaque, | |
58 | }; | |
d7c99a12 | 59 | qemu_lockcnt_lock(&ctx->list_lock); |
5b8bb359 PB |
60 | bh->next = ctx->first_bh; |
61 | bh->scheduled = 1; | |
62 | bh->deleted = 1; | |
63 | /* Make sure that the members are ready before putting bh into list */ | |
64 | smp_wmb(); | |
65 | ctx->first_bh = bh; | |
d7c99a12 | 66 | qemu_lockcnt_unlock(&ctx->list_lock); |
c9d1a561 | 67 | aio_notify(ctx); |
5b8bb359 PB |
68 | } |
69 | ||
f627aab1 | 70 | QEMUBH *aio_bh_new(AioContext *ctx, QEMUBHFunc *cb, void *opaque) |
4f999d05 KW |
71 | { |
72 | QEMUBH *bh; | |
ee82310f PB |
73 | bh = g_new(QEMUBH, 1); |
74 | *bh = (QEMUBH){ | |
75 | .ctx = ctx, | |
76 | .cb = cb, | |
77 | .opaque = opaque, | |
78 | }; | |
d7c99a12 | 79 | qemu_lockcnt_lock(&ctx->list_lock); |
f627aab1 | 80 | bh->next = ctx->first_bh; |
dcc772e2 LPF |
81 | /* Make sure that the members are ready before putting bh into list */ |
82 | smp_wmb(); | |
f627aab1 | 83 | ctx->first_bh = bh; |
d7c99a12 | 84 | qemu_lockcnt_unlock(&ctx->list_lock); |
4f999d05 KW |
85 | return bh; |
86 | } | |
87 | ||
df281b80 PD |
88 | void aio_bh_call(QEMUBH *bh) |
89 | { | |
90 | bh->cb(bh->opaque); | |
91 | } | |
92 | ||
dcc772e2 | 93 | /* Multiple occurrences of aio_bh_poll cannot be called concurrently */ |
f627aab1 | 94 | int aio_bh_poll(AioContext *ctx) |
4f999d05 | 95 | { |
7887f620 | 96 | QEMUBH *bh, **bhp, *next; |
4f999d05 | 97 | int ret; |
7d506c90 | 98 | bool deleted = false; |
648fb0ea | 99 | |
d7c99a12 | 100 | qemu_lockcnt_inc(&ctx->list_lock); |
4f999d05 KW |
101 | |
102 | ret = 0; | |
d7c99a12 PB |
103 | for (bh = atomic_rcu_read(&ctx->first_bh); bh; bh = next) { |
104 | next = atomic_rcu_read(&bh->next); | |
e8d3b1a2 PB |
105 | /* The atomic_xchg is paired with the one in qemu_bh_schedule. The |
106 | * implicit memory barrier ensures that the callback sees all writes | |
107 | * done by the scheduling thread. It also ensures that the scheduling | |
108 | * thread sees the zero before bh->cb has run, and thus will call | |
109 | * aio_notify again if necessary. | |
110 | */ | |
5b8bb359 | 111 | if (atomic_xchg(&bh->scheduled, 0)) { |
65c1b5b6 PB |
112 | /* Idle BHs don't count as progress */ |
113 | if (!bh->idle) { | |
4f999d05 | 114 | ret = 1; |
ca96ac44 | 115 | } |
4f999d05 | 116 | bh->idle = 0; |
0836c72f | 117 | aio_context_acquire(ctx); |
df281b80 | 118 | aio_bh_call(bh); |
0836c72f | 119 | aio_context_release(ctx); |
4f999d05 | 120 | } |
7d506c90 PB |
121 | if (bh->deleted) { |
122 | deleted = true; | |
123 | } | |
4f999d05 KW |
124 | } |
125 | ||
126 | /* remove deleted bhs */ | |
7d506c90 PB |
127 | if (!deleted) { |
128 | qemu_lockcnt_dec(&ctx->list_lock); | |
129 | return ret; | |
130 | } | |
131 | ||
d7c99a12 | 132 | if (qemu_lockcnt_dec_and_lock(&ctx->list_lock)) { |
f627aab1 | 133 | bhp = &ctx->first_bh; |
648fb0ea KW |
134 | while (*bhp) { |
135 | bh = *bhp; | |
5b8bb359 | 136 | if (bh->deleted && !bh->scheduled) { |
648fb0ea KW |
137 | *bhp = bh->next; |
138 | g_free(bh); | |
139 | } else { | |
140 | bhp = &bh->next; | |
141 | } | |
142 | } | |
d7c99a12 | 143 | qemu_lockcnt_unlock(&ctx->list_lock); |
4f999d05 | 144 | } |
4f999d05 KW |
145 | return ret; |
146 | } | |
147 | ||
148 | void qemu_bh_schedule_idle(QEMUBH *bh) | |
149 | { | |
4f999d05 | 150 | bh->idle = 1; |
dcc772e2 LPF |
151 | /* Make sure that idle & any writes needed by the callback are done |
152 | * before the locations are read in the aio_bh_poll. | |
153 | */ | |
e8d3b1a2 | 154 | atomic_mb_set(&bh->scheduled, 1); |
4f999d05 KW |
155 | } |
156 | ||
157 | void qemu_bh_schedule(QEMUBH *bh) | |
158 | { | |
924fe129 SH |
159 | AioContext *ctx; |
160 | ||
924fe129 | 161 | ctx = bh->ctx; |
4f999d05 | 162 | bh->idle = 0; |
e8d3b1a2 | 163 | /* The memory barrier implicit in atomic_xchg makes sure that: |
924fe129 SH |
164 | * 1. idle & any writes needed by the callback are done before the |
165 | * locations are read in the aio_bh_poll. | |
166 | * 2. ctx is loaded before scheduled is set and the callback has a chance | |
167 | * to execute. | |
dcc772e2 | 168 | */ |
e8d3b1a2 PB |
169 | if (atomic_xchg(&bh->scheduled, 1) == 0) { |
170 | aio_notify(ctx); | |
171 | } | |
4f999d05 KW |
172 | } |
173 | ||
dcc772e2 LPF |
174 | |
175 | /* This func is async. | |
176 | */ | |
4f999d05 KW |
177 | void qemu_bh_cancel(QEMUBH *bh) |
178 | { | |
179 | bh->scheduled = 0; | |
180 | } | |
181 | ||
dcc772e2 LPF |
182 | /* This func is async.The bottom half will do the delete action at the finial |
183 | * end. | |
184 | */ | |
4f999d05 KW |
185 | void qemu_bh_delete(QEMUBH *bh) |
186 | { | |
187 | bh->scheduled = 0; | |
188 | bh->deleted = 1; | |
189 | } | |
190 | ||
845ca10d PB |
191 | int64_t |
192 | aio_compute_timeout(AioContext *ctx) | |
4f999d05 | 193 | { |
845ca10d PB |
194 | int64_t deadline; |
195 | int timeout = -1; | |
4f999d05 KW |
196 | QEMUBH *bh; |
197 | ||
d7c99a12 PB |
198 | for (bh = atomic_rcu_read(&ctx->first_bh); bh; |
199 | bh = atomic_rcu_read(&bh->next)) { | |
5b8bb359 | 200 | if (bh->scheduled) { |
4f999d05 KW |
201 | if (bh->idle) { |
202 | /* idle bottom halves will be polled at least | |
203 | * every 10ms */ | |
845ca10d | 204 | timeout = 10000000; |
4f999d05 KW |
205 | } else { |
206 | /* non-idle bottom halves will be executed | |
207 | * immediately */ | |
845ca10d | 208 | return 0; |
4f999d05 KW |
209 | } |
210 | } | |
211 | } | |
e3713e00 | 212 | |
845ca10d | 213 | deadline = timerlistgroup_deadline_ns(&ctx->tlg); |
533a8cf3 | 214 | if (deadline == 0) { |
845ca10d | 215 | return 0; |
533a8cf3 | 216 | } else { |
845ca10d | 217 | return qemu_soonest_timeout(timeout, deadline); |
533a8cf3 | 218 | } |
845ca10d | 219 | } |
533a8cf3 | 220 | |
845ca10d PB |
221 | static gboolean |
222 | aio_ctx_prepare(GSource *source, gint *timeout) | |
223 | { | |
224 | AioContext *ctx = (AioContext *) source; | |
225 | ||
eabc9779 PB |
226 | atomic_or(&ctx->notify_me, 1); |
227 | ||
845ca10d PB |
228 | /* We assume there is no timeout already supplied */ |
229 | *timeout = qemu_timeout_ns_to_ms(aio_compute_timeout(ctx)); | |
a3462c65 PB |
230 | |
231 | if (aio_prepare(ctx)) { | |
232 | *timeout = 0; | |
233 | } | |
234 | ||
845ca10d | 235 | return *timeout == 0; |
e3713e00 PB |
236 | } |
237 | ||
238 | static gboolean | |
239 | aio_ctx_check(GSource *source) | |
240 | { | |
241 | AioContext *ctx = (AioContext *) source; | |
242 | QEMUBH *bh; | |
243 | ||
eabc9779 | 244 | atomic_and(&ctx->notify_me, ~1); |
05e514b1 | 245 | aio_notify_accept(ctx); |
21a03d17 | 246 | |
e3713e00 | 247 | for (bh = ctx->first_bh; bh; bh = bh->next) { |
5b8bb359 | 248 | if (bh->scheduled) { |
e3713e00 | 249 | return true; |
6977d901 | 250 | } |
e3713e00 | 251 | } |
533a8cf3 | 252 | return aio_pending(ctx) || (timerlistgroup_deadline_ns(&ctx->tlg) == 0); |
e3713e00 PB |
253 | } |
254 | ||
255 | static gboolean | |
256 | aio_ctx_dispatch(GSource *source, | |
257 | GSourceFunc callback, | |
258 | gpointer user_data) | |
259 | { | |
260 | AioContext *ctx = (AioContext *) source; | |
261 | ||
262 | assert(callback == NULL); | |
721671ad | 263 | aio_dispatch(ctx, true); |
e3713e00 PB |
264 | return true; |
265 | } | |
266 | ||
2f4dc3c1 PB |
267 | static void |
268 | aio_ctx_finalize(GSource *source) | |
269 | { | |
270 | AioContext *ctx = (AioContext *) source; | |
271 | ||
9b34277d | 272 | thread_pool_free(ctx->thread_pool); |
a076972a | 273 | |
0187f5c9 PB |
274 | #ifdef CONFIG_LINUX_AIO |
275 | if (ctx->linux_aio) { | |
276 | laio_detach_aio_context(ctx->linux_aio, ctx); | |
277 | laio_cleanup(ctx->linux_aio); | |
278 | ctx->linux_aio = NULL; | |
279 | } | |
280 | #endif | |
281 | ||
0c330a73 PB |
282 | assert(QSLIST_EMPTY(&ctx->scheduled_coroutines)); |
283 | qemu_bh_delete(ctx->co_schedule_bh); | |
284 | ||
d7c99a12 PB |
285 | qemu_lockcnt_lock(&ctx->list_lock); |
286 | assert(!qemu_lockcnt_count(&ctx->list_lock)); | |
a076972a SH |
287 | while (ctx->first_bh) { |
288 | QEMUBH *next = ctx->first_bh->next; | |
289 | ||
290 | /* qemu_bh_delete() must have been called on BHs in this AioContext */ | |
291 | assert(ctx->first_bh->deleted); | |
292 | ||
293 | g_free(ctx->first_bh); | |
294 | ctx->first_bh = next; | |
295 | } | |
d7c99a12 | 296 | qemu_lockcnt_unlock(&ctx->list_lock); |
a076972a | 297 | |
f6a51c84 | 298 | aio_set_event_notifier(ctx, &ctx->notifier, false, NULL, NULL); |
2f4dc3c1 | 299 | event_notifier_cleanup(&ctx->notifier); |
3fe71223 | 300 | qemu_rec_mutex_destroy(&ctx->lock); |
d7c99a12 | 301 | qemu_lockcnt_destroy(&ctx->list_lock); |
dae21b98 | 302 | timerlistgroup_deinit(&ctx->tlg); |
2f4dc3c1 PB |
303 | } |
304 | ||
e3713e00 PB |
305 | static GSourceFuncs aio_source_funcs = { |
306 | aio_ctx_prepare, | |
307 | aio_ctx_check, | |
308 | aio_ctx_dispatch, | |
2f4dc3c1 | 309 | aio_ctx_finalize |
e3713e00 PB |
310 | }; |
311 | ||
312 | GSource *aio_get_g_source(AioContext *ctx) | |
313 | { | |
314 | g_source_ref(&ctx->source); | |
315 | return &ctx->source; | |
316 | } | |
a915f4bc | 317 | |
9b34277d SH |
318 | ThreadPool *aio_get_thread_pool(AioContext *ctx) |
319 | { | |
320 | if (!ctx->thread_pool) { | |
321 | ctx->thread_pool = thread_pool_new(ctx); | |
322 | } | |
323 | return ctx->thread_pool; | |
324 | } | |
325 | ||
0187f5c9 PB |
326 | #ifdef CONFIG_LINUX_AIO |
327 | LinuxAioState *aio_get_linux_aio(AioContext *ctx) | |
328 | { | |
329 | if (!ctx->linux_aio) { | |
330 | ctx->linux_aio = laio_init(); | |
331 | laio_attach_aio_context(ctx->linux_aio, ctx); | |
332 | } | |
333 | return ctx->linux_aio; | |
334 | } | |
335 | #endif | |
336 | ||
2f4dc3c1 PB |
337 | void aio_notify(AioContext *ctx) |
338 | { | |
eabc9779 PB |
339 | /* Write e.g. bh->scheduled before reading ctx->notify_me. Pairs |
340 | * with atomic_or in aio_ctx_prepare or atomic_add in aio_poll. | |
341 | */ | |
0ceb849b | 342 | smp_mb(); |
eabc9779 | 343 | if (ctx->notify_me) { |
0ceb849b | 344 | event_notifier_set(&ctx->notifier); |
05e514b1 PB |
345 | atomic_mb_set(&ctx->notified, true); |
346 | } | |
347 | } | |
348 | ||
349 | void aio_notify_accept(AioContext *ctx) | |
350 | { | |
351 | if (atomic_xchg(&ctx->notified, false)) { | |
352 | event_notifier_test_and_clear(&ctx->notifier); | |
0ceb849b | 353 | } |
2f4dc3c1 PB |
354 | } |
355 | ||
d5541d86 AB |
356 | static void aio_timerlist_notify(void *opaque) |
357 | { | |
358 | aio_notify(opaque); | |
359 | } | |
360 | ||
21a03d17 PB |
361 | static void event_notifier_dummy_cb(EventNotifier *e) |
362 | { | |
363 | } | |
364 | ||
4a1cba38 SH |
365 | /* Returns true if aio_notify() was called (e.g. a BH was scheduled) */ |
366 | static bool event_notifier_poll(void *opaque) | |
367 | { | |
368 | EventNotifier *e = opaque; | |
369 | AioContext *ctx = container_of(e, AioContext, notifier); | |
370 | ||
371 | return atomic_read(&ctx->notified); | |
372 | } | |
373 | ||
0c330a73 PB |
374 | static void co_schedule_bh_cb(void *opaque) |
375 | { | |
376 | AioContext *ctx = opaque; | |
377 | QSLIST_HEAD(, Coroutine) straight, reversed; | |
378 | ||
379 | QSLIST_MOVE_ATOMIC(&reversed, &ctx->scheduled_coroutines); | |
380 | QSLIST_INIT(&straight); | |
381 | ||
382 | while (!QSLIST_EMPTY(&reversed)) { | |
383 | Coroutine *co = QSLIST_FIRST(&reversed); | |
384 | QSLIST_REMOVE_HEAD(&reversed, co_scheduled_next); | |
385 | QSLIST_INSERT_HEAD(&straight, co, co_scheduled_next); | |
386 | } | |
387 | ||
388 | while (!QSLIST_EMPTY(&straight)) { | |
389 | Coroutine *co = QSLIST_FIRST(&straight); | |
390 | QSLIST_REMOVE_HEAD(&straight, co_scheduled_next); | |
391 | trace_aio_co_schedule_bh_cb(ctx, co); | |
392 | qemu_coroutine_enter(co); | |
393 | } | |
394 | } | |
395 | ||
2f78e491 | 396 | AioContext *aio_context_new(Error **errp) |
f627aab1 | 397 | { |
2f78e491 | 398 | int ret; |
2f4dc3c1 | 399 | AioContext *ctx; |
37fcee5d | 400 | |
2f4dc3c1 | 401 | ctx = (AioContext *) g_source_new(&aio_source_funcs, sizeof(AioContext)); |
7e003465 C |
402 | aio_context_setup(ctx); |
403 | ||
2f78e491 CN |
404 | ret = event_notifier_init(&ctx->notifier, false); |
405 | if (ret < 0) { | |
2f78e491 | 406 | error_setg_errno(errp, -ret, "Failed to initialize event notifier"); |
37fcee5d | 407 | goto fail; |
2f78e491 | 408 | } |
fcf5def1 | 409 | g_source_set_can_recurse(&ctx->source, true); |
d7c99a12 | 410 | qemu_lockcnt_init(&ctx->list_lock); |
0c330a73 PB |
411 | |
412 | ctx->co_schedule_bh = aio_bh_new(ctx, co_schedule_bh_cb, ctx); | |
413 | QSLIST_INIT(&ctx->scheduled_coroutines); | |
414 | ||
2f78e491 | 415 | aio_set_event_notifier(ctx, &ctx->notifier, |
dca21ef2 | 416 | false, |
2f78e491 | 417 | (EventNotifierHandler *) |
f6a51c84 | 418 | event_notifier_dummy_cb, |
4a1cba38 | 419 | event_notifier_poll); |
0187f5c9 PB |
420 | #ifdef CONFIG_LINUX_AIO |
421 | ctx->linux_aio = NULL; | |
422 | #endif | |
9b34277d | 423 | ctx->thread_pool = NULL; |
3fe71223 | 424 | qemu_rec_mutex_init(&ctx->lock); |
d5541d86 | 425 | timerlistgroup_init(&ctx->tlg, aio_timerlist_notify, ctx); |
2f4dc3c1 | 426 | |
82a41186 | 427 | ctx->poll_ns = 0; |
4a1cba38 | 428 | ctx->poll_max_ns = 0; |
82a41186 SH |
429 | ctx->poll_grow = 0; |
430 | ctx->poll_shrink = 0; | |
4a1cba38 | 431 | |
2f4dc3c1 | 432 | return ctx; |
37fcee5d FZ |
433 | fail: |
434 | g_source_destroy(&ctx->source); | |
435 | return NULL; | |
e3713e00 PB |
436 | } |
437 | ||
0c330a73 PB |
438 | void aio_co_schedule(AioContext *ctx, Coroutine *co) |
439 | { | |
440 | trace_aio_co_schedule(ctx, co); | |
441 | QSLIST_INSERT_HEAD_ATOMIC(&ctx->scheduled_coroutines, | |
442 | co, co_scheduled_next); | |
443 | qemu_bh_schedule(ctx->co_schedule_bh); | |
444 | } | |
445 | ||
446 | void aio_co_wake(struct Coroutine *co) | |
447 | { | |
448 | AioContext *ctx; | |
449 | ||
450 | /* Read coroutine before co->ctx. Matches smp_wmb in | |
451 | * qemu_coroutine_enter. | |
452 | */ | |
453 | smp_read_barrier_depends(); | |
454 | ctx = atomic_read(&co->ctx); | |
455 | ||
456 | if (ctx != qemu_get_current_aio_context()) { | |
457 | aio_co_schedule(ctx, co); | |
458 | return; | |
459 | } | |
460 | ||
461 | if (qemu_in_coroutine()) { | |
462 | Coroutine *self = qemu_coroutine_self(); | |
463 | assert(self != co); | |
464 | QSIMPLEQ_INSERT_TAIL(&self->co_queue_wakeup, co, co_queue_next); | |
465 | } else { | |
466 | aio_context_acquire(ctx); | |
467 | qemu_coroutine_enter(co); | |
468 | aio_context_release(ctx); | |
469 | } | |
470 | } | |
471 | ||
e3713e00 PB |
472 | void aio_context_ref(AioContext *ctx) |
473 | { | |
474 | g_source_ref(&ctx->source); | |
475 | } | |
476 | ||
477 | void aio_context_unref(AioContext *ctx) | |
478 | { | |
479 | g_source_unref(&ctx->source); | |
f627aab1 | 480 | } |
98563fc3 SH |
481 | |
482 | void aio_context_acquire(AioContext *ctx) | |
483 | { | |
3fe71223 | 484 | qemu_rec_mutex_lock(&ctx->lock); |
98563fc3 SH |
485 | } |
486 | ||
487 | void aio_context_release(AioContext *ctx) | |
488 | { | |
3fe71223 | 489 | qemu_rec_mutex_unlock(&ctx->lock); |
98563fc3 | 490 | } |