]>
Commit | Line | Data |
---|---|---|
00dccaf1 KW |
1 | /* |
2 | * ucontext coroutine initialization code | |
3 | * | |
4 | * Copyright (C) 2006 Anthony Liguori <anthony@codemonkey.ws> | |
5 | * Copyright (C) 2011 Kevin Wolf <kwolf@redhat.com> | |
6 | * | |
7 | * This library is free software; you can redistribute it and/or | |
8 | * modify it under the terms of the GNU Lesser General Public | |
9 | * License as published by the Free Software Foundation; either | |
10 | * version 2.0 of the License, or (at your option) any later version. | |
11 | * | |
12 | * This library is distributed in the hope that it will be useful, | |
13 | * but WITHOUT ANY WARRANTY; without even the implied warranty of | |
14 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU | |
15 | * Lesser General Public License for more details. | |
16 | * | |
17 | * You should have received a copy of the GNU Lesser General Public | |
18 | * License along with this library; if not, see <http://www.gnu.org/licenses/>. | |
19 | */ | |
20 | ||
21 | /* XXX Is there a nicer way to disable glibc's stack check for longjmp? */ | |
00dccaf1 | 22 | #undef _FORTIFY_SOURCE |
9afa888c DB |
23 | #define _FORTIFY_SOURCE 0 |
24 | ||
aafd7584 | 25 | #include "qemu/osdep.h" |
00dccaf1 | 26 | #include <ucontext.h> |
10817bf0 | 27 | #include "qemu/coroutine_int.h" |
34145a30 | 28 | #include "qemu/coroutine-tls.h" |
00dccaf1 | 29 | |
3f4349dc KW |
30 | #ifdef CONFIG_VALGRIND_H |
31 | #include <valgrind/valgrind.h> | |
32 | #endif | |
33 | ||
638466f7 | 34 | #ifdef QEMU_SANITIZE_ADDRESS |
d83414e1 MAL |
35 | #ifdef CONFIG_ASAN_IFACE_FIBER |
36 | #define CONFIG_ASAN 1 | |
37 | #include <sanitizer/asan_interface.h> | |
38 | #endif | |
39 | #endif | |
40 | ||
0aebab04 LY |
41 | #ifdef CONFIG_TSAN |
42 | #include <sanitizer/tsan_interface.h> | |
43 | #endif | |
44 | ||
00dccaf1 KW |
45 | typedef struct { |
46 | Coroutine base; | |
47 | void *stack; | |
ddba1591 | 48 | size_t stack_size; |
58ebc2c3 DB |
49 | #ifdef CONFIG_SAFESTACK |
50 | /* Need an unsafe stack for each coroutine */ | |
51 | void *unsafe_stack; | |
52 | size_t unsafe_stack_size; | |
53 | #endif | |
6ab7e546 | 54 | sigjmp_buf env; |
3f4349dc | 55 | |
995f5c3c | 56 | #ifdef CONFIG_TSAN |
0aebab04 LY |
57 | void *tsan_co_fiber; |
58 | void *tsan_caller_fiber; | |
995f5c3c | 59 | #endif |
0aebab04 | 60 | |
3f4349dc KW |
61 | #ifdef CONFIG_VALGRIND_H |
62 | unsigned int valgrind_stack_id; | |
63 | #endif | |
64 | ||
00dccaf1 KW |
65 | } CoroutineUContext; |
66 | ||
67 | /** | |
68 | * Per-thread coroutine bookkeeping | |
69 | */ | |
34145a30 SH |
70 | QEMU_DEFINE_STATIC_CO_TLS(Coroutine *, current); |
71 | QEMU_DEFINE_STATIC_CO_TLS(CoroutineUContext, leader); | |
00dccaf1 KW |
72 | |
73 | /* | |
74 | * va_args to makecontext() must be type 'int', so passing | |
75 | * the pointer we need may require several int args. This | |
76 | * union is a quick hack to let us do that | |
77 | */ | |
78 | union cc_arg { | |
79 | void *p; | |
80 | int i[2]; | |
81 | }; | |
82 | ||
995f5c3c RF |
83 | /* |
84 | * QEMU_ALWAYS_INLINE only does so if __OPTIMIZE__, so we cannot use it. | |
85 | * always_inline is required to avoid TSan runtime fatal errors. | |
86 | */ | |
0aebab04 LY |
87 | static inline __attribute__((always_inline)) |
88 | void on_new_fiber(CoroutineUContext *co) | |
89 | { | |
90 | #ifdef CONFIG_TSAN | |
91 | co->tsan_co_fiber = __tsan_create_fiber(0); /* flags: sync on switch */ | |
92 | co->tsan_caller_fiber = __tsan_get_current_fiber(); | |
93 | #endif | |
94 | } | |
95 | ||
995f5c3c | 96 | /* always_inline is required to avoid TSan runtime fatal errors. */ |
0aebab04 LY |
97 | static inline __attribute__((always_inline)) |
98 | void finish_switch_fiber(void *fake_stack_save) | |
d83414e1 MAL |
99 | { |
100 | #ifdef CONFIG_ASAN | |
34145a30 | 101 | CoroutineUContext *leaderp = get_ptr_leader(); |
d83414e1 MAL |
102 | const void *bottom_old; |
103 | size_t size_old; | |
104 | ||
105 | __sanitizer_finish_switch_fiber(fake_stack_save, &bottom_old, &size_old); | |
106 | ||
34145a30 SH |
107 | if (!leaderp->stack) { |
108 | leaderp->stack = (void *)bottom_old; | |
109 | leaderp->stack_size = size_old; | |
d83414e1 MAL |
110 | } |
111 | #endif | |
0aebab04 LY |
112 | #ifdef CONFIG_TSAN |
113 | if (fake_stack_save) { | |
114 | __tsan_release(fake_stack_save); | |
115 | __tsan_switch_to_fiber(fake_stack_save, 0); /* 0=synchronize */ | |
116 | } | |
117 | #endif | |
d83414e1 MAL |
118 | } |
119 | ||
995f5c3c RF |
120 | /* always_inline is required to avoid TSan runtime fatal errors. */ |
121 | static inline __attribute__((always_inline)) | |
d9945ccd | 122 | void start_switch_fiber_asan(void **fake_stack_save, |
995f5c3c | 123 | const void *bottom, size_t size) |
d83414e1 MAL |
124 | { |
125 | #ifdef CONFIG_ASAN | |
d9945ccd | 126 | __sanitizer_start_switch_fiber(fake_stack_save, bottom, size); |
0aebab04 | 127 | #endif |
995f5c3c RF |
128 | } |
129 | ||
130 | /* always_inline is required to avoid TSan runtime fatal errors. */ | |
131 | static inline __attribute__((always_inline)) | |
132 | void start_switch_fiber_tsan(void **fake_stack_save, | |
133 | CoroutineUContext *co, | |
134 | bool caller) | |
135 | { | |
0aebab04 | 136 | #ifdef CONFIG_TSAN |
995f5c3c RF |
137 | void *new_fiber = caller ? |
138 | co->tsan_caller_fiber : | |
139 | co->tsan_co_fiber; | |
140 | void *curr_fiber = __tsan_get_current_fiber(); | |
0aebab04 LY |
141 | __tsan_acquire(curr_fiber); |
142 | ||
143 | *fake_stack_save = curr_fiber; | |
144 | __tsan_switch_to_fiber(new_fiber, 0); /* 0=synchronize */ | |
d83414e1 MAL |
145 | #endif |
146 | } | |
147 | ||
00dccaf1 KW |
148 | static void coroutine_trampoline(int i0, int i1) |
149 | { | |
150 | union cc_arg arg; | |
151 | CoroutineUContext *self; | |
152 | Coroutine *co; | |
d83414e1 MAL |
153 | void *fake_stack_save = NULL; |
154 | ||
155 | finish_switch_fiber(NULL); | |
00dccaf1 KW |
156 | |
157 | arg.i[0] = i0; | |
158 | arg.i[1] = i1; | |
159 | self = arg.p; | |
160 | co = &self->base; | |
161 | ||
162 | /* Initialize longjmp environment and switch back the caller */ | |
6ab7e546 | 163 | if (!sigsetjmp(self->env, 0)) { |
34145a30 SH |
164 | CoroutineUContext *leaderp = get_ptr_leader(); |
165 | ||
d9945ccd | 166 | start_switch_fiber_asan(&fake_stack_save, |
34145a30 | 167 | leaderp->stack, leaderp->stack_size); |
995f5c3c | 168 | start_switch_fiber_tsan(&fake_stack_save, self, true); /* true=caller */ |
6ab7e546 | 169 | siglongjmp(*(sigjmp_buf *)co->entry_arg, 1); |
00dccaf1 KW |
170 | } |
171 | ||
d83414e1 MAL |
172 | finish_switch_fiber(fake_stack_save); |
173 | ||
00dccaf1 KW |
174 | while (true) { |
175 | co->entry(co->entry_arg); | |
176 | qemu_coroutine_switch(co, co->caller, COROUTINE_TERMINATE); | |
177 | } | |
178 | } | |
179 | ||
40239784 | 180 | Coroutine *qemu_coroutine_new(void) |
00dccaf1 | 181 | { |
00dccaf1 KW |
182 | CoroutineUContext *co; |
183 | ucontext_t old_uc, uc; | |
6ab7e546 | 184 | sigjmp_buf old_env; |
32b74677 | 185 | union cc_arg arg = {0}; |
d83414e1 | 186 | void *fake_stack_save = NULL; |
00dccaf1 | 187 | |
6ab7e546 PM |
188 | /* The ucontext functions preserve signal masks which incurs a |
189 | * system call overhead. sigsetjmp(buf, 0)/siglongjmp() does not | |
190 | * preserve signal masks but only works on the current stack. | |
191 | * Since we need a way to create and switch to a new stack, use | |
192 | * the ucontext functions for that but sigsetjmp()/siglongjmp() for | |
193 | * everything else. | |
00dccaf1 KW |
194 | */ |
195 | ||
196 | if (getcontext(&uc) == -1) { | |
197 | abort(); | |
198 | } | |
199 | ||
7267c094 | 200 | co = g_malloc0(sizeof(*co)); |
ddba1591 PL |
201 | co->stack_size = COROUTINE_STACK_SIZE; |
202 | co->stack = qemu_alloc_stack(&co->stack_size); | |
58ebc2c3 DB |
203 | #ifdef CONFIG_SAFESTACK |
204 | co->unsafe_stack_size = COROUTINE_STACK_SIZE; | |
205 | co->unsafe_stack = qemu_alloc_stack(&co->unsafe_stack_size); | |
206 | #endif | |
00dccaf1 KW |
207 | co->base.entry_arg = &old_env; /* stash away our jmp_buf */ |
208 | ||
209 | uc.uc_link = &old_uc; | |
210 | uc.uc_stack.ss_sp = co->stack; | |
ddba1591 | 211 | uc.uc_stack.ss_size = co->stack_size; |
00dccaf1 KW |
212 | uc.uc_stack.ss_flags = 0; |
213 | ||
3f4349dc KW |
214 | #ifdef CONFIG_VALGRIND_H |
215 | co->valgrind_stack_id = | |
ddba1591 | 216 | VALGRIND_STACK_REGISTER(co->stack, co->stack + co->stack_size); |
3f4349dc KW |
217 | #endif |
218 | ||
00dccaf1 KW |
219 | arg.p = co; |
220 | ||
0aebab04 | 221 | on_new_fiber(co); |
00dccaf1 KW |
222 | makecontext(&uc, (void (*)(void))coroutine_trampoline, |
223 | 2, arg.i[0], arg.i[1]); | |
224 | ||
6ab7e546 PM |
225 | /* swapcontext() in, siglongjmp() back out */ |
226 | if (!sigsetjmp(old_env, 0)) { | |
d9945ccd | 227 | start_switch_fiber_asan(&fake_stack_save, co->stack, co->stack_size); |
995f5c3c RF |
228 | start_switch_fiber_tsan(&fake_stack_save, |
229 | co, false); /* false=not caller */ | |
58ebc2c3 DB |
230 | |
231 | #ifdef CONFIG_SAFESTACK | |
232 | /* | |
233 | * Before we swap the context, set the new unsafe stack | |
234 | * The unsafe stack grows just like the normal stack, so start from | |
235 | * the last usable location of the memory area. | |
236 | * NOTE: we don't have to re-set the usp afterwards because we are | |
237 | * coming back to this context through a siglongjmp. | |
238 | * The compiler already wrapped the corresponding sigsetjmp call with | |
239 | * code that saves the usp on the (safe) stack before the call, and | |
240 | * restores it right after (which is where we return with siglongjmp). | |
241 | */ | |
242 | void *usp = co->unsafe_stack + co->unsafe_stack_size; | |
243 | __safestack_unsafe_stack_ptr = usp; | |
244 | #endif | |
245 | ||
00dccaf1 KW |
246 | swapcontext(&old_uc, &uc); |
247 | } | |
d83414e1 MAL |
248 | |
249 | finish_switch_fiber(fake_stack_save); | |
250 | ||
00dccaf1 KW |
251 | return &co->base; |
252 | } | |
253 | ||
3f4349dc KW |
254 | #ifdef CONFIG_VALGRIND_H |
255 | /* Work around an unused variable in the valgrind.h macro... */ | |
7aa12aa2 | 256 | #if !defined(__clang__) |
e6f53fd5 | 257 | #pragma GCC diagnostic push |
3f4349dc | 258 | #pragma GCC diagnostic ignored "-Wunused-but-set-variable" |
06d71fa1 | 259 | #endif |
3f4349dc KW |
260 | static inline void valgrind_stack_deregister(CoroutineUContext *co) |
261 | { | |
262 | VALGRIND_STACK_DEREGISTER(co->valgrind_stack_id); | |
263 | } | |
7aa12aa2 | 264 | #if !defined(__clang__) |
e6f53fd5 | 265 | #pragma GCC diagnostic pop |
3f4349dc | 266 | #endif |
06d71fa1 | 267 | #endif |
3f4349dc | 268 | |
d9945ccd AO |
269 | #if defined(CONFIG_ASAN) && defined(CONFIG_COROUTINE_POOL) |
270 | static void coroutine_fn terminate_asan(void *opaque) | |
271 | { | |
272 | CoroutineUContext *to = DO_UPCAST(CoroutineUContext, base, opaque); | |
273 | ||
274 | set_current(opaque); | |
275 | start_switch_fiber_asan(NULL, to->stack, to->stack_size); | |
276 | G_STATIC_ASSERT(!IS_ENABLED(CONFIG_TSAN)); | |
277 | siglongjmp(to->env, COROUTINE_ENTER); | |
278 | } | |
279 | #endif | |
280 | ||
00dccaf1 KW |
281 | void qemu_coroutine_delete(Coroutine *co_) |
282 | { | |
00dccaf1 KW |
283 | CoroutineUContext *co = DO_UPCAST(CoroutineUContext, base, co_); |
284 | ||
d9945ccd AO |
285 | #if defined(CONFIG_ASAN) && defined(CONFIG_COROUTINE_POOL) |
286 | co_->entry_arg = qemu_coroutine_self(); | |
287 | co_->entry = terminate_asan; | |
288 | qemu_coroutine_switch(co_->entry_arg, co_, COROUTINE_ENTER); | |
289 | #endif | |
290 | ||
3f4349dc KW |
291 | #ifdef CONFIG_VALGRIND_H |
292 | valgrind_stack_deregister(co); | |
293 | #endif | |
294 | ||
ddba1591 | 295 | qemu_free_stack(co->stack, co->stack_size); |
58ebc2c3 DB |
296 | #ifdef CONFIG_SAFESTACK |
297 | qemu_free_stack(co->unsafe_stack, co->unsafe_stack_size); | |
298 | #endif | |
7267c094 | 299 | g_free(co); |
00dccaf1 KW |
300 | } |
301 | ||
d1d1b206 PB |
302 | /* This function is marked noinline to prevent GCC from inlining it |
303 | * into coroutine_trampoline(). If we allow it to do that then it | |
304 | * hoists the code to get the address of the TLS variable "current" | |
305 | * out of the while() loop. This is an invalid transformation because | |
306 | * the sigsetjmp() call may be called when running thread A but | |
307 | * return in thread B, and so we might be in a different thread | |
308 | * context each time round the loop. | |
309 | */ | |
310 | CoroutineAction __attribute__((noinline)) | |
311 | qemu_coroutine_switch(Coroutine *from_, Coroutine *to_, | |
312 | CoroutineAction action) | |
00dccaf1 KW |
313 | { |
314 | CoroutineUContext *from = DO_UPCAST(CoroutineUContext, base, from_); | |
315 | CoroutineUContext *to = DO_UPCAST(CoroutineUContext, base, to_); | |
00dccaf1 | 316 | int ret; |
d83414e1 | 317 | void *fake_stack_save = NULL; |
00dccaf1 | 318 | |
34145a30 | 319 | set_current(to_); |
00dccaf1 | 320 | |
6ab7e546 | 321 | ret = sigsetjmp(from->env, 0); |
00dccaf1 | 322 | if (ret == 0) { |
d9945ccd AO |
323 | start_switch_fiber_asan(IS_ENABLED(CONFIG_COROUTINE_POOL) || |
324 | action != COROUTINE_TERMINATE ? | |
325 | &fake_stack_save : NULL, | |
326 | to->stack, to->stack_size); | |
995f5c3c RF |
327 | start_switch_fiber_tsan(&fake_stack_save, |
328 | to, false); /* false=not caller */ | |
6ab7e546 | 329 | siglongjmp(to->env, action); |
00dccaf1 | 330 | } |
d83414e1 MAL |
331 | |
332 | finish_switch_fiber(fake_stack_save); | |
333 | ||
00dccaf1 KW |
334 | return ret; |
335 | } | |
336 | ||
337 | Coroutine *qemu_coroutine_self(void) | |
338 | { | |
34145a30 SH |
339 | Coroutine *self = get_current(); |
340 | CoroutineUContext *leaderp = get_ptr_leader(); | |
341 | ||
342 | if (!self) { | |
343 | self = &leaderp->base; | |
344 | set_current(self); | |
d1d1b206 | 345 | } |
0aebab04 | 346 | #ifdef CONFIG_TSAN |
34145a30 SH |
347 | if (!leaderp->tsan_co_fiber) { |
348 | leaderp->tsan_co_fiber = __tsan_get_current_fiber(); | |
0aebab04 LY |
349 | } |
350 | #endif | |
34145a30 | 351 | return self; |
00dccaf1 KW |
352 | } |
353 | ||
354 | bool qemu_in_coroutine(void) | |
355 | { | |
34145a30 SH |
356 | Coroutine *self = get_current(); |
357 | ||
358 | return self && self->caller; | |
00dccaf1 | 359 | } |