]>
Commit | Line | Data |
---|---|---|
49789fd0 IS |
1 | /* coroutine expansion and optimisation passes. |
2 | ||
aeee4812 | 3 | Copyright (C) 2018-2023 Free Software Foundation, Inc. |
49789fd0 IS |
4 | |
5 | Contributed by Iain Sandoe <iain@sandoe.co.uk> under contract to Facebook. | |
6 | ||
7 | This file is part of GCC. | |
8 | ||
9 | GCC is free software; you can redistribute it and/or modify it under | |
10 | the terms of the GNU General Public License as published by the Free | |
11 | Software Foundation; either version 3, or (at your option) any later | |
12 | version. | |
13 | ||
14 | GCC is distributed in the hope that it will be useful, but WITHOUT ANY | |
15 | WARRANTY; without even the implied warranty of MERCHANTABILITY or | |
16 | FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License | |
17 | for more details. | |
18 | ||
19 | You should have received a copy of the GNU General Public License | |
20 | along with GCC; see the file COPYING3. If not see | |
21 | <http://www.gnu.org/licenses/>. */ | |
22 | ||
23 | #include "config.h" | |
24 | #include "system.h" | |
25 | #include "coretypes.h" | |
26 | #include "backend.h" | |
27 | #include "target.h" | |
28 | #include "tree.h" | |
29 | #include "gimple.h" | |
30 | #include "tree-pass.h" | |
31 | #include "ssa.h" | |
32 | #include "cgraph.h" | |
33 | #include "pretty-print.h" | |
34 | #include "diagnostic-core.h" | |
35 | #include "fold-const.h" | |
36 | #include "internal-fn.h" | |
37 | #include "langhooks.h" | |
38 | #include "gimplify.h" | |
39 | #include "gimple-iterator.h" | |
40 | #include "gimplify-me.h" | |
41 | #include "gimple-walk.h" | |
42 | #include "gimple-fold.h" | |
43 | #include "tree-cfg.h" | |
44 | #include "tree-into-ssa.h" | |
45 | #include "tree-ssa-propagate.h" | |
46 | #include "gimple-pretty-print.h" | |
47 | #include "cfghooks.h" | |
48 | ||
49 | /* Here we: | |
50 | * lower the internal function that implements an exit from scope. | |
51 | * expand the builtins that are used to implement the library | |
52 | interfaces to the coroutine frame. */ | |
53 | ||
54 | static tree | |
55 | lower_coro_builtin (gimple_stmt_iterator *gsi, bool *handled_ops_p, | |
56 | struct walk_stmt_info *wi ATTRIBUTE_UNUSED) | |
57 | { | |
58 | gimple *stmt = gsi_stmt (*gsi); | |
59 | *handled_ops_p = !gimple_has_substatements (stmt); | |
60 | ||
61 | if (gimple_code (stmt) != GIMPLE_CALL) | |
62 | return NULL_TREE; | |
63 | ||
64 | /* This internal function implements an exit from scope without | |
65 | performing any cleanups; it jumps directly to the label provided. */ | |
66 | if (gimple_call_internal_p (stmt) | |
67 | && gimple_call_internal_fn (stmt) == IFN_CO_SUSPN) | |
68 | { | |
69 | tree dest = TREE_OPERAND (gimple_call_arg (stmt, 0), 0); | |
70 | ggoto *g = gimple_build_goto (dest); | |
71 | gsi_replace (gsi, g, /* do EH */ false); | |
72 | *handled_ops_p = true; | |
73 | return NULL_TREE; | |
74 | } | |
75 | ||
76 | tree decl = gimple_call_fndecl (stmt); | |
77 | if (!decl || !fndecl_built_in_p (decl, BUILT_IN_NORMAL)) | |
78 | return NULL_TREE; | |
79 | ||
80 | /* The remaining builtins implement the library interfaces to the coro | |
81 | frame. */ | |
82 | unsigned call_idx = 0; | |
83 | ||
84 | switch (DECL_FUNCTION_CODE (decl)) | |
85 | { | |
86 | default: | |
87 | break; | |
88 | case BUILT_IN_CORO_PROMISE: | |
89 | { | |
90 | /* If we are discarding this, then skip it; the function has no | |
91 | side-effects. */ | |
92 | tree lhs = gimple_call_lhs (stmt); | |
93 | if (!lhs) | |
94 | { | |
95 | gsi_remove (gsi, true); | |
96 | *handled_ops_p = true; | |
97 | return NULL_TREE; | |
98 | } | |
99 | /* The coro frame starts with two pointers (to the resume and | |
100 | destroy() functions). These are followed by the promise which | |
101 | is aligned as per type [or user attribute]. | |
102 | The input pointer is the first argument. | |
103 | The promise alignment is the second and the third is a bool | |
104 | that is true when we are converting from a promise ptr to a | |
105 | frame pointer, and false for the inverse. */ | |
106 | tree ptr = gimple_call_arg (stmt, 0); | |
107 | tree align_t = gimple_call_arg (stmt, 1); | |
108 | tree from = gimple_call_arg (stmt, 2); | |
109 | gcc_checking_assert (TREE_CODE (align_t) == INTEGER_CST); | |
110 | gcc_checking_assert (TREE_CODE (from) == INTEGER_CST); | |
111 | bool dir = wi::to_wide (from) != 0; | |
112 | HOST_WIDE_INT promise_align = TREE_INT_CST_LOW (align_t); | |
113 | HOST_WIDE_INT psize = | |
114 | TREE_INT_CST_LOW (TYPE_SIZE_UNIT (ptr_type_node)); | |
115 | HOST_WIDE_INT align = TYPE_ALIGN_UNIT (ptr_type_node); | |
116 | align = MAX (align, promise_align); | |
117 | psize *= 2; /* Start with two pointers. */ | |
118 | psize = ROUND_UP (psize, align); | |
119 | HOST_WIDE_INT offs = dir ? -psize : psize; | |
120 | tree repl = build2 (POINTER_PLUS_EXPR, ptr_type_node, ptr, | |
121 | size_int (offs)); | |
122 | gassign *grpl = gimple_build_assign (lhs, repl); | |
123 | gsi_replace (gsi, grpl, true); | |
124 | *handled_ops_p = true; | |
125 | } | |
126 | break; | |
127 | case BUILT_IN_CORO_DESTROY: | |
128 | call_idx = 1; | |
129 | /* FALLTHROUGH */ | |
130 | case BUILT_IN_CORO_RESUME: | |
131 | { | |
132 | tree ptr = gimple_call_arg (stmt, 0); /* frame ptr. */ | |
133 | HOST_WIDE_INT psize = | |
134 | TREE_INT_CST_LOW (TYPE_SIZE_UNIT (ptr_type_node)); | |
135 | HOST_WIDE_INT offset = call_idx * psize; | |
136 | tree fntype = TREE_TYPE (decl); | |
137 | tree fntype_ptr = build_pointer_type (fntype); | |
138 | tree fntype_ppp = build_pointer_type (fntype_ptr); | |
139 | tree indirect = fold_build2 (MEM_REF, fntype_ptr, ptr, | |
140 | build_int_cst (fntype_ppp, offset)); | |
141 | tree f_ptr_tmp = make_ssa_name (TYPE_MAIN_VARIANT (fntype_ptr)); | |
142 | gassign *get_fptr = gimple_build_assign (f_ptr_tmp, indirect); | |
143 | gsi_insert_before (gsi, get_fptr, GSI_SAME_STMT); | |
144 | gimple_call_set_fn (static_cast<gcall *> (stmt), f_ptr_tmp); | |
145 | *handled_ops_p = true; | |
146 | } | |
147 | break; | |
148 | case BUILT_IN_CORO_DONE: | |
149 | { | |
150 | /* If we are discarding this, then skip it; the function has no | |
151 | side-effects. */ | |
152 | tree lhs = gimple_call_lhs (stmt); | |
153 | if (!lhs) | |
154 | { | |
155 | gsi_remove (gsi, true); | |
156 | *handled_ops_p = true; | |
157 | return NULL_TREE; | |
158 | } | |
159 | /* When we're done, the resume fn is set to NULL. */ | |
160 | tree ptr = gimple_call_arg (stmt, 0); /* frame ptr. */ | |
161 | tree vpp = build_pointer_type (ptr_type_node); | |
162 | tree indirect | |
163 | = fold_build2 (MEM_REF, vpp, ptr, build_int_cst (vpp, 0)); | |
164 | tree d_ptr_tmp = make_ssa_name (ptr_type_node); | |
165 | gassign *get_dptr = gimple_build_assign (d_ptr_tmp, indirect); | |
166 | gsi_insert_before (gsi, get_dptr, GSI_SAME_STMT); | |
167 | tree done = fold_build2 (EQ_EXPR, boolean_type_node, d_ptr_tmp, | |
168 | null_pointer_node); | |
169 | gassign *get_res = gimple_build_assign (lhs, done); | |
170 | gsi_replace (gsi, get_res, true); | |
171 | *handled_ops_p = true; | |
172 | } | |
173 | break; | |
174 | } | |
175 | return NULL_TREE; | |
176 | } | |
177 | ||
178 | /* Main entry point for lowering coroutine FE builtins. */ | |
179 | ||
180 | static unsigned int | |
181 | execute_lower_coro_builtins (void) | |
182 | { | |
183 | struct walk_stmt_info wi; | |
184 | gimple_seq body; | |
185 | ||
186 | body = gimple_body (current_function_decl); | |
187 | memset (&wi, 0, sizeof (wi)); | |
188 | walk_gimple_seq_mod (&body, lower_coro_builtin, NULL, &wi); | |
189 | gimple_set_body (current_function_decl, body); | |
190 | ||
191 | return 0; | |
192 | } | |
193 | ||
194 | namespace { | |
195 | ||
196 | const pass_data pass_data_coroutine_lower_builtins = { | |
197 | GIMPLE_PASS, /* type */ | |
198 | "coro-lower-builtins", /* name */ | |
199 | OPTGROUP_NONE, /* optinfo_flags */ | |
200 | TV_NONE, /* tv_id */ | |
201 | 0, /* properties_required */ | |
202 | 0, /* properties_provided */ | |
203 | 0, /* properties_destroyed */ | |
204 | 0, /* todo_flags_start */ | |
205 | 0 /* todo_flags_finish */ | |
206 | }; | |
207 | ||
208 | class pass_coroutine_lower_builtins : public gimple_opt_pass | |
209 | { | |
210 | public: | |
211 | pass_coroutine_lower_builtins (gcc::context *ctxt) | |
212 | : gimple_opt_pass (pass_data_coroutine_lower_builtins, ctxt) | |
213 | {} | |
214 | ||
215 | /* opt_pass methods: */ | |
725793af | 216 | bool gate (function *) final override { return flag_coroutines; }; |
49789fd0 | 217 | |
725793af | 218 | unsigned int execute (function *f ATTRIBUTE_UNUSED) final override |
49789fd0 IS |
219 | { |
220 | return execute_lower_coro_builtins (); | |
221 | } | |
222 | ||
223 | }; // class pass_coroutine_lower_builtins | |
224 | ||
225 | } // namespace | |
226 | ||
227 | gimple_opt_pass * | |
228 | make_pass_coroutine_lower_builtins (gcc::context *ctxt) | |
229 | { | |
230 | return new pass_coroutine_lower_builtins (ctxt); | |
231 | } | |
232 | ||
233 | /* Expand the remaining coroutine IFNs. | |
234 | ||
235 | In the front end we construct a single actor function that contains | |
236 | the coroutine state machine. | |
237 | ||
238 | The actor function has three entry conditions: | |
239 | 1. from the ramp, resume point 0 - to initial-suspend. | |
240 | 2. when resume () is executed (resume point N). | |
241 | 3. from the destroy () shim when that is executed. | |
242 | ||
243 | The actor function begins with two dispatchers; one for resume and | |
244 | one for destroy (where the initial entry from the ramp is a special- | |
245 | case of resume point 0). | |
246 | ||
247 | Each suspend point and each dispatch entry is marked with an IFN such | |
248 | that we can connect the relevant dispatchers to their target labels. | |
249 | ||
250 | So, if we have: | |
251 | ||
252 | CO_YIELD (NUM, FINAL, RES_LAB, DEST_LAB, FRAME_PTR) | |
253 | ||
254 | This is await point NUM, and is the final await if FINAL is non-zero. | |
255 | The resume point is RES_LAB, and the destroy point is DEST_LAB. | |
256 | ||
257 | We expect to find a CO_ACTOR (NUM) in the resume dispatcher and a | |
258 | CO_ACTOR (NUM+1) in the destroy dispatcher. | |
259 | ||
260 | Initially, the intent of keeping the resume and destroy paths together | |
261 | is that the conditionals controlling them are identical, and thus there | |
262 | would be duplication of any optimisation of those paths if the split | |
263 | were earlier. | |
264 | ||
265 | Subsequent inlining of the actor (and DCE) is then able to extract the | |
266 | resume and destroy paths as separate functions if that is found | |
267 | profitable by the optimisers. | |
268 | ||
269 | Once we have remade the connections to their correct postions, we elide | |
270 | the labels that the front end inserted. */ | |
271 | ||
272 | static void | |
273 | move_edge_and_update (edge e, basic_block old_bb, basic_block new_bb) | |
274 | { | |
275 | if (dump_file) | |
276 | fprintf (dump_file, "redirecting edge from bb %u to bb %u\n", old_bb->index, | |
277 | new_bb->index); | |
278 | ||
279 | e = redirect_edge_and_branch (e, new_bb); | |
280 | if (!e && dump_file) | |
281 | fprintf (dump_file, "failed to redirect edge .. \n"); | |
282 | ||
283 | /* Die if we failed. */ | |
284 | gcc_checking_assert (e); | |
285 | } | |
286 | ||
287 | static unsigned int | |
288 | execute_early_expand_coro_ifns (void) | |
289 | { | |
290 | /* Don't rebuild stuff unless we have to. */ | |
291 | unsigned int todoflags = 0; | |
292 | bool changed = false; | |
293 | /* Some of the possible YIELD points will hopefully have been removed by | |
294 | earlier optimisations; record the ones that are still present. */ | |
295 | hash_map<int_hash<HOST_WIDE_INT, -1, -2>, tree> destinations; | |
296 | /* Labels we added to carry the CFG changes, we need to remove these to | |
297 | avoid confusing EH. */ | |
298 | hash_set<tree> to_remove; | |
299 | /* List of dispatch points to update. */ | |
300 | auto_vec<gimple_stmt_iterator, 16> actor_worklist; | |
301 | basic_block bb; | |
302 | gimple_stmt_iterator gsi; | |
303 | ||
304 | FOR_EACH_BB_FN (bb, cfun) | |
305 | for (gsi = gsi_start_bb (bb); !gsi_end_p (gsi);) | |
306 | { | |
307 | gimple *stmt = gsi_stmt (gsi); | |
308 | ||
309 | if (!is_gimple_call (stmt) || !gimple_call_internal_p (stmt)) | |
310 | { | |
311 | gsi_next (&gsi); | |
312 | continue; | |
313 | } | |
314 | switch (gimple_call_internal_fn (stmt)) | |
315 | { | |
316 | case IFN_CO_FRAME: | |
317 | { | |
318 | /* This internal function is a placeholder for the frame | |
319 | size. In principle, we might lower it later (after some | |
320 | optimisation had reduced the frame size). At present, | |
321 | without any such optimisation, we just set it here. */ | |
322 | tree lhs = gimple_call_lhs (stmt); | |
323 | tree size = gimple_call_arg (stmt, 0); | |
324 | /* Right now, this is a trivial operation - copy through | |
325 | the size computed during initial layout. */ | |
326 | gassign *grpl = gimple_build_assign (lhs, size); | |
327 | gsi_replace (&gsi, grpl, true); | |
328 | gsi_next (&gsi); | |
329 | } | |
330 | break; | |
331 | case IFN_CO_ACTOR: | |
332 | changed = true; | |
333 | actor_worklist.safe_push (gsi); /* Save for later. */ | |
334 | gsi_next (&gsi); | |
335 | break; | |
336 | case IFN_CO_YIELD: | |
337 | { | |
338 | changed = true; | |
339 | /* .CO_YIELD (NUM, FINAL, RES_LAB, DEST_LAB, FRAME_PTR); | |
340 | NUM = await number. | |
341 | FINAL = 1 if this is the final_suspend() await. | |
342 | RES_LAB = resume point label. | |
343 | DEST_LAB = destroy point label. | |
344 | FRAME_PTR = is a null pointer with the type of the coro | |
345 | frame, so that we can resize, if needed. */ | |
346 | if (dump_file) | |
347 | fprintf (dump_file, "saw CO_YIELD in BB %u\n", bb->index); | |
348 | tree num = gimple_call_arg (stmt, 0); /* yield point. */ | |
349 | HOST_WIDE_INT idx = TREE_INT_CST_LOW (num); | |
350 | bool existed; | |
351 | tree res_tgt = TREE_OPERAND (gimple_call_arg (stmt, 2), 0); | |
352 | tree &res_dest = destinations.get_or_insert (idx, &existed); | |
353 | if (existed && dump_file) | |
354 | { | |
355 | fprintf ( | |
356 | dump_file, | |
357 | "duplicate YIELD RESUME point (" HOST_WIDE_INT_PRINT_DEC | |
358 | ") ?\n", | |
359 | idx); | |
360 | print_gimple_stmt (dump_file, stmt, 0, TDF_VOPS|TDF_MEMSYMS); | |
361 | } | |
362 | else | |
363 | res_dest = res_tgt; | |
364 | tree dst_tgt = TREE_OPERAND (gimple_call_arg (stmt, 3), 0); | |
365 | tree &dst_dest = destinations.get_or_insert (idx + 1, &existed); | |
366 | if (existed && dump_file) | |
367 | { | |
368 | fprintf ( | |
369 | dump_file, | |
370 | "duplicate YIELD DESTROY point (" HOST_WIDE_INT_PRINT_DEC | |
371 | ") ?\n", | |
372 | idx + 1); | |
373 | print_gimple_stmt (dump_file, stmt, 0, TDF_VOPS|TDF_MEMSYMS); | |
374 | } | |
375 | else | |
376 | dst_dest = dst_tgt; | |
377 | to_remove.add (res_tgt); | |
378 | to_remove.add (dst_tgt); | |
379 | /* lose the co_yield. */ | |
380 | gsi_remove (&gsi, true); | |
381 | stmt = gsi_stmt (gsi); /* next. */ | |
382 | /* lose the copy present at O0. */ | |
383 | if (is_gimple_assign (stmt)) | |
384 | { | |
385 | gsi_remove (&gsi, true); | |
386 | stmt = gsi_stmt (gsi); | |
387 | } | |
388 | /* Simplify the switch or if following. */ | |
389 | if (gswitch *gsw = dyn_cast<gswitch *> (stmt)) | |
390 | { | |
391 | gimple_switch_set_index (gsw, integer_zero_node); | |
392 | fold_stmt (&gsi); | |
393 | } | |
394 | else if (gcond *gif = dyn_cast<gcond *> (stmt)) | |
395 | { | |
396 | if (gimple_cond_code (gif) == EQ_EXPR) | |
397 | gimple_cond_make_true (gif); | |
398 | else | |
399 | gimple_cond_make_false (gif); | |
400 | fold_stmt (&gsi); | |
401 | } | |
402 | else if (dump_file) | |
403 | print_gimple_stmt (dump_file, stmt, 0, TDF_VOPS|TDF_MEMSYMS); | |
404 | if (gsi_end_p (gsi)) | |
405 | break; | |
406 | continue; | |
407 | } | |
408 | default: | |
409 | gsi_next (&gsi); | |
410 | break; | |
411 | } | |
412 | } | |
413 | ||
414 | if (!changed) | |
415 | { | |
416 | if (dump_file) | |
417 | fprintf (dump_file, "coro: nothing to do\n"); | |
418 | return todoflags; | |
419 | } | |
420 | ||
421 | while (!actor_worklist.is_empty ()) | |
422 | { | |
423 | gsi = actor_worklist.pop (); | |
424 | gimple *stmt = gsi_stmt (gsi); | |
425 | gcc_checking_assert (is_gimple_call (stmt) | |
426 | && gimple_call_internal_p (stmt) | |
427 | && gimple_call_internal_fn (stmt) == IFN_CO_ACTOR); | |
428 | bb = gsi_bb (gsi); | |
429 | HOST_WIDE_INT idx = TREE_INT_CST_LOW (gimple_call_arg (stmt, 0)); | |
430 | tree *seen = destinations.get (idx); | |
431 | changed = true; | |
432 | ||
433 | if (dump_file) | |
434 | fprintf (dump_file, "saw CO_ACTOR in BB %u\n", bb->index); | |
435 | ||
436 | if (!seen) | |
437 | { | |
438 | /* If we never saw this index, it means that the CO_YIELD | |
439 | associated was elided during earlier optimisations, so we | |
440 | don't need to fix up the switch targets. */ | |
441 | if (dump_file) | |
442 | fprintf (dump_file, "yield point " HOST_WIDE_INT_PRINT_DEC | |
443 | " not used, removing it .. \n", idx); | |
444 | gsi_remove (&gsi, true); | |
445 | release_defs (stmt); | |
446 | } | |
447 | else | |
448 | { | |
449 | /* So we need to switch the target of this switch case to the | |
450 | relevant BB. */ | |
451 | basic_block new_bb = label_to_block (cfun, *seen); | |
452 | /* We expect the block we're modifying to contain a single | |
453 | CO_ACTOR() followed by a goto <switch default bb>. */ | |
454 | gcc_checking_assert (EDGE_COUNT (bb->succs) == 1); | |
455 | edge e; | |
456 | edge_iterator ei; | |
457 | FOR_EACH_EDGE (e, ei, bb->succs) | |
458 | { | |
459 | basic_block old_bb = e->dest; | |
460 | move_edge_and_update (e, old_bb, new_bb); | |
461 | } | |
462 | gsi_remove (&gsi, true); | |
463 | } | |
464 | } | |
465 | ||
466 | /* Remove the labels we inserted to map our hidden CFG, this | |
467 | avoids confusing block merges when there are also EH labels. */ | |
468 | FOR_EACH_BB_FN (bb, cfun) | |
469 | for (gsi = gsi_start_bb (bb); !gsi_end_p (gsi);) | |
470 | { | |
471 | gimple *stmt = gsi_stmt (gsi); | |
472 | if (glabel *glab = dyn_cast<glabel *> (stmt)) | |
473 | { | |
474 | tree rem = gimple_label_label (glab); | |
475 | if (to_remove.contains (rem)) | |
476 | { | |
477 | gsi_remove (&gsi, true); | |
478 | to_remove.remove (rem); | |
479 | continue; /* We already moved to the next insn. */ | |
480 | } | |
481 | } | |
482 | else | |
483 | break; | |
484 | gsi_next (&gsi); | |
485 | } | |
486 | ||
487 | /* Changed the CFG. */ | |
488 | todoflags |= TODO_cleanup_cfg; | |
489 | return todoflags; | |
490 | } | |
491 | ||
492 | namespace { | |
493 | ||
494 | const pass_data pass_data_coroutine_early_expand_ifns = { | |
495 | GIMPLE_PASS, /* type */ | |
496 | "coro-early-expand-ifns", /* name */ | |
497 | OPTGROUP_NONE, /* optinfo_flags */ | |
498 | TV_NONE, /* tv_id */ | |
499 | (PROP_cfg), /* properties_required */ | |
500 | 0, /* properties_provided */ | |
501 | 0, /* properties_destroyed */ | |
502 | 0, /* todo_flags_start */ | |
503 | 0 /* todo_flags_finish, set this in the fn. */ | |
504 | }; | |
505 | ||
506 | class pass_coroutine_early_expand_ifns : public gimple_opt_pass | |
507 | { | |
508 | public: | |
509 | pass_coroutine_early_expand_ifns (gcc::context *ctxt) | |
510 | : gimple_opt_pass (pass_data_coroutine_early_expand_ifns, ctxt) | |
511 | {} | |
512 | ||
513 | /* opt_pass methods: */ | |
725793af | 514 | bool gate (function *f) final override |
49789fd0 IS |
515 | { |
516 | return flag_coroutines && f->coroutine_component; | |
517 | } | |
518 | ||
725793af | 519 | unsigned int execute (function *f ATTRIBUTE_UNUSED) final override |
49789fd0 IS |
520 | { |
521 | return execute_early_expand_coro_ifns (); | |
522 | } | |
523 | ||
524 | }; // class pass_coroutine_expand_ifns | |
525 | ||
526 | } // namespace | |
527 | ||
528 | gimple_opt_pass * | |
529 | make_pass_coroutine_early_expand_ifns (gcc::context *ctxt) | |
530 | { | |
531 | return new pass_coroutine_early_expand_ifns (ctxt); | |
532 | } |