]>
Commit | Line | Data |
---|---|---|
f0a90c7d | 1 | /* strub (stack scrubbing) support. |
a945c346 | 2 | Copyright (C) 2021-2024 Free Software Foundation, Inc. |
f0a90c7d AO |
3 | Contributed by Alexandre Oliva <oliva@adacore.com>. |
4 | ||
5 | This file is part of GCC. | |
6 | ||
7 | GCC is free software; you can redistribute it and/or modify it under | |
8 | the terms of the GNU General Public License as published by the Free | |
9 | Software Foundation; either version 3, or (at your option) any later | |
10 | version. | |
11 | ||
12 | GCC is distributed in the hope that it will be useful, but WITHOUT ANY | |
13 | WARRANTY; without even the implied warranty of MERCHANTABILITY or | |
14 | FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License | |
15 | for more details. | |
16 | ||
17 | You should have received a copy of the GNU General Public License | |
18 | along with GCC; see the file COPYING3. If not see | |
19 | <http://www.gnu.org/licenses/>. */ | |
20 | ||
21 | #include "config.h" | |
22 | #include "system.h" | |
23 | #include "coretypes.h" | |
24 | #include "backend.h" | |
25 | #include "tree.h" | |
26 | #include "gimple.h" | |
27 | #include "gimplify.h" | |
28 | #include "tree-pass.h" | |
29 | #include "ssa.h" | |
30 | #include "gimple-iterator.h" | |
31 | #include "gimplify-me.h" | |
32 | #include "tree-into-ssa.h" | |
33 | #include "tree-ssa.h" | |
34 | #include "tree-cfg.h" | |
35 | #include "cfghooks.h" | |
36 | #include "cfgloop.h" | |
37 | #include "cfgcleanup.h" | |
38 | #include "tree-eh.h" | |
39 | #include "except.h" | |
40 | #include "builtins.h" | |
41 | #include "attribs.h" | |
42 | #include "tree-inline.h" | |
43 | #include "cgraph.h" | |
44 | #include "alloc-pool.h" | |
45 | #include "symbol-summary.h" | |
c8742849 MJ |
46 | #include "sreal.h" |
47 | #include "ipa-cp.h" | |
f0a90c7d AO |
48 | #include "ipa-prop.h" |
49 | #include "ipa-fnsummary.h" | |
50 | #include "gimple-fold.h" | |
51 | #include "fold-const.h" | |
52 | #include "gimple-walk.h" | |
53 | #include "tree-dfa.h" | |
54 | #include "langhooks.h" | |
55 | #include "calls.h" | |
56 | #include "vec.h" | |
57 | #include "stor-layout.h" | |
58 | #include "varasm.h" | |
59 | #include "alias.h" | |
60 | #include "diagnostic.h" | |
61 | #include "intl.h" | |
62 | #include "ipa-strub.h" | |
63 | #include "symtab-thunks.h" | |
64 | #include "attr-fnspec.h" | |
f908368d | 65 | #include "target.h" |
f0a90c7d AO |
66 | |
67 | /* This file introduces two passes that, together, implement | |
68 | machine-independent stack scrubbing, strub for short. It arranges | |
69 | for stack frames that have strub enabled to be zeroed-out after | |
70 | relinquishing control to a caller, whether by returning or by | |
71 | propagating an exception. This admittedly unusual design decision | |
72 | was driven by exception support (one needs a stack frame to be | |
73 | active to propagate exceptions out of it), and it enabled an | |
74 | implementation that is entirely machine-independent (no custom | |
75 | epilogue code is required). | |
76 | ||
77 | Strub modes can be selected for stack frames by attaching attribute | |
78 | strub to functions or to variables (to their types, actually). | |
79 | Different strub modes, with different implementation details, are | |
80 | available, and they can be selected by an argument to the strub | |
81 | attribute. When enabled by strub-enabled variables, whether by | |
82 | accessing (as in reading from) statically-allocated ones, or by | |
83 | introducing (as in declaring) automatically-allocated ones, a | |
84 | suitable mode is selected automatically. | |
85 | ||
86 | At-calls mode modifies the interface of a function, adding a stack | |
87 | watermark argument, that callers use to clean up the stack frame of | |
88 | the called function. Because of the interface change, it can only | |
89 | be used when explicitly selected, or when a function is internal to | |
90 | a translation unit. Strub-at-calls function types are distinct | |
91 | from their original types (they're not modified in-place), and they | |
92 | are not interchangeable with other function types. | |
93 | ||
94 | Internal mode, in turn, does not modify the type or the interface | |
95 | of a function. It is currently implemented by turning the function | |
96 | into a wrapper, moving the function body to a separate wrapped | |
97 | function, and scrubbing the wrapped body's stack in the wrapper. | |
98 | Internal-strub function types are mostly interface-compatible with | |
99 | other strub modes, namely callable (from strub functions, though | |
100 | not strub-enabled) and disabled (not callable from strub | |
101 | functions). | |
102 | ||
103 | Always_inline functions can be strub functions, but they can only | |
104 | be called from other strub functions, because strub functions must | |
105 | never be inlined into non-strub functions. Internal and at-calls | |
106 | modes are indistinguishable when it comes to always_inline | |
107 | functions: they will necessarily be inlined into another strub | |
108 | function, and will thus be integrated into the caller's stack | |
109 | frame, whatever the mode. (Contrast with non-always_inline strub | |
110 | functions: an at-calls function can be called from other strub | |
111 | functions, ensuring no discontinuity in stack erasing, whereas an | |
112 | internal-strub function can only be called from other strub | |
113 | functions if it happens to be inlined, or if -fstrub=relaxed mode | |
114 | is in effect (that's the default). In -fstrub=strict mode, | |
115 | internal-strub functions are not callable from strub functions, | |
116 | because the wrapper itself is not strubbed. | |
117 | ||
118 | The implementation involves two simple-IPA passes. The earliest | |
119 | one, strub-mode, assigns strub modes to functions. It needs to run | |
120 | before any inlining, so that we can prevent inlining of strub | |
121 | functions into non-strub functions. It notes explicit strub mode | |
122 | requests, enables strub in response to strub variables and testing | |
123 | options, and flags unsatisfiable requests. | |
124 | ||
125 | Three possibilities of unsatisfiable requests come to mind: (a) | |
126 | when a strub mode is explicitly selected, but the function uses | |
127 | features that make it ineligible for that mode (e.g. at-calls rules | |
128 | out calling __builtin_apply_args, because of the interface changes, | |
129 | and internal mode rules out noclone or otherwise non-versionable | |
130 | functions, non-default varargs, non-local or forced labels, and | |
131 | functions with far too many arguments); (b) when some strub mode | |
132 | must be enabled because of a strub variable, but the function is | |
133 | not eligible or not viable for any mode; and (c) when | |
134 | -fstrub=strict is enabled, and calls are found in strub functions | |
135 | to functions that are not callable from strub contexts. | |
136 | compute_strub_mode implements (a) and (b), and verify_strub | |
137 | implements (c). | |
138 | ||
139 | The second IPA pass modifies interfaces of at-calls-strub functions | |
140 | and types, introduces strub calls in and around them. and splits | |
141 | internal-strub functions. It is placed after early inlining, so | |
142 | that even internal-strub functions get a chance of being inlined | |
143 | into other strub functions, but before non-early inlining, so that | |
144 | internal-strub wrapper functions still get a chance of inlining | |
145 | after splitting. | |
146 | ||
147 | Wrappers avoid duplicating the copying of large arguments again by | |
148 | passing them by reference to the wrapped bodies. This involves | |
149 | occasional SSA rewriting of address computations, because of the | |
150 | additional indirection. Besides these changes, and the | |
151 | introduction of the stack watermark parameter, wrappers and wrapped | |
152 | functions cooperate to handle variable argument lists (performing | |
153 | va_start in the wrapper, passing the list as an argument, and | |
154 | replacing va_start calls in the wrapped body with va_copy), and | |
155 | __builtin_apply_args (also called in the wrapper and passed to the | |
156 | wrapped body as an argument). | |
157 | ||
158 | Strub bodies (both internal-mode wrapped bodies, and at-calls | |
159 | functions) always start by adjusting the watermark parameter, by | |
160 | calling __builtin___strub_update. The compiler inserts them in the | |
161 | main strub pass. Allocations of additional stack space for the | |
162 | frame (__builtin_alloca) are also followed by watermark updates. | |
163 | Stack space temporarily allocated to pass arguments to other | |
164 | functions, released right after the call, is not regarded as part | |
165 | of the frame. Around calls to them, i.e., in internal-mode | |
166 | wrappers and at-calls callers (even calls through pointers), calls | |
167 | to __builtin___strub_enter and __builtin___strub_leave are | |
168 | inserted, the latter as a __finally block, so that it runs at | |
169 | regular and exceptional exit paths. strub_enter only initializes | |
170 | the stack watermark, and strub_leave is where the scrubbing takes | |
171 | place, overwriting with zeros the stack space from the top of the | |
172 | stack to the watermark. | |
173 | ||
174 | These calls can be optimized in various cases. In | |
175 | pass_ipa_strub::adjust_at_calls_call, for example, we enable | |
176 | tail-calling and other optimized calls from one strub body to | |
177 | another by passing on the watermark parameter. The builtins | |
178 | themselves may undergo inline substitution during expansion, | |
179 | dependign on optimization levels. This involves dealing with stack | |
180 | red zones (when the builtins are called out-of-line, the red zone | |
181 | cannot be used) and other ugly details related with inlining strub | |
182 | bodies into other strub bodies (see expand_builtin_strub_update). | |
183 | expand_builtin_strub_leave may even perform partial inline | |
184 | substitution. */ | |
185 | ||
186 | /* Const and pure functions that gain a watermark parameter for strub purposes | |
187 | are still regarded as such, which may cause the inline expansions of the | |
188 | __strub builtins to malfunction. Ideally, attribute "fn spec" would enable | |
189 | us to inform the backend about requirements and side effects of the call, but | |
190 | call_fusage building in calls.c:expand_call does not even look at | |
191 | attr_fnspec, so we resort to asm loads and updates to attain an equivalent | |
192 | effect. Once expand_call gains the ability to issue extra memory uses and | |
193 | clobbers based on pure/const function's fnspec, we can define this to 1. */ | |
194 | #define ATTR_FNSPEC_DECONST_WATERMARK 0 | |
195 | ||
196 | enum strub_mode { | |
197 | /* This mode denotes a regular function, that does not require stack | |
198 | scrubbing (strubbing). It may call any other functions, but if | |
199 | it calls AT_CALLS (or WRAPPED) ones, strubbing logic is | |
200 | automatically introduced around those calls (the latter, by | |
201 | inlining INTERNAL wrappers). */ | |
202 | STRUB_DISABLED = 0, | |
203 | ||
204 | /* This denotes a function whose signature is (to be) modified to | |
205 | take an extra parameter, for stack use annotation, and its | |
206 | callers must initialize and pass that argument, and perform the | |
207 | strubbing. Functions that are explicitly marked with attribute | |
208 | strub must have the mark visible wherever the function is, | |
209 | including aliases, and overriders and overriding methods. | |
210 | Functions that are implicitly marked for strubbing, for accessing | |
211 | variables explicitly marked as such, will only select this | |
212 | strubbing method if they are internal to a translation unit. It | |
213 | can only be inlined into other strubbing functions, i.e., | |
214 | STRUB_AT_CALLS or STRUB_WRAPPED. */ | |
215 | STRUB_AT_CALLS = 1, | |
216 | ||
217 | /* This denotes a function that is to perform strubbing internally, | |
218 | without any changes to its interface (the function is turned into | |
219 | a strubbing wrapper, and its original body is moved to a separate | |
220 | STRUB_WRAPPED function, with a modified interface). Functions | |
221 | may be explicitly marked with attribute strub(2), and the | |
222 | attribute must be visible at the point of definition. Functions | |
223 | that are explicitly marked for strubbing, for accessing variables | |
224 | explicitly marked as such, may select this strubbing mode if | |
225 | their interface cannot change, e.g. because its interface is | |
226 | visible to other translation units, directly, by indirection | |
227 | (having its address taken), inheritance, etc. Functions that use | |
228 | this method must not have the noclone attribute, nor the noipa | |
229 | one. Functions marked as always_inline may select this mode, but | |
230 | they are NOT wrapped, they remain unchanged, and are only inlined | |
231 | into strubbed contexts. Once non-always_inline functions are | |
232 | wrapped, the wrapper becomes STRUB_WRAPPER, and the wrapped becomes | |
233 | STRUB_WRAPPED. */ | |
234 | STRUB_INTERNAL = 2, | |
235 | ||
236 | /* This denotes a function whose stack is not strubbed, but that is | |
237 | nevertheless explicitly or implicitly marked as callable from strubbing | |
238 | functions. Normally, only STRUB_AT_CALLS (and STRUB_INTERNAL -> | |
239 | STRUB_WRAPPED) functions can be called from strubbing contexts (bodies of | |
240 | STRUB_AT_CALLS, STRUB_INTERNAL and STRUB_WRAPPED functions), but attribute | |
241 | strub(3) enables other functions to be (indirectly) called from these | |
242 | contexts. Some builtins and internal functions may be implicitly marked as | |
243 | STRUB_CALLABLE. */ | |
244 | STRUB_CALLABLE = 3, | |
245 | ||
246 | /* This denotes the function that took over the body of a | |
247 | STRUB_INTERNAL function. At first, it's only called by its | |
248 | wrapper, but the wrapper may be inlined. The wrapped function, | |
249 | in turn, can only be inlined into other functions whose stack | |
250 | frames are strubbed, i.e., that are STRUB_WRAPPED or | |
251 | STRUB_AT_CALLS. */ | |
252 | STRUB_WRAPPED = -1, | |
253 | ||
254 | /* This denotes the wrapper function that replaced the STRUB_INTERNAL | |
255 | function. This mode overrides the STRUB_INTERNAL mode at the time the | |
256 | internal to-be-wrapped function becomes a wrapper, so that inlining logic | |
257 | can tell one from the other. */ | |
258 | STRUB_WRAPPER = -2, | |
259 | ||
260 | /* This denotes an always_inline function that requires strubbing. It can | |
261 | only be called from, and inlined into, other strubbing contexts. */ | |
262 | STRUB_INLINABLE = -3, | |
263 | ||
264 | /* This denotes a function that accesses strub variables, so it would call for | |
265 | internal strubbing (whether or not it's eligible for that), but since | |
266 | at-calls strubbing is viable, that's selected as an optimization. This | |
267 | mode addresses the inconvenience that such functions may have different | |
268 | modes selected depending on optimization flags, and get a different | |
269 | callable status depending on that choice: if we assigned them | |
270 | STRUB_AT_CALLS mode, they would be callable when optimizing, whereas | |
271 | STRUB_INTERNAL would not be callable. */ | |
272 | STRUB_AT_CALLS_OPT = -4, | |
273 | ||
274 | }; | |
275 | ||
276 | /* Look up a strub attribute in TYPE, and return it. */ | |
277 | ||
278 | static tree | |
279 | get_strub_attr_from_type (tree type) | |
280 | { | |
281 | return lookup_attribute ("strub", TYPE_ATTRIBUTES (type)); | |
282 | } | |
283 | ||
284 | /* Look up a strub attribute in DECL or in its type, and return it. */ | |
285 | ||
286 | static tree | |
287 | get_strub_attr_from_decl (tree decl) | |
288 | { | |
289 | tree ret = lookup_attribute ("strub", DECL_ATTRIBUTES (decl)); | |
290 | if (ret) | |
291 | return ret; | |
292 | return get_strub_attr_from_type (TREE_TYPE (decl)); | |
293 | } | |
294 | ||
295 | #define STRUB_ID_COUNT 8 | |
296 | #define STRUB_IDENT_COUNT 3 | |
297 | #define STRUB_TYPE_COUNT 5 | |
298 | ||
299 | #define STRUB_ID_BASE 0 | |
300 | #define STRUB_IDENT_BASE (STRUB_ID_BASE + STRUB_ID_COUNT) | |
301 | #define STRUB_TYPE_BASE (STRUB_IDENT_BASE + STRUB_IDENT_COUNT) | |
302 | #define STRUB_CACHE_SIZE (STRUB_TYPE_BASE + STRUB_TYPE_COUNT) | |
303 | ||
304 | /* Keep the strub mode and temp identifiers and types from being GC'd. */ | |
305 | static GTY((deletable)) tree strub_cache[STRUB_CACHE_SIZE]; | |
306 | ||
307 | /* Define a function to cache identifier ID, to be used as a strub attribute | |
308 | parameter for a strub mode named after NAME. */ | |
309 | #define DEF_STRUB_IDS(IDX, NAME, ID) \ | |
310 | static inline tree get_strub_mode_id_ ## NAME () { \ | |
311 | int idx = STRUB_ID_BASE + IDX; \ | |
312 | tree identifier = strub_cache[idx]; \ | |
313 | if (!identifier) \ | |
314 | strub_cache[idx] = identifier = get_identifier (ID); \ | |
315 | return identifier; \ | |
316 | } | |
317 | /* Same as DEF_STRUB_IDS, but use the string expansion of NAME as ID. */ | |
318 | #define DEF_STRUB_ID(IDX, NAME) \ | |
319 | DEF_STRUB_IDS (IDX, NAME, #NAME) | |
320 | ||
321 | /* Define functions for each of the strub mode identifiers. | |
322 | Expose dashes rather than underscores. */ | |
323 | DEF_STRUB_ID (0, disabled) | |
324 | DEF_STRUB_IDS (1, at_calls, "at-calls") | |
325 | DEF_STRUB_ID (2, internal) | |
326 | DEF_STRUB_ID (3, callable) | |
327 | DEF_STRUB_ID (4, wrapped) | |
328 | DEF_STRUB_ID (5, wrapper) | |
329 | DEF_STRUB_ID (6, inlinable) | |
330 | DEF_STRUB_IDS (7, at_calls_opt, "at-calls-opt") | |
331 | ||
332 | /* Release the temporary macro names. */ | |
333 | #undef DEF_STRUB_IDS | |
334 | #undef DEF_STRUB_ID | |
335 | ||
336 | /* Return the identifier corresponding to strub MODE. */ | |
337 | ||
338 | static tree | |
339 | get_strub_mode_attr_parm (enum strub_mode mode) | |
340 | { | |
341 | switch (mode) | |
342 | { | |
343 | case STRUB_DISABLED: | |
344 | return get_strub_mode_id_disabled (); | |
345 | ||
346 | case STRUB_AT_CALLS: | |
347 | return get_strub_mode_id_at_calls (); | |
348 | ||
349 | case STRUB_INTERNAL: | |
350 | return get_strub_mode_id_internal (); | |
351 | ||
352 | case STRUB_CALLABLE: | |
353 | return get_strub_mode_id_callable (); | |
354 | ||
355 | case STRUB_WRAPPED: | |
356 | return get_strub_mode_id_wrapped (); | |
357 | ||
358 | case STRUB_WRAPPER: | |
359 | return get_strub_mode_id_wrapper (); | |
360 | ||
361 | case STRUB_INLINABLE: | |
362 | return get_strub_mode_id_inlinable (); | |
363 | ||
364 | case STRUB_AT_CALLS_OPT: | |
365 | return get_strub_mode_id_at_calls_opt (); | |
366 | ||
367 | default: | |
368 | gcc_unreachable (); | |
369 | } | |
370 | } | |
371 | ||
372 | /* Return the parmeters (TREE_VALUE) for a strub attribute of MODE. | |
373 | We know we use a single parameter, so we bypass the creation of a | |
374 | tree list. */ | |
375 | ||
376 | static tree | |
377 | get_strub_mode_attr_value (enum strub_mode mode) | |
378 | { | |
379 | return get_strub_mode_attr_parm (mode); | |
380 | } | |
381 | ||
382 | /* Determine whether ID is a well-formed strub mode-specifying attribute | |
383 | parameter for a function (type). Only user-visible modes are accepted, and | |
384 | ID must be non-NULL. | |
385 | ||
386 | For unacceptable parms, return 0, otherwise a nonzero value as below. | |
387 | ||
388 | If the parm enables strub, return positive, otherwise negative. | |
389 | ||
390 | If the affected type must be a distinct, incompatible type,return an integer | |
391 | of absolute value 2, otherwise 1. */ | |
392 | ||
393 | int | |
394 | strub_validate_fn_attr_parm (tree id) | |
395 | { | |
396 | int ret; | |
397 | const char *s = NULL; | |
398 | size_t len = 0; | |
399 | ||
400 | /* do NOT test for NULL. This is only to be called with non-NULL arguments. | |
401 | We assume that the strub parameter applies to a function, because only | |
402 | functions accept an explicit argument. If we accepted NULL, and we | |
403 | happened to be called to verify the argument for a variable, our return | |
404 | values would be wrong. */ | |
405 | if (TREE_CODE (id) == STRING_CST) | |
406 | { | |
407 | s = TREE_STRING_POINTER (id); | |
408 | len = TREE_STRING_LENGTH (id) - 1; | |
409 | } | |
410 | else if (TREE_CODE (id) == IDENTIFIER_NODE) | |
411 | { | |
412 | s = IDENTIFIER_POINTER (id); | |
413 | len = IDENTIFIER_LENGTH (id); | |
414 | } | |
415 | else | |
416 | return 0; | |
417 | ||
418 | enum strub_mode mode; | |
419 | ||
420 | if (len != 8) | |
421 | return 0; | |
422 | ||
423 | switch (s[0]) | |
424 | { | |
425 | case 'd': | |
426 | mode = STRUB_DISABLED; | |
427 | ret = -1; | |
428 | break; | |
429 | ||
430 | case 'a': | |
431 | mode = STRUB_AT_CALLS; | |
432 | ret = 2; | |
433 | break; | |
434 | ||
435 | case 'i': | |
436 | mode = STRUB_INTERNAL; | |
437 | ret = 1; | |
438 | break; | |
439 | ||
440 | case 'c': | |
441 | mode = STRUB_CALLABLE; | |
442 | ret = -2; | |
443 | break; | |
444 | ||
445 | default: | |
446 | /* Other parms are for internal use only. */ | |
447 | return 0; | |
448 | } | |
449 | ||
450 | tree mode_id = get_strub_mode_attr_parm (mode); | |
451 | ||
452 | if (TREE_CODE (id) == IDENTIFIER_NODE | |
453 | ? id != mode_id | |
454 | : strncmp (s, IDENTIFIER_POINTER (mode_id), len) != 0) | |
455 | return 0; | |
456 | ||
457 | return ret; | |
458 | } | |
459 | ||
460 | /* Return the strub mode from STRUB_ATTR. VAR_P should be TRUE if the attribute | |
461 | is taken from a variable, rather than from a function, or a type thereof. */ | |
462 | ||
463 | static enum strub_mode | |
464 | get_strub_mode_from_attr (tree strub_attr, bool var_p = false) | |
465 | { | |
466 | enum strub_mode mode = STRUB_DISABLED; | |
467 | ||
468 | if (strub_attr) | |
469 | { | |
470 | if (!TREE_VALUE (strub_attr)) | |
471 | mode = !var_p ? STRUB_AT_CALLS : STRUB_INTERNAL; | |
472 | else | |
473 | { | |
474 | gcc_checking_assert (!var_p); | |
475 | tree id = TREE_VALUE (strub_attr); | |
476 | if (TREE_CODE (id) == TREE_LIST) | |
477 | id = TREE_VALUE (id); | |
478 | const char *s = (TREE_CODE (id) == STRING_CST | |
479 | ? TREE_STRING_POINTER (id) | |
480 | : IDENTIFIER_POINTER (id)); | |
481 | size_t len = (TREE_CODE (id) == STRING_CST | |
482 | ? TREE_STRING_LENGTH (id) - 1 | |
483 | : IDENTIFIER_LENGTH (id)); | |
484 | ||
485 | switch (len) | |
486 | { | |
487 | case 7: | |
488 | switch (s[6]) | |
489 | { | |
490 | case 'r': | |
491 | mode = STRUB_WRAPPER; | |
492 | break; | |
493 | ||
494 | case 'd': | |
495 | mode = STRUB_WRAPPED; | |
496 | break; | |
497 | ||
498 | default: | |
499 | gcc_unreachable (); | |
500 | } | |
501 | break; | |
502 | ||
503 | case 8: | |
504 | switch (s[0]) | |
505 | { | |
506 | case 'd': | |
507 | mode = STRUB_DISABLED; | |
508 | break; | |
509 | ||
510 | case 'a': | |
511 | mode = STRUB_AT_CALLS; | |
512 | break; | |
513 | ||
514 | case 'i': | |
515 | mode = STRUB_INTERNAL; | |
516 | break; | |
517 | ||
518 | case 'c': | |
519 | mode = STRUB_CALLABLE; | |
520 | break; | |
521 | ||
522 | default: | |
523 | gcc_unreachable (); | |
524 | } | |
525 | break; | |
526 | ||
527 | case 9: | |
528 | mode = STRUB_INLINABLE; | |
529 | break; | |
530 | ||
531 | case 12: | |
532 | mode = STRUB_AT_CALLS_OPT; | |
533 | break; | |
534 | ||
535 | default: | |
536 | gcc_unreachable (); | |
537 | } | |
538 | ||
539 | gcc_checking_assert (TREE_CODE (id) == IDENTIFIER_NODE | |
540 | ? id == get_strub_mode_attr_parm (mode) | |
541 | : strncmp (IDENTIFIER_POINTER | |
542 | (get_strub_mode_attr_parm (mode)), | |
543 | s, len) == 0); | |
544 | } | |
545 | } | |
546 | ||
547 | return mode; | |
548 | } | |
549 | ||
550 | /* Look up, decode and return the strub mode associated with FNDECL. */ | |
551 | ||
552 | static enum strub_mode | |
553 | get_strub_mode_from_fndecl (tree fndecl) | |
554 | { | |
555 | return get_strub_mode_from_attr (get_strub_attr_from_decl (fndecl)); | |
556 | } | |
557 | ||
558 | /* Look up, decode and return the strub mode associated with NODE. */ | |
559 | ||
560 | static enum strub_mode | |
561 | get_strub_mode (cgraph_node *node) | |
562 | { | |
563 | return get_strub_mode_from_fndecl (node->decl); | |
564 | } | |
565 | ||
566 | /* Look up, decode and return the strub mode associated with TYPE. */ | |
567 | ||
568 | static enum strub_mode | |
569 | get_strub_mode_from_type (tree type) | |
570 | { | |
571 | bool var_p = !FUNC_OR_METHOD_TYPE_P (type); | |
572 | tree attr = get_strub_attr_from_type (type); | |
573 | ||
574 | if (attr) | |
575 | return get_strub_mode_from_attr (attr, var_p); | |
576 | ||
577 | if (flag_strub >= -1 && !var_p) | |
578 | return STRUB_CALLABLE; | |
579 | ||
580 | return STRUB_DISABLED; | |
581 | } | |
582 | ||
583 | \f | |
584 | /* Return TRUE iff NODE calls builtin va_start. */ | |
585 | ||
586 | static bool | |
587 | calls_builtin_va_start_p (cgraph_node *node) | |
588 | { | |
589 | bool result = false; | |
590 | ||
591 | for (cgraph_edge *e = node->callees; e; e = e->next_callee) | |
592 | { | |
593 | tree cdecl = e->callee->decl; | |
594 | if (fndecl_built_in_p (cdecl, BUILT_IN_VA_START)) | |
595 | return true; | |
596 | } | |
597 | ||
598 | return result; | |
599 | } | |
600 | ||
601 | /* Return TRUE iff NODE calls builtin apply_args, and optionally REPORT it. */ | |
602 | ||
603 | static bool | |
604 | calls_builtin_apply_args_p (cgraph_node *node, bool report = false) | |
605 | { | |
606 | bool result = false; | |
607 | ||
608 | for (cgraph_edge *e = node->callees; e; e = e->next_callee) | |
609 | { | |
610 | tree cdecl = e->callee->decl; | |
611 | if (!fndecl_built_in_p (cdecl, BUILT_IN_APPLY_ARGS)) | |
612 | continue; | |
613 | ||
614 | result = true; | |
615 | ||
616 | if (!report) | |
617 | break; | |
618 | ||
619 | sorry_at (e->call_stmt | |
620 | ? gimple_location (e->call_stmt) | |
621 | : DECL_SOURCE_LOCATION (node->decl), | |
622 | "at-calls %<strub%> does not support call to %qD", | |
623 | cdecl); | |
624 | } | |
625 | ||
626 | return result; | |
627 | } | |
628 | ||
629 | /* Return TRUE iff NODE carries the always_inline attribute. */ | |
630 | ||
631 | static inline bool | |
632 | strub_always_inline_p (cgraph_node *node) | |
633 | { | |
634 | return lookup_attribute ("always_inline", DECL_ATTRIBUTES (node->decl)); | |
635 | } | |
636 | ||
f908368d AO |
637 | /* Return TRUE iff the target has strub support for T, a function |
638 | decl, or a type used in an indirect call, and optionally REPORT the | |
639 | reasons for ineligibility. If T is a type and error REPORTing is | |
640 | enabled, the LOCation (of the indirect call) should be provided. */ | |
641 | static inline bool | |
642 | strub_target_support_p (tree t, bool report = false, | |
643 | location_t loc = UNKNOWN_LOCATION) | |
644 | { | |
645 | bool result = true; | |
646 | ||
647 | if (!targetm.have_strub_support_for (t)) | |
648 | { | |
649 | result = false; | |
650 | ||
651 | if (!report) | |
652 | return result; | |
653 | ||
654 | if (DECL_P (t)) | |
655 | sorry_at (DECL_SOURCE_LOCATION (t), | |
656 | "%qD is not eligible for %<strub%>" | |
657 | " on the target system", t); | |
658 | else | |
659 | sorry_at (loc, | |
660 | "unsupported %<strub%> call" | |
661 | " on the target system"); | |
662 | } | |
663 | ||
664 | return result; | |
665 | } | |
666 | ||
f0a90c7d AO |
667 | /* Return TRUE iff NODE is potentially eligible for any strub-enabled mode, and |
668 | optionally REPORT the reasons for ineligibility. */ | |
669 | ||
670 | static inline bool | |
671 | can_strub_p (cgraph_node *node, bool report = false) | |
672 | { | |
f908368d | 673 | bool result = strub_target_support_p (node->decl, report); |
f0a90c7d | 674 | |
f908368d | 675 | if (!report && (!result || strub_always_inline_p (node))) |
f0a90c7d AO |
676 | return result; |
677 | ||
f908368d AO |
678 | if (flag_split_stack) |
679 | { | |
680 | result = false; | |
681 | ||
682 | if (!report) | |
683 | return result; | |
684 | ||
685 | sorry_at (DECL_SOURCE_LOCATION (node->decl), | |
686 | "%qD is not eligible for %<strub%>" | |
687 | " because %<-fsplit-stack%> is enabled", | |
688 | node->decl); | |
689 | } | |
690 | ||
f0a90c7d AO |
691 | if (lookup_attribute ("noipa", DECL_ATTRIBUTES (node->decl))) |
692 | { | |
693 | result = false; | |
694 | ||
695 | if (!report) | |
696 | return result; | |
697 | ||
698 | sorry_at (DECL_SOURCE_LOCATION (node->decl), | |
699 | "%qD is not eligible for %<strub%>" | |
700 | " because of attribute %<noipa%>", | |
701 | node->decl); | |
702 | } | |
703 | ||
704 | /* We can't, and don't want to vectorize the watermark and other | |
705 | strub-introduced parms. */ | |
706 | if (lookup_attribute ("simd", DECL_ATTRIBUTES (node->decl))) | |
707 | { | |
708 | result = false; | |
709 | ||
710 | if (!report) | |
711 | return result; | |
712 | ||
713 | sorry_at (DECL_SOURCE_LOCATION (node->decl), | |
714 | "%qD is not eligible for %<strub%>" | |
715 | " because of attribute %<simd%>", | |
716 | node->decl); | |
717 | } | |
718 | ||
719 | return result; | |
720 | } | |
721 | ||
722 | /* Return TRUE iff NODE is eligible for at-calls strub, and optionally REPORT | |
723 | the reasons for ineligibility. Besides general non-eligibility for | |
724 | strub-enabled modes, at-calls rules out calling builtin apply_args. */ | |
725 | ||
726 | static bool | |
727 | can_strub_at_calls_p (cgraph_node *node, bool report = false) | |
728 | { | |
729 | bool result = !report || can_strub_p (node, report); | |
730 | ||
731 | if (!result && !report) | |
732 | return result; | |
733 | ||
734 | return !calls_builtin_apply_args_p (node, report); | |
735 | } | |
736 | ||
737 | /* Return TRUE iff the called function (pointer or, if available, | |
738 | decl) undergoes a significant type conversion for the call. Strub | |
739 | mode changes between function types, and other non-useless type | |
740 | conversions, are regarded as significant. When the function type | |
741 | is overridden, the effective strub mode for the call is that of the | |
742 | call fntype, rather than that of the pointer or of the decl. | |
743 | Functions called with type overrides cannot undergo type changes; | |
744 | it's as if their address was taken, so they're considered | |
745 | non-viable for implicit at-calls strub mode. */ | |
746 | ||
747 | static inline bool | |
748 | strub_call_fntype_override_p (const gcall *gs) | |
749 | { | |
750 | if (gimple_call_internal_p (gs)) | |
751 | return false; | |
752 | tree fn_type = TREE_TYPE (TREE_TYPE (gimple_call_fn (gs))); | |
753 | if (tree decl = gimple_call_fndecl (gs)) | |
754 | fn_type = TREE_TYPE (decl); | |
755 | ||
756 | /* We do NOT want to take the mode from the decl here. This | |
757 | function is used to tell whether we can change the strub mode of | |
758 | a function, and whether the effective mode for the call is to be | |
759 | taken from the decl or from an overrider type. When the strub | |
760 | mode is explicitly declared, or overridden with a type cast, the | |
761 | difference will be noticed in function types. However, if the | |
762 | strub mode is implicit due to e.g. strub variables or -fstrub=* | |
763 | command-line flags, we will adjust call types along with function | |
764 | types. In either case, the presence of type or strub mode | |
765 | overriders in calls will prevent a function from having its strub | |
766 | modes changed in ways that would imply type changes, but taking | |
767 | strub modes from decls would defeat this, since we set strub | |
768 | modes and then call this function to tell whether the original | |
769 | type was overridden to decide whether to adjust the call. We | |
770 | need the answer to be about the type, not the decl. */ | |
771 | enum strub_mode mode = get_strub_mode_from_type (fn_type); | |
772 | return (get_strub_mode_from_type (gs->u.fntype) != mode | |
773 | || !useless_type_conversion_p (gs->u.fntype, fn_type)); | |
774 | } | |
775 | ||
776 | /* Return TRUE iff NODE is called directly with a type override. */ | |
777 | ||
778 | static bool | |
779 | called_directly_with_type_override_p (cgraph_node *node, void *) | |
780 | { | |
781 | for (cgraph_edge *e = node->callers; e; e = e->next_caller) | |
782 | if (e->call_stmt && strub_call_fntype_override_p (e->call_stmt)) | |
783 | return true; | |
784 | ||
785 | return false; | |
786 | } | |
787 | ||
788 | /* Return TRUE iff NODE or any other nodes aliased to it are called | |
789 | with type overrides. We can't safely change the type of such | |
790 | functions. */ | |
791 | ||
792 | static bool | |
793 | called_with_type_override_p (cgraph_node *node) | |
794 | { | |
795 | return (node->call_for_symbol_thunks_and_aliases | |
796 | (called_directly_with_type_override_p, NULL, true, true)); | |
797 | } | |
798 | ||
799 | /* Symbolic macro for the max number of arguments that internal strub may add to | |
800 | a function. */ | |
801 | ||
802 | #define STRUB_INTERNAL_MAX_EXTRA_ARGS 3 | |
803 | ||
804 | /* We can't perform internal strubbing if the function body involves certain | |
805 | features: | |
806 | ||
807 | - a non-default __builtin_va_start (e.g. x86's __builtin_ms_va_start) is | |
808 | currently unsupported because we can't discover the corresponding va_copy and | |
809 | va_end decls in the wrapper, and we don't convey the alternate variable | |
810 | arguments ABI to the modified wrapped function. The default | |
811 | __builtin_va_start is supported by calling va_start/va_end at the wrapper, | |
812 | that takes variable arguments, passing a pointer to the va_list object to the | |
813 | wrapped function, that runs va_copy from it where the original function ran | |
814 | va_start. | |
815 | ||
816 | __builtin_next_arg is currently unsupported because the wrapped function | |
817 | won't be a variable argument function. We could process it in the wrapper, | |
818 | that remains a variable argument function, and replace calls in the wrapped | |
819 | body, but we currently don't. | |
820 | ||
821 | __builtin_return_address is rejected because it's generally used when the | |
822 | actual caller matters, and introducing a wrapper breaks such uses as those in | |
823 | the unwinder. */ | |
824 | ||
825 | static bool | |
826 | can_strub_internally_p (cgraph_node *node, bool report = false) | |
827 | { | |
828 | bool result = !report || can_strub_p (node, report); | |
829 | ||
830 | if (!result && !report) | |
831 | return result; | |
832 | ||
833 | if (!report && strub_always_inline_p (node)) | |
834 | return result; | |
835 | ||
836 | /* Since we're not changing the function identity proper, just | |
837 | moving its full implementation, we *could* disable | |
838 | fun->cannot_be_copied_reason and/or temporarily drop a noclone | |
839 | attribute, but we'd have to prevent remapping of the labels. */ | |
840 | if (lookup_attribute ("noclone", DECL_ATTRIBUTES (node->decl))) | |
841 | { | |
842 | result = false; | |
843 | ||
844 | if (!report) | |
845 | return result; | |
846 | ||
847 | sorry_at (DECL_SOURCE_LOCATION (node->decl), | |
848 | "%qD is not eligible for internal %<strub%>" | |
849 | " because of attribute %<noclone%>", | |
850 | node->decl); | |
851 | } | |
852 | ||
853 | if (node->has_gimple_body_p ()) | |
854 | { | |
855 | for (cgraph_edge *e = node->callees; e; e = e->next_callee) | |
856 | { | |
857 | tree cdecl = e->callee->decl; | |
858 | if (!((fndecl_built_in_p (cdecl, BUILT_IN_VA_START) | |
859 | && cdecl != builtin_decl_explicit (BUILT_IN_VA_START)) | |
860 | || fndecl_built_in_p (cdecl, BUILT_IN_NEXT_ARG) | |
861 | || fndecl_built_in_p (cdecl, BUILT_IN_RETURN_ADDRESS))) | |
862 | continue; | |
863 | ||
864 | result = false; | |
865 | ||
866 | if (!report) | |
867 | return result; | |
868 | ||
869 | sorry_at (e->call_stmt | |
870 | ? gimple_location (e->call_stmt) | |
871 | : DECL_SOURCE_LOCATION (node->decl), | |
872 | "%qD is not eligible for internal %<strub%> " | |
873 | "because it calls %qD", | |
874 | node->decl, cdecl); | |
875 | } | |
876 | ||
877 | struct function *fun = DECL_STRUCT_FUNCTION (node->decl); | |
878 | if (fun->has_nonlocal_label) | |
879 | { | |
880 | result = false; | |
881 | ||
882 | if (!report) | |
883 | return result; | |
884 | ||
885 | sorry_at (DECL_SOURCE_LOCATION (node->decl), | |
886 | "%qD is not eligible for internal %<strub%> " | |
887 | "because it contains a non-local goto target", | |
888 | node->decl); | |
889 | } | |
890 | ||
891 | if (fun->has_forced_label_in_static) | |
892 | { | |
893 | result = false; | |
894 | ||
895 | if (!report) | |
896 | return result; | |
897 | ||
898 | sorry_at (DECL_SOURCE_LOCATION (node->decl), | |
899 | "%qD is not eligible for internal %<strub%> " | |
900 | "because the address of a local label escapes", | |
901 | node->decl); | |
902 | } | |
903 | ||
904 | /* Catch any other case that would prevent versioning/cloning | |
905 | so as to also have it covered above. */ | |
906 | gcc_checking_assert (!result /* || !node->has_gimple_body_p () */ | |
907 | || tree_versionable_function_p (node->decl)); | |
908 | ||
909 | ||
910 | /* Label values references are not preserved when copying. If referenced | |
911 | in nested functions, as in 920415-1.c and 920721-4.c their decls get | |
912 | remapped independently. The exclusion below might be too broad, in | |
913 | that we might be able to support correctly cases in which the labels | |
914 | are only used internally in a function, but disconnecting forced labels | |
915 | from their original declarations is undesirable in general. */ | |
916 | basic_block bb; | |
917 | FOR_EACH_BB_FN (bb, DECL_STRUCT_FUNCTION (node->decl)) | |
918 | for (gimple_stmt_iterator gsi = gsi_start_bb (bb); | |
919 | !gsi_end_p (gsi); gsi_next (&gsi)) | |
920 | { | |
921 | glabel *label_stmt = dyn_cast <glabel *> (gsi_stmt (gsi)); | |
922 | tree target; | |
923 | ||
924 | if (!label_stmt) | |
925 | break; | |
926 | ||
927 | target = gimple_label_label (label_stmt); | |
928 | ||
929 | if (!FORCED_LABEL (target)) | |
930 | continue; | |
931 | ||
932 | result = false; | |
933 | ||
934 | if (!report) | |
935 | return result; | |
936 | ||
937 | sorry_at (gimple_location (label_stmt), | |
938 | "internal %<strub%> does not support forced labels"); | |
939 | } | |
940 | } | |
941 | ||
942 | if (list_length (TYPE_ARG_TYPES (TREE_TYPE (node->decl))) | |
d8b74d0b | 943 | >= ((HOST_WIDE_INT_1 << IPA_PARAM_MAX_INDEX_BITS) |
f0a90c7d AO |
944 | - STRUB_INTERNAL_MAX_EXTRA_ARGS)) |
945 | { | |
946 | result = false; | |
947 | ||
948 | if (!report) | |
949 | return result; | |
950 | ||
951 | sorry_at (DECL_SOURCE_LOCATION (node->decl), | |
952 | "%qD has too many arguments for internal %<strub%>", | |
953 | node->decl); | |
954 | } | |
955 | ||
956 | return result; | |
957 | } | |
958 | ||
959 | /* Return TRUE iff NODE has any strub-requiring local variable, or accesses (as | |
960 | in reading) any variable through a strub-requiring type. */ | |
961 | ||
962 | static bool | |
963 | strub_from_body_p (cgraph_node *node) | |
964 | { | |
965 | if (!node->has_gimple_body_p ()) | |
966 | return false; | |
967 | ||
968 | /* If any local variable is marked for strub... */ | |
969 | unsigned i; | |
970 | tree var; | |
971 | FOR_EACH_LOCAL_DECL (DECL_STRUCT_FUNCTION (node->decl), | |
972 | i, var) | |
973 | if (get_strub_mode_from_type (TREE_TYPE (var)) | |
974 | != STRUB_DISABLED) | |
975 | return true; | |
976 | ||
977 | /* Now scan the body for loads with strub-requiring types. | |
978 | ??? Compound types don't propagate the strub requirement to | |
979 | component types. */ | |
980 | basic_block bb; | |
981 | FOR_EACH_BB_FN (bb, DECL_STRUCT_FUNCTION (node->decl)) | |
982 | for (gimple_stmt_iterator gsi = gsi_start_bb (bb); | |
983 | !gsi_end_p (gsi); gsi_next (&gsi)) | |
984 | { | |
985 | gimple *stmt = gsi_stmt (gsi); | |
986 | ||
987 | if (!gimple_assign_load_p (stmt)) | |
988 | continue; | |
989 | ||
990 | tree rhs = gimple_assign_rhs1 (stmt); | |
991 | if (get_strub_mode_from_type (TREE_TYPE (rhs)) | |
992 | != STRUB_DISABLED) | |
993 | return true; | |
994 | } | |
995 | ||
996 | return false; | |
997 | } | |
998 | ||
999 | /* Return TRUE iff node is associated with a builtin that should be callable | |
1000 | from strub contexts. */ | |
1001 | ||
1002 | static inline bool | |
1003 | strub_callable_builtin_p (cgraph_node *node) | |
1004 | { | |
1005 | if (DECL_BUILT_IN_CLASS (node->decl) != BUILT_IN_NORMAL) | |
1006 | return false; | |
1007 | ||
1008 | enum built_in_function fcode = DECL_FUNCTION_CODE (node->decl); | |
1009 | ||
1010 | switch (fcode) | |
1011 | { | |
1012 | case BUILT_IN_NONE: | |
1013 | gcc_unreachable (); | |
1014 | ||
1015 | /* This temporarily allocates stack for the call, and we can't reasonably | |
1016 | update the watermark for that. Besides, we don't check the actual call | |
1017 | target, nor its signature, and it seems to be overkill to as much as | |
1018 | try to do so. */ | |
1019 | case BUILT_IN_APPLY: | |
1020 | return false; | |
1021 | ||
1022 | /* Conversely, this shouldn't be called from within strub contexts, since | |
1023 | the caller may have had its signature modified. STRUB_INTERNAL is ok, | |
1024 | the call will remain in the STRUB_WRAPPER, and removed from the | |
1025 | STRUB_WRAPPED clone. */ | |
1026 | case BUILT_IN_APPLY_ARGS: | |
1027 | return false; | |
1028 | ||
1029 | /* ??? Make all other builtins callable. We wish to make any builtin call | |
1030 | the compiler might introduce on its own callable. Anything that is | |
1031 | predictable enough as to be known not to allow stack data that should | |
1032 | be strubbed to unintentionally escape to non-strub contexts can be | |
1033 | allowed, and pretty much every builtin appears to fit this description. | |
1034 | The exceptions to this rule seem to be rare, and only available as | |
1035 | explicit __builtin calls, so let's keep it simple and allow all of | |
1036 | them... */ | |
1037 | default: | |
1038 | return true; | |
1039 | } | |
1040 | } | |
1041 | ||
1042 | /* Compute the strub mode to be used for NODE. STRUB_ATTR should be the strub | |
1043 | attribute,found for NODE, if any. */ | |
1044 | ||
1045 | static enum strub_mode | |
1046 | compute_strub_mode (cgraph_node *node, tree strub_attr) | |
1047 | { | |
1048 | enum strub_mode req_mode = get_strub_mode_from_attr (strub_attr); | |
1049 | ||
1050 | gcc_checking_assert (flag_strub >= -2 && flag_strub <= 3); | |
1051 | ||
1052 | /* Symbolic encodings of the -fstrub-* flags. */ | |
1053 | /* Enable strub when explicitly requested through attributes to functions or | |
1054 | variables, reporting errors if the requests cannot be satisfied. */ | |
1055 | const bool strub_flag_auto = flag_strub < 0; | |
1056 | /* strub_flag_auto with strub call verification; without this, functions are | |
1057 | implicitly callable. */ | |
1058 | const bool strub_flag_strict = flag_strub < -1; | |
1059 | /* Disable strub altogether, ignore attributes entirely. */ | |
1060 | const bool strub_flag_disabled = flag_strub == 0; | |
1061 | /* On top of _auto, also enable strub implicitly for functions that can | |
1062 | safely undergo at-calls strubbing. Internal mode will still be used in | |
1063 | functions that request it explicitly with attribute strub(2), or when the | |
1064 | function body requires strubbing and at-calls strubbing is not viable. */ | |
1065 | const bool strub_flag_at_calls = flag_strub == 1; | |
1066 | /* On top of default, also enable strub implicitly for functions that can | |
1067 | safely undergo internal strubbing. At-calls mode will still be used in | |
1068 | functions that requiest it explicitly with attribute strub() or strub(1), | |
1069 | or when the function body requires strubbing and internal strubbing is not | |
1070 | viable. */ | |
1071 | const bool strub_flag_internal = flag_strub == 2; | |
1072 | /* On top of default, also enable strub implicitly for functions that can | |
1073 | safely undergo strubbing in either mode. When both modes are viable, | |
1074 | at-calls is preferred. */ | |
1075 | const bool strub_flag_either = flag_strub == 3; | |
1076 | /* Besides the default behavior, enable strub implicitly for all viable | |
1077 | functions. */ | |
1078 | const bool strub_flag_viable = flag_strub > 0; | |
1079 | ||
1080 | /* The consider_* variables should be TRUE if selecting the corresponding | |
1081 | strub modes would be consistent with requests from attributes and command | |
1082 | line flags. Attributes associated with functions pretty much mandate a | |
1083 | selection, and should report an error if not satisfied; strub_flag_auto | |
1084 | implicitly enables some viable strub mode if that's required by references | |
1085 | to variables marked for strub; strub_flag_viable enables strub if viable | |
1086 | (even when favoring one mode, body-requested strub can still be satisfied | |
1087 | by either mode), and falls back to callable, silently unless variables | |
1088 | require strubbing. */ | |
1089 | ||
1090 | const bool consider_at_calls | |
1091 | = (!strub_flag_disabled | |
1092 | && (strub_attr | |
1093 | ? req_mode == STRUB_AT_CALLS | |
1094 | : true)); | |
1095 | const bool consider_internal | |
1096 | = (!strub_flag_disabled | |
1097 | && (strub_attr | |
1098 | ? req_mode == STRUB_INTERNAL | |
1099 | : true)); | |
1100 | ||
1101 | const bool consider_callable | |
1102 | = (!strub_flag_disabled | |
1103 | && (strub_attr | |
1104 | ? req_mode == STRUB_CALLABLE | |
1105 | : (!strub_flag_strict | |
1106 | || strub_callable_builtin_p (node)))); | |
1107 | ||
1108 | /* This is a shorthand for either strub-enabled mode. */ | |
1109 | const bool consider_strub | |
1110 | = (consider_at_calls || consider_internal); | |
1111 | ||
1112 | /* We can cope with always_inline functions even with noipa and noclone, | |
1113 | because we just leave them alone. */ | |
1114 | const bool is_always_inline | |
1115 | = strub_always_inline_p (node); | |
1116 | ||
1117 | /* Strubbing in general, and each specific strub mode, may have its own set of | |
1118 | requirements. We require noipa for strubbing, either because of cloning | |
1119 | required for internal strub, or because of caller enumeration required for | |
1120 | at-calls strub. We don't consider the at-calls mode eligible if it's not | |
1121 | even considered, it has no further requirements. Internal mode requires | |
1122 | cloning and the absence of certain features in the body and, like at-calls, | |
1123 | it's not eligible if it's not even under consideration. | |
1124 | ||
1125 | ??? Do we need target hooks for further constraints? E.g., x86's | |
1126 | "interrupt" attribute breaks internal strubbing because the wrapped clone | |
1127 | carries the attribute and thus isn't callable; in this case, we could use a | |
1128 | target hook to adjust the clone instead. */ | |
1129 | const bool strub_eligible | |
1130 | = (consider_strub | |
1131 | && (is_always_inline || can_strub_p (node))); | |
1132 | const bool at_calls_eligible | |
1133 | = (consider_at_calls && strub_eligible | |
1134 | && can_strub_at_calls_p (node)); | |
1135 | const bool internal_eligible | |
1136 | = (consider_internal && strub_eligible | |
1137 | && (is_always_inline | |
1138 | || can_strub_internally_p (node))); | |
1139 | ||
1140 | /* In addition to the strict eligibility requirements, some additional | |
1141 | constraints are placed on implicit selection of certain modes. These do | |
1142 | not prevent the selection of a mode if explicitly specified as part of a | |
1143 | function interface (the strub attribute), but they may prevent modes from | |
1144 | being selected by the command line or by function bodies. The only actual | |
1145 | constraint is on at-calls mode: since we change the function's exposed | |
1146 | signature, we won't do it implicitly if the function can possibly be used | |
1147 | in ways that do not expect the signature change, e.g., if the function is | |
1148 | available to or interposable by other units, if its address is taken, | |
1149 | etc. */ | |
1150 | const bool at_calls_viable | |
1151 | = (at_calls_eligible | |
1152 | && (strub_attr | |
1153 | || (node->has_gimple_body_p () | |
1154 | && (!node->externally_visible | |
1155 | || (node->binds_to_current_def_p () | |
1156 | && node->can_be_local_p ())) | |
1157 | && node->only_called_directly_p () | |
1158 | && !called_with_type_override_p (node)))); | |
1159 | const bool internal_viable | |
1160 | = (internal_eligible); | |
1161 | ||
1162 | /* Shorthand. */ | |
1163 | const bool strub_viable | |
1164 | = (at_calls_viable || internal_viable); | |
1165 | ||
1166 | /* We wish to analyze the body, to look for implicit requests for strub, both | |
1167 | to implicitly enable it when the body calls for it, and to report errors if | |
1168 | the body calls for it but neither mode is viable (even if that follows from | |
1169 | non-eligibility because of the explicit specification of some non-strubbing | |
1170 | mode). We can refrain from scanning the body only in rare circumstances: | |
1171 | when strub is enabled by a function attribute (scanning might be redundant | |
1172 | in telling us to also enable it), and when we are enabling strub implicitly | |
1173 | but there are non-viable modes: we want to know whether strubbing is | |
1174 | required, to fallback to another mode, even if we're only enabling a | |
1175 | certain mode, or, when either mode would do, to report an error if neither | |
1176 | happens to be viable. */ | |
1177 | const bool analyze_body | |
1178 | = (strub_attr | |
1179 | ? !consider_strub | |
1180 | : (strub_flag_auto | |
1181 | || (strub_flag_viable && (!at_calls_viable && !internal_viable)) | |
1182 | || (strub_flag_either && !strub_viable))); | |
1183 | ||
1184 | /* Cases in which strubbing is enabled or disabled by strub_flag_auto. | |
1185 | Unsatisfiable requests ought to be reported. */ | |
1186 | const bool strub_required | |
1187 | = ((strub_attr && consider_strub) | |
1188 | || (analyze_body && strub_from_body_p (node))); | |
1189 | ||
1190 | /* Besides the required cases, we want to abide by the requests to enabling on | |
1191 | an if-viable basis. */ | |
1192 | const bool strub_enable | |
1193 | = (strub_required | |
1194 | || (strub_flag_at_calls && at_calls_viable) | |
1195 | || (strub_flag_internal && internal_viable) | |
1196 | || (strub_flag_either && strub_viable)); | |
1197 | ||
1198 | /* And now we're finally ready to select a mode that abides by the viability | |
1199 | and eligibility constraints, and that satisfies the strubbing requirements | |
1200 | and requests, subject to the constraints. If both modes are viable and | |
1201 | strub is to be enabled, pick STRUB_AT_CALLS unless STRUB_INTERNAL was named | |
1202 | as preferred. */ | |
1203 | const enum strub_mode mode | |
1204 | = ((strub_enable && is_always_inline) | |
1205 | ? (strub_required ? STRUB_INLINABLE : STRUB_CALLABLE) | |
1206 | : (strub_enable && internal_viable | |
1207 | && (strub_flag_internal || !at_calls_viable)) | |
1208 | ? STRUB_INTERNAL | |
1209 | : (strub_enable && at_calls_viable) | |
1210 | ? (strub_required && !strub_attr | |
1211 | ? STRUB_AT_CALLS_OPT | |
1212 | : STRUB_AT_CALLS) | |
1213 | : consider_callable | |
1214 | ? STRUB_CALLABLE | |
1215 | : STRUB_DISABLED); | |
1216 | ||
1217 | switch (mode) | |
1218 | { | |
1219 | case STRUB_CALLABLE: | |
1220 | if (is_always_inline) | |
1221 | break; | |
1222 | /* Fall through. */ | |
1223 | ||
1224 | case STRUB_DISABLED: | |
1225 | if (strub_enable && !strub_attr) | |
1226 | { | |
1227 | gcc_checking_assert (analyze_body); | |
1228 | error_at (DECL_SOURCE_LOCATION (node->decl), | |
1229 | "%qD requires %<strub%>," | |
1230 | " but no viable %<strub%> mode was found", | |
1231 | node->decl); | |
1232 | break; | |
1233 | } | |
1234 | /* Fall through. */ | |
1235 | ||
1236 | case STRUB_AT_CALLS: | |
1237 | case STRUB_INTERNAL: | |
1238 | case STRUB_INLINABLE: | |
1239 | /* Differences from an mode requested through a function attribute are | |
1240 | reported in set_strub_mode_to. */ | |
1241 | break; | |
1242 | ||
1243 | case STRUB_AT_CALLS_OPT: | |
1244 | /* Functions that select this mode do so because of references to strub | |
1245 | variables. Even if we choose at-calls as an optimization, the | |
1246 | requirements for internal strub must still be satisfied. Optimization | |
1247 | options may render implicit at-calls strub not viable (-O0 sets | |
1248 | force_output for static non-inline functions), and it would not be good | |
1249 | if changing optimization options turned a well-formed into an | |
1250 | ill-formed one. */ | |
1251 | if (!internal_viable) | |
1252 | can_strub_internally_p (node, true); | |
1253 | break; | |
1254 | ||
1255 | case STRUB_WRAPPED: | |
1256 | case STRUB_WRAPPER: | |
1257 | default: | |
1258 | gcc_unreachable (); | |
1259 | } | |
1260 | ||
1261 | return mode; | |
1262 | } | |
1263 | ||
1264 | /* Set FNDT's strub mode to MODE; FNDT may be a function decl or | |
1265 | function type. If OVERRIDE, do not check whether a mode is already | |
1266 | set. */ | |
1267 | ||
1268 | static void | |
1269 | strub_set_fndt_mode_to (tree fndt, enum strub_mode mode, bool override) | |
1270 | { | |
1271 | gcc_checking_assert (override | |
1272 | || !(DECL_P (fndt) | |
1273 | ? get_strub_attr_from_decl (fndt) | |
1274 | : get_strub_attr_from_type (fndt))); | |
1275 | ||
1276 | tree attr = tree_cons (get_identifier ("strub"), | |
1277 | get_strub_mode_attr_value (mode), | |
1278 | NULL_TREE); | |
1279 | tree *attrp = NULL; | |
1280 | if (DECL_P (fndt)) | |
1281 | { | |
1282 | gcc_checking_assert (FUNC_OR_METHOD_TYPE_P (TREE_TYPE (fndt))); | |
1283 | attrp = &DECL_ATTRIBUTES (fndt); | |
1284 | } | |
1285 | else if (FUNC_OR_METHOD_TYPE_P (fndt)) | |
1286 | attrp = &TYPE_ATTRIBUTES (fndt); | |
1287 | else | |
1288 | gcc_unreachable (); | |
1289 | ||
1290 | TREE_CHAIN (attr) = *attrp; | |
1291 | *attrp = attr; | |
1292 | } | |
1293 | ||
1294 | /* Set FNDT's strub mode to callable. | |
1295 | FNDT may be a function decl or a function type. */ | |
1296 | ||
1297 | void | |
1298 | strub_make_callable (tree fndt) | |
1299 | { | |
1300 | strub_set_fndt_mode_to (fndt, STRUB_CALLABLE, false); | |
1301 | } | |
1302 | ||
1303 | /* Set NODE to strub MODE. Report incompatibilities between MODE and the mode | |
1304 | requested through explicit attributes, and cases of non-eligibility. */ | |
1305 | ||
1306 | static void | |
1307 | set_strub_mode_to (cgraph_node *node, enum strub_mode mode) | |
1308 | { | |
1309 | tree attr = get_strub_attr_from_decl (node->decl); | |
1310 | enum strub_mode req_mode = get_strub_mode_from_attr (attr); | |
1311 | ||
1312 | if (attr) | |
1313 | { | |
1314 | /* Check for and report incompatible mode changes. */ | |
1315 | if (mode != req_mode | |
1316 | && !(req_mode == STRUB_INTERNAL | |
1317 | && (mode == STRUB_WRAPPED | |
1318 | || mode == STRUB_WRAPPER)) | |
1319 | && !((req_mode == STRUB_INTERNAL | |
1320 | || req_mode == STRUB_AT_CALLS | |
1321 | || req_mode == STRUB_CALLABLE) | |
1322 | && mode == STRUB_INLINABLE)) | |
1323 | { | |
1324 | error_at (DECL_SOURCE_LOCATION (node->decl), | |
1325 | "%<strub%> mode %qE selected for %qD, when %qE was requested", | |
1326 | get_strub_mode_attr_parm (mode), | |
1327 | node->decl, | |
1328 | get_strub_mode_attr_parm (req_mode)); | |
1329 | if (node->alias) | |
1330 | { | |
1331 | cgraph_node *target = node->ultimate_alias_target (); | |
1332 | if (target != node) | |
1333 | error_at (DECL_SOURCE_LOCATION (target->decl), | |
1334 | "the incompatible selection was determined" | |
1335 | " by ultimate alias target %qD", | |
1336 | target->decl); | |
1337 | } | |
1338 | ||
1339 | /* Report any incompatibilities with explicitly-requested strub. */ | |
1340 | switch (req_mode) | |
1341 | { | |
1342 | case STRUB_AT_CALLS: | |
1343 | can_strub_at_calls_p (node, true); | |
1344 | break; | |
1345 | ||
1346 | case STRUB_INTERNAL: | |
1347 | can_strub_internally_p (node, true); | |
1348 | break; | |
1349 | ||
1350 | default: | |
1351 | break; | |
1352 | } | |
1353 | } | |
1354 | ||
1355 | /* Drop any incompatible strub attributes leading the decl attribute | |
1356 | chain. Return if we find one with the mode we need. */ | |
1357 | for (;;) | |
1358 | { | |
1359 | if (mode == req_mode) | |
1360 | return; | |
1361 | ||
1362 | if (DECL_ATTRIBUTES (node->decl) != attr) | |
1363 | break; | |
1364 | ||
1365 | DECL_ATTRIBUTES (node->decl) = TREE_CHAIN (attr); | |
1366 | attr = get_strub_attr_from_decl (node->decl); | |
1367 | if (!attr) | |
1368 | break; | |
1369 | ||
1370 | req_mode = get_strub_mode_from_attr (attr); | |
1371 | } | |
1372 | } | |
1373 | else if (mode == req_mode) | |
1374 | return; | |
1375 | ||
1376 | strub_set_fndt_mode_to (node->decl, mode, attr); | |
1377 | } | |
1378 | ||
1379 | /* Compute and set NODE's strub mode. */ | |
1380 | ||
1381 | static void | |
1382 | set_strub_mode (cgraph_node *node) | |
1383 | { | |
1384 | tree attr = get_strub_attr_from_decl (node->decl); | |
1385 | ||
1386 | if (attr) | |
1387 | switch (get_strub_mode_from_attr (attr)) | |
1388 | { | |
1389 | /* These can't have been requested through user attributes, so we must | |
1390 | have already gone through them. */ | |
1391 | case STRUB_WRAPPER: | |
1392 | case STRUB_WRAPPED: | |
1393 | case STRUB_INLINABLE: | |
1394 | case STRUB_AT_CALLS_OPT: | |
1395 | return; | |
1396 | ||
1397 | case STRUB_DISABLED: | |
1398 | case STRUB_AT_CALLS: | |
1399 | case STRUB_INTERNAL: | |
1400 | case STRUB_CALLABLE: | |
1401 | break; | |
1402 | ||
1403 | default: | |
1404 | gcc_unreachable (); | |
1405 | } | |
1406 | ||
1407 | cgraph_node *xnode = node; | |
1408 | if (node->alias) | |
1409 | xnode = node->ultimate_alias_target (); | |
1410 | /* Weakrefs may remain unresolved (the above will return node) if | |
1411 | their targets are not defined, so make sure we compute a strub | |
1412 | mode for them, instead of defaulting to STRUB_DISABLED and | |
1413 | rendering them uncallable. */ | |
1414 | enum strub_mode mode = (xnode != node && !xnode->alias | |
1415 | ? get_strub_mode (xnode) | |
1416 | : compute_strub_mode (node, attr)); | |
1417 | ||
1418 | set_strub_mode_to (node, mode); | |
1419 | } | |
1420 | ||
1421 | \f | |
1422 | /* Non-strub functions shouldn't be called from within strub contexts, | |
1423 | except through callable ones. Always inline strub functions can | |
1424 | only be called from strub functions. */ | |
1425 | ||
1426 | static bool | |
1427 | strub_callable_from_p (strub_mode caller_mode, strub_mode callee_mode) | |
1428 | { | |
1429 | switch (caller_mode) | |
1430 | { | |
1431 | case STRUB_WRAPPED: | |
1432 | case STRUB_AT_CALLS_OPT: | |
1433 | case STRUB_AT_CALLS: | |
1434 | case STRUB_INTERNAL: | |
1435 | case STRUB_INLINABLE: | |
1436 | break; | |
1437 | ||
1438 | case STRUB_WRAPPER: | |
1439 | case STRUB_DISABLED: | |
1440 | case STRUB_CALLABLE: | |
1441 | return callee_mode != STRUB_INLINABLE; | |
1442 | ||
1443 | default: | |
1444 | gcc_unreachable (); | |
1445 | } | |
1446 | ||
1447 | switch (callee_mode) | |
1448 | { | |
1449 | case STRUB_WRAPPED: | |
1450 | case STRUB_AT_CALLS: | |
1451 | case STRUB_INLINABLE: | |
1452 | break; | |
1453 | ||
1454 | case STRUB_AT_CALLS_OPT: | |
1455 | case STRUB_INTERNAL: | |
1456 | case STRUB_WRAPPER: | |
1457 | return (flag_strub >= -1); | |
1458 | ||
1459 | case STRUB_DISABLED: | |
1460 | return false; | |
1461 | ||
1462 | case STRUB_CALLABLE: | |
1463 | break; | |
1464 | ||
1465 | default: | |
1466 | gcc_unreachable (); | |
1467 | } | |
1468 | ||
1469 | return true; | |
1470 | } | |
1471 | ||
1472 | /* Return TRUE iff CALLEE can be inlined into CALLER. We wish to avoid inlining | |
1473 | WRAPPED functions back into their WRAPPERs. More generally, we wish to avoid | |
1474 | inlining strubbed functions into non-strubbed ones. CALLER doesn't have to | |
1475 | be an immediate caller of CALLEE: the immediate caller may have already been | |
1476 | cloned for inlining, and then CALLER may be further up the original call | |
1477 | chain. ??? It would be nice if our own caller would retry inlining callee | |
1478 | if caller gets inlined. */ | |
1479 | ||
1480 | bool | |
1481 | strub_inlinable_to_p (cgraph_node *callee, cgraph_node *caller) | |
1482 | { | |
1483 | strub_mode callee_mode = get_strub_mode (callee); | |
1484 | ||
1485 | switch (callee_mode) | |
1486 | { | |
1487 | case STRUB_WRAPPED: | |
1488 | case STRUB_AT_CALLS: | |
1489 | case STRUB_INTERNAL: | |
1490 | case STRUB_INLINABLE: | |
1491 | case STRUB_AT_CALLS_OPT: | |
1492 | break; | |
1493 | ||
1494 | case STRUB_WRAPPER: | |
1495 | case STRUB_DISABLED: | |
1496 | case STRUB_CALLABLE: | |
1497 | /* When we consider inlining, we've already verified callability, so we | |
1498 | can even inline callable and then disabled into a strub context. That | |
1499 | will get strubbed along with the context, so it's hopefully not a | |
1500 | problem. */ | |
1501 | return true; | |
1502 | ||
1503 | default: | |
1504 | gcc_unreachable (); | |
1505 | } | |
1506 | ||
1507 | strub_mode caller_mode = get_strub_mode (caller); | |
1508 | ||
1509 | switch (caller_mode) | |
1510 | { | |
1511 | case STRUB_WRAPPED: | |
1512 | case STRUB_AT_CALLS: | |
1513 | case STRUB_INTERNAL: | |
1514 | case STRUB_INLINABLE: | |
1515 | case STRUB_AT_CALLS_OPT: | |
1516 | return true; | |
1517 | ||
1518 | case STRUB_WRAPPER: | |
1519 | case STRUB_DISABLED: | |
1520 | case STRUB_CALLABLE: | |
1521 | break; | |
1522 | ||
1523 | default: | |
1524 | gcc_unreachable (); | |
1525 | } | |
1526 | ||
1527 | return false; | |
1528 | } | |
1529 | ||
1530 | /* Check that types T1 and T2 are strub-compatible. Return 1 if the strub modes | |
1531 | are the same, 2 if they are interchangeable, and 0 otherwise. */ | |
1532 | ||
1533 | int | |
1534 | strub_comptypes (tree t1, tree t2) | |
1535 | { | |
1536 | if (TREE_CODE (t1) != TREE_CODE (t2)) | |
1537 | return 0; | |
1538 | ||
1539 | enum strub_mode m1 = get_strub_mode_from_type (t1); | |
1540 | enum strub_mode m2 = get_strub_mode_from_type (t2); | |
1541 | ||
1542 | if (m1 == m2) | |
1543 | return 1; | |
1544 | ||
1545 | /* We're dealing with types, so only strub modes that can be selected by | |
1546 | attributes in the front end matter. If either mode is at-calls (for | |
1547 | functions) or internal (for variables), the conversion is not | |
1548 | compatible. */ | |
1549 | bool var_p = !FUNC_OR_METHOD_TYPE_P (t1); | |
1550 | enum strub_mode mr = var_p ? STRUB_INTERNAL : STRUB_AT_CALLS; | |
1551 | if (m1 == mr || m2 == mr) | |
1552 | return 0; | |
1553 | ||
1554 | return 2; | |
1555 | } | |
1556 | ||
1557 | /* Return the effective strub mode used for CALL, and set *TYPEP to | |
1558 | the effective type used for the call. The effective type and mode | |
1559 | are those of the callee, unless the call involves a typecast. */ | |
1560 | ||
1561 | static enum strub_mode | |
1562 | effective_strub_mode_for_call (gcall *call, tree *typep) | |
1563 | { | |
1564 | tree type; | |
1565 | enum strub_mode mode; | |
1566 | ||
1567 | if (strub_call_fntype_override_p (call)) | |
1568 | { | |
1569 | type = gimple_call_fntype (call); | |
1570 | mode = get_strub_mode_from_type (type); | |
1571 | } | |
1572 | else | |
1573 | { | |
1574 | type = TREE_TYPE (TREE_TYPE (gimple_call_fn (call))); | |
1575 | tree decl = gimple_call_fndecl (call); | |
1576 | if (decl) | |
1577 | mode = get_strub_mode_from_fndecl (decl); | |
1578 | else | |
1579 | mode = get_strub_mode_from_type (type); | |
1580 | } | |
1581 | ||
1582 | if (typep) | |
1583 | *typep = type; | |
1584 | ||
1585 | return mode; | |
1586 | } | |
1587 | ||
1588 | /* Create a distinct copy of the type of NODE's function, and change | |
1589 | the fntype of all calls to it with the same main type to the new | |
1590 | type. */ | |
1591 | ||
1592 | static void | |
1593 | distinctify_node_type (cgraph_node *node) | |
1594 | { | |
1595 | tree old_type = TREE_TYPE (node->decl); | |
1596 | tree new_type = build_distinct_type_copy (old_type); | |
1597 | tree new_ptr_type = NULL_TREE; | |
1598 | ||
1599 | /* Remap any calls to node->decl that use old_type, or a variant | |
1600 | thereof, to new_type as well. We don't look for aliases, their | |
1601 | declarations will have their types changed independently, and | |
1602 | we'll adjust their fntypes then. */ | |
1603 | for (cgraph_edge *e = node->callers; e; e = e->next_caller) | |
1604 | { | |
1605 | if (!e->call_stmt) | |
1606 | continue; | |
1607 | tree fnaddr = gimple_call_fn (e->call_stmt); | |
1608 | gcc_checking_assert (TREE_CODE (fnaddr) == ADDR_EXPR | |
1609 | && TREE_OPERAND (fnaddr, 0) == node->decl); | |
1610 | if (strub_call_fntype_override_p (e->call_stmt)) | |
1611 | continue; | |
1612 | if (!new_ptr_type) | |
1613 | new_ptr_type = build_pointer_type (new_type); | |
1614 | TREE_TYPE (fnaddr) = new_ptr_type; | |
1615 | gimple_call_set_fntype (e->call_stmt, new_type); | |
1616 | } | |
1617 | ||
1618 | TREE_TYPE (node->decl) = new_type; | |
1619 | } | |
1620 | ||
1621 | /* Return TRUE iff TYPE and any variants have the same strub mode. */ | |
1622 | ||
1623 | static bool | |
1624 | same_strub_mode_in_variants_p (tree type) | |
1625 | { | |
1626 | enum strub_mode mode = get_strub_mode_from_type (type); | |
1627 | ||
1628 | for (tree other = TYPE_MAIN_VARIANT (type); | |
1629 | other != NULL_TREE; other = TYPE_NEXT_VARIANT (other)) | |
1630 | if (type != other && mode != get_strub_mode_from_type (other)) | |
1631 | return false; | |
1632 | ||
1633 | /* Check that the canonical type, if set, either is in the same | |
1634 | variant chain, or has the same strub mode as type. Also check | |
1635 | the variants of the canonical type. */ | |
1636 | if (TYPE_CANONICAL (type) | |
1637 | && (TYPE_MAIN_VARIANT (TYPE_CANONICAL (type)) | |
1638 | != TYPE_MAIN_VARIANT (type))) | |
1639 | { | |
1640 | if (mode != get_strub_mode_from_type (TYPE_CANONICAL (type))) | |
1641 | return false; | |
1642 | else | |
1643 | return same_strub_mode_in_variants_p (TYPE_CANONICAL (type)); | |
1644 | } | |
1645 | ||
1646 | return true; | |
1647 | } | |
1648 | ||
1649 | /* Check that strub functions don't call non-strub functions, and that | |
1650 | always_inline strub functions are only called by strub | |
1651 | functions. */ | |
1652 | ||
1653 | static void | |
1654 | verify_strub () | |
1655 | { | |
1656 | cgraph_node *node; | |
1657 | ||
1658 | /* It's expected that check strub-wise pointer type compatibility of variables | |
1659 | and of functions is already taken care of by front-ends, on account of the | |
1660 | attribute's being marked as affecting type identity and of the creation of | |
1661 | distinct types. */ | |
1662 | ||
1663 | /* Check that call targets in strub contexts have strub-callable types. */ | |
1664 | ||
1665 | FOR_EACH_FUNCTION_WITH_GIMPLE_BODY (node) | |
1666 | { | |
1667 | enum strub_mode caller_mode = get_strub_mode (node); | |
1668 | ||
1669 | for (cgraph_edge *e = node->indirect_calls; e; e = e->next_callee) | |
1670 | { | |
1671 | gcc_checking_assert (e->indirect_unknown_callee); | |
1672 | ||
1673 | if (!e->call_stmt) | |
1674 | continue; | |
1675 | ||
1676 | enum strub_mode callee_mode | |
1677 | = effective_strub_mode_for_call (e->call_stmt, NULL); | |
1678 | ||
1679 | if (!strub_callable_from_p (caller_mode, callee_mode)) | |
1680 | error_at (gimple_location (e->call_stmt), | |
1681 | "indirect non-%<strub%> call in %<strub%> context %qD", | |
1682 | node->decl); | |
1683 | } | |
1684 | ||
1685 | for (cgraph_edge *e = node->callees; e; e = e->next_callee) | |
1686 | { | |
1687 | gcc_checking_assert (!e->indirect_unknown_callee); | |
1688 | ||
1689 | if (!e->call_stmt) | |
1690 | continue; | |
1691 | ||
1692 | tree callee_fntype; | |
1693 | enum strub_mode callee_mode | |
1694 | = effective_strub_mode_for_call (e->call_stmt, &callee_fntype); | |
1695 | ||
1696 | if (!strub_callable_from_p (caller_mode, callee_mode)) | |
1697 | { | |
1698 | if (callee_mode == STRUB_INLINABLE) | |
1699 | error_at (gimple_location (e->call_stmt), | |
1700 | "calling %<always_inline%> %<strub%> %qD" | |
1701 | " in non-%<strub%> context %qD", | |
1702 | e->callee->decl, node->decl); | |
1703 | else if (fndecl_built_in_p (e->callee->decl, BUILT_IN_APPLY_ARGS) | |
1704 | && caller_mode == STRUB_INTERNAL) | |
1705 | /* This is ok, it will be kept in the STRUB_WRAPPER, and removed | |
1706 | from the STRUB_WRAPPED's strub context. */ | |
1707 | continue; | |
1708 | else if (!strub_call_fntype_override_p (e->call_stmt)) | |
1709 | error_at (gimple_location (e->call_stmt), | |
1710 | "calling non-%<strub%> %qD in %<strub%> context %qD", | |
1711 | e->callee->decl, node->decl); | |
1712 | else | |
1713 | error_at (gimple_location (e->call_stmt), | |
1714 | "calling %qD using non-%<strub%> type %qT" | |
1715 | " in %<strub%> context %qD", | |
1716 | e->callee->decl, callee_fntype, node->decl); | |
1717 | } | |
1718 | } | |
1719 | } | |
1720 | } | |
1721 | ||
1722 | namespace { | |
1723 | ||
1724 | /* Define a pass to compute strub modes. */ | |
1725 | const pass_data pass_data_ipa_strub_mode = { | |
1726 | SIMPLE_IPA_PASS, | |
1727 | "strubm", | |
1728 | OPTGROUP_NONE, | |
1729 | TV_NONE, | |
1730 | PROP_cfg, // properties_required | |
1731 | 0, // properties_provided | |
1732 | 0, // properties_destroyed | |
1733 | 0, // properties_start | |
1734 | 0, // properties_finish | |
1735 | }; | |
1736 | ||
1737 | class pass_ipa_strub_mode : public simple_ipa_opt_pass | |
1738 | { | |
1739 | public: | |
1740 | pass_ipa_strub_mode (gcc::context *ctxt) | |
1741 | : simple_ipa_opt_pass (pass_data_ipa_strub_mode, ctxt) | |
1742 | {} | |
1743 | opt_pass *clone () { return new pass_ipa_strub_mode (m_ctxt); } | |
1744 | virtual bool gate (function *) { | |
1745 | /* In relaxed (-3) and strict (-4) settings, that only enable strub at a | |
1746 | function or variable attribute's request, the attribute handler changes | |
1747 | flag_strub to -1 or -2, respectively, if any strub-enabling occurence of | |
1748 | the attribute is found. Therefore, if it remains at -3 or -4, nothing | |
1749 | that would enable strub was found, so we can disable it and avoid the | |
1750 | overhead. */ | |
1751 | if (flag_strub < -2) | |
1752 | flag_strub = 0; | |
1753 | return flag_strub; | |
1754 | } | |
1755 | virtual unsigned int execute (function *); | |
1756 | }; | |
1757 | ||
1758 | /* Define a pass to introduce strub transformations. */ | |
1759 | const pass_data pass_data_ipa_strub = { | |
1760 | SIMPLE_IPA_PASS, | |
1761 | "strub", | |
1762 | OPTGROUP_NONE, | |
1763 | TV_NONE, | |
1764 | PROP_cfg | PROP_ssa, // properties_required | |
1765 | 0, // properties_provided | |
1766 | 0, // properties_destroyed | |
1767 | 0, // properties_start | |
1768 | TODO_update_ssa | |
1769 | | TODO_cleanup_cfg | |
1770 | | TODO_rebuild_cgraph_edges | |
1771 | | TODO_verify_il, // properties_finish | |
1772 | }; | |
1773 | ||
1774 | class pass_ipa_strub : public simple_ipa_opt_pass | |
1775 | { | |
1776 | public: | |
1777 | pass_ipa_strub (gcc::context *ctxt) | |
1778 | : simple_ipa_opt_pass (pass_data_ipa_strub, ctxt) | |
1779 | {} | |
1780 | opt_pass *clone () { return new pass_ipa_strub (m_ctxt); } | |
1781 | virtual bool gate (function *) { return flag_strub && !seen_error (); } | |
1782 | virtual unsigned int execute (function *); | |
1783 | ||
1784 | /* Define on demand and cache some types we use often. */ | |
1785 | #define DEF_TYPE(IDX, NAME, INIT) \ | |
1786 | static inline tree get_ ## NAME () { \ | |
1787 | int idx = STRUB_TYPE_BASE + IDX; \ | |
1788 | static tree type = strub_cache[idx]; \ | |
1789 | if (!type) \ | |
1790 | strub_cache[idx] = type = (INIT); \ | |
1791 | return type; \ | |
1792 | } | |
1793 | ||
1794 | /* Use a distinct ptr_type_node to denote the watermark, so that we can | |
1795 | recognize it in arg lists and avoid modifying types twice. */ | |
1796 | DEF_TYPE (0, wmt, build_variant_type_copy (ptr_type_node)) | |
1797 | ||
1798 | DEF_TYPE (1, pwmt, build_reference_type (get_wmt ())) | |
1799 | ||
1800 | DEF_TYPE (2, qpwmt, | |
1801 | build_qualified_type (get_pwmt (), | |
1802 | TYPE_QUAL_RESTRICT | |
1803 | /* | TYPE_QUAL_CONST */)) | |
1804 | ||
1805 | DEF_TYPE (3, qptr, | |
1806 | build_qualified_type (ptr_type_node, | |
1807 | TYPE_QUAL_RESTRICT | |
1808 | | TYPE_QUAL_CONST)) | |
1809 | ||
1810 | DEF_TYPE (4, qpvalst, | |
1811 | build_qualified_type (build_reference_type | |
1812 | (va_list_type_node), | |
1813 | TYPE_QUAL_RESTRICT | |
1814 | /* | TYPE_QUAL_CONST */)) | |
1815 | ||
1816 | #undef DEF_TYPE | |
1817 | ||
1818 | /* Define non-strub builtins on demand. */ | |
1819 | #define DEF_NM_BUILTIN(NAME, CODE, FNTYPELIST) \ | |
1820 | static tree get_ ## NAME () { \ | |
1821 | tree decl = builtin_decl_explicit (CODE); \ | |
1822 | if (!decl) \ | |
1823 | { \ | |
1824 | tree type = build_function_type_list FNTYPELIST; \ | |
1825 | decl = add_builtin_function \ | |
1826 | ("__builtin_" #NAME, \ | |
1827 | type, CODE, BUILT_IN_NORMAL, \ | |
1828 | NULL, NULL); \ | |
1829 | TREE_NOTHROW (decl) = true; \ | |
1830 | set_builtin_decl ((CODE), decl, true); \ | |
1831 | } \ | |
1832 | return decl; \ | |
1833 | } | |
1834 | ||
1835 | DEF_NM_BUILTIN (stack_address, | |
1836 | BUILT_IN_STACK_ADDRESS, | |
1837 | (ptr_type_node, NULL)) | |
1838 | ||
1839 | #undef DEF_NM_BUILTIN | |
1840 | ||
1841 | /* Define strub builtins on demand. */ | |
1842 | #define DEF_SS_BUILTIN(NAME, FNSPEC, CODE, FNTYPELIST) \ | |
1843 | static tree get_ ## NAME () { \ | |
1844 | tree decl = builtin_decl_explicit (CODE); \ | |
1845 | if (!decl) \ | |
1846 | { \ | |
1847 | tree type = build_function_type_list FNTYPELIST; \ | |
1848 | tree attrs = NULL; \ | |
1849 | if (FNSPEC) \ | |
1850 | attrs = tree_cons (get_identifier ("fn spec"), \ | |
1851 | build_tree_list \ | |
1852 | (NULL_TREE, \ | |
1853 | build_string (strlen (FNSPEC), \ | |
1854 | (FNSPEC))), \ | |
1855 | attrs); \ | |
1856 | decl = add_builtin_function_ext_scope \ | |
1857 | ("__builtin___strub_" #NAME, \ | |
1858 | type, CODE, BUILT_IN_NORMAL, \ | |
1859 | "__strub_" #NAME, attrs); \ | |
1860 | TREE_NOTHROW (decl) = true; \ | |
1861 | set_builtin_decl ((CODE), decl, true); \ | |
1862 | } \ | |
1863 | return decl; \ | |
1864 | } | |
1865 | ||
1866 | DEF_SS_BUILTIN (enter, ". Ot", | |
1867 | BUILT_IN___STRUB_ENTER, | |
1868 | (void_type_node, get_qpwmt (), NULL)) | |
1869 | DEF_SS_BUILTIN (update, ". Wt", | |
1870 | BUILT_IN___STRUB_UPDATE, | |
1871 | (void_type_node, get_qpwmt (), NULL)) | |
1872 | DEF_SS_BUILTIN (leave, ". w ", | |
1873 | BUILT_IN___STRUB_LEAVE, | |
1874 | (void_type_node, get_qpwmt (), NULL)) | |
1875 | ||
1876 | #undef DEF_SS_BUILTIN | |
1877 | ||
1878 | /* Define strub identifiers on demand. */ | |
1879 | #define DEF_IDENT(IDX, NAME) \ | |
1880 | static inline tree get_ ## NAME () { \ | |
1881 | int idx = STRUB_IDENT_BASE + IDX; \ | |
1882 | tree identifier = strub_cache[idx]; \ | |
1883 | if (!identifier) \ | |
1884 | strub_cache[idx] = identifier = get_identifier (".strub." #NAME); \ | |
1885 | return identifier; \ | |
1886 | } | |
1887 | ||
1888 | DEF_IDENT (0, watermark_ptr) | |
1889 | DEF_IDENT (1, va_list_ptr) | |
1890 | DEF_IDENT (2, apply_args) | |
1891 | ||
1892 | #undef DEF_IDENT | |
1893 | ||
1894 | static inline int adjust_at_calls_type (tree); | |
1895 | static inline void adjust_at_calls_call (cgraph_edge *, int, tree); | |
1896 | static inline void adjust_at_calls_calls (cgraph_node *); | |
1897 | ||
1898 | /* Add to SEQ a call to the strub watermark update builtin, taking NODE's | |
1899 | location if given. Optionally add the corresponding edge from NODE, with | |
1900 | execution frequency COUNT. Return the modified SEQ. */ | |
1901 | ||
1902 | static inline gimple_seq | |
1903 | call_update_watermark (tree wmptr, cgraph_node *node, profile_count count, | |
1904 | gimple_seq seq = NULL) | |
1905 | { | |
1906 | tree uwm = get_update (); | |
1907 | gcall *update = gimple_build_call (uwm, 1, wmptr); | |
1908 | if (node) | |
1909 | gimple_set_location (update, DECL_SOURCE_LOCATION (node->decl)); | |
1910 | gimple_seq_add_stmt (&seq, update); | |
1911 | if (node) | |
1912 | node->create_edge (cgraph_node::get_create (uwm), update, count, false); | |
1913 | return seq; | |
1914 | } | |
1915 | ||
1916 | }; | |
1917 | ||
1918 | } // anon namespace | |
1919 | ||
1920 | /* Gather with this type a collection of parameters that we're turning into | |
1921 | explicit references. */ | |
1922 | ||
1923 | typedef hash_set<tree> indirect_parms_t; | |
1924 | ||
1925 | /* Dereference OP's incoming turned-into-reference parm if it's an | |
1926 | INDIRECT_PARMS or an ADDR_EXPR thereof. Set *REC and return according to | |
1927 | gimple-walking expectations. */ | |
1928 | ||
1929 | static tree | |
1930 | maybe_make_indirect (indirect_parms_t &indirect_parms, tree op, int *rec) | |
1931 | { | |
1932 | if (DECL_P (op)) | |
1933 | { | |
1934 | *rec = 0; | |
1935 | if (indirect_parms.contains (op)) | |
1936 | { | |
1937 | tree ret = gimple_fold_indirect_ref (op); | |
1938 | if (!ret) | |
1939 | ret = build2 (MEM_REF, | |
1940 | TREE_TYPE (TREE_TYPE (op)), | |
1941 | op, | |
1942 | build_int_cst (TREE_TYPE (op), 0)); | |
c39dc5bb AO |
1943 | if (TYPE_VOLATILE (TREE_TYPE (TREE_TYPE (op))) |
1944 | && !TREE_THIS_VOLATILE (ret)) | |
1945 | TREE_SIDE_EFFECTS (ret) = TREE_THIS_VOLATILE (ret) = 1; | |
f0a90c7d AO |
1946 | return ret; |
1947 | } | |
1948 | } | |
1949 | else if (TREE_CODE (op) == ADDR_EXPR | |
1950 | && DECL_P (TREE_OPERAND (op, 0))) | |
1951 | { | |
1952 | *rec = 0; | |
1953 | if (indirect_parms.contains (TREE_OPERAND (op, 0))) | |
1954 | { | |
1955 | op = TREE_OPERAND (op, 0); | |
1956 | return op; | |
1957 | } | |
1958 | } | |
1959 | ||
1960 | return NULL_TREE; | |
1961 | } | |
1962 | ||
1963 | /* A gimple-walking function that adds dereferencing to indirect parms. */ | |
1964 | ||
1965 | static tree | |
1966 | walk_make_indirect (tree *op, int *rec, void *arg) | |
1967 | { | |
1968 | walk_stmt_info *wi = (walk_stmt_info *)arg; | |
1969 | indirect_parms_t &indirect_parms = *(indirect_parms_t *)wi->info; | |
1970 | ||
1971 | if (!*op || TYPE_P (*op)) | |
1972 | { | |
1973 | *rec = 0; | |
1974 | return NULL_TREE; | |
1975 | } | |
1976 | ||
1977 | if (tree repl = maybe_make_indirect (indirect_parms, *op, rec)) | |
1978 | { | |
1979 | *op = repl; | |
1980 | wi->changed = true; | |
1981 | } | |
1982 | ||
1983 | return NULL_TREE; | |
1984 | } | |
1985 | ||
1986 | /* A gimple-walking function that turns any non-gimple-val ADDR_EXPRs into a | |
1987 | separate SSA. Though addresses of e.g. parameters, and of members thereof, | |
1988 | are gimple vals, turning parameters into references, with an extra layer of | |
1989 | indirection and thus explicit dereferencing, need to be regimplified. */ | |
1990 | ||
1991 | static tree | |
1992 | walk_regimplify_addr_expr (tree *op, int *rec, void *arg) | |
1993 | { | |
1994 | walk_stmt_info *wi = (walk_stmt_info *)arg; | |
1995 | gimple_stmt_iterator &gsi = *(gimple_stmt_iterator *)wi->info; | |
1996 | ||
1997 | *rec = 0; | |
1998 | ||
1999 | if (!*op || TREE_CODE (*op) != ADDR_EXPR) | |
2000 | return NULL_TREE; | |
2001 | ||
2002 | if (!is_gimple_val (*op)) | |
2003 | { | |
2004 | tree ret = force_gimple_operand_gsi (&gsi, *op, true, | |
2005 | NULL_TREE, true, GSI_SAME_STMT); | |
2006 | gcc_assert (ret != *op); | |
2007 | *op = ret; | |
2008 | wi->changed = true; | |
2009 | } | |
2010 | ||
2011 | return NULL_TREE; | |
2012 | } | |
2013 | ||
2014 | /* Turn STMT's PHI arg defs into separate SSA defs if they've become | |
2015 | non-gimple_val. Return TRUE if any edge insertions need to be committed. */ | |
2016 | ||
2017 | static bool | |
2018 | walk_regimplify_phi (gphi *stmt) | |
2019 | { | |
2020 | bool needs_commit = false; | |
2021 | ||
2022 | for (unsigned i = 0, n = gimple_phi_num_args (stmt); i < n; i++) | |
2023 | { | |
2024 | tree op = gimple_phi_arg_def (stmt, i); | |
2025 | if ((TREE_CODE (op) == ADDR_EXPR | |
2026 | && !is_gimple_val (op)) | |
2027 | /* ??? A PARM_DECL that was addressable in the original function and | |
2028 | had its address in PHI nodes, but that became a reference in the | |
2029 | wrapped clone would NOT be updated by update_ssa in PHI nodes. | |
2030 | Alas, if we were to create a default def for it now, update_ssa | |
2031 | would complain that the symbol that needed rewriting already has | |
2032 | SSA names associated with it. OTOH, leaving the PARM_DECL alone, | |
2033 | it eventually causes errors because it remains unchanged in PHI | |
2034 | nodes, but it gets rewritten as expected if it appears in other | |
2035 | stmts. So we cheat a little here, and force the PARM_DECL out of | |
2036 | the PHI node and into an assignment. It's a little expensive, | |
2037 | because we insert it at the edge, which introduces a basic block | |
2038 | that's entirely unnecessary, but it works, and the block will be | |
2039 | removed as the default def gets propagated back into the PHI node, | |
2040 | so the final optimized code looks just as expected. */ | |
2041 | || (TREE_CODE (op) == PARM_DECL | |
2042 | && !TREE_ADDRESSABLE (op))) | |
2043 | { | |
2044 | tree temp = make_ssa_name (TREE_TYPE (op), stmt); | |
2045 | if (TREE_CODE (op) == PARM_DECL) | |
2046 | SET_SSA_NAME_VAR_OR_IDENTIFIER (temp, DECL_NAME (op)); | |
2047 | SET_PHI_ARG_DEF (stmt, i, temp); | |
2048 | ||
2049 | gimple *assign = gimple_build_assign (temp, op); | |
2050 | if (gimple_phi_arg_has_location (stmt, i)) | |
2051 | gimple_set_location (assign, gimple_phi_arg_location (stmt, i)); | |
2052 | gsi_insert_on_edge (gimple_phi_arg_edge (stmt, i), assign); | |
2053 | needs_commit = true; | |
2054 | } | |
2055 | } | |
2056 | ||
2057 | return needs_commit; | |
2058 | } | |
2059 | ||
d826596a AO |
2060 | /* Create a reference type to use for PARM when turning it into a |
2061 | reference. */ | |
f0a90c7d AO |
2062 | |
2063 | static tree | |
d826596a | 2064 | build_ref_type_for (tree parm) |
f0a90c7d AO |
2065 | { |
2066 | gcc_checking_assert (TREE_CODE (parm) == PARM_DECL); | |
2067 | ||
2068 | tree ref_type = build_reference_type (TREE_TYPE (parm)); | |
2069 | ||
d826596a | 2070 | return ref_type; |
f0a90c7d AO |
2071 | } |
2072 | ||
2073 | /* Add cgraph edges from current_function_decl to callees in SEQ with frequency | |
2074 | COUNT, assuming all calls in SEQ are direct. */ | |
2075 | ||
2076 | static void | |
2077 | add_call_edges_for_seq (gimple_seq seq, profile_count count) | |
2078 | { | |
2079 | cgraph_node *node = cgraph_node::get_create (current_function_decl); | |
2080 | ||
2081 | for (gimple_stmt_iterator gsi = gsi_start (seq); | |
2082 | !gsi_end_p (gsi); gsi_next (&gsi)) | |
2083 | { | |
2084 | gimple *stmt = gsi_stmt (gsi); | |
2085 | ||
2086 | gcall *call = dyn_cast <gcall *> (stmt); | |
2087 | if (!call) | |
2088 | continue; | |
2089 | ||
2090 | tree callee = gimple_call_fndecl (call); | |
2091 | gcc_checking_assert (callee); | |
2092 | node->create_edge (cgraph_node::get_create (callee), call, count, false); | |
2093 | } | |
2094 | } | |
2095 | ||
2096 | /* Insert SEQ after the call at GSI, as if the call was in a try block with SEQ | |
2097 | as finally, i.e., SEQ will run after the call whether it returns or | |
2098 | propagates an exception. This handles block splitting, EH edge and block | |
2099 | creation, noreturn and nothrow optimizations, and even throwing calls without | |
2100 | preexisting local handlers. */ | |
2101 | ||
2102 | static void | |
2103 | gsi_insert_finally_seq_after_call (gimple_stmt_iterator gsi, gimple_seq seq) | |
2104 | { | |
2105 | if (!seq) | |
2106 | return; | |
2107 | ||
2108 | gimple *stmt = gsi_stmt (gsi); | |
2109 | ||
2110 | if (gimple_has_location (stmt)) | |
2111 | annotate_all_with_location (seq, gimple_location (stmt)); | |
2112 | ||
2113 | gcall *call = dyn_cast <gcall *> (stmt); | |
2114 | bool noreturn_p = call && gimple_call_noreturn_p (call); | |
2115 | int eh_lp = lookup_stmt_eh_lp (stmt); | |
2116 | bool must_not_throw_p = eh_lp < 0; | |
2117 | bool nothrow_p = (must_not_throw_p | |
2118 | || (call && gimple_call_nothrow_p (call)) | |
2119 | || (eh_lp <= 0 | |
2120 | && (TREE_NOTHROW (cfun->decl) | |
181917d5 | 2121 | || !opt_for_fn (cfun->decl, flag_exceptions)))); |
f0a90c7d AO |
2122 | |
2123 | if (noreturn_p && nothrow_p) | |
2124 | return; | |
2125 | ||
2126 | /* Don't expect an EH edge if we're not to throw, or if we're not in an EH | |
2127 | region yet. */ | |
2128 | bool no_eh_edge_p = (nothrow_p || !eh_lp); | |
2129 | bool must_end_bb = stmt_ends_bb_p (stmt); | |
2130 | ||
2131 | edge eft = NULL, eeh = NULL; | |
2132 | if (must_end_bb && !(noreturn_p && no_eh_edge_p)) | |
2133 | { | |
2134 | gcc_checking_assert (gsi_one_before_end_p (gsi)); | |
2135 | ||
2136 | edge e; | |
2137 | edge_iterator ei; | |
2138 | FOR_EACH_EDGE (e, ei, gsi_bb (gsi)->succs) | |
2139 | { | |
2140 | if ((e->flags & EDGE_EH)) | |
2141 | { | |
2142 | gcc_checking_assert (!eeh); | |
2143 | eeh = e; | |
2144 | #if !CHECKING_P | |
2145 | if (eft || noreturn_p) | |
2146 | break; | |
2147 | #endif | |
2148 | } | |
2149 | if ((e->flags & EDGE_FALLTHRU)) | |
2150 | { | |
2151 | gcc_checking_assert (!eft); | |
2152 | eft = e; | |
2153 | #if !CHECKING_P | |
2154 | if (eeh || no_eh_edge_p) | |
2155 | break; | |
2156 | #endif | |
2157 | } | |
2158 | } | |
2159 | ||
2160 | gcc_checking_assert (!(eft && (eft->flags & EDGE_FALLTHRU)) | |
2161 | == noreturn_p); | |
2162 | gcc_checking_assert (!(eeh && (eeh->flags & EDGE_EH)) | |
2163 | == no_eh_edge_p); | |
2164 | gcc_checking_assert (eft != eeh); | |
2165 | } | |
2166 | ||
2167 | if (!noreturn_p) | |
2168 | { | |
2169 | gimple_seq nseq = nothrow_p ? seq : gimple_seq_copy (seq); | |
2170 | ||
2171 | if (must_end_bb) | |
2172 | { | |
2173 | gcc_checking_assert (gsi_one_before_end_p (gsi)); | |
2174 | add_call_edges_for_seq (nseq, eft->count ()); | |
2175 | gsi_insert_seq_on_edge_immediate (eft, nseq); | |
2176 | } | |
2177 | else | |
2178 | { | |
2179 | add_call_edges_for_seq (nseq, gsi_bb (gsi)->count); | |
2180 | gsi_insert_seq_after (&gsi, nseq, GSI_SAME_STMT); | |
2181 | } | |
2182 | } | |
2183 | ||
2184 | if (nothrow_p) | |
2185 | return; | |
2186 | ||
2187 | if (eh_lp) | |
2188 | { | |
2189 | add_call_edges_for_seq (seq, eeh->count ()); | |
2190 | gsi_insert_seq_on_edge_immediate (eeh, seq); | |
2191 | return; | |
2192 | } | |
2193 | ||
2194 | /* A throwing call may appear within a basic block in a function that doesn't | |
2195 | have any EH regions. We're going to add a cleanup if so, therefore the | |
2196 | block will have to be split. */ | |
2197 | basic_block bb = gsi_bb (gsi); | |
2198 | if (!gsi_one_before_end_p (gsi)) | |
2199 | split_block (bb, stmt); | |
2200 | ||
2201 | /* Create a new block for the EH cleanup. */ | |
2202 | basic_block bb_eh_cleanup = create_empty_bb (bb); | |
2203 | if (dom_info_available_p (CDI_DOMINATORS)) | |
2204 | set_immediate_dominator (CDI_DOMINATORS, bb_eh_cleanup, bb); | |
2205 | if (current_loops) | |
2206 | add_bb_to_loop (bb_eh_cleanup, current_loops->tree_root); | |
2207 | ||
2208 | /* Make the new block an EH cleanup for the call. */ | |
2209 | eh_region new_r = gen_eh_region_cleanup (NULL); | |
2210 | eh_landing_pad lp = gen_eh_landing_pad (new_r); | |
2211 | tree label = gimple_block_label (bb_eh_cleanup); | |
2212 | lp->post_landing_pad = label; | |
2213 | EH_LANDING_PAD_NR (label) = lp->index; | |
2214 | add_stmt_to_eh_lp (stmt, lp->index); | |
2215 | ||
2216 | /* Add the cleanup code to the EH cleanup block. */ | |
2217 | gsi = gsi_after_labels (bb_eh_cleanup); | |
2218 | gsi_insert_seq_before (&gsi, seq, GSI_SAME_STMT); | |
2219 | ||
2220 | /* And then propagate the exception further. */ | |
2221 | gresx *resx = gimple_build_resx (new_r->index); | |
2222 | if (gimple_has_location (stmt)) | |
2223 | gimple_set_location (resx, gimple_location (stmt)); | |
2224 | gsi_insert_before (&gsi, resx, GSI_SAME_STMT); | |
2225 | ||
2226 | /* Finally, wire the EH cleanup block into the CFG. */ | |
2227 | edge neeh = make_eh_edge (stmt); | |
2228 | neeh->probability = profile_probability::never (); | |
2229 | gcc_checking_assert (neeh->dest == bb_eh_cleanup); | |
2230 | gcc_checking_assert (!neeh->dest->count.initialized_p ()); | |
2231 | neeh->dest->count = neeh->count (); | |
2232 | add_call_edges_for_seq (seq, neeh->dest->count); | |
2233 | } | |
2234 | ||
2235 | /* Copy the attribute list at *ATTRS, minus any NAME attributes, leaving | |
2236 | shareable trailing nodes alone. */ | |
2237 | ||
2238 | static inline void | |
2239 | remove_named_attribute_unsharing (const char *name, tree *attrs) | |
2240 | { | |
2241 | while (tree found = lookup_attribute (name, *attrs)) | |
2242 | { | |
2243 | /* Copy nodes up to the next NAME attribute. */ | |
2244 | while (*attrs != found) | |
2245 | { | |
2246 | *attrs = tree_cons (TREE_PURPOSE (*attrs), | |
2247 | TREE_VALUE (*attrs), | |
2248 | TREE_CHAIN (*attrs)); | |
2249 | attrs = &TREE_CHAIN (*attrs); | |
2250 | } | |
2251 | /* Then drop it. */ | |
2252 | gcc_checking_assert (*attrs == found); | |
2253 | *attrs = TREE_CHAIN (*attrs); | |
2254 | } | |
2255 | } | |
2256 | ||
2257 | /* Record the order of the last cgraph entry whose mode we've already set, so | |
2258 | that we can perform mode setting incrementally without duplication. */ | |
2259 | static int last_cgraph_order; | |
2260 | ||
2261 | /* Set strub modes for functions introduced since the last call. */ | |
2262 | ||
2263 | static void | |
2264 | ipa_strub_set_mode_for_new_functions () | |
2265 | { | |
2266 | if (symtab->order == last_cgraph_order) | |
2267 | return; | |
2268 | ||
2269 | cgraph_node *node; | |
2270 | ||
2271 | /* Go through the functions twice, once over non-aliases, and then over | |
2272 | aliases, so that aliases can reuse the mode computation of their ultimate | |
2273 | targets. */ | |
2274 | for (int aliases = 0; aliases <= 1; aliases++) | |
2275 | FOR_EACH_FUNCTION (node) | |
2276 | { | |
2277 | if (!node->alias != !aliases) | |
2278 | continue; | |
2279 | ||
2280 | /* Already done. */ | |
2281 | if (node->order < last_cgraph_order) | |
2282 | continue; | |
2283 | ||
2284 | set_strub_mode (node); | |
2285 | } | |
2286 | ||
2287 | last_cgraph_order = symtab->order; | |
2288 | } | |
2289 | ||
2290 | /* Return FALSE if NODE is a strub context, and TRUE otherwise. */ | |
2291 | ||
2292 | bool | |
2293 | strub_splittable_p (cgraph_node *node) | |
2294 | { | |
2295 | switch (get_strub_mode (node)) | |
2296 | { | |
2297 | case STRUB_WRAPPED: | |
2298 | case STRUB_AT_CALLS: | |
2299 | case STRUB_AT_CALLS_OPT: | |
2300 | case STRUB_INLINABLE: | |
2301 | case STRUB_INTERNAL: | |
2302 | case STRUB_WRAPPER: | |
2303 | return false; | |
2304 | ||
2305 | case STRUB_CALLABLE: | |
2306 | case STRUB_DISABLED: | |
2307 | break; | |
2308 | ||
2309 | default: | |
2310 | gcc_unreachable (); | |
2311 | } | |
2312 | ||
2313 | return true; | |
2314 | } | |
2315 | ||
2316 | /* Return the PARM_DECL of the incoming watermark pointer, if there is one. */ | |
2317 | ||
2318 | tree | |
2319 | strub_watermark_parm (tree fndecl) | |
2320 | { | |
2321 | switch (get_strub_mode_from_fndecl (fndecl)) | |
2322 | { | |
2323 | case STRUB_WRAPPED: | |
2324 | case STRUB_AT_CALLS: | |
2325 | case STRUB_AT_CALLS_OPT: | |
2326 | break; | |
2327 | ||
2328 | case STRUB_INTERNAL: | |
2329 | case STRUB_WRAPPER: | |
2330 | case STRUB_CALLABLE: | |
2331 | case STRUB_DISABLED: | |
2332 | case STRUB_INLINABLE: | |
2333 | return NULL_TREE; | |
2334 | ||
2335 | default: | |
2336 | gcc_unreachable (); | |
2337 | } | |
2338 | ||
2339 | for (tree parm = DECL_ARGUMENTS (fndecl); parm; parm = DECL_CHAIN (parm)) | |
2340 | /* The type (variant) compare finds the parameter even in a just-created | |
2341 | clone, before we set its name, but the type-based compare doesn't work | |
2342 | during builtin expansion within the lto compiler, because we'll have | |
2343 | created a separate variant in that run. */ | |
2344 | if (TREE_TYPE (parm) == pass_ipa_strub::get_qpwmt () | |
2345 | || DECL_NAME (parm) == pass_ipa_strub::get_watermark_ptr ()) | |
2346 | return parm; | |
2347 | ||
2348 | gcc_unreachable (); | |
2349 | } | |
2350 | ||
2351 | /* Adjust a STRUB_AT_CALLS function TYPE, adding a watermark pointer if it | |
2352 | hasn't been added yet. Return the named argument count. */ | |
2353 | ||
2354 | int | |
2355 | pass_ipa_strub::adjust_at_calls_type (tree type) | |
2356 | { | |
2357 | int named_args = 0; | |
2358 | ||
2359 | gcc_checking_assert (same_strub_mode_in_variants_p (type)); | |
2360 | ||
2361 | if (!TYPE_ARG_TYPES (type)) | |
2362 | return named_args; | |
2363 | ||
2364 | tree *tlist = &TYPE_ARG_TYPES (type); | |
2365 | tree qpwmptrt = get_qpwmt (); | |
2366 | while (*tlist && TREE_VALUE (*tlist) != void_type_node) | |
2367 | { | |
2368 | /* The type has already been adjusted. */ | |
2369 | if (TREE_VALUE (*tlist) == qpwmptrt) | |
2370 | return named_args; | |
2371 | named_args++; | |
2372 | *tlist = tree_cons (TREE_PURPOSE (*tlist), | |
2373 | TREE_VALUE (*tlist), | |
2374 | TREE_CHAIN (*tlist)); | |
2375 | tlist = &TREE_CHAIN (*tlist); | |
2376 | } | |
2377 | ||
2378 | /* Add the new argument after all named arguments, so as to not mess with | |
2379 | attributes that reference parameters. */ | |
2380 | *tlist = tree_cons (NULL_TREE, get_qpwmt (), *tlist); | |
2381 | ||
2382 | #if ATTR_FNSPEC_DECONST_WATERMARK | |
2383 | if (!type_already_adjusted) | |
2384 | { | |
2385 | int flags = flags_from_decl_or_type (type); | |
2386 | tree fnspec = lookup_attribute ("fn spec", type); | |
2387 | ||
2388 | if ((flags & (ECF_CONST | ECF_PURE | ECF_NOVOPS)) || fnspec) | |
2389 | { | |
2390 | size_t xargs = 1; | |
2391 | size_t curlen = 0, tgtlen = 2 + 2 * (named_args + xargs); | |
2392 | auto_vec<char> nspecv (tgtlen); | |
2393 | char *nspec = &nspecv[0]; /* It will *not* be NUL-terminated! */ | |
2394 | if (fnspec) | |
2395 | { | |
2396 | tree fnspecstr = TREE_VALUE (TREE_VALUE (fnspec)); | |
2397 | curlen = TREE_STRING_LENGTH (fnspecstr); | |
2398 | memcpy (nspec, TREE_STRING_POINTER (fnspecstr), curlen); | |
2399 | } | |
2400 | if (!curlen) | |
2401 | { | |
2402 | nspec[curlen++] = '.'; | |
2403 | nspec[curlen++] = ((flags & ECF_CONST) | |
2404 | ? 'c' | |
2405 | : (flags & ECF_PURE) | |
2406 | ? 'p' | |
2407 | : ' '); | |
2408 | } | |
2409 | while (curlen < tgtlen - 2 * xargs) | |
2410 | { | |
2411 | nspec[curlen++] = '.'; | |
2412 | nspec[curlen++] = ' '; | |
2413 | } | |
2414 | nspec[curlen++] = 'W'; | |
2415 | nspec[curlen++] = 't'; | |
2416 | ||
2417 | /* The type has already been copied, if needed, before adding | |
2418 | parameters. */ | |
2419 | TYPE_ATTRIBUTES (type) | |
2420 | = tree_cons (get_identifier ("fn spec"), | |
2421 | build_tree_list (NULL_TREE, | |
2422 | build_string (tgtlen, nspec)), | |
2423 | TYPE_ATTRIBUTES (type)); | |
2424 | } | |
2425 | } | |
2426 | #endif | |
2427 | ||
2428 | return named_args; | |
2429 | } | |
2430 | ||
2431 | /* Adjust a call to an at-calls call target. Create a watermark local variable | |
2432 | if needed, initialize it before, pass it to the callee according to the | |
2433 | modified at-calls interface, and release the callee's stack space after the | |
2434 | call, if not deferred. If the call is const or pure, arrange for the | |
2435 | watermark to not be assumed unused or unchanged. */ | |
2436 | ||
2437 | void | |
2438 | pass_ipa_strub::adjust_at_calls_call (cgraph_edge *e, int named_args, | |
2439 | tree callee_fntype) | |
2440 | { | |
2441 | gcc_checking_assert (e->call_stmt); | |
2442 | gcall *ocall = e->call_stmt; | |
2443 | gimple_stmt_iterator gsi = gsi_for_stmt (ocall); | |
2444 | ||
2445 | /* Make sure we haven't modified this call yet. */ | |
2446 | gcc_checking_assert (!(int (gimple_call_num_args (ocall)) > named_args | |
2447 | && (TREE_TYPE (gimple_call_arg (ocall, named_args)) | |
2448 | == get_pwmt ()))); | |
2449 | ||
f908368d AO |
2450 | tree tsup; |
2451 | if (!(tsup = gimple_call_fndecl (ocall))) | |
2452 | tsup = TREE_TYPE (TREE_TYPE (gimple_call_fn (ocall))); | |
2453 | if (!strub_target_support_p (tsup, true, gimple_location (ocall))) | |
2454 | return; | |
2455 | ||
f0a90c7d AO |
2456 | /* If we're already within a strub context, pass on the incoming watermark |
2457 | pointer, and omit the enter and leave calls around the modified call, as an | |
2458 | optimization, or as a means to satisfy a tail-call requirement. */ | |
181917d5 AO |
2459 | tree swmp = ((opt_for_fn (e->caller->decl, optimize_size) |
2460 | || opt_for_fn (e->caller->decl, optimize) > 2 | |
f0a90c7d | 2461 | || gimple_call_must_tail_p (ocall) |
181917d5 AO |
2462 | || (opt_for_fn (e->caller->decl, optimize) == 2 |
2463 | && gimple_call_tail_p (ocall))) | |
f0a90c7d AO |
2464 | ? strub_watermark_parm (e->caller->decl) |
2465 | : NULL_TREE); | |
2466 | bool omit_own_watermark = swmp; | |
2467 | tree swm = NULL_TREE; | |
2468 | if (!omit_own_watermark) | |
2469 | { | |
2470 | swm = create_tmp_var (get_wmt (), ".strub.watermark"); | |
2471 | TREE_ADDRESSABLE (swm) = true; | |
2472 | swmp = build1 (ADDR_EXPR, get_pwmt (), swm); | |
2473 | ||
2474 | /* Initialize the watermark before the call. */ | |
2475 | tree enter = get_enter (); | |
2476 | gcall *stptr = gimple_build_call (enter, 1, | |
2477 | unshare_expr (swmp)); | |
2478 | if (gimple_has_location (ocall)) | |
2479 | gimple_set_location (stptr, gimple_location (ocall)); | |
2480 | gsi_insert_before (&gsi, stptr, GSI_SAME_STMT); | |
2481 | e->caller->create_edge (cgraph_node::get_create (enter), | |
2482 | stptr, gsi_bb (gsi)->count, false); | |
2483 | } | |
2484 | ||
2485 | ||
2486 | /* Replace the call with one that passes the swmp argument first. */ | |
2487 | gcall *wrcall; | |
2488 | { gcall *stmt = ocall; | |
2489 | // Mostly copied from gimple_call_copy_skip_args. | |
2490 | int i = 0; | |
2491 | int nargs = gimple_call_num_args (stmt); | |
2492 | auto_vec<tree> vargs (MAX (nargs, named_args) + 1); | |
2493 | gcall *new_stmt; | |
2494 | ||
2495 | /* pr71109.c calls a prototypeless function, then defines it with | |
2496 | additional arguments. It's ill-formed, but after it's inlined, | |
2497 | it somehow works out. */ | |
2498 | for (; i < named_args && i < nargs; i++) | |
2499 | vargs.quick_push (gimple_call_arg (stmt, i)); | |
2500 | for (; i < named_args; i++) | |
2501 | vargs.quick_push (null_pointer_node); | |
2502 | ||
2503 | vargs.quick_push (unshare_expr (swmp)); | |
2504 | ||
2505 | for (; i < nargs; i++) | |
2506 | vargs.quick_push (gimple_call_arg (stmt, i)); | |
2507 | ||
2508 | if (gimple_call_internal_p (stmt)) | |
2509 | gcc_unreachable (); | |
2510 | else | |
2511 | new_stmt = gimple_build_call_vec (gimple_call_fn (stmt), vargs); | |
2512 | gimple_call_set_fntype (new_stmt, callee_fntype); | |
2513 | ||
2514 | if (gimple_call_lhs (stmt)) | |
2515 | gimple_call_set_lhs (new_stmt, gimple_call_lhs (stmt)); | |
2516 | ||
2517 | gimple_move_vops (new_stmt, stmt); | |
2518 | ||
2519 | if (gimple_has_location (stmt)) | |
2520 | gimple_set_location (new_stmt, gimple_location (stmt)); | |
2521 | gimple_call_copy_flags (new_stmt, stmt); | |
2522 | gimple_call_set_chain (new_stmt, gimple_call_chain (stmt)); | |
2523 | ||
2524 | gimple_set_modified (new_stmt, true); | |
2525 | ||
2526 | wrcall = new_stmt; | |
2527 | } | |
2528 | ||
2529 | update_stmt (wrcall); | |
2530 | gsi_replace (&gsi, wrcall, true); | |
2531 | cgraph_edge::set_call_stmt (e, wrcall, false); | |
2532 | ||
2533 | /* Insert the strub code after the call. */ | |
2534 | gimple_seq seq = NULL; | |
2535 | ||
2536 | #if !ATTR_FNSPEC_DECONST_WATERMARK | |
2537 | /* If the call will be assumed to not modify or even read the | |
2538 | watermark, make it read and modified ourselves. */ | |
2539 | if ((gimple_call_flags (wrcall) | |
2540 | & (ECF_CONST | ECF_PURE | ECF_NOVOPS))) | |
2541 | { | |
2542 | if (!swm) | |
2543 | swm = build2 (MEM_REF, | |
2544 | TREE_TYPE (TREE_TYPE (swmp)), | |
2545 | swmp, | |
2546 | build_int_cst (TREE_TYPE (swmp), 0)); | |
2547 | ||
2548 | vec<tree, va_gc> *inputs = NULL; | |
2549 | vec<tree, va_gc> *outputs = NULL; | |
2550 | vec_safe_push (outputs, | |
2551 | build_tree_list | |
2552 | (build_tree_list | |
2553 | (NULL_TREE, build_string (2, "=m")), | |
2554 | unshare_expr (swm))); | |
2555 | vec_safe_push (inputs, | |
2556 | build_tree_list | |
2557 | (build_tree_list | |
2558 | (NULL_TREE, build_string (1, "m")), | |
2559 | unshare_expr (swm))); | |
2560 | gasm *forcemod = gimple_build_asm_vec ("", inputs, outputs, | |
2561 | NULL, NULL); | |
2562 | gimple_seq_add_stmt (&seq, forcemod); | |
2563 | ||
2564 | /* If the call will be assumed to not even read the watermark, | |
2565 | make sure it is already in memory before the call. */ | |
2566 | if ((gimple_call_flags (wrcall) & ECF_CONST)) | |
2567 | { | |
2568 | vec<tree, va_gc> *inputs = NULL; | |
2569 | vec_safe_push (inputs, | |
2570 | build_tree_list | |
2571 | (build_tree_list | |
2572 | (NULL_TREE, build_string (1, "m")), | |
2573 | unshare_expr (swm))); | |
2574 | gasm *force_store = gimple_build_asm_vec ("", inputs, NULL, | |
2575 | NULL, NULL); | |
2576 | if (gimple_has_location (wrcall)) | |
2577 | gimple_set_location (force_store, gimple_location (wrcall)); | |
2578 | gsi_insert_before (&gsi, force_store, GSI_SAME_STMT); | |
2579 | } | |
2580 | } | |
2581 | #endif | |
2582 | ||
2583 | if (!omit_own_watermark) | |
2584 | { | |
2585 | gcall *sleave = gimple_build_call (get_leave (), 1, | |
2586 | unshare_expr (swmp)); | |
2587 | gimple_seq_add_stmt (&seq, sleave); | |
2588 | ||
2589 | gassign *clobber = gimple_build_assign (swm, | |
2590 | build_clobber | |
2591 | (TREE_TYPE (swm))); | |
2592 | gimple_seq_add_stmt (&seq, clobber); | |
2593 | } | |
2594 | ||
2595 | gsi_insert_finally_seq_after_call (gsi, seq); | |
2596 | } | |
2597 | ||
2598 | /* Adjust all at-calls calls in NODE. */ | |
2599 | ||
2600 | void | |
2601 | pass_ipa_strub::adjust_at_calls_calls (cgraph_node *node) | |
2602 | { | |
2603 | /* Adjust unknown-callee indirect calls with STRUB_AT_CALLS types within | |
2604 | onode. */ | |
2605 | if (node->indirect_calls) | |
2606 | { | |
2607 | push_cfun (DECL_STRUCT_FUNCTION (node->decl)); | |
2608 | for (cgraph_edge *e = node->indirect_calls; e; e = e->next_callee) | |
2609 | { | |
2610 | gcc_checking_assert (e->indirect_unknown_callee); | |
2611 | ||
2612 | if (!e->call_stmt) | |
2613 | continue; | |
2614 | ||
2615 | tree callee_fntype; | |
2616 | enum strub_mode callee_mode | |
2617 | = effective_strub_mode_for_call (e->call_stmt, &callee_fntype); | |
2618 | ||
2619 | if (callee_mode != STRUB_AT_CALLS | |
2620 | && callee_mode != STRUB_AT_CALLS_OPT) | |
2621 | continue; | |
2622 | ||
2623 | int named_args = adjust_at_calls_type (callee_fntype); | |
2624 | ||
2625 | adjust_at_calls_call (e, named_args, callee_fntype); | |
2626 | } | |
2627 | pop_cfun (); | |
2628 | } | |
2629 | ||
2630 | if (node->callees) | |
2631 | { | |
2632 | push_cfun (DECL_STRUCT_FUNCTION (node->decl)); | |
2633 | for (cgraph_edge *e = node->callees; e; e = e->next_callee) | |
2634 | { | |
2635 | gcc_checking_assert (!e->indirect_unknown_callee); | |
2636 | ||
2637 | if (!e->call_stmt) | |
2638 | continue; | |
2639 | ||
2640 | tree callee_fntype; | |
2641 | enum strub_mode callee_mode | |
2642 | = effective_strub_mode_for_call (e->call_stmt, &callee_fntype); | |
2643 | ||
2644 | if (callee_mode != STRUB_AT_CALLS | |
2645 | && callee_mode != STRUB_AT_CALLS_OPT) | |
2646 | continue; | |
2647 | ||
2648 | int named_args = adjust_at_calls_type (callee_fntype); | |
2649 | ||
2650 | adjust_at_calls_call (e, named_args, callee_fntype); | |
2651 | } | |
2652 | pop_cfun (); | |
2653 | } | |
2654 | } | |
2655 | ||
2656 | /* The strubm (strub mode) pass computes a strub mode for each function in the | |
2657 | call graph, and checks, before any inlining, that strub callability | |
2658 | requirements in effect are satisfied. */ | |
2659 | ||
2660 | unsigned int | |
2661 | pass_ipa_strub_mode::execute (function *) | |
2662 | { | |
2663 | last_cgraph_order = 0; | |
2664 | ipa_strub_set_mode_for_new_functions (); | |
2665 | ||
2666 | /* Verify before any inlining or other transformations. */ | |
2667 | verify_strub (); | |
2668 | ||
2669 | return 0; | |
2670 | } | |
2671 | ||
2672 | /* Create a strub mode pass. */ | |
2673 | ||
2674 | simple_ipa_opt_pass * | |
2675 | make_pass_ipa_strub_mode (gcc::context *ctxt) | |
2676 | { | |
2677 | return new pass_ipa_strub_mode (ctxt); | |
2678 | } | |
2679 | ||
2680 | /* The strub pass proper adjusts types, signatures, and at-calls calls, and | |
2681 | splits internal-strub functions. */ | |
2682 | ||
2683 | unsigned int | |
2684 | pass_ipa_strub::execute (function *) | |
2685 | { | |
2686 | cgraph_node *onode; | |
2687 | ||
2688 | ipa_strub_set_mode_for_new_functions (); | |
2689 | ||
2690 | /* First, adjust the signature of at-calls functions. We adjust types of | |
2691 | at-calls functions first, so that we don't modify types in place unless | |
2692 | strub is explicitly requested. */ | |
2693 | FOR_EACH_FUNCTION (onode) | |
2694 | { | |
2695 | enum strub_mode mode = get_strub_mode (onode); | |
2696 | ||
2697 | if (mode == STRUB_AT_CALLS | |
2698 | || mode == STRUB_AT_CALLS_OPT) | |
2699 | { | |
2700 | /* Create a type variant if strubbing was not explicitly requested in | |
2701 | the function type. */ | |
2702 | if (get_strub_mode_from_type (TREE_TYPE (onode->decl)) != mode) | |
2703 | distinctify_node_type (onode); | |
2704 | ||
2705 | int named_args = adjust_at_calls_type (TREE_TYPE (onode->decl)); | |
2706 | ||
2707 | /* An external function explicitly declared with strub won't have a | |
2708 | body. Even with implicit at-calls strub, a function may have had its | |
2709 | body removed after we selected the mode, and then we have nothing | |
2710 | further to do. */ | |
2711 | if (!onode->has_gimple_body_p ()) | |
2712 | continue; | |
2713 | ||
2714 | tree *pargs = &DECL_ARGUMENTS (onode->decl); | |
2715 | ||
2716 | /* A noninterposable_alias reuses the same parm decl chain, don't add | |
2717 | the parm twice. */ | |
2718 | bool aliased_parms = (onode->alias && *pargs | |
2719 | && DECL_CONTEXT (*pargs) != onode->decl); | |
2720 | ||
2721 | if (aliased_parms) | |
2722 | continue; | |
2723 | ||
2724 | for (int i = 0; i < named_args; i++) | |
2725 | pargs = &DECL_CHAIN (*pargs); | |
2726 | ||
2727 | tree wmptr = build_decl (DECL_SOURCE_LOCATION (onode->decl), | |
2728 | PARM_DECL, | |
2729 | get_watermark_ptr (), | |
2730 | get_qpwmt ()); | |
2731 | DECL_ARTIFICIAL (wmptr) = 1; | |
2732 | DECL_ARG_TYPE (wmptr) = get_qpwmt (); | |
2733 | DECL_CONTEXT (wmptr) = onode->decl; | |
2734 | TREE_USED (wmptr) = 1; | |
2735 | DECL_CHAIN (wmptr) = *pargs; | |
2736 | *pargs = wmptr; | |
2737 | ||
2738 | if (onode->alias) | |
2739 | continue; | |
2740 | ||
2741 | cgraph_node *nnode = onode; | |
2742 | push_cfun (DECL_STRUCT_FUNCTION (nnode->decl)); | |
2743 | ||
2744 | { | |
2745 | edge e = single_succ_edge (ENTRY_BLOCK_PTR_FOR_FN (cfun)); | |
2746 | gimple_seq seq = call_update_watermark (wmptr, nnode, e->src->count); | |
2747 | gsi_insert_seq_on_edge_immediate (e, seq); | |
2748 | } | |
2749 | ||
2750 | if (DECL_STRUCT_FUNCTION (nnode->decl)->calls_alloca) | |
2751 | { | |
2752 | basic_block bb; | |
2753 | FOR_EACH_BB_FN (bb, cfun) | |
2754 | for (gimple_stmt_iterator gsi = gsi_start_bb (bb); | |
2755 | !gsi_end_p (gsi); gsi_next (&gsi)) | |
2756 | { | |
2757 | gimple *stmt = gsi_stmt (gsi); | |
2758 | ||
2759 | gcall *call = dyn_cast <gcall *> (stmt); | |
2760 | ||
2761 | if (!call) | |
2762 | continue; | |
2763 | ||
2764 | if (gimple_alloca_call_p (call)) | |
2765 | { | |
2766 | /* Capture stack growth. */ | |
2767 | gimple_seq seq = call_update_watermark (wmptr, NULL, | |
2768 | gsi_bb (gsi) | |
2769 | ->count); | |
2770 | gsi_insert_finally_seq_after_call (gsi, seq); | |
2771 | } | |
2772 | } | |
2773 | } | |
2774 | ||
2775 | pop_cfun (); | |
2776 | } | |
2777 | } | |
2778 | ||
2779 | FOR_EACH_FUNCTION (onode) | |
2780 | { | |
2781 | if (!onode->has_gimple_body_p ()) | |
2782 | continue; | |
2783 | ||
2784 | enum strub_mode mode = get_strub_mode (onode); | |
2785 | ||
2786 | if (mode != STRUB_INTERNAL) | |
2787 | { | |
2788 | adjust_at_calls_calls (onode); | |
2789 | continue; | |
2790 | } | |
2791 | ||
2792 | bool is_stdarg = calls_builtin_va_start_p (onode);; | |
2793 | bool apply_args = calls_builtin_apply_args_p (onode); | |
2794 | ||
2795 | vec<ipa_adjusted_param, va_gc> *nparms = NULL; | |
2796 | unsigned j = 0; | |
2797 | { | |
2798 | // The following loop copied from ipa-split.c:split_function. | |
2799 | for (tree parm = DECL_ARGUMENTS (onode->decl); | |
2800 | parm; parm = DECL_CHAIN (parm), j++) | |
2801 | { | |
2802 | ipa_adjusted_param adj = {}; | |
2803 | adj.op = IPA_PARAM_OP_COPY; | |
2804 | adj.base_index = j; | |
2805 | adj.prev_clone_index = j; | |
2806 | vec_safe_push (nparms, adj); | |
2807 | } | |
2808 | ||
2809 | if (apply_args) | |
2810 | { | |
2811 | ipa_adjusted_param aaadj = {}; | |
2812 | aaadj.op = IPA_PARAM_OP_NEW; | |
2813 | aaadj.type = get_qptr (); | |
2814 | vec_safe_push (nparms, aaadj); | |
2815 | } | |
2816 | ||
2817 | if (is_stdarg) | |
2818 | { | |
2819 | ipa_adjusted_param vladj = {}; | |
2820 | vladj.op = IPA_PARAM_OP_NEW; | |
2821 | vladj.type = get_qpvalst (); | |
2822 | vec_safe_push (nparms, vladj); | |
2823 | } | |
2824 | ||
2825 | ipa_adjusted_param wmadj = {}; | |
2826 | wmadj.op = IPA_PARAM_OP_NEW; | |
2827 | wmadj.type = get_qpwmt (); | |
2828 | vec_safe_push (nparms, wmadj); | |
2829 | } | |
2830 | ipa_param_adjustments adj (nparms, -1, false); | |
2831 | ||
2832 | cgraph_node *nnode = onode->create_version_clone_with_body | |
2833 | (auto_vec<cgraph_edge *> (0), | |
2834 | NULL, &adj, NULL, NULL, "strub", NULL); | |
2835 | ||
2836 | if (!nnode) | |
2837 | { | |
2838 | error_at (DECL_SOURCE_LOCATION (onode->decl), | |
2839 | "failed to split %qD for %<strub%>", | |
2840 | onode->decl); | |
2841 | continue; | |
2842 | } | |
2843 | ||
2844 | onode->split_part = true; | |
2845 | if (onode->calls_comdat_local) | |
2846 | nnode->add_to_same_comdat_group (onode); | |
2847 | ||
2848 | set_strub_mode_to (onode, STRUB_WRAPPER); | |
2849 | set_strub_mode_to (nnode, STRUB_WRAPPED); | |
2850 | ||
2851 | adjust_at_calls_calls (nnode); | |
2852 | ||
2853 | /* Decide which of the wrapped function's parms we want to turn into | |
2854 | references to the argument passed to the wrapper. In general, we want to | |
2855 | copy small arguments, and avoid copying large ones. Variable-sized array | |
2856 | lengths given by other arguments, as in 20020210-1.c, would lead to | |
2857 | problems if passed by value, after resetting the original function and | |
2858 | dropping the length computation; passing them by reference works. | |
2859 | DECL_BY_REFERENCE is *not* a substitute for this: it involves copying | |
2860 | anyway, but performed at the caller. */ | |
2861 | indirect_parms_t indirect_nparms (3, false); | |
2862 | unsigned adjust_ftype = 0; | |
2863 | unsigned named_args = 0; | |
2864 | for (tree parm = DECL_ARGUMENTS (onode->decl), | |
2865 | nparm = DECL_ARGUMENTS (nnode->decl), | |
2866 | nparmt = TYPE_ARG_TYPES (TREE_TYPE (nnode->decl)); | |
2867 | parm; | |
2868 | named_args++, | |
2869 | parm = DECL_CHAIN (parm), | |
2870 | nparm = DECL_CHAIN (nparm), | |
2871 | nparmt = nparmt ? TREE_CHAIN (nparmt) : NULL_TREE) | |
e1e71b4e AO |
2872 | if (TREE_THIS_VOLATILE (parm) |
2873 | || !(0 /* DECL_BY_REFERENCE (narg) */ | |
2874 | || is_gimple_reg_type (TREE_TYPE (nparm)) | |
2875 | || VECTOR_TYPE_P (TREE_TYPE (nparm)) | |
2876 | || TREE_CODE (TREE_TYPE (nparm)) == COMPLEX_TYPE | |
2877 | || (tree_fits_uhwi_p (TYPE_SIZE_UNIT (TREE_TYPE (nparm))) | |
2878 | && (tree_to_uhwi (TYPE_SIZE_UNIT (TREE_TYPE (nparm))) | |
2879 | <= 4 * UNITS_PER_WORD)))) | |
f0a90c7d | 2880 | { |
e4fc59e0 AO |
2881 | /* No point in indirecting pointer types. Presumably they |
2882 | won't ever pass the size-based test above, but check the | |
2883 | assumption here, because getting this wrong would mess | |
2884 | with attribute access and possibly others. We deal with | |
2885 | fn spec below. */ | |
2886 | gcc_checking_assert (!POINTER_TYPE_P (TREE_TYPE (nparm))); | |
2887 | ||
f0a90c7d AO |
2888 | indirect_nparms.add (nparm); |
2889 | ||
2890 | /* ??? Is there any case in which it is not safe to suggest the parms | |
2891 | turned indirect don't alias anything else? They are distinct, | |
2892 | unaliased memory in the wrapper, and the wrapped can't possibly | |
2893 | take pointers into them because none of the pointers passed to the | |
2894 | wrapper can alias other incoming parameters passed by value, even | |
2895 | if with transparent reference, and the wrapper doesn't take any | |
2896 | extra parms that could point into wrapper's parms. So we can | |
2897 | probably drop the TREE_ADDRESSABLE and keep the TRUE. */ | |
d826596a | 2898 | tree ref_type = build_ref_type_for (nparm); |
f0a90c7d | 2899 | |
c39dc5bb AO |
2900 | if (TREE_THIS_VOLATILE (nparm) |
2901 | && TYPE_VOLATILE (TREE_TYPE (nparm)) | |
2902 | && !TYPE_VOLATILE (ref_type)) | |
2903 | TREE_SIDE_EFFECTS (nparm) = TREE_THIS_VOLATILE (nparm) = 0; | |
f0a90c7d AO |
2904 | DECL_ARG_TYPE (nparm) = TREE_TYPE (nparm) = ref_type; |
2905 | relayout_decl (nparm); | |
2906 | TREE_ADDRESSABLE (nparm) = 0; | |
2907 | DECL_BY_REFERENCE (nparm) = 0; | |
2908 | DECL_NOT_GIMPLE_REG_P (nparm) = 0; | |
2909 | /* ??? This avoids mismatches in debug info bind stmts in | |
2910 | e.g. a-chahan . */ | |
2911 | DECL_ABSTRACT_ORIGIN (nparm) = NULL; | |
2912 | ||
2913 | if (nparmt) | |
2914 | adjust_ftype++; | |
2915 | } | |
2916 | ||
2917 | /* Also adjust the wrapped function type, if needed. */ | |
2918 | if (adjust_ftype) | |
2919 | { | |
2920 | tree nftype = TREE_TYPE (nnode->decl); | |
2921 | ||
2922 | /* We always add at least one argument at the end of the signature, when | |
2923 | cloning the function, so we don't expect to need to duplicate the | |
2924 | type here. */ | |
2925 | gcc_checking_assert (TYPE_ARG_TYPES (nftype) | |
2926 | != TYPE_ARG_TYPES (TREE_TYPE (onode->decl))); | |
2927 | ||
2928 | /* Check that fnspec still works for the modified function signature, | |
2929 | and drop it otherwise. */ | |
2930 | bool drop_fnspec = false; | |
2931 | tree fnspec = lookup_attribute ("fn spec", TYPE_ATTRIBUTES (nftype)); | |
2932 | attr_fnspec spec = fnspec ? attr_fnspec (fnspec) : attr_fnspec (""); | |
2933 | ||
2934 | unsigned retcopy; | |
2935 | if (!(fnspec && spec.returns_arg (&retcopy))) | |
2936 | retcopy = (unsigned) -1; | |
2937 | ||
2938 | unsigned i = 0; | |
2939 | for (tree nparm = DECL_ARGUMENTS (nnode->decl), | |
2940 | nparmt = TYPE_ARG_TYPES (nftype); | |
2941 | adjust_ftype > 0; | |
2942 | i++, nparm = DECL_CHAIN (nparm), nparmt = TREE_CHAIN (nparmt)) | |
2943 | if (indirect_nparms.contains (nparm)) | |
2944 | { | |
2945 | TREE_VALUE (nparmt) = TREE_TYPE (nparm); | |
2946 | adjust_ftype--; | |
2947 | ||
2948 | if (fnspec && !drop_fnspec) | |
2949 | { | |
2950 | if (i == retcopy) | |
2951 | drop_fnspec = true; | |
2952 | else if (spec.arg_specified_p (i)) | |
2953 | { | |
2954 | /* Properties that apply to pointers only must not be | |
2955 | present, because we don't make pointers further | |
2956 | indirect. */ | |
2957 | gcc_checking_assert | |
2958 | (!spec.arg_max_access_size_given_by_arg_p (i, NULL)); | |
2959 | gcc_checking_assert (!spec.arg_copied_to_arg_p (i, NULL)); | |
2960 | ||
2961 | /* Any claim of direct access only is invalidated by | |
2962 | adding an indirection level. */ | |
2963 | if (spec.arg_direct_p (i)) | |
2964 | drop_fnspec = true; | |
2965 | ||
2966 | /* If there's a claim the argument is not read from, the | |
2967 | added indirection invalidates it: if the argument is | |
2968 | used at all, then the pointer will necessarily be | |
2969 | read. */ | |
2970 | if (!spec.arg_maybe_read_p (i) | |
2971 | && spec.arg_used_p (i)) | |
2972 | drop_fnspec = true; | |
2973 | } | |
2974 | } | |
2975 | } | |
2976 | ||
e4fc59e0 AO |
2977 | /* ??? Maybe we could adjust it instead. Note we don't need |
2978 | to mess with attribute access: pointer-typed parameters are | |
2979 | not modified, so they can remain unchanged. */ | |
f0a90c7d AO |
2980 | if (drop_fnspec) |
2981 | remove_named_attribute_unsharing ("fn spec", | |
2982 | &TYPE_ATTRIBUTES (nftype)); | |
2983 | ||
2984 | TREE_TYPE (nnode->decl) = nftype; | |
2985 | } | |
2986 | ||
2987 | #if ATTR_FNSPEC_DECONST_WATERMARK | |
2988 | { | |
2989 | int flags = flags_from_decl_or_type (nnode->decl); | |
2990 | tree fnspec = lookup_attribute ("fn spec", TREE_TYPE (nnode->decl)); | |
2991 | ||
2992 | if ((flags & (ECF_CONST | ECF_PURE | ECF_NOVOPS)) || fnspec) | |
2993 | { | |
2994 | size_t xargs = 1 + int (is_stdarg) + int (apply_args); | |
2995 | size_t curlen = 0, tgtlen = 2 + 2 * (named_args + xargs); | |
2996 | auto_vec<char> nspecv (tgtlen); | |
2997 | char *nspec = &nspecv[0]; /* It will *not* be NUL-terminated! */ | |
2998 | bool no_writes_p = true; | |
2999 | if (fnspec) | |
3000 | { | |
3001 | tree fnspecstr = TREE_VALUE (TREE_VALUE (fnspec)); | |
3002 | curlen = TREE_STRING_LENGTH (fnspecstr); | |
3003 | memcpy (nspec, TREE_STRING_POINTER (fnspecstr), curlen); | |
3004 | if (!(flags & (ECF_CONST | ECF_PURE | ECF_NOVOPS)) | |
3005 | && curlen >= 2 | |
3006 | && nspec[1] != 'c' && nspec[1] != 'C' | |
3007 | && nspec[1] != 'p' && nspec[1] != 'P') | |
3008 | no_writes_p = false; | |
3009 | } | |
3010 | if (!curlen) | |
3011 | { | |
3012 | nspec[curlen++] = '.'; | |
3013 | nspec[curlen++] = ((flags & ECF_CONST) | |
3014 | ? 'c' | |
3015 | : (flags & ECF_PURE) | |
3016 | ? 'p' | |
3017 | : ' '); | |
3018 | } | |
3019 | while (curlen < tgtlen - 2 * xargs) | |
3020 | { | |
3021 | nspec[curlen++] = '.'; | |
3022 | nspec[curlen++] = ' '; | |
3023 | } | |
3024 | ||
3025 | /* These extra args are unlikely to be present in const or pure | |
3026 | functions. It's conceivable that a function that takes variable | |
3027 | arguments, or that passes its arguments on to another function, | |
3028 | could be const or pure, but it would not modify the arguments, and, | |
3029 | being pure or const, it couldn't possibly modify or even access | |
3030 | memory referenced by them. But it can read from these internal | |
3031 | data structures created by the wrapper, and from any | |
3032 | argument-passing memory referenced by them, so we denote the | |
3033 | possibility of reading from multiple levels of indirection, but | |
3034 | only of reading because const/pure. */ | |
3035 | if (apply_args) | |
3036 | { | |
3037 | nspec[curlen++] = 'r'; | |
3038 | nspec[curlen++] = ' '; | |
3039 | } | |
3040 | if (is_stdarg) | |
3041 | { | |
3042 | nspec[curlen++] = (no_writes_p ? 'r' : '.'); | |
3043 | nspec[curlen++] = (no_writes_p ? 't' : ' '); | |
3044 | } | |
3045 | ||
3046 | nspec[curlen++] = 'W'; | |
3047 | nspec[curlen++] = 't'; | |
3048 | ||
3049 | /* The type has already been copied before adding parameters. */ | |
3050 | gcc_checking_assert (TYPE_ARG_TYPES (TREE_TYPE (nnode->decl)) | |
3051 | != TYPE_ARG_TYPES (TREE_TYPE (onode->decl))); | |
3052 | TYPE_ATTRIBUTES (TREE_TYPE (nnode->decl)) | |
3053 | = tree_cons (get_identifier ("fn spec"), | |
3054 | build_tree_list (NULL_TREE, | |
3055 | build_string (tgtlen, nspec)), | |
3056 | TYPE_ATTRIBUTES (TREE_TYPE (nnode->decl))); | |
3057 | } | |
3058 | } | |
3059 | #endif | |
3060 | ||
3061 | { | |
3062 | tree decl = onode->decl; | |
3063 | cgraph_node *target = nnode; | |
3064 | ||
3065 | { // copied from create_wrapper | |
3066 | ||
3067 | /* Preserve DECL_RESULT so we get right by reference flag. */ | |
3068 | tree decl_result = DECL_RESULT (decl); | |
3069 | ||
3070 | /* Remove the function's body but keep arguments to be reused | |
3071 | for thunk. */ | |
3072 | onode->release_body (true); | |
3073 | onode->reset (/* unlike create_wrapper: preserve_comdat_group = */true); | |
3074 | ||
3075 | DECL_UNINLINABLE (decl) = false; | |
3076 | DECL_RESULT (decl) = decl_result; | |
3077 | DECL_INITIAL (decl) = NULL; | |
3078 | allocate_struct_function (decl, false); | |
3079 | set_cfun (NULL); | |
3080 | ||
3081 | /* Turn alias into thunk and expand it into GIMPLE representation. */ | |
3082 | onode->definition = true; | |
3083 | ||
3084 | thunk_info::get_create (onode); | |
3085 | onode->thunk = true; | |
3086 | onode->create_edge (target, NULL, onode->count); | |
3087 | onode->callees->can_throw_external = !TREE_NOTHROW (target->decl); | |
3088 | ||
3089 | tree arguments = DECL_ARGUMENTS (decl); | |
3090 | ||
3091 | while (arguments) | |
3092 | { | |
3093 | TREE_ADDRESSABLE (arguments) = false; | |
3094 | arguments = TREE_CHAIN (arguments); | |
3095 | } | |
3096 | ||
3097 | { | |
3098 | tree alias = onode->callees->callee->decl; | |
3099 | tree thunk_fndecl = decl; | |
3100 | tree a; | |
3101 | ||
3102 | int nxargs = 1 + is_stdarg + apply_args; | |
3103 | ||
3104 | { // Simplified from expand_thunk. | |
3105 | tree restype; | |
3106 | basic_block bb, then_bb, else_bb, return_bb; | |
3107 | gimple_stmt_iterator bsi; | |
3108 | int nargs = 0; | |
3109 | tree arg; | |
3110 | int i; | |
3111 | tree resdecl; | |
3112 | tree restmp = NULL; | |
3113 | ||
3114 | gcall *call; | |
3115 | greturn *ret; | |
3116 | bool alias_is_noreturn = TREE_THIS_VOLATILE (alias); | |
3117 | ||
3118 | a = DECL_ARGUMENTS (thunk_fndecl); | |
3119 | ||
3120 | current_function_decl = thunk_fndecl; | |
3121 | ||
3122 | /* Ensure thunks are emitted in their correct sections. */ | |
3123 | resolve_unique_section (thunk_fndecl, 0, | |
3124 | flag_function_sections); | |
3125 | ||
3126 | bitmap_obstack_initialize (NULL); | |
3127 | ||
3128 | /* Build the return declaration for the function. */ | |
3129 | restype = TREE_TYPE (TREE_TYPE (thunk_fndecl)); | |
3130 | if (DECL_RESULT (thunk_fndecl) == NULL_TREE) | |
3131 | { | |
3132 | resdecl = build_decl (input_location, RESULT_DECL, 0, restype); | |
3133 | DECL_ARTIFICIAL (resdecl) = 1; | |
3134 | DECL_IGNORED_P (resdecl) = 1; | |
3135 | DECL_CONTEXT (resdecl) = thunk_fndecl; | |
3136 | DECL_RESULT (thunk_fndecl) = resdecl; | |
3137 | } | |
3138 | else | |
3139 | resdecl = DECL_RESULT (thunk_fndecl); | |
3140 | ||
3141 | profile_count cfg_count = onode->count; | |
3142 | if (!cfg_count.initialized_p ()) | |
3143 | cfg_count = profile_count::from_gcov_type (BB_FREQ_MAX).guessed_local (); | |
3144 | ||
3145 | bb = then_bb = else_bb = return_bb | |
3146 | = init_lowered_empty_function (thunk_fndecl, true, cfg_count); | |
3147 | ||
3148 | bsi = gsi_start_bb (bb); | |
3149 | ||
3150 | /* Build call to the function being thunked. */ | |
3151 | if (!VOID_TYPE_P (restype) | |
3152 | && (!alias_is_noreturn | |
3153 | || TREE_ADDRESSABLE (restype) | |
3154 | || TREE_CODE (TYPE_SIZE_UNIT (restype)) != INTEGER_CST)) | |
3155 | { | |
3156 | if (DECL_BY_REFERENCE (resdecl)) | |
3157 | { | |
3158 | restmp = gimple_fold_indirect_ref (resdecl); | |
3159 | if (!restmp) | |
3160 | restmp = build2 (MEM_REF, | |
3161 | TREE_TYPE (TREE_TYPE (resdecl)), | |
3162 | resdecl, | |
3163 | build_int_cst (TREE_TYPE (resdecl), 0)); | |
3164 | } | |
2c74d5c0 | 3165 | else if (aggregate_value_p (resdecl, TREE_TYPE (thunk_fndecl))) |
f0a90c7d | 3166 | { |
2c74d5c0 JJ |
3167 | restmp = resdecl; |
3168 | ||
3169 | if (VAR_P (restmp)) | |
f0a90c7d | 3170 | { |
2c74d5c0 JJ |
3171 | add_local_decl (cfun, restmp); |
3172 | BLOCK_VARS (DECL_INITIAL (current_function_decl)) | |
3173 | = restmp; | |
f0a90c7d | 3174 | } |
f0a90c7d AO |
3175 | } |
3176 | else | |
3177 | restmp = create_tmp_reg (restype, "retval"); | |
3178 | } | |
3179 | ||
3180 | for (arg = a; arg; arg = DECL_CHAIN (arg)) | |
3181 | nargs++; | |
3182 | auto_vec<tree> vargs (nargs + nxargs); | |
3183 | i = 0; | |
3184 | arg = a; | |
3185 | ||
3186 | if (nargs) | |
3187 | for (tree nparm = DECL_ARGUMENTS (nnode->decl); | |
3188 | i < nargs; | |
3189 | i++, arg = DECL_CHAIN (arg), nparm = DECL_CHAIN (nparm)) | |
3190 | { | |
3191 | tree save_arg = arg; | |
f0a90c7d AO |
3192 | |
3193 | /* Arrange to pass indirectly the parms, if we decided to do | |
3194 | so, and revert its type in the wrapper. */ | |
3195 | if (indirect_nparms.contains (nparm)) | |
3196 | { | |
3197 | tree ref_type = TREE_TYPE (nparm); | |
3198 | TREE_ADDRESSABLE (arg) = true; | |
37afeec8 | 3199 | arg = build1 (ADDR_EXPR, ref_type, arg); |
f0a90c7d | 3200 | } |
37afeec8 | 3201 | else if (!TREE_THIS_VOLATILE (arg)) |
f0a90c7d AO |
3202 | DECL_NOT_GIMPLE_REG_P (arg) = 0; |
3203 | ||
3204 | /* Convert the argument back to the type used by the calling | |
3205 | conventions, e.g. a non-prototyped float type is passed as | |
3206 | double, as in 930603-1.c, and needs to be converted back to | |
3207 | double to be passed on unchanged to the wrapped | |
3208 | function. */ | |
3209 | if (TREE_TYPE (nparm) != DECL_ARG_TYPE (nparm)) | |
37afeec8 AO |
3210 | { |
3211 | tree tmp = arg; | |
3212 | /* If ARG is e.g. volatile, we must copy and | |
e1e71b4e | 3213 | convert in separate statements. */ |
37afeec8 AO |
3214 | if (!is_gimple_val (arg)) |
3215 | { | |
3216 | tmp = create_tmp_reg (TYPE_MAIN_VARIANT | |
3217 | (TREE_TYPE (arg)), "arg"); | |
3218 | gimple *stmt = gimple_build_assign (tmp, arg); | |
3219 | gsi_insert_after (&bsi, stmt, GSI_NEW_STMT); | |
3220 | } | |
3221 | arg = fold_convert (DECL_ARG_TYPE (nparm), tmp); | |
3222 | } | |
f0a90c7d AO |
3223 | |
3224 | if (!is_gimple_val (arg)) | |
3225 | { | |
37afeec8 AO |
3226 | tree tmp = create_tmp_reg (TYPE_MAIN_VARIANT |
3227 | (TREE_TYPE (arg)), "arg"); | |
f0a90c7d AO |
3228 | gimple *stmt = gimple_build_assign (tmp, arg); |
3229 | gsi_insert_after (&bsi, stmt, GSI_NEW_STMT); | |
37afeec8 | 3230 | arg = tmp; |
f0a90c7d | 3231 | } |
37afeec8 | 3232 | vargs.quick_push (arg); |
f0a90c7d AO |
3233 | arg = save_arg; |
3234 | } | |
3235 | /* These strub arguments are adjusted later. */ | |
3236 | if (apply_args) | |
3237 | vargs.quick_push (null_pointer_node); | |
3238 | if (is_stdarg) | |
3239 | vargs.quick_push (null_pointer_node); | |
3240 | vargs.quick_push (null_pointer_node); | |
3241 | call = gimple_build_call_vec (build_fold_addr_expr_loc (0, alias), | |
3242 | vargs); | |
3243 | onode->callees->call_stmt = call; | |
3244 | // gimple_call_set_from_thunk (call, true); | |
3245 | if (DECL_STATIC_CHAIN (alias)) | |
3246 | { | |
3247 | tree p = DECL_STRUCT_FUNCTION (alias)->static_chain_decl; | |
3248 | tree type = TREE_TYPE (p); | |
3249 | tree decl = build_decl (DECL_SOURCE_LOCATION (thunk_fndecl), | |
3250 | PARM_DECL, create_tmp_var_name ("CHAIN"), | |
3251 | type); | |
3252 | DECL_ARTIFICIAL (decl) = 1; | |
3253 | DECL_IGNORED_P (decl) = 1; | |
3254 | TREE_USED (decl) = 1; | |
3255 | DECL_CONTEXT (decl) = thunk_fndecl; | |
3256 | DECL_ARG_TYPE (decl) = type; | |
3257 | TREE_READONLY (decl) = 1; | |
3258 | ||
3259 | struct function *sf = DECL_STRUCT_FUNCTION (thunk_fndecl); | |
3260 | sf->static_chain_decl = decl; | |
3261 | ||
3262 | gimple_call_set_chain (call, decl); | |
3263 | } | |
3264 | ||
3265 | /* Return slot optimization is always possible and in fact required to | |
3266 | return values with DECL_BY_REFERENCE. */ | |
3267 | if (aggregate_value_p (resdecl, TREE_TYPE (thunk_fndecl)) | |
3268 | && (!is_gimple_reg_type (TREE_TYPE (resdecl)) | |
3269 | || DECL_BY_REFERENCE (resdecl))) | |
3270 | gimple_call_set_return_slot_opt (call, true); | |
3271 | ||
3272 | if (restmp) | |
3273 | { | |
3274 | gimple_call_set_lhs (call, restmp); | |
3275 | gcc_assert (useless_type_conversion_p (TREE_TYPE (restmp), | |
3276 | TREE_TYPE (TREE_TYPE (alias)))); | |
3277 | } | |
3278 | gsi_insert_after (&bsi, call, GSI_NEW_STMT); | |
3279 | if (!alias_is_noreturn) | |
3280 | { | |
3281 | /* Build return value. */ | |
3282 | if (!DECL_BY_REFERENCE (resdecl)) | |
3283 | ret = gimple_build_return (restmp); | |
3284 | else | |
3285 | ret = gimple_build_return (resdecl); | |
3286 | ||
3287 | gsi_insert_after (&bsi, ret, GSI_NEW_STMT); | |
3288 | } | |
3289 | else | |
3290 | { | |
3291 | remove_edge (single_succ_edge (bb)); | |
3292 | } | |
3293 | ||
3294 | cfun->gimple_df->in_ssa_p = true; | |
3295 | update_max_bb_count (); | |
3296 | profile_status_for_fn (cfun) | |
3297 | = cfg_count.initialized_p () && cfg_count.ipa_p () | |
3298 | ? PROFILE_READ : PROFILE_GUESSED; | |
3299 | /* FIXME: C++ FE should stop setting TREE_ASM_WRITTEN on thunks. */ | |
3300 | // TREE_ASM_WRITTEN (thunk_fndecl) = false; | |
3301 | delete_unreachable_blocks (); | |
3302 | update_ssa (TODO_update_ssa); | |
3303 | checking_verify_flow_info (); | |
3304 | free_dominance_info (CDI_DOMINATORS); | |
3305 | ||
3306 | /* Since we want to emit the thunk, we explicitly mark its name as | |
3307 | referenced. */ | |
3308 | onode->thunk = false; | |
3309 | onode->lowered = true; | |
3310 | bitmap_obstack_release (NULL); | |
3311 | } | |
3312 | current_function_decl = NULL; | |
3313 | set_cfun (NULL); | |
3314 | } | |
3315 | ||
3316 | thunk_info::remove (onode); | |
3317 | ||
3318 | // some more of create_wrapper at the end of the next block. | |
3319 | } | |
3320 | } | |
3321 | ||
3322 | { | |
3323 | tree aaval = NULL_TREE; | |
3324 | tree vaptr = NULL_TREE; | |
3325 | tree wmptr = NULL_TREE; | |
3326 | for (tree arg = DECL_ARGUMENTS (nnode->decl); arg; arg = DECL_CHAIN (arg)) | |
3327 | { | |
3328 | aaval = vaptr; | |
3329 | vaptr = wmptr; | |
3330 | wmptr = arg; | |
3331 | } | |
3332 | ||
3333 | if (!apply_args) | |
3334 | aaval = NULL_TREE; | |
3335 | /* The trailing args are [apply_args], [va_list_ptr], and | |
3336 | watermark. If we don't have a va_list_ptr, the penultimate | |
3337 | argument is apply_args. | |
3338 | */ | |
3339 | else if (!is_stdarg) | |
3340 | aaval = vaptr; | |
3341 | ||
3342 | if (!is_stdarg) | |
3343 | vaptr = NULL_TREE; | |
3344 | ||
3345 | DECL_NAME (wmptr) = get_watermark_ptr (); | |
3346 | DECL_ARTIFICIAL (wmptr) = 1; | |
3347 | DECL_IGNORED_P (wmptr) = 1; | |
3348 | TREE_USED (wmptr) = 1; | |
3349 | ||
3350 | if (is_stdarg) | |
3351 | { | |
3352 | DECL_NAME (vaptr) = get_va_list_ptr (); | |
3353 | DECL_ARTIFICIAL (vaptr) = 1; | |
3354 | DECL_IGNORED_P (vaptr) = 1; | |
3355 | TREE_USED (vaptr) = 1; | |
3356 | } | |
3357 | ||
3358 | if (apply_args) | |
3359 | { | |
3360 | DECL_NAME (aaval) = get_apply_args (); | |
3361 | DECL_ARTIFICIAL (aaval) = 1; | |
3362 | DECL_IGNORED_P (aaval) = 1; | |
3363 | TREE_USED (aaval) = 1; | |
3364 | } | |
3365 | ||
3366 | push_cfun (DECL_STRUCT_FUNCTION (nnode->decl)); | |
3367 | ||
3368 | { | |
3369 | edge e = single_succ_edge (ENTRY_BLOCK_PTR_FOR_FN (cfun)); | |
3370 | gimple_seq seq = call_update_watermark (wmptr, nnode, e->src->count); | |
3371 | gsi_insert_seq_on_edge_immediate (e, seq); | |
3372 | } | |
3373 | ||
3374 | bool any_indirect = !indirect_nparms.is_empty (); | |
3375 | ||
3376 | if (any_indirect) | |
3377 | { | |
3378 | basic_block bb; | |
3379 | bool needs_commit = false; | |
3380 | FOR_EACH_BB_FN (bb, cfun) | |
3381 | { | |
3382 | for (gphi_iterator gsi = gsi_start_nonvirtual_phis (bb); | |
3383 | !gsi_end_p (gsi); | |
3384 | gsi_next_nonvirtual_phi (&gsi)) | |
3385 | { | |
3386 | gphi *stmt = gsi.phi (); | |
3387 | ||
3388 | walk_stmt_info wi = {}; | |
3389 | wi.info = &indirect_nparms; | |
3390 | walk_gimple_op (stmt, walk_make_indirect, &wi); | |
3391 | if (wi.changed && !is_gimple_debug (gsi_stmt (gsi))) | |
3392 | if (walk_regimplify_phi (stmt)) | |
3393 | needs_commit = true; | |
3394 | } | |
3395 | ||
3396 | for (gimple_stmt_iterator gsi = gsi_start_bb (bb); | |
3397 | !gsi_end_p (gsi); gsi_next (&gsi)) | |
3398 | { | |
3399 | gimple *stmt = gsi_stmt (gsi); | |
3400 | ||
3401 | walk_stmt_info wi = {}; | |
3402 | wi.info = &indirect_nparms; | |
3403 | walk_gimple_op (stmt, walk_make_indirect, &wi); | |
3404 | if (wi.changed) | |
3405 | { | |
3406 | if (!is_gimple_debug (stmt)) | |
3407 | { | |
3408 | wi.info = &gsi; | |
3409 | walk_gimple_op (stmt, walk_regimplify_addr_expr, | |
3410 | &wi); | |
3411 | } | |
3412 | update_stmt (stmt); | |
3413 | } | |
3414 | } | |
3415 | } | |
3416 | if (needs_commit) | |
3417 | gsi_commit_edge_inserts (); | |
3418 | } | |
3419 | ||
3420 | if (DECL_STRUCT_FUNCTION (nnode->decl)->calls_alloca | |
3421 | || is_stdarg || apply_args) | |
3422 | for (cgraph_edge *e = nnode->callees, *enext; e; e = enext) | |
3423 | { | |
3424 | if (!e->call_stmt) | |
3425 | continue; | |
3426 | ||
3427 | gcall *call = e->call_stmt; | |
3428 | gimple_stmt_iterator gsi = gsi_for_stmt (call); | |
3429 | tree fndecl = e->callee->decl; | |
3430 | ||
3431 | enext = e->next_callee; | |
3432 | ||
3433 | if (gimple_alloca_call_p (call)) | |
3434 | { | |
3435 | gimple_seq seq = call_update_watermark (wmptr, NULL, | |
3436 | gsi_bb (gsi)->count); | |
3437 | gsi_insert_finally_seq_after_call (gsi, seq); | |
3438 | } | |
3439 | else if (fndecl && is_stdarg | |
3440 | && fndecl_built_in_p (fndecl, BUILT_IN_VA_START)) | |
3441 | { | |
3442 | /* Using a non-default stdarg ABI makes the function ineligible | |
3443 | for internal strub. */ | |
3444 | gcc_checking_assert (builtin_decl_explicit (BUILT_IN_VA_START) | |
3445 | == fndecl); | |
3446 | tree bvacopy = builtin_decl_explicit (BUILT_IN_VA_COPY); | |
3447 | gimple_call_set_fndecl (call, bvacopy); | |
3448 | tree arg = vaptr; | |
3449 | /* The va_copy source must be dereferenced, unless it's an array | |
3450 | type, that would have decayed to a pointer. */ | |
3451 | if (TREE_CODE (TREE_TYPE (TREE_TYPE (vaptr))) != ARRAY_TYPE) | |
3452 | { | |
3453 | arg = gimple_fold_indirect_ref (vaptr); | |
3454 | if (!arg) | |
3455 | arg = build2 (MEM_REF, | |
3456 | TREE_TYPE (TREE_TYPE (vaptr)), | |
3457 | vaptr, | |
3458 | build_int_cst (TREE_TYPE (vaptr), 0)); | |
3459 | if (!is_gimple_val (arg)) | |
3460 | arg = force_gimple_operand_gsi (&gsi, arg, true, | |
3461 | NULL_TREE, true, GSI_SAME_STMT); | |
3462 | } | |
3463 | gimple_call_set_arg (call, 1, arg); | |
3464 | update_stmt (call); | |
3465 | e->redirect_callee (cgraph_node::get_create (bvacopy)); | |
3466 | } | |
3467 | else if (fndecl && apply_args | |
3468 | && fndecl_built_in_p (fndecl, BUILT_IN_APPLY_ARGS)) | |
3469 | { | |
3470 | tree lhs = gimple_call_lhs (call); | |
3471 | gimple *assign = (lhs | |
3472 | ? gimple_build_assign (lhs, aaval) | |
3473 | : gimple_build_nop ()); | |
3474 | gsi_replace (&gsi, assign, true); | |
3475 | cgraph_edge::remove (e); | |
3476 | } | |
3477 | } | |
3478 | ||
3479 | { // a little more copied from create_wrapper | |
3480 | ||
3481 | /* Inline summary set-up. */ | |
3482 | nnode->analyze (); | |
3483 | // inline_analyze_function (nnode); | |
3484 | } | |
3485 | ||
3486 | pop_cfun (); | |
3487 | } | |
3488 | ||
3489 | { | |
3490 | push_cfun (DECL_STRUCT_FUNCTION (onode->decl)); | |
3491 | gimple_stmt_iterator gsi | |
3492 | = gsi_after_labels (single_succ (ENTRY_BLOCK_PTR_FOR_FN (cfun))); | |
3493 | ||
3494 | gcall *wrcall; | |
3495 | while (!(wrcall = dyn_cast <gcall *> (gsi_stmt (gsi)))) | |
3496 | gsi_next (&gsi); | |
3497 | ||
3498 | tree swm = create_tmp_var (get_wmt (), ".strub.watermark"); | |
3499 | TREE_ADDRESSABLE (swm) = true; | |
3500 | tree swmp = build1 (ADDR_EXPR, get_pwmt (), swm); | |
3501 | ||
3502 | tree enter = get_enter (); | |
3503 | gcall *stptr = gimple_build_call (enter, 1, unshare_expr (swmp)); | |
3504 | gimple_set_location (stptr, gimple_location (wrcall)); | |
3505 | gsi_insert_before (&gsi, stptr, GSI_SAME_STMT); | |
3506 | onode->create_edge (cgraph_node::get_create (enter), | |
3507 | stptr, gsi_bb (gsi)->count, false); | |
3508 | ||
3509 | int nargs = gimple_call_num_args (wrcall); | |
3510 | ||
3511 | gimple_seq seq = NULL; | |
3512 | ||
3513 | if (apply_args) | |
3514 | { | |
3515 | tree aalst = create_tmp_var (ptr_type_node, ".strub.apply_args"); | |
3516 | tree bappargs = builtin_decl_explicit (BUILT_IN_APPLY_ARGS); | |
3517 | gcall *appargs = gimple_build_call (bappargs, 0); | |
3518 | gimple_call_set_lhs (appargs, aalst); | |
3519 | gimple_set_location (appargs, gimple_location (wrcall)); | |
3520 | gsi_insert_before (&gsi, appargs, GSI_SAME_STMT); | |
3521 | gimple_call_set_arg (wrcall, nargs - 2 - is_stdarg, aalst); | |
3522 | onode->create_edge (cgraph_node::get_create (bappargs), | |
3523 | appargs, gsi_bb (gsi)->count, false); | |
3524 | } | |
3525 | ||
3526 | if (is_stdarg) | |
3527 | { | |
3528 | tree valst = create_tmp_var (va_list_type_node, ".strub.va_list"); | |
3529 | TREE_ADDRESSABLE (valst) = true; | |
3530 | tree vaptr = build1 (ADDR_EXPR, | |
3531 | build_pointer_type (va_list_type_node), | |
3532 | valst); | |
3533 | gimple_call_set_arg (wrcall, nargs - 2, unshare_expr (vaptr)); | |
3534 | ||
3535 | tree bvastart = builtin_decl_explicit (BUILT_IN_VA_START); | |
3536 | gcall *vastart = gimple_build_call (bvastart, 2, | |
3537 | unshare_expr (vaptr), | |
3538 | integer_zero_node); | |
3539 | gimple_set_location (vastart, gimple_location (wrcall)); | |
3540 | gsi_insert_before (&gsi, vastart, GSI_SAME_STMT); | |
3541 | onode->create_edge (cgraph_node::get_create (bvastart), | |
3542 | vastart, gsi_bb (gsi)->count, false); | |
3543 | ||
3544 | tree bvaend = builtin_decl_explicit (BUILT_IN_VA_END); | |
3545 | gcall *vaend = gimple_build_call (bvaend, 1, unshare_expr (vaptr)); | |
3546 | gimple_set_location (vaend, gimple_location (wrcall)); | |
3547 | gimple_seq_add_stmt (&seq, vaend); | |
3548 | } | |
3549 | ||
3550 | gimple_call_set_arg (wrcall, nargs - 1, unshare_expr (swmp)); | |
3551 | // gimple_call_set_tail (wrcall, false); | |
3552 | update_stmt (wrcall); | |
3553 | ||
3554 | { | |
3555 | #if !ATTR_FNSPEC_DECONST_WATERMARK | |
3556 | /* If the call will be assumed to not modify or even read the | |
3557 | watermark, make it read and modified ourselves. */ | |
3558 | if ((gimple_call_flags (wrcall) | |
3559 | & (ECF_CONST | ECF_PURE | ECF_NOVOPS))) | |
3560 | { | |
3561 | vec<tree, va_gc> *inputs = NULL; | |
3562 | vec<tree, va_gc> *outputs = NULL; | |
3563 | vec_safe_push (outputs, | |
3564 | build_tree_list | |
3565 | (build_tree_list | |
3566 | (NULL_TREE, build_string (2, "=m")), | |
3567 | swm)); | |
3568 | vec_safe_push (inputs, | |
3569 | build_tree_list | |
3570 | (build_tree_list | |
3571 | (NULL_TREE, build_string (1, "m")), | |
3572 | swm)); | |
3573 | gasm *forcemod = gimple_build_asm_vec ("", inputs, outputs, | |
3574 | NULL, NULL); | |
3575 | gimple_seq_add_stmt (&seq, forcemod); | |
3576 | ||
3577 | /* If the call will be assumed to not even read the watermark, | |
3578 | make sure it is already in memory before the call. */ | |
3579 | if ((gimple_call_flags (wrcall) & ECF_CONST)) | |
3580 | { | |
3581 | vec<tree, va_gc> *inputs = NULL; | |
3582 | vec_safe_push (inputs, | |
3583 | build_tree_list | |
3584 | (build_tree_list | |
3585 | (NULL_TREE, build_string (1, "m")), | |
3586 | swm)); | |
3587 | gasm *force_store = gimple_build_asm_vec ("", inputs, NULL, | |
3588 | NULL, NULL); | |
3589 | gimple_set_location (force_store, gimple_location (wrcall)); | |
3590 | gsi_insert_before (&gsi, force_store, GSI_SAME_STMT); | |
3591 | } | |
3592 | } | |
3593 | #endif | |
3594 | ||
3595 | gcall *sleave = gimple_build_call (get_leave (), 1, | |
3596 | unshare_expr (swmp)); | |
3597 | gimple_seq_add_stmt (&seq, sleave); | |
3598 | ||
3599 | gassign *clobber = gimple_build_assign (swm, | |
3600 | build_clobber | |
3601 | (TREE_TYPE (swm))); | |
3602 | gimple_seq_add_stmt (&seq, clobber); | |
3603 | } | |
3604 | ||
3605 | gsi_insert_finally_seq_after_call (gsi, seq); | |
3606 | ||
3607 | /* For nnode, we don't rebuild edges because we wish to retain | |
3608 | any redirections copied to it from earlier passes, so we add | |
3609 | call graph edges explicitly there, but for onode, we create a | |
3610 | fresh function, so we may as well just issue the calls and | |
3611 | then rebuild all cgraph edges. */ | |
3612 | // cgraph_edge::rebuild_edges (); | |
3613 | onode->analyze (); | |
3614 | // inline_analyze_function (onode); | |
3615 | ||
3616 | pop_cfun (); | |
3617 | } | |
3618 | } | |
3619 | ||
3620 | return 0; | |
3621 | } | |
3622 | ||
3623 | simple_ipa_opt_pass * | |
3624 | make_pass_ipa_strub (gcc::context *ctxt) | |
3625 | { | |
3626 | return new pass_ipa_strub (ctxt); | |
3627 | } | |
3628 | ||
3629 | #include "gt-ipa-strub.h" |