]>
Commit | Line | Data |
---|---|---|
65c1a668 | 1 | /* Inlining decision heuristics. |
071bc3bd | 2 | Copyright (C) 2003, 2004, 2007, 2008, 2009, 2010, 2011 |
7cf0dbf3 | 3 | Free Software Foundation, Inc. |
65c1a668 | 4 | Contributed by Jan Hubicka |
5 | ||
6 | This file is part of GCC. | |
7 | ||
8 | GCC is free software; you can redistribute it and/or modify it under | |
9 | the terms of the GNU General Public License as published by the Free | |
8c4c00c1 | 10 | Software Foundation; either version 3, or (at your option) any later |
65c1a668 | 11 | version. |
12 | ||
13 | GCC is distributed in the hope that it will be useful, but WITHOUT ANY | |
14 | WARRANTY; without even the implied warranty of MERCHANTABILITY or | |
15 | FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License | |
16 | for more details. | |
17 | ||
18 | You should have received a copy of the GNU General Public License | |
8c4c00c1 | 19 | along with GCC; see the file COPYING3. If not see |
20 | <http://www.gnu.org/licenses/>. */ | |
65c1a668 | 21 | |
22 | /* Inlining decision heuristics | |
23 | ||
4869c23f | 24 | The implementation of inliner is organized as follows: |
65c1a668 | 25 | |
65c1a668 | 26 | inlining heuristics limits |
27 | ||
4869c23f | 28 | can_inline_edge_p allow to check that particular inlining is allowed |
29 | by the limits specified by user (allowed function growth, growth and so | |
30 | on). | |
31 | ||
32 | Functions are inlined when it is obvious the result is profitable (such | |
33 | as functions called once or when inlining reduce code size). | |
34 | In addition to that we perform inlining of small functions and recursive | |
35 | inlining. | |
65c1a668 | 36 | |
37 | inlining heuristics | |
38 | ||
4869c23f | 39 | The inliner itself is split into two passes: |
40 | ||
41 | pass_early_inlining | |
65c1a668 | 42 | |
4869c23f | 43 | Simple local inlining pass inlining callees into current function. |
44 | This pass makes no use of whole unit analysis and thus it can do only | |
45 | very simple decisions based on local properties. | |
65c1a668 | 46 | |
4869c23f | 47 | The strength of the pass is that it is run in topological order |
48 | (reverse postorder) on the callgraph. Functions are converted into SSA | |
49 | form just before this pass and optimized subsequently. As a result, the | |
50 | callees of the function seen by the early inliner was already optimized | |
4055a556 | 51 | and results of early inlining adds a lot of optimization opportunities |
4869c23f | 52 | for the local optimization. |
65c1a668 | 53 | |
4055a556 | 54 | The pass handle the obvious inlining decisions within the compilation |
4869c23f | 55 | unit - inlining auto inline functions, inlining for size and |
56 | flattening. | |
65c1a668 | 57 | |
4869c23f | 58 | main strength of the pass is the ability to eliminate abstraction |
59 | penalty in C++ code (via combination of inlining and early | |
60 | optimization) and thus improve quality of analysis done by real IPA | |
61 | optimizers. | |
09a2e412 | 62 | |
4869c23f | 63 | Because of lack of whole unit knowledge, the pass can not really make |
64 | good code size/performance tradeoffs. It however does very simple | |
65 | speculative inlining allowing code size to grow by | |
4055a556 | 66 | EARLY_INLINING_INSNS when callee is leaf function. In this case the |
67 | optimizations performed later are very likely to eliminate the cost. | |
09a2e412 | 68 | |
4869c23f | 69 | pass_ipa_inline |
09a2e412 | 70 | |
4869c23f | 71 | This is the real inliner able to handle inlining with whole program |
72 | knowledge. It performs following steps: | |
09a2e412 | 73 | |
4869c23f | 74 | 1) inlining of small functions. This is implemented by greedy |
75 | algorithm ordering all inlinable cgraph edges by their badness and | |
76 | inlining them in this order as long as inline limits allows doing so. | |
09a2e412 | 77 | |
4869c23f | 78 | This heuristics is not very good on inlining recursive calls. Recursive |
79 | calls can be inlined with results similar to loop unrolling. To do so, | |
80 | special purpose recursive inliner is executed on function when | |
81 | recursive edge is met as viable candidate. | |
09a2e412 | 82 | |
4869c23f | 83 | 2) Unreachable functions are removed from callgraph. Inlining leads |
84 | to devirtualization and other modification of callgraph so functions | |
85 | may become unreachable during the process. Also functions declared as | |
86 | extern inline or virtual functions are removed, since after inlining | |
87 | we no longer need the offline bodies. | |
88 | ||
89 | 3) Functions called once and not exported from the unit are inlined. | |
90 | This should almost always lead to reduction of code size by eliminating | |
91 | the need for offline copy of the function. */ | |
65c1a668 | 92 | |
93 | #include "config.h" | |
94 | #include "system.h" | |
95 | #include "coretypes.h" | |
96 | #include "tm.h" | |
97 | #include "tree.h" | |
98 | #include "tree-inline.h" | |
99 | #include "langhooks.h" | |
100 | #include "flags.h" | |
101 | #include "cgraph.h" | |
102 | #include "diagnostic.h" | |
ce084dfc | 103 | #include "gimple-pretty-print.h" |
65c1a668 | 104 | #include "timevar.h" |
105 | #include "params.h" | |
106 | #include "fibheap.h" | |
107 | #include "intl.h" | |
108 | #include "tree-pass.h" | |
a49506c7 | 109 | #include "coverage.h" |
9e0baf4d | 110 | #include "ggc.h" |
4ae20857 | 111 | #include "rtl.h" |
4869c23f | 112 | #include "tree-flow.h" |
f8daee9b | 113 | #include "ipa-prop.h" |
97343302 | 114 | #include "except.h" |
4869c23f | 115 | #include "target.h" |
99c67f24 | 116 | #include "ipa-inline.h" |
7771d558 | 117 | #include "ipa-utils.h" |
97343302 | 118 | |
65c1a668 | 119 | /* Statistics we collect about inlining algorithm. */ |
97343302 | 120 | static int overall_size; |
a41f2a28 | 121 | static gcov_type max_count; |
65c1a668 | 122 | |
4869c23f | 123 | /* Return false when inlining edge E would lead to violating |
124 | limits on function unit growth or stack usage growth. | |
125 | ||
126 | The relative function body growth limit is present generally | |
4055a556 | 127 | to avoid problems with non-linear behavior of the compiler. |
4869c23f | 128 | To allow inlining huge functions into tiny wrapper, the limit |
129 | is always based on the bigger of the two functions considered. | |
130 | ||
131 | For stack growth limits we always base the growth in stack usage | |
132 | of the callers. We want to prevent applications from segfaulting | |
133 | on stack overflow when functions with huge stack frames gets | |
134 | inlined. */ | |
65c1a668 | 135 | |
136 | static bool | |
4869c23f | 137 | caller_growth_limits (struct cgraph_edge *e) |
65c1a668 | 138 | { |
17c205c9 | 139 | struct cgraph_node *to = e->caller; |
82626cb0 | 140 | struct cgraph_node *what = cgraph_function_or_thunk_node (e->callee, NULL); |
65c1a668 | 141 | int newsize; |
4869c23f | 142 | int limit = 0; |
143 | HOST_WIDE_INT stack_size_limit = 0, inlined_stack; | |
144 | struct inline_summary *info, *what_info, *outer_info = inline_summary (to); | |
145 | ||
146 | /* Look for function e->caller is inlined to. While doing | |
147 | so work out the largest function body on the way. As | |
148 | described above, we want to base our function growth | |
149 | limits based on that. Not on the self size of the | |
150 | outer function, not on the self size of inline code | |
151 | we immediately inline to. This is the most relaxed | |
152 | interpretation of the rule "do not grow large functions | |
153 | too much in order to prevent compiler from exploding". */ | |
0a0ca4d6 | 154 | while (true) |
4869c23f | 155 | { |
156 | info = inline_summary (to); | |
157 | if (limit < info->self_size) | |
158 | limit = info->self_size; | |
159 | if (stack_size_limit < info->estimated_self_stack_size) | |
160 | stack_size_limit = info->estimated_self_stack_size; | |
161 | if (to->global.inlined_to) | |
162 | to = to->callers->caller; | |
0a0ca4d6 | 163 | else |
164 | break; | |
4869c23f | 165 | } |
4b4d4c92 | 166 | |
cbd7f5a0 | 167 | what_info = inline_summary (what); |
168 | ||
4869c23f | 169 | if (limit < what_info->self_size) |
cbd7f5a0 | 170 | limit = what_info->self_size; |
65c1a668 | 171 | |
172 | limit += limit * PARAM_VALUE (PARAM_LARGE_FUNCTION_GROWTH) / 100; | |
173 | ||
4b4d4c92 | 174 | /* Check the size after inlining against the function limits. But allow |
175 | the function to shrink if it went over the limits by forced inlining. */ | |
99c67f24 | 176 | newsize = estimate_size_after_inlining (to, e); |
cbd7f5a0 | 177 | if (newsize >= info->size |
4b4d4c92 | 178 | && newsize > PARAM_VALUE (PARAM_LARGE_FUNCTION_INSNS) |
65c1a668 | 179 | && newsize > limit) |
180 | { | |
4869c23f | 181 | e->inline_failed = CIF_LARGE_FUNCTION_GROWTH_LIMIT; |
65c1a668 | 182 | return false; |
183 | } | |
5a02d67b | 184 | |
0a0ca4d6 | 185 | if (!what_info->estimated_stack_size) |
186 | return true; | |
187 | ||
4055a556 | 188 | /* FIXME: Stack size limit often prevents inlining in Fortran programs |
189 | due to large i/o datastructures used by the Fortran front-end. | |
4869c23f | 190 | We ought to ignore this limit when we know that the edge is executed |
191 | on every invocation of the caller (i.e. its call statement dominates | |
192 | exit block). We do not track this information, yet. */ | |
0a0ca4d6 | 193 | stack_size_limit += ((gcov_type)stack_size_limit |
4869c23f | 194 | * PARAM_VALUE (PARAM_STACK_FRAME_GROWTH) / 100); |
5a02d67b | 195 | |
4869c23f | 196 | inlined_stack = (outer_info->stack_frame_offset |
197 | + outer_info->estimated_self_stack_size | |
cbd7f5a0 | 198 | + what_info->estimated_stack_size); |
4869c23f | 199 | /* Check new stack consumption with stack consumption at the place |
200 | stack is used. */ | |
201 | if (inlined_stack > stack_size_limit | |
4055a556 | 202 | /* If function already has large stack usage from sibling |
4869c23f | 203 | inline call, we can inline, too. |
204 | This bit overoptimistically assume that we are good at stack | |
205 | packing. */ | |
206 | && inlined_stack > info->estimated_stack_size | |
5a02d67b | 207 | && inlined_stack > PARAM_VALUE (PARAM_LARGE_STACK_FRAME)) |
208 | { | |
4869c23f | 209 | e->inline_failed = CIF_LARGE_STACK_FRAME_GROWTH_LIMIT; |
5a02d67b | 210 | return false; |
211 | } | |
65c1a668 | 212 | return true; |
213 | } | |
214 | ||
4869c23f | 215 | /* Dump info about why inlining has failed. */ |
216 | ||
217 | static void | |
218 | report_inline_failed_reason (struct cgraph_edge *e) | |
219 | { | |
220 | if (dump_file) | |
221 | { | |
222 | fprintf (dump_file, " not inlinable: %s/%i -> %s/%i, %s\n", | |
223 | cgraph_node_name (e->caller), e->caller->uid, | |
224 | cgraph_node_name (e->callee), e->callee->uid, | |
225 | cgraph_inline_failed_string (e->inline_failed)); | |
226 | } | |
227 | } | |
228 | ||
229 | /* Decide if we can inline the edge and possibly update | |
230 | inline_failed reason. | |
231 | We check whether inlining is possible at all and whether | |
232 | caller growth limits allow doing so. | |
233 | ||
234 | if REPORT is true, output reason to the dump file. */ | |
65c1a668 | 235 | |
326a9581 | 236 | static bool |
4869c23f | 237 | can_inline_edge_p (struct cgraph_edge *e, bool report) |
65c1a668 | 238 | { |
4869c23f | 239 | bool inlinable = true; |
82626cb0 | 240 | enum availability avail; |
69d925d0 | 241 | struct cgraph_node *callee |
242 | = cgraph_function_or_thunk_node (e->callee, &avail); | |
7d0d0ce1 | 243 | tree caller_tree = DECL_FUNCTION_SPECIFIC_OPTIMIZATION (e->caller->symbol.decl); |
69d925d0 | 244 | tree callee_tree |
7d0d0ce1 | 245 | = callee ? DECL_FUNCTION_SPECIFIC_OPTIMIZATION (callee->symbol.decl) : NULL; |
246 | struct function *caller_cfun = DECL_STRUCT_FUNCTION (e->caller->symbol.decl); | |
69d925d0 | 247 | struct function *callee_cfun |
7d0d0ce1 | 248 | = callee ? DECL_STRUCT_FUNCTION (callee->symbol.decl) : NULL; |
69d925d0 | 249 | |
250 | if (!caller_cfun && e->caller->clone_of) | |
7d0d0ce1 | 251 | caller_cfun = DECL_STRUCT_FUNCTION (e->caller->clone_of->symbol.decl); |
69d925d0 | 252 | |
253 | if (!callee_cfun && callee && callee->clone_of) | |
7d0d0ce1 | 254 | callee_cfun = DECL_STRUCT_FUNCTION (callee->clone_of->symbol.decl); |
469679ab | 255 | |
4869c23f | 256 | gcc_assert (e->inline_failed); |
d160af41 | 257 | |
82626cb0 | 258 | if (!callee || !callee->analyzed) |
4869c23f | 259 | { |
260 | e->inline_failed = CIF_BODY_NOT_AVAILABLE; | |
261 | inlinable = false; | |
262 | } | |
82626cb0 | 263 | else if (!inline_summary (callee)->inlinable) |
4869c23f | 264 | { |
265 | e->inline_failed = CIF_FUNCTION_NOT_INLINABLE; | |
266 | inlinable = false; | |
267 | } | |
82626cb0 | 268 | else if (avail <= AVAIL_OVERWRITABLE) |
b30512dd | 269 | { |
4869c23f | 270 | e->inline_failed = CIF_OVERWRITABLE; |
b30512dd | 271 | return false; |
272 | } | |
f883da84 | 273 | else if (e->call_stmt_cannot_inline_p) |
4869c23f | 274 | { |
275 | e->inline_failed = CIF_MISMATCHED_ARGUMENTS; | |
276 | inlinable = false; | |
277 | } | |
278 | /* Don't inline if the functions have different EH personalities. */ | |
7d0d0ce1 | 279 | else if (DECL_FUNCTION_PERSONALITY (e->caller->symbol.decl) |
280 | && DECL_FUNCTION_PERSONALITY (callee->symbol.decl) | |
281 | && (DECL_FUNCTION_PERSONALITY (e->caller->symbol.decl) | |
282 | != DECL_FUNCTION_PERSONALITY (callee->symbol.decl))) | |
4869c23f | 283 | { |
284 | e->inline_failed = CIF_EH_PERSONALITY; | |
285 | inlinable = false; | |
286 | } | |
3bd76a99 | 287 | /* TM pure functions should not be inlined into non-TM_pure |
288 | functions. */ | |
7d0d0ce1 | 289 | else if (is_tm_pure (callee->symbol.decl) |
290 | && !is_tm_pure (e->caller->symbol.decl)) | |
4c0315d0 | 291 | { |
292 | e->inline_failed = CIF_UNSPECIFIED; | |
293 | inlinable = false; | |
294 | } | |
4869c23f | 295 | /* Don't inline if the callee can throw non-call exceptions but the |
296 | caller cannot. | |
297 | FIXME: this is obviously wrong for LTO where STRUCT_FUNCTION is missing. | |
298 | Move the flag into cgraph node or mirror it in the inline summary. */ | |
69d925d0 | 299 | else if (callee_cfun && callee_cfun->can_throw_non_call_exceptions |
300 | && !(caller_cfun && caller_cfun->can_throw_non_call_exceptions)) | |
4869c23f | 301 | { |
302 | e->inline_failed = CIF_NON_CALL_EXCEPTIONS; | |
303 | inlinable = false; | |
304 | } | |
4055a556 | 305 | /* Check compatibility of target optimization options. */ |
7d0d0ce1 | 306 | else if (!targetm.target_option.can_inline_p (e->caller->symbol.decl, |
307 | callee->symbol.decl)) | |
4869c23f | 308 | { |
309 | e->inline_failed = CIF_TARGET_OPTION_MISMATCH; | |
310 | inlinable = false; | |
311 | } | |
312 | /* Check if caller growth allows the inlining. */ | |
7d0d0ce1 | 313 | else if (!DECL_DISREGARD_INLINE_LIMITS (callee->symbol.decl) |
6f60f0b6 | 314 | && !lookup_attribute ("flatten", |
315 | DECL_ATTRIBUTES | |
316 | (e->caller->global.inlined_to | |
7d0d0ce1 | 317 | ? e->caller->global.inlined_to->symbol.decl |
318 | : e->caller->symbol.decl)) | |
4869c23f | 319 | && !caller_growth_limits (e)) |
320 | inlinable = false; | |
321 | /* Don't inline a function with a higher optimization level than the | |
322 | caller. FIXME: this is really just tip of iceberg of handling | |
323 | optimization attribute. */ | |
324 | else if (caller_tree != callee_tree) | |
b30512dd | 325 | { |
4869c23f | 326 | struct cl_optimization *caller_opt |
327 | = TREE_OPTIMIZATION ((caller_tree) | |
328 | ? caller_tree | |
329 | : optimization_default_node); | |
330 | ||
331 | struct cl_optimization *callee_opt | |
332 | = TREE_OPTIMIZATION ((callee_tree) | |
333 | ? callee_tree | |
334 | : optimization_default_node); | |
335 | ||
438719a9 | 336 | if (((caller_opt->x_optimize > callee_opt->x_optimize) |
337 | || (caller_opt->x_optimize_size != callee_opt->x_optimize_size)) | |
338 | /* gcc.dg/pr43564.c. Look at forced inline even in -O0. */ | |
7d0d0ce1 | 339 | && !DECL_DISREGARD_INLINE_LIMITS (e->callee->symbol.decl)) |
4869c23f | 340 | { |
b588156f | 341 | e->inline_failed = CIF_OPTIMIZATION_MISMATCH; |
4869c23f | 342 | inlinable = false; |
343 | } | |
344 | } | |
345 | ||
4869c23f | 346 | if (!inlinable && report) |
347 | report_inline_failed_reason (e); | |
348 | return inlinable; | |
349 | } | |
350 | ||
351 | ||
352 | /* Return true if the edge E is inlinable during early inlining. */ | |
353 | ||
354 | static bool | |
355 | can_early_inline_edge_p (struct cgraph_edge *e) | |
356 | { | |
82626cb0 | 357 | struct cgraph_node *callee = cgraph_function_or_thunk_node (e->callee, |
358 | NULL); | |
4869c23f | 359 | /* Early inliner might get called at WPA stage when IPA pass adds new |
360 | function. In this case we can not really do any of early inlining | |
361 | because function bodies are missing. */ | |
7d0d0ce1 | 362 | if (!gimple_has_body_p (callee->symbol.decl)) |
4869c23f | 363 | { |
364 | e->inline_failed = CIF_BODY_NOT_AVAILABLE; | |
b30512dd | 365 | return false; |
366 | } | |
4869c23f | 367 | /* In early inliner some of callees may not be in SSA form yet |
368 | (i.e. the callgraph is cyclic and we did not process | |
369 | the callee by early inliner, yet). We don't have CIF code for this | |
370 | case; later we will re-do the decision in the real inliner. */ | |
7d0d0ce1 | 371 | if (!gimple_in_ssa_p (DECL_STRUCT_FUNCTION (e->caller->symbol.decl)) |
372 | || !gimple_in_ssa_p (DECL_STRUCT_FUNCTION (callee->symbol.decl))) | |
af9e0580 | 373 | { |
4869c23f | 374 | if (dump_file) |
375 | fprintf (dump_file, " edge not inlinable: not in SSA form\n"); | |
af9e0580 | 376 | return false; |
377 | } | |
4869c23f | 378 | if (!can_inline_edge_p (e, true)) |
379 | return false; | |
380 | return true; | |
381 | } | |
382 | ||
383 | ||
384 | /* Return true when N is leaf function. Accept cheap builtins | |
385 | in leaf functions. */ | |
386 | ||
387 | static bool | |
388 | leaf_node_p (struct cgraph_node *n) | |
389 | { | |
390 | struct cgraph_edge *e; | |
391 | for (e = n->callees; e; e = e->next_callee) | |
7d0d0ce1 | 392 | if (!is_inexpensive_builtin (e->callee->symbol.decl)) |
4869c23f | 393 | return false; |
394 | return true; | |
395 | } | |
396 | ||
af9e0580 | 397 | |
4869c23f | 398 | /* Return true if we are interested in inlining small function. */ |
b30512dd | 399 | |
4869c23f | 400 | static bool |
401 | want_early_inline_function_p (struct cgraph_edge *e) | |
402 | { | |
403 | bool want_inline = true; | |
82626cb0 | 404 | struct cgraph_node *callee = cgraph_function_or_thunk_node (e->callee, NULL); |
4869c23f | 405 | |
7d0d0ce1 | 406 | if (DECL_DISREGARD_INLINE_LIMITS (callee->symbol.decl)) |
4869c23f | 407 | ; |
7d0d0ce1 | 408 | else if (!DECL_DECLARED_INLINE_P (callee->symbol.decl) |
4869c23f | 409 | && !flag_inline_small_functions) |
410 | { | |
411 | e->inline_failed = CIF_FUNCTION_NOT_INLINE_CANDIDATE; | |
412 | report_inline_failed_reason (e); | |
413 | want_inline = false; | |
414 | } | |
415 | else | |
b30512dd | 416 | { |
4869c23f | 417 | int growth = estimate_edge_growth (e); |
418 | if (growth <= 0) | |
419 | ; | |
420 | else if (!cgraph_maybe_hot_edge_p (e) | |
421 | && growth > 0) | |
422 | { | |
423 | if (dump_file) | |
424 | fprintf (dump_file, " will not early inline: %s/%i->%s/%i, " | |
425 | "call is cold and code would grow by %i\n", | |
426 | cgraph_node_name (e->caller), e->caller->uid, | |
82626cb0 | 427 | cgraph_node_name (callee), callee->uid, |
4869c23f | 428 | growth); |
429 | want_inline = false; | |
430 | } | |
82626cb0 | 431 | else if (!leaf_node_p (callee) |
4869c23f | 432 | && growth > 0) |
b30512dd | 433 | { |
4869c23f | 434 | if (dump_file) |
435 | fprintf (dump_file, " will not early inline: %s/%i->%s/%i, " | |
436 | "callee is not leaf and code would grow by %i\n", | |
437 | cgraph_node_name (e->caller), e->caller->uid, | |
82626cb0 | 438 | cgraph_node_name (callee), callee->uid, |
4869c23f | 439 | growth); |
440 | want_inline = false; | |
b30512dd | 441 | } |
4869c23f | 442 | else if (growth > PARAM_VALUE (PARAM_EARLY_INLINING_INSNS)) |
443 | { | |
444 | if (dump_file) | |
445 | fprintf (dump_file, " will not early inline: %s/%i->%s/%i, " | |
446 | "growth %i exceeds --param early-inlining-insns\n", | |
447 | cgraph_node_name (e->caller), e->caller->uid, | |
82626cb0 | 448 | cgraph_node_name (callee), callee->uid, |
4869c23f | 449 | growth); |
450 | want_inline = false; | |
451 | } | |
452 | } | |
453 | return want_inline; | |
454 | } | |
455 | ||
456 | /* Return true if we are interested in inlining small function. | |
457 | When REPORT is true, report reason to dump file. */ | |
458 | ||
459 | static bool | |
460 | want_inline_small_function_p (struct cgraph_edge *e, bool report) | |
461 | { | |
462 | bool want_inline = true; | |
82626cb0 | 463 | struct cgraph_node *callee = cgraph_function_or_thunk_node (e->callee, NULL); |
4869c23f | 464 | |
7d0d0ce1 | 465 | if (DECL_DISREGARD_INLINE_LIMITS (callee->symbol.decl)) |
4869c23f | 466 | ; |
7d0d0ce1 | 467 | else if (!DECL_DECLARED_INLINE_P (callee->symbol.decl) |
4869c23f | 468 | && !flag_inline_small_functions) |
469 | { | |
470 | e->inline_failed = CIF_FUNCTION_NOT_INLINE_CANDIDATE; | |
471 | want_inline = false; | |
b30512dd | 472 | } |
65c1a668 | 473 | else |
b30512dd | 474 | { |
4869c23f | 475 | int growth = estimate_edge_growth (e); |
476 | ||
477 | if (growth <= 0) | |
478 | ; | |
7d0d0ce1 | 479 | else if (DECL_DECLARED_INLINE_P (callee->symbol.decl) |
4869c23f | 480 | && growth >= MAX_INLINE_INSNS_SINGLE) |
481 | { | |
482 | e->inline_failed = CIF_MAX_INLINE_INSNS_SINGLE_LIMIT; | |
483 | want_inline = false; | |
484 | } | |
a844747e | 485 | /* Before giving up based on fact that caller size will grow, allow |
486 | functions that are called few times and eliminating the offline | |
487 | copy will lead to overall code size reduction. | |
488 | Not all of these will be handled by subsequent inlining of functions | |
489 | called once: in particular weak functions are not handled or funcitons | |
490 | that inline to multiple calls but a lot of bodies is optimized out. | |
491 | Finally we want to inline earlier to allow inlining of callbacks. | |
492 | ||
493 | This is slightly wrong on aggressive side: it is entirely possible | |
494 | that function is called many times with a context where inlining | |
495 | reduces code size and few times with a context where inlining increase | |
496 | code size. Resoluting growth estimate will be negative even if it | |
497 | would make more sense to keep offline copy and do not inline into the | |
498 | call sites that makes the code size grow. | |
499 | ||
500 | When badness orders the calls in a way that code reducing calls come | |
501 | first, this situation is not a problem at all: after inlining all | |
502 | "good" calls, we will realize that keeping the function around is | |
503 | better. */ | |
504 | else if (growth <= MAX_INLINE_INSNS_SINGLE | |
505 | /* Unlike for functions called once, we play unsafe with | |
506 | COMDATs. We can allow that since we know functions | |
507 | in consideration are small (and thus risk is small) and | |
508 | moreover grow estimates already accounts that COMDAT | |
509 | functions may or may not disappear when eliminated from | |
510 | current unit. With good probability making aggressive | |
511 | choice in all units is going to make overall program | |
512 | smaller. | |
513 | ||
514 | Consequently we ask cgraph_can_remove_if_no_direct_calls_p | |
515 | instead of | |
516 | cgraph_will_be_removed_from_program_if_no_direct_calls */ | |
7d0d0ce1 | 517 | && !DECL_EXTERNAL (callee->symbol.decl) |
a844747e | 518 | && cgraph_can_remove_if_no_direct_calls_p (callee) |
519 | && estimate_growth (callee) <= 0) | |
520 | ; | |
7d0d0ce1 | 521 | else if (!DECL_DECLARED_INLINE_P (callee->symbol.decl) |
4869c23f | 522 | && !flag_inline_functions) |
523 | { | |
524 | e->inline_failed = CIF_NOT_DECLARED_INLINED; | |
525 | want_inline = false; | |
526 | } | |
7d0d0ce1 | 527 | else if (!DECL_DECLARED_INLINE_P (callee->symbol.decl) |
4869c23f | 528 | && growth >= MAX_INLINE_INSNS_AUTO) |
529 | { | |
530 | e->inline_failed = CIF_MAX_INLINE_INSNS_AUTO_LIMIT; | |
531 | want_inline = false; | |
532 | } | |
a844747e | 533 | /* If call is cold, do not inline when function body would grow. */ |
534 | else if (!cgraph_maybe_hot_edge_p (e)) | |
b30512dd | 535 | { |
4869c23f | 536 | e->inline_failed = CIF_UNLIKELY_CALL; |
537 | want_inline = false; | |
b30512dd | 538 | } |
539 | } | |
4869c23f | 540 | if (!want_inline && report) |
541 | report_inline_failed_reason (e); | |
542 | return want_inline; | |
543 | } | |
b30512dd | 544 | |
4869c23f | 545 | /* EDGE is self recursive edge. |
546 | We hand two cases - when function A is inlining into itself | |
547 | or when function A is being inlined into another inliner copy of function | |
548 | A within function B. | |
549 | ||
550 | In first case OUTER_NODE points to the toplevel copy of A, while | |
551 | in the second case OUTER_NODE points to the outermost copy of A in B. | |
552 | ||
553 | In both cases we want to be extra selective since | |
554 | inlining the call will just introduce new recursive calls to appear. */ | |
4055a556 | 555 | |
4869c23f | 556 | static bool |
557 | want_inline_self_recursive_call_p (struct cgraph_edge *edge, | |
558 | struct cgraph_node *outer_node, | |
559 | bool peeling, | |
560 | int depth) | |
561 | { | |
562 | char const *reason = NULL; | |
563 | bool want_inline = true; | |
564 | int caller_freq = CGRAPH_FREQ_BASE; | |
565 | int max_depth = PARAM_VALUE (PARAM_MAX_INLINE_RECURSIVE_DEPTH_AUTO); | |
566 | ||
7d0d0ce1 | 567 | if (DECL_DECLARED_INLINE_P (edge->caller->symbol.decl)) |
4869c23f | 568 | max_depth = PARAM_VALUE (PARAM_MAX_INLINE_RECURSIVE_DEPTH); |
569 | ||
570 | if (!cgraph_maybe_hot_edge_p (edge)) | |
571 | { | |
572 | reason = "recursive call is cold"; | |
573 | want_inline = false; | |
574 | } | |
575 | else if (max_count && !outer_node->count) | |
576 | { | |
577 | reason = "not executed in profile"; | |
578 | want_inline = false; | |
579 | } | |
580 | else if (depth > max_depth) | |
581 | { | |
582 | reason = "--param max-inline-recursive-depth exceeded."; | |
583 | want_inline = false; | |
584 | } | |
585 | ||
586 | if (outer_node->global.inlined_to) | |
587 | caller_freq = outer_node->callers->frequency; | |
588 | ||
589 | if (!want_inline) | |
590 | ; | |
591 | /* Inlining of self recursive function into copy of itself within other function | |
592 | is transformation similar to loop peeling. | |
593 | ||
4055a556 | 594 | Peeling is profitable if we can inline enough copies to make probability |
4869c23f | 595 | of actual call to the self recursive function very small. Be sure that |
596 | the probability of recursion is small. | |
597 | ||
4055a556 | 598 | We ensure that the frequency of recursing is at most 1 - (1/max_depth). |
599 | This way the expected number of recision is at most max_depth. */ | |
4869c23f | 600 | else if (peeling) |
601 | { | |
602 | int max_prob = CGRAPH_FREQ_BASE - ((CGRAPH_FREQ_BASE + max_depth - 1) | |
603 | / max_depth); | |
604 | int i; | |
605 | for (i = 1; i < depth; i++) | |
606 | max_prob = max_prob * max_prob / CGRAPH_FREQ_BASE; | |
607 | if (max_count | |
608 | && (edge->count * CGRAPH_FREQ_BASE / outer_node->count | |
609 | >= max_prob)) | |
610 | { | |
611 | reason = "profile of recursive call is too large"; | |
612 | want_inline = false; | |
613 | } | |
614 | if (!max_count | |
615 | && (edge->frequency * CGRAPH_FREQ_BASE / caller_freq | |
616 | >= max_prob)) | |
617 | { | |
618 | reason = "frequency of recursive call is too large"; | |
619 | want_inline = false; | |
620 | } | |
621 | } | |
4055a556 | 622 | /* Recursive inlining, i.e. equivalent of unrolling, is profitable if recursion |
4869c23f | 623 | depth is large. We reduce function call overhead and increase chances that |
624 | things fit in hardware return predictor. | |
625 | ||
626 | Recursive inlining might however increase cost of stack frame setup | |
627 | actually slowing down functions whose recursion tree is wide rather than | |
628 | deep. | |
629 | ||
4055a556 | 630 | Deciding reliably on when to do recursive inlining without profile feedback |
4869c23f | 631 | is tricky. For now we disable recursive inlining when probability of self |
632 | recursion is low. | |
633 | ||
634 | Recursive inlining of self recursive call within loop also results in large loop | |
635 | depths that generally optimize badly. We may want to throttle down inlining | |
636 | in those cases. In particular this seems to happen in one of libstdc++ rb tree | |
637 | methods. */ | |
638 | else | |
639 | { | |
640 | if (max_count | |
641 | && (edge->count * 100 / outer_node->count | |
642 | <= PARAM_VALUE (PARAM_MIN_INLINE_RECURSIVE_PROBABILITY))) | |
643 | { | |
644 | reason = "profile of recursive call is too small"; | |
645 | want_inline = false; | |
646 | } | |
647 | else if (!max_count | |
648 | && (edge->frequency * 100 / caller_freq | |
649 | <= PARAM_VALUE (PARAM_MIN_INLINE_RECURSIVE_PROBABILITY))) | |
650 | { | |
651 | reason = "frequency of recursive call is too small"; | |
652 | want_inline = false; | |
653 | } | |
654 | } | |
655 | if (!want_inline && dump_file) | |
656 | fprintf (dump_file, " not inlining recursively: %s\n", reason); | |
657 | return want_inline; | |
65c1a668 | 658 | } |
659 | ||
794fd282 | 660 | /* Return true when NODE has caller other than EDGE. |
661 | Worker for cgraph_for_node_and_aliases. */ | |
662 | ||
663 | static bool | |
664 | check_caller_edge (struct cgraph_node *node, void *edge) | |
665 | { | |
666 | return (node->callers | |
667 | && node->callers != edge); | |
668 | } | |
669 | ||
4055a556 | 670 | |
671 | /* Decide if NODE is called once inlining it would eliminate need | |
672 | for the offline copy of function. */ | |
673 | ||
674 | static bool | |
675 | want_inline_function_called_once_p (struct cgraph_node *node) | |
676 | { | |
794fd282 | 677 | struct cgraph_node *function = cgraph_function_or_thunk_node (node, NULL); |
4055a556 | 678 | /* Already inlined? */ |
794fd282 | 679 | if (function->global.inlined_to) |
4055a556 | 680 | return false; |
681 | /* Zero or more then one callers? */ | |
682 | if (!node->callers | |
683 | || node->callers->next_caller) | |
684 | return false; | |
794fd282 | 685 | /* Maybe other aliases has more direct calls. */ |
686 | if (cgraph_for_node_and_aliases (node, check_caller_edge, node->callers, true)) | |
687 | return false; | |
4055a556 | 688 | /* Recursive call makes no sense to inline. */ |
794fd282 | 689 | if (cgraph_edge_recursive_p (node->callers)) |
4055a556 | 690 | return false; |
691 | /* External functions are not really in the unit, so inlining | |
692 | them when called once would just increase the program size. */ | |
7d0d0ce1 | 693 | if (DECL_EXTERNAL (function->symbol.decl)) |
4055a556 | 694 | return false; |
695 | /* Offline body must be optimized out. */ | |
794fd282 | 696 | if (!cgraph_will_be_removed_from_program_if_no_direct_calls (function)) |
4055a556 | 697 | return false; |
698 | if (!can_inline_edge_p (node->callers, true)) | |
699 | return false; | |
700 | return true; | |
701 | } | |
702 | ||
0656d247 | 703 | |
704 | /* Return relative time improvement for inlining EDGE in range | |
705 | 1...2^9. */ | |
706 | ||
707 | static inline int | |
708 | relative_time_benefit (struct inline_summary *callee_info, | |
709 | struct cgraph_edge *edge, | |
710 | int time_growth) | |
711 | { | |
712 | int relbenefit; | |
713 | gcov_type uninlined_call_time; | |
714 | ||
715 | uninlined_call_time = | |
716 | ((gcov_type) | |
717 | (callee_info->time | |
0a97420b | 718 | + inline_edge_summary (edge)->call_stmt_time) * edge->frequency |
719 | + CGRAPH_FREQ_BASE / 2) / CGRAPH_FREQ_BASE; | |
0656d247 | 720 | /* Compute relative time benefit, i.e. how much the call becomes faster. |
721 | ??? perhaps computing how much the caller+calle together become faster | |
722 | would lead to more realistic results. */ | |
723 | if (!uninlined_call_time) | |
724 | uninlined_call_time = 1; | |
725 | relbenefit = | |
726 | (uninlined_call_time - time_growth) * 256 / (uninlined_call_time); | |
727 | relbenefit = MIN (relbenefit, 512); | |
728 | relbenefit = MAX (relbenefit, 1); | |
729 | return relbenefit; | |
730 | } | |
731 | ||
732 | ||
a49506c7 | 733 | /* A cost model driving the inlining heuristics in a way so the edges with |
734 | smallest badness are inlined first. After each inlining is performed | |
442e3cb9 | 735 | the costs of all caller edges of nodes affected are recomputed so the |
a49506c7 | 736 | metrics may accurately depend on values such as number of inlinable callers |
4ae20857 | 737 | of the function or function body size. */ |
a49506c7 | 738 | |
739 | static int | |
4869c23f | 740 | edge_badness (struct cgraph_edge *edge, bool dump) |
a49506c7 | 741 | { |
97343302 | 742 | gcov_type badness; |
a41f2a28 | 743 | int growth, time_growth; |
82626cb0 | 744 | struct cgraph_node *callee = cgraph_function_or_thunk_node (edge->callee, |
745 | NULL); | |
746 | struct inline_summary *callee_info = inline_summary (callee); | |
4ae20857 | 747 | |
7d0d0ce1 | 748 | if (DECL_DISREGARD_INLINE_LIMITS (callee->symbol.decl)) |
960dff4c | 749 | return INT_MIN; |
750 | ||
99c67f24 | 751 | growth = estimate_edge_growth (edge); |
a41f2a28 | 752 | time_growth = estimate_edge_time (edge); |
5cd33168 | 753 | |
022b3380 | 754 | if (dump) |
755 | { | |
0a10fd82 | 756 | fprintf (dump_file, " Badness calculation for %s -> %s\n", |
022b3380 | 757 | cgraph_node_name (edge->caller), |
82626cb0 | 758 | cgraph_node_name (callee)); |
0656d247 | 759 | fprintf (dump_file, " size growth %i, time growth %i\n", |
022b3380 | 760 | growth, |
a41f2a28 | 761 | time_growth); |
022b3380 | 762 | } |
4ae20857 | 763 | |
764 | /* Always prefer inlining saving code size. */ | |
765 | if (growth <= 0) | |
022b3380 | 766 | { |
0656d247 | 767 | badness = INT_MIN / 2 + growth; |
022b3380 | 768 | if (dump) |
0656d247 | 769 | fprintf (dump_file, " %i: Growth %i <= 0\n", (int) badness, |
022b3380 | 770 | growth); |
771 | } | |
4ae20857 | 772 | |
0656d247 | 773 | /* When profiling is available, compute badness as: |
774 | ||
775 | relative_edge_count * relative_time_benefit | |
776 | goodness = ------------------------------------------- | |
777 | edge_growth | |
778 | badness = -goodness | |
779 | ||
780 | The fraction is upside down, becuase on edge counts and time beneits | |
781 | the bounds are known. Edge growth is essentially unlimited. */ | |
782 | ||
54e3de71 | 783 | else if (max_count) |
022b3380 | 784 | { |
0656d247 | 785 | int relbenefit = relative_time_benefit (callee_info, edge, time_growth); |
022b3380 | 786 | badness = |
787 | ((int) | |
0656d247 | 788 | ((double) edge->count * INT_MIN / 2 / max_count / 512) * |
789 | relative_time_benefit (callee_info, edge, time_growth)) / growth; | |
4055a556 | 790 | |
791 | /* Be sure that insanity of the profile won't lead to increasing counts | |
792 | in the scalling and thus to overflow in the computation above. */ | |
793 | gcc_assert (max_count >= edge->count); | |
022b3380 | 794 | if (dump) |
795 | { | |
796 | fprintf (dump_file, | |
797 | " %i (relative %f): profile info. Relative count %f" | |
798 | " * Relative benefit %f\n", | |
799 | (int) badness, (double) badness / INT_MIN, | |
800 | (double) edge->count / max_count, | |
0656d247 | 801 | relbenefit * 100 / 256.0); |
022b3380 | 802 | } |
803 | } | |
4ae20857 | 804 | |
0656d247 | 805 | /* When function local profile is available. Compute badness as: |
4ae20857 | 806 | |
0656d247 | 807 | |
808 | growth_of_callee | |
809 | badness = -------------------------------------- + growth_for-all | |
810 | relative_time_benefit * edge_frequency | |
811 | ||
812 | */ | |
4ae20857 | 813 | else if (flag_guess_branch_prob) |
a49506c7 | 814 | { |
0656d247 | 815 | int div = edge->frequency * (1<<10) / CGRAPH_FREQ_MAX; |
48e1416a | 816 | |
0656d247 | 817 | div = MAX (div, 1); |
818 | gcc_checking_assert (edge->frequency <= CGRAPH_FREQ_MAX); | |
819 | div *= relative_time_benefit (callee_info, edge, time_growth); | |
820 | ||
821 | /* frequency is normalized in range 1...2^10. | |
822 | relbenefit in range 1...2^9 | |
823 | DIV should be in range 1....2^19. */ | |
824 | gcc_checking_assert (div >= 1 && div <= (1<<19)); | |
825 | ||
826 | /* Result must be integer in range 0...INT_MAX. | |
827 | Set the base of fixed point calculation so we don't lose much of | |
828 | precision for small bandesses (those are interesting) yet we don't | |
48976c07 | 829 | overflow for growths that are still in interesting range. |
830 | ||
831 | Fixed point arithmetic with point at 8th bit. */ | |
832 | badness = ((gcov_type)growth) * (1<<(19+8)); | |
0656d247 | 833 | badness = (badness + div / 2) / div; |
834 | ||
835 | /* Overall growth of inlining all calls of function matters: we want to | |
836 | inline so offline copy of function is no longer needed. | |
837 | ||
838 | Additionally functions that can be fully inlined without much of | |
839 | effort are better inline candidates than functions that can be fully | |
840 | inlined only after noticeable overall unit growths. The latter | |
841 | are better in a sense compressing of code size by factoring out common | |
842 | code into separate function shared by multiple code paths. | |
843 | ||
844 | We might mix the valud into the fraction by taking into account | |
845 | relative growth of the unit, but for now just add the number | |
846 | into resulting fraction. */ | |
48976c07 | 847 | if (badness > INT_MAX / 2) |
848 | { | |
849 | badness = INT_MAX / 2; | |
850 | if (dump) | |
851 | fprintf (dump_file, "Badness overflow\n"); | |
852 | } | |
022b3380 | 853 | if (dump) |
854 | { | |
855 | fprintf (dump_file, | |
db86e6d4 | 856 | " %i: guessed profile. frequency %f," |
0656d247 | 857 | " benefit %f%%, divisor %i\n", |
db86e6d4 | 858 | (int) badness, (double)edge->frequency / CGRAPH_FREQ_BASE, |
0656d247 | 859 | relative_time_benefit (callee_info, edge, time_growth) * 100 / 256.0, div); |
022b3380 | 860 | } |
4ae20857 | 861 | } |
862 | /* When function local profile is not available or it does not give | |
863 | useful information (ie frequency is zero), base the cost on | |
864 | loop nest and overall size growth, so we optimize for overall number | |
865 | of functions fully inlined in program. */ | |
866 | else | |
867 | { | |
0835ad03 | 868 | int nest = MIN (inline_edge_summary (edge)->loop_depth, 8); |
10694fa2 | 869 | badness = growth * 256; |
a49506c7 | 870 | |
4ae20857 | 871 | /* Decrease badness if call is nested. */ |
48e1416a | 872 | if (badness > 0) |
4ae20857 | 873 | badness >>= nest; |
874 | else | |
022b3380 | 875 | { |
4ae20857 | 876 | badness <<= nest; |
022b3380 | 877 | } |
878 | if (dump) | |
879 | fprintf (dump_file, " %i: no profile. nest %i\n", (int) badness, | |
880 | nest); | |
a49506c7 | 881 | } |
022b3380 | 882 | |
883 | /* Ensure that we did not overflow in all the fixed point math above. */ | |
884 | gcc_assert (badness >= INT_MIN); | |
885 | gcc_assert (badness <= INT_MAX - 1); | |
4ae20857 | 886 | /* Make recursive inlining happen always after other inlining is done. */ |
17c205c9 | 887 | if (cgraph_edge_recursive_p (edge)) |
4ae20857 | 888 | return badness + 1; |
a49506c7 | 889 | else |
4ae20857 | 890 | return badness; |
a49506c7 | 891 | } |
892 | ||
9f3c2a90 | 893 | /* Recompute badness of EDGE and update its key in HEAP if needed. */ |
4869c23f | 894 | static inline void |
9f3c2a90 | 895 | update_edge_key (fibheap_t heap, struct cgraph_edge *edge) |
896 | { | |
4869c23f | 897 | int badness = edge_badness (edge, false); |
9f3c2a90 | 898 | if (edge->aux) |
899 | { | |
900 | fibnode_t n = (fibnode_t) edge->aux; | |
901 | gcc_checking_assert (n->data == edge); | |
902 | ||
903 | /* fibheap_replace_key only decrease the keys. | |
904 | When we increase the key we do not update heap | |
905 | and instead re-insert the element once it becomes | |
0a10fd82 | 906 | a minimum of heap. */ |
9f3c2a90 | 907 | if (badness < n->key) |
908 | { | |
4869c23f | 909 | if (dump_file && (dump_flags & TDF_DETAILS)) |
910 | { | |
911 | fprintf (dump_file, | |
912 | " decreasing badness %s/%i -> %s/%i, %i to %i\n", | |
913 | cgraph_node_name (edge->caller), edge->caller->uid, | |
914 | cgraph_node_name (edge->callee), edge->callee->uid, | |
915 | (int)n->key, | |
916 | badness); | |
917 | } | |
0656d247 | 918 | fibheap_replace_key (heap, n, badness); |
9f3c2a90 | 919 | gcc_checking_assert (n->key == badness); |
920 | } | |
921 | } | |
922 | else | |
4869c23f | 923 | { |
924 | if (dump_file && (dump_flags & TDF_DETAILS)) | |
925 | { | |
926 | fprintf (dump_file, | |
927 | " enqueuing call %s/%i -> %s/%i, badness %i\n", | |
928 | cgraph_node_name (edge->caller), edge->caller->uid, | |
929 | cgraph_node_name (edge->callee), edge->callee->uid, | |
930 | badness); | |
931 | } | |
932 | edge->aux = fibheap_insert (heap, badness, edge); | |
933 | } | |
9f3c2a90 | 934 | } |
935 | ||
ba5b0608 | 936 | |
937 | /* NODE was inlined. | |
938 | All caller edges needs to be resetted because | |
939 | size estimates change. Similarly callees needs reset | |
940 | because better context may be known. */ | |
941 | ||
942 | static void | |
943 | reset_edge_caches (struct cgraph_node *node) | |
944 | { | |
945 | struct cgraph_edge *edge; | |
946 | struct cgraph_edge *e = node->callees; | |
947 | struct cgraph_node *where = node; | |
f30e87e9 | 948 | int i; |
949 | struct ipa_ref *ref; | |
ba5b0608 | 950 | |
951 | if (where->global.inlined_to) | |
952 | where = where->global.inlined_to; | |
953 | ||
954 | /* WHERE body size has changed, the cached growth is invalid. */ | |
955 | reset_node_growth_cache (where); | |
956 | ||
957 | for (edge = where->callers; edge; edge = edge->next_caller) | |
958 | if (edge->inline_failed) | |
959 | reset_edge_growth_cache (edge); | |
7d0d0ce1 | 960 | for (i = 0; ipa_ref_list_refering_iterate (&where->symbol.ref_list, |
961 | i, ref); i++) | |
f30e87e9 | 962 | if (ref->use == IPA_REF_ALIAS) |
963 | reset_edge_caches (ipa_ref_refering_node (ref)); | |
ba5b0608 | 964 | |
965 | if (!e) | |
966 | return; | |
967 | ||
968 | while (true) | |
969 | if (!e->inline_failed && e->callee->callees) | |
970 | e = e->callee->callees; | |
971 | else | |
972 | { | |
973 | if (e->inline_failed) | |
974 | reset_edge_growth_cache (e); | |
975 | if (e->next_callee) | |
976 | e = e->next_callee; | |
977 | else | |
978 | { | |
979 | do | |
980 | { | |
981 | if (e->caller == node) | |
982 | return; | |
983 | e = e->caller->callers; | |
984 | } | |
985 | while (!e->next_callee); | |
986 | e = e->next_callee; | |
987 | } | |
988 | } | |
989 | } | |
990 | ||
991 | /* Recompute HEAP nodes for each of caller of NODE. | |
992 | UPDATED_NODES track nodes we already visited, to avoid redundant work. | |
993 | When CHECK_INLINABLITY_FOR is set, re-check for specified edge that | |
994 | it is inlinable. Otherwise check all edges. */ | |
a49506c7 | 995 | |
996 | static void | |
997 | update_caller_keys (fibheap_t heap, struct cgraph_node *node, | |
ba5b0608 | 998 | bitmap updated_nodes, |
999 | struct cgraph_edge *check_inlinablity_for) | |
a49506c7 | 1000 | { |
1001 | struct cgraph_edge *edge; | |
c70f46b0 | 1002 | int i; |
1003 | struct ipa_ref *ref; | |
a49506c7 | 1004 | |
f30e87e9 | 1005 | if ((!node->alias && !inline_summary (node)->inlinable) |
af9e0580 | 1006 | || cgraph_function_body_availability (node) <= AVAIL_OVERWRITABLE |
a49506c7 | 1007 | || node->global.inlined_to) |
1008 | return; | |
6ef9bbe0 | 1009 | if (!bitmap_set_bit (updated_nodes, node->uid)) |
a49506c7 | 1010 | return; |
a49506c7 | 1011 | |
7d0d0ce1 | 1012 | for (i = 0; ipa_ref_list_refering_iterate (&node->symbol.ref_list, |
1013 | i, ref); i++) | |
c70f46b0 | 1014 | if (ref->use == IPA_REF_ALIAS) |
1015 | { | |
1016 | struct cgraph_node *alias = ipa_ref_refering_node (ref); | |
1017 | update_caller_keys (heap, alias, updated_nodes, check_inlinablity_for); | |
1018 | } | |
1019 | ||
854efde4 | 1020 | for (edge = node->callers; edge; edge = edge->next_caller) |
4869c23f | 1021 | if (edge->inline_failed) |
1022 | { | |
ba5b0608 | 1023 | if (!check_inlinablity_for |
1024 | || check_inlinablity_for == edge) | |
109bf1e3 | 1025 | { |
ba5b0608 | 1026 | if (can_inline_edge_p (edge, false) |
1027 | && want_inline_small_function_p (edge, false)) | |
1028 | update_edge_key (heap, edge); | |
1029 | else if (edge->aux) | |
1030 | { | |
1031 | report_inline_failed_reason (edge); | |
1032 | fibheap_delete_node (heap, (fibnode_t) edge->aux); | |
1033 | edge->aux = NULL; | |
1034 | } | |
109bf1e3 | 1035 | } |
ba5b0608 | 1036 | else if (edge->aux) |
1037 | update_edge_key (heap, edge); | |
4869c23f | 1038 | } |
9f3c2a90 | 1039 | } |
1040 | ||
ba5b0608 | 1041 | /* Recompute HEAP nodes for each uninlined call in NODE. |
9f3c2a90 | 1042 | This is used when we know that edge badnesses are going only to increase |
1043 | (we introduced new call site) and thus all we need is to insert newly | |
1044 | created edges into heap. */ | |
1045 | ||
1046 | static void | |
1047 | update_callee_keys (fibheap_t heap, struct cgraph_node *node, | |
1048 | bitmap updated_nodes) | |
1049 | { | |
1050 | struct cgraph_edge *e = node->callees; | |
4055a556 | 1051 | |
9f3c2a90 | 1052 | if (!e) |
1053 | return; | |
1054 | while (true) | |
1055 | if (!e->inline_failed && e->callee->callees) | |
1056 | e = e->callee->callees; | |
1057 | else | |
a49506c7 | 1058 | { |
82626cb0 | 1059 | enum availability avail; |
1060 | struct cgraph_node *callee; | |
e825447c | 1061 | /* We do not reset callee growth cache here. Since we added a new call, |
1062 | growth chould have just increased and consequentely badness metric | |
1063 | don't need updating. */ | |
9f3c2a90 | 1064 | if (e->inline_failed |
82626cb0 | 1065 | && (callee = cgraph_function_or_thunk_node (e->callee, &avail)) |
1066 | && inline_summary (callee)->inlinable | |
1067 | && cgraph_function_body_availability (callee) >= AVAIL_AVAILABLE | |
1068 | && !bitmap_bit_p (updated_nodes, callee->uid)) | |
a49506c7 | 1069 | { |
ba5b0608 | 1070 | if (can_inline_edge_p (e, false) |
1071 | && want_inline_small_function_p (e, false)) | |
1072 | update_edge_key (heap, e); | |
1073 | else if (e->aux) | |
1074 | { | |
1075 | report_inline_failed_reason (e); | |
1076 | fibheap_delete_node (heap, (fibnode_t) e->aux); | |
1077 | e->aux = NULL; | |
1078 | } | |
9f3c2a90 | 1079 | } |
1080 | if (e->next_callee) | |
1081 | e = e->next_callee; | |
1082 | else | |
1083 | { | |
1084 | do | |
022b3380 | 1085 | { |
9f3c2a90 | 1086 | if (e->caller == node) |
1087 | return; | |
1088 | e = e->caller->callers; | |
022b3380 | 1089 | } |
9f3c2a90 | 1090 | while (!e->next_callee); |
1091 | e = e->next_callee; | |
a49506c7 | 1092 | } |
a49506c7 | 1093 | } |
1094 | } | |
1095 | ||
854efde4 | 1096 | /* Recompute heap nodes for each of caller edges of each of callees. |
1097 | Walk recursively into all inline clones. */ | |
a49506c7 | 1098 | |
65c1a668 | 1099 | static void |
9f3c2a90 | 1100 | update_all_callee_keys (fibheap_t heap, struct cgraph_node *node, |
1101 | bitmap updated_nodes) | |
65c1a668 | 1102 | { |
854efde4 | 1103 | struct cgraph_edge *e = node->callees; |
854efde4 | 1104 | if (!e) |
1105 | return; | |
1106 | while (true) | |
1107 | if (!e->inline_failed && e->callee->callees) | |
1108 | e = e->callee->callees; | |
1109 | else | |
1110 | { | |
82626cb0 | 1111 | struct cgraph_node *callee = cgraph_function_or_thunk_node (e->callee, |
1112 | NULL); | |
1113 | ||
ba5b0608 | 1114 | /* We inlined and thus callees might have different number of calls. |
1115 | Reset their caches */ | |
82626cb0 | 1116 | reset_node_growth_cache (callee); |
854efde4 | 1117 | if (e->inline_failed) |
82626cb0 | 1118 | update_caller_keys (heap, callee, updated_nodes, e); |
854efde4 | 1119 | if (e->next_callee) |
1120 | e = e->next_callee; | |
1121 | else | |
1122 | { | |
1123 | do | |
1124 | { | |
1125 | if (e->caller == node) | |
1126 | return; | |
1127 | e = e->caller->callers; | |
1128 | } | |
1129 | while (!e->next_callee); | |
1130 | e = e->next_callee; | |
1131 | } | |
1132 | } | |
65c1a668 | 1133 | } |
1134 | ||
a49506c7 | 1135 | /* Enqueue all recursive calls from NODE into priority queue depending on |
442e3cb9 | 1136 | how likely we want to recursively inline the call. */ |
a49506c7 | 1137 | |
65c1a668 | 1138 | static void |
1139 | lookup_recursive_calls (struct cgraph_node *node, struct cgraph_node *where, | |
a49506c7 | 1140 | fibheap_t heap) |
65c1a668 | 1141 | { |
1142 | struct cgraph_edge *e; | |
82626cb0 | 1143 | enum availability avail; |
1144 | ||
65c1a668 | 1145 | for (e = where->callees; e; e = e->next_callee) |
82626cb0 | 1146 | if (e->callee == node |
1147 | || (cgraph_function_or_thunk_node (e->callee, &avail) == node | |
1148 | && avail > AVAIL_OVERWRITABLE)) | |
65c1a668 | 1149 | { |
0aca0eb6 | 1150 | /* When profile feedback is available, prioritize by expected number |
4055a556 | 1151 | of calls. */ |
0aca0eb6 | 1152 | fibheap_insert (heap, |
4055a556 | 1153 | !max_count ? -e->frequency |
0aca0eb6 | 1154 | : -(e->count / ((max_count + (1<<24) - 1) / (1<<24))), |
1155 | e); | |
65c1a668 | 1156 | } |
1157 | for (e = where->callees; e; e = e->next_callee) | |
1158 | if (!e->inline_failed) | |
a49506c7 | 1159 | lookup_recursive_calls (node, e->callee, heap); |
65c1a668 | 1160 | } |
1161 | ||
1162 | /* Decide on recursive inlining: in the case function has recursive calls, | |
f8daee9b | 1163 | inline until body size reaches given argument. If any new indirect edges |
6db08adc | 1164 | are discovered in the process, add them to *NEW_EDGES, unless NEW_EDGES |
1165 | is NULL. */ | |
a49506c7 | 1166 | |
1167 | static bool | |
4869c23f | 1168 | recursive_inlining (struct cgraph_edge *edge, |
1169 | VEC (cgraph_edge_p, heap) **new_edges) | |
65c1a668 | 1170 | { |
1171 | int limit = PARAM_VALUE (PARAM_MAX_INLINE_INSNS_RECURSIVE_AUTO); | |
a49506c7 | 1172 | fibheap_t heap; |
17c205c9 | 1173 | struct cgraph_node *node; |
65c1a668 | 1174 | struct cgraph_edge *e; |
4869c23f | 1175 | struct cgraph_node *master_clone = NULL, *next; |
65c1a668 | 1176 | int depth = 0; |
1177 | int n = 0; | |
1178 | ||
17c205c9 | 1179 | node = edge->caller; |
1180 | if (node->global.inlined_to) | |
1181 | node = node->global.inlined_to; | |
1182 | ||
7d0d0ce1 | 1183 | if (DECL_DECLARED_INLINE_P (node->symbol.decl)) |
4869c23f | 1184 | limit = PARAM_VALUE (PARAM_MAX_INLINE_INSNS_RECURSIVE); |
65c1a668 | 1185 | |
1186 | /* Make sure that function is small enough to be considered for inlining. */ | |
4869c23f | 1187 | if (estimate_size_after_inlining (node, edge) >= limit) |
a49506c7 | 1188 | return false; |
1189 | heap = fibheap_new (); | |
1190 | lookup_recursive_calls (node, node, heap); | |
1191 | if (fibheap_empty (heap)) | |
1192 | { | |
1193 | fibheap_delete (heap); | |
1194 | return false; | |
1195 | } | |
65c1a668 | 1196 | |
1197 | if (dump_file) | |
48e1416a | 1198 | fprintf (dump_file, |
a49506c7 | 1199 | " Performing recursive inlining on %s\n", |
65c1a668 | 1200 | cgraph_node_name (node)); |
1201 | ||
65c1a668 | 1202 | /* Do the inlining and update list of recursive call during process. */ |
17c205c9 | 1203 | while (!fibheap_empty (heap)) |
65c1a668 | 1204 | { |
cda6870f | 1205 | struct cgraph_edge *curr |
1206 | = (struct cgraph_edge *) fibheap_extract_min (heap); | |
0aca0eb6 | 1207 | struct cgraph_node *cnode; |
a49506c7 | 1208 | |
99c67f24 | 1209 | if (estimate_size_after_inlining (node, curr) > limit) |
17c205c9 | 1210 | break; |
1211 | ||
4869c23f | 1212 | if (!can_inline_edge_p (curr, true)) |
1213 | continue; | |
1214 | ||
0aca0eb6 | 1215 | depth = 1; |
1216 | for (cnode = curr->caller; | |
1217 | cnode->global.inlined_to; cnode = cnode->callers->caller) | |
7d0d0ce1 | 1218 | if (node->symbol.decl |
1219 | == cgraph_function_or_thunk_node (curr->callee, NULL)->symbol.decl) | |
67baa302 | 1220 | depth++; |
0aca0eb6 | 1221 | |
4869c23f | 1222 | if (!want_inline_self_recursive_call_p (curr, node, false, depth)) |
1223 | continue; | |
65c1a668 | 1224 | |
a49506c7 | 1225 | if (dump_file) |
0aca0eb6 | 1226 | { |
48e1416a | 1227 | fprintf (dump_file, |
0aca0eb6 | 1228 | " Inlining call of depth %i", depth); |
1229 | if (node->count) | |
1230 | { | |
1231 | fprintf (dump_file, " called approx. %.2f times per call", | |
1232 | (double)curr->count / node->count); | |
1233 | } | |
1234 | fprintf (dump_file, "\n"); | |
1235 | } | |
4869c23f | 1236 | if (!master_clone) |
1237 | { | |
1238 | /* We need original clone to copy around. */ | |
7d0d0ce1 | 1239 | master_clone = cgraph_clone_node (node, node->symbol.decl, |
0835ad03 | 1240 | node->count, CGRAPH_FREQ_BASE, |
8bae3ea4 | 1241 | false, NULL, true); |
4869c23f | 1242 | for (e = master_clone->callees; e; e = e->next_callee) |
1243 | if (!e->inline_failed) | |
8cbc43ff | 1244 | clone_inlined_nodes (e, true, false, NULL); |
4869c23f | 1245 | } |
1246 | ||
65c1a668 | 1247 | cgraph_redirect_edge_callee (curr, master_clone); |
8cbc43ff | 1248 | inline_call (curr, false, new_edges, &overall_size); |
a49506c7 | 1249 | lookup_recursive_calls (node, curr->callee, heap); |
65c1a668 | 1250 | n++; |
1251 | } | |
4869c23f | 1252 | |
0aca0eb6 | 1253 | if (!fibheap_empty (heap) && dump_file) |
1254 | fprintf (dump_file, " Recursive inlining growth limit met.\n"); | |
a49506c7 | 1255 | fibheap_delete (heap); |
4869c23f | 1256 | |
1257 | if (!master_clone) | |
1258 | return false; | |
1259 | ||
65c1a668 | 1260 | if (dump_file) |
48e1416a | 1261 | fprintf (dump_file, |
4869c23f | 1262 | "\n Inlined %i times, " |
1263 | "body grown from size %i to %i, time %i to %i\n", n, | |
cbd7f5a0 | 1264 | inline_summary (master_clone)->size, inline_summary (node)->size, |
1265 | inline_summary (master_clone)->time, inline_summary (node)->time); | |
65c1a668 | 1266 | |
1267 | /* Remove master clone we used for inlining. We rely that clones inlined | |
1268 | into master clone gets queued just before master clone so we don't | |
1269 | need recursion. */ | |
0704fb2e | 1270 | for (node = cgraph_first_function (); node != master_clone; |
f4ec5ce1 | 1271 | node = next) |
1272 | { | |
0704fb2e | 1273 | next = cgraph_next_function (node); |
f4ec5ce1 | 1274 | if (node->global.inlined_to == master_clone) |
1275 | cgraph_remove_node (node); | |
1276 | } | |
65c1a668 | 1277 | cgraph_remove_node (master_clone); |
4869c23f | 1278 | return true; |
65c1a668 | 1279 | } |
1280 | ||
4055a556 | 1281 | |
0d424440 | 1282 | /* Given whole compilation unit estimate of INSNS, compute how large we can |
5c121ffe | 1283 | allow the unit to grow. */ |
4055a556 | 1284 | |
5c121ffe | 1285 | static int |
1286 | compute_max_insns (int insns) | |
1287 | { | |
1288 | int max_insns = insns; | |
1289 | if (max_insns < PARAM_VALUE (PARAM_LARGE_UNIT_INSNS)) | |
1290 | max_insns = PARAM_VALUE (PARAM_LARGE_UNIT_INSNS); | |
1291 | ||
773aeca3 | 1292 | return ((HOST_WIDEST_INT) max_insns |
1293 | * (100 + PARAM_VALUE (PARAM_INLINE_UNIT_GROWTH)) / 100); | |
5c121ffe | 1294 | } |
1295 | ||
4055a556 | 1296 | |
f8daee9b | 1297 | /* Compute badness of all edges in NEW_EDGES and add them to the HEAP. */ |
4055a556 | 1298 | |
f8daee9b | 1299 | static void |
1300 | add_new_edges_to_heap (fibheap_t heap, VEC (cgraph_edge_p, heap) *new_edges) | |
1301 | { | |
1302 | while (VEC_length (cgraph_edge_p, new_edges) > 0) | |
1303 | { | |
1304 | struct cgraph_edge *edge = VEC_pop (cgraph_edge_p, new_edges); | |
1305 | ||
1306 | gcc_assert (!edge->aux); | |
82626cb0 | 1307 | if (edge->inline_failed |
4869c23f | 1308 | && can_inline_edge_p (edge, true) |
1309 | && want_inline_small_function_p (edge, true)) | |
1310 | edge->aux = fibheap_insert (heap, edge_badness (edge, false), edge); | |
f8daee9b | 1311 | } |
1312 | } | |
1313 | ||
1314 | ||
65c1a668 | 1315 | /* We use greedy algorithm for inlining of small functions: |
4055a556 | 1316 | All inline candidates are put into prioritized heap ordered in |
1317 | increasing badness. | |
65c1a668 | 1318 | |
4055a556 | 1319 | The inlining of small functions is bounded by unit growth parameters. */ |
65c1a668 | 1320 | |
1321 | static void | |
4869c23f | 1322 | inline_small_functions (void) |
65c1a668 | 1323 | { |
1324 | struct cgraph_node *node; | |
a49506c7 | 1325 | struct cgraph_edge *edge; |
65c1a668 | 1326 | fibheap_t heap = fibheap_new (); |
a49506c7 | 1327 | bitmap updated_nodes = BITMAP_ALLOC (NULL); |
97343302 | 1328 | int min_size, max_size; |
f8daee9b | 1329 | VEC (cgraph_edge_p, heap) *new_indirect_edges = NULL; |
4055a556 | 1330 | int initial_size = 0; |
f8daee9b | 1331 | |
00e1f01e | 1332 | if (flag_indirect_inlining) |
f8daee9b | 1333 | new_indirect_edges = VEC_alloc (cgraph_edge_p, heap, 8); |
a49506c7 | 1334 | |
1335 | if (dump_file) | |
4055a556 | 1336 | fprintf (dump_file, |
1337 | "\nDeciding on inlining of small functions. Starting with size %i.\n", | |
1338 | initial_size); | |
65c1a668 | 1339 | |
d826e131 | 1340 | /* Compute overall unit size and other global parameters used by badness |
1341 | metrics. */ | |
65c1a668 | 1342 | |
4055a556 | 1343 | max_count = 0; |
a41f2a28 | 1344 | initialize_growth_caches (); |
d826e131 | 1345 | |
91bf9d9a | 1346 | FOR_EACH_DEFINED_FUNCTION (node) |
1347 | if (!node->global.inlined_to) | |
cbd7f5a0 | 1348 | { |
82626cb0 | 1349 | if (cgraph_function_with_gimple_body_p (node) |
1350 | || node->thunk.thunk_p) | |
1351 | { | |
1352 | struct inline_summary *info = inline_summary (node); | |
65c1a668 | 1353 | |
7d0d0ce1 | 1354 | if (!DECL_EXTERNAL (node->symbol.decl)) |
82626cb0 | 1355 | initial_size += info->size; |
1356 | } | |
4055a556 | 1357 | |
cbd7f5a0 | 1358 | for (edge = node->callers; edge; edge = edge->next_caller) |
a41f2a28 | 1359 | if (max_count < edge->count) |
1360 | max_count = edge->count; | |
cbd7f5a0 | 1361 | } |
5c121ffe | 1362 | |
33b2724f | 1363 | overall_size = initial_size; |
97343302 | 1364 | max_size = compute_max_insns (overall_size); |
1365 | min_size = overall_size; | |
d826e131 | 1366 | |
1367 | /* Populate the heeap with all edges we might inline. */ | |
1368 | ||
91bf9d9a | 1369 | FOR_EACH_DEFINED_FUNCTION (node) |
1370 | if (!node->global.inlined_to) | |
d826e131 | 1371 | { |
1372 | if (dump_file) | |
1373 | fprintf (dump_file, "Enqueueing calls of %s/%i.\n", | |
1374 | cgraph_node_name (node), node->uid); | |
1375 | ||
1376 | for (edge = node->callers; edge; edge = edge->next_caller) | |
1377 | if (edge->inline_failed | |
1378 | && can_inline_edge_p (edge, true) | |
1379 | && want_inline_small_function_p (edge, true) | |
1380 | && edge->inline_failed) | |
1381 | { | |
1382 | gcc_assert (!edge->aux); | |
1383 | update_edge_key (heap, edge); | |
1384 | } | |
1385 | } | |
1386 | ||
4055a556 | 1387 | gcc_assert (in_lto_p |
1388 | || !max_count | |
1389 | || (profile_info && flag_branch_probabilities)); | |
5c121ffe | 1390 | |
4055a556 | 1391 | while (!fibheap_empty (heap)) |
65c1a668 | 1392 | { |
97343302 | 1393 | int old_size = overall_size; |
022b3380 | 1394 | struct cgraph_node *where, *callee; |
1395 | int badness = fibheap_min_key (heap); | |
854efde4 | 1396 | int current_badness; |
60ac8a3c | 1397 | int cached_badness; |
022b3380 | 1398 | int growth; |
a49506c7 | 1399 | |
022b3380 | 1400 | edge = (struct cgraph_edge *) fibheap_extract_min (heap); |
1401 | gcc_assert (edge->aux); | |
1402 | edge->aux = NULL; | |
1403 | if (!edge->inline_failed) | |
1404 | continue; | |
854efde4 | 1405 | |
60ac8a3c | 1406 | /* Be sure that caches are maintained consistent. |
1407 | We can not make this ENABLE_CHECKING only because it cause differnt | |
1408 | updates of the fibheap queue. */ | |
1409 | cached_badness = edge_badness (edge, false); | |
ba5b0608 | 1410 | reset_edge_growth_cache (edge); |
1411 | reset_node_growth_cache (edge->callee); | |
ba5b0608 | 1412 | |
854efde4 | 1413 | /* When updating the edge costs, we only decrease badness in the keys. |
4055a556 | 1414 | Increases of badness are handled lazilly; when we see key with out |
1415 | of date value on it, we re-insert it now. */ | |
4869c23f | 1416 | current_badness = edge_badness (edge, false); |
60ac8a3c | 1417 | gcc_assert (cached_badness == current_badness); |
854efde4 | 1418 | gcc_assert (current_badness >= badness); |
1419 | if (current_badness != badness) | |
1420 | { | |
1421 | edge->aux = fibheap_insert (heap, current_badness, edge); | |
1422 | continue; | |
1423 | } | |
4869c23f | 1424 | |
1425 | if (!can_inline_edge_p (edge, true)) | |
1426 | continue; | |
854efde4 | 1427 | |
82626cb0 | 1428 | callee = cgraph_function_or_thunk_node (edge->callee, NULL); |
99c67f24 | 1429 | growth = estimate_edge_growth (edge); |
65c1a668 | 1430 | if (dump_file) |
65c1a668 | 1431 | { |
48e1416a | 1432 | fprintf (dump_file, |
97343302 | 1433 | "\nConsidering %s with %i size\n", |
82626cb0 | 1434 | cgraph_node_name (callee), |
1435 | inline_summary (callee)->size); | |
48e1416a | 1436 | fprintf (dump_file, |
bc6e5ec3 | 1437 | " to be inlined into %s in %s:%i\n" |
4869c23f | 1438 | " Estimated growth after inlined into all is %+i insns.\n" |
4ae20857 | 1439 | " Estimated badness is %i, frequency %.2f.\n", |
a49506c7 | 1440 | cgraph_node_name (edge->caller), |
6d61f3f9 | 1441 | flag_wpa ? "unknown" |
1442 | : gimple_filename ((const_gimple) edge->call_stmt), | |
4869c23f | 1443 | flag_wpa ? -1 |
1444 | : gimple_lineno ((const_gimple) edge->call_stmt), | |
82626cb0 | 1445 | estimate_growth (callee), |
022b3380 | 1446 | badness, |
4ae20857 | 1447 | edge->frequency / (double)CGRAPH_FREQ_BASE); |
a49506c7 | 1448 | if (edge->count) |
4869c23f | 1449 | fprintf (dump_file," Called "HOST_WIDEST_INT_PRINT_DEC"x\n", |
1450 | edge->count); | |
022b3380 | 1451 | if (dump_flags & TDF_DETAILS) |
4869c23f | 1452 | edge_badness (edge, true); |
65c1a668 | 1453 | } |
1454 | ||
4869c23f | 1455 | if (overall_size + growth > max_size |
7d0d0ce1 | 1456 | && !DECL_DISREGARD_INLINE_LIMITS (callee->symbol.decl)) |
a49506c7 | 1457 | { |
4869c23f | 1458 | edge->inline_failed = CIF_INLINE_UNIT_GROWTH_LIMIT; |
1459 | report_inline_failed_reason (edge); | |
a49506c7 | 1460 | continue; |
1461 | } | |
4869c23f | 1462 | |
1463 | if (!want_inline_small_function_p (edge, true)) | |
4055a556 | 1464 | continue; |
1465 | ||
1466 | /* Heuristics for inlining small functions works poorly for | |
1467 | recursive calls where we do efect similar to loop unrolling. | |
1468 | When inliing such edge seems profitable, leave decision on | |
1469 | specific inliner. */ | |
17c205c9 | 1470 | if (cgraph_edge_recursive_p (edge)) |
a49506c7 | 1471 | { |
1472 | where = edge->caller; | |
1473 | if (where->global.inlined_to) | |
1474 | where = where->global.inlined_to; | |
4869c23f | 1475 | if (!recursive_inlining (edge, |
1476 | flag_indirect_inlining | |
1477 | ? &new_indirect_edges : NULL)) | |
17c205c9 | 1478 | { |
1479 | edge->inline_failed = CIF_RECURSIVE_INLINING; | |
1480 | continue; | |
1481 | } | |
ba5b0608 | 1482 | reset_edge_caches (where); |
4055a556 | 1483 | /* Recursive inliner inlines all recursive calls of the function |
1484 | at once. Consequently we need to update all callee keys. */ | |
00e1f01e | 1485 | if (flag_indirect_inlining) |
f8daee9b | 1486 | add_new_edges_to_heap (heap, new_indirect_edges); |
9f3c2a90 | 1487 | update_all_callee_keys (heap, where, updated_nodes); |
a49506c7 | 1488 | } |
1489 | else | |
1490 | { | |
4869c23f | 1491 | struct cgraph_node *outer_node = NULL; |
1492 | int depth = 0; | |
1493 | ||
1494 | /* Consider the case where self recursive function A is inlined into B. | |
1495 | This is desired optimization in some cases, since it leads to effect | |
1496 | similar of loop peeling and we might completely optimize out the | |
1497 | recursive call. However we must be extra selective. */ | |
1498 | ||
1499 | where = edge->caller; | |
1500 | while (where->global.inlined_to) | |
a49506c7 | 1501 | { |
7d0d0ce1 | 1502 | if (where->symbol.decl == callee->symbol.decl) |
4869c23f | 1503 | outer_node = where, depth++; |
1504 | where = where->callers->caller; | |
1505 | } | |
1506 | if (outer_node | |
1507 | && !want_inline_self_recursive_call_p (edge, outer_node, | |
1508 | true, depth)) | |
1509 | { | |
1510 | edge->inline_failed | |
7d0d0ce1 | 1511 | = (DECL_DISREGARD_INLINE_LIMITS (edge->callee->symbol.decl) |
4869c23f | 1512 | ? CIF_RECURSIVE_INLINING : CIF_UNSPECIFIED); |
a49506c7 | 1513 | continue; |
1514 | } | |
4869c23f | 1515 | else if (depth && dump_file) |
1516 | fprintf (dump_file, " Peeling recursion with depth %i\n", depth); | |
1517 | ||
9f3c2a90 | 1518 | gcc_checking_assert (!callee->global.inlined_to); |
8cbc43ff | 1519 | inline_call (edge, true, &new_indirect_edges, &overall_size); |
00e1f01e | 1520 | if (flag_indirect_inlining) |
3f2ff969 | 1521 | add_new_edges_to_heap (heap, new_indirect_edges); |
1522 | ||
ba5b0608 | 1523 | reset_edge_caches (edge->callee); |
1524 | reset_node_growth_cache (callee); | |
1525 | ||
9f3c2a90 | 1526 | /* We inlined last offline copy to the body. This might lead |
1527 | to callees of function having fewer call sites and thus they | |
eb78a892 | 1528 | may need updating. |
1529 | ||
1530 | FIXME: the callee size could also shrink because more information | |
1531 | is propagated from caller. We don't track when this happen and | |
1532 | thus we need to recompute everything all the time. Once this is | |
1533 | solved, "|| 1" should go away. */ | |
1534 | if (callee->global.inlined_to || 1) | |
9f3c2a90 | 1535 | update_all_callee_keys (heap, callee, updated_nodes); |
1536 | else | |
1537 | update_callee_keys (heap, edge->callee, updated_nodes); | |
a49506c7 | 1538 | } |
1539 | where = edge->caller; | |
1540 | if (where->global.inlined_to) | |
1541 | where = where->global.inlined_to; | |
1542 | ||
1543 | /* Our profitability metric can depend on local properties | |
1544 | such as number of inlinable calls and size of the function body. | |
1545 | After inlining these properties might change for the function we | |
1546 | inlined into (since it's body size changed) and for the functions | |
1547 | called by function we inlined (since number of it inlinable callers | |
1548 | might change). */ | |
ba5b0608 | 1549 | update_caller_keys (heap, where, updated_nodes, NULL); |
022b3380 | 1550 | |
1551 | /* We removed one call of the function we just inlined. If offline | |
1552 | copy is still needed, be sure to update the keys. */ | |
1553 | if (callee != where && !callee->global.inlined_to) | |
ba5b0608 | 1554 | update_caller_keys (heap, callee, updated_nodes, NULL); |
a49506c7 | 1555 | bitmap_clear (updated_nodes); |
65c1a668 | 1556 | |
a49506c7 | 1557 | if (dump_file) |
71cadde7 | 1558 | { |
48e1416a | 1559 | fprintf (dump_file, |
ef725e2a | 1560 | " Inlined into %s which now has time %i and size %i," |
97343302 | 1561 | "net change of %+i.\n", |
71cadde7 | 1562 | cgraph_node_name (edge->caller), |
cbd7f5a0 | 1563 | inline_summary (edge->caller)->time, |
1564 | inline_summary (edge->caller)->size, | |
97343302 | 1565 | overall_size - old_size); |
71cadde7 | 1566 | } |
97343302 | 1567 | if (min_size > overall_size) |
5c121ffe | 1568 | { |
97343302 | 1569 | min_size = overall_size; |
1570 | max_size = compute_max_insns (min_size); | |
5c121ffe | 1571 | |
1572 | if (dump_file) | |
97343302 | 1573 | fprintf (dump_file, "New minimal size reached: %i\n", min_size); |
5c121ffe | 1574 | } |
65c1a668 | 1575 | } |
f8daee9b | 1576 | |
a41f2a28 | 1577 | free_growth_caches (); |
f8daee9b | 1578 | if (new_indirect_edges) |
1579 | VEC_free (cgraph_edge_p, heap, new_indirect_edges); | |
65c1a668 | 1580 | fibheap_delete (heap); |
4055a556 | 1581 | if (dump_file) |
1582 | fprintf (dump_file, | |
1583 | "Unit growth for small function inlining: %i->%i (%i%%)\n", | |
a41f2a28 | 1584 | initial_size, overall_size, |
1585 | initial_size ? overall_size * 100 / (initial_size) - 100: 0); | |
a49506c7 | 1586 | BITMAP_FREE (updated_nodes); |
65c1a668 | 1587 | } |
1588 | ||
4055a556 | 1589 | /* Flatten NODE. Performed both during early inlining and |
1590 | at IPA inlining time. */ | |
d160af41 | 1591 | |
1592 | static void | |
a41f2a28 | 1593 | flatten_function (struct cgraph_node *node, bool early) |
d160af41 | 1594 | { |
1595 | struct cgraph_edge *e; | |
1596 | ||
1597 | /* We shouldn't be called recursively when we are being processed. */ | |
7d0d0ce1 | 1598 | gcc_assert (node->symbol.aux == NULL); |
d160af41 | 1599 | |
7d0d0ce1 | 1600 | node->symbol.aux = (void *) node; |
d160af41 | 1601 | |
1602 | for (e = node->callees; e; e = e->next_callee) | |
1603 | { | |
1604 | struct cgraph_node *orig_callee; | |
82626cb0 | 1605 | struct cgraph_node *callee = cgraph_function_or_thunk_node (e->callee, NULL); |
d160af41 | 1606 | |
d160af41 | 1607 | /* We've hit cycle? It is time to give up. */ |
7d0d0ce1 | 1608 | if (callee->symbol.aux) |
d160af41 | 1609 | { |
1610 | if (dump_file) | |
1611 | fprintf (dump_file, | |
1612 | "Not inlining %s into %s to avoid cycle.\n", | |
82626cb0 | 1613 | cgraph_node_name (callee), |
d160af41 | 1614 | cgraph_node_name (e->caller)); |
1615 | e->inline_failed = CIF_RECURSIVE_INLINING; | |
1616 | continue; | |
1617 | } | |
1618 | ||
1619 | /* When the edge is already inlined, we just need to recurse into | |
1620 | it in order to fully flatten the leaves. */ | |
1621 | if (!e->inline_failed) | |
1622 | { | |
82626cb0 | 1623 | flatten_function (callee, early); |
d160af41 | 1624 | continue; |
1625 | } | |
1626 | ||
4869c23f | 1627 | /* Flatten attribute needs to be processed during late inlining. For |
1628 | extra code quality we however do flattening during early optimization, | |
1629 | too. */ | |
a41f2a28 | 1630 | if (!early |
4869c23f | 1631 | ? !can_inline_edge_p (e, true) |
1632 | : !can_early_inline_edge_p (e)) | |
1633 | continue; | |
1634 | ||
17c205c9 | 1635 | if (cgraph_edge_recursive_p (e)) |
d160af41 | 1636 | { |
1637 | if (dump_file) | |
1638 | fprintf (dump_file, "Not inlining: recursive call.\n"); | |
1639 | continue; | |
1640 | } | |
1641 | ||
7d0d0ce1 | 1642 | if (gimple_in_ssa_p (DECL_STRUCT_FUNCTION (node->symbol.decl)) |
1643 | != gimple_in_ssa_p (DECL_STRUCT_FUNCTION (callee->symbol.decl))) | |
ae576fce | 1644 | { |
1645 | if (dump_file) | |
1646 | fprintf (dump_file, "Not inlining: SSA form does not match.\n"); | |
1647 | continue; | |
1648 | } | |
1649 | ||
d160af41 | 1650 | /* Inline the edge and flatten the inline clone. Avoid |
1651 | recursing through the original node if the node was cloned. */ | |
1652 | if (dump_file) | |
1653 | fprintf (dump_file, " Inlining %s into %s.\n", | |
82626cb0 | 1654 | cgraph_node_name (callee), |
d160af41 | 1655 | cgraph_node_name (e->caller)); |
82626cb0 | 1656 | orig_callee = callee; |
8cbc43ff | 1657 | inline_call (e, true, NULL, NULL); |
d160af41 | 1658 | if (e->callee != orig_callee) |
7d0d0ce1 | 1659 | orig_callee->symbol.aux = (void *) node; |
a41f2a28 | 1660 | flatten_function (e->callee, early); |
d160af41 | 1661 | if (e->callee != orig_callee) |
7d0d0ce1 | 1662 | orig_callee->symbol.aux = NULL; |
d160af41 | 1663 | } |
1664 | ||
7d0d0ce1 | 1665 | node->symbol.aux = NULL; |
d160af41 | 1666 | } |
1667 | ||
65c1a668 | 1668 | /* Decide on the inlining. We do so in the topological order to avoid |
1669 | expenses on updating data structures. */ | |
1670 | ||
2a1990e9 | 1671 | static unsigned int |
4869c23f | 1672 | ipa_inline (void) |
65c1a668 | 1673 | { |
1674 | struct cgraph_node *node; | |
1675 | int nnodes; | |
1676 | struct cgraph_node **order = | |
4c36ffe6 | 1677 | XCNEWVEC (struct cgraph_node *, cgraph_n_nodes); |
65c1a668 | 1678 | int i; |
1679 | ||
a226c368 | 1680 | if (in_lto_p && optimize) |
8867b500 | 1681 | ipa_update_after_lto_read (); |
9ca785fc | 1682 | |
c7b2cc59 | 1683 | if (dump_file) |
1684 | dump_inline_summaries (dump_file); | |
a49506c7 | 1685 | |
7771d558 | 1686 | nnodes = ipa_reverse_postorder (order); |
65c1a668 | 1687 | |
7c455d87 | 1688 | FOR_EACH_FUNCTION (node) |
7d0d0ce1 | 1689 | node->symbol.aux = 0; |
65c1a668 | 1690 | |
1691 | if (dump_file) | |
d160af41 | 1692 | fprintf (dump_file, "\nFlattening functions:\n"); |
65c1a668 | 1693 | |
d160af41 | 1694 | /* In the first pass handle functions to be flattened. Do this with |
1695 | a priority so none of our later choices will make this impossible. */ | |
1696 | for (i = nnodes - 1; i >= 0; i--) | |
65c1a668 | 1697 | { |
d160af41 | 1698 | node = order[i]; |
1699 | ||
4055a556 | 1700 | /* Handle nodes to be flattened. |
d160af41 | 1701 | Ideally when processing callees we stop inlining at the |
1702 | entry of cycles, possibly cloning that entry point and | |
1703 | try to flatten itself turning it into a self-recursive | |
1704 | function. */ | |
1705 | if (lookup_attribute ("flatten", | |
7d0d0ce1 | 1706 | DECL_ATTRIBUTES (node->symbol.decl)) != NULL) |
3f2ff969 | 1707 | { |
65c1a668 | 1708 | if (dump_file) |
48e1416a | 1709 | fprintf (dump_file, |
d160af41 | 1710 | "Flattening %s\n", cgraph_node_name (node)); |
a41f2a28 | 1711 | flatten_function (node, false); |
65c1a668 | 1712 | } |
65c1a668 | 1713 | } |
1714 | ||
4869c23f | 1715 | inline_small_functions (); |
1716 | cgraph_remove_unreachable_nodes (true, dump_file); | |
1717 | free (order); | |
65c1a668 | 1718 | |
4869c23f | 1719 | /* We already perform some inlining of functions called once during |
1720 | inlining small functions above. After unreachable nodes are removed, | |
1721 | we still might do a quick check that nothing new is found. */ | |
1c2f0012 | 1722 | if (flag_inline_functions_called_once) |
f1aa280c | 1723 | { |
4055a556 | 1724 | int cold; |
65c1a668 | 1725 | if (dump_file) |
1726 | fprintf (dump_file, "\nDeciding on functions called once:\n"); | |
1727 | ||
4055a556 | 1728 | /* Inlining one function called once has good chance of preventing |
1729 | inlining other function into the same callee. Ideally we should | |
1730 | work in priority order, but probably inlining hot functions first | |
1731 | is good cut without the extra pain of maintaining the queue. | |
1732 | ||
1733 | ??? this is not really fitting the bill perfectly: inlining function | |
1734 | into callee often leads to better optimization of callee due to | |
1735 | increased context for optimization. | |
1736 | For example if main() function calls a function that outputs help | |
1737 | and then function that does the main optmization, we should inline | |
1738 | the second with priority even if both calls are cold by themselves. | |
1739 | ||
1740 | We probably want to implement new predicate replacing our use of | |
1741 | maybe_hot_edge interpreted as maybe_hot_edge || callee is known | |
1742 | to be hot. */ | |
1743 | for (cold = 0; cold <= 1; cold ++) | |
65c1a668 | 1744 | { |
7c455d87 | 1745 | FOR_EACH_DEFINED_FUNCTION (node) |
65c1a668 | 1746 | { |
4055a556 | 1747 | if (want_inline_function_called_once_p (node) |
1748 | && (cold | |
1749 | || cgraph_maybe_hot_edge_p (node->callers))) | |
a30b29a7 | 1750 | { |
4055a556 | 1751 | struct cgraph_node *caller = node->callers->caller; |
1752 | ||
1753 | if (dump_file) | |
1754 | { | |
1755 | fprintf (dump_file, | |
1756 | "\nInlining %s size %i.\n", | |
1757 | cgraph_node_name (node), inline_summary (node)->size); | |
1758 | fprintf (dump_file, | |
1759 | " Called once from %s %i insns.\n", | |
1760 | cgraph_node_name (node->callers->caller), | |
1761 | inline_summary (node->callers->caller)->size); | |
1762 | } | |
1763 | ||
8cbc43ff | 1764 | inline_call (node->callers, true, NULL, NULL); |
4055a556 | 1765 | if (dump_file) |
1766 | fprintf (dump_file, | |
1767 | " Inlined into %s which now has %i size\n", | |
1768 | cgraph_node_name (caller), | |
1769 | inline_summary (caller)->size); | |
a30b29a7 | 1770 | } |
65c1a668 | 1771 | } |
1772 | } | |
1773 | } | |
1774 | ||
3f2ff969 | 1775 | /* Free ipa-prop structures if they are no longer needed. */ |
a226c368 | 1776 | if (optimize) |
799c8711 | 1777 | ipa_free_all_structures_after_iinln (); |
3f2ff969 | 1778 | |
65c1a668 | 1779 | if (dump_file) |
1780 | fprintf (dump_file, | |
4055a556 | 1781 | "\nInlined %i calls, eliminated %i functions\n\n", |
1782 | ncalls_inlined, nfunctions_inlined); | |
1783 | ||
0835ad03 | 1784 | if (dump_file) |
1785 | dump_inline_summaries (dump_file); | |
c7b2cc59 | 1786 | /* In WPA we use inline summaries for partitioning process. */ |
1787 | if (!flag_wpa) | |
1788 | inline_free_summary (); | |
2a1990e9 | 1789 | return 0; |
65c1a668 | 1790 | } |
1791 | ||
cd800728 | 1792 | /* Inline always-inline function calls in NODE. */ |
1793 | ||
1794 | static bool | |
4869c23f | 1795 | inline_always_inline_functions (struct cgraph_node *node) |
cd800728 | 1796 | { |
1797 | struct cgraph_edge *e; | |
1798 | bool inlined = false; | |
1799 | ||
1800 | for (e = node->callees; e; e = e->next_callee) | |
1801 | { | |
82626cb0 | 1802 | struct cgraph_node *callee = cgraph_function_or_thunk_node (e->callee, NULL); |
7d0d0ce1 | 1803 | if (!DECL_DISREGARD_INLINE_LIMITS (callee->symbol.decl)) |
cd800728 | 1804 | continue; |
1805 | ||
cd800728 | 1806 | if (cgraph_edge_recursive_p (e)) |
1807 | { | |
1808 | if (dump_file) | |
4869c23f | 1809 | fprintf (dump_file, " Not inlining recursive call to %s.\n", |
1810 | cgraph_node_name (e->callee)); | |
cd800728 | 1811 | e->inline_failed = CIF_RECURSIVE_INLINING; |
1812 | continue; | |
1813 | } | |
1814 | ||
4869c23f | 1815 | if (!can_early_inline_edge_p (e)) |
cd800728 | 1816 | continue; |
1817 | ||
1818 | if (dump_file) | |
4869c23f | 1819 | fprintf (dump_file, " Inlining %s into %s (always_inline).\n", |
cd800728 | 1820 | cgraph_node_name (e->callee), |
1821 | cgraph_node_name (e->caller)); | |
8cbc43ff | 1822 | inline_call (e, true, NULL, NULL); |
cd800728 | 1823 | inlined = true; |
1824 | } | |
1825 | ||
1826 | return inlined; | |
1827 | } | |
1828 | ||
65c1a668 | 1829 | /* Decide on the inlining. We do so in the topological order to avoid |
d160af41 | 1830 | expenses on updating data structures. */ |
65c1a668 | 1831 | |
436a2379 | 1832 | static bool |
4869c23f | 1833 | early_inline_small_functions (struct cgraph_node *node) |
65c1a668 | 1834 | { |
1835 | struct cgraph_edge *e; | |
9e0baf4d | 1836 | bool inlined = false; |
436a2379 | 1837 | |
cd800728 | 1838 | for (e = node->callees; e; e = e->next_callee) |
a223d5ed | 1839 | { |
82626cb0 | 1840 | struct cgraph_node *callee = cgraph_function_or_thunk_node (e->callee, NULL); |
1841 | if (!inline_summary (callee)->inlinable | |
4869c23f | 1842 | || !e->inline_failed) |
cd800728 | 1843 | continue; |
1844 | ||
1845 | /* Do not consider functions not declared inline. */ | |
7d0d0ce1 | 1846 | if (!DECL_DECLARED_INLINE_P (callee->symbol.decl) |
cd800728 | 1847 | && !flag_inline_small_functions |
1848 | && !flag_inline_functions) | |
1849 | continue; | |
1850 | ||
a223d5ed | 1851 | if (dump_file) |
cd800728 | 1852 | fprintf (dump_file, "Considering inline candidate %s.\n", |
82626cb0 | 1853 | cgraph_node_name (callee)); |
65c1a668 | 1854 | |
4869c23f | 1855 | if (!can_early_inline_edge_p (e)) |
1856 | continue; | |
1857 | ||
cd800728 | 1858 | if (cgraph_edge_recursive_p (e)) |
1859 | { | |
1860 | if (dump_file) | |
4869c23f | 1861 | fprintf (dump_file, " Not inlining: recursive call.\n"); |
f41629b6 | 1862 | continue; |
cd800728 | 1863 | } |
d160af41 | 1864 | |
4869c23f | 1865 | if (!want_early_inline_function_p (e)) |
cd800728 | 1866 | continue; |
65c1a668 | 1867 | |
4869c23f | 1868 | if (dump_file) |
1869 | fprintf (dump_file, " Inlining %s into %s.\n", | |
82626cb0 | 1870 | cgraph_node_name (callee), |
4869c23f | 1871 | cgraph_node_name (e->caller)); |
8cbc43ff | 1872 | inline_call (e, true, NULL, NULL); |
4869c23f | 1873 | inlined = true; |
00efe249 | 1874 | } |
cd800728 | 1875 | |
436a2379 | 1876 | return inlined; |
65c1a668 | 1877 | } |
1878 | ||
9e0baf4d | 1879 | /* Do inlining of small functions. Doing so early helps profiling and other |
1880 | passes to be somewhat more effective and avoids some code duplication in | |
1881 | later real inlining pass for testcases with very many function calls. */ | |
2a1990e9 | 1882 | static unsigned int |
4869c23f | 1883 | early_inliner (void) |
9e0baf4d | 1884 | { |
fd6a3c41 | 1885 | struct cgraph_node *node = cgraph_get_node (current_function_decl); |
c7b2cc59 | 1886 | struct cgraph_edge *edge; |
436a2379 | 1887 | unsigned int todo = 0; |
a7b61d8c | 1888 | int iterations = 0; |
cd800728 | 1889 | bool inlined = false; |
9e0baf4d | 1890 | |
852f689e | 1891 | if (seen_error ()) |
2a1990e9 | 1892 | return 0; |
d160af41 | 1893 | |
9da15f94 | 1894 | /* Do nothing if datastructures for ipa-inliner are already computed. This |
1895 | happens when some pass decides to construct new function and | |
1896 | cgraph_add_new_function calls lowering passes and early optimization on | |
1897 | it. This may confuse ourself when early inliner decide to inline call to | |
1898 | function clone, because function clones don't have parameter list in | |
1899 | ipa-prop matching their signature. */ | |
1900 | if (ipa_node_params_vector) | |
1901 | return 0; | |
1902 | ||
cd800728 | 1903 | #ifdef ENABLE_CHECKING |
1904 | verify_cgraph_node (node); | |
1905 | #endif | |
1906 | ||
1907 | /* Even when not optimizing or not inlining inline always-inline | |
1908 | functions. */ | |
4869c23f | 1909 | inlined = inline_always_inline_functions (node); |
cd800728 | 1910 | |
d160af41 | 1911 | if (!optimize |
1912 | || flag_no_inline | |
4869c23f | 1913 | || !flag_early_inlining |
1914 | /* Never inline regular functions into always-inline functions | |
1915 | during incremental inlining. This sucks as functions calling | |
1916 | always inline functions will get less optimized, but at the | |
1917 | same time inlining of functions calling always inline | |
4055a556 | 1918 | function into an always inline function might introduce |
4869c23f | 1919 | cycles of edges to be always inlined in the callgraph. |
1920 | ||
1921 | We might want to be smarter and just avoid this type of inlining. */ | |
7d0d0ce1 | 1922 | || DECL_DISREGARD_INLINE_LIMITS (node->symbol.decl)) |
cd800728 | 1923 | ; |
1924 | else if (lookup_attribute ("flatten", | |
7d0d0ce1 | 1925 | DECL_ATTRIBUTES (node->symbol.decl)) != NULL) |
436a2379 | 1926 | { |
cd800728 | 1927 | /* When the function is marked to be flattened, recursively inline |
1928 | all calls in it. */ | |
1929 | if (dump_file) | |
1930 | fprintf (dump_file, | |
1931 | "Flattening %s\n", cgraph_node_name (node)); | |
a41f2a28 | 1932 | flatten_function (node, true); |
cd800728 | 1933 | inlined = true; |
436a2379 | 1934 | } |
d160af41 | 1935 | else |
1936 | { | |
1937 | /* We iterate incremental inlining to get trivial cases of indirect | |
1938 | inlining. */ | |
1939 | while (iterations < PARAM_VALUE (PARAM_EARLY_INLINER_MAX_ITERATIONS) | |
4869c23f | 1940 | && early_inline_small_functions (node)) |
d160af41 | 1941 | { |
1942 | timevar_push (TV_INTEGRATION); | |
1943 | todo |= optimize_inline_calls (current_function_decl); | |
4869c23f | 1944 | |
1945 | /* Technically we ought to recompute inline parameters so the new | |
1946 | iteration of early inliner works as expected. We however have | |
1947 | values approximately right and thus we only need to update edge | |
1948 | info that might be cleared out for newly discovered edges. */ | |
1949 | for (edge = node->callees; edge; edge = edge->next_callee) | |
1950 | { | |
0835ad03 | 1951 | struct inline_edge_summary *es = inline_edge_summary (edge); |
1952 | es->call_stmt_size | |
4869c23f | 1953 | = estimate_num_insns (edge->call_stmt, &eni_size_weights); |
0835ad03 | 1954 | es->call_stmt_time |
4869c23f | 1955 | = estimate_num_insns (edge->call_stmt, &eni_time_weights); |
7d0d0ce1 | 1956 | if (edge->callee->symbol.decl |
f883da84 | 1957 | && !gimple_check_call_matching_types (edge->call_stmt, |
7d0d0ce1 | 1958 | edge->callee->symbol.decl)) |
f883da84 | 1959 | edge->call_stmt_cannot_inline_p = true; |
4869c23f | 1960 | } |
d160af41 | 1961 | timevar_pop (TV_INTEGRATION); |
cd800728 | 1962 | iterations++; |
1963 | inlined = false; | |
d160af41 | 1964 | } |
1965 | if (dump_file) | |
1966 | fprintf (dump_file, "Iterations: %i\n", iterations); | |
1967 | } | |
1968 | ||
cd800728 | 1969 | if (inlined) |
1970 | { | |
1971 | timevar_push (TV_INTEGRATION); | |
1972 | todo |= optimize_inline_calls (current_function_decl); | |
1973 | timevar_pop (TV_INTEGRATION); | |
1974 | } | |
1975 | ||
198622c0 | 1976 | cfun->always_inline_functions_inlined = true; |
9e0baf4d | 1977 | |
d160af41 | 1978 | return todo; |
9e0baf4d | 1979 | } |
1980 | ||
48e1416a | 1981 | struct gimple_opt_pass pass_early_inline = |
9e0baf4d | 1982 | { |
20099e35 | 1983 | { |
1984 | GIMPLE_PASS, | |
9e0baf4d | 1985 | "einline", /* name */ |
d160af41 | 1986 | NULL, /* gate */ |
4869c23f | 1987 | early_inliner, /* execute */ |
9e0baf4d | 1988 | NULL, /* sub */ |
1989 | NULL, /* next */ | |
1990 | 0, /* static_pass_number */ | |
f37a5008 | 1991 | TV_INLINE_HEURISTICS, /* tv_id */ |
cd800728 | 1992 | PROP_ssa, /* properties_required */ |
41709826 | 1993 | 0, /* properties_provided */ |
65c1a668 | 1994 | 0, /* properties_destroyed */ |
1995 | 0, /* todo_flags_start */ | |
771e2890 | 1996 | 0 /* todo_flags_finish */ |
20099e35 | 1997 | } |
09a2e412 | 1998 | }; |
1999 | ||
09a2e412 | 2000 | |
d160af41 | 2001 | /* When to run IPA inlining. Inlining of always-inline functions |
657e3a56 | 2002 | happens during early inlining. |
2003 | ||
2004 | Enable inlining unconditoinally at -flto. We need size estimates to | |
2005 | drive partitioning. */ | |
d160af41 | 2006 | |
2007 | static bool | |
4869c23f | 2008 | gate_ipa_inline (void) |
d160af41 | 2009 | { |
657e3a56 | 2010 | return optimize || flag_lto || flag_wpa; |
d160af41 | 2011 | } |
2012 | ||
26dbec0a | 2013 | struct ipa_opt_pass_d pass_ipa_inline = |
09a2e412 | 2014 | { |
20099e35 | 2015 | { |
68e3904e | 2016 | IPA_PASS, |
2017 | "inline", /* name */ | |
4869c23f | 2018 | gate_ipa_inline, /* gate */ |
2019 | ipa_inline, /* execute */ | |
09a2e412 | 2020 | NULL, /* sub */ |
2021 | NULL, /* next */ | |
2022 | 0, /* static_pass_number */ | |
2023 | TV_INLINE_HEURISTICS, /* tv_id */ | |
2024 | 0, /* properties_required */ | |
41709826 | 2025 | 0, /* properties_provided */ |
09a2e412 | 2026 | 0, /* properties_destroyed */ |
68e3904e | 2027 | TODO_remove_functions, /* todo_flags_finish */ |
18841b0c | 2028 | TODO_dump_symtab |
57305941 | 2029 | | TODO_remove_functions | TODO_ggc_collect /* todo_flags_finish */ |
68e3904e | 2030 | }, |
9c1bff7a | 2031 | inline_generate_summary, /* generate_summary */ |
8867b500 | 2032 | inline_write_summary, /* write_summary */ |
2033 | inline_read_summary, /* read_summary */ | |
ddc90d88 | 2034 | NULL, /* write_optimization_summary */ |
2035 | NULL, /* read_optimization_summary */ | |
799c8711 | 2036 | NULL, /* stmt_fixup */ |
68e3904e | 2037 | 0, /* TODOs */ |
2038 | inline_transform, /* function_transform */ | |
2039 | NULL, /* variable_transform */ | |
65c1a668 | 2040 | }; |