1 /* Inlining decision heuristics.
2 Copyright (C) 2003-2019 Free Software Foundation, Inc.
3 Contributed by Jan Hubicka
5 This file is part of GCC.
7 GCC is free software; you can redistribute it and/or modify it under
8 the terms of the GNU General Public License as published by the Free
9 Software Foundation; either version 3, or (at your option) any later
12 GCC is distributed in the hope that it will be useful, but WITHOUT ANY
13 WARRANTY; without even the implied warranty of MERCHANTABILITY or
14 FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
17 You should have received a copy of the GNU General Public License
18 along with GCC; see the file COPYING3. If not see
19 <http://www.gnu.org/licenses/>. */
21 /* Inlining decision heuristics
23 The implementation of inliner is organized as follows:
25 inlining heuristics limits
27 can_inline_edge_p allow to check that particular inlining is allowed
28 by the limits specified by user (allowed function growth, growth and so
31 Functions are inlined when it is obvious the result is profitable (such
32 as functions called once or when inlining reduce code size).
33 In addition to that we perform inlining of small functions and recursive
38 The inliner itself is split into two passes:
42 Simple local inlining pass inlining callees into current function.
43 This pass makes no use of whole unit analysis and thus it can do only
44 very simple decisions based on local properties.
46 The strength of the pass is that it is run in topological order
47 (reverse postorder) on the callgraph. Functions are converted into SSA
48 form just before this pass and optimized subsequently. As a result, the
49 callees of the function seen by the early inliner was already optimized
50 and results of early inlining adds a lot of optimization opportunities
51 for the local optimization.
53 The pass handle the obvious inlining decisions within the compilation
54 unit - inlining auto inline functions, inlining for size and
57 main strength of the pass is the ability to eliminate abstraction
58 penalty in C++ code (via combination of inlining and early
59 optimization) and thus improve quality of analysis done by real IPA
62 Because of lack of whole unit knowledge, the pass cannot really make
63 good code size/performance tradeoffs. It however does very simple
64 speculative inlining allowing code size to grow by
65 EARLY_INLINING_INSNS when callee is leaf function. In this case the
66 optimizations performed later are very likely to eliminate the cost.
70 This is the real inliner able to handle inlining with whole program
71 knowledge. It performs following steps:
73 1) inlining of small functions. This is implemented by greedy
74 algorithm ordering all inlinable cgraph edges by their badness and
75 inlining them in this order as long as inline limits allows doing so.
77 This heuristics is not very good on inlining recursive calls. Recursive
78 calls can be inlined with results similar to loop unrolling. To do so,
79 special purpose recursive inliner is executed on function when
80 recursive edge is met as viable candidate.
82 2) Unreachable functions are removed from callgraph. Inlining leads
83 to devirtualization and other modification of callgraph so functions
84 may become unreachable during the process. Also functions declared as
85 extern inline or virtual functions are removed, since after inlining
86 we no longer need the offline bodies.
88 3) Functions called once and not exported from the unit are inlined.
89 This should almost always lead to reduction of code size by eliminating
90 the need for offline copy of the function. */
94 #include "coretypes.h"
100 #include "alloc-pool.h"
101 #include "tree-pass.h"
102 #include "gimple-ssa.h"
104 #include "lto-streamer.h"
105 #include "trans-mem.h"
107 #include "tree-inline.h"
110 #include "symbol-summary.h"
111 #include "tree-vrp.h"
112 #include "ipa-prop.h"
113 #include "ipa-fnsummary.h"
114 #include "ipa-inline.h"
115 #include "ipa-utils.h"
117 #include "auto-profile.h"
118 #include "builtins.h"
119 #include "fibonacci_heap.h"
120 #include "stringpool.h"
124 typedef fibonacci_heap
<sreal
, cgraph_edge
> edge_heap_t
;
125 typedef fibonacci_node
<sreal
, cgraph_edge
> edge_heap_node_t
;
127 /* Statistics we collect about inlining algorithm. */
128 static int overall_size
;
129 static profile_count max_count
;
130 static profile_count spec_rem
;
132 /* Return false when inlining edge E would lead to violating
133 limits on function unit growth or stack usage growth.
135 The relative function body growth limit is present generally
136 to avoid problems with non-linear behavior of the compiler.
137 To allow inlining huge functions into tiny wrapper, the limit
138 is always based on the bigger of the two functions considered.
140 For stack growth limits we always base the growth in stack usage
141 of the callers. We want to prevent applications from segfaulting
142 on stack overflow when functions with huge stack frames gets
146 caller_growth_limits (struct cgraph_edge
*e
)
148 struct cgraph_node
*to
= e
->caller
;
149 struct cgraph_node
*what
= e
->callee
->ultimate_alias_target ();
152 HOST_WIDE_INT stack_size_limit
= 0, inlined_stack
;
153 ipa_size_summary
*outer_info
= ipa_size_summaries
->get (to
);
155 /* Look for function e->caller is inlined to. While doing
156 so work out the largest function body on the way. As
157 described above, we want to base our function growth
158 limits based on that. Not on the self size of the
159 outer function, not on the self size of inline code
160 we immediately inline to. This is the most relaxed
161 interpretation of the rule "do not grow large functions
162 too much in order to prevent compiler from exploding". */
165 ipa_size_summary
*size_info
= ipa_size_summaries
->get (to
);
166 if (limit
< size_info
->self_size
)
167 limit
= size_info
->self_size
;
168 if (stack_size_limit
< size_info
->estimated_self_stack_size
)
169 stack_size_limit
= size_info
->estimated_self_stack_size
;
171 to
= to
->callers
->caller
;
176 ipa_fn_summary
*what_info
= ipa_fn_summaries
->get (what
);
177 ipa_size_summary
*what_size_info
= ipa_size_summaries
->get (what
);
179 if (limit
< what_size_info
->self_size
)
180 limit
= what_size_info
->self_size
;
182 limit
+= limit
* PARAM_VALUE (PARAM_LARGE_FUNCTION_GROWTH
) / 100;
184 /* Check the size after inlining against the function limits. But allow
185 the function to shrink if it went over the limits by forced inlining. */
186 newsize
= estimate_size_after_inlining (to
, e
);
187 if (newsize
>= ipa_size_summaries
->get (what
)->size
188 && newsize
> PARAM_VALUE (PARAM_LARGE_FUNCTION_INSNS
)
191 e
->inline_failed
= CIF_LARGE_FUNCTION_GROWTH_LIMIT
;
195 if (!what_info
->estimated_stack_size
)
198 /* FIXME: Stack size limit often prevents inlining in Fortran programs
199 due to large i/o datastructures used by the Fortran front-end.
200 We ought to ignore this limit when we know that the edge is executed
201 on every invocation of the caller (i.e. its call statement dominates
202 exit block). We do not track this information, yet. */
203 stack_size_limit
+= ((gcov_type
)stack_size_limit
204 * PARAM_VALUE (PARAM_STACK_FRAME_GROWTH
) / 100);
206 inlined_stack
= (ipa_get_stack_frame_offset (to
)
207 + outer_info
->estimated_self_stack_size
208 + what_info
->estimated_stack_size
);
209 /* Check new stack consumption with stack consumption at the place
211 if (inlined_stack
> stack_size_limit
212 /* If function already has large stack usage from sibling
213 inline call, we can inline, too.
214 This bit overoptimistically assume that we are good at stack
216 && inlined_stack
> ipa_fn_summaries
->get (to
)->estimated_stack_size
217 && inlined_stack
> PARAM_VALUE (PARAM_LARGE_STACK_FRAME
))
219 e
->inline_failed
= CIF_LARGE_STACK_FRAME_GROWTH_LIMIT
;
225 /* Dump info about why inlining has failed. */
228 report_inline_failed_reason (struct cgraph_edge
*e
)
230 if (dump_enabled_p ())
232 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, e
->call_stmt
,
233 " not inlinable: %C -> %C, %s\n",
234 e
->caller
, e
->callee
,
235 cgraph_inline_failed_string (e
->inline_failed
));
236 if ((e
->inline_failed
== CIF_TARGET_OPTION_MISMATCH
237 || e
->inline_failed
== CIF_OPTIMIZATION_MISMATCH
)
238 && e
->caller
->lto_file_data
239 && e
->callee
->ultimate_alias_target ()->lto_file_data
)
241 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, e
->call_stmt
,
242 " LTO objects: %s, %s\n",
243 e
->caller
->lto_file_data
->file_name
,
244 e
->callee
->ultimate_alias_target ()->lto_file_data
->file_name
);
246 if (e
->inline_failed
== CIF_TARGET_OPTION_MISMATCH
)
248 cl_target_option_print_diff
249 (dump_file
, 2, target_opts_for_fn (e
->caller
->decl
),
250 target_opts_for_fn (e
->callee
->ultimate_alias_target ()->decl
));
251 if (e
->inline_failed
== CIF_OPTIMIZATION_MISMATCH
)
253 cl_optimization_print_diff
254 (dump_file
, 2, opts_for_fn (e
->caller
->decl
),
255 opts_for_fn (e
->callee
->ultimate_alias_target ()->decl
));
259 /* Decide whether sanitizer-related attributes allow inlining. */
262 sanitize_attrs_match_for_inline_p (const_tree caller
, const_tree callee
)
264 if (!caller
|| !callee
)
267 /* Allow inlining always_inline functions into no_sanitize_address
269 if (!sanitize_flags_p (SANITIZE_ADDRESS
, caller
)
270 && lookup_attribute ("always_inline", DECL_ATTRIBUTES (callee
)))
273 return ((sanitize_flags_p (SANITIZE_ADDRESS
, caller
)
274 == sanitize_flags_p (SANITIZE_ADDRESS
, callee
))
275 && (sanitize_flags_p (SANITIZE_POINTER_COMPARE
, caller
)
276 == sanitize_flags_p (SANITIZE_POINTER_COMPARE
, callee
))
277 && (sanitize_flags_p (SANITIZE_POINTER_SUBTRACT
, caller
)
278 == sanitize_flags_p (SANITIZE_POINTER_SUBTRACT
, callee
)));
281 /* Used for flags where it is safe to inline when caller's value is
282 grater than callee's. */
283 #define check_maybe_up(flag) \
284 (opts_for_fn (caller->decl)->x_##flag \
285 != opts_for_fn (callee->decl)->x_##flag \
287 || opts_for_fn (caller->decl)->x_##flag \
288 < opts_for_fn (callee->decl)->x_##flag))
289 /* Used for flags where it is safe to inline when caller's value is
290 smaller than callee's. */
291 #define check_maybe_down(flag) \
292 (opts_for_fn (caller->decl)->x_##flag \
293 != opts_for_fn (callee->decl)->x_##flag \
295 || opts_for_fn (caller->decl)->x_##flag \
296 > opts_for_fn (callee->decl)->x_##flag))
297 /* Used for flags where exact match is needed for correctness. */
298 #define check_match(flag) \
299 (opts_for_fn (caller->decl)->x_##flag \
300 != opts_for_fn (callee->decl)->x_##flag)
302 /* Decide if we can inline the edge and possibly update
303 inline_failed reason.
304 We check whether inlining is possible at all and whether
305 caller growth limits allow doing so.
307 if REPORT is true, output reason to the dump file. */
310 can_inline_edge_p (struct cgraph_edge
*e
, bool report
,
313 gcc_checking_assert (e
->inline_failed
);
315 if (cgraph_inline_failed_type (e
->inline_failed
) == CIF_FINAL_ERROR
)
318 report_inline_failed_reason (e
);
322 bool inlinable
= true;
323 enum availability avail
;
324 cgraph_node
*caller
= (e
->caller
->inlined_to
325 ? e
->caller
->inlined_to
: e
->caller
);
326 cgraph_node
*callee
= e
->callee
->ultimate_alias_target (&avail
, caller
);
328 if (!callee
->definition
)
330 e
->inline_failed
= CIF_BODY_NOT_AVAILABLE
;
333 if (!early
&& (!opt_for_fn (callee
->decl
, optimize
)
334 || !opt_for_fn (caller
->decl
, optimize
)))
336 e
->inline_failed
= CIF_FUNCTION_NOT_OPTIMIZED
;
339 else if (callee
->calls_comdat_local
)
341 e
->inline_failed
= CIF_USES_COMDAT_LOCAL
;
344 else if (avail
<= AVAIL_INTERPOSABLE
)
346 e
->inline_failed
= CIF_OVERWRITABLE
;
349 /* All edges with call_stmt_cannot_inline_p should have inline_failed
350 initialized to one of FINAL_ERROR reasons. */
351 else if (e
->call_stmt_cannot_inline_p
)
353 /* Don't inline if the functions have different EH personalities. */
354 else if (DECL_FUNCTION_PERSONALITY (caller
->decl
)
355 && DECL_FUNCTION_PERSONALITY (callee
->decl
)
356 && (DECL_FUNCTION_PERSONALITY (caller
->decl
)
357 != DECL_FUNCTION_PERSONALITY (callee
->decl
)))
359 e
->inline_failed
= CIF_EH_PERSONALITY
;
362 /* TM pure functions should not be inlined into non-TM_pure
364 else if (is_tm_pure (callee
->decl
) && !is_tm_pure (caller
->decl
))
366 e
->inline_failed
= CIF_UNSPECIFIED
;
369 /* Check compatibility of target optimization options. */
370 else if (!targetm
.target_option
.can_inline_p (caller
->decl
,
373 e
->inline_failed
= CIF_TARGET_OPTION_MISMATCH
;
376 else if (ipa_fn_summaries
->get (callee
) == NULL
377 || !ipa_fn_summaries
->get (callee
)->inlinable
)
379 e
->inline_failed
= CIF_FUNCTION_NOT_INLINABLE
;
382 /* Don't inline a function with mismatched sanitization attributes. */
383 else if (!sanitize_attrs_match_for_inline_p (caller
->decl
, callee
->decl
))
385 e
->inline_failed
= CIF_ATTRIBUTE_MISMATCH
;
388 if (!inlinable
&& report
)
389 report_inline_failed_reason (e
);
393 /* Return inlining_insns_single limit for function N. If HINT is true
394 scale up the bound. */
397 inline_insns_single (cgraph_node
*n
, bool hint
)
399 if (opt_for_fn (n
->decl
, optimize
) >= 3)
402 return PARAM_VALUE (PARAM_MAX_INLINE_INSNS_SINGLE
)
403 * PARAM_VALUE (PARAM_INLINE_HEURISTICS_HINT_PERCENT
) / 100;
404 return PARAM_VALUE (PARAM_MAX_INLINE_INSNS_SINGLE
);
409 return PARAM_VALUE (PARAM_MAX_INLINE_INSNS_SINGLE_O2
)
410 * PARAM_VALUE (PARAM_INLINE_HEURISTICS_HINT_PERCENT_O2
) / 100;
411 return PARAM_VALUE (PARAM_MAX_INLINE_INSNS_SINGLE_O2
);
415 /* Return inlining_insns_auto limit for function N. If HINT is true
416 scale up the bound. */
419 inline_insns_auto (cgraph_node
*n
, bool hint
)
421 if (opt_for_fn (n
->decl
, optimize
) >= 3)
424 return PARAM_VALUE (PARAM_MAX_INLINE_INSNS_AUTO
)
425 * PARAM_VALUE (PARAM_INLINE_HEURISTICS_HINT_PERCENT
) / 100;
426 return PARAM_VALUE (PARAM_MAX_INLINE_INSNS_AUTO
);
431 return PARAM_VALUE (PARAM_MAX_INLINE_INSNS_AUTO_O2
)
432 * PARAM_VALUE (PARAM_INLINE_HEURISTICS_HINT_PERCENT_O2
) / 100;
433 return PARAM_VALUE (PARAM_MAX_INLINE_INSNS_AUTO_O2
);
437 /* Decide if we can inline the edge and possibly update
438 inline_failed reason.
439 We check whether inlining is possible at all and whether
440 caller growth limits allow doing so.
442 if REPORT is true, output reason to the dump file.
444 if DISREGARD_LIMITS is true, ignore size limits. */
447 can_inline_edge_by_limits_p (struct cgraph_edge
*e
, bool report
,
448 bool disregard_limits
= false, bool early
= false)
450 gcc_checking_assert (e
->inline_failed
);
452 if (cgraph_inline_failed_type (e
->inline_failed
) == CIF_FINAL_ERROR
)
455 report_inline_failed_reason (e
);
459 bool inlinable
= true;
460 enum availability avail
;
461 cgraph_node
*caller
= (e
->caller
->inlined_to
462 ? e
->caller
->inlined_to
: e
->caller
);
463 cgraph_node
*callee
= e
->callee
->ultimate_alias_target (&avail
, caller
);
464 tree caller_tree
= DECL_FUNCTION_SPECIFIC_OPTIMIZATION (caller
->decl
);
466 = callee
? DECL_FUNCTION_SPECIFIC_OPTIMIZATION (callee
->decl
) : NULL
;
467 /* Check if caller growth allows the inlining. */
468 if (!DECL_DISREGARD_INLINE_LIMITS (callee
->decl
)
470 && !lookup_attribute ("flatten",
471 DECL_ATTRIBUTES (caller
->decl
))
472 && !caller_growth_limits (e
))
474 else if (callee
->externally_visible
475 && !DECL_DISREGARD_INLINE_LIMITS (callee
->decl
)
476 && flag_live_patching
== LIVE_PATCHING_INLINE_ONLY_STATIC
)
478 e
->inline_failed
= CIF_EXTERN_LIVE_ONLY_STATIC
;
481 /* Don't inline a function with a higher optimization level than the
482 caller. FIXME: this is really just tip of iceberg of handling
483 optimization attribute. */
484 else if (caller_tree
!= callee_tree
)
487 (DECL_DISREGARD_INLINE_LIMITS (callee
->decl
)
488 && lookup_attribute ("always_inline",
489 DECL_ATTRIBUTES (callee
->decl
)));
490 ipa_fn_summary
*caller_info
= ipa_fn_summaries
->get (caller
);
491 ipa_fn_summary
*callee_info
= ipa_fn_summaries
->get (callee
);
493 /* Until GCC 4.9 we did not check the semantics-altering flags
494 below and inlined across optimization boundaries.
495 Enabling checks below breaks several packages by refusing
496 to inline library always_inline functions. See PR65873.
497 Disable the check for early inlining for now until better solution
499 if (always_inline
&& early
)
501 /* There are some options that change IL semantics which means
502 we cannot inline in these cases for correctness reason.
503 Not even for always_inline declared functions. */
504 else if (check_match (flag_wrapv
)
505 || check_match (flag_trapv
)
506 || check_match (flag_pcc_struct_return
)
507 /* When caller or callee does FP math, be sure FP codegen flags
509 || ((caller_info
->fp_expressions
&& callee_info
->fp_expressions
)
510 && (check_maybe_up (flag_rounding_math
)
511 || check_maybe_up (flag_trapping_math
)
512 || check_maybe_down (flag_unsafe_math_optimizations
)
513 || check_maybe_down (flag_finite_math_only
)
514 || check_maybe_up (flag_signaling_nans
)
515 || check_maybe_down (flag_cx_limited_range
)
516 || check_maybe_up (flag_signed_zeros
)
517 || check_maybe_down (flag_associative_math
)
518 || check_maybe_down (flag_reciprocal_math
)
519 || check_maybe_down (flag_fp_int_builtin_inexact
)
520 /* Strictly speaking only when the callee contains function
521 calls that may end up setting errno. */
522 || check_maybe_up (flag_errno_math
)))
523 /* We do not want to make code compiled with exceptions to be
524 brought into a non-EH function unless we know that the callee
526 This is tracked by DECL_FUNCTION_PERSONALITY. */
527 || (check_maybe_up (flag_non_call_exceptions
)
528 && DECL_FUNCTION_PERSONALITY (callee
->decl
))
529 || (check_maybe_up (flag_exceptions
)
530 && DECL_FUNCTION_PERSONALITY (callee
->decl
))
531 /* When devirtualization is diabled for callee, it is not safe
532 to inline it as we possibly mangled the type info.
533 Allow early inlining of always inlines. */
534 || (!early
&& check_maybe_down (flag_devirtualize
)))
536 e
->inline_failed
= CIF_OPTIMIZATION_MISMATCH
;
539 /* gcc.dg/pr43564.c. Apply user-forced inline even at -O0. */
540 else if (always_inline
)
542 /* When user added an attribute to the callee honor it. */
543 else if (lookup_attribute ("optimize", DECL_ATTRIBUTES (callee
->decl
))
544 && opts_for_fn (caller
->decl
) != opts_for_fn (callee
->decl
))
546 e
->inline_failed
= CIF_OPTIMIZATION_MISMATCH
;
549 /* If explicit optimize attribute are not used, the mismatch is caused
550 by different command line options used to build different units.
551 Do not care about COMDAT functions - those are intended to be
552 optimized with the optimization flags of module they are used in.
553 Also do not care about mixing up size/speed optimization when
554 DECL_DISREGARD_INLINE_LIMITS is set. */
555 else if ((callee
->merged_comdat
556 && !lookup_attribute ("optimize",
557 DECL_ATTRIBUTES (caller
->decl
)))
558 || DECL_DISREGARD_INLINE_LIMITS (callee
->decl
))
560 /* If mismatch is caused by merging two LTO units with different
561 optimizationflags we want to be bit nicer. However never inline
562 if one of functions is not optimized at all. */
563 else if (!opt_for_fn (callee
->decl
, optimize
)
564 || !opt_for_fn (caller
->decl
, optimize
))
566 e
->inline_failed
= CIF_OPTIMIZATION_MISMATCH
;
569 /* If callee is optimized for size and caller is not, allow inlining if
570 code shrinks or we are in MAX_INLINE_INSNS_SINGLE limit and callee
571 is inline (and thus likely an unified comdat). This will allow caller
573 else if (opt_for_fn (callee
->decl
, optimize_size
)
574 > opt_for_fn (caller
->decl
, optimize_size
))
576 int growth
= estimate_edge_growth (e
);
577 if (growth
> PARAM_VALUE (PARAM_MAX_INLINE_INSNS_SIZE
)
578 && (!DECL_DECLARED_INLINE_P (callee
->decl
)
579 && growth
>= MAX (inline_insns_single (caller
, false),
580 inline_insns_auto (caller
, false))))
582 e
->inline_failed
= CIF_OPTIMIZATION_MISMATCH
;
586 /* If callee is more aggressively optimized for performance than caller,
587 we generally want to inline only cheap (runtime wise) functions. */
588 else if (opt_for_fn (callee
->decl
, optimize_size
)
589 < opt_for_fn (caller
->decl
, optimize_size
)
590 || (opt_for_fn (callee
->decl
, optimize
)
591 > opt_for_fn (caller
->decl
, optimize
)))
593 if (estimate_edge_time (e
)
594 >= 20 + ipa_call_summaries
->get (e
)->call_stmt_time
)
596 e
->inline_failed
= CIF_OPTIMIZATION_MISMATCH
;
603 if (!inlinable
&& report
)
604 report_inline_failed_reason (e
);
609 /* Return true if the edge E is inlinable during early inlining. */
612 can_early_inline_edge_p (struct cgraph_edge
*e
)
614 struct cgraph_node
*callee
= e
->callee
->ultimate_alias_target ();
615 /* Early inliner might get called at WPA stage when IPA pass adds new
616 function. In this case we cannot really do any of early inlining
617 because function bodies are missing. */
618 if (cgraph_inline_failed_type (e
->inline_failed
) == CIF_FINAL_ERROR
)
620 if (!gimple_has_body_p (callee
->decl
))
622 e
->inline_failed
= CIF_BODY_NOT_AVAILABLE
;
625 /* In early inliner some of callees may not be in SSA form yet
626 (i.e. the callgraph is cyclic and we did not process
627 the callee by early inliner, yet). We don't have CIF code for this
628 case; later we will re-do the decision in the real inliner. */
629 if (!gimple_in_ssa_p (DECL_STRUCT_FUNCTION (e
->caller
->decl
))
630 || !gimple_in_ssa_p (DECL_STRUCT_FUNCTION (callee
->decl
)))
632 if (dump_enabled_p ())
633 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, e
->call_stmt
,
634 " edge not inlinable: not in SSA form\n");
637 if (!can_inline_edge_p (e
, true, true)
638 || !can_inline_edge_by_limits_p (e
, true, false, true))
644 /* Return number of calls in N. Ignore cheap builtins. */
647 num_calls (struct cgraph_node
*n
)
649 struct cgraph_edge
*e
;
652 for (e
= n
->callees
; e
; e
= e
->next_callee
)
653 if (!is_inexpensive_builtin (e
->callee
->decl
))
659 /* Return true if we are interested in inlining small function. */
662 want_early_inline_function_p (struct cgraph_edge
*e
)
664 bool want_inline
= true;
665 struct cgraph_node
*callee
= e
->callee
->ultimate_alias_target ();
667 if (DECL_DISREGARD_INLINE_LIMITS (callee
->decl
))
669 /* For AutoFDO, we need to make sure that before profile summary, all
670 hot paths' IR look exactly the same as profiled binary. As a result,
671 in einliner, we will disregard size limit and inline those callsites
673 * inlined in the profiled binary, and
674 * the cloned callee has enough samples to be considered "hot". */
675 else if (flag_auto_profile
&& afdo_callsite_hot_enough_for_early_inline (e
))
677 else if (!DECL_DECLARED_INLINE_P (callee
->decl
)
678 && !opt_for_fn (e
->caller
->decl
, flag_inline_small_functions
))
680 e
->inline_failed
= CIF_FUNCTION_NOT_INLINE_CANDIDATE
;
681 report_inline_failed_reason (e
);
686 int growth
= estimate_edge_growth (e
);
688 int early_inlining_insns
= opt_for_fn (e
->caller
->decl
, optimize
) >= 3
689 ? PARAM_VALUE (PARAM_EARLY_INLINING_INSNS
)
690 : PARAM_VALUE (PARAM_EARLY_INLINING_INSNS_O2
);
693 if (growth
<= PARAM_VALUE (PARAM_MAX_INLINE_INSNS_SIZE
))
695 else if (!e
->maybe_hot_p ())
697 if (dump_enabled_p ())
698 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, e
->call_stmt
,
699 " will not early inline: %C->%C, "
700 "call is cold and code would grow by %i\n",
705 else if (growth
> early_inlining_insns
)
707 if (dump_enabled_p ())
708 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, e
->call_stmt
,
709 " will not early inline: %C->%C, "
710 "growth %i exceeds --param early-inlining-insns%s\n",
711 e
->caller
, callee
, growth
,
712 opt_for_fn (e
->caller
->decl
, optimize
) >= 3
716 else if ((n
= num_calls (callee
)) != 0
717 && growth
* (n
+ 1) > early_inlining_insns
)
719 if (dump_enabled_p ())
720 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, e
->call_stmt
,
721 " will not early inline: %C->%C, "
722 "growth %i exceeds --param early-inlining-insns%s "
723 "divided by number of calls\n",
724 e
->caller
, callee
, growth
,
725 opt_for_fn (e
->caller
->decl
, optimize
) >= 3
733 /* Compute time of the edge->caller + edge->callee execution when inlining
737 compute_uninlined_call_time (struct cgraph_edge
*edge
,
738 sreal uninlined_call_time
)
740 cgraph_node
*caller
= (edge
->caller
->inlined_to
741 ? edge
->caller
->inlined_to
744 sreal freq
= edge
->sreal_frequency ();
746 uninlined_call_time
*= freq
;
748 uninlined_call_time
= uninlined_call_time
>> 11;
750 sreal caller_time
= ipa_fn_summaries
->get (caller
)->time
;
751 return uninlined_call_time
+ caller_time
;
754 /* Same as compute_uinlined_call_time but compute time when inlining
758 compute_inlined_call_time (struct cgraph_edge
*edge
,
761 cgraph_node
*caller
= (edge
->caller
->inlined_to
762 ? edge
->caller
->inlined_to
764 sreal caller_time
= ipa_fn_summaries
->get (caller
)->time
;
766 sreal freq
= edge
->sreal_frequency ();
772 /* This calculation should match one in ipa-inline-analysis.c
773 (estimate_edge_size_and_time). */
774 time
-= (sreal
)ipa_call_summaries
->get (edge
)->call_stmt_time
* freq
;
777 time
= ((sreal
) 1) >> 8;
778 gcc_checking_assert (time
>= 0);
782 /* Return true if the speedup for inlining E is bigger than
783 PARAM_MAX_INLINE_MIN_SPEEDUP. */
786 big_speedup_p (struct cgraph_edge
*e
)
789 sreal spec_time
= estimate_edge_time (e
, &unspec_time
);
790 sreal time
= compute_uninlined_call_time (e
, unspec_time
);
791 sreal inlined_time
= compute_inlined_call_time (e
, spec_time
);
792 cgraph_node
*caller
= (e
->caller
->inlined_to
793 ? e
->caller
->inlined_to
795 int limit
= opt_for_fn (caller
->decl
, optimize
) >= 3
796 ? PARAM_VALUE (PARAM_INLINE_MIN_SPEEDUP
)
797 : PARAM_VALUE (PARAM_INLINE_MIN_SPEEDUP_O2
);
799 if ((time
- inlined_time
) * 100 > time
* limit
)
804 /* Return true if we are interested in inlining small function.
805 When REPORT is true, report reason to dump file. */
808 want_inline_small_function_p (struct cgraph_edge
*e
, bool report
)
810 bool want_inline
= true;
811 struct cgraph_node
*callee
= e
->callee
->ultimate_alias_target ();
813 /* Allow this function to be called before can_inline_edge_p,
814 since it's usually cheaper. */
815 if (cgraph_inline_failed_type (e
->inline_failed
) == CIF_FINAL_ERROR
)
817 else if (DECL_DISREGARD_INLINE_LIMITS (callee
->decl
))
819 else if (!DECL_DECLARED_INLINE_P (callee
->decl
)
820 && !opt_for_fn (e
->caller
->decl
, flag_inline_small_functions
))
822 e
->inline_failed
= CIF_FUNCTION_NOT_INLINE_CANDIDATE
;
825 /* Do fast and conservative check if the function can be good
827 else if ((!DECL_DECLARED_INLINE_P (callee
->decl
)
828 && (!e
->count
.ipa ().initialized_p () || !e
->maybe_hot_p ()))
829 && ipa_fn_summaries
->get (callee
)->min_size
830 - ipa_call_summaries
->get (e
)->call_stmt_size
831 > inline_insns_auto (e
->caller
, true))
833 if (opt_for_fn (e
->caller
->decl
, optimize
) >= 3)
834 e
->inline_failed
= CIF_MAX_INLINE_INSNS_AUTO_LIMIT
;
836 e
->inline_failed
= CIF_MAX_INLINE_INSNS_AUTO_O2_LIMIT
;
839 else if ((DECL_DECLARED_INLINE_P (callee
->decl
)
840 || e
->count
.ipa ().nonzero_p ())
841 && ipa_fn_summaries
->get (callee
)->min_size
842 - ipa_call_summaries
->get (e
)->call_stmt_size
843 > inline_insns_single (e
->caller
, true))
845 if (opt_for_fn (e
->caller
->decl
, optimize
) >= 3)
846 e
->inline_failed
= (DECL_DECLARED_INLINE_P (callee
->decl
)
847 ? CIF_MAX_INLINE_INSNS_SINGLE_LIMIT
848 : CIF_MAX_INLINE_INSNS_AUTO_LIMIT
);
850 e
->inline_failed
= (DECL_DECLARED_INLINE_P (callee
->decl
)
851 ? CIF_MAX_INLINE_INSNS_SINGLE_O2_LIMIT
852 : CIF_MAX_INLINE_INSNS_AUTO_O2_LIMIT
);
857 int growth
= estimate_edge_growth (e
);
858 ipa_hints hints
= estimate_edge_hints (e
);
859 bool apply_hints
= (hints
& (INLINE_HINT_indirect_call
860 | INLINE_HINT_known_hot
861 | INLINE_HINT_loop_iterations
862 | INLINE_HINT_loop_stride
));
864 if (growth
<= PARAM_VALUE (PARAM_MAX_INLINE_INSNS_SIZE
))
866 /* Apply MAX_INLINE_INSNS_SINGLE limit. Do not do so when
867 hints suggests that inlining given function is very profitable.
868 Avoid computation of big_speedup_p when not necessary to change
869 outcome of decision. */
870 else if (DECL_DECLARED_INLINE_P (callee
->decl
)
871 && growth
>= inline_insns_single (e
->caller
, apply_hints
)
873 || growth
>= inline_insns_single (e
->caller
, true)
874 || !big_speedup_p (e
)))
876 if (opt_for_fn (e
->caller
->decl
, optimize
) >= 3)
877 e
->inline_failed
= CIF_MAX_INLINE_INSNS_SINGLE_LIMIT
;
879 e
->inline_failed
= CIF_MAX_INLINE_INSNS_SINGLE_O2_LIMIT
;
882 else if (!DECL_DECLARED_INLINE_P (callee
->decl
)
883 && !opt_for_fn (e
->caller
->decl
, flag_inline_functions
)
884 && growth
>= PARAM_VALUE (PARAM_MAX_INLINE_INSNS_SMALL
))
886 /* growth_likely_positive is expensive, always test it last. */
887 if (growth
>= inline_insns_single (e
->caller
, false)
888 || growth_likely_positive (callee
, growth
))
890 e
->inline_failed
= CIF_NOT_DECLARED_INLINED
;
894 /* Apply MAX_INLINE_INSNS_AUTO limit for functions not declared inline.
895 Bypass the limit when speedup seems big. */
896 else if (!DECL_DECLARED_INLINE_P (callee
->decl
)
897 && growth
>= inline_insns_auto (e
->caller
, apply_hints
)
899 || growth
>= inline_insns_auto (e
->caller
, true)
900 || !big_speedup_p (e
)))
902 /* growth_likely_positive is expensive, always test it last. */
903 if (growth
>= inline_insns_single (e
->caller
, false)
904 || growth_likely_positive (callee
, growth
))
906 if (opt_for_fn (e
->caller
->decl
, optimize
) >= 3)
907 e
->inline_failed
= CIF_MAX_INLINE_INSNS_AUTO_LIMIT
;
909 e
->inline_failed
= CIF_MAX_INLINE_INSNS_AUTO_O2_LIMIT
;
913 /* If call is cold, do not inline when function body would grow. */
914 else if (!e
->maybe_hot_p ()
915 && (growth
>= inline_insns_single (e
->caller
, false)
916 || growth_likely_positive (callee
, growth
)))
918 e
->inline_failed
= CIF_UNLIKELY_CALL
;
922 if (!want_inline
&& report
)
923 report_inline_failed_reason (e
);
927 /* EDGE is self recursive edge.
928 We hand two cases - when function A is inlining into itself
929 or when function A is being inlined into another inliner copy of function
932 In first case OUTER_NODE points to the toplevel copy of A, while
933 in the second case OUTER_NODE points to the outermost copy of A in B.
935 In both cases we want to be extra selective since
936 inlining the call will just introduce new recursive calls to appear. */
939 want_inline_self_recursive_call_p (struct cgraph_edge
*edge
,
940 struct cgraph_node
*outer_node
,
944 char const *reason
= NULL
;
945 bool want_inline
= true;
946 sreal caller_freq
= 1;
947 int max_depth
= PARAM_VALUE (PARAM_MAX_INLINE_RECURSIVE_DEPTH_AUTO
);
949 if (DECL_DECLARED_INLINE_P (edge
->caller
->decl
))
950 max_depth
= PARAM_VALUE (PARAM_MAX_INLINE_RECURSIVE_DEPTH
);
952 if (!edge
->maybe_hot_p ())
954 reason
= "recursive call is cold";
957 else if (depth
> max_depth
)
959 reason
= "--param max-inline-recursive-depth exceeded.";
962 else if (outer_node
->inlined_to
963 && (caller_freq
= outer_node
->callers
->sreal_frequency ()) == 0)
965 reason
= "caller frequency is 0";
971 /* Inlining of self recursive function into copy of itself within other
972 function is transformation similar to loop peeling.
974 Peeling is profitable if we can inline enough copies to make probability
975 of actual call to the self recursive function very small. Be sure that
976 the probability of recursion is small.
978 We ensure that the frequency of recursing is at most 1 - (1/max_depth).
979 This way the expected number of recursion is at most max_depth. */
982 sreal max_prob
= (sreal
)1 - ((sreal
)1 / (sreal
)max_depth
);
984 for (i
= 1; i
< depth
; i
++)
985 max_prob
= max_prob
* max_prob
;
986 if (edge
->sreal_frequency () >= max_prob
* caller_freq
)
988 reason
= "frequency of recursive call is too large";
992 /* Recursive inlining, i.e. equivalent of unrolling, is profitable if
993 recursion depth is large. We reduce function call overhead and increase
994 chances that things fit in hardware return predictor.
996 Recursive inlining might however increase cost of stack frame setup
997 actually slowing down functions whose recursion tree is wide rather than
1000 Deciding reliably on when to do recursive inlining without profile feedback
1001 is tricky. For now we disable recursive inlining when probability of self
1004 Recursive inlining of self recursive call within loop also results in
1005 large loop depths that generally optimize badly. We may want to throttle
1006 down inlining in those cases. In particular this seems to happen in one
1007 of libstdc++ rb tree methods. */
1010 if (edge
->sreal_frequency () * 100
1012 * PARAM_VALUE (PARAM_MIN_INLINE_RECURSIVE_PROBABILITY
))
1014 reason
= "frequency of recursive call is too small";
1015 want_inline
= false;
1018 if (!want_inline
&& dump_enabled_p ())
1019 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, edge
->call_stmt
,
1020 " not inlining recursively: %s\n", reason
);
1024 /* Return true when NODE has uninlinable caller;
1025 set HAS_HOT_CALL if it has hot call.
1026 Worker for cgraph_for_node_and_aliases. */
1029 check_callers (struct cgraph_node
*node
, void *has_hot_call
)
1031 struct cgraph_edge
*e
;
1032 for (e
= node
->callers
; e
; e
= e
->next_caller
)
1034 if (!opt_for_fn (e
->caller
->decl
, flag_inline_functions_called_once
)
1035 || !opt_for_fn (e
->caller
->decl
, optimize
))
1037 if (!can_inline_edge_p (e
, true))
1039 if (e
->recursive_p ())
1041 if (!can_inline_edge_by_limits_p (e
, true))
1043 if (!(*(bool *)has_hot_call
) && e
->maybe_hot_p ())
1044 *(bool *)has_hot_call
= true;
1049 /* If NODE has a caller, return true. */
1052 has_caller_p (struct cgraph_node
*node
, void *data ATTRIBUTE_UNUSED
)
1059 /* Decide if inlining NODE would reduce unit size by eliminating
1060 the offline copy of function.
1061 When COLD is true the cold calls are considered, too. */
1064 want_inline_function_to_all_callers_p (struct cgraph_node
*node
, bool cold
)
1066 bool has_hot_call
= false;
1068 /* Aliases gets inlined along with the function they alias. */
1071 /* Already inlined? */
1072 if (node
->inlined_to
)
1074 /* Does it have callers? */
1075 if (!node
->call_for_symbol_and_aliases (has_caller_p
, NULL
, true))
1077 /* Inlining into all callers would increase size? */
1078 if (estimate_growth (node
) > 0)
1080 /* All inlines must be possible. */
1081 if (node
->call_for_symbol_and_aliases (check_callers
, &has_hot_call
,
1084 if (!cold
&& !has_hot_call
)
1089 /* A cost model driving the inlining heuristics in a way so the edges with
1090 smallest badness are inlined first. After each inlining is performed
1091 the costs of all caller edges of nodes affected are recomputed so the
1092 metrics may accurately depend on values such as number of inlinable callers
1093 of the function or function body size. */
1096 edge_badness (struct cgraph_edge
*edge
, bool dump
)
1100 sreal edge_time
, unspec_edge_time
;
1101 struct cgraph_node
*callee
= edge
->callee
->ultimate_alias_target ();
1102 class ipa_fn_summary
*callee_info
= ipa_fn_summaries
->get (callee
);
1104 cgraph_node
*caller
= (edge
->caller
->inlined_to
1105 ? edge
->caller
->inlined_to
1108 growth
= estimate_edge_growth (edge
);
1109 edge_time
= estimate_edge_time (edge
, &unspec_edge_time
);
1110 hints
= estimate_edge_hints (edge
);
1111 gcc_checking_assert (edge_time
>= 0);
1112 /* Check that inlined time is better, but tolerate some roundoff issues.
1113 FIXME: When callee profile drops to 0 we account calls more. This
1114 should be fixed by never doing that. */
1115 gcc_checking_assert ((edge_time
* 100
1116 - callee_info
->time
* 101).to_int () <= 0
1117 || callee
->count
.ipa ().initialized_p ());
1118 gcc_checking_assert (growth
<= ipa_size_summaries
->get (callee
)->size
);
1122 fprintf (dump_file
, " Badness calculation for %s -> %s\n",
1123 edge
->caller
->dump_name (),
1124 edge
->callee
->dump_name ());
1125 fprintf (dump_file
, " size growth %i, time %f unspec %f ",
1127 edge_time
.to_double (),
1128 unspec_edge_time
.to_double ());
1129 ipa_dump_hints (dump_file
, hints
);
1130 if (big_speedup_p (edge
))
1131 fprintf (dump_file
, " big_speedup");
1132 fprintf (dump_file
, "\n");
1135 /* Always prefer inlining saving code size. */
1138 badness
= (sreal
) (-SREAL_MIN_SIG
+ growth
) << (SREAL_MAX_EXP
/ 256);
1140 fprintf (dump_file
, " %f: Growth %d <= 0\n", badness
.to_double (),
1143 /* Inlining into EXTERNAL functions is not going to change anything unless
1144 they are themselves inlined. */
1145 else if (DECL_EXTERNAL (caller
->decl
))
1148 fprintf (dump_file
, " max: function is external\n");
1149 return sreal::max ();
1151 /* When profile is available. Compute badness as:
1153 time_saved * caller_count
1154 goodness = -------------------------------------------------
1155 growth_of_caller * overall_growth * combined_size
1157 badness = - goodness
1159 Again use negative value to make calls with profile appear hotter
1162 else if (opt_for_fn (caller
->decl
, flag_guess_branch_prob
)
1163 || caller
->count
.ipa ().nonzero_p ())
1165 sreal numerator
, denominator
;
1167 sreal inlined_time
= compute_inlined_call_time (edge
, edge_time
);
1169 numerator
= (compute_uninlined_call_time (edge
, unspec_edge_time
)
1172 numerator
= ((sreal
) 1 >> 8);
1173 if (caller
->count
.ipa ().nonzero_p ())
1174 numerator
*= caller
->count
.ipa ().to_gcov_type ();
1175 else if (caller
->count
.ipa ().initialized_p ())
1176 numerator
= numerator
>> 11;
1177 denominator
= growth
;
1179 overall_growth
= callee_info
->growth
;
1181 /* Look for inliner wrappers of the form:
1187 noninline_callee ();
1189 Withhout penalizing this case, we usually inline noninline_callee
1190 into the inline_caller because overall_growth is small preventing
1191 further inlining of inline_caller.
1193 Penalize only callgraph edges to functions with small overall
1196 if (growth
> overall_growth
1197 /* ... and having only one caller which is not inlined ... */
1198 && callee_info
->single_caller
1199 && !edge
->caller
->inlined_to
1200 /* ... and edges executed only conditionally ... */
1201 && edge
->sreal_frequency () < 1
1202 /* ... consider case where callee is not inline but caller is ... */
1203 && ((!DECL_DECLARED_INLINE_P (edge
->callee
->decl
)
1204 && DECL_DECLARED_INLINE_P (caller
->decl
))
1205 /* ... or when early optimizers decided to split and edge
1206 frequency still indicates splitting is a win ... */
1207 || (callee
->split_part
&& !caller
->split_part
1208 && edge
->sreal_frequency () * 100
1210 (PARAM_PARTIAL_INLINING_ENTRY_PROBABILITY
)
1211 /* ... and do not overwrite user specified hints. */
1212 && (!DECL_DECLARED_INLINE_P (edge
->callee
->decl
)
1213 || DECL_DECLARED_INLINE_P (caller
->decl
)))))
1215 ipa_fn_summary
*caller_info
= ipa_fn_summaries
->get (caller
);
1216 int caller_growth
= caller_info
->growth
;
1218 /* Only apply the penalty when caller looks like inline candidate,
1219 and it is not called once. */
1220 if (!caller_info
->single_caller
&& overall_growth
< caller_growth
1221 && caller_info
->inlinable
1222 && ipa_size_summaries
->get (caller
)->size
1223 < (DECL_DECLARED_INLINE_P (caller
->decl
)
1224 ? inline_insns_single (caller
, false)
1225 : inline_insns_auto (caller
, false)))
1229 " Wrapper penalty. Increasing growth %i to %i\n",
1230 overall_growth
, caller_growth
);
1231 overall_growth
= caller_growth
;
1234 if (overall_growth
> 0)
1236 /* Strongly preffer functions with few callers that can be inlined
1237 fully. The square root here leads to smaller binaries at average.
1238 Watch however for extreme cases and return to linear function
1239 when growth is large. */
1240 if (overall_growth
< 256)
1241 overall_growth
*= overall_growth
;
1243 overall_growth
+= 256 * 256 - 256;
1244 denominator
*= overall_growth
;
1246 denominator
*= ipa_size_summaries
->get (caller
)->size
+ growth
;
1248 badness
= - numerator
/ denominator
;
1253 " %f: guessed profile. frequency %f, count %" PRId64
1254 " caller count %" PRId64
1255 " time w/o inlining %f, time with inlining %f"
1256 " overall growth %i (current) %i (original)"
1257 " %i (compensated)\n",
1258 badness
.to_double (),
1259 edge
->sreal_frequency ().to_double (),
1260 edge
->count
.ipa ().initialized_p () ? edge
->count
.ipa ().to_gcov_type () : -1,
1261 caller
->count
.ipa ().initialized_p () ? caller
->count
.ipa ().to_gcov_type () : -1,
1262 compute_uninlined_call_time (edge
,
1263 unspec_edge_time
).to_double (),
1264 inlined_time
.to_double (),
1265 estimate_growth (callee
),
1266 callee_info
->growth
, overall_growth
);
1269 /* When function local profile is not available or it does not give
1270 useful information (ie frequency is zero), base the cost on
1271 loop nest and overall size growth, so we optimize for overall number
1272 of functions fully inlined in program. */
1275 int nest
= MIN (ipa_call_summaries
->get (edge
)->loop_depth
, 8);
1278 /* Decrease badness if call is nested. */
1280 badness
= badness
>> nest
;
1282 badness
= badness
<< nest
;
1284 fprintf (dump_file
, " %f: no profile. nest %i\n",
1285 badness
.to_double (), nest
);
1287 gcc_checking_assert (badness
!= 0);
1289 if (edge
->recursive_p ())
1290 badness
= badness
.shift (badness
> 0 ? 4 : -4);
1291 if ((hints
& (INLINE_HINT_indirect_call
1292 | INLINE_HINT_loop_iterations
1293 | INLINE_HINT_loop_stride
))
1294 || callee_info
->growth
<= 0)
1295 badness
= badness
.shift (badness
> 0 ? -2 : 2);
1296 if (hints
& (INLINE_HINT_same_scc
))
1297 badness
= badness
.shift (badness
> 0 ? 3 : -3);
1298 else if (hints
& (INLINE_HINT_in_scc
))
1299 badness
= badness
.shift (badness
> 0 ? 2 : -2);
1300 else if (hints
& (INLINE_HINT_cross_module
))
1301 badness
= badness
.shift (badness
> 0 ? 1 : -1);
1302 if (DECL_DISREGARD_INLINE_LIMITS (callee
->decl
))
1303 badness
= badness
.shift (badness
> 0 ? -4 : 4);
1304 else if ((hints
& INLINE_HINT_declared_inline
))
1305 badness
= badness
.shift (badness
> 0 ? -3 : 3);
1307 fprintf (dump_file
, " Adjusted by hints %f\n", badness
.to_double ());
1311 /* Recompute badness of EDGE and update its key in HEAP if needed. */
1313 update_edge_key (edge_heap_t
*heap
, struct cgraph_edge
*edge
)
1315 sreal badness
= edge_badness (edge
, false);
1318 edge_heap_node_t
*n
= (edge_heap_node_t
*) edge
->aux
;
1319 gcc_checking_assert (n
->get_data () == edge
);
1321 /* fibonacci_heap::replace_key does busy updating of the
1322 heap that is unnecesarily expensive.
1323 We do lazy increases: after extracting minimum if the key
1324 turns out to be out of date, it is re-inserted into heap
1325 with correct value. */
1326 if (badness
< n
->get_key ())
1328 if (dump_file
&& (dump_flags
& TDF_DETAILS
))
1331 " decreasing badness %s -> %s, %f to %f\n",
1332 edge
->caller
->dump_name (),
1333 edge
->callee
->dump_name (),
1334 n
->get_key ().to_double (),
1335 badness
.to_double ());
1337 heap
->decrease_key (n
, badness
);
1342 if (dump_file
&& (dump_flags
& TDF_DETAILS
))
1345 " enqueuing call %s -> %s, badness %f\n",
1346 edge
->caller
->dump_name (),
1347 edge
->callee
->dump_name (),
1348 badness
.to_double ());
1350 edge
->aux
= heap
->insert (badness
, edge
);
1355 /* NODE was inlined.
1356 All caller edges needs to be resetted because
1357 size estimates change. Similarly callees needs reset
1358 because better context may be known. */
1361 reset_edge_caches (struct cgraph_node
*node
)
1363 struct cgraph_edge
*edge
;
1364 struct cgraph_edge
*e
= node
->callees
;
1365 struct cgraph_node
*where
= node
;
1366 struct ipa_ref
*ref
;
1368 if (where
->inlined_to
)
1369 where
= where
->inlined_to
;
1371 if (edge_growth_cache
!= NULL
)
1372 for (edge
= where
->callers
; edge
; edge
= edge
->next_caller
)
1373 if (edge
->inline_failed
)
1374 edge_growth_cache
->remove (edge
);
1376 FOR_EACH_ALIAS (where
, ref
)
1377 reset_edge_caches (dyn_cast
<cgraph_node
*> (ref
->referring
));
1383 if (!e
->inline_failed
&& e
->callee
->callees
)
1384 e
= e
->callee
->callees
;
1387 if (edge_growth_cache
!= NULL
&& e
->inline_failed
)
1388 edge_growth_cache
->remove (e
);
1395 if (e
->caller
== node
)
1397 e
= e
->caller
->callers
;
1399 while (!e
->next_callee
);
1405 /* Recompute HEAP nodes for each of caller of NODE.
1406 UPDATED_NODES track nodes we already visited, to avoid redundant work.
1407 When CHECK_INLINABLITY_FOR is set, re-check for specified edge that
1408 it is inlinable. Otherwise check all edges. */
1411 update_caller_keys (edge_heap_t
*heap
, struct cgraph_node
*node
,
1412 bitmap updated_nodes
,
1413 struct cgraph_edge
*check_inlinablity_for
)
1415 struct cgraph_edge
*edge
;
1416 struct ipa_ref
*ref
;
1418 if ((!node
->alias
&& !ipa_fn_summaries
->get (node
)->inlinable
)
1419 || node
->inlined_to
)
1421 if (!bitmap_set_bit (updated_nodes
, node
->get_uid ()))
1424 FOR_EACH_ALIAS (node
, ref
)
1426 struct cgraph_node
*alias
= dyn_cast
<cgraph_node
*> (ref
->referring
);
1427 update_caller_keys (heap
, alias
, updated_nodes
, check_inlinablity_for
);
1430 for (edge
= node
->callers
; edge
; edge
= edge
->next_caller
)
1431 if (edge
->inline_failed
)
1433 if (!check_inlinablity_for
1434 || check_inlinablity_for
== edge
)
1436 if (can_inline_edge_p (edge
, false)
1437 && want_inline_small_function_p (edge
, false)
1438 && can_inline_edge_by_limits_p (edge
, false))
1439 update_edge_key (heap
, edge
);
1442 report_inline_failed_reason (edge
);
1443 heap
->delete_node ((edge_heap_node_t
*) edge
->aux
);
1448 update_edge_key (heap
, edge
);
1452 /* Recompute HEAP nodes for each uninlined call in NODE.
1453 This is used when we know that edge badnesses are going only to increase
1454 (we introduced new call site) and thus all we need is to insert newly
1455 created edges into heap. */
1458 update_callee_keys (edge_heap_t
*heap
, struct cgraph_node
*node
,
1459 bitmap updated_nodes
)
1461 struct cgraph_edge
*e
= node
->callees
;
1466 if (!e
->inline_failed
&& e
->callee
->callees
)
1467 e
= e
->callee
->callees
;
1470 enum availability avail
;
1471 struct cgraph_node
*callee
;
1472 /* We do not reset callee growth cache here. Since we added a new call,
1473 growth chould have just increased and consequentely badness metric
1474 don't need updating. */
1475 if (e
->inline_failed
1476 && (callee
= e
->callee
->ultimate_alias_target (&avail
, e
->caller
))
1477 && ipa_fn_summaries
->get (callee
) != NULL
1478 && ipa_fn_summaries
->get (callee
)->inlinable
1479 && avail
>= AVAIL_AVAILABLE
1480 && !bitmap_bit_p (updated_nodes
, callee
->get_uid ()))
1482 if (can_inline_edge_p (e
, false)
1483 && want_inline_small_function_p (e
, false)
1484 && can_inline_edge_by_limits_p (e
, false))
1485 update_edge_key (heap
, e
);
1488 report_inline_failed_reason (e
);
1489 heap
->delete_node ((edge_heap_node_t
*) e
->aux
);
1499 if (e
->caller
== node
)
1501 e
= e
->caller
->callers
;
1503 while (!e
->next_callee
);
1509 /* Enqueue all recursive calls from NODE into priority queue depending on
1510 how likely we want to recursively inline the call. */
1513 lookup_recursive_calls (struct cgraph_node
*node
, struct cgraph_node
*where
,
1516 struct cgraph_edge
*e
;
1517 enum availability avail
;
1519 for (e
= where
->callees
; e
; e
= e
->next_callee
)
1520 if (e
->callee
== node
1521 || (e
->callee
->ultimate_alias_target (&avail
, e
->caller
) == node
1522 && avail
> AVAIL_INTERPOSABLE
))
1523 heap
->insert (-e
->sreal_frequency (), e
);
1524 for (e
= where
->callees
; e
; e
= e
->next_callee
)
1525 if (!e
->inline_failed
)
1526 lookup_recursive_calls (node
, e
->callee
, heap
);
1529 /* Decide on recursive inlining: in the case function has recursive calls,
1530 inline until body size reaches given argument. If any new indirect edges
1531 are discovered in the process, add them to *NEW_EDGES, unless NEW_EDGES
1535 recursive_inlining (struct cgraph_edge
*edge
,
1536 vec
<cgraph_edge
*> *new_edges
)
1538 int limit
= PARAM_VALUE (PARAM_MAX_INLINE_INSNS_RECURSIVE_AUTO
);
1539 edge_heap_t
heap (sreal::min ());
1540 struct cgraph_node
*node
;
1541 struct cgraph_edge
*e
;
1542 struct cgraph_node
*master_clone
= NULL
, *next
;
1546 node
= edge
->caller
;
1547 if (node
->inlined_to
)
1548 node
= node
->inlined_to
;
1550 if (DECL_DECLARED_INLINE_P (node
->decl
))
1551 limit
= PARAM_VALUE (PARAM_MAX_INLINE_INSNS_RECURSIVE
);
1553 /* Make sure that function is small enough to be considered for inlining. */
1554 if (estimate_size_after_inlining (node
, edge
) >= limit
)
1556 lookup_recursive_calls (node
, node
, &heap
);
1562 " Performing recursive inlining on %s\n",
1565 /* Do the inlining and update list of recursive call during process. */
1566 while (!heap
.empty ())
1568 struct cgraph_edge
*curr
= heap
.extract_min ();
1569 struct cgraph_node
*cnode
, *dest
= curr
->callee
;
1571 if (!can_inline_edge_p (curr
, true)
1572 || !can_inline_edge_by_limits_p (curr
, true))
1575 /* MASTER_CLONE is produced in the case we already started modified
1576 the function. Be sure to redirect edge to the original body before
1577 estimating growths otherwise we will be seeing growths after inlining
1578 the already modified body. */
1581 curr
->redirect_callee (master_clone
);
1582 if (edge_growth_cache
!= NULL
)
1583 edge_growth_cache
->remove (curr
);
1586 if (estimate_size_after_inlining (node
, curr
) > limit
)
1588 curr
->redirect_callee (dest
);
1589 if (edge_growth_cache
!= NULL
)
1590 edge_growth_cache
->remove (curr
);
1595 for (cnode
= curr
->caller
;
1596 cnode
->inlined_to
; cnode
= cnode
->callers
->caller
)
1598 == curr
->callee
->ultimate_alias_target ()->decl
)
1601 if (!want_inline_self_recursive_call_p (curr
, node
, false, depth
))
1603 curr
->redirect_callee (dest
);
1604 if (edge_growth_cache
!= NULL
)
1605 edge_growth_cache
->remove (curr
);
1612 " Inlining call of depth %i", depth
);
1613 if (node
->count
.nonzero_p () && curr
->count
.initialized_p ())
1615 fprintf (dump_file
, " called approx. %.2f times per call",
1616 (double)curr
->count
.to_gcov_type ()
1617 / node
->count
.to_gcov_type ());
1619 fprintf (dump_file
, "\n");
1623 /* We need original clone to copy around. */
1624 master_clone
= node
->create_clone (node
->decl
, node
->count
,
1625 false, vNULL
, true, NULL
, NULL
);
1626 for (e
= master_clone
->callees
; e
; e
= e
->next_callee
)
1627 if (!e
->inline_failed
)
1628 clone_inlined_nodes (e
, true, false, NULL
);
1629 curr
->redirect_callee (master_clone
);
1630 if (edge_growth_cache
!= NULL
)
1631 edge_growth_cache
->remove (curr
);
1634 inline_call (curr
, false, new_edges
, &overall_size
, true);
1635 lookup_recursive_calls (node
, curr
->callee
, &heap
);
1639 if (!heap
.empty () && dump_file
)
1640 fprintf (dump_file
, " Recursive inlining growth limit met.\n");
1645 if (dump_enabled_p ())
1646 dump_printf_loc (MSG_NOTE
, edge
->call_stmt
,
1647 "\n Inlined %i times, "
1648 "body grown from size %i to %i, time %f to %f\n", n
,
1649 ipa_size_summaries
->get (master_clone
)->size
,
1650 ipa_size_summaries
->get (node
)->size
,
1651 ipa_fn_summaries
->get (master_clone
)->time
.to_double (),
1652 ipa_fn_summaries
->get (node
)->time
.to_double ());
1654 /* Remove master clone we used for inlining. We rely that clones inlined
1655 into master clone gets queued just before master clone so we don't
1657 for (node
= symtab
->first_function (); node
!= master_clone
;
1660 next
= symtab
->next_function (node
);
1661 if (node
->inlined_to
== master_clone
)
1664 master_clone
->remove ();
1669 /* Given whole compilation unit estimate of INSNS, compute how large we can
1670 allow the unit to grow. */
1673 compute_max_insns (int insns
)
1675 int max_insns
= insns
;
1676 if (max_insns
< PARAM_VALUE (PARAM_LARGE_UNIT_INSNS
))
1677 max_insns
= PARAM_VALUE (PARAM_LARGE_UNIT_INSNS
);
1679 return ((int64_t) max_insns
1680 * (100 + PARAM_VALUE (PARAM_INLINE_UNIT_GROWTH
)) / 100);
1684 /* Compute badness of all edges in NEW_EDGES and add them to the HEAP. */
1687 add_new_edges_to_heap (edge_heap_t
*heap
, vec
<cgraph_edge
*> new_edges
)
1689 while (new_edges
.length () > 0)
1691 struct cgraph_edge
*edge
= new_edges
.pop ();
1693 gcc_assert (!edge
->aux
);
1694 gcc_assert (edge
->callee
);
1695 if (edge
->inline_failed
1696 && can_inline_edge_p (edge
, true)
1697 && want_inline_small_function_p (edge
, true)
1698 && can_inline_edge_by_limits_p (edge
, true))
1699 edge
->aux
= heap
->insert (edge_badness (edge
, false), edge
);
1703 /* Remove EDGE from the fibheap. */
1706 heap_edge_removal_hook (struct cgraph_edge
*e
, void *data
)
1710 ((edge_heap_t
*)data
)->delete_node ((edge_heap_node_t
*)e
->aux
);
1715 /* Return true if speculation of edge E seems useful.
1716 If ANTICIPATE_INLINING is true, be conservative and hope that E
1720 speculation_useful_p (struct cgraph_edge
*e
, bool anticipate_inlining
)
1722 /* If we have already decided to inline the edge, it seems useful. */
1723 if (!e
->inline_failed
)
1726 enum availability avail
;
1727 struct cgraph_node
*target
= e
->callee
->ultimate_alias_target (&avail
,
1729 struct cgraph_edge
*direct
, *indirect
;
1730 struct ipa_ref
*ref
;
1732 gcc_assert (e
->speculative
&& !e
->indirect_unknown_callee
);
1734 if (!e
->maybe_hot_p ())
1737 /* See if IP optimizations found something potentially useful about the
1738 function. For now we look only for CONST/PURE flags. Almost everything
1739 else we propagate is useless. */
1740 if (avail
>= AVAIL_AVAILABLE
)
1742 int ecf_flags
= flags_from_decl_or_type (target
->decl
);
1743 if (ecf_flags
& ECF_CONST
)
1745 e
->speculative_call_info (direct
, indirect
, ref
);
1746 if (!(indirect
->indirect_info
->ecf_flags
& ECF_CONST
))
1749 else if (ecf_flags
& ECF_PURE
)
1751 e
->speculative_call_info (direct
, indirect
, ref
);
1752 if (!(indirect
->indirect_info
->ecf_flags
& ECF_PURE
))
1756 /* If we did not managed to inline the function nor redirect
1757 to an ipa-cp clone (that are seen by having local flag set),
1758 it is probably pointless to inline it unless hardware is missing
1759 indirect call predictor. */
1760 if (!anticipate_inlining
&& !target
->local
)
1762 /* For overwritable targets there is not much to do. */
1763 if (!can_inline_edge_p (e
, false)
1764 || !can_inline_edge_by_limits_p (e
, false, true))
1766 /* OK, speculation seems interesting. */
1770 /* We know that EDGE is not going to be inlined.
1771 See if we can remove speculation. */
1774 resolve_noninline_speculation (edge_heap_t
*edge_heap
, struct cgraph_edge
*edge
)
1776 if (edge
->speculative
&& !speculation_useful_p (edge
, false))
1778 struct cgraph_node
*node
= edge
->caller
;
1779 struct cgraph_node
*where
= node
->inlined_to
1780 ? node
->inlined_to
: node
;
1781 auto_bitmap updated_nodes
;
1783 if (edge
->count
.ipa ().initialized_p ())
1784 spec_rem
+= edge
->count
.ipa ();
1785 edge
->resolve_speculation ();
1786 reset_edge_caches (where
);
1787 ipa_update_overall_fn_summary (where
);
1788 update_caller_keys (edge_heap
, where
,
1789 updated_nodes
, NULL
);
1790 update_callee_keys (edge_heap
, where
,
1795 /* Return true if NODE should be accounted for overall size estimate.
1796 Skip all nodes optimized for size so we can measure the growth of hot
1797 part of program no matter of the padding. */
1800 inline_account_function_p (struct cgraph_node
*node
)
1802 return (!DECL_EXTERNAL (node
->decl
)
1803 && !opt_for_fn (node
->decl
, optimize_size
)
1804 && node
->frequency
!= NODE_FREQUENCY_UNLIKELY_EXECUTED
);
1807 /* Count number of callers of NODE and store it into DATA (that
1808 points to int. Worker for cgraph_for_node_and_aliases. */
1811 sum_callers (struct cgraph_node
*node
, void *data
)
1813 struct cgraph_edge
*e
;
1814 int *num_calls
= (int *)data
;
1816 for (e
= node
->callers
; e
; e
= e
->next_caller
)
1821 /* We only propagate across edges with non-interposable callee. */
1824 ignore_edge_p (struct cgraph_edge
*e
)
1826 enum availability avail
;
1827 e
->callee
->function_or_virtual_thunk_symbol (&avail
, e
->caller
);
1828 return (avail
<= AVAIL_INTERPOSABLE
);
1831 /* We use greedy algorithm for inlining of small functions:
1832 All inline candidates are put into prioritized heap ordered in
1835 The inlining of small functions is bounded by unit growth parameters. */
1838 inline_small_functions (void)
1840 struct cgraph_node
*node
;
1841 struct cgraph_edge
*edge
;
1842 edge_heap_t
edge_heap (sreal::min ());
1843 auto_bitmap updated_nodes
;
1844 int min_size
, max_size
;
1845 auto_vec
<cgraph_edge
*> new_indirect_edges
;
1846 int initial_size
= 0;
1847 struct cgraph_node
**order
= XCNEWVEC (cgraph_node
*, symtab
->cgraph_count
);
1848 struct cgraph_edge_hook_list
*edge_removal_hook_holder
;
1849 new_indirect_edges
.create (8);
1851 edge_removal_hook_holder
1852 = symtab
->add_edge_removal_hook (&heap_edge_removal_hook
, &edge_heap
);
1854 /* Compute overall unit size and other global parameters used by badness
1857 max_count
= profile_count::uninitialized ();
1858 ipa_reduced_postorder (order
, true, ignore_edge_p
);
1861 FOR_EACH_DEFINED_FUNCTION (node
)
1862 if (!node
->inlined_to
)
1864 if (!node
->alias
&& node
->analyzed
1865 && (node
->has_gimple_body_p () || node
->thunk
.thunk_p
)
1866 && opt_for_fn (node
->decl
, optimize
))
1868 class ipa_fn_summary
*info
= ipa_fn_summaries
->get (node
);
1869 struct ipa_dfs_info
*dfs
= (struct ipa_dfs_info
*) node
->aux
;
1871 /* Do not account external functions, they will be optimized out
1872 if not inlined. Also only count the non-cold portion of program. */
1873 if (inline_account_function_p (node
))
1874 initial_size
+= ipa_size_summaries
->get (node
)->size
;
1875 info
->growth
= estimate_growth (node
);
1878 node
->call_for_symbol_and_aliases (sum_callers
, &num_calls
,
1881 info
->single_caller
= true;
1882 if (dfs
&& dfs
->next_cycle
)
1884 struct cgraph_node
*n2
;
1885 int id
= dfs
->scc_no
+ 1;
1887 n2
= ((struct ipa_dfs_info
*) n2
->aux
)->next_cycle
)
1888 if (opt_for_fn (n2
->decl
, optimize
))
1890 ipa_fn_summary
*info2
= ipa_fn_summaries
->get
1891 (n2
->inlined_to
? n2
->inlined_to
: n2
);
1899 for (edge
= node
->callers
; edge
; edge
= edge
->next_caller
)
1900 max_count
= max_count
.max (edge
->count
.ipa ());
1902 ipa_free_postorder_info ();
1904 = new call_summary
<edge_growth_cache_entry
*> (symtab
, false);
1908 "\nDeciding on inlining of small functions. Starting with size %i.\n",
1911 overall_size
= initial_size
;
1912 max_size
= compute_max_insns (overall_size
);
1913 min_size
= overall_size
;
1915 /* Populate the heap with all edges we might inline. */
1917 FOR_EACH_DEFINED_FUNCTION (node
)
1919 bool update
= false;
1920 struct cgraph_edge
*next
= NULL
;
1921 bool has_speculative
= false;
1923 if (!opt_for_fn (node
->decl
, optimize
))
1927 fprintf (dump_file
, "Enqueueing calls in %s.\n", node
->dump_name ());
1929 for (edge
= node
->callees
; edge
; edge
= next
)
1931 next
= edge
->next_callee
;
1932 if (edge
->inline_failed
1934 && can_inline_edge_p (edge
, true)
1935 && want_inline_small_function_p (edge
, true)
1936 && can_inline_edge_by_limits_p (edge
, true)
1937 && edge
->inline_failed
)
1939 gcc_assert (!edge
->aux
);
1940 update_edge_key (&edge_heap
, edge
);
1942 if (edge
->speculative
)
1943 has_speculative
= true;
1945 if (has_speculative
)
1946 for (edge
= node
->callees
; edge
; edge
= next
)
1947 if (edge
->speculative
&& !speculation_useful_p (edge
,
1950 edge
->resolve_speculation ();
1955 struct cgraph_node
*where
= node
->inlined_to
1956 ? node
->inlined_to
: node
;
1957 ipa_update_overall_fn_summary (where
);
1958 reset_edge_caches (where
);
1959 update_caller_keys (&edge_heap
, where
,
1960 updated_nodes
, NULL
);
1961 update_callee_keys (&edge_heap
, where
,
1963 bitmap_clear (updated_nodes
);
1967 gcc_assert (in_lto_p
1969 || (profile_info
&& flag_branch_probabilities
));
1971 while (!edge_heap
.empty ())
1973 int old_size
= overall_size
;
1974 struct cgraph_node
*where
, *callee
;
1975 sreal badness
= edge_heap
.min_key ();
1976 sreal current_badness
;
1979 edge
= edge_heap
.extract_min ();
1980 gcc_assert (edge
->aux
);
1982 if (!edge
->inline_failed
|| !edge
->callee
->analyzed
)
1986 /* Be sure that caches are maintained consistent.
1987 This check is affected by scaling roundoff errors when compiling for
1988 IPA this we skip it in that case. */
1989 if (!edge
->callee
->count
.ipa_p ()
1990 && (!max_count
.initialized_p () || !max_count
.nonzero_p ()))
1992 sreal cached_badness
= edge_badness (edge
, false);
1994 int old_size_est
= estimate_edge_size (edge
);
1995 sreal old_time_est
= estimate_edge_time (edge
);
1996 int old_hints_est
= estimate_edge_hints (edge
);
1998 if (edge_growth_cache
!= NULL
)
1999 edge_growth_cache
->remove (edge
);
2000 gcc_assert (old_size_est
== estimate_edge_size (edge
));
2001 gcc_assert (old_time_est
== estimate_edge_time (edge
));
2004 gcc_assert (old_hints_est == estimate_edge_hints (edge));
2006 fails with profile feedback because some hints depends on
2007 maybe_hot_edge_p predicate and because callee gets inlined to other
2008 calls, the edge may become cold.
2009 This ought to be fixed by computing relative probabilities
2010 for given invocation but that will be better done once whole
2011 code is converted to sreals. Disable for now and revert to "wrong"
2012 value so enable/disable checking paths agree. */
2013 edge_growth_cache
->get (edge
)->hints
= old_hints_est
+ 1;
2015 /* When updating the edge costs, we only decrease badness in the keys.
2016 Increases of badness are handled lazilly; when we see key with out
2017 of date value on it, we re-insert it now. */
2018 current_badness
= edge_badness (edge
, false);
2019 gcc_assert (cached_badness
== current_badness
);
2020 gcc_assert (current_badness
>= badness
);
2023 current_badness
= edge_badness (edge
, false);
2025 current_badness
= edge_badness (edge
, false);
2027 if (current_badness
!= badness
)
2029 if (edge_heap
.min () && current_badness
> edge_heap
.min_key ())
2031 edge
->aux
= edge_heap
.insert (current_badness
, edge
);
2035 badness
= current_badness
;
2038 if (!can_inline_edge_p (edge
, true)
2039 || !can_inline_edge_by_limits_p (edge
, true))
2041 resolve_noninline_speculation (&edge_heap
, edge
);
2045 callee
= edge
->callee
->ultimate_alias_target ();
2046 growth
= estimate_edge_growth (edge
);
2050 "\nConsidering %s with %i size\n",
2051 callee
->dump_name (),
2052 ipa_size_summaries
->get (callee
)->size
);
2054 " to be inlined into %s in %s:%i\n"
2055 " Estimated badness is %f, frequency %.2f.\n",
2056 edge
->caller
->dump_name (),
2058 && (LOCATION_LOCUS (gimple_location ((const gimple
*)
2060 > BUILTINS_LOCATION
)
2061 ? gimple_filename ((const gimple
*) edge
->call_stmt
)
2064 ? gimple_lineno ((const gimple
*) edge
->call_stmt
)
2066 badness
.to_double (),
2067 edge
->sreal_frequency ().to_double ());
2068 if (edge
->count
.ipa ().initialized_p ())
2070 fprintf (dump_file
, " Called ");
2071 edge
->count
.ipa ().dump (dump_file
);
2072 fprintf (dump_file
, " times\n");
2074 if (dump_flags
& TDF_DETAILS
)
2075 edge_badness (edge
, true);
2078 if (overall_size
+ growth
> max_size
2079 && !DECL_DISREGARD_INLINE_LIMITS (callee
->decl
))
2081 edge
->inline_failed
= CIF_INLINE_UNIT_GROWTH_LIMIT
;
2082 report_inline_failed_reason (edge
);
2083 resolve_noninline_speculation (&edge_heap
, edge
);
2087 if (!want_inline_small_function_p (edge
, true))
2089 resolve_noninline_speculation (&edge_heap
, edge
);
2093 /* Heuristics for inlining small functions work poorly for
2094 recursive calls where we do effects similar to loop unrolling.
2095 When inlining such edge seems profitable, leave decision on
2096 specific inliner. */
2097 if (edge
->recursive_p ())
2099 where
= edge
->caller
;
2100 if (where
->inlined_to
)
2101 where
= where
->inlined_to
;
2102 if (!recursive_inlining (edge
,
2103 opt_for_fn (edge
->caller
->decl
,
2104 flag_indirect_inlining
)
2105 ? &new_indirect_edges
: NULL
))
2107 edge
->inline_failed
= CIF_RECURSIVE_INLINING
;
2108 resolve_noninline_speculation (&edge_heap
, edge
);
2111 reset_edge_caches (where
);
2112 /* Recursive inliner inlines all recursive calls of the function
2113 at once. Consequently we need to update all callee keys. */
2114 if (opt_for_fn (edge
->caller
->decl
, flag_indirect_inlining
))
2115 add_new_edges_to_heap (&edge_heap
, new_indirect_edges
);
2116 update_callee_keys (&edge_heap
, where
, updated_nodes
);
2117 bitmap_clear (updated_nodes
);
2121 struct cgraph_node
*outer_node
= NULL
;
2124 /* Consider the case where self recursive function A is inlined
2125 into B. This is desired optimization in some cases, since it
2126 leads to effect similar of loop peeling and we might completely
2127 optimize out the recursive call. However we must be extra
2130 where
= edge
->caller
;
2131 while (where
->inlined_to
)
2133 if (where
->decl
== callee
->decl
)
2134 outer_node
= where
, depth
++;
2135 where
= where
->callers
->caller
;
2138 && !want_inline_self_recursive_call_p (edge
, outer_node
,
2142 = (DECL_DISREGARD_INLINE_LIMITS (edge
->callee
->decl
)
2143 ? CIF_RECURSIVE_INLINING
: CIF_UNSPECIFIED
);
2144 resolve_noninline_speculation (&edge_heap
, edge
);
2147 else if (depth
&& dump_file
)
2148 fprintf (dump_file
, " Peeling recursion with depth %i\n", depth
);
2150 gcc_checking_assert (!callee
->inlined_to
);
2151 inline_call (edge
, true, &new_indirect_edges
, &overall_size
, true);
2152 add_new_edges_to_heap (&edge_heap
, new_indirect_edges
);
2154 reset_edge_caches (edge
->callee
);
2156 update_callee_keys (&edge_heap
, where
, updated_nodes
);
2158 where
= edge
->caller
;
2159 if (where
->inlined_to
)
2160 where
= where
->inlined_to
;
2162 /* Our profitability metric can depend on local properties
2163 such as number of inlinable calls and size of the function body.
2164 After inlining these properties might change for the function we
2165 inlined into (since it's body size changed) and for the functions
2166 called by function we inlined (since number of it inlinable callers
2168 update_caller_keys (&edge_heap
, where
, updated_nodes
, NULL
);
2169 /* Offline copy count has possibly changed, recompute if profile is
2171 struct cgraph_node
*n
= cgraph_node::get (edge
->callee
->decl
);
2172 if (n
!= edge
->callee
&& n
->analyzed
&& n
->count
.ipa ().initialized_p ())
2173 update_callee_keys (&edge_heap
, n
, updated_nodes
);
2174 bitmap_clear (updated_nodes
);
2176 if (dump_enabled_p ())
2178 ipa_fn_summary
*s
= ipa_fn_summaries
->get (where
);
2180 /* dump_printf can't handle %+i. */
2181 char buf_net_change
[100];
2182 snprintf (buf_net_change
, sizeof buf_net_change
, "%+i",
2183 overall_size
- old_size
);
2185 dump_printf_loc (MSG_OPTIMIZED_LOCATIONS
, edge
->call_stmt
,
2186 " Inlined %C into %C which now has time %f and "
2187 "size %i, net change of %s.\n",
2188 edge
->callee
, edge
->caller
,
2189 s
->time
.to_double (),
2190 ipa_size_summaries
->get (edge
->caller
)->size
,
2193 if (min_size
> overall_size
)
2195 min_size
= overall_size
;
2196 max_size
= compute_max_insns (min_size
);
2199 fprintf (dump_file
, "New minimal size reached: %i\n", min_size
);
2203 free_growth_caches ();
2204 if (dump_enabled_p ())
2205 dump_printf (MSG_NOTE
,
2206 "Unit growth for small function inlining: %i->%i (%i%%)\n",
2207 initial_size
, overall_size
,
2208 initial_size
? overall_size
* 100 / (initial_size
) - 100: 0);
2209 symtab
->remove_edge_removal_hook (edge_removal_hook_holder
);
2212 /* Flatten NODE. Performed both during early inlining and
2213 at IPA inlining time. */
2216 flatten_function (struct cgraph_node
*node
, bool early
, bool update
)
2218 struct cgraph_edge
*e
;
2220 /* We shouldn't be called recursively when we are being processed. */
2221 gcc_assert (node
->aux
== NULL
);
2223 node
->aux
= (void *) node
;
2225 for (e
= node
->callees
; e
; e
= e
->next_callee
)
2227 struct cgraph_node
*orig_callee
;
2228 struct cgraph_node
*callee
= e
->callee
->ultimate_alias_target ();
2230 /* We've hit cycle? It is time to give up. */
2233 if (dump_enabled_p ())
2234 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, e
->call_stmt
,
2235 "Not inlining %C into %C to avoid cycle.\n",
2237 if (cgraph_inline_failed_type (e
->inline_failed
) != CIF_FINAL_ERROR
)
2238 e
->inline_failed
= CIF_RECURSIVE_INLINING
;
2242 /* When the edge is already inlined, we just need to recurse into
2243 it in order to fully flatten the leaves. */
2244 if (!e
->inline_failed
)
2246 flatten_function (callee
, early
, false);
2250 /* Flatten attribute needs to be processed during late inlining. For
2251 extra code quality we however do flattening during early optimization,
2254 ? !can_inline_edge_p (e
, true)
2255 && !can_inline_edge_by_limits_p (e
, true)
2256 : !can_early_inline_edge_p (e
))
2259 if (e
->recursive_p ())
2261 if (dump_enabled_p ())
2262 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, e
->call_stmt
,
2263 "Not inlining: recursive call.\n");
2267 if (gimple_in_ssa_p (DECL_STRUCT_FUNCTION (node
->decl
))
2268 != gimple_in_ssa_p (DECL_STRUCT_FUNCTION (callee
->decl
)))
2270 if (dump_enabled_p ())
2271 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, e
->call_stmt
,
2272 "Not inlining: SSA form does not match.\n");
2276 /* Inline the edge and flatten the inline clone. Avoid
2277 recursing through the original node if the node was cloned. */
2278 if (dump_enabled_p ())
2279 dump_printf_loc (MSG_OPTIMIZED_LOCATIONS
, e
->call_stmt
,
2280 " Inlining %C into %C.\n",
2282 orig_callee
= callee
;
2283 inline_call (e
, true, NULL
, NULL
, false);
2284 if (e
->callee
!= orig_callee
)
2285 orig_callee
->aux
= (void *) node
;
2286 flatten_function (e
->callee
, early
, false);
2287 if (e
->callee
!= orig_callee
)
2288 orig_callee
->aux
= NULL
;
2293 ipa_update_overall_fn_summary (node
->inlined_to
2294 ? node
->inlined_to
: node
);
2297 /* Inline NODE to all callers. Worker for cgraph_for_node_and_aliases.
2298 DATA points to number of calls originally found so we avoid infinite
2302 inline_to_all_callers_1 (struct cgraph_node
*node
, void *data
,
2303 hash_set
<cgraph_node
*> *callers
)
2305 int *num_calls
= (int *)data
;
2306 bool callee_removed
= false;
2308 while (node
->callers
&& !node
->inlined_to
)
2310 struct cgraph_node
*caller
= node
->callers
->caller
;
2312 if (!can_inline_edge_p (node
->callers
, true)
2313 || !can_inline_edge_by_limits_p (node
->callers
, true)
2314 || node
->callers
->recursive_p ())
2317 fprintf (dump_file
, "Uninlinable call found; giving up.\n");
2324 cgraph_node
*ultimate
= node
->ultimate_alias_target ();
2326 "\nInlining %s size %i.\n",
2328 ipa_size_summaries
->get (ultimate
)->size
);
2330 " Called once from %s %i insns.\n",
2331 node
->callers
->caller
->name (),
2332 ipa_size_summaries
->get (node
->callers
->caller
)->size
);
2335 /* Remember which callers we inlined to, delaying updating the
2337 callers
->add (node
->callers
->caller
);
2338 inline_call (node
->callers
, true, NULL
, NULL
, false, &callee_removed
);
2341 " Inlined into %s which now has %i size\n",
2343 ipa_size_summaries
->get (caller
)->size
);
2344 if (!(*num_calls
)--)
2347 fprintf (dump_file
, "New calls found; giving up.\n");
2348 return callee_removed
;
2356 /* Wrapper around inline_to_all_callers_1 doing delayed overall summary
2360 inline_to_all_callers (struct cgraph_node
*node
, void *data
)
2362 hash_set
<cgraph_node
*> callers
;
2363 bool res
= inline_to_all_callers_1 (node
, data
, &callers
);
2364 /* Perform the delayed update of the overall summary of all callers
2365 processed. This avoids quadratic behavior in the cases where
2366 we have a lot of calls to the same function. */
2367 for (hash_set
<cgraph_node
*>::iterator i
= callers
.begin ();
2368 i
!= callers
.end (); ++i
)
2369 ipa_update_overall_fn_summary (*i
);
2373 /* Output overall time estimate. */
2375 dump_overall_stats (void)
2377 sreal sum_weighted
= 0, sum
= 0;
2378 struct cgraph_node
*node
;
2380 FOR_EACH_DEFINED_FUNCTION (node
)
2381 if (!node
->inlined_to
2384 ipa_fn_summary
*s
= ipa_fn_summaries
->get (node
);
2388 if (node
->count
.ipa ().initialized_p ())
2389 sum_weighted
+= s
->time
* node
->count
.ipa ().to_gcov_type ();
2392 fprintf (dump_file
, "Overall time estimate: "
2393 "%f weighted by profile: "
2394 "%f\n", sum
.to_double (), sum_weighted
.to_double ());
2397 /* Output some useful stats about inlining. */
2400 dump_inline_stats (void)
2402 int64_t inlined_cnt
= 0, inlined_indir_cnt
= 0;
2403 int64_t inlined_virt_cnt
= 0, inlined_virt_indir_cnt
= 0;
2404 int64_t noninlined_cnt
= 0, noninlined_indir_cnt
= 0;
2405 int64_t noninlined_virt_cnt
= 0, noninlined_virt_indir_cnt
= 0;
2406 int64_t inlined_speculative
= 0, inlined_speculative_ply
= 0;
2407 int64_t indirect_poly_cnt
= 0, indirect_cnt
= 0;
2408 int64_t reason
[CIF_N_REASONS
][2];
2409 sreal reason_freq
[CIF_N_REASONS
];
2411 struct cgraph_node
*node
;
2413 memset (reason
, 0, sizeof (reason
));
2414 for (i
=0; i
< CIF_N_REASONS
; i
++)
2416 FOR_EACH_DEFINED_FUNCTION (node
)
2418 struct cgraph_edge
*e
;
2419 for (e
= node
->callees
; e
; e
= e
->next_callee
)
2421 if (e
->inline_failed
)
2423 if (e
->count
.ipa ().initialized_p ())
2424 reason
[(int) e
->inline_failed
][0] += e
->count
.ipa ().to_gcov_type ();
2425 reason_freq
[(int) e
->inline_failed
] += e
->sreal_frequency ();
2426 reason
[(int) e
->inline_failed
][1] ++;
2427 if (DECL_VIRTUAL_P (e
->callee
->decl
)
2428 && e
->count
.ipa ().initialized_p ())
2430 if (e
->indirect_inlining_edge
)
2431 noninlined_virt_indir_cnt
+= e
->count
.ipa ().to_gcov_type ();
2433 noninlined_virt_cnt
+= e
->count
.ipa ().to_gcov_type ();
2435 else if (e
->count
.ipa ().initialized_p ())
2437 if (e
->indirect_inlining_edge
)
2438 noninlined_indir_cnt
+= e
->count
.ipa ().to_gcov_type ();
2440 noninlined_cnt
+= e
->count
.ipa ().to_gcov_type ();
2443 else if (e
->count
.ipa ().initialized_p ())
2447 if (DECL_VIRTUAL_P (e
->callee
->decl
))
2448 inlined_speculative_ply
+= e
->count
.ipa ().to_gcov_type ();
2450 inlined_speculative
+= e
->count
.ipa ().to_gcov_type ();
2452 else if (DECL_VIRTUAL_P (e
->callee
->decl
))
2454 if (e
->indirect_inlining_edge
)
2455 inlined_virt_indir_cnt
+= e
->count
.ipa ().to_gcov_type ();
2457 inlined_virt_cnt
+= e
->count
.ipa ().to_gcov_type ();
2461 if (e
->indirect_inlining_edge
)
2462 inlined_indir_cnt
+= e
->count
.ipa ().to_gcov_type ();
2464 inlined_cnt
+= e
->count
.ipa ().to_gcov_type ();
2468 for (e
= node
->indirect_calls
; e
; e
= e
->next_callee
)
2469 if (e
->indirect_info
->polymorphic
2470 & e
->count
.ipa ().initialized_p ())
2471 indirect_poly_cnt
+= e
->count
.ipa ().to_gcov_type ();
2472 else if (e
->count
.ipa ().initialized_p ())
2473 indirect_cnt
+= e
->count
.ipa ().to_gcov_type ();
2475 if (max_count
.initialized_p ())
2478 "Inlined %" PRId64
" + speculative "
2479 "%" PRId64
" + speculative polymorphic "
2480 "%" PRId64
" + previously indirect "
2481 "%" PRId64
" + virtual "
2482 "%" PRId64
" + virtual and previously indirect "
2483 "%" PRId64
"\n" "Not inlined "
2484 "%" PRId64
" + previously indirect "
2485 "%" PRId64
" + virtual "
2486 "%" PRId64
" + virtual and previously indirect "
2487 "%" PRId64
" + stil indirect "
2488 "%" PRId64
" + still indirect polymorphic "
2489 "%" PRId64
"\n", inlined_cnt
,
2490 inlined_speculative
, inlined_speculative_ply
,
2491 inlined_indir_cnt
, inlined_virt_cnt
, inlined_virt_indir_cnt
,
2492 noninlined_cnt
, noninlined_indir_cnt
, noninlined_virt_cnt
,
2493 noninlined_virt_indir_cnt
, indirect_cnt
, indirect_poly_cnt
);
2494 fprintf (dump_file
, "Removed speculations ");
2495 spec_rem
.dump (dump_file
);
2496 fprintf (dump_file
, "\n");
2498 dump_overall_stats ();
2499 fprintf (dump_file
, "\nWhy inlining failed?\n");
2500 for (i
= 0; i
< CIF_N_REASONS
; i
++)
2502 fprintf (dump_file
, "%-50s: %8i calls, %8f freq, %" PRId64
" count\n",
2503 cgraph_inline_failed_string ((cgraph_inline_failed_t
) i
),
2504 (int) reason
[i
][1], reason_freq
[i
].to_double (), reason
[i
][0]);
2507 /* Called when node is removed. */
2510 flatten_remove_node_hook (struct cgraph_node
*node
, void *data
)
2512 if (lookup_attribute ("flatten", DECL_ATTRIBUTES (node
->decl
)) == NULL
)
2515 hash_set
<struct cgraph_node
*> *removed
2516 = (hash_set
<struct cgraph_node
*> *) data
;
2517 removed
->add (node
);
2520 /* Decide on the inlining. We do so in the topological order to avoid
2521 expenses on updating data structures. */
2526 struct cgraph_node
*node
;
2528 struct cgraph_node
**order
;
2531 bool remove_functions
= false;
2533 order
= XCNEWVEC (struct cgraph_node
*, symtab
->cgraph_count
);
2536 ipa_dump_fn_summaries (dump_file
);
2538 nnodes
= ipa_reverse_postorder (order
);
2539 spec_rem
= profile_count::zero ();
2541 FOR_EACH_FUNCTION (node
)
2545 /* Recompute the default reasons for inlining because they may have
2546 changed during merging. */
2549 for (cgraph_edge
*e
= node
->callees
; e
; e
= e
->next_callee
)
2551 gcc_assert (e
->inline_failed
);
2552 initialize_inline_failed (e
);
2554 for (cgraph_edge
*e
= node
->indirect_calls
; e
; e
= e
->next_callee
)
2555 initialize_inline_failed (e
);
2560 fprintf (dump_file
, "\nFlattening functions:\n");
2562 /* First shrink order array, so that it only contains nodes with
2563 flatten attribute. */
2564 for (i
= nnodes
- 1, j
= i
; i
>= 0; i
--)
2567 if (lookup_attribute ("flatten",
2568 DECL_ATTRIBUTES (node
->decl
)) != NULL
)
2569 order
[j
--] = order
[i
];
2572 /* After the above loop, order[j + 1] ... order[nnodes - 1] contain
2573 nodes with flatten attribute. If there is more than one such
2574 node, we need to register a node removal hook, as flatten_function
2575 could remove other nodes with flatten attribute. See PR82801. */
2576 struct cgraph_node_hook_list
*node_removal_hook_holder
= NULL
;
2577 hash_set
<struct cgraph_node
*> *flatten_removed_nodes
= NULL
;
2580 flatten_removed_nodes
= new hash_set
<struct cgraph_node
*>;
2581 node_removal_hook_holder
2582 = symtab
->add_cgraph_removal_hook (&flatten_remove_node_hook
,
2583 flatten_removed_nodes
);
2586 /* In the first pass handle functions to be flattened. Do this with
2587 a priority so none of our later choices will make this impossible. */
2588 for (i
= nnodes
- 1; i
> j
; i
--)
2591 if (flatten_removed_nodes
2592 && flatten_removed_nodes
->contains (node
))
2595 /* Handle nodes to be flattened.
2596 Ideally when processing callees we stop inlining at the
2597 entry of cycles, possibly cloning that entry point and
2598 try to flatten itself turning it into a self-recursive
2601 fprintf (dump_file
, "Flattening %s\n", node
->name ());
2602 flatten_function (node
, false, true);
2607 symtab
->remove_cgraph_removal_hook (node_removal_hook_holder
);
2608 delete flatten_removed_nodes
;
2613 dump_overall_stats ();
2615 inline_small_functions ();
2617 gcc_assert (symtab
->state
== IPA_SSA
);
2618 symtab
->state
= IPA_SSA_AFTER_INLINING
;
2619 /* Do first after-inlining removal. We want to remove all "stale" extern
2620 inline functions and virtual functions so we really know what is called
2622 symtab
->remove_unreachable_nodes (dump_file
);
2624 /* Inline functions with a property that after inlining into all callers the
2625 code size will shrink because the out-of-line copy is eliminated.
2626 We do this regardless on the callee size as long as function growth limits
2630 "\nDeciding on functions to be inlined into all callers and "
2631 "removing useless speculations:\n");
2633 /* Inlining one function called once has good chance of preventing
2634 inlining other function into the same callee. Ideally we should
2635 work in priority order, but probably inlining hot functions first
2636 is good cut without the extra pain of maintaining the queue.
2638 ??? this is not really fitting the bill perfectly: inlining function
2639 into callee often leads to better optimization of callee due to
2640 increased context for optimization.
2641 For example if main() function calls a function that outputs help
2642 and then function that does the main optmization, we should inline
2643 the second with priority even if both calls are cold by themselves.
2645 We probably want to implement new predicate replacing our use of
2646 maybe_hot_edge interpreted as maybe_hot_edge || callee is known
2648 for (cold
= 0; cold
<= 1; cold
++)
2650 FOR_EACH_DEFINED_FUNCTION (node
)
2652 struct cgraph_edge
*edge
, *next
;
2655 if (!opt_for_fn (node
->decl
, optimize
)
2656 || !opt_for_fn (node
->decl
, flag_inline_functions_called_once
))
2659 for (edge
= node
->callees
; edge
; edge
= next
)
2661 next
= edge
->next_callee
;
2662 if (edge
->speculative
&& !speculation_useful_p (edge
, false))
2664 if (edge
->count
.ipa ().initialized_p ())
2665 spec_rem
+= edge
->count
.ipa ();
2666 edge
->resolve_speculation ();
2668 remove_functions
= true;
2673 struct cgraph_node
*where
= node
->inlined_to
2674 ? node
->inlined_to
: node
;
2675 reset_edge_caches (where
);
2676 ipa_update_overall_fn_summary (where
);
2678 if (want_inline_function_to_all_callers_p (node
, cold
))
2681 node
->call_for_symbol_and_aliases (sum_callers
, &num_calls
,
2683 while (node
->call_for_symbol_and_aliases
2684 (inline_to_all_callers
, &num_calls
, true))
2686 remove_functions
= true;
2691 /* Free ipa-prop structures if they are no longer needed. */
2692 ipa_free_all_structures_after_iinln ();
2694 if (dump_enabled_p ())
2695 dump_printf (MSG_NOTE
,
2696 "\nInlined %i calls, eliminated %i functions\n\n",
2697 ncalls_inlined
, nfunctions_inlined
);
2699 dump_inline_stats ();
2702 ipa_dump_fn_summaries (dump_file
);
2703 return remove_functions
? TODO_remove_functions
: 0;
2706 /* Inline always-inline function calls in NODE. */
2709 inline_always_inline_functions (struct cgraph_node
*node
)
2711 struct cgraph_edge
*e
;
2712 bool inlined
= false;
2714 for (e
= node
->callees
; e
; e
= e
->next_callee
)
2716 struct cgraph_node
*callee
= e
->callee
->ultimate_alias_target ();
2717 if (!DECL_DISREGARD_INLINE_LIMITS (callee
->decl
))
2720 if (e
->recursive_p ())
2722 if (dump_enabled_p ())
2723 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, e
->call_stmt
,
2724 " Not inlining recursive call to %C.\n",
2726 e
->inline_failed
= CIF_RECURSIVE_INLINING
;
2730 if (!can_early_inline_edge_p (e
))
2732 /* Set inlined to true if the callee is marked "always_inline" but
2733 is not inlinable. This will allow flagging an error later in
2734 expand_call_inline in tree-inline.c. */
2735 if (lookup_attribute ("always_inline",
2736 DECL_ATTRIBUTES (callee
->decl
)) != NULL
)
2741 if (dump_enabled_p ())
2742 dump_printf_loc (MSG_OPTIMIZED_LOCATIONS
, e
->call_stmt
,
2743 " Inlining %C into %C (always_inline).\n",
2744 e
->callee
, e
->caller
);
2745 inline_call (e
, true, NULL
, NULL
, false);
2749 ipa_update_overall_fn_summary (node
);
2754 /* Decide on the inlining. We do so in the topological order to avoid
2755 expenses on updating data structures. */
2758 early_inline_small_functions (struct cgraph_node
*node
)
2760 struct cgraph_edge
*e
;
2761 bool inlined
= false;
2763 for (e
= node
->callees
; e
; e
= e
->next_callee
)
2765 struct cgraph_node
*callee
= e
->callee
->ultimate_alias_target ();
2767 /* We can enounter not-yet-analyzed function during
2768 early inlining on callgraphs with strongly
2769 connected components. */
2770 ipa_fn_summary
*s
= ipa_fn_summaries
->get (callee
);
2771 if (s
== NULL
|| !s
->inlinable
|| !e
->inline_failed
)
2774 /* Do not consider functions not declared inline. */
2775 if (!DECL_DECLARED_INLINE_P (callee
->decl
)
2776 && !opt_for_fn (node
->decl
, flag_inline_small_functions
)
2777 && !opt_for_fn (node
->decl
, flag_inline_functions
))
2780 if (dump_enabled_p ())
2781 dump_printf_loc (MSG_NOTE
, e
->call_stmt
,
2782 "Considering inline candidate %C.\n",
2785 if (!can_early_inline_edge_p (e
))
2788 if (e
->recursive_p ())
2790 if (dump_enabled_p ())
2791 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, e
->call_stmt
,
2792 " Not inlining: recursive call.\n");
2796 if (!want_early_inline_function_p (e
))
2799 if (dump_enabled_p ())
2800 dump_printf_loc (MSG_OPTIMIZED_LOCATIONS
, e
->call_stmt
,
2801 " Inlining %C into %C.\n",
2803 inline_call (e
, true, NULL
, NULL
, false);
2808 ipa_update_overall_fn_summary (node
);
2814 early_inliner (function
*fun
)
2816 struct cgraph_node
*node
= cgraph_node::get (current_function_decl
);
2817 struct cgraph_edge
*edge
;
2818 unsigned int todo
= 0;
2820 bool inlined
= false;
2825 /* Do nothing if datastructures for ipa-inliner are already computed. This
2826 happens when some pass decides to construct new function and
2827 cgraph_add_new_function calls lowering passes and early optimization on
2828 it. This may confuse ourself when early inliner decide to inline call to
2829 function clone, because function clones don't have parameter list in
2830 ipa-prop matching their signature. */
2831 if (ipa_node_params_sum
)
2836 node
->remove_all_references ();
2838 /* Even when not optimizing or not inlining inline always-inline
2840 inlined
= inline_always_inline_functions (node
);
2844 || !flag_early_inlining
2845 /* Never inline regular functions into always-inline functions
2846 during incremental inlining. This sucks as functions calling
2847 always inline functions will get less optimized, but at the
2848 same time inlining of functions calling always inline
2849 function into an always inline function might introduce
2850 cycles of edges to be always inlined in the callgraph.
2852 We might want to be smarter and just avoid this type of inlining. */
2853 || (DECL_DISREGARD_INLINE_LIMITS (node
->decl
)
2854 && lookup_attribute ("always_inline",
2855 DECL_ATTRIBUTES (node
->decl
))))
2857 else if (lookup_attribute ("flatten",
2858 DECL_ATTRIBUTES (node
->decl
)) != NULL
)
2860 /* When the function is marked to be flattened, recursively inline
2862 if (dump_enabled_p ())
2863 dump_printf (MSG_OPTIMIZED_LOCATIONS
,
2864 "Flattening %C\n", node
);
2865 flatten_function (node
, true, true);
2870 /* If some always_inline functions was inlined, apply the changes.
2871 This way we will not account always inline into growth limits and
2872 moreover we will inline calls from always inlines that we skipped
2873 previously because of conditional above. */
2876 timevar_push (TV_INTEGRATION
);
2877 todo
|= optimize_inline_calls (current_function_decl
);
2878 /* optimize_inline_calls call above might have introduced new
2879 statements that don't have inline parameters computed. */
2880 for (edge
= node
->callees
; edge
; edge
= edge
->next_callee
)
2882 /* We can enounter not-yet-analyzed function during
2883 early inlining on callgraphs with strongly
2884 connected components. */
2885 ipa_call_summary
*es
= ipa_call_summaries
->get_create (edge
);
2887 = estimate_num_insns (edge
->call_stmt
, &eni_size_weights
);
2889 = estimate_num_insns (edge
->call_stmt
, &eni_time_weights
);
2891 ipa_update_overall_fn_summary (node
);
2893 timevar_pop (TV_INTEGRATION
);
2895 /* We iterate incremental inlining to get trivial cases of indirect
2897 while (iterations
< PARAM_VALUE (PARAM_EARLY_INLINER_MAX_ITERATIONS
)
2898 && early_inline_small_functions (node
))
2900 timevar_push (TV_INTEGRATION
);
2901 todo
|= optimize_inline_calls (current_function_decl
);
2903 /* Technically we ought to recompute inline parameters so the new
2904 iteration of early inliner works as expected. We however have
2905 values approximately right and thus we only need to update edge
2906 info that might be cleared out for newly discovered edges. */
2907 for (edge
= node
->callees
; edge
; edge
= edge
->next_callee
)
2909 /* We have no summary for new bound store calls yet. */
2910 ipa_call_summary
*es
= ipa_call_summaries
->get_create (edge
);
2912 = estimate_num_insns (edge
->call_stmt
, &eni_size_weights
);
2914 = estimate_num_insns (edge
->call_stmt
, &eni_time_weights
);
2916 if (edge
->callee
->decl
2917 && !gimple_check_call_matching_types (
2918 edge
->call_stmt
, edge
->callee
->decl
, false))
2920 edge
->inline_failed
= CIF_MISMATCHED_ARGUMENTS
;
2921 edge
->call_stmt_cannot_inline_p
= true;
2924 if (iterations
< PARAM_VALUE (PARAM_EARLY_INLINER_MAX_ITERATIONS
) - 1)
2925 ipa_update_overall_fn_summary (node
);
2926 timevar_pop (TV_INTEGRATION
);
2931 fprintf (dump_file
, "Iterations: %i\n", iterations
);
2936 timevar_push (TV_INTEGRATION
);
2937 todo
|= optimize_inline_calls (current_function_decl
);
2938 timevar_pop (TV_INTEGRATION
);
2941 fun
->always_inline_functions_inlined
= true;
2946 /* Do inlining of small functions. Doing so early helps profiling and other
2947 passes to be somewhat more effective and avoids some code duplication in
2948 later real inlining pass for testcases with very many function calls. */
2952 const pass_data pass_data_early_inline
=
2954 GIMPLE_PASS
, /* type */
2955 "einline", /* name */
2956 OPTGROUP_INLINE
, /* optinfo_flags */
2957 TV_EARLY_INLINING
, /* tv_id */
2958 PROP_ssa
, /* properties_required */
2959 0, /* properties_provided */
2960 0, /* properties_destroyed */
2961 0, /* todo_flags_start */
2962 0, /* todo_flags_finish */
2965 class pass_early_inline
: public gimple_opt_pass
2968 pass_early_inline (gcc::context
*ctxt
)
2969 : gimple_opt_pass (pass_data_early_inline
, ctxt
)
2972 /* opt_pass methods: */
2973 virtual unsigned int execute (function
*);
2975 }; // class pass_early_inline
2978 pass_early_inline::execute (function
*fun
)
2980 return early_inliner (fun
);
2986 make_pass_early_inline (gcc::context
*ctxt
)
2988 return new pass_early_inline (ctxt
);
2993 const pass_data pass_data_ipa_inline
=
2995 IPA_PASS
, /* type */
2996 "inline", /* name */
2997 OPTGROUP_INLINE
, /* optinfo_flags */
2998 TV_IPA_INLINING
, /* tv_id */
2999 0, /* properties_required */
3000 0, /* properties_provided */
3001 0, /* properties_destroyed */
3002 0, /* todo_flags_start */
3003 ( TODO_dump_symtab
), /* todo_flags_finish */
3006 class pass_ipa_inline
: public ipa_opt_pass_d
3009 pass_ipa_inline (gcc::context
*ctxt
)
3010 : ipa_opt_pass_d (pass_data_ipa_inline
, ctxt
,
3011 NULL
, /* generate_summary */
3012 NULL
, /* write_summary */
3013 NULL
, /* read_summary */
3014 NULL
, /* write_optimization_summary */
3015 NULL
, /* read_optimization_summary */
3016 NULL
, /* stmt_fixup */
3017 0, /* function_transform_todo_flags_start */
3018 inline_transform
, /* function_transform */
3019 NULL
) /* variable_transform */
3022 /* opt_pass methods: */
3023 virtual unsigned int execute (function
*) { return ipa_inline (); }
3025 }; // class pass_ipa_inline
3030 make_pass_ipa_inline (gcc::context
*ctxt
)
3032 return new pass_ipa_inline (ctxt
);