1 /* Inlining decision heuristics.
2 Copyright (C) 2003-2019 Free Software Foundation, Inc.
3 Contributed by Jan Hubicka
5 This file is part of GCC.
7 GCC is free software; you can redistribute it and/or modify it under
8 the terms of the GNU General Public License as published by the Free
9 Software Foundation; either version 3, or (at your option) any later
12 GCC is distributed in the hope that it will be useful, but WITHOUT ANY
13 WARRANTY; without even the implied warranty of MERCHANTABILITY or
14 FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
17 You should have received a copy of the GNU General Public License
18 along with GCC; see the file COPYING3. If not see
19 <http://www.gnu.org/licenses/>. */
21 /* Inlining decision heuristics
23 The implementation of inliner is organized as follows:
25 inlining heuristics limits
27 can_inline_edge_p allow to check that particular inlining is allowed
28 by the limits specified by user (allowed function growth, growth and so
31 Functions are inlined when it is obvious the result is profitable (such
32 as functions called once or when inlining reduce code size).
33 In addition to that we perform inlining of small functions and recursive
38 The inliner itself is split into two passes:
42 Simple local inlining pass inlining callees into current function.
43 This pass makes no use of whole unit analysis and thus it can do only
44 very simple decisions based on local properties.
46 The strength of the pass is that it is run in topological order
47 (reverse postorder) on the callgraph. Functions are converted into SSA
48 form just before this pass and optimized subsequently. As a result, the
49 callees of the function seen by the early inliner was already optimized
50 and results of early inlining adds a lot of optimization opportunities
51 for the local optimization.
53 The pass handle the obvious inlining decisions within the compilation
54 unit - inlining auto inline functions, inlining for size and
57 main strength of the pass is the ability to eliminate abstraction
58 penalty in C++ code (via combination of inlining and early
59 optimization) and thus improve quality of analysis done by real IPA
62 Because of lack of whole unit knowledge, the pass cannot really make
63 good code size/performance tradeoffs. It however does very simple
64 speculative inlining allowing code size to grow by
65 EARLY_INLINING_INSNS when callee is leaf function. In this case the
66 optimizations performed later are very likely to eliminate the cost.
70 This is the real inliner able to handle inlining with whole program
71 knowledge. It performs following steps:
73 1) inlining of small functions. This is implemented by greedy
74 algorithm ordering all inlinable cgraph edges by their badness and
75 inlining them in this order as long as inline limits allows doing so.
77 This heuristics is not very good on inlining recursive calls. Recursive
78 calls can be inlined with results similar to loop unrolling. To do so,
79 special purpose recursive inliner is executed on function when
80 recursive edge is met as viable candidate.
82 2) Unreachable functions are removed from callgraph. Inlining leads
83 to devirtualization and other modification of callgraph so functions
84 may become unreachable during the process. Also functions declared as
85 extern inline or virtual functions are removed, since after inlining
86 we no longer need the offline bodies.
88 3) Functions called once and not exported from the unit are inlined.
89 This should almost always lead to reduction of code size by eliminating
90 the need for offline copy of the function. */
94 #include "coretypes.h"
100 #include "alloc-pool.h"
101 #include "tree-pass.h"
102 #include "gimple-ssa.h"
104 #include "lto-streamer.h"
105 #include "trans-mem.h"
107 #include "tree-inline.h"
110 #include "symbol-summary.h"
111 #include "tree-vrp.h"
112 #include "ipa-prop.h"
113 #include "ipa-fnsummary.h"
114 #include "ipa-inline.h"
115 #include "ipa-utils.h"
117 #include "auto-profile.h"
118 #include "builtins.h"
119 #include "fibonacci_heap.h"
120 #include "stringpool.h"
124 typedef fibonacci_heap
<sreal
, cgraph_edge
> edge_heap_t
;
125 typedef fibonacci_node
<sreal
, cgraph_edge
> edge_heap_node_t
;
127 /* Statistics we collect about inlining algorithm. */
128 static int overall_size
;
129 static profile_count max_count
;
130 static profile_count spec_rem
;
132 /* Return false when inlining edge E would lead to violating
133 limits on function unit growth or stack usage growth.
135 The relative function body growth limit is present generally
136 to avoid problems with non-linear behavior of the compiler.
137 To allow inlining huge functions into tiny wrapper, the limit
138 is always based on the bigger of the two functions considered.
140 For stack growth limits we always base the growth in stack usage
141 of the callers. We want to prevent applications from segfaulting
142 on stack overflow when functions with huge stack frames gets
146 caller_growth_limits (struct cgraph_edge
*e
)
148 struct cgraph_node
*to
= e
->caller
;
149 struct cgraph_node
*what
= e
->callee
->ultimate_alias_target ();
152 HOST_WIDE_INT stack_size_limit
= 0, inlined_stack
;
153 ipa_fn_summary
*info
, *what_info
;
154 ipa_fn_summary
*outer_info
= ipa_fn_summaries
->get (to
);
156 /* Look for function e->caller is inlined to. While doing
157 so work out the largest function body on the way. As
158 described above, we want to base our function growth
159 limits based on that. Not on the self size of the
160 outer function, not on the self size of inline code
161 we immediately inline to. This is the most relaxed
162 interpretation of the rule "do not grow large functions
163 too much in order to prevent compiler from exploding". */
166 info
= ipa_fn_summaries
->get (to
);
167 if (limit
< info
->self_size
)
168 limit
= info
->self_size
;
169 if (stack_size_limit
< info
->estimated_self_stack_size
)
170 stack_size_limit
= info
->estimated_self_stack_size
;
171 if (to
->global
.inlined_to
)
172 to
= to
->callers
->caller
;
177 what_info
= ipa_fn_summaries
->get (what
);
179 if (limit
< what_info
->self_size
)
180 limit
= what_info
->self_size
;
182 limit
+= limit
* PARAM_VALUE (PARAM_LARGE_FUNCTION_GROWTH
) / 100;
184 /* Check the size after inlining against the function limits. But allow
185 the function to shrink if it went over the limits by forced inlining. */
186 newsize
= estimate_size_after_inlining (to
, e
);
187 if (newsize
>= info
->size
188 && newsize
> PARAM_VALUE (PARAM_LARGE_FUNCTION_INSNS
)
191 e
->inline_failed
= CIF_LARGE_FUNCTION_GROWTH_LIMIT
;
195 if (!what_info
->estimated_stack_size
)
198 /* FIXME: Stack size limit often prevents inlining in Fortran programs
199 due to large i/o datastructures used by the Fortran front-end.
200 We ought to ignore this limit when we know that the edge is executed
201 on every invocation of the caller (i.e. its call statement dominates
202 exit block). We do not track this information, yet. */
203 stack_size_limit
+= ((gcov_type
)stack_size_limit
204 * PARAM_VALUE (PARAM_STACK_FRAME_GROWTH
) / 100);
206 inlined_stack
= (outer_info
->stack_frame_offset
207 + outer_info
->estimated_self_stack_size
208 + what_info
->estimated_stack_size
);
209 /* Check new stack consumption with stack consumption at the place
211 if (inlined_stack
> stack_size_limit
212 /* If function already has large stack usage from sibling
213 inline call, we can inline, too.
214 This bit overoptimistically assume that we are good at stack
216 && inlined_stack
> info
->estimated_stack_size
217 && inlined_stack
> PARAM_VALUE (PARAM_LARGE_STACK_FRAME
))
219 e
->inline_failed
= CIF_LARGE_STACK_FRAME_GROWTH_LIMIT
;
225 /* Dump info about why inlining has failed. */
228 report_inline_failed_reason (struct cgraph_edge
*e
)
230 if (dump_enabled_p ())
232 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, e
->call_stmt
,
233 " not inlinable: %C -> %C, %s\n",
234 e
->caller
, e
->callee
,
235 cgraph_inline_failed_string (e
->inline_failed
));
236 if ((e
->inline_failed
== CIF_TARGET_OPTION_MISMATCH
237 || e
->inline_failed
== CIF_OPTIMIZATION_MISMATCH
)
238 && e
->caller
->lto_file_data
239 && e
->callee
->ultimate_alias_target ()->lto_file_data
)
241 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, e
->call_stmt
,
242 " LTO objects: %s, %s\n",
243 e
->caller
->lto_file_data
->file_name
,
244 e
->callee
->ultimate_alias_target ()->lto_file_data
->file_name
);
246 if (e
->inline_failed
== CIF_TARGET_OPTION_MISMATCH
)
248 cl_target_option_print_diff
249 (dump_file
, 2, target_opts_for_fn (e
->caller
->decl
),
250 target_opts_for_fn (e
->callee
->ultimate_alias_target ()->decl
));
251 if (e
->inline_failed
== CIF_OPTIMIZATION_MISMATCH
)
253 cl_optimization_print_diff
254 (dump_file
, 2, opts_for_fn (e
->caller
->decl
),
255 opts_for_fn (e
->callee
->ultimate_alias_target ()->decl
));
259 /* Decide whether sanitizer-related attributes allow inlining. */
262 sanitize_attrs_match_for_inline_p (const_tree caller
, const_tree callee
)
264 if (!caller
|| !callee
)
267 /* Allow inlining always_inline functions into no_sanitize_address
269 if (!sanitize_flags_p (SANITIZE_ADDRESS
, caller
)
270 && lookup_attribute ("always_inline", DECL_ATTRIBUTES (callee
)))
273 return ((sanitize_flags_p (SANITIZE_ADDRESS
, caller
)
274 == sanitize_flags_p (SANITIZE_ADDRESS
, callee
))
275 && (sanitize_flags_p (SANITIZE_POINTER_COMPARE
, caller
)
276 == sanitize_flags_p (SANITIZE_POINTER_COMPARE
, callee
))
277 && (sanitize_flags_p (SANITIZE_POINTER_SUBTRACT
, caller
)
278 == sanitize_flags_p (SANITIZE_POINTER_SUBTRACT
, callee
)));
281 /* Used for flags where it is safe to inline when caller's value is
282 grater than callee's. */
283 #define check_maybe_up(flag) \
284 (opts_for_fn (caller->decl)->x_##flag \
285 != opts_for_fn (callee->decl)->x_##flag \
287 || opts_for_fn (caller->decl)->x_##flag \
288 < opts_for_fn (callee->decl)->x_##flag))
289 /* Used for flags where it is safe to inline when caller's value is
290 smaller than callee's. */
291 #define check_maybe_down(flag) \
292 (opts_for_fn (caller->decl)->x_##flag \
293 != opts_for_fn (callee->decl)->x_##flag \
295 || opts_for_fn (caller->decl)->x_##flag \
296 > opts_for_fn (callee->decl)->x_##flag))
297 /* Used for flags where exact match is needed for correctness. */
298 #define check_match(flag) \
299 (opts_for_fn (caller->decl)->x_##flag \
300 != opts_for_fn (callee->decl)->x_##flag)
302 /* Decide if we can inline the edge and possibly update
303 inline_failed reason.
304 We check whether inlining is possible at all and whether
305 caller growth limits allow doing so.
307 if REPORT is true, output reason to the dump file. */
310 can_inline_edge_p (struct cgraph_edge
*e
, bool report
,
313 gcc_checking_assert (e
->inline_failed
);
315 if (cgraph_inline_failed_type (e
->inline_failed
) == CIF_FINAL_ERROR
)
318 report_inline_failed_reason (e
);
322 bool inlinable
= true;
323 enum availability avail
;
324 cgraph_node
*caller
= e
->caller
->global
.inlined_to
325 ? e
->caller
->global
.inlined_to
: e
->caller
;
326 cgraph_node
*callee
= e
->callee
->ultimate_alias_target (&avail
, caller
);
328 if (!callee
->definition
)
330 e
->inline_failed
= CIF_BODY_NOT_AVAILABLE
;
333 if (!early
&& (!opt_for_fn (callee
->decl
, optimize
)
334 || !opt_for_fn (caller
->decl
, optimize
)))
336 e
->inline_failed
= CIF_FUNCTION_NOT_OPTIMIZED
;
339 else if (callee
->calls_comdat_local
)
341 e
->inline_failed
= CIF_USES_COMDAT_LOCAL
;
344 else if (avail
<= AVAIL_INTERPOSABLE
)
346 e
->inline_failed
= CIF_OVERWRITABLE
;
349 /* All edges with call_stmt_cannot_inline_p should have inline_failed
350 initialized to one of FINAL_ERROR reasons. */
351 else if (e
->call_stmt_cannot_inline_p
)
353 /* Don't inline if the functions have different EH personalities. */
354 else if (DECL_FUNCTION_PERSONALITY (caller
->decl
)
355 && DECL_FUNCTION_PERSONALITY (callee
->decl
)
356 && (DECL_FUNCTION_PERSONALITY (caller
->decl
)
357 != DECL_FUNCTION_PERSONALITY (callee
->decl
)))
359 e
->inline_failed
= CIF_EH_PERSONALITY
;
362 /* TM pure functions should not be inlined into non-TM_pure
364 else if (is_tm_pure (callee
->decl
) && !is_tm_pure (caller
->decl
))
366 e
->inline_failed
= CIF_UNSPECIFIED
;
369 /* Check compatibility of target optimization options. */
370 else if (!targetm
.target_option
.can_inline_p (caller
->decl
,
373 e
->inline_failed
= CIF_TARGET_OPTION_MISMATCH
;
376 else if (ipa_fn_summaries
->get (callee
) == NULL
377 || !ipa_fn_summaries
->get (callee
)->inlinable
)
379 e
->inline_failed
= CIF_FUNCTION_NOT_INLINABLE
;
382 /* Don't inline a function with mismatched sanitization attributes. */
383 else if (!sanitize_attrs_match_for_inline_p (caller
->decl
, callee
->decl
))
385 e
->inline_failed
= CIF_ATTRIBUTE_MISMATCH
;
388 else if (callee
->externally_visible
389 && flag_live_patching
== LIVE_PATCHING_INLINE_ONLY_STATIC
)
391 e
->inline_failed
= CIF_EXTERN_LIVE_ONLY_STATIC
;
394 if (!inlinable
&& report
)
395 report_inline_failed_reason (e
);
399 /* Decide if we can inline the edge and possibly update
400 inline_failed reason.
401 We check whether inlining is possible at all and whether
402 caller growth limits allow doing so.
404 if REPORT is true, output reason to the dump file.
406 if DISREGARD_LIMITS is true, ignore size limits. */
409 can_inline_edge_by_limits_p (struct cgraph_edge
*e
, bool report
,
410 bool disregard_limits
= false, bool early
= false)
412 gcc_checking_assert (e
->inline_failed
);
414 if (cgraph_inline_failed_type (e
->inline_failed
) == CIF_FINAL_ERROR
)
417 report_inline_failed_reason (e
);
421 bool inlinable
= true;
422 enum availability avail
;
423 cgraph_node
*caller
= e
->caller
->global
.inlined_to
424 ? e
->caller
->global
.inlined_to
: e
->caller
;
425 cgraph_node
*callee
= e
->callee
->ultimate_alias_target (&avail
, caller
);
426 tree caller_tree
= DECL_FUNCTION_SPECIFIC_OPTIMIZATION (caller
->decl
);
428 = callee
? DECL_FUNCTION_SPECIFIC_OPTIMIZATION (callee
->decl
) : NULL
;
429 /* Check if caller growth allows the inlining. */
430 if (!DECL_DISREGARD_INLINE_LIMITS (callee
->decl
)
432 && !lookup_attribute ("flatten",
433 DECL_ATTRIBUTES (caller
->decl
))
434 && !caller_growth_limits (e
))
436 /* Don't inline a function with a higher optimization level than the
437 caller. FIXME: this is really just tip of iceberg of handling
438 optimization attribute. */
439 else if (caller_tree
!= callee_tree
)
442 (DECL_DISREGARD_INLINE_LIMITS (callee
->decl
)
443 && lookup_attribute ("always_inline",
444 DECL_ATTRIBUTES (callee
->decl
)));
445 ipa_fn_summary
*caller_info
= ipa_fn_summaries
->get (caller
);
446 ipa_fn_summary
*callee_info
= ipa_fn_summaries
->get (callee
);
448 /* Until GCC 4.9 we did not check the semantics-altering flags
449 below and inlined across optimization boundaries.
450 Enabling checks below breaks several packages by refusing
451 to inline library always_inline functions. See PR65873.
452 Disable the check for early inlining for now until better solution
454 if (always_inline
&& early
)
456 /* There are some options that change IL semantics which means
457 we cannot inline in these cases for correctness reason.
458 Not even for always_inline declared functions. */
459 else if (check_match (flag_wrapv
)
460 || check_match (flag_trapv
)
461 || check_match (flag_pcc_struct_return
)
462 /* When caller or callee does FP math, be sure FP codegen flags
464 || ((caller_info
->fp_expressions
&& callee_info
->fp_expressions
)
465 && (check_maybe_up (flag_rounding_math
)
466 || check_maybe_up (flag_trapping_math
)
467 || check_maybe_down (flag_unsafe_math_optimizations
)
468 || check_maybe_down (flag_finite_math_only
)
469 || check_maybe_up (flag_signaling_nans
)
470 || check_maybe_down (flag_cx_limited_range
)
471 || check_maybe_up (flag_signed_zeros
)
472 || check_maybe_down (flag_associative_math
)
473 || check_maybe_down (flag_reciprocal_math
)
474 || check_maybe_down (flag_fp_int_builtin_inexact
)
475 /* Strictly speaking only when the callee contains function
476 calls that may end up setting errno. */
477 || check_maybe_up (flag_errno_math
)))
478 /* We do not want to make code compiled with exceptions to be
479 brought into a non-EH function unless we know that the callee
481 This is tracked by DECL_FUNCTION_PERSONALITY. */
482 || (check_maybe_up (flag_non_call_exceptions
)
483 && DECL_FUNCTION_PERSONALITY (callee
->decl
))
484 || (check_maybe_up (flag_exceptions
)
485 && DECL_FUNCTION_PERSONALITY (callee
->decl
))
486 /* When devirtualization is diabled for callee, it is not safe
487 to inline it as we possibly mangled the type info.
488 Allow early inlining of always inlines. */
489 || (!early
&& check_maybe_down (flag_devirtualize
)))
491 e
->inline_failed
= CIF_OPTIMIZATION_MISMATCH
;
494 /* gcc.dg/pr43564.c. Apply user-forced inline even at -O0. */
495 else if (always_inline
)
497 /* When user added an attribute to the callee honor it. */
498 else if (lookup_attribute ("optimize", DECL_ATTRIBUTES (callee
->decl
))
499 && opts_for_fn (caller
->decl
) != opts_for_fn (callee
->decl
))
501 e
->inline_failed
= CIF_OPTIMIZATION_MISMATCH
;
504 /* If explicit optimize attribute are not used, the mismatch is caused
505 by different command line options used to build different units.
506 Do not care about COMDAT functions - those are intended to be
507 optimized with the optimization flags of module they are used in.
508 Also do not care about mixing up size/speed optimization when
509 DECL_DISREGARD_INLINE_LIMITS is set. */
510 else if ((callee
->merged_comdat
511 && !lookup_attribute ("optimize",
512 DECL_ATTRIBUTES (caller
->decl
)))
513 || DECL_DISREGARD_INLINE_LIMITS (callee
->decl
))
515 /* If mismatch is caused by merging two LTO units with different
516 optimizationflags we want to be bit nicer. However never inline
517 if one of functions is not optimized at all. */
518 else if (!opt_for_fn (callee
->decl
, optimize
)
519 || !opt_for_fn (caller
->decl
, optimize
))
521 e
->inline_failed
= CIF_OPTIMIZATION_MISMATCH
;
524 /* If callee is optimized for size and caller is not, allow inlining if
525 code shrinks or we are in MAX_INLINE_INSNS_SINGLE limit and callee
526 is inline (and thus likely an unified comdat). This will allow caller
528 else if (opt_for_fn (callee
->decl
, optimize_size
)
529 > opt_for_fn (caller
->decl
, optimize_size
))
531 int growth
= estimate_edge_growth (e
);
532 if (growth
> PARAM_VALUE (PARAM_MAX_INLINE_INSNS_SIZE
)
533 && (!DECL_DECLARED_INLINE_P (callee
->decl
)
534 && growth
>= MAX (MAX_INLINE_INSNS_SINGLE
,
535 MAX_INLINE_INSNS_AUTO
)))
537 e
->inline_failed
= CIF_OPTIMIZATION_MISMATCH
;
541 /* If callee is more aggressively optimized for performance than caller,
542 we generally want to inline only cheap (runtime wise) functions. */
543 else if (opt_for_fn (callee
->decl
, optimize_size
)
544 < opt_for_fn (caller
->decl
, optimize_size
)
545 || (opt_for_fn (callee
->decl
, optimize
)
546 > opt_for_fn (caller
->decl
, optimize
)))
548 if (estimate_edge_time (e
)
549 >= 20 + ipa_call_summaries
->get (e
)->call_stmt_time
)
551 e
->inline_failed
= CIF_OPTIMIZATION_MISMATCH
;
558 if (!inlinable
&& report
)
559 report_inline_failed_reason (e
);
564 /* Return true if the edge E is inlinable during early inlining. */
567 can_early_inline_edge_p (struct cgraph_edge
*e
)
569 struct cgraph_node
*callee
= e
->callee
->ultimate_alias_target ();
570 /* Early inliner might get called at WPA stage when IPA pass adds new
571 function. In this case we cannot really do any of early inlining
572 because function bodies are missing. */
573 if (cgraph_inline_failed_type (e
->inline_failed
) == CIF_FINAL_ERROR
)
575 if (!gimple_has_body_p (callee
->decl
))
577 e
->inline_failed
= CIF_BODY_NOT_AVAILABLE
;
580 /* In early inliner some of callees may not be in SSA form yet
581 (i.e. the callgraph is cyclic and we did not process
582 the callee by early inliner, yet). We don't have CIF code for this
583 case; later we will re-do the decision in the real inliner. */
584 if (!gimple_in_ssa_p (DECL_STRUCT_FUNCTION (e
->caller
->decl
))
585 || !gimple_in_ssa_p (DECL_STRUCT_FUNCTION (callee
->decl
)))
587 if (dump_enabled_p ())
588 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, e
->call_stmt
,
589 " edge not inlinable: not in SSA form\n");
592 if (!can_inline_edge_p (e
, true, true)
593 || !can_inline_edge_by_limits_p (e
, true, false, true))
599 /* Return number of calls in N. Ignore cheap builtins. */
602 num_calls (struct cgraph_node
*n
)
604 struct cgraph_edge
*e
;
607 for (e
= n
->callees
; e
; e
= e
->next_callee
)
608 if (!is_inexpensive_builtin (e
->callee
->decl
))
614 /* Return true if we are interested in inlining small function. */
617 want_early_inline_function_p (struct cgraph_edge
*e
)
619 bool want_inline
= true;
620 struct cgraph_node
*callee
= e
->callee
->ultimate_alias_target ();
622 if (DECL_DISREGARD_INLINE_LIMITS (callee
->decl
))
624 /* For AutoFDO, we need to make sure that before profile summary, all
625 hot paths' IR look exactly the same as profiled binary. As a result,
626 in einliner, we will disregard size limit and inline those callsites
628 * inlined in the profiled binary, and
629 * the cloned callee has enough samples to be considered "hot". */
630 else if (flag_auto_profile
&& afdo_callsite_hot_enough_for_early_inline (e
))
632 else if (!DECL_DECLARED_INLINE_P (callee
->decl
)
633 && !opt_for_fn (e
->caller
->decl
, flag_inline_small_functions
))
635 e
->inline_failed
= CIF_FUNCTION_NOT_INLINE_CANDIDATE
;
636 report_inline_failed_reason (e
);
641 int growth
= estimate_edge_growth (e
);
644 if (growth
<= PARAM_VALUE (PARAM_MAX_INLINE_INSNS_SIZE
))
646 else if (!e
->maybe_hot_p ())
648 if (dump_enabled_p ())
649 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, e
->call_stmt
,
650 " will not early inline: %C->%C, "
651 "call is cold and code would grow by %i\n",
656 else if (growth
> PARAM_VALUE (PARAM_EARLY_INLINING_INSNS
))
658 if (dump_enabled_p ())
659 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, e
->call_stmt
,
660 " will not early inline: %C->%C, "
661 "growth %i exceeds --param early-inlining-insns\n",
666 else if ((n
= num_calls (callee
)) != 0
667 && growth
* (n
+ 1) > PARAM_VALUE (PARAM_EARLY_INLINING_INSNS
))
669 if (dump_enabled_p ())
670 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, e
->call_stmt
,
671 " will not early inline: %C->%C, "
672 "growth %i exceeds --param early-inlining-insns "
673 "divided by number of calls\n",
682 /* Compute time of the edge->caller + edge->callee execution when inlining
686 compute_uninlined_call_time (struct cgraph_edge
*edge
,
687 sreal uninlined_call_time
)
689 cgraph_node
*caller
= (edge
->caller
->global
.inlined_to
690 ? edge
->caller
->global
.inlined_to
693 sreal freq
= edge
->sreal_frequency ();
695 uninlined_call_time
*= freq
;
697 uninlined_call_time
= uninlined_call_time
>> 11;
699 sreal caller_time
= ipa_fn_summaries
->get (caller
)->time
;
700 return uninlined_call_time
+ caller_time
;
703 /* Same as compute_uinlined_call_time but compute time when inlining
707 compute_inlined_call_time (struct cgraph_edge
*edge
,
710 cgraph_node
*caller
= (edge
->caller
->global
.inlined_to
711 ? edge
->caller
->global
.inlined_to
713 sreal caller_time
= ipa_fn_summaries
->get (caller
)->time
;
715 sreal freq
= edge
->sreal_frequency ();
721 /* This calculation should match one in ipa-inline-analysis.c
722 (estimate_edge_size_and_time). */
723 time
-= (sreal
)ipa_call_summaries
->get (edge
)->call_stmt_time
* freq
;
726 time
= ((sreal
) 1) >> 8;
727 gcc_checking_assert (time
>= 0);
731 /* Return true if the speedup for inlining E is bigger than
732 PARAM_MAX_INLINE_MIN_SPEEDUP. */
735 big_speedup_p (struct cgraph_edge
*e
)
738 sreal spec_time
= estimate_edge_time (e
, &unspec_time
);
739 sreal time
= compute_uninlined_call_time (e
, unspec_time
);
740 sreal inlined_time
= compute_inlined_call_time (e
, spec_time
);
742 if ((time
- inlined_time
) * 100
743 > (sreal
) (time
* PARAM_VALUE (PARAM_INLINE_MIN_SPEEDUP
)))
748 /* Return true if we are interested in inlining small function.
749 When REPORT is true, report reason to dump file. */
752 want_inline_small_function_p (struct cgraph_edge
*e
, bool report
)
754 bool want_inline
= true;
755 struct cgraph_node
*callee
= e
->callee
->ultimate_alias_target ();
757 /* Allow this function to be called before can_inline_edge_p,
758 since it's usually cheaper. */
759 if (cgraph_inline_failed_type (e
->inline_failed
) == CIF_FINAL_ERROR
)
761 else if (DECL_DISREGARD_INLINE_LIMITS (callee
->decl
))
763 else if (!DECL_DECLARED_INLINE_P (callee
->decl
)
764 && !opt_for_fn (e
->caller
->decl
, flag_inline_small_functions
))
766 e
->inline_failed
= CIF_FUNCTION_NOT_INLINE_CANDIDATE
;
769 /* Do fast and conservative check if the function can be good
770 inline candidate. At the moment we allow inline hints to
771 promote non-inline functions to inline and we increase
772 MAX_INLINE_INSNS_SINGLE 16-fold for inline functions. */
773 else if ((!DECL_DECLARED_INLINE_P (callee
->decl
)
774 && (!e
->count
.ipa ().initialized_p () || !e
->maybe_hot_p ()))
775 && ipa_fn_summaries
->get (callee
)->min_size
776 - ipa_call_summaries
->get (e
)->call_stmt_size
777 > MAX (MAX_INLINE_INSNS_SINGLE
, MAX_INLINE_INSNS_AUTO
))
779 e
->inline_failed
= CIF_MAX_INLINE_INSNS_AUTO_LIMIT
;
782 else if ((DECL_DECLARED_INLINE_P (callee
->decl
)
783 || e
->count
.ipa ().nonzero_p ())
784 && ipa_fn_summaries
->get (callee
)->min_size
785 - ipa_call_summaries
->get (e
)->call_stmt_size
786 > 16 * MAX_INLINE_INSNS_SINGLE
)
788 e
->inline_failed
= (DECL_DECLARED_INLINE_P (callee
->decl
)
789 ? CIF_MAX_INLINE_INSNS_SINGLE_LIMIT
790 : CIF_MAX_INLINE_INSNS_AUTO_LIMIT
);
795 int growth
= estimate_edge_growth (e
);
796 ipa_hints hints
= estimate_edge_hints (e
);
797 int big_speedup
= -1; /* compute this lazily */
799 if (growth
<= PARAM_VALUE (PARAM_MAX_INLINE_INSNS_SIZE
))
801 /* Apply MAX_INLINE_INSNS_SINGLE limit. Do not do so when
802 hints suggests that inlining given function is very profitable. */
803 else if (DECL_DECLARED_INLINE_P (callee
->decl
)
804 && growth
>= MAX_INLINE_INSNS_SINGLE
805 && (growth
>= MAX_INLINE_INSNS_SINGLE
* 16
806 || (!(hints
& (INLINE_HINT_indirect_call
807 | INLINE_HINT_known_hot
808 | INLINE_HINT_loop_iterations
809 | INLINE_HINT_array_index
810 | INLINE_HINT_loop_stride
))
811 && !(big_speedup
= big_speedup_p (e
)))))
813 e
->inline_failed
= CIF_MAX_INLINE_INSNS_SINGLE_LIMIT
;
816 else if (!DECL_DECLARED_INLINE_P (callee
->decl
)
817 && !opt_for_fn (e
->caller
->decl
, flag_inline_functions
)
818 && growth
>= PARAM_VALUE (PARAM_MAX_INLINE_INSNS_SMALL
))
820 /* growth_likely_positive is expensive, always test it last. */
821 if (growth
>= MAX_INLINE_INSNS_SINGLE
822 || growth_likely_positive (callee
, growth
))
824 e
->inline_failed
= CIF_NOT_DECLARED_INLINED
;
828 /* Apply MAX_INLINE_INSNS_AUTO limit for functions not declared inline
829 Upgrade it to MAX_INLINE_INSNS_SINGLE when hints suggests that
830 inlining given function is very profitable. */
831 else if (!DECL_DECLARED_INLINE_P (callee
->decl
)
832 && !(hints
& INLINE_HINT_known_hot
)
833 && growth
>= ((hints
& (INLINE_HINT_indirect_call
834 | INLINE_HINT_loop_iterations
835 | INLINE_HINT_array_index
836 | INLINE_HINT_loop_stride
))
837 ? MAX (MAX_INLINE_INSNS_AUTO
,
838 MAX_INLINE_INSNS_SINGLE
)
839 : MAX_INLINE_INSNS_AUTO
)
840 && !(big_speedup
== -1 ? big_speedup_p (e
) : big_speedup
))
842 /* growth_likely_positive is expensive, always test it last. */
843 if (growth
>= MAX_INLINE_INSNS_SINGLE
844 || growth_likely_positive (callee
, growth
))
846 e
->inline_failed
= CIF_MAX_INLINE_INSNS_AUTO_LIMIT
;
850 /* If call is cold, do not inline when function body would grow. */
851 else if (!e
->maybe_hot_p ()
852 && (growth
>= MAX_INLINE_INSNS_SINGLE
853 || growth_likely_positive (callee
, growth
)))
855 e
->inline_failed
= CIF_UNLIKELY_CALL
;
859 if (!want_inline
&& report
)
860 report_inline_failed_reason (e
);
864 /* EDGE is self recursive edge.
865 We hand two cases - when function A is inlining into itself
866 or when function A is being inlined into another inliner copy of function
869 In first case OUTER_NODE points to the toplevel copy of A, while
870 in the second case OUTER_NODE points to the outermost copy of A in B.
872 In both cases we want to be extra selective since
873 inlining the call will just introduce new recursive calls to appear. */
876 want_inline_self_recursive_call_p (struct cgraph_edge
*edge
,
877 struct cgraph_node
*outer_node
,
881 char const *reason
= NULL
;
882 bool want_inline
= true;
883 sreal caller_freq
= 1;
884 int max_depth
= PARAM_VALUE (PARAM_MAX_INLINE_RECURSIVE_DEPTH_AUTO
);
886 if (DECL_DECLARED_INLINE_P (edge
->caller
->decl
))
887 max_depth
= PARAM_VALUE (PARAM_MAX_INLINE_RECURSIVE_DEPTH
);
889 if (!edge
->maybe_hot_p ())
891 reason
= "recursive call is cold";
894 else if (depth
> max_depth
)
896 reason
= "--param max-inline-recursive-depth exceeded.";
899 else if (outer_node
->global
.inlined_to
900 && (caller_freq
= outer_node
->callers
->sreal_frequency ()) == 0)
902 reason
= "caller frequency is 0";
908 /* Inlining of self recursive function into copy of itself within other
909 function is transformation similar to loop peeling.
911 Peeling is profitable if we can inline enough copies to make probability
912 of actual call to the self recursive function very small. Be sure that
913 the probability of recursion is small.
915 We ensure that the frequency of recursing is at most 1 - (1/max_depth).
916 This way the expected number of recursion is at most max_depth. */
919 sreal max_prob
= (sreal
)1 - ((sreal
)1 / (sreal
)max_depth
);
921 for (i
= 1; i
< depth
; i
++)
922 max_prob
= max_prob
* max_prob
;
923 if (edge
->sreal_frequency () >= max_prob
* caller_freq
)
925 reason
= "frequency of recursive call is too large";
929 /* Recursive inlining, i.e. equivalent of unrolling, is profitable if
930 recursion depth is large. We reduce function call overhead and increase
931 chances that things fit in hardware return predictor.
933 Recursive inlining might however increase cost of stack frame setup
934 actually slowing down functions whose recursion tree is wide rather than
937 Deciding reliably on when to do recursive inlining without profile feedback
938 is tricky. For now we disable recursive inlining when probability of self
941 Recursive inlining of self recursive call within loop also results in
942 large loop depths that generally optimize badly. We may want to throttle
943 down inlining in those cases. In particular this seems to happen in one
944 of libstdc++ rb tree methods. */
947 if (edge
->sreal_frequency () * 100
949 * PARAM_VALUE (PARAM_MIN_INLINE_RECURSIVE_PROBABILITY
))
951 reason
= "frequency of recursive call is too small";
955 if (!want_inline
&& dump_enabled_p ())
956 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, edge
->call_stmt
,
957 " not inlining recursively: %s\n", reason
);
961 /* Return true when NODE has uninlinable caller;
962 set HAS_HOT_CALL if it has hot call.
963 Worker for cgraph_for_node_and_aliases. */
966 check_callers (struct cgraph_node
*node
, void *has_hot_call
)
968 struct cgraph_edge
*e
;
969 for (e
= node
->callers
; e
; e
= e
->next_caller
)
971 if (!opt_for_fn (e
->caller
->decl
, flag_inline_functions_called_once
)
972 || !opt_for_fn (e
->caller
->decl
, optimize
))
974 if (!can_inline_edge_p (e
, true))
976 if (e
->recursive_p ())
978 if (!can_inline_edge_by_limits_p (e
, true))
980 if (!(*(bool *)has_hot_call
) && e
->maybe_hot_p ())
981 *(bool *)has_hot_call
= true;
986 /* If NODE has a caller, return true. */
989 has_caller_p (struct cgraph_node
*node
, void *data ATTRIBUTE_UNUSED
)
996 /* Decide if inlining NODE would reduce unit size by eliminating
997 the offline copy of function.
998 When COLD is true the cold calls are considered, too. */
1001 want_inline_function_to_all_callers_p (struct cgraph_node
*node
, bool cold
)
1003 bool has_hot_call
= false;
1005 /* Aliases gets inlined along with the function they alias. */
1008 /* Already inlined? */
1009 if (node
->global
.inlined_to
)
1011 /* Does it have callers? */
1012 if (!node
->call_for_symbol_and_aliases (has_caller_p
, NULL
, true))
1014 /* Inlining into all callers would increase size? */
1015 if (estimate_growth (node
) > 0)
1017 /* All inlines must be possible. */
1018 if (node
->call_for_symbol_and_aliases (check_callers
, &has_hot_call
,
1021 if (!cold
&& !has_hot_call
)
1026 /* A cost model driving the inlining heuristics in a way so the edges with
1027 smallest badness are inlined first. After each inlining is performed
1028 the costs of all caller edges of nodes affected are recomputed so the
1029 metrics may accurately depend on values such as number of inlinable callers
1030 of the function or function body size. */
1033 edge_badness (struct cgraph_edge
*edge
, bool dump
)
1037 sreal edge_time
, unspec_edge_time
;
1038 struct cgraph_node
*callee
= edge
->callee
->ultimate_alias_target ();
1039 struct ipa_fn_summary
*callee_info
= ipa_fn_summaries
->get (callee
);
1041 cgraph_node
*caller
= (edge
->caller
->global
.inlined_to
1042 ? edge
->caller
->global
.inlined_to
1045 growth
= estimate_edge_growth (edge
);
1046 edge_time
= estimate_edge_time (edge
, &unspec_edge_time
);
1047 hints
= estimate_edge_hints (edge
);
1048 gcc_checking_assert (edge_time
>= 0);
1049 /* Check that inlined time is better, but tolerate some roundoff issues.
1050 FIXME: When callee profile drops to 0 we account calls more. This
1051 should be fixed by never doing that. */
1052 gcc_checking_assert ((edge_time
* 100
1053 - callee_info
->time
* 101).to_int () <= 0
1054 || callee
->count
.ipa ().initialized_p ());
1055 gcc_checking_assert (growth
<= callee_info
->size
);
1059 fprintf (dump_file
, " Badness calculation for %s -> %s\n",
1060 edge
->caller
->dump_name (),
1061 edge
->callee
->dump_name ());
1062 fprintf (dump_file
, " size growth %i, time %f unspec %f ",
1064 edge_time
.to_double (),
1065 unspec_edge_time
.to_double ());
1066 ipa_dump_hints (dump_file
, hints
);
1067 if (big_speedup_p (edge
))
1068 fprintf (dump_file
, " big_speedup");
1069 fprintf (dump_file
, "\n");
1072 /* Always prefer inlining saving code size. */
1075 badness
= (sreal
) (-SREAL_MIN_SIG
+ growth
) << (SREAL_MAX_EXP
/ 256);
1077 fprintf (dump_file
, " %f: Growth %d <= 0\n", badness
.to_double (),
1080 /* Inlining into EXTERNAL functions is not going to change anything unless
1081 they are themselves inlined. */
1082 else if (DECL_EXTERNAL (caller
->decl
))
1085 fprintf (dump_file
, " max: function is external\n");
1086 return sreal::max ();
1088 /* When profile is available. Compute badness as:
1090 time_saved * caller_count
1091 goodness = -------------------------------------------------
1092 growth_of_caller * overall_growth * combined_size
1094 badness = - goodness
1096 Again use negative value to make calls with profile appear hotter
1099 else if (opt_for_fn (caller
->decl
, flag_guess_branch_prob
)
1100 || caller
->count
.ipa ().nonzero_p ())
1102 sreal numerator
, denominator
;
1104 sreal inlined_time
= compute_inlined_call_time (edge
, edge_time
);
1106 numerator
= (compute_uninlined_call_time (edge
, unspec_edge_time
)
1109 numerator
= ((sreal
) 1 >> 8);
1110 if (caller
->count
.ipa ().nonzero_p ())
1111 numerator
*= caller
->count
.ipa ().to_gcov_type ();
1112 else if (caller
->count
.ipa ().initialized_p ())
1113 numerator
= numerator
>> 11;
1114 denominator
= growth
;
1116 overall_growth
= callee_info
->growth
;
1118 /* Look for inliner wrappers of the form:
1124 noninline_callee ();
1126 Withhout panilizing this case, we usually inline noninline_callee
1127 into the inline_caller because overall_growth is small preventing
1128 further inlining of inline_caller.
1130 Penalize only callgraph edges to functions with small overall
1133 if (growth
> overall_growth
1134 /* ... and having only one caller which is not inlined ... */
1135 && callee_info
->single_caller
1136 && !edge
->caller
->global
.inlined_to
1137 /* ... and edges executed only conditionally ... */
1138 && edge
->sreal_frequency () < 1
1139 /* ... consider case where callee is not inline but caller is ... */
1140 && ((!DECL_DECLARED_INLINE_P (edge
->callee
->decl
)
1141 && DECL_DECLARED_INLINE_P (caller
->decl
))
1142 /* ... or when early optimizers decided to split and edge
1143 frequency still indicates splitting is a win ... */
1144 || (callee
->split_part
&& !caller
->split_part
1145 && edge
->sreal_frequency () * 100
1147 (PARAM_PARTIAL_INLINING_ENTRY_PROBABILITY
)
1148 /* ... and do not overwrite user specified hints. */
1149 && (!DECL_DECLARED_INLINE_P (edge
->callee
->decl
)
1150 || DECL_DECLARED_INLINE_P (caller
->decl
)))))
1152 ipa_fn_summary
*caller_info
= ipa_fn_summaries
->get (caller
);
1153 int caller_growth
= caller_info
->growth
;
1155 /* Only apply the penalty when caller looks like inline candidate,
1156 and it is not called once and. */
1157 if (!caller_info
->single_caller
&& overall_growth
< caller_growth
1158 && caller_info
->inlinable
1159 && caller_info
->size
1160 < (DECL_DECLARED_INLINE_P (caller
->decl
)
1161 ? MAX_INLINE_INSNS_SINGLE
: MAX_INLINE_INSNS_AUTO
))
1165 " Wrapper penalty. Increasing growth %i to %i\n",
1166 overall_growth
, caller_growth
);
1167 overall_growth
= caller_growth
;
1170 if (overall_growth
> 0)
1172 /* Strongly preffer functions with few callers that can be inlined
1173 fully. The square root here leads to smaller binaries at average.
1174 Watch however for extreme cases and return to linear function
1175 when growth is large. */
1176 if (overall_growth
< 256)
1177 overall_growth
*= overall_growth
;
1179 overall_growth
+= 256 * 256 - 256;
1180 denominator
*= overall_growth
;
1182 denominator
*= ipa_fn_summaries
->get (caller
)->self_size
+ growth
;
1184 badness
= - numerator
/ denominator
;
1189 " %f: guessed profile. frequency %f, count %" PRId64
1190 " caller count %" PRId64
1191 " time w/o inlining %f, time with inlining %f"
1192 " overall growth %i (current) %i (original)"
1193 " %i (compensated)\n",
1194 badness
.to_double (),
1195 edge
->sreal_frequency ().to_double (),
1196 edge
->count
.ipa ().initialized_p () ? edge
->count
.ipa ().to_gcov_type () : -1,
1197 caller
->count
.ipa ().initialized_p () ? caller
->count
.ipa ().to_gcov_type () : -1,
1198 compute_uninlined_call_time (edge
,
1199 unspec_edge_time
).to_double (),
1200 inlined_time
.to_double (),
1201 estimate_growth (callee
),
1202 callee_info
->growth
, overall_growth
);
1205 /* When function local profile is not available or it does not give
1206 useful information (ie frequency is zero), base the cost on
1207 loop nest and overall size growth, so we optimize for overall number
1208 of functions fully inlined in program. */
1211 int nest
= MIN (ipa_call_summaries
->get (edge
)->loop_depth
, 8);
1214 /* Decrease badness if call is nested. */
1216 badness
= badness
>> nest
;
1218 badness
= badness
<< nest
;
1220 fprintf (dump_file
, " %f: no profile. nest %i\n",
1221 badness
.to_double (), nest
);
1223 gcc_checking_assert (badness
!= 0);
1225 if (edge
->recursive_p ())
1226 badness
= badness
.shift (badness
> 0 ? 4 : -4);
1227 if ((hints
& (INLINE_HINT_indirect_call
1228 | INLINE_HINT_loop_iterations
1229 | INLINE_HINT_array_index
1230 | INLINE_HINT_loop_stride
))
1231 || callee_info
->growth
<= 0)
1232 badness
= badness
.shift (badness
> 0 ? -2 : 2);
1233 if (hints
& (INLINE_HINT_same_scc
))
1234 badness
= badness
.shift (badness
> 0 ? 3 : -3);
1235 else if (hints
& (INLINE_HINT_in_scc
))
1236 badness
= badness
.shift (badness
> 0 ? 2 : -2);
1237 else if (hints
& (INLINE_HINT_cross_module
))
1238 badness
= badness
.shift (badness
> 0 ? 1 : -1);
1239 if (DECL_DISREGARD_INLINE_LIMITS (callee
->decl
))
1240 badness
= badness
.shift (badness
> 0 ? -4 : 4);
1241 else if ((hints
& INLINE_HINT_declared_inline
))
1242 badness
= badness
.shift (badness
> 0 ? -3 : 3);
1244 fprintf (dump_file
, " Adjusted by hints %f\n", badness
.to_double ());
1248 /* Recompute badness of EDGE and update its key in HEAP if needed. */
1250 update_edge_key (edge_heap_t
*heap
, struct cgraph_edge
*edge
)
1252 sreal badness
= edge_badness (edge
, false);
1255 edge_heap_node_t
*n
= (edge_heap_node_t
*) edge
->aux
;
1256 gcc_checking_assert (n
->get_data () == edge
);
1258 /* fibonacci_heap::replace_key does busy updating of the
1259 heap that is unnecesarily expensive.
1260 We do lazy increases: after extracting minimum if the key
1261 turns out to be out of date, it is re-inserted into heap
1262 with correct value. */
1263 if (badness
< n
->get_key ())
1265 if (dump_file
&& (dump_flags
& TDF_DETAILS
))
1268 " decreasing badness %s -> %s, %f to %f\n",
1269 edge
->caller
->dump_name (),
1270 edge
->callee
->dump_name (),
1271 n
->get_key ().to_double (),
1272 badness
.to_double ());
1274 heap
->decrease_key (n
, badness
);
1279 if (dump_file
&& (dump_flags
& TDF_DETAILS
))
1282 " enqueuing call %s -> %s, badness %f\n",
1283 edge
->caller
->dump_name (),
1284 edge
->callee
->dump_name (),
1285 badness
.to_double ());
1287 edge
->aux
= heap
->insert (badness
, edge
);
1292 /* NODE was inlined.
1293 All caller edges needs to be resetted because
1294 size estimates change. Similarly callees needs reset
1295 because better context may be known. */
1298 reset_edge_caches (struct cgraph_node
*node
)
1300 struct cgraph_edge
*edge
;
1301 struct cgraph_edge
*e
= node
->callees
;
1302 struct cgraph_node
*where
= node
;
1303 struct ipa_ref
*ref
;
1305 if (where
->global
.inlined_to
)
1306 where
= where
->global
.inlined_to
;
1308 if (edge_growth_cache
!= NULL
)
1309 for (edge
= where
->callers
; edge
; edge
= edge
->next_caller
)
1310 if (edge
->inline_failed
)
1311 edge_growth_cache
->remove (edge
);
1313 FOR_EACH_ALIAS (where
, ref
)
1314 reset_edge_caches (dyn_cast
<cgraph_node
*> (ref
->referring
));
1320 if (!e
->inline_failed
&& e
->callee
->callees
)
1321 e
= e
->callee
->callees
;
1324 if (edge_growth_cache
!= NULL
&& e
->inline_failed
)
1325 edge_growth_cache
->remove (e
);
1332 if (e
->caller
== node
)
1334 e
= e
->caller
->callers
;
1336 while (!e
->next_callee
);
1342 /* Recompute HEAP nodes for each of caller of NODE.
1343 UPDATED_NODES track nodes we already visited, to avoid redundant work.
1344 When CHECK_INLINABLITY_FOR is set, re-check for specified edge that
1345 it is inlinable. Otherwise check all edges. */
1348 update_caller_keys (edge_heap_t
*heap
, struct cgraph_node
*node
,
1349 bitmap updated_nodes
,
1350 struct cgraph_edge
*check_inlinablity_for
)
1352 struct cgraph_edge
*edge
;
1353 struct ipa_ref
*ref
;
1355 if ((!node
->alias
&& !ipa_fn_summaries
->get (node
)->inlinable
)
1356 || node
->global
.inlined_to
)
1358 if (!bitmap_set_bit (updated_nodes
, node
->get_uid ()))
1361 FOR_EACH_ALIAS (node
, ref
)
1363 struct cgraph_node
*alias
= dyn_cast
<cgraph_node
*> (ref
->referring
);
1364 update_caller_keys (heap
, alias
, updated_nodes
, check_inlinablity_for
);
1367 for (edge
= node
->callers
; edge
; edge
= edge
->next_caller
)
1368 if (edge
->inline_failed
)
1370 if (!check_inlinablity_for
1371 || check_inlinablity_for
== edge
)
1373 if (can_inline_edge_p (edge
, false)
1374 && want_inline_small_function_p (edge
, false)
1375 && can_inline_edge_by_limits_p (edge
, false))
1376 update_edge_key (heap
, edge
);
1379 report_inline_failed_reason (edge
);
1380 heap
->delete_node ((edge_heap_node_t
*) edge
->aux
);
1385 update_edge_key (heap
, edge
);
1389 /* Recompute HEAP nodes for each uninlined call in NODE.
1390 This is used when we know that edge badnesses are going only to increase
1391 (we introduced new call site) and thus all we need is to insert newly
1392 created edges into heap. */
1395 update_callee_keys (edge_heap_t
*heap
, struct cgraph_node
*node
,
1396 bitmap updated_nodes
)
1398 struct cgraph_edge
*e
= node
->callees
;
1403 if (!e
->inline_failed
&& e
->callee
->callees
)
1404 e
= e
->callee
->callees
;
1407 enum availability avail
;
1408 struct cgraph_node
*callee
;
1409 /* We do not reset callee growth cache here. Since we added a new call,
1410 growth chould have just increased and consequentely badness metric
1411 don't need updating. */
1412 if (e
->inline_failed
1413 && (callee
= e
->callee
->ultimate_alias_target (&avail
, e
->caller
))
1414 && ipa_fn_summaries
->get (callee
) != NULL
1415 && ipa_fn_summaries
->get (callee
)->inlinable
1416 && avail
>= AVAIL_AVAILABLE
1417 && !bitmap_bit_p (updated_nodes
, callee
->get_uid ()))
1419 if (can_inline_edge_p (e
, false)
1420 && want_inline_small_function_p (e
, false)
1421 && can_inline_edge_by_limits_p (e
, false))
1422 update_edge_key (heap
, e
);
1425 report_inline_failed_reason (e
);
1426 heap
->delete_node ((edge_heap_node_t
*) e
->aux
);
1436 if (e
->caller
== node
)
1438 e
= e
->caller
->callers
;
1440 while (!e
->next_callee
);
1446 /* Enqueue all recursive calls from NODE into priority queue depending on
1447 how likely we want to recursively inline the call. */
1450 lookup_recursive_calls (struct cgraph_node
*node
, struct cgraph_node
*where
,
1453 struct cgraph_edge
*e
;
1454 enum availability avail
;
1456 for (e
= where
->callees
; e
; e
= e
->next_callee
)
1457 if (e
->callee
== node
1458 || (e
->callee
->ultimate_alias_target (&avail
, e
->caller
) == node
1459 && avail
> AVAIL_INTERPOSABLE
))
1460 heap
->insert (-e
->sreal_frequency (), e
);
1461 for (e
= where
->callees
; e
; e
= e
->next_callee
)
1462 if (!e
->inline_failed
)
1463 lookup_recursive_calls (node
, e
->callee
, heap
);
1466 /* Decide on recursive inlining: in the case function has recursive calls,
1467 inline until body size reaches given argument. If any new indirect edges
1468 are discovered in the process, add them to *NEW_EDGES, unless NEW_EDGES
1472 recursive_inlining (struct cgraph_edge
*edge
,
1473 vec
<cgraph_edge
*> *new_edges
)
1475 int limit
= PARAM_VALUE (PARAM_MAX_INLINE_INSNS_RECURSIVE_AUTO
);
1476 edge_heap_t
heap (sreal::min ());
1477 struct cgraph_node
*node
;
1478 struct cgraph_edge
*e
;
1479 struct cgraph_node
*master_clone
= NULL
, *next
;
1483 node
= edge
->caller
;
1484 if (node
->global
.inlined_to
)
1485 node
= node
->global
.inlined_to
;
1487 if (DECL_DECLARED_INLINE_P (node
->decl
))
1488 limit
= PARAM_VALUE (PARAM_MAX_INLINE_INSNS_RECURSIVE
);
1490 /* Make sure that function is small enough to be considered for inlining. */
1491 if (estimate_size_after_inlining (node
, edge
) >= limit
)
1493 lookup_recursive_calls (node
, node
, &heap
);
1499 " Performing recursive inlining on %s\n",
1502 /* Do the inlining and update list of recursive call during process. */
1503 while (!heap
.empty ())
1505 struct cgraph_edge
*curr
= heap
.extract_min ();
1506 struct cgraph_node
*cnode
, *dest
= curr
->callee
;
1508 if (!can_inline_edge_p (curr
, true)
1509 || can_inline_edge_by_limits_p (curr
, true))
1512 /* MASTER_CLONE is produced in the case we already started modified
1513 the function. Be sure to redirect edge to the original body before
1514 estimating growths otherwise we will be seeing growths after inlining
1515 the already modified body. */
1518 curr
->redirect_callee (master_clone
);
1519 if (edge_growth_cache
!= NULL
)
1520 edge_growth_cache
->remove (curr
);
1523 if (estimate_size_after_inlining (node
, curr
) > limit
)
1525 curr
->redirect_callee (dest
);
1526 if (edge_growth_cache
!= NULL
)
1527 edge_growth_cache
->remove (curr
);
1532 for (cnode
= curr
->caller
;
1533 cnode
->global
.inlined_to
; cnode
= cnode
->callers
->caller
)
1535 == curr
->callee
->ultimate_alias_target ()->decl
)
1538 if (!want_inline_self_recursive_call_p (curr
, node
, false, depth
))
1540 curr
->redirect_callee (dest
);
1541 if (edge_growth_cache
!= NULL
)
1542 edge_growth_cache
->remove (curr
);
1549 " Inlining call of depth %i", depth
);
1550 if (node
->count
.nonzero_p ())
1552 fprintf (dump_file
, " called approx. %.2f times per call",
1553 (double)curr
->count
.to_gcov_type ()
1554 / node
->count
.to_gcov_type ());
1556 fprintf (dump_file
, "\n");
1560 /* We need original clone to copy around. */
1561 master_clone
= node
->create_clone (node
->decl
, node
->count
,
1562 false, vNULL
, true, NULL
, NULL
);
1563 for (e
= master_clone
->callees
; e
; e
= e
->next_callee
)
1564 if (!e
->inline_failed
)
1565 clone_inlined_nodes (e
, true, false, NULL
);
1566 curr
->redirect_callee (master_clone
);
1567 if (edge_growth_cache
!= NULL
)
1568 edge_growth_cache
->remove (curr
);
1571 inline_call (curr
, false, new_edges
, &overall_size
, true);
1572 lookup_recursive_calls (node
, curr
->callee
, &heap
);
1576 if (!heap
.empty () && dump_file
)
1577 fprintf (dump_file
, " Recursive inlining growth limit met.\n");
1582 if (dump_enabled_p ())
1583 dump_printf_loc (MSG_NOTE
, edge
->call_stmt
,
1584 "\n Inlined %i times, "
1585 "body grown from size %i to %i, time %f to %f\n", n
,
1586 ipa_fn_summaries
->get (master_clone
)->size
,
1587 ipa_fn_summaries
->get (node
)->size
,
1588 ipa_fn_summaries
->get (master_clone
)->time
.to_double (),
1589 ipa_fn_summaries
->get (node
)->time
.to_double ());
1591 /* Remove master clone we used for inlining. We rely that clones inlined
1592 into master clone gets queued just before master clone so we don't
1594 for (node
= symtab
->first_function (); node
!= master_clone
;
1597 next
= symtab
->next_function (node
);
1598 if (node
->global
.inlined_to
== master_clone
)
1601 master_clone
->remove ();
1606 /* Given whole compilation unit estimate of INSNS, compute how large we can
1607 allow the unit to grow. */
1610 compute_max_insns (int insns
)
1612 int max_insns
= insns
;
1613 if (max_insns
< PARAM_VALUE (PARAM_LARGE_UNIT_INSNS
))
1614 max_insns
= PARAM_VALUE (PARAM_LARGE_UNIT_INSNS
);
1616 return ((int64_t) max_insns
1617 * (100 + PARAM_VALUE (PARAM_INLINE_UNIT_GROWTH
)) / 100);
1621 /* Compute badness of all edges in NEW_EDGES and add them to the HEAP. */
1624 add_new_edges_to_heap (edge_heap_t
*heap
, vec
<cgraph_edge
*> new_edges
)
1626 while (new_edges
.length () > 0)
1628 struct cgraph_edge
*edge
= new_edges
.pop ();
1630 gcc_assert (!edge
->aux
);
1631 if (edge
->inline_failed
1632 && can_inline_edge_p (edge
, true)
1633 && want_inline_small_function_p (edge
, true)
1634 && can_inline_edge_by_limits_p (edge
, true))
1635 edge
->aux
= heap
->insert (edge_badness (edge
, false), edge
);
1639 /* Remove EDGE from the fibheap. */
1642 heap_edge_removal_hook (struct cgraph_edge
*e
, void *data
)
1646 ((edge_heap_t
*)data
)->delete_node ((edge_heap_node_t
*)e
->aux
);
1651 /* Return true if speculation of edge E seems useful.
1652 If ANTICIPATE_INLINING is true, be conservative and hope that E
1656 speculation_useful_p (struct cgraph_edge
*e
, bool anticipate_inlining
)
1658 enum availability avail
;
1659 struct cgraph_node
*target
= e
->callee
->ultimate_alias_target (&avail
,
1661 struct cgraph_edge
*direct
, *indirect
;
1662 struct ipa_ref
*ref
;
1664 gcc_assert (e
->speculative
&& !e
->indirect_unknown_callee
);
1666 if (!e
->maybe_hot_p ())
1669 /* See if IP optimizations found something potentially useful about the
1670 function. For now we look only for CONST/PURE flags. Almost everything
1671 else we propagate is useless. */
1672 if (avail
>= AVAIL_AVAILABLE
)
1674 int ecf_flags
= flags_from_decl_or_type (target
->decl
);
1675 if (ecf_flags
& ECF_CONST
)
1677 e
->speculative_call_info (direct
, indirect
, ref
);
1678 if (!(indirect
->indirect_info
->ecf_flags
& ECF_CONST
))
1681 else if (ecf_flags
& ECF_PURE
)
1683 e
->speculative_call_info (direct
, indirect
, ref
);
1684 if (!(indirect
->indirect_info
->ecf_flags
& ECF_PURE
))
1688 /* If we did not managed to inline the function nor redirect
1689 to an ipa-cp clone (that are seen by having local flag set),
1690 it is probably pointless to inline it unless hardware is missing
1691 indirect call predictor. */
1692 if (!anticipate_inlining
&& e
->inline_failed
&& !target
->local
.local
)
1694 /* For overwritable targets there is not much to do. */
1695 if (e
->inline_failed
1696 && (!can_inline_edge_p (e
, false)
1697 || !can_inline_edge_by_limits_p (e
, false, true)))
1699 /* OK, speculation seems interesting. */
1703 /* We know that EDGE is not going to be inlined.
1704 See if we can remove speculation. */
1707 resolve_noninline_speculation (edge_heap_t
*edge_heap
, struct cgraph_edge
*edge
)
1709 if (edge
->speculative
&& !speculation_useful_p (edge
, false))
1711 struct cgraph_node
*node
= edge
->caller
;
1712 struct cgraph_node
*where
= node
->global
.inlined_to
1713 ? node
->global
.inlined_to
: node
;
1714 auto_bitmap updated_nodes
;
1716 if (edge
->count
.ipa ().initialized_p ())
1717 spec_rem
+= edge
->count
.ipa ();
1718 edge
->resolve_speculation ();
1719 reset_edge_caches (where
);
1720 ipa_update_overall_fn_summary (where
);
1721 update_caller_keys (edge_heap
, where
,
1722 updated_nodes
, NULL
);
1723 update_callee_keys (edge_heap
, where
,
1728 /* Return true if NODE should be accounted for overall size estimate.
1729 Skip all nodes optimized for size so we can measure the growth of hot
1730 part of program no matter of the padding. */
1733 inline_account_function_p (struct cgraph_node
*node
)
1735 return (!DECL_EXTERNAL (node
->decl
)
1736 && !opt_for_fn (node
->decl
, optimize_size
)
1737 && node
->frequency
!= NODE_FREQUENCY_UNLIKELY_EXECUTED
);
1740 /* Count number of callers of NODE and store it into DATA (that
1741 points to int. Worker for cgraph_for_node_and_aliases. */
1744 sum_callers (struct cgraph_node
*node
, void *data
)
1746 struct cgraph_edge
*e
;
1747 int *num_calls
= (int *)data
;
1749 for (e
= node
->callers
; e
; e
= e
->next_caller
)
1754 /* We use greedy algorithm for inlining of small functions:
1755 All inline candidates are put into prioritized heap ordered in
1758 The inlining of small functions is bounded by unit growth parameters. */
1761 inline_small_functions (void)
1763 struct cgraph_node
*node
;
1764 struct cgraph_edge
*edge
;
1765 edge_heap_t
edge_heap (sreal::min ());
1766 auto_bitmap updated_nodes
;
1767 int min_size
, max_size
;
1768 auto_vec
<cgraph_edge
*> new_indirect_edges
;
1769 int initial_size
= 0;
1770 struct cgraph_node
**order
= XCNEWVEC (cgraph_node
*, symtab
->cgraph_count
);
1771 struct cgraph_edge_hook_list
*edge_removal_hook_holder
;
1772 new_indirect_edges
.create (8);
1774 edge_removal_hook_holder
1775 = symtab
->add_edge_removal_hook (&heap_edge_removal_hook
, &edge_heap
);
1777 /* Compute overall unit size and other global parameters used by badness
1780 max_count
= profile_count::uninitialized ();
1781 ipa_reduced_postorder (order
, true, NULL
);
1784 FOR_EACH_DEFINED_FUNCTION (node
)
1785 if (!node
->global
.inlined_to
)
1787 if (!node
->alias
&& node
->analyzed
1788 && (node
->has_gimple_body_p () || node
->thunk
.thunk_p
)
1789 && opt_for_fn (node
->decl
, optimize
))
1791 struct ipa_fn_summary
*info
= ipa_fn_summaries
->get (node
);
1792 struct ipa_dfs_info
*dfs
= (struct ipa_dfs_info
*) node
->aux
;
1794 /* Do not account external functions, they will be optimized out
1795 if not inlined. Also only count the non-cold portion of program. */
1796 if (inline_account_function_p (node
))
1797 initial_size
+= info
->size
;
1798 info
->growth
= estimate_growth (node
);
1801 node
->call_for_symbol_and_aliases (sum_callers
, &num_calls
,
1804 info
->single_caller
= true;
1805 if (dfs
&& dfs
->next_cycle
)
1807 struct cgraph_node
*n2
;
1808 int id
= dfs
->scc_no
+ 1;
1810 n2
= ((struct ipa_dfs_info
*) n2
->aux
)->next_cycle
)
1811 if (opt_for_fn (n2
->decl
, optimize
))
1813 ipa_fn_summary
*info2
= ipa_fn_summaries
->get (n2
);
1821 for (edge
= node
->callers
; edge
; edge
= edge
->next_caller
)
1822 max_count
= max_count
.max (edge
->count
.ipa ());
1824 ipa_free_postorder_info ();
1826 = new call_summary
<edge_growth_cache_entry
*> (symtab
, false);
1830 "\nDeciding on inlining of small functions. Starting with size %i.\n",
1833 overall_size
= initial_size
;
1834 max_size
= compute_max_insns (overall_size
);
1835 min_size
= overall_size
;
1837 /* Populate the heap with all edges we might inline. */
1839 FOR_EACH_DEFINED_FUNCTION (node
)
1841 bool update
= false;
1842 struct cgraph_edge
*next
= NULL
;
1843 bool has_speculative
= false;
1845 if (!opt_for_fn (node
->decl
, optimize
))
1849 fprintf (dump_file
, "Enqueueing calls in %s.\n", node
->dump_name ());
1851 for (edge
= node
->callees
; edge
; edge
= next
)
1853 next
= edge
->next_callee
;
1854 if (edge
->inline_failed
1856 && can_inline_edge_p (edge
, true)
1857 && want_inline_small_function_p (edge
, true)
1858 && can_inline_edge_by_limits_p (edge
, true)
1859 && edge
->inline_failed
)
1861 gcc_assert (!edge
->aux
);
1862 update_edge_key (&edge_heap
, edge
);
1864 if (edge
->speculative
)
1865 has_speculative
= true;
1867 if (has_speculative
)
1868 for (edge
= node
->callees
; edge
; edge
= next
)
1869 if (edge
->speculative
&& !speculation_useful_p (edge
,
1872 edge
->resolve_speculation ();
1877 struct cgraph_node
*where
= node
->global
.inlined_to
1878 ? node
->global
.inlined_to
: node
;
1879 ipa_update_overall_fn_summary (where
);
1880 reset_edge_caches (where
);
1881 update_caller_keys (&edge_heap
, where
,
1882 updated_nodes
, NULL
);
1883 update_callee_keys (&edge_heap
, where
,
1885 bitmap_clear (updated_nodes
);
1889 gcc_assert (in_lto_p
1891 || (profile_info
&& flag_branch_probabilities
));
1893 while (!edge_heap
.empty ())
1895 int old_size
= overall_size
;
1896 struct cgraph_node
*where
, *callee
;
1897 sreal badness
= edge_heap
.min_key ();
1898 sreal current_badness
;
1901 edge
= edge_heap
.extract_min ();
1902 gcc_assert (edge
->aux
);
1904 if (!edge
->inline_failed
|| !edge
->callee
->analyzed
)
1908 /* Be sure that caches are maintained consistent.
1909 This check is affected by scaling roundoff errors when compiling for
1910 IPA this we skip it in that case. */
1911 if (!edge
->callee
->count
.ipa_p ()
1912 && (!max_count
.initialized_p () || !max_count
.nonzero_p ()))
1914 sreal cached_badness
= edge_badness (edge
, false);
1916 int old_size_est
= estimate_edge_size (edge
);
1917 sreal old_time_est
= estimate_edge_time (edge
);
1918 int old_hints_est
= estimate_edge_hints (edge
);
1920 if (edge_growth_cache
!= NULL
)
1921 edge_growth_cache
->remove (edge
);
1922 gcc_assert (old_size_est
== estimate_edge_size (edge
));
1923 gcc_assert (old_time_est
== estimate_edge_time (edge
));
1926 gcc_assert (old_hints_est == estimate_edge_hints (edge));
1928 fails with profile feedback because some hints depends on
1929 maybe_hot_edge_p predicate and because callee gets inlined to other
1930 calls, the edge may become cold.
1931 This ought to be fixed by computing relative probabilities
1932 for given invocation but that will be better done once whole
1933 code is converted to sreals. Disable for now and revert to "wrong"
1934 value so enable/disable checking paths agree. */
1935 edge_growth_cache
->get (edge
)->hints
= old_hints_est
+ 1;
1937 /* When updating the edge costs, we only decrease badness in the keys.
1938 Increases of badness are handled lazilly; when we see key with out
1939 of date value on it, we re-insert it now. */
1940 current_badness
= edge_badness (edge
, false);
1941 gcc_assert (cached_badness
== current_badness
);
1942 gcc_assert (current_badness
>= badness
);
1945 current_badness
= edge_badness (edge
, false);
1947 current_badness
= edge_badness (edge
, false);
1949 if (current_badness
!= badness
)
1951 if (edge_heap
.min () && current_badness
> edge_heap
.min_key ())
1953 edge
->aux
= edge_heap
.insert (current_badness
, edge
);
1957 badness
= current_badness
;
1960 if (!can_inline_edge_p (edge
, true)
1961 || !can_inline_edge_by_limits_p (edge
, true))
1963 resolve_noninline_speculation (&edge_heap
, edge
);
1967 callee
= edge
->callee
->ultimate_alias_target ();
1968 growth
= estimate_edge_growth (edge
);
1972 "\nConsidering %s with %i size\n",
1973 callee
->dump_name (),
1974 ipa_fn_summaries
->get (callee
)->size
);
1976 " to be inlined into %s in %s:%i\n"
1977 " Estimated badness is %f, frequency %.2f.\n",
1978 edge
->caller
->dump_name (),
1980 && (LOCATION_LOCUS (gimple_location ((const gimple
*)
1982 > BUILTINS_LOCATION
)
1983 ? gimple_filename ((const gimple
*) edge
->call_stmt
)
1986 ? gimple_lineno ((const gimple
*) edge
->call_stmt
)
1988 badness
.to_double (),
1989 edge
->sreal_frequency ().to_double ());
1990 if (edge
->count
.ipa ().initialized_p ())
1992 fprintf (dump_file
, " Called ");
1993 edge
->count
.ipa ().dump (dump_file
);
1994 fprintf (dump_file
, " times\n");
1996 if (dump_flags
& TDF_DETAILS
)
1997 edge_badness (edge
, true);
2000 if (overall_size
+ growth
> max_size
2001 && !DECL_DISREGARD_INLINE_LIMITS (callee
->decl
))
2003 edge
->inline_failed
= CIF_INLINE_UNIT_GROWTH_LIMIT
;
2004 report_inline_failed_reason (edge
);
2005 resolve_noninline_speculation (&edge_heap
, edge
);
2009 if (!want_inline_small_function_p (edge
, true))
2011 resolve_noninline_speculation (&edge_heap
, edge
);
2015 /* Heuristics for inlining small functions work poorly for
2016 recursive calls where we do effects similar to loop unrolling.
2017 When inlining such edge seems profitable, leave decision on
2018 specific inliner. */
2019 if (edge
->recursive_p ())
2021 where
= edge
->caller
;
2022 if (where
->global
.inlined_to
)
2023 where
= where
->global
.inlined_to
;
2024 if (!recursive_inlining (edge
,
2025 opt_for_fn (edge
->caller
->decl
,
2026 flag_indirect_inlining
)
2027 ? &new_indirect_edges
: NULL
))
2029 edge
->inline_failed
= CIF_RECURSIVE_INLINING
;
2030 resolve_noninline_speculation (&edge_heap
, edge
);
2033 reset_edge_caches (where
);
2034 /* Recursive inliner inlines all recursive calls of the function
2035 at once. Consequently we need to update all callee keys. */
2036 if (opt_for_fn (edge
->caller
->decl
, flag_indirect_inlining
))
2037 add_new_edges_to_heap (&edge_heap
, new_indirect_edges
);
2038 update_callee_keys (&edge_heap
, where
, updated_nodes
);
2039 bitmap_clear (updated_nodes
);
2043 struct cgraph_node
*outer_node
= NULL
;
2046 /* Consider the case where self recursive function A is inlined
2047 into B. This is desired optimization in some cases, since it
2048 leads to effect similar of loop peeling and we might completely
2049 optimize out the recursive call. However we must be extra
2052 where
= edge
->caller
;
2053 while (where
->global
.inlined_to
)
2055 if (where
->decl
== callee
->decl
)
2056 outer_node
= where
, depth
++;
2057 where
= where
->callers
->caller
;
2060 && !want_inline_self_recursive_call_p (edge
, outer_node
,
2064 = (DECL_DISREGARD_INLINE_LIMITS (edge
->callee
->decl
)
2065 ? CIF_RECURSIVE_INLINING
: CIF_UNSPECIFIED
);
2066 resolve_noninline_speculation (&edge_heap
, edge
);
2069 else if (depth
&& dump_file
)
2070 fprintf (dump_file
, " Peeling recursion with depth %i\n", depth
);
2072 gcc_checking_assert (!callee
->global
.inlined_to
);
2073 inline_call (edge
, true, &new_indirect_edges
, &overall_size
, true);
2074 add_new_edges_to_heap (&edge_heap
, new_indirect_edges
);
2076 reset_edge_caches (edge
->callee
);
2078 update_callee_keys (&edge_heap
, where
, updated_nodes
);
2080 where
= edge
->caller
;
2081 if (where
->global
.inlined_to
)
2082 where
= where
->global
.inlined_to
;
2084 /* Our profitability metric can depend on local properties
2085 such as number of inlinable calls and size of the function body.
2086 After inlining these properties might change for the function we
2087 inlined into (since it's body size changed) and for the functions
2088 called by function we inlined (since number of it inlinable callers
2090 update_caller_keys (&edge_heap
, where
, updated_nodes
, NULL
);
2091 /* Offline copy count has possibly changed, recompute if profile is
2093 struct cgraph_node
*n
= cgraph_node::get (edge
->callee
->decl
);
2094 if (n
!= edge
->callee
&& n
->analyzed
&& n
->count
.ipa ().initialized_p ())
2095 update_callee_keys (&edge_heap
, n
, updated_nodes
);
2096 bitmap_clear (updated_nodes
);
2098 if (dump_enabled_p ())
2100 ipa_fn_summary
*s
= ipa_fn_summaries
->get (edge
->caller
);
2102 /* dump_printf can't handle %+i. */
2103 char buf_net_change
[100];
2104 snprintf (buf_net_change
, sizeof buf_net_change
, "%+i",
2105 overall_size
- old_size
);
2107 dump_printf_loc (MSG_OPTIMIZED_LOCATIONS
, edge
->call_stmt
,
2108 " Inlined %C into %C which now has time %f and "
2109 "size %i, net change of %s.\n",
2110 edge
->callee
, edge
->caller
,
2111 s
->time
.to_double (), s
->size
, buf_net_change
);
2113 if (min_size
> overall_size
)
2115 min_size
= overall_size
;
2116 max_size
= compute_max_insns (min_size
);
2119 fprintf (dump_file
, "New minimal size reached: %i\n", min_size
);
2123 free_growth_caches ();
2124 if (dump_enabled_p ())
2125 dump_printf (MSG_NOTE
,
2126 "Unit growth for small function inlining: %i->%i (%i%%)\n",
2127 initial_size
, overall_size
,
2128 initial_size
? overall_size
* 100 / (initial_size
) - 100: 0);
2129 symtab
->remove_edge_removal_hook (edge_removal_hook_holder
);
2132 /* Flatten NODE. Performed both during early inlining and
2133 at IPA inlining time. */
2136 flatten_function (struct cgraph_node
*node
, bool early
)
2138 struct cgraph_edge
*e
;
2140 /* We shouldn't be called recursively when we are being processed. */
2141 gcc_assert (node
->aux
== NULL
);
2143 node
->aux
= (void *) node
;
2145 for (e
= node
->callees
; e
; e
= e
->next_callee
)
2147 struct cgraph_node
*orig_callee
;
2148 struct cgraph_node
*callee
= e
->callee
->ultimate_alias_target ();
2150 /* We've hit cycle? It is time to give up. */
2153 if (dump_enabled_p ())
2154 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, e
->call_stmt
,
2155 "Not inlining %C into %C to avoid cycle.\n",
2157 if (cgraph_inline_failed_type (e
->inline_failed
) != CIF_FINAL_ERROR
)
2158 e
->inline_failed
= CIF_RECURSIVE_INLINING
;
2162 /* When the edge is already inlined, we just need to recurse into
2163 it in order to fully flatten the leaves. */
2164 if (!e
->inline_failed
)
2166 flatten_function (callee
, early
);
2170 /* Flatten attribute needs to be processed during late inlining. For
2171 extra code quality we however do flattening during early optimization,
2174 ? !can_inline_edge_p (e
, true)
2175 && !can_inline_edge_by_limits_p (e
, true)
2176 : !can_early_inline_edge_p (e
))
2179 if (e
->recursive_p ())
2181 if (dump_enabled_p ())
2182 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, e
->call_stmt
,
2183 "Not inlining: recursive call.\n");
2187 if (gimple_in_ssa_p (DECL_STRUCT_FUNCTION (node
->decl
))
2188 != gimple_in_ssa_p (DECL_STRUCT_FUNCTION (callee
->decl
)))
2190 if (dump_enabled_p ())
2191 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, e
->call_stmt
,
2192 "Not inlining: SSA form does not match.\n");
2196 /* Inline the edge and flatten the inline clone. Avoid
2197 recursing through the original node if the node was cloned. */
2198 if (dump_enabled_p ())
2199 dump_printf_loc (MSG_OPTIMIZED_LOCATIONS
, e
->call_stmt
,
2200 " Inlining %C into %C.\n",
2202 orig_callee
= callee
;
2203 inline_call (e
, true, NULL
, NULL
, false);
2204 if (e
->callee
!= orig_callee
)
2205 orig_callee
->aux
= (void *) node
;
2206 flatten_function (e
->callee
, early
);
2207 if (e
->callee
!= orig_callee
)
2208 orig_callee
->aux
= NULL
;
2212 if (!node
->global
.inlined_to
)
2213 ipa_update_overall_fn_summary (node
);
2216 /* Inline NODE to all callers. Worker for cgraph_for_node_and_aliases.
2217 DATA points to number of calls originally found so we avoid infinite
2221 inline_to_all_callers_1 (struct cgraph_node
*node
, void *data
,
2222 hash_set
<cgraph_node
*> *callers
)
2224 int *num_calls
= (int *)data
;
2225 bool callee_removed
= false;
2227 while (node
->callers
&& !node
->global
.inlined_to
)
2229 struct cgraph_node
*caller
= node
->callers
->caller
;
2231 if (!can_inline_edge_p (node
->callers
, true)
2232 || !can_inline_edge_by_limits_p (node
->callers
, true)
2233 || node
->callers
->recursive_p ())
2236 fprintf (dump_file
, "Uninlinable call found; giving up.\n");
2243 cgraph_node
*ultimate
= node
->ultimate_alias_target ();
2245 "\nInlining %s size %i.\n",
2247 ipa_fn_summaries
->get (ultimate
)->size
);
2249 " Called once from %s %i insns.\n",
2250 node
->callers
->caller
->name (),
2251 ipa_fn_summaries
->get (node
->callers
->caller
)->size
);
2254 /* Remember which callers we inlined to, delaying updating the
2256 callers
->add (node
->callers
->caller
);
2257 inline_call (node
->callers
, true, NULL
, NULL
, false, &callee_removed
);
2260 " Inlined into %s which now has %i size\n",
2262 ipa_fn_summaries
->get (caller
)->size
);
2263 if (!(*num_calls
)--)
2266 fprintf (dump_file
, "New calls found; giving up.\n");
2267 return callee_removed
;
2275 /* Wrapper around inline_to_all_callers_1 doing delayed overall summary
2279 inline_to_all_callers (struct cgraph_node
*node
, void *data
)
2281 hash_set
<cgraph_node
*> callers
;
2282 bool res
= inline_to_all_callers_1 (node
, data
, &callers
);
2283 /* Perform the delayed update of the overall summary of all callers
2284 processed. This avoids quadratic behavior in the cases where
2285 we have a lot of calls to the same function. */
2286 for (hash_set
<cgraph_node
*>::iterator i
= callers
.begin ();
2287 i
!= callers
.end (); ++i
)
2288 ipa_update_overall_fn_summary (*i
);
2292 /* Output overall time estimate. */
2294 dump_overall_stats (void)
2296 sreal sum_weighted
= 0, sum
= 0;
2297 struct cgraph_node
*node
;
2299 FOR_EACH_DEFINED_FUNCTION (node
)
2300 if (!node
->global
.inlined_to
2303 ipa_fn_summary
*s
= ipa_fn_summaries
->get (node
);
2307 if (node
->count
.ipa ().initialized_p ())
2308 sum_weighted
+= s
->time
* node
->count
.ipa ().to_gcov_type ();
2311 fprintf (dump_file
, "Overall time estimate: "
2312 "%f weighted by profile: "
2313 "%f\n", sum
.to_double (), sum_weighted
.to_double ());
2316 /* Output some useful stats about inlining. */
2319 dump_inline_stats (void)
2321 int64_t inlined_cnt
= 0, inlined_indir_cnt
= 0;
2322 int64_t inlined_virt_cnt
= 0, inlined_virt_indir_cnt
= 0;
2323 int64_t noninlined_cnt
= 0, noninlined_indir_cnt
= 0;
2324 int64_t noninlined_virt_cnt
= 0, noninlined_virt_indir_cnt
= 0;
2325 int64_t inlined_speculative
= 0, inlined_speculative_ply
= 0;
2326 int64_t indirect_poly_cnt
= 0, indirect_cnt
= 0;
2327 int64_t reason
[CIF_N_REASONS
][2];
2328 sreal reason_freq
[CIF_N_REASONS
];
2330 struct cgraph_node
*node
;
2332 memset (reason
, 0, sizeof (reason
));
2333 for (i
=0; i
< CIF_N_REASONS
; i
++)
2335 FOR_EACH_DEFINED_FUNCTION (node
)
2337 struct cgraph_edge
*e
;
2338 for (e
= node
->callees
; e
; e
= e
->next_callee
)
2340 if (e
->inline_failed
)
2342 if (e
->count
.ipa ().initialized_p ())
2343 reason
[(int) e
->inline_failed
][0] += e
->count
.ipa ().to_gcov_type ();
2344 reason_freq
[(int) e
->inline_failed
] += e
->sreal_frequency ();
2345 reason
[(int) e
->inline_failed
][1] ++;
2346 if (DECL_VIRTUAL_P (e
->callee
->decl
)
2347 && e
->count
.ipa ().initialized_p ())
2349 if (e
->indirect_inlining_edge
)
2350 noninlined_virt_indir_cnt
+= e
->count
.ipa ().to_gcov_type ();
2352 noninlined_virt_cnt
+= e
->count
.ipa ().to_gcov_type ();
2354 else if (e
->count
.ipa ().initialized_p ())
2356 if (e
->indirect_inlining_edge
)
2357 noninlined_indir_cnt
+= e
->count
.ipa ().to_gcov_type ();
2359 noninlined_cnt
+= e
->count
.ipa ().to_gcov_type ();
2362 else if (e
->count
.ipa ().initialized_p ())
2366 if (DECL_VIRTUAL_P (e
->callee
->decl
))
2367 inlined_speculative_ply
+= e
->count
.ipa ().to_gcov_type ();
2369 inlined_speculative
+= e
->count
.ipa ().to_gcov_type ();
2371 else if (DECL_VIRTUAL_P (e
->callee
->decl
))
2373 if (e
->indirect_inlining_edge
)
2374 inlined_virt_indir_cnt
+= e
->count
.ipa ().to_gcov_type ();
2376 inlined_virt_cnt
+= e
->count
.ipa ().to_gcov_type ();
2380 if (e
->indirect_inlining_edge
)
2381 inlined_indir_cnt
+= e
->count
.ipa ().to_gcov_type ();
2383 inlined_cnt
+= e
->count
.ipa ().to_gcov_type ();
2387 for (e
= node
->indirect_calls
; e
; e
= e
->next_callee
)
2388 if (e
->indirect_info
->polymorphic
2389 & e
->count
.ipa ().initialized_p ())
2390 indirect_poly_cnt
+= e
->count
.ipa ().to_gcov_type ();
2391 else if (e
->count
.ipa ().initialized_p ())
2392 indirect_cnt
+= e
->count
.ipa ().to_gcov_type ();
2394 if (max_count
.initialized_p ())
2397 "Inlined %" PRId64
" + speculative "
2398 "%" PRId64
" + speculative polymorphic "
2399 "%" PRId64
" + previously indirect "
2400 "%" PRId64
" + virtual "
2401 "%" PRId64
" + virtual and previously indirect "
2402 "%" PRId64
"\n" "Not inlined "
2403 "%" PRId64
" + previously indirect "
2404 "%" PRId64
" + virtual "
2405 "%" PRId64
" + virtual and previously indirect "
2406 "%" PRId64
" + stil indirect "
2407 "%" PRId64
" + still indirect polymorphic "
2408 "%" PRId64
"\n", inlined_cnt
,
2409 inlined_speculative
, inlined_speculative_ply
,
2410 inlined_indir_cnt
, inlined_virt_cnt
, inlined_virt_indir_cnt
,
2411 noninlined_cnt
, noninlined_indir_cnt
, noninlined_virt_cnt
,
2412 noninlined_virt_indir_cnt
, indirect_cnt
, indirect_poly_cnt
);
2413 fprintf (dump_file
, "Removed speculations ");
2414 spec_rem
.dump (dump_file
);
2415 fprintf (dump_file
, "\n");
2417 dump_overall_stats ();
2418 fprintf (dump_file
, "\nWhy inlining failed?\n");
2419 for (i
= 0; i
< CIF_N_REASONS
; i
++)
2421 fprintf (dump_file
, "%-50s: %8i calls, %8f freq, %" PRId64
" count\n",
2422 cgraph_inline_failed_string ((cgraph_inline_failed_t
) i
),
2423 (int) reason
[i
][1], reason_freq
[i
].to_double (), reason
[i
][0]);
2426 /* Called when node is removed. */
2429 flatten_remove_node_hook (struct cgraph_node
*node
, void *data
)
2431 if (lookup_attribute ("flatten", DECL_ATTRIBUTES (node
->decl
)) == NULL
)
2434 hash_set
<struct cgraph_node
*> *removed
2435 = (hash_set
<struct cgraph_node
*> *) data
;
2436 removed
->add (node
);
2439 /* Decide on the inlining. We do so in the topological order to avoid
2440 expenses on updating data structures. */
2445 struct cgraph_node
*node
;
2447 struct cgraph_node
**order
;
2450 bool remove_functions
= false;
2452 order
= XCNEWVEC (struct cgraph_node
*, symtab
->cgraph_count
);
2455 ipa_dump_fn_summaries (dump_file
);
2457 nnodes
= ipa_reverse_postorder (order
);
2458 spec_rem
= profile_count::zero ();
2460 FOR_EACH_FUNCTION (node
)
2464 /* Recompute the default reasons for inlining because they may have
2465 changed during merging. */
2468 for (cgraph_edge
*e
= node
->callees
; e
; e
= e
->next_callee
)
2470 gcc_assert (e
->inline_failed
);
2471 initialize_inline_failed (e
);
2473 for (cgraph_edge
*e
= node
->indirect_calls
; e
; e
= e
->next_callee
)
2474 initialize_inline_failed (e
);
2479 fprintf (dump_file
, "\nFlattening functions:\n");
2481 /* First shrink order array, so that it only contains nodes with
2482 flatten attribute. */
2483 for (i
= nnodes
- 1, j
= i
; i
>= 0; i
--)
2486 if (lookup_attribute ("flatten",
2487 DECL_ATTRIBUTES (node
->decl
)) != NULL
)
2488 order
[j
--] = order
[i
];
2491 /* After the above loop, order[j + 1] ... order[nnodes - 1] contain
2492 nodes with flatten attribute. If there is more than one such
2493 node, we need to register a node removal hook, as flatten_function
2494 could remove other nodes with flatten attribute. See PR82801. */
2495 struct cgraph_node_hook_list
*node_removal_hook_holder
= NULL
;
2496 hash_set
<struct cgraph_node
*> *flatten_removed_nodes
= NULL
;
2499 flatten_removed_nodes
= new hash_set
<struct cgraph_node
*>;
2500 node_removal_hook_holder
2501 = symtab
->add_cgraph_removal_hook (&flatten_remove_node_hook
,
2502 flatten_removed_nodes
);
2505 /* In the first pass handle functions to be flattened. Do this with
2506 a priority so none of our later choices will make this impossible. */
2507 for (i
= nnodes
- 1; i
> j
; i
--)
2510 if (flatten_removed_nodes
2511 && flatten_removed_nodes
->contains (node
))
2514 /* Handle nodes to be flattened.
2515 Ideally when processing callees we stop inlining at the
2516 entry of cycles, possibly cloning that entry point and
2517 try to flatten itself turning it into a self-recursive
2520 fprintf (dump_file
, "Flattening %s\n", node
->name ());
2521 flatten_function (node
, false);
2526 symtab
->remove_cgraph_removal_hook (node_removal_hook_holder
);
2527 delete flatten_removed_nodes
;
2532 dump_overall_stats ();
2534 inline_small_functions ();
2536 gcc_assert (symtab
->state
== IPA_SSA
);
2537 symtab
->state
= IPA_SSA_AFTER_INLINING
;
2538 /* Do first after-inlining removal. We want to remove all "stale" extern
2539 inline functions and virtual functions so we really know what is called
2541 symtab
->remove_unreachable_nodes (dump_file
);
2543 /* Inline functions with a property that after inlining into all callers the
2544 code size will shrink because the out-of-line copy is eliminated.
2545 We do this regardless on the callee size as long as function growth limits
2549 "\nDeciding on functions to be inlined into all callers and "
2550 "removing useless speculations:\n");
2552 /* Inlining one function called once has good chance of preventing
2553 inlining other function into the same callee. Ideally we should
2554 work in priority order, but probably inlining hot functions first
2555 is good cut without the extra pain of maintaining the queue.
2557 ??? this is not really fitting the bill perfectly: inlining function
2558 into callee often leads to better optimization of callee due to
2559 increased context for optimization.
2560 For example if main() function calls a function that outputs help
2561 and then function that does the main optmization, we should inline
2562 the second with priority even if both calls are cold by themselves.
2564 We probably want to implement new predicate replacing our use of
2565 maybe_hot_edge interpreted as maybe_hot_edge || callee is known
2567 for (cold
= 0; cold
<= 1; cold
++)
2569 FOR_EACH_DEFINED_FUNCTION (node
)
2571 struct cgraph_edge
*edge
, *next
;
2574 if (!opt_for_fn (node
->decl
, optimize
)
2575 || !opt_for_fn (node
->decl
, flag_inline_functions_called_once
))
2578 for (edge
= node
->callees
; edge
; edge
= next
)
2580 next
= edge
->next_callee
;
2581 if (edge
->speculative
&& !speculation_useful_p (edge
, false))
2583 if (edge
->count
.ipa ().initialized_p ())
2584 spec_rem
+= edge
->count
.ipa ();
2585 edge
->resolve_speculation ();
2587 remove_functions
= true;
2592 struct cgraph_node
*where
= node
->global
.inlined_to
2593 ? node
->global
.inlined_to
: node
;
2594 reset_edge_caches (where
);
2595 ipa_update_overall_fn_summary (where
);
2597 if (want_inline_function_to_all_callers_p (node
, cold
))
2600 node
->call_for_symbol_and_aliases (sum_callers
, &num_calls
,
2602 while (node
->call_for_symbol_and_aliases
2603 (inline_to_all_callers
, &num_calls
, true))
2605 remove_functions
= true;
2610 /* Free ipa-prop structures if they are no longer needed. */
2611 ipa_free_all_structures_after_iinln ();
2613 if (dump_enabled_p ())
2614 dump_printf (MSG_NOTE
,
2615 "\nInlined %i calls, eliminated %i functions\n\n",
2616 ncalls_inlined
, nfunctions_inlined
);
2618 dump_inline_stats ();
2621 ipa_dump_fn_summaries (dump_file
);
2622 return remove_functions
? TODO_remove_functions
: 0;
2625 /* Inline always-inline function calls in NODE. */
2628 inline_always_inline_functions (struct cgraph_node
*node
)
2630 struct cgraph_edge
*e
;
2631 bool inlined
= false;
2633 for (e
= node
->callees
; e
; e
= e
->next_callee
)
2635 struct cgraph_node
*callee
= e
->callee
->ultimate_alias_target ();
2636 if (!DECL_DISREGARD_INLINE_LIMITS (callee
->decl
))
2639 if (e
->recursive_p ())
2641 if (dump_enabled_p ())
2642 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, e
->call_stmt
,
2643 " Not inlining recursive call to %C.\n",
2645 e
->inline_failed
= CIF_RECURSIVE_INLINING
;
2649 if (!can_early_inline_edge_p (e
))
2651 /* Set inlined to true if the callee is marked "always_inline" but
2652 is not inlinable. This will allow flagging an error later in
2653 expand_call_inline in tree-inline.c. */
2654 if (lookup_attribute ("always_inline",
2655 DECL_ATTRIBUTES (callee
->decl
)) != NULL
)
2660 if (dump_enabled_p ())
2661 dump_printf_loc (MSG_OPTIMIZED_LOCATIONS
, e
->call_stmt
,
2662 " Inlining %C into %C (always_inline).\n",
2663 e
->callee
, e
->caller
);
2664 inline_call (e
, true, NULL
, NULL
, false);
2668 ipa_update_overall_fn_summary (node
);
2673 /* Decide on the inlining. We do so in the topological order to avoid
2674 expenses on updating data structures. */
2677 early_inline_small_functions (struct cgraph_node
*node
)
2679 struct cgraph_edge
*e
;
2680 bool inlined
= false;
2682 for (e
= node
->callees
; e
; e
= e
->next_callee
)
2684 struct cgraph_node
*callee
= e
->callee
->ultimate_alias_target ();
2686 /* We can enounter not-yet-analyzed function during
2687 early inlining on callgraphs with strongly
2688 connected components. */
2689 ipa_fn_summary
*s
= ipa_fn_summaries
->get (callee
);
2690 if (s
== NULL
|| !s
->inlinable
|| !e
->inline_failed
)
2693 /* Do not consider functions not declared inline. */
2694 if (!DECL_DECLARED_INLINE_P (callee
->decl
)
2695 && !opt_for_fn (node
->decl
, flag_inline_small_functions
)
2696 && !opt_for_fn (node
->decl
, flag_inline_functions
))
2699 if (dump_enabled_p ())
2700 dump_printf_loc (MSG_NOTE
, e
->call_stmt
,
2701 "Considering inline candidate %C.\n",
2704 if (!can_early_inline_edge_p (e
))
2707 if (e
->recursive_p ())
2709 if (dump_enabled_p ())
2710 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, e
->call_stmt
,
2711 " Not inlining: recursive call.\n");
2715 if (!want_early_inline_function_p (e
))
2718 if (dump_enabled_p ())
2719 dump_printf_loc (MSG_OPTIMIZED_LOCATIONS
, e
->call_stmt
,
2720 " Inlining %C into %C.\n",
2722 inline_call (e
, true, NULL
, NULL
, false);
2727 ipa_update_overall_fn_summary (node
);
2733 early_inliner (function
*fun
)
2735 struct cgraph_node
*node
= cgraph_node::get (current_function_decl
);
2736 struct cgraph_edge
*edge
;
2737 unsigned int todo
= 0;
2739 bool inlined
= false;
2744 /* Do nothing if datastructures for ipa-inliner are already computed. This
2745 happens when some pass decides to construct new function and
2746 cgraph_add_new_function calls lowering passes and early optimization on
2747 it. This may confuse ourself when early inliner decide to inline call to
2748 function clone, because function clones don't have parameter list in
2749 ipa-prop matching their signature. */
2750 if (ipa_node_params_sum
)
2755 node
->remove_all_references ();
2757 /* Even when not optimizing or not inlining inline always-inline
2759 inlined
= inline_always_inline_functions (node
);
2763 || !flag_early_inlining
2764 /* Never inline regular functions into always-inline functions
2765 during incremental inlining. This sucks as functions calling
2766 always inline functions will get less optimized, but at the
2767 same time inlining of functions calling always inline
2768 function into an always inline function might introduce
2769 cycles of edges to be always inlined in the callgraph.
2771 We might want to be smarter and just avoid this type of inlining. */
2772 || (DECL_DISREGARD_INLINE_LIMITS (node
->decl
)
2773 && lookup_attribute ("always_inline",
2774 DECL_ATTRIBUTES (node
->decl
))))
2776 else if (lookup_attribute ("flatten",
2777 DECL_ATTRIBUTES (node
->decl
)) != NULL
)
2779 /* When the function is marked to be flattened, recursively inline
2781 if (dump_enabled_p ())
2782 dump_printf (MSG_OPTIMIZED_LOCATIONS
,
2783 "Flattening %C\n", node
);
2784 flatten_function (node
, true);
2789 /* If some always_inline functions was inlined, apply the changes.
2790 This way we will not account always inline into growth limits and
2791 moreover we will inline calls from always inlines that we skipped
2792 previously because of conditional above. */
2795 timevar_push (TV_INTEGRATION
);
2796 todo
|= optimize_inline_calls (current_function_decl
);
2797 /* optimize_inline_calls call above might have introduced new
2798 statements that don't have inline parameters computed. */
2799 for (edge
= node
->callees
; edge
; edge
= edge
->next_callee
)
2801 /* We can enounter not-yet-analyzed function during
2802 early inlining on callgraphs with strongly
2803 connected components. */
2804 ipa_call_summary
*es
= ipa_call_summaries
->get_create (edge
);
2806 = estimate_num_insns (edge
->call_stmt
, &eni_size_weights
);
2808 = estimate_num_insns (edge
->call_stmt
, &eni_time_weights
);
2810 ipa_update_overall_fn_summary (node
);
2812 timevar_pop (TV_INTEGRATION
);
2814 /* We iterate incremental inlining to get trivial cases of indirect
2816 while (iterations
< PARAM_VALUE (PARAM_EARLY_INLINER_MAX_ITERATIONS
)
2817 && early_inline_small_functions (node
))
2819 timevar_push (TV_INTEGRATION
);
2820 todo
|= optimize_inline_calls (current_function_decl
);
2822 /* Technically we ought to recompute inline parameters so the new
2823 iteration of early inliner works as expected. We however have
2824 values approximately right and thus we only need to update edge
2825 info that might be cleared out for newly discovered edges. */
2826 for (edge
= node
->callees
; edge
; edge
= edge
->next_callee
)
2828 /* We have no summary for new bound store calls yet. */
2829 ipa_call_summary
*es
= ipa_call_summaries
->get_create (edge
);
2831 = estimate_num_insns (edge
->call_stmt
, &eni_size_weights
);
2833 = estimate_num_insns (edge
->call_stmt
, &eni_time_weights
);
2835 if (edge
->callee
->decl
2836 && !gimple_check_call_matching_types (
2837 edge
->call_stmt
, edge
->callee
->decl
, false))
2839 edge
->inline_failed
= CIF_MISMATCHED_ARGUMENTS
;
2840 edge
->call_stmt_cannot_inline_p
= true;
2843 if (iterations
< PARAM_VALUE (PARAM_EARLY_INLINER_MAX_ITERATIONS
) - 1)
2844 ipa_update_overall_fn_summary (node
);
2845 timevar_pop (TV_INTEGRATION
);
2850 fprintf (dump_file
, "Iterations: %i\n", iterations
);
2855 timevar_push (TV_INTEGRATION
);
2856 todo
|= optimize_inline_calls (current_function_decl
);
2857 timevar_pop (TV_INTEGRATION
);
2860 fun
->always_inline_functions_inlined
= true;
2865 /* Do inlining of small functions. Doing so early helps profiling and other
2866 passes to be somewhat more effective and avoids some code duplication in
2867 later real inlining pass for testcases with very many function calls. */
2871 const pass_data pass_data_early_inline
=
2873 GIMPLE_PASS
, /* type */
2874 "einline", /* name */
2875 OPTGROUP_INLINE
, /* optinfo_flags */
2876 TV_EARLY_INLINING
, /* tv_id */
2877 PROP_ssa
, /* properties_required */
2878 0, /* properties_provided */
2879 0, /* properties_destroyed */
2880 0, /* todo_flags_start */
2881 0, /* todo_flags_finish */
2884 class pass_early_inline
: public gimple_opt_pass
2887 pass_early_inline (gcc::context
*ctxt
)
2888 : gimple_opt_pass (pass_data_early_inline
, ctxt
)
2891 /* opt_pass methods: */
2892 virtual unsigned int execute (function
*);
2894 }; // class pass_early_inline
2897 pass_early_inline::execute (function
*fun
)
2899 return early_inliner (fun
);
2905 make_pass_early_inline (gcc::context
*ctxt
)
2907 return new pass_early_inline (ctxt
);
2912 const pass_data pass_data_ipa_inline
=
2914 IPA_PASS
, /* type */
2915 "inline", /* name */
2916 OPTGROUP_INLINE
, /* optinfo_flags */
2917 TV_IPA_INLINING
, /* tv_id */
2918 0, /* properties_required */
2919 0, /* properties_provided */
2920 0, /* properties_destroyed */
2921 0, /* todo_flags_start */
2922 ( TODO_dump_symtab
), /* todo_flags_finish */
2925 class pass_ipa_inline
: public ipa_opt_pass_d
2928 pass_ipa_inline (gcc::context
*ctxt
)
2929 : ipa_opt_pass_d (pass_data_ipa_inline
, ctxt
,
2930 NULL
, /* generate_summary */
2931 NULL
, /* write_summary */
2932 NULL
, /* read_summary */
2933 NULL
, /* write_optimization_summary */
2934 NULL
, /* read_optimization_summary */
2935 NULL
, /* stmt_fixup */
2936 0, /* function_transform_todo_flags_start */
2937 inline_transform
, /* function_transform */
2938 NULL
) /* variable_transform */
2941 /* opt_pass methods: */
2942 virtual unsigned int execute (function
*) { return ipa_inline (); }
2944 }; // class pass_ipa_inline
2949 make_pass_ipa_inline (gcc::context
*ctxt
)
2951 return new pass_ipa_inline (ctxt
);