1 /* Inlining decision heuristics.
2 Copyright (C) 2003-2018 Free Software Foundation, Inc.
3 Contributed by Jan Hubicka
5 This file is part of GCC.
7 GCC is free software; you can redistribute it and/or modify it under
8 the terms of the GNU General Public License as published by the Free
9 Software Foundation; either version 3, or (at your option) any later
12 GCC is distributed in the hope that it will be useful, but WITHOUT ANY
13 WARRANTY; without even the implied warranty of MERCHANTABILITY or
14 FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
17 You should have received a copy of the GNU General Public License
18 along with GCC; see the file COPYING3. If not see
19 <http://www.gnu.org/licenses/>. */
21 /* Inlining decision heuristics
23 The implementation of inliner is organized as follows:
25 inlining heuristics limits
27 can_inline_edge_p allow to check that particular inlining is allowed
28 by the limits specified by user (allowed function growth, growth and so
31 Functions are inlined when it is obvious the result is profitable (such
32 as functions called once or when inlining reduce code size).
33 In addition to that we perform inlining of small functions and recursive
38 The inliner itself is split into two passes:
42 Simple local inlining pass inlining callees into current function.
43 This pass makes no use of whole unit analysis and thus it can do only
44 very simple decisions based on local properties.
46 The strength of the pass is that it is run in topological order
47 (reverse postorder) on the callgraph. Functions are converted into SSA
48 form just before this pass and optimized subsequently. As a result, the
49 callees of the function seen by the early inliner was already optimized
50 and results of early inlining adds a lot of optimization opportunities
51 for the local optimization.
53 The pass handle the obvious inlining decisions within the compilation
54 unit - inlining auto inline functions, inlining for size and
57 main strength of the pass is the ability to eliminate abstraction
58 penalty in C++ code (via combination of inlining and early
59 optimization) and thus improve quality of analysis done by real IPA
62 Because of lack of whole unit knowledge, the pass can not really make
63 good code size/performance tradeoffs. It however does very simple
64 speculative inlining allowing code size to grow by
65 EARLY_INLINING_INSNS when callee is leaf function. In this case the
66 optimizations performed later are very likely to eliminate the cost.
70 This is the real inliner able to handle inlining with whole program
71 knowledge. It performs following steps:
73 1) inlining of small functions. This is implemented by greedy
74 algorithm ordering all inlinable cgraph edges by their badness and
75 inlining them in this order as long as inline limits allows doing so.
77 This heuristics is not very good on inlining recursive calls. Recursive
78 calls can be inlined with results similar to loop unrolling. To do so,
79 special purpose recursive inliner is executed on function when
80 recursive edge is met as viable candidate.
82 2) Unreachable functions are removed from callgraph. Inlining leads
83 to devirtualization and other modification of callgraph so functions
84 may become unreachable during the process. Also functions declared as
85 extern inline or virtual functions are removed, since after inlining
86 we no longer need the offline bodies.
88 3) Functions called once and not exported from the unit are inlined.
89 This should almost always lead to reduction of code size by eliminating
90 the need for offline copy of the function. */
94 #include "coretypes.h"
100 #include "alloc-pool.h"
101 #include "tree-pass.h"
102 #include "gimple-ssa.h"
104 #include "lto-streamer.h"
105 #include "trans-mem.h"
107 #include "tree-inline.h"
110 #include "symbol-summary.h"
111 #include "tree-vrp.h"
112 #include "ipa-prop.h"
113 #include "ipa-fnsummary.h"
114 #include "ipa-inline.h"
115 #include "ipa-utils.h"
117 #include "auto-profile.h"
118 #include "builtins.h"
119 #include "fibonacci_heap.h"
120 #include "stringpool.h"
124 typedef fibonacci_heap
<sreal
, cgraph_edge
> edge_heap_t
;
125 typedef fibonacci_node
<sreal
, cgraph_edge
> edge_heap_node_t
;
127 /* Statistics we collect about inlining algorithm. */
128 static int overall_size
;
129 static profile_count max_count
;
130 static profile_count spec_rem
;
132 /* Return false when inlining edge E would lead to violating
133 limits on function unit growth or stack usage growth.
135 The relative function body growth limit is present generally
136 to avoid problems with non-linear behavior of the compiler.
137 To allow inlining huge functions into tiny wrapper, the limit
138 is always based on the bigger of the two functions considered.
140 For stack growth limits we always base the growth in stack usage
141 of the callers. We want to prevent applications from segfaulting
142 on stack overflow when functions with huge stack frames gets
146 caller_growth_limits (struct cgraph_edge
*e
)
148 struct cgraph_node
*to
= e
->caller
;
149 struct cgraph_node
*what
= e
->callee
->ultimate_alias_target ();
152 HOST_WIDE_INT stack_size_limit
= 0, inlined_stack
;
153 ipa_fn_summary
*info
, *what_info
;
154 ipa_fn_summary
*outer_info
= ipa_fn_summaries
->get (to
);
156 /* Look for function e->caller is inlined to. While doing
157 so work out the largest function body on the way. As
158 described above, we want to base our function growth
159 limits based on that. Not on the self size of the
160 outer function, not on the self size of inline code
161 we immediately inline to. This is the most relaxed
162 interpretation of the rule "do not grow large functions
163 too much in order to prevent compiler from exploding". */
166 info
= ipa_fn_summaries
->get (to
);
167 if (limit
< info
->self_size
)
168 limit
= info
->self_size
;
169 if (stack_size_limit
< info
->estimated_self_stack_size
)
170 stack_size_limit
= info
->estimated_self_stack_size
;
171 if (to
->global
.inlined_to
)
172 to
= to
->callers
->caller
;
177 what_info
= ipa_fn_summaries
->get (what
);
179 if (limit
< what_info
->self_size
)
180 limit
= what_info
->self_size
;
182 limit
+= limit
* PARAM_VALUE (PARAM_LARGE_FUNCTION_GROWTH
) / 100;
184 /* Check the size after inlining against the function limits. But allow
185 the function to shrink if it went over the limits by forced inlining. */
186 newsize
= estimate_size_after_inlining (to
, e
);
187 if (newsize
>= info
->size
188 && newsize
> PARAM_VALUE (PARAM_LARGE_FUNCTION_INSNS
)
191 e
->inline_failed
= CIF_LARGE_FUNCTION_GROWTH_LIMIT
;
195 if (!what_info
->estimated_stack_size
)
198 /* FIXME: Stack size limit often prevents inlining in Fortran programs
199 due to large i/o datastructures used by the Fortran front-end.
200 We ought to ignore this limit when we know that the edge is executed
201 on every invocation of the caller (i.e. its call statement dominates
202 exit block). We do not track this information, yet. */
203 stack_size_limit
+= ((gcov_type
)stack_size_limit
204 * PARAM_VALUE (PARAM_STACK_FRAME_GROWTH
) / 100);
206 inlined_stack
= (outer_info
->stack_frame_offset
207 + outer_info
->estimated_self_stack_size
208 + what_info
->estimated_stack_size
);
209 /* Check new stack consumption with stack consumption at the place
211 if (inlined_stack
> stack_size_limit
212 /* If function already has large stack usage from sibling
213 inline call, we can inline, too.
214 This bit overoptimistically assume that we are good at stack
216 && inlined_stack
> info
->estimated_stack_size
217 && inlined_stack
> PARAM_VALUE (PARAM_LARGE_STACK_FRAME
))
219 e
->inline_failed
= CIF_LARGE_STACK_FRAME_GROWTH_LIMIT
;
225 /* Dump info about why inlining has failed. */
228 report_inline_failed_reason (struct cgraph_edge
*e
)
230 if (dump_enabled_p ())
232 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, e
->call_stmt
,
233 " not inlinable: %C -> %C, %s\n",
234 e
->caller
, e
->callee
,
235 cgraph_inline_failed_string (e
->inline_failed
));
236 if ((e
->inline_failed
== CIF_TARGET_OPTION_MISMATCH
237 || e
->inline_failed
== CIF_OPTIMIZATION_MISMATCH
)
238 && e
->caller
->lto_file_data
239 && e
->callee
->ultimate_alias_target ()->lto_file_data
)
241 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, e
->call_stmt
,
242 " LTO objects: %s, %s\n",
243 e
->caller
->lto_file_data
->file_name
,
244 e
->callee
->ultimate_alias_target ()->lto_file_data
->file_name
);
246 if (e
->inline_failed
== CIF_TARGET_OPTION_MISMATCH
)
248 cl_target_option_print_diff
249 (dump_file
, 2, target_opts_for_fn (e
->caller
->decl
),
250 target_opts_for_fn (e
->callee
->ultimate_alias_target ()->decl
));
251 if (e
->inline_failed
== CIF_OPTIMIZATION_MISMATCH
)
253 cl_optimization_print_diff
254 (dump_file
, 2, opts_for_fn (e
->caller
->decl
),
255 opts_for_fn (e
->callee
->ultimate_alias_target ()->decl
));
259 /* Decide whether sanitizer-related attributes allow inlining. */
262 sanitize_attrs_match_for_inline_p (const_tree caller
, const_tree callee
)
264 if (!caller
|| !callee
)
267 return ((sanitize_flags_p (SANITIZE_ADDRESS
, caller
)
268 == sanitize_flags_p (SANITIZE_ADDRESS
, callee
))
269 && (sanitize_flags_p (SANITIZE_POINTER_COMPARE
, caller
)
270 == sanitize_flags_p (SANITIZE_POINTER_COMPARE
, callee
))
271 && (sanitize_flags_p (SANITIZE_POINTER_SUBTRACT
, caller
)
272 == sanitize_flags_p (SANITIZE_POINTER_SUBTRACT
, callee
)));
275 /* Used for flags where it is safe to inline when caller's value is
276 grater than callee's. */
277 #define check_maybe_up(flag) \
278 (opts_for_fn (caller->decl)->x_##flag \
279 != opts_for_fn (callee->decl)->x_##flag \
281 || opts_for_fn (caller->decl)->x_##flag \
282 < opts_for_fn (callee->decl)->x_##flag))
283 /* Used for flags where it is safe to inline when caller's value is
284 smaller than callee's. */
285 #define check_maybe_down(flag) \
286 (opts_for_fn (caller->decl)->x_##flag \
287 != opts_for_fn (callee->decl)->x_##flag \
289 || opts_for_fn (caller->decl)->x_##flag \
290 > opts_for_fn (callee->decl)->x_##flag))
291 /* Used for flags where exact match is needed for correctness. */
292 #define check_match(flag) \
293 (opts_for_fn (caller->decl)->x_##flag \
294 != opts_for_fn (callee->decl)->x_##flag)
296 /* Decide if we can inline the edge and possibly update
297 inline_failed reason.
298 We check whether inlining is possible at all and whether
299 caller growth limits allow doing so.
301 if REPORT is true, output reason to the dump file. */
304 can_inline_edge_p (struct cgraph_edge
*e
, bool report
,
307 gcc_checking_assert (e
->inline_failed
);
309 if (cgraph_inline_failed_type (e
->inline_failed
) == CIF_FINAL_ERROR
)
312 report_inline_failed_reason (e
);
316 bool inlinable
= true;
317 enum availability avail
;
318 cgraph_node
*caller
= e
->caller
->global
.inlined_to
319 ? e
->caller
->global
.inlined_to
: e
->caller
;
320 cgraph_node
*callee
= e
->callee
->ultimate_alias_target (&avail
, caller
);
322 if (!callee
->definition
)
324 e
->inline_failed
= CIF_BODY_NOT_AVAILABLE
;
327 if (!early
&& (!opt_for_fn (callee
->decl
, optimize
)
328 || !opt_for_fn (caller
->decl
, optimize
)))
330 e
->inline_failed
= CIF_FUNCTION_NOT_OPTIMIZED
;
333 else if (callee
->calls_comdat_local
)
335 e
->inline_failed
= CIF_USES_COMDAT_LOCAL
;
338 else if (avail
<= AVAIL_INTERPOSABLE
)
340 e
->inline_failed
= CIF_OVERWRITABLE
;
343 /* All edges with call_stmt_cannot_inline_p should have inline_failed
344 initialized to one of FINAL_ERROR reasons. */
345 else if (e
->call_stmt_cannot_inline_p
)
347 /* Don't inline if the functions have different EH personalities. */
348 else if (DECL_FUNCTION_PERSONALITY (caller
->decl
)
349 && DECL_FUNCTION_PERSONALITY (callee
->decl
)
350 && (DECL_FUNCTION_PERSONALITY (caller
->decl
)
351 != DECL_FUNCTION_PERSONALITY (callee
->decl
)))
353 e
->inline_failed
= CIF_EH_PERSONALITY
;
356 /* TM pure functions should not be inlined into non-TM_pure
358 else if (is_tm_pure (callee
->decl
) && !is_tm_pure (caller
->decl
))
360 e
->inline_failed
= CIF_UNSPECIFIED
;
363 /* Check compatibility of target optimization options. */
364 else if (!targetm
.target_option
.can_inline_p (caller
->decl
,
367 e
->inline_failed
= CIF_TARGET_OPTION_MISMATCH
;
370 else if (ipa_fn_summaries
->get (callee
) == NULL
371 || !ipa_fn_summaries
->get (callee
)->inlinable
)
373 e
->inline_failed
= CIF_FUNCTION_NOT_INLINABLE
;
376 /* Don't inline a function with mismatched sanitization attributes. */
377 else if (!sanitize_attrs_match_for_inline_p (caller
->decl
, callee
->decl
))
379 e
->inline_failed
= CIF_ATTRIBUTE_MISMATCH
;
382 if (!inlinable
&& report
)
383 report_inline_failed_reason (e
);
387 /* Decide if we can inline the edge and possibly update
388 inline_failed reason.
389 We check whether inlining is possible at all and whether
390 caller growth limits allow doing so.
392 if REPORT is true, output reason to the dump file.
394 if DISREGARD_LIMITS is true, ignore size limits. */
397 can_inline_edge_by_limits_p (struct cgraph_edge
*e
, bool report
,
398 bool disregard_limits
= false, bool early
= false)
400 gcc_checking_assert (e
->inline_failed
);
402 if (cgraph_inline_failed_type (e
->inline_failed
) == CIF_FINAL_ERROR
)
405 report_inline_failed_reason (e
);
409 bool inlinable
= true;
410 enum availability avail
;
411 cgraph_node
*caller
= e
->caller
->global
.inlined_to
412 ? e
->caller
->global
.inlined_to
: e
->caller
;
413 cgraph_node
*callee
= e
->callee
->ultimate_alias_target (&avail
, caller
);
414 tree caller_tree
= DECL_FUNCTION_SPECIFIC_OPTIMIZATION (caller
->decl
);
416 = callee
? DECL_FUNCTION_SPECIFIC_OPTIMIZATION (callee
->decl
) : NULL
;
417 /* Check if caller growth allows the inlining. */
418 if (!DECL_DISREGARD_INLINE_LIMITS (callee
->decl
)
420 && !lookup_attribute ("flatten",
421 DECL_ATTRIBUTES (caller
->decl
))
422 && !caller_growth_limits (e
))
424 /* Don't inline a function with a higher optimization level than the
425 caller. FIXME: this is really just tip of iceberg of handling
426 optimization attribute. */
427 else if (caller_tree
!= callee_tree
)
430 (DECL_DISREGARD_INLINE_LIMITS (callee
->decl
)
431 && lookup_attribute ("always_inline",
432 DECL_ATTRIBUTES (callee
->decl
)));
433 ipa_fn_summary
*caller_info
= ipa_fn_summaries
->get (caller
);
434 ipa_fn_summary
*callee_info
= ipa_fn_summaries
->get (callee
);
436 /* Until GCC 4.9 we did not check the semantics-altering flags
437 below and inlined across optimization boundaries.
438 Enabling checks below breaks several packages by refusing
439 to inline library always_inline functions. See PR65873.
440 Disable the check for early inlining for now until better solution
442 if (always_inline
&& early
)
444 /* There are some options that change IL semantics which means
445 we cannot inline in these cases for correctness reason.
446 Not even for always_inline declared functions. */
447 else if (check_match (flag_wrapv
)
448 || check_match (flag_trapv
)
449 || check_match (flag_pcc_struct_return
)
450 /* When caller or callee does FP math, be sure FP codegen flags
452 || ((caller_info
->fp_expressions
&& callee_info
->fp_expressions
)
453 && (check_maybe_up (flag_rounding_math
)
454 || check_maybe_up (flag_trapping_math
)
455 || check_maybe_down (flag_unsafe_math_optimizations
)
456 || check_maybe_down (flag_finite_math_only
)
457 || check_maybe_up (flag_signaling_nans
)
458 || check_maybe_down (flag_cx_limited_range
)
459 || check_maybe_up (flag_signed_zeros
)
460 || check_maybe_down (flag_associative_math
)
461 || check_maybe_down (flag_reciprocal_math
)
462 || check_maybe_down (flag_fp_int_builtin_inexact
)
463 /* Strictly speaking only when the callee contains function
464 calls that may end up setting errno. */
465 || check_maybe_up (flag_errno_math
)))
466 /* We do not want to make code compiled with exceptions to be
467 brought into a non-EH function unless we know that the callee
469 This is tracked by DECL_FUNCTION_PERSONALITY. */
470 || (check_maybe_up (flag_non_call_exceptions
)
471 && DECL_FUNCTION_PERSONALITY (callee
->decl
))
472 || (check_maybe_up (flag_exceptions
)
473 && DECL_FUNCTION_PERSONALITY (callee
->decl
))
474 /* When devirtualization is diabled for callee, it is not safe
475 to inline it as we possibly mangled the type info.
476 Allow early inlining of always inlines. */
477 || (!early
&& check_maybe_down (flag_devirtualize
)))
479 e
->inline_failed
= CIF_OPTIMIZATION_MISMATCH
;
482 /* gcc.dg/pr43564.c. Apply user-forced inline even at -O0. */
483 else if (always_inline
)
485 /* When user added an attribute to the callee honor it. */
486 else if (lookup_attribute ("optimize", DECL_ATTRIBUTES (callee
->decl
))
487 && opts_for_fn (caller
->decl
) != opts_for_fn (callee
->decl
))
489 e
->inline_failed
= CIF_OPTIMIZATION_MISMATCH
;
492 /* If explicit optimize attribute are not used, the mismatch is caused
493 by different command line options used to build different units.
494 Do not care about COMDAT functions - those are intended to be
495 optimized with the optimization flags of module they are used in.
496 Also do not care about mixing up size/speed optimization when
497 DECL_DISREGARD_INLINE_LIMITS is set. */
498 else if ((callee
->merged_comdat
499 && !lookup_attribute ("optimize",
500 DECL_ATTRIBUTES (caller
->decl
)))
501 || DECL_DISREGARD_INLINE_LIMITS (callee
->decl
))
503 /* If mismatch is caused by merging two LTO units with different
504 optimizationflags we want to be bit nicer. However never inline
505 if one of functions is not optimized at all. */
506 else if (!opt_for_fn (callee
->decl
, optimize
)
507 || !opt_for_fn (caller
->decl
, optimize
))
509 e
->inline_failed
= CIF_OPTIMIZATION_MISMATCH
;
512 /* If callee is optimized for size and caller is not, allow inlining if
513 code shrinks or we are in MAX_INLINE_INSNS_SINGLE limit and callee
514 is inline (and thus likely an unified comdat). This will allow caller
516 else if (opt_for_fn (callee
->decl
, optimize_size
)
517 > opt_for_fn (caller
->decl
, optimize_size
))
519 int growth
= estimate_edge_growth (e
);
521 && (!DECL_DECLARED_INLINE_P (callee
->decl
)
522 && growth
>= MAX (MAX_INLINE_INSNS_SINGLE
,
523 MAX_INLINE_INSNS_AUTO
)))
525 e
->inline_failed
= CIF_OPTIMIZATION_MISMATCH
;
529 /* If callee is more aggressively optimized for performance than caller,
530 we generally want to inline only cheap (runtime wise) functions. */
531 else if (opt_for_fn (callee
->decl
, optimize_size
)
532 < opt_for_fn (caller
->decl
, optimize_size
)
533 || (opt_for_fn (callee
->decl
, optimize
)
534 > opt_for_fn (caller
->decl
, optimize
)))
536 if (estimate_edge_time (e
)
537 >= 20 + ipa_call_summaries
->get (e
)->call_stmt_time
)
539 e
->inline_failed
= CIF_OPTIMIZATION_MISMATCH
;
546 if (!inlinable
&& report
)
547 report_inline_failed_reason (e
);
552 /* Return true if the edge E is inlinable during early inlining. */
555 can_early_inline_edge_p (struct cgraph_edge
*e
)
557 struct cgraph_node
*callee
= e
->callee
->ultimate_alias_target ();
558 /* Early inliner might get called at WPA stage when IPA pass adds new
559 function. In this case we can not really do any of early inlining
560 because function bodies are missing. */
561 if (cgraph_inline_failed_type (e
->inline_failed
) == CIF_FINAL_ERROR
)
563 if (!gimple_has_body_p (callee
->decl
))
565 e
->inline_failed
= CIF_BODY_NOT_AVAILABLE
;
568 /* In early inliner some of callees may not be in SSA form yet
569 (i.e. the callgraph is cyclic and we did not process
570 the callee by early inliner, yet). We don't have CIF code for this
571 case; later we will re-do the decision in the real inliner. */
572 if (!gimple_in_ssa_p (DECL_STRUCT_FUNCTION (e
->caller
->decl
))
573 || !gimple_in_ssa_p (DECL_STRUCT_FUNCTION (callee
->decl
)))
575 if (dump_enabled_p ())
576 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, e
->call_stmt
,
577 " edge not inlinable: not in SSA form\n");
580 if (!can_inline_edge_p (e
, true, true)
581 || !can_inline_edge_by_limits_p (e
, true, false, true))
587 /* Return number of calls in N. Ignore cheap builtins. */
590 num_calls (struct cgraph_node
*n
)
592 struct cgraph_edge
*e
;
595 for (e
= n
->callees
; e
; e
= e
->next_callee
)
596 if (!is_inexpensive_builtin (e
->callee
->decl
))
602 /* Return true if we are interested in inlining small function. */
605 want_early_inline_function_p (struct cgraph_edge
*e
)
607 bool want_inline
= true;
608 struct cgraph_node
*callee
= e
->callee
->ultimate_alias_target ();
610 if (DECL_DISREGARD_INLINE_LIMITS (callee
->decl
))
612 /* For AutoFDO, we need to make sure that before profile summary, all
613 hot paths' IR look exactly the same as profiled binary. As a result,
614 in einliner, we will disregard size limit and inline those callsites
616 * inlined in the profiled binary, and
617 * the cloned callee has enough samples to be considered "hot". */
618 else if (flag_auto_profile
&& afdo_callsite_hot_enough_for_early_inline (e
))
620 else if (!DECL_DECLARED_INLINE_P (callee
->decl
)
621 && !opt_for_fn (e
->caller
->decl
, flag_inline_small_functions
))
623 e
->inline_failed
= CIF_FUNCTION_NOT_INLINE_CANDIDATE
;
624 report_inline_failed_reason (e
);
629 int growth
= estimate_edge_growth (e
);
634 else if (!e
->maybe_hot_p ()
637 if (dump_enabled_p ())
638 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, e
->call_stmt
,
639 " will not early inline: %C->%C, "
640 "call is cold and code would grow by %i\n",
645 else if (growth
> PARAM_VALUE (PARAM_EARLY_INLINING_INSNS
))
647 if (dump_enabled_p ())
648 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, e
->call_stmt
,
649 " will not early inline: %C->%C, "
650 "growth %i exceeds --param early-inlining-insns\n",
655 else if ((n
= num_calls (callee
)) != 0
656 && growth
* (n
+ 1) > PARAM_VALUE (PARAM_EARLY_INLINING_INSNS
))
658 if (dump_enabled_p ())
659 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, e
->call_stmt
,
660 " will not early inline: %C->%C, "
661 "growth %i exceeds --param early-inlining-insns "
662 "divided by number of calls\n",
671 /* Compute time of the edge->caller + edge->callee execution when inlining
675 compute_uninlined_call_time (struct cgraph_edge
*edge
,
676 sreal uninlined_call_time
)
678 cgraph_node
*caller
= (edge
->caller
->global
.inlined_to
679 ? edge
->caller
->global
.inlined_to
682 sreal freq
= edge
->sreal_frequency ();
684 uninlined_call_time
*= freq
;
686 uninlined_call_time
= uninlined_call_time
>> 11;
688 sreal caller_time
= ipa_fn_summaries
->get (caller
)->time
;
689 return uninlined_call_time
+ caller_time
;
692 /* Same as compute_uinlined_call_time but compute time when inlining
696 compute_inlined_call_time (struct cgraph_edge
*edge
,
699 cgraph_node
*caller
= (edge
->caller
->global
.inlined_to
700 ? edge
->caller
->global
.inlined_to
702 sreal caller_time
= ipa_fn_summaries
->get (caller
)->time
;
704 sreal freq
= edge
->sreal_frequency ();
710 /* This calculation should match one in ipa-inline-analysis.c
711 (estimate_edge_size_and_time). */
712 time
-= (sreal
)ipa_call_summaries
->get (edge
)->call_stmt_time
* freq
;
715 time
= ((sreal
) 1) >> 8;
716 gcc_checking_assert (time
>= 0);
720 /* Return true if the speedup for inlining E is bigger than
721 PARAM_MAX_INLINE_MIN_SPEEDUP. */
724 big_speedup_p (struct cgraph_edge
*e
)
727 sreal spec_time
= estimate_edge_time (e
, &unspec_time
);
728 sreal time
= compute_uninlined_call_time (e
, unspec_time
);
729 sreal inlined_time
= compute_inlined_call_time (e
, spec_time
);
731 if ((time
- inlined_time
) * 100
732 > (sreal
) (time
* PARAM_VALUE (PARAM_INLINE_MIN_SPEEDUP
)))
737 /* Return true if we are interested in inlining small function.
738 When REPORT is true, report reason to dump file. */
741 want_inline_small_function_p (struct cgraph_edge
*e
, bool report
)
743 bool want_inline
= true;
744 struct cgraph_node
*callee
= e
->callee
->ultimate_alias_target ();
746 /* Allow this function to be called before can_inline_edge_p,
747 since it's usually cheaper. */
748 if (cgraph_inline_failed_type (e
->inline_failed
) == CIF_FINAL_ERROR
)
750 else if (DECL_DISREGARD_INLINE_LIMITS (callee
->decl
))
752 else if (!DECL_DECLARED_INLINE_P (callee
->decl
)
753 && !opt_for_fn (e
->caller
->decl
, flag_inline_small_functions
))
755 e
->inline_failed
= CIF_FUNCTION_NOT_INLINE_CANDIDATE
;
758 /* Do fast and conservative check if the function can be good
759 inline candidate. At the moment we allow inline hints to
760 promote non-inline functions to inline and we increase
761 MAX_INLINE_INSNS_SINGLE 16-fold for inline functions. */
762 else if ((!DECL_DECLARED_INLINE_P (callee
->decl
)
763 && (!e
->count
.ipa ().initialized_p () || !e
->maybe_hot_p ()))
764 && ipa_fn_summaries
->get (callee
)->min_size
765 - ipa_call_summaries
->get (e
)->call_stmt_size
766 > MAX (MAX_INLINE_INSNS_SINGLE
, MAX_INLINE_INSNS_AUTO
))
768 e
->inline_failed
= CIF_MAX_INLINE_INSNS_AUTO_LIMIT
;
771 else if ((DECL_DECLARED_INLINE_P (callee
->decl
)
772 || e
->count
.ipa ().nonzero_p ())
773 && ipa_fn_summaries
->get (callee
)->min_size
774 - ipa_call_summaries
->get (e
)->call_stmt_size
775 > 16 * MAX_INLINE_INSNS_SINGLE
)
777 e
->inline_failed
= (DECL_DECLARED_INLINE_P (callee
->decl
)
778 ? CIF_MAX_INLINE_INSNS_SINGLE_LIMIT
779 : CIF_MAX_INLINE_INSNS_AUTO_LIMIT
);
784 int growth
= estimate_edge_growth (e
);
785 ipa_hints hints
= estimate_edge_hints (e
);
786 int big_speedup
= -1; /* compute this lazily */
790 /* Apply MAX_INLINE_INSNS_SINGLE limit. Do not do so when
791 hints suggests that inlining given function is very profitable. */
792 else if (DECL_DECLARED_INLINE_P (callee
->decl
)
793 && growth
>= MAX_INLINE_INSNS_SINGLE
794 && (growth
>= MAX_INLINE_INSNS_SINGLE
* 16
795 || (!(hints
& (INLINE_HINT_indirect_call
796 | INLINE_HINT_known_hot
797 | INLINE_HINT_loop_iterations
798 | INLINE_HINT_array_index
799 | INLINE_HINT_loop_stride
))
800 && !(big_speedup
= big_speedup_p (e
)))))
802 e
->inline_failed
= CIF_MAX_INLINE_INSNS_SINGLE_LIMIT
;
805 else if (!DECL_DECLARED_INLINE_P (callee
->decl
)
806 && !opt_for_fn (e
->caller
->decl
, flag_inline_functions
))
808 /* growth_likely_positive is expensive, always test it last. */
809 if (growth
>= MAX_INLINE_INSNS_SINGLE
810 || growth_likely_positive (callee
, growth
))
812 e
->inline_failed
= CIF_NOT_DECLARED_INLINED
;
816 /* Apply MAX_INLINE_INSNS_AUTO limit for functions not declared inline
817 Upgrade it to MAX_INLINE_INSNS_SINGLE when hints suggests that
818 inlining given function is very profitable. */
819 else if (!DECL_DECLARED_INLINE_P (callee
->decl
)
820 && !(hints
& INLINE_HINT_known_hot
)
821 && growth
>= ((hints
& (INLINE_HINT_indirect_call
822 | INLINE_HINT_loop_iterations
823 | INLINE_HINT_array_index
824 | INLINE_HINT_loop_stride
))
825 ? MAX (MAX_INLINE_INSNS_AUTO
,
826 MAX_INLINE_INSNS_SINGLE
)
827 : MAX_INLINE_INSNS_AUTO
)
828 && !(big_speedup
== -1 ? big_speedup_p (e
) : big_speedup
))
830 /* growth_likely_positive is expensive, always test it last. */
831 if (growth
>= MAX_INLINE_INSNS_SINGLE
832 || growth_likely_positive (callee
, growth
))
834 e
->inline_failed
= CIF_MAX_INLINE_INSNS_AUTO_LIMIT
;
838 /* If call is cold, do not inline when function body would grow. */
839 else if (!e
->maybe_hot_p ()
840 && (growth
>= MAX_INLINE_INSNS_SINGLE
841 || growth_likely_positive (callee
, growth
)))
843 e
->inline_failed
= CIF_UNLIKELY_CALL
;
847 if (!want_inline
&& report
)
848 report_inline_failed_reason (e
);
852 /* EDGE is self recursive edge.
853 We hand two cases - when function A is inlining into itself
854 or when function A is being inlined into another inliner copy of function
857 In first case OUTER_NODE points to the toplevel copy of A, while
858 in the second case OUTER_NODE points to the outermost copy of A in B.
860 In both cases we want to be extra selective since
861 inlining the call will just introduce new recursive calls to appear. */
864 want_inline_self_recursive_call_p (struct cgraph_edge
*edge
,
865 struct cgraph_node
*outer_node
,
869 char const *reason
= NULL
;
870 bool want_inline
= true;
871 sreal caller_freq
= 1;
872 int max_depth
= PARAM_VALUE (PARAM_MAX_INLINE_RECURSIVE_DEPTH_AUTO
);
874 if (DECL_DECLARED_INLINE_P (edge
->caller
->decl
))
875 max_depth
= PARAM_VALUE (PARAM_MAX_INLINE_RECURSIVE_DEPTH
);
877 if (!edge
->maybe_hot_p ())
879 reason
= "recursive call is cold";
882 else if (depth
> max_depth
)
884 reason
= "--param max-inline-recursive-depth exceeded.";
887 else if (outer_node
->global
.inlined_to
888 && (caller_freq
= outer_node
->callers
->sreal_frequency ()) == 0)
890 reason
= "caller frequency is 0";
896 /* Inlining of self recursive function into copy of itself within other
897 function is transformation similar to loop peeling.
899 Peeling is profitable if we can inline enough copies to make probability
900 of actual call to the self recursive function very small. Be sure that
901 the probability of recursion is small.
903 We ensure that the frequency of recursing is at most 1 - (1/max_depth).
904 This way the expected number of recursion is at most max_depth. */
907 sreal max_prob
= (sreal
)1 - ((sreal
)1 / (sreal
)max_depth
);
909 for (i
= 1; i
< depth
; i
++)
910 max_prob
= max_prob
* max_prob
;
911 if (edge
->sreal_frequency () >= max_prob
* caller_freq
)
913 reason
= "frequency of recursive call is too large";
917 /* Recursive inlining, i.e. equivalent of unrolling, is profitable if
918 recursion depth is large. We reduce function call overhead and increase
919 chances that things fit in hardware return predictor.
921 Recursive inlining might however increase cost of stack frame setup
922 actually slowing down functions whose recursion tree is wide rather than
925 Deciding reliably on when to do recursive inlining without profile feedback
926 is tricky. For now we disable recursive inlining when probability of self
929 Recursive inlining of self recursive call within loop also results in
930 large loop depths that generally optimize badly. We may want to throttle
931 down inlining in those cases. In particular this seems to happen in one
932 of libstdc++ rb tree methods. */
935 if (edge
->sreal_frequency () * 100
937 * PARAM_VALUE (PARAM_MIN_INLINE_RECURSIVE_PROBABILITY
))
939 reason
= "frequency of recursive call is too small";
943 if (!want_inline
&& dump_enabled_p ())
944 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, edge
->call_stmt
,
945 " not inlining recursively: %s\n", reason
);
949 /* Return true when NODE has uninlinable caller;
950 set HAS_HOT_CALL if it has hot call.
951 Worker for cgraph_for_node_and_aliases. */
954 check_callers (struct cgraph_node
*node
, void *has_hot_call
)
956 struct cgraph_edge
*e
;
957 for (e
= node
->callers
; e
; e
= e
->next_caller
)
959 if (!opt_for_fn (e
->caller
->decl
, flag_inline_functions_called_once
)
960 || !opt_for_fn (e
->caller
->decl
, optimize
))
962 if (!can_inline_edge_p (e
, true))
964 if (e
->recursive_p ())
966 if (!can_inline_edge_by_limits_p (e
, true))
968 if (!(*(bool *)has_hot_call
) && e
->maybe_hot_p ())
969 *(bool *)has_hot_call
= true;
974 /* If NODE has a caller, return true. */
977 has_caller_p (struct cgraph_node
*node
, void *data ATTRIBUTE_UNUSED
)
984 /* Decide if inlining NODE would reduce unit size by eliminating
985 the offline copy of function.
986 When COLD is true the cold calls are considered, too. */
989 want_inline_function_to_all_callers_p (struct cgraph_node
*node
, bool cold
)
991 bool has_hot_call
= false;
993 /* Aliases gets inlined along with the function they alias. */
996 /* Already inlined? */
997 if (node
->global
.inlined_to
)
999 /* Does it have callers? */
1000 if (!node
->call_for_symbol_and_aliases (has_caller_p
, NULL
, true))
1002 /* Inlining into all callers would increase size? */
1003 if (estimate_growth (node
) > 0)
1005 /* All inlines must be possible. */
1006 if (node
->call_for_symbol_and_aliases (check_callers
, &has_hot_call
,
1009 if (!cold
&& !has_hot_call
)
1014 /* A cost model driving the inlining heuristics in a way so the edges with
1015 smallest badness are inlined first. After each inlining is performed
1016 the costs of all caller edges of nodes affected are recomputed so the
1017 metrics may accurately depend on values such as number of inlinable callers
1018 of the function or function body size. */
1021 edge_badness (struct cgraph_edge
*edge
, bool dump
)
1025 sreal edge_time
, unspec_edge_time
;
1026 struct cgraph_node
*callee
= edge
->callee
->ultimate_alias_target ();
1027 struct ipa_fn_summary
*callee_info
= ipa_fn_summaries
->get (callee
);
1029 cgraph_node
*caller
= (edge
->caller
->global
.inlined_to
1030 ? edge
->caller
->global
.inlined_to
1033 growth
= estimate_edge_growth (edge
);
1034 edge_time
= estimate_edge_time (edge
, &unspec_edge_time
);
1035 hints
= estimate_edge_hints (edge
);
1036 gcc_checking_assert (edge_time
>= 0);
1037 /* Check that inlined time is better, but tolerate some roundoff issues.
1038 FIXME: When callee profile drops to 0 we account calls more. This
1039 should be fixed by never doing that. */
1040 gcc_checking_assert ((edge_time
* 100
1041 - callee_info
->time
* 101).to_int () <= 0
1042 || callee
->count
.ipa ().initialized_p ());
1043 gcc_checking_assert (growth
<= callee_info
->size
);
1047 fprintf (dump_file
, " Badness calculation for %s -> %s\n",
1048 edge
->caller
->dump_name (),
1049 edge
->callee
->dump_name ());
1050 fprintf (dump_file
, " size growth %i, time %f unspec %f ",
1052 edge_time
.to_double (),
1053 unspec_edge_time
.to_double ());
1054 ipa_dump_hints (dump_file
, hints
);
1055 if (big_speedup_p (edge
))
1056 fprintf (dump_file
, " big_speedup");
1057 fprintf (dump_file
, "\n");
1060 /* Always prefer inlining saving code size. */
1063 badness
= (sreal
) (-SREAL_MIN_SIG
+ growth
) << (SREAL_MAX_EXP
/ 256);
1065 fprintf (dump_file
, " %f: Growth %d <= 0\n", badness
.to_double (),
1068 /* Inlining into EXTERNAL functions is not going to change anything unless
1069 they are themselves inlined. */
1070 else if (DECL_EXTERNAL (caller
->decl
))
1073 fprintf (dump_file
, " max: function is external\n");
1074 return sreal::max ();
1076 /* When profile is available. Compute badness as:
1078 time_saved * caller_count
1079 goodness = -------------------------------------------------
1080 growth_of_caller * overall_growth * combined_size
1082 badness = - goodness
1084 Again use negative value to make calls with profile appear hotter
1087 else if (opt_for_fn (caller
->decl
, flag_guess_branch_prob
)
1088 || caller
->count
.ipa ().nonzero_p ())
1090 sreal numerator
, denominator
;
1092 sreal inlined_time
= compute_inlined_call_time (edge
, edge_time
);
1094 numerator
= (compute_uninlined_call_time (edge
, unspec_edge_time
)
1097 numerator
= ((sreal
) 1 >> 8);
1098 if (caller
->count
.ipa ().nonzero_p ())
1099 numerator
*= caller
->count
.ipa ().to_gcov_type ();
1100 else if (caller
->count
.ipa ().initialized_p ())
1101 numerator
= numerator
>> 11;
1102 denominator
= growth
;
1104 overall_growth
= callee_info
->growth
;
1106 /* Look for inliner wrappers of the form:
1112 noninline_callee ();
1114 Withhout panilizing this case, we usually inline noninline_callee
1115 into the inline_caller because overall_growth is small preventing
1116 further inlining of inline_caller.
1118 Penalize only callgraph edges to functions with small overall
1121 if (growth
> overall_growth
1122 /* ... and having only one caller which is not inlined ... */
1123 && callee_info
->single_caller
1124 && !edge
->caller
->global
.inlined_to
1125 /* ... and edges executed only conditionally ... */
1126 && edge
->sreal_frequency () < 1
1127 /* ... consider case where callee is not inline but caller is ... */
1128 && ((!DECL_DECLARED_INLINE_P (edge
->callee
->decl
)
1129 && DECL_DECLARED_INLINE_P (caller
->decl
))
1130 /* ... or when early optimizers decided to split and edge
1131 frequency still indicates splitting is a win ... */
1132 || (callee
->split_part
&& !caller
->split_part
1133 && edge
->sreal_frequency () * 100
1135 (PARAM_PARTIAL_INLINING_ENTRY_PROBABILITY
)
1136 /* ... and do not overwrite user specified hints. */
1137 && (!DECL_DECLARED_INLINE_P (edge
->callee
->decl
)
1138 || DECL_DECLARED_INLINE_P (caller
->decl
)))))
1140 ipa_fn_summary
*caller_info
= ipa_fn_summaries
->get (caller
);
1141 int caller_growth
= caller_info
->growth
;
1143 /* Only apply the penalty when caller looks like inline candidate,
1144 and it is not called once and. */
1145 if (!caller_info
->single_caller
&& overall_growth
< caller_growth
1146 && caller_info
->inlinable
1147 && caller_info
->size
1148 < (DECL_DECLARED_INLINE_P (caller
->decl
)
1149 ? MAX_INLINE_INSNS_SINGLE
: MAX_INLINE_INSNS_AUTO
))
1153 " Wrapper penalty. Increasing growth %i to %i\n",
1154 overall_growth
, caller_growth
);
1155 overall_growth
= caller_growth
;
1158 if (overall_growth
> 0)
1160 /* Strongly preffer functions with few callers that can be inlined
1161 fully. The square root here leads to smaller binaries at average.
1162 Watch however for extreme cases and return to linear function
1163 when growth is large. */
1164 if (overall_growth
< 256)
1165 overall_growth
*= overall_growth
;
1167 overall_growth
+= 256 * 256 - 256;
1168 denominator
*= overall_growth
;
1170 denominator
*= inlined_time
;
1172 badness
= - numerator
/ denominator
;
1177 " %f: guessed profile. frequency %f, count %" PRId64
1178 " caller count %" PRId64
1179 " time w/o inlining %f, time with inlining %f"
1180 " overall growth %i (current) %i (original)"
1181 " %i (compensated)\n",
1182 badness
.to_double (),
1183 edge
->sreal_frequency ().to_double (),
1184 edge
->count
.ipa ().initialized_p () ? edge
->count
.ipa ().to_gcov_type () : -1,
1185 caller
->count
.ipa ().initialized_p () ? caller
->count
.ipa ().to_gcov_type () : -1,
1186 compute_uninlined_call_time (edge
,
1187 unspec_edge_time
).to_double (),
1188 inlined_time
.to_double (),
1189 estimate_growth (callee
),
1190 callee_info
->growth
, overall_growth
);
1193 /* When function local profile is not available or it does not give
1194 useful information (ie frequency is zero), base the cost on
1195 loop nest and overall size growth, so we optimize for overall number
1196 of functions fully inlined in program. */
1199 int nest
= MIN (ipa_call_summaries
->get (edge
)->loop_depth
, 8);
1202 /* Decrease badness if call is nested. */
1204 badness
= badness
>> nest
;
1206 badness
= badness
<< nest
;
1208 fprintf (dump_file
, " %f: no profile. nest %i\n",
1209 badness
.to_double (), nest
);
1211 gcc_checking_assert (badness
!= 0);
1213 if (edge
->recursive_p ())
1214 badness
= badness
.shift (badness
> 0 ? 4 : -4);
1215 if ((hints
& (INLINE_HINT_indirect_call
1216 | INLINE_HINT_loop_iterations
1217 | INLINE_HINT_array_index
1218 | INLINE_HINT_loop_stride
))
1219 || callee_info
->growth
<= 0)
1220 badness
= badness
.shift (badness
> 0 ? -2 : 2);
1221 if (hints
& (INLINE_HINT_same_scc
))
1222 badness
= badness
.shift (badness
> 0 ? 3 : -3);
1223 else if (hints
& (INLINE_HINT_in_scc
))
1224 badness
= badness
.shift (badness
> 0 ? 2 : -2);
1225 else if (hints
& (INLINE_HINT_cross_module
))
1226 badness
= badness
.shift (badness
> 0 ? 1 : -1);
1227 if (DECL_DISREGARD_INLINE_LIMITS (callee
->decl
))
1228 badness
= badness
.shift (badness
> 0 ? -4 : 4);
1229 else if ((hints
& INLINE_HINT_declared_inline
))
1230 badness
= badness
.shift (badness
> 0 ? -3 : 3);
1232 fprintf (dump_file
, " Adjusted by hints %f\n", badness
.to_double ());
1236 /* Recompute badness of EDGE and update its key in HEAP if needed. */
1238 update_edge_key (edge_heap_t
*heap
, struct cgraph_edge
*edge
)
1240 sreal badness
= edge_badness (edge
, false);
1243 edge_heap_node_t
*n
= (edge_heap_node_t
*) edge
->aux
;
1244 gcc_checking_assert (n
->get_data () == edge
);
1246 /* fibonacci_heap::replace_key does busy updating of the
1247 heap that is unnecesarily expensive.
1248 We do lazy increases: after extracting minimum if the key
1249 turns out to be out of date, it is re-inserted into heap
1250 with correct value. */
1251 if (badness
< n
->get_key ())
1253 if (dump_file
&& (dump_flags
& TDF_DETAILS
))
1256 " decreasing badness %s -> %s, %f to %f\n",
1257 edge
->caller
->dump_name (),
1258 edge
->callee
->dump_name (),
1259 n
->get_key ().to_double (),
1260 badness
.to_double ());
1262 heap
->decrease_key (n
, badness
);
1267 if (dump_file
&& (dump_flags
& TDF_DETAILS
))
1270 " enqueuing call %s -> %s, badness %f\n",
1271 edge
->caller
->dump_name (),
1272 edge
->callee
->dump_name (),
1273 badness
.to_double ());
1275 edge
->aux
= heap
->insert (badness
, edge
);
1280 /* NODE was inlined.
1281 All caller edges needs to be resetted because
1282 size estimates change. Similarly callees needs reset
1283 because better context may be known. */
1286 reset_edge_caches (struct cgraph_node
*node
)
1288 struct cgraph_edge
*edge
;
1289 struct cgraph_edge
*e
= node
->callees
;
1290 struct cgraph_node
*where
= node
;
1291 struct ipa_ref
*ref
;
1293 if (where
->global
.inlined_to
)
1294 where
= where
->global
.inlined_to
;
1296 if (edge_growth_cache
!= NULL
)
1297 for (edge
= where
->callers
; edge
; edge
= edge
->next_caller
)
1298 if (edge
->inline_failed
)
1299 edge_growth_cache
->remove (edge
);
1301 FOR_EACH_ALIAS (where
, ref
)
1302 reset_edge_caches (dyn_cast
<cgraph_node
*> (ref
->referring
));
1308 if (!e
->inline_failed
&& e
->callee
->callees
)
1309 e
= e
->callee
->callees
;
1312 if (edge_growth_cache
!= NULL
&& e
->inline_failed
)
1313 edge_growth_cache
->remove (e
);
1320 if (e
->caller
== node
)
1322 e
= e
->caller
->callers
;
1324 while (!e
->next_callee
);
1330 /* Recompute HEAP nodes for each of caller of NODE.
1331 UPDATED_NODES track nodes we already visited, to avoid redundant work.
1332 When CHECK_INLINABLITY_FOR is set, re-check for specified edge that
1333 it is inlinable. Otherwise check all edges. */
1336 update_caller_keys (edge_heap_t
*heap
, struct cgraph_node
*node
,
1337 bitmap updated_nodes
,
1338 struct cgraph_edge
*check_inlinablity_for
)
1340 struct cgraph_edge
*edge
;
1341 struct ipa_ref
*ref
;
1343 if ((!node
->alias
&& !ipa_fn_summaries
->get (node
)->inlinable
)
1344 || node
->global
.inlined_to
)
1346 if (!bitmap_set_bit (updated_nodes
, node
->get_uid ()))
1349 FOR_EACH_ALIAS (node
, ref
)
1351 struct cgraph_node
*alias
= dyn_cast
<cgraph_node
*> (ref
->referring
);
1352 update_caller_keys (heap
, alias
, updated_nodes
, check_inlinablity_for
);
1355 for (edge
= node
->callers
; edge
; edge
= edge
->next_caller
)
1356 if (edge
->inline_failed
)
1358 if (!check_inlinablity_for
1359 || check_inlinablity_for
== edge
)
1361 if (can_inline_edge_p (edge
, false)
1362 && want_inline_small_function_p (edge
, false)
1363 && can_inline_edge_by_limits_p (edge
, false))
1364 update_edge_key (heap
, edge
);
1367 report_inline_failed_reason (edge
);
1368 heap
->delete_node ((edge_heap_node_t
*) edge
->aux
);
1373 update_edge_key (heap
, edge
);
1377 /* Recompute HEAP nodes for each uninlined call in NODE.
1378 This is used when we know that edge badnesses are going only to increase
1379 (we introduced new call site) and thus all we need is to insert newly
1380 created edges into heap. */
1383 update_callee_keys (edge_heap_t
*heap
, struct cgraph_node
*node
,
1384 bitmap updated_nodes
)
1386 struct cgraph_edge
*e
= node
->callees
;
1391 if (!e
->inline_failed
&& e
->callee
->callees
)
1392 e
= e
->callee
->callees
;
1395 enum availability avail
;
1396 struct cgraph_node
*callee
;
1397 /* We do not reset callee growth cache here. Since we added a new call,
1398 growth chould have just increased and consequentely badness metric
1399 don't need updating. */
1400 if (e
->inline_failed
1401 && (callee
= e
->callee
->ultimate_alias_target (&avail
, e
->caller
))
1402 && ipa_fn_summaries
->get (callee
) != NULL
1403 && ipa_fn_summaries
->get (callee
)->inlinable
1404 && avail
>= AVAIL_AVAILABLE
1405 && !bitmap_bit_p (updated_nodes
, callee
->get_uid ()))
1407 if (can_inline_edge_p (e
, false)
1408 && want_inline_small_function_p (e
, false)
1409 && can_inline_edge_by_limits_p (e
, false))
1410 update_edge_key (heap
, e
);
1413 report_inline_failed_reason (e
);
1414 heap
->delete_node ((edge_heap_node_t
*) e
->aux
);
1424 if (e
->caller
== node
)
1426 e
= e
->caller
->callers
;
1428 while (!e
->next_callee
);
1434 /* Enqueue all recursive calls from NODE into priority queue depending on
1435 how likely we want to recursively inline the call. */
1438 lookup_recursive_calls (struct cgraph_node
*node
, struct cgraph_node
*where
,
1441 struct cgraph_edge
*e
;
1442 enum availability avail
;
1444 for (e
= where
->callees
; e
; e
= e
->next_callee
)
1445 if (e
->callee
== node
1446 || (e
->callee
->ultimate_alias_target (&avail
, e
->caller
) == node
1447 && avail
> AVAIL_INTERPOSABLE
))
1448 heap
->insert (-e
->sreal_frequency (), e
);
1449 for (e
= where
->callees
; e
; e
= e
->next_callee
)
1450 if (!e
->inline_failed
)
1451 lookup_recursive_calls (node
, e
->callee
, heap
);
1454 /* Decide on recursive inlining: in the case function has recursive calls,
1455 inline until body size reaches given argument. If any new indirect edges
1456 are discovered in the process, add them to *NEW_EDGES, unless NEW_EDGES
1460 recursive_inlining (struct cgraph_edge
*edge
,
1461 vec
<cgraph_edge
*> *new_edges
)
1463 int limit
= PARAM_VALUE (PARAM_MAX_INLINE_INSNS_RECURSIVE_AUTO
);
1464 edge_heap_t
heap (sreal::min ());
1465 struct cgraph_node
*node
;
1466 struct cgraph_edge
*e
;
1467 struct cgraph_node
*master_clone
= NULL
, *next
;
1471 node
= edge
->caller
;
1472 if (node
->global
.inlined_to
)
1473 node
= node
->global
.inlined_to
;
1475 if (DECL_DECLARED_INLINE_P (node
->decl
))
1476 limit
= PARAM_VALUE (PARAM_MAX_INLINE_INSNS_RECURSIVE
);
1478 /* Make sure that function is small enough to be considered for inlining. */
1479 if (estimate_size_after_inlining (node
, edge
) >= limit
)
1481 lookup_recursive_calls (node
, node
, &heap
);
1487 " Performing recursive inlining on %s\n",
1490 /* Do the inlining and update list of recursive call during process. */
1491 while (!heap
.empty ())
1493 struct cgraph_edge
*curr
= heap
.extract_min ();
1494 struct cgraph_node
*cnode
, *dest
= curr
->callee
;
1496 if (!can_inline_edge_p (curr
, true)
1497 || can_inline_edge_by_limits_p (curr
, true))
1500 /* MASTER_CLONE is produced in the case we already started modified
1501 the function. Be sure to redirect edge to the original body before
1502 estimating growths otherwise we will be seeing growths after inlining
1503 the already modified body. */
1506 curr
->redirect_callee (master_clone
);
1507 if (edge_growth_cache
!= NULL
)
1508 edge_growth_cache
->remove (curr
);
1511 if (estimate_size_after_inlining (node
, curr
) > limit
)
1513 curr
->redirect_callee (dest
);
1514 if (edge_growth_cache
!= NULL
)
1515 edge_growth_cache
->remove (curr
);
1520 for (cnode
= curr
->caller
;
1521 cnode
->global
.inlined_to
; cnode
= cnode
->callers
->caller
)
1523 == curr
->callee
->ultimate_alias_target ()->decl
)
1526 if (!want_inline_self_recursive_call_p (curr
, node
, false, depth
))
1528 curr
->redirect_callee (dest
);
1529 if (edge_growth_cache
!= NULL
)
1530 edge_growth_cache
->remove (curr
);
1537 " Inlining call of depth %i", depth
);
1538 if (node
->count
.nonzero_p ())
1540 fprintf (dump_file
, " called approx. %.2f times per call",
1541 (double)curr
->count
.to_gcov_type ()
1542 / node
->count
.to_gcov_type ());
1544 fprintf (dump_file
, "\n");
1548 /* We need original clone to copy around. */
1549 master_clone
= node
->create_clone (node
->decl
, node
->count
,
1550 false, vNULL
, true, NULL
, NULL
);
1551 for (e
= master_clone
->callees
; e
; e
= e
->next_callee
)
1552 if (!e
->inline_failed
)
1553 clone_inlined_nodes (e
, true, false, NULL
);
1554 curr
->redirect_callee (master_clone
);
1555 if (edge_growth_cache
!= NULL
)
1556 edge_growth_cache
->remove (curr
);
1559 inline_call (curr
, false, new_edges
, &overall_size
, true);
1560 lookup_recursive_calls (node
, curr
->callee
, &heap
);
1564 if (!heap
.empty () && dump_file
)
1565 fprintf (dump_file
, " Recursive inlining growth limit met.\n");
1570 if (dump_enabled_p ())
1571 dump_printf_loc (MSG_NOTE
, edge
->call_stmt
,
1572 "\n Inlined %i times, "
1573 "body grown from size %i to %i, time %f to %f\n", n
,
1574 ipa_fn_summaries
->get (master_clone
)->size
,
1575 ipa_fn_summaries
->get (node
)->size
,
1576 ipa_fn_summaries
->get (master_clone
)->time
.to_double (),
1577 ipa_fn_summaries
->get (node
)->time
.to_double ());
1579 /* Remove master clone we used for inlining. We rely that clones inlined
1580 into master clone gets queued just before master clone so we don't
1582 for (node
= symtab
->first_function (); node
!= master_clone
;
1585 next
= symtab
->next_function (node
);
1586 if (node
->global
.inlined_to
== master_clone
)
1589 master_clone
->remove ();
1594 /* Given whole compilation unit estimate of INSNS, compute how large we can
1595 allow the unit to grow. */
1598 compute_max_insns (int insns
)
1600 int max_insns
= insns
;
1601 if (max_insns
< PARAM_VALUE (PARAM_LARGE_UNIT_INSNS
))
1602 max_insns
= PARAM_VALUE (PARAM_LARGE_UNIT_INSNS
);
1604 return ((int64_t) max_insns
1605 * (100 + PARAM_VALUE (PARAM_INLINE_UNIT_GROWTH
)) / 100);
1609 /* Compute badness of all edges in NEW_EDGES and add them to the HEAP. */
1612 add_new_edges_to_heap (edge_heap_t
*heap
, vec
<cgraph_edge
*> new_edges
)
1614 while (new_edges
.length () > 0)
1616 struct cgraph_edge
*edge
= new_edges
.pop ();
1618 gcc_assert (!edge
->aux
);
1619 if (edge
->inline_failed
1620 && can_inline_edge_p (edge
, true)
1621 && want_inline_small_function_p (edge
, true)
1622 && can_inline_edge_by_limits_p (edge
, true))
1623 edge
->aux
= heap
->insert (edge_badness (edge
, false), edge
);
1627 /* Remove EDGE from the fibheap. */
1630 heap_edge_removal_hook (struct cgraph_edge
*e
, void *data
)
1634 ((edge_heap_t
*)data
)->delete_node ((edge_heap_node_t
*)e
->aux
);
1639 /* Return true if speculation of edge E seems useful.
1640 If ANTICIPATE_INLINING is true, be conservative and hope that E
1644 speculation_useful_p (struct cgraph_edge
*e
, bool anticipate_inlining
)
1646 enum availability avail
;
1647 struct cgraph_node
*target
= e
->callee
->ultimate_alias_target (&avail
,
1649 struct cgraph_edge
*direct
, *indirect
;
1650 struct ipa_ref
*ref
;
1652 gcc_assert (e
->speculative
&& !e
->indirect_unknown_callee
);
1654 if (!e
->maybe_hot_p ())
1657 /* See if IP optimizations found something potentially useful about the
1658 function. For now we look only for CONST/PURE flags. Almost everything
1659 else we propagate is useless. */
1660 if (avail
>= AVAIL_AVAILABLE
)
1662 int ecf_flags
= flags_from_decl_or_type (target
->decl
);
1663 if (ecf_flags
& ECF_CONST
)
1665 e
->speculative_call_info (direct
, indirect
, ref
);
1666 if (!(indirect
->indirect_info
->ecf_flags
& ECF_CONST
))
1669 else if (ecf_flags
& ECF_PURE
)
1671 e
->speculative_call_info (direct
, indirect
, ref
);
1672 if (!(indirect
->indirect_info
->ecf_flags
& ECF_PURE
))
1676 /* If we did not managed to inline the function nor redirect
1677 to an ipa-cp clone (that are seen by having local flag set),
1678 it is probably pointless to inline it unless hardware is missing
1679 indirect call predictor. */
1680 if (!anticipate_inlining
&& e
->inline_failed
&& !target
->local
.local
)
1682 /* For overwritable targets there is not much to do. */
1683 if (e
->inline_failed
1684 && (!can_inline_edge_p (e
, false)
1685 || !can_inline_edge_by_limits_p (e
, false, true)))
1687 /* OK, speculation seems interesting. */
1691 /* We know that EDGE is not going to be inlined.
1692 See if we can remove speculation. */
1695 resolve_noninline_speculation (edge_heap_t
*edge_heap
, struct cgraph_edge
*edge
)
1697 if (edge
->speculative
&& !speculation_useful_p (edge
, false))
1699 struct cgraph_node
*node
= edge
->caller
;
1700 struct cgraph_node
*where
= node
->global
.inlined_to
1701 ? node
->global
.inlined_to
: node
;
1702 auto_bitmap updated_nodes
;
1704 if (edge
->count
.ipa ().initialized_p ())
1705 spec_rem
+= edge
->count
.ipa ();
1706 edge
->resolve_speculation ();
1707 reset_edge_caches (where
);
1708 ipa_update_overall_fn_summary (where
);
1709 update_caller_keys (edge_heap
, where
,
1710 updated_nodes
, NULL
);
1711 update_callee_keys (edge_heap
, where
,
1716 /* Return true if NODE should be accounted for overall size estimate.
1717 Skip all nodes optimized for size so we can measure the growth of hot
1718 part of program no matter of the padding. */
1721 inline_account_function_p (struct cgraph_node
*node
)
1723 return (!DECL_EXTERNAL (node
->decl
)
1724 && !opt_for_fn (node
->decl
, optimize_size
)
1725 && node
->frequency
!= NODE_FREQUENCY_UNLIKELY_EXECUTED
);
1728 /* Count number of callers of NODE and store it into DATA (that
1729 points to int. Worker for cgraph_for_node_and_aliases. */
1732 sum_callers (struct cgraph_node
*node
, void *data
)
1734 struct cgraph_edge
*e
;
1735 int *num_calls
= (int *)data
;
1737 for (e
= node
->callers
; e
; e
= e
->next_caller
)
1742 /* We use greedy algorithm for inlining of small functions:
1743 All inline candidates are put into prioritized heap ordered in
1746 The inlining of small functions is bounded by unit growth parameters. */
1749 inline_small_functions (void)
1751 struct cgraph_node
*node
;
1752 struct cgraph_edge
*edge
;
1753 edge_heap_t
edge_heap (sreal::min ());
1754 auto_bitmap updated_nodes
;
1755 int min_size
, max_size
;
1756 auto_vec
<cgraph_edge
*> new_indirect_edges
;
1757 int initial_size
= 0;
1758 struct cgraph_node
**order
= XCNEWVEC (cgraph_node
*, symtab
->cgraph_count
);
1759 struct cgraph_edge_hook_list
*edge_removal_hook_holder
;
1760 new_indirect_edges
.create (8);
1762 edge_removal_hook_holder
1763 = symtab
->add_edge_removal_hook (&heap_edge_removal_hook
, &edge_heap
);
1765 /* Compute overall unit size and other global parameters used by badness
1768 max_count
= profile_count::uninitialized ();
1769 ipa_reduced_postorder (order
, true, true, NULL
);
1772 FOR_EACH_DEFINED_FUNCTION (node
)
1773 if (!node
->global
.inlined_to
)
1775 if (!node
->alias
&& node
->analyzed
1776 && (node
->has_gimple_body_p () || node
->thunk
.thunk_p
)
1777 && opt_for_fn (node
->decl
, optimize
))
1779 struct ipa_fn_summary
*info
= ipa_fn_summaries
->get (node
);
1780 struct ipa_dfs_info
*dfs
= (struct ipa_dfs_info
*) node
->aux
;
1782 /* Do not account external functions, they will be optimized out
1783 if not inlined. Also only count the non-cold portion of program. */
1784 if (inline_account_function_p (node
))
1785 initial_size
+= info
->size
;
1786 info
->growth
= estimate_growth (node
);
1789 node
->call_for_symbol_and_aliases (sum_callers
, &num_calls
,
1792 info
->single_caller
= true;
1793 if (dfs
&& dfs
->next_cycle
)
1795 struct cgraph_node
*n2
;
1796 int id
= dfs
->scc_no
+ 1;
1798 n2
= ((struct ipa_dfs_info
*) n2
->aux
)->next_cycle
)
1799 if (opt_for_fn (n2
->decl
, optimize
))
1801 ipa_fn_summary
*info2
= ipa_fn_summaries
->get (n2
);
1809 for (edge
= node
->callers
; edge
; edge
= edge
->next_caller
)
1810 max_count
= max_count
.max (edge
->count
.ipa ());
1812 ipa_free_postorder_info ();
1814 = new call_summary
<edge_growth_cache_entry
*> (symtab
, false);
1818 "\nDeciding on inlining of small functions. Starting with size %i.\n",
1821 overall_size
= initial_size
;
1822 max_size
= compute_max_insns (overall_size
);
1823 min_size
= overall_size
;
1825 /* Populate the heap with all edges we might inline. */
1827 FOR_EACH_DEFINED_FUNCTION (node
)
1829 bool update
= false;
1830 struct cgraph_edge
*next
= NULL
;
1831 bool has_speculative
= false;
1833 if (!opt_for_fn (node
->decl
, optimize
))
1837 fprintf (dump_file
, "Enqueueing calls in %s.\n", node
->dump_name ());
1839 for (edge
= node
->callees
; edge
; edge
= next
)
1841 next
= edge
->next_callee
;
1842 if (edge
->inline_failed
1844 && can_inline_edge_p (edge
, true)
1845 && want_inline_small_function_p (edge
, true)
1846 && can_inline_edge_by_limits_p (edge
, true)
1847 && edge
->inline_failed
)
1849 gcc_assert (!edge
->aux
);
1850 update_edge_key (&edge_heap
, edge
);
1852 if (edge
->speculative
)
1853 has_speculative
= true;
1855 if (has_speculative
)
1856 for (edge
= node
->callees
; edge
; edge
= next
)
1857 if (edge
->speculative
&& !speculation_useful_p (edge
,
1860 edge
->resolve_speculation ();
1865 struct cgraph_node
*where
= node
->global
.inlined_to
1866 ? node
->global
.inlined_to
: node
;
1867 ipa_update_overall_fn_summary (where
);
1868 reset_edge_caches (where
);
1869 update_caller_keys (&edge_heap
, where
,
1870 updated_nodes
, NULL
);
1871 update_callee_keys (&edge_heap
, where
,
1873 bitmap_clear (updated_nodes
);
1877 gcc_assert (in_lto_p
1879 || (profile_info
&& flag_branch_probabilities
));
1881 while (!edge_heap
.empty ())
1883 int old_size
= overall_size
;
1884 struct cgraph_node
*where
, *callee
;
1885 sreal badness
= edge_heap
.min_key ();
1886 sreal current_badness
;
1889 edge
= edge_heap
.extract_min ();
1890 gcc_assert (edge
->aux
);
1892 if (!edge
->inline_failed
|| !edge
->callee
->analyzed
)
1896 /* Be sure that caches are maintained consistent.
1897 This check is affected by scaling roundoff errors when compiling for
1898 IPA this we skip it in that case. */
1899 if (!edge
->callee
->count
.ipa_p ()
1900 && (!max_count
.initialized_p () || !max_count
.nonzero_p ()))
1902 sreal cached_badness
= edge_badness (edge
, false);
1904 int old_size_est
= estimate_edge_size (edge
);
1905 sreal old_time_est
= estimate_edge_time (edge
);
1906 int old_hints_est
= estimate_edge_hints (edge
);
1908 if (edge_growth_cache
!= NULL
)
1909 edge_growth_cache
->remove (edge
);
1910 gcc_assert (old_size_est
== estimate_edge_size (edge
));
1911 gcc_assert (old_time_est
== estimate_edge_time (edge
));
1914 gcc_assert (old_hints_est == estimate_edge_hints (edge));
1916 fails with profile feedback because some hints depends on
1917 maybe_hot_edge_p predicate and because callee gets inlined to other
1918 calls, the edge may become cold.
1919 This ought to be fixed by computing relative probabilities
1920 for given invocation but that will be better done once whole
1921 code is converted to sreals. Disable for now and revert to "wrong"
1922 value so enable/disable checking paths agree. */
1923 edge_growth_cache
->get (edge
)->hints
= old_hints_est
+ 1;
1925 /* When updating the edge costs, we only decrease badness in the keys.
1926 Increases of badness are handled lazilly; when we see key with out
1927 of date value on it, we re-insert it now. */
1928 current_badness
= edge_badness (edge
, false);
1929 gcc_assert (cached_badness
== current_badness
);
1930 gcc_assert (current_badness
>= badness
);
1933 current_badness
= edge_badness (edge
, false);
1935 current_badness
= edge_badness (edge
, false);
1937 if (current_badness
!= badness
)
1939 if (edge_heap
.min () && current_badness
> edge_heap
.min_key ())
1941 edge
->aux
= edge_heap
.insert (current_badness
, edge
);
1945 badness
= current_badness
;
1948 if (!can_inline_edge_p (edge
, true)
1949 || !can_inline_edge_by_limits_p (edge
, true))
1951 resolve_noninline_speculation (&edge_heap
, edge
);
1955 callee
= edge
->callee
->ultimate_alias_target ();
1956 growth
= estimate_edge_growth (edge
);
1960 "\nConsidering %s with %i size\n",
1961 callee
->dump_name (),
1962 ipa_fn_summaries
->get (callee
)->size
);
1964 " to be inlined into %s in %s:%i\n"
1965 " Estimated badness is %f, frequency %.2f.\n",
1966 edge
->caller
->dump_name (),
1968 && (LOCATION_LOCUS (gimple_location ((const gimple
*)
1970 > BUILTINS_LOCATION
)
1971 ? gimple_filename ((const gimple
*) edge
->call_stmt
)
1974 ? gimple_lineno ((const gimple
*) edge
->call_stmt
)
1976 badness
.to_double (),
1977 edge
->sreal_frequency ().to_double ());
1978 if (edge
->count
.ipa ().initialized_p ())
1980 fprintf (dump_file
, " Called ");
1981 edge
->count
.ipa ().dump (dump_file
);
1982 fprintf (dump_file
, " times\n");
1984 if (dump_flags
& TDF_DETAILS
)
1985 edge_badness (edge
, true);
1988 if (overall_size
+ growth
> max_size
1989 && !DECL_DISREGARD_INLINE_LIMITS (callee
->decl
))
1991 edge
->inline_failed
= CIF_INLINE_UNIT_GROWTH_LIMIT
;
1992 report_inline_failed_reason (edge
);
1993 resolve_noninline_speculation (&edge_heap
, edge
);
1997 if (!want_inline_small_function_p (edge
, true))
1999 resolve_noninline_speculation (&edge_heap
, edge
);
2003 /* Heuristics for inlining small functions work poorly for
2004 recursive calls where we do effects similar to loop unrolling.
2005 When inlining such edge seems profitable, leave decision on
2006 specific inliner. */
2007 if (edge
->recursive_p ())
2009 where
= edge
->caller
;
2010 if (where
->global
.inlined_to
)
2011 where
= where
->global
.inlined_to
;
2012 if (!recursive_inlining (edge
,
2013 opt_for_fn (edge
->caller
->decl
,
2014 flag_indirect_inlining
)
2015 ? &new_indirect_edges
: NULL
))
2017 edge
->inline_failed
= CIF_RECURSIVE_INLINING
;
2018 resolve_noninline_speculation (&edge_heap
, edge
);
2021 reset_edge_caches (where
);
2022 /* Recursive inliner inlines all recursive calls of the function
2023 at once. Consequently we need to update all callee keys. */
2024 if (opt_for_fn (edge
->caller
->decl
, flag_indirect_inlining
))
2025 add_new_edges_to_heap (&edge_heap
, new_indirect_edges
);
2026 update_callee_keys (&edge_heap
, where
, updated_nodes
);
2027 bitmap_clear (updated_nodes
);
2031 struct cgraph_node
*outer_node
= NULL
;
2034 /* Consider the case where self recursive function A is inlined
2035 into B. This is desired optimization in some cases, since it
2036 leads to effect similar of loop peeling and we might completely
2037 optimize out the recursive call. However we must be extra
2040 where
= edge
->caller
;
2041 while (where
->global
.inlined_to
)
2043 if (where
->decl
== callee
->decl
)
2044 outer_node
= where
, depth
++;
2045 where
= where
->callers
->caller
;
2048 && !want_inline_self_recursive_call_p (edge
, outer_node
,
2052 = (DECL_DISREGARD_INLINE_LIMITS (edge
->callee
->decl
)
2053 ? CIF_RECURSIVE_INLINING
: CIF_UNSPECIFIED
);
2054 resolve_noninline_speculation (&edge_heap
, edge
);
2057 else if (depth
&& dump_file
)
2058 fprintf (dump_file
, " Peeling recursion with depth %i\n", depth
);
2060 gcc_checking_assert (!callee
->global
.inlined_to
);
2061 inline_call (edge
, true, &new_indirect_edges
, &overall_size
, true);
2062 add_new_edges_to_heap (&edge_heap
, new_indirect_edges
);
2064 reset_edge_caches (edge
->callee
);
2066 update_callee_keys (&edge_heap
, where
, updated_nodes
);
2068 where
= edge
->caller
;
2069 if (where
->global
.inlined_to
)
2070 where
= where
->global
.inlined_to
;
2072 /* Our profitability metric can depend on local properties
2073 such as number of inlinable calls and size of the function body.
2074 After inlining these properties might change for the function we
2075 inlined into (since it's body size changed) and for the functions
2076 called by function we inlined (since number of it inlinable callers
2078 update_caller_keys (&edge_heap
, where
, updated_nodes
, NULL
);
2079 /* Offline copy count has possibly changed, recompute if profile is
2081 struct cgraph_node
*n
= cgraph_node::get (edge
->callee
->decl
);
2082 if (n
!= edge
->callee
&& n
->analyzed
&& n
->count
.ipa ().initialized_p ())
2083 update_callee_keys (&edge_heap
, n
, updated_nodes
);
2084 bitmap_clear (updated_nodes
);
2086 if (dump_enabled_p ())
2088 ipa_fn_summary
*s
= ipa_fn_summaries
->get (edge
->caller
);
2090 /* dump_printf can't handle %+i. */
2091 char buf_net_change
[100];
2092 snprintf (buf_net_change
, sizeof buf_net_change
, "%+i",
2093 overall_size
- old_size
);
2095 dump_printf_loc (MSG_OPTIMIZED_LOCATIONS
, edge
->call_stmt
,
2096 " Inlined %C into %C which now has time %f and "
2097 "size %i, net change of %s.\n",
2098 edge
->callee
, edge
->caller
,
2099 s
->time
.to_double (), s
->size
, buf_net_change
);
2101 if (min_size
> overall_size
)
2103 min_size
= overall_size
;
2104 max_size
= compute_max_insns (min_size
);
2107 fprintf (dump_file
, "New minimal size reached: %i\n", min_size
);
2111 free_growth_caches ();
2112 if (dump_enabled_p ())
2113 dump_printf (MSG_NOTE
,
2114 "Unit growth for small function inlining: %i->%i (%i%%)\n",
2115 initial_size
, overall_size
,
2116 initial_size
? overall_size
* 100 / (initial_size
) - 100: 0);
2117 symtab
->remove_edge_removal_hook (edge_removal_hook_holder
);
2120 /* Flatten NODE. Performed both during early inlining and
2121 at IPA inlining time. */
2124 flatten_function (struct cgraph_node
*node
, bool early
)
2126 struct cgraph_edge
*e
;
2128 /* We shouldn't be called recursively when we are being processed. */
2129 gcc_assert (node
->aux
== NULL
);
2131 node
->aux
= (void *) node
;
2133 for (e
= node
->callees
; e
; e
= e
->next_callee
)
2135 struct cgraph_node
*orig_callee
;
2136 struct cgraph_node
*callee
= e
->callee
->ultimate_alias_target ();
2138 /* We've hit cycle? It is time to give up. */
2141 if (dump_enabled_p ())
2142 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, e
->call_stmt
,
2143 "Not inlining %C into %C to avoid cycle.\n",
2145 if (cgraph_inline_failed_type (e
->inline_failed
) != CIF_FINAL_ERROR
)
2146 e
->inline_failed
= CIF_RECURSIVE_INLINING
;
2150 /* When the edge is already inlined, we just need to recurse into
2151 it in order to fully flatten the leaves. */
2152 if (!e
->inline_failed
)
2154 flatten_function (callee
, early
);
2158 /* Flatten attribute needs to be processed during late inlining. For
2159 extra code quality we however do flattening during early optimization,
2162 ? !can_inline_edge_p (e
, true)
2163 && !can_inline_edge_by_limits_p (e
, true)
2164 : !can_early_inline_edge_p (e
))
2167 if (e
->recursive_p ())
2169 if (dump_enabled_p ())
2170 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, e
->call_stmt
,
2171 "Not inlining: recursive call.\n");
2175 if (gimple_in_ssa_p (DECL_STRUCT_FUNCTION (node
->decl
))
2176 != gimple_in_ssa_p (DECL_STRUCT_FUNCTION (callee
->decl
)))
2178 if (dump_enabled_p ())
2179 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, e
->call_stmt
,
2180 "Not inlining: SSA form does not match.\n");
2184 /* Inline the edge and flatten the inline clone. Avoid
2185 recursing through the original node if the node was cloned. */
2186 if (dump_enabled_p ())
2187 dump_printf_loc (MSG_OPTIMIZED_LOCATIONS
, e
->call_stmt
,
2188 " Inlining %C into %C.\n",
2190 orig_callee
= callee
;
2191 inline_call (e
, true, NULL
, NULL
, false);
2192 if (e
->callee
!= orig_callee
)
2193 orig_callee
->aux
= (void *) node
;
2194 flatten_function (e
->callee
, early
);
2195 if (e
->callee
!= orig_callee
)
2196 orig_callee
->aux
= NULL
;
2200 if (!node
->global
.inlined_to
)
2201 ipa_update_overall_fn_summary (node
);
2204 /* Inline NODE to all callers. Worker for cgraph_for_node_and_aliases.
2205 DATA points to number of calls originally found so we avoid infinite
2209 inline_to_all_callers_1 (struct cgraph_node
*node
, void *data
,
2210 hash_set
<cgraph_node
*> *callers
)
2212 int *num_calls
= (int *)data
;
2213 bool callee_removed
= false;
2215 while (node
->callers
&& !node
->global
.inlined_to
)
2217 struct cgraph_node
*caller
= node
->callers
->caller
;
2219 if (!can_inline_edge_p (node
->callers
, true)
2220 || !can_inline_edge_by_limits_p (node
->callers
, true)
2221 || node
->callers
->recursive_p ())
2224 fprintf (dump_file
, "Uninlinable call found; giving up.\n");
2231 cgraph_node
*ultimate
= node
->ultimate_alias_target ();
2233 "\nInlining %s size %i.\n",
2235 ipa_fn_summaries
->get (ultimate
)->size
);
2237 " Called once from %s %i insns.\n",
2238 node
->callers
->caller
->name (),
2239 ipa_fn_summaries
->get (node
->callers
->caller
)->size
);
2242 /* Remember which callers we inlined to, delaying updating the
2244 callers
->add (node
->callers
->caller
);
2245 inline_call (node
->callers
, true, NULL
, NULL
, false, &callee_removed
);
2248 " Inlined into %s which now has %i size\n",
2250 ipa_fn_summaries
->get (caller
)->size
);
2251 if (!(*num_calls
)--)
2254 fprintf (dump_file
, "New calls found; giving up.\n");
2255 return callee_removed
;
2263 /* Wrapper around inline_to_all_callers_1 doing delayed overall summary
2267 inline_to_all_callers (struct cgraph_node
*node
, void *data
)
2269 hash_set
<cgraph_node
*> callers
;
2270 bool res
= inline_to_all_callers_1 (node
, data
, &callers
);
2271 /* Perform the delayed update of the overall summary of all callers
2272 processed. This avoids quadratic behavior in the cases where
2273 we have a lot of calls to the same function. */
2274 for (hash_set
<cgraph_node
*>::iterator i
= callers
.begin ();
2275 i
!= callers
.end (); ++i
)
2276 ipa_update_overall_fn_summary (*i
);
2280 /* Output overall time estimate. */
2282 dump_overall_stats (void)
2284 sreal sum_weighted
= 0, sum
= 0;
2285 struct cgraph_node
*node
;
2287 FOR_EACH_DEFINED_FUNCTION (node
)
2288 if (!node
->global
.inlined_to
2291 ipa_fn_summary
*s
= ipa_fn_summaries
->get (node
);
2295 if (node
->count
.ipa ().initialized_p ())
2296 sum_weighted
+= s
->time
* node
->count
.ipa ().to_gcov_type ();
2299 fprintf (dump_file
, "Overall time estimate: "
2300 "%f weighted by profile: "
2301 "%f\n", sum
.to_double (), sum_weighted
.to_double ());
2304 /* Output some useful stats about inlining. */
2307 dump_inline_stats (void)
2309 int64_t inlined_cnt
= 0, inlined_indir_cnt
= 0;
2310 int64_t inlined_virt_cnt
= 0, inlined_virt_indir_cnt
= 0;
2311 int64_t noninlined_cnt
= 0, noninlined_indir_cnt
= 0;
2312 int64_t noninlined_virt_cnt
= 0, noninlined_virt_indir_cnt
= 0;
2313 int64_t inlined_speculative
= 0, inlined_speculative_ply
= 0;
2314 int64_t indirect_poly_cnt
= 0, indirect_cnt
= 0;
2315 int64_t reason
[CIF_N_REASONS
][2];
2316 sreal reason_freq
[CIF_N_REASONS
];
2318 struct cgraph_node
*node
;
2320 memset (reason
, 0, sizeof (reason
));
2321 for (i
=0; i
< CIF_N_REASONS
; i
++)
2323 FOR_EACH_DEFINED_FUNCTION (node
)
2325 struct cgraph_edge
*e
;
2326 for (e
= node
->callees
; e
; e
= e
->next_callee
)
2328 if (e
->inline_failed
)
2330 if (e
->count
.ipa ().initialized_p ())
2331 reason
[(int) e
->inline_failed
][0] += e
->count
.ipa ().to_gcov_type ();
2332 reason_freq
[(int) e
->inline_failed
] += e
->sreal_frequency ();
2333 reason
[(int) e
->inline_failed
][1] ++;
2334 if (DECL_VIRTUAL_P (e
->callee
->decl
)
2335 && e
->count
.ipa ().initialized_p ())
2337 if (e
->indirect_inlining_edge
)
2338 noninlined_virt_indir_cnt
+= e
->count
.ipa ().to_gcov_type ();
2340 noninlined_virt_cnt
+= e
->count
.ipa ().to_gcov_type ();
2342 else if (e
->count
.ipa ().initialized_p ())
2344 if (e
->indirect_inlining_edge
)
2345 noninlined_indir_cnt
+= e
->count
.ipa ().to_gcov_type ();
2347 noninlined_cnt
+= e
->count
.ipa ().to_gcov_type ();
2350 else if (e
->count
.ipa ().initialized_p ())
2354 if (DECL_VIRTUAL_P (e
->callee
->decl
))
2355 inlined_speculative_ply
+= e
->count
.ipa ().to_gcov_type ();
2357 inlined_speculative
+= e
->count
.ipa ().to_gcov_type ();
2359 else if (DECL_VIRTUAL_P (e
->callee
->decl
))
2361 if (e
->indirect_inlining_edge
)
2362 inlined_virt_indir_cnt
+= e
->count
.ipa ().to_gcov_type ();
2364 inlined_virt_cnt
+= e
->count
.ipa ().to_gcov_type ();
2368 if (e
->indirect_inlining_edge
)
2369 inlined_indir_cnt
+= e
->count
.ipa ().to_gcov_type ();
2371 inlined_cnt
+= e
->count
.ipa ().to_gcov_type ();
2375 for (e
= node
->indirect_calls
; e
; e
= e
->next_callee
)
2376 if (e
->indirect_info
->polymorphic
2377 & e
->count
.ipa ().initialized_p ())
2378 indirect_poly_cnt
+= e
->count
.ipa ().to_gcov_type ();
2379 else if (e
->count
.ipa ().initialized_p ())
2380 indirect_cnt
+= e
->count
.ipa ().to_gcov_type ();
2382 if (max_count
.initialized_p ())
2385 "Inlined %" PRId64
" + speculative "
2386 "%" PRId64
" + speculative polymorphic "
2387 "%" PRId64
" + previously indirect "
2388 "%" PRId64
" + virtual "
2389 "%" PRId64
" + virtual and previously indirect "
2390 "%" PRId64
"\n" "Not inlined "
2391 "%" PRId64
" + previously indirect "
2392 "%" PRId64
" + virtual "
2393 "%" PRId64
" + virtual and previously indirect "
2394 "%" PRId64
" + stil indirect "
2395 "%" PRId64
" + still indirect polymorphic "
2396 "%" PRId64
"\n", inlined_cnt
,
2397 inlined_speculative
, inlined_speculative_ply
,
2398 inlined_indir_cnt
, inlined_virt_cnt
, inlined_virt_indir_cnt
,
2399 noninlined_cnt
, noninlined_indir_cnt
, noninlined_virt_cnt
,
2400 noninlined_virt_indir_cnt
, indirect_cnt
, indirect_poly_cnt
);
2401 fprintf (dump_file
, "Removed speculations ");
2402 spec_rem
.dump (dump_file
);
2403 fprintf (dump_file
, "\n");
2405 dump_overall_stats ();
2406 fprintf (dump_file
, "\nWhy inlining failed?\n");
2407 for (i
= 0; i
< CIF_N_REASONS
; i
++)
2409 fprintf (dump_file
, "%-50s: %8i calls, %8f freq, %" PRId64
" count\n",
2410 cgraph_inline_failed_string ((cgraph_inline_failed_t
) i
),
2411 (int) reason
[i
][1], reason_freq
[i
].to_double (), reason
[i
][0]);
2414 /* Called when node is removed. */
2417 flatten_remove_node_hook (struct cgraph_node
*node
, void *data
)
2419 if (lookup_attribute ("flatten", DECL_ATTRIBUTES (node
->decl
)) == NULL
)
2422 hash_set
<struct cgraph_node
*> *removed
2423 = (hash_set
<struct cgraph_node
*> *) data
;
2424 removed
->add (node
);
2427 /* Decide on the inlining. We do so in the topological order to avoid
2428 expenses on updating data structures. */
2433 struct cgraph_node
*node
;
2435 struct cgraph_node
**order
;
2438 bool remove_functions
= false;
2440 order
= XCNEWVEC (struct cgraph_node
*, symtab
->cgraph_count
);
2443 ipa_dump_fn_summaries (dump_file
);
2445 nnodes
= ipa_reverse_postorder (order
);
2446 spec_rem
= profile_count::zero ();
2448 FOR_EACH_FUNCTION (node
)
2452 /* Recompute the default reasons for inlining because they may have
2453 changed during merging. */
2456 for (cgraph_edge
*e
= node
->callees
; e
; e
= e
->next_callee
)
2458 gcc_assert (e
->inline_failed
);
2459 initialize_inline_failed (e
);
2461 for (cgraph_edge
*e
= node
->indirect_calls
; e
; e
= e
->next_callee
)
2462 initialize_inline_failed (e
);
2467 fprintf (dump_file
, "\nFlattening functions:\n");
2469 /* First shrink order array, so that it only contains nodes with
2470 flatten attribute. */
2471 for (i
= nnodes
- 1, j
= i
; i
>= 0; i
--)
2474 if (lookup_attribute ("flatten",
2475 DECL_ATTRIBUTES (node
->decl
)) != NULL
)
2476 order
[j
--] = order
[i
];
2479 /* After the above loop, order[j + 1] ... order[nnodes - 1] contain
2480 nodes with flatten attribute. If there is more than one such
2481 node, we need to register a node removal hook, as flatten_function
2482 could remove other nodes with flatten attribute. See PR82801. */
2483 struct cgraph_node_hook_list
*node_removal_hook_holder
= NULL
;
2484 hash_set
<struct cgraph_node
*> *flatten_removed_nodes
= NULL
;
2487 flatten_removed_nodes
= new hash_set
<struct cgraph_node
*>;
2488 node_removal_hook_holder
2489 = symtab
->add_cgraph_removal_hook (&flatten_remove_node_hook
,
2490 flatten_removed_nodes
);
2493 /* In the first pass handle functions to be flattened. Do this with
2494 a priority so none of our later choices will make this impossible. */
2495 for (i
= nnodes
- 1; i
> j
; i
--)
2498 if (flatten_removed_nodes
2499 && flatten_removed_nodes
->contains (node
))
2502 /* Handle nodes to be flattened.
2503 Ideally when processing callees we stop inlining at the
2504 entry of cycles, possibly cloning that entry point and
2505 try to flatten itself turning it into a self-recursive
2508 fprintf (dump_file
, "Flattening %s\n", node
->name ());
2509 flatten_function (node
, false);
2514 symtab
->remove_cgraph_removal_hook (node_removal_hook_holder
);
2515 delete flatten_removed_nodes
;
2520 dump_overall_stats ();
2522 inline_small_functions ();
2524 gcc_assert (symtab
->state
== IPA_SSA
);
2525 symtab
->state
= IPA_SSA_AFTER_INLINING
;
2526 /* Do first after-inlining removal. We want to remove all "stale" extern
2527 inline functions and virtual functions so we really know what is called
2529 symtab
->remove_unreachable_nodes (dump_file
);
2531 /* Inline functions with a property that after inlining into all callers the
2532 code size will shrink because the out-of-line copy is eliminated.
2533 We do this regardless on the callee size as long as function growth limits
2537 "\nDeciding on functions to be inlined into all callers and "
2538 "removing useless speculations:\n");
2540 /* Inlining one function called once has good chance of preventing
2541 inlining other function into the same callee. Ideally we should
2542 work in priority order, but probably inlining hot functions first
2543 is good cut without the extra pain of maintaining the queue.
2545 ??? this is not really fitting the bill perfectly: inlining function
2546 into callee often leads to better optimization of callee due to
2547 increased context for optimization.
2548 For example if main() function calls a function that outputs help
2549 and then function that does the main optmization, we should inline
2550 the second with priority even if both calls are cold by themselves.
2552 We probably want to implement new predicate replacing our use of
2553 maybe_hot_edge interpreted as maybe_hot_edge || callee is known
2555 for (cold
= 0; cold
<= 1; cold
++)
2557 FOR_EACH_DEFINED_FUNCTION (node
)
2559 struct cgraph_edge
*edge
, *next
;
2562 if (!opt_for_fn (node
->decl
, optimize
)
2563 || !opt_for_fn (node
->decl
, flag_inline_functions_called_once
))
2566 for (edge
= node
->callees
; edge
; edge
= next
)
2568 next
= edge
->next_callee
;
2569 if (edge
->speculative
&& !speculation_useful_p (edge
, false))
2571 if (edge
->count
.ipa ().initialized_p ())
2572 spec_rem
+= edge
->count
.ipa ();
2573 edge
->resolve_speculation ();
2575 remove_functions
= true;
2580 struct cgraph_node
*where
= node
->global
.inlined_to
2581 ? node
->global
.inlined_to
: node
;
2582 reset_edge_caches (where
);
2583 ipa_update_overall_fn_summary (where
);
2585 if (want_inline_function_to_all_callers_p (node
, cold
))
2588 node
->call_for_symbol_and_aliases (sum_callers
, &num_calls
,
2590 while (node
->call_for_symbol_and_aliases
2591 (inline_to_all_callers
, &num_calls
, true))
2593 remove_functions
= true;
2598 /* Free ipa-prop structures if they are no longer needed. */
2599 ipa_free_all_structures_after_iinln ();
2601 if (dump_enabled_p ())
2602 dump_printf (MSG_NOTE
,
2603 "\nInlined %i calls, eliminated %i functions\n\n",
2604 ncalls_inlined
, nfunctions_inlined
);
2606 dump_inline_stats ();
2609 ipa_dump_fn_summaries (dump_file
);
2610 return remove_functions
? TODO_remove_functions
: 0;
2613 /* Inline always-inline function calls in NODE. */
2616 inline_always_inline_functions (struct cgraph_node
*node
)
2618 struct cgraph_edge
*e
;
2619 bool inlined
= false;
2621 for (e
= node
->callees
; e
; e
= e
->next_callee
)
2623 struct cgraph_node
*callee
= e
->callee
->ultimate_alias_target ();
2624 if (!DECL_DISREGARD_INLINE_LIMITS (callee
->decl
))
2627 if (e
->recursive_p ())
2629 if (dump_enabled_p ())
2630 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, e
->call_stmt
,
2631 " Not inlining recursive call to %C.\n",
2633 e
->inline_failed
= CIF_RECURSIVE_INLINING
;
2637 if (!can_early_inline_edge_p (e
))
2639 /* Set inlined to true if the callee is marked "always_inline" but
2640 is not inlinable. This will allow flagging an error later in
2641 expand_call_inline in tree-inline.c. */
2642 if (lookup_attribute ("always_inline",
2643 DECL_ATTRIBUTES (callee
->decl
)) != NULL
)
2648 if (dump_enabled_p ())
2649 dump_printf_loc (MSG_OPTIMIZED_LOCATIONS
, e
->call_stmt
,
2650 " Inlining %C into %C (always_inline).\n",
2651 e
->callee
, e
->caller
);
2652 inline_call (e
, true, NULL
, NULL
, false);
2656 ipa_update_overall_fn_summary (node
);
2661 /* Decide on the inlining. We do so in the topological order to avoid
2662 expenses on updating data structures. */
2665 early_inline_small_functions (struct cgraph_node
*node
)
2667 struct cgraph_edge
*e
;
2668 bool inlined
= false;
2670 for (e
= node
->callees
; e
; e
= e
->next_callee
)
2672 struct cgraph_node
*callee
= e
->callee
->ultimate_alias_target ();
2674 /* We can enounter not-yet-analyzed function during
2675 early inlining on callgraphs with strongly
2676 connected components. */
2677 ipa_fn_summary
*s
= ipa_fn_summaries
->get (callee
);
2678 if (s
== NULL
|| !s
->inlinable
|| !e
->inline_failed
)
2681 /* Do not consider functions not declared inline. */
2682 if (!DECL_DECLARED_INLINE_P (callee
->decl
)
2683 && !opt_for_fn (node
->decl
, flag_inline_small_functions
)
2684 && !opt_for_fn (node
->decl
, flag_inline_functions
))
2687 if (dump_enabled_p ())
2688 dump_printf_loc (MSG_NOTE
, e
->call_stmt
,
2689 "Considering inline candidate %C.\n",
2692 if (!can_early_inline_edge_p (e
))
2695 if (e
->recursive_p ())
2697 if (dump_enabled_p ())
2698 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, e
->call_stmt
,
2699 " Not inlining: recursive call.\n");
2703 if (!want_early_inline_function_p (e
))
2706 if (dump_enabled_p ())
2707 dump_printf_loc (MSG_OPTIMIZED_LOCATIONS
, e
->call_stmt
,
2708 " Inlining %C into %C.\n",
2710 inline_call (e
, true, NULL
, NULL
, false);
2715 ipa_update_overall_fn_summary (node
);
2721 early_inliner (function
*fun
)
2723 struct cgraph_node
*node
= cgraph_node::get (current_function_decl
);
2724 struct cgraph_edge
*edge
;
2725 unsigned int todo
= 0;
2727 bool inlined
= false;
2732 /* Do nothing if datastructures for ipa-inliner are already computed. This
2733 happens when some pass decides to construct new function and
2734 cgraph_add_new_function calls lowering passes and early optimization on
2735 it. This may confuse ourself when early inliner decide to inline call to
2736 function clone, because function clones don't have parameter list in
2737 ipa-prop matching their signature. */
2738 if (ipa_node_params_sum
)
2743 node
->remove_all_references ();
2745 /* Even when not optimizing or not inlining inline always-inline
2747 inlined
= inline_always_inline_functions (node
);
2751 || !flag_early_inlining
2752 /* Never inline regular functions into always-inline functions
2753 during incremental inlining. This sucks as functions calling
2754 always inline functions will get less optimized, but at the
2755 same time inlining of functions calling always inline
2756 function into an always inline function might introduce
2757 cycles of edges to be always inlined in the callgraph.
2759 We might want to be smarter and just avoid this type of inlining. */
2760 || (DECL_DISREGARD_INLINE_LIMITS (node
->decl
)
2761 && lookup_attribute ("always_inline",
2762 DECL_ATTRIBUTES (node
->decl
))))
2764 else if (lookup_attribute ("flatten",
2765 DECL_ATTRIBUTES (node
->decl
)) != NULL
)
2767 /* When the function is marked to be flattened, recursively inline
2769 if (dump_enabled_p ())
2770 dump_printf (MSG_OPTIMIZED_LOCATIONS
,
2771 "Flattening %C\n", node
);
2772 flatten_function (node
, true);
2777 /* If some always_inline functions was inlined, apply the changes.
2778 This way we will not account always inline into growth limits and
2779 moreover we will inline calls from always inlines that we skipped
2780 previously because of conditional above. */
2783 timevar_push (TV_INTEGRATION
);
2784 todo
|= optimize_inline_calls (current_function_decl
);
2785 /* optimize_inline_calls call above might have introduced new
2786 statements that don't have inline parameters computed. */
2787 for (edge
= node
->callees
; edge
; edge
= edge
->next_callee
)
2789 /* We can enounter not-yet-analyzed function during
2790 early inlining on callgraphs with strongly
2791 connected components. */
2792 ipa_call_summary
*es
= ipa_call_summaries
->get_create (edge
);
2794 = estimate_num_insns (edge
->call_stmt
, &eni_size_weights
);
2796 = estimate_num_insns (edge
->call_stmt
, &eni_time_weights
);
2798 ipa_update_overall_fn_summary (node
);
2800 timevar_pop (TV_INTEGRATION
);
2802 /* We iterate incremental inlining to get trivial cases of indirect
2804 while (iterations
< PARAM_VALUE (PARAM_EARLY_INLINER_MAX_ITERATIONS
)
2805 && early_inline_small_functions (node
))
2807 timevar_push (TV_INTEGRATION
);
2808 todo
|= optimize_inline_calls (current_function_decl
);
2810 /* Technically we ought to recompute inline parameters so the new
2811 iteration of early inliner works as expected. We however have
2812 values approximately right and thus we only need to update edge
2813 info that might be cleared out for newly discovered edges. */
2814 for (edge
= node
->callees
; edge
; edge
= edge
->next_callee
)
2816 /* We have no summary for new bound store calls yet. */
2817 ipa_call_summary
*es
= ipa_call_summaries
->get_create (edge
);
2819 = estimate_num_insns (edge
->call_stmt
, &eni_size_weights
);
2821 = estimate_num_insns (edge
->call_stmt
, &eni_time_weights
);
2823 if (edge
->callee
->decl
2824 && !gimple_check_call_matching_types (
2825 edge
->call_stmt
, edge
->callee
->decl
, false))
2827 edge
->inline_failed
= CIF_MISMATCHED_ARGUMENTS
;
2828 edge
->call_stmt_cannot_inline_p
= true;
2831 if (iterations
< PARAM_VALUE (PARAM_EARLY_INLINER_MAX_ITERATIONS
) - 1)
2832 ipa_update_overall_fn_summary (node
);
2833 timevar_pop (TV_INTEGRATION
);
2838 fprintf (dump_file
, "Iterations: %i\n", iterations
);
2843 timevar_push (TV_INTEGRATION
);
2844 todo
|= optimize_inline_calls (current_function_decl
);
2845 timevar_pop (TV_INTEGRATION
);
2848 fun
->always_inline_functions_inlined
= true;
2853 /* Do inlining of small functions. Doing so early helps profiling and other
2854 passes to be somewhat more effective and avoids some code duplication in
2855 later real inlining pass for testcases with very many function calls. */
2859 const pass_data pass_data_early_inline
=
2861 GIMPLE_PASS
, /* type */
2862 "einline", /* name */
2863 OPTGROUP_INLINE
, /* optinfo_flags */
2864 TV_EARLY_INLINING
, /* tv_id */
2865 PROP_ssa
, /* properties_required */
2866 0, /* properties_provided */
2867 0, /* properties_destroyed */
2868 0, /* todo_flags_start */
2869 0, /* todo_flags_finish */
2872 class pass_early_inline
: public gimple_opt_pass
2875 pass_early_inline (gcc::context
*ctxt
)
2876 : gimple_opt_pass (pass_data_early_inline
, ctxt
)
2879 /* opt_pass methods: */
2880 virtual unsigned int execute (function
*);
2882 }; // class pass_early_inline
2885 pass_early_inline::execute (function
*fun
)
2887 return early_inliner (fun
);
2893 make_pass_early_inline (gcc::context
*ctxt
)
2895 return new pass_early_inline (ctxt
);
2900 const pass_data pass_data_ipa_inline
=
2902 IPA_PASS
, /* type */
2903 "inline", /* name */
2904 OPTGROUP_INLINE
, /* optinfo_flags */
2905 TV_IPA_INLINING
, /* tv_id */
2906 0, /* properties_required */
2907 0, /* properties_provided */
2908 0, /* properties_destroyed */
2909 0, /* todo_flags_start */
2910 ( TODO_dump_symtab
), /* todo_flags_finish */
2913 class pass_ipa_inline
: public ipa_opt_pass_d
2916 pass_ipa_inline (gcc::context
*ctxt
)
2917 : ipa_opt_pass_d (pass_data_ipa_inline
, ctxt
,
2918 NULL
, /* generate_summary */
2919 NULL
, /* write_summary */
2920 NULL
, /* read_summary */
2921 NULL
, /* write_optimization_summary */
2922 NULL
, /* read_optimization_summary */
2923 NULL
, /* stmt_fixup */
2924 0, /* function_transform_todo_flags_start */
2925 inline_transform
, /* function_transform */
2926 NULL
) /* variable_transform */
2929 /* opt_pass methods: */
2930 virtual unsigned int execute (function
*) { return ipa_inline (); }
2932 }; // class pass_ipa_inline
2937 make_pass_ipa_inline (gcc::context
*ctxt
)
2939 return new pass_ipa_inline (ctxt
);