/* Inlining decision heuristics.
- Copyright (C) 2003, 2004, 2007, 2008, 2009, 2010, 2011, 2012
- Free Software Foundation, Inc.
+ Copyright (C) 2003-2014 Free Software Foundation, Inc.
Contributed by Jan Hubicka
This file is part of GCC.
#include "coretypes.h"
#include "tm.h"
#include "tree.h"
+#include "trans-mem.h"
+#include "calls.h"
#include "tree-inline.h"
#include "langhooks.h"
#include "flags.h"
-#include "cgraph.h"
#include "diagnostic.h"
#include "gimple-pretty-print.h"
#include "params.h"
#include "intl.h"
#include "tree-pass.h"
#include "coverage.h"
-#include "ggc.h"
#include "rtl.h"
-#include "tree-flow.h"
+#include "bitmap.h"
+#include "profile.h"
+#include "predict.h"
+#include "vec.h"
+#include "hashtab.h"
+#include "hash-set.h"
+#include "machmode.h"
+#include "hard-reg-set.h"
+#include "input.h"
+#include "function.h"
+#include "basic-block.h"
+#include "tree-ssa-alias.h"
+#include "internal-fn.h"
+#include "gimple-expr.h"
+#include "is-a.h"
+#include "gimple.h"
+#include "gimple-ssa.h"
+#include "hash-map.h"
+#include "plugin-api.h"
+#include "ipa-ref.h"
+#include "cgraph.h"
+#include "alloc-pool.h"
#include "ipa-prop.h"
#include "except.h"
#include "target.h"
#include "ipa-inline.h"
#include "ipa-utils.h"
+#include "sreal.h"
+#include "auto-profile.h"
+#include "cilk.h"
+#include "builtins.h"
/* Statistics we collect about inlining algorithm. */
static int overall_size;
static gcov_type max_count;
+static sreal max_count_real, max_relbenefit_real, half_int_min_real;
+static gcov_type spec_rem;
/* Return false when inlining edge E would lead to violating
limits on function unit growth or stack usage growth.
caller_growth_limits (struct cgraph_edge *e)
{
struct cgraph_node *to = e->caller;
- struct cgraph_node *what = cgraph_function_or_thunk_node (e->callee, NULL);
+ struct cgraph_node *what = e->callee->ultimate_alias_target ();
int newsize;
int limit = 0;
HOST_WIDE_INT stack_size_limit = 0, inlined_stack;
if (dump_file)
{
fprintf (dump_file, " not inlinable: %s/%i -> %s/%i, %s\n",
- xstrdup (cgraph_node_name (e->caller)), e->caller->uid,
- xstrdup (cgraph_node_name (e->callee)), e->callee->uid,
+ xstrdup (e->caller->name ()), e->caller->order,
+ xstrdup (e->callee->name ()), e->callee->order,
cgraph_inline_failed_string (e->inline_failed));
}
}
-/* Decide if we can inline the edge and possibly update
+ /* Decide whether sanitizer-related attributes allow inlining. */
+
+static bool
+sanitize_attrs_match_for_inline_p (const_tree caller, const_tree callee)
+{
+ /* Don't care if sanitizer is disabled */
+ if (!(flag_sanitize & SANITIZE_ADDRESS))
+ return true;
+
+ if (!caller || !callee)
+ return true;
+
+ return !!lookup_attribute ("no_sanitize_address",
+ DECL_ATTRIBUTES (caller)) ==
+ !!lookup_attribute ("no_sanitize_address",
+ DECL_ATTRIBUTES (callee));
+}
+
+ /* Decide if we can inline the edge and possibly update
inline_failed reason.
We check whether inlining is possible at all and whether
caller growth limits allow doing so.
- if REPORT is true, output reason to the dump file. */
+ if REPORT is true, output reason to the dump file.
+
+ if DISREGARD_LIMITS is true, ignore size limits.*/
static bool
-can_inline_edge_p (struct cgraph_edge *e, bool report)
+can_inline_edge_p (struct cgraph_edge *e, bool report,
+ bool disregard_limits = false)
{
bool inlinable = true;
enum availability avail;
- struct cgraph_node *callee
- = cgraph_function_or_thunk_node (e->callee, &avail);
- tree caller_tree = DECL_FUNCTION_SPECIFIC_OPTIMIZATION (e->caller->symbol.decl);
+ cgraph_node *callee = e->callee->ultimate_alias_target (&avail);
+ tree caller_tree = DECL_FUNCTION_SPECIFIC_OPTIMIZATION (e->caller->decl);
tree callee_tree
- = callee ? DECL_FUNCTION_SPECIFIC_OPTIMIZATION (callee->symbol.decl) : NULL;
- struct function *caller_cfun = DECL_STRUCT_FUNCTION (e->caller->symbol.decl);
- struct function *callee_cfun
- = callee ? DECL_STRUCT_FUNCTION (callee->symbol.decl) : NULL;
-
- if (!caller_cfun && e->caller->clone_of)
- caller_cfun = DECL_STRUCT_FUNCTION (e->caller->clone_of->symbol.decl);
-
- if (!callee_cfun && callee && callee->clone_of)
- callee_cfun = DECL_STRUCT_FUNCTION (callee->clone_of->symbol.decl);
+ = callee ? DECL_FUNCTION_SPECIFIC_OPTIMIZATION (callee->decl) : NULL;
+ struct function *caller_fun = e->caller->get_fun ();
+ struct function *callee_fun = callee ? callee->get_fun () : NULL;
gcc_assert (e->inline_failed);
- if (!callee || !callee->analyzed)
+ if (!callee || !callee->definition)
{
e->inline_failed = CIF_BODY_NOT_AVAILABLE;
inlinable = false;
}
- else if (!inline_summary (callee)->inlinable)
+ else if (callee->calls_comdat_local)
+ {
+ e->inline_failed = CIF_USES_COMDAT_LOCAL;
+ inlinable = false;
+ }
+ else if (!inline_summary (callee)->inlinable
+ || (caller_fun && fn_contains_cilk_spawn_p (caller_fun)))
{
e->inline_failed = CIF_FUNCTION_NOT_INLINABLE;
inlinable = false;
}
- else if (avail <= AVAIL_OVERWRITABLE)
+ else if (avail <= AVAIL_INTERPOSABLE)
{
e->inline_failed = CIF_OVERWRITABLE;
- return false;
+ inlinable = false;
}
else if (e->call_stmt_cannot_inline_p)
{
- e->inline_failed = CIF_MISMATCHED_ARGUMENTS;
+ if (e->inline_failed != CIF_FUNCTION_NOT_OPTIMIZED)
+ e->inline_failed = CIF_MISMATCHED_ARGUMENTS;
inlinable = false;
}
/* Don't inline if the functions have different EH personalities. */
- else if (DECL_FUNCTION_PERSONALITY (e->caller->symbol.decl)
- && DECL_FUNCTION_PERSONALITY (callee->symbol.decl)
- && (DECL_FUNCTION_PERSONALITY (e->caller->symbol.decl)
- != DECL_FUNCTION_PERSONALITY (callee->symbol.decl)))
+ else if (DECL_FUNCTION_PERSONALITY (e->caller->decl)
+ && DECL_FUNCTION_PERSONALITY (callee->decl)
+ && (DECL_FUNCTION_PERSONALITY (e->caller->decl)
+ != DECL_FUNCTION_PERSONALITY (callee->decl)))
{
e->inline_failed = CIF_EH_PERSONALITY;
inlinable = false;
}
/* TM pure functions should not be inlined into non-TM_pure
functions. */
- else if (is_tm_pure (callee->symbol.decl)
- && !is_tm_pure (e->caller->symbol.decl))
+ else if (is_tm_pure (callee->decl)
+ && !is_tm_pure (e->caller->decl))
{
e->inline_failed = CIF_UNSPECIFIED;
inlinable = false;
caller cannot.
FIXME: this is obviously wrong for LTO where STRUCT_FUNCTION is missing.
Move the flag into cgraph node or mirror it in the inline summary. */
- else if (callee_cfun && callee_cfun->can_throw_non_call_exceptions
- && !(caller_cfun && caller_cfun->can_throw_non_call_exceptions))
+ else if (callee_fun && callee_fun->can_throw_non_call_exceptions
+ && !(caller_fun && caller_fun->can_throw_non_call_exceptions))
{
e->inline_failed = CIF_NON_CALL_EXCEPTIONS;
inlinable = false;
}
/* Check compatibility of target optimization options. */
- else if (!targetm.target_option.can_inline_p (e->caller->symbol.decl,
- callee->symbol.decl))
+ else if (!targetm.target_option.can_inline_p (e->caller->decl,
+ callee->decl))
{
e->inline_failed = CIF_TARGET_OPTION_MISMATCH;
inlinable = false;
}
+ /* Don't inline a function with mismatched sanitization attributes. */
+ else if (!sanitize_attrs_match_for_inline_p (e->caller->decl, callee->decl))
+ {
+ e->inline_failed = CIF_ATTRIBUTE_MISMATCH;
+ inlinable = false;
+ }
/* Check if caller growth allows the inlining. */
- else if (!DECL_DISREGARD_INLINE_LIMITS (callee->symbol.decl)
+ else if (!DECL_DISREGARD_INLINE_LIMITS (callee->decl)
+ && !disregard_limits
&& !lookup_attribute ("flatten",
DECL_ATTRIBUTES
(e->caller->global.inlined_to
- ? e->caller->global.inlined_to->symbol.decl
- : e->caller->symbol.decl))
+ ? e->caller->global.inlined_to->decl
+ : e->caller->decl))
&& !caller_growth_limits (e))
inlinable = false;
/* Don't inline a function with a higher optimization level than the
if (((caller_opt->x_optimize > callee_opt->x_optimize)
|| (caller_opt->x_optimize_size != callee_opt->x_optimize_size))
/* gcc.dg/pr43564.c. Look at forced inline even in -O0. */
- && !DECL_DISREGARD_INLINE_LIMITS (e->callee->symbol.decl))
+ && !DECL_DISREGARD_INLINE_LIMITS (e->callee->decl))
{
e->inline_failed = CIF_OPTIMIZATION_MISMATCH;
inlinable = false;
static bool
can_early_inline_edge_p (struct cgraph_edge *e)
{
- struct cgraph_node *callee = cgraph_function_or_thunk_node (e->callee,
- NULL);
+ struct cgraph_node *callee = e->callee->ultimate_alias_target ();
/* Early inliner might get called at WPA stage when IPA pass adds new
function. In this case we can not really do any of early inlining
because function bodies are missing. */
- if (!gimple_has_body_p (callee->symbol.decl))
+ if (!gimple_has_body_p (callee->decl))
{
e->inline_failed = CIF_BODY_NOT_AVAILABLE;
return false;
(i.e. the callgraph is cyclic and we did not process
the callee by early inliner, yet). We don't have CIF code for this
case; later we will re-do the decision in the real inliner. */
- if (!gimple_in_ssa_p (DECL_STRUCT_FUNCTION (e->caller->symbol.decl))
- || !gimple_in_ssa_p (DECL_STRUCT_FUNCTION (callee->symbol.decl)))
+ if (!gimple_in_ssa_p (DECL_STRUCT_FUNCTION (e->caller->decl))
+ || !gimple_in_ssa_p (DECL_STRUCT_FUNCTION (callee->decl)))
{
if (dump_file)
fprintf (dump_file, " edge not inlinable: not in SSA form\n");
int num = 0;
for (e = n->callees; e; e = e->next_callee)
- if (!is_inexpensive_builtin (e->callee->symbol.decl))
+ if (!is_inexpensive_builtin (e->callee->decl))
num++;
return num;
}
want_early_inline_function_p (struct cgraph_edge *e)
{
bool want_inline = true;
- struct cgraph_node *callee = cgraph_function_or_thunk_node (e->callee, NULL);
+ struct cgraph_node *callee = e->callee->ultimate_alias_target ();
- if (DECL_DISREGARD_INLINE_LIMITS (callee->symbol.decl))
+ if (DECL_DISREGARD_INLINE_LIMITS (callee->decl))
;
- else if (!DECL_DECLARED_INLINE_P (callee->symbol.decl)
+ /* For AutoFDO, we need to make sure that before profile annotation, all
+ hot paths' IR look exactly the same as profiled binary. As a result,
+ in einliner, we will disregard size limit and inline those callsites
+ that are:
+ * inlined in the profiled binary, and
+ * the cloned callee has enough samples to be considered "hot". */
+ else if (flag_auto_profile && afdo_callsite_hot_enough_for_early_inline (e))
+ ;
+ else if (!DECL_DECLARED_INLINE_P (callee->decl)
&& !flag_inline_small_functions)
{
e->inline_failed = CIF_FUNCTION_NOT_INLINE_CANDIDATE;
if (growth <= 0)
;
- else if (!cgraph_maybe_hot_edge_p (e)
+ else if (!e->maybe_hot_p ()
&& growth > 0)
{
if (dump_file)
fprintf (dump_file, " will not early inline: %s/%i->%s/%i, "
"call is cold and code would grow by %i\n",
- xstrdup (cgraph_node_name (e->caller)), e->caller->uid,
- xstrdup (cgraph_node_name (callee)), callee->uid,
+ xstrdup (e->caller->name ()),
+ e->caller->order,
+ xstrdup (callee->name ()), callee->order,
growth);
want_inline = false;
}
if (dump_file)
fprintf (dump_file, " will not early inline: %s/%i->%s/%i, "
"growth %i exceeds --param early-inlining-insns\n",
- xstrdup (cgraph_node_name (e->caller)), e->caller->uid,
- xstrdup (cgraph_node_name (callee)), callee->uid,
+ xstrdup (e->caller->name ()),
+ e->caller->order,
+ xstrdup (callee->name ()), callee->order,
growth);
want_inline = false;
}
fprintf (dump_file, " will not early inline: %s/%i->%s/%i, "
"growth %i exceeds --param early-inlining-insns "
"divided by number of calls\n",
- xstrdup (cgraph_node_name (e->caller)), e->caller->uid,
- xstrdup (cgraph_node_name (callee)), callee->uid,
+ xstrdup (e->caller->name ()),
+ e->caller->order,
+ xstrdup (callee->name ()), callee->order,
growth);
want_inline = false;
}
want_inline_small_function_p (struct cgraph_edge *e, bool report)
{
bool want_inline = true;
- struct cgraph_node *callee = cgraph_function_or_thunk_node (e->callee, NULL);
+ struct cgraph_node *callee = e->callee->ultimate_alias_target ();
- if (DECL_DISREGARD_INLINE_LIMITS (callee->symbol.decl))
+ if (DECL_DISREGARD_INLINE_LIMITS (callee->decl))
;
- else if (!DECL_DECLARED_INLINE_P (callee->symbol.decl)
+ else if (!DECL_DECLARED_INLINE_P (callee->decl)
&& !flag_inline_small_functions)
{
e->inline_failed = CIF_FUNCTION_NOT_INLINE_CANDIDATE;
want_inline = false;
}
+ /* Do fast and conservative check if the function can be good
+ inline candidate. At the moment we allow inline hints to
+ promote non-inline functions to inline and we increase
+ MAX_INLINE_INSNS_SINGLE 16-fold for inline functions. */
+ else if ((!DECL_DECLARED_INLINE_P (callee->decl)
+ && (!e->count || !e->maybe_hot_p ()))
+ && inline_summary (callee)->min_size
+ - inline_edge_summary (e)->call_stmt_size
+ > MAX (MAX_INLINE_INSNS_SINGLE, MAX_INLINE_INSNS_AUTO))
+ {
+ e->inline_failed = CIF_MAX_INLINE_INSNS_AUTO_LIMIT;
+ want_inline = false;
+ }
+ else if ((DECL_DECLARED_INLINE_P (callee->decl) || e->count)
+ && inline_summary (callee)->min_size
+ - inline_edge_summary (e)->call_stmt_size
+ > 16 * MAX_INLINE_INSNS_SINGLE)
+ {
+ e->inline_failed = (DECL_DECLARED_INLINE_P (callee->decl)
+ ? CIF_MAX_INLINE_INSNS_SINGLE_LIMIT
+ : CIF_MAX_INLINE_INSNS_AUTO_LIMIT);
+ want_inline = false;
+ }
else
{
int growth = estimate_edge_growth (e);
;
/* Apply MAX_INLINE_INSNS_SINGLE limit. Do not do so when
hints suggests that inlining given function is very profitable. */
- else if (DECL_DECLARED_INLINE_P (callee->symbol.decl)
+ else if (DECL_DECLARED_INLINE_P (callee->decl)
&& growth >= MAX_INLINE_INSNS_SINGLE
- && !big_speedup
- && !(hints & (INLINE_HINT_indirect_call
- | INLINE_HINT_loop_iterations
- | INLINE_HINT_array_index
- | INLINE_HINT_loop_stride)))
+ && ((!big_speedup
+ && !(hints & (INLINE_HINT_indirect_call
+ | INLINE_HINT_known_hot
+ | INLINE_HINT_loop_iterations
+ | INLINE_HINT_array_index
+ | INLINE_HINT_loop_stride)))
+ || growth >= MAX_INLINE_INSNS_SINGLE * 16))
{
e->inline_failed = CIF_MAX_INLINE_INSNS_SINGLE_LIMIT;
want_inline = false;
}
- /* Before giving up based on fact that caller size will grow, allow
- functions that are called few times and eliminating the offline
- copy will lead to overall code size reduction.
- Not all of these will be handled by subsequent inlining of functions
- called once: in particular weak functions are not handled or funcitons
- that inline to multiple calls but a lot of bodies is optimized out.
- Finally we want to inline earlier to allow inlining of callbacks.
-
- This is slightly wrong on aggressive side: it is entirely possible
- that function is called many times with a context where inlining
- reduces code size and few times with a context where inlining increase
- code size. Resoluting growth estimate will be negative even if it
- would make more sense to keep offline copy and do not inline into the
- call sites that makes the code size grow.
-
- When badness orders the calls in a way that code reducing calls come
- first, this situation is not a problem at all: after inlining all
- "good" calls, we will realize that keeping the function around is
- better. */
- else if (growth <= MAX_INLINE_INSNS_SINGLE
- /* Unlike for functions called once, we play unsafe with
- COMDATs. We can allow that since we know functions
- in consideration are small (and thus risk is small) and
- moreover grow estimates already accounts that COMDAT
- functions may or may not disappear when eliminated from
- current unit. With good probability making aggressive
- choice in all units is going to make overall program
- smaller.
-
- Consequently we ask cgraph_can_remove_if_no_direct_calls_p
- instead of
- cgraph_will_be_removed_from_program_if_no_direct_calls */
- && !DECL_EXTERNAL (callee->symbol.decl)
- && cgraph_can_remove_if_no_direct_calls_p (callee)
- && estimate_growth (callee) <= 0)
- ;
- else if (!DECL_DECLARED_INLINE_P (callee->symbol.decl)
+ else if (!DECL_DECLARED_INLINE_P (callee->decl)
&& !flag_inline_functions)
{
- e->inline_failed = CIF_NOT_DECLARED_INLINED;
- want_inline = false;
+ /* growth_likely_positive is expensive, always test it last. */
+ if (growth >= MAX_INLINE_INSNS_SINGLE
+ || growth_likely_positive (callee, growth))
+ {
+ e->inline_failed = CIF_NOT_DECLARED_INLINED;
+ want_inline = false;
+ }
}
/* Apply MAX_INLINE_INSNS_AUTO limit for functions not declared inline
Upgrade it to MAX_INLINE_INSNS_SINGLE when hints suggests that
inlining given function is very profitable. */
- else if (!DECL_DECLARED_INLINE_P (callee->symbol.decl)
+ else if (!DECL_DECLARED_INLINE_P (callee->decl)
&& !big_speedup
+ && !(hints & INLINE_HINT_known_hot)
&& growth >= ((hints & (INLINE_HINT_indirect_call
| INLINE_HINT_loop_iterations
| INLINE_HINT_array_index
MAX_INLINE_INSNS_SINGLE)
: MAX_INLINE_INSNS_AUTO))
{
- e->inline_failed = CIF_MAX_INLINE_INSNS_AUTO_LIMIT;
- want_inline = false;
+ /* growth_likely_positive is expensive, always test it last. */
+ if (growth >= MAX_INLINE_INSNS_SINGLE
+ || growth_likely_positive (callee, growth))
+ {
+ e->inline_failed = CIF_MAX_INLINE_INSNS_AUTO_LIMIT;
+ want_inline = false;
+ }
}
/* If call is cold, do not inline when function body would grow. */
- else if (!cgraph_maybe_hot_edge_p (e))
+ else if (!e->maybe_hot_p ()
+ && (growth >= MAX_INLINE_INSNS_SINGLE
+ || growth_likely_positive (callee, growth)))
{
e->inline_failed = CIF_UNLIKELY_CALL;
want_inline = false;
int caller_freq = CGRAPH_FREQ_BASE;
int max_depth = PARAM_VALUE (PARAM_MAX_INLINE_RECURSIVE_DEPTH_AUTO);
- if (DECL_DECLARED_INLINE_P (edge->caller->symbol.decl))
+ if (DECL_DECLARED_INLINE_P (edge->caller->decl))
max_depth = PARAM_VALUE (PARAM_MAX_INLINE_RECURSIVE_DEPTH);
- if (!cgraph_maybe_hot_edge_p (edge))
+ if (!edge->maybe_hot_p ())
{
reason = "recursive call is cold";
want_inline = false;
if (outer_node->global.inlined_to)
caller_freq = outer_node->callers->frequency;
+ if (!caller_freq)
+ {
+ reason = "function is inlined and unlikely";
+ want_inline = false;
+ }
+
if (!want_inline)
;
/* Inlining of self recursive function into copy of itself within other function
return want_inline;
}
-/* Return true when NODE has caller other than EDGE.
+/* Return true when NODE has uninlinable caller;
+ set HAS_HOT_CALL if it has hot call.
Worker for cgraph_for_node_and_aliases. */
static bool
-check_caller_edge (struct cgraph_node *node, void *edge)
+check_callers (struct cgraph_node *node, void *has_hot_call)
{
- return (node->callers
- && node->callers != edge);
+ struct cgraph_edge *e;
+ for (e = node->callers; e; e = e->next_caller)
+ {
+ if (!can_inline_edge_p (e, true))
+ return true;
+ if (!(*(bool *)has_hot_call) && e->maybe_hot_p ())
+ *(bool *)has_hot_call = true;
+ }
+ return false;
}
+/* If NODE has a caller, return true. */
+
+static bool
+has_caller_p (struct cgraph_node *node, void *data ATTRIBUTE_UNUSED)
+{
+ if (node->callers)
+ return true;
+ return false;
+}
/* Decide if inlining NODE would reduce unit size by eliminating
the offline copy of function.
static bool
want_inline_function_to_all_callers_p (struct cgraph_node *node, bool cold)
{
- struct cgraph_node *function = cgraph_function_or_thunk_node (node, NULL);
- struct cgraph_edge *e;
- bool has_hot_call = false;
-
- /* Does it have callers? */
- if (!node->callers)
- return false;
- /* Already inlined? */
- if (function->global.inlined_to)
- return false;
- if (cgraph_function_or_thunk_node (node, NULL) != node)
- return false;
- /* Inlining into all callers would increase size? */
- if (estimate_growth (node) > 0)
- return false;
- /* Maybe other aliases has more direct calls. */
- if (cgraph_for_node_and_aliases (node, check_caller_edge, node->callers, true))
- return false;
- /* All inlines must be possible. */
- for (e = node->callers; e; e = e->next_caller)
- {
- if (!can_inline_edge_p (e, true))
- return false;
- if (!has_hot_call && cgraph_maybe_hot_edge_p (e))
- has_hot_call = 1;
- }
+ bool has_hot_call = false;
- if (!cold && !has_hot_call)
- return false;
- return true;
+ if (node->ultimate_alias_target () != node)
+ return false;
+ /* Already inlined? */
+ if (node->global.inlined_to)
+ return false;
+ /* Does it have callers? */
+ if (!node->call_for_symbol_thunks_and_aliases (has_caller_p, NULL, true))
+ return false;
+ /* Inlining into all callers would increase size? */
+ if (estimate_growth (node) > 0)
+ return false;
+ /* All inlines must be possible. */
+ if (node->call_for_symbol_thunks_and_aliases (check_callers, &has_hot_call,
+ true))
+ return false;
+ if (!cold && !has_hot_call)
+ return false;
+ return true;
}
#define RELATIVE_TIME_BENEFIT_RANGE (INT_MAX / 64)
/* Inlining into extern inline function is not a win. */
if (DECL_EXTERNAL (edge->caller->global.inlined_to
- ? edge->caller->global.inlined_to->symbol.decl
- : edge->caller->symbol.decl))
+ ? edge->caller->global.inlined_to->decl
+ : edge->caller->decl))
return 1;
/* Watch overflows. */
{
gcov_type badness;
int growth, edge_time;
- struct cgraph_node *callee = cgraph_function_or_thunk_node (edge->callee,
- NULL);
+ struct cgraph_node *callee = edge->callee->ultimate_alias_target ();
struct inline_summary *callee_info = inline_summary (callee);
inline_hints hints;
- if (DECL_DISREGARD_INLINE_LIMITS (callee->symbol.decl))
+ if (DECL_DISREGARD_INLINE_LIMITS (callee->decl))
return INT_MIN;
growth = estimate_edge_growth (edge);
if (dump)
{
fprintf (dump_file, " Badness calculation for %s/%i -> %s/%i\n",
- xstrdup (cgraph_node_name (edge->caller)),
- edge->caller->uid,
- xstrdup (cgraph_node_name (callee)),
- edge->callee->uid);
+ xstrdup (edge->caller->name ()),
+ edge->caller->order,
+ xstrdup (callee->name ()),
+ edge->callee->order);
fprintf (dump_file, " size growth %i, time %i ",
growth,
edge_time);
else if (max_count)
{
int relbenefit = relative_time_benefit (callee_info, edge, edge_time);
- badness =
- ((int)
- ((double) edge->count * INT_MIN / 2 / max_count / RELATIVE_TIME_BENEFIT_RANGE) *
- relbenefit) / growth;
-
- /* Be sure that insanity of the profile won't lead to increasing counts
- in the scalling and thus to overflow in the computation above. */
- gcc_assert (max_count >= edge->count);
+ /* Capping edge->count to max_count. edge->count can be larger than
+ max_count if an inline adds new edges which increase max_count
+ after max_count is computed. */
+ gcov_type edge_count = edge->count > max_count ? max_count : edge->count;
+
+ sreal relbenefit_real (relbenefit, 0);
+ sreal growth_real (growth, 0);
+
+ /* relative_edge_count. */
+ sreal tmp (edge_count, 0);
+ tmp /= max_count_real;
+
+ /* relative_time_benefit. */
+ tmp *= relbenefit_real;
+ tmp /= max_relbenefit_real;
+
+ /* growth_f_caller. */
+ tmp *= half_int_min_real;
+ tmp /= growth_real;
+
+ badness = -1 * tmp.to_int ();
+
if (dump)
{
fprintf (dump_file,
- " %i (relative %f): profile info. Relative count %f"
+ " %i (relative %f): profile info. Relative count %f%s"
" * Relative benefit %f\n",
(int) badness, (double) badness / INT_MIN,
- (double) edge->count / max_count,
+ (double) edge_count / max_count,
+ edge->count > max_count ? " (capped to max_count)" : "",
relbenefit * 100.0 / RELATIVE_TIME_BENEFIT_RANGE);
}
}
gcc_assert (badness >= INT_MIN);
gcc_assert (badness <= INT_MAX - 1);
/* Make recursive inlining happen always after other inlining is done. */
- if (cgraph_edge_recursive_p (edge))
+ if (edge->recursive_p ())
return badness + 1;
else
return badness;
{
fprintf (dump_file,
" decreasing badness %s/%i -> %s/%i, %i to %i\n",
- xstrdup (cgraph_node_name (edge->caller)),
- edge->caller->uid,
- xstrdup (cgraph_node_name (edge->callee)),
- edge->callee->uid,
+ xstrdup (edge->caller->name ()),
+ edge->caller->order,
+ xstrdup (edge->callee->name ()),
+ edge->callee->order,
(int)n->key,
badness);
}
{
fprintf (dump_file,
" enqueuing call %s/%i -> %s/%i, badness %i\n",
- xstrdup (cgraph_node_name (edge->caller)),
- edge->caller->uid,
- xstrdup (cgraph_node_name (edge->callee)),
- edge->callee->uid,
+ xstrdup (edge->caller->name ()),
+ edge->caller->order,
+ xstrdup (edge->callee->name ()),
+ edge->callee->order,
badness);
}
edge->aux = fibheap_insert (heap, badness, edge);
struct cgraph_edge *edge;
struct cgraph_edge *e = node->callees;
struct cgraph_node *where = node;
- int i;
struct ipa_ref *ref;
if (where->global.inlined_to)
for (edge = where->callers; edge; edge = edge->next_caller)
if (edge->inline_failed)
reset_edge_growth_cache (edge);
- for (i = 0; ipa_ref_list_referring_iterate (&where->symbol.ref_list,
- i, ref); i++)
- if (ref->use == IPA_REF_ALIAS)
- reset_edge_caches (ipa_ref_referring_node (ref));
+
+ FOR_EACH_ALIAS (where, ref)
+ reset_edge_caches (dyn_cast <cgraph_node *> (ref->referring));
if (!e)
return;
struct cgraph_edge *check_inlinablity_for)
{
struct cgraph_edge *edge;
- int i;
struct ipa_ref *ref;
if ((!node->alias && !inline_summary (node)->inlinable)
- || cgraph_function_body_availability (node) <= AVAIL_OVERWRITABLE
|| node->global.inlined_to)
return;
if (!bitmap_set_bit (updated_nodes, node->uid))
return;
- for (i = 0; ipa_ref_list_referring_iterate (&node->symbol.ref_list,
- i, ref); i++)
- if (ref->use == IPA_REF_ALIAS)
- {
- struct cgraph_node *alias = ipa_ref_referring_node (ref);
- update_caller_keys (heap, alias, updated_nodes, check_inlinablity_for);
- }
+ FOR_EACH_ALIAS (node, ref)
+ {
+ struct cgraph_node *alias = dyn_cast <cgraph_node *> (ref->referring);
+ update_caller_keys (heap, alias, updated_nodes, check_inlinablity_for);
+ }
for (edge = node->callers; edge; edge = edge->next_caller)
if (edge->inline_failed)
growth chould have just increased and consequentely badness metric
don't need updating. */
if (e->inline_failed
- && (callee = cgraph_function_or_thunk_node (e->callee, &avail))
+ && (callee = e->callee->ultimate_alias_target (&avail))
&& inline_summary (callee)->inlinable
- && cgraph_function_body_availability (callee) >= AVAIL_AVAILABLE
+ && avail >= AVAIL_AVAILABLE
&& !bitmap_bit_p (updated_nodes, callee->uid))
{
if (can_inline_edge_p (e, false)
for (e = where->callees; e; e = e->next_callee)
if (e->callee == node
- || (cgraph_function_or_thunk_node (e->callee, &avail) == node
- && avail > AVAIL_OVERWRITABLE))
+ || (e->callee->ultimate_alias_target (&avail) == node
+ && avail > AVAIL_INTERPOSABLE))
{
/* When profile feedback is available, prioritize by expected number
of calls. */
static bool
recursive_inlining (struct cgraph_edge *edge,
- vec<cgraph_edge_p> *new_edges)
+ vec<cgraph_edge *> *new_edges)
{
int limit = PARAM_VALUE (PARAM_MAX_INLINE_INSNS_RECURSIVE_AUTO);
fibheap_t heap;
if (node->global.inlined_to)
node = node->global.inlined_to;
- if (DECL_DECLARED_INLINE_P (node->symbol.decl))
+ if (DECL_DECLARED_INLINE_P (node->decl))
limit = PARAM_VALUE (PARAM_MAX_INLINE_INSNS_RECURSIVE);
/* Make sure that function is small enough to be considered for inlining. */
if (dump_file)
fprintf (dump_file,
" Performing recursive inlining on %s\n",
- cgraph_node_name (node));
+ node->name ());
/* Do the inlining and update list of recursive call during process. */
while (!fibheap_empty (heap))
the already modified body. */
if (master_clone)
{
- cgraph_redirect_edge_callee (curr, master_clone);
- reset_edge_growth_cache (curr);
+ curr->redirect_callee (master_clone);
+ reset_edge_growth_cache (curr);
}
if (estimate_size_after_inlining (node, curr) > limit)
{
- cgraph_redirect_edge_callee (curr, dest);
+ curr->redirect_callee (dest);
reset_edge_growth_cache (curr);
break;
}
depth = 1;
for (cnode = curr->caller;
cnode->global.inlined_to; cnode = cnode->callers->caller)
- if (node->symbol.decl
- == cgraph_function_or_thunk_node (curr->callee, NULL)->symbol.decl)
+ if (node->decl
+ == curr->callee->ultimate_alias_target ()->decl)
depth++;
if (!want_inline_self_recursive_call_p (curr, node, false, depth))
{
- cgraph_redirect_edge_callee (curr, dest);
+ curr->redirect_callee (dest);
reset_edge_growth_cache (curr);
continue;
}
if (!master_clone)
{
/* We need original clone to copy around. */
- master_clone = cgraph_clone_node (node, node->symbol.decl,
- node->count, CGRAPH_FREQ_BASE,
- false, vNULL, true);
+ master_clone = node->create_clone (node->decl, node->count,
+ CGRAPH_FREQ_BASE, false, vNULL,
+ true, NULL, NULL);
for (e = master_clone->callees; e; e = e->next_callee)
if (!e->inline_failed)
- clone_inlined_nodes (e, true, false, NULL);
- cgraph_redirect_edge_callee (curr, master_clone);
+ clone_inlined_nodes (e, true, false, NULL, CGRAPH_FREQ_BASE);
+ curr->redirect_callee (master_clone);
reset_edge_growth_cache (curr);
}
/* Remove master clone we used for inlining. We rely that clones inlined
into master clone gets queued just before master clone so we don't
need recursion. */
- for (node = cgraph_first_function (); node != master_clone;
+ for (node = symtab->first_function (); node != master_clone;
node = next)
{
- next = cgraph_next_function (node);
+ next = symtab->next_function (node);
if (node->global.inlined_to == master_clone)
- cgraph_remove_node (node);
+ node->remove ();
}
- cgraph_remove_node (master_clone);
+ master_clone->remove ();
return true;
}
if (max_insns < PARAM_VALUE (PARAM_LARGE_UNIT_INSNS))
max_insns = PARAM_VALUE (PARAM_LARGE_UNIT_INSNS);
- return ((HOST_WIDEST_INT) max_insns
+ return ((int64_t) max_insns
* (100 + PARAM_VALUE (PARAM_INLINE_UNIT_GROWTH)) / 100);
}
/* Compute badness of all edges in NEW_EDGES and add them to the HEAP. */
static void
-add_new_edges_to_heap (fibheap_t heap, vec<cgraph_edge_p> new_edges)
+add_new_edges_to_heap (fibheap_t heap, vec<cgraph_edge *> new_edges)
{
while (new_edges.length () > 0)
{
}
}
+/* Remove EDGE from the fibheap. */
+
+static void
+heap_edge_removal_hook (struct cgraph_edge *e, void *data)
+{
+ if (e->callee)
+ reset_node_growth_cache (e->callee);
+ if (e->aux)
+ {
+ fibheap_delete_node ((fibheap_t)data, (fibnode_t)e->aux);
+ e->aux = NULL;
+ }
+}
+
+/* Return true if speculation of edge E seems useful.
+ If ANTICIPATE_INLINING is true, be conservative and hope that E
+ may get inlined. */
+
+bool
+speculation_useful_p (struct cgraph_edge *e, bool anticipate_inlining)
+{
+ enum availability avail;
+ struct cgraph_node *target = e->callee->ultimate_alias_target (&avail);
+ struct cgraph_edge *direct, *indirect;
+ struct ipa_ref *ref;
+
+ gcc_assert (e->speculative && !e->indirect_unknown_callee);
+
+ if (!e->maybe_hot_p ())
+ return false;
+
+ /* See if IP optimizations found something potentially useful about the
+ function. For now we look only for CONST/PURE flags. Almost everything
+ else we propagate is useless. */
+ if (avail >= AVAIL_AVAILABLE)
+ {
+ int ecf_flags = flags_from_decl_or_type (target->decl);
+ if (ecf_flags & ECF_CONST)
+ {
+ e->speculative_call_info (direct, indirect, ref);
+ if (!(indirect->indirect_info->ecf_flags & ECF_CONST))
+ return true;
+ }
+ else if (ecf_flags & ECF_PURE)
+ {
+ e->speculative_call_info (direct, indirect, ref);
+ if (!(indirect->indirect_info->ecf_flags & ECF_PURE))
+ return true;
+ }
+ }
+ /* If we did not managed to inline the function nor redirect
+ to an ipa-cp clone (that are seen by having local flag set),
+ it is probably pointless to inline it unless hardware is missing
+ indirect call predictor. */
+ if (!anticipate_inlining && e->inline_failed && !target->local.local)
+ return false;
+ /* For overwritable targets there is not much to do. */
+ if (e->inline_failed && !can_inline_edge_p (e, false, true))
+ return false;
+ /* OK, speculation seems interesting. */
+ return true;
+}
+
+/* We know that EDGE is not going to be inlined.
+ See if we can remove speculation. */
+
+static void
+resolve_noninline_speculation (fibheap_t edge_heap, struct cgraph_edge *edge)
+{
+ if (edge->speculative && !speculation_useful_p (edge, false))
+ {
+ struct cgraph_node *node = edge->caller;
+ struct cgraph_node *where = node->global.inlined_to
+ ? node->global.inlined_to : node;
+ bitmap updated_nodes = BITMAP_ALLOC (NULL);
+
+ spec_rem += edge->count;
+ edge->resolve_speculation ();
+ reset_edge_caches (where);
+ inline_update_overall_summary (where);
+ update_caller_keys (edge_heap, where,
+ updated_nodes, NULL);
+ update_callee_keys (edge_heap, where,
+ updated_nodes);
+ BITMAP_FREE (updated_nodes);
+ }
+}
/* We use greedy algorithm for inlining of small functions:
All inline candidates are put into prioritized heap ordered in
fibheap_t edge_heap = fibheap_new ();
bitmap updated_nodes = BITMAP_ALLOC (NULL);
int min_size, max_size;
- vec<cgraph_edge_p> new_indirect_edges = vNULL;
+ auto_vec<cgraph_edge *> new_indirect_edges;
int initial_size = 0;
- struct cgraph_node **order = XCNEWVEC (struct cgraph_node *, cgraph_n_nodes);
-
+ struct cgraph_node **order = XCNEWVEC (cgraph_node *, symtab->cgraph_count);
+ struct cgraph_edge_hook_list *edge_removal_hook_holder;
if (flag_indirect_inlining)
new_indirect_edges.create (8);
+ edge_removal_hook_holder
+ = symtab->add_edge_removal_hook (&heap_edge_removal_hook, edge_heap);
+
/* Compute overall unit size and other global parameters used by badness
metrics. */
FOR_EACH_DEFINED_FUNCTION (node)
if (!node->global.inlined_to)
{
- if (cgraph_function_with_gimple_body_p (node)
+ if (node->has_gimple_body_p ()
|| node->thunk.thunk_p)
{
struct inline_summary *info = inline_summary (node);
- struct ipa_dfs_info *dfs = (struct ipa_dfs_info *) node->symbol.aux;
+ struct ipa_dfs_info *dfs = (struct ipa_dfs_info *) node->aux;
- if (!DECL_EXTERNAL (node->symbol.decl))
+ /* Do not account external functions, they will be optimized out
+ if not inlined. Also only count the non-cold portion of program. */
+ if (!DECL_EXTERNAL (node->decl)
+ && node->frequency != NODE_FREQUENCY_UNLIKELY_EXECUTED)
initial_size += info->size;
info->growth = estimate_growth (node);
if (dfs && dfs->next_cycle)
struct cgraph_node *n2;
int id = dfs->scc_no + 1;
for (n2 = node; n2;
- n2 = ((struct ipa_dfs_info *) node->symbol.aux)->next_cycle)
+ n2 = ((struct ipa_dfs_info *) node->aux)->next_cycle)
{
struct inline_summary *info2 = inline_summary (n2);
if (info2->scc_no)
if (max_count < edge->count)
max_count = edge->count;
}
+ max_count_real = sreal (max_count, 0);
+ max_relbenefit_real = sreal (RELATIVE_TIME_BENEFIT_RANGE, 0);
+ half_int_min_real = sreal (INT_MAX / 2, 0);
ipa_free_postorder_info ();
initialize_growth_caches ();
max_size = compute_max_insns (overall_size);
min_size = overall_size;
- /* Populate the heeap with all edges we might inline. */
+ /* Populate the heap with all edges we might inline. */
FOR_EACH_DEFINED_FUNCTION (node)
- if (!node->global.inlined_to)
- {
- if (dump_file)
- fprintf (dump_file, "Enqueueing calls of %s/%i.\n",
- cgraph_node_name (node), node->uid);
+ {
+ bool update = false;
+ struct cgraph_edge *next;
- for (edge = node->callers; edge; edge = edge->next_caller)
+ if (dump_file)
+ fprintf (dump_file, "Enqueueing calls in %s/%i.\n",
+ node->name (), node->order);
+
+ for (edge = node->callees; edge; edge = next)
+ {
+ next = edge->next_callee;
if (edge->inline_failed
+ && !edge->aux
&& can_inline_edge_p (edge, true)
&& want_inline_small_function_p (edge, true)
&& edge->inline_failed)
gcc_assert (!edge->aux);
update_edge_key (edge_heap, edge);
}
- }
+ if (edge->speculative && !speculation_useful_p (edge, edge->aux != NULL))
+ {
+ edge->resolve_speculation ();
+ update = true;
+ }
+ }
+ if (update)
+ {
+ struct cgraph_node *where = node->global.inlined_to
+ ? node->global.inlined_to : node;
+ inline_update_overall_summary (where);
+ reset_node_growth_cache (where);
+ reset_edge_caches (where);
+ update_caller_keys (edge_heap, where,
+ updated_nodes, NULL);
+ bitmap_clear (updated_nodes);
+ }
+ }
gcc_assert (in_lto_p
|| !max_count
edge = (struct cgraph_edge *) fibheap_extract_min (edge_heap);
gcc_assert (edge->aux);
edge->aux = NULL;
- if (!edge->inline_failed)
+ if (!edge->inline_failed || !edge->callee->analyzed)
continue;
/* Be sure that caches are maintained consistent.
}
if (!can_inline_edge_p (edge, true))
- continue;
+ {
+ resolve_noninline_speculation (edge_heap, edge);
+ continue;
+ }
- callee = cgraph_function_or_thunk_node (edge->callee, NULL);
+ callee = edge->callee->ultimate_alias_target ();
growth = estimate_edge_growth (edge);
if (dump_file)
{
fprintf (dump_file,
- "\nConsidering %s with %i size\n",
- cgraph_node_name (callee),
+ "\nConsidering %s/%i with %i size\n",
+ callee->name (), callee->order,
inline_summary (callee)->size);
fprintf (dump_file,
- " to be inlined into %s in %s:%i\n"
- " Estimated growth after inlined into all is %+i insns.\n"
+ " to be inlined into %s/%i in %s:%i\n"
" Estimated badness is %i, frequency %.2f.\n",
- cgraph_node_name (edge->caller),
+ edge->caller->name (), edge->caller->order,
flag_wpa ? "unknown"
: gimple_filename ((const_gimple) edge->call_stmt),
flag_wpa ? -1
: gimple_lineno ((const_gimple) edge->call_stmt),
- estimate_growth (callee),
badness,
edge->frequency / (double)CGRAPH_FREQ_BASE);
if (edge->count)
- fprintf (dump_file," Called "HOST_WIDEST_INT_PRINT_DEC"x\n",
+ fprintf (dump_file," Called %"PRId64"x\n",
edge->count);
if (dump_flags & TDF_DETAILS)
edge_badness (edge, true);
}
if (overall_size + growth > max_size
- && !DECL_DISREGARD_INLINE_LIMITS (callee->symbol.decl))
+ && !DECL_DISREGARD_INLINE_LIMITS (callee->decl))
{
edge->inline_failed = CIF_INLINE_UNIT_GROWTH_LIMIT;
report_inline_failed_reason (edge);
+ resolve_noninline_speculation (edge_heap, edge);
continue;
}
if (!want_inline_small_function_p (edge, true))
- continue;
+ {
+ resolve_noninline_speculation (edge_heap, edge);
+ continue;
+ }
- /* Heuristics for inlining small functions works poorly for
- recursive calls where we do efect similar to loop unrolling.
- When inliing such edge seems profitable, leave decision on
+ /* Heuristics for inlining small functions work poorly for
+ recursive calls where we do effects similar to loop unrolling.
+ When inlining such edge seems profitable, leave decision on
specific inliner. */
- if (cgraph_edge_recursive_p (edge))
+ if (edge->recursive_p ())
{
where = edge->caller;
if (where->global.inlined_to)
? &new_indirect_edges : NULL))
{
edge->inline_failed = CIF_RECURSIVE_INLINING;
+ resolve_noninline_speculation (edge_heap, edge);
continue;
}
reset_edge_caches (where);
if (flag_indirect_inlining)
add_new_edges_to_heap (edge_heap, new_indirect_edges);
update_callee_keys (edge_heap, where, updated_nodes);
+ bitmap_clear (updated_nodes);
}
else
{
struct cgraph_node *outer_node = NULL;
int depth = 0;
- /* Consider the case where self recursive function A is inlined into B.
- This is desired optimization in some cases, since it leads to effect
- similar of loop peeling and we might completely optimize out the
- recursive call. However we must be extra selective. */
+ /* Consider the case where self recursive function A is inlined
+ into B. This is desired optimization in some cases, since it
+ leads to effect similar of loop peeling and we might completely
+ optimize out the recursive call. However we must be extra
+ selective. */
where = edge->caller;
while (where->global.inlined_to)
{
- if (where->symbol.decl == callee->symbol.decl)
+ if (where->decl == callee->decl)
outer_node = where, depth++;
where = where->callers->caller;
}
true, depth))
{
edge->inline_failed
- = (DECL_DISREGARD_INLINE_LIMITS (edge->callee->symbol.decl)
+ = (DECL_DISREGARD_INLINE_LIMITS (edge->callee->decl)
? CIF_RECURSIVE_INLINING : CIF_UNSPECIFIED);
+ resolve_noninline_speculation (edge_heap, edge);
continue;
}
else if (depth && dump_file)
fprintf (dump_file,
" Inlined into %s which now has time %i and size %i,"
"net change of %+i.\n",
- cgraph_node_name (edge->caller),
+ edge->caller->name (),
inline_summary (edge->caller)->time,
inline_summary (edge->caller)->size,
overall_size - old_size);
}
free_growth_caches ();
- new_indirect_edges.release ();
fibheap_delete (edge_heap);
if (dump_file)
fprintf (dump_file,
initial_size, overall_size,
initial_size ? overall_size * 100 / (initial_size) - 100: 0);
BITMAP_FREE (updated_nodes);
+ symtab->remove_edge_removal_hook (edge_removal_hook_holder);
}
/* Flatten NODE. Performed both during early inlining and
struct cgraph_edge *e;
/* We shouldn't be called recursively when we are being processed. */
- gcc_assert (node->symbol.aux == NULL);
+ gcc_assert (node->aux == NULL);
- node->symbol.aux = (void *) node;
+ node->aux = (void *) node;
for (e = node->callees; e; e = e->next_callee)
{
struct cgraph_node *orig_callee;
- struct cgraph_node *callee = cgraph_function_or_thunk_node (e->callee, NULL);
+ struct cgraph_node *callee = e->callee->ultimate_alias_target ();
/* We've hit cycle? It is time to give up. */
- if (callee->symbol.aux)
+ if (callee->aux)
{
if (dump_file)
fprintf (dump_file,
"Not inlining %s into %s to avoid cycle.\n",
- xstrdup (cgraph_node_name (callee)),
- xstrdup (cgraph_node_name (e->caller)));
+ xstrdup (callee->name ()),
+ xstrdup (e->caller->name ()));
e->inline_failed = CIF_RECURSIVE_INLINING;
continue;
}
: !can_early_inline_edge_p (e))
continue;
- if (cgraph_edge_recursive_p (e))
+ if (e->recursive_p ())
{
if (dump_file)
fprintf (dump_file, "Not inlining: recursive call.\n");
continue;
}
- if (gimple_in_ssa_p (DECL_STRUCT_FUNCTION (node->symbol.decl))
- != gimple_in_ssa_p (DECL_STRUCT_FUNCTION (callee->symbol.decl)))
+ if (gimple_in_ssa_p (DECL_STRUCT_FUNCTION (node->decl))
+ != gimple_in_ssa_p (DECL_STRUCT_FUNCTION (callee->decl)))
{
if (dump_file)
fprintf (dump_file, "Not inlining: SSA form does not match.\n");
recursing through the original node if the node was cloned. */
if (dump_file)
fprintf (dump_file, " Inlining %s into %s.\n",
- xstrdup (cgraph_node_name (callee)),
- xstrdup (cgraph_node_name (e->caller)));
+ xstrdup (callee->name ()),
+ xstrdup (e->caller->name ()));
orig_callee = callee;
inline_call (e, true, NULL, NULL, false);
if (e->callee != orig_callee)
- orig_callee->symbol.aux = (void *) node;
+ orig_callee->aux = (void *) node;
flatten_function (e->callee, early);
if (e->callee != orig_callee)
- orig_callee->symbol.aux = NULL;
+ orig_callee->aux = NULL;
}
- node->symbol.aux = NULL;
+ node->aux = NULL;
if (!node->global.inlined_to)
inline_update_overall_summary (node);
}
+/* Count number of callers of NODE and store it into DATA (that
+ points to int. Worker for cgraph_for_node_and_aliases. */
+
+static bool
+sum_callers (struct cgraph_node *node, void *data)
+{
+ struct cgraph_edge *e;
+ int *num_calls = (int *)data;
+
+ for (e = node->callers; e; e = e->next_caller)
+ (*num_calls)++;
+ return false;
+}
+
+/* Inline NODE to all callers. Worker for cgraph_for_node_and_aliases.
+ DATA points to number of calls originally found so we avoid infinite
+ recursion. */
+
+static bool
+inline_to_all_callers (struct cgraph_node *node, void *data)
+{
+ int *num_calls = (int *)data;
+ bool callee_removed = false;
+
+ while (node->callers && !node->global.inlined_to)
+ {
+ struct cgraph_node *caller = node->callers->caller;
+
+ if (dump_file)
+ {
+ fprintf (dump_file,
+ "\nInlining %s size %i.\n",
+ node->name (),
+ inline_summary (node)->size);
+ fprintf (dump_file,
+ " Called once from %s %i insns.\n",
+ node->callers->caller->name (),
+ inline_summary (node->callers->caller)->size);
+ }
+
+ inline_call (node->callers, true, NULL, NULL, true, &callee_removed);
+ if (dump_file)
+ fprintf (dump_file,
+ " Inlined into %s which now has %i size\n",
+ caller->name (),
+ inline_summary (caller)->size);
+ if (!(*num_calls)--)
+ {
+ if (dump_file)
+ fprintf (dump_file, "New calls found; giving up.\n");
+ return callee_removed;
+ }
+ if (callee_removed)
+ return true;
+ }
+ return false;
+}
+
+/* Output overall time estimate. */
+static void
+dump_overall_stats (void)
+{
+ int64_t sum_weighted = 0, sum = 0;
+ struct cgraph_node *node;
+
+ FOR_EACH_DEFINED_FUNCTION (node)
+ if (!node->global.inlined_to
+ && !node->alias)
+ {
+ int time = inline_summary (node)->time;
+ sum += time;
+ sum_weighted += time * node->count;
+ }
+ fprintf (dump_file, "Overall time estimate: "
+ "%"PRId64" weighted by profile: "
+ "%"PRId64"\n", sum, sum_weighted);
+}
+
+/* Output some useful stats about inlining. */
+
+static void
+dump_inline_stats (void)
+{
+ int64_t inlined_cnt = 0, inlined_indir_cnt = 0;
+ int64_t inlined_virt_cnt = 0, inlined_virt_indir_cnt = 0;
+ int64_t noninlined_cnt = 0, noninlined_indir_cnt = 0;
+ int64_t noninlined_virt_cnt = 0, noninlined_virt_indir_cnt = 0;
+ int64_t inlined_speculative = 0, inlined_speculative_ply = 0;
+ int64_t indirect_poly_cnt = 0, indirect_cnt = 0;
+ int64_t reason[CIF_N_REASONS][3];
+ int i;
+ struct cgraph_node *node;
+
+ memset (reason, 0, sizeof (reason));
+ FOR_EACH_DEFINED_FUNCTION (node)
+ {
+ struct cgraph_edge *e;
+ for (e = node->callees; e; e = e->next_callee)
+ {
+ if (e->inline_failed)
+ {
+ reason[(int) e->inline_failed][0] += e->count;
+ reason[(int) e->inline_failed][1] += e->frequency;
+ reason[(int) e->inline_failed][2] ++;
+ if (DECL_VIRTUAL_P (e->callee->decl))
+ {
+ if (e->indirect_inlining_edge)
+ noninlined_virt_indir_cnt += e->count;
+ else
+ noninlined_virt_cnt += e->count;
+ }
+ else
+ {
+ if (e->indirect_inlining_edge)
+ noninlined_indir_cnt += e->count;
+ else
+ noninlined_cnt += e->count;
+ }
+ }
+ else
+ {
+ if (e->speculative)
+ {
+ if (DECL_VIRTUAL_P (e->callee->decl))
+ inlined_speculative_ply += e->count;
+ else
+ inlined_speculative += e->count;
+ }
+ else if (DECL_VIRTUAL_P (e->callee->decl))
+ {
+ if (e->indirect_inlining_edge)
+ inlined_virt_indir_cnt += e->count;
+ else
+ inlined_virt_cnt += e->count;
+ }
+ else
+ {
+ if (e->indirect_inlining_edge)
+ inlined_indir_cnt += e->count;
+ else
+ inlined_cnt += e->count;
+ }
+ }
+ }
+ for (e = node->indirect_calls; e; e = e->next_callee)
+ if (e->indirect_info->polymorphic)
+ indirect_poly_cnt += e->count;
+ else
+ indirect_cnt += e->count;
+ }
+ if (max_count)
+ {
+ fprintf (dump_file,
+ "Inlined %"PRId64 " + speculative "
+ "%"PRId64 " + speculative polymorphic "
+ "%"PRId64 " + previously indirect "
+ "%"PRId64 " + virtual "
+ "%"PRId64 " + virtual and previously indirect "
+ "%"PRId64 "\n" "Not inlined "
+ "%"PRId64 " + previously indirect "
+ "%"PRId64 " + virtual "
+ "%"PRId64 " + virtual and previously indirect "
+ "%"PRId64 " + stil indirect "
+ "%"PRId64 " + still indirect polymorphic "
+ "%"PRId64 "\n", inlined_cnt,
+ inlined_speculative, inlined_speculative_ply,
+ inlined_indir_cnt, inlined_virt_cnt, inlined_virt_indir_cnt,
+ noninlined_cnt, noninlined_indir_cnt, noninlined_virt_cnt,
+ noninlined_virt_indir_cnt, indirect_cnt, indirect_poly_cnt);
+ fprintf (dump_file,
+ "Removed speculations %"PRId64 "\n",
+ spec_rem);
+ }
+ dump_overall_stats ();
+ fprintf (dump_file, "\nWhy inlining failed?\n");
+ for (i = 0; i < CIF_N_REASONS; i++)
+ if (reason[i][2])
+ fprintf (dump_file, "%-50s: %8i calls, %8i freq, %"PRId64" count\n",
+ cgraph_inline_failed_string ((cgraph_inline_failed_t) i),
+ (int) reason[i][2], (int) reason[i][1], reason[i][0]);
+}
+
/* Decide on the inlining. We do so in the topological order to avoid
expenses on updating data structures. */
{
struct cgraph_node *node;
int nnodes;
- struct cgraph_node **order =
- XCNEWVEC (struct cgraph_node *, cgraph_n_nodes);
+ struct cgraph_node **order;
int i;
+ int cold;
+ bool remove_functions = false;
+
+ if (!optimize)
+ return 0;
+
+ order = XCNEWVEC (struct cgraph_node *, symtab->cgraph_count);
if (in_lto_p && optimize)
ipa_update_after_lto_read ();
nnodes = ipa_reverse_postorder (order);
FOR_EACH_FUNCTION (node)
- node->symbol.aux = 0;
+ node->aux = 0;
if (dump_file)
fprintf (dump_file, "\nFlattening functions:\n");
try to flatten itself turning it into a self-recursive
function. */
if (lookup_attribute ("flatten",
- DECL_ATTRIBUTES (node->symbol.decl)) != NULL)
+ DECL_ATTRIBUTES (node->decl)) != NULL)
{
if (dump_file)
fprintf (dump_file,
- "Flattening %s\n", cgraph_node_name (node));
+ "Flattening %s\n", node->name ());
flatten_function (node, false);
}
}
+ if (dump_file)
+ dump_overall_stats ();
inline_small_functions ();
- symtab_remove_unreachable_nodes (true, dump_file);
+
+ /* Do first after-inlining removal. We want to remove all "stale" extern inline
+ functions and virtual functions so we really know what is called once. */
+ symtab->remove_unreachable_nodes (false, dump_file);
free (order);
/* Inline functions with a property that after inlining into all callers the
code size will shrink because the out-of-line copy is eliminated.
We do this regardless on the callee size as long as function growth limits
are met. */
- if (flag_inline_functions_called_once)
+ if (dump_file)
+ fprintf (dump_file,
+ "\nDeciding on functions to be inlined into all callers and removing useless speculations:\n");
+
+ /* Inlining one function called once has good chance of preventing
+ inlining other function into the same callee. Ideally we should
+ work in priority order, but probably inlining hot functions first
+ is good cut without the extra pain of maintaining the queue.
+
+ ??? this is not really fitting the bill perfectly: inlining function
+ into callee often leads to better optimization of callee due to
+ increased context for optimization.
+ For example if main() function calls a function that outputs help
+ and then function that does the main optmization, we should inline
+ the second with priority even if both calls are cold by themselves.
+
+ We probably want to implement new predicate replacing our use of
+ maybe_hot_edge interpreted as maybe_hot_edge || callee is known
+ to be hot. */
+ for (cold = 0; cold <= 1; cold ++)
{
- int cold;
- if (dump_file)
- fprintf (dump_file,
- "\nDeciding on functions to be inlined into all callers:\n");
-
- /* Inlining one function called once has good chance of preventing
- inlining other function into the same callee. Ideally we should
- work in priority order, but probably inlining hot functions first
- is good cut without the extra pain of maintaining the queue.
-
- ??? this is not really fitting the bill perfectly: inlining function
- into callee often leads to better optimization of callee due to
- increased context for optimization.
- For example if main() function calls a function that outputs help
- and then function that does the main optmization, we should inline
- the second with priority even if both calls are cold by themselves.
-
- We probably want to implement new predicate replacing our use of
- maybe_hot_edge interpreted as maybe_hot_edge || callee is known
- to be hot. */
- for (cold = 0; cold <= 1; cold ++)
+ FOR_EACH_DEFINED_FUNCTION (node)
{
- FOR_EACH_DEFINED_FUNCTION (node)
+ struct cgraph_edge *edge, *next;
+ bool update=false;
+
+ for (edge = node->callees; edge; edge = next)
{
- if (want_inline_function_to_all_callers_p (node, cold))
+ next = edge->next_callee;
+ if (edge->speculative && !speculation_useful_p (edge, false))
{
- int num_calls = 0;
- struct cgraph_edge *e;
- for (e = node->callers; e; e = e->next_caller)
- num_calls++;
- while (node->callers && !node->global.inlined_to)
- {
- struct cgraph_node *caller = node->callers->caller;
-
- if (dump_file)
- {
- fprintf (dump_file,
- "\nInlining %s size %i.\n",
- cgraph_node_name (node),
- inline_summary (node)->size);
- fprintf (dump_file,
- " Called once from %s %i insns.\n",
- cgraph_node_name (node->callers->caller),
- inline_summary (node->callers->caller)->size);
- }
-
- inline_call (node->callers, true, NULL, NULL, true);
- if (dump_file)
- fprintf (dump_file,
- " Inlined into %s which now has %i size\n",
- cgraph_node_name (caller),
- inline_summary (caller)->size);
- if (!num_calls--)
- {
- if (dump_file)
- fprintf (dump_file, "New calls found; giving up.\n");
- break;
- }
- }
+ edge->resolve_speculation ();
+ spec_rem += edge->count;
+ update = true;
+ remove_functions = true;
}
}
+ if (update)
+ {
+ struct cgraph_node *where = node->global.inlined_to
+ ? node->global.inlined_to : node;
+ reset_node_growth_cache (where);
+ reset_edge_caches (where);
+ inline_update_overall_summary (where);
+ }
+ if (flag_inline_functions_called_once
+ && want_inline_function_to_all_callers_p (node, cold))
+ {
+ int num_calls = 0;
+ node->call_for_symbol_thunks_and_aliases (sum_callers, &num_calls,
+ true);
+ while (node->call_for_symbol_thunks_and_aliases (inline_to_all_callers,
+ &num_calls, true))
+ ;
+ remove_functions = true;
+ }
}
}
ipa_free_all_structures_after_iinln ();
if (dump_file)
- fprintf (dump_file,
- "\nInlined %i calls, eliminated %i functions\n\n",
- ncalls_inlined, nfunctions_inlined);
+ {
+ fprintf (dump_file,
+ "\nInlined %i calls, eliminated %i functions\n\n",
+ ncalls_inlined, nfunctions_inlined);
+ dump_inline_stats ();
+ }
if (dump_file)
dump_inline_summaries (dump_file);
/* In WPA we use inline summaries for partitioning process. */
if (!flag_wpa)
inline_free_summary ();
- return 0;
+ return remove_functions ? TODO_remove_functions : 0;
}
/* Inline always-inline function calls in NODE. */
for (e = node->callees; e; e = e->next_callee)
{
- struct cgraph_node *callee = cgraph_function_or_thunk_node (e->callee, NULL);
- if (!DECL_DISREGARD_INLINE_LIMITS (callee->symbol.decl))
+ struct cgraph_node *callee = e->callee->ultimate_alias_target ();
+ if (!DECL_DISREGARD_INLINE_LIMITS (callee->decl))
continue;
- if (cgraph_edge_recursive_p (e))
+ if (e->recursive_p ())
{
if (dump_file)
fprintf (dump_file, " Not inlining recursive call to %s.\n",
- cgraph_node_name (e->callee));
+ e->callee->name ());
e->inline_failed = CIF_RECURSIVE_INLINING;
continue;
}
if (!can_early_inline_edge_p (e))
- continue;
+ {
+ /* Set inlined to true if the callee is marked "always_inline" but
+ is not inlinable. This will allow flagging an error later in
+ expand_call_inline in tree-inline.c. */
+ if (lookup_attribute ("always_inline",
+ DECL_ATTRIBUTES (callee->decl)) != NULL)
+ inlined = true;
+ continue;
+ }
if (dump_file)
fprintf (dump_file, " Inlining %s into %s (always_inline).\n",
- xstrdup (cgraph_node_name (e->callee)),
- xstrdup (cgraph_node_name (e->caller)));
+ xstrdup (e->callee->name ()),
+ xstrdup (e->caller->name ()));
inline_call (e, true, NULL, NULL, false);
inlined = true;
}
for (e = node->callees; e; e = e->next_callee)
{
- struct cgraph_node *callee = cgraph_function_or_thunk_node (e->callee, NULL);
+ struct cgraph_node *callee = e->callee->ultimate_alias_target ();
if (!inline_summary (callee)->inlinable
|| !e->inline_failed)
continue;
/* Do not consider functions not declared inline. */
- if (!DECL_DECLARED_INLINE_P (callee->symbol.decl)
+ if (!DECL_DECLARED_INLINE_P (callee->decl)
&& !flag_inline_small_functions
&& !flag_inline_functions)
continue;
if (dump_file)
fprintf (dump_file, "Considering inline candidate %s.\n",
- cgraph_node_name (callee));
+ callee->name ());
if (!can_early_inline_edge_p (e))
continue;
- if (cgraph_edge_recursive_p (e))
+ if (e->recursive_p ())
{
if (dump_file)
fprintf (dump_file, " Not inlining: recursive call.\n");
if (dump_file)
fprintf (dump_file, " Inlining %s into %s.\n",
- xstrdup (cgraph_node_name (callee)),
- xstrdup (cgraph_node_name (e->caller)));
+ xstrdup (callee->name ()),
+ xstrdup (e->caller->name ()));
inline_call (e, true, NULL, NULL, true);
inlined = true;
}
return inlined;
}
-/* Do inlining of small functions. Doing so early helps profiling and other
- passes to be somewhat more effective and avoids some code duplication in
- later real inlining pass for testcases with very many function calls. */
-static unsigned int
-early_inliner (void)
+unsigned int
+early_inliner (function *fun)
{
- struct cgraph_node *node = cgraph_get_node (current_function_decl);
+ struct cgraph_node *node = cgraph_node::get (current_function_decl);
struct cgraph_edge *edge;
unsigned int todo = 0;
int iterations = 0;
return 0;
#ifdef ENABLE_CHECKING
- verify_cgraph_node (node);
+ node->verify ();
#endif
+ node->remove_all_references ();
/* Even when not optimizing or not inlining inline always-inline
functions. */
cycles of edges to be always inlined in the callgraph.
We might want to be smarter and just avoid this type of inlining. */
- || DECL_DISREGARD_INLINE_LIMITS (node->symbol.decl))
+ || DECL_DISREGARD_INLINE_LIMITS (node->decl))
;
else if (lookup_attribute ("flatten",
- DECL_ATTRIBUTES (node->symbol.decl)) != NULL)
+ DECL_ATTRIBUTES (node->decl)) != NULL)
{
/* When the function is marked to be flattened, recursively inline
all calls in it. */
if (dump_file)
fprintf (dump_file,
- "Flattening %s\n", cgraph_node_name (node));
+ "Flattening %s\n", node->name ());
flatten_function (node, true);
inlined = true;
}
info that might be cleared out for newly discovered edges. */
for (edge = node->callees; edge; edge = edge->next_callee)
{
- struct inline_edge_summary *es = inline_edge_summary (edge);
- es->call_stmt_size
- = estimate_num_insns (edge->call_stmt, &eni_size_weights);
- es->call_stmt_time
- = estimate_num_insns (edge->call_stmt, &eni_time_weights);
- if (edge->callee->symbol.decl
- && !gimple_check_call_matching_types (edge->call_stmt,
- edge->callee->symbol.decl))
+ /* We have no summary for new bound store calls yet. */
+ if (inline_edge_summary_vec.length () > (unsigned)edge->uid)
+ {
+ struct inline_edge_summary *es = inline_edge_summary (edge);
+ es->call_stmt_size
+ = estimate_num_insns (edge->call_stmt, &eni_size_weights);
+ es->call_stmt_time
+ = estimate_num_insns (edge->call_stmt, &eni_time_weights);
+ }
+ if (edge->callee->decl
+ && !gimple_check_call_matching_types (
+ edge->call_stmt, edge->callee->decl, false))
edge->call_stmt_cannot_inline_p = true;
}
+ if (iterations < PARAM_VALUE (PARAM_EARLY_INLINER_MAX_ITERATIONS) - 1)
+ inline_update_overall_summary (node);
timevar_pop (TV_INTEGRATION);
iterations++;
inlined = false;
timevar_pop (TV_INTEGRATION);
}
- cfun->always_inline_functions_inlined = true;
+ fun->always_inline_functions_inlined = true;
return todo;
}
-struct gimple_opt_pass pass_early_inline =
+/* Do inlining of small functions. Doing so early helps profiling and other
+ passes to be somewhat more effective and avoids some code duplication in
+ later real inlining pass for testcases with very many function calls. */
+
+namespace {
+
+const pass_data pass_data_early_inline =
{
- {
- GIMPLE_PASS,
- "einline", /* name */
- OPTGROUP_INLINE, /* optinfo_flags */
- NULL, /* gate */
- early_inliner, /* execute */
- NULL, /* sub */
- NULL, /* next */
- 0, /* static_pass_number */
- TV_EARLY_INLINING, /* tv_id */
- PROP_ssa, /* properties_required */
- 0, /* properties_provided */
- 0, /* properties_destroyed */
- 0, /* todo_flags_start */
- 0 /* todo_flags_finish */
- }
+ GIMPLE_PASS, /* type */
+ "einline", /* name */
+ OPTGROUP_INLINE, /* optinfo_flags */
+ TV_EARLY_INLINING, /* tv_id */
+ PROP_ssa, /* properties_required */
+ 0, /* properties_provided */
+ 0, /* properties_destroyed */
+ 0, /* todo_flags_start */
+ 0, /* todo_flags_finish */
};
+class pass_early_inline : public gimple_opt_pass
+{
+public:
+ pass_early_inline (gcc::context *ctxt)
+ : gimple_opt_pass (pass_data_early_inline, ctxt)
+ {}
-/* When to run IPA inlining. Inlining of always-inline functions
- happens during early inlining.
+ /* opt_pass methods: */
+ virtual unsigned int execute (function *);
- Enable inlining unconditoinally at -flto. We need size estimates to
- drive partitioning. */
+}; // class pass_early_inline
-static bool
-gate_ipa_inline (void)
+unsigned int
+pass_early_inline::execute (function *fun)
{
- return optimize || flag_lto || flag_wpa;
+ return early_inliner (fun);
}
-struct ipa_opt_pass_d pass_ipa_inline =
+} // anon namespace
+
+gimple_opt_pass *
+make_pass_early_inline (gcc::context *ctxt)
{
- {
- IPA_PASS,
- "inline", /* name */
- OPTGROUP_INLINE, /* optinfo_flags */
- gate_ipa_inline, /* gate */
- ipa_inline, /* execute */
- NULL, /* sub */
- NULL, /* next */
- 0, /* static_pass_number */
- TV_IPA_INLINING, /* tv_id */
- 0, /* properties_required */
- 0, /* properties_provided */
- 0, /* properties_destroyed */
- TODO_remove_functions, /* todo_flags_finish */
- TODO_dump_symtab
- | TODO_remove_functions | TODO_ggc_collect /* todo_flags_finish */
- },
- inline_generate_summary, /* generate_summary */
- inline_write_summary, /* write_summary */
- inline_read_summary, /* read_summary */
- NULL, /* write_optimization_summary */
- NULL, /* read_optimization_summary */
- NULL, /* stmt_fixup */
- 0, /* TODOs */
- inline_transform, /* function_transform */
- NULL, /* variable_transform */
+ return new pass_early_inline (ctxt);
+}
+
+namespace {
+
+const pass_data pass_data_ipa_inline =
+{
+ IPA_PASS, /* type */
+ "inline", /* name */
+ OPTGROUP_INLINE, /* optinfo_flags */
+ TV_IPA_INLINING, /* tv_id */
+ 0, /* properties_required */
+ 0, /* properties_provided */
+ 0, /* properties_destroyed */
+ 0, /* todo_flags_start */
+ ( TODO_dump_symtab ), /* todo_flags_finish */
};
+
+class pass_ipa_inline : public ipa_opt_pass_d
+{
+public:
+ pass_ipa_inline (gcc::context *ctxt)
+ : ipa_opt_pass_d (pass_data_ipa_inline, ctxt,
+ inline_generate_summary, /* generate_summary */
+ inline_write_summary, /* write_summary */
+ inline_read_summary, /* read_summary */
+ NULL, /* write_optimization_summary */
+ NULL, /* read_optimization_summary */
+ NULL, /* stmt_fixup */
+ 0, /* function_transform_todo_flags_start */
+ inline_transform, /* function_transform */
+ NULL) /* variable_transform */
+ {}
+
+ /* opt_pass methods: */
+ virtual unsigned int execute (function *) { return ipa_inline (); }
+
+}; // class pass_ipa_inline
+
+} // anon namespace
+
+ipa_opt_pass_d *
+make_pass_ipa_inline (gcc::context *ctxt)
+{
+ return new pass_ipa_inline (ctxt);
+}
projects / thirdparty / gcc.git / blobdiff