/* Inlining decision heuristics.
- Copyright (C) 2003-2019 Free Software Foundation, Inc.
+ Copyright (C) 2003-2021 Free Software Foundation, Inc.
Contributed by Jan Hubicka
This file is part of GCC.
if (!caller || !callee)
return true;
- /* Allow inlining always_inline functions into no_sanitize_address
- functions. */
- if (!sanitize_flags_p (SANITIZE_ADDRESS, caller)
- && lookup_attribute ("always_inline", DECL_ATTRIBUTES (callee)))
+ /* Follow clang and allow inlining for always_inline functions. */
+ if (lookup_attribute ("always_inline", DECL_ATTRIBUTES (callee)))
return true;
- return ((sanitize_flags_p (SANITIZE_ADDRESS, caller)
- == sanitize_flags_p (SANITIZE_ADDRESS, callee))
- && (sanitize_flags_p (SANITIZE_POINTER_COMPARE, caller)
- == sanitize_flags_p (SANITIZE_POINTER_COMPARE, callee))
- && (sanitize_flags_p (SANITIZE_POINTER_SUBTRACT, caller)
- == sanitize_flags_p (SANITIZE_POINTER_SUBTRACT, callee)));
+ const sanitize_code codes[] =
+ {
+ SANITIZE_ADDRESS,
+ SANITIZE_THREAD,
+ SANITIZE_UNDEFINED,
+ SANITIZE_UNDEFINED_NONDEFAULT,
+ SANITIZE_POINTER_COMPARE,
+ SANITIZE_POINTER_SUBTRACT
+ };
+
+ for (unsigned i = 0; i < sizeof (codes) / sizeof (codes[0]); i++)
+ if (sanitize_flags_p (codes[i], caller)
+ != sanitize_flags_p (codes[i], callee))
+ return false;
+
+ if (sanitize_coverage_p (caller) != sanitize_coverage_p (callee))
+ return false;
+
+ return true;
}
/* Used for flags where it is safe to inline when caller's value is
/* Don't inline a function with mismatched sanitization attributes. */
else if (!sanitize_attrs_match_for_inline_p (caller->decl, callee->decl))
{
- e->inline_failed = CIF_ATTRIBUTE_MISMATCH;
+ e->inline_failed = CIF_SANITIZE_ATTRIBUTE_MISMATCH;
inlinable = false;
}
if (!inlinable && report)
return inlinable;
}
-/* Return inlining_insns_single limit for function N. If HINT is true
+/* Return inlining_insns_single limit for function N. If HINT or HINT2 is true
scale up the bound. */
static int
-inline_insns_single (cgraph_node *n, bool hint)
+inline_insns_single (cgraph_node *n, bool hint, bool hint2)
{
- if (hint)
+ if (hint && hint2)
+ {
+ int64_t spd = opt_for_fn (n->decl, param_inline_heuristics_hint_percent);
+ spd = spd * spd;
+ if (spd > 1000000)
+ spd = 1000000;
+ return opt_for_fn (n->decl, param_max_inline_insns_single) * spd / 100;
+ }
+ if (hint || hint2)
return opt_for_fn (n->decl, param_max_inline_insns_single)
* opt_for_fn (n->decl, param_inline_heuristics_hint_percent) / 100;
return opt_for_fn (n->decl, param_max_inline_insns_single);
}
-/* Return inlining_insns_auto limit for function N. If HINT is true
+/* Return inlining_insns_auto limit for function N. If HINT or HINT2 is true
scale up the bound. */
static int
-inline_insns_auto (cgraph_node *n, bool hint)
+inline_insns_auto (cgraph_node *n, bool hint, bool hint2)
{
int max_inline_insns_auto = opt_for_fn (n->decl, param_max_inline_insns_auto);
- if (hint)
+ if (hint && hint2)
+ {
+ int64_t spd = opt_for_fn (n->decl, param_inline_heuristics_hint_percent);
+ spd = spd * spd;
+ if (spd > 1000000)
+ spd = 1000000;
+ return max_inline_insns_auto * spd / 100;
+ }
+ if (hint || hint2)
return max_inline_insns_auto
* opt_for_fn (n->decl, param_inline_heuristics_hint_percent) / 100;
return max_inline_insns_auto;
else if (check_match (flag_wrapv)
|| check_match (flag_trapv)
|| check_match (flag_pcc_struct_return)
+ || check_maybe_down (optimize_debug)
/* When caller or callee does FP math, be sure FP codegen flags
compatible. */
|| ((caller_info->fp_expressions && callee_info->fp_expressions)
int growth = estimate_edge_growth (e);
if (growth > opt_for_fn (caller->decl, param_max_inline_insns_size)
&& (!DECL_DECLARED_INLINE_P (callee->decl)
- && growth >= MAX (inline_insns_single (caller, false),
- inline_insns_auto (caller, false))))
+ && growth >= MAX (inline_insns_single (caller, false, false),
+ inline_insns_auto (caller, false, false))))
{
e->inline_failed = CIF_OPTIMIZATION_MISMATCH;
inlinable = false;
}
/* Return true if the speedup for inlining E is bigger than
- PARAM_MAX_INLINE_MIN_SPEEDUP. */
+ param_inline_min_speedup. */
static bool
big_speedup_p (struct cgraph_edge *e)
&& (!e->count.ipa ().initialized_p () || !e->maybe_hot_p ()))
&& ipa_fn_summaries->get (callee)->min_size
- ipa_call_summaries->get (e)->call_stmt_size
- > inline_insns_auto (e->caller, true))
+ > inline_insns_auto (e->caller, true, true))
{
e->inline_failed = CIF_MAX_INLINE_INSNS_AUTO_LIMIT;
want_inline = false;
|| e->count.ipa ().nonzero_p ())
&& ipa_fn_summaries->get (callee)->min_size
- ipa_call_summaries->get (e)->call_stmt_size
- > inline_insns_single (e->caller, true))
+ > inline_insns_single (e->caller, true, true))
{
e->inline_failed = (DECL_DECLARED_INLINE_P (callee->decl)
? CIF_MAX_INLINE_INSNS_SINGLE_LIMIT
{
int growth = estimate_edge_growth (e);
ipa_hints hints = estimate_edge_hints (e);
+ /* We have two independent groups of hints. If one matches in each
+ of groups the limits are inreased. If both groups matches, limit
+ is increased even more. */
bool apply_hints = (hints & (INLINE_HINT_indirect_call
| INLINE_HINT_known_hot
| INLINE_HINT_loop_iterations
| INLINE_HINT_loop_stride));
+ bool apply_hints2 = (hints & INLINE_HINT_builtin_constant_p);
if (growth <= opt_for_fn (to->decl,
param_max_inline_insns_size))
Avoid computation of big_speedup_p when not necessary to change
outcome of decision. */
else if (DECL_DECLARED_INLINE_P (callee->decl)
- && growth >= inline_insns_single (e->caller, apply_hints)
- && (apply_hints
- || growth >= inline_insns_single (e->caller, true)
+ && growth >= inline_insns_single (e->caller, apply_hints,
+ apply_hints2)
+ && (apply_hints || apply_hints2
+ || growth >= inline_insns_single (e->caller, true,
+ apply_hints2)
|| !big_speedup_p (e)))
{
e->inline_failed = CIF_MAX_INLINE_INSNS_SINGLE_LIMIT;
param_max_inline_insns_small))
{
/* growth_positive_p is expensive, always test it last. */
- if (growth >= inline_insns_single (e->caller, false)
+ if (growth >= inline_insns_single (e->caller, false, false)
|| growth_positive_p (callee, e, growth))
{
e->inline_failed = CIF_NOT_DECLARED_INLINED;
/* Apply param_max_inline_insns_auto limit for functions not declared
inline. Bypass the limit when speedup seems big. */
else if (!DECL_DECLARED_INLINE_P (callee->decl)
- && growth >= inline_insns_auto (e->caller, apply_hints)
- && (apply_hints
- || growth >= inline_insns_auto (e->caller, true)
+ && growth >= inline_insns_auto (e->caller, apply_hints,
+ apply_hints2)
+ && (apply_hints || apply_hints2
+ || growth >= inline_insns_auto (e->caller, true,
+ apply_hints2)
|| !big_speedup_p (e)))
{
/* growth_positive_p is expensive, always test it last. */
- if (growth >= inline_insns_single (e->caller, false)
+ if (growth >= inline_insns_single (e->caller, false, false)
|| growth_positive_p (callee, e, growth))
{
e->inline_failed = CIF_MAX_INLINE_INSNS_AUTO_LIMIT;
}
/* If call is cold, do not inline when function body would grow. */
else if (!e->maybe_hot_p ()
- && (growth >= inline_insns_single (e->caller, false)
+ && (growth >= inline_insns_single (e->caller, false, false)
|| growth_positive_p (callee, e, growth)))
{
e->inline_failed = CIF_UNLIKELY_CALL;
wrapper_heuristics_may_apply (struct cgraph_node *where, int size)
{
return size < (DECL_DECLARED_INLINE_P (where->decl)
- ? inline_insns_single (where, false)
- : inline_insns_auto (where, false));
+ ? inline_insns_single (where, false, false)
+ : inline_insns_auto (where, false, false));
}
/* A cost model driving the inlining heuristics in a way so the edges with
| INLINE_HINT_loop_stride))
|| callee_info->growth <= 0)
badness = badness.shift (badness > 0 ? -2 : 2);
+ if (hints & INLINE_HINT_builtin_constant_p)
+ badness = badness.shift (badness > 0 ? -4 : 4);
if (hints & (INLINE_HINT_same_scc))
badness = badness.shift (badness > 0 ? 3 : -3);
else if (hints & (INLINE_HINT_in_scc))
if (dump_file)
fprintf (dump_file,
- " Performing recursive inlining on %s\n",
- node->name ());
+ " Performing recursive inlining on %s\n", node->dump_name ());
/* Do the inlining and update list of recursive call during process. */
while (!heap.empty ())
/* Given whole compilation unit estimate of INSNS, compute how large we can
allow the unit to grow. */
-static int
+static int64_t
compute_max_insns (cgraph_node *node, int insns)
{
int max_insns = insns;
/* Compute badness of all edges in NEW_EDGES and add them to the HEAP. */
static void
-add_new_edges_to_heap (edge_heap_t *heap, vec<cgraph_edge *> new_edges)
+add_new_edges_to_heap (edge_heap_t *heap, vec<cgraph_edge *> &new_edges)
{
while (new_edges.length () > 0)
{
enum availability avail;
struct cgraph_node *target = e->callee->ultimate_alias_target (&avail,
e->caller);
- struct cgraph_edge *direct, *indirect;
- struct ipa_ref *ref;
gcc_assert (e->speculative && !e->indirect_unknown_callee);
int ecf_flags = flags_from_decl_or_type (target->decl);
if (ecf_flags & ECF_CONST)
{
- e->speculative_call_info (direct, indirect, ref);
- if (!(indirect->indirect_info->ecf_flags & ECF_CONST))
+ if (!(e->speculative_call_indirect_edge ()->indirect_info
+ ->ecf_flags & ECF_CONST))
return true;
}
else if (ecf_flags & ECF_PURE)
{
- e->speculative_call_info (direct, indirect, ref);
- if (!(indirect->indirect_info->ecf_flags & ECF_PURE))
+ if (!(e->speculative_call_indirect_edge ()->indirect_info
+ ->ecf_flags & ECF_PURE))
return true;
}
}
if (edge->count.ipa ().initialized_p ())
spec_rem += edge->count.ipa ();
- edge->resolve_speculation ();
+ cgraph_edge::resolve_speculation (edge);
reset_edge_caches (where);
ipa_update_overall_fn_summary (where);
update_caller_keys (edge_heap, where,
if (!node->inlined_to)
{
if (!node->alias && node->analyzed
- && (node->has_gimple_body_p () || node->thunk.thunk_p)
+ && (node->has_gimple_body_p () || node->thunk)
&& opt_for_fn (node->decl, optimize))
{
class ipa_fn_summary *info = ipa_fn_summaries->get (node);
if (dump_file)
fprintf (dump_file, "Enqueueing calls in %s.\n", node->dump_name ());
- for (edge = node->callees; edge; edge = next)
+ for (edge = node->callees; edge; edge = edge->next_callee)
{
- next = edge->next_callee;
if (edge->inline_failed
&& !edge->aux
&& can_inline_edge_p (edge, true)
if (edge->speculative
&& !speculation_useful_p (edge, edge->aux != NULL))
{
- edge->resolve_speculation ();
+ cgraph_edge::resolve_speculation (edge);
update = true;
}
}
dump_printf_loc (MSG_OPTIMIZED_LOCATIONS, edge->call_stmt,
" Inlined %C into %C which now has time %f and "
- "size %i, net change of %s.\n",
+ "size %i, net change of %s%s.\n",
edge->callee, edge->caller,
s->time.to_double (),
ipa_size_summaries->get (edge->caller)->size,
- buf_net_change);
+ buf_net_change,
+ cross_module_call_p (edge) ? " (cross module)":"");
}
if (min_size > overall_size)
{
cgraph_node *ultimate = node->ultimate_alias_target ();
fprintf (dump_file,
"\nInlining %s size %i.\n",
- ultimate->name (),
+ ultimate->dump_name (),
ipa_size_summaries->get (ultimate)->size);
fprintf (dump_file,
" Called once from %s %i insns.\n",
- node->callers->caller->name (),
+ node->callers->caller->dump_name (),
ipa_size_summaries->get (node->callers->caller)->size);
}
if (dump_file)
fprintf (dump_file,
" Inlined into %s which now has %i size\n",
- caller->name (),
+ caller->dump_name (),
ipa_size_summaries->get (caller)->size);
if (!(*num_calls)--)
{
{
node = order[i];
if (node->definition
+ /* Do not try to flatten aliases. These may happen for example when
+ creating local aliases. */
+ && !node->alias
&& lookup_attribute ("flatten",
DECL_ATTRIBUTES (node->decl)) != NULL)
order[j--] = order[i];
try to flatten itself turning it into a self-recursive
function. */
if (dump_file)
- fprintf (dump_file, "Flattening %s\n", node->name ());
+ fprintf (dump_file, "Flattening %s\n", node->dump_name ());
flatten_function (node, false, true);
}
{
if (edge->count.ipa ().initialized_p ())
spec_rem += edge->count.ipa ();
- edge->resolve_speculation ();
+ cgraph_edge::resolve_speculation (edge);
update = true;
remove_functions = true;
}
}
}
- /* Free ipa-prop structures if they are no longer needed. */
- ipa_free_all_structures_after_iinln ();
-
if (dump_enabled_p ())
dump_printf (MSG_NOTE,
"\nInlined %i calls, eliminated %i functions\n\n",
}
/* We iterate incremental inlining to get trivial cases of indirect
inlining. */
- while (iterations < param_early_inliner_max_iterations
+ while (iterations < opt_for_fn (node->decl,
+ param_early_inliner_max_iterations)
&& early_inline_small_functions (node))
{
timevar_push (TV_INTEGRATION);
es->call_stmt_time
= estimate_num_insns (edge->call_stmt, &eni_time_weights);
}
- if (iterations < param_early_inliner_max_iterations - 1)
+ if (iterations < opt_for_fn (node->decl,
+ param_early_inliner_max_iterations) - 1)
ipa_update_overall_fn_summary (node);
timevar_pop (TV_INTEGRATION);
iterations++;
projects / thirdparty / gcc.git / blobdiff