/* Read and annotate call graph profile from the auto profile data file.
- Copyright (C) 2014-2017 Free Software Foundation, Inc.
+ Copyright (C) 2014-2020 Free Software Foundation, Inc.
Contributed by Dehao Chen (dehao@google.com)
This file is part of GCC.
#include "tree-into-ssa.h"
#include "gimple-iterator.h"
#include "value-prof.h"
-#include "params.h"
#include "symbol-summary.h"
#include "ipa-prop.h"
#include "ipa-fnsummary.h"
namespace autofdo
{
+/* Intermediate edge info used when propagating AutoFDO profile information.
+ We can't edge->count() directly since it's computed from edge's probability
+ while probability is yet not decided during propagation. */
+#define AFDO_EINFO(e) ((class edge_info *) e->aux)
+class edge_info
+{
+public:
+ edge_info () : count_ (profile_count::zero ().afdo ()), annotated_ (false) {}
+ bool is_annotated () const { return annotated_; }
+ void set_annotated () { annotated_ = true; }
+ profile_count get_count () const { return count_; }
+ void set_count (profile_count count) { count_ = count; }
+private:
+ profile_count count_;
+ bool annotated_;
+};
+
/* Represent a source location: (function_decl, lineno). */
typedef std::pair<tree, unsigned> decl_lineno;
typedef std::set<gimple *> stmt_set;
/* Represent count info of an inline stack. */
-struct count_info
+class count_info
{
+public:
/* Sampled count of the inline stack. */
gcov_type count;
/* Store the AutoFDO source profile. */
static autofdo_source_profile *afdo_source_profile;
-/* gcov_ctr_summary structure to store the profile_info. */
-static struct gcov_ctr_summary *afdo_profile_info;
+/* gcov_summary structure to store the profile_info. */
+static gcov_summary *afdo_profile_info;
/* Helper functions. */
static tree
get_function_decl_from_block (tree block)
{
- tree decl;
-
- if (LOCATION_LOCUS (BLOCK_SOURCE_LOCATION (block)) == UNKNOWN_LOCATION)
+ if (!inlined_function_outer_scope_p (block))
return NULL_TREE;
- for (decl = BLOCK_ABSTRACT_ORIGIN (block);
- decl && (TREE_CODE (decl) == BLOCK);
- decl = BLOCK_ABSTRACT_ORIGIN (decl))
- if (TREE_CODE (decl) == FUNCTION_DECL)
- break;
- return decl;
+ return BLOCK_ABSTRACT_ORIGIN (block);
}
/* Store inline stack for STMT in STACK. */
ret = get_index (lang_hooks.dwarf_name (decl, 0));
if (ret != -1)
return ret;
- if (DECL_ABSTRACT_ORIGIN (decl) && DECL_ABSTRACT_ORIGIN (decl) != decl)
+ if (DECL_FROM_INLINE (decl))
return get_index_by_decl (DECL_ABSTRACT_ORIGIN (decl));
return -1;
if (ret != callsites.end ())
return ret->second;
}
- if (DECL_ABSTRACT_ORIGIN (decl))
+ if (DECL_FROM_INLINE (decl))
return get_function_instance_by_decl (lineno, DECL_ABSTRACT_ORIGIN (decl));
return NULL;
get_identifier (afdo_string_table->get_name (callee)));
if (node == NULL)
continue;
- if (!check_ic_target (stmt, node))
- continue;
(*map)[callee] = iter->second->total_count ();
ret += iter->second->total_count ();
}
function_instance::function_instance_stack stack;
function_instance *s = function_instance::read_function_instance (
&stack, gcov_read_counter ());
- afdo_profile_info->sum_all += s->total_count ();
map_[s->name ()] = s;
}
return true;
/* autofdo_module_profile. */
fake_read_autofdo_module_profile ();
-
- /* Read in the working set. */
- if (gcov_read_unsigned () != GCOV_TAG_AFDO_WORKING_SET)
- {
- error ("cannot read working set from %s", auto_profile_file);
- return;
- }
-
- /* Skip the length of the section. */
- gcov_read_unsigned ();
- gcov_working_set_t set[128];
- for (unsigned i = 0; i < 128; i++)
- {
- set[i].num_counters = gcov_read_unsigned ();
- set[i].min_counter = gcov_read_counter ();
- }
- add_working_set (set);
}
/* From AutoFDO profiles, find values inside STMT for that we want to measure
if (map.size () == 0)
return;
gcall *stmt = dyn_cast <gcall *> (gs);
- if ((!stmt) || gimple_call_fndecl (stmt) != NULL_TREE)
+ if (!stmt
+ || gimple_call_internal_p (stmt)
+ || gimple_call_fndecl (stmt) != NULL_TREE)
return;
- callee = gimple_call_fn (stmt);
-
- histogram_value hist = gimple_alloc_histogram_value (
- cfun, HIST_TYPE_INDIR_CALL, stmt, callee);
- hist->n_counters = 3;
- hist->hvalue.counters = XNEWVEC (gcov_type, hist->n_counters);
- gimple_add_histogram_value (cfun, stmt, hist);
-
gcov_type total = 0;
icall_target_map::const_iterator max_iter = map.end ();
if (max_iter == map.end () || max_iter->second < iter->second)
max_iter = iter;
}
+ struct cgraph_node *direct_call = cgraph_node::get_for_asmname (
+ get_identifier (afdo_string_table->get_name (max_iter->first)));
+ if (direct_call == NULL || !direct_call->profile_id)
+ return;
+
+ callee = gimple_call_fn (stmt);
+
+ histogram_value hist = gimple_alloc_histogram_value (
+ cfun, HIST_TYPE_INDIR_CALL, stmt, callee);
+ hist->n_counters = 3;
+ hist->hvalue.counters = XNEWVEC (gcov_type, hist->n_counters);
+ gimple_add_histogram_value (cfun, stmt, hist);
- hist->hvalue.counters[0]
- = (unsigned long long)afdo_string_table->get_name (max_iter->first);
+ hist->hvalue.counters[0] = direct_call->profile_id;
hist->hvalue.counters[1] = max_iter->second;
hist->hvalue.counters[2] = total;
struct cgraph_edge *indirect_edge
= cgraph_node::get (current_function_decl)->get_edge (stmt);
- struct cgraph_node *direct_call = cgraph_node::get_for_asmname (
- get_identifier ((const char *) hist->hvalue.counters[0]));
if (dump_file)
{
print_generic_expr (dump_file, direct_call->decl, TDF_SLIM);
}
- if (direct_call == NULL || !check_ic_target (stmt, direct_call))
+ if (direct_call == NULL)
{
if (dump_file)
fprintf (dump_file, " not transforming\n");
struct cgraph_edge *new_edge
= indirect_edge->make_speculative (direct_call,
profile_count::uninitialized ());
- new_edge->redirect_call_stmt_to_callee ();
+ cgraph_edge::redirect_call_stmt_to_callee (new_edge);
gimple_remove_histogram_value (cfun, stmt, hist);
inline_call (new_edge, true, NULL, NULL, false);
}
annotated->insert (bb);
}
-static bool
-is_edge_annotated (const edge e, const edge_set &annotated)
-{
- return annotated.find (e) != annotated.end ();
-}
-
-static void
-set_edge_annotated (edge e, edge_set *annotated)
-{
- annotated->insert (e);
-}
-
/* For a given BB, set its execution count. Attach value profile if a stmt
is not in PROMOTED, because we only want to promote an indirect call once.
Return TRUE if BB is annotated. */
edges' counts are known, then the basic block's unknown count can also be
calculated.
IS_SUCC is true if out edges of a basic blocks are examined.
- Update ANNOTATED_BB and ANNOTATED_EDGE accordingly.
+ Update ANNOTATED_BB accordingly.
Return TRUE if any basic block/edge count is changed. */
static bool
-afdo_propagate_edge (bool is_succ, bb_set *annotated_bb,
- edge_set *annotated_edge)
+afdo_propagate_edge (bool is_succ, bb_set *annotated_bb)
{
basic_block bb;
bool changed = false;
profile_count total_known_count = profile_count::zero ().afdo ();
FOR_EACH_EDGE (e, ei, is_succ ? bb->succs : bb->preds)
- if (!is_edge_annotated (e, *annotated_edge))
- num_unknown_edge++, unknown_edge = e;
- else
- total_known_count += e->count ();
+ {
+ gcc_assert (AFDO_EINFO (e) != NULL);
+ if (! AFDO_EINFO (e)->is_annotated ())
+ num_unknown_edge++, unknown_edge = e;
+ else
+ total_known_count += AFDO_EINFO (e)->get_count ();
+ }
- if (num_unknown_edge == 0)
+ /* Be careful not to annotate block with no successor in special cases. */
+ if (num_unknown_edge == 0 && total_known_count > bb->count)
{
- if (total_known_count > bb->count)
- {
- bb->count = total_known_count;
- changed = true;
- }
- if (!is_bb_annotated (bb, *annotated_bb))
- {
- set_bb_annotated (bb, annotated_bb);
- changed = true;
- }
+ bb->count = total_known_count;
+ if (!is_bb_annotated (bb, *annotated_bb))
+ set_bb_annotated (bb, annotated_bb);
+ changed = true;
}
else if (num_unknown_edge == 1 && is_bb_annotated (bb, *annotated_bb))
{
- unknown_edge->probability
- = total_known_count.probability_in (bb->count);
- set_edge_annotated (unknown_edge, annotated_edge);
- changed = true;
+ if (bb->count > total_known_count)
+ AFDO_EINFO (unknown_edge)->set_count (bb->count - total_known_count);
+ else
+ AFDO_EINFO (unknown_edge)->set_count (profile_count::zero().afdo ());
+ AFDO_EINFO (unknown_edge)->set_annotated ();
+ changed = true;
}
}
return changed;
goto BB3
In this case, we need to propagate through PHI to determine the edge
- count of BB1->BB.t1, BB.t1->BB.t2.
- Update ANNOTATED_EDGE accordingly. */
+ count of BB1->BB.t1, BB.t1->BB.t2. */
static void
-afdo_propagate_circuit (const bb_set &annotated_bb, edge_set *annotated_edge)
+afdo_propagate_circuit (const bb_set &annotated_bb)
{
basic_block bb;
FOR_ALL_BB_FN (bb, cfun)
bool check_value_one = (((integer_onep (cmp_rhs))
^ (gimple_cond_code (cmp_stmt) == EQ_EXPR))
^ ((e->flags & EDGE_TRUE_VALUE) != 0));
- if (!is_edge_annotated (e, *annotated_edge))
+ if (! AFDO_EINFO (e)->is_annotated ())
continue;
for (i = 0; i < gimple_phi_num_args (phi_stmt); i++)
{
continue;
total++;
only_one = ep;
- if (!e->probability.initialized_p ()
- && !is_edge_annotated (ep, *annotated_edge))
- {
- ep->probability = profile_probability::never ().afdo ();
- set_edge_annotated (ep, annotated_edge);
- }
- }
- if (total == 1 && !is_edge_annotated (only_one, *annotated_edge))
- {
- only_one->probability = e->probability;
- set_edge_annotated (only_one, annotated_edge);
- }
+ if (! (AFDO_EINFO (e)->get_count ()).nonzero_p ()
+ && ! AFDO_EINFO (ep)->is_annotated ())
+ {
+ AFDO_EINFO (ep)->set_count (profile_count::zero ().afdo ());
+ AFDO_EINFO (ep)->set_annotated ();
+ }
+ }
+ if (total == 1 && ! AFDO_EINFO (only_one)->is_annotated ())
+ {
+ AFDO_EINFO (only_one)->set_count (AFDO_EINFO (e)->get_count ());
+ AFDO_EINFO (only_one)->set_annotated ();
+ }
}
}
}
graph. We do the propagation iteratively until stablize. */
static void
-afdo_propagate (bb_set *annotated_bb, edge_set *annotated_edge)
+afdo_propagate (bb_set *annotated_bb)
{
basic_block bb;
bool changed = true;
{
changed = false;
- if (afdo_propagate_edge (true, annotated_bb, annotated_edge))
+ if (afdo_propagate_edge (true, annotated_bb))
changed = true;
- if (afdo_propagate_edge (false, annotated_bb, annotated_edge))
+ if (afdo_propagate_edge (false, annotated_bb))
changed = true;
- afdo_propagate_circuit (*annotated_bb, annotated_edge);
+ afdo_propagate_circuit (*annotated_bb);
}
}
probabilities. */
static void
-afdo_calculate_branch_prob (bb_set *annotated_bb, edge_set *annotated_edge)
+afdo_calculate_branch_prob (bb_set *annotated_bb)
{
+ edge e;
+ edge_iterator ei;
basic_block bb;
- bool has_sample = false;
-
- FOR_EACH_BB_FN (bb, cfun)
- {
- if (bb->count > profile_count::zero ())
- {
- has_sample = true;
- break;
- }
- }
-
- if (!has_sample)
- return;
calculate_dominance_info (CDI_POST_DOMINATORS);
calculate_dominance_info (CDI_DOMINATORS);
loop_optimizer_init (0);
+ FOR_ALL_BB_FN (bb, cfun)
+ {
+ gcc_assert (bb->aux == NULL);
+ FOR_EACH_EDGE (e, ei, bb->succs)
+ {
+ gcc_assert (e->aux == NULL);
+ e->aux = new edge_info ();
+ }
+ }
+
afdo_find_equiv_class (annotated_bb);
- afdo_propagate (annotated_bb, annotated_edge);
+ afdo_propagate (annotated_bb);
FOR_EACH_BB_FN (bb, cfun)
{
- edge e;
- edge_iterator ei;
int num_unknown_succ = 0;
- profile_count total_count = profile_count::zero ();
+ profile_count total_count = profile_count::zero ().afdo ();
FOR_EACH_EDGE (e, ei, bb->succs)
{
- if (!is_edge_annotated (e, *annotated_edge))
+ gcc_assert (AFDO_EINFO (e) != NULL);
+ if (! AFDO_EINFO (e)->is_annotated ())
num_unknown_succ++;
else
- total_count += e->count ();
+ total_count += AFDO_EINFO (e)->get_count ();
}
if (num_unknown_succ == 0 && total_count > profile_count::zero ())
{
- FOR_EACH_EDGE (e, ei, bb->succs)
- e->probability = e->count ().probability_in (total_count);
+ FOR_EACH_EDGE (e, ei, bb->succs)
+ e->probability
+ = AFDO_EINFO (e)->get_count ().probability_in (total_count);
}
}
FOR_ALL_BB_FN (bb, cfun)
- bb->aux = NULL;
+ {
+ bb->aux = NULL;
+ FOR_EACH_EDGE (e, ei, bb->succs)
+ if (AFDO_EINFO (e) != NULL)
+ {
+ delete AFDO_EINFO (e);
+ e->aux = NULL;
+ }
+ }
loop_optimizer_finalize ();
free_dominance_info (CDI_DOMINATORS);
{
basic_block bb;
bb_set annotated_bb;
- edge_set annotated_edge;
const function_instance *s
= afdo_source_profile->get_function_instance_by_decl (
current_function_decl);
= profile_count::from_gcov_type (s->head_count ()).afdo ();
ENTRY_BLOCK_PTR_FOR_FN (cfun)->count
= profile_count::from_gcov_type (s->head_count ()).afdo ();
+ EXIT_BLOCK_PTR_FOR_FN (cfun)->count = profile_count::zero ().afdo ();
profile_count max_count = ENTRY_BLOCK_PTR_FOR_FN (cfun)->count;
FOR_EACH_BB_FN (bb, cfun)
- {
- edge e;
- edge_iterator ei;
-
- /* As autoFDO uses sampling approach, we have to assume that all
- counters are zero when not seen by autoFDO. */
- bb->count = profile_count::zero ().afdo ();
- FOR_EACH_EDGE (e, ei, bb->succs)
- e->probability = profile_probability::uninitialized ();
-
- if (afdo_set_bb_count (bb, promoted_stmts))
- set_bb_annotated (bb, &annotated_bb);
- if (bb->count > max_count)
- max_count = bb->count;
- }
+ {
+ /* As autoFDO uses sampling approach, we have to assume that all
+ counters are zero when not seen by autoFDO. */
+ bb->count = profile_count::zero ().afdo ();
+ if (afdo_set_bb_count (bb, promoted_stmts))
+ set_bb_annotated (bb, &annotated_bb);
+ if (bb->count > max_count)
+ max_count = bb->count;
+ }
if (ENTRY_BLOCK_PTR_FOR_FN (cfun)->count
> ENTRY_BLOCK_PTR_FOR_FN (cfun)->next_bb->count)
{
afdo_source_profile->mark_annotated (cfun->function_end_locus);
if (max_count > profile_count::zero ())
{
- afdo_calculate_branch_prob (&annotated_bb, &annotated_edge);
- update_max_bb_count ();
- profile_status_for_fn (cfun) = PROFILE_READ;
+ /* Calculate, propagate count and probability information on CFG. */
+ afdo_calculate_branch_prob (&annotated_bb);
}
+ update_max_bb_count ();
+ profile_status_for_fn (cfun) = PROFILE_READ;
if (flag_value_profile_transformations)
{
gimple_value_profile_transformations ();
function before annotation, so the profile inside bar@loc_foo2
will be useful. */
autofdo::stmt_set promoted_stmts;
- for (int i = 0; i < PARAM_VALUE (PARAM_EARLY_INLINER_MAX_ITERATIONS); i++)
+ for (int i = 0; i < opt_for_fn (node->decl,
+ param_early_inliner_max_iterations); i++)
{
if (!flag_value_profile_transformations
|| !autofdo::afdo_vpt_for_early_inline (&promoted_stmts))
if (auto_profile_file == NULL)
auto_profile_file = DEFAULT_AUTO_PROFILE_FILE;
- autofdo::afdo_profile_info = (struct gcov_ctr_summary *)xcalloc (
- 1, sizeof (struct gcov_ctr_summary));
+ autofdo::afdo_profile_info = XNEW (gcov_summary);
autofdo::afdo_profile_info->runs = 1;
autofdo::afdo_profile_info->sum_max = 0;
- autofdo::afdo_profile_info->sum_all = 0;
/* Read the profile from the profile file. */
autofdo::read_profile ();
if (count > 0)
{
bool is_hot;
- const struct gcov_ctr_summary *saved_profile_info = profile_info;
+ profile_count pcount = profile_count::from_gcov_type (count).afdo ();
+ gcov_summary *saved_profile_info = profile_info;
/* At early inline stage, profile_info is not set yet. We need to
temporarily set it to afdo_profile_info to calculate hotness. */
profile_info = autofdo::afdo_profile_info;
- is_hot = maybe_hot_count_p (NULL, profile_count::from_gcov_type (count));
+ is_hot = maybe_hot_count_p (NULL, pcount);
profile_info = saved_profile_info;
return is_hot;
}
projects / thirdparty / gcc.git / blobdiff