/* Read and annotate call graph profile from the auto profile data file.
- Copyright (C) 2014-2015 Free Software Foundation, Inc.
+ Copyright (C) 2014-2020 Free Software Foundation, Inc.
Contributed by Dehao Chen (dehao@google.com)
This file is part of GCC.
<http://www.gnu.org/licenses/>. */
#include "config.h"
+#define INCLUDE_MAP
+#define INCLUDE_SET
#include "system.h"
-
-#include <string.h>
-#include <map>
-#include <set>
-
#include "coretypes.h"
-#include "alias.h"
#include "backend.h"
#include "tree.h"
#include "gimple.h"
-#include "hard-reg-set.h"
-#include "ssa.h"
-#include "options.h"
-#include "fold-const.h"
+#include "predict.h"
+#include "alloc-pool.h"
#include "tree-pass.h"
-#include "flags.h"
-#include "diagnostic-core.h"
+#include "ssa.h"
+#include "cgraph.h"
#include "gcov-io.h"
+#include "diagnostic-core.h"
#include "profile.h"
#include "langhooks.h"
-#include "opts.h"
-#include "tree-pass.h"
#include "cfgloop.h"
#include "tree-cfg.h"
#include "tree-cfgcleanup.h"
#include "tree-into-ssa.h"
-#include "internal-fn.h"
#include "gimple-iterator.h"
-#include "cgraph.h"
#include "value-prof.h"
-#include "coverage.h"
-#include "params.h"
-#include "alloc-pool.h"
#include "symbol-summary.h"
#include "ipa-prop.h"
+#include "ipa-fnsummary.h"
#include "ipa-inline.h"
#include "tree-inline.h"
#include "auto-profile.h"
+#include "tree-pretty-print.h"
+#include "gimple-pretty-print.h"
/* The following routines implements AutoFDO optimization.
namespace autofdo
{
+/* Intermediate edge info used when propagating AutoFDO profile information.
+ We can't edge->count() directly since it's computed from edge's probability
+ while probability is yet not decided during propagation. */
+#define AFDO_EINFO(e) ((class edge_info *) e->aux)
+class edge_info
+{
+public:
+ edge_info () : count_ (profile_count::zero ().afdo ()), annotated_ (false) {}
+ bool is_annotated () const { return annotated_; }
+ void set_annotated () { annotated_ = true; }
+ profile_count get_count () const { return count_; }
+ void set_count (profile_count count) { count_ = count; }
+private:
+ profile_count count_;
+ bool annotated_;
+};
+
/* Represent a source location: (function_decl, lineno). */
typedef std::pair<tree, unsigned> decl_lineno;
/* Set of gimple stmts. Used to track if the stmt has already been promoted
to direct call. */
-typedef std::set<gimple> stmt_set;
+typedef std::set<gimple *> stmt_set;
/* Represent count info of an inline stack. */
-struct count_info
+class count_info
{
+public:
/* Sampled count of the inline stack. */
gcov_type count;
/* Find count_info for a given gimple STMT. If found, store the count_info
in INFO and return true; otherwise return false. */
- bool get_count_info (gimple stmt, count_info *info) const;
+ bool get_count_info (gimple *stmt, count_info *info) const;
/* Find total count of the callee of EDGE. */
gcov_type get_callsite_total_count (struct cgraph_edge *edge) const;
/* Store the AutoFDO source profile. */
static autofdo_source_profile *afdo_source_profile;
-/* gcov_ctr_summary structure to store the profile_info. */
-static struct gcov_ctr_summary *afdo_profile_info;
+/* gcov_summary structure to store the profile_info. */
+static gcov_summary *afdo_profile_info;
/* Helper functions. */
{
/* TODO: allow more bits for line and less bits for discriminator. */
if (LOCATION_LINE (loc) - DECL_SOURCE_LINE (decl) >= (1<<16))
- warning_at (loc, OPT_Woverflow, "Offset exceeds 16 bytes.");
+ warning_at (loc, OPT_Woverflow, "offset exceeds 16 bytes");
return ((LOCATION_LINE (loc) - DECL_SOURCE_LINE (decl)) << 16);
}
static tree
get_function_decl_from_block (tree block)
{
- tree decl;
-
- if (LOCATION_LOCUS (BLOCK_SOURCE_LOCATION (block) == UNKNOWN_LOCATION))
+ if (!inlined_function_outer_scope_p (block))
return NULL_TREE;
- for (decl = BLOCK_ABSTRACT_ORIGIN (block);
- decl && (TREE_CODE (decl) == BLOCK);
- decl = BLOCK_ABSTRACT_ORIGIN (decl))
- if (TREE_CODE (decl) == FUNCTION_DECL)
- break;
- return decl;
+ return BLOCK_ABSTRACT_ORIGIN (block);
}
/* Store inline stack for STMT in STACK. */
of DECL, The lower 16 bits stores the discriminator. */
static unsigned
-get_relative_location_for_stmt (gimple stmt)
+get_relative_location_for_stmt (gimple *stmt)
{
location_t locus = gimple_location (stmt);
if (LOCATION_LOCUS (locus) == UNKNOWN_LOCATION)
for (gsi = gsi_start_bb (bb); !gsi_end_p (gsi); gsi_next (&gsi))
{
- gimple stmt = gsi_stmt (gsi);
+ gimple *stmt = gsi_stmt (gsi);
if (gimple_code (stmt) == GIMPLE_CALL && !gimple_call_internal_p (stmt)
&& (gimple_call_fn (stmt) == NULL
|| TREE_CODE (gimple_call_fn (stmt)) != FUNCTION_DECL))
ret = get_index (lang_hooks.dwarf_name (decl, 0));
if (ret != -1)
return ret;
- if (DECL_ABSTRACT_ORIGIN (decl))
+ if (DECL_FROM_INLINE (decl))
return get_index_by_decl (DECL_ABSTRACT_ORIGIN (decl));
return -1;
if (ret != callsites.end ())
return ret->second;
}
- if (DECL_ABSTRACT_ORIGIN (decl))
+ if (DECL_FROM_INLINE (decl))
return get_function_instance_by_decl (lineno, DECL_ABSTRACT_ORIGIN (decl));
return NULL;
get_identifier (afdo_string_table->get_name (callee)));
if (node == NULL)
continue;
- if (!check_ic_target (stmt, node))
- continue;
(*map)[callee] = iter->second->total_count ();
ret += iter->second->total_count ();
}
in INFO and return true; otherwise return false. */
bool
-autofdo_source_profile::get_count_info (gimple stmt, count_info *info) const
+autofdo_source_profile::get_count_info (gimple *stmt, count_info *info) const
{
if (LOCATION_LOCUS (gimple_location (stmt)) == cfun->function_end_locus)
return false;
autofdo_source_profile::update_inlined_ind_target (gcall *stmt,
count_info *info)
{
+ if (dump_file)
+ {
+ fprintf (dump_file, "Checking indirect call -> direct call ");
+ print_gimple_stmt (dump_file, stmt, 0, TDF_SLIM);
+ }
+
if (LOCATION_LOCUS (gimple_location (stmt)) == cfun->function_end_locus)
- return false;
+ {
+ if (dump_file)
+ fprintf (dump_file, " good locus\n");
+ return false;
+ }
count_info old_info;
get_count_info (stmt, &old_info);
hot any more. Will avoid promote the original target.
To check if original promoted target is still hot, we check the total
- count of the unpromoted targets (stored in old_info). If it is no less
- than half of the callsite count (stored in INFO), the original promoted
- target is considered not hot any more. */
- if (total >= info->count / 2)
- return false;
+ count of the unpromoted targets (stored in TOTAL). If a callsite count
+ (stored in INFO) is smaller than half of the total count, the original
+ promoted target is considered not hot any more. */
+ if (info->count < total / 2)
+ {
+ if (dump_file)
+ fprintf (dump_file, " not hot anymore %ld < %ld",
+ (long)info->count,
+ (long)total /2);
+ return false;
+ }
inline_stack stack;
get_inline_stack (gimple_location (stmt), &stack);
if (stack.length () == 0)
- return false;
+ {
+ if (dump_file)
+ fprintf (dump_file, " no inline stack\n");
+ return false;
+ }
function_instance *s = get_function_instance_by_inline_stack (stack);
if (s == NULL)
- return false;
+ {
+ if (dump_file)
+ fprintf (dump_file, " function not found in inline stack\n");
+ return false;
+ }
icall_target_map map;
if (s->find_icall_target_map (stmt, &map) == 0)
- return false;
+ {
+ if (dump_file)
+ fprintf (dump_file, " no target map\n");
+ return false;
+ }
for (icall_target_map::const_iterator iter = map.begin ();
iter != map.end (); ++iter)
info->targets[iter->first] = iter->second;
+ if (dump_file)
+ fprintf (dump_file, " looks good\n");
return true;
}
{
if (gcov_read_unsigned () != GCOV_TAG_AFDO_FUNCTION)
{
- inform (0, "Not expected TAG.");
+ inform (UNKNOWN_LOCATION, "Not expected TAG.");
return false;
}
function_instance::function_instance_stack stack;
function_instance *s = function_instance::read_function_instance (
&stack, gcov_read_counter ());
- afdo_profile_info->sum_all += s->total_count ();
map_[s->name ()] = s;
}
return true;
read_profile (void)
{
if (gcov_open (auto_profile_file, 1) == 0)
- error ("Cannot open profile file %s.", auto_profile_file);
+ {
+ error ("cannot open profile file %s", auto_profile_file);
+ return;
+ }
if (gcov_read_unsigned () != GCOV_DATA_MAGIC)
- error ("AutoFDO profile magic number does not mathch.");
+ {
+ error ("AutoFDO profile magic number does not match");
+ return;
+ }
/* Skip the version number. */
unsigned version = gcov_read_unsigned ();
if (version != AUTO_PROFILE_VERSION)
- error ("AutoFDO profile version %u does match %u.",
- version, AUTO_PROFILE_VERSION);
+ {
+ error ("AutoFDO profile version %u does match %u",
+ version, AUTO_PROFILE_VERSION);
+ return;
+ }
/* Skip the empty integer. */
gcov_read_unsigned ();
/* string_table. */
afdo_string_table = new string_table ();
if (!afdo_string_table->read())
- error ("Cannot read string table from %s.", auto_profile_file);
+ {
+ error ("cannot read string table from %s", auto_profile_file);
+ return;
+ }
/* autofdo_source_profile. */
afdo_source_profile = autofdo_source_profile::create ();
if (afdo_source_profile == NULL)
- error ("Cannot read function profile from %s.", auto_profile_file);
+ {
+ error ("cannot read function profile from %s", auto_profile_file);
+ return;
+ }
/* autofdo_module_profile. */
fake_read_autofdo_module_profile ();
-
- /* Read in the working set. */
- if (gcov_read_unsigned () != GCOV_TAG_AFDO_WORKING_SET)
- error ("Cannot read working set from %s.", auto_profile_file);
-
- /* Skip the length of the section. */
- gcov_read_unsigned ();
- gcov_working_set_t set[128];
- for (unsigned i = 0; i < 128; i++)
- {
- set[i].num_counters = gcov_read_unsigned ();
- set[i].min_counter = gcov_read_counter ();
- }
- add_working_set (set);
}
/* From AutoFDO profiles, find values inside STMT for that we want to measure
afdo_indirect_call (gimple_stmt_iterator *gsi, const icall_target_map &map,
bool transform)
{
- gimple gs = gsi_stmt (*gsi);
+ gimple *gs = gsi_stmt (*gsi);
tree callee;
if (map.size () == 0)
return;
gcall *stmt = dyn_cast <gcall *> (gs);
- if ((!stmt) || gimple_call_fndecl (stmt) != NULL_TREE)
+ if (!stmt
+ || gimple_call_internal_p (stmt)
+ || gimple_call_fndecl (stmt) != NULL_TREE)
return;
- callee = gimple_call_fn (stmt);
-
- histogram_value hist = gimple_alloc_histogram_value (
- cfun, HIST_TYPE_INDIR_CALL, stmt, callee);
- hist->n_counters = 3;
- hist->hvalue.counters = XNEWVEC (gcov_type, hist->n_counters);
- gimple_add_histogram_value (cfun, stmt, hist);
-
gcov_type total = 0;
icall_target_map::const_iterator max_iter = map.end ();
if (max_iter == map.end () || max_iter->second < iter->second)
max_iter = iter;
}
+ struct cgraph_node *direct_call = cgraph_node::get_for_asmname (
+ get_identifier (afdo_string_table->get_name (max_iter->first)));
+ if (direct_call == NULL || !direct_call->profile_id)
+ return;
- hist->hvalue.counters[0]
- = (unsigned long long)afdo_string_table->get_name (max_iter->first);
+ callee = gimple_call_fn (stmt);
+
+ histogram_value hist = gimple_alloc_histogram_value (
+ cfun, HIST_TYPE_INDIR_CALL, stmt, callee);
+ hist->n_counters = 3;
+ hist->hvalue.counters = XNEWVEC (gcov_type, hist->n_counters);
+ gimple_add_histogram_value (cfun, stmt, hist);
+
+ hist->hvalue.counters[0] = direct_call->profile_id;
hist->hvalue.counters[1] = max_iter->second;
hist->hvalue.counters[2] = total;
struct cgraph_edge *indirect_edge
= cgraph_node::get (current_function_decl)->get_edge (stmt);
- struct cgraph_node *direct_call = cgraph_node::get_for_asmname (
- get_identifier ((const char *) hist->hvalue.counters[0]));
- if (direct_call == NULL || !check_ic_target (stmt, direct_call))
- return;
+ if (dump_file)
+ {
+ fprintf (dump_file, "Indirect call -> direct call ");
+ print_generic_expr (dump_file, callee, TDF_SLIM);
+ fprintf (dump_file, " => ");
+ print_generic_expr (dump_file, direct_call->decl, TDF_SLIM);
+ }
+
+ if (direct_call == NULL)
+ {
+ if (dump_file)
+ fprintf (dump_file, " not transforming\n");
+ return;
+ }
if (DECL_STRUCT_FUNCTION (direct_call->decl) == NULL)
- return;
+ {
+ if (dump_file)
+ fprintf (dump_file, " no declaration\n");
+ return;
+ }
+
+ if (dump_file)
+ {
+ fprintf (dump_file, " transformation on insn ");
+ print_gimple_stmt (dump_file, stmt, 0, TDF_SLIM);
+ fprintf (dump_file, "\n");
+ }
+
+ /* FIXME: Count should be initialized. */
struct cgraph_edge *new_edge
- = indirect_edge->make_speculative (direct_call, 0, 0);
- new_edge->redirect_call_stmt_to_callee ();
+ = indirect_edge->make_speculative (direct_call,
+ profile_count::uninitialized ());
+ cgraph_edge::redirect_call_stmt_to_callee (new_edge);
gimple_remove_histogram_value (cfun, stmt, hist);
inline_call (new_edge, true, NULL, NULL, false);
}
annotated->insert (bb);
}
-static bool
-is_edge_annotated (const edge e, const edge_set &annotated)
-{
- return annotated.find (e) != annotated.end ();
-}
-
-static void
-set_edge_annotated (edge e, edge_set *annotated)
-{
- annotated->insert (e);
-}
-
/* For a given BB, set its execution count. Attach value profile if a stmt
is not in PROMOTED, because we only want to promote an indirect call once.
Return TRUE if BB is annotated. */
for (gsi = gsi_start_bb (bb); !gsi_end_p (gsi); gsi_next (&gsi))
{
count_info info;
- gimple stmt = gsi_stmt (gsi);
+ gimple *stmt = gsi_stmt (gsi);
if (gimple_clobber_p (stmt) || is_gimple_debug (stmt))
continue;
if (afdo_source_profile->get_count_info (stmt, &info))
FOR_EACH_EDGE (e, ei, bb->succs)
afdo_source_profile->mark_annotated (e->goto_locus);
- bb->count = max_count;
+ bb->count = profile_count::from_gcov_type (max_count).afdo ();
return true;
}
edges' counts are known, then the basic block's unknown count can also be
calculated.
IS_SUCC is true if out edges of a basic blocks are examined.
- Update ANNOTATED_BB and ANNOTATED_EDGE accordingly.
+ Update ANNOTATED_BB accordingly.
Return TRUE if any basic block/edge count is changed. */
static bool
-afdo_propagate_edge (bool is_succ, bb_set *annotated_bb,
- edge_set *annotated_edge)
+afdo_propagate_edge (bool is_succ, bb_set *annotated_bb)
{
basic_block bb;
bool changed = false;
edge e, unknown_edge = NULL;
edge_iterator ei;
int num_unknown_edge = 0;
- gcov_type total_known_count = 0;
+ profile_count total_known_count = profile_count::zero ().afdo ();
FOR_EACH_EDGE (e, ei, is_succ ? bb->succs : bb->preds)
- if (!is_edge_annotated (e, *annotated_edge))
- num_unknown_edge++, unknown_edge = e;
- else
- total_known_count += e->count;
+ {
+ gcc_assert (AFDO_EINFO (e) != NULL);
+ if (! AFDO_EINFO (e)->is_annotated ())
+ num_unknown_edge++, unknown_edge = e;
+ else
+ total_known_count += AFDO_EINFO (e)->get_count ();
+ }
- if (num_unknown_edge == 0)
+ /* Be careful not to annotate block with no successor in special cases. */
+ if (num_unknown_edge == 0 && total_known_count > bb->count)
{
- if (total_known_count > bb->count)
- {
- bb->count = total_known_count;
- changed = true;
- }
- if (!is_bb_annotated (bb, *annotated_bb))
- {
- set_bb_annotated (bb, annotated_bb);
- changed = true;
- }
+ bb->count = total_known_count;
+ if (!is_bb_annotated (bb, *annotated_bb))
+ set_bb_annotated (bb, annotated_bb);
+ changed = true;
}
else if (num_unknown_edge == 1 && is_bb_annotated (bb, *annotated_bb))
{
- if (bb->count >= total_known_count)
- unknown_edge->count = bb->count - total_known_count;
- else
- unknown_edge->count = 0;
- set_edge_annotated (unknown_edge, annotated_edge);
- changed = true;
+ if (bb->count > total_known_count)
+ AFDO_EINFO (unknown_edge)->set_count (bb->count - total_known_count);
+ else
+ AFDO_EINFO (unknown_edge)->set_count (profile_count::zero().afdo ());
+ AFDO_EINFO (unknown_edge)->set_annotated ();
+ changed = true;
}
}
return changed;
goto BB3
In this case, we need to propagate through PHI to determine the edge
- count of BB1->BB.t1, BB.t1->BB.t2.
- Update ANNOTATED_EDGE accordingly. */
+ count of BB1->BB.t1, BB.t1->BB.t2. */
static void
-afdo_propagate_circuit (const bb_set &annotated_bb, edge_set *annotated_edge)
+afdo_propagate_circuit (const bb_set &annotated_bb)
{
basic_block bb;
FOR_ALL_BB_FN (bb, cfun)
{
- gimple def_stmt;
+ gimple *def_stmt;
tree cmp_rhs, cmp_lhs;
- gimple cmp_stmt = last_stmt (bb);
+ gimple *cmp_stmt = last_stmt (bb);
edge e;
edge_iterator ei;
bool check_value_one = (((integer_onep (cmp_rhs))
^ (gimple_cond_code (cmp_stmt) == EQ_EXPR))
^ ((e->flags & EDGE_TRUE_VALUE) != 0));
- if (!is_edge_annotated (e, *annotated_edge))
+ if (! AFDO_EINFO (e)->is_annotated ())
continue;
for (i = 0; i < gimple_phi_num_args (phi_stmt); i++)
{
continue;
total++;
only_one = ep;
- if (e->probability == 0 && !is_edge_annotated (ep, *annotated_edge))
- {
- ep->probability = 0;
- ep->count = 0;
- set_edge_annotated (ep, annotated_edge);
- }
- }
- if (total == 1 && !is_edge_annotated (only_one, *annotated_edge))
- {
- only_one->probability = e->probability;
- only_one->count = e->count;
- set_edge_annotated (only_one, annotated_edge);
- }
+ if (! (AFDO_EINFO (e)->get_count ()).nonzero_p ()
+ && ! AFDO_EINFO (ep)->is_annotated ())
+ {
+ AFDO_EINFO (ep)->set_count (profile_count::zero ().afdo ());
+ AFDO_EINFO (ep)->set_annotated ();
+ }
+ }
+ if (total == 1 && ! AFDO_EINFO (only_one)->is_annotated ())
+ {
+ AFDO_EINFO (only_one)->set_count (AFDO_EINFO (e)->get_count ());
+ AFDO_EINFO (only_one)->set_annotated ();
+ }
}
}
}
graph. We do the propagation iteratively until stablize. */
static void
-afdo_propagate (bb_set *annotated_bb, edge_set *annotated_edge)
+afdo_propagate (bb_set *annotated_bb)
{
basic_block bb;
bool changed = true;
FOR_ALL_BB_FN (bb, cfun)
{
bb->count = ((basic_block)bb->aux)->count;
- if (is_bb_annotated ((const basic_block)bb->aux, *annotated_bb))
+ if (is_bb_annotated ((basic_block)bb->aux, *annotated_bb))
set_bb_annotated (bb, annotated_bb);
}
{
changed = false;
- if (afdo_propagate_edge (true, annotated_bb, annotated_edge))
+ if (afdo_propagate_edge (true, annotated_bb))
changed = true;
- if (afdo_propagate_edge (false, annotated_bb, annotated_edge))
+ if (afdo_propagate_edge (false, annotated_bb))
changed = true;
- afdo_propagate_circuit (*annotated_bb, annotated_edge);
+ afdo_propagate_circuit (*annotated_bb);
}
}
probabilities. */
static void
-afdo_calculate_branch_prob (bb_set *annotated_bb, edge_set *annotated_edge)
+afdo_calculate_branch_prob (bb_set *annotated_bb)
{
+ edge e;
+ edge_iterator ei;
basic_block bb;
- bool has_sample = false;
-
- FOR_EACH_BB_FN (bb, cfun)
- {
- if (bb->count > 0)
- {
- has_sample = true;
- break;
- }
- }
-
- if (!has_sample)
- return;
calculate_dominance_info (CDI_POST_DOMINATORS);
calculate_dominance_info (CDI_DOMINATORS);
loop_optimizer_init (0);
+ FOR_ALL_BB_FN (bb, cfun)
+ {
+ gcc_assert (bb->aux == NULL);
+ FOR_EACH_EDGE (e, ei, bb->succs)
+ {
+ gcc_assert (e->aux == NULL);
+ e->aux = new edge_info ();
+ }
+ }
+
afdo_find_equiv_class (annotated_bb);
- afdo_propagate (annotated_bb, annotated_edge);
+ afdo_propagate (annotated_bb);
FOR_EACH_BB_FN (bb, cfun)
{
- edge e;
- edge_iterator ei;
int num_unknown_succ = 0;
- gcov_type total_count = 0;
+ profile_count total_count = profile_count::zero ().afdo ();
FOR_EACH_EDGE (e, ei, bb->succs)
{
- if (!is_edge_annotated (e, *annotated_edge))
+ gcc_assert (AFDO_EINFO (e) != NULL);
+ if (! AFDO_EINFO (e)->is_annotated ())
num_unknown_succ++;
else
- total_count += e->count;
+ total_count += AFDO_EINFO (e)->get_count ();
}
- if (num_unknown_succ == 0 && total_count > 0)
+ if (num_unknown_succ == 0 && total_count > profile_count::zero ())
{
- FOR_EACH_EDGE (e, ei, bb->succs)
- e->probability = (double)e->count * REG_BR_PROB_BASE / total_count;
+ FOR_EACH_EDGE (e, ei, bb->succs)
+ e->probability
+ = AFDO_EINFO (e)->get_count ().probability_in (total_count);
}
}
FOR_ALL_BB_FN (bb, cfun)
- {
- edge e;
- edge_iterator ei;
-
- FOR_EACH_EDGE (e, ei, bb->succs)
- e->count = (double)bb->count * e->probability / REG_BR_PROB_BASE;
- bb->aux = NULL;
- }
+ {
+ bb->aux = NULL;
+ FOR_EACH_EDGE (e, ei, bb->succs)
+ if (AFDO_EINFO (e) != NULL)
+ {
+ delete AFDO_EINFO (e);
+ e->aux = NULL;
+ }
+ }
loop_optimizer_finalize ();
free_dominance_info (CDI_DOMINATORS);
current_function_decl) == NULL)
return false;
- compute_inline_parameters (cgraph_node::get (current_function_decl), true);
+ compute_fn_summary (cgraph_node::get (current_function_decl), true);
bool has_vpt = false;
FOR_EACH_BB_FN (bb, cfun)
for (gsi = gsi_start_bb (bb); !gsi_end_p (gsi); gsi_next (&gsi))
{
count_info info;
- gimple stmt = gsi_stmt (gsi);
+ gimple *stmt = gsi_stmt (gsi);
if (afdo_source_profile->get_count_info (stmt, &info))
bb_count = MAX (bb_count, info.count);
}
if (has_vpt)
{
- optimize_inline_calls (current_function_decl);
+ unsigned todo = optimize_inline_calls (current_function_decl);
+ if (todo & TODO_update_ssa_any)
+ update_ssa (TODO_update_ssa);
return true;
}
{
basic_block bb;
bb_set annotated_bb;
- edge_set annotated_edge;
const function_instance *s
= afdo_source_profile->get_function_instance_by_decl (
current_function_decl);
if (s == NULL)
return;
- cgraph_node::get (current_function_decl)->count = s->head_count ();
- ENTRY_BLOCK_PTR_FOR_FN (cfun)->count = s->head_count ();
- gcov_type max_count = ENTRY_BLOCK_PTR_FOR_FN (cfun)->count;
+ cgraph_node::get (current_function_decl)->count
+ = profile_count::from_gcov_type (s->head_count ()).afdo ();
+ ENTRY_BLOCK_PTR_FOR_FN (cfun)->count
+ = profile_count::from_gcov_type (s->head_count ()).afdo ();
+ EXIT_BLOCK_PTR_FOR_FN (cfun)->count = profile_count::zero ().afdo ();
+ profile_count max_count = ENTRY_BLOCK_PTR_FOR_FN (cfun)->count;
FOR_EACH_BB_FN (bb, cfun)
- {
- edge e;
- edge_iterator ei;
-
- bb->count = 0;
- FOR_EACH_EDGE (e, ei, bb->succs)
- e->count = 0;
-
- if (afdo_set_bb_count (bb, promoted_stmts))
- set_bb_annotated (bb, &annotated_bb);
- if (bb->count > max_count)
- max_count = bb->count;
- }
+ {
+ /* As autoFDO uses sampling approach, we have to assume that all
+ counters are zero when not seen by autoFDO. */
+ bb->count = profile_count::zero ().afdo ();
+ if (afdo_set_bb_count (bb, promoted_stmts))
+ set_bb_annotated (bb, &annotated_bb);
+ if (bb->count > max_count)
+ max_count = bb->count;
+ }
if (ENTRY_BLOCK_PTR_FOR_FN (cfun)->count
> ENTRY_BLOCK_PTR_FOR_FN (cfun)->next_bb->count)
{
DECL_SOURCE_LOCATION (current_function_decl));
afdo_source_profile->mark_annotated (cfun->function_start_locus);
afdo_source_profile->mark_annotated (cfun->function_end_locus);
- if (max_count > 0)
+ if (max_count > profile_count::zero ())
{
- afdo_calculate_branch_prob (&annotated_bb, &annotated_edge);
- counts_to_freqs ();
- profile_status_for_fn (cfun) = PROFILE_READ;
+ /* Calculate, propagate count and probability information on CFG. */
+ afdo_calculate_branch_prob (&annotated_bb);
}
+ update_max_bb_count ();
+ profile_status_for_fn (cfun) = PROFILE_READ;
if (flag_value_profile_transformations)
{
gimple_value_profile_transformations ();
static void
early_inline ()
{
- compute_inline_parameters (cgraph_node::get (current_function_decl), true);
+ compute_fn_summary (cgraph_node::get (current_function_decl), true);
unsigned todo = early_inliner (cfun);
if (todo & TODO_update_ssa_any)
update_ssa (TODO_update_ssa);
function before annotation, so the profile inside bar@loc_foo2
will be useful. */
autofdo::stmt_set promoted_stmts;
- for (int i = 0; i < PARAM_VALUE (PARAM_EARLY_INLINER_MAX_ITERATIONS); i++)
+ for (int i = 0; i < opt_for_fn (node->decl,
+ param_early_inliner_max_iterations); i++)
{
if (!flag_value_profile_transformations
|| !autofdo::afdo_vpt_for_early_inline (&promoted_stmts))
free_dominance_info (CDI_DOMINATORS);
free_dominance_info (CDI_POST_DOMINATORS);
cgraph_edge::rebuild_edges ();
- compute_inline_parameters (cgraph_node::get (current_function_decl), true);
+ compute_fn_summary (cgraph_node::get (current_function_decl), true);
pop_cfun ();
}
if (auto_profile_file == NULL)
auto_profile_file = DEFAULT_AUTO_PROFILE_FILE;
- autofdo::afdo_profile_info = (struct gcov_ctr_summary *)xcalloc (
- 1, sizeof (struct gcov_ctr_summary));
+ autofdo::afdo_profile_info = XNEW (gcov_summary);
autofdo::afdo_profile_info->runs = 1;
autofdo::afdo_profile_info->sum_max = 0;
- autofdo::afdo_profile_info->sum_all = 0;
/* Read the profile from the profile file. */
autofdo::read_profile ();
if (count > 0)
{
bool is_hot;
- const struct gcov_ctr_summary *saved_profile_info = profile_info;
+ profile_count pcount = profile_count::from_gcov_type (count).afdo ();
+ gcov_summary *saved_profile_info = profile_info;
/* At early inline stage, profile_info is not set yet. We need to
temporarily set it to afdo_profile_info to calculate hotness. */
profile_info = autofdo::afdo_profile_info;
- is_hot = maybe_hot_count_p (NULL, count);
+ is_hot = maybe_hot_count_p (NULL, pcount);
profile_info = saved_profile_info;
return is_hot;
}