+2017-10-19 Jan Hubicka <hubicka@ucw.cz>
+
+ * asan.c (create_cond_insert_point): Do not update edge count.
+ * auto-profile.c (afdo_propagate_edge): Update for edge count removal.
+ (afdo_propagate_circuit): Likewise.
+ (afdo_calculate_branch_prob): Likewise.
+ (afdo_annotate_cfg): Likewise.
+ * basic-block.h (struct edge_def): Remove count.
+ (edge_def::count): New accessor.
+ * bb-reorder.c (rotate_loop): Update.
+ (find_traces_1_round): Update.
+ (connect_traces): Update.
+ (sanitize_hot_paths): Update.
+ * cfg.c (unchecked_make_edge): Update.
+ (make_single_succ_edge): Update.
+ (check_bb_profile): Update.
+ (dump_edge_info): Update.
+ (update_bb_profile_for_threading): Update.
+ (scale_bbs_frequencies_int): Update.
+ (scale_bbs_frequencies_gcov_type): Update.
+ (scale_bbs_frequencies_profile_count): Update.
+ (scale_bbs_frequencies): Update.
+ * cfganal.c (connect_infinite_loops_to_exit): Update.
+ * cfgbuild.c (compute_outgoing_frequencies): Update.
+ (find_many_sub_basic_blocks): Update.
+ * cfgcleanup.c (try_forward_edges): Update.
+ (try_crossjump_to_edge): Update
+ * cfgexpand.c (expand_gimple_cond): Update
+ (expand_gimple_tailcall): Update
+ (construct_exit_block): Update
+ * cfghooks.c (verify_flow_info): Update
+ (redirect_edge_succ_nodup): Update
+ (split_edge): Update
+ (make_forwarder_block): Update
+ (duplicate_block): Update
+ (account_profile_record): Update
+ * cfgloop.c (find_subloop_latch_edge_by_profile): Update.
+ * cfgloopanal.c (expected_loop_iterations_unbounded): Update.
+ * cfgloopmanip.c (scale_loop_profile): Update.
+ (loopify): Update.
+ (lv_adjust_loop_entry_edge): Update.
+ * cfgrtl.c (try_redirect_by_replacing_jump): Update.
+ (force_nonfallthru_and_redirect): Update.
+ (purge_dead_edges): Update.
+ (rtl_flow_call_edges_add): Update.
+ * cgraphunit.c (init_lowered_empty_function): Update.
+ (cgraph_node::expand_thunk): Update.
+ * gimple-pretty-print.c (dump_probability): Update.
+ (dump_edge_probability): Update.
+ * gimple-ssa-isolate-paths.c (isolate_path): Update.
+ * haifa-sched.c (sched_create_recovery_edges): Update.
+ * hsa-gen.c (convert_switch_statements): Update.
+ * ifcvt.c (dead_or_predicable): Update.
+ * ipa-inline-transform.c (inline_transform): Update.
+ * ipa-split.c (split_function): Update.
+ * ipa-utils.c (ipa_merge_profiles): Update.
+ * loop-doloop.c (add_test): Update.
+ * loop-unroll.c (unroll_loop_runtime_iterations): Update.
+ * lto-streamer-in.c (input_cfg): Update.
+ (input_function): Update.
+ * lto-streamer-out.c (output_cfg): Update.
+ * modulo-sched.c (sms_schedule): Update.
+ * postreload-gcse.c (eliminate_partially_redundant_load): Update.
+ * predict.c (maybe_hot_edge_p): Update.
+ (unlikely_executed_edge_p): Update.
+ (probably_never_executed_edge_p): Update.
+ (dump_prediction): Update.
+ (drop_profile): Update.
+ (propagate_unlikely_bbs_forward): Update.
+ (determine_unlikely_bbs): Update.
+ (force_edge_cold): Update.
+ * profile.c (compute_branch_probabilities): Update.
+ * reg-stack.c (better_edge): Update.
+ * shrink-wrap.c (handle_simple_exit): Update.
+ * tracer.c (better_p): Update.
+ * trans-mem.c (expand_transaction): Update.
+ (split_bb_make_tm_edge): Update.
+ * tree-call-cdce.c: Update.
+ * tree-cfg.c (gimple_find_sub_bbs): Update.
+ (gimple_split_edge): Update.
+ (gimple_duplicate_sese_region): Update.
+ (gimple_duplicate_sese_tail): Update.
+ (gimple_flow_call_edges_add): Update.
+ (insert_cond_bb): Update.
+ (execute_fixup_cfg): Update.
+ * tree-cfgcleanup.c (cleanup_control_expr_graph): Update.
+ * tree-complex.c (expand_complex_div_wide): Update.
+ * tree-eh.c (lower_resx): Update.
+ (unsplit_eh): Update.
+ (cleanup_empty_eh_move_lp): Update.
+ * tree-inline.c (copy_edges_for_bb): Update.
+ (freqs_to_counts): Update.
+ (copy_cfg_body): Update.
+ * tree-ssa-dce.c (remove_dead_stmt): Update.
+ * tree-ssa-ifcombine.c (update_profile_after_ifcombine): Update.
+ * tree-ssa-loop-im.c (execute_sm_if_changed): Update.
+ * tree-ssa-loop-ivcanon.c (remove_exits_and_undefined_stmts): Update.
+ (unloop_loops): Update.
+ * tree-ssa-loop-manip.c (tree_transform_and_unroll_loop): Update.
+ * tree-ssa-loop-split.c (connect_loops): Update.
+ (split_loop): Update.
+ * tree-ssa-loop-unswitch.c (hoist_guard): Update.
+ * tree-ssa-phionlycprop.c (propagate_rhs_into_lhs): Update.
+ * tree-ssa-phiopt.c (replace_phi_edge_with_variable): Update.
+ * tree-ssa-reassoc.c (branch_fixup): Update.
+ * tree-ssa-tail-merge.c (replace_block_by): Update.
+ * tree-ssa-threadupdate.c (remove_ctrl_stmt_and_useless_edges): Update.
+ (compute_path_counts): Update.
+ (update_profile): Update.
+ (recompute_probabilities): Update.
+ (update_joiner_offpath_counts): Update.
+ (estimated_freqs_path): Update.
+ (freqs_to_counts_path): Update.
+ (clear_counts_path): Update.
+ (ssa_fix_duplicate_block_edges): Update.
+ (duplicate_thread_path): Update.
+ * tree-switch-conversion.c (hoist_edge_and_branch_if_true): Update.
+ (case_bit_test_cmp): Update.
+ (collect_switch_conv_info): Update.
+ (gen_inbound_check): Update.
+ (do_jump_if_equal): Update.
+ (emit_cmp_and_jump_insns): Update.
+ * tree-tailcall.c (decrease_profile): Update.
+ (eliminate_tail_call): Update.
+ * tree-vect-loop-manip.c (slpeel_add_loop_guard): Update.
+ (vect_do_peeling): Update.
+ * tree-vect-loop.c (scale_profile_for_vect_loop): Update.
+ * ubsan.c (ubsan_expand_null_ifn): Update.
+ (ubsan_expand_ptr_ifn): Update.
+ * value-prof.c (gimple_divmod_fixed_value): Update.
+ (gimple_mod_pow2): Update.
+ (gimple_mod_subtract): Update.
+ (gimple_ic): Update.
+ (gimple_stringop_fixed_value): Update.
+
2017-10-19 Uros Bizjak <ubizjak@gmail.com>
PR target/82618
/* Set up the fallthrough basic block. */
e = find_edge (cond_bb, fallthru_bb);
e->flags = EDGE_FALSE_VALUE;
- e->count = cond_bb->count;
e->probability = fallthrough_probability;
/* Update dominance info for the newly created then_bb; note that
if (!is_edge_annotated (e, *annotated_edge))
num_unknown_edge++, unknown_edge = e;
else
- total_known_count += e->count;
+ total_known_count += e->count ();
if (num_unknown_edge == 0)
{
}
else if (num_unknown_edge == 1 && is_bb_annotated (bb, *annotated_bb))
{
- unknown_edge->count = bb->count - total_known_count;
+ unknown_edge->probability
+ = total_known_count.probability_in (bb->count);
set_edge_annotated (unknown_edge, annotated_edge);
changed = true;
}
if (!e->probability.initialized_p ()
&& !is_edge_annotated (ep, *annotated_edge))
{
- ep->probability = profile_probability::never ();
- ep->count = profile_count::zero ().afdo ();
+ ep->probability = profile_probability::never ().afdo ();
set_edge_annotated (ep, annotated_edge);
}
}
if (total == 1 && !is_edge_annotated (only_one, *annotated_edge))
{
only_one->probability = e->probability;
- only_one->count = e->count;
set_edge_annotated (only_one, annotated_edge);
}
}
if (!is_edge_annotated (e, *annotated_edge))
num_unknown_succ++;
else
- total_count += e->count;
+ total_count += e->count ();
}
if (num_unknown_succ == 0 && total_count > profile_count::zero ())
{
FOR_EACH_EDGE (e, ei, bb->succs)
- e->probability = e->count.probability_in (total_count);
+ e->probability = e->count ().probability_in (total_count);
}
}
FOR_ALL_BB_FN (bb, cfun)
- {
- edge e;
- edge_iterator ei;
-
- FOR_EACH_EDGE (e, ei, bb->succs)
- e->count = bb->count.apply_probability (e->probability);
bb->aux = NULL;
- }
loop_optimizer_finalize ();
free_dominance_info (CDI_DOMINATORS);
counters are zero when not seen by autoFDO. */
bb->count = profile_count::zero ().afdo ();
FOR_EACH_EDGE (e, ei, bb->succs)
- e->count = profile_count::zero ().afdo ();
+ e->probability = profile_probability::uninitialized ();
if (afdo_set_bb_count (bb, promoted_stmts))
set_bb_annotated (bb, &annotated_bb);
int flags; /* see cfg-flags.def */
profile_probability probability;
- profile_count count; /* Expected number of executions calculated
- in profile.c */
+
+ /* Return count of edge E. */
+ inline profile_count count () const;
};
/* Masks for edge.flags. */
return false;
}
+/* Return count of edge E. */
+inline profile_count edge_def::count () const
+{
+ return src->count.apply_probability (probability);
+}
+
#endif /* GCC_BASIC_BLOCK_H */
{
/* The current edge E is also preferred. */
int freq = EDGE_FREQUENCY (e);
- if (freq > best_freq || e->count > best_count)
+ if (freq > best_freq || e->count () > best_count)
{
best_freq = freq;
- if (e->count.initialized_p ())
- best_count = e->count;
+ if (e->count ().initialized_p ())
+ best_count = e->count ();
best_edge = e;
best_bb = bb;
}
/* The current edge E is preferred. */
is_preferred = true;
best_freq = EDGE_FREQUENCY (e);
- best_count = e->count;
+ best_count = e->count ();
best_edge = e;
best_bb = bb;
}
else
{
int freq = EDGE_FREQUENCY (e);
- if (!best_edge || freq > best_freq || e->count > best_count)
+ if (!best_edge || freq > best_freq || e->count () > best_count)
{
best_freq = freq;
- best_count = e->count;
+ best_count = e->count ();
best_edge = e;
best_bb = bb;
}
|| !prob.initialized_p ()
|| ((prob.to_reg_br_prob_base () < branch_th
|| EDGE_FREQUENCY (e) < exec_th
- || e->count < count_th) && (!for_size)))
+ || e->count () < count_th) && (!for_size)))
continue;
/* If partitioning hot/cold basic blocks, don't consider edges
|| !prob.initialized_p ()
|| prob.to_reg_br_prob_base () < branch_th
|| freq < exec_th
- || e->count < count_th)
+ || e->count () < count_th)
{
/* When partitioning hot/cold basic blocks, make sure
the cold blocks (and only the cold blocks) all get
&& !connected[bbd[di].start_of_trace]
&& BB_PARTITION (e2->dest) == current_partition
&& EDGE_FREQUENCY (e2) >= freq_threshold
- && e2->count >= count_threshold
+ && e2->count () >= count_threshold
&& (!best2
|| e2->probability > best2->probability
|| (e2->probability == best2->probability
&& copy_bb_p (best->dest,
optimize_edge_for_speed_p (best)
&& EDGE_FREQUENCY (best) >= freq_threshold
- && (!best->count.initialized_p ()
- || best->count >= count_threshold)))
+ && (!best->count ().initialized_p ()
+ || best->count () >= count_threshold)))
{
basic_block new_bb;
/* Do not expect profile insanities when profile was not adjusted. */
if (e->probability == profile_probability::never ()
- || e->count == profile_count::zero ())
+ || e->count () == profile_count::zero ())
continue;
if (BB_PARTITION (reach_bb) != BB_COLD_PARTITION)
/* The following loop will look for the hottest edge via
the edge count, if it is non-zero, then fallback to the edge
frequency and finally the edge probability. */
- if (!highest_count.initialized_p () || e->count > highest_count)
- highest_count = e->count;
+ if (!highest_count.initialized_p () || e->count () > highest_count)
+ highest_count = e->count ();
int edge_freq = EDGE_FREQUENCY (e);
if (edge_freq > highest_freq)
highest_freq = edge_freq;
continue;
/* Do not expect profile insanities when profile was not adjusted. */
if (e->probability == profile_probability::never ()
- || e->count == profile_count::zero ())
+ || e->count () == profile_count::zero ())
continue;
/* Select the hottest edge using the edge count, if it is non-zero,
then fallback to the edge frequency and finally the edge
probability. */
if (highest_count > 0)
{
- if (e->count < highest_count)
+ if (e->count () < highest_count)
continue;
}
else if (highest_freq)
e = ggc_cleared_alloc<edge_def> ();
n_edges_for_fn (cfun)++;
- e->count = profile_count::uninitialized ();
e->probability = profile_probability::uninitialized ();
e->src = src;
e->dest = dst;
edge e = make_edge (src, dest, flags);
e->probability = profile_probability::always ();
- e->count = src->count;
return e;
}
";; %sInvalid sum of outgoing probabilities %.1f%%\n",
s_indent, isum * 100.0 / REG_BR_PROB_BASE);
}
- profile_count lsum = profile_count::zero ();
- FOR_EACH_EDGE (e, ei, bb->succs)
- lsum += e->count;
- if (EDGE_COUNT (bb->succs) && lsum.differs_from_p (bb->count))
- {
- fprintf (file, ";; %sInvalid sum of outgoing counts ",
- s_indent);
- lsum.dump (file);
- fprintf (file, ", should be ");
- bb->count.dump (file);
- fprintf (file, "\n");
- }
}
}
if (bb != ENTRY_BLOCK_PTR_FOR_FN (fun))
fprintf (file,
";; %sInvalid sum of incoming frequencies %i, should be %i\n",
s_indent, sum, bb->frequency);
- profile_count lsum = profile_count::zero ();
- FOR_EACH_EDGE (e, ei, bb->preds)
- lsum += e->count;
- if (lsum.differs_from_p (bb->count))
- {
- fprintf (file, ";; %sInvalid sum of incoming counts ",
- s_indent);
- lsum.dump (file);
- fprintf (file, ", should be ");
- bb->count.dump (file);
- fprintf (file, "\n");
- }
}
if (BB_PARTITION (bb) == BB_COLD_PARTITION)
{
fprintf (file, "] ");
}
- if (e->count.initialized_p () && do_details)
+ if (e->count ().initialized_p () && do_details)
{
fputs (" count:", file);
- e->count.dump (file);
+ e->count ().dump (file);
}
if (e->flags && do_details)
}
gcc_assert (bb == taken_edge->src);
- if (dump_file && taken_edge->count < count)
- fprintf (dump_file, "edge %i->%i count became negative after threading",
- taken_edge->src->index, taken_edge->dest->index);
- taken_edge->count -= count;
}
/* Multiply all frequencies of basic blocks in array BBS of length NBBS
scale_bbs_frequencies_int (basic_block *bbs, int nbbs, int num, int den)
{
int i;
- edge e;
if (num < 0)
num = 0;
for (i = 0; i < nbbs; i++)
{
- edge_iterator ei;
bbs[i]->frequency = RDIV (bbs[i]->frequency * num, den);
/* Make sure the frequencies do not grow over BB_FREQ_MAX. */
if (bbs[i]->frequency > BB_FREQ_MAX)
bbs[i]->frequency = BB_FREQ_MAX;
bbs[i]->count = bbs[i]->count.apply_scale (num, den);
- FOR_EACH_EDGE (e, ei, bbs[i]->succs)
- e->count = e->count.apply_scale (num, den);
}
}
gcov_type den)
{
int i;
- edge e;
gcov_type fraction = RDIV (num * 65536, den);
gcc_assert (fraction >= 0);
if (num < MAX_SAFE_MULTIPLIER)
for (i = 0; i < nbbs; i++)
{
- edge_iterator ei;
bbs[i]->frequency = RDIV (bbs[i]->frequency * num, den);
if (bbs[i]->count <= MAX_SAFE_MULTIPLIER)
bbs[i]->count = bbs[i]->count.apply_scale (num, den);
else
bbs[i]->count = bbs[i]->count.apply_scale (fraction, 65536);
- FOR_EACH_EDGE (e, ei, bbs[i]->succs)
- if (bbs[i]->count <= MAX_SAFE_MULTIPLIER)
- e->count = e->count.apply_scale (num, den);
- else
- e->count = e->count.apply_scale (fraction, 65536);
}
else
for (i = 0; i < nbbs; i++)
{
- edge_iterator ei;
if (sizeof (gcov_type) > sizeof (int))
bbs[i]->frequency = RDIV (bbs[i]->frequency * num, den);
else
bbs[i]->frequency = RDIV (bbs[i]->frequency * fraction, 65536);
bbs[i]->count = bbs[i]->count.apply_scale (fraction, 65536);
- FOR_EACH_EDGE (e, ei, bbs[i]->succs)
- e->count = e->count.apply_scale (fraction, 65536);
}
}
profile_count num, profile_count den)
{
int i;
- edge e;
for (i = 0; i < nbbs; i++)
{
- edge_iterator ei;
bbs[i]->frequency = RDIV (bbs[i]->frequency * num.to_gcov_type (),
den.to_gcov_type ());
bbs[i]->count = bbs[i]->count.apply_scale (num, den);
- FOR_EACH_EDGE (e, ei, bbs[i]->succs)
- e->count = e->count.apply_scale (num, den);
}
}
profile_probability p)
{
int i;
- edge e;
for (i = 0; i < nbbs; i++)
{
- edge_iterator ei;
bbs[i]->frequency = p.apply (bbs[i]->frequency);
bbs[i]->count = bbs[i]->count.apply_probability (p);
- FOR_EACH_EDGE (e, ei, bbs[i]->succs)
- e->count = e->count.apply_probability (p);
}
}
basic_block deadend_block = dfs_find_deadend (unvisited_block);
edge e = make_edge (deadend_block, EXIT_BLOCK_PTR_FOR_FN (cfun),
EDGE_FAKE);
- e->count = profile_count::zero ();
e->probability = profile_probability::never ();
dfs.add_bb (deadend_block);
}
e = BRANCH_EDGE (b);
e->probability
= profile_probability::from_reg_br_prob_note (probability);
- e->count = b->count.apply_probability (e->probability);
f = FALLTHRU_EDGE (b);
f->probability = e->probability.invert ();
- f->count = b->count - e->count;
return;
}
else
{
e = single_succ_edge (b);
e->probability = profile_probability::always ();
- e->count = b->count;
return;
}
else
if (complex_edge)
guess_outgoing_edge_probabilities (b);
}
-
- if (b->count.initialized_p ())
- FOR_EACH_EDGE (e, ei, b->succs)
- e->count = b->count.apply_probability (e->probability);
}
/* Assume that some pass has inserted labels or control flow
bb->frequency = 0;
FOR_EACH_EDGE (e, ei, bb->preds)
{
- if (e->count.initialized_p ())
+ if (e->count ().initialized_p ())
{
- bb->count += e->count;
+ bb->count += e->count ();
initialized_src = true;
}
else
else
{
/* Save the values now, as the edge may get removed. */
- profile_count edge_count = e->count;
+ profile_count edge_count = e->count ();
profile_probability edge_probability = e->probability;
int edge_frequency;
int n = 0;
t = single_succ_edge (first);
}
- t->count -= edge_count;
first = t->dest;
}
while (first != target);
break;
}
- s->count += s2->count;
-
/* Take care to update possible forwarder blocks. We verified
that there is no more than one in the chain, so we can't run
into infinite loop. */
if (FORWARDER_BLOCK_P (s->dest))
{
- single_succ_edge (s->dest)->count += s2->count;
- s->dest->count += s2->count;
s->dest->frequency += EDGE_FREQUENCY (s);
}
if (FORWARDER_BLOCK_P (s2->dest))
{
- single_succ_edge (s2->dest)->count -= s2->count;
- s2->dest->count -= s2->count;
s2->dest->frequency -= EDGE_FREQUENCY (s);
if (s2->dest->frequency < 0)
s2->dest->frequency = 0;
dest = false_edge->dest;
redirect_edge_succ (false_edge, new_bb);
false_edge->flags |= EDGE_FALLTHRU;
- new_bb->count = false_edge->count;
+ new_bb->count = false_edge->count ();
new_bb->frequency = EDGE_FREQUENCY (false_edge);
loop_p loop = find_common_loop (bb->loop_father, dest->loop_father);
add_bb_to_loop (new_bb, loop);
the exit block. */
probability = profile_probability::never ();
- profile_count count = profile_count::zero ();
for (ei = ei_start (bb->succs); (e = ei_safe_edge (ei)); )
{
{
if (e->dest != EXIT_BLOCK_PTR_FOR_FN (cfun))
{
- e->dest->count -= e->count;
e->dest->frequency -= EDGE_FREQUENCY (e);
if (e->dest->frequency < 0)
e->dest->frequency = 0;
}
- count += e->count;
probability += e->probability;
remove_edge (e);
}
e = make_edge (bb, EXIT_BLOCK_PTR_FOR_FN (cfun), EDGE_ABNORMAL
| EDGE_SIBCALL);
e->probability = probability;
- e->count = count;
BB_END (bb) = last;
update_bb_for_insn (bb);
FOR_EACH_EDGE (e2, ei, EXIT_BLOCK_PTR_FOR_FN (cfun)->preds)
if (e2 != e)
{
- e->count -= e2->count;
- exit_block->count -= e2->count;
+ exit_block->count -= e2->count ();
exit_block->frequency -= EDGE_FREQUENCY (e2);
}
if (exit_block->frequency < 0)
e->src->index, e->dest->index);
err = 1;
}
- if (!e->count.verify ())
- {
- error ("verify_flow_info: Wrong count of edge %i->%i",
- e->src->index, e->dest->index);
- err = 1;
- }
last_visited [e->dest->index] = bb;
{
s->flags |= e->flags;
s->probability += e->probability;
- s->count += e->count;
/* FIXME: This should be called via a hook and only for IR_GIMPLE. */
redirect_edge_var_map_dup (s, e);
remove_edge (e);
split_edge (edge e)
{
basic_block ret;
- profile_count count = e->count;
+ profile_count count = e->count ();
int freq = EDGE_FREQUENCY (e);
edge f;
bool irr = (e->flags & EDGE_IRREDUCIBLE_LOOP) != 0;
ret->count = count;
ret->frequency = freq;
single_succ_edge (ret)->probability = profile_probability::always ();
- single_succ_edge (ret)->count = count;
if (irr)
{
dummy = fallthru->src;
dummy->count = profile_count::zero ();
dummy->frequency = 0;
- fallthru->count = profile_count::zero ();
bb = fallthru->dest;
/* Redirect back edges we want to keep. */
if (dummy->frequency > BB_FREQ_MAX)
dummy->frequency = BB_FREQ_MAX;
- dummy->count += e->count;
- fallthru->count += e->count;
+ dummy->count += e->count ();
ei_next (&ei);
continue;
}
{
edge s, n;
basic_block new_bb;
- profile_count new_count = e ? e->count : profile_count::uninitialized ();
+ profile_count new_count = e ? e->count (): profile_count::uninitialized ();
edge_iterator ei;
if (!cfg_hooks->duplicate_block)
is no need to actually check for duplicated edges. */
n = unchecked_make_edge (new_bb, s->dest, s->flags);
n->probability = s->probability;
- if (e && bb->count > profile_count::zero ())
- {
- n->count = s->count.apply_scale (new_count, bb->count);
- s->count -= n->count;
- }
- else
- n->count = s->count;
n->aux = s->aux;
}
record->num_mismatched_freq_out[after_pass]++;
profile_count lsum = profile_count::zero ();
FOR_EACH_EDGE (e, ei, bb->succs)
- lsum += e->count;
+ lsum += e->count ();
if (EDGE_COUNT (bb->succs) && (lsum.differs_from_p (bb->count)))
record->num_mismatched_count_out[after_pass]++;
}
record->num_mismatched_freq_in[after_pass]++;
profile_count lsum = profile_count::zero ();
FOR_EACH_EDGE (e, ei, bb->preds)
- lsum += e->count;
+ lsum += e->count ();
if (lsum.differs_from_p (bb->count))
record->num_mismatched_count_in[after_pass]++;
}
FOR_EACH_VEC_ELT (latches, i, e)
{
- if (e->count > mcount)
+ if (e->count ()> mcount)
{
me = e;
- mcount = e->count;
+ mcount = e->count();
}
- tcount += e->count;
+ tcount += e->count();
}
if (!tcount.initialized_p () || tcount < HEAVY_EDGE_MIN_SAMPLES
FOR_EACH_EDGE (e, ei, loop->header->preds)
if (e->src == loop->latch)
- count_latch = e->count;
+ count_latch = e->count ();
else
- count_in += e->count;
+ count_in += e->count ();
if (!count_latch.initialized_p ())
;
/* Probability of exit must be 1/iterations. */
freq_delta = EDGE_FREQUENCY (e);
+ count_delta = e->count ();
e->probability = profile_probability::always ()
.apply_scale (1, iteration_bound);
other_e->probability = e->probability.invert ();
freq_delta -= EDGE_FREQUENCY (e);
-
- /* Adjust counts accordingly. */
- count_delta = e->count;
- e->count = e->src->count.apply_probability (e->probability);
- other_e->count = e->src->count.apply_probability (other_e->probability);
- count_delta -= e->count;
+ count_delta -= e->count ();
/* If latch exists, change its frequency and count, since we changed
probability of exit. Theoretically we should update everything from
FOR_EACH_EDGE (e, ei, loop->header->preds)
if (e->src != loop->latch)
- count_in += e->count;
+ count_in += e->count ();
if (count_in > profile_count::zero () )
{
struct loop *outer = loop_outer (succ_bb->loop_father);
int freq;
profile_count cnt;
- edge e;
- edge_iterator ei;
loop->header = header_edge->dest;
loop->latch = latch_edge->src;
freq = EDGE_FREQUENCY (header_edge);
- cnt = header_edge->count;
+ cnt = header_edge->count ();
/* Redirect edges. */
loop_redirect_edge (latch_edge, loop->header);
{
switch_bb->frequency = freq;
switch_bb->count = cnt;
- FOR_EACH_EDGE (e, ei, switch_bb->succs)
- {
- e->count = switch_bb->count.apply_probability (e->probability);
- }
}
scale_loop_frequencies (loop, false_scale);
scale_loop_frequencies (succ_bb->loop_father, true_scale);
current_ir_type () == IR_GIMPLE ? EDGE_TRUE_VALUE : 0);
e1->probability = then_prob;
e->probability = else_prob;
- e1->count = e->count.apply_probability (e1->probability);
- e->count = e->count.apply_probability (e->probability);
set_immediate_dominator (CDI_DOMINATORS, first_head, new_head);
set_immediate_dominator (CDI_DOMINATORS, second_head, new_head);
e->flags = 0;
e->probability = profile_probability::always ();
- e->count = src->count;
if (e->dest != target)
redirect_edge_succ (e, target);
int prob = XINT (note, 0);
b->probability = profile_probability::from_reg_br_prob_note (prob);
- b->count = e->count.apply_probability (b->probability);
e->probability -= e->probability;
- e->count -= b->count;
}
}
if (EDGE_COUNT (e->src->succs) >= 2 || abnormal_edge_flags || asm_goto_edge)
{
rtx_insn *new_head;
- profile_count count = e->count;
+ profile_count count = e->count ();
profile_probability probability = e->probability;
/* Create the new structures. */
/* Wire edge in. */
new_edge = make_edge (e->src, jump_block, EDGE_FALLTHRU);
new_edge->probability = probability;
- new_edge->count = count;
/* Redirect old edge. */
redirect_edge_pred (e, jump_block);
if (asm_goto_edge)
{
new_edge->probability = new_edge->probability.apply_scale (1, 2);
- new_edge->count = new_edge->count.apply_scale (1, 2);
jump_block->count = jump_block->count.apply_scale (1, 2);
jump_block->frequency /= 2;
edge new_edge2 = make_edge (new_edge->src, target,
e->flags & ~EDGE_FALLTHRU);
new_edge2->probability = probability - new_edge->probability;
- new_edge2->count = count - new_edge->count;
}
new_bb = jump_block;
if (single_succ_p (bb))
{
single_succ_edge (bb)->probability = profile_probability::always ();
- single_succ_edge (bb)->count = bb->count;
}
else
{
b->probability = profile_probability::from_reg_br_prob_note
(XINT (note, 0));
f->probability = b->probability.invert ();
- b->count = bb->count.apply_probability (b->probability);
- f->count = bb->count.apply_probability (f->probability);
}
return purged;
gcc_assert (single_succ_p (bb));
single_succ_edge (bb)->probability = profile_probability::always ();
- single_succ_edge (bb)->count = bb->count;
if (dump_file)
fprintf (dump_file, "Purged non-fallthru edges from bb %i\n",
edge ne = make_edge (bb, EXIT_BLOCK_PTR_FOR_FN (cfun), EDGE_FAKE);
ne->probability = profile_probability::guessed_never ();
- ne->count = profile_count::guessed_zero ();
}
if (insn == BB_HEAD (bb))
bb->count = count;
bb->frequency = BB_FREQ_MAX;
e = make_edge (ENTRY_BLOCK_PTR_FOR_FN (cfun), bb, EDGE_FALLTHRU);
- e->count = count;
e->probability = profile_probability::always ();
e = make_edge (bb, EXIT_BLOCK_PTR_FOR_FN (cfun), 0);
- e->count = count;
e->probability = profile_probability::always ();
add_bb_to_loop (bb, ENTRY_BLOCK_PTR_FOR_FN (cfun)->loop_father);
e = make_edge (bb, then_bb, EDGE_TRUE_VALUE);
e->probability = profile_probability::guessed_always ()
.apply_scale (1, 16);
- e->count = count - count.apply_scale (1, 16);
e = make_edge (bb, else_bb, EDGE_FALSE_VALUE);
e->probability = profile_probability::guessed_always ()
.apply_scale (1, 16);
- e->count = count.apply_scale (1, 16);
make_single_succ_edge (return_bb,
EXIT_BLOCK_PTR_FOR_FN (cfun), 0);
make_single_succ_edge (then_bb, return_bb, EDGE_FALLTHRU);
e = make_edge (else_bb, return_bb, EDGE_FALLTHRU);
e->probability = profile_probability::always ();
- e->count = count.apply_scale (1, 16);
bsi = gsi_last_bb (then_bb);
}
by xstrdup_for_dump. */
static const char *
-dump_probability (profile_probability probability, profile_count &count)
+dump_probability (profile_probability probability)
{
float minimum = 0.01f;
float fvalue = -1;
}
char *buf;
- if (count.initialized_p ())
- buf = xasprintf ("[%.2f%%] [count: %" PRId64 "]", fvalue,
- count.to_gcov_type ());
- else if (probability.initialized_p ())
- buf = xasprintf ("[%.2f%%] [count: INV]", fvalue);
+ if (probability.initialized_p ())
+ buf = xasprintf ("[%.2f%%]", fvalue);
else
- buf = xasprintf ("[INV] [count: INV]");
+ buf = xasprintf ("[INV]");
const char *ret = xstrdup_for_dump (buf);
free (buf);
static void
dump_edge_probability (pretty_printer *buffer, edge e)
{
- pp_scalar (buffer, " %s", dump_probability (e->probability, e->count));
+ pp_scalar (buffer, " %s", dump_probability (e->probability));
}
/* Print GIMPLE statement G to FILE using SPC indentation spaces and
/* Update profile only when redirection is really processed. */
bb->frequency += EDGE_FREQUENCY (e);
- bb->count += e->count;
}
/* There may be more than one statement in DUPLICATE which exhibits
'todo_spec' variable in create_check_block_twin and
in sel-sched.c `check_ds' in create_speculation_check. */
e->probability = profile_probability::very_unlikely ();
- e->count = first_bb->count.apply_probability (e->probability);
- rec->count = e->count;
+ rec->count = e->count ();
rec->frequency = EDGE_FREQUENCY (e);
e2->probability = e->probability.invert ();
- e2->count = first_bb->count - e2->count;
rtx_code_label *label = block_label (second_bb);
rtx_jump_insn *jump = emit_jump_insn_after (targetm.gen_jump (label),
tree label = gimple_switch_label (s, i);
basic_block label_bb = label_to_block_fn (func, CASE_LABEL (label));
edge e = find_edge (bb, label_bb);
- edge_counts.safe_push (e->count);
+ edge_counts.safe_push (e->count ());
edge_probabilities.safe_push (e->probability);
gphi_iterator phi_gsi;
if (prob_sum.initialized_p ())
new_edge->probability = edge_probabilities[i] / prob_sum;
- new_edge->count = edge_counts[i];
new_edges.safe_push (new_edge);
if (i < labels - 1)
edge next_edge = make_edge (cur_bb, next_bb, EDGE_FALSE_VALUE);
next_edge->probability = new_edge->probability.invert ();
- next_edge->count = edge_counts[0]
- + sum_slice <profile_count> (edge_counts, i, labels,
- profile_count::zero ());
next_bb->frequency = EDGE_FREQUENCY (next_edge);
cur_bb = next_bb;
}
{
edge e = make_edge (cur_bb, default_label_bb, EDGE_FALSE_VALUE);
e->probability = new_edge->probability.invert ();
- e->count = edge_counts[0];
new_edges.safe_insert (0, e);
}
}
redirect_edge_succ (BRANCH_EDGE (test_bb), new_dest);
if (reversep)
{
- std::swap (BRANCH_EDGE (test_bb)->count,
- FALLTHRU_EDGE (test_bb)->count);
std::swap (BRANCH_EDGE (test_bb)->probability,
FALLTHRU_EDGE (test_bb)->probability);
update_br_prob_note (test_bb);
basic_block bb;
FOR_ALL_BB_FN (bb, cfun)
- {
- bb->count = bb->count.apply_scale (num, den);
-
- edge e;
- edge_iterator ei;
- FOR_EACH_EDGE (e, ei, bb->succs)
- e->count = e->count.apply_scale (num, den);
- }
+ bb->count = bb->count.apply_scale (num, den);
ENTRY_BLOCK_PTR_FOR_FN (cfun)->count = node->count;
}
todo = optimize_inline_calls (current_function_decl);
FOR_EACH_EDGE (e, ei, return_bb->preds)
if (bitmap_bit_p (split_point->split_bbs, e->src->index))
{
- new_return_bb->count += e->count;
new_return_bb->frequency += EDGE_FREQUENCY (e);
redirect_edge_and_branch (e, new_return_bb);
redirected = true;
unsigned int i;
dstbb = BASIC_BLOCK_FOR_FN (dstcfun, srcbb->index);
- if (dstbb->count.initialized_p ())
- dstbb->count += srcbb->count;
- else
- dstbb->count = srcbb->count;
- for (i = 0; i < EDGE_COUNT (srcbb->succs); i++)
+ if (!dstbb->count.initialized_p ())
{
- edge srce = EDGE_SUCC (srcbb, i);
- edge dste = EDGE_SUCC (dstbb, i);
- if (dstbb->count.initialized_p ())
- dste->count += srce->count;
- else
- dste->count = srce->count;
- if (dstbb->count > 0 && dste->count.initialized_p ())
- dste->probability = dste->count.probability_in (dstbb->count);
+ dstbb->count = srcbb->count;
+ for (i = 0; i < EDGE_COUNT (srcbb->succs); i++)
+ {
+ edge srce = EDGE_SUCC (srcbb, i);
+ edge dste = EDGE_SUCC (dstbb, i);
+ if (srce->probability.initialized_p ())
+ dste->probability = srce->probability;
+ }
+ }
+ else if (srcbb->count.initialized_p ())
+ {
+ for (i = 0; i < EDGE_COUNT (srcbb->succs); i++)
+ {
+ edge srce = EDGE_SUCC (srcbb, i);
+ edge dste = EDGE_SUCC (dstbb, i);
+ dste->probability =
+ dste->probability * dstbb->count.probability_in (dstbb->count + srcbb->count)
+ + srce->probability * srcbb->count.probability_in (dstbb->count + srcbb->count);
+ }
+ dstbb->count += srcbb->count;
}
}
push_cfun (dstcfun);
edge e2 = make_edge (bb, dest, (*e)->flags & ~EDGE_FALLTHRU);
e2->probability = prob;
- e2->count = e2->src->count.apply_probability (prob);
(*e)->probability = prob.invert ();
- (*e)->count = (*e)->count.apply_probability (prob);
update_br_prob_note (e2->src);
return true;
}
iter_count = new_count = swtch->count.apply_scale (1, max_unroll + 1);
swtch->frequency = new_freq;
swtch->count = new_count;
- single_succ_edge (swtch)->count = new_count;
for (i = 0; i < n_peel; i++)
{
/* Add in frequency/count of edge from switch block. */
preheader->frequency += iter_freq;
preheader->count += iter_count;
- single_succ_edge (preheader)->count = preheader->count;
branch_code = compare_and_jump_seq (copy_rtx (niter), GEN_INT (j), EQ,
block_label (preheader), p,
NULL);
swtch = split_edge_and_insert (single_pred_edge (swtch), branch_code);
set_immediate_dominator (CDI_DOMINATORS, preheader, swtch);
single_succ_edge (swtch)->probability = p.invert ();
- single_succ_edge (swtch)->count = new_count;
new_freq += iter_freq;
new_count += iter_count;
swtch->frequency = new_freq;
swtch->count = new_count;
e = make_edge (swtch, preheader,
single_succ_edge (swtch)->flags & EDGE_IRREDUCIBLE_LOOP);
- e->count = iter_count;
e->probability = p;
}
/* Add in frequency/count of edge from switch block. */
preheader->frequency += iter_freq;
preheader->count += iter_count;
- single_succ_edge (preheader)->count = preheader->count;
branch_code = compare_and_jump_seq (copy_rtx (niter), const0_rtx, EQ,
block_label (preheader), p,
NULL);
swtch = split_edge_and_insert (single_succ_edge (swtch), branch_code);
set_immediate_dominator (CDI_DOMINATORS, preheader, swtch);
single_succ_edge (swtch)->probability = p.invert ();
- single_succ_edge (swtch)->count -= iter_count;
e = make_edge (swtch, preheader,
single_succ_edge (swtch)->flags & EDGE_IRREDUCIBLE_LOOP);
- e->count = iter_count;
e->probability = p;
}
static void
input_cfg (struct lto_input_block *ib, struct data_in *data_in,
- struct function *fn,
- int count_materialization_scale)
+ struct function *fn)
{
unsigned int bb_count;
basic_block p_bb;
unsigned int edge_flags;
basic_block dest;
profile_probability probability;
- profile_count count;
edge e;
dest_index = streamer_read_uhwi (ib);
probability = profile_probability::stream_in (ib);
- count = profile_count::stream_in (ib).apply_scale
- (count_materialization_scale, REG_BR_PROB_BASE);
edge_flags = streamer_read_uhwi (ib);
dest = BASIC_BLOCK_FOR_FN (fn, dest_index);
e = make_edge (bb, dest, edge_flags);
e->probability = probability;
- e->count = count;
}
index = streamer_read_hwi (ib);
if (!node)
node = cgraph_node::create (fn_decl);
input_struct_function_base (fn, data_in, ib);
- input_cfg (ib_cfg, data_in, fn, node->count_materialization_scale);
+ input_cfg (ib_cfg, data_in, fn);
/* Read all the SSA names. */
input_ssa_names (ib, data_in, fn);
{
streamer_write_uhwi (ob, e->dest->index);
e->probability.stream_out (ob);
- e->count.stream_out (ob);
streamer_write_uhwi (ob, e->flags);
}
}
get_ebb_head_tail (bb, bb, &head, &tail);
latch_edge = loop_latch_edge (loop);
gcc_assert (single_exit (loop));
- if (single_exit (loop)->count > profile_count::zero ())
- trip_count = latch_edge->count.to_gcov_type ()
- / single_exit (loop)->count.to_gcov_type ();
+ if (single_exit (loop)->count () > profile_count::zero ())
+ trip_count = latch_edge->count ().to_gcov_type ()
+ / single_exit (loop)->count ().to_gcov_type ();
/* Perform SMS only on loops that their average count is above threshold. */
- if ( latch_edge->count > profile_count::zero ()
- && (latch_edge->count
- < single_exit (loop)->count.apply_scale
+ if ( latch_edge->count () > profile_count::zero ()
+ && (latch_edge->count()
+ < single_exit (loop)->count ().apply_scale
(SMS_LOOP_AVERAGE_COUNT_THRESHOLD, 1)))
{
if (dump_file)
latch_edge = loop_latch_edge (loop);
gcc_assert (single_exit (loop));
- if (single_exit (loop)->count > profile_count::zero ())
- trip_count = latch_edge->count.to_gcov_type ()
- / single_exit (loop)->count.to_gcov_type ();
+ if (single_exit (loop)->count ()> profile_count::zero ())
+ trip_count = latch_edge->count ().to_gcov_type ()
+ / single_exit (loop)->count ().to_gcov_type ();
if (dump_file)
{
avail_insn = NULL;
}
- if (EDGE_CRITICAL_P (pred) && pred->count.initialized_p ())
- critical_count += pred->count;
+ if (EDGE_CRITICAL_P (pred) && pred->count ().initialized_p ())
+ critical_count += pred->count ();
if (avail_insn != NULL_RTX)
{
npred_ok++;
- if (pred->count.initialized_p ())
- ok_count = ok_count + pred->count;
+ if (pred->count ().initialized_p ())
+ ok_count = ok_count + pred->count ();
if (! set_noop_p (PATTERN (gen_move_insn (copy_rtx (dest),
copy_rtx (avail_reg)))))
{
/* Adding a load on a critical edge will cause a split. */
if (EDGE_CRITICAL_P (pred))
critical_edge_split = true;
- if (pred->count.initialized_p ())
- not_ok_count = not_ok_count + pred->count;
+ if (pred->count ().initialized_p ())
+ not_ok_count = not_ok_count + pred->count ();
unoccr = (struct unoccr *) obstack_alloc (&unoccr_obstack,
sizeof (struct unoccr));
unoccr->insn = NULL;
bool
maybe_hot_edge_p (edge e)
{
- if (!maybe_hot_count_p (cfun, e->count))
+ if (!maybe_hot_count_p (cfun, e->count ()))
return false;
return maybe_hot_frequency_p (cfun, EDGE_FREQUENCY (e));
}
static bool
unlikely_executed_edge_p (edge e)
{
- return (e->count == profile_count::zero ()
+ return (e->count () == profile_count::zero ()
|| e->probability == profile_probability::never ())
|| (e->flags & (EDGE_EH | EDGE_FAKE));
}
{
if (unlikely_executed_edge_p (e))
return true;
- return probably_never_executed (fun, e->count, EDGE_FREQUENCY (e));
+ return probably_never_executed (fun, e->count (), EDGE_FREQUENCY (e));
}
/* Return true when current function should always be optimized for size. */
if (e)
{
fprintf (file, " hit ");
- e->count.dump (file);
- fprintf (file, " (%.1f%%)", e->count.to_gcov_type() * 100.0
+ e->count ().dump (file);
+ fprintf (file, " (%.1f%%)", e->count ().to_gcov_type() * 100.0
/ bb->count.to_gcov_type ());
}
}
FOR_ALL_BB_FN (bb, fn)
{
bb->count = profile_count::uninitialized ();
-
- edge_iterator ei;
- edge e;
- FOR_EACH_EDGE (e, ei, bb->preds)
- e->count = profile_count::uninitialized ();
}
struct cgraph_edge *e;
for (e = node->callees; e; e = e->next_caller)
{
- e->count = profile_count::uninitialized ();
e->frequency = compute_call_stmt_bb_frequency (e->caller->decl,
gimple_bb (e->call_stmt));
}
- node->count = profile_count::uninitialized ();
profile_status_for_fn (fn)
= (flag_guess_branch_prob ? PROFILE_GUESSED : PROFILE_ABSENT);
{
bb = worklist.pop ();
FOR_EACH_EDGE (e, ei, bb->succs)
- if (!(e->count == profile_count::zero ())
+ if (!(e->count () == profile_count::zero ())
&& !(e->dest->count == profile_count::zero ())
&& !e->dest->aux)
{
bb->index);
bb->count = profile_count::zero ();
bb->frequency = 0;
- FOR_EACH_EDGE (e, ei, bb->succs)
- e->count = profile_count::zero ();
}
else
bb->aux = NULL;
}
if (bb->count == profile_count::zero ())
- {
- bb->frequency = 0;
- FOR_EACH_EDGE (e, ei, bb->preds)
- e->count = profile_count::zero ();
- }
+ bb->frequency = 0;
FOR_EACH_EDGE (e, ei, bb->succs)
- if (!(e->count == profile_count::zero ())
+ if (!(e->probability == profile_probability::never ())
&& unlikely_executed_edge_p (e))
{
if (dump_file && (dump_flags & TDF_DETAILS))
fprintf (dump_file, "Edge %i->%i is locally unlikely\n",
bb->index, e->dest->index);
- e->count = profile_count::zero ();
+ e->probability = profile_probability::never ();
}
gcc_checking_assert (!bb->aux);
{
nsuccs[bb->index] = 0;
FOR_EACH_EDGE (e, ei, bb->succs)
- if (!(e->count == profile_count::zero ()))
+ if (!(e->probability == profile_probability::never ())
+ && !(e->dest->count == profile_count::zero ()))
nsuccs[bb->index]++;
if (!nsuccs[bb->index])
worklist.safe_push (bb);
bb->count = profile_count::zero ();
bb->frequency = 0;
FOR_EACH_EDGE (e, ei, bb->preds)
- if (!(e->count == profile_count::zero ()))
+ if (!(e->probability == profile_probability::never ()))
{
- e->count = profile_count::zero ();
+ e->probability = profile_probability::never ();
if (!(e->src->count == profile_count::zero ()))
{
nsuccs[e->src->index]--;
profile_probability prob_sum = profile_probability::never ();
edge_iterator ei;
edge e2;
- profile_count old_count = e->count;
- profile_probability old_probability = e->probability;
bool uninitialized_exit = false;
profile_probability goal = (impossible ? profile_probability::never ()
/* If edge is already improbably or cold, just return. */
if (e->probability <= goal
- && (!impossible || e->count == profile_count::zero ()))
+ && (!impossible || e->count () == profile_count::zero ()))
return;
FOR_EACH_EDGE (e2, ei, e->src->succs)
if (e2 != e)
{
- if (e2->count.initialized_p ())
- count_sum += e2->count;
+ if (e2->count ().initialized_p ())
+ count_sum += e2->count ();
else
uninitialized_exit = true;
if (e2->probability.initialized_p ())
{
if (!(e->probability < goal))
e->probability = goal;
- if (impossible)
- e->count = profile_count::zero ();
- else if (old_probability > profile_probability::never ())
- e->count = e->count.apply_probability (e->probability
- / old_probability);
- else
- e->count = e->count.apply_scale (1, REG_BR_PROB_BASE);
profile_probability prob_comp = prob_sum / e->probability.invert ();
"probability to other edges.\n",
e->src->index, e->dest->index,
impossible ? "impossible" : "cold");
- profile_count count_sum2 = count_sum + old_count - e->count;
FOR_EACH_EDGE (e2, ei, e->src->succs)
if (e2 != e)
{
- if (count_sum > 0)
- e2->count.apply_scale (count_sum2, count_sum);
e2->probability /= prob_comp;
}
if (current_ir_type () != IR_GIMPLE
fprintf (dump_file,
"Making bb %i impossible and dropping count to 0.\n",
e->src->index);
- e->count = profile_count::zero ();
e->src->count = profile_count::zero ();
FOR_EACH_EDGE (e2, ei, e->src->preds)
force_edge_cold (e2, impossible);
impossible ? "impossible" : "cold");
e->src->frequency = MIN (e->src->frequency, impossible ? 0 : 1);
if (impossible)
- e->src->count = e->count = profile_count::zero ();
+ e->src->count = profile_count::zero ();
else
- e->src->count = e->count = e->count.apply_scale (e->src->frequency,
- old_frequency);
+ e->src->count = e->count ().apply_scale (e->src->frequency,
+ old_frequency);
force_edge_cold (single_pred_edge (e->src), impossible);
}
else if (dump_file && (dump_flags & TDF_DETAILS)
FOR_ALL_BB_FN (bb, cfun)
{
- edge e;
- edge_iterator ei;
-
bb->count = profile_count::from_gcov_type (bb_gcov_count (bb));
- FOR_EACH_EDGE (e, ei, bb->succs)
- e->count = profile_count::from_gcov_type (edge_gcov_count (e));
}
bb_gcov_counts.release ();
delete edge_gcov_counts;
if (EDGE_FREQUENCY (e1) < EDGE_FREQUENCY (e2))
return e2;
- if (e1->count > e2->count)
+ if (e1->count () > e2->count ())
return e1;
- if (e1->count < e2->count)
+ if (e1->count () < e2->count ())
return e2;
/* Prefer critical edges to minimize inserting compensation code on
BB_END (old_bb) = end;
redirect_edge_succ (e, new_bb);
- new_bb->count = e->count;
new_bb->frequency = EDGE_FREQUENCY (e);
e->flags |= EDGE_FALLTHRU;
static bool
better_p (const_edge e1, const_edge e2)
{
- if (e1->count.initialized_p () && e2->count.initialized_p ()
- && !(e1->count == e2->count))
- return e1->count > e2->count;
+ if (e1->count ().initialized_p () && e2->count ().initialized_p ()
+ && ((e1->count () > e2->count ()) || (e1->count () < e2->count ())))
+ return e1->count () > e2->count ();
if (EDGE_FREQUENCY (e1) != EDGE_FREQUENCY (e2))
return EDGE_FREQUENCY (e1) > EDGE_FREQUENCY (e2);
/* This is needed to avoid changes in the decision after
ei->probability = profile_probability::always ();
et->probability = profile_probability::likely ();
ef->probability = profile_probability::unlikely ();
- et->count = test_bb->count.apply_probability (et->probability);
- ef->count = test_bb->count.apply_probability (ef->probability);
- code_bb->count = et->count;
+ code_bb->count = et->count ();
code_bb->frequency = EDGE_FREQUENCY (et);
transaction_bb = join_bb;
redirect_edge_pred (fallthru_edge, test_bb);
fallthru_edge->flags = EDGE_FALSE_VALUE;
fallthru_edge->probability = profile_probability::very_likely ();
- fallthru_edge->count = test_bb->count.apply_probability
- (fallthru_edge->probability);
// Abort/over edge.
redirect_edge_pred (abort_edge, test_bb);
abort_edge->flags = EDGE_TRUE_VALUE;
abort_edge->probability = profile_probability::unlikely ();
- abort_edge->count = test_bb->count.apply_probability
- (abort_edge->probability);
transaction_bb = test_bb;
}
// out of the fallthru edge.
edge e = make_edge (transaction_bb, test_bb, fallthru_edge->flags);
e->probability = fallthru_edge->probability;
- test_bb->count = e->count = fallthru_edge->count;
+ test_bb->count = fallthru_edge->count ();
test_bb->frequency = EDGE_FREQUENCY (e);
// Now update the edges to the inst/uninist implementations.
redirect_edge_pred (inst_edge, test_bb);
inst_edge->flags = EDGE_FALSE_VALUE;
inst_edge->probability = profile_probability::even ();
- inst_edge->count
- = test_bb->count.apply_probability (inst_edge->probability);
redirect_edge_pred (uninst_edge, test_bb);
uninst_edge->flags = EDGE_TRUE_VALUE;
uninst_edge->probability = profile_probability::even ();
- uninst_edge->count
- = test_bb->count.apply_probability (uninst_edge->probability);
}
// If we have no previous special cases, and we have PHIs at the beginning
}
edge e = make_edge (bb, dest_bb, EDGE_ABNORMAL);
if (e)
- {
- e->probability = profile_probability::guessed_never ();
- e->count = profile_count::guessed_zero ();
- }
+ e->probability = profile_probability::guessed_never ();
// Record the need for the edge for the benefit of the rtl passes.
if (cfun->gimple_df->tm_restart == NULL)
gcc_assert (src_bb == nocall_edge->src);
call_edge->probability = profile_probability::very_unlikely ();
- call_edge->count
- = src_bb->count.apply_probability (call_edge->probability);
nocall_edge->probability = profile_probability::always ()
- call_edge->probability;
- nocall_edge->count = src_bb->count - call_edge->count;
unsigned int call_frequency
= call_edge->probability.apply (src_bb->frequency);
- bi_call_bb->count += call_edge->count;
+ bi_call_bb->count += call_edge->count ();
bi_call_bb->frequency += call_frequency;
if (nocall_edge->dest != join_tgt_bb)
{
- nocall_edge->dest->count = nocall_edge->count;
nocall_edge->dest->frequency = src_bb->frequency - call_frequency;
}
}
edge_iterator ei;
FOR_EACH_EDGE (e, ei, bb->preds)
{
- if (e->count.initialized_p ())
- cnt += e->count;
+ if (e->count ().initialized_p ())
+ cnt += e->count ();
else
all = false;
freq += EDGE_FREQUENCY (e);
if (all || profile_status_for_fn (cfun) == PROFILE_READ)
bb->count = cnt;
bb->frequency = freq;
- FOR_EACH_EDGE (e, ei, bb->succs)
- e->count = bb->count.apply_probability (e->probability);
bb = bb->next_bb;
}
new_bb = create_empty_bb (after_bb);
new_bb->frequency = EDGE_FREQUENCY (edge_in);
- new_bb->count = edge_in->count;
+ new_bb->count = edge_in->count ();
e = redirect_edge_and_branch (edge_in, new_bb);
gcc_assert (e == edge_in);
if (entry->dest->count.initialized_p ())
{
total_count = entry->dest->count;
- entry_count = entry->count;
+ entry_count = entry->count ();
/* Fix up corner cases, to avoid division by zero or creation of negative
frequencies. */
if (entry_count > total_count)
if (exit->src->count > 0)
{
total_count = exit->src->count;
- exit_count = exit->count;
+ exit_count = exit->count ();
/* Fix up corner cases, to avoid division by zero or creation of negative
frequencies. */
if (exit_count > total_count)
sorig = single_succ_edge (switch_bb);
sorig->flags = exits[1]->flags;
sorig->probability = exits[1]->probability;
- sorig->count = exits[1]->count;
snew = make_edge (switch_bb, nentry_bb, exits[0]->flags);
snew->probability = exits[0]->probability;
- snew->count = exits[1]->count;
/* Register the new edge from SWITCH_BB in loop exit lists. */
}
e = make_edge (bb, EXIT_BLOCK_PTR_FOR_FN (cfun), EDGE_FAKE);
e->probability = profile_probability::guessed_never ();
- e->count = profile_count::guessed_zero ();
}
gsi_prev (&gsi);
}
new_bb = create_empty_bb (bb);
edge e = make_edge (bb, new_bb, EDGE_TRUE_VALUE);
e->probability = prob;
- e->count = bb->count.apply_probability (prob);
- new_bb->count = e->count;
+ new_bb->count = e->count ();
new_bb->frequency = prob.apply (bb->frequency);
make_single_succ_edge (new_bb, fall->dest, EDGE_FALLTHRU);
/* Fix edge for split bb. */
fall->flags = EDGE_FALSE_VALUE;
- fall->count -= e->count;
fall->probability -= e->probability;
/* Update dominance info. */
basic_block bb;
gimple_stmt_iterator gsi;
int todo = 0;
- edge e;
- edge_iterator ei;
cgraph_node *node = cgraph_node::get (current_function_decl);
profile_count num = node->count;
profile_count den = ENTRY_BLOCK_PTR_FOR_FN (cfun)->count;
ENTRY_BLOCK_PTR_FOR_FN (cfun)->count = node->count;
EXIT_BLOCK_PTR_FOR_FN (cfun)->count
= EXIT_BLOCK_PTR_FOR_FN (cfun)->count.apply_scale (num, den);
-
- FOR_EACH_EDGE (e, ei, ENTRY_BLOCK_PTR_FOR_FN (cfun)->succs)
- e->count = e->count.apply_scale (num, den);
}
FOR_EACH_BB_FN (bb, cfun)
gsi_next (&gsi);
}
- if (scale)
- FOR_EACH_EDGE (e, ei, bb->succs)
- e->count = e->count.apply_scale (num, den);
-
/* If we have a basic block with no successors that does not
end with a control statement or a noreturn call end it with
a call to __builtin_unreachable. This situation can occur
}
taken_edge->probability += e->probability;
- taken_edge->count += e->count;
remove_edge_and_dominated_blocks (e);
retval = true;
}
/* Wire the blocks together. */
e->flags = EDGE_TRUE_VALUE;
- e->count = bb_true->count;
/* TODO: With value profile we could add an historgram to determine real
branch outcome. */
e->probability = profile_probability::even ();
redirect_edge_succ (e, bb_true);
edge e2 = make_edge (bb_cond, bb_false, EDGE_FALSE_VALUE);
- e2->count = bb_false->count;
e2->probability = profile_probability::even ();
make_single_succ_edge (bb_true, bb_join, EDGE_FALLTHRU);
make_single_succ_edge (bb_false, bb_join, EDGE_FALLTHRU);
gcc_assert (e->flags & EDGE_EH);
e->flags = (e->flags & ~EDGE_EH) | EDGE_FALLTHRU;
e->probability = profile_probability::always ();
- e->count = bb->count;
/* If there are no more EH users of the landing pad, delete it. */
FOR_EACH_EDGE (e, ei, e->dest->preds)
redirect_edge_pred (e_out, e_in->src);
e_out->flags = e_in->flags;
e_out->probability = e_in->probability;
- e_out->count = e_in->count;
remove_edge (e_in);
return true;
/* Clean up E_OUT for the fallthru. */
e_out->flags = (e_out->flags & ~EDGE_EH) | EDGE_FALLTHRU;
e_out->probability = profile_probability::always ();
- e_out->count = e_out->src->count;
}
/* A subroutine of cleanup_empty_eh. Handle more complex cases of
debug stmts are left after a statement that must end the basic block. */
static bool
-copy_edges_for_bb (basic_block bb, profile_count num, profile_count den,
+copy_edges_for_bb (basic_block bb,
basic_block ret_bb, basic_block abnormal_goto_dest)
{
basic_block new_bb = (basic_block) bb->aux;
gimple_stmt_iterator si;
int flags;
bool need_debug_cleanup = false;
- bool scale = num.initialized_p ()
- && (den > 0 || num == profile_count::zero ());
/* Use the indices from the original blocks to create edges for the
new ones. */
&& old_edge->dest->aux != EXIT_BLOCK_PTR_FOR_FN (cfun))
flags |= EDGE_FALLTHRU;
new_edge = make_edge (new_bb, (basic_block) old_edge->dest->aux, flags);
- if (scale)
- new_edge->count = old_edge->count.apply_scale (num, den);
new_edge->probability = old_edge->probability;
}
&& (e = find_edge (copy_stmt_bb,
(basic_block) old_edge->dest->aux))
&& (e->flags & EDGE_EH))
- {
- e->probability = old_edge->probability;
- e->count = old_edge->count;
- }
+ e->probability = old_edge->probability;
FOR_EACH_EDGE (e, ei, copy_stmt_bb->succs)
if ((e->flags & EDGE_EH) && !e->probability.initialized_p ())
- {
- e->probability = profile_probability::never ();
- e->count = profile_count::zero ();
- }
+ e->probability = profile_probability::never ();
}
freqs_to_counts (struct cgraph_node *node, profile_count count)
{
basic_block bb;
- edge_iterator ei;
- edge e;
struct function *fn = DECL_STRUCT_FUNCTION (node->decl);
FOR_ALL_BB_FN(bb, fn)
- {
- bb->count = count.apply_scale (bb->frequency, BB_FREQ_MAX);
- FOR_EACH_EDGE (e, ei, bb->succs)
- e->count = e->src->count.apply_probability (e->probability);
- }
+ bb->count = count.apply_scale (bb->frequency, BB_FREQ_MAX);
}
/* Make a copy of the body of FN so that it can be inserted inline in
FOR_EACH_EDGE (e, ei, new_entry->preds)
if (!e->src->aux)
- {
- incoming_frequency += EDGE_FREQUENCY (e);
- incoming_count += e->count;
- }
+ incoming_frequency += EDGE_FREQUENCY (e);
if (scale)
incoming_count = incoming_count.apply_scale (num, den);
else
FOR_ALL_BB_FN (bb, cfun_to_copy)
if (!id->blocks_to_copy
|| (bb->index > 0 && bitmap_bit_p (id->blocks_to_copy, bb->index)))
- need_debug_cleanup |= copy_edges_for_bb (bb, num, den, exit_block_map,
+ need_debug_cleanup |= copy_edges_for_bb (bb, exit_block_map,
abnormal_goto_dest);
if (new_entry)
{
edge e = make_edge (entry_block_map, (basic_block)new_entry->aux, EDGE_FALLTHRU);
e->probability = profile_probability::always ();
- e->count = incoming_count;
}
/* Duplicate the loop tree, if available and wanted. */
}
gcc_assert (e);
e->probability = profile_probability::always ();
- e->count = bb->count;
/* The edge is no longer associated with a conditional, so it does
not have TRUE/FALSE flags.
outer_cond_bb->(outer_to_inner)->inner_cond_bb->(inner_taken)
and probability of inner_not_taken updated. */
- outer_to_inner->count = outer_cond_bb->count;
inner_cond_bb->count = outer_cond_bb->count;
- inner_taken->count += outer2->count;
- outer2->count = profile_count::zero ();
inner_taken->probability = outer2->probability + outer_to_inner->probability
* inner_taken->probability;
if (flag_probability.initialized_p ())
;
- else if (ncount == nbbs && count_sum > 0 && preheader->count >= count_sum)
+ else if (ncount == nbbs && count_sum > 0 && preheader->count () >= count_sum)
{
- flag_probability = count_sum.probability_in (preheader->count);
+ flag_probability = count_sum.probability_in (preheader->count ());
if (flag_probability > cap)
flag_probability = cap;
}
edge e2 = make_edge (new_bb, then_bb,
EDGE_TRUE_VALUE | (irr ? EDGE_IRREDUCIBLE_LOOP : 0));
e2->probability = flag_probability;
- e2->count = then_bb->count;
e1->flags |= EDGE_FALSE_VALUE | (irr ? EDGE_IRREDUCIBLE_LOOP : 0);
e1->flags &= ~EDGE_FALLTHRU;
e1->probability = flag_probability.invert ();
- e1->count = new_bb->count - then_bb->count;
then_old_edge = make_single_succ_edge (then_bb, old_dest,
EDGE_FALLTHRU | (irr ? EDGE_IRREDUCIBLE_LOOP : 0));
if (!loop_exit_edge_p (loop, exit_edge))
exit_edge = EDGE_SUCC (bb, 1);
exit_edge->probability = profile_probability::always ();
- exit_edge->count = exit_edge->src->count;
gcc_checking_assert (loop_exit_edge_p (loop, exit_edge));
gcond *cond_stmt = as_a <gcond *> (elt->stmt);
if (exit_edge->flags & EDGE_TRUE_VALUE)
stmt = gimple_build_call (builtin_decl_implicit (BUILT_IN_UNREACHABLE), 0);
latch_edge = make_edge (latch, create_basic_block (NULL, NULL, latch), flags);
latch_edge->probability = profile_probability::never ();
- latch_edge->count = profile_count::zero ();
latch_edge->flags |= flags;
latch_edge->goto_locus = locus;
/* Set the probability of new exit to the same of the old one. Fix
the frequency of the latch block, by scaling it back by
1 - exit->probability. */
- new_exit->count = exit->count;
new_exit->probability = exit->probability;
new_nonexit = single_pred_edge (loop->latch);
new_nonexit->probability = exit->probability.invert ();
new_nonexit->flags = EDGE_TRUE_VALUE;
- new_nonexit->count -= exit->count;
if (new_nonexit->probability.initialized_p ())
scale_bbs_frequencies (&loop->latch, 1, new_nonexit->probability);
exit edge. */
freq_h = loop->header->count;
- freq_e = (loop_preheader_edge (loop))->count;
+ freq_e = (loop_preheader_edge (loop))->count ();
/* Use frequency only if counts are zero. */
if (!(freq_h > 0) && !(freq_e > 0))
{
exit_bb = single_pred (loop->latch);
new_exit = find_edge (exit_bb, rest);
- new_exit->count = loop_preheader_edge (loop)->count;
new_exit->probability = profile_probability::always ()
.apply_scale (1, new_est_niter + 1);
- rest->count += new_exit->count;
+ rest->count += new_exit->count ();
rest->frequency += EDGE_FREQUENCY (new_exit);
new_nonexit = single_pred_edge (loop->latch);
prob = new_nonexit->probability;
new_nonexit->probability = new_exit->probability.invert ();
- new_nonexit->count = exit_bb->count - new_exit->count;
prob = new_nonexit->probability / prob;
if (prob.initialized_p ())
scale_bbs_frequencies (&loop->latch, 1, prob);
new_e->flags |= EDGE_TRUE_VALUE;
}
- new_e->count = skip_bb->count;
new_e->probability = profile_probability::likely ();
- new_e->count = skip_e->count.apply_probability (PROB_LIKELY);
- skip_e->count -= new_e->count;
- skip_e->probability = profile_probability::unlikely ();
+ skip_e->probability = new_e->probability.invert ();
return new_e;
}
initialize_original_copy_tables ();
basic_block cond_bb;
- /* FIXME: probabilities seems wrong here. */
struct loop *loop2 = loop_version (loop1, cond, &cond_bb,
profile_probability::always (),
profile_probability::always (),
same average number of iterations regardless outcome of guard. */
new_edge->probability = guard->probability;
profile_count skip_count = guard->src->count > 0
- ? guard->count.apply_scale (pre_header->count,
+ ? guard->count ().apply_scale (pre_header->count,
guard->src->count)
- : guard->count.apply_probability (new_edge->probability);
+ : guard->count ().apply_probability (new_edge->probability);
- if (skip_count > e->count)
+ if (skip_count > e->count ())
{
fprintf (dump_file, " Capping count; expect profile inconsistency\n");
- skip_count = e->count;
+ skip_count = e->count ();
}
- new_edge->count = skip_count;
if (dump_file && (dump_flags & TDF_DETAILS))
{
fprintf (dump_file, " Estimated probability of skipping loop is ");
First decrease count of path from newly hoisted loop guard
to loop header... */
- e->count -= skip_count;
e->probability = new_edge->probability.invert ();
- e->dest->count = e->count;
+ e->dest->count = e->count ();
e->dest->frequency = EDGE_FREQUENCY (e);
/* ... now update profile to represent that original guard will be optimized
away ... */
guard->probability = profile_probability::never ();
- guard->count = profile_count::zero ();
not_guard->probability = profile_probability::always ();
- /* This count is wrong (frequency of not_guard does not change),
- but will be scaled later. */
- not_guard->count = guard->src->count;
/* ... finally scale everything in the loop except for guarded basic blocks
where profile does not change. */
te->probability += e->probability;
- te->count += e->count;
remove_edge (e);
cfg_altered = true;
}
EDGE_SUCC (cond_block, 0)->flags |= EDGE_FALLTHRU;
EDGE_SUCC (cond_block, 0)->flags &= ~(EDGE_TRUE_VALUE | EDGE_FALSE_VALUE);
EDGE_SUCC (cond_block, 0)->probability = profile_probability::always ();
- EDGE_SUCC (cond_block, 0)->count += EDGE_SUCC (cond_block, 1)->count;
block_to_remove = EDGE_SUCC (cond_block, 1)->dest;
}
EDGE_SUCC (cond_block, 1)->flags
&= ~(EDGE_TRUE_VALUE | EDGE_FALSE_VALUE);
EDGE_SUCC (cond_block, 1)->probability = profile_probability::always ();
- EDGE_SUCC (cond_block, 1)->count += EDGE_SUCC (cond_block, 0)->count;
block_to_remove = EDGE_SUCC (cond_block, 0)->dest;
}
edge etrue = make_edge (cond_bb, merge_bb, EDGE_TRUE_VALUE);
etrue->probability = profile_probability::even ();
- etrue->count = cond_bb->count.apply_scale (1, 2);
edge efalse = find_edge (cond_bb, then_bb);
efalse->flags = EDGE_FALSE_VALUE;
efalse->probability -= etrue->probability;
- efalse->count -= etrue->count;
- then_bb->count -= etrue->count;
+ then_bb->count -= etrue->count ();
tree othervar = NULL_TREE;
if (gimple_assign_rhs1 (use_stmt) == var)
making the bb count inconsistent with the edge weights. */
FOR_EACH_EDGE (e1, ei, bb1->succs)
{
- if (e1->count.initialized_p ())
- out_sum += e1->count;
+ if (e1->count ().initialized_p ())
+ out_sum += e1->count ();
out_freq_sum += EDGE_FREQUENCY (e1);
}
FOR_EACH_EDGE (e1, ei, bb2->succs)
{
- if (e1->count.initialized_p ())
- out_sum += e1->count;
+ if (e1->count ().initialized_p ())
+ out_sum += e1->count ();
out_freq_sum += EDGE_FREQUENCY (e1);
}
{
e2 = find_edge (bb2, e1->dest);
gcc_assert (e2);
- e2->count += e1->count;
- if (out_sum > 0 && e2->count.initialized_p ())
+ if (out_sum > 0 && e2->count ().initialized_p ())
{
- e2->probability = e2->count.probability_in (bb2->count);
+ e2->probability = e2->count ().probability_in (bb2->count);
}
else if (bb1->frequency && bb2->frequency)
e2->probability = e1->probability;
(GCOV_COMPUTE_SCALE (EDGE_FREQUENCY (e1)
+ EDGE_FREQUENCY (e2),
out_freq_sum));
- out_sum += e2->count;
+ out_sum += e2->count ();
}
bb2->frequency += bb1->frequency;
if (bb2->frequency > BB_FREQ_MAX)
else
{
e->probability = profile_probability::always ();
- e->count = bb->count;
ei_next (&ei);
}
}
same last path edge in the case where the last edge has a nocopy
source block. */
gcc_assert (ein_path->last ()->e == elast);
- path_in_count += ein->count;
+ path_in_count += ein->count ();
path_in_freq += EDGE_FREQUENCY (ein);
}
else if (!ein_path)
/* Keep track of the incoming edges that are not on any jump-threading
path. These counts will still flow out of original path after all
jump threading is complete. */
- nonpath_count += ein->count;
+ nonpath_count += ein->count ();
}
}
for (unsigned int i = 1; i < path->length (); i++)
{
edge epath = (*path)[i]->e;
- profile_count cur_count = epath->count;
+ profile_count cur_count = epath->count ();
if ((*path)[i]->type == EDGE_COPY_SRC_JOINER_BLOCK)
{
has_joiner = true;
they are redirected by an invocation of this routine. */
&& !bitmap_bit_p (local_info->duplicate_blocks,
ein->src->index))
- nonpath_count += ein->count;
+ nonpath_count += ein->count ();
}
}
if (cur_count < path_out_count)
path_out_count = cur_count;
- if (epath->count < min_path_count)
- min_path_count = epath->count;
+ if (epath->count () < min_path_count)
+ min_path_count = epath->count ();
}
/* We computed path_out_count above assuming that this path targeted
(since any path through the joiner with a different elast will not
include a copy of this elast in its duplicated path).
So ensure that this path's path_out_count is at least the
- difference between elast->count and nonpath_count. Otherwise the edge
+ difference between elast->count () and nonpath_count. Otherwise the edge
counts after threading will not be sane. */
if (local_info->need_profile_correction
- && has_joiner && path_out_count < elast->count - nonpath_count)
+ && has_joiner && path_out_count < elast->count () - nonpath_count)
{
- path_out_count = elast->count - nonpath_count;
+ path_out_count = elast->count () - nonpath_count;
/* But neither can we go above the minimum count along the path
we are duplicating. This can be an issue due to profile
insanities coming in to this pass. */
update_profile (edge epath, edge edup, profile_count path_in_count,
profile_count path_out_count, int path_in_freq)
{
+ if (!(path_in_count > 0))
+ return;
/* First update the duplicated block's count / frequency. */
if (edup)
{
basic_block dup_block = edup->src;
- gcc_assert (!dup_block->count.initialized_p ());
+
+ /* Edup's count is reduced by path_out_count. We need to redistribute
+ probabilities to the remaining edges. */
+
+ edge esucc;
+ edge_iterator ei;
+ profile_probability edup_prob
+ = path_out_count.probability_in (path_in_count);
+
+ /* Either scale up or down the remaining edges.
+ probabilities are always in range <0,1> and thus we can't do
+ both by same loop. */
+ if (edup->probability > edup_prob)
+ {
+ profile_probability rev_scale
+ = (profile_probability::always () - edup->probability)
+ / (profile_probability::always () - edup_prob);
+ FOR_EACH_EDGE (esucc, ei, dup_block->succs)
+ if (esucc != edup)
+ esucc->probability /= rev_scale;
+ }
+ else if (edup->probability < edup_prob)
+ {
+ profile_probability scale
+ = (profile_probability::always () - edup_prob)
+ / (profile_probability::always () - edup->probability);
+ FOR_EACH_EDGE (esucc, ei, dup_block->succs)
+ if (esucc != edup)
+ esucc->probability *= scale;
+ }
+ edup->probability = edup_prob;
+
+ /* FIXME once freqs_to_counts is dropped re-enable this check. */
+ gcc_assert (!dup_block->count.initialized_p () || 1);
gcc_assert (dup_block->frequency == 0);
dup_block->count = path_in_count;
dup_block->frequency = path_in_freq;
}
+ profile_count final_count = epath->count () - path_out_count;
+
/* Now update the original block's count and frequency in the
opposite manner - remove the counts/freq that will flow
into the duplicated block. Handle underflow due to precision/
out of it (in the joiner case this is the count along the duplicated path
out of the duplicated joiner). This count can then be removed from the
original path edge. */
- if (edup)
- edup->count = path_out_count;
- epath->count -= path_out_count;
- /* FIXME: can epath->count be legally uninitialized here? */
-}
-
-
-/* The duplicate and original joiner blocks may end up with different
- probabilities (different from both the original and from each other).
- Recompute the probabilities here once we have updated the edge
- counts and frequencies. */
-
-static void
-recompute_probabilities (basic_block bb)
-{
- edge esucc;
- edge_iterator ei;
- FOR_EACH_EDGE (esucc, ei, bb->succs)
- {
- if (!(bb->count > 0))
- continue;
-
- /* Prevent overflow computation due to insane profiles. */
- if (esucc->count < bb->count)
- esucc->probability = esucc->count.probability_in (bb->count).guessed ();
- else
- /* Can happen with missing/guessed probabilities, since we
- may determine that more is flowing along duplicated
- path than joiner succ probabilities allowed.
- Counts and freqs will be insane after jump threading,
- at least make sure probability is sane or we will
- get a flow verification error.
- Not much we can do to make counts/freqs sane without
- redoing the profile estimation. */
- esucc->probability = profile_probability::guessed_always ();
- }
-}
-
-
-/* Update the counts of the original and duplicated edges from a joiner
- that go off path, given that we have already determined that the
- duplicate joiner DUP_BB has incoming count PATH_IN_COUNT and
- outgoing count along the path PATH_OUT_COUNT. The original (on-)path
- edge from joiner is EPATH. */
-
-static void
-update_joiner_offpath_counts (edge epath, basic_block dup_bb,
- profile_count path_in_count,
- profile_count path_out_count)
-{
- /* Compute the count that currently flows off path from the joiner.
- In other words, the total count of joiner's out edges other than
- epath. Compute this by walking the successors instead of
- subtracting epath's count from the joiner bb count, since there
- are sometimes slight insanities where the total out edge count is
- larger than the bb count (possibly due to rounding/truncation
- errors). */
- profile_count total_orig_off_path_count = profile_count::zero ();
- edge enonpath;
- edge_iterator ei;
- FOR_EACH_EDGE (enonpath, ei, epath->src->succs)
+ if (epath->src->count > 0)
{
- if (enonpath == epath)
- continue;
- total_orig_off_path_count += enonpath->count;
- }
-
- /* For the path that we are duplicating, the amount that will flow
- off path from the duplicated joiner is the delta between the
- path's cumulative in count and the portion of that count we
- estimated above as flowing from the joiner along the duplicated
- path. */
- profile_count total_dup_off_path_count = path_in_count - path_out_count;
-
- /* Now do the actual updates of the off-path edges. */
- FOR_EACH_EDGE (enonpath, ei, epath->src->succs)
- {
- /* Look for edges going off of the threading path. */
- if (enonpath == epath)
- continue;
+ edge esucc;
+ edge_iterator ei;
+ profile_probability epath_prob = final_count.probability_in (epath->src->count);
- /* Find the corresponding edge out of the duplicated joiner. */
- edge enonpathdup = find_edge (dup_bb, enonpath->dest);
- gcc_assert (enonpathdup);
-
- /* We can't use the original probability of the joiner's out
- edges, since the probabilities of the original branch
- and the duplicated branches may vary after all threading is
- complete. But apportion the duplicated joiner's off-path
- total edge count computed earlier (total_dup_off_path_count)
- among the duplicated off-path edges based on their original
- ratio to the full off-path count (total_orig_off_path_count).
- */
- profile_probability scale
- = enonpath->count.probability_in (total_orig_off_path_count);
- /* Give the duplicated offpath edge a portion of the duplicated
- total. */
- enonpathdup->count = total_dup_off_path_count.apply_probability (scale);
- /* Now update the original offpath edge count, handling underflow
- due to rounding errors. */
- enonpath->count -= enonpathdup->count;
+ if (epath->probability > epath_prob)
+ {
+ profile_probability rev_scale
+ = (profile_probability::always () - epath->probability)
+ / (profile_probability::always () - epath_prob);
+ FOR_EACH_EDGE (esucc, ei, epath->src->succs)
+ if (esucc != epath)
+ esucc->probability /= rev_scale;
+ }
+ else if (epath->probability < epath_prob)
+ {
+ profile_probability scale
+ = (profile_probability::always () - epath_prob)
+ / (profile_probability::always () - epath->probability);
+ FOR_EACH_EDGE (esucc, ei, epath->src->succs)
+ if (esucc != epath)
+ esucc->probability *= scale;
+ }
+ epath->probability = epath_prob;
}
}
bool non_zero_freq = false;
FOR_EACH_EDGE (ein, ei, e->dest->preds)
{
- if (ein->count > 0)
+ if (ein->count () > 0)
return false;
non_zero_freq |= ein->src->frequency != 0;
}
edge esucc;
FOR_EACH_EDGE (esucc, ei, epath->src->succs)
{
- if (esucc->count > 0)
+ if (esucc->count () > 0)
return false;
non_zero_freq |= esucc->src->frequency != 0;
}
vec<jump_thread_edge *> *path = THREAD_PATH (e);
edge ein;
edge_iterator ei;
- FOR_EACH_EDGE (ein, ei, e->dest->preds)
- {
- /* Scale up the frequency by REG_BR_PROB_BASE, to avoid rounding
- errors applying the probability when the frequencies are very
- small. */
- if (ein->probability.initialized_p ())
- ein->count = profile_count::from_gcov_type
- (apply_probability (ein->src->frequency * REG_BR_PROB_BASE,
- ein->probability
- .to_reg_br_prob_base ())).guessed ();
- else
- /* FIXME: this is hack; we should track uninitialized values. */
- ein->count = profile_count::zero ();
- }
+ FOR_EACH_EDGE (ein, ei, e->dest->preds)
+ ein->src->count = profile_count::from_gcov_type
+ (ein->src->frequency * REG_BR_PROB_BASE);
for (unsigned int i = 1; i < path->length (); i++)
{
edge epath = (*path)[i]->e;
- edge esucc;
/* Scale up the frequency by REG_BR_PROB_BASE, to avoid rounding
errors applying the edge probability when the frequencies are very
small. */
epath->src->count =
profile_count::from_gcov_type
(epath->src->frequency * REG_BR_PROB_BASE);
- FOR_EACH_EDGE (esucc, ei, epath->src->succs)
- esucc->count =
- esucc->src->count.apply_probability (esucc->probability);
}
}
{
edge e = rd->incoming_edges->e;
vec<jump_thread_edge *> *path = THREAD_PATH (e);
- edge ein, esucc;
- edge_iterator ei;
profile_count val = profile_count::uninitialized ();
if (profile_status_for_fn (cfun) == PROFILE_READ)
val = profile_count::zero ();
+ edge ein;
+ edge_iterator ei;
+
FOR_EACH_EDGE (ein, ei, e->dest->preds)
- ein->count = val;
+ ein->src->count = val;
/* First clear counts along original path. */
for (unsigned int i = 1; i < path->length (); i++)
{
edge epath = (*path)[i]->e;
- FOR_EACH_EDGE (esucc, ei, epath->src->succs)
- esucc->count = val;
epath->src->count = val;
}
/* Also need to clear the counts along duplicated path. */
basic_block dup = rd->dup_blocks[i];
if (!dup)
continue;
- FOR_EACH_EDGE (esucc, ei, dup->succs)
- esucc->count = val;
dup->count = val;
}
}
update_profile (epath, e2, path_in_count, path_out_count,
path_in_freq);
- /* Next we need to update the counts of the original and duplicated
- edges from the joiner that go off path. */
- update_joiner_offpath_counts (epath, e2->src, path_in_count,
- path_out_count);
-
- /* Finally, we need to set the probabilities on the duplicated
- edges out of the duplicated joiner (e2->src). The probabilities
- along the original path will all be updated below after we finish
- processing the whole path. */
- recompute_probabilities (e2->src);
-
/* Record the frequency flowing to the downstream duplicated
path blocks. */
cur_path_freq = EDGE_FREQUENCY (e2);
been updated at the end of that handling to the edge frequency
along the duplicated joiner path edge. */
update_profile (epath, EDGE_SUCC (rd->dup_blocks[count], 0),
- path_out_count, path_out_count,
- cur_path_freq);
+ path_out_count, path_out_count, cur_path_freq);
}
else
{
}
}
- /* Now walk orig blocks and update their probabilities, since the
- counts and freqs should be updated properly by above loop. */
- for (unsigned int i = 1; i < path->length (); i++)
- {
- edge epath = (*path)[i]->e;
- recompute_probabilities (epath->src);
- }
-
/* Done with all profile and frequency updates, clear counts if they
were copied. */
if (do_freqs_to_counts)
invalidating the property that is propagated by executing all the blocks of
the jump-thread path in order. */
- curr_count = entry->count;
+ curr_count = entry->count ();
curr_freq = EDGE_FREQUENCY (entry);
for (i = 0; i < n_region; i++)
if (i + 1 != n_region)
{
curr_freq = EDGE_FREQUENCY (single_succ_edge (bb));
- curr_count = single_succ_edge (bb)->count;
+ curr_count = single_succ_edge (bb)->count ();
}
continue;
}
else
{
curr_freq = EDGE_FREQUENCY (e);
- curr_count = e->count;
+ curr_count = e->count ();
}
}
{
rescan_loop_exit (e, true, false);
e->probability = profile_probability::always ();
- e->count = region_copy[n_region - 1]->count;
}
/* Redirect the entry and add the phi node arguments. */
e_false->flags &= ~EDGE_FALLTHRU;
e_false->flags |= EDGE_FALSE_VALUE;
e_false->probability = e_true->probability.invert ();
- e_false->count = split_bb->count - e_true->count;
- new_bb->count = e_false->count;
+ new_bb->count = e_false->count ();
if (update_dominators)
{
const struct case_bit_test *const d1 = (const struct case_bit_test *) p1;
const struct case_bit_test *const d2 = (const struct case_bit_test *) p2;
- if (d2->target_edge->count < d1->target_edge->count)
+ if (d2->target_edge->count () < d1->target_edge->count ())
return -1;
- if (d2->target_edge->count > d1->target_edge->count)
+ if (d2->target_edge->count () > d1->target_edge->count ())
return 1;
if (d2->bits != d1->bits)
return d2->bits - d1->bits;
= label_to_block (CASE_LABEL (gimple_switch_default_label (swtch)));
e_default = find_edge (info->switch_bb, info->default_bb);
info->default_prob = e_default->probability;
- info->default_count = e_default->count;
+ info->default_count = e_default->count ();
FOR_EACH_EDGE (e, ei, info->switch_bb->succs)
if (e != e_default)
- info->other_count += e->count;
+ info->other_count += e->count ();
/* Get upper and lower bounds of case values, and the covered range. */
min_case = gimple_switch_label (swtch, 1);
if (!info->default_case_nonstandard)
e01 = make_edge (bb0, bb1, EDGE_TRUE_VALUE);
e01->probability = info->default_prob.invert ();
- e01->count = info->other_count;
/* flags and profiles of the edge taking care of out-of-range values */
e02->flags &= ~EDGE_FALLTHRU;
e02->flags |= EDGE_FALSE_VALUE;
e02->probability = info->default_prob;
- e02->count = info->default_count;
bbf = info->final_bb;
e1f = make_edge (bb1, bbf, EDGE_FALLTHRU);
e1f->probability = profile_probability::always ();
- e1f->count = info->other_count;
if (info->default_case_nonstandard)
e2f = NULL;
{
e2f = make_edge (bb2, bbf, EDGE_FALLTHRU);
e2f->probability = profile_probability::always ();
- e2f->count = info->default_count;
}
/* frequencies of the new BBs */
edge false_edge = split_block (bb, cond);
false_edge->flags = EDGE_FALSE_VALUE;
false_edge->probability = prob.invert ();
- false_edge->count = bb->count.apply_probability (false_edge->probability);
edge true_edge = make_edge (bb, label_bb, EDGE_TRUE_VALUE);
fix_phi_operands_for_edge (true_edge, phi_mapping);
true_edge->probability = prob;
- true_edge->count = bb->count.apply_probability (true_edge->probability);
return false_edge->dest;
}
edge false_edge = split_block (bb, cond);
false_edge->flags = EDGE_FALSE_VALUE;
false_edge->probability = prob.invert ();
- false_edge->count = bb->count.apply_probability (false_edge->probability);
edge true_edge = make_edge (bb, label_bb, EDGE_TRUE_VALUE);
fix_phi_operands_for_edge (true_edge, phi_mapping);
true_edge->probability = prob;
- true_edge->count = bb->count.apply_probability (true_edge->probability);
return false_edge->dest;
}
static void
decrease_profile (basic_block bb, profile_count count, int frequency)
{
- edge e;
bb->count = bb->count - count;
bb->frequency -= frequency;
if (bb->frequency < 0)
gcc_assert (!EDGE_COUNT (bb->succs));
return;
}
- e = single_succ_edge (bb);
- e->count -= count;
}
/* Returns true if argument PARAM of the tail recursive call needs to be copied
/* Number of executions of function has reduced by the tailcall. */
e = single_succ_edge (gsi_bb (t->call_gsi));
- decrease_profile (EXIT_BLOCK_PTR_FOR_FN (cfun), e->count, EDGE_FREQUENCY (e));
- decrease_profile (ENTRY_BLOCK_PTR_FOR_FN (cfun), e->count,
+ decrease_profile (EXIT_BLOCK_PTR_FOR_FN (cfun), e->count (), EDGE_FREQUENCY (e));
+ decrease_profile (ENTRY_BLOCK_PTR_FOR_FN (cfun), e->count (),
EDGE_FREQUENCY (e));
if (e->dest != EXIT_BLOCK_PTR_FOR_FN (cfun))
- decrease_profile (e->dest, e->count, EDGE_FREQUENCY (e));
+ decrease_profile (e->dest, e->count (), EDGE_FREQUENCY (e));
/* Replace the call by a jump to the start of function. */
e = redirect_edge_and_branch (single_succ_edge (gsi_bb (t->call_gsi)),
/* Add new edge to connect guard block to the merge/loop-exit block. */
new_e = make_edge (guard_bb, guard_to, EDGE_TRUE_VALUE);
- new_e->count = guard_bb->count;
new_e->probability = probability;
- new_e->count = enter_e->count.apply_probability (probability);
if (irreducible_p)
new_e->flags |= EDGE_IRREDUCIBLE_LOOP;
- enter_e->count -= new_e->count;
enter_e->probability = probability.invert ();
set_immediate_dominator (CDI_DOMINATORS, guard_to, dom_bb);
a merge point of control flow. */
guard_to->frequency = guard_bb->frequency;
guard_to->count = guard_bb->count;
- single_succ_edge (guard_to)->count = guard_to->count;
/* Scale probability of epilog loop back.
FIXME: We should avoid scaling down and back up. Profile may
get lost if we scale down to 0. */
edge preheader = loop_preheader_edge (loop);
/* Reduce loop iterations by the vectorization factor. */
gcov_type new_est_niter = niter_for_unrolled_loop (loop, vf);
- profile_count freq_h = loop->header->count, freq_e = preheader->count;
+ profile_count freq_h = loop->header->count, freq_e = preheader->count ();
/* Use frequency only if counts are zero. */
if (!(freq_h > 0) && !(freq_e > 0))
scale_loop_frequencies (loop, p);
}
- basic_block exit_bb = single_pred (loop->latch);
edge exit_e = single_exit (loop);
- exit_e->count = loop_preheader_edge (loop)->count;
exit_e->probability = profile_probability::always ()
.apply_scale (1, new_est_niter + 1);
edge exit_l = single_pred_edge (loop->latch);
profile_probability prob = exit_l->probability;
exit_l->probability = exit_e->probability.invert ();
- exit_l->count = exit_bb->count - exit_e->count;
if (prob.initialized_p () && exit_l->probability.initialized_p ())
scale_bbs_frequencies (&loop->latch, 1, exit_l->probability / prob);
}
/* Set up the fallthrough basic block. */
e = find_edge (cond_bb, fallthru_bb);
e->flags = EDGE_FALSE_VALUE;
- e->count = cond_bb->count;
e->probability = profile_probability::very_likely ();
/* Update dominance info for the newly created then_bb; note that
/* Set up the fallthrough basic block. */
e = find_edge (cond1_bb, cond2_bb);
e->flags = EDGE_FALSE_VALUE;
- e->count = cond1_bb->count;
e->probability = profile_probability::very_likely ();
/* Update dominance info. */
e->flags = EDGE_FALSE_VALUE;
if (pos_neg != 3)
{
- e->count = cond_bb->count;
e->probability = profile_probability::very_likely ();
/* Connect 'then block' with the 'else block'. This is needed
}
else
{
- profile_count count = cond_bb->count.apply_probability (PROB_EVEN);
- e->count = count;
e->probability = profile_probability::even ();
e = split_block (fallthru_bb, (gimple *) NULL);
cond_neg_bb = e->src;
fallthru_bb = e->dest;
- e->count = count;
e->probability = profile_probability::very_likely ();
e->flags = EDGE_FALSE_VALUE;
add_bb_to_loop (cond_pos_bb, cond_bb->loop_father);
e = make_edge (cond_bb, cond_pos_bb, EDGE_TRUE_VALUE);
- e->count = count;
e->probability = profile_probability::even ();
e = make_edge (cond_pos_bb, then_bb, EDGE_TRUE_VALUE);
e->probability = profile_probability::very_unlikely ();
e = make_edge (cond_pos_bb, fallthru_bb, EDGE_FALSE_VALUE);
- e->count = count;
e->probability = profile_probability::very_likely ();
make_single_succ_edge (then_bb, fallthru_bb, EDGE_FALLTHRU);
e12->flags &= ~EDGE_FALLTHRU;
e12->flags |= EDGE_FALSE_VALUE;
e12->probability = prob;
- e12->count = profile_count::from_gcov_type (count);
e13 = make_edge (bb, bb3, EDGE_TRUE_VALUE);
e13->probability = prob.invert ();
- e13->count = profile_count::from_gcov_type (all - count);
remove_edge (e23);
e24 = make_edge (bb2, bb4, EDGE_FALLTHRU);
e24->probability = profile_probability::always ();
- e24->count = profile_count::from_gcov_type (count);
e34->probability = profile_probability::always ();
- e34->count = profile_count::from_gcov_type (all - count);
return tmp2;
}
e12->flags &= ~EDGE_FALLTHRU;
e12->flags |= EDGE_FALSE_VALUE;
e12->probability = prob;
- e12->count = profile_count::from_gcov_type (count);
e13 = make_edge (bb, bb3, EDGE_TRUE_VALUE);
e13->probability = prob.invert ();
- e13->count = profile_count::from_gcov_type (all - count);
remove_edge (e23);
e24 = make_edge (bb2, bb4, EDGE_FALLTHRU);
e24->probability = profile_probability::always ();
- e24->count = profile_count::from_gcov_type (count);
e34->probability = profile_probability::always ();
- e34->count = profile_count::from_gcov_type (all - count);
return result;
}
e12->flags &= ~EDGE_FALLTHRU;
e12->flags |= EDGE_FALSE_VALUE;
e12->probability = prob1.invert ();
- e12->count = profile_count::from_gcov_type (all - count1);
e14 = make_edge (bb, bb4, EDGE_TRUE_VALUE);
e14->probability = prob1;
- e14->count = profile_count::from_gcov_type (count1);
if (ncounts) /* Assumed to be 0 or 1. */
{
e23->flags &= ~EDGE_FALLTHRU;
e23->flags |= EDGE_FALSE_VALUE;
- e23->count = profile_count::from_gcov_type (all - count1 - count2);
e23->probability = prob2.invert ();
e24 = make_edge (bb2, bb4, EDGE_TRUE_VALUE);
e24->probability = prob2;
- e24->count = profile_count::from_gcov_type (count2);
}
e34->probability = profile_probability::always ();
- e34->count = profile_count::from_gcov_type (all - count1 - count2);
return result;
}
if (e_ij != NULL)
{
e_ij->probability = profile_probability::always ();
- e_ij->count = all - count;
e_ij = single_pred_edge (split_edge (e_ij));
}
}
e_cd->flags = (e_cd->flags & ~EDGE_FALLTHRU) | EDGE_TRUE_VALUE;
e_cd->probability = prob;
- e_cd->count = count;
e_ci = make_edge (cond_bb, icall_bb, EDGE_FALSE_VALUE);
e_ci->probability = prob.invert ();
- e_ci->count = all - count;
remove_edge (e_di);
if (e_ij != NULL)
{
- if ((dflags & ECF_NORETURN) != 0)
- e_ij->count = all;
- else
+ if ((dflags & ECF_NORETURN) == 0)
{
e_dj = make_edge (dcall_bb, join_bb, EDGE_FALLTHRU);
e_dj->probability = profile_probability::always ();
- e_dj->count = count;
-
- e_ij->count = all - count;
}
e_ij->probability = profile_probability::always ();
}
{
e = make_edge (dcall_bb, e_eh->dest, e_eh->flags);
e->probability = e_eh->probability;
- e->count = e_eh->count;
for (gphi_iterator psi = gsi_start_phis (e_eh->dest);
!gsi_end_p (psi); gsi_next (&psi))
{
e_ci->flags = (e_ci->flags & ~EDGE_FALLTHRU) | EDGE_TRUE_VALUE;
e_ci->probability = prob;
- e_ci->count = profile_count::from_gcov_type (count);
e_cv = make_edge (cond_bb, vcall_bb, EDGE_FALSE_VALUE);
e_cv->probability = prob.invert ();
- e_cv->count = profile_count::from_gcov_type (all - count);
remove_edge (e_iv);
e_ij = make_edge (icall_bb, join_bb, EDGE_FALLTHRU);
e_ij->probability = profile_probability::always ();
- e_ij->count = profile_count::from_gcov_type (count);
e_vj->probability = profile_probability::always ();
- e_vj->count = profile_count::from_gcov_type (all - count);
/* Insert PHI node for the call result if necessary. */
if (gimple_call_lhs (vcall_stmt)
projects / thirdparty / gcc.git / commitdiff
