/* Instruction scheduling pass.
- Copyright (C) 1992-2017 Free Software Foundation, Inc.
+ Copyright (C) 1992-2021 Free Software Foundation, Inc.
Contributed by Michael Tiemann (tiemann@cygnus.com) Enhanced by,
and currently maintained by, Jim Wilson (wilson@cygnus.com)
#include "cfgbuild.h"
#include "sched-int.h"
#include "common/common-target.h"
-#include "params.h"
#include "dbgcnt.h"
#include "cfgloop.h"
#include "dumpfile.h"
#include "print-rtl.h"
+#include "function-abi.h"
#ifdef INSN_SCHEDULING
modulo_max_stages = max_stages;
modulo_n_insns = insns;
modulo_iter0_max_uid = max_uid;
- modulo_backtracks_left = PARAM_VALUE (PARAM_MAX_MODULO_BACKTRACK_ATTEMPTS);
+ modulo_backtracks_left = param_max_modulo_backtrack_attempts;
}
/* A structure to record a pair of insns where the first one is a real
\f
/* Forward declarations. */
-static int priority (rtx_insn *);
+static int priority (rtx_insn *, bool force_recompute = false);
static int autopref_rank_for_schedule (const rtx_insn *, const rtx_insn *);
static int rank_for_schedule (const void *, const void *);
static void swap_sort (rtx_insn **, int);
static void move_succs (vec<edge, va_gc> **, basic_block);
static void sched_remove_insn (rtx_insn *);
static void clear_priorities (rtx_insn *, rtx_vec_t *);
-static void calc_priorities (rtx_vec_t);
+static void calc_priorities (const rtx_vec_t &);
static void add_jump_dependencies (rtx_insn *, rtx_insn *);
#endif /* INSN_SCHEDULING */
/* Effective number of available registers of a given class (see comment
in sched_pressure_start_bb). */
static int sched_class_regs_num[N_REG_CLASSES];
-/* Number of call_saved_regs and fixed_regs. Helpers for calculating of
+/* The number of registers that the function would need to save before it
+ uses them, and the number of fixed_regs. Helpers for calculating of
sched_class_regs_num. */
static int call_saved_regs_num[N_REG_CLASSES];
static int fixed_regs_num[N_REG_CLASSES];
/* Compute the priority number for INSN. */
static int
-priority (rtx_insn *insn)
+priority (rtx_insn *insn, bool force_recompute)
{
if (! INSN_P (insn))
return 0;
/* We should not be interested in priority of an already scheduled insn. */
gcc_assert (QUEUE_INDEX (insn) != QUEUE_SCHEDULED);
- if (!INSN_PRIORITY_KNOWN (insn))
+ if (force_recompute || !INSN_PRIORITY_KNOWN (insn))
{
int this_priority = -1;
enum rfs_decision {
RFS_LIVE_RANGE_SHRINK1, RFS_LIVE_RANGE_SHRINK2,
RFS_SCHED_GROUP, RFS_PRESSURE_DELAY, RFS_PRESSURE_TICK,
- RFS_FEEDS_BACKTRACK_INSN, RFS_PRIORITY, RFS_SPECULATION,
+ RFS_FEEDS_BACKTRACK_INSN, RFS_PRIORITY, RFS_AUTOPREF, RFS_SPECULATION,
RFS_SCHED_RANK, RFS_LAST_INSN, RFS_PRESSURE_INDEX,
- RFS_DEP_COUNT, RFS_TIE, RFS_FUSION, RFS_N };
+ RFS_DEP_COUNT, RFS_TIE, RFS_FUSION, RFS_COST, RFS_N };
/* Corresponding strings for print outs. */
static const char *rfs_str[RFS_N] = {
"RFS_LIVE_RANGE_SHRINK1", "RFS_LIVE_RANGE_SHRINK2",
"RFS_SCHED_GROUP", "RFS_PRESSURE_DELAY", "RFS_PRESSURE_TICK",
- "RFS_FEEDS_BACKTRACK_INSN", "RFS_PRIORITY", "RFS_SPECULATION",
+ "RFS_FEEDS_BACKTRACK_INSN", "RFS_PRIORITY", "RFS_AUTOPREF", "RFS_SPECULATION",
"RFS_SCHED_RANK", "RFS_LAST_INSN", "RFS_PRESSURE_INDEX",
- "RFS_DEP_COUNT", "RFS_TIE", "RFS_FUSION" };
+ "RFS_DEP_COUNT", "RFS_TIE", "RFS_FUSION", "RFS_COST" };
/* Statistical breakdown of rank_for_schedule decisions. */
struct rank_for_schedule_stats_t { unsigned stats[RFS_N]; };
if (flag_sched_critical_path_heuristic && priority_val)
return rfs_result (RFS_PRIORITY, priority_val, tmp, tmp2);
- if (PARAM_VALUE (PARAM_SCHED_AUTOPREF_QUEUE_DEPTH) >= 0)
+ if (param_sched_autopref_queue_depth >= 0)
{
int autopref = autopref_rank_for_schedule (tmp, tmp2);
if (autopref != 0)
- return autopref;
+ return rfs_result (RFS_AUTOPREF, autopref, tmp, tmp2);
}
/* Prefer speculative insn with greater dependencies weakness. */
}
/* Prefer instructions that occur earlier in the model schedule. */
- if (sched_pressure == SCHED_PRESSURE_MODEL
- && INSN_BB (tmp) == target_bb && INSN_BB (tmp2) == target_bb)
+ if (sched_pressure == SCHED_PRESSURE_MODEL)
{
diff = model_index (tmp) - model_index (tmp2);
- gcc_assert (diff != 0);
- return rfs_result (RFS_PRESSURE_INDEX, diff, tmp, tmp2);
+ if (diff != 0)
+ return rfs_result (RFS_PRESSURE_INDEX, diff, tmp, tmp2);
}
/* Prefer the insn which has more later insns that depend on it.
if (flag_sched_dep_count_heuristic && val != 0)
return rfs_result (RFS_DEP_COUNT, val, tmp, tmp2);
+ /* Sort by INSN_COST rather than INSN_LUID. This means that instructions
+ which take longer to execute are prioritised and it leads to more
+ dual-issue opportunities on in-order cores which have this feature. */
+
+ if (INSN_COST (tmp) != INSN_COST (tmp2))
+ return rfs_result (RFS_COST, INSN_COST (tmp2) - INSN_COST (tmp),
+ tmp, tmp2);
+
/* If insns are equally good, sort by INSN_LUID (original insn order),
so that we make the sort stable. This minimizes instruction movement,
thus minimizing sched's effect on debugging and cross-jumping. */
HAIFA_INLINE static void
advance_one_cycle (void)
{
+ int i;
+
advance_state (curr_state);
- if (sched_verbose >= 4)
- fprintf (sched_dump, ";;\tAdvance the current state.\n");
+ for (i = 4; i <= sched_verbose; ++i)
+ fprintf (sched_dump, ";;\tAdvance the current state: %d.\n", clock_var);
}
/* Update register pressure after scheduling INSN. */
}
/* Add INSN to the model worklist. Start looking for a suitable position
- between neighbors PREV and NEXT, testing at most MAX_SCHED_READY_INSNS
+ between neighbors PREV and NEXT, testing at most param_max_sched_ready_insns
insns either side. A null PREV indicates the beginning of the list and
a null NEXT indicates the end. */
{
int count;
- count = MAX_SCHED_READY_INSNS;
+ count = param_max_sched_ready_insns;
if (count > 0 && prev && model_order_p (insn, prev))
do
{
int count;
prev = insn->prev;
- count = MAX_SCHED_READY_INSNS;
+ count = param_max_sched_ready_insns;
while (count > 0 && prev && model_order_p (insn, prev))
{
count--;
{
fprintf (sched_dump, ";;\t+--- worklist:\n");
insn = model_worklist;
- count = MAX_SCHED_READY_INSNS;
+ count = param_max_sched_ready_insns;
while (count > 0 && insn)
{
fprintf (sched_dump, ";;\t+--- %d [%d, %d, %d, %d]\n",
Failing that, just pick the highest-priority instruction in the
worklist. */
- count = MAX_SCHED_READY_INSNS;
+ count = param_max_sched_ready_insns;
insn = model_worklist;
fallback = 0;
for (;;)
- call_saved_regs_num[cl]). */
{
int i;
- int entry_freq = ENTRY_BLOCK_PTR_FOR_FN (cfun)->frequency;
- int bb_freq = bb->frequency;
+ int entry_freq = ENTRY_BLOCK_PTR_FOR_FN (cfun)->count.to_frequency (cfun);
+ int bb_freq = bb->count.to_frequency (cfun);
if (bb_freq == 0)
{
gcc_assert (sd_lists_empty_p (insn, SD_LIST_HARD_BACK));
/* Reset debug insns invalidated by moving this insn. */
- if (MAY_HAVE_DEBUG_INSNS && !DEBUG_INSN_P (insn))
+ if (MAY_HAVE_DEBUG_BIND_INSNS && !DEBUG_INSN_P (insn))
for (sd_it = sd_iterator_start (insn, SD_LIST_BACK);
sd_iterator_cond (&sd_it, &dep);)
{
continue;
}
- gcc_assert (DEBUG_INSN_P (dbg));
+ gcc_assert (DEBUG_BIND_INSN_P (dbg));
if (sched_verbose >= 6)
fprintf (sched_dump, ";;\t\tresetting: debug insn %d\n",
if (insn != tail)
{
remove_insn (insn);
+ /* If an insn was split just before the EPILOGUE_BEG note and
+ that split created new basic blocks, we could have a
+ BASIC_BLOCK note here. Safely advance over it in that case
+ and assert that we land on a real insn. */
+ if (NOTE_P (next)
+ && NOTE_KIND (next) == NOTE_INSN_BASIC_BLOCK
+ && next != next_tail)
+ next = NEXT_INSN (next);
+ gcc_assert (INSN_P (next));
add_reg_note (next, REG_SAVE_NOTE,
GEN_INT (NOTE_INSN_EPILOGUE_BEG));
break;
success = validate_change (desc->insn, desc->loc, desc->newval, 0);
gcc_assert (success);
+ rtx_insn *insn = DEP_PRO (dep);
+
+ /* Recompute priority since dependent priorities may have changed. */
+ priority (insn, true);
update_insn_after_change (desc->insn);
+
if ((TODO_SPEC (desc->insn) & (HARD_DEP | DEP_POSTPONED)) == 0)
fix_tick_ready (desc->insn);
success = validate_change (desc->insn, desc->loc, desc->orig, 0);
gcc_assert (success);
+
+ rtx_insn *insn = DEP_PRO (dep);
+
+ if (QUEUE_INDEX (insn) != QUEUE_SCHEDULED)
+ {
+ /* Recompute priority since dependent priorities may have changed. */
+ priority (insn, true);
+ }
+
update_insn_after_change (desc->insn);
+
if (backtrack_queue != NULL)
{
backtrack_queue->replacement_deps.safe_push (dep);
/* If the ready list is full, delay the insn for 1 cycle.
See the comment in schedule_block for the rationale. */
if (!reload_completed
- && (ready->n_ready - ready->n_debug > MAX_SCHED_READY_INSNS
+ && (ready->n_ready - ready->n_debug > param_max_sched_ready_insns
|| (sched_pressure == SCHED_PRESSURE_MODEL
- /* Limit pressure recalculations to MAX_SCHED_READY_INSNS
- instructions too. */
+ /* Limit pressure recalculations to
+ param_max_sched_ready_insns instructions too. */
&& model_index (insn) > (model_curr_point
- + MAX_SCHED_READY_INSNS)))
+ + param_max_sched_ready_insns)))
&& !(sched_pressure == SCHED_PRESSURE_MODEL
&& model_curr_point < model_num_insns
/* Always allow the next model instruction to issue. */
last = emit_note_before (note_type, last);
remove_note (insn, note);
+ df_insn_create_insn_record (last);
}
}
}
int irrel2 = data2->status == AUTOPREF_MULTIPASS_DATA_IRRELEVANT;
if (!irrel1 && !irrel2)
+ /* Sort memory references from lowest offset to the largest. */
r = data1->offset - data2->offset;
- else
+ else if (write)
+ /* Schedule "irrelevant" insns before memory stores to resolve
+ as many producer dependencies of stores as possible. */
r = irrel2 - irrel1;
+ else
+ /* Schedule "irrelevant" insns after memory reads to avoid breaking
+ memory read sequences. */
+ r = irrel1 - irrel2;
}
return r;
/* Exit early if the param forbids this or if we're not entering here through
normal haifa scheduling. This can happen if selective scheduling is
explicitly enabled. */
- if (!insn_queue || PARAM_VALUE (PARAM_SCHED_AUTOPREF_QUEUE_DEPTH) <= 0)
+ if (!insn_queue || param_sched_autopref_queue_depth <= 0)
return 0;
if (sched_verbose >= 2 && ready_index == 0)
}
}
- if (PARAM_VALUE (PARAM_SCHED_AUTOPREF_QUEUE_DEPTH) == 1)
+ if (param_sched_autopref_queue_depth == 1)
continue;
/* Everything from the current queue slot should have been moved to
the ready list. */
gcc_assert (insn_queue[NEXT_Q_AFTER (q_ptr, 0)] == NULL_RTX);
- int n_stalls = PARAM_VALUE (PARAM_SCHED_AUTOPREF_QUEUE_DEPTH) - 1;
+ int n_stalls = param_sched_autopref_queue_depth - 1;
if (n_stalls > max_insn_queue_index)
n_stalls = max_insn_queue_index;
{
int i, pass;
bool sched_group_found = false;
- int min_cost_group = 1;
+ int min_cost_group = 0;
if (sched_fusion)
return;
}
/* Make two passes if there's a SCHED_GROUP_P insn; make sure to handle
- such an insn first and note its cost, then schedule all other insns
- for one cycle later. */
+ such an insn first and note its cost. If at least one SCHED_GROUP_P insn
+ gets queued, then all other insns get queued for one cycle later. */
for (pass = sched_group_found ? 0 : 1; pass < 2; )
{
int n = ready.n_ready;
if (DEBUG_INSN_P (insn))
continue;
- if (sched_group_found && !SCHED_GROUP_P (insn))
+ if (sched_group_found && !SCHED_GROUP_P (insn)
+ && ((pass == 0) || (min_cost_group >= 1)))
{
if (pass == 0)
continue;
time in the worst case. Before reload we are more likely to have
big lists so truncate them to a reasonable size. */
if (!reload_completed
- && ready.n_ready - ready.n_debug > MAX_SCHED_READY_INSNS)
+ && ready.n_ready - ready.n_debug > param_max_sched_ready_insns)
{
ready_sort_debug (&ready);
ready_sort_real (&ready);
- /* Find first free-standing insn past MAX_SCHED_READY_INSNS.
+ /* Find first free-standing insn past param_max_sched_ready_insns.
If there are debug insns, we know they're first. */
- for (i = MAX_SCHED_READY_INSNS + ready.n_debug; i < ready.n_ready; i++)
+ for (i = param_max_sched_ready_insns + ready.n_debug; i < ready.n_ready;
+ i++)
if (!SCHED_GROUP_P (ready_element (&ready, i)))
break;
fixed_regs_num[cl] = 0;
for (int i = 0; i < ira_class_hard_regs_num[cl]; ++i)
- if (!call_used_regs[ira_class_hard_regs[cl][i]])
- ++call_saved_regs_num[cl];
- else if (fixed_regs[ira_class_hard_regs[cl][i]])
- ++fixed_regs_num[cl];
+ {
+ unsigned int regno = ira_class_hard_regs[cl][i];
+ if (fixed_regs[regno])
+ ++fixed_regs_num[cl];
+ else if (!crtl->abi->clobbers_full_reg_p (regno))
+ ++call_saved_regs_num[cl];
+ }
}
}
}
void
sched_init (void)
{
- /* Disable speculative loads in their presence if cc0 defined. */
- if (HAVE_cc0)
- flag_schedule_speculative_load = 0;
-
if (targetm.sched.dispatch (NULL, IS_DISPATCH_ON))
targetm.sched.dispatch_do (NULL, DISPATCH_INIT);
&& !reload_completed
&& common_sched_info->sched_pass_id == SCHED_RGN_PASS)
sched_pressure = ((enum sched_pressure_algorithm)
- PARAM_VALUE (PARAM_SCHED_PRESSURE_ALGORITHM));
+ param_sched_pressure_algorithm);
else
sched_pressure = SCHED_PRESSURE_NONE;
if (spec_info->mask != 0)
{
- spec_info->data_weakness_cutoff =
- (PARAM_VALUE (PARAM_SCHED_SPEC_PROB_CUTOFF) * MAX_DEP_WEAK) / 100;
- spec_info->control_weakness_cutoff =
- (PARAM_VALUE (PARAM_SCHED_SPEC_PROB_CUTOFF)
- * REG_BR_PROB_BASE) / 100;
+ spec_info->data_weakness_cutoff
+ = (param_sched_spec_prob_cutoff * MAX_DEP_WEAK) / 100;
+ spec_info->control_weakness_cutoff
+ = (param_sched_spec_prob_cutoff * REG_BR_PROB_BASE) / 100;
}
else
/* So we won't read anything accidentally. */
if (e)
{
- gcc_assert (e->dest == succ);
+ gcc_assert (e->dest == succ || e->dest->index == EXIT_BLOCK);
return e;
}
}
|| (!NOTE_P (insn)
&& !LABEL_P (insn)
/* Don't emit a NOTE if it would end up before a BARRIER. */
- && !BARRIER_P (NEXT_INSN (end))))
+ && !BARRIER_P (next_nondebug_insn (end))))
{
rtx_note *note = emit_note_after (NOTE_INSN_DELETED, end);
/* Make note appear outside BB. */
single->count = last->count;
empty->count = last->count;
- single->frequency = last->frequency;
- empty->frequency = last->frequency;
BB_COPY_PARTITION (single, last);
BB_COPY_PARTITION (empty, last);
'todo_spec' variable in create_check_block_twin and
in sel-sched.c `check_ds' in create_speculation_check. */
e->probability = profile_probability::very_unlikely ();
- e->count = first_bb->count.apply_probability (e->probability);
- rec->count = e->count;
- rec->frequency = EDGE_FREQUENCY (e);
+ rec->count = e->count ();
e2->probability = e->probability.invert ();
- e2->count = first_bb->count - e2->count;
rtx_code_label *label = block_label (second_bb);
rtx_jump_insn *jump = emit_jump_insn_after (targetm.gen_jump (label),
changed. ROOTS is a vector of instructions whose priority computation will
trigger initialization of all cleared priorities. */
static void
-calc_priorities (rtx_vec_t roots)
+calc_priorities (const rtx_vec_t &roots)
{
int i;
rtx_insn *insn;
{
int new_luids_max_uid = get_max_uid () + 1;
- sched_luids.safe_grow_cleared (new_luids_max_uid);
+ sched_luids.safe_grow_cleared (new_luids_max_uid, true);
}
/* Initialize LUID for INSN. */
The hook common_sched_info->luid_for_non_insn () is used to determine
if notes, labels, etc. need luids. */
void
-sched_init_luids (bb_vec_t bbs)
+sched_init_luids (const bb_vec_t &bbs)
{
int i;
basic_block bb;
if (reserve > 0
&& ! h_i_d.space (reserve))
{
- h_i_d.safe_grow_cleared (3 * get_max_uid () / 2);
+ h_i_d.safe_grow_cleared (3 * get_max_uid () / 2, true);
sched_extend_target ();
}
}
/* Initialize haifa_insn_data for BBS. */
void
-haifa_init_h_i_d (bb_vec_t bbs)
+haifa_init_h_i_d (const bb_vec_t &bbs)
{
int i;
basic_block bb;