/* Instruction scheduling pass.
- Copyright (C) 1992-2014 Free Software Foundation, Inc.
+ Copyright (C) 1992-2020 Free Software Foundation, Inc.
Contributed by Michael Tiemann (tiemann@cygnus.com) Enhanced by,
and currently maintained by, Jim Wilson (wilson@cygnus.com)
#include "config.h"
#include "system.h"
#include "coretypes.h"
-#include "tm.h"
-#include "diagnostic-core.h"
-#include "hard-reg-set.h"
+#include "backend.h"
+#include "target.h"
#include "rtl.h"
+#include "cfghooks.h"
+#include "df.h"
+#include "memmodel.h"
#include "tm_p.h"
-#include "regs.h"
-#include "hashtab.h"
-#include "hash-set.h"
-#include "vec.h"
-#include "machmode.h"
-#include "input.h"
-#include "function.h"
-#include "flags.h"
#include "insn-config.h"
-#include "insn-attr.h"
-#include "except.h"
+#include "regs.h"
+#include "ira.h"
#include "recog.h"
+#include "insn-attr.h"
+#include "cfgrtl.h"
+#include "cfgbuild.h"
#include "sched-int.h"
-#include "target.h"
#include "common/common-target.h"
-#include "params.h"
#include "dbgcnt.h"
#include "cfgloop.h"
-#include "ira.h"
-#include "emit-rtl.h" /* FIXME: Can go away once crtl is moved to rtl.h. */
-#include "hash-table.h"
#include "dumpfile.h"
+#include "print-rtl.h"
+#include "function-abi.h"
#ifdef INSN_SCHEDULING
/* sched-verbose controls the amount of debugging output the
scheduler prints. It is controlled by -fsched-verbose=N:
- N>0 and no -DSR : the output is directed to stderr.
- N>=10 will direct the printouts to stderr (regardless of -dSR).
- N=1: same as -dSR.
+ N=0: no debugging output.
+ N=1: default value.
N=2: bb's probabilities, detailed ready list info, unit/insn info.
N=3: rtl at abort point, control-flow, regions info.
N=5: dependences info. */
-
int sched_verbose = 0;
-/* Debugging file. All printouts are sent to dump, which is always set,
- either to stderr, or to the dump listing file (-dRS). */
+/* Debugging file. All printouts are sent to dump. */
FILE *sched_dump = 0;
/* This is a placeholder for the scheduler parameters common
#define FEEDS_BACKTRACK_INSN(INSN) (HID (INSN)->feeds_backtrack_insn)
#define SHADOW_P(INSN) (HID (INSN)->shadow_p)
#define MUST_RECOMPUTE_SPEC_P(INSN) (HID (INSN)->must_recompute_spec)
-/* Cached cost of the instruction. Use insn_cost to get cost of the
+/* Cached cost of the instruction. Use insn_sched_cost to get cost of the
insn. -1 here means that the field is not initialized. */
#define INSN_COST(INSN) (HID (INSN)->cost)
/* The minimal value of the INSN_TICK of an instruction. */
#define MIN_TICK (-max_insn_queue_index)
+/* Original order of insns in the ready list.
+ Used to keep order of normal insns while separating DEBUG_INSNs. */
+#define INSN_RFS_DEBUG_ORIG_ORDER(INSN) (HID (INSN)->rfs_debug_orig_order)
+
/* The deciding reason for INSN's place in the ready list. */
#define INSN_LAST_RFS_WIN(INSN) (HID (INSN)->last_rfs_win)
};
/* Mapping from instruction UID to its Logical UID. */
-vec<int> sched_luids = vNULL;
+vec<int> sched_luids;
/* Next LUID to assign to an instruction. */
int sched_max_luid = 1;
/* Haifa Instruction Data. */
-vec<haifa_insn_data_def> h_i_d = vNULL;
+vec<haifa_insn_data_def> h_i_d;
void (* sched_init_only_bb) (basic_block, basic_block);
modulo_max_stages = max_stages;
modulo_n_insns = insns;
modulo_iter0_max_uid = max_uid;
- modulo_backtracks_left = PARAM_VALUE (PARAM_MAX_MODULO_BACKTRACK_ATTEMPTS);
+ modulo_backtracks_left = param_max_modulo_backtrack_attempts;
}
/* A structure to record a pair of insns where the first one is a real
/* Helpers for delay hashing. */
-struct delay_i1_hasher : typed_noop_remove <delay_pair>
+struct delay_i1_hasher : nofree_ptr_hash <delay_pair>
{
- typedef delay_pair value_type;
- typedef void compare_type;
- static inline hashval_t hash (const value_type *);
- static inline bool equal (const value_type *, const compare_type *);
+ typedef void *compare_type;
+ static inline hashval_t hash (const delay_pair *);
+ static inline bool equal (const delay_pair *, const void *);
};
/* Returns a hash value for X, based on hashing just I1. */
inline hashval_t
-delay_i1_hasher::hash (const value_type *x)
+delay_i1_hasher::hash (const delay_pair *x)
{
return htab_hash_pointer (x->i1);
}
/* Return true if I1 of pair X is the same as that of pair Y. */
inline bool
-delay_i1_hasher::equal (const value_type *x, const compare_type *y)
+delay_i1_hasher::equal (const delay_pair *x, const void *y)
{
return x->i1 == y;
}
-struct delay_i2_hasher : typed_free_remove <delay_pair>
+struct delay_i2_hasher : free_ptr_hash <delay_pair>
{
- typedef delay_pair value_type;
- typedef void compare_type;
- static inline hashval_t hash (const value_type *);
- static inline bool equal (const value_type *, const compare_type *);
+ typedef void *compare_type;
+ static inline hashval_t hash (const delay_pair *);
+ static inline bool equal (const delay_pair *, const void *);
};
/* Returns a hash value for X, based on hashing just I2. */
inline hashval_t
-delay_i2_hasher::hash (const value_type *x)
+delay_i2_hasher::hash (const delay_pair *x)
{
return htab_hash_pointer (x->i2);
}
/* Return true if I2 of pair X is the same as that of pair Y. */
inline bool
-delay_i2_hasher::equal (const value_type *x, const compare_type *y)
+delay_i2_hasher::equal (const delay_pair *x, const void *y)
{
return x->i2 == y;
}
\f
/* Forward declarations. */
-static int priority (rtx_insn *);
+static int priority (rtx_insn *, bool force_recompute = false);
+static int autopref_rank_for_schedule (const rtx_insn *, const rtx_insn *);
static int rank_for_schedule (const void *, const void *);
static void swap_sort (rtx_insn **, int);
static void queue_insn (rtx_insn *, int, const char *);
/* The following functions are used to implement multi-pass scheduling
on the first cycle. */
static rtx_insn *ready_remove (struct ready_list *, int);
-static void ready_remove_insn (rtx);
+static void ready_remove_insn (rtx_insn *);
static void fix_inter_tick (rtx_insn *, rtx_insn *);
static int fix_tick_ready (rtx_insn *);
static void init_h_i_d (rtx_insn *);
static int haifa_speculate_insn (rtx_insn *, ds_t, rtx *);
static void generate_recovery_code (rtx_insn *);
-static void process_insn_forw_deps_be_in_spec (rtx, rtx_insn *, ds_t);
+static void process_insn_forw_deps_be_in_spec (rtx_insn *, rtx_insn *, ds_t);
static void begin_speculative_block (rtx_insn *);
static void add_to_speculative_block (rtx_insn *);
static void init_before_recovery (basic_block *);
/* Registers mentioned in the current region. */
static bitmap region_ref_regs;
+/* Temporary bitmap used for SCHED_PRESSURE_MODEL. */
+static bitmap tmp_bitmap;
+
/* Effective number of available registers of a given class (see comment
in sched_pressure_start_bb). */
static int sched_class_regs_num[N_REG_CLASSES];
-/* Number of call_used_regs. This is a helper for calculating of
+/* The number of registers that the function would need to save before it
+ uses them, and the number of fixed_regs. Helpers for calculating of
sched_class_regs_num. */
-static int call_used_regs_num[N_REG_CLASSES];
+static int call_saved_regs_num[N_REG_CLASSES];
+static int fixed_regs_num[N_REG_CLASSES];
/* Initiate register pressure relative info for scheduling the current
region. Currently it is only clearing register mentioned in the
static void
setup_ref_regs (rtx x)
{
- int i, j, regno;
+ int i, j;
const RTX_CODE code = GET_CODE (x);
const char *fmt;
if (REG_P (x))
{
- regno = REGNO (x);
- if (HARD_REGISTER_NUM_P (regno))
- bitmap_set_range (region_ref_regs, regno,
- hard_regno_nregs[regno][GET_MODE (x)]);
- else
- bitmap_set_bit (region_ref_regs, REGNO (x));
+ bitmap_set_range (region_ref_regs, REGNO (x), REG_NREGS (x));
return;
}
fmt = GET_RTX_FORMAT (code);
if (NONDEBUG_INSN_P (insn))
setup_ref_regs (PATTERN (insn));
initiate_reg_pressure_info (df_get_live_in (bb));
-#ifdef EH_RETURN_DATA_REGNO
if (bb_has_eh_pred (bb))
for (i = 0; ; ++i)
{
mark_regno_birth_or_death (curr_reg_live, curr_reg_pressure,
regno, true);
}
-#endif
}
/* Save current register pressure related info. */
INSN_COST (insn) = -1;
/* Invalidate INSN_TICK, so it'll be recalculated. */
INSN_TICK (insn) = INVALID_TICK;
+
+ /* Invalidate autoprefetch data entry. */
+ INSN_AUTOPREF_MULTIPASS_DATA (insn)[0].status
+ = AUTOPREF_MULTIPASS_DATA_UNINITIALIZED;
+ INSN_AUTOPREF_MULTIPASS_DATA (insn)[1].status
+ = AUTOPREF_MULTIPASS_DATA_UNINITIALIZED;
}
if (!sd_lists_empty_p (next, SD_LIST_HARD_BACK))
return HARD_DEP;
+ /* If NEXT is intended to sit adjacent to this instruction, we don't
+ want to try to break any dependencies. Treat it as a HARD_DEP. */
+ if (SCHED_GROUP_P (next))
+ return HARD_DEP;
+
/* Now we've got NEXT with speculative deps only.
1. Look at the deps to see what we have to do.
2. Check if we can do 'todo'. */
block, or the prev_head of the scheduling block. Used by
rank_for_schedule, so that insns independent of the last scheduled
insn will be preferred over dependent instructions. */
-static rtx last_nondebug_scheduled_insn;
+static rtx_insn *last_nondebug_scheduled_insn;
/* Pointer that iterates through the list of unscheduled insns if we
have a dbg_cnt enabled. It always points at an insn prior to the
This is the number of cycles between instruction issue and
instruction results. */
int
-insn_cost (rtx_insn *insn)
+insn_sched_cost (rtx_insn *insn)
{
int cost;
+ if (sched_fusion)
+ return 0;
+
if (sel_sched_p ())
{
if (recog_memoized (insn) < 0)
{
enum reg_note dep_type = DEP_TYPE (link);
- cost = insn_cost (insn);
+ cost = insn_sched_cost (insn);
if (INSN_CODE (insn) >= 0)
{
}
- if (targetm.sched.adjust_cost_2)
- cost = targetm.sched.adjust_cost_2 (used, (int) dep_type, insn, cost,
- dw);
- else if (targetm.sched.adjust_cost != NULL)
- {
- /* This variable is used for backward compatibility with the
- targets. */
- rtx_insn_list *dep_cost_rtx_link =
- alloc_INSN_LIST (NULL_RTX, NULL);
-
- /* Make it self-cycled, so that if some tries to walk over this
- incomplete list he/she will be caught in an endless loop. */
- XEXP (dep_cost_rtx_link, 1) = dep_cost_rtx_link;
-
- /* Targets use only REG_NOTE_KIND of the link. */
- PUT_REG_NOTE_KIND (dep_cost_rtx_link, DEP_TYPE (link));
-
- cost = targetm.sched.adjust_cost (used, dep_cost_rtx_link,
- insn, cost);
-
- free_INSN_LIST_node (dep_cost_rtx_link);
- }
+ if (targetm.sched.adjust_cost)
+ cost = targetm.sched.adjust_cost (used, (int) dep_type, insn, cost,
+ dw);
if (cost < 0)
cost = 0;
/* Compute the number of nondebug deps in list LIST for INSN. */
static int
-dep_list_size (rtx insn, sd_list_types_def list)
+dep_list_size (rtx_insn *insn, sd_list_types_def list)
{
sd_iterator_def sd_it;
dep_t dep;
return nodbgcount;
}
+bool sched_fusion;
+
/* Compute the priority number for INSN. */
static int
-priority (rtx_insn *insn)
+priority (rtx_insn *insn, bool force_recompute)
{
if (! INSN_P (insn))
return 0;
/* We should not be interested in priority of an already scheduled insn. */
gcc_assert (QUEUE_INDEX (insn) != QUEUE_SCHEDULED);
- if (!INSN_PRIORITY_KNOWN (insn))
+ if (force_recompute || !INSN_PRIORITY_KNOWN (insn))
{
int this_priority = -1;
- if (dep_list_size (insn, SD_LIST_FORW) == 0)
- /* ??? We should set INSN_PRIORITY to insn_cost when and insn has
- some forward deps but all of them are ignored by
+ if (sched_fusion)
+ {
+ int this_fusion_priority;
+
+ targetm.sched.fusion_priority (insn, FUSION_MAX_PRIORITY,
+ &this_fusion_priority, &this_priority);
+ INSN_FUSION_PRIORITY (insn) = this_fusion_priority;
+ }
+ else if (dep_list_size (insn, SD_LIST_FORW) == 0)
+ /* ??? We should set INSN_PRIORITY to insn_sched_cost when and insn
+ has some forward deps but all of them are ignored by
contributes_to_priority hook. At the moment we set priority of
such insn to 0. */
- this_priority = insn_cost (insn);
+ this_priority = insn_sched_cost (insn);
else
{
rtx_insn *prev_first, *twin;
{
gcc_assert (this_priority == -1);
- this_priority = insn_cost (insn);
+ this_priority = insn_sched_cost (insn);
}
INSN_PRIORITY (insn) = this_priority;
{
int i, change, before, after, hard_regno;
int excess_cost_change;
- enum machine_mode mode;
+ machine_mode mode;
enum reg_class cl;
struct reg_pressure_data *pressure_info;
int *max_reg_pressure;
registers that will be born in the range [model_curr_point, POINT). */
num_uses = 0;
num_pending_births = 0;
+ bitmap_clear (tmp_bitmap);
for (use = INSN_REG_USE_LIST (insn); use != NULL; use = use->next_insn_use)
{
new_last = model_last_use_except (use);
- if (new_last < point)
+ if (new_last < point && bitmap_set_bit (tmp_bitmap, use->regno))
{
gcc_assert (num_uses < ARRAY_SIZE (uses));
uses[num_uses].last_use = new_last;
/* Enum of rank_for_schedule heuristic decisions. */
enum rfs_decision {
- RFS_DEBUG, RFS_LIVE_RANGE_SHRINK1, RFS_LIVE_RANGE_SHRINK2,
+ RFS_LIVE_RANGE_SHRINK1, RFS_LIVE_RANGE_SHRINK2,
RFS_SCHED_GROUP, RFS_PRESSURE_DELAY, RFS_PRESSURE_TICK,
RFS_FEEDS_BACKTRACK_INSN, RFS_PRIORITY, RFS_SPECULATION,
RFS_SCHED_RANK, RFS_LAST_INSN, RFS_PRESSURE_INDEX,
- RFS_DEP_COUNT, RFS_TIE, RFS_N };
+ RFS_DEP_COUNT, RFS_TIE, RFS_FUSION, RFS_COST, RFS_N };
/* Corresponding strings for print outs. */
static const char *rfs_str[RFS_N] = {
- "RFS_DEBUG", "RFS_LIVE_RANGE_SHRINK1", "RFS_LIVE_RANGE_SHRINK2",
+ "RFS_LIVE_RANGE_SHRINK1", "RFS_LIVE_RANGE_SHRINK2",
"RFS_SCHED_GROUP", "RFS_PRESSURE_DELAY", "RFS_PRESSURE_TICK",
"RFS_FEEDS_BACKTRACK_INSN", "RFS_PRIORITY", "RFS_SPECULATION",
"RFS_SCHED_RANK", "RFS_LAST_INSN", "RFS_PRESSURE_INDEX",
- "RFS_DEP_COUNT", "RFS_TIE" };
+ "RFS_DEP_COUNT", "RFS_TIE", "RFS_FUSION", "RFS_COST" };
/* Statistical breakdown of rank_for_schedule decisions. */
-typedef struct { unsigned stats[RFS_N]; } rank_for_schedule_stats_t;
+struct rank_for_schedule_stats_t { unsigned stats[RFS_N]; };
static rank_for_schedule_stats_t rank_for_schedule_stats;
/* Return the result of comparing insns TMP and TMP2 and update
return result;
}
+/* Sorting predicate to move DEBUG_INSNs to the top of ready list, while
+ keeping normal insns in original order. */
+
+static int
+rank_for_schedule_debug (const void *x, const void *y)
+{
+ rtx_insn *tmp = *(rtx_insn * const *) y;
+ rtx_insn *tmp2 = *(rtx_insn * const *) x;
+
+ /* Schedule debug insns as early as possible. */
+ if (DEBUG_INSN_P (tmp) && !DEBUG_INSN_P (tmp2))
+ return -1;
+ else if (!DEBUG_INSN_P (tmp) && DEBUG_INSN_P (tmp2))
+ return 1;
+ else if (DEBUG_INSN_P (tmp) && DEBUG_INSN_P (tmp2))
+ return INSN_LUID (tmp) - INSN_LUID (tmp2);
+ else
+ return INSN_RFS_DEBUG_ORIG_ORDER (tmp2) - INSN_RFS_DEBUG_ORIG_ORDER (tmp);
+}
+
/* Returns a positive value if x is preferred; returns a negative value if
y is preferred. Should never return 0, since that will make the sort
unstable. */
int tmp_class, tmp2_class;
int val, priority_val, info_val, diff;
- if (MAY_HAVE_DEBUG_INSNS)
- {
- /* Schedule debug insns as early as possible. */
- if (DEBUG_INSN_P (tmp) && !DEBUG_INSN_P (tmp2))
- return rfs_result (RFS_DEBUG, -1, tmp, tmp2);
- else if (!DEBUG_INSN_P (tmp) && DEBUG_INSN_P (tmp2))
- return rfs_result (RFS_DEBUG, 1, tmp, tmp2);
- else if (DEBUG_INSN_P (tmp) && DEBUG_INSN_P (tmp2))
- return rfs_result (RFS_DEBUG, INSN_LUID (tmp) - INSN_LUID (tmp2),
- tmp, tmp2);
- }
-
if (live_range_shrinkage_p)
{
/* Don't use SCHED_PRESSURE_MODEL -- it results in much worse
/* Make sure that priority of TMP and TMP2 are initialized. */
gcc_assert (INSN_PRIORITY_KNOWN (tmp) && INSN_PRIORITY_KNOWN (tmp2));
+ if (sched_fusion)
+ {
+ /* The instruction that has the same fusion priority as the last
+ instruction is the instruction we picked next. If that is not
+ the case, we sort ready list firstly by fusion priority, then
+ by priority, and at last by INSN_LUID. */
+ int a = INSN_FUSION_PRIORITY (tmp);
+ int b = INSN_FUSION_PRIORITY (tmp2);
+ int last = -1;
+
+ if (last_nondebug_scheduled_insn
+ && !NOTE_P (last_nondebug_scheduled_insn)
+ && BLOCK_FOR_INSN (tmp)
+ == BLOCK_FOR_INSN (last_nondebug_scheduled_insn))
+ last = INSN_FUSION_PRIORITY (last_nondebug_scheduled_insn);
+
+ if (a != last && b != last)
+ {
+ if (a == b)
+ {
+ a = INSN_PRIORITY (tmp);
+ b = INSN_PRIORITY (tmp2);
+ }
+ if (a != b)
+ return rfs_result (RFS_FUSION, b - a, tmp, tmp2);
+ else
+ return rfs_result (RFS_FUSION,
+ INSN_LUID (tmp) - INSN_LUID (tmp2), tmp, tmp2);
+ }
+ else if (a == b)
+ {
+ gcc_assert (last_nondebug_scheduled_insn
+ && !NOTE_P (last_nondebug_scheduled_insn));
+ last = INSN_PRIORITY (last_nondebug_scheduled_insn);
+
+ a = abs (INSN_PRIORITY (tmp) - last);
+ b = abs (INSN_PRIORITY (tmp2) - last);
+ if (a != b)
+ return rfs_result (RFS_FUSION, a - b, tmp, tmp2);
+ else
+ return rfs_result (RFS_FUSION,
+ INSN_LUID (tmp) - INSN_LUID (tmp2), tmp, tmp2);
+ }
+ else if (a == last)
+ return rfs_result (RFS_FUSION, -1, tmp, tmp2);
+ else
+ return rfs_result (RFS_FUSION, 1, tmp, tmp2);
+ }
+
if (sched_pressure != SCHED_PRESSURE_NONE)
{
/* Prefer insn whose scheduling results in the smallest register
if (flag_sched_critical_path_heuristic && priority_val)
return rfs_result (RFS_PRIORITY, priority_val, tmp, tmp2);
+ if (param_sched_autopref_queue_depth >= 0)
+ {
+ int autopref = autopref_rank_for_schedule (tmp, tmp2);
+ if (autopref != 0)
+ return autopref;
+ }
+
/* Prefer speculative insn with greater dependencies weakness. */
if (flag_sched_spec_insn_heuristic && spec_info)
{
{
dep_t dep1;
dep_t dep2;
- rtx last = last_nondebug_scheduled_insn;
+ rtx_insn *last = last_nondebug_scheduled_insn;
/* Classify the instructions into three classes:
1) Data dependent on last schedule insn.
}
/* Prefer instructions that occur earlier in the model schedule. */
- if (sched_pressure == SCHED_PRESSURE_MODEL
- && INSN_BB (tmp) == target_bb && INSN_BB (tmp2) == target_bb)
+ if (sched_pressure == SCHED_PRESSURE_MODEL)
{
diff = model_index (tmp) - model_index (tmp2);
- gcc_assert (diff != 0);
- return rfs_result (RFS_PRESSURE_INDEX, diff, tmp, tmp2);
+ if (diff != 0)
+ return rfs_result (RFS_PRESSURE_INDEX, diff, tmp, tmp2);
}
/* Prefer the insn which has more later insns that depend on it.
if (flag_sched_dep_count_heuristic && val != 0)
return rfs_result (RFS_DEP_COUNT, val, tmp, tmp2);
+ /* Sort by INSN_COST rather than INSN_LUID. This means that instructions
+ which take longer to execute are prioritised and it leads to more
+ dual-issue opportunities on in-order cores which have this feature. */
+
+ if (INSN_COST (tmp) != INSN_COST (tmp2))
+ return rfs_result (RFS_COST, INSN_COST (tmp2) - INSN_COST (tmp),
+ tmp, tmp2);
+
/* If insns are equally good, sort by INSN_LUID (original insn order),
so that we make the sort stable. This minimizes instruction movement,
thus minimizing sched's effect on debugging and cross-jumping. */
/* Remove INSN from the ready list. */
static void
-ready_remove_insn (rtx insn)
+ready_remove_insn (rtx_insn *insn)
{
int i;
}
}
-/* Sort the ready list READY by ascending priority, using the SCHED_SORT
- macro. */
+/* Separate DEBUG_INSNS from normal insns. DEBUG_INSNs go to the end
+ of array. */
+static void
+ready_sort_debug (struct ready_list *ready)
+{
+ int i;
+ rtx_insn **first = ready_lastpos (ready);
-void
-ready_sort (struct ready_list *ready)
+ for (i = 0; i < ready->n_ready; ++i)
+ if (!DEBUG_INSN_P (first[i]))
+ INSN_RFS_DEBUG_ORIG_ORDER (first[i]) = i;
+
+ qsort (first, ready->n_ready, sizeof (rtx), rank_for_schedule_debug);
+}
+
+/* Sort non-debug insns in the ready list READY by ascending priority.
+ Assumes that all debug insns are separated from the real insns. */
+static void
+ready_sort_real (struct ready_list *ready)
{
int i;
rtx_insn **first = ready_lastpos (ready);
+ int n_ready_real = ready->n_ready - ready->n_debug;
if (sched_pressure == SCHED_PRESSURE_WEIGHTED)
- {
- for (i = 0; i < ready->n_ready; i++)
- if (!DEBUG_INSN_P (first[i]))
- setup_insn_reg_pressure_info (first[i]);
- }
- if (sched_pressure == SCHED_PRESSURE_MODEL
- && model_curr_point < model_num_insns)
- model_set_excess_costs (first, ready->n_ready);
+ for (i = 0; i < n_ready_real; ++i)
+ setup_insn_reg_pressure_info (first[i]);
+ else if (sched_pressure == SCHED_PRESSURE_MODEL
+ && model_curr_point < model_num_insns)
+ model_set_excess_costs (first, n_ready_real);
rank_for_schedule_stats_t stats1;
if (sched_verbose >= 4)
stats1 = rank_for_schedule_stats;
- if (ready->n_ready == 2)
- swap_sort (first, ready->n_ready);
- else if (ready->n_ready > 2)
- qsort (first, ready->n_ready, sizeof (rtx), rank_for_schedule);
+ if (n_ready_real == 2)
+ swap_sort (first, n_ready_real);
+ else if (n_ready_real > 2)
+ qsort (first, n_ready_real, sizeof (rtx), rank_for_schedule);
if (sched_verbose >= 4)
{
}
}
+/* Sort the ready list READY by ascending priority. */
+static void
+ready_sort (struct ready_list *ready)
+{
+ if (ready->n_debug > 0)
+ ready_sort_debug (ready);
+ else
+ ready_sort_real (ready);
+}
+
/* PREV is an insn that is ready to execute. Adjust its priority if that
will help shorten or lengthen register lifetimes as appropriate. Also
provide a hook for the target to tweak itself. */
only be scheduled once their control dependency is resolved. */
static void
-check_clobbered_conditions (rtx insn)
+check_clobbered_conditions (rtx_insn *insn)
{
HARD_REG_SET t;
int i;
}
/* Add INSN to the model worklist. Start looking for a suitable position
- between neighbors PREV and NEXT, testing at most MAX_SCHED_READY_INSNS
+ between neighbors PREV and NEXT, testing at most param_max_sched_ready_insns
insns either side. A null PREV indicates the beginning of the list and
a null NEXT indicates the end. */
{
int count;
- count = MAX_SCHED_READY_INSNS;
+ count = param_max_sched_ready_insns;
if (count > 0 && prev && model_order_p (insn, prev))
do
{
int count;
prev = insn->prev;
- count = MAX_SCHED_READY_INSNS;
+ count = param_max_sched_ready_insns;
while (count > 0 && prev && model_order_p (insn, prev))
{
count--;
{
fprintf (sched_dump, ";;\t+--- worklist:\n");
insn = model_worklist;
- count = MAX_SCHED_READY_INSNS;
+ count = param_max_sched_ready_insns;
while (count > 0 && insn)
{
fprintf (sched_dump, ";;\t+--- %d [%d, %d, %d, %d]\n",
Failing that, just pick the highest-priority instruction in the
worklist. */
- count = MAX_SCHED_READY_INSNS;
+ count = param_max_sched_ready_insns;
insn = model_worklist;
fallback = 0;
for (;;)
* If the basic block executes much more often than the prologue/epilogue
(e.g., inside a hot loop), then cost of spill in the prologue is close to
nil, so the effective number of available registers is
- (ira_class_hard_regs_num[cl] - 0).
+ (ira_class_hard_regs_num[cl] - fixed_regs_num[cl] - 0).
* If the basic block executes as often as the prologue/epilogue,
then spill in the block is as costly as in the prologue, so the effective
number of available registers is
- (ira_class_hard_regs_num[cl] - call_used_regs_num[cl]).
+ (ira_class_hard_regs_num[cl] - fixed_regs_num[cl]
+ - call_saved_regs_num[cl]).
Note that all-else-equal, we prefer to spill in the prologue, since that
allows "extra" registers for other basic blocks of the function.
* If the basic block is on the cold path of the function and executes
rarely, then we should always prefer to spill in the block, rather than
in the prologue/epilogue. The effective number of available register is
- (ira_class_hard_regs_num[cl] - call_used_regs_num[cl]). */
+ (ira_class_hard_regs_num[cl] - fixed_regs_num[cl]
+ - call_saved_regs_num[cl]). */
{
int i;
- int entry_freq = ENTRY_BLOCK_PTR_FOR_FN (cfun)->frequency;
- int bb_freq = bb->frequency;
+ int entry_freq = ENTRY_BLOCK_PTR_FOR_FN (cfun)->count.to_frequency (cfun);
+ int bb_freq = bb->count.to_frequency (cfun);
if (bb_freq == 0)
{
for (i = 0; i < ira_pressure_classes_num; ++i)
{
enum reg_class cl = ira_pressure_classes[i];
- sched_class_regs_num[cl] = ira_class_hard_regs_num[cl];
+ sched_class_regs_num[cl] = ira_class_hard_regs_num[cl]
+ - fixed_regs_num[cl];
sched_class_regs_num[cl]
- -= (call_used_regs_num[cl] * entry_freq) / bb_freq;
+ -= (call_saved_regs_num[cl] * entry_freq) / bb_freq;
}
}
gcc_assert (sd_lists_empty_p (insn, SD_LIST_HARD_BACK));
/* Reset debug insns invalidated by moving this insn. */
- if (MAY_HAVE_DEBUG_INSNS && !DEBUG_INSN_P (insn))
+ if (MAY_HAVE_DEBUG_BIND_INSNS && !DEBUG_INSN_P (insn))
for (sd_it = sd_iterator_start (insn, SD_LIST_BACK);
sd_iterator_cond (&sd_it, &dep);)
{
continue;
}
- gcc_assert (DEBUG_INSN_P (dbg));
+ gcc_assert (DEBUG_BIND_INSN_P (dbg));
if (sched_verbose >= 6)
fprintf (sched_dump, ";;\t\tresetting: debug insn %d\n",
gcc_assert (INSN_TICK (insn) >= MIN_TICK);
if (INSN_TICK (insn) > clock_var)
/* INSN has been prematurely moved from the queue to the ready list.
- This is possible only if following flag is set. */
- gcc_assert (flag_sched_stalled_insns);
+ This is possible only if following flags are set. */
+ gcc_assert (flag_sched_stalled_insns || sched_fusion);
/* ??? Probably, if INSN is scheduled prematurely, we should leave
INSN_TICK untouched. This is a machine-dependent issue, actually. */
state_t curr_state;
rtx_insn *last_scheduled_insn;
- rtx last_nondebug_scheduled_insn;
+ rtx_insn *last_nondebug_scheduled_insn;
rtx_insn *nonscheduled_insns_begin;
int cycle_issued_insns;
/* For every dependency of INSN, set the FEEDS_BACKTRACK_INSN bit according
to SET_P. */
static void
-mark_backtrack_feeds (rtx insn, int set_p)
+mark_backtrack_feeds (rtx_insn *insn, int set_p)
{
sd_iterator_def sd_it;
dep_t dep;
queued nowhere. */
static void
-unschedule_insns_until (rtx insn)
+unschedule_insns_until (rtx_insn *insn)
{
auto_vec<rtx_insn *> recompute_vec;
success = validate_change (desc->insn, desc->loc, desc->newval, 0);
gcc_assert (success);
+ rtx_insn *insn = DEP_PRO (dep);
+
+ /* Recompute priority since dependent priorities may have changed. */
+ priority (insn, true);
update_insn_after_change (desc->insn);
+
if ((TODO_SPEC (desc->insn) & (HARD_DEP | DEP_POSTPONED)) == 0)
fix_tick_ready (desc->insn);
success = validate_change (desc->insn, desc->loc, desc->orig, 0);
gcc_assert (success);
+
+ rtx_insn *insn = DEP_PRO (dep);
+
+ if (QUEUE_INDEX (insn) != QUEUE_SCHEDULED)
+ {
+ /* Recompute priority since dependent priorities may have changed. */
+ priority (insn, true);
+ }
+
update_insn_after_change (desc->insn);
+
if (backtrack_queue != NULL)
{
backtrack_queue->replacement_deps.safe_push (dep);
static int
estimate_shadow_tick (struct delay_pair *p)
{
- bitmap_head processed;
+ auto_bitmap processed;
int t;
bool cutoff;
- bitmap_initialize (&processed, 0);
- cutoff = !estimate_insn_tick (&processed, p->i2,
+ cutoff = !estimate_insn_tick (processed, p->i2,
max_insn_queue_index + pair_delay (p));
- bitmap_clear (&processed);
if (cutoff)
return max_insn_queue_index;
t = INSN_TICK_ESTIMATE (p->i2) - (clock_var + pair_delay (p) + 1);
{
rtx_insn *insn;
rtx_insn_list *link;
- rtx skip_insn;
+ rtx_insn *skip_insn;
q_ptr = NEXT_Q (q_ptr);
nonscheduled insn. */
skip_insn = first_nonscheduled_insn ();
else
- skip_insn = NULL_RTX;
+ skip_insn = NULL;
/* Add all pending insns that can be scheduled without stalls to the
ready list. */
/* If the ready list is full, delay the insn for 1 cycle.
See the comment in schedule_block for the rationale. */
if (!reload_completed
- && (ready->n_ready - ready->n_debug > MAX_SCHED_READY_INSNS
+ && (ready->n_ready - ready->n_debug > param_max_sched_ready_insns
|| (sched_pressure == SCHED_PRESSURE_MODEL
- /* Limit pressure recalculations to MAX_SCHED_READY_INSNS
- instructions too. */
+ /* Limit pressure recalculations to
+ param_max_sched_ready_insns instructions too. */
&& model_index (insn) > (model_curr_point
- + MAX_SCHED_READY_INSNS)))
+ + param_max_sched_ready_insns)))
&& !(sched_pressure == SCHED_PRESSURE_MODEL
&& model_curr_point < model_num_insns
/* Always allow the next model instruction to issue. */
addition) depending on user flags and target hooks. */
static bool
-ok_for_early_queue_removal (rtx insn)
+ok_for_early_queue_removal (rtx_insn *insn)
{
if (targetm.sched.is_costly_dependence)
{
- rtx prev_insn;
int n_cycles;
int i = scheduled_insns.length ();
for (n_cycles = flag_sched_stalled_insns_dep; n_cycles; n_cycles--)
{
int cost;
- prev_insn = scheduled_insns[i];
+ rtx_insn *prev_insn = scheduled_insns[i];
if (!NOTE_P (prev_insn))
{
last = emit_note_before (note_type, last);
remove_note (insn, note);
+ df_insn_create_insn_record (last);
}
}
}
return false;
}
+/* Helper for autopref_multipass_init. Given a SET in PAT and whether
+ we're expecting a memory WRITE or not, check that the insn is relevant to
+ the autoprefetcher modelling code. Return true iff that is the case.
+ If it is relevant, record the base register of the memory op in BASE and
+ the offset in OFFSET. */
+
+static bool
+analyze_set_insn_for_autopref (rtx pat, bool write, rtx *base, int *offset)
+{
+ if (GET_CODE (pat) != SET)
+ return false;
+
+ rtx mem = write ? SET_DEST (pat) : SET_SRC (pat);
+ if (!MEM_P (mem))
+ return false;
+
+ struct address_info info;
+ decompose_mem_address (&info, mem);
+
+ /* TODO: Currently only (base+const) addressing is supported. */
+ if (info.base == NULL || !REG_P (*info.base)
+ || (info.disp != NULL && !CONST_INT_P (*info.disp)))
+ return false;
+
+ *base = *info.base;
+ *offset = info.disp ? INTVAL (*info.disp) : 0;
+ return true;
+}
+
+/* Functions to model cache auto-prefetcher.
+
+ Some of the CPUs have cache auto-prefetcher, which /seems/ to initiate
+ memory prefetches if it sees instructions with consequitive memory accesses
+ in the instruction stream. Details of such hardware units are not published,
+ so we can only guess what exactly is going on there.
+ In the scheduler, we model abstract auto-prefetcher. If there are memory
+ insns in the ready list (or the queue) that have same memory base, but
+ different offsets, then we delay the insns with larger offsets until insns
+ with smaller offsets get scheduled. If PARAM_SCHED_AUTOPREF_QUEUE_DEPTH
+ is "1", then we look at the ready list; if it is N>1, then we also look
+ through N-1 queue entries.
+ If the param is N>=0, then rank_for_schedule will consider auto-prefetching
+ among its heuristics.
+ Param value of "-1" disables modelling of the auto-prefetcher. */
+
+/* Initialize autoprefetcher model data for INSN. */
+static void
+autopref_multipass_init (const rtx_insn *insn, int write)
+{
+ autopref_multipass_data_t data = &INSN_AUTOPREF_MULTIPASS_DATA (insn)[write];
+
+ gcc_assert (data->status == AUTOPREF_MULTIPASS_DATA_UNINITIALIZED);
+ data->base = NULL_RTX;
+ data->offset = 0;
+ /* Set insn entry initialized, but not relevant for auto-prefetcher. */
+ data->status = AUTOPREF_MULTIPASS_DATA_IRRELEVANT;
+
+ rtx pat = PATTERN (insn);
+
+ /* We have a multi-set insn like a load-multiple or store-multiple.
+ We care about these as long as all the memory ops inside the PARALLEL
+ have the same base register. We care about the minimum and maximum
+ offsets from that base but don't check for the order of those offsets
+ within the PARALLEL insn itself. */
+ if (GET_CODE (pat) == PARALLEL)
+ {
+ int n_elems = XVECLEN (pat, 0);
+
+ int i, offset;
+ rtx base, prev_base = NULL_RTX;
+ int min_offset = INT_MAX;
+
+ for (i = 0; i < n_elems; i++)
+ {
+ rtx set = XVECEXP (pat, 0, i);
+ if (GET_CODE (set) != SET)
+ return;
+
+ if (!analyze_set_insn_for_autopref (set, write, &base, &offset))
+ return;
+
+ /* Ensure that all memory operations in the PARALLEL use the same
+ base register. */
+ if (i > 0 && REGNO (base) != REGNO (prev_base))
+ return;
+ prev_base = base;
+ min_offset = MIN (min_offset, offset);
+ }
+
+ /* If we reached here then we have a valid PARALLEL of multiple memory ops
+ with prev_base as the base and min_offset containing the offset. */
+ gcc_assert (prev_base);
+ data->base = prev_base;
+ data->offset = min_offset;
+ data->status = AUTOPREF_MULTIPASS_DATA_NORMAL;
+ return;
+ }
+
+ /* Otherwise this is a single set memory operation. */
+ rtx set = single_set (insn);
+ if (set == NULL_RTX)
+ return;
+
+ if (!analyze_set_insn_for_autopref (set, write, &data->base,
+ &data->offset))
+ return;
+
+ /* This insn is relevant for the auto-prefetcher.
+ The base and offset fields will have been filled in the
+ analyze_set_insn_for_autopref call above. */
+ data->status = AUTOPREF_MULTIPASS_DATA_NORMAL;
+}
+
+/* Helper function for rank_for_schedule sorting. */
+static int
+autopref_rank_for_schedule (const rtx_insn *insn1, const rtx_insn *insn2)
+{
+ int r = 0;
+ for (int write = 0; write < 2 && !r; ++write)
+ {
+ autopref_multipass_data_t data1
+ = &INSN_AUTOPREF_MULTIPASS_DATA (insn1)[write];
+ autopref_multipass_data_t data2
+ = &INSN_AUTOPREF_MULTIPASS_DATA (insn2)[write];
+
+ if (data1->status == AUTOPREF_MULTIPASS_DATA_UNINITIALIZED)
+ autopref_multipass_init (insn1, write);
+
+ if (data2->status == AUTOPREF_MULTIPASS_DATA_UNINITIALIZED)
+ autopref_multipass_init (insn2, write);
+
+ int irrel1 = data1->status == AUTOPREF_MULTIPASS_DATA_IRRELEVANT;
+ int irrel2 = data2->status == AUTOPREF_MULTIPASS_DATA_IRRELEVANT;
+
+ if (!irrel1 && !irrel2)
+ r = data1->offset - data2->offset;
+ else
+ r = irrel2 - irrel1;
+ }
+
+ return r;
+}
+
+/* True if header of debug dump was printed. */
+static bool autopref_multipass_dfa_lookahead_guard_started_dump_p;
+
+/* Helper for autopref_multipass_dfa_lookahead_guard.
+ Return "1" if INSN1 should be delayed in favor of INSN2. */
+static int
+autopref_multipass_dfa_lookahead_guard_1 (const rtx_insn *insn1,
+ const rtx_insn *insn2, int write)
+{
+ autopref_multipass_data_t data1
+ = &INSN_AUTOPREF_MULTIPASS_DATA (insn1)[write];
+ autopref_multipass_data_t data2
+ = &INSN_AUTOPREF_MULTIPASS_DATA (insn2)[write];
+
+ if (data2->status == AUTOPREF_MULTIPASS_DATA_UNINITIALIZED)
+ autopref_multipass_init (insn2, write);
+ if (data2->status == AUTOPREF_MULTIPASS_DATA_IRRELEVANT)
+ return 0;
+
+ if (rtx_equal_p (data1->base, data2->base)
+ && data1->offset > data2->offset)
+ {
+ if (sched_verbose >= 2)
+ {
+ if (!autopref_multipass_dfa_lookahead_guard_started_dump_p)
+ {
+ fprintf (sched_dump,
+ ";;\t\tnot trying in max_issue due to autoprefetch "
+ "model: ");
+ autopref_multipass_dfa_lookahead_guard_started_dump_p = true;
+ }
+
+ fprintf (sched_dump, " %d(%d)", INSN_UID (insn1), INSN_UID (insn2));
+ }
+
+ return 1;
+ }
+
+ return 0;
+}
+
+/* General note:
+
+ We could have also hooked autoprefetcher model into
+ first_cycle_multipass_backtrack / first_cycle_multipass_issue hooks
+ to enable intelligent selection of "[r1+0]=r2; [r1+4]=r3" on the same cycle
+ (e.g., once "[r1+0]=r2" is issued in max_issue(), "[r1+4]=r3" gets
+ unblocked). We don't bother about this yet because target of interest
+ (ARM Cortex-A15) can issue only 1 memory operation per cycle. */
+
+/* Implementation of first_cycle_multipass_dfa_lookahead_guard hook.
+ Return "1" if INSN1 should not be considered in max_issue due to
+ auto-prefetcher considerations. */
+int
+autopref_multipass_dfa_lookahead_guard (rtx_insn *insn1, int ready_index)
+{
+ int r = 0;
+
+ /* Exit early if the param forbids this or if we're not entering here through
+ normal haifa scheduling. This can happen if selective scheduling is
+ explicitly enabled. */
+ if (!insn_queue || param_sched_autopref_queue_depth <= 0)
+ return 0;
+
+ if (sched_verbose >= 2 && ready_index == 0)
+ autopref_multipass_dfa_lookahead_guard_started_dump_p = false;
+
+ for (int write = 0; write < 2; ++write)
+ {
+ autopref_multipass_data_t data1
+ = &INSN_AUTOPREF_MULTIPASS_DATA (insn1)[write];
+
+ if (data1->status == AUTOPREF_MULTIPASS_DATA_UNINITIALIZED)
+ autopref_multipass_init (insn1, write);
+ if (data1->status == AUTOPREF_MULTIPASS_DATA_IRRELEVANT)
+ continue;
+
+ if (ready_index == 0
+ && data1->status == AUTOPREF_MULTIPASS_DATA_DONT_DELAY)
+ /* We allow only a single delay on priviledged instructions.
+ Doing otherwise would cause infinite loop. */
+ {
+ if (sched_verbose >= 2)
+ {
+ if (!autopref_multipass_dfa_lookahead_guard_started_dump_p)
+ {
+ fprintf (sched_dump,
+ ";;\t\tnot trying in max_issue due to autoprefetch "
+ "model: ");
+ autopref_multipass_dfa_lookahead_guard_started_dump_p = true;
+ }
+
+ fprintf (sched_dump, " *%d*", INSN_UID (insn1));
+ }
+ continue;
+ }
+
+ for (int i2 = 0; i2 < ready.n_ready; ++i2)
+ {
+ rtx_insn *insn2 = get_ready_element (i2);
+ if (insn1 == insn2)
+ continue;
+ r = autopref_multipass_dfa_lookahead_guard_1 (insn1, insn2, write);
+ if (r)
+ {
+ if (ready_index == 0)
+ {
+ r = -1;
+ data1->status = AUTOPREF_MULTIPASS_DATA_DONT_DELAY;
+ }
+ goto finish;
+ }
+ }
+
+ if (param_sched_autopref_queue_depth == 1)
+ continue;
+
+ /* Everything from the current queue slot should have been moved to
+ the ready list. */
+ gcc_assert (insn_queue[NEXT_Q_AFTER (q_ptr, 0)] == NULL_RTX);
+
+ int n_stalls = param_sched_autopref_queue_depth - 1;
+ if (n_stalls > max_insn_queue_index)
+ n_stalls = max_insn_queue_index;
+
+ for (int stalls = 1; stalls <= n_stalls; ++stalls)
+ {
+ for (rtx_insn_list *link = insn_queue[NEXT_Q_AFTER (q_ptr, stalls)];
+ link != NULL_RTX;
+ link = link->next ())
+ {
+ rtx_insn *insn2 = link->insn ();
+ r = autopref_multipass_dfa_lookahead_guard_1 (insn1, insn2,
+ write);
+ if (r)
+ {
+ /* Queue INSN1 until INSN2 can issue. */
+ r = -stalls;
+ if (ready_index == 0)
+ data1->status = AUTOPREF_MULTIPASS_DATA_DONT_DELAY;
+ goto finish;
+ }
+ }
+ }
+ }
+
+ finish:
+ if (sched_verbose >= 2
+ && autopref_multipass_dfa_lookahead_guard_started_dump_p
+ && (ready_index == ready.n_ready - 1 || r < 0))
+ /* This does not /always/ trigger. We don't output EOL if the last
+ insn is not recognized (INSN_CODE < 0) and lookahead_guard is not
+ called. We can live with this. */
+ fprintf (sched_dump, "\n");
+
+ return r;
+}
+
/* Define type for target data used in multipass scheduling. */
#ifndef TARGET_SCHED_FIRST_CYCLE_MULTIPASS_DATA_T
# define TARGET_SCHED_FIRST_CYCLE_MULTIPASS_DATA_T int
struct choice_entry *top;
rtx_insn *insn;
+ if (sched_fusion)
+ return 0;
+
n_ready = ready->n_ready;
gcc_assert (dfa_lookahead >= 1 && privileged_n >= 0
&& privileged_n <= n_ready);
{
int i, pass;
bool sched_group_found = false;
- int min_cost_group = 1;
+ int min_cost_group = 0;
+
+ if (sched_fusion)
+ return;
for (i = 0; i < ready.n_ready; i++)
{
}
/* Make two passes if there's a SCHED_GROUP_P insn; make sure to handle
- such an insn first and note its cost, then schedule all other insns
- for one cycle later. */
+ such an insn first and note its cost. If at least one SCHED_GROUP_P insn
+ gets queued, then all other insns get queued for one cycle later. */
for (pass = sched_group_found ? 0 : 1; pass < 2; )
{
int n = ready.n_ready;
if (DEBUG_INSN_P (insn))
continue;
- if (sched_group_found && !SCHED_GROUP_P (insn))
+ if (sched_group_found && !SCHED_GROUP_P (insn)
+ && ((pass == 0) || (min_cost_group >= 1)))
{
if (pass == 0)
continue;
if (SCHED_GROUP_P (insn) && cost > min_cost_group)
min_cost_group = cost;
ready_remove (&ready, i);
- queue_insn (insn, cost, reason);
+ /* Normally we'd want to queue INSN for COST cycles. However,
+ if SCHED_GROUP_P is set, then we must ensure that nothing
+ else comes between INSN and its predecessor. If there is
+ some other insn ready to fire on the next cycle, then that
+ invariant would be broken.
+
+ So when SCHED_GROUP_P is set, just queue this insn for a
+ single cycle. */
+ queue_insn (insn, SCHED_GROUP_P (insn) ? 1 : cost, reason);
if (i + 1 < n)
break;
}
if (sched_verbose >= 4)
{
- if (NOTE_P (insn) || recog_memoized (insn) < 0)
+ if (NOTE_P (insn) || LABEL_P (insn) || recog_memoized (insn) < 0)
fprintf (sched_dump, "nothing");
else
print_reservation (sched_dump, insn);
rtx_insn *tail = PREV_INSN (next_tail);
if ((current_sched_info->flags & DONT_BREAK_DEPENDENCIES) == 0
- && sched_pressure != SCHED_PRESSURE_MODEL)
+ && sched_pressure != SCHED_PRESSURE_MODEL && !sched_fusion)
find_modifiable_mems (head, tail);
/* We used to have code to avoid getting parameters moved from hard
/* We start inserting insns after PREV_HEAD. */
last_scheduled_insn = prev_head;
- last_nondebug_scheduled_insn = NULL_RTX;
+ last_nondebug_scheduled_insn = NULL;
nonscheduled_insns_begin = NULL;
gcc_assert ((NOTE_P (last_scheduled_insn)
time in the worst case. Before reload we are more likely to have
big lists so truncate them to a reasonable size. */
if (!reload_completed
- && ready.n_ready - ready.n_debug > MAX_SCHED_READY_INSNS)
+ && ready.n_ready - ready.n_debug > param_max_sched_ready_insns)
{
- ready_sort (&ready);
+ ready_sort_debug (&ready);
+ ready_sort_real (&ready);
- /* Find first free-standing insn past MAX_SCHED_READY_INSNS.
+ /* Find first free-standing insn past param_max_sched_ready_insns.
If there are debug insns, we know they're first. */
- for (i = MAX_SCHED_READY_INSNS + ready.n_debug; i < ready.n_ready; i++)
+ for (i = param_max_sched_ready_insns + ready.n_debug; i < ready.n_ready;
+ i++)
if (!SCHED_GROUP_P (ready_element (&ready, i)))
break;
if (sched_verbose >= 2)
{
fprintf (sched_dump,
- ";;\t\tReady list on entry: %d insns\n", ready.n_ready);
+ ";;\t\tReady list on entry: %d insns: ", ready.n_ready);
+ debug_ready_list (&ready);
fprintf (sched_dump,
";;\t\t before reload => truncated to %d insns\n", i);
}
{
memcpy (temp_state, curr_state, dfa_state_size);
cost = state_transition (curr_state, insn);
- if (sched_pressure != SCHED_PRESSURE_WEIGHTED)
+ if (sched_pressure != SCHED_PRESSURE_WEIGHTED && !sched_fusion)
gcc_assert (cost < 0);
if (memcmp (temp_state, curr_state, dfa_state_size) != 0)
cycle_issued_insns++;
return n_insn;
}
-/* Set dump and sched_verbose for the desired debugging output. If no
- dump-file was specified, but -fsched-verbose=N (any N), print to stderr.
- For -fsched-verbose=N, N>=10, print everything to stderr. */
+/* Set sched_dump and sched_verbose for the desired debugging output. */
void
setup_sched_dump (void)
{
sched_verbose = sched_verbose_param;
- if (sched_verbose_param == 0 && dump_file)
- sched_verbose = 1;
- sched_dump = ((sched_verbose_param >= 10 || !dump_file)
- ? stderr : dump_file);
+ sched_dump = dump_file;
+ if (!dump_file)
+ sched_verbose = 0;
}
/* Allocate data for register pressure sensitive scheduling. */
saved_reg_live = BITMAP_ALLOC (NULL);
region_ref_regs = BITMAP_ALLOC (NULL);
}
+ if (sched_pressure == SCHED_PRESSURE_MODEL)
+ tmp_bitmap = BITMAP_ALLOC (NULL);
- /* Calculate number of CALL_USED_REGS in register classes that
- we calculate register pressure for. */
+ /* Calculate number of CALL_SAVED_REGS and FIXED_REGS in register classes
+ that we calculate register pressure for. */
for (int c = 0; c < ira_pressure_classes_num; ++c)
{
enum reg_class cl = ira_pressure_classes[c];
- call_used_regs_num[cl] = 0;
+ call_saved_regs_num[cl] = 0;
+ fixed_regs_num[cl] = 0;
for (int i = 0; i < ira_class_hard_regs_num[cl]; ++i)
- if (call_used_regs[ira_class_hard_regs[cl][i]])
- ++call_used_regs_num[cl];
+ {
+ unsigned int regno = ira_class_hard_regs[cl][i];
+ if (fixed_regs[regno])
+ ++fixed_regs_num[cl];
+ else if (!crtl->abi->clobbers_full_reg_p (regno))
+ ++call_saved_regs_num[cl];
+ }
}
}
}
BITMAP_FREE (region_ref_regs);
BITMAP_FREE (saved_reg_live);
}
+ if (sched_pressure == SCHED_PRESSURE_MODEL)
+ BITMAP_FREE (tmp_bitmap);
BITMAP_FREE (curr_reg_live);
free (sched_regno_pressure_class);
}
sched_init (void)
{
/* Disable speculative loads in their presence if cc0 defined. */
-#ifdef HAVE_cc0
+ if (HAVE_cc0)
flag_schedule_speculative_load = 0;
-#endif
if (targetm.sched.dispatch (NULL, IS_DISPATCH_ON))
targetm.sched.dispatch_do (NULL, DISPATCH_INIT);
&& !reload_completed
&& common_sched_info->sched_pass_id == SCHED_RGN_PASS)
sched_pressure = ((enum sched_pressure_algorithm)
- PARAM_VALUE (PARAM_SCHED_PRESSURE_ALGORITHM));
+ param_sched_pressure_algorithm);
else
sched_pressure = SCHED_PRESSURE_NONE;
if (spec_info->mask != 0)
{
- spec_info->data_weakness_cutoff =
- (PARAM_VALUE (PARAM_SCHED_SPEC_PROB_CUTOFF) * MAX_DEP_WEAK) / 100;
- spec_info->control_weakness_cutoff =
- (PARAM_VALUE (PARAM_SCHED_SPEC_PROB_CUTOFF)
- * REG_BR_PROB_BASE) / 100;
+ spec_info->data_weakness_cutoff
+ = (param_sched_spec_prob_cutoff * MAX_DEP_WEAK) / 100;
+ spec_info->control_weakness_cutoff
+ = (param_sched_spec_prob_cutoff * REG_BR_PROB_BASE) / 100;
}
else
/* So we won't read anything accidentally. */
else
issue_rate = 1;
- if (targetm.sched.first_cycle_multipass_dfa_lookahead)
+ if (targetm.sched.first_cycle_multipass_dfa_lookahead
+ /* Don't use max_issue with reg_pressure scheduling. Multipass
+ scheduling and reg_pressure scheduling undo each other's decisions. */
+ && sched_pressure == SCHED_PRESSURE_NONE)
dfa_lookahead = targetm.sched.first_cycle_multipass_dfa_lookahead ();
else
dfa_lookahead = 0;
/* Initialize luids, dependency caches, target and h_i_d for the
whole function. */
{
- bb_vec_t bbs;
- bbs.create (n_basic_blocks_for_fn (cfun));
- basic_block bb;
-
sched_init_bbs ();
+ auto_vec<basic_block> bbs (n_basic_blocks_for_fn (cfun));
+ basic_block bb;
FOR_EACH_BB_FN (bb, cfun)
bbs.quick_push (bb);
sched_init_luids (bbs);
sched_deps_init (true);
sched_extend_target ();
haifa_init_h_i_d (bbs);
-
- bbs.release ();
}
sched_init_only_bb = haifa_init_only_bb;
sched_deps_finish ();
sched_finish_luids ();
current_sched_info = NULL;
+ insn_queue = NULL;
sched_finish ();
}
fix_inter_tick (rtx_insn *head, rtx_insn *tail)
{
/* Set of instructions with corrected INSN_TICK. */
- bitmap_head processed;
+ auto_bitmap processed;
/* ??? It is doubtful if we should assume that cycle advance happens on
basic block boundaries. Basically insns that are unconditionally ready
on the start of the block are more preferable then those which have
a one cycle dependency over insn from the previous block. */
int next_clock = clock_var + 1;
- bitmap_initialize (&processed, 0);
-
/* Iterates over scheduled instructions and fix their INSN_TICKs and
INSN_TICKs of dependent instructions, so that INSN_TICKs are consistent
across different blocks. */
gcc_assert (tick >= MIN_TICK);
/* Fix INSN_TICK of instruction from just scheduled block. */
- if (bitmap_set_bit (&processed, INSN_LUID (head)))
+ if (bitmap_set_bit (processed, INSN_LUID (head)))
{
tick -= next_clock;
/* If NEXT has its INSN_TICK calculated, fix it.
If not - it will be properly calculated from
scratch later in fix_tick_ready. */
- && bitmap_set_bit (&processed, INSN_LUID (next)))
+ && bitmap_set_bit (processed, INSN_LUID (next)))
{
tick -= next_clock;
}
}
}
- bitmap_clear (&processed);
}
/* Check if NEXT is ready to be added to the ready or queue list.
INSN_TICK (next) = tick;
delay = tick - clock_var;
- if (delay <= 0 || sched_pressure != SCHED_PRESSURE_NONE)
+ if (delay <= 0 || sched_pressure != SCHED_PRESSURE_NONE || sched_fusion)
delay = QUEUE_READY;
change_queue_index (next, delay);
Tries to add speculative dependencies of type FS between instructions
in deps_list L and TWIN. */
static void
-process_insn_forw_deps_be_in_spec (rtx insn, rtx_insn *twin, ds_t fs)
+process_insn_forw_deps_be_in_spec (rtx_insn *insn, rtx_insn *twin, ds_t fs)
{
sd_iterator_def sd_it;
dep_t dep;
ds_t ts;
sd_iterator_def sd_it;
dep_t dep;
- rtx_insn_list *twins = NULL;
- rtx_vec_t priorities_roots;
+ auto_vec<rtx_insn *, 10> twins;
ts = TODO_SPEC (insn);
gcc_assert (!(ts & ~BE_IN_SPEC));
sd_iterator_next (&sd_it);
}
- priorities_roots.create (0);
+ auto_vec<rtx_insn *> priorities_roots;
clear_priorities (insn, &priorities_roots);
while (1)
fprintf (spec_info->dump, ";;\t\tGenerated twin insn : %d/rec%d\n",
INSN_UID (twin), rec->index);
- twins = alloc_INSN_LIST (twin, twins);
+ twins.safe_push (twin);
/* Add dependences between TWIN and all appropriate
instructions from REC. */
/* We couldn't have added the dependencies between INSN and TWINS earlier
because that would make TWINS appear in the INSN_BACK_DEPS (INSN). */
- while (twins)
+ unsigned int i;
+ rtx_insn *twin;
+ FOR_EACH_VEC_ELT_REVERSE (twins, i, twin)
{
- rtx_insn *twin;
- rtx_insn_list *next_node;
+ dep_def _new_dep, *new_dep = &_new_dep;
- twin = twins->insn ();
-
- {
- dep_def _new_dep, *new_dep = &_new_dep;
-
- init_dep (new_dep, insn, twin, REG_DEP_OUTPUT);
- sd_add_dep (new_dep, false);
- }
-
- next_node = twins->next ();
- free_INSN_LIST_node (twins);
- twins = next_node;
+ init_dep (new_dep, insn, twin, REG_DEP_OUTPUT);
+ sd_add_dep (new_dep, false);
}
calc_priorities (priorities_roots);
- priorities_roots.release ();
}
/* Extends and fills with zeros (only the new part) array pointed to by P. */
if (e)
{
- gcc_assert (e->dest == succ);
+ gcc_assert (e->dest == succ || e->dest->index == EXIT_BLOCK);
return e;
}
}
|| (!NOTE_P (insn)
&& !LABEL_P (insn)
/* Don't emit a NOTE if it would end up before a BARRIER. */
- && !BARRIER_P (NEXT_INSN (end))))
+ && !BARRIER_P (next_nondebug_insn (end))))
{
rtx_note *note = emit_note_after (NOTE_INSN_DELETED, end);
/* Make note appear outside BB. */
Between these two blocks recovery blocks will be emitted. */
basic_block single, empty;
- rtx_insn *x;
- rtx label;
/* If the fallthrough edge to exit we've found is from the block we've
created before, don't do anything more. */
single->count = last->count;
empty->count = last->count;
- single->frequency = last->frequency;
- empty->frequency = last->frequency;
BB_COPY_PARTITION (single, last);
BB_COPY_PARTITION (empty, last);
make_single_succ_edge (empty, EXIT_BLOCK_PTR_FOR_FN (cfun),
EDGE_FALLTHRU);
- label = block_label (empty);
- x = emit_jump_insn_after (gen_jump (label), BB_END (single));
+ rtx_code_label *label = block_label (empty);
+ rtx_jump_insn *x = emit_jump_insn_after (targetm.gen_jump (label),
+ BB_END (single));
JUMP_LABEL (x) = label;
LABEL_NUSES (label)++;
haifa_init_insn (x);
basic_block
sched_create_recovery_block (basic_block *before_recovery_ptr)
{
- rtx label;
rtx_insn *barrier;
basic_block rec;
barrier = get_last_bb_insn (before_recovery);
gcc_assert (BARRIER_P (barrier));
- label = emit_label_after (gen_label_rtx (), barrier);
+ rtx_insn *label = emit_label_after (gen_label_rtx (), barrier);
rec = create_basic_block (label, label, before_recovery);
sched_create_recovery_edges (basic_block first_bb, basic_block rec,
basic_block second_bb)
{
- rtx label;
- rtx jump;
int edge_flags;
/* This is fixing of incoming edge. */
else
edge_flags = 0;
- make_edge (first_bb, rec, edge_flags);
- label = block_label (second_bb);
- jump = emit_jump_insn_after (gen_jump (label), BB_END (rec));
+ edge e2 = single_succ_edge (first_bb);
+ edge e = make_edge (first_bb, rec, edge_flags);
+
+ /* TODO: The actual probability can be determined and is computed as
+ 'todo_spec' variable in create_check_block_twin and
+ in sel-sched.c `check_ds' in create_speculation_check. */
+ e->probability = profile_probability::very_unlikely ();
+ rec->count = e->count ();
+ e2->probability = e->probability.invert ();
+
+ rtx_code_label *label = block_label (second_bb);
+ rtx_jump_insn *jump = emit_jump_insn_after (targetm.gen_jump (label),
+ BB_END (rec));
JUMP_LABEL (jump) = label;
LABEL_NUSES (label)++;
/* Partition type is the same, if it is "unpartitioned". */
{
/* Rewritten from cfgrtl.c. */
- if (flag_reorder_blocks_and_partition
- && targetm_common.have_named_sections)
+ if (crtl->has_bb_partition && targetm_common.have_named_sections)
{
/* We don't need the same note for the check because
any_condjump_p (check) == true. */
/* Fix priorities. If MUTATE_P is nonzero, this is not necessary,
because it'll be done later in add_to_speculative_block. */
{
- rtx_vec_t priorities_roots = rtx_vec_t ();
+ auto_vec<rtx_insn *> priorities_roots;
clear_priorities (twin, &priorities_roots);
calc_priorities (priorities_roots);
- priorities_roots.release ();
}
}
fix_recovery_deps (basic_block rec)
{
rtx_insn *note, *insn, *jump;
- rtx_insn_list *ready_list = 0;
- bitmap_head in_ready;
- rtx_insn_list *link;
-
- bitmap_initialize (&in_ready, 0);
+ auto_vec<rtx_insn *, 10> ready_list;
+ auto_bitmap in_ready;
/* NOTE - a basic block note. */
note = NEXT_INSN (BB_HEAD (rec));
{
sd_delete_dep (sd_it);
- if (bitmap_set_bit (&in_ready, INSN_LUID (consumer)))
- ready_list = alloc_INSN_LIST (consumer, ready_list);
+ if (bitmap_set_bit (in_ready, INSN_LUID (consumer)))
+ ready_list.safe_push (consumer);
}
else
{
}
while (insn != note);
- bitmap_clear (&in_ready);
-
/* Try to add instructions to the ready or queue list. */
- for (link = ready_list; link; link = link->next ())
- try_ready (link->insn ());
- free_INSN_LIST_list (&ready_list);
+ unsigned int i;
+ rtx_insn *temp;
+ FOR_EACH_VEC_ELT_REVERSE (ready_list, i, temp)
+ try_ready (temp);
/* Fixing jump's dependences. */
insn = BB_HEAD (rec);
INSN_EXACT_TICK (insn) = INVALID_TICK;
INTER_TICK (insn) = INVALID_TICK;
TODO_SPEC (insn) = HARD_DEP;
+ INSN_AUTOPREF_MULTIPASS_DATA (insn)[0].status
+ = AUTOPREF_MULTIPASS_DATA_UNINITIALIZED;
+ INSN_AUTOPREF_MULTIPASS_DATA (insn)[1].status
+ = AUTOPREF_MULTIPASS_DATA_UNINITIALIZED;
}
}
{
int i;
haifa_insn_data_t data;
- struct reg_use_data *use, *next;
+ reg_use_data *use, *next_use;
+ reg_set_data *set, *next_set;
FOR_EACH_VEC_ELT (h_i_d, i, data)
{
free (data->max_reg_pressure);
free (data->reg_pressure);
- for (use = data->reg_use_list; use != NULL; use = next)
+ for (use = data->reg_use_list; use != NULL; use = next_use)
{
- next = use->next_insn_use;
+ next_use = use->next_insn_use;
free (use);
}
+ for (set = data->reg_set_list; set != NULL; set = next_set)
+ {
+ next_set = set->next_insn_set;
+ free (set);
+ }
+
}
h_i_d.release ();
}
projects / thirdparty / gcc.git / blobdiff