1 /* Instruction scheduling pass.
2 Copyright (C) 1992-2015 Free Software Foundation, Inc.
3 Contributed by Michael Tiemann (tiemann@cygnus.com) Enhanced by,
4 and currently maintained by, Jim Wilson (wilson@cygnus.com)
6 This file is part of GCC.
8 GCC is free software; you can redistribute it and/or modify it under
9 the terms of the GNU General Public License as published by the Free
10 Software Foundation; either version 3, or (at your option) any later
13 GCC is distributed in the hope that it will be useful, but WITHOUT ANY
14 WARRANTY; without even the implied warranty of MERCHANTABILITY or
15 FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
18 You should have received a copy of the GNU General Public License
19 along with GCC; see the file COPYING3. If not see
20 <http://www.gnu.org/licenses/>. */
22 /* Instruction scheduling pass. This file, along with sched-deps.c,
23 contains the generic parts. The actual entry point for
24 the normal instruction scheduling pass is found in sched-rgn.c.
26 We compute insn priorities based on data dependencies. Flow
27 analysis only creates a fraction of the data-dependencies we must
28 observe: namely, only those dependencies which the combiner can be
29 expected to use. For this pass, we must therefore create the
30 remaining dependencies we need to observe: register dependencies,
31 memory dependencies, dependencies to keep function calls in order,
32 and the dependence between a conditional branch and the setting of
33 condition codes are all dealt with here.
35 The scheduler first traverses the data flow graph, starting with
36 the last instruction, and proceeding to the first, assigning values
37 to insn_priority as it goes. This sorts the instructions
38 topologically by data dependence.
40 Once priorities have been established, we order the insns using
41 list scheduling. This works as follows: starting with a list of
42 all the ready insns, and sorted according to priority number, we
43 schedule the insn from the end of the list by placing its
44 predecessors in the list according to their priority order. We
45 consider this insn scheduled by setting the pointer to the "end" of
46 the list to point to the previous insn. When an insn has no
47 predecessors, we either queue it until sufficient time has elapsed
48 or add it to the ready list. As the instructions are scheduled or
49 when stalls are introduced, the queue advances and dumps insns into
50 the ready list. When all insns down to the lowest priority have
51 been scheduled, the critical path of the basic block has been made
52 as short as possible. The remaining insns are then scheduled in
55 The following list shows the order in which we want to break ties
56 among insns in the ready list:
58 1. choose insn with the longest path to end of bb, ties
60 2. choose insn with least contribution to register pressure,
62 3. prefer in-block upon interblock motion, ties broken by
63 4. prefer useful upon speculative motion, ties broken by
64 5. choose insn with largest control flow probability, ties
66 6. choose insn with the least dependences upon the previously
67 scheduled insn, or finally
68 7 choose the insn which has the most insns dependent on it.
69 8. choose insn with lowest UID.
71 Memory references complicate matters. Only if we can be certain
72 that memory references are not part of the data dependency graph
73 (via true, anti, or output dependence), can we move operations past
74 memory references. To first approximation, reads can be done
75 independently, while writes introduce dependencies. Better
76 approximations will yield fewer dependencies.
78 Before reload, an extended analysis of interblock data dependences
79 is required for interblock scheduling. This is performed in
80 compute_block_dependences ().
82 Dependencies set up by memory references are treated in exactly the
83 same way as other dependencies, by using insn backward dependences
84 INSN_BACK_DEPS. INSN_BACK_DEPS are translated into forward dependences
85 INSN_FORW_DEPS for the purpose of forward list scheduling.
87 Having optimized the critical path, we may have also unduly
88 extended the lifetimes of some registers. If an operation requires
89 that constants be loaded into registers, it is certainly desirable
90 to load those constants as early as necessary, but no earlier.
91 I.e., it will not do to load up a bunch of registers at the
92 beginning of a basic block only to use them at the end, if they
93 could be loaded later, since this may result in excessive register
96 Note that since branches are never in basic blocks, but only end
97 basic blocks, this pass will not move branches. But that is ok,
98 since we can use GNU's delayed branch scheduling pass to take care
101 Also note that no further optimizations based on algebraic
102 identities are performed, so this pass would be a good one to
103 perform instruction splitting, such as breaking up a multiply
104 instruction into shifts and adds where that is profitable.
106 Given the memory aliasing analysis that this pass should perform,
107 it should be possible to remove redundant stores to memory, and to
108 load values from registers instead of hitting memory.
110 Before reload, speculative insns are moved only if a 'proof' exists
111 that no exception will be caused by this, and if no live registers
112 exist that inhibit the motion (live registers constraints are not
113 represented by data dependence edges).
115 This pass must update information that subsequent passes expect to
116 be correct. Namely: reg_n_refs, reg_n_sets, reg_n_deaths,
117 reg_n_calls_crossed, and reg_live_length. Also, BB_HEAD, BB_END.
119 The information in the line number notes is carefully retained by
120 this pass. Notes that refer to the starting and ending of
121 exception regions are also carefully retained by this pass. All
122 other NOTE insns are grouped in their same relative order at the
123 beginning of basic blocks and regions that have been scheduled. */
127 #include "coretypes.h"
131 #include "diagnostic-core.h"
135 #include "insn-config.h"
136 #include "insn-attr.h"
140 #include "cfgbuild.h"
141 #include "sched-int.h"
143 #include "common/common-target.h"
148 #include "emit-rtl.h" /* FIXME: Can go away once crtl is moved to rtl.h. */
149 #include "dumpfile.h"
151 #ifdef INSN_SCHEDULING
153 /* True if we do register pressure relief through live-range
155 static bool live_range_shrinkage_p
;
157 /* Switch on live range shrinkage. */
159 initialize_live_range_shrinkage (void)
161 live_range_shrinkage_p
= true;
164 /* Switch off live range shrinkage. */
166 finish_live_range_shrinkage (void)
168 live_range_shrinkage_p
= false;
171 /* issue_rate is the number of insns that can be scheduled in the same
172 machine cycle. It can be defined in the config/mach/mach.h file,
173 otherwise we set it to 1. */
177 /* This can be set to true by a backend if the scheduler should not
178 enable a DCE pass. */
181 /* The current initiation interval used when modulo scheduling. */
182 static int modulo_ii
;
184 /* The maximum number of stages we are prepared to handle. */
185 static int modulo_max_stages
;
187 /* The number of insns that exist in each iteration of the loop. We use this
188 to detect when we've scheduled all insns from the first iteration. */
189 static int modulo_n_insns
;
191 /* The current count of insns in the first iteration of the loop that have
192 already been scheduled. */
193 static int modulo_insns_scheduled
;
195 /* The maximum uid of insns from the first iteration of the loop. */
196 static int modulo_iter0_max_uid
;
198 /* The number of times we should attempt to backtrack when modulo scheduling.
199 Decreased each time we have to backtrack. */
200 static int modulo_backtracks_left
;
202 /* The stage in which the last insn from the original loop was
204 static int modulo_last_stage
;
206 /* sched-verbose controls the amount of debugging output the
207 scheduler prints. It is controlled by -fsched-verbose=N:
208 N>0 and no -DSR : the output is directed to stderr.
209 N>=10 will direct the printouts to stderr (regardless of -dSR).
211 N=2: bb's probabilities, detailed ready list info, unit/insn info.
212 N=3: rtl at abort point, control-flow, regions info.
213 N=5: dependences info. */
215 int sched_verbose
= 0;
217 /* Debugging file. All printouts are sent to dump, which is always set,
218 either to stderr, or to the dump listing file (-dRS). */
219 FILE *sched_dump
= 0;
221 /* This is a placeholder for the scheduler parameters common
222 to all schedulers. */
223 struct common_sched_info_def
*common_sched_info
;
225 #define INSN_TICK(INSN) (HID (INSN)->tick)
226 #define INSN_EXACT_TICK(INSN) (HID (INSN)->exact_tick)
227 #define INSN_TICK_ESTIMATE(INSN) (HID (INSN)->tick_estimate)
228 #define INTER_TICK(INSN) (HID (INSN)->inter_tick)
229 #define FEEDS_BACKTRACK_INSN(INSN) (HID (INSN)->feeds_backtrack_insn)
230 #define SHADOW_P(INSN) (HID (INSN)->shadow_p)
231 #define MUST_RECOMPUTE_SPEC_P(INSN) (HID (INSN)->must_recompute_spec)
232 /* Cached cost of the instruction. Use insn_cost to get cost of the
233 insn. -1 here means that the field is not initialized. */
234 #define INSN_COST(INSN) (HID (INSN)->cost)
236 /* If INSN_TICK of an instruction is equal to INVALID_TICK,
237 then it should be recalculated from scratch. */
238 #define INVALID_TICK (-(max_insn_queue_index + 1))
239 /* The minimal value of the INSN_TICK of an instruction. */
240 #define MIN_TICK (-max_insn_queue_index)
242 /* Original order of insns in the ready list.
243 Used to keep order of normal insns while separating DEBUG_INSNs. */
244 #define INSN_RFS_DEBUG_ORIG_ORDER(INSN) (HID (INSN)->rfs_debug_orig_order)
246 /* The deciding reason for INSN's place in the ready list. */
247 #define INSN_LAST_RFS_WIN(INSN) (HID (INSN)->last_rfs_win)
249 /* List of important notes we must keep around. This is a pointer to the
250 last element in the list. */
253 static struct spec_info_def spec_info_var
;
254 /* Description of the speculative part of the scheduling.
255 If NULL - no speculation. */
256 spec_info_t spec_info
= NULL
;
258 /* True, if recovery block was added during scheduling of current block.
259 Used to determine, if we need to fix INSN_TICKs. */
260 static bool haifa_recovery_bb_recently_added_p
;
262 /* True, if recovery block was added during this scheduling pass.
263 Used to determine if we should have empty memory pools of dependencies
264 after finishing current region. */
265 bool haifa_recovery_bb_ever_added_p
;
267 /* Counters of different types of speculative instructions. */
268 static int nr_begin_data
, nr_be_in_data
, nr_begin_control
, nr_be_in_control
;
270 /* Array used in {unlink, restore}_bb_notes. */
271 static rtx_insn
**bb_header
= 0;
273 /* Basic block after which recovery blocks will be created. */
274 static basic_block before_recovery
;
276 /* Basic block just before the EXIT_BLOCK and after recovery, if we have
278 basic_block after_recovery
;
280 /* FALSE if we add bb to another region, so we don't need to initialize it. */
281 bool adding_bb_to_current_region_p
= true;
285 /* An instruction is ready to be scheduled when all insns preceding it
286 have already been scheduled. It is important to ensure that all
287 insns which use its result will not be executed until its result
288 has been computed. An insn is maintained in one of four structures:
290 (P) the "Pending" set of insns which cannot be scheduled until
291 their dependencies have been satisfied.
292 (Q) the "Queued" set of insns that can be scheduled when sufficient
294 (R) the "Ready" list of unscheduled, uncommitted insns.
295 (S) the "Scheduled" list of insns.
297 Initially, all insns are either "Pending" or "Ready" depending on
298 whether their dependencies are satisfied.
300 Insns move from the "Ready" list to the "Scheduled" list as they
301 are committed to the schedule. As this occurs, the insns in the
302 "Pending" list have their dependencies satisfied and move to either
303 the "Ready" list or the "Queued" set depending on whether
304 sufficient time has passed to make them ready. As time passes,
305 insns move from the "Queued" set to the "Ready" list.
307 The "Pending" list (P) are the insns in the INSN_FORW_DEPS of the
308 unscheduled insns, i.e., those that are ready, queued, and pending.
309 The "Queued" set (Q) is implemented by the variable `insn_queue'.
310 The "Ready" list (R) is implemented by the variables `ready' and
312 The "Scheduled" list (S) is the new insn chain built by this pass.
314 The transition (R->S) is implemented in the scheduling loop in
315 `schedule_block' when the best insn to schedule is chosen.
316 The transitions (P->R and P->Q) are implemented in `schedule_insn' as
317 insns move from the ready list to the scheduled list.
318 The transition (Q->R) is implemented in 'queue_to_insn' as time
319 passes or stalls are introduced. */
321 /* Implement a circular buffer to delay instructions until sufficient
322 time has passed. For the new pipeline description interface,
323 MAX_INSN_QUEUE_INDEX is a power of two minus one which is not less
324 than maximal time of instruction execution computed by genattr.c on
325 the base maximal time of functional unit reservations and getting a
326 result. This is the longest time an insn may be queued. */
328 static rtx_insn_list
**insn_queue
;
329 static int q_ptr
= 0;
330 static int q_size
= 0;
331 #define NEXT_Q(X) (((X)+1) & max_insn_queue_index)
332 #define NEXT_Q_AFTER(X, C) (((X)+C) & max_insn_queue_index)
334 #define QUEUE_SCHEDULED (-3)
335 #define QUEUE_NOWHERE (-2)
336 #define QUEUE_READY (-1)
337 /* QUEUE_SCHEDULED - INSN is scheduled.
338 QUEUE_NOWHERE - INSN isn't scheduled yet and is neither in
340 QUEUE_READY - INSN is in ready list.
341 N >= 0 - INSN queued for X [where NEXT_Q_AFTER (q_ptr, X) == N] cycles. */
343 #define QUEUE_INDEX(INSN) (HID (INSN)->queue_index)
345 /* The following variable value refers for all current and future
346 reservations of the processor units. */
349 /* The following variable value is size of memory representing all
350 current and future reservations of the processor units. */
351 size_t dfa_state_size
;
353 /* The following array is used to find the best insn from ready when
354 the automaton pipeline interface is used. */
355 signed char *ready_try
= NULL
;
357 /* The ready list. */
358 struct ready_list ready
= {NULL
, 0, 0, 0, 0};
360 /* The pointer to the ready list (to be removed). */
361 static struct ready_list
*readyp
= &ready
;
363 /* Scheduling clock. */
364 static int clock_var
;
366 /* Clock at which the previous instruction was issued. */
367 static int last_clock_var
;
369 /* Set to true if, when queuing a shadow insn, we discover that it would be
370 scheduled too late. */
371 static bool must_backtrack
;
373 /* The following variable value is number of essential insns issued on
374 the current cycle. An insn is essential one if it changes the
376 int cycle_issued_insns
;
378 /* This records the actual schedule. It is built up during the main phase
379 of schedule_block, and afterwards used to reorder the insns in the RTL. */
380 static vec
<rtx_insn
*> scheduled_insns
;
382 static int may_trap_exp (const_rtx
, int);
384 /* Nonzero iff the address is comprised from at most 1 register. */
385 #define CONST_BASED_ADDRESS_P(x) \
387 || ((GET_CODE (x) == PLUS || GET_CODE (x) == MINUS \
388 || (GET_CODE (x) == LO_SUM)) \
389 && (CONSTANT_P (XEXP (x, 0)) \
390 || CONSTANT_P (XEXP (x, 1)))))
392 /* Returns a class that insn with GET_DEST(insn)=x may belong to,
393 as found by analyzing insn's expression. */
396 static int haifa_luid_for_non_insn (rtx x
);
398 /* Haifa version of sched_info hooks common to all headers. */
399 const struct common_sched_info_def haifa_common_sched_info
=
401 NULL
, /* fix_recovery_cfg */
402 NULL
, /* add_block */
403 NULL
, /* estimate_number_of_insns */
404 haifa_luid_for_non_insn
, /* luid_for_non_insn */
405 SCHED_PASS_UNKNOWN
/* sched_pass_id */
408 /* Mapping from instruction UID to its Logical UID. */
409 vec
<int> sched_luids
= vNULL
;
411 /* Next LUID to assign to an instruction. */
412 int sched_max_luid
= 1;
414 /* Haifa Instruction Data. */
415 vec
<haifa_insn_data_def
> h_i_d
= vNULL
;
417 void (* sched_init_only_bb
) (basic_block
, basic_block
);
419 /* Split block function. Different schedulers might use different functions
420 to handle their internal data consistent. */
421 basic_block (* sched_split_block
) (basic_block
, rtx
);
423 /* Create empty basic block after the specified block. */
424 basic_block (* sched_create_empty_bb
) (basic_block
);
426 /* Return the number of cycles until INSN is expected to be ready.
427 Return zero if it already is. */
429 insn_delay (rtx_insn
*insn
)
431 return MAX (INSN_TICK (insn
) - clock_var
, 0);
435 may_trap_exp (const_rtx x
, int is_store
)
444 if (code
== MEM
&& may_trap_p (x
))
451 /* The insn uses memory: a volatile load. */
452 if (MEM_VOLATILE_P (x
))
454 /* An exception-free load. */
457 /* A load with 1 base register, to be further checked. */
458 if (CONST_BASED_ADDRESS_P (XEXP (x
, 0)))
459 return PFREE_CANDIDATE
;
460 /* No info on the load, to be further checked. */
461 return PRISKY_CANDIDATE
;
466 int i
, insn_class
= TRAP_FREE
;
468 /* Neither store nor load, check if it may cause a trap. */
471 /* Recursive step: walk the insn... */
472 fmt
= GET_RTX_FORMAT (code
);
473 for (i
= GET_RTX_LENGTH (code
) - 1; i
>= 0; i
--)
477 int tmp_class
= may_trap_exp (XEXP (x
, i
), is_store
);
478 insn_class
= WORST_CLASS (insn_class
, tmp_class
);
480 else if (fmt
[i
] == 'E')
483 for (j
= 0; j
< XVECLEN (x
, i
); j
++)
485 int tmp_class
= may_trap_exp (XVECEXP (x
, i
, j
), is_store
);
486 insn_class
= WORST_CLASS (insn_class
, tmp_class
);
487 if (insn_class
== TRAP_RISKY
|| insn_class
== IRISKY
)
491 if (insn_class
== TRAP_RISKY
|| insn_class
== IRISKY
)
498 /* Classifies rtx X of an insn for the purpose of verifying that X can be
499 executed speculatively (and consequently the insn can be moved
500 speculatively), by examining X, returning:
501 TRAP_RISKY: store, or risky non-load insn (e.g. division by variable).
502 TRAP_FREE: non-load insn.
503 IFREE: load from a globally safe location.
504 IRISKY: volatile load.
505 PFREE_CANDIDATE, PRISKY_CANDIDATE: load that need to be checked for
506 being either PFREE or PRISKY. */
509 haifa_classify_rtx (const_rtx x
)
511 int tmp_class
= TRAP_FREE
;
512 int insn_class
= TRAP_FREE
;
515 if (GET_CODE (x
) == PARALLEL
)
517 int i
, len
= XVECLEN (x
, 0);
519 for (i
= len
- 1; i
>= 0; i
--)
521 tmp_class
= haifa_classify_rtx (XVECEXP (x
, 0, i
));
522 insn_class
= WORST_CLASS (insn_class
, tmp_class
);
523 if (insn_class
== TRAP_RISKY
|| insn_class
== IRISKY
)
533 /* Test if it is a 'store'. */
534 tmp_class
= may_trap_exp (XEXP (x
, 0), 1);
537 /* Test if it is a store. */
538 tmp_class
= may_trap_exp (SET_DEST (x
), 1);
539 if (tmp_class
== TRAP_RISKY
)
541 /* Test if it is a load. */
543 WORST_CLASS (tmp_class
,
544 may_trap_exp (SET_SRC (x
), 0));
547 tmp_class
= haifa_classify_rtx (COND_EXEC_CODE (x
));
548 if (tmp_class
== TRAP_RISKY
)
550 tmp_class
= WORST_CLASS (tmp_class
,
551 may_trap_exp (COND_EXEC_TEST (x
), 0));
554 tmp_class
= TRAP_RISKY
;
558 insn_class
= tmp_class
;
565 haifa_classify_insn (const_rtx insn
)
567 return haifa_classify_rtx (PATTERN (insn
));
570 /* After the scheduler initialization function has been called, this function
571 can be called to enable modulo scheduling. II is the initiation interval
572 we should use, it affects the delays for delay_pairs that were recorded as
573 separated by a given number of stages.
575 MAX_STAGES provides us with a limit
576 after which we give up scheduling; the caller must have unrolled at least
577 as many copies of the loop body and recorded delay_pairs for them.
579 INSNS is the number of real (non-debug) insns in one iteration of
580 the loop. MAX_UID can be used to test whether an insn belongs to
581 the first iteration of the loop; all of them have a uid lower than
584 set_modulo_params (int ii
, int max_stages
, int insns
, int max_uid
)
587 modulo_max_stages
= max_stages
;
588 modulo_n_insns
= insns
;
589 modulo_iter0_max_uid
= max_uid
;
590 modulo_backtracks_left
= PARAM_VALUE (PARAM_MAX_MODULO_BACKTRACK_ATTEMPTS
);
593 /* A structure to record a pair of insns where the first one is a real
594 insn that has delay slots, and the second is its delayed shadow.
595 I1 is scheduled normally and will emit an assembly instruction,
596 while I2 describes the side effect that takes place at the
597 transition between cycles CYCLES and (CYCLES + 1) after I1. */
600 struct delay_pair
*next_same_i1
;
603 /* When doing modulo scheduling, we a delay_pair can also be used to
604 show that I1 and I2 are the same insn in a different stage. If that
605 is the case, STAGES will be nonzero. */
609 /* Helpers for delay hashing. */
611 struct delay_i1_hasher
: nofree_ptr_hash
<delay_pair
>
613 typedef void *compare_type
;
614 static inline hashval_t
hash (const delay_pair
*);
615 static inline bool equal (const delay_pair
*, const void *);
618 /* Returns a hash value for X, based on hashing just I1. */
621 delay_i1_hasher::hash (const delay_pair
*x
)
623 return htab_hash_pointer (x
->i1
);
626 /* Return true if I1 of pair X is the same as that of pair Y. */
629 delay_i1_hasher::equal (const delay_pair
*x
, const void *y
)
634 struct delay_i2_hasher
: free_ptr_hash
<delay_pair
>
636 typedef void *compare_type
;
637 static inline hashval_t
hash (const delay_pair
*);
638 static inline bool equal (const delay_pair
*, const void *);
641 /* Returns a hash value for X, based on hashing just I2. */
644 delay_i2_hasher::hash (const delay_pair
*x
)
646 return htab_hash_pointer (x
->i2
);
649 /* Return true if I2 of pair X is the same as that of pair Y. */
652 delay_i2_hasher::equal (const delay_pair
*x
, const void *y
)
657 /* Two hash tables to record delay_pairs, one indexed by I1 and the other
659 static hash_table
<delay_i1_hasher
> *delay_htab
;
660 static hash_table
<delay_i2_hasher
> *delay_htab_i2
;
662 /* Called through htab_traverse. Walk the hashtable using I2 as
663 index, and delete all elements involving an UID higher than
664 that pointed to by *DATA. */
666 haifa_htab_i2_traverse (delay_pair
**slot
, int *data
)
669 struct delay_pair
*p
= *slot
;
670 if (INSN_UID (p
->i2
) >= maxuid
|| INSN_UID (p
->i1
) >= maxuid
)
672 delay_htab_i2
->clear_slot (slot
);
677 /* Called through htab_traverse. Walk the hashtable using I2 as
678 index, and delete all elements involving an UID higher than
679 that pointed to by *DATA. */
681 haifa_htab_i1_traverse (delay_pair
**pslot
, int *data
)
684 struct delay_pair
*p
, *first
, **pprev
;
686 if (INSN_UID ((*pslot
)->i1
) >= maxuid
)
688 delay_htab
->clear_slot (pslot
);
692 for (p
= *pslot
; p
; p
= p
->next_same_i1
)
694 if (INSN_UID (p
->i2
) < maxuid
)
697 pprev
= &p
->next_same_i1
;
702 delay_htab
->clear_slot (pslot
);
708 /* Discard all delay pairs which involve an insn with an UID higher
711 discard_delay_pairs_above (int max_uid
)
713 delay_htab
->traverse
<int *, haifa_htab_i1_traverse
> (&max_uid
);
714 delay_htab_i2
->traverse
<int *, haifa_htab_i2_traverse
> (&max_uid
);
717 /* This function can be called by a port just before it starts the final
718 scheduling pass. It records the fact that an instruction with delay
719 slots has been split into two insns, I1 and I2. The first one will be
720 scheduled normally and initiates the operation. The second one is a
721 shadow which must follow a specific number of cycles after I1; its only
722 purpose is to show the side effect that occurs at that cycle in the RTL.
723 If a JUMP_INSN or a CALL_INSN has been split, I1 should be a normal INSN,
724 while I2 retains the original insn type.
726 There are two ways in which the number of cycles can be specified,
727 involving the CYCLES and STAGES arguments to this function. If STAGES
728 is zero, we just use the value of CYCLES. Otherwise, STAGES is a factor
729 which is multiplied by MODULO_II to give the number of cycles. This is
730 only useful if the caller also calls set_modulo_params to enable modulo
734 record_delay_slot_pair (rtx_insn
*i1
, rtx_insn
*i2
, int cycles
, int stages
)
736 struct delay_pair
*p
= XNEW (struct delay_pair
);
737 struct delay_pair
**slot
;
746 delay_htab
= new hash_table
<delay_i1_hasher
> (10);
747 delay_htab_i2
= new hash_table
<delay_i2_hasher
> (10);
749 slot
= delay_htab
->find_slot_with_hash (i1
, htab_hash_pointer (i1
), INSERT
);
750 p
->next_same_i1
= *slot
;
752 slot
= delay_htab_i2
->find_slot (p
, INSERT
);
756 /* Examine the delay pair hashtable to see if INSN is a shadow for another,
757 and return the other insn if so. Return NULL otherwise. */
759 real_insn_for_shadow (rtx_insn
*insn
)
761 struct delay_pair
*pair
;
766 pair
= delay_htab_i2
->find_with_hash (insn
, htab_hash_pointer (insn
));
767 if (!pair
|| pair
->stages
> 0)
772 /* For a pair P of insns, return the fixed distance in cycles from the first
773 insn after which the second must be scheduled. */
775 pair_delay (struct delay_pair
*p
)
780 return p
->stages
* modulo_ii
;
783 /* Given an insn INSN, add a dependence on its delayed shadow if it
784 has one. Also try to find situations where shadows depend on each other
785 and add dependencies to the real insns to limit the amount of backtracking
788 add_delay_dependencies (rtx_insn
*insn
)
790 struct delay_pair
*pair
;
791 sd_iterator_def sd_it
;
797 pair
= delay_htab_i2
->find_with_hash (insn
, htab_hash_pointer (insn
));
800 add_dependence (insn
, pair
->i1
, REG_DEP_ANTI
);
804 FOR_EACH_DEP (pair
->i2
, SD_LIST_BACK
, sd_it
, dep
)
806 rtx_insn
*pro
= DEP_PRO (dep
);
807 struct delay_pair
*other_pair
808 = delay_htab_i2
->find_with_hash (pro
, htab_hash_pointer (pro
));
809 if (!other_pair
|| other_pair
->stages
)
811 if (pair_delay (other_pair
) >= pair_delay (pair
))
813 if (sched_verbose
>= 4)
815 fprintf (sched_dump
, ";;\tadding dependence %d <- %d\n",
816 INSN_UID (other_pair
->i1
),
817 INSN_UID (pair
->i1
));
818 fprintf (sched_dump
, ";;\tpair1 %d <- %d, cost %d\n",
822 fprintf (sched_dump
, ";;\tpair2 %d <- %d, cost %d\n",
823 INSN_UID (other_pair
->i1
),
824 INSN_UID (other_pair
->i2
),
825 pair_delay (other_pair
));
827 add_dependence (pair
->i1
, other_pair
->i1
, REG_DEP_ANTI
);
832 /* Forward declarations. */
834 static int priority (rtx_insn
*);
835 static int autopref_rank_for_schedule (const rtx_insn
*, const rtx_insn
*);
836 static int rank_for_schedule (const void *, const void *);
837 static void swap_sort (rtx_insn
**, int);
838 static void queue_insn (rtx_insn
*, int, const char *);
839 static int schedule_insn (rtx_insn
*);
840 static void adjust_priority (rtx_insn
*);
841 static void advance_one_cycle (void);
842 static void extend_h_i_d (void);
845 /* Notes handling mechanism:
846 =========================
847 Generally, NOTES are saved before scheduling and restored after scheduling.
848 The scheduler distinguishes between two types of notes:
850 (1) LOOP_BEGIN, LOOP_END, SETJMP, EHREGION_BEG, EHREGION_END notes:
851 Before scheduling a region, a pointer to the note is added to the insn
852 that follows or precedes it. (This happens as part of the data dependence
853 computation). After scheduling an insn, the pointer contained in it is
854 used for regenerating the corresponding note (in reemit_notes).
856 (2) All other notes (e.g. INSN_DELETED): Before scheduling a block,
857 these notes are put in a list (in rm_other_notes() and
858 unlink_other_notes ()). After scheduling the block, these notes are
859 inserted at the beginning of the block (in schedule_block()). */
861 static void ready_add (struct ready_list
*, rtx_insn
*, bool);
862 static rtx_insn
*ready_remove_first (struct ready_list
*);
863 static rtx_insn
*ready_remove_first_dispatch (struct ready_list
*ready
);
865 static void queue_to_ready (struct ready_list
*);
866 static int early_queue_to_ready (state_t
, struct ready_list
*);
868 /* The following functions are used to implement multi-pass scheduling
869 on the first cycle. */
870 static rtx_insn
*ready_remove (struct ready_list
*, int);
871 static void ready_remove_insn (rtx_insn
*);
873 static void fix_inter_tick (rtx_insn
*, rtx_insn
*);
874 static int fix_tick_ready (rtx_insn
*);
875 static void change_queue_index (rtx_insn
*, int);
877 /* The following functions are used to implement scheduling of data/control
878 speculative instructions. */
880 static void extend_h_i_d (void);
881 static void init_h_i_d (rtx_insn
*);
882 static int haifa_speculate_insn (rtx_insn
*, ds_t
, rtx
*);
883 static void generate_recovery_code (rtx_insn
*);
884 static void process_insn_forw_deps_be_in_spec (rtx_insn
*, rtx_insn
*, ds_t
);
885 static void begin_speculative_block (rtx_insn
*);
886 static void add_to_speculative_block (rtx_insn
*);
887 static void init_before_recovery (basic_block
*);
888 static void create_check_block_twin (rtx_insn
*, bool);
889 static void fix_recovery_deps (basic_block
);
890 static bool haifa_change_pattern (rtx_insn
*, rtx
);
891 static void dump_new_block_header (int, basic_block
, rtx_insn
*, rtx_insn
*);
892 static void restore_bb_notes (basic_block
);
893 static void fix_jump_move (rtx_insn
*);
894 static void move_block_after_check (rtx_insn
*);
895 static void move_succs (vec
<edge
, va_gc
> **, basic_block
);
896 static void sched_remove_insn (rtx_insn
*);
897 static void clear_priorities (rtx_insn
*, rtx_vec_t
*);
898 static void calc_priorities (rtx_vec_t
);
899 static void add_jump_dependencies (rtx_insn
*, rtx_insn
*);
901 #endif /* INSN_SCHEDULING */
903 /* Point to state used for the current scheduling pass. */
904 struct haifa_sched_info
*current_sched_info
;
906 #ifndef INSN_SCHEDULING
908 schedule_insns (void)
913 /* Do register pressure sensitive insn scheduling if the flag is set
915 enum sched_pressure_algorithm sched_pressure
;
917 /* Map regno -> its pressure class. The map defined only when
918 SCHED_PRESSURE != SCHED_PRESSURE_NONE. */
919 enum reg_class
*sched_regno_pressure_class
;
921 /* The current register pressure. Only elements corresponding pressure
922 classes are defined. */
923 static int curr_reg_pressure
[N_REG_CLASSES
];
925 /* Saved value of the previous array. */
926 static int saved_reg_pressure
[N_REG_CLASSES
];
928 /* Register living at given scheduling point. */
929 static bitmap curr_reg_live
;
931 /* Saved value of the previous array. */
932 static bitmap saved_reg_live
;
934 /* Registers mentioned in the current region. */
935 static bitmap region_ref_regs
;
937 /* Effective number of available registers of a given class (see comment
938 in sched_pressure_start_bb). */
939 static int sched_class_regs_num
[N_REG_CLASSES
];
940 /* Number of call_used_regs. This is a helper for calculating of
941 sched_class_regs_num. */
942 static int call_used_regs_num
[N_REG_CLASSES
];
944 /* Initiate register pressure relative info for scheduling the current
945 region. Currently it is only clearing register mentioned in the
948 sched_init_region_reg_pressure_info (void)
950 bitmap_clear (region_ref_regs
);
953 /* PRESSURE[CL] describes the pressure on register class CL. Update it
954 for the birth (if BIRTH_P) or death (if !BIRTH_P) of register REGNO.
955 LIVE tracks the set of live registers; if it is null, assume that
956 every birth or death is genuine. */
958 mark_regno_birth_or_death (bitmap live
, int *pressure
, int regno
, bool birth_p
)
960 enum reg_class pressure_class
;
962 pressure_class
= sched_regno_pressure_class
[regno
];
963 if (regno
>= FIRST_PSEUDO_REGISTER
)
965 if (pressure_class
!= NO_REGS
)
969 if (!live
|| bitmap_set_bit (live
, regno
))
970 pressure
[pressure_class
]
971 += (ira_reg_class_max_nregs
972 [pressure_class
][PSEUDO_REGNO_MODE (regno
)]);
976 if (!live
|| bitmap_clear_bit (live
, regno
))
977 pressure
[pressure_class
]
978 -= (ira_reg_class_max_nregs
979 [pressure_class
][PSEUDO_REGNO_MODE (regno
)]);
983 else if (pressure_class
!= NO_REGS
984 && ! TEST_HARD_REG_BIT (ira_no_alloc_regs
, regno
))
988 if (!live
|| bitmap_set_bit (live
, regno
))
989 pressure
[pressure_class
]++;
993 if (!live
|| bitmap_clear_bit (live
, regno
))
994 pressure
[pressure_class
]--;
999 /* Initiate current register pressure related info from living
1000 registers given by LIVE. */
1002 initiate_reg_pressure_info (bitmap live
)
1008 for (i
= 0; i
< ira_pressure_classes_num
; i
++)
1009 curr_reg_pressure
[ira_pressure_classes
[i
]] = 0;
1010 bitmap_clear (curr_reg_live
);
1011 EXECUTE_IF_SET_IN_BITMAP (live
, 0, j
, bi
)
1012 if (sched_pressure
== SCHED_PRESSURE_MODEL
1013 || current_nr_blocks
== 1
1014 || bitmap_bit_p (region_ref_regs
, j
))
1015 mark_regno_birth_or_death (curr_reg_live
, curr_reg_pressure
, j
, true);
1018 /* Mark registers in X as mentioned in the current region. */
1020 setup_ref_regs (rtx x
)
1023 const RTX_CODE code
= GET_CODE (x
);
1028 bitmap_set_range (region_ref_regs
, REGNO (x
), REG_NREGS (x
));
1031 fmt
= GET_RTX_FORMAT (code
);
1032 for (i
= GET_RTX_LENGTH (code
) - 1; i
>= 0; i
--)
1034 setup_ref_regs (XEXP (x
, i
));
1035 else if (fmt
[i
] == 'E')
1037 for (j
= 0; j
< XVECLEN (x
, i
); j
++)
1038 setup_ref_regs (XVECEXP (x
, i
, j
));
1042 /* Initiate current register pressure related info at the start of
1045 initiate_bb_reg_pressure_info (basic_block bb
)
1047 unsigned int i ATTRIBUTE_UNUSED
;
1050 if (current_nr_blocks
> 1)
1051 FOR_BB_INSNS (bb
, insn
)
1052 if (NONDEBUG_INSN_P (insn
))
1053 setup_ref_regs (PATTERN (insn
));
1054 initiate_reg_pressure_info (df_get_live_in (bb
));
1055 if (bb_has_eh_pred (bb
))
1058 unsigned int regno
= EH_RETURN_DATA_REGNO (i
);
1060 if (regno
== INVALID_REGNUM
)
1062 if (! bitmap_bit_p (df_get_live_in (bb
), regno
))
1063 mark_regno_birth_or_death (curr_reg_live
, curr_reg_pressure
,
1068 /* Save current register pressure related info. */
1070 save_reg_pressure (void)
1074 for (i
= 0; i
< ira_pressure_classes_num
; i
++)
1075 saved_reg_pressure
[ira_pressure_classes
[i
]]
1076 = curr_reg_pressure
[ira_pressure_classes
[i
]];
1077 bitmap_copy (saved_reg_live
, curr_reg_live
);
1080 /* Restore saved register pressure related info. */
1082 restore_reg_pressure (void)
1086 for (i
= 0; i
< ira_pressure_classes_num
; i
++)
1087 curr_reg_pressure
[ira_pressure_classes
[i
]]
1088 = saved_reg_pressure
[ira_pressure_classes
[i
]];
1089 bitmap_copy (curr_reg_live
, saved_reg_live
);
1092 /* Return TRUE if the register is dying after its USE. */
1094 dying_use_p (struct reg_use_data
*use
)
1096 struct reg_use_data
*next
;
1098 for (next
= use
->next_regno_use
; next
!= use
; next
= next
->next_regno_use
)
1099 if (NONDEBUG_INSN_P (next
->insn
)
1100 && QUEUE_INDEX (next
->insn
) != QUEUE_SCHEDULED
)
1105 /* Print info about the current register pressure and its excess for
1106 each pressure class. */
1108 print_curr_reg_pressure (void)
1113 fprintf (sched_dump
, ";;\t");
1114 for (i
= 0; i
< ira_pressure_classes_num
; i
++)
1116 cl
= ira_pressure_classes
[i
];
1117 gcc_assert (curr_reg_pressure
[cl
] >= 0);
1118 fprintf (sched_dump
, " %s:%d(%d)", reg_class_names
[cl
],
1119 curr_reg_pressure
[cl
],
1120 curr_reg_pressure
[cl
] - sched_class_regs_num
[cl
]);
1122 fprintf (sched_dump
, "\n");
1125 /* Determine if INSN has a condition that is clobbered if a register
1126 in SET_REGS is modified. */
1128 cond_clobbered_p (rtx_insn
*insn
, HARD_REG_SET set_regs
)
1130 rtx pat
= PATTERN (insn
);
1131 gcc_assert (GET_CODE (pat
) == COND_EXEC
);
1132 if (TEST_HARD_REG_BIT (set_regs
, REGNO (XEXP (COND_EXEC_TEST (pat
), 0))))
1134 sd_iterator_def sd_it
;
1136 haifa_change_pattern (insn
, ORIG_PAT (insn
));
1137 FOR_EACH_DEP (insn
, SD_LIST_BACK
, sd_it
, dep
)
1138 DEP_STATUS (dep
) &= ~DEP_CANCELLED
;
1139 TODO_SPEC (insn
) = HARD_DEP
;
1140 if (sched_verbose
>= 2)
1141 fprintf (sched_dump
,
1142 ";;\t\tdequeue insn %s because of clobbered condition\n",
1143 (*current_sched_info
->print_insn
) (insn
, 0));
1150 /* This function should be called after modifying the pattern of INSN,
1151 to update scheduler data structures as needed. */
1153 update_insn_after_change (rtx_insn
*insn
)
1155 sd_iterator_def sd_it
;
1158 dfa_clear_single_insn_cache (insn
);
1160 sd_it
= sd_iterator_start (insn
,
1161 SD_LIST_FORW
| SD_LIST_BACK
| SD_LIST_RES_BACK
);
1162 while (sd_iterator_cond (&sd_it
, &dep
))
1164 DEP_COST (dep
) = UNKNOWN_DEP_COST
;
1165 sd_iterator_next (&sd_it
);
1168 /* Invalidate INSN_COST, so it'll be recalculated. */
1169 INSN_COST (insn
) = -1;
1170 /* Invalidate INSN_TICK, so it'll be recalculated. */
1171 INSN_TICK (insn
) = INVALID_TICK
;
1173 /* Invalidate autoprefetch data entry. */
1174 INSN_AUTOPREF_MULTIPASS_DATA (insn
)[0].status
1175 = AUTOPREF_MULTIPASS_DATA_UNINITIALIZED
;
1176 INSN_AUTOPREF_MULTIPASS_DATA (insn
)[1].status
1177 = AUTOPREF_MULTIPASS_DATA_UNINITIALIZED
;
1181 /* Two VECs, one to hold dependencies for which pattern replacements
1182 need to be applied or restored at the start of the next cycle, and
1183 another to hold an integer that is either one, to apply the
1184 corresponding replacement, or zero to restore it. */
1185 static vec
<dep_t
> next_cycle_replace_deps
;
1186 static vec
<int> next_cycle_apply
;
1188 static void apply_replacement (dep_t
, bool);
1189 static void restore_pattern (dep_t
, bool);
1191 /* Look at the remaining dependencies for insn NEXT, and compute and return
1192 the TODO_SPEC value we should use for it. This is called after one of
1193 NEXT's dependencies has been resolved.
1194 We also perform pattern replacements for predication, and for broken
1195 replacement dependencies. The latter is only done if FOR_BACKTRACK is
1199 recompute_todo_spec (rtx_insn
*next
, bool for_backtrack
)
1202 sd_iterator_def sd_it
;
1203 dep_t dep
, modify_dep
= NULL
;
1207 bool first_p
= true;
1209 if (sd_lists_empty_p (next
, SD_LIST_BACK
))
1210 /* NEXT has all its dependencies resolved. */
1213 if (!sd_lists_empty_p (next
, SD_LIST_HARD_BACK
))
1216 /* If NEXT is intended to sit adjacent to this instruction, we don't
1217 want to try to break any dependencies. Treat it as a HARD_DEP. */
1218 if (SCHED_GROUP_P (next
))
1221 /* Now we've got NEXT with speculative deps only.
1222 1. Look at the deps to see what we have to do.
1223 2. Check if we can do 'todo'. */
1226 FOR_EACH_DEP (next
, SD_LIST_BACK
, sd_it
, dep
)
1228 rtx_insn
*pro
= DEP_PRO (dep
);
1229 ds_t ds
= DEP_STATUS (dep
) & SPECULATIVE
;
1231 if (DEBUG_INSN_P (pro
) && !DEBUG_INSN_P (next
))
1244 new_ds
= ds_merge (new_ds
, ds
);
1246 else if (DEP_TYPE (dep
) == REG_DEP_CONTROL
)
1248 if (QUEUE_INDEX (pro
) != QUEUE_SCHEDULED
)
1253 DEP_STATUS (dep
) &= ~DEP_CANCELLED
;
1255 else if (DEP_REPLACE (dep
) != NULL
)
1257 if (QUEUE_INDEX (pro
) != QUEUE_SCHEDULED
)
1262 DEP_STATUS (dep
) &= ~DEP_CANCELLED
;
1266 if (n_replace
> 0 && n_control
== 0 && n_spec
== 0)
1268 if (!dbg_cnt (sched_breakdep
))
1270 FOR_EACH_DEP (next
, SD_LIST_BACK
, sd_it
, dep
)
1272 struct dep_replacement
*desc
= DEP_REPLACE (dep
);
1275 if (desc
->insn
== next
&& !for_backtrack
)
1277 gcc_assert (n_replace
== 1);
1278 apply_replacement (dep
, true);
1280 DEP_STATUS (dep
) |= DEP_CANCELLED
;
1286 else if (n_control
== 1 && n_replace
== 0 && n_spec
== 0)
1288 rtx_insn
*pro
, *other
;
1290 rtx cond
= NULL_RTX
;
1292 rtx_insn
*prev
= NULL
;
1296 if ((current_sched_info
->flags
& DO_PREDICATION
) == 0
1297 || (ORIG_PAT (next
) != NULL_RTX
1298 && PREDICATED_PAT (next
) == NULL_RTX
))
1301 pro
= DEP_PRO (modify_dep
);
1302 other
= real_insn_for_shadow (pro
);
1303 if (other
!= NULL_RTX
)
1306 cond
= sched_get_reverse_condition_uncached (pro
);
1307 regno
= REGNO (XEXP (cond
, 0));
1309 /* Find the last scheduled insn that modifies the condition register.
1310 We can stop looking once we find the insn we depend on through the
1311 REG_DEP_CONTROL; if the condition register isn't modified after it,
1312 we know that it still has the right value. */
1313 if (QUEUE_INDEX (pro
) == QUEUE_SCHEDULED
)
1314 FOR_EACH_VEC_ELT_REVERSE (scheduled_insns
, i
, prev
)
1318 find_all_hard_reg_sets (prev
, &t
, true);
1319 if (TEST_HARD_REG_BIT (t
, regno
))
1324 if (ORIG_PAT (next
) == NULL_RTX
)
1326 ORIG_PAT (next
) = PATTERN (next
);
1328 new_pat
= gen_rtx_COND_EXEC (VOIDmode
, cond
, PATTERN (next
));
1329 success
= haifa_change_pattern (next
, new_pat
);
1332 PREDICATED_PAT (next
) = new_pat
;
1334 else if (PATTERN (next
) != PREDICATED_PAT (next
))
1336 bool success
= haifa_change_pattern (next
,
1337 PREDICATED_PAT (next
));
1338 gcc_assert (success
);
1340 DEP_STATUS (modify_dep
) |= DEP_CANCELLED
;
1344 if (PREDICATED_PAT (next
) != NULL_RTX
)
1346 int tick
= INSN_TICK (next
);
1347 bool success
= haifa_change_pattern (next
,
1349 INSN_TICK (next
) = tick
;
1350 gcc_assert (success
);
1353 /* We can't handle the case where there are both speculative and control
1354 dependencies, so we return HARD_DEP in such a case. Also fail if
1355 we have speculative dependencies with not enough points, or more than
1356 one control dependency. */
1357 if ((n_spec
> 0 && (n_control
> 0 || n_replace
> 0))
1359 /* Too few points? */
1360 && ds_weak (new_ds
) < spec_info
->data_weakness_cutoff
)
1368 /* Pointer to the last instruction scheduled. */
1369 static rtx_insn
*last_scheduled_insn
;
1371 /* Pointer to the last nondebug instruction scheduled within the
1372 block, or the prev_head of the scheduling block. Used by
1373 rank_for_schedule, so that insns independent of the last scheduled
1374 insn will be preferred over dependent instructions. */
1375 static rtx_insn
*last_nondebug_scheduled_insn
;
1377 /* Pointer that iterates through the list of unscheduled insns if we
1378 have a dbg_cnt enabled. It always points at an insn prior to the
1379 first unscheduled one. */
1380 static rtx_insn
*nonscheduled_insns_begin
;
1382 /* Compute cost of executing INSN.
1383 This is the number of cycles between instruction issue and
1384 instruction results. */
1386 insn_cost (rtx_insn
*insn
)
1395 if (recog_memoized (insn
) < 0)
1398 cost
= insn_default_latency (insn
);
1405 cost
= INSN_COST (insn
);
1409 /* A USE insn, or something else we don't need to
1410 understand. We can't pass these directly to
1411 result_ready_cost or insn_default_latency because it will
1412 trigger a fatal error for unrecognizable insns. */
1413 if (recog_memoized (insn
) < 0)
1415 INSN_COST (insn
) = 0;
1420 cost
= insn_default_latency (insn
);
1424 INSN_COST (insn
) = cost
;
1431 /* Compute cost of dependence LINK.
1432 This is the number of cycles between instruction issue and
1433 instruction results.
1434 ??? We also use this function to call recog_memoized on all insns. */
1436 dep_cost_1 (dep_t link
, dw_t dw
)
1438 rtx_insn
*insn
= DEP_PRO (link
);
1439 rtx_insn
*used
= DEP_CON (link
);
1442 if (DEP_COST (link
) != UNKNOWN_DEP_COST
)
1443 return DEP_COST (link
);
1447 struct delay_pair
*delay_entry
;
1449 = delay_htab_i2
->find_with_hash (used
, htab_hash_pointer (used
));
1452 if (delay_entry
->i1
== insn
)
1454 DEP_COST (link
) = pair_delay (delay_entry
);
1455 return DEP_COST (link
);
1460 /* A USE insn should never require the value used to be computed.
1461 This allows the computation of a function's result and parameter
1462 values to overlap the return and call. We don't care about the
1463 dependence cost when only decreasing register pressure. */
1464 if (recog_memoized (used
) < 0)
1467 recog_memoized (insn
);
1471 enum reg_note dep_type
= DEP_TYPE (link
);
1473 cost
= insn_cost (insn
);
1475 if (INSN_CODE (insn
) >= 0)
1477 if (dep_type
== REG_DEP_ANTI
)
1479 else if (dep_type
== REG_DEP_OUTPUT
)
1481 cost
= (insn_default_latency (insn
)
1482 - insn_default_latency (used
));
1486 else if (bypass_p (insn
))
1487 cost
= insn_latency (insn
, used
);
1491 if (targetm
.sched
.adjust_cost_2
)
1492 cost
= targetm
.sched
.adjust_cost_2 (used
, (int) dep_type
, insn
, cost
,
1494 else if (targetm
.sched
.adjust_cost
!= NULL
)
1496 /* This variable is used for backward compatibility with the
1498 rtx_insn_list
*dep_cost_rtx_link
=
1499 alloc_INSN_LIST (NULL_RTX
, NULL
);
1501 /* Make it self-cycled, so that if some tries to walk over this
1502 incomplete list he/she will be caught in an endless loop. */
1503 XEXP (dep_cost_rtx_link
, 1) = dep_cost_rtx_link
;
1505 /* Targets use only REG_NOTE_KIND of the link. */
1506 PUT_REG_NOTE_KIND (dep_cost_rtx_link
, DEP_TYPE (link
));
1508 cost
= targetm
.sched
.adjust_cost (used
, dep_cost_rtx_link
,
1511 free_INSN_LIST_node (dep_cost_rtx_link
);
1518 DEP_COST (link
) = cost
;
1522 /* Compute cost of dependence LINK.
1523 This is the number of cycles between instruction issue and
1524 instruction results. */
1526 dep_cost (dep_t link
)
1528 return dep_cost_1 (link
, 0);
1531 /* Use this sel-sched.c friendly function in reorder2 instead of increasing
1532 INSN_PRIORITY explicitly. */
1534 increase_insn_priority (rtx_insn
*insn
, int amount
)
1536 if (!sel_sched_p ())
1538 /* We're dealing with haifa-sched.c INSN_PRIORITY. */
1539 if (INSN_PRIORITY_KNOWN (insn
))
1540 INSN_PRIORITY (insn
) += amount
;
1544 /* In sel-sched.c INSN_PRIORITY is not kept up to date.
1545 Use EXPR_PRIORITY instead. */
1546 sel_add_to_insn_priority (insn
, amount
);
1550 /* Return 'true' if DEP should be included in priority calculations. */
1552 contributes_to_priority_p (dep_t dep
)
1554 if (DEBUG_INSN_P (DEP_CON (dep
))
1555 || DEBUG_INSN_P (DEP_PRO (dep
)))
1558 /* Critical path is meaningful in block boundaries only. */
1559 if (!current_sched_info
->contributes_to_priority (DEP_CON (dep
),
1563 if (DEP_REPLACE (dep
) != NULL
)
1566 /* If flag COUNT_SPEC_IN_CRITICAL_PATH is set,
1567 then speculative instructions will less likely be
1568 scheduled. That is because the priority of
1569 their producers will increase, and, thus, the
1570 producers will more likely be scheduled, thus,
1571 resolving the dependence. */
1572 if (sched_deps_info
->generate_spec_deps
1573 && !(spec_info
->flags
& COUNT_SPEC_IN_CRITICAL_PATH
)
1574 && (DEP_STATUS (dep
) & SPECULATIVE
))
1580 /* Compute the number of nondebug deps in list LIST for INSN. */
1583 dep_list_size (rtx_insn
*insn
, sd_list_types_def list
)
1585 sd_iterator_def sd_it
;
1587 int dbgcount
= 0, nodbgcount
= 0;
1589 if (!MAY_HAVE_DEBUG_INSNS
)
1590 return sd_lists_size (insn
, list
);
1592 FOR_EACH_DEP (insn
, list
, sd_it
, dep
)
1594 if (DEBUG_INSN_P (DEP_CON (dep
)))
1596 else if (!DEBUG_INSN_P (DEP_PRO (dep
)))
1600 gcc_assert (dbgcount
+ nodbgcount
== sd_lists_size (insn
, list
));
1607 /* Compute the priority number for INSN. */
1609 priority (rtx_insn
*insn
)
1611 if (! INSN_P (insn
))
1614 /* We should not be interested in priority of an already scheduled insn. */
1615 gcc_assert (QUEUE_INDEX (insn
) != QUEUE_SCHEDULED
);
1617 if (!INSN_PRIORITY_KNOWN (insn
))
1619 int this_priority
= -1;
1623 int this_fusion_priority
;
1625 targetm
.sched
.fusion_priority (insn
, FUSION_MAX_PRIORITY
,
1626 &this_fusion_priority
, &this_priority
);
1627 INSN_FUSION_PRIORITY (insn
) = this_fusion_priority
;
1629 else if (dep_list_size (insn
, SD_LIST_FORW
) == 0)
1630 /* ??? We should set INSN_PRIORITY to insn_cost when and insn has
1631 some forward deps but all of them are ignored by
1632 contributes_to_priority hook. At the moment we set priority of
1634 this_priority
= insn_cost (insn
);
1637 rtx_insn
*prev_first
, *twin
;
1640 /* For recovery check instructions we calculate priority slightly
1641 different than that of normal instructions. Instead of walking
1642 through INSN_FORW_DEPS (check) list, we walk through
1643 INSN_FORW_DEPS list of each instruction in the corresponding
1646 /* Selective scheduling does not define RECOVERY_BLOCK macro. */
1647 rec
= sel_sched_p () ? NULL
: RECOVERY_BLOCK (insn
);
1648 if (!rec
|| rec
== EXIT_BLOCK_PTR_FOR_FN (cfun
))
1650 prev_first
= PREV_INSN (insn
);
1655 prev_first
= NEXT_INSN (BB_HEAD (rec
));
1656 twin
= PREV_INSN (BB_END (rec
));
1661 sd_iterator_def sd_it
;
1664 FOR_EACH_DEP (twin
, SD_LIST_FORW
, sd_it
, dep
)
1669 next
= DEP_CON (dep
);
1671 if (BLOCK_FOR_INSN (next
) != rec
)
1675 if (!contributes_to_priority_p (dep
))
1679 cost
= dep_cost (dep
);
1682 struct _dep _dep1
, *dep1
= &_dep1
;
1684 init_dep (dep1
, insn
, next
, REG_DEP_ANTI
);
1686 cost
= dep_cost (dep1
);
1689 next_priority
= cost
+ priority (next
);
1691 if (next_priority
> this_priority
)
1692 this_priority
= next_priority
;
1696 twin
= PREV_INSN (twin
);
1698 while (twin
!= prev_first
);
1701 if (this_priority
< 0)
1703 gcc_assert (this_priority
== -1);
1705 this_priority
= insn_cost (insn
);
1708 INSN_PRIORITY (insn
) = this_priority
;
1709 INSN_PRIORITY_STATUS (insn
) = 1;
1712 return INSN_PRIORITY (insn
);
1715 /* Macros and functions for keeping the priority queue sorted, and
1716 dealing with queuing and dequeuing of instructions. */
1718 /* For each pressure class CL, set DEATH[CL] to the number of registers
1719 in that class that die in INSN. */
1722 calculate_reg_deaths (rtx_insn
*insn
, int *death
)
1725 struct reg_use_data
*use
;
1727 for (i
= 0; i
< ira_pressure_classes_num
; i
++)
1728 death
[ira_pressure_classes
[i
]] = 0;
1729 for (use
= INSN_REG_USE_LIST (insn
); use
!= NULL
; use
= use
->next_insn_use
)
1730 if (dying_use_p (use
))
1731 mark_regno_birth_or_death (0, death
, use
->regno
, true);
1734 /* Setup info about the current register pressure impact of scheduling
1735 INSN at the current scheduling point. */
1737 setup_insn_reg_pressure_info (rtx_insn
*insn
)
1739 int i
, change
, before
, after
, hard_regno
;
1740 int excess_cost_change
;
1743 struct reg_pressure_data
*pressure_info
;
1744 int *max_reg_pressure
;
1745 static int death
[N_REG_CLASSES
];
1747 gcc_checking_assert (!DEBUG_INSN_P (insn
));
1749 excess_cost_change
= 0;
1750 calculate_reg_deaths (insn
, death
);
1751 pressure_info
= INSN_REG_PRESSURE (insn
);
1752 max_reg_pressure
= INSN_MAX_REG_PRESSURE (insn
);
1753 gcc_assert (pressure_info
!= NULL
&& max_reg_pressure
!= NULL
);
1754 for (i
= 0; i
< ira_pressure_classes_num
; i
++)
1756 cl
= ira_pressure_classes
[i
];
1757 gcc_assert (curr_reg_pressure
[cl
] >= 0);
1758 change
= (int) pressure_info
[i
].set_increase
- death
[cl
];
1759 before
= MAX (0, max_reg_pressure
[i
] - sched_class_regs_num
[cl
]);
1760 after
= MAX (0, max_reg_pressure
[i
] + change
1761 - sched_class_regs_num
[cl
]);
1762 hard_regno
= ira_class_hard_regs
[cl
][0];
1763 gcc_assert (hard_regno
>= 0);
1764 mode
= reg_raw_mode
[hard_regno
];
1765 excess_cost_change
+= ((after
- before
)
1766 * (ira_memory_move_cost
[mode
][cl
][0]
1767 + ira_memory_move_cost
[mode
][cl
][1]));
1769 INSN_REG_PRESSURE_EXCESS_COST_CHANGE (insn
) = excess_cost_change
;
1772 /* This is the first page of code related to SCHED_PRESSURE_MODEL.
1773 It tries to make the scheduler take register pressure into account
1774 without introducing too many unnecessary stalls. It hooks into the
1775 main scheduling algorithm at several points:
1777 - Before scheduling starts, model_start_schedule constructs a
1778 "model schedule" for the current block. This model schedule is
1779 chosen solely to keep register pressure down. It does not take the
1780 target's pipeline or the original instruction order into account,
1781 except as a tie-breaker. It also doesn't work to a particular
1784 This model schedule gives us an idea of what pressure can be
1785 achieved for the block and gives us an example of a schedule that
1786 keeps to that pressure. It also makes the final schedule less
1787 dependent on the original instruction order. This is important
1788 because the original order can either be "wide" (many values live
1789 at once, such as in user-scheduled code) or "narrow" (few values
1790 live at once, such as after loop unrolling, where several
1791 iterations are executed sequentially).
1793 We do not apply this model schedule to the rtx stream. We simply
1794 record it in model_schedule. We also compute the maximum pressure,
1795 MP, that was seen during this schedule.
1797 - Instructions are added to the ready queue even if they require
1798 a stall. The length of the stall is instead computed as:
1800 MAX (INSN_TICK (INSN) - clock_var, 0)
1802 (= insn_delay). This allows rank_for_schedule to choose between
1803 introducing a deliberate stall or increasing pressure.
1805 - Before sorting the ready queue, model_set_excess_costs assigns
1806 a pressure-based cost to each ready instruction in the queue.
1807 This is the instruction's INSN_REG_PRESSURE_EXCESS_COST_CHANGE
1808 (ECC for short) and is effectively measured in cycles.
1810 - rank_for_schedule ranks instructions based on:
1812 ECC (insn) + insn_delay (insn)
1818 So, for example, an instruction X1 with an ECC of 1 that can issue
1819 now will win over an instruction X0 with an ECC of zero that would
1820 introduce a stall of one cycle. However, an instruction X2 with an
1821 ECC of 2 that can issue now will lose to both X0 and X1.
1823 - When an instruction is scheduled, model_recompute updates the model
1824 schedule with the new pressures (some of which might now exceed the
1825 original maximum pressure MP). model_update_limit_points then searches
1826 for the new point of maximum pressure, if not already known. */
1828 /* Used to separate high-verbosity debug information for SCHED_PRESSURE_MODEL
1829 from surrounding debug information. */
1831 ";;\t\t+------------------------------------------------------\n"
1833 /* Information about the pressure on a particular register class at a
1834 particular point of the model schedule. */
1835 struct model_pressure_data
{
1836 /* The pressure at this point of the model schedule, or -1 if the
1837 point is associated with an instruction that has already been
1841 /* The maximum pressure during or after this point of the model schedule. */
1845 /* Per-instruction information that is used while building the model
1846 schedule. Here, "schedule" refers to the model schedule rather
1847 than the main schedule. */
1848 struct model_insn_info
{
1849 /* The instruction itself. */
1852 /* If this instruction is in model_worklist, these fields link to the
1853 previous (higher-priority) and next (lower-priority) instructions
1855 struct model_insn_info
*prev
;
1856 struct model_insn_info
*next
;
1858 /* While constructing the schedule, QUEUE_INDEX describes whether an
1859 instruction has already been added to the schedule (QUEUE_SCHEDULED),
1860 is in model_worklist (QUEUE_READY), or neither (QUEUE_NOWHERE).
1861 old_queue records the value that QUEUE_INDEX had before scheduling
1862 started, so that we can restore it once the schedule is complete. */
1865 /* The relative importance of an unscheduled instruction. Higher
1866 values indicate greater importance. */
1867 unsigned int model_priority
;
1869 /* The length of the longest path of satisfied true dependencies
1870 that leads to this instruction. */
1873 /* The length of the longest path of dependencies of any kind
1874 that leads from this instruction. */
1877 /* The number of predecessor nodes that must still be scheduled. */
1878 int unscheduled_preds
;
1881 /* Information about the pressure limit for a particular register class.
1882 This structure is used when applying a model schedule to the main
1884 struct model_pressure_limit
{
1885 /* The maximum register pressure seen in the original model schedule. */
1888 /* The maximum register pressure seen in the current model schedule
1889 (which excludes instructions that have already been scheduled). */
1892 /* The point of the current model schedule at which PRESSURE is first
1893 reached. It is set to -1 if the value needs to be recomputed. */
1897 /* Describes a particular way of measuring register pressure. */
1898 struct model_pressure_group
{
1899 /* Index PCI describes the maximum pressure on ira_pressure_classes[PCI]. */
1900 struct model_pressure_limit limits
[N_REG_CLASSES
];
1902 /* Index (POINT * ira_num_pressure_classes + PCI) describes the pressure
1903 on register class ira_pressure_classes[PCI] at point POINT of the
1904 current model schedule. A POINT of model_num_insns describes the
1905 pressure at the end of the schedule. */
1906 struct model_pressure_data
*model
;
1909 /* Index POINT gives the instruction at point POINT of the model schedule.
1910 This array doesn't change during main scheduling. */
1911 static vec
<rtx_insn
*> model_schedule
;
1913 /* The list of instructions in the model worklist, sorted in order of
1914 decreasing priority. */
1915 static struct model_insn_info
*model_worklist
;
1917 /* Index I describes the instruction with INSN_LUID I. */
1918 static struct model_insn_info
*model_insns
;
1920 /* The number of instructions in the model schedule. */
1921 static int model_num_insns
;
1923 /* The index of the first instruction in model_schedule that hasn't yet been
1924 added to the main schedule, or model_num_insns if all of them have. */
1925 static int model_curr_point
;
1927 /* Describes the pressure before each instruction in the model schedule. */
1928 static struct model_pressure_group model_before_pressure
;
1930 /* The first unused model_priority value (as used in model_insn_info). */
1931 static unsigned int model_next_priority
;
1934 /* The model_pressure_data for ira_pressure_classes[PCI] in GROUP
1935 at point POINT of the model schedule. */
1936 #define MODEL_PRESSURE_DATA(GROUP, POINT, PCI) \
1937 (&(GROUP)->model[(POINT) * ira_pressure_classes_num + (PCI)])
1939 /* The maximum pressure on ira_pressure_classes[PCI] in GROUP at or
1940 after point POINT of the model schedule. */
1941 #define MODEL_MAX_PRESSURE(GROUP, POINT, PCI) \
1942 (MODEL_PRESSURE_DATA (GROUP, POINT, PCI)->max_pressure)
1944 /* The pressure on ira_pressure_classes[PCI] in GROUP at point POINT
1945 of the model schedule. */
1946 #define MODEL_REF_PRESSURE(GROUP, POINT, PCI) \
1947 (MODEL_PRESSURE_DATA (GROUP, POINT, PCI)->ref_pressure)
1949 /* Information about INSN that is used when creating the model schedule. */
1950 #define MODEL_INSN_INFO(INSN) \
1951 (&model_insns[INSN_LUID (INSN)])
1953 /* The instruction at point POINT of the model schedule. */
1954 #define MODEL_INSN(POINT) \
1955 (model_schedule[POINT])
1958 /* Return INSN's index in the model schedule, or model_num_insns if it
1959 doesn't belong to that schedule. */
1962 model_index (rtx_insn
*insn
)
1964 if (INSN_MODEL_INDEX (insn
) == 0)
1965 return model_num_insns
;
1966 return INSN_MODEL_INDEX (insn
) - 1;
1969 /* Make sure that GROUP->limits is up-to-date for the current point
1970 of the model schedule. */
1973 model_update_limit_points_in_group (struct model_pressure_group
*group
)
1975 int pci
, max_pressure
, point
;
1977 for (pci
= 0; pci
< ira_pressure_classes_num
; pci
++)
1979 /* We may have passed the final point at which the pressure in
1980 group->limits[pci].pressure was reached. Update the limit if so. */
1981 max_pressure
= MODEL_MAX_PRESSURE (group
, model_curr_point
, pci
);
1982 group
->limits
[pci
].pressure
= max_pressure
;
1984 /* Find the point at which MAX_PRESSURE is first reached. We need
1985 to search in three cases:
1987 - We've already moved past the previous pressure point.
1988 In this case we search forward from model_curr_point.
1990 - We scheduled the previous point of maximum pressure ahead of
1991 its position in the model schedule, but doing so didn't bring
1992 the pressure point earlier. In this case we search forward
1993 from that previous pressure point.
1995 - Scheduling an instruction early caused the maximum pressure
1996 to decrease. In this case we will have set the pressure
1997 point to -1, and we search forward from model_curr_point. */
1998 point
= MAX (group
->limits
[pci
].point
, model_curr_point
);
1999 while (point
< model_num_insns
2000 && MODEL_REF_PRESSURE (group
, point
, pci
) < max_pressure
)
2002 group
->limits
[pci
].point
= point
;
2004 gcc_assert (MODEL_REF_PRESSURE (group
, point
, pci
) == max_pressure
);
2005 gcc_assert (MODEL_MAX_PRESSURE (group
, point
, pci
) == max_pressure
);
2009 /* Make sure that all register-pressure limits are up-to-date for the
2010 current position in the model schedule. */
2013 model_update_limit_points (void)
2015 model_update_limit_points_in_group (&model_before_pressure
);
2018 /* Return the model_index of the last unscheduled use in chain USE
2019 outside of USE's instruction. Return -1 if there are no other uses,
2020 or model_num_insns if the register is live at the end of the block. */
2023 model_last_use_except (struct reg_use_data
*use
)
2025 struct reg_use_data
*next
;
2029 for (next
= use
->next_regno_use
; next
!= use
; next
= next
->next_regno_use
)
2030 if (NONDEBUG_INSN_P (next
->insn
)
2031 && QUEUE_INDEX (next
->insn
) != QUEUE_SCHEDULED
)
2033 index
= model_index (next
->insn
);
2034 if (index
== model_num_insns
)
2035 return model_num_insns
;
2042 /* An instruction with model_index POINT has just been scheduled, and it
2043 adds DELTA to the pressure on ira_pressure_classes[PCI] after POINT - 1.
2044 Update MODEL_REF_PRESSURE (GROUP, POINT, PCI) and
2045 MODEL_MAX_PRESSURE (GROUP, POINT, PCI) accordingly. */
2048 model_start_update_pressure (struct model_pressure_group
*group
,
2049 int point
, int pci
, int delta
)
2051 int next_max_pressure
;
2053 if (point
== model_num_insns
)
2055 /* The instruction wasn't part of the model schedule; it was moved
2056 from a different block. Update the pressure for the end of
2057 the model schedule. */
2058 MODEL_REF_PRESSURE (group
, point
, pci
) += delta
;
2059 MODEL_MAX_PRESSURE (group
, point
, pci
) += delta
;
2063 /* Record that this instruction has been scheduled. Nothing now
2064 changes between POINT and POINT + 1, so get the maximum pressure
2065 from the latter. If the maximum pressure decreases, the new
2066 pressure point may be before POINT. */
2067 MODEL_REF_PRESSURE (group
, point
, pci
) = -1;
2068 next_max_pressure
= MODEL_MAX_PRESSURE (group
, point
+ 1, pci
);
2069 if (MODEL_MAX_PRESSURE (group
, point
, pci
) > next_max_pressure
)
2071 MODEL_MAX_PRESSURE (group
, point
, pci
) = next_max_pressure
;
2072 if (group
->limits
[pci
].point
== point
)
2073 group
->limits
[pci
].point
= -1;
2078 /* Record that scheduling a later instruction has changed the pressure
2079 at point POINT of the model schedule by DELTA (which might be 0).
2080 Update GROUP accordingly. Return nonzero if these changes might
2081 trigger changes to previous points as well. */
2084 model_update_pressure (struct model_pressure_group
*group
,
2085 int point
, int pci
, int delta
)
2087 int ref_pressure
, max_pressure
, next_max_pressure
;
2089 /* If POINT hasn't yet been scheduled, update its pressure. */
2090 ref_pressure
= MODEL_REF_PRESSURE (group
, point
, pci
);
2091 if (ref_pressure
>= 0 && delta
!= 0)
2093 ref_pressure
+= delta
;
2094 MODEL_REF_PRESSURE (group
, point
, pci
) = ref_pressure
;
2096 /* Check whether the maximum pressure in the overall schedule
2097 has increased. (This means that the MODEL_MAX_PRESSURE of
2098 every point <= POINT will need to increase too; see below.) */
2099 if (group
->limits
[pci
].pressure
< ref_pressure
)
2100 group
->limits
[pci
].pressure
= ref_pressure
;
2102 /* If we are at maximum pressure, and the maximum pressure
2103 point was previously unknown or later than POINT,
2104 bring it forward. */
2105 if (group
->limits
[pci
].pressure
== ref_pressure
2106 && !IN_RANGE (group
->limits
[pci
].point
, 0, point
))
2107 group
->limits
[pci
].point
= point
;
2109 /* If POINT used to be the point of maximum pressure, but isn't
2110 any longer, we need to recalculate it using a forward walk. */
2111 if (group
->limits
[pci
].pressure
> ref_pressure
2112 && group
->limits
[pci
].point
== point
)
2113 group
->limits
[pci
].point
= -1;
2116 /* Update the maximum pressure at POINT. Changes here might also
2117 affect the maximum pressure at POINT - 1. */
2118 next_max_pressure
= MODEL_MAX_PRESSURE (group
, point
+ 1, pci
);
2119 max_pressure
= MAX (ref_pressure
, next_max_pressure
);
2120 if (MODEL_MAX_PRESSURE (group
, point
, pci
) != max_pressure
)
2122 MODEL_MAX_PRESSURE (group
, point
, pci
) = max_pressure
;
2128 /* INSN has just been scheduled. Update the model schedule accordingly. */
2131 model_recompute (rtx_insn
*insn
)
2136 } uses
[FIRST_PSEUDO_REGISTER
+ MAX_RECOG_OPERANDS
];
2137 struct reg_use_data
*use
;
2138 struct reg_pressure_data
*reg_pressure
;
2139 int delta
[N_REG_CLASSES
];
2140 int pci
, point
, mix
, new_last
, cl
, ref_pressure
, queue
;
2141 unsigned int i
, num_uses
, num_pending_births
;
2144 /* The destinations of INSN were previously live from POINT onwards, but are
2145 now live from model_curr_point onwards. Set up DELTA accordingly. */
2146 point
= model_index (insn
);
2147 reg_pressure
= INSN_REG_PRESSURE (insn
);
2148 for (pci
= 0; pci
< ira_pressure_classes_num
; pci
++)
2150 cl
= ira_pressure_classes
[pci
];
2151 delta
[cl
] = reg_pressure
[pci
].set_increase
;
2154 /* Record which registers previously died at POINT, but which now die
2155 before POINT. Adjust DELTA so that it represents the effect of
2156 this change after POINT - 1. Set NUM_PENDING_BIRTHS to the number of
2157 registers that will be born in the range [model_curr_point, POINT). */
2159 num_pending_births
= 0;
2160 for (use
= INSN_REG_USE_LIST (insn
); use
!= NULL
; use
= use
->next_insn_use
)
2162 new_last
= model_last_use_except (use
);
2163 if (new_last
< point
)
2165 gcc_assert (num_uses
< ARRAY_SIZE (uses
));
2166 uses
[num_uses
].last_use
= new_last
;
2167 uses
[num_uses
].regno
= use
->regno
;
2168 /* This register is no longer live after POINT - 1. */
2169 mark_regno_birth_or_death (NULL
, delta
, use
->regno
, false);
2172 num_pending_births
++;
2176 /* Update the MODEL_REF_PRESSURE and MODEL_MAX_PRESSURE for POINT.
2177 Also set each group pressure limit for POINT. */
2178 for (pci
= 0; pci
< ira_pressure_classes_num
; pci
++)
2180 cl
= ira_pressure_classes
[pci
];
2181 model_start_update_pressure (&model_before_pressure
,
2182 point
, pci
, delta
[cl
]);
2185 /* Walk the model schedule backwards, starting immediately before POINT. */
2187 if (point
!= model_curr_point
)
2191 insn
= MODEL_INSN (point
);
2192 queue
= QUEUE_INDEX (insn
);
2194 if (queue
!= QUEUE_SCHEDULED
)
2196 /* DELTA describes the effect of the move on the register pressure
2197 after POINT. Make it describe the effect on the pressure
2200 while (i
< num_uses
)
2202 if (uses
[i
].last_use
== point
)
2204 /* This register is now live again. */
2205 mark_regno_birth_or_death (NULL
, delta
,
2206 uses
[i
].regno
, true);
2208 /* Remove this use from the array. */
2209 uses
[i
] = uses
[num_uses
- 1];
2211 num_pending_births
--;
2217 if (sched_verbose
>= 5)
2221 fprintf (sched_dump
, MODEL_BAR
);
2222 fprintf (sched_dump
, ";;\t\t| New pressure for model"
2224 fprintf (sched_dump
, MODEL_BAR
);
2228 fprintf (sched_dump
, ";;\t\t| %3d %4d %-30s ",
2229 point
, INSN_UID (insn
),
2230 str_pattern_slim (PATTERN (insn
)));
2231 for (pci
= 0; pci
< ira_pressure_classes_num
; pci
++)
2233 cl
= ira_pressure_classes
[pci
];
2234 ref_pressure
= MODEL_REF_PRESSURE (&model_before_pressure
,
2236 fprintf (sched_dump
, " %s:[%d->%d]",
2237 reg_class_names
[ira_pressure_classes
[pci
]],
2238 ref_pressure
, ref_pressure
+ delta
[cl
]);
2240 fprintf (sched_dump
, "\n");
2244 /* Adjust the pressure at POINT. Set MIX to nonzero if POINT - 1
2245 might have changed as well. */
2246 mix
= num_pending_births
;
2247 for (pci
= 0; pci
< ira_pressure_classes_num
; pci
++)
2249 cl
= ira_pressure_classes
[pci
];
2251 mix
|= model_update_pressure (&model_before_pressure
,
2252 point
, pci
, delta
[cl
]);
2255 while (mix
&& point
> model_curr_point
);
2258 fprintf (sched_dump
, MODEL_BAR
);
2261 /* After DEP, which was cancelled, has been resolved for insn NEXT,
2262 check whether the insn's pattern needs restoring. */
2264 must_restore_pattern_p (rtx_insn
*next
, dep_t dep
)
2266 if (QUEUE_INDEX (next
) == QUEUE_SCHEDULED
)
2269 if (DEP_TYPE (dep
) == REG_DEP_CONTROL
)
2271 gcc_assert (ORIG_PAT (next
) != NULL_RTX
);
2272 gcc_assert (next
== DEP_CON (dep
));
2276 struct dep_replacement
*desc
= DEP_REPLACE (dep
);
2277 if (desc
->insn
!= next
)
2279 gcc_assert (*desc
->loc
== desc
->orig
);
2286 /* model_spill_cost (CL, P, P') returns the cost of increasing the
2287 pressure on CL from P to P'. We use this to calculate a "base ECC",
2288 baseECC (CL, X), for each pressure class CL and each instruction X.
2289 Supposing X changes the pressure on CL from P to P', and that the
2290 maximum pressure on CL in the current model schedule is MP', then:
2292 * if X occurs before or at the next point of maximum pressure in
2293 the model schedule and P' > MP', then:
2295 baseECC (CL, X) = model_spill_cost (CL, MP, P')
2297 The idea is that the pressure after scheduling a fixed set of
2298 instructions -- in this case, the set up to and including the
2299 next maximum pressure point -- is going to be the same regardless
2300 of the order; we simply want to keep the intermediate pressure
2301 under control. Thus X has a cost of zero unless scheduling it
2302 now would exceed MP'.
2304 If all increases in the set are by the same amount, no zero-cost
2305 instruction will ever cause the pressure to exceed MP'. However,
2306 if X is instead moved past an instruction X' with pressure in the
2307 range (MP' - (P' - P), MP'), the pressure at X' will increase
2308 beyond MP'. Since baseECC is very much a heuristic anyway,
2309 it doesn't seem worth the overhead of tracking cases like these.
2311 The cost of exceeding MP' is always based on the original maximum
2312 pressure MP. This is so that going 2 registers over the original
2313 limit has the same cost regardless of whether it comes from two
2314 separate +1 deltas or from a single +2 delta.
2316 * if X occurs after the next point of maximum pressure in the model
2317 schedule and P' > P, then:
2319 baseECC (CL, X) = model_spill_cost (CL, MP, MP' + (P' - P))
2321 That is, if we move X forward across a point of maximum pressure,
2322 and if X increases the pressure by P' - P, then we conservatively
2323 assume that scheduling X next would increase the maximum pressure
2324 by P' - P. Again, the cost of doing this is based on the original
2325 maximum pressure MP, for the same reason as above.
2327 * if P' < P, P > MP, and X occurs at or after the next point of
2328 maximum pressure, then:
2330 baseECC (CL, X) = -model_spill_cost (CL, MAX (MP, P'), P)
2332 That is, if we have already exceeded the original maximum pressure MP,
2333 and if X might reduce the maximum pressure again -- or at least push
2334 it further back, and thus allow more scheduling freedom -- it is given
2335 a negative cost to reflect the improvement.
2341 In this case, X is not expected to affect the maximum pressure MP',
2342 so it has zero cost.
2344 We then create a combined value baseECC (X) that is the sum of
2345 baseECC (CL, X) for each pressure class CL.
2347 baseECC (X) could itself be used as the ECC value described above.
2348 However, this is often too conservative, in the sense that it
2349 tends to make high-priority instructions that increase pressure
2350 wait too long in cases where introducing a spill would be better.
2351 For this reason the final ECC is a priority-adjusted form of
2352 baseECC (X). Specifically, we calculate:
2354 P (X) = INSN_PRIORITY (X) - insn_delay (X) - baseECC (X)
2355 baseP = MAX { P (X) | baseECC (X) <= 0 }
2359 ECC (X) = MAX (MIN (baseP - P (X), baseECC (X)), 0)
2361 Thus an instruction's effect on pressure is ignored if it has a high
2362 enough priority relative to the ones that don't increase pressure.
2363 Negative values of baseECC (X) do not increase the priority of X
2364 itself, but they do make it harder for other instructions to
2365 increase the pressure further.
2367 This pressure cost is deliberately timid. The intention has been
2368 to choose a heuristic that rarely interferes with the normal list
2369 scheduler in cases where that scheduler would produce good code.
2370 We simply want to curb some of its worst excesses. */
2372 /* Return the cost of increasing the pressure in class CL from FROM to TO.
2374 Here we use the very simplistic cost model that every register above
2375 sched_class_regs_num[CL] has a spill cost of 1. We could use other
2376 measures instead, such as one based on MEMORY_MOVE_COST. However:
2378 (1) In order for an instruction to be scheduled, the higher cost
2379 would need to be justified in a single saving of that many stalls.
2380 This is overly pessimistic, because the benefit of spilling is
2381 often to avoid a sequence of several short stalls rather than
2384 (2) The cost is still arbitrary. Because we are not allocating
2385 registers during scheduling, we have no way of knowing for
2386 sure how many memory accesses will be required by each spill,
2387 where the spills will be placed within the block, or even
2388 which block(s) will contain the spills.
2390 So a higher cost than 1 is often too conservative in practice,
2391 forcing blocks to contain unnecessary stalls instead of spill code.
2392 The simple cost below seems to be the best compromise. It reduces
2393 the interference with the normal list scheduler, which helps make
2394 it more suitable for a default-on option. */
2397 model_spill_cost (int cl
, int from
, int to
)
2399 from
= MAX (from
, sched_class_regs_num
[cl
]);
2400 return MAX (to
, from
) - from
;
2403 /* Return baseECC (ira_pressure_classes[PCI], POINT), given that
2404 P = curr_reg_pressure[ira_pressure_classes[PCI]] and that
2408 model_excess_group_cost (struct model_pressure_group
*group
,
2409 int point
, int pci
, int delta
)
2413 cl
= ira_pressure_classes
[pci
];
2414 if (delta
< 0 && point
>= group
->limits
[pci
].point
)
2416 pressure
= MAX (group
->limits
[pci
].orig_pressure
,
2417 curr_reg_pressure
[cl
] + delta
);
2418 return -model_spill_cost (cl
, pressure
, curr_reg_pressure
[cl
]);
2423 if (point
> group
->limits
[pci
].point
)
2424 pressure
= group
->limits
[pci
].pressure
+ delta
;
2426 pressure
= curr_reg_pressure
[cl
] + delta
;
2428 if (pressure
> group
->limits
[pci
].pressure
)
2429 return model_spill_cost (cl
, group
->limits
[pci
].orig_pressure
,
2436 /* Return baseECC (MODEL_INSN (INSN)). Dump the costs to sched_dump
2440 model_excess_cost (rtx_insn
*insn
, bool print_p
)
2442 int point
, pci
, cl
, cost
, this_cost
, delta
;
2443 struct reg_pressure_data
*insn_reg_pressure
;
2444 int insn_death
[N_REG_CLASSES
];
2446 calculate_reg_deaths (insn
, insn_death
);
2447 point
= model_index (insn
);
2448 insn_reg_pressure
= INSN_REG_PRESSURE (insn
);
2452 fprintf (sched_dump
, ";;\t\t| %3d %4d | %4d %+3d |", point
,
2453 INSN_UID (insn
), INSN_PRIORITY (insn
), insn_delay (insn
));
2455 /* Sum up the individual costs for each register class. */
2456 for (pci
= 0; pci
< ira_pressure_classes_num
; pci
++)
2458 cl
= ira_pressure_classes
[pci
];
2459 delta
= insn_reg_pressure
[pci
].set_increase
- insn_death
[cl
];
2460 this_cost
= model_excess_group_cost (&model_before_pressure
,
2464 fprintf (sched_dump
, " %s:[%d base cost %d]",
2465 reg_class_names
[cl
], delta
, this_cost
);
2469 fprintf (sched_dump
, "\n");
2474 /* Dump the next points of maximum pressure for GROUP. */
2477 model_dump_pressure_points (struct model_pressure_group
*group
)
2481 fprintf (sched_dump
, ";;\t\t| pressure points");
2482 for (pci
= 0; pci
< ira_pressure_classes_num
; pci
++)
2484 cl
= ira_pressure_classes
[pci
];
2485 fprintf (sched_dump
, " %s:[%d->%d at ", reg_class_names
[cl
],
2486 curr_reg_pressure
[cl
], group
->limits
[pci
].pressure
);
2487 if (group
->limits
[pci
].point
< model_num_insns
)
2488 fprintf (sched_dump
, "%d:%d]", group
->limits
[pci
].point
,
2489 INSN_UID (MODEL_INSN (group
->limits
[pci
].point
)));
2491 fprintf (sched_dump
, "end]");
2493 fprintf (sched_dump
, "\n");
2496 /* Set INSN_REG_PRESSURE_EXCESS_COST_CHANGE for INSNS[0...COUNT-1]. */
2499 model_set_excess_costs (rtx_insn
**insns
, int count
)
2501 int i
, cost
, priority_base
, priority
;
2504 /* Record the baseECC value for each instruction in the model schedule,
2505 except that negative costs are converted to zero ones now rather than
2506 later. Do not assign a cost to debug instructions, since they must
2507 not change code-generation decisions. Experiments suggest we also
2508 get better results by not assigning a cost to instructions from
2511 Set PRIORITY_BASE to baseP in the block comment above. This is the
2512 maximum priority of the "cheap" instructions, which should always
2513 include the next model instruction. */
2516 for (i
= 0; i
< count
; i
++)
2517 if (INSN_MODEL_INDEX (insns
[i
]))
2519 if (sched_verbose
>= 6 && !print_p
)
2521 fprintf (sched_dump
, MODEL_BAR
);
2522 fprintf (sched_dump
, ";;\t\t| Pressure costs for ready queue\n");
2523 model_dump_pressure_points (&model_before_pressure
);
2524 fprintf (sched_dump
, MODEL_BAR
);
2527 cost
= model_excess_cost (insns
[i
], print_p
);
2530 priority
= INSN_PRIORITY (insns
[i
]) - insn_delay (insns
[i
]) - cost
;
2531 priority_base
= MAX (priority_base
, priority
);
2534 INSN_REG_PRESSURE_EXCESS_COST_CHANGE (insns
[i
]) = cost
;
2537 fprintf (sched_dump
, MODEL_BAR
);
2539 /* Use MAX (baseECC, 0) and baseP to calculcate ECC for each
2541 for (i
= 0; i
< count
; i
++)
2543 cost
= INSN_REG_PRESSURE_EXCESS_COST_CHANGE (insns
[i
]);
2544 priority
= INSN_PRIORITY (insns
[i
]) - insn_delay (insns
[i
]);
2545 if (cost
> 0 && priority
> priority_base
)
2547 cost
+= priority_base
- priority
;
2548 INSN_REG_PRESSURE_EXCESS_COST_CHANGE (insns
[i
]) = MAX (cost
, 0);
2554 /* Enum of rank_for_schedule heuristic decisions. */
2556 RFS_LIVE_RANGE_SHRINK1
, RFS_LIVE_RANGE_SHRINK2
,
2557 RFS_SCHED_GROUP
, RFS_PRESSURE_DELAY
, RFS_PRESSURE_TICK
,
2558 RFS_FEEDS_BACKTRACK_INSN
, RFS_PRIORITY
, RFS_SPECULATION
,
2559 RFS_SCHED_RANK
, RFS_LAST_INSN
, RFS_PRESSURE_INDEX
,
2560 RFS_DEP_COUNT
, RFS_TIE
, RFS_FUSION
, RFS_N
};
2562 /* Corresponding strings for print outs. */
2563 static const char *rfs_str
[RFS_N
] = {
2564 "RFS_LIVE_RANGE_SHRINK1", "RFS_LIVE_RANGE_SHRINK2",
2565 "RFS_SCHED_GROUP", "RFS_PRESSURE_DELAY", "RFS_PRESSURE_TICK",
2566 "RFS_FEEDS_BACKTRACK_INSN", "RFS_PRIORITY", "RFS_SPECULATION",
2567 "RFS_SCHED_RANK", "RFS_LAST_INSN", "RFS_PRESSURE_INDEX",
2568 "RFS_DEP_COUNT", "RFS_TIE", "RFS_FUSION" };
2570 /* Statistical breakdown of rank_for_schedule decisions. */
2571 typedef struct { unsigned stats
[RFS_N
]; } rank_for_schedule_stats_t
;
2572 static rank_for_schedule_stats_t rank_for_schedule_stats
;
2574 /* Return the result of comparing insns TMP and TMP2 and update
2575 Rank_For_Schedule statistics. */
2577 rfs_result (enum rfs_decision decision
, int result
, rtx tmp
, rtx tmp2
)
2579 ++rank_for_schedule_stats
.stats
[decision
];
2581 INSN_LAST_RFS_WIN (tmp
) = decision
;
2582 else if (result
> 0)
2583 INSN_LAST_RFS_WIN (tmp2
) = decision
;
2589 /* Sorting predicate to move DEBUG_INSNs to the top of ready list, while
2590 keeping normal insns in original order. */
2593 rank_for_schedule_debug (const void *x
, const void *y
)
2595 rtx_insn
*tmp
= *(rtx_insn
* const *) y
;
2596 rtx_insn
*tmp2
= *(rtx_insn
* const *) x
;
2598 /* Schedule debug insns as early as possible. */
2599 if (DEBUG_INSN_P (tmp
) && !DEBUG_INSN_P (tmp2
))
2601 else if (!DEBUG_INSN_P (tmp
) && DEBUG_INSN_P (tmp2
))
2603 else if (DEBUG_INSN_P (tmp
) && DEBUG_INSN_P (tmp2
))
2604 return INSN_LUID (tmp
) - INSN_LUID (tmp2
);
2606 return INSN_RFS_DEBUG_ORIG_ORDER (tmp2
) - INSN_RFS_DEBUG_ORIG_ORDER (tmp
);
2609 /* Returns a positive value if x is preferred; returns a negative value if
2610 y is preferred. Should never return 0, since that will make the sort
2614 rank_for_schedule (const void *x
, const void *y
)
2616 rtx_insn
*tmp
= *(rtx_insn
* const *) y
;
2617 rtx_insn
*tmp2
= *(rtx_insn
* const *) x
;
2618 int tmp_class
, tmp2_class
;
2619 int val
, priority_val
, info_val
, diff
;
2621 if (live_range_shrinkage_p
)
2623 /* Don't use SCHED_PRESSURE_MODEL -- it results in much worse
2625 gcc_assert (sched_pressure
== SCHED_PRESSURE_WEIGHTED
);
2626 if ((INSN_REG_PRESSURE_EXCESS_COST_CHANGE (tmp
) < 0
2627 || INSN_REG_PRESSURE_EXCESS_COST_CHANGE (tmp2
) < 0)
2628 && (diff
= (INSN_REG_PRESSURE_EXCESS_COST_CHANGE (tmp
)
2629 - INSN_REG_PRESSURE_EXCESS_COST_CHANGE (tmp2
))) != 0)
2630 return rfs_result (RFS_LIVE_RANGE_SHRINK1
, diff
, tmp
, tmp2
);
2631 /* Sort by INSN_LUID (original insn order), so that we make the
2632 sort stable. This minimizes instruction movement, thus
2633 minimizing sched's effect on debugging and cross-jumping. */
2634 return rfs_result (RFS_LIVE_RANGE_SHRINK2
,
2635 INSN_LUID (tmp
) - INSN_LUID (tmp2
), tmp
, tmp2
);
2638 /* The insn in a schedule group should be issued the first. */
2639 if (flag_sched_group_heuristic
&&
2640 SCHED_GROUP_P (tmp
) != SCHED_GROUP_P (tmp2
))
2641 return rfs_result (RFS_SCHED_GROUP
, SCHED_GROUP_P (tmp2
) ? 1 : -1,
2644 /* Make sure that priority of TMP and TMP2 are initialized. */
2645 gcc_assert (INSN_PRIORITY_KNOWN (tmp
) && INSN_PRIORITY_KNOWN (tmp2
));
2649 /* The instruction that has the same fusion priority as the last
2650 instruction is the instruction we picked next. If that is not
2651 the case, we sort ready list firstly by fusion priority, then
2652 by priority, and at last by INSN_LUID. */
2653 int a
= INSN_FUSION_PRIORITY (tmp
);
2654 int b
= INSN_FUSION_PRIORITY (tmp2
);
2657 if (last_nondebug_scheduled_insn
2658 && !NOTE_P (last_nondebug_scheduled_insn
)
2659 && BLOCK_FOR_INSN (tmp
)
2660 == BLOCK_FOR_INSN (last_nondebug_scheduled_insn
))
2661 last
= INSN_FUSION_PRIORITY (last_nondebug_scheduled_insn
);
2663 if (a
!= last
&& b
!= last
)
2667 a
= INSN_PRIORITY (tmp
);
2668 b
= INSN_PRIORITY (tmp2
);
2671 return rfs_result (RFS_FUSION
, b
- a
, tmp
, tmp2
);
2673 return rfs_result (RFS_FUSION
,
2674 INSN_LUID (tmp
) - INSN_LUID (tmp2
), tmp
, tmp2
);
2678 gcc_assert (last_nondebug_scheduled_insn
2679 && !NOTE_P (last_nondebug_scheduled_insn
));
2680 last
= INSN_PRIORITY (last_nondebug_scheduled_insn
);
2682 a
= abs (INSN_PRIORITY (tmp
) - last
);
2683 b
= abs (INSN_PRIORITY (tmp2
) - last
);
2685 return rfs_result (RFS_FUSION
, a
- b
, tmp
, tmp2
);
2687 return rfs_result (RFS_FUSION
,
2688 INSN_LUID (tmp
) - INSN_LUID (tmp2
), tmp
, tmp2
);
2691 return rfs_result (RFS_FUSION
, -1, tmp
, tmp2
);
2693 return rfs_result (RFS_FUSION
, 1, tmp
, tmp2
);
2696 if (sched_pressure
!= SCHED_PRESSURE_NONE
)
2698 /* Prefer insn whose scheduling results in the smallest register
2700 if ((diff
= (INSN_REG_PRESSURE_EXCESS_COST_CHANGE (tmp
)
2702 - INSN_REG_PRESSURE_EXCESS_COST_CHANGE (tmp2
)
2703 - insn_delay (tmp2
))))
2704 return rfs_result (RFS_PRESSURE_DELAY
, diff
, tmp
, tmp2
);
2707 if (sched_pressure
!= SCHED_PRESSURE_NONE
2708 && (INSN_TICK (tmp2
) > clock_var
|| INSN_TICK (tmp
) > clock_var
)
2709 && INSN_TICK (tmp2
) != INSN_TICK (tmp
))
2711 diff
= INSN_TICK (tmp
) - INSN_TICK (tmp2
);
2712 return rfs_result (RFS_PRESSURE_TICK
, diff
, tmp
, tmp2
);
2715 /* If we are doing backtracking in this schedule, prefer insns that
2716 have forward dependencies with negative cost against an insn that
2717 was already scheduled. */
2718 if (current_sched_info
->flags
& DO_BACKTRACKING
)
2720 priority_val
= FEEDS_BACKTRACK_INSN (tmp2
) - FEEDS_BACKTRACK_INSN (tmp
);
2722 return rfs_result (RFS_FEEDS_BACKTRACK_INSN
, priority_val
, tmp
, tmp2
);
2725 /* Prefer insn with higher priority. */
2726 priority_val
= INSN_PRIORITY (tmp2
) - INSN_PRIORITY (tmp
);
2728 if (flag_sched_critical_path_heuristic
&& priority_val
)
2729 return rfs_result (RFS_PRIORITY
, priority_val
, tmp
, tmp2
);
2731 if (PARAM_VALUE (PARAM_SCHED_AUTOPREF_QUEUE_DEPTH
) >= 0)
2733 int autopref
= autopref_rank_for_schedule (tmp
, tmp2
);
2738 /* Prefer speculative insn with greater dependencies weakness. */
2739 if (flag_sched_spec_insn_heuristic
&& spec_info
)
2745 ds1
= TODO_SPEC (tmp
) & SPECULATIVE
;
2747 dw1
= ds_weak (ds1
);
2751 ds2
= TODO_SPEC (tmp2
) & SPECULATIVE
;
2753 dw2
= ds_weak (ds2
);
2758 if (dw
> (NO_DEP_WEAK
/ 8) || dw
< -(NO_DEP_WEAK
/ 8))
2759 return rfs_result (RFS_SPECULATION
, dw
, tmp
, tmp2
);
2762 info_val
= (*current_sched_info
->rank
) (tmp
, tmp2
);
2763 if (flag_sched_rank_heuristic
&& info_val
)
2764 return rfs_result (RFS_SCHED_RANK
, info_val
, tmp
, tmp2
);
2766 /* Compare insns based on their relation to the last scheduled
2768 if (flag_sched_last_insn_heuristic
&& last_nondebug_scheduled_insn
)
2772 rtx_insn
*last
= last_nondebug_scheduled_insn
;
2774 /* Classify the instructions into three classes:
2775 1) Data dependent on last schedule insn.
2776 2) Anti/Output dependent on last scheduled insn.
2777 3) Independent of last scheduled insn, or has latency of one.
2778 Choose the insn from the highest numbered class if different. */
2779 dep1
= sd_find_dep_between (last
, tmp
, true);
2781 if (dep1
== NULL
|| dep_cost (dep1
) == 1)
2783 else if (/* Data dependence. */
2784 DEP_TYPE (dep1
) == REG_DEP_TRUE
)
2789 dep2
= sd_find_dep_between (last
, tmp2
, true);
2791 if (dep2
== NULL
|| dep_cost (dep2
) == 1)
2793 else if (/* Data dependence. */
2794 DEP_TYPE (dep2
) == REG_DEP_TRUE
)
2799 if ((val
= tmp2_class
- tmp_class
))
2800 return rfs_result (RFS_LAST_INSN
, val
, tmp
, tmp2
);
2803 /* Prefer instructions that occur earlier in the model schedule. */
2804 if (sched_pressure
== SCHED_PRESSURE_MODEL
2805 && INSN_BB (tmp
) == target_bb
&& INSN_BB (tmp2
) == target_bb
)
2807 diff
= model_index (tmp
) - model_index (tmp2
);
2808 gcc_assert (diff
!= 0);
2809 return rfs_result (RFS_PRESSURE_INDEX
, diff
, tmp
, tmp2
);
2812 /* Prefer the insn which has more later insns that depend on it.
2813 This gives the scheduler more freedom when scheduling later
2814 instructions at the expense of added register pressure. */
2816 val
= (dep_list_size (tmp2
, SD_LIST_FORW
)
2817 - dep_list_size (tmp
, SD_LIST_FORW
));
2819 if (flag_sched_dep_count_heuristic
&& val
!= 0)
2820 return rfs_result (RFS_DEP_COUNT
, val
, tmp
, tmp2
);
2822 /* If insns are equally good, sort by INSN_LUID (original insn order),
2823 so that we make the sort stable. This minimizes instruction movement,
2824 thus minimizing sched's effect on debugging and cross-jumping. */
2825 return rfs_result (RFS_TIE
, INSN_LUID (tmp
) - INSN_LUID (tmp2
), tmp
, tmp2
);
2828 /* Resort the array A in which only element at index N may be out of order. */
2830 HAIFA_INLINE
static void
2831 swap_sort (rtx_insn
**a
, int n
)
2833 rtx_insn
*insn
= a
[n
- 1];
2836 while (i
>= 0 && rank_for_schedule (a
+ i
, &insn
) >= 0)
2844 /* Add INSN to the insn queue so that it can be executed at least
2845 N_CYCLES after the currently executing insn. Preserve insns
2846 chain for debugging purposes. REASON will be printed in debugging
2849 HAIFA_INLINE
static void
2850 queue_insn (rtx_insn
*insn
, int n_cycles
, const char *reason
)
2852 int next_q
= NEXT_Q_AFTER (q_ptr
, n_cycles
);
2853 rtx_insn_list
*link
= alloc_INSN_LIST (insn
, insn_queue
[next_q
]);
2856 gcc_assert (n_cycles
<= max_insn_queue_index
);
2857 gcc_assert (!DEBUG_INSN_P (insn
));
2859 insn_queue
[next_q
] = link
;
2862 if (sched_verbose
>= 2)
2864 fprintf (sched_dump
, ";;\t\tReady-->Q: insn %s: ",
2865 (*current_sched_info
->print_insn
) (insn
, 0));
2867 fprintf (sched_dump
, "queued for %d cycles (%s).\n", n_cycles
, reason
);
2870 QUEUE_INDEX (insn
) = next_q
;
2872 if (current_sched_info
->flags
& DO_BACKTRACKING
)
2874 new_tick
= clock_var
+ n_cycles
;
2875 if (INSN_TICK (insn
) == INVALID_TICK
|| INSN_TICK (insn
) < new_tick
)
2876 INSN_TICK (insn
) = new_tick
;
2878 if (INSN_EXACT_TICK (insn
) != INVALID_TICK
2879 && INSN_EXACT_TICK (insn
) < clock_var
+ n_cycles
)
2881 must_backtrack
= true;
2882 if (sched_verbose
>= 2)
2883 fprintf (sched_dump
, ";;\t\tcausing a backtrack.\n");
2888 /* Remove INSN from queue. */
2890 queue_remove (rtx_insn
*insn
)
2892 gcc_assert (QUEUE_INDEX (insn
) >= 0);
2893 remove_free_INSN_LIST_elem (insn
, &insn_queue
[QUEUE_INDEX (insn
)]);
2895 QUEUE_INDEX (insn
) = QUEUE_NOWHERE
;
2898 /* Return a pointer to the bottom of the ready list, i.e. the insn
2899 with the lowest priority. */
2902 ready_lastpos (struct ready_list
*ready
)
2904 gcc_assert (ready
->n_ready
>= 1);
2905 return ready
->vec
+ ready
->first
- ready
->n_ready
+ 1;
2908 /* Add an element INSN to the ready list so that it ends up with the
2909 lowest/highest priority depending on FIRST_P. */
2911 HAIFA_INLINE
static void
2912 ready_add (struct ready_list
*ready
, rtx_insn
*insn
, bool first_p
)
2916 if (ready
->first
== ready
->n_ready
)
2918 memmove (ready
->vec
+ ready
->veclen
- ready
->n_ready
,
2919 ready_lastpos (ready
),
2920 ready
->n_ready
* sizeof (rtx
));
2921 ready
->first
= ready
->veclen
- 1;
2923 ready
->vec
[ready
->first
- ready
->n_ready
] = insn
;
2927 if (ready
->first
== ready
->veclen
- 1)
2930 /* ready_lastpos() fails when called with (ready->n_ready == 0). */
2931 memmove (ready
->vec
+ ready
->veclen
- ready
->n_ready
- 1,
2932 ready_lastpos (ready
),
2933 ready
->n_ready
* sizeof (rtx
));
2934 ready
->first
= ready
->veclen
- 2;
2936 ready
->vec
[++(ready
->first
)] = insn
;
2940 if (DEBUG_INSN_P (insn
))
2943 gcc_assert (QUEUE_INDEX (insn
) != QUEUE_READY
);
2944 QUEUE_INDEX (insn
) = QUEUE_READY
;
2946 if (INSN_EXACT_TICK (insn
) != INVALID_TICK
2947 && INSN_EXACT_TICK (insn
) < clock_var
)
2949 must_backtrack
= true;
2953 /* Remove the element with the highest priority from the ready list and
2956 HAIFA_INLINE
static rtx_insn
*
2957 ready_remove_first (struct ready_list
*ready
)
2961 gcc_assert (ready
->n_ready
);
2962 t
= ready
->vec
[ready
->first
--];
2964 if (DEBUG_INSN_P (t
))
2966 /* If the queue becomes empty, reset it. */
2967 if (ready
->n_ready
== 0)
2968 ready
->first
= ready
->veclen
- 1;
2970 gcc_assert (QUEUE_INDEX (t
) == QUEUE_READY
);
2971 QUEUE_INDEX (t
) = QUEUE_NOWHERE
;
2976 /* The following code implements multi-pass scheduling for the first
2977 cycle. In other words, we will try to choose ready insn which
2978 permits to start maximum number of insns on the same cycle. */
2980 /* Return a pointer to the element INDEX from the ready. INDEX for
2981 insn with the highest priority is 0, and the lowest priority has
2985 ready_element (struct ready_list
*ready
, int index
)
2987 gcc_assert (ready
->n_ready
&& index
< ready
->n_ready
);
2989 return ready
->vec
[ready
->first
- index
];
2992 /* Remove the element INDEX from the ready list and return it. INDEX
2993 for insn with the highest priority is 0, and the lowest priority
2996 HAIFA_INLINE
static rtx_insn
*
2997 ready_remove (struct ready_list
*ready
, int index
)
3003 return ready_remove_first (ready
);
3004 gcc_assert (ready
->n_ready
&& index
< ready
->n_ready
);
3005 t
= ready
->vec
[ready
->first
- index
];
3007 if (DEBUG_INSN_P (t
))
3009 for (i
= index
; i
< ready
->n_ready
; i
++)
3010 ready
->vec
[ready
->first
- i
] = ready
->vec
[ready
->first
- i
- 1];
3011 QUEUE_INDEX (t
) = QUEUE_NOWHERE
;
3015 /* Remove INSN from the ready list. */
3017 ready_remove_insn (rtx_insn
*insn
)
3021 for (i
= 0; i
< readyp
->n_ready
; i
++)
3022 if (ready_element (readyp
, i
) == insn
)
3024 ready_remove (readyp
, i
);
3030 /* Calculate difference of two statistics set WAS and NOW.
3031 Result returned in WAS. */
3033 rank_for_schedule_stats_diff (rank_for_schedule_stats_t
*was
,
3034 const rank_for_schedule_stats_t
*now
)
3036 for (int i
= 0; i
< RFS_N
; ++i
)
3037 was
->stats
[i
] = now
->stats
[i
] - was
->stats
[i
];
3040 /* Print rank_for_schedule statistics. */
3042 print_rank_for_schedule_stats (const char *prefix
,
3043 const rank_for_schedule_stats_t
*stats
,
3044 struct ready_list
*ready
)
3046 for (int i
= 0; i
< RFS_N
; ++i
)
3047 if (stats
->stats
[i
])
3049 fprintf (sched_dump
, "%s%20s: %u", prefix
, rfs_str
[i
], stats
->stats
[i
]);
3052 /* Print out insns that won due to RFS_<I>. */
3054 rtx_insn
**p
= ready_lastpos (ready
);
3056 fprintf (sched_dump
, ":");
3057 /* Start with 1 since least-priority insn didn't have any wins. */
3058 for (int j
= 1; j
< ready
->n_ready
; ++j
)
3059 if (INSN_LAST_RFS_WIN (p
[j
]) == i
)
3060 fprintf (sched_dump
, " %s",
3061 (*current_sched_info
->print_insn
) (p
[j
], 0));
3063 fprintf (sched_dump
, "\n");
3067 /* Separate DEBUG_INSNS from normal insns. DEBUG_INSNs go to the end
3070 ready_sort_debug (struct ready_list
*ready
)
3073 rtx_insn
**first
= ready_lastpos (ready
);
3075 for (i
= 0; i
< ready
->n_ready
; ++i
)
3076 if (!DEBUG_INSN_P (first
[i
]))
3077 INSN_RFS_DEBUG_ORIG_ORDER (first
[i
]) = i
;
3079 qsort (first
, ready
->n_ready
, sizeof (rtx
), rank_for_schedule_debug
);
3082 /* Sort non-debug insns in the ready list READY by ascending priority.
3083 Assumes that all debug insns are separated from the real insns. */
3085 ready_sort_real (struct ready_list
*ready
)
3088 rtx_insn
**first
= ready_lastpos (ready
);
3089 int n_ready_real
= ready
->n_ready
- ready
->n_debug
;
3091 if (sched_pressure
== SCHED_PRESSURE_WEIGHTED
)
3092 for (i
= 0; i
< n_ready_real
; ++i
)
3093 setup_insn_reg_pressure_info (first
[i
]);
3094 else if (sched_pressure
== SCHED_PRESSURE_MODEL
3095 && model_curr_point
< model_num_insns
)
3096 model_set_excess_costs (first
, n_ready_real
);
3098 rank_for_schedule_stats_t stats1
;
3099 if (sched_verbose
>= 4)
3100 stats1
= rank_for_schedule_stats
;
3102 if (n_ready_real
== 2)
3103 swap_sort (first
, n_ready_real
);
3104 else if (n_ready_real
> 2)
3105 qsort (first
, n_ready_real
, sizeof (rtx
), rank_for_schedule
);
3107 if (sched_verbose
>= 4)
3109 rank_for_schedule_stats_diff (&stats1
, &rank_for_schedule_stats
);
3110 print_rank_for_schedule_stats (";;\t\t", &stats1
, ready
);
3114 /* Sort the ready list READY by ascending priority. */
3116 ready_sort (struct ready_list
*ready
)
3118 if (ready
->n_debug
> 0)
3119 ready_sort_debug (ready
);
3121 ready_sort_real (ready
);
3124 /* PREV is an insn that is ready to execute. Adjust its priority if that
3125 will help shorten or lengthen register lifetimes as appropriate. Also
3126 provide a hook for the target to tweak itself. */
3128 HAIFA_INLINE
static void
3129 adjust_priority (rtx_insn
*prev
)
3131 /* ??? There used to be code here to try and estimate how an insn
3132 affected register lifetimes, but it did it by looking at REG_DEAD
3133 notes, which we removed in schedule_region. Nor did it try to
3134 take into account register pressure or anything useful like that.
3136 Revisit when we have a machine model to work with and not before. */
3138 if (targetm
.sched
.adjust_priority
)
3139 INSN_PRIORITY (prev
) =
3140 targetm
.sched
.adjust_priority (prev
, INSN_PRIORITY (prev
));
3143 /* Advance DFA state STATE on one cycle. */
3145 advance_state (state_t state
)
3147 if (targetm
.sched
.dfa_pre_advance_cycle
)
3148 targetm
.sched
.dfa_pre_advance_cycle ();
3150 if (targetm
.sched
.dfa_pre_cycle_insn
)
3151 state_transition (state
,
3152 targetm
.sched
.dfa_pre_cycle_insn ());
3154 state_transition (state
, NULL
);
3156 if (targetm
.sched
.dfa_post_cycle_insn
)
3157 state_transition (state
,
3158 targetm
.sched
.dfa_post_cycle_insn ());
3160 if (targetm
.sched
.dfa_post_advance_cycle
)
3161 targetm
.sched
.dfa_post_advance_cycle ();
3164 /* Advance time on one cycle. */
3165 HAIFA_INLINE
static void
3166 advance_one_cycle (void)
3168 advance_state (curr_state
);
3169 if (sched_verbose
>= 4)
3170 fprintf (sched_dump
, ";;\tAdvance the current state.\n");
3173 /* Update register pressure after scheduling INSN. */
3175 update_register_pressure (rtx_insn
*insn
)
3177 struct reg_use_data
*use
;
3178 struct reg_set_data
*set
;
3180 gcc_checking_assert (!DEBUG_INSN_P (insn
));
3182 for (use
= INSN_REG_USE_LIST (insn
); use
!= NULL
; use
= use
->next_insn_use
)
3183 if (dying_use_p (use
))
3184 mark_regno_birth_or_death (curr_reg_live
, curr_reg_pressure
,
3186 for (set
= INSN_REG_SET_LIST (insn
); set
!= NULL
; set
= set
->next_insn_set
)
3187 mark_regno_birth_or_death (curr_reg_live
, curr_reg_pressure
,
3191 /* Set up or update (if UPDATE_P) max register pressure (see its
3192 meaning in sched-int.h::_haifa_insn_data) for all current BB insns
3193 after insn AFTER. */
3195 setup_insn_max_reg_pressure (rtx_insn
*after
, bool update_p
)
3200 static int max_reg_pressure
[N_REG_CLASSES
];
3202 save_reg_pressure ();
3203 for (i
= 0; i
< ira_pressure_classes_num
; i
++)
3204 max_reg_pressure
[ira_pressure_classes
[i
]]
3205 = curr_reg_pressure
[ira_pressure_classes
[i
]];
3206 for (insn
= NEXT_INSN (after
);
3207 insn
!= NULL_RTX
&& ! BARRIER_P (insn
)
3208 && BLOCK_FOR_INSN (insn
) == BLOCK_FOR_INSN (after
);
3209 insn
= NEXT_INSN (insn
))
3210 if (NONDEBUG_INSN_P (insn
))
3213 for (i
= 0; i
< ira_pressure_classes_num
; i
++)
3215 p
= max_reg_pressure
[ira_pressure_classes
[i
]];
3216 if (INSN_MAX_REG_PRESSURE (insn
)[i
] != p
)
3219 INSN_MAX_REG_PRESSURE (insn
)[i
]
3220 = max_reg_pressure
[ira_pressure_classes
[i
]];
3223 if (update_p
&& eq_p
)
3225 update_register_pressure (insn
);
3226 for (i
= 0; i
< ira_pressure_classes_num
; i
++)
3227 if (max_reg_pressure
[ira_pressure_classes
[i
]]
3228 < curr_reg_pressure
[ira_pressure_classes
[i
]])
3229 max_reg_pressure
[ira_pressure_classes
[i
]]
3230 = curr_reg_pressure
[ira_pressure_classes
[i
]];
3232 restore_reg_pressure ();
3235 /* Update the current register pressure after scheduling INSN. Update
3236 also max register pressure for unscheduled insns of the current
3239 update_reg_and_insn_max_reg_pressure (rtx_insn
*insn
)
3242 int before
[N_REG_CLASSES
];
3244 for (i
= 0; i
< ira_pressure_classes_num
; i
++)
3245 before
[i
] = curr_reg_pressure
[ira_pressure_classes
[i
]];
3246 update_register_pressure (insn
);
3247 for (i
= 0; i
< ira_pressure_classes_num
; i
++)
3248 if (curr_reg_pressure
[ira_pressure_classes
[i
]] != before
[i
])
3250 if (i
< ira_pressure_classes_num
)
3251 setup_insn_max_reg_pressure (insn
, true);
3254 /* Set up register pressure at the beginning of basic block BB whose
3255 insns starting after insn AFTER. Set up also max register pressure
3256 for all insns of the basic block. */
3258 sched_setup_bb_reg_pressure_info (basic_block bb
, rtx_insn
*after
)
3260 gcc_assert (sched_pressure
== SCHED_PRESSURE_WEIGHTED
);
3261 initiate_bb_reg_pressure_info (bb
);
3262 setup_insn_max_reg_pressure (after
, false);
3265 /* If doing predication while scheduling, verify whether INSN, which
3266 has just been scheduled, clobbers the conditions of any
3267 instructions that must be predicated in order to break their
3268 dependencies. If so, remove them from the queues so that they will
3269 only be scheduled once their control dependency is resolved. */
3272 check_clobbered_conditions (rtx_insn
*insn
)
3277 if ((current_sched_info
->flags
& DO_PREDICATION
) == 0)
3280 find_all_hard_reg_sets (insn
, &t
, true);
3283 for (i
= 0; i
< ready
.n_ready
; i
++)
3285 rtx_insn
*x
= ready_element (&ready
, i
);
3286 if (TODO_SPEC (x
) == DEP_CONTROL
&& cond_clobbered_p (x
, t
))
3288 ready_remove_insn (x
);
3292 for (i
= 0; i
<= max_insn_queue_index
; i
++)
3294 rtx_insn_list
*link
;
3295 int q
= NEXT_Q_AFTER (q_ptr
, i
);
3298 for (link
= insn_queue
[q
]; link
; link
= link
->next ())
3300 rtx_insn
*x
= link
->insn ();
3301 if (TODO_SPEC (x
) == DEP_CONTROL
&& cond_clobbered_p (x
, t
))
3310 /* Return (in order):
3312 - positive if INSN adversely affects the pressure on one
3315 - negative if INSN reduces the pressure on one register class
3317 - 0 if INSN doesn't affect the pressure on any register class. */
3320 model_classify_pressure (struct model_insn_info
*insn
)
3322 struct reg_pressure_data
*reg_pressure
;
3323 int death
[N_REG_CLASSES
];
3326 calculate_reg_deaths (insn
->insn
, death
);
3327 reg_pressure
= INSN_REG_PRESSURE (insn
->insn
);
3329 for (pci
= 0; pci
< ira_pressure_classes_num
; pci
++)
3331 cl
= ira_pressure_classes
[pci
];
3332 if (death
[cl
] < reg_pressure
[pci
].set_increase
)
3334 sum
+= reg_pressure
[pci
].set_increase
- death
[cl
];
3339 /* Return true if INSN1 should come before INSN2 in the model schedule. */
3342 model_order_p (struct model_insn_info
*insn1
, struct model_insn_info
*insn2
)
3344 unsigned int height1
, height2
;
3345 unsigned int priority1
, priority2
;
3347 /* Prefer instructions with a higher model priority. */
3348 if (insn1
->model_priority
!= insn2
->model_priority
)
3349 return insn1
->model_priority
> insn2
->model_priority
;
3351 /* Combine the length of the longest path of satisfied true dependencies
3352 that leads to each instruction (depth) with the length of the longest
3353 path of any dependencies that leads from the instruction (alap).
3354 Prefer instructions with the greatest combined length. If the combined
3355 lengths are equal, prefer instructions with the greatest depth.
3357 The idea is that, if we have a set S of "equal" instructions that each
3358 have ALAP value X, and we pick one such instruction I, any true-dependent
3359 successors of I that have ALAP value X - 1 should be preferred over S.
3360 This encourages the schedule to be "narrow" rather than "wide".
3361 However, if I is a low-priority instruction that we decided to
3362 schedule because of its model_classify_pressure, and if there
3363 is a set of higher-priority instructions T, the aforementioned
3364 successors of I should not have the edge over T. */
3365 height1
= insn1
->depth
+ insn1
->alap
;
3366 height2
= insn2
->depth
+ insn2
->alap
;
3367 if (height1
!= height2
)
3368 return height1
> height2
;
3369 if (insn1
->depth
!= insn2
->depth
)
3370 return insn1
->depth
> insn2
->depth
;
3372 /* We have no real preference between INSN1 an INSN2 as far as attempts
3373 to reduce pressure go. Prefer instructions with higher priorities. */
3374 priority1
= INSN_PRIORITY (insn1
->insn
);
3375 priority2
= INSN_PRIORITY (insn2
->insn
);
3376 if (priority1
!= priority2
)
3377 return priority1
> priority2
;
3379 /* Use the original rtl sequence as a tie-breaker. */
3380 return insn1
< insn2
;
3383 /* Add INSN to the model worklist immediately after PREV. Add it to the
3384 beginning of the list if PREV is null. */
3387 model_add_to_worklist_at (struct model_insn_info
*insn
,
3388 struct model_insn_info
*prev
)
3390 gcc_assert (QUEUE_INDEX (insn
->insn
) == QUEUE_NOWHERE
);
3391 QUEUE_INDEX (insn
->insn
) = QUEUE_READY
;
3396 insn
->next
= prev
->next
;
3401 insn
->next
= model_worklist
;
3402 model_worklist
= insn
;
3405 insn
->next
->prev
= insn
;
3408 /* Remove INSN from the model worklist. */
3411 model_remove_from_worklist (struct model_insn_info
*insn
)
3413 gcc_assert (QUEUE_INDEX (insn
->insn
) == QUEUE_READY
);
3414 QUEUE_INDEX (insn
->insn
) = QUEUE_NOWHERE
;
3417 insn
->prev
->next
= insn
->next
;
3419 model_worklist
= insn
->next
;
3421 insn
->next
->prev
= insn
->prev
;
3424 /* Add INSN to the model worklist. Start looking for a suitable position
3425 between neighbors PREV and NEXT, testing at most MAX_SCHED_READY_INSNS
3426 insns either side. A null PREV indicates the beginning of the list and
3427 a null NEXT indicates the end. */
3430 model_add_to_worklist (struct model_insn_info
*insn
,
3431 struct model_insn_info
*prev
,
3432 struct model_insn_info
*next
)
3436 count
= MAX_SCHED_READY_INSNS
;
3437 if (count
> 0 && prev
&& model_order_p (insn
, prev
))
3443 while (count
> 0 && prev
&& model_order_p (insn
, prev
));
3445 while (count
> 0 && next
&& model_order_p (next
, insn
))
3451 model_add_to_worklist_at (insn
, prev
);
3454 /* INSN may now have a higher priority (in the model_order_p sense)
3455 than before. Move it up the worklist if necessary. */
3458 model_promote_insn (struct model_insn_info
*insn
)
3460 struct model_insn_info
*prev
;
3464 count
= MAX_SCHED_READY_INSNS
;
3465 while (count
> 0 && prev
&& model_order_p (insn
, prev
))
3470 if (prev
!= insn
->prev
)
3472 model_remove_from_worklist (insn
);
3473 model_add_to_worklist_at (insn
, prev
);
3477 /* Add INSN to the end of the model schedule. */
3480 model_add_to_schedule (rtx_insn
*insn
)
3484 gcc_assert (QUEUE_INDEX (insn
) == QUEUE_NOWHERE
);
3485 QUEUE_INDEX (insn
) = QUEUE_SCHEDULED
;
3487 point
= model_schedule
.length ();
3488 model_schedule
.quick_push (insn
);
3489 INSN_MODEL_INDEX (insn
) = point
+ 1;
3492 /* Analyze the instructions that are to be scheduled, setting up
3493 MODEL_INSN_INFO (...) and model_num_insns accordingly. Add ready
3494 instructions to model_worklist. */
3497 model_analyze_insns (void)
3499 rtx_insn
*start
, *end
, *iter
;
3500 sd_iterator_def sd_it
;
3502 struct model_insn_info
*insn
, *con
;
3504 model_num_insns
= 0;
3505 start
= PREV_INSN (current_sched_info
->next_tail
);
3506 end
= current_sched_info
->prev_head
;
3507 for (iter
= start
; iter
!= end
; iter
= PREV_INSN (iter
))
3508 if (NONDEBUG_INSN_P (iter
))
3510 insn
= MODEL_INSN_INFO (iter
);
3512 FOR_EACH_DEP (iter
, SD_LIST_FORW
, sd_it
, dep
)
3514 con
= MODEL_INSN_INFO (DEP_CON (dep
));
3515 if (con
->insn
&& insn
->alap
< con
->alap
+ 1)
3516 insn
->alap
= con
->alap
+ 1;
3519 insn
->old_queue
= QUEUE_INDEX (iter
);
3520 QUEUE_INDEX (iter
) = QUEUE_NOWHERE
;
3522 insn
->unscheduled_preds
= dep_list_size (iter
, SD_LIST_HARD_BACK
);
3523 if (insn
->unscheduled_preds
== 0)
3524 model_add_to_worklist (insn
, NULL
, model_worklist
);
3530 /* The global state describes the register pressure at the start of the
3531 model schedule. Initialize GROUP accordingly. */
3534 model_init_pressure_group (struct model_pressure_group
*group
)
3538 for (pci
= 0; pci
< ira_pressure_classes_num
; pci
++)
3540 cl
= ira_pressure_classes
[pci
];
3541 group
->limits
[pci
].pressure
= curr_reg_pressure
[cl
];
3542 group
->limits
[pci
].point
= 0;
3544 /* Use index model_num_insns to record the state after the last
3545 instruction in the model schedule. */
3546 group
->model
= XNEWVEC (struct model_pressure_data
,
3547 (model_num_insns
+ 1) * ira_pressure_classes_num
);
3550 /* Record that MODEL_REF_PRESSURE (GROUP, POINT, PCI) is PRESSURE.
3551 Update the maximum pressure for the whole schedule. */
3554 model_record_pressure (struct model_pressure_group
*group
,
3555 int point
, int pci
, int pressure
)
3557 MODEL_REF_PRESSURE (group
, point
, pci
) = pressure
;
3558 if (group
->limits
[pci
].pressure
< pressure
)
3560 group
->limits
[pci
].pressure
= pressure
;
3561 group
->limits
[pci
].point
= point
;
3565 /* INSN has just been added to the end of the model schedule. Record its
3566 register-pressure information. */
3569 model_record_pressures (struct model_insn_info
*insn
)
3571 struct reg_pressure_data
*reg_pressure
;
3572 int point
, pci
, cl
, delta
;
3573 int death
[N_REG_CLASSES
];
3575 point
= model_index (insn
->insn
);
3576 if (sched_verbose
>= 2)
3580 fprintf (sched_dump
, "\n;;\tModel schedule:\n;;\n");
3581 fprintf (sched_dump
, ";;\t| idx insn | mpri hght dpth prio |\n");
3583 fprintf (sched_dump
, ";;\t| %3d %4d | %4d %4d %4d %4d | %-30s ",
3584 point
, INSN_UID (insn
->insn
), insn
->model_priority
,
3585 insn
->depth
+ insn
->alap
, insn
->depth
,
3586 INSN_PRIORITY (insn
->insn
),
3587 str_pattern_slim (PATTERN (insn
->insn
)));
3589 calculate_reg_deaths (insn
->insn
, death
);
3590 reg_pressure
= INSN_REG_PRESSURE (insn
->insn
);
3591 for (pci
= 0; pci
< ira_pressure_classes_num
; pci
++)
3593 cl
= ira_pressure_classes
[pci
];
3594 delta
= reg_pressure
[pci
].set_increase
- death
[cl
];
3595 if (sched_verbose
>= 2)
3596 fprintf (sched_dump
, " %s:[%d,%+d]", reg_class_names
[cl
],
3597 curr_reg_pressure
[cl
], delta
);
3598 model_record_pressure (&model_before_pressure
, point
, pci
,
3599 curr_reg_pressure
[cl
]);
3601 if (sched_verbose
>= 2)
3602 fprintf (sched_dump
, "\n");
3605 /* All instructions have been added to the model schedule. Record the
3606 final register pressure in GROUP and set up all MODEL_MAX_PRESSUREs. */
3609 model_record_final_pressures (struct model_pressure_group
*group
)
3611 int point
, pci
, max_pressure
, ref_pressure
, cl
;
3613 for (pci
= 0; pci
< ira_pressure_classes_num
; pci
++)
3615 /* Record the final pressure for this class. */
3616 cl
= ira_pressure_classes
[pci
];
3617 point
= model_num_insns
;
3618 ref_pressure
= curr_reg_pressure
[cl
];
3619 model_record_pressure (group
, point
, pci
, ref_pressure
);
3621 /* Record the original maximum pressure. */
3622 group
->limits
[pci
].orig_pressure
= group
->limits
[pci
].pressure
;
3624 /* Update the MODEL_MAX_PRESSURE for every point of the schedule. */
3625 max_pressure
= ref_pressure
;
3626 MODEL_MAX_PRESSURE (group
, point
, pci
) = max_pressure
;
3630 ref_pressure
= MODEL_REF_PRESSURE (group
, point
, pci
);
3631 max_pressure
= MAX (max_pressure
, ref_pressure
);
3632 MODEL_MAX_PRESSURE (group
, point
, pci
) = max_pressure
;
3637 /* Update all successors of INSN, given that INSN has just been scheduled. */
3640 model_add_successors_to_worklist (struct model_insn_info
*insn
)
3642 sd_iterator_def sd_it
;
3643 struct model_insn_info
*con
;
3646 FOR_EACH_DEP (insn
->insn
, SD_LIST_FORW
, sd_it
, dep
)
3648 con
= MODEL_INSN_INFO (DEP_CON (dep
));
3649 /* Ignore debug instructions, and instructions from other blocks. */
3652 con
->unscheduled_preds
--;
3654 /* Update the depth field of each true-dependent successor.
3655 Increasing the depth gives them a higher priority than
3657 if (DEP_TYPE (dep
) == REG_DEP_TRUE
&& con
->depth
< insn
->depth
+ 1)
3659 con
->depth
= insn
->depth
+ 1;
3660 if (QUEUE_INDEX (con
->insn
) == QUEUE_READY
)
3661 model_promote_insn (con
);
3664 /* If this is a true dependency, or if there are no remaining
3665 dependencies for CON (meaning that CON only had non-true
3666 dependencies), make sure that CON is on the worklist.
3667 We don't bother otherwise because it would tend to fill the
3668 worklist with a lot of low-priority instructions that are not
3669 yet ready to issue. */
3670 if ((con
->depth
> 0 || con
->unscheduled_preds
== 0)
3671 && QUEUE_INDEX (con
->insn
) == QUEUE_NOWHERE
)
3672 model_add_to_worklist (con
, insn
, insn
->next
);
3677 /* Give INSN a higher priority than any current instruction, then give
3678 unscheduled predecessors of INSN a higher priority still. If any of
3679 those predecessors are not on the model worklist, do the same for its
3680 predecessors, and so on. */
3683 model_promote_predecessors (struct model_insn_info
*insn
)
3685 struct model_insn_info
*pro
, *first
;
3686 sd_iterator_def sd_it
;
3689 if (sched_verbose
>= 7)
3690 fprintf (sched_dump
, ";;\t+--- priority of %d = %d, priority of",
3691 INSN_UID (insn
->insn
), model_next_priority
);
3692 insn
->model_priority
= model_next_priority
++;
3693 model_remove_from_worklist (insn
);
3694 model_add_to_worklist_at (insn
, NULL
);
3699 FOR_EACH_DEP (insn
->insn
, SD_LIST_HARD_BACK
, sd_it
, dep
)
3701 pro
= MODEL_INSN_INFO (DEP_PRO (dep
));
3702 /* The first test is to ignore debug instructions, and instructions
3703 from other blocks. */
3705 && pro
->model_priority
!= model_next_priority
3706 && QUEUE_INDEX (pro
->insn
) != QUEUE_SCHEDULED
)
3708 pro
->model_priority
= model_next_priority
;
3709 if (sched_verbose
>= 7)
3710 fprintf (sched_dump
, " %d", INSN_UID (pro
->insn
));
3711 if (QUEUE_INDEX (pro
->insn
) == QUEUE_READY
)
3713 /* PRO is already in the worklist, but it now has
3714 a higher priority than before. Move it at the
3715 appropriate place. */
3716 model_remove_from_worklist (pro
);
3717 model_add_to_worklist (pro
, NULL
, model_worklist
);
3721 /* PRO isn't in the worklist. Recursively process
3722 its predecessors until we find one that is. */
3733 if (sched_verbose
>= 7)
3734 fprintf (sched_dump
, " = %d\n", model_next_priority
);
3735 model_next_priority
++;
3738 /* Pick one instruction from model_worklist and process it. */
3741 model_choose_insn (void)
3743 struct model_insn_info
*insn
, *fallback
;
3746 if (sched_verbose
>= 7)
3748 fprintf (sched_dump
, ";;\t+--- worklist:\n");
3749 insn
= model_worklist
;
3750 count
= MAX_SCHED_READY_INSNS
;
3751 while (count
> 0 && insn
)
3753 fprintf (sched_dump
, ";;\t+--- %d [%d, %d, %d, %d]\n",
3754 INSN_UID (insn
->insn
), insn
->model_priority
,
3755 insn
->depth
+ insn
->alap
, insn
->depth
,
3756 INSN_PRIORITY (insn
->insn
));
3762 /* Look for a ready instruction whose model_classify_priority is zero
3763 or negative, picking the highest-priority one. Adding such an
3764 instruction to the schedule now should do no harm, and may actually
3767 Failing that, see whether there is an instruction with the highest
3768 extant model_priority that is not yet ready, but which would reduce
3769 pressure if it became ready. This is designed to catch cases like:
3771 (set (mem (reg R1)) (reg R2))
3773 where the instruction is the last remaining use of R1 and where the
3774 value of R2 is not yet available (or vice versa). The death of R1
3775 means that this instruction already reduces pressure. It is of
3776 course possible that the computation of R2 involves other registers
3777 that are hard to kill, but such cases are rare enough for this
3778 heuristic to be a win in general.
3780 Failing that, just pick the highest-priority instruction in the
3782 count
= MAX_SCHED_READY_INSNS
;
3783 insn
= model_worklist
;
3787 if (count
== 0 || !insn
)
3789 insn
= fallback
? fallback
: model_worklist
;
3792 if (insn
->unscheduled_preds
)
3794 if (model_worklist
->model_priority
== insn
->model_priority
3796 && model_classify_pressure (insn
) < 0)
3801 if (model_classify_pressure (insn
) <= 0)
3808 if (sched_verbose
>= 7 && insn
!= model_worklist
)
3810 if (insn
->unscheduled_preds
)
3811 fprintf (sched_dump
, ";;\t+--- promoting insn %d, with dependencies\n",
3812 INSN_UID (insn
->insn
));
3814 fprintf (sched_dump
, ";;\t+--- promoting insn %d, which is ready\n",
3815 INSN_UID (insn
->insn
));
3817 if (insn
->unscheduled_preds
)
3818 /* INSN isn't yet ready to issue. Give all its predecessors the
3819 highest priority. */
3820 model_promote_predecessors (insn
);
3823 /* INSN is ready. Add it to the end of model_schedule and
3824 process its successors. */
3825 model_add_successors_to_worklist (insn
);
3826 model_remove_from_worklist (insn
);
3827 model_add_to_schedule (insn
->insn
);
3828 model_record_pressures (insn
);
3829 update_register_pressure (insn
->insn
);
3833 /* Restore all QUEUE_INDEXs to the values that they had before
3834 model_start_schedule was called. */
3837 model_reset_queue_indices (void)
3842 FOR_EACH_VEC_ELT (model_schedule
, i
, insn
)
3843 QUEUE_INDEX (insn
) = MODEL_INSN_INFO (insn
)->old_queue
;
3846 /* We have calculated the model schedule and spill costs. Print a summary
3850 model_dump_pressure_summary (void)
3854 fprintf (sched_dump
, ";; Pressure summary:");
3855 for (pci
= 0; pci
< ira_pressure_classes_num
; pci
++)
3857 cl
= ira_pressure_classes
[pci
];
3858 fprintf (sched_dump
, " %s:%d", reg_class_names
[cl
],
3859 model_before_pressure
.limits
[pci
].pressure
);
3861 fprintf (sched_dump
, "\n\n");
3864 /* Initialize the SCHED_PRESSURE_MODEL information for the current
3865 scheduling region. */
3868 model_start_schedule (basic_block bb
)
3870 model_next_priority
= 1;
3871 model_schedule
.create (sched_max_luid
);
3872 model_insns
= XCNEWVEC (struct model_insn_info
, sched_max_luid
);
3874 gcc_assert (bb
== BLOCK_FOR_INSN (NEXT_INSN (current_sched_info
->prev_head
)));
3875 initiate_reg_pressure_info (df_get_live_in (bb
));
3877 model_analyze_insns ();
3878 model_init_pressure_group (&model_before_pressure
);
3879 while (model_worklist
)
3880 model_choose_insn ();
3881 gcc_assert (model_num_insns
== (int) model_schedule
.length ());
3882 if (sched_verbose
>= 2)
3883 fprintf (sched_dump
, "\n");
3885 model_record_final_pressures (&model_before_pressure
);
3886 model_reset_queue_indices ();
3888 XDELETEVEC (model_insns
);
3890 model_curr_point
= 0;
3891 initiate_reg_pressure_info (df_get_live_in (bb
));
3892 if (sched_verbose
>= 1)
3893 model_dump_pressure_summary ();
3896 /* Free the information associated with GROUP. */
3899 model_finalize_pressure_group (struct model_pressure_group
*group
)
3901 XDELETEVEC (group
->model
);
3904 /* Free the information created by model_start_schedule. */
3907 model_end_schedule (void)
3909 model_finalize_pressure_group (&model_before_pressure
);
3910 model_schedule
.release ();
3913 /* Prepare reg pressure scheduling for basic block BB. */
3915 sched_pressure_start_bb (basic_block bb
)
3917 /* Set the number of available registers for each class taking into account
3918 relative probability of current basic block versus function prologue and
3920 * If the basic block executes much more often than the prologue/epilogue
3921 (e.g., inside a hot loop), then cost of spill in the prologue is close to
3922 nil, so the effective number of available registers is
3923 (ira_class_hard_regs_num[cl] - 0).
3924 * If the basic block executes as often as the prologue/epilogue,
3925 then spill in the block is as costly as in the prologue, so the effective
3926 number of available registers is
3927 (ira_class_hard_regs_num[cl] - call_used_regs_num[cl]).
3928 Note that all-else-equal, we prefer to spill in the prologue, since that
3929 allows "extra" registers for other basic blocks of the function.
3930 * If the basic block is on the cold path of the function and executes
3931 rarely, then we should always prefer to spill in the block, rather than
3932 in the prologue/epilogue. The effective number of available register is
3933 (ira_class_hard_regs_num[cl] - call_used_regs_num[cl]). */
3936 int entry_freq
= ENTRY_BLOCK_PTR_FOR_FN (cfun
)->frequency
;
3937 int bb_freq
= bb
->frequency
;
3941 if (entry_freq
== 0)
3942 entry_freq
= bb_freq
= 1;
3944 if (bb_freq
< entry_freq
)
3945 bb_freq
= entry_freq
;
3947 for (i
= 0; i
< ira_pressure_classes_num
; ++i
)
3949 enum reg_class cl
= ira_pressure_classes
[i
];
3950 sched_class_regs_num
[cl
] = ira_class_hard_regs_num
[cl
];
3951 sched_class_regs_num
[cl
]
3952 -= (call_used_regs_num
[cl
] * entry_freq
) / bb_freq
;
3956 if (sched_pressure
== SCHED_PRESSURE_MODEL
)
3957 model_start_schedule (bb
);
3960 /* A structure that holds local state for the loop in schedule_block. */
3961 struct sched_block_state
3963 /* True if no real insns have been scheduled in the current cycle. */
3964 bool first_cycle_insn_p
;
3965 /* True if a shadow insn has been scheduled in the current cycle, which
3966 means that no more normal insns can be issued. */
3967 bool shadows_only_p
;
3968 /* True if we're winding down a modulo schedule, which means that we only
3969 issue insns with INSN_EXACT_TICK set. */
3970 bool modulo_epilogue
;
3971 /* Initialized with the machine's issue rate every cycle, and updated
3972 by calls to the variable_issue hook. */
3976 /* INSN is the "currently executing insn". Launch each insn which was
3977 waiting on INSN. READY is the ready list which contains the insns
3978 that are ready to fire. CLOCK is the current cycle. The function
3979 returns necessary cycle advance after issuing the insn (it is not
3980 zero for insns in a schedule group). */
3983 schedule_insn (rtx_insn
*insn
)
3985 sd_iterator_def sd_it
;
3990 if (sched_verbose
>= 1)
3992 struct reg_pressure_data
*pressure_info
;
3993 fprintf (sched_dump
, ";;\t%3i--> %s %-40s:",
3994 clock_var
, (*current_sched_info
->print_insn
) (insn
, 1),
3995 str_pattern_slim (PATTERN (insn
)));
3997 if (recog_memoized (insn
) < 0)
3998 fprintf (sched_dump
, "nothing");
4000 print_reservation (sched_dump
, insn
);
4001 pressure_info
= INSN_REG_PRESSURE (insn
);
4002 if (pressure_info
!= NULL
)
4004 fputc (':', sched_dump
);
4005 for (i
= 0; i
< ira_pressure_classes_num
; i
++)
4006 fprintf (sched_dump
, "%s%s%+d(%d)",
4007 scheduled_insns
.length () > 1
4009 < INSN_LUID (scheduled_insns
[scheduled_insns
.length () - 2]) ? "@" : "",
4010 reg_class_names
[ira_pressure_classes
[i
]],
4011 pressure_info
[i
].set_increase
, pressure_info
[i
].change
);
4013 if (sched_pressure
== SCHED_PRESSURE_MODEL
4014 && model_curr_point
< model_num_insns
4015 && model_index (insn
) == model_curr_point
)
4016 fprintf (sched_dump
, ":model %d", model_curr_point
);
4017 fputc ('\n', sched_dump
);
4020 if (sched_pressure
== SCHED_PRESSURE_WEIGHTED
&& !DEBUG_INSN_P (insn
))
4021 update_reg_and_insn_max_reg_pressure (insn
);
4023 /* Scheduling instruction should have all its dependencies resolved and
4024 should have been removed from the ready list. */
4025 gcc_assert (sd_lists_empty_p (insn
, SD_LIST_HARD_BACK
));
4027 /* Reset debug insns invalidated by moving this insn. */
4028 if (MAY_HAVE_DEBUG_INSNS
&& !DEBUG_INSN_P (insn
))
4029 for (sd_it
= sd_iterator_start (insn
, SD_LIST_BACK
);
4030 sd_iterator_cond (&sd_it
, &dep
);)
4032 rtx_insn
*dbg
= DEP_PRO (dep
);
4033 struct reg_use_data
*use
, *next
;
4035 if (DEP_STATUS (dep
) & DEP_CANCELLED
)
4037 sd_iterator_next (&sd_it
);
4041 gcc_assert (DEBUG_INSN_P (dbg
));
4043 if (sched_verbose
>= 6)
4044 fprintf (sched_dump
, ";;\t\tresetting: debug insn %d\n",
4047 /* ??? Rather than resetting the debug insn, we might be able
4048 to emit a debug temp before the just-scheduled insn, but
4049 this would involve checking that the expression at the
4050 point of the debug insn is equivalent to the expression
4051 before the just-scheduled insn. They might not be: the
4052 expression in the debug insn may depend on other insns not
4053 yet scheduled that set MEMs, REGs or even other debug
4054 insns. It's not clear that attempting to preserve debug
4055 information in these cases is worth the effort, given how
4056 uncommon these resets are and the likelihood that the debug
4057 temps introduced won't survive the schedule change. */
4058 INSN_VAR_LOCATION_LOC (dbg
) = gen_rtx_UNKNOWN_VAR_LOC ();
4059 df_insn_rescan (dbg
);
4061 /* Unknown location doesn't use any registers. */
4062 for (use
= INSN_REG_USE_LIST (dbg
); use
!= NULL
; use
= next
)
4064 struct reg_use_data
*prev
= use
;
4066 /* Remove use from the cyclic next_regno_use chain first. */
4067 while (prev
->next_regno_use
!= use
)
4068 prev
= prev
->next_regno_use
;
4069 prev
->next_regno_use
= use
->next_regno_use
;
4070 next
= use
->next_insn_use
;
4073 INSN_REG_USE_LIST (dbg
) = NULL
;
4075 /* We delete rather than resolve these deps, otherwise we
4076 crash in sched_free_deps(), because forward deps are
4077 expected to be released before backward deps. */
4078 sd_delete_dep (sd_it
);
4081 gcc_assert (QUEUE_INDEX (insn
) == QUEUE_NOWHERE
);
4082 QUEUE_INDEX (insn
) = QUEUE_SCHEDULED
;
4084 if (sched_pressure
== SCHED_PRESSURE_MODEL
4085 && model_curr_point
< model_num_insns
4086 && NONDEBUG_INSN_P (insn
))
4088 if (model_index (insn
) == model_curr_point
)
4091 while (model_curr_point
< model_num_insns
4092 && (QUEUE_INDEX (MODEL_INSN (model_curr_point
))
4093 == QUEUE_SCHEDULED
));
4095 model_recompute (insn
);
4096 model_update_limit_points ();
4097 update_register_pressure (insn
);
4098 if (sched_verbose
>= 2)
4099 print_curr_reg_pressure ();
4102 gcc_assert (INSN_TICK (insn
) >= MIN_TICK
);
4103 if (INSN_TICK (insn
) > clock_var
)
4104 /* INSN has been prematurely moved from the queue to the ready list.
4105 This is possible only if following flags are set. */
4106 gcc_assert (flag_sched_stalled_insns
|| sched_fusion
);
4108 /* ??? Probably, if INSN is scheduled prematurely, we should leave
4109 INSN_TICK untouched. This is a machine-dependent issue, actually. */
4110 INSN_TICK (insn
) = clock_var
;
4112 check_clobbered_conditions (insn
);
4114 /* Update dependent instructions. First, see if by scheduling this insn
4115 now we broke a dependence in a way that requires us to change another
4117 for (sd_it
= sd_iterator_start (insn
, SD_LIST_SPEC_BACK
);
4118 sd_iterator_cond (&sd_it
, &dep
); sd_iterator_next (&sd_it
))
4120 struct dep_replacement
*desc
= DEP_REPLACE (dep
);
4121 rtx_insn
*pro
= DEP_PRO (dep
);
4122 if (QUEUE_INDEX (pro
) != QUEUE_SCHEDULED
4123 && desc
!= NULL
&& desc
->insn
== pro
)
4124 apply_replacement (dep
, false);
4127 /* Go through and resolve forward dependencies. */
4128 for (sd_it
= sd_iterator_start (insn
, SD_LIST_FORW
);
4129 sd_iterator_cond (&sd_it
, &dep
);)
4131 rtx_insn
*next
= DEP_CON (dep
);
4132 bool cancelled
= (DEP_STATUS (dep
) & DEP_CANCELLED
) != 0;
4134 /* Resolve the dependence between INSN and NEXT.
4135 sd_resolve_dep () moves current dep to another list thus
4136 advancing the iterator. */
4137 sd_resolve_dep (sd_it
);
4141 if (must_restore_pattern_p (next
, dep
))
4142 restore_pattern (dep
, false);
4146 /* Don't bother trying to mark next as ready if insn is a debug
4147 insn. If insn is the last hard dependency, it will have
4148 already been discounted. */
4149 if (DEBUG_INSN_P (insn
) && !DEBUG_INSN_P (next
))
4152 if (!IS_SPECULATION_BRANCHY_CHECK_P (insn
))
4156 effective_cost
= try_ready (next
);
4158 if (effective_cost
>= 0
4159 && SCHED_GROUP_P (next
)
4160 && advance
< effective_cost
)
4161 advance
= effective_cost
;
4164 /* Check always has only one forward dependence (to the first insn in
4165 the recovery block), therefore, this will be executed only once. */
4167 gcc_assert (sd_lists_empty_p (insn
, SD_LIST_FORW
));
4168 fix_recovery_deps (RECOVERY_BLOCK (insn
));
4172 /* Annotate the instruction with issue information -- TImode
4173 indicates that the instruction is expected not to be able
4174 to issue on the same cycle as the previous insn. A machine
4175 may use this information to decide how the instruction should
4178 && GET_CODE (PATTERN (insn
)) != USE
4179 && GET_CODE (PATTERN (insn
)) != CLOBBER
4180 && !DEBUG_INSN_P (insn
))
4182 if (reload_completed
)
4183 PUT_MODE (insn
, clock_var
> last_clock_var
? TImode
: VOIDmode
);
4184 last_clock_var
= clock_var
;
4187 if (nonscheduled_insns_begin
!= NULL_RTX
)
4188 /* Indicate to debug counters that INSN is scheduled. */
4189 nonscheduled_insns_begin
= insn
;
4194 /* Functions for handling of notes. */
4196 /* Add note list that ends on FROM_END to the end of TO_ENDP. */
4198 concat_note_lists (rtx_insn
*from_end
, rtx_insn
**to_endp
)
4200 rtx_insn
*from_start
;
4202 /* It's easy when have nothing to concat. */
4203 if (from_end
== NULL
)
4206 /* It's also easy when destination is empty. */
4207 if (*to_endp
== NULL
)
4209 *to_endp
= from_end
;
4213 from_start
= from_end
;
4214 while (PREV_INSN (from_start
) != NULL
)
4215 from_start
= PREV_INSN (from_start
);
4217 SET_PREV_INSN (from_start
) = *to_endp
;
4218 SET_NEXT_INSN (*to_endp
) = from_start
;
4219 *to_endp
= from_end
;
4222 /* Delete notes between HEAD and TAIL and put them in the chain
4223 of notes ended by NOTE_LIST. */
4225 remove_notes (rtx_insn
*head
, rtx_insn
*tail
)
4227 rtx_insn
*next_tail
, *insn
, *next
;
4230 if (head
== tail
&& !INSN_P (head
))
4233 next_tail
= NEXT_INSN (tail
);
4234 for (insn
= head
; insn
!= next_tail
; insn
= next
)
4236 next
= NEXT_INSN (insn
);
4240 switch (NOTE_KIND (insn
))
4242 case NOTE_INSN_BASIC_BLOCK
:
4245 case NOTE_INSN_EPILOGUE_BEG
:
4249 add_reg_note (next
, REG_SAVE_NOTE
,
4250 GEN_INT (NOTE_INSN_EPILOGUE_BEG
));
4258 /* Add the note to list that ends at NOTE_LIST. */
4259 SET_PREV_INSN (insn
) = note_list
;
4260 SET_NEXT_INSN (insn
) = NULL_RTX
;
4262 SET_NEXT_INSN (note_list
) = insn
;
4267 gcc_assert ((sel_sched_p () || insn
!= tail
) && insn
!= head
);
4271 /* A structure to record enough data to allow us to backtrack the scheduler to
4272 a previous state. */
4273 struct haifa_saved_data
4275 /* Next entry on the list. */
4276 struct haifa_saved_data
*next
;
4278 /* Backtracking is associated with scheduling insns that have delay slots.
4279 DELAY_PAIR points to the structure that contains the insns involved, and
4280 the number of cycles between them. */
4281 struct delay_pair
*delay_pair
;
4283 /* Data used by the frontend (e.g. sched-ebb or sched-rgn). */
4284 void *fe_saved_data
;
4285 /* Data used by the backend. */
4286 void *be_saved_data
;
4288 /* Copies of global state. */
4289 int clock_var
, last_clock_var
;
4290 struct ready_list ready
;
4293 rtx_insn
*last_scheduled_insn
;
4294 rtx_insn
*last_nondebug_scheduled_insn
;
4295 rtx_insn
*nonscheduled_insns_begin
;
4296 int cycle_issued_insns
;
4298 /* Copies of state used in the inner loop of schedule_block. */
4299 struct sched_block_state sched_block
;
4301 /* We don't need to save q_ptr, as its value is arbitrary and we can set it
4302 to 0 when restoring. */
4304 rtx_insn_list
**insn_queue
;
4306 /* Describe pattern replacements that occurred since this backtrack point
4308 vec
<dep_t
> replacement_deps
;
4309 vec
<int> replace_apply
;
4311 /* A copy of the next-cycle replacement vectors at the time of the backtrack
4313 vec
<dep_t
> next_cycle_deps
;
4314 vec
<int> next_cycle_apply
;
4317 /* A record, in reverse order, of all scheduled insns which have delay slots
4318 and may require backtracking. */
4319 static struct haifa_saved_data
*backtrack_queue
;
4321 /* For every dependency of INSN, set the FEEDS_BACKTRACK_INSN bit according
4324 mark_backtrack_feeds (rtx_insn
*insn
, int set_p
)
4326 sd_iterator_def sd_it
;
4328 FOR_EACH_DEP (insn
, SD_LIST_HARD_BACK
, sd_it
, dep
)
4330 FEEDS_BACKTRACK_INSN (DEP_PRO (dep
)) = set_p
;
4334 /* Save the current scheduler state so that we can backtrack to it
4335 later if necessary. PAIR gives the insns that make it necessary to
4336 save this point. SCHED_BLOCK is the local state of schedule_block
4337 that need to be saved. */
4339 save_backtrack_point (struct delay_pair
*pair
,
4340 struct sched_block_state sched_block
)
4343 struct haifa_saved_data
*save
= XNEW (struct haifa_saved_data
);
4345 save
->curr_state
= xmalloc (dfa_state_size
);
4346 memcpy (save
->curr_state
, curr_state
, dfa_state_size
);
4348 save
->ready
.first
= ready
.first
;
4349 save
->ready
.n_ready
= ready
.n_ready
;
4350 save
->ready
.n_debug
= ready
.n_debug
;
4351 save
->ready
.veclen
= ready
.veclen
;
4352 save
->ready
.vec
= XNEWVEC (rtx_insn
*, ready
.veclen
);
4353 memcpy (save
->ready
.vec
, ready
.vec
, ready
.veclen
* sizeof (rtx
));
4355 save
->insn_queue
= XNEWVEC (rtx_insn_list
*, max_insn_queue_index
+ 1);
4356 save
->q_size
= q_size
;
4357 for (i
= 0; i
<= max_insn_queue_index
; i
++)
4359 int q
= NEXT_Q_AFTER (q_ptr
, i
);
4360 save
->insn_queue
[i
] = copy_INSN_LIST (insn_queue
[q
]);
4363 save
->clock_var
= clock_var
;
4364 save
->last_clock_var
= last_clock_var
;
4365 save
->cycle_issued_insns
= cycle_issued_insns
;
4366 save
->last_scheduled_insn
= last_scheduled_insn
;
4367 save
->last_nondebug_scheduled_insn
= last_nondebug_scheduled_insn
;
4368 save
->nonscheduled_insns_begin
= nonscheduled_insns_begin
;
4370 save
->sched_block
= sched_block
;
4372 save
->replacement_deps
.create (0);
4373 save
->replace_apply
.create (0);
4374 save
->next_cycle_deps
= next_cycle_replace_deps
.copy ();
4375 save
->next_cycle_apply
= next_cycle_apply
.copy ();
4377 if (current_sched_info
->save_state
)
4378 save
->fe_saved_data
= (*current_sched_info
->save_state
) ();
4380 if (targetm
.sched
.alloc_sched_context
)
4382 save
->be_saved_data
= targetm
.sched
.alloc_sched_context ();
4383 targetm
.sched
.init_sched_context (save
->be_saved_data
, false);
4386 save
->be_saved_data
= NULL
;
4388 save
->delay_pair
= pair
;
4390 save
->next
= backtrack_queue
;
4391 backtrack_queue
= save
;
4395 mark_backtrack_feeds (pair
->i2
, 1);
4396 INSN_TICK (pair
->i2
) = INVALID_TICK
;
4397 INSN_EXACT_TICK (pair
->i2
) = clock_var
+ pair_delay (pair
);
4398 SHADOW_P (pair
->i2
) = pair
->stages
== 0;
4399 pair
= pair
->next_same_i1
;
4403 /* Walk the ready list and all queues. If any insns have unresolved backwards
4404 dependencies, these must be cancelled deps, broken by predication. Set or
4405 clear (depending on SET) the DEP_CANCELLED bit in DEP_STATUS. */
4408 toggle_cancelled_flags (bool set
)
4411 sd_iterator_def sd_it
;
4414 if (ready
.n_ready
> 0)
4416 rtx_insn
**first
= ready_lastpos (&ready
);
4417 for (i
= 0; i
< ready
.n_ready
; i
++)
4418 FOR_EACH_DEP (first
[i
], SD_LIST_BACK
, sd_it
, dep
)
4419 if (!DEBUG_INSN_P (DEP_PRO (dep
)))
4422 DEP_STATUS (dep
) |= DEP_CANCELLED
;
4424 DEP_STATUS (dep
) &= ~DEP_CANCELLED
;
4427 for (i
= 0; i
<= max_insn_queue_index
; i
++)
4429 int q
= NEXT_Q_AFTER (q_ptr
, i
);
4430 rtx_insn_list
*link
;
4431 for (link
= insn_queue
[q
]; link
; link
= link
->next ())
4433 rtx_insn
*insn
= link
->insn ();
4434 FOR_EACH_DEP (insn
, SD_LIST_BACK
, sd_it
, dep
)
4435 if (!DEBUG_INSN_P (DEP_PRO (dep
)))
4438 DEP_STATUS (dep
) |= DEP_CANCELLED
;
4440 DEP_STATUS (dep
) &= ~DEP_CANCELLED
;
4446 /* Undo the replacements that have occurred after backtrack point SAVE
4449 undo_replacements_for_backtrack (struct haifa_saved_data
*save
)
4451 while (!save
->replacement_deps
.is_empty ())
4453 dep_t dep
= save
->replacement_deps
.pop ();
4454 int apply_p
= save
->replace_apply
.pop ();
4457 restore_pattern (dep
, true);
4459 apply_replacement (dep
, true);
4461 save
->replacement_deps
.release ();
4462 save
->replace_apply
.release ();
4465 /* Pop entries from the SCHEDULED_INSNS vector up to and including INSN.
4466 Restore their dependencies to an unresolved state, and mark them as
4470 unschedule_insns_until (rtx_insn
*insn
)
4472 auto_vec
<rtx_insn
*> recompute_vec
;
4474 /* Make two passes over the insns to be unscheduled. First, we clear out
4475 dependencies and other trivial bookkeeping. */
4479 sd_iterator_def sd_it
;
4482 last
= scheduled_insns
.pop ();
4484 /* This will be changed by restore_backtrack_point if the insn is in
4486 QUEUE_INDEX (last
) = QUEUE_NOWHERE
;
4488 INSN_TICK (last
) = INVALID_TICK
;
4490 if (modulo_ii
> 0 && INSN_UID (last
) < modulo_iter0_max_uid
)
4491 modulo_insns_scheduled
--;
4493 for (sd_it
= sd_iterator_start (last
, SD_LIST_RES_FORW
);
4494 sd_iterator_cond (&sd_it
, &dep
);)
4496 rtx_insn
*con
= DEP_CON (dep
);
4497 sd_unresolve_dep (sd_it
);
4498 if (!MUST_RECOMPUTE_SPEC_P (con
))
4500 MUST_RECOMPUTE_SPEC_P (con
) = 1;
4501 recompute_vec
.safe_push (con
);
4509 /* A second pass, to update ready and speculation status for insns
4510 depending on the unscheduled ones. The first pass must have
4511 popped the scheduled_insns vector up to the point where we
4512 restart scheduling, as recompute_todo_spec requires it to be
4514 while (!recompute_vec
.is_empty ())
4518 con
= recompute_vec
.pop ();
4519 MUST_RECOMPUTE_SPEC_P (con
) = 0;
4520 if (!sd_lists_empty_p (con
, SD_LIST_HARD_BACK
))
4522 TODO_SPEC (con
) = HARD_DEP
;
4523 INSN_TICK (con
) = INVALID_TICK
;
4524 if (PREDICATED_PAT (con
) != NULL_RTX
)
4525 haifa_change_pattern (con
, ORIG_PAT (con
));
4527 else if (QUEUE_INDEX (con
) != QUEUE_SCHEDULED
)
4528 TODO_SPEC (con
) = recompute_todo_spec (con
, true);
4532 /* Restore scheduler state from the topmost entry on the backtracking queue.
4533 PSCHED_BLOCK_P points to the local data of schedule_block that we must
4534 overwrite with the saved data.
4535 The caller must already have called unschedule_insns_until. */
4538 restore_last_backtrack_point (struct sched_block_state
*psched_block
)
4541 struct haifa_saved_data
*save
= backtrack_queue
;
4543 backtrack_queue
= save
->next
;
4545 if (current_sched_info
->restore_state
)
4546 (*current_sched_info
->restore_state
) (save
->fe_saved_data
);
4548 if (targetm
.sched
.alloc_sched_context
)
4550 targetm
.sched
.set_sched_context (save
->be_saved_data
);
4551 targetm
.sched
.free_sched_context (save
->be_saved_data
);
4554 /* Do this first since it clobbers INSN_TICK of the involved
4556 undo_replacements_for_backtrack (save
);
4558 /* Clear the QUEUE_INDEX of everything in the ready list or one
4560 if (ready
.n_ready
> 0)
4562 rtx_insn
**first
= ready_lastpos (&ready
);
4563 for (i
= 0; i
< ready
.n_ready
; i
++)
4565 rtx_insn
*insn
= first
[i
];
4566 QUEUE_INDEX (insn
) = QUEUE_NOWHERE
;
4567 INSN_TICK (insn
) = INVALID_TICK
;
4570 for (i
= 0; i
<= max_insn_queue_index
; i
++)
4572 int q
= NEXT_Q_AFTER (q_ptr
, i
);
4574 for (rtx_insn_list
*link
= insn_queue
[q
]; link
; link
= link
->next ())
4576 rtx_insn
*x
= link
->insn ();
4577 QUEUE_INDEX (x
) = QUEUE_NOWHERE
;
4578 INSN_TICK (x
) = INVALID_TICK
;
4580 free_INSN_LIST_list (&insn_queue
[q
]);
4584 ready
= save
->ready
;
4586 if (ready
.n_ready
> 0)
4588 rtx_insn
**first
= ready_lastpos (&ready
);
4589 for (i
= 0; i
< ready
.n_ready
; i
++)
4591 rtx_insn
*insn
= first
[i
];
4592 QUEUE_INDEX (insn
) = QUEUE_READY
;
4593 TODO_SPEC (insn
) = recompute_todo_spec (insn
, true);
4594 INSN_TICK (insn
) = save
->clock_var
;
4599 q_size
= save
->q_size
;
4600 for (i
= 0; i
<= max_insn_queue_index
; i
++)
4602 int q
= NEXT_Q_AFTER (q_ptr
, i
);
4604 insn_queue
[q
] = save
->insn_queue
[q
];
4606 for (rtx_insn_list
*link
= insn_queue
[q
]; link
; link
= link
->next ())
4608 rtx_insn
*x
= link
->insn ();
4609 QUEUE_INDEX (x
) = i
;
4610 TODO_SPEC (x
) = recompute_todo_spec (x
, true);
4611 INSN_TICK (x
) = save
->clock_var
+ i
;
4614 free (save
->insn_queue
);
4616 toggle_cancelled_flags (true);
4618 clock_var
= save
->clock_var
;
4619 last_clock_var
= save
->last_clock_var
;
4620 cycle_issued_insns
= save
->cycle_issued_insns
;
4621 last_scheduled_insn
= save
->last_scheduled_insn
;
4622 last_nondebug_scheduled_insn
= save
->last_nondebug_scheduled_insn
;
4623 nonscheduled_insns_begin
= save
->nonscheduled_insns_begin
;
4625 *psched_block
= save
->sched_block
;
4627 memcpy (curr_state
, save
->curr_state
, dfa_state_size
);
4628 free (save
->curr_state
);
4630 mark_backtrack_feeds (save
->delay_pair
->i2
, 0);
4632 gcc_assert (next_cycle_replace_deps
.is_empty ());
4633 next_cycle_replace_deps
= save
->next_cycle_deps
.copy ();
4634 next_cycle_apply
= save
->next_cycle_apply
.copy ();
4638 for (save
= backtrack_queue
; save
; save
= save
->next
)
4640 mark_backtrack_feeds (save
->delay_pair
->i2
, 1);
4644 /* Discard all data associated with the topmost entry in the backtrack
4645 queue. If RESET_TICK is false, we just want to free the data. If true,
4646 we are doing this because we discovered a reason to backtrack. In the
4647 latter case, also reset the INSN_TICK for the shadow insn. */
4649 free_topmost_backtrack_point (bool reset_tick
)
4651 struct haifa_saved_data
*save
= backtrack_queue
;
4654 backtrack_queue
= save
->next
;
4658 struct delay_pair
*pair
= save
->delay_pair
;
4661 INSN_TICK (pair
->i2
) = INVALID_TICK
;
4662 INSN_EXACT_TICK (pair
->i2
) = INVALID_TICK
;
4663 pair
= pair
->next_same_i1
;
4665 undo_replacements_for_backtrack (save
);
4669 save
->replacement_deps
.release ();
4670 save
->replace_apply
.release ();
4673 if (targetm
.sched
.free_sched_context
)
4674 targetm
.sched
.free_sched_context (save
->be_saved_data
);
4675 if (current_sched_info
->restore_state
)
4676 free (save
->fe_saved_data
);
4677 for (i
= 0; i
<= max_insn_queue_index
; i
++)
4678 free_INSN_LIST_list (&save
->insn_queue
[i
]);
4679 free (save
->insn_queue
);
4680 free (save
->curr_state
);
4681 free (save
->ready
.vec
);
4685 /* Free the entire backtrack queue. */
4687 free_backtrack_queue (void)
4689 while (backtrack_queue
)
4690 free_topmost_backtrack_point (false);
4693 /* Apply a replacement described by DESC. If IMMEDIATELY is false, we
4694 may have to postpone the replacement until the start of the next cycle,
4695 at which point we will be called again with IMMEDIATELY true. This is
4696 only done for machines which have instruction packets with explicit
4697 parallelism however. */
4699 apply_replacement (dep_t dep
, bool immediately
)
4701 struct dep_replacement
*desc
= DEP_REPLACE (dep
);
4702 if (!immediately
&& targetm
.sched
.exposed_pipeline
&& reload_completed
)
4704 next_cycle_replace_deps
.safe_push (dep
);
4705 next_cycle_apply
.safe_push (1);
4711 if (QUEUE_INDEX (desc
->insn
) == QUEUE_SCHEDULED
)
4714 if (sched_verbose
>= 5)
4715 fprintf (sched_dump
, "applying replacement for insn %d\n",
4716 INSN_UID (desc
->insn
));
4718 success
= validate_change (desc
->insn
, desc
->loc
, desc
->newval
, 0);
4719 gcc_assert (success
);
4721 update_insn_after_change (desc
->insn
);
4722 if ((TODO_SPEC (desc
->insn
) & (HARD_DEP
| DEP_POSTPONED
)) == 0)
4723 fix_tick_ready (desc
->insn
);
4725 if (backtrack_queue
!= NULL
)
4727 backtrack_queue
->replacement_deps
.safe_push (dep
);
4728 backtrack_queue
->replace_apply
.safe_push (1);
4733 /* We have determined that a pattern involved in DEP must be restored.
4734 If IMMEDIATELY is false, we may have to postpone the replacement
4735 until the start of the next cycle, at which point we will be called
4736 again with IMMEDIATELY true. */
4738 restore_pattern (dep_t dep
, bool immediately
)
4740 rtx_insn
*next
= DEP_CON (dep
);
4741 int tick
= INSN_TICK (next
);
4743 /* If we already scheduled the insn, the modified version is
4745 if (QUEUE_INDEX (next
) == QUEUE_SCHEDULED
)
4748 if (!immediately
&& targetm
.sched
.exposed_pipeline
&& reload_completed
)
4750 next_cycle_replace_deps
.safe_push (dep
);
4751 next_cycle_apply
.safe_push (0);
4756 if (DEP_TYPE (dep
) == REG_DEP_CONTROL
)
4758 if (sched_verbose
>= 5)
4759 fprintf (sched_dump
, "restoring pattern for insn %d\n",
4761 haifa_change_pattern (next
, ORIG_PAT (next
));
4765 struct dep_replacement
*desc
= DEP_REPLACE (dep
);
4768 if (sched_verbose
>= 5)
4769 fprintf (sched_dump
, "restoring pattern for insn %d\n",
4770 INSN_UID (desc
->insn
));
4771 tick
= INSN_TICK (desc
->insn
);
4773 success
= validate_change (desc
->insn
, desc
->loc
, desc
->orig
, 0);
4774 gcc_assert (success
);
4775 update_insn_after_change (desc
->insn
);
4776 if (backtrack_queue
!= NULL
)
4778 backtrack_queue
->replacement_deps
.safe_push (dep
);
4779 backtrack_queue
->replace_apply
.safe_push (0);
4782 INSN_TICK (next
) = tick
;
4783 if (TODO_SPEC (next
) == DEP_POSTPONED
)
4786 if (sd_lists_empty_p (next
, SD_LIST_BACK
))
4787 TODO_SPEC (next
) = 0;
4788 else if (!sd_lists_empty_p (next
, SD_LIST_HARD_BACK
))
4789 TODO_SPEC (next
) = HARD_DEP
;
4792 /* Perform pattern replacements that were queued up until the next
4795 perform_replacements_new_cycle (void)
4799 FOR_EACH_VEC_ELT (next_cycle_replace_deps
, i
, dep
)
4801 int apply_p
= next_cycle_apply
[i
];
4803 apply_replacement (dep
, true);
4805 restore_pattern (dep
, true);
4807 next_cycle_replace_deps
.truncate (0);
4808 next_cycle_apply
.truncate (0);
4811 /* Compute INSN_TICK_ESTIMATE for INSN. PROCESSED is a bitmap of
4812 instructions we've previously encountered, a set bit prevents
4813 recursion. BUDGET is a limit on how far ahead we look, it is
4814 reduced on recursive calls. Return true if we produced a good
4815 estimate, or false if we exceeded the budget. */
4817 estimate_insn_tick (bitmap processed
, rtx_insn
*insn
, int budget
)
4819 sd_iterator_def sd_it
;
4821 int earliest
= INSN_TICK (insn
);
4823 FOR_EACH_DEP (insn
, SD_LIST_BACK
, sd_it
, dep
)
4825 rtx_insn
*pro
= DEP_PRO (dep
);
4828 if (DEP_STATUS (dep
) & DEP_CANCELLED
)
4831 if (QUEUE_INDEX (pro
) == QUEUE_SCHEDULED
)
4832 gcc_assert (INSN_TICK (pro
) + dep_cost (dep
) <= INSN_TICK (insn
));
4835 int cost
= dep_cost (dep
);
4838 if (!bitmap_bit_p (processed
, INSN_LUID (pro
)))
4840 if (!estimate_insn_tick (processed
, pro
, budget
- cost
))
4843 gcc_assert (INSN_TICK_ESTIMATE (pro
) != INVALID_TICK
);
4844 t
= INSN_TICK_ESTIMATE (pro
) + cost
;
4845 if (earliest
== INVALID_TICK
|| t
> earliest
)
4849 bitmap_set_bit (processed
, INSN_LUID (insn
));
4850 INSN_TICK_ESTIMATE (insn
) = earliest
;
4854 /* Examine the pair of insns in P, and estimate (optimistically, assuming
4855 infinite resources) the cycle in which the delayed shadow can be issued.
4856 Return the number of cycles that must pass before the real insn can be
4857 issued in order to meet this constraint. */
4859 estimate_shadow_tick (struct delay_pair
*p
)
4861 bitmap_head processed
;
4864 bitmap_initialize (&processed
, 0);
4866 cutoff
= !estimate_insn_tick (&processed
, p
->i2
,
4867 max_insn_queue_index
+ pair_delay (p
));
4868 bitmap_clear (&processed
);
4870 return max_insn_queue_index
;
4871 t
= INSN_TICK_ESTIMATE (p
->i2
) - (clock_var
+ pair_delay (p
) + 1);
4877 /* If INSN has no unresolved backwards dependencies, add it to the schedule and
4878 recursively resolve all its forward dependencies. */
4880 resolve_dependencies (rtx_insn
*insn
)
4882 sd_iterator_def sd_it
;
4885 /* Don't use sd_lists_empty_p; it ignores debug insns. */
4886 if (DEPS_LIST_FIRST (INSN_HARD_BACK_DEPS (insn
)) != NULL
4887 || DEPS_LIST_FIRST (INSN_SPEC_BACK_DEPS (insn
)) != NULL
)
4890 if (sched_verbose
>= 4)
4891 fprintf (sched_dump
, ";;\tquickly resolving %d\n", INSN_UID (insn
));
4893 if (QUEUE_INDEX (insn
) >= 0)
4894 queue_remove (insn
);
4896 scheduled_insns
.safe_push (insn
);
4898 /* Update dependent instructions. */
4899 for (sd_it
= sd_iterator_start (insn
, SD_LIST_FORW
);
4900 sd_iterator_cond (&sd_it
, &dep
);)
4902 rtx_insn
*next
= DEP_CON (dep
);
4904 if (sched_verbose
>= 4)
4905 fprintf (sched_dump
, ";;\t\tdep %d against %d\n", INSN_UID (insn
),
4908 /* Resolve the dependence between INSN and NEXT.
4909 sd_resolve_dep () moves current dep to another list thus
4910 advancing the iterator. */
4911 sd_resolve_dep (sd_it
);
4913 if (!IS_SPECULATION_BRANCHY_CHECK_P (insn
))
4915 resolve_dependencies (next
);
4918 /* Check always has only one forward dependence (to the first insn in
4919 the recovery block), therefore, this will be executed only once. */
4921 gcc_assert (sd_lists_empty_p (insn
, SD_LIST_FORW
));
4927 /* Return the head and tail pointers of ebb starting at BEG and ending
4930 get_ebb_head_tail (basic_block beg
, basic_block end
,
4931 rtx_insn
**headp
, rtx_insn
**tailp
)
4933 rtx_insn
*beg_head
= BB_HEAD (beg
);
4934 rtx_insn
* beg_tail
= BB_END (beg
);
4935 rtx_insn
* end_head
= BB_HEAD (end
);
4936 rtx_insn
* end_tail
= BB_END (end
);
4938 /* Don't include any notes or labels at the beginning of the BEG
4939 basic block, or notes at the end of the END basic blocks. */
4941 if (LABEL_P (beg_head
))
4942 beg_head
= NEXT_INSN (beg_head
);
4944 while (beg_head
!= beg_tail
)
4945 if (NOTE_P (beg_head
))
4946 beg_head
= NEXT_INSN (beg_head
);
4947 else if (DEBUG_INSN_P (beg_head
))
4949 rtx_insn
* note
, *next
;
4951 for (note
= NEXT_INSN (beg_head
);
4955 next
= NEXT_INSN (note
);
4958 if (sched_verbose
>= 9)
4959 fprintf (sched_dump
, "reorder %i\n", INSN_UID (note
));
4961 reorder_insns_nobb (note
, note
, PREV_INSN (beg_head
));
4963 if (BLOCK_FOR_INSN (note
) != beg
)
4964 df_insn_change_bb (note
, beg
);
4966 else if (!DEBUG_INSN_P (note
))
4978 end_head
= beg_head
;
4979 else if (LABEL_P (end_head
))
4980 end_head
= NEXT_INSN (end_head
);
4982 while (end_head
!= end_tail
)
4983 if (NOTE_P (end_tail
))
4984 end_tail
= PREV_INSN (end_tail
);
4985 else if (DEBUG_INSN_P (end_tail
))
4987 rtx_insn
* note
, *prev
;
4989 for (note
= PREV_INSN (end_tail
);
4993 prev
= PREV_INSN (note
);
4996 if (sched_verbose
>= 9)
4997 fprintf (sched_dump
, "reorder %i\n", INSN_UID (note
));
4999 reorder_insns_nobb (note
, note
, end_tail
);
5001 if (end_tail
== BB_END (end
))
5002 BB_END (end
) = note
;
5004 if (BLOCK_FOR_INSN (note
) != end
)
5005 df_insn_change_bb (note
, end
);
5007 else if (!DEBUG_INSN_P (note
))
5019 /* Return nonzero if there are no real insns in the range [ HEAD, TAIL ]. */
5022 no_real_insns_p (const rtx_insn
*head
, const rtx_insn
*tail
)
5024 while (head
!= NEXT_INSN (tail
))
5026 if (!NOTE_P (head
) && !LABEL_P (head
))
5028 head
= NEXT_INSN (head
);
5033 /* Restore-other-notes: NOTE_LIST is the end of a chain of notes
5034 previously found among the insns. Insert them just before HEAD. */
5036 restore_other_notes (rtx_insn
*head
, basic_block head_bb
)
5040 rtx_insn
*note_head
= note_list
;
5043 head_bb
= BLOCK_FOR_INSN (head
);
5045 head
= NEXT_INSN (bb_note (head_bb
));
5047 while (PREV_INSN (note_head
))
5049 set_block_for_insn (note_head
, head_bb
);
5050 note_head
= PREV_INSN (note_head
);
5052 /* In the above cycle we've missed this note. */
5053 set_block_for_insn (note_head
, head_bb
);
5055 SET_PREV_INSN (note_head
) = PREV_INSN (head
);
5056 SET_NEXT_INSN (PREV_INSN (head
)) = note_head
;
5057 SET_PREV_INSN (head
) = note_list
;
5058 SET_NEXT_INSN (note_list
) = head
;
5060 if (BLOCK_FOR_INSN (head
) != head_bb
)
5061 BB_END (head_bb
) = note_list
;
5069 /* When we know we are going to discard the schedule due to a failed attempt
5070 at modulo scheduling, undo all replacements. */
5072 undo_all_replacements (void)
5077 FOR_EACH_VEC_ELT (scheduled_insns
, i
, insn
)
5079 sd_iterator_def sd_it
;
5082 /* See if we must undo a replacement. */
5083 for (sd_it
= sd_iterator_start (insn
, SD_LIST_RES_FORW
);
5084 sd_iterator_cond (&sd_it
, &dep
); sd_iterator_next (&sd_it
))
5086 struct dep_replacement
*desc
= DEP_REPLACE (dep
);
5088 validate_change (desc
->insn
, desc
->loc
, desc
->orig
, 0);
5093 /* Return first non-scheduled insn in the current scheduling block.
5094 This is mostly used for debug-counter purposes. */
5096 first_nonscheduled_insn (void)
5098 rtx_insn
*insn
= (nonscheduled_insns_begin
!= NULL_RTX
5099 ? nonscheduled_insns_begin
5100 : current_sched_info
->prev_head
);
5104 insn
= next_nonnote_nondebug_insn (insn
);
5106 while (QUEUE_INDEX (insn
) == QUEUE_SCHEDULED
);
5111 /* Move insns that became ready to fire from queue to ready list. */
5114 queue_to_ready (struct ready_list
*ready
)
5117 rtx_insn_list
*link
;
5118 rtx_insn
*skip_insn
;
5120 q_ptr
= NEXT_Q (q_ptr
);
5122 if (dbg_cnt (sched_insn
) == false)
5123 /* If debug counter is activated do not requeue the first
5124 nonscheduled insn. */
5125 skip_insn
= first_nonscheduled_insn ();
5129 /* Add all pending insns that can be scheduled without stalls to the
5131 for (link
= insn_queue
[q_ptr
]; link
; link
= link
->next ())
5133 insn
= link
->insn ();
5136 if (sched_verbose
>= 2)
5137 fprintf (sched_dump
, ";;\t\tQ-->Ready: insn %s: ",
5138 (*current_sched_info
->print_insn
) (insn
, 0));
5140 /* If the ready list is full, delay the insn for 1 cycle.
5141 See the comment in schedule_block for the rationale. */
5142 if (!reload_completed
5143 && (ready
->n_ready
- ready
->n_debug
> MAX_SCHED_READY_INSNS
5144 || (sched_pressure
== SCHED_PRESSURE_MODEL
5145 /* Limit pressure recalculations to MAX_SCHED_READY_INSNS
5146 instructions too. */
5147 && model_index (insn
) > (model_curr_point
5148 + MAX_SCHED_READY_INSNS
)))
5149 && !(sched_pressure
== SCHED_PRESSURE_MODEL
5150 && model_curr_point
< model_num_insns
5151 /* Always allow the next model instruction to issue. */
5152 && model_index (insn
) == model_curr_point
)
5153 && !SCHED_GROUP_P (insn
)
5154 && insn
!= skip_insn
)
5156 if (sched_verbose
>= 2)
5157 fprintf (sched_dump
, "keeping in queue, ready full\n");
5158 queue_insn (insn
, 1, "ready full");
5162 ready_add (ready
, insn
, false);
5163 if (sched_verbose
>= 2)
5164 fprintf (sched_dump
, "moving to ready without stalls\n");
5167 free_INSN_LIST_list (&insn_queue
[q_ptr
]);
5169 /* If there are no ready insns, stall until one is ready and add all
5170 of the pending insns at that point to the ready list. */
5171 if (ready
->n_ready
== 0)
5175 for (stalls
= 1; stalls
<= max_insn_queue_index
; stalls
++)
5177 if ((link
= insn_queue
[NEXT_Q_AFTER (q_ptr
, stalls
)]))
5179 for (; link
; link
= link
->next ())
5181 insn
= link
->insn ();
5184 if (sched_verbose
>= 2)
5185 fprintf (sched_dump
, ";;\t\tQ-->Ready: insn %s: ",
5186 (*current_sched_info
->print_insn
) (insn
, 0));
5188 ready_add (ready
, insn
, false);
5189 if (sched_verbose
>= 2)
5190 fprintf (sched_dump
, "moving to ready with %d stalls\n", stalls
);
5192 free_INSN_LIST_list (&insn_queue
[NEXT_Q_AFTER (q_ptr
, stalls
)]);
5194 advance_one_cycle ();
5199 advance_one_cycle ();
5202 q_ptr
= NEXT_Q_AFTER (q_ptr
, stalls
);
5203 clock_var
+= stalls
;
5204 if (sched_verbose
>= 2)
5205 fprintf (sched_dump
, ";;\tAdvancing clock by %d cycle[s] to %d\n",
5210 /* Used by early_queue_to_ready. Determines whether it is "ok" to
5211 prematurely move INSN from the queue to the ready list. Currently,
5212 if a target defines the hook 'is_costly_dependence', this function
5213 uses the hook to check whether there exist any dependences which are
5214 considered costly by the target, between INSN and other insns that
5215 have already been scheduled. Dependences are checked up to Y cycles
5216 back, with default Y=1; The flag -fsched-stalled-insns-dep=Y allows
5217 controlling this value.
5218 (Other considerations could be taken into account instead (or in
5219 addition) depending on user flags and target hooks. */
5222 ok_for_early_queue_removal (rtx_insn
*insn
)
5224 if (targetm
.sched
.is_costly_dependence
)
5227 int i
= scheduled_insns
.length ();
5228 for (n_cycles
= flag_sched_stalled_insns_dep
; n_cycles
; n_cycles
--)
5234 rtx_insn
*prev_insn
= scheduled_insns
[i
];
5236 if (!NOTE_P (prev_insn
))
5240 dep
= sd_find_dep_between (prev_insn
, insn
, true);
5244 cost
= dep_cost (dep
);
5246 if (targetm
.sched
.is_costly_dependence (dep
, cost
,
5247 flag_sched_stalled_insns_dep
- n_cycles
))
5252 if (GET_MODE (prev_insn
) == TImode
) /* end of dispatch group */
5265 /* Remove insns from the queue, before they become "ready" with respect
5266 to FU latency considerations. */
5269 early_queue_to_ready (state_t state
, struct ready_list
*ready
)
5272 rtx_insn_list
*link
;
5273 rtx_insn_list
*next_link
;
5274 rtx_insn_list
*prev_link
;
5277 state_t temp_state
= alloca (dfa_state_size
);
5279 int insns_removed
= 0;
5282 Flag '-fsched-stalled-insns=X' determines the aggressiveness of this
5285 X == 0: There is no limit on how many queued insns can be removed
5286 prematurely. (flag_sched_stalled_insns = -1).
5288 X >= 1: Only X queued insns can be removed prematurely in each
5289 invocation. (flag_sched_stalled_insns = X).
5291 Otherwise: Early queue removal is disabled.
5292 (flag_sched_stalled_insns = 0)
5295 if (! flag_sched_stalled_insns
)
5298 for (stalls
= 0; stalls
<= max_insn_queue_index
; stalls
++)
5300 if ((link
= insn_queue
[NEXT_Q_AFTER (q_ptr
, stalls
)]))
5302 if (sched_verbose
> 6)
5303 fprintf (sched_dump
, ";; look at index %d + %d\n", q_ptr
, stalls
);
5308 next_link
= link
->next ();
5309 insn
= link
->insn ();
5310 if (insn
&& sched_verbose
> 6)
5311 print_rtl_single (sched_dump
, insn
);
5313 memcpy (temp_state
, state
, dfa_state_size
);
5314 if (recog_memoized (insn
) < 0)
5315 /* non-negative to indicate that it's not ready
5316 to avoid infinite Q->R->Q->R... */
5319 cost
= state_transition (temp_state
, insn
);
5321 if (sched_verbose
>= 6)
5322 fprintf (sched_dump
, "transition cost = %d\n", cost
);
5324 move_to_ready
= false;
5327 move_to_ready
= ok_for_early_queue_removal (insn
);
5328 if (move_to_ready
== true)
5330 /* move from Q to R */
5332 ready_add (ready
, insn
, false);
5335 XEXP (prev_link
, 1) = next_link
;
5337 insn_queue
[NEXT_Q_AFTER (q_ptr
, stalls
)] = next_link
;
5339 free_INSN_LIST_node (link
);
5341 if (sched_verbose
>= 2)
5342 fprintf (sched_dump
, ";;\t\tEarly Q-->Ready: insn %s\n",
5343 (*current_sched_info
->print_insn
) (insn
, 0));
5346 if (insns_removed
== flag_sched_stalled_insns
)
5347 /* Remove no more than flag_sched_stalled_insns insns
5348 from Q at a time. */
5349 return insns_removed
;
5353 if (move_to_ready
== false)
5360 } /* for stalls.. */
5362 return insns_removed
;
5366 /* Print the ready list for debugging purposes.
5367 If READY_TRY is non-zero then only print insns that max_issue
5370 debug_ready_list_1 (struct ready_list
*ready
, signed char *ready_try
)
5375 if (ready
->n_ready
== 0)
5377 fprintf (sched_dump
, "\n");
5381 p
= ready_lastpos (ready
);
5382 for (i
= 0; i
< ready
->n_ready
; i
++)
5384 if (ready_try
!= NULL
&& ready_try
[ready
->n_ready
- i
- 1])
5387 fprintf (sched_dump
, " %s:%d",
5388 (*current_sched_info
->print_insn
) (p
[i
], 0),
5390 if (sched_pressure
!= SCHED_PRESSURE_NONE
)
5391 fprintf (sched_dump
, "(cost=%d",
5392 INSN_REG_PRESSURE_EXCESS_COST_CHANGE (p
[i
]));
5393 fprintf (sched_dump
, ":prio=%d", INSN_PRIORITY (p
[i
]));
5394 if (INSN_TICK (p
[i
]) > clock_var
)
5395 fprintf (sched_dump
, ":delay=%d", INSN_TICK (p
[i
]) - clock_var
);
5396 if (sched_pressure
== SCHED_PRESSURE_MODEL
)
5397 fprintf (sched_dump
, ":idx=%d",
5398 model_index (p
[i
]));
5399 if (sched_pressure
!= SCHED_PRESSURE_NONE
)
5400 fprintf (sched_dump
, ")");
5402 fprintf (sched_dump
, "\n");
5405 /* Print the ready list. Callable from debugger. */
5407 debug_ready_list (struct ready_list
*ready
)
5409 debug_ready_list_1 (ready
, NULL
);
5412 /* Search INSN for REG_SAVE_NOTE notes and convert them back into insn
5413 NOTEs. This is used for NOTE_INSN_EPILOGUE_BEG, so that sched-ebb
5414 replaces the epilogue note in the correct basic block. */
5416 reemit_notes (rtx_insn
*insn
)
5419 rtx_insn
*last
= insn
;
5421 for (note
= REG_NOTES (insn
); note
; note
= XEXP (note
, 1))
5423 if (REG_NOTE_KIND (note
) == REG_SAVE_NOTE
)
5425 enum insn_note note_type
= (enum insn_note
) INTVAL (XEXP (note
, 0));
5427 last
= emit_note_before (note_type
, last
);
5428 remove_note (insn
, note
);
5433 /* Move INSN. Reemit notes if needed. Update CFG, if needed. */
5435 move_insn (rtx_insn
*insn
, rtx_insn
*last
, rtx nt
)
5437 if (PREV_INSN (insn
) != last
)
5443 bb
= BLOCK_FOR_INSN (insn
);
5445 /* BB_HEAD is either LABEL or NOTE. */
5446 gcc_assert (BB_HEAD (bb
) != insn
);
5448 if (BB_END (bb
) == insn
)
5449 /* If this is last instruction in BB, move end marker one
5452 /* Jumps are always placed at the end of basic block. */
5453 jump_p
= control_flow_insn_p (insn
);
5456 || ((common_sched_info
->sched_pass_id
== SCHED_RGN_PASS
)
5457 && IS_SPECULATION_BRANCHY_CHECK_P (insn
))
5458 || (common_sched_info
->sched_pass_id
5459 == SCHED_EBB_PASS
));
5461 gcc_assert (BLOCK_FOR_INSN (PREV_INSN (insn
)) == bb
);
5463 BB_END (bb
) = PREV_INSN (insn
);
5466 gcc_assert (BB_END (bb
) != last
);
5469 /* We move the block note along with jump. */
5473 note
= NEXT_INSN (insn
);
5474 while (NOTE_NOT_BB_P (note
) && note
!= nt
)
5475 note
= NEXT_INSN (note
);
5479 || BARRIER_P (note
)))
5480 note
= NEXT_INSN (note
);
5482 gcc_assert (NOTE_INSN_BASIC_BLOCK_P (note
));
5487 SET_NEXT_INSN (PREV_INSN (insn
)) = NEXT_INSN (note
);
5488 SET_PREV_INSN (NEXT_INSN (note
)) = PREV_INSN (insn
);
5490 SET_NEXT_INSN (note
) = NEXT_INSN (last
);
5491 SET_PREV_INSN (NEXT_INSN (last
)) = note
;
5493 SET_NEXT_INSN (last
) = insn
;
5494 SET_PREV_INSN (insn
) = last
;
5496 bb
= BLOCK_FOR_INSN (last
);
5500 fix_jump_move (insn
);
5502 if (BLOCK_FOR_INSN (insn
) != bb
)
5503 move_block_after_check (insn
);
5505 gcc_assert (BB_END (bb
) == last
);
5508 df_insn_change_bb (insn
, bb
);
5510 /* Update BB_END, if needed. */
5511 if (BB_END (bb
) == last
)
5515 SCHED_GROUP_P (insn
) = 0;
5518 /* Return true if scheduling INSN will finish current clock cycle. */
5520 insn_finishes_cycle_p (rtx_insn
*insn
)
5522 if (SCHED_GROUP_P (insn
))
5523 /* After issuing INSN, rest of the sched_group will be forced to issue
5524 in order. Don't make any plans for the rest of cycle. */
5527 /* Finishing the block will, apparently, finish the cycle. */
5528 if (current_sched_info
->insn_finishes_block_p
5529 && current_sched_info
->insn_finishes_block_p (insn
))
5535 /* Functions to model cache auto-prefetcher.
5537 Some of the CPUs have cache auto-prefetcher, which /seems/ to initiate
5538 memory prefetches if it sees instructions with consequitive memory accesses
5539 in the instruction stream. Details of such hardware units are not published,
5540 so we can only guess what exactly is going on there.
5541 In the scheduler, we model abstract auto-prefetcher. If there are memory
5542 insns in the ready list (or the queue) that have same memory base, but
5543 different offsets, then we delay the insns with larger offsets until insns
5544 with smaller offsets get scheduled. If PARAM_SCHED_AUTOPREF_QUEUE_DEPTH
5545 is "1", then we look at the ready list; if it is N>1, then we also look
5546 through N-1 queue entries.
5547 If the param is N>=0, then rank_for_schedule will consider auto-prefetching
5548 among its heuristics.
5549 Param value of "-1" disables modelling of the auto-prefetcher. */
5551 /* Initialize autoprefetcher model data for INSN. */
5553 autopref_multipass_init (const rtx_insn
*insn
, int write
)
5555 autopref_multipass_data_t data
= &INSN_AUTOPREF_MULTIPASS_DATA (insn
)[write
];
5557 gcc_assert (data
->status
== AUTOPREF_MULTIPASS_DATA_UNINITIALIZED
);
5558 data
->base
= NULL_RTX
;
5560 /* Set insn entry initialized, but not relevant for auto-prefetcher. */
5561 data
->status
= AUTOPREF_MULTIPASS_DATA_IRRELEVANT
;
5563 rtx set
= single_set (insn
);
5564 if (set
== NULL_RTX
)
5567 rtx mem
= write
? SET_DEST (set
) : SET_SRC (set
);
5571 struct address_info info
;
5572 decompose_mem_address (&info
, mem
);
5574 /* TODO: Currently only (base+const) addressing is supported. */
5575 if (info
.base
== NULL
|| !REG_P (*info
.base
)
5576 || (info
.disp
!= NULL
&& !CONST_INT_P (*info
.disp
)))
5579 /* This insn is relevant for auto-prefetcher. */
5580 data
->base
= *info
.base
;
5581 data
->offset
= info
.disp
? INTVAL (*info
.disp
) : 0;
5582 data
->status
= AUTOPREF_MULTIPASS_DATA_NORMAL
;
5585 /* Helper function for rank_for_schedule sorting. */
5587 autopref_rank_for_schedule (const rtx_insn
*insn1
, const rtx_insn
*insn2
)
5589 for (int write
= 0; write
< 2; ++write
)
5591 autopref_multipass_data_t data1
5592 = &INSN_AUTOPREF_MULTIPASS_DATA (insn1
)[write
];
5593 autopref_multipass_data_t data2
5594 = &INSN_AUTOPREF_MULTIPASS_DATA (insn2
)[write
];
5596 if (data1
->status
== AUTOPREF_MULTIPASS_DATA_UNINITIALIZED
)
5597 autopref_multipass_init (insn1
, write
);
5598 if (data1
->status
== AUTOPREF_MULTIPASS_DATA_IRRELEVANT
)
5601 if (data2
->status
== AUTOPREF_MULTIPASS_DATA_UNINITIALIZED
)
5602 autopref_multipass_init (insn2
, write
);
5603 if (data2
->status
== AUTOPREF_MULTIPASS_DATA_IRRELEVANT
)
5606 if (!rtx_equal_p (data1
->base
, data2
->base
))
5609 return data1
->offset
- data2
->offset
;
5615 /* True if header of debug dump was printed. */
5616 static bool autopref_multipass_dfa_lookahead_guard_started_dump_p
;
5618 /* Helper for autopref_multipass_dfa_lookahead_guard.
5619 Return "1" if INSN1 should be delayed in favor of INSN2. */
5621 autopref_multipass_dfa_lookahead_guard_1 (const rtx_insn
*insn1
,
5622 const rtx_insn
*insn2
, int write
)
5624 autopref_multipass_data_t data1
5625 = &INSN_AUTOPREF_MULTIPASS_DATA (insn1
)[write
];
5626 autopref_multipass_data_t data2
5627 = &INSN_AUTOPREF_MULTIPASS_DATA (insn2
)[write
];
5629 if (data2
->status
== AUTOPREF_MULTIPASS_DATA_UNINITIALIZED
)
5630 autopref_multipass_init (insn2
, write
);
5631 if (data2
->status
== AUTOPREF_MULTIPASS_DATA_IRRELEVANT
)
5634 if (rtx_equal_p (data1
->base
, data2
->base
)
5635 && data1
->offset
> data2
->offset
)
5637 if (sched_verbose
>= 2)
5639 if (!autopref_multipass_dfa_lookahead_guard_started_dump_p
)
5641 fprintf (sched_dump
,
5642 ";;\t\tnot trying in max_issue due to autoprefetch "
5644 autopref_multipass_dfa_lookahead_guard_started_dump_p
= true;
5647 fprintf (sched_dump
, " %d(%d)", INSN_UID (insn1
), INSN_UID (insn2
));
5658 We could have also hooked autoprefetcher model into
5659 first_cycle_multipass_backtrack / first_cycle_multipass_issue hooks
5660 to enable intelligent selection of "[r1+0]=r2; [r1+4]=r3" on the same cycle
5661 (e.g., once "[r1+0]=r2" is issued in max_issue(), "[r1+4]=r3" gets
5662 unblocked). We don't bother about this yet because target of interest
5663 (ARM Cortex-A15) can issue only 1 memory operation per cycle. */
5665 /* Implementation of first_cycle_multipass_dfa_lookahead_guard hook.
5666 Return "1" if INSN1 should not be considered in max_issue due to
5667 auto-prefetcher considerations. */
5669 autopref_multipass_dfa_lookahead_guard (rtx_insn
*insn1
, int ready_index
)
5673 if (PARAM_VALUE (PARAM_SCHED_AUTOPREF_QUEUE_DEPTH
) <= 0)
5676 if (sched_verbose
>= 2 && ready_index
== 0)
5677 autopref_multipass_dfa_lookahead_guard_started_dump_p
= false;
5679 for (int write
= 0; write
< 2; ++write
)
5681 autopref_multipass_data_t data1
5682 = &INSN_AUTOPREF_MULTIPASS_DATA (insn1
)[write
];
5684 if (data1
->status
== AUTOPREF_MULTIPASS_DATA_UNINITIALIZED
)
5685 autopref_multipass_init (insn1
, write
);
5686 if (data1
->status
== AUTOPREF_MULTIPASS_DATA_IRRELEVANT
)
5689 if (ready_index
== 0
5690 && data1
->status
== AUTOPREF_MULTIPASS_DATA_DONT_DELAY
)
5691 /* We allow only a single delay on priviledged instructions.
5692 Doing otherwise would cause infinite loop. */
5694 if (sched_verbose
>= 2)
5696 if (!autopref_multipass_dfa_lookahead_guard_started_dump_p
)
5698 fprintf (sched_dump
,
5699 ";;\t\tnot trying in max_issue due to autoprefetch "
5701 autopref_multipass_dfa_lookahead_guard_started_dump_p
= true;
5704 fprintf (sched_dump
, " *%d*", INSN_UID (insn1
));
5709 for (int i2
= 0; i2
< ready
.n_ready
; ++i2
)
5711 rtx_insn
*insn2
= get_ready_element (i2
);
5714 r
= autopref_multipass_dfa_lookahead_guard_1 (insn1
, insn2
, write
);
5717 if (ready_index
== 0)
5720 data1
->status
= AUTOPREF_MULTIPASS_DATA_DONT_DELAY
;
5726 if (PARAM_VALUE (PARAM_SCHED_AUTOPREF_QUEUE_DEPTH
) == 1)
5729 /* Everything from the current queue slot should have been moved to
5731 gcc_assert (insn_queue
[NEXT_Q_AFTER (q_ptr
, 0)] == NULL_RTX
);
5733 int n_stalls
= PARAM_VALUE (PARAM_SCHED_AUTOPREF_QUEUE_DEPTH
) - 1;
5734 if (n_stalls
> max_insn_queue_index
)
5735 n_stalls
= max_insn_queue_index
;
5737 for (int stalls
= 1; stalls
<= n_stalls
; ++stalls
)
5739 for (rtx_insn_list
*link
= insn_queue
[NEXT_Q_AFTER (q_ptr
, stalls
)];
5741 link
= link
->next ())
5743 rtx_insn
*insn2
= link
->insn ();
5744 r
= autopref_multipass_dfa_lookahead_guard_1 (insn1
, insn2
,
5748 /* Queue INSN1 until INSN2 can issue. */
5750 if (ready_index
== 0)
5751 data1
->status
= AUTOPREF_MULTIPASS_DATA_DONT_DELAY
;
5759 if (sched_verbose
>= 2
5760 && autopref_multipass_dfa_lookahead_guard_started_dump_p
5761 && (ready_index
== ready
.n_ready
- 1 || r
< 0))
5762 /* This does not /always/ trigger. We don't output EOL if the last
5763 insn is not recognized (INSN_CODE < 0) and lookahead_guard is not
5764 called. We can live with this. */
5765 fprintf (sched_dump
, "\n");
5770 /* Define type for target data used in multipass scheduling. */
5771 #ifndef TARGET_SCHED_FIRST_CYCLE_MULTIPASS_DATA_T
5772 # define TARGET_SCHED_FIRST_CYCLE_MULTIPASS_DATA_T int
5774 typedef TARGET_SCHED_FIRST_CYCLE_MULTIPASS_DATA_T first_cycle_multipass_data_t
;
5776 /* The following structure describe an entry of the stack of choices. */
5779 /* Ordinal number of the issued insn in the ready queue. */
5781 /* The number of the rest insns whose issues we should try. */
5783 /* The number of issued essential insns. */
5785 /* State after issuing the insn. */
5787 /* Target-specific data. */
5788 first_cycle_multipass_data_t target_data
;
5791 /* The following array is used to implement a stack of choices used in
5792 function max_issue. */
5793 static struct choice_entry
*choice_stack
;
5795 /* This holds the value of the target dfa_lookahead hook. */
5798 /* The following variable value is maximal number of tries of issuing
5799 insns for the first cycle multipass insn scheduling. We define
5800 this value as constant*(DFA_LOOKAHEAD**ISSUE_RATE). We would not
5801 need this constraint if all real insns (with non-negative codes)
5802 had reservations because in this case the algorithm complexity is
5803 O(DFA_LOOKAHEAD**ISSUE_RATE). Unfortunately, the dfa descriptions
5804 might be incomplete and such insn might occur. For such
5805 descriptions, the complexity of algorithm (without the constraint)
5806 could achieve DFA_LOOKAHEAD ** N , where N is the queue length. */
5807 static int max_lookahead_tries
;
5809 /* The following function returns maximal (or close to maximal) number
5810 of insns which can be issued on the same cycle and one of which
5811 insns is insns with the best rank (the first insn in READY). To
5812 make this function tries different samples of ready insns. READY
5813 is current queue `ready'. Global array READY_TRY reflects what
5814 insns are already issued in this try. The function stops immediately,
5815 if it reached the such a solution, that all instruction can be issued.
5816 INDEX will contain index of the best insn in READY. The following
5817 function is used only for first cycle multipass scheduling.
5821 This function expects recognized insns only. All USEs,
5822 CLOBBERs, etc must be filtered elsewhere. */
5824 max_issue (struct ready_list
*ready
, int privileged_n
, state_t state
,
5825 bool first_cycle_insn_p
, int *index
)
5827 int n
, i
, all
, n_ready
, best
, delay
, tries_num
;
5829 struct choice_entry
*top
;
5835 n_ready
= ready
->n_ready
;
5836 gcc_assert (dfa_lookahead
>= 1 && privileged_n
>= 0
5837 && privileged_n
<= n_ready
);
5839 /* Init MAX_LOOKAHEAD_TRIES. */
5840 if (max_lookahead_tries
== 0)
5842 max_lookahead_tries
= 100;
5843 for (i
= 0; i
< issue_rate
; i
++)
5844 max_lookahead_tries
*= dfa_lookahead
;
5847 /* Init max_points. */
5848 more_issue
= issue_rate
- cycle_issued_insns
;
5849 gcc_assert (more_issue
>= 0);
5851 /* The number of the issued insns in the best solution. */
5856 /* Set initial state of the search. */
5857 memcpy (top
->state
, state
, dfa_state_size
);
5858 top
->rest
= dfa_lookahead
;
5860 if (targetm
.sched
.first_cycle_multipass_begin
)
5861 targetm
.sched
.first_cycle_multipass_begin (&top
->target_data
,
5863 first_cycle_insn_p
);
5865 /* Count the number of the insns to search among. */
5866 for (all
= i
= 0; i
< n_ready
; i
++)
5870 if (sched_verbose
>= 2)
5872 fprintf (sched_dump
, ";;\t\tmax_issue among %d insns:", all
);
5873 debug_ready_list_1 (ready
, ready_try
);
5876 /* I is the index of the insn to try next. */
5881 if (/* If we've reached a dead end or searched enough of what we have
5884 /* or have nothing else to try... */
5886 /* or should not issue more. */
5887 || top
->n
>= more_issue
)
5889 /* ??? (... || i == n_ready). */
5890 gcc_assert (i
<= n_ready
);
5892 /* We should not issue more than issue_rate instructions. */
5893 gcc_assert (top
->n
<= more_issue
);
5895 if (top
== choice_stack
)
5898 if (best
< top
- choice_stack
)
5903 /* Try to find issued privileged insn. */
5904 while (n
&& !ready_try
[--n
])
5908 if (/* If all insns are equally good... */
5910 /* Or a privileged insn will be issued. */
5912 /* Then we have a solution. */
5914 best
= top
- choice_stack
;
5915 /* This is the index of the insn issued first in this
5917 *index
= choice_stack
[1].index
;
5918 if (top
->n
== more_issue
|| best
== all
)
5923 /* Set ready-list index to point to the last insn
5924 ('i++' below will advance it to the next insn). */
5930 if (targetm
.sched
.first_cycle_multipass_backtrack
)
5931 targetm
.sched
.first_cycle_multipass_backtrack (&top
->target_data
,
5932 ready_try
, n_ready
);
5935 memcpy (state
, top
->state
, dfa_state_size
);
5937 else if (!ready_try
[i
])
5940 if (tries_num
> max_lookahead_tries
)
5942 insn
= ready_element (ready
, i
);
5943 delay
= state_transition (state
, insn
);
5946 if (state_dead_lock_p (state
)
5947 || insn_finishes_cycle_p (insn
))
5948 /* We won't issue any more instructions in the next
5955 if (memcmp (top
->state
, state
, dfa_state_size
) != 0)
5958 /* Advance to the next choice_entry. */
5960 /* Initialize it. */
5961 top
->rest
= dfa_lookahead
;
5964 memcpy (top
->state
, state
, dfa_state_size
);
5967 if (targetm
.sched
.first_cycle_multipass_issue
)
5968 targetm
.sched
.first_cycle_multipass_issue (&top
->target_data
,
5978 /* Increase ready-list index. */
5982 if (targetm
.sched
.first_cycle_multipass_end
)
5983 targetm
.sched
.first_cycle_multipass_end (best
!= 0
5984 ? &choice_stack
[1].target_data
5987 /* Restore the original state of the DFA. */
5988 memcpy (state
, choice_stack
->state
, dfa_state_size
);
5993 /* The following function chooses insn from READY and modifies
5994 READY. The following function is used only for first
5995 cycle multipass scheduling.
5997 -1 if cycle should be advanced,
5998 0 if INSN_PTR is set to point to the desirable insn,
5999 1 if choose_ready () should be restarted without advancing the cycle. */
6001 choose_ready (struct ready_list
*ready
, bool first_cycle_insn_p
,
6002 rtx_insn
**insn_ptr
)
6004 if (dbg_cnt (sched_insn
) == false)
6006 if (nonscheduled_insns_begin
== NULL_RTX
)
6007 nonscheduled_insns_begin
= current_sched_info
->prev_head
;
6009 rtx_insn
*insn
= first_nonscheduled_insn ();
6011 if (QUEUE_INDEX (insn
) == QUEUE_READY
)
6012 /* INSN is in the ready_list. */
6014 ready_remove_insn (insn
);
6019 /* INSN is in the queue. Advance cycle to move it to the ready list. */
6020 gcc_assert (QUEUE_INDEX (insn
) >= 0);
6024 if (dfa_lookahead
<= 0 || SCHED_GROUP_P (ready_element (ready
, 0))
6025 || DEBUG_INSN_P (ready_element (ready
, 0)))
6027 if (targetm
.sched
.dispatch (NULL
, IS_DISPATCH_ON
))
6028 *insn_ptr
= ready_remove_first_dispatch (ready
);
6030 *insn_ptr
= ready_remove_first (ready
);
6036 /* Try to choose the best insn. */
6040 insn
= ready_element (ready
, 0);
6041 if (INSN_CODE (insn
) < 0)
6043 *insn_ptr
= ready_remove_first (ready
);
6047 /* Filter the search space. */
6048 for (i
= 0; i
< ready
->n_ready
; i
++)
6052 insn
= ready_element (ready
, i
);
6054 /* If this insn is recognizable we should have already
6055 recognized it earlier.
6056 ??? Not very clear where this is supposed to be done.
6058 gcc_checking_assert (INSN_CODE (insn
) >= 0
6059 || recog_memoized (insn
) < 0);
6060 if (INSN_CODE (insn
) < 0)
6062 /* Non-recognized insns at position 0 are handled above. */
6068 if (targetm
.sched
.first_cycle_multipass_dfa_lookahead_guard
)
6071 = (targetm
.sched
.first_cycle_multipass_dfa_lookahead_guard
6074 if (ready_try
[i
] < 0)
6075 /* Queue instruction for several cycles.
6076 We need to restart choose_ready as we have changed
6079 change_queue_index (insn
, -ready_try
[i
]);
6083 /* Make sure that we didn't end up with 0'th insn filtered out.
6084 Don't be tempted to make life easier for backends and just
6085 requeue 0'th insn if (ready_try[0] == 0) and restart
6086 choose_ready. Backends should be very considerate about
6087 requeueing instructions -- especially the highest priority
6088 one at position 0. */
6089 gcc_assert (ready_try
[i
] == 0 || i
> 0);
6094 gcc_assert (ready_try
[i
] == 0);
6095 /* INSN made it through the scrutiny of filters! */
6098 if (max_issue (ready
, 1, curr_state
, first_cycle_insn_p
, &index
) == 0)
6100 *insn_ptr
= ready_remove_first (ready
);
6101 if (sched_verbose
>= 4)
6102 fprintf (sched_dump
, ";;\t\tChosen insn (but can't issue) : %s \n",
6103 (*current_sched_info
->print_insn
) (*insn_ptr
, 0));
6108 if (sched_verbose
>= 4)
6109 fprintf (sched_dump
, ";;\t\tChosen insn : %s\n",
6110 (*current_sched_info
->print_insn
)
6111 (ready_element (ready
, index
), 0));
6113 *insn_ptr
= ready_remove (ready
, index
);
6119 /* This function is called when we have successfully scheduled a
6120 block. It uses the schedule stored in the scheduled_insns vector
6121 to rearrange the RTL. PREV_HEAD is used as the anchor to which we
6122 append the scheduled insns; TAIL is the insn after the scheduled
6123 block. TARGET_BB is the argument passed to schedule_block. */
6126 commit_schedule (rtx_insn
*prev_head
, rtx_insn
*tail
, basic_block
*target_bb
)
6131 last_scheduled_insn
= prev_head
;
6133 scheduled_insns
.iterate (i
, &insn
);
6136 if (control_flow_insn_p (last_scheduled_insn
)
6137 || current_sched_info
->advance_target_bb (*target_bb
, insn
))
6139 *target_bb
= current_sched_info
->advance_target_bb (*target_bb
, 0);
6145 x
= next_real_insn (last_scheduled_insn
);
6147 dump_new_block_header (1, *target_bb
, x
, tail
);
6150 last_scheduled_insn
= bb_note (*target_bb
);
6153 if (current_sched_info
->begin_move_insn
)
6154 (*current_sched_info
->begin_move_insn
) (insn
, last_scheduled_insn
);
6155 move_insn (insn
, last_scheduled_insn
,
6156 current_sched_info
->next_tail
);
6157 if (!DEBUG_INSN_P (insn
))
6158 reemit_notes (insn
);
6159 last_scheduled_insn
= insn
;
6162 scheduled_insns
.truncate (0);
6165 /* Examine all insns on the ready list and queue those which can't be
6166 issued in this cycle. TEMP_STATE is temporary scheduler state we
6167 can use as scratch space. If FIRST_CYCLE_INSN_P is true, no insns
6168 have been issued for the current cycle, which means it is valid to
6169 issue an asm statement.
6171 If SHADOWS_ONLY_P is true, we eliminate all real insns and only
6172 leave those for which SHADOW_P is true. If MODULO_EPILOGUE is true,
6173 we only leave insns which have an INSN_EXACT_TICK. */
6176 prune_ready_list (state_t temp_state
, bool first_cycle_insn_p
,
6177 bool shadows_only_p
, bool modulo_epilogue_p
)
6180 bool sched_group_found
= false;
6181 int min_cost_group
= 1;
6186 for (i
= 0; i
< ready
.n_ready
; i
++)
6188 rtx_insn
*insn
= ready_element (&ready
, i
);
6189 if (SCHED_GROUP_P (insn
))
6191 sched_group_found
= true;
6196 /* Make two passes if there's a SCHED_GROUP_P insn; make sure to handle
6197 such an insn first and note its cost, then schedule all other insns
6198 for one cycle later. */
6199 for (pass
= sched_group_found
? 0 : 1; pass
< 2; )
6201 int n
= ready
.n_ready
;
6202 for (i
= 0; i
< n
; i
++)
6204 rtx_insn
*insn
= ready_element (&ready
, i
);
6206 const char *reason
= "resource conflict";
6208 if (DEBUG_INSN_P (insn
))
6211 if (sched_group_found
&& !SCHED_GROUP_P (insn
))
6215 cost
= min_cost_group
;
6216 reason
= "not in sched group";
6218 else if (modulo_epilogue_p
6219 && INSN_EXACT_TICK (insn
) == INVALID_TICK
)
6221 cost
= max_insn_queue_index
;
6222 reason
= "not an epilogue insn";
6224 else if (shadows_only_p
&& !SHADOW_P (insn
))
6227 reason
= "not a shadow";
6229 else if (recog_memoized (insn
) < 0)
6231 if (!first_cycle_insn_p
6232 && (GET_CODE (PATTERN (insn
)) == ASM_INPUT
6233 || asm_noperands (PATTERN (insn
)) >= 0))
6237 else if (sched_pressure
!= SCHED_PRESSURE_NONE
)
6239 if (sched_pressure
== SCHED_PRESSURE_MODEL
6240 && INSN_TICK (insn
) <= clock_var
)
6242 memcpy (temp_state
, curr_state
, dfa_state_size
);
6243 if (state_transition (temp_state
, insn
) >= 0)
6244 INSN_TICK (insn
) = clock_var
+ 1;
6254 struct delay_pair
*delay_entry
;
6256 = delay_htab
->find_with_hash (insn
,
6257 htab_hash_pointer (insn
));
6258 while (delay_entry
&& delay_cost
== 0)
6260 delay_cost
= estimate_shadow_tick (delay_entry
);
6261 if (delay_cost
> max_insn_queue_index
)
6262 delay_cost
= max_insn_queue_index
;
6263 delay_entry
= delay_entry
->next_same_i1
;
6267 memcpy (temp_state
, curr_state
, dfa_state_size
);
6268 cost
= state_transition (temp_state
, insn
);
6273 if (cost
< delay_cost
)
6276 reason
= "shadow tick";
6281 if (SCHED_GROUP_P (insn
) && cost
> min_cost_group
)
6282 min_cost_group
= cost
;
6283 ready_remove (&ready
, i
);
6284 /* Normally we'd want to queue INSN for COST cycles. However,
6285 if SCHED_GROUP_P is set, then we must ensure that nothing
6286 else comes between INSN and its predecessor. If there is
6287 some other insn ready to fire on the next cycle, then that
6288 invariant would be broken.
6290 So when SCHED_GROUP_P is set, just queue this insn for a
6292 queue_insn (insn
, SCHED_GROUP_P (insn
) ? 1 : cost
, reason
);
6302 /* Called when we detect that the schedule is impossible. We examine the
6303 backtrack queue to find the earliest insn that caused this condition. */
6305 static struct haifa_saved_data
*
6306 verify_shadows (void)
6308 struct haifa_saved_data
*save
, *earliest_fail
= NULL
;
6309 for (save
= backtrack_queue
; save
; save
= save
->next
)
6312 struct delay_pair
*pair
= save
->delay_pair
;
6313 rtx_insn
*i1
= pair
->i1
;
6315 for (; pair
; pair
= pair
->next_same_i1
)
6317 rtx_insn
*i2
= pair
->i2
;
6319 if (QUEUE_INDEX (i2
) == QUEUE_SCHEDULED
)
6322 t
= INSN_TICK (i1
) + pair_delay (pair
);
6325 if (sched_verbose
>= 2)
6326 fprintf (sched_dump
,
6327 ";;\t\tfailed delay requirements for %d/%d (%d->%d)"
6329 INSN_UID (pair
->i1
), INSN_UID (pair
->i2
),
6330 INSN_TICK (pair
->i1
), INSN_EXACT_TICK (pair
->i2
));
6331 earliest_fail
= save
;
6334 if (QUEUE_INDEX (i2
) >= 0)
6336 int queued_for
= INSN_TICK (i2
);
6340 if (sched_verbose
>= 2)
6341 fprintf (sched_dump
,
6342 ";;\t\tfailed delay requirements for %d/%d"
6343 " (%d->%d), queued too late\n",
6344 INSN_UID (pair
->i1
), INSN_UID (pair
->i2
),
6345 INSN_TICK (pair
->i1
), INSN_EXACT_TICK (pair
->i2
));
6346 earliest_fail
= save
;
6353 return earliest_fail
;
6356 /* Print instructions together with useful scheduling information between
6357 HEAD and TAIL (inclusive). */
6359 dump_insn_stream (rtx_insn
*head
, rtx_insn
*tail
)
6361 fprintf (sched_dump
, ";;\t| insn | prio |\n");
6363 rtx_insn
*next_tail
= NEXT_INSN (tail
);
6364 for (rtx_insn
*insn
= head
; insn
!= next_tail
; insn
= NEXT_INSN (insn
))
6366 int priority
= NOTE_P (insn
) ? 0 : INSN_PRIORITY (insn
);
6367 const char *pattern
= (NOTE_P (insn
)
6369 : str_pattern_slim (PATTERN (insn
)));
6371 fprintf (sched_dump
, ";;\t| %4d | %4d | %-30s ",
6372 INSN_UID (insn
), priority
, pattern
);
6374 if (sched_verbose
>= 4)
6376 if (NOTE_P (insn
) || recog_memoized (insn
) < 0)
6377 fprintf (sched_dump
, "nothing");
6379 print_reservation (sched_dump
, insn
);
6381 fprintf (sched_dump
, "\n");
6385 /* Use forward list scheduling to rearrange insns of block pointed to by
6386 TARGET_BB, possibly bringing insns from subsequent blocks in the same
6390 schedule_block (basic_block
*target_bb
, state_t init_state
)
6393 bool success
= modulo_ii
== 0;
6394 struct sched_block_state ls
;
6395 state_t temp_state
= NULL
; /* It is used for multipass scheduling. */
6396 int sort_p
, advance
, start_clock_var
;
6398 /* Head/tail info for this block. */
6399 rtx_insn
*prev_head
= current_sched_info
->prev_head
;
6400 rtx_insn
*next_tail
= current_sched_info
->next_tail
;
6401 rtx_insn
*head
= NEXT_INSN (prev_head
);
6402 rtx_insn
*tail
= PREV_INSN (next_tail
);
6404 if ((current_sched_info
->flags
& DONT_BREAK_DEPENDENCIES
) == 0
6405 && sched_pressure
!= SCHED_PRESSURE_MODEL
&& !sched_fusion
)
6406 find_modifiable_mems (head
, tail
);
6408 /* We used to have code to avoid getting parameters moved from hard
6409 argument registers into pseudos.
6411 However, it was removed when it proved to be of marginal benefit
6412 and caused problems because schedule_block and compute_forward_dependences
6413 had different notions of what the "head" insn was. */
6415 gcc_assert (head
!= tail
|| INSN_P (head
));
6417 haifa_recovery_bb_recently_added_p
= false;
6419 backtrack_queue
= NULL
;
6424 dump_new_block_header (0, *target_bb
, head
, tail
);
6426 if (sched_verbose
>= 2)
6428 dump_insn_stream (head
, tail
);
6429 memset (&rank_for_schedule_stats
, 0,
6430 sizeof (rank_for_schedule_stats
));
6434 if (init_state
== NULL
)
6435 state_reset (curr_state
);
6437 memcpy (curr_state
, init_state
, dfa_state_size
);
6439 /* Clear the ready list. */
6440 ready
.first
= ready
.veclen
- 1;
6444 /* It is used for first cycle multipass scheduling. */
6445 temp_state
= alloca (dfa_state_size
);
6447 if (targetm
.sched
.init
)
6448 targetm
.sched
.init (sched_dump
, sched_verbose
, ready
.veclen
);
6450 /* We start inserting insns after PREV_HEAD. */
6451 last_scheduled_insn
= prev_head
;
6452 last_nondebug_scheduled_insn
= NULL
;
6453 nonscheduled_insns_begin
= NULL
;
6455 gcc_assert ((NOTE_P (last_scheduled_insn
)
6456 || DEBUG_INSN_P (last_scheduled_insn
))
6457 && BLOCK_FOR_INSN (last_scheduled_insn
) == *target_bb
);
6459 /* Initialize INSN_QUEUE. Q_SIZE is the total number of insns in the
6464 insn_queue
= XALLOCAVEC (rtx_insn_list
*, max_insn_queue_index
+ 1);
6465 memset (insn_queue
, 0, (max_insn_queue_index
+ 1) * sizeof (rtx
));
6467 /* Start just before the beginning of time. */
6470 /* We need queue and ready lists and clock_var be initialized
6471 in try_ready () (which is called through init_ready_list ()). */
6472 (*current_sched_info
->init_ready_list
) ();
6475 sched_pressure_start_bb (*target_bb
);
6477 /* The algorithm is O(n^2) in the number of ready insns at any given
6478 time in the worst case. Before reload we are more likely to have
6479 big lists so truncate them to a reasonable size. */
6480 if (!reload_completed
6481 && ready
.n_ready
- ready
.n_debug
> MAX_SCHED_READY_INSNS
)
6483 ready_sort_debug (&ready
);
6484 ready_sort_real (&ready
);
6486 /* Find first free-standing insn past MAX_SCHED_READY_INSNS.
6487 If there are debug insns, we know they're first. */
6488 for (i
= MAX_SCHED_READY_INSNS
+ ready
.n_debug
; i
< ready
.n_ready
; i
++)
6489 if (!SCHED_GROUP_P (ready_element (&ready
, i
)))
6492 if (sched_verbose
>= 2)
6494 fprintf (sched_dump
,
6495 ";;\t\tReady list on entry: %d insns: ", ready
.n_ready
);
6496 debug_ready_list (&ready
);
6497 fprintf (sched_dump
,
6498 ";;\t\t before reload => truncated to %d insns\n", i
);
6501 /* Delay all insns past it for 1 cycle. If debug counter is
6502 activated make an exception for the insn right after
6503 nonscheduled_insns_begin. */
6505 rtx_insn
*skip_insn
;
6507 if (dbg_cnt (sched_insn
) == false)
6508 skip_insn
= first_nonscheduled_insn ();
6512 while (i
< ready
.n_ready
)
6516 insn
= ready_remove (&ready
, i
);
6518 if (insn
!= skip_insn
)
6519 queue_insn (insn
, 1, "list truncated");
6522 ready_add (&ready
, skip_insn
, true);
6526 /* Now we can restore basic block notes and maintain precise cfg. */
6527 restore_bb_notes (*target_bb
);
6529 last_clock_var
= -1;
6533 gcc_assert (scheduled_insns
.length () == 0);
6535 must_backtrack
= false;
6536 modulo_insns_scheduled
= 0;
6538 ls
.modulo_epilogue
= false;
6539 ls
.first_cycle_insn_p
= true;
6541 /* Loop until all the insns in BB are scheduled. */
6542 while ((*current_sched_info
->schedule_more_p
) ())
6544 perform_replacements_new_cycle ();
6547 start_clock_var
= clock_var
;
6551 advance_one_cycle ();
6553 /* Add to the ready list all pending insns that can be issued now.
6554 If there are no ready insns, increment clock until one
6555 is ready and add all pending insns at that point to the ready
6557 queue_to_ready (&ready
);
6559 gcc_assert (ready
.n_ready
);
6561 if (sched_verbose
>= 2)
6563 fprintf (sched_dump
, ";;\t\tReady list after queue_to_ready:");
6564 debug_ready_list (&ready
);
6566 advance
-= clock_var
- start_clock_var
;
6568 while (advance
> 0);
6570 if (ls
.modulo_epilogue
)
6572 int stage
= clock_var
/ modulo_ii
;
6573 if (stage
> modulo_last_stage
* 2 + 2)
6575 if (sched_verbose
>= 2)
6576 fprintf (sched_dump
,
6577 ";;\t\tmodulo scheduled succeeded at II %d\n",
6583 else if (modulo_ii
> 0)
6585 int stage
= clock_var
/ modulo_ii
;
6586 if (stage
> modulo_max_stages
)
6588 if (sched_verbose
>= 2)
6589 fprintf (sched_dump
,
6590 ";;\t\tfailing schedule due to excessive stages\n");
6593 if (modulo_n_insns
== modulo_insns_scheduled
6594 && stage
> modulo_last_stage
)
6596 if (sched_verbose
>= 2)
6597 fprintf (sched_dump
,
6598 ";;\t\tfound kernel after %d stages, II %d\n",
6600 ls
.modulo_epilogue
= true;
6604 prune_ready_list (temp_state
, true, false, ls
.modulo_epilogue
);
6605 if (ready
.n_ready
== 0)
6610 ls
.shadows_only_p
= false;
6611 cycle_issued_insns
= 0;
6612 ls
.can_issue_more
= issue_rate
;
6619 if (sort_p
&& ready
.n_ready
> 0)
6621 /* Sort the ready list based on priority. This must be
6622 done every iteration through the loop, as schedule_insn
6623 may have readied additional insns that will not be
6624 sorted correctly. */
6625 ready_sort (&ready
);
6627 if (sched_verbose
>= 2)
6629 fprintf (sched_dump
,
6630 ";;\t\tReady list after ready_sort: ");
6631 debug_ready_list (&ready
);
6635 /* We don't want md sched reorder to even see debug isns, so put
6636 them out right away. */
6637 if (ready
.n_ready
&& DEBUG_INSN_P (ready_element (&ready
, 0))
6638 && (*current_sched_info
->schedule_more_p
) ())
6640 while (ready
.n_ready
&& DEBUG_INSN_P (ready_element (&ready
, 0)))
6642 rtx_insn
*insn
= ready_remove_first (&ready
);
6643 gcc_assert (DEBUG_INSN_P (insn
));
6644 (*current_sched_info
->begin_schedule_ready
) (insn
);
6645 scheduled_insns
.safe_push (insn
);
6646 last_scheduled_insn
= insn
;
6647 advance
= schedule_insn (insn
);
6648 gcc_assert (advance
== 0);
6649 if (ready
.n_ready
> 0)
6650 ready_sort (&ready
);
6654 if (ls
.first_cycle_insn_p
&& !ready
.n_ready
)
6657 resume_after_backtrack
:
6658 /* Allow the target to reorder the list, typically for
6659 better instruction bundling. */
6661 && (ready
.n_ready
== 0
6662 || !SCHED_GROUP_P (ready_element (&ready
, 0))))
6664 if (ls
.first_cycle_insn_p
&& targetm
.sched
.reorder
)
6666 = targetm
.sched
.reorder (sched_dump
, sched_verbose
,
6667 ready_lastpos (&ready
),
6668 &ready
.n_ready
, clock_var
);
6669 else if (!ls
.first_cycle_insn_p
&& targetm
.sched
.reorder2
)
6671 = targetm
.sched
.reorder2 (sched_dump
, sched_verbose
,
6673 ? ready_lastpos (&ready
) : NULL
,
6674 &ready
.n_ready
, clock_var
);
6677 restart_choose_ready
:
6678 if (sched_verbose
>= 2)
6680 fprintf (sched_dump
, ";;\tReady list (t = %3d): ",
6682 debug_ready_list (&ready
);
6683 if (sched_pressure
== SCHED_PRESSURE_WEIGHTED
)
6684 print_curr_reg_pressure ();
6687 if (ready
.n_ready
== 0
6688 && ls
.can_issue_more
6689 && reload_completed
)
6691 /* Allow scheduling insns directly from the queue in case
6692 there's nothing better to do (ready list is empty) but
6693 there are still vacant dispatch slots in the current cycle. */
6694 if (sched_verbose
>= 6)
6695 fprintf (sched_dump
,";;\t\tSecond chance\n");
6696 memcpy (temp_state
, curr_state
, dfa_state_size
);
6697 if (early_queue_to_ready (temp_state
, &ready
))
6698 ready_sort (&ready
);
6701 if (ready
.n_ready
== 0
6702 || !ls
.can_issue_more
6703 || state_dead_lock_p (curr_state
)
6704 || !(*current_sched_info
->schedule_more_p
) ())
6707 /* Select and remove the insn from the ready list. */
6713 res
= choose_ready (&ready
, ls
.first_cycle_insn_p
, &insn
);
6719 goto restart_choose_ready
;
6721 gcc_assert (insn
!= NULL_RTX
);
6724 insn
= ready_remove_first (&ready
);
6726 if (sched_pressure
!= SCHED_PRESSURE_NONE
6727 && INSN_TICK (insn
) > clock_var
)
6729 ready_add (&ready
, insn
, true);
6734 if (targetm
.sched
.dfa_new_cycle
6735 && targetm
.sched
.dfa_new_cycle (sched_dump
, sched_verbose
,
6736 insn
, last_clock_var
,
6737 clock_var
, &sort_p
))
6738 /* SORT_P is used by the target to override sorting
6739 of the ready list. This is needed when the target
6740 has modified its internal structures expecting that
6741 the insn will be issued next. As we need the insn
6742 to have the highest priority (so it will be returned by
6743 the ready_remove_first call above), we invoke
6744 ready_add (&ready, insn, true).
6745 But, still, there is one issue: INSN can be later
6746 discarded by scheduler's front end through
6747 current_sched_info->can_schedule_ready_p, hence, won't
6750 ready_add (&ready
, insn
, true);
6756 if (current_sched_info
->can_schedule_ready_p
6757 && ! (*current_sched_info
->can_schedule_ready_p
) (insn
))
6758 /* We normally get here only if we don't want to move
6759 insn from the split block. */
6761 TODO_SPEC (insn
) = DEP_POSTPONED
;
6762 goto restart_choose_ready
;
6767 /* If this insn is the first part of a delay-slot pair, record a
6769 struct delay_pair
*delay_entry
;
6771 = delay_htab
->find_with_hash (insn
, htab_hash_pointer (insn
));
6774 save_backtrack_point (delay_entry
, ls
);
6775 if (sched_verbose
>= 2)
6776 fprintf (sched_dump
, ";;\t\tsaving backtrack point\n");
6780 /* DECISION is made. */
6782 if (modulo_ii
> 0 && INSN_UID (insn
) < modulo_iter0_max_uid
)
6784 modulo_insns_scheduled
++;
6785 modulo_last_stage
= clock_var
/ modulo_ii
;
6787 if (TODO_SPEC (insn
) & SPECULATIVE
)
6788 generate_recovery_code (insn
);
6790 if (targetm
.sched
.dispatch (NULL
, IS_DISPATCH_ON
))
6791 targetm
.sched
.dispatch_do (insn
, ADD_TO_DISPATCH_WINDOW
);
6793 /* Update counters, etc in the scheduler's front end. */
6794 (*current_sched_info
->begin_schedule_ready
) (insn
);
6795 scheduled_insns
.safe_push (insn
);
6796 gcc_assert (NONDEBUG_INSN_P (insn
));
6797 last_nondebug_scheduled_insn
= last_scheduled_insn
= insn
;
6799 if (recog_memoized (insn
) >= 0)
6801 memcpy (temp_state
, curr_state
, dfa_state_size
);
6802 cost
= state_transition (curr_state
, insn
);
6803 if (sched_pressure
!= SCHED_PRESSURE_WEIGHTED
&& !sched_fusion
)
6804 gcc_assert (cost
< 0);
6805 if (memcmp (temp_state
, curr_state
, dfa_state_size
) != 0)
6806 cycle_issued_insns
++;
6810 asm_p
= (GET_CODE (PATTERN (insn
)) == ASM_INPUT
6811 || asm_noperands (PATTERN (insn
)) >= 0);
6813 if (targetm
.sched
.variable_issue
)
6815 targetm
.sched
.variable_issue (sched_dump
, sched_verbose
,
6816 insn
, ls
.can_issue_more
);
6817 /* A naked CLOBBER or USE generates no instruction, so do
6818 not count them against the issue rate. */
6819 else if (GET_CODE (PATTERN (insn
)) != USE
6820 && GET_CODE (PATTERN (insn
)) != CLOBBER
)
6821 ls
.can_issue_more
--;
6822 advance
= schedule_insn (insn
);
6824 if (SHADOW_P (insn
))
6825 ls
.shadows_only_p
= true;
6827 /* After issuing an asm insn we should start a new cycle. */
6828 if (advance
== 0 && asm_p
)
6837 ls
.first_cycle_insn_p
= false;
6838 if (ready
.n_ready
> 0)
6839 prune_ready_list (temp_state
, false, ls
.shadows_only_p
,
6840 ls
.modulo_epilogue
);
6844 if (!must_backtrack
)
6845 for (i
= 0; i
< ready
.n_ready
; i
++)
6847 rtx_insn
*insn
= ready_element (&ready
, i
);
6848 if (INSN_EXACT_TICK (insn
) == clock_var
)
6850 must_backtrack
= true;
6855 if (must_backtrack
&& modulo_ii
> 0)
6857 if (modulo_backtracks_left
== 0)
6859 modulo_backtracks_left
--;
6861 while (must_backtrack
)
6863 struct haifa_saved_data
*failed
;
6864 rtx_insn
*failed_insn
;
6866 must_backtrack
= false;
6867 failed
= verify_shadows ();
6868 gcc_assert (failed
);
6870 failed_insn
= failed
->delay_pair
->i1
;
6871 /* Clear these queues. */
6872 perform_replacements_new_cycle ();
6873 toggle_cancelled_flags (false);
6874 unschedule_insns_until (failed_insn
);
6875 while (failed
!= backtrack_queue
)
6876 free_topmost_backtrack_point (true);
6877 restore_last_backtrack_point (&ls
);
6878 if (sched_verbose
>= 2)
6879 fprintf (sched_dump
, ";;\t\trewind to cycle %d\n", clock_var
);
6880 /* Delay by at least a cycle. This could cause additional
6882 queue_insn (failed_insn
, 1, "backtracked");
6886 if (ready
.n_ready
> 0)
6887 goto resume_after_backtrack
;
6890 if (clock_var
== 0 && ls
.first_cycle_insn_p
)
6896 ls
.first_cycle_insn_p
= true;
6898 if (ls
.modulo_epilogue
)
6901 if (!ls
.first_cycle_insn_p
|| advance
)
6902 advance_one_cycle ();
6903 perform_replacements_new_cycle ();
6906 /* Once again, debug insn suckiness: they can be on the ready list
6907 even if they have unresolved dependencies. To make our view
6908 of the world consistent, remove such "ready" insns. */
6909 restart_debug_insn_loop
:
6910 for (i
= ready
.n_ready
- 1; i
>= 0; i
--)
6914 x
= ready_element (&ready
, i
);
6915 if (DEPS_LIST_FIRST (INSN_HARD_BACK_DEPS (x
)) != NULL
6916 || DEPS_LIST_FIRST (INSN_SPEC_BACK_DEPS (x
)) != NULL
)
6918 ready_remove (&ready
, i
);
6919 goto restart_debug_insn_loop
;
6922 for (i
= ready
.n_ready
- 1; i
>= 0; i
--)
6926 x
= ready_element (&ready
, i
);
6927 resolve_dependencies (x
);
6929 for (i
= 0; i
<= max_insn_queue_index
; i
++)
6931 rtx_insn_list
*link
;
6932 while ((link
= insn_queue
[i
]) != NULL
)
6934 rtx_insn
*x
= link
->insn ();
6935 insn_queue
[i
] = link
->next ();
6936 QUEUE_INDEX (x
) = QUEUE_NOWHERE
;
6937 free_INSN_LIST_node (link
);
6938 resolve_dependencies (x
);
6944 undo_all_replacements ();
6949 fprintf (sched_dump
, ";;\tReady list (final): ");
6950 debug_ready_list (&ready
);
6953 if (modulo_ii
== 0 && current_sched_info
->queue_must_finish_empty
)
6954 /* Sanity check -- queue must be empty now. Meaningless if region has
6956 gcc_assert (!q_size
&& !ready
.n_ready
&& !ready
.n_debug
);
6957 else if (modulo_ii
== 0)
6959 /* We must maintain QUEUE_INDEX between blocks in region. */
6960 for (i
= ready
.n_ready
- 1; i
>= 0; i
--)
6964 x
= ready_element (&ready
, i
);
6965 QUEUE_INDEX (x
) = QUEUE_NOWHERE
;
6966 TODO_SPEC (x
) = HARD_DEP
;
6970 for (i
= 0; i
<= max_insn_queue_index
; i
++)
6972 rtx_insn_list
*link
;
6973 for (link
= insn_queue
[i
]; link
; link
= link
->next ())
6978 QUEUE_INDEX (x
) = QUEUE_NOWHERE
;
6979 TODO_SPEC (x
) = HARD_DEP
;
6981 free_INSN_LIST_list (&insn_queue
[i
]);
6985 if (sched_pressure
== SCHED_PRESSURE_MODEL
)
6986 model_end_schedule ();
6990 commit_schedule (prev_head
, tail
, target_bb
);
6992 fprintf (sched_dump
, ";; total time = %d\n", clock_var
);
6995 last_scheduled_insn
= tail
;
6997 scheduled_insns
.truncate (0);
6999 if (!current_sched_info
->queue_must_finish_empty
7000 || haifa_recovery_bb_recently_added_p
)
7002 /* INSN_TICK (minimum clock tick at which the insn becomes
7003 ready) may be not correct for the insn in the subsequent
7004 blocks of the region. We should use a correct value of
7005 `clock_var' or modify INSN_TICK. It is better to keep
7006 clock_var value equal to 0 at the start of a basic block.
7007 Therefore we modify INSN_TICK here. */
7008 fix_inter_tick (NEXT_INSN (prev_head
), last_scheduled_insn
);
7011 if (targetm
.sched
.finish
)
7013 targetm
.sched
.finish (sched_dump
, sched_verbose
);
7014 /* Target might have added some instructions to the scheduled block
7015 in its md_finish () hook. These new insns don't have any data
7016 initialized and to identify them we extend h_i_d so that they'll
7018 sched_extend_luids ();
7021 /* Update head/tail boundaries. */
7022 head
= NEXT_INSN (prev_head
);
7023 tail
= last_scheduled_insn
;
7027 fprintf (sched_dump
, ";; new head = %d\n;; new tail = %d\n",
7028 INSN_UID (head
), INSN_UID (tail
));
7030 if (sched_verbose
>= 2)
7032 dump_insn_stream (head
, tail
);
7033 print_rank_for_schedule_stats (";; TOTAL ", &rank_for_schedule_stats
,
7037 fprintf (sched_dump
, "\n");
7040 head
= restore_other_notes (head
, NULL
);
7042 current_sched_info
->head
= head
;
7043 current_sched_info
->tail
= tail
;
7045 free_backtrack_queue ();
7050 /* Set_priorities: compute priority of each insn in the block. */
7053 set_priorities (rtx_insn
*head
, rtx_insn
*tail
)
7057 int sched_max_insns_priority
=
7058 current_sched_info
->sched_max_insns_priority
;
7059 rtx_insn
*prev_head
;
7061 if (head
== tail
&& ! INSN_P (head
))
7066 prev_head
= PREV_INSN (head
);
7067 for (insn
= tail
; insn
!= prev_head
; insn
= PREV_INSN (insn
))
7073 (void) priority (insn
);
7075 gcc_assert (INSN_PRIORITY_KNOWN (insn
));
7077 sched_max_insns_priority
= MAX (sched_max_insns_priority
,
7078 INSN_PRIORITY (insn
));
7081 current_sched_info
->sched_max_insns_priority
= sched_max_insns_priority
;
7086 /* Set dump and sched_verbose for the desired debugging output. If no
7087 dump-file was specified, but -fsched-verbose=N (any N), print to stderr.
7088 For -fsched-verbose=N, N>=10, print everything to stderr. */
7090 setup_sched_dump (void)
7092 sched_verbose
= sched_verbose_param
;
7093 if (sched_verbose_param
== 0 && dump_file
)
7095 sched_dump
= ((sched_verbose_param
>= 10 || !dump_file
)
7096 ? stderr
: dump_file
);
7099 /* Allocate data for register pressure sensitive scheduling. */
7101 alloc_global_sched_pressure_data (void)
7103 if (sched_pressure
!= SCHED_PRESSURE_NONE
)
7105 int i
, max_regno
= max_reg_num ();
7107 if (sched_dump
!= NULL
)
7108 /* We need info about pseudos for rtl dumps about pseudo
7109 classes and costs. */
7110 regstat_init_n_sets_and_refs ();
7111 ira_set_pseudo_classes (true, sched_verbose
? sched_dump
: NULL
);
7112 sched_regno_pressure_class
7113 = (enum reg_class
*) xmalloc (max_regno
* sizeof (enum reg_class
));
7114 for (i
= 0; i
< max_regno
; i
++)
7115 sched_regno_pressure_class
[i
]
7116 = (i
< FIRST_PSEUDO_REGISTER
7117 ? ira_pressure_class_translate
[REGNO_REG_CLASS (i
)]
7118 : ira_pressure_class_translate
[reg_allocno_class (i
)]);
7119 curr_reg_live
= BITMAP_ALLOC (NULL
);
7120 if (sched_pressure
== SCHED_PRESSURE_WEIGHTED
)
7122 saved_reg_live
= BITMAP_ALLOC (NULL
);
7123 region_ref_regs
= BITMAP_ALLOC (NULL
);
7126 /* Calculate number of CALL_USED_REGS in register classes that
7127 we calculate register pressure for. */
7128 for (int c
= 0; c
< ira_pressure_classes_num
; ++c
)
7130 enum reg_class cl
= ira_pressure_classes
[c
];
7132 call_used_regs_num
[cl
] = 0;
7134 for (int i
= 0; i
< ira_class_hard_regs_num
[cl
]; ++i
)
7135 if (call_used_regs
[ira_class_hard_regs
[cl
][i
]])
7136 ++call_used_regs_num
[cl
];
7141 /* Free data for register pressure sensitive scheduling. Also called
7142 from schedule_region when stopping sched-pressure early. */
7144 free_global_sched_pressure_data (void)
7146 if (sched_pressure
!= SCHED_PRESSURE_NONE
)
7148 if (regstat_n_sets_and_refs
!= NULL
)
7149 regstat_free_n_sets_and_refs ();
7150 if (sched_pressure
== SCHED_PRESSURE_WEIGHTED
)
7152 BITMAP_FREE (region_ref_regs
);
7153 BITMAP_FREE (saved_reg_live
);
7155 BITMAP_FREE (curr_reg_live
);
7156 free (sched_regno_pressure_class
);
7160 /* Initialize some global state for the scheduler. This function works
7161 with the common data shared between all the schedulers. It is called
7162 from the scheduler specific initialization routine. */
7167 /* Disable speculative loads in their presence if cc0 defined. */
7169 flag_schedule_speculative_load
= 0;
7171 if (targetm
.sched
.dispatch (NULL
, IS_DISPATCH_ON
))
7172 targetm
.sched
.dispatch_do (NULL
, DISPATCH_INIT
);
7174 if (live_range_shrinkage_p
)
7175 sched_pressure
= SCHED_PRESSURE_WEIGHTED
;
7176 else if (flag_sched_pressure
7177 && !reload_completed
7178 && common_sched_info
->sched_pass_id
== SCHED_RGN_PASS
)
7179 sched_pressure
= ((enum sched_pressure_algorithm
)
7180 PARAM_VALUE (PARAM_SCHED_PRESSURE_ALGORITHM
));
7182 sched_pressure
= SCHED_PRESSURE_NONE
;
7184 if (sched_pressure
!= SCHED_PRESSURE_NONE
)
7185 ira_setup_eliminable_regset ();
7187 /* Initialize SPEC_INFO. */
7188 if (targetm
.sched
.set_sched_flags
)
7190 spec_info
= &spec_info_var
;
7191 targetm
.sched
.set_sched_flags (spec_info
);
7193 if (spec_info
->mask
!= 0)
7195 spec_info
->data_weakness_cutoff
=
7196 (PARAM_VALUE (PARAM_SCHED_SPEC_PROB_CUTOFF
) * MAX_DEP_WEAK
) / 100;
7197 spec_info
->control_weakness_cutoff
=
7198 (PARAM_VALUE (PARAM_SCHED_SPEC_PROB_CUTOFF
)
7199 * REG_BR_PROB_BASE
) / 100;
7202 /* So we won't read anything accidentally. */
7207 /* So we won't read anything accidentally. */
7210 /* Initialize issue_rate. */
7211 if (targetm
.sched
.issue_rate
)
7212 issue_rate
= targetm
.sched
.issue_rate ();
7216 if (targetm
.sched
.first_cycle_multipass_dfa_lookahead
7217 /* Don't use max_issue with reg_pressure scheduling. Multipass
7218 scheduling and reg_pressure scheduling undo each other's decisions. */
7219 && sched_pressure
== SCHED_PRESSURE_NONE
)
7220 dfa_lookahead
= targetm
.sched
.first_cycle_multipass_dfa_lookahead ();
7224 /* Set to "0" so that we recalculate. */
7225 max_lookahead_tries
= 0;
7227 if (targetm
.sched
.init_dfa_pre_cycle_insn
)
7228 targetm
.sched
.init_dfa_pre_cycle_insn ();
7230 if (targetm
.sched
.init_dfa_post_cycle_insn
)
7231 targetm
.sched
.init_dfa_post_cycle_insn ();
7234 dfa_state_size
= state_size ();
7236 init_alias_analysis ();
7239 df_set_flags (DF_LR_RUN_DCE
);
7240 df_note_add_problem ();
7242 /* More problems needed for interloop dep calculation in SMS. */
7243 if (common_sched_info
->sched_pass_id
== SCHED_SMS_PASS
)
7245 df_rd_add_problem ();
7246 df_chain_add_problem (DF_DU_CHAIN
+ DF_UD_CHAIN
);
7251 /* Do not run DCE after reload, as this can kill nops inserted
7253 if (reload_completed
)
7254 df_clear_flags (DF_LR_RUN_DCE
);
7256 regstat_compute_calls_crossed ();
7258 if (targetm
.sched
.init_global
)
7259 targetm
.sched
.init_global (sched_dump
, sched_verbose
, get_max_uid () + 1);
7261 alloc_global_sched_pressure_data ();
7263 curr_state
= xmalloc (dfa_state_size
);
7266 static void haifa_init_only_bb (basic_block
, basic_block
);
7268 /* Initialize data structures specific to the Haifa scheduler. */
7270 haifa_sched_init (void)
7272 setup_sched_dump ();
7275 scheduled_insns
.create (0);
7277 if (spec_info
!= NULL
)
7279 sched_deps_info
->use_deps_list
= 1;
7280 sched_deps_info
->generate_spec_deps
= 1;
7283 /* Initialize luids, dependency caches, target and h_i_d for the
7287 bbs
.create (n_basic_blocks_for_fn (cfun
));
7292 FOR_EACH_BB_FN (bb
, cfun
)
7293 bbs
.quick_push (bb
);
7294 sched_init_luids (bbs
);
7295 sched_deps_init (true);
7296 sched_extend_target ();
7297 haifa_init_h_i_d (bbs
);
7302 sched_init_only_bb
= haifa_init_only_bb
;
7303 sched_split_block
= sched_split_block_1
;
7304 sched_create_empty_bb
= sched_create_empty_bb_1
;
7305 haifa_recovery_bb_ever_added_p
= false;
7307 nr_begin_data
= nr_begin_control
= nr_be_in_data
= nr_be_in_control
= 0;
7308 before_recovery
= 0;
7314 /* Finish work with the data specific to the Haifa scheduler. */
7316 haifa_sched_finish (void)
7318 sched_create_empty_bb
= NULL
;
7319 sched_split_block
= NULL
;
7320 sched_init_only_bb
= NULL
;
7322 if (spec_info
&& spec_info
->dump
)
7324 char c
= reload_completed
? 'a' : 'b';
7326 fprintf (spec_info
->dump
,
7327 ";; %s:\n", current_function_name ());
7329 fprintf (spec_info
->dump
,
7330 ";; Procedure %cr-begin-data-spec motions == %d\n",
7332 fprintf (spec_info
->dump
,
7333 ";; Procedure %cr-be-in-data-spec motions == %d\n",
7335 fprintf (spec_info
->dump
,
7336 ";; Procedure %cr-begin-control-spec motions == %d\n",
7337 c
, nr_begin_control
);
7338 fprintf (spec_info
->dump
,
7339 ";; Procedure %cr-be-in-control-spec motions == %d\n",
7340 c
, nr_be_in_control
);
7343 scheduled_insns
.release ();
7345 /* Finalize h_i_d, dependency caches, and luids for the whole
7346 function. Target will be finalized in md_global_finish (). */
7347 sched_deps_finish ();
7348 sched_finish_luids ();
7349 current_sched_info
= NULL
;
7353 /* Free global data used during insn scheduling. This function works with
7354 the common data shared between the schedulers. */
7359 haifa_finish_h_i_d ();
7360 free_global_sched_pressure_data ();
7363 if (targetm
.sched
.finish_global
)
7364 targetm
.sched
.finish_global (sched_dump
, sched_verbose
);
7366 end_alias_analysis ();
7368 regstat_free_calls_crossed ();
7373 /* Free all delay_pair structures that were recorded. */
7375 free_delay_pairs (void)
7379 delay_htab
->empty ();
7380 delay_htab_i2
->empty ();
7384 /* Fix INSN_TICKs of the instructions in the current block as well as
7385 INSN_TICKs of their dependents.
7386 HEAD and TAIL are the begin and the end of the current scheduled block. */
7388 fix_inter_tick (rtx_insn
*head
, rtx_insn
*tail
)
7390 /* Set of instructions with corrected INSN_TICK. */
7391 bitmap_head processed
;
7392 /* ??? It is doubtful if we should assume that cycle advance happens on
7393 basic block boundaries. Basically insns that are unconditionally ready
7394 on the start of the block are more preferable then those which have
7395 a one cycle dependency over insn from the previous block. */
7396 int next_clock
= clock_var
+ 1;
7398 bitmap_initialize (&processed
, 0);
7400 /* Iterates over scheduled instructions and fix their INSN_TICKs and
7401 INSN_TICKs of dependent instructions, so that INSN_TICKs are consistent
7402 across different blocks. */
7403 for (tail
= NEXT_INSN (tail
); head
!= tail
; head
= NEXT_INSN (head
))
7408 sd_iterator_def sd_it
;
7411 tick
= INSN_TICK (head
);
7412 gcc_assert (tick
>= MIN_TICK
);
7414 /* Fix INSN_TICK of instruction from just scheduled block. */
7415 if (bitmap_set_bit (&processed
, INSN_LUID (head
)))
7419 if (tick
< MIN_TICK
)
7422 INSN_TICK (head
) = tick
;
7425 if (DEBUG_INSN_P (head
))
7428 FOR_EACH_DEP (head
, SD_LIST_RES_FORW
, sd_it
, dep
)
7432 next
= DEP_CON (dep
);
7433 tick
= INSN_TICK (next
);
7435 if (tick
!= INVALID_TICK
7436 /* If NEXT has its INSN_TICK calculated, fix it.
7437 If not - it will be properly calculated from
7438 scratch later in fix_tick_ready. */
7439 && bitmap_set_bit (&processed
, INSN_LUID (next
)))
7443 if (tick
< MIN_TICK
)
7446 if (tick
> INTER_TICK (next
))
7447 INTER_TICK (next
) = tick
;
7449 tick
= INTER_TICK (next
);
7451 INSN_TICK (next
) = tick
;
7456 bitmap_clear (&processed
);
7459 /* Check if NEXT is ready to be added to the ready or queue list.
7460 If "yes", add it to the proper list.
7462 -1 - is not ready yet,
7463 0 - added to the ready list,
7464 0 < N - queued for N cycles. */
7466 try_ready (rtx_insn
*next
)
7468 ds_t old_ts
, new_ts
;
7470 old_ts
= TODO_SPEC (next
);
7472 gcc_assert (!(old_ts
& ~(SPECULATIVE
| HARD_DEP
| DEP_CONTROL
| DEP_POSTPONED
))
7473 && (old_ts
== HARD_DEP
7474 || old_ts
== DEP_POSTPONED
7475 || (old_ts
& SPECULATIVE
)
7476 || old_ts
== DEP_CONTROL
));
7478 new_ts
= recompute_todo_spec (next
, false);
7480 if (new_ts
& (HARD_DEP
| DEP_POSTPONED
))
7481 gcc_assert (new_ts
== old_ts
7482 && QUEUE_INDEX (next
) == QUEUE_NOWHERE
);
7483 else if (current_sched_info
->new_ready
)
7484 new_ts
= current_sched_info
->new_ready (next
, new_ts
);
7486 /* * if !(old_ts & SPECULATIVE) (e.g. HARD_DEP or 0), then insn might
7487 have its original pattern or changed (speculative) one. This is due
7488 to changing ebb in region scheduling.
7489 * But if (old_ts & SPECULATIVE), then we are pretty sure that insn
7490 has speculative pattern.
7492 We can't assert (!(new_ts & HARD_DEP) || new_ts == old_ts) here because
7493 control-speculative NEXT could have been discarded by sched-rgn.c
7494 (the same case as when discarded by can_schedule_ready_p ()). */
7496 if ((new_ts
& SPECULATIVE
)
7497 /* If (old_ts == new_ts), then (old_ts & SPECULATIVE) and we don't
7498 need to change anything. */
7499 && new_ts
!= old_ts
)
7504 gcc_assert ((new_ts
& SPECULATIVE
) && !(new_ts
& ~SPECULATIVE
));
7506 res
= haifa_speculate_insn (next
, new_ts
, &new_pat
);
7511 /* It would be nice to change DEP_STATUS of all dependences,
7512 which have ((DEP_STATUS & SPECULATIVE) == new_ts) to HARD_DEP,
7513 so we won't reanalyze anything. */
7518 /* We follow the rule, that every speculative insn
7519 has non-null ORIG_PAT. */
7520 if (!ORIG_PAT (next
))
7521 ORIG_PAT (next
) = PATTERN (next
);
7525 if (!ORIG_PAT (next
))
7526 /* If we gonna to overwrite the original pattern of insn,
7528 ORIG_PAT (next
) = PATTERN (next
);
7530 res
= haifa_change_pattern (next
, new_pat
);
7539 /* We need to restore pattern only if (new_ts == 0), because otherwise it is
7540 either correct (new_ts & SPECULATIVE),
7541 or we simply don't care (new_ts & HARD_DEP). */
7543 gcc_assert (!ORIG_PAT (next
)
7544 || !IS_SPECULATION_BRANCHY_CHECK_P (next
));
7546 TODO_SPEC (next
) = new_ts
;
7548 if (new_ts
& (HARD_DEP
| DEP_POSTPONED
))
7550 /* We can't assert (QUEUE_INDEX (next) == QUEUE_NOWHERE) here because
7551 control-speculative NEXT could have been discarded by sched-rgn.c
7552 (the same case as when discarded by can_schedule_ready_p ()). */
7553 /*gcc_assert (QUEUE_INDEX (next) == QUEUE_NOWHERE);*/
7555 change_queue_index (next
, QUEUE_NOWHERE
);
7559 else if (!(new_ts
& BEGIN_SPEC
)
7560 && ORIG_PAT (next
) && PREDICATED_PAT (next
) == NULL_RTX
7561 && !IS_SPECULATION_CHECK_P (next
))
7562 /* We should change pattern of every previously speculative
7563 instruction - and we determine if NEXT was speculative by using
7564 ORIG_PAT field. Except one case - speculation checks have ORIG_PAT
7565 pat too, so skip them. */
7567 bool success
= haifa_change_pattern (next
, ORIG_PAT (next
));
7568 gcc_assert (success
);
7569 ORIG_PAT (next
) = 0;
7572 if (sched_verbose
>= 2)
7574 fprintf (sched_dump
, ";;\t\tdependencies resolved: insn %s",
7575 (*current_sched_info
->print_insn
) (next
, 0));
7577 if (spec_info
&& spec_info
->dump
)
7579 if (new_ts
& BEGIN_DATA
)
7580 fprintf (spec_info
->dump
, "; data-spec;");
7581 if (new_ts
& BEGIN_CONTROL
)
7582 fprintf (spec_info
->dump
, "; control-spec;");
7583 if (new_ts
& BE_IN_CONTROL
)
7584 fprintf (spec_info
->dump
, "; in-control-spec;");
7586 if (TODO_SPEC (next
) & DEP_CONTROL
)
7587 fprintf (sched_dump
, " predicated");
7588 fprintf (sched_dump
, "\n");
7591 adjust_priority (next
);
7593 return fix_tick_ready (next
);
7596 /* Calculate INSN_TICK of NEXT and add it to either ready or queue list. */
7598 fix_tick_ready (rtx_insn
*next
)
7602 if (!DEBUG_INSN_P (next
) && !sd_lists_empty_p (next
, SD_LIST_RES_BACK
))
7605 sd_iterator_def sd_it
;
7608 tick
= INSN_TICK (next
);
7609 /* if tick is not equal to INVALID_TICK, then update
7610 INSN_TICK of NEXT with the most recent resolved dependence
7611 cost. Otherwise, recalculate from scratch. */
7612 full_p
= (tick
== INVALID_TICK
);
7614 FOR_EACH_DEP (next
, SD_LIST_RES_BACK
, sd_it
, dep
)
7616 rtx_insn
*pro
= DEP_PRO (dep
);
7619 gcc_assert (INSN_TICK (pro
) >= MIN_TICK
);
7621 tick1
= INSN_TICK (pro
) + dep_cost (dep
);
7632 INSN_TICK (next
) = tick
;
7634 delay
= tick
- clock_var
;
7635 if (delay
<= 0 || sched_pressure
!= SCHED_PRESSURE_NONE
|| sched_fusion
)
7636 delay
= QUEUE_READY
;
7638 change_queue_index (next
, delay
);
7643 /* Move NEXT to the proper queue list with (DELAY >= 1),
7644 or add it to the ready list (DELAY == QUEUE_READY),
7645 or remove it from ready and queue lists at all (DELAY == QUEUE_NOWHERE). */
7647 change_queue_index (rtx_insn
*next
, int delay
)
7649 int i
= QUEUE_INDEX (next
);
7651 gcc_assert (QUEUE_NOWHERE
<= delay
&& delay
<= max_insn_queue_index
7653 gcc_assert (i
!= QUEUE_SCHEDULED
);
7655 if ((delay
> 0 && NEXT_Q_AFTER (q_ptr
, delay
) == i
)
7656 || (delay
< 0 && delay
== i
))
7657 /* We have nothing to do. */
7660 /* Remove NEXT from wherever it is now. */
7661 if (i
== QUEUE_READY
)
7662 ready_remove_insn (next
);
7664 queue_remove (next
);
7666 /* Add it to the proper place. */
7667 if (delay
== QUEUE_READY
)
7668 ready_add (readyp
, next
, false);
7669 else if (delay
>= 1)
7670 queue_insn (next
, delay
, "change queue index");
7672 if (sched_verbose
>= 2)
7674 fprintf (sched_dump
, ";;\t\ttick updated: insn %s",
7675 (*current_sched_info
->print_insn
) (next
, 0));
7677 if (delay
== QUEUE_READY
)
7678 fprintf (sched_dump
, " into ready\n");
7679 else if (delay
>= 1)
7680 fprintf (sched_dump
, " into queue with cost=%d\n", delay
);
7682 fprintf (sched_dump
, " removed from ready or queue lists\n");
7686 static int sched_ready_n_insns
= -1;
7688 /* Initialize per region data structures. */
7690 sched_extend_ready_list (int new_sched_ready_n_insns
)
7694 if (sched_ready_n_insns
== -1)
7695 /* At the first call we need to initialize one more choice_stack
7699 sched_ready_n_insns
= 0;
7700 scheduled_insns
.reserve (new_sched_ready_n_insns
);
7703 i
= sched_ready_n_insns
+ 1;
7705 ready
.veclen
= new_sched_ready_n_insns
+ issue_rate
;
7706 ready
.vec
= XRESIZEVEC (rtx_insn
*, ready
.vec
, ready
.veclen
);
7708 gcc_assert (new_sched_ready_n_insns
>= sched_ready_n_insns
);
7710 ready_try
= (signed char *) xrecalloc (ready_try
, new_sched_ready_n_insns
,
7711 sched_ready_n_insns
,
7712 sizeof (*ready_try
));
7714 /* We allocate +1 element to save initial state in the choice_stack[0]
7716 choice_stack
= XRESIZEVEC (struct choice_entry
, choice_stack
,
7717 new_sched_ready_n_insns
+ 1);
7719 for (; i
<= new_sched_ready_n_insns
; i
++)
7721 choice_stack
[i
].state
= xmalloc (dfa_state_size
);
7723 if (targetm
.sched
.first_cycle_multipass_init
)
7724 targetm
.sched
.first_cycle_multipass_init (&(choice_stack
[i
]
7728 sched_ready_n_insns
= new_sched_ready_n_insns
;
7731 /* Free per region data structures. */
7733 sched_finish_ready_list (void)
7744 for (i
= 0; i
<= sched_ready_n_insns
; i
++)
7746 if (targetm
.sched
.first_cycle_multipass_fini
)
7747 targetm
.sched
.first_cycle_multipass_fini (&(choice_stack
[i
]
7750 free (choice_stack
[i
].state
);
7752 free (choice_stack
);
7753 choice_stack
= NULL
;
7755 sched_ready_n_insns
= -1;
7759 haifa_luid_for_non_insn (rtx x
)
7761 gcc_assert (NOTE_P (x
) || LABEL_P (x
));
7766 /* Generates recovery code for INSN. */
7768 generate_recovery_code (rtx_insn
*insn
)
7770 if (TODO_SPEC (insn
) & BEGIN_SPEC
)
7771 begin_speculative_block (insn
);
7773 /* Here we have insn with no dependencies to
7774 instructions other then CHECK_SPEC ones. */
7776 if (TODO_SPEC (insn
) & BE_IN_SPEC
)
7777 add_to_speculative_block (insn
);
7781 Tries to add speculative dependencies of type FS between instructions
7782 in deps_list L and TWIN. */
7784 process_insn_forw_deps_be_in_spec (rtx_insn
*insn
, rtx_insn
*twin
, ds_t fs
)
7786 sd_iterator_def sd_it
;
7789 FOR_EACH_DEP (insn
, SD_LIST_FORW
, sd_it
, dep
)
7794 consumer
= DEP_CON (dep
);
7796 ds
= DEP_STATUS (dep
);
7798 if (/* If we want to create speculative dep. */
7800 /* And we can do that because this is a true dep. */
7801 && (ds
& DEP_TYPES
) == DEP_TRUE
)
7803 gcc_assert (!(ds
& BE_IN_SPEC
));
7805 if (/* If this dep can be overcome with 'begin speculation'. */
7807 /* Then we have a choice: keep the dep 'begin speculative'
7808 or transform it into 'be in speculative'. */
7810 if (/* In try_ready we assert that if insn once became ready
7811 it can be removed from the ready (or queue) list only
7812 due to backend decision. Hence we can't let the
7813 probability of the speculative dep to decrease. */
7814 ds_weak (ds
) <= ds_weak (fs
))
7818 new_ds
= (ds
& ~BEGIN_SPEC
) | fs
;
7820 if (/* consumer can 'be in speculative'. */
7821 sched_insn_is_legitimate_for_speculation_p (consumer
,
7823 /* Transform it to be in speculative. */
7828 /* Mark the dep as 'be in speculative'. */
7833 dep_def _new_dep
, *new_dep
= &_new_dep
;
7835 init_dep_1 (new_dep
, twin
, consumer
, DEP_TYPE (dep
), ds
);
7836 sd_add_dep (new_dep
, false);
7841 /* Generates recovery code for BEGIN speculative INSN. */
7843 begin_speculative_block (rtx_insn
*insn
)
7845 if (TODO_SPEC (insn
) & BEGIN_DATA
)
7847 if (TODO_SPEC (insn
) & BEGIN_CONTROL
)
7850 create_check_block_twin (insn
, false);
7852 TODO_SPEC (insn
) &= ~BEGIN_SPEC
;
7855 static void haifa_init_insn (rtx_insn
*);
7857 /* Generates recovery code for BE_IN speculative INSN. */
7859 add_to_speculative_block (rtx_insn
*insn
)
7862 sd_iterator_def sd_it
;
7864 rtx_insn_list
*twins
= NULL
;
7865 rtx_vec_t priorities_roots
;
7867 ts
= TODO_SPEC (insn
);
7868 gcc_assert (!(ts
& ~BE_IN_SPEC
));
7870 if (ts
& BE_IN_DATA
)
7872 if (ts
& BE_IN_CONTROL
)
7875 TODO_SPEC (insn
) &= ~BE_IN_SPEC
;
7876 gcc_assert (!TODO_SPEC (insn
));
7878 DONE_SPEC (insn
) |= ts
;
7880 /* First we convert all simple checks to branchy. */
7881 for (sd_it
= sd_iterator_start (insn
, SD_LIST_SPEC_BACK
);
7882 sd_iterator_cond (&sd_it
, &dep
);)
7884 rtx_insn
*check
= DEP_PRO (dep
);
7886 if (IS_SPECULATION_SIMPLE_CHECK_P (check
))
7888 create_check_block_twin (check
, true);
7890 /* Restart search. */
7891 sd_it
= sd_iterator_start (insn
, SD_LIST_SPEC_BACK
);
7894 /* Continue search. */
7895 sd_iterator_next (&sd_it
);
7898 priorities_roots
.create (0);
7899 clear_priorities (insn
, &priorities_roots
);
7903 rtx_insn
*check
, *twin
;
7906 /* Get the first backward dependency of INSN. */
7907 sd_it
= sd_iterator_start (insn
, SD_LIST_SPEC_BACK
);
7908 if (!sd_iterator_cond (&sd_it
, &dep
))
7909 /* INSN has no backward dependencies left. */
7912 gcc_assert ((DEP_STATUS (dep
) & BEGIN_SPEC
) == 0
7913 && (DEP_STATUS (dep
) & BE_IN_SPEC
) != 0
7914 && (DEP_STATUS (dep
) & DEP_TYPES
) == DEP_TRUE
);
7916 check
= DEP_PRO (dep
);
7918 gcc_assert (!IS_SPECULATION_CHECK_P (check
) && !ORIG_PAT (check
)
7919 && QUEUE_INDEX (check
) == QUEUE_NOWHERE
);
7921 rec
= BLOCK_FOR_INSN (check
);
7923 twin
= emit_insn_before (copy_insn (PATTERN (insn
)), BB_END (rec
));
7924 haifa_init_insn (twin
);
7926 sd_copy_back_deps (twin
, insn
, true);
7928 if (sched_verbose
&& spec_info
->dump
)
7929 /* INSN_BB (insn) isn't determined for twin insns yet.
7930 So we can't use current_sched_info->print_insn. */
7931 fprintf (spec_info
->dump
, ";;\t\tGenerated twin insn : %d/rec%d\n",
7932 INSN_UID (twin
), rec
->index
);
7934 twins
= alloc_INSN_LIST (twin
, twins
);
7936 /* Add dependences between TWIN and all appropriate
7937 instructions from REC. */
7938 FOR_EACH_DEP (insn
, SD_LIST_SPEC_BACK
, sd_it
, dep
)
7940 rtx_insn
*pro
= DEP_PRO (dep
);
7942 gcc_assert (DEP_TYPE (dep
) == REG_DEP_TRUE
);
7944 /* INSN might have dependencies from the instructions from
7945 several recovery blocks. At this iteration we process those
7946 producers that reside in REC. */
7947 if (BLOCK_FOR_INSN (pro
) == rec
)
7949 dep_def _new_dep
, *new_dep
= &_new_dep
;
7951 init_dep (new_dep
, pro
, twin
, REG_DEP_TRUE
);
7952 sd_add_dep (new_dep
, false);
7956 process_insn_forw_deps_be_in_spec (insn
, twin
, ts
);
7958 /* Remove all dependencies between INSN and insns in REC. */
7959 for (sd_it
= sd_iterator_start (insn
, SD_LIST_SPEC_BACK
);
7960 sd_iterator_cond (&sd_it
, &dep
);)
7962 rtx_insn
*pro
= DEP_PRO (dep
);
7964 if (BLOCK_FOR_INSN (pro
) == rec
)
7965 sd_delete_dep (sd_it
);
7967 sd_iterator_next (&sd_it
);
7971 /* We couldn't have added the dependencies between INSN and TWINS earlier
7972 because that would make TWINS appear in the INSN_BACK_DEPS (INSN). */
7976 rtx_insn_list
*next_node
;
7978 twin
= twins
->insn ();
7981 dep_def _new_dep
, *new_dep
= &_new_dep
;
7983 init_dep (new_dep
, insn
, twin
, REG_DEP_OUTPUT
);
7984 sd_add_dep (new_dep
, false);
7987 next_node
= twins
->next ();
7988 free_INSN_LIST_node (twins
);
7992 calc_priorities (priorities_roots
);
7993 priorities_roots
.release ();
7996 /* Extends and fills with zeros (only the new part) array pointed to by P. */
7998 xrecalloc (void *p
, size_t new_nmemb
, size_t old_nmemb
, size_t size
)
8000 gcc_assert (new_nmemb
>= old_nmemb
);
8001 p
= XRESIZEVAR (void, p
, new_nmemb
* size
);
8002 memset (((char *) p
) + old_nmemb
* size
, 0, (new_nmemb
- old_nmemb
) * size
);
8007 Find fallthru edge from PRED. */
8009 find_fallthru_edge_from (basic_block pred
)
8014 succ
= pred
->next_bb
;
8015 gcc_assert (succ
->prev_bb
== pred
);
8017 if (EDGE_COUNT (pred
->succs
) <= EDGE_COUNT (succ
->preds
))
8019 e
= find_fallthru_edge (pred
->succs
);
8023 gcc_assert (e
->dest
== succ
);
8029 e
= find_fallthru_edge (succ
->preds
);
8033 gcc_assert (e
->src
== pred
);
8041 /* Extend per basic block data structures. */
8043 sched_extend_bb (void)
8045 /* The following is done to keep current_sched_info->next_tail non null. */
8046 rtx_insn
*end
= BB_END (EXIT_BLOCK_PTR_FOR_FN (cfun
)->prev_bb
);
8047 rtx_insn
*insn
= DEBUG_INSN_P (end
) ? prev_nondebug_insn (end
) : end
;
8048 if (NEXT_INSN (end
) == 0
8051 /* Don't emit a NOTE if it would end up before a BARRIER. */
8052 && !BARRIER_P (NEXT_INSN (end
))))
8054 rtx_note
*note
= emit_note_after (NOTE_INSN_DELETED
, end
);
8055 /* Make note appear outside BB. */
8056 set_block_for_insn (note
, NULL
);
8057 BB_END (EXIT_BLOCK_PTR_FOR_FN (cfun
)->prev_bb
) = end
;
8061 /* Init per basic block data structures. */
8063 sched_init_bbs (void)
8068 /* Initialize BEFORE_RECOVERY variable. */
8070 init_before_recovery (basic_block
*before_recovery_ptr
)
8075 last
= EXIT_BLOCK_PTR_FOR_FN (cfun
)->prev_bb
;
8076 e
= find_fallthru_edge_from (last
);
8080 /* We create two basic blocks:
8081 1. Single instruction block is inserted right after E->SRC
8083 2. Empty block right before EXIT_BLOCK.
8084 Between these two blocks recovery blocks will be emitted. */
8086 basic_block single
, empty
;
8088 /* If the fallthrough edge to exit we've found is from the block we've
8089 created before, don't do anything more. */
8090 if (last
== after_recovery
)
8093 adding_bb_to_current_region_p
= false;
8095 single
= sched_create_empty_bb (last
);
8096 empty
= sched_create_empty_bb (single
);
8098 /* Add new blocks to the root loop. */
8099 if (current_loops
!= NULL
)
8101 add_bb_to_loop (single
, (*current_loops
->larray
)[0]);
8102 add_bb_to_loop (empty
, (*current_loops
->larray
)[0]);
8105 single
->count
= last
->count
;
8106 empty
->count
= last
->count
;
8107 single
->frequency
= last
->frequency
;
8108 empty
->frequency
= last
->frequency
;
8109 BB_COPY_PARTITION (single
, last
);
8110 BB_COPY_PARTITION (empty
, last
);
8112 redirect_edge_succ (e
, single
);
8113 make_single_succ_edge (single
, empty
, 0);
8114 make_single_succ_edge (empty
, EXIT_BLOCK_PTR_FOR_FN (cfun
),
8117 rtx_code_label
*label
= block_label (empty
);
8118 rtx_jump_insn
*x
= emit_jump_insn_after (targetm
.gen_jump (label
),
8120 JUMP_LABEL (x
) = label
;
8121 LABEL_NUSES (label
)++;
8122 haifa_init_insn (x
);
8124 emit_barrier_after (x
);
8126 sched_init_only_bb (empty
, NULL
);
8127 sched_init_only_bb (single
, NULL
);
8130 adding_bb_to_current_region_p
= true;
8131 before_recovery
= single
;
8132 after_recovery
= empty
;
8134 if (before_recovery_ptr
)
8135 *before_recovery_ptr
= before_recovery
;
8137 if (sched_verbose
>= 2 && spec_info
->dump
)
8138 fprintf (spec_info
->dump
,
8139 ";;\t\tFixed fallthru to EXIT : %d->>%d->%d->>EXIT\n",
8140 last
->index
, single
->index
, empty
->index
);
8143 before_recovery
= last
;
8146 /* Returns new recovery block. */
8148 sched_create_recovery_block (basic_block
*before_recovery_ptr
)
8153 haifa_recovery_bb_recently_added_p
= true;
8154 haifa_recovery_bb_ever_added_p
= true;
8156 init_before_recovery (before_recovery_ptr
);
8158 barrier
= get_last_bb_insn (before_recovery
);
8159 gcc_assert (BARRIER_P (barrier
));
8161 rtx_insn
*label
= emit_label_after (gen_label_rtx (), barrier
);
8163 rec
= create_basic_block (label
, label
, before_recovery
);
8165 /* A recovery block always ends with an unconditional jump. */
8166 emit_barrier_after (BB_END (rec
));
8168 if (BB_PARTITION (before_recovery
) != BB_UNPARTITIONED
)
8169 BB_SET_PARTITION (rec
, BB_COLD_PARTITION
);
8171 if (sched_verbose
&& spec_info
->dump
)
8172 fprintf (spec_info
->dump
, ";;\t\tGenerated recovery block rec%d\n",
8178 /* Create edges: FIRST_BB -> REC; FIRST_BB -> SECOND_BB; REC -> SECOND_BB
8179 and emit necessary jumps. */
8181 sched_create_recovery_edges (basic_block first_bb
, basic_block rec
,
8182 basic_block second_bb
)
8186 /* This is fixing of incoming edge. */
8187 /* ??? Which other flags should be specified? */
8188 if (BB_PARTITION (first_bb
) != BB_PARTITION (rec
))
8189 /* Partition type is the same, if it is "unpartitioned". */
8190 edge_flags
= EDGE_CROSSING
;
8194 make_edge (first_bb
, rec
, edge_flags
);
8195 rtx_code_label
*label
= block_label (second_bb
);
8196 rtx_jump_insn
*jump
= emit_jump_insn_after (targetm
.gen_jump (label
),
8198 JUMP_LABEL (jump
) = label
;
8199 LABEL_NUSES (label
)++;
8201 if (BB_PARTITION (second_bb
) != BB_PARTITION (rec
))
8202 /* Partition type is the same, if it is "unpartitioned". */
8204 /* Rewritten from cfgrtl.c. */
8205 if (flag_reorder_blocks_and_partition
8206 && targetm_common
.have_named_sections
)
8208 /* We don't need the same note for the check because
8209 any_condjump_p (check) == true. */
8210 CROSSING_JUMP_P (jump
) = 1;
8212 edge_flags
= EDGE_CROSSING
;
8217 make_single_succ_edge (rec
, second_bb
, edge_flags
);
8218 if (dom_info_available_p (CDI_DOMINATORS
))
8219 set_immediate_dominator (CDI_DOMINATORS
, rec
, first_bb
);
8222 /* This function creates recovery code for INSN. If MUTATE_P is nonzero,
8223 INSN is a simple check, that should be converted to branchy one. */
8225 create_check_block_twin (rtx_insn
*insn
, bool mutate_p
)
8228 rtx_insn
*label
, *check
, *twin
;
8231 sd_iterator_def sd_it
;
8233 dep_def _new_dep
, *new_dep
= &_new_dep
;
8236 gcc_assert (ORIG_PAT (insn
) != NULL_RTX
);
8239 todo_spec
= TODO_SPEC (insn
);
8242 gcc_assert (IS_SPECULATION_SIMPLE_CHECK_P (insn
)
8243 && (TODO_SPEC (insn
) & SPECULATIVE
) == 0);
8245 todo_spec
= CHECK_SPEC (insn
);
8248 todo_spec
&= SPECULATIVE
;
8250 /* Create recovery block. */
8251 if (mutate_p
|| targetm
.sched
.needs_block_p (todo_spec
))
8253 rec
= sched_create_recovery_block (NULL
);
8254 label
= BB_HEAD (rec
);
8258 rec
= EXIT_BLOCK_PTR_FOR_FN (cfun
);
8263 check_pat
= targetm
.sched
.gen_spec_check (insn
, label
, todo_spec
);
8265 if (rec
!= EXIT_BLOCK_PTR_FOR_FN (cfun
))
8267 /* To have mem_reg alive at the beginning of second_bb,
8268 we emit check BEFORE insn, so insn after splitting
8269 insn will be at the beginning of second_bb, which will
8270 provide us with the correct life information. */
8271 check
= emit_jump_insn_before (check_pat
, insn
);
8272 JUMP_LABEL (check
) = label
;
8273 LABEL_NUSES (label
)++;
8276 check
= emit_insn_before (check_pat
, insn
);
8278 /* Extend data structures. */
8279 haifa_init_insn (check
);
8281 /* CHECK is being added to current region. Extend ready list. */
8282 gcc_assert (sched_ready_n_insns
!= -1);
8283 sched_extend_ready_list (sched_ready_n_insns
+ 1);
8285 if (current_sched_info
->add_remove_insn
)
8286 current_sched_info
->add_remove_insn (insn
, 0);
8288 RECOVERY_BLOCK (check
) = rec
;
8290 if (sched_verbose
&& spec_info
->dump
)
8291 fprintf (spec_info
->dump
, ";;\t\tGenerated check insn : %s\n",
8292 (*current_sched_info
->print_insn
) (check
, 0));
8294 gcc_assert (ORIG_PAT (insn
));
8296 /* Initialize TWIN (twin is a duplicate of original instruction
8297 in the recovery block). */
8298 if (rec
!= EXIT_BLOCK_PTR_FOR_FN (cfun
))
8300 sd_iterator_def sd_it
;
8303 FOR_EACH_DEP (insn
, SD_LIST_RES_BACK
, sd_it
, dep
)
8304 if ((DEP_STATUS (dep
) & DEP_OUTPUT
) != 0)
8306 struct _dep _dep2
, *dep2
= &_dep2
;
8308 init_dep (dep2
, DEP_PRO (dep
), check
, REG_DEP_TRUE
);
8310 sd_add_dep (dep2
, true);
8313 twin
= emit_insn_after (ORIG_PAT (insn
), BB_END (rec
));
8314 haifa_init_insn (twin
);
8316 if (sched_verbose
&& spec_info
->dump
)
8317 /* INSN_BB (insn) isn't determined for twin insns yet.
8318 So we can't use current_sched_info->print_insn. */
8319 fprintf (spec_info
->dump
, ";;\t\tGenerated twin insn : %d/rec%d\n",
8320 INSN_UID (twin
), rec
->index
);
8324 ORIG_PAT (check
) = ORIG_PAT (insn
);
8325 HAS_INTERNAL_DEP (check
) = 1;
8327 /* ??? We probably should change all OUTPUT dependencies to
8331 /* Copy all resolved back dependencies of INSN to TWIN. This will
8332 provide correct value for INSN_TICK (TWIN). */
8333 sd_copy_back_deps (twin
, insn
, true);
8335 if (rec
!= EXIT_BLOCK_PTR_FOR_FN (cfun
))
8336 /* In case of branchy check, fix CFG. */
8338 basic_block first_bb
, second_bb
;
8341 first_bb
= BLOCK_FOR_INSN (check
);
8342 second_bb
= sched_split_block (first_bb
, check
);
8344 sched_create_recovery_edges (first_bb
, rec
, second_bb
);
8346 sched_init_only_bb (second_bb
, first_bb
);
8347 sched_init_only_bb (rec
, EXIT_BLOCK_PTR_FOR_FN (cfun
));
8349 jump
= BB_END (rec
);
8350 haifa_init_insn (jump
);
8353 /* Move backward dependences from INSN to CHECK and
8354 move forward dependences from INSN to TWIN. */
8356 /* First, create dependencies between INSN's producers and CHECK & TWIN. */
8357 FOR_EACH_DEP (insn
, SD_LIST_BACK
, sd_it
, dep
)
8359 rtx_insn
*pro
= DEP_PRO (dep
);
8362 /* If BEGIN_DATA: [insn ~~TRUE~~> producer]:
8363 check --TRUE--> producer ??? or ANTI ???
8364 twin --TRUE--> producer
8365 twin --ANTI--> check
8367 If BEGIN_CONTROL: [insn ~~ANTI~~> producer]:
8368 check --ANTI--> producer
8369 twin --ANTI--> producer
8370 twin --ANTI--> check
8372 If BE_IN_SPEC: [insn ~~TRUE~~> producer]:
8373 check ~~TRUE~~> producer
8374 twin ~~TRUE~~> producer
8375 twin --ANTI--> check */
8377 ds
= DEP_STATUS (dep
);
8379 if (ds
& BEGIN_SPEC
)
8381 gcc_assert (!mutate_p
);
8385 init_dep_1 (new_dep
, pro
, check
, DEP_TYPE (dep
), ds
);
8386 sd_add_dep (new_dep
, false);
8388 if (rec
!= EXIT_BLOCK_PTR_FOR_FN (cfun
))
8390 DEP_CON (new_dep
) = twin
;
8391 sd_add_dep (new_dep
, false);
8395 /* Second, remove backward dependencies of INSN. */
8396 for (sd_it
= sd_iterator_start (insn
, SD_LIST_SPEC_BACK
);
8397 sd_iterator_cond (&sd_it
, &dep
);)
8399 if ((DEP_STATUS (dep
) & BEGIN_SPEC
)
8401 /* We can delete this dep because we overcome it with
8402 BEGIN_SPECULATION. */
8403 sd_delete_dep (sd_it
);
8405 sd_iterator_next (&sd_it
);
8408 /* Future Speculations. Determine what BE_IN speculations will be like. */
8411 /* Fields (DONE_SPEC (x) & BEGIN_SPEC) and CHECK_SPEC (x) are set only
8414 gcc_assert (!DONE_SPEC (insn
));
8418 ds_t ts
= TODO_SPEC (insn
);
8420 DONE_SPEC (insn
) = ts
& BEGIN_SPEC
;
8421 CHECK_SPEC (check
) = ts
& BEGIN_SPEC
;
8423 /* Luckiness of future speculations solely depends upon initial
8424 BEGIN speculation. */
8425 if (ts
& BEGIN_DATA
)
8426 fs
= set_dep_weak (fs
, BE_IN_DATA
, get_dep_weak (ts
, BEGIN_DATA
));
8427 if (ts
& BEGIN_CONTROL
)
8428 fs
= set_dep_weak (fs
, BE_IN_CONTROL
,
8429 get_dep_weak (ts
, BEGIN_CONTROL
));
8432 CHECK_SPEC (check
) = CHECK_SPEC (insn
);
8434 /* Future speculations: call the helper. */
8435 process_insn_forw_deps_be_in_spec (insn
, twin
, fs
);
8437 if (rec
!= EXIT_BLOCK_PTR_FOR_FN (cfun
))
8439 /* Which types of dependencies should we use here is,
8440 generally, machine-dependent question... But, for now,
8445 init_dep (new_dep
, insn
, check
, REG_DEP_TRUE
);
8446 sd_add_dep (new_dep
, false);
8448 init_dep (new_dep
, insn
, twin
, REG_DEP_OUTPUT
);
8449 sd_add_dep (new_dep
, false);
8453 if (spec_info
->dump
)
8454 fprintf (spec_info
->dump
, ";;\t\tRemoved simple check : %s\n",
8455 (*current_sched_info
->print_insn
) (insn
, 0));
8457 /* Remove all dependencies of the INSN. */
8459 sd_it
= sd_iterator_start (insn
, (SD_LIST_FORW
8461 | SD_LIST_RES_BACK
));
8462 while (sd_iterator_cond (&sd_it
, &dep
))
8463 sd_delete_dep (sd_it
);
8466 /* If former check (INSN) already was moved to the ready (or queue)
8467 list, add new check (CHECK) there too. */
8468 if (QUEUE_INDEX (insn
) != QUEUE_NOWHERE
)
8471 /* Remove old check from instruction stream and free its
8473 sched_remove_insn (insn
);
8476 init_dep (new_dep
, check
, twin
, REG_DEP_ANTI
);
8477 sd_add_dep (new_dep
, false);
8481 init_dep_1 (new_dep
, insn
, check
, REG_DEP_TRUE
, DEP_TRUE
| DEP_OUTPUT
);
8482 sd_add_dep (new_dep
, false);
8486 /* Fix priorities. If MUTATE_P is nonzero, this is not necessary,
8487 because it'll be done later in add_to_speculative_block. */
8489 rtx_vec_t priorities_roots
= rtx_vec_t ();
8491 clear_priorities (twin
, &priorities_roots
);
8492 calc_priorities (priorities_roots
);
8493 priorities_roots
.release ();
8497 /* Removes dependency between instructions in the recovery block REC
8498 and usual region instructions. It keeps inner dependences so it
8499 won't be necessary to recompute them. */
8501 fix_recovery_deps (basic_block rec
)
8503 rtx_insn
*note
, *insn
, *jump
;
8504 rtx_insn_list
*ready_list
= 0;
8505 bitmap_head in_ready
;
8506 rtx_insn_list
*link
;
8508 bitmap_initialize (&in_ready
, 0);
8510 /* NOTE - a basic block note. */
8511 note
= NEXT_INSN (BB_HEAD (rec
));
8512 gcc_assert (NOTE_INSN_BASIC_BLOCK_P (note
));
8513 insn
= BB_END (rec
);
8514 gcc_assert (JUMP_P (insn
));
8515 insn
= PREV_INSN (insn
);
8519 sd_iterator_def sd_it
;
8522 for (sd_it
= sd_iterator_start (insn
, SD_LIST_FORW
);
8523 sd_iterator_cond (&sd_it
, &dep
);)
8525 rtx_insn
*consumer
= DEP_CON (dep
);
8527 if (BLOCK_FOR_INSN (consumer
) != rec
)
8529 sd_delete_dep (sd_it
);
8531 if (bitmap_set_bit (&in_ready
, INSN_LUID (consumer
)))
8532 ready_list
= alloc_INSN_LIST (consumer
, ready_list
);
8536 gcc_assert ((DEP_STATUS (dep
) & DEP_TYPES
) == DEP_TRUE
);
8538 sd_iterator_next (&sd_it
);
8542 insn
= PREV_INSN (insn
);
8544 while (insn
!= note
);
8546 bitmap_clear (&in_ready
);
8548 /* Try to add instructions to the ready or queue list. */
8549 for (link
= ready_list
; link
; link
= link
->next ())
8550 try_ready (link
->insn ());
8551 free_INSN_LIST_list (&ready_list
);
8553 /* Fixing jump's dependences. */
8554 insn
= BB_HEAD (rec
);
8555 jump
= BB_END (rec
);
8557 gcc_assert (LABEL_P (insn
));
8558 insn
= NEXT_INSN (insn
);
8560 gcc_assert (NOTE_INSN_BASIC_BLOCK_P (insn
));
8561 add_jump_dependencies (insn
, jump
);
8564 /* Change pattern of INSN to NEW_PAT. Invalidate cached haifa
8565 instruction data. */
8567 haifa_change_pattern (rtx_insn
*insn
, rtx new_pat
)
8571 t
= validate_change (insn
, &PATTERN (insn
), new_pat
, 0);
8575 update_insn_after_change (insn
);
8579 /* -1 - can't speculate,
8580 0 - for speculation with REQUEST mode it is OK to use
8581 current instruction pattern,
8582 1 - need to change pattern for *NEW_PAT to be speculative. */
8584 sched_speculate_insn (rtx_insn
*insn
, ds_t request
, rtx
*new_pat
)
8586 gcc_assert (current_sched_info
->flags
& DO_SPECULATION
8587 && (request
& SPECULATIVE
)
8588 && sched_insn_is_legitimate_for_speculation_p (insn
, request
));
8590 if ((request
& spec_info
->mask
) != request
)
8593 if (request
& BE_IN_SPEC
8594 && !(request
& BEGIN_SPEC
))
8597 return targetm
.sched
.speculate_insn (insn
, request
, new_pat
);
8601 haifa_speculate_insn (rtx_insn
*insn
, ds_t request
, rtx
*new_pat
)
8603 gcc_assert (sched_deps_info
->generate_spec_deps
8604 && !IS_SPECULATION_CHECK_P (insn
));
8606 if (HAS_INTERNAL_DEP (insn
)
8607 || SCHED_GROUP_P (insn
))
8610 return sched_speculate_insn (insn
, request
, new_pat
);
8613 /* Print some information about block BB, which starts with HEAD and
8614 ends with TAIL, before scheduling it.
8615 I is zero, if scheduler is about to start with the fresh ebb. */
8617 dump_new_block_header (int i
, basic_block bb
, rtx_insn
*head
, rtx_insn
*tail
)
8620 fprintf (sched_dump
,
8621 ";; ======================================================\n");
8623 fprintf (sched_dump
,
8624 ";; =====================ADVANCING TO=====================\n");
8625 fprintf (sched_dump
,
8626 ";; -- basic block %d from %d to %d -- %s reload\n",
8627 bb
->index
, INSN_UID (head
), INSN_UID (tail
),
8628 (reload_completed
? "after" : "before"));
8629 fprintf (sched_dump
,
8630 ";; ======================================================\n");
8631 fprintf (sched_dump
, "\n");
8634 /* Unlink basic block notes and labels and saves them, so they
8635 can be easily restored. We unlink basic block notes in EBB to
8636 provide back-compatibility with the previous code, as target backends
8637 assume, that there'll be only instructions between
8638 current_sched_info->{head and tail}. We restore these notes as soon
8640 FIRST (LAST) is the first (last) basic block in the ebb.
8641 NB: In usual case (FIRST == LAST) nothing is really done. */
8643 unlink_bb_notes (basic_block first
, basic_block last
)
8645 /* We DON'T unlink basic block notes of the first block in the ebb. */
8649 bb_header
= XNEWVEC (rtx_insn
*, last_basic_block_for_fn (cfun
));
8651 /* Make a sentinel. */
8652 if (last
->next_bb
!= EXIT_BLOCK_PTR_FOR_FN (cfun
))
8653 bb_header
[last
->next_bb
->index
] = 0;
8655 first
= first
->next_bb
;
8658 rtx_insn
*prev
, *label
, *note
, *next
;
8660 label
= BB_HEAD (last
);
8661 if (LABEL_P (label
))
8662 note
= NEXT_INSN (label
);
8665 gcc_assert (NOTE_INSN_BASIC_BLOCK_P (note
));
8667 prev
= PREV_INSN (label
);
8668 next
= NEXT_INSN (note
);
8669 gcc_assert (prev
&& next
);
8671 SET_NEXT_INSN (prev
) = next
;
8672 SET_PREV_INSN (next
) = prev
;
8674 bb_header
[last
->index
] = label
;
8679 last
= last
->prev_bb
;
8684 /* Restore basic block notes.
8685 FIRST is the first basic block in the ebb. */
8687 restore_bb_notes (basic_block first
)
8692 /* We DON'T unlink basic block notes of the first block in the ebb. */
8693 first
= first
->next_bb
;
8694 /* Remember: FIRST is actually a second basic block in the ebb. */
8696 while (first
!= EXIT_BLOCK_PTR_FOR_FN (cfun
)
8697 && bb_header
[first
->index
])
8699 rtx_insn
*prev
, *label
, *note
, *next
;
8701 label
= bb_header
[first
->index
];
8702 prev
= PREV_INSN (label
);
8703 next
= NEXT_INSN (prev
);
8705 if (LABEL_P (label
))
8706 note
= NEXT_INSN (label
);
8709 gcc_assert (NOTE_INSN_BASIC_BLOCK_P (note
));
8711 bb_header
[first
->index
] = 0;
8713 SET_NEXT_INSN (prev
) = label
;
8714 SET_NEXT_INSN (note
) = next
;
8715 SET_PREV_INSN (next
) = note
;
8717 first
= first
->next_bb
;
8725 Fix CFG after both in- and inter-block movement of
8726 control_flow_insn_p JUMP. */
8728 fix_jump_move (rtx_insn
*jump
)
8730 basic_block bb
, jump_bb
, jump_bb_next
;
8732 bb
= BLOCK_FOR_INSN (PREV_INSN (jump
));
8733 jump_bb
= BLOCK_FOR_INSN (jump
);
8734 jump_bb_next
= jump_bb
->next_bb
;
8736 gcc_assert (common_sched_info
->sched_pass_id
== SCHED_EBB_PASS
8737 || IS_SPECULATION_BRANCHY_CHECK_P (jump
));
8739 if (!NOTE_INSN_BASIC_BLOCK_P (BB_END (jump_bb_next
)))
8740 /* if jump_bb_next is not empty. */
8741 BB_END (jump_bb
) = BB_END (jump_bb_next
);
8743 if (BB_END (bb
) != PREV_INSN (jump
))
8744 /* Then there are instruction after jump that should be placed
8746 BB_END (jump_bb_next
) = BB_END (bb
);
8748 /* Otherwise jump_bb_next is empty. */
8749 BB_END (jump_bb_next
) = NEXT_INSN (BB_HEAD (jump_bb_next
));
8751 /* To make assertion in move_insn happy. */
8752 BB_END (bb
) = PREV_INSN (jump
);
8754 update_bb_for_insn (jump_bb_next
);
8757 /* Fix CFG after interblock movement of control_flow_insn_p JUMP. */
8759 move_block_after_check (rtx_insn
*jump
)
8761 basic_block bb
, jump_bb
, jump_bb_next
;
8762 vec
<edge
, va_gc
> *t
;
8764 bb
= BLOCK_FOR_INSN (PREV_INSN (jump
));
8765 jump_bb
= BLOCK_FOR_INSN (jump
);
8766 jump_bb_next
= jump_bb
->next_bb
;
8768 update_bb_for_insn (jump_bb
);
8770 gcc_assert (IS_SPECULATION_CHECK_P (jump
)
8771 || IS_SPECULATION_CHECK_P (BB_END (jump_bb_next
)));
8773 unlink_block (jump_bb_next
);
8774 link_block (jump_bb_next
, bb
);
8778 move_succs (&(jump_bb
->succs
), bb
);
8779 move_succs (&(jump_bb_next
->succs
), jump_bb
);
8780 move_succs (&t
, jump_bb_next
);
8782 df_mark_solutions_dirty ();
8784 common_sched_info
->fix_recovery_cfg
8785 (bb
->index
, jump_bb
->index
, jump_bb_next
->index
);
8788 /* Helper function for move_block_after_check.
8789 This functions attaches edge vector pointed to by SUCCSP to
8792 move_succs (vec
<edge
, va_gc
> **succsp
, basic_block to
)
8797 gcc_assert (to
->succs
== 0);
8799 to
->succs
= *succsp
;
8801 FOR_EACH_EDGE (e
, ei
, to
->succs
)
8807 /* Remove INSN from the instruction stream.
8808 INSN should have any dependencies. */
8810 sched_remove_insn (rtx_insn
*insn
)
8812 sd_finish_insn (insn
);
8814 change_queue_index (insn
, QUEUE_NOWHERE
);
8815 current_sched_info
->add_remove_insn (insn
, 1);
8819 /* Clear priorities of all instructions, that are forward dependent on INSN.
8820 Store in vector pointed to by ROOTS_PTR insns on which priority () should
8821 be invoked to initialize all cleared priorities. */
8823 clear_priorities (rtx_insn
*insn
, rtx_vec_t
*roots_ptr
)
8825 sd_iterator_def sd_it
;
8827 bool insn_is_root_p
= true;
8829 gcc_assert (QUEUE_INDEX (insn
) != QUEUE_SCHEDULED
);
8831 FOR_EACH_DEP (insn
, SD_LIST_BACK
, sd_it
, dep
)
8833 rtx_insn
*pro
= DEP_PRO (dep
);
8835 if (INSN_PRIORITY_STATUS (pro
) >= 0
8836 && QUEUE_INDEX (insn
) != QUEUE_SCHEDULED
)
8838 /* If DEP doesn't contribute to priority then INSN itself should
8839 be added to priority roots. */
8840 if (contributes_to_priority_p (dep
))
8841 insn_is_root_p
= false;
8843 INSN_PRIORITY_STATUS (pro
) = -1;
8844 clear_priorities (pro
, roots_ptr
);
8849 roots_ptr
->safe_push (insn
);
8852 /* Recompute priorities of instructions, whose priorities might have been
8853 changed. ROOTS is a vector of instructions whose priority computation will
8854 trigger initialization of all cleared priorities. */
8856 calc_priorities (rtx_vec_t roots
)
8861 FOR_EACH_VEC_ELT (roots
, i
, insn
)
8866 /* Add dependences between JUMP and other instructions in the recovery
8867 block. INSN is the first insn the recovery block. */
8869 add_jump_dependencies (rtx_insn
*insn
, rtx_insn
*jump
)
8873 insn
= NEXT_INSN (insn
);
8877 if (dep_list_size (insn
, SD_LIST_FORW
) == 0)
8879 dep_def _new_dep
, *new_dep
= &_new_dep
;
8881 init_dep (new_dep
, insn
, jump
, REG_DEP_ANTI
);
8882 sd_add_dep (new_dep
, false);
8887 gcc_assert (!sd_lists_empty_p (jump
, SD_LIST_BACK
));
8890 /* Extend data structures for logical insn UID. */
8892 sched_extend_luids (void)
8894 int new_luids_max_uid
= get_max_uid () + 1;
8896 sched_luids
.safe_grow_cleared (new_luids_max_uid
);
8899 /* Initialize LUID for INSN. */
8901 sched_init_insn_luid (rtx_insn
*insn
)
8903 int i
= INSN_P (insn
) ? 1 : common_sched_info
->luid_for_non_insn (insn
);
8908 luid
= sched_max_luid
;
8909 sched_max_luid
+= i
;
8914 SET_INSN_LUID (insn
, luid
);
8917 /* Initialize luids for BBS.
8918 The hook common_sched_info->luid_for_non_insn () is used to determine
8919 if notes, labels, etc. need luids. */
8921 sched_init_luids (bb_vec_t bbs
)
8926 sched_extend_luids ();
8927 FOR_EACH_VEC_ELT (bbs
, i
, bb
)
8931 FOR_BB_INSNS (bb
, insn
)
8932 sched_init_insn_luid (insn
);
8938 sched_finish_luids (void)
8940 sched_luids
.release ();
8944 /* Return logical uid of INSN. Helpful while debugging. */
8946 insn_luid (rtx_insn
*insn
)
8948 return INSN_LUID (insn
);
8951 /* Extend per insn data in the target. */
8953 sched_extend_target (void)
8955 if (targetm
.sched
.h_i_d_extended
)
8956 targetm
.sched
.h_i_d_extended ();
8959 /* Extend global scheduler structures (those, that live across calls to
8960 schedule_block) to include information about just emitted INSN. */
8964 int reserve
= (get_max_uid () + 1 - h_i_d
.length ());
8966 && ! h_i_d
.space (reserve
))
8968 h_i_d
.safe_grow_cleared (3 * get_max_uid () / 2);
8969 sched_extend_target ();
8973 /* Initialize h_i_d entry of the INSN with default values.
8974 Values, that are not explicitly initialized here, hold zero. */
8976 init_h_i_d (rtx_insn
*insn
)
8978 if (INSN_LUID (insn
) > 0)
8980 INSN_COST (insn
) = -1;
8981 QUEUE_INDEX (insn
) = QUEUE_NOWHERE
;
8982 INSN_TICK (insn
) = INVALID_TICK
;
8983 INSN_EXACT_TICK (insn
) = INVALID_TICK
;
8984 INTER_TICK (insn
) = INVALID_TICK
;
8985 TODO_SPEC (insn
) = HARD_DEP
;
8986 INSN_AUTOPREF_MULTIPASS_DATA (insn
)[0].status
8987 = AUTOPREF_MULTIPASS_DATA_UNINITIALIZED
;
8988 INSN_AUTOPREF_MULTIPASS_DATA (insn
)[1].status
8989 = AUTOPREF_MULTIPASS_DATA_UNINITIALIZED
;
8993 /* Initialize haifa_insn_data for BBS. */
8995 haifa_init_h_i_d (bb_vec_t bbs
)
9001 FOR_EACH_VEC_ELT (bbs
, i
, bb
)
9005 FOR_BB_INSNS (bb
, insn
)
9010 /* Finalize haifa_insn_data. */
9012 haifa_finish_h_i_d (void)
9015 haifa_insn_data_t data
;
9016 struct reg_use_data
*use
, *next
;
9018 FOR_EACH_VEC_ELT (h_i_d
, i
, data
)
9020 free (data
->max_reg_pressure
);
9021 free (data
->reg_pressure
);
9022 for (use
= data
->reg_use_list
; use
!= NULL
; use
= next
)
9024 next
= use
->next_insn_use
;
9031 /* Init data for the new insn INSN. */
9033 haifa_init_insn (rtx_insn
*insn
)
9035 gcc_assert (insn
!= NULL
);
9037 sched_extend_luids ();
9038 sched_init_insn_luid (insn
);
9039 sched_extend_target ();
9040 sched_deps_init (false);
9044 if (adding_bb_to_current_region_p
)
9046 sd_init_insn (insn
);
9048 /* Extend dependency caches by one element. */
9049 extend_dependency_caches (1, false);
9051 if (sched_pressure
!= SCHED_PRESSURE_NONE
)
9052 init_insn_reg_pressure_info (insn
);
9055 /* Init data for the new basic block BB which comes after AFTER. */
9057 haifa_init_only_bb (basic_block bb
, basic_block after
)
9059 gcc_assert (bb
!= NULL
);
9063 if (common_sched_info
->add_block
)
9064 /* This changes only data structures of the front-end. */
9065 common_sched_info
->add_block (bb
, after
);
9068 /* A generic version of sched_split_block (). */
9070 sched_split_block_1 (basic_block first_bb
, rtx after
)
9074 e
= split_block (first_bb
, after
);
9075 gcc_assert (e
->src
== first_bb
);
9077 /* sched_split_block emits note if *check == BB_END. Probably it
9078 is better to rip that note off. */
9083 /* A generic version of sched_create_empty_bb (). */
9085 sched_create_empty_bb_1 (basic_block after
)
9087 return create_empty_bb (after
);
9090 /* Insert PAT as an INSN into the schedule and update the necessary data
9091 structures to account for it. */
9093 sched_emit_insn (rtx pat
)
9095 rtx_insn
*insn
= emit_insn_before (pat
, first_nonscheduled_insn ());
9096 haifa_init_insn (insn
);
9098 if (current_sched_info
->add_remove_insn
)
9099 current_sched_info
->add_remove_insn (insn
, 0);
9101 (*current_sched_info
->begin_schedule_ready
) (insn
);
9102 scheduled_insns
.safe_push (insn
);
9104 last_scheduled_insn
= insn
;
9108 /* This function returns a candidate satisfying dispatch constraints from
9112 ready_remove_first_dispatch (struct ready_list
*ready
)
9115 rtx_insn
*insn
= ready_element (ready
, 0);
9117 if (ready
->n_ready
== 1
9119 || INSN_CODE (insn
) < 0
9120 || !active_insn_p (insn
)
9121 || targetm
.sched
.dispatch (insn
, FITS_DISPATCH_WINDOW
))
9122 return ready_remove_first (ready
);
9124 for (i
= 1; i
< ready
->n_ready
; i
++)
9126 insn
= ready_element (ready
, i
);
9129 || INSN_CODE (insn
) < 0
9130 || !active_insn_p (insn
))
9133 if (targetm
.sched
.dispatch (insn
, FITS_DISPATCH_WINDOW
))
9135 /* Return ith element of ready. */
9136 insn
= ready_remove (ready
, i
);
9141 if (targetm
.sched
.dispatch (NULL
, DISPATCH_VIOLATION
))
9142 return ready_remove_first (ready
);
9144 for (i
= 1; i
< ready
->n_ready
; i
++)
9146 insn
= ready_element (ready
, i
);
9149 || INSN_CODE (insn
) < 0
9150 || !active_insn_p (insn
))
9153 /* Return i-th element of ready. */
9154 if (targetm
.sched
.dispatch (insn
, IS_CMP
))
9155 return ready_remove (ready
, i
);
9158 return ready_remove_first (ready
);
9161 /* Get number of ready insn in the ready list. */
9164 number_in_ready (void)
9166 return ready
.n_ready
;
9169 /* Get number of ready's in the ready list. */
9172 get_ready_element (int i
)
9174 return ready_element (&ready
, i
);
9177 #endif /* INSN_SCHEDULING */