1 /* Instruction scheduling pass.
2 Copyright (C) 1992-2015 Free Software Foundation, Inc.
3 Contributed by Michael Tiemann (tiemann@cygnus.com) Enhanced by,
4 and currently maintained by, Jim Wilson (wilson@cygnus.com)
6 This file is part of GCC.
8 GCC is free software; you can redistribute it and/or modify it under
9 the terms of the GNU General Public License as published by the Free
10 Software Foundation; either version 3, or (at your option) any later
13 GCC is distributed in the hope that it will be useful, but WITHOUT ANY
14 WARRANTY; without even the implied warranty of MERCHANTABILITY or
15 FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
18 You should have received a copy of the GNU General Public License
19 along with GCC; see the file COPYING3. If not see
20 <http://www.gnu.org/licenses/>. */
22 /* Instruction scheduling pass. This file, along with sched-deps.c,
23 contains the generic parts. The actual entry point for
24 the normal instruction scheduling pass is found in sched-rgn.c.
26 We compute insn priorities based on data dependencies. Flow
27 analysis only creates a fraction of the data-dependencies we must
28 observe: namely, only those dependencies which the combiner can be
29 expected to use. For this pass, we must therefore create the
30 remaining dependencies we need to observe: register dependencies,
31 memory dependencies, dependencies to keep function calls in order,
32 and the dependence between a conditional branch and the setting of
33 condition codes are all dealt with here.
35 The scheduler first traverses the data flow graph, starting with
36 the last instruction, and proceeding to the first, assigning values
37 to insn_priority as it goes. This sorts the instructions
38 topologically by data dependence.
40 Once priorities have been established, we order the insns using
41 list scheduling. This works as follows: starting with a list of
42 all the ready insns, and sorted according to priority number, we
43 schedule the insn from the end of the list by placing its
44 predecessors in the list according to their priority order. We
45 consider this insn scheduled by setting the pointer to the "end" of
46 the list to point to the previous insn. When an insn has no
47 predecessors, we either queue it until sufficient time has elapsed
48 or add it to the ready list. As the instructions are scheduled or
49 when stalls are introduced, the queue advances and dumps insns into
50 the ready list. When all insns down to the lowest priority have
51 been scheduled, the critical path of the basic block has been made
52 as short as possible. The remaining insns are then scheduled in
55 The following list shows the order in which we want to break ties
56 among insns in the ready list:
58 1. choose insn with the longest path to end of bb, ties
60 2. choose insn with least contribution to register pressure,
62 3. prefer in-block upon interblock motion, ties broken by
63 4. prefer useful upon speculative motion, ties broken by
64 5. choose insn with largest control flow probability, ties
66 6. choose insn with the least dependences upon the previously
67 scheduled insn, or finally
68 7 choose the insn which has the most insns dependent on it.
69 8. choose insn with lowest UID.
71 Memory references complicate matters. Only if we can be certain
72 that memory references are not part of the data dependency graph
73 (via true, anti, or output dependence), can we move operations past
74 memory references. To first approximation, reads can be done
75 independently, while writes introduce dependencies. Better
76 approximations will yield fewer dependencies.
78 Before reload, an extended analysis of interblock data dependences
79 is required for interblock scheduling. This is performed in
80 compute_block_dependences ().
82 Dependencies set up by memory references are treated in exactly the
83 same way as other dependencies, by using insn backward dependences
84 INSN_BACK_DEPS. INSN_BACK_DEPS are translated into forward dependences
85 INSN_FORW_DEPS for the purpose of forward list scheduling.
87 Having optimized the critical path, we may have also unduly
88 extended the lifetimes of some registers. If an operation requires
89 that constants be loaded into registers, it is certainly desirable
90 to load those constants as early as necessary, but no earlier.
91 I.e., it will not do to load up a bunch of registers at the
92 beginning of a basic block only to use them at the end, if they
93 could be loaded later, since this may result in excessive register
96 Note that since branches are never in basic blocks, but only end
97 basic blocks, this pass will not move branches. But that is ok,
98 since we can use GNU's delayed branch scheduling pass to take care
101 Also note that no further optimizations based on algebraic
102 identities are performed, so this pass would be a good one to
103 perform instruction splitting, such as breaking up a multiply
104 instruction into shifts and adds where that is profitable.
106 Given the memory aliasing analysis that this pass should perform,
107 it should be possible to remove redundant stores to memory, and to
108 load values from registers instead of hitting memory.
110 Before reload, speculative insns are moved only if a 'proof' exists
111 that no exception will be caused by this, and if no live registers
112 exist that inhibit the motion (live registers constraints are not
113 represented by data dependence edges).
115 This pass must update information that subsequent passes expect to
116 be correct. Namely: reg_n_refs, reg_n_sets, reg_n_deaths,
117 reg_n_calls_crossed, and reg_live_length. Also, BB_HEAD, BB_END.
119 The information in the line number notes is carefully retained by
120 this pass. Notes that refer to the starting and ending of
121 exception regions are also carefully retained by this pass. All
122 other NOTE insns are grouped in their same relative order at the
123 beginning of basic blocks and regions that have been scheduled. */
127 #include "coretypes.h"
131 #include "cfghooks.h"
134 #include "insn-config.h"
138 #include "diagnostic-core.h"
140 #include "insn-attr.h"
143 #include "cfgbuild.h"
144 #include "sched-int.h"
145 #include "common/common-target.h"
149 #include "dumpfile.h"
150 #include "print-rtl.h"
152 #ifdef INSN_SCHEDULING
154 /* True if we do register pressure relief through live-range
156 static bool live_range_shrinkage_p
;
158 /* Switch on live range shrinkage. */
160 initialize_live_range_shrinkage (void)
162 live_range_shrinkage_p
= true;
165 /* Switch off live range shrinkage. */
167 finish_live_range_shrinkage (void)
169 live_range_shrinkage_p
= false;
172 /* issue_rate is the number of insns that can be scheduled in the same
173 machine cycle. It can be defined in the config/mach/mach.h file,
174 otherwise we set it to 1. */
178 /* This can be set to true by a backend if the scheduler should not
179 enable a DCE pass. */
182 /* The current initiation interval used when modulo scheduling. */
183 static int modulo_ii
;
185 /* The maximum number of stages we are prepared to handle. */
186 static int modulo_max_stages
;
188 /* The number of insns that exist in each iteration of the loop. We use this
189 to detect when we've scheduled all insns from the first iteration. */
190 static int modulo_n_insns
;
192 /* The current count of insns in the first iteration of the loop that have
193 already been scheduled. */
194 static int modulo_insns_scheduled
;
196 /* The maximum uid of insns from the first iteration of the loop. */
197 static int modulo_iter0_max_uid
;
199 /* The number of times we should attempt to backtrack when modulo scheduling.
200 Decreased each time we have to backtrack. */
201 static int modulo_backtracks_left
;
203 /* The stage in which the last insn from the original loop was
205 static int modulo_last_stage
;
207 /* sched-verbose controls the amount of debugging output the
208 scheduler prints. It is controlled by -fsched-verbose=N:
209 N>0 and no -DSR : the output is directed to stderr.
210 N>=10 will direct the printouts to stderr (regardless of -dSR).
212 N=2: bb's probabilities, detailed ready list info, unit/insn info.
213 N=3: rtl at abort point, control-flow, regions info.
214 N=5: dependences info. */
216 int sched_verbose
= 0;
218 /* Debugging file. All printouts are sent to dump, which is always set,
219 either to stderr, or to the dump listing file (-dRS). */
220 FILE *sched_dump
= 0;
222 /* This is a placeholder for the scheduler parameters common
223 to all schedulers. */
224 struct common_sched_info_def
*common_sched_info
;
226 #define INSN_TICK(INSN) (HID (INSN)->tick)
227 #define INSN_EXACT_TICK(INSN) (HID (INSN)->exact_tick)
228 #define INSN_TICK_ESTIMATE(INSN) (HID (INSN)->tick_estimate)
229 #define INTER_TICK(INSN) (HID (INSN)->inter_tick)
230 #define FEEDS_BACKTRACK_INSN(INSN) (HID (INSN)->feeds_backtrack_insn)
231 #define SHADOW_P(INSN) (HID (INSN)->shadow_p)
232 #define MUST_RECOMPUTE_SPEC_P(INSN) (HID (INSN)->must_recompute_spec)
233 /* Cached cost of the instruction. Use insn_cost to get cost of the
234 insn. -1 here means that the field is not initialized. */
235 #define INSN_COST(INSN) (HID (INSN)->cost)
237 /* If INSN_TICK of an instruction is equal to INVALID_TICK,
238 then it should be recalculated from scratch. */
239 #define INVALID_TICK (-(max_insn_queue_index + 1))
240 /* The minimal value of the INSN_TICK of an instruction. */
241 #define MIN_TICK (-max_insn_queue_index)
243 /* Original order of insns in the ready list.
244 Used to keep order of normal insns while separating DEBUG_INSNs. */
245 #define INSN_RFS_DEBUG_ORIG_ORDER(INSN) (HID (INSN)->rfs_debug_orig_order)
247 /* The deciding reason for INSN's place in the ready list. */
248 #define INSN_LAST_RFS_WIN(INSN) (HID (INSN)->last_rfs_win)
250 /* List of important notes we must keep around. This is a pointer to the
251 last element in the list. */
254 static struct spec_info_def spec_info_var
;
255 /* Description of the speculative part of the scheduling.
256 If NULL - no speculation. */
257 spec_info_t spec_info
= NULL
;
259 /* True, if recovery block was added during scheduling of current block.
260 Used to determine, if we need to fix INSN_TICKs. */
261 static bool haifa_recovery_bb_recently_added_p
;
263 /* True, if recovery block was added during this scheduling pass.
264 Used to determine if we should have empty memory pools of dependencies
265 after finishing current region. */
266 bool haifa_recovery_bb_ever_added_p
;
268 /* Counters of different types of speculative instructions. */
269 static int nr_begin_data
, nr_be_in_data
, nr_begin_control
, nr_be_in_control
;
271 /* Array used in {unlink, restore}_bb_notes. */
272 static rtx_insn
**bb_header
= 0;
274 /* Basic block after which recovery blocks will be created. */
275 static basic_block before_recovery
;
277 /* Basic block just before the EXIT_BLOCK and after recovery, if we have
279 basic_block after_recovery
;
281 /* FALSE if we add bb to another region, so we don't need to initialize it. */
282 bool adding_bb_to_current_region_p
= true;
286 /* An instruction is ready to be scheduled when all insns preceding it
287 have already been scheduled. It is important to ensure that all
288 insns which use its result will not be executed until its result
289 has been computed. An insn is maintained in one of four structures:
291 (P) the "Pending" set of insns which cannot be scheduled until
292 their dependencies have been satisfied.
293 (Q) the "Queued" set of insns that can be scheduled when sufficient
295 (R) the "Ready" list of unscheduled, uncommitted insns.
296 (S) the "Scheduled" list of insns.
298 Initially, all insns are either "Pending" or "Ready" depending on
299 whether their dependencies are satisfied.
301 Insns move from the "Ready" list to the "Scheduled" list as they
302 are committed to the schedule. As this occurs, the insns in the
303 "Pending" list have their dependencies satisfied and move to either
304 the "Ready" list or the "Queued" set depending on whether
305 sufficient time has passed to make them ready. As time passes,
306 insns move from the "Queued" set to the "Ready" list.
308 The "Pending" list (P) are the insns in the INSN_FORW_DEPS of the
309 unscheduled insns, i.e., those that are ready, queued, and pending.
310 The "Queued" set (Q) is implemented by the variable `insn_queue'.
311 The "Ready" list (R) is implemented by the variables `ready' and
313 The "Scheduled" list (S) is the new insn chain built by this pass.
315 The transition (R->S) is implemented in the scheduling loop in
316 `schedule_block' when the best insn to schedule is chosen.
317 The transitions (P->R and P->Q) are implemented in `schedule_insn' as
318 insns move from the ready list to the scheduled list.
319 The transition (Q->R) is implemented in 'queue_to_insn' as time
320 passes or stalls are introduced. */
322 /* Implement a circular buffer to delay instructions until sufficient
323 time has passed. For the new pipeline description interface,
324 MAX_INSN_QUEUE_INDEX is a power of two minus one which is not less
325 than maximal time of instruction execution computed by genattr.c on
326 the base maximal time of functional unit reservations and getting a
327 result. This is the longest time an insn may be queued. */
329 static rtx_insn_list
**insn_queue
;
330 static int q_ptr
= 0;
331 static int q_size
= 0;
332 #define NEXT_Q(X) (((X)+1) & max_insn_queue_index)
333 #define NEXT_Q_AFTER(X, C) (((X)+C) & max_insn_queue_index)
335 #define QUEUE_SCHEDULED (-3)
336 #define QUEUE_NOWHERE (-2)
337 #define QUEUE_READY (-1)
338 /* QUEUE_SCHEDULED - INSN is scheduled.
339 QUEUE_NOWHERE - INSN isn't scheduled yet and is neither in
341 QUEUE_READY - INSN is in ready list.
342 N >= 0 - INSN queued for X [where NEXT_Q_AFTER (q_ptr, X) == N] cycles. */
344 #define QUEUE_INDEX(INSN) (HID (INSN)->queue_index)
346 /* The following variable value refers for all current and future
347 reservations of the processor units. */
350 /* The following variable value is size of memory representing all
351 current and future reservations of the processor units. */
352 size_t dfa_state_size
;
354 /* The following array is used to find the best insn from ready when
355 the automaton pipeline interface is used. */
356 signed char *ready_try
= NULL
;
358 /* The ready list. */
359 struct ready_list ready
= {NULL
, 0, 0, 0, 0};
361 /* The pointer to the ready list (to be removed). */
362 static struct ready_list
*readyp
= &ready
;
364 /* Scheduling clock. */
365 static int clock_var
;
367 /* Clock at which the previous instruction was issued. */
368 static int last_clock_var
;
370 /* Set to true if, when queuing a shadow insn, we discover that it would be
371 scheduled too late. */
372 static bool must_backtrack
;
374 /* The following variable value is number of essential insns issued on
375 the current cycle. An insn is essential one if it changes the
377 int cycle_issued_insns
;
379 /* This records the actual schedule. It is built up during the main phase
380 of schedule_block, and afterwards used to reorder the insns in the RTL. */
381 static vec
<rtx_insn
*> scheduled_insns
;
383 static int may_trap_exp (const_rtx
, int);
385 /* Nonzero iff the address is comprised from at most 1 register. */
386 #define CONST_BASED_ADDRESS_P(x) \
388 || ((GET_CODE (x) == PLUS || GET_CODE (x) == MINUS \
389 || (GET_CODE (x) == LO_SUM)) \
390 && (CONSTANT_P (XEXP (x, 0)) \
391 || CONSTANT_P (XEXP (x, 1)))))
393 /* Returns a class that insn with GET_DEST(insn)=x may belong to,
394 as found by analyzing insn's expression. */
397 static int haifa_luid_for_non_insn (rtx x
);
399 /* Haifa version of sched_info hooks common to all headers. */
400 const struct common_sched_info_def haifa_common_sched_info
=
402 NULL
, /* fix_recovery_cfg */
403 NULL
, /* add_block */
404 NULL
, /* estimate_number_of_insns */
405 haifa_luid_for_non_insn
, /* luid_for_non_insn */
406 SCHED_PASS_UNKNOWN
/* sched_pass_id */
409 /* Mapping from instruction UID to its Logical UID. */
410 vec
<int> sched_luids
= vNULL
;
412 /* Next LUID to assign to an instruction. */
413 int sched_max_luid
= 1;
415 /* Haifa Instruction Data. */
416 vec
<haifa_insn_data_def
> h_i_d
= vNULL
;
418 void (* sched_init_only_bb
) (basic_block
, basic_block
);
420 /* Split block function. Different schedulers might use different functions
421 to handle their internal data consistent. */
422 basic_block (* sched_split_block
) (basic_block
, rtx
);
424 /* Create empty basic block after the specified block. */
425 basic_block (* sched_create_empty_bb
) (basic_block
);
427 /* Return the number of cycles until INSN is expected to be ready.
428 Return zero if it already is. */
430 insn_delay (rtx_insn
*insn
)
432 return MAX (INSN_TICK (insn
) - clock_var
, 0);
436 may_trap_exp (const_rtx x
, int is_store
)
445 if (code
== MEM
&& may_trap_p (x
))
452 /* The insn uses memory: a volatile load. */
453 if (MEM_VOLATILE_P (x
))
455 /* An exception-free load. */
458 /* A load with 1 base register, to be further checked. */
459 if (CONST_BASED_ADDRESS_P (XEXP (x
, 0)))
460 return PFREE_CANDIDATE
;
461 /* No info on the load, to be further checked. */
462 return PRISKY_CANDIDATE
;
467 int i
, insn_class
= TRAP_FREE
;
469 /* Neither store nor load, check if it may cause a trap. */
472 /* Recursive step: walk the insn... */
473 fmt
= GET_RTX_FORMAT (code
);
474 for (i
= GET_RTX_LENGTH (code
) - 1; i
>= 0; i
--)
478 int tmp_class
= may_trap_exp (XEXP (x
, i
), is_store
);
479 insn_class
= WORST_CLASS (insn_class
, tmp_class
);
481 else if (fmt
[i
] == 'E')
484 for (j
= 0; j
< XVECLEN (x
, i
); j
++)
486 int tmp_class
= may_trap_exp (XVECEXP (x
, i
, j
), is_store
);
487 insn_class
= WORST_CLASS (insn_class
, tmp_class
);
488 if (insn_class
== TRAP_RISKY
|| insn_class
== IRISKY
)
492 if (insn_class
== TRAP_RISKY
|| insn_class
== IRISKY
)
499 /* Classifies rtx X of an insn for the purpose of verifying that X can be
500 executed speculatively (and consequently the insn can be moved
501 speculatively), by examining X, returning:
502 TRAP_RISKY: store, or risky non-load insn (e.g. division by variable).
503 TRAP_FREE: non-load insn.
504 IFREE: load from a globally safe location.
505 IRISKY: volatile load.
506 PFREE_CANDIDATE, PRISKY_CANDIDATE: load that need to be checked for
507 being either PFREE or PRISKY. */
510 haifa_classify_rtx (const_rtx x
)
512 int tmp_class
= TRAP_FREE
;
513 int insn_class
= TRAP_FREE
;
516 if (GET_CODE (x
) == PARALLEL
)
518 int i
, len
= XVECLEN (x
, 0);
520 for (i
= len
- 1; i
>= 0; i
--)
522 tmp_class
= haifa_classify_rtx (XVECEXP (x
, 0, i
));
523 insn_class
= WORST_CLASS (insn_class
, tmp_class
);
524 if (insn_class
== TRAP_RISKY
|| insn_class
== IRISKY
)
534 /* Test if it is a 'store'. */
535 tmp_class
= may_trap_exp (XEXP (x
, 0), 1);
538 /* Test if it is a store. */
539 tmp_class
= may_trap_exp (SET_DEST (x
), 1);
540 if (tmp_class
== TRAP_RISKY
)
542 /* Test if it is a load. */
544 WORST_CLASS (tmp_class
,
545 may_trap_exp (SET_SRC (x
), 0));
548 tmp_class
= haifa_classify_rtx (COND_EXEC_CODE (x
));
549 if (tmp_class
== TRAP_RISKY
)
551 tmp_class
= WORST_CLASS (tmp_class
,
552 may_trap_exp (COND_EXEC_TEST (x
), 0));
555 tmp_class
= TRAP_RISKY
;
559 insn_class
= tmp_class
;
566 haifa_classify_insn (const_rtx insn
)
568 return haifa_classify_rtx (PATTERN (insn
));
571 /* After the scheduler initialization function has been called, this function
572 can be called to enable modulo scheduling. II is the initiation interval
573 we should use, it affects the delays for delay_pairs that were recorded as
574 separated by a given number of stages.
576 MAX_STAGES provides us with a limit
577 after which we give up scheduling; the caller must have unrolled at least
578 as many copies of the loop body and recorded delay_pairs for them.
580 INSNS is the number of real (non-debug) insns in one iteration of
581 the loop. MAX_UID can be used to test whether an insn belongs to
582 the first iteration of the loop; all of them have a uid lower than
585 set_modulo_params (int ii
, int max_stages
, int insns
, int max_uid
)
588 modulo_max_stages
= max_stages
;
589 modulo_n_insns
= insns
;
590 modulo_iter0_max_uid
= max_uid
;
591 modulo_backtracks_left
= PARAM_VALUE (PARAM_MAX_MODULO_BACKTRACK_ATTEMPTS
);
594 /* A structure to record a pair of insns where the first one is a real
595 insn that has delay slots, and the second is its delayed shadow.
596 I1 is scheduled normally and will emit an assembly instruction,
597 while I2 describes the side effect that takes place at the
598 transition between cycles CYCLES and (CYCLES + 1) after I1. */
601 struct delay_pair
*next_same_i1
;
604 /* When doing modulo scheduling, we a delay_pair can also be used to
605 show that I1 and I2 are the same insn in a different stage. If that
606 is the case, STAGES will be nonzero. */
610 /* Helpers for delay hashing. */
612 struct delay_i1_hasher
: nofree_ptr_hash
<delay_pair
>
614 typedef void *compare_type
;
615 static inline hashval_t
hash (const delay_pair
*);
616 static inline bool equal (const delay_pair
*, const void *);
619 /* Returns a hash value for X, based on hashing just I1. */
622 delay_i1_hasher::hash (const delay_pair
*x
)
624 return htab_hash_pointer (x
->i1
);
627 /* Return true if I1 of pair X is the same as that of pair Y. */
630 delay_i1_hasher::equal (const delay_pair
*x
, const void *y
)
635 struct delay_i2_hasher
: free_ptr_hash
<delay_pair
>
637 typedef void *compare_type
;
638 static inline hashval_t
hash (const delay_pair
*);
639 static inline bool equal (const delay_pair
*, const void *);
642 /* Returns a hash value for X, based on hashing just I2. */
645 delay_i2_hasher::hash (const delay_pair
*x
)
647 return htab_hash_pointer (x
->i2
);
650 /* Return true if I2 of pair X is the same as that of pair Y. */
653 delay_i2_hasher::equal (const delay_pair
*x
, const void *y
)
658 /* Two hash tables to record delay_pairs, one indexed by I1 and the other
660 static hash_table
<delay_i1_hasher
> *delay_htab
;
661 static hash_table
<delay_i2_hasher
> *delay_htab_i2
;
663 /* Called through htab_traverse. Walk the hashtable using I2 as
664 index, and delete all elements involving an UID higher than
665 that pointed to by *DATA. */
667 haifa_htab_i2_traverse (delay_pair
**slot
, int *data
)
670 struct delay_pair
*p
= *slot
;
671 if (INSN_UID (p
->i2
) >= maxuid
|| INSN_UID (p
->i1
) >= maxuid
)
673 delay_htab_i2
->clear_slot (slot
);
678 /* Called through htab_traverse. Walk the hashtable using I2 as
679 index, and delete all elements involving an UID higher than
680 that pointed to by *DATA. */
682 haifa_htab_i1_traverse (delay_pair
**pslot
, int *data
)
685 struct delay_pair
*p
, *first
, **pprev
;
687 if (INSN_UID ((*pslot
)->i1
) >= maxuid
)
689 delay_htab
->clear_slot (pslot
);
693 for (p
= *pslot
; p
; p
= p
->next_same_i1
)
695 if (INSN_UID (p
->i2
) < maxuid
)
698 pprev
= &p
->next_same_i1
;
703 delay_htab
->clear_slot (pslot
);
709 /* Discard all delay pairs which involve an insn with an UID higher
712 discard_delay_pairs_above (int max_uid
)
714 delay_htab
->traverse
<int *, haifa_htab_i1_traverse
> (&max_uid
);
715 delay_htab_i2
->traverse
<int *, haifa_htab_i2_traverse
> (&max_uid
);
718 /* This function can be called by a port just before it starts the final
719 scheduling pass. It records the fact that an instruction with delay
720 slots has been split into two insns, I1 and I2. The first one will be
721 scheduled normally and initiates the operation. The second one is a
722 shadow which must follow a specific number of cycles after I1; its only
723 purpose is to show the side effect that occurs at that cycle in the RTL.
724 If a JUMP_INSN or a CALL_INSN has been split, I1 should be a normal INSN,
725 while I2 retains the original insn type.
727 There are two ways in which the number of cycles can be specified,
728 involving the CYCLES and STAGES arguments to this function. If STAGES
729 is zero, we just use the value of CYCLES. Otherwise, STAGES is a factor
730 which is multiplied by MODULO_II to give the number of cycles. This is
731 only useful if the caller also calls set_modulo_params to enable modulo
735 record_delay_slot_pair (rtx_insn
*i1
, rtx_insn
*i2
, int cycles
, int stages
)
737 struct delay_pair
*p
= XNEW (struct delay_pair
);
738 struct delay_pair
**slot
;
747 delay_htab
= new hash_table
<delay_i1_hasher
> (10);
748 delay_htab_i2
= new hash_table
<delay_i2_hasher
> (10);
750 slot
= delay_htab
->find_slot_with_hash (i1
, htab_hash_pointer (i1
), INSERT
);
751 p
->next_same_i1
= *slot
;
753 slot
= delay_htab_i2
->find_slot (p
, INSERT
);
757 /* Examine the delay pair hashtable to see if INSN is a shadow for another,
758 and return the other insn if so. Return NULL otherwise. */
760 real_insn_for_shadow (rtx_insn
*insn
)
762 struct delay_pair
*pair
;
767 pair
= delay_htab_i2
->find_with_hash (insn
, htab_hash_pointer (insn
));
768 if (!pair
|| pair
->stages
> 0)
773 /* For a pair P of insns, return the fixed distance in cycles from the first
774 insn after which the second must be scheduled. */
776 pair_delay (struct delay_pair
*p
)
781 return p
->stages
* modulo_ii
;
784 /* Given an insn INSN, add a dependence on its delayed shadow if it
785 has one. Also try to find situations where shadows depend on each other
786 and add dependencies to the real insns to limit the amount of backtracking
789 add_delay_dependencies (rtx_insn
*insn
)
791 struct delay_pair
*pair
;
792 sd_iterator_def sd_it
;
798 pair
= delay_htab_i2
->find_with_hash (insn
, htab_hash_pointer (insn
));
801 add_dependence (insn
, pair
->i1
, REG_DEP_ANTI
);
805 FOR_EACH_DEP (pair
->i2
, SD_LIST_BACK
, sd_it
, dep
)
807 rtx_insn
*pro
= DEP_PRO (dep
);
808 struct delay_pair
*other_pair
809 = delay_htab_i2
->find_with_hash (pro
, htab_hash_pointer (pro
));
810 if (!other_pair
|| other_pair
->stages
)
812 if (pair_delay (other_pair
) >= pair_delay (pair
))
814 if (sched_verbose
>= 4)
816 fprintf (sched_dump
, ";;\tadding dependence %d <- %d\n",
817 INSN_UID (other_pair
->i1
),
818 INSN_UID (pair
->i1
));
819 fprintf (sched_dump
, ";;\tpair1 %d <- %d, cost %d\n",
823 fprintf (sched_dump
, ";;\tpair2 %d <- %d, cost %d\n",
824 INSN_UID (other_pair
->i1
),
825 INSN_UID (other_pair
->i2
),
826 pair_delay (other_pair
));
828 add_dependence (pair
->i1
, other_pair
->i1
, REG_DEP_ANTI
);
833 /* Forward declarations. */
835 static int priority (rtx_insn
*);
836 static int autopref_rank_for_schedule (const rtx_insn
*, const rtx_insn
*);
837 static int rank_for_schedule (const void *, const void *);
838 static void swap_sort (rtx_insn
**, int);
839 static void queue_insn (rtx_insn
*, int, const char *);
840 static int schedule_insn (rtx_insn
*);
841 static void adjust_priority (rtx_insn
*);
842 static void advance_one_cycle (void);
843 static void extend_h_i_d (void);
846 /* Notes handling mechanism:
847 =========================
848 Generally, NOTES are saved before scheduling and restored after scheduling.
849 The scheduler distinguishes between two types of notes:
851 (1) LOOP_BEGIN, LOOP_END, SETJMP, EHREGION_BEG, EHREGION_END notes:
852 Before scheduling a region, a pointer to the note is added to the insn
853 that follows or precedes it. (This happens as part of the data dependence
854 computation). After scheduling an insn, the pointer contained in it is
855 used for regenerating the corresponding note (in reemit_notes).
857 (2) All other notes (e.g. INSN_DELETED): Before scheduling a block,
858 these notes are put in a list (in rm_other_notes() and
859 unlink_other_notes ()). After scheduling the block, these notes are
860 inserted at the beginning of the block (in schedule_block()). */
862 static void ready_add (struct ready_list
*, rtx_insn
*, bool);
863 static rtx_insn
*ready_remove_first (struct ready_list
*);
864 static rtx_insn
*ready_remove_first_dispatch (struct ready_list
*ready
);
866 static void queue_to_ready (struct ready_list
*);
867 static int early_queue_to_ready (state_t
, struct ready_list
*);
869 /* The following functions are used to implement multi-pass scheduling
870 on the first cycle. */
871 static rtx_insn
*ready_remove (struct ready_list
*, int);
872 static void ready_remove_insn (rtx_insn
*);
874 static void fix_inter_tick (rtx_insn
*, rtx_insn
*);
875 static int fix_tick_ready (rtx_insn
*);
876 static void change_queue_index (rtx_insn
*, int);
878 /* The following functions are used to implement scheduling of data/control
879 speculative instructions. */
881 static void extend_h_i_d (void);
882 static void init_h_i_d (rtx_insn
*);
883 static int haifa_speculate_insn (rtx_insn
*, ds_t
, rtx
*);
884 static void generate_recovery_code (rtx_insn
*);
885 static void process_insn_forw_deps_be_in_spec (rtx_insn
*, rtx_insn
*, ds_t
);
886 static void begin_speculative_block (rtx_insn
*);
887 static void add_to_speculative_block (rtx_insn
*);
888 static void init_before_recovery (basic_block
*);
889 static void create_check_block_twin (rtx_insn
*, bool);
890 static void fix_recovery_deps (basic_block
);
891 static bool haifa_change_pattern (rtx_insn
*, rtx
);
892 static void dump_new_block_header (int, basic_block
, rtx_insn
*, rtx_insn
*);
893 static void restore_bb_notes (basic_block
);
894 static void fix_jump_move (rtx_insn
*);
895 static void move_block_after_check (rtx_insn
*);
896 static void move_succs (vec
<edge
, va_gc
> **, basic_block
);
897 static void sched_remove_insn (rtx_insn
*);
898 static void clear_priorities (rtx_insn
*, rtx_vec_t
*);
899 static void calc_priorities (rtx_vec_t
);
900 static void add_jump_dependencies (rtx_insn
*, rtx_insn
*);
902 #endif /* INSN_SCHEDULING */
904 /* Point to state used for the current scheduling pass. */
905 struct haifa_sched_info
*current_sched_info
;
907 #ifndef INSN_SCHEDULING
909 schedule_insns (void)
914 /* Do register pressure sensitive insn scheduling if the flag is set
916 enum sched_pressure_algorithm sched_pressure
;
918 /* Map regno -> its pressure class. The map defined only when
919 SCHED_PRESSURE != SCHED_PRESSURE_NONE. */
920 enum reg_class
*sched_regno_pressure_class
;
922 /* The current register pressure. Only elements corresponding pressure
923 classes are defined. */
924 static int curr_reg_pressure
[N_REG_CLASSES
];
926 /* Saved value of the previous array. */
927 static int saved_reg_pressure
[N_REG_CLASSES
];
929 /* Register living at given scheduling point. */
930 static bitmap curr_reg_live
;
932 /* Saved value of the previous array. */
933 static bitmap saved_reg_live
;
935 /* Registers mentioned in the current region. */
936 static bitmap region_ref_regs
;
938 /* Effective number of available registers of a given class (see comment
939 in sched_pressure_start_bb). */
940 static int sched_class_regs_num
[N_REG_CLASSES
];
941 /* Number of call_used_regs. This is a helper for calculating of
942 sched_class_regs_num. */
943 static int call_used_regs_num
[N_REG_CLASSES
];
945 /* Initiate register pressure relative info for scheduling the current
946 region. Currently it is only clearing register mentioned in the
949 sched_init_region_reg_pressure_info (void)
951 bitmap_clear (region_ref_regs
);
954 /* PRESSURE[CL] describes the pressure on register class CL. Update it
955 for the birth (if BIRTH_P) or death (if !BIRTH_P) of register REGNO.
956 LIVE tracks the set of live registers; if it is null, assume that
957 every birth or death is genuine. */
959 mark_regno_birth_or_death (bitmap live
, int *pressure
, int regno
, bool birth_p
)
961 enum reg_class pressure_class
;
963 pressure_class
= sched_regno_pressure_class
[regno
];
964 if (regno
>= FIRST_PSEUDO_REGISTER
)
966 if (pressure_class
!= NO_REGS
)
970 if (!live
|| bitmap_set_bit (live
, regno
))
971 pressure
[pressure_class
]
972 += (ira_reg_class_max_nregs
973 [pressure_class
][PSEUDO_REGNO_MODE (regno
)]);
977 if (!live
|| bitmap_clear_bit (live
, regno
))
978 pressure
[pressure_class
]
979 -= (ira_reg_class_max_nregs
980 [pressure_class
][PSEUDO_REGNO_MODE (regno
)]);
984 else if (pressure_class
!= NO_REGS
985 && ! TEST_HARD_REG_BIT (ira_no_alloc_regs
, regno
))
989 if (!live
|| bitmap_set_bit (live
, regno
))
990 pressure
[pressure_class
]++;
994 if (!live
|| bitmap_clear_bit (live
, regno
))
995 pressure
[pressure_class
]--;
1000 /* Initiate current register pressure related info from living
1001 registers given by LIVE. */
1003 initiate_reg_pressure_info (bitmap live
)
1009 for (i
= 0; i
< ira_pressure_classes_num
; i
++)
1010 curr_reg_pressure
[ira_pressure_classes
[i
]] = 0;
1011 bitmap_clear (curr_reg_live
);
1012 EXECUTE_IF_SET_IN_BITMAP (live
, 0, j
, bi
)
1013 if (sched_pressure
== SCHED_PRESSURE_MODEL
1014 || current_nr_blocks
== 1
1015 || bitmap_bit_p (region_ref_regs
, j
))
1016 mark_regno_birth_or_death (curr_reg_live
, curr_reg_pressure
, j
, true);
1019 /* Mark registers in X as mentioned in the current region. */
1021 setup_ref_regs (rtx x
)
1024 const RTX_CODE code
= GET_CODE (x
);
1029 bitmap_set_range (region_ref_regs
, REGNO (x
), REG_NREGS (x
));
1032 fmt
= GET_RTX_FORMAT (code
);
1033 for (i
= GET_RTX_LENGTH (code
) - 1; i
>= 0; i
--)
1035 setup_ref_regs (XEXP (x
, i
));
1036 else if (fmt
[i
] == 'E')
1038 for (j
= 0; j
< XVECLEN (x
, i
); j
++)
1039 setup_ref_regs (XVECEXP (x
, i
, j
));
1043 /* Initiate current register pressure related info at the start of
1046 initiate_bb_reg_pressure_info (basic_block bb
)
1048 unsigned int i ATTRIBUTE_UNUSED
;
1051 if (current_nr_blocks
> 1)
1052 FOR_BB_INSNS (bb
, insn
)
1053 if (NONDEBUG_INSN_P (insn
))
1054 setup_ref_regs (PATTERN (insn
));
1055 initiate_reg_pressure_info (df_get_live_in (bb
));
1056 if (bb_has_eh_pred (bb
))
1059 unsigned int regno
= EH_RETURN_DATA_REGNO (i
);
1061 if (regno
== INVALID_REGNUM
)
1063 if (! bitmap_bit_p (df_get_live_in (bb
), regno
))
1064 mark_regno_birth_or_death (curr_reg_live
, curr_reg_pressure
,
1069 /* Save current register pressure related info. */
1071 save_reg_pressure (void)
1075 for (i
= 0; i
< ira_pressure_classes_num
; i
++)
1076 saved_reg_pressure
[ira_pressure_classes
[i
]]
1077 = curr_reg_pressure
[ira_pressure_classes
[i
]];
1078 bitmap_copy (saved_reg_live
, curr_reg_live
);
1081 /* Restore saved register pressure related info. */
1083 restore_reg_pressure (void)
1087 for (i
= 0; i
< ira_pressure_classes_num
; i
++)
1088 curr_reg_pressure
[ira_pressure_classes
[i
]]
1089 = saved_reg_pressure
[ira_pressure_classes
[i
]];
1090 bitmap_copy (curr_reg_live
, saved_reg_live
);
1093 /* Return TRUE if the register is dying after its USE. */
1095 dying_use_p (struct reg_use_data
*use
)
1097 struct reg_use_data
*next
;
1099 for (next
= use
->next_regno_use
; next
!= use
; next
= next
->next_regno_use
)
1100 if (NONDEBUG_INSN_P (next
->insn
)
1101 && QUEUE_INDEX (next
->insn
) != QUEUE_SCHEDULED
)
1106 /* Print info about the current register pressure and its excess for
1107 each pressure class. */
1109 print_curr_reg_pressure (void)
1114 fprintf (sched_dump
, ";;\t");
1115 for (i
= 0; i
< ira_pressure_classes_num
; i
++)
1117 cl
= ira_pressure_classes
[i
];
1118 gcc_assert (curr_reg_pressure
[cl
] >= 0);
1119 fprintf (sched_dump
, " %s:%d(%d)", reg_class_names
[cl
],
1120 curr_reg_pressure
[cl
],
1121 curr_reg_pressure
[cl
] - sched_class_regs_num
[cl
]);
1123 fprintf (sched_dump
, "\n");
1126 /* Determine if INSN has a condition that is clobbered if a register
1127 in SET_REGS is modified. */
1129 cond_clobbered_p (rtx_insn
*insn
, HARD_REG_SET set_regs
)
1131 rtx pat
= PATTERN (insn
);
1132 gcc_assert (GET_CODE (pat
) == COND_EXEC
);
1133 if (TEST_HARD_REG_BIT (set_regs
, REGNO (XEXP (COND_EXEC_TEST (pat
), 0))))
1135 sd_iterator_def sd_it
;
1137 haifa_change_pattern (insn
, ORIG_PAT (insn
));
1138 FOR_EACH_DEP (insn
, SD_LIST_BACK
, sd_it
, dep
)
1139 DEP_STATUS (dep
) &= ~DEP_CANCELLED
;
1140 TODO_SPEC (insn
) = HARD_DEP
;
1141 if (sched_verbose
>= 2)
1142 fprintf (sched_dump
,
1143 ";;\t\tdequeue insn %s because of clobbered condition\n",
1144 (*current_sched_info
->print_insn
) (insn
, 0));
1151 /* This function should be called after modifying the pattern of INSN,
1152 to update scheduler data structures as needed. */
1154 update_insn_after_change (rtx_insn
*insn
)
1156 sd_iterator_def sd_it
;
1159 dfa_clear_single_insn_cache (insn
);
1161 sd_it
= sd_iterator_start (insn
,
1162 SD_LIST_FORW
| SD_LIST_BACK
| SD_LIST_RES_BACK
);
1163 while (sd_iterator_cond (&sd_it
, &dep
))
1165 DEP_COST (dep
) = UNKNOWN_DEP_COST
;
1166 sd_iterator_next (&sd_it
);
1169 /* Invalidate INSN_COST, so it'll be recalculated. */
1170 INSN_COST (insn
) = -1;
1171 /* Invalidate INSN_TICK, so it'll be recalculated. */
1172 INSN_TICK (insn
) = INVALID_TICK
;
1174 /* Invalidate autoprefetch data entry. */
1175 INSN_AUTOPREF_MULTIPASS_DATA (insn
)[0].status
1176 = AUTOPREF_MULTIPASS_DATA_UNINITIALIZED
;
1177 INSN_AUTOPREF_MULTIPASS_DATA (insn
)[1].status
1178 = AUTOPREF_MULTIPASS_DATA_UNINITIALIZED
;
1182 /* Two VECs, one to hold dependencies for which pattern replacements
1183 need to be applied or restored at the start of the next cycle, and
1184 another to hold an integer that is either one, to apply the
1185 corresponding replacement, or zero to restore it. */
1186 static vec
<dep_t
> next_cycle_replace_deps
;
1187 static vec
<int> next_cycle_apply
;
1189 static void apply_replacement (dep_t
, bool);
1190 static void restore_pattern (dep_t
, bool);
1192 /* Look at the remaining dependencies for insn NEXT, and compute and return
1193 the TODO_SPEC value we should use for it. This is called after one of
1194 NEXT's dependencies has been resolved.
1195 We also perform pattern replacements for predication, and for broken
1196 replacement dependencies. The latter is only done if FOR_BACKTRACK is
1200 recompute_todo_spec (rtx_insn
*next
, bool for_backtrack
)
1203 sd_iterator_def sd_it
;
1204 dep_t dep
, modify_dep
= NULL
;
1208 bool first_p
= true;
1210 if (sd_lists_empty_p (next
, SD_LIST_BACK
))
1211 /* NEXT has all its dependencies resolved. */
1214 if (!sd_lists_empty_p (next
, SD_LIST_HARD_BACK
))
1217 /* If NEXT is intended to sit adjacent to this instruction, we don't
1218 want to try to break any dependencies. Treat it as a HARD_DEP. */
1219 if (SCHED_GROUP_P (next
))
1222 /* Now we've got NEXT with speculative deps only.
1223 1. Look at the deps to see what we have to do.
1224 2. Check if we can do 'todo'. */
1227 FOR_EACH_DEP (next
, SD_LIST_BACK
, sd_it
, dep
)
1229 rtx_insn
*pro
= DEP_PRO (dep
);
1230 ds_t ds
= DEP_STATUS (dep
) & SPECULATIVE
;
1232 if (DEBUG_INSN_P (pro
) && !DEBUG_INSN_P (next
))
1245 new_ds
= ds_merge (new_ds
, ds
);
1247 else if (DEP_TYPE (dep
) == REG_DEP_CONTROL
)
1249 if (QUEUE_INDEX (pro
) != QUEUE_SCHEDULED
)
1254 DEP_STATUS (dep
) &= ~DEP_CANCELLED
;
1256 else if (DEP_REPLACE (dep
) != NULL
)
1258 if (QUEUE_INDEX (pro
) != QUEUE_SCHEDULED
)
1263 DEP_STATUS (dep
) &= ~DEP_CANCELLED
;
1267 if (n_replace
> 0 && n_control
== 0 && n_spec
== 0)
1269 if (!dbg_cnt (sched_breakdep
))
1271 FOR_EACH_DEP (next
, SD_LIST_BACK
, sd_it
, dep
)
1273 struct dep_replacement
*desc
= DEP_REPLACE (dep
);
1276 if (desc
->insn
== next
&& !for_backtrack
)
1278 gcc_assert (n_replace
== 1);
1279 apply_replacement (dep
, true);
1281 DEP_STATUS (dep
) |= DEP_CANCELLED
;
1287 else if (n_control
== 1 && n_replace
== 0 && n_spec
== 0)
1289 rtx_insn
*pro
, *other
;
1291 rtx cond
= NULL_RTX
;
1293 rtx_insn
*prev
= NULL
;
1297 if ((current_sched_info
->flags
& DO_PREDICATION
) == 0
1298 || (ORIG_PAT (next
) != NULL_RTX
1299 && PREDICATED_PAT (next
) == NULL_RTX
))
1302 pro
= DEP_PRO (modify_dep
);
1303 other
= real_insn_for_shadow (pro
);
1304 if (other
!= NULL_RTX
)
1307 cond
= sched_get_reverse_condition_uncached (pro
);
1308 regno
= REGNO (XEXP (cond
, 0));
1310 /* Find the last scheduled insn that modifies the condition register.
1311 We can stop looking once we find the insn we depend on through the
1312 REG_DEP_CONTROL; if the condition register isn't modified after it,
1313 we know that it still has the right value. */
1314 if (QUEUE_INDEX (pro
) == QUEUE_SCHEDULED
)
1315 FOR_EACH_VEC_ELT_REVERSE (scheduled_insns
, i
, prev
)
1319 find_all_hard_reg_sets (prev
, &t
, true);
1320 if (TEST_HARD_REG_BIT (t
, regno
))
1325 if (ORIG_PAT (next
) == NULL_RTX
)
1327 ORIG_PAT (next
) = PATTERN (next
);
1329 new_pat
= gen_rtx_COND_EXEC (VOIDmode
, cond
, PATTERN (next
));
1330 success
= haifa_change_pattern (next
, new_pat
);
1333 PREDICATED_PAT (next
) = new_pat
;
1335 else if (PATTERN (next
) != PREDICATED_PAT (next
))
1337 bool success
= haifa_change_pattern (next
,
1338 PREDICATED_PAT (next
));
1339 gcc_assert (success
);
1341 DEP_STATUS (modify_dep
) |= DEP_CANCELLED
;
1345 if (PREDICATED_PAT (next
) != NULL_RTX
)
1347 int tick
= INSN_TICK (next
);
1348 bool success
= haifa_change_pattern (next
,
1350 INSN_TICK (next
) = tick
;
1351 gcc_assert (success
);
1354 /* We can't handle the case where there are both speculative and control
1355 dependencies, so we return HARD_DEP in such a case. Also fail if
1356 we have speculative dependencies with not enough points, or more than
1357 one control dependency. */
1358 if ((n_spec
> 0 && (n_control
> 0 || n_replace
> 0))
1360 /* Too few points? */
1361 && ds_weak (new_ds
) < spec_info
->data_weakness_cutoff
)
1369 /* Pointer to the last instruction scheduled. */
1370 static rtx_insn
*last_scheduled_insn
;
1372 /* Pointer to the last nondebug instruction scheduled within the
1373 block, or the prev_head of the scheduling block. Used by
1374 rank_for_schedule, so that insns independent of the last scheduled
1375 insn will be preferred over dependent instructions. */
1376 static rtx_insn
*last_nondebug_scheduled_insn
;
1378 /* Pointer that iterates through the list of unscheduled insns if we
1379 have a dbg_cnt enabled. It always points at an insn prior to the
1380 first unscheduled one. */
1381 static rtx_insn
*nonscheduled_insns_begin
;
1383 /* Compute cost of executing INSN.
1384 This is the number of cycles between instruction issue and
1385 instruction results. */
1387 insn_cost (rtx_insn
*insn
)
1396 if (recog_memoized (insn
) < 0)
1399 cost
= insn_default_latency (insn
);
1406 cost
= INSN_COST (insn
);
1410 /* A USE insn, or something else we don't need to
1411 understand. We can't pass these directly to
1412 result_ready_cost or insn_default_latency because it will
1413 trigger a fatal error for unrecognizable insns. */
1414 if (recog_memoized (insn
) < 0)
1416 INSN_COST (insn
) = 0;
1421 cost
= insn_default_latency (insn
);
1425 INSN_COST (insn
) = cost
;
1432 /* Compute cost of dependence LINK.
1433 This is the number of cycles between instruction issue and
1434 instruction results.
1435 ??? We also use this function to call recog_memoized on all insns. */
1437 dep_cost_1 (dep_t link
, dw_t dw
)
1439 rtx_insn
*insn
= DEP_PRO (link
);
1440 rtx_insn
*used
= DEP_CON (link
);
1443 if (DEP_COST (link
) != UNKNOWN_DEP_COST
)
1444 return DEP_COST (link
);
1448 struct delay_pair
*delay_entry
;
1450 = delay_htab_i2
->find_with_hash (used
, htab_hash_pointer (used
));
1453 if (delay_entry
->i1
== insn
)
1455 DEP_COST (link
) = pair_delay (delay_entry
);
1456 return DEP_COST (link
);
1461 /* A USE insn should never require the value used to be computed.
1462 This allows the computation of a function's result and parameter
1463 values to overlap the return and call. We don't care about the
1464 dependence cost when only decreasing register pressure. */
1465 if (recog_memoized (used
) < 0)
1468 recog_memoized (insn
);
1472 enum reg_note dep_type
= DEP_TYPE (link
);
1474 cost
= insn_cost (insn
);
1476 if (INSN_CODE (insn
) >= 0)
1478 if (dep_type
== REG_DEP_ANTI
)
1480 else if (dep_type
== REG_DEP_OUTPUT
)
1482 cost
= (insn_default_latency (insn
)
1483 - insn_default_latency (used
));
1487 else if (bypass_p (insn
))
1488 cost
= insn_latency (insn
, used
);
1492 if (targetm
.sched
.adjust_cost_2
)
1493 cost
= targetm
.sched
.adjust_cost_2 (used
, (int) dep_type
, insn
, cost
,
1495 else if (targetm
.sched
.adjust_cost
!= NULL
)
1497 /* This variable is used for backward compatibility with the
1499 rtx_insn_list
*dep_cost_rtx_link
=
1500 alloc_INSN_LIST (NULL_RTX
, NULL
);
1502 /* Make it self-cycled, so that if some tries to walk over this
1503 incomplete list he/she will be caught in an endless loop. */
1504 XEXP (dep_cost_rtx_link
, 1) = dep_cost_rtx_link
;
1506 /* Targets use only REG_NOTE_KIND of the link. */
1507 PUT_REG_NOTE_KIND (dep_cost_rtx_link
, DEP_TYPE (link
));
1509 cost
= targetm
.sched
.adjust_cost (used
, dep_cost_rtx_link
,
1512 free_INSN_LIST_node (dep_cost_rtx_link
);
1519 DEP_COST (link
) = cost
;
1523 /* Compute cost of dependence LINK.
1524 This is the number of cycles between instruction issue and
1525 instruction results. */
1527 dep_cost (dep_t link
)
1529 return dep_cost_1 (link
, 0);
1532 /* Use this sel-sched.c friendly function in reorder2 instead of increasing
1533 INSN_PRIORITY explicitly. */
1535 increase_insn_priority (rtx_insn
*insn
, int amount
)
1537 if (!sel_sched_p ())
1539 /* We're dealing with haifa-sched.c INSN_PRIORITY. */
1540 if (INSN_PRIORITY_KNOWN (insn
))
1541 INSN_PRIORITY (insn
) += amount
;
1545 /* In sel-sched.c INSN_PRIORITY is not kept up to date.
1546 Use EXPR_PRIORITY instead. */
1547 sel_add_to_insn_priority (insn
, amount
);
1551 /* Return 'true' if DEP should be included in priority calculations. */
1553 contributes_to_priority_p (dep_t dep
)
1555 if (DEBUG_INSN_P (DEP_CON (dep
))
1556 || DEBUG_INSN_P (DEP_PRO (dep
)))
1559 /* Critical path is meaningful in block boundaries only. */
1560 if (!current_sched_info
->contributes_to_priority (DEP_CON (dep
),
1564 if (DEP_REPLACE (dep
) != NULL
)
1567 /* If flag COUNT_SPEC_IN_CRITICAL_PATH is set,
1568 then speculative instructions will less likely be
1569 scheduled. That is because the priority of
1570 their producers will increase, and, thus, the
1571 producers will more likely be scheduled, thus,
1572 resolving the dependence. */
1573 if (sched_deps_info
->generate_spec_deps
1574 && !(spec_info
->flags
& COUNT_SPEC_IN_CRITICAL_PATH
)
1575 && (DEP_STATUS (dep
) & SPECULATIVE
))
1581 /* Compute the number of nondebug deps in list LIST for INSN. */
1584 dep_list_size (rtx_insn
*insn
, sd_list_types_def list
)
1586 sd_iterator_def sd_it
;
1588 int dbgcount
= 0, nodbgcount
= 0;
1590 if (!MAY_HAVE_DEBUG_INSNS
)
1591 return sd_lists_size (insn
, list
);
1593 FOR_EACH_DEP (insn
, list
, sd_it
, dep
)
1595 if (DEBUG_INSN_P (DEP_CON (dep
)))
1597 else if (!DEBUG_INSN_P (DEP_PRO (dep
)))
1601 gcc_assert (dbgcount
+ nodbgcount
== sd_lists_size (insn
, list
));
1608 /* Compute the priority number for INSN. */
1610 priority (rtx_insn
*insn
)
1612 if (! INSN_P (insn
))
1615 /* We should not be interested in priority of an already scheduled insn. */
1616 gcc_assert (QUEUE_INDEX (insn
) != QUEUE_SCHEDULED
);
1618 if (!INSN_PRIORITY_KNOWN (insn
))
1620 int this_priority
= -1;
1624 int this_fusion_priority
;
1626 targetm
.sched
.fusion_priority (insn
, FUSION_MAX_PRIORITY
,
1627 &this_fusion_priority
, &this_priority
);
1628 INSN_FUSION_PRIORITY (insn
) = this_fusion_priority
;
1630 else if (dep_list_size (insn
, SD_LIST_FORW
) == 0)
1631 /* ??? We should set INSN_PRIORITY to insn_cost when and insn has
1632 some forward deps but all of them are ignored by
1633 contributes_to_priority hook. At the moment we set priority of
1635 this_priority
= insn_cost (insn
);
1638 rtx_insn
*prev_first
, *twin
;
1641 /* For recovery check instructions we calculate priority slightly
1642 different than that of normal instructions. Instead of walking
1643 through INSN_FORW_DEPS (check) list, we walk through
1644 INSN_FORW_DEPS list of each instruction in the corresponding
1647 /* Selective scheduling does not define RECOVERY_BLOCK macro. */
1648 rec
= sel_sched_p () ? NULL
: RECOVERY_BLOCK (insn
);
1649 if (!rec
|| rec
== EXIT_BLOCK_PTR_FOR_FN (cfun
))
1651 prev_first
= PREV_INSN (insn
);
1656 prev_first
= NEXT_INSN (BB_HEAD (rec
));
1657 twin
= PREV_INSN (BB_END (rec
));
1662 sd_iterator_def sd_it
;
1665 FOR_EACH_DEP (twin
, SD_LIST_FORW
, sd_it
, dep
)
1670 next
= DEP_CON (dep
);
1672 if (BLOCK_FOR_INSN (next
) != rec
)
1676 if (!contributes_to_priority_p (dep
))
1680 cost
= dep_cost (dep
);
1683 struct _dep _dep1
, *dep1
= &_dep1
;
1685 init_dep (dep1
, insn
, next
, REG_DEP_ANTI
);
1687 cost
= dep_cost (dep1
);
1690 next_priority
= cost
+ priority (next
);
1692 if (next_priority
> this_priority
)
1693 this_priority
= next_priority
;
1697 twin
= PREV_INSN (twin
);
1699 while (twin
!= prev_first
);
1702 if (this_priority
< 0)
1704 gcc_assert (this_priority
== -1);
1706 this_priority
= insn_cost (insn
);
1709 INSN_PRIORITY (insn
) = this_priority
;
1710 INSN_PRIORITY_STATUS (insn
) = 1;
1713 return INSN_PRIORITY (insn
);
1716 /* Macros and functions for keeping the priority queue sorted, and
1717 dealing with queuing and dequeuing of instructions. */
1719 /* For each pressure class CL, set DEATH[CL] to the number of registers
1720 in that class that die in INSN. */
1723 calculate_reg_deaths (rtx_insn
*insn
, int *death
)
1726 struct reg_use_data
*use
;
1728 for (i
= 0; i
< ira_pressure_classes_num
; i
++)
1729 death
[ira_pressure_classes
[i
]] = 0;
1730 for (use
= INSN_REG_USE_LIST (insn
); use
!= NULL
; use
= use
->next_insn_use
)
1731 if (dying_use_p (use
))
1732 mark_regno_birth_or_death (0, death
, use
->regno
, true);
1735 /* Setup info about the current register pressure impact of scheduling
1736 INSN at the current scheduling point. */
1738 setup_insn_reg_pressure_info (rtx_insn
*insn
)
1740 int i
, change
, before
, after
, hard_regno
;
1741 int excess_cost_change
;
1744 struct reg_pressure_data
*pressure_info
;
1745 int *max_reg_pressure
;
1746 static int death
[N_REG_CLASSES
];
1748 gcc_checking_assert (!DEBUG_INSN_P (insn
));
1750 excess_cost_change
= 0;
1751 calculate_reg_deaths (insn
, death
);
1752 pressure_info
= INSN_REG_PRESSURE (insn
);
1753 max_reg_pressure
= INSN_MAX_REG_PRESSURE (insn
);
1754 gcc_assert (pressure_info
!= NULL
&& max_reg_pressure
!= NULL
);
1755 for (i
= 0; i
< ira_pressure_classes_num
; i
++)
1757 cl
= ira_pressure_classes
[i
];
1758 gcc_assert (curr_reg_pressure
[cl
] >= 0);
1759 change
= (int) pressure_info
[i
].set_increase
- death
[cl
];
1760 before
= MAX (0, max_reg_pressure
[i
] - sched_class_regs_num
[cl
]);
1761 after
= MAX (0, max_reg_pressure
[i
] + change
1762 - sched_class_regs_num
[cl
]);
1763 hard_regno
= ira_class_hard_regs
[cl
][0];
1764 gcc_assert (hard_regno
>= 0);
1765 mode
= reg_raw_mode
[hard_regno
];
1766 excess_cost_change
+= ((after
- before
)
1767 * (ira_memory_move_cost
[mode
][cl
][0]
1768 + ira_memory_move_cost
[mode
][cl
][1]));
1770 INSN_REG_PRESSURE_EXCESS_COST_CHANGE (insn
) = excess_cost_change
;
1773 /* This is the first page of code related to SCHED_PRESSURE_MODEL.
1774 It tries to make the scheduler take register pressure into account
1775 without introducing too many unnecessary stalls. It hooks into the
1776 main scheduling algorithm at several points:
1778 - Before scheduling starts, model_start_schedule constructs a
1779 "model schedule" for the current block. This model schedule is
1780 chosen solely to keep register pressure down. It does not take the
1781 target's pipeline or the original instruction order into account,
1782 except as a tie-breaker. It also doesn't work to a particular
1785 This model schedule gives us an idea of what pressure can be
1786 achieved for the block and gives us an example of a schedule that
1787 keeps to that pressure. It also makes the final schedule less
1788 dependent on the original instruction order. This is important
1789 because the original order can either be "wide" (many values live
1790 at once, such as in user-scheduled code) or "narrow" (few values
1791 live at once, such as after loop unrolling, where several
1792 iterations are executed sequentially).
1794 We do not apply this model schedule to the rtx stream. We simply
1795 record it in model_schedule. We also compute the maximum pressure,
1796 MP, that was seen during this schedule.
1798 - Instructions are added to the ready queue even if they require
1799 a stall. The length of the stall is instead computed as:
1801 MAX (INSN_TICK (INSN) - clock_var, 0)
1803 (= insn_delay). This allows rank_for_schedule to choose between
1804 introducing a deliberate stall or increasing pressure.
1806 - Before sorting the ready queue, model_set_excess_costs assigns
1807 a pressure-based cost to each ready instruction in the queue.
1808 This is the instruction's INSN_REG_PRESSURE_EXCESS_COST_CHANGE
1809 (ECC for short) and is effectively measured in cycles.
1811 - rank_for_schedule ranks instructions based on:
1813 ECC (insn) + insn_delay (insn)
1819 So, for example, an instruction X1 with an ECC of 1 that can issue
1820 now will win over an instruction X0 with an ECC of zero that would
1821 introduce a stall of one cycle. However, an instruction X2 with an
1822 ECC of 2 that can issue now will lose to both X0 and X1.
1824 - When an instruction is scheduled, model_recompute updates the model
1825 schedule with the new pressures (some of which might now exceed the
1826 original maximum pressure MP). model_update_limit_points then searches
1827 for the new point of maximum pressure, if not already known. */
1829 /* Used to separate high-verbosity debug information for SCHED_PRESSURE_MODEL
1830 from surrounding debug information. */
1832 ";;\t\t+------------------------------------------------------\n"
1834 /* Information about the pressure on a particular register class at a
1835 particular point of the model schedule. */
1836 struct model_pressure_data
{
1837 /* The pressure at this point of the model schedule, or -1 if the
1838 point is associated with an instruction that has already been
1842 /* The maximum pressure during or after this point of the model schedule. */
1846 /* Per-instruction information that is used while building the model
1847 schedule. Here, "schedule" refers to the model schedule rather
1848 than the main schedule. */
1849 struct model_insn_info
{
1850 /* The instruction itself. */
1853 /* If this instruction is in model_worklist, these fields link to the
1854 previous (higher-priority) and next (lower-priority) instructions
1856 struct model_insn_info
*prev
;
1857 struct model_insn_info
*next
;
1859 /* While constructing the schedule, QUEUE_INDEX describes whether an
1860 instruction has already been added to the schedule (QUEUE_SCHEDULED),
1861 is in model_worklist (QUEUE_READY), or neither (QUEUE_NOWHERE).
1862 old_queue records the value that QUEUE_INDEX had before scheduling
1863 started, so that we can restore it once the schedule is complete. */
1866 /* The relative importance of an unscheduled instruction. Higher
1867 values indicate greater importance. */
1868 unsigned int model_priority
;
1870 /* The length of the longest path of satisfied true dependencies
1871 that leads to this instruction. */
1874 /* The length of the longest path of dependencies of any kind
1875 that leads from this instruction. */
1878 /* The number of predecessor nodes that must still be scheduled. */
1879 int unscheduled_preds
;
1882 /* Information about the pressure limit for a particular register class.
1883 This structure is used when applying a model schedule to the main
1885 struct model_pressure_limit
{
1886 /* The maximum register pressure seen in the original model schedule. */
1889 /* The maximum register pressure seen in the current model schedule
1890 (which excludes instructions that have already been scheduled). */
1893 /* The point of the current model schedule at which PRESSURE is first
1894 reached. It is set to -1 if the value needs to be recomputed. */
1898 /* Describes a particular way of measuring register pressure. */
1899 struct model_pressure_group
{
1900 /* Index PCI describes the maximum pressure on ira_pressure_classes[PCI]. */
1901 struct model_pressure_limit limits
[N_REG_CLASSES
];
1903 /* Index (POINT * ira_num_pressure_classes + PCI) describes the pressure
1904 on register class ira_pressure_classes[PCI] at point POINT of the
1905 current model schedule. A POINT of model_num_insns describes the
1906 pressure at the end of the schedule. */
1907 struct model_pressure_data
*model
;
1910 /* Index POINT gives the instruction at point POINT of the model schedule.
1911 This array doesn't change during main scheduling. */
1912 static vec
<rtx_insn
*> model_schedule
;
1914 /* The list of instructions in the model worklist, sorted in order of
1915 decreasing priority. */
1916 static struct model_insn_info
*model_worklist
;
1918 /* Index I describes the instruction with INSN_LUID I. */
1919 static struct model_insn_info
*model_insns
;
1921 /* The number of instructions in the model schedule. */
1922 static int model_num_insns
;
1924 /* The index of the first instruction in model_schedule that hasn't yet been
1925 added to the main schedule, or model_num_insns if all of them have. */
1926 static int model_curr_point
;
1928 /* Describes the pressure before each instruction in the model schedule. */
1929 static struct model_pressure_group model_before_pressure
;
1931 /* The first unused model_priority value (as used in model_insn_info). */
1932 static unsigned int model_next_priority
;
1935 /* The model_pressure_data for ira_pressure_classes[PCI] in GROUP
1936 at point POINT of the model schedule. */
1937 #define MODEL_PRESSURE_DATA(GROUP, POINT, PCI) \
1938 (&(GROUP)->model[(POINT) * ira_pressure_classes_num + (PCI)])
1940 /* The maximum pressure on ira_pressure_classes[PCI] in GROUP at or
1941 after point POINT of the model schedule. */
1942 #define MODEL_MAX_PRESSURE(GROUP, POINT, PCI) \
1943 (MODEL_PRESSURE_DATA (GROUP, POINT, PCI)->max_pressure)
1945 /* The pressure on ira_pressure_classes[PCI] in GROUP at point POINT
1946 of the model schedule. */
1947 #define MODEL_REF_PRESSURE(GROUP, POINT, PCI) \
1948 (MODEL_PRESSURE_DATA (GROUP, POINT, PCI)->ref_pressure)
1950 /* Information about INSN that is used when creating the model schedule. */
1951 #define MODEL_INSN_INFO(INSN) \
1952 (&model_insns[INSN_LUID (INSN)])
1954 /* The instruction at point POINT of the model schedule. */
1955 #define MODEL_INSN(POINT) \
1956 (model_schedule[POINT])
1959 /* Return INSN's index in the model schedule, or model_num_insns if it
1960 doesn't belong to that schedule. */
1963 model_index (rtx_insn
*insn
)
1965 if (INSN_MODEL_INDEX (insn
) == 0)
1966 return model_num_insns
;
1967 return INSN_MODEL_INDEX (insn
) - 1;
1970 /* Make sure that GROUP->limits is up-to-date for the current point
1971 of the model schedule. */
1974 model_update_limit_points_in_group (struct model_pressure_group
*group
)
1976 int pci
, max_pressure
, point
;
1978 for (pci
= 0; pci
< ira_pressure_classes_num
; pci
++)
1980 /* We may have passed the final point at which the pressure in
1981 group->limits[pci].pressure was reached. Update the limit if so. */
1982 max_pressure
= MODEL_MAX_PRESSURE (group
, model_curr_point
, pci
);
1983 group
->limits
[pci
].pressure
= max_pressure
;
1985 /* Find the point at which MAX_PRESSURE is first reached. We need
1986 to search in three cases:
1988 - We've already moved past the previous pressure point.
1989 In this case we search forward from model_curr_point.
1991 - We scheduled the previous point of maximum pressure ahead of
1992 its position in the model schedule, but doing so didn't bring
1993 the pressure point earlier. In this case we search forward
1994 from that previous pressure point.
1996 - Scheduling an instruction early caused the maximum pressure
1997 to decrease. In this case we will have set the pressure
1998 point to -1, and we search forward from model_curr_point. */
1999 point
= MAX (group
->limits
[pci
].point
, model_curr_point
);
2000 while (point
< model_num_insns
2001 && MODEL_REF_PRESSURE (group
, point
, pci
) < max_pressure
)
2003 group
->limits
[pci
].point
= point
;
2005 gcc_assert (MODEL_REF_PRESSURE (group
, point
, pci
) == max_pressure
);
2006 gcc_assert (MODEL_MAX_PRESSURE (group
, point
, pci
) == max_pressure
);
2010 /* Make sure that all register-pressure limits are up-to-date for the
2011 current position in the model schedule. */
2014 model_update_limit_points (void)
2016 model_update_limit_points_in_group (&model_before_pressure
);
2019 /* Return the model_index of the last unscheduled use in chain USE
2020 outside of USE's instruction. Return -1 if there are no other uses,
2021 or model_num_insns if the register is live at the end of the block. */
2024 model_last_use_except (struct reg_use_data
*use
)
2026 struct reg_use_data
*next
;
2030 for (next
= use
->next_regno_use
; next
!= use
; next
= next
->next_regno_use
)
2031 if (NONDEBUG_INSN_P (next
->insn
)
2032 && QUEUE_INDEX (next
->insn
) != QUEUE_SCHEDULED
)
2034 index
= model_index (next
->insn
);
2035 if (index
== model_num_insns
)
2036 return model_num_insns
;
2043 /* An instruction with model_index POINT has just been scheduled, and it
2044 adds DELTA to the pressure on ira_pressure_classes[PCI] after POINT - 1.
2045 Update MODEL_REF_PRESSURE (GROUP, POINT, PCI) and
2046 MODEL_MAX_PRESSURE (GROUP, POINT, PCI) accordingly. */
2049 model_start_update_pressure (struct model_pressure_group
*group
,
2050 int point
, int pci
, int delta
)
2052 int next_max_pressure
;
2054 if (point
== model_num_insns
)
2056 /* The instruction wasn't part of the model schedule; it was moved
2057 from a different block. Update the pressure for the end of
2058 the model schedule. */
2059 MODEL_REF_PRESSURE (group
, point
, pci
) += delta
;
2060 MODEL_MAX_PRESSURE (group
, point
, pci
) += delta
;
2064 /* Record that this instruction has been scheduled. Nothing now
2065 changes between POINT and POINT + 1, so get the maximum pressure
2066 from the latter. If the maximum pressure decreases, the new
2067 pressure point may be before POINT. */
2068 MODEL_REF_PRESSURE (group
, point
, pci
) = -1;
2069 next_max_pressure
= MODEL_MAX_PRESSURE (group
, point
+ 1, pci
);
2070 if (MODEL_MAX_PRESSURE (group
, point
, pci
) > next_max_pressure
)
2072 MODEL_MAX_PRESSURE (group
, point
, pci
) = next_max_pressure
;
2073 if (group
->limits
[pci
].point
== point
)
2074 group
->limits
[pci
].point
= -1;
2079 /* Record that scheduling a later instruction has changed the pressure
2080 at point POINT of the model schedule by DELTA (which might be 0).
2081 Update GROUP accordingly. Return nonzero if these changes might
2082 trigger changes to previous points as well. */
2085 model_update_pressure (struct model_pressure_group
*group
,
2086 int point
, int pci
, int delta
)
2088 int ref_pressure
, max_pressure
, next_max_pressure
;
2090 /* If POINT hasn't yet been scheduled, update its pressure. */
2091 ref_pressure
= MODEL_REF_PRESSURE (group
, point
, pci
);
2092 if (ref_pressure
>= 0 && delta
!= 0)
2094 ref_pressure
+= delta
;
2095 MODEL_REF_PRESSURE (group
, point
, pci
) = ref_pressure
;
2097 /* Check whether the maximum pressure in the overall schedule
2098 has increased. (This means that the MODEL_MAX_PRESSURE of
2099 every point <= POINT will need to increase too; see below.) */
2100 if (group
->limits
[pci
].pressure
< ref_pressure
)
2101 group
->limits
[pci
].pressure
= ref_pressure
;
2103 /* If we are at maximum pressure, and the maximum pressure
2104 point was previously unknown or later than POINT,
2105 bring it forward. */
2106 if (group
->limits
[pci
].pressure
== ref_pressure
2107 && !IN_RANGE (group
->limits
[pci
].point
, 0, point
))
2108 group
->limits
[pci
].point
= point
;
2110 /* If POINT used to be the point of maximum pressure, but isn't
2111 any longer, we need to recalculate it using a forward walk. */
2112 if (group
->limits
[pci
].pressure
> ref_pressure
2113 && group
->limits
[pci
].point
== point
)
2114 group
->limits
[pci
].point
= -1;
2117 /* Update the maximum pressure at POINT. Changes here might also
2118 affect the maximum pressure at POINT - 1. */
2119 next_max_pressure
= MODEL_MAX_PRESSURE (group
, point
+ 1, pci
);
2120 max_pressure
= MAX (ref_pressure
, next_max_pressure
);
2121 if (MODEL_MAX_PRESSURE (group
, point
, pci
) != max_pressure
)
2123 MODEL_MAX_PRESSURE (group
, point
, pci
) = max_pressure
;
2129 /* INSN has just been scheduled. Update the model schedule accordingly. */
2132 model_recompute (rtx_insn
*insn
)
2137 } uses
[FIRST_PSEUDO_REGISTER
+ MAX_RECOG_OPERANDS
];
2138 struct reg_use_data
*use
;
2139 struct reg_pressure_data
*reg_pressure
;
2140 int delta
[N_REG_CLASSES
];
2141 int pci
, point
, mix
, new_last
, cl
, ref_pressure
, queue
;
2142 unsigned int i
, num_uses
, num_pending_births
;
2145 /* The destinations of INSN were previously live from POINT onwards, but are
2146 now live from model_curr_point onwards. Set up DELTA accordingly. */
2147 point
= model_index (insn
);
2148 reg_pressure
= INSN_REG_PRESSURE (insn
);
2149 for (pci
= 0; pci
< ira_pressure_classes_num
; pci
++)
2151 cl
= ira_pressure_classes
[pci
];
2152 delta
[cl
] = reg_pressure
[pci
].set_increase
;
2155 /* Record which registers previously died at POINT, but which now die
2156 before POINT. Adjust DELTA so that it represents the effect of
2157 this change after POINT - 1. Set NUM_PENDING_BIRTHS to the number of
2158 registers that will be born in the range [model_curr_point, POINT). */
2160 num_pending_births
= 0;
2161 for (use
= INSN_REG_USE_LIST (insn
); use
!= NULL
; use
= use
->next_insn_use
)
2163 new_last
= model_last_use_except (use
);
2164 if (new_last
< point
)
2166 gcc_assert (num_uses
< ARRAY_SIZE (uses
));
2167 uses
[num_uses
].last_use
= new_last
;
2168 uses
[num_uses
].regno
= use
->regno
;
2169 /* This register is no longer live after POINT - 1. */
2170 mark_regno_birth_or_death (NULL
, delta
, use
->regno
, false);
2173 num_pending_births
++;
2177 /* Update the MODEL_REF_PRESSURE and MODEL_MAX_PRESSURE for POINT.
2178 Also set each group pressure limit for POINT. */
2179 for (pci
= 0; pci
< ira_pressure_classes_num
; pci
++)
2181 cl
= ira_pressure_classes
[pci
];
2182 model_start_update_pressure (&model_before_pressure
,
2183 point
, pci
, delta
[cl
]);
2186 /* Walk the model schedule backwards, starting immediately before POINT. */
2188 if (point
!= model_curr_point
)
2192 insn
= MODEL_INSN (point
);
2193 queue
= QUEUE_INDEX (insn
);
2195 if (queue
!= QUEUE_SCHEDULED
)
2197 /* DELTA describes the effect of the move on the register pressure
2198 after POINT. Make it describe the effect on the pressure
2201 while (i
< num_uses
)
2203 if (uses
[i
].last_use
== point
)
2205 /* This register is now live again. */
2206 mark_regno_birth_or_death (NULL
, delta
,
2207 uses
[i
].regno
, true);
2209 /* Remove this use from the array. */
2210 uses
[i
] = uses
[num_uses
- 1];
2212 num_pending_births
--;
2218 if (sched_verbose
>= 5)
2222 fprintf (sched_dump
, MODEL_BAR
);
2223 fprintf (sched_dump
, ";;\t\t| New pressure for model"
2225 fprintf (sched_dump
, MODEL_BAR
);
2229 fprintf (sched_dump
, ";;\t\t| %3d %4d %-30s ",
2230 point
, INSN_UID (insn
),
2231 str_pattern_slim (PATTERN (insn
)));
2232 for (pci
= 0; pci
< ira_pressure_classes_num
; pci
++)
2234 cl
= ira_pressure_classes
[pci
];
2235 ref_pressure
= MODEL_REF_PRESSURE (&model_before_pressure
,
2237 fprintf (sched_dump
, " %s:[%d->%d]",
2238 reg_class_names
[ira_pressure_classes
[pci
]],
2239 ref_pressure
, ref_pressure
+ delta
[cl
]);
2241 fprintf (sched_dump
, "\n");
2245 /* Adjust the pressure at POINT. Set MIX to nonzero if POINT - 1
2246 might have changed as well. */
2247 mix
= num_pending_births
;
2248 for (pci
= 0; pci
< ira_pressure_classes_num
; pci
++)
2250 cl
= ira_pressure_classes
[pci
];
2252 mix
|= model_update_pressure (&model_before_pressure
,
2253 point
, pci
, delta
[cl
]);
2256 while (mix
&& point
> model_curr_point
);
2259 fprintf (sched_dump
, MODEL_BAR
);
2262 /* After DEP, which was cancelled, has been resolved for insn NEXT,
2263 check whether the insn's pattern needs restoring. */
2265 must_restore_pattern_p (rtx_insn
*next
, dep_t dep
)
2267 if (QUEUE_INDEX (next
) == QUEUE_SCHEDULED
)
2270 if (DEP_TYPE (dep
) == REG_DEP_CONTROL
)
2272 gcc_assert (ORIG_PAT (next
) != NULL_RTX
);
2273 gcc_assert (next
== DEP_CON (dep
));
2277 struct dep_replacement
*desc
= DEP_REPLACE (dep
);
2278 if (desc
->insn
!= next
)
2280 gcc_assert (*desc
->loc
== desc
->orig
);
2287 /* model_spill_cost (CL, P, P') returns the cost of increasing the
2288 pressure on CL from P to P'. We use this to calculate a "base ECC",
2289 baseECC (CL, X), for each pressure class CL and each instruction X.
2290 Supposing X changes the pressure on CL from P to P', and that the
2291 maximum pressure on CL in the current model schedule is MP', then:
2293 * if X occurs before or at the next point of maximum pressure in
2294 the model schedule and P' > MP', then:
2296 baseECC (CL, X) = model_spill_cost (CL, MP, P')
2298 The idea is that the pressure after scheduling a fixed set of
2299 instructions -- in this case, the set up to and including the
2300 next maximum pressure point -- is going to be the same regardless
2301 of the order; we simply want to keep the intermediate pressure
2302 under control. Thus X has a cost of zero unless scheduling it
2303 now would exceed MP'.
2305 If all increases in the set are by the same amount, no zero-cost
2306 instruction will ever cause the pressure to exceed MP'. However,
2307 if X is instead moved past an instruction X' with pressure in the
2308 range (MP' - (P' - P), MP'), the pressure at X' will increase
2309 beyond MP'. Since baseECC is very much a heuristic anyway,
2310 it doesn't seem worth the overhead of tracking cases like these.
2312 The cost of exceeding MP' is always based on the original maximum
2313 pressure MP. This is so that going 2 registers over the original
2314 limit has the same cost regardless of whether it comes from two
2315 separate +1 deltas or from a single +2 delta.
2317 * if X occurs after the next point of maximum pressure in the model
2318 schedule and P' > P, then:
2320 baseECC (CL, X) = model_spill_cost (CL, MP, MP' + (P' - P))
2322 That is, if we move X forward across a point of maximum pressure,
2323 and if X increases the pressure by P' - P, then we conservatively
2324 assume that scheduling X next would increase the maximum pressure
2325 by P' - P. Again, the cost of doing this is based on the original
2326 maximum pressure MP, for the same reason as above.
2328 * if P' < P, P > MP, and X occurs at or after the next point of
2329 maximum pressure, then:
2331 baseECC (CL, X) = -model_spill_cost (CL, MAX (MP, P'), P)
2333 That is, if we have already exceeded the original maximum pressure MP,
2334 and if X might reduce the maximum pressure again -- or at least push
2335 it further back, and thus allow more scheduling freedom -- it is given
2336 a negative cost to reflect the improvement.
2342 In this case, X is not expected to affect the maximum pressure MP',
2343 so it has zero cost.
2345 We then create a combined value baseECC (X) that is the sum of
2346 baseECC (CL, X) for each pressure class CL.
2348 baseECC (X) could itself be used as the ECC value described above.
2349 However, this is often too conservative, in the sense that it
2350 tends to make high-priority instructions that increase pressure
2351 wait too long in cases where introducing a spill would be better.
2352 For this reason the final ECC is a priority-adjusted form of
2353 baseECC (X). Specifically, we calculate:
2355 P (X) = INSN_PRIORITY (X) - insn_delay (X) - baseECC (X)
2356 baseP = MAX { P (X) | baseECC (X) <= 0 }
2360 ECC (X) = MAX (MIN (baseP - P (X), baseECC (X)), 0)
2362 Thus an instruction's effect on pressure is ignored if it has a high
2363 enough priority relative to the ones that don't increase pressure.
2364 Negative values of baseECC (X) do not increase the priority of X
2365 itself, but they do make it harder for other instructions to
2366 increase the pressure further.
2368 This pressure cost is deliberately timid. The intention has been
2369 to choose a heuristic that rarely interferes with the normal list
2370 scheduler in cases where that scheduler would produce good code.
2371 We simply want to curb some of its worst excesses. */
2373 /* Return the cost of increasing the pressure in class CL from FROM to TO.
2375 Here we use the very simplistic cost model that every register above
2376 sched_class_regs_num[CL] has a spill cost of 1. We could use other
2377 measures instead, such as one based on MEMORY_MOVE_COST. However:
2379 (1) In order for an instruction to be scheduled, the higher cost
2380 would need to be justified in a single saving of that many stalls.
2381 This is overly pessimistic, because the benefit of spilling is
2382 often to avoid a sequence of several short stalls rather than
2385 (2) The cost is still arbitrary. Because we are not allocating
2386 registers during scheduling, we have no way of knowing for
2387 sure how many memory accesses will be required by each spill,
2388 where the spills will be placed within the block, or even
2389 which block(s) will contain the spills.
2391 So a higher cost than 1 is often too conservative in practice,
2392 forcing blocks to contain unnecessary stalls instead of spill code.
2393 The simple cost below seems to be the best compromise. It reduces
2394 the interference with the normal list scheduler, which helps make
2395 it more suitable for a default-on option. */
2398 model_spill_cost (int cl
, int from
, int to
)
2400 from
= MAX (from
, sched_class_regs_num
[cl
]);
2401 return MAX (to
, from
) - from
;
2404 /* Return baseECC (ira_pressure_classes[PCI], POINT), given that
2405 P = curr_reg_pressure[ira_pressure_classes[PCI]] and that
2409 model_excess_group_cost (struct model_pressure_group
*group
,
2410 int point
, int pci
, int delta
)
2414 cl
= ira_pressure_classes
[pci
];
2415 if (delta
< 0 && point
>= group
->limits
[pci
].point
)
2417 pressure
= MAX (group
->limits
[pci
].orig_pressure
,
2418 curr_reg_pressure
[cl
] + delta
);
2419 return -model_spill_cost (cl
, pressure
, curr_reg_pressure
[cl
]);
2424 if (point
> group
->limits
[pci
].point
)
2425 pressure
= group
->limits
[pci
].pressure
+ delta
;
2427 pressure
= curr_reg_pressure
[cl
] + delta
;
2429 if (pressure
> group
->limits
[pci
].pressure
)
2430 return model_spill_cost (cl
, group
->limits
[pci
].orig_pressure
,
2437 /* Return baseECC (MODEL_INSN (INSN)). Dump the costs to sched_dump
2441 model_excess_cost (rtx_insn
*insn
, bool print_p
)
2443 int point
, pci
, cl
, cost
, this_cost
, delta
;
2444 struct reg_pressure_data
*insn_reg_pressure
;
2445 int insn_death
[N_REG_CLASSES
];
2447 calculate_reg_deaths (insn
, insn_death
);
2448 point
= model_index (insn
);
2449 insn_reg_pressure
= INSN_REG_PRESSURE (insn
);
2453 fprintf (sched_dump
, ";;\t\t| %3d %4d | %4d %+3d |", point
,
2454 INSN_UID (insn
), INSN_PRIORITY (insn
), insn_delay (insn
));
2456 /* Sum up the individual costs for each register class. */
2457 for (pci
= 0; pci
< ira_pressure_classes_num
; pci
++)
2459 cl
= ira_pressure_classes
[pci
];
2460 delta
= insn_reg_pressure
[pci
].set_increase
- insn_death
[cl
];
2461 this_cost
= model_excess_group_cost (&model_before_pressure
,
2465 fprintf (sched_dump
, " %s:[%d base cost %d]",
2466 reg_class_names
[cl
], delta
, this_cost
);
2470 fprintf (sched_dump
, "\n");
2475 /* Dump the next points of maximum pressure for GROUP. */
2478 model_dump_pressure_points (struct model_pressure_group
*group
)
2482 fprintf (sched_dump
, ";;\t\t| pressure points");
2483 for (pci
= 0; pci
< ira_pressure_classes_num
; pci
++)
2485 cl
= ira_pressure_classes
[pci
];
2486 fprintf (sched_dump
, " %s:[%d->%d at ", reg_class_names
[cl
],
2487 curr_reg_pressure
[cl
], group
->limits
[pci
].pressure
);
2488 if (group
->limits
[pci
].point
< model_num_insns
)
2489 fprintf (sched_dump
, "%d:%d]", group
->limits
[pci
].point
,
2490 INSN_UID (MODEL_INSN (group
->limits
[pci
].point
)));
2492 fprintf (sched_dump
, "end]");
2494 fprintf (sched_dump
, "\n");
2497 /* Set INSN_REG_PRESSURE_EXCESS_COST_CHANGE for INSNS[0...COUNT-1]. */
2500 model_set_excess_costs (rtx_insn
**insns
, int count
)
2502 int i
, cost
, priority_base
, priority
;
2505 /* Record the baseECC value for each instruction in the model schedule,
2506 except that negative costs are converted to zero ones now rather than
2507 later. Do not assign a cost to debug instructions, since they must
2508 not change code-generation decisions. Experiments suggest we also
2509 get better results by not assigning a cost to instructions from
2512 Set PRIORITY_BASE to baseP in the block comment above. This is the
2513 maximum priority of the "cheap" instructions, which should always
2514 include the next model instruction. */
2517 for (i
= 0; i
< count
; i
++)
2518 if (INSN_MODEL_INDEX (insns
[i
]))
2520 if (sched_verbose
>= 6 && !print_p
)
2522 fprintf (sched_dump
, MODEL_BAR
);
2523 fprintf (sched_dump
, ";;\t\t| Pressure costs for ready queue\n");
2524 model_dump_pressure_points (&model_before_pressure
);
2525 fprintf (sched_dump
, MODEL_BAR
);
2528 cost
= model_excess_cost (insns
[i
], print_p
);
2531 priority
= INSN_PRIORITY (insns
[i
]) - insn_delay (insns
[i
]) - cost
;
2532 priority_base
= MAX (priority_base
, priority
);
2535 INSN_REG_PRESSURE_EXCESS_COST_CHANGE (insns
[i
]) = cost
;
2538 fprintf (sched_dump
, MODEL_BAR
);
2540 /* Use MAX (baseECC, 0) and baseP to calculcate ECC for each
2542 for (i
= 0; i
< count
; i
++)
2544 cost
= INSN_REG_PRESSURE_EXCESS_COST_CHANGE (insns
[i
]);
2545 priority
= INSN_PRIORITY (insns
[i
]) - insn_delay (insns
[i
]);
2546 if (cost
> 0 && priority
> priority_base
)
2548 cost
+= priority_base
- priority
;
2549 INSN_REG_PRESSURE_EXCESS_COST_CHANGE (insns
[i
]) = MAX (cost
, 0);
2555 /* Enum of rank_for_schedule heuristic decisions. */
2557 RFS_LIVE_RANGE_SHRINK1
, RFS_LIVE_RANGE_SHRINK2
,
2558 RFS_SCHED_GROUP
, RFS_PRESSURE_DELAY
, RFS_PRESSURE_TICK
,
2559 RFS_FEEDS_BACKTRACK_INSN
, RFS_PRIORITY
, RFS_SPECULATION
,
2560 RFS_SCHED_RANK
, RFS_LAST_INSN
, RFS_PRESSURE_INDEX
,
2561 RFS_DEP_COUNT
, RFS_TIE
, RFS_FUSION
, RFS_N
};
2563 /* Corresponding strings for print outs. */
2564 static const char *rfs_str
[RFS_N
] = {
2565 "RFS_LIVE_RANGE_SHRINK1", "RFS_LIVE_RANGE_SHRINK2",
2566 "RFS_SCHED_GROUP", "RFS_PRESSURE_DELAY", "RFS_PRESSURE_TICK",
2567 "RFS_FEEDS_BACKTRACK_INSN", "RFS_PRIORITY", "RFS_SPECULATION",
2568 "RFS_SCHED_RANK", "RFS_LAST_INSN", "RFS_PRESSURE_INDEX",
2569 "RFS_DEP_COUNT", "RFS_TIE", "RFS_FUSION" };
2571 /* Statistical breakdown of rank_for_schedule decisions. */
2572 struct rank_for_schedule_stats_t
{ unsigned stats
[RFS_N
]; };
2573 static rank_for_schedule_stats_t rank_for_schedule_stats
;
2575 /* Return the result of comparing insns TMP and TMP2 and update
2576 Rank_For_Schedule statistics. */
2578 rfs_result (enum rfs_decision decision
, int result
, rtx tmp
, rtx tmp2
)
2580 ++rank_for_schedule_stats
.stats
[decision
];
2582 INSN_LAST_RFS_WIN (tmp
) = decision
;
2583 else if (result
> 0)
2584 INSN_LAST_RFS_WIN (tmp2
) = decision
;
2590 /* Sorting predicate to move DEBUG_INSNs to the top of ready list, while
2591 keeping normal insns in original order. */
2594 rank_for_schedule_debug (const void *x
, const void *y
)
2596 rtx_insn
*tmp
= *(rtx_insn
* const *) y
;
2597 rtx_insn
*tmp2
= *(rtx_insn
* const *) x
;
2599 /* Schedule debug insns as early as possible. */
2600 if (DEBUG_INSN_P (tmp
) && !DEBUG_INSN_P (tmp2
))
2602 else if (!DEBUG_INSN_P (tmp
) && DEBUG_INSN_P (tmp2
))
2604 else if (DEBUG_INSN_P (tmp
) && DEBUG_INSN_P (tmp2
))
2605 return INSN_LUID (tmp
) - INSN_LUID (tmp2
);
2607 return INSN_RFS_DEBUG_ORIG_ORDER (tmp2
) - INSN_RFS_DEBUG_ORIG_ORDER (tmp
);
2610 /* Returns a positive value if x is preferred; returns a negative value if
2611 y is preferred. Should never return 0, since that will make the sort
2615 rank_for_schedule (const void *x
, const void *y
)
2617 rtx_insn
*tmp
= *(rtx_insn
* const *) y
;
2618 rtx_insn
*tmp2
= *(rtx_insn
* const *) x
;
2619 int tmp_class
, tmp2_class
;
2620 int val
, priority_val
, info_val
, diff
;
2622 if (live_range_shrinkage_p
)
2624 /* Don't use SCHED_PRESSURE_MODEL -- it results in much worse
2626 gcc_assert (sched_pressure
== SCHED_PRESSURE_WEIGHTED
);
2627 if ((INSN_REG_PRESSURE_EXCESS_COST_CHANGE (tmp
) < 0
2628 || INSN_REG_PRESSURE_EXCESS_COST_CHANGE (tmp2
) < 0)
2629 && (diff
= (INSN_REG_PRESSURE_EXCESS_COST_CHANGE (tmp
)
2630 - INSN_REG_PRESSURE_EXCESS_COST_CHANGE (tmp2
))) != 0)
2631 return rfs_result (RFS_LIVE_RANGE_SHRINK1
, diff
, tmp
, tmp2
);
2632 /* Sort by INSN_LUID (original insn order), so that we make the
2633 sort stable. This minimizes instruction movement, thus
2634 minimizing sched's effect on debugging and cross-jumping. */
2635 return rfs_result (RFS_LIVE_RANGE_SHRINK2
,
2636 INSN_LUID (tmp
) - INSN_LUID (tmp2
), tmp
, tmp2
);
2639 /* The insn in a schedule group should be issued the first. */
2640 if (flag_sched_group_heuristic
&&
2641 SCHED_GROUP_P (tmp
) != SCHED_GROUP_P (tmp2
))
2642 return rfs_result (RFS_SCHED_GROUP
, SCHED_GROUP_P (tmp2
) ? 1 : -1,
2645 /* Make sure that priority of TMP and TMP2 are initialized. */
2646 gcc_assert (INSN_PRIORITY_KNOWN (tmp
) && INSN_PRIORITY_KNOWN (tmp2
));
2650 /* The instruction that has the same fusion priority as the last
2651 instruction is the instruction we picked next. If that is not
2652 the case, we sort ready list firstly by fusion priority, then
2653 by priority, and at last by INSN_LUID. */
2654 int a
= INSN_FUSION_PRIORITY (tmp
);
2655 int b
= INSN_FUSION_PRIORITY (tmp2
);
2658 if (last_nondebug_scheduled_insn
2659 && !NOTE_P (last_nondebug_scheduled_insn
)
2660 && BLOCK_FOR_INSN (tmp
)
2661 == BLOCK_FOR_INSN (last_nondebug_scheduled_insn
))
2662 last
= INSN_FUSION_PRIORITY (last_nondebug_scheduled_insn
);
2664 if (a
!= last
&& b
!= last
)
2668 a
= INSN_PRIORITY (tmp
);
2669 b
= INSN_PRIORITY (tmp2
);
2672 return rfs_result (RFS_FUSION
, b
- a
, tmp
, tmp2
);
2674 return rfs_result (RFS_FUSION
,
2675 INSN_LUID (tmp
) - INSN_LUID (tmp2
), tmp
, tmp2
);
2679 gcc_assert (last_nondebug_scheduled_insn
2680 && !NOTE_P (last_nondebug_scheduled_insn
));
2681 last
= INSN_PRIORITY (last_nondebug_scheduled_insn
);
2683 a
= abs (INSN_PRIORITY (tmp
) - last
);
2684 b
= abs (INSN_PRIORITY (tmp2
) - last
);
2686 return rfs_result (RFS_FUSION
, a
- b
, tmp
, tmp2
);
2688 return rfs_result (RFS_FUSION
,
2689 INSN_LUID (tmp
) - INSN_LUID (tmp2
), tmp
, tmp2
);
2692 return rfs_result (RFS_FUSION
, -1, tmp
, tmp2
);
2694 return rfs_result (RFS_FUSION
, 1, tmp
, tmp2
);
2697 if (sched_pressure
!= SCHED_PRESSURE_NONE
)
2699 /* Prefer insn whose scheduling results in the smallest register
2701 if ((diff
= (INSN_REG_PRESSURE_EXCESS_COST_CHANGE (tmp
)
2703 - INSN_REG_PRESSURE_EXCESS_COST_CHANGE (tmp2
)
2704 - insn_delay (tmp2
))))
2705 return rfs_result (RFS_PRESSURE_DELAY
, diff
, tmp
, tmp2
);
2708 if (sched_pressure
!= SCHED_PRESSURE_NONE
2709 && (INSN_TICK (tmp2
) > clock_var
|| INSN_TICK (tmp
) > clock_var
)
2710 && INSN_TICK (tmp2
) != INSN_TICK (tmp
))
2712 diff
= INSN_TICK (tmp
) - INSN_TICK (tmp2
);
2713 return rfs_result (RFS_PRESSURE_TICK
, diff
, tmp
, tmp2
);
2716 /* If we are doing backtracking in this schedule, prefer insns that
2717 have forward dependencies with negative cost against an insn that
2718 was already scheduled. */
2719 if (current_sched_info
->flags
& DO_BACKTRACKING
)
2721 priority_val
= FEEDS_BACKTRACK_INSN (tmp2
) - FEEDS_BACKTRACK_INSN (tmp
);
2723 return rfs_result (RFS_FEEDS_BACKTRACK_INSN
, priority_val
, tmp
, tmp2
);
2726 /* Prefer insn with higher priority. */
2727 priority_val
= INSN_PRIORITY (tmp2
) - INSN_PRIORITY (tmp
);
2729 if (flag_sched_critical_path_heuristic
&& priority_val
)
2730 return rfs_result (RFS_PRIORITY
, priority_val
, tmp
, tmp2
);
2732 if (PARAM_VALUE (PARAM_SCHED_AUTOPREF_QUEUE_DEPTH
) >= 0)
2734 int autopref
= autopref_rank_for_schedule (tmp
, tmp2
);
2739 /* Prefer speculative insn with greater dependencies weakness. */
2740 if (flag_sched_spec_insn_heuristic
&& spec_info
)
2746 ds1
= TODO_SPEC (tmp
) & SPECULATIVE
;
2748 dw1
= ds_weak (ds1
);
2752 ds2
= TODO_SPEC (tmp2
) & SPECULATIVE
;
2754 dw2
= ds_weak (ds2
);
2759 if (dw
> (NO_DEP_WEAK
/ 8) || dw
< -(NO_DEP_WEAK
/ 8))
2760 return rfs_result (RFS_SPECULATION
, dw
, tmp
, tmp2
);
2763 info_val
= (*current_sched_info
->rank
) (tmp
, tmp2
);
2764 if (flag_sched_rank_heuristic
&& info_val
)
2765 return rfs_result (RFS_SCHED_RANK
, info_val
, tmp
, tmp2
);
2767 /* Compare insns based on their relation to the last scheduled
2769 if (flag_sched_last_insn_heuristic
&& last_nondebug_scheduled_insn
)
2773 rtx_insn
*last
= last_nondebug_scheduled_insn
;
2775 /* Classify the instructions into three classes:
2776 1) Data dependent on last schedule insn.
2777 2) Anti/Output dependent on last scheduled insn.
2778 3) Independent of last scheduled insn, or has latency of one.
2779 Choose the insn from the highest numbered class if different. */
2780 dep1
= sd_find_dep_between (last
, tmp
, true);
2782 if (dep1
== NULL
|| dep_cost (dep1
) == 1)
2784 else if (/* Data dependence. */
2785 DEP_TYPE (dep1
) == REG_DEP_TRUE
)
2790 dep2
= sd_find_dep_between (last
, tmp2
, true);
2792 if (dep2
== NULL
|| dep_cost (dep2
) == 1)
2794 else if (/* Data dependence. */
2795 DEP_TYPE (dep2
) == REG_DEP_TRUE
)
2800 if ((val
= tmp2_class
- tmp_class
))
2801 return rfs_result (RFS_LAST_INSN
, val
, tmp
, tmp2
);
2804 /* Prefer instructions that occur earlier in the model schedule. */
2805 if (sched_pressure
== SCHED_PRESSURE_MODEL
2806 && INSN_BB (tmp
) == target_bb
&& INSN_BB (tmp2
) == target_bb
)
2808 diff
= model_index (tmp
) - model_index (tmp2
);
2809 gcc_assert (diff
!= 0);
2810 return rfs_result (RFS_PRESSURE_INDEX
, diff
, tmp
, tmp2
);
2813 /* Prefer the insn which has more later insns that depend on it.
2814 This gives the scheduler more freedom when scheduling later
2815 instructions at the expense of added register pressure. */
2817 val
= (dep_list_size (tmp2
, SD_LIST_FORW
)
2818 - dep_list_size (tmp
, SD_LIST_FORW
));
2820 if (flag_sched_dep_count_heuristic
&& val
!= 0)
2821 return rfs_result (RFS_DEP_COUNT
, val
, tmp
, tmp2
);
2823 /* If insns are equally good, sort by INSN_LUID (original insn order),
2824 so that we make the sort stable. This minimizes instruction movement,
2825 thus minimizing sched's effect on debugging and cross-jumping. */
2826 return rfs_result (RFS_TIE
, INSN_LUID (tmp
) - INSN_LUID (tmp2
), tmp
, tmp2
);
2829 /* Resort the array A in which only element at index N may be out of order. */
2831 HAIFA_INLINE
static void
2832 swap_sort (rtx_insn
**a
, int n
)
2834 rtx_insn
*insn
= a
[n
- 1];
2837 while (i
>= 0 && rank_for_schedule (a
+ i
, &insn
) >= 0)
2845 /* Add INSN to the insn queue so that it can be executed at least
2846 N_CYCLES after the currently executing insn. Preserve insns
2847 chain for debugging purposes. REASON will be printed in debugging
2850 HAIFA_INLINE
static void
2851 queue_insn (rtx_insn
*insn
, int n_cycles
, const char *reason
)
2853 int next_q
= NEXT_Q_AFTER (q_ptr
, n_cycles
);
2854 rtx_insn_list
*link
= alloc_INSN_LIST (insn
, insn_queue
[next_q
]);
2857 gcc_assert (n_cycles
<= max_insn_queue_index
);
2858 gcc_assert (!DEBUG_INSN_P (insn
));
2860 insn_queue
[next_q
] = link
;
2863 if (sched_verbose
>= 2)
2865 fprintf (sched_dump
, ";;\t\tReady-->Q: insn %s: ",
2866 (*current_sched_info
->print_insn
) (insn
, 0));
2868 fprintf (sched_dump
, "queued for %d cycles (%s).\n", n_cycles
, reason
);
2871 QUEUE_INDEX (insn
) = next_q
;
2873 if (current_sched_info
->flags
& DO_BACKTRACKING
)
2875 new_tick
= clock_var
+ n_cycles
;
2876 if (INSN_TICK (insn
) == INVALID_TICK
|| INSN_TICK (insn
) < new_tick
)
2877 INSN_TICK (insn
) = new_tick
;
2879 if (INSN_EXACT_TICK (insn
) != INVALID_TICK
2880 && INSN_EXACT_TICK (insn
) < clock_var
+ n_cycles
)
2882 must_backtrack
= true;
2883 if (sched_verbose
>= 2)
2884 fprintf (sched_dump
, ";;\t\tcausing a backtrack.\n");
2889 /* Remove INSN from queue. */
2891 queue_remove (rtx_insn
*insn
)
2893 gcc_assert (QUEUE_INDEX (insn
) >= 0);
2894 remove_free_INSN_LIST_elem (insn
, &insn_queue
[QUEUE_INDEX (insn
)]);
2896 QUEUE_INDEX (insn
) = QUEUE_NOWHERE
;
2899 /* Return a pointer to the bottom of the ready list, i.e. the insn
2900 with the lowest priority. */
2903 ready_lastpos (struct ready_list
*ready
)
2905 gcc_assert (ready
->n_ready
>= 1);
2906 return ready
->vec
+ ready
->first
- ready
->n_ready
+ 1;
2909 /* Add an element INSN to the ready list so that it ends up with the
2910 lowest/highest priority depending on FIRST_P. */
2912 HAIFA_INLINE
static void
2913 ready_add (struct ready_list
*ready
, rtx_insn
*insn
, bool first_p
)
2917 if (ready
->first
== ready
->n_ready
)
2919 memmove (ready
->vec
+ ready
->veclen
- ready
->n_ready
,
2920 ready_lastpos (ready
),
2921 ready
->n_ready
* sizeof (rtx
));
2922 ready
->first
= ready
->veclen
- 1;
2924 ready
->vec
[ready
->first
- ready
->n_ready
] = insn
;
2928 if (ready
->first
== ready
->veclen
- 1)
2931 /* ready_lastpos() fails when called with (ready->n_ready == 0). */
2932 memmove (ready
->vec
+ ready
->veclen
- ready
->n_ready
- 1,
2933 ready_lastpos (ready
),
2934 ready
->n_ready
* sizeof (rtx
));
2935 ready
->first
= ready
->veclen
- 2;
2937 ready
->vec
[++(ready
->first
)] = insn
;
2941 if (DEBUG_INSN_P (insn
))
2944 gcc_assert (QUEUE_INDEX (insn
) != QUEUE_READY
);
2945 QUEUE_INDEX (insn
) = QUEUE_READY
;
2947 if (INSN_EXACT_TICK (insn
) != INVALID_TICK
2948 && INSN_EXACT_TICK (insn
) < clock_var
)
2950 must_backtrack
= true;
2954 /* Remove the element with the highest priority from the ready list and
2957 HAIFA_INLINE
static rtx_insn
*
2958 ready_remove_first (struct ready_list
*ready
)
2962 gcc_assert (ready
->n_ready
);
2963 t
= ready
->vec
[ready
->first
--];
2965 if (DEBUG_INSN_P (t
))
2967 /* If the queue becomes empty, reset it. */
2968 if (ready
->n_ready
== 0)
2969 ready
->first
= ready
->veclen
- 1;
2971 gcc_assert (QUEUE_INDEX (t
) == QUEUE_READY
);
2972 QUEUE_INDEX (t
) = QUEUE_NOWHERE
;
2977 /* The following code implements multi-pass scheduling for the first
2978 cycle. In other words, we will try to choose ready insn which
2979 permits to start maximum number of insns on the same cycle. */
2981 /* Return a pointer to the element INDEX from the ready. INDEX for
2982 insn with the highest priority is 0, and the lowest priority has
2986 ready_element (struct ready_list
*ready
, int index
)
2988 gcc_assert (ready
->n_ready
&& index
< ready
->n_ready
);
2990 return ready
->vec
[ready
->first
- index
];
2993 /* Remove the element INDEX from the ready list and return it. INDEX
2994 for insn with the highest priority is 0, and the lowest priority
2997 HAIFA_INLINE
static rtx_insn
*
2998 ready_remove (struct ready_list
*ready
, int index
)
3004 return ready_remove_first (ready
);
3005 gcc_assert (ready
->n_ready
&& index
< ready
->n_ready
);
3006 t
= ready
->vec
[ready
->first
- index
];
3008 if (DEBUG_INSN_P (t
))
3010 for (i
= index
; i
< ready
->n_ready
; i
++)
3011 ready
->vec
[ready
->first
- i
] = ready
->vec
[ready
->first
- i
- 1];
3012 QUEUE_INDEX (t
) = QUEUE_NOWHERE
;
3016 /* Remove INSN from the ready list. */
3018 ready_remove_insn (rtx_insn
*insn
)
3022 for (i
= 0; i
< readyp
->n_ready
; i
++)
3023 if (ready_element (readyp
, i
) == insn
)
3025 ready_remove (readyp
, i
);
3031 /* Calculate difference of two statistics set WAS and NOW.
3032 Result returned in WAS. */
3034 rank_for_schedule_stats_diff (rank_for_schedule_stats_t
*was
,
3035 const rank_for_schedule_stats_t
*now
)
3037 for (int i
= 0; i
< RFS_N
; ++i
)
3038 was
->stats
[i
] = now
->stats
[i
] - was
->stats
[i
];
3041 /* Print rank_for_schedule statistics. */
3043 print_rank_for_schedule_stats (const char *prefix
,
3044 const rank_for_schedule_stats_t
*stats
,
3045 struct ready_list
*ready
)
3047 for (int i
= 0; i
< RFS_N
; ++i
)
3048 if (stats
->stats
[i
])
3050 fprintf (sched_dump
, "%s%20s: %u", prefix
, rfs_str
[i
], stats
->stats
[i
]);
3053 /* Print out insns that won due to RFS_<I>. */
3055 rtx_insn
**p
= ready_lastpos (ready
);
3057 fprintf (sched_dump
, ":");
3058 /* Start with 1 since least-priority insn didn't have any wins. */
3059 for (int j
= 1; j
< ready
->n_ready
; ++j
)
3060 if (INSN_LAST_RFS_WIN (p
[j
]) == i
)
3061 fprintf (sched_dump
, " %s",
3062 (*current_sched_info
->print_insn
) (p
[j
], 0));
3064 fprintf (sched_dump
, "\n");
3068 /* Separate DEBUG_INSNS from normal insns. DEBUG_INSNs go to the end
3071 ready_sort_debug (struct ready_list
*ready
)
3074 rtx_insn
**first
= ready_lastpos (ready
);
3076 for (i
= 0; i
< ready
->n_ready
; ++i
)
3077 if (!DEBUG_INSN_P (first
[i
]))
3078 INSN_RFS_DEBUG_ORIG_ORDER (first
[i
]) = i
;
3080 qsort (first
, ready
->n_ready
, sizeof (rtx
), rank_for_schedule_debug
);
3083 /* Sort non-debug insns in the ready list READY by ascending priority.
3084 Assumes that all debug insns are separated from the real insns. */
3086 ready_sort_real (struct ready_list
*ready
)
3089 rtx_insn
**first
= ready_lastpos (ready
);
3090 int n_ready_real
= ready
->n_ready
- ready
->n_debug
;
3092 if (sched_pressure
== SCHED_PRESSURE_WEIGHTED
)
3093 for (i
= 0; i
< n_ready_real
; ++i
)
3094 setup_insn_reg_pressure_info (first
[i
]);
3095 else if (sched_pressure
== SCHED_PRESSURE_MODEL
3096 && model_curr_point
< model_num_insns
)
3097 model_set_excess_costs (first
, n_ready_real
);
3099 rank_for_schedule_stats_t stats1
;
3100 if (sched_verbose
>= 4)
3101 stats1
= rank_for_schedule_stats
;
3103 if (n_ready_real
== 2)
3104 swap_sort (first
, n_ready_real
);
3105 else if (n_ready_real
> 2)
3106 qsort (first
, n_ready_real
, sizeof (rtx
), rank_for_schedule
);
3108 if (sched_verbose
>= 4)
3110 rank_for_schedule_stats_diff (&stats1
, &rank_for_schedule_stats
);
3111 print_rank_for_schedule_stats (";;\t\t", &stats1
, ready
);
3115 /* Sort the ready list READY by ascending priority. */
3117 ready_sort (struct ready_list
*ready
)
3119 if (ready
->n_debug
> 0)
3120 ready_sort_debug (ready
);
3122 ready_sort_real (ready
);
3125 /* PREV is an insn that is ready to execute. Adjust its priority if that
3126 will help shorten or lengthen register lifetimes as appropriate. Also
3127 provide a hook for the target to tweak itself. */
3129 HAIFA_INLINE
static void
3130 adjust_priority (rtx_insn
*prev
)
3132 /* ??? There used to be code here to try and estimate how an insn
3133 affected register lifetimes, but it did it by looking at REG_DEAD
3134 notes, which we removed in schedule_region. Nor did it try to
3135 take into account register pressure or anything useful like that.
3137 Revisit when we have a machine model to work with and not before. */
3139 if (targetm
.sched
.adjust_priority
)
3140 INSN_PRIORITY (prev
) =
3141 targetm
.sched
.adjust_priority (prev
, INSN_PRIORITY (prev
));
3144 /* Advance DFA state STATE on one cycle. */
3146 advance_state (state_t state
)
3148 if (targetm
.sched
.dfa_pre_advance_cycle
)
3149 targetm
.sched
.dfa_pre_advance_cycle ();
3151 if (targetm
.sched
.dfa_pre_cycle_insn
)
3152 state_transition (state
,
3153 targetm
.sched
.dfa_pre_cycle_insn ());
3155 state_transition (state
, NULL
);
3157 if (targetm
.sched
.dfa_post_cycle_insn
)
3158 state_transition (state
,
3159 targetm
.sched
.dfa_post_cycle_insn ());
3161 if (targetm
.sched
.dfa_post_advance_cycle
)
3162 targetm
.sched
.dfa_post_advance_cycle ();
3165 /* Advance time on one cycle. */
3166 HAIFA_INLINE
static void
3167 advance_one_cycle (void)
3169 advance_state (curr_state
);
3170 if (sched_verbose
>= 4)
3171 fprintf (sched_dump
, ";;\tAdvance the current state.\n");
3174 /* Update register pressure after scheduling INSN. */
3176 update_register_pressure (rtx_insn
*insn
)
3178 struct reg_use_data
*use
;
3179 struct reg_set_data
*set
;
3181 gcc_checking_assert (!DEBUG_INSN_P (insn
));
3183 for (use
= INSN_REG_USE_LIST (insn
); use
!= NULL
; use
= use
->next_insn_use
)
3184 if (dying_use_p (use
))
3185 mark_regno_birth_or_death (curr_reg_live
, curr_reg_pressure
,
3187 for (set
= INSN_REG_SET_LIST (insn
); set
!= NULL
; set
= set
->next_insn_set
)
3188 mark_regno_birth_or_death (curr_reg_live
, curr_reg_pressure
,
3192 /* Set up or update (if UPDATE_P) max register pressure (see its
3193 meaning in sched-int.h::_haifa_insn_data) for all current BB insns
3194 after insn AFTER. */
3196 setup_insn_max_reg_pressure (rtx_insn
*after
, bool update_p
)
3201 static int max_reg_pressure
[N_REG_CLASSES
];
3203 save_reg_pressure ();
3204 for (i
= 0; i
< ira_pressure_classes_num
; i
++)
3205 max_reg_pressure
[ira_pressure_classes
[i
]]
3206 = curr_reg_pressure
[ira_pressure_classes
[i
]];
3207 for (insn
= NEXT_INSN (after
);
3208 insn
!= NULL_RTX
&& ! BARRIER_P (insn
)
3209 && BLOCK_FOR_INSN (insn
) == BLOCK_FOR_INSN (after
);
3210 insn
= NEXT_INSN (insn
))
3211 if (NONDEBUG_INSN_P (insn
))
3214 for (i
= 0; i
< ira_pressure_classes_num
; i
++)
3216 p
= max_reg_pressure
[ira_pressure_classes
[i
]];
3217 if (INSN_MAX_REG_PRESSURE (insn
)[i
] != p
)
3220 INSN_MAX_REG_PRESSURE (insn
)[i
]
3221 = max_reg_pressure
[ira_pressure_classes
[i
]];
3224 if (update_p
&& eq_p
)
3226 update_register_pressure (insn
);
3227 for (i
= 0; i
< ira_pressure_classes_num
; i
++)
3228 if (max_reg_pressure
[ira_pressure_classes
[i
]]
3229 < curr_reg_pressure
[ira_pressure_classes
[i
]])
3230 max_reg_pressure
[ira_pressure_classes
[i
]]
3231 = curr_reg_pressure
[ira_pressure_classes
[i
]];
3233 restore_reg_pressure ();
3236 /* Update the current register pressure after scheduling INSN. Update
3237 also max register pressure for unscheduled insns of the current
3240 update_reg_and_insn_max_reg_pressure (rtx_insn
*insn
)
3243 int before
[N_REG_CLASSES
];
3245 for (i
= 0; i
< ira_pressure_classes_num
; i
++)
3246 before
[i
] = curr_reg_pressure
[ira_pressure_classes
[i
]];
3247 update_register_pressure (insn
);
3248 for (i
= 0; i
< ira_pressure_classes_num
; i
++)
3249 if (curr_reg_pressure
[ira_pressure_classes
[i
]] != before
[i
])
3251 if (i
< ira_pressure_classes_num
)
3252 setup_insn_max_reg_pressure (insn
, true);
3255 /* Set up register pressure at the beginning of basic block BB whose
3256 insns starting after insn AFTER. Set up also max register pressure
3257 for all insns of the basic block. */
3259 sched_setup_bb_reg_pressure_info (basic_block bb
, rtx_insn
*after
)
3261 gcc_assert (sched_pressure
== SCHED_PRESSURE_WEIGHTED
);
3262 initiate_bb_reg_pressure_info (bb
);
3263 setup_insn_max_reg_pressure (after
, false);
3266 /* If doing predication while scheduling, verify whether INSN, which
3267 has just been scheduled, clobbers the conditions of any
3268 instructions that must be predicated in order to break their
3269 dependencies. If so, remove them from the queues so that they will
3270 only be scheduled once their control dependency is resolved. */
3273 check_clobbered_conditions (rtx_insn
*insn
)
3278 if ((current_sched_info
->flags
& DO_PREDICATION
) == 0)
3281 find_all_hard_reg_sets (insn
, &t
, true);
3284 for (i
= 0; i
< ready
.n_ready
; i
++)
3286 rtx_insn
*x
= ready_element (&ready
, i
);
3287 if (TODO_SPEC (x
) == DEP_CONTROL
&& cond_clobbered_p (x
, t
))
3289 ready_remove_insn (x
);
3293 for (i
= 0; i
<= max_insn_queue_index
; i
++)
3295 rtx_insn_list
*link
;
3296 int q
= NEXT_Q_AFTER (q_ptr
, i
);
3299 for (link
= insn_queue
[q
]; link
; link
= link
->next ())
3301 rtx_insn
*x
= link
->insn ();
3302 if (TODO_SPEC (x
) == DEP_CONTROL
&& cond_clobbered_p (x
, t
))
3311 /* Return (in order):
3313 - positive if INSN adversely affects the pressure on one
3316 - negative if INSN reduces the pressure on one register class
3318 - 0 if INSN doesn't affect the pressure on any register class. */
3321 model_classify_pressure (struct model_insn_info
*insn
)
3323 struct reg_pressure_data
*reg_pressure
;
3324 int death
[N_REG_CLASSES
];
3327 calculate_reg_deaths (insn
->insn
, death
);
3328 reg_pressure
= INSN_REG_PRESSURE (insn
->insn
);
3330 for (pci
= 0; pci
< ira_pressure_classes_num
; pci
++)
3332 cl
= ira_pressure_classes
[pci
];
3333 if (death
[cl
] < reg_pressure
[pci
].set_increase
)
3335 sum
+= reg_pressure
[pci
].set_increase
- death
[cl
];
3340 /* Return true if INSN1 should come before INSN2 in the model schedule. */
3343 model_order_p (struct model_insn_info
*insn1
, struct model_insn_info
*insn2
)
3345 unsigned int height1
, height2
;
3346 unsigned int priority1
, priority2
;
3348 /* Prefer instructions with a higher model priority. */
3349 if (insn1
->model_priority
!= insn2
->model_priority
)
3350 return insn1
->model_priority
> insn2
->model_priority
;
3352 /* Combine the length of the longest path of satisfied true dependencies
3353 that leads to each instruction (depth) with the length of the longest
3354 path of any dependencies that leads from the instruction (alap).
3355 Prefer instructions with the greatest combined length. If the combined
3356 lengths are equal, prefer instructions with the greatest depth.
3358 The idea is that, if we have a set S of "equal" instructions that each
3359 have ALAP value X, and we pick one such instruction I, any true-dependent
3360 successors of I that have ALAP value X - 1 should be preferred over S.
3361 This encourages the schedule to be "narrow" rather than "wide".
3362 However, if I is a low-priority instruction that we decided to
3363 schedule because of its model_classify_pressure, and if there
3364 is a set of higher-priority instructions T, the aforementioned
3365 successors of I should not have the edge over T. */
3366 height1
= insn1
->depth
+ insn1
->alap
;
3367 height2
= insn2
->depth
+ insn2
->alap
;
3368 if (height1
!= height2
)
3369 return height1
> height2
;
3370 if (insn1
->depth
!= insn2
->depth
)
3371 return insn1
->depth
> insn2
->depth
;
3373 /* We have no real preference between INSN1 an INSN2 as far as attempts
3374 to reduce pressure go. Prefer instructions with higher priorities. */
3375 priority1
= INSN_PRIORITY (insn1
->insn
);
3376 priority2
= INSN_PRIORITY (insn2
->insn
);
3377 if (priority1
!= priority2
)
3378 return priority1
> priority2
;
3380 /* Use the original rtl sequence as a tie-breaker. */
3381 return insn1
< insn2
;
3384 /* Add INSN to the model worklist immediately after PREV. Add it to the
3385 beginning of the list if PREV is null. */
3388 model_add_to_worklist_at (struct model_insn_info
*insn
,
3389 struct model_insn_info
*prev
)
3391 gcc_assert (QUEUE_INDEX (insn
->insn
) == QUEUE_NOWHERE
);
3392 QUEUE_INDEX (insn
->insn
) = QUEUE_READY
;
3397 insn
->next
= prev
->next
;
3402 insn
->next
= model_worklist
;
3403 model_worklist
= insn
;
3406 insn
->next
->prev
= insn
;
3409 /* Remove INSN from the model worklist. */
3412 model_remove_from_worklist (struct model_insn_info
*insn
)
3414 gcc_assert (QUEUE_INDEX (insn
->insn
) == QUEUE_READY
);
3415 QUEUE_INDEX (insn
->insn
) = QUEUE_NOWHERE
;
3418 insn
->prev
->next
= insn
->next
;
3420 model_worklist
= insn
->next
;
3422 insn
->next
->prev
= insn
->prev
;
3425 /* Add INSN to the model worklist. Start looking for a suitable position
3426 between neighbors PREV and NEXT, testing at most MAX_SCHED_READY_INSNS
3427 insns either side. A null PREV indicates the beginning of the list and
3428 a null NEXT indicates the end. */
3431 model_add_to_worklist (struct model_insn_info
*insn
,
3432 struct model_insn_info
*prev
,
3433 struct model_insn_info
*next
)
3437 count
= MAX_SCHED_READY_INSNS
;
3438 if (count
> 0 && prev
&& model_order_p (insn
, prev
))
3444 while (count
> 0 && prev
&& model_order_p (insn
, prev
));
3446 while (count
> 0 && next
&& model_order_p (next
, insn
))
3452 model_add_to_worklist_at (insn
, prev
);
3455 /* INSN may now have a higher priority (in the model_order_p sense)
3456 than before. Move it up the worklist if necessary. */
3459 model_promote_insn (struct model_insn_info
*insn
)
3461 struct model_insn_info
*prev
;
3465 count
= MAX_SCHED_READY_INSNS
;
3466 while (count
> 0 && prev
&& model_order_p (insn
, prev
))
3471 if (prev
!= insn
->prev
)
3473 model_remove_from_worklist (insn
);
3474 model_add_to_worklist_at (insn
, prev
);
3478 /* Add INSN to the end of the model schedule. */
3481 model_add_to_schedule (rtx_insn
*insn
)
3485 gcc_assert (QUEUE_INDEX (insn
) == QUEUE_NOWHERE
);
3486 QUEUE_INDEX (insn
) = QUEUE_SCHEDULED
;
3488 point
= model_schedule
.length ();
3489 model_schedule
.quick_push (insn
);
3490 INSN_MODEL_INDEX (insn
) = point
+ 1;
3493 /* Analyze the instructions that are to be scheduled, setting up
3494 MODEL_INSN_INFO (...) and model_num_insns accordingly. Add ready
3495 instructions to model_worklist. */
3498 model_analyze_insns (void)
3500 rtx_insn
*start
, *end
, *iter
;
3501 sd_iterator_def sd_it
;
3503 struct model_insn_info
*insn
, *con
;
3505 model_num_insns
= 0;
3506 start
= PREV_INSN (current_sched_info
->next_tail
);
3507 end
= current_sched_info
->prev_head
;
3508 for (iter
= start
; iter
!= end
; iter
= PREV_INSN (iter
))
3509 if (NONDEBUG_INSN_P (iter
))
3511 insn
= MODEL_INSN_INFO (iter
);
3513 FOR_EACH_DEP (iter
, SD_LIST_FORW
, sd_it
, dep
)
3515 con
= MODEL_INSN_INFO (DEP_CON (dep
));
3516 if (con
->insn
&& insn
->alap
< con
->alap
+ 1)
3517 insn
->alap
= con
->alap
+ 1;
3520 insn
->old_queue
= QUEUE_INDEX (iter
);
3521 QUEUE_INDEX (iter
) = QUEUE_NOWHERE
;
3523 insn
->unscheduled_preds
= dep_list_size (iter
, SD_LIST_HARD_BACK
);
3524 if (insn
->unscheduled_preds
== 0)
3525 model_add_to_worklist (insn
, NULL
, model_worklist
);
3531 /* The global state describes the register pressure at the start of the
3532 model schedule. Initialize GROUP accordingly. */
3535 model_init_pressure_group (struct model_pressure_group
*group
)
3539 for (pci
= 0; pci
< ira_pressure_classes_num
; pci
++)
3541 cl
= ira_pressure_classes
[pci
];
3542 group
->limits
[pci
].pressure
= curr_reg_pressure
[cl
];
3543 group
->limits
[pci
].point
= 0;
3545 /* Use index model_num_insns to record the state after the last
3546 instruction in the model schedule. */
3547 group
->model
= XNEWVEC (struct model_pressure_data
,
3548 (model_num_insns
+ 1) * ira_pressure_classes_num
);
3551 /* Record that MODEL_REF_PRESSURE (GROUP, POINT, PCI) is PRESSURE.
3552 Update the maximum pressure for the whole schedule. */
3555 model_record_pressure (struct model_pressure_group
*group
,
3556 int point
, int pci
, int pressure
)
3558 MODEL_REF_PRESSURE (group
, point
, pci
) = pressure
;
3559 if (group
->limits
[pci
].pressure
< pressure
)
3561 group
->limits
[pci
].pressure
= pressure
;
3562 group
->limits
[pci
].point
= point
;
3566 /* INSN has just been added to the end of the model schedule. Record its
3567 register-pressure information. */
3570 model_record_pressures (struct model_insn_info
*insn
)
3572 struct reg_pressure_data
*reg_pressure
;
3573 int point
, pci
, cl
, delta
;
3574 int death
[N_REG_CLASSES
];
3576 point
= model_index (insn
->insn
);
3577 if (sched_verbose
>= 2)
3581 fprintf (sched_dump
, "\n;;\tModel schedule:\n;;\n");
3582 fprintf (sched_dump
, ";;\t| idx insn | mpri hght dpth prio |\n");
3584 fprintf (sched_dump
, ";;\t| %3d %4d | %4d %4d %4d %4d | %-30s ",
3585 point
, INSN_UID (insn
->insn
), insn
->model_priority
,
3586 insn
->depth
+ insn
->alap
, insn
->depth
,
3587 INSN_PRIORITY (insn
->insn
),
3588 str_pattern_slim (PATTERN (insn
->insn
)));
3590 calculate_reg_deaths (insn
->insn
, death
);
3591 reg_pressure
= INSN_REG_PRESSURE (insn
->insn
);
3592 for (pci
= 0; pci
< ira_pressure_classes_num
; pci
++)
3594 cl
= ira_pressure_classes
[pci
];
3595 delta
= reg_pressure
[pci
].set_increase
- death
[cl
];
3596 if (sched_verbose
>= 2)
3597 fprintf (sched_dump
, " %s:[%d,%+d]", reg_class_names
[cl
],
3598 curr_reg_pressure
[cl
], delta
);
3599 model_record_pressure (&model_before_pressure
, point
, pci
,
3600 curr_reg_pressure
[cl
]);
3602 if (sched_verbose
>= 2)
3603 fprintf (sched_dump
, "\n");
3606 /* All instructions have been added to the model schedule. Record the
3607 final register pressure in GROUP and set up all MODEL_MAX_PRESSUREs. */
3610 model_record_final_pressures (struct model_pressure_group
*group
)
3612 int point
, pci
, max_pressure
, ref_pressure
, cl
;
3614 for (pci
= 0; pci
< ira_pressure_classes_num
; pci
++)
3616 /* Record the final pressure for this class. */
3617 cl
= ira_pressure_classes
[pci
];
3618 point
= model_num_insns
;
3619 ref_pressure
= curr_reg_pressure
[cl
];
3620 model_record_pressure (group
, point
, pci
, ref_pressure
);
3622 /* Record the original maximum pressure. */
3623 group
->limits
[pci
].orig_pressure
= group
->limits
[pci
].pressure
;
3625 /* Update the MODEL_MAX_PRESSURE for every point of the schedule. */
3626 max_pressure
= ref_pressure
;
3627 MODEL_MAX_PRESSURE (group
, point
, pci
) = max_pressure
;
3631 ref_pressure
= MODEL_REF_PRESSURE (group
, point
, pci
);
3632 max_pressure
= MAX (max_pressure
, ref_pressure
);
3633 MODEL_MAX_PRESSURE (group
, point
, pci
) = max_pressure
;
3638 /* Update all successors of INSN, given that INSN has just been scheduled. */
3641 model_add_successors_to_worklist (struct model_insn_info
*insn
)
3643 sd_iterator_def sd_it
;
3644 struct model_insn_info
*con
;
3647 FOR_EACH_DEP (insn
->insn
, SD_LIST_FORW
, sd_it
, dep
)
3649 con
= MODEL_INSN_INFO (DEP_CON (dep
));
3650 /* Ignore debug instructions, and instructions from other blocks. */
3653 con
->unscheduled_preds
--;
3655 /* Update the depth field of each true-dependent successor.
3656 Increasing the depth gives them a higher priority than
3658 if (DEP_TYPE (dep
) == REG_DEP_TRUE
&& con
->depth
< insn
->depth
+ 1)
3660 con
->depth
= insn
->depth
+ 1;
3661 if (QUEUE_INDEX (con
->insn
) == QUEUE_READY
)
3662 model_promote_insn (con
);
3665 /* If this is a true dependency, or if there are no remaining
3666 dependencies for CON (meaning that CON only had non-true
3667 dependencies), make sure that CON is on the worklist.
3668 We don't bother otherwise because it would tend to fill the
3669 worklist with a lot of low-priority instructions that are not
3670 yet ready to issue. */
3671 if ((con
->depth
> 0 || con
->unscheduled_preds
== 0)
3672 && QUEUE_INDEX (con
->insn
) == QUEUE_NOWHERE
)
3673 model_add_to_worklist (con
, insn
, insn
->next
);
3678 /* Give INSN a higher priority than any current instruction, then give
3679 unscheduled predecessors of INSN a higher priority still. If any of
3680 those predecessors are not on the model worklist, do the same for its
3681 predecessors, and so on. */
3684 model_promote_predecessors (struct model_insn_info
*insn
)
3686 struct model_insn_info
*pro
, *first
;
3687 sd_iterator_def sd_it
;
3690 if (sched_verbose
>= 7)
3691 fprintf (sched_dump
, ";;\t+--- priority of %d = %d, priority of",
3692 INSN_UID (insn
->insn
), model_next_priority
);
3693 insn
->model_priority
= model_next_priority
++;
3694 model_remove_from_worklist (insn
);
3695 model_add_to_worklist_at (insn
, NULL
);
3700 FOR_EACH_DEP (insn
->insn
, SD_LIST_HARD_BACK
, sd_it
, dep
)
3702 pro
= MODEL_INSN_INFO (DEP_PRO (dep
));
3703 /* The first test is to ignore debug instructions, and instructions
3704 from other blocks. */
3706 && pro
->model_priority
!= model_next_priority
3707 && QUEUE_INDEX (pro
->insn
) != QUEUE_SCHEDULED
)
3709 pro
->model_priority
= model_next_priority
;
3710 if (sched_verbose
>= 7)
3711 fprintf (sched_dump
, " %d", INSN_UID (pro
->insn
));
3712 if (QUEUE_INDEX (pro
->insn
) == QUEUE_READY
)
3714 /* PRO is already in the worklist, but it now has
3715 a higher priority than before. Move it at the
3716 appropriate place. */
3717 model_remove_from_worklist (pro
);
3718 model_add_to_worklist (pro
, NULL
, model_worklist
);
3722 /* PRO isn't in the worklist. Recursively process
3723 its predecessors until we find one that is. */
3734 if (sched_verbose
>= 7)
3735 fprintf (sched_dump
, " = %d\n", model_next_priority
);
3736 model_next_priority
++;
3739 /* Pick one instruction from model_worklist and process it. */
3742 model_choose_insn (void)
3744 struct model_insn_info
*insn
, *fallback
;
3747 if (sched_verbose
>= 7)
3749 fprintf (sched_dump
, ";;\t+--- worklist:\n");
3750 insn
= model_worklist
;
3751 count
= MAX_SCHED_READY_INSNS
;
3752 while (count
> 0 && insn
)
3754 fprintf (sched_dump
, ";;\t+--- %d [%d, %d, %d, %d]\n",
3755 INSN_UID (insn
->insn
), insn
->model_priority
,
3756 insn
->depth
+ insn
->alap
, insn
->depth
,
3757 INSN_PRIORITY (insn
->insn
));
3763 /* Look for a ready instruction whose model_classify_priority is zero
3764 or negative, picking the highest-priority one. Adding such an
3765 instruction to the schedule now should do no harm, and may actually
3768 Failing that, see whether there is an instruction with the highest
3769 extant model_priority that is not yet ready, but which would reduce
3770 pressure if it became ready. This is designed to catch cases like:
3772 (set (mem (reg R1)) (reg R2))
3774 where the instruction is the last remaining use of R1 and where the
3775 value of R2 is not yet available (or vice versa). The death of R1
3776 means that this instruction already reduces pressure. It is of
3777 course possible that the computation of R2 involves other registers
3778 that are hard to kill, but such cases are rare enough for this
3779 heuristic to be a win in general.
3781 Failing that, just pick the highest-priority instruction in the
3783 count
= MAX_SCHED_READY_INSNS
;
3784 insn
= model_worklist
;
3788 if (count
== 0 || !insn
)
3790 insn
= fallback
? fallback
: model_worklist
;
3793 if (insn
->unscheduled_preds
)
3795 if (model_worklist
->model_priority
== insn
->model_priority
3797 && model_classify_pressure (insn
) < 0)
3802 if (model_classify_pressure (insn
) <= 0)
3809 if (sched_verbose
>= 7 && insn
!= model_worklist
)
3811 if (insn
->unscheduled_preds
)
3812 fprintf (sched_dump
, ";;\t+--- promoting insn %d, with dependencies\n",
3813 INSN_UID (insn
->insn
));
3815 fprintf (sched_dump
, ";;\t+--- promoting insn %d, which is ready\n",
3816 INSN_UID (insn
->insn
));
3818 if (insn
->unscheduled_preds
)
3819 /* INSN isn't yet ready to issue. Give all its predecessors the
3820 highest priority. */
3821 model_promote_predecessors (insn
);
3824 /* INSN is ready. Add it to the end of model_schedule and
3825 process its successors. */
3826 model_add_successors_to_worklist (insn
);
3827 model_remove_from_worklist (insn
);
3828 model_add_to_schedule (insn
->insn
);
3829 model_record_pressures (insn
);
3830 update_register_pressure (insn
->insn
);
3834 /* Restore all QUEUE_INDEXs to the values that they had before
3835 model_start_schedule was called. */
3838 model_reset_queue_indices (void)
3843 FOR_EACH_VEC_ELT (model_schedule
, i
, insn
)
3844 QUEUE_INDEX (insn
) = MODEL_INSN_INFO (insn
)->old_queue
;
3847 /* We have calculated the model schedule and spill costs. Print a summary
3851 model_dump_pressure_summary (void)
3855 fprintf (sched_dump
, ";; Pressure summary:");
3856 for (pci
= 0; pci
< ira_pressure_classes_num
; pci
++)
3858 cl
= ira_pressure_classes
[pci
];
3859 fprintf (sched_dump
, " %s:%d", reg_class_names
[cl
],
3860 model_before_pressure
.limits
[pci
].pressure
);
3862 fprintf (sched_dump
, "\n\n");
3865 /* Initialize the SCHED_PRESSURE_MODEL information for the current
3866 scheduling region. */
3869 model_start_schedule (basic_block bb
)
3871 model_next_priority
= 1;
3872 model_schedule
.create (sched_max_luid
);
3873 model_insns
= XCNEWVEC (struct model_insn_info
, sched_max_luid
);
3875 gcc_assert (bb
== BLOCK_FOR_INSN (NEXT_INSN (current_sched_info
->prev_head
)));
3876 initiate_reg_pressure_info (df_get_live_in (bb
));
3878 model_analyze_insns ();
3879 model_init_pressure_group (&model_before_pressure
);
3880 while (model_worklist
)
3881 model_choose_insn ();
3882 gcc_assert (model_num_insns
== (int) model_schedule
.length ());
3883 if (sched_verbose
>= 2)
3884 fprintf (sched_dump
, "\n");
3886 model_record_final_pressures (&model_before_pressure
);
3887 model_reset_queue_indices ();
3889 XDELETEVEC (model_insns
);
3891 model_curr_point
= 0;
3892 initiate_reg_pressure_info (df_get_live_in (bb
));
3893 if (sched_verbose
>= 1)
3894 model_dump_pressure_summary ();
3897 /* Free the information associated with GROUP. */
3900 model_finalize_pressure_group (struct model_pressure_group
*group
)
3902 XDELETEVEC (group
->model
);
3905 /* Free the information created by model_start_schedule. */
3908 model_end_schedule (void)
3910 model_finalize_pressure_group (&model_before_pressure
);
3911 model_schedule
.release ();
3914 /* Prepare reg pressure scheduling for basic block BB. */
3916 sched_pressure_start_bb (basic_block bb
)
3918 /* Set the number of available registers for each class taking into account
3919 relative probability of current basic block versus function prologue and
3921 * If the basic block executes much more often than the prologue/epilogue
3922 (e.g., inside a hot loop), then cost of spill in the prologue is close to
3923 nil, so the effective number of available registers is
3924 (ira_class_hard_regs_num[cl] - 0).
3925 * If the basic block executes as often as the prologue/epilogue,
3926 then spill in the block is as costly as in the prologue, so the effective
3927 number of available registers is
3928 (ira_class_hard_regs_num[cl] - call_used_regs_num[cl]).
3929 Note that all-else-equal, we prefer to spill in the prologue, since that
3930 allows "extra" registers for other basic blocks of the function.
3931 * If the basic block is on the cold path of the function and executes
3932 rarely, then we should always prefer to spill in the block, rather than
3933 in the prologue/epilogue. The effective number of available register is
3934 (ira_class_hard_regs_num[cl] - call_used_regs_num[cl]). */
3937 int entry_freq
= ENTRY_BLOCK_PTR_FOR_FN (cfun
)->frequency
;
3938 int bb_freq
= bb
->frequency
;
3942 if (entry_freq
== 0)
3943 entry_freq
= bb_freq
= 1;
3945 if (bb_freq
< entry_freq
)
3946 bb_freq
= entry_freq
;
3948 for (i
= 0; i
< ira_pressure_classes_num
; ++i
)
3950 enum reg_class cl
= ira_pressure_classes
[i
];
3951 sched_class_regs_num
[cl
] = ira_class_hard_regs_num
[cl
];
3952 sched_class_regs_num
[cl
]
3953 -= (call_used_regs_num
[cl
] * entry_freq
) / bb_freq
;
3957 if (sched_pressure
== SCHED_PRESSURE_MODEL
)
3958 model_start_schedule (bb
);
3961 /* A structure that holds local state for the loop in schedule_block. */
3962 struct sched_block_state
3964 /* True if no real insns have been scheduled in the current cycle. */
3965 bool first_cycle_insn_p
;
3966 /* True if a shadow insn has been scheduled in the current cycle, which
3967 means that no more normal insns can be issued. */
3968 bool shadows_only_p
;
3969 /* True if we're winding down a modulo schedule, which means that we only
3970 issue insns with INSN_EXACT_TICK set. */
3971 bool modulo_epilogue
;
3972 /* Initialized with the machine's issue rate every cycle, and updated
3973 by calls to the variable_issue hook. */
3977 /* INSN is the "currently executing insn". Launch each insn which was
3978 waiting on INSN. READY is the ready list which contains the insns
3979 that are ready to fire. CLOCK is the current cycle. The function
3980 returns necessary cycle advance after issuing the insn (it is not
3981 zero for insns in a schedule group). */
3984 schedule_insn (rtx_insn
*insn
)
3986 sd_iterator_def sd_it
;
3991 if (sched_verbose
>= 1)
3993 struct reg_pressure_data
*pressure_info
;
3994 fprintf (sched_dump
, ";;\t%3i--> %s %-40s:",
3995 clock_var
, (*current_sched_info
->print_insn
) (insn
, 1),
3996 str_pattern_slim (PATTERN (insn
)));
3998 if (recog_memoized (insn
) < 0)
3999 fprintf (sched_dump
, "nothing");
4001 print_reservation (sched_dump
, insn
);
4002 pressure_info
= INSN_REG_PRESSURE (insn
);
4003 if (pressure_info
!= NULL
)
4005 fputc (':', sched_dump
);
4006 for (i
= 0; i
< ira_pressure_classes_num
; i
++)
4007 fprintf (sched_dump
, "%s%s%+d(%d)",
4008 scheduled_insns
.length () > 1
4010 < INSN_LUID (scheduled_insns
[scheduled_insns
.length () - 2]) ? "@" : "",
4011 reg_class_names
[ira_pressure_classes
[i
]],
4012 pressure_info
[i
].set_increase
, pressure_info
[i
].change
);
4014 if (sched_pressure
== SCHED_PRESSURE_MODEL
4015 && model_curr_point
< model_num_insns
4016 && model_index (insn
) == model_curr_point
)
4017 fprintf (sched_dump
, ":model %d", model_curr_point
);
4018 fputc ('\n', sched_dump
);
4021 if (sched_pressure
== SCHED_PRESSURE_WEIGHTED
&& !DEBUG_INSN_P (insn
))
4022 update_reg_and_insn_max_reg_pressure (insn
);
4024 /* Scheduling instruction should have all its dependencies resolved and
4025 should have been removed from the ready list. */
4026 gcc_assert (sd_lists_empty_p (insn
, SD_LIST_HARD_BACK
));
4028 /* Reset debug insns invalidated by moving this insn. */
4029 if (MAY_HAVE_DEBUG_INSNS
&& !DEBUG_INSN_P (insn
))
4030 for (sd_it
= sd_iterator_start (insn
, SD_LIST_BACK
);
4031 sd_iterator_cond (&sd_it
, &dep
);)
4033 rtx_insn
*dbg
= DEP_PRO (dep
);
4034 struct reg_use_data
*use
, *next
;
4036 if (DEP_STATUS (dep
) & DEP_CANCELLED
)
4038 sd_iterator_next (&sd_it
);
4042 gcc_assert (DEBUG_INSN_P (dbg
));
4044 if (sched_verbose
>= 6)
4045 fprintf (sched_dump
, ";;\t\tresetting: debug insn %d\n",
4048 /* ??? Rather than resetting the debug insn, we might be able
4049 to emit a debug temp before the just-scheduled insn, but
4050 this would involve checking that the expression at the
4051 point of the debug insn is equivalent to the expression
4052 before the just-scheduled insn. They might not be: the
4053 expression in the debug insn may depend on other insns not
4054 yet scheduled that set MEMs, REGs or even other debug
4055 insns. It's not clear that attempting to preserve debug
4056 information in these cases is worth the effort, given how
4057 uncommon these resets are and the likelihood that the debug
4058 temps introduced won't survive the schedule change. */
4059 INSN_VAR_LOCATION_LOC (dbg
) = gen_rtx_UNKNOWN_VAR_LOC ();
4060 df_insn_rescan (dbg
);
4062 /* Unknown location doesn't use any registers. */
4063 for (use
= INSN_REG_USE_LIST (dbg
); use
!= NULL
; use
= next
)
4065 struct reg_use_data
*prev
= use
;
4067 /* Remove use from the cyclic next_regno_use chain first. */
4068 while (prev
->next_regno_use
!= use
)
4069 prev
= prev
->next_regno_use
;
4070 prev
->next_regno_use
= use
->next_regno_use
;
4071 next
= use
->next_insn_use
;
4074 INSN_REG_USE_LIST (dbg
) = NULL
;
4076 /* We delete rather than resolve these deps, otherwise we
4077 crash in sched_free_deps(), because forward deps are
4078 expected to be released before backward deps. */
4079 sd_delete_dep (sd_it
);
4082 gcc_assert (QUEUE_INDEX (insn
) == QUEUE_NOWHERE
);
4083 QUEUE_INDEX (insn
) = QUEUE_SCHEDULED
;
4085 if (sched_pressure
== SCHED_PRESSURE_MODEL
4086 && model_curr_point
< model_num_insns
4087 && NONDEBUG_INSN_P (insn
))
4089 if (model_index (insn
) == model_curr_point
)
4092 while (model_curr_point
< model_num_insns
4093 && (QUEUE_INDEX (MODEL_INSN (model_curr_point
))
4094 == QUEUE_SCHEDULED
));
4096 model_recompute (insn
);
4097 model_update_limit_points ();
4098 update_register_pressure (insn
);
4099 if (sched_verbose
>= 2)
4100 print_curr_reg_pressure ();
4103 gcc_assert (INSN_TICK (insn
) >= MIN_TICK
);
4104 if (INSN_TICK (insn
) > clock_var
)
4105 /* INSN has been prematurely moved from the queue to the ready list.
4106 This is possible only if following flags are set. */
4107 gcc_assert (flag_sched_stalled_insns
|| sched_fusion
);
4109 /* ??? Probably, if INSN is scheduled prematurely, we should leave
4110 INSN_TICK untouched. This is a machine-dependent issue, actually. */
4111 INSN_TICK (insn
) = clock_var
;
4113 check_clobbered_conditions (insn
);
4115 /* Update dependent instructions. First, see if by scheduling this insn
4116 now we broke a dependence in a way that requires us to change another
4118 for (sd_it
= sd_iterator_start (insn
, SD_LIST_SPEC_BACK
);
4119 sd_iterator_cond (&sd_it
, &dep
); sd_iterator_next (&sd_it
))
4121 struct dep_replacement
*desc
= DEP_REPLACE (dep
);
4122 rtx_insn
*pro
= DEP_PRO (dep
);
4123 if (QUEUE_INDEX (pro
) != QUEUE_SCHEDULED
4124 && desc
!= NULL
&& desc
->insn
== pro
)
4125 apply_replacement (dep
, false);
4128 /* Go through and resolve forward dependencies. */
4129 for (sd_it
= sd_iterator_start (insn
, SD_LIST_FORW
);
4130 sd_iterator_cond (&sd_it
, &dep
);)
4132 rtx_insn
*next
= DEP_CON (dep
);
4133 bool cancelled
= (DEP_STATUS (dep
) & DEP_CANCELLED
) != 0;
4135 /* Resolve the dependence between INSN and NEXT.
4136 sd_resolve_dep () moves current dep to another list thus
4137 advancing the iterator. */
4138 sd_resolve_dep (sd_it
);
4142 if (must_restore_pattern_p (next
, dep
))
4143 restore_pattern (dep
, false);
4147 /* Don't bother trying to mark next as ready if insn is a debug
4148 insn. If insn is the last hard dependency, it will have
4149 already been discounted. */
4150 if (DEBUG_INSN_P (insn
) && !DEBUG_INSN_P (next
))
4153 if (!IS_SPECULATION_BRANCHY_CHECK_P (insn
))
4157 effective_cost
= try_ready (next
);
4159 if (effective_cost
>= 0
4160 && SCHED_GROUP_P (next
)
4161 && advance
< effective_cost
)
4162 advance
= effective_cost
;
4165 /* Check always has only one forward dependence (to the first insn in
4166 the recovery block), therefore, this will be executed only once. */
4168 gcc_assert (sd_lists_empty_p (insn
, SD_LIST_FORW
));
4169 fix_recovery_deps (RECOVERY_BLOCK (insn
));
4173 /* Annotate the instruction with issue information -- TImode
4174 indicates that the instruction is expected not to be able
4175 to issue on the same cycle as the previous insn. A machine
4176 may use this information to decide how the instruction should
4179 && GET_CODE (PATTERN (insn
)) != USE
4180 && GET_CODE (PATTERN (insn
)) != CLOBBER
4181 && !DEBUG_INSN_P (insn
))
4183 if (reload_completed
)
4184 PUT_MODE (insn
, clock_var
> last_clock_var
? TImode
: VOIDmode
);
4185 last_clock_var
= clock_var
;
4188 if (nonscheduled_insns_begin
!= NULL_RTX
)
4189 /* Indicate to debug counters that INSN is scheduled. */
4190 nonscheduled_insns_begin
= insn
;
4195 /* Functions for handling of notes. */
4197 /* Add note list that ends on FROM_END to the end of TO_ENDP. */
4199 concat_note_lists (rtx_insn
*from_end
, rtx_insn
**to_endp
)
4201 rtx_insn
*from_start
;
4203 /* It's easy when have nothing to concat. */
4204 if (from_end
== NULL
)
4207 /* It's also easy when destination is empty. */
4208 if (*to_endp
== NULL
)
4210 *to_endp
= from_end
;
4214 from_start
= from_end
;
4215 while (PREV_INSN (from_start
) != NULL
)
4216 from_start
= PREV_INSN (from_start
);
4218 SET_PREV_INSN (from_start
) = *to_endp
;
4219 SET_NEXT_INSN (*to_endp
) = from_start
;
4220 *to_endp
= from_end
;
4223 /* Delete notes between HEAD and TAIL and put them in the chain
4224 of notes ended by NOTE_LIST. */
4226 remove_notes (rtx_insn
*head
, rtx_insn
*tail
)
4228 rtx_insn
*next_tail
, *insn
, *next
;
4231 if (head
== tail
&& !INSN_P (head
))
4234 next_tail
= NEXT_INSN (tail
);
4235 for (insn
= head
; insn
!= next_tail
; insn
= next
)
4237 next
= NEXT_INSN (insn
);
4241 switch (NOTE_KIND (insn
))
4243 case NOTE_INSN_BASIC_BLOCK
:
4246 case NOTE_INSN_EPILOGUE_BEG
:
4250 add_reg_note (next
, REG_SAVE_NOTE
,
4251 GEN_INT (NOTE_INSN_EPILOGUE_BEG
));
4259 /* Add the note to list that ends at NOTE_LIST. */
4260 SET_PREV_INSN (insn
) = note_list
;
4261 SET_NEXT_INSN (insn
) = NULL_RTX
;
4263 SET_NEXT_INSN (note_list
) = insn
;
4268 gcc_assert ((sel_sched_p () || insn
!= tail
) && insn
!= head
);
4272 /* A structure to record enough data to allow us to backtrack the scheduler to
4273 a previous state. */
4274 struct haifa_saved_data
4276 /* Next entry on the list. */
4277 struct haifa_saved_data
*next
;
4279 /* Backtracking is associated with scheduling insns that have delay slots.
4280 DELAY_PAIR points to the structure that contains the insns involved, and
4281 the number of cycles between them. */
4282 struct delay_pair
*delay_pair
;
4284 /* Data used by the frontend (e.g. sched-ebb or sched-rgn). */
4285 void *fe_saved_data
;
4286 /* Data used by the backend. */
4287 void *be_saved_data
;
4289 /* Copies of global state. */
4290 int clock_var
, last_clock_var
;
4291 struct ready_list ready
;
4294 rtx_insn
*last_scheduled_insn
;
4295 rtx_insn
*last_nondebug_scheduled_insn
;
4296 rtx_insn
*nonscheduled_insns_begin
;
4297 int cycle_issued_insns
;
4299 /* Copies of state used in the inner loop of schedule_block. */
4300 struct sched_block_state sched_block
;
4302 /* We don't need to save q_ptr, as its value is arbitrary and we can set it
4303 to 0 when restoring. */
4305 rtx_insn_list
**insn_queue
;
4307 /* Describe pattern replacements that occurred since this backtrack point
4309 vec
<dep_t
> replacement_deps
;
4310 vec
<int> replace_apply
;
4312 /* A copy of the next-cycle replacement vectors at the time of the backtrack
4314 vec
<dep_t
> next_cycle_deps
;
4315 vec
<int> next_cycle_apply
;
4318 /* A record, in reverse order, of all scheduled insns which have delay slots
4319 and may require backtracking. */
4320 static struct haifa_saved_data
*backtrack_queue
;
4322 /* For every dependency of INSN, set the FEEDS_BACKTRACK_INSN bit according
4325 mark_backtrack_feeds (rtx_insn
*insn
, int set_p
)
4327 sd_iterator_def sd_it
;
4329 FOR_EACH_DEP (insn
, SD_LIST_HARD_BACK
, sd_it
, dep
)
4331 FEEDS_BACKTRACK_INSN (DEP_PRO (dep
)) = set_p
;
4335 /* Save the current scheduler state so that we can backtrack to it
4336 later if necessary. PAIR gives the insns that make it necessary to
4337 save this point. SCHED_BLOCK is the local state of schedule_block
4338 that need to be saved. */
4340 save_backtrack_point (struct delay_pair
*pair
,
4341 struct sched_block_state sched_block
)
4344 struct haifa_saved_data
*save
= XNEW (struct haifa_saved_data
);
4346 save
->curr_state
= xmalloc (dfa_state_size
);
4347 memcpy (save
->curr_state
, curr_state
, dfa_state_size
);
4349 save
->ready
.first
= ready
.first
;
4350 save
->ready
.n_ready
= ready
.n_ready
;
4351 save
->ready
.n_debug
= ready
.n_debug
;
4352 save
->ready
.veclen
= ready
.veclen
;
4353 save
->ready
.vec
= XNEWVEC (rtx_insn
*, ready
.veclen
);
4354 memcpy (save
->ready
.vec
, ready
.vec
, ready
.veclen
* sizeof (rtx
));
4356 save
->insn_queue
= XNEWVEC (rtx_insn_list
*, max_insn_queue_index
+ 1);
4357 save
->q_size
= q_size
;
4358 for (i
= 0; i
<= max_insn_queue_index
; i
++)
4360 int q
= NEXT_Q_AFTER (q_ptr
, i
);
4361 save
->insn_queue
[i
] = copy_INSN_LIST (insn_queue
[q
]);
4364 save
->clock_var
= clock_var
;
4365 save
->last_clock_var
= last_clock_var
;
4366 save
->cycle_issued_insns
= cycle_issued_insns
;
4367 save
->last_scheduled_insn
= last_scheduled_insn
;
4368 save
->last_nondebug_scheduled_insn
= last_nondebug_scheduled_insn
;
4369 save
->nonscheduled_insns_begin
= nonscheduled_insns_begin
;
4371 save
->sched_block
= sched_block
;
4373 save
->replacement_deps
.create (0);
4374 save
->replace_apply
.create (0);
4375 save
->next_cycle_deps
= next_cycle_replace_deps
.copy ();
4376 save
->next_cycle_apply
= next_cycle_apply
.copy ();
4378 if (current_sched_info
->save_state
)
4379 save
->fe_saved_data
= (*current_sched_info
->save_state
) ();
4381 if (targetm
.sched
.alloc_sched_context
)
4383 save
->be_saved_data
= targetm
.sched
.alloc_sched_context ();
4384 targetm
.sched
.init_sched_context (save
->be_saved_data
, false);
4387 save
->be_saved_data
= NULL
;
4389 save
->delay_pair
= pair
;
4391 save
->next
= backtrack_queue
;
4392 backtrack_queue
= save
;
4396 mark_backtrack_feeds (pair
->i2
, 1);
4397 INSN_TICK (pair
->i2
) = INVALID_TICK
;
4398 INSN_EXACT_TICK (pair
->i2
) = clock_var
+ pair_delay (pair
);
4399 SHADOW_P (pair
->i2
) = pair
->stages
== 0;
4400 pair
= pair
->next_same_i1
;
4404 /* Walk the ready list and all queues. If any insns have unresolved backwards
4405 dependencies, these must be cancelled deps, broken by predication. Set or
4406 clear (depending on SET) the DEP_CANCELLED bit in DEP_STATUS. */
4409 toggle_cancelled_flags (bool set
)
4412 sd_iterator_def sd_it
;
4415 if (ready
.n_ready
> 0)
4417 rtx_insn
**first
= ready_lastpos (&ready
);
4418 for (i
= 0; i
< ready
.n_ready
; i
++)
4419 FOR_EACH_DEP (first
[i
], SD_LIST_BACK
, sd_it
, dep
)
4420 if (!DEBUG_INSN_P (DEP_PRO (dep
)))
4423 DEP_STATUS (dep
) |= DEP_CANCELLED
;
4425 DEP_STATUS (dep
) &= ~DEP_CANCELLED
;
4428 for (i
= 0; i
<= max_insn_queue_index
; i
++)
4430 int q
= NEXT_Q_AFTER (q_ptr
, i
);
4431 rtx_insn_list
*link
;
4432 for (link
= insn_queue
[q
]; link
; link
= link
->next ())
4434 rtx_insn
*insn
= link
->insn ();
4435 FOR_EACH_DEP (insn
, SD_LIST_BACK
, sd_it
, dep
)
4436 if (!DEBUG_INSN_P (DEP_PRO (dep
)))
4439 DEP_STATUS (dep
) |= DEP_CANCELLED
;
4441 DEP_STATUS (dep
) &= ~DEP_CANCELLED
;
4447 /* Undo the replacements that have occurred after backtrack point SAVE
4450 undo_replacements_for_backtrack (struct haifa_saved_data
*save
)
4452 while (!save
->replacement_deps
.is_empty ())
4454 dep_t dep
= save
->replacement_deps
.pop ();
4455 int apply_p
= save
->replace_apply
.pop ();
4458 restore_pattern (dep
, true);
4460 apply_replacement (dep
, true);
4462 save
->replacement_deps
.release ();
4463 save
->replace_apply
.release ();
4466 /* Pop entries from the SCHEDULED_INSNS vector up to and including INSN.
4467 Restore their dependencies to an unresolved state, and mark them as
4471 unschedule_insns_until (rtx_insn
*insn
)
4473 auto_vec
<rtx_insn
*> recompute_vec
;
4475 /* Make two passes over the insns to be unscheduled. First, we clear out
4476 dependencies and other trivial bookkeeping. */
4480 sd_iterator_def sd_it
;
4483 last
= scheduled_insns
.pop ();
4485 /* This will be changed by restore_backtrack_point if the insn is in
4487 QUEUE_INDEX (last
) = QUEUE_NOWHERE
;
4489 INSN_TICK (last
) = INVALID_TICK
;
4491 if (modulo_ii
> 0 && INSN_UID (last
) < modulo_iter0_max_uid
)
4492 modulo_insns_scheduled
--;
4494 for (sd_it
= sd_iterator_start (last
, SD_LIST_RES_FORW
);
4495 sd_iterator_cond (&sd_it
, &dep
);)
4497 rtx_insn
*con
= DEP_CON (dep
);
4498 sd_unresolve_dep (sd_it
);
4499 if (!MUST_RECOMPUTE_SPEC_P (con
))
4501 MUST_RECOMPUTE_SPEC_P (con
) = 1;
4502 recompute_vec
.safe_push (con
);
4510 /* A second pass, to update ready and speculation status for insns
4511 depending on the unscheduled ones. The first pass must have
4512 popped the scheduled_insns vector up to the point where we
4513 restart scheduling, as recompute_todo_spec requires it to be
4515 while (!recompute_vec
.is_empty ())
4519 con
= recompute_vec
.pop ();
4520 MUST_RECOMPUTE_SPEC_P (con
) = 0;
4521 if (!sd_lists_empty_p (con
, SD_LIST_HARD_BACK
))
4523 TODO_SPEC (con
) = HARD_DEP
;
4524 INSN_TICK (con
) = INVALID_TICK
;
4525 if (PREDICATED_PAT (con
) != NULL_RTX
)
4526 haifa_change_pattern (con
, ORIG_PAT (con
));
4528 else if (QUEUE_INDEX (con
) != QUEUE_SCHEDULED
)
4529 TODO_SPEC (con
) = recompute_todo_spec (con
, true);
4533 /* Restore scheduler state from the topmost entry on the backtracking queue.
4534 PSCHED_BLOCK_P points to the local data of schedule_block that we must
4535 overwrite with the saved data.
4536 The caller must already have called unschedule_insns_until. */
4539 restore_last_backtrack_point (struct sched_block_state
*psched_block
)
4542 struct haifa_saved_data
*save
= backtrack_queue
;
4544 backtrack_queue
= save
->next
;
4546 if (current_sched_info
->restore_state
)
4547 (*current_sched_info
->restore_state
) (save
->fe_saved_data
);
4549 if (targetm
.sched
.alloc_sched_context
)
4551 targetm
.sched
.set_sched_context (save
->be_saved_data
);
4552 targetm
.sched
.free_sched_context (save
->be_saved_data
);
4555 /* Do this first since it clobbers INSN_TICK of the involved
4557 undo_replacements_for_backtrack (save
);
4559 /* Clear the QUEUE_INDEX of everything in the ready list or one
4561 if (ready
.n_ready
> 0)
4563 rtx_insn
**first
= ready_lastpos (&ready
);
4564 for (i
= 0; i
< ready
.n_ready
; i
++)
4566 rtx_insn
*insn
= first
[i
];
4567 QUEUE_INDEX (insn
) = QUEUE_NOWHERE
;
4568 INSN_TICK (insn
) = INVALID_TICK
;
4571 for (i
= 0; i
<= max_insn_queue_index
; i
++)
4573 int q
= NEXT_Q_AFTER (q_ptr
, i
);
4575 for (rtx_insn_list
*link
= insn_queue
[q
]; link
; link
= link
->next ())
4577 rtx_insn
*x
= link
->insn ();
4578 QUEUE_INDEX (x
) = QUEUE_NOWHERE
;
4579 INSN_TICK (x
) = INVALID_TICK
;
4581 free_INSN_LIST_list (&insn_queue
[q
]);
4585 ready
= save
->ready
;
4587 if (ready
.n_ready
> 0)
4589 rtx_insn
**first
= ready_lastpos (&ready
);
4590 for (i
= 0; i
< ready
.n_ready
; i
++)
4592 rtx_insn
*insn
= first
[i
];
4593 QUEUE_INDEX (insn
) = QUEUE_READY
;
4594 TODO_SPEC (insn
) = recompute_todo_spec (insn
, true);
4595 INSN_TICK (insn
) = save
->clock_var
;
4600 q_size
= save
->q_size
;
4601 for (i
= 0; i
<= max_insn_queue_index
; i
++)
4603 int q
= NEXT_Q_AFTER (q_ptr
, i
);
4605 insn_queue
[q
] = save
->insn_queue
[q
];
4607 for (rtx_insn_list
*link
= insn_queue
[q
]; link
; link
= link
->next ())
4609 rtx_insn
*x
= link
->insn ();
4610 QUEUE_INDEX (x
) = i
;
4611 TODO_SPEC (x
) = recompute_todo_spec (x
, true);
4612 INSN_TICK (x
) = save
->clock_var
+ i
;
4615 free (save
->insn_queue
);
4617 toggle_cancelled_flags (true);
4619 clock_var
= save
->clock_var
;
4620 last_clock_var
= save
->last_clock_var
;
4621 cycle_issued_insns
= save
->cycle_issued_insns
;
4622 last_scheduled_insn
= save
->last_scheduled_insn
;
4623 last_nondebug_scheduled_insn
= save
->last_nondebug_scheduled_insn
;
4624 nonscheduled_insns_begin
= save
->nonscheduled_insns_begin
;
4626 *psched_block
= save
->sched_block
;
4628 memcpy (curr_state
, save
->curr_state
, dfa_state_size
);
4629 free (save
->curr_state
);
4631 mark_backtrack_feeds (save
->delay_pair
->i2
, 0);
4633 gcc_assert (next_cycle_replace_deps
.is_empty ());
4634 next_cycle_replace_deps
= save
->next_cycle_deps
.copy ();
4635 next_cycle_apply
= save
->next_cycle_apply
.copy ();
4639 for (save
= backtrack_queue
; save
; save
= save
->next
)
4641 mark_backtrack_feeds (save
->delay_pair
->i2
, 1);
4645 /* Discard all data associated with the topmost entry in the backtrack
4646 queue. If RESET_TICK is false, we just want to free the data. If true,
4647 we are doing this because we discovered a reason to backtrack. In the
4648 latter case, also reset the INSN_TICK for the shadow insn. */
4650 free_topmost_backtrack_point (bool reset_tick
)
4652 struct haifa_saved_data
*save
= backtrack_queue
;
4655 backtrack_queue
= save
->next
;
4659 struct delay_pair
*pair
= save
->delay_pair
;
4662 INSN_TICK (pair
->i2
) = INVALID_TICK
;
4663 INSN_EXACT_TICK (pair
->i2
) = INVALID_TICK
;
4664 pair
= pair
->next_same_i1
;
4666 undo_replacements_for_backtrack (save
);
4670 save
->replacement_deps
.release ();
4671 save
->replace_apply
.release ();
4674 if (targetm
.sched
.free_sched_context
)
4675 targetm
.sched
.free_sched_context (save
->be_saved_data
);
4676 if (current_sched_info
->restore_state
)
4677 free (save
->fe_saved_data
);
4678 for (i
= 0; i
<= max_insn_queue_index
; i
++)
4679 free_INSN_LIST_list (&save
->insn_queue
[i
]);
4680 free (save
->insn_queue
);
4681 free (save
->curr_state
);
4682 free (save
->ready
.vec
);
4686 /* Free the entire backtrack queue. */
4688 free_backtrack_queue (void)
4690 while (backtrack_queue
)
4691 free_topmost_backtrack_point (false);
4694 /* Apply a replacement described by DESC. If IMMEDIATELY is false, we
4695 may have to postpone the replacement until the start of the next cycle,
4696 at which point we will be called again with IMMEDIATELY true. This is
4697 only done for machines which have instruction packets with explicit
4698 parallelism however. */
4700 apply_replacement (dep_t dep
, bool immediately
)
4702 struct dep_replacement
*desc
= DEP_REPLACE (dep
);
4703 if (!immediately
&& targetm
.sched
.exposed_pipeline
&& reload_completed
)
4705 next_cycle_replace_deps
.safe_push (dep
);
4706 next_cycle_apply
.safe_push (1);
4712 if (QUEUE_INDEX (desc
->insn
) == QUEUE_SCHEDULED
)
4715 if (sched_verbose
>= 5)
4716 fprintf (sched_dump
, "applying replacement for insn %d\n",
4717 INSN_UID (desc
->insn
));
4719 success
= validate_change (desc
->insn
, desc
->loc
, desc
->newval
, 0);
4720 gcc_assert (success
);
4722 update_insn_after_change (desc
->insn
);
4723 if ((TODO_SPEC (desc
->insn
) & (HARD_DEP
| DEP_POSTPONED
)) == 0)
4724 fix_tick_ready (desc
->insn
);
4726 if (backtrack_queue
!= NULL
)
4728 backtrack_queue
->replacement_deps
.safe_push (dep
);
4729 backtrack_queue
->replace_apply
.safe_push (1);
4734 /* We have determined that a pattern involved in DEP must be restored.
4735 If IMMEDIATELY is false, we may have to postpone the replacement
4736 until the start of the next cycle, at which point we will be called
4737 again with IMMEDIATELY true. */
4739 restore_pattern (dep_t dep
, bool immediately
)
4741 rtx_insn
*next
= DEP_CON (dep
);
4742 int tick
= INSN_TICK (next
);
4744 /* If we already scheduled the insn, the modified version is
4746 if (QUEUE_INDEX (next
) == QUEUE_SCHEDULED
)
4749 if (!immediately
&& targetm
.sched
.exposed_pipeline
&& reload_completed
)
4751 next_cycle_replace_deps
.safe_push (dep
);
4752 next_cycle_apply
.safe_push (0);
4757 if (DEP_TYPE (dep
) == REG_DEP_CONTROL
)
4759 if (sched_verbose
>= 5)
4760 fprintf (sched_dump
, "restoring pattern for insn %d\n",
4762 haifa_change_pattern (next
, ORIG_PAT (next
));
4766 struct dep_replacement
*desc
= DEP_REPLACE (dep
);
4769 if (sched_verbose
>= 5)
4770 fprintf (sched_dump
, "restoring pattern for insn %d\n",
4771 INSN_UID (desc
->insn
));
4772 tick
= INSN_TICK (desc
->insn
);
4774 success
= validate_change (desc
->insn
, desc
->loc
, desc
->orig
, 0);
4775 gcc_assert (success
);
4776 update_insn_after_change (desc
->insn
);
4777 if (backtrack_queue
!= NULL
)
4779 backtrack_queue
->replacement_deps
.safe_push (dep
);
4780 backtrack_queue
->replace_apply
.safe_push (0);
4783 INSN_TICK (next
) = tick
;
4784 if (TODO_SPEC (next
) == DEP_POSTPONED
)
4787 if (sd_lists_empty_p (next
, SD_LIST_BACK
))
4788 TODO_SPEC (next
) = 0;
4789 else if (!sd_lists_empty_p (next
, SD_LIST_HARD_BACK
))
4790 TODO_SPEC (next
) = HARD_DEP
;
4793 /* Perform pattern replacements that were queued up until the next
4796 perform_replacements_new_cycle (void)
4800 FOR_EACH_VEC_ELT (next_cycle_replace_deps
, i
, dep
)
4802 int apply_p
= next_cycle_apply
[i
];
4804 apply_replacement (dep
, true);
4806 restore_pattern (dep
, true);
4808 next_cycle_replace_deps
.truncate (0);
4809 next_cycle_apply
.truncate (0);
4812 /* Compute INSN_TICK_ESTIMATE for INSN. PROCESSED is a bitmap of
4813 instructions we've previously encountered, a set bit prevents
4814 recursion. BUDGET is a limit on how far ahead we look, it is
4815 reduced on recursive calls. Return true if we produced a good
4816 estimate, or false if we exceeded the budget. */
4818 estimate_insn_tick (bitmap processed
, rtx_insn
*insn
, int budget
)
4820 sd_iterator_def sd_it
;
4822 int earliest
= INSN_TICK (insn
);
4824 FOR_EACH_DEP (insn
, SD_LIST_BACK
, sd_it
, dep
)
4826 rtx_insn
*pro
= DEP_PRO (dep
);
4829 if (DEP_STATUS (dep
) & DEP_CANCELLED
)
4832 if (QUEUE_INDEX (pro
) == QUEUE_SCHEDULED
)
4833 gcc_assert (INSN_TICK (pro
) + dep_cost (dep
) <= INSN_TICK (insn
));
4836 int cost
= dep_cost (dep
);
4839 if (!bitmap_bit_p (processed
, INSN_LUID (pro
)))
4841 if (!estimate_insn_tick (processed
, pro
, budget
- cost
))
4844 gcc_assert (INSN_TICK_ESTIMATE (pro
) != INVALID_TICK
);
4845 t
= INSN_TICK_ESTIMATE (pro
) + cost
;
4846 if (earliest
== INVALID_TICK
|| t
> earliest
)
4850 bitmap_set_bit (processed
, INSN_LUID (insn
));
4851 INSN_TICK_ESTIMATE (insn
) = earliest
;
4855 /* Examine the pair of insns in P, and estimate (optimistically, assuming
4856 infinite resources) the cycle in which the delayed shadow can be issued.
4857 Return the number of cycles that must pass before the real insn can be
4858 issued in order to meet this constraint. */
4860 estimate_shadow_tick (struct delay_pair
*p
)
4862 bitmap_head processed
;
4865 bitmap_initialize (&processed
, 0);
4867 cutoff
= !estimate_insn_tick (&processed
, p
->i2
,
4868 max_insn_queue_index
+ pair_delay (p
));
4869 bitmap_clear (&processed
);
4871 return max_insn_queue_index
;
4872 t
= INSN_TICK_ESTIMATE (p
->i2
) - (clock_var
+ pair_delay (p
) + 1);
4878 /* If INSN has no unresolved backwards dependencies, add it to the schedule and
4879 recursively resolve all its forward dependencies. */
4881 resolve_dependencies (rtx_insn
*insn
)
4883 sd_iterator_def sd_it
;
4886 /* Don't use sd_lists_empty_p; it ignores debug insns. */
4887 if (DEPS_LIST_FIRST (INSN_HARD_BACK_DEPS (insn
)) != NULL
4888 || DEPS_LIST_FIRST (INSN_SPEC_BACK_DEPS (insn
)) != NULL
)
4891 if (sched_verbose
>= 4)
4892 fprintf (sched_dump
, ";;\tquickly resolving %d\n", INSN_UID (insn
));
4894 if (QUEUE_INDEX (insn
) >= 0)
4895 queue_remove (insn
);
4897 scheduled_insns
.safe_push (insn
);
4899 /* Update dependent instructions. */
4900 for (sd_it
= sd_iterator_start (insn
, SD_LIST_FORW
);
4901 sd_iterator_cond (&sd_it
, &dep
);)
4903 rtx_insn
*next
= DEP_CON (dep
);
4905 if (sched_verbose
>= 4)
4906 fprintf (sched_dump
, ";;\t\tdep %d against %d\n", INSN_UID (insn
),
4909 /* Resolve the dependence between INSN and NEXT.
4910 sd_resolve_dep () moves current dep to another list thus
4911 advancing the iterator. */
4912 sd_resolve_dep (sd_it
);
4914 if (!IS_SPECULATION_BRANCHY_CHECK_P (insn
))
4916 resolve_dependencies (next
);
4919 /* Check always has only one forward dependence (to the first insn in
4920 the recovery block), therefore, this will be executed only once. */
4922 gcc_assert (sd_lists_empty_p (insn
, SD_LIST_FORW
));
4928 /* Return the head and tail pointers of ebb starting at BEG and ending
4931 get_ebb_head_tail (basic_block beg
, basic_block end
,
4932 rtx_insn
**headp
, rtx_insn
**tailp
)
4934 rtx_insn
*beg_head
= BB_HEAD (beg
);
4935 rtx_insn
* beg_tail
= BB_END (beg
);
4936 rtx_insn
* end_head
= BB_HEAD (end
);
4937 rtx_insn
* end_tail
= BB_END (end
);
4939 /* Don't include any notes or labels at the beginning of the BEG
4940 basic block, or notes at the end of the END basic blocks. */
4942 if (LABEL_P (beg_head
))
4943 beg_head
= NEXT_INSN (beg_head
);
4945 while (beg_head
!= beg_tail
)
4946 if (NOTE_P (beg_head
))
4947 beg_head
= NEXT_INSN (beg_head
);
4948 else if (DEBUG_INSN_P (beg_head
))
4950 rtx_insn
* note
, *next
;
4952 for (note
= NEXT_INSN (beg_head
);
4956 next
= NEXT_INSN (note
);
4959 if (sched_verbose
>= 9)
4960 fprintf (sched_dump
, "reorder %i\n", INSN_UID (note
));
4962 reorder_insns_nobb (note
, note
, PREV_INSN (beg_head
));
4964 if (BLOCK_FOR_INSN (note
) != beg
)
4965 df_insn_change_bb (note
, beg
);
4967 else if (!DEBUG_INSN_P (note
))
4979 end_head
= beg_head
;
4980 else if (LABEL_P (end_head
))
4981 end_head
= NEXT_INSN (end_head
);
4983 while (end_head
!= end_tail
)
4984 if (NOTE_P (end_tail
))
4985 end_tail
= PREV_INSN (end_tail
);
4986 else if (DEBUG_INSN_P (end_tail
))
4988 rtx_insn
* note
, *prev
;
4990 for (note
= PREV_INSN (end_tail
);
4994 prev
= PREV_INSN (note
);
4997 if (sched_verbose
>= 9)
4998 fprintf (sched_dump
, "reorder %i\n", INSN_UID (note
));
5000 reorder_insns_nobb (note
, note
, end_tail
);
5002 if (end_tail
== BB_END (end
))
5003 BB_END (end
) = note
;
5005 if (BLOCK_FOR_INSN (note
) != end
)
5006 df_insn_change_bb (note
, end
);
5008 else if (!DEBUG_INSN_P (note
))
5020 /* Return nonzero if there are no real insns in the range [ HEAD, TAIL ]. */
5023 no_real_insns_p (const rtx_insn
*head
, const rtx_insn
*tail
)
5025 while (head
!= NEXT_INSN (tail
))
5027 if (!NOTE_P (head
) && !LABEL_P (head
))
5029 head
= NEXT_INSN (head
);
5034 /* Restore-other-notes: NOTE_LIST is the end of a chain of notes
5035 previously found among the insns. Insert them just before HEAD. */
5037 restore_other_notes (rtx_insn
*head
, basic_block head_bb
)
5041 rtx_insn
*note_head
= note_list
;
5044 head_bb
= BLOCK_FOR_INSN (head
);
5046 head
= NEXT_INSN (bb_note (head_bb
));
5048 while (PREV_INSN (note_head
))
5050 set_block_for_insn (note_head
, head_bb
);
5051 note_head
= PREV_INSN (note_head
);
5053 /* In the above cycle we've missed this note. */
5054 set_block_for_insn (note_head
, head_bb
);
5056 SET_PREV_INSN (note_head
) = PREV_INSN (head
);
5057 SET_NEXT_INSN (PREV_INSN (head
)) = note_head
;
5058 SET_PREV_INSN (head
) = note_list
;
5059 SET_NEXT_INSN (note_list
) = head
;
5061 if (BLOCK_FOR_INSN (head
) != head_bb
)
5062 BB_END (head_bb
) = note_list
;
5070 /* When we know we are going to discard the schedule due to a failed attempt
5071 at modulo scheduling, undo all replacements. */
5073 undo_all_replacements (void)
5078 FOR_EACH_VEC_ELT (scheduled_insns
, i
, insn
)
5080 sd_iterator_def sd_it
;
5083 /* See if we must undo a replacement. */
5084 for (sd_it
= sd_iterator_start (insn
, SD_LIST_RES_FORW
);
5085 sd_iterator_cond (&sd_it
, &dep
); sd_iterator_next (&sd_it
))
5087 struct dep_replacement
*desc
= DEP_REPLACE (dep
);
5089 validate_change (desc
->insn
, desc
->loc
, desc
->orig
, 0);
5094 /* Return first non-scheduled insn in the current scheduling block.
5095 This is mostly used for debug-counter purposes. */
5097 first_nonscheduled_insn (void)
5099 rtx_insn
*insn
= (nonscheduled_insns_begin
!= NULL_RTX
5100 ? nonscheduled_insns_begin
5101 : current_sched_info
->prev_head
);
5105 insn
= next_nonnote_nondebug_insn (insn
);
5107 while (QUEUE_INDEX (insn
) == QUEUE_SCHEDULED
);
5112 /* Move insns that became ready to fire from queue to ready list. */
5115 queue_to_ready (struct ready_list
*ready
)
5118 rtx_insn_list
*link
;
5119 rtx_insn
*skip_insn
;
5121 q_ptr
= NEXT_Q (q_ptr
);
5123 if (dbg_cnt (sched_insn
) == false)
5124 /* If debug counter is activated do not requeue the first
5125 nonscheduled insn. */
5126 skip_insn
= first_nonscheduled_insn ();
5130 /* Add all pending insns that can be scheduled without stalls to the
5132 for (link
= insn_queue
[q_ptr
]; link
; link
= link
->next ())
5134 insn
= link
->insn ();
5137 if (sched_verbose
>= 2)
5138 fprintf (sched_dump
, ";;\t\tQ-->Ready: insn %s: ",
5139 (*current_sched_info
->print_insn
) (insn
, 0));
5141 /* If the ready list is full, delay the insn for 1 cycle.
5142 See the comment in schedule_block for the rationale. */
5143 if (!reload_completed
5144 && (ready
->n_ready
- ready
->n_debug
> MAX_SCHED_READY_INSNS
5145 || (sched_pressure
== SCHED_PRESSURE_MODEL
5146 /* Limit pressure recalculations to MAX_SCHED_READY_INSNS
5147 instructions too. */
5148 && model_index (insn
) > (model_curr_point
5149 + MAX_SCHED_READY_INSNS
)))
5150 && !(sched_pressure
== SCHED_PRESSURE_MODEL
5151 && model_curr_point
< model_num_insns
5152 /* Always allow the next model instruction to issue. */
5153 && model_index (insn
) == model_curr_point
)
5154 && !SCHED_GROUP_P (insn
)
5155 && insn
!= skip_insn
)
5157 if (sched_verbose
>= 2)
5158 fprintf (sched_dump
, "keeping in queue, ready full\n");
5159 queue_insn (insn
, 1, "ready full");
5163 ready_add (ready
, insn
, false);
5164 if (sched_verbose
>= 2)
5165 fprintf (sched_dump
, "moving to ready without stalls\n");
5168 free_INSN_LIST_list (&insn_queue
[q_ptr
]);
5170 /* If there are no ready insns, stall until one is ready and add all
5171 of the pending insns at that point to the ready list. */
5172 if (ready
->n_ready
== 0)
5176 for (stalls
= 1; stalls
<= max_insn_queue_index
; stalls
++)
5178 if ((link
= insn_queue
[NEXT_Q_AFTER (q_ptr
, stalls
)]))
5180 for (; link
; link
= link
->next ())
5182 insn
= link
->insn ();
5185 if (sched_verbose
>= 2)
5186 fprintf (sched_dump
, ";;\t\tQ-->Ready: insn %s: ",
5187 (*current_sched_info
->print_insn
) (insn
, 0));
5189 ready_add (ready
, insn
, false);
5190 if (sched_verbose
>= 2)
5191 fprintf (sched_dump
, "moving to ready with %d stalls\n", stalls
);
5193 free_INSN_LIST_list (&insn_queue
[NEXT_Q_AFTER (q_ptr
, stalls
)]);
5195 advance_one_cycle ();
5200 advance_one_cycle ();
5203 q_ptr
= NEXT_Q_AFTER (q_ptr
, stalls
);
5204 clock_var
+= stalls
;
5205 if (sched_verbose
>= 2)
5206 fprintf (sched_dump
, ";;\tAdvancing clock by %d cycle[s] to %d\n",
5211 /* Used by early_queue_to_ready. Determines whether it is "ok" to
5212 prematurely move INSN from the queue to the ready list. Currently,
5213 if a target defines the hook 'is_costly_dependence', this function
5214 uses the hook to check whether there exist any dependences which are
5215 considered costly by the target, between INSN and other insns that
5216 have already been scheduled. Dependences are checked up to Y cycles
5217 back, with default Y=1; The flag -fsched-stalled-insns-dep=Y allows
5218 controlling this value.
5219 (Other considerations could be taken into account instead (or in
5220 addition) depending on user flags and target hooks. */
5223 ok_for_early_queue_removal (rtx_insn
*insn
)
5225 if (targetm
.sched
.is_costly_dependence
)
5228 int i
= scheduled_insns
.length ();
5229 for (n_cycles
= flag_sched_stalled_insns_dep
; n_cycles
; n_cycles
--)
5235 rtx_insn
*prev_insn
= scheduled_insns
[i
];
5237 if (!NOTE_P (prev_insn
))
5241 dep
= sd_find_dep_between (prev_insn
, insn
, true);
5245 cost
= dep_cost (dep
);
5247 if (targetm
.sched
.is_costly_dependence (dep
, cost
,
5248 flag_sched_stalled_insns_dep
- n_cycles
))
5253 if (GET_MODE (prev_insn
) == TImode
) /* end of dispatch group */
5266 /* Remove insns from the queue, before they become "ready" with respect
5267 to FU latency considerations. */
5270 early_queue_to_ready (state_t state
, struct ready_list
*ready
)
5273 rtx_insn_list
*link
;
5274 rtx_insn_list
*next_link
;
5275 rtx_insn_list
*prev_link
;
5278 state_t temp_state
= alloca (dfa_state_size
);
5280 int insns_removed
= 0;
5283 Flag '-fsched-stalled-insns=X' determines the aggressiveness of this
5286 X == 0: There is no limit on how many queued insns can be removed
5287 prematurely. (flag_sched_stalled_insns = -1).
5289 X >= 1: Only X queued insns can be removed prematurely in each
5290 invocation. (flag_sched_stalled_insns = X).
5292 Otherwise: Early queue removal is disabled.
5293 (flag_sched_stalled_insns = 0)
5296 if (! flag_sched_stalled_insns
)
5299 for (stalls
= 0; stalls
<= max_insn_queue_index
; stalls
++)
5301 if ((link
= insn_queue
[NEXT_Q_AFTER (q_ptr
, stalls
)]))
5303 if (sched_verbose
> 6)
5304 fprintf (sched_dump
, ";; look at index %d + %d\n", q_ptr
, stalls
);
5309 next_link
= link
->next ();
5310 insn
= link
->insn ();
5311 if (insn
&& sched_verbose
> 6)
5312 print_rtl_single (sched_dump
, insn
);
5314 memcpy (temp_state
, state
, dfa_state_size
);
5315 if (recog_memoized (insn
) < 0)
5316 /* non-negative to indicate that it's not ready
5317 to avoid infinite Q->R->Q->R... */
5320 cost
= state_transition (temp_state
, insn
);
5322 if (sched_verbose
>= 6)
5323 fprintf (sched_dump
, "transition cost = %d\n", cost
);
5325 move_to_ready
= false;
5328 move_to_ready
= ok_for_early_queue_removal (insn
);
5329 if (move_to_ready
== true)
5331 /* move from Q to R */
5333 ready_add (ready
, insn
, false);
5336 XEXP (prev_link
, 1) = next_link
;
5338 insn_queue
[NEXT_Q_AFTER (q_ptr
, stalls
)] = next_link
;
5340 free_INSN_LIST_node (link
);
5342 if (sched_verbose
>= 2)
5343 fprintf (sched_dump
, ";;\t\tEarly Q-->Ready: insn %s\n",
5344 (*current_sched_info
->print_insn
) (insn
, 0));
5347 if (insns_removed
== flag_sched_stalled_insns
)
5348 /* Remove no more than flag_sched_stalled_insns insns
5349 from Q at a time. */
5350 return insns_removed
;
5354 if (move_to_ready
== false)
5361 } /* for stalls.. */
5363 return insns_removed
;
5367 /* Print the ready list for debugging purposes.
5368 If READY_TRY is non-zero then only print insns that max_issue
5371 debug_ready_list_1 (struct ready_list
*ready
, signed char *ready_try
)
5376 if (ready
->n_ready
== 0)
5378 fprintf (sched_dump
, "\n");
5382 p
= ready_lastpos (ready
);
5383 for (i
= 0; i
< ready
->n_ready
; i
++)
5385 if (ready_try
!= NULL
&& ready_try
[ready
->n_ready
- i
- 1])
5388 fprintf (sched_dump
, " %s:%d",
5389 (*current_sched_info
->print_insn
) (p
[i
], 0),
5391 if (sched_pressure
!= SCHED_PRESSURE_NONE
)
5392 fprintf (sched_dump
, "(cost=%d",
5393 INSN_REG_PRESSURE_EXCESS_COST_CHANGE (p
[i
]));
5394 fprintf (sched_dump
, ":prio=%d", INSN_PRIORITY (p
[i
]));
5395 if (INSN_TICK (p
[i
]) > clock_var
)
5396 fprintf (sched_dump
, ":delay=%d", INSN_TICK (p
[i
]) - clock_var
);
5397 if (sched_pressure
== SCHED_PRESSURE_MODEL
)
5398 fprintf (sched_dump
, ":idx=%d",
5399 model_index (p
[i
]));
5400 if (sched_pressure
!= SCHED_PRESSURE_NONE
)
5401 fprintf (sched_dump
, ")");
5403 fprintf (sched_dump
, "\n");
5406 /* Print the ready list. Callable from debugger. */
5408 debug_ready_list (struct ready_list
*ready
)
5410 debug_ready_list_1 (ready
, NULL
);
5413 /* Search INSN for REG_SAVE_NOTE notes and convert them back into insn
5414 NOTEs. This is used for NOTE_INSN_EPILOGUE_BEG, so that sched-ebb
5415 replaces the epilogue note in the correct basic block. */
5417 reemit_notes (rtx_insn
*insn
)
5420 rtx_insn
*last
= insn
;
5422 for (note
= REG_NOTES (insn
); note
; note
= XEXP (note
, 1))
5424 if (REG_NOTE_KIND (note
) == REG_SAVE_NOTE
)
5426 enum insn_note note_type
= (enum insn_note
) INTVAL (XEXP (note
, 0));
5428 last
= emit_note_before (note_type
, last
);
5429 remove_note (insn
, note
);
5434 /* Move INSN. Reemit notes if needed. Update CFG, if needed. */
5436 move_insn (rtx_insn
*insn
, rtx_insn
*last
, rtx nt
)
5438 if (PREV_INSN (insn
) != last
)
5444 bb
= BLOCK_FOR_INSN (insn
);
5446 /* BB_HEAD is either LABEL or NOTE. */
5447 gcc_assert (BB_HEAD (bb
) != insn
);
5449 if (BB_END (bb
) == insn
)
5450 /* If this is last instruction in BB, move end marker one
5453 /* Jumps are always placed at the end of basic block. */
5454 jump_p
= control_flow_insn_p (insn
);
5457 || ((common_sched_info
->sched_pass_id
== SCHED_RGN_PASS
)
5458 && IS_SPECULATION_BRANCHY_CHECK_P (insn
))
5459 || (common_sched_info
->sched_pass_id
5460 == SCHED_EBB_PASS
));
5462 gcc_assert (BLOCK_FOR_INSN (PREV_INSN (insn
)) == bb
);
5464 BB_END (bb
) = PREV_INSN (insn
);
5467 gcc_assert (BB_END (bb
) != last
);
5470 /* We move the block note along with jump. */
5474 note
= NEXT_INSN (insn
);
5475 while (NOTE_NOT_BB_P (note
) && note
!= nt
)
5476 note
= NEXT_INSN (note
);
5480 || BARRIER_P (note
)))
5481 note
= NEXT_INSN (note
);
5483 gcc_assert (NOTE_INSN_BASIC_BLOCK_P (note
));
5488 SET_NEXT_INSN (PREV_INSN (insn
)) = NEXT_INSN (note
);
5489 SET_PREV_INSN (NEXT_INSN (note
)) = PREV_INSN (insn
);
5491 SET_NEXT_INSN (note
) = NEXT_INSN (last
);
5492 SET_PREV_INSN (NEXT_INSN (last
)) = note
;
5494 SET_NEXT_INSN (last
) = insn
;
5495 SET_PREV_INSN (insn
) = last
;
5497 bb
= BLOCK_FOR_INSN (last
);
5501 fix_jump_move (insn
);
5503 if (BLOCK_FOR_INSN (insn
) != bb
)
5504 move_block_after_check (insn
);
5506 gcc_assert (BB_END (bb
) == last
);
5509 df_insn_change_bb (insn
, bb
);
5511 /* Update BB_END, if needed. */
5512 if (BB_END (bb
) == last
)
5516 SCHED_GROUP_P (insn
) = 0;
5519 /* Return true if scheduling INSN will finish current clock cycle. */
5521 insn_finishes_cycle_p (rtx_insn
*insn
)
5523 if (SCHED_GROUP_P (insn
))
5524 /* After issuing INSN, rest of the sched_group will be forced to issue
5525 in order. Don't make any plans for the rest of cycle. */
5528 /* Finishing the block will, apparently, finish the cycle. */
5529 if (current_sched_info
->insn_finishes_block_p
5530 && current_sched_info
->insn_finishes_block_p (insn
))
5536 /* Helper for autopref_multipass_init. Given a SET in PAT and whether
5537 we're expecting a memory WRITE or not, check that the insn is relevant to
5538 the autoprefetcher modelling code. Return true iff that is the case.
5539 If it is relevant, record the base register of the memory op in BASE and
5540 the offset in OFFSET. */
5543 analyze_set_insn_for_autopref (rtx pat
, bool write
, rtx
*base
, int *offset
)
5545 if (GET_CODE (pat
) != SET
)
5548 rtx mem
= write
? SET_DEST (pat
) : SET_SRC (pat
);
5552 struct address_info info
;
5553 decompose_mem_address (&info
, mem
);
5555 /* TODO: Currently only (base+const) addressing is supported. */
5556 if (info
.base
== NULL
|| !REG_P (*info
.base
)
5557 || (info
.disp
!= NULL
&& !CONST_INT_P (*info
.disp
)))
5561 *offset
= info
.disp
? INTVAL (*info
.disp
) : 0;
5565 /* Functions to model cache auto-prefetcher.
5567 Some of the CPUs have cache auto-prefetcher, which /seems/ to initiate
5568 memory prefetches if it sees instructions with consequitive memory accesses
5569 in the instruction stream. Details of such hardware units are not published,
5570 so we can only guess what exactly is going on there.
5571 In the scheduler, we model abstract auto-prefetcher. If there are memory
5572 insns in the ready list (or the queue) that have same memory base, but
5573 different offsets, then we delay the insns with larger offsets until insns
5574 with smaller offsets get scheduled. If PARAM_SCHED_AUTOPREF_QUEUE_DEPTH
5575 is "1", then we look at the ready list; if it is N>1, then we also look
5576 through N-1 queue entries.
5577 If the param is N>=0, then rank_for_schedule will consider auto-prefetching
5578 among its heuristics.
5579 Param value of "-1" disables modelling of the auto-prefetcher. */
5581 /* Initialize autoprefetcher model data for INSN. */
5583 autopref_multipass_init (const rtx_insn
*insn
, int write
)
5585 autopref_multipass_data_t data
= &INSN_AUTOPREF_MULTIPASS_DATA (insn
)[write
];
5587 gcc_assert (data
->status
== AUTOPREF_MULTIPASS_DATA_UNINITIALIZED
);
5588 data
->base
= NULL_RTX
;
5589 data
->min_offset
= 0;
5590 data
->max_offset
= 0;
5591 data
->multi_mem_insn_p
= false;
5592 /* Set insn entry initialized, but not relevant for auto-prefetcher. */
5593 data
->status
= AUTOPREF_MULTIPASS_DATA_IRRELEVANT
;
5595 rtx pat
= PATTERN (insn
);
5597 /* We have a multi-set insn like a load-multiple or store-multiple.
5598 We care about these as long as all the memory ops inside the PARALLEL
5599 have the same base register. We care about the minimum and maximum
5600 offsets from that base but don't check for the order of those offsets
5601 within the PARALLEL insn itself. */
5602 if (GET_CODE (pat
) == PARALLEL
)
5604 int n_elems
= XVECLEN (pat
, 0);
5607 rtx prev_base
= NULL_RTX
;
5611 for (i
= 0; i
< n_elems
; i
++)
5613 rtx set
= XVECEXP (pat
, 0, i
);
5614 if (GET_CODE (set
) != SET
)
5617 rtx base
= NULL_RTX
;
5619 if (!analyze_set_insn_for_autopref (set
, write
, &base
, &offset
))
5625 min_offset
= offset
;
5626 max_offset
= offset
;
5628 /* Ensure that all memory operations in the PARALLEL use the same
5630 else if (REGNO (base
) != REGNO (prev_base
))
5634 min_offset
= MIN (min_offset
, offset
);
5635 max_offset
= MAX (max_offset
, offset
);
5639 /* If we reached here then we have a valid PARALLEL of multiple memory
5640 ops with prev_base as the base and min_offset and max_offset
5641 containing the offsets range. */
5642 gcc_assert (prev_base
);
5643 data
->base
= prev_base
;
5644 data
->min_offset
= min_offset
;
5645 data
->max_offset
= max_offset
;
5646 data
->multi_mem_insn_p
= true;
5647 data
->status
= AUTOPREF_MULTIPASS_DATA_NORMAL
;
5652 /* Otherwise this is a single set memory operation. */
5653 rtx set
= single_set (insn
);
5654 if (set
== NULL_RTX
)
5657 if (!analyze_set_insn_for_autopref (set
, write
, &data
->base
,
5661 /* This insn is relevant for the auto-prefetcher.
5662 The base and offset fields will have been filled in the
5663 analyze_set_insn_for_autopref call above. */
5664 data
->status
= AUTOPREF_MULTIPASS_DATA_NORMAL
;
5668 /* Helper for autopref_rank_for_schedule. Given the data of two
5669 insns relevant to the auto-prefetcher modelling code DATA1 and DATA2
5670 return their comparison result. Return 0 if there is no sensible
5671 ranking order for the two insns. */
5674 autopref_rank_data (autopref_multipass_data_t data1
,
5675 autopref_multipass_data_t data2
)
5677 /* Simple case when both insns are simple single memory ops. */
5678 if (!data1
->multi_mem_insn_p
&& !data2
->multi_mem_insn_p
)
5679 return data1
->min_offset
- data2
->min_offset
;
5681 /* Two load/store multiple insns. Return 0 if the offset ranges
5682 overlap and the difference between the minimum offsets otherwise. */
5683 else if (data1
->multi_mem_insn_p
&& data2
->multi_mem_insn_p
)
5685 int min1
= data1
->min_offset
;
5686 int max1
= data1
->max_offset
;
5687 int min2
= data2
->min_offset
;
5688 int max2
= data2
->max_offset
;
5690 if (max1
< min2
|| min1
> max2
)
5696 /* The other two cases is a pair of a load/store multiple and
5697 a simple memory op. Return 0 if the single op's offset is within the
5698 range of the multi-op insn and the difference between the single offset
5699 and the minimum offset of the multi-set insn otherwise. */
5700 else if (data1
->multi_mem_insn_p
&& !data2
->multi_mem_insn_p
)
5702 int max1
= data1
->max_offset
;
5703 int min1
= data1
->min_offset
;
5705 if (data2
->min_offset
>= min1
5706 && data2
->min_offset
<= max1
)
5709 return min1
- data2
->min_offset
;
5713 int max2
= data2
->max_offset
;
5714 int min2
= data2
->min_offset
;
5716 if (data1
->min_offset
>= min2
5717 && data1
->min_offset
<= max2
)
5720 return data1
->min_offset
- min2
;
5724 /* Helper function for rank_for_schedule sorting. */
5726 autopref_rank_for_schedule (const rtx_insn
*insn1
, const rtx_insn
*insn2
)
5728 for (int write
= 0; write
< 2; ++write
)
5730 autopref_multipass_data_t data1
5731 = &INSN_AUTOPREF_MULTIPASS_DATA (insn1
)[write
];
5732 autopref_multipass_data_t data2
5733 = &INSN_AUTOPREF_MULTIPASS_DATA (insn2
)[write
];
5735 if (data1
->status
== AUTOPREF_MULTIPASS_DATA_UNINITIALIZED
)
5736 autopref_multipass_init (insn1
, write
);
5737 if (data1
->status
== AUTOPREF_MULTIPASS_DATA_IRRELEVANT
)
5740 if (data2
->status
== AUTOPREF_MULTIPASS_DATA_UNINITIALIZED
)
5741 autopref_multipass_init (insn2
, write
);
5742 if (data2
->status
== AUTOPREF_MULTIPASS_DATA_IRRELEVANT
)
5745 if (!rtx_equal_p (data1
->base
, data2
->base
))
5748 return autopref_rank_data (data1
, data2
);
5754 /* True if header of debug dump was printed. */
5755 static bool autopref_multipass_dfa_lookahead_guard_started_dump_p
;
5757 /* Helper for autopref_multipass_dfa_lookahead_guard.
5758 Return "1" if INSN1 should be delayed in favor of INSN2. */
5760 autopref_multipass_dfa_lookahead_guard_1 (const rtx_insn
*insn1
,
5761 const rtx_insn
*insn2
, int write
)
5763 autopref_multipass_data_t data1
5764 = &INSN_AUTOPREF_MULTIPASS_DATA (insn1
)[write
];
5765 autopref_multipass_data_t data2
5766 = &INSN_AUTOPREF_MULTIPASS_DATA (insn2
)[write
];
5768 if (data2
->status
== AUTOPREF_MULTIPASS_DATA_UNINITIALIZED
)
5769 autopref_multipass_init (insn2
, write
);
5770 if (data2
->status
== AUTOPREF_MULTIPASS_DATA_IRRELEVANT
)
5773 if (rtx_equal_p (data1
->base
, data2
->base
)
5774 && autopref_rank_data (data1
, data2
) > 0)
5776 if (sched_verbose
>= 2)
5778 if (!autopref_multipass_dfa_lookahead_guard_started_dump_p
)
5780 fprintf (sched_dump
,
5781 ";;\t\tnot trying in max_issue due to autoprefetch "
5783 autopref_multipass_dfa_lookahead_guard_started_dump_p
= true;
5786 fprintf (sched_dump
, " %d(%d)", INSN_UID (insn1
), INSN_UID (insn2
));
5797 We could have also hooked autoprefetcher model into
5798 first_cycle_multipass_backtrack / first_cycle_multipass_issue hooks
5799 to enable intelligent selection of "[r1+0]=r2; [r1+4]=r3" on the same cycle
5800 (e.g., once "[r1+0]=r2" is issued in max_issue(), "[r1+4]=r3" gets
5801 unblocked). We don't bother about this yet because target of interest
5802 (ARM Cortex-A15) can issue only 1 memory operation per cycle. */
5804 /* Implementation of first_cycle_multipass_dfa_lookahead_guard hook.
5805 Return "1" if INSN1 should not be considered in max_issue due to
5806 auto-prefetcher considerations. */
5808 autopref_multipass_dfa_lookahead_guard (rtx_insn
*insn1
, int ready_index
)
5812 if (PARAM_VALUE (PARAM_SCHED_AUTOPREF_QUEUE_DEPTH
) <= 0)
5815 if (sched_verbose
>= 2 && ready_index
== 0)
5816 autopref_multipass_dfa_lookahead_guard_started_dump_p
= false;
5818 for (int write
= 0; write
< 2; ++write
)
5820 autopref_multipass_data_t data1
5821 = &INSN_AUTOPREF_MULTIPASS_DATA (insn1
)[write
];
5823 if (data1
->status
== AUTOPREF_MULTIPASS_DATA_UNINITIALIZED
)
5824 autopref_multipass_init (insn1
, write
);
5825 if (data1
->status
== AUTOPREF_MULTIPASS_DATA_IRRELEVANT
)
5828 if (ready_index
== 0
5829 && data1
->status
== AUTOPREF_MULTIPASS_DATA_DONT_DELAY
)
5830 /* We allow only a single delay on priviledged instructions.
5831 Doing otherwise would cause infinite loop. */
5833 if (sched_verbose
>= 2)
5835 if (!autopref_multipass_dfa_lookahead_guard_started_dump_p
)
5837 fprintf (sched_dump
,
5838 ";;\t\tnot trying in max_issue due to autoprefetch "
5840 autopref_multipass_dfa_lookahead_guard_started_dump_p
= true;
5843 fprintf (sched_dump
, " *%d*", INSN_UID (insn1
));
5848 for (int i2
= 0; i2
< ready
.n_ready
; ++i2
)
5850 rtx_insn
*insn2
= get_ready_element (i2
);
5853 r
= autopref_multipass_dfa_lookahead_guard_1 (insn1
, insn2
, write
);
5856 if (ready_index
== 0)
5859 data1
->status
= AUTOPREF_MULTIPASS_DATA_DONT_DELAY
;
5865 if (PARAM_VALUE (PARAM_SCHED_AUTOPREF_QUEUE_DEPTH
) == 1)
5868 /* Everything from the current queue slot should have been moved to
5870 gcc_assert (insn_queue
[NEXT_Q_AFTER (q_ptr
, 0)] == NULL_RTX
);
5872 int n_stalls
= PARAM_VALUE (PARAM_SCHED_AUTOPREF_QUEUE_DEPTH
) - 1;
5873 if (n_stalls
> max_insn_queue_index
)
5874 n_stalls
= max_insn_queue_index
;
5876 for (int stalls
= 1; stalls
<= n_stalls
; ++stalls
)
5878 for (rtx_insn_list
*link
= insn_queue
[NEXT_Q_AFTER (q_ptr
, stalls
)];
5880 link
= link
->next ())
5882 rtx_insn
*insn2
= link
->insn ();
5883 r
= autopref_multipass_dfa_lookahead_guard_1 (insn1
, insn2
,
5887 /* Queue INSN1 until INSN2 can issue. */
5889 if (ready_index
== 0)
5890 data1
->status
= AUTOPREF_MULTIPASS_DATA_DONT_DELAY
;
5898 if (sched_verbose
>= 2
5899 && autopref_multipass_dfa_lookahead_guard_started_dump_p
5900 && (ready_index
== ready
.n_ready
- 1 || r
< 0))
5901 /* This does not /always/ trigger. We don't output EOL if the last
5902 insn is not recognized (INSN_CODE < 0) and lookahead_guard is not
5903 called. We can live with this. */
5904 fprintf (sched_dump
, "\n");
5909 /* Define type for target data used in multipass scheduling. */
5910 #ifndef TARGET_SCHED_FIRST_CYCLE_MULTIPASS_DATA_T
5911 # define TARGET_SCHED_FIRST_CYCLE_MULTIPASS_DATA_T int
5913 typedef TARGET_SCHED_FIRST_CYCLE_MULTIPASS_DATA_T first_cycle_multipass_data_t
;
5915 /* The following structure describe an entry of the stack of choices. */
5918 /* Ordinal number of the issued insn in the ready queue. */
5920 /* The number of the rest insns whose issues we should try. */
5922 /* The number of issued essential insns. */
5924 /* State after issuing the insn. */
5926 /* Target-specific data. */
5927 first_cycle_multipass_data_t target_data
;
5930 /* The following array is used to implement a stack of choices used in
5931 function max_issue. */
5932 static struct choice_entry
*choice_stack
;
5934 /* This holds the value of the target dfa_lookahead hook. */
5937 /* The following variable value is maximal number of tries of issuing
5938 insns for the first cycle multipass insn scheduling. We define
5939 this value as constant*(DFA_LOOKAHEAD**ISSUE_RATE). We would not
5940 need this constraint if all real insns (with non-negative codes)
5941 had reservations because in this case the algorithm complexity is
5942 O(DFA_LOOKAHEAD**ISSUE_RATE). Unfortunately, the dfa descriptions
5943 might be incomplete and such insn might occur. For such
5944 descriptions, the complexity of algorithm (without the constraint)
5945 could achieve DFA_LOOKAHEAD ** N , where N is the queue length. */
5946 static int max_lookahead_tries
;
5948 /* The following function returns maximal (or close to maximal) number
5949 of insns which can be issued on the same cycle and one of which
5950 insns is insns with the best rank (the first insn in READY). To
5951 make this function tries different samples of ready insns. READY
5952 is current queue `ready'. Global array READY_TRY reflects what
5953 insns are already issued in this try. The function stops immediately,
5954 if it reached the such a solution, that all instruction can be issued.
5955 INDEX will contain index of the best insn in READY. The following
5956 function is used only for first cycle multipass scheduling.
5960 This function expects recognized insns only. All USEs,
5961 CLOBBERs, etc must be filtered elsewhere. */
5963 max_issue (struct ready_list
*ready
, int privileged_n
, state_t state
,
5964 bool first_cycle_insn_p
, int *index
)
5966 int n
, i
, all
, n_ready
, best
, delay
, tries_num
;
5968 struct choice_entry
*top
;
5974 n_ready
= ready
->n_ready
;
5975 gcc_assert (dfa_lookahead
>= 1 && privileged_n
>= 0
5976 && privileged_n
<= n_ready
);
5978 /* Init MAX_LOOKAHEAD_TRIES. */
5979 if (max_lookahead_tries
== 0)
5981 max_lookahead_tries
= 100;
5982 for (i
= 0; i
< issue_rate
; i
++)
5983 max_lookahead_tries
*= dfa_lookahead
;
5986 /* Init max_points. */
5987 more_issue
= issue_rate
- cycle_issued_insns
;
5988 gcc_assert (more_issue
>= 0);
5990 /* The number of the issued insns in the best solution. */
5995 /* Set initial state of the search. */
5996 memcpy (top
->state
, state
, dfa_state_size
);
5997 top
->rest
= dfa_lookahead
;
5999 if (targetm
.sched
.first_cycle_multipass_begin
)
6000 targetm
.sched
.first_cycle_multipass_begin (&top
->target_data
,
6002 first_cycle_insn_p
);
6004 /* Count the number of the insns to search among. */
6005 for (all
= i
= 0; i
< n_ready
; i
++)
6009 if (sched_verbose
>= 2)
6011 fprintf (sched_dump
, ";;\t\tmax_issue among %d insns:", all
);
6012 debug_ready_list_1 (ready
, ready_try
);
6015 /* I is the index of the insn to try next. */
6020 if (/* If we've reached a dead end or searched enough of what we have
6023 /* or have nothing else to try... */
6025 /* or should not issue more. */
6026 || top
->n
>= more_issue
)
6028 /* ??? (... || i == n_ready). */
6029 gcc_assert (i
<= n_ready
);
6031 /* We should not issue more than issue_rate instructions. */
6032 gcc_assert (top
->n
<= more_issue
);
6034 if (top
== choice_stack
)
6037 if (best
< top
- choice_stack
)
6042 /* Try to find issued privileged insn. */
6043 while (n
&& !ready_try
[--n
])
6047 if (/* If all insns are equally good... */
6049 /* Or a privileged insn will be issued. */
6051 /* Then we have a solution. */
6053 best
= top
- choice_stack
;
6054 /* This is the index of the insn issued first in this
6056 *index
= choice_stack
[1].index
;
6057 if (top
->n
== more_issue
|| best
== all
)
6062 /* Set ready-list index to point to the last insn
6063 ('i++' below will advance it to the next insn). */
6069 if (targetm
.sched
.first_cycle_multipass_backtrack
)
6070 targetm
.sched
.first_cycle_multipass_backtrack (&top
->target_data
,
6071 ready_try
, n_ready
);
6074 memcpy (state
, top
->state
, dfa_state_size
);
6076 else if (!ready_try
[i
])
6079 if (tries_num
> max_lookahead_tries
)
6081 insn
= ready_element (ready
, i
);
6082 delay
= state_transition (state
, insn
);
6085 if (state_dead_lock_p (state
)
6086 || insn_finishes_cycle_p (insn
))
6087 /* We won't issue any more instructions in the next
6094 if (memcmp (top
->state
, state
, dfa_state_size
) != 0)
6097 /* Advance to the next choice_entry. */
6099 /* Initialize it. */
6100 top
->rest
= dfa_lookahead
;
6103 memcpy (top
->state
, state
, dfa_state_size
);
6106 if (targetm
.sched
.first_cycle_multipass_issue
)
6107 targetm
.sched
.first_cycle_multipass_issue (&top
->target_data
,
6117 /* Increase ready-list index. */
6121 if (targetm
.sched
.first_cycle_multipass_end
)
6122 targetm
.sched
.first_cycle_multipass_end (best
!= 0
6123 ? &choice_stack
[1].target_data
6126 /* Restore the original state of the DFA. */
6127 memcpy (state
, choice_stack
->state
, dfa_state_size
);
6132 /* The following function chooses insn from READY and modifies
6133 READY. The following function is used only for first
6134 cycle multipass scheduling.
6136 -1 if cycle should be advanced,
6137 0 if INSN_PTR is set to point to the desirable insn,
6138 1 if choose_ready () should be restarted without advancing the cycle. */
6140 choose_ready (struct ready_list
*ready
, bool first_cycle_insn_p
,
6141 rtx_insn
**insn_ptr
)
6143 if (dbg_cnt (sched_insn
) == false)
6145 if (nonscheduled_insns_begin
== NULL_RTX
)
6146 nonscheduled_insns_begin
= current_sched_info
->prev_head
;
6148 rtx_insn
*insn
= first_nonscheduled_insn ();
6150 if (QUEUE_INDEX (insn
) == QUEUE_READY
)
6151 /* INSN is in the ready_list. */
6153 ready_remove_insn (insn
);
6158 /* INSN is in the queue. Advance cycle to move it to the ready list. */
6159 gcc_assert (QUEUE_INDEX (insn
) >= 0);
6163 if (dfa_lookahead
<= 0 || SCHED_GROUP_P (ready_element (ready
, 0))
6164 || DEBUG_INSN_P (ready_element (ready
, 0)))
6166 if (targetm
.sched
.dispatch (NULL
, IS_DISPATCH_ON
))
6167 *insn_ptr
= ready_remove_first_dispatch (ready
);
6169 *insn_ptr
= ready_remove_first (ready
);
6175 /* Try to choose the best insn. */
6179 insn
= ready_element (ready
, 0);
6180 if (INSN_CODE (insn
) < 0)
6182 *insn_ptr
= ready_remove_first (ready
);
6186 /* Filter the search space. */
6187 for (i
= 0; i
< ready
->n_ready
; i
++)
6191 insn
= ready_element (ready
, i
);
6193 /* If this insn is recognizable we should have already
6194 recognized it earlier.
6195 ??? Not very clear where this is supposed to be done.
6197 gcc_checking_assert (INSN_CODE (insn
) >= 0
6198 || recog_memoized (insn
) < 0);
6199 if (INSN_CODE (insn
) < 0)
6201 /* Non-recognized insns at position 0 are handled above. */
6207 if (targetm
.sched
.first_cycle_multipass_dfa_lookahead_guard
)
6210 = (targetm
.sched
.first_cycle_multipass_dfa_lookahead_guard
6213 if (ready_try
[i
] < 0)
6214 /* Queue instruction for several cycles.
6215 We need to restart choose_ready as we have changed
6218 change_queue_index (insn
, -ready_try
[i
]);
6222 /* Make sure that we didn't end up with 0'th insn filtered out.
6223 Don't be tempted to make life easier for backends and just
6224 requeue 0'th insn if (ready_try[0] == 0) and restart
6225 choose_ready. Backends should be very considerate about
6226 requeueing instructions -- especially the highest priority
6227 one at position 0. */
6228 gcc_assert (ready_try
[i
] == 0 || i
> 0);
6233 gcc_assert (ready_try
[i
] == 0);
6234 /* INSN made it through the scrutiny of filters! */
6237 if (max_issue (ready
, 1, curr_state
, first_cycle_insn_p
, &index
) == 0)
6239 *insn_ptr
= ready_remove_first (ready
);
6240 if (sched_verbose
>= 4)
6241 fprintf (sched_dump
, ";;\t\tChosen insn (but can't issue) : %s \n",
6242 (*current_sched_info
->print_insn
) (*insn_ptr
, 0));
6247 if (sched_verbose
>= 4)
6248 fprintf (sched_dump
, ";;\t\tChosen insn : %s\n",
6249 (*current_sched_info
->print_insn
)
6250 (ready_element (ready
, index
), 0));
6252 *insn_ptr
= ready_remove (ready
, index
);
6258 /* This function is called when we have successfully scheduled a
6259 block. It uses the schedule stored in the scheduled_insns vector
6260 to rearrange the RTL. PREV_HEAD is used as the anchor to which we
6261 append the scheduled insns; TAIL is the insn after the scheduled
6262 block. TARGET_BB is the argument passed to schedule_block. */
6265 commit_schedule (rtx_insn
*prev_head
, rtx_insn
*tail
, basic_block
*target_bb
)
6270 last_scheduled_insn
= prev_head
;
6272 scheduled_insns
.iterate (i
, &insn
);
6275 if (control_flow_insn_p (last_scheduled_insn
)
6276 || current_sched_info
->advance_target_bb (*target_bb
, insn
))
6278 *target_bb
= current_sched_info
->advance_target_bb (*target_bb
, 0);
6284 x
= next_real_insn (last_scheduled_insn
);
6286 dump_new_block_header (1, *target_bb
, x
, tail
);
6289 last_scheduled_insn
= bb_note (*target_bb
);
6292 if (current_sched_info
->begin_move_insn
)
6293 (*current_sched_info
->begin_move_insn
) (insn
, last_scheduled_insn
);
6294 move_insn (insn
, last_scheduled_insn
,
6295 current_sched_info
->next_tail
);
6296 if (!DEBUG_INSN_P (insn
))
6297 reemit_notes (insn
);
6298 last_scheduled_insn
= insn
;
6301 scheduled_insns
.truncate (0);
6304 /* Examine all insns on the ready list and queue those which can't be
6305 issued in this cycle. TEMP_STATE is temporary scheduler state we
6306 can use as scratch space. If FIRST_CYCLE_INSN_P is true, no insns
6307 have been issued for the current cycle, which means it is valid to
6308 issue an asm statement.
6310 If SHADOWS_ONLY_P is true, we eliminate all real insns and only
6311 leave those for which SHADOW_P is true. If MODULO_EPILOGUE is true,
6312 we only leave insns which have an INSN_EXACT_TICK. */
6315 prune_ready_list (state_t temp_state
, bool first_cycle_insn_p
,
6316 bool shadows_only_p
, bool modulo_epilogue_p
)
6319 bool sched_group_found
= false;
6320 int min_cost_group
= 1;
6325 for (i
= 0; i
< ready
.n_ready
; i
++)
6327 rtx_insn
*insn
= ready_element (&ready
, i
);
6328 if (SCHED_GROUP_P (insn
))
6330 sched_group_found
= true;
6335 /* Make two passes if there's a SCHED_GROUP_P insn; make sure to handle
6336 such an insn first and note its cost, then schedule all other insns
6337 for one cycle later. */
6338 for (pass
= sched_group_found
? 0 : 1; pass
< 2; )
6340 int n
= ready
.n_ready
;
6341 for (i
= 0; i
< n
; i
++)
6343 rtx_insn
*insn
= ready_element (&ready
, i
);
6345 const char *reason
= "resource conflict";
6347 if (DEBUG_INSN_P (insn
))
6350 if (sched_group_found
&& !SCHED_GROUP_P (insn
))
6354 cost
= min_cost_group
;
6355 reason
= "not in sched group";
6357 else if (modulo_epilogue_p
6358 && INSN_EXACT_TICK (insn
) == INVALID_TICK
)
6360 cost
= max_insn_queue_index
;
6361 reason
= "not an epilogue insn";
6363 else if (shadows_only_p
&& !SHADOW_P (insn
))
6366 reason
= "not a shadow";
6368 else if (recog_memoized (insn
) < 0)
6370 if (!first_cycle_insn_p
6371 && (GET_CODE (PATTERN (insn
)) == ASM_INPUT
6372 || asm_noperands (PATTERN (insn
)) >= 0))
6376 else if (sched_pressure
!= SCHED_PRESSURE_NONE
)
6378 if (sched_pressure
== SCHED_PRESSURE_MODEL
6379 && INSN_TICK (insn
) <= clock_var
)
6381 memcpy (temp_state
, curr_state
, dfa_state_size
);
6382 if (state_transition (temp_state
, insn
) >= 0)
6383 INSN_TICK (insn
) = clock_var
+ 1;
6393 struct delay_pair
*delay_entry
;
6395 = delay_htab
->find_with_hash (insn
,
6396 htab_hash_pointer (insn
));
6397 while (delay_entry
&& delay_cost
== 0)
6399 delay_cost
= estimate_shadow_tick (delay_entry
);
6400 if (delay_cost
> max_insn_queue_index
)
6401 delay_cost
= max_insn_queue_index
;
6402 delay_entry
= delay_entry
->next_same_i1
;
6406 memcpy (temp_state
, curr_state
, dfa_state_size
);
6407 cost
= state_transition (temp_state
, insn
);
6412 if (cost
< delay_cost
)
6415 reason
= "shadow tick";
6420 if (SCHED_GROUP_P (insn
) && cost
> min_cost_group
)
6421 min_cost_group
= cost
;
6422 ready_remove (&ready
, i
);
6423 /* Normally we'd want to queue INSN for COST cycles. However,
6424 if SCHED_GROUP_P is set, then we must ensure that nothing
6425 else comes between INSN and its predecessor. If there is
6426 some other insn ready to fire on the next cycle, then that
6427 invariant would be broken.
6429 So when SCHED_GROUP_P is set, just queue this insn for a
6431 queue_insn (insn
, SCHED_GROUP_P (insn
) ? 1 : cost
, reason
);
6441 /* Called when we detect that the schedule is impossible. We examine the
6442 backtrack queue to find the earliest insn that caused this condition. */
6444 static struct haifa_saved_data
*
6445 verify_shadows (void)
6447 struct haifa_saved_data
*save
, *earliest_fail
= NULL
;
6448 for (save
= backtrack_queue
; save
; save
= save
->next
)
6451 struct delay_pair
*pair
= save
->delay_pair
;
6452 rtx_insn
*i1
= pair
->i1
;
6454 for (; pair
; pair
= pair
->next_same_i1
)
6456 rtx_insn
*i2
= pair
->i2
;
6458 if (QUEUE_INDEX (i2
) == QUEUE_SCHEDULED
)
6461 t
= INSN_TICK (i1
) + pair_delay (pair
);
6464 if (sched_verbose
>= 2)
6465 fprintf (sched_dump
,
6466 ";;\t\tfailed delay requirements for %d/%d (%d->%d)"
6468 INSN_UID (pair
->i1
), INSN_UID (pair
->i2
),
6469 INSN_TICK (pair
->i1
), INSN_EXACT_TICK (pair
->i2
));
6470 earliest_fail
= save
;
6473 if (QUEUE_INDEX (i2
) >= 0)
6475 int queued_for
= INSN_TICK (i2
);
6479 if (sched_verbose
>= 2)
6480 fprintf (sched_dump
,
6481 ";;\t\tfailed delay requirements for %d/%d"
6482 " (%d->%d), queued too late\n",
6483 INSN_UID (pair
->i1
), INSN_UID (pair
->i2
),
6484 INSN_TICK (pair
->i1
), INSN_EXACT_TICK (pair
->i2
));
6485 earliest_fail
= save
;
6492 return earliest_fail
;
6495 /* Print instructions together with useful scheduling information between
6496 HEAD and TAIL (inclusive). */
6498 dump_insn_stream (rtx_insn
*head
, rtx_insn
*tail
)
6500 fprintf (sched_dump
, ";;\t| insn | prio |\n");
6502 rtx_insn
*next_tail
= NEXT_INSN (tail
);
6503 for (rtx_insn
*insn
= head
; insn
!= next_tail
; insn
= NEXT_INSN (insn
))
6505 int priority
= NOTE_P (insn
) ? 0 : INSN_PRIORITY (insn
);
6506 const char *pattern
= (NOTE_P (insn
)
6508 : str_pattern_slim (PATTERN (insn
)));
6510 fprintf (sched_dump
, ";;\t| %4d | %4d | %-30s ",
6511 INSN_UID (insn
), priority
, pattern
);
6513 if (sched_verbose
>= 4)
6515 if (NOTE_P (insn
) || recog_memoized (insn
) < 0)
6516 fprintf (sched_dump
, "nothing");
6518 print_reservation (sched_dump
, insn
);
6520 fprintf (sched_dump
, "\n");
6524 /* Use forward list scheduling to rearrange insns of block pointed to by
6525 TARGET_BB, possibly bringing insns from subsequent blocks in the same
6529 schedule_block (basic_block
*target_bb
, state_t init_state
)
6532 bool success
= modulo_ii
== 0;
6533 struct sched_block_state ls
;
6534 state_t temp_state
= NULL
; /* It is used for multipass scheduling. */
6535 int sort_p
, advance
, start_clock_var
;
6537 /* Head/tail info for this block. */
6538 rtx_insn
*prev_head
= current_sched_info
->prev_head
;
6539 rtx_insn
*next_tail
= current_sched_info
->next_tail
;
6540 rtx_insn
*head
= NEXT_INSN (prev_head
);
6541 rtx_insn
*tail
= PREV_INSN (next_tail
);
6543 if ((current_sched_info
->flags
& DONT_BREAK_DEPENDENCIES
) == 0
6544 && sched_pressure
!= SCHED_PRESSURE_MODEL
&& !sched_fusion
)
6545 find_modifiable_mems (head
, tail
);
6547 /* We used to have code to avoid getting parameters moved from hard
6548 argument registers into pseudos.
6550 However, it was removed when it proved to be of marginal benefit
6551 and caused problems because schedule_block and compute_forward_dependences
6552 had different notions of what the "head" insn was. */
6554 gcc_assert (head
!= tail
|| INSN_P (head
));
6556 haifa_recovery_bb_recently_added_p
= false;
6558 backtrack_queue
= NULL
;
6563 dump_new_block_header (0, *target_bb
, head
, tail
);
6565 if (sched_verbose
>= 2)
6567 dump_insn_stream (head
, tail
);
6568 memset (&rank_for_schedule_stats
, 0,
6569 sizeof (rank_for_schedule_stats
));
6573 if (init_state
== NULL
)
6574 state_reset (curr_state
);
6576 memcpy (curr_state
, init_state
, dfa_state_size
);
6578 /* Clear the ready list. */
6579 ready
.first
= ready
.veclen
- 1;
6583 /* It is used for first cycle multipass scheduling. */
6584 temp_state
= alloca (dfa_state_size
);
6586 if (targetm
.sched
.init
)
6587 targetm
.sched
.init (sched_dump
, sched_verbose
, ready
.veclen
);
6589 /* We start inserting insns after PREV_HEAD. */
6590 last_scheduled_insn
= prev_head
;
6591 last_nondebug_scheduled_insn
= NULL
;
6592 nonscheduled_insns_begin
= NULL
;
6594 gcc_assert ((NOTE_P (last_scheduled_insn
)
6595 || DEBUG_INSN_P (last_scheduled_insn
))
6596 && BLOCK_FOR_INSN (last_scheduled_insn
) == *target_bb
);
6598 /* Initialize INSN_QUEUE. Q_SIZE is the total number of insns in the
6603 insn_queue
= XALLOCAVEC (rtx_insn_list
*, max_insn_queue_index
+ 1);
6604 memset (insn_queue
, 0, (max_insn_queue_index
+ 1) * sizeof (rtx
));
6606 /* Start just before the beginning of time. */
6609 /* We need queue and ready lists and clock_var be initialized
6610 in try_ready () (which is called through init_ready_list ()). */
6611 (*current_sched_info
->init_ready_list
) ();
6614 sched_pressure_start_bb (*target_bb
);
6616 /* The algorithm is O(n^2) in the number of ready insns at any given
6617 time in the worst case. Before reload we are more likely to have
6618 big lists so truncate them to a reasonable size. */
6619 if (!reload_completed
6620 && ready
.n_ready
- ready
.n_debug
> MAX_SCHED_READY_INSNS
)
6622 ready_sort_debug (&ready
);
6623 ready_sort_real (&ready
);
6625 /* Find first free-standing insn past MAX_SCHED_READY_INSNS.
6626 If there are debug insns, we know they're first. */
6627 for (i
= MAX_SCHED_READY_INSNS
+ ready
.n_debug
; i
< ready
.n_ready
; i
++)
6628 if (!SCHED_GROUP_P (ready_element (&ready
, i
)))
6631 if (sched_verbose
>= 2)
6633 fprintf (sched_dump
,
6634 ";;\t\tReady list on entry: %d insns: ", ready
.n_ready
);
6635 debug_ready_list (&ready
);
6636 fprintf (sched_dump
,
6637 ";;\t\t before reload => truncated to %d insns\n", i
);
6640 /* Delay all insns past it for 1 cycle. If debug counter is
6641 activated make an exception for the insn right after
6642 nonscheduled_insns_begin. */
6644 rtx_insn
*skip_insn
;
6646 if (dbg_cnt (sched_insn
) == false)
6647 skip_insn
= first_nonscheduled_insn ();
6651 while (i
< ready
.n_ready
)
6655 insn
= ready_remove (&ready
, i
);
6657 if (insn
!= skip_insn
)
6658 queue_insn (insn
, 1, "list truncated");
6661 ready_add (&ready
, skip_insn
, true);
6665 /* Now we can restore basic block notes and maintain precise cfg. */
6666 restore_bb_notes (*target_bb
);
6668 last_clock_var
= -1;
6672 gcc_assert (scheduled_insns
.length () == 0);
6674 must_backtrack
= false;
6675 modulo_insns_scheduled
= 0;
6677 ls
.modulo_epilogue
= false;
6678 ls
.first_cycle_insn_p
= true;
6680 /* Loop until all the insns in BB are scheduled. */
6681 while ((*current_sched_info
->schedule_more_p
) ())
6683 perform_replacements_new_cycle ();
6686 start_clock_var
= clock_var
;
6690 advance_one_cycle ();
6692 /* Add to the ready list all pending insns that can be issued now.
6693 If there are no ready insns, increment clock until one
6694 is ready and add all pending insns at that point to the ready
6696 queue_to_ready (&ready
);
6698 gcc_assert (ready
.n_ready
);
6700 if (sched_verbose
>= 2)
6702 fprintf (sched_dump
, ";;\t\tReady list after queue_to_ready:");
6703 debug_ready_list (&ready
);
6705 advance
-= clock_var
- start_clock_var
;
6707 while (advance
> 0);
6709 if (ls
.modulo_epilogue
)
6711 int stage
= clock_var
/ modulo_ii
;
6712 if (stage
> modulo_last_stage
* 2 + 2)
6714 if (sched_verbose
>= 2)
6715 fprintf (sched_dump
,
6716 ";;\t\tmodulo scheduled succeeded at II %d\n",
6722 else if (modulo_ii
> 0)
6724 int stage
= clock_var
/ modulo_ii
;
6725 if (stage
> modulo_max_stages
)
6727 if (sched_verbose
>= 2)
6728 fprintf (sched_dump
,
6729 ";;\t\tfailing schedule due to excessive stages\n");
6732 if (modulo_n_insns
== modulo_insns_scheduled
6733 && stage
> modulo_last_stage
)
6735 if (sched_verbose
>= 2)
6736 fprintf (sched_dump
,
6737 ";;\t\tfound kernel after %d stages, II %d\n",
6739 ls
.modulo_epilogue
= true;
6743 prune_ready_list (temp_state
, true, false, ls
.modulo_epilogue
);
6744 if (ready
.n_ready
== 0)
6749 ls
.shadows_only_p
= false;
6750 cycle_issued_insns
= 0;
6751 ls
.can_issue_more
= issue_rate
;
6758 if (sort_p
&& ready
.n_ready
> 0)
6760 /* Sort the ready list based on priority. This must be
6761 done every iteration through the loop, as schedule_insn
6762 may have readied additional insns that will not be
6763 sorted correctly. */
6764 ready_sort (&ready
);
6766 if (sched_verbose
>= 2)
6768 fprintf (sched_dump
,
6769 ";;\t\tReady list after ready_sort: ");
6770 debug_ready_list (&ready
);
6774 /* We don't want md sched reorder to even see debug isns, so put
6775 them out right away. */
6776 if (ready
.n_ready
&& DEBUG_INSN_P (ready_element (&ready
, 0))
6777 && (*current_sched_info
->schedule_more_p
) ())
6779 while (ready
.n_ready
&& DEBUG_INSN_P (ready_element (&ready
, 0)))
6781 rtx_insn
*insn
= ready_remove_first (&ready
);
6782 gcc_assert (DEBUG_INSN_P (insn
));
6783 (*current_sched_info
->begin_schedule_ready
) (insn
);
6784 scheduled_insns
.safe_push (insn
);
6785 last_scheduled_insn
= insn
;
6786 advance
= schedule_insn (insn
);
6787 gcc_assert (advance
== 0);
6788 if (ready
.n_ready
> 0)
6789 ready_sort (&ready
);
6793 if (ls
.first_cycle_insn_p
&& !ready
.n_ready
)
6796 resume_after_backtrack
:
6797 /* Allow the target to reorder the list, typically for
6798 better instruction bundling. */
6800 && (ready
.n_ready
== 0
6801 || !SCHED_GROUP_P (ready_element (&ready
, 0))))
6803 if (ls
.first_cycle_insn_p
&& targetm
.sched
.reorder
)
6805 = targetm
.sched
.reorder (sched_dump
, sched_verbose
,
6806 ready_lastpos (&ready
),
6807 &ready
.n_ready
, clock_var
);
6808 else if (!ls
.first_cycle_insn_p
&& targetm
.sched
.reorder2
)
6810 = targetm
.sched
.reorder2 (sched_dump
, sched_verbose
,
6812 ? ready_lastpos (&ready
) : NULL
,
6813 &ready
.n_ready
, clock_var
);
6816 restart_choose_ready
:
6817 if (sched_verbose
>= 2)
6819 fprintf (sched_dump
, ";;\tReady list (t = %3d): ",
6821 debug_ready_list (&ready
);
6822 if (sched_pressure
== SCHED_PRESSURE_WEIGHTED
)
6823 print_curr_reg_pressure ();
6826 if (ready
.n_ready
== 0
6827 && ls
.can_issue_more
6828 && reload_completed
)
6830 /* Allow scheduling insns directly from the queue in case
6831 there's nothing better to do (ready list is empty) but
6832 there are still vacant dispatch slots in the current cycle. */
6833 if (sched_verbose
>= 6)
6834 fprintf (sched_dump
,";;\t\tSecond chance\n");
6835 memcpy (temp_state
, curr_state
, dfa_state_size
);
6836 if (early_queue_to_ready (temp_state
, &ready
))
6837 ready_sort (&ready
);
6840 if (ready
.n_ready
== 0
6841 || !ls
.can_issue_more
6842 || state_dead_lock_p (curr_state
)
6843 || !(*current_sched_info
->schedule_more_p
) ())
6846 /* Select and remove the insn from the ready list. */
6852 res
= choose_ready (&ready
, ls
.first_cycle_insn_p
, &insn
);
6858 goto restart_choose_ready
;
6860 gcc_assert (insn
!= NULL_RTX
);
6863 insn
= ready_remove_first (&ready
);
6865 if (sched_pressure
!= SCHED_PRESSURE_NONE
6866 && INSN_TICK (insn
) > clock_var
)
6868 ready_add (&ready
, insn
, true);
6873 if (targetm
.sched
.dfa_new_cycle
6874 && targetm
.sched
.dfa_new_cycle (sched_dump
, sched_verbose
,
6875 insn
, last_clock_var
,
6876 clock_var
, &sort_p
))
6877 /* SORT_P is used by the target to override sorting
6878 of the ready list. This is needed when the target
6879 has modified its internal structures expecting that
6880 the insn will be issued next. As we need the insn
6881 to have the highest priority (so it will be returned by
6882 the ready_remove_first call above), we invoke
6883 ready_add (&ready, insn, true).
6884 But, still, there is one issue: INSN can be later
6885 discarded by scheduler's front end through
6886 current_sched_info->can_schedule_ready_p, hence, won't
6889 ready_add (&ready
, insn
, true);
6895 if (current_sched_info
->can_schedule_ready_p
6896 && ! (*current_sched_info
->can_schedule_ready_p
) (insn
))
6897 /* We normally get here only if we don't want to move
6898 insn from the split block. */
6900 TODO_SPEC (insn
) = DEP_POSTPONED
;
6901 goto restart_choose_ready
;
6906 /* If this insn is the first part of a delay-slot pair, record a
6908 struct delay_pair
*delay_entry
;
6910 = delay_htab
->find_with_hash (insn
, htab_hash_pointer (insn
));
6913 save_backtrack_point (delay_entry
, ls
);
6914 if (sched_verbose
>= 2)
6915 fprintf (sched_dump
, ";;\t\tsaving backtrack point\n");
6919 /* DECISION is made. */
6921 if (modulo_ii
> 0 && INSN_UID (insn
) < modulo_iter0_max_uid
)
6923 modulo_insns_scheduled
++;
6924 modulo_last_stage
= clock_var
/ modulo_ii
;
6926 if (TODO_SPEC (insn
) & SPECULATIVE
)
6927 generate_recovery_code (insn
);
6929 if (targetm
.sched
.dispatch (NULL
, IS_DISPATCH_ON
))
6930 targetm
.sched
.dispatch_do (insn
, ADD_TO_DISPATCH_WINDOW
);
6932 /* Update counters, etc in the scheduler's front end. */
6933 (*current_sched_info
->begin_schedule_ready
) (insn
);
6934 scheduled_insns
.safe_push (insn
);
6935 gcc_assert (NONDEBUG_INSN_P (insn
));
6936 last_nondebug_scheduled_insn
= last_scheduled_insn
= insn
;
6938 if (recog_memoized (insn
) >= 0)
6940 memcpy (temp_state
, curr_state
, dfa_state_size
);
6941 cost
= state_transition (curr_state
, insn
);
6942 if (sched_pressure
!= SCHED_PRESSURE_WEIGHTED
&& !sched_fusion
)
6943 gcc_assert (cost
< 0);
6944 if (memcmp (temp_state
, curr_state
, dfa_state_size
) != 0)
6945 cycle_issued_insns
++;
6949 asm_p
= (GET_CODE (PATTERN (insn
)) == ASM_INPUT
6950 || asm_noperands (PATTERN (insn
)) >= 0);
6952 if (targetm
.sched
.variable_issue
)
6954 targetm
.sched
.variable_issue (sched_dump
, sched_verbose
,
6955 insn
, ls
.can_issue_more
);
6956 /* A naked CLOBBER or USE generates no instruction, so do
6957 not count them against the issue rate. */
6958 else if (GET_CODE (PATTERN (insn
)) != USE
6959 && GET_CODE (PATTERN (insn
)) != CLOBBER
)
6960 ls
.can_issue_more
--;
6961 advance
= schedule_insn (insn
);
6963 if (SHADOW_P (insn
))
6964 ls
.shadows_only_p
= true;
6966 /* After issuing an asm insn we should start a new cycle. */
6967 if (advance
== 0 && asm_p
)
6976 ls
.first_cycle_insn_p
= false;
6977 if (ready
.n_ready
> 0)
6978 prune_ready_list (temp_state
, false, ls
.shadows_only_p
,
6979 ls
.modulo_epilogue
);
6983 if (!must_backtrack
)
6984 for (i
= 0; i
< ready
.n_ready
; i
++)
6986 rtx_insn
*insn
= ready_element (&ready
, i
);
6987 if (INSN_EXACT_TICK (insn
) == clock_var
)
6989 must_backtrack
= true;
6994 if (must_backtrack
&& modulo_ii
> 0)
6996 if (modulo_backtracks_left
== 0)
6998 modulo_backtracks_left
--;
7000 while (must_backtrack
)
7002 struct haifa_saved_data
*failed
;
7003 rtx_insn
*failed_insn
;
7005 must_backtrack
= false;
7006 failed
= verify_shadows ();
7007 gcc_assert (failed
);
7009 failed_insn
= failed
->delay_pair
->i1
;
7010 /* Clear these queues. */
7011 perform_replacements_new_cycle ();
7012 toggle_cancelled_flags (false);
7013 unschedule_insns_until (failed_insn
);
7014 while (failed
!= backtrack_queue
)
7015 free_topmost_backtrack_point (true);
7016 restore_last_backtrack_point (&ls
);
7017 if (sched_verbose
>= 2)
7018 fprintf (sched_dump
, ";;\t\trewind to cycle %d\n", clock_var
);
7019 /* Delay by at least a cycle. This could cause additional
7021 queue_insn (failed_insn
, 1, "backtracked");
7025 if (ready
.n_ready
> 0)
7026 goto resume_after_backtrack
;
7029 if (clock_var
== 0 && ls
.first_cycle_insn_p
)
7035 ls
.first_cycle_insn_p
= true;
7037 if (ls
.modulo_epilogue
)
7040 if (!ls
.first_cycle_insn_p
|| advance
)
7041 advance_one_cycle ();
7042 perform_replacements_new_cycle ();
7045 /* Once again, debug insn suckiness: they can be on the ready list
7046 even if they have unresolved dependencies. To make our view
7047 of the world consistent, remove such "ready" insns. */
7048 restart_debug_insn_loop
:
7049 for (i
= ready
.n_ready
- 1; i
>= 0; i
--)
7053 x
= ready_element (&ready
, i
);
7054 if (DEPS_LIST_FIRST (INSN_HARD_BACK_DEPS (x
)) != NULL
7055 || DEPS_LIST_FIRST (INSN_SPEC_BACK_DEPS (x
)) != NULL
)
7057 ready_remove (&ready
, i
);
7058 goto restart_debug_insn_loop
;
7061 for (i
= ready
.n_ready
- 1; i
>= 0; i
--)
7065 x
= ready_element (&ready
, i
);
7066 resolve_dependencies (x
);
7068 for (i
= 0; i
<= max_insn_queue_index
; i
++)
7070 rtx_insn_list
*link
;
7071 while ((link
= insn_queue
[i
]) != NULL
)
7073 rtx_insn
*x
= link
->insn ();
7074 insn_queue
[i
] = link
->next ();
7075 QUEUE_INDEX (x
) = QUEUE_NOWHERE
;
7076 free_INSN_LIST_node (link
);
7077 resolve_dependencies (x
);
7083 undo_all_replacements ();
7088 fprintf (sched_dump
, ";;\tReady list (final): ");
7089 debug_ready_list (&ready
);
7092 if (modulo_ii
== 0 && current_sched_info
->queue_must_finish_empty
)
7093 /* Sanity check -- queue must be empty now. Meaningless if region has
7095 gcc_assert (!q_size
&& !ready
.n_ready
&& !ready
.n_debug
);
7096 else if (modulo_ii
== 0)
7098 /* We must maintain QUEUE_INDEX between blocks in region. */
7099 for (i
= ready
.n_ready
- 1; i
>= 0; i
--)
7103 x
= ready_element (&ready
, i
);
7104 QUEUE_INDEX (x
) = QUEUE_NOWHERE
;
7105 TODO_SPEC (x
) = HARD_DEP
;
7109 for (i
= 0; i
<= max_insn_queue_index
; i
++)
7111 rtx_insn_list
*link
;
7112 for (link
= insn_queue
[i
]; link
; link
= link
->next ())
7117 QUEUE_INDEX (x
) = QUEUE_NOWHERE
;
7118 TODO_SPEC (x
) = HARD_DEP
;
7120 free_INSN_LIST_list (&insn_queue
[i
]);
7124 if (sched_pressure
== SCHED_PRESSURE_MODEL
)
7125 model_end_schedule ();
7129 commit_schedule (prev_head
, tail
, target_bb
);
7131 fprintf (sched_dump
, ";; total time = %d\n", clock_var
);
7134 last_scheduled_insn
= tail
;
7136 scheduled_insns
.truncate (0);
7138 if (!current_sched_info
->queue_must_finish_empty
7139 || haifa_recovery_bb_recently_added_p
)
7141 /* INSN_TICK (minimum clock tick at which the insn becomes
7142 ready) may be not correct for the insn in the subsequent
7143 blocks of the region. We should use a correct value of
7144 `clock_var' or modify INSN_TICK. It is better to keep
7145 clock_var value equal to 0 at the start of a basic block.
7146 Therefore we modify INSN_TICK here. */
7147 fix_inter_tick (NEXT_INSN (prev_head
), last_scheduled_insn
);
7150 if (targetm
.sched
.finish
)
7152 targetm
.sched
.finish (sched_dump
, sched_verbose
);
7153 /* Target might have added some instructions to the scheduled block
7154 in its md_finish () hook. These new insns don't have any data
7155 initialized and to identify them we extend h_i_d so that they'll
7157 sched_extend_luids ();
7160 /* Update head/tail boundaries. */
7161 head
= NEXT_INSN (prev_head
);
7162 tail
= last_scheduled_insn
;
7166 fprintf (sched_dump
, ";; new head = %d\n;; new tail = %d\n",
7167 INSN_UID (head
), INSN_UID (tail
));
7169 if (sched_verbose
>= 2)
7171 dump_insn_stream (head
, tail
);
7172 print_rank_for_schedule_stats (";; TOTAL ", &rank_for_schedule_stats
,
7176 fprintf (sched_dump
, "\n");
7179 head
= restore_other_notes (head
, NULL
);
7181 current_sched_info
->head
= head
;
7182 current_sched_info
->tail
= tail
;
7184 free_backtrack_queue ();
7189 /* Set_priorities: compute priority of each insn in the block. */
7192 set_priorities (rtx_insn
*head
, rtx_insn
*tail
)
7196 int sched_max_insns_priority
=
7197 current_sched_info
->sched_max_insns_priority
;
7198 rtx_insn
*prev_head
;
7200 if (head
== tail
&& ! INSN_P (head
))
7205 prev_head
= PREV_INSN (head
);
7206 for (insn
= tail
; insn
!= prev_head
; insn
= PREV_INSN (insn
))
7212 (void) priority (insn
);
7214 gcc_assert (INSN_PRIORITY_KNOWN (insn
));
7216 sched_max_insns_priority
= MAX (sched_max_insns_priority
,
7217 INSN_PRIORITY (insn
));
7220 current_sched_info
->sched_max_insns_priority
= sched_max_insns_priority
;
7225 /* Set dump and sched_verbose for the desired debugging output. If no
7226 dump-file was specified, but -fsched-verbose=N (any N), print to stderr.
7227 For -fsched-verbose=N, N>=10, print everything to stderr. */
7229 setup_sched_dump (void)
7231 sched_verbose
= sched_verbose_param
;
7232 if (sched_verbose_param
== 0 && dump_file
)
7234 sched_dump
= ((sched_verbose_param
>= 10 || !dump_file
)
7235 ? stderr
: dump_file
);
7238 /* Allocate data for register pressure sensitive scheduling. */
7240 alloc_global_sched_pressure_data (void)
7242 if (sched_pressure
!= SCHED_PRESSURE_NONE
)
7244 int i
, max_regno
= max_reg_num ();
7246 if (sched_dump
!= NULL
)
7247 /* We need info about pseudos for rtl dumps about pseudo
7248 classes and costs. */
7249 regstat_init_n_sets_and_refs ();
7250 ira_set_pseudo_classes (true, sched_verbose
? sched_dump
: NULL
);
7251 sched_regno_pressure_class
7252 = (enum reg_class
*) xmalloc (max_regno
* sizeof (enum reg_class
));
7253 for (i
= 0; i
< max_regno
; i
++)
7254 sched_regno_pressure_class
[i
]
7255 = (i
< FIRST_PSEUDO_REGISTER
7256 ? ira_pressure_class_translate
[REGNO_REG_CLASS (i
)]
7257 : ira_pressure_class_translate
[reg_allocno_class (i
)]);
7258 curr_reg_live
= BITMAP_ALLOC (NULL
);
7259 if (sched_pressure
== SCHED_PRESSURE_WEIGHTED
)
7261 saved_reg_live
= BITMAP_ALLOC (NULL
);
7262 region_ref_regs
= BITMAP_ALLOC (NULL
);
7265 /* Calculate number of CALL_USED_REGS in register classes that
7266 we calculate register pressure for. */
7267 for (int c
= 0; c
< ira_pressure_classes_num
; ++c
)
7269 enum reg_class cl
= ira_pressure_classes
[c
];
7271 call_used_regs_num
[cl
] = 0;
7273 for (int i
= 0; i
< ira_class_hard_regs_num
[cl
]; ++i
)
7274 if (call_used_regs
[ira_class_hard_regs
[cl
][i
]])
7275 ++call_used_regs_num
[cl
];
7280 /* Free data for register pressure sensitive scheduling. Also called
7281 from schedule_region when stopping sched-pressure early. */
7283 free_global_sched_pressure_data (void)
7285 if (sched_pressure
!= SCHED_PRESSURE_NONE
)
7287 if (regstat_n_sets_and_refs
!= NULL
)
7288 regstat_free_n_sets_and_refs ();
7289 if (sched_pressure
== SCHED_PRESSURE_WEIGHTED
)
7291 BITMAP_FREE (region_ref_regs
);
7292 BITMAP_FREE (saved_reg_live
);
7294 BITMAP_FREE (curr_reg_live
);
7295 free (sched_regno_pressure_class
);
7299 /* Initialize some global state for the scheduler. This function works
7300 with the common data shared between all the schedulers. It is called
7301 from the scheduler specific initialization routine. */
7306 /* Disable speculative loads in their presence if cc0 defined. */
7308 flag_schedule_speculative_load
= 0;
7310 if (targetm
.sched
.dispatch (NULL
, IS_DISPATCH_ON
))
7311 targetm
.sched
.dispatch_do (NULL
, DISPATCH_INIT
);
7313 if (live_range_shrinkage_p
)
7314 sched_pressure
= SCHED_PRESSURE_WEIGHTED
;
7315 else if (flag_sched_pressure
7316 && !reload_completed
7317 && common_sched_info
->sched_pass_id
== SCHED_RGN_PASS
)
7318 sched_pressure
= ((enum sched_pressure_algorithm
)
7319 PARAM_VALUE (PARAM_SCHED_PRESSURE_ALGORITHM
));
7321 sched_pressure
= SCHED_PRESSURE_NONE
;
7323 if (sched_pressure
!= SCHED_PRESSURE_NONE
)
7324 ira_setup_eliminable_regset ();
7326 /* Initialize SPEC_INFO. */
7327 if (targetm
.sched
.set_sched_flags
)
7329 spec_info
= &spec_info_var
;
7330 targetm
.sched
.set_sched_flags (spec_info
);
7332 if (spec_info
->mask
!= 0)
7334 spec_info
->data_weakness_cutoff
=
7335 (PARAM_VALUE (PARAM_SCHED_SPEC_PROB_CUTOFF
) * MAX_DEP_WEAK
) / 100;
7336 spec_info
->control_weakness_cutoff
=
7337 (PARAM_VALUE (PARAM_SCHED_SPEC_PROB_CUTOFF
)
7338 * REG_BR_PROB_BASE
) / 100;
7341 /* So we won't read anything accidentally. */
7346 /* So we won't read anything accidentally. */
7349 /* Initialize issue_rate. */
7350 if (targetm
.sched
.issue_rate
)
7351 issue_rate
= targetm
.sched
.issue_rate ();
7355 if (targetm
.sched
.first_cycle_multipass_dfa_lookahead
7356 /* Don't use max_issue with reg_pressure scheduling. Multipass
7357 scheduling and reg_pressure scheduling undo each other's decisions. */
7358 && sched_pressure
== SCHED_PRESSURE_NONE
)
7359 dfa_lookahead
= targetm
.sched
.first_cycle_multipass_dfa_lookahead ();
7363 /* Set to "0" so that we recalculate. */
7364 max_lookahead_tries
= 0;
7366 if (targetm
.sched
.init_dfa_pre_cycle_insn
)
7367 targetm
.sched
.init_dfa_pre_cycle_insn ();
7369 if (targetm
.sched
.init_dfa_post_cycle_insn
)
7370 targetm
.sched
.init_dfa_post_cycle_insn ();
7373 dfa_state_size
= state_size ();
7375 init_alias_analysis ();
7378 df_set_flags (DF_LR_RUN_DCE
);
7379 df_note_add_problem ();
7381 /* More problems needed for interloop dep calculation in SMS. */
7382 if (common_sched_info
->sched_pass_id
== SCHED_SMS_PASS
)
7384 df_rd_add_problem ();
7385 df_chain_add_problem (DF_DU_CHAIN
+ DF_UD_CHAIN
);
7390 /* Do not run DCE after reload, as this can kill nops inserted
7392 if (reload_completed
)
7393 df_clear_flags (DF_LR_RUN_DCE
);
7395 regstat_compute_calls_crossed ();
7397 if (targetm
.sched
.init_global
)
7398 targetm
.sched
.init_global (sched_dump
, sched_verbose
, get_max_uid () + 1);
7400 alloc_global_sched_pressure_data ();
7402 curr_state
= xmalloc (dfa_state_size
);
7405 static void haifa_init_only_bb (basic_block
, basic_block
);
7407 /* Initialize data structures specific to the Haifa scheduler. */
7409 haifa_sched_init (void)
7411 setup_sched_dump ();
7414 scheduled_insns
.create (0);
7416 if (spec_info
!= NULL
)
7418 sched_deps_info
->use_deps_list
= 1;
7419 sched_deps_info
->generate_spec_deps
= 1;
7422 /* Initialize luids, dependency caches, target and h_i_d for the
7426 bbs
.create (n_basic_blocks_for_fn (cfun
));
7431 FOR_EACH_BB_FN (bb
, cfun
)
7432 bbs
.quick_push (bb
);
7433 sched_init_luids (bbs
);
7434 sched_deps_init (true);
7435 sched_extend_target ();
7436 haifa_init_h_i_d (bbs
);
7441 sched_init_only_bb
= haifa_init_only_bb
;
7442 sched_split_block
= sched_split_block_1
;
7443 sched_create_empty_bb
= sched_create_empty_bb_1
;
7444 haifa_recovery_bb_ever_added_p
= false;
7446 nr_begin_data
= nr_begin_control
= nr_be_in_data
= nr_be_in_control
= 0;
7447 before_recovery
= 0;
7453 /* Finish work with the data specific to the Haifa scheduler. */
7455 haifa_sched_finish (void)
7457 sched_create_empty_bb
= NULL
;
7458 sched_split_block
= NULL
;
7459 sched_init_only_bb
= NULL
;
7461 if (spec_info
&& spec_info
->dump
)
7463 char c
= reload_completed
? 'a' : 'b';
7465 fprintf (spec_info
->dump
,
7466 ";; %s:\n", current_function_name ());
7468 fprintf (spec_info
->dump
,
7469 ";; Procedure %cr-begin-data-spec motions == %d\n",
7471 fprintf (spec_info
->dump
,
7472 ";; Procedure %cr-be-in-data-spec motions == %d\n",
7474 fprintf (spec_info
->dump
,
7475 ";; Procedure %cr-begin-control-spec motions == %d\n",
7476 c
, nr_begin_control
);
7477 fprintf (spec_info
->dump
,
7478 ";; Procedure %cr-be-in-control-spec motions == %d\n",
7479 c
, nr_be_in_control
);
7482 scheduled_insns
.release ();
7484 /* Finalize h_i_d, dependency caches, and luids for the whole
7485 function. Target will be finalized in md_global_finish (). */
7486 sched_deps_finish ();
7487 sched_finish_luids ();
7488 current_sched_info
= NULL
;
7492 /* Free global data used during insn scheduling. This function works with
7493 the common data shared between the schedulers. */
7498 haifa_finish_h_i_d ();
7499 free_global_sched_pressure_data ();
7502 if (targetm
.sched
.finish_global
)
7503 targetm
.sched
.finish_global (sched_dump
, sched_verbose
);
7505 end_alias_analysis ();
7507 regstat_free_calls_crossed ();
7512 /* Free all delay_pair structures that were recorded. */
7514 free_delay_pairs (void)
7518 delay_htab
->empty ();
7519 delay_htab_i2
->empty ();
7523 /* Fix INSN_TICKs of the instructions in the current block as well as
7524 INSN_TICKs of their dependents.
7525 HEAD and TAIL are the begin and the end of the current scheduled block. */
7527 fix_inter_tick (rtx_insn
*head
, rtx_insn
*tail
)
7529 /* Set of instructions with corrected INSN_TICK. */
7530 bitmap_head processed
;
7531 /* ??? It is doubtful if we should assume that cycle advance happens on
7532 basic block boundaries. Basically insns that are unconditionally ready
7533 on the start of the block are more preferable then those which have
7534 a one cycle dependency over insn from the previous block. */
7535 int next_clock
= clock_var
+ 1;
7537 bitmap_initialize (&processed
, 0);
7539 /* Iterates over scheduled instructions and fix their INSN_TICKs and
7540 INSN_TICKs of dependent instructions, so that INSN_TICKs are consistent
7541 across different blocks. */
7542 for (tail
= NEXT_INSN (tail
); head
!= tail
; head
= NEXT_INSN (head
))
7547 sd_iterator_def sd_it
;
7550 tick
= INSN_TICK (head
);
7551 gcc_assert (tick
>= MIN_TICK
);
7553 /* Fix INSN_TICK of instruction from just scheduled block. */
7554 if (bitmap_set_bit (&processed
, INSN_LUID (head
)))
7558 if (tick
< MIN_TICK
)
7561 INSN_TICK (head
) = tick
;
7564 if (DEBUG_INSN_P (head
))
7567 FOR_EACH_DEP (head
, SD_LIST_RES_FORW
, sd_it
, dep
)
7571 next
= DEP_CON (dep
);
7572 tick
= INSN_TICK (next
);
7574 if (tick
!= INVALID_TICK
7575 /* If NEXT has its INSN_TICK calculated, fix it.
7576 If not - it will be properly calculated from
7577 scratch later in fix_tick_ready. */
7578 && bitmap_set_bit (&processed
, INSN_LUID (next
)))
7582 if (tick
< MIN_TICK
)
7585 if (tick
> INTER_TICK (next
))
7586 INTER_TICK (next
) = tick
;
7588 tick
= INTER_TICK (next
);
7590 INSN_TICK (next
) = tick
;
7595 bitmap_clear (&processed
);
7598 /* Check if NEXT is ready to be added to the ready or queue list.
7599 If "yes", add it to the proper list.
7601 -1 - is not ready yet,
7602 0 - added to the ready list,
7603 0 < N - queued for N cycles. */
7605 try_ready (rtx_insn
*next
)
7607 ds_t old_ts
, new_ts
;
7609 old_ts
= TODO_SPEC (next
);
7611 gcc_assert (!(old_ts
& ~(SPECULATIVE
| HARD_DEP
| DEP_CONTROL
| DEP_POSTPONED
))
7612 && (old_ts
== HARD_DEP
7613 || old_ts
== DEP_POSTPONED
7614 || (old_ts
& SPECULATIVE
)
7615 || old_ts
== DEP_CONTROL
));
7617 new_ts
= recompute_todo_spec (next
, false);
7619 if (new_ts
& (HARD_DEP
| DEP_POSTPONED
))
7620 gcc_assert (new_ts
== old_ts
7621 && QUEUE_INDEX (next
) == QUEUE_NOWHERE
);
7622 else if (current_sched_info
->new_ready
)
7623 new_ts
= current_sched_info
->new_ready (next
, new_ts
);
7625 /* * if !(old_ts & SPECULATIVE) (e.g. HARD_DEP or 0), then insn might
7626 have its original pattern or changed (speculative) one. This is due
7627 to changing ebb in region scheduling.
7628 * But if (old_ts & SPECULATIVE), then we are pretty sure that insn
7629 has speculative pattern.
7631 We can't assert (!(new_ts & HARD_DEP) || new_ts == old_ts) here because
7632 control-speculative NEXT could have been discarded by sched-rgn.c
7633 (the same case as when discarded by can_schedule_ready_p ()). */
7635 if ((new_ts
& SPECULATIVE
)
7636 /* If (old_ts == new_ts), then (old_ts & SPECULATIVE) and we don't
7637 need to change anything. */
7638 && new_ts
!= old_ts
)
7643 gcc_assert ((new_ts
& SPECULATIVE
) && !(new_ts
& ~SPECULATIVE
));
7645 res
= haifa_speculate_insn (next
, new_ts
, &new_pat
);
7650 /* It would be nice to change DEP_STATUS of all dependences,
7651 which have ((DEP_STATUS & SPECULATIVE) == new_ts) to HARD_DEP,
7652 so we won't reanalyze anything. */
7657 /* We follow the rule, that every speculative insn
7658 has non-null ORIG_PAT. */
7659 if (!ORIG_PAT (next
))
7660 ORIG_PAT (next
) = PATTERN (next
);
7664 if (!ORIG_PAT (next
))
7665 /* If we gonna to overwrite the original pattern of insn,
7667 ORIG_PAT (next
) = PATTERN (next
);
7669 res
= haifa_change_pattern (next
, new_pat
);
7678 /* We need to restore pattern only if (new_ts == 0), because otherwise it is
7679 either correct (new_ts & SPECULATIVE),
7680 or we simply don't care (new_ts & HARD_DEP). */
7682 gcc_assert (!ORIG_PAT (next
)
7683 || !IS_SPECULATION_BRANCHY_CHECK_P (next
));
7685 TODO_SPEC (next
) = new_ts
;
7687 if (new_ts
& (HARD_DEP
| DEP_POSTPONED
))
7689 /* We can't assert (QUEUE_INDEX (next) == QUEUE_NOWHERE) here because
7690 control-speculative NEXT could have been discarded by sched-rgn.c
7691 (the same case as when discarded by can_schedule_ready_p ()). */
7692 /*gcc_assert (QUEUE_INDEX (next) == QUEUE_NOWHERE);*/
7694 change_queue_index (next
, QUEUE_NOWHERE
);
7698 else if (!(new_ts
& BEGIN_SPEC
)
7699 && ORIG_PAT (next
) && PREDICATED_PAT (next
) == NULL_RTX
7700 && !IS_SPECULATION_CHECK_P (next
))
7701 /* We should change pattern of every previously speculative
7702 instruction - and we determine if NEXT was speculative by using
7703 ORIG_PAT field. Except one case - speculation checks have ORIG_PAT
7704 pat too, so skip them. */
7706 bool success
= haifa_change_pattern (next
, ORIG_PAT (next
));
7707 gcc_assert (success
);
7708 ORIG_PAT (next
) = 0;
7711 if (sched_verbose
>= 2)
7713 fprintf (sched_dump
, ";;\t\tdependencies resolved: insn %s",
7714 (*current_sched_info
->print_insn
) (next
, 0));
7716 if (spec_info
&& spec_info
->dump
)
7718 if (new_ts
& BEGIN_DATA
)
7719 fprintf (spec_info
->dump
, "; data-spec;");
7720 if (new_ts
& BEGIN_CONTROL
)
7721 fprintf (spec_info
->dump
, "; control-spec;");
7722 if (new_ts
& BE_IN_CONTROL
)
7723 fprintf (spec_info
->dump
, "; in-control-spec;");
7725 if (TODO_SPEC (next
) & DEP_CONTROL
)
7726 fprintf (sched_dump
, " predicated");
7727 fprintf (sched_dump
, "\n");
7730 adjust_priority (next
);
7732 return fix_tick_ready (next
);
7735 /* Calculate INSN_TICK of NEXT and add it to either ready or queue list. */
7737 fix_tick_ready (rtx_insn
*next
)
7741 if (!DEBUG_INSN_P (next
) && !sd_lists_empty_p (next
, SD_LIST_RES_BACK
))
7744 sd_iterator_def sd_it
;
7747 tick
= INSN_TICK (next
);
7748 /* if tick is not equal to INVALID_TICK, then update
7749 INSN_TICK of NEXT with the most recent resolved dependence
7750 cost. Otherwise, recalculate from scratch. */
7751 full_p
= (tick
== INVALID_TICK
);
7753 FOR_EACH_DEP (next
, SD_LIST_RES_BACK
, sd_it
, dep
)
7755 rtx_insn
*pro
= DEP_PRO (dep
);
7758 gcc_assert (INSN_TICK (pro
) >= MIN_TICK
);
7760 tick1
= INSN_TICK (pro
) + dep_cost (dep
);
7771 INSN_TICK (next
) = tick
;
7773 delay
= tick
- clock_var
;
7774 if (delay
<= 0 || sched_pressure
!= SCHED_PRESSURE_NONE
|| sched_fusion
)
7775 delay
= QUEUE_READY
;
7777 change_queue_index (next
, delay
);
7782 /* Move NEXT to the proper queue list with (DELAY >= 1),
7783 or add it to the ready list (DELAY == QUEUE_READY),
7784 or remove it from ready and queue lists at all (DELAY == QUEUE_NOWHERE). */
7786 change_queue_index (rtx_insn
*next
, int delay
)
7788 int i
= QUEUE_INDEX (next
);
7790 gcc_assert (QUEUE_NOWHERE
<= delay
&& delay
<= max_insn_queue_index
7792 gcc_assert (i
!= QUEUE_SCHEDULED
);
7794 if ((delay
> 0 && NEXT_Q_AFTER (q_ptr
, delay
) == i
)
7795 || (delay
< 0 && delay
== i
))
7796 /* We have nothing to do. */
7799 /* Remove NEXT from wherever it is now. */
7800 if (i
== QUEUE_READY
)
7801 ready_remove_insn (next
);
7803 queue_remove (next
);
7805 /* Add it to the proper place. */
7806 if (delay
== QUEUE_READY
)
7807 ready_add (readyp
, next
, false);
7808 else if (delay
>= 1)
7809 queue_insn (next
, delay
, "change queue index");
7811 if (sched_verbose
>= 2)
7813 fprintf (sched_dump
, ";;\t\ttick updated: insn %s",
7814 (*current_sched_info
->print_insn
) (next
, 0));
7816 if (delay
== QUEUE_READY
)
7817 fprintf (sched_dump
, " into ready\n");
7818 else if (delay
>= 1)
7819 fprintf (sched_dump
, " into queue with cost=%d\n", delay
);
7821 fprintf (sched_dump
, " removed from ready or queue lists\n");
7825 static int sched_ready_n_insns
= -1;
7827 /* Initialize per region data structures. */
7829 sched_extend_ready_list (int new_sched_ready_n_insns
)
7833 if (sched_ready_n_insns
== -1)
7834 /* At the first call we need to initialize one more choice_stack
7838 sched_ready_n_insns
= 0;
7839 scheduled_insns
.reserve (new_sched_ready_n_insns
);
7842 i
= sched_ready_n_insns
+ 1;
7844 ready
.veclen
= new_sched_ready_n_insns
+ issue_rate
;
7845 ready
.vec
= XRESIZEVEC (rtx_insn
*, ready
.vec
, ready
.veclen
);
7847 gcc_assert (new_sched_ready_n_insns
>= sched_ready_n_insns
);
7849 ready_try
= (signed char *) xrecalloc (ready_try
, new_sched_ready_n_insns
,
7850 sched_ready_n_insns
,
7851 sizeof (*ready_try
));
7853 /* We allocate +1 element to save initial state in the choice_stack[0]
7855 choice_stack
= XRESIZEVEC (struct choice_entry
, choice_stack
,
7856 new_sched_ready_n_insns
+ 1);
7858 for (; i
<= new_sched_ready_n_insns
; i
++)
7860 choice_stack
[i
].state
= xmalloc (dfa_state_size
);
7862 if (targetm
.sched
.first_cycle_multipass_init
)
7863 targetm
.sched
.first_cycle_multipass_init (&(choice_stack
[i
]
7867 sched_ready_n_insns
= new_sched_ready_n_insns
;
7870 /* Free per region data structures. */
7872 sched_finish_ready_list (void)
7883 for (i
= 0; i
<= sched_ready_n_insns
; i
++)
7885 if (targetm
.sched
.first_cycle_multipass_fini
)
7886 targetm
.sched
.first_cycle_multipass_fini (&(choice_stack
[i
]
7889 free (choice_stack
[i
].state
);
7891 free (choice_stack
);
7892 choice_stack
= NULL
;
7894 sched_ready_n_insns
= -1;
7898 haifa_luid_for_non_insn (rtx x
)
7900 gcc_assert (NOTE_P (x
) || LABEL_P (x
));
7905 /* Generates recovery code for INSN. */
7907 generate_recovery_code (rtx_insn
*insn
)
7909 if (TODO_SPEC (insn
) & BEGIN_SPEC
)
7910 begin_speculative_block (insn
);
7912 /* Here we have insn with no dependencies to
7913 instructions other then CHECK_SPEC ones. */
7915 if (TODO_SPEC (insn
) & BE_IN_SPEC
)
7916 add_to_speculative_block (insn
);
7920 Tries to add speculative dependencies of type FS between instructions
7921 in deps_list L and TWIN. */
7923 process_insn_forw_deps_be_in_spec (rtx_insn
*insn
, rtx_insn
*twin
, ds_t fs
)
7925 sd_iterator_def sd_it
;
7928 FOR_EACH_DEP (insn
, SD_LIST_FORW
, sd_it
, dep
)
7933 consumer
= DEP_CON (dep
);
7935 ds
= DEP_STATUS (dep
);
7937 if (/* If we want to create speculative dep. */
7939 /* And we can do that because this is a true dep. */
7940 && (ds
& DEP_TYPES
) == DEP_TRUE
)
7942 gcc_assert (!(ds
& BE_IN_SPEC
));
7944 if (/* If this dep can be overcome with 'begin speculation'. */
7946 /* Then we have a choice: keep the dep 'begin speculative'
7947 or transform it into 'be in speculative'. */
7949 if (/* In try_ready we assert that if insn once became ready
7950 it can be removed from the ready (or queue) list only
7951 due to backend decision. Hence we can't let the
7952 probability of the speculative dep to decrease. */
7953 ds_weak (ds
) <= ds_weak (fs
))
7957 new_ds
= (ds
& ~BEGIN_SPEC
) | fs
;
7959 if (/* consumer can 'be in speculative'. */
7960 sched_insn_is_legitimate_for_speculation_p (consumer
,
7962 /* Transform it to be in speculative. */
7967 /* Mark the dep as 'be in speculative'. */
7972 dep_def _new_dep
, *new_dep
= &_new_dep
;
7974 init_dep_1 (new_dep
, twin
, consumer
, DEP_TYPE (dep
), ds
);
7975 sd_add_dep (new_dep
, false);
7980 /* Generates recovery code for BEGIN speculative INSN. */
7982 begin_speculative_block (rtx_insn
*insn
)
7984 if (TODO_SPEC (insn
) & BEGIN_DATA
)
7986 if (TODO_SPEC (insn
) & BEGIN_CONTROL
)
7989 create_check_block_twin (insn
, false);
7991 TODO_SPEC (insn
) &= ~BEGIN_SPEC
;
7994 static void haifa_init_insn (rtx_insn
*);
7996 /* Generates recovery code for BE_IN speculative INSN. */
7998 add_to_speculative_block (rtx_insn
*insn
)
8001 sd_iterator_def sd_it
;
8003 rtx_insn_list
*twins
= NULL
;
8004 rtx_vec_t priorities_roots
;
8006 ts
= TODO_SPEC (insn
);
8007 gcc_assert (!(ts
& ~BE_IN_SPEC
));
8009 if (ts
& BE_IN_DATA
)
8011 if (ts
& BE_IN_CONTROL
)
8014 TODO_SPEC (insn
) &= ~BE_IN_SPEC
;
8015 gcc_assert (!TODO_SPEC (insn
));
8017 DONE_SPEC (insn
) |= ts
;
8019 /* First we convert all simple checks to branchy. */
8020 for (sd_it
= sd_iterator_start (insn
, SD_LIST_SPEC_BACK
);
8021 sd_iterator_cond (&sd_it
, &dep
);)
8023 rtx_insn
*check
= DEP_PRO (dep
);
8025 if (IS_SPECULATION_SIMPLE_CHECK_P (check
))
8027 create_check_block_twin (check
, true);
8029 /* Restart search. */
8030 sd_it
= sd_iterator_start (insn
, SD_LIST_SPEC_BACK
);
8033 /* Continue search. */
8034 sd_iterator_next (&sd_it
);
8037 priorities_roots
.create (0);
8038 clear_priorities (insn
, &priorities_roots
);
8042 rtx_insn
*check
, *twin
;
8045 /* Get the first backward dependency of INSN. */
8046 sd_it
= sd_iterator_start (insn
, SD_LIST_SPEC_BACK
);
8047 if (!sd_iterator_cond (&sd_it
, &dep
))
8048 /* INSN has no backward dependencies left. */
8051 gcc_assert ((DEP_STATUS (dep
) & BEGIN_SPEC
) == 0
8052 && (DEP_STATUS (dep
) & BE_IN_SPEC
) != 0
8053 && (DEP_STATUS (dep
) & DEP_TYPES
) == DEP_TRUE
);
8055 check
= DEP_PRO (dep
);
8057 gcc_assert (!IS_SPECULATION_CHECK_P (check
) && !ORIG_PAT (check
)
8058 && QUEUE_INDEX (check
) == QUEUE_NOWHERE
);
8060 rec
= BLOCK_FOR_INSN (check
);
8062 twin
= emit_insn_before (copy_insn (PATTERN (insn
)), BB_END (rec
));
8063 haifa_init_insn (twin
);
8065 sd_copy_back_deps (twin
, insn
, true);
8067 if (sched_verbose
&& spec_info
->dump
)
8068 /* INSN_BB (insn) isn't determined for twin insns yet.
8069 So we can't use current_sched_info->print_insn. */
8070 fprintf (spec_info
->dump
, ";;\t\tGenerated twin insn : %d/rec%d\n",
8071 INSN_UID (twin
), rec
->index
);
8073 twins
= alloc_INSN_LIST (twin
, twins
);
8075 /* Add dependences between TWIN and all appropriate
8076 instructions from REC. */
8077 FOR_EACH_DEP (insn
, SD_LIST_SPEC_BACK
, sd_it
, dep
)
8079 rtx_insn
*pro
= DEP_PRO (dep
);
8081 gcc_assert (DEP_TYPE (dep
) == REG_DEP_TRUE
);
8083 /* INSN might have dependencies from the instructions from
8084 several recovery blocks. At this iteration we process those
8085 producers that reside in REC. */
8086 if (BLOCK_FOR_INSN (pro
) == rec
)
8088 dep_def _new_dep
, *new_dep
= &_new_dep
;
8090 init_dep (new_dep
, pro
, twin
, REG_DEP_TRUE
);
8091 sd_add_dep (new_dep
, false);
8095 process_insn_forw_deps_be_in_spec (insn
, twin
, ts
);
8097 /* Remove all dependencies between INSN and insns in REC. */
8098 for (sd_it
= sd_iterator_start (insn
, SD_LIST_SPEC_BACK
);
8099 sd_iterator_cond (&sd_it
, &dep
);)
8101 rtx_insn
*pro
= DEP_PRO (dep
);
8103 if (BLOCK_FOR_INSN (pro
) == rec
)
8104 sd_delete_dep (sd_it
);
8106 sd_iterator_next (&sd_it
);
8110 /* We couldn't have added the dependencies between INSN and TWINS earlier
8111 because that would make TWINS appear in the INSN_BACK_DEPS (INSN). */
8115 rtx_insn_list
*next_node
;
8117 twin
= twins
->insn ();
8120 dep_def _new_dep
, *new_dep
= &_new_dep
;
8122 init_dep (new_dep
, insn
, twin
, REG_DEP_OUTPUT
);
8123 sd_add_dep (new_dep
, false);
8126 next_node
= twins
->next ();
8127 free_INSN_LIST_node (twins
);
8131 calc_priorities (priorities_roots
);
8132 priorities_roots
.release ();
8135 /* Extends and fills with zeros (only the new part) array pointed to by P. */
8137 xrecalloc (void *p
, size_t new_nmemb
, size_t old_nmemb
, size_t size
)
8139 gcc_assert (new_nmemb
>= old_nmemb
);
8140 p
= XRESIZEVAR (void, p
, new_nmemb
* size
);
8141 memset (((char *) p
) + old_nmemb
* size
, 0, (new_nmemb
- old_nmemb
) * size
);
8146 Find fallthru edge from PRED. */
8148 find_fallthru_edge_from (basic_block pred
)
8153 succ
= pred
->next_bb
;
8154 gcc_assert (succ
->prev_bb
== pred
);
8156 if (EDGE_COUNT (pred
->succs
) <= EDGE_COUNT (succ
->preds
))
8158 e
= find_fallthru_edge (pred
->succs
);
8162 gcc_assert (e
->dest
== succ
);
8168 e
= find_fallthru_edge (succ
->preds
);
8172 gcc_assert (e
->src
== pred
);
8180 /* Extend per basic block data structures. */
8182 sched_extend_bb (void)
8184 /* The following is done to keep current_sched_info->next_tail non null. */
8185 rtx_insn
*end
= BB_END (EXIT_BLOCK_PTR_FOR_FN (cfun
)->prev_bb
);
8186 rtx_insn
*insn
= DEBUG_INSN_P (end
) ? prev_nondebug_insn (end
) : end
;
8187 if (NEXT_INSN (end
) == 0
8190 /* Don't emit a NOTE if it would end up before a BARRIER. */
8191 && !BARRIER_P (NEXT_INSN (end
))))
8193 rtx_note
*note
= emit_note_after (NOTE_INSN_DELETED
, end
);
8194 /* Make note appear outside BB. */
8195 set_block_for_insn (note
, NULL
);
8196 BB_END (EXIT_BLOCK_PTR_FOR_FN (cfun
)->prev_bb
) = end
;
8200 /* Init per basic block data structures. */
8202 sched_init_bbs (void)
8207 /* Initialize BEFORE_RECOVERY variable. */
8209 init_before_recovery (basic_block
*before_recovery_ptr
)
8214 last
= EXIT_BLOCK_PTR_FOR_FN (cfun
)->prev_bb
;
8215 e
= find_fallthru_edge_from (last
);
8219 /* We create two basic blocks:
8220 1. Single instruction block is inserted right after E->SRC
8222 2. Empty block right before EXIT_BLOCK.
8223 Between these two blocks recovery blocks will be emitted. */
8225 basic_block single
, empty
;
8227 /* If the fallthrough edge to exit we've found is from the block we've
8228 created before, don't do anything more. */
8229 if (last
== after_recovery
)
8232 adding_bb_to_current_region_p
= false;
8234 single
= sched_create_empty_bb (last
);
8235 empty
= sched_create_empty_bb (single
);
8237 /* Add new blocks to the root loop. */
8238 if (current_loops
!= NULL
)
8240 add_bb_to_loop (single
, (*current_loops
->larray
)[0]);
8241 add_bb_to_loop (empty
, (*current_loops
->larray
)[0]);
8244 single
->count
= last
->count
;
8245 empty
->count
= last
->count
;
8246 single
->frequency
= last
->frequency
;
8247 empty
->frequency
= last
->frequency
;
8248 BB_COPY_PARTITION (single
, last
);
8249 BB_COPY_PARTITION (empty
, last
);
8251 redirect_edge_succ (e
, single
);
8252 make_single_succ_edge (single
, empty
, 0);
8253 make_single_succ_edge (empty
, EXIT_BLOCK_PTR_FOR_FN (cfun
),
8256 rtx_code_label
*label
= block_label (empty
);
8257 rtx_jump_insn
*x
= emit_jump_insn_after (targetm
.gen_jump (label
),
8259 JUMP_LABEL (x
) = label
;
8260 LABEL_NUSES (label
)++;
8261 haifa_init_insn (x
);
8263 emit_barrier_after (x
);
8265 sched_init_only_bb (empty
, NULL
);
8266 sched_init_only_bb (single
, NULL
);
8269 adding_bb_to_current_region_p
= true;
8270 before_recovery
= single
;
8271 after_recovery
= empty
;
8273 if (before_recovery_ptr
)
8274 *before_recovery_ptr
= before_recovery
;
8276 if (sched_verbose
>= 2 && spec_info
->dump
)
8277 fprintf (spec_info
->dump
,
8278 ";;\t\tFixed fallthru to EXIT : %d->>%d->%d->>EXIT\n",
8279 last
->index
, single
->index
, empty
->index
);
8282 before_recovery
= last
;
8285 /* Returns new recovery block. */
8287 sched_create_recovery_block (basic_block
*before_recovery_ptr
)
8292 haifa_recovery_bb_recently_added_p
= true;
8293 haifa_recovery_bb_ever_added_p
= true;
8295 init_before_recovery (before_recovery_ptr
);
8297 barrier
= get_last_bb_insn (before_recovery
);
8298 gcc_assert (BARRIER_P (barrier
));
8300 rtx_insn
*label
= emit_label_after (gen_label_rtx (), barrier
);
8302 rec
= create_basic_block (label
, label
, before_recovery
);
8304 /* A recovery block always ends with an unconditional jump. */
8305 emit_barrier_after (BB_END (rec
));
8307 if (BB_PARTITION (before_recovery
) != BB_UNPARTITIONED
)
8308 BB_SET_PARTITION (rec
, BB_COLD_PARTITION
);
8310 if (sched_verbose
&& spec_info
->dump
)
8311 fprintf (spec_info
->dump
, ";;\t\tGenerated recovery block rec%d\n",
8317 /* Create edges: FIRST_BB -> REC; FIRST_BB -> SECOND_BB; REC -> SECOND_BB
8318 and emit necessary jumps. */
8320 sched_create_recovery_edges (basic_block first_bb
, basic_block rec
,
8321 basic_block second_bb
)
8325 /* This is fixing of incoming edge. */
8326 /* ??? Which other flags should be specified? */
8327 if (BB_PARTITION (first_bb
) != BB_PARTITION (rec
))
8328 /* Partition type is the same, if it is "unpartitioned". */
8329 edge_flags
= EDGE_CROSSING
;
8333 make_edge (first_bb
, rec
, edge_flags
);
8334 rtx_code_label
*label
= block_label (second_bb
);
8335 rtx_jump_insn
*jump
= emit_jump_insn_after (targetm
.gen_jump (label
),
8337 JUMP_LABEL (jump
) = label
;
8338 LABEL_NUSES (label
)++;
8340 if (BB_PARTITION (second_bb
) != BB_PARTITION (rec
))
8341 /* Partition type is the same, if it is "unpartitioned". */
8343 /* Rewritten from cfgrtl.c. */
8344 if (flag_reorder_blocks_and_partition
8345 && targetm_common
.have_named_sections
)
8347 /* We don't need the same note for the check because
8348 any_condjump_p (check) == true. */
8349 CROSSING_JUMP_P (jump
) = 1;
8351 edge_flags
= EDGE_CROSSING
;
8356 make_single_succ_edge (rec
, second_bb
, edge_flags
);
8357 if (dom_info_available_p (CDI_DOMINATORS
))
8358 set_immediate_dominator (CDI_DOMINATORS
, rec
, first_bb
);
8361 /* This function creates recovery code for INSN. If MUTATE_P is nonzero,
8362 INSN is a simple check, that should be converted to branchy one. */
8364 create_check_block_twin (rtx_insn
*insn
, bool mutate_p
)
8367 rtx_insn
*label
, *check
, *twin
;
8370 sd_iterator_def sd_it
;
8372 dep_def _new_dep
, *new_dep
= &_new_dep
;
8375 gcc_assert (ORIG_PAT (insn
) != NULL_RTX
);
8378 todo_spec
= TODO_SPEC (insn
);
8381 gcc_assert (IS_SPECULATION_SIMPLE_CHECK_P (insn
)
8382 && (TODO_SPEC (insn
) & SPECULATIVE
) == 0);
8384 todo_spec
= CHECK_SPEC (insn
);
8387 todo_spec
&= SPECULATIVE
;
8389 /* Create recovery block. */
8390 if (mutate_p
|| targetm
.sched
.needs_block_p (todo_spec
))
8392 rec
= sched_create_recovery_block (NULL
);
8393 label
= BB_HEAD (rec
);
8397 rec
= EXIT_BLOCK_PTR_FOR_FN (cfun
);
8402 check_pat
= targetm
.sched
.gen_spec_check (insn
, label
, todo_spec
);
8404 if (rec
!= EXIT_BLOCK_PTR_FOR_FN (cfun
))
8406 /* To have mem_reg alive at the beginning of second_bb,
8407 we emit check BEFORE insn, so insn after splitting
8408 insn will be at the beginning of second_bb, which will
8409 provide us with the correct life information. */
8410 check
= emit_jump_insn_before (check_pat
, insn
);
8411 JUMP_LABEL (check
) = label
;
8412 LABEL_NUSES (label
)++;
8415 check
= emit_insn_before (check_pat
, insn
);
8417 /* Extend data structures. */
8418 haifa_init_insn (check
);
8420 /* CHECK is being added to current region. Extend ready list. */
8421 gcc_assert (sched_ready_n_insns
!= -1);
8422 sched_extend_ready_list (sched_ready_n_insns
+ 1);
8424 if (current_sched_info
->add_remove_insn
)
8425 current_sched_info
->add_remove_insn (insn
, 0);
8427 RECOVERY_BLOCK (check
) = rec
;
8429 if (sched_verbose
&& spec_info
->dump
)
8430 fprintf (spec_info
->dump
, ";;\t\tGenerated check insn : %s\n",
8431 (*current_sched_info
->print_insn
) (check
, 0));
8433 gcc_assert (ORIG_PAT (insn
));
8435 /* Initialize TWIN (twin is a duplicate of original instruction
8436 in the recovery block). */
8437 if (rec
!= EXIT_BLOCK_PTR_FOR_FN (cfun
))
8439 sd_iterator_def sd_it
;
8442 FOR_EACH_DEP (insn
, SD_LIST_RES_BACK
, sd_it
, dep
)
8443 if ((DEP_STATUS (dep
) & DEP_OUTPUT
) != 0)
8445 struct _dep _dep2
, *dep2
= &_dep2
;
8447 init_dep (dep2
, DEP_PRO (dep
), check
, REG_DEP_TRUE
);
8449 sd_add_dep (dep2
, true);
8452 twin
= emit_insn_after (ORIG_PAT (insn
), BB_END (rec
));
8453 haifa_init_insn (twin
);
8455 if (sched_verbose
&& spec_info
->dump
)
8456 /* INSN_BB (insn) isn't determined for twin insns yet.
8457 So we can't use current_sched_info->print_insn. */
8458 fprintf (spec_info
->dump
, ";;\t\tGenerated twin insn : %d/rec%d\n",
8459 INSN_UID (twin
), rec
->index
);
8463 ORIG_PAT (check
) = ORIG_PAT (insn
);
8464 HAS_INTERNAL_DEP (check
) = 1;
8466 /* ??? We probably should change all OUTPUT dependencies to
8470 /* Copy all resolved back dependencies of INSN to TWIN. This will
8471 provide correct value for INSN_TICK (TWIN). */
8472 sd_copy_back_deps (twin
, insn
, true);
8474 if (rec
!= EXIT_BLOCK_PTR_FOR_FN (cfun
))
8475 /* In case of branchy check, fix CFG. */
8477 basic_block first_bb
, second_bb
;
8480 first_bb
= BLOCK_FOR_INSN (check
);
8481 second_bb
= sched_split_block (first_bb
, check
);
8483 sched_create_recovery_edges (first_bb
, rec
, second_bb
);
8485 sched_init_only_bb (second_bb
, first_bb
);
8486 sched_init_only_bb (rec
, EXIT_BLOCK_PTR_FOR_FN (cfun
));
8488 jump
= BB_END (rec
);
8489 haifa_init_insn (jump
);
8492 /* Move backward dependences from INSN to CHECK and
8493 move forward dependences from INSN to TWIN. */
8495 /* First, create dependencies between INSN's producers and CHECK & TWIN. */
8496 FOR_EACH_DEP (insn
, SD_LIST_BACK
, sd_it
, dep
)
8498 rtx_insn
*pro
= DEP_PRO (dep
);
8501 /* If BEGIN_DATA: [insn ~~TRUE~~> producer]:
8502 check --TRUE--> producer ??? or ANTI ???
8503 twin --TRUE--> producer
8504 twin --ANTI--> check
8506 If BEGIN_CONTROL: [insn ~~ANTI~~> producer]:
8507 check --ANTI--> producer
8508 twin --ANTI--> producer
8509 twin --ANTI--> check
8511 If BE_IN_SPEC: [insn ~~TRUE~~> producer]:
8512 check ~~TRUE~~> producer
8513 twin ~~TRUE~~> producer
8514 twin --ANTI--> check */
8516 ds
= DEP_STATUS (dep
);
8518 if (ds
& BEGIN_SPEC
)
8520 gcc_assert (!mutate_p
);
8524 init_dep_1 (new_dep
, pro
, check
, DEP_TYPE (dep
), ds
);
8525 sd_add_dep (new_dep
, false);
8527 if (rec
!= EXIT_BLOCK_PTR_FOR_FN (cfun
))
8529 DEP_CON (new_dep
) = twin
;
8530 sd_add_dep (new_dep
, false);
8534 /* Second, remove backward dependencies of INSN. */
8535 for (sd_it
= sd_iterator_start (insn
, SD_LIST_SPEC_BACK
);
8536 sd_iterator_cond (&sd_it
, &dep
);)
8538 if ((DEP_STATUS (dep
) & BEGIN_SPEC
)
8540 /* We can delete this dep because we overcome it with
8541 BEGIN_SPECULATION. */
8542 sd_delete_dep (sd_it
);
8544 sd_iterator_next (&sd_it
);
8547 /* Future Speculations. Determine what BE_IN speculations will be like. */
8550 /* Fields (DONE_SPEC (x) & BEGIN_SPEC) and CHECK_SPEC (x) are set only
8553 gcc_assert (!DONE_SPEC (insn
));
8557 ds_t ts
= TODO_SPEC (insn
);
8559 DONE_SPEC (insn
) = ts
& BEGIN_SPEC
;
8560 CHECK_SPEC (check
) = ts
& BEGIN_SPEC
;
8562 /* Luckiness of future speculations solely depends upon initial
8563 BEGIN speculation. */
8564 if (ts
& BEGIN_DATA
)
8565 fs
= set_dep_weak (fs
, BE_IN_DATA
, get_dep_weak (ts
, BEGIN_DATA
));
8566 if (ts
& BEGIN_CONTROL
)
8567 fs
= set_dep_weak (fs
, BE_IN_CONTROL
,
8568 get_dep_weak (ts
, BEGIN_CONTROL
));
8571 CHECK_SPEC (check
) = CHECK_SPEC (insn
);
8573 /* Future speculations: call the helper. */
8574 process_insn_forw_deps_be_in_spec (insn
, twin
, fs
);
8576 if (rec
!= EXIT_BLOCK_PTR_FOR_FN (cfun
))
8578 /* Which types of dependencies should we use here is,
8579 generally, machine-dependent question... But, for now,
8584 init_dep (new_dep
, insn
, check
, REG_DEP_TRUE
);
8585 sd_add_dep (new_dep
, false);
8587 init_dep (new_dep
, insn
, twin
, REG_DEP_OUTPUT
);
8588 sd_add_dep (new_dep
, false);
8592 if (spec_info
->dump
)
8593 fprintf (spec_info
->dump
, ";;\t\tRemoved simple check : %s\n",
8594 (*current_sched_info
->print_insn
) (insn
, 0));
8596 /* Remove all dependencies of the INSN. */
8598 sd_it
= sd_iterator_start (insn
, (SD_LIST_FORW
8600 | SD_LIST_RES_BACK
));
8601 while (sd_iterator_cond (&sd_it
, &dep
))
8602 sd_delete_dep (sd_it
);
8605 /* If former check (INSN) already was moved to the ready (or queue)
8606 list, add new check (CHECK) there too. */
8607 if (QUEUE_INDEX (insn
) != QUEUE_NOWHERE
)
8610 /* Remove old check from instruction stream and free its
8612 sched_remove_insn (insn
);
8615 init_dep (new_dep
, check
, twin
, REG_DEP_ANTI
);
8616 sd_add_dep (new_dep
, false);
8620 init_dep_1 (new_dep
, insn
, check
, REG_DEP_TRUE
, DEP_TRUE
| DEP_OUTPUT
);
8621 sd_add_dep (new_dep
, false);
8625 /* Fix priorities. If MUTATE_P is nonzero, this is not necessary,
8626 because it'll be done later in add_to_speculative_block. */
8628 rtx_vec_t priorities_roots
= rtx_vec_t ();
8630 clear_priorities (twin
, &priorities_roots
);
8631 calc_priorities (priorities_roots
);
8632 priorities_roots
.release ();
8636 /* Removes dependency between instructions in the recovery block REC
8637 and usual region instructions. It keeps inner dependences so it
8638 won't be necessary to recompute them. */
8640 fix_recovery_deps (basic_block rec
)
8642 rtx_insn
*note
, *insn
, *jump
;
8643 rtx_insn_list
*ready_list
= 0;
8644 bitmap_head in_ready
;
8645 rtx_insn_list
*link
;
8647 bitmap_initialize (&in_ready
, 0);
8649 /* NOTE - a basic block note. */
8650 note
= NEXT_INSN (BB_HEAD (rec
));
8651 gcc_assert (NOTE_INSN_BASIC_BLOCK_P (note
));
8652 insn
= BB_END (rec
);
8653 gcc_assert (JUMP_P (insn
));
8654 insn
= PREV_INSN (insn
);
8658 sd_iterator_def sd_it
;
8661 for (sd_it
= sd_iterator_start (insn
, SD_LIST_FORW
);
8662 sd_iterator_cond (&sd_it
, &dep
);)
8664 rtx_insn
*consumer
= DEP_CON (dep
);
8666 if (BLOCK_FOR_INSN (consumer
) != rec
)
8668 sd_delete_dep (sd_it
);
8670 if (bitmap_set_bit (&in_ready
, INSN_LUID (consumer
)))
8671 ready_list
= alloc_INSN_LIST (consumer
, ready_list
);
8675 gcc_assert ((DEP_STATUS (dep
) & DEP_TYPES
) == DEP_TRUE
);
8677 sd_iterator_next (&sd_it
);
8681 insn
= PREV_INSN (insn
);
8683 while (insn
!= note
);
8685 bitmap_clear (&in_ready
);
8687 /* Try to add instructions to the ready or queue list. */
8688 for (link
= ready_list
; link
; link
= link
->next ())
8689 try_ready (link
->insn ());
8690 free_INSN_LIST_list (&ready_list
);
8692 /* Fixing jump's dependences. */
8693 insn
= BB_HEAD (rec
);
8694 jump
= BB_END (rec
);
8696 gcc_assert (LABEL_P (insn
));
8697 insn
= NEXT_INSN (insn
);
8699 gcc_assert (NOTE_INSN_BASIC_BLOCK_P (insn
));
8700 add_jump_dependencies (insn
, jump
);
8703 /* Change pattern of INSN to NEW_PAT. Invalidate cached haifa
8704 instruction data. */
8706 haifa_change_pattern (rtx_insn
*insn
, rtx new_pat
)
8710 t
= validate_change (insn
, &PATTERN (insn
), new_pat
, 0);
8714 update_insn_after_change (insn
);
8718 /* -1 - can't speculate,
8719 0 - for speculation with REQUEST mode it is OK to use
8720 current instruction pattern,
8721 1 - need to change pattern for *NEW_PAT to be speculative. */
8723 sched_speculate_insn (rtx_insn
*insn
, ds_t request
, rtx
*new_pat
)
8725 gcc_assert (current_sched_info
->flags
& DO_SPECULATION
8726 && (request
& SPECULATIVE
)
8727 && sched_insn_is_legitimate_for_speculation_p (insn
, request
));
8729 if ((request
& spec_info
->mask
) != request
)
8732 if (request
& BE_IN_SPEC
8733 && !(request
& BEGIN_SPEC
))
8736 return targetm
.sched
.speculate_insn (insn
, request
, new_pat
);
8740 haifa_speculate_insn (rtx_insn
*insn
, ds_t request
, rtx
*new_pat
)
8742 gcc_assert (sched_deps_info
->generate_spec_deps
8743 && !IS_SPECULATION_CHECK_P (insn
));
8745 if (HAS_INTERNAL_DEP (insn
)
8746 || SCHED_GROUP_P (insn
))
8749 return sched_speculate_insn (insn
, request
, new_pat
);
8752 /* Print some information about block BB, which starts with HEAD and
8753 ends with TAIL, before scheduling it.
8754 I is zero, if scheduler is about to start with the fresh ebb. */
8756 dump_new_block_header (int i
, basic_block bb
, rtx_insn
*head
, rtx_insn
*tail
)
8759 fprintf (sched_dump
,
8760 ";; ======================================================\n");
8762 fprintf (sched_dump
,
8763 ";; =====================ADVANCING TO=====================\n");
8764 fprintf (sched_dump
,
8765 ";; -- basic block %d from %d to %d -- %s reload\n",
8766 bb
->index
, INSN_UID (head
), INSN_UID (tail
),
8767 (reload_completed
? "after" : "before"));
8768 fprintf (sched_dump
,
8769 ";; ======================================================\n");
8770 fprintf (sched_dump
, "\n");
8773 /* Unlink basic block notes and labels and saves them, so they
8774 can be easily restored. We unlink basic block notes in EBB to
8775 provide back-compatibility with the previous code, as target backends
8776 assume, that there'll be only instructions between
8777 current_sched_info->{head and tail}. We restore these notes as soon
8779 FIRST (LAST) is the first (last) basic block in the ebb.
8780 NB: In usual case (FIRST == LAST) nothing is really done. */
8782 unlink_bb_notes (basic_block first
, basic_block last
)
8784 /* We DON'T unlink basic block notes of the first block in the ebb. */
8788 bb_header
= XNEWVEC (rtx_insn
*, last_basic_block_for_fn (cfun
));
8790 /* Make a sentinel. */
8791 if (last
->next_bb
!= EXIT_BLOCK_PTR_FOR_FN (cfun
))
8792 bb_header
[last
->next_bb
->index
] = 0;
8794 first
= first
->next_bb
;
8797 rtx_insn
*prev
, *label
, *note
, *next
;
8799 label
= BB_HEAD (last
);
8800 if (LABEL_P (label
))
8801 note
= NEXT_INSN (label
);
8804 gcc_assert (NOTE_INSN_BASIC_BLOCK_P (note
));
8806 prev
= PREV_INSN (label
);
8807 next
= NEXT_INSN (note
);
8808 gcc_assert (prev
&& next
);
8810 SET_NEXT_INSN (prev
) = next
;
8811 SET_PREV_INSN (next
) = prev
;
8813 bb_header
[last
->index
] = label
;
8818 last
= last
->prev_bb
;
8823 /* Restore basic block notes.
8824 FIRST is the first basic block in the ebb. */
8826 restore_bb_notes (basic_block first
)
8831 /* We DON'T unlink basic block notes of the first block in the ebb. */
8832 first
= first
->next_bb
;
8833 /* Remember: FIRST is actually a second basic block in the ebb. */
8835 while (first
!= EXIT_BLOCK_PTR_FOR_FN (cfun
)
8836 && bb_header
[first
->index
])
8838 rtx_insn
*prev
, *label
, *note
, *next
;
8840 label
= bb_header
[first
->index
];
8841 prev
= PREV_INSN (label
);
8842 next
= NEXT_INSN (prev
);
8844 if (LABEL_P (label
))
8845 note
= NEXT_INSN (label
);
8848 gcc_assert (NOTE_INSN_BASIC_BLOCK_P (note
));
8850 bb_header
[first
->index
] = 0;
8852 SET_NEXT_INSN (prev
) = label
;
8853 SET_NEXT_INSN (note
) = next
;
8854 SET_PREV_INSN (next
) = note
;
8856 first
= first
->next_bb
;
8864 Fix CFG after both in- and inter-block movement of
8865 control_flow_insn_p JUMP. */
8867 fix_jump_move (rtx_insn
*jump
)
8869 basic_block bb
, jump_bb
, jump_bb_next
;
8871 bb
= BLOCK_FOR_INSN (PREV_INSN (jump
));
8872 jump_bb
= BLOCK_FOR_INSN (jump
);
8873 jump_bb_next
= jump_bb
->next_bb
;
8875 gcc_assert (common_sched_info
->sched_pass_id
== SCHED_EBB_PASS
8876 || IS_SPECULATION_BRANCHY_CHECK_P (jump
));
8878 if (!NOTE_INSN_BASIC_BLOCK_P (BB_END (jump_bb_next
)))
8879 /* if jump_bb_next is not empty. */
8880 BB_END (jump_bb
) = BB_END (jump_bb_next
);
8882 if (BB_END (bb
) != PREV_INSN (jump
))
8883 /* Then there are instruction after jump that should be placed
8885 BB_END (jump_bb_next
) = BB_END (bb
);
8887 /* Otherwise jump_bb_next is empty. */
8888 BB_END (jump_bb_next
) = NEXT_INSN (BB_HEAD (jump_bb_next
));
8890 /* To make assertion in move_insn happy. */
8891 BB_END (bb
) = PREV_INSN (jump
);
8893 update_bb_for_insn (jump_bb_next
);
8896 /* Fix CFG after interblock movement of control_flow_insn_p JUMP. */
8898 move_block_after_check (rtx_insn
*jump
)
8900 basic_block bb
, jump_bb
, jump_bb_next
;
8901 vec
<edge
, va_gc
> *t
;
8903 bb
= BLOCK_FOR_INSN (PREV_INSN (jump
));
8904 jump_bb
= BLOCK_FOR_INSN (jump
);
8905 jump_bb_next
= jump_bb
->next_bb
;
8907 update_bb_for_insn (jump_bb
);
8909 gcc_assert (IS_SPECULATION_CHECK_P (jump
)
8910 || IS_SPECULATION_CHECK_P (BB_END (jump_bb_next
)));
8912 unlink_block (jump_bb_next
);
8913 link_block (jump_bb_next
, bb
);
8917 move_succs (&(jump_bb
->succs
), bb
);
8918 move_succs (&(jump_bb_next
->succs
), jump_bb
);
8919 move_succs (&t
, jump_bb_next
);
8921 df_mark_solutions_dirty ();
8923 common_sched_info
->fix_recovery_cfg
8924 (bb
->index
, jump_bb
->index
, jump_bb_next
->index
);
8927 /* Helper function for move_block_after_check.
8928 This functions attaches edge vector pointed to by SUCCSP to
8931 move_succs (vec
<edge
, va_gc
> **succsp
, basic_block to
)
8936 gcc_assert (to
->succs
== 0);
8938 to
->succs
= *succsp
;
8940 FOR_EACH_EDGE (e
, ei
, to
->succs
)
8946 /* Remove INSN from the instruction stream.
8947 INSN should have any dependencies. */
8949 sched_remove_insn (rtx_insn
*insn
)
8951 sd_finish_insn (insn
);
8953 change_queue_index (insn
, QUEUE_NOWHERE
);
8954 current_sched_info
->add_remove_insn (insn
, 1);
8958 /* Clear priorities of all instructions, that are forward dependent on INSN.
8959 Store in vector pointed to by ROOTS_PTR insns on which priority () should
8960 be invoked to initialize all cleared priorities. */
8962 clear_priorities (rtx_insn
*insn
, rtx_vec_t
*roots_ptr
)
8964 sd_iterator_def sd_it
;
8966 bool insn_is_root_p
= true;
8968 gcc_assert (QUEUE_INDEX (insn
) != QUEUE_SCHEDULED
);
8970 FOR_EACH_DEP (insn
, SD_LIST_BACK
, sd_it
, dep
)
8972 rtx_insn
*pro
= DEP_PRO (dep
);
8974 if (INSN_PRIORITY_STATUS (pro
) >= 0
8975 && QUEUE_INDEX (insn
) != QUEUE_SCHEDULED
)
8977 /* If DEP doesn't contribute to priority then INSN itself should
8978 be added to priority roots. */
8979 if (contributes_to_priority_p (dep
))
8980 insn_is_root_p
= false;
8982 INSN_PRIORITY_STATUS (pro
) = -1;
8983 clear_priorities (pro
, roots_ptr
);
8988 roots_ptr
->safe_push (insn
);
8991 /* Recompute priorities of instructions, whose priorities might have been
8992 changed. ROOTS is a vector of instructions whose priority computation will
8993 trigger initialization of all cleared priorities. */
8995 calc_priorities (rtx_vec_t roots
)
9000 FOR_EACH_VEC_ELT (roots
, i
, insn
)
9005 /* Add dependences between JUMP and other instructions in the recovery
9006 block. INSN is the first insn the recovery block. */
9008 add_jump_dependencies (rtx_insn
*insn
, rtx_insn
*jump
)
9012 insn
= NEXT_INSN (insn
);
9016 if (dep_list_size (insn
, SD_LIST_FORW
) == 0)
9018 dep_def _new_dep
, *new_dep
= &_new_dep
;
9020 init_dep (new_dep
, insn
, jump
, REG_DEP_ANTI
);
9021 sd_add_dep (new_dep
, false);
9026 gcc_assert (!sd_lists_empty_p (jump
, SD_LIST_BACK
));
9029 /* Extend data structures for logical insn UID. */
9031 sched_extend_luids (void)
9033 int new_luids_max_uid
= get_max_uid () + 1;
9035 sched_luids
.safe_grow_cleared (new_luids_max_uid
);
9038 /* Initialize LUID for INSN. */
9040 sched_init_insn_luid (rtx_insn
*insn
)
9042 int i
= INSN_P (insn
) ? 1 : common_sched_info
->luid_for_non_insn (insn
);
9047 luid
= sched_max_luid
;
9048 sched_max_luid
+= i
;
9053 SET_INSN_LUID (insn
, luid
);
9056 /* Initialize luids for BBS.
9057 The hook common_sched_info->luid_for_non_insn () is used to determine
9058 if notes, labels, etc. need luids. */
9060 sched_init_luids (bb_vec_t bbs
)
9065 sched_extend_luids ();
9066 FOR_EACH_VEC_ELT (bbs
, i
, bb
)
9070 FOR_BB_INSNS (bb
, insn
)
9071 sched_init_insn_luid (insn
);
9077 sched_finish_luids (void)
9079 sched_luids
.release ();
9083 /* Return logical uid of INSN. Helpful while debugging. */
9085 insn_luid (rtx_insn
*insn
)
9087 return INSN_LUID (insn
);
9090 /* Extend per insn data in the target. */
9092 sched_extend_target (void)
9094 if (targetm
.sched
.h_i_d_extended
)
9095 targetm
.sched
.h_i_d_extended ();
9098 /* Extend global scheduler structures (those, that live across calls to
9099 schedule_block) to include information about just emitted INSN. */
9103 int reserve
= (get_max_uid () + 1 - h_i_d
.length ());
9105 && ! h_i_d
.space (reserve
))
9107 h_i_d
.safe_grow_cleared (3 * get_max_uid () / 2);
9108 sched_extend_target ();
9112 /* Initialize h_i_d entry of the INSN with default values.
9113 Values, that are not explicitly initialized here, hold zero. */
9115 init_h_i_d (rtx_insn
*insn
)
9117 if (INSN_LUID (insn
) > 0)
9119 INSN_COST (insn
) = -1;
9120 QUEUE_INDEX (insn
) = QUEUE_NOWHERE
;
9121 INSN_TICK (insn
) = INVALID_TICK
;
9122 INSN_EXACT_TICK (insn
) = INVALID_TICK
;
9123 INTER_TICK (insn
) = INVALID_TICK
;
9124 TODO_SPEC (insn
) = HARD_DEP
;
9125 INSN_AUTOPREF_MULTIPASS_DATA (insn
)[0].status
9126 = AUTOPREF_MULTIPASS_DATA_UNINITIALIZED
;
9127 INSN_AUTOPREF_MULTIPASS_DATA (insn
)[1].status
9128 = AUTOPREF_MULTIPASS_DATA_UNINITIALIZED
;
9132 /* Initialize haifa_insn_data for BBS. */
9134 haifa_init_h_i_d (bb_vec_t bbs
)
9140 FOR_EACH_VEC_ELT (bbs
, i
, bb
)
9144 FOR_BB_INSNS (bb
, insn
)
9149 /* Finalize haifa_insn_data. */
9151 haifa_finish_h_i_d (void)
9154 haifa_insn_data_t data
;
9155 struct reg_use_data
*use
, *next
;
9157 FOR_EACH_VEC_ELT (h_i_d
, i
, data
)
9159 free (data
->max_reg_pressure
);
9160 free (data
->reg_pressure
);
9161 for (use
= data
->reg_use_list
; use
!= NULL
; use
= next
)
9163 next
= use
->next_insn_use
;
9170 /* Init data for the new insn INSN. */
9172 haifa_init_insn (rtx_insn
*insn
)
9174 gcc_assert (insn
!= NULL
);
9176 sched_extend_luids ();
9177 sched_init_insn_luid (insn
);
9178 sched_extend_target ();
9179 sched_deps_init (false);
9183 if (adding_bb_to_current_region_p
)
9185 sd_init_insn (insn
);
9187 /* Extend dependency caches by one element. */
9188 extend_dependency_caches (1, false);
9190 if (sched_pressure
!= SCHED_PRESSURE_NONE
)
9191 init_insn_reg_pressure_info (insn
);
9194 /* Init data for the new basic block BB which comes after AFTER. */
9196 haifa_init_only_bb (basic_block bb
, basic_block after
)
9198 gcc_assert (bb
!= NULL
);
9202 if (common_sched_info
->add_block
)
9203 /* This changes only data structures of the front-end. */
9204 common_sched_info
->add_block (bb
, after
);
9207 /* A generic version of sched_split_block (). */
9209 sched_split_block_1 (basic_block first_bb
, rtx after
)
9213 e
= split_block (first_bb
, after
);
9214 gcc_assert (e
->src
== first_bb
);
9216 /* sched_split_block emits note if *check == BB_END. Probably it
9217 is better to rip that note off. */
9222 /* A generic version of sched_create_empty_bb (). */
9224 sched_create_empty_bb_1 (basic_block after
)
9226 return create_empty_bb (after
);
9229 /* Insert PAT as an INSN into the schedule and update the necessary data
9230 structures to account for it. */
9232 sched_emit_insn (rtx pat
)
9234 rtx_insn
*insn
= emit_insn_before (pat
, first_nonscheduled_insn ());
9235 haifa_init_insn (insn
);
9237 if (current_sched_info
->add_remove_insn
)
9238 current_sched_info
->add_remove_insn (insn
, 0);
9240 (*current_sched_info
->begin_schedule_ready
) (insn
);
9241 scheduled_insns
.safe_push (insn
);
9243 last_scheduled_insn
= insn
;
9247 /* This function returns a candidate satisfying dispatch constraints from
9251 ready_remove_first_dispatch (struct ready_list
*ready
)
9254 rtx_insn
*insn
= ready_element (ready
, 0);
9256 if (ready
->n_ready
== 1
9258 || INSN_CODE (insn
) < 0
9259 || !active_insn_p (insn
)
9260 || targetm
.sched
.dispatch (insn
, FITS_DISPATCH_WINDOW
))
9261 return ready_remove_first (ready
);
9263 for (i
= 1; i
< ready
->n_ready
; i
++)
9265 insn
= ready_element (ready
, i
);
9268 || INSN_CODE (insn
) < 0
9269 || !active_insn_p (insn
))
9272 if (targetm
.sched
.dispatch (insn
, FITS_DISPATCH_WINDOW
))
9274 /* Return ith element of ready. */
9275 insn
= ready_remove (ready
, i
);
9280 if (targetm
.sched
.dispatch (NULL
, DISPATCH_VIOLATION
))
9281 return ready_remove_first (ready
);
9283 for (i
= 1; i
< ready
->n_ready
; i
++)
9285 insn
= ready_element (ready
, i
);
9288 || INSN_CODE (insn
) < 0
9289 || !active_insn_p (insn
))
9292 /* Return i-th element of ready. */
9293 if (targetm
.sched
.dispatch (insn
, IS_CMP
))
9294 return ready_remove (ready
, i
);
9297 return ready_remove_first (ready
);
9300 /* Get number of ready insn in the ready list. */
9303 number_in_ready (void)
9305 return ready
.n_ready
;
9308 /* Get number of ready's in the ready list. */
9311 get_ready_element (int i
)
9313 return ready_element (&ready
, i
);
9316 #endif /* INSN_SCHEDULING */