]>
Commit | Line | Data |
---|---|---|
3eb9a99d | 1 | /* Instruction scheduling pass. |
711789cc | 2 | Copyright (C) 1992-2013 Free Software Foundation, Inc. |
3eb9a99d | 3 | Contributed by Michael Tiemann (tiemann@cygnus.com) Enhanced by, |
4 | and currently maintained by, Jim Wilson (wilson@cygnus.com) | |
5 | ||
f12b58b3 | 6 | This file is part of GCC. |
b1820f75 | 7 | |
f12b58b3 | 8 | GCC is free software; you can redistribute it and/or modify it under |
9 | the terms of the GNU General Public License as published by the Free | |
8c4c00c1 | 10 | Software Foundation; either version 3, or (at your option) any later |
f12b58b3 | 11 | version. |
b1820f75 | 12 | |
f12b58b3 | 13 | GCC is distributed in the hope that it will be useful, but WITHOUT ANY |
14 | WARRANTY; without even the implied warranty of MERCHANTABILITY or | |
b1820f75 | 15 | FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License |
16 | for more details. | |
17 | ||
18 | You should have received a copy of the GNU General Public License | |
8c4c00c1 | 19 | along with GCC; see the file COPYING3. If not see |
20 | <http://www.gnu.org/licenses/>. */ | |
3eb9a99d | 21 | |
7a31a7bd | 22 | /* Instruction scheduling pass. This file, along with sched-deps.c, |
23 | contains the generic parts. The actual entry point is found for | |
24 | the normal instruction scheduling pass is found in sched-rgn.c. | |
3eb9a99d | 25 | |
26 | We compute insn priorities based on data dependencies. Flow | |
27 | analysis only creates a fraction of the data-dependencies we must | |
28 | observe: namely, only those dependencies which the combiner can be | |
29 | expected to use. For this pass, we must therefore create the | |
30 | remaining dependencies we need to observe: register dependencies, | |
31 | memory dependencies, dependencies to keep function calls in order, | |
32 | and the dependence between a conditional branch and the setting of | |
33 | condition codes are all dealt with here. | |
34 | ||
35 | The scheduler first traverses the data flow graph, starting with | |
36 | the last instruction, and proceeding to the first, assigning values | |
37 | to insn_priority as it goes. This sorts the instructions | |
38 | topologically by data dependence. | |
39 | ||
40 | Once priorities have been established, we order the insns using | |
41 | list scheduling. This works as follows: starting with a list of | |
42 | all the ready insns, and sorted according to priority number, we | |
43 | schedule the insn from the end of the list by placing its | |
44 | predecessors in the list according to their priority order. We | |
45 | consider this insn scheduled by setting the pointer to the "end" of | |
46 | the list to point to the previous insn. When an insn has no | |
47 | predecessors, we either queue it until sufficient time has elapsed | |
48 | or add it to the ready list. As the instructions are scheduled or | |
49 | when stalls are introduced, the queue advances and dumps insns into | |
50 | the ready list. When all insns down to the lowest priority have | |
51 | been scheduled, the critical path of the basic block has been made | |
52 | as short as possible. The remaining insns are then scheduled in | |
53 | remaining slots. | |
54 | ||
3eb9a99d | 55 | The following list shows the order in which we want to break ties |
56 | among insns in the ready list: | |
57 | ||
58 | 1. choose insn with the longest path to end of bb, ties | |
59 | broken by | |
60 | 2. choose insn with least contribution to register pressure, | |
61 | ties broken by | |
62 | 3. prefer in-block upon interblock motion, ties broken by | |
63 | 4. prefer useful upon speculative motion, ties broken by | |
64 | 5. choose insn with largest control flow probability, ties | |
65 | broken by | |
66 | 6. choose insn with the least dependences upon the previously | |
67 | scheduled insn, or finally | |
02aef853 | 68 | 7 choose the insn which has the most insns dependent on it. |
69 | 8. choose insn with lowest UID. | |
3eb9a99d | 70 | |
71 | Memory references complicate matters. Only if we can be certain | |
72 | that memory references are not part of the data dependency graph | |
73 | (via true, anti, or output dependence), can we move operations past | |
74 | memory references. To first approximation, reads can be done | |
75 | independently, while writes introduce dependencies. Better | |
76 | approximations will yield fewer dependencies. | |
77 | ||
78 | Before reload, an extended analysis of interblock data dependences | |
79 | is required for interblock scheduling. This is performed in | |
80 | compute_block_backward_dependences (). | |
81 | ||
82 | Dependencies set up by memory references are treated in exactly the | |
9997bd27 | 83 | same way as other dependencies, by using insn backward dependences |
84 | INSN_BACK_DEPS. INSN_BACK_DEPS are translated into forward dependences | |
85 | INSN_FORW_DEPS the purpose of forward list scheduling. | |
3eb9a99d | 86 | |
87 | Having optimized the critical path, we may have also unduly | |
88 | extended the lifetimes of some registers. If an operation requires | |
89 | that constants be loaded into registers, it is certainly desirable | |
90 | to load those constants as early as necessary, but no earlier. | |
91 | I.e., it will not do to load up a bunch of registers at the | |
92 | beginning of a basic block only to use them at the end, if they | |
93 | could be loaded later, since this may result in excessive register | |
94 | utilization. | |
95 | ||
96 | Note that since branches are never in basic blocks, but only end | |
97 | basic blocks, this pass will not move branches. But that is ok, | |
98 | since we can use GNU's delayed branch scheduling pass to take care | |
99 | of this case. | |
100 | ||
101 | Also note that no further optimizations based on algebraic | |
102 | identities are performed, so this pass would be a good one to | |
103 | perform instruction splitting, such as breaking up a multiply | |
104 | instruction into shifts and adds where that is profitable. | |
105 | ||
106 | Given the memory aliasing analysis that this pass should perform, | |
107 | it should be possible to remove redundant stores to memory, and to | |
108 | load values from registers instead of hitting memory. | |
109 | ||
110 | Before reload, speculative insns are moved only if a 'proof' exists | |
111 | that no exception will be caused by this, and if no live registers | |
112 | exist that inhibit the motion (live registers constraints are not | |
113 | represented by data dependence edges). | |
114 | ||
115 | This pass must update information that subsequent passes expect to | |
116 | be correct. Namely: reg_n_refs, reg_n_sets, reg_n_deaths, | |
5496dbfc | 117 | reg_n_calls_crossed, and reg_live_length. Also, BB_HEAD, BB_END. |
3eb9a99d | 118 | |
119 | The information in the line number notes is carefully retained by | |
120 | this pass. Notes that refer to the starting and ending of | |
121 | exception regions are also carefully retained by this pass. All | |
122 | other NOTE insns are grouped in their same relative order at the | |
7a31a7bd | 123 | beginning of basic blocks and regions that have been scheduled. */ |
3eb9a99d | 124 | \f |
3eb9a99d | 125 | #include "config.h" |
46c1a957 | 126 | #include "system.h" |
805e22b2 | 127 | #include "coretypes.h" |
128 | #include "tm.h" | |
0b205f4c | 129 | #include "diagnostic-core.h" |
effd1640 | 130 | #include "hard-reg-set.h" |
3eb9a99d | 131 | #include "rtl.h" |
7953c610 | 132 | #include "tm_p.h" |
3eb9a99d | 133 | #include "regs.h" |
0a893c29 | 134 | #include "function.h" |
3eb9a99d | 135 | #include "flags.h" |
136 | #include "insn-config.h" | |
137 | #include "insn-attr.h" | |
138 | #include "except.h" | |
ba1c8484 | 139 | #include "recog.h" |
c2069298 | 140 | #include "sched-int.h" |
747af5e7 | 141 | #include "target.h" |
218e3e4e | 142 | #include "common/common-target.h" |
6a1cdb4d | 143 | #include "params.h" |
3072d30e | 144 | #include "dbgcnt.h" |
e1ab7874 | 145 | #include "cfgloop.h" |
a7dcf969 | 146 | #include "ira.h" |
06f9d6ef | 147 | #include "emit-rtl.h" /* FIXME: Can go away once crtl is moved to rtl.h. */ |
d9dd21a8 | 148 | #include "hash-table.h" |
b9ed1410 | 149 | #include "dumpfile.h" |
3eb9a99d | 150 | |
3eb9a99d | 151 | #ifdef INSN_SCHEDULING |
152 | ||
57a8bf1b | 153 | /* True if we do register pressure relief through live-range |
154 | shrinkage. */ | |
155 | static bool live_range_shrinkage_p; | |
156 | ||
157 | /* Switch on live range shrinkage. */ | |
158 | void | |
159 | initialize_live_range_shrinkage (void) | |
160 | { | |
161 | live_range_shrinkage_p = true; | |
162 | } | |
163 | ||
164 | /* Switch off live range shrinkage. */ | |
165 | void | |
166 | finish_live_range_shrinkage (void) | |
167 | { | |
168 | live_range_shrinkage_p = false; | |
169 | } | |
170 | ||
3eb9a99d | 171 | /* issue_rate is the number of insns that can be scheduled in the same |
172 | machine cycle. It can be defined in the config/mach/mach.h file, | |
173 | otherwise we set it to 1. */ | |
174 | ||
e1ab7874 | 175 | int issue_rate; |
3eb9a99d | 176 | |
e2f4a6ff | 177 | /* This can be set to true by a backend if the scheduler should not |
178 | enable a DCE pass. */ | |
179 | bool sched_no_dce; | |
180 | ||
c0457559 | 181 | /* The current initiation interval used when modulo scheduling. */ |
182 | static int modulo_ii; | |
183 | ||
184 | /* The maximum number of stages we are prepared to handle. */ | |
185 | static int modulo_max_stages; | |
186 | ||
187 | /* The number of insns that exist in each iteration of the loop. We use this | |
188 | to detect when we've scheduled all insns from the first iteration. */ | |
189 | static int modulo_n_insns; | |
190 | ||
191 | /* The current count of insns in the first iteration of the loop that have | |
192 | already been scheduled. */ | |
193 | static int modulo_insns_scheduled; | |
194 | ||
195 | /* The maximum uid of insns from the first iteration of the loop. */ | |
196 | static int modulo_iter0_max_uid; | |
197 | ||
198 | /* The number of times we should attempt to backtrack when modulo scheduling. | |
199 | Decreased each time we have to backtrack. */ | |
200 | static int modulo_backtracks_left; | |
201 | ||
202 | /* The stage in which the last insn from the original loop was | |
203 | scheduled. */ | |
204 | static int modulo_last_stage; | |
205 | ||
cc13a078 | 206 | /* sched-verbose controls the amount of debugging output the |
5c69f8a7 | 207 | scheduler prints. It is controlled by -fsched-verbose=N: |
3eb9a99d | 208 | N>0 and no -DSR : the output is directed to stderr. |
209 | N>=10 will direct the printouts to stderr (regardless of -dSR). | |
210 | N=1: same as -dSR. | |
211 | N=2: bb's probabilities, detailed ready list info, unit/insn info. | |
212 | N=3: rtl at abort point, control-flow, regions info. | |
cc13a078 | 213 | N=5: dependences info. */ |
3eb9a99d | 214 | |
7a31a7bd | 215 | int sched_verbose = 0; |
3eb9a99d | 216 | |
c4cd519a | 217 | /* Debugging file. All printouts are sent to dump, which is always set, |
3eb9a99d | 218 | either to stderr, or to the dump listing file (-dRS). */ |
10c06114 | 219 | FILE *sched_dump = 0; |
d0768316 | 220 | |
48e1416a | 221 | /* This is a placeholder for the scheduler parameters common |
e1ab7874 | 222 | to all schedulers. */ |
223 | struct common_sched_info_def *common_sched_info; | |
d28d5327 | 224 | |
e1ab7874 | 225 | #define INSN_TICK(INSN) (HID (INSN)->tick) |
e2f4a6ff | 226 | #define INSN_EXACT_TICK(INSN) (HID (INSN)->exact_tick) |
227 | #define INSN_TICK_ESTIMATE(INSN) (HID (INSN)->tick_estimate) | |
e1ab7874 | 228 | #define INTER_TICK(INSN) (HID (INSN)->inter_tick) |
e2f4a6ff | 229 | #define FEEDS_BACKTRACK_INSN(INSN) (HID (INSN)->feeds_backtrack_insn) |
230 | #define SHADOW_P(INSN) (HID (INSN)->shadow_p) | |
effd1640 | 231 | #define MUST_RECOMPUTE_SPEC_P(INSN) (HID (INSN)->must_recompute_spec) |
d452a169 | 232 | /* Cached cost of the instruction. Use insn_cost to get cost of the |
233 | insn. -1 here means that the field is not initialized. */ | |
234 | #define INSN_COST(INSN) (HID (INSN)->cost) | |
e4897000 | 235 | |
236 | /* If INSN_TICK of an instruction is equal to INVALID_TICK, | |
237 | then it should be recalculated from scratch. */ | |
238 | #define INVALID_TICK (-(max_insn_queue_index + 1)) | |
239 | /* The minimal value of the INSN_TICK of an instruction. */ | |
240 | #define MIN_TICK (-max_insn_queue_index) | |
3eb9a99d | 241 | |
3eb9a99d | 242 | /* List of important notes we must keep around. This is a pointer to the |
243 | last element in the list. */ | |
e1ab7874 | 244 | rtx note_list; |
3eb9a99d | 245 | |
6a1cdb4d | 246 | static struct spec_info_def spec_info_var; |
247 | /* Description of the speculative part of the scheduling. | |
248 | If NULL - no speculation. */ | |
e1ab7874 | 249 | spec_info_t spec_info = NULL; |
6a1cdb4d | 250 | |
251 | /* True, if recovery block was added during scheduling of current block. | |
252 | Used to determine, if we need to fix INSN_TICKs. */ | |
93f6b030 | 253 | static bool haifa_recovery_bb_recently_added_p; |
254 | ||
255 | /* True, if recovery block was added during this scheduling pass. | |
256 | Used to determine if we should have empty memory pools of dependencies | |
257 | after finishing current region. */ | |
258 | bool haifa_recovery_bb_ever_added_p; | |
6a1cdb4d | 259 | |
9ca2c29a | 260 | /* Counters of different types of speculative instructions. */ |
6a1cdb4d | 261 | static int nr_begin_data, nr_be_in_data, nr_begin_control, nr_be_in_control; |
262 | ||
6a1cdb4d | 263 | /* Array used in {unlink, restore}_bb_notes. */ |
264 | static rtx *bb_header = 0; | |
265 | ||
6a1cdb4d | 266 | /* Basic block after which recovery blocks will be created. */ |
267 | static basic_block before_recovery; | |
268 | ||
e1ab7874 | 269 | /* Basic block just before the EXIT_BLOCK and after recovery, if we have |
270 | created it. */ | |
271 | basic_block after_recovery; | |
272 | ||
273 | /* FALSE if we add bb to another region, so we don't need to initialize it. */ | |
274 | bool adding_bb_to_current_region_p = true; | |
275 | ||
3eb9a99d | 276 | /* Queues, etc. */ |
277 | ||
278 | /* An instruction is ready to be scheduled when all insns preceding it | |
279 | have already been scheduled. It is important to ensure that all | |
280 | insns which use its result will not be executed until its result | |
281 | has been computed. An insn is maintained in one of four structures: | |
282 | ||
283 | (P) the "Pending" set of insns which cannot be scheduled until | |
284 | their dependencies have been satisfied. | |
285 | (Q) the "Queued" set of insns that can be scheduled when sufficient | |
286 | time has passed. | |
287 | (R) the "Ready" list of unscheduled, uncommitted insns. | |
288 | (S) the "Scheduled" list of insns. | |
289 | ||
290 | Initially, all insns are either "Pending" or "Ready" depending on | |
291 | whether their dependencies are satisfied. | |
292 | ||
293 | Insns move from the "Ready" list to the "Scheduled" list as they | |
294 | are committed to the schedule. As this occurs, the insns in the | |
295 | "Pending" list have their dependencies satisfied and move to either | |
296 | the "Ready" list or the "Queued" set depending on whether | |
297 | sufficient time has passed to make them ready. As time passes, | |
67900a4f | 298 | insns move from the "Queued" set to the "Ready" list. |
3eb9a99d | 299 | |
9997bd27 | 300 | The "Pending" list (P) are the insns in the INSN_FORW_DEPS of the |
301 | unscheduled insns, i.e., those that are ready, queued, and pending. | |
3eb9a99d | 302 | The "Queued" set (Q) is implemented by the variable `insn_queue'. |
303 | The "Ready" list (R) is implemented by the variables `ready' and | |
304 | `n_ready'. | |
305 | The "Scheduled" list (S) is the new insn chain built by this pass. | |
306 | ||
307 | The transition (R->S) is implemented in the scheduling loop in | |
308 | `schedule_block' when the best insn to schedule is chosen. | |
3eb9a99d | 309 | The transitions (P->R and P->Q) are implemented in `schedule_insn' as |
310 | insns move from the ready list to the scheduled list. | |
311 | The transition (Q->R) is implemented in 'queue_to_insn' as time | |
312 | passes or stalls are introduced. */ | |
313 | ||
314 | /* Implement a circular buffer to delay instructions until sufficient | |
67900a4f | 315 | time has passed. For the new pipeline description interface, |
e4897000 | 316 | MAX_INSN_QUEUE_INDEX is a power of two minus one which is not less |
67900a4f | 317 | than maximal time of instruction execution computed by genattr.c on |
318 | the base maximal time of functional unit reservations and getting a | |
319 | result. This is the longest time an insn may be queued. */ | |
bea4bad2 | 320 | |
321 | static rtx *insn_queue; | |
3eb9a99d | 322 | static int q_ptr = 0; |
323 | static int q_size = 0; | |
67900a4f | 324 | #define NEXT_Q(X) (((X)+1) & max_insn_queue_index) |
325 | #define NEXT_Q_AFTER(X, C) (((X)+C) & max_insn_queue_index) | |
bea4bad2 | 326 | |
e4897000 | 327 | #define QUEUE_SCHEDULED (-3) |
328 | #define QUEUE_NOWHERE (-2) | |
329 | #define QUEUE_READY (-1) | |
330 | /* QUEUE_SCHEDULED - INSN is scheduled. | |
331 | QUEUE_NOWHERE - INSN isn't scheduled yet and is neither in | |
332 | queue or ready list. | |
333 | QUEUE_READY - INSN is in ready list. | |
334 | N >= 0 - INSN queued for X [where NEXT_Q_AFTER (q_ptr, X) == N] cycles. */ | |
48e1416a | 335 | |
e1ab7874 | 336 | #define QUEUE_INDEX(INSN) (HID (INSN)->queue_index) |
e4897000 | 337 | |
bea4bad2 | 338 | /* The following variable value refers for all current and future |
339 | reservations of the processor units. */ | |
340 | state_t curr_state; | |
341 | ||
342 | /* The following variable value is size of memory representing all | |
67900a4f | 343 | current and future reservations of the processor units. */ |
e1ab7874 | 344 | size_t dfa_state_size; |
bea4bad2 | 345 | |
346 | /* The following array is used to find the best insn from ready when | |
347 | the automaton pipeline interface is used. */ | |
e1ab7874 | 348 | char *ready_try = NULL; |
3eb9a99d | 349 | |
e1ab7874 | 350 | /* The ready list. */ |
9845d120 | 351 | struct ready_list ready = {NULL, 0, 0, 0, 0}; |
30b1ec30 | 352 | |
e1ab7874 | 353 | /* The pointer to the ready list (to be removed). */ |
354 | static struct ready_list *readyp = &ready; | |
e4897000 | 355 | |
356 | /* Scheduling clock. */ | |
357 | static int clock_var; | |
358 | ||
e2f4a6ff | 359 | /* Clock at which the previous instruction was issued. */ |
360 | static int last_clock_var; | |
361 | ||
362 | /* Set to true if, when queuing a shadow insn, we discover that it would be | |
363 | scheduled too late. */ | |
364 | static bool must_backtrack; | |
365 | ||
366 | /* The following variable value is number of essential insns issued on | |
367 | the current cycle. An insn is essential one if it changes the | |
368 | processors state. */ | |
369 | int cycle_issued_insns; | |
370 | ||
d2412f57 | 371 | /* This records the actual schedule. It is built up during the main phase |
372 | of schedule_block, and afterwards used to reorder the insns in the RTL. */ | |
f1f41a6c | 373 | static vec<rtx> scheduled_insns; |
d2412f57 | 374 | |
5493cb9a | 375 | static int may_trap_exp (const_rtx, int); |
9b7c6f02 | 376 | |
377 | /* Nonzero iff the address is comprised from at most 1 register. */ | |
378 | #define CONST_BASED_ADDRESS_P(x) \ | |
8ad4c111 | 379 | (REG_P (x) \ |
9b7c6f02 | 380 | || ((GET_CODE (x) == PLUS || GET_CODE (x) == MINUS \ |
381 | || (GET_CODE (x) == LO_SUM)) \ | |
382 | && (CONSTANT_P (XEXP (x, 0)) \ | |
383 | || CONSTANT_P (XEXP (x, 1))))) | |
384 | ||
385 | /* Returns a class that insn with GET_DEST(insn)=x may belong to, | |
386 | as found by analyzing insn's expression. */ | |
387 | ||
e1ab7874 | 388 | \f |
389 | static int haifa_luid_for_non_insn (rtx x); | |
390 | ||
391 | /* Haifa version of sched_info hooks common to all headers. */ | |
48e1416a | 392 | const struct common_sched_info_def haifa_common_sched_info = |
e1ab7874 | 393 | { |
394 | NULL, /* fix_recovery_cfg */ | |
395 | NULL, /* add_block */ | |
396 | NULL, /* estimate_number_of_insns */ | |
397 | haifa_luid_for_non_insn, /* luid_for_non_insn */ | |
398 | SCHED_PASS_UNKNOWN /* sched_pass_id */ | |
399 | }; | |
400 | ||
e1ab7874 | 401 | /* Mapping from instruction UID to its Logical UID. */ |
1e094109 | 402 | vec<int> sched_luids = vNULL; |
e1ab7874 | 403 | |
404 | /* Next LUID to assign to an instruction. */ | |
405 | int sched_max_luid = 1; | |
406 | ||
407 | /* Haifa Instruction Data. */ | |
1e094109 | 408 | vec<haifa_insn_data_def> h_i_d = vNULL; |
e1ab7874 | 409 | |
410 | void (* sched_init_only_bb) (basic_block, basic_block); | |
411 | ||
412 | /* Split block function. Different schedulers might use different functions | |
413 | to handle their internal data consistent. */ | |
414 | basic_block (* sched_split_block) (basic_block, rtx); | |
415 | ||
416 | /* Create empty basic block after the specified block. */ | |
417 | basic_block (* sched_create_empty_bb) (basic_block); | |
418 | ||
b30b031c | 419 | /* Return the number of cycles until INSN is expected to be ready. |
420 | Return zero if it already is. */ | |
421 | static int | |
422 | insn_delay (rtx insn) | |
423 | { | |
424 | return MAX (INSN_TICK (insn) - clock_var, 0); | |
425 | } | |
426 | ||
9b7c6f02 | 427 | static int |
5493cb9a | 428 | may_trap_exp (const_rtx x, int is_store) |
9b7c6f02 | 429 | { |
430 | enum rtx_code code; | |
431 | ||
432 | if (x == 0) | |
433 | return TRAP_FREE; | |
434 | code = GET_CODE (x); | |
435 | if (is_store) | |
436 | { | |
437 | if (code == MEM && may_trap_p (x)) | |
438 | return TRAP_RISKY; | |
439 | else | |
440 | return TRAP_FREE; | |
441 | } | |
442 | if (code == MEM) | |
443 | { | |
444 | /* The insn uses memory: a volatile load. */ | |
445 | if (MEM_VOLATILE_P (x)) | |
446 | return IRISKY; | |
447 | /* An exception-free load. */ | |
448 | if (!may_trap_p (x)) | |
449 | return IFREE; | |
450 | /* A load with 1 base register, to be further checked. */ | |
451 | if (CONST_BASED_ADDRESS_P (XEXP (x, 0))) | |
452 | return PFREE_CANDIDATE; | |
453 | /* No info on the load, to be further checked. */ | |
454 | return PRISKY_CANDIDATE; | |
455 | } | |
456 | else | |
457 | { | |
458 | const char *fmt; | |
459 | int i, insn_class = TRAP_FREE; | |
460 | ||
461 | /* Neither store nor load, check if it may cause a trap. */ | |
462 | if (may_trap_p (x)) | |
463 | return TRAP_RISKY; | |
464 | /* Recursive step: walk the insn... */ | |
465 | fmt = GET_RTX_FORMAT (code); | |
466 | for (i = GET_RTX_LENGTH (code) - 1; i >= 0; i--) | |
467 | { | |
468 | if (fmt[i] == 'e') | |
469 | { | |
470 | int tmp_class = may_trap_exp (XEXP (x, i), is_store); | |
471 | insn_class = WORST_CLASS (insn_class, tmp_class); | |
472 | } | |
473 | else if (fmt[i] == 'E') | |
474 | { | |
475 | int j; | |
476 | for (j = 0; j < XVECLEN (x, i); j++) | |
477 | { | |
478 | int tmp_class = may_trap_exp (XVECEXP (x, i, j), is_store); | |
479 | insn_class = WORST_CLASS (insn_class, tmp_class); | |
480 | if (insn_class == TRAP_RISKY || insn_class == IRISKY) | |
481 | break; | |
482 | } | |
483 | } | |
484 | if (insn_class == TRAP_RISKY || insn_class == IRISKY) | |
485 | break; | |
486 | } | |
487 | return insn_class; | |
488 | } | |
489 | } | |
490 | ||
f1b700e6 | 491 | /* Classifies rtx X of an insn for the purpose of verifying that X can be |
492 | executed speculatively (and consequently the insn can be moved | |
493 | speculatively), by examining X, returning: | |
9b7c6f02 | 494 | TRAP_RISKY: store, or risky non-load insn (e.g. division by variable). |
495 | TRAP_FREE: non-load insn. | |
91c82c20 | 496 | IFREE: load from a globally safe location. |
9b7c6f02 | 497 | IRISKY: volatile load. |
498 | PFREE_CANDIDATE, PRISKY_CANDIDATE: load that need to be checked for | |
499 | being either PFREE or PRISKY. */ | |
500 | ||
f1b700e6 | 501 | static int |
502 | haifa_classify_rtx (const_rtx x) | |
9b7c6f02 | 503 | { |
9b7c6f02 | 504 | int tmp_class = TRAP_FREE; |
505 | int insn_class = TRAP_FREE; | |
506 | enum rtx_code code; | |
507 | ||
f1b700e6 | 508 | if (GET_CODE (x) == PARALLEL) |
9b7c6f02 | 509 | { |
f1b700e6 | 510 | int i, len = XVECLEN (x, 0); |
9b7c6f02 | 511 | |
512 | for (i = len - 1; i >= 0; i--) | |
513 | { | |
f1b700e6 | 514 | tmp_class = haifa_classify_rtx (XVECEXP (x, 0, i)); |
9b7c6f02 | 515 | insn_class = WORST_CLASS (insn_class, tmp_class); |
516 | if (insn_class == TRAP_RISKY || insn_class == IRISKY) | |
517 | break; | |
518 | } | |
519 | } | |
520 | else | |
521 | { | |
f1b700e6 | 522 | code = GET_CODE (x); |
9b7c6f02 | 523 | switch (code) |
524 | { | |
525 | case CLOBBER: | |
526 | /* Test if it is a 'store'. */ | |
f1b700e6 | 527 | tmp_class = may_trap_exp (XEXP (x, 0), 1); |
9b7c6f02 | 528 | break; |
529 | case SET: | |
530 | /* Test if it is a store. */ | |
f1b700e6 | 531 | tmp_class = may_trap_exp (SET_DEST (x), 1); |
9b7c6f02 | 532 | if (tmp_class == TRAP_RISKY) |
533 | break; | |
534 | /* Test if it is a load. */ | |
535 | tmp_class = | |
536 | WORST_CLASS (tmp_class, | |
f1b700e6 | 537 | may_trap_exp (SET_SRC (x), 0)); |
9b7c6f02 | 538 | break; |
539 | case COND_EXEC: | |
f1b700e6 | 540 | tmp_class = haifa_classify_rtx (COND_EXEC_CODE (x)); |
541 | if (tmp_class == TRAP_RISKY) | |
542 | break; | |
543 | tmp_class = WORST_CLASS (tmp_class, | |
544 | may_trap_exp (COND_EXEC_TEST (x), 0)); | |
545 | break; | |
9b7c6f02 | 546 | case TRAP_IF: |
547 | tmp_class = TRAP_RISKY; | |
548 | break; | |
549 | default:; | |
550 | } | |
551 | insn_class = tmp_class; | |
552 | } | |
553 | ||
554 | return insn_class; | |
555 | } | |
556 | ||
f1b700e6 | 557 | int |
558 | haifa_classify_insn (const_rtx insn) | |
559 | { | |
560 | return haifa_classify_rtx (PATTERN (insn)); | |
561 | } | |
c0457559 | 562 | \f |
563 | /* After the scheduler initialization function has been called, this function | |
564 | can be called to enable modulo scheduling. II is the initiation interval | |
565 | we should use, it affects the delays for delay_pairs that were recorded as | |
566 | separated by a given number of stages. | |
567 | ||
568 | MAX_STAGES provides us with a limit | |
569 | after which we give up scheduling; the caller must have unrolled at least | |
570 | as many copies of the loop body and recorded delay_pairs for them. | |
571 | ||
572 | INSNS is the number of real (non-debug) insns in one iteration of | |
573 | the loop. MAX_UID can be used to test whether an insn belongs to | |
574 | the first iteration of the loop; all of them have a uid lower than | |
575 | MAX_UID. */ | |
576 | void | |
577 | set_modulo_params (int ii, int max_stages, int insns, int max_uid) | |
578 | { | |
579 | modulo_ii = ii; | |
580 | modulo_max_stages = max_stages; | |
581 | modulo_n_insns = insns; | |
582 | modulo_iter0_max_uid = max_uid; | |
583 | modulo_backtracks_left = PARAM_VALUE (PARAM_MAX_MODULO_BACKTRACK_ATTEMPTS); | |
584 | } | |
f1b700e6 | 585 | |
e2f4a6ff | 586 | /* A structure to record a pair of insns where the first one is a real |
587 | insn that has delay slots, and the second is its delayed shadow. | |
588 | I1 is scheduled normally and will emit an assembly instruction, | |
589 | while I2 describes the side effect that takes place at the | |
590 | transition between cycles CYCLES and (CYCLES + 1) after I1. */ | |
591 | struct delay_pair | |
592 | { | |
593 | struct delay_pair *next_same_i1; | |
594 | rtx i1, i2; | |
595 | int cycles; | |
c0457559 | 596 | /* When doing modulo scheduling, we a delay_pair can also be used to |
597 | show that I1 and I2 are the same insn in a different stage. If that | |
598 | is the case, STAGES will be nonzero. */ | |
599 | int stages; | |
e2f4a6ff | 600 | }; |
601 | ||
d9dd21a8 | 602 | /* Helpers for delay hashing. */ |
603 | ||
604 | struct delay_i1_hasher : typed_noop_remove <delay_pair> | |
605 | { | |
606 | typedef delay_pair value_type; | |
607 | typedef void compare_type; | |
608 | static inline hashval_t hash (const value_type *); | |
609 | static inline bool equal (const value_type *, const compare_type *); | |
610 | }; | |
611 | ||
612 | /* Returns a hash value for X, based on hashing just I1. */ | |
613 | ||
614 | inline hashval_t | |
615 | delay_i1_hasher::hash (const value_type *x) | |
616 | { | |
617 | return htab_hash_pointer (x->i1); | |
618 | } | |
619 | ||
620 | /* Return true if I1 of pair X is the same as that of pair Y. */ | |
621 | ||
622 | inline bool | |
623 | delay_i1_hasher::equal (const value_type *x, const compare_type *y) | |
624 | { | |
625 | return x->i1 == y; | |
626 | } | |
627 | ||
628 | struct delay_i2_hasher : typed_free_remove <delay_pair> | |
629 | { | |
630 | typedef delay_pair value_type; | |
631 | typedef void compare_type; | |
632 | static inline hashval_t hash (const value_type *); | |
633 | static inline bool equal (const value_type *, const compare_type *); | |
634 | }; | |
635 | ||
636 | /* Returns a hash value for X, based on hashing just I2. */ | |
637 | ||
638 | inline hashval_t | |
639 | delay_i2_hasher::hash (const value_type *x) | |
640 | { | |
641 | return htab_hash_pointer (x->i2); | |
642 | } | |
643 | ||
644 | /* Return true if I2 of pair X is the same as that of pair Y. */ | |
645 | ||
646 | inline bool | |
647 | delay_i2_hasher::equal (const value_type *x, const compare_type *y) | |
648 | { | |
649 | return x->i2 == y; | |
650 | } | |
651 | ||
e2f4a6ff | 652 | /* Two hash tables to record delay_pairs, one indexed by I1 and the other |
653 | indexed by I2. */ | |
d9dd21a8 | 654 | static hash_table <delay_i1_hasher> delay_htab; |
655 | static hash_table <delay_i2_hasher> delay_htab_i2; | |
e2f4a6ff | 656 | |
c0457559 | 657 | /* Called through htab_traverse. Walk the hashtable using I2 as |
658 | index, and delete all elements involving an UID higher than | |
659 | that pointed to by *DATA. */ | |
d9dd21a8 | 660 | int |
661 | haifa_htab_i2_traverse (delay_pair **slot, int *data) | |
c0457559 | 662 | { |
d9dd21a8 | 663 | int maxuid = *data; |
664 | struct delay_pair *p = *slot; | |
c0457559 | 665 | if (INSN_UID (p->i2) >= maxuid || INSN_UID (p->i1) >= maxuid) |
666 | { | |
d9dd21a8 | 667 | delay_htab_i2.clear_slot (slot); |
c0457559 | 668 | } |
669 | return 1; | |
670 | } | |
671 | ||
672 | /* Called through htab_traverse. Walk the hashtable using I2 as | |
673 | index, and delete all elements involving an UID higher than | |
674 | that pointed to by *DATA. */ | |
d9dd21a8 | 675 | int |
676 | haifa_htab_i1_traverse (delay_pair **pslot, int *data) | |
c0457559 | 677 | { |
d9dd21a8 | 678 | int maxuid = *data; |
c0457559 | 679 | struct delay_pair *p, *first, **pprev; |
680 | ||
681 | if (INSN_UID ((*pslot)->i1) >= maxuid) | |
682 | { | |
d9dd21a8 | 683 | delay_htab.clear_slot (pslot); |
c0457559 | 684 | return 1; |
685 | } | |
686 | pprev = &first; | |
687 | for (p = *pslot; p; p = p->next_same_i1) | |
688 | { | |
689 | if (INSN_UID (p->i2) < maxuid) | |
690 | { | |
691 | *pprev = p; | |
692 | pprev = &p->next_same_i1; | |
693 | } | |
694 | } | |
695 | *pprev = NULL; | |
696 | if (first == NULL) | |
d9dd21a8 | 697 | delay_htab.clear_slot (pslot); |
c0457559 | 698 | else |
699 | *pslot = first; | |
700 | return 1; | |
701 | } | |
702 | ||
703 | /* Discard all delay pairs which involve an insn with an UID higher | |
704 | than MAX_UID. */ | |
705 | void | |
706 | discard_delay_pairs_above (int max_uid) | |
707 | { | |
d9dd21a8 | 708 | delay_htab.traverse <int *, haifa_htab_i1_traverse> (&max_uid); |
709 | delay_htab_i2.traverse <int *, haifa_htab_i2_traverse> (&max_uid); | |
e2f4a6ff | 710 | } |
711 | ||
c0457559 | 712 | /* This function can be called by a port just before it starts the final |
713 | scheduling pass. It records the fact that an instruction with delay | |
714 | slots has been split into two insns, I1 and I2. The first one will be | |
715 | scheduled normally and initiates the operation. The second one is a | |
716 | shadow which must follow a specific number of cycles after I1; its only | |
717 | purpose is to show the side effect that occurs at that cycle in the RTL. | |
718 | If a JUMP_INSN or a CALL_INSN has been split, I1 should be a normal INSN, | |
719 | while I2 retains the original insn type. | |
720 | ||
721 | There are two ways in which the number of cycles can be specified, | |
722 | involving the CYCLES and STAGES arguments to this function. If STAGES | |
723 | is zero, we just use the value of CYCLES. Otherwise, STAGES is a factor | |
724 | which is multiplied by MODULO_II to give the number of cycles. This is | |
725 | only useful if the caller also calls set_modulo_params to enable modulo | |
726 | scheduling. */ | |
e2f4a6ff | 727 | |
728 | void | |
c0457559 | 729 | record_delay_slot_pair (rtx i1, rtx i2, int cycles, int stages) |
e2f4a6ff | 730 | { |
731 | struct delay_pair *p = XNEW (struct delay_pair); | |
732 | struct delay_pair **slot; | |
733 | ||
734 | p->i1 = i1; | |
735 | p->i2 = i2; | |
736 | p->cycles = cycles; | |
c0457559 | 737 | p->stages = stages; |
e2f4a6ff | 738 | |
d9dd21a8 | 739 | if (!delay_htab.is_created ()) |
e2f4a6ff | 740 | { |
d9dd21a8 | 741 | delay_htab.create (10); |
742 | delay_htab_i2.create (10); | |
e2f4a6ff | 743 | } |
d9dd21a8 | 744 | slot = delay_htab.find_slot_with_hash (i1, htab_hash_pointer (i1), INSERT); |
e2f4a6ff | 745 | p->next_same_i1 = *slot; |
746 | *slot = p; | |
d9dd21a8 | 747 | slot = delay_htab_i2.find_slot_with_hash (i2, htab_hash_pointer (i2), INSERT); |
e2f4a6ff | 748 | *slot = p; |
749 | } | |
750 | ||
effd1640 | 751 | /* Examine the delay pair hashtable to see if INSN is a shadow for another, |
752 | and return the other insn if so. Return NULL otherwise. */ | |
753 | rtx | |
754 | real_insn_for_shadow (rtx insn) | |
755 | { | |
756 | struct delay_pair *pair; | |
757 | ||
d9dd21a8 | 758 | if (!delay_htab.is_created ()) |
effd1640 | 759 | return NULL_RTX; |
760 | ||
d9dd21a8 | 761 | pair = delay_htab_i2.find_with_hash (insn, htab_hash_pointer (insn)); |
effd1640 | 762 | if (!pair || pair->stages > 0) |
763 | return NULL_RTX; | |
764 | return pair->i1; | |
765 | } | |
766 | ||
e2f4a6ff | 767 | /* For a pair P of insns, return the fixed distance in cycles from the first |
768 | insn after which the second must be scheduled. */ | |
769 | static int | |
770 | pair_delay (struct delay_pair *p) | |
771 | { | |
c0457559 | 772 | if (p->stages == 0) |
773 | return p->cycles; | |
774 | else | |
775 | return p->stages * modulo_ii; | |
e2f4a6ff | 776 | } |
777 | ||
778 | /* Given an insn INSN, add a dependence on its delayed shadow if it | |
779 | has one. Also try to find situations where shadows depend on each other | |
780 | and add dependencies to the real insns to limit the amount of backtracking | |
781 | needed. */ | |
782 | void | |
783 | add_delay_dependencies (rtx insn) | |
784 | { | |
785 | struct delay_pair *pair; | |
786 | sd_iterator_def sd_it; | |
787 | dep_t dep; | |
788 | ||
d9dd21a8 | 789 | if (!delay_htab.is_created ()) |
e2f4a6ff | 790 | return; |
791 | ||
d9dd21a8 | 792 | pair = delay_htab_i2.find_with_hash (insn, htab_hash_pointer (insn)); |
e2f4a6ff | 793 | if (!pair) |
794 | return; | |
795 | add_dependence (insn, pair->i1, REG_DEP_ANTI); | |
c0457559 | 796 | if (pair->stages) |
797 | return; | |
e2f4a6ff | 798 | |
799 | FOR_EACH_DEP (pair->i2, SD_LIST_BACK, sd_it, dep) | |
800 | { | |
801 | rtx pro = DEP_PRO (dep); | |
802 | struct delay_pair *other_pair | |
d9dd21a8 | 803 | = delay_htab_i2.find_with_hash (pro, htab_hash_pointer (pro)); |
c0457559 | 804 | if (!other_pair || other_pair->stages) |
e2f4a6ff | 805 | continue; |
806 | if (pair_delay (other_pair) >= pair_delay (pair)) | |
807 | { | |
808 | if (sched_verbose >= 4) | |
809 | { | |
810 | fprintf (sched_dump, ";;\tadding dependence %d <- %d\n", | |
811 | INSN_UID (other_pair->i1), | |
812 | INSN_UID (pair->i1)); | |
813 | fprintf (sched_dump, ";;\tpair1 %d <- %d, cost %d\n", | |
814 | INSN_UID (pair->i1), | |
815 | INSN_UID (pair->i2), | |
816 | pair_delay (pair)); | |
817 | fprintf (sched_dump, ";;\tpair2 %d <- %d, cost %d\n", | |
818 | INSN_UID (other_pair->i1), | |
819 | INSN_UID (other_pair->i2), | |
820 | pair_delay (other_pair)); | |
821 | } | |
822 | add_dependence (pair->i1, other_pair->i1, REG_DEP_ANTI); | |
823 | } | |
824 | } | |
825 | } | |
826 | \f | |
3eb9a99d | 827 | /* Forward declarations. */ |
bea4bad2 | 828 | |
952f0048 | 829 | static int priority (rtx); |
830 | static int rank_for_schedule (const void *, const void *); | |
831 | static void swap_sort (rtx *, int); | |
b49b2e5c | 832 | static void queue_insn (rtx, int, const char *); |
e4897000 | 833 | static int schedule_insn (rtx); |
952f0048 | 834 | static void adjust_priority (rtx); |
835 | static void advance_one_cycle (void); | |
e1ab7874 | 836 | static void extend_h_i_d (void); |
837 | ||
3eb9a99d | 838 | |
3eb9a99d | 839 | /* Notes handling mechanism: |
840 | ========================= | |
841 | Generally, NOTES are saved before scheduling and restored after scheduling. | |
3953ee1c | 842 | The scheduler distinguishes between two types of notes: |
3eb9a99d | 843 | |
f2aa2401 | 844 | (1) LOOP_BEGIN, LOOP_END, SETJMP, EHREGION_BEG, EHREGION_END notes: |
3eb9a99d | 845 | Before scheduling a region, a pointer to the note is added to the insn |
846 | that follows or precedes it. (This happens as part of the data dependence | |
847 | computation). After scheduling an insn, the pointer contained in it is | |
848 | used for regenerating the corresponding note (in reemit_notes). | |
849 | ||
f2aa2401 | 850 | (2) All other notes (e.g. INSN_DELETED): Before scheduling a block, |
3eb9a99d | 851 | these notes are put in a list (in rm_other_notes() and |
852 | unlink_other_notes ()). After scheduling the block, these notes are | |
853 | inserted at the beginning of the block (in schedule_block()). */ | |
854 | ||
e4897000 | 855 | static void ready_add (struct ready_list *, rtx, bool); |
952f0048 | 856 | static rtx ready_remove_first (struct ready_list *); |
0ac9454e | 857 | static rtx ready_remove_first_dispatch (struct ready_list *ready); |
38b9004f | 858 | |
952f0048 | 859 | static void queue_to_ready (struct ready_list *); |
52c4b43f | 860 | static int early_queue_to_ready (state_t, struct ready_list *); |
30b1ec30 | 861 | |
952f0048 | 862 | static void debug_ready_list (struct ready_list *); |
38b9004f | 863 | |
bea4bad2 | 864 | /* The following functions are used to implement multi-pass scheduling |
67900a4f | 865 | on the first cycle. */ |
952f0048 | 866 | static rtx ready_remove (struct ready_list *, int); |
e4897000 | 867 | static void ready_remove_insn (rtx); |
bea4bad2 | 868 | |
e4897000 | 869 | static void fix_inter_tick (rtx, rtx); |
870 | static int fix_tick_ready (rtx); | |
871 | static void change_queue_index (rtx, int); | |
e4897000 | 872 | |
6a1cdb4d | 873 | /* The following functions are used to implement scheduling of data/control |
874 | speculative instructions. */ | |
875 | ||
876 | static void extend_h_i_d (void); | |
6a1cdb4d | 877 | static void init_h_i_d (rtx); |
effd1640 | 878 | static int haifa_speculate_insn (rtx, ds_t, rtx *); |
6a1cdb4d | 879 | static void generate_recovery_code (rtx); |
93f6b030 | 880 | static void process_insn_forw_deps_be_in_spec (rtx, rtx, ds_t); |
6a1cdb4d | 881 | static void begin_speculative_block (rtx); |
882 | static void add_to_speculative_block (rtx); | |
e1ab7874 | 883 | static void init_before_recovery (basic_block *); |
6a1cdb4d | 884 | static void create_check_block_twin (rtx, bool); |
885 | static void fix_recovery_deps (basic_block); | |
effd1640 | 886 | static bool haifa_change_pattern (rtx, rtx); |
6a1cdb4d | 887 | static void dump_new_block_header (int, basic_block, rtx, rtx); |
888 | static void restore_bb_notes (basic_block); | |
6a1cdb4d | 889 | static void fix_jump_move (rtx); |
890 | static void move_block_after_check (rtx); | |
f1f41a6c | 891 | static void move_succs (vec<edge, va_gc> **, basic_block); |
6a1cdb4d | 892 | static void sched_remove_insn (rtx); |
3588ab9c | 893 | static void clear_priorities (rtx, rtx_vec_t *); |
894 | static void calc_priorities (rtx_vec_t); | |
6a1cdb4d | 895 | static void add_jump_dependencies (rtx, rtx); |
6a1cdb4d | 896 | |
3eb9a99d | 897 | #endif /* INSN_SCHEDULING */ |
898 | \f | |
c2069298 | 899 | /* Point to state used for the current scheduling pass. */ |
e1ab7874 | 900 | struct haifa_sched_info *current_sched_info; |
3eb9a99d | 901 | \f |
902 | #ifndef INSN_SCHEDULING | |
903 | void | |
3f5be5f4 | 904 | schedule_insns (void) |
3eb9a99d | 905 | { |
906 | } | |
907 | #else | |
3e016693 | 908 | |
a7dcf969 | 909 | /* Do register pressure sensitive insn scheduling if the flag is set |
910 | up. */ | |
11189c7a | 911 | enum sched_pressure_algorithm sched_pressure; |
a7dcf969 | 912 | |
66d9a7b9 | 913 | /* Map regno -> its pressure class. The map defined only when |
b30b031c | 914 | SCHED_PRESSURE != SCHED_PRESSURE_NONE. */ |
66d9a7b9 | 915 | enum reg_class *sched_regno_pressure_class; |
a7dcf969 | 916 | |
66d9a7b9 | 917 | /* The current register pressure. Only elements corresponding pressure |
a7dcf969 | 918 | classes are defined. */ |
919 | static int curr_reg_pressure[N_REG_CLASSES]; | |
920 | ||
921 | /* Saved value of the previous array. */ | |
922 | static int saved_reg_pressure[N_REG_CLASSES]; | |
923 | ||
924 | /* Register living at given scheduling point. */ | |
925 | static bitmap curr_reg_live; | |
926 | ||
927 | /* Saved value of the previous array. */ | |
928 | static bitmap saved_reg_live; | |
929 | ||
930 | /* Registers mentioned in the current region. */ | |
931 | static bitmap region_ref_regs; | |
932 | ||
933 | /* Initiate register pressure relative info for scheduling the current | |
934 | region. Currently it is only clearing register mentioned in the | |
935 | current region. */ | |
936 | void | |
937 | sched_init_region_reg_pressure_info (void) | |
938 | { | |
939 | bitmap_clear (region_ref_regs); | |
940 | } | |
941 | ||
b30b031c | 942 | /* PRESSURE[CL] describes the pressure on register class CL. Update it |
943 | for the birth (if BIRTH_P) or death (if !BIRTH_P) of register REGNO. | |
944 | LIVE tracks the set of live registers; if it is null, assume that | |
945 | every birth or death is genuine. */ | |
946 | static inline void | |
947 | mark_regno_birth_or_death (bitmap live, int *pressure, int regno, bool birth_p) | |
a7dcf969 | 948 | { |
66d9a7b9 | 949 | enum reg_class pressure_class; |
a7dcf969 | 950 | |
66d9a7b9 | 951 | pressure_class = sched_regno_pressure_class[regno]; |
a7dcf969 | 952 | if (regno >= FIRST_PSEUDO_REGISTER) |
953 | { | |
66d9a7b9 | 954 | if (pressure_class != NO_REGS) |
a7dcf969 | 955 | { |
956 | if (birth_p) | |
957 | { | |
b30b031c | 958 | if (!live || bitmap_set_bit (live, regno)) |
959 | pressure[pressure_class] | |
960 | += (ira_reg_class_max_nregs | |
961 | [pressure_class][PSEUDO_REGNO_MODE (regno)]); | |
a7dcf969 | 962 | } |
963 | else | |
964 | { | |
b30b031c | 965 | if (!live || bitmap_clear_bit (live, regno)) |
966 | pressure[pressure_class] | |
967 | -= (ira_reg_class_max_nregs | |
968 | [pressure_class][PSEUDO_REGNO_MODE (regno)]); | |
a7dcf969 | 969 | } |
970 | } | |
971 | } | |
66d9a7b9 | 972 | else if (pressure_class != NO_REGS |
a7dcf969 | 973 | && ! TEST_HARD_REG_BIT (ira_no_alloc_regs, regno)) |
974 | { | |
975 | if (birth_p) | |
976 | { | |
b30b031c | 977 | if (!live || bitmap_set_bit (live, regno)) |
978 | pressure[pressure_class]++; | |
a7dcf969 | 979 | } |
980 | else | |
981 | { | |
b30b031c | 982 | if (!live || bitmap_clear_bit (live, regno)) |
983 | pressure[pressure_class]--; | |
a7dcf969 | 984 | } |
985 | } | |
986 | } | |
987 | ||
988 | /* Initiate current register pressure related info from living | |
989 | registers given by LIVE. */ | |
990 | static void | |
991 | initiate_reg_pressure_info (bitmap live) | |
992 | { | |
993 | int i; | |
994 | unsigned int j; | |
995 | bitmap_iterator bi; | |
996 | ||
66d9a7b9 | 997 | for (i = 0; i < ira_pressure_classes_num; i++) |
998 | curr_reg_pressure[ira_pressure_classes[i]] = 0; | |
a7dcf969 | 999 | bitmap_clear (curr_reg_live); |
1000 | EXECUTE_IF_SET_IN_BITMAP (live, 0, j, bi) | |
b30b031c | 1001 | if (sched_pressure == SCHED_PRESSURE_MODEL |
1002 | || current_nr_blocks == 1 | |
1003 | || bitmap_bit_p (region_ref_regs, j)) | |
1004 | mark_regno_birth_or_death (curr_reg_live, curr_reg_pressure, j, true); | |
a7dcf969 | 1005 | } |
1006 | ||
1007 | /* Mark registers in X as mentioned in the current region. */ | |
1008 | static void | |
1009 | setup_ref_regs (rtx x) | |
1010 | { | |
1011 | int i, j, regno; | |
1012 | const RTX_CODE code = GET_CODE (x); | |
1013 | const char *fmt; | |
1014 | ||
1015 | if (REG_P (x)) | |
1016 | { | |
1017 | regno = REGNO (x); | |
771d4616 | 1018 | if (HARD_REGISTER_NUM_P (regno)) |
1019 | bitmap_set_range (region_ref_regs, regno, | |
1020 | hard_regno_nregs[regno][GET_MODE (x)]); | |
a7dcf969 | 1021 | else |
771d4616 | 1022 | bitmap_set_bit (region_ref_regs, REGNO (x)); |
a7dcf969 | 1023 | return; |
1024 | } | |
1025 | fmt = GET_RTX_FORMAT (code); | |
1026 | for (i = GET_RTX_LENGTH (code) - 1; i >= 0; i--) | |
1027 | if (fmt[i] == 'e') | |
1028 | setup_ref_regs (XEXP (x, i)); | |
1029 | else if (fmt[i] == 'E') | |
1030 | { | |
1031 | for (j = 0; j < XVECLEN (x, i); j++) | |
1032 | setup_ref_regs (XVECEXP (x, i, j)); | |
1033 | } | |
1034 | } | |
1035 | ||
1036 | /* Initiate current register pressure related info at the start of | |
1037 | basic block BB. */ | |
1038 | static void | |
1039 | initiate_bb_reg_pressure_info (basic_block bb) | |
1040 | { | |
d92517d3 | 1041 | unsigned int i ATTRIBUTE_UNUSED; |
a7dcf969 | 1042 | rtx insn; |
1043 | ||
1044 | if (current_nr_blocks > 1) | |
1045 | FOR_BB_INSNS (bb, insn) | |
20f8d64b | 1046 | if (NONDEBUG_INSN_P (insn)) |
a7dcf969 | 1047 | setup_ref_regs (PATTERN (insn)); |
1048 | initiate_reg_pressure_info (df_get_live_in (bb)); | |
1049 | #ifdef EH_RETURN_DATA_REGNO | |
1050 | if (bb_has_eh_pred (bb)) | |
1051 | for (i = 0; ; ++i) | |
1052 | { | |
1053 | unsigned int regno = EH_RETURN_DATA_REGNO (i); | |
48e1416a | 1054 | |
a7dcf969 | 1055 | if (regno == INVALID_REGNUM) |
1056 | break; | |
1057 | if (! bitmap_bit_p (df_get_live_in (bb), regno)) | |
b30b031c | 1058 | mark_regno_birth_or_death (curr_reg_live, curr_reg_pressure, |
1059 | regno, true); | |
a7dcf969 | 1060 | } |
1061 | #endif | |
1062 | } | |
1063 | ||
1064 | /* Save current register pressure related info. */ | |
1065 | static void | |
1066 | save_reg_pressure (void) | |
1067 | { | |
1068 | int i; | |
48e1416a | 1069 | |
66d9a7b9 | 1070 | for (i = 0; i < ira_pressure_classes_num; i++) |
1071 | saved_reg_pressure[ira_pressure_classes[i]] | |
1072 | = curr_reg_pressure[ira_pressure_classes[i]]; | |
a7dcf969 | 1073 | bitmap_copy (saved_reg_live, curr_reg_live); |
1074 | } | |
1075 | ||
1076 | /* Restore saved register pressure related info. */ | |
1077 | static void | |
1078 | restore_reg_pressure (void) | |
1079 | { | |
1080 | int i; | |
48e1416a | 1081 | |
66d9a7b9 | 1082 | for (i = 0; i < ira_pressure_classes_num; i++) |
1083 | curr_reg_pressure[ira_pressure_classes[i]] | |
1084 | = saved_reg_pressure[ira_pressure_classes[i]]; | |
a7dcf969 | 1085 | bitmap_copy (curr_reg_live, saved_reg_live); |
1086 | } | |
1087 | ||
1088 | /* Return TRUE if the register is dying after its USE. */ | |
1089 | static bool | |
1090 | dying_use_p (struct reg_use_data *use) | |
1091 | { | |
1092 | struct reg_use_data *next; | |
1093 | ||
1094 | for (next = use->next_regno_use; next != use; next = next->next_regno_use) | |
08f2a34b | 1095 | if (NONDEBUG_INSN_P (next->insn) |
1096 | && QUEUE_INDEX (next->insn) != QUEUE_SCHEDULED) | |
a7dcf969 | 1097 | return false; |
1098 | return true; | |
1099 | } | |
1100 | ||
1101 | /* Print info about the current register pressure and its excess for | |
66d9a7b9 | 1102 | each pressure class. */ |
a7dcf969 | 1103 | static void |
1104 | print_curr_reg_pressure (void) | |
1105 | { | |
1106 | int i; | |
1107 | enum reg_class cl; | |
1108 | ||
1109 | fprintf (sched_dump, ";;\t"); | |
66d9a7b9 | 1110 | for (i = 0; i < ira_pressure_classes_num; i++) |
a7dcf969 | 1111 | { |
66d9a7b9 | 1112 | cl = ira_pressure_classes[i]; |
a7dcf969 | 1113 | gcc_assert (curr_reg_pressure[cl] >= 0); |
1114 | fprintf (sched_dump, " %s:%d(%d)", reg_class_names[cl], | |
1115 | curr_reg_pressure[cl], | |
1072fecf | 1116 | curr_reg_pressure[cl] - ira_class_hard_regs_num[cl]); |
a7dcf969 | 1117 | } |
1118 | fprintf (sched_dump, "\n"); | |
1119 | } | |
effd1640 | 1120 | \f |
1121 | /* Determine if INSN has a condition that is clobbered if a register | |
1122 | in SET_REGS is modified. */ | |
1123 | static bool | |
1124 | cond_clobbered_p (rtx insn, HARD_REG_SET set_regs) | |
1125 | { | |
1126 | rtx pat = PATTERN (insn); | |
1127 | gcc_assert (GET_CODE (pat) == COND_EXEC); | |
1128 | if (TEST_HARD_REG_BIT (set_regs, REGNO (XEXP (COND_EXEC_TEST (pat), 0)))) | |
1129 | { | |
1130 | sd_iterator_def sd_it; | |
1131 | dep_t dep; | |
1132 | haifa_change_pattern (insn, ORIG_PAT (insn)); | |
1133 | FOR_EACH_DEP (insn, SD_LIST_BACK, sd_it, dep) | |
1134 | DEP_STATUS (dep) &= ~DEP_CANCELLED; | |
1135 | TODO_SPEC (insn) = HARD_DEP; | |
1136 | if (sched_verbose >= 2) | |
1137 | fprintf (sched_dump, | |
1138 | ";;\t\tdequeue insn %s because of clobbered condition\n", | |
1139 | (*current_sched_info->print_insn) (insn, 0)); | |
1140 | return true; | |
1141 | } | |
1142 | ||
1143 | return false; | |
1144 | } | |
1145 | ||
d452a169 | 1146 | /* This function should be called after modifying the pattern of INSN, |
1147 | to update scheduler data structures as needed. */ | |
1148 | static void | |
1149 | update_insn_after_change (rtx insn) | |
1150 | { | |
1151 | sd_iterator_def sd_it; | |
1152 | dep_t dep; | |
1153 | ||
1154 | dfa_clear_single_insn_cache (insn); | |
1155 | ||
1156 | sd_it = sd_iterator_start (insn, | |
1157 | SD_LIST_FORW | SD_LIST_BACK | SD_LIST_RES_BACK); | |
1158 | while (sd_iterator_cond (&sd_it, &dep)) | |
1159 | { | |
1160 | DEP_COST (dep) = UNKNOWN_DEP_COST; | |
1161 | sd_iterator_next (&sd_it); | |
1162 | } | |
1163 | ||
1164 | /* Invalidate INSN_COST, so it'll be recalculated. */ | |
1165 | INSN_COST (insn) = -1; | |
1166 | /* Invalidate INSN_TICK, so it'll be recalculated. */ | |
1167 | INSN_TICK (insn) = INVALID_TICK; | |
1168 | } | |
1169 | ||
d452a169 | 1170 | |
1171 | /* Two VECs, one to hold dependencies for which pattern replacements | |
1172 | need to be applied or restored at the start of the next cycle, and | |
1173 | another to hold an integer that is either one, to apply the | |
1174 | corresponding replacement, or zero to restore it. */ | |
f1f41a6c | 1175 | static vec<dep_t> next_cycle_replace_deps; |
1176 | static vec<int> next_cycle_apply; | |
d452a169 | 1177 | |
1178 | static void apply_replacement (dep_t, bool); | |
1179 | static void restore_pattern (dep_t, bool); | |
1180 | ||
effd1640 | 1181 | /* Look at the remaining dependencies for insn NEXT, and compute and return |
1182 | the TODO_SPEC value we should use for it. This is called after one of | |
d452a169 | 1183 | NEXT's dependencies has been resolved. |
1184 | We also perform pattern replacements for predication, and for broken | |
1185 | replacement dependencies. The latter is only done if FOR_BACKTRACK is | |
1186 | false. */ | |
a7dcf969 | 1187 | |
effd1640 | 1188 | static ds_t |
d452a169 | 1189 | recompute_todo_spec (rtx next, bool for_backtrack) |
effd1640 | 1190 | { |
1191 | ds_t new_ds; | |
1192 | sd_iterator_def sd_it; | |
d452a169 | 1193 | dep_t dep, modify_dep = NULL; |
effd1640 | 1194 | int n_spec = 0; |
1195 | int n_control = 0; | |
d452a169 | 1196 | int n_replace = 0; |
effd1640 | 1197 | bool first_p = true; |
1198 | ||
1199 | if (sd_lists_empty_p (next, SD_LIST_BACK)) | |
1200 | /* NEXT has all its dependencies resolved. */ | |
1201 | return 0; | |
1202 | ||
1203 | if (!sd_lists_empty_p (next, SD_LIST_HARD_BACK)) | |
1204 | return HARD_DEP; | |
1205 | ||
1206 | /* Now we've got NEXT with speculative deps only. | |
1207 | 1. Look at the deps to see what we have to do. | |
1208 | 2. Check if we can do 'todo'. */ | |
1209 | new_ds = 0; | |
1210 | ||
1211 | FOR_EACH_DEP (next, SD_LIST_BACK, sd_it, dep) | |
1212 | { | |
d452a169 | 1213 | rtx pro = DEP_PRO (dep); |
effd1640 | 1214 | ds_t ds = DEP_STATUS (dep) & SPECULATIVE; |
1215 | ||
d452a169 | 1216 | if (DEBUG_INSN_P (pro) && !DEBUG_INSN_P (next)) |
effd1640 | 1217 | continue; |
1218 | ||
1219 | if (ds) | |
1220 | { | |
1221 | n_spec++; | |
1222 | if (first_p) | |
1223 | { | |
1224 | first_p = false; | |
1225 | ||
1226 | new_ds = ds; | |
1227 | } | |
1228 | else | |
1229 | new_ds = ds_merge (new_ds, ds); | |
1230 | } | |
d452a169 | 1231 | else if (DEP_TYPE (dep) == REG_DEP_CONTROL) |
effd1640 | 1232 | { |
d452a169 | 1233 | if (QUEUE_INDEX (pro) != QUEUE_SCHEDULED) |
1234 | { | |
1235 | n_control++; | |
1236 | modify_dep = dep; | |
1237 | } | |
1238 | DEP_STATUS (dep) &= ~DEP_CANCELLED; | |
1239 | } | |
1240 | else if (DEP_REPLACE (dep) != NULL) | |
1241 | { | |
1242 | if (QUEUE_INDEX (pro) != QUEUE_SCHEDULED) | |
1243 | { | |
1244 | n_replace++; | |
1245 | modify_dep = dep; | |
1246 | } | |
effd1640 | 1247 | DEP_STATUS (dep) &= ~DEP_CANCELLED; |
1248 | } | |
1249 | } | |
1250 | ||
d452a169 | 1251 | if (n_replace > 0 && n_control == 0 && n_spec == 0) |
1252 | { | |
1253 | if (!dbg_cnt (sched_breakdep)) | |
1254 | return HARD_DEP; | |
1255 | FOR_EACH_DEP (next, SD_LIST_BACK, sd_it, dep) | |
1256 | { | |
1257 | struct dep_replacement *desc = DEP_REPLACE (dep); | |
1258 | if (desc != NULL) | |
1259 | { | |
1260 | if (desc->insn == next && !for_backtrack) | |
1261 | { | |
1262 | gcc_assert (n_replace == 1); | |
1263 | apply_replacement (dep, true); | |
1264 | } | |
1265 | DEP_STATUS (dep) |= DEP_CANCELLED; | |
1266 | } | |
1267 | } | |
1268 | return 0; | |
1269 | } | |
1270 | ||
1271 | else if (n_control == 1 && n_replace == 0 && n_spec == 0) | |
effd1640 | 1272 | { |
1273 | rtx pro, other, new_pat; | |
1274 | rtx cond = NULL_RTX; | |
1275 | bool success; | |
1276 | rtx prev = NULL_RTX; | |
1277 | int i; | |
1278 | unsigned regno; | |
1279 | ||
1280 | if ((current_sched_info->flags & DO_PREDICATION) == 0 | |
1281 | || (ORIG_PAT (next) != NULL_RTX | |
1282 | && PREDICATED_PAT (next) == NULL_RTX)) | |
1283 | return HARD_DEP; | |
1284 | ||
d452a169 | 1285 | pro = DEP_PRO (modify_dep); |
effd1640 | 1286 | other = real_insn_for_shadow (pro); |
1287 | if (other != NULL_RTX) | |
1288 | pro = other; | |
1289 | ||
1290 | cond = sched_get_reverse_condition_uncached (pro); | |
1291 | regno = REGNO (XEXP (cond, 0)); | |
1292 | ||
1293 | /* Find the last scheduled insn that modifies the condition register. | |
7aa6606c | 1294 | We can stop looking once we find the insn we depend on through the |
1295 | REG_DEP_CONTROL; if the condition register isn't modified after it, | |
1296 | we know that it still has the right value. */ | |
1297 | if (QUEUE_INDEX (pro) == QUEUE_SCHEDULED) | |
f1f41a6c | 1298 | FOR_EACH_VEC_ELT_REVERSE (scheduled_insns, i, prev) |
7aa6606c | 1299 | { |
1300 | HARD_REG_SET t; | |
1301 | ||
1302 | find_all_hard_reg_sets (prev, &t); | |
1303 | if (TEST_HARD_REG_BIT (t, regno)) | |
1304 | return HARD_DEP; | |
1305 | if (prev == pro) | |
1306 | break; | |
1307 | } | |
effd1640 | 1308 | if (ORIG_PAT (next) == NULL_RTX) |
1309 | { | |
1310 | ORIG_PAT (next) = PATTERN (next); | |
1311 | ||
1312 | new_pat = gen_rtx_COND_EXEC (VOIDmode, cond, PATTERN (next)); | |
1313 | success = haifa_change_pattern (next, new_pat); | |
1314 | if (!success) | |
1315 | return HARD_DEP; | |
1316 | PREDICATED_PAT (next) = new_pat; | |
1317 | } | |
1318 | else if (PATTERN (next) != PREDICATED_PAT (next)) | |
1319 | { | |
1320 | bool success = haifa_change_pattern (next, | |
1321 | PREDICATED_PAT (next)); | |
1322 | gcc_assert (success); | |
1323 | } | |
d452a169 | 1324 | DEP_STATUS (modify_dep) |= DEP_CANCELLED; |
effd1640 | 1325 | return DEP_CONTROL; |
1326 | } | |
1327 | ||
1328 | if (PREDICATED_PAT (next) != NULL_RTX) | |
1329 | { | |
1330 | int tick = INSN_TICK (next); | |
1331 | bool success = haifa_change_pattern (next, | |
1332 | ORIG_PAT (next)); | |
1333 | INSN_TICK (next) = tick; | |
1334 | gcc_assert (success); | |
1335 | } | |
1336 | ||
1337 | /* We can't handle the case where there are both speculative and control | |
1338 | dependencies, so we return HARD_DEP in such a case. Also fail if | |
1339 | we have speculative dependencies with not enough points, or more than | |
1340 | one control dependency. */ | |
d452a169 | 1341 | if ((n_spec > 0 && (n_control > 0 || n_replace > 0)) |
effd1640 | 1342 | || (n_spec > 0 |
1343 | /* Too few points? */ | |
1344 | && ds_weak (new_ds) < spec_info->data_weakness_cutoff) | |
d452a169 | 1345 | || n_control > 0 |
1346 | || n_replace > 0) | |
effd1640 | 1347 | return HARD_DEP; |
1348 | ||
1349 | return new_ds; | |
1350 | } | |
1351 | \f | |
b5f4bfde | 1352 | /* Pointer to the last instruction scheduled. */ |
3eb9a99d | 1353 | static rtx last_scheduled_insn; |
1354 | ||
d0aa45d0 | 1355 | /* Pointer to the last nondebug instruction scheduled within the |
1356 | block, or the prev_head of the scheduling block. Used by | |
1357 | rank_for_schedule, so that insns independent of the last scheduled | |
1358 | insn will be preferred over dependent instructions. */ | |
1359 | static rtx last_nondebug_scheduled_insn; | |
1360 | ||
b5f4bfde | 1361 | /* Pointer that iterates through the list of unscheduled insns if we |
1362 | have a dbg_cnt enabled. It always points at an insn prior to the | |
1363 | first unscheduled one. */ | |
1364 | static rtx nonscheduled_insns_begin; | |
1365 | ||
9997bd27 | 1366 | /* Compute cost of executing INSN. |
6a1cdb4d | 1367 | This is the number of cycles between instruction issue and |
1368 | instruction results. */ | |
c623bf22 | 1369 | int |
9997bd27 | 1370 | insn_cost (rtx insn) |
3eb9a99d | 1371 | { |
e1ab7874 | 1372 | int cost; |
1373 | ||
1374 | if (sel_sched_p ()) | |
1375 | { | |
1376 | if (recog_memoized (insn) < 0) | |
1377 | return 0; | |
1378 | ||
1379 | cost = insn_default_latency (insn); | |
1380 | if (cost < 0) | |
1381 | cost = 0; | |
1382 | ||
1383 | return cost; | |
1384 | } | |
1385 | ||
1386 | cost = INSN_COST (insn); | |
3eb9a99d | 1387 | |
bea4bad2 | 1388 | if (cost < 0) |
3eb9a99d | 1389 | { |
bea4bad2 | 1390 | /* A USE insn, or something else we don't need to |
1391 | understand. We can't pass these directly to | |
1392 | result_ready_cost or insn_default_latency because it will | |
1393 | trigger a fatal error for unrecognizable insns. */ | |
1394 | if (recog_memoized (insn) < 0) | |
3eb9a99d | 1395 | { |
bea4bad2 | 1396 | INSN_COST (insn) = 0; |
1397 | return 0; | |
3eb9a99d | 1398 | } |
1399 | else | |
1400 | { | |
67900a4f | 1401 | cost = insn_default_latency (insn); |
bea4bad2 | 1402 | if (cost < 0) |
1403 | cost = 0; | |
952f0048 | 1404 | |
3eb9a99d | 1405 | INSN_COST (insn) = cost; |
1406 | } | |
1407 | } | |
1408 | ||
9997bd27 | 1409 | return cost; |
1410 | } | |
1411 | ||
1412 | /* Compute cost of dependence LINK. | |
1413 | This is the number of cycles between instruction issue and | |
cdf13311 | 1414 | instruction results. |
1415 | ??? We also use this function to call recog_memoized on all insns. */ | |
9997bd27 | 1416 | int |
e1ab7874 | 1417 | dep_cost_1 (dep_t link, dw_t dw) |
9997bd27 | 1418 | { |
cdf13311 | 1419 | rtx insn = DEP_PRO (link); |
9997bd27 | 1420 | rtx used = DEP_CON (link); |
1421 | int cost; | |
3eb9a99d | 1422 | |
38354bb6 | 1423 | if (DEP_COST (link) != UNKNOWN_DEP_COST) |
1424 | return DEP_COST (link); | |
1425 | ||
d9dd21a8 | 1426 | if (delay_htab.is_created ()) |
e2f4a6ff | 1427 | { |
1428 | struct delay_pair *delay_entry; | |
1429 | delay_entry | |
d9dd21a8 | 1430 | = delay_htab_i2.find_with_hash (used, htab_hash_pointer (used)); |
e2f4a6ff | 1431 | if (delay_entry) |
1432 | { | |
1433 | if (delay_entry->i1 == insn) | |
1434 | { | |
1435 | DEP_COST (link) = pair_delay (delay_entry); | |
1436 | return DEP_COST (link); | |
1437 | } | |
1438 | } | |
1439 | } | |
1440 | ||
bea4bad2 | 1441 | /* A USE insn should never require the value used to be computed. |
1442 | This allows the computation of a function's result and parameter | |
a7dcf969 | 1443 | values to overlap the return and call. We don't care about the |
851d9296 | 1444 | dependence cost when only decreasing register pressure. */ |
bea4bad2 | 1445 | if (recog_memoized (used) < 0) |
cdf13311 | 1446 | { |
1447 | cost = 0; | |
1448 | recog_memoized (insn); | |
1449 | } | |
bea4bad2 | 1450 | else |
3eb9a99d | 1451 | { |
93f6b030 | 1452 | enum reg_note dep_type = DEP_TYPE (link); |
9997bd27 | 1453 | |
1454 | cost = insn_cost (insn); | |
6a1cdb4d | 1455 | |
67900a4f | 1456 | if (INSN_CODE (insn) >= 0) |
de206697 | 1457 | { |
6a1cdb4d | 1458 | if (dep_type == REG_DEP_ANTI) |
67900a4f | 1459 | cost = 0; |
6a1cdb4d | 1460 | else if (dep_type == REG_DEP_OUTPUT) |
bea4bad2 | 1461 | { |
67900a4f | 1462 | cost = (insn_default_latency (insn) |
1463 | - insn_default_latency (used)); | |
1464 | if (cost <= 0) | |
1465 | cost = 1; | |
bea4bad2 | 1466 | } |
67900a4f | 1467 | else if (bypass_p (insn)) |
1468 | cost = insn_latency (insn, used); | |
de206697 | 1469 | } |
48e1416a | 1470 | |
17c7a04d | 1471 | |
e1ab7874 | 1472 | if (targetm.sched.adjust_cost_2) |
a7dcf969 | 1473 | cost = targetm.sched.adjust_cost_2 (used, (int) dep_type, insn, cost, |
1474 | dw); | |
e1ab7874 | 1475 | else if (targetm.sched.adjust_cost != NULL) |
6a1cdb4d | 1476 | { |
9997bd27 | 1477 | /* This variable is used for backward compatibility with the |
1478 | targets. */ | |
1479 | rtx dep_cost_rtx_link = alloc_INSN_LIST (NULL_RTX, NULL_RTX); | |
1480 | ||
1481 | /* Make it self-cycled, so that if some tries to walk over this | |
f2b32076 | 1482 | incomplete list he/she will be caught in an endless loop. */ |
9997bd27 | 1483 | XEXP (dep_cost_rtx_link, 1) = dep_cost_rtx_link; |
1484 | ||
1485 | /* Targets use only REG_NOTE_KIND of the link. */ | |
93f6b030 | 1486 | PUT_REG_NOTE_KIND (dep_cost_rtx_link, DEP_TYPE (link)); |
9997bd27 | 1487 | |
1488 | cost = targetm.sched.adjust_cost (used, dep_cost_rtx_link, | |
1489 | insn, cost); | |
1490 | ||
1491 | free_INSN_LIST_node (dep_cost_rtx_link); | |
6a1cdb4d | 1492 | } |
bea4bad2 | 1493 | |
1494 | if (cost < 0) | |
1495 | cost = 0; | |
1496 | } | |
952f0048 | 1497 | |
38354bb6 | 1498 | DEP_COST (link) = cost; |
3eb9a99d | 1499 | return cost; |
1500 | } | |
1501 | ||
e1ab7874 | 1502 | /* Compute cost of dependence LINK. |
1503 | This is the number of cycles between instruction issue and | |
1504 | instruction results. */ | |
1505 | int | |
1506 | dep_cost (dep_t link) | |
1507 | { | |
1508 | return dep_cost_1 (link, 0); | |
1509 | } | |
1510 | ||
1511 | /* Use this sel-sched.c friendly function in reorder2 instead of increasing | |
1512 | INSN_PRIORITY explicitly. */ | |
1513 | void | |
1514 | increase_insn_priority (rtx insn, int amount) | |
1515 | { | |
1516 | if (!sel_sched_p ()) | |
1517 | { | |
1518 | /* We're dealing with haifa-sched.c INSN_PRIORITY. */ | |
1519 | if (INSN_PRIORITY_KNOWN (insn)) | |
1520 | INSN_PRIORITY (insn) += amount; | |
1521 | } | |
1522 | else | |
1523 | { | |
48e1416a | 1524 | /* In sel-sched.c INSN_PRIORITY is not kept up to date. |
e1ab7874 | 1525 | Use EXPR_PRIORITY instead. */ |
1526 | sel_add_to_insn_priority (insn, amount); | |
1527 | } | |
1528 | } | |
1529 | ||
3588ab9c | 1530 | /* Return 'true' if DEP should be included in priority calculations. */ |
1531 | static bool | |
1532 | contributes_to_priority_p (dep_t dep) | |
1533 | { | |
9845d120 | 1534 | if (DEBUG_INSN_P (DEP_CON (dep)) |
1535 | || DEBUG_INSN_P (DEP_PRO (dep))) | |
1536 | return false; | |
1537 | ||
3588ab9c | 1538 | /* Critical path is meaningful in block boundaries only. */ |
1539 | if (!current_sched_info->contributes_to_priority (DEP_CON (dep), | |
1540 | DEP_PRO (dep))) | |
1541 | return false; | |
1542 | ||
d452a169 | 1543 | if (DEP_REPLACE (dep) != NULL) |
1544 | return false; | |
1545 | ||
3588ab9c | 1546 | /* If flag COUNT_SPEC_IN_CRITICAL_PATH is set, |
1547 | then speculative instructions will less likely be | |
1548 | scheduled. That is because the priority of | |
1549 | their producers will increase, and, thus, the | |
1550 | producers will more likely be scheduled, thus, | |
1551 | resolving the dependence. */ | |
e1ab7874 | 1552 | if (sched_deps_info->generate_spec_deps |
3588ab9c | 1553 | && !(spec_info->flags & COUNT_SPEC_IN_CRITICAL_PATH) |
1554 | && (DEP_STATUS (dep) & SPECULATIVE)) | |
1555 | return false; | |
3eb9a99d | 1556 | |
3588ab9c | 1557 | return true; |
1558 | } | |
1559 | ||
b30b031c | 1560 | /* Compute the number of nondebug deps in list LIST for INSN. */ |
9845d120 | 1561 | |
1562 | static int | |
b30b031c | 1563 | dep_list_size (rtx insn, sd_list_types_def list) |
9845d120 | 1564 | { |
1565 | sd_iterator_def sd_it; | |
1566 | dep_t dep; | |
1567 | int dbgcount = 0, nodbgcount = 0; | |
1568 | ||
1569 | if (!MAY_HAVE_DEBUG_INSNS) | |
b30b031c | 1570 | return sd_lists_size (insn, list); |
9845d120 | 1571 | |
b30b031c | 1572 | FOR_EACH_DEP (insn, list, sd_it, dep) |
9845d120 | 1573 | { |
1574 | if (DEBUG_INSN_P (DEP_CON (dep))) | |
1575 | dbgcount++; | |
c971b445 | 1576 | else if (!DEBUG_INSN_P (DEP_PRO (dep))) |
9845d120 | 1577 | nodbgcount++; |
1578 | } | |
1579 | ||
b30b031c | 1580 | gcc_assert (dbgcount + nodbgcount == sd_lists_size (insn, list)); |
9845d120 | 1581 | |
1582 | return nodbgcount; | |
1583 | } | |
1584 | ||
3588ab9c | 1585 | /* Compute the priority number for INSN. */ |
3eb9a99d | 1586 | static int |
952f0048 | 1587 | priority (rtx insn) |
3eb9a99d | 1588 | { |
9204e736 | 1589 | if (! INSN_P (insn)) |
3eb9a99d | 1590 | return 0; |
1591 | ||
310d2511 | 1592 | /* We should not be interested in priority of an already scheduled insn. */ |
3588ab9c | 1593 | gcc_assert (QUEUE_INDEX (insn) != QUEUE_SCHEDULED); |
1594 | ||
1595 | if (!INSN_PRIORITY_KNOWN (insn)) | |
3eb9a99d | 1596 | { |
e1ab7874 | 1597 | int this_priority = -1; |
89beeed3 | 1598 | |
b30b031c | 1599 | if (dep_list_size (insn, SD_LIST_FORW) == 0) |
9997bd27 | 1600 | /* ??? We should set INSN_PRIORITY to insn_cost when and insn has |
1601 | some forward deps but all of them are ignored by | |
1602 | contributes_to_priority hook. At the moment we set priority of | |
1603 | such insn to 0. */ | |
1604 | this_priority = insn_cost (insn); | |
3eb9a99d | 1605 | else |
89beeed3 | 1606 | { |
6a1cdb4d | 1607 | rtx prev_first, twin; |
1608 | basic_block rec; | |
3eb9a99d | 1609 | |
6a1cdb4d | 1610 | /* For recovery check instructions we calculate priority slightly |
1611 | different than that of normal instructions. Instead of walking | |
9997bd27 | 1612 | through INSN_FORW_DEPS (check) list, we walk through |
1613 | INSN_FORW_DEPS list of each instruction in the corresponding | |
48e1416a | 1614 | recovery block. */ |
3eb9a99d | 1615 | |
e1ab7874 | 1616 | /* Selective scheduling does not define RECOVERY_BLOCK macro. */ |
1617 | rec = sel_sched_p () ? NULL : RECOVERY_BLOCK (insn); | |
34154e27 | 1618 | if (!rec || rec == EXIT_BLOCK_PTR_FOR_FN (cfun)) |
6a1cdb4d | 1619 | { |
1620 | prev_first = PREV_INSN (insn); | |
1621 | twin = insn; | |
1622 | } | |
1623 | else | |
1624 | { | |
1625 | prev_first = NEXT_INSN (BB_HEAD (rec)); | |
1626 | twin = PREV_INSN (BB_END (rec)); | |
1627 | } | |
3eb9a99d | 1628 | |
6a1cdb4d | 1629 | do |
1630 | { | |
93f6b030 | 1631 | sd_iterator_def sd_it; |
1632 | dep_t dep; | |
1633 | ||
1634 | FOR_EACH_DEP (twin, SD_LIST_FORW, sd_it, dep) | |
6a1cdb4d | 1635 | { |
1636 | rtx next; | |
1637 | int next_priority; | |
9997bd27 | 1638 | |
1639 | next = DEP_CON (dep); | |
1640 | ||
6a1cdb4d | 1641 | if (BLOCK_FOR_INSN (next) != rec) |
1642 | { | |
9997bd27 | 1643 | int cost; |
1644 | ||
3588ab9c | 1645 | if (!contributes_to_priority_p (dep)) |
6a1cdb4d | 1646 | continue; |
9997bd27 | 1647 | |
1648 | if (twin == insn) | |
1649 | cost = dep_cost (dep); | |
1650 | else | |
1651 | { | |
1652 | struct _dep _dep1, *dep1 = &_dep1; | |
1653 | ||
1654 | init_dep (dep1, insn, next, REG_DEP_ANTI); | |
1655 | ||
1656 | cost = dep_cost (dep1); | |
1657 | } | |
1658 | ||
1659 | next_priority = cost + priority (next); | |
6a1cdb4d | 1660 | |
1661 | if (next_priority > this_priority) | |
1662 | this_priority = next_priority; | |
1663 | } | |
1664 | } | |
48e1416a | 1665 | |
6a1cdb4d | 1666 | twin = PREV_INSN (twin); |
89beeed3 | 1667 | } |
6a1cdb4d | 1668 | while (twin != prev_first); |
89beeed3 | 1669 | } |
e1ab7874 | 1670 | |
1671 | if (this_priority < 0) | |
1672 | { | |
1673 | gcc_assert (this_priority == -1); | |
1674 | ||
1675 | this_priority = insn_cost (insn); | |
1676 | } | |
1677 | ||
3eb9a99d | 1678 | INSN_PRIORITY (insn) = this_priority; |
3588ab9c | 1679 | INSN_PRIORITY_STATUS (insn) = 1; |
3eb9a99d | 1680 | } |
89beeed3 | 1681 | |
1682 | return INSN_PRIORITY (insn); | |
3eb9a99d | 1683 | } |
1684 | \f | |
3eb9a99d | 1685 | /* Macros and functions for keeping the priority queue sorted, and |
40e55fbb | 1686 | dealing with queuing and dequeuing of instructions. */ |
3eb9a99d | 1687 | |
1688 | #define SCHED_SORT(READY, N_READY) \ | |
1689 | do { if ((N_READY) == 2) \ | |
1690 | swap_sort (READY, N_READY); \ | |
1691 | else if ((N_READY) > 2) \ | |
1692 | qsort (READY, N_READY, sizeof (rtx), rank_for_schedule); } \ | |
1693 | while (0) | |
1694 | ||
b30b031c | 1695 | /* For each pressure class CL, set DEATH[CL] to the number of registers |
1696 | in that class that die in INSN. */ | |
1697 | ||
1698 | static void | |
1699 | calculate_reg_deaths (rtx insn, int *death) | |
1700 | { | |
1701 | int i; | |
1702 | struct reg_use_data *use; | |
1703 | ||
1704 | for (i = 0; i < ira_pressure_classes_num; i++) | |
1705 | death[ira_pressure_classes[i]] = 0; | |
1706 | for (use = INSN_REG_USE_LIST (insn); use != NULL; use = use->next_insn_use) | |
1707 | if (dying_use_p (use)) | |
1708 | mark_regno_birth_or_death (0, death, use->regno, true); | |
1709 | } | |
1710 | ||
a7dcf969 | 1711 | /* Setup info about the current register pressure impact of scheduling |
1712 | INSN at the current scheduling point. */ | |
1713 | static void | |
1714 | setup_insn_reg_pressure_info (rtx insn) | |
1715 | { | |
1716 | int i, change, before, after, hard_regno; | |
1717 | int excess_cost_change; | |
1718 | enum machine_mode mode; | |
1719 | enum reg_class cl; | |
1720 | struct reg_pressure_data *pressure_info; | |
1721 | int *max_reg_pressure; | |
a7dcf969 | 1722 | static int death[N_REG_CLASSES]; |
1723 | ||
10272f4e | 1724 | gcc_checking_assert (!DEBUG_INSN_P (insn)); |
1725 | ||
a7dcf969 | 1726 | excess_cost_change = 0; |
b30b031c | 1727 | calculate_reg_deaths (insn, death); |
a7dcf969 | 1728 | pressure_info = INSN_REG_PRESSURE (insn); |
1729 | max_reg_pressure = INSN_MAX_REG_PRESSURE (insn); | |
1730 | gcc_assert (pressure_info != NULL && max_reg_pressure != NULL); | |
66d9a7b9 | 1731 | for (i = 0; i < ira_pressure_classes_num; i++) |
a7dcf969 | 1732 | { |
66d9a7b9 | 1733 | cl = ira_pressure_classes[i]; |
a7dcf969 | 1734 | gcc_assert (curr_reg_pressure[cl] >= 0); |
1735 | change = (int) pressure_info[i].set_increase - death[cl]; | |
1072fecf | 1736 | before = MAX (0, max_reg_pressure[i] - ira_class_hard_regs_num[cl]); |
a7dcf969 | 1737 | after = MAX (0, max_reg_pressure[i] + change |
1072fecf | 1738 | - ira_class_hard_regs_num[cl]); |
a7dcf969 | 1739 | hard_regno = ira_class_hard_regs[cl][0]; |
1740 | gcc_assert (hard_regno >= 0); | |
1741 | mode = reg_raw_mode[hard_regno]; | |
1742 | excess_cost_change += ((after - before) | |
1743 | * (ira_memory_move_cost[mode][cl][0] | |
1744 | + ira_memory_move_cost[mode][cl][1])); | |
1745 | } | |
1746 | INSN_REG_PRESSURE_EXCESS_COST_CHANGE (insn) = excess_cost_change; | |
1747 | } | |
b30b031c | 1748 | \f |
1749 | /* This is the first page of code related to SCHED_PRESSURE_MODEL. | |
1750 | It tries to make the scheduler take register pressure into account | |
1751 | without introducing too many unnecessary stalls. It hooks into the | |
1752 | main scheduling algorithm at several points: | |
1753 | ||
1754 | - Before scheduling starts, model_start_schedule constructs a | |
1755 | "model schedule" for the current block. This model schedule is | |
1756 | chosen solely to keep register pressure down. It does not take the | |
1757 | target's pipeline or the original instruction order into account, | |
1758 | except as a tie-breaker. It also doesn't work to a particular | |
1759 | pressure limit. | |
1760 | ||
1761 | This model schedule gives us an idea of what pressure can be | |
1762 | achieved for the block and gives us an example of a schedule that | |
1763 | keeps to that pressure. It also makes the final schedule less | |
1764 | dependent on the original instruction order. This is important | |
1765 | because the original order can either be "wide" (many values live | |
1766 | at once, such as in user-scheduled code) or "narrow" (few values | |
1767 | live at once, such as after loop unrolling, where several | |
1768 | iterations are executed sequentially). | |
1769 | ||
1770 | We do not apply this model schedule to the rtx stream. We simply | |
1771 | record it in model_schedule. We also compute the maximum pressure, | |
1772 | MP, that was seen during this schedule. | |
1773 | ||
1774 | - Instructions are added to the ready queue even if they require | |
1775 | a stall. The length of the stall is instead computed as: | |
1776 | ||
1777 | MAX (INSN_TICK (INSN) - clock_var, 0) | |
1778 | ||
1779 | (= insn_delay). This allows rank_for_schedule to choose between | |
1780 | introducing a deliberate stall or increasing pressure. | |
1781 | ||
1782 | - Before sorting the ready queue, model_set_excess_costs assigns | |
1783 | a pressure-based cost to each ready instruction in the queue. | |
1784 | This is the instruction's INSN_REG_PRESSURE_EXCESS_COST_CHANGE | |
1785 | (ECC for short) and is effectively measured in cycles. | |
1786 | ||
1787 | - rank_for_schedule ranks instructions based on: | |
1788 | ||
1789 | ECC (insn) + insn_delay (insn) | |
1790 | ||
1791 | then as: | |
1792 | ||
1793 | insn_delay (insn) | |
1794 | ||
1795 | So, for example, an instruction X1 with an ECC of 1 that can issue | |
1796 | now will win over an instruction X0 with an ECC of zero that would | |
1797 | introduce a stall of one cycle. However, an instruction X2 with an | |
1798 | ECC of 2 that can issue now will lose to both X0 and X1. | |
1799 | ||
1800 | - When an instruction is scheduled, model_recompute updates the model | |
1801 | schedule with the new pressures (some of which might now exceed the | |
1802 | original maximum pressure MP). model_update_limit_points then searches | |
1803 | for the new point of maximum pressure, if not already known. */ | |
1804 | ||
1805 | /* Used to separate high-verbosity debug information for SCHED_PRESSURE_MODEL | |
1806 | from surrounding debug information. */ | |
1807 | #define MODEL_BAR \ | |
1808 | ";;\t\t+------------------------------------------------------\n" | |
1809 | ||
1810 | /* Information about the pressure on a particular register class at a | |
1811 | particular point of the model schedule. */ | |
1812 | struct model_pressure_data { | |
1813 | /* The pressure at this point of the model schedule, or -1 if the | |
1814 | point is associated with an instruction that has already been | |
1815 | scheduled. */ | |
1816 | int ref_pressure; | |
1817 | ||
1818 | /* The maximum pressure during or after this point of the model schedule. */ | |
1819 | int max_pressure; | |
1820 | }; | |
1821 | ||
1822 | /* Per-instruction information that is used while building the model | |
1823 | schedule. Here, "schedule" refers to the model schedule rather | |
1824 | than the main schedule. */ | |
1825 | struct model_insn_info { | |
1826 | /* The instruction itself. */ | |
1827 | rtx insn; | |
1828 | ||
1829 | /* If this instruction is in model_worklist, these fields link to the | |
1830 | previous (higher-priority) and next (lower-priority) instructions | |
1831 | in the list. */ | |
1832 | struct model_insn_info *prev; | |
1833 | struct model_insn_info *next; | |
1834 | ||
1835 | /* While constructing the schedule, QUEUE_INDEX describes whether an | |
1836 | instruction has already been added to the schedule (QUEUE_SCHEDULED), | |
1837 | is in model_worklist (QUEUE_READY), or neither (QUEUE_NOWHERE). | |
1838 | old_queue records the value that QUEUE_INDEX had before scheduling | |
1839 | started, so that we can restore it once the schedule is complete. */ | |
1840 | int old_queue; | |
1841 | ||
1842 | /* The relative importance of an unscheduled instruction. Higher | |
1843 | values indicate greater importance. */ | |
1844 | unsigned int model_priority; | |
1845 | ||
1846 | /* The length of the longest path of satisfied true dependencies | |
1847 | that leads to this instruction. */ | |
1848 | unsigned int depth; | |
1849 | ||
1850 | /* The length of the longest path of dependencies of any kind | |
1851 | that leads from this instruction. */ | |
1852 | unsigned int alap; | |
1853 | ||
1854 | /* The number of predecessor nodes that must still be scheduled. */ | |
1855 | int unscheduled_preds; | |
1856 | }; | |
1857 | ||
1858 | /* Information about the pressure limit for a particular register class. | |
1859 | This structure is used when applying a model schedule to the main | |
1860 | schedule. */ | |
1861 | struct model_pressure_limit { | |
1862 | /* The maximum register pressure seen in the original model schedule. */ | |
1863 | int orig_pressure; | |
1864 | ||
1865 | /* The maximum register pressure seen in the current model schedule | |
1866 | (which excludes instructions that have already been scheduled). */ | |
1867 | int pressure; | |
1868 | ||
1869 | /* The point of the current model schedule at which PRESSURE is first | |
1870 | reached. It is set to -1 if the value needs to be recomputed. */ | |
1871 | int point; | |
1872 | }; | |
1873 | ||
1874 | /* Describes a particular way of measuring register pressure. */ | |
1875 | struct model_pressure_group { | |
1876 | /* Index PCI describes the maximum pressure on ira_pressure_classes[PCI]. */ | |
1877 | struct model_pressure_limit limits[N_REG_CLASSES]; | |
1878 | ||
1879 | /* Index (POINT * ira_num_pressure_classes + PCI) describes the pressure | |
1880 | on register class ira_pressure_classes[PCI] at point POINT of the | |
1881 | current model schedule. A POINT of model_num_insns describes the | |
1882 | pressure at the end of the schedule. */ | |
1883 | struct model_pressure_data *model; | |
1884 | }; | |
1885 | ||
1886 | /* Index POINT gives the instruction at point POINT of the model schedule. | |
1887 | This array doesn't change during main scheduling. */ | |
f1f41a6c | 1888 | static vec<rtx> model_schedule; |
b30b031c | 1889 | |
1890 | /* The list of instructions in the model worklist, sorted in order of | |
1891 | decreasing priority. */ | |
1892 | static struct model_insn_info *model_worklist; | |
1893 | ||
1894 | /* Index I describes the instruction with INSN_LUID I. */ | |
1895 | static struct model_insn_info *model_insns; | |
1896 | ||
1897 | /* The number of instructions in the model schedule. */ | |
1898 | static int model_num_insns; | |
1899 | ||
1900 | /* The index of the first instruction in model_schedule that hasn't yet been | |
1901 | added to the main schedule, or model_num_insns if all of them have. */ | |
1902 | static int model_curr_point; | |
1903 | ||
1904 | /* Describes the pressure before each instruction in the model schedule. */ | |
1905 | static struct model_pressure_group model_before_pressure; | |
1906 | ||
1907 | /* The first unused model_priority value (as used in model_insn_info). */ | |
1908 | static unsigned int model_next_priority; | |
1909 | ||
1910 | ||
1911 | /* The model_pressure_data for ira_pressure_classes[PCI] in GROUP | |
1912 | at point POINT of the model schedule. */ | |
1913 | #define MODEL_PRESSURE_DATA(GROUP, POINT, PCI) \ | |
1914 | (&(GROUP)->model[(POINT) * ira_pressure_classes_num + (PCI)]) | |
1915 | ||
1916 | /* The maximum pressure on ira_pressure_classes[PCI] in GROUP at or | |
1917 | after point POINT of the model schedule. */ | |
1918 | #define MODEL_MAX_PRESSURE(GROUP, POINT, PCI) \ | |
1919 | (MODEL_PRESSURE_DATA (GROUP, POINT, PCI)->max_pressure) | |
1920 | ||
1921 | /* The pressure on ira_pressure_classes[PCI] in GROUP at point POINT | |
1922 | of the model schedule. */ | |
1923 | #define MODEL_REF_PRESSURE(GROUP, POINT, PCI) \ | |
1924 | (MODEL_PRESSURE_DATA (GROUP, POINT, PCI)->ref_pressure) | |
1925 | ||
1926 | /* Information about INSN that is used when creating the model schedule. */ | |
1927 | #define MODEL_INSN_INFO(INSN) \ | |
1928 | (&model_insns[INSN_LUID (INSN)]) | |
1929 | ||
1930 | /* The instruction at point POINT of the model schedule. */ | |
1931 | #define MODEL_INSN(POINT) \ | |
f1f41a6c | 1932 | (model_schedule[POINT]) |
b30b031c | 1933 | |
1934 | ||
1935 | /* Return INSN's index in the model schedule, or model_num_insns if it | |
1936 | doesn't belong to that schedule. */ | |
1937 | ||
1938 | static int | |
1939 | model_index (rtx insn) | |
1940 | { | |
1941 | if (INSN_MODEL_INDEX (insn) == 0) | |
1942 | return model_num_insns; | |
1943 | return INSN_MODEL_INDEX (insn) - 1; | |
1944 | } | |
1945 | ||
1946 | /* Make sure that GROUP->limits is up-to-date for the current point | |
1947 | of the model schedule. */ | |
1948 | ||
1949 | static void | |
1950 | model_update_limit_points_in_group (struct model_pressure_group *group) | |
1951 | { | |
1952 | int pci, max_pressure, point; | |
1953 | ||
1954 | for (pci = 0; pci < ira_pressure_classes_num; pci++) | |
1955 | { | |
1956 | /* We may have passed the final point at which the pressure in | |
1957 | group->limits[pci].pressure was reached. Update the limit if so. */ | |
1958 | max_pressure = MODEL_MAX_PRESSURE (group, model_curr_point, pci); | |
1959 | group->limits[pci].pressure = max_pressure; | |
1960 | ||
1961 | /* Find the point at which MAX_PRESSURE is first reached. We need | |
1962 | to search in three cases: | |
1963 | ||
1964 | - We've already moved past the previous pressure point. | |
1965 | In this case we search forward from model_curr_point. | |
1966 | ||
1967 | - We scheduled the previous point of maximum pressure ahead of | |
1968 | its position in the model schedule, but doing so didn't bring | |
1969 | the pressure point earlier. In this case we search forward | |
1970 | from that previous pressure point. | |
1971 | ||
1972 | - Scheduling an instruction early caused the maximum pressure | |
1973 | to decrease. In this case we will have set the pressure | |
1974 | point to -1, and we search forward from model_curr_point. */ | |
1975 | point = MAX (group->limits[pci].point, model_curr_point); | |
1976 | while (point < model_num_insns | |
1977 | && MODEL_REF_PRESSURE (group, point, pci) < max_pressure) | |
1978 | point++; | |
1979 | group->limits[pci].point = point; | |
1980 | ||
1981 | gcc_assert (MODEL_REF_PRESSURE (group, point, pci) == max_pressure); | |
1982 | gcc_assert (MODEL_MAX_PRESSURE (group, point, pci) == max_pressure); | |
1983 | } | |
1984 | } | |
1985 | ||
1986 | /* Make sure that all register-pressure limits are up-to-date for the | |
1987 | current position in the model schedule. */ | |
1988 | ||
1989 | static void | |
1990 | model_update_limit_points (void) | |
1991 | { | |
1992 | model_update_limit_points_in_group (&model_before_pressure); | |
1993 | } | |
1994 | ||
1995 | /* Return the model_index of the last unscheduled use in chain USE | |
1996 | outside of USE's instruction. Return -1 if there are no other uses, | |
1997 | or model_num_insns if the register is live at the end of the block. */ | |
1998 | ||
1999 | static int | |
2000 | model_last_use_except (struct reg_use_data *use) | |
2001 | { | |
2002 | struct reg_use_data *next; | |
2003 | int last, index; | |
2004 | ||
2005 | last = -1; | |
2006 | for (next = use->next_regno_use; next != use; next = next->next_regno_use) | |
2007 | if (NONDEBUG_INSN_P (next->insn) | |
2008 | && QUEUE_INDEX (next->insn) != QUEUE_SCHEDULED) | |
2009 | { | |
2010 | index = model_index (next->insn); | |
2011 | if (index == model_num_insns) | |
2012 | return model_num_insns; | |
2013 | if (last < index) | |
2014 | last = index; | |
2015 | } | |
2016 | return last; | |
2017 | } | |
2018 | ||
2019 | /* An instruction with model_index POINT has just been scheduled, and it | |
2020 | adds DELTA to the pressure on ira_pressure_classes[PCI] after POINT - 1. | |
2021 | Update MODEL_REF_PRESSURE (GROUP, POINT, PCI) and | |
2022 | MODEL_MAX_PRESSURE (GROUP, POINT, PCI) accordingly. */ | |
2023 | ||
2024 | static void | |
2025 | model_start_update_pressure (struct model_pressure_group *group, | |
2026 | int point, int pci, int delta) | |
2027 | { | |
2028 | int next_max_pressure; | |
2029 | ||
2030 | if (point == model_num_insns) | |
2031 | { | |
2032 | /* The instruction wasn't part of the model schedule; it was moved | |
2033 | from a different block. Update the pressure for the end of | |
2034 | the model schedule. */ | |
2035 | MODEL_REF_PRESSURE (group, point, pci) += delta; | |
2036 | MODEL_MAX_PRESSURE (group, point, pci) += delta; | |
2037 | } | |
2038 | else | |
2039 | { | |
2040 | /* Record that this instruction has been scheduled. Nothing now | |
2041 | changes between POINT and POINT + 1, so get the maximum pressure | |
2042 | from the latter. If the maximum pressure decreases, the new | |
2043 | pressure point may be before POINT. */ | |
2044 | MODEL_REF_PRESSURE (group, point, pci) = -1; | |
2045 | next_max_pressure = MODEL_MAX_PRESSURE (group, point + 1, pci); | |
2046 | if (MODEL_MAX_PRESSURE (group, point, pci) > next_max_pressure) | |
2047 | { | |
2048 | MODEL_MAX_PRESSURE (group, point, pci) = next_max_pressure; | |
2049 | if (group->limits[pci].point == point) | |
2050 | group->limits[pci].point = -1; | |
2051 | } | |
2052 | } | |
2053 | } | |
2054 | ||
2055 | /* Record that scheduling a later instruction has changed the pressure | |
2056 | at point POINT of the model schedule by DELTA (which might be 0). | |
2057 | Update GROUP accordingly. Return nonzero if these changes might | |
2058 | trigger changes to previous points as well. */ | |
2059 | ||
2060 | static int | |
2061 | model_update_pressure (struct model_pressure_group *group, | |
2062 | int point, int pci, int delta) | |
2063 | { | |
2064 | int ref_pressure, max_pressure, next_max_pressure; | |
2065 | ||
2066 | /* If POINT hasn't yet been scheduled, update its pressure. */ | |
2067 | ref_pressure = MODEL_REF_PRESSURE (group, point, pci); | |
2068 | if (ref_pressure >= 0 && delta != 0) | |
2069 | { | |
2070 | ref_pressure += delta; | |
2071 | MODEL_REF_PRESSURE (group, point, pci) = ref_pressure; | |
2072 | ||
2073 | /* Check whether the maximum pressure in the overall schedule | |
2074 | has increased. (This means that the MODEL_MAX_PRESSURE of | |
2075 | every point <= POINT will need to increae too; see below.) */ | |
2076 | if (group->limits[pci].pressure < ref_pressure) | |
2077 | group->limits[pci].pressure = ref_pressure; | |
2078 | ||
2079 | /* If we are at maximum pressure, and the maximum pressure | |
2080 | point was previously unknown or later than POINT, | |
2081 | bring it forward. */ | |
2082 | if (group->limits[pci].pressure == ref_pressure | |
2083 | && !IN_RANGE (group->limits[pci].point, 0, point)) | |
2084 | group->limits[pci].point = point; | |
2085 | ||
2086 | /* If POINT used to be the point of maximum pressure, but isn't | |
2087 | any longer, we need to recalculate it using a forward walk. */ | |
2088 | if (group->limits[pci].pressure > ref_pressure | |
2089 | && group->limits[pci].point == point) | |
2090 | group->limits[pci].point = -1; | |
2091 | } | |
2092 | ||
2093 | /* Update the maximum pressure at POINT. Changes here might also | |
2094 | affect the maximum pressure at POINT - 1. */ | |
2095 | next_max_pressure = MODEL_MAX_PRESSURE (group, point + 1, pci); | |
2096 | max_pressure = MAX (ref_pressure, next_max_pressure); | |
2097 | if (MODEL_MAX_PRESSURE (group, point, pci) != max_pressure) | |
2098 | { | |
2099 | MODEL_MAX_PRESSURE (group, point, pci) = max_pressure; | |
2100 | return 1; | |
2101 | } | |
2102 | return 0; | |
2103 | } | |
2104 | ||
2105 | /* INSN has just been scheduled. Update the model schedule accordingly. */ | |
2106 | ||
2107 | static void | |
2108 | model_recompute (rtx insn) | |
2109 | { | |
2110 | struct { | |
2111 | int last_use; | |
2112 | int regno; | |
2113 | } uses[FIRST_PSEUDO_REGISTER + MAX_RECOG_OPERANDS]; | |
2114 | struct reg_use_data *use; | |
2115 | struct reg_pressure_data *reg_pressure; | |
2116 | int delta[N_REG_CLASSES]; | |
2117 | int pci, point, mix, new_last, cl, ref_pressure, queue; | |
2118 | unsigned int i, num_uses, num_pending_births; | |
2119 | bool print_p; | |
2120 | ||
2121 | /* The destinations of INSN were previously live from POINT onwards, but are | |
2122 | now live from model_curr_point onwards. Set up DELTA accordingly. */ | |
2123 | point = model_index (insn); | |
2124 | reg_pressure = INSN_REG_PRESSURE (insn); | |
2125 | for (pci = 0; pci < ira_pressure_classes_num; pci++) | |
2126 | { | |
2127 | cl = ira_pressure_classes[pci]; | |
2128 | delta[cl] = reg_pressure[pci].set_increase; | |
2129 | } | |
2130 | ||
2131 | /* Record which registers previously died at POINT, but which now die | |
2132 | before POINT. Adjust DELTA so that it represents the effect of | |
2133 | this change after POINT - 1. Set NUM_PENDING_BIRTHS to the number of | |
2134 | registers that will be born in the range [model_curr_point, POINT). */ | |
2135 | num_uses = 0; | |
2136 | num_pending_births = 0; | |
2137 | for (use = INSN_REG_USE_LIST (insn); use != NULL; use = use->next_insn_use) | |
2138 | { | |
2139 | new_last = model_last_use_except (use); | |
2140 | if (new_last < point) | |
2141 | { | |
2142 | gcc_assert (num_uses < ARRAY_SIZE (uses)); | |
2143 | uses[num_uses].last_use = new_last; | |
2144 | uses[num_uses].regno = use->regno; | |
2145 | /* This register is no longer live after POINT - 1. */ | |
2146 | mark_regno_birth_or_death (NULL, delta, use->regno, false); | |
2147 | num_uses++; | |
2148 | if (new_last >= 0) | |
2149 | num_pending_births++; | |
2150 | } | |
2151 | } | |
2152 | ||
2153 | /* Update the MODEL_REF_PRESSURE and MODEL_MAX_PRESSURE for POINT. | |
2154 | Also set each group pressure limit for POINT. */ | |
2155 | for (pci = 0; pci < ira_pressure_classes_num; pci++) | |
2156 | { | |
2157 | cl = ira_pressure_classes[pci]; | |
2158 | model_start_update_pressure (&model_before_pressure, | |
2159 | point, pci, delta[cl]); | |
2160 | } | |
2161 | ||
2162 | /* Walk the model schedule backwards, starting immediately before POINT. */ | |
2163 | print_p = false; | |
2164 | if (point != model_curr_point) | |
2165 | do | |
2166 | { | |
2167 | point--; | |
2168 | insn = MODEL_INSN (point); | |
2169 | queue = QUEUE_INDEX (insn); | |
2170 | ||
2171 | if (queue != QUEUE_SCHEDULED) | |
2172 | { | |
2173 | /* DELTA describes the effect of the move on the register pressure | |
2174 | after POINT. Make it describe the effect on the pressure | |
2175 | before POINT. */ | |
2176 | i = 0; | |
2177 | while (i < num_uses) | |
2178 | { | |
2179 | if (uses[i].last_use == point) | |
2180 | { | |
2181 | /* This register is now live again. */ | |
2182 | mark_regno_birth_or_death (NULL, delta, | |
2183 | uses[i].regno, true); | |
2184 | ||
2185 | /* Remove this use from the array. */ | |
2186 | uses[i] = uses[num_uses - 1]; | |
2187 | num_uses--; | |
2188 | num_pending_births--; | |
2189 | } | |
2190 | else | |
2191 | i++; | |
2192 | } | |
2193 | ||
2194 | if (sched_verbose >= 5) | |
2195 | { | |
b30b031c | 2196 | if (!print_p) |
2197 | { | |
2198 | fprintf (sched_dump, MODEL_BAR); | |
2199 | fprintf (sched_dump, ";;\t\t| New pressure for model" | |
2200 | " schedule\n"); | |
2201 | fprintf (sched_dump, MODEL_BAR); | |
2202 | print_p = true; | |
2203 | } | |
2204 | ||
b30b031c | 2205 | fprintf (sched_dump, ";;\t\t| %3d %4d %-30s ", |
3d9a4504 | 2206 | point, INSN_UID (insn), |
2207 | str_pattern_slim (PATTERN (insn))); | |
b30b031c | 2208 | for (pci = 0; pci < ira_pressure_classes_num; pci++) |
2209 | { | |
2210 | cl = ira_pressure_classes[pci]; | |
2211 | ref_pressure = MODEL_REF_PRESSURE (&model_before_pressure, | |
2212 | point, pci); | |
2213 | fprintf (sched_dump, " %s:[%d->%d]", | |
2214 | reg_class_names[ira_pressure_classes[pci]], | |
2215 | ref_pressure, ref_pressure + delta[cl]); | |
2216 | } | |
2217 | fprintf (sched_dump, "\n"); | |
2218 | } | |
2219 | } | |
2220 | ||
2221 | /* Adjust the pressure at POINT. Set MIX to nonzero if POINT - 1 | |
2222 | might have changed as well. */ | |
2223 | mix = num_pending_births; | |
2224 | for (pci = 0; pci < ira_pressure_classes_num; pci++) | |
2225 | { | |
2226 | cl = ira_pressure_classes[pci]; | |
2227 | mix |= delta[cl]; | |
2228 | mix |= model_update_pressure (&model_before_pressure, | |
2229 | point, pci, delta[cl]); | |
2230 | } | |
2231 | } | |
2232 | while (mix && point > model_curr_point); | |
2233 | ||
2234 | if (print_p) | |
2235 | fprintf (sched_dump, MODEL_BAR); | |
2236 | } | |
d452a169 | 2237 | |
2238 | /* After DEP, which was cancelled, has been resolved for insn NEXT, | |
2239 | check whether the insn's pattern needs restoring. */ | |
2240 | static bool | |
2241 | must_restore_pattern_p (rtx next, dep_t dep) | |
2242 | { | |
2243 | if (QUEUE_INDEX (next) == QUEUE_SCHEDULED) | |
2244 | return false; | |
2245 | ||
2246 | if (DEP_TYPE (dep) == REG_DEP_CONTROL) | |
2247 | { | |
2248 | gcc_assert (ORIG_PAT (next) != NULL_RTX); | |
2249 | gcc_assert (next == DEP_CON (dep)); | |
2250 | } | |
2251 | else | |
2252 | { | |
2253 | struct dep_replacement *desc = DEP_REPLACE (dep); | |
2254 | if (desc->insn != next) | |
2255 | { | |
2256 | gcc_assert (*desc->loc == desc->orig); | |
2257 | return false; | |
2258 | } | |
2259 | } | |
2260 | return true; | |
2261 | } | |
b30b031c | 2262 | \f |
2263 | /* model_spill_cost (CL, P, P') returns the cost of increasing the | |
2264 | pressure on CL from P to P'. We use this to calculate a "base ECC", | |
2265 | baseECC (CL, X), for each pressure class CL and each instruction X. | |
2266 | Supposing X changes the pressure on CL from P to P', and that the | |
2267 | maximum pressure on CL in the current model schedule is MP', then: | |
2268 | ||
2269 | * if X occurs before or at the next point of maximum pressure in | |
2270 | the model schedule and P' > MP', then: | |
2271 | ||
2272 | baseECC (CL, X) = model_spill_cost (CL, MP, P') | |
2273 | ||
2274 | The idea is that the pressure after scheduling a fixed set of | |
2275 | instructions -- in this case, the set up to and including the | |
2276 | next maximum pressure point -- is going to be the same regardless | |
2277 | of the order; we simply want to keep the intermediate pressure | |
2278 | under control. Thus X has a cost of zero unless scheduling it | |
2279 | now would exceed MP'. | |
2280 | ||
2281 | If all increases in the set are by the same amount, no zero-cost | |
2282 | instruction will ever cause the pressure to exceed MP'. However, | |
2283 | if X is instead moved past an instruction X' with pressure in the | |
2284 | range (MP' - (P' - P), MP'), the pressure at X' will increase | |
2285 | beyond MP'. Since baseECC is very much a heuristic anyway, | |
2286 | it doesn't seem worth the overhead of tracking cases like these. | |
2287 | ||
2288 | The cost of exceeding MP' is always based on the original maximum | |
2289 | pressure MP. This is so that going 2 registers over the original | |
2290 | limit has the same cost regardless of whether it comes from two | |
2291 | separate +1 deltas or from a single +2 delta. | |
2292 | ||
2293 | * if X occurs after the next point of maximum pressure in the model | |
2294 | schedule and P' > P, then: | |
2295 | ||
2296 | baseECC (CL, X) = model_spill_cost (CL, MP, MP' + (P' - P)) | |
2297 | ||
2298 | That is, if we move X forward across a point of maximum pressure, | |
2299 | and if X increases the pressure by P' - P, then we conservatively | |
2300 | assume that scheduling X next would increase the maximum pressure | |
2301 | by P' - P. Again, the cost of doing this is based on the original | |
2302 | maximum pressure MP, for the same reason as above. | |
2303 | ||
2304 | * if P' < P, P > MP, and X occurs at or after the next point of | |
2305 | maximum pressure, then: | |
2306 | ||
2307 | baseECC (CL, X) = -model_spill_cost (CL, MAX (MP, P'), P) | |
a7dcf969 | 2308 | |
b30b031c | 2309 | That is, if we have already exceeded the original maximum pressure MP, |
2310 | and if X might reduce the maximum pressure again -- or at least push | |
2311 | it further back, and thus allow more scheduling freedom -- it is given | |
2312 | a negative cost to reflect the improvement. | |
2313 | ||
2314 | * otherwise, | |
2315 | ||
2316 | baseECC (CL, X) = 0 | |
2317 | ||
2318 | In this case, X is not expected to affect the maximum pressure MP', | |
2319 | so it has zero cost. | |
2320 | ||
2321 | We then create a combined value baseECC (X) that is the sum of | |
2322 | baseECC (CL, X) for each pressure class CL. | |
2323 | ||
2324 | baseECC (X) could itself be used as the ECC value described above. | |
2325 | However, this is often too conservative, in the sense that it | |
2326 | tends to make high-priority instructions that increase pressure | |
2327 | wait too long in cases where introducing a spill would be better. | |
2328 | For this reason the final ECC is a priority-adjusted form of | |
2329 | baseECC (X). Specifically, we calculate: | |
2330 | ||
2331 | P (X) = INSN_PRIORITY (X) - insn_delay (X) - baseECC (X) | |
2332 | baseP = MAX { P (X) | baseECC (X) <= 0 } | |
2333 | ||
2334 | Then: | |
2335 | ||
2336 | ECC (X) = MAX (MIN (baseP - P (X), baseECC (X)), 0) | |
2337 | ||
2338 | Thus an instruction's effect on pressure is ignored if it has a high | |
2339 | enough priority relative to the ones that don't increase pressure. | |
2340 | Negative values of baseECC (X) do not increase the priority of X | |
2341 | itself, but they do make it harder for other instructions to | |
2342 | increase the pressure further. | |
2343 | ||
2344 | This pressure cost is deliberately timid. The intention has been | |
2345 | to choose a heuristic that rarely interferes with the normal list | |
2346 | scheduler in cases where that scheduler would produce good code. | |
2347 | We simply want to curb some of its worst excesses. */ | |
2348 | ||
2349 | /* Return the cost of increasing the pressure in class CL from FROM to TO. | |
2350 | ||
2351 | Here we use the very simplistic cost model that every register above | |
1072fecf | 2352 | ira_class_hard_regs_num[CL] has a spill cost of 1. We could use other |
b30b031c | 2353 | measures instead, such as one based on MEMORY_MOVE_COST. However: |
2354 | ||
2355 | (1) In order for an instruction to be scheduled, the higher cost | |
2356 | would need to be justified in a single saving of that many stalls. | |
2357 | This is overly pessimistic, because the benefit of spilling is | |
2358 | often to avoid a sequence of several short stalls rather than | |
2359 | a single long one. | |
2360 | ||
2361 | (2) The cost is still arbitrary. Because we are not allocating | |
2362 | registers during scheduling, we have no way of knowing for | |
2363 | sure how many memory accesses will be required by each spill, | |
2364 | where the spills will be placed within the block, or even | |
2365 | which block(s) will contain the spills. | |
2366 | ||
2367 | So a higher cost than 1 is often too conservative in practice, | |
2368 | forcing blocks to contain unnecessary stalls instead of spill code. | |
2369 | The simple cost below seems to be the best compromise. It reduces | |
2370 | the interference with the normal list scheduler, which helps make | |
2371 | it more suitable for a default-on option. */ | |
2372 | ||
2373 | static int | |
2374 | model_spill_cost (int cl, int from, int to) | |
2375 | { | |
1072fecf | 2376 | from = MAX (from, ira_class_hard_regs_num[cl]); |
b30b031c | 2377 | return MAX (to, from) - from; |
2378 | } | |
2379 | ||
2380 | /* Return baseECC (ira_pressure_classes[PCI], POINT), given that | |
2381 | P = curr_reg_pressure[ira_pressure_classes[PCI]] and that | |
2382 | P' = P + DELTA. */ | |
2383 | ||
2384 | static int | |
2385 | model_excess_group_cost (struct model_pressure_group *group, | |
2386 | int point, int pci, int delta) | |
2387 | { | |
2388 | int pressure, cl; | |
2389 | ||
2390 | cl = ira_pressure_classes[pci]; | |
2391 | if (delta < 0 && point >= group->limits[pci].point) | |
2392 | { | |
2393 | pressure = MAX (group->limits[pci].orig_pressure, | |
2394 | curr_reg_pressure[cl] + delta); | |
2395 | return -model_spill_cost (cl, pressure, curr_reg_pressure[cl]); | |
2396 | } | |
2397 | ||
2398 | if (delta > 0) | |
2399 | { | |
2400 | if (point > group->limits[pci].point) | |
2401 | pressure = group->limits[pci].pressure + delta; | |
2402 | else | |
2403 | pressure = curr_reg_pressure[cl] + delta; | |
2404 | ||
2405 | if (pressure > group->limits[pci].pressure) | |
2406 | return model_spill_cost (cl, group->limits[pci].orig_pressure, | |
2407 | pressure); | |
2408 | } | |
2409 | ||
2410 | return 0; | |
2411 | } | |
2412 | ||
2413 | /* Return baseECC (MODEL_INSN (INSN)). Dump the costs to sched_dump | |
2414 | if PRINT_P. */ | |
2415 | ||
2416 | static int | |
2417 | model_excess_cost (rtx insn, bool print_p) | |
2418 | { | |
2419 | int point, pci, cl, cost, this_cost, delta; | |
2420 | struct reg_pressure_data *insn_reg_pressure; | |
2421 | int insn_death[N_REG_CLASSES]; | |
2422 | ||
2423 | calculate_reg_deaths (insn, insn_death); | |
2424 | point = model_index (insn); | |
2425 | insn_reg_pressure = INSN_REG_PRESSURE (insn); | |
2426 | cost = 0; | |
2427 | ||
2428 | if (print_p) | |
2429 | fprintf (sched_dump, ";;\t\t| %3d %4d | %4d %+3d |", point, | |
2430 | INSN_UID (insn), INSN_PRIORITY (insn), insn_delay (insn)); | |
2431 | ||
2432 | /* Sum up the individual costs for each register class. */ | |
2433 | for (pci = 0; pci < ira_pressure_classes_num; pci++) | |
2434 | { | |
2435 | cl = ira_pressure_classes[pci]; | |
2436 | delta = insn_reg_pressure[pci].set_increase - insn_death[cl]; | |
2437 | this_cost = model_excess_group_cost (&model_before_pressure, | |
2438 | point, pci, delta); | |
2439 | cost += this_cost; | |
2440 | if (print_p) | |
2441 | fprintf (sched_dump, " %s:[%d base cost %d]", | |
2442 | reg_class_names[cl], delta, this_cost); | |
2443 | } | |
2444 | ||
2445 | if (print_p) | |
2446 | fprintf (sched_dump, "\n"); | |
2447 | ||
2448 | return cost; | |
2449 | } | |
2450 | ||
2451 | /* Dump the next points of maximum pressure for GROUP. */ | |
2452 | ||
2453 | static void | |
2454 | model_dump_pressure_points (struct model_pressure_group *group) | |
2455 | { | |
2456 | int pci, cl; | |
2457 | ||
2458 | fprintf (sched_dump, ";;\t\t| pressure points"); | |
2459 | for (pci = 0; pci < ira_pressure_classes_num; pci++) | |
2460 | { | |
2461 | cl = ira_pressure_classes[pci]; | |
2462 | fprintf (sched_dump, " %s:[%d->%d at ", reg_class_names[cl], | |
2463 | curr_reg_pressure[cl], group->limits[pci].pressure); | |
2464 | if (group->limits[pci].point < model_num_insns) | |
2465 | fprintf (sched_dump, "%d:%d]", group->limits[pci].point, | |
2466 | INSN_UID (MODEL_INSN (group->limits[pci].point))); | |
2467 | else | |
2468 | fprintf (sched_dump, "end]"); | |
2469 | } | |
2470 | fprintf (sched_dump, "\n"); | |
2471 | } | |
2472 | ||
2473 | /* Set INSN_REG_PRESSURE_EXCESS_COST_CHANGE for INSNS[0...COUNT-1]. */ | |
2474 | ||
2475 | static void | |
2476 | model_set_excess_costs (rtx *insns, int count) | |
2477 | { | |
2478 | int i, cost, priority_base, priority; | |
2479 | bool print_p; | |
2480 | ||
2481 | /* Record the baseECC value for each instruction in the model schedule, | |
2482 | except that negative costs are converted to zero ones now rather thatn | |
2483 | later. Do not assign a cost to debug instructions, since they must | |
2484 | not change code-generation decisions. Experiments suggest we also | |
2485 | get better results by not assigning a cost to instructions from | |
2486 | a different block. | |
2487 | ||
2488 | Set PRIORITY_BASE to baseP in the block comment above. This is the | |
2489 | maximum priority of the "cheap" instructions, which should always | |
2490 | include the next model instruction. */ | |
2491 | priority_base = 0; | |
2492 | print_p = false; | |
2493 | for (i = 0; i < count; i++) | |
2494 | if (INSN_MODEL_INDEX (insns[i])) | |
2495 | { | |
2496 | if (sched_verbose >= 6 && !print_p) | |
2497 | { | |
2498 | fprintf (sched_dump, MODEL_BAR); | |
2499 | fprintf (sched_dump, ";;\t\t| Pressure costs for ready queue\n"); | |
2500 | model_dump_pressure_points (&model_before_pressure); | |
2501 | fprintf (sched_dump, MODEL_BAR); | |
2502 | print_p = true; | |
2503 | } | |
2504 | cost = model_excess_cost (insns[i], print_p); | |
2505 | if (cost <= 0) | |
2506 | { | |
2507 | priority = INSN_PRIORITY (insns[i]) - insn_delay (insns[i]) - cost; | |
2508 | priority_base = MAX (priority_base, priority); | |
2509 | cost = 0; | |
2510 | } | |
2511 | INSN_REG_PRESSURE_EXCESS_COST_CHANGE (insns[i]) = cost; | |
2512 | } | |
2513 | if (print_p) | |
2514 | fprintf (sched_dump, MODEL_BAR); | |
2515 | ||
2516 | /* Use MAX (baseECC, 0) and baseP to calculcate ECC for each | |
2517 | instruction. */ | |
2518 | for (i = 0; i < count; i++) | |
2519 | { | |
2520 | cost = INSN_REG_PRESSURE_EXCESS_COST_CHANGE (insns[i]); | |
2521 | priority = INSN_PRIORITY (insns[i]) - insn_delay (insns[i]); | |
2522 | if (cost > 0 && priority > priority_base) | |
2523 | { | |
2524 | cost += priority_base - priority; | |
2525 | INSN_REG_PRESSURE_EXCESS_COST_CHANGE (insns[i]) = MAX (cost, 0); | |
2526 | } | |
2527 | } | |
2528 | } | |
2529 | \f | |
3eb9a99d | 2530 | /* Returns a positive value if x is preferred; returns a negative value if |
2531 | y is preferred. Should never return 0, since that will make the sort | |
2532 | unstable. */ | |
2533 | ||
2534 | static int | |
952f0048 | 2535 | rank_for_schedule (const void *x, const void *y) |
3eb9a99d | 2536 | { |
896c2bfe | 2537 | rtx tmp = *(const rtx *) y; |
2538 | rtx tmp2 = *(const rtx *) x; | |
9997bd27 | 2539 | int tmp_class, tmp2_class; |
57a8bf1b | 2540 | int val, priority_val, info_val, diff; |
3eb9a99d | 2541 | |
9845d120 | 2542 | if (MAY_HAVE_DEBUG_INSNS) |
2543 | { | |
2544 | /* Schedule debug insns as early as possible. */ | |
2545 | if (DEBUG_INSN_P (tmp) && !DEBUG_INSN_P (tmp2)) | |
2546 | return -1; | |
148868c2 | 2547 | else if (!DEBUG_INSN_P (tmp) && DEBUG_INSN_P (tmp2)) |
9845d120 | 2548 | return 1; |
148868c2 | 2549 | else if (DEBUG_INSN_P (tmp) && DEBUG_INSN_P (tmp2)) |
2550 | return INSN_LUID (tmp) - INSN_LUID (tmp2); | |
9845d120 | 2551 | } |
2552 | ||
57a8bf1b | 2553 | if (live_range_shrinkage_p) |
2554 | { | |
2555 | /* Don't use SCHED_PRESSURE_MODEL -- it results in much worse | |
2556 | code. */ | |
2557 | gcc_assert (sched_pressure == SCHED_PRESSURE_WEIGHTED); | |
2558 | if ((INSN_REG_PRESSURE_EXCESS_COST_CHANGE (tmp) < 0 | |
2559 | || INSN_REG_PRESSURE_EXCESS_COST_CHANGE (tmp2) < 0) | |
2560 | && (diff = (INSN_REG_PRESSURE_EXCESS_COST_CHANGE (tmp) | |
2561 | - INSN_REG_PRESSURE_EXCESS_COST_CHANGE (tmp2))) != 0) | |
2562 | return diff; | |
2563 | /* Sort by INSN_LUID (original insn order), so that we make the | |
2564 | sort stable. This minimizes instruction movement, thus | |
2565 | minimizing sched's effect on debugging and cross-jumping. */ | |
2566 | return INSN_LUID (tmp) - INSN_LUID (tmp2); | |
2567 | } | |
2568 | ||
e26579fc | 2569 | /* The insn in a schedule group should be issued the first. */ |
48e1416a | 2570 | if (flag_sched_group_heuristic && |
4d569386 | 2571 | SCHED_GROUP_P (tmp) != SCHED_GROUP_P (tmp2)) |
e26579fc | 2572 | return SCHED_GROUP_P (tmp2) ? 1 : -1; |
2573 | ||
3588ab9c | 2574 | /* Make sure that priority of TMP and TMP2 are initialized. */ |
2575 | gcc_assert (INSN_PRIORITY_KNOWN (tmp) && INSN_PRIORITY_KNOWN (tmp2)); | |
2576 | ||
b30b031c | 2577 | if (sched_pressure != SCHED_PRESSURE_NONE) |
a7dcf969 | 2578 | { |
a7dcf969 | 2579 | /* Prefer insn whose scheduling results in the smallest register |
2580 | pressure excess. */ | |
2581 | if ((diff = (INSN_REG_PRESSURE_EXCESS_COST_CHANGE (tmp) | |
b30b031c | 2582 | + insn_delay (tmp) |
a7dcf969 | 2583 | - INSN_REG_PRESSURE_EXCESS_COST_CHANGE (tmp2) |
b30b031c | 2584 | - insn_delay (tmp2)))) |
a7dcf969 | 2585 | return diff; |
2586 | } | |
2587 | ||
b30b031c | 2588 | if (sched_pressure != SCHED_PRESSURE_NONE |
a7dcf969 | 2589 | && (INSN_TICK (tmp2) > clock_var || INSN_TICK (tmp) > clock_var)) |
2590 | { | |
2591 | if (INSN_TICK (tmp) <= clock_var) | |
2592 | return -1; | |
2593 | else if (INSN_TICK (tmp2) <= clock_var) | |
2594 | return 1; | |
2595 | else | |
2596 | return INSN_TICK (tmp) - INSN_TICK (tmp2); | |
2597 | } | |
e2f4a6ff | 2598 | |
2599 | /* If we are doing backtracking in this schedule, prefer insns that | |
2600 | have forward dependencies with negative cost against an insn that | |
2601 | was already scheduled. */ | |
2602 | if (current_sched_info->flags & DO_BACKTRACKING) | |
2603 | { | |
2604 | priority_val = FEEDS_BACKTRACK_INSN (tmp2) - FEEDS_BACKTRACK_INSN (tmp); | |
2605 | if (priority_val) | |
2606 | return priority_val; | |
2607 | } | |
2608 | ||
c4cd519a | 2609 | /* Prefer insn with higher priority. */ |
3eb9a99d | 2610 | priority_val = INSN_PRIORITY (tmp2) - INSN_PRIORITY (tmp); |
58ada791 | 2611 | |
4d569386 | 2612 | if (flag_sched_critical_path_heuristic && priority_val) |
3eb9a99d | 2613 | return priority_val; |
48e1416a | 2614 | |
6a1cdb4d | 2615 | /* Prefer speculative insn with greater dependencies weakness. */ |
4d569386 | 2616 | if (flag_sched_spec_insn_heuristic && spec_info) |
6a1cdb4d | 2617 | { |
2618 | ds_t ds1, ds2; | |
2619 | dw_t dw1, dw2; | |
2620 | int dw; | |
2621 | ||
2622 | ds1 = TODO_SPEC (tmp) & SPECULATIVE; | |
2623 | if (ds1) | |
e1ab7874 | 2624 | dw1 = ds_weak (ds1); |
6a1cdb4d | 2625 | else |
2626 | dw1 = NO_DEP_WEAK; | |
48e1416a | 2627 | |
6a1cdb4d | 2628 | ds2 = TODO_SPEC (tmp2) & SPECULATIVE; |
2629 | if (ds2) | |
e1ab7874 | 2630 | dw2 = ds_weak (ds2); |
6a1cdb4d | 2631 | else |
2632 | dw2 = NO_DEP_WEAK; | |
2633 | ||
2634 | dw = dw2 - dw1; | |
2635 | if (dw > (NO_DEP_WEAK / 8) || dw < -(NO_DEP_WEAK / 8)) | |
2636 | return dw; | |
2637 | } | |
2638 | ||
c2069298 | 2639 | info_val = (*current_sched_info->rank) (tmp, tmp2); |
9af5ce0c | 2640 | if (flag_sched_rank_heuristic && info_val) |
c2069298 | 2641 | return info_val; |
3eb9a99d | 2642 | |
9845d120 | 2643 | /* Compare insns based on their relation to the last scheduled |
2644 | non-debug insn. */ | |
d0aa45d0 | 2645 | if (flag_sched_last_insn_heuristic && last_nondebug_scheduled_insn) |
3eb9a99d | 2646 | { |
93f6b030 | 2647 | dep_t dep1; |
2648 | dep_t dep2; | |
d0aa45d0 | 2649 | rtx last = last_nondebug_scheduled_insn; |
93f6b030 | 2650 | |
3eb9a99d | 2651 | /* Classify the instructions into three classes: |
2652 | 1) Data dependent on last schedule insn. | |
2653 | 2) Anti/Output dependent on last scheduled insn. | |
2654 | 3) Independent of last scheduled insn, or has latency of one. | |
2655 | Choose the insn from the highest numbered class if different. */ | |
9845d120 | 2656 | dep1 = sd_find_dep_between (last, tmp, true); |
9997bd27 | 2657 | |
93f6b030 | 2658 | if (dep1 == NULL || dep_cost (dep1) == 1) |
3eb9a99d | 2659 | tmp_class = 3; |
9997bd27 | 2660 | else if (/* Data dependence. */ |
93f6b030 | 2661 | DEP_TYPE (dep1) == REG_DEP_TRUE) |
3eb9a99d | 2662 | tmp_class = 1; |
2663 | else | |
2664 | tmp_class = 2; | |
2665 | ||
9845d120 | 2666 | dep2 = sd_find_dep_between (last, tmp2, true); |
9997bd27 | 2667 | |
93f6b030 | 2668 | if (dep2 == NULL || dep_cost (dep2) == 1) |
3eb9a99d | 2669 | tmp2_class = 3; |
9997bd27 | 2670 | else if (/* Data dependence. */ |
93f6b030 | 2671 | DEP_TYPE (dep2) == REG_DEP_TRUE) |
3eb9a99d | 2672 | tmp2_class = 1; |
2673 | else | |
2674 | tmp2_class = 2; | |
2675 | ||
2676 | if ((val = tmp2_class - tmp_class)) | |
2677 | return val; | |
2678 | } | |
2679 | ||
b30b031c | 2680 | /* Prefer instructions that occur earlier in the model schedule. */ |
2681 | if (sched_pressure == SCHED_PRESSURE_MODEL) | |
2682 | { | |
2683 | int diff; | |
2684 | ||
2685 | diff = model_index (tmp) - model_index (tmp2); | |
2686 | if (diff != 0) | |
2687 | return diff; | |
2688 | } | |
2689 | ||
896c2bfe | 2690 | /* Prefer the insn which has more later insns that depend on it. |
02aef853 | 2691 | This gives the scheduler more freedom when scheduling later |
2692 | instructions at the expense of added register pressure. */ | |
02aef853 | 2693 | |
b30b031c | 2694 | val = (dep_list_size (tmp2, SD_LIST_FORW) |
2695 | - dep_list_size (tmp, SD_LIST_FORW)); | |
02aef853 | 2696 | |
4d569386 | 2697 | if (flag_sched_dep_count_heuristic && val != 0) |
93f6b030 | 2698 | return val; |
896c2bfe | 2699 | |
3eb9a99d | 2700 | /* If insns are equally good, sort by INSN_LUID (original insn order), |
2701 | so that we make the sort stable. This minimizes instruction movement, | |
2702 | thus minimizing sched's effect on debugging and cross-jumping. */ | |
2703 | return INSN_LUID (tmp) - INSN_LUID (tmp2); | |
2704 | } | |
2705 | ||
2706 | /* Resort the array A in which only element at index N may be out of order. */ | |
2707 | ||
3e016693 | 2708 | HAIFA_INLINE static void |
952f0048 | 2709 | swap_sort (rtx *a, int n) |
3eb9a99d | 2710 | { |
2711 | rtx insn = a[n - 1]; | |
2712 | int i = n - 2; | |
2713 | ||
2714 | while (i >= 0 && rank_for_schedule (a + i, &insn) >= 0) | |
2715 | { | |
2716 | a[i + 1] = a[i]; | |
2717 | i -= 1; | |
2718 | } | |
2719 | a[i + 1] = insn; | |
2720 | } | |
2721 | ||
3eb9a99d | 2722 | /* Add INSN to the insn queue so that it can be executed at least |
2723 | N_CYCLES after the currently executing insn. Preserve insns | |
b49b2e5c | 2724 | chain for debugging purposes. REASON will be printed in debugging |
2725 | output. */ | |
3eb9a99d | 2726 | |
3e016693 | 2727 | HAIFA_INLINE static void |
b49b2e5c | 2728 | queue_insn (rtx insn, int n_cycles, const char *reason) |
3eb9a99d | 2729 | { |
2730 | int next_q = NEXT_Q_AFTER (q_ptr, n_cycles); | |
7ce0700a | 2731 | rtx link = alloc_INSN_LIST (insn, insn_queue[next_q]); |
e2f4a6ff | 2732 | int new_tick; |
e4897000 | 2733 | |
2734 | gcc_assert (n_cycles <= max_insn_queue_index); | |
9845d120 | 2735 | gcc_assert (!DEBUG_INSN_P (insn)); |
e4897000 | 2736 | |
3eb9a99d | 2737 | insn_queue[next_q] = link; |
2738 | q_size += 1; | |
2739 | ||
2740 | if (sched_verbose >= 2) | |
2741 | { | |
c2069298 | 2742 | fprintf (sched_dump, ";;\t\tReady-->Q: insn %s: ", |
2743 | (*current_sched_info->print_insn) (insn, 0)); | |
3eb9a99d | 2744 | |
b49b2e5c | 2745 | fprintf (sched_dump, "queued for %d cycles (%s).\n", n_cycles, reason); |
3eb9a99d | 2746 | } |
e1ab7874 | 2747 | |
e4897000 | 2748 | QUEUE_INDEX (insn) = next_q; |
e2f4a6ff | 2749 | |
2750 | if (current_sched_info->flags & DO_BACKTRACKING) | |
2751 | { | |
2752 | new_tick = clock_var + n_cycles; | |
2753 | if (INSN_TICK (insn) == INVALID_TICK || INSN_TICK (insn) < new_tick) | |
2754 | INSN_TICK (insn) = new_tick; | |
2755 | ||
2756 | if (INSN_EXACT_TICK (insn) != INVALID_TICK | |
2757 | && INSN_EXACT_TICK (insn) < clock_var + n_cycles) | |
2758 | { | |
2759 | must_backtrack = true; | |
2760 | if (sched_verbose >= 2) | |
2761 | fprintf (sched_dump, ";;\t\tcausing a backtrack.\n"); | |
2762 | } | |
2763 | } | |
e4897000 | 2764 | } |
2765 | ||
2766 | /* Remove INSN from queue. */ | |
2767 | static void | |
2768 | queue_remove (rtx insn) | |
2769 | { | |
2770 | gcc_assert (QUEUE_INDEX (insn) >= 0); | |
2771 | remove_free_INSN_LIST_elem (insn, &insn_queue[QUEUE_INDEX (insn)]); | |
2772 | q_size--; | |
2773 | QUEUE_INDEX (insn) = QUEUE_NOWHERE; | |
30b1ec30 | 2774 | } |
2775 | ||
2776 | /* Return a pointer to the bottom of the ready list, i.e. the insn | |
2777 | with the lowest priority. */ | |
2778 | ||
e1ab7874 | 2779 | rtx * |
952f0048 | 2780 | ready_lastpos (struct ready_list *ready) |
30b1ec30 | 2781 | { |
e4897000 | 2782 | gcc_assert (ready->n_ready >= 1); |
30b1ec30 | 2783 | return ready->vec + ready->first - ready->n_ready + 1; |
2784 | } | |
2785 | ||
e4897000 | 2786 | /* Add an element INSN to the ready list so that it ends up with the |
9ca2c29a | 2787 | lowest/highest priority depending on FIRST_P. */ |
30b1ec30 | 2788 | |
e4897000 | 2789 | HAIFA_INLINE static void |
2790 | ready_add (struct ready_list *ready, rtx insn, bool first_p) | |
30b1ec30 | 2791 | { |
e4897000 | 2792 | if (!first_p) |
30b1ec30 | 2793 | { |
e4897000 | 2794 | if (ready->first == ready->n_ready) |
2795 | { | |
2796 | memmove (ready->vec + ready->veclen - ready->n_ready, | |
2797 | ready_lastpos (ready), | |
2798 | ready->n_ready * sizeof (rtx)); | |
2799 | ready->first = ready->veclen - 1; | |
2800 | } | |
2801 | ready->vec[ready->first - ready->n_ready] = insn; | |
30b1ec30 | 2802 | } |
e4897000 | 2803 | else |
2804 | { | |
2805 | if (ready->first == ready->veclen - 1) | |
2806 | { | |
2807 | if (ready->n_ready) | |
2808 | /* ready_lastpos() fails when called with (ready->n_ready == 0). */ | |
2809 | memmove (ready->vec + ready->veclen - ready->n_ready - 1, | |
2810 | ready_lastpos (ready), | |
2811 | ready->n_ready * sizeof (rtx)); | |
2812 | ready->first = ready->veclen - 2; | |
2813 | } | |
2814 | ready->vec[++(ready->first)] = insn; | |
2815 | } | |
2816 | ||
30b1ec30 | 2817 | ready->n_ready++; |
9845d120 | 2818 | if (DEBUG_INSN_P (insn)) |
2819 | ready->n_debug++; | |
e4897000 | 2820 | |
2821 | gcc_assert (QUEUE_INDEX (insn) != QUEUE_READY); | |
2822 | QUEUE_INDEX (insn) = QUEUE_READY; | |
e2f4a6ff | 2823 | |
2824 | if (INSN_EXACT_TICK (insn) != INVALID_TICK | |
2825 | && INSN_EXACT_TICK (insn) < clock_var) | |
2826 | { | |
2827 | must_backtrack = true; | |
2828 | } | |
30b1ec30 | 2829 | } |
3eb9a99d | 2830 | |
30b1ec30 | 2831 | /* Remove the element with the highest priority from the ready list and |
2832 | return it. */ | |
2833 | ||
2834 | HAIFA_INLINE static rtx | |
952f0048 | 2835 | ready_remove_first (struct ready_list *ready) |
30b1ec30 | 2836 | { |
2837 | rtx t; | |
48e1416a | 2838 | |
89f18f73 | 2839 | gcc_assert (ready->n_ready); |
30b1ec30 | 2840 | t = ready->vec[ready->first--]; |
2841 | ready->n_ready--; | |
9845d120 | 2842 | if (DEBUG_INSN_P (t)) |
2843 | ready->n_debug--; | |
30b1ec30 | 2844 | /* If the queue becomes empty, reset it. */ |
2845 | if (ready->n_ready == 0) | |
2846 | ready->first = ready->veclen - 1; | |
e4897000 | 2847 | |
2848 | gcc_assert (QUEUE_INDEX (t) == QUEUE_READY); | |
2849 | QUEUE_INDEX (t) = QUEUE_NOWHERE; | |
2850 | ||
30b1ec30 | 2851 | return t; |
2852 | } | |
2853 | ||
bea4bad2 | 2854 | /* The following code implements multi-pass scheduling for the first |
2855 | cycle. In other words, we will try to choose ready insn which | |
2856 | permits to start maximum number of insns on the same cycle. */ | |
2857 | ||
2858 | /* Return a pointer to the element INDEX from the ready. INDEX for | |
2859 | insn with the highest priority is 0, and the lowest priority has | |
2860 | N_READY - 1. */ | |
2861 | ||
e1ab7874 | 2862 | rtx |
952f0048 | 2863 | ready_element (struct ready_list *ready, int index) |
bea4bad2 | 2864 | { |
89f18f73 | 2865 | gcc_assert (ready->n_ready && index < ready->n_ready); |
48e1416a | 2866 | |
bea4bad2 | 2867 | return ready->vec[ready->first - index]; |
2868 | } | |
2869 | ||
2870 | /* Remove the element INDEX from the ready list and return it. INDEX | |
2871 | for insn with the highest priority is 0, and the lowest priority | |
2872 | has N_READY - 1. */ | |
2873 | ||
2874 | HAIFA_INLINE static rtx | |
952f0048 | 2875 | ready_remove (struct ready_list *ready, int index) |
bea4bad2 | 2876 | { |
2877 | rtx t; | |
2878 | int i; | |
2879 | ||
2880 | if (index == 0) | |
2881 | return ready_remove_first (ready); | |
89f18f73 | 2882 | gcc_assert (ready->n_ready && index < ready->n_ready); |
bea4bad2 | 2883 | t = ready->vec[ready->first - index]; |
2884 | ready->n_ready--; | |
9845d120 | 2885 | if (DEBUG_INSN_P (t)) |
2886 | ready->n_debug--; | |
bea4bad2 | 2887 | for (i = index; i < ready->n_ready; i++) |
2888 | ready->vec[ready->first - i] = ready->vec[ready->first - i - 1]; | |
e4897000 | 2889 | QUEUE_INDEX (t) = QUEUE_NOWHERE; |
bea4bad2 | 2890 | return t; |
2891 | } | |
2892 | ||
e4897000 | 2893 | /* Remove INSN from the ready list. */ |
2894 | static void | |
2895 | ready_remove_insn (rtx insn) | |
2896 | { | |
2897 | int i; | |
2898 | ||
2899 | for (i = 0; i < readyp->n_ready; i++) | |
2900 | if (ready_element (readyp, i) == insn) | |
2901 | { | |
2902 | ready_remove (readyp, i); | |
2903 | return; | |
2904 | } | |
2905 | gcc_unreachable (); | |
2906 | } | |
bea4bad2 | 2907 | |
30b1ec30 | 2908 | /* Sort the ready list READY by ascending priority, using the SCHED_SORT |
2909 | macro. */ | |
2910 | ||
e1ab7874 | 2911 | void |
952f0048 | 2912 | ready_sort (struct ready_list *ready) |
30b1ec30 | 2913 | { |
a7dcf969 | 2914 | int i; |
30b1ec30 | 2915 | rtx *first = ready_lastpos (ready); |
a7dcf969 | 2916 | |
11189c7a | 2917 | if (sched_pressure == SCHED_PRESSURE_WEIGHTED) |
a7dcf969 | 2918 | { |
2919 | for (i = 0; i < ready->n_ready; i++) | |
10272f4e | 2920 | if (!DEBUG_INSN_P (first[i])) |
2921 | setup_insn_reg_pressure_info (first[i]); | |
a7dcf969 | 2922 | } |
b30b031c | 2923 | if (sched_pressure == SCHED_PRESSURE_MODEL |
2924 | && model_curr_point < model_num_insns) | |
2925 | model_set_excess_costs (first, ready->n_ready); | |
30b1ec30 | 2926 | SCHED_SORT (first, ready->n_ready); |
3eb9a99d | 2927 | } |
2928 | ||
3eb9a99d | 2929 | /* PREV is an insn that is ready to execute. Adjust its priority if that |
ba57cb24 | 2930 | will help shorten or lengthen register lifetimes as appropriate. Also |
f0b5f617 | 2931 | provide a hook for the target to tweak itself. */ |
3eb9a99d | 2932 | |
3e016693 | 2933 | HAIFA_INLINE static void |
952f0048 | 2934 | adjust_priority (rtx prev) |
3eb9a99d | 2935 | { |
ba57cb24 | 2936 | /* ??? There used to be code here to try and estimate how an insn |
2937 | affected register lifetimes, but it did it by looking at REG_DEAD | |
896c2bfe | 2938 | notes, which we removed in schedule_region. Nor did it try to |
ba57cb24 | 2939 | take into account register pressure or anything useful like that. |
3eb9a99d | 2940 | |
ba57cb24 | 2941 | Revisit when we have a machine model to work with and not before. */ |
de206697 | 2942 | |
747af5e7 | 2943 | if (targetm.sched.adjust_priority) |
2944 | INSN_PRIORITY (prev) = | |
883b2e73 | 2945 | targetm.sched.adjust_priority (prev, INSN_PRIORITY (prev)); |
3eb9a99d | 2946 | } |
2947 | ||
e1ab7874 | 2948 | /* Advance DFA state STATE on one cycle. */ |
2949 | void | |
2950 | advance_state (state_t state) | |
bea4bad2 | 2951 | { |
07317795 | 2952 | if (targetm.sched.dfa_pre_advance_cycle) |
2953 | targetm.sched.dfa_pre_advance_cycle (); | |
2954 | ||
67900a4f | 2955 | if (targetm.sched.dfa_pre_cycle_insn) |
e1ab7874 | 2956 | state_transition (state, |
67900a4f | 2957 | targetm.sched.dfa_pre_cycle_insn ()); |
2958 | ||
e1ab7874 | 2959 | state_transition (state, NULL); |
48e1416a | 2960 | |
67900a4f | 2961 | if (targetm.sched.dfa_post_cycle_insn) |
e1ab7874 | 2962 | state_transition (state, |
67900a4f | 2963 | targetm.sched.dfa_post_cycle_insn ()); |
07317795 | 2964 | |
2965 | if (targetm.sched.dfa_post_advance_cycle) | |
2966 | targetm.sched.dfa_post_advance_cycle (); | |
bea4bad2 | 2967 | } |
2968 | ||
e1ab7874 | 2969 | /* Advance time on one cycle. */ |
2970 | HAIFA_INLINE static void | |
2971 | advance_one_cycle (void) | |
2972 | { | |
2973 | advance_state (curr_state); | |
2974 | if (sched_verbose >= 6) | |
76f8cfc9 | 2975 | fprintf (sched_dump, ";;\tAdvanced a state.\n"); |
e1ab7874 | 2976 | } |
2977 | ||
a7dcf969 | 2978 | /* Update register pressure after scheduling INSN. */ |
2979 | static void | |
2980 | update_register_pressure (rtx insn) | |
2981 | { | |
2982 | struct reg_use_data *use; | |
2983 | struct reg_set_data *set; | |
2984 | ||
10272f4e | 2985 | gcc_checking_assert (!DEBUG_INSN_P (insn)); |
2986 | ||
a7dcf969 | 2987 | for (use = INSN_REG_USE_LIST (insn); use != NULL; use = use->next_insn_use) |
b30b031c | 2988 | if (dying_use_p (use)) |
2989 | mark_regno_birth_or_death (curr_reg_live, curr_reg_pressure, | |
2990 | use->regno, false); | |
a7dcf969 | 2991 | for (set = INSN_REG_SET_LIST (insn); set != NULL; set = set->next_insn_set) |
b30b031c | 2992 | mark_regno_birth_or_death (curr_reg_live, curr_reg_pressure, |
2993 | set->regno, true); | |
a7dcf969 | 2994 | } |
2995 | ||
2996 | /* Set up or update (if UPDATE_P) max register pressure (see its | |
2997 | meaning in sched-int.h::_haifa_insn_data) for all current BB insns | |
2998 | after insn AFTER. */ | |
2999 | static void | |
3000 | setup_insn_max_reg_pressure (rtx after, bool update_p) | |
3001 | { | |
3002 | int i, p; | |
3003 | bool eq_p; | |
3004 | rtx insn; | |
3005 | static int max_reg_pressure[N_REG_CLASSES]; | |
3006 | ||
3007 | save_reg_pressure (); | |
66d9a7b9 | 3008 | for (i = 0; i < ira_pressure_classes_num; i++) |
3009 | max_reg_pressure[ira_pressure_classes[i]] | |
3010 | = curr_reg_pressure[ira_pressure_classes[i]]; | |
a7dcf969 | 3011 | for (insn = NEXT_INSN (after); |
3a16ebc2 | 3012 | insn != NULL_RTX && ! BARRIER_P (insn) |
3013 | && BLOCK_FOR_INSN (insn) == BLOCK_FOR_INSN (after); | |
a7dcf969 | 3014 | insn = NEXT_INSN (insn)) |
3015 | if (NONDEBUG_INSN_P (insn)) | |
3016 | { | |
3017 | eq_p = true; | |
66d9a7b9 | 3018 | for (i = 0; i < ira_pressure_classes_num; i++) |
a7dcf969 | 3019 | { |
66d9a7b9 | 3020 | p = max_reg_pressure[ira_pressure_classes[i]]; |
a7dcf969 | 3021 | if (INSN_MAX_REG_PRESSURE (insn)[i] != p) |
3022 | { | |
3023 | eq_p = false; | |
3024 | INSN_MAX_REG_PRESSURE (insn)[i] | |
66d9a7b9 | 3025 | = max_reg_pressure[ira_pressure_classes[i]]; |
a7dcf969 | 3026 | } |
3027 | } | |
3028 | if (update_p && eq_p) | |
3029 | break; | |
3030 | update_register_pressure (insn); | |
66d9a7b9 | 3031 | for (i = 0; i < ira_pressure_classes_num; i++) |
3032 | if (max_reg_pressure[ira_pressure_classes[i]] | |
3033 | < curr_reg_pressure[ira_pressure_classes[i]]) | |
3034 | max_reg_pressure[ira_pressure_classes[i]] | |
3035 | = curr_reg_pressure[ira_pressure_classes[i]]; | |
a7dcf969 | 3036 | } |
3037 | restore_reg_pressure (); | |
3038 | } | |
3039 | ||
3040 | /* Update the current register pressure after scheduling INSN. Update | |
3041 | also max register pressure for unscheduled insns of the current | |
3042 | BB. */ | |
3043 | static void | |
3044 | update_reg_and_insn_max_reg_pressure (rtx insn) | |
3045 | { | |
3046 | int i; | |
3047 | int before[N_REG_CLASSES]; | |
3048 | ||
66d9a7b9 | 3049 | for (i = 0; i < ira_pressure_classes_num; i++) |
3050 | before[i] = curr_reg_pressure[ira_pressure_classes[i]]; | |
a7dcf969 | 3051 | update_register_pressure (insn); |
66d9a7b9 | 3052 | for (i = 0; i < ira_pressure_classes_num; i++) |
3053 | if (curr_reg_pressure[ira_pressure_classes[i]] != before[i]) | |
a7dcf969 | 3054 | break; |
66d9a7b9 | 3055 | if (i < ira_pressure_classes_num) |
a7dcf969 | 3056 | setup_insn_max_reg_pressure (insn, true); |
3057 | } | |
3058 | ||
3059 | /* Set up register pressure at the beginning of basic block BB whose | |
3060 | insns starting after insn AFTER. Set up also max register pressure | |
3061 | for all insns of the basic block. */ | |
3062 | void | |
3063 | sched_setup_bb_reg_pressure_info (basic_block bb, rtx after) | |
3064 | { | |
11189c7a | 3065 | gcc_assert (sched_pressure == SCHED_PRESSURE_WEIGHTED); |
a7dcf969 | 3066 | initiate_bb_reg_pressure_info (bb); |
3067 | setup_insn_max_reg_pressure (after, false); | |
3068 | } | |
0c775e95 | 3069 | \f |
effd1640 | 3070 | /* If doing predication while scheduling, verify whether INSN, which |
3071 | has just been scheduled, clobbers the conditions of any | |
3072 | instructions that must be predicated in order to break their | |
3073 | dependencies. If so, remove them from the queues so that they will | |
3074 | only be scheduled once their control dependency is resolved. */ | |
3075 | ||
3076 | static void | |
3077 | check_clobbered_conditions (rtx insn) | |
3078 | { | |
3079 | HARD_REG_SET t; | |
3080 | int i; | |
3081 | ||
3082 | if ((current_sched_info->flags & DO_PREDICATION) == 0) | |
3083 | return; | |
3084 | ||
3085 | find_all_hard_reg_sets (insn, &t); | |
3086 | ||
3087 | restart: | |
3088 | for (i = 0; i < ready.n_ready; i++) | |
3089 | { | |
3090 | rtx x = ready_element (&ready, i); | |
3091 | if (TODO_SPEC (x) == DEP_CONTROL && cond_clobbered_p (x, t)) | |
3092 | { | |
3093 | ready_remove_insn (x); | |
3094 | goto restart; | |
3095 | } | |
3096 | } | |
3097 | for (i = 0; i <= max_insn_queue_index; i++) | |
3098 | { | |
3099 | rtx link; | |
3100 | int q = NEXT_Q_AFTER (q_ptr, i); | |
3101 | ||
3102 | restart_queue: | |
3103 | for (link = insn_queue[q]; link; link = XEXP (link, 1)) | |
3104 | { | |
3105 | rtx x = XEXP (link, 0); | |
3106 | if (TODO_SPEC (x) == DEP_CONTROL && cond_clobbered_p (x, t)) | |
3107 | { | |
3108 | queue_remove (x); | |
3109 | goto restart_queue; | |
3110 | } | |
3111 | } | |
3112 | } | |
3113 | } | |
3114 | \f | |
b30b031c | 3115 | /* Return (in order): |
3116 | ||
3117 | - positive if INSN adversely affects the pressure on one | |
3118 | register class | |
3119 | ||
3120 | - negative if INSN reduces the pressure on one register class | |
3121 | ||
3122 | - 0 if INSN doesn't affect the pressure on any register class. */ | |
3123 | ||
3124 | static int | |
3125 | model_classify_pressure (struct model_insn_info *insn) | |
3126 | { | |
3127 | struct reg_pressure_data *reg_pressure; | |
3128 | int death[N_REG_CLASSES]; | |
3129 | int pci, cl, sum; | |
3130 | ||
3131 | calculate_reg_deaths (insn->insn, death); | |
3132 | reg_pressure = INSN_REG_PRESSURE (insn->insn); | |
3133 | sum = 0; | |
3134 | for (pci = 0; pci < ira_pressure_classes_num; pci++) | |
3135 | { | |
3136 | cl = ira_pressure_classes[pci]; | |
3137 | if (death[cl] < reg_pressure[pci].set_increase) | |
3138 | return 1; | |
3139 | sum += reg_pressure[pci].set_increase - death[cl]; | |
3140 | } | |
3141 | return sum; | |
3142 | } | |
3143 | ||
3144 | /* Return true if INSN1 should come before INSN2 in the model schedule. */ | |
3145 | ||
3146 | static int | |
3147 | model_order_p (struct model_insn_info *insn1, struct model_insn_info *insn2) | |
3148 | { | |
3149 | unsigned int height1, height2; | |
3150 | unsigned int priority1, priority2; | |
3151 | ||
3152 | /* Prefer instructions with a higher model priority. */ | |
3153 | if (insn1->model_priority != insn2->model_priority) | |
3154 | return insn1->model_priority > insn2->model_priority; | |
3155 | ||
3156 | /* Combine the length of the longest path of satisfied true dependencies | |
3157 | that leads to each instruction (depth) with the length of the longest | |
3158 | path of any dependencies that leads from the instruction (alap). | |
3159 | Prefer instructions with the greatest combined length. If the combined | |
3160 | lengths are equal, prefer instructions with the greatest depth. | |
3161 | ||
3162 | The idea is that, if we have a set S of "equal" instructions that each | |
3163 | have ALAP value X, and we pick one such instruction I, any true-dependent | |
3164 | successors of I that have ALAP value X - 1 should be preferred over S. | |
3165 | This encourages the schedule to be "narrow" rather than "wide". | |
3166 | However, if I is a low-priority instruction that we decided to | |
3167 | schedule because of its model_classify_pressure, and if there | |
3168 | is a set of higher-priority instructions T, the aforementioned | |
3169 | successors of I should not have the edge over T. */ | |
3170 | height1 = insn1->depth + insn1->alap; | |
3171 | height2 = insn2->depth + insn2->alap; | |
3172 | if (height1 != height2) | |
3173 | return height1 > height2; | |
3174 | if (insn1->depth != insn2->depth) | |
3175 | return insn1->depth > insn2->depth; | |
3176 | ||
3177 | /* We have no real preference between INSN1 an INSN2 as far as attempts | |
3178 | to reduce pressure go. Prefer instructions with higher priorities. */ | |
3179 | priority1 = INSN_PRIORITY (insn1->insn); | |
3180 | priority2 = INSN_PRIORITY (insn2->insn); | |
3181 | if (priority1 != priority2) | |
3182 | return priority1 > priority2; | |
3183 | ||
3184 | /* Use the original rtl sequence as a tie-breaker. */ | |
3185 | return insn1 < insn2; | |
3186 | } | |
3187 | ||
3188 | /* Add INSN to the model worklist immediately after PREV. Add it to the | |
3189 | beginning of the list if PREV is null. */ | |
3190 | ||
3191 | static void | |
3192 | model_add_to_worklist_at (struct model_insn_info *insn, | |
3193 | struct model_insn_info *prev) | |
3194 | { | |
3195 | gcc_assert (QUEUE_INDEX (insn->insn) == QUEUE_NOWHERE); | |
3196 | QUEUE_INDEX (insn->insn) = QUEUE_READY; | |
3197 | ||
3198 | insn->prev = prev; | |
3199 | if (prev) | |
3200 | { | |
3201 | insn->next = prev->next; | |
3202 | prev->next = insn; | |
3203 | } | |
3204 | else | |
3205 | { | |
3206 | insn->next = model_worklist; | |
3207 | model_worklist = insn; | |
3208 | } | |
3209 | if (insn->next) | |
3210 | insn->next->prev = insn; | |
3211 | } | |
3212 | ||
3213 | /* Remove INSN from the model worklist. */ | |
3214 | ||
3215 | static void | |
3216 | model_remove_from_worklist (struct model_insn_info *insn) | |
3217 | { | |
3218 | gcc_assert (QUEUE_INDEX (insn->insn) == QUEUE_READY); | |
3219 | QUEUE_INDEX (insn->insn) = QUEUE_NOWHERE; | |
3220 | ||
3221 | if (insn->prev) | |
3222 | insn->prev->next = insn->next; | |
3223 | else | |
3224 | model_worklist = insn->next; | |
3225 | if (insn->next) | |
3226 | insn->next->prev = insn->prev; | |
3227 | } | |
3228 | ||
3229 | /* Add INSN to the model worklist. Start looking for a suitable position | |
3230 | between neighbors PREV and NEXT, testing at most MAX_SCHED_READY_INSNS | |
3231 | insns either side. A null PREV indicates the beginning of the list and | |
3232 | a null NEXT indicates the end. */ | |
3233 | ||
3234 | static void | |
3235 | model_add_to_worklist (struct model_insn_info *insn, | |
3236 | struct model_insn_info *prev, | |
3237 | struct model_insn_info *next) | |
3238 | { | |
3239 | int count; | |
3240 | ||
3241 | count = MAX_SCHED_READY_INSNS; | |
3242 | if (count > 0 && prev && model_order_p (insn, prev)) | |
3243 | do | |
3244 | { | |
3245 | count--; | |
3246 | prev = prev->prev; | |
3247 | } | |
3248 | while (count > 0 && prev && model_order_p (insn, prev)); | |
3249 | else | |
3250 | while (count > 0 && next && model_order_p (next, insn)) | |
3251 | { | |
3252 | count--; | |
3253 | prev = next; | |
3254 | next = next->next; | |
3255 | } | |
3256 | model_add_to_worklist_at (insn, prev); | |
3257 | } | |
3258 | ||
3259 | /* INSN may now have a higher priority (in the model_order_p sense) | |
3260 | than before. Move it up the worklist if necessary. */ | |
3261 | ||
3262 | static void | |
3263 | model_promote_insn (struct model_insn_info *insn) | |
3264 | { | |
3265 | struct model_insn_info *prev; | |
3266 | int count; | |
3267 | ||
3268 | prev = insn->prev; | |
3269 | count = MAX_SCHED_READY_INSNS; | |
3270 | while (count > 0 && prev && model_order_p (insn, prev)) | |
3271 | { | |
3272 | count--; | |
3273 | prev = prev->prev; | |
3274 | } | |
3275 | if (prev != insn->prev) | |
3276 | { | |
3277 | model_remove_from_worklist (insn); | |
3278 | model_add_to_worklist_at (insn, prev); | |
3279 | } | |
3280 | } | |
3281 | ||
3282 | /* Add INSN to the end of the model schedule. */ | |
3283 | ||
3284 | static void | |
3285 | model_add_to_schedule (rtx insn) | |
3286 | { | |
3287 | unsigned int point; | |
3288 | ||
3289 | gcc_assert (QUEUE_INDEX (insn) == QUEUE_NOWHERE); | |
3290 | QUEUE_INDEX (insn) = QUEUE_SCHEDULED; | |
3291 | ||
f1f41a6c | 3292 | point = model_schedule.length (); |
3293 | model_schedule.quick_push (insn); | |
b30b031c | 3294 | INSN_MODEL_INDEX (insn) = point + 1; |
3295 | } | |
3296 | ||
3297 | /* Analyze the instructions that are to be scheduled, setting up | |
3298 | MODEL_INSN_INFO (...) and model_num_insns accordingly. Add ready | |
3299 | instructions to model_worklist. */ | |
3300 | ||
3301 | static void | |
3302 | model_analyze_insns (void) | |
3303 | { | |
3304 | rtx start, end, iter; | |
3305 | sd_iterator_def sd_it; | |
3306 | dep_t dep; | |
3307 | struct model_insn_info *insn, *con; | |
3308 | ||
3309 | model_num_insns = 0; | |
3310 | start = PREV_INSN (current_sched_info->next_tail); | |
3311 | end = current_sched_info->prev_head; | |
3312 | for (iter = start; iter != end; iter = PREV_INSN (iter)) | |
3313 | if (NONDEBUG_INSN_P (iter)) | |
3314 | { | |
3315 | insn = MODEL_INSN_INFO (iter); | |
3316 | insn->insn = iter; | |
3317 | FOR_EACH_DEP (iter, SD_LIST_FORW, sd_it, dep) | |
3318 | { | |
3319 | con = MODEL_INSN_INFO (DEP_CON (dep)); | |
3320 | if (con->insn && insn->alap < con->alap + 1) | |
3321 | insn->alap = con->alap + 1; | |
3322 | } | |
3323 | ||
3324 | insn->old_queue = QUEUE_INDEX (iter); | |
3325 | QUEUE_INDEX (iter) = QUEUE_NOWHERE; | |
3326 | ||
3327 | insn->unscheduled_preds = dep_list_size (iter, SD_LIST_HARD_BACK); | |
3328 | if (insn->unscheduled_preds == 0) | |
3329 | model_add_to_worklist (insn, NULL, model_worklist); | |
3330 | ||
3331 | model_num_insns++; | |
3332 | } | |
3333 | } | |
3334 | ||
3335 | /* The global state describes the register pressure at the start of the | |
3336 | model schedule. Initialize GROUP accordingly. */ | |
3337 | ||
3338 | static void | |
3339 | model_init_pressure_group (struct model_pressure_group *group) | |
3340 | { | |
3341 | int pci, cl; | |
3342 | ||
3343 | for (pci = 0; pci < ira_pressure_classes_num; pci++) | |
3344 | { | |
3345 | cl = ira_pressure_classes[pci]; | |
3346 | group->limits[pci].pressure = curr_reg_pressure[cl]; | |
3347 | group->limits[pci].point = 0; | |
3348 | } | |
3349 | /* Use index model_num_insns to record the state after the last | |
3350 | instruction in the model schedule. */ | |
3351 | group->model = XNEWVEC (struct model_pressure_data, | |
3352 | (model_num_insns + 1) * ira_pressure_classes_num); | |
3353 | } | |
3354 | ||
3355 | /* Record that MODEL_REF_PRESSURE (GROUP, POINT, PCI) is PRESSURE. | |
3356 | Update the maximum pressure for the whole schedule. */ | |
3357 | ||
3358 | static void | |
3359 | model_record_pressure (struct model_pressure_group *group, | |
3360 | int point, int pci, int pressure) | |
3361 | { | |
3362 | MODEL_REF_PRESSURE (group, point, pci) = pressure; | |
3363 | if (group->limits[pci].pressure < pressure) | |
3364 | { | |
3365 | group->limits[pci].pressure = pressure; | |
3366 | group->limits[pci].point = point; | |
3367 | } | |
3368 | } | |
3369 | ||
3370 | /* INSN has just been added to the end of the model schedule. Record its | |
3371 | register-pressure information. */ | |
3372 | ||
3373 | static void | |
3374 | model_record_pressures (struct model_insn_info *insn) | |
3375 | { | |
3376 | struct reg_pressure_data *reg_pressure; | |
3377 | int point, pci, cl, delta; | |
3378 | int death[N_REG_CLASSES]; | |
3379 | ||
3380 | point = model_index (insn->insn); | |
3381 | if (sched_verbose >= 2) | |
3382 | { | |
b30b031c | 3383 | if (point == 0) |
3384 | { | |
3385 | fprintf (sched_dump, "\n;;\tModel schedule:\n;;\n"); | |
3386 | fprintf (sched_dump, ";;\t| idx insn | mpri hght dpth prio |\n"); | |
3387 | } | |
b30b031c | 3388 | fprintf (sched_dump, ";;\t| %3d %4d | %4d %4d %4d %4d | %-30s ", |
3389 | point, INSN_UID (insn->insn), insn->model_priority, | |
3390 | insn->depth + insn->alap, insn->depth, | |
3d9a4504 | 3391 | INSN_PRIORITY (insn->insn), |
3392 | str_pattern_slim (PATTERN (insn->insn))); | |
b30b031c | 3393 | } |
3394 | calculate_reg_deaths (insn->insn, death); | |
3395 | reg_pressure = INSN_REG_PRESSURE (insn->insn); | |
3396 | for (pci = 0; pci < ira_pressure_classes_num; pci++) | |
3397 | { | |
3398 | cl = ira_pressure_classes[pci]; | |
3399 | delta = reg_pressure[pci].set_increase - death[cl]; | |
3400 | if (sched_verbose >= 2) | |
3401 | fprintf (sched_dump, " %s:[%d,%+d]", reg_class_names[cl], | |
3402 | curr_reg_pressure[cl], delta); | |
3403 | model_record_pressure (&model_before_pressure, point, pci, | |
3404 | curr_reg_pressure[cl]); | |
3405 | } | |
3406 | if (sched_verbose >= 2) | |
3407 | fprintf (sched_dump, "\n"); | |
3408 | } | |
3409 | ||
3410 | /* All instructions have been added to the model schedule. Record the | |
3411 | final register pressure in GROUP and set up all MODEL_MAX_PRESSUREs. */ | |
3412 | ||
3413 | static void | |
3414 | model_record_final_pressures (struct model_pressure_group *group) | |
3415 | { | |
3416 | int point, pci, max_pressure, ref_pressure, cl; | |
3417 | ||
3418 | for (pci = 0; pci < ira_pressure_classes_num; pci++) | |
3419 | { | |
3420 | /* Record the final pressure for this class. */ | |
3421 | cl = ira_pressure_classes[pci]; | |
3422 | point = model_num_insns; | |
3423 | ref_pressure = curr_reg_pressure[cl]; | |
3424 | model_record_pressure (group, point, pci, ref_pressure); | |
3425 | ||
3426 | /* Record the original maximum pressure. */ | |
3427 | group->limits[pci].orig_pressure = group->limits[pci].pressure; | |
3428 | ||
3429 | /* Update the MODEL_MAX_PRESSURE for every point of the schedule. */ | |
3430 | max_pressure = ref_pressure; | |
3431 | MODEL_MAX_PRESSURE (group, point, pci) = max_pressure; | |
3432 | while (point > 0) | |
3433 | { | |
3434 | point--; | |
3435 | ref_pressure = MODEL_REF_PRESSURE (group, point, pci); | |
3436 | max_pressure = MAX (max_pressure, ref_pressure); | |
3437 | MODEL_MAX_PRESSURE (group, point, pci) = max_pressure; | |
3438 | } | |
3439 | } | |
3440 | } | |
3441 | ||
3442 | /* Update all successors of INSN, given that INSN has just been scheduled. */ | |
3443 | ||
3444 | static void | |
3445 | model_add_successors_to_worklist (struct model_insn_info *insn) | |
3446 | { | |
3447 | sd_iterator_def sd_it; | |
3448 | struct model_insn_info *con; | |
3449 | dep_t dep; | |
3450 | ||
3451 | FOR_EACH_DEP (insn->insn, SD_LIST_FORW, sd_it, dep) | |
3452 | { | |
3453 | con = MODEL_INSN_INFO (DEP_CON (dep)); | |
3454 | /* Ignore debug instructions, and instructions from other blocks. */ | |
3455 | if (con->insn) | |
3456 | { | |
3457 | con->unscheduled_preds--; | |
3458 | ||
3459 | /* Update the depth field of each true-dependent successor. | |
3460 | Increasing the depth gives them a higher priority than | |
3461 | before. */ | |
3462 | if (DEP_TYPE (dep) == REG_DEP_TRUE && con->depth < insn->depth + 1) | |
3463 | { | |
3464 | con->depth = insn->depth + 1; | |
3465 | if (QUEUE_INDEX (con->insn) == QUEUE_READY) | |
3466 | model_promote_insn (con); | |
3467 | } | |
3468 | ||
3469 | /* If this is a true dependency, or if there are no remaining | |
3470 | dependencies for CON (meaning that CON only had non-true | |
3471 | dependencies), make sure that CON is on the worklist. | |
3472 | We don't bother otherwise because it would tend to fill the | |
3473 | worklist with a lot of low-priority instructions that are not | |
3474 | yet ready to issue. */ | |
3475 | if ((con->depth > 0 || con->unscheduled_preds == 0) | |
3476 | && QUEUE_INDEX (con->insn) == QUEUE_NOWHERE) | |
3477 | model_add_to_worklist (con, insn, insn->next); | |
3478 | } | |
3479 | } | |
3480 | } | |
3481 | ||
3482 | /* Give INSN a higher priority than any current instruction, then give | |
3483 | unscheduled predecessors of INSN a higher priority still. If any of | |
3484 | those predecessors are not on the model worklist, do the same for its | |
3485 | predecessors, and so on. */ | |
3486 | ||
3487 | static void | |
3488 | model_promote_predecessors (struct model_insn_info *insn) | |
3489 | { | |
3490 | struct model_insn_info *pro, *first; | |
3491 | sd_iterator_def sd_it; | |
3492 | dep_t dep; | |
3493 | ||
3494 | if (sched_verbose >= 7) | |
3495 | fprintf (sched_dump, ";;\t+--- priority of %d = %d, priority of", | |
3496 | INSN_UID (insn->insn), model_next_priority); | |
3497 | insn->model_priority = model_next_priority++; | |
3498 | model_remove_from_worklist (insn); | |
3499 | model_add_to_worklist_at (insn, NULL); | |
3500 | ||
3501 | first = NULL; | |
3502 | for (;;) | |
3503 | { | |
3504 | FOR_EACH_DEP (insn->insn, SD_LIST_HARD_BACK, sd_it, dep) | |
3505 | { | |
3506 | pro = MODEL_INSN_INFO (DEP_PRO (dep)); | |
3507 | /* The first test is to ignore debug instructions, and instructions | |
3508 | from other blocks. */ | |
3509 | if (pro->insn | |
3510 | && pro->model_priority != model_next_priority | |
3511 | && QUEUE_INDEX (pro->insn) != QUEUE_SCHEDULED) | |
3512 | { | |
3513 | pro->model_priority = model_next_priority; | |
3514 | if (sched_verbose >= 7) | |
3515 | fprintf (sched_dump, " %d", INSN_UID (pro->insn)); | |
3516 | if (QUEUE_INDEX (pro->insn) == QUEUE_READY) | |
3517 | { | |
3518 | /* PRO is already in the worklist, but it now has | |
3519 | a higher priority than before. Move it at the | |
3520 | appropriate place. */ | |
3521 | model_remove_from_worklist (pro); | |
3522 | model_add_to_worklist (pro, NULL, model_worklist); | |
3523 | } | |
3524 | else | |
3525 | { | |
3526 | /* PRO isn't in the worklist. Recursively process | |
3527 | its predecessors until we find one that is. */ | |
3528 | pro->next = first; | |
3529 | first = pro; | |
3530 | } | |
3531 | } | |
3532 | } | |
3533 | if (!first) | |
3534 | break; | |
3535 | insn = first; | |
3536 | first = insn->next; | |
3537 | } | |
3538 | if (sched_verbose >= 7) | |
3539 | fprintf (sched_dump, " = %d\n", model_next_priority); | |
3540 | model_next_priority++; | |
3541 | } | |
3542 | ||
3543 | /* Pick one instruction from model_worklist and process it. */ | |
3544 | ||
3545 | static void | |
3546 | model_choose_insn (void) | |
3547 | { | |
3548 | struct model_insn_info *insn, *fallback; | |
3549 | int count; | |
3550 | ||
3551 | if (sched_verbose >= 7) | |
3552 | { | |
3553 | fprintf (sched_dump, ";;\t+--- worklist:\n"); | |
3554 | insn = model_worklist; | |
3555 | count = MAX_SCHED_READY_INSNS; | |
3556 | while (count > 0 && insn) | |
3557 | { | |
3558 | fprintf (sched_dump, ";;\t+--- %d [%d, %d, %d, %d]\n", | |
3559 | INSN_UID (insn->insn), insn->model_priority, | |
3560 | insn->depth + insn->alap, insn->depth, | |
3561 | INSN_PRIORITY (insn->insn)); | |
3562 | count--; | |
3563 | insn = insn->next; | |
3564 | } | |
3565 | } | |
3566 | ||
3567 | /* Look for a ready instruction whose model_classify_priority is zero | |
3568 | or negative, picking the highest-priority one. Adding such an | |
3569 | instruction to the schedule now should do no harm, and may actually | |
3570 | do some good. | |
3571 | ||
3572 | Failing that, see whether there is an instruction with the highest | |
3573 | extant model_priority that is not yet ready, but which would reduce | |
3574 | pressure if it became ready. This is designed to catch cases like: | |
3575 | ||
3576 | (set (mem (reg R1)) (reg R2)) | |
3577 | ||
3578 | where the instruction is the last remaining use of R1 and where the | |
3579 | value of R2 is not yet available (or vice versa). The death of R1 | |
3580 | means that this instruction already reduces pressure. It is of | |
3581 | course possible that the computation of R2 involves other registers | |
3582 | that are hard to kill, but such cases are rare enough for this | |
3583 | heuristic to be a win in general. | |
3584 | ||
3585 | Failing that, just pick the highest-priority instruction in the | |
3586 | worklist. */ | |
3587 | count = MAX_SCHED_READY_INSNS; | |
3588 | insn = model_worklist; | |
3589 | fallback = 0; | |
3590 | for (;;) | |
3591 | { | |
3592 | if (count == 0 || !insn) | |
3593 | { | |
3594 | insn = fallback ? fallback : model_worklist; | |
3595 | break; | |
3596 | } | |
3597 | if (insn->unscheduled_preds) | |
3598 | { | |
3599 | if (model_worklist->model_priority == insn->model_priority | |
3600 | && !fallback | |
3601 | && model_classify_pressure (insn) < 0) | |
3602 | fallback = insn; | |
3603 | } | |
3604 | else | |
3605 | { | |
3606 | if (model_classify_pressure (insn) <= 0) | |
3607 | break; | |
3608 | } | |
3609 | count--; | |
3610 | insn = insn->next; | |
3611 | } | |
3612 | ||
3613 | if (sched_verbose >= 7 && insn != model_worklist) | |
3614 | { | |
3615 | if (insn->unscheduled_preds) | |
3616 | fprintf (sched_dump, ";;\t+--- promoting insn %d, with dependencies\n", | |
3617 | INSN_UID (insn->insn)); | |
3618 | else | |
3619 | fprintf (sched_dump, ";;\t+--- promoting insn %d, which is ready\n", | |
3620 | INSN_UID (insn->insn)); | |
3621 | } | |
3622 | if (insn->unscheduled_preds) | |
3623 | /* INSN isn't yet ready to issue. Give all its predecessors the | |
3624 | highest priority. */ | |
3625 | model_promote_predecessors (insn); | |
3626 | else | |
3627 | { | |
3628 | /* INSN is ready. Add it to the end of model_schedule and | |
3629 | process its successors. */ | |
3630 | model_add_successors_to_worklist (insn); | |
3631 | model_remove_from_worklist (insn); | |
3632 | model_add_to_schedule (insn->insn); | |
3633 | model_record_pressures (insn); | |
3634 | update_register_pressure (insn->insn); | |
3635 | } | |
3636 | } | |
3637 | ||
3638 | /* Restore all QUEUE_INDEXs to the values that they had before | |
3639 | model_start_schedule was called. */ | |
3640 | ||
3641 | static void | |
3642 | model_reset_queue_indices (void) | |
3643 | { | |
3644 | unsigned int i; | |
3645 | rtx insn; | |
3646 | ||
f1f41a6c | 3647 | FOR_EACH_VEC_ELT (model_schedule, i, insn) |
b30b031c | 3648 | QUEUE_INDEX (insn) = MODEL_INSN_INFO (insn)->old_queue; |
3649 | } | |
3650 | ||
3651 | /* We have calculated the model schedule and spill costs. Print a summary | |
3652 | to sched_dump. */ | |
3653 | ||
3654 | static void | |
3655 | model_dump_pressure_summary (void) | |
3656 | { | |
3657 | int pci, cl; | |
3658 | ||
3659 | fprintf (sched_dump, ";; Pressure summary:"); | |
3660 | for (pci = 0; pci < ira_pressure_classes_num; pci++) | |
3661 | { | |
3662 | cl = ira_pressure_classes[pci]; | |
3663 | fprintf (sched_dump, " %s:%d", reg_class_names[cl], | |
3664 | model_before_pressure.limits[pci].pressure); | |
3665 | } | |
3666 | fprintf (sched_dump, "\n\n"); | |
3667 | } | |
3668 | ||
3669 | /* Initialize the SCHED_PRESSURE_MODEL information for the current | |
3670 | scheduling region. */ | |
3671 | ||
3672 | static void | |
3673 | model_start_schedule (void) | |
3674 | { | |
3675 | basic_block bb; | |
3676 | ||
3677 | model_next_priority = 1; | |
f1f41a6c | 3678 | model_schedule.create (sched_max_luid); |
b30b031c | 3679 | model_insns = XCNEWVEC (struct model_insn_info, sched_max_luid); |
3680 | ||
3681 | bb = BLOCK_FOR_INSN (NEXT_INSN (current_sched_info->prev_head)); | |
3682 | initiate_reg_pressure_info (df_get_live_in (bb)); | |
3683 | ||
3684 | model_analyze_insns (); | |
3685 | model_init_pressure_group (&model_before_pressure); | |
3686 | while (model_worklist) | |
3687 | model_choose_insn (); | |
f1f41a6c | 3688 | gcc_assert (model_num_insns == (int) model_schedule.length ()); |
b30b031c | 3689 | if (sched_verbose >= 2) |
3690 | fprintf (sched_dump, "\n"); | |
3691 | ||
3692 | model_record_final_pressures (&model_before_pressure); | |
3693 | model_reset_queue_indices (); | |
3694 | ||
3695 | XDELETEVEC (model_insns); | |
3696 | ||
3697 | model_curr_point = 0; | |
3698 | initiate_reg_pressure_info (df_get_live_in (bb)); | |
3699 | if (sched_verbose >= 1) | |
3700 | model_dump_pressure_summary (); | |
3701 | } | |
3702 | ||
3703 | /* Free the information associated with GROUP. */ | |
3704 | ||
3705 | static void | |
3706 | model_finalize_pressure_group (struct model_pressure_group *group) | |
3707 | { | |
3708 | XDELETEVEC (group->model); | |
3709 | } | |
3710 | ||
3711 | /* Free the information created by model_start_schedule. */ | |
3712 | ||
3713 | static void | |
3714 | model_end_schedule (void) | |
3715 | { | |
3716 | model_finalize_pressure_group (&model_before_pressure); | |
f1f41a6c | 3717 | model_schedule.release (); |
b30b031c | 3718 | } |
3719 | \f | |
0c775e95 | 3720 | /* A structure that holds local state for the loop in schedule_block. */ |
3721 | struct sched_block_state | |
3722 | { | |
3723 | /* True if no real insns have been scheduled in the current cycle. */ | |
3724 | bool first_cycle_insn_p; | |
e2f4a6ff | 3725 | /* True if a shadow insn has been scheduled in the current cycle, which |
3726 | means that no more normal insns can be issued. */ | |
3727 | bool shadows_only_p; | |
c0457559 | 3728 | /* True if we're winding down a modulo schedule, which means that we only |
3729 | issue insns with INSN_EXACT_TICK set. */ | |
3730 | bool modulo_epilogue; | |
0c775e95 | 3731 | /* Initialized with the machine's issue rate every cycle, and updated |
3732 | by calls to the variable_issue hook. */ | |
3733 | int can_issue_more; | |
3734 | }; | |
a7dcf969 | 3735 | |
3eb9a99d | 3736 | /* INSN is the "currently executing insn". Launch each insn which was |
30b1ec30 | 3737 | waiting on INSN. READY is the ready list which contains the insns |
e26579fc | 3738 | that are ready to fire. CLOCK is the current cycle. The function |
3739 | returns necessary cycle advance after issuing the insn (it is not | |
3740 | zero for insns in a schedule group). */ | |
3eb9a99d | 3741 | |
e26579fc | 3742 | static int |
e4897000 | 3743 | schedule_insn (rtx insn) |
3eb9a99d | 3744 | { |
93f6b030 | 3745 | sd_iterator_def sd_it; |
3746 | dep_t dep; | |
a7dcf969 | 3747 | int i; |
e26579fc | 3748 | int advance = 0; |
3eb9a99d | 3749 | |
67900a4f | 3750 | if (sched_verbose >= 1) |
3eb9a99d | 3751 | { |
a7dcf969 | 3752 | struct reg_pressure_data *pressure_info; |
6dde9719 | 3753 | fprintf (sched_dump, ";;\t%3i--> %s%-40s:", |
3d9a4504 | 3754 | clock_var, (*current_sched_info->print_insn) (insn, 1), |
3755 | str_pattern_slim (PATTERN (insn))); | |
bea4bad2 | 3756 | |
0bc18d66 | 3757 | if (recog_memoized (insn) < 0) |
3758 | fprintf (sched_dump, "nothing"); | |
3759 | else | |
3760 | print_reservation (sched_dump, insn); | |
a7dcf969 | 3761 | pressure_info = INSN_REG_PRESSURE (insn); |
3762 | if (pressure_info != NULL) | |
3763 | { | |
3764 | fputc (':', sched_dump); | |
66d9a7b9 | 3765 | for (i = 0; i < ira_pressure_classes_num; i++) |
57a8bf1b | 3766 | fprintf (sched_dump, "%s%s%+d(%d)", |
3767 | scheduled_insns.length () > 1 | |
3768 | && INSN_LUID (insn) | |
3769 | < INSN_LUID (scheduled_insns[scheduled_insns.length () - 2]) ? "@" : "", | |
66d9a7b9 | 3770 | reg_class_names[ira_pressure_classes[i]], |
a7dcf969 | 3771 | pressure_info[i].set_increase, pressure_info[i].change); |
3772 | } | |
b30b031c | 3773 | if (sched_pressure == SCHED_PRESSURE_MODEL |
3774 | && model_curr_point < model_num_insns | |
3775 | && model_index (insn) == model_curr_point) | |
3776 | fprintf (sched_dump, ":model %d", model_curr_point); | |
0bc18d66 | 3777 | fputc ('\n', sched_dump); |
3778 | } | |
3eb9a99d | 3779 | |
11189c7a | 3780 | if (sched_pressure == SCHED_PRESSURE_WEIGHTED && !DEBUG_INSN_P (insn)) |
a7dcf969 | 3781 | update_reg_and_insn_max_reg_pressure (insn); |
3782 | ||
e4897000 | 3783 | /* Scheduling instruction should have all its dependencies resolved and |
3784 | should have been removed from the ready list. */ | |
effd1640 | 3785 | gcc_assert (sd_lists_empty_p (insn, SD_LIST_HARD_BACK)); |
52c4b43f | 3786 | |
c971b445 | 3787 | /* Reset debug insns invalidated by moving this insn. */ |
3788 | if (MAY_HAVE_DEBUG_INSNS && !DEBUG_INSN_P (insn)) | |
3789 | for (sd_it = sd_iterator_start (insn, SD_LIST_BACK); | |
3790 | sd_iterator_cond (&sd_it, &dep);) | |
3791 | { | |
3792 | rtx dbg = DEP_PRO (dep); | |
f145fac4 | 3793 | struct reg_use_data *use, *next; |
c971b445 | 3794 | |
effd1640 | 3795 | if (DEP_STATUS (dep) & DEP_CANCELLED) |
3796 | { | |
3797 | sd_iterator_next (&sd_it); | |
3798 | continue; | |
3799 | } | |
3800 | ||
c971b445 | 3801 | gcc_assert (DEBUG_INSN_P (dbg)); |
3802 | ||
3803 | if (sched_verbose >= 6) | |
3804 | fprintf (sched_dump, ";;\t\tresetting: debug insn %d\n", | |
3805 | INSN_UID (dbg)); | |
3806 | ||
3807 | /* ??? Rather than resetting the debug insn, we might be able | |
3808 | to emit a debug temp before the just-scheduled insn, but | |
3809 | this would involve checking that the expression at the | |
3810 | point of the debug insn is equivalent to the expression | |
3811 | before the just-scheduled insn. They might not be: the | |
3812 | expression in the debug insn may depend on other insns not | |
3813 | yet scheduled that set MEMs, REGs or even other debug | |
3814 | insns. It's not clear that attempting to preserve debug | |
3815 | information in these cases is worth the effort, given how | |
3816 | uncommon these resets are and the likelihood that the debug | |
3817 | temps introduced won't survive the schedule change. */ | |
3818 | INSN_VAR_LOCATION_LOC (dbg) = gen_rtx_UNKNOWN_VAR_LOC (); | |
3819 | df_insn_rescan (dbg); | |
3820 | ||
f145fac4 | 3821 | /* Unknown location doesn't use any registers. */ |
3822 | for (use = INSN_REG_USE_LIST (dbg); use != NULL; use = next) | |
3823 | { | |
47086a61 | 3824 | struct reg_use_data *prev = use; |
3825 | ||
3826 | /* Remove use from the cyclic next_regno_use chain first. */ | |
3827 | while (prev->next_regno_use != use) | |
3828 | prev = prev->next_regno_use; | |
3829 | prev->next_regno_use = use->next_regno_use; | |
f145fac4 | 3830 | next = use->next_insn_use; |
3831 | free (use); | |
3832 | } | |
3833 | INSN_REG_USE_LIST (dbg) = NULL; | |
3834 | ||
c971b445 | 3835 | /* We delete rather than resolve these deps, otherwise we |
3836 | crash in sched_free_deps(), because forward deps are | |
3837 | expected to be released before backward deps. */ | |
3838 | sd_delete_dep (sd_it); | |
3839 | } | |
3840 | ||
9997bd27 | 3841 | gcc_assert (QUEUE_INDEX (insn) == QUEUE_NOWHERE); |
e4897000 | 3842 | QUEUE_INDEX (insn) = QUEUE_SCHEDULED; |
9997bd27 | 3843 | |
b30b031c | 3844 | if (sched_pressure == SCHED_PRESSURE_MODEL |
3845 | && model_curr_point < model_num_insns | |
3846 | && NONDEBUG_INSN_P (insn)) | |
3847 | { | |
3848 | if (model_index (insn) == model_curr_point) | |
3849 | do | |
3850 | model_curr_point++; | |
3851 | while (model_curr_point < model_num_insns | |
3852 | && (QUEUE_INDEX (MODEL_INSN (model_curr_point)) | |
3853 | == QUEUE_SCHEDULED)); | |
3854 | else | |
3855 | model_recompute (insn); | |
3856 | model_update_limit_points (); | |
3857 | update_register_pressure (insn); | |
3858 | if (sched_verbose >= 2) | |
3859 | print_curr_reg_pressure (); | |
3860 | } | |
3861 | ||
e4897000 | 3862 | gcc_assert (INSN_TICK (insn) >= MIN_TICK); |
3863 | if (INSN_TICK (insn) > clock_var) | |
3864 | /* INSN has been prematurely moved from the queue to the ready list. | |
3865 | This is possible only if following flag is set. */ | |
48e1416a | 3866 | gcc_assert (flag_sched_stalled_insns); |
e4897000 | 3867 | |
3868 | /* ??? Probably, if INSN is scheduled prematurely, we should leave | |
3869 | INSN_TICK untouched. This is a machine-dependent issue, actually. */ | |
3870 | INSN_TICK (insn) = clock_var; | |
3871 | ||
effd1640 | 3872 | check_clobbered_conditions (insn); |
3873 | ||
d452a169 | 3874 | /* Update dependent instructions. First, see if by scheduling this insn |
3875 | now we broke a dependence in a way that requires us to change another | |
3876 | insn. */ | |
3877 | for (sd_it = sd_iterator_start (insn, SD_LIST_SPEC_BACK); | |
3878 | sd_iterator_cond (&sd_it, &dep); sd_iterator_next (&sd_it)) | |
3879 | { | |
3880 | struct dep_replacement *desc = DEP_REPLACE (dep); | |
3881 | rtx pro = DEP_PRO (dep); | |
3882 | if (QUEUE_INDEX (pro) != QUEUE_SCHEDULED | |
3883 | && desc != NULL && desc->insn == pro) | |
3884 | apply_replacement (dep, false); | |
3885 | } | |
3886 | ||
3887 | /* Go through and resolve forward dependencies. */ | |
93f6b030 | 3888 | for (sd_it = sd_iterator_start (insn, SD_LIST_FORW); |
3889 | sd_iterator_cond (&sd_it, &dep);) | |
3eb9a99d | 3890 | { |
93f6b030 | 3891 | rtx next = DEP_CON (dep); |
effd1640 | 3892 | bool cancelled = (DEP_STATUS (dep) & DEP_CANCELLED) != 0; |
3eb9a99d | 3893 | |
93f6b030 | 3894 | /* Resolve the dependence between INSN and NEXT. |
3895 | sd_resolve_dep () moves current dep to another list thus | |
3896 | advancing the iterator. */ | |
3897 | sd_resolve_dep (sd_it); | |
3eb9a99d | 3898 | |
effd1640 | 3899 | if (cancelled) |
3900 | { | |
d452a169 | 3901 | if (must_restore_pattern_p (next, dep)) |
3902 | restore_pattern (dep, false); | |
effd1640 | 3903 | continue; |
3904 | } | |
3905 | ||
c971b445 | 3906 | /* Don't bother trying to mark next as ready if insn is a debug |
3907 | insn. If insn is the last hard dependency, it will have | |
3908 | already been discounted. */ | |
3909 | if (DEBUG_INSN_P (insn) && !DEBUG_INSN_P (next)) | |
3910 | continue; | |
3911 | ||
fd27912f | 3912 | if (!IS_SPECULATION_BRANCHY_CHECK_P (insn)) |
6a1cdb4d | 3913 | { |
48e1416a | 3914 | int effective_cost; |
3915 | ||
6a1cdb4d | 3916 | effective_cost = try_ready (next); |
48e1416a | 3917 | |
6a1cdb4d | 3918 | if (effective_cost >= 0 |
3919 | && SCHED_GROUP_P (next) | |
3920 | && advance < effective_cost) | |
3921 | advance = effective_cost; | |
3922 | } | |
3923 | else | |
3924 | /* Check always has only one forward dependence (to the first insn in | |
3925 | the recovery block), therefore, this will be executed only once. */ | |
3926 | { | |
93f6b030 | 3927 | gcc_assert (sd_lists_empty_p (insn, SD_LIST_FORW)); |
6a1cdb4d | 3928 | fix_recovery_deps (RECOVERY_BLOCK (insn)); |
3929 | } | |
3eb9a99d | 3930 | } |
3931 | ||
896c2bfe | 3932 | /* Annotate the instruction with issue information -- TImode |
8da6595b | 3933 | indicates that the instruction is expected not to be able |
3934 | to issue on the same cycle as the previous insn. A machine | |
3935 | may use this information to decide how the instruction should | |
3936 | be aligned. */ | |
58ada791 | 3937 | if (issue_rate > 1 |
bea4bad2 | 3938 | && GET_CODE (PATTERN (insn)) != USE |
9845d120 | 3939 | && GET_CODE (PATTERN (insn)) != CLOBBER |
3940 | && !DEBUG_INSN_P (insn)) | |
8da6595b | 3941 | { |
58ada791 | 3942 | if (reload_completed) |
e4897000 | 3943 | PUT_MODE (insn, clock_var > last_clock_var ? TImode : VOIDmode); |
3944 | last_clock_var = clock_var; | |
8da6595b | 3945 | } |
e4897000 | 3946 | |
e26579fc | 3947 | return advance; |
3eb9a99d | 3948 | } |
3949 | ||
c4cd519a | 3950 | /* Functions for handling of notes. */ |
3eb9a99d | 3951 | |
e1ab7874 | 3952 | /* Add note list that ends on FROM_END to the end of TO_ENDP. */ |
3953 | void | |
3954 | concat_note_lists (rtx from_end, rtx *to_endp) | |
3955 | { | |
3956 | rtx from_start; | |
3957 | ||
15a03e56 | 3958 | /* It's easy when have nothing to concat. */ |
e1ab7874 | 3959 | if (from_end == NULL) |
e1ab7874 | 3960 | return; |
3961 | ||
15a03e56 | 3962 | /* It's also easy when destination is empty. */ |
e1ab7874 | 3963 | if (*to_endp == NULL) |
e1ab7874 | 3964 | { |
3965 | *to_endp = from_end; | |
3966 | return; | |
3967 | } | |
3968 | ||
3969 | from_start = from_end; | |
48e1416a | 3970 | while (PREV_INSN (from_start) != NULL) |
e1ab7874 | 3971 | from_start = PREV_INSN (from_start); |
3972 | ||
15a03e56 | 3973 | PREV_INSN (from_start) = *to_endp; |
3974 | NEXT_INSN (*to_endp) = from_start; | |
e1ab7874 | 3975 | *to_endp = from_end; |
3976 | } | |
3977 | ||
15a03e56 | 3978 | /* Delete notes between HEAD and TAIL and put them in the chain |
3979 | of notes ended by NOTE_LIST. */ | |
3980 | void | |
3981 | remove_notes (rtx head, rtx tail) | |
3eb9a99d | 3982 | { |
57ab8ec3 | 3983 | rtx next_tail, insn, next; |
3eb9a99d | 3984 | |
15a03e56 | 3985 | note_list = 0; |
3986 | if (head == tail && !INSN_P (head)) | |
3987 | return; | |
3988 | ||
3989 | next_tail = NEXT_INSN (tail); | |
15a03e56 | 3990 | for (insn = head; insn != next_tail; insn = next) |
3eb9a99d | 3991 | { |
15a03e56 | 3992 | next = NEXT_INSN (insn); |
3993 | if (!NOTE_P (insn)) | |
57ab8ec3 | 3994 | continue; |
792f1838 | 3995 | |
15a03e56 | 3996 | switch (NOTE_KIND (insn)) |
3997 | { | |
3998 | case NOTE_INSN_BASIC_BLOCK: | |
15a03e56 | 3999 | continue; |
3eb9a99d | 4000 | |
15a03e56 | 4001 | case NOTE_INSN_EPILOGUE_BEG: |
4002 | if (insn != tail) | |
4003 | { | |
4004 | remove_insn (insn); | |
4005 | add_reg_note (next, REG_SAVE_NOTE, | |
4006 | GEN_INT (NOTE_INSN_EPILOGUE_BEG)); | |
4007 | break; | |
4008 | } | |
4009 | /* FALLTHRU */ | |
e1ab7874 | 4010 | |
15a03e56 | 4011 | default: |
4012 | remove_insn (insn); | |
e1ab7874 | 4013 | |
15a03e56 | 4014 | /* Add the note to list that ends at NOTE_LIST. */ |
4015 | PREV_INSN (insn) = note_list; | |
4016 | NEXT_INSN (insn) = NULL_RTX; | |
4017 | if (note_list) | |
4018 | NEXT_INSN (note_list) = insn; | |
4019 | note_list = insn; | |
4020 | break; | |
4021 | } | |
4022 | ||
4023 | gcc_assert ((sel_sched_p () || insn != tail) && insn != head); | |
4024 | } | |
3eb9a99d | 4025 | } |
4026 | ||
e2f4a6ff | 4027 | /* A structure to record enough data to allow us to backtrack the scheduler to |
4028 | a previous state. */ | |
4029 | struct haifa_saved_data | |
4030 | { | |
4031 | /* Next entry on the list. */ | |
4032 | struct haifa_saved_data *next; | |
4033 | ||
4034 | /* Backtracking is associated with scheduling insns that have delay slots. | |
4035 | DELAY_PAIR points to the structure that contains the insns involved, and | |
4036 | the number of cycles between them. */ | |
4037 | struct delay_pair *delay_pair; | |
4038 | ||
4039 | /* Data used by the frontend (e.g. sched-ebb or sched-rgn). */ | |
4040 | void *fe_saved_data; | |
4041 | /* Data used by the backend. */ | |
4042 | void *be_saved_data; | |
4043 | ||
4044 | /* Copies of global state. */ | |
4045 | int clock_var, last_clock_var; | |
4046 | struct ready_list ready; | |
4047 | state_t curr_state; | |
4048 | ||
4049 | rtx last_scheduled_insn; | |
4050 | rtx last_nondebug_scheduled_insn; | |
4051 | int cycle_issued_insns; | |
4052 | ||
4053 | /* Copies of state used in the inner loop of schedule_block. */ | |
4054 | struct sched_block_state sched_block; | |
4055 | ||
4056 | /* We don't need to save q_ptr, as its value is arbitrary and we can set it | |
4057 | to 0 when restoring. */ | |
4058 | int q_size; | |
4059 | rtx *insn_queue; | |
d452a169 | 4060 | |
4061 | /* Describe pattern replacements that occurred since this backtrack point | |
4062 | was queued. */ | |
f1f41a6c | 4063 | vec<dep_t> replacement_deps; |
4064 | vec<int> replace_apply; | |
d452a169 | 4065 | |
4066 | /* A copy of the next-cycle replacement vectors at the time of the backtrack | |
4067 | point. */ | |
f1f41a6c | 4068 | vec<dep_t> next_cycle_deps; |
4069 | vec<int> next_cycle_apply; | |
e2f4a6ff | 4070 | }; |
4071 | ||
4072 | /* A record, in reverse order, of all scheduled insns which have delay slots | |
4073 | and may require backtracking. */ | |
4074 | static struct haifa_saved_data *backtrack_queue; | |
4075 | ||
4076 | /* For every dependency of INSN, set the FEEDS_BACKTRACK_INSN bit according | |
4077 | to SET_P. */ | |
4078 | static void | |
4079 | mark_backtrack_feeds (rtx insn, int set_p) | |
4080 | { | |
4081 | sd_iterator_def sd_it; | |
4082 | dep_t dep; | |
4083 | FOR_EACH_DEP (insn, SD_LIST_HARD_BACK, sd_it, dep) | |
4084 | { | |
4085 | FEEDS_BACKTRACK_INSN (DEP_PRO (dep)) = set_p; | |
4086 | } | |
4087 | } | |
4088 | ||
e2f4a6ff | 4089 | /* Save the current scheduler state so that we can backtrack to it |
4090 | later if necessary. PAIR gives the insns that make it necessary to | |
4091 | save this point. SCHED_BLOCK is the local state of schedule_block | |
4092 | that need to be saved. */ | |
4093 | static void | |
4094 | save_backtrack_point (struct delay_pair *pair, | |
4095 | struct sched_block_state sched_block) | |
4096 | { | |
4097 | int i; | |
4098 | struct haifa_saved_data *save = XNEW (struct haifa_saved_data); | |
4099 | ||
4100 | save->curr_state = xmalloc (dfa_state_size); | |
4101 | memcpy (save->curr_state, curr_state, dfa_state_size); | |
4102 | ||
4103 | save->ready.first = ready.first; | |
4104 | save->ready.n_ready = ready.n_ready; | |
4105 | save->ready.n_debug = ready.n_debug; | |
4106 | save->ready.veclen = ready.veclen; | |
4107 | save->ready.vec = XNEWVEC (rtx, ready.veclen); | |
4108 | memcpy (save->ready.vec, ready.vec, ready.veclen * sizeof (rtx)); | |
4109 | ||
4110 | save->insn_queue = XNEWVEC (rtx, max_insn_queue_index + 1); | |
4111 | save->q_size = q_size; | |
4112 | for (i = 0; i <= max_insn_queue_index; i++) | |
4113 | { | |
4114 | int q = NEXT_Q_AFTER (q_ptr, i); | |
effd1640 | 4115 | save->insn_queue[i] = copy_INSN_LIST (insn_queue[q]); |
e2f4a6ff | 4116 | } |
4117 | ||
4118 | save->clock_var = clock_var; | |
4119 | save->last_clock_var = last_clock_var; | |
4120 | save->cycle_issued_insns = cycle_issued_insns; | |
4121 | save->last_scheduled_insn = last_scheduled_insn; | |
4122 | save->last_nondebug_scheduled_insn = last_nondebug_scheduled_insn; | |
4123 | ||
4124 | save->sched_block = sched_block; | |
4125 | ||
f1f41a6c | 4126 | save->replacement_deps.create (0); |
4127 | save->replace_apply.create (0); | |
4128 | save->next_cycle_deps = next_cycle_replace_deps.copy (); | |
4129 | save->next_cycle_apply = next_cycle_apply.copy (); | |
d452a169 | 4130 | |
e2f4a6ff | 4131 | if (current_sched_info->save_state) |
4132 | save->fe_saved_data = (*current_sched_info->save_state) (); | |
4133 | ||
4134 | if (targetm.sched.alloc_sched_context) | |
4135 | { | |
4136 | save->be_saved_data = targetm.sched.alloc_sched_context (); | |
4137 | targetm.sched.init_sched_context (save->be_saved_data, false); | |
4138 | } | |
4139 | else | |
4140 | save->be_saved_data = NULL; | |
4141 | ||
4142 | save->delay_pair = pair; | |
4143 | ||
4144 | save->next = backtrack_queue; | |
4145 | backtrack_queue = save; | |
4146 | ||
4147 | while (pair) | |
4148 | { | |
4149 | mark_backtrack_feeds (pair->i2, 1); | |
4150 | INSN_TICK (pair->i2) = INVALID_TICK; | |
4151 | INSN_EXACT_TICK (pair->i2) = clock_var + pair_delay (pair); | |
c0457559 | 4152 | SHADOW_P (pair->i2) = pair->stages == 0; |
e2f4a6ff | 4153 | pair = pair->next_same_i1; |
4154 | } | |
4155 | } | |
4156 | ||
effd1640 | 4157 | /* Walk the ready list and all queues. If any insns have unresolved backwards |
4158 | dependencies, these must be cancelled deps, broken by predication. Set or | |
4159 | clear (depending on SET) the DEP_CANCELLED bit in DEP_STATUS. */ | |
4160 | ||
4161 | static void | |
4162 | toggle_cancelled_flags (bool set) | |
4163 | { | |
4164 | int i; | |
4165 | sd_iterator_def sd_it; | |
4166 | dep_t dep; | |
4167 | ||
4168 | if (ready.n_ready > 0) | |
4169 | { | |
4170 | rtx *first = ready_lastpos (&ready); | |
4171 | for (i = 0; i < ready.n_ready; i++) | |
4172 | FOR_EACH_DEP (first[i], SD_LIST_BACK, sd_it, dep) | |
4173 | if (!DEBUG_INSN_P (DEP_PRO (dep))) | |
4174 | { | |
4175 | if (set) | |
4176 | DEP_STATUS (dep) |= DEP_CANCELLED; | |
4177 | else | |
4178 | DEP_STATUS (dep) &= ~DEP_CANCELLED; | |
4179 | } | |
4180 | } | |
4181 | for (i = 0; i <= max_insn_queue_index; i++) | |
4182 | { | |
4183 | int q = NEXT_Q_AFTER (q_ptr, i); | |
4184 | rtx link; | |
4185 | for (link = insn_queue[q]; link; link = XEXP (link, 1)) | |
4186 | { | |
4187 | rtx insn = XEXP (link, 0); | |
4188 | FOR_EACH_DEP (insn, SD_LIST_BACK, sd_it, dep) | |
4189 | if (!DEBUG_INSN_P (DEP_PRO (dep))) | |
4190 | { | |
4191 | if (set) | |
4192 | DEP_STATUS (dep) |= DEP_CANCELLED; | |
4193 | else | |
4194 | DEP_STATUS (dep) &= ~DEP_CANCELLED; | |
4195 | } | |
4196 | } | |
4197 | } | |
4198 | } | |
4199 | ||
d452a169 | 4200 | /* Undo the replacements that have occurred after backtrack point SAVE |
4201 | was placed. */ | |
4202 | static void | |
4203 | undo_replacements_for_backtrack (struct haifa_saved_data *save) | |
4204 | { | |
f1f41a6c | 4205 | while (!save->replacement_deps.is_empty ()) |
d452a169 | 4206 | { |
f1f41a6c | 4207 | dep_t dep = save->replacement_deps.pop (); |
4208 | int apply_p = save->replace_apply.pop (); | |
d452a169 | 4209 | |
4210 | if (apply_p) | |
4211 | restore_pattern (dep, true); | |
4212 | else | |
4213 | apply_replacement (dep, true); | |
4214 | } | |
f1f41a6c | 4215 | save->replacement_deps.release (); |
4216 | save->replace_apply.release (); | |
d452a169 | 4217 | } |
4218 | ||
e2f4a6ff | 4219 | /* Pop entries from the SCHEDULED_INSNS vector up to and including INSN. |
4220 | Restore their dependencies to an unresolved state, and mark them as | |
4221 | queued nowhere. */ | |
4222 | ||
4223 | static void | |
4224 | unschedule_insns_until (rtx insn) | |
4225 | { | |
c2078b80 | 4226 | auto_vec<rtx> recompute_vec; |
effd1640 | 4227 | |
4228 | /* Make two passes over the insns to be unscheduled. First, we clear out | |
4229 | dependencies and other trivial bookkeeping. */ | |
e2f4a6ff | 4230 | for (;;) |
4231 | { | |
4232 | rtx last; | |
4233 | sd_iterator_def sd_it; | |
4234 | dep_t dep; | |
4235 | ||
f1f41a6c | 4236 | last = scheduled_insns.pop (); |
e2f4a6ff | 4237 | |
4238 | /* This will be changed by restore_backtrack_point if the insn is in | |
4239 | any queue. */ | |
4240 | QUEUE_INDEX (last) = QUEUE_NOWHERE; | |
4241 | if (last != insn) | |
4242 | INSN_TICK (last) = INVALID_TICK; | |
4243 | ||
c0457559 | 4244 | if (modulo_ii > 0 && INSN_UID (last) < modulo_iter0_max_uid) |
4245 | modulo_insns_scheduled--; | |
4246 | ||
e2f4a6ff | 4247 | for (sd_it = sd_iterator_start (last, SD_LIST_RES_FORW); |
4248 | sd_iterator_cond (&sd_it, &dep);) | |
4249 | { | |
4250 | rtx con = DEP_CON (dep); | |
e2f4a6ff | 4251 | sd_unresolve_dep (sd_it); |
effd1640 | 4252 | if (!MUST_RECOMPUTE_SPEC_P (con)) |
4253 | { | |
4254 | MUST_RECOMPUTE_SPEC_P (con) = 1; | |
f1f41a6c | 4255 | recompute_vec.safe_push (con); |
effd1640 | 4256 | } |
e2f4a6ff | 4257 | } |
4258 | ||
4259 | if (last == insn) | |
4260 | break; | |
4261 | } | |
effd1640 | 4262 | |
4263 | /* A second pass, to update ready and speculation status for insns | |
4264 | depending on the unscheduled ones. The first pass must have | |
4265 | popped the scheduled_insns vector up to the point where we | |
4266 | restart scheduling, as recompute_todo_spec requires it to be | |
4267 | up-to-date. */ | |
f1f41a6c | 4268 | while (!recompute_vec.is_empty ()) |
effd1640 | 4269 | { |
4270 | rtx con; | |
4271 | ||
f1f41a6c | 4272 | con = recompute_vec.pop (); |
effd1640 | 4273 | MUST_RECOMPUTE_SPEC_P (con) = 0; |
4274 | if (!sd_lists_empty_p (con, SD_LIST_HARD_BACK)) | |
4275 | { | |
4276 | TODO_SPEC (con) = HARD_DEP; | |
4277 | INSN_TICK (con) = INVALID_TICK; | |
4278 | if (PREDICATED_PAT (con) != NULL_RTX) | |
4279 | haifa_change_pattern (con, ORIG_PAT (con)); | |
4280 | } | |
4281 | else if (QUEUE_INDEX (con) != QUEUE_SCHEDULED) | |
d452a169 | 4282 | TODO_SPEC (con) = recompute_todo_spec (con, true); |
effd1640 | 4283 | } |
e2f4a6ff | 4284 | } |
4285 | ||
4286 | /* Restore scheduler state from the topmost entry on the backtracking queue. | |
4287 | PSCHED_BLOCK_P points to the local data of schedule_block that we must | |
4288 | overwrite with the saved data. | |
4289 | The caller must already have called unschedule_insns_until. */ | |
4290 | ||
4291 | static void | |
4292 | restore_last_backtrack_point (struct sched_block_state *psched_block) | |
e2f4a6ff | 4293 | { |
4294 | rtx link; | |
4295 | int i; | |
4296 | struct haifa_saved_data *save = backtrack_queue; | |
4297 | ||
4298 | backtrack_queue = save->next; | |
4299 | ||
4300 | if (current_sched_info->restore_state) | |
4301 | (*current_sched_info->restore_state) (save->fe_saved_data); | |
4302 | ||
4303 | if (targetm.sched.alloc_sched_context) | |
4304 | { | |
4305 | targetm.sched.set_sched_context (save->be_saved_data); | |
4306 | targetm.sched.free_sched_context (save->be_saved_data); | |
4307 | } | |
4308 | ||
d452a169 | 4309 | /* Do this first since it clobbers INSN_TICK of the involved |
4310 | instructions. */ | |
4311 | undo_replacements_for_backtrack (save); | |
4312 | ||
e2f4a6ff | 4313 | /* Clear the QUEUE_INDEX of everything in the ready list or one |
4314 | of the queues. */ | |
4315 | if (ready.n_ready > 0) | |
4316 | { | |
4317 | rtx *first = ready_lastpos (&ready); | |
4318 | for (i = 0; i < ready.n_ready; i++) | |
4319 | { | |
effd1640 | 4320 | rtx insn = first[i]; |
4321 | QUEUE_INDEX (insn) = QUEUE_NOWHERE; | |
4322 | INSN_TICK (insn) = INVALID_TICK; | |
e2f4a6ff | 4323 | } |
4324 | } | |
4325 | for (i = 0; i <= max_insn_queue_index; i++) | |
4326 | { | |
4327 | int q = NEXT_Q_AFTER (q_ptr, i); | |
4328 | ||
4329 | for (link = insn_queue[q]; link; link = XEXP (link, 1)) | |
4330 | { | |
4331 | rtx x = XEXP (link, 0); | |
4332 | QUEUE_INDEX (x) = QUEUE_NOWHERE; | |
4333 | INSN_TICK (x) = INVALID_TICK; | |
4334 | } | |
4335 | free_INSN_LIST_list (&insn_queue[q]); | |
4336 | } | |
4337 | ||
4338 | free (ready.vec); | |
4339 | ready = save->ready; | |
4340 | ||
4341 | if (ready.n_ready > 0) | |
4342 | { | |
4343 | rtx *first = ready_lastpos (&ready); | |
4344 | for (i = 0; i < ready.n_ready; i++) | |
4345 | { | |
effd1640 | 4346 | rtx insn = first[i]; |
4347 | QUEUE_INDEX (insn) = QUEUE_READY; | |
d452a169 | 4348 | TODO_SPEC (insn) = recompute_todo_spec (insn, true); |
effd1640 | 4349 | INSN_TICK (insn) = save->clock_var; |
e2f4a6ff | 4350 | } |
4351 | } | |
4352 | ||
4353 | q_ptr = 0; | |
4354 | q_size = save->q_size; | |
4355 | for (i = 0; i <= max_insn_queue_index; i++) | |
4356 | { | |
4357 | int q = NEXT_Q_AFTER (q_ptr, i); | |
4358 | ||
4359 | insn_queue[q] = save->insn_queue[q]; | |
4360 | ||
4361 | for (link = insn_queue[q]; link; link = XEXP (link, 1)) | |
4362 | { | |
4363 | rtx x = XEXP (link, 0); | |
4364 | QUEUE_INDEX (x) = i; | |
d452a169 | 4365 | TODO_SPEC (x) = recompute_todo_spec (x, true); |
e2f4a6ff | 4366 | INSN_TICK (x) = save->clock_var + i; |
4367 | } | |
4368 | } | |
4369 | free (save->insn_queue); | |
4370 | ||
effd1640 | 4371 | toggle_cancelled_flags (true); |
4372 | ||
e2f4a6ff | 4373 | clock_var = save->clock_var; |
4374 | last_clock_var = save->last_clock_var; | |
4375 | cycle_issued_insns = save->cycle_issued_insns; | |
4376 | last_scheduled_insn = save->last_scheduled_insn; | |
4377 | last_nondebug_scheduled_insn = save->last_nondebug_scheduled_insn; | |
4378 | ||
4379 | *psched_block = save->sched_block; | |
4380 | ||
4381 | memcpy (curr_state, save->curr_state, dfa_state_size); | |
4382 | free (save->curr_state); | |
4383 | ||
4384 | mark_backtrack_feeds (save->delay_pair->i2, 0); | |
4385 | ||
f1f41a6c | 4386 | gcc_assert (next_cycle_replace_deps.is_empty ()); |
4387 | next_cycle_replace_deps = save->next_cycle_deps.copy (); | |
4388 | next_cycle_apply = save->next_cycle_apply.copy (); | |
d452a169 | 4389 | |
e2f4a6ff | 4390 | free (save); |
4391 | ||
4392 | for (save = backtrack_queue; save; save = save->next) | |
4393 | { | |
4394 | mark_backtrack_feeds (save->delay_pair->i2, 1); | |
4395 | } | |
4396 | } | |
4397 | ||
4398 | /* Discard all data associated with the topmost entry in the backtrack | |
4399 | queue. If RESET_TICK is false, we just want to free the data. If true, | |
4400 | we are doing this because we discovered a reason to backtrack. In the | |
4401 | latter case, also reset the INSN_TICK for the shadow insn. */ | |
4402 | static void | |
4403 | free_topmost_backtrack_point (bool reset_tick) | |
4404 | { | |
4405 | struct haifa_saved_data *save = backtrack_queue; | |
4406 | int i; | |
4407 | ||
4408 | backtrack_queue = save->next; | |
4409 | ||
4410 | if (reset_tick) | |
4411 | { | |
4412 | struct delay_pair *pair = save->delay_pair; | |
4413 | while (pair) | |
4414 | { | |
4415 | INSN_TICK (pair->i2) = INVALID_TICK; | |
4416 | INSN_EXACT_TICK (pair->i2) = INVALID_TICK; | |
4417 | pair = pair->next_same_i1; | |
4418 | } | |
d452a169 | 4419 | undo_replacements_for_backtrack (save); |
4420 | } | |
4421 | else | |
4422 | { | |
f1f41a6c | 4423 | save->replacement_deps.release (); |
4424 | save->replace_apply.release (); | |
e2f4a6ff | 4425 | } |
d452a169 | 4426 | |
e2f4a6ff | 4427 | if (targetm.sched.free_sched_context) |
4428 | targetm.sched.free_sched_context (save->be_saved_data); | |
4429 | if (current_sched_info->restore_state) | |
4430 | free (save->fe_saved_data); | |
4431 | for (i = 0; i <= max_insn_queue_index; i++) | |
4432 | free_INSN_LIST_list (&save->insn_queue[i]); | |
4433 | free (save->insn_queue); | |
4434 | free (save->curr_state); | |
4435 | free (save->ready.vec); | |
4436 | free (save); | |
4437 | } | |
4438 | ||
4439 | /* Free the entire backtrack queue. */ | |
4440 | static void | |
4441 | free_backtrack_queue (void) | |
4442 | { | |
4443 | while (backtrack_queue) | |
4444 | free_topmost_backtrack_point (false); | |
4445 | } | |
4446 | ||
d452a169 | 4447 | /* Apply a replacement described by DESC. If IMMEDIATELY is false, we |
4448 | may have to postpone the replacement until the start of the next cycle, | |
4449 | at which point we will be called again with IMMEDIATELY true. This is | |
4450 | only done for machines which have instruction packets with explicit | |
4451 | parallelism however. */ | |
4452 | static void | |
4453 | apply_replacement (dep_t dep, bool immediately) | |
4454 | { | |
4455 | struct dep_replacement *desc = DEP_REPLACE (dep); | |
4456 | if (!immediately && targetm.sched.exposed_pipeline && reload_completed) | |
4457 | { | |
f1f41a6c | 4458 | next_cycle_replace_deps.safe_push (dep); |
4459 | next_cycle_apply.safe_push (1); | |
d452a169 | 4460 | } |
4461 | else | |
4462 | { | |
4463 | bool success; | |
4464 | ||
4465 | if (QUEUE_INDEX (desc->insn) == QUEUE_SCHEDULED) | |
4466 | return; | |
4467 | ||
4468 | if (sched_verbose >= 5) | |
4469 | fprintf (sched_dump, "applying replacement for insn %d\n", | |
4470 | INSN_UID (desc->insn)); | |
4471 | ||
4472 | success = validate_change (desc->insn, desc->loc, desc->newval, 0); | |
4473 | gcc_assert (success); | |
4474 | ||
4475 | update_insn_after_change (desc->insn); | |
4476 | if ((TODO_SPEC (desc->insn) & (HARD_DEP | DEP_POSTPONED)) == 0) | |
4477 | fix_tick_ready (desc->insn); | |
4478 | ||
4479 | if (backtrack_queue != NULL) | |
4480 | { | |
f1f41a6c | 4481 | backtrack_queue->replacement_deps.safe_push (dep); |
4482 | backtrack_queue->replace_apply.safe_push (1); | |
d452a169 | 4483 | } |
4484 | } | |
4485 | } | |
4486 | ||
4487 | /* We have determined that a pattern involved in DEP must be restored. | |
4488 | If IMMEDIATELY is false, we may have to postpone the replacement | |
4489 | until the start of the next cycle, at which point we will be called | |
4490 | again with IMMEDIATELY true. */ | |
4491 | static void | |
4492 | restore_pattern (dep_t dep, bool immediately) | |
4493 | { | |
4494 | rtx next = DEP_CON (dep); | |
4495 | int tick = INSN_TICK (next); | |
4496 | ||
4497 | /* If we already scheduled the insn, the modified version is | |
4498 | correct. */ | |
4499 | if (QUEUE_INDEX (next) == QUEUE_SCHEDULED) | |
4500 | return; | |
4501 | ||
4502 | if (!immediately && targetm.sched.exposed_pipeline && reload_completed) | |
4503 | { | |
f1f41a6c | 4504 | next_cycle_replace_deps.safe_push (dep); |
4505 | next_cycle_apply.safe_push (0); | |
d452a169 | 4506 | return; |
4507 | } | |
4508 | ||
4509 | ||
4510 | if (DEP_TYPE (dep) == REG_DEP_CONTROL) | |
4511 | { | |
4512 | if (sched_verbose >= 5) | |
4513 | fprintf (sched_dump, "restoring pattern for insn %d\n", | |
4514 | INSN_UID (next)); | |
4515 | haifa_change_pattern (next, ORIG_PAT (next)); | |
4516 | } | |
4517 | else | |
4518 | { | |
4519 | struct dep_replacement *desc = DEP_REPLACE (dep); | |
4520 | bool success; | |
4521 | ||
4522 | if (sched_verbose >= 5) | |
4523 | fprintf (sched_dump, "restoring pattern for insn %d\n", | |
4524 | INSN_UID (desc->insn)); | |
4525 | tick = INSN_TICK (desc->insn); | |
4526 | ||
4527 | success = validate_change (desc->insn, desc->loc, desc->orig, 0); | |
4528 | gcc_assert (success); | |
4529 | update_insn_after_change (desc->insn); | |
4530 | if (backtrack_queue != NULL) | |
4531 | { | |
f1f41a6c | 4532 | backtrack_queue->replacement_deps.safe_push (dep); |
4533 | backtrack_queue->replace_apply.safe_push (0); | |
d452a169 | 4534 | } |
4535 | } | |
4536 | INSN_TICK (next) = tick; | |
4537 | if (TODO_SPEC (next) == DEP_POSTPONED) | |
4538 | return; | |
4539 | ||
4540 | if (sd_lists_empty_p (next, SD_LIST_BACK)) | |
4541 | TODO_SPEC (next) = 0; | |
4542 | else if (!sd_lists_empty_p (next, SD_LIST_HARD_BACK)) | |
4543 | TODO_SPEC (next) = HARD_DEP; | |
4544 | } | |
4545 | ||
4546 | /* Perform pattern replacements that were queued up until the next | |
4547 | cycle. */ | |
4548 | static void | |
4549 | perform_replacements_new_cycle (void) | |
4550 | { | |
4551 | int i; | |
4552 | dep_t dep; | |
f1f41a6c | 4553 | FOR_EACH_VEC_ELT (next_cycle_replace_deps, i, dep) |
d452a169 | 4554 | { |
f1f41a6c | 4555 | int apply_p = next_cycle_apply[i]; |
d452a169 | 4556 | if (apply_p) |
4557 | apply_replacement (dep, true); | |
4558 | else | |
4559 | restore_pattern (dep, true); | |
4560 | } | |
f1f41a6c | 4561 | next_cycle_replace_deps.truncate (0); |
4562 | next_cycle_apply.truncate (0); | |
d452a169 | 4563 | } |
4564 | ||
e2f4a6ff | 4565 | /* Compute INSN_TICK_ESTIMATE for INSN. PROCESSED is a bitmap of |
4566 | instructions we've previously encountered, a set bit prevents | |
4567 | recursion. BUDGET is a limit on how far ahead we look, it is | |
4568 | reduced on recursive calls. Return true if we produced a good | |
4569 | estimate, or false if we exceeded the budget. */ | |
4570 | static bool | |
4571 | estimate_insn_tick (bitmap processed, rtx insn, int budget) | |
4572 | { | |
4573 | sd_iterator_def sd_it; | |
4574 | dep_t dep; | |
4575 | int earliest = INSN_TICK (insn); | |
4576 | ||
4577 | FOR_EACH_DEP (insn, SD_LIST_BACK, sd_it, dep) | |
4578 | { | |
4579 | rtx pro = DEP_PRO (dep); | |
4580 | int t; | |
4581 | ||
effd1640 | 4582 | if (DEP_STATUS (dep) & DEP_CANCELLED) |
4583 | continue; | |
4584 | ||
e2f4a6ff | 4585 | if (QUEUE_INDEX (pro) == QUEUE_SCHEDULED) |
4586 | gcc_assert (INSN_TICK (pro) + dep_cost (dep) <= INSN_TICK (insn)); | |
4587 | else | |
4588 | { | |
4589 | int cost = dep_cost (dep); | |
4590 | if (cost >= budget) | |
4591 | return false; | |
4592 | if (!bitmap_bit_p (processed, INSN_LUID (pro))) | |
4593 | { | |
4594 | if (!estimate_insn_tick (processed, pro, budget - cost)) | |
4595 | return false; | |
4596 | } | |
4597 | gcc_assert (INSN_TICK_ESTIMATE (pro) != INVALID_TICK); | |
4598 | t = INSN_TICK_ESTIMATE (pro) + cost; | |
4599 | if (earliest == INVALID_TICK || t > earliest) | |
4600 | earliest = t; | |
4601 | } | |
4602 | } | |
4603 | bitmap_set_bit (processed, INSN_LUID (insn)); | |
4604 | INSN_TICK_ESTIMATE (insn) = earliest; | |
4605 | return true; | |
4606 | } | |
4607 | ||
4608 | /* Examine the pair of insns in P, and estimate (optimistically, assuming | |
4609 | infinite resources) the cycle in which the delayed shadow can be issued. | |
4610 | Return the number of cycles that must pass before the real insn can be | |
4611 | issued in order to meet this constraint. */ | |
4612 | static int | |
4613 | estimate_shadow_tick (struct delay_pair *p) | |
4614 | { | |
4615 | bitmap_head processed; | |
4616 | int t; | |
4617 | bool cutoff; | |
4618 | bitmap_initialize (&processed, 0); | |
4619 | ||
4620 | cutoff = !estimate_insn_tick (&processed, p->i2, | |
4621 | max_insn_queue_index + pair_delay (p)); | |
4622 | bitmap_clear (&processed); | |
4623 | if (cutoff) | |
4624 | return max_insn_queue_index; | |
4625 | t = INSN_TICK_ESTIMATE (p->i2) - (clock_var + pair_delay (p) + 1); | |
4626 | if (t > 0) | |
4627 | return t; | |
4628 | return 0; | |
4629 | } | |
15a03e56 | 4630 | |
c0457559 | 4631 | /* If INSN has no unresolved backwards dependencies, add it to the schedule and |
4632 | recursively resolve all its forward dependencies. */ | |
4633 | static void | |
4634 | resolve_dependencies (rtx insn) | |
4635 | { | |
4636 | sd_iterator_def sd_it; | |
4637 | dep_t dep; | |
4638 | ||
4639 | /* Don't use sd_lists_empty_p; it ignores debug insns. */ | |
4640 | if (DEPS_LIST_FIRST (INSN_HARD_BACK_DEPS (insn)) != NULL | |
4641 | || DEPS_LIST_FIRST (INSN_SPEC_BACK_DEPS (insn)) != NULL) | |
4642 | return; | |
4643 | ||
4644 | if (sched_verbose >= 4) | |
4645 | fprintf (sched_dump, ";;\tquickly resolving %d\n", INSN_UID (insn)); | |
4646 | ||
4647 | if (QUEUE_INDEX (insn) >= 0) | |
4648 | queue_remove (insn); | |
4649 | ||
f1f41a6c | 4650 | scheduled_insns.safe_push (insn); |
c0457559 | 4651 | |
4652 | /* Update dependent instructions. */ | |
4653 | for (sd_it = sd_iterator_start (insn, SD_LIST_FORW); | |
4654 | sd_iterator_cond (&sd_it, &dep);) | |
4655 | { | |
4656 | rtx next = DEP_CON (dep); | |
4657 | ||
4658 | if (sched_verbose >= 4) | |
4659 | fprintf (sched_dump, ";;\t\tdep %d against %d\n", INSN_UID (insn), | |
4660 | INSN_UID (next)); | |
4661 | ||
4662 | /* Resolve the dependence between INSN and NEXT. | |
4663 | sd_resolve_dep () moves current dep to another list thus | |
4664 | advancing the iterator. */ | |
4665 | sd_resolve_dep (sd_it); | |
4666 | ||
4667 | if (!IS_SPECULATION_BRANCHY_CHECK_P (insn)) | |
4668 | { | |
4669 | resolve_dependencies (next); | |
4670 | } | |
4671 | else | |
4672 | /* Check always has only one forward dependence (to the first insn in | |
4673 | the recovery block), therefore, this will be executed only once. */ | |
4674 | { | |
4675 | gcc_assert (sd_lists_empty_p (insn, SD_LIST_FORW)); | |
4676 | } | |
4677 | } | |
4678 | } | |
4679 | ||
4680 | ||
6a1cdb4d | 4681 | /* Return the head and tail pointers of ebb starting at BEG and ending |
4682 | at END. */ | |
7a31a7bd | 4683 | void |
6a1cdb4d | 4684 | get_ebb_head_tail (basic_block beg, basic_block end, rtx *headp, rtx *tailp) |
4685 | { | |
4686 | rtx beg_head = BB_HEAD (beg); | |
4687 | rtx beg_tail = BB_END (beg); | |
4688 | rtx end_head = BB_HEAD (end); | |
4689 | rtx end_tail = BB_END (end); | |
4690 | ||
4691 | /* Don't include any notes or labels at the beginning of the BEG | |
4692 | basic block, or notes at the end of the END basic blocks. */ | |
4693 | ||
4694 | if (LABEL_P (beg_head)) | |
4695 | beg_head = NEXT_INSN (beg_head); | |
4696 | ||
4697 | while (beg_head != beg_tail) | |
9221ec44 | 4698 | if (NOTE_P (beg_head)) |
6a1cdb4d | 4699 | beg_head = NEXT_INSN (beg_head); |
9221ec44 | 4700 | else if (DEBUG_INSN_P (beg_head)) |
4701 | { | |
4702 | rtx note, next; | |
4703 | ||
4704 | for (note = NEXT_INSN (beg_head); | |
4705 | note != beg_tail; | |
4706 | note = next) | |
4707 | { | |
4708 | next = NEXT_INSN (note); | |
4709 | if (NOTE_P (note)) | |
4710 | { | |
4711 | if (sched_verbose >= 9) | |
4712 | fprintf (sched_dump, "reorder %i\n", INSN_UID (note)); | |
4713 | ||
4714 | reorder_insns_nobb (note, note, PREV_INSN (beg_head)); | |
4715 | ||
4716 | if (BLOCK_FOR_INSN (note) != beg) | |
4717 | df_insn_change_bb (note, beg); | |
4718 | } | |
4719 | else if (!DEBUG_INSN_P (note)) | |
4720 | break; | |
4721 | } | |
4722 | ||
4723 | break; | |
4724 | } | |
6a1cdb4d | 4725 | else |
4726 | break; | |
4727 | ||
4728 | *headp = beg_head; | |
4729 | ||
4730 | if (beg == end) | |
4731 | end_head = beg_head; | |
4732 | else if (LABEL_P (end_head)) | |
4733 | end_head = NEXT_INSN (end_head); | |
4734 | ||
4735 | while (end_head != end_tail) | |
9221ec44 | 4736 | if (NOTE_P (end_tail)) |
6a1cdb4d | 4737 | end_tail = PREV_INSN (end_tail); |
9221ec44 | 4738 | else if (DEBUG_INSN_P (end_tail)) |
4739 | { | |
4740 | rtx note, prev; | |
4741 | ||
4742 | for (note = PREV_INSN (end_tail); | |
4743 | note != end_head; | |
4744 | note = prev) | |
4745 | { | |
4746 | prev = PREV_INSN (note); | |
4747 | if (NOTE_P (note)) | |
4748 | { | |
4749 | if (sched_verbose >= 9) | |
4750 | fprintf (sched_dump, "reorder %i\n", INSN_UID (note)); | |
4751 | ||
4752 | reorder_insns_nobb (note, note, end_tail); | |
4753 | ||
4754 | if (end_tail == BB_END (end)) | |
4755 | BB_END (end) = note; | |
4756 | ||
4757 | if (BLOCK_FOR_INSN (note) != end) | |
4758 | df_insn_change_bb (note, end); | |
4759 | } | |
4760 | else if (!DEBUG_INSN_P (note)) | |
4761 | break; | |
4762 | } | |
4763 | ||
4764 | break; | |
4765 | } | |
6a1cdb4d | 4766 | else |
4767 | break; | |
3eb9a99d | 4768 | |
6a1cdb4d | 4769 | *tailp = end_tail; |
3eb9a99d | 4770 | } |
4771 | ||
c2069298 | 4772 | /* Return nonzero if there are no real insns in the range [ HEAD, TAIL ]. */ |
4773 | ||
7a31a7bd | 4774 | int |
5493cb9a | 4775 | no_real_insns_p (const_rtx head, const_rtx tail) |
c2069298 | 4776 | { |
4777 | while (head != NEXT_INSN (tail)) | |
4778 | { | |
9221ec44 | 4779 | if (!NOTE_P (head) && !LABEL_P (head)) |
c2069298 | 4780 | return 0; |
4781 | head = NEXT_INSN (head); | |
4782 | } | |
4783 | return 1; | |
4784 | } | |
4785 | ||
e1ab7874 | 4786 | /* Restore-other-notes: NOTE_LIST is the end of a chain of notes |
4787 | previously found among the insns. Insert them just before HEAD. */ | |
4788 | rtx | |
4789 | restore_other_notes (rtx head, basic_block head_bb) | |
4790 | { | |
4791 | if (note_list != 0) | |
4792 | { | |
4793 | rtx note_head = note_list; | |
4794 | ||
4795 | if (head) | |
4796 | head_bb = BLOCK_FOR_INSN (head); | |
4797 | else | |
4798 | head = NEXT_INSN (bb_note (head_bb)); | |
4799 | ||
4800 | while (PREV_INSN (note_head)) | |
4801 | { | |
4802 | set_block_for_insn (note_head, head_bb); | |
4803 | note_head = PREV_INSN (note_head); | |
3eb9a99d | 4804 | } |
e1ab7874 | 4805 | /* In the above cycle we've missed this note. */ |
4806 | set_block_for_insn (note_head, head_bb); | |
4807 | ||
4808 | PREV_INSN (note_head) = PREV_INSN (head); | |
4809 | NEXT_INSN (PREV_INSN (head)) = note_head; | |
4810 | PREV_INSN (head) = note_list; | |
4811 | NEXT_INSN (note_list) = head; | |
4812 | ||
4813 | if (BLOCK_FOR_INSN (head) != head_bb) | |
4814 | BB_END (head_bb) = note_list; | |
4815 | ||
4816 | head = note_head; | |
3eb9a99d | 4817 | } |
e1ab7874 | 4818 | |
4819 | return head; | |
3eb9a99d | 4820 | } |
4821 | ||
d452a169 | 4822 | /* When we know we are going to discard the schedule due to a failed attempt |
4823 | at modulo scheduling, undo all replacements. */ | |
4824 | static void | |
4825 | undo_all_replacements (void) | |
4826 | { | |
4827 | rtx insn; | |
4828 | int i; | |
4829 | ||
f1f41a6c | 4830 | FOR_EACH_VEC_ELT (scheduled_insns, i, insn) |
d452a169 | 4831 | { |
4832 | sd_iterator_def sd_it; | |
4833 | dep_t dep; | |
4834 | ||
4835 | /* See if we must undo a replacement. */ | |
4836 | for (sd_it = sd_iterator_start (insn, SD_LIST_RES_FORW); | |
4837 | sd_iterator_cond (&sd_it, &dep); sd_iterator_next (&sd_it)) | |
4838 | { | |
4839 | struct dep_replacement *desc = DEP_REPLACE (dep); | |
4840 | if (desc != NULL) | |
4841 | validate_change (desc->insn, desc->loc, desc->orig, 0); | |
4842 | } | |
4843 | } | |
4844 | } | |
4845 | ||
3eb9a99d | 4846 | /* Move insns that became ready to fire from queue to ready list. */ |
4847 | ||
30b1ec30 | 4848 | static void |
952f0048 | 4849 | queue_to_ready (struct ready_list *ready) |
3eb9a99d | 4850 | { |
4851 | rtx insn; | |
4852 | rtx link; | |
bdeaf25e | 4853 | rtx skip_insn; |
3eb9a99d | 4854 | |
4855 | q_ptr = NEXT_Q (q_ptr); | |
4856 | ||
bdeaf25e | 4857 | if (dbg_cnt (sched_insn) == false) |
b5f4bfde | 4858 | { |
4859 | /* If debug counter is activated do not requeue the first | |
4860 | nonscheduled insn. */ | |
4861 | skip_insn = nonscheduled_insns_begin; | |
4862 | do | |
4863 | { | |
4864 | skip_insn = next_nonnote_nondebug_insn (skip_insn); | |
4865 | } | |
4866 | while (QUEUE_INDEX (skip_insn) == QUEUE_SCHEDULED); | |
4867 | } | |
bdeaf25e | 4868 | else |
4869 | skip_insn = NULL_RTX; | |
4870 | ||
3eb9a99d | 4871 | /* Add all pending insns that can be scheduled without stalls to the |
7a31a7bd | 4872 | ready list. */ |
4873 | for (link = insn_queue[q_ptr]; link; link = XEXP (link, 1)) | |
4874 | { | |
4875 | insn = XEXP (link, 0); | |
4876 | q_size -= 1; | |
c2069298 | 4877 | |
7a31a7bd | 4878 | if (sched_verbose >= 2) |
4879 | fprintf (sched_dump, ";;\t\tQ-->Ready: insn %s: ", | |
4880 | (*current_sched_info->print_insn) (insn, 0)); | |
c2069298 | 4881 | |
23df5e5b | 4882 | /* If the ready list is full, delay the insn for 1 cycle. |
4883 | See the comment in schedule_block for the rationale. */ | |
4884 | if (!reload_completed | |
b30b031c | 4885 | && (ready->n_ready - ready->n_debug > MAX_SCHED_READY_INSNS |
4886 | || (sched_pressure == SCHED_PRESSURE_MODEL | |
4887 | /* Limit pressure recalculations to MAX_SCHED_READY_INSNS | |
4888 | instructions too. */ | |
4889 | && model_index (insn) > (model_curr_point | |
4890 | + MAX_SCHED_READY_INSNS))) | |
4891 | && !(sched_pressure == SCHED_PRESSURE_MODEL | |
4892 | && model_curr_point < model_num_insns | |
4893 | /* Always allow the next model instruction to issue. */ | |
4894 | && model_index (insn) == model_curr_point) | |
bdeaf25e | 4895 | && !SCHED_GROUP_P (insn) |
4896 | && insn != skip_insn) | |
b49b2e5c | 4897 | queue_insn (insn, 1, "ready full"); |
23df5e5b | 4898 | else |
4899 | { | |
4900 | ready_add (ready, insn, false); | |
4901 | if (sched_verbose >= 2) | |
4902 | fprintf (sched_dump, "moving to ready without stalls\n"); | |
4903 | } | |
c2069298 | 4904 | } |
e4897000 | 4905 | free_INSN_LIST_list (&insn_queue[q_ptr]); |
7a31a7bd | 4906 | |
4907 | /* If there are no ready insns, stall until one is ready and add all | |
4908 | of the pending insns at that point to the ready list. */ | |
4909 | if (ready->n_ready == 0) | |
c2069298 | 4910 | { |
19cb6b50 | 4911 | int stalls; |
c2069298 | 4912 | |
67900a4f | 4913 | for (stalls = 1; stalls <= max_insn_queue_index; stalls++) |
7a31a7bd | 4914 | { |
4915 | if ((link = insn_queue[NEXT_Q_AFTER (q_ptr, stalls)])) | |
4916 | { | |
4917 | for (; link; link = XEXP (link, 1)) | |
4918 | { | |
4919 | insn = XEXP (link, 0); | |
4920 | q_size -= 1; | |
c2069298 | 4921 | |
7a31a7bd | 4922 | if (sched_verbose >= 2) |
4923 | fprintf (sched_dump, ";;\t\tQ-->Ready: insn %s: ", | |
4924 | (*current_sched_info->print_insn) (insn, 0)); | |
c2069298 | 4925 | |
e4897000 | 4926 | ready_add (ready, insn, false); |
7a31a7bd | 4927 | if (sched_verbose >= 2) |
4928 | fprintf (sched_dump, "moving to ready with %d stalls\n", stalls); | |
4929 | } | |
e4897000 | 4930 | free_INSN_LIST_list (&insn_queue[NEXT_Q_AFTER (q_ptr, stalls)]); |
c2069298 | 4931 | |
bea4bad2 | 4932 | advance_one_cycle (); |
4933 | ||
4934 | break; | |
7a31a7bd | 4935 | } |
bea4bad2 | 4936 | |
4937 | advance_one_cycle (); | |
7a31a7bd | 4938 | } |
c2069298 | 4939 | |
7a31a7bd | 4940 | q_ptr = NEXT_Q_AFTER (q_ptr, stalls); |
4941 | clock_var += stalls; | |
c2069298 | 4942 | } |
c2069298 | 4943 | } |
4944 | ||
52c4b43f | 4945 | /* Used by early_queue_to_ready. Determines whether it is "ok" to |
48e1416a | 4946 | prematurely move INSN from the queue to the ready list. Currently, |
4947 | if a target defines the hook 'is_costly_dependence', this function | |
52c4b43f | 4948 | uses the hook to check whether there exist any dependences which are |
48e1416a | 4949 | considered costly by the target, between INSN and other insns that |
52c4b43f | 4950 | have already been scheduled. Dependences are checked up to Y cycles |
4951 | back, with default Y=1; The flag -fsched-stalled-insns-dep=Y allows | |
48e1416a | 4952 | controlling this value. |
4953 | (Other considerations could be taken into account instead (or in | |
52c4b43f | 4954 | addition) depending on user flags and target hooks. */ |
4955 | ||
48e1416a | 4956 | static bool |
52c4b43f | 4957 | ok_for_early_queue_removal (rtx insn) |
4958 | { | |
52c4b43f | 4959 | if (targetm.sched.is_costly_dependence) |
4960 | { | |
b5f4bfde | 4961 | rtx prev_insn; |
4962 | int n_cycles; | |
f1f41a6c | 4963 | int i = scheduled_insns.length (); |
52c4b43f | 4964 | for (n_cycles = flag_sched_stalled_insns_dep; n_cycles; n_cycles--) |
4965 | { | |
b5f4bfde | 4966 | while (i-- > 0) |
52c4b43f | 4967 | { |
9997bd27 | 4968 | int cost; |
52c4b43f | 4969 | |
f1f41a6c | 4970 | prev_insn = scheduled_insns[i]; |
305799d4 | 4971 | |
6d7dc5b9 | 4972 | if (!NOTE_P (prev_insn)) |
52c4b43f | 4973 | { |
93f6b030 | 4974 | dep_t dep; |
9997bd27 | 4975 | |
93f6b030 | 4976 | dep = sd_find_dep_between (prev_insn, insn, true); |
9997bd27 | 4977 | |
93f6b030 | 4978 | if (dep != NULL) |
52c4b43f | 4979 | { |
9997bd27 | 4980 | cost = dep_cost (dep); |
4981 | ||
4982 | if (targetm.sched.is_costly_dependence (dep, cost, | |
52c4b43f | 4983 | flag_sched_stalled_insns_dep - n_cycles)) |
4984 | return false; | |
4985 | } | |
4986 | } | |
4987 | ||
4988 | if (GET_MODE (prev_insn) == TImode) /* end of dispatch group */ | |
4989 | break; | |
4990 | } | |
4991 | ||
b5f4bfde | 4992 | if (i == 0) |
52c4b43f | 4993 | break; |
52c4b43f | 4994 | } |
4995 | } | |
4996 | ||
4997 | return true; | |
4998 | } | |
4999 | ||
5000 | ||
5001 | /* Remove insns from the queue, before they become "ready" with respect | |
8b332087 | 5002 | to FU latency considerations. */ |
52c4b43f | 5003 | |
48e1416a | 5004 | static int |
52c4b43f | 5005 | early_queue_to_ready (state_t state, struct ready_list *ready) |
5006 | { | |
5007 | rtx insn; | |
5008 | rtx link; | |
5009 | rtx next_link; | |
5010 | rtx prev_link; | |
5011 | bool move_to_ready; | |
5012 | int cost; | |
5013 | state_t temp_state = alloca (dfa_state_size); | |
5014 | int stalls; | |
5015 | int insns_removed = 0; | |
5016 | ||
5017 | /* | |
48e1416a | 5018 | Flag '-fsched-stalled-insns=X' determines the aggressiveness of this |
5019 | function: | |
52c4b43f | 5020 | |
48e1416a | 5021 | X == 0: There is no limit on how many queued insns can be removed |
52c4b43f | 5022 | prematurely. (flag_sched_stalled_insns = -1). |
5023 | ||
48e1416a | 5024 | X >= 1: Only X queued insns can be removed prematurely in each |
52c4b43f | 5025 | invocation. (flag_sched_stalled_insns = X). |
5026 | ||
5027 | Otherwise: Early queue removal is disabled. | |
5028 | (flag_sched_stalled_insns = 0) | |
5029 | */ | |
5030 | ||
48e1416a | 5031 | if (! flag_sched_stalled_insns) |
52c4b43f | 5032 | return 0; |
5033 | ||
67900a4f | 5034 | for (stalls = 0; stalls <= max_insn_queue_index; stalls++) |
52c4b43f | 5035 | { |
5036 | if ((link = insn_queue[NEXT_Q_AFTER (q_ptr, stalls)])) | |
5037 | { | |
5038 | if (sched_verbose > 6) | |
5039 | fprintf (sched_dump, ";; look at index %d + %d\n", q_ptr, stalls); | |
5040 | ||
5041 | prev_link = 0; | |
5042 | while (link) | |
5043 | { | |
5044 | next_link = XEXP (link, 1); | |
5045 | insn = XEXP (link, 0); | |
5046 | if (insn && sched_verbose > 6) | |
5047 | print_rtl_single (sched_dump, insn); | |
5048 | ||
5049 | memcpy (temp_state, state, dfa_state_size); | |
48e1416a | 5050 | if (recog_memoized (insn) < 0) |
52c4b43f | 5051 | /* non-negative to indicate that it's not ready |
5052 | to avoid infinite Q->R->Q->R... */ | |
5053 | cost = 0; | |
5054 | else | |
5055 | cost = state_transition (temp_state, insn); | |
5056 | ||
5057 | if (sched_verbose >= 6) | |
5058 | fprintf (sched_dump, "transition cost = %d\n", cost); | |
5059 | ||
5060 | move_to_ready = false; | |
48e1416a | 5061 | if (cost < 0) |
52c4b43f | 5062 | { |
5063 | move_to_ready = ok_for_early_queue_removal (insn); | |
5064 | if (move_to_ready == true) | |
5065 | { | |
5066 | /* move from Q to R */ | |
5067 | q_size -= 1; | |
e4897000 | 5068 | ready_add (ready, insn, false); |
52c4b43f | 5069 | |
48e1416a | 5070 | if (prev_link) |
52c4b43f | 5071 | XEXP (prev_link, 1) = next_link; |
5072 | else | |
5073 | insn_queue[NEXT_Q_AFTER (q_ptr, stalls)] = next_link; | |
5074 | ||
5075 | free_INSN_LIST_node (link); | |
5076 | ||
5077 | if (sched_verbose >= 2) | |
5078 | fprintf (sched_dump, ";;\t\tEarly Q-->Ready: insn %s\n", | |
5079 | (*current_sched_info->print_insn) (insn, 0)); | |
5080 | ||
5081 | insns_removed++; | |
5082 | if (insns_removed == flag_sched_stalled_insns) | |
e4897000 | 5083 | /* Remove no more than flag_sched_stalled_insns insns |
5084 | from Q at a time. */ | |
52c4b43f | 5085 | return insns_removed; |
5086 | } | |
5087 | } | |
5088 | ||
5089 | if (move_to_ready == false) | |
5090 | prev_link = link; | |
5091 | ||
5092 | link = next_link; | |
5093 | } /* while link */ | |
48e1416a | 5094 | } /* if link */ |
52c4b43f | 5095 | |
5096 | } /* for stalls.. */ | |
5097 | ||
48e1416a | 5098 | return insns_removed; |
52c4b43f | 5099 | } |
5100 | ||
5101 | ||
7a31a7bd | 5102 | /* Print the ready list for debugging purposes. Callable from debugger. */ |
c2069298 | 5103 | |
7a31a7bd | 5104 | static void |
952f0048 | 5105 | debug_ready_list (struct ready_list *ready) |
c2069298 | 5106 | { |
7a31a7bd | 5107 | rtx *p; |
5108 | int i; | |
c2069298 | 5109 | |
7a31a7bd | 5110 | if (ready->n_ready == 0) |
bea4bad2 | 5111 | { |
5112 | fprintf (sched_dump, "\n"); | |
5113 | return; | |
5114 | } | |
c2069298 | 5115 | |
7a31a7bd | 5116 | p = ready_lastpos (ready); |
5117 | for (i = 0; i < ready->n_ready; i++) | |
a7dcf969 | 5118 | { |
5119 | fprintf (sched_dump, " %s:%d", | |
5120 | (*current_sched_info->print_insn) (p[i], 0), | |
5121 | INSN_LUID (p[i])); | |
b30b031c | 5122 | if (sched_pressure != SCHED_PRESSURE_NONE) |
a7dcf969 | 5123 | fprintf (sched_dump, "(cost=%d", |
5124 | INSN_REG_PRESSURE_EXCESS_COST_CHANGE (p[i])); | |
5125 | if (INSN_TICK (p[i]) > clock_var) | |
5126 | fprintf (sched_dump, ":delay=%d", INSN_TICK (p[i]) - clock_var); | |
b30b031c | 5127 | if (sched_pressure != SCHED_PRESSURE_NONE) |
a7dcf969 | 5128 | fprintf (sched_dump, ")"); |
5129 | } | |
7a31a7bd | 5130 | fprintf (sched_dump, "\n"); |
5131 | } | |
c2069298 | 5132 | |
15a03e56 | 5133 | /* Search INSN for REG_SAVE_NOTE notes and convert them back into insn |
5134 | NOTEs. This is used for NOTE_INSN_EPILOGUE_BEG, so that sched-ebb | |
5135 | replaces the epilogue note in the correct basic block. */ | |
e1ab7874 | 5136 | void |
6a1cdb4d | 5137 | reemit_notes (rtx insn) |
3eb9a99d | 5138 | { |
6a1cdb4d | 5139 | rtx note, last = insn; |
3eb9a99d | 5140 | |
3eb9a99d | 5141 | for (note = REG_NOTES (insn); note; note = XEXP (note, 1)) |
5142 | { | |
ba57cb24 | 5143 | if (REG_NOTE_KIND (note) == REG_SAVE_NOTE) |
3eb9a99d | 5144 | { |
bc620c5c | 5145 | enum insn_note note_type = (enum insn_note) INTVAL (XEXP (note, 0)); |
87e9bf87 | 5146 | |
f16b6102 | 5147 | last = emit_note_before (note_type, last); |
5148 | remove_note (insn, note); | |
3eb9a99d | 5149 | } |
5150 | } | |
3eb9a99d | 5151 | } |
5152 | ||
6a1cdb4d | 5153 | /* Move INSN. Reemit notes if needed. Update CFG, if needed. */ |
5154 | static void | |
e1ab7874 | 5155 | move_insn (rtx insn, rtx last, rtx nt) |
6a1cdb4d | 5156 | { |
6a1cdb4d | 5157 | if (PREV_INSN (insn) != last) |
5158 | { | |
5159 | basic_block bb; | |
5160 | rtx note; | |
5161 | int jump_p = 0; | |
3eb9a99d | 5162 | |
6a1cdb4d | 5163 | bb = BLOCK_FOR_INSN (insn); |
48e1416a | 5164 | |
6a1cdb4d | 5165 | /* BB_HEAD is either LABEL or NOTE. */ |
48e1416a | 5166 | gcc_assert (BB_HEAD (bb) != insn); |
6a1cdb4d | 5167 | |
5168 | if (BB_END (bb) == insn) | |
5169 | /* If this is last instruction in BB, move end marker one | |
5170 | instruction up. */ | |
5171 | { | |
5172 | /* Jumps are always placed at the end of basic block. */ | |
5173 | jump_p = control_flow_insn_p (insn); | |
5174 | ||
5175 | gcc_assert (!jump_p | |
e1ab7874 | 5176 | || ((common_sched_info->sched_pass_id == SCHED_RGN_PASS) |
fd27912f | 5177 | && IS_SPECULATION_BRANCHY_CHECK_P (insn)) |
e1ab7874 | 5178 | || (common_sched_info->sched_pass_id |
5179 | == SCHED_EBB_PASS)); | |
48e1416a | 5180 | |
6a1cdb4d | 5181 | gcc_assert (BLOCK_FOR_INSN (PREV_INSN (insn)) == bb); |
5182 | ||
5183 | BB_END (bb) = PREV_INSN (insn); | |
5184 | } | |
3eb9a99d | 5185 | |
6a1cdb4d | 5186 | gcc_assert (BB_END (bb) != last); |
44f831ee | 5187 | |
6a1cdb4d | 5188 | if (jump_p) |
5189 | /* We move the block note along with jump. */ | |
5190 | { | |
e1ab7874 | 5191 | gcc_assert (nt); |
6a1cdb4d | 5192 | |
5193 | note = NEXT_INSN (insn); | |
5194 | while (NOTE_NOT_BB_P (note) && note != nt) | |
5195 | note = NEXT_INSN (note); | |
5196 | ||
5197 | if (note != nt | |
5198 | && (LABEL_P (note) | |
5199 | || BARRIER_P (note))) | |
5200 | note = NEXT_INSN (note); | |
48e1416a | 5201 | |
6a1cdb4d | 5202 | gcc_assert (NOTE_INSN_BASIC_BLOCK_P (note)); |
5203 | } | |
5204 | else | |
5205 | note = insn; | |
5206 | ||
5207 | NEXT_INSN (PREV_INSN (insn)) = NEXT_INSN (note); | |
5208 | PREV_INSN (NEXT_INSN (note)) = PREV_INSN (insn); | |
5209 | ||
5210 | NEXT_INSN (note) = NEXT_INSN (last); | |
5211 | PREV_INSN (NEXT_INSN (last)) = note; | |
44f831ee | 5212 | |
6a1cdb4d | 5213 | NEXT_INSN (last) = insn; |
5214 | PREV_INSN (insn) = last; | |
5215 | ||
5216 | bb = BLOCK_FOR_INSN (last); | |
5217 | ||
5218 | if (jump_p) | |
5219 | { | |
5220 | fix_jump_move (insn); | |
5221 | ||
5222 | if (BLOCK_FOR_INSN (insn) != bb) | |
5223 | move_block_after_check (insn); | |
5224 | ||
5225 | gcc_assert (BB_END (bb) == last); | |
5226 | } | |
5227 | ||
a2bdd643 | 5228 | df_insn_change_bb (insn, bb); |
48e1416a | 5229 | |
6a1cdb4d | 5230 | /* Update BB_END, if needed. */ |
5231 | if (BB_END (bb) == last) | |
48e1416a | 5232 | BB_END (bb) = insn; |
6a1cdb4d | 5233 | } |
e26579fc | 5234 | |
48e1416a | 5235 | SCHED_GROUP_P (insn) = 0; |
3eb9a99d | 5236 | } |
5237 | ||
4db82bc9 | 5238 | /* Return true if scheduling INSN will finish current clock cycle. */ |
5239 | static bool | |
5240 | insn_finishes_cycle_p (rtx insn) | |
5241 | { | |
5242 | if (SCHED_GROUP_P (insn)) | |
5243 | /* After issuing INSN, rest of the sched_group will be forced to issue | |
5244 | in order. Don't make any plans for the rest of cycle. */ | |
5245 | return true; | |
5246 | ||
5247 | /* Finishing the block will, apparently, finish the cycle. */ | |
5248 | if (current_sched_info->insn_finishes_block_p | |
5249 | && current_sched_info->insn_finishes_block_p (insn)) | |
5250 | return true; | |
5251 | ||
5252 | return false; | |
5253 | } | |
5254 | ||
44ad1e56 | 5255 | /* Define type for target data used in multipass scheduling. */ |
5256 | #ifndef TARGET_SCHED_FIRST_CYCLE_MULTIPASS_DATA_T | |
5257 | # define TARGET_SCHED_FIRST_CYCLE_MULTIPASS_DATA_T int | |
5258 | #endif | |
5259 | typedef TARGET_SCHED_FIRST_CYCLE_MULTIPASS_DATA_T first_cycle_multipass_data_t; | |
5260 | ||
58ada791 | 5261 | /* The following structure describe an entry of the stack of choices. */ |
5262 | struct choice_entry | |
5263 | { | |
5264 | /* Ordinal number of the issued insn in the ready queue. */ | |
5265 | int index; | |
5266 | /* The number of the rest insns whose issues we should try. */ | |
5267 | int rest; | |
5268 | /* The number of issued essential insns. */ | |
5269 | int n; | |
5270 | /* State after issuing the insn. */ | |
5271 | state_t state; | |
44ad1e56 | 5272 | /* Target-specific data. */ |
5273 | first_cycle_multipass_data_t target_data; | |
58ada791 | 5274 | }; |
5275 | ||
5276 | /* The following array is used to implement a stack of choices used in | |
5277 | function max_issue. */ | |
5278 | static struct choice_entry *choice_stack; | |
5279 | ||
e1ab7874 | 5280 | /* This holds the value of the target dfa_lookahead hook. */ |
5281 | int dfa_lookahead; | |
58ada791 | 5282 | |
29fde7f1 | 5283 | /* The following variable value is maximal number of tries of issuing |
5284 | insns for the first cycle multipass insn scheduling. We define | |
5285 | this value as constant*(DFA_LOOKAHEAD**ISSUE_RATE). We would not | |
5286 | need this constraint if all real insns (with non-negative codes) | |
5287 | had reservations because in this case the algorithm complexity is | |
5288 | O(DFA_LOOKAHEAD**ISSUE_RATE). Unfortunately, the dfa descriptions | |
5289 | might be incomplete and such insn might occur. For such | |
5290 | descriptions, the complexity of algorithm (without the constraint) | |
5291 | could achieve DFA_LOOKAHEAD ** N , where N is the queue length. */ | |
5292 | static int max_lookahead_tries; | |
5293 | ||
5294 | /* The following value is value of hook | |
5295 | `first_cycle_multipass_dfa_lookahead' at the last call of | |
5296 | `max_issue'. */ | |
5297 | static int cached_first_cycle_multipass_dfa_lookahead = 0; | |
5298 | ||
5299 | /* The following value is value of `issue_rate' at the last call of | |
5300 | `sched_init'. */ | |
5301 | static int cached_issue_rate = 0; | |
5302 | ||
bea4bad2 | 5303 | /* The following function returns maximal (or close to maximal) number |
5304 | of insns which can be issued on the same cycle and one of which | |
58ada791 | 5305 | insns is insns with the best rank (the first insn in READY). To |
bea4bad2 | 5306 | make this function tries different samples of ready insns. READY |
5307 | is current queue `ready'. Global array READY_TRY reflects what | |
7db79429 | 5308 | insns are already issued in this try. The function stops immediately, |
6a1cdb4d | 5309 | if it reached the such a solution, that all instruction can be issued. |
5310 | INDEX will contain index of the best insn in READY. The following | |
e1ab7874 | 5311 | function is used only for first cycle multipass scheduling. |
5312 | ||
5313 | PRIVILEGED_N >= 0 | |
5314 | ||
5315 | This function expects recognized insns only. All USEs, | |
5316 | CLOBBERs, etc must be filtered elsewhere. */ | |
5317 | int | |
5318 | max_issue (struct ready_list *ready, int privileged_n, state_t state, | |
44ad1e56 | 5319 | bool first_cycle_insn_p, int *index) |
bea4bad2 | 5320 | { |
7db79429 | 5321 | int n, i, all, n_ready, best, delay, tries_num; |
e1ab7874 | 5322 | int more_issue; |
58ada791 | 5323 | struct choice_entry *top; |
bea4bad2 | 5324 | rtx insn; |
bea4bad2 | 5325 | |
e1ab7874 | 5326 | n_ready = ready->n_ready; |
5327 | gcc_assert (dfa_lookahead >= 1 && privileged_n >= 0 | |
5328 | && privileged_n <= n_ready); | |
5329 | ||
5330 | /* Init MAX_LOOKAHEAD_TRIES. */ | |
5331 | if (cached_first_cycle_multipass_dfa_lookahead != dfa_lookahead) | |
5332 | { | |
5333 | cached_first_cycle_multipass_dfa_lookahead = dfa_lookahead; | |
5334 | max_lookahead_tries = 100; | |
5335 | for (i = 0; i < issue_rate; i++) | |
5336 | max_lookahead_tries *= dfa_lookahead; | |
5337 | } | |
5338 | ||
5339 | /* Init max_points. */ | |
e1ab7874 | 5340 | more_issue = issue_rate - cycle_issued_insns; |
8658e086 | 5341 | gcc_assert (more_issue >= 0); |
e1ab7874 | 5342 | |
e1ab7874 | 5343 | /* The number of the issued insns in the best solution. */ |
bea4bad2 | 5344 | best = 0; |
e1ab7874 | 5345 | |
58ada791 | 5346 | top = choice_stack; |
e1ab7874 | 5347 | |
5348 | /* Set initial state of the search. */ | |
5349 | memcpy (top->state, state, dfa_state_size); | |
5350 | top->rest = dfa_lookahead; | |
58ada791 | 5351 | top->n = 0; |
44ad1e56 | 5352 | if (targetm.sched.first_cycle_multipass_begin) |
5353 | targetm.sched.first_cycle_multipass_begin (&top->target_data, | |
5354 | ready_try, n_ready, | |
5355 | first_cycle_insn_p); | |
e1ab7874 | 5356 | |
5357 | /* Count the number of the insns to search among. */ | |
58ada791 | 5358 | for (all = i = 0; i < n_ready; i++) |
bea4bad2 | 5359 | if (!ready_try [i]) |
58ada791 | 5360 | all++; |
e1ab7874 | 5361 | |
5362 | /* I is the index of the insn to try next. */ | |
58ada791 | 5363 | i = 0; |
29fde7f1 | 5364 | tries_num = 0; |
58ada791 | 5365 | for (;;) |
5366 | { | |
e1ab7874 | 5367 | if (/* If we've reached a dead end or searched enough of what we have |
5368 | been asked... */ | |
5369 | top->rest == 0 | |
7db79429 | 5370 | /* or have nothing else to try... */ |
5371 | || i >= n_ready | |
5372 | /* or should not issue more. */ | |
5373 | || top->n >= more_issue) | |
58ada791 | 5374 | { |
e1ab7874 | 5375 | /* ??? (... || i == n_ready). */ |
5376 | gcc_assert (i <= n_ready); | |
5377 | ||
7db79429 | 5378 | /* We should not issue more than issue_rate instructions. */ |
5379 | gcc_assert (top->n <= more_issue); | |
5380 | ||
58ada791 | 5381 | if (top == choice_stack) |
5382 | break; | |
e1ab7874 | 5383 | |
5384 | if (best < top - choice_stack) | |
58ada791 | 5385 | { |
e1ab7874 | 5386 | if (privileged_n) |
5387 | { | |
5388 | n = privileged_n; | |
5389 | /* Try to find issued privileged insn. */ | |
3c802a1e | 5390 | while (n && !ready_try[--n]) |
5391 | ; | |
e1ab7874 | 5392 | } |
5393 | ||
5394 | if (/* If all insns are equally good... */ | |
5395 | privileged_n == 0 | |
5396 | /* Or a privileged insn will be issued. */ | |
5397 | || ready_try[n]) | |
5398 | /* Then we have a solution. */ | |
5399 | { | |
5400 | best = top - choice_stack; | |
5401 | /* This is the index of the insn issued first in this | |
5402 | solution. */ | |
5403 | *index = choice_stack [1].index; | |
7db79429 | 5404 | if (top->n == more_issue || best == all) |
e1ab7874 | 5405 | break; |
5406 | } | |
58ada791 | 5407 | } |
e1ab7874 | 5408 | |
5409 | /* Set ready-list index to point to the last insn | |
5410 | ('i++' below will advance it to the next insn). */ | |
58ada791 | 5411 | i = top->index; |
e1ab7874 | 5412 | |
5413 | /* Backtrack. */ | |
58ada791 | 5414 | ready_try [i] = 0; |
44ad1e56 | 5415 | |
5416 | if (targetm.sched.first_cycle_multipass_backtrack) | |
5417 | targetm.sched.first_cycle_multipass_backtrack (&top->target_data, | |
5418 | ready_try, n_ready); | |
5419 | ||
58ada791 | 5420 | top--; |
e1ab7874 | 5421 | memcpy (state, top->state, dfa_state_size); |
58ada791 | 5422 | } |
5423 | else if (!ready_try [i]) | |
5424 | { | |
29fde7f1 | 5425 | tries_num++; |
5426 | if (tries_num > max_lookahead_tries) | |
5427 | break; | |
58ada791 | 5428 | insn = ready_element (ready, i); |
e1ab7874 | 5429 | delay = state_transition (state, insn); |
58ada791 | 5430 | if (delay < 0) |
5431 | { | |
4db82bc9 | 5432 | if (state_dead_lock_p (state) |
5433 | || insn_finishes_cycle_p (insn)) | |
a0fc5ef1 | 5434 | /* We won't issue any more instructions in the next |
5435 | choice_state. */ | |
58ada791 | 5436 | top->rest = 0; |
5437 | else | |
5438 | top->rest--; | |
e1ab7874 | 5439 | |
58ada791 | 5440 | n = top->n; |
e1ab7874 | 5441 | if (memcmp (top->state, state, dfa_state_size) != 0) |
7db79429 | 5442 | n++; |
e1ab7874 | 5443 | |
5444 | /* Advance to the next choice_entry. */ | |
58ada791 | 5445 | top++; |
e1ab7874 | 5446 | /* Initialize it. */ |
5447 | top->rest = dfa_lookahead; | |
58ada791 | 5448 | top->index = i; |
5449 | top->n = n; | |
e1ab7874 | 5450 | memcpy (top->state, state, dfa_state_size); |
58ada791 | 5451 | ready_try [i] = 1; |
44ad1e56 | 5452 | |
5453 | if (targetm.sched.first_cycle_multipass_issue) | |
5454 | targetm.sched.first_cycle_multipass_issue (&top->target_data, | |
5455 | ready_try, n_ready, | |
5456 | insn, | |
5457 | &((top - 1) | |
5458 | ->target_data)); | |
5459 | ||
58ada791 | 5460 | i = -1; |
5461 | } | |
5462 | } | |
e1ab7874 | 5463 | |
5464 | /* Increase ready-list index. */ | |
58ada791 | 5465 | i++; |
5466 | } | |
6a1cdb4d | 5467 | |
44ad1e56 | 5468 | if (targetm.sched.first_cycle_multipass_end) |
5469 | targetm.sched.first_cycle_multipass_end (best != 0 | |
5470 | ? &choice_stack[1].target_data | |
5471 | : NULL); | |
5472 | ||
e1ab7874 | 5473 | /* Restore the original state of the DFA. */ |
48e1416a | 5474 | memcpy (state, choice_stack->state, dfa_state_size); |
e1ab7874 | 5475 | |
bea4bad2 | 5476 | return best; |
5477 | } | |
5478 | ||
5479 | /* The following function chooses insn from READY and modifies | |
e1ab7874 | 5480 | READY. The following function is used only for first |
bdeaf25e | 5481 | cycle multipass scheduling. |
5482 | Return: | |
5483 | -1 if cycle should be advanced, | |
5484 | 0 if INSN_PTR is set to point to the desirable insn, | |
5485 | 1 if choose_ready () should be restarted without advancing the cycle. */ | |
5486 | static int | |
44ad1e56 | 5487 | choose_ready (struct ready_list *ready, bool first_cycle_insn_p, |
5488 | rtx *insn_ptr) | |
bea4bad2 | 5489 | { |
bdeaf25e | 5490 | int lookahead; |
5491 | ||
5492 | if (dbg_cnt (sched_insn) == false) | |
5493 | { | |
b5f4bfde | 5494 | rtx insn = nonscheduled_insns_begin; |
5495 | do | |
5496 | { | |
5497 | insn = next_nonnote_insn (insn); | |
5498 | } | |
5499 | while (QUEUE_INDEX (insn) == QUEUE_SCHEDULED); | |
bdeaf25e | 5500 | |
5501 | if (QUEUE_INDEX (insn) == QUEUE_READY) | |
5502 | /* INSN is in the ready_list. */ | |
5503 | { | |
b5f4bfde | 5504 | nonscheduled_insns_begin = insn; |
bdeaf25e | 5505 | ready_remove_insn (insn); |
5506 | *insn_ptr = insn; | |
5507 | return 0; | |
5508 | } | |
5509 | ||
5510 | /* INSN is in the queue. Advance cycle to move it to the ready list. */ | |
5511 | return -1; | |
5512 | } | |
5513 | ||
5514 | lookahead = 0; | |
29fde7f1 | 5515 | |
5516 | if (targetm.sched.first_cycle_multipass_dfa_lookahead) | |
883b2e73 | 5517 | lookahead = targetm.sched.first_cycle_multipass_dfa_lookahead (); |
9845d120 | 5518 | if (lookahead <= 0 || SCHED_GROUP_P (ready_element (ready, 0)) |
5519 | || DEBUG_INSN_P (ready_element (ready, 0))) | |
bdeaf25e | 5520 | { |
0ac9454e | 5521 | if (targetm.sched.dispatch (NULL_RTX, IS_DISPATCH_ON)) |
5522 | *insn_ptr = ready_remove_first_dispatch (ready); | |
5523 | else | |
5524 | *insn_ptr = ready_remove_first (ready); | |
5525 | ||
bdeaf25e | 5526 | return 0; |
5527 | } | |
bea4bad2 | 5528 | else |
5529 | { | |
5530 | /* Try to choose the better insn. */ | |
6a1cdb4d | 5531 | int index = 0, i, n; |
58ada791 | 5532 | rtx insn; |
e1ab7874 | 5533 | int try_data = 1, try_control = 1; |
5534 | ds_t ts; | |
48e1416a | 5535 | |
58ada791 | 5536 | insn = ready_element (ready, 0); |
5537 | if (INSN_CODE (insn) < 0) | |
bdeaf25e | 5538 | { |
5539 | *insn_ptr = ready_remove_first (ready); | |
5540 | return 0; | |
5541 | } | |
6a1cdb4d | 5542 | |
5543 | if (spec_info | |
5544 | && spec_info->flags & (PREFER_NON_DATA_SPEC | |
5545 | | PREFER_NON_CONTROL_SPEC)) | |
5546 | { | |
6a1cdb4d | 5547 | for (i = 0, n = ready->n_ready; i < n; i++) |
5548 | { | |
e5f7d6da | 5549 | rtx x; |
5550 | ds_t s; | |
5551 | ||
6a1cdb4d | 5552 | x = ready_element (ready, i); |
5553 | s = TODO_SPEC (x); | |
48e1416a | 5554 | |
6a1cdb4d | 5555 | if (spec_info->flags & PREFER_NON_DATA_SPEC |
5556 | && !(s & DATA_SPEC)) | |
48e1416a | 5557 | { |
6a1cdb4d | 5558 | try_data = 0; |
5559 | if (!(spec_info->flags & PREFER_NON_CONTROL_SPEC) | |
5560 | || !try_control) | |
5561 | break; | |
5562 | } | |
48e1416a | 5563 | |
6a1cdb4d | 5564 | if (spec_info->flags & PREFER_NON_CONTROL_SPEC |
5565 | && !(s & CONTROL_SPEC)) | |
5566 | { | |
5567 | try_control = 0; | |
5568 | if (!(spec_info->flags & PREFER_NON_DATA_SPEC) || !try_data) | |
5569 | break; | |
5570 | } | |
5571 | } | |
5572 | } | |
5573 | ||
e1ab7874 | 5574 | ts = TODO_SPEC (insn); |
5575 | if ((ts & SPECULATIVE) | |
5576 | && (((!try_data && (ts & DATA_SPEC)) | |
5577 | || (!try_control && (ts & CONTROL_SPEC))) | |
5578 | || (targetm.sched.first_cycle_multipass_dfa_lookahead_guard_spec | |
5579 | && !targetm.sched | |
5580 | .first_cycle_multipass_dfa_lookahead_guard_spec (insn)))) | |
e5f7d6da | 5581 | /* Discard speculative instruction that stands first in the ready |
5582 | list. */ | |
6a1cdb4d | 5583 | { |
5584 | change_queue_index (insn, 1); | |
bdeaf25e | 5585 | return 1; |
6a1cdb4d | 5586 | } |
5587 | ||
e1ab7874 | 5588 | ready_try[0] = 0; |
6a1cdb4d | 5589 | |
58ada791 | 5590 | for (i = 1; i < ready->n_ready; i++) |
5591 | { | |
5592 | insn = ready_element (ready, i); | |
e1ab7874 | 5593 | |
58ada791 | 5594 | ready_try [i] |
e1ab7874 | 5595 | = ((!try_data && (TODO_SPEC (insn) & DATA_SPEC)) |
5596 | || (!try_control && (TODO_SPEC (insn) & CONTROL_SPEC))); | |
58ada791 | 5597 | } |
6a1cdb4d | 5598 | |
e1ab7874 | 5599 | /* Let the target filter the search space. */ |
5600 | for (i = 1; i < ready->n_ready; i++) | |
5601 | if (!ready_try[i]) | |
5602 | { | |
5603 | insn = ready_element (ready, i); | |
5604 | ||
cdf13311 | 5605 | /* If this insn is recognizable we should have already |
5606 | recognized it earlier. | |
5607 | ??? Not very clear where this is supposed to be done. | |
5608 | See dep_cost_1. */ | |
1b4345f7 | 5609 | gcc_checking_assert (INSN_CODE (insn) >= 0 |
5610 | || recog_memoized (insn) < 0); | |
e1ab7874 | 5611 | |
5612 | ready_try [i] | |
5613 | = (/* INSN_CODE check can be omitted here as it is also done later | |
5614 | in max_issue (). */ | |
5615 | INSN_CODE (insn) < 0 | |
5616 | || (targetm.sched.first_cycle_multipass_dfa_lookahead_guard | |
5617 | && !targetm.sched.first_cycle_multipass_dfa_lookahead_guard | |
5618 | (insn))); | |
5619 | } | |
5620 | ||
44ad1e56 | 5621 | if (max_issue (ready, 1, curr_state, first_cycle_insn_p, &index) == 0) |
bdeaf25e | 5622 | { |
5623 | *insn_ptr = ready_remove_first (ready); | |
76f8cfc9 | 5624 | if (sched_verbose >= 4) |
48e1416a | 5625 | fprintf (sched_dump, ";;\t\tChosen insn (but can't issue) : %s \n", |
76f8cfc9 | 5626 | (*current_sched_info->print_insn) (*insn_ptr, 0)); |
bdeaf25e | 5627 | return 0; |
5628 | } | |
bea4bad2 | 5629 | else |
bdeaf25e | 5630 | { |
48e1416a | 5631 | if (sched_verbose >= 4) |
e1ab7874 | 5632 | fprintf (sched_dump, ";;\t\tChosen insn : %s\n", |
5633 | (*current_sched_info->print_insn) | |
5634 | (ready_element (ready, index), 0)); | |
48e1416a | 5635 | |
bdeaf25e | 5636 | *insn_ptr = ready_remove (ready, index); |
5637 | return 0; | |
5638 | } | |
bea4bad2 | 5639 | } |
5640 | } | |
5641 | ||
d2412f57 | 5642 | /* This function is called when we have successfully scheduled a |
5643 | block. It uses the schedule stored in the scheduled_insns vector | |
5644 | to rearrange the RTL. PREV_HEAD is used as the anchor to which we | |
5645 | append the scheduled insns; TAIL is the insn after the scheduled | |
5646 | block. TARGET_BB is the argument passed to schedule_block. */ | |
5647 | ||
5648 | static void | |
5649 | commit_schedule (rtx prev_head, rtx tail, basic_block *target_bb) | |
5650 | { | |
b5f4bfde | 5651 | unsigned int i; |
5652 | rtx insn; | |
d2412f57 | 5653 | |
5654 | last_scheduled_insn = prev_head; | |
b5f4bfde | 5655 | for (i = 0; |
f1f41a6c | 5656 | scheduled_insns.iterate (i, &insn); |
b5f4bfde | 5657 | i++) |
d2412f57 | 5658 | { |
d2412f57 | 5659 | if (control_flow_insn_p (last_scheduled_insn) |
5660 | || current_sched_info->advance_target_bb (*target_bb, insn)) | |
5661 | { | |
5662 | *target_bb = current_sched_info->advance_target_bb (*target_bb, 0); | |
5663 | ||
5664 | if (sched_verbose) | |
5665 | { | |
5666 | rtx x; | |
5667 | ||
5668 | x = next_real_insn (last_scheduled_insn); | |
5669 | gcc_assert (x); | |
5670 | dump_new_block_header (1, *target_bb, x, tail); | |
5671 | } | |
5672 | ||
5673 | last_scheduled_insn = bb_note (*target_bb); | |
5674 | } | |
5675 | ||
5676 | if (current_sched_info->begin_move_insn) | |
5677 | (*current_sched_info->begin_move_insn) (insn, last_scheduled_insn); | |
5678 | move_insn (insn, last_scheduled_insn, | |
5679 | current_sched_info->next_tail); | |
5680 | if (!DEBUG_INSN_P (insn)) | |
5681 | reemit_notes (insn); | |
5682 | last_scheduled_insn = insn; | |
5683 | } | |
5684 | ||
f1f41a6c | 5685 | scheduled_insns.truncate (0); |
d2412f57 | 5686 | } |
5687 | ||
a0fc5ef1 | 5688 | /* Examine all insns on the ready list and queue those which can't be |
5689 | issued in this cycle. TEMP_STATE is temporary scheduler state we | |
5690 | can use as scratch space. If FIRST_CYCLE_INSN_P is true, no insns | |
5691 | have been issued for the current cycle, which means it is valid to | |
e2f4a6ff | 5692 | issue an asm statement. |
5693 | ||
5694 | If SHADOWS_ONLY_P is true, we eliminate all real insns and only | |
c0457559 | 5695 | leave those for which SHADOW_P is true. If MODULO_EPILOGUE is true, |
5696 | we only leave insns which have an INSN_EXACT_TICK. */ | |
a0fc5ef1 | 5697 | |
5698 | static void | |
e2f4a6ff | 5699 | prune_ready_list (state_t temp_state, bool first_cycle_insn_p, |
c0457559 | 5700 | bool shadows_only_p, bool modulo_epilogue_p) |
a0fc5ef1 | 5701 | { |
ca5e50e4 | 5702 | int i, pass; |
38d8f4bb | 5703 | bool sched_group_found = false; |
ca5e50e4 | 5704 | int min_cost_group = 1; |
a0fc5ef1 | 5705 | |
a0fc5ef1 | 5706 | for (i = 0; i < ready.n_ready; i++) |
5707 | { | |
5708 | rtx insn = ready_element (&ready, i); | |
ca5e50e4 | 5709 | if (SCHED_GROUP_P (insn)) |
38d8f4bb | 5710 | { |
5711 | sched_group_found = true; | |
ca5e50e4 | 5712 | break; |
38d8f4bb | 5713 | } |
ca5e50e4 | 5714 | } |
38d8f4bb | 5715 | |
ca5e50e4 | 5716 | /* Make two passes if there's a SCHED_GROUP_P insn; make sure to handle |
5717 | such an insn first and note its cost, then schedule all other insns | |
5718 | for one cycle later. */ | |
5719 | for (pass = sched_group_found ? 0 : 1; pass < 2; ) | |
5720 | { | |
5721 | int n = ready.n_ready; | |
5722 | for (i = 0; i < n; i++) | |
a0fc5ef1 | 5723 | { |
ca5e50e4 | 5724 | rtx insn = ready_element (&ready, i); |
5725 | int cost = 0; | |
5726 | const char *reason = "resource conflict"; | |
e2f4a6ff | 5727 | |
ca5e50e4 | 5728 | if (DEBUG_INSN_P (insn)) |
5729 | continue; | |
5730 | ||
5731 | if (sched_group_found && !SCHED_GROUP_P (insn)) | |
5732 | { | |
5733 | if (pass == 0) | |
5734 | continue; | |
5735 | cost = min_cost_group; | |
5736 | reason = "not in sched group"; | |
5737 | } | |
5738 | else if (modulo_epilogue_p | |
5739 | && INSN_EXACT_TICK (insn) == INVALID_TICK) | |
5740 | { | |
5741 | cost = max_insn_queue_index; | |
5742 | reason = "not an epilogue insn"; | |
5743 | } | |
5744 | else if (shadows_only_p && !SHADOW_P (insn)) | |
e2f4a6ff | 5745 | { |
ca5e50e4 | 5746 | cost = 1; |
5747 | reason = "not a shadow"; | |
5748 | } | |
5749 | else if (recog_memoized (insn) < 0) | |
5750 | { | |
5751 | if (!first_cycle_insn_p | |
5752 | && (GET_CODE (PATTERN (insn)) == ASM_INPUT | |
5753 | || asm_noperands (PATTERN (insn)) >= 0)) | |
5754 | cost = 1; | |
5755 | reason = "asm"; | |
5756 | } | |
b30b031c | 5757 | else if (sched_pressure != SCHED_PRESSURE_NONE) |
5758 | { | |
5759 | if (sched_pressure == SCHED_PRESSURE_MODEL | |
5760 | && INSN_TICK (insn) <= clock_var) | |
5761 | { | |
5762 | memcpy (temp_state, curr_state, dfa_state_size); | |
5763 | if (state_transition (temp_state, insn) >= 0) | |
5764 | INSN_TICK (insn) = clock_var + 1; | |
5765 | } | |
5766 | cost = 0; | |
5767 | } | |
ca5e50e4 | 5768 | else |
5769 | { | |
5770 | int delay_cost = 0; | |
5771 | ||
d9dd21a8 | 5772 | if (delay_htab.is_created ()) |
e2f4a6ff | 5773 | { |
ca5e50e4 | 5774 | struct delay_pair *delay_entry; |
5775 | delay_entry | |
d9dd21a8 | 5776 | = delay_htab.find_with_hash (insn, |
5777 | htab_hash_pointer (insn)); | |
ca5e50e4 | 5778 | while (delay_entry && delay_cost == 0) |
5779 | { | |
5780 | delay_cost = estimate_shadow_tick (delay_entry); | |
5781 | if (delay_cost > max_insn_queue_index) | |
5782 | delay_cost = max_insn_queue_index; | |
5783 | delay_entry = delay_entry->next_same_i1; | |
5784 | } | |
e2f4a6ff | 5785 | } |
e2f4a6ff | 5786 | |
ca5e50e4 | 5787 | memcpy (temp_state, curr_state, dfa_state_size); |
5788 | cost = state_transition (temp_state, insn); | |
5789 | if (cost < 0) | |
5790 | cost = 0; | |
5791 | else if (cost == 0) | |
5792 | cost = 1; | |
5793 | if (cost < delay_cost) | |
5794 | { | |
5795 | cost = delay_cost; | |
5796 | reason = "shadow tick"; | |
5797 | } | |
5798 | } | |
5799 | if (cost >= 1) | |
e2f4a6ff | 5800 | { |
ca5e50e4 | 5801 | if (SCHED_GROUP_P (insn) && cost > min_cost_group) |
5802 | min_cost_group = cost; | |
5803 | ready_remove (&ready, i); | |
5804 | queue_insn (insn, cost, reason); | |
5805 | if (i + 1 < n) | |
5806 | break; | |
e2f4a6ff | 5807 | } |
a0fc5ef1 | 5808 | } |
ca5e50e4 | 5809 | if (i == n) |
5810 | pass++; | |
a0fc5ef1 | 5811 | } |
5812 | } | |
5813 | ||
e2f4a6ff | 5814 | /* Called when we detect that the schedule is impossible. We examine the |
5815 | backtrack queue to find the earliest insn that caused this condition. */ | |
5816 | ||
5817 | static struct haifa_saved_data * | |
5818 | verify_shadows (void) | |
5819 | { | |
5820 | struct haifa_saved_data *save, *earliest_fail = NULL; | |
5821 | for (save = backtrack_queue; save; save = save->next) | |
5822 | { | |
5823 | int t; | |
5824 | struct delay_pair *pair = save->delay_pair; | |
5825 | rtx i1 = pair->i1; | |
5826 | ||
5827 | for (; pair; pair = pair->next_same_i1) | |
5828 | { | |
5829 | rtx i2 = pair->i2; | |
5830 | ||
5831 | if (QUEUE_INDEX (i2) == QUEUE_SCHEDULED) | |
5832 | continue; | |
5833 | ||
5834 | t = INSN_TICK (i1) + pair_delay (pair); | |
5835 | if (t < clock_var) | |
5836 | { | |
5837 | if (sched_verbose >= 2) | |
5838 | fprintf (sched_dump, | |
5839 | ";;\t\tfailed delay requirements for %d/%d (%d->%d)" | |
5840 | ", not ready\n", | |
5841 | INSN_UID (pair->i1), INSN_UID (pair->i2), | |
5842 | INSN_TICK (pair->i1), INSN_EXACT_TICK (pair->i2)); | |
5843 | earliest_fail = save; | |
5844 | break; | |
5845 | } | |
5846 | if (QUEUE_INDEX (i2) >= 0) | |
5847 | { | |
5848 | int queued_for = INSN_TICK (i2); | |
5849 | ||
5850 | if (t < queued_for) | |
5851 | { | |
5852 | if (sched_verbose >= 2) | |
5853 | fprintf (sched_dump, | |
5854 | ";;\t\tfailed delay requirements for %d/%d" | |
5855 | " (%d->%d), queued too late\n", | |
5856 | INSN_UID (pair->i1), INSN_UID (pair->i2), | |
5857 | INSN_TICK (pair->i1), INSN_EXACT_TICK (pair->i2)); | |
5858 | earliest_fail = save; | |
5859 | break; | |
5860 | } | |
5861 | } | |
5862 | } | |
5863 | } | |
5864 | ||
5865 | return earliest_fail; | |
5866 | } | |
5867 | ||
6a1cdb4d | 5868 | /* Use forward list scheduling to rearrange insns of block pointed to by |
5869 | TARGET_BB, possibly bringing insns from subsequent blocks in the same | |
5870 | region. */ | |
3eb9a99d | 5871 | |
c0457559 | 5872 | bool |
0a15667c | 5873 | schedule_block (basic_block *target_bb, state_t init_state) |
3eb9a99d | 5874 | { |
44ad1e56 | 5875 | int i; |
c0457559 | 5876 | bool success = modulo_ii == 0; |
0c775e95 | 5877 | struct sched_block_state ls; |
bea4bad2 | 5878 | state_t temp_state = NULL; /* It is used for multipass scheduling. */ |
e26579fc | 5879 | int sort_p, advance, start_clock_var; |
3eb9a99d | 5880 | |
c4cd519a | 5881 | /* Head/tail info for this block. */ |
c2069298 | 5882 | rtx prev_head = current_sched_info->prev_head; |
5883 | rtx next_tail = current_sched_info->next_tail; | |
5884 | rtx head = NEXT_INSN (prev_head); | |
5885 | rtx tail = PREV_INSN (next_tail); | |
3eb9a99d | 5886 | |
8058bf10 | 5887 | if ((current_sched_info->flags & DONT_BREAK_DEPENDENCIES) == 0 |
5888 | && sched_pressure != SCHED_PRESSURE_MODEL) | |
d452a169 | 5889 | find_modifiable_mems (head, tail); |
5890 | ||
6106ad68 | 5891 | /* We used to have code to avoid getting parameters moved from hard |
5892 | argument registers into pseudos. | |
3eb9a99d | 5893 | |
6106ad68 | 5894 | However, it was removed when it proved to be of marginal benefit |
5895 | and caused problems because schedule_block and compute_forward_dependences | |
5896 | had different notions of what the "head" insn was. */ | |
3eb9a99d | 5897 | |
89f18f73 | 5898 | gcc_assert (head != tail || INSN_P (head)); |
3eb9a99d | 5899 | |
93f6b030 | 5900 | haifa_recovery_bb_recently_added_p = false; |
6a1cdb4d | 5901 | |
e2f4a6ff | 5902 | backtrack_queue = NULL; |
5903 | ||
c4cd519a | 5904 | /* Debug info. */ |
3eb9a99d | 5905 | if (sched_verbose) |
6a1cdb4d | 5906 | dump_new_block_header (0, *target_bb, head, tail); |
3eb9a99d | 5907 | |
0a15667c | 5908 | if (init_state == NULL) |
5909 | state_reset (curr_state); | |
5910 | else | |
5911 | memcpy (curr_state, init_state, dfa_state_size); | |
3eb9a99d | 5912 | |
e1ab7874 | 5913 | /* Clear the ready list. */ |
30b1ec30 | 5914 | ready.first = ready.veclen - 1; |
30b1ec30 | 5915 | ready.n_ready = 0; |
9845d120 | 5916 | ready.n_debug = 0; |
3eb9a99d | 5917 | |
67900a4f | 5918 | /* It is used for first cycle multipass scheduling. */ |
5919 | temp_state = alloca (dfa_state_size); | |
bea4bad2 | 5920 | |
202d6e5f | 5921 | if (targetm.sched.init) |
5922 | targetm.sched.init (sched_dump, sched_verbose, ready.veclen); | |
663b9536 | 5923 | |
0b923690 | 5924 | /* We start inserting insns after PREV_HEAD. */ |
b5f4bfde | 5925 | last_scheduled_insn = nonscheduled_insns_begin = prev_head; |
d0aa45d0 | 5926 | last_nondebug_scheduled_insn = NULL_RTX; |
3eb9a99d | 5927 | |
9845d120 | 5928 | gcc_assert ((NOTE_P (last_scheduled_insn) |
9221ec44 | 5929 | || DEBUG_INSN_P (last_scheduled_insn)) |
6a1cdb4d | 5930 | && BLOCK_FOR_INSN (last_scheduled_insn) == *target_bb); |
5931 | ||
c2069298 | 5932 | /* Initialize INSN_QUEUE. Q_SIZE is the total number of insns in the |
5933 | queue. */ | |
3eb9a99d | 5934 | q_ptr = 0; |
5935 | q_size = 0; | |
bea4bad2 | 5936 | |
4077bf7a | 5937 | insn_queue = XALLOCAVEC (rtx, max_insn_queue_index + 1); |
67900a4f | 5938 | memset (insn_queue, 0, (max_insn_queue_index + 1) * sizeof (rtx)); |
3eb9a99d | 5939 | |
de206697 | 5940 | /* Start just before the beginning of time. */ |
5941 | clock_var = -1; | |
e4897000 | 5942 | |
48e1416a | 5943 | /* We need queue and ready lists and clock_var be initialized |
e4897000 | 5944 | in try_ready () (which is called through init_ready_list ()). */ |
5945 | (*current_sched_info->init_ready_list) (); | |
5946 | ||
b30b031c | 5947 | if (sched_pressure == SCHED_PRESSURE_MODEL) |
5948 | model_start_schedule (); | |
5949 | ||
23df5e5b | 5950 | /* The algorithm is O(n^2) in the number of ready insns at any given |
5951 | time in the worst case. Before reload we are more likely to have | |
5952 | big lists so truncate them to a reasonable size. */ | |
9845d120 | 5953 | if (!reload_completed |
5954 | && ready.n_ready - ready.n_debug > MAX_SCHED_READY_INSNS) | |
23df5e5b | 5955 | { |
5956 | ready_sort (&ready); | |
5957 | ||
9845d120 | 5958 | /* Find first free-standing insn past MAX_SCHED_READY_INSNS. |
5959 | If there are debug insns, we know they're first. */ | |
5960 | for (i = MAX_SCHED_READY_INSNS + ready.n_debug; i < ready.n_ready; i++) | |
23df5e5b | 5961 | if (!SCHED_GROUP_P (ready_element (&ready, i))) |
5962 | break; | |
5963 | ||
5964 | if (sched_verbose >= 2) | |
5965 | { | |
5966 | fprintf (sched_dump, | |
5967 | ";;\t\tReady list on entry: %d insns\n", ready.n_ready); | |
5968 | fprintf (sched_dump, | |
5969 | ";;\t\t before reload => truncated to %d insns\n", i); | |
5970 | } | |
5971 | ||
bdeaf25e | 5972 | /* Delay all insns past it for 1 cycle. If debug counter is |
5973 | activated make an exception for the insn right after | |
b5f4bfde | 5974 | nonscheduled_insns_begin. */ |
bdeaf25e | 5975 | { |
5976 | rtx skip_insn; | |
5977 | ||
5978 | if (dbg_cnt (sched_insn) == false) | |
b5f4bfde | 5979 | skip_insn = next_nonnote_insn (nonscheduled_insns_begin); |
bdeaf25e | 5980 | else |
5981 | skip_insn = NULL_RTX; | |
5982 | ||
5983 | while (i < ready.n_ready) | |
5984 | { | |
5985 | rtx insn; | |
5986 | ||
5987 | insn = ready_remove (&ready, i); | |
5988 | ||
5989 | if (insn != skip_insn) | |
b49b2e5c | 5990 | queue_insn (insn, 1, "list truncated"); |
bdeaf25e | 5991 | } |
b5f4bfde | 5992 | if (skip_insn) |
5993 | ready_add (&ready, skip_insn, true); | |
bdeaf25e | 5994 | } |
23df5e5b | 5995 | } |
5996 | ||
6a1cdb4d | 5997 | /* Now we can restore basic block notes and maintain precise cfg. */ |
5998 | restore_bb_notes (*target_bb); | |
5999 | ||
e4897000 | 6000 | last_clock_var = -1; |
6001 | ||
e26579fc | 6002 | advance = 0; |
de206697 | 6003 | |
f1f41a6c | 6004 | gcc_assert (scheduled_insns.length () == 0); |
58ada791 | 6005 | sort_p = TRUE; |
e2f4a6ff | 6006 | must_backtrack = false; |
c0457559 | 6007 | modulo_insns_scheduled = 0; |
6008 | ||
6009 | ls.modulo_epilogue = false; | |
e2f4a6ff | 6010 | |
c4cd519a | 6011 | /* Loop until all the insns in BB are scheduled. */ |
c2069298 | 6012 | while ((*current_sched_info->schedule_more_p) ()) |
3eb9a99d | 6013 | { |
d452a169 | 6014 | perform_replacements_new_cycle (); |
e26579fc | 6015 | do |
3eb9a99d | 6016 | { |
e26579fc | 6017 | start_clock_var = clock_var; |
6018 | ||
6019 | clock_var++; | |
952f0048 | 6020 | |
e26579fc | 6021 | advance_one_cycle (); |
952f0048 | 6022 | |
e26579fc | 6023 | /* Add to the ready list all pending insns that can be issued now. |
6024 | If there are no ready insns, increment clock until one | |
6025 | is ready and add all pending insns at that point to the ready | |
6026 | list. */ | |
6027 | queue_to_ready (&ready); | |
952f0048 | 6028 | |
89f18f73 | 6029 | gcc_assert (ready.n_ready); |
952f0048 | 6030 | |
e26579fc | 6031 | if (sched_verbose >= 2) |
6032 | { | |
6033 | fprintf (sched_dump, ";;\t\tReady list after queue_to_ready: "); | |
6034 | debug_ready_list (&ready); | |
6035 | } | |
6036 | advance -= clock_var - start_clock_var; | |
3eb9a99d | 6037 | } |
e26579fc | 6038 | while (advance > 0); |
3eb9a99d | 6039 | |
c0457559 | 6040 | if (ls.modulo_epilogue) |
6041 | { | |
6042 | int stage = clock_var / modulo_ii; | |
6043 | if (stage > modulo_last_stage * 2 + 2) | |
6044 | { | |
6045 | if (sched_verbose >= 2) | |
6046 | fprintf (sched_dump, | |
6047 | ";;\t\tmodulo scheduled succeeded at II %d\n", | |
6048 | modulo_ii); | |
6049 | success = true; | |
6050 | goto end_schedule; | |
6051 | } | |
6052 | } | |
6053 | else if (modulo_ii > 0) | |
6054 | { | |
6055 | int stage = clock_var / modulo_ii; | |
6056 | if (stage > modulo_max_stages) | |
6057 | { | |
6058 | if (sched_verbose >= 2) | |
6059 | fprintf (sched_dump, | |
6060 | ";;\t\tfailing schedule due to excessive stages\n"); | |
6061 | goto end_schedule; | |
6062 | } | |
6063 | if (modulo_n_insns == modulo_insns_scheduled | |
6064 | && stage > modulo_last_stage) | |
6065 | { | |
6066 | if (sched_verbose >= 2) | |
6067 | fprintf (sched_dump, | |
6068 | ";;\t\tfound kernel after %d stages, II %d\n", | |
6069 | stage, modulo_ii); | |
6070 | ls.modulo_epilogue = true; | |
6071 | } | |
6072 | } | |
6073 | ||
6074 | prune_ready_list (temp_state, true, false, ls.modulo_epilogue); | |
a0fc5ef1 | 6075 | if (ready.n_ready == 0) |
8a83ab3e | 6076 | continue; |
e2f4a6ff | 6077 | if (must_backtrack) |
6078 | goto do_backtrack; | |
a0fc5ef1 | 6079 | |
0c775e95 | 6080 | ls.first_cycle_insn_p = true; |
e2f4a6ff | 6081 | ls.shadows_only_p = false; |
8a83ab3e | 6082 | cycle_issued_insns = 0; |
0c775e95 | 6083 | ls.can_issue_more = issue_rate; |
8a83ab3e | 6084 | for (;;) |
58ada791 | 6085 | { |
8a83ab3e | 6086 | rtx insn; |
6087 | int cost; | |
e2f4a6ff | 6088 | bool asm_p; |
952f0048 | 6089 | |
8a83ab3e | 6090 | if (sort_p && ready.n_ready > 0) |
58ada791 | 6091 | { |
8a83ab3e | 6092 | /* Sort the ready list based on priority. This must be |
6093 | done every iteration through the loop, as schedule_insn | |
6094 | may have readied additional insns that will not be | |
6095 | sorted correctly. */ | |
6096 | ready_sort (&ready); | |
6097 | ||
6098 | if (sched_verbose >= 2) | |
6099 | { | |
6100 | fprintf (sched_dump, ";;\t\tReady list after ready_sort: "); | |
6101 | debug_ready_list (&ready); | |
6102 | } | |
58ada791 | 6103 | } |
de206697 | 6104 | |
8a83ab3e | 6105 | /* We don't want md sched reorder to even see debug isns, so put |
6106 | them out right away. */ | |
6107 | if (ready.n_ready && DEBUG_INSN_P (ready_element (&ready, 0)) | |
6108 | && (*current_sched_info->schedule_more_p) ()) | |
9845d120 | 6109 | { |
8a83ab3e | 6110 | while (ready.n_ready && DEBUG_INSN_P (ready_element (&ready, 0))) |
6111 | { | |
6112 | rtx insn = ready_remove_first (&ready); | |
6113 | gcc_assert (DEBUG_INSN_P (insn)); | |
6114 | (*current_sched_info->begin_schedule_ready) (insn); | |
f1f41a6c | 6115 | scheduled_insns.safe_push (insn); |
8a83ab3e | 6116 | last_scheduled_insn = insn; |
6117 | advance = schedule_insn (insn); | |
6118 | gcc_assert (advance == 0); | |
6119 | if (ready.n_ready > 0) | |
6120 | ready_sort (&ready); | |
6121 | } | |
9845d120 | 6122 | } |
6123 | ||
0c775e95 | 6124 | if (ls.first_cycle_insn_p && !ready.n_ready) |
8a83ab3e | 6125 | break; |
9bf1996d | 6126 | |
e2f4a6ff | 6127 | resume_after_backtrack: |
8a83ab3e | 6128 | /* Allow the target to reorder the list, typically for |
6129 | better instruction bundling. */ | |
6130 | if (sort_p | |
6131 | && (ready.n_ready == 0 | |
6132 | || !SCHED_GROUP_P (ready_element (&ready, 0)))) | |
6133 | { | |
0c775e95 | 6134 | if (ls.first_cycle_insn_p && targetm.sched.reorder) |
6135 | ls.can_issue_more | |
8a83ab3e | 6136 | = targetm.sched.reorder (sched_dump, sched_verbose, |
6137 | ready_lastpos (&ready), | |
6138 | &ready.n_ready, clock_var); | |
0c775e95 | 6139 | else if (!ls.first_cycle_insn_p && targetm.sched.reorder2) |
6140 | ls.can_issue_more | |
8a83ab3e | 6141 | = targetm.sched.reorder2 (sched_dump, sched_verbose, |
6142 | ready.n_ready | |
6143 | ? ready_lastpos (&ready) : NULL, | |
6144 | &ready.n_ready, clock_var); | |
6145 | } | |
bea4bad2 | 6146 | |
8a83ab3e | 6147 | restart_choose_ready: |
0bc18d66 | 6148 | if (sched_verbose >= 2) |
bea4bad2 | 6149 | { |
6a1cdb4d | 6150 | fprintf (sched_dump, ";;\tReady list (t = %3d): ", |
bea4bad2 | 6151 | clock_var); |
6152 | debug_ready_list (&ready); | |
11189c7a | 6153 | if (sched_pressure == SCHED_PRESSURE_WEIGHTED) |
a7dcf969 | 6154 | print_curr_reg_pressure (); |
bea4bad2 | 6155 | } |
6156 | ||
48e1416a | 6157 | if (ready.n_ready == 0 |
0c775e95 | 6158 | && ls.can_issue_more |
48e1416a | 6159 | && reload_completed) |
bea4bad2 | 6160 | { |
67900a4f | 6161 | /* Allow scheduling insns directly from the queue in case |
6162 | there's nothing better to do (ready list is empty) but | |
6163 | there are still vacant dispatch slots in the current cycle. */ | |
6164 | if (sched_verbose >= 6) | |
d4473c84 | 6165 | fprintf (sched_dump,";;\t\tSecond chance\n"); |
67900a4f | 6166 | memcpy (temp_state, curr_state, dfa_state_size); |
6167 | if (early_queue_to_ready (temp_state, &ready)) | |
6168 | ready_sort (&ready); | |
bea4bad2 | 6169 | } |
52c4b43f | 6170 | |
9845d120 | 6171 | if (ready.n_ready == 0 |
0c775e95 | 6172 | || !ls.can_issue_more |
67900a4f | 6173 | || state_dead_lock_p (curr_state) |
6174 | || !(*current_sched_info->schedule_more_p) ()) | |
6175 | break; | |
952f0048 | 6176 | |
67900a4f | 6177 | /* Select and remove the insn from the ready list. */ |
6178 | if (sort_p) | |
6a1cdb4d | 6179 | { |
bdeaf25e | 6180 | int res; |
6181 | ||
6182 | insn = NULL_RTX; | |
0c775e95 | 6183 | res = choose_ready (&ready, ls.first_cycle_insn_p, &insn); |
bdeaf25e | 6184 | |
6185 | if (res < 0) | |
6186 | /* Finish cycle. */ | |
6187 | break; | |
6188 | if (res > 0) | |
8a83ab3e | 6189 | goto restart_choose_ready; |
bdeaf25e | 6190 | |
6191 | gcc_assert (insn != NULL_RTX); | |
6a1cdb4d | 6192 | } |
67900a4f | 6193 | else |
6194 | insn = ready_remove_first (&ready); | |
952f0048 | 6195 | |
b30b031c | 6196 | if (sched_pressure != SCHED_PRESSURE_NONE |
11189c7a | 6197 | && INSN_TICK (insn) > clock_var) |
a7dcf969 | 6198 | { |
6199 | ready_add (&ready, insn, true); | |
6200 | advance = 1; | |
6201 | break; | |
6202 | } | |
6203 | ||
67900a4f | 6204 | if (targetm.sched.dfa_new_cycle |
6205 | && targetm.sched.dfa_new_cycle (sched_dump, sched_verbose, | |
6206 | insn, last_clock_var, | |
6207 | clock_var, &sort_p)) | |
e4897000 | 6208 | /* SORT_P is used by the target to override sorting |
6209 | of the ready list. This is needed when the target | |
6210 | has modified its internal structures expecting that | |
6211 | the insn will be issued next. As we need the insn | |
6212 | to have the highest priority (so it will be returned by | |
6213 | the ready_remove_first call above), we invoke | |
6214 | ready_add (&ready, insn, true). | |
48e1416a | 6215 | But, still, there is one issue: INSN can be later |
6216 | discarded by scheduler's front end through | |
e4897000 | 6217 | current_sched_info->can_schedule_ready_p, hence, won't |
48e1416a | 6218 | be issued next. */ |
67900a4f | 6219 | { |
e4897000 | 6220 | ready_add (&ready, insn, true); |
6221 | break; | |
67900a4f | 6222 | } |
952f0048 | 6223 | |
67900a4f | 6224 | sort_p = TRUE; |
9bf1996d | 6225 | |
6a1cdb4d | 6226 | if (current_sched_info->can_schedule_ready_p |
6227 | && ! (*current_sched_info->can_schedule_ready_p) (insn)) | |
6228 | /* We normally get here only if we don't want to move | |
6229 | insn from the split block. */ | |
6230 | { | |
d452a169 | 6231 | TODO_SPEC (insn) = DEP_POSTPONED; |
8a83ab3e | 6232 | goto restart_choose_ready; |
6a1cdb4d | 6233 | } |
6234 | ||
d9dd21a8 | 6235 | if (delay_htab.is_created ()) |
e2f4a6ff | 6236 | { |
6237 | /* If this insn is the first part of a delay-slot pair, record a | |
6238 | backtrack point. */ | |
6239 | struct delay_pair *delay_entry; | |
6240 | delay_entry | |
d9dd21a8 | 6241 | = delay_htab.find_with_hash (insn, htab_hash_pointer (insn)); |
e2f4a6ff | 6242 | if (delay_entry) |
6243 | { | |
6244 | save_backtrack_point (delay_entry, ls); | |
6245 | if (sched_verbose >= 2) | |
6246 | fprintf (sched_dump, ";;\t\tsaving backtrack point\n"); | |
6247 | } | |
6248 | } | |
6249 | ||
48e1416a | 6250 | /* DECISION is made. */ |
6251 | ||
c0457559 | 6252 | if (modulo_ii > 0 && INSN_UID (insn) < modulo_iter0_max_uid) |
6253 | { | |
6254 | modulo_insns_scheduled++; | |
6255 | modulo_last_stage = clock_var / modulo_ii; | |
6256 | } | |
6a1cdb4d | 6257 | if (TODO_SPEC (insn) & SPECULATIVE) |
6258 | generate_recovery_code (insn); | |
6259 | ||
0ac9454e | 6260 | if (targetm.sched.dispatch (NULL_RTX, IS_DISPATCH_ON)) |
6261 | targetm.sched.dispatch_do (insn, ADD_TO_DISPATCH_WINDOW); | |
6262 | ||
d2412f57 | 6263 | /* Update counters, etc in the scheduler's front end. */ |
6264 | (*current_sched_info->begin_schedule_ready) (insn); | |
f1f41a6c | 6265 | scheduled_insns.safe_push (insn); |
d0aa45d0 | 6266 | gcc_assert (NONDEBUG_INSN_P (insn)); |
6267 | last_nondebug_scheduled_insn = last_scheduled_insn = insn; | |
48e1416a | 6268 | |
a0fc5ef1 | 6269 | if (recog_memoized (insn) >= 0) |
6270 | { | |
2cd04cca | 6271 | memcpy (temp_state, curr_state, dfa_state_size); |
a0fc5ef1 | 6272 | cost = state_transition (curr_state, insn); |
11189c7a | 6273 | if (sched_pressure != SCHED_PRESSURE_WEIGHTED) |
a0fc5ef1 | 6274 | gcc_assert (cost < 0); |
2cd04cca | 6275 | if (memcmp (temp_state, curr_state, dfa_state_size) != 0) |
6276 | cycle_issued_insns++; | |
a0fc5ef1 | 6277 | asm_p = false; |
6278 | } | |
6279 | else | |
6280 | asm_p = (GET_CODE (PATTERN (insn)) == ASM_INPUT | |
6281 | || asm_noperands (PATTERN (insn)) >= 0); | |
952f0048 | 6282 | |
747af5e7 | 6283 | if (targetm.sched.variable_issue) |
0c775e95 | 6284 | ls.can_issue_more = |
883b2e73 | 6285 | targetm.sched.variable_issue (sched_dump, sched_verbose, |
0c775e95 | 6286 | insn, ls.can_issue_more); |
37cde9fb | 6287 | /* A naked CLOBBER or USE generates no instruction, so do |
6288 | not count them against the issue rate. */ | |
6289 | else if (GET_CODE (PATTERN (insn)) != USE | |
6290 | && GET_CODE (PATTERN (insn)) != CLOBBER) | |
0c775e95 | 6291 | ls.can_issue_more--; |
e4897000 | 6292 | advance = schedule_insn (insn); |
deeb83d6 | 6293 | |
e2f4a6ff | 6294 | if (SHADOW_P (insn)) |
6295 | ls.shadows_only_p = true; | |
6296 | ||
deeb83d6 | 6297 | /* After issuing an asm insn we should start a new cycle. */ |
6298 | if (advance == 0 && asm_p) | |
6299 | advance = 1; | |
e2f4a6ff | 6300 | |
6301 | if (must_backtrack) | |
6302 | break; | |
6303 | ||
e26579fc | 6304 | if (advance != 0) |
6305 | break; | |
3eb9a99d | 6306 | |
0c775e95 | 6307 | ls.first_cycle_insn_p = false; |
58ada791 | 6308 | if (ready.n_ready > 0) |
c0457559 | 6309 | prune_ready_list (temp_state, false, ls.shadows_only_p, |
6310 | ls.modulo_epilogue); | |
7a31a7bd | 6311 | } |
3eb9a99d | 6312 | |
e2f4a6ff | 6313 | do_backtrack: |
6314 | if (!must_backtrack) | |
6315 | for (i = 0; i < ready.n_ready; i++) | |
6316 | { | |
6317 | rtx insn = ready_element (&ready, i); | |
6318 | if (INSN_EXACT_TICK (insn) == clock_var) | |
6319 | { | |
6320 | must_backtrack = true; | |
6321 | clock_var++; | |
6322 | break; | |
6323 | } | |
6324 | } | |
c0457559 | 6325 | if (must_backtrack && modulo_ii > 0) |
6326 | { | |
6327 | if (modulo_backtracks_left == 0) | |
6328 | goto end_schedule; | |
6329 | modulo_backtracks_left--; | |
6330 | } | |
e2f4a6ff | 6331 | while (must_backtrack) |
6332 | { | |
6333 | struct haifa_saved_data *failed; | |
6334 | rtx failed_insn; | |
6335 | ||
6336 | must_backtrack = false; | |
6337 | failed = verify_shadows (); | |
6338 | gcc_assert (failed); | |
6339 | ||
6340 | failed_insn = failed->delay_pair->i1; | |
d452a169 | 6341 | /* Clear these queues. */ |
6342 | perform_replacements_new_cycle (); | |
effd1640 | 6343 | toggle_cancelled_flags (false); |
e2f4a6ff | 6344 | unschedule_insns_until (failed_insn); |
6345 | while (failed != backtrack_queue) | |
6346 | free_topmost_backtrack_point (true); | |
6347 | restore_last_backtrack_point (&ls); | |
6348 | if (sched_verbose >= 2) | |
6349 | fprintf (sched_dump, ";;\t\trewind to cycle %d\n", clock_var); | |
6350 | /* Delay by at least a cycle. This could cause additional | |
6351 | backtracking. */ | |
6352 | queue_insn (failed_insn, 1, "backtracked"); | |
6353 | advance = 0; | |
6354 | if (must_backtrack) | |
6355 | continue; | |
6356 | if (ready.n_ready > 0) | |
6357 | goto resume_after_backtrack; | |
6358 | else | |
6359 | { | |
6360 | if (clock_var == 0 && ls.first_cycle_insn_p) | |
6361 | goto end_schedule; | |
6362 | advance = 1; | |
6363 | break; | |
6364 | } | |
6365 | } | |
6366 | } | |
c0457559 | 6367 | if (ls.modulo_epilogue) |
6368 | success = true; | |
e2f4a6ff | 6369 | end_schedule: |
0a15667c | 6370 | advance_one_cycle (); |
d452a169 | 6371 | perform_replacements_new_cycle (); |
c0457559 | 6372 | if (modulo_ii > 0) |
6373 | { | |
6374 | /* Once again, debug insn suckiness: they can be on the ready list | |
6375 | even if they have unresolved dependencies. To make our view | |
6376 | of the world consistent, remove such "ready" insns. */ | |
6377 | restart_debug_insn_loop: | |
6378 | for (i = ready.n_ready - 1; i >= 0; i--) | |
6379 | { | |
6380 | rtx x; | |
6381 | ||
6382 | x = ready_element (&ready, i); | |
6383 | if (DEPS_LIST_FIRST (INSN_HARD_BACK_DEPS (x)) != NULL | |
6384 | || DEPS_LIST_FIRST (INSN_SPEC_BACK_DEPS (x)) != NULL) | |
6385 | { | |
6386 | ready_remove (&ready, i); | |
6387 | goto restart_debug_insn_loop; | |
6388 | } | |
6389 | } | |
6390 | for (i = ready.n_ready - 1; i >= 0; i--) | |
6391 | { | |
6392 | rtx x; | |
6393 | ||
6394 | x = ready_element (&ready, i); | |
6395 | resolve_dependencies (x); | |
6396 | } | |
6397 | for (i = 0; i <= max_insn_queue_index; i++) | |
6398 | { | |
6399 | rtx link; | |
6400 | while ((link = insn_queue[i]) != NULL) | |
6401 | { | |
6402 | rtx x = XEXP (link, 0); | |
6403 | insn_queue[i] = XEXP (link, 1); | |
6404 | QUEUE_INDEX (x) = QUEUE_NOWHERE; | |
6405 | free_INSN_LIST_node (link); | |
6406 | resolve_dependencies (x); | |
6407 | } | |
6408 | } | |
6409 | } | |
6410 | ||
d452a169 | 6411 | if (!success) |
6412 | undo_all_replacements (); | |
6413 | ||
7a31a7bd | 6414 | /* Debug info. */ |
6415 | if (sched_verbose) | |
6416 | { | |
6417 | fprintf (sched_dump, ";;\tReady list (final): "); | |
6418 | debug_ready_list (&ready); | |
7a31a7bd | 6419 | } |
3eb9a99d | 6420 | |
c0457559 | 6421 | if (modulo_ii == 0 && current_sched_info->queue_must_finish_empty) |
e4897000 | 6422 | /* Sanity check -- queue must be empty now. Meaningless if region has |
6423 | multiple bbs. */ | |
9845d120 | 6424 | gcc_assert (!q_size && !ready.n_ready && !ready.n_debug); |
c0457559 | 6425 | else if (modulo_ii == 0) |
58ada791 | 6426 | { |
e4897000 | 6427 | /* We must maintain QUEUE_INDEX between blocks in region. */ |
6428 | for (i = ready.n_ready - 1; i >= 0; i--) | |
6a1cdb4d | 6429 | { |
6430 | rtx x; | |
48e1416a | 6431 | |
6a1cdb4d | 6432 | x = ready_element (&ready, i); |
6433 | QUEUE_INDEX (x) = QUEUE_NOWHERE; | |
2261c559 | 6434 | TODO_SPEC (x) = HARD_DEP; |
6a1cdb4d | 6435 | } |
e4897000 | 6436 | |
48e1416a | 6437 | if (q_size) |
e4897000 | 6438 | for (i = 0; i <= max_insn_queue_index; i++) |
6439 | { | |
6440 | rtx link; | |
6441 | for (link = insn_queue[i]; link; link = XEXP (link, 1)) | |
6a1cdb4d | 6442 | { |
6443 | rtx x; | |
6444 | ||
6445 | x = XEXP (link, 0); | |
6446 | QUEUE_INDEX (x) = QUEUE_NOWHERE; | |
2261c559 | 6447 | TODO_SPEC (x) = HARD_DEP; |
6a1cdb4d | 6448 | } |
e4897000 | 6449 | free_INSN_LIST_list (&insn_queue[i]); |
6450 | } | |
6a1cdb4d | 6451 | } |
952f0048 | 6452 | |
b30b031c | 6453 | if (sched_pressure == SCHED_PRESSURE_MODEL) |
6454 | model_end_schedule (); | |
6455 | ||
c0457559 | 6456 | if (success) |
6457 | { | |
6458 | commit_schedule (prev_head, tail, target_bb); | |
6459 | if (sched_verbose) | |
6460 | fprintf (sched_dump, ";; total time = %d\n", clock_var); | |
6461 | } | |
6462 | else | |
6463 | last_scheduled_insn = tail; | |
6464 | ||
f1f41a6c | 6465 | scheduled_insns.truncate (0); |
e1ab7874 | 6466 | |
6a1cdb4d | 6467 | if (!current_sched_info->queue_must_finish_empty |
93f6b030 | 6468 | || haifa_recovery_bb_recently_added_p) |
6a1cdb4d | 6469 | { |
58ada791 | 6470 | /* INSN_TICK (minimum clock tick at which the insn becomes |
6471 | ready) may be not correct for the insn in the subsequent | |
6472 | blocks of the region. We should use a correct value of | |
6473 | `clock_var' or modify INSN_TICK. It is better to keep | |
6474 | clock_var value equal to 0 at the start of a basic block. | |
6475 | Therefore we modify INSN_TICK here. */ | |
e4897000 | 6476 | fix_inter_tick (NEXT_INSN (prev_head), last_scheduled_insn); |
58ada791 | 6477 | } |
6478 | ||
202d6e5f | 6479 | if (targetm.sched.finish) |
93f6b030 | 6480 | { |
202d6e5f | 6481 | targetm.sched.finish (sched_dump, sched_verbose); |
f0b5f617 | 6482 | /* Target might have added some instructions to the scheduled block |
93f6b030 | 6483 | in its md_finish () hook. These new insns don't have any data |
6484 | initialized and to identify them we extend h_i_d so that they'll | |
e1ab7874 | 6485 | get zero luids. */ |
52d7e28c | 6486 | sched_extend_luids (); |
93f6b030 | 6487 | } |
e4897000 | 6488 | |
e1ab7874 | 6489 | if (sched_verbose) |
6490 | fprintf (sched_dump, ";; new head = %d\n;; new tail = %d\n\n", | |
6491 | INSN_UID (head), INSN_UID (tail)); | |
6492 | ||
e4897000 | 6493 | /* Update head/tail boundaries. */ |
6494 | head = NEXT_INSN (prev_head); | |
6495 | tail = last_scheduled_insn; | |
6496 | ||
e1ab7874 | 6497 | head = restore_other_notes (head, NULL); |
3eb9a99d | 6498 | |
7a31a7bd | 6499 | current_sched_info->head = head; |
6500 | current_sched_info->tail = tail; | |
e2f4a6ff | 6501 | |
6502 | free_backtrack_queue (); | |
c0457559 | 6503 | |
6504 | return success; | |
3eb9a99d | 6505 | } |
7a31a7bd | 6506 | \f |
c4cd519a | 6507 | /* Set_priorities: compute priority of each insn in the block. */ |
3eb9a99d | 6508 | |
7a31a7bd | 6509 | int |
952f0048 | 6510 | set_priorities (rtx head, rtx tail) |
3eb9a99d | 6511 | { |
6512 | rtx insn; | |
6513 | int n_insn; | |
48e1416a | 6514 | int sched_max_insns_priority = |
09542196 | 6515 | current_sched_info->sched_max_insns_priority; |
3eb9a99d | 6516 | rtx prev_head; |
3eb9a99d | 6517 | |
9221ec44 | 6518 | if (head == tail && ! INSN_P (head)) |
9845d120 | 6519 | gcc_unreachable (); |
3eb9a99d | 6520 | |
6521 | n_insn = 0; | |
e4897000 | 6522 | |
6523 | prev_head = PREV_INSN (head); | |
3eb9a99d | 6524 | for (insn = tail; insn != prev_head; insn = PREV_INSN (insn)) |
6525 | { | |
e4897000 | 6526 | if (!INSN_P (insn)) |
3eb9a99d | 6527 | continue; |
6528 | ||
e26579fc | 6529 | n_insn++; |
3eb9a99d | 6530 | (void) priority (insn); |
09542196 | 6531 | |
3588ab9c | 6532 | gcc_assert (INSN_PRIORITY_KNOWN (insn)); |
6533 | ||
6534 | sched_max_insns_priority = MAX (sched_max_insns_priority, | |
6535 | INSN_PRIORITY (insn)); | |
3eb9a99d | 6536 | } |
e4897000 | 6537 | |
6538 | current_sched_info->sched_max_insns_priority = sched_max_insns_priority; | |
3eb9a99d | 6539 | |
6540 | return n_insn; | |
6541 | } | |
6542 | ||
e1ab7874 | 6543 | /* Set dump and sched_verbose for the desired debugging output. If no |
6544 | dump-file was specified, but -fsched-verbose=N (any N), print to stderr. | |
6545 | For -fsched-verbose=N, N>=10, print everything to stderr. */ | |
6546 | void | |
6547 | setup_sched_dump (void) | |
6548 | { | |
6549 | sched_verbose = sched_verbose_param; | |
6550 | if (sched_verbose_param == 0 && dump_file) | |
6551 | sched_verbose = 1; | |
6552 | sched_dump = ((sched_verbose_param >= 10 || !dump_file) | |
6553 | ? stderr : dump_file); | |
6554 | } | |
6a1cdb4d | 6555 | |
48e1416a | 6556 | /* Initialize some global state for the scheduler. This function works |
e1ab7874 | 6557 | with the common data shared between all the schedulers. It is called |
6558 | from the scheduler specific initialization routine. */ | |
3eb9a99d | 6559 | |
7a31a7bd | 6560 | void |
3f5be5f4 | 6561 | sched_init (void) |
3eb9a99d | 6562 | { |
c4cd519a | 6563 | /* Disable speculative loads in their presence if cc0 defined. */ |
3eb9a99d | 6564 | #ifdef HAVE_cc0 |
6565 | flag_schedule_speculative_load = 0; | |
6566 | #endif | |
6567 | ||
0ac9454e | 6568 | if (targetm.sched.dispatch (NULL_RTX, IS_DISPATCH_ON)) |
6569 | targetm.sched.dispatch_do (NULL_RTX, DISPATCH_INIT); | |
6570 | ||
57a8bf1b | 6571 | if (live_range_shrinkage_p) |
6572 | sched_pressure = SCHED_PRESSURE_WEIGHTED; | |
6573 | else if (flag_sched_pressure | |
6574 | && !reload_completed | |
6575 | && common_sched_info->sched_pass_id == SCHED_RGN_PASS) | |
d86dbf92 | 6576 | sched_pressure = ((enum sched_pressure_algorithm) |
6577 | PARAM_VALUE (PARAM_SCHED_PRESSURE_ALGORITHM)); | |
11189c7a | 6578 | else |
6579 | sched_pressure = SCHED_PRESSURE_NONE; | |
0ac9454e | 6580 | |
b30b031c | 6581 | if (sched_pressure != SCHED_PRESSURE_NONE) |
3b3a5e5f | 6582 | ira_setup_eliminable_regset (); |
a7dcf969 | 6583 | |
6a1cdb4d | 6584 | /* Initialize SPEC_INFO. */ |
6585 | if (targetm.sched.set_sched_flags) | |
6586 | { | |
6587 | spec_info = &spec_info_var; | |
6588 | targetm.sched.set_sched_flags (spec_info); | |
e1ab7874 | 6589 | |
6590 | if (spec_info->mask != 0) | |
6591 | { | |
6592 | spec_info->data_weakness_cutoff = | |
6593 | (PARAM_VALUE (PARAM_SCHED_SPEC_PROB_CUTOFF) * MAX_DEP_WEAK) / 100; | |
6594 | spec_info->control_weakness_cutoff = | |
6595 | (PARAM_VALUE (PARAM_SCHED_SPEC_PROB_CUTOFF) | |
6596 | * REG_BR_PROB_BASE) / 100; | |
6597 | } | |
6a1cdb4d | 6598 | else |
554f2707 | 6599 | /* So we won't read anything accidentally. */ |
e1ab7874 | 6600 | spec_info = NULL; |
6601 | ||
6a1cdb4d | 6602 | } |
6603 | else | |
554f2707 | 6604 | /* So we won't read anything accidentally. */ |
6a1cdb4d | 6605 | spec_info = 0; |
6606 | ||
c4cd519a | 6607 | /* Initialize issue_rate. */ |
747af5e7 | 6608 | if (targetm.sched.issue_rate) |
883b2e73 | 6609 | issue_rate = targetm.sched.issue_rate (); |
747af5e7 | 6610 | else |
6611 | issue_rate = 1; | |
3eb9a99d | 6612 | |
29fde7f1 | 6613 | if (cached_issue_rate != issue_rate) |
6614 | { | |
6615 | cached_issue_rate = issue_rate; | |
6616 | /* To invalidate max_lookahead_tries: */ | |
6617 | cached_first_cycle_multipass_dfa_lookahead = 0; | |
6618 | } | |
6619 | ||
e1ab7874 | 6620 | if (targetm.sched.first_cycle_multipass_dfa_lookahead) |
6621 | dfa_lookahead = targetm.sched.first_cycle_multipass_dfa_lookahead (); | |
6622 | else | |
6623 | dfa_lookahead = 0; | |
bea4bad2 | 6624 | |
67900a4f | 6625 | if (targetm.sched.init_dfa_pre_cycle_insn) |
6626 | targetm.sched.init_dfa_pre_cycle_insn (); | |
952f0048 | 6627 | |
67900a4f | 6628 | if (targetm.sched.init_dfa_post_cycle_insn) |
6629 | targetm.sched.init_dfa_post_cycle_insn (); | |
952f0048 | 6630 | |
67900a4f | 6631 | dfa_start (); |
6632 | dfa_state_size = state_size (); | |
bea4bad2 | 6633 | |
e1ab7874 | 6634 | init_alias_analysis (); |
62276683 | 6635 | |
e2f4a6ff | 6636 | if (!sched_no_dce) |
6637 | df_set_flags (DF_LR_RUN_DCE); | |
e1ab7874 | 6638 | df_note_add_problem (); |
62276683 | 6639 | |
e1ab7874 | 6640 | /* More problems needed for interloop dep calculation in SMS. */ |
6641 | if (common_sched_info->sched_pass_id == SCHED_SMS_PASS) | |
6642 | { | |
6643 | df_rd_add_problem (); | |
6644 | df_chain_add_problem (DF_DU_CHAIN + DF_UD_CHAIN); | |
6645 | } | |
896c2bfe | 6646 | |
e1ab7874 | 6647 | df_analyze (); |
48e1416a | 6648 | |
6649 | /* Do not run DCE after reload, as this can kill nops inserted | |
e1ab7874 | 6650 | by bundling. */ |
6651 | if (reload_completed) | |
6652 | df_clear_flags (DF_LR_RUN_DCE); | |
d0768316 | 6653 | |
e1ab7874 | 6654 | regstat_compute_calls_crossed (); |
d0768316 | 6655 | |
202d6e5f | 6656 | if (targetm.sched.init_global) |
6657 | targetm.sched.init_global (sched_dump, sched_verbose, get_max_uid () + 1); | |
d0768316 | 6658 | |
b30b031c | 6659 | if (sched_pressure != SCHED_PRESSURE_NONE) |
a7dcf969 | 6660 | { |
6661 | int i, max_regno = max_reg_num (); | |
6662 | ||
169d9eaf | 6663 | if (sched_dump != NULL) |
6664 | /* We need info about pseudos for rtl dumps about pseudo | |
6665 | classes and costs. */ | |
6666 | regstat_init_n_sets_and_refs (); | |
1ec78e16 | 6667 | ira_set_pseudo_classes (true, sched_verbose ? sched_dump : NULL); |
66d9a7b9 | 6668 | sched_regno_pressure_class |
a7dcf969 | 6669 | = (enum reg_class *) xmalloc (max_regno * sizeof (enum reg_class)); |
6670 | for (i = 0; i < max_regno; i++) | |
66d9a7b9 | 6671 | sched_regno_pressure_class[i] |
a7dcf969 | 6672 | = (i < FIRST_PSEUDO_REGISTER |
66d9a7b9 | 6673 | ? ira_pressure_class_translate[REGNO_REG_CLASS (i)] |
6674 | : ira_pressure_class_translate[reg_allocno_class (i)]); | |
a7dcf969 | 6675 | curr_reg_live = BITMAP_ALLOC (NULL); |
b30b031c | 6676 | if (sched_pressure == SCHED_PRESSURE_WEIGHTED) |
6677 | { | |
6678 | saved_reg_live = BITMAP_ALLOC (NULL); | |
6679 | region_ref_regs = BITMAP_ALLOC (NULL); | |
6680 | } | |
a7dcf969 | 6681 | } |
48e1416a | 6682 | |
e1ab7874 | 6683 | curr_state = xmalloc (dfa_state_size); |
6684 | } | |
f77a5bb0 | 6685 | |
e1ab7874 | 6686 | static void haifa_init_only_bb (basic_block, basic_block); |
6a1cdb4d | 6687 | |
e1ab7874 | 6688 | /* Initialize data structures specific to the Haifa scheduler. */ |
6689 | void | |
6690 | haifa_sched_init (void) | |
6691 | { | |
6692 | setup_sched_dump (); | |
6693 | sched_init (); | |
6694 | ||
f1f41a6c | 6695 | scheduled_insns.create (0); |
b5f4bfde | 6696 | |
e1ab7874 | 6697 | if (spec_info != NULL) |
6698 | { | |
6699 | sched_deps_info->use_deps_list = 1; | |
6700 | sched_deps_info->generate_spec_deps = 1; | |
6701 | } | |
6702 | ||
6703 | /* Initialize luids, dependency caches, target and h_i_d for the | |
6704 | whole function. */ | |
6705 | { | |
f1f41a6c | 6706 | bb_vec_t bbs; |
a28770e1 | 6707 | bbs.create (n_basic_blocks_for_fn (cfun)); |
e1ab7874 | 6708 | basic_block bb; |
6a1cdb4d | 6709 | |
e1ab7874 | 6710 | sched_init_bbs (); |
6711 | ||
6712 | FOR_EACH_BB (bb) | |
f1f41a6c | 6713 | bbs.quick_push (bb); |
52d7e28c | 6714 | sched_init_luids (bbs); |
e1ab7874 | 6715 | sched_deps_init (true); |
6716 | sched_extend_target (); | |
52d7e28c | 6717 | haifa_init_h_i_d (bbs); |
e1ab7874 | 6718 | |
f1f41a6c | 6719 | bbs.release (); |
e1ab7874 | 6720 | } |
6721 | ||
6722 | sched_init_only_bb = haifa_init_only_bb; | |
6723 | sched_split_block = sched_split_block_1; | |
6724 | sched_create_empty_bb = sched_create_empty_bb_1; | |
93f6b030 | 6725 | haifa_recovery_bb_ever_added_p = false; |
6726 | ||
e1ab7874 | 6727 | nr_begin_data = nr_begin_control = nr_be_in_data = nr_be_in_control = 0; |
6728 | before_recovery = 0; | |
6729 | after_recovery = 0; | |
c0457559 | 6730 | |
6731 | modulo_ii = 0; | |
e1ab7874 | 6732 | } |
3eb9a99d | 6733 | |
e1ab7874 | 6734 | /* Finish work with the data specific to the Haifa scheduler. */ |
d0768316 | 6735 | void |
e1ab7874 | 6736 | haifa_sched_finish (void) |
d0768316 | 6737 | { |
e1ab7874 | 6738 | sched_create_empty_bb = NULL; |
6739 | sched_split_block = NULL; | |
6740 | sched_init_only_bb = NULL; | |
f77a5bb0 | 6741 | |
6a1cdb4d | 6742 | if (spec_info && spec_info->dump) |
6743 | { | |
6744 | char c = reload_completed ? 'a' : 'b'; | |
6745 | ||
6746 | fprintf (spec_info->dump, | |
6747 | ";; %s:\n", current_function_name ()); | |
6748 | ||
6749 | fprintf (spec_info->dump, | |
6750 | ";; Procedure %cr-begin-data-spec motions == %d\n", | |
6751 | c, nr_begin_data); | |
6752 | fprintf (spec_info->dump, | |
6753 | ";; Procedure %cr-be-in-data-spec motions == %d\n", | |
6754 | c, nr_be_in_data); | |
6755 | fprintf (spec_info->dump, | |
6756 | ";; Procedure %cr-begin-control-spec motions == %d\n", | |
6757 | c, nr_begin_control); | |
6758 | fprintf (spec_info->dump, | |
6759 | ";; Procedure %cr-be-in-control-spec motions == %d\n", | |
6760 | c, nr_be_in_control); | |
6761 | } | |
6762 | ||
f1f41a6c | 6763 | scheduled_insns.release (); |
b5f4bfde | 6764 | |
e1ab7874 | 6765 | /* Finalize h_i_d, dependency caches, and luids for the whole |
6766 | function. Target will be finalized in md_global_finish (). */ | |
6767 | sched_deps_finish (); | |
6768 | sched_finish_luids (); | |
6769 | current_sched_info = NULL; | |
6770 | sched_finish (); | |
6771 | } | |
6772 | ||
48e1416a | 6773 | /* Free global data used during insn scheduling. This function works with |
e1ab7874 | 6774 | the common data shared between the schedulers. */ |
6775 | ||
6776 | void | |
6777 | sched_finish (void) | |
6778 | { | |
6779 | haifa_finish_h_i_d (); | |
b30b031c | 6780 | if (sched_pressure != SCHED_PRESSURE_NONE) |
a7dcf969 | 6781 | { |
169d9eaf | 6782 | if (regstat_n_sets_and_refs != NULL) |
6783 | regstat_free_n_sets_and_refs (); | |
b30b031c | 6784 | if (sched_pressure == SCHED_PRESSURE_WEIGHTED) |
6785 | { | |
6786 | BITMAP_FREE (region_ref_regs); | |
6787 | BITMAP_FREE (saved_reg_live); | |
6788 | } | |
a7dcf969 | 6789 | BITMAP_FREE (curr_reg_live); |
b30b031c | 6790 | free (sched_regno_pressure_class); |
a7dcf969 | 6791 | } |
e1ab7874 | 6792 | free (curr_state); |
6793 | ||
202d6e5f | 6794 | if (targetm.sched.finish_global) |
6795 | targetm.sched.finish_global (sched_dump, sched_verbose); | |
e1ab7874 | 6796 | |
6797 | end_alias_analysis (); | |
6798 | ||
6799 | regstat_free_calls_crossed (); | |
6800 | ||
6801 | dfa_finish (); | |
3eb9a99d | 6802 | } |
e4897000 | 6803 | |
e2f4a6ff | 6804 | /* Free all delay_pair structures that were recorded. */ |
6805 | void | |
6806 | free_delay_pairs (void) | |
6807 | { | |
d9dd21a8 | 6808 | if (delay_htab.is_created ()) |
e2f4a6ff | 6809 | { |
d9dd21a8 | 6810 | delay_htab.empty (); |
6811 | delay_htab_i2.empty (); | |
e2f4a6ff | 6812 | } |
6813 | } | |
6814 | ||
e4897000 | 6815 | /* Fix INSN_TICKs of the instructions in the current block as well as |
9ca2c29a | 6816 | INSN_TICKs of their dependents. |
e4897000 | 6817 | HEAD and TAIL are the begin and the end of the current scheduled block. */ |
6818 | static void | |
6819 | fix_inter_tick (rtx head, rtx tail) | |
6820 | { | |
6821 | /* Set of instructions with corrected INSN_TICK. */ | |
6822 | bitmap_head processed; | |
93f6b030 | 6823 | /* ??? It is doubtful if we should assume that cycle advance happens on |
6824 | basic block boundaries. Basically insns that are unconditionally ready | |
6825 | on the start of the block are more preferable then those which have | |
6826 | a one cycle dependency over insn from the previous block. */ | |
e4897000 | 6827 | int next_clock = clock_var + 1; |
6828 | ||
6829 | bitmap_initialize (&processed, 0); | |
48e1416a | 6830 | |
e4897000 | 6831 | /* Iterates over scheduled instructions and fix their INSN_TICKs and |
6832 | INSN_TICKs of dependent instructions, so that INSN_TICKs are consistent | |
6833 | across different blocks. */ | |
6834 | for (tail = NEXT_INSN (tail); head != tail; head = NEXT_INSN (head)) | |
6835 | { | |
6836 | if (INSN_P (head)) | |
6837 | { | |
6838 | int tick; | |
93f6b030 | 6839 | sd_iterator_def sd_it; |
6840 | dep_t dep; | |
48e1416a | 6841 | |
e4897000 | 6842 | tick = INSN_TICK (head); |
6843 | gcc_assert (tick >= MIN_TICK); | |
48e1416a | 6844 | |
e4897000 | 6845 | /* Fix INSN_TICK of instruction from just scheduled block. */ |
6ef9bbe0 | 6846 | if (bitmap_set_bit (&processed, INSN_LUID (head))) |
e4897000 | 6847 | { |
e4897000 | 6848 | tick -= next_clock; |
48e1416a | 6849 | |
e4897000 | 6850 | if (tick < MIN_TICK) |
6851 | tick = MIN_TICK; | |
48e1416a | 6852 | |
6853 | INSN_TICK (head) = tick; | |
e4897000 | 6854 | } |
48e1416a | 6855 | |
ae3eaa0f | 6856 | if (DEBUG_INSN_P (head)) |
6857 | continue; | |
6858 | ||
93f6b030 | 6859 | FOR_EACH_DEP (head, SD_LIST_RES_FORW, sd_it, dep) |
e4897000 | 6860 | { |
6861 | rtx next; | |
48e1416a | 6862 | |
93f6b030 | 6863 | next = DEP_CON (dep); |
e4897000 | 6864 | tick = INSN_TICK (next); |
6865 | ||
6866 | if (tick != INVALID_TICK | |
6867 | /* If NEXT has its INSN_TICK calculated, fix it. | |
6868 | If not - it will be properly calculated from | |
6869 | scratch later in fix_tick_ready. */ | |
6ef9bbe0 | 6870 | && bitmap_set_bit (&processed, INSN_LUID (next))) |
e4897000 | 6871 | { |
e4897000 | 6872 | tick -= next_clock; |
48e1416a | 6873 | |
e4897000 | 6874 | if (tick < MIN_TICK) |
6875 | tick = MIN_TICK; | |
48e1416a | 6876 | |
e4897000 | 6877 | if (tick > INTER_TICK (next)) |
6878 | INTER_TICK (next) = tick; | |
6879 | else | |
6880 | tick = INTER_TICK (next); | |
93f6b030 | 6881 | |
e4897000 | 6882 | INSN_TICK (next) = tick; |
6883 | } | |
6884 | } | |
6885 | } | |
6886 | } | |
6887 | bitmap_clear (&processed); | |
6888 | } | |
e1ab7874 | 6889 | |
e4897000 | 6890 | /* Check if NEXT is ready to be added to the ready or queue list. |
6891 | If "yes", add it to the proper list. | |
6892 | Returns: | |
6893 | -1 - is not ready yet, | |
6894 | 0 - added to the ready list, | |
6895 | 0 < N - queued for N cycles. */ | |
6896 | int | |
6897 | try_ready (rtx next) | |
48e1416a | 6898 | { |
2261c559 | 6899 | ds_t old_ts, new_ts; |
e4897000 | 6900 | |
2261c559 | 6901 | old_ts = TODO_SPEC (next); |
e4897000 | 6902 | |
d452a169 | 6903 | gcc_assert (!(old_ts & ~(SPECULATIVE | HARD_DEP | DEP_CONTROL | DEP_POSTPONED)) |
6904 | && (old_ts == HARD_DEP | |
6905 | || old_ts == DEP_POSTPONED | |
effd1640 | 6906 | || (old_ts & SPECULATIVE) |
d452a169 | 6907 | || old_ts == DEP_CONTROL)); |
93f6b030 | 6908 | |
d452a169 | 6909 | new_ts = recompute_todo_spec (next, false); |
9997bd27 | 6910 | |
d452a169 | 6911 | if (new_ts & (HARD_DEP | DEP_POSTPONED)) |
effd1640 | 6912 | gcc_assert (new_ts == old_ts |
6a1cdb4d | 6913 | && QUEUE_INDEX (next) == QUEUE_NOWHERE); |
6914 | else if (current_sched_info->new_ready) | |
2261c559 | 6915 | new_ts = current_sched_info->new_ready (next, new_ts); |
6a1cdb4d | 6916 | |
9997bd27 | 6917 | /* * if !(old_ts & SPECULATIVE) (e.g. HARD_DEP or 0), then insn might |
6a1cdb4d | 6918 | have its original pattern or changed (speculative) one. This is due |
6919 | to changing ebb in region scheduling. | |
6920 | * But if (old_ts & SPECULATIVE), then we are pretty sure that insn | |
6921 | has speculative pattern. | |
9997bd27 | 6922 | |
2261c559 | 6923 | We can't assert (!(new_ts & HARD_DEP) || new_ts == old_ts) here because |
6a1cdb4d | 6924 | control-speculative NEXT could have been discarded by sched-rgn.c |
6925 | (the same case as when discarded by can_schedule_ready_p ()). */ | |
9997bd27 | 6926 | |
2261c559 | 6927 | if ((new_ts & SPECULATIVE) |
6928 | /* If (old_ts == new_ts), then (old_ts & SPECULATIVE) and we don't | |
6a1cdb4d | 6929 | need to change anything. */ |
2261c559 | 6930 | && new_ts != old_ts) |
6a1cdb4d | 6931 | { |
6932 | int res; | |
6933 | rtx new_pat; | |
48e1416a | 6934 | |
effd1640 | 6935 | gcc_assert ((new_ts & SPECULATIVE) && !(new_ts & ~SPECULATIVE)); |
48e1416a | 6936 | |
2261c559 | 6937 | res = haifa_speculate_insn (next, new_ts, &new_pat); |
48e1416a | 6938 | |
6a1cdb4d | 6939 | switch (res) |
6940 | { | |
6941 | case -1: | |
6942 | /* It would be nice to change DEP_STATUS of all dependences, | |
2261c559 | 6943 | which have ((DEP_STATUS & SPECULATIVE) == new_ts) to HARD_DEP, |
6a1cdb4d | 6944 | so we won't reanalyze anything. */ |
2261c559 | 6945 | new_ts = HARD_DEP; |
6a1cdb4d | 6946 | break; |
48e1416a | 6947 | |
6a1cdb4d | 6948 | case 0: |
6949 | /* We follow the rule, that every speculative insn | |
6950 | has non-null ORIG_PAT. */ | |
6951 | if (!ORIG_PAT (next)) | |
6952 | ORIG_PAT (next) = PATTERN (next); | |
6953 | break; | |
48e1416a | 6954 | |
6955 | case 1: | |
6a1cdb4d | 6956 | if (!ORIG_PAT (next)) |
6957 | /* If we gonna to overwrite the original pattern of insn, | |
6958 | save it. */ | |
6959 | ORIG_PAT (next) = PATTERN (next); | |
48e1416a | 6960 | |
effd1640 | 6961 | res = haifa_change_pattern (next, new_pat); |
6962 | gcc_assert (res); | |
6a1cdb4d | 6963 | break; |
48e1416a | 6964 | |
6a1cdb4d | 6965 | default: |
6966 | gcc_unreachable (); | |
6967 | } | |
6968 | } | |
48e1416a | 6969 | |
2261c559 | 6970 | /* We need to restore pattern only if (new_ts == 0), because otherwise it is |
6971 | either correct (new_ts & SPECULATIVE), | |
6972 | or we simply don't care (new_ts & HARD_DEP). */ | |
48e1416a | 6973 | |
6a1cdb4d | 6974 | gcc_assert (!ORIG_PAT (next) |
fd27912f | 6975 | || !IS_SPECULATION_BRANCHY_CHECK_P (next)); |
48e1416a | 6976 | |
2261c559 | 6977 | TODO_SPEC (next) = new_ts; |
6978 | ||
d452a169 | 6979 | if (new_ts & (HARD_DEP | DEP_POSTPONED)) |
6a1cdb4d | 6980 | { |
6981 | /* We can't assert (QUEUE_INDEX (next) == QUEUE_NOWHERE) here because | |
6982 | control-speculative NEXT could have been discarded by sched-rgn.c | |
6983 | (the same case as when discarded by can_schedule_ready_p ()). */ | |
6984 | /*gcc_assert (QUEUE_INDEX (next) == QUEUE_NOWHERE);*/ | |
48e1416a | 6985 | |
6a1cdb4d | 6986 | change_queue_index (next, QUEUE_NOWHERE); |
effd1640 | 6987 | |
6a1cdb4d | 6988 | return -1; |
6989 | } | |
2261c559 | 6990 | else if (!(new_ts & BEGIN_SPEC) |
effd1640 | 6991 | && ORIG_PAT (next) && PREDICATED_PAT (next) == NULL_RTX |
6992 | && !IS_SPECULATION_CHECK_P (next)) | |
48e1416a | 6993 | /* We should change pattern of every previously speculative |
611a596b | 6994 | instruction - and we determine if NEXT was speculative by using |
fd27912f | 6995 | ORIG_PAT field. Except one case - speculation checks have ORIG_PAT |
6996 | pat too, so skip them. */ | |
611a596b | 6997 | { |
effd1640 | 6998 | bool success = haifa_change_pattern (next, ORIG_PAT (next)); |
6999 | gcc_assert (success); | |
611a596b | 7000 | ORIG_PAT (next) = 0; |
7001 | } | |
6a1cdb4d | 7002 | |
7003 | if (sched_verbose >= 2) | |
48e1416a | 7004 | { |
6a1cdb4d | 7005 | fprintf (sched_dump, ";;\t\tdependencies resolved: insn %s", |
7006 | (*current_sched_info->print_insn) (next, 0)); | |
48e1416a | 7007 | |
6a1cdb4d | 7008 | if (spec_info && spec_info->dump) |
7009 | { | |
2261c559 | 7010 | if (new_ts & BEGIN_DATA) |
6a1cdb4d | 7011 | fprintf (spec_info->dump, "; data-spec;"); |
2261c559 | 7012 | if (new_ts & BEGIN_CONTROL) |
6a1cdb4d | 7013 | fprintf (spec_info->dump, "; control-spec;"); |
2261c559 | 7014 | if (new_ts & BE_IN_CONTROL) |
6a1cdb4d | 7015 | fprintf (spec_info->dump, "; in-control-spec;"); |
7016 | } | |
effd1640 | 7017 | if (TODO_SPEC (next) & DEP_CONTROL) |
7018 | fprintf (sched_dump, " predicated"); | |
6a1cdb4d | 7019 | fprintf (sched_dump, "\n"); |
48e1416a | 7020 | } |
7021 | ||
6a1cdb4d | 7022 | adjust_priority (next); |
48e1416a | 7023 | |
6a1cdb4d | 7024 | return fix_tick_ready (next); |
7025 | } | |
7026 | ||
7027 | /* Calculate INSN_TICK of NEXT and add it to either ready or queue list. */ | |
7028 | static int | |
e4897000 | 7029 | fix_tick_ready (rtx next) |
7030 | { | |
e4897000 | 7031 | int tick, delay; |
7032 | ||
82467e60 | 7033 | if (!DEBUG_INSN_P (next) && !sd_lists_empty_p (next, SD_LIST_RES_BACK)) |
e4897000 | 7034 | { |
7035 | int full_p; | |
93f6b030 | 7036 | sd_iterator_def sd_it; |
7037 | dep_t dep; | |
e4897000 | 7038 | |
7039 | tick = INSN_TICK (next); | |
6a1cdb4d | 7040 | /* if tick is not equal to INVALID_TICK, then update |
e4897000 | 7041 | INSN_TICK of NEXT with the most recent resolved dependence |
9ca2c29a | 7042 | cost. Otherwise, recalculate from scratch. */ |
9997bd27 | 7043 | full_p = (tick == INVALID_TICK); |
7044 | ||
93f6b030 | 7045 | FOR_EACH_DEP (next, SD_LIST_RES_BACK, sd_it, dep) |
48e1416a | 7046 | { |
9997bd27 | 7047 | rtx pro = DEP_PRO (dep); |
e4897000 | 7048 | int tick1; |
48e1416a | 7049 | |
e4897000 | 7050 | gcc_assert (INSN_TICK (pro) >= MIN_TICK); |
6a1cdb4d | 7051 | |
9997bd27 | 7052 | tick1 = INSN_TICK (pro) + dep_cost (dep); |
e4897000 | 7053 | if (tick1 > tick) |
7054 | tick = tick1; | |
9997bd27 | 7055 | |
7056 | if (!full_p) | |
7057 | break; | |
e4897000 | 7058 | } |
e4897000 | 7059 | } |
7060 | else | |
7061 | tick = -1; | |
7062 | ||
7063 | INSN_TICK (next) = tick; | |
7064 | ||
7065 | delay = tick - clock_var; | |
b30b031c | 7066 | if (delay <= 0 || sched_pressure != SCHED_PRESSURE_NONE) |
e4897000 | 7067 | delay = QUEUE_READY; |
7068 | ||
7069 | change_queue_index (next, delay); | |
6a1cdb4d | 7070 | |
e4897000 | 7071 | return delay; |
7072 | } | |
7073 | ||
7074 | /* Move NEXT to the proper queue list with (DELAY >= 1), | |
7075 | or add it to the ready list (DELAY == QUEUE_READY), | |
7076 | or remove it from ready and queue lists at all (DELAY == QUEUE_NOWHERE). */ | |
7077 | static void | |
7078 | change_queue_index (rtx next, int delay) | |
7079 | { | |
7080 | int i = QUEUE_INDEX (next); | |
7081 | ||
48e1416a | 7082 | gcc_assert (QUEUE_NOWHERE <= delay && delay <= max_insn_queue_index |
e4897000 | 7083 | && delay != 0); |
7084 | gcc_assert (i != QUEUE_SCHEDULED); | |
48e1416a | 7085 | |
e4897000 | 7086 | if ((delay > 0 && NEXT_Q_AFTER (q_ptr, delay) == i) |
7087 | || (delay < 0 && delay == i)) | |
7088 | /* We have nothing to do. */ | |
7089 | return; | |
7090 | ||
9ca2c29a | 7091 | /* Remove NEXT from wherever it is now. */ |
e4897000 | 7092 | if (i == QUEUE_READY) |
7093 | ready_remove_insn (next); | |
7094 | else if (i >= 0) | |
7095 | queue_remove (next); | |
48e1416a | 7096 | |
e4897000 | 7097 | /* Add it to the proper place. */ |
7098 | if (delay == QUEUE_READY) | |
7099 | ready_add (readyp, next, false); | |
7100 | else if (delay >= 1) | |
b49b2e5c | 7101 | queue_insn (next, delay, "change queue index"); |
48e1416a | 7102 | |
e4897000 | 7103 | if (sched_verbose >= 2) |
48e1416a | 7104 | { |
e4897000 | 7105 | fprintf (sched_dump, ";;\t\ttick updated: insn %s", |
7106 | (*current_sched_info->print_insn) (next, 0)); | |
48e1416a | 7107 | |
e4897000 | 7108 | if (delay == QUEUE_READY) |
7109 | fprintf (sched_dump, " into ready\n"); | |
7110 | else if (delay >= 1) | |
7111 | fprintf (sched_dump, " into queue with cost=%d\n", delay); | |
7112 | else | |
7113 | fprintf (sched_dump, " removed from ready or queue lists\n"); | |
7114 | } | |
7115 | } | |
7116 | ||
e1ab7874 | 7117 | static int sched_ready_n_insns = -1; |
6a1cdb4d | 7118 | |
e1ab7874 | 7119 | /* Initialize per region data structures. */ |
7120 | void | |
7121 | sched_extend_ready_list (int new_sched_ready_n_insns) | |
6a1cdb4d | 7122 | { |
7123 | int i; | |
7124 | ||
e1ab7874 | 7125 | if (sched_ready_n_insns == -1) |
7126 | /* At the first call we need to initialize one more choice_stack | |
7127 | entry. */ | |
7128 | { | |
7129 | i = 0; | |
7130 | sched_ready_n_insns = 0; | |
f1f41a6c | 7131 | scheduled_insns.reserve (new_sched_ready_n_insns); |
e1ab7874 | 7132 | } |
7133 | else | |
7134 | i = sched_ready_n_insns + 1; | |
6a1cdb4d | 7135 | |
e1ab7874 | 7136 | ready.veclen = new_sched_ready_n_insns + issue_rate; |
7137 | ready.vec = XRESIZEVEC (rtx, ready.vec, ready.veclen); | |
6a1cdb4d | 7138 | |
e1ab7874 | 7139 | gcc_assert (new_sched_ready_n_insns >= sched_ready_n_insns); |
6a1cdb4d | 7140 | |
e1ab7874 | 7141 | ready_try = (char *) xrecalloc (ready_try, new_sched_ready_n_insns, |
7142 | sched_ready_n_insns, sizeof (*ready_try)); | |
6a1cdb4d | 7143 | |
e1ab7874 | 7144 | /* We allocate +1 element to save initial state in the choice_stack[0] |
7145 | entry. */ | |
7146 | choice_stack = XRESIZEVEC (struct choice_entry, choice_stack, | |
7147 | new_sched_ready_n_insns + 1); | |
93f6b030 | 7148 | |
e1ab7874 | 7149 | for (; i <= new_sched_ready_n_insns; i++) |
44ad1e56 | 7150 | { |
7151 | choice_stack[i].state = xmalloc (dfa_state_size); | |
7152 | ||
7153 | if (targetm.sched.first_cycle_multipass_init) | |
7154 | targetm.sched.first_cycle_multipass_init (&(choice_stack[i] | |
7155 | .target_data)); | |
7156 | } | |
6a1cdb4d | 7157 | |
e1ab7874 | 7158 | sched_ready_n_insns = new_sched_ready_n_insns; |
6a1cdb4d | 7159 | } |
7160 | ||
e1ab7874 | 7161 | /* Free per region data structures. */ |
7162 | void | |
7163 | sched_finish_ready_list (void) | |
97ba5522 | 7164 | { |
e1ab7874 | 7165 | int i; |
97ba5522 | 7166 | |
e1ab7874 | 7167 | free (ready.vec); |
7168 | ready.vec = NULL; | |
7169 | ready.veclen = 0; | |
97ba5522 | 7170 | |
e1ab7874 | 7171 | free (ready_try); |
7172 | ready_try = NULL; | |
6a1cdb4d | 7173 | |
e1ab7874 | 7174 | for (i = 0; i <= sched_ready_n_insns; i++) |
44ad1e56 | 7175 | { |
7176 | if (targetm.sched.first_cycle_multipass_fini) | |
7177 | targetm.sched.first_cycle_multipass_fini (&(choice_stack[i] | |
7178 | .target_data)); | |
7179 | ||
7180 | free (choice_stack [i].state); | |
7181 | } | |
e1ab7874 | 7182 | free (choice_stack); |
7183 | choice_stack = NULL; | |
97ba5522 | 7184 | |
e1ab7874 | 7185 | sched_ready_n_insns = -1; |
6a1cdb4d | 7186 | } |
7187 | ||
e1ab7874 | 7188 | static int |
7189 | haifa_luid_for_non_insn (rtx x) | |
6a1cdb4d | 7190 | { |
e1ab7874 | 7191 | gcc_assert (NOTE_P (x) || LABEL_P (x)); |
7192 | ||
7193 | return 0; | |
6a1cdb4d | 7194 | } |
7195 | ||
7196 | /* Generates recovery code for INSN. */ | |
7197 | static void | |
7198 | generate_recovery_code (rtx insn) | |
7199 | { | |
7200 | if (TODO_SPEC (insn) & BEGIN_SPEC) | |
7201 | begin_speculative_block (insn); | |
48e1416a | 7202 | |
6a1cdb4d | 7203 | /* Here we have insn with no dependencies to |
7204 | instructions other then CHECK_SPEC ones. */ | |
48e1416a | 7205 | |
6a1cdb4d | 7206 | if (TODO_SPEC (insn) & BE_IN_SPEC) |
7207 | add_to_speculative_block (insn); | |
7208 | } | |
7209 | ||
7210 | /* Helper function. | |
7211 | Tries to add speculative dependencies of type FS between instructions | |
9997bd27 | 7212 | in deps_list L and TWIN. */ |
6a1cdb4d | 7213 | static void |
93f6b030 | 7214 | process_insn_forw_deps_be_in_spec (rtx insn, rtx twin, ds_t fs) |
6a1cdb4d | 7215 | { |
93f6b030 | 7216 | sd_iterator_def sd_it; |
7217 | dep_t dep; | |
9997bd27 | 7218 | |
93f6b030 | 7219 | FOR_EACH_DEP (insn, SD_LIST_FORW, sd_it, dep) |
6a1cdb4d | 7220 | { |
7221 | ds_t ds; | |
7222 | rtx consumer; | |
7223 | ||
93f6b030 | 7224 | consumer = DEP_CON (dep); |
6a1cdb4d | 7225 | |
93f6b030 | 7226 | ds = DEP_STATUS (dep); |
6a1cdb4d | 7227 | |
611a596b | 7228 | if (/* If we want to create speculative dep. */ |
7229 | fs | |
7230 | /* And we can do that because this is a true dep. */ | |
7231 | && (ds & DEP_TYPES) == DEP_TRUE) | |
7232 | { | |
7233 | gcc_assert (!(ds & BE_IN_SPEC)); | |
7234 | ||
9ca2c29a | 7235 | if (/* If this dep can be overcome with 'begin speculation'. */ |
611a596b | 7236 | ds & BEGIN_SPEC) |
7237 | /* Then we have a choice: keep the dep 'begin speculative' | |
7238 | or transform it into 'be in speculative'. */ | |
7239 | { | |
7240 | if (/* In try_ready we assert that if insn once became ready | |
7241 | it can be removed from the ready (or queue) list only | |
7242 | due to backend decision. Hence we can't let the | |
7243 | probability of the speculative dep to decrease. */ | |
e1ab7874 | 7244 | ds_weak (ds) <= ds_weak (fs)) |
b06b98c8 | 7245 | { |
7246 | ds_t new_ds; | |
7247 | ||
7248 | new_ds = (ds & ~BEGIN_SPEC) | fs; | |
48e1416a | 7249 | |
b06b98c8 | 7250 | if (/* consumer can 'be in speculative'. */ |
7251 | sched_insn_is_legitimate_for_speculation_p (consumer, | |
7252 | new_ds)) | |
7253 | /* Transform it to be in speculative. */ | |
7254 | ds = new_ds; | |
7255 | } | |
611a596b | 7256 | } |
7257 | else | |
7258 | /* Mark the dep as 'be in speculative'. */ | |
7259 | ds |= fs; | |
7260 | } | |
6a1cdb4d | 7261 | |
93f6b030 | 7262 | { |
7263 | dep_def _new_dep, *new_dep = &_new_dep; | |
7264 | ||
7265 | init_dep_1 (new_dep, twin, consumer, DEP_TYPE (dep), ds); | |
7266 | sd_add_dep (new_dep, false); | |
7267 | } | |
6a1cdb4d | 7268 | } |
7269 | } | |
7270 | ||
7271 | /* Generates recovery code for BEGIN speculative INSN. */ | |
7272 | static void | |
7273 | begin_speculative_block (rtx insn) | |
7274 | { | |
7275 | if (TODO_SPEC (insn) & BEGIN_DATA) | |
48e1416a | 7276 | nr_begin_data++; |
6a1cdb4d | 7277 | if (TODO_SPEC (insn) & BEGIN_CONTROL) |
7278 | nr_begin_control++; | |
7279 | ||
7280 | create_check_block_twin (insn, false); | |
7281 | ||
7282 | TODO_SPEC (insn) &= ~BEGIN_SPEC; | |
7283 | } | |
7284 | ||
e1ab7874 | 7285 | static void haifa_init_insn (rtx); |
7286 | ||
6a1cdb4d | 7287 | /* Generates recovery code for BE_IN speculative INSN. */ |
7288 | static void | |
7289 | add_to_speculative_block (rtx insn) | |
7290 | { | |
7291 | ds_t ts; | |
93f6b030 | 7292 | sd_iterator_def sd_it; |
7293 | dep_t dep; | |
9997bd27 | 7294 | rtx twins = NULL; |
3588ab9c | 7295 | rtx_vec_t priorities_roots; |
6a1cdb4d | 7296 | |
7297 | ts = TODO_SPEC (insn); | |
7298 | gcc_assert (!(ts & ~BE_IN_SPEC)); | |
7299 | ||
7300 | if (ts & BE_IN_DATA) | |
7301 | nr_be_in_data++; | |
7302 | if (ts & BE_IN_CONTROL) | |
7303 | nr_be_in_control++; | |
7304 | ||
7305 | TODO_SPEC (insn) &= ~BE_IN_SPEC; | |
7306 | gcc_assert (!TODO_SPEC (insn)); | |
48e1416a | 7307 | |
6a1cdb4d | 7308 | DONE_SPEC (insn) |= ts; |
7309 | ||
7310 | /* First we convert all simple checks to branchy. */ | |
93f6b030 | 7311 | for (sd_it = sd_iterator_start (insn, SD_LIST_SPEC_BACK); |
7312 | sd_iterator_cond (&sd_it, &dep);) | |
6a1cdb4d | 7313 | { |
93f6b030 | 7314 | rtx check = DEP_PRO (dep); |
6a1cdb4d | 7315 | |
fd27912f | 7316 | if (IS_SPECULATION_SIMPLE_CHECK_P (check)) |
6a1cdb4d | 7317 | { |
7318 | create_check_block_twin (check, true); | |
9997bd27 | 7319 | |
7320 | /* Restart search. */ | |
93f6b030 | 7321 | sd_it = sd_iterator_start (insn, SD_LIST_SPEC_BACK); |
6a1cdb4d | 7322 | } |
7323 | else | |
9997bd27 | 7324 | /* Continue search. */ |
93f6b030 | 7325 | sd_iterator_next (&sd_it); |
6a1cdb4d | 7326 | } |
7327 | ||
f1f41a6c | 7328 | priorities_roots.create (0); |
3588ab9c | 7329 | clear_priorities (insn, &priorities_roots); |
93f6b030 | 7330 | |
7331 | while (1) | |
6a1cdb4d | 7332 | { |
9997bd27 | 7333 | rtx check, twin; |
6a1cdb4d | 7334 | basic_block rec; |
7335 | ||
93f6b030 | 7336 | /* Get the first backward dependency of INSN. */ |
7337 | sd_it = sd_iterator_start (insn, SD_LIST_SPEC_BACK); | |
7338 | if (!sd_iterator_cond (&sd_it, &dep)) | |
7339 | /* INSN has no backward dependencies left. */ | |
7340 | break; | |
6a1cdb4d | 7341 | |
93f6b030 | 7342 | gcc_assert ((DEP_STATUS (dep) & BEGIN_SPEC) == 0 |
7343 | && (DEP_STATUS (dep) & BE_IN_SPEC) != 0 | |
7344 | && (DEP_STATUS (dep) & DEP_TYPES) == DEP_TRUE); | |
9997bd27 | 7345 | |
93f6b030 | 7346 | check = DEP_PRO (dep); |
fd27912f | 7347 | |
7348 | gcc_assert (!IS_SPECULATION_CHECK_P (check) && !ORIG_PAT (check) | |
6a1cdb4d | 7349 | && QUEUE_INDEX (check) == QUEUE_NOWHERE); |
93f6b030 | 7350 | |
6a1cdb4d | 7351 | rec = BLOCK_FOR_INSN (check); |
93f6b030 | 7352 | |
0f9736f3 | 7353 | twin = emit_insn_before (copy_insn (PATTERN (insn)), BB_END (rec)); |
e1ab7874 | 7354 | haifa_init_insn (twin); |
6a1cdb4d | 7355 | |
93f6b030 | 7356 | sd_copy_back_deps (twin, insn, true); |
6a1cdb4d | 7357 | |
7358 | if (sched_verbose && spec_info->dump) | |
7359 | /* INSN_BB (insn) isn't determined for twin insns yet. | |
7360 | So we can't use current_sched_info->print_insn. */ | |
7361 | fprintf (spec_info->dump, ";;\t\tGenerated twin insn : %d/rec%d\n", | |
7362 | INSN_UID (twin), rec->index); | |
7363 | ||
7364 | twins = alloc_INSN_LIST (twin, twins); | |
7365 | ||
9ca2c29a | 7366 | /* Add dependences between TWIN and all appropriate |
6a1cdb4d | 7367 | instructions from REC. */ |
93f6b030 | 7368 | FOR_EACH_DEP (insn, SD_LIST_SPEC_BACK, sd_it, dep) |
7369 | { | |
7370 | rtx pro = DEP_PRO (dep); | |
9997bd27 | 7371 | |
93f6b030 | 7372 | gcc_assert (DEP_TYPE (dep) == REG_DEP_TRUE); |
7373 | ||
7374 | /* INSN might have dependencies from the instructions from | |
7375 | several recovery blocks. At this iteration we process those | |
7376 | producers that reside in REC. */ | |
7377 | if (BLOCK_FOR_INSN (pro) == rec) | |
7378 | { | |
7379 | dep_def _new_dep, *new_dep = &_new_dep; | |
7380 | ||
7381 | init_dep (new_dep, pro, twin, REG_DEP_TRUE); | |
7382 | sd_add_dep (new_dep, false); | |
6a1cdb4d | 7383 | } |
6a1cdb4d | 7384 | } |
6a1cdb4d | 7385 | |
93f6b030 | 7386 | process_insn_forw_deps_be_in_spec (insn, twin, ts); |
6a1cdb4d | 7387 | |
9997bd27 | 7388 | /* Remove all dependencies between INSN and insns in REC. */ |
93f6b030 | 7389 | for (sd_it = sd_iterator_start (insn, SD_LIST_SPEC_BACK); |
7390 | sd_iterator_cond (&sd_it, &dep);) | |
6a1cdb4d | 7391 | { |
93f6b030 | 7392 | rtx pro = DEP_PRO (dep); |
9997bd27 | 7393 | |
93f6b030 | 7394 | if (BLOCK_FOR_INSN (pro) == rec) |
7395 | sd_delete_dep (sd_it); | |
6a1cdb4d | 7396 | else |
93f6b030 | 7397 | sd_iterator_next (&sd_it); |
6a1cdb4d | 7398 | } |
7399 | } | |
6a1cdb4d | 7400 | |
9997bd27 | 7401 | /* We couldn't have added the dependencies between INSN and TWINS earlier |
7402 | because that would make TWINS appear in the INSN_BACK_DEPS (INSN). */ | |
6a1cdb4d | 7403 | while (twins) |
7404 | { | |
7405 | rtx twin; | |
7406 | ||
7407 | twin = XEXP (twins, 0); | |
93f6b030 | 7408 | |
7409 | { | |
7410 | dep_def _new_dep, *new_dep = &_new_dep; | |
7411 | ||
7412 | init_dep (new_dep, insn, twin, REG_DEP_OUTPUT); | |
7413 | sd_add_dep (new_dep, false); | |
7414 | } | |
6a1cdb4d | 7415 | |
7416 | twin = XEXP (twins, 1); | |
7417 | free_INSN_LIST_node (twins); | |
48e1416a | 7418 | twins = twin; |
6a1cdb4d | 7419 | } |
3588ab9c | 7420 | |
7421 | calc_priorities (priorities_roots); | |
f1f41a6c | 7422 | priorities_roots.release (); |
6a1cdb4d | 7423 | } |
7424 | ||
7425 | /* Extends and fills with zeros (only the new part) array pointed to by P. */ | |
7426 | void * | |
7427 | xrecalloc (void *p, size_t new_nmemb, size_t old_nmemb, size_t size) | |
7428 | { | |
7429 | gcc_assert (new_nmemb >= old_nmemb); | |
7430 | p = XRESIZEVAR (void, p, new_nmemb * size); | |
7431 | memset (((char *) p) + old_nmemb * size, 0, (new_nmemb - old_nmemb) * size); | |
7432 | return p; | |
7433 | } | |
7434 | ||
6a1cdb4d | 7435 | /* Helper function. |
7436 | Find fallthru edge from PRED. */ | |
e1ab7874 | 7437 | edge |
7f58c05e | 7438 | find_fallthru_edge_from (basic_block pred) |
6a1cdb4d | 7439 | { |
7440 | edge e; | |
6a1cdb4d | 7441 | basic_block succ; |
7442 | ||
7443 | succ = pred->next_bb; | |
7444 | gcc_assert (succ->prev_bb == pred); | |
7445 | ||
7446 | if (EDGE_COUNT (pred->succs) <= EDGE_COUNT (succ->preds)) | |
7447 | { | |
7f58c05e | 7448 | e = find_fallthru_edge (pred->succs); |
7449 | ||
7450 | if (e) | |
7451 | { | |
7452 | gcc_assert (e->dest == succ); | |
7453 | return e; | |
7454 | } | |
6a1cdb4d | 7455 | } |
7456 | else | |
7457 | { | |
7f58c05e | 7458 | e = find_fallthru_edge (succ->preds); |
7459 | ||
7460 | if (e) | |
7461 | { | |
7462 | gcc_assert (e->src == pred); | |
7463 | return e; | |
7464 | } | |
6a1cdb4d | 7465 | } |
7466 | ||
7467 | return NULL; | |
7468 | } | |
7469 | ||
e1ab7874 | 7470 | /* Extend per basic block data structures. */ |
7471 | static void | |
7472 | sched_extend_bb (void) | |
7473 | { | |
e1ab7874 | 7474 | /* The following is done to keep current_sched_info->next_tail non null. */ |
34154e27 | 7475 | rtx end = BB_END (EXIT_BLOCK_PTR_FOR_FN (cfun)->prev_bb); |
2fdeadc1 | 7476 | rtx insn = DEBUG_INSN_P (end) ? prev_nondebug_insn (end) : end; |
7477 | if (NEXT_INSN (end) == 0 | |
e1ab7874 | 7478 | || (!NOTE_P (insn) |
7479 | && !LABEL_P (insn) | |
7480 | /* Don't emit a NOTE if it would end up before a BARRIER. */ | |
2fdeadc1 | 7481 | && !BARRIER_P (NEXT_INSN (end)))) |
e1ab7874 | 7482 | { |
2fdeadc1 | 7483 | rtx note = emit_note_after (NOTE_INSN_DELETED, end); |
7484 | /* Make note appear outside BB. */ | |
e1ab7874 | 7485 | set_block_for_insn (note, NULL); |
34154e27 | 7486 | BB_END (EXIT_BLOCK_PTR_FOR_FN (cfun)->prev_bb) = end; |
e1ab7874 | 7487 | } |
7488 | } | |
7489 | ||
7490 | /* Init per basic block data structures. */ | |
7491 | void | |
7492 | sched_init_bbs (void) | |
7493 | { | |
7494 | sched_extend_bb (); | |
7495 | } | |
7496 | ||
6a1cdb4d | 7497 | /* Initialize BEFORE_RECOVERY variable. */ |
7498 | static void | |
e1ab7874 | 7499 | init_before_recovery (basic_block *before_recovery_ptr) |
6a1cdb4d | 7500 | { |
7501 | basic_block last; | |
7502 | edge e; | |
7503 | ||
34154e27 | 7504 | last = EXIT_BLOCK_PTR_FOR_FN (cfun)->prev_bb; |
7f58c05e | 7505 | e = find_fallthru_edge_from (last); |
6a1cdb4d | 7506 | |
7507 | if (e) | |
7508 | { | |
48e1416a | 7509 | /* We create two basic blocks: |
6a1cdb4d | 7510 | 1. Single instruction block is inserted right after E->SRC |
48e1416a | 7511 | and has jump to |
6a1cdb4d | 7512 | 2. Empty block right before EXIT_BLOCK. |
7513 | Between these two blocks recovery blocks will be emitted. */ | |
7514 | ||
7515 | basic_block single, empty; | |
7516 | rtx x, label; | |
7517 | ||
48e1416a | 7518 | /* If the fallthrough edge to exit we've found is from the block we've |
e1ab7874 | 7519 | created before, don't do anything more. */ |
7520 | if (last == after_recovery) | |
7521 | return; | |
6a1cdb4d | 7522 | |
e1ab7874 | 7523 | adding_bb_to_current_region_p = false; |
7524 | ||
7525 | single = sched_create_empty_bb (last); | |
7526 | empty = sched_create_empty_bb (single); | |
7527 | ||
7528 | /* Add new blocks to the root loop. */ | |
7529 | if (current_loops != NULL) | |
7530 | { | |
f1f41a6c | 7531 | add_bb_to_loop (single, (*current_loops->larray)[0]); |
7532 | add_bb_to_loop (empty, (*current_loops->larray)[0]); | |
e1ab7874 | 7533 | } |
7534 | ||
7535 | single->count = last->count; | |
6a1cdb4d | 7536 | empty->count = last->count; |
7537 | single->frequency = last->frequency; | |
7538 | empty->frequency = last->frequency; | |
7539 | BB_COPY_PARTITION (single, last); | |
7540 | BB_COPY_PARTITION (empty, last); | |
7541 | ||
7542 | redirect_edge_succ (e, single); | |
7543 | make_single_succ_edge (single, empty, 0); | |
34154e27 | 7544 | make_single_succ_edge (empty, EXIT_BLOCK_PTR_FOR_FN (cfun), |
7545 | EDGE_FALLTHRU); | |
6a1cdb4d | 7546 | |
7547 | label = block_label (empty); | |
7548 | x = emit_jump_insn_after (gen_jump (label), BB_END (single)); | |
7549 | JUMP_LABEL (x) = label; | |
7550 | LABEL_NUSES (label)++; | |
e1ab7874 | 7551 | haifa_init_insn (x); |
48e1416a | 7552 | |
6a1cdb4d | 7553 | emit_barrier_after (x); |
7554 | ||
e1ab7874 | 7555 | sched_init_only_bb (empty, NULL); |
7556 | sched_init_only_bb (single, NULL); | |
7557 | sched_extend_bb (); | |
6a1cdb4d | 7558 | |
e1ab7874 | 7559 | adding_bb_to_current_region_p = true; |
6a1cdb4d | 7560 | before_recovery = single; |
e1ab7874 | 7561 | after_recovery = empty; |
7562 | ||
7563 | if (before_recovery_ptr) | |
7564 | *before_recovery_ptr = before_recovery; | |
6a1cdb4d | 7565 | |
7566 | if (sched_verbose >= 2 && spec_info->dump) | |
7567 | fprintf (spec_info->dump, | |
48e1416a | 7568 | ";;\t\tFixed fallthru to EXIT : %d->>%d->%d->>EXIT\n", |
7569 | last->index, single->index, empty->index); | |
6a1cdb4d | 7570 | } |
7571 | else | |
7572 | before_recovery = last; | |
7573 | } | |
7574 | ||
7575 | /* Returns new recovery block. */ | |
e1ab7874 | 7576 | basic_block |
7577 | sched_create_recovery_block (basic_block *before_recovery_ptr) | |
6a1cdb4d | 7578 | { |
7579 | rtx label; | |
5d65a39c | 7580 | rtx barrier; |
6a1cdb4d | 7581 | basic_block rec; |
48e1416a | 7582 | |
93f6b030 | 7583 | haifa_recovery_bb_recently_added_p = true; |
7584 | haifa_recovery_bb_ever_added_p = true; | |
6a1cdb4d | 7585 | |
e1ab7874 | 7586 | init_before_recovery (before_recovery_ptr); |
6a1cdb4d | 7587 | |
5d65a39c | 7588 | barrier = get_last_bb_insn (before_recovery); |
7589 | gcc_assert (BARRIER_P (barrier)); | |
7590 | ||
7591 | label = emit_label_after (gen_label_rtx (), barrier); | |
7592 | ||
7593 | rec = create_basic_block (label, label, before_recovery); | |
7594 | ||
e1ab7874 | 7595 | /* A recovery block always ends with an unconditional jump. */ |
6a1cdb4d | 7596 | emit_barrier_after (BB_END (rec)); |
7597 | ||
7598 | if (BB_PARTITION (before_recovery) != BB_UNPARTITIONED) | |
7599 | BB_SET_PARTITION (rec, BB_COLD_PARTITION); | |
48e1416a | 7600 | |
7601 | if (sched_verbose && spec_info->dump) | |
6a1cdb4d | 7602 | fprintf (spec_info->dump, ";;\t\tGenerated recovery block rec%d\n", |
7603 | rec->index); | |
7604 | ||
6a1cdb4d | 7605 | return rec; |
7606 | } | |
7607 | ||
e1ab7874 | 7608 | /* Create edges: FIRST_BB -> REC; FIRST_BB -> SECOND_BB; REC -> SECOND_BB |
7609 | and emit necessary jumps. */ | |
7610 | void | |
7611 | sched_create_recovery_edges (basic_block first_bb, basic_block rec, | |
7612 | basic_block second_bb) | |
7613 | { | |
7614 | rtx label; | |
7615 | rtx jump; | |
e1ab7874 | 7616 | int edge_flags; |
7617 | ||
7618 | /* This is fixing of incoming edge. */ | |
48e1416a | 7619 | /* ??? Which other flags should be specified? */ |
e1ab7874 | 7620 | if (BB_PARTITION (first_bb) != BB_PARTITION (rec)) |
7621 | /* Partition type is the same, if it is "unpartitioned". */ | |
7622 | edge_flags = EDGE_CROSSING; | |
7623 | else | |
7624 | edge_flags = 0; | |
48e1416a | 7625 | |
d52d7a3a | 7626 | make_edge (first_bb, rec, edge_flags); |
e1ab7874 | 7627 | label = block_label (second_bb); |
7628 | jump = emit_jump_insn_after (gen_jump (label), BB_END (rec)); | |
7629 | JUMP_LABEL (jump) = label; | |
7630 | LABEL_NUSES (label)++; | |
7631 | ||
7632 | if (BB_PARTITION (second_bb) != BB_PARTITION (rec)) | |
7633 | /* Partition type is the same, if it is "unpartitioned". */ | |
7634 | { | |
7635 | /* Rewritten from cfgrtl.c. */ | |
7636 | if (flag_reorder_blocks_and_partition | |
218e3e4e | 7637 | && targetm_common.have_named_sections) |
e1ab7874 | 7638 | { |
5859ee98 | 7639 | /* We don't need the same note for the check because |
7640 | any_condjump_p (check) == true. */ | |
7641 | add_reg_note (jump, REG_CROSSING_JUMP, NULL_RTX); | |
e1ab7874 | 7642 | } |
7643 | edge_flags = EDGE_CROSSING; | |
7644 | } | |
7645 | else | |
48e1416a | 7646 | edge_flags = 0; |
e1ab7874 | 7647 | |
48e1416a | 7648 | make_single_succ_edge (rec, second_bb, edge_flags); |
1a5dbaab | 7649 | if (dom_info_available_p (CDI_DOMINATORS)) |
7650 | set_immediate_dominator (CDI_DOMINATORS, rec, first_bb); | |
e1ab7874 | 7651 | } |
7652 | ||
6a1cdb4d | 7653 | /* This function creates recovery code for INSN. If MUTATE_P is nonzero, |
7654 | INSN is a simple check, that should be converted to branchy one. */ | |
7655 | static void | |
7656 | create_check_block_twin (rtx insn, bool mutate_p) | |
7657 | { | |
7658 | basic_block rec; | |
9997bd27 | 7659 | rtx label, check, twin; |
6a1cdb4d | 7660 | ds_t fs; |
93f6b030 | 7661 | sd_iterator_def sd_it; |
7662 | dep_t dep; | |
7663 | dep_def _new_dep, *new_dep = &_new_dep; | |
e1ab7874 | 7664 | ds_t todo_spec; |
6a1cdb4d | 7665 | |
e1ab7874 | 7666 | gcc_assert (ORIG_PAT (insn) != NULL_RTX); |
7667 | ||
7668 | if (!mutate_p) | |
7669 | todo_spec = TODO_SPEC (insn); | |
7670 | else | |
7671 | { | |
7672 | gcc_assert (IS_SPECULATION_SIMPLE_CHECK_P (insn) | |
7673 | && (TODO_SPEC (insn) & SPECULATIVE) == 0); | |
7674 | ||
7675 | todo_spec = CHECK_SPEC (insn); | |
7676 | } | |
7677 | ||
7678 | todo_spec &= SPECULATIVE; | |
6a1cdb4d | 7679 | |
7680 | /* Create recovery block. */ | |
cf7898a6 | 7681 | if (mutate_p || targetm.sched.needs_block_p (todo_spec)) |
6a1cdb4d | 7682 | { |
e1ab7874 | 7683 | rec = sched_create_recovery_block (NULL); |
6a1cdb4d | 7684 | label = BB_HEAD (rec); |
7685 | } | |
7686 | else | |
7687 | { | |
34154e27 | 7688 | rec = EXIT_BLOCK_PTR_FOR_FN (cfun); |
e1ab7874 | 7689 | label = NULL_RTX; |
6a1cdb4d | 7690 | } |
7691 | ||
7692 | /* Emit CHECK. */ | |
cf7898a6 | 7693 | check = targetm.sched.gen_spec_check (insn, label, todo_spec); |
6a1cdb4d | 7694 | |
34154e27 | 7695 | if (rec != EXIT_BLOCK_PTR_FOR_FN (cfun)) |
6a1cdb4d | 7696 | { |
7697 | /* To have mem_reg alive at the beginning of second_bb, | |
48e1416a | 7698 | we emit check BEFORE insn, so insn after splitting |
6a1cdb4d | 7699 | insn will be at the beginning of second_bb, which will |
7700 | provide us with the correct life information. */ | |
7701 | check = emit_jump_insn_before (check, insn); | |
7702 | JUMP_LABEL (check) = label; | |
7703 | LABEL_NUSES (label)++; | |
7704 | } | |
7705 | else | |
7706 | check = emit_insn_before (check, insn); | |
7707 | ||
7708 | /* Extend data structures. */ | |
e1ab7874 | 7709 | haifa_init_insn (check); |
7710 | ||
7711 | /* CHECK is being added to current region. Extend ready list. */ | |
7712 | gcc_assert (sched_ready_n_insns != -1); | |
7713 | sched_extend_ready_list (sched_ready_n_insns + 1); | |
7714 | ||
7715 | if (current_sched_info->add_remove_insn) | |
7716 | current_sched_info->add_remove_insn (insn, 0); | |
7717 | ||
6a1cdb4d | 7718 | RECOVERY_BLOCK (check) = rec; |
7719 | ||
7720 | if (sched_verbose && spec_info->dump) | |
7721 | fprintf (spec_info->dump, ";;\t\tGenerated check insn : %s\n", | |
7722 | (*current_sched_info->print_insn) (check, 0)); | |
7723 | ||
7724 | gcc_assert (ORIG_PAT (insn)); | |
7725 | ||
9ca2c29a | 7726 | /* Initialize TWIN (twin is a duplicate of original instruction |
6a1cdb4d | 7727 | in the recovery block). */ |
34154e27 | 7728 | if (rec != EXIT_BLOCK_PTR_FOR_FN (cfun)) |
6a1cdb4d | 7729 | { |
93f6b030 | 7730 | sd_iterator_def sd_it; |
7731 | dep_t dep; | |
7732 | ||
7733 | FOR_EACH_DEP (insn, SD_LIST_RES_BACK, sd_it, dep) | |
7734 | if ((DEP_STATUS (dep) & DEP_OUTPUT) != 0) | |
6a1cdb4d | 7735 | { |
93f6b030 | 7736 | struct _dep _dep2, *dep2 = &_dep2; |
9997bd27 | 7737 | |
93f6b030 | 7738 | init_dep (dep2, DEP_PRO (dep), check, REG_DEP_TRUE); |
9997bd27 | 7739 | |
93f6b030 | 7740 | sd_add_dep (dep2, true); |
6a1cdb4d | 7741 | } |
7742 | ||
7743 | twin = emit_insn_after (ORIG_PAT (insn), BB_END (rec)); | |
e1ab7874 | 7744 | haifa_init_insn (twin); |
6a1cdb4d | 7745 | |
7746 | if (sched_verbose && spec_info->dump) | |
7747 | /* INSN_BB (insn) isn't determined for twin insns yet. | |
7748 | So we can't use current_sched_info->print_insn. */ | |
7749 | fprintf (spec_info->dump, ";;\t\tGenerated twin insn : %d/rec%d\n", | |
7750 | INSN_UID (twin), rec->index); | |
7751 | } | |
7752 | else | |
7753 | { | |
7754 | ORIG_PAT (check) = ORIG_PAT (insn); | |
7755 | HAS_INTERNAL_DEP (check) = 1; | |
7756 | twin = check; | |
7757 | /* ??? We probably should change all OUTPUT dependencies to | |
7758 | (TRUE | OUTPUT). */ | |
7759 | } | |
7760 | ||
93f6b030 | 7761 | /* Copy all resolved back dependencies of INSN to TWIN. This will |
7762 | provide correct value for INSN_TICK (TWIN). */ | |
7763 | sd_copy_back_deps (twin, insn, true); | |
6a1cdb4d | 7764 | |
34154e27 | 7765 | if (rec != EXIT_BLOCK_PTR_FOR_FN (cfun)) |
6a1cdb4d | 7766 | /* In case of branchy check, fix CFG. */ |
7767 | { | |
7768 | basic_block first_bb, second_bb; | |
7769 | rtx jump; | |
6a1cdb4d | 7770 | |
7771 | first_bb = BLOCK_FOR_INSN (check); | |
e1ab7874 | 7772 | second_bb = sched_split_block (first_bb, check); |
6a1cdb4d | 7773 | |
e1ab7874 | 7774 | sched_create_recovery_edges (first_bb, rec, second_bb); |
6a1cdb4d | 7775 | |
48e1416a | 7776 | sched_init_only_bb (second_bb, first_bb); |
34154e27 | 7777 | sched_init_only_bb (rec, EXIT_BLOCK_PTR_FOR_FN (cfun)); |
e1ab7874 | 7778 | |
7779 | jump = BB_END (rec); | |
7780 | haifa_init_insn (jump); | |
6a1cdb4d | 7781 | } |
7782 | ||
48e1416a | 7783 | /* Move backward dependences from INSN to CHECK and |
6a1cdb4d | 7784 | move forward dependences from INSN to TWIN. */ |
93f6b030 | 7785 | |
7786 | /* First, create dependencies between INSN's producers and CHECK & TWIN. */ | |
7787 | FOR_EACH_DEP (insn, SD_LIST_BACK, sd_it, dep) | |
6a1cdb4d | 7788 | { |
93f6b030 | 7789 | rtx pro = DEP_PRO (dep); |
6a1cdb4d | 7790 | ds_t ds; |
7791 | ||
7792 | /* If BEGIN_DATA: [insn ~~TRUE~~> producer]: | |
7793 | check --TRUE--> producer ??? or ANTI ??? | |
7794 | twin --TRUE--> producer | |
7795 | twin --ANTI--> check | |
48e1416a | 7796 | |
6a1cdb4d | 7797 | If BEGIN_CONTROL: [insn ~~ANTI~~> producer]: |
7798 | check --ANTI--> producer | |
7799 | twin --ANTI--> producer | |
7800 | twin --ANTI--> check | |
7801 | ||
7802 | If BE_IN_SPEC: [insn ~~TRUE~~> producer]: | |
7803 | check ~~TRUE~~> producer | |
7804 | twin ~~TRUE~~> producer | |
48e1416a | 7805 | twin --ANTI--> check */ |
6a1cdb4d | 7806 | |
93f6b030 | 7807 | ds = DEP_STATUS (dep); |
6a1cdb4d | 7808 | |
7809 | if (ds & BEGIN_SPEC) | |
7810 | { | |
7811 | gcc_assert (!mutate_p); | |
7812 | ds &= ~BEGIN_SPEC; | |
7813 | } | |
7814 | ||
93f6b030 | 7815 | init_dep_1 (new_dep, pro, check, DEP_TYPE (dep), ds); |
7816 | sd_add_dep (new_dep, false); | |
7817 | ||
34154e27 | 7818 | if (rec != EXIT_BLOCK_PTR_FOR_FN (cfun)) |
6a1cdb4d | 7819 | { |
93f6b030 | 7820 | DEP_CON (new_dep) = twin; |
7821 | sd_add_dep (new_dep, false); | |
48e1416a | 7822 | } |
6a1cdb4d | 7823 | } |
7824 | ||
93f6b030 | 7825 | /* Second, remove backward dependencies of INSN. */ |
7826 | for (sd_it = sd_iterator_start (insn, SD_LIST_SPEC_BACK); | |
7827 | sd_iterator_cond (&sd_it, &dep);) | |
7828 | { | |
7829 | if ((DEP_STATUS (dep) & BEGIN_SPEC) | |
7830 | || mutate_p) | |
7831 | /* We can delete this dep because we overcome it with | |
7832 | BEGIN_SPECULATION. */ | |
7833 | sd_delete_dep (sd_it); | |
7834 | else | |
7835 | sd_iterator_next (&sd_it); | |
7836 | } | |
6a1cdb4d | 7837 | |
93f6b030 | 7838 | /* Future Speculations. Determine what BE_IN speculations will be like. */ |
6a1cdb4d | 7839 | fs = 0; |
7840 | ||
7841 | /* Fields (DONE_SPEC (x) & BEGIN_SPEC) and CHECK_SPEC (x) are set only | |
7842 | here. */ | |
48e1416a | 7843 | |
6a1cdb4d | 7844 | gcc_assert (!DONE_SPEC (insn)); |
48e1416a | 7845 | |
6a1cdb4d | 7846 | if (!mutate_p) |
48e1416a | 7847 | { |
6a1cdb4d | 7848 | ds_t ts = TODO_SPEC (insn); |
7849 | ||
7850 | DONE_SPEC (insn) = ts & BEGIN_SPEC; | |
7851 | CHECK_SPEC (check) = ts & BEGIN_SPEC; | |
7852 | ||
becfaa62 | 7853 | /* Luckiness of future speculations solely depends upon initial |
93f6b030 | 7854 | BEGIN speculation. */ |
6a1cdb4d | 7855 | if (ts & BEGIN_DATA) |
7856 | fs = set_dep_weak (fs, BE_IN_DATA, get_dep_weak (ts, BEGIN_DATA)); | |
7857 | if (ts & BEGIN_CONTROL) | |
93f6b030 | 7858 | fs = set_dep_weak (fs, BE_IN_CONTROL, |
7859 | get_dep_weak (ts, BEGIN_CONTROL)); | |
6a1cdb4d | 7860 | } |
7861 | else | |
7862 | CHECK_SPEC (check) = CHECK_SPEC (insn); | |
7863 | ||
7864 | /* Future speculations: call the helper. */ | |
93f6b030 | 7865 | process_insn_forw_deps_be_in_spec (insn, twin, fs); |
6a1cdb4d | 7866 | |
34154e27 | 7867 | if (rec != EXIT_BLOCK_PTR_FOR_FN (cfun)) |
6a1cdb4d | 7868 | { |
7869 | /* Which types of dependencies should we use here is, | |
7870 | generally, machine-dependent question... But, for now, | |
7871 | it is not. */ | |
7872 | ||
7873 | if (!mutate_p) | |
7874 | { | |
93f6b030 | 7875 | init_dep (new_dep, insn, check, REG_DEP_TRUE); |
7876 | sd_add_dep (new_dep, false); | |
7877 | ||
7878 | init_dep (new_dep, insn, twin, REG_DEP_OUTPUT); | |
7879 | sd_add_dep (new_dep, false); | |
6a1cdb4d | 7880 | } |
7881 | else | |
7882 | { | |
48e1416a | 7883 | if (spec_info->dump) |
6a1cdb4d | 7884 | fprintf (spec_info->dump, ";;\t\tRemoved simple check : %s\n", |
7885 | (*current_sched_info->print_insn) (insn, 0)); | |
7886 | ||
93f6b030 | 7887 | /* Remove all dependencies of the INSN. */ |
7888 | { | |
7889 | sd_it = sd_iterator_start (insn, (SD_LIST_FORW | |
7890 | | SD_LIST_BACK | |
7891 | | SD_LIST_RES_BACK)); | |
7892 | while (sd_iterator_cond (&sd_it, &dep)) | |
7893 | sd_delete_dep (sd_it); | |
7894 | } | |
6a1cdb4d | 7895 | |
93f6b030 | 7896 | /* If former check (INSN) already was moved to the ready (or queue) |
7897 | list, add new check (CHECK) there too. */ | |
6a1cdb4d | 7898 | if (QUEUE_INDEX (insn) != QUEUE_NOWHERE) |
7899 | try_ready (check); | |
7900 | ||
93f6b030 | 7901 | /* Remove old check from instruction stream and free its |
7902 | data. */ | |
6a1cdb4d | 7903 | sched_remove_insn (insn); |
7904 | } | |
7905 | ||
93f6b030 | 7906 | init_dep (new_dep, check, twin, REG_DEP_ANTI); |
7907 | sd_add_dep (new_dep, false); | |
6a1cdb4d | 7908 | } |
7909 | else | |
93f6b030 | 7910 | { |
7911 | init_dep_1 (new_dep, insn, check, REG_DEP_TRUE, DEP_TRUE | DEP_OUTPUT); | |
7912 | sd_add_dep (new_dep, false); | |
7913 | } | |
6a1cdb4d | 7914 | |
7915 | if (!mutate_p) | |
9ca2c29a | 7916 | /* Fix priorities. If MUTATE_P is nonzero, this is not necessary, |
6a1cdb4d | 7917 | because it'll be done later in add_to_speculative_block. */ |
7918 | { | |
9af5ce0c | 7919 | rtx_vec_t priorities_roots = rtx_vec_t (); |
3588ab9c | 7920 | |
7921 | clear_priorities (twin, &priorities_roots); | |
7922 | calc_priorities (priorities_roots); | |
f1f41a6c | 7923 | priorities_roots.release (); |
6a1cdb4d | 7924 | } |
7925 | } | |
7926 | ||
7927 | /* Removes dependency between instructions in the recovery block REC | |
7928 | and usual region instructions. It keeps inner dependences so it | |
9ca2c29a | 7929 | won't be necessary to recompute them. */ |
6a1cdb4d | 7930 | static void |
7931 | fix_recovery_deps (basic_block rec) | |
7932 | { | |
9997bd27 | 7933 | rtx note, insn, jump, ready_list = 0; |
6a1cdb4d | 7934 | bitmap_head in_ready; |
93f6b030 | 7935 | rtx link; |
6a1cdb4d | 7936 | |
7937 | bitmap_initialize (&in_ready, 0); | |
48e1416a | 7938 | |
6a1cdb4d | 7939 | /* NOTE - a basic block note. */ |
7940 | note = NEXT_INSN (BB_HEAD (rec)); | |
7941 | gcc_assert (NOTE_INSN_BASIC_BLOCK_P (note)); | |
7942 | insn = BB_END (rec); | |
7943 | gcc_assert (JUMP_P (insn)); | |
7944 | insn = PREV_INSN (insn); | |
7945 | ||
7946 | do | |
93f6b030 | 7947 | { |
7948 | sd_iterator_def sd_it; | |
7949 | dep_t dep; | |
6a1cdb4d | 7950 | |
93f6b030 | 7951 | for (sd_it = sd_iterator_start (insn, SD_LIST_FORW); |
7952 | sd_iterator_cond (&sd_it, &dep);) | |
7953 | { | |
7954 | rtx consumer = DEP_CON (dep); | |
6a1cdb4d | 7955 | |
7956 | if (BLOCK_FOR_INSN (consumer) != rec) | |
7957 | { | |
93f6b030 | 7958 | sd_delete_dep (sd_it); |
6a1cdb4d | 7959 | |
6ef9bbe0 | 7960 | if (bitmap_set_bit (&in_ready, INSN_LUID (consumer))) |
7961 | ready_list = alloc_INSN_LIST (consumer, ready_list); | |
6a1cdb4d | 7962 | } |
7963 | else | |
7964 | { | |
93f6b030 | 7965 | gcc_assert ((DEP_STATUS (dep) & DEP_TYPES) == DEP_TRUE); |
6a1cdb4d | 7966 | |
93f6b030 | 7967 | sd_iterator_next (&sd_it); |
6a1cdb4d | 7968 | } |
7969 | } | |
48e1416a | 7970 | |
6a1cdb4d | 7971 | insn = PREV_INSN (insn); |
7972 | } | |
7973 | while (insn != note); | |
7974 | ||
7975 | bitmap_clear (&in_ready); | |
7976 | ||
7977 | /* Try to add instructions to the ready or queue list. */ | |
93f6b030 | 7978 | for (link = ready_list; link; link = XEXP (link, 1)) |
7979 | try_ready (XEXP (link, 0)); | |
6a1cdb4d | 7980 | free_INSN_LIST_list (&ready_list); |
7981 | ||
7982 | /* Fixing jump's dependences. */ | |
7983 | insn = BB_HEAD (rec); | |
7984 | jump = BB_END (rec); | |
48e1416a | 7985 | |
6a1cdb4d | 7986 | gcc_assert (LABEL_P (insn)); |
7987 | insn = NEXT_INSN (insn); | |
48e1416a | 7988 | |
6a1cdb4d | 7989 | gcc_assert (NOTE_INSN_BASIC_BLOCK_P (insn)); |
7990 | add_jump_dependencies (insn, jump); | |
7991 | } | |
7992 | ||
effd1640 | 7993 | /* Change pattern of INSN to NEW_PAT. Invalidate cached haifa |
7994 | instruction data. */ | |
7995 | static bool | |
7996 | haifa_change_pattern (rtx insn, rtx new_pat) | |
6a1cdb4d | 7997 | { |
7998 | int t; | |
7999 | ||
8000 | t = validate_change (insn, &PATTERN (insn), new_pat, 0); | |
effd1640 | 8001 | if (!t) |
8002 | return false; | |
e1ab7874 | 8003 | |
d452a169 | 8004 | update_insn_after_change (insn); |
effd1640 | 8005 | return true; |
6a1cdb4d | 8006 | } |
8007 | ||
6a1cdb4d | 8008 | /* -1 - can't speculate, |
8009 | 0 - for speculation with REQUEST mode it is OK to use | |
8010 | current instruction pattern, | |
8011 | 1 - need to change pattern for *NEW_PAT to be speculative. */ | |
e1ab7874 | 8012 | int |
8013 | sched_speculate_insn (rtx insn, ds_t request, rtx *new_pat) | |
6a1cdb4d | 8014 | { |
8015 | gcc_assert (current_sched_info->flags & DO_SPECULATION | |
93f6b030 | 8016 | && (request & SPECULATIVE) |
8017 | && sched_insn_is_legitimate_for_speculation_p (insn, request)); | |
6a1cdb4d | 8018 | |
93f6b030 | 8019 | if ((request & spec_info->mask) != request) |
6a1cdb4d | 8020 | return -1; |
6a1cdb4d | 8021 | |
93f6b030 | 8022 | if (request & BE_IN_SPEC |
8023 | && !(request & BEGIN_SPEC)) | |
8024 | return 0; | |
6a1cdb4d | 8025 | |
e1ab7874 | 8026 | return targetm.sched.speculate_insn (insn, request, new_pat); |
8027 | } | |
8028 | ||
8029 | static int | |
8030 | haifa_speculate_insn (rtx insn, ds_t request, rtx *new_pat) | |
8031 | { | |
8032 | gcc_assert (sched_deps_info->generate_spec_deps | |
8033 | && !IS_SPECULATION_CHECK_P (insn)); | |
8034 | ||
8035 | if (HAS_INTERNAL_DEP (insn) | |
8036 | || SCHED_GROUP_P (insn)) | |
8037 | return -1; | |
8038 | ||
8039 | return sched_speculate_insn (insn, request, new_pat); | |
6a1cdb4d | 8040 | } |
8041 | ||
8042 | /* Print some information about block BB, which starts with HEAD and | |
8043 | ends with TAIL, before scheduling it. | |
8044 | I is zero, if scheduler is about to start with the fresh ebb. */ | |
8045 | static void | |
8046 | dump_new_block_header (int i, basic_block bb, rtx head, rtx tail) | |
8047 | { | |
8048 | if (!i) | |
8049 | fprintf (sched_dump, | |
8050 | ";; ======================================================\n"); | |
8051 | else | |
8052 | fprintf (sched_dump, | |
8053 | ";; =====================ADVANCING TO=====================\n"); | |
8054 | fprintf (sched_dump, | |
8055 | ";; -- basic block %d from %d to %d -- %s reload\n", | |
8056 | bb->index, INSN_UID (head), INSN_UID (tail), | |
8057 | (reload_completed ? "after" : "before")); | |
8058 | fprintf (sched_dump, | |
8059 | ";; ======================================================\n"); | |
8060 | fprintf (sched_dump, "\n"); | |
8061 | } | |
8062 | ||
8063 | /* Unlink basic block notes and labels and saves them, so they | |
8064 | can be easily restored. We unlink basic block notes in EBB to | |
9ca2c29a | 8065 | provide back-compatibility with the previous code, as target backends |
6a1cdb4d | 8066 | assume, that there'll be only instructions between |
8067 | current_sched_info->{head and tail}. We restore these notes as soon | |
8068 | as we can. | |
8069 | FIRST (LAST) is the first (last) basic block in the ebb. | |
8070 | NB: In usual case (FIRST == LAST) nothing is really done. */ | |
8071 | void | |
8072 | unlink_bb_notes (basic_block first, basic_block last) | |
8073 | { | |
8074 | /* We DON'T unlink basic block notes of the first block in the ebb. */ | |
8075 | if (first == last) | |
8076 | return; | |
8077 | ||
fe672ac0 | 8078 | bb_header = XNEWVEC (rtx, last_basic_block_for_fn (cfun)); |
6a1cdb4d | 8079 | |
8080 | /* Make a sentinel. */ | |
34154e27 | 8081 | if (last->next_bb != EXIT_BLOCK_PTR_FOR_FN (cfun)) |
6a1cdb4d | 8082 | bb_header[last->next_bb->index] = 0; |
8083 | ||
8084 | first = first->next_bb; | |
8085 | do | |
8086 | { | |
8087 | rtx prev, label, note, next; | |
8088 | ||
8089 | label = BB_HEAD (last); | |
8090 | if (LABEL_P (label)) | |
8091 | note = NEXT_INSN (label); | |
8092 | else | |
48e1416a | 8093 | note = label; |
6a1cdb4d | 8094 | gcc_assert (NOTE_INSN_BASIC_BLOCK_P (note)); |
8095 | ||
8096 | prev = PREV_INSN (label); | |
8097 | next = NEXT_INSN (note); | |
8098 | gcc_assert (prev && next); | |
8099 | ||
8100 | NEXT_INSN (prev) = next; | |
8101 | PREV_INSN (next) = prev; | |
8102 | ||
8103 | bb_header[last->index] = label; | |
8104 | ||
8105 | if (last == first) | |
8106 | break; | |
48e1416a | 8107 | |
6a1cdb4d | 8108 | last = last->prev_bb; |
8109 | } | |
8110 | while (1); | |
8111 | } | |
8112 | ||
8113 | /* Restore basic block notes. | |
8114 | FIRST is the first basic block in the ebb. */ | |
8115 | static void | |
8116 | restore_bb_notes (basic_block first) | |
8117 | { | |
8118 | if (!bb_header) | |
8119 | return; | |
8120 | ||
8121 | /* We DON'T unlink basic block notes of the first block in the ebb. */ | |
48e1416a | 8122 | first = first->next_bb; |
6a1cdb4d | 8123 | /* Remember: FIRST is actually a second basic block in the ebb. */ |
8124 | ||
34154e27 | 8125 | while (first != EXIT_BLOCK_PTR_FOR_FN (cfun) |
6a1cdb4d | 8126 | && bb_header[first->index]) |
8127 | { | |
8128 | rtx prev, label, note, next; | |
48e1416a | 8129 | |
6a1cdb4d | 8130 | label = bb_header[first->index]; |
8131 | prev = PREV_INSN (label); | |
8132 | next = NEXT_INSN (prev); | |
8133 | ||
8134 | if (LABEL_P (label)) | |
8135 | note = NEXT_INSN (label); | |
8136 | else | |
48e1416a | 8137 | note = label; |
6a1cdb4d | 8138 | gcc_assert (NOTE_INSN_BASIC_BLOCK_P (note)); |
8139 | ||
8140 | bb_header[first->index] = 0; | |
8141 | ||
8142 | NEXT_INSN (prev) = label; | |
8143 | NEXT_INSN (note) = next; | |
8144 | PREV_INSN (next) = note; | |
48e1416a | 8145 | |
6a1cdb4d | 8146 | first = first->next_bb; |
8147 | } | |
8148 | ||
8149 | free (bb_header); | |
8150 | bb_header = 0; | |
8151 | } | |
8152 | ||
6a1cdb4d | 8153 | /* Helper function. |
8154 | Fix CFG after both in- and inter-block movement of | |
8155 | control_flow_insn_p JUMP. */ | |
8156 | static void | |
8157 | fix_jump_move (rtx jump) | |
8158 | { | |
8159 | basic_block bb, jump_bb, jump_bb_next; | |
8160 | ||
8161 | bb = BLOCK_FOR_INSN (PREV_INSN (jump)); | |
8162 | jump_bb = BLOCK_FOR_INSN (jump); | |
8163 | jump_bb_next = jump_bb->next_bb; | |
8164 | ||
e1ab7874 | 8165 | gcc_assert (common_sched_info->sched_pass_id == SCHED_EBB_PASS |
fd27912f | 8166 | || IS_SPECULATION_BRANCHY_CHECK_P (jump)); |
48e1416a | 8167 | |
6a1cdb4d | 8168 | if (!NOTE_INSN_BASIC_BLOCK_P (BB_END (jump_bb_next))) |
8169 | /* if jump_bb_next is not empty. */ | |
8170 | BB_END (jump_bb) = BB_END (jump_bb_next); | |
8171 | ||
8172 | if (BB_END (bb) != PREV_INSN (jump)) | |
8173 | /* Then there are instruction after jump that should be placed | |
8174 | to jump_bb_next. */ | |
8175 | BB_END (jump_bb_next) = BB_END (bb); | |
8176 | else | |
8177 | /* Otherwise jump_bb_next is empty. */ | |
8178 | BB_END (jump_bb_next) = NEXT_INSN (BB_HEAD (jump_bb_next)); | |
8179 | ||
8180 | /* To make assertion in move_insn happy. */ | |
8181 | BB_END (bb) = PREV_INSN (jump); | |
8182 | ||
8183 | update_bb_for_insn (jump_bb_next); | |
8184 | } | |
8185 | ||
8186 | /* Fix CFG after interblock movement of control_flow_insn_p JUMP. */ | |
8187 | static void | |
8188 | move_block_after_check (rtx jump) | |
8189 | { | |
8190 | basic_block bb, jump_bb, jump_bb_next; | |
f1f41a6c | 8191 | vec<edge, va_gc> *t; |
6a1cdb4d | 8192 | |
8193 | bb = BLOCK_FOR_INSN (PREV_INSN (jump)); | |
8194 | jump_bb = BLOCK_FOR_INSN (jump); | |
8195 | jump_bb_next = jump_bb->next_bb; | |
48e1416a | 8196 | |
6a1cdb4d | 8197 | update_bb_for_insn (jump_bb); |
48e1416a | 8198 | |
fd27912f | 8199 | gcc_assert (IS_SPECULATION_CHECK_P (jump) |
8200 | || IS_SPECULATION_CHECK_P (BB_END (jump_bb_next))); | |
6a1cdb4d | 8201 | |
8202 | unlink_block (jump_bb_next); | |
8203 | link_block (jump_bb_next, bb); | |
8204 | ||
8205 | t = bb->succs; | |
8206 | bb->succs = 0; | |
8207 | move_succs (&(jump_bb->succs), bb); | |
8208 | move_succs (&(jump_bb_next->succs), jump_bb); | |
8209 | move_succs (&t, jump_bb_next); | |
3072d30e | 8210 | |
8211 | df_mark_solutions_dirty (); | |
48e1416a | 8212 | |
e1ab7874 | 8213 | common_sched_info->fix_recovery_cfg |
8214 | (bb->index, jump_bb->index, jump_bb_next->index); | |
6a1cdb4d | 8215 | } |
8216 | ||
8217 | /* Helper function for move_block_after_check. | |
8218 | This functions attaches edge vector pointed to by SUCCSP to | |
8219 | block TO. */ | |
8220 | static void | |
f1f41a6c | 8221 | move_succs (vec<edge, va_gc> **succsp, basic_block to) |
6a1cdb4d | 8222 | { |
8223 | edge e; | |
8224 | edge_iterator ei; | |
8225 | ||
8226 | gcc_assert (to->succs == 0); | |
8227 | ||
8228 | to->succs = *succsp; | |
8229 | ||
8230 | FOR_EACH_EDGE (e, ei, to->succs) | |
8231 | e->src = to; | |
8232 | ||
8233 | *succsp = 0; | |
8234 | } | |
8235 | ||
6a1cdb4d | 8236 | /* Remove INSN from the instruction stream. |
8237 | INSN should have any dependencies. */ | |
8238 | static void | |
8239 | sched_remove_insn (rtx insn) | |
8240 | { | |
93f6b030 | 8241 | sd_finish_insn (insn); |
8242 | ||
6a1cdb4d | 8243 | change_queue_index (insn, QUEUE_NOWHERE); |
8244 | current_sched_info->add_remove_insn (insn, 1); | |
93ff53d3 | 8245 | delete_insn (insn); |
6a1cdb4d | 8246 | } |
8247 | ||
3588ab9c | 8248 | /* Clear priorities of all instructions, that are forward dependent on INSN. |
8249 | Store in vector pointed to by ROOTS_PTR insns on which priority () should | |
8250 | be invoked to initialize all cleared priorities. */ | |
6a1cdb4d | 8251 | static void |
3588ab9c | 8252 | clear_priorities (rtx insn, rtx_vec_t *roots_ptr) |
6a1cdb4d | 8253 | { |
93f6b030 | 8254 | sd_iterator_def sd_it; |
8255 | dep_t dep; | |
3588ab9c | 8256 | bool insn_is_root_p = true; |
8257 | ||
8258 | gcc_assert (QUEUE_INDEX (insn) != QUEUE_SCHEDULED); | |
6a1cdb4d | 8259 | |
93f6b030 | 8260 | FOR_EACH_DEP (insn, SD_LIST_BACK, sd_it, dep) |
6a1cdb4d | 8261 | { |
3588ab9c | 8262 | rtx pro = DEP_PRO (dep); |
6a1cdb4d | 8263 | |
3588ab9c | 8264 | if (INSN_PRIORITY_STATUS (pro) >= 0 |
8265 | && QUEUE_INDEX (insn) != QUEUE_SCHEDULED) | |
6a1cdb4d | 8266 | { |
3588ab9c | 8267 | /* If DEP doesn't contribute to priority then INSN itself should |
8268 | be added to priority roots. */ | |
8269 | if (contributes_to_priority_p (dep)) | |
8270 | insn_is_root_p = false; | |
8271 | ||
8272 | INSN_PRIORITY_STATUS (pro) = -1; | |
8273 | clear_priorities (pro, roots_ptr); | |
6a1cdb4d | 8274 | } |
8275 | } | |
3588ab9c | 8276 | |
8277 | if (insn_is_root_p) | |
f1f41a6c | 8278 | roots_ptr->safe_push (insn); |
6a1cdb4d | 8279 | } |
8280 | ||
8281 | /* Recompute priorities of instructions, whose priorities might have been | |
3588ab9c | 8282 | changed. ROOTS is a vector of instructions whose priority computation will |
8283 | trigger initialization of all cleared priorities. */ | |
6a1cdb4d | 8284 | static void |
3588ab9c | 8285 | calc_priorities (rtx_vec_t roots) |
6a1cdb4d | 8286 | { |
3588ab9c | 8287 | int i; |
8288 | rtx insn; | |
6a1cdb4d | 8289 | |
f1f41a6c | 8290 | FOR_EACH_VEC_ELT (roots, i, insn) |
3588ab9c | 8291 | priority (insn); |
6a1cdb4d | 8292 | } |
8293 | ||
8294 | ||
8295 | /* Add dependences between JUMP and other instructions in the recovery | |
8296 | block. INSN is the first insn the recovery block. */ | |
8297 | static void | |
8298 | add_jump_dependencies (rtx insn, rtx jump) | |
8299 | { | |
8300 | do | |
8301 | { | |
8302 | insn = NEXT_INSN (insn); | |
8303 | if (insn == jump) | |
8304 | break; | |
48e1416a | 8305 | |
b30b031c | 8306 | if (dep_list_size (insn, SD_LIST_FORW) == 0) |
93f6b030 | 8307 | { |
8308 | dep_def _new_dep, *new_dep = &_new_dep; | |
8309 | ||
8310 | init_dep (new_dep, insn, jump, REG_DEP_ANTI); | |
8311 | sd_add_dep (new_dep, false); | |
8312 | } | |
6a1cdb4d | 8313 | } |
8314 | while (1); | |
9997bd27 | 8315 | |
93f6b030 | 8316 | gcc_assert (!sd_lists_empty_p (jump, SD_LIST_BACK)); |
6a1cdb4d | 8317 | } |
8318 | ||
e1ab7874 | 8319 | /* Extend data structures for logical insn UID. */ |
52d7e28c | 8320 | void |
8321 | sched_extend_luids (void) | |
e1ab7874 | 8322 | { |
8323 | int new_luids_max_uid = get_max_uid () + 1; | |
8324 | ||
f1f41a6c | 8325 | sched_luids.safe_grow_cleared (new_luids_max_uid); |
e1ab7874 | 8326 | } |
8327 | ||
8328 | /* Initialize LUID for INSN. */ | |
52d7e28c | 8329 | void |
8330 | sched_init_insn_luid (rtx insn) | |
e1ab7874 | 8331 | { |
8332 | int i = INSN_P (insn) ? 1 : common_sched_info->luid_for_non_insn (insn); | |
8333 | int luid; | |
8334 | ||
8335 | if (i >= 0) | |
8336 | { | |
8337 | luid = sched_max_luid; | |
8338 | sched_max_luid += i; | |
8339 | } | |
8340 | else | |
8341 | luid = -1; | |
8342 | ||
8343 | SET_INSN_LUID (insn, luid); | |
8344 | } | |
8345 | ||
52d7e28c | 8346 | /* Initialize luids for BBS. |
e1ab7874 | 8347 | The hook common_sched_info->luid_for_non_insn () is used to determine |
8348 | if notes, labels, etc. need luids. */ | |
8349 | void | |
52d7e28c | 8350 | sched_init_luids (bb_vec_t bbs) |
e1ab7874 | 8351 | { |
52d7e28c | 8352 | int i; |
8353 | basic_block bb; | |
8354 | ||
8355 | sched_extend_luids (); | |
f1f41a6c | 8356 | FOR_EACH_VEC_ELT (bbs, i, bb) |
e1ab7874 | 8357 | { |
52d7e28c | 8358 | rtx insn; |
e1ab7874 | 8359 | |
52d7e28c | 8360 | FOR_BB_INSNS (bb, insn) |
8361 | sched_init_insn_luid (insn); | |
8362 | } | |
e1ab7874 | 8363 | } |
8364 | ||
8365 | /* Free LUIDs. */ | |
8366 | void | |
8367 | sched_finish_luids (void) | |
8368 | { | |
f1f41a6c | 8369 | sched_luids.release (); |
e1ab7874 | 8370 | sched_max_luid = 1; |
8371 | } | |
8372 | ||
8373 | /* Return logical uid of INSN. Helpful while debugging. */ | |
8374 | int | |
8375 | insn_luid (rtx insn) | |
8376 | { | |
8377 | return INSN_LUID (insn); | |
8378 | } | |
8379 | ||
8380 | /* Extend per insn data in the target. */ | |
8381 | void | |
8382 | sched_extend_target (void) | |
8383 | { | |
8384 | if (targetm.sched.h_i_d_extended) | |
8385 | targetm.sched.h_i_d_extended (); | |
8386 | } | |
8387 | ||
8388 | /* Extend global scheduler structures (those, that live across calls to | |
8389 | schedule_block) to include information about just emitted INSN. */ | |
8390 | static void | |
8391 | extend_h_i_d (void) | |
8392 | { | |
f1f41a6c | 8393 | int reserve = (get_max_uid () + 1 - h_i_d.length ()); |
48e1416a | 8394 | if (reserve > 0 |
f1f41a6c | 8395 | && ! h_i_d.space (reserve)) |
e1ab7874 | 8396 | { |
f1f41a6c | 8397 | h_i_d.safe_grow_cleared (3 * get_max_uid () / 2); |
e1ab7874 | 8398 | sched_extend_target (); |
8399 | } | |
8400 | } | |
8401 | ||
8402 | /* Initialize h_i_d entry of the INSN with default values. | |
8403 | Values, that are not explicitly initialized here, hold zero. */ | |
8404 | static void | |
8405 | init_h_i_d (rtx insn) | |
8406 | { | |
8407 | if (INSN_LUID (insn) > 0) | |
8408 | { | |
8409 | INSN_COST (insn) = -1; | |
e1ab7874 | 8410 | QUEUE_INDEX (insn) = QUEUE_NOWHERE; |
8411 | INSN_TICK (insn) = INVALID_TICK; | |
e2f4a6ff | 8412 | INSN_EXACT_TICK (insn) = INVALID_TICK; |
e1ab7874 | 8413 | INTER_TICK (insn) = INVALID_TICK; |
8414 | TODO_SPEC (insn) = HARD_DEP; | |
8415 | } | |
8416 | } | |
8417 | ||
52d7e28c | 8418 | /* Initialize haifa_insn_data for BBS. */ |
e1ab7874 | 8419 | void |
52d7e28c | 8420 | haifa_init_h_i_d (bb_vec_t bbs) |
e1ab7874 | 8421 | { |
52d7e28c | 8422 | int i; |
8423 | basic_block bb; | |
8424 | ||
8425 | extend_h_i_d (); | |
f1f41a6c | 8426 | FOR_EACH_VEC_ELT (bbs, i, bb) |
e1ab7874 | 8427 | { |
52d7e28c | 8428 | rtx insn; |
e1ab7874 | 8429 | |
52d7e28c | 8430 | FOR_BB_INSNS (bb, insn) |
8431 | init_h_i_d (insn); | |
8432 | } | |
e1ab7874 | 8433 | } |
8434 | ||
8435 | /* Finalize haifa_insn_data. */ | |
8436 | void | |
8437 | haifa_finish_h_i_d (void) | |
8438 | { | |
a7dcf969 | 8439 | int i; |
8440 | haifa_insn_data_t data; | |
8441 | struct reg_use_data *use, *next; | |
8442 | ||
f1f41a6c | 8443 | FOR_EACH_VEC_ELT (h_i_d, i, data) |
a7dcf969 | 8444 | { |
b30b031c | 8445 | free (data->max_reg_pressure); |
dd045aee | 8446 | free (data->reg_pressure); |
a7dcf969 | 8447 | for (use = data->reg_use_list; use != NULL; use = next) |
8448 | { | |
8449 | next = use->next_insn_use; | |
8450 | free (use); | |
8451 | } | |
8452 | } | |
f1f41a6c | 8453 | h_i_d.release (); |
e1ab7874 | 8454 | } |
8455 | ||
8456 | /* Init data for the new insn INSN. */ | |
8457 | static void | |
8458 | haifa_init_insn (rtx insn) | |
8459 | { | |
8460 | gcc_assert (insn != NULL); | |
8461 | ||
52d7e28c | 8462 | sched_extend_luids (); |
8463 | sched_init_insn_luid (insn); | |
e1ab7874 | 8464 | sched_extend_target (); |
8465 | sched_deps_init (false); | |
52d7e28c | 8466 | extend_h_i_d (); |
8467 | init_h_i_d (insn); | |
e1ab7874 | 8468 | |
8469 | if (adding_bb_to_current_region_p) | |
8470 | { | |
8471 | sd_init_insn (insn); | |
8472 | ||
8473 | /* Extend dependency caches by one element. */ | |
8474 | extend_dependency_caches (1, false); | |
8475 | } | |
b30b031c | 8476 | if (sched_pressure != SCHED_PRESSURE_NONE) |
c15d7785 | 8477 | init_insn_reg_pressure_info (insn); |
e1ab7874 | 8478 | } |
8479 | ||
8480 | /* Init data for the new basic block BB which comes after AFTER. */ | |
8481 | static void | |
8482 | haifa_init_only_bb (basic_block bb, basic_block after) | |
8483 | { | |
8484 | gcc_assert (bb != NULL); | |
8485 | ||
8486 | sched_init_bbs (); | |
8487 | ||
8488 | if (common_sched_info->add_block) | |
8489 | /* This changes only data structures of the front-end. */ | |
8490 | common_sched_info->add_block (bb, after); | |
8491 | } | |
8492 | ||
8493 | /* A generic version of sched_split_block (). */ | |
8494 | basic_block | |
8495 | sched_split_block_1 (basic_block first_bb, rtx after) | |
8496 | { | |
8497 | edge e; | |
8498 | ||
8499 | e = split_block (first_bb, after); | |
8500 | gcc_assert (e->src == first_bb); | |
8501 | ||
48e1416a | 8502 | /* sched_split_block emits note if *check == BB_END. Probably it |
e1ab7874 | 8503 | is better to rip that note off. */ |
8504 | ||
8505 | return e->dest; | |
8506 | } | |
8507 | ||
8508 | /* A generic version of sched_create_empty_bb (). */ | |
8509 | basic_block | |
8510 | sched_create_empty_bb_1 (basic_block after) | |
8511 | { | |
8512 | return create_empty_bb (after); | |
8513 | } | |
8514 | ||
5a976006 | 8515 | /* Insert PAT as an INSN into the schedule and update the necessary data |
8516 | structures to account for it. */ | |
8517 | rtx | |
8518 | sched_emit_insn (rtx pat) | |
8519 | { | |
c03ae898 | 8520 | rtx insn = emit_insn_before (pat, nonscheduled_insns_begin); |
5a976006 | 8521 | haifa_init_insn (insn); |
c03ae898 | 8522 | |
8523 | if (current_sched_info->add_remove_insn) | |
8524 | current_sched_info->add_remove_insn (insn, 0); | |
8525 | ||
8526 | (*current_sched_info->begin_schedule_ready) (insn); | |
f1f41a6c | 8527 | scheduled_insns.safe_push (insn); |
c03ae898 | 8528 | |
8529 | last_scheduled_insn = insn; | |
5a976006 | 8530 | return insn; |
8531 | } | |
8532 | ||
0ac9454e | 8533 | /* This function returns a candidate satisfying dispatch constraints from |
8534 | the ready list. */ | |
8535 | ||
8536 | static rtx | |
8537 | ready_remove_first_dispatch (struct ready_list *ready) | |
8538 | { | |
8539 | int i; | |
8540 | rtx insn = ready_element (ready, 0); | |
8541 | ||
8542 | if (ready->n_ready == 1 | |
0ac9454e | 8543 | || !INSN_P (insn) |
cd316116 | 8544 | || INSN_CODE (insn) < 0 |
0ac9454e | 8545 | || !active_insn_p (insn) |
8546 | || targetm.sched.dispatch (insn, FITS_DISPATCH_WINDOW)) | |
8547 | return ready_remove_first (ready); | |
8548 | ||
8549 | for (i = 1; i < ready->n_ready; i++) | |
8550 | { | |
8551 | insn = ready_element (ready, i); | |
8552 | ||
cd316116 | 8553 | if (!INSN_P (insn) |
8554 | || INSN_CODE (insn) < 0 | |
0ac9454e | 8555 | || !active_insn_p (insn)) |
8556 | continue; | |
8557 | ||
8558 | if (targetm.sched.dispatch (insn, FITS_DISPATCH_WINDOW)) | |
8559 | { | |
8560 | /* Return ith element of ready. */ | |
8561 | insn = ready_remove (ready, i); | |
8562 | return insn; | |
8563 | } | |
8564 | } | |
8565 | ||
8566 | if (targetm.sched.dispatch (NULL_RTX, DISPATCH_VIOLATION)) | |
8567 | return ready_remove_first (ready); | |
8568 | ||
8569 | for (i = 1; i < ready->n_ready; i++) | |
8570 | { | |
8571 | insn = ready_element (ready, i); | |
8572 | ||
04ed582d | 8573 | if (!INSN_P (insn) |
cd316116 | 8574 | || INSN_CODE (insn) < 0 |
0ac9454e | 8575 | || !active_insn_p (insn)) |
8576 | continue; | |
8577 | ||
8578 | /* Return i-th element of ready. */ | |
8579 | if (targetm.sched.dispatch (insn, IS_CMP)) | |
8580 | return ready_remove (ready, i); | |
8581 | } | |
8582 | ||
8583 | return ready_remove_first (ready); | |
8584 | } | |
8585 | ||
8586 | /* Get number of ready insn in the ready list. */ | |
8587 | ||
8588 | int | |
8589 | number_in_ready (void) | |
8590 | { | |
8591 | return ready.n_ready; | |
8592 | } | |
8593 | ||
8594 | /* Get number of ready's in the ready list. */ | |
8595 | ||
8596 | rtx | |
8597 | get_ready_element (int i) | |
8598 | { | |
8599 | return ready_element (&ready, i); | |
8600 | } | |
8601 | ||
3eb9a99d | 8602 | #endif /* INSN_SCHEDULING */ |